Chili Palmer
fb7d0c6836
Some checks failed
Native code generation / deterministic (macos-latest) (push) Has been cancelled
Native code generation / deterministic (ubuntu-latest) (push) Has been cancelled
Native code generation / deterministic (windows-latest) (push) Has been cancelled
Imaging and meshing gate / native (macos-latest) (push) Has been cancelled
Imaging and meshing gate / native (ubuntu-latest) (push) Has been cancelled
Imaging and meshing gate / native (windows-latest) (push) Has been cancelled
JPEG 2000 feature / linux (push) Has been cancelled
JPEG 2000 feature / macos (push) Has been cancelled
JPEG 2000 feature / windows (push) Has been cancelled
Skia feature / linux (push) Has been cancelled
Skia feature / macos (push) Has been cancelled
Skia feature / windows (push) Has been cancelled
1679 lines
60 KiB
Rust
1679 lines
60 KiB
Rust
//! Deterministic framework shared by the `LibreMetaverse` data generators.
|
|
|
|
#![allow(clippy::missing_errors_doc)] // The CLI renders the complete error at its boundary.
|
|
#![allow(clippy::must_use_candidate)] // Generator helpers are also exercised for validation.
|
|
|
|
use serde::Deserialize;
|
|
use sha2::{Digest, Sha256};
|
|
use std::collections::{BTreeMap, BTreeSet};
|
|
use std::fmt;
|
|
use std::fmt::Write as _;
|
|
use std::fs;
|
|
use std::path::{Component, Path, PathBuf};
|
|
|
|
pub const INVENTORY_PATH: &str = "codegen/sources.json";
|
|
pub const MANIFEST_OUTPUT: &str = "codegen/generated/source_manifest.rs";
|
|
pub const PACKET_OUTPUT: &str = "crates/libremetaverse/src/packet_catalog.rs";
|
|
pub const PUBLIC_API_PATH: &str = "api/public-api.json";
|
|
const RUST_KEYWORDS: &[&str] = &[
|
|
"as", "break", "const", "continue", "crate", "else", "enum", "extern", "false", "fn", "for",
|
|
"if", "impl", "in", "let", "loop", "match", "mod", "move", "mut", "pub", "ref", "return",
|
|
"self", "Self", "static", "struct", "super", "trait", "true", "type", "unsafe", "use", "where",
|
|
"while", "async", "await", "dyn", "abstract", "become", "box", "do", "final", "macro",
|
|
"override", "priv", "typeof", "unsized", "virtual", "yield", "try", "from",
|
|
];
|
|
|
|
#[derive(Clone, Debug, Deserialize, Eq, PartialEq)]
|
|
pub struct Inventory {
|
|
pub schema: u32,
|
|
pub upstream_commit: String,
|
|
pub upstream_repository: String,
|
|
pub generators: Vec<GeneratorSpec>,
|
|
pub inputs: Vec<InputSpec>,
|
|
}
|
|
|
|
#[derive(Clone, Debug, Deserialize, Eq, PartialEq)]
|
|
pub struct GeneratorSpec {
|
|
pub id: String,
|
|
pub reference_source: String,
|
|
pub sha256: String,
|
|
pub license: String,
|
|
pub inputs: Vec<String>,
|
|
#[serde(default)]
|
|
pub note: String,
|
|
}
|
|
|
|
#[derive(Clone, Debug, Deserialize, Eq, PartialEq)]
|
|
pub struct InputSpec {
|
|
pub id: String,
|
|
pub reference_path: String,
|
|
pub vendored_path: String,
|
|
pub sha256: String,
|
|
pub license: String,
|
|
pub format: String,
|
|
}
|
|
|
|
#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)]
|
|
pub enum Severity {
|
|
Error,
|
|
Warning,
|
|
}
|
|
|
|
#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)]
|
|
pub struct Diagnostic {
|
|
pub code: &'static str,
|
|
pub severity: Severity,
|
|
pub path: String,
|
|
pub line: usize,
|
|
pub column: usize,
|
|
pub message: String,
|
|
}
|
|
|
|
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
|
pub enum PacketFrequency {
|
|
Low,
|
|
Medium,
|
|
High,
|
|
}
|
|
|
|
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
|
pub enum BlockRepetition {
|
|
Single,
|
|
Multiple(usize),
|
|
Variable,
|
|
}
|
|
|
|
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
|
pub enum FieldKind {
|
|
U8,
|
|
U16,
|
|
U32,
|
|
U64,
|
|
S8,
|
|
S16,
|
|
S32,
|
|
F32,
|
|
F64,
|
|
LlUuid,
|
|
Bool,
|
|
LlVector3,
|
|
LlVector3d,
|
|
LlVector4,
|
|
LlQuaternion,
|
|
IpAddr,
|
|
IpPort,
|
|
Variable,
|
|
Fixed,
|
|
}
|
|
|
|
#[derive(Clone, Debug, Eq, PartialEq)]
|
|
pub struct PacketField {
|
|
pub name: String,
|
|
pub kind: FieldKind,
|
|
pub count: usize,
|
|
pub line: usize,
|
|
pub column: usize,
|
|
}
|
|
|
|
#[derive(Clone, Debug, Eq, PartialEq)]
|
|
pub struct PacketBlock {
|
|
pub name: String,
|
|
pub repetition: BlockRepetition,
|
|
pub fields: Vec<PacketField>,
|
|
pub line: usize,
|
|
pub column: usize,
|
|
}
|
|
|
|
#[derive(Clone, Debug, Eq, PartialEq)]
|
|
pub struct PacketDefinition {
|
|
pub id: u16,
|
|
pub name: String,
|
|
pub frequency: PacketFrequency,
|
|
pub trusted: bool,
|
|
pub zerocoded: bool,
|
|
pub flags: Vec<String>,
|
|
pub blocks: Vec<PacketBlock>,
|
|
pub line: usize,
|
|
pub column: usize,
|
|
}
|
|
|
|
#[derive(Clone, Debug, Eq, PartialEq)]
|
|
pub struct PacketProtocol {
|
|
pub packets: Vec<PacketDefinition>,
|
|
}
|
|
|
|
#[derive(Clone, Debug, Eq, PartialEq)]
|
|
struct Token {
|
|
text: String,
|
|
line: usize,
|
|
column: usize,
|
|
}
|
|
|
|
impl fmt::Display for Diagnostic {
|
|
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
|
let severity = match self.severity {
|
|
Severity::Error => "error",
|
|
Severity::Warning => "warning",
|
|
};
|
|
write!(
|
|
formatter,
|
|
"{}:{}:{}: {severity}[{}]: {}",
|
|
self.path, self.line, self.column, self.code, self.message
|
|
)
|
|
}
|
|
}
|
|
|
|
pub fn workspace_root() -> PathBuf {
|
|
Path::new(env!("CARGO_MANIFEST_DIR")).join("../..")
|
|
}
|
|
|
|
pub fn normalize_text(path: &str, bytes: &[u8]) -> Result<String, Diagnostic> {
|
|
let bytes = bytes.strip_prefix(&[0xef, 0xbb, 0xbf]).unwrap_or(bytes);
|
|
let text = std::str::from_utf8(bytes).map_err(|error| {
|
|
let valid = &bytes[..error.valid_up_to()];
|
|
let (line, column) = line_column(valid);
|
|
Diagnostic {
|
|
code: "CG001",
|
|
severity: Severity::Error,
|
|
path: path.replace('\\', "/"),
|
|
line,
|
|
column,
|
|
message: "input is not valid UTF-8".to_owned(),
|
|
}
|
|
})?;
|
|
if let Some(position) = text.as_bytes().iter().position(|byte| *byte == 0) {
|
|
let prefix = &text.as_bytes()[..position];
|
|
let (line, column) = line_column(prefix);
|
|
return Err(Diagnostic {
|
|
code: "CG002",
|
|
severity: Severity::Error,
|
|
path: path.replace('\\', "/"),
|
|
line,
|
|
column,
|
|
message: "input contains a NUL byte".to_owned(),
|
|
});
|
|
}
|
|
Ok(text.replace("\r\n", "\n").replace('\r', "\n"))
|
|
}
|
|
|
|
fn line_column(bytes: &[u8]) -> (usize, usize) {
|
|
let mut line = 1;
|
|
let mut column = 1;
|
|
for byte in bytes {
|
|
if *byte == b'\n' {
|
|
line += 1;
|
|
column = 1;
|
|
} else {
|
|
column += 1;
|
|
}
|
|
}
|
|
(line, column)
|
|
}
|
|
|
|
fn packet_diagnostic(
|
|
path: &str,
|
|
token: Option<&Token>,
|
|
code: &'static str,
|
|
message: impl Into<String>,
|
|
) -> Diagnostic {
|
|
Diagnostic {
|
|
code,
|
|
severity: Severity::Error,
|
|
path: path.replace('\\', "/"),
|
|
line: token.map_or(1, |token| token.line),
|
|
column: token.map_or(1, |token| token.column),
|
|
message: message.into(),
|
|
}
|
|
}
|
|
|
|
fn tokenize_packet_template(path: &str, text: &str) -> Result<Vec<Token>, Vec<Diagnostic>> {
|
|
let mut tokens = Vec::new();
|
|
let mut characters = text.char_indices().peekable();
|
|
let mut line = 1usize;
|
|
let mut column = 1usize;
|
|
while let Some((start, character)) = characters.next() {
|
|
if character == '\n' {
|
|
line += 1;
|
|
column = 1;
|
|
continue;
|
|
}
|
|
if character.is_whitespace() {
|
|
column += 1;
|
|
continue;
|
|
}
|
|
if character == '/' && characters.peek().is_some_and(|(_, next)| *next == '/') {
|
|
characters.next();
|
|
column += 2;
|
|
for (_, next) in characters.by_ref() {
|
|
if next == '\n' {
|
|
line += 1;
|
|
column = 1;
|
|
break;
|
|
}
|
|
column += 1;
|
|
}
|
|
continue;
|
|
}
|
|
if matches!(character, '{' | '}') {
|
|
tokens.push(Token {
|
|
text: character.to_string(),
|
|
line,
|
|
column,
|
|
});
|
|
column += 1;
|
|
continue;
|
|
}
|
|
let token_line = line;
|
|
let token_column = column;
|
|
let mut end = start + character.len_utf8();
|
|
column += 1;
|
|
while let Some((offset, next)) = characters.peek().copied() {
|
|
if next.is_whitespace() || matches!(next, '{' | '}') {
|
|
break;
|
|
}
|
|
if next == '/' {
|
|
let mut lookahead = characters.clone();
|
|
lookahead.next();
|
|
if lookahead
|
|
.peek()
|
|
.is_some_and(|(_, following)| *following == '/')
|
|
{
|
|
break;
|
|
}
|
|
}
|
|
characters.next();
|
|
end = offset + next.len_utf8();
|
|
column += 1;
|
|
}
|
|
let value = &text[start..end];
|
|
if !value.is_ascii() {
|
|
return Err(vec![packet_diagnostic(
|
|
path,
|
|
Some(&Token {
|
|
text: value.to_owned(),
|
|
line: token_line,
|
|
column: token_column,
|
|
}),
|
|
"PG002",
|
|
"protocol identifiers and values must be ASCII",
|
|
)]);
|
|
}
|
|
tokens.push(Token {
|
|
text: value.to_owned(),
|
|
line: token_line,
|
|
column: token_column,
|
|
});
|
|
}
|
|
Ok(tokens)
|
|
}
|
|
|
|
struct PacketParser<'a> {
|
|
path: &'a str,
|
|
tokens: Vec<Token>,
|
|
position: usize,
|
|
diagnostics: Vec<Diagnostic>,
|
|
}
|
|
|
|
impl PacketParser<'_> {
|
|
fn current(&self) -> Option<&Token> {
|
|
self.tokens.get(self.position)
|
|
}
|
|
|
|
fn advance(&mut self) -> Option<Token> {
|
|
let token = self.tokens.get(self.position).cloned();
|
|
self.position += usize::from(token.is_some());
|
|
token
|
|
}
|
|
|
|
fn expect(&mut self, value: &str, code: &'static str, context: &str) -> bool {
|
|
if self.current().is_some_and(|token| token.text == value) {
|
|
self.position += 1;
|
|
true
|
|
} else {
|
|
self.diagnostics.push(packet_diagnostic(
|
|
self.path,
|
|
self.current(),
|
|
code,
|
|
format!("expected {value:?} {context}"),
|
|
));
|
|
false
|
|
}
|
|
}
|
|
|
|
fn atom(&mut self, code: &'static str, context: &str) -> Option<Token> {
|
|
match self.current() {
|
|
Some(token) if !matches!(token.text.as_str(), "{" | "}") => self.advance(),
|
|
_ => {
|
|
self.diagnostics.push(packet_diagnostic(
|
|
self.path,
|
|
self.current(),
|
|
code,
|
|
format!("expected {context}"),
|
|
));
|
|
None
|
|
}
|
|
}
|
|
}
|
|
|
|
fn recover_to_close(&mut self) {
|
|
let mut depth = 0usize;
|
|
while let Some(token) = self.advance() {
|
|
match token.text.as_str() {
|
|
"{" => depth += 1,
|
|
"}" if depth == 0 => break,
|
|
"}" => depth -= 1,
|
|
_ => {}
|
|
}
|
|
}
|
|
}
|
|
|
|
fn parse_field(&mut self) -> Option<PacketField> {
|
|
if !self.expect("{", "PG020", "to begin a field") {
|
|
return None;
|
|
}
|
|
let name = self.atom("PG021", "a field name")?;
|
|
let kind_token = self.atom("PG022", "a field type")?;
|
|
let (kind, known_kind) = match kind_token.text.as_str() {
|
|
"U8" => (FieldKind::U8, true),
|
|
"U16" => (FieldKind::U16, true),
|
|
"U32" => (FieldKind::U32, true),
|
|
"U64" => (FieldKind::U64, true),
|
|
"S8" => (FieldKind::S8, true),
|
|
"S16" => (FieldKind::S16, true),
|
|
"S32" => (FieldKind::S32, true),
|
|
"F32" => (FieldKind::F32, true),
|
|
"F64" => (FieldKind::F64, true),
|
|
"LLUUID" => (FieldKind::LlUuid, true),
|
|
"BOOL" => (FieldKind::Bool, true),
|
|
"LLVector3" => (FieldKind::LlVector3, true),
|
|
"LLVector3d" => (FieldKind::LlVector3d, true),
|
|
"LLVector4" => (FieldKind::LlVector4, true),
|
|
"LLQuaternion" => (FieldKind::LlQuaternion, true),
|
|
"IPADDR" => (FieldKind::IpAddr, true),
|
|
"IPPORT" => (FieldKind::IpPort, true),
|
|
"Variable" => (FieldKind::Variable, true),
|
|
"Fixed" => (FieldKind::Fixed, true),
|
|
other => {
|
|
self.diagnostics.push(packet_diagnostic(
|
|
self.path,
|
|
Some(&kind_token),
|
|
"PG023",
|
|
format!("unknown field type {other:?}"),
|
|
));
|
|
(FieldKind::Variable, false)
|
|
}
|
|
};
|
|
let count = if self.current().is_some_and(|token| token.text != "}") {
|
|
let count_token = self.atom("PG024", "a field width")?;
|
|
match count_token.text.parse::<usize>() {
|
|
Ok(value) if value > 0 => value,
|
|
_ => {
|
|
self.diagnostics.push(packet_diagnostic(
|
|
self.path,
|
|
Some(&count_token),
|
|
"PG025",
|
|
"field width must be a positive decimal integer",
|
|
));
|
|
1
|
|
}
|
|
}
|
|
} else {
|
|
1
|
|
};
|
|
if known_kind && matches!(kind, FieldKind::Variable) && !matches!(count, 1 | 2) {
|
|
self.diagnostics.push(packet_diagnostic(
|
|
self.path,
|
|
Some(&name),
|
|
"PG026",
|
|
"Variable fields require a one- or two-byte length prefix",
|
|
));
|
|
}
|
|
if known_kind && !matches!(kind, FieldKind::Variable | FieldKind::Fixed) && count != 1 {
|
|
self.diagnostics.push(packet_diagnostic(
|
|
self.path,
|
|
Some(&name),
|
|
"PG027",
|
|
"only Variable and Fixed fields may specify a width",
|
|
));
|
|
}
|
|
if !self.expect("}", "PG028", "after a field") {
|
|
self.recover_to_close();
|
|
}
|
|
Some(PacketField {
|
|
name: name.text,
|
|
kind,
|
|
count,
|
|
line: name.line,
|
|
column: name.column,
|
|
})
|
|
}
|
|
|
|
fn parse_block(&mut self) -> Option<PacketBlock> {
|
|
if !self.expect("{", "PG010", "to begin a block") {
|
|
return None;
|
|
}
|
|
let name = self.atom("PG011", "a block name")?;
|
|
let repetition_token = self.atom("PG012", "a block repetition")?;
|
|
let repetition = match repetition_token.text.as_str() {
|
|
"Single" => BlockRepetition::Single,
|
|
"Variable" => BlockRepetition::Variable,
|
|
"Multiple" => {
|
|
let count = self.atom("PG013", "a Multiple block count")?;
|
|
match count.text.parse::<usize>() {
|
|
Ok(value) if value > 0 => BlockRepetition::Multiple(value),
|
|
_ => {
|
|
self.diagnostics.push(packet_diagnostic(
|
|
self.path,
|
|
Some(&count),
|
|
"PG014",
|
|
"Multiple block count must be a positive decimal integer",
|
|
));
|
|
BlockRepetition::Multiple(1)
|
|
}
|
|
}
|
|
}
|
|
other => {
|
|
self.diagnostics.push(packet_diagnostic(
|
|
self.path,
|
|
Some(&repetition_token),
|
|
"PG015",
|
|
format!("unknown block repetition {other:?}"),
|
|
));
|
|
BlockRepetition::Single
|
|
}
|
|
};
|
|
let mut fields = Vec::new();
|
|
while self.current().is_some_and(|token| token.text == "{") {
|
|
if let Some(field) = self.parse_field() {
|
|
fields.push(field);
|
|
}
|
|
}
|
|
if !self.expect("}", "PG016", "after a block") {
|
|
self.recover_to_close();
|
|
}
|
|
let mut names = BTreeSet::new();
|
|
for field in &fields {
|
|
if !names.insert(&field.name) {
|
|
self.diagnostics.push(packet_diagnostic(
|
|
self.path,
|
|
Some(&Token {
|
|
text: field.name.clone(),
|
|
line: field.line,
|
|
column: field.column,
|
|
}),
|
|
"PG017",
|
|
format!("duplicate field {:?} in block {:?}", field.name, name.text),
|
|
));
|
|
}
|
|
}
|
|
Some(PacketBlock {
|
|
name: name.text,
|
|
repetition,
|
|
fields,
|
|
line: name.line,
|
|
column: name.column,
|
|
})
|
|
}
|
|
|
|
#[allow(clippy::too_many_lines)]
|
|
fn parse_packet(&mut self) -> Option<PacketDefinition> {
|
|
if !self.expect("{", "PG003", "to begin a packet") {
|
|
return None;
|
|
}
|
|
let name = self.atom("PG004", "a packet name")?;
|
|
let frequency_token = self.atom("PG005", "a packet frequency")?;
|
|
let id_token = self.atom("PG006", "a packet ID")?;
|
|
let trust_token = self.atom("PG007", "a packet trust marker")?;
|
|
let coding_token = self.atom("PG008", "a packet coding marker")?;
|
|
let frequency = match frequency_token.text.as_str() {
|
|
"Fixed" | "Low" => PacketFrequency::Low,
|
|
"Medium" | "Mid" => PacketFrequency::Medium,
|
|
"High" => PacketFrequency::High,
|
|
other => {
|
|
self.diagnostics.push(packet_diagnostic(
|
|
self.path,
|
|
Some(&frequency_token),
|
|
"PG030",
|
|
format!("unknown packet frequency {other:?}"),
|
|
));
|
|
PacketFrequency::Low
|
|
}
|
|
};
|
|
let parsed_id = if let Some(hex) = id_token
|
|
.text
|
|
.strip_prefix("0x")
|
|
.or_else(|| id_token.text.strip_prefix("0X"))
|
|
{
|
|
u32::from_str_radix(hex, 16)
|
|
} else {
|
|
id_token.text.parse::<u32>()
|
|
};
|
|
let id = if let Ok(value) = parsed_id {
|
|
(value & 0xffff) as u16
|
|
} else {
|
|
self.diagnostics.push(packet_diagnostic(
|
|
self.path,
|
|
Some(&id_token),
|
|
"PG031",
|
|
"packet ID must be a decimal or hexadecimal unsigned integer",
|
|
));
|
|
0
|
|
};
|
|
let trusted = match trust_token.text.as_str() {
|
|
"Trusted" => true,
|
|
"NotTrusted" => false,
|
|
other => {
|
|
self.diagnostics.push(packet_diagnostic(
|
|
self.path,
|
|
Some(&trust_token),
|
|
"PG032",
|
|
format!("unknown packet trust marker {other:?}"),
|
|
));
|
|
false
|
|
}
|
|
};
|
|
let zerocoded = match coding_token.text.as_str() {
|
|
"Zerocoded" => true,
|
|
"Unencoded" => false,
|
|
other => {
|
|
self.diagnostics.push(packet_diagnostic(
|
|
self.path,
|
|
Some(&coding_token),
|
|
"PG033",
|
|
format!("unknown packet coding marker {other:?}"),
|
|
));
|
|
false
|
|
}
|
|
};
|
|
let mut flags = Vec::new();
|
|
while self
|
|
.current()
|
|
.is_some_and(|token| !matches!(token.text.as_str(), "{" | "}"))
|
|
{
|
|
if let Some(flag) = self.advance() {
|
|
flags.push(flag.text);
|
|
}
|
|
}
|
|
let mut blocks = Vec::new();
|
|
while self.current().is_some_and(|token| token.text == "{") {
|
|
if let Some(block) = self.parse_block() {
|
|
blocks.push(block);
|
|
}
|
|
}
|
|
if !self.expect("}", "PG009", "after a packet") {
|
|
self.recover_to_close();
|
|
}
|
|
let mut block_names = BTreeSet::new();
|
|
for block in &blocks {
|
|
if !block_names.insert(&block.name) {
|
|
self.diagnostics.push(packet_diagnostic(
|
|
self.path,
|
|
Some(&Token {
|
|
text: block.name.clone(),
|
|
line: block.line,
|
|
column: block.column,
|
|
}),
|
|
"PG034",
|
|
format!("duplicate block {:?} in packet {:?}", block.name, name.text),
|
|
));
|
|
}
|
|
}
|
|
Some(PacketDefinition {
|
|
id,
|
|
name: name.text,
|
|
frequency,
|
|
trusted,
|
|
zerocoded,
|
|
flags,
|
|
blocks,
|
|
line: name.line,
|
|
column: name.column,
|
|
})
|
|
}
|
|
}
|
|
|
|
pub fn parse_packet_template(path: &str, text: &str) -> Result<PacketProtocol, Vec<Diagnostic>> {
|
|
let tokens = tokenize_packet_template(path, text)?;
|
|
let mut parser = PacketParser {
|
|
path,
|
|
tokens,
|
|
position: 0,
|
|
diagnostics: Vec::new(),
|
|
};
|
|
while parser.current().is_some_and(|token| token.text != "{") {
|
|
parser.position += 1;
|
|
}
|
|
let mut packets = Vec::new();
|
|
while parser.current().is_some() {
|
|
if parser.current().is_some_and(|token| token.text != "{") {
|
|
parser.diagnostics.push(packet_diagnostic(
|
|
path,
|
|
parser.current(),
|
|
"PG035",
|
|
"unexpected token outside a packet",
|
|
));
|
|
parser.position += 1;
|
|
continue;
|
|
}
|
|
if let Some(packet) = parser.parse_packet() {
|
|
packets.push(packet);
|
|
} else {
|
|
parser.recover_to_close();
|
|
}
|
|
}
|
|
let mut names = BTreeMap::new();
|
|
let mut ids = BTreeMap::new();
|
|
for packet in &packets {
|
|
if let Some((line, column)) =
|
|
names.insert(packet.name.clone(), (packet.line, packet.column))
|
|
{
|
|
parser.diagnostics.push(packet_diagnostic(
|
|
path,
|
|
Some(&Token {
|
|
text: packet.name.clone(),
|
|
line: packet.line,
|
|
column: packet.column,
|
|
}),
|
|
"PG036",
|
|
format!(
|
|
"duplicate packet {:?}; first declared at {line}:{column}",
|
|
packet.name
|
|
),
|
|
));
|
|
}
|
|
let key = (packet.frequency as u8, packet.id);
|
|
if let Some((other, line, column)) =
|
|
ids.insert(key, (&packet.name, packet.line, packet.column))
|
|
{
|
|
parser.diagnostics.push(packet_diagnostic(
|
|
path,
|
|
Some(&Token {
|
|
text: packet.name.clone(),
|
|
line: packet.line,
|
|
column: packet.column,
|
|
}),
|
|
"PG037",
|
|
format!(
|
|
"duplicate {:?} packet ID {}; {:?} was declared at {line}:{column}",
|
|
packet.frequency, packet.id, other
|
|
),
|
|
));
|
|
}
|
|
}
|
|
parser.diagnostics.sort_by(|left, right| {
|
|
(&left.path, left.line, left.column, left.code, &left.message).cmp(&(
|
|
&right.path,
|
|
right.line,
|
|
right.column,
|
|
right.code,
|
|
&right.message,
|
|
))
|
|
});
|
|
if parser.diagnostics.is_empty() {
|
|
Ok(PacketProtocol { packets })
|
|
} else {
|
|
Err(parser.diagnostics)
|
|
}
|
|
}
|
|
|
|
pub fn generated_rust(generator: &str, sources: &[&InputSpec], body: &str) -> Vec<u8> {
|
|
let mut output = String::new();
|
|
output.push_str("// @generated by libremetaverse-codegen; DO NOT EDIT.\n");
|
|
output.push_str("// Regenerate: cargo run -p libremetaverse-codegen -- generate\n");
|
|
let _ = writeln!(output, "// Generator: {generator}");
|
|
for source in sources {
|
|
let _ = writeln!(
|
|
output,
|
|
"// Source: {} sha256={} license={}",
|
|
source.vendored_path, source.sha256, source.license
|
|
);
|
|
}
|
|
output.push('\n');
|
|
output.push_str(body.trim_end());
|
|
output.push('\n');
|
|
output.into_bytes()
|
|
}
|
|
|
|
fn rust_snake(name: &str) -> String {
|
|
let characters: Vec<char> = name.trim_start_matches('@').chars().collect();
|
|
let mut output = String::new();
|
|
for (index, character) in characters.iter().copied().enumerate() {
|
|
let previous = index
|
|
.checked_sub(1)
|
|
.and_then(|i| characters.get(i))
|
|
.copied();
|
|
let next = characters.get(index + 1).copied();
|
|
if character.is_ascii_uppercase()
|
|
&& !output.is_empty()
|
|
&& (previous.is_some_and(|value| value.is_ascii_lowercase() || value.is_ascii_digit())
|
|
|| (previous.is_some_and(|value| value.is_ascii_uppercase())
|
|
&& next.is_some_and(|value| value.is_ascii_lowercase())))
|
|
{
|
|
output.push('_');
|
|
}
|
|
if character.is_ascii_alphanumeric() || character == '_' {
|
|
output.push(character.to_ascii_lowercase());
|
|
} else {
|
|
output.push('_');
|
|
}
|
|
}
|
|
while output.contains("__") {
|
|
output = output.replace("__", "_");
|
|
}
|
|
let mut output = output.trim_matches('_').to_owned();
|
|
if output.is_empty() {
|
|
output.push_str("item");
|
|
}
|
|
if output.starts_with(|character: char| character.is_ascii_digit())
|
|
|| RUST_KEYWORDS.contains(&output.as_str())
|
|
{
|
|
output.push('_');
|
|
}
|
|
output
|
|
}
|
|
|
|
const fn field_fixed_length(field: &PacketField) -> usize {
|
|
match field.kind {
|
|
FieldKind::Bool | FieldKind::U8 | FieldKind::S8 => 1,
|
|
FieldKind::U16 | FieldKind::S16 | FieldKind::IpPort => 2,
|
|
FieldKind::U32 | FieldKind::S32 | FieldKind::F32 | FieldKind::IpAddr => 4,
|
|
FieldKind::U64 | FieldKind::F64 => 8,
|
|
FieldKind::LlVector3 | FieldKind::LlQuaternion => 12,
|
|
FieldKind::LlUuid | FieldKind::LlVector4 => 16,
|
|
FieldKind::LlVector3d => 24,
|
|
FieldKind::Fixed => field.count,
|
|
FieldKind::Variable => 0,
|
|
}
|
|
}
|
|
|
|
fn field_kind_name(kind: FieldKind) -> &'static str {
|
|
match kind {
|
|
FieldKind::U8 => "U8",
|
|
FieldKind::U16 => "U16",
|
|
FieldKind::U32 => "U32",
|
|
FieldKind::U64 => "U64",
|
|
FieldKind::S8 => "S8",
|
|
FieldKind::S16 => "S16",
|
|
FieldKind::S32 => "S32",
|
|
FieldKind::F32 => "F32",
|
|
FieldKind::F64 => "F64",
|
|
FieldKind::LlUuid => "LlUuid",
|
|
FieldKind::Bool => "Bool",
|
|
FieldKind::LlVector3 => "LlVector3",
|
|
FieldKind::LlVector3d => "LlVector3d",
|
|
FieldKind::LlVector4 => "LlVector4",
|
|
FieldKind::LlQuaternion => "LlQuaternion",
|
|
FieldKind::IpAddr => "IpAddr",
|
|
FieldKind::IpPort => "IpPort",
|
|
FieldKind::Variable => "Variable",
|
|
FieldKind::Fixed => "Fixed",
|
|
}
|
|
}
|
|
|
|
fn frequency_name(frequency: PacketFrequency) -> &'static str {
|
|
match frequency {
|
|
PacketFrequency::Low => "Low",
|
|
PacketFrequency::Medium => "Medium",
|
|
PacketFrequency::High => "High",
|
|
}
|
|
}
|
|
|
|
fn field_default(kind: FieldKind) -> &'static str {
|
|
match kind {
|
|
FieldKind::Bool => "false",
|
|
FieldKind::U8
|
|
| FieldKind::U16
|
|
| FieldKind::U32
|
|
| FieldKind::U64
|
|
| FieldKind::S8
|
|
| FieldKind::S16
|
|
| FieldKind::S32
|
|
| FieldKind::IpAddr
|
|
| FieldKind::IpPort => "0",
|
|
FieldKind::F32 => "0.0_f32",
|
|
FieldKind::F64 => "0.0_f64",
|
|
FieldKind::LlUuid => "libremetaverse_types::UUID::zero()",
|
|
FieldKind::LlVector3 => "libremetaverse_types::Vector3::zero()",
|
|
FieldKind::LlVector3d => "libremetaverse_types::Vector3d::zero()",
|
|
FieldKind::LlVector4 => "libremetaverse_types::Vector4::zero()",
|
|
FieldKind::LlQuaternion => {
|
|
"libremetaverse_types::Quaternion { x: 0.0, y: 0.0, z: 0.0, w: 0.0 }"
|
|
}
|
|
FieldKind::Variable | FieldKind::Fixed => "Vec::new()",
|
|
}
|
|
}
|
|
|
|
fn ordered_packets(protocol: &PacketProtocol) -> Vec<&PacketDefinition> {
|
|
[
|
|
PacketFrequency::Low,
|
|
PacketFrequency::Medium,
|
|
PacketFrequency::High,
|
|
]
|
|
.into_iter()
|
|
.flat_map(|frequency| {
|
|
protocol
|
|
.packets
|
|
.iter()
|
|
.filter(move |packet| packet.frequency == frequency)
|
|
})
|
|
.collect()
|
|
}
|
|
|
|
fn packet_type_value(packet: &PacketDefinition) -> u32 {
|
|
let prefix = match packet.frequency {
|
|
PacketFrequency::Low => 0x1_0000,
|
|
PacketFrequency::Medium => 0x2_0000,
|
|
PacketFrequency::High => 0x3_0000,
|
|
};
|
|
prefix | u32::from(packet.id)
|
|
}
|
|
|
|
fn append_descriptor_types(body: &mut String) {
|
|
body.push_str(
|
|
"#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]\n\
|
|
pub enum FieldKind { U8, U16, U32, U64, S8, S16, S32, F32, F64, LlUuid, Bool, LlVector3, LlVector3d, LlVector4, LlQuaternion, IpAddr, IpPort, Variable, Fixed }\n\n\
|
|
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]\n\
|
|
pub enum BlockRepetition { Single, Multiple(usize), Variable }\n\n\
|
|
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]\n\
|
|
pub struct FieldDescriptor { pub name: &'static str, pub kind: FieldKind, pub count: usize, pub fixed_width: usize }\n\n\
|
|
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]\n\
|
|
pub struct BlockDescriptor { pub name: &'static str, pub repetition: BlockRepetition, pub fields: &'static [FieldDescriptor] }\n\n\
|
|
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]\n\
|
|
pub struct PacketDescriptor { pub name: &'static str, pub packet_type: PacketType, pub id: u16, pub frequency: crate::PacketFrequency, pub trusted: bool, pub zerocoded: bool, pub flags: &'static [&'static str], pub blocks: &'static [BlockDescriptor] }\n\n",
|
|
);
|
|
}
|
|
|
|
fn append_packet_type(body: &mut String, packets: &[&PacketDefinition]) {
|
|
body.push_str("#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]\n#[repr(i32)]\npub enum PacketType {\n Default = 0,\n");
|
|
for packet in packets {
|
|
let prefix = match packet.frequency {
|
|
PacketFrequency::Low => 1,
|
|
PacketFrequency::Medium => 2,
|
|
PacketFrequency::High => 3,
|
|
};
|
|
let _ = writeln!(body, " {} = 0x{prefix}_{:04x},", packet.name, packet.id);
|
|
}
|
|
body.push_str("}\n\n");
|
|
}
|
|
|
|
fn append_descriptors(body: &mut String, packets: &[&PacketDefinition]) {
|
|
for (packet_index, packet) in packets.iter().enumerate() {
|
|
for (block_index, block) in packet.blocks.iter().enumerate() {
|
|
let _ = writeln!(
|
|
body,
|
|
"static PACKET_{packet_index}_BLOCK_{block_index}_FIELDS: &[FieldDescriptor] = &["
|
|
);
|
|
for field in &block.fields {
|
|
let _ = writeln!(
|
|
body,
|
|
" FieldDescriptor {{ name: {:?}, kind: FieldKind::{}, count: {}, fixed_width: {} }},",
|
|
field.name,
|
|
field_kind_name(field.kind),
|
|
field.count,
|
|
field_fixed_length(field)
|
|
);
|
|
}
|
|
body.push_str("];\n");
|
|
}
|
|
let _ = writeln!(
|
|
body,
|
|
"static PACKET_{packet_index}_BLOCKS: &[BlockDescriptor] = &["
|
|
);
|
|
for (block_index, block) in packet.blocks.iter().enumerate() {
|
|
let repetition = match block.repetition {
|
|
BlockRepetition::Single => "BlockRepetition::Single".to_owned(),
|
|
BlockRepetition::Variable => "BlockRepetition::Variable".to_owned(),
|
|
BlockRepetition::Multiple(count) => {
|
|
format!("BlockRepetition::Multiple({count})")
|
|
}
|
|
};
|
|
let _ = writeln!(
|
|
body,
|
|
" BlockDescriptor {{ name: {:?}, repetition: {repetition}, fields: PACKET_{packet_index}_BLOCK_{block_index}_FIELDS }},",
|
|
block.name
|
|
);
|
|
}
|
|
body.push_str("];\n\n");
|
|
}
|
|
body.push_str("pub static PACKETS: &[PacketDescriptor] = &[\n");
|
|
for (packet_index, packet) in packets.iter().enumerate() {
|
|
let flags = packet
|
|
.flags
|
|
.iter()
|
|
.map(|flag| format!("{flag:?}"))
|
|
.collect::<Vec<_>>()
|
|
.join(", ");
|
|
let _ = writeln!(
|
|
body,
|
|
" PacketDescriptor {{ name: {:?}, packet_type: PacketType::{}, id: {}, frequency: crate::PacketFrequency::{}, trusted: {}, zerocoded: {}, flags: &[{}], blocks: PACKET_{packet_index}_BLOCKS }},",
|
|
packet.name,
|
|
packet.name,
|
|
packet.id,
|
|
frequency_name(packet.frequency),
|
|
packet.trusted,
|
|
packet.zerocoded,
|
|
flags
|
|
);
|
|
}
|
|
body.push_str("];\n\n");
|
|
}
|
|
|
|
fn append_dispatch(body: &mut String, packets: &[&PacketDefinition]) {
|
|
body.push_str("#[must_use]\n#[allow(clippy::too_many_lines)]\npub fn packet_type(id: u16, frequency: crate::PacketFrequency) -> PacketType {\n match (frequency, id) {\n");
|
|
for packet in packets {
|
|
let _ = writeln!(
|
|
body,
|
|
" (crate::PacketFrequency::{}, {}) => PacketType::{},",
|
|
frequency_name(packet.frequency),
|
|
packet.id,
|
|
packet.name
|
|
);
|
|
}
|
|
body.push_str(" _ => PacketType::Default,\n }\n}\n\n");
|
|
body.push_str("#[must_use]\n#[allow(clippy::too_many_lines)]\npub fn descriptor_by_type(packet_type: PacketType) -> Option<&'static PacketDescriptor> {\n match packet_type {\n");
|
|
for (index, packet) in packets.iter().enumerate() {
|
|
let _ = writeln!(
|
|
body,
|
|
" PacketType::{} => Some(&PACKETS[{index}]),",
|
|
packet.name
|
|
);
|
|
}
|
|
body.push_str(" PacketType::Default => None,\n }\n}\n\n");
|
|
body.push_str("#[must_use]\n#[allow(clippy::too_many_lines)]\npub fn descriptor_by_name(name: &str) -> Option<&'static PacketDescriptor> {\n match name {\n");
|
|
for (index, packet) in packets.iter().enumerate() {
|
|
let _ = writeln!(
|
|
body,
|
|
" {:?} => Some(&PACKETS[{index}]),",
|
|
packet.name
|
|
);
|
|
}
|
|
body.push_str(" _ => None,\n }\n}\n\n");
|
|
body.push_str(
|
|
"pub(crate) fn build_packet(packet_type: PacketType) -> Result<crate::packets::Packet, crate::Error> {\n\
|
|
let descriptor = descriptor_by_type(packet_type).ok_or(crate::Error::InvalidOperation)?;\n\
|
|
Ok(crate::packets::Packet {\n\
|
|
has_variable_blocks: descriptor.blocks.iter().any(|block| matches!(block.repetition, BlockRepetition::Variable)),\n\
|
|
header: new_header(descriptor.frequency, descriptor.id, descriptor.zerocoded),\n\
|
|
type_: packet_type,\n\
|
|
})\n\
|
|
}\n\n\
|
|
pub(crate) const fn new_header(frequency: crate::PacketFrequency, id: u16, zerocoded: bool) -> crate::packets::Header {\n\
|
|
crate::packets::Header { ack_list: None, appended_acks: false, frequency, id, reliable: true, resent: false, sequence: 0, zerocoded }\n\
|
|
}\n\n\
|
|
fn usize_to_i32(value: usize) -> i32 { i32::try_from(value).unwrap_or(i32::MAX) }\n\n\
|
|
pub(crate) trait GeneratedBlock { fn new_generated() -> Self; fn generated_length(&self) -> i32; }\n\n\
|
|
pub(crate) trait GeneratedPacket { fn new_generated() -> Self; fn generated_length(&self) -> i32; const USES_BUFFER_POOLING: bool; }\n\n",
|
|
);
|
|
}
|
|
|
|
fn append_block_impls(body: &mut String, packets: &[&PacketDefinition]) {
|
|
for packet in packets {
|
|
for block in &packet.blocks {
|
|
let type_name = format!("{}Packet{}Block", packet.name, block.name);
|
|
let _ = writeln!(
|
|
body,
|
|
"impl GeneratedBlock for crate::packets::{type_name} {{\n fn new_generated() -> Self {{\n Self {{"
|
|
);
|
|
for field in &block.fields {
|
|
let _ = writeln!(
|
|
body,
|
|
" {}: {},",
|
|
rust_snake(&field.name),
|
|
field_default(field.kind)
|
|
);
|
|
}
|
|
body.push_str(" }\n }\n\n fn generated_length(&self) -> i32 {\n");
|
|
let fixed = block.fields.iter().map(field_fixed_length).sum::<usize>();
|
|
let variable_fields = block
|
|
.fields
|
|
.iter()
|
|
.filter(|field| field.kind == FieldKind::Variable)
|
|
.collect::<Vec<_>>();
|
|
if variable_fields.is_empty() {
|
|
let _ = writeln!(body, " {fixed}_i32");
|
|
} else {
|
|
let _ = writeln!(body, " let mut length = {fixed}_i32;");
|
|
for field in variable_fields {
|
|
let _ = writeln!(
|
|
body,
|
|
" length = length.saturating_add({}).saturating_add(usize_to_i32(self.{}.len()));",
|
|
field.count,
|
|
rust_snake(&field.name)
|
|
);
|
|
}
|
|
body.push_str(" length\n");
|
|
}
|
|
body.push_str(" }\n}\n\n");
|
|
}
|
|
}
|
|
}
|
|
|
|
fn packet_has_composed_base(name: &str) -> bool {
|
|
matches!(name, "DirPlacesReply" | "TestMessage")
|
|
}
|
|
|
|
fn append_packet_impls(body: &mut String, packets: &[&PacketDefinition]) {
|
|
for packet in packets {
|
|
let type_name = format!("{}Packet", packet.name);
|
|
let variable_count = packet
|
|
.blocks
|
|
.iter()
|
|
.filter(|block| block.repetition == BlockRepetition::Variable)
|
|
.count();
|
|
let base_length = match packet.frequency {
|
|
PacketFrequency::Low => 10,
|
|
PacketFrequency::Medium => 8,
|
|
PacketFrequency::High => 7,
|
|
};
|
|
let _ = writeln!(
|
|
body,
|
|
"impl GeneratedPacket for crate::packets::{type_name} {{\n const USES_BUFFER_POOLING: bool = {};\n\n fn new_generated() -> Self {{\n Self {{",
|
|
variable_count == 0
|
|
);
|
|
if packet_has_composed_base(&packet.name) {
|
|
let _ = writeln!(
|
|
body,
|
|
" base: crate::packets::Packet {{ has_variable_blocks: {}, header: new_header(crate::PacketFrequency::{}, {}, {}), type_: PacketType::{} }},",
|
|
variable_count > 0,
|
|
frequency_name(packet.frequency),
|
|
packet.id,
|
|
packet.zerocoded,
|
|
packet.name
|
|
);
|
|
}
|
|
for block in &packet.blocks {
|
|
let field_name = rust_snake(&block.name);
|
|
let block_type = format!("crate::packets::{}Packet{}Block", packet.name, block.name);
|
|
match block.repetition {
|
|
BlockRepetition::Single => {
|
|
let _ = writeln!(
|
|
body,
|
|
" {field_name}: <{block_type} as GeneratedBlock>::new_generated(),"
|
|
);
|
|
}
|
|
BlockRepetition::Variable => {
|
|
let _ = writeln!(body, " {field_name}: Vec::new(),");
|
|
}
|
|
BlockRepetition::Multiple(count) => {
|
|
let _ = writeln!(
|
|
body,
|
|
" {field_name}: (0..{count}).map(|_| <{block_type} as GeneratedBlock>::new_generated()).collect(),"
|
|
);
|
|
}
|
|
}
|
|
}
|
|
body.push_str(" }\n }\n\n fn generated_length(&self) -> i32 {\n");
|
|
if packet.blocks.is_empty() {
|
|
let _ = writeln!(body, " {base_length}_i32");
|
|
body.push_str(" }\n}\n\n");
|
|
continue;
|
|
}
|
|
let _ = writeln!(
|
|
body,
|
|
" let mut length = {}_i32;",
|
|
base_length + variable_count
|
|
);
|
|
for block in &packet.blocks {
|
|
let field_name = rust_snake(&block.name);
|
|
match block.repetition {
|
|
BlockRepetition::Single => {
|
|
let _ = writeln!(
|
|
body,
|
|
" length = length.saturating_add(GeneratedBlock::generated_length(&self.{field_name}));"
|
|
);
|
|
}
|
|
BlockRepetition::Variable => {
|
|
let _ = writeln!(body, " length = length.saturating_add(1);");
|
|
let _ = writeln!(
|
|
body,
|
|
" for block in &self.{field_name} {{ length = length.saturating_add(GeneratedBlock::generated_length(block)); }}"
|
|
);
|
|
}
|
|
BlockRepetition::Multiple(_) => {
|
|
let _ = writeln!(
|
|
body,
|
|
" for block in &self.{field_name} {{ length = length.saturating_add(GeneratedBlock::generated_length(block)); }}"
|
|
);
|
|
}
|
|
}
|
|
}
|
|
body.push_str(" length\n }\n}\n\n");
|
|
}
|
|
}
|
|
|
|
fn csharp_field_type(kind: FieldKind) -> &'static str {
|
|
match kind {
|
|
FieldKind::U8 => "System.Byte",
|
|
FieldKind::U16 | FieldKind::IpPort => "System.UInt16",
|
|
FieldKind::U32 | FieldKind::IpAddr => "System.UInt32",
|
|
FieldKind::U64 => "System.UInt64",
|
|
FieldKind::S8 => "System.SByte",
|
|
FieldKind::S16 => "System.Int16",
|
|
FieldKind::S32 => "System.Int32",
|
|
FieldKind::F32 => "System.Single",
|
|
FieldKind::F64 => "System.Double",
|
|
FieldKind::LlUuid => "LibreMetaverse.UUID",
|
|
FieldKind::Bool => "System.Boolean",
|
|
FieldKind::LlVector3 => "LibreMetaverse.Vector3",
|
|
FieldKind::LlVector3d => "LibreMetaverse.Vector3d",
|
|
FieldKind::LlVector4 => "LibreMetaverse.Vector4",
|
|
FieldKind::LlQuaternion => "LibreMetaverse.Quaternion",
|
|
FieldKind::Variable | FieldKind::Fixed => "System.Byte[]",
|
|
}
|
|
}
|
|
|
|
fn catalog_fields(value: &serde_json::Value) -> Result<BTreeMap<String, String>, String> {
|
|
let members = value["members"]
|
|
.as_array()
|
|
.ok_or("catalog type has no members array")?;
|
|
members
|
|
.iter()
|
|
.filter(|member| member["kind"] == "field" && member["static"] == false)
|
|
.map(|member| {
|
|
let name = member["name"]
|
|
.as_str()
|
|
.ok_or("catalog field has no name")?
|
|
.to_owned();
|
|
let field_type = member["type"]
|
|
.as_str()
|
|
.ok_or("catalog field has no type")?
|
|
.to_owned();
|
|
Ok((name, field_type))
|
|
})
|
|
.collect()
|
|
}
|
|
|
|
#[allow(clippy::too_many_lines)]
|
|
pub fn validate_packet_api_catalog(root: &Path, protocol: &PacketProtocol) -> Result<(), String> {
|
|
let path = root.join(PUBLIC_API_PATH);
|
|
let bytes = fs::read(&path).map_err(|error| format!("{}: {error}", path.display()))?;
|
|
let catalog: serde_json::Value =
|
|
serde_json::from_slice(&bytes).map_err(|error| format!("{}: {error}", path.display()))?;
|
|
let assembly = catalog["assemblies"]
|
|
.as_array()
|
|
.and_then(|assemblies| {
|
|
assemblies
|
|
.iter()
|
|
.find(|assembly| assembly["identity"]["name"] == "LibreMetaverse")
|
|
})
|
|
.ok_or("public API catalog has no LibreMetaverse assembly")?;
|
|
let types = assembly["types"]
|
|
.as_array()
|
|
.ok_or("LibreMetaverse catalog has no types array")?;
|
|
let types_by_id: BTreeMap<&str, &serde_json::Value> = types
|
|
.iter()
|
|
.map(|item| {
|
|
item["doc_id"]
|
|
.as_str()
|
|
.map(|id| (id, item))
|
|
.ok_or("catalog type has no doc_id")
|
|
})
|
|
.collect::<Result<_, _>>()?;
|
|
|
|
let expected_packet_ids: BTreeSet<String> = protocol
|
|
.packets
|
|
.iter()
|
|
.map(|packet| format!("T:LibreMetaverse.Packets.{}Packet", packet.name))
|
|
.collect();
|
|
let expected_block_ids: BTreeSet<String> = protocol
|
|
.packets
|
|
.iter()
|
|
.flat_map(|packet| {
|
|
packet.blocks.iter().map(move |block| {
|
|
format!(
|
|
"T:LibreMetaverse.Packets.{}Packet.{}Block",
|
|
packet.name, block.name
|
|
)
|
|
})
|
|
})
|
|
.collect();
|
|
let actual_packet_ids: BTreeSet<String> = types
|
|
.iter()
|
|
.filter(|item| item["base_type"] == "LibreMetaverse.Packets.Packet")
|
|
.filter_map(|item| item["doc_id"].as_str().map(str::to_owned))
|
|
.collect();
|
|
let actual_block_ids: BTreeSet<String> = types
|
|
.iter()
|
|
.filter(|item| item["base_type"] == "LibreMetaverse.Packets.PacketBlock")
|
|
.filter_map(|item| item["doc_id"].as_str().map(str::to_owned))
|
|
.collect();
|
|
if expected_packet_ids != actual_packet_ids {
|
|
return Err(format!(
|
|
"packet API catalog mismatch: missing={:?}, stale={:?}",
|
|
expected_packet_ids
|
|
.difference(&actual_packet_ids)
|
|
.collect::<Vec<_>>(),
|
|
actual_packet_ids
|
|
.difference(&expected_packet_ids)
|
|
.collect::<Vec<_>>()
|
|
));
|
|
}
|
|
if expected_block_ids != actual_block_ids {
|
|
return Err(format!(
|
|
"packet block API catalog mismatch: missing={:?}, stale={:?}",
|
|
expected_block_ids
|
|
.difference(&actual_block_ids)
|
|
.collect::<Vec<_>>(),
|
|
actual_block_ids
|
|
.difference(&expected_block_ids)
|
|
.collect::<Vec<_>>()
|
|
));
|
|
}
|
|
|
|
for packet in &protocol.packets {
|
|
let packet_id = format!("T:LibreMetaverse.Packets.{}Packet", packet.name);
|
|
let actual = catalog_fields(types_by_id[packet_id.as_str()])?;
|
|
let expected = packet
|
|
.blocks
|
|
.iter()
|
|
.map(|block| {
|
|
let name = if block.name == "Header" {
|
|
"_Header".to_owned()
|
|
} else {
|
|
block.name.clone()
|
|
};
|
|
let array = if block.repetition == BlockRepetition::Single {
|
|
""
|
|
} else {
|
|
"[]"
|
|
};
|
|
(
|
|
name,
|
|
format!(
|
|
"LibreMetaverse.Packets.{}Packet.{}Block{array}",
|
|
packet.name, block.name
|
|
),
|
|
)
|
|
})
|
|
.collect::<BTreeMap<_, _>>();
|
|
if expected != actual {
|
|
return Err(format!(
|
|
"{packet_id} field mismatch: expected={expected:?}, actual={actual:?}"
|
|
));
|
|
}
|
|
for block in &packet.blocks {
|
|
let block_id = format!(
|
|
"T:LibreMetaverse.Packets.{}Packet.{}Block",
|
|
packet.name, block.name
|
|
);
|
|
let actual = catalog_fields(types_by_id[block_id.as_str()])?;
|
|
let expected = block
|
|
.fields
|
|
.iter()
|
|
.map(|field| (field.name.clone(), csharp_field_type(field.kind).to_owned()))
|
|
.collect::<BTreeMap<_, _>>();
|
|
if expected != actual {
|
|
return Err(format!(
|
|
"{block_id} field mismatch: expected={expected:?}, actual={actual:?}"
|
|
));
|
|
}
|
|
}
|
|
}
|
|
|
|
let packet_type = types_by_id
|
|
.get("T:LibreMetaverse.Packets.PacketType")
|
|
.ok_or("public API catalog has no PacketType")?;
|
|
let actual_values = packet_type["members"]
|
|
.as_array()
|
|
.ok_or("PacketType has no members array")?
|
|
.iter()
|
|
.filter(|member| member["kind"] == "enum_value")
|
|
.map(|member| {
|
|
let name = member["name"]
|
|
.as_str()
|
|
.ok_or("PacketType value has no name")?
|
|
.to_owned();
|
|
let value = member["value"]["value"]
|
|
.as_str()
|
|
.ok_or("PacketType value has no numeric value")?
|
|
.parse::<u32>()
|
|
.map_err(|_| "PacketType value is not a u32")?;
|
|
Ok::<_, &'static str>((name, value))
|
|
})
|
|
.collect::<Result<BTreeMap<_, _>, _>>()?;
|
|
let mut expected_values = BTreeMap::from([("Default".to_owned(), 0)]);
|
|
expected_values.extend(
|
|
protocol
|
|
.packets
|
|
.iter()
|
|
.map(|packet| (packet.name.clone(), packet_type_value(packet))),
|
|
);
|
|
if expected_values != actual_values {
|
|
return Err(format!(
|
|
"PacketType enum mismatch: missing/stale values = {}/{}",
|
|
expected_values
|
|
.keys()
|
|
.filter(|name| !actual_values.contains_key(*name))
|
|
.count(),
|
|
actual_values
|
|
.keys()
|
|
.filter(|name| !expected_values.contains_key(*name))
|
|
.count()
|
|
));
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
pub fn packet_catalog_bytes(root: &Path) -> Result<Vec<u8>, String> {
|
|
let inventory = load_inventory(root)?;
|
|
verify_inputs(root, &inventory)?;
|
|
let input = inventory
|
|
.inputs
|
|
.iter()
|
|
.find(|input| input.id == "message_template")
|
|
.ok_or("source inventory has no message_template input")?;
|
|
let generator = inventory
|
|
.generators
|
|
.iter()
|
|
.find(|generator| generator.id == "packets")
|
|
.ok_or("source inventory has no packets generator")?;
|
|
let path = root.join(&input.vendored_path);
|
|
let bytes = fs::read(&path).map_err(|error| format!("{}: {error}", path.display()))?;
|
|
let text = normalize_text(&input.vendored_path, &bytes).map_err(|error| error.to_string())?;
|
|
let protocol = parse_packet_template(&input.vendored_path, &text).map_err(|diagnostics| {
|
|
diagnostics
|
|
.into_iter()
|
|
.map(|item| item.to_string())
|
|
.collect::<Vec<_>>()
|
|
.join("\n")
|
|
})?;
|
|
validate_packet_api_catalog(root, &protocol)?;
|
|
let packets = ordered_packets(&protocol);
|
|
let mut body = String::new();
|
|
let _ = writeln!(
|
|
body,
|
|
"pub const GOLDEN_GENERATOR_SOURCE: &str = {:?};",
|
|
generator.reference_source
|
|
);
|
|
let _ = writeln!(
|
|
body,
|
|
"pub const GOLDEN_GENERATOR_SHA256: &str = {:?};",
|
|
generator.sha256
|
|
);
|
|
let _ = writeln!(
|
|
body,
|
|
"pub const GOLDEN_GENERATOR_LICENSE: &str = {:?};\n",
|
|
generator.license
|
|
);
|
|
append_descriptor_types(&mut body);
|
|
append_packet_type(&mut body, &packets);
|
|
append_descriptors(&mut body, &packets);
|
|
append_dispatch(&mut body, &packets);
|
|
append_block_impls(&mut body, &packets);
|
|
append_packet_impls(&mut body, &packets);
|
|
let syntax = syn::parse_file(&body)
|
|
.map_err(|error| format!("generated packet Rust is invalid: {error}"))?;
|
|
let formatted = prettyplease::unparse(&syntax);
|
|
Ok(generated_rust("packets", &[input], &formatted))
|
|
}
|
|
|
|
pub fn load_inventory(root: &Path) -> Result<Inventory, String> {
|
|
let path = root.join(INVENTORY_PATH);
|
|
let bytes = fs::read(&path).map_err(|error| format!("{}: {error}", path.display()))?;
|
|
let inventory: Inventory =
|
|
serde_json::from_slice(&bytes).map_err(|error| format!("{}: {error}", path.display()))?;
|
|
validate_inventory(&inventory)?;
|
|
Ok(inventory)
|
|
}
|
|
|
|
fn validate_inventory(inventory: &Inventory) -> Result<(), String> {
|
|
if inventory.schema != 1 || inventory.upstream_commit.len() != 40 {
|
|
return Err("unsupported or malformed source inventory header".to_owned());
|
|
}
|
|
let mut input_ids = BTreeSet::new();
|
|
for input in &inventory.inputs {
|
|
validate_relative(&input.reference_path)?;
|
|
validate_relative(&input.vendored_path)?;
|
|
validate_hash(&input.sha256)?;
|
|
if input.license != "BSD-3-Clause" || !input_ids.insert(&input.id) {
|
|
return Err(format!("invalid or duplicate input {}", input.id));
|
|
}
|
|
}
|
|
let mut generator_ids = BTreeSet::new();
|
|
for generator in &inventory.generators {
|
|
validate_relative(&generator.reference_source)?;
|
|
validate_hash(&generator.sha256)?;
|
|
if generator.license != "BSD-3-Clause" || !generator_ids.insert(&generator.id) {
|
|
return Err(format!("invalid or duplicate generator {}", generator.id));
|
|
}
|
|
for input in &generator.inputs {
|
|
if !input_ids.contains(input) {
|
|
return Err(format!(
|
|
"generator {} references unknown input {input}",
|
|
generator.id
|
|
));
|
|
}
|
|
}
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
fn validate_relative(value: &str) -> Result<(), String> {
|
|
let path = Path::new(value);
|
|
if path.is_absolute()
|
|
|| path
|
|
.components()
|
|
.any(|part| !matches!(part, Component::Normal(_)))
|
|
{
|
|
return Err(format!("unsafe inventory path {value}"));
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
fn validate_hash(value: &str) -> Result<(), String> {
|
|
if value.len() == 64 && value.bytes().all(|byte| byte.is_ascii_hexdigit()) {
|
|
Ok(())
|
|
} else {
|
|
Err(format!("invalid SHA-256 {value}"))
|
|
}
|
|
}
|
|
|
|
fn sha256(bytes: &[u8]) -> String {
|
|
const HEX: &[u8; 16] = b"0123456789abcdef";
|
|
let mut output = String::with_capacity(64);
|
|
for byte in Sha256::digest(bytes) {
|
|
output.push(char::from(HEX[usize::from(byte >> 4)]));
|
|
output.push(char::from(HEX[usize::from(byte & 0x0f)]));
|
|
}
|
|
output
|
|
}
|
|
|
|
pub fn verify_inputs(root: &Path, inventory: &Inventory) -> Result<(), String> {
|
|
for input in &inventory.inputs {
|
|
let path = root.join(&input.vendored_path);
|
|
let bytes = fs::read(&path).map_err(|error| format!("{}: {error}", path.display()))?;
|
|
let actual = sha256(&bytes);
|
|
if actual != input.sha256 {
|
|
return Err(format!(
|
|
"{}: SHA-256 mismatch: expected {}, found {actual}",
|
|
path.display(),
|
|
input.sha256
|
|
));
|
|
}
|
|
normalize_text(&input.vendored_path, &bytes).map_err(|error| error.to_string())?;
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
pub fn vendor_inputs(root: &Path, reference: &Path) -> Result<(), String> {
|
|
let inventory = load_inventory(root)?;
|
|
for generator in &inventory.generators {
|
|
let source = reference.join(&generator.reference_source);
|
|
let bytes = fs::read(&source).map_err(|error| format!("{}: {error}", source.display()))?;
|
|
let actual = sha256(&bytes);
|
|
if actual != generator.sha256 {
|
|
return Err(format!(
|
|
"{}: pinned generator SHA-256 mismatch: expected {}, found {actual}",
|
|
source.display(),
|
|
generator.sha256
|
|
));
|
|
}
|
|
}
|
|
let mut snapshots = Vec::with_capacity(inventory.inputs.len());
|
|
for input in &inventory.inputs {
|
|
let source = reference.join(&input.reference_path);
|
|
let bytes = fs::read(&source).map_err(|error| format!("{}: {error}", source.display()))?;
|
|
let actual = sha256(&bytes);
|
|
if actual != input.sha256 {
|
|
return Err(format!(
|
|
"{}: pinned SHA-256 mismatch: expected {}, found {actual}",
|
|
source.display(),
|
|
input.sha256
|
|
));
|
|
}
|
|
snapshots.push((root.join(&input.vendored_path), bytes));
|
|
}
|
|
for (target, bytes) in snapshots {
|
|
fs::create_dir_all(target.parent().ok_or("vendored input has no parent")?)
|
|
.map_err(|error| error.to_string())?;
|
|
fs::write(&target, bytes).map_err(|error| format!("{}: {error}", target.display()))?;
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
pub fn source_manifest_bytes(root: &Path) -> Result<Vec<u8>, String> {
|
|
let inventory = load_inventory(root)?;
|
|
verify_inputs(root, &inventory)?;
|
|
let mut inputs: Vec<&InputSpec> = inventory.inputs.iter().collect();
|
|
inputs.sort_by_key(|input| &input.id);
|
|
let mut generators: Vec<&GeneratorSpec> = inventory.generators.iter().collect();
|
|
generators.sort_by_key(|generator| &generator.id);
|
|
let mut body = String::new();
|
|
let _ = writeln!(
|
|
body,
|
|
"pub const UPSTREAM_COMMIT: &str = {:?};",
|
|
inventory.upstream_commit
|
|
);
|
|
let _ = writeln!(
|
|
body,
|
|
"pub const UPSTREAM_REPOSITORY: &str = {:?};",
|
|
inventory.upstream_repository
|
|
);
|
|
body.push_str("pub const SOURCES: &[(&str, &str, &str, &str)] = &[\n");
|
|
for input in &inputs {
|
|
let _ = writeln!(
|
|
body,
|
|
" ({:?}, {:?}, {:?}, {:?}),",
|
|
input.id, input.vendored_path, input.sha256, input.license
|
|
);
|
|
}
|
|
body.push_str("];\n");
|
|
body.push_str("pub const GENERATORS: &[(&str, &str, &str, &str)] = &[\n");
|
|
for generator in generators {
|
|
let _ = writeln!(
|
|
body,
|
|
" ({:?}, {:?}, {:?}, {:?}),",
|
|
generator.id, generator.reference_source, generator.sha256, generator.license
|
|
);
|
|
}
|
|
body.push_str("];\n");
|
|
Ok(generated_rust("source-manifest", &inputs, &body))
|
|
}
|
|
|
|
pub fn regenerate(root: &Path, check: bool) -> Result<(), String> {
|
|
let outputs = [
|
|
(root.join(MANIFEST_OUTPUT), source_manifest_bytes(root)?),
|
|
(root.join(PACKET_OUTPUT), packet_catalog_bytes(root)?),
|
|
];
|
|
if check {
|
|
for (path, expected) in outputs {
|
|
let actual = fs::read(&path).map_err(|error| format!("{}: {error}", path.display()))?;
|
|
if actual != expected {
|
|
return Err(format!(
|
|
"{} is stale; run the regeneration command",
|
|
path.display()
|
|
));
|
|
}
|
|
}
|
|
} else {
|
|
for (path, expected) in outputs {
|
|
fs::create_dir_all(path.parent().ok_or("generated output has no parent")?)
|
|
.map_err(|error| error.to_string())?;
|
|
fs::write(&path, expected).map_err(|error| format!("{}: {error}", path.display()))?;
|
|
}
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn checked_in_inputs_and_manifest_are_current() {
|
|
let root = workspace_root();
|
|
let first = source_manifest_bytes(&root).expect("first deterministic generation");
|
|
let second = source_manifest_bytes(&root).expect("second deterministic generation");
|
|
assert_eq!(first, second);
|
|
let first_packets = packet_catalog_bytes(&root).expect("first packet generation");
|
|
let second_packets = packet_catalog_bytes(&root).expect("second packet generation");
|
|
assert_eq!(first_packets, second_packets);
|
|
}
|
|
|
|
#[test]
|
|
fn text_normalization_and_diagnostics_are_stable() {
|
|
assert_eq!(
|
|
normalize_text("a\\b.xml", b"\xef\xbb\xbfa\r\nb\r").unwrap(),
|
|
"a\nb\n"
|
|
);
|
|
let error = normalize_text("a\\b.xml", b"one\n\0two").unwrap_err();
|
|
assert_eq!(
|
|
error.to_string(),
|
|
"a/b.xml:2:1: error[CG002]: input contains a NUL byte"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn inventory_rejects_unsafe_paths_hashes_and_unknown_inputs() {
|
|
let root = workspace_root();
|
|
let mut inventory = load_inventory(&root).expect("pinned inventory");
|
|
inventory.inputs[0].vendored_path = "../escape".to_owned();
|
|
assert!(validate_inventory(&inventory).is_err());
|
|
|
|
let mut inventory = load_inventory(&root).expect("pinned inventory");
|
|
inventory.generators[0].sha256 = "not-a-hash".to_owned();
|
|
assert!(validate_inventory(&inventory).is_err());
|
|
|
|
let mut inventory = load_inventory(&root).expect("pinned inventory");
|
|
inventory.generators[0].inputs.push("missing".to_owned());
|
|
assert!(validate_inventory(&inventory).is_err());
|
|
}
|
|
|
|
#[test]
|
|
fn packet_template_parser_preserves_ids_order_and_repetition() {
|
|
let text = "version 2.0\n{ Alpha Fixed 0xFFFFFFFB Trusted Zerocoded UDPDeprecated\n { One Single { Value U32 } }\n { Many Multiple 2 { Data Fixed 4 } }\n { Rest Variable { Name Variable 2 } }\n}\n{ Beta High 7 NotTrusted Unencoded }\n";
|
|
let protocol = parse_packet_template("fixture.msg", text).expect("valid protocol");
|
|
assert_eq!(protocol.packets.len(), 2);
|
|
let alpha = &protocol.packets[0];
|
|
assert_eq!(alpha.id, 0xfffb);
|
|
assert_eq!(alpha.frequency, PacketFrequency::Low);
|
|
assert!(alpha.trusted && alpha.zerocoded);
|
|
assert_eq!(alpha.flags, ["UDPDeprecated"]);
|
|
assert_eq!(alpha.blocks[1].repetition, BlockRepetition::Multiple(2));
|
|
assert_eq!(alpha.blocks[2].repetition, BlockRepetition::Variable);
|
|
assert_eq!(alpha.blocks[2].fields[0].count, 2);
|
|
assert_eq!(protocol.packets[1].frequency, PacketFrequency::High);
|
|
}
|
|
|
|
#[test]
|
|
fn packet_template_diagnostics_are_source_located_and_stable() {
|
|
let text = "{ Bad Nope xx Maybe Compressed\n { Data Multiple 0 { Value Mystery 3 } }\n}\n";
|
|
let diagnostics = parse_packet_template("data\\bad.msg", text).unwrap_err();
|
|
let rendered = diagnostics
|
|
.iter()
|
|
.map(ToString::to_string)
|
|
.collect::<Vec<_>>();
|
|
assert_eq!(
|
|
rendered,
|
|
[
|
|
"data/bad.msg:1:7: error[PG030]: unknown packet frequency \"Nope\"",
|
|
"data/bad.msg:1:12: error[PG031]: packet ID must be a decimal or hexadecimal unsigned integer",
|
|
"data/bad.msg:1:15: error[PG032]: unknown packet trust marker \"Maybe\"",
|
|
"data/bad.msg:1:21: error[PG033]: unknown packet coding marker \"Compressed\"",
|
|
"data/bad.msg:2:18: error[PG014]: Multiple block count must be a positive decimal integer",
|
|
"data/bad.msg:2:28: error[PG023]: unknown field type \"Mystery\"",
|
|
]
|
|
);
|
|
}
|
|
}
|