//! Deterministic framework shared by the `LibreMetaverse` data generators. #![allow(clippy::missing_errors_doc)] // The CLI renders the complete error at its boundary. #![allow(clippy::must_use_candidate)] // Generator helpers are also exercised for validation. use serde::Deserialize; use sha2::{Digest, Sha256}; use std::collections::{BTreeMap, BTreeSet}; use std::fmt; use std::fmt::Write as _; use std::fs; use std::path::{Component, Path, PathBuf}; mod avatar_catalog; mod xml_catalog; pub const INVENTORY_PATH: &str = "codegen/sources.json"; pub const MANIFEST_OUTPUT: &str = "codegen/generated/source_manifest.rs"; pub const PACKET_OUTPUT: &str = "crates/libremetaverse/src/packet_catalog.rs"; pub const VISUAL_OUTPUT: &str = "crates/libremetaverse/src/visual_catalog.rs"; pub const FOLIAGE_OUTPUT: &str = "crates/libremetaverse/src/foliage_catalog.rs"; pub const SKELETON_OUTPUT: &str = "crates/libremetaverse/src/skeleton_catalog.rs"; pub const ATTENTION_OUTPUT: &str = "crates/libremetaverse/src/attention_catalog.rs"; pub const GENEPOOL_OUTPUT: &str = "crates/libremetaverse/src/genepool_catalog.rs"; pub const PUBLIC_API_PATH: &str = "api/public-api.json"; pub const GENERATED_OUTPUT_PATHS: [&str; 7] = [ MANIFEST_OUTPUT, PACKET_OUTPUT, VISUAL_OUTPUT, FOLIAGE_OUTPUT, SKELETON_OUTPUT, ATTENTION_OUTPUT, GENEPOOL_OUTPUT, ]; const RUST_KEYWORDS: &[&str] = &[ "as", "break", "const", "continue", "crate", "else", "enum", "extern", "false", "fn", "for", "if", "impl", "in", "let", "loop", "match", "mod", "move", "mut", "pub", "ref", "return", "self", "Self", "static", "struct", "super", "trait", "true", "type", "unsafe", "use", "where", "while", "async", "await", "dyn", "abstract", "become", "box", "do", "final", "macro", "override", "priv", "typeof", "unsized", "virtual", "yield", "try", "from", ]; #[derive(Clone, Debug, Deserialize, Eq, PartialEq)] pub struct Inventory { pub schema: u32, pub upstream_commit: String, pub upstream_repository: String, pub generators: Vec, pub inputs: Vec, } #[derive(Clone, Debug, Deserialize, Eq, PartialEq)] pub struct GeneratorSpec { pub id: String, pub reference_source: String, pub sha256: String, pub license: String, pub inputs: Vec, #[serde(default)] pub note: String, } #[derive(Clone, Debug, Deserialize, Eq, PartialEq)] pub struct InputSpec { pub id: String, pub reference_path: String, pub vendored_path: String, pub sha256: String, pub license: String, pub format: String, } #[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] pub enum Severity { Error, Warning, } #[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] pub struct Diagnostic { pub code: &'static str, pub severity: Severity, pub path: String, pub line: usize, pub column: usize, pub message: String, } #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum PacketFrequency { Low, Medium, High, } #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum BlockRepetition { Single, Multiple(usize), Variable, } #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum FieldKind { U8, U16, U32, U64, S8, S16, S32, F32, F64, LlUuid, Bool, LlVector3, LlVector3d, LlVector4, LlQuaternion, IpAddr, IpPort, Variable, Fixed, } #[derive(Clone, Debug, Eq, PartialEq)] pub struct PacketField { pub name: String, pub kind: FieldKind, pub count: usize, pub line: usize, pub column: usize, } #[derive(Clone, Debug, Eq, PartialEq)] pub struct PacketBlock { pub name: String, pub repetition: BlockRepetition, pub fields: Vec, pub line: usize, pub column: usize, } #[derive(Clone, Debug, Eq, PartialEq)] pub struct PacketDefinition { pub id: u16, pub name: String, pub frequency: PacketFrequency, pub trusted: bool, pub zerocoded: bool, pub flags: Vec, pub blocks: Vec, pub line: usize, pub column: usize, } #[derive(Clone, Debug, Eq, PartialEq)] pub struct PacketProtocol { pub packets: Vec, } #[derive(Clone, Debug, Eq, PartialEq)] struct Token { text: String, line: usize, column: usize, } impl fmt::Display for Diagnostic { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { let severity = match self.severity { Severity::Error => "error", Severity::Warning => "warning", }; write!( formatter, "{}:{}:{}: {severity}[{}]: {}", self.path, self.line, self.column, self.code, self.message ) } } pub fn workspace_root() -> PathBuf { Path::new(env!("CARGO_MANIFEST_DIR")).join("../..") } pub fn normalize_text(path: &str, bytes: &[u8]) -> Result { let bytes = bytes.strip_prefix(&[0xef, 0xbb, 0xbf]).unwrap_or(bytes); let text = std::str::from_utf8(bytes).map_err(|error| { let valid = &bytes[..error.valid_up_to()]; let (line, column) = line_column(valid); Diagnostic { code: "CG001", severity: Severity::Error, path: path.replace('\\', "/"), line, column, message: "input is not valid UTF-8".to_owned(), } })?; if let Some(position) = text.as_bytes().iter().position(|byte| *byte == 0) { let prefix = &text.as_bytes()[..position]; let (line, column) = line_column(prefix); return Err(Diagnostic { code: "CG002", severity: Severity::Error, path: path.replace('\\', "/"), line, column, message: "input contains a NUL byte".to_owned(), }); } Ok(text.replace("\r\n", "\n").replace('\r', "\n")) } fn line_column(bytes: &[u8]) -> (usize, usize) { let mut line = 1; let mut column = 1; for byte in bytes { if *byte == b'\n' { line += 1; column = 1; } else { column += 1; } } (line, column) } fn packet_diagnostic( path: &str, token: Option<&Token>, code: &'static str, message: impl Into, ) -> Diagnostic { Diagnostic { code, severity: Severity::Error, path: path.replace('\\', "/"), line: token.map_or(1, |token| token.line), column: token.map_or(1, |token| token.column), message: message.into(), } } fn tokenize_packet_template(path: &str, text: &str) -> Result, Vec> { let mut tokens = Vec::new(); let mut characters = text.char_indices().peekable(); let mut line = 1usize; let mut column = 1usize; while let Some((start, character)) = characters.next() { if character == '\n' { line += 1; column = 1; continue; } if character.is_whitespace() { column += 1; continue; } if character == '/' && characters.peek().is_some_and(|(_, next)| *next == '/') { characters.next(); column += 2; for (_, next) in characters.by_ref() { if next == '\n' { line += 1; column = 1; break; } column += 1; } continue; } if matches!(character, '{' | '}') { tokens.push(Token { text: character.to_string(), line, column, }); column += 1; continue; } let token_line = line; let token_column = column; let mut end = start + character.len_utf8(); column += 1; while let Some((offset, next)) = characters.peek().copied() { if next.is_whitespace() || matches!(next, '{' | '}') { break; } if next == '/' { let mut lookahead = characters.clone(); lookahead.next(); if lookahead .peek() .is_some_and(|(_, following)| *following == '/') { break; } } characters.next(); end = offset + next.len_utf8(); column += 1; } let value = &text[start..end]; if !value.is_ascii() { return Err(vec![packet_diagnostic( path, Some(&Token { text: value.to_owned(), line: token_line, column: token_column, }), "PG002", "protocol identifiers and values must be ASCII", )]); } tokens.push(Token { text: value.to_owned(), line: token_line, column: token_column, }); } Ok(tokens) } struct PacketParser<'a> { path: &'a str, tokens: Vec, position: usize, diagnostics: Vec, } impl PacketParser<'_> { fn current(&self) -> Option<&Token> { self.tokens.get(self.position) } fn advance(&mut self) -> Option { let token = self.tokens.get(self.position).cloned(); self.position += usize::from(token.is_some()); token } fn expect(&mut self, value: &str, code: &'static str, context: &str) -> bool { if self.current().is_some_and(|token| token.text == value) { self.position += 1; true } else { self.diagnostics.push(packet_diagnostic( self.path, self.current(), code, format!("expected {value:?} {context}"), )); false } } fn atom(&mut self, code: &'static str, context: &str) -> Option { match self.current() { Some(token) if !matches!(token.text.as_str(), "{" | "}") => self.advance(), _ => { self.diagnostics.push(packet_diagnostic( self.path, self.current(), code, format!("expected {context}"), )); None } } } fn recover_to_close(&mut self) { let mut depth = 0usize; while let Some(token) = self.advance() { match token.text.as_str() { "{" => depth += 1, "}" if depth == 0 => break, "}" => depth -= 1, _ => {} } } } fn parse_field(&mut self) -> Option { if !self.expect("{", "PG020", "to begin a field") { return None; } let name = self.atom("PG021", "a field name")?; let kind_token = self.atom("PG022", "a field type")?; let (kind, known_kind) = match kind_token.text.as_str() { "U8" => (FieldKind::U8, true), "U16" => (FieldKind::U16, true), "U32" => (FieldKind::U32, true), "U64" => (FieldKind::U64, true), "S8" => (FieldKind::S8, true), "S16" => (FieldKind::S16, true), "S32" => (FieldKind::S32, true), "F32" => (FieldKind::F32, true), "F64" => (FieldKind::F64, true), "LLUUID" => (FieldKind::LlUuid, true), "BOOL" => (FieldKind::Bool, true), "LLVector3" => (FieldKind::LlVector3, true), "LLVector3d" => (FieldKind::LlVector3d, true), "LLVector4" => (FieldKind::LlVector4, true), "LLQuaternion" => (FieldKind::LlQuaternion, true), "IPADDR" => (FieldKind::IpAddr, true), "IPPORT" => (FieldKind::IpPort, true), "Variable" => (FieldKind::Variable, true), "Fixed" => (FieldKind::Fixed, true), other => { self.diagnostics.push(packet_diagnostic( self.path, Some(&kind_token), "PG023", format!("unknown field type {other:?}"), )); (FieldKind::Variable, false) } }; let count = if self.current().is_some_and(|token| token.text != "}") { let count_token = self.atom("PG024", "a field width")?; match count_token.text.parse::() { Ok(value) if value > 0 => value, _ => { self.diagnostics.push(packet_diagnostic( self.path, Some(&count_token), "PG025", "field width must be a positive decimal integer", )); 1 } } } else { 1 }; if known_kind && matches!(kind, FieldKind::Variable) && !matches!(count, 1 | 2) { self.diagnostics.push(packet_diagnostic( self.path, Some(&name), "PG026", "Variable fields require a one- or two-byte length prefix", )); } if known_kind && !matches!(kind, FieldKind::Variable | FieldKind::Fixed) && count != 1 { self.diagnostics.push(packet_diagnostic( self.path, Some(&name), "PG027", "only Variable and Fixed fields may specify a width", )); } if !self.expect("}", "PG028", "after a field") { self.recover_to_close(); } Some(PacketField { name: name.text, kind, count, line: name.line, column: name.column, }) } fn parse_block(&mut self) -> Option { if !self.expect("{", "PG010", "to begin a block") { return None; } let name = self.atom("PG011", "a block name")?; let repetition_token = self.atom("PG012", "a block repetition")?; let repetition = match repetition_token.text.as_str() { "Single" => BlockRepetition::Single, "Variable" => BlockRepetition::Variable, "Multiple" => { let count = self.atom("PG013", "a Multiple block count")?; match count.text.parse::() { Ok(value) if value > 0 => BlockRepetition::Multiple(value), _ => { self.diagnostics.push(packet_diagnostic( self.path, Some(&count), "PG014", "Multiple block count must be a positive decimal integer", )); BlockRepetition::Multiple(1) } } } other => { self.diagnostics.push(packet_diagnostic( self.path, Some(&repetition_token), "PG015", format!("unknown block repetition {other:?}"), )); BlockRepetition::Single } }; let mut fields = Vec::new(); while self.current().is_some_and(|token| token.text == "{") { if let Some(field) = self.parse_field() { fields.push(field); } } if !self.expect("}", "PG016", "after a block") { self.recover_to_close(); } let mut names = BTreeSet::new(); for field in &fields { if !names.insert(&field.name) { self.diagnostics.push(packet_diagnostic( self.path, Some(&Token { text: field.name.clone(), line: field.line, column: field.column, }), "PG017", format!("duplicate field {:?} in block {:?}", field.name, name.text), )); } } Some(PacketBlock { name: name.text, repetition, fields, line: name.line, column: name.column, }) } #[allow(clippy::too_many_lines)] fn parse_packet(&mut self) -> Option { if !self.expect("{", "PG003", "to begin a packet") { return None; } let name = self.atom("PG004", "a packet name")?; let frequency_token = self.atom("PG005", "a packet frequency")?; let id_token = self.atom("PG006", "a packet ID")?; let trust_token = self.atom("PG007", "a packet trust marker")?; let coding_token = self.atom("PG008", "a packet coding marker")?; let frequency = match frequency_token.text.as_str() { "Fixed" | "Low" => PacketFrequency::Low, "Medium" | "Mid" => PacketFrequency::Medium, "High" => PacketFrequency::High, other => { self.diagnostics.push(packet_diagnostic( self.path, Some(&frequency_token), "PG030", format!("unknown packet frequency {other:?}"), )); PacketFrequency::Low } }; let parsed_id = if let Some(hex) = id_token .text .strip_prefix("0x") .or_else(|| id_token.text.strip_prefix("0X")) { u32::from_str_radix(hex, 16) } else { id_token.text.parse::() }; let id = if let Ok(value) = parsed_id { (value & 0xffff) as u16 } else { self.diagnostics.push(packet_diagnostic( self.path, Some(&id_token), "PG031", "packet ID must be a decimal or hexadecimal unsigned integer", )); 0 }; let trusted = match trust_token.text.as_str() { "Trusted" => true, "NotTrusted" => false, other => { self.diagnostics.push(packet_diagnostic( self.path, Some(&trust_token), "PG032", format!("unknown packet trust marker {other:?}"), )); false } }; let zerocoded = match coding_token.text.as_str() { "Zerocoded" => true, "Unencoded" => false, other => { self.diagnostics.push(packet_diagnostic( self.path, Some(&coding_token), "PG033", format!("unknown packet coding marker {other:?}"), )); false } }; let mut flags = Vec::new(); while self .current() .is_some_and(|token| !matches!(token.text.as_str(), "{" | "}")) { if let Some(flag) = self.advance() { flags.push(flag.text); } } let mut blocks = Vec::new(); while self.current().is_some_and(|token| token.text == "{") { if let Some(block) = self.parse_block() { blocks.push(block); } } if !self.expect("}", "PG009", "after a packet") { self.recover_to_close(); } let mut block_names = BTreeSet::new(); for block in &blocks { if !block_names.insert(&block.name) { self.diagnostics.push(packet_diagnostic( self.path, Some(&Token { text: block.name.clone(), line: block.line, column: block.column, }), "PG034", format!("duplicate block {:?} in packet {:?}", block.name, name.text), )); } } Some(PacketDefinition { id, name: name.text, frequency, trusted, zerocoded, flags, blocks, line: name.line, column: name.column, }) } } pub fn parse_packet_template(path: &str, text: &str) -> Result> { let tokens = tokenize_packet_template(path, text)?; let mut parser = PacketParser { path, tokens, position: 0, diagnostics: Vec::new(), }; while parser.current().is_some_and(|token| token.text != "{") { parser.position += 1; } let mut packets = Vec::new(); while parser.current().is_some() { if parser.current().is_some_and(|token| token.text != "{") { parser.diagnostics.push(packet_diagnostic( path, parser.current(), "PG035", "unexpected token outside a packet", )); parser.position += 1; continue; } if let Some(packet) = parser.parse_packet() { packets.push(packet); } else { parser.recover_to_close(); } } let mut names = BTreeMap::new(); let mut ids = BTreeMap::new(); for packet in &packets { if let Some((line, column)) = names.insert(packet.name.clone(), (packet.line, packet.column)) { parser.diagnostics.push(packet_diagnostic( path, Some(&Token { text: packet.name.clone(), line: packet.line, column: packet.column, }), "PG036", format!( "duplicate packet {:?}; first declared at {line}:{column}", packet.name ), )); } let key = (packet.frequency as u8, packet.id); if let Some((other, line, column)) = ids.insert(key, (&packet.name, packet.line, packet.column)) { parser.diagnostics.push(packet_diagnostic( path, Some(&Token { text: packet.name.clone(), line: packet.line, column: packet.column, }), "PG037", format!( "duplicate {:?} packet ID {}; {:?} was declared at {line}:{column}", packet.frequency, packet.id, other ), )); } } parser.diagnostics.sort_by(|left, right| { (&left.path, left.line, left.column, left.code, &left.message).cmp(&( &right.path, right.line, right.column, right.code, &right.message, )) }); if parser.diagnostics.is_empty() { Ok(PacketProtocol { packets }) } else { Err(parser.diagnostics) } } pub fn generated_rust(generator: &str, sources: &[&InputSpec], body: &str) -> Vec { let mut output = String::new(); output.push_str("// @generated by libremetaverse-codegen; DO NOT EDIT.\n"); output.push_str("// Regenerate: cargo run -p libremetaverse-codegen -- generate\n"); let _ = writeln!(output, "// Generator: {generator}"); for source in sources { let _ = writeln!( output, "// Source: {} sha256={} license={}", source.vendored_path, source.sha256, source.license ); } output.push('\n'); output.push_str(body.trim_end()); output.push('\n'); output.into_bytes() } fn rust_snake(name: &str) -> String { let characters: Vec = name.trim_start_matches('@').chars().collect(); let mut output = String::new(); for (index, character) in characters.iter().copied().enumerate() { let previous = index .checked_sub(1) .and_then(|i| characters.get(i)) .copied(); let next = characters.get(index + 1).copied(); if character.is_ascii_uppercase() && !output.is_empty() && (previous.is_some_and(|value| value.is_ascii_lowercase() || value.is_ascii_digit()) || (previous.is_some_and(|value| value.is_ascii_uppercase()) && next.is_some_and(|value| value.is_ascii_lowercase()))) { output.push('_'); } if character.is_ascii_alphanumeric() || character == '_' { output.push(character.to_ascii_lowercase()); } else { output.push('_'); } } while output.contains("__") { output = output.replace("__", "_"); } let mut output = output.trim_matches('_').to_owned(); if output.is_empty() { output.push_str("item"); } if output.starts_with(|character: char| character.is_ascii_digit()) || RUST_KEYWORDS.contains(&output.as_str()) { output.push('_'); } output } const fn field_fixed_length(field: &PacketField) -> usize { match field.kind { FieldKind::Bool | FieldKind::U8 | FieldKind::S8 => 1, FieldKind::U16 | FieldKind::S16 | FieldKind::IpPort => 2, FieldKind::U32 | FieldKind::S32 | FieldKind::F32 | FieldKind::IpAddr => 4, FieldKind::U64 | FieldKind::F64 => 8, FieldKind::LlVector3 | FieldKind::LlQuaternion => 12, FieldKind::LlUuid | FieldKind::LlVector4 => 16, FieldKind::LlVector3d => 24, FieldKind::Fixed => field.count, FieldKind::Variable => 0, } } fn field_kind_name(kind: FieldKind) -> &'static str { match kind { FieldKind::U8 => "U8", FieldKind::U16 => "U16", FieldKind::U32 => "U32", FieldKind::U64 => "U64", FieldKind::S8 => "S8", FieldKind::S16 => "S16", FieldKind::S32 => "S32", FieldKind::F32 => "F32", FieldKind::F64 => "F64", FieldKind::LlUuid => "LlUuid", FieldKind::Bool => "Bool", FieldKind::LlVector3 => "LlVector3", FieldKind::LlVector3d => "LlVector3d", FieldKind::LlVector4 => "LlVector4", FieldKind::LlQuaternion => "LlQuaternion", FieldKind::IpAddr => "IpAddr", FieldKind::IpPort => "IpPort", FieldKind::Variable => "Variable", FieldKind::Fixed => "Fixed", } } fn frequency_name(frequency: PacketFrequency) -> &'static str { match frequency { PacketFrequency::Low => "Low", PacketFrequency::Medium => "Medium", PacketFrequency::High => "High", } } fn field_default(kind: FieldKind) -> &'static str { match kind { FieldKind::Bool => "false", FieldKind::U8 | FieldKind::U16 | FieldKind::U32 | FieldKind::U64 | FieldKind::S8 | FieldKind::S16 | FieldKind::S32 | FieldKind::IpAddr | FieldKind::IpPort => "0", FieldKind::F32 => "0.0_f32", FieldKind::F64 => "0.0_f64", FieldKind::LlUuid => "libremetaverse_types::UUID::zero()", FieldKind::LlVector3 => "libremetaverse_types::Vector3::zero()", FieldKind::LlVector3d => "libremetaverse_types::Vector3d::zero()", FieldKind::LlVector4 => "libremetaverse_types::Vector4::zero()", FieldKind::LlQuaternion => { "libremetaverse_types::Quaternion { x: 0.0, y: 0.0, z: 0.0, w: 0.0 }" } FieldKind::Variable | FieldKind::Fixed => "Vec::new()", } } fn field_sample(field: &PacketField) -> String { match field.kind { FieldKind::Bool => "true".to_owned(), FieldKind::U8 => "0xa5_u8".to_owned(), FieldKind::U16 | FieldKind::IpPort => "0x1234_u16".to_owned(), FieldKind::U32 | FieldKind::IpAddr => "0x1122_3344_u32".to_owned(), FieldKind::U64 => "0x0102_0304_0506_0708_u64".to_owned(), FieldKind::S8 => "-37_i8".to_owned(), FieldKind::S16 => "-0x1234_i16".to_owned(), FieldKind::S32 => "-0x1122_3344_i32".to_owned(), FieldKind::F32 => "1.5_f32".to_owned(), FieldKind::F64 => "-2.25_f64".to_owned(), FieldKind::LlUuid => "libremetaverse_types::UUID::new_with_bytes_int32((0_u8..16).collect(), 0).expect(\"sample UUID\")".to_owned(), FieldKind::LlVector3 => "libremetaverse_types::Vector3 { x: 1.25, y: -2.5, z: 3.75 }".to_owned(), FieldKind::LlVector3d => "libremetaverse_types::Vector3d { x: 1.25, y: -2.5, z: 3.75 }".to_owned(), FieldKind::LlVector4 => "libremetaverse_types::Vector4 { x: 1.25, y: -2.5, z: 3.75, w: -4.5 }".to_owned(), FieldKind::LlQuaternion => "libremetaverse_types::Quaternion::identity()".to_owned(), FieldKind::Variable => "vec![0_u8, 1, 2, 0, 0xff]".to_owned(), FieldKind::Fixed => format!("vec![0xa5_u8; {}]", field.count), } } fn field_decode_statement(field: &PacketField) -> String { let name = rust_snake(&field.name); let expression = match field.kind { FieldKind::Bool => "reader.read_u8()? != 0".to_owned(), FieldKind::U8 => "reader.read_u8()?".to_owned(), FieldKind::U16 => "reader.read_u16_le()?".to_owned(), FieldKind::U32 | FieldKind::IpAddr => "reader.read_u32_le()?".to_owned(), FieldKind::U64 => "reader.read_u64_le()?".to_owned(), FieldKind::S8 => "reader.read_i8()?".to_owned(), FieldKind::S16 => "reader.read_i16_le()?".to_owned(), FieldKind::S32 => "reader.read_i32_le()?".to_owned(), FieldKind::F32 => "reader.read_f32_le()?".to_owned(), FieldKind::F64 => "reader.read_f64_le()?".to_owned(), FieldKind::LlUuid => "reader.read_uuid()?".to_owned(), FieldKind::LlVector3 => "reader.read_vector3()?".to_owned(), FieldKind::LlVector3d => "reader.read_vector3d()?".to_owned(), FieldKind::LlVector4 => "reader.read_vector4()?".to_owned(), FieldKind::LlQuaternion => "reader.read_quaternion()?".to_owned(), FieldKind::IpPort => "reader.read_u16_be()?".to_owned(), FieldKind::Variable => format!("reader.read_variable({})?", field.count), FieldKind::Fixed => format!( "reader.read_bytes({}, \"truncated fixed packet field\")?", field.count ), }; format!("self.{name} = {expression};") } fn field_encode_statement(field: &PacketField) -> String { let name = rust_snake(&field.name); match field.kind { FieldKind::Bool => format!("writer.write_u8(u8::from(self.{name}))?;"), FieldKind::U8 => format!("writer.write_u8(self.{name})?;"), FieldKind::U16 => format!("writer.write_u16_le(self.{name})?;"), FieldKind::U32 | FieldKind::IpAddr => { format!("writer.write_u32_le(self.{name})?;") } FieldKind::U64 => format!("writer.write_u64_le(self.{name})?;"), FieldKind::S8 => format!("writer.write_i8(self.{name})?;"), FieldKind::S16 => format!("writer.write_i16_le(self.{name})?;"), FieldKind::S32 => format!("writer.write_i32_le(self.{name})?;"), FieldKind::F32 => format!("writer.write_f32_le(self.{name})?;"), FieldKind::F64 => format!("writer.write_f64_le(self.{name})?;"), FieldKind::LlUuid => format!("writer.write_uuid(&self.{name})?;"), FieldKind::LlVector3 => format!("writer.write_vector3(&self.{name})?;"), FieldKind::LlVector3d => format!("writer.write_vector3d(&self.{name})?;"), FieldKind::LlVector4 => format!("writer.write_vector4(&self.{name})?;"), FieldKind::LlQuaternion => format!("writer.write_quaternion(&self.{name})?;"), FieldKind::IpPort => format!("writer.write_u16_be(self.{name})?;"), FieldKind::Variable => { format!("writer.write_variable(&self.{name}, {})?;", field.count) } FieldKind::Fixed => format!("writer.write_fixed(&self.{name}, {})?;", field.count), } } fn field_to_osd_expression(field: &PacketField) -> String { let name = rust_snake(&field.name); match field.kind { FieldKind::Bool => { format!("libremetaverse_structured_data::OSD::Boolean(self.{name})") } FieldKind::U8 | FieldKind::U16 | FieldKind::IpPort | FieldKind::S8 | FieldKind::S16 => { format!("libremetaverse_structured_data::OSD::Integer(i32::from(self.{name}))") } FieldKind::S32 => { format!("libremetaverse_structured_data::OSD::Integer(self.{name})") } FieldKind::U32 | FieldKind::IpAddr | FieldKind::U64 => format!( "libremetaverse_structured_data::OSD::Binary(self.{name}.to_le_bytes().to_vec())" ), FieldKind::F32 => { format!("libremetaverse_structured_data::OSD::Real(f64::from(self.{name}))") } FieldKind::F64 => { format!("libremetaverse_structured_data::OSD::Real(self.{name})") } FieldKind::LlUuid => { format!("libremetaverse_structured_data::OSD::UUID(self.{name})") } FieldKind::LlVector3 => { format!("libremetaverse_structured_data::OSD::from_vector3(self.{name})?") } FieldKind::LlVector3d => { format!("libremetaverse_structured_data::OSD::from_vector3d(self.{name})?") } FieldKind::LlVector4 => { format!("libremetaverse_structured_data::OSD::from_vector4(self.{name})?") } FieldKind::LlQuaternion => { format!("libremetaverse_structured_data::OSD::from_quaternion(self.{name})?") } FieldKind::Variable | FieldKind::Fixed => { format!("libremetaverse_structured_data::OSD::Binary(self.{name}.clone())") } } } fn field_from_osd_statement(field: &PacketField) -> String { let name = rust_snake(&field.name); let conversion = match field.kind { FieldKind::Bool => "value.as_boolean()?".to_owned(), FieldKind::U8 => "value.as_integer()? as u8".to_owned(), FieldKind::U16 | FieldKind::IpPort => "value.as_integer()? as u16".to_owned(), FieldKind::U32 | FieldKind::IpAddr => "osd_u32(&value)?".to_owned(), FieldKind::U64 => "osd_u64(&value)?".to_owned(), FieldKind::S8 => "value.as_integer()? as i8".to_owned(), FieldKind::S16 => "value.as_integer()? as i16".to_owned(), FieldKind::S32 => "value.as_integer()?".to_owned(), FieldKind::F32 => "value.as_real()? as f32".to_owned(), FieldKind::F64 => "value.as_real()?".to_owned(), FieldKind::LlUuid => "value.as_uuid()?".to_owned(), FieldKind::LlVector3 => "value.as_vector3()?".to_owned(), FieldKind::LlVector3d => "value.as_vector3d()?".to_owned(), FieldKind::LlVector4 => "value.as_vector4()?".to_owned(), FieldKind::LlQuaternion => "value.as_quaternion()?".to_owned(), FieldKind::Variable | FieldKind::Fixed => "value.as_binary()?".to_owned(), }; format!( "if let Some(value) = map.get({:?}) {{ self.{name} = {conversion}; }}", field.name ) } fn ordered_packets(protocol: &PacketProtocol) -> Vec<&PacketDefinition> { [ PacketFrequency::Low, PacketFrequency::Medium, PacketFrequency::High, ] .into_iter() .flat_map(|frequency| { protocol .packets .iter() .filter(move |packet| packet.frequency == frequency) }) .collect() } fn packet_type_value(packet: &PacketDefinition) -> u32 { let prefix = match packet.frequency { PacketFrequency::Low => 0x1_0000, PacketFrequency::Medium => 0x2_0000, PacketFrequency::High => 0x3_0000, }; prefix | u32::from(packet.id) } fn append_descriptor_types(body: &mut String) { body.push_str( "#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]\n\ pub enum FieldKind { U8, U16, U32, U64, S8, S16, S32, F32, F64, LlUuid, Bool, LlVector3, LlVector3d, LlVector4, LlQuaternion, IpAddr, IpPort, Variable, Fixed }\n\n\ #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]\n\ pub enum BlockRepetition { Single, Multiple(usize), Variable }\n\n\ #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]\n\ pub struct FieldDescriptor { pub name: &'static str, pub kind: FieldKind, pub count: usize, pub fixed_width: usize }\n\n\ #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]\n\ pub struct BlockDescriptor { pub name: &'static str, pub repetition: BlockRepetition, pub fields: &'static [FieldDescriptor] }\n\n\ #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]\n\ pub struct PacketDescriptor { pub name: &'static str, pub packet_type: PacketType, pub id: u16, pub frequency: crate::PacketFrequency, pub trusted: bool, pub zerocoded: bool, pub flags: &'static [&'static str], pub blocks: &'static [BlockDescriptor] }\n\n", ); } fn append_packet_type(body: &mut String, packets: &[&PacketDefinition]) { body.push_str("#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]\n#[repr(i32)]\npub enum PacketType {\n Default = 0,\n"); for packet in packets { let prefix = match packet.frequency { PacketFrequency::Low => 1, PacketFrequency::Medium => 2, PacketFrequency::High => 3, }; let _ = writeln!(body, " {} = 0x{prefix}_{:04x},", packet.name, packet.id); } body.push_str("}\n\n"); } fn append_descriptors(body: &mut String, packets: &[&PacketDefinition]) { for (packet_index, packet) in packets.iter().enumerate() { for (block_index, block) in packet.blocks.iter().enumerate() { let _ = writeln!( body, "static PACKET_{packet_index}_BLOCK_{block_index}_FIELDS: &[FieldDescriptor] = &[" ); for field in &block.fields { let _ = writeln!( body, " FieldDescriptor {{ name: {:?}, kind: FieldKind::{}, count: {}, fixed_width: {} }},", field.name, field_kind_name(field.kind), field.count, field_fixed_length(field) ); } body.push_str("];\n"); } let _ = writeln!( body, "static PACKET_{packet_index}_BLOCKS: &[BlockDescriptor] = &[" ); for (block_index, block) in packet.blocks.iter().enumerate() { let repetition = match block.repetition { BlockRepetition::Single => "BlockRepetition::Single".to_owned(), BlockRepetition::Variable => "BlockRepetition::Variable".to_owned(), BlockRepetition::Multiple(count) => { format!("BlockRepetition::Multiple({count})") } }; let _ = writeln!( body, " BlockDescriptor {{ name: {:?}, repetition: {repetition}, fields: PACKET_{packet_index}_BLOCK_{block_index}_FIELDS }},", block.name ); } body.push_str("];\n\n"); } body.push_str("pub static PACKETS: &[PacketDescriptor] = &[\n"); for (packet_index, packet) in packets.iter().enumerate() { let flags = packet .flags .iter() .map(|flag| format!("{flag:?}")) .collect::>() .join(", "); let _ = writeln!( body, " PacketDescriptor {{ name: {:?}, packet_type: PacketType::{}, id: {}, frequency: crate::PacketFrequency::{}, trusted: {}, zerocoded: {}, flags: &[{}], blocks: PACKET_{packet_index}_BLOCKS }},", packet.name, packet.name, packet.id, frequency_name(packet.frequency), packet.trusted, packet.zerocoded, flags ); } body.push_str("];\n\n"); } #[allow(clippy::too_many_lines)] fn append_dispatch(body: &mut String, packets: &[&PacketDefinition]) { body.push_str("#[must_use]\n#[allow(clippy::too_many_lines)]\npub fn packet_type(id: u16, frequency: crate::PacketFrequency) -> PacketType {\n match (frequency, id) {\n"); for packet in packets { let _ = writeln!( body, " (crate::PacketFrequency::{}, {}) => PacketType::{},", frequency_name(packet.frequency), packet.id, packet.name ); } body.push_str(" _ => PacketType::Default,\n }\n}\n\n"); body.push_str("#[must_use]\n#[allow(clippy::too_many_lines)]\npub fn descriptor_by_type(packet_type: PacketType) -> Option<&'static PacketDescriptor> {\n match packet_type {\n"); for (index, packet) in packets.iter().enumerate() { let _ = writeln!( body, " PacketType::{} => Some(&PACKETS[{index}]),", packet.name ); } body.push_str(" PacketType::Default => None,\n }\n}\n\n"); body.push_str("#[must_use]\n#[allow(clippy::too_many_lines)]\npub fn descriptor_by_name(name: &str) -> Option<&'static PacketDescriptor> {\n match name {\n"); for (index, packet) in packets.iter().enumerate() { let _ = writeln!( body, " {:?} => Some(&PACKETS[{index}]),", packet.name ); } body.push_str(" _ => None,\n }\n}\n\n"); body.push_str( "pub(crate) fn build_packet(packet_type: PacketType) -> Result {\n\ let descriptor = descriptor_by_type(packet_type).ok_or(crate::Error::InvalidOperation)?;\n\ Ok(crate::packets::Packet {\n\ has_variable_blocks: descriptor.blocks.iter().any(|block| matches!(block.repetition, BlockRepetition::Variable)),\n\ header: new_header(descriptor.frequency, descriptor.id, descriptor.zerocoded),\n\ type_: packet_type,\n\ })\n\ }\n\n\ pub(crate) const fn new_header(frequency: crate::PacketFrequency, id: u16, zerocoded: bool) -> crate::packets::Header {\n\ crate::packets::Header { ack_list: None, appended_acks: false, frequency, id, reliable: true, resent: false, sequence: 0, zerocoded }\n\ }\n\n\ fn usize_to_i32(value: usize) -> i32 { i32::try_from(value).unwrap_or(i32::MAX) }\n\n\ fn osd_u32(value: &libremetaverse_structured_data::OSD) -> Result {\n\ let bytes = value.as_binary()?;\n\ Ok(bytes.get(..4).and_then(|value| value.try_into().ok()).map_or(0, u32::from_le_bytes))\n\ }\n\n\ fn osd_u64(value: &libremetaverse_structured_data::OSD) -> Result {\n\ let bytes = value.as_binary()?;\n\ Ok(bytes.get(..8).and_then(|value| value.try_into().ok()).map_or(0, u64::from_le_bytes))\n\ }\n\n\ pub(crate) fn empty_packet_osd() -> Result {\n\ libremetaverse_structured_data::OSDMap::new_with_constructor()\n\ }\n\n\ pub(crate) fn ignore_packet_osd(body: &libremetaverse_structured_data::OSDMap) -> Result<(), crate::Error> {\n\ let _ = body;\n\ Ok(())\n\ }\n\n\ pub(crate) trait GeneratedBlock: Sized {\n\ fn new_generated() -> Self;\n\ fn generated_length(&self) -> i32;\n\ fn decode_payload(&mut self, reader: &mut crate::packet_wire::WireReader<'_>) -> Result<(), crate::Error>;\n\ fn encode_payload(&self, writer: &mut crate::packet_wire::WireWriter) -> Result<(), crate::Error>;\n\ fn to_osd(&self) -> Result;\n\ fn from_osd(&mut self, map: &libremetaverse_structured_data::OSDMap) -> Result<(), crate::Error>;\n\ fn new_from_bytes(bytes: &[u8], position: &mut i32) -> Result {\n\ let mut value = Self::new_generated();\n\ Self::decode_from_bytes(&mut value, bytes, position)?;\n\ Ok(value)\n\ }\n\ fn decode_from_bytes(&mut self, bytes: &[u8], position: &mut i32) -> Result<(), crate::Error> {\n\ if *position < 0 { return Err(crate::Error::Argument); }\n\ let start = usize::try_from(*position).map_err(|_| crate::Error::Argument)?;\n\ let mut reader = crate::packet_wire::WireReader::new(bytes, start, bytes.len())?;\n\ self.decode_payload(&mut reader)?;\n\ reader.commit_position(position)\n\ }\n\ fn encode_to_slice(&self, bytes: &mut [u8], position: &mut i32) -> Result<(), crate::Error> {\n\ if *position < 0 { return Err(crate::Error::Argument); }\n\ let start = usize::try_from(*position).map_err(|_| crate::Error::Argument)?;\n\ let capacity = usize::try_from(self.generated_length()).map_err(|_| crate::Error::Argument)?;\n\ let mut writer = crate::packet_wire::WireWriter::with_capacity(capacity)?;\n\ self.encode_payload(&mut writer)?;\n\ let encoded = writer.into_inner();\n\ let end = start.checked_add(encoded.len()).ok_or(crate::Error::Argument)?;\n\ if end > bytes.len() { return Err(crate::Error::IndexOutOfRange); }\n\ bytes[start..end].copy_from_slice(&encoded);\n\ *position = i32::try_from(end).map_err(|_| crate::Error::Argument)?;\n\ Ok(())\n\ }\n\ }\n\n\ pub(crate) trait GeneratedPacket: Sized {\n\ const USES_BUFFER_POOLING: bool;\n\ fn new_generated() -> Self;\n\ fn generated_length(&self) -> i32;\n\ fn generated_header(&self) -> crate::packets::Header;\n\ fn store_header(&mut self, header: crate::packets::Header);\n\ fn decode_payload(&mut self, reader: &mut crate::packet_wire::WireReader<'_>) -> Result<(), crate::Error>;\n\ fn encode_payload(&self, writer: &mut crate::packet_wire::WireWriter) -> Result<(), crate::Error>;\n\ fn packet_to_osd(&self) -> Result;\n\ fn packet_from_osd(&mut self, body: &libremetaverse_structured_data::OSDMap) -> Result<(), crate::Error>;\n\ fn encode_multiple(&self) -> Result>, crate::Error>;\n\ fn encode_packet(&self) -> Result, crate::Error> {\n\ let header = self.generated_header();\n\ let payload = usize::try_from(self.generated_length()).map_err(|_| crate::Error::Argument)?;\n\ let capacity = payload.checked_add(crate::packet_wire::ack_length(&header)?).ok_or(crate::Error::Argument)?;\n\ let mut writer = crate::packet_wire::WireWriter::with_capacity(capacity)?;\n\ crate::packet_wire::encode_header(&header, &mut writer)?;\n\ self.encode_payload(&mut writer)?;\n\ crate::packet_wire::encode_acks(&header, &mut writer)?;\n\ Ok(writer.into_inner())\n\ }\n\ fn new_from_bytes(bytes: &[u8], position: &mut i32) -> Result {\n\ let mut value = Self::new_generated();\n\ let mut packet_end = i32::try_from(bytes.len()).map_err(|_| crate::Error::Argument)?.checked_sub(1).ok_or(crate::Error::Argument)?;\n\ Self::decode_from_bytes(&mut value, bytes, position, &mut packet_end, None)?;\n\ Ok(value)\n\ }\n\ fn new_from_header(header: crate::packets::Header, bytes: &[u8], position: &mut i32) -> Result {\n\ let mut value = Self::new_generated();\n\ let mut packet_end = i32::try_from(bytes.len()).map_err(|_| crate::Error::Argument)?.checked_sub(1).ok_or(crate::Error::Argument)?;\n\ Self::decode_from_header(&mut value, header, bytes, position, &mut packet_end)?;\n\ Ok(value)\n\ }\n\ fn decode_from_header(&mut self, header: crate::packets::Header, bytes: &[u8], position: &mut i32, packet_end: &mut i32) -> Result<(), crate::Error> {\n\ if *position < 0 || *packet_end < 0 { return Err(crate::Error::Argument); }\n\ let start = usize::try_from(*position).map_err(|_| crate::Error::Argument)?;\n\ let end = usize::try_from(packet_end.checked_add(1).ok_or(crate::Error::Argument)?).map_err(|_| crate::Error::Argument)?;\n\ let mut reader = crate::packet_wire::WireReader::new(bytes, start, end)?;\n\ self.decode_payload(&mut reader)?;\n\ reader.finish(position)?;\n\ self.store_header(header);\n\ Ok(())\n\ }\n\ fn decode_from_bytes(&mut self, bytes: &[u8], position: &mut i32, packet_end: &mut i32, zero_buffer: Option<&mut [u8]>) -> Result<(), crate::Error> {\n\ let header = crate::packet_wire::decode_header(bytes, position, packet_end)?;\n\ if header.zerocoded && let Some(buffer) = zero_buffer {\n\ let decoded_length = crate::packet_wire::zero_decode(Some(bytes), packet_end.checked_add(1).ok_or(crate::Error::Argument)?, Some(buffer))?;\n\ *packet_end = decoded_length.checked_sub(1).ok_or(crate::Error::Argument)?;\n\ let end = usize::try_from(decoded_length).map_err(|_| crate::Error::Argument)?;\n\ return self.decode_from_header(header, &buffer[..end], position, packet_end);\n\ }\n\ self.decode_from_header(header, bytes, position, packet_end)\n\ }\n\ }\n\n", ); body.push_str( "#[allow(clippy::too_many_lines)]\npub(crate) fn validate_packet_payload(\n packet_type: PacketType,\n header: crate::packets::Header,\n bytes: &[u8],\n position: &mut i32,\n packet_end: &mut i32,\n) -> Result<(), crate::Error> {\n match packet_type {\n", ); for packet in packets { let _ = writeln!( body, " PacketType::{} => {{ let mut packet = ::new_generated(); GeneratedPacket::decode_from_header(&mut packet, header, bytes, position, packet_end) }},", packet.name, packet.name ); } body.push_str( " PacketType::Default => Err(crate::Error::InvalidOperation),\n }\n}\n\n", ); body.push_str( "#[allow(clippy::too_many_lines)]\npub(crate) fn build_packet_from_osd(\n name: &str,\n body: &libremetaverse_structured_data::OSDMap,\n) -> Result, crate::Error> {\n let Some(descriptor) = descriptor_by_name(name) else { return Ok(None); };\n match descriptor.packet_type {\n", ); for packet in packets { let _ = writeln!( body, " PacketType::{} => {{ let mut packet = ::new_generated(); GeneratedPacket::packet_from_osd(&mut packet, body)?; }},", packet.name, packet.name ); } body.push_str( " PacketType::Default => return Ok(None),\n }\n build_packet(descriptor.packet_type).map(Some)\n}\n\n", ); } fn append_block_impls(body: &mut String, packets: &[&PacketDefinition]) { for packet in packets { for block in &packet.blocks { let type_name = format!("{}Packet{}Block", packet.name, block.name); let _ = writeln!( body, "impl GeneratedBlock for crate::packets::{type_name} {{\n fn new_generated() -> Self {{\n Self {{" ); for field in &block.fields { let _ = writeln!( body, " {}: {},", rust_snake(&field.name), field_default(field.kind) ); } body.push_str(" }\n }\n\n fn generated_length(&self) -> i32 {\n"); let fixed = block.fields.iter().map(field_fixed_length).sum::(); let variable_fields = block .fields .iter() .filter(|field| field.kind == FieldKind::Variable) .collect::>(); if variable_fields.is_empty() { let _ = writeln!(body, " {fixed}_i32"); } else { let _ = writeln!(body, " let mut length = {fixed}_i32;"); for field in variable_fields { let _ = writeln!( body, " length = length.saturating_add({}).saturating_add(usize_to_i32(self.{}.len()));", field.count, rust_snake(&field.name) ); } body.push_str(" length\n"); } body.push_str( " }\n\n fn decode_payload(&mut self, reader: &mut crate::packet_wire::WireReader<'_>) -> Result<(), crate::Error> {\n", ); for field in &block.fields { let _ = writeln!(body, " {}", field_decode_statement(field)); } body.push_str(" Ok(())\n }\n\n fn encode_payload(&self, writer: &mut crate::packet_wire::WireWriter) -> Result<(), crate::Error> {\n"); for field in &block.fields { let _ = writeln!(body, " {}", field_encode_statement(field)); } body.push_str(" Ok(())\n }\n\n fn to_osd(&self) -> Result {\n let map = libremetaverse_structured_data::OSDMap::new_with_constructor()?;\n"); for field in &block.fields { let _ = writeln!( body, " map.add_with_string_osd({:?}.to_owned(), {})?;", field.name, field_to_osd_expression(field) ); } body.push_str(" Ok(map)\n }\n\n fn from_osd(&mut self, map: &libremetaverse_structured_data::OSDMap) -> Result<(), crate::Error> {\n"); for field in &block.fields { let _ = writeln!(body, " {}", field_from_osd_statement(field)); } if block.fields.is_empty() { body.push_str(" let _ = map;\n"); } body.push_str(" Ok(())\n }\n}\n\n"); } } } fn packet_has_composed_base(name: &str) -> bool { matches!(name, "DirPlacesReply" | "TestMessage") } fn packet_can_split(packet: &PacketDefinition) -> bool { let mut saw_variable = false; let mut has_variable = false; for block in &packet.blocks { if block.repetition == BlockRepetition::Variable { saw_variable = true; has_variable = true; } else if saw_variable { return false; } } has_variable } #[allow(clippy::too_many_lines)] fn append_packet_encode_multiple(body: &mut String, packet: &PacketDefinition) { if !packet_can_split(packet) { body.push_str( " fn encode_multiple(&self) -> Result>, crate::Error> {\n Ok(vec![self.encode_packet()?])\n }\n", ); return; } let variable_blocks = packet .blocks .iter() .filter(|block| block.repetition == BlockRepetition::Variable) .collect::>(); body.push_str( " #[allow(clippy::too_many_lines)]\n fn encode_multiple(&self) -> Result>, crate::Error> {\n let header = self.generated_header();\n let mut packets = Vec::new();\n", ); for block in &variable_blocks { let _ = writeln!( body, " let mut {}_start = 0_usize;", rust_snake(&block.name) ); } body.push_str(" loop {\n let first_packet = packets.is_empty();\n let mut packet_header = crate::packet_wire::clone_header(&header);\n if !first_packet { packet_header.appended_acks = false; packet_header.ack_list = None; }\n let acks_length = crate::packet_wire::ack_length(&packet_header)?;\n let mut fixed_length = crate::packet_wire::header_length(packet_header.frequency); "); for block in &packet.blocks { let field_name = rust_snake(&block.name); match block.repetition { BlockRepetition::Single => { let _ = writeln!( body, " fixed_length = fixed_length.checked_add(usize::try_from(GeneratedBlock::generated_length(&self.{field_name})).map_err(|_| crate::Error::Argument)?).ok_or(crate::Error::Argument)?;" ); } BlockRepetition::Multiple(_) => { let _ = writeln!( body, " for block in &self.{field_name} {{ fixed_length = fixed_length.checked_add(usize::try_from(GeneratedBlock::generated_length(block)).map_err(|_| crate::Error::Argument)?).ok_or(crate::Error::Argument)?; }}" ); } BlockRepetition::Variable => {} } } let _ = writeln!( body, " fixed_length = fixed_length.checked_add({}).ok_or(crate::Error::Argument)?;", variable_blocks.len() ); body.push_str(" let mut variable_length = 0_usize;\n"); for block in &variable_blocks { let field_name = rust_snake(&block.name); let _ = writeln!(body, " let mut {field_name}_count = 0_usize;"); let _ = writeln!( body, " while {field_name}_start + {field_name}_count < self.{field_name}.len() && {field_name}_count < usize::from(u8::MAX) {{" ); let _ = writeln!( body, " let block_length = usize::try_from(GeneratedBlock::generated_length(&self.{field_name}[{field_name}_start + {field_name}_count])).map_err(|_| crate::Error::Argument)?;" ); let _ = writeln!( body, " if fixed_length.saturating_add(variable_length).saturating_add(block_length).saturating_add(acks_length) <= usize::try_from(crate::packets::Packet::MTU).map_err(|_| crate::Error::Argument)? || {field_name}_count == 0 {{ variable_length = variable_length.checked_add(block_length).ok_or(crate::Error::Argument)?; {field_name}_count += 1; }} else {{ break; }}" ); body.push_str(" }\n"); } body.push_str(" let capacity = fixed_length.checked_add(variable_length).and_then(|length| length.checked_add(acks_length)).ok_or(crate::Error::Argument)?;\n let mut writer = crate::packet_wire::WireWriter::with_capacity(capacity)?;\n crate::packet_wire::encode_header(&packet_header, &mut writer)?;\n"); for block in &packet.blocks { let field_name = rust_snake(&block.name); match block.repetition { BlockRepetition::Single => { let _ = writeln!( body, " GeneratedBlock::encode_payload(&self.{field_name}, &mut writer)?;" ); } BlockRepetition::Multiple(count) => { let _ = writeln!( body, " if self.{field_name}.len() != {count} {{ return Err(crate::Error::Argument); }}" ); let _ = writeln!( body, " for block in &self.{field_name} {{ GeneratedBlock::encode_payload(block, &mut writer)?; }}" ); } BlockRepetition::Variable => { let _ = writeln!( body, " writer.write_u8(u8::try_from({field_name}_count).map_err(|_| crate::Error::Argument)?)?;" ); let _ = writeln!( body, " for block in &self.{field_name}[{field_name}_start..{field_name}_start + {field_name}_count] {{ GeneratedBlock::encode_payload(block, &mut writer)?; }}" ); let _ = writeln!( body, " {field_name}_start += {field_name}_count;" ); } } } body.push_str(" crate::packet_wire::encode_acks(&packet_header, &mut writer)?;\n packets.push(writer.into_inner());\n if "); for (index, block) in variable_blocks.iter().enumerate() { if index != 0 { body.push_str(" && "); } let field_name = rust_snake(&block.name); let _ = write!(body, "{field_name}_start >= self.{field_name}.len()"); } body.push_str(" { break; }\n }\n Ok(packets)\n }\n"); } fn append_packet_osd(body: &mut String, packet: &PacketDefinition) { body.push_str( " fn packet_to_osd(&self) -> Result {\n let body = libremetaverse_structured_data::OSDMap::new_with_constructor()?;\n", ); for block in &packet.blocks { let field_name = rust_snake(&block.name); let capacity = match block.repetition { BlockRepetition::Single => "1".to_owned(), BlockRepetition::Multiple(_) | BlockRepetition::Variable => { format!("self.{field_name}.len()") } }; let _ = writeln!( body, " let mut {field_name}_values = Vec::with_capacity({capacity});" ); match block.repetition { BlockRepetition::Single => { let _ = writeln!( body, " {field_name}_values.push(libremetaverse_structured_data::OSD::Map(GeneratedBlock::to_osd(&self.{field_name})?.snapshot()));" ); } BlockRepetition::Multiple(_) | BlockRepetition::Variable => { let _ = writeln!( body, " for block in &self.{field_name} {{ {field_name}_values.push(libremetaverse_structured_data::OSD::Map(GeneratedBlock::to_osd(block)?.snapshot())); }}" ); } } let _ = writeln!( body, " body.add_with_string_osd({:?}.to_owned(), libremetaverse_structured_data::OSD::Array({field_name}_values))?;", block.name ); } body.push_str(" Ok(body)\n }\n\n fn packet_from_osd(&mut self, body: &libremetaverse_structured_data::OSDMap) -> Result<(), crate::Error> {\n"); for block in &packet.blocks { let field_name = rust_snake(&block.name); let block_type = format!("crate::packets::{}Packet{}Block", packet.name, block.name); let _ = writeln!( body, " if let Some(libremetaverse_structured_data::OSD::Array(values)) = body.get({:?}) {{", block.name ); match block.repetition { BlockRepetition::Single => { let _ = writeln!( body, " if let Some(libremetaverse_structured_data::OSD::Map(values)) = values.first() {{ let map = libremetaverse_structured_data::OSDMap::new_with_dictionary(values.clone())?; GeneratedBlock::from_osd(&mut self.{field_name}, &map)?; }}" ); } BlockRepetition::Variable => { let _ = writeln!( body, " let mut blocks = Vec::new();\n blocks.try_reserve_exact(values.len()).map_err(|_| crate::Error::InvalidOperation)?;\n for value in values {{ let mut block = <{block_type} as GeneratedBlock>::new_generated(); if let libremetaverse_structured_data::OSD::Map(values) = value {{ let map = libremetaverse_structured_data::OSDMap::new_with_dictionary(values.clone())?; GeneratedBlock::from_osd(&mut block, &map)?; }} blocks.push(block); }}\n self.{field_name} = blocks;" ); } BlockRepetition::Multiple(count) => { let _ = writeln!( body, " if self.{field_name}.len() != {count} {{ self.{field_name} = (0..{count}).map(|_| <{block_type} as GeneratedBlock>::new_generated()).collect(); }}\n for (block, value) in self.{field_name}.iter_mut().zip(values.iter().take({count})) {{ if let libremetaverse_structured_data::OSD::Map(values) = value {{ let map = libremetaverse_structured_data::OSDMap::new_with_dictionary(values.clone())?; GeneratedBlock::from_osd(block, &map)?; }} }}" ); } } body.push_str(" }\n"); } if packet.blocks.is_empty() { body.push_str(" let _ = body;\n"); } body.push_str(" Ok(())\n }\n\n"); } #[allow(clippy::too_many_lines)] fn append_packet_impls(body: &mut String, packets: &[&PacketDefinition]) { for packet in packets { let type_name = format!("{}Packet", packet.name); let variable_count = packet .blocks .iter() .filter(|block| block.repetition == BlockRepetition::Variable) .count(); let base_length = match packet.frequency { PacketFrequency::Low => 10, PacketFrequency::Medium => 8, PacketFrequency::High => 7, }; let _ = writeln!( body, "impl GeneratedPacket for crate::packets::{type_name} {{\n const USES_BUFFER_POOLING: bool = {};\n\n fn new_generated() -> Self {{\n Self {{", variable_count == 0 ); if packet_has_composed_base(&packet.name) { let _ = writeln!( body, " base: crate::packets::Packet {{ has_variable_blocks: {}, header: new_header(crate::PacketFrequency::{}, {}, {}), type_: PacketType::{} }},", variable_count > 0, frequency_name(packet.frequency), packet.id, packet.zerocoded, packet.name ); } for block in &packet.blocks { let field_name = rust_snake(&block.name); let block_type = format!("crate::packets::{}Packet{}Block", packet.name, block.name); match block.repetition { BlockRepetition::Single => { let _ = writeln!( body, " {field_name}: <{block_type} as GeneratedBlock>::new_generated()," ); } BlockRepetition::Variable => { let _ = writeln!(body, " {field_name}: Vec::new(),"); } BlockRepetition::Multiple(count) => { let _ = writeln!( body, " {field_name}: (0..{count}).map(|_| <{block_type} as GeneratedBlock>::new_generated()).collect()," ); } } } body.push_str(" }\n }\n\n fn generated_length(&self) -> i32 {\n"); if packet.blocks.is_empty() { let _ = writeln!(body, " {base_length}_i32"); } else { let _ = writeln!(body, " let mut length = {base_length}_i32;"); for block in &packet.blocks { let field_name = rust_snake(&block.name); match block.repetition { BlockRepetition::Single => { let _ = writeln!( body, " length = length.saturating_add(GeneratedBlock::generated_length(&self.{field_name}));" ); } BlockRepetition::Variable => { let _ = writeln!(body, " length = length.saturating_add(1);"); let _ = writeln!( body, " for block in &self.{field_name} {{ length = length.saturating_add(GeneratedBlock::generated_length(block)); }}" ); } BlockRepetition::Multiple(_) => { let _ = writeln!( body, " for block in &self.{field_name} {{ length = length.saturating_add(GeneratedBlock::generated_length(block)); }}" ); } } } body.push_str(" length\n"); } body.push_str(" }\n\n fn generated_header(&self) -> crate::packets::Header {\n"); if packet_has_composed_base(&packet.name) { body.push_str(" crate::packet_wire::clone_header(&self.base.header)\n"); } else { let _ = writeln!( body, " new_header(crate::PacketFrequency::{}, {}, {})", frequency_name(packet.frequency), packet.id, packet.zerocoded ); } body.push_str( " }\n\n fn store_header(&mut self, header: crate::packets::Header) {\n", ); if packet_has_composed_base(&packet.name) { body.push_str(" self.base.header = header;\n"); } else { body.push_str(" let _ = header;\n"); } body.push_str(" }\n\n fn decode_payload(&mut self, reader: &mut crate::packet_wire::WireReader<'_>) -> Result<(), crate::Error> {\n"); for block in &packet.blocks { let field_name = rust_snake(&block.name); let block_type = format!("crate::packets::{}Packet{}Block", packet.name, block.name); match block.repetition { BlockRepetition::Single => { let _ = writeln!( body, " GeneratedBlock::decode_payload(&mut self.{field_name}, reader)?;" ); } BlockRepetition::Variable => { let _ = writeln!( body, " let {field_name}_count = usize::from(reader.read_u8()?);" ); let _ = writeln!(body, " let mut {field_name} = Vec::new();"); let _ = writeln!( body, " {field_name}.try_reserve_exact({field_name}_count).map_err(|_| crate::Error::InvalidOperation)?;" ); let _ = writeln!( body, " for _ in 0..{field_name}_count {{ let mut block = <{block_type} as GeneratedBlock>::new_generated(); GeneratedBlock::decode_payload(&mut block, reader)?; {field_name}.push(block); }}" ); let _ = writeln!(body, " self.{field_name} = {field_name};"); } BlockRepetition::Multiple(count) => { let _ = writeln!(body, " let mut {field_name} = Vec::new();"); let _ = writeln!( body, " {field_name}.try_reserve_exact({count}).map_err(|_| crate::Error::InvalidOperation)?;" ); let _ = writeln!( body, " for _ in 0..{count} {{ let mut block = <{block_type} as GeneratedBlock>::new_generated(); GeneratedBlock::decode_payload(&mut block, reader)?; {field_name}.push(block); }}" ); let _ = writeln!(body, " self.{field_name} = {field_name};"); } } } if packet.blocks.is_empty() { body.push_str(" let _ = reader;\n"); } body.push_str(" Ok(())\n }\n\n fn encode_payload(&self, writer: &mut crate::packet_wire::WireWriter) -> Result<(), crate::Error> {\n"); for block in &packet.blocks { let field_name = rust_snake(&block.name); match block.repetition { BlockRepetition::Single => { let _ = writeln!( body, " GeneratedBlock::encode_payload(&self.{field_name}, writer)?;" ); } BlockRepetition::Variable => { let _ = writeln!( body, " writer.write_u8(u8::try_from(self.{field_name}.len()).map_err(|_| crate::Error::Argument)?)?;" ); let _ = writeln!( body, " for block in &self.{field_name} {{ GeneratedBlock::encode_payload(block, writer)?; }}" ); } BlockRepetition::Multiple(count) => { let _ = writeln!( body, " if self.{field_name}.len() != {count} {{ return Err(crate::Error::Argument); }}" ); let _ = writeln!( body, " for block in &self.{field_name} {{ GeneratedBlock::encode_payload(block, writer)?; }}" ); } } } if packet.blocks.is_empty() { body.push_str(" let _ = writer;\n"); } body.push_str(" Ok(())\n }\n\n"); append_packet_osd(body, packet); append_packet_encode_multiple(body, packet); body.push_str("}\n\n"); } } fn append_packet_codec_tests(body: &mut String, packets: &[&PacketDefinition]) { body.push_str( "#[cfg(test)]\nmod generated_codec_tests {\n use super::{GeneratedBlock, GeneratedPacket};\n\n fn assert_round_trip(packet: &T, name: &str) {\n let expected = packet.encode_packet().unwrap_or_else(|error| panic!(\"{name}: encode failed: {error}\"));\n let (wire, mut zero_buffer) = if expected[0] & crate::Helpers::MSG_ZEROCODED != 0 {\n let mut encoded = vec![0_u8; expected.len().saturating_mul(2).saturating_add(2)];\n let encoded_length = crate::packet_wire::zero_encode(\n Some(&expected),\n i32::try_from(expected.len()).expect(\"packet length fits i32\"),\n Some(&mut encoded),\n )\n .unwrap_or_else(|error| panic!(\"{name}: zero encode failed: {error}\"));\n encoded.truncate(usize::try_from(encoded_length).expect(\"encoded length fits usize\"));\n (encoded, vec![0_u8; expected.len()])\n } else {\n (expected.clone(), Vec::new())\n };\n let mut decoded = T::new_generated();\n let mut position = 0_i32;\n let mut packet_end = i32::try_from(wire.len()).expect(\"wire length fits i32\") - 1;\n let buffer = (!zero_buffer.is_empty()).then_some(zero_buffer.as_mut_slice());\n decoded\n .decode_from_bytes(&wire, &mut position, &mut packet_end, buffer)\n .unwrap_or_else(|error| panic!(\"{name}: decode failed: {error}\"));\n let actual = decoded.encode_packet().unwrap_or_else(|error| panic!(\"{name}: re-encode failed: {error}\"));\n assert_eq!(actual, expected, \"{name}: round-trip bytes differ\");\n assert_eq!(position, i32::try_from(expected.len()).expect(\"packet length fits i32\"), \"{name}: final position\");\n\n for truncated_length in 0..wire.len() {\n let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {\n let mut candidate = T::new_generated();\n let mut truncated_position = 0_i32;\n let mut truncated_end = i32::try_from(truncated_length).expect(\"truncated length fits i32\") - 1;\n let mut scratch = vec![0_u8; expected.len()];\n let truncated_buffer = (!zero_buffer.is_empty()).then_some(scratch.as_mut_slice());\n candidate.decode_from_bytes(\n &wire[..truncated_length],\n &mut truncated_position,\n &mut truncated_end,\n truncated_buffer,\n )\n }));\n assert!(outcome.is_ok(), \"{name}: truncation at {truncated_length} panicked\");\n }\n }\n\n #[test]\n #[allow(clippy::too_many_lines)]\n fn every_generated_packet_round_trips_all_fields() {\n", ); let truncation_loop = body .rfind(" for truncated_length in 0..wire.len()") .expect("packet test scaffold contains truncation loop"); body.insert_str( truncation_loop, " let expected_osd = packet.packet_to_osd().unwrap_or_else(|error| panic!(\"{name}: OSD serialization failed: {error}\"));\n let mut osd_round_trip = T::new_generated();\n osd_round_trip.packet_from_osd(&expected_osd).unwrap_or_else(|error| panic!(\"{name}: OSD deserialization failed: {error}\"));\n assert_eq!(osd_round_trip.encode_packet().unwrap_or_else(|error| panic!(\"{name}: OSD re-encode failed: {error}\")), expected, \"{name}: OSD round-trip bytes differ\");\n assert_eq!(osd_round_trip.packet_to_osd().unwrap_or_else(|error| panic!(\"{name}: OSD reserialization failed: {error}\")).snapshot(), expected_osd.snapshot(), \"{name}: OSD round-trip differs\");\n\n", ); for packet in packets { let mutates_packet = packet .blocks .iter() .any(|block| block.repetition == BlockRepetition::Variable || !block.fields.is_empty()); let _ = writeln!( body, " let {}packet = ::new_generated();", if mutates_packet { "mut " } else { "" }, packet.name, ); for block in &packet.blocks { let block_name = rust_snake(&block.name); match block.repetition { BlockRepetition::Single => { for field in &block.fields { let field_name = rust_snake(&field.name); let sample = field_sample(field); let _ = writeln!(body, " packet.{block_name}.{field_name} = {sample};"); } } BlockRepetition::Multiple(_) => { let _ = writeln!(body, " for block in &mut packet.{block_name} {{"); for field in &block.fields { let field_name = rust_snake(&field.name); let sample = field_sample(field); let _ = writeln!(body, " block.{field_name} = {sample};"); } body.push_str(" }\n"); } BlockRepetition::Variable => { let block_type = format!("{}Packet{}Block", packet.name, block.name); let _ = writeln!( body, " let mut block = ::new_generated();" ); for field in &block.fields { let field_name = rust_snake(&field.name); let sample = field_sample(field); let _ = writeln!(body, " block.{field_name} = {sample};"); } let _ = writeln!(body, " packet.{block_name}.push(block);"); } } } let _ = writeln!( body, " assert_round_trip(&packet, {:?});", packet.name ); } body.push_str(" }\n}\n"); } fn csharp_field_type(kind: FieldKind) -> &'static str { match kind { FieldKind::U8 => "System.Byte", FieldKind::U16 | FieldKind::IpPort => "System.UInt16", FieldKind::U32 | FieldKind::IpAddr => "System.UInt32", FieldKind::U64 => "System.UInt64", FieldKind::S8 => "System.SByte", FieldKind::S16 => "System.Int16", FieldKind::S32 => "System.Int32", FieldKind::F32 => "System.Single", FieldKind::F64 => "System.Double", FieldKind::LlUuid => "LibreMetaverse.UUID", FieldKind::Bool => "System.Boolean", FieldKind::LlVector3 => "LibreMetaverse.Vector3", FieldKind::LlVector3d => "LibreMetaverse.Vector3d", FieldKind::LlVector4 => "LibreMetaverse.Vector4", FieldKind::LlQuaternion => "LibreMetaverse.Quaternion", FieldKind::Variable | FieldKind::Fixed => "System.Byte[]", } } fn catalog_fields(value: &serde_json::Value) -> Result, String> { let members = value["members"] .as_array() .ok_or("catalog type has no members array")?; members .iter() .filter(|member| member["kind"] == "field" && member["static"] == false) .map(|member| { let name = member["name"] .as_str() .ok_or("catalog field has no name")? .to_owned(); let field_type = member["type"] .as_str() .ok_or("catalog field has no type")? .to_owned(); Ok((name, field_type)) }) .collect() } #[allow(clippy::too_many_lines)] pub fn validate_packet_api_catalog(root: &Path, protocol: &PacketProtocol) -> Result<(), String> { let path = root.join(PUBLIC_API_PATH); let bytes = fs::read(&path).map_err(|error| format!("{}: {error}", path.display()))?; let catalog: serde_json::Value = serde_json::from_slice(&bytes).map_err(|error| format!("{}: {error}", path.display()))?; let assembly = catalog["assemblies"] .as_array() .and_then(|assemblies| { assemblies .iter() .find(|assembly| assembly["identity"]["name"] == "LibreMetaverse") }) .ok_or("public API catalog has no LibreMetaverse assembly")?; let types = assembly["types"] .as_array() .ok_or("LibreMetaverse catalog has no types array")?; let types_by_id: BTreeMap<&str, &serde_json::Value> = types .iter() .map(|item| { item["doc_id"] .as_str() .map(|id| (id, item)) .ok_or("catalog type has no doc_id") }) .collect::>()?; let expected_packet_ids: BTreeSet = protocol .packets .iter() .map(|packet| format!("T:LibreMetaverse.Packets.{}Packet", packet.name)) .collect(); let expected_block_ids: BTreeSet = protocol .packets .iter() .flat_map(|packet| { packet.blocks.iter().map(move |block| { format!( "T:LibreMetaverse.Packets.{}Packet.{}Block", packet.name, block.name ) }) }) .collect(); let actual_packet_ids: BTreeSet = types .iter() .filter(|item| item["base_type"] == "LibreMetaverse.Packets.Packet") .filter_map(|item| item["doc_id"].as_str().map(str::to_owned)) .collect(); let actual_block_ids: BTreeSet = types .iter() .filter(|item| item["base_type"] == "LibreMetaverse.Packets.PacketBlock") .filter_map(|item| item["doc_id"].as_str().map(str::to_owned)) .collect(); if expected_packet_ids != actual_packet_ids { return Err(format!( "packet API catalog mismatch: missing={:?}, stale={:?}", expected_packet_ids .difference(&actual_packet_ids) .collect::>(), actual_packet_ids .difference(&expected_packet_ids) .collect::>() )); } if expected_block_ids != actual_block_ids { return Err(format!( "packet block API catalog mismatch: missing={:?}, stale={:?}", expected_block_ids .difference(&actual_block_ids) .collect::>(), actual_block_ids .difference(&expected_block_ids) .collect::>() )); } for packet in &protocol.packets { let packet_id = format!("T:LibreMetaverse.Packets.{}Packet", packet.name); let actual = catalog_fields(types_by_id[packet_id.as_str()])?; let expected = packet .blocks .iter() .map(|block| { let name = if block.name == "Header" { "_Header".to_owned() } else { block.name.clone() }; let array = if block.repetition == BlockRepetition::Single { "" } else { "[]" }; ( name, format!( "LibreMetaverse.Packets.{}Packet.{}Block{array}", packet.name, block.name ), ) }) .collect::>(); if expected != actual { return Err(format!( "{packet_id} field mismatch: expected={expected:?}, actual={actual:?}" )); } for block in &packet.blocks { let block_id = format!( "T:LibreMetaverse.Packets.{}Packet.{}Block", packet.name, block.name ); let actual = catalog_fields(types_by_id[block_id.as_str()])?; let expected = block .fields .iter() .map(|field| (field.name.clone(), csharp_field_type(field.kind).to_owned())) .collect::>(); if expected != actual { return Err(format!( "{block_id} field mismatch: expected={expected:?}, actual={actual:?}" )); } } } let packet_type = types_by_id .get("T:LibreMetaverse.Packets.PacketType") .ok_or("public API catalog has no PacketType")?; let actual_values = packet_type["members"] .as_array() .ok_or("PacketType has no members array")? .iter() .filter(|member| member["kind"] == "enum_value") .map(|member| { let name = member["name"] .as_str() .ok_or("PacketType value has no name")? .to_owned(); let value = member["value"]["value"] .as_str() .ok_or("PacketType value has no numeric value")? .parse::() .map_err(|_| "PacketType value is not a u32")?; Ok::<_, &'static str>((name, value)) }) .collect::, _>>()?; let mut expected_values = BTreeMap::from([("Default".to_owned(), 0)]); expected_values.extend( protocol .packets .iter() .map(|packet| (packet.name.clone(), packet_type_value(packet))), ); if expected_values != actual_values { return Err(format!( "PacketType enum mismatch: missing/stale values = {}/{}", expected_values .keys() .filter(|name| !actual_values.contains_key(*name)) .count(), actual_values .keys() .filter(|name| !expected_values.contains_key(*name)) .count() )); } Ok(()) } pub fn packet_catalog_bytes(root: &Path) -> Result, String> { let inventory = load_inventory(root)?; verify_inputs(root, &inventory)?; let input = inventory .inputs .iter() .find(|input| input.id == "message_template") .ok_or("source inventory has no message_template input")?; let generator = inventory .generators .iter() .find(|generator| generator.id == "packets") .ok_or("source inventory has no packets generator")?; let path = root.join(&input.vendored_path); let bytes = fs::read(&path).map_err(|error| format!("{}: {error}", path.display()))?; let text = normalize_text(&input.vendored_path, &bytes).map_err(|error| error.to_string())?; let protocol = parse_packet_template(&input.vendored_path, &text).map_err(|diagnostics| { diagnostics .into_iter() .map(|item| item.to_string()) .collect::>() .join("\n") })?; validate_packet_api_catalog(root, &protocol)?; let packets = ordered_packets(&protocol); let mut body = String::new(); body.push_str("#![allow(clippy::cast_possible_truncation, clippy::cast_possible_wrap, clippy::cast_sign_loss, clippy::collapsible_if, clippy::similar_names, clippy::too_many_lines, clippy::unnecessary_wraps, clippy::vec_init_then_push, clippy::wrong_self_convention)]\n\n"); let _ = writeln!( body, "pub const GOLDEN_GENERATOR_SOURCE: &str = {:?};", generator.reference_source ); let _ = writeln!( body, "pub const GOLDEN_GENERATOR_SHA256: &str = {:?};", generator.sha256 ); let _ = writeln!( body, "pub const GOLDEN_GENERATOR_LICENSE: &str = {:?};\n", generator.license ); append_descriptor_types(&mut body); append_packet_type(&mut body, &packets); append_descriptors(&mut body, &packets); append_dispatch(&mut body, &packets); append_block_impls(&mut body, &packets); append_packet_impls(&mut body, &packets); append_packet_codec_tests(&mut body, &packets); let syntax = syn::parse_file(&body) .map_err(|error| format!("generated packet Rust is invalid: {error}"))?; let formatted = prettyplease::unparse(&syntax); Ok(generated_rust("packets", &[input], &formatted)) } pub fn load_inventory(root: &Path) -> Result { let path = root.join(INVENTORY_PATH); let bytes = fs::read(&path).map_err(|error| format!("{}: {error}", path.display()))?; let inventory: Inventory = serde_json::from_slice(&bytes).map_err(|error| format!("{}: {error}", path.display()))?; validate_inventory(&inventory)?; Ok(inventory) } fn validate_inventory(inventory: &Inventory) -> Result<(), String> { if inventory.schema != 1 || inventory.upstream_commit.len() != 40 { return Err("unsupported or malformed source inventory header".to_owned()); } let mut input_ids = BTreeSet::new(); for input in &inventory.inputs { validate_relative(&input.reference_path)?; validate_relative(&input.vendored_path)?; validate_hash(&input.sha256)?; if input.license != "BSD-3-Clause" || !input_ids.insert(&input.id) { return Err(format!("invalid or duplicate input {}", input.id)); } } let mut generator_ids = BTreeSet::new(); for generator in &inventory.generators { validate_relative(&generator.reference_source)?; validate_hash(&generator.sha256)?; if generator.license != "BSD-3-Clause" || !generator_ids.insert(&generator.id) { return Err(format!("invalid or duplicate generator {}", generator.id)); } for input in &generator.inputs { if !input_ids.contains(input) { return Err(format!( "generator {} references unknown input {input}", generator.id )); } } } Ok(()) } fn validate_relative(value: &str) -> Result<(), String> { let path = Path::new(value); if path.is_absolute() || path .components() .any(|part| !matches!(part, Component::Normal(_))) { return Err(format!("unsafe inventory path {value}")); } Ok(()) } fn validate_hash(value: &str) -> Result<(), String> { if value.len() == 64 && value.bytes().all(|byte| byte.is_ascii_hexdigit()) { Ok(()) } else { Err(format!("invalid SHA-256 {value}")) } } fn sha256(bytes: &[u8]) -> String { const HEX: &[u8; 16] = b"0123456789abcdef"; let mut output = String::with_capacity(64); for byte in Sha256::digest(bytes) { output.push(char::from(HEX[usize::from(byte >> 4)])); output.push(char::from(HEX[usize::from(byte & 0x0f)])); } output } pub fn verify_inputs(root: &Path, inventory: &Inventory) -> Result<(), String> { for input in &inventory.inputs { let path = root.join(&input.vendored_path); let bytes = fs::read(&path).map_err(|error| format!("{}: {error}", path.display()))?; let actual = sha256(&bytes); if actual != input.sha256 { return Err(format!( "{}: SHA-256 mismatch: expected {}, found {actual}", path.display(), input.sha256 )); } normalize_text(&input.vendored_path, &bytes).map_err(|error| error.to_string())?; } Ok(()) } pub fn vendor_inputs(root: &Path, reference: &Path) -> Result<(), String> { let inventory = load_inventory(root)?; for generator in &inventory.generators { let source = reference.join(&generator.reference_source); let bytes = fs::read(&source).map_err(|error| format!("{}: {error}", source.display()))?; let actual = sha256(&bytes); if actual != generator.sha256 { return Err(format!( "{}: pinned generator SHA-256 mismatch: expected {}, found {actual}", source.display(), generator.sha256 )); } } let mut snapshots = Vec::with_capacity(inventory.inputs.len()); for input in &inventory.inputs { let source = reference.join(&input.reference_path); let bytes = fs::read(&source).map_err(|error| format!("{}: {error}", source.display()))?; let actual = sha256(&bytes); if actual != input.sha256 { return Err(format!( "{}: pinned SHA-256 mismatch: expected {}, found {actual}", source.display(), input.sha256 )); } snapshots.push((root.join(&input.vendored_path), bytes)); } for (target, bytes) in snapshots { fs::create_dir_all(target.parent().ok_or("vendored input has no parent")?) .map_err(|error| error.to_string())?; fs::write(&target, bytes).map_err(|error| format!("{}: {error}", target.display()))?; } Ok(()) } pub fn source_manifest_bytes(root: &Path) -> Result, String> { let inventory = load_inventory(root)?; verify_inputs(root, &inventory)?; let mut inputs: Vec<&InputSpec> = inventory.inputs.iter().collect(); inputs.sort_by_key(|input| &input.id); let mut generators: Vec<&GeneratorSpec> = inventory.generators.iter().collect(); generators.sort_by_key(|generator| &generator.id); let mut body = String::new(); let _ = writeln!( body, "pub const UPSTREAM_COMMIT: &str = {:?};", inventory.upstream_commit ); let _ = writeln!( body, "pub const UPSTREAM_REPOSITORY: &str = {:?};", inventory.upstream_repository ); body.push_str("pub const SOURCES: &[(&str, &str, &str, &str)] = &[\n"); for input in &inputs { let _ = writeln!( body, " ({:?}, {:?}, {:?}, {:?}),", input.id, input.vendored_path, input.sha256, input.license ); } body.push_str("];\n"); body.push_str("pub const GENERATORS: &[(&str, &str, &str, &str)] = &[\n"); for generator in generators { let _ = writeln!( body, " ({:?}, {:?}, {:?}, {:?}),", generator.id, generator.reference_source, generator.sha256, generator.license ); } body.push_str("];\n"); Ok(generated_rust("source-manifest", &inputs, &body)) } fn generated_outputs(root: &Path) -> Result)>, String> { Ok(vec![ (MANIFEST_OUTPUT, source_manifest_bytes(root)?), (PACKET_OUTPUT, packet_catalog_bytes(root)?), (VISUAL_OUTPUT, xml_catalog::visual_catalog_bytes(root)?), (FOLIAGE_OUTPUT, xml_catalog::foliage_catalog_bytes(root)?), ( SKELETON_OUTPUT, avatar_catalog::skeleton_catalog_bytes(root)?, ), ( ATTENTION_OUTPUT, avatar_catalog::attention_catalog_bytes(root)?, ), ( GENEPOOL_OUTPUT, avatar_catalog::genepool_catalog_bytes(root)?, ), ]) } pub fn regenerate_to(root: &Path, output_root: &Path) -> Result<(), String> { for (relative, expected) in generated_outputs(root)? { let path = output_root.join(relative); fs::create_dir_all(path.parent().ok_or("generated output has no parent")?) .map_err(|error| error.to_string())?; fs::write(&path, expected).map_err(|error| format!("{}: {error}", path.display()))?; } Ok(()) } pub fn regenerate(root: &Path, check: bool) -> Result<(), String> { let outputs = generated_outputs(root)?; if check { for (relative, expected) in outputs { let path = root.join(relative); let actual = fs::read(&path).map_err(|error| format!("{}: {error}", path.display()))?; if actual != expected { return Err(format!( "{} is stale; run the regeneration command", path.display() )); } } } else { for (relative, expected) in outputs { let path = root.join(relative); fs::create_dir_all(path.parent().ok_or("generated output has no parent")?) .map_err(|error| error.to_string())?; fs::write(&path, expected).map_err(|error| format!("{}: {error}", path.display()))?; } } Ok(()) } #[cfg(test)] mod tests { use super::*; #[test] fn checked_in_inputs_and_manifest_are_current() { let root = workspace_root(); let first = source_manifest_bytes(&root).expect("first deterministic generation"); let second = source_manifest_bytes(&root).expect("second deterministic generation"); assert_eq!(first, second); let first_packets = packet_catalog_bytes(&root).expect("first packet generation"); let second_packets = packet_catalog_bytes(&root).expect("second packet generation"); assert_eq!(first_packets, second_packets); let first_visual = xml_catalog::visual_catalog_bytes(&root).expect("first visual generation"); let second_visual = xml_catalog::visual_catalog_bytes(&root).expect("second visual generation"); assert_eq!(first_visual, second_visual); let first_foliage = xml_catalog::foliage_catalog_bytes(&root).expect("first foliage generation"); let second_foliage = xml_catalog::foliage_catalog_bytes(&root).expect("second foliage generation"); assert_eq!(first_foliage, second_foliage); let first_skeleton = avatar_catalog::skeleton_catalog_bytes(&root).expect("first skeleton generation"); let second_skeleton = avatar_catalog::skeleton_catalog_bytes(&root).expect("second skeleton generation"); assert_eq!(first_skeleton, second_skeleton); let first_attention = avatar_catalog::attention_catalog_bytes(&root).expect("first attention generation"); let second_attention = avatar_catalog::attention_catalog_bytes(&root).expect("second attention generation"); assert_eq!(first_attention, second_attention); let first_genepool = avatar_catalog::genepool_catalog_bytes(&root).expect("first genepool generation"); let second_genepool = avatar_catalog::genepool_catalog_bytes(&root).expect("second genepool generation"); assert_eq!(first_genepool, second_genepool); } #[test] fn text_normalization_and_diagnostics_are_stable() { assert_eq!( normalize_text("a\\b.xml", b"\xef\xbb\xbfa\r\nb\r").unwrap(), "a\nb\n" ); let error = normalize_text("a\\b.xml", b"one\n\0two").unwrap_err(); assert_eq!( error.to_string(), "a/b.xml:2:1: error[CG002]: input contains a NUL byte" ); } #[test] fn inventory_rejects_unsafe_paths_hashes_and_unknown_inputs() { let root = workspace_root(); let mut inventory = load_inventory(&root).expect("pinned inventory"); inventory.inputs[0].vendored_path = "../escape".to_owned(); assert!(validate_inventory(&inventory).is_err()); let mut inventory = load_inventory(&root).expect("pinned inventory"); inventory.generators[0].sha256 = "not-a-hash".to_owned(); assert!(validate_inventory(&inventory).is_err()); let mut inventory = load_inventory(&root).expect("pinned inventory"); inventory.generators[0].inputs.push("missing".to_owned()); assert!(validate_inventory(&inventory).is_err()); } #[test] fn packet_template_parser_preserves_ids_order_and_repetition() { let text = "version 2.0\n{ Alpha Fixed 0xFFFFFFFB Trusted Zerocoded UDPDeprecated\n { One Single { Value U32 } }\n { Many Multiple 2 { Data Fixed 4 } }\n { Rest Variable { Name Variable 2 } }\n}\n{ Beta High 7 NotTrusted Unencoded }\n"; let protocol = parse_packet_template("fixture.msg", text).expect("valid protocol"); assert_eq!(protocol.packets.len(), 2); let alpha = &protocol.packets[0]; assert_eq!(alpha.id, 0xfffb); assert_eq!(alpha.frequency, PacketFrequency::Low); assert!(alpha.trusted && alpha.zerocoded); assert_eq!(alpha.flags, ["UDPDeprecated"]); assert_eq!(alpha.blocks[1].repetition, BlockRepetition::Multiple(2)); assert_eq!(alpha.blocks[2].repetition, BlockRepetition::Variable); assert_eq!(alpha.blocks[2].fields[0].count, 2); assert_eq!(protocol.packets[1].frequency, PacketFrequency::High); } #[test] fn packet_template_diagnostics_are_source_located_and_stable() { let text = "{ Bad Nope xx Maybe Compressed\n { Data Multiple 0 { Value Mystery 3 } }\n}\n"; let diagnostics = parse_packet_template("data\\bad.msg", text).unwrap_err(); let rendered = diagnostics .iter() .map(ToString::to_string) .collect::>(); assert_eq!( rendered, [ "data/bad.msg:1:7: error[PG030]: unknown packet frequency \"Nope\"", "data/bad.msg:1:12: error[PG031]: packet ID must be a decimal or hexadecimal unsigned integer", "data/bad.msg:1:15: error[PG032]: unknown packet trust marker \"Maybe\"", "data/bad.msg:1:21: error[PG033]: unknown packet coding marker \"Compressed\"", "data/bad.msg:2:18: error[PG014]: Multiple block count must be a positive decimal integer", "data/bad.msg:2:28: error[PG023]: unknown field type \"Mystery\"", ] ); } }