//! Deterministic framework shared by the `LibreMetaverse` data generators. #![allow(clippy::missing_errors_doc)] // The CLI renders the complete error at its boundary. #![allow(clippy::must_use_candidate)] // Generator helpers are also exercised for validation. use serde::Deserialize; use sha2::{Digest, Sha256}; use std::collections::{BTreeMap, BTreeSet}; use std::fmt; use std::fmt::Write as _; use std::fs; use std::path::{Component, Path, PathBuf}; pub const INVENTORY_PATH: &str = "codegen/sources.json"; pub const MANIFEST_OUTPUT: &str = "codegen/generated/source_manifest.rs"; pub const PACKET_OUTPUT: &str = "crates/libremetaverse/src/packet_catalog.rs"; pub const PUBLIC_API_PATH: &str = "api/public-api.json"; const RUST_KEYWORDS: &[&str] = &[ "as", "break", "const", "continue", "crate", "else", "enum", "extern", "false", "fn", "for", "if", "impl", "in", "let", "loop", "match", "mod", "move", "mut", "pub", "ref", "return", "self", "Self", "static", "struct", "super", "trait", "true", "type", "unsafe", "use", "where", "while", "async", "await", "dyn", "abstract", "become", "box", "do", "final", "macro", "override", "priv", "typeof", "unsized", "virtual", "yield", "try", "from", ]; #[derive(Clone, Debug, Deserialize, Eq, PartialEq)] pub struct Inventory { pub schema: u32, pub upstream_commit: String, pub upstream_repository: String, pub generators: Vec, pub inputs: Vec, } #[derive(Clone, Debug, Deserialize, Eq, PartialEq)] pub struct GeneratorSpec { pub id: String, pub reference_source: String, pub sha256: String, pub license: String, pub inputs: Vec, #[serde(default)] pub note: String, } #[derive(Clone, Debug, Deserialize, Eq, PartialEq)] pub struct InputSpec { pub id: String, pub reference_path: String, pub vendored_path: String, pub sha256: String, pub license: String, pub format: String, } #[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] pub enum Severity { Error, Warning, } #[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] pub struct Diagnostic { pub code: &'static str, pub severity: Severity, pub path: String, pub line: usize, pub column: usize, pub message: String, } #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum PacketFrequency { Low, Medium, High, } #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum BlockRepetition { Single, Multiple(usize), Variable, } #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum FieldKind { U8, U16, U32, U64, S8, S16, S32, F32, F64, LlUuid, Bool, LlVector3, LlVector3d, LlVector4, LlQuaternion, IpAddr, IpPort, Variable, Fixed, } #[derive(Clone, Debug, Eq, PartialEq)] pub struct PacketField { pub name: String, pub kind: FieldKind, pub count: usize, pub line: usize, pub column: usize, } #[derive(Clone, Debug, Eq, PartialEq)] pub struct PacketBlock { pub name: String, pub repetition: BlockRepetition, pub fields: Vec, pub line: usize, pub column: usize, } #[derive(Clone, Debug, Eq, PartialEq)] pub struct PacketDefinition { pub id: u16, pub name: String, pub frequency: PacketFrequency, pub trusted: bool, pub zerocoded: bool, pub flags: Vec, pub blocks: Vec, pub line: usize, pub column: usize, } #[derive(Clone, Debug, Eq, PartialEq)] pub struct PacketProtocol { pub packets: Vec, } #[derive(Clone, Debug, Eq, PartialEq)] struct Token { text: String, line: usize, column: usize, } impl fmt::Display for Diagnostic { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { let severity = match self.severity { Severity::Error => "error", Severity::Warning => "warning", }; write!( formatter, "{}:{}:{}: {severity}[{}]: {}", self.path, self.line, self.column, self.code, self.message ) } } pub fn workspace_root() -> PathBuf { Path::new(env!("CARGO_MANIFEST_DIR")).join("../..") } pub fn normalize_text(path: &str, bytes: &[u8]) -> Result { let bytes = bytes.strip_prefix(&[0xef, 0xbb, 0xbf]).unwrap_or(bytes); let text = std::str::from_utf8(bytes).map_err(|error| { let valid = &bytes[..error.valid_up_to()]; let (line, column) = line_column(valid); Diagnostic { code: "CG001", severity: Severity::Error, path: path.replace('\\', "/"), line, column, message: "input is not valid UTF-8".to_owned(), } })?; if let Some(position) = text.as_bytes().iter().position(|byte| *byte == 0) { let prefix = &text.as_bytes()[..position]; let (line, column) = line_column(prefix); return Err(Diagnostic { code: "CG002", severity: Severity::Error, path: path.replace('\\', "/"), line, column, message: "input contains a NUL byte".to_owned(), }); } Ok(text.replace("\r\n", "\n").replace('\r', "\n")) } fn line_column(bytes: &[u8]) -> (usize, usize) { let mut line = 1; let mut column = 1; for byte in bytes { if *byte == b'\n' { line += 1; column = 1; } else { column += 1; } } (line, column) } fn packet_diagnostic( path: &str, token: Option<&Token>, code: &'static str, message: impl Into, ) -> Diagnostic { Diagnostic { code, severity: Severity::Error, path: path.replace('\\', "/"), line: token.map_or(1, |token| token.line), column: token.map_or(1, |token| token.column), message: message.into(), } } fn tokenize_packet_template(path: &str, text: &str) -> Result, Vec> { let mut tokens = Vec::new(); let mut characters = text.char_indices().peekable(); let mut line = 1usize; let mut column = 1usize; while let Some((start, character)) = characters.next() { if character == '\n' { line += 1; column = 1; continue; } if character.is_whitespace() { column += 1; continue; } if character == '/' && characters.peek().is_some_and(|(_, next)| *next == '/') { characters.next(); column += 2; for (_, next) in characters.by_ref() { if next == '\n' { line += 1; column = 1; break; } column += 1; } continue; } if matches!(character, '{' | '}') { tokens.push(Token { text: character.to_string(), line, column, }); column += 1; continue; } let token_line = line; let token_column = column; let mut end = start + character.len_utf8(); column += 1; while let Some((offset, next)) = characters.peek().copied() { if next.is_whitespace() || matches!(next, '{' | '}') { break; } if next == '/' { let mut lookahead = characters.clone(); lookahead.next(); if lookahead .peek() .is_some_and(|(_, following)| *following == '/') { break; } } characters.next(); end = offset + next.len_utf8(); column += 1; } let value = &text[start..end]; if !value.is_ascii() { return Err(vec![packet_diagnostic( path, Some(&Token { text: value.to_owned(), line: token_line, column: token_column, }), "PG002", "protocol identifiers and values must be ASCII", )]); } tokens.push(Token { text: value.to_owned(), line: token_line, column: token_column, }); } Ok(tokens) } struct PacketParser<'a> { path: &'a str, tokens: Vec, position: usize, diagnostics: Vec, } impl PacketParser<'_> { fn current(&self) -> Option<&Token> { self.tokens.get(self.position) } fn advance(&mut self) -> Option { let token = self.tokens.get(self.position).cloned(); self.position += usize::from(token.is_some()); token } fn expect(&mut self, value: &str, code: &'static str, context: &str) -> bool { if self.current().is_some_and(|token| token.text == value) { self.position += 1; true } else { self.diagnostics.push(packet_diagnostic( self.path, self.current(), code, format!("expected {value:?} {context}"), )); false } } fn atom(&mut self, code: &'static str, context: &str) -> Option { match self.current() { Some(token) if !matches!(token.text.as_str(), "{" | "}") => self.advance(), _ => { self.diagnostics.push(packet_diagnostic( self.path, self.current(), code, format!("expected {context}"), )); None } } } fn recover_to_close(&mut self) { let mut depth = 0usize; while let Some(token) = self.advance() { match token.text.as_str() { "{" => depth += 1, "}" if depth == 0 => break, "}" => depth -= 1, _ => {} } } } fn parse_field(&mut self) -> Option { if !self.expect("{", "PG020", "to begin a field") { return None; } let name = self.atom("PG021", "a field name")?; let kind_token = self.atom("PG022", "a field type")?; let (kind, known_kind) = match kind_token.text.as_str() { "U8" => (FieldKind::U8, true), "U16" => (FieldKind::U16, true), "U32" => (FieldKind::U32, true), "U64" => (FieldKind::U64, true), "S8" => (FieldKind::S8, true), "S16" => (FieldKind::S16, true), "S32" => (FieldKind::S32, true), "F32" => (FieldKind::F32, true), "F64" => (FieldKind::F64, true), "LLUUID" => (FieldKind::LlUuid, true), "BOOL" => (FieldKind::Bool, true), "LLVector3" => (FieldKind::LlVector3, true), "LLVector3d" => (FieldKind::LlVector3d, true), "LLVector4" => (FieldKind::LlVector4, true), "LLQuaternion" => (FieldKind::LlQuaternion, true), "IPADDR" => (FieldKind::IpAddr, true), "IPPORT" => (FieldKind::IpPort, true), "Variable" => (FieldKind::Variable, true), "Fixed" => (FieldKind::Fixed, true), other => { self.diagnostics.push(packet_diagnostic( self.path, Some(&kind_token), "PG023", format!("unknown field type {other:?}"), )); (FieldKind::Variable, false) } }; let count = if self.current().is_some_and(|token| token.text != "}") { let count_token = self.atom("PG024", "a field width")?; match count_token.text.parse::() { Ok(value) if value > 0 => value, _ => { self.diagnostics.push(packet_diagnostic( self.path, Some(&count_token), "PG025", "field width must be a positive decimal integer", )); 1 } } } else { 1 }; if known_kind && matches!(kind, FieldKind::Variable) && !matches!(count, 1 | 2) { self.diagnostics.push(packet_diagnostic( self.path, Some(&name), "PG026", "Variable fields require a one- or two-byte length prefix", )); } if known_kind && !matches!(kind, FieldKind::Variable | FieldKind::Fixed) && count != 1 { self.diagnostics.push(packet_diagnostic( self.path, Some(&name), "PG027", "only Variable and Fixed fields may specify a width", )); } if !self.expect("}", "PG028", "after a field") { self.recover_to_close(); } Some(PacketField { name: name.text, kind, count, line: name.line, column: name.column, }) } fn parse_block(&mut self) -> Option { if !self.expect("{", "PG010", "to begin a block") { return None; } let name = self.atom("PG011", "a block name")?; let repetition_token = self.atom("PG012", "a block repetition")?; let repetition = match repetition_token.text.as_str() { "Single" => BlockRepetition::Single, "Variable" => BlockRepetition::Variable, "Multiple" => { let count = self.atom("PG013", "a Multiple block count")?; match count.text.parse::() { Ok(value) if value > 0 => BlockRepetition::Multiple(value), _ => { self.diagnostics.push(packet_diagnostic( self.path, Some(&count), "PG014", "Multiple block count must be a positive decimal integer", )); BlockRepetition::Multiple(1) } } } other => { self.diagnostics.push(packet_diagnostic( self.path, Some(&repetition_token), "PG015", format!("unknown block repetition {other:?}"), )); BlockRepetition::Single } }; let mut fields = Vec::new(); while self.current().is_some_and(|token| token.text == "{") { if let Some(field) = self.parse_field() { fields.push(field); } } if !self.expect("}", "PG016", "after a block") { self.recover_to_close(); } let mut names = BTreeSet::new(); for field in &fields { if !names.insert(&field.name) { self.diagnostics.push(packet_diagnostic( self.path, Some(&Token { text: field.name.clone(), line: field.line, column: field.column, }), "PG017", format!("duplicate field {:?} in block {:?}", field.name, name.text), )); } } Some(PacketBlock { name: name.text, repetition, fields, line: name.line, column: name.column, }) } #[allow(clippy::too_many_lines)] fn parse_packet(&mut self) -> Option { if !self.expect("{", "PG003", "to begin a packet") { return None; } let name = self.atom("PG004", "a packet name")?; let frequency_token = self.atom("PG005", "a packet frequency")?; let id_token = self.atom("PG006", "a packet ID")?; let trust_token = self.atom("PG007", "a packet trust marker")?; let coding_token = self.atom("PG008", "a packet coding marker")?; let frequency = match frequency_token.text.as_str() { "Fixed" | "Low" => PacketFrequency::Low, "Medium" | "Mid" => PacketFrequency::Medium, "High" => PacketFrequency::High, other => { self.diagnostics.push(packet_diagnostic( self.path, Some(&frequency_token), "PG030", format!("unknown packet frequency {other:?}"), )); PacketFrequency::Low } }; let parsed_id = if let Some(hex) = id_token .text .strip_prefix("0x") .or_else(|| id_token.text.strip_prefix("0X")) { u32::from_str_radix(hex, 16) } else { id_token.text.parse::() }; let id = if let Ok(value) = parsed_id { (value & 0xffff) as u16 } else { self.diagnostics.push(packet_diagnostic( self.path, Some(&id_token), "PG031", "packet ID must be a decimal or hexadecimal unsigned integer", )); 0 }; let trusted = match trust_token.text.as_str() { "Trusted" => true, "NotTrusted" => false, other => { self.diagnostics.push(packet_diagnostic( self.path, Some(&trust_token), "PG032", format!("unknown packet trust marker {other:?}"), )); false } }; let zerocoded = match coding_token.text.as_str() { "Zerocoded" => true, "Unencoded" => false, other => { self.diagnostics.push(packet_diagnostic( self.path, Some(&coding_token), "PG033", format!("unknown packet coding marker {other:?}"), )); false } }; let mut flags = Vec::new(); while self .current() .is_some_and(|token| !matches!(token.text.as_str(), "{" | "}")) { if let Some(flag) = self.advance() { flags.push(flag.text); } } let mut blocks = Vec::new(); while self.current().is_some_and(|token| token.text == "{") { if let Some(block) = self.parse_block() { blocks.push(block); } } if !self.expect("}", "PG009", "after a packet") { self.recover_to_close(); } let mut block_names = BTreeSet::new(); for block in &blocks { if !block_names.insert(&block.name) { self.diagnostics.push(packet_diagnostic( self.path, Some(&Token { text: block.name.clone(), line: block.line, column: block.column, }), "PG034", format!("duplicate block {:?} in packet {:?}", block.name, name.text), )); } } Some(PacketDefinition { id, name: name.text, frequency, trusted, zerocoded, flags, blocks, line: name.line, column: name.column, }) } } pub fn parse_packet_template(path: &str, text: &str) -> Result> { let tokens = tokenize_packet_template(path, text)?; let mut parser = PacketParser { path, tokens, position: 0, diagnostics: Vec::new(), }; while parser.current().is_some_and(|token| token.text != "{") { parser.position += 1; } let mut packets = Vec::new(); while parser.current().is_some() { if parser.current().is_some_and(|token| token.text != "{") { parser.diagnostics.push(packet_diagnostic( path, parser.current(), "PG035", "unexpected token outside a packet", )); parser.position += 1; continue; } if let Some(packet) = parser.parse_packet() { packets.push(packet); } else { parser.recover_to_close(); } } let mut names = BTreeMap::new(); let mut ids = BTreeMap::new(); for packet in &packets { if let Some((line, column)) = names.insert(packet.name.clone(), (packet.line, packet.column)) { parser.diagnostics.push(packet_diagnostic( path, Some(&Token { text: packet.name.clone(), line: packet.line, column: packet.column, }), "PG036", format!( "duplicate packet {:?}; first declared at {line}:{column}", packet.name ), )); } let key = (packet.frequency as u8, packet.id); if let Some((other, line, column)) = ids.insert(key, (&packet.name, packet.line, packet.column)) { parser.diagnostics.push(packet_diagnostic( path, Some(&Token { text: packet.name.clone(), line: packet.line, column: packet.column, }), "PG037", format!( "duplicate {:?} packet ID {}; {:?} was declared at {line}:{column}", packet.frequency, packet.id, other ), )); } } parser.diagnostics.sort_by(|left, right| { (&left.path, left.line, left.column, left.code, &left.message).cmp(&( &right.path, right.line, right.column, right.code, &right.message, )) }); if parser.diagnostics.is_empty() { Ok(PacketProtocol { packets }) } else { Err(parser.diagnostics) } } pub fn generated_rust(generator: &str, sources: &[&InputSpec], body: &str) -> Vec { let mut output = String::new(); output.push_str("// @generated by libremetaverse-codegen; DO NOT EDIT.\n"); output.push_str("// Regenerate: cargo run -p libremetaverse-codegen -- generate\n"); let _ = writeln!(output, "// Generator: {generator}"); for source in sources { let _ = writeln!( output, "// Source: {} sha256={} license={}", source.vendored_path, source.sha256, source.license ); } output.push('\n'); output.push_str(body.trim_end()); output.push('\n'); output.into_bytes() } fn rust_snake(name: &str) -> String { let characters: Vec = name.trim_start_matches('@').chars().collect(); let mut output = String::new(); for (index, character) in characters.iter().copied().enumerate() { let previous = index .checked_sub(1) .and_then(|i| characters.get(i)) .copied(); let next = characters.get(index + 1).copied(); if character.is_ascii_uppercase() && !output.is_empty() && (previous.is_some_and(|value| value.is_ascii_lowercase() || value.is_ascii_digit()) || (previous.is_some_and(|value| value.is_ascii_uppercase()) && next.is_some_and(|value| value.is_ascii_lowercase()))) { output.push('_'); } if character.is_ascii_alphanumeric() || character == '_' { output.push(character.to_ascii_lowercase()); } else { output.push('_'); } } while output.contains("__") { output = output.replace("__", "_"); } let mut output = output.trim_matches('_').to_owned(); if output.is_empty() { output.push_str("item"); } if output.starts_with(|character: char| character.is_ascii_digit()) || RUST_KEYWORDS.contains(&output.as_str()) { output.push('_'); } output } const fn field_fixed_length(field: &PacketField) -> usize { match field.kind { FieldKind::Bool | FieldKind::U8 | FieldKind::S8 => 1, FieldKind::U16 | FieldKind::S16 | FieldKind::IpPort => 2, FieldKind::U32 | FieldKind::S32 | FieldKind::F32 | FieldKind::IpAddr => 4, FieldKind::U64 | FieldKind::F64 => 8, FieldKind::LlVector3 | FieldKind::LlQuaternion => 12, FieldKind::LlUuid | FieldKind::LlVector4 => 16, FieldKind::LlVector3d => 24, FieldKind::Fixed => field.count, FieldKind::Variable => 0, } } fn field_kind_name(kind: FieldKind) -> &'static str { match kind { FieldKind::U8 => "U8", FieldKind::U16 => "U16", FieldKind::U32 => "U32", FieldKind::U64 => "U64", FieldKind::S8 => "S8", FieldKind::S16 => "S16", FieldKind::S32 => "S32", FieldKind::F32 => "F32", FieldKind::F64 => "F64", FieldKind::LlUuid => "LlUuid", FieldKind::Bool => "Bool", FieldKind::LlVector3 => "LlVector3", FieldKind::LlVector3d => "LlVector3d", FieldKind::LlVector4 => "LlVector4", FieldKind::LlQuaternion => "LlQuaternion", FieldKind::IpAddr => "IpAddr", FieldKind::IpPort => "IpPort", FieldKind::Variable => "Variable", FieldKind::Fixed => "Fixed", } } fn frequency_name(frequency: PacketFrequency) -> &'static str { match frequency { PacketFrequency::Low => "Low", PacketFrequency::Medium => "Medium", PacketFrequency::High => "High", } } fn field_default(kind: FieldKind) -> &'static str { match kind { FieldKind::Bool => "false", FieldKind::U8 | FieldKind::U16 | FieldKind::U32 | FieldKind::U64 | FieldKind::S8 | FieldKind::S16 | FieldKind::S32 | FieldKind::IpAddr | FieldKind::IpPort => "0", FieldKind::F32 => "0.0_f32", FieldKind::F64 => "0.0_f64", FieldKind::LlUuid => "libremetaverse_types::UUID::zero()", FieldKind::LlVector3 => "libremetaverse_types::Vector3::zero()", FieldKind::LlVector3d => "libremetaverse_types::Vector3d::zero()", FieldKind::LlVector4 => "libremetaverse_types::Vector4::zero()", FieldKind::LlQuaternion => { "libremetaverse_types::Quaternion { x: 0.0, y: 0.0, z: 0.0, w: 0.0 }" } FieldKind::Variable | FieldKind::Fixed => "Vec::new()", } } fn ordered_packets(protocol: &PacketProtocol) -> Vec<&PacketDefinition> { [ PacketFrequency::Low, PacketFrequency::Medium, PacketFrequency::High, ] .into_iter() .flat_map(|frequency| { protocol .packets .iter() .filter(move |packet| packet.frequency == frequency) }) .collect() } fn packet_type_value(packet: &PacketDefinition) -> u32 { let prefix = match packet.frequency { PacketFrequency::Low => 0x1_0000, PacketFrequency::Medium => 0x2_0000, PacketFrequency::High => 0x3_0000, }; prefix | u32::from(packet.id) } fn append_descriptor_types(body: &mut String) { body.push_str( "#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]\n\ pub enum FieldKind { U8, U16, U32, U64, S8, S16, S32, F32, F64, LlUuid, Bool, LlVector3, LlVector3d, LlVector4, LlQuaternion, IpAddr, IpPort, Variable, Fixed }\n\n\ #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]\n\ pub enum BlockRepetition { Single, Multiple(usize), Variable }\n\n\ #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]\n\ pub struct FieldDescriptor { pub name: &'static str, pub kind: FieldKind, pub count: usize, pub fixed_width: usize }\n\n\ #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]\n\ pub struct BlockDescriptor { pub name: &'static str, pub repetition: BlockRepetition, pub fields: &'static [FieldDescriptor] }\n\n\ #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]\n\ pub struct PacketDescriptor { pub name: &'static str, pub packet_type: PacketType, pub id: u16, pub frequency: crate::PacketFrequency, pub trusted: bool, pub zerocoded: bool, pub flags: &'static [&'static str], pub blocks: &'static [BlockDescriptor] }\n\n", ); } fn append_packet_type(body: &mut String, packets: &[&PacketDefinition]) { body.push_str("#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]\n#[repr(i32)]\npub enum PacketType {\n Default = 0,\n"); for packet in packets { let prefix = match packet.frequency { PacketFrequency::Low => 1, PacketFrequency::Medium => 2, PacketFrequency::High => 3, }; let _ = writeln!(body, " {} = 0x{prefix}_{:04x},", packet.name, packet.id); } body.push_str("}\n\n"); } fn append_descriptors(body: &mut String, packets: &[&PacketDefinition]) { for (packet_index, packet) in packets.iter().enumerate() { for (block_index, block) in packet.blocks.iter().enumerate() { let _ = writeln!( body, "static PACKET_{packet_index}_BLOCK_{block_index}_FIELDS: &[FieldDescriptor] = &[" ); for field in &block.fields { let _ = writeln!( body, " FieldDescriptor {{ name: {:?}, kind: FieldKind::{}, count: {}, fixed_width: {} }},", field.name, field_kind_name(field.kind), field.count, field_fixed_length(field) ); } body.push_str("];\n"); } let _ = writeln!( body, "static PACKET_{packet_index}_BLOCKS: &[BlockDescriptor] = &[" ); for (block_index, block) in packet.blocks.iter().enumerate() { let repetition = match block.repetition { BlockRepetition::Single => "BlockRepetition::Single".to_owned(), BlockRepetition::Variable => "BlockRepetition::Variable".to_owned(), BlockRepetition::Multiple(count) => { format!("BlockRepetition::Multiple({count})") } }; let _ = writeln!( body, " BlockDescriptor {{ name: {:?}, repetition: {repetition}, fields: PACKET_{packet_index}_BLOCK_{block_index}_FIELDS }},", block.name ); } body.push_str("];\n\n"); } body.push_str("pub static PACKETS: &[PacketDescriptor] = &[\n"); for (packet_index, packet) in packets.iter().enumerate() { let flags = packet .flags .iter() .map(|flag| format!("{flag:?}")) .collect::>() .join(", "); let _ = writeln!( body, " PacketDescriptor {{ name: {:?}, packet_type: PacketType::{}, id: {}, frequency: crate::PacketFrequency::{}, trusted: {}, zerocoded: {}, flags: &[{}], blocks: PACKET_{packet_index}_BLOCKS }},", packet.name, packet.name, packet.id, frequency_name(packet.frequency), packet.trusted, packet.zerocoded, flags ); } body.push_str("];\n\n"); } fn append_dispatch(body: &mut String, packets: &[&PacketDefinition]) { body.push_str("#[must_use]\n#[allow(clippy::too_many_lines)]\npub fn packet_type(id: u16, frequency: crate::PacketFrequency) -> PacketType {\n match (frequency, id) {\n"); for packet in packets { let _ = writeln!( body, " (crate::PacketFrequency::{}, {}) => PacketType::{},", frequency_name(packet.frequency), packet.id, packet.name ); } body.push_str(" _ => PacketType::Default,\n }\n}\n\n"); body.push_str("#[must_use]\n#[allow(clippy::too_many_lines)]\npub fn descriptor_by_type(packet_type: PacketType) -> Option<&'static PacketDescriptor> {\n match packet_type {\n"); for (index, packet) in packets.iter().enumerate() { let _ = writeln!( body, " PacketType::{} => Some(&PACKETS[{index}]),", packet.name ); } body.push_str(" PacketType::Default => None,\n }\n}\n\n"); body.push_str("#[must_use]\n#[allow(clippy::too_many_lines)]\npub fn descriptor_by_name(name: &str) -> Option<&'static PacketDescriptor> {\n match name {\n"); for (index, packet) in packets.iter().enumerate() { let _ = writeln!( body, " {:?} => Some(&PACKETS[{index}]),", packet.name ); } body.push_str(" _ => None,\n }\n}\n\n"); body.push_str( "pub(crate) fn build_packet(packet_type: PacketType) -> Result {\n\ let descriptor = descriptor_by_type(packet_type).ok_or(crate::Error::InvalidOperation)?;\n\ Ok(crate::packets::Packet {\n\ has_variable_blocks: descriptor.blocks.iter().any(|block| matches!(block.repetition, BlockRepetition::Variable)),\n\ header: new_header(descriptor.frequency, descriptor.id, descriptor.zerocoded),\n\ type_: packet_type,\n\ })\n\ }\n\n\ pub(crate) const fn new_header(frequency: crate::PacketFrequency, id: u16, zerocoded: bool) -> crate::packets::Header {\n\ crate::packets::Header { ack_list: None, appended_acks: false, frequency, id, reliable: true, resent: false, sequence: 0, zerocoded }\n\ }\n\n\ fn usize_to_i32(value: usize) -> i32 { i32::try_from(value).unwrap_or(i32::MAX) }\n\n\ pub(crate) trait GeneratedBlock { fn new_generated() -> Self; fn generated_length(&self) -> i32; }\n\n\ pub(crate) trait GeneratedPacket { fn new_generated() -> Self; fn generated_length(&self) -> i32; const USES_BUFFER_POOLING: bool; }\n\n", ); } fn append_block_impls(body: &mut String, packets: &[&PacketDefinition]) { for packet in packets { for block in &packet.blocks { let type_name = format!("{}Packet{}Block", packet.name, block.name); let _ = writeln!( body, "impl GeneratedBlock for crate::packets::{type_name} {{\n fn new_generated() -> Self {{\n Self {{" ); for field in &block.fields { let _ = writeln!( body, " {}: {},", rust_snake(&field.name), field_default(field.kind) ); } body.push_str(" }\n }\n\n fn generated_length(&self) -> i32 {\n"); let fixed = block.fields.iter().map(field_fixed_length).sum::(); let variable_fields = block .fields .iter() .filter(|field| field.kind == FieldKind::Variable) .collect::>(); if variable_fields.is_empty() { let _ = writeln!(body, " {fixed}_i32"); } else { let _ = writeln!(body, " let mut length = {fixed}_i32;"); for field in variable_fields { let _ = writeln!( body, " length = length.saturating_add({}).saturating_add(usize_to_i32(self.{}.len()));", field.count, rust_snake(&field.name) ); } body.push_str(" length\n"); } body.push_str(" }\n}\n\n"); } } } fn packet_has_composed_base(name: &str) -> bool { matches!(name, "DirPlacesReply" | "TestMessage") } fn append_packet_impls(body: &mut String, packets: &[&PacketDefinition]) { for packet in packets { let type_name = format!("{}Packet", packet.name); let variable_count = packet .blocks .iter() .filter(|block| block.repetition == BlockRepetition::Variable) .count(); let base_length = match packet.frequency { PacketFrequency::Low => 10, PacketFrequency::Medium => 8, PacketFrequency::High => 7, }; let _ = writeln!( body, "impl GeneratedPacket for crate::packets::{type_name} {{\n const USES_BUFFER_POOLING: bool = {};\n\n fn new_generated() -> Self {{\n Self {{", variable_count == 0 ); if packet_has_composed_base(&packet.name) { let _ = writeln!( body, " base: crate::packets::Packet {{ has_variable_blocks: {}, header: new_header(crate::PacketFrequency::{}, {}, {}), type_: PacketType::{} }},", variable_count > 0, frequency_name(packet.frequency), packet.id, packet.zerocoded, packet.name ); } for block in &packet.blocks { let field_name = rust_snake(&block.name); let block_type = format!("crate::packets::{}Packet{}Block", packet.name, block.name); match block.repetition { BlockRepetition::Single => { let _ = writeln!( body, " {field_name}: <{block_type} as GeneratedBlock>::new_generated()," ); } BlockRepetition::Variable => { let _ = writeln!(body, " {field_name}: Vec::new(),"); } BlockRepetition::Multiple(count) => { let _ = writeln!( body, " {field_name}: (0..{count}).map(|_| <{block_type} as GeneratedBlock>::new_generated()).collect()," ); } } } body.push_str(" }\n }\n\n fn generated_length(&self) -> i32 {\n"); if packet.blocks.is_empty() { let _ = writeln!(body, " {base_length}_i32"); body.push_str(" }\n}\n\n"); continue; } let _ = writeln!( body, " let mut length = {}_i32;", base_length + variable_count ); for block in &packet.blocks { let field_name = rust_snake(&block.name); match block.repetition { BlockRepetition::Single => { let _ = writeln!( body, " length = length.saturating_add(GeneratedBlock::generated_length(&self.{field_name}));" ); } BlockRepetition::Variable => { let _ = writeln!(body, " length = length.saturating_add(1);"); let _ = writeln!( body, " for block in &self.{field_name} {{ length = length.saturating_add(GeneratedBlock::generated_length(block)); }}" ); } BlockRepetition::Multiple(_) => { let _ = writeln!( body, " for block in &self.{field_name} {{ length = length.saturating_add(GeneratedBlock::generated_length(block)); }}" ); } } } body.push_str(" length\n }\n}\n\n"); } } fn csharp_field_type(kind: FieldKind) -> &'static str { match kind { FieldKind::U8 => "System.Byte", FieldKind::U16 | FieldKind::IpPort => "System.UInt16", FieldKind::U32 | FieldKind::IpAddr => "System.UInt32", FieldKind::U64 => "System.UInt64", FieldKind::S8 => "System.SByte", FieldKind::S16 => "System.Int16", FieldKind::S32 => "System.Int32", FieldKind::F32 => "System.Single", FieldKind::F64 => "System.Double", FieldKind::LlUuid => "LibreMetaverse.UUID", FieldKind::Bool => "System.Boolean", FieldKind::LlVector3 => "LibreMetaverse.Vector3", FieldKind::LlVector3d => "LibreMetaverse.Vector3d", FieldKind::LlVector4 => "LibreMetaverse.Vector4", FieldKind::LlQuaternion => "LibreMetaverse.Quaternion", FieldKind::Variable | FieldKind::Fixed => "System.Byte[]", } } fn catalog_fields(value: &serde_json::Value) -> Result, String> { let members = value["members"] .as_array() .ok_or("catalog type has no members array")?; members .iter() .filter(|member| member["kind"] == "field" && member["static"] == false) .map(|member| { let name = member["name"] .as_str() .ok_or("catalog field has no name")? .to_owned(); let field_type = member["type"] .as_str() .ok_or("catalog field has no type")? .to_owned(); Ok((name, field_type)) }) .collect() } #[allow(clippy::too_many_lines)] pub fn validate_packet_api_catalog(root: &Path, protocol: &PacketProtocol) -> Result<(), String> { let path = root.join(PUBLIC_API_PATH); let bytes = fs::read(&path).map_err(|error| format!("{}: {error}", path.display()))?; let catalog: serde_json::Value = serde_json::from_slice(&bytes).map_err(|error| format!("{}: {error}", path.display()))?; let assembly = catalog["assemblies"] .as_array() .and_then(|assemblies| { assemblies .iter() .find(|assembly| assembly["identity"]["name"] == "LibreMetaverse") }) .ok_or("public API catalog has no LibreMetaverse assembly")?; let types = assembly["types"] .as_array() .ok_or("LibreMetaverse catalog has no types array")?; let types_by_id: BTreeMap<&str, &serde_json::Value> = types .iter() .map(|item| { item["doc_id"] .as_str() .map(|id| (id, item)) .ok_or("catalog type has no doc_id") }) .collect::>()?; let expected_packet_ids: BTreeSet = protocol .packets .iter() .map(|packet| format!("T:LibreMetaverse.Packets.{}Packet", packet.name)) .collect(); let expected_block_ids: BTreeSet = protocol .packets .iter() .flat_map(|packet| { packet.blocks.iter().map(move |block| { format!( "T:LibreMetaverse.Packets.{}Packet.{}Block", packet.name, block.name ) }) }) .collect(); let actual_packet_ids: BTreeSet = types .iter() .filter(|item| item["base_type"] == "LibreMetaverse.Packets.Packet") .filter_map(|item| item["doc_id"].as_str().map(str::to_owned)) .collect(); let actual_block_ids: BTreeSet = types .iter() .filter(|item| item["base_type"] == "LibreMetaverse.Packets.PacketBlock") .filter_map(|item| item["doc_id"].as_str().map(str::to_owned)) .collect(); if expected_packet_ids != actual_packet_ids { return Err(format!( "packet API catalog mismatch: missing={:?}, stale={:?}", expected_packet_ids .difference(&actual_packet_ids) .collect::>(), actual_packet_ids .difference(&expected_packet_ids) .collect::>() )); } if expected_block_ids != actual_block_ids { return Err(format!( "packet block API catalog mismatch: missing={:?}, stale={:?}", expected_block_ids .difference(&actual_block_ids) .collect::>(), actual_block_ids .difference(&expected_block_ids) .collect::>() )); } for packet in &protocol.packets { let packet_id = format!("T:LibreMetaverse.Packets.{}Packet", packet.name); let actual = catalog_fields(types_by_id[packet_id.as_str()])?; let expected = packet .blocks .iter() .map(|block| { let name = if block.name == "Header" { "_Header".to_owned() } else { block.name.clone() }; let array = if block.repetition == BlockRepetition::Single { "" } else { "[]" }; ( name, format!( "LibreMetaverse.Packets.{}Packet.{}Block{array}", packet.name, block.name ), ) }) .collect::>(); if expected != actual { return Err(format!( "{packet_id} field mismatch: expected={expected:?}, actual={actual:?}" )); } for block in &packet.blocks { let block_id = format!( "T:LibreMetaverse.Packets.{}Packet.{}Block", packet.name, block.name ); let actual = catalog_fields(types_by_id[block_id.as_str()])?; let expected = block .fields .iter() .map(|field| (field.name.clone(), csharp_field_type(field.kind).to_owned())) .collect::>(); if expected != actual { return Err(format!( "{block_id} field mismatch: expected={expected:?}, actual={actual:?}" )); } } } let packet_type = types_by_id .get("T:LibreMetaverse.Packets.PacketType") .ok_or("public API catalog has no PacketType")?; let actual_values = packet_type["members"] .as_array() .ok_or("PacketType has no members array")? .iter() .filter(|member| member["kind"] == "enum_value") .map(|member| { let name = member["name"] .as_str() .ok_or("PacketType value has no name")? .to_owned(); let value = member["value"]["value"] .as_str() .ok_or("PacketType value has no numeric value")? .parse::() .map_err(|_| "PacketType value is not a u32")?; Ok::<_, &'static str>((name, value)) }) .collect::, _>>()?; let mut expected_values = BTreeMap::from([("Default".to_owned(), 0)]); expected_values.extend( protocol .packets .iter() .map(|packet| (packet.name.clone(), packet_type_value(packet))), ); if expected_values != actual_values { return Err(format!( "PacketType enum mismatch: missing/stale values = {}/{}", expected_values .keys() .filter(|name| !actual_values.contains_key(*name)) .count(), actual_values .keys() .filter(|name| !expected_values.contains_key(*name)) .count() )); } Ok(()) } pub fn packet_catalog_bytes(root: &Path) -> Result, String> { let inventory = load_inventory(root)?; verify_inputs(root, &inventory)?; let input = inventory .inputs .iter() .find(|input| input.id == "message_template") .ok_or("source inventory has no message_template input")?; let generator = inventory .generators .iter() .find(|generator| generator.id == "packets") .ok_or("source inventory has no packets generator")?; let path = root.join(&input.vendored_path); let bytes = fs::read(&path).map_err(|error| format!("{}: {error}", path.display()))?; let text = normalize_text(&input.vendored_path, &bytes).map_err(|error| error.to_string())?; let protocol = parse_packet_template(&input.vendored_path, &text).map_err(|diagnostics| { diagnostics .into_iter() .map(|item| item.to_string()) .collect::>() .join("\n") })?; validate_packet_api_catalog(root, &protocol)?; let packets = ordered_packets(&protocol); let mut body = String::new(); let _ = writeln!( body, "pub const GOLDEN_GENERATOR_SOURCE: &str = {:?};", generator.reference_source ); let _ = writeln!( body, "pub const GOLDEN_GENERATOR_SHA256: &str = {:?};", generator.sha256 ); let _ = writeln!( body, "pub const GOLDEN_GENERATOR_LICENSE: &str = {:?};\n", generator.license ); append_descriptor_types(&mut body); append_packet_type(&mut body, &packets); append_descriptors(&mut body, &packets); append_dispatch(&mut body, &packets); append_block_impls(&mut body, &packets); append_packet_impls(&mut body, &packets); let syntax = syn::parse_file(&body) .map_err(|error| format!("generated packet Rust is invalid: {error}"))?; let formatted = prettyplease::unparse(&syntax); Ok(generated_rust("packets", &[input], &formatted)) } pub fn load_inventory(root: &Path) -> Result { let path = root.join(INVENTORY_PATH); let bytes = fs::read(&path).map_err(|error| format!("{}: {error}", path.display()))?; let inventory: Inventory = serde_json::from_slice(&bytes).map_err(|error| format!("{}: {error}", path.display()))?; validate_inventory(&inventory)?; Ok(inventory) } fn validate_inventory(inventory: &Inventory) -> Result<(), String> { if inventory.schema != 1 || inventory.upstream_commit.len() != 40 { return Err("unsupported or malformed source inventory header".to_owned()); } let mut input_ids = BTreeSet::new(); for input in &inventory.inputs { validate_relative(&input.reference_path)?; validate_relative(&input.vendored_path)?; validate_hash(&input.sha256)?; if input.license != "BSD-3-Clause" || !input_ids.insert(&input.id) { return Err(format!("invalid or duplicate input {}", input.id)); } } let mut generator_ids = BTreeSet::new(); for generator in &inventory.generators { validate_relative(&generator.reference_source)?; validate_hash(&generator.sha256)?; if generator.license != "BSD-3-Clause" || !generator_ids.insert(&generator.id) { return Err(format!("invalid or duplicate generator {}", generator.id)); } for input in &generator.inputs { if !input_ids.contains(input) { return Err(format!( "generator {} references unknown input {input}", generator.id )); } } } Ok(()) } fn validate_relative(value: &str) -> Result<(), String> { let path = Path::new(value); if path.is_absolute() || path .components() .any(|part| !matches!(part, Component::Normal(_))) { return Err(format!("unsafe inventory path {value}")); } Ok(()) } fn validate_hash(value: &str) -> Result<(), String> { if value.len() == 64 && value.bytes().all(|byte| byte.is_ascii_hexdigit()) { Ok(()) } else { Err(format!("invalid SHA-256 {value}")) } } fn sha256(bytes: &[u8]) -> String { const HEX: &[u8; 16] = b"0123456789abcdef"; let mut output = String::with_capacity(64); for byte in Sha256::digest(bytes) { output.push(char::from(HEX[usize::from(byte >> 4)])); output.push(char::from(HEX[usize::from(byte & 0x0f)])); } output } pub fn verify_inputs(root: &Path, inventory: &Inventory) -> Result<(), String> { for input in &inventory.inputs { let path = root.join(&input.vendored_path); let bytes = fs::read(&path).map_err(|error| format!("{}: {error}", path.display()))?; let actual = sha256(&bytes); if actual != input.sha256 { return Err(format!( "{}: SHA-256 mismatch: expected {}, found {actual}", path.display(), input.sha256 )); } normalize_text(&input.vendored_path, &bytes).map_err(|error| error.to_string())?; } Ok(()) } pub fn vendor_inputs(root: &Path, reference: &Path) -> Result<(), String> { let inventory = load_inventory(root)?; for generator in &inventory.generators { let source = reference.join(&generator.reference_source); let bytes = fs::read(&source).map_err(|error| format!("{}: {error}", source.display()))?; let actual = sha256(&bytes); if actual != generator.sha256 { return Err(format!( "{}: pinned generator SHA-256 mismatch: expected {}, found {actual}", source.display(), generator.sha256 )); } } let mut snapshots = Vec::with_capacity(inventory.inputs.len()); for input in &inventory.inputs { let source = reference.join(&input.reference_path); let bytes = fs::read(&source).map_err(|error| format!("{}: {error}", source.display()))?; let actual = sha256(&bytes); if actual != input.sha256 { return Err(format!( "{}: pinned SHA-256 mismatch: expected {}, found {actual}", source.display(), input.sha256 )); } snapshots.push((root.join(&input.vendored_path), bytes)); } for (target, bytes) in snapshots { fs::create_dir_all(target.parent().ok_or("vendored input has no parent")?) .map_err(|error| error.to_string())?; fs::write(&target, bytes).map_err(|error| format!("{}: {error}", target.display()))?; } Ok(()) } pub fn source_manifest_bytes(root: &Path) -> Result, String> { let inventory = load_inventory(root)?; verify_inputs(root, &inventory)?; let mut inputs: Vec<&InputSpec> = inventory.inputs.iter().collect(); inputs.sort_by_key(|input| &input.id); let mut generators: Vec<&GeneratorSpec> = inventory.generators.iter().collect(); generators.sort_by_key(|generator| &generator.id); let mut body = String::new(); let _ = writeln!( body, "pub const UPSTREAM_COMMIT: &str = {:?};", inventory.upstream_commit ); let _ = writeln!( body, "pub const UPSTREAM_REPOSITORY: &str = {:?};", inventory.upstream_repository ); body.push_str("pub const SOURCES: &[(&str, &str, &str, &str)] = &[\n"); for input in &inputs { let _ = writeln!( body, " ({:?}, {:?}, {:?}, {:?}),", input.id, input.vendored_path, input.sha256, input.license ); } body.push_str("];\n"); body.push_str("pub const GENERATORS: &[(&str, &str, &str, &str)] = &[\n"); for generator in generators { let _ = writeln!( body, " ({:?}, {:?}, {:?}, {:?}),", generator.id, generator.reference_source, generator.sha256, generator.license ); } body.push_str("];\n"); Ok(generated_rust("source-manifest", &inputs, &body)) } pub fn regenerate(root: &Path, check: bool) -> Result<(), String> { let outputs = [ (root.join(MANIFEST_OUTPUT), source_manifest_bytes(root)?), (root.join(PACKET_OUTPUT), packet_catalog_bytes(root)?), ]; if check { for (path, expected) in outputs { let actual = fs::read(&path).map_err(|error| format!("{}: {error}", path.display()))?; if actual != expected { return Err(format!( "{} is stale; run the regeneration command", path.display() )); } } } else { for (path, expected) in outputs { fs::create_dir_all(path.parent().ok_or("generated output has no parent")?) .map_err(|error| error.to_string())?; fs::write(&path, expected).map_err(|error| format!("{}: {error}", path.display()))?; } } Ok(()) } #[cfg(test)] mod tests { use super::*; #[test] fn checked_in_inputs_and_manifest_are_current() { let root = workspace_root(); let first = source_manifest_bytes(&root).expect("first deterministic generation"); let second = source_manifest_bytes(&root).expect("second deterministic generation"); assert_eq!(first, second); let first_packets = packet_catalog_bytes(&root).expect("first packet generation"); let second_packets = packet_catalog_bytes(&root).expect("second packet generation"); assert_eq!(first_packets, second_packets); } #[test] fn text_normalization_and_diagnostics_are_stable() { assert_eq!( normalize_text("a\\b.xml", b"\xef\xbb\xbfa\r\nb\r").unwrap(), "a\nb\n" ); let error = normalize_text("a\\b.xml", b"one\n\0two").unwrap_err(); assert_eq!( error.to_string(), "a/b.xml:2:1: error[CG002]: input contains a NUL byte" ); } #[test] fn inventory_rejects_unsafe_paths_hashes_and_unknown_inputs() { let root = workspace_root(); let mut inventory = load_inventory(&root).expect("pinned inventory"); inventory.inputs[0].vendored_path = "../escape".to_owned(); assert!(validate_inventory(&inventory).is_err()); let mut inventory = load_inventory(&root).expect("pinned inventory"); inventory.generators[0].sha256 = "not-a-hash".to_owned(); assert!(validate_inventory(&inventory).is_err()); let mut inventory = load_inventory(&root).expect("pinned inventory"); inventory.generators[0].inputs.push("missing".to_owned()); assert!(validate_inventory(&inventory).is_err()); } #[test] fn packet_template_parser_preserves_ids_order_and_repetition() { let text = "version 2.0\n{ Alpha Fixed 0xFFFFFFFB Trusted Zerocoded UDPDeprecated\n { One Single { Value U32 } }\n { Many Multiple 2 { Data Fixed 4 } }\n { Rest Variable { Name Variable 2 } }\n}\n{ Beta High 7 NotTrusted Unencoded }\n"; let protocol = parse_packet_template("fixture.msg", text).expect("valid protocol"); assert_eq!(protocol.packets.len(), 2); let alpha = &protocol.packets[0]; assert_eq!(alpha.id, 0xfffb); assert_eq!(alpha.frequency, PacketFrequency::Low); assert!(alpha.trusted && alpha.zerocoded); assert_eq!(alpha.flags, ["UDPDeprecated"]); assert_eq!(alpha.blocks[1].repetition, BlockRepetition::Multiple(2)); assert_eq!(alpha.blocks[2].repetition, BlockRepetition::Variable); assert_eq!(alpha.blocks[2].fields[0].count, 2); assert_eq!(protocol.packets[1].frequency, PacketFrequency::High); } #[test] fn packet_template_diagnostics_are_source_located_and_stable() { let text = "{ Bad Nope xx Maybe Compressed\n { Data Multiple 0 { Value Mystery 3 } }\n}\n"; let diagnostics = parse_packet_template("data\\bad.msg", text).unwrap_err(); let rendered = diagnostics .iter() .map(ToString::to_string) .collect::>(); assert_eq!( rendered, [ "data/bad.msg:1:7: error[PG030]: unknown packet frequency \"Nope\"", "data/bad.msg:1:12: error[PG031]: packet ID must be a decimal or hexadecimal unsigned integer", "data/bad.msg:1:15: error[PG032]: unknown packet trust marker \"Maybe\"", "data/bad.msg:1:21: error[PG033]: unknown packet coding marker \"Compressed\"", "data/bad.msg:2:18: error[PG014]: Multiple block count must be a positive decimal integer", "data/bad.msg:2:28: error[PG023]: unknown field type \"Mystery\"", ] ); } }