hugo/MetaCrate
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity
Files
b413cd6f4de7b0befb0155b8e2d30854dadaec4d
MetaCrate/tools/codegen/src/lib.rs
Chili Palmer 167f469fc2
Some checks failed
Native code generation / deterministic (push) Failing after 6m6s
Details
Imaging and meshing gate / native (push) Failing after 17s
Details
JPEG 2000 feature / linux (push) Failing after 59s
Details
Skia feature / linux (push) Failing after 1m57s
Details
Complete native codegen milestone gate (#50)
2026-08-09 10:06:11 +00:00

2418 lines
102 KiB
Rust
Raw Blame History

//! Deterministic framework shared by the `LibreMetaverse` data generators.
#![allow(clippy::missing_errors_doc)] // The CLI renders the complete error at its boundary.
#![allow(clippy::must_use_candidate)] // Generator helpers are also exercised for validation.
use serde::Deserialize;
use sha2::{Digest, Sha256};
use std::collections::{BTreeMap, BTreeSet};
use std::fmt;
use std::fmt::Write as _;
use std::fs;
use std::path::{Component, Path, PathBuf};
mod avatar_catalog;
mod xml_catalog;
pub const INVENTORY_PATH: &str = "codegen/sources.json";
pub const MANIFEST_OUTPUT: &str = "codegen/generated/source_manifest.rs";
pub const PACKET_OUTPUT: &str = "crates/libremetaverse/src/packet_catalog.rs";
pub const VISUAL_OUTPUT: &str = "crates/libremetaverse/src/visual_catalog.rs";
pub const FOLIAGE_OUTPUT: &str = "crates/libremetaverse/src/foliage_catalog.rs";
pub const SKELETON_OUTPUT: &str = "crates/libremetaverse/src/skeleton_catalog.rs";
pub const ATTENTION_OUTPUT: &str = "crates/libremetaverse/src/attention_catalog.rs";
pub const GENEPOOL_OUTPUT: &str = "crates/libremetaverse/src/genepool_catalog.rs";
pub const PUBLIC_API_PATH: &str = "api/public-api.json";
pub const GENERATED_OUTPUT_PATHS: [&str; 7] = [
MANIFEST_OUTPUT,
PACKET_OUTPUT,
VISUAL_OUTPUT,
FOLIAGE_OUTPUT,
SKELETON_OUTPUT,
ATTENTION_OUTPUT,
GENEPOOL_OUTPUT,
];
const RUST_KEYWORDS: &[&str] = &[
"as", "break", "const", "continue", "crate", "else", "enum", "extern", "false", "fn", "for",
"if", "impl", "in", "let", "loop", "match", "mod", "move", "mut", "pub", "ref", "return",
"self", "Self", "static", "struct", "super", "trait", "true", "type", "unsafe", "use", "where",
"while", "async", "await", "dyn", "abstract", "become", "box", "do", "final", "macro",
"override", "priv", "typeof", "unsized", "virtual", "yield", "try", "from",
];
#[derive(Clone, Debug, Deserialize, Eq, PartialEq)]
pub struct Inventory {
pub schema: u32,
pub upstream_commit: String,
pub upstream_repository: String,
pub generators: Vec<GeneratorSpec>,
pub inputs: Vec<InputSpec>,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq)]
pub struct GeneratorSpec {
pub id: String,
pub reference_source: String,
pub sha256: String,
pub license: String,
pub inputs: Vec<String>,
#[serde(default)]
pub note: String,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq)]
pub struct InputSpec {
pub id: String,
pub reference_path: String,
pub vendored_path: String,
pub sha256: String,
pub license: String,
pub format: String,
}
#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)]
pub enum Severity {
Error,
Warning,
}
#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)]
pub struct Diagnostic {
pub code: &'static str,
pub severity: Severity,
pub path: String,
pub line: usize,
pub column: usize,
pub message: String,
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum PacketFrequency {
Low,
Medium,
High,
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum BlockRepetition {
Single,
Multiple(usize),
Variable,
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum FieldKind {
U8,
U16,
U32,
U64,
S8,
S16,
S32,
F32,
F64,
LlUuid,
Bool,
LlVector3,
LlVector3d,
LlVector4,
LlQuaternion,
IpAddr,
IpPort,
Variable,
Fixed,
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct PacketField {
pub name: String,
pub kind: FieldKind,
pub count: usize,
pub line: usize,
pub column: usize,
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct PacketBlock {
pub name: String,
pub repetition: BlockRepetition,
pub fields: Vec<PacketField>,
pub line: usize,
pub column: usize,
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct PacketDefinition {
pub id: u16,
pub name: String,
pub frequency: PacketFrequency,
pub trusted: bool,
pub zerocoded: bool,
pub flags: Vec<String>,
pub blocks: Vec<PacketBlock>,
pub line: usize,
pub column: usize,
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct PacketProtocol {
pub packets: Vec<PacketDefinition>,
}
#[derive(Clone, Debug, Eq, PartialEq)]
struct Token {
text: String,
line: usize,
column: usize,
}
impl fmt::Display for Diagnostic {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
let severity = match self.severity {
Severity::Error => "error",
Severity::Warning => "warning",
};
write!(
formatter,
"{}:{}:{}: {severity}[{}]: {}",
self.path, self.line, self.column, self.code, self.message
)
}
}
pub fn workspace_root() -> PathBuf {
Path::new(env!("CARGO_MANIFEST_DIR")).join("../..")
}
pub fn normalize_text(path: &str, bytes: &[u8]) -> Result<String, Diagnostic> {
let bytes = bytes.strip_prefix(&[0xef, 0xbb, 0xbf]).unwrap_or(bytes);
let text = std::str::from_utf8(bytes).map_err(|error| {
let valid = &bytes[..error.valid_up_to()];
let (line, column) = line_column(valid);
Diagnostic {
code: "CG001",
severity: Severity::Error,
path: path.replace('\\', "/"),
line,
column,
message: "input is not valid UTF-8".to_owned(),
}
})?;
if let Some(position) = text.as_bytes().iter().position(|byte| *byte == 0) {
let prefix = &text.as_bytes()[..position];
let (line, column) = line_column(prefix);
return Err(Diagnostic {
code: "CG002",
severity: Severity::Error,
path: path.replace('\\', "/"),
line,
column,
message: "input contains a NUL byte".to_owned(),
});
}
Ok(text.replace("\r\n", "\n").replace('\r', "\n"))
}
fn line_column(bytes: &[u8]) -> (usize, usize) {
let mut line = 1;
let mut column = 1;
for byte in bytes {
if *byte == b'\n' {
line += 1;
column = 1;
} else {
column += 1;
}
}
(line, column)
}
fn packet_diagnostic(
path: &str,
token: Option<&Token>,
code: &'static str,
message: impl Into<String>,
) -> Diagnostic {
Diagnostic {
code,
severity: Severity::Error,
path: path.replace('\\', "/"),
line: token.map_or(1, |token| token.line),
column: token.map_or(1, |token| token.column),
message: message.into(),
}
}
fn tokenize_packet_template(path: &str, text: &str) -> Result<Vec<Token>, Vec<Diagnostic>> {
let mut tokens = Vec::new();
let mut characters = text.char_indices().peekable();
let mut line = 1usize;
let mut column = 1usize;
while let Some((start, character)) = characters.next() {
if character == '\n' {
line += 1;
column = 1;
continue;
}
if character.is_whitespace() {
column += 1;
continue;
}
if character == '/' && characters.peek().is_some_and(|(_, next)| *next == '/') {
characters.next();
column += 2;
for (_, next) in characters.by_ref() {
if next == '\n' {
line += 1;
column = 1;
break;
}
column += 1;
}
continue;
}
if matches!(character, '{' | '}') {
tokens.push(Token {
text: character.to_string(),
line,
column,
});
column += 1;
continue;
}
let token_line = line;
let token_column = column;
let mut end = start + character.len_utf8();
column += 1;
while let Some((offset, next)) = characters.peek().copied() {
if next.is_whitespace() || matches!(next, '{' | '}') {
break;
}
if next == '/' {
let mut lookahead = characters.clone();
lookahead.next();
if lookahead
.peek()
.is_some_and(|(_, following)| *following == '/')
{
break;
}
}
characters.next();
end = offset + next.len_utf8();
column += 1;
}
let value = &text[start..end];
if !value.is_ascii() {
return Err(vec![packet_diagnostic(
path,
Some(&Token {
text: value.to_owned(),
line: token_line,
column: token_column,
}),
"PG002",
"protocol identifiers and values must be ASCII",
)]);
}
tokens.push(Token {
text: value.to_owned(),
line: token_line,
column: token_column,
});
}
Ok(tokens)
}
struct PacketParser<'a> {
path: &'a str,
tokens: Vec<Token>,
position: usize,
diagnostics: Vec<Diagnostic>,
}
impl PacketParser<'_> {
fn current(&self) -> Option<&Token> {
self.tokens.get(self.position)
}
fn advance(&mut self) -> Option<Token> {
let token = self.tokens.get(self.position).cloned();
self.position += usize::from(token.is_some());
token
}
fn expect(&mut self, value: &str, code: &'static str, context: &str) -> bool {
if self.current().is_some_and(|token| token.text == value) {
self.position += 1;
true
} else {
self.diagnostics.push(packet_diagnostic(
self.path,
self.current(),
code,
format!("expected {value:?} {context}"),
));
false
}
}
fn atom(&mut self, code: &'static str, context: &str) -> Option<Token> {
match self.current() {
Some(token) if !matches!(token.text.as_str(), "{" | "}") => self.advance(),
_ => {
self.diagnostics.push(packet_diagnostic(
self.path,
self.current(),
code,
format!("expected {context}"),
));
None
}
}
}
fn recover_to_close(&mut self) {
let mut depth = 0usize;
while let Some(token) = self.advance() {
match token.text.as_str() {
"{" => depth += 1,
"}" if depth == 0 => break,
"}" => depth -= 1,
_ => {}
}
}
}
fn parse_field(&mut self) -> Option<PacketField> {
if !self.expect("{", "PG020", "to begin a field") {
return None;
}
let name = self.atom("PG021", "a field name")?;
let kind_token = self.atom("PG022", "a field type")?;
let (kind, known_kind) = match kind_token.text.as_str() {
"U8" => (FieldKind::U8, true),
"U16" => (FieldKind::U16, true),
"U32" => (FieldKind::U32, true),
"U64" => (FieldKind::U64, true),
"S8" => (FieldKind::S8, true),
"S16" => (FieldKind::S16, true),
"S32" => (FieldKind::S32, true),
"F32" => (FieldKind::F32, true),
"F64" => (FieldKind::F64, true),
"LLUUID" => (FieldKind::LlUuid, true),
"BOOL" => (FieldKind::Bool, true),
"LLVector3" => (FieldKind::LlVector3, true),
"LLVector3d" => (FieldKind::LlVector3d, true),
"LLVector4" => (FieldKind::LlVector4, true),
"LLQuaternion" => (FieldKind::LlQuaternion, true),
"IPADDR" => (FieldKind::IpAddr, true),
"IPPORT" => (FieldKind::IpPort, true),
"Variable" => (FieldKind::Variable, true),
"Fixed" => (FieldKind::Fixed, true),
other => {
self.diagnostics.push(packet_diagnostic(
self.path,
Some(&kind_token),
"PG023",
format!("unknown field type {other:?}"),
));
(FieldKind::Variable, false)
}
};
let count = if self.current().is_some_and(|token| token.text != "}") {
let count_token = self.atom("PG024", "a field width")?;
match count_token.text.parse::<usize>() {
Ok(value) if value > 0 => value,
_ => {
self.diagnostics.push(packet_diagnostic(
self.path,
Some(&count_token),
"PG025",
"field width must be a positive decimal integer",
));
1
}
}
} else {
1
};
if known_kind && matches!(kind, FieldKind::Variable) && !matches!(count, 1 | 2) {
self.diagnostics.push(packet_diagnostic(
self.path,
Some(&name),
"PG026",
"Variable fields require a one- or two-byte length prefix",
));
}
if known_kind && !matches!(kind, FieldKind::Variable | FieldKind::Fixed) && count != 1 {
self.diagnostics.push(packet_diagnostic(
self.path,
Some(&name),
"PG027",
"only Variable and Fixed fields may specify a width",
));
}
if !self.expect("}", "PG028", "after a field") {
self.recover_to_close();
}
Some(PacketField {
name: name.text,
kind,
count,
line: name.line,
column: name.column,
})
}
fn parse_block(&mut self) -> Option<PacketBlock> {
if !self.expect("{", "PG010", "to begin a block") {
return None;
}
let name = self.atom("PG011", "a block name")?;
let repetition_token = self.atom("PG012", "a block repetition")?;
let repetition = match repetition_token.text.as_str() {
"Single" => BlockRepetition::Single,
"Variable" => BlockRepetition::Variable,
"Multiple" => {
let count = self.atom("PG013", "a Multiple block count")?;
match count.text.parse::<usize>() {
Ok(value) if value > 0 => BlockRepetition::Multiple(value),
_ => {
self.diagnostics.push(packet_diagnostic(
self.path,
Some(&count),
"PG014",
"Multiple block count must be a positive decimal integer",
));
BlockRepetition::Multiple(1)
}
}
}
other => {
self.diagnostics.push(packet_diagnostic(
self.path,
Some(&repetition_token),
"PG015",
format!("unknown block repetition {other:?}"),
));
BlockRepetition::Single
}
};
let mut fields = Vec::new();
while self.current().is_some_and(|token| token.text == "{") {
if let Some(field) = self.parse_field() {
fields.push(field);
}
}
if !self.expect("}", "PG016", "after a block") {
self.recover_to_close();
}
let mut names = BTreeSet::new();
for field in &fields {
if !names.insert(&field.name) {
self.diagnostics.push(packet_diagnostic(
self.path,
Some(&Token {
text: field.name.clone(),
line: field.line,
column: field.column,
}),
"PG017",
format!("duplicate field {:?} in block {:?}", field.name, name.text),
));
}
}
Some(PacketBlock {
name: name.text,
repetition,
fields,
line: name.line,
column: name.column,
})
}
#[allow(clippy::too_many_lines)]
fn parse_packet(&mut self) -> Option<PacketDefinition> {
if !self.expect("{", "PG003", "to begin a packet") {
return None;
}
let name = self.atom("PG004", "a packet name")?;
let frequency_token = self.atom("PG005", "a packet frequency")?;
let id_token = self.atom("PG006", "a packet ID")?;
let trust_token = self.atom("PG007", "a packet trust marker")?;
let coding_token = self.atom("PG008", "a packet coding marker")?;
let frequency = match frequency_token.text.as_str() {
"Fixed" | "Low" => PacketFrequency::Low,
"Medium" | "Mid" => PacketFrequency::Medium,
"High" => PacketFrequency::High,
other => {
self.diagnostics.push(packet_diagnostic(
self.path,
Some(&frequency_token),
"PG030",
format!("unknown packet frequency {other:?}"),
));
PacketFrequency::Low
}
};
let parsed_id = if let Some(hex) = id_token
.text
.strip_prefix("0x")
.or_else(|| id_token.text.strip_prefix("0X"))
{
u32::from_str_radix(hex, 16)
} else {
id_token.text.parse::<u32>()
};
let id = if let Ok(value) = parsed_id {
(value & 0xffff) as u16
} else {
self.diagnostics.push(packet_diagnostic(
self.path,
Some(&id_token),
"PG031",
"packet ID must be a decimal or hexadecimal unsigned integer",
));
0
};
let trusted = match trust_token.text.as_str() {
"Trusted" => true,
"NotTrusted" => false,
other => {
self.diagnostics.push(packet_diagnostic(
self.path,
Some(&trust_token),
"PG032",
format!("unknown packet trust marker {other:?}"),
));
false
}
};
let zerocoded = match coding_token.text.as_str() {
"Zerocoded" => true,
"Unencoded" => false,
other => {
self.diagnostics.push(packet_diagnostic(
self.path,
Some(&coding_token),
"PG033",
format!("unknown packet coding marker {other:?}"),
));
false
}
};
let mut flags = Vec::new();
while self
.current()
.is_some_and(|token| !matches!(token.text.as_str(), "{" | "}"))
{
if let Some(flag) = self.advance() {
flags.push(flag.text);
}
}
let mut blocks = Vec::new();
while self.current().is_some_and(|token| token.text == "{") {
if let Some(block) = self.parse_block() {
blocks.push(block);
}
}
if !self.expect("}", "PG009", "after a packet") {
self.recover_to_close();
}
let mut block_names = BTreeSet::new();
for block in &blocks {
if !block_names.insert(&block.name) {
self.diagnostics.push(packet_diagnostic(
self.path,
Some(&Token {
text: block.name.clone(),
line: block.line,
column: block.column,
}),
"PG034",
format!("duplicate block {:?} in packet {:?}", block.name, name.text),
));
}
}
Some(PacketDefinition {
id,
name: name.text,
frequency,
trusted,
zerocoded,
flags,
blocks,
line: name.line,
column: name.column,
})
}
}
pub fn parse_packet_template(path: &str, text: &str) -> Result<PacketProtocol, Vec<Diagnostic>> {
let tokens = tokenize_packet_template(path, text)?;
let mut parser = PacketParser {
path,
tokens,
position: 0,
diagnostics: Vec::new(),
};
while parser.current().is_some_and(|token| token.text != "{") {
parser.position += 1;
}
let mut packets = Vec::new();
while parser.current().is_some() {
if parser.current().is_some_and(|token| token.text != "{") {
parser.diagnostics.push(packet_diagnostic(
path,
parser.current(),
"PG035",
"unexpected token outside a packet",
));
parser.position += 1;
continue;
}
if let Some(packet) = parser.parse_packet() {
packets.push(packet);
} else {
parser.recover_to_close();
}
}
let mut names = BTreeMap::new();
let mut ids = BTreeMap::new();
for packet in &packets {
if let Some((line, column)) =
names.insert(packet.name.clone(), (packet.line, packet.column))
{
parser.diagnostics.push(packet_diagnostic(
path,
Some(&Token {
text: packet.name.clone(),
line: packet.line,
column: packet.column,
}),
"PG036",
format!(
"duplicate packet {:?}; first declared at {line}:{column}",
packet.name
),
));
}
let key = (packet.frequency as u8, packet.id);
if let Some((other, line, column)) =
ids.insert(key, (&packet.name, packet.line, packet.column))
{
parser.diagnostics.push(packet_diagnostic(
path,
Some(&Token {
text: packet.name.clone(),
line: packet.line,
column: packet.column,
}),
"PG037",
format!(
"duplicate {:?} packet ID {}; {:?} was declared at {line}:{column}",
packet.frequency, packet.id, other
),
));
}
}
parser.diagnostics.sort_by(|left, right| {
(&left.path, left.line, left.column, left.code, &left.message).cmp(&(
&right.path,
right.line,
right.column,
right.code,
&right.message,
))
});
if parser.diagnostics.is_empty() {
Ok(PacketProtocol { packets })
} else {
Err(parser.diagnostics)
}
}
pub fn generated_rust(generator: &str, sources: &[&InputSpec], body: &str) -> Vec<u8> {
let mut output = String::new();
output.push_str("// @generated by libremetaverse-codegen; DO NOT EDIT.\n");
output.push_str("// Regenerate: cargo run -p libremetaverse-codegen -- generate\n");
let _ = writeln!(output, "// Generator: {generator}");
for source in sources {
let _ = writeln!(
output,
"// Source: {} sha256={} license={}",
source.vendored_path, source.sha256, source.license
);
}
output.push('\n');
output.push_str(body.trim_end());
output.push('\n');
output.into_bytes()
}
fn rust_snake(name: &str) -> String {
let characters: Vec<char> = name.trim_start_matches('@').chars().collect();
let mut output = String::new();
for (index, character) in characters.iter().copied().enumerate() {
let previous = index
.checked_sub(1)
.and_then(|i| characters.get(i))
.copied();
let next = characters.get(index + 1).copied();
if character.is_ascii_uppercase()
&& !output.is_empty()
&& (previous.is_some_and(|value| value.is_ascii_lowercase() || value.is_ascii_digit())
|| (previous.is_some_and(|value| value.is_ascii_uppercase())
&& next.is_some_and(|value| value.is_ascii_lowercase())))
{
output.push('_');
}
if character.is_ascii_alphanumeric() || character == '_' {
output.push(character.to_ascii_lowercase());
} else {
output.push('_');
}
}
while output.contains("__") {
output = output.replace("__", "_");
}
let mut output = output.trim_matches('_').to_owned();
if output.is_empty() {
output.push_str("item");
}
if output.starts_with(|character: char| character.is_ascii_digit())
|| RUST_KEYWORDS.contains(&output.as_str())
{
output.push('_');
}
output
}
const fn field_fixed_length(field: &PacketField) -> usize {
match field.kind {
FieldKind::Bool | FieldKind::U8 | FieldKind::S8 => 1,
FieldKind::U16 | FieldKind::S16 | FieldKind::IpPort => 2,
FieldKind::U32 | FieldKind::S32 | FieldKind::F32 | FieldKind::IpAddr => 4,
FieldKind::U64 | FieldKind::F64 => 8,
FieldKind::LlVector3 | FieldKind::LlQuaternion => 12,
FieldKind::LlUuid | FieldKind::LlVector4 => 16,
FieldKind::LlVector3d => 24,
FieldKind::Fixed => field.count,
FieldKind::Variable => 0,
}
}
fn field_kind_name(kind: FieldKind) -> &'static str {
match kind {
FieldKind::U8 => "U8",
FieldKind::U16 => "U16",
FieldKind::U32 => "U32",
FieldKind::U64 => "U64",
FieldKind::S8 => "S8",
FieldKind::S16 => "S16",
FieldKind::S32 => "S32",
FieldKind::F32 => "F32",
FieldKind::F64 => "F64",
FieldKind::LlUuid => "LlUuid",
FieldKind::Bool => "Bool",
FieldKind::LlVector3 => "LlVector3",
FieldKind::LlVector3d => "LlVector3d",
FieldKind::LlVector4 => "LlVector4",
FieldKind::LlQuaternion => "LlQuaternion",
FieldKind::IpAddr => "IpAddr",
FieldKind::IpPort => "IpPort",
FieldKind::Variable => "Variable",
FieldKind::Fixed => "Fixed",
}
}
fn frequency_name(frequency: PacketFrequency) -> &'static str {
match frequency {
PacketFrequency::Low => "Low",
PacketFrequency::Medium => "Medium",
PacketFrequency::High => "High",
}
}
fn field_default(kind: FieldKind) -> &'static str {
match kind {
FieldKind::Bool => "false",
FieldKind::U8
| FieldKind::U16
| FieldKind::U32
| FieldKind::U64
| FieldKind::S8
| FieldKind::S16
| FieldKind::S32
| FieldKind::IpAddr
| FieldKind::IpPort => "0",
FieldKind::F32 => "0.0_f32",
FieldKind::F64 => "0.0_f64",
FieldKind::LlUuid => "libremetaverse_types::UUID::zero()",
FieldKind::LlVector3 => "libremetaverse_types::Vector3::zero()",
FieldKind::LlVector3d => "libremetaverse_types::Vector3d::zero()",
FieldKind::LlVector4 => "libremetaverse_types::Vector4::zero()",
FieldKind::LlQuaternion => {
"libremetaverse_types::Quaternion { x: 0.0, y: 0.0, z: 0.0, w: 0.0 }"
}
FieldKind::Variable | FieldKind::Fixed => "Vec::new()",
}
}
fn field_sample(field: &PacketField) -> String {
match field.kind {
FieldKind::Bool => "true".to_owned(),
FieldKind::U8 => "0xa5_u8".to_owned(),
FieldKind::U16 | FieldKind::IpPort => "0x1234_u16".to_owned(),
FieldKind::U32 | FieldKind::IpAddr => "0x1122_3344_u32".to_owned(),
FieldKind::U64 => "0x0102_0304_0506_0708_u64".to_owned(),
FieldKind::S8 => "-37_i8".to_owned(),
FieldKind::S16 => "-0x1234_i16".to_owned(),
FieldKind::S32 => "-0x1122_3344_i32".to_owned(),
FieldKind::F32 => "1.5_f32".to_owned(),
FieldKind::F64 => "-2.25_f64".to_owned(),
FieldKind::LlUuid => "libremetaverse_types::UUID::new_with_bytes_int32((0_u8..16).collect(), 0).expect(\"sample UUID\")".to_owned(),
FieldKind::LlVector3 => "libremetaverse_types::Vector3 { x: 1.25, y: -2.5, z: 3.75 }".to_owned(),
FieldKind::LlVector3d => "libremetaverse_types::Vector3d { x: 1.25, y: -2.5, z: 3.75 }".to_owned(),
FieldKind::LlVector4 => "libremetaverse_types::Vector4 { x: 1.25, y: -2.5, z: 3.75, w: -4.5 }".to_owned(),
FieldKind::LlQuaternion => "libremetaverse_types::Quaternion::identity()".to_owned(),
FieldKind::Variable => "vec![0_u8, 1, 2, 0, 0xff]".to_owned(),
FieldKind::Fixed => format!("vec![0xa5_u8; {}]", field.count),
}
}
fn field_decode_statement(field: &PacketField) -> String {
let name = rust_snake(&field.name);
let expression = match field.kind {
FieldKind::Bool => "reader.read_u8()? != 0".to_owned(),
FieldKind::U8 => "reader.read_u8()?".to_owned(),
FieldKind::U16 => "reader.read_u16_le()?".to_owned(),
FieldKind::U32 | FieldKind::IpAddr => "reader.read_u32_le()?".to_owned(),
FieldKind::U64 => "reader.read_u64_le()?".to_owned(),
FieldKind::S8 => "reader.read_i8()?".to_owned(),
FieldKind::S16 => "reader.read_i16_le()?".to_owned(),
FieldKind::S32 => "reader.read_i32_le()?".to_owned(),
FieldKind::F32 => "reader.read_f32_le()?".to_owned(),
FieldKind::F64 => "reader.read_f64_le()?".to_owned(),
FieldKind::LlUuid => "reader.read_uuid()?".to_owned(),
FieldKind::LlVector3 => "reader.read_vector3()?".to_owned(),
FieldKind::LlVector3d => "reader.read_vector3d()?".to_owned(),
FieldKind::LlVector4 => "reader.read_vector4()?".to_owned(),
FieldKind::LlQuaternion => "reader.read_quaternion()?".to_owned(),
FieldKind::IpPort => "reader.read_u16_be()?".to_owned(),
FieldKind::Variable => format!("reader.read_variable({})?", field.count),
FieldKind::Fixed => format!(
"reader.read_bytes({}, \"truncated fixed packet field\")?",
field.count
),
};
format!("self.{name} = {expression};")
}
fn field_encode_statement(field: &PacketField) -> String {
let name = rust_snake(&field.name);
match field.kind {
FieldKind::Bool => format!("writer.write_u8(u8::from(self.{name}))?;"),
FieldKind::U8 => format!("writer.write_u8(self.{name})?;"),
FieldKind::U16 => format!("writer.write_u16_le(self.{name})?;"),
FieldKind::U32 | FieldKind::IpAddr => {
format!("writer.write_u32_le(self.{name})?;")
}
FieldKind::U64 => format!("writer.write_u64_le(self.{name})?;"),
FieldKind::S8 => format!("writer.write_i8(self.{name})?;"),
FieldKind::S16 => format!("writer.write_i16_le(self.{name})?;"),
FieldKind::S32 => format!("writer.write_i32_le(self.{name})?;"),
FieldKind::F32 => format!("writer.write_f32_le(self.{name})?;"),
FieldKind::F64 => format!("writer.write_f64_le(self.{name})?;"),
FieldKind::LlUuid => format!("writer.write_uuid(&self.{name})?;"),
FieldKind::LlVector3 => format!("writer.write_vector3(&self.{name})?;"),
FieldKind::LlVector3d => format!("writer.write_vector3d(&self.{name})?;"),
FieldKind::LlVector4 => format!("writer.write_vector4(&self.{name})?;"),
FieldKind::LlQuaternion => format!("writer.write_quaternion(&self.{name})?;"),
FieldKind::IpPort => format!("writer.write_u16_be(self.{name})?;"),
FieldKind::Variable => {
format!("writer.write_variable(&self.{name}, {})?;", field.count)
}
FieldKind::Fixed => format!("writer.write_fixed(&self.{name}, {})?;", field.count),
}
}
fn field_to_osd_expression(field: &PacketField) -> String {
let name = rust_snake(&field.name);
match field.kind {
FieldKind::Bool => {
format!("libremetaverse_structured_data::OSD::Boolean(self.{name})")
}
FieldKind::U8 | FieldKind::U16 | FieldKind::IpPort | FieldKind::S8 | FieldKind::S16 => {
format!("libremetaverse_structured_data::OSD::Integer(i32::from(self.{name}))")
}
FieldKind::S32 => {
format!("libremetaverse_structured_data::OSD::Integer(self.{name})")
}
FieldKind::U32 | FieldKind::IpAddr | FieldKind::U64 => format!(
"libremetaverse_structured_data::OSD::Binary(self.{name}.to_le_bytes().to_vec())"
),
FieldKind::F32 => {
format!("libremetaverse_structured_data::OSD::Real(f64::from(self.{name}))")
}
FieldKind::F64 => {
format!("libremetaverse_structured_data::OSD::Real(self.{name})")
}
FieldKind::LlUuid => {
format!("libremetaverse_structured_data::OSD::UUID(self.{name})")
}
FieldKind::LlVector3 => {
format!("libremetaverse_structured_data::OSD::from_vector3(self.{name})?")
}
FieldKind::LlVector3d => {
format!("libremetaverse_structured_data::OSD::from_vector3d(self.{name})?")
}
FieldKind::LlVector4 => {
format!("libremetaverse_structured_data::OSD::from_vector4(self.{name})?")
}
FieldKind::LlQuaternion => {
format!("libremetaverse_structured_data::OSD::from_quaternion(self.{name})?")
}
FieldKind::Variable | FieldKind::Fixed => {
format!("libremetaverse_structured_data::OSD::Binary(self.{name}.clone())")
}
}
}
fn field_from_osd_statement(field: &PacketField) -> String {
let name = rust_snake(&field.name);
let conversion = match field.kind {
FieldKind::Bool => "value.as_boolean()?".to_owned(),
FieldKind::U8 => "value.as_integer()? as u8".to_owned(),
FieldKind::U16 | FieldKind::IpPort => "value.as_integer()? as u16".to_owned(),
FieldKind::U32 | FieldKind::IpAddr => "osd_u32(&value)?".to_owned(),
FieldKind::U64 => "osd_u64(&value)?".to_owned(),
FieldKind::S8 => "value.as_integer()? as i8".to_owned(),
FieldKind::S16 => "value.as_integer()? as i16".to_owned(),
FieldKind::S32 => "value.as_integer()?".to_owned(),
FieldKind::F32 => "value.as_real()? as f32".to_owned(),
FieldKind::F64 => "value.as_real()?".to_owned(),
FieldKind::LlUuid => "value.as_uuid()?".to_owned(),
FieldKind::LlVector3 => "value.as_vector3()?".to_owned(),
FieldKind::LlVector3d => "value.as_vector3d()?".to_owned(),
FieldKind::LlVector4 => "value.as_vector4()?".to_owned(),
FieldKind::LlQuaternion => "value.as_quaternion()?".to_owned(),
FieldKind::Variable | FieldKind::Fixed => "value.as_binary()?".to_owned(),
};
format!(
"if let Some(value) = map.get({:?}) {{ self.{name} = {conversion}; }}",
field.name
)
}
fn ordered_packets(protocol: &PacketProtocol) -> Vec<&PacketDefinition> {
[
PacketFrequency::Low,
PacketFrequency::Medium,
PacketFrequency::High,
]
.into_iter()
.flat_map(|frequency| {
protocol
.packets
.iter()
.filter(move |packet| packet.frequency == frequency)
})
.collect()
}
fn packet_type_value(packet: &PacketDefinition) -> u32 {
let prefix = match packet.frequency {
PacketFrequency::Low => 0x1_0000,
PacketFrequency::Medium => 0x2_0000,
PacketFrequency::High => 0x3_0000,
};
prefix | u32::from(packet.id)
}
fn append_descriptor_types(body: &mut String) {
body.push_str(
"#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]\n\
pub enum FieldKind { U8, U16, U32, U64, S8, S16, S32, F32, F64, LlUuid, Bool, LlVector3, LlVector3d, LlVector4, LlQuaternion, IpAddr, IpPort, Variable, Fixed }\n\n\
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]\n\
pub enum BlockRepetition { Single, Multiple(usize), Variable }\n\n\
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]\n\
pub struct FieldDescriptor { pub name: &'static str, pub kind: FieldKind, pub count: usize, pub fixed_width: usize }\n\n\
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]\n\
pub struct BlockDescriptor { pub name: &'static str, pub repetition: BlockRepetition, pub fields: &'static [FieldDescriptor] }\n\n\
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]\n\
pub struct PacketDescriptor { pub name: &'static str, pub packet_type: PacketType, pub id: u16, pub frequency: crate::PacketFrequency, pub trusted: bool, pub zerocoded: bool, pub flags: &'static [&'static str], pub blocks: &'static [BlockDescriptor] }\n\n",
);
}
fn append_packet_type(body: &mut String, packets: &[&PacketDefinition]) {
body.push_str("#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]\n#[repr(i32)]\npub enum PacketType {\n Default = 0,\n");
for packet in packets {
let prefix = match packet.frequency {
PacketFrequency::Low => 1,
PacketFrequency::Medium => 2,
PacketFrequency::High => 3,
};
let _ = writeln!(body, " {} = 0x{prefix}_{:04x},", packet.name, packet.id);
}
body.push_str("}\n\n");
}
fn append_descriptors(body: &mut String, packets: &[&PacketDefinition]) {
for (packet_index, packet) in packets.iter().enumerate() {
for (block_index, block) in packet.blocks.iter().enumerate() {
let _ = writeln!(
body,
"static PACKET_{packet_index}_BLOCK_{block_index}_FIELDS: &[FieldDescriptor] = &["
);
for field in &block.fields {
let _ = writeln!(
body,
" FieldDescriptor {{ name: {:?}, kind: FieldKind::{}, count: {}, fixed_width: {} }},",
field.name,
field_kind_name(field.kind),
field.count,
field_fixed_length(field)
);
}
body.push_str("];\n");
}
let _ = writeln!(
body,
"static PACKET_{packet_index}_BLOCKS: &[BlockDescriptor] = &["
);
for (block_index, block) in packet.blocks.iter().enumerate() {
let repetition = match block.repetition {
BlockRepetition::Single => "BlockRepetition::Single".to_owned(),
BlockRepetition::Variable => "BlockRepetition::Variable".to_owned(),
BlockRepetition::Multiple(count) => {
format!("BlockRepetition::Multiple({count})")
}
};
let _ = writeln!(
body,
" BlockDescriptor {{ name: {:?}, repetition: {repetition}, fields: PACKET_{packet_index}_BLOCK_{block_index}_FIELDS }},",
block.name
);
}
body.push_str("];\n\n");
}
body.push_str("pub static PACKETS: &[PacketDescriptor] = &[\n");
for (packet_index, packet) in packets.iter().enumerate() {
let flags = packet
.flags
.iter()
.map(|flag| format!("{flag:?}"))
.collect::<Vec<_>>()
.join(", ");
let _ = writeln!(
body,
" PacketDescriptor {{ name: {:?}, packet_type: PacketType::{}, id: {}, frequency: crate::PacketFrequency::{}, trusted: {}, zerocoded: {}, flags: &[{}], blocks: PACKET_{packet_index}_BLOCKS }},",
packet.name,
packet.name,
packet.id,
frequency_name(packet.frequency),
packet.trusted,
packet.zerocoded,
flags
);
}
body.push_str("];\n\n");
}
#[allow(clippy::too_many_lines)]
fn append_dispatch(body: &mut String, packets: &[&PacketDefinition]) {
body.push_str("#[must_use]\n#[allow(clippy::too_many_lines)]\npub fn packet_type(id: u16, frequency: crate::PacketFrequency) -> PacketType {\n match (frequency, id) {\n");
for packet in packets {
let _ = writeln!(
body,
" (crate::PacketFrequency::{}, {}) => PacketType::{},",
frequency_name(packet.frequency),
packet.id,
packet.name
);
}
body.push_str(" _ => PacketType::Default,\n }\n}\n\n");
body.push_str("#[must_use]\n#[allow(clippy::too_many_lines)]\npub fn descriptor_by_type(packet_type: PacketType) -> Option<&'static PacketDescriptor> {\n match packet_type {\n");
for (index, packet) in packets.iter().enumerate() {
let _ = writeln!(
body,
" PacketType::{} => Some(&PACKETS[{index}]),",
packet.name
);
}
body.push_str(" PacketType::Default => None,\n }\n}\n\n");
body.push_str("#[must_use]\n#[allow(clippy::too_many_lines)]\npub fn descriptor_by_name(name: &str) -> Option<&'static PacketDescriptor> {\n match name {\n");
for (index, packet) in packets.iter().enumerate() {
let _ = writeln!(
body,
" {:?} => Some(&PACKETS[{index}]),",
packet.name
);
}
body.push_str(" _ => None,\n }\n}\n\n");
body.push_str(
"pub(crate) fn build_packet(packet_type: PacketType) -> Result<crate::packets::Packet, crate::Error> {\n\
let descriptor = descriptor_by_type(packet_type).ok_or(crate::Error::InvalidOperation)?;\n\
Ok(crate::packets::Packet {\n\
has_variable_blocks: descriptor.blocks.iter().any(|block| matches!(block.repetition, BlockRepetition::Variable)),\n\
header: new_header(descriptor.frequency, descriptor.id, descriptor.zerocoded),\n\
type_: packet_type,\n\
})\n\
}\n\n\
pub(crate) const fn new_header(frequency: crate::PacketFrequency, id: u16, zerocoded: bool) -> crate::packets::Header {\n\
crate::packets::Header { ack_list: None, appended_acks: false, frequency, id, reliable: true, resent: false, sequence: 0, zerocoded }\n\
}\n\n\
fn usize_to_i32(value: usize) -> i32 { i32::try_from(value).unwrap_or(i32::MAX) }\n\n\
fn osd_u32(value: &libremetaverse_structured_data::OSD) -> Result<u32, crate::Error> {\n\
let bytes = value.as_binary()?;\n\
Ok(bytes.get(..4).and_then(|value| value.try_into().ok()).map_or(0, u32::from_le_bytes))\n\
}\n\n\
fn osd_u64(value: &libremetaverse_structured_data::OSD) -> Result<u64, crate::Error> {\n\
let bytes = value.as_binary()?;\n\
Ok(bytes.get(..8).and_then(|value| value.try_into().ok()).map_or(0, u64::from_le_bytes))\n\
}\n\n\
pub(crate) fn empty_packet_osd() -> Result<libremetaverse_structured_data::OSDMap, crate::Error> {\n\
libremetaverse_structured_data::OSDMap::new_with_constructor()\n\
}\n\n\
pub(crate) fn ignore_packet_osd(body: &libremetaverse_structured_data::OSDMap) -> Result<(), crate::Error> {\n\
let _ = body;\n\
Ok(())\n\
}\n\n\
pub(crate) trait GeneratedBlock: Sized {\n\
fn new_generated() -> Self;\n\
fn generated_length(&self) -> i32;\n\
fn decode_payload(&mut self, reader: &mut crate::packet_wire::WireReader<'_>) -> Result<(), crate::Error>;\n\
fn encode_payload(&self, writer: &mut crate::packet_wire::WireWriter) -> Result<(), crate::Error>;\n\
fn to_osd(&self) -> Result<libremetaverse_structured_data::OSDMap, crate::Error>;\n\
fn from_osd(&mut self, map: &libremetaverse_structured_data::OSDMap) -> Result<(), crate::Error>;\n\
fn new_from_bytes(bytes: &[u8], position: &mut i32) -> Result<Self, crate::Error> {\n\
let mut value = Self::new_generated();\n\
Self::decode_from_bytes(&mut value, bytes, position)?;\n\
Ok(value)\n\
}\n\
fn decode_from_bytes(&mut self, bytes: &[u8], position: &mut i32) -> Result<(), crate::Error> {\n\
if *position < 0 { return Err(crate::Error::Argument); }\n\
let start = usize::try_from(*position).map_err(|_| crate::Error::Argument)?;\n\
let mut reader = crate::packet_wire::WireReader::new(bytes, start, bytes.len())?;\n\
self.decode_payload(&mut reader)?;\n\
reader.commit_position(position)\n\
}\n\
fn encode_to_slice(&self, bytes: &mut [u8], position: &mut i32) -> Result<(), crate::Error> {\n\
if *position < 0 { return Err(crate::Error::Argument); }\n\
let start = usize::try_from(*position).map_err(|_| crate::Error::Argument)?;\n\
let capacity = usize::try_from(self.generated_length()).map_err(|_| crate::Error::Argument)?;\n\
let mut writer = crate::packet_wire::WireWriter::with_capacity(capacity)?;\n\
self.encode_payload(&mut writer)?;\n\
let encoded = writer.into_inner();\n\
let end = start.checked_add(encoded.len()).ok_or(crate::Error::Argument)?;\n\
if end > bytes.len() { return Err(crate::Error::IndexOutOfRange); }\n\
bytes[start..end].copy_from_slice(&encoded);\n\
*position = i32::try_from(end).map_err(|_| crate::Error::Argument)?;\n\
Ok(())\n\
}\n\
}\n\n\
pub(crate) trait GeneratedPacket: Sized {\n\
const USES_BUFFER_POOLING: bool;\n\
fn new_generated() -> Self;\n\
fn generated_length(&self) -> i32;\n\
fn generated_header(&self) -> crate::packets::Header;\n\
fn store_header(&mut self, header: crate::packets::Header);\n\
fn decode_payload(&mut self, reader: &mut crate::packet_wire::WireReader<'_>) -> Result<(), crate::Error>;\n\
fn encode_payload(&self, writer: &mut crate::packet_wire::WireWriter) -> Result<(), crate::Error>;\n\
fn packet_to_osd(&self) -> Result<libremetaverse_structured_data::OSDMap, crate::Error>;\n\
fn packet_from_osd(&mut self, body: &libremetaverse_structured_data::OSDMap) -> Result<(), crate::Error>;\n\
fn encode_multiple(&self) -> Result<Vec<Vec<u8>>, crate::Error>;\n\
fn encode_packet(&self) -> Result<Vec<u8>, crate::Error> {\n\
let header = self.generated_header();\n\
let payload = usize::try_from(self.generated_length()).map_err(|_| crate::Error::Argument)?;\n\
let capacity = payload.checked_add(crate::packet_wire::ack_length(&header)?).ok_or(crate::Error::Argument)?;\n\
let mut writer = crate::packet_wire::WireWriter::with_capacity(capacity)?;\n\
crate::packet_wire::encode_header(&header, &mut writer)?;\n\
self.encode_payload(&mut writer)?;\n\
crate::packet_wire::encode_acks(&header, &mut writer)?;\n\
Ok(writer.into_inner())\n\
}\n\
fn new_from_bytes(bytes: &[u8], position: &mut i32) -> Result<Self, crate::Error> {\n\
let mut value = Self::new_generated();\n\
let mut packet_end = i32::try_from(bytes.len()).map_err(|_| crate::Error::Argument)?.checked_sub(1).ok_or(crate::Error::Argument)?;\n\
Self::decode_from_bytes(&mut value, bytes, position, &mut packet_end, None)?;\n\
Ok(value)\n\
}\n\
fn new_from_header(header: crate::packets::Header, bytes: &[u8], position: &mut i32) -> Result<Self, crate::Error> {\n\
let mut value = Self::new_generated();\n\
let mut packet_end = i32::try_from(bytes.len()).map_err(|_| crate::Error::Argument)?.checked_sub(1).ok_or(crate::Error::Argument)?;\n\
Self::decode_from_header(&mut value, header, bytes, position, &mut packet_end)?;\n\
Ok(value)\n\
}\n\
fn decode_from_header(&mut self, header: crate::packets::Header, bytes: &[u8], position: &mut i32, packet_end: &mut i32) -> Result<(), crate::Error> {\n\
if *position < 0 || *packet_end < 0 { return Err(crate::Error::Argument); }\n\
let start = usize::try_from(*position).map_err(|_| crate::Error::Argument)?;\n\
let end = usize::try_from(packet_end.checked_add(1).ok_or(crate::Error::Argument)?).map_err(|_| crate::Error::Argument)?;\n\
let mut reader = crate::packet_wire::WireReader::new(bytes, start, end)?;\n\
self.decode_payload(&mut reader)?;\n\
reader.finish(position)?;\n\
self.store_header(header);\n\
Ok(())\n\
}\n\
fn decode_from_bytes(&mut self, bytes: &[u8], position: &mut i32, packet_end: &mut i32, zero_buffer: Option<&mut [u8]>) -> Result<(), crate::Error> {\n\
let header = crate::packet_wire::decode_header(bytes, position, packet_end)?;\n\
if header.zerocoded && let Some(buffer) = zero_buffer {\n\
let decoded_length = crate::packet_wire::zero_decode(Some(bytes), packet_end.checked_add(1).ok_or(crate::Error::Argument)?, Some(buffer))?;\n\
*packet_end = decoded_length.checked_sub(1).ok_or(crate::Error::Argument)?;\n\
let end = usize::try_from(decoded_length).map_err(|_| crate::Error::Argument)?;\n\
return self.decode_from_header(header, &buffer[..end], position, packet_end);\n\
}\n\
self.decode_from_header(header, bytes, position, packet_end)\n\
}\n\
}\n\n",
);
body.push_str(
"#[allow(clippy::too_many_lines)]\npub(crate) fn validate_packet_payload(\n packet_type: PacketType,\n header: crate::packets::Header,\n bytes: &[u8],\n position: &mut i32,\n packet_end: &mut i32,\n) -> Result<(), crate::Error> {\n match packet_type {\n",
);
for packet in packets {
let _ = writeln!(
body,
" PacketType::{} => {{ let mut packet = <crate::packets::{}Packet as GeneratedPacket>::new_generated(); GeneratedPacket::decode_from_header(&mut packet, header, bytes, position, packet_end) }},",
packet.name, packet.name
);
}
body.push_str(
" PacketType::Default => Err(crate::Error::InvalidOperation),\n }\n}\n\n",
);
body.push_str(
"#[allow(clippy::too_many_lines)]\npub(crate) fn build_packet_from_osd(\n name: &str,\n body: &libremetaverse_structured_data::OSDMap,\n) -> Result<Option<crate::packets::Packet>, crate::Error> {\n let Some(descriptor) = descriptor_by_name(name) else { return Ok(None); };\n match descriptor.packet_type {\n",
);
for packet in packets {
let _ = writeln!(
body,
" PacketType::{} => {{ let mut packet = <crate::packets::{}Packet as GeneratedPacket>::new_generated(); GeneratedPacket::packet_from_osd(&mut packet, body)?; }},",
packet.name, packet.name
);
}
body.push_str(
" PacketType::Default => return Ok(None),\n }\n build_packet(descriptor.packet_type).map(Some)\n}\n\n",
);
}
fn append_block_impls(body: &mut String, packets: &[&PacketDefinition]) {
for packet in packets {
for block in &packet.blocks {
let type_name = format!("{}Packet{}Block", packet.name, block.name);
let _ = writeln!(
body,
"impl GeneratedBlock for crate::packets::{type_name} {{\n fn new_generated() -> Self {{\n Self {{"
);
for field in &block.fields {
let _ = writeln!(
body,
" {}: {},",
rust_snake(&field.name),
field_default(field.kind)
);
}
body.push_str(" }\n }\n\n fn generated_length(&self) -> i32 {\n");
let fixed = block.fields.iter().map(field_fixed_length).sum::<usize>();
let variable_fields = block
.fields
.iter()
.filter(|field| field.kind == FieldKind::Variable)
.collect::<Vec<_>>();
if variable_fields.is_empty() {
let _ = writeln!(body, " {fixed}_i32");
} else {
let _ = writeln!(body, " let mut length = {fixed}_i32;");
for field in variable_fields {
let _ = writeln!(
body,
" length = length.saturating_add({}).saturating_add(usize_to_i32(self.{}.len()));",
field.count,
rust_snake(&field.name)
);
}
body.push_str(" length\n");
}
body.push_str(
" }\n\n fn decode_payload(&mut self, reader: &mut crate::packet_wire::WireReader<'_>) -> Result<(), crate::Error> {\n",
);
for field in &block.fields {
let _ = writeln!(body, " {}", field_decode_statement(field));
}
body.push_str(" Ok(())\n }\n\n fn encode_payload(&self, writer: &mut crate::packet_wire::WireWriter) -> Result<(), crate::Error> {\n");
for field in &block.fields {
let _ = writeln!(body, " {}", field_encode_statement(field));
}
body.push_str(" Ok(())\n }\n\n fn to_osd(&self) -> Result<libremetaverse_structured_data::OSDMap, crate::Error> {\n let map = libremetaverse_structured_data::OSDMap::new_with_constructor()?;\n");
for field in &block.fields {
let _ = writeln!(
body,
" map.add_with_string_osd({:?}.to_owned(), {})?;",
field.name,
field_to_osd_expression(field)
);
}
body.push_str(" Ok(map)\n }\n\n fn from_osd(&mut self, map: &libremetaverse_structured_data::OSDMap) -> Result<(), crate::Error> {\n");
for field in &block.fields {
let _ = writeln!(body, " {}", field_from_osd_statement(field));
}
if block.fields.is_empty() {
body.push_str(" let _ = map;\n");
}
body.push_str(" Ok(())\n }\n}\n\n");
}
}
}
fn packet_has_composed_base(name: &str) -> bool {
matches!(name, "DirPlacesReply" | "TestMessage")
}
fn packet_can_split(packet: &PacketDefinition) -> bool {
let mut saw_variable = false;
let mut has_variable = false;
for block in &packet.blocks {
if block.repetition == BlockRepetition::Variable {
saw_variable = true;
has_variable = true;
} else if saw_variable {
return false;
}
}
has_variable
}
#[allow(clippy::too_many_lines)]
fn append_packet_encode_multiple(body: &mut String, packet: &PacketDefinition) {
if !packet_can_split(packet) {
body.push_str(
" fn encode_multiple(&self) -> Result<Vec<Vec<u8>>, crate::Error> {\n Ok(vec![self.encode_packet()?])\n }\n",
);
return;
}
let variable_blocks = packet
.blocks
.iter()
.filter(|block| block.repetition == BlockRepetition::Variable)
.collect::<Vec<_>>();
body.push_str(
" #[allow(clippy::too_many_lines)]\n fn encode_multiple(&self) -> Result<Vec<Vec<u8>>, crate::Error> {\n let header = self.generated_header();\n let mut packets = Vec::new();\n",
);
for block in &variable_blocks {
let _ = writeln!(
body,
" let mut {}_start = 0_usize;",
rust_snake(&block.name)
);
}
body.push_str(" loop {\n let first_packet = packets.is_empty();\n let mut packet_header = crate::packet_wire::clone_header(&header);\n if !first_packet { packet_header.appended_acks = false; packet_header.ack_list = None; }\n let acks_length = crate::packet_wire::ack_length(&packet_header)?;\n let mut fixed_length = crate::packet_wire::header_length(packet_header.frequency);
");
for block in &packet.blocks {
let field_name = rust_snake(&block.name);
match block.repetition {
BlockRepetition::Single => {
let _ = writeln!(
body,
" fixed_length = fixed_length.checked_add(usize::try_from(GeneratedBlock::generated_length(&self.{field_name})).map_err(|_| crate::Error::Argument)?).ok_or(crate::Error::Argument)?;"
);
}
BlockRepetition::Multiple(_) => {
let _ = writeln!(
body,
" for block in &self.{field_name} {{ fixed_length = fixed_length.checked_add(usize::try_from(GeneratedBlock::generated_length(block)).map_err(|_| crate::Error::Argument)?).ok_or(crate::Error::Argument)?; }}"
);
}
BlockRepetition::Variable => {}
}
}
let _ = writeln!(
body,
" fixed_length = fixed_length.checked_add({}).ok_or(crate::Error::Argument)?;",
variable_blocks.len()
);
body.push_str(" let mut variable_length = 0_usize;\n");
for block in &variable_blocks {
let field_name = rust_snake(&block.name);
let _ = writeln!(body, " let mut {field_name}_count = 0_usize;");
let _ = writeln!(
body,
" while {field_name}_start + {field_name}_count < self.{field_name}.len() && {field_name}_count < usize::from(u8::MAX) {{"
);
let _ = writeln!(
body,
" let block_length = usize::try_from(GeneratedBlock::generated_length(&self.{field_name}[{field_name}_start + {field_name}_count])).map_err(|_| crate::Error::Argument)?;"
);
let _ = writeln!(
body,
" if fixed_length.saturating_add(variable_length).saturating_add(block_length).saturating_add(acks_length) <= usize::try_from(crate::packets::Packet::MTU).map_err(|_| crate::Error::Argument)? || {field_name}_count == 0 {{ variable_length = variable_length.checked_add(block_length).ok_or(crate::Error::Argument)?; {field_name}_count += 1; }} else {{ break; }}"
);
body.push_str(" }\n");
}
body.push_str(" let capacity = fixed_length.checked_add(variable_length).and_then(|length| length.checked_add(acks_length)).ok_or(crate::Error::Argument)?;\n let mut writer = crate::packet_wire::WireWriter::with_capacity(capacity)?;\n crate::packet_wire::encode_header(&packet_header, &mut writer)?;\n");
for block in &packet.blocks {
let field_name = rust_snake(&block.name);
match block.repetition {
BlockRepetition::Single => {
let _ = writeln!(
body,
" GeneratedBlock::encode_payload(&self.{field_name}, &mut writer)?;"
);
}
BlockRepetition::Multiple(count) => {
let _ = writeln!(
body,
" if self.{field_name}.len() != {count} {{ return Err(crate::Error::Argument); }}"
);
let _ = writeln!(
body,
" for block in &self.{field_name} {{ GeneratedBlock::encode_payload(block, &mut writer)?; }}"
);
}
BlockRepetition::Variable => {
let _ = writeln!(
body,
" writer.write_u8(u8::try_from({field_name}_count).map_err(|_| crate::Error::Argument)?)?;"
);
let _ = writeln!(
body,
" for block in &self.{field_name}[{field_name}_start..{field_name}_start + {field_name}_count] {{ GeneratedBlock::encode_payload(block, &mut writer)?; }}"
);
let _ = writeln!(
body,
" {field_name}_start += {field_name}_count;"
);
}
}
}
body.push_str(" crate::packet_wire::encode_acks(&packet_header, &mut writer)?;\n packets.push(writer.into_inner());\n if ");
for (index, block) in variable_blocks.iter().enumerate() {
if index != 0 {
body.push_str(" && ");
}
let field_name = rust_snake(&block.name);
let _ = write!(body, "{field_name}_start >= self.{field_name}.len()");
}
body.push_str(" { break; }\n }\n Ok(packets)\n }\n");
}
fn append_packet_osd(body: &mut String, packet: &PacketDefinition) {
body.push_str(
" fn packet_to_osd(&self) -> Result<libremetaverse_structured_data::OSDMap, crate::Error> {\n let body = libremetaverse_structured_data::OSDMap::new_with_constructor()?;\n",
);
for block in &packet.blocks {
let field_name = rust_snake(&block.name);
let capacity = match block.repetition {
BlockRepetition::Single => "1".to_owned(),
BlockRepetition::Multiple(_) | BlockRepetition::Variable => {
format!("self.{field_name}.len()")
}
};
let _ = writeln!(
body,
" let mut {field_name}_values = Vec::with_capacity({capacity});"
);
match block.repetition {
BlockRepetition::Single => {
let _ = writeln!(
body,
" {field_name}_values.push(libremetaverse_structured_data::OSD::Map(GeneratedBlock::to_osd(&self.{field_name})?.snapshot()));"
);
}
BlockRepetition::Multiple(_) | BlockRepetition::Variable => {
let _ = writeln!(
body,
" for block in &self.{field_name} {{ {field_name}_values.push(libremetaverse_structured_data::OSD::Map(GeneratedBlock::to_osd(block)?.snapshot())); }}"
);
}
}
let _ = writeln!(
body,
" body.add_with_string_osd({:?}.to_owned(), libremetaverse_structured_data::OSD::Array({field_name}_values))?;",
block.name
);
}
body.push_str(" Ok(body)\n }\n\n fn packet_from_osd(&mut self, body: &libremetaverse_structured_data::OSDMap) -> Result<(), crate::Error> {\n");
for block in &packet.blocks {
let field_name = rust_snake(&block.name);
let block_type = format!("crate::packets::{}Packet{}Block", packet.name, block.name);
let _ = writeln!(
body,
" if let Some(libremetaverse_structured_data::OSD::Array(values)) = body.get({:?}) {{",
block.name
);
match block.repetition {
BlockRepetition::Single => {
let _ = writeln!(
body,
" if let Some(libremetaverse_structured_data::OSD::Map(values)) = values.first() {{ let map = libremetaverse_structured_data::OSDMap::new_with_dictionary(values.clone())?; GeneratedBlock::from_osd(&mut self.{field_name}, &map)?; }}"
);
}
BlockRepetition::Variable => {
let _ = writeln!(
body,
" let mut blocks = Vec::new();\n blocks.try_reserve_exact(values.len()).map_err(|_| crate::Error::InvalidOperation)?;\n for value in values {{ let mut block = <{block_type} as GeneratedBlock>::new_generated(); if let libremetaverse_structured_data::OSD::Map(values) = value {{ let map = libremetaverse_structured_data::OSDMap::new_with_dictionary(values.clone())?; GeneratedBlock::from_osd(&mut block, &map)?; }} blocks.push(block); }}\n self.{field_name} = blocks;"
);
}
BlockRepetition::Multiple(count) => {
let _ = writeln!(
body,
" if self.{field_name}.len() != {count} {{ self.{field_name} = (0..{count}).map(|_| <{block_type} as GeneratedBlock>::new_generated()).collect(); }}\n for (block, value) in self.{field_name}.iter_mut().zip(values.iter().take({count})) {{ if let libremetaverse_structured_data::OSD::Map(values) = value {{ let map = libremetaverse_structured_data::OSDMap::new_with_dictionary(values.clone())?; GeneratedBlock::from_osd(block, &map)?; }} }}"
);
}
}
body.push_str(" }\n");
}
if packet.blocks.is_empty() {
body.push_str(" let _ = body;\n");
}
body.push_str(" Ok(())\n }\n\n");
}
#[allow(clippy::too_many_lines)]
fn append_packet_impls(body: &mut String, packets: &[&PacketDefinition]) {
for packet in packets {
let type_name = format!("{}Packet", packet.name);
let variable_count = packet
.blocks
.iter()
.filter(|block| block.repetition == BlockRepetition::Variable)
.count();
let base_length = match packet.frequency {
PacketFrequency::Low => 10,
PacketFrequency::Medium => 8,
PacketFrequency::High => 7,
};
let _ = writeln!(
body,
"impl GeneratedPacket for crate::packets::{type_name} {{\n const USES_BUFFER_POOLING: bool = {};\n\n fn new_generated() -> Self {{\n Self {{",
variable_count == 0
);
if packet_has_composed_base(&packet.name) {
let _ = writeln!(
body,
" base: crate::packets::Packet {{ has_variable_blocks: {}, header: new_header(crate::PacketFrequency::{}, {}, {}), type_: PacketType::{} }},",
variable_count > 0,
frequency_name(packet.frequency),
packet.id,
packet.zerocoded,
packet.name
);
}
for block in &packet.blocks {
let field_name = rust_snake(&block.name);
let block_type = format!("crate::packets::{}Packet{}Block", packet.name, block.name);
match block.repetition {
BlockRepetition::Single => {
let _ = writeln!(
body,
" {field_name}: <{block_type} as GeneratedBlock>::new_generated(),"
);
}
BlockRepetition::Variable => {
let _ = writeln!(body, " {field_name}: Vec::new(),");
}
BlockRepetition::Multiple(count) => {
let _ = writeln!(
body,
" {field_name}: (0..{count}).map(|_| <{block_type} as GeneratedBlock>::new_generated()).collect(),"
);
}
}
}
body.push_str(" }\n }\n\n fn generated_length(&self) -> i32 {\n");
if packet.blocks.is_empty() {
let _ = writeln!(body, " {base_length}_i32");
} else {
let _ = writeln!(body, " let mut length = {base_length}_i32;");
for block in &packet.blocks {
let field_name = rust_snake(&block.name);
match block.repetition {
BlockRepetition::Single => {
let _ = writeln!(
body,
" length = length.saturating_add(GeneratedBlock::generated_length(&self.{field_name}));"
);
}
BlockRepetition::Variable => {
let _ = writeln!(body, " length = length.saturating_add(1);");
let _ = writeln!(
body,
" for block in &self.{field_name} {{ length = length.saturating_add(GeneratedBlock::generated_length(block)); }}"
);
}
BlockRepetition::Multiple(_) => {
let _ = writeln!(
body,
" for block in &self.{field_name} {{ length = length.saturating_add(GeneratedBlock::generated_length(block)); }}"
);
}
}
}
body.push_str(" length\n");
}
body.push_str(" }\n\n fn generated_header(&self) -> crate::packets::Header {\n");
if packet_has_composed_base(&packet.name) {
body.push_str(" crate::packet_wire::clone_header(&self.base.header)\n");
} else {
let _ = writeln!(
body,
" new_header(crate::PacketFrequency::{}, {}, {})",
frequency_name(packet.frequency),
packet.id,
packet.zerocoded
);
}
body.push_str(
" }\n\n fn store_header(&mut self, header: crate::packets::Header) {\n",
);
if packet_has_composed_base(&packet.name) {
body.push_str(" self.base.header = header;\n");
} else {
body.push_str(" let _ = header;\n");
}
body.push_str(" }\n\n fn decode_payload(&mut self, reader: &mut crate::packet_wire::WireReader<'_>) -> Result<(), crate::Error> {\n");
for block in &packet.blocks {
let field_name = rust_snake(&block.name);
let block_type = format!("crate::packets::{}Packet{}Block", packet.name, block.name);
match block.repetition {
BlockRepetition::Single => {
let _ = writeln!(
body,
" GeneratedBlock::decode_payload(&mut self.{field_name}, reader)?;"
);
}
BlockRepetition::Variable => {
let _ = writeln!(
body,
" let {field_name}_count = usize::from(reader.read_u8()?);"
);
let _ = writeln!(body, " let mut {field_name} = Vec::new();");
let _ = writeln!(
body,
" {field_name}.try_reserve_exact({field_name}_count).map_err(|_| crate::Error::InvalidOperation)?;"
);
let _ = writeln!(
body,
" for _ in 0..{field_name}_count {{ let mut block = <{block_type} as GeneratedBlock>::new_generated(); GeneratedBlock::decode_payload(&mut block, reader)?; {field_name}.push(block); }}"
);
let _ = writeln!(body, " self.{field_name} = {field_name};");
}
BlockRepetition::Multiple(count) => {
let _ = writeln!(body, " let mut {field_name} = Vec::new();");
let _ = writeln!(
body,
" {field_name}.try_reserve_exact({count}).map_err(|_| crate::Error::InvalidOperation)?;"
);
let _ = writeln!(
body,
" for _ in 0..{count} {{ let mut block = <{block_type} as GeneratedBlock>::new_generated(); GeneratedBlock::decode_payload(&mut block, reader)?; {field_name}.push(block); }}"
);
let _ = writeln!(body, " self.{field_name} = {field_name};");
}
}
}
if packet.blocks.is_empty() {
body.push_str(" let _ = reader;\n");
}
body.push_str(" Ok(())\n }\n\n fn encode_payload(&self, writer: &mut crate::packet_wire::WireWriter) -> Result<(), crate::Error> {\n");
for block in &packet.blocks {
let field_name = rust_snake(&block.name);
match block.repetition {
BlockRepetition::Single => {
let _ = writeln!(
body,
" GeneratedBlock::encode_payload(&self.{field_name}, writer)?;"
);
}
BlockRepetition::Variable => {
let _ = writeln!(
body,
" writer.write_u8(u8::try_from(self.{field_name}.len()).map_err(|_| crate::Error::Argument)?)?;"
);
let _ = writeln!(
body,
" for block in &self.{field_name} {{ GeneratedBlock::encode_payload(block, writer)?; }}"
);
}
BlockRepetition::Multiple(count) => {
let _ = writeln!(
body,
" if self.{field_name}.len() != {count} {{ return Err(crate::Error::Argument); }}"
);
let _ = writeln!(
body,
" for block in &self.{field_name} {{ GeneratedBlock::encode_payload(block, writer)?; }}"
);
}
}
}
if packet.blocks.is_empty() {
body.push_str(" let _ = writer;\n");
}
body.push_str(" Ok(())\n }\n\n");
append_packet_osd(body, packet);
append_packet_encode_multiple(body, packet);
body.push_str("}\n\n");
}
}
fn append_packet_codec_tests(body: &mut String, packets: &[&PacketDefinition]) {
body.push_str(
"#[cfg(test)]\nmod generated_codec_tests {\n use super::{GeneratedBlock, GeneratedPacket};\n\n fn assert_round_trip<T: GeneratedPacket>(packet: &T, name: &str) {\n let expected = packet.encode_packet().unwrap_or_else(|error| panic!(\"{name}: encode failed: {error}\"));\n let (wire, mut zero_buffer) = if expected[0] & crate::Helpers::MSG_ZEROCODED != 0 {\n let mut encoded = vec![0_u8; expected.len().saturating_mul(2).saturating_add(2)];\n let encoded_length = crate::packet_wire::zero_encode(\n Some(&expected),\n i32::try_from(expected.len()).expect(\"packet length fits i32\"),\n Some(&mut encoded),\n )\n .unwrap_or_else(|error| panic!(\"{name}: zero encode failed: {error}\"));\n encoded.truncate(usize::try_from(encoded_length).expect(\"encoded length fits usize\"));\n (encoded, vec![0_u8; expected.len()])\n } else {\n (expected.clone(), Vec::new())\n };\n let mut decoded = T::new_generated();\n let mut position = 0_i32;\n let mut packet_end = i32::try_from(wire.len()).expect(\"wire length fits i32\") - 1;\n let buffer = (!zero_buffer.is_empty()).then_some(zero_buffer.as_mut_slice());\n decoded\n .decode_from_bytes(&wire, &mut position, &mut packet_end, buffer)\n .unwrap_or_else(|error| panic!(\"{name}: decode failed: {error}\"));\n let actual = decoded.encode_packet().unwrap_or_else(|error| panic!(\"{name}: re-encode failed: {error}\"));\n assert_eq!(actual, expected, \"{name}: round-trip bytes differ\");\n assert_eq!(position, i32::try_from(expected.len()).expect(\"packet length fits i32\"), \"{name}: final position\");\n\n for truncated_length in 0..wire.len() {\n let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {\n let mut candidate = T::new_generated();\n let mut truncated_position = 0_i32;\n let mut truncated_end = i32::try_from(truncated_length).expect(\"truncated length fits i32\") - 1;\n let mut scratch = vec![0_u8; expected.len()];\n let truncated_buffer = (!zero_buffer.is_empty()).then_some(scratch.as_mut_slice());\n candidate.decode_from_bytes(\n &wire[..truncated_length],\n &mut truncated_position,\n &mut truncated_end,\n truncated_buffer,\n )\n }));\n assert!(outcome.is_ok(), \"{name}: truncation at {truncated_length} panicked\");\n }\n }\n\n #[test]\n #[allow(clippy::too_many_lines)]\n fn every_generated_packet_round_trips_all_fields() {\n",
);
let truncation_loop = body
.rfind(" for truncated_length in 0..wire.len()")
.expect("packet test scaffold contains truncation loop");
body.insert_str(
truncation_loop,
" let expected_osd = packet.packet_to_osd().unwrap_or_else(|error| panic!(\"{name}: OSD serialization failed: {error}\"));\n let mut osd_round_trip = T::new_generated();\n osd_round_trip.packet_from_osd(&expected_osd).unwrap_or_else(|error| panic!(\"{name}: OSD deserialization failed: {error}\"));\n assert_eq!(osd_round_trip.encode_packet().unwrap_or_else(|error| panic!(\"{name}: OSD re-encode failed: {error}\")), expected, \"{name}: OSD round-trip bytes differ\");\n assert_eq!(osd_round_trip.packet_to_osd().unwrap_or_else(|error| panic!(\"{name}: OSD reserialization failed: {error}\")).snapshot(), expected_osd.snapshot(), \"{name}: OSD round-trip differs\");\n\n",
);
for packet in packets {
let mutates_packet = packet
.blocks
.iter()
.any(|block| block.repetition == BlockRepetition::Variable || !block.fields.is_empty());
let _ = writeln!(
body,
" let {}packet = <crate::packets::{}Packet as GeneratedPacket>::new_generated();",
if mutates_packet { "mut " } else { "" },
packet.name,
);
for block in &packet.blocks {
let block_name = rust_snake(&block.name);
match block.repetition {
BlockRepetition::Single => {
for field in &block.fields {
let field_name = rust_snake(&field.name);
let sample = field_sample(field);
let _ =
writeln!(body, " packet.{block_name}.{field_name} = {sample};");
}
}
BlockRepetition::Multiple(_) => {
let _ = writeln!(body, " for block in &mut packet.{block_name} {{");
for field in &block.fields {
let field_name = rust_snake(&field.name);
let sample = field_sample(field);
let _ = writeln!(body, " block.{field_name} = {sample};");
}
body.push_str(" }\n");
}
BlockRepetition::Variable => {
let block_type = format!("{}Packet{}Block", packet.name, block.name);
let _ = writeln!(
body,
" let mut block = <crate::packets::{block_type} as GeneratedBlock>::new_generated();"
);
for field in &block.fields {
let field_name = rust_snake(&field.name);
let sample = field_sample(field);
let _ = writeln!(body, " block.{field_name} = {sample};");
}
let _ = writeln!(body, " packet.{block_name}.push(block);");
}
}
}
let _ = writeln!(
body,
" assert_round_trip(&packet, {:?});",
packet.name
);
}
body.push_str(" }\n}\n");
}
fn csharp_field_type(kind: FieldKind) -> &'static str {
match kind {
FieldKind::U8 => "System.Byte",
FieldKind::U16 | FieldKind::IpPort => "System.UInt16",
FieldKind::U32 | FieldKind::IpAddr => "System.UInt32",
FieldKind::U64 => "System.UInt64",
FieldKind::S8 => "System.SByte",
FieldKind::S16 => "System.Int16",
FieldKind::S32 => "System.Int32",
FieldKind::F32 => "System.Single",
FieldKind::F64 => "System.Double",
FieldKind::LlUuid => "LibreMetaverse.UUID",
FieldKind::Bool => "System.Boolean",
FieldKind::LlVector3 => "LibreMetaverse.Vector3",
FieldKind::LlVector3d => "LibreMetaverse.Vector3d",
FieldKind::LlVector4 => "LibreMetaverse.Vector4",
FieldKind::LlQuaternion => "LibreMetaverse.Quaternion",
FieldKind::Variable | FieldKind::Fixed => "System.Byte[]",
}
}
fn catalog_fields(value: &serde_json::Value) -> Result<BTreeMap<String, String>, String> {
let members = value["members"]
.as_array()
.ok_or("catalog type has no members array")?;
members
.iter()
.filter(|member| member["kind"] == "field" && member["static"] == false)
.map(|member| {
let name = member["name"]
.as_str()
.ok_or("catalog field has no name")?
.to_owned();
let field_type = member["type"]
.as_str()
.ok_or("catalog field has no type")?
.to_owned();
Ok((name, field_type))
})
.collect()
}
#[allow(clippy::too_many_lines)]
pub fn validate_packet_api_catalog(root: &Path, protocol: &PacketProtocol) -> Result<(), String> {
let path = root.join(PUBLIC_API_PATH);
let bytes = fs::read(&path).map_err(|error| format!("{}: {error}", path.display()))?;
let catalog: serde_json::Value =
serde_json::from_slice(&bytes).map_err(|error| format!("{}: {error}", path.display()))?;
let assembly = catalog["assemblies"]
.as_array()
.and_then(|assemblies| {
assemblies
.iter()
.find(|assembly| assembly["identity"]["name"] == "LibreMetaverse")
})
.ok_or("public API catalog has no LibreMetaverse assembly")?;
let types = assembly["types"]
.as_array()
.ok_or("LibreMetaverse catalog has no types array")?;
let types_by_id: BTreeMap<&str, &serde_json::Value> = types
.iter()
.map(|item| {
item["doc_id"]
.as_str()
.map(|id| (id, item))
.ok_or("catalog type has no doc_id")
})
.collect::<Result<_, _>>()?;
let expected_packet_ids: BTreeSet<String> = protocol
.packets
.iter()
.map(|packet| format!("T:LibreMetaverse.Packets.{}Packet", packet.name))
.collect();
let expected_block_ids: BTreeSet<String> = protocol
.packets
.iter()
.flat_map(|packet| {
packet.blocks.iter().map(move |block| {
format!(
"T:LibreMetaverse.Packets.{}Packet.{}Block",
packet.name, block.name
)
})
})
.collect();
let actual_packet_ids: BTreeSet<String> = types
.iter()
.filter(|item| item["base_type"] == "LibreMetaverse.Packets.Packet")
.filter_map(|item| item["doc_id"].as_str().map(str::to_owned))
.collect();
let actual_block_ids: BTreeSet<String> = types
.iter()
.filter(|item| item["base_type"] == "LibreMetaverse.Packets.PacketBlock")
.filter_map(|item| item["doc_id"].as_str().map(str::to_owned))
.collect();
if expected_packet_ids != actual_packet_ids {
return Err(format!(
"packet API catalog mismatch: missing={:?}, stale={:?}",
expected_packet_ids
.difference(&actual_packet_ids)
.collect::<Vec<_>>(),
actual_packet_ids
.difference(&expected_packet_ids)
.collect::<Vec<_>>()
));
}
if expected_block_ids != actual_block_ids {
return Err(format!(
"packet block API catalog mismatch: missing={:?}, stale={:?}",
expected_block_ids
.difference(&actual_block_ids)
.collect::<Vec<_>>(),
actual_block_ids
.difference(&expected_block_ids)
.collect::<Vec<_>>()
));
}
for packet in &protocol.packets {
let packet_id = format!("T:LibreMetaverse.Packets.{}Packet", packet.name);
let actual = catalog_fields(types_by_id[packet_id.as_str()])?;
let expected = packet
.blocks
.iter()
.map(|block| {
let name = if block.name == "Header" {
"_Header".to_owned()
} else {
block.name.clone()
};
let array = if block.repetition == BlockRepetition::Single {
""
} else {
"[]"
};
(
name,
format!(
"LibreMetaverse.Packets.{}Packet.{}Block{array}",
packet.name, block.name
),
)
})
.collect::<BTreeMap<_, _>>();
if expected != actual {
return Err(format!(
"{packet_id} field mismatch: expected={expected:?}, actual={actual:?}"
));
}
for block in &packet.blocks {
let block_id = format!(
"T:LibreMetaverse.Packets.{}Packet.{}Block",
packet.name, block.name
);
let actual = catalog_fields(types_by_id[block_id.as_str()])?;
let expected = block
.fields
.iter()
.map(|field| (field.name.clone(), csharp_field_type(field.kind).to_owned()))
.collect::<BTreeMap<_, _>>();
if expected != actual {
return Err(format!(
"{block_id} field mismatch: expected={expected:?}, actual={actual:?}"
));
}
}
}
let packet_type = types_by_id
.get("T:LibreMetaverse.Packets.PacketType")
.ok_or("public API catalog has no PacketType")?;
let actual_values = packet_type["members"]
.as_array()
.ok_or("PacketType has no members array")?
.iter()
.filter(|member| member["kind"] == "enum_value")
.map(|member| {
let name = member["name"]
.as_str()
.ok_or("PacketType value has no name")?
.to_owned();
let value = member["value"]["value"]
.as_str()
.ok_or("PacketType value has no numeric value")?
.parse::<u32>()
.map_err(|_| "PacketType value is not a u32")?;
Ok::<_, &'static str>((name, value))
})
.collect::<Result<BTreeMap<_, _>, _>>()?;
let mut expected_values = BTreeMap::from([("Default".to_owned(), 0)]);
expected_values.extend(
protocol
.packets
.iter()
.map(|packet| (packet.name.clone(), packet_type_value(packet))),
);
if expected_values != actual_values {
return Err(format!(
"PacketType enum mismatch: missing/stale values = {}/{}",
expected_values
.keys()
.filter(|name| !actual_values.contains_key(*name))
.count(),
actual_values
.keys()
.filter(|name| !expected_values.contains_key(*name))
.count()
));
}
Ok(())
}
pub fn packet_catalog_bytes(root: &Path) -> Result<Vec<u8>, String> {
let inventory = load_inventory(root)?;
verify_inputs(root, &inventory)?;
let input = inventory
.inputs
.iter()
.find(|input| input.id == "message_template")
.ok_or("source inventory has no message_template input")?;
let generator = inventory
.generators
.iter()
.find(|generator| generator.id == "packets")
.ok_or("source inventory has no packets generator")?;
let path = root.join(&input.vendored_path);
let bytes = fs::read(&path).map_err(|error| format!("{}: {error}", path.display()))?;
let text = normalize_text(&input.vendored_path, &bytes).map_err(|error| error.to_string())?;
let protocol = parse_packet_template(&input.vendored_path, &text).map_err(|diagnostics| {
diagnostics
.into_iter()
.map(|item| item.to_string())
.collect::<Vec<_>>()
.join("\n")
})?;
validate_packet_api_catalog(root, &protocol)?;
let packets = ordered_packets(&protocol);
let mut body = String::new();
body.push_str("#![allow(clippy::cast_possible_truncation, clippy::cast_possible_wrap, clippy::cast_sign_loss, clippy::collapsible_if, clippy::similar_names, clippy::too_many_lines, clippy::unnecessary_wraps, clippy::vec_init_then_push, clippy::wrong_self_convention)]\n\n");
let _ = writeln!(
body,
"pub const GOLDEN_GENERATOR_SOURCE: &str = {:?};",
generator.reference_source
);
let _ = writeln!(
body,
"pub const GOLDEN_GENERATOR_SHA256: &str = {:?};",
generator.sha256
);
let _ = writeln!(
body,
"pub const GOLDEN_GENERATOR_LICENSE: &str = {:?};\n",
generator.license
);
append_descriptor_types(&mut body);
append_packet_type(&mut body, &packets);
append_descriptors(&mut body, &packets);
append_dispatch(&mut body, &packets);
append_block_impls(&mut body, &packets);
append_packet_impls(&mut body, &packets);
append_packet_codec_tests(&mut body, &packets);
let syntax = syn::parse_file(&body)
.map_err(|error| format!("generated packet Rust is invalid: {error}"))?;
let formatted = prettyplease::unparse(&syntax);
Ok(generated_rust("packets", &[input], &formatted))
}
pub fn load_inventory(root: &Path) -> Result<Inventory, String> {
let path = root.join(INVENTORY_PATH);
let bytes = fs::read(&path).map_err(|error| format!("{}: {error}", path.display()))?;
let inventory: Inventory =
serde_json::from_slice(&bytes).map_err(|error| format!("{}: {error}", path.display()))?;
validate_inventory(&inventory)?;
Ok(inventory)
}
fn validate_inventory(inventory: &Inventory) -> Result<(), String> {
if inventory.schema != 1 || inventory.upstream_commit.len() != 40 {
return Err("unsupported or malformed source inventory header".to_owned());
}
let mut input_ids = BTreeSet::new();
for input in &inventory.inputs {
validate_relative(&input.reference_path)?;
validate_relative(&input.vendored_path)?;
validate_hash(&input.sha256)?;
if input.license != "BSD-3-Clause" || !input_ids.insert(&input.id) {
return Err(format!("invalid or duplicate input {}", input.id));
}
}
let mut generator_ids = BTreeSet::new();
for generator in &inventory.generators {
validate_relative(&generator.reference_source)?;
validate_hash(&generator.sha256)?;
if generator.license != "BSD-3-Clause" || !generator_ids.insert(&generator.id) {
return Err(format!("invalid or duplicate generator {}", generator.id));
}
for input in &generator.inputs {
if !input_ids.contains(input) {
return Err(format!(
"generator {} references unknown input {input}",
generator.id
));
}
}
}
Ok(())
}
fn validate_relative(value: &str) -> Result<(), String> {
let path = Path::new(value);
if path.is_absolute()
|| path
.components()
.any(|part| !matches!(part, Component::Normal(_)))
{
return Err(format!("unsafe inventory path {value}"));
}
Ok(())
}
fn validate_hash(value: &str) -> Result<(), String> {
if value.len() == 64 && value.bytes().all(|byte| byte.is_ascii_hexdigit()) {
Ok(())
} else {
Err(format!("invalid SHA-256 {value}"))
}
}
fn sha256(bytes: &[u8]) -> String {
const HEX: &[u8; 16] = b"0123456789abcdef";
let mut output = String::with_capacity(64);
for byte in Sha256::digest(bytes) {
output.push(char::from(HEX[usize::from(byte >> 4)]));
output.push(char::from(HEX[usize::from(byte & 0x0f)]));
}
output
}
pub fn verify_inputs(root: &Path, inventory: &Inventory) -> Result<(), String> {
for input in &inventory.inputs {
let path = root.join(&input.vendored_path);
let bytes = fs::read(&path).map_err(|error| format!("{}: {error}", path.display()))?;
let actual = sha256(&bytes);
if actual != input.sha256 {
return Err(format!(
"{}: SHA-256 mismatch: expected {}, found {actual}",
path.display(),
input.sha256
));
}
normalize_text(&input.vendored_path, &bytes).map_err(|error| error.to_string())?;
}
Ok(())
}
pub fn vendor_inputs(root: &Path, reference: &Path) -> Result<(), String> {
let inventory = load_inventory(root)?;
for generator in &inventory.generators {
let source = reference.join(&generator.reference_source);
let bytes = fs::read(&source).map_err(|error| format!("{}: {error}", source.display()))?;
let actual = sha256(&bytes);
if actual != generator.sha256 {
return Err(format!(
"{}: pinned generator SHA-256 mismatch: expected {}, found {actual}",
source.display(),
generator.sha256
));
}
}
let mut snapshots = Vec::with_capacity(inventory.inputs.len());
for input in &inventory.inputs {
let source = reference.join(&input.reference_path);
let bytes = fs::read(&source).map_err(|error| format!("{}: {error}", source.display()))?;
let actual = sha256(&bytes);
if actual != input.sha256 {
return Err(format!(
"{}: pinned SHA-256 mismatch: expected {}, found {actual}",
source.display(),
input.sha256
));
}
snapshots.push((root.join(&input.vendored_path), bytes));
}
for (target, bytes) in snapshots {
fs::create_dir_all(target.parent().ok_or("vendored input has no parent")?)
.map_err(|error| error.to_string())?;
fs::write(&target, bytes).map_err(|error| format!("{}: {error}", target.display()))?;
}
Ok(())
}
pub fn source_manifest_bytes(root: &Path) -> Result<Vec<u8>, String> {
let inventory = load_inventory(root)?;
verify_inputs(root, &inventory)?;
let mut inputs: Vec<&InputSpec> = inventory.inputs.iter().collect();
inputs.sort_by_key(|input| &input.id);
let mut generators: Vec<&GeneratorSpec> = inventory.generators.iter().collect();
generators.sort_by_key(|generator| &generator.id);
let mut body = String::new();
let _ = writeln!(
body,
"pub const UPSTREAM_COMMIT: &str = {:?};",
inventory.upstream_commit
);
let _ = writeln!(
body,
"pub const UPSTREAM_REPOSITORY: &str = {:?};",
inventory.upstream_repository
);
body.push_str("pub const SOURCES: &[(&str, &str, &str, &str)] = &[\n");
for input in &inputs {
let _ = writeln!(
body,
" ({:?}, {:?}, {:?}, {:?}),",
input.id, input.vendored_path, input.sha256, input.license
);
}
body.push_str("];\n");
body.push_str("pub const GENERATORS: &[(&str, &str, &str, &str)] = &[\n");
for generator in generators {
let _ = writeln!(
body,
" ({:?}, {:?}, {:?}, {:?}),",
generator.id, generator.reference_source, generator.sha256, generator.license
);
}
body.push_str("];\n");
Ok(generated_rust("source-manifest", &inputs, &body))
}
fn generated_outputs(root: &Path) -> Result<Vec<(&'static str, Vec<u8>)>, String> {
Ok(vec![
(MANIFEST_OUTPUT, source_manifest_bytes(root)?),
(PACKET_OUTPUT, packet_catalog_bytes(root)?),
(VISUAL_OUTPUT, xml_catalog::visual_catalog_bytes(root)?),
(FOLIAGE_OUTPUT, xml_catalog::foliage_catalog_bytes(root)?),
(
SKELETON_OUTPUT,
avatar_catalog::skeleton_catalog_bytes(root)?,
),
(
ATTENTION_OUTPUT,
avatar_catalog::attention_catalog_bytes(root)?,
),
(
GENEPOOL_OUTPUT,
avatar_catalog::genepool_catalog_bytes(root)?,
),
])
}
pub fn regenerate_to(root: &Path, output_root: &Path) -> Result<(), String> {
for (relative, expected) in generated_outputs(root)? {
let path = output_root.join(relative);
fs::create_dir_all(path.parent().ok_or("generated output has no parent")?)
.map_err(|error| error.to_string())?;
fs::write(&path, expected).map_err(|error| format!("{}: {error}", path.display()))?;
}
Ok(())
}
pub fn regenerate(root: &Path, check: bool) -> Result<(), String> {
let outputs = generated_outputs(root)?;
if check {
for (relative, expected) in outputs {
let path = root.join(relative);
let actual = fs::read(&path).map_err(|error| format!("{}: {error}", path.display()))?;
if actual != expected {
return Err(format!(
"{} is stale; run the regeneration command",
path.display()
));
}
}
} else {
for (relative, expected) in outputs {
let path = root.join(relative);
fs::create_dir_all(path.parent().ok_or("generated output has no parent")?)
.map_err(|error| error.to_string())?;
fs::write(&path, expected).map_err(|error| format!("{}: {error}", path.display()))?;
}
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn checked_in_inputs_and_manifest_are_current() {
let root = workspace_root();
let first = source_manifest_bytes(&root).expect("first deterministic generation");
let second = source_manifest_bytes(&root).expect("second deterministic generation");
assert_eq!(first, second);
let first_packets = packet_catalog_bytes(&root).expect("first packet generation");
let second_packets = packet_catalog_bytes(&root).expect("second packet generation");
assert_eq!(first_packets, second_packets);
let first_visual =
xml_catalog::visual_catalog_bytes(&root).expect("first visual generation");
let second_visual =
xml_catalog::visual_catalog_bytes(&root).expect("second visual generation");
assert_eq!(first_visual, second_visual);
let first_foliage =
xml_catalog::foliage_catalog_bytes(&root).expect("first foliage generation");
let second_foliage =
xml_catalog::foliage_catalog_bytes(&root).expect("second foliage generation");
assert_eq!(first_foliage, second_foliage);
let first_skeleton =
avatar_catalog::skeleton_catalog_bytes(&root).expect("first skeleton generation");
let second_skeleton =
avatar_catalog::skeleton_catalog_bytes(&root).expect("second skeleton generation");
assert_eq!(first_skeleton, second_skeleton);
let first_attention =
avatar_catalog::attention_catalog_bytes(&root).expect("first attention generation");
let second_attention =
avatar_catalog::attention_catalog_bytes(&root).expect("second attention generation");
assert_eq!(first_attention, second_attention);
let first_genepool =
avatar_catalog::genepool_catalog_bytes(&root).expect("first genepool generation");
let second_genepool =
avatar_catalog::genepool_catalog_bytes(&root).expect("second genepool generation");
assert_eq!(first_genepool, second_genepool);
}
#[test]
fn text_normalization_and_diagnostics_are_stable() {
assert_eq!(
normalize_text("a\\b.xml", b"\xef\xbb\xbfa\r\nb\r").unwrap(),
"a\nb\n"
);
let error = normalize_text("a\\b.xml", b"one\n\0two").unwrap_err();
assert_eq!(
error.to_string(),
"a/b.xml:2:1: error[CG002]: input contains a NUL byte"
);
}
#[test]
fn inventory_rejects_unsafe_paths_hashes_and_unknown_inputs() {
let root = workspace_root();
let mut inventory = load_inventory(&root).expect("pinned inventory");
inventory.inputs[0].vendored_path = "../escape".to_owned();
assert!(validate_inventory(&inventory).is_err());
let mut inventory = load_inventory(&root).expect("pinned inventory");
inventory.generators[0].sha256 = "not-a-hash".to_owned();
assert!(validate_inventory(&inventory).is_err());
let mut inventory = load_inventory(&root).expect("pinned inventory");
inventory.generators[0].inputs.push("missing".to_owned());
assert!(validate_inventory(&inventory).is_err());
}
#[test]
fn packet_template_parser_preserves_ids_order_and_repetition() {
let text = "version 2.0\n{ Alpha Fixed 0xFFFFFFFB Trusted Zerocoded UDPDeprecated\n { One Single { Value U32 } }\n { Many Multiple 2 { Data Fixed 4 } }\n { Rest Variable { Name Variable 2 } }\n}\n{ Beta High 7 NotTrusted Unencoded }\n";
let protocol = parse_packet_template("fixture.msg", text).expect("valid protocol");
assert_eq!(protocol.packets.len(), 2);
let alpha = &protocol.packets[0];
assert_eq!(alpha.id, 0xfffb);
assert_eq!(alpha.frequency, PacketFrequency::Low);
assert!(alpha.trusted && alpha.zerocoded);
assert_eq!(alpha.flags, ["UDPDeprecated"]);
assert_eq!(alpha.blocks[1].repetition, BlockRepetition::Multiple(2));
assert_eq!(alpha.blocks[2].repetition, BlockRepetition::Variable);
assert_eq!(alpha.blocks[2].fields[0].count, 2);
assert_eq!(protocol.packets[1].frequency, PacketFrequency::High);
}
#[test]
fn packet_template_diagnostics_are_source_located_and_stable() {
let text = "{ Bad Nope xx Maybe Compressed\n { Data Multiple 0 { Value Mystery 3 } }\n}\n";
let diagnostics = parse_packet_template("data\\bad.msg", text).unwrap_err();
let rendered = diagnostics
.iter()
.map(ToString::to_string)
.collect::<Vec<_>>();
assert_eq!(
rendered,
[
"data/bad.msg:1:7: error[PG030]: unknown packet frequency \"Nope\"",
"data/bad.msg:1:12: error[PG031]: packet ID must be a decimal or hexadecimal unsigned integer",
"data/bad.msg:1:15: error[PG032]: unknown packet trust marker \"Maybe\"",
"data/bad.msg:1:21: error[PG033]: unknown packet coding marker \"Compressed\"",
"data/bad.msg:2:18: error[PG014]: Multiple block count must be a positive decimal integer",
"data/bad.msg:2:28: error[PG023]: unknown field type \"Mystery\"",
]
);
}
}
Reference in New Issue View Git Blame Copy Permalink