Complete native codegen milestone gate (#50)

tools/check_codegen_gate.py

@@ -0,0 +1,314 @@
+#!/usr/bin/env python3
+"""Enforce the complete native source-generation milestone gate."""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import re
+import subprocess
+import tempfile
+import tomllib
+from pathlib import Path
+
+import check_api_coverage
+import generate_api_shims
+import generate_rust_mapping
+
+ROOT = Path(__file__).resolve().parents[1]
+INVENTORY_PATH = ROOT / "codegen/sources.json"
+CATALOG_PATH = ROOT / "api/public-api.json"
+CODEGEN_README = ROOT / "codegen/README.md"
+API_REPORT = ROOT / "api/API-COVERAGE.md"
+
+OUTPUT_OWNERS = {
+    "codegen/generated/source_manifest.rs": ("source-manifest", None),
+    "crates/libremetaverse/src/packet_catalog.rs": ("packets", "packets"),
+    "crates/libremetaverse/src/visual_catalog.rs": ("visual-params", "visual_params"),
+    "crates/libremetaverse/src/foliage_catalog.rs": ("foliage", "trees"),
+    "crates/libremetaverse/src/skeleton_catalog.rs": ("skeleton", "skeleton"),
+    "crates/libremetaverse/src/attention_catalog.rs": ("attentions", "attentions"),
+    "crates/libremetaverse/src/genepool_catalog.rs": ("genepool", "genepool"),
+}
+
+SOURCE_GENERATED_NATIVE_TYPES = {
+    "T:LibreMetaverse.ArchetypeParam",
+    "T:LibreMetaverse.AttentionData",
+    "T:LibreMetaverse.AttentionSet",
+    "T:LibreMetaverse.DrivenParamInfo",
+    "T:LibreMetaverse.Genepool",
+    "T:LibreMetaverse.GenepoolArchetype",
+    "T:LibreMetaverse.GrassDefinition",
+    "T:LibreMetaverse.GrassDefinitions",
+    "T:LibreMetaverse.LindenAttentions",
+    "T:LibreMetaverse.SkeletalBoneInfo",
+    "T:LibreMetaverse.TreeDefinition",
+    "T:LibreMetaverse.TreeDefinitions",
+    "T:LibreMetaverse.VisualAlphaParam",
+    "T:LibreMetaverse.VisualColorParam",
+    "T:LibreMetaverse.VisualParam",
+    "T:LibreMetaverse.VisualParams",
+    "T:LibreMetaverse.VolumeMorphInfo",
+    "T:LibreMetaverse.Rendering.CollisionVolume",
+    "T:LibreMetaverse.Rendering.Joint",
+    "T:LibreMetaverse.Rendering.JointBase",
+    "T:LibreMetaverse.Rendering.LindenSkeleton",
+}
+
+FORBIDDEN_RUNTIME_NAMES = ("dotnet", "roslyn", "microsoft.codeanalysis", "coreclr")
+FORBIDDEN_RUNTIME_BYTES = tuple(value.encode() for value in FORBIDDEN_RUNTIME_NAMES)
+
+
+def fail(message: str) -> None:
+    raise ValueError(message)
+
+
+def sha256(path: Path) -> str:
+    return hashlib.sha256(path.read_bytes()).hexdigest()
+
+
+def run_temp_generation() -> None:
+    with tempfile.TemporaryDirectory(prefix="metacrate-codegen-") as directory:
+        output_root = Path(directory)
+        subprocess.run(
+            [
+                "cargo",
+                "run",
+                "--locked",
+                "--quiet",
+                "-p",
+                "libremetaverse-codegen",
+                "--",
+                "generate-to",
+                str(output_root),
+            ],
+            cwd=ROOT,
+            check=True,
+        )
+        actual_paths = {
+            path.relative_to(output_root).as_posix()
+            for path in output_root.rglob("*")
+            if path.is_file()
+        }
+        expected_paths = set(OUTPUT_OWNERS)
+        if actual_paths != expected_paths:
+            fail(
+                "temporary generation output mismatch: "
+                f"missing={sorted(expected_paths - actual_paths)}, "
+                f"unexpected={sorted(actual_paths - expected_paths)}"
+            )
+        for relative in sorted(expected_paths):
+            generated = output_root / relative
+            checked_in = ROOT / relative
+            if generated.read_bytes() != checked_in.read_bytes():
+                fail(f"{relative} differs from clean temporary regeneration")
+
+
+def audit_inventory_and_headers() -> tuple[dict, dict]:
+    inventory = json.loads(INVENTORY_PATH.read_text())
+    catalog = json.loads(CATALOG_PATH.read_text())
+    if inventory.get("schema") != 1:
+        fail("unsupported codegen inventory schema")
+    if catalog["upstream"]["commit"] != inventory["upstream_commit"]:
+        fail("public API catalog and source inventory pin different upstream commits")
+    if catalog["upstream"]["repository"] != inventory["upstream_repository"]:
+        fail("public API catalog and source inventory pin different repositories")
+
+    generators = {item["id"]: item for item in inventory["generators"]}
+    inputs = {item["id"]: item for item in inventory["inputs"]}
+    if len(generators) != 6 or len(inputs) != len(inventory["inputs"]):
+        fail("source inventory must contain six unique generators and unique inputs")
+    public_generators = {item["name"] for item in catalog["source_generators"]}
+    inventoried_generators = {
+        Path(item["reference_source"]).stem for item in generators.values()
+    }
+    if public_generators != inventoried_generators:
+        fail("source inventory does not exactly cover the authoritative generator catalog")
+
+    referenced_inputs: set[str] = set()
+    for generator in generators.values():
+        if generator["license"] != "BSD-3-Clause":
+            fail(f"generator {generator['id']} has an unexpected license")
+        if not re.fullmatch(r"[0-9a-f]{64}", generator["sha256"]):
+            fail(f"generator {generator['id']} has an invalid SHA-256")
+        referenced_inputs.update(generator["inputs"])
+    if referenced_inputs != set(inputs):
+        fail("generator input lists do not exactly cover the input inventory")
+    for item in inputs.values():
+        if item["license"] != "BSD-3-Clause":
+            fail(f"input {item['id']} has an unexpected license")
+        path = ROOT / item["vendored_path"]
+        if sha256(path) != item["sha256"]:
+            fail(f"{item['vendored_path']} does not match its pinned SHA-256")
+
+    for relative, (header_name, owner_id) in OUTPUT_OWNERS.items():
+        text = (ROOT / relative).read_text()
+        if not text.startswith("// @generated by libremetaverse-codegen; DO NOT EDIT.\n"):
+            fail(f"{relative} has no generated-file header")
+        if f"// Generator: {header_name}\n" not in text[:1000]:
+            fail(f"{relative} has the wrong generator header")
+        expected_inputs = inputs.values() if owner_id is None else (
+            inputs[input_id] for input_id in generators[owner_id]["inputs"]
+        )
+        for item in expected_inputs:
+            source_line = (
+                f"// Source: {item['vendored_path']} sha256={item['sha256']} "
+                f"license={item['license']}"
+            )
+            if source_line not in text[:4000]:
+                fail(f"{relative} omits provenance for {item['id']}")
+    return inventory, catalog
+
+
+def require_current(path: Path, expected: str, label: str) -> None:
+    if not path.exists() or path.read_text() != expected:
+        fail(f"{label} is stale: {path.relative_to(ROOT)}")
+
+
+def audit_api_outputs(catalog: dict) -> tuple[int, int]:
+    for path, expected in generate_rust_mapping.generate().items():
+        require_current(path, expected, "Rust API mapping")
+    for path, expected in generate_api_shims.generate().items():
+        require_current(path, expected, "Rust API shim")
+    require_current(API_REPORT, check_api_coverage.audit(), "complete API coverage report")
+
+    catalog_types = {
+        item["doc_id"]
+        for assembly in catalog["assemblies"]
+        for item in assembly["types"]
+    }
+    missing_native = SOURCE_GENERATED_NATIVE_TYPES - catalog_types
+    if missing_native:
+        fail(f"source-generated native type catalog entries are missing: {sorted(missing_native)}")
+    unmapped_native = SOURCE_GENERATED_NATIVE_TYPES - set(generate_api_shims.NATIVE_TYPES)
+    if unmapped_native:
+        fail(f"source-generated types are not backed by native Rust: {sorted(unmapped_native)}")
+
+    packet_type = re.compile(
+        r"T:LibreMetaverse\.Packets\.[^.]+Packet(?:\.[^.]+Block)?"
+    )
+    packet_osd = re.compile(
+        r"M:LibreMetaverse\.Packets\.[^.]+Packet(?:\.[^.]+Block)?\."
+        r"(?:PacketToOSD|PacketFromOSD|ToOSD|FromOSD)(?:\([^)]*\))?"
+    )
+    expected_members = {
+        member["doc_id"]
+        for assembly in catalog["assemblies"]
+        for item in assembly["types"]
+        for member in item["members"]
+        if packet_type.fullmatch(item["doc_id"])
+    }
+    osd_members = {member for member in expected_members if packet_osd.fullmatch(member)}
+    generated = generate_api_shims.MAIN_TARGET.read_text()
+    markers = list(re.finditer(r"/// C# member: `([^`]+)`\.", generated))
+    blocks = {
+        match.group(1): generated[
+            match.start() : markers[index + 1].start() if index + 1 < len(markers) else len(generated)
+        ]
+        for index, match in enumerate(markers)
+    }
+    if not osd_members or expected_members - set(blocks):
+        fail("packet source-generator API coverage is missing generated members")
+    for member_id in expected_members:
+        block = blocks[member_id]
+        if "not_implemented(" in block or "unimplemented_api(" in block:
+            fail(f"packet source-generator API remains a failure shim: {member_id}")
+    return len(expected_members), len(osd_members)
+
+
+def dependency_names() -> set[str]:
+    names: set[str] = set()
+    lock = tomllib.loads((ROOT / "Cargo.lock").read_text())
+    names.update(package["name"].lower() for package in lock["package"])
+    for manifest in ROOT.rglob("Cargo.toml"):
+        if "target" in manifest.parts:
+            continue
+        data = tomllib.loads(manifest.read_text())
+        for table_name, value in data.items():
+            if table_name.endswith("dependencies") and isinstance(value, dict):
+                names.update(name.lower() for name in value)
+        for target in data.get("target", {}).values():
+            if isinstance(target, dict):
+                for table_name, value in target.items():
+                    if table_name.endswith("dependencies") and isinstance(value, dict):
+                        names.update(name.lower() for name in value)
+    return names
+
+
+def audit_rust_only_release(release_dir: Path | None) -> int:
+    bad_dependencies = sorted(
+        name
+        for name in dependency_names()
+        if any(forbidden in name for forbidden in FORBIDDEN_RUNTIME_NAMES)
+    )
+    if bad_dependencies:
+        fail(f"managed-runtime dependencies found: {bad_dependencies}")
+    for build_script in ROOT.rglob("build.rs"):
+        if "target" in build_script.parts:
+            continue
+        lowered = build_script.read_bytes().lower()
+        if any(value in lowered for value in FORBIDDEN_RUNTIME_BYTES):
+            fail(f"managed-runtime invocation found in {build_script.relative_to(ROOT)}")
+
+    if release_dir is None:
+        return 0
+    if not release_dir.is_dir():
+        fail(f"release artifact directory does not exist: {release_dir}")
+    scanned = 0
+    for path in release_dir.rglob("*"):
+        if not path.is_file() or path.suffix in {".d", ".rmeta"}:
+            continue
+        lowered_name = path.name.lower()
+        if any(value in lowered_name for value in FORBIDDEN_RUNTIME_NAMES):
+            fail(f"managed-runtime release artifact found: {path}")
+        data = path.read_bytes().lower()
+        if any(value in data for value in FORBIDDEN_RUNTIME_BYTES):
+            fail(f"managed-runtime marker embedded in release artifact: {path}")
+        scanned += 1
+    if scanned == 0:
+        fail(f"no release artifacts found under {release_dir}")
+    return scanned
+
+
+def audit_documentation(inventory: dict) -> None:
+    text = CODEGEN_README.read_text()
+    required = [
+        *(Path(path).name for path in OUTPUT_OWNERS),
+        *(Path(item["reference_source"]).stem for item in inventory["generators"]),
+        *(Path(item["vendored_path"]).name for item in inventory["inputs"]),
+        "generate-to",
+        "check_codegen_gate.py",
+    ]
+    missing = sorted(value for value in required if value not in text)
+    if missing:
+        fail(f"codegen documentation omits audited inputs/outputs/commands: {missing}")
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--release-dir",
+        type=Path,
+        help="scan a completed Rust release build for managed-runtime artifacts",
+    )
+    args = parser.parse_args()
+
+    run_temp_generation()
+    inventory, catalog = audit_inventory_and_headers()
+    packet_members, osd_members = audit_api_outputs(catalog)
+    audit_documentation(inventory)
+    artifacts = audit_rust_only_release(args.release_dir)
+    print(
+        "native codegen gate passed: "
+        f"{len(OUTPUT_OWNERS)} byte-identical outputs, "
+        f"{len(inventory['generators'])} generators, "
+        f"{len(inventory['inputs'])} inputs, "
+        f"{packet_members} native packet members, "
+        f"{osd_members} packet OSD members, "
+        f"{artifacts} release artifacts scanned"
+    )
+
+
+if __name__ == "__main__":
+    main()

tools/codegen/src/lib.rs

@@ -23,6 +23,15 @@ pub const SKELETON_OUTPUT: &str = "crates/libremetaverse/src/skeleton_catalog.rs
 pub const ATTENTION_OUTPUT: &str = "crates/libremetaverse/src/attention_catalog.rs";
 pub const GENEPOOL_OUTPUT: &str = "crates/libremetaverse/src/genepool_catalog.rs";
 pub const PUBLIC_API_PATH: &str = "api/public-api.json";
+pub const GENERATED_OUTPUT_PATHS: [&str; 7] = [
+    MANIFEST_OUTPUT,
+    PACKET_OUTPUT,
+    VISUAL_OUTPUT,
+    FOLIAGE_OUTPUT,
+    SKELETON_OUTPUT,
+    ATTENTION_OUTPUT,
+    GENEPOOL_OUTPUT,
+];
 const RUST_KEYWORDS: &[&str] = &[
     "as", "break", "const", "continue", "crate", "else", "enum", "extern", "false", "fn", "for",
     "if", "impl", "in", "let", "loop", "match", "mod", "move", "mut", "pub", "ref", "return",
@@ -929,6 +938,74 @@ fn field_encode_statement(field: &PacketField) -> String {
     }
 }
 
+fn field_to_osd_expression(field: &PacketField) -> String {
+    let name = rust_snake(&field.name);
+    match field.kind {
+        FieldKind::Bool => {
+            format!("libremetaverse_structured_data::OSD::Boolean(self.{name})")
+        }
+        FieldKind::U8 | FieldKind::U16 | FieldKind::IpPort | FieldKind::S8 | FieldKind::S16 => {
+            format!("libremetaverse_structured_data::OSD::Integer(i32::from(self.{name}))")
+        }
+        FieldKind::S32 => {
+            format!("libremetaverse_structured_data::OSD::Integer(self.{name})")
+        }
+        FieldKind::U32 | FieldKind::IpAddr | FieldKind::U64 => format!(
+            "libremetaverse_structured_data::OSD::Binary(self.{name}.to_le_bytes().to_vec())"
+        ),
+        FieldKind::F32 => {
+            format!("libremetaverse_structured_data::OSD::Real(f64::from(self.{name}))")
+        }
+        FieldKind::F64 => {
+            format!("libremetaverse_structured_data::OSD::Real(self.{name})")
+        }
+        FieldKind::LlUuid => {
+            format!("libremetaverse_structured_data::OSD::UUID(self.{name})")
+        }
+        FieldKind::LlVector3 => {
+            format!("libremetaverse_structured_data::OSD::from_vector3(self.{name})?")
+        }
+        FieldKind::LlVector3d => {
+            format!("libremetaverse_structured_data::OSD::from_vector3d(self.{name})?")
+        }
+        FieldKind::LlVector4 => {
+            format!("libremetaverse_structured_data::OSD::from_vector4(self.{name})?")
+        }
+        FieldKind::LlQuaternion => {
+            format!("libremetaverse_structured_data::OSD::from_quaternion(self.{name})?")
+        }
+        FieldKind::Variable | FieldKind::Fixed => {
+            format!("libremetaverse_structured_data::OSD::Binary(self.{name}.clone())")
+        }
+    }
+}
+
+fn field_from_osd_statement(field: &PacketField) -> String {
+    let name = rust_snake(&field.name);
+    let conversion = match field.kind {
+        FieldKind::Bool => "value.as_boolean()?".to_owned(),
+        FieldKind::U8 => "value.as_integer()? as u8".to_owned(),
+        FieldKind::U16 | FieldKind::IpPort => "value.as_integer()? as u16".to_owned(),
+        FieldKind::U32 | FieldKind::IpAddr => "osd_u32(&value)?".to_owned(),
+        FieldKind::U64 => "osd_u64(&value)?".to_owned(),
+        FieldKind::S8 => "value.as_integer()? as i8".to_owned(),
+        FieldKind::S16 => "value.as_integer()? as i16".to_owned(),
+        FieldKind::S32 => "value.as_integer()?".to_owned(),
+        FieldKind::F32 => "value.as_real()? as f32".to_owned(),
+        FieldKind::F64 => "value.as_real()?".to_owned(),
+        FieldKind::LlUuid => "value.as_uuid()?".to_owned(),
+        FieldKind::LlVector3 => "value.as_vector3()?".to_owned(),
+        FieldKind::LlVector3d => "value.as_vector3d()?".to_owned(),
+        FieldKind::LlVector4 => "value.as_vector4()?".to_owned(),
+        FieldKind::LlQuaternion => "value.as_quaternion()?".to_owned(),
+        FieldKind::Variable | FieldKind::Fixed => "value.as_binary()?".to_owned(),
+    };
+    format!(
+        "if let Some(value) = map.get({:?}) {{ self.{name} = {conversion}; }}",
+        field.name
+    )
+}
+
 fn ordered_packets(protocol: &PacketProtocol) -> Vec<&PacketDefinition> {
     [
         PacketFrequency::Low,
@@ -1088,11 +1165,28 @@ fn append_dispatch(body: &mut String, packets: &[&PacketDefinition]) {
              crate::packets::Header { ack_list: None, appended_acks: false, frequency, id, reliable: true, resent: false, sequence: 0, zerocoded }\n\
          }\n\n\
          fn usize_to_i32(value: usize) -> i32 { i32::try_from(value).unwrap_or(i32::MAX) }\n\n\
+         fn osd_u32(value: &libremetaverse_structured_data::OSD) -> Result<u32, crate::Error> {\n\
+             let bytes = value.as_binary()?;\n\
+             Ok(bytes.get(..4).and_then(|value| value.try_into().ok()).map_or(0, u32::from_le_bytes))\n\
+         }\n\n\
+         fn osd_u64(value: &libremetaverse_structured_data::OSD) -> Result<u64, crate::Error> {\n\
+             let bytes = value.as_binary()?;\n\
+             Ok(bytes.get(..8).and_then(|value| value.try_into().ok()).map_or(0, u64::from_le_bytes))\n\
+         }\n\n\
+         pub(crate) fn empty_packet_osd() -> Result<libremetaverse_structured_data::OSDMap, crate::Error> {\n\
+             libremetaverse_structured_data::OSDMap::new_with_constructor()\n\
+         }\n\n\
+         pub(crate) fn ignore_packet_osd(body: &libremetaverse_structured_data::OSDMap) -> Result<(), crate::Error> {\n\
+             let _ = body;\n\
+             Ok(())\n\
+         }\n\n\
          pub(crate) trait GeneratedBlock: Sized {\n\
              fn new_generated() -> Self;\n\
              fn generated_length(&self) -> i32;\n\
              fn decode_payload(&mut self, reader: &mut crate::packet_wire::WireReader<'_>) -> Result<(), crate::Error>;\n\
              fn encode_payload(&self, writer: &mut crate::packet_wire::WireWriter) -> Result<(), crate::Error>;\n\
+             fn to_osd(&self) -> Result<libremetaverse_structured_data::OSDMap, crate::Error>;\n\
+             fn from_osd(&mut self, map: &libremetaverse_structured_data::OSDMap) -> Result<(), crate::Error>;\n\
              fn new_from_bytes(bytes: &[u8], position: &mut i32) -> Result<Self, crate::Error> {\n\
                  let mut value = Self::new_generated();\n\
                  Self::decode_from_bytes(&mut value, bytes, position)?;\n\
@@ -1127,6 +1221,8 @@ fn append_dispatch(body: &mut String, packets: &[&PacketDefinition]) {
              fn store_header(&mut self, header: crate::packets::Header);\n\
              fn decode_payload(&mut self, reader: &mut crate::packet_wire::WireReader<'_>) -> Result<(), crate::Error>;\n\
              fn encode_payload(&self, writer: &mut crate::packet_wire::WireWriter) -> Result<(), crate::Error>;\n\
+             fn packet_to_osd(&self) -> Result<libremetaverse_structured_data::OSDMap, crate::Error>;\n\
+             fn packet_from_osd(&mut self, body: &libremetaverse_structured_data::OSDMap) -> Result<(), crate::Error>;\n\
              fn encode_multiple(&self) -> Result<Vec<Vec<u8>>, crate::Error>;\n\
              fn encode_packet(&self) -> Result<Vec<u8>, crate::Error> {\n\
                  let header = self.generated_header();\n\
@@ -1185,6 +1281,19 @@ fn append_dispatch(body: &mut String, packets: &[&PacketDefinition]) {
     body.push_str(
         "        PacketType::Default => Err(crate::Error::InvalidOperation),\n    }\n}\n\n",
     );
+    body.push_str(
+        "#[allow(clippy::too_many_lines)]\npub(crate) fn build_packet_from_osd(\n    name: &str,\n    body: &libremetaverse_structured_data::OSDMap,\n) -> Result<Option<crate::packets::Packet>, crate::Error> {\n    let Some(descriptor) = descriptor_by_name(name) else { return Ok(None); };\n    match descriptor.packet_type {\n",
+    );
+    for packet in packets {
+        let _ = writeln!(
+            body,
+            "        PacketType::{} => {{ let mut packet = <crate::packets::{}Packet as GeneratedPacket>::new_generated(); GeneratedPacket::packet_from_osd(&mut packet, body)?; }},",
+            packet.name, packet.name
+        );
+    }
+    body.push_str(
+        "        PacketType::Default => return Ok(None),\n    }\n    build_packet(descriptor.packet_type).map(Some)\n}\n\n",
+    );
 }
 
 fn append_block_impls(body: &mut String, packets: &[&PacketDefinition]) {
@@ -1234,6 +1343,22 @@ fn append_block_impls(body: &mut String, packets: &[&PacketDefinition]) {
             for field in &block.fields {
                 let _ = writeln!(body, "        {}", field_encode_statement(field));
             }
+            body.push_str("        Ok(())\n    }\n\n    fn to_osd(&self) -> Result<libremetaverse_structured_data::OSDMap, crate::Error> {\n        let map = libremetaverse_structured_data::OSDMap::new_with_constructor()?;\n");
+            for field in &block.fields {
+                let _ = writeln!(
+                    body,
+                    "        map.add_with_string_osd({:?}.to_owned(), {})?;",
+                    field.name,
+                    field_to_osd_expression(field)
+                );
+            }
+            body.push_str("        Ok(map)\n    }\n\n    fn from_osd(&mut self, map: &libremetaverse_structured_data::OSDMap) -> Result<(), crate::Error> {\n");
+            for field in &block.fields {
+                let _ = writeln!(body, "        {}", field_from_osd_statement(field));
+            }
+            if block.fields.is_empty() {
+                body.push_str("        let _ = map;\n");
+            }
             body.push_str("        Ok(())\n    }\n}\n\n");
         }
     }
@@ -1371,6 +1496,79 @@ fn append_packet_encode_multiple(body: &mut String, packet: &PacketDefinition) {
     body.push_str(" { break; }\n        }\n        Ok(packets)\n    }\n");
 }
 
+fn append_packet_osd(body: &mut String, packet: &PacketDefinition) {
+    body.push_str(
+        "    fn packet_to_osd(&self) -> Result<libremetaverse_structured_data::OSDMap, crate::Error> {\n        let body = libremetaverse_structured_data::OSDMap::new_with_constructor()?;\n",
+    );
+    for block in &packet.blocks {
+        let field_name = rust_snake(&block.name);
+        let capacity = match block.repetition {
+            BlockRepetition::Single => "1".to_owned(),
+            BlockRepetition::Multiple(_) | BlockRepetition::Variable => {
+                format!("self.{field_name}.len()")
+            }
+        };
+        let _ = writeln!(
+            body,
+            "        let mut {field_name}_values = Vec::with_capacity({capacity});"
+        );
+        match block.repetition {
+            BlockRepetition::Single => {
+                let _ = writeln!(
+                    body,
+                    "        {field_name}_values.push(libremetaverse_structured_data::OSD::Map(GeneratedBlock::to_osd(&self.{field_name})?.snapshot()));"
+                );
+            }
+            BlockRepetition::Multiple(_) | BlockRepetition::Variable => {
+                let _ = writeln!(
+                    body,
+                    "        for block in &self.{field_name} {{ {field_name}_values.push(libremetaverse_structured_data::OSD::Map(GeneratedBlock::to_osd(block)?.snapshot())); }}"
+                );
+            }
+        }
+        let _ = writeln!(
+            body,
+            "        body.add_with_string_osd({:?}.to_owned(), libremetaverse_structured_data::OSD::Array({field_name}_values))?;",
+            block.name
+        );
+    }
+    body.push_str("        Ok(body)\n    }\n\n    fn packet_from_osd(&mut self, body: &libremetaverse_structured_data::OSDMap) -> Result<(), crate::Error> {\n");
+    for block in &packet.blocks {
+        let field_name = rust_snake(&block.name);
+        let block_type = format!("crate::packets::{}Packet{}Block", packet.name, block.name);
+        let _ = writeln!(
+            body,
+            "        if let Some(libremetaverse_structured_data::OSD::Array(values)) = body.get({:?}) {{",
+            block.name
+        );
+        match block.repetition {
+            BlockRepetition::Single => {
+                let _ = writeln!(
+                    body,
+                    "            if let Some(libremetaverse_structured_data::OSD::Map(values)) = values.first() {{ let map = libremetaverse_structured_data::OSDMap::new_with_dictionary(values.clone())?; GeneratedBlock::from_osd(&mut self.{field_name}, &map)?; }}"
+                );
+            }
+            BlockRepetition::Variable => {
+                let _ = writeln!(
+                    body,
+                    "            let mut blocks = Vec::new();\n            blocks.try_reserve_exact(values.len()).map_err(|_| crate::Error::InvalidOperation)?;\n            for value in values {{ let mut block = <{block_type} as GeneratedBlock>::new_generated(); if let libremetaverse_structured_data::OSD::Map(values) = value {{ let map = libremetaverse_structured_data::OSDMap::new_with_dictionary(values.clone())?; GeneratedBlock::from_osd(&mut block, &map)?; }} blocks.push(block); }}\n            self.{field_name} = blocks;"
+                );
+            }
+            BlockRepetition::Multiple(count) => {
+                let _ = writeln!(
+                    body,
+                    "            if self.{field_name}.len() != {count} {{ self.{field_name} = (0..{count}).map(|_| <{block_type} as GeneratedBlock>::new_generated()).collect(); }}\n            for (block, value) in self.{field_name}.iter_mut().zip(values.iter().take({count})) {{ if let libremetaverse_structured_data::OSD::Map(values) = value {{ let map = libremetaverse_structured_data::OSDMap::new_with_dictionary(values.clone())?; GeneratedBlock::from_osd(block, &map)?; }} }}"
+                );
+            }
+        }
+        body.push_str("        }\n");
+    }
+    if packet.blocks.is_empty() {
+        body.push_str("        let _ = body;\n");
+    }
+    body.push_str("        Ok(())\n    }\n\n");
+}
+
 #[allow(clippy::too_many_lines)]
 fn append_packet_impls(body: &mut String, packets: &[&PacketDefinition]) {
     for packet in packets {
@@ -1553,6 +1751,7 @@ fn append_packet_impls(body: &mut String, packets: &[&PacketDefinition]) {
             body.push_str("        let _ = writer;\n");
         }
         body.push_str("        Ok(())\n    }\n\n");
+        append_packet_osd(body, packet);
         append_packet_encode_multiple(body, packet);
         body.push_str("}\n\n");
     }
@@ -1562,6 +1761,13 @@ fn append_packet_codec_tests(body: &mut String, packets: &[&PacketDefinition]) {
     body.push_str(
         "#[cfg(test)]\nmod generated_codec_tests {\n    use super::{GeneratedBlock, GeneratedPacket};\n\n    fn assert_round_trip<T: GeneratedPacket>(packet: &T, name: &str) {\n        let expected = packet.encode_packet().unwrap_or_else(|error| panic!(\"{name}: encode failed: {error}\"));\n        let (wire, mut zero_buffer) = if expected[0] & crate::Helpers::MSG_ZEROCODED != 0 {\n            let mut encoded = vec![0_u8; expected.len().saturating_mul(2).saturating_add(2)];\n            let encoded_length = crate::packet_wire::zero_encode(\n                Some(&expected),\n                i32::try_from(expected.len()).expect(\"packet length fits i32\"),\n                Some(&mut encoded),\n            )\n            .unwrap_or_else(|error| panic!(\"{name}: zero encode failed: {error}\"));\n            encoded.truncate(usize::try_from(encoded_length).expect(\"encoded length fits usize\"));\n            (encoded, vec![0_u8; expected.len()])\n        } else {\n            (expected.clone(), Vec::new())\n        };\n        let mut decoded = T::new_generated();\n        let mut position = 0_i32;\n        let mut packet_end = i32::try_from(wire.len()).expect(\"wire length fits i32\") - 1;\n        let buffer = (!zero_buffer.is_empty()).then_some(zero_buffer.as_mut_slice());\n        decoded\n            .decode_from_bytes(&wire, &mut position, &mut packet_end, buffer)\n            .unwrap_or_else(|error| panic!(\"{name}: decode failed: {error}\"));\n        let actual = decoded.encode_packet().unwrap_or_else(|error| panic!(\"{name}: re-encode failed: {error}\"));\n        assert_eq!(actual, expected, \"{name}: round-trip bytes differ\");\n        assert_eq!(position, i32::try_from(expected.len()).expect(\"packet length fits i32\"), \"{name}: final position\");\n\n        for truncated_length in 0..wire.len() {\n            let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {\n                let mut candidate = T::new_generated();\n                let mut truncated_position = 0_i32;\n                let mut truncated_end = i32::try_from(truncated_length).expect(\"truncated length fits i32\") - 1;\n                let mut scratch = vec![0_u8; expected.len()];\n                let truncated_buffer = (!zero_buffer.is_empty()).then_some(scratch.as_mut_slice());\n                candidate.decode_from_bytes(\n                    &wire[..truncated_length],\n                    &mut truncated_position,\n                    &mut truncated_end,\n                    truncated_buffer,\n                )\n            }));\n            assert!(outcome.is_ok(), \"{name}: truncation at {truncated_length} panicked\");\n        }\n    }\n\n    #[test]\n    #[allow(clippy::too_many_lines)]\n    fn every_generated_packet_round_trips_all_fields() {\n",
     );
+    let truncation_loop = body
+        .rfind("        for truncated_length in 0..wire.len()")
+        .expect("packet test scaffold contains truncation loop");
+    body.insert_str(
+        truncation_loop,
+        "        let expected_osd = packet.packet_to_osd().unwrap_or_else(|error| panic!(\"{name}: OSD serialization failed: {error}\"));\n        let mut osd_round_trip = T::new_generated();\n        osd_round_trip.packet_from_osd(&expected_osd).unwrap_or_else(|error| panic!(\"{name}: OSD deserialization failed: {error}\"));\n        assert_eq!(osd_round_trip.encode_packet().unwrap_or_else(|error| panic!(\"{name}: OSD re-encode failed: {error}\")), expected, \"{name}: OSD round-trip bytes differ\");\n        assert_eq!(osd_round_trip.packet_to_osd().unwrap_or_else(|error| panic!(\"{name}: OSD reserialization failed: {error}\")).snapshot(), expected_osd.snapshot(), \"{name}: OSD round-trip differs\");\n\n",
+    );
     for packet in packets {
         let mutates_packet = packet
             .blocks
@@ -1856,6 +2062,7 @@ pub fn packet_catalog_bytes(root: &Path) -> Result<Vec<u8>, String> {
     validate_packet_api_catalog(root, &protocol)?;
     let packets = ordered_packets(&protocol);
     let mut body = String::new();
+    body.push_str("#![allow(clippy::cast_possible_truncation, clippy::cast_possible_wrap, clippy::cast_sign_loss, clippy::collapsible_if, clippy::similar_names, clippy::too_many_lines, clippy::unnecessary_wraps, clippy::vec_init_then_push, clippy::wrong_self_convention)]\n\n");
     let _ = writeln!(
         body,
         "pub const GOLDEN_GENERATOR_SOURCE: &str = {:?};",
@@ -2047,33 +2254,42 @@ pub fn source_manifest_bytes(root: &Path) -> Result<Vec<u8>, String> {
     Ok(generated_rust("source-manifest", &inputs, &body))
 }
 
-pub fn regenerate(root: &Path, check: bool) -> Result<(), String> {
-    let outputs = [
-        (root.join(MANIFEST_OUTPUT), source_manifest_bytes(root)?),
-        (root.join(PACKET_OUTPUT), packet_catalog_bytes(root)?),
+fn generated_outputs(root: &Path) -> Result<Vec<(&'static str, Vec<u8>)>, String> {
+    Ok(vec![
+        (MANIFEST_OUTPUT, source_manifest_bytes(root)?),
+        (PACKET_OUTPUT, packet_catalog_bytes(root)?),
+        (VISUAL_OUTPUT, xml_catalog::visual_catalog_bytes(root)?),
+        (FOLIAGE_OUTPUT, xml_catalog::foliage_catalog_bytes(root)?),
         (
-            root.join(VISUAL_OUTPUT),
-            xml_catalog::visual_catalog_bytes(root)?,
-        ),
-        (
-            root.join(FOLIAGE_OUTPUT),
-            xml_catalog::foliage_catalog_bytes(root)?,
-        ),
-        (
-            root.join(SKELETON_OUTPUT),
+            SKELETON_OUTPUT,
             avatar_catalog::skeleton_catalog_bytes(root)?,
         ),
         (
-            root.join(ATTENTION_OUTPUT),
+            ATTENTION_OUTPUT,
             avatar_catalog::attention_catalog_bytes(root)?,
         ),
         (
-            root.join(GENEPOOL_OUTPUT),
+            GENEPOOL_OUTPUT,
             avatar_catalog::genepool_catalog_bytes(root)?,
         ),
-    ];
+    ])
+}
+
+pub fn regenerate_to(root: &Path, output_root: &Path) -> Result<(), String> {
+    for (relative, expected) in generated_outputs(root)? {
+        let path = output_root.join(relative);
+        fs::create_dir_all(path.parent().ok_or("generated output has no parent")?)
+            .map_err(|error| error.to_string())?;
+        fs::write(&path, expected).map_err(|error| format!("{}: {error}", path.display()))?;
+    }
+    Ok(())
+}
+
+pub fn regenerate(root: &Path, check: bool) -> Result<(), String> {
+    let outputs = generated_outputs(root)?;
     if check {
-        for (path, expected) in outputs {
+        for (relative, expected) in outputs {
+            let path = root.join(relative);
             let actual = fs::read(&path).map_err(|error| format!("{}: {error}", path.display()))?;
             if actual != expected {
                 return Err(format!(
@@ -2083,7 +2299,8 @@ pub fn regenerate(root: &Path, check: bool) -> Result<(), String> {
             }
         }
     } else {
-        for (path, expected) in outputs {
+        for (relative, expected) in outputs {
+            let path = root.join(relative);
             fs::create_dir_all(path.parent().ok_or("generated output has no parent")?)
                 .map_err(|error| error.to_string())?;
             fs::write(&path, expected).map_err(|error| format!("{}: {error}", path.display()))?;

tools/codegen/src/main.rs

@@ -1,4 +1,4 @@
-use libremetaverse_codegen::{regenerate, vendor_inputs, workspace_root};
+use libremetaverse_codegen::{regenerate, regenerate_to, vendor_inputs, workspace_root};
 use std::path::Path;
 
 fn main() {
@@ -7,9 +7,10 @@ fn main() {
     let result = match arguments.as_slice() {
         [command] if command == "generate" => regenerate(&root, false),
         [command] if command == "check" => regenerate(&root, true),
+        [command, output] if command == "generate-to" => regenerate_to(&root, Path::new(output)),
         [command, reference] if command == "vendor" => vendor_inputs(&root, Path::new(reference)),
         _ => Err(
-            "usage: libremetaverse-codegen <generate|check|vendor REFERENCE_CHECKOUT>".to_owned(),
+            "usage: libremetaverse-codegen <generate|check|generate-to OUTPUT_ROOT|vendor REFERENCE_CHECKOUT>".to_owned(),
         ),
     };
     if let Err(error) = result {

tools/generate_api_shims.py

@@ -869,6 +869,16 @@ def native_member_body(member_id: str) -> str | None:
         member_id,
     ):
         return "<Self as crate::packet_catalog::GeneratedBlock>::encode_to_slice(self, bytes, i)"
+    if re.fullmatch(
+        r"M:LibreMetaverse\.Packets\.[^.]+Packet\.[^.]+Block\.FromOSD\(LibreMetaverse\.StructuredData\.OSDMap\)",
+        member_id,
+    ):
+        return "<Self as crate::packet_catalog::GeneratedBlock>::from_osd(self, &map)"
+    if re.fullmatch(
+        r"M:LibreMetaverse\.Packets\.[^.]+Packet\.[^.]+Block\.ToOSD",
+        member_id,
+    ):
+        return "<Self as crate::packet_catalog::GeneratedBlock>::to_osd(self)"
     if re.fullmatch(
         r"M:LibreMetaverse\.Packets\.[^.]+Packet\.FromBytes\(LibreMetaverse\.Packets\.Header,System\.Byte\[\],System\.Int32@,System\.Int32@\)",
         member_id,
@@ -881,6 +891,13 @@ def native_member_body(member_id: str) -> str | None:
         return "<Self as crate::packet_catalog::GeneratedPacket>::decode_from_bytes(self, &bytes, i, packet_end, zero_buffer)"
     if re.fullmatch(r"M:LibreMetaverse\.Packets\.[^.]+Packet\.ToBytes", member_id):
         return "<Self as crate::packet_catalog::GeneratedPacket>::encode_packet(self)"
+    if re.fullmatch(
+        r"M:LibreMetaverse\.Packets\.[^.]+Packet\.PacketFromOSD\(LibreMetaverse\.StructuredData\.OSDMap\)",
+        member_id,
+    ):
+        return "<Self as crate::packet_catalog::GeneratedPacket>::packet_from_osd(self, &body)"
+    if re.fullmatch(r"M:LibreMetaverse\.Packets\.[^.]+Packet\.PacketToOSD", member_id):
+        return "<Self as crate::packet_catalog::GeneratedPacket>::packet_to_osd(self)"
     if re.fullmatch(
         r"M:LibreMetaverse\.Packets\.[^.]+Packet\.ToBytes\(LibreMetaverse\.Interfaces\.IByteBufferPool,System\.Int32@\)",
         member_id,
@@ -903,6 +920,11 @@ def native_member_body(member_id: str) -> str | None:
         return "Ok(crate::packet_catalog::packet_type(id, frequency))"
     if member_id == "M:LibreMetaverse.Packets.Packet.BuildPacket(LibreMetaverse.Packets.PacketType)":
         return "crate::packet_catalog::build_packet(type_)"
+    if member_id in {
+        "M:LibreMetaverse.Packets.Packet.BuildPacket(System.String,LibreMetaverse.StructuredData.OSDMap)",
+        "M:LibreMetaverse.Packets.Packet.BuildPacketFromOSD(System.String,LibreMetaverse.StructuredData.OSDMap)",
+    }:
+        return "crate::packet_catalog::build_packet_from_osd(&caps_event_name, &body)"
     if member_id == "M:LibreMetaverse.Packets.Packet.BuildPacket(System.Byte[],System.Int32@,System.Byte[])":
         return "{ let mut zero_buffer = zero_buffer; crate::packet_wire::build_packet_from_bytes(&packet_buffer, packet_end, &mut zero_buffer) }"
     if member_id == "M:LibreMetaverse.Packets.Header.BuildHeader(System.Byte[],System.Int32@,System.Int32@)":
@@ -913,6 +935,10 @@ def native_member_body(member_id: str) -> str | None:
         return "crate::packet_wire::header_to_slice(self, bytes, i)"
     if member_id == "M:LibreMetaverse.Packets.Header.AcksToBytes(System.Byte[],System.Int32@)":
         return "crate::packet_wire::acks_to_slice(self, bytes, i)"
+    if member_id == "M:LibreMetaverse.Packets.Packet.PacketFromOSD(LibreMetaverse.StructuredData.OSDMap)":
+        return "crate::packet_catalog::ignore_packet_osd(&osd)"
+    if member_id == "M:LibreMetaverse.Packets.Packet.PacketToOSD":
+        return "crate::packet_catalog::empty_packet_osd()"
     if member_id == "M:LibreMetaverse.Helpers.ZeroDecode(System.Byte[],System.Int32,System.Byte[])":
         return "crate::packet_wire::zero_decode(src, srclen, dest)"
     if member_id == "M:LibreMetaverse.Helpers.ZeroEncode(System.Byte[],System.Int32,System.Byte[])":

tools/generate_rust_mapping.py

@@ -1164,8 +1164,8 @@ def method_receiver(owner: dict, item: dict) -> tuple[str, str]:
         return "&self", "shared_self"
     if owner["doc_id"] == "T:LibreMetaverse.BitPack" and item["name"].startswith(("Pack", "Unpack")):
         return "&mut self", "mutable_self"
-    if owner["doc_id"].startswith("T:LibreMetaverse.Packets.") and item["name"].startswith(
-        "FromBytes"
+    if owner["doc_id"].startswith("T:LibreMetaverse.Packets.") and (
+        item["name"].startswith("FromBytes") or item["name"] in {"FromOSD", "PacketFromOSD"}
     ):
         return "&mut self", "mutable_self"
     observers = ("Get", "TryGet", "Find", "Contains", "Has", "Is", "Can", "To", "Equals", "Compare")