Generate native packet catalog (#46)

.gitea/workflows/codegen.yml

@@ -24,7 +24,7 @@ jobs:
         run: |
           cargo run -p libremetaverse-codegen -- generate
           cargo run -p libremetaverse-codegen -- generate
-          git diff --exit-code -- codegen/generated
+          git diff --exit-code -- codegen/generated crates/libremetaverse/src/packet_catalog.rs
       - name: Lint native generator
         run: cargo clippy -p libremetaverse-codegen --all-targets -- -D warnings
       - name: Prove normal consumer build does not execute generation

Cargo.lock

@@ -374,9 +374,11 @@ dependencies = [
 name = "libremetaverse-codegen"
 version = "0.0.1"
 dependencies = [
+ "prettyplease",
  "serde",
  "serde_json",
  "sha2",
+ "syn 2.0.119",
 ]
 
 [[package]]

README.md

@@ -181,3 +181,10 @@ member ID and rejects unrelated fixture, assertion, compile, or symbol errors.
 The deterministic, Rust-only source-data generator framework and its complete
 pinned input inventory are documented in
 [`codegen/README.md`](codegen/README.md).
+The first native data generator now parses the pinned protocol template into a
+checked-in catalog covering all 483 packets, 905 blocks, exact field widths,
+repetition rules, flags, IDs, and frequencies. It supplies the public
+`PacketType`, deterministic lookup tables, and real default construction and
+sizing behavior for every mapped packet/block while validating the complete
+shape against the pinned compiled C# API catalog. Wire encoding and decoding
+remain the separately gated next codegen issue.

api/SHIM-COVERAGE.md

@@ -4,7 +4,7 @@ Generated by `python3 tools/generate_api_shims.py`; do not edit by hand.
 
 | Assembly | Types | Members | Status |
 |---|---:|---:|---|
-| `LibreMetaverse` | 2,711 | 27,281 | native implementation: 3 types / 7 members; remaining surface is callable failure-only shims |
+| `LibreMetaverse` | 2,711 | 27,281 | native implementation: 4 types / 3,752 members; remaining surface is callable failure-only shims |
 | `LibreMetaverse.Imaging.Abstractions` | 3 | 20 | native implementation: 3 types / 20 members; no generated shims remain |
 | `LibreMetaverse.Imaging.Skia` | 1 | 3 | native implementation: 1 type / 3 members; no generated shims remain |
 | `LibreMetaverse.LslTools` | 164 | 768 | callable failure-only shim |

codegen/README.md

@@ -31,6 +31,33 @@ copyright attribution are covered by the repository
 [`LICENSE.md`](../LICENSE.md). The generated source manifest repeats each hash
 and license in its header and data table.
 
+## Packet generation
+
+The packet generator tokenizes the pinned `message_template.msg` directly in
+Rust. It preserves source order, all 483 packet names, the 434 low/17 medium/32
+high frequency assignments, 16-bit IDs (including the low-frequency `Fixed`
+IDs), trust and zerocode flags, optional protocol flags, 905 blocks, repetition
+rules, and every field width. Braces, identifiers, numeric values, field kinds,
+prefix widths, duplicate names, and duplicate frequency/ID pairs are validated
+with stable source-located `PGxxx` diagnostics.
+
+[`packet_catalog.rs`](../crates/libremetaverse/src/packet_catalog.rs) is the
+checked-in native output. It contains the public `PacketType` discriminants,
+ordered immutable schema descriptors, name/type/frequency-ID dispatch tables,
+and native construction and sizing implementations for every mapped packet and
+block. Variable blocks start empty, fixed-repeat blocks receive their declared
+number of independently constructed elements, and scalar/vector/UUID fields
+receive the same zero defaults as the golden generator. The generated `Length`
+metadata intentionally preserves the golden generator's variable-block count
+semantics.
+
+Generation also compares all packet, block, field, field-type, and `PacketType`
+entries against the pinned compiled public API catalog. This rejects a template
+parser or naming change even if the emitted Rust would otherwise compile. The
+Rust API compile fixture then compiles every mapped generated member, while the
+packet catalog test exercises representative low/medium/high, fixed, multiple,
+variable, and unknown dispatch behavior.
+
 ## Commands
 
 Regenerate every currently registered Rust output using only checked-in data:
@@ -56,5 +83,5 @@ cargo run -p libremetaverse-codegen -- vendor ../libremetaverse
 Generation uses UTF-8 with an optional BOM, normalizes CRLF/CR to LF, rejects
 NUL input, reports stable `path:line:column: severity[code]` diagnostics, sorts
 manifest records by stable IDs, emits provenance-rich generated headers, and
-always terminates output with one LF. Tests generate twice in memory and
+always terminates output with one LF. Tests generate both outputs twice in
-require byte identity before comparing with the checked-in file.
+memory and require byte identity before comparing with the checked-in files.

crates/libremetaverse/src/lib.rs

@@ -4,6 +4,8 @@ extern crate self as libremetaverse;
 
 mod generated;
 mod j2k;
+#[rustfmt::skip]
+pub mod packet_catalog;
 mod targa;
 
 #[cfg(test)]

crates/libremetaverse/tests/packet_catalog.rs

@@ -0,0 +1,143 @@
+use libremetaverse::PacketFrequency;
+use libremetaverse::packet_catalog::{
+    BlockRepetition, FieldKind, PACKETS, descriptor_by_name, descriptor_by_type,
+};
+use libremetaverse::packets::{
+    Packet, PacketAckPacket, PacketType, TestMessagePacket, UseCircuitCodePacket,
+};
+use std::collections::BTreeSet;
+
+#[test]
+fn generated_catalog_matches_the_complete_pinned_protocol() {
+    assert_eq!(PACKETS.len(), 483);
+    assert_eq!(
+        PACKETS
+            .iter()
+            .filter(|packet| packet.frequency == PacketFrequency::Low)
+            .count(),
+        434
+    );
+    assert_eq!(
+        PACKETS
+            .iter()
+            .filter(|packet| packet.frequency == PacketFrequency::Medium)
+            .count(),
+        17
+    );
+    assert_eq!(
+        PACKETS
+            .iter()
+            .filter(|packet| packet.frequency == PacketFrequency::High)
+            .count(),
+        32
+    );
+
+    let names = PACKETS
+        .iter()
+        .map(|packet| packet.name)
+        .collect::<BTreeSet<_>>();
+    let ids = PACKETS
+        .iter()
+        .map(|packet| (packet.frequency as u8, packet.id))
+        .collect::<BTreeSet<_>>();
+    assert_eq!(names.len(), PACKETS.len());
+    assert_eq!(ids.len(), PACKETS.len());
+
+    for descriptor in PACKETS {
+        assert_eq!(descriptor_by_name(descriptor.name), Some(descriptor));
+        assert_eq!(descriptor_by_type(descriptor.packet_type), Some(descriptor));
+        assert_eq!(
+            Packet::get_type(descriptor.id, descriptor.frequency).unwrap(),
+            descriptor.packet_type
+        );
+    }
+}
+
+#[test]
+fn generated_ids_repetition_and_field_widths_match_golden_entries() {
+    assert_eq!(PacketType::TestMessage as i32, 0x1_0001);
+    assert_eq!(PacketType::PacketAck as i32, 0x1_fffb);
+    assert_eq!(PacketType::ObjectUpdate as i32, 0x3_000c);
+
+    let test = descriptor_by_name("TestMessage").unwrap();
+    assert_eq!(test.id, 1);
+    assert_eq!(test.blocks.len(), 2);
+    assert_eq!(test.blocks[0].repetition, BlockRepetition::Single);
+    assert_eq!(test.blocks[1].repetition, BlockRepetition::Multiple(4));
+    assert_eq!(test.blocks[1].fields.len(), 3);
+    assert!(
+        test.blocks[1]
+            .fields
+            .iter()
+            .all(|field| field.kind == FieldKind::U32 && field.fixed_width == 4)
+    );
+
+    let ack = descriptor_by_name("PacketAck").unwrap();
+    assert_eq!(ack.id, 0xfffb);
+    assert_eq!(ack.frequency, PacketFrequency::Low);
+    assert_eq!(ack.blocks[0].repetition, BlockRepetition::Variable);
+    assert_eq!(ack.blocks[0].fields[0].kind, FieldKind::U32);
+
+    let object_update = descriptor_by_name("ObjectUpdate").unwrap();
+    assert!(object_update.zerocoded);
+    assert!(object_update.trusted);
+    assert!(
+        object_update
+            .blocks
+            .iter()
+            .flat_map(|block| block.fields)
+            .any(|field| field.kind == FieldKind::Fixed)
+    );
+}
+
+#[test]
+fn generated_constructors_initialize_blocks_headers_and_sizing_metadata() {
+    let test = TestMessagePacket::new_with_constructor().unwrap();
+    assert_eq!(test.base.type_, PacketType::TestMessage);
+    assert_eq!(test.base.header.frequency, PacketFrequency::Low);
+    assert_eq!(test.base.header.id, 1);
+    assert!(test.base.header.reliable);
+    assert!(test.base.header.zerocoded);
+    assert_eq!(test.neighbor_block.len(), 4);
+    assert_eq!(test.length(), 62);
+    assert!(test.uses_buffer_pooling());
+
+    let ack = PacketAckPacket::new_with_constructor().unwrap();
+    assert!(ack.packets.is_empty());
+    // The source generator includes one variable-block metadata byte in the
+    // base and one in the block contribution; preserve that exact Length API.
+    assert_eq!(ack.length(), 12);
+    assert!(!ack.uses_buffer_pooling());
+
+    let circuit = UseCircuitCodePacket::new_with_constructor().unwrap();
+    assert_eq!(circuit.circuit_code.code, 0);
+    assert_eq!(circuit.circuit_code.id, libremetaverse_types::UUID::zero());
+    assert_eq!(
+        circuit.circuit_code.session_id,
+        libremetaverse_types::UUID::zero()
+    );
+    assert_eq!(circuit.length(), 46);
+    assert!(circuit.uses_buffer_pooling());
+}
+
+#[test]
+fn generated_dispatch_handles_known_and_unknown_packets_without_fallbacks() {
+    let packet = Packet::build_packet_with_packet_type(PacketType::UseCircuitCode).unwrap();
+    assert_eq!(packet.type_, PacketType::UseCircuitCode);
+    assert_eq!(packet.header.frequency, PacketFrequency::Low);
+    assert_eq!(packet.header.id, 3);
+    assert!(packet.header.reliable);
+    assert!(!packet.header.zerocoded);
+    assert!(!packet.has_variable_blocks);
+
+    assert_eq!(
+        Packet::get_type(u16::MAX, PacketFrequency::High).unwrap(),
+        PacketType::Default
+    );
+    assert!(matches!(
+        Packet::build_packet_with_packet_type(PacketType::Default),
+        Err(libremetaverse::Error::InvalidOperation)
+    ));
+    assert_eq!(descriptor_by_name("NotAPacket"), None);
+    assert_eq!(descriptor_by_type(PacketType::Default), None);
+}

tests/compat/tests/wire_shims.rs

@@ -16,10 +16,9 @@ fn member_id<T>(result: Result<T, libremetaverse::Error>) -> &'static str {
 
 #[test]
 fn each_wire_data_domain_is_callable_and_failure_only() {
-    assert_eq!(
+    let packet = UseCircuitCodePacket::new_with_constructor()
-        member_id(UseCircuitCodePacket::new_with_constructor()),
+        .expect("generated packet constructor is native");
-        "M:LibreMetaverse.Packets.UseCircuitCodePacket.#ctor"
+    assert_eq!(packet.circuit_code.code, 0);
-    );
     assert_eq!(
         member_id(AgentDropGroupMessage::new()),
         "M:LibreMetaverse.Messages.Linden.AgentDropGroupMessage.#ctor"

tests/red-suite-baseline.json

@@ -111,6 +111,7 @@
     "LibreMetaverse.Tests/NotationLLSDTests.cs::NotationSDTests.SerializeURI::test",
     "LibreMetaverse.Tests/NotationLLSDTests.cs::NotationSDTests.SerializeUUID::test",
     "LibreMetaverse.Tests/NotationLLSDTests.cs::NotationSDTests.SerializeUndef::test",
+    "LibreMetaverse.Tests/PacketTests.cs::PacketTests.HeaderFlags::test",
     "LibreMetaverse.Tests/ProtobufTests.cs::ProtobufTests.AutoDetectProtobuf::test",
     "LibreMetaverse.Tests/ProtobufTests.cs::ProtobufTests.SerializeArray::test",
     "LibreMetaverse.Tests/ProtobufTests.cs::ProtobufTests.SerializeBinary::test",
@@ -146,5 +147,5 @@
     "LibreMetaverse.Tests/XmlLLSDTests.cs::XmlSDTests.DeserializeUUID::test",
     "LibreMetaverse.Tests/XmlLLSDTests.cs::XmlSDTests.DeserializeUndef::test"
   ],
-  "support_passes": 137
+  "support_passes": 143
 }

tools/codegen/Cargo.toml

@@ -9,9 +9,11 @@ description = "Deterministic native generators for pinned LibreMetaverse data"
 publish = false
 
 [dependencies]
+prettyplease = "0.2"
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
 sha2 = "0.11"
+syn = { version = "2", features = ["full"] }
 
 [lints]
 workspace = true

tools/generate_api_shims.py

@@ -38,6 +38,7 @@ TARGETS = {
 # implementations. The generated module keeps catalog markers and re-exports
 # the hand-written type so coverage remains deterministic.
 NATIVE_TYPES = {
+    "T:LibreMetaverse.Packets.PacketType": "crate::packet_catalog::PacketType",
     "T:LibreMetaverse.PrimMesher.Coord": "crate::prim_mesher::Coord",
     "T:LibreMetaverse.PrimMesher.Face": "crate::prim_mesher::Face",
     "T:LibreMetaverse.PrimMesher.Path": "crate::prim_mesher::Path",
@@ -741,11 +742,32 @@ def render_enum(item: dict, rust_name: str) -> list[str]:
     return lines
 
 
+def native_member_body(member_id: str) -> str | None:
+    body = NATIVE_MEMBER_BODIES.get(member_id)
+    if body is not None:
+        return body
+    if re.fullmatch(r"M:LibreMetaverse\.Packets\.[^.]+Packet\.#ctor", member_id):
+        return "Ok(<Self as crate::packet_catalog::GeneratedPacket>::new_generated())"
+    if re.fullmatch(r"M:LibreMetaverse\.Packets\.[^.]+Packet\.[^.]+Block\.#ctor", member_id):
+        return "Ok(<Self as crate::packet_catalog::GeneratedBlock>::new_generated())"
+    if re.fullmatch(r"P:LibreMetaverse\.Packets\.[^.]+Packet\.Length", member_id):
+        return "<Self as crate::packet_catalog::GeneratedPacket>::generated_length(self)"
+    if re.fullmatch(r"P:LibreMetaverse\.Packets\.[^.]+Packet\.UsesBufferPooling", member_id):
+        return "<Self as crate::packet_catalog::GeneratedPacket>::USES_BUFFER_POOLING"
+    if re.fullmatch(r"P:LibreMetaverse\.Packets\.[^.]+Packet\.[^.]+Block\.Length", member_id):
+        return "<Self as crate::packet_catalog::GeneratedBlock>::generated_length(self)"
+    if member_id == "M:LibreMetaverse.Packets.Packet.GetType(System.UInt16,LibreMetaverse.PacketFrequency)":
+        return "Ok(crate::packet_catalog::packet_type(id, frequency))"
+    if member_id == "M:LibreMetaverse.Packets.Packet.BuildPacket(LibreMetaverse.Packets.PacketType)":
+        return "crate::packet_catalog::build_packet(type_)"
+    return None
+
+
 def render_body(signature: str, member_id: str, error_model: str, asyncness: str, trait: bool) -> str:
     if trait:
         signature = signature.removeprefix("pub ")
-    if member_id in NATIVE_MEMBER_BODIES:
+    if body := native_member_body(member_id):
-        return f"    {signature} {{ {NATIVE_MEMBER_BODIES[member_id]} }}"
+        return f"    {signature} {{ {body} }}"
     failure = (
         f"libremetaverse_types::not_implemented({json.dumps(member_id)})"
         if error_model.startswith("Result")
@@ -1035,7 +1057,7 @@ def coverage_report(catalog: dict, coverage: dict[str, tuple[int, int, bool]]) -
                 member["doc_id"]
                 for item in assembly["types"]
                 for member in item["members"]
-                if item["doc_id"] in native_type_ids or member["doc_id"] in NATIVE_MEMBER_BODIES
+                if item["doc_id"] in native_type_ids or native_member_body(member["doc_id"]) is not None
             }
             if native_type_ids or native_member_ids:
                 type_noun = "type" if len(native_type_ids) == 1 else "types"

tools/generate_rust_mapping.py

@@ -984,6 +984,7 @@ def validate_generated_shims() -> None:
                     "crate::dispatch::",
                     "crate::json_codec::",
                     "crate::notation::",
+                    "crate::packet_catalog::",
                     "crate::protobuf::",
                     "crate::xml_codec::",
                 )