Complete native codegen milestone gate (#50)

tools/codegen/src/lib.rs

@@ -23,6 +23,15 @@ pub const SKELETON_OUTPUT: &str = "crates/libremetaverse/src/skeleton_catalog.rs
 pub const ATTENTION_OUTPUT: &str = "crates/libremetaverse/src/attention_catalog.rs";
 pub const GENEPOOL_OUTPUT: &str = "crates/libremetaverse/src/genepool_catalog.rs";
 pub const PUBLIC_API_PATH: &str = "api/public-api.json";
+pub const GENERATED_OUTPUT_PATHS: [&str; 7] = [
+    MANIFEST_OUTPUT,
+    PACKET_OUTPUT,
+    VISUAL_OUTPUT,
+    FOLIAGE_OUTPUT,
+    SKELETON_OUTPUT,
+    ATTENTION_OUTPUT,
+    GENEPOOL_OUTPUT,
+];
 const RUST_KEYWORDS: &[&str] = &[
     "as", "break", "const", "continue", "crate", "else", "enum", "extern", "false", "fn", "for",
     "if", "impl", "in", "let", "loop", "match", "mod", "move", "mut", "pub", "ref", "return",
@@ -929,6 +938,74 @@ fn field_encode_statement(field: &PacketField) -> String {
     }
 }
 
+fn field_to_osd_expression(field: &PacketField) -> String {
+    let name = rust_snake(&field.name);
+    match field.kind {
+        FieldKind::Bool => {
+            format!("libremetaverse_structured_data::OSD::Boolean(self.{name})")
+        }
+        FieldKind::U8 | FieldKind::U16 | FieldKind::IpPort | FieldKind::S8 | FieldKind::S16 => {
+            format!("libremetaverse_structured_data::OSD::Integer(i32::from(self.{name}))")
+        }
+        FieldKind::S32 => {
+            format!("libremetaverse_structured_data::OSD::Integer(self.{name})")
+        }
+        FieldKind::U32 | FieldKind::IpAddr | FieldKind::U64 => format!(
+            "libremetaverse_structured_data::OSD::Binary(self.{name}.to_le_bytes().to_vec())"
+        ),
+        FieldKind::F32 => {
+            format!("libremetaverse_structured_data::OSD::Real(f64::from(self.{name}))")
+        }
+        FieldKind::F64 => {
+            format!("libremetaverse_structured_data::OSD::Real(self.{name})")
+        }
+        FieldKind::LlUuid => {
+            format!("libremetaverse_structured_data::OSD::UUID(self.{name})")
+        }
+        FieldKind::LlVector3 => {
+            format!("libremetaverse_structured_data::OSD::from_vector3(self.{name})?")
+        }
+        FieldKind::LlVector3d => {
+            format!("libremetaverse_structured_data::OSD::from_vector3d(self.{name})?")
+        }
+        FieldKind::LlVector4 => {
+            format!("libremetaverse_structured_data::OSD::from_vector4(self.{name})?")
+        }
+        FieldKind::LlQuaternion => {
+            format!("libremetaverse_structured_data::OSD::from_quaternion(self.{name})?")
+        }
+        FieldKind::Variable | FieldKind::Fixed => {
+            format!("libremetaverse_structured_data::OSD::Binary(self.{name}.clone())")
+        }
+    }
+}
+
+fn field_from_osd_statement(field: &PacketField) -> String {
+    let name = rust_snake(&field.name);
+    let conversion = match field.kind {
+        FieldKind::Bool => "value.as_boolean()?".to_owned(),
+        FieldKind::U8 => "value.as_integer()? as u8".to_owned(),
+        FieldKind::U16 | FieldKind::IpPort => "value.as_integer()? as u16".to_owned(),
+        FieldKind::U32 | FieldKind::IpAddr => "osd_u32(&value)?".to_owned(),
+        FieldKind::U64 => "osd_u64(&value)?".to_owned(),
+        FieldKind::S8 => "value.as_integer()? as i8".to_owned(),
+        FieldKind::S16 => "value.as_integer()? as i16".to_owned(),
+        FieldKind::S32 => "value.as_integer()?".to_owned(),
+        FieldKind::F32 => "value.as_real()? as f32".to_owned(),
+        FieldKind::F64 => "value.as_real()?".to_owned(),
+        FieldKind::LlUuid => "value.as_uuid()?".to_owned(),
+        FieldKind::LlVector3 => "value.as_vector3()?".to_owned(),
+        FieldKind::LlVector3d => "value.as_vector3d()?".to_owned(),
+        FieldKind::LlVector4 => "value.as_vector4()?".to_owned(),
+        FieldKind::LlQuaternion => "value.as_quaternion()?".to_owned(),
+        FieldKind::Variable | FieldKind::Fixed => "value.as_binary()?".to_owned(),
+    };
+    format!(
+        "if let Some(value) = map.get({:?}) {{ self.{name} = {conversion}; }}",
+        field.name
+    )
+}
+
 fn ordered_packets(protocol: &PacketProtocol) -> Vec<&PacketDefinition> {
     [
         PacketFrequency::Low,
@@ -1088,11 +1165,28 @@ fn append_dispatch(body: &mut String, packets: &[&PacketDefinition]) {
              crate::packets::Header { ack_list: None, appended_acks: false, frequency, id, reliable: true, resent: false, sequence: 0, zerocoded }\n\
          }\n\n\
          fn usize_to_i32(value: usize) -> i32 { i32::try_from(value).unwrap_or(i32::MAX) }\n\n\
+         fn osd_u32(value: &libremetaverse_structured_data::OSD) -> Result<u32, crate::Error> {\n\
+             let bytes = value.as_binary()?;\n\
+             Ok(bytes.get(..4).and_then(|value| value.try_into().ok()).map_or(0, u32::from_le_bytes))\n\
+         }\n\n\
+         fn osd_u64(value: &libremetaverse_structured_data::OSD) -> Result<u64, crate::Error> {\n\
+             let bytes = value.as_binary()?;\n\
+             Ok(bytes.get(..8).and_then(|value| value.try_into().ok()).map_or(0, u64::from_le_bytes))\n\
+         }\n\n\
+         pub(crate) fn empty_packet_osd() -> Result<libremetaverse_structured_data::OSDMap, crate::Error> {\n\
+             libremetaverse_structured_data::OSDMap::new_with_constructor()\n\
+         }\n\n\
+         pub(crate) fn ignore_packet_osd(body: &libremetaverse_structured_data::OSDMap) -> Result<(), crate::Error> {\n\
+             let _ = body;\n\
+             Ok(())\n\
+         }\n\n\
          pub(crate) trait GeneratedBlock: Sized {\n\
              fn new_generated() -> Self;\n\
              fn generated_length(&self) -> i32;\n\
              fn decode_payload(&mut self, reader: &mut crate::packet_wire::WireReader<'_>) -> Result<(), crate::Error>;\n\
              fn encode_payload(&self, writer: &mut crate::packet_wire::WireWriter) -> Result<(), crate::Error>;\n\
+             fn to_osd(&self) -> Result<libremetaverse_structured_data::OSDMap, crate::Error>;\n\
+             fn from_osd(&mut self, map: &libremetaverse_structured_data::OSDMap) -> Result<(), crate::Error>;\n\
              fn new_from_bytes(bytes: &[u8], position: &mut i32) -> Result<Self, crate::Error> {\n\
                  let mut value = Self::new_generated();\n\
                  Self::decode_from_bytes(&mut value, bytes, position)?;\n\
@@ -1127,6 +1221,8 @@ fn append_dispatch(body: &mut String, packets: &[&PacketDefinition]) {
              fn store_header(&mut self, header: crate::packets::Header);\n\
              fn decode_payload(&mut self, reader: &mut crate::packet_wire::WireReader<'_>) -> Result<(), crate::Error>;\n\
              fn encode_payload(&self, writer: &mut crate::packet_wire::WireWriter) -> Result<(), crate::Error>;\n\
+             fn packet_to_osd(&self) -> Result<libremetaverse_structured_data::OSDMap, crate::Error>;\n\
+             fn packet_from_osd(&mut self, body: &libremetaverse_structured_data::OSDMap) -> Result<(), crate::Error>;\n\
              fn encode_multiple(&self) -> Result<Vec<Vec<u8>>, crate::Error>;\n\
              fn encode_packet(&self) -> Result<Vec<u8>, crate::Error> {\n\
                  let header = self.generated_header();\n\
@@ -1185,6 +1281,19 @@ fn append_dispatch(body: &mut String, packets: &[&PacketDefinition]) {
     body.push_str(
         "        PacketType::Default => Err(crate::Error::InvalidOperation),\n    }\n}\n\n",
     );
+    body.push_str(
+        "#[allow(clippy::too_many_lines)]\npub(crate) fn build_packet_from_osd(\n    name: &str,\n    body: &libremetaverse_structured_data::OSDMap,\n) -> Result<Option<crate::packets::Packet>, crate::Error> {\n    let Some(descriptor) = descriptor_by_name(name) else { return Ok(None); };\n    match descriptor.packet_type {\n",
+    );
+    for packet in packets {
+        let _ = writeln!(
+            body,
+            "        PacketType::{} => {{ let mut packet = <crate::packets::{}Packet as GeneratedPacket>::new_generated(); GeneratedPacket::packet_from_osd(&mut packet, body)?; }},",
+            packet.name, packet.name
+        );
+    }
+    body.push_str(
+        "        PacketType::Default => return Ok(None),\n    }\n    build_packet(descriptor.packet_type).map(Some)\n}\n\n",
+    );
 }
 
 fn append_block_impls(body: &mut String, packets: &[&PacketDefinition]) {
@@ -1234,6 +1343,22 @@ fn append_block_impls(body: &mut String, packets: &[&PacketDefinition]) {
             for field in &block.fields {
                 let _ = writeln!(body, "        {}", field_encode_statement(field));
             }
+            body.push_str("        Ok(())\n    }\n\n    fn to_osd(&self) -> Result<libremetaverse_structured_data::OSDMap, crate::Error> {\n        let map = libremetaverse_structured_data::OSDMap::new_with_constructor()?;\n");
+            for field in &block.fields {
+                let _ = writeln!(
+                    body,
+                    "        map.add_with_string_osd({:?}.to_owned(), {})?;",
+                    field.name,
+                    field_to_osd_expression(field)
+                );
+            }
+            body.push_str("        Ok(map)\n    }\n\n    fn from_osd(&mut self, map: &libremetaverse_structured_data::OSDMap) -> Result<(), crate::Error> {\n");
+            for field in &block.fields {
+                let _ = writeln!(body, "        {}", field_from_osd_statement(field));
+            }
+            if block.fields.is_empty() {
+                body.push_str("        let _ = map;\n");
+            }
             body.push_str("        Ok(())\n    }\n}\n\n");
         }
     }
@@ -1371,6 +1496,79 @@ fn append_packet_encode_multiple(body: &mut String, packet: &PacketDefinition) {
     body.push_str(" { break; }\n        }\n        Ok(packets)\n    }\n");
 }
 
+fn append_packet_osd(body: &mut String, packet: &PacketDefinition) {
+    body.push_str(
+        "    fn packet_to_osd(&self) -> Result<libremetaverse_structured_data::OSDMap, crate::Error> {\n        let body = libremetaverse_structured_data::OSDMap::new_with_constructor()?;\n",
+    );
+    for block in &packet.blocks {
+        let field_name = rust_snake(&block.name);
+        let capacity = match block.repetition {
+            BlockRepetition::Single => "1".to_owned(),
+            BlockRepetition::Multiple(_) | BlockRepetition::Variable => {
+                format!("self.{field_name}.len()")
+            }
+        };
+        let _ = writeln!(
+            body,
+            "        let mut {field_name}_values = Vec::with_capacity({capacity});"
+        );
+        match block.repetition {
+            BlockRepetition::Single => {
+                let _ = writeln!(
+                    body,
+                    "        {field_name}_values.push(libremetaverse_structured_data::OSD::Map(GeneratedBlock::to_osd(&self.{field_name})?.snapshot()));"
+                );
+            }
+            BlockRepetition::Multiple(_) | BlockRepetition::Variable => {
+                let _ = writeln!(
+                    body,
+                    "        for block in &self.{field_name} {{ {field_name}_values.push(libremetaverse_structured_data::OSD::Map(GeneratedBlock::to_osd(block)?.snapshot())); }}"
+                );
+            }
+        }
+        let _ = writeln!(
+            body,
+            "        body.add_with_string_osd({:?}.to_owned(), libremetaverse_structured_data::OSD::Array({field_name}_values))?;",
+            block.name
+        );
+    }
+    body.push_str("        Ok(body)\n    }\n\n    fn packet_from_osd(&mut self, body: &libremetaverse_structured_data::OSDMap) -> Result<(), crate::Error> {\n");
+    for block in &packet.blocks {
+        let field_name = rust_snake(&block.name);
+        let block_type = format!("crate::packets::{}Packet{}Block", packet.name, block.name);
+        let _ = writeln!(
+            body,
+            "        if let Some(libremetaverse_structured_data::OSD::Array(values)) = body.get({:?}) {{",
+            block.name
+        );
+        match block.repetition {
+            BlockRepetition::Single => {
+                let _ = writeln!(
+                    body,
+                    "            if let Some(libremetaverse_structured_data::OSD::Map(values)) = values.first() {{ let map = libremetaverse_structured_data::OSDMap::new_with_dictionary(values.clone())?; GeneratedBlock::from_osd(&mut self.{field_name}, &map)?; }}"
+                );
+            }
+            BlockRepetition::Variable => {
+                let _ = writeln!(
+                    body,
+                    "            let mut blocks = Vec::new();\n            blocks.try_reserve_exact(values.len()).map_err(|_| crate::Error::InvalidOperation)?;\n            for value in values {{ let mut block = <{block_type} as GeneratedBlock>::new_generated(); if let libremetaverse_structured_data::OSD::Map(values) = value {{ let map = libremetaverse_structured_data::OSDMap::new_with_dictionary(values.clone())?; GeneratedBlock::from_osd(&mut block, &map)?; }} blocks.push(block); }}\n            self.{field_name} = blocks;"
+                );
+            }
+            BlockRepetition::Multiple(count) => {
+                let _ = writeln!(
+                    body,
+                    "            if self.{field_name}.len() != {count} {{ self.{field_name} = (0..{count}).map(|_| <{block_type} as GeneratedBlock>::new_generated()).collect(); }}\n            for (block, value) in self.{field_name}.iter_mut().zip(values.iter().take({count})) {{ if let libremetaverse_structured_data::OSD::Map(values) = value {{ let map = libremetaverse_structured_data::OSDMap::new_with_dictionary(values.clone())?; GeneratedBlock::from_osd(block, &map)?; }} }}"
+                );
+            }
+        }
+        body.push_str("        }\n");
+    }
+    if packet.blocks.is_empty() {
+        body.push_str("        let _ = body;\n");
+    }
+    body.push_str("        Ok(())\n    }\n\n");
+}
+
 #[allow(clippy::too_many_lines)]
 fn append_packet_impls(body: &mut String, packets: &[&PacketDefinition]) {
     for packet in packets {
@@ -1553,6 +1751,7 @@ fn append_packet_impls(body: &mut String, packets: &[&PacketDefinition]) {
             body.push_str("        let _ = writer;\n");
         }
         body.push_str("        Ok(())\n    }\n\n");
+        append_packet_osd(body, packet);
         append_packet_encode_multiple(body, packet);
         body.push_str("}\n\n");
     }
@@ -1562,6 +1761,13 @@ fn append_packet_codec_tests(body: &mut String, packets: &[&PacketDefinition]) {
     body.push_str(
         "#[cfg(test)]\nmod generated_codec_tests {\n    use super::{GeneratedBlock, GeneratedPacket};\n\n    fn assert_round_trip<T: GeneratedPacket>(packet: &T, name: &str) {\n        let expected = packet.encode_packet().unwrap_or_else(|error| panic!(\"{name}: encode failed: {error}\"));\n        let (wire, mut zero_buffer) = if expected[0] & crate::Helpers::MSG_ZEROCODED != 0 {\n            let mut encoded = vec![0_u8; expected.len().saturating_mul(2).saturating_add(2)];\n            let encoded_length = crate::packet_wire::zero_encode(\n                Some(&expected),\n                i32::try_from(expected.len()).expect(\"packet length fits i32\"),\n                Some(&mut encoded),\n            )\n            .unwrap_or_else(|error| panic!(\"{name}: zero encode failed: {error}\"));\n            encoded.truncate(usize::try_from(encoded_length).expect(\"encoded length fits usize\"));\n            (encoded, vec![0_u8; expected.len()])\n        } else {\n            (expected.clone(), Vec::new())\n        };\n        let mut decoded = T::new_generated();\n        let mut position = 0_i32;\n        let mut packet_end = i32::try_from(wire.len()).expect(\"wire length fits i32\") - 1;\n        let buffer = (!zero_buffer.is_empty()).then_some(zero_buffer.as_mut_slice());\n        decoded\n            .decode_from_bytes(&wire, &mut position, &mut packet_end, buffer)\n            .unwrap_or_else(|error| panic!(\"{name}: decode failed: {error}\"));\n        let actual = decoded.encode_packet().unwrap_or_else(|error| panic!(\"{name}: re-encode failed: {error}\"));\n        assert_eq!(actual, expected, \"{name}: round-trip bytes differ\");\n        assert_eq!(position, i32::try_from(expected.len()).expect(\"packet length fits i32\"), \"{name}: final position\");\n\n        for truncated_length in 0..wire.len() {\n            let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {\n                let mut candidate = T::new_generated();\n                let mut truncated_position = 0_i32;\n                let mut truncated_end = i32::try_from(truncated_length).expect(\"truncated length fits i32\") - 1;\n                let mut scratch = vec![0_u8; expected.len()];\n                let truncated_buffer = (!zero_buffer.is_empty()).then_some(scratch.as_mut_slice());\n                candidate.decode_from_bytes(\n                    &wire[..truncated_length],\n                    &mut truncated_position,\n                    &mut truncated_end,\n                    truncated_buffer,\n                )\n            }));\n            assert!(outcome.is_ok(), \"{name}: truncation at {truncated_length} panicked\");\n        }\n    }\n\n    #[test]\n    #[allow(clippy::too_many_lines)]\n    fn every_generated_packet_round_trips_all_fields() {\n",
     );
+    let truncation_loop = body
+        .rfind("        for truncated_length in 0..wire.len()")
+        .expect("packet test scaffold contains truncation loop");
+    body.insert_str(
+        truncation_loop,
+        "        let expected_osd = packet.packet_to_osd().unwrap_or_else(|error| panic!(\"{name}: OSD serialization failed: {error}\"));\n        let mut osd_round_trip = T::new_generated();\n        osd_round_trip.packet_from_osd(&expected_osd).unwrap_or_else(|error| panic!(\"{name}: OSD deserialization failed: {error}\"));\n        assert_eq!(osd_round_trip.encode_packet().unwrap_or_else(|error| panic!(\"{name}: OSD re-encode failed: {error}\")), expected, \"{name}: OSD round-trip bytes differ\");\n        assert_eq!(osd_round_trip.packet_to_osd().unwrap_or_else(|error| panic!(\"{name}: OSD reserialization failed: {error}\")).snapshot(), expected_osd.snapshot(), \"{name}: OSD round-trip differs\");\n\n",
+    );
     for packet in packets {
         let mutates_packet = packet
             .blocks
@@ -1856,6 +2062,7 @@ pub fn packet_catalog_bytes(root: &Path) -> Result<Vec<u8>, String> {
     validate_packet_api_catalog(root, &protocol)?;
     let packets = ordered_packets(&protocol);
     let mut body = String::new();
+    body.push_str("#![allow(clippy::cast_possible_truncation, clippy::cast_possible_wrap, clippy::cast_sign_loss, clippy::collapsible_if, clippy::similar_names, clippy::too_many_lines, clippy::unnecessary_wraps, clippy::vec_init_then_push, clippy::wrong_self_convention)]\n\n");
     let _ = writeln!(
         body,
         "pub const GOLDEN_GENERATOR_SOURCE: &str = {:?};",
@@ -2047,33 +2254,42 @@ pub fn source_manifest_bytes(root: &Path) -> Result<Vec<u8>, String> {
     Ok(generated_rust("source-manifest", &inputs, &body))
 }
 
-pub fn regenerate(root: &Path, check: bool) -> Result<(), String> {
-    let outputs = [
-        (root.join(MANIFEST_OUTPUT), source_manifest_bytes(root)?),
-        (root.join(PACKET_OUTPUT), packet_catalog_bytes(root)?),
+fn generated_outputs(root: &Path) -> Result<Vec<(&'static str, Vec<u8>)>, String> {
+    Ok(vec![
+        (MANIFEST_OUTPUT, source_manifest_bytes(root)?),
+        (PACKET_OUTPUT, packet_catalog_bytes(root)?),
+        (VISUAL_OUTPUT, xml_catalog::visual_catalog_bytes(root)?),
+        (FOLIAGE_OUTPUT, xml_catalog::foliage_catalog_bytes(root)?),
         (
-            root.join(VISUAL_OUTPUT),
-            xml_catalog::visual_catalog_bytes(root)?,
-        ),
-        (
-            root.join(FOLIAGE_OUTPUT),
-            xml_catalog::foliage_catalog_bytes(root)?,
-        ),
-        (
-            root.join(SKELETON_OUTPUT),
+            SKELETON_OUTPUT,
             avatar_catalog::skeleton_catalog_bytes(root)?,
         ),
         (
-            root.join(ATTENTION_OUTPUT),
+            ATTENTION_OUTPUT,
             avatar_catalog::attention_catalog_bytes(root)?,
         ),
         (
-            root.join(GENEPOOL_OUTPUT),
+            GENEPOOL_OUTPUT,
             avatar_catalog::genepool_catalog_bytes(root)?,
         ),
-    ];
+    ])
+}
+
+pub fn regenerate_to(root: &Path, output_root: &Path) -> Result<(), String> {
+    for (relative, expected) in generated_outputs(root)? {
+        let path = output_root.join(relative);
+        fs::create_dir_all(path.parent().ok_or("generated output has no parent")?)
+            .map_err(|error| error.to_string())?;
+        fs::write(&path, expected).map_err(|error| format!("{}: {error}", path.display()))?;
+    }
+    Ok(())
+}
+
+pub fn regenerate(root: &Path, check: bool) -> Result<(), String> {
+    let outputs = generated_outputs(root)?;
     if check {
-        for (path, expected) in outputs {
+        for (relative, expected) in outputs {
+            let path = root.join(relative);
             let actual = fs::read(&path).map_err(|error| format!("{}: {error}", path.display()))?;
             if actual != expected {
                 return Err(format!(
@@ -2083,7 +2299,8 @@ pub fn regenerate(root: &Path, check: bool) -> Result<(), String> {
             }
         }
     } else {
-        for (path, expected) in outputs {
+        for (relative, expected) in outputs {
+            let path = root.join(relative);
             fs::create_dir_all(path.parent().ok_or("generated output has no parent")?)
                 .map_err(|error| error.to_string())?;
             fs::write(&path, expected).map_err(|error| format!("{}: {error}", path.display()))?;