Complete StructuredData parity and parser hardening (#37)

README.md

@@ -107,5 +107,20 @@ ZigZag `int32`, little-endian IEEE-754 fixed64 values, 16-byte UUIDs, bounded
 length-delimited containers, and wire-type-aware unknown-field skipping. The
 schema remains an internal compatibility format; native encoders emit stable
 sorted maps and accept the exact optional LLSD Protobuf header.
+Cross-format conversion follows the reference's format-specific boundaries:
+
+| Format | Round-trip behavior |
+| --- | --- |
+| Binary LLSD | All regular OSD variants are lossless; raw `OSDLlsdXml` is unsupported. |
+| Notation LLSD | All regular OSD variants are lossless; raw `OSDLlsdXml` is unsupported. |
+| XML LLSD | Regular variants are lossless; raw `OSDLlsdXml` is injected as an element and therefore decodes as that element's ordinary OSD value. |
+| JSON OSD | Booleans, integers, reals, nonempty strings, maps, and arrays retain structure; UUID, date, and URI become strings, binary becomes an integer array, and undefined, empty strings, and raw XML become JSON `null`. |
+| Protobuf OSD | Regular variants are lossless within the reference schema's date range and whole-second precision; raw `OSDLlsdXml` maps to undefined. |
+
+Checked-in malformed corpora and bounded deterministic mutations exercise all
+five parsers without an external fuzzing runtime. They cover truncation, tags,
+lengths, payloads, delimiters, nesting, entity rejection, contextual errors,
+and the parser resource limits. Every encoder sorts map keys where necessary,
+so serialization is deterministic across platforms.
 The controlled audit aggregates every expected failure by standardized C#
 member ID and rejects unrelated fixture, assertion, compile, or symbol errors.

api/SHIM-COVERAGE.md

@@ -12,7 +12,7 @@ Generated by `python3 tools/generate_api_shims.py`; do not edit by hand.
 | `LibreMetaverse.RLV` | 28 | 499 | callable failure-only shim |
 | `LibreMetaverse.Rendering.MeshFoundry` | 1 | 14 | callable failure-only shim |
 | `LibreMetaverse.Rendering.Simple` | 1 | 6 | callable failure-only shim |
-| `LibreMetaverse.StructuredData` | 16 | 295 | native implementation: 15 types / 295 members; no generated shims remain |
+| `LibreMetaverse.StructuredData` | 16 | 295 | native implementation: 16 types / 295 members; no generated shims remain |
 | `LibreMetaverse.Types` | 45 | 942 | native implementation: 45 types / 942 members; no generated shims remain |
 | `LibreMetaverse.Utilities` | 3 | 13 | callable failure-only shim |
 | `LibreMetaverse.Voice.Vivox` | 64 | 531 | callable failure-only shim |

crates/libremetaverse-structured-data/src/generated.rs

@@ -234,7 +234,7 @@ pub use crate::model::OSDLlsdXml;
 pub use crate::model::OSDMap;
 
 /// C# type: `T:LibreMetaverse.StructuredData.OSDParser`.
-pub struct OSDParser;
+pub use crate::model::OSDParser;
 impl OSDParser {
     /// C# member: `M:LibreMetaverse.StructuredData.OSDParser.BufferCharactersEqual(System.IO.StringReader,System.Char[],System.Int32)`.
     pub fn buffer_characters_equal(

crates/libremetaverse-structured-data/src/hardening.rs

@@ -0,0 +1,356 @@
+//! Cross-format compatibility and deterministic mutation tests.
+
+use crate::{Error, OSD};
+use libremetaverse_types::UUID;
+use libremetaverse_types::compat::Uri;
+use std::collections::HashMap;
+use std::panic::{AssertUnwindSafe, catch_unwind};
+use std::time::{Duration, UNIX_EPOCH};
+
+fn variants() -> Vec<OSD> {
+    vec![
+        OSD::Undefined,
+        OSD::Boolean(true),
+        OSD::Integer(-42),
+        OSD::Real(12.5),
+        OSD::String("text".into()),
+        OSD::UUID(UUID::new_with_string("97f4aeca-88a1-42a1-b385-b97b18abb255".into()).unwrap()),
+        OSD::Date(UNIX_EPOCH + Duration::from_secs(1_705_314_645)),
+        OSD::Uri(Uri("https://example.com/path".into())),
+        OSD::Binary(vec![0, 1, 255]),
+        OSD::Map(HashMap::from([("key".into(), OSD::Integer(7))])),
+        OSD::Array(vec![OSD::String("nested".into()), OSD::Boolean(false)]),
+        OSD::LlsdXml("<string>raw XML</string>".into()),
+    ]
+}
+
+#[test]
+fn every_osd_variant_has_documented_cross_format_behavior() {
+    for value in variants() {
+        if matches!(value, OSD::LlsdXml(_)) {
+            assert!(crate::binary::serialize(value.clone()).is_err());
+            assert!(crate::notation::serialize(value.clone()).is_err());
+        } else {
+            assert_eq!(
+                crate::binary::deserialize_bytes(crate::binary::serialize(value.clone()).unwrap())
+                    .unwrap(),
+                value
+            );
+            assert_eq!(
+                crate::notation::deserialize_string(
+                    crate::notation::serialize(value.clone()).unwrap()
+                )
+                .unwrap(),
+                value
+            );
+        }
+
+        let xml = crate::xml_codec::deserialize_bytes(
+            crate::xml_codec::serialize_bytes(value.clone()).unwrap(),
+        )
+        .unwrap();
+        let expected_xml = if matches!(value, OSD::LlsdXml(_)) {
+            OSD::String("raw XML".into())
+        } else {
+            value.clone()
+        };
+        assert_eq!(xml, expected_xml);
+
+        let json = crate::json_codec::deserialize_string(
+            crate::json_codec::serialize(value.clone(), Some(true)).unwrap(),
+        )
+        .unwrap();
+        assert_eq!(json, expected_json(&value));
+
+        let protobuf = crate::protobuf::deserialize_bytes(
+            crate::protobuf::serialize(value.clone(), Some(false)).unwrap(),
+        )
+        .unwrap();
+        let expected_protobuf = if matches!(value, OSD::LlsdXml(_)) {
+            OSD::Undefined
+        } else {
+            value
+        };
+        assert_eq!(protobuf, expected_protobuf);
+    }
+}
+
+#[test]
+fn every_encoder_is_deterministic_for_unordered_maps() {
+    let mut forward = HashMap::new();
+    forward.insert("a".into(), OSD::String("first".into()));
+    forward.insert("m".into(), OSD::Boolean(true));
+    forward.insert("z".into(), OSD::Integer(1));
+    let mut reverse = HashMap::new();
+    reverse.insert("z".into(), OSD::Integer(1));
+    reverse.insert("m".into(), OSD::Boolean(true));
+    reverse.insert("a".into(), OSD::String("first".into()));
+    let forward = OSD::Map(forward);
+    let reverse = OSD::Map(reverse);
+    assert_eq!(
+        crate::binary::serialize(forward.clone()).unwrap(),
+        crate::binary::serialize(reverse.clone()).unwrap()
+    );
+    assert_eq!(
+        crate::notation::serialize(forward.clone()).unwrap(),
+        crate::notation::serialize(reverse.clone()).unwrap()
+    );
+    assert_eq!(
+        crate::xml_codec::serialize_bytes(forward.clone()).unwrap(),
+        crate::xml_codec::serialize_bytes(reverse.clone()).unwrap()
+    );
+    assert_eq!(
+        crate::json_codec::serialize(forward.clone(), Some(true)).unwrap(),
+        crate::json_codec::serialize(reverse.clone(), Some(true)).unwrap()
+    );
+    assert_eq!(
+        crate::protobuf::serialize(forward, Some(false)).unwrap(),
+        crate::protobuf::serialize(reverse, Some(false)).unwrap()
+    );
+}
+
+#[test]
+fn malformed_corpora_and_focused_mutations_never_panic() {
+    for (name, seed) in hex_corpus(include_str!(
+        "../../../fuzz/corpus/binary_llsd/malformed.hex"
+    )) {
+        assert_no_panic(
+            &format!("binary corpus {name}"),
+            seed,
+            crate::binary::deserialize_bytes,
+        );
+    }
+    for (name, seed) in text_corpus(include_str!(
+        "../../../fuzz/corpus/notation_llsd/malformed.txt"
+    )) {
+        assert_no_panic(&format!("notation corpus {name}"), seed, notation_bytes);
+    }
+    for (name, seed) in text_corpus(include_str!("../../../fuzz/corpus/json_osd/malformed.txt")) {
+        assert_no_panic(&format!("JSON corpus {name}"), seed, json_bytes);
+    }
+    for (name, seed) in hex_corpus(include_str!(
+        "../../../fuzz/corpus/protobuf_osd/malformed.hex"
+    )) {
+        assert_no_panic(
+            &format!("Protobuf corpus {name}"),
+            seed,
+            crate::protobuf::deserialize_bytes,
+        );
+    }
+    for (name, seed) in [
+        (
+            "XML DTD",
+            include_bytes!("../../../fuzz/corpus/xml_llsd/doctype_entity.xml").as_slice(),
+        ),
+        (
+            "XML external entity",
+            include_bytes!("../../../fuzz/corpus/xml_llsd/external_entity.xml").as_slice(),
+        ),
+        (
+            "XML named entity",
+            include_bytes!("../../../fuzz/corpus/xml_llsd/unknown_entity.xml").as_slice(),
+        ),
+        (
+            "XML nesting",
+            include_bytes!("../../../fuzz/corpus/xml_llsd/malformed_nesting.xml").as_slice(),
+        ),
+        (
+            "XML truncated scalar",
+            include_bytes!("../../../fuzz/corpus/xml_llsd/truncated_scalar.xml").as_slice(),
+        ),
+    ] {
+        assert_no_panic(name, seed.to_vec(), crate::xml_codec::deserialize_bytes);
+    }
+
+    let value = OSD::Map(HashMap::from([
+        ("array".into(), OSD::Array(variants())),
+        ("binary".into(), OSD::Binary(vec![0, 1, 2, 255])),
+    ]));
+    let binary = crate::binary::serialize(value.clone()).unwrap_err();
+    assert!(matches!(binary, Error::Parse { .. }));
+    let fuzzable = OSD::Map(HashMap::from([
+        (
+            "array".into(),
+            OSD::Array(variants().into_iter().take(11).collect()),
+        ),
+        ("binary".into(), OSD::Binary(vec![0, 1, 2, 255])),
+    ]));
+    mutate(
+        "binary",
+        &crate::binary::serialize(fuzzable.clone()).unwrap(),
+        crate::binary::deserialize_bytes,
+    );
+    mutate(
+        "notation",
+        &crate::notation::serialize(fuzzable.clone())
+            .unwrap()
+            .into_bytes(),
+        notation_bytes,
+    );
+    mutate(
+        "XML",
+        &crate::xml_codec::serialize_bytes(fuzzable.clone()).unwrap(),
+        crate::xml_codec::deserialize_bytes,
+    );
+    mutate(
+        "JSON",
+        &crate::json_codec::serialize(fuzzable.clone(), Some(true))
+            .unwrap()
+            .into_bytes(),
+        json_bytes,
+    );
+    mutate(
+        "Protobuf",
+        &crate::protobuf::serialize(fuzzable, Some(false)).unwrap(),
+        crate::protobuf::deserialize_bytes,
+    );
+}
+
+#[test]
+fn malformed_errors_retain_positions_and_context() {
+    let cases = [
+        crate::binary::deserialize_bytes(vec![b'i']),
+        crate::notation::deserialize_string("s(4)\"x".into()),
+        crate::xml_codec::deserialize_string("<llsd><integer>1</real></llsd>".into()),
+        crate::json_codec::deserialize_string("[1,".into()),
+        crate::protobuf::deserialize_bytes(vec![0x08]),
+    ];
+    for result in cases {
+        match result {
+            Err(Error::Parse { position, context }) => {
+                assert!(position < OSD::DEFAULT_MAX_BINARY_BYTES);
+                assert!(!context.is_empty());
+            }
+            result => panic!("expected contextual parse error, got {result:?}"),
+        }
+    }
+}
+
+fn expected_json(value: &OSD) -> OSD {
+    match value {
+        OSD::Undefined | OSD::LlsdXml(_) => OSD::Undefined,
+        OSD::Boolean(value) => OSD::Boolean(*value),
+        OSD::Integer(value) => OSD::Integer(*value),
+        OSD::Real(value) => OSD::Real(*value),
+        OSD::String(value) => {
+            if value.is_empty() {
+                OSD::Undefined
+            } else {
+                OSD::String(value.clone())
+            }
+        }
+        OSD::UUID(value) => OSD::String(value.to_string()),
+        OSD::Date(value) => OSD::String(crate::model::format_system_time_for_codec(*value)),
+        OSD::Uri(Uri(value)) => OSD::String(crate::model::format_uri_for_codec(value)),
+        OSD::Binary(values) => OSD::Array(
+            values
+                .iter()
+                .map(|value| OSD::Integer(i32::from(*value)))
+                .collect(),
+        ),
+        OSD::Map(values) => OSD::Map(
+            values
+                .iter()
+                .map(|(key, value)| (key.clone(), expected_json(value)))
+                .collect(),
+        ),
+        OSD::Array(values) => OSD::Array(values.iter().map(expected_json).collect()),
+    }
+}
+
+fn hex_corpus(input: &'static str) -> Vec<(&'static str, Vec<u8>)> {
+    input
+        .lines()
+        .filter(|line| !line.is_empty() && !line.starts_with('#'))
+        .map(|line| {
+            let (name, value) = line.split_once(':').expect("named hex corpus seed");
+            let bytes = if value.is_empty() {
+                Vec::new()
+            } else {
+                decode_hex(value)
+            };
+            (name, bytes)
+        })
+        .collect()
+}
+
+fn decode_hex(value: &str) -> Vec<u8> {
+    assert_eq!(value.len() % 2, 0, "corpus hex must have byte pairs");
+    value
+        .as_bytes()
+        .chunks_exact(2)
+        .map(|pair| {
+            u8::from_str_radix(std::str::from_utf8(pair).expect("ASCII corpus hex"), 16)
+                .expect("valid corpus hex")
+        })
+        .collect()
+}
+
+fn text_corpus(input: &'static str) -> Vec<(&'static str, Vec<u8>)> {
+    input
+        .lines()
+        .filter(|line| !line.is_empty() && !line.starts_with('#'))
+        .map(|line| {
+            let (name, value) = line.split_once(':').expect("named text corpus seed");
+            (name, value.as_bytes().to_vec())
+        })
+        .collect()
+}
+
+fn notation_bytes(bytes: Vec<u8>) -> Result<OSD, Error> {
+    crate::notation::deserialize_string(String::from_utf8(bytes).map_err(|_| Error::Argument)?)
+}
+
+fn json_bytes(bytes: Vec<u8>) -> Result<OSD, Error> {
+    crate::json_codec::deserialize_string(String::from_utf8(bytes).map_err(|_| Error::Argument)?)
+}
+
+fn assert_no_panic<F>(name: &str, bytes: Vec<u8>, decoder: F)
+where
+    F: Fn(Vec<u8>) -> Result<OSD, Error>,
+{
+    assert!(
+        catch_unwind(AssertUnwindSafe(|| decoder(bytes))).is_ok(),
+        "{name} panicked"
+    );
+}
+
+fn mutate<F>(name: &str, seed: &[u8], decoder: F)
+where
+    F: Fn(Vec<u8>) -> Result<OSD, Error>,
+{
+    let mut truncations = vec![
+        0,
+        1.min(seed.len()),
+        2.min(seed.len()),
+        seed.len() / 4,
+        seed.len() / 2,
+        seed.len() * 3 / 4,
+        seed.len().saturating_sub(2),
+        seed.len().saturating_sub(1),
+        seed.len(),
+    ];
+    truncations.sort_unstable();
+    truncations.dedup();
+    for length in truncations {
+        assert_no_panic(name, seed[..length].to_vec(), &decoder);
+    }
+
+    // Sample the complete input at a bounded number of evenly spaced points.
+    // This covers tags, lengths, payloads and delimiters without turning this
+    // deterministic regression test into an unbounded fuzzing job in CI.
+    let step = seed.len().div_ceil(64).max(1);
+    let mut indexes = (0..seed.len()).step_by(step).collect::<Vec<_>>();
+    if let Some(last) = seed.len().checked_sub(1) {
+        indexes.push(last);
+    }
+    indexes.sort_unstable();
+    indexes.dedup();
+    for index in indexes {
+        for mask in [0x01, 0x80] {
+            let mut mutation = seed.to_owned();
+            mutation[index] ^= mask;
+            assert_no_panic(name, mutation, &decoder);
+        }
+    }
+}

crates/libremetaverse-structured-data/src/lib.rs

@@ -5,6 +5,8 @@ extern crate self as libremetaverse_structured_data;
 mod binary;
 mod dispatch;
 mod generated;
+#[cfg(test)]
+mod hardening;
 mod json_codec;
 mod model;
 mod notation;

crates/libremetaverse-structured-data/src/model.rs

@@ -688,6 +688,9 @@ impl OSD {
     }
 }
 
+/// Format-neutral parser/serializer namespace matching the C# static type.
+pub struct OSDParser;
+
 macro_rules! scalar_wrapper {
     ($name:ident, $value:ty, $variant:ident, $kind:ident) => {
         #[derive(Clone, Debug)]

crates/libremetaverse-structured-data/src/xml_codec.rs

@@ -296,6 +296,9 @@ impl<'a> Parser<'a> {
             if self.at_end(name) {
                 break;
             }
+            if self.rest().is_empty() {
+                return Err(self.error("unterminated XML LLSD scalar"));
+            }
             if self.rest().starts_with("<!--") {
                 self.skip_comment()?;
                 continue;

fuzz/corpus/json_osd/README.md

@@ -0,0 +1,4 @@
+# JSON OSD malformed corpus
+
+The normal Rust hardening suite feeds these parser-edge cases and deterministic
+mutations of a valid all-variant JSON document to the bounded JSON adapter.

fuzz/corpus/json_osd/malformed.txt

@@ -0,0 +1,7 @@
+empty:
+trailing:[]x
+unterminated_string:"x
+unterminated_array:[1
+unterminated_object:{"x":1
+duplicate_property:{"x":1,"x":2}
+non_finite_number:1e99999

fuzz/corpus/notation_llsd/README.md

@@ -0,0 +1,5 @@
+# Notation LLSD malformed corpus
+
+Each line in `malformed.txt` has a seed name followed by the raw notation input.
+The normal Rust hardening suite executes every seed and deterministic mutations
+of a valid all-variant document under no-panic and resource-limit checks.

fuzz/corpus/notation_llsd/malformed.txt

@@ -0,0 +1,7 @@
+empty:
+unknown:?
+unterminated_string:s(10)"x
+missing_array_end:[i1
+missing_map_value:{'key':
+bad_base64:b64"***"
+bad_date:d"not-a-date"

fuzz/corpus/protobuf_osd/README.md

@@ -0,0 +1,5 @@
+# Protobuf OSD malformed corpus
+
+Each named seed is hex-encoded. The normal Rust hardening suite checks every
+seed and deterministic mutations of a valid all-variant wire document without
+requiring an external fuzzing runtime.

fuzz/corpus/protobuf_osd/malformed.hex

@@ -0,0 +1,8 @@
+empty:
+truncated_varint:80
+zero_field:00
+wrong_type_wire:0d00000000
+oversized_length:2affffffff0f
+truncated_fixed64:210000
+unknown_group:0b
+bad_uuid:0805320100

fuzz/corpus/xml_llsd/README.md

@@ -3,4 +3,6 @@
 These inputs exercise the XML LLSD parser's entity boundary. DTD declarations,
 external entities, and undeclared named entities must be rejected without file
 or network access and without entity expansion. The structured-data unit tests
-load every fixture directly.
+load every fixture directly. `malformed_nesting.xml` additionally covers
+crossed container end tags, while `truncated_scalar.xml` guards against parser
+loops when a scalar reaches end-of-input without a closing tag.

fuzz/corpus/xml_llsd/malformed_nesting.xml

@@ -0,0 +1 @@
+<llsd><map><key>value</key><array><integer>1</map></array></llsd>

fuzz/corpus/xml_llsd/truncated_scalar.xml

@@ -0,0 +1 @@
+<llsd><string>unterminated

tests/red-suite-baseline.json

@@ -98,5 +98,5 @@
     "LibreMetaverse.Tests/XmlLLSDTests.cs::XmlSDTests.DeserializeUUID::test",
     "LibreMetaverse.Tests/XmlLLSDTests.cs::XmlSDTests.DeserializeUndef::test"
   ],
-  "support_passes": 93
+  "support_passes": 97
 }

tools/generate_api_shims.py

@@ -78,6 +78,13 @@ NATIVE_TYPES = {
     "T:LibreMetaverse.Utils": "crate::utils::Utils",
 }
 
+# Native declarations whose cataloged method implementations are still emitted
+# below. This is used for static namespace types such as OSDParser: the type is
+# hand-written, while its fixed public methods remain generator-audited.
+NATIVE_DECLARATIONS = {
+    "T:LibreMetaverse.StructuredData.OSDParser": "crate::model::OSDParser",
+}
+
 NATIVE_MEMBER_BODIES = {
     "M:LibreMetaverse.StructuredData.OSDParser.BufferCharactersEqual(System.IO.StringReader,System.Char[],System.Int32)":
         "crate::notation::buffer_characters_equal(reader, buffer, offset)",
@@ -762,9 +769,14 @@ def render_type(item: dict, type_row: dict, member_rows: dict[str, dict[str, str
         fields = [member for member in item["members"] if member["kind"] == "field" and not member.get("static")]
         private_fields = PRIVATE_LAYOUTS.get(item["doc_id"], [])
         base_type = item.get("base_type") if item["doc_id"] in COMPOSED_BASE_TYPES else None
-        if derives := VALUE_DERIVES.get(item["doc_id"]):
+        native_declaration = NATIVE_DECLARATIONS.get(item["doc_id"])
+        if native_declaration:
+            lines.append(f"pub use {native_declaration} as {rust_name};")
+        elif derives := VALUE_DERIVES.get(item["doc_id"]):
             lines.append(f"#[derive({derives})]")
-        if item["doc_id"] == "T:LibreMetaverse.StructuredData.OSD":
+        if native_declaration:
+            pass
+        elif item["doc_id"] == "T:LibreMetaverse.StructuredData.OSD":
             lines += [
                 "#[non_exhaustive]",
                 "pub enum OSD {",
@@ -990,7 +1002,7 @@ def coverage_report(catalog: dict, coverage: dict[str, tuple[int, int, bool]]) -
             native_type_ids = {
                 item["doc_id"]
                 for item in assembly["types"]
-                if item["doc_id"] in NATIVE_TYPES
+                if item["doc_id"] in NATIVE_TYPES or item["doc_id"] in NATIVE_DECLARATIONS
                 or (name == "LibreMetaverse.Types" and item["kind"] == "enum")
             }
             native_member_ids = {