From d980692933622140566fbbf71172240238ad34df Mon Sep 17 00:00:00 2001 From: Chili Palmer Date: Sun, 9 Aug 2026 01:51:29 +0000 Subject: [PATCH] Complete StructuredData parity and parser hardening (#37) --- README.md | 15 + api/SHIM-COVERAGE.md | 2 +- .../src/generated.rs | 2 +- .../src/hardening.rs | 356 ++++++++++++++++++ .../libremetaverse-structured-data/src/lib.rs | 2 + .../src/model.rs | 3 + .../src/xml_codec.rs | 3 + fuzz/corpus/json_osd/README.md | 4 + fuzz/corpus/json_osd/malformed.txt | 7 + fuzz/corpus/notation_llsd/README.md | 5 + fuzz/corpus/notation_llsd/malformed.txt | 7 + fuzz/corpus/protobuf_osd/README.md | 5 + fuzz/corpus/protobuf_osd/malformed.hex | 8 + fuzz/corpus/xml_llsd/README.md | 4 +- fuzz/corpus/xml_llsd/malformed_nesting.xml | 1 + fuzz/corpus/xml_llsd/truncated_scalar.xml | 1 + tests/red-suite-baseline.json | 2 +- tools/generate_api_shims.py | 18 +- 18 files changed, 438 insertions(+), 7 deletions(-) create mode 100644 crates/libremetaverse-structured-data/src/hardening.rs create mode 100644 fuzz/corpus/json_osd/README.md create mode 100644 fuzz/corpus/json_osd/malformed.txt create mode 100644 fuzz/corpus/notation_llsd/README.md create mode 100644 fuzz/corpus/notation_llsd/malformed.txt create mode 100644 fuzz/corpus/protobuf_osd/README.md create mode 100644 fuzz/corpus/protobuf_osd/malformed.hex create mode 100644 fuzz/corpus/xml_llsd/malformed_nesting.xml create mode 100644 fuzz/corpus/xml_llsd/truncated_scalar.xml diff --git a/README.md b/README.md index f099bd4..11ae4a2 100644 --- a/README.md +++ b/README.md @@ -107,5 +107,20 @@ ZigZag `int32`, little-endian IEEE-754 fixed64 values, 16-byte UUIDs, bounded length-delimited containers, and wire-type-aware unknown-field skipping. The schema remains an internal compatibility format; native encoders emit stable sorted maps and accept the exact optional LLSD Protobuf header. +Cross-format conversion follows the reference's format-specific boundaries: + +| Format | Round-trip behavior | +| --- | --- | +| Binary LLSD | All regular OSD variants are lossless; raw `OSDLlsdXml` is unsupported. | +| Notation LLSD | All regular OSD variants are lossless; raw `OSDLlsdXml` is unsupported. | +| XML LLSD | Regular variants are lossless; raw `OSDLlsdXml` is injected as an element and therefore decodes as that element's ordinary OSD value. | +| JSON OSD | Booleans, integers, reals, nonempty strings, maps, and arrays retain structure; UUID, date, and URI become strings, binary becomes an integer array, and undefined, empty strings, and raw XML become JSON `null`. | +| Protobuf OSD | Regular variants are lossless within the reference schema's date range and whole-second precision; raw `OSDLlsdXml` maps to undefined. | + +Checked-in malformed corpora and bounded deterministic mutations exercise all +five parsers without an external fuzzing runtime. They cover truncation, tags, +lengths, payloads, delimiters, nesting, entity rejection, contextual errors, +and the parser resource limits. Every encoder sorts map keys where necessary, +so serialization is deterministic across platforms. The controlled audit aggregates every expected failure by standardized C# member ID and rejects unrelated fixture, assertion, compile, or symbol errors. diff --git a/api/SHIM-COVERAGE.md b/api/SHIM-COVERAGE.md index 01a829c..7e696ef 100644 --- a/api/SHIM-COVERAGE.md +++ b/api/SHIM-COVERAGE.md @@ -12,7 +12,7 @@ Generated by `python3 tools/generate_api_shims.py`; do not edit by hand. | `LibreMetaverse.RLV` | 28 | 499 | callable failure-only shim | | `LibreMetaverse.Rendering.MeshFoundry` | 1 | 14 | callable failure-only shim | | `LibreMetaverse.Rendering.Simple` | 1 | 6 | callable failure-only shim | -| `LibreMetaverse.StructuredData` | 16 | 295 | native implementation: 15 types / 295 members; no generated shims remain | +| `LibreMetaverse.StructuredData` | 16 | 295 | native implementation: 16 types / 295 members; no generated shims remain | | `LibreMetaverse.Types` | 45 | 942 | native implementation: 45 types / 942 members; no generated shims remain | | `LibreMetaverse.Utilities` | 3 | 13 | callable failure-only shim | | `LibreMetaverse.Voice.Vivox` | 64 | 531 | callable failure-only shim | diff --git a/crates/libremetaverse-structured-data/src/generated.rs b/crates/libremetaverse-structured-data/src/generated.rs index 34ce76b..ddd2b08 100644 --- a/crates/libremetaverse-structured-data/src/generated.rs +++ b/crates/libremetaverse-structured-data/src/generated.rs @@ -234,7 +234,7 @@ pub use crate::model::OSDLlsdXml; pub use crate::model::OSDMap; /// C# type: `T:LibreMetaverse.StructuredData.OSDParser`. -pub struct OSDParser; +pub use crate::model::OSDParser; impl OSDParser { /// C# member: `M:LibreMetaverse.StructuredData.OSDParser.BufferCharactersEqual(System.IO.StringReader,System.Char[],System.Int32)`. pub fn buffer_characters_equal( diff --git a/crates/libremetaverse-structured-data/src/hardening.rs b/crates/libremetaverse-structured-data/src/hardening.rs new file mode 100644 index 0000000..ed67224 --- /dev/null +++ b/crates/libremetaverse-structured-data/src/hardening.rs @@ -0,0 +1,356 @@ +//! Cross-format compatibility and deterministic mutation tests. + +use crate::{Error, OSD}; +use libremetaverse_types::UUID; +use libremetaverse_types::compat::Uri; +use std::collections::HashMap; +use std::panic::{AssertUnwindSafe, catch_unwind}; +use std::time::{Duration, UNIX_EPOCH}; + +fn variants() -> Vec { + vec![ + OSD::Undefined, + OSD::Boolean(true), + OSD::Integer(-42), + OSD::Real(12.5), + OSD::String("text".into()), + OSD::UUID(UUID::new_with_string("97f4aeca-88a1-42a1-b385-b97b18abb255".into()).unwrap()), + OSD::Date(UNIX_EPOCH + Duration::from_secs(1_705_314_645)), + OSD::Uri(Uri("https://example.com/path".into())), + OSD::Binary(vec![0, 1, 255]), + OSD::Map(HashMap::from([("key".into(), OSD::Integer(7))])), + OSD::Array(vec![OSD::String("nested".into()), OSD::Boolean(false)]), + OSD::LlsdXml("raw XML".into()), + ] +} + +#[test] +fn every_osd_variant_has_documented_cross_format_behavior() { + for value in variants() { + if matches!(value, OSD::LlsdXml(_)) { + assert!(crate::binary::serialize(value.clone()).is_err()); + assert!(crate::notation::serialize(value.clone()).is_err()); + } else { + assert_eq!( + crate::binary::deserialize_bytes(crate::binary::serialize(value.clone()).unwrap()) + .unwrap(), + value + ); + assert_eq!( + crate::notation::deserialize_string( + crate::notation::serialize(value.clone()).unwrap() + ) + .unwrap(), + value + ); + } + + let xml = crate::xml_codec::deserialize_bytes( + crate::xml_codec::serialize_bytes(value.clone()).unwrap(), + ) + .unwrap(); + let expected_xml = if matches!(value, OSD::LlsdXml(_)) { + OSD::String("raw XML".into()) + } else { + value.clone() + }; + assert_eq!(xml, expected_xml); + + let json = crate::json_codec::deserialize_string( + crate::json_codec::serialize(value.clone(), Some(true)).unwrap(), + ) + .unwrap(); + assert_eq!(json, expected_json(&value)); + + let protobuf = crate::protobuf::deserialize_bytes( + crate::protobuf::serialize(value.clone(), Some(false)).unwrap(), + ) + .unwrap(); + let expected_protobuf = if matches!(value, OSD::LlsdXml(_)) { + OSD::Undefined + } else { + value + }; + assert_eq!(protobuf, expected_protobuf); + } +} + +#[test] +fn every_encoder_is_deterministic_for_unordered_maps() { + let mut forward = HashMap::new(); + forward.insert("a".into(), OSD::String("first".into())); + forward.insert("m".into(), OSD::Boolean(true)); + forward.insert("z".into(), OSD::Integer(1)); + let mut reverse = HashMap::new(); + reverse.insert("z".into(), OSD::Integer(1)); + reverse.insert("m".into(), OSD::Boolean(true)); + reverse.insert("a".into(), OSD::String("first".into())); + let forward = OSD::Map(forward); + let reverse = OSD::Map(reverse); + assert_eq!( + crate::binary::serialize(forward.clone()).unwrap(), + crate::binary::serialize(reverse.clone()).unwrap() + ); + assert_eq!( + crate::notation::serialize(forward.clone()).unwrap(), + crate::notation::serialize(reverse.clone()).unwrap() + ); + assert_eq!( + crate::xml_codec::serialize_bytes(forward.clone()).unwrap(), + crate::xml_codec::serialize_bytes(reverse.clone()).unwrap() + ); + assert_eq!( + crate::json_codec::serialize(forward.clone(), Some(true)).unwrap(), + crate::json_codec::serialize(reverse.clone(), Some(true)).unwrap() + ); + assert_eq!( + crate::protobuf::serialize(forward, Some(false)).unwrap(), + crate::protobuf::serialize(reverse, Some(false)).unwrap() + ); +} + +#[test] +fn malformed_corpora_and_focused_mutations_never_panic() { + for (name, seed) in hex_corpus(include_str!( + "../../../fuzz/corpus/binary_llsd/malformed.hex" + )) { + assert_no_panic( + &format!("binary corpus {name}"), + seed, + crate::binary::deserialize_bytes, + ); + } + for (name, seed) in text_corpus(include_str!( + "../../../fuzz/corpus/notation_llsd/malformed.txt" + )) { + assert_no_panic(&format!("notation corpus {name}"), seed, notation_bytes); + } + for (name, seed) in text_corpus(include_str!("../../../fuzz/corpus/json_osd/malformed.txt")) { + assert_no_panic(&format!("JSON corpus {name}"), seed, json_bytes); + } + for (name, seed) in hex_corpus(include_str!( + "../../../fuzz/corpus/protobuf_osd/malformed.hex" + )) { + assert_no_panic( + &format!("Protobuf corpus {name}"), + seed, + crate::protobuf::deserialize_bytes, + ); + } + for (name, seed) in [ + ( + "XML DTD", + include_bytes!("../../../fuzz/corpus/xml_llsd/doctype_entity.xml").as_slice(), + ), + ( + "XML external entity", + include_bytes!("../../../fuzz/corpus/xml_llsd/external_entity.xml").as_slice(), + ), + ( + "XML named entity", + include_bytes!("../../../fuzz/corpus/xml_llsd/unknown_entity.xml").as_slice(), + ), + ( + "XML nesting", + include_bytes!("../../../fuzz/corpus/xml_llsd/malformed_nesting.xml").as_slice(), + ), + ( + "XML truncated scalar", + include_bytes!("../../../fuzz/corpus/xml_llsd/truncated_scalar.xml").as_slice(), + ), + ] { + assert_no_panic(name, seed.to_vec(), crate::xml_codec::deserialize_bytes); + } + + let value = OSD::Map(HashMap::from([ + ("array".into(), OSD::Array(variants())), + ("binary".into(), OSD::Binary(vec![0, 1, 2, 255])), + ])); + let binary = crate::binary::serialize(value.clone()).unwrap_err(); + assert!(matches!(binary, Error::Parse { .. })); + let fuzzable = OSD::Map(HashMap::from([ + ( + "array".into(), + OSD::Array(variants().into_iter().take(11).collect()), + ), + ("binary".into(), OSD::Binary(vec![0, 1, 2, 255])), + ])); + mutate( + "binary", + &crate::binary::serialize(fuzzable.clone()).unwrap(), + crate::binary::deserialize_bytes, + ); + mutate( + "notation", + &crate::notation::serialize(fuzzable.clone()) + .unwrap() + .into_bytes(), + notation_bytes, + ); + mutate( + "XML", + &crate::xml_codec::serialize_bytes(fuzzable.clone()).unwrap(), + crate::xml_codec::deserialize_bytes, + ); + mutate( + "JSON", + &crate::json_codec::serialize(fuzzable.clone(), Some(true)) + .unwrap() + .into_bytes(), + json_bytes, + ); + mutate( + "Protobuf", + &crate::protobuf::serialize(fuzzable, Some(false)).unwrap(), + crate::protobuf::deserialize_bytes, + ); +} + +#[test] +fn malformed_errors_retain_positions_and_context() { + let cases = [ + crate::binary::deserialize_bytes(vec![b'i']), + crate::notation::deserialize_string("s(4)\"x".into()), + crate::xml_codec::deserialize_string("1".into()), + crate::json_codec::deserialize_string("[1,".into()), + crate::protobuf::deserialize_bytes(vec![0x08]), + ]; + for result in cases { + match result { + Err(Error::Parse { position, context }) => { + assert!(position < OSD::DEFAULT_MAX_BINARY_BYTES); + assert!(!context.is_empty()); + } + result => panic!("expected contextual parse error, got {result:?}"), + } + } +} + +fn expected_json(value: &OSD) -> OSD { + match value { + OSD::Undefined | OSD::LlsdXml(_) => OSD::Undefined, + OSD::Boolean(value) => OSD::Boolean(*value), + OSD::Integer(value) => OSD::Integer(*value), + OSD::Real(value) => OSD::Real(*value), + OSD::String(value) => { + if value.is_empty() { + OSD::Undefined + } else { + OSD::String(value.clone()) + } + } + OSD::UUID(value) => OSD::String(value.to_string()), + OSD::Date(value) => OSD::String(crate::model::format_system_time_for_codec(*value)), + OSD::Uri(Uri(value)) => OSD::String(crate::model::format_uri_for_codec(value)), + OSD::Binary(values) => OSD::Array( + values + .iter() + .map(|value| OSD::Integer(i32::from(*value))) + .collect(), + ), + OSD::Map(values) => OSD::Map( + values + .iter() + .map(|(key, value)| (key.clone(), expected_json(value))) + .collect(), + ), + OSD::Array(values) => OSD::Array(values.iter().map(expected_json).collect()), + } +} + +fn hex_corpus(input: &'static str) -> Vec<(&'static str, Vec)> { + input + .lines() + .filter(|line| !line.is_empty() && !line.starts_with('#')) + .map(|line| { + let (name, value) = line.split_once(':').expect("named hex corpus seed"); + let bytes = if value.is_empty() { + Vec::new() + } else { + decode_hex(value) + }; + (name, bytes) + }) + .collect() +} + +fn decode_hex(value: &str) -> Vec { + assert_eq!(value.len() % 2, 0, "corpus hex must have byte pairs"); + value + .as_bytes() + .chunks_exact(2) + .map(|pair| { + u8::from_str_radix(std::str::from_utf8(pair).expect("ASCII corpus hex"), 16) + .expect("valid corpus hex") + }) + .collect() +} + +fn text_corpus(input: &'static str) -> Vec<(&'static str, Vec)> { + input + .lines() + .filter(|line| !line.is_empty() && !line.starts_with('#')) + .map(|line| { + let (name, value) = line.split_once(':').expect("named text corpus seed"); + (name, value.as_bytes().to_vec()) + }) + .collect() +} + +fn notation_bytes(bytes: Vec) -> Result { + crate::notation::deserialize_string(String::from_utf8(bytes).map_err(|_| Error::Argument)?) +} + +fn json_bytes(bytes: Vec) -> Result { + crate::json_codec::deserialize_string(String::from_utf8(bytes).map_err(|_| Error::Argument)?) +} + +fn assert_no_panic(name: &str, bytes: Vec, decoder: F) +where + F: Fn(Vec) -> Result, +{ + assert!( + catch_unwind(AssertUnwindSafe(|| decoder(bytes))).is_ok(), + "{name} panicked" + ); +} + +fn mutate(name: &str, seed: &[u8], decoder: F) +where + F: Fn(Vec) -> Result, +{ + let mut truncations = vec![ + 0, + 1.min(seed.len()), + 2.min(seed.len()), + seed.len() / 4, + seed.len() / 2, + seed.len() * 3 / 4, + seed.len().saturating_sub(2), + seed.len().saturating_sub(1), + seed.len(), + ]; + truncations.sort_unstable(); + truncations.dedup(); + for length in truncations { + assert_no_panic(name, seed[..length].to_vec(), &decoder); + } + + // Sample the complete input at a bounded number of evenly spaced points. + // This covers tags, lengths, payloads and delimiters without turning this + // deterministic regression test into an unbounded fuzzing job in CI. + let step = seed.len().div_ceil(64).max(1); + let mut indexes = (0..seed.len()).step_by(step).collect::>(); + if let Some(last) = seed.len().checked_sub(1) { + indexes.push(last); + } + indexes.sort_unstable(); + indexes.dedup(); + for index in indexes { + for mask in [0x01, 0x80] { + let mut mutation = seed.to_owned(); + mutation[index] ^= mask; + assert_no_panic(name, mutation, &decoder); + } + } +} diff --git a/crates/libremetaverse-structured-data/src/lib.rs b/crates/libremetaverse-structured-data/src/lib.rs index 357cf6d..9bc123d 100644 --- a/crates/libremetaverse-structured-data/src/lib.rs +++ b/crates/libremetaverse-structured-data/src/lib.rs @@ -5,6 +5,8 @@ extern crate self as libremetaverse_structured_data; mod binary; mod dispatch; mod generated; +#[cfg(test)] +mod hardening; mod json_codec; mod model; mod notation; diff --git a/crates/libremetaverse-structured-data/src/model.rs b/crates/libremetaverse-structured-data/src/model.rs index a31f22d..ebb1b0d 100644 --- a/crates/libremetaverse-structured-data/src/model.rs +++ b/crates/libremetaverse-structured-data/src/model.rs @@ -688,6 +688,9 @@ impl OSD { } } +/// Format-neutral parser/serializer namespace matching the C# static type. +pub struct OSDParser; + macro_rules! scalar_wrapper { ($name:ident, $value:ty, $variant:ident, $kind:ident) => { #[derive(Clone, Debug)] diff --git a/crates/libremetaverse-structured-data/src/xml_codec.rs b/crates/libremetaverse-structured-data/src/xml_codec.rs index 8010e45..181c156 100644 --- a/crates/libremetaverse-structured-data/src/xml_codec.rs +++ b/crates/libremetaverse-structured-data/src/xml_codec.rs @@ -296,6 +296,9 @@ impl<'a> Parser<'a> { if self.at_end(name) { break; } + if self.rest().is_empty() { + return Err(self.error("unterminated XML LLSD scalar")); + } if self.rest().starts_with("