Complete StructuredData parity and parser hardening (#37)

crates/libremetaverse-structured-data/src/generated.rs

@@ -234,7 +234,7 @@ pub use crate::model::OSDLlsdXml;
 pub use crate::model::OSDMap;
 
 /// C# type: `T:LibreMetaverse.StructuredData.OSDParser`.
-pub struct OSDParser;
+pub use crate::model::OSDParser;
 impl OSDParser {
     /// C# member: `M:LibreMetaverse.StructuredData.OSDParser.BufferCharactersEqual(System.IO.StringReader,System.Char[],System.Int32)`.
     pub fn buffer_characters_equal(

crates/libremetaverse-structured-data/src/hardening.rs

@@ -0,0 +1,356 @@
+//! Cross-format compatibility and deterministic mutation tests.
+
+use crate::{Error, OSD};
+use libremetaverse_types::UUID;
+use libremetaverse_types::compat::Uri;
+use std::collections::HashMap;
+use std::panic::{AssertUnwindSafe, catch_unwind};
+use std::time::{Duration, UNIX_EPOCH};
+
+fn variants() -> Vec<OSD> {
+    vec![
+        OSD::Undefined,
+        OSD::Boolean(true),
+        OSD::Integer(-42),
+        OSD::Real(12.5),
+        OSD::String("text".into()),
+        OSD::UUID(UUID::new_with_string("97f4aeca-88a1-42a1-b385-b97b18abb255".into()).unwrap()),
+        OSD::Date(UNIX_EPOCH + Duration::from_secs(1_705_314_645)),
+        OSD::Uri(Uri("https://example.com/path".into())),
+        OSD::Binary(vec![0, 1, 255]),
+        OSD::Map(HashMap::from([("key".into(), OSD::Integer(7))])),
+        OSD::Array(vec![OSD::String("nested".into()), OSD::Boolean(false)]),
+        OSD::LlsdXml("<string>raw XML</string>".into()),
+    ]
+}
+
+#[test]
+fn every_osd_variant_has_documented_cross_format_behavior() {
+    for value in variants() {
+        if matches!(value, OSD::LlsdXml(_)) {
+            assert!(crate::binary::serialize(value.clone()).is_err());
+            assert!(crate::notation::serialize(value.clone()).is_err());
+        } else {
+            assert_eq!(
+                crate::binary::deserialize_bytes(crate::binary::serialize(value.clone()).unwrap())
+                    .unwrap(),
+                value
+            );
+            assert_eq!(
+                crate::notation::deserialize_string(
+                    crate::notation::serialize(value.clone()).unwrap()
+                )
+                .unwrap(),
+                value
+            );
+        }
+
+        let xml = crate::xml_codec::deserialize_bytes(
+            crate::xml_codec::serialize_bytes(value.clone()).unwrap(),
+        )
+        .unwrap();
+        let expected_xml = if matches!(value, OSD::LlsdXml(_)) {
+            OSD::String("raw XML".into())
+        } else {
+            value.clone()
+        };
+        assert_eq!(xml, expected_xml);
+
+        let json = crate::json_codec::deserialize_string(
+            crate::json_codec::serialize(value.clone(), Some(true)).unwrap(),
+        )
+        .unwrap();
+        assert_eq!(json, expected_json(&value));
+
+        let protobuf = crate::protobuf::deserialize_bytes(
+            crate::protobuf::serialize(value.clone(), Some(false)).unwrap(),
+        )
+        .unwrap();
+        let expected_protobuf = if matches!(value, OSD::LlsdXml(_)) {
+            OSD::Undefined
+        } else {
+            value
+        };
+        assert_eq!(protobuf, expected_protobuf);
+    }
+}
+
+#[test]
+fn every_encoder_is_deterministic_for_unordered_maps() {
+    let mut forward = HashMap::new();
+    forward.insert("a".into(), OSD::String("first".into()));
+    forward.insert("m".into(), OSD::Boolean(true));
+    forward.insert("z".into(), OSD::Integer(1));
+    let mut reverse = HashMap::new();
+    reverse.insert("z".into(), OSD::Integer(1));
+    reverse.insert("m".into(), OSD::Boolean(true));
+    reverse.insert("a".into(), OSD::String("first".into()));
+    let forward = OSD::Map(forward);
+    let reverse = OSD::Map(reverse);
+    assert_eq!(
+        crate::binary::serialize(forward.clone()).unwrap(),
+        crate::binary::serialize(reverse.clone()).unwrap()
+    );
+    assert_eq!(
+        crate::notation::serialize(forward.clone()).unwrap(),
+        crate::notation::serialize(reverse.clone()).unwrap()
+    );
+    assert_eq!(
+        crate::xml_codec::serialize_bytes(forward.clone()).unwrap(),
+        crate::xml_codec::serialize_bytes(reverse.clone()).unwrap()
+    );
+    assert_eq!(
+        crate::json_codec::serialize(forward.clone(), Some(true)).unwrap(),
+        crate::json_codec::serialize(reverse.clone(), Some(true)).unwrap()
+    );
+    assert_eq!(
+        crate::protobuf::serialize(forward, Some(false)).unwrap(),
+        crate::protobuf::serialize(reverse, Some(false)).unwrap()
+    );
+}
+
+#[test]
+fn malformed_corpora_and_focused_mutations_never_panic() {
+    for (name, seed) in hex_corpus(include_str!(
+        "../../../fuzz/corpus/binary_llsd/malformed.hex"
+    )) {
+        assert_no_panic(
+            &format!("binary corpus {name}"),
+            seed,
+            crate::binary::deserialize_bytes,
+        );
+    }
+    for (name, seed) in text_corpus(include_str!(
+        "../../../fuzz/corpus/notation_llsd/malformed.txt"
+    )) {
+        assert_no_panic(&format!("notation corpus {name}"), seed, notation_bytes);
+    }
+    for (name, seed) in text_corpus(include_str!("../../../fuzz/corpus/json_osd/malformed.txt")) {
+        assert_no_panic(&format!("JSON corpus {name}"), seed, json_bytes);
+    }
+    for (name, seed) in hex_corpus(include_str!(
+        "../../../fuzz/corpus/protobuf_osd/malformed.hex"
+    )) {
+        assert_no_panic(
+            &format!("Protobuf corpus {name}"),
+            seed,
+            crate::protobuf::deserialize_bytes,
+        );
+    }
+    for (name, seed) in [
+        (
+            "XML DTD",
+            include_bytes!("../../../fuzz/corpus/xml_llsd/doctype_entity.xml").as_slice(),
+        ),
+        (
+            "XML external entity",
+            include_bytes!("../../../fuzz/corpus/xml_llsd/external_entity.xml").as_slice(),
+        ),
+        (
+            "XML named entity",
+            include_bytes!("../../../fuzz/corpus/xml_llsd/unknown_entity.xml").as_slice(),
+        ),
+        (
+            "XML nesting",
+            include_bytes!("../../../fuzz/corpus/xml_llsd/malformed_nesting.xml").as_slice(),
+        ),
+        (
+            "XML truncated scalar",
+            include_bytes!("../../../fuzz/corpus/xml_llsd/truncated_scalar.xml").as_slice(),
+        ),
+    ] {
+        assert_no_panic(name, seed.to_vec(), crate::xml_codec::deserialize_bytes);
+    }
+
+    let value = OSD::Map(HashMap::from([
+        ("array".into(), OSD::Array(variants())),
+        ("binary".into(), OSD::Binary(vec![0, 1, 2, 255])),
+    ]));
+    let binary = crate::binary::serialize(value.clone()).unwrap_err();
+    assert!(matches!(binary, Error::Parse { .. }));
+    let fuzzable = OSD::Map(HashMap::from([
+        (
+            "array".into(),
+            OSD::Array(variants().into_iter().take(11).collect()),
+        ),
+        ("binary".into(), OSD::Binary(vec![0, 1, 2, 255])),
+    ]));
+    mutate(
+        "binary",
+        &crate::binary::serialize(fuzzable.clone()).unwrap(),
+        crate::binary::deserialize_bytes,
+    );
+    mutate(
+        "notation",
+        &crate::notation::serialize(fuzzable.clone())
+            .unwrap()
+            .into_bytes(),
+        notation_bytes,
+    );
+    mutate(
+        "XML",
+        &crate::xml_codec::serialize_bytes(fuzzable.clone()).unwrap(),
+        crate::xml_codec::deserialize_bytes,
+    );
+    mutate(
+        "JSON",
+        &crate::json_codec::serialize(fuzzable.clone(), Some(true))
+            .unwrap()
+            .into_bytes(),
+        json_bytes,
+    );
+    mutate(
+        "Protobuf",
+        &crate::protobuf::serialize(fuzzable, Some(false)).unwrap(),
+        crate::protobuf::deserialize_bytes,
+    );
+}
+
+#[test]
+fn malformed_errors_retain_positions_and_context() {
+    let cases = [
+        crate::binary::deserialize_bytes(vec![b'i']),
+        crate::notation::deserialize_string("s(4)\"x".into()),
+        crate::xml_codec::deserialize_string("<llsd><integer>1</real></llsd>".into()),
+        crate::json_codec::deserialize_string("[1,".into()),
+        crate::protobuf::deserialize_bytes(vec![0x08]),
+    ];
+    for result in cases {
+        match result {
+            Err(Error::Parse { position, context }) => {
+                assert!(position < OSD::DEFAULT_MAX_BINARY_BYTES);
+                assert!(!context.is_empty());
+            }
+            result => panic!("expected contextual parse error, got {result:?}"),
+        }
+    }
+}
+
+fn expected_json(value: &OSD) -> OSD {
+    match value {
+        OSD::Undefined | OSD::LlsdXml(_) => OSD::Undefined,
+        OSD::Boolean(value) => OSD::Boolean(*value),
+        OSD::Integer(value) => OSD::Integer(*value),
+        OSD::Real(value) => OSD::Real(*value),
+        OSD::String(value) => {
+            if value.is_empty() {
+                OSD::Undefined
+            } else {
+                OSD::String(value.clone())
+            }
+        }
+        OSD::UUID(value) => OSD::String(value.to_string()),
+        OSD::Date(value) => OSD::String(crate::model::format_system_time_for_codec(*value)),
+        OSD::Uri(Uri(value)) => OSD::String(crate::model::format_uri_for_codec(value)),
+        OSD::Binary(values) => OSD::Array(
+            values
+                .iter()
+                .map(|value| OSD::Integer(i32::from(*value)))
+                .collect(),
+        ),
+        OSD::Map(values) => OSD::Map(
+            values
+                .iter()
+                .map(|(key, value)| (key.clone(), expected_json(value)))
+                .collect(),
+        ),
+        OSD::Array(values) => OSD::Array(values.iter().map(expected_json).collect()),
+    }
+}
+
+fn hex_corpus(input: &'static str) -> Vec<(&'static str, Vec<u8>)> {
+    input
+        .lines()
+        .filter(|line| !line.is_empty() && !line.starts_with('#'))
+        .map(|line| {
+            let (name, value) = line.split_once(':').expect("named hex corpus seed");
+            let bytes = if value.is_empty() {
+                Vec::new()
+            } else {
+                decode_hex(value)
+            };
+            (name, bytes)
+        })
+        .collect()
+}
+
+fn decode_hex(value: &str) -> Vec<u8> {
+    assert_eq!(value.len() % 2, 0, "corpus hex must have byte pairs");
+    value
+        .as_bytes()
+        .chunks_exact(2)
+        .map(|pair| {
+            u8::from_str_radix(std::str::from_utf8(pair).expect("ASCII corpus hex"), 16)
+                .expect("valid corpus hex")
+        })
+        .collect()
+}
+
+fn text_corpus(input: &'static str) -> Vec<(&'static str, Vec<u8>)> {
+    input
+        .lines()
+        .filter(|line| !line.is_empty() && !line.starts_with('#'))
+        .map(|line| {
+            let (name, value) = line.split_once(':').expect("named text corpus seed");
+            (name, value.as_bytes().to_vec())
+        })
+        .collect()
+}
+
+fn notation_bytes(bytes: Vec<u8>) -> Result<OSD, Error> {
+    crate::notation::deserialize_string(String::from_utf8(bytes).map_err(|_| Error::Argument)?)
+}
+
+fn json_bytes(bytes: Vec<u8>) -> Result<OSD, Error> {
+    crate::json_codec::deserialize_string(String::from_utf8(bytes).map_err(|_| Error::Argument)?)
+}
+
+fn assert_no_panic<F>(name: &str, bytes: Vec<u8>, decoder: F)
+where
+    F: Fn(Vec<u8>) -> Result<OSD, Error>,
+{
+    assert!(
+        catch_unwind(AssertUnwindSafe(|| decoder(bytes))).is_ok(),
+        "{name} panicked"
+    );
+}
+
+fn mutate<F>(name: &str, seed: &[u8], decoder: F)
+where
+    F: Fn(Vec<u8>) -> Result<OSD, Error>,
+{
+    let mut truncations = vec![
+        0,
+        1.min(seed.len()),
+        2.min(seed.len()),
+        seed.len() / 4,
+        seed.len() / 2,
+        seed.len() * 3 / 4,
+        seed.len().saturating_sub(2),
+        seed.len().saturating_sub(1),
+        seed.len(),
+    ];
+    truncations.sort_unstable();
+    truncations.dedup();
+    for length in truncations {
+        assert_no_panic(name, seed[..length].to_vec(), &decoder);
+    }
+
+    // Sample the complete input at a bounded number of evenly spaced points.
+    // This covers tags, lengths, payloads and delimiters without turning this
+    // deterministic regression test into an unbounded fuzzing job in CI.
+    let step = seed.len().div_ceil(64).max(1);
+    let mut indexes = (0..seed.len()).step_by(step).collect::<Vec<_>>();
+    if let Some(last) = seed.len().checked_sub(1) {
+        indexes.push(last);
+    }
+    indexes.sort_unstable();
+    indexes.dedup();
+    for index in indexes {
+        for mask in [0x01, 0x80] {
+            let mut mutation = seed.to_owned();
+            mutation[index] ^= mask;
+            assert_no_panic(name, mutation, &decoder);
+        }
+    }
+}

crates/libremetaverse-structured-data/src/lib.rs

@@ -5,6 +5,8 @@ extern crate self as libremetaverse_structured_data;
 mod binary;
 mod dispatch;
 mod generated;
+#[cfg(test)]
+mod hardening;
 mod json_codec;
 mod model;
 mod notation;

crates/libremetaverse-structured-data/src/model.rs

@@ -688,6 +688,9 @@ impl OSD {
     }
 }
 
+/// Format-neutral parser/serializer namespace matching the C# static type.
+pub struct OSDParser;
+
 macro_rules! scalar_wrapper {
     ($name:ident, $value:ty, $variant:ident, $kind:ident) => {
         #[derive(Clone, Debug)]

crates/libremetaverse-structured-data/src/xml_codec.rs

@@ -296,6 +296,9 @@ impl<'a> Parser<'a> {
             if self.at_end(name) {
                 break;
             }
+            if self.rest().is_empty() {
+                return Err(self.error("unterminated XML LLSD scalar"));
+            }
             if self.rest().starts_with("<!--") {
                 self.skip_comment()?;
                 continue;