diff --git a/Cargo.lock b/Cargo.lock index 3671f57..3b72472 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -156,6 +156,12 @@ dependencies = [ "typenum", ] +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + [[package]] name = "js-sys" version = "0.3.104" @@ -274,6 +280,8 @@ version = "0.0.1" dependencies = [ "base64", "libremetaverse-types", + "serde", + "serde_json", ] [[package]] @@ -326,6 +334,12 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + [[package]] name = "once_cell" version = "1.21.4" @@ -368,6 +382,48 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + [[package]] name = "sha1" version = "0.10.7" @@ -413,6 +469,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "typenum" version = "1.20.1" @@ -474,7 +541,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn", + "syn 2.0.119", "wasm-bindgen-shared", ] @@ -486,3 +553,9 @@ checksum = "7ef64dbcc55df09c7e5a46182d181c2cfa3e925f3da937ea764728b4bbb9dcbf" dependencies = [ "unicode-ident", ] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/README.md b/README.md index ee0b783..f099bd4 100644 --- a/README.md +++ b/README.md @@ -97,5 +97,15 @@ instruction handling. Serialization is compact and deterministic for maps. DTD declarations and named entity expansion are disabled; input, output, nesting, node, and aggregate allocation limits are enforced with byte-positioned parse errors. +JSON conversion is explicit rather than a derived Serde mapping: integral +width coercion, empty strings, binary arrays, typed values, and the reference's +default omission rules retain their C# behavior. Its reader rejects duplicate +properties and enforces byte, depth, node, and allocation limits while compact +output uses System.Text.Json-compatible escaping and stable map-key ordering. +The private OSD Protobuf schema is implemented directly with fixed field tags, +ZigZag `int32`, little-endian IEEE-754 fixed64 values, 16-byte UUIDs, bounded +length-delimited containers, and wire-type-aware unknown-field skipping. The +schema remains an internal compatibility format; native encoders emit stable +sorted maps and accept the exact optional LLSD Protobuf header. The controlled audit aggregates every expected failure by standardized C# member ID and rejects unrelated fixture, assertion, compile, or symbol errors. diff --git a/api/SHIM-COVERAGE.md b/api/SHIM-COVERAGE.md index c77e7ce..01a829c 100644 --- a/api/SHIM-COVERAGE.md +++ b/api/SHIM-COVERAGE.md @@ -12,7 +12,7 @@ Generated by `python3 tools/generate_api_shims.py`; do not edit by hand. | `LibreMetaverse.RLV` | 28 | 499 | callable failure-only shim | | `LibreMetaverse.Rendering.MeshFoundry` | 1 | 14 | callable failure-only shim | | `LibreMetaverse.Rendering.Simple` | 1 | 6 | callable failure-only shim | -| `LibreMetaverse.StructuredData` | 16 | 295 | native implementation: 15 types / 289 members; remaining surface is callable failure-only shims | +| `LibreMetaverse.StructuredData` | 16 | 295 | native implementation: 15 types / 295 members; no generated shims remain | | `LibreMetaverse.Types` | 45 | 942 | native implementation: 45 types / 942 members; no generated shims remain | | `LibreMetaverse.Utilities` | 3 | 13 | callable failure-only shim | | `LibreMetaverse.Voice.Vivox` | 64 | 531 | callable failure-only shim | diff --git a/crates/libremetaverse-structured-data/Cargo.toml b/crates/libremetaverse-structured-data/Cargo.toml index 5d00949..6463a17 100644 --- a/crates/libremetaverse-structured-data/Cargo.toml +++ b/crates/libremetaverse-structured-data/Cargo.toml @@ -10,6 +10,8 @@ description = "OSD and LLSD shims for the MetaCrate LibreMetaverse rewrite" [dependencies] base64 = "0.22" libremetaverse-types = { path = "../libremetaverse-types" } +serde = "1" +serde_json = "1" [lints] workspace = true diff --git a/crates/libremetaverse-structured-data/src/dispatch.rs b/crates/libremetaverse-structured-data/src/dispatch.rs index b6fcec7..a3afd4d 100644 --- a/crates/libremetaverse-structured-data/src/dispatch.rs +++ b/crates/libremetaverse-structured-data/src/dispatch.rs @@ -120,6 +120,7 @@ fn starts_with_ignore_ascii_case(value: &str, prefix: &str) -> bool { #[cfg(test)] mod tests { use super::*; + use std::collections::HashMap; #[test] fn byte_dispatch_recognizes_headers_bom_whitespace_and_json_tokens() { @@ -164,31 +165,16 @@ mod tests { deserialize_bytes(b"".to_vec()), Ok(OSD::Undefined) ); - let cases: &[(&[u8], &str)] = &[ - ( - b"", - "M:LibreMetaverse.StructuredData.OSDParser.DeserializeLLSDProtobuf(System.Byte[])", - ), - ( - b"{}", - "M:LibreMetaverse.StructuredData.OSDParser.DeserializeJson(System.String)", - ), - ]; - for (input, expected_member) in cases { - assert_eq!( - deserialize_bytes(input.to_vec()) - .expect_err("format decoder is intentionally owned by a later issue") - .csharp_member(), - Some(*expected_member) - ); - } + assert_eq!( + deserialize_bytes(b"".to_vec()), + Ok(OSD::Undefined) + ); + assert_eq!( + deserialize_bytes(b"{}".to_vec()), + Ok(OSD::Map(HashMap::new())) + ); let stream = Box::new(std::io::Cursor::new(b"{}".to_vec())); - assert_eq!( - deserialize_stream(stream) - .expect_err("JSON decoder is intentionally owned by issue #36") - .csharp_member(), - Some("M:LibreMetaverse.StructuredData.OSDParser.DeserializeJson(System.String)") - ); + assert_eq!(deserialize_stream(stream), Ok(OSD::Map(HashMap::new()))); } } diff --git a/crates/libremetaverse-structured-data/src/generated.rs b/crates/libremetaverse-structured-data/src/generated.rs index 8321972..34ce76b 100644 --- a/crates/libremetaverse-structured-data/src/generated.rs +++ b/crates/libremetaverse-structured-data/src/generated.rs @@ -273,17 +273,13 @@ impl OSDParser { pub fn deserialize_json_with_stream( json: Box, ) -> Result { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.StructuredData.OSDParser.DeserializeJson(System.IO.Stream)", - ) + crate::json_codec::deserialize_stream(json) } /// C# member: `M:LibreMetaverse.StructuredData.OSDParser.DeserializeJson(System.String)`. pub fn deserialize_json_with_string( json: String, ) -> Result { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.StructuredData.OSDParser.DeserializeJson(System.String)", - ) + crate::json_codec::deserialize_string(json) } /// C# member: `M:LibreMetaverse.StructuredData.OSDParser.DeserializeLLSDBinary(System.Byte[])`. pub fn deserialize_llsd_binary_with_bytes( @@ -313,17 +309,13 @@ impl OSDParser { pub fn deserialize_llsd_protobuf_with_bytes( data: Vec, ) -> Result { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.StructuredData.OSDParser.DeserializeLLSDProtobuf(System.Byte[])", - ) + crate::protobuf::deserialize_bytes(data) } /// C# member: `M:LibreMetaverse.StructuredData.OSDParser.DeserializeLLSDProtobuf(System.IO.Stream)`. pub fn deserialize_llsd_protobuf_with_stream( stream: Box, ) -> Result { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.StructuredData.OSDParser.DeserializeLLSDProtobuf(System.IO.Stream)", - ) + crate::protobuf::deserialize_stream(stream) } /// C# member: `M:LibreMetaverse.StructuredData.OSDParser.DeserializeLLSDXml(System.Byte[])`. pub fn deserialize_llsd_xml_with_bytes( @@ -412,9 +404,7 @@ impl OSDParser { osd: libremetaverse_structured_data::OSD, preserve_defaults: Option, ) -> Result { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.StructuredData.OSDParser.SerializeJsonString(LibreMetaverse.StructuredData.OSD,System.Boolean)", - ) + crate::json_codec::serialize(osd, preserve_defaults) } /// C# member: `M:LibreMetaverse.StructuredData.OSDParser.SerializeLLSDBinary(LibreMetaverse.StructuredData.OSD)`. pub fn serialize_llsd_binary_with_osd( @@ -477,9 +467,7 @@ impl OSDParser { osd: libremetaverse_structured_data::OSD, prepend_header: Option, ) -> Result, crate::Error> { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.StructuredData.OSDParser.SerializeLLSDProtobuf(LibreMetaverse.StructuredData.OSD,System.Boolean)", - ) + crate::protobuf::serialize(osd, prepend_header) } /// C# member: `M:LibreMetaverse.StructuredData.OSDParser.SerializeLLSDXmlBytes(LibreMetaverse.StructuredData.OSD)`. pub fn serialize_llsd_xml_bytes( diff --git a/crates/libremetaverse-structured-data/src/json_codec.rs b/crates/libremetaverse-structured-data/src/json_codec.rs new file mode 100644 index 0000000..9a9d61c --- /dev/null +++ b/crates/libremetaverse-structured-data/src/json_codec.rs @@ -0,0 +1,467 @@ +//! Bounded JSON parsing with explicit `LibreMetaverse` OSD coercions. + +#![allow(clippy::missing_errors_doc)] +#![allow(clippy::needless_pass_by_value)] + +use crate::{Error, OSD}; +use libremetaverse_types::compat::{ReadWrite, Uri}; +use serde::de::{self, DeserializeSeed, Error as _, MapAccess, SeqAccess, Visitor}; +use std::cell::Cell; +use std::collections::HashMap; +use std::fmt; +use std::io::Read as _; +use std::rc::Rc; + +const MAX_BYTES: usize = OSD::DEFAULT_MAX_BINARY_BYTES; + +pub(crate) fn deserialize_string(json: String) -> Result { + if json.len() > MAX_BYTES { + return Err(parse_error(0, "JSON OSD input exceeds allocation limit")); + } + let json = json.strip_prefix('\u{feff}').unwrap_or(&json); + let limits = Limits::default(); + let mut deserializer = serde_json::Deserializer::from_str(json); + let value = Seed { depth: 0, limits } + .deserialize(&mut deserializer) + .map_err(|error| json_error(json, &error))?; + deserializer + .end() + .map_err(|error| json_error(json, &error))?; + value.validate_limits( + OSD::DEFAULT_MAX_DEPTH, + OSD::DEFAULT_MAX_NODES, + OSD::DEFAULT_MAX_BINARY_BYTES, + )?; + Ok(value) +} + +pub(crate) fn deserialize_stream(mut json: Box) -> Result { + let mut bytes = Vec::new(); + (&mut *json) + .take((MAX_BYTES + 1) as u64) + .read_to_end(&mut bytes) + .map_err(|_| Error::InvalidOperation)?; + if bytes.len() > MAX_BYTES { + return Err(parse_error(0, "JSON OSD input exceeds allocation limit")); + } + let text = String::from_utf8(bytes) + .map_err(|_| parse_error(0, "JSON OSD input is not valid UTF-8"))?; + deserialize_string(text) +} + +pub(crate) fn serialize(osd: OSD, preserve_defaults: Option) -> Result { + osd.validate_limits( + OSD::DEFAULT_MAX_DEPTH, + OSD::DEFAULT_MAX_NODES, + OSD::DEFAULT_MAX_BINARY_BYTES, + )?; + let mut encoder = Encoder::new(); + encoder.write_value(&osd, preserve_defaults.unwrap_or(false), 0)?; + Ok(encoder.output) +} + +#[derive(Clone, Default)] +struct Limits { + nodes: Rc>, + allocated: Rc>, +} + +impl Limits { + fn add_node(&self, depth: usize) -> Result<(), E> { + if depth > OSD::DEFAULT_MAX_DEPTH { + return Err(E::custom("JSON OSD nesting depth exceeded")); + } + let nodes = self + .nodes + .get() + .checked_add(1) + .ok_or_else(|| E::custom("JSON OSD node count overflow"))?; + if nodes > OSD::DEFAULT_MAX_NODES { + return Err(E::custom("JSON OSD node limit exceeded")); + } + self.nodes.set(nodes); + Ok(()) + } + + fn add_allocation(&self, amount: usize) -> Result<(), E> { + let allocated = self + .allocated + .get() + .checked_add(amount) + .ok_or_else(|| E::custom("JSON OSD allocation overflow"))?; + if allocated > MAX_BYTES { + return Err(E::custom("JSON OSD allocation limit exceeded")); + } + self.allocated.set(allocated); + Ok(()) + } +} + +struct Seed { + depth: usize, + limits: Limits, +} + +impl<'de> DeserializeSeed<'de> for Seed { + type Value = OSD; + + fn deserialize>(self, deserializer: D) -> Result { + self.limits.add_node(self.depth)?; + deserializer.deserialize_any(OsdVisitor { + depth: self.depth, + limits: self.limits, + }) + } +} + +struct OsdVisitor { + depth: usize, + limits: Limits, +} + +impl<'de> Visitor<'de> for OsdVisitor { + type Value = OSD; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("a JSON value convertible to OSD") + } + + fn visit_bool(self, value: bool) -> Result { + Ok(OSD::Boolean(value)) + } + + fn visit_i64(self, value: i64) -> Result { + if let Ok(value) = i32::try_from(value) { + Ok(OSD::Integer(value)) + } else { + self.limits.add_allocation(8)?; + Ok(OSD::Binary(value.to_be_bytes().to_vec())) + } + } + + #[allow(clippy::cast_precision_loss)] + fn visit_u64(self, value: u64) -> Result { + if let Ok(value) = i32::try_from(value) { + Ok(OSD::Integer(value)) + } else if let Ok(value) = i64::try_from(value) { + self.limits.add_allocation(8)?; + Ok(OSD::Binary(value.to_be_bytes().to_vec())) + } else { + Ok(OSD::Real(value as f64)) + } + } + + fn visit_f64(self, value: f64) -> Result { + if value.is_finite() { + Ok(OSD::Real(value)) + } else { + Err(E::custom( + "JSON OSD number is outside the finite double range", + )) + } + } + + fn visit_str(self, value: &str) -> Result { + self.limits.add_allocation(value.len())?; + Ok(if value.is_empty() { + OSD::Undefined + } else { + OSD::String(value.to_owned()) + }) + } + + fn visit_string(self, value: String) -> Result { + self.limits.add_allocation(value.len())?; + Ok(if value.is_empty() { + OSD::Undefined + } else { + OSD::String(value) + }) + } + + fn visit_none(self) -> Result { + Ok(OSD::Undefined) + } + + fn visit_unit(self) -> Result { + Ok(OSD::Undefined) + } + + fn visit_seq>(self, mut sequence: A) -> Result { + let mut values = Vec::with_capacity(sequence.size_hint().unwrap_or(0).min(1024)); + while let Some(value) = sequence.next_element_seed(Seed { + depth: self.depth + 1, + limits: self.limits.clone(), + })? { + self.limits + .add_allocation::(std::mem::size_of::())?; + values.push(value); + } + Ok(OSD::Array(values)) + } + + fn visit_map>(self, mut object: A) -> Result { + let mut values = HashMap::with_capacity(object.size_hint().unwrap_or(0).min(1024)); + while let Some(key) = object.next_key::()? { + self.limits + .add_allocation::(key.len() + std::mem::size_of::<(String, OSD)>())?; + if values.contains_key(&key) { + return Err(A::Error::custom("duplicate JSON OSD object property")); + } + let value = object.next_value_seed(Seed { + depth: self.depth + 1, + limits: self.limits.clone(), + })?; + values.insert(key, value); + } + Ok(OSD::Map(values)) + } +} + +struct Encoder { + output: String, +} + +impl Encoder { + fn new() -> Self { + Self { + output: String::with_capacity(128), + } + } + + fn write_value(&mut self, value: &OSD, preserve: bool, depth: usize) -> Result<(), Error> { + if depth > OSD::DEFAULT_MAX_DEPTH { + return Err(Error::Argument); + } + if !preserve && is_default(value) { + return self.push("null"); + } + match value { + OSD::Undefined | OSD::LlsdXml(_) => self.push("null"), + OSD::Boolean(value) => self.push(if *value { "true" } else { "false" }), + OSD::Integer(value) => self.push(&value.to_string()), + OSD::Real(value) => { + let number = serde_json::Number::from_f64(*value).ok_or(Error::Argument)?; + self.push(&number.to_string()) + } + OSD::String(value) => self.write_string(value), + OSD::UUID(value) => self.write_string(&value.to_string()), + OSD::Date(value) => { + self.write_string(&crate::model::format_system_time_for_codec(*value)) + } + OSD::Uri(Uri(value)) => self.write_string(&crate::model::format_uri_for_codec(value)), + OSD::Binary(values) => { + self.push("[")?; + for (index, value) in values.iter().enumerate() { + if index != 0 { + self.push(",")?; + } + self.push(&value.to_string())?; + } + self.push("]") + } + OSD::Array(values) => { + self.push("[")?; + for (index, value) in values.iter().enumerate() { + if index != 0 { + self.push(",")?; + } + self.write_value(value, preserve, depth + 1)?; + } + self.push("]") + } + OSD::Map(values) => { + self.push("{")?; + let mut entries: Vec<_> = values.iter().collect(); + entries.sort_unstable_by_key(|(key, _)| *key); + let mut written = 0; + for (key, value) in entries { + if !preserve && is_default(value) { + continue; + } + if written != 0 { + self.push(",")?; + } + self.write_string(key)?; + self.push(":")?; + self.write_value(value, preserve, depth + 1)?; + written += 1; + } + self.push("}") + } + } + } + + fn write_string(&mut self, value: &str) -> Result<(), Error> { + self.push("\"")?; + let mut start = 0; + for (index, character) in value.char_indices() { + let escape = match character { + '\u{08}' => Some("\\b"), + '\t' => Some("\\t"), + '\n' => Some("\\n"), + '\u{0c}' => Some("\\f"), + '\r' => Some("\\r"), + '"' => Some("\\u0022"), + '&' => Some("\\u0026"), + '\'' => Some("\\u0027"), + '+' => Some("\\u002B"), + '<' => Some("\\u003C"), + '>' => Some("\\u003E"), + '\\' => Some("\\\\"), + character if character < ' ' || !character.is_ascii() => None, + _ => continue, + }; + self.push(&value[start..index])?; + if let Some(escape) = escape { + self.push(escape)?; + } else { + let mut units = [0_u16; 2]; + for unit in character.encode_utf16(&mut units) { + self.push(&format!("\\u{unit:04X}"))?; + } + } + start = index + character.len_utf8(); + } + self.push(&value[start..])?; + self.push("\"") + } + + fn push(&mut self, value: &str) -> Result<(), Error> { + let length = self + .output + .len() + .checked_add(value.len()) + .ok_or(Error::Argument)?; + if length > MAX_BYTES { + return Err(Error::Argument); + } + self.output.push_str(value); + Ok(()) + } +} + +fn is_default(value: &OSD) -> bool { + match value { + OSD::Undefined | OSD::LlsdXml(_) => true, + OSD::Boolean(value) => !value, + OSD::Integer(value) => *value == 0, + OSD::Real(value) => *value == 0.0, + OSD::String(value) | OSD::Uri(Uri(value)) => value.is_empty(), + OSD::UUID(value) => value.equals_with_uuid(libremetaverse_types::UUID::zero()), + OSD::Binary(value) => value.is_empty(), + OSD::Date(_) | OSD::Array(_) | OSD::Map(_) => false, + } +} + +fn json_error(input: &str, error: &serde_json::Error) -> Error { + let line = error.line().max(1); + let column = error.column().max(1); + let line_start = input + .match_indices('\n') + .take(line.saturating_sub(1)) + .last() + .map_or(0, |(index, _)| index + 1); + parse_error( + line_start.saturating_add(column - 1).min(input.len()), + "malformed or resource-limited JSON OSD", + ) +} + +const fn parse_error(position: usize, context: &'static str) -> Error { + Error::Parse { position, context } +} + +#[cfg(test)] +mod tests { + use super::*; + use libremetaverse_types::UUID; + + #[test] + fn explicit_json_coercions_and_default_policy_match_reference() { + assert_eq!(deserialize_string("\"\"".into()).unwrap(), OSD::Undefined); + assert_eq!(deserialize_string("null".into()).unwrap(), OSD::Undefined); + let stream: Box = + Box::new(std::io::Cursor::new(b"{\"stream\":true}".to_vec())); + assert_eq!( + deserialize_stream(stream).unwrap(), + OSD::Map(HashMap::from([("stream".into(), OSD::Boolean(true))])) + ); + assert_eq!( + deserialize_string("2147483648".into()).unwrap(), + OSD::Binary(2_147_483_648_i64.to_be_bytes().to_vec()) + ); + let value = OSD::Map(HashMap::from([ + ("false".into(), OSD::Boolean(false)), + ("true".into(), OSD::Boolean(true)), + ("zero".into(), OSD::Integer(0)), + ( + "array".into(), + OSD::Array(vec![OSD::Integer(0), OSD::String("x".into())]), + ), + ])); + assert_eq!( + serialize(value.clone(), None).unwrap(), + "{\"array\":[null,\"x\"],\"true\":true}" + ); + assert_eq!( + serialize(value, Some(true)).unwrap(), + include_str!("../../../tests/fixtures/structured_data/json_reference.json").trim() + ); + } + + #[test] + fn json_strings_use_system_text_json_compatible_escaping() { + let value = OSD::String("<&+\"'\\ ж 𠮟".into()); + assert_eq!( + serialize(value.clone(), Some(true)).unwrap(), + "\"\\u003C\\u0026\\u002B\\u0022\\u0027\\\\ \\u0436 \\uD842\\uDF9F\"" + ); + assert_eq!( + deserialize_string(serialize(value.clone(), Some(true)).unwrap()).unwrap(), + value + ); + } + + #[test] + fn malformed_duplicate_and_deep_json_are_positioned_errors() { + for input in ["{\"x\":1,\"x\":2}", "[1,", "1 trailing"] { + assert!(matches!( + deserialize_string(input.into()), + Err(Error::Parse { .. }) + )); + } + let too_deep = format!( + "{}null{}", + "[".repeat(OSD::DEFAULT_MAX_DEPTH + 2), + "]".repeat(OSD::DEFAULT_MAX_DEPTH + 2) + ); + assert!(matches!( + deserialize_string(too_deep), + Err(Error::Parse { .. }) + )); + } + + #[test] + fn json_round_trip_preserves_representable_osd_values() { + let value = OSD::Map(HashMap::from([ + ("boolean".into(), OSD::Boolean(false)), + ("integer".into(), OSD::Integer(-4)), + ("real".into(), OSD::Real(1.5)), + ("string".into(), OSD::String("text".into())), + ("uuid".into(), OSD::UUID(UUID::zero())), + ])); + let decoded = deserialize_string(serialize(value, Some(true)).unwrap()).unwrap(); + let OSD::Map(decoded) = decoded else { + panic!("expected JSON object"); + }; + assert_eq!(decoded["boolean"], OSD::Boolean(false)); + assert_eq!(decoded["integer"], OSD::Integer(-4)); + assert_eq!(decoded["real"], OSD::Real(1.5)); + assert_eq!(decoded["string"], OSD::String("text".into())); + assert_eq!( + decoded["uuid"], + OSD::String("00000000-0000-0000-0000-000000000000".into()) + ); + } +} diff --git a/crates/libremetaverse-structured-data/src/lib.rs b/crates/libremetaverse-structured-data/src/lib.rs index a655798..357cf6d 100644 --- a/crates/libremetaverse-structured-data/src/lib.rs +++ b/crates/libremetaverse-structured-data/src/lib.rs @@ -5,8 +5,10 @@ extern crate self as libremetaverse_structured_data; mod binary; mod dispatch; mod generated; +mod json_codec; mod model; mod notation; +mod protobuf; mod xml_codec; pub mod xml { diff --git a/crates/libremetaverse-structured-data/src/protobuf.rs b/crates/libremetaverse-structured-data/src/protobuf.rs new file mode 100644 index 0000000..1ddab28 --- /dev/null +++ b/crates/libremetaverse-structured-data/src/protobuf.rs @@ -0,0 +1,689 @@ +//! Private, bounded implementation of the `LibreMetaverse` OSD Protobuf schema. +//! +//! The pinned reference assigns fields 1 through 11 to the OSD type and value +//! alternatives. Integers use `ZigZag` varints; real and date values are IEEE-754 +//! fixed64 in little-endian Protobuf wire order. Unknown top-level and entry +//! fields are skipped according to their wire type. Map ordering is not part of +//! the schema, so this encoder sorts keys to make native output deterministic. + +#![allow(clippy::missing_errors_doc)] +#![allow(clippy::needless_pass_by_value)] + +use crate::{Error, OSD}; +use libremetaverse_types::UUID; +use libremetaverse_types::compat::{ReadWrite, Uri}; +use std::collections::HashMap; +use std::io::Read as _; +use std::time::{Duration, UNIX_EPOCH}; + +const HEADER: &[u8] = b""; +const MAX_BYTES: usize = OSD::DEFAULT_MAX_BINARY_BYTES; + +const WIRE_VARINT: u8 = 0; +const WIRE_FIXED64: u8 = 1; +const WIRE_LENGTH: u8 = 2; +const WIRE_FIXED32: u8 = 5; + +const FIELD_TYPE: u32 = 1; +const FIELD_BOOLEAN: u32 = 2; +const FIELD_INTEGER: u32 = 3; +const FIELD_REAL: u32 = 4; +const FIELD_STRING: u32 = 5; +const FIELD_UUID: u32 = 6; +const FIELD_DATE: u32 = 7; +const FIELD_URI: u32 = 8; +const FIELD_BINARY: u32 = 9; +const FIELD_MAP_ENTRIES: u32 = 10; +const FIELD_ARRAY_ELEMENTS: u32 = 11; + +pub(crate) fn deserialize_bytes(data: Vec) -> Result { + if data.len() > MAX_BYTES { + return Err(parse_error( + 0, + "Protobuf OSD input exceeds allocation limit", + )); + } + let (payload, origin) = strip_header(&data); + let mut budget = Budget::default(); + let value = parse_value(payload, origin, 0, &mut budget)?; + value.validate_limits( + OSD::DEFAULT_MAX_DEPTH, + OSD::DEFAULT_MAX_NODES, + OSD::DEFAULT_MAX_BINARY_BYTES, + )?; + Ok(value) +} + +pub(crate) fn deserialize_stream(mut stream: Box) -> Result { + let mut data = Vec::new(); + (&mut *stream) + .take((MAX_BYTES + 1) as u64) + .read_to_end(&mut data) + .map_err(|_| Error::InvalidOperation)?; + deserialize_bytes(data) +} + +pub(crate) fn serialize(osd: OSD, prepend_header: Option) -> Result, Error> { + osd.validate_limits( + OSD::DEFAULT_MAX_DEPTH, + OSD::DEFAULT_MAX_NODES, + OSD::DEFAULT_MAX_BINARY_BYTES, + )?; + let mut output = Vec::new(); + if prepend_header.unwrap_or(true) { + push(&mut output, HEADER)?; + push(&mut output, b"\n")?; + } + push(&mut output, &encode_value(&osd, 0)?)?; + Ok(output) +} + +fn strip_header(data: &[u8]) -> (&[u8], usize) { + if let Some(rest) = data.strip_prefix(HEADER) { + if let Some(rest) = rest.strip_prefix(b"\n") { + (rest, HEADER.len() + 1) + } else { + (rest, HEADER.len()) + } + } else { + (data, 0) + } +} + +#[derive(Default)] +struct Budget { + nodes: usize, + allocated: usize, +} + +impl Budget { + fn node(&mut self, depth: usize, position: usize) -> Result<(), Error> { + if depth > OSD::DEFAULT_MAX_DEPTH { + return Err(parse_error(position, "Protobuf OSD nesting depth exceeded")); + } + self.nodes = self + .nodes + .checked_add(1) + .ok_or_else(|| parse_error(position, "Protobuf OSD node overflow"))?; + if self.nodes > OSD::DEFAULT_MAX_NODES { + return Err(parse_error(position, "Protobuf OSD node limit exceeded")); + } + Ok(()) + } + + fn allocate(&mut self, amount: usize, position: usize) -> Result<(), Error> { + self.allocated = self + .allocated + .checked_add(amount) + .ok_or_else(|| parse_error(position, "Protobuf OSD allocation overflow"))?; + if self.allocated > MAX_BYTES { + return Err(parse_error( + position, + "Protobuf OSD allocation limit exceeded", + )); + } + Ok(()) + } +} + +enum Scalar { + Boolean(bool), + Integer(i32), + Real(f64), + String(String), + Uuid(UUID), + Date(std::time::SystemTime), + Binary(Vec), +} + +#[allow(clippy::too_many_lines)] +fn parse_value( + input: &[u8], + origin: usize, + depth: usize, + budget: &mut Budget, +) -> Result { + budget.node(depth, origin)?; + let mut cursor = Cursor::new(input, origin); + let mut type_code = 0_u64; + let mut scalar = None; + let mut map = None; + let mut array = None; + + while !cursor.is_empty() { + let tag_position = cursor.absolute_position(); + let tag = cursor.varint()?; + let field = u32::try_from(tag >> 3) + .map_err(|_| parse_error(tag_position, "Protobuf OSD field number overflow"))?; + let wire = (tag & 7) as u8; + if field == 0 { + return Err(parse_error( + tag_position, + "Protobuf OSD field number zero is invalid", + )); + } + match field { + FIELD_TYPE => { + expect_wire(wire, WIRE_VARINT, tag_position, "OSD type")?; + type_code = cursor.varint()?; + } + FIELD_BOOLEAN => { + expect_wire(wire, WIRE_VARINT, tag_position, "boolean")?; + scalar = Some(Scalar::Boolean(cursor.varint()? != 0)); + } + FIELD_INTEGER => { + expect_wire(wire, WIRE_VARINT, tag_position, "integer")?; + let encoded = cursor.varint()?; + let magnitude = i64::try_from(encoded >> 1).map_err(|_| { + parse_error(tag_position, "Protobuf OSD ZigZag integer overflow") + })?; + let sign = i64::try_from(encoded & 1) + .map_err(|_| parse_error(tag_position, "Protobuf OSD ZigZag sign overflow"))?; + let decoded = magnitude ^ -sign; + let bytes = decoded.to_le_bytes(); + scalar = Some(Scalar::Integer(i32::from_le_bytes([ + bytes[0], bytes[1], bytes[2], bytes[3], + ]))); + } + FIELD_REAL => { + expect_wire(wire, WIRE_FIXED64, tag_position, "real")?; + scalar = Some(Scalar::Real(cursor.fixed64()?)); + } + FIELD_STRING | FIELD_URI => { + expect_wire(wire, WIRE_LENGTH, tag_position, "string/URI")?; + let position = cursor.absolute_position(); + let bytes = cursor.length_delimited()?; + budget.allocate(bytes.len(), position)?; + let value = std::str::from_utf8(bytes) + .map_err(|_| parse_error(position, "Protobuf OSD string is not valid UTF-8"))?; + scalar = Some(Scalar::String(value.to_owned())); + } + FIELD_UUID => { + expect_wire(wire, WIRE_LENGTH, tag_position, "UUID")?; + let position = cursor.absolute_position(); + let bytes = cursor.length_delimited()?; + if bytes.len() != 16 { + return Err(parse_error( + position, + "Protobuf OSD UUID must contain 16 bytes", + )); + } + scalar = Some(Scalar::Uuid( + UUID::from_bytes(bytes.to_vec(), 0) + .map_err(|_| parse_error(position, "invalid Protobuf OSD UUID"))?, + )); + } + FIELD_DATE => { + expect_wire(wire, WIRE_FIXED64, tag_position, "date")?; + let seconds = cursor.fixed64()?; + if !seconds.is_finite() { + return Err(parse_error(tag_position, "invalid Protobuf OSD date")); + } + scalar = Some(Scalar::Date( + UNIX_EPOCH + Duration::from_secs(u64::from(reference_u32_from_f64(seconds))), + )); + } + FIELD_BINARY => { + expect_wire(wire, WIRE_LENGTH, tag_position, "binary")?; + let position = cursor.absolute_position(); + let bytes = cursor.length_delimited()?; + budget.allocate(bytes.len(), position)?; + scalar = Some(Scalar::Binary(bytes.to_vec())); + } + FIELD_MAP_ENTRIES => { + expect_wire(wire, WIRE_LENGTH, tag_position, "map entry")?; + let entry = cursor.length_delimited()?; + let entry_origin = cursor.absolute_position() - entry.len(); + let values = map.get_or_insert_with(HashMap::new); + parse_map_entry(entry, entry_origin, depth, budget, values)?; + } + FIELD_ARRAY_ELEMENTS => { + expect_wire(wire, WIRE_LENGTH, tag_position, "array element")?; + let element = cursor.length_delimited()?; + let element_origin = cursor.absolute_position() - element.len(); + budget.allocate(std::mem::size_of::(), element_origin)?; + array.get_or_insert_with(Vec::new).push(parse_value( + element, + element_origin, + depth + 1, + budget, + )?); + } + _ => cursor.skip(wire, tag_position)?, + } + } + + construct(type_code, scalar, map, array, origin) +} + +fn construct( + type_code: u64, + scalar: Option, + map: Option>, + array: Option>, + position: usize, +) -> Result { + match type_code { + 1 => match scalar { + Some(Scalar::Boolean(value)) => Ok(OSD::Boolean(value)), + None => Ok(OSD::Undefined), + _ => Err(parse_error(position, "Protobuf OSD boolean field mismatch")), + }, + 2 => match scalar { + Some(Scalar::Integer(value)) => Ok(OSD::Integer(value)), + None => Ok(OSD::Undefined), + _ => Err(parse_error(position, "Protobuf OSD integer field mismatch")), + }, + 3 => match scalar { + Some(Scalar::Real(value)) => Ok(OSD::Real(value)), + None => Ok(OSD::Undefined), + _ => Err(parse_error(position, "Protobuf OSD real field mismatch")), + }, + 4 => match scalar { + Some(Scalar::String(value)) => Ok(OSD::String(value)), + None => Ok(OSD::String(String::new())), + _ => Err(parse_error(position, "Protobuf OSD string field mismatch")), + }, + 5 => match scalar { + Some(Scalar::Uuid(value)) => Ok(OSD::UUID(value)), + None => Ok(OSD::UUID(UUID::zero())), + _ => Err(parse_error(position, "Protobuf OSD UUID field mismatch")), + }, + 6 => match scalar { + Some(Scalar::Date(value)) => Ok(OSD::Date(value)), + None => Ok(OSD::Date(UNIX_EPOCH)), + _ => Err(parse_error(position, "Protobuf OSD date field mismatch")), + }, + 7 => match scalar { + Some(Scalar::String(value)) => Ok(OSD::Uri(Uri(value))), + None => Ok(OSD::Uri(Uri(String::new()))), + _ => Err(parse_error(position, "Protobuf OSD URI field mismatch")), + }, + 8 => match scalar { + Some(Scalar::Binary(value)) => Ok(OSD::Binary(value)), + None => Ok(OSD::Binary(Vec::new())), + _ => Err(parse_error(position, "Protobuf OSD binary field mismatch")), + }, + 9 => Ok(OSD::Map(map.unwrap_or_default())), + 10 => Ok(OSD::Array(array.unwrap_or_default())), + _ => Ok(OSD::Undefined), + } +} + +fn parse_map_entry( + input: &[u8], + origin: usize, + depth: usize, + budget: &mut Budget, + map: &mut HashMap, +) -> Result<(), Error> { + let mut cursor = Cursor::new(input, origin); + let mut key = None; + let mut value = None; + while !cursor.is_empty() { + let tag_position = cursor.absolute_position(); + let tag = cursor.varint()?; + let field = u32::try_from(tag >> 3) + .map_err(|_| parse_error(tag_position, "Protobuf map field number overflow"))?; + let wire = (tag & 7) as u8; + if field == 0 { + return Err(parse_error( + tag_position, + "Protobuf map field number zero is invalid", + )); + } + match field { + 1 => { + expect_wire(wire, WIRE_LENGTH, tag_position, "map key")?; + let position = cursor.absolute_position(); + let bytes = cursor.length_delimited()?; + budget.allocate(bytes.len(), position)?; + key = Some( + std::str::from_utf8(bytes) + .map_err(|_| parse_error(position, "Protobuf map key is not valid UTF-8"))? + .to_owned(), + ); + } + 2 => { + expect_wire(wire, WIRE_LENGTH, tag_position, "map value")?; + let bytes = cursor.length_delimited()?; + let value_origin = cursor.absolute_position() - bytes.len(); + value = Some(parse_value(bytes, value_origin, depth + 1, budget)?); + } + _ => cursor.skip(wire, tag_position)?, + } + } + if let (Some(key), Some(value)) = (key, value) { + budget.allocate(std::mem::size_of::<(String, OSD)>(), origin)?; + map.insert(key, value); + } + Ok(()) +} + +struct Cursor<'a> { + input: &'a [u8], + position: usize, + origin: usize, +} + +impl<'a> Cursor<'a> { + const fn new(input: &'a [u8], origin: usize) -> Self { + Self { + input, + position: 0, + origin, + } + } + + fn varint(&mut self) -> Result { + let start = self.absolute_position(); + let mut value = 0_u64; + for shift in (0..=63).step_by(7) { + let byte = self.byte()?; + if shift == 63 && byte > 1 { + return Err(parse_error(start, "Protobuf OSD varint overflow")); + } + value |= u64::from(byte & 0x7f) << shift; + if byte & 0x80 == 0 { + return Ok(value); + } + } + Err(parse_error(start, "Protobuf OSD varint exceeds ten bytes")) + } + + fn fixed64(&mut self) -> Result { + let bytes: [u8; 8] = self + .bytes(8)? + .try_into() + .map_err(|_| parse_error(self.absolute_position(), "truncated fixed64"))?; + Ok(f64::from_le_bytes(bytes)) + } + + fn length_delimited(&mut self) -> Result<&'a [u8], Error> { + let position = self.absolute_position(); + let length = usize::try_from(self.varint()?) + .map_err(|_| parse_error(position, "Protobuf OSD length overflow"))?; + self.bytes(length) + } + + fn skip(&mut self, wire: u8, position: usize) -> Result<(), Error> { + match wire { + WIRE_VARINT => { + self.varint()?; + } + WIRE_FIXED64 => { + self.bytes(8)?; + } + WIRE_LENGTH => { + self.length_delimited()?; + } + WIRE_FIXED32 => { + self.bytes(4)?; + } + _ => { + return Err(parse_error(position, "unknown Protobuf OSD wire type")); + } + } + Ok(()) + } + + fn byte(&mut self) -> Result { + let byte = + self.input.get(self.position).copied().ok_or_else(|| { + parse_error(self.absolute_position(), "truncated Protobuf OSD value") + })?; + self.position += 1; + Ok(byte) + } + + fn bytes(&mut self, length: usize) -> Result<&'a [u8], Error> { + let end = self + .position + .checked_add(length) + .ok_or_else(|| parse_error(self.absolute_position(), "Protobuf OSD length overflow"))?; + let bytes = self + .input + .get(self.position..end) + .ok_or_else(|| parse_error(self.absolute_position(), "truncated Protobuf OSD value"))?; + self.position = end; + Ok(bytes) + } + + const fn is_empty(&self) -> bool { + self.position == self.input.len() + } + + const fn absolute_position(&self) -> usize { + self.origin + self.position + } +} + +fn expect_wire( + actual: u8, + expected: u8, + position: usize, + _field: &'static str, +) -> Result<(), Error> { + if actual == expected { + Ok(()) + } else { + Err(parse_error(position, "incorrect Protobuf OSD wire type")) + } +} + +fn encode_value(value: &OSD, depth: usize) -> Result, Error> { + if depth > OSD::DEFAULT_MAX_DEPTH { + return Err(Error::Argument); + } + let mut output = Vec::new(); + write_varint(&mut output, make_tag(FIELD_TYPE, WIRE_VARINT))?; + write_varint(&mut output, value.type_() as u64)?; + match value { + OSD::Undefined | OSD::LlsdXml(_) => {} + OSD::Boolean(value) => { + write_varint(&mut output, make_tag(FIELD_BOOLEAN, WIRE_VARINT))?; + write_varint(&mut output, u64::from(*value))?; + } + OSD::Integer(value) => { + write_varint(&mut output, make_tag(FIELD_INTEGER, WIRE_VARINT))?; + let bits = u32::from_ne_bytes(value.to_ne_bytes()); + let sign = u32::from_ne_bytes((*value >> 31).to_ne_bytes()); + write_varint(&mut output, u64::from(bits.wrapping_shl(1) ^ sign))?; + } + OSD::Real(value) => { + write_varint(&mut output, make_tag(FIELD_REAL, WIRE_FIXED64))?; + push(&mut output, &value.to_le_bytes())?; + } + OSD::String(value) => { + write_varint(&mut output, make_tag(FIELD_STRING, WIRE_LENGTH))?; + write_length_delimited(&mut output, value.as_bytes())?; + } + OSD::UUID(value) => { + write_varint(&mut output, make_tag(FIELD_UUID, WIRE_LENGTH))?; + write_length_delimited(&mut output, &value.get_bytes()?)?; + } + OSD::Date(value) => { + write_varint(&mut output, make_tag(FIELD_DATE, WIRE_FIXED64))?; + let seconds = match value.duration_since(UNIX_EPOCH) { + Ok(duration) => low_u32(duration.as_secs()), + Err(error) => low_u32(error.duration().as_secs()).wrapping_neg(), + }; + push(&mut output, &f64::from(seconds).to_le_bytes())?; + } + OSD::Uri(Uri(value)) => { + write_varint(&mut output, make_tag(FIELD_URI, WIRE_LENGTH))?; + write_length_delimited( + &mut output, + crate::model::format_uri_for_codec(value).as_bytes(), + )?; + } + OSD::Binary(value) => { + write_varint(&mut output, make_tag(FIELD_BINARY, WIRE_LENGTH))?; + write_length_delimited(&mut output, value)?; + } + OSD::Map(values) => { + let mut entries: Vec<_> = values.iter().collect(); + entries.sort_unstable_by_key(|(key, _)| *key); + for (key, value) in entries { + write_varint(&mut output, make_tag(FIELD_MAP_ENTRIES, WIRE_LENGTH))?; + let mut entry = Vec::new(); + write_varint(&mut entry, make_tag(1, WIRE_LENGTH))?; + write_length_delimited(&mut entry, key.as_bytes())?; + write_varint(&mut entry, make_tag(2, WIRE_LENGTH))?; + write_length_delimited(&mut entry, &encode_value(value, depth + 1)?)?; + write_length_delimited(&mut output, &entry)?; + } + } + OSD::Array(values) => { + for value in values { + write_varint(&mut output, make_tag(FIELD_ARRAY_ELEMENTS, WIRE_LENGTH))?; + write_length_delimited(&mut output, &encode_value(value, depth + 1)?)?; + } + } + } + Ok(output) +} + +const fn make_tag(field: u32, wire: u8) -> u64 { + ((field as u64) << 3) | wire as u64 +} + +#[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] +fn reference_u32_from_f64(value: f64) -> u32 { + value as u32 +} + +const fn low_u32(value: u64) -> u32 { + let bytes = value.to_le_bytes(); + u32::from_le_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]) +} + +fn write_varint(output: &mut Vec, mut value: u64) -> Result<(), Error> { + loop { + let mut byte = (value & 0x7f) as u8; + value >>= 7; + if value != 0 { + byte |= 0x80; + } + push(output, &[byte])?; + if value == 0 { + return Ok(()); + } + } +} + +fn write_length_delimited(output: &mut Vec, value: &[u8]) -> Result<(), Error> { + write_varint(output, value.len() as u64)?; + push(output, value) +} + +fn push(output: &mut Vec, value: &[u8]) -> Result<(), Error> { + let length = output + .len() + .checked_add(value.len()) + .ok_or(Error::Argument)?; + if length > MAX_BYTES { + return Err(Error::Argument); + } + output.extend_from_slice(value); + Ok(()) +} + +const fn parse_error(position: usize, context: &'static str) -> Error { + Error::Parse { position, context } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn scalar_golden_bytes_match_the_private_reference_schema() { + assert_eq!( + serialize(OSD::Boolean(true), Some(false)).unwrap(), + [0x08, 0x01, 0x10, 0x01] + ); + assert_eq!( + serialize(OSD::Integer(-1), Some(false)).unwrap(), + [0x08, 0x02, 0x18, 0x01] + ); + assert_eq!( + serialize(OSD::String("A".into()), Some(false)).unwrap(), + [0x08, 0x04, 0x2a, 0x01, b'A'] + ); + let mut real = vec![0x08, 0x03, 0x21]; + real.extend_from_slice(&1.5_f64.to_le_bytes()); + assert_eq!(serialize(OSD::Real(1.5), Some(false)).unwrap(), real); + } + + #[test] + fn composite_golden_bytes_and_header_are_stable() { + let value = OSD::Map(HashMap::from([("a".into(), OSD::Integer(1))])); + let fixture: Vec = + include_str!("../../../tests/fixtures/structured_data/protobuf_reference.hex") + .split_ascii_whitespace() + .map(|byte| u8::from_str_radix(byte, 16).unwrap()) + .collect(); + assert_eq!(serialize(value.clone(), Some(false)).unwrap(), fixture); + let encoded = serialize(value.clone(), None).unwrap(); + assert!(encoded.starts_with(b"\n")); + assert_eq!(deserialize_bytes(encoded.clone()).unwrap(), value); + let stream: Box = Box::new(std::io::Cursor::new(encoded)); + assert_eq!(deserialize_stream(stream).unwrap(), value); + } + + #[test] + fn every_osd_variant_round_trips_as_defined_by_schema() { + let values = [ + OSD::Undefined, + OSD::Boolean(false), + OSD::Integer(i32::MIN), + OSD::Real(-12.5), + OSD::String(String::new()), + OSD::UUID(UUID::zero()), + OSD::Date(UNIX_EPOCH + Duration::from_secs(42)), + OSD::Uri(Uri("relative/path".into())), + OSD::Binary(vec![0, 255]), + OSD::Map(HashMap::new()), + OSD::Array(Vec::new()), + ]; + for value in values { + assert_eq!( + deserialize_bytes(serialize(value.clone(), Some(false)).unwrap()).unwrap(), + value + ); + } + assert_eq!( + deserialize_bytes(serialize(OSD::LlsdXml("".into()), Some(false)).unwrap()) + .unwrap(), + OSD::Undefined + ); + } + + #[test] + fn unknown_fields_are_skipped_and_malformed_lengths_are_positioned() { + let mut encoded = serialize(OSD::Boolean(true), Some(false)).unwrap(); + encoded.extend_from_slice(&[0x78, 0x7f]); + assert_eq!(deserialize_bytes(encoded).unwrap(), OSD::Boolean(true)); + for malformed in [ + vec![0x08], + vec![0x08, 0x04, 0x2a, 0x7f], + vec![0x0b], + vec![0x08, 0x01, 0x15, 0, 0, 0, 0], + ] { + assert!(matches!( + deserialize_bytes(malformed), + Err(Error::Parse { .. }) + )); + } + } + + #[test] + fn nested_depth_is_bounded_before_recursive_descent() { + let mut value = OSD::Undefined; + for _ in 0..=OSD::DEFAULT_MAX_DEPTH { + value = OSD::Array(vec![value]); + } + assert_eq!(serialize(value, Some(false)), Err(Error::Argument)); + } +} diff --git a/tests/api-compile/Cargo.lock b/tests/api-compile/Cargo.lock index 539b097..94ff9dd 100644 --- a/tests/api-compile/Cargo.lock +++ b/tests/api-compile/Cargo.lock @@ -156,6 +156,12 @@ dependencies = [ "typenum", ] +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + [[package]] name = "js-sys" version = "0.3.104" @@ -245,6 +251,8 @@ version = "0.0.1" dependencies = [ "base64", "libremetaverse-types", + "serde", + "serde_json", ] [[package]] @@ -296,6 +304,12 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + [[package]] name = "metacrate-api-compile" version = "0.0.0" @@ -357,6 +371,48 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + [[package]] name = "sha1" version = "0.10.7" @@ -396,6 +452,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "typenum" version = "1.20.1" @@ -457,7 +524,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn", + "syn 2.0.119", "wasm-bindgen-shared", ] @@ -469,3 +536,9 @@ checksum = "7ef64dbcc55df09c7e5a46182d181c2cfa3e925f3da937ea764728b4bbb9dcbf" dependencies = [ "unicode-ident", ] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/tests/fixtures/structured_data/json_reference.json b/tests/fixtures/structured_data/json_reference.json new file mode 100644 index 0000000..79cab45 --- /dev/null +++ b/tests/fixtures/structured_data/json_reference.json @@ -0,0 +1 @@ +{"array":[0,"x"],"false":false,"true":true,"zero":0} diff --git a/tests/fixtures/structured_data/protobuf_reference.hex b/tests/fixtures/structured_data/protobuf_reference.hex new file mode 100644 index 0000000..326656c --- /dev/null +++ b/tests/fixtures/structured_data/protobuf_reference.hex @@ -0,0 +1 @@ +08 09 52 09 0a 01 61 12 04 08 02 18 02 diff --git a/tests/red-suite-baseline.json b/tests/red-suite-baseline.json index a27680d..2a9bbe4 100644 --- a/tests/red-suite-baseline.json +++ b/tests/red-suite-baseline.json @@ -63,6 +63,18 @@ "LibreMetaverse.Tests/NotationLLSDTests.cs::NotationSDTests.SerializeURI::test", "LibreMetaverse.Tests/NotationLLSDTests.cs::NotationSDTests.SerializeUUID::test", "LibreMetaverse.Tests/NotationLLSDTests.cs::NotationSDTests.SerializeUndef::test", + "LibreMetaverse.Tests/ProtobufTests.cs::ProtobufTests.AutoDetectProtobuf::test", + "LibreMetaverse.Tests/ProtobufTests.cs::ProtobufTests.SerializeArray::test", + "LibreMetaverse.Tests/ProtobufTests.cs::ProtobufTests.SerializeBinary::test", + "LibreMetaverse.Tests/ProtobufTests.cs::ProtobufTests.SerializeBoolean::test", + "LibreMetaverse.Tests/ProtobufTests.cs::ProtobufTests.SerializeDate::test", + "LibreMetaverse.Tests/ProtobufTests.cs::ProtobufTests.SerializeInteger::test", + "LibreMetaverse.Tests/ProtobufTests.cs::ProtobufTests.SerializeMap::test", + "LibreMetaverse.Tests/ProtobufTests.cs::ProtobufTests.SerializeNestedComposite::test", + "LibreMetaverse.Tests/ProtobufTests.cs::ProtobufTests.SerializeReal::test", + "LibreMetaverse.Tests/ProtobufTests.cs::ProtobufTests.SerializeString::test", + "LibreMetaverse.Tests/ProtobufTests.cs::ProtobufTests.SerializeUUID::test", + "LibreMetaverse.Tests/ProtobufTests.cs::ProtobufTests.SerializeUri::test", "LibreMetaverse.Tests/TypeTests.cs::TypeTests.LLSDTerseParsing::test", "LibreMetaverse.Tests/TypeTests.cs::TypeTests.Quaternions::test", "LibreMetaverse.Tests/TypeTests.cs::TypeTests.TestMatrix::test", @@ -86,5 +98,5 @@ "LibreMetaverse.Tests/XmlLLSDTests.cs::XmlSDTests.DeserializeUUID::test", "LibreMetaverse.Tests/XmlLLSDTests.cs::XmlSDTests.DeserializeUndef::test" ], - "support_passes": 84 + "support_passes": 93 } diff --git a/tools/generate_api_shims.py b/tools/generate_api_shims.py index c580a66..831bd54 100644 --- a/tools/generate_api_shims.py +++ b/tools/generate_api_shims.py @@ -91,6 +91,14 @@ NATIVE_MEMBER_BODIES = { "crate::notation::deserialize_reader(reader)", "M:LibreMetaverse.StructuredData.OSDParser.DeserializeLLSDNotation(System.String)": "crate::notation::deserialize_string(notation_data)", + "M:LibreMetaverse.StructuredData.OSDParser.DeserializeLLSDProtobuf(System.Byte[])": + "crate::protobuf::deserialize_bytes(data)", + "M:LibreMetaverse.StructuredData.OSDParser.DeserializeLLSDProtobuf(System.IO.Stream)": + "crate::protobuf::deserialize_stream(stream)", + "M:LibreMetaverse.StructuredData.OSDParser.DeserializeJson(System.IO.Stream)": + "crate::json_codec::deserialize_stream(json)", + "M:LibreMetaverse.StructuredData.OSDParser.DeserializeJson(System.String)": + "crate::json_codec::deserialize_string(json)", "M:LibreMetaverse.StructuredData.OSDParser.DeserializeLLSDXml(System.Byte[])": "crate::xml_codec::deserialize_bytes(xml_data)", "M:LibreMetaverse.StructuredData.OSDParser.DeserializeLLSDXml(System.IO.Stream)": @@ -127,6 +135,8 @@ NATIVE_MEMBER_BODIES = { "crate::binary::serialize_stream(data)", "M:LibreMetaverse.StructuredData.OSDParser.SerializeLLSDBinaryStream(LibreMetaverse.StructuredData.OSD,System.Boolean)": "crate::binary::serialize_stream_with_header(data, prepend_header)", + "M:LibreMetaverse.StructuredData.OSDParser.SerializeJsonString(LibreMetaverse.StructuredData.OSD,System.Boolean)": + "crate::json_codec::serialize(osd, preserve_defaults)", "M:LibreMetaverse.StructuredData.OSDParser.SerializeLLSDInnerXmlString(LibreMetaverse.StructuredData.OSD)": "crate::xml_codec::serialize_inner(data)", "M:LibreMetaverse.StructuredData.OSDParser.SerializeLLSDNotation(LibreMetaverse.StructuredData.OSD)": @@ -137,6 +147,8 @@ NATIVE_MEMBER_BODIES = { "crate::notation::serialize_stream(osd)", "M:LibreMetaverse.StructuredData.OSDParser.SerializeLLSDNotationStreamFormatted(LibreMetaverse.StructuredData.OSD)": "crate::notation::serialize_stream_formatted(osd)", + "M:LibreMetaverse.StructuredData.OSDParser.SerializeLLSDProtobuf(LibreMetaverse.StructuredData.OSD,System.Boolean)": + "crate::protobuf::serialize(osd, prepend_header)", "M:LibreMetaverse.StructuredData.OSDParser.SerializeLLSDXmlBytes(LibreMetaverse.StructuredData.OSD)": "crate::xml_codec::serialize_bytes(data)", "M:LibreMetaverse.StructuredData.OSDParser.SerializeLLSDXmlElement(System.Xml.XmlWriter,LibreMetaverse.StructuredData.OSD)": @@ -988,7 +1000,7 @@ def coverage_report(catalog: dict, coverage: dict[str, tuple[int, int, bool]]) - if item["doc_id"] in native_type_ids or member["doc_id"] in NATIVE_MEMBER_BODIES } if native_type_ids or native_member_ids: - if (len(native_type_ids), len(native_member_ids)) == (types, members): + if len(native_member_ids) == members: status = ( f"native implementation: {len(native_type_ids):,} types / " f"{len(native_member_ids):,} members; no generated shims remain" diff --git a/tools/generate_rust_mapping.py b/tools/generate_rust_mapping.py index 4fb2398..2fcd9a9 100644 --- a/tools/generate_rust_mapping.py +++ b/tools/generate_rust_mapping.py @@ -982,7 +982,9 @@ def validate_generated_shims() -> None: "crate::byte_order::", "crate::binary::", "crate::dispatch::", + "crate::json_codec::", "crate::notation::", + "crate::protobuf::", "crate::xml_codec::", ) ):