hugo/MetaCrate
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

Implement JSON and Protobuf OSD codecs

Browse Source
This commit is contained in:
Chili Palmer
2026-08-09 01:30:59 +00:00
parent 84bc31d031
commit 6d02db8a9a
15 changed files with 1365 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

75
Cargo.lock generated
View File

@@ -156,6 +156,12 @@ dependencies = [
"typenum",
]
[[package]]
name = "itoa"
version = "1.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
[[package]]
name = "js-sys"
version = "0.3.104"
@@ -274,6 +280,8 @@ version = "0.0.1"
dependencies = [
"base64",
"libremetaverse-types",
"serde",
"serde_json",
]
[[package]]
@@ -326,6 +334,12 @@ dependencies = [
"digest 0.10.7",
]
[[package]]
name = "memchr"
version = "2.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
[[package]]
name = "once_cell"
version = "1.21.4"
@@ -368,6 +382,48 @@ version = "1.0.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f"
[[package]]
name = "serde"
version = "1.0.229"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba"
dependencies = [
"serde_core",
]
[[package]]
name = "serde_core"
version = "1.0.229"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48"
dependencies = [
"serde_derive",
]
[[package]]
name = "serde_derive"
version = "1.0.229"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
dependencies = [
"proc-macro2",
"quote",
"syn 3.0.3",
]
[[package]]
name = "serde_json"
version = "1.0.151"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14"
dependencies = [
"itoa",
"memchr",
"serde",
"serde_core",
"zmij",
]
[[package]]
name = "sha1"
version = "0.10.7"
@@ -413,6 +469,17 @@ dependencies = [
"unicode-ident",
]
[[package]]
name = "syn"
version = "3.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3"
dependencies = [
"proc-macro2",
"quote",
"unicode-ident",
]
[[package]]
name = "typenum"
version = "1.20.1"
@@ -474,7 +541,7 @@ dependencies = [
"bumpalo",
"proc-macro2",
"quote",
"syn",
"syn 2.0.119",
"wasm-bindgen-shared",
]
@@ -486,3 +553,9 @@ checksum = "7ef64dbcc55df09c7e5a46182d181c2cfa3e925f3da937ea764728b4bbb9dcbf"
dependencies = [
"unicode-ident",
]
[[package]]
name = "zmij"
version = "1.0.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"

10
README.md
View File

@@ -97,5 +97,15 @@ instruction handling. Serialization is compact and deterministic for maps.
DTD declarations and named entity expansion are disabled; input, output,
nesting, node, and aggregate allocation limits are enforced with byte-positioned
parse errors.
JSON conversion is explicit rather than a derived Serde mapping: integral
width coercion, empty strings, binary arrays, typed values, and the reference's
default omission rules retain their C# behavior. Its reader rejects duplicate
properties and enforces byte, depth, node, and allocation limits while compact
output uses System.Text.Json-compatible escaping and stable map-key ordering.
The private OSD Protobuf schema is implemented directly with fixed field tags,
ZigZag `int32`, little-endian IEEE-754 fixed64 values, 16-byte UUIDs, bounded
length-delimited containers, and wire-type-aware unknown-field skipping. The
schema remains an internal compatibility format; native encoders emit stable
sorted maps and accept the exact optional LLSD Protobuf header.
The controlled audit aggregates every expected failure by standardized C#
member ID and rejects unrelated fixture, assertion, compile, or symbol errors.

2
api/SHIM-COVERAGE.md
View File

@@ -12,7 +12,7 @@ Generated by `python3 tools/generate_api_shims.py`; do not edit by hand.
| `LibreMetaverse.RLV` | 28 | 499 | callable failure-only shim |
| `LibreMetaverse.Rendering.MeshFoundry` | 1 | 14 | callable failure-only shim |
| `LibreMetaverse.Rendering.Simple` | 1 | 6 | callable failure-only shim |
| `LibreMetaverse.StructuredData` | 16 | 295 | native implementation: 15 types / 289 members; remaining surface is callable failure-only shims |
| `LibreMetaverse.StructuredData` | 16 | 295 | native implementation: 15 types / 295 members; no generated shims remain |
| `LibreMetaverse.Types` | 45 | 942 | native implementation: 45 types / 942 members; no generated shims remain |
| `LibreMetaverse.Utilities` | 3 | 13 | callable failure-only shim |
| `LibreMetaverse.Voice.Vivox` | 64 | 531 | callable failure-only shim |

2
crates/libremetaverse-structured-data/Cargo.toml
View File

@@ -10,6 +10,8 @@ description = "OSD and LLSD shims for the MetaCrate LibreMetaverse rewrite"
[dependencies]
base64 = "0.22"
libremetaverse-types = { path = "../libremetaverse-types" }
serde = "1"
serde_json = "1"
[lints]
workspace = true

30
crates/libremetaverse-structured-data/src/dispatch.rs
View File

@@ -120,6 +120,7 @@ fn starts_with_ignore_ascii_case(value: &str, prefix: &str) -> bool {
#[cfg(test)]
mod tests {
use super::*;
use std::collections::HashMap;
#[test]
fn byte_dispatch_recognizes_headers_bom_whitespace_and_json_tokens() {
@@ -164,31 +165,16 @@ mod tests {
deserialize_bytes(b"<llsd><undef /></llsd>".to_vec()),
Ok(OSD::Undefined)
);
let cases: &[(&[u8], &str)] = &[
(
b"<? llsd/protobuf ?>",
"M:LibreMetaverse.StructuredData.OSDParser.DeserializeLLSDProtobuf(System.Byte[])",
),
(
b"{}",
"M:LibreMetaverse.StructuredData.OSDParser.DeserializeJson(System.String)",
),
];
for (input, expected_member) in cases {
assert_eq!(
deserialize_bytes(input.to_vec())
.expect_err("format decoder is intentionally owned by a later issue")
.csharp_member(),
Some(*expected_member)
deserialize_bytes(b"<? llsd/protobuf ?>".to_vec()),
Ok(OSD::Undefined)
);
assert_eq!(
deserialize_bytes(b"{}".to_vec()),
Ok(OSD::Map(HashMap::new()))
);
}
let stream = Box::new(std::io::Cursor::new(b"{}".to_vec()));
assert_eq!(
deserialize_stream(stream)
.expect_err("JSON decoder is intentionally owned by issue #36")
.csharp_member(),
Some("M:LibreMetaverse.StructuredData.OSDParser.DeserializeJson(System.String)")
);
assert_eq!(deserialize_stream(stream), Ok(OSD::Map(HashMap::new())));
}
}

24
crates/libremetaverse-structured-data/src/generated.rs
View File

@@ -273,17 +273,13 @@ impl OSDParser {
pub fn deserialize_json_with_stream(
json: Box<dyn libremetaverse_types::compat::ReadWrite + Send>,
) -> Result<libremetaverse_structured_data::OSD, crate::Error> {
libremetaverse_types::not_implemented(
"M:LibreMetaverse.StructuredData.OSDParser.DeserializeJson(System.IO.Stream)",
)
crate::json_codec::deserialize_stream(json)
}
/// C# member: `M:LibreMetaverse.StructuredData.OSDParser.DeserializeJson(System.String)`.
pub fn deserialize_json_with_string(
json: String,
) -> Result<libremetaverse_structured_data::OSD, crate::Error> {
libremetaverse_types::not_implemented(
"M:LibreMetaverse.StructuredData.OSDParser.DeserializeJson(System.String)",
)
crate::json_codec::deserialize_string(json)
}
/// C# member: `M:LibreMetaverse.StructuredData.OSDParser.DeserializeLLSDBinary(System.Byte[])`.
pub fn deserialize_llsd_binary_with_bytes(
@@ -313,17 +309,13 @@ impl OSDParser {
pub fn deserialize_llsd_protobuf_with_bytes(
data: Vec<u8>,
) -> Result<libremetaverse_structured_data::OSD, crate::Error> {
libremetaverse_types::not_implemented(
"M:LibreMetaverse.StructuredData.OSDParser.DeserializeLLSDProtobuf(System.Byte[])",
)
crate::protobuf::deserialize_bytes(data)
}
/// C# member: `M:LibreMetaverse.StructuredData.OSDParser.DeserializeLLSDProtobuf(System.IO.Stream)`.
pub fn deserialize_llsd_protobuf_with_stream(
stream: Box<dyn libremetaverse_types::compat::ReadWrite + Send>,
) -> Result<libremetaverse_structured_data::OSD, crate::Error> {
libremetaverse_types::not_implemented(
"M:LibreMetaverse.StructuredData.OSDParser.DeserializeLLSDProtobuf(System.IO.Stream)",
)
crate::protobuf::deserialize_stream(stream)
}
/// C# member: `M:LibreMetaverse.StructuredData.OSDParser.DeserializeLLSDXml(System.Byte[])`.
pub fn deserialize_llsd_xml_with_bytes(
@@ -412,9 +404,7 @@ impl OSDParser {
osd: libremetaverse_structured_data::OSD,
preserve_defaults: Option<bool>,
) -> Result<String, crate::Error> {
libremetaverse_types::not_implemented(
"M:LibreMetaverse.StructuredData.OSDParser.SerializeJsonString(LibreMetaverse.StructuredData.OSD,System.Boolean)",
)
crate::json_codec::serialize(osd, preserve_defaults)
}
/// C# member: `M:LibreMetaverse.StructuredData.OSDParser.SerializeLLSDBinary(LibreMetaverse.StructuredData.OSD)`.
pub fn serialize_llsd_binary_with_osd(
@@ -477,9 +467,7 @@ impl OSDParser {
osd: libremetaverse_structured_data::OSD,
prepend_header: Option<bool>,
) -> Result<Vec<u8>, crate::Error> {
libremetaverse_types::not_implemented(
"M:LibreMetaverse.StructuredData.OSDParser.SerializeLLSDProtobuf(LibreMetaverse.StructuredData.OSD,System.Boolean)",
)
crate::protobuf::serialize(osd, prepend_header)
}
/// C# member: `M:LibreMetaverse.StructuredData.OSDParser.SerializeLLSDXmlBytes(LibreMetaverse.StructuredData.OSD)`.
pub fn serialize_llsd_xml_bytes(

467
crates/libremetaverse-structured-data/src/json_codec.rs Normal file
View File

@@ -0,0 +1,467 @@
//! Bounded JSON parsing with explicit `LibreMetaverse` OSD coercions.
#![allow(clippy::missing_errors_doc)]
#![allow(clippy::needless_pass_by_value)]
use crate::{Error, OSD};
use libremetaverse_types::compat::{ReadWrite, Uri};
use serde::de::{self, DeserializeSeed, Error as _, MapAccess, SeqAccess, Visitor};
use std::cell::Cell;
use std::collections::HashMap;
use std::fmt;
use std::io::Read as _;
use std::rc::Rc;
const MAX_BYTES: usize = OSD::DEFAULT_MAX_BINARY_BYTES;
pub(crate) fn deserialize_string(json: String) -> Result<OSD, Error> {
if json.len() > MAX_BYTES {
return Err(parse_error(0, "JSON OSD input exceeds allocation limit"));
}
let json = json.strip_prefix('\u{feff}').unwrap_or(&json);
let limits = Limits::default();
let mut deserializer = serde_json::Deserializer::from_str(json);
let value = Seed { depth: 0, limits }
.deserialize(&mut deserializer)
.map_err(|error| json_error(json, &error))?;
deserializer
.end()
.map_err(|error| json_error(json, &error))?;
value.validate_limits(
OSD::DEFAULT_MAX_DEPTH,
OSD::DEFAULT_MAX_NODES,
OSD::DEFAULT_MAX_BINARY_BYTES,
)?;
Ok(value)
}
pub(crate) fn deserialize_stream(mut json: Box<dyn ReadWrite + Send>) -> Result<OSD, Error> {
let mut bytes = Vec::new();
(&mut *json)
.take((MAX_BYTES + 1) as u64)
.read_to_end(&mut bytes)
.map_err(|_| Error::InvalidOperation)?;
if bytes.len() > MAX_BYTES {
return Err(parse_error(0, "JSON OSD input exceeds allocation limit"));
}
let text = String::from_utf8(bytes)
.map_err(|_| parse_error(0, "JSON OSD input is not valid UTF-8"))?;
deserialize_string(text)
}
pub(crate) fn serialize(osd: OSD, preserve_defaults: Option<bool>) -> Result<String, Error> {
osd.validate_limits(
OSD::DEFAULT_MAX_DEPTH,
OSD::DEFAULT_MAX_NODES,
OSD::DEFAULT_MAX_BINARY_BYTES,
)?;
let mut encoder = Encoder::new();
encoder.write_value(&osd, preserve_defaults.unwrap_or(false), 0)?;
Ok(encoder.output)
}
#[derive(Clone, Default)]
struct Limits {
nodes: Rc<Cell<usize>>,
allocated: Rc<Cell<usize>>,
}
impl Limits {
fn add_node<E: de::Error>(&self, depth: usize) -> Result<(), E> {
if depth > OSD::DEFAULT_MAX_DEPTH {
return Err(E::custom("JSON OSD nesting depth exceeded"));
}
let nodes = self
.nodes
.get()
.checked_add(1)
.ok_or_else(|| E::custom("JSON OSD node count overflow"))?;
if nodes > OSD::DEFAULT_MAX_NODES {
return Err(E::custom("JSON OSD node limit exceeded"));
}
self.nodes.set(nodes);
Ok(())
}
fn add_allocation<E: de::Error>(&self, amount: usize) -> Result<(), E> {
let allocated = self
.allocated
.get()
.checked_add(amount)
.ok_or_else(|| E::custom("JSON OSD allocation overflow"))?;
if allocated > MAX_BYTES {
return Err(E::custom("JSON OSD allocation limit exceeded"));
}
self.allocated.set(allocated);
Ok(())
}
}
struct Seed {
depth: usize,
limits: Limits,
}
impl<'de> DeserializeSeed<'de> for Seed {
type Value = OSD;
fn deserialize<D: serde::Deserializer<'de>>(self, deserializer: D) -> Result<OSD, D::Error> {
self.limits.add_node(self.depth)?;
deserializer.deserialize_any(OsdVisitor {
depth: self.depth,
limits: self.limits,
})
}
}
struct OsdVisitor {
depth: usize,
limits: Limits,
}
impl<'de> Visitor<'de> for OsdVisitor {
type Value = OSD;
fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str("a JSON value convertible to OSD")
}
fn visit_bool<E: de::Error>(self, value: bool) -> Result<OSD, E> {
Ok(OSD::Boolean(value))
}
fn visit_i64<E: de::Error>(self, value: i64) -> Result<OSD, E> {
if let Ok(value) = i32::try_from(value) {
Ok(OSD::Integer(value))
} else {
self.limits.add_allocation(8)?;
Ok(OSD::Binary(value.to_be_bytes().to_vec()))
}
}
#[allow(clippy::cast_precision_loss)]
fn visit_u64<E: de::Error>(self, value: u64) -> Result<OSD, E> {
if let Ok(value) = i32::try_from(value) {
Ok(OSD::Integer(value))
} else if let Ok(value) = i64::try_from(value) {
self.limits.add_allocation(8)?;
Ok(OSD::Binary(value.to_be_bytes().to_vec()))
} else {
Ok(OSD::Real(value as f64))
}
}
fn visit_f64<E: de::Error>(self, value: f64) -> Result<OSD, E> {
if value.is_finite() {
Ok(OSD::Real(value))
} else {
Err(E::custom(
"JSON OSD number is outside the finite double range",
))
}
}
fn visit_str<E: de::Error>(self, value: &str) -> Result<OSD, E> {
self.limits.add_allocation(value.len())?;
Ok(if value.is_empty() {
OSD::Undefined
} else {
OSD::String(value.to_owned())
})
}
fn visit_string<E: de::Error>(self, value: String) -> Result<OSD, E> {
self.limits.add_allocation(value.len())?;
Ok(if value.is_empty() {
OSD::Undefined
} else {
OSD::String(value)
})
}
fn visit_none<E: de::Error>(self) -> Result<OSD, E> {
Ok(OSD::Undefined)
}
fn visit_unit<E: de::Error>(self) -> Result<OSD, E> {
Ok(OSD::Undefined)
}
fn visit_seq<A: SeqAccess<'de>>(self, mut sequence: A) -> Result<OSD, A::Error> {
let mut values = Vec::with_capacity(sequence.size_hint().unwrap_or(0).min(1024));
while let Some(value) = sequence.next_element_seed(Seed {
depth: self.depth + 1,
limits: self.limits.clone(),
})? {
self.limits
.add_allocation::<A::Error>(std::mem::size_of::<OSD>())?;
values.push(value);
}
Ok(OSD::Array(values))
}
fn visit_map<A: MapAccess<'de>>(self, mut object: A) -> Result<OSD, A::Error> {
let mut values = HashMap::with_capacity(object.size_hint().unwrap_or(0).min(1024));
while let Some(key) = object.next_key::<String>()? {
self.limits
.add_allocation::<A::Error>(key.len() + std::mem::size_of::<(String, OSD)>())?;
if values.contains_key(&key) {
return Err(A::Error::custom("duplicate JSON OSD object property"));
}
let value = object.next_value_seed(Seed {
depth: self.depth + 1,
limits: self.limits.clone(),
})?;
values.insert(key, value);
}
Ok(OSD::Map(values))
}
}
struct Encoder {
output: String,
}
impl Encoder {
fn new() -> Self {
Self {
output: String::with_capacity(128),
}
}
fn write_value(&mut self, value: &OSD, preserve: bool, depth: usize) -> Result<(), Error> {
if depth > OSD::DEFAULT_MAX_DEPTH {
return Err(Error::Argument);
}
if !preserve && is_default(value) {
return self.push("null");
}
match value {
OSD::Undefined | OSD::LlsdXml(_) => self.push("null"),
OSD::Boolean(value) => self.push(if *value { "true" } else { "false" }),
OSD::Integer(value) => self.push(&value.to_string()),
OSD::Real(value) => {
let number = serde_json::Number::from_f64(*value).ok_or(Error::Argument)?;
self.push(&number.to_string())
}
OSD::String(value) => self.write_string(value),
OSD::UUID(value) => self.write_string(&value.to_string()),
OSD::Date(value) => {
self.write_string(&crate::model::format_system_time_for_codec(*value))
}
OSD::Uri(Uri(value)) => self.write_string(&crate::model::format_uri_for_codec(value)),
OSD::Binary(values) => {
self.push("[")?;
for (index, value) in values.iter().enumerate() {
if index != 0 {
self.push(",")?;
}
self.push(&value.to_string())?;
}
self.push("]")
}
OSD::Array(values) => {
self.push("[")?;
for (index, value) in values.iter().enumerate() {
if index != 0 {
self.push(",")?;
}
self.write_value(value, preserve, depth + 1)?;
}
self.push("]")
}
OSD::Map(values) => {
self.push("{")?;
let mut entries: Vec<_> = values.iter().collect();
entries.sort_unstable_by_key(|(key, _)| *key);
let mut written = 0;
for (key, value) in entries {
if !preserve && is_default(value) {
continue;
}
if written != 0 {
self.push(",")?;
}
self.write_string(key)?;
self.push(":")?;
self.write_value(value, preserve, depth + 1)?;
written += 1;
}
self.push("}")
}
}
}
fn write_string(&mut self, value: &str) -> Result<(), Error> {
self.push("\"")?;
let mut start = 0;
for (index, character) in value.char_indices() {
let escape = match character {
'\u{08}' => Some("\\b"),
'\t' => Some("\\t"),
'\n' => Some("\\n"),
'\u{0c}' => Some("\\f"),
'\r' => Some("\\r"),
'"' => Some("\\u0022"),
'&' => Some("\\u0026"),
'\'' => Some("\\u0027"),
'+' => Some("\\u002B"),
'<' => Some("\\u003C"),
'>' => Some("\\u003E"),
'\\' => Some("\\\\"),
character if character < ' ' || !character.is_ascii() => None,
_ => continue,
};
self.push(&value[start..index])?;
if let Some(escape) = escape {
self.push(escape)?;
} else {
let mut units = [0_u16; 2];
for unit in character.encode_utf16(&mut units) {
self.push(&format!("\\u{unit:04X}"))?;
}
}
start = index + character.len_utf8();
}
self.push(&value[start..])?;
self.push("\"")
}
fn push(&mut self, value: &str) -> Result<(), Error> {
let length = self
.output
.len()
.checked_add(value.len())
.ok_or(Error::Argument)?;
if length > MAX_BYTES {
return Err(Error::Argument);
}
self.output.push_str(value);
Ok(())
}
}
fn is_default(value: &OSD) -> bool {
match value {
OSD::Undefined | OSD::LlsdXml(_) => true,
OSD::Boolean(value) => !value,
OSD::Integer(value) => *value == 0,
OSD::Real(value) => *value == 0.0,
OSD::String(value) | OSD::Uri(Uri(value)) => value.is_empty(),
OSD::UUID(value) => value.equals_with_uuid(libremetaverse_types::UUID::zero()),
OSD::Binary(value) => value.is_empty(),
OSD::Date(_) | OSD::Array(_) | OSD::Map(_) => false,
}
}
fn json_error(input: &str, error: &serde_json::Error) -> Error {
let line = error.line().max(1);
let column = error.column().max(1);
let line_start = input
.match_indices('\n')
.take(line.saturating_sub(1))
.last()
.map_or(0, |(index, _)| index + 1);
parse_error(
line_start.saturating_add(column - 1).min(input.len()),
"malformed or resource-limited JSON OSD",
)
}
const fn parse_error(position: usize, context: &'static str) -> Error {
Error::Parse { position, context }
}
#[cfg(test)]
mod tests {
use super::*;
use libremetaverse_types::UUID;
#[test]
fn explicit_json_coercions_and_default_policy_match_reference() {
assert_eq!(deserialize_string("\"\"".into()).unwrap(), OSD::Undefined);
assert_eq!(deserialize_string("null".into()).unwrap(), OSD::Undefined);
let stream: Box<dyn ReadWrite + Send> =
Box::new(std::io::Cursor::new(b"{\"stream\":true}".to_vec()));
assert_eq!(
deserialize_stream(stream).unwrap(),
OSD::Map(HashMap::from([("stream".into(), OSD::Boolean(true))]))
);
assert_eq!(
deserialize_string("2147483648".into()).unwrap(),
OSD::Binary(2_147_483_648_i64.to_be_bytes().to_vec())
);
let value = OSD::Map(HashMap::from([
("false".into(), OSD::Boolean(false)),
("true".into(), OSD::Boolean(true)),
("zero".into(), OSD::Integer(0)),
(
"array".into(),
OSD::Array(vec![OSD::Integer(0), OSD::String("x".into())]),
),
]));
assert_eq!(
serialize(value.clone(), None).unwrap(),
"{\"array\":[null,\"x\"],\"true\":true}"
);
assert_eq!(
serialize(value, Some(true)).unwrap(),
include_str!("../../../tests/fixtures/structured_data/json_reference.json").trim()
);
}
#[test]
fn json_strings_use_system_text_json_compatible_escaping() {
let value = OSD::String("<&+\"'\\ ж 𠮟".into());
assert_eq!(
serialize(value.clone(), Some(true)).unwrap(),
"\"\\u003C\\u0026\\u002B\\u0022\\u0027\\\\ \\u0436 \\uD842\\uDF9F\""
);
assert_eq!(
deserialize_string(serialize(value.clone(), Some(true)).unwrap()).unwrap(),
value
);
}
#[test]
fn malformed_duplicate_and_deep_json_are_positioned_errors() {
for input in ["{\"x\":1,\"x\":2}", "[1,", "1 trailing"] {
assert!(matches!(
deserialize_string(input.into()),
Err(Error::Parse { .. })
));
}
let too_deep = format!(
"{}null{}",
"[".repeat(OSD::DEFAULT_MAX_DEPTH + 2),
"]".repeat(OSD::DEFAULT_MAX_DEPTH + 2)
);
assert!(matches!(
deserialize_string(too_deep),
Err(Error::Parse { .. })
));
}
#[test]
fn json_round_trip_preserves_representable_osd_values() {
let value = OSD::Map(HashMap::from([
("boolean".into(), OSD::Boolean(false)),
("integer".into(), OSD::Integer(-4)),
("real".into(), OSD::Real(1.5)),
("string".into(), OSD::String("text".into())),
("uuid".into(), OSD::UUID(UUID::zero())),
]));
let decoded = deserialize_string(serialize(value, Some(true)).unwrap()).unwrap();
let OSD::Map(decoded) = decoded else {
panic!("expected JSON object");
};
assert_eq!(decoded["boolean"], OSD::Boolean(false));
assert_eq!(decoded["integer"], OSD::Integer(-4));
assert_eq!(decoded["real"], OSD::Real(1.5));
assert_eq!(decoded["string"], OSD::String("text".into()));
assert_eq!(
decoded["uuid"],
OSD::String("00000000-0000-0000-0000-000000000000".into())
);
}
}

2
crates/libremetaverse-structured-data/src/lib.rs
View File

@@ -5,8 +5,10 @@ extern crate self as libremetaverse_structured_data;
mod binary;
mod dispatch;
mod generated;
mod json_codec;
mod model;
mod notation;
mod protobuf;
mod xml_codec;
pub mod xml {

689
crates/libremetaverse-structured-data/src/protobuf.rs Normal file
View File

@@ -0,0 +1,689 @@
//! Private, bounded implementation of the `LibreMetaverse` OSD Protobuf schema.
//!
//! The pinned reference assigns fields 1 through 11 to the OSD type and value
//! alternatives. Integers use `ZigZag` varints; real and date values are IEEE-754
//! fixed64 in little-endian Protobuf wire order. Unknown top-level and entry
//! fields are skipped according to their wire type. Map ordering is not part of
//! the schema, so this encoder sorts keys to make native output deterministic.
#![allow(clippy::missing_errors_doc)]
#![allow(clippy::needless_pass_by_value)]
use crate::{Error, OSD};
use libremetaverse_types::UUID;
use libremetaverse_types::compat::{ReadWrite, Uri};
use std::collections::HashMap;
use std::io::Read as _;
use std::time::{Duration, UNIX_EPOCH};
const HEADER: &[u8] = b"<? llsd/protobuf ?>";
const MAX_BYTES: usize = OSD::DEFAULT_MAX_BINARY_BYTES;
const WIRE_VARINT: u8 = 0;
const WIRE_FIXED64: u8 = 1;
const WIRE_LENGTH: u8 = 2;
const WIRE_FIXED32: u8 = 5;
const FIELD_TYPE: u32 = 1;
const FIELD_BOOLEAN: u32 = 2;
const FIELD_INTEGER: u32 = 3;
const FIELD_REAL: u32 = 4;
const FIELD_STRING: u32 = 5;
const FIELD_UUID: u32 = 6;
const FIELD_DATE: u32 = 7;
const FIELD_URI: u32 = 8;
const FIELD_BINARY: u32 = 9;
const FIELD_MAP_ENTRIES: u32 = 10;
const FIELD_ARRAY_ELEMENTS: u32 = 11;
pub(crate) fn deserialize_bytes(data: Vec<u8>) -> Result<OSD, Error> {
if data.len() > MAX_BYTES {
return Err(parse_error(
0,
"Protobuf OSD input exceeds allocation limit",
));
}
let (payload, origin) = strip_header(&data);
let mut budget = Budget::default();
let value = parse_value(payload, origin, 0, &mut budget)?;
value.validate_limits(
OSD::DEFAULT_MAX_DEPTH,
OSD::DEFAULT_MAX_NODES,
OSD::DEFAULT_MAX_BINARY_BYTES,
)?;
Ok(value)
}
pub(crate) fn deserialize_stream(mut stream: Box<dyn ReadWrite + Send>) -> Result<OSD, Error> {
let mut data = Vec::new();
(&mut *stream)
.take((MAX_BYTES + 1) as u64)
.read_to_end(&mut data)
.map_err(|_| Error::InvalidOperation)?;
deserialize_bytes(data)
}
pub(crate) fn serialize(osd: OSD, prepend_header: Option<bool>) -> Result<Vec<u8>, Error> {
osd.validate_limits(
OSD::DEFAULT_MAX_DEPTH,
OSD::DEFAULT_MAX_NODES,
OSD::DEFAULT_MAX_BINARY_BYTES,
)?;
let mut output = Vec::new();
if prepend_header.unwrap_or(true) {
push(&mut output, HEADER)?;
push(&mut output, b"\n")?;
}
push(&mut output, &encode_value(&osd, 0)?)?;
Ok(output)
}
fn strip_header(data: &[u8]) -> (&[u8], usize) {
if let Some(rest) = data.strip_prefix(HEADER) {
if let Some(rest) = rest.strip_prefix(b"\n") {
(rest, HEADER.len() + 1)
} else {
(rest, HEADER.len())
}
} else {
(data, 0)
}
}
#[derive(Default)]
struct Budget {
nodes: usize,
allocated: usize,
}
impl Budget {
fn node(&mut self, depth: usize, position: usize) -> Result<(), Error> {
if depth > OSD::DEFAULT_MAX_DEPTH {
return Err(parse_error(position, "Protobuf OSD nesting depth exceeded"));
}
self.nodes = self
.nodes
.checked_add(1)
.ok_or_else(|| parse_error(position, "Protobuf OSD node overflow"))?;
if self.nodes > OSD::DEFAULT_MAX_NODES {
return Err(parse_error(position, "Protobuf OSD node limit exceeded"));
}
Ok(())
}
fn allocate(&mut self, amount: usize, position: usize) -> Result<(), Error> {
self.allocated = self
.allocated
.checked_add(amount)
.ok_or_else(|| parse_error(position, "Protobuf OSD allocation overflow"))?;
if self.allocated > MAX_BYTES {
return Err(parse_error(
position,
"Protobuf OSD allocation limit exceeded",
));
}
Ok(())
}
}
enum Scalar {
Boolean(bool),
Integer(i32),
Real(f64),
String(String),
Uuid(UUID),
Date(std::time::SystemTime),
Binary(Vec<u8>),
}
#[allow(clippy::too_many_lines)]
fn parse_value(
input: &[u8],
origin: usize,
depth: usize,
budget: &mut Budget,
) -> Result<OSD, Error> {
budget.node(depth, origin)?;
let mut cursor = Cursor::new(input, origin);
let mut type_code = 0_u64;
let mut scalar = None;
let mut map = None;
let mut array = None;
while !cursor.is_empty() {
let tag_position = cursor.absolute_position();
let tag = cursor.varint()?;
let field = u32::try_from(tag >> 3)
.map_err(|_| parse_error(tag_position, "Protobuf OSD field number overflow"))?;
let wire = (tag & 7) as u8;
if field == 0 {
return Err(parse_error(
tag_position,
"Protobuf OSD field number zero is invalid",
));
}
match field {
FIELD_TYPE => {
expect_wire(wire, WIRE_VARINT, tag_position, "OSD type")?;
type_code = cursor.varint()?;
}
FIELD_BOOLEAN => {
expect_wire(wire, WIRE_VARINT, tag_position, "boolean")?;
scalar = Some(Scalar::Boolean(cursor.varint()? != 0));
}
FIELD_INTEGER => {
expect_wire(wire, WIRE_VARINT, tag_position, "integer")?;
let encoded = cursor.varint()?;
let magnitude = i64::try_from(encoded >> 1).map_err(|_| {
parse_error(tag_position, "Protobuf OSD ZigZag integer overflow")
})?;
let sign = i64::try_from(encoded & 1)
.map_err(|_| parse_error(tag_position, "Protobuf OSD ZigZag sign overflow"))?;
let decoded = magnitude ^ -sign;
let bytes = decoded.to_le_bytes();
scalar = Some(Scalar::Integer(i32::from_le_bytes([
bytes[0], bytes[1], bytes[2], bytes[3],
])));
}
FIELD_REAL => {
expect_wire(wire, WIRE_FIXED64, tag_position, "real")?;
scalar = Some(Scalar::Real(cursor.fixed64()?));
}
FIELD_STRING | FIELD_URI => {
expect_wire(wire, WIRE_LENGTH, tag_position, "string/URI")?;
let position = cursor.absolute_position();
let bytes = cursor.length_delimited()?;
budget.allocate(bytes.len(), position)?;
let value = std::str::from_utf8(bytes)
.map_err(|_| parse_error(position, "Protobuf OSD string is not valid UTF-8"))?;
scalar = Some(Scalar::String(value.to_owned()));
}
FIELD_UUID => {
expect_wire(wire, WIRE_LENGTH, tag_position, "UUID")?;
let position = cursor.absolute_position();
let bytes = cursor.length_delimited()?;
if bytes.len() != 16 {
return Err(parse_error(
position,
"Protobuf OSD UUID must contain 16 bytes",
));
}
scalar = Some(Scalar::Uuid(
UUID::from_bytes(bytes.to_vec(), 0)
.map_err(|_| parse_error(position, "invalid Protobuf OSD UUID"))?,
));
}
FIELD_DATE => {
expect_wire(wire, WIRE_FIXED64, tag_position, "date")?;
let seconds = cursor.fixed64()?;
if !seconds.is_finite() {
return Err(parse_error(tag_position, "invalid Protobuf OSD date"));
}
scalar = Some(Scalar::Date(
UNIX_EPOCH + Duration::from_secs(u64::from(reference_u32_from_f64(seconds))),
));
}
FIELD_BINARY => {
expect_wire(wire, WIRE_LENGTH, tag_position, "binary")?;
let position = cursor.absolute_position();
let bytes = cursor.length_delimited()?;
budget.allocate(bytes.len(), position)?;
scalar = Some(Scalar::Binary(bytes.to_vec()));
}
FIELD_MAP_ENTRIES => {
expect_wire(wire, WIRE_LENGTH, tag_position, "map entry")?;
let entry = cursor.length_delimited()?;
let entry_origin = cursor.absolute_position() - entry.len();
let values = map.get_or_insert_with(HashMap::new);
parse_map_entry(entry, entry_origin, depth, budget, values)?;
}
FIELD_ARRAY_ELEMENTS => {
expect_wire(wire, WIRE_LENGTH, tag_position, "array element")?;
let element = cursor.length_delimited()?;
let element_origin = cursor.absolute_position() - element.len();
budget.allocate(std::mem::size_of::<OSD>(), element_origin)?;
array.get_or_insert_with(Vec::new).push(parse_value(
element,
element_origin,
depth + 1,
budget,
)?);
}
_ => cursor.skip(wire, tag_position)?,
}
}
construct(type_code, scalar, map, array, origin)
}
fn construct(
type_code: u64,
scalar: Option<Scalar>,
map: Option<HashMap<String, OSD>>,
array: Option<Vec<OSD>>,
position: usize,
) -> Result<OSD, Error> {
match type_code {
1 => match scalar {
Some(Scalar::Boolean(value)) => Ok(OSD::Boolean(value)),
None => Ok(OSD::Undefined),
_ => Err(parse_error(position, "Protobuf OSD boolean field mismatch")),
},
2 => match scalar {
Some(Scalar::Integer(value)) => Ok(OSD::Integer(value)),
None => Ok(OSD::Undefined),
_ => Err(parse_error(position, "Protobuf OSD integer field mismatch")),
},
3 => match scalar {
Some(Scalar::Real(value)) => Ok(OSD::Real(value)),
None => Ok(OSD::Undefined),
_ => Err(parse_error(position, "Protobuf OSD real field mismatch")),
},
4 => match scalar {
Some(Scalar::String(value)) => Ok(OSD::String(value)),
None => Ok(OSD::String(String::new())),
_ => Err(parse_error(position, "Protobuf OSD string field mismatch")),
},
5 => match scalar {
Some(Scalar::Uuid(value)) => Ok(OSD::UUID(value)),
None => Ok(OSD::UUID(UUID::zero())),
_ => Err(parse_error(position, "Protobuf OSD UUID field mismatch")),
},
6 => match scalar {
Some(Scalar::Date(value)) => Ok(OSD::Date(value)),
None => Ok(OSD::Date(UNIX_EPOCH)),
_ => Err(parse_error(position, "Protobuf OSD date field mismatch")),
},
7 => match scalar {
Some(Scalar::String(value)) => Ok(OSD::Uri(Uri(value))),
None => Ok(OSD::Uri(Uri(String::new()))),
_ => Err(parse_error(position, "Protobuf OSD URI field mismatch")),
},
8 => match scalar {
Some(Scalar::Binary(value)) => Ok(OSD::Binary(value)),
None => Ok(OSD::Binary(Vec::new())),
_ => Err(parse_error(position, "Protobuf OSD binary field mismatch")),
},
9 => Ok(OSD::Map(map.unwrap_or_default())),
10 => Ok(OSD::Array(array.unwrap_or_default())),
_ => Ok(OSD::Undefined),
}
}
fn parse_map_entry(
input: &[u8],
origin: usize,
depth: usize,
budget: &mut Budget,
map: &mut HashMap<String, OSD>,
) -> Result<(), Error> {
let mut cursor = Cursor::new(input, origin);
let mut key = None;
let mut value = None;
while !cursor.is_empty() {
let tag_position = cursor.absolute_position();
let tag = cursor.varint()?;
let field = u32::try_from(tag >> 3)
.map_err(|_| parse_error(tag_position, "Protobuf map field number overflow"))?;
let wire = (tag & 7) as u8;
if field == 0 {
return Err(parse_error(
tag_position,
"Protobuf map field number zero is invalid",
));
}
match field {
1 => {
expect_wire(wire, WIRE_LENGTH, tag_position, "map key")?;
let position = cursor.absolute_position();
let bytes = cursor.length_delimited()?;
budget.allocate(bytes.len(), position)?;
key = Some(
std::str::from_utf8(bytes)
.map_err(|_| parse_error(position, "Protobuf map key is not valid UTF-8"))?
.to_owned(),
);
}
2 => {
expect_wire(wire, WIRE_LENGTH, tag_position, "map value")?;
let bytes = cursor.length_delimited()?;
let value_origin = cursor.absolute_position() - bytes.len();
value = Some(parse_value(bytes, value_origin, depth + 1, budget)?);
}
_ => cursor.skip(wire, tag_position)?,
}
}
if let (Some(key), Some(value)) = (key, value) {
budget.allocate(std::mem::size_of::<(String, OSD)>(), origin)?;
map.insert(key, value);
}
Ok(())
}
struct Cursor<'a> {
input: &'a [u8],
position: usize,
origin: usize,
}
impl<'a> Cursor<'a> {
const fn new(input: &'a [u8], origin: usize) -> Self {
Self {
input,
position: 0,
origin,
}
}
fn varint(&mut self) -> Result<u64, Error> {
let start = self.absolute_position();
let mut value = 0_u64;
for shift in (0..=63).step_by(7) {
let byte = self.byte()?;
if shift == 63 && byte > 1 {
return Err(parse_error(start, "Protobuf OSD varint overflow"));
}
value |= u64::from(byte & 0x7f) << shift;
if byte & 0x80 == 0 {
return Ok(value);
}
}
Err(parse_error(start, "Protobuf OSD varint exceeds ten bytes"))
}
fn fixed64(&mut self) -> Result<f64, Error> {
let bytes: [u8; 8] = self
.bytes(8)?
.try_into()
.map_err(|_| parse_error(self.absolute_position(), "truncated fixed64"))?;
Ok(f64::from_le_bytes(bytes))
}
fn length_delimited(&mut self) -> Result<&'a [u8], Error> {
let position = self.absolute_position();
let length = usize::try_from(self.varint()?)
.map_err(|_| parse_error(position, "Protobuf OSD length overflow"))?;
self.bytes(length)
}
fn skip(&mut self, wire: u8, position: usize) -> Result<(), Error> {
match wire {
WIRE_VARINT => {
self.varint()?;
}
WIRE_FIXED64 => {
self.bytes(8)?;
}
WIRE_LENGTH => {
self.length_delimited()?;
}
WIRE_FIXED32 => {
self.bytes(4)?;
}
_ => {
return Err(parse_error(position, "unknown Protobuf OSD wire type"));
}
}
Ok(())
}
fn byte(&mut self) -> Result<u8, Error> {
let byte =
self.input.get(self.position).copied().ok_or_else(|| {
parse_error(self.absolute_position(), "truncated Protobuf OSD value")
})?;
self.position += 1;
Ok(byte)
}
fn bytes(&mut self, length: usize) -> Result<&'a [u8], Error> {
let end = self
.position
.checked_add(length)
.ok_or_else(|| parse_error(self.absolute_position(), "Protobuf OSD length overflow"))?;
let bytes = self
.input
.get(self.position..end)
.ok_or_else(|| parse_error(self.absolute_position(), "truncated Protobuf OSD value"))?;
self.position = end;
Ok(bytes)
}
const fn is_empty(&self) -> bool {
self.position == self.input.len()
}
const fn absolute_position(&self) -> usize {
self.origin + self.position
}
}
fn expect_wire(
actual: u8,
expected: u8,
position: usize,
_field: &'static str,
) -> Result<(), Error> {
if actual == expected {
Ok(())
} else {
Err(parse_error(position, "incorrect Protobuf OSD wire type"))
}
}
fn encode_value(value: &OSD, depth: usize) -> Result<Vec<u8>, Error> {
if depth > OSD::DEFAULT_MAX_DEPTH {
return Err(Error::Argument);
}
let mut output = Vec::new();
write_varint(&mut output, make_tag(FIELD_TYPE, WIRE_VARINT))?;
write_varint(&mut output, value.type_() as u64)?;
match value {
OSD::Undefined | OSD::LlsdXml(_) => {}
OSD::Boolean(value) => {
write_varint(&mut output, make_tag(FIELD_BOOLEAN, WIRE_VARINT))?;
write_varint(&mut output, u64::from(*value))?;
}
OSD::Integer(value) => {
write_varint(&mut output, make_tag(FIELD_INTEGER, WIRE_VARINT))?;
let bits = u32::from_ne_bytes(value.to_ne_bytes());
let sign = u32::from_ne_bytes((*value >> 31).to_ne_bytes());
write_varint(&mut output, u64::from(bits.wrapping_shl(1) ^ sign))?;
}
OSD::Real(value) => {
write_varint(&mut output, make_tag(FIELD_REAL, WIRE_FIXED64))?;
push(&mut output, &value.to_le_bytes())?;
}
OSD::String(value) => {
write_varint(&mut output, make_tag(FIELD_STRING, WIRE_LENGTH))?;
write_length_delimited(&mut output, value.as_bytes())?;
}
OSD::UUID(value) => {
write_varint(&mut output, make_tag(FIELD_UUID, WIRE_LENGTH))?;
write_length_delimited(&mut output, &value.get_bytes()?)?;
}
OSD::Date(value) => {
write_varint(&mut output, make_tag(FIELD_DATE, WIRE_FIXED64))?;
let seconds = match value.duration_since(UNIX_EPOCH) {
Ok(duration) => low_u32(duration.as_secs()),
Err(error) => low_u32(error.duration().as_secs()).wrapping_neg(),
};
push(&mut output, &f64::from(seconds).to_le_bytes())?;
}
OSD::Uri(Uri(value)) => {
write_varint(&mut output, make_tag(FIELD_URI, WIRE_LENGTH))?;
write_length_delimited(
&mut output,
crate::model::format_uri_for_codec(value).as_bytes(),
)?;
}
OSD::Binary(value) => {
write_varint(&mut output, make_tag(FIELD_BINARY, WIRE_LENGTH))?;
write_length_delimited(&mut output, value)?;
}
OSD::Map(values) => {
let mut entries: Vec<_> = values.iter().collect();
entries.sort_unstable_by_key(|(key, _)| *key);
for (key, value) in entries {
write_varint(&mut output, make_tag(FIELD_MAP_ENTRIES, WIRE_LENGTH))?;
let mut entry = Vec::new();
write_varint(&mut entry, make_tag(1, WIRE_LENGTH))?;
write_length_delimited(&mut entry, key.as_bytes())?;
write_varint(&mut entry, make_tag(2, WIRE_LENGTH))?;
write_length_delimited(&mut entry, &encode_value(value, depth + 1)?)?;
write_length_delimited(&mut output, &entry)?;
}
}
OSD::Array(values) => {
for value in values {
write_varint(&mut output, make_tag(FIELD_ARRAY_ELEMENTS, WIRE_LENGTH))?;
write_length_delimited(&mut output, &encode_value(value, depth + 1)?)?;
}
}
}
Ok(output)
}
const fn make_tag(field: u32, wire: u8) -> u64 {
((field as u64) << 3) | wire as u64
}
#[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)]
fn reference_u32_from_f64(value: f64) -> u32 {
value as u32
}
const fn low_u32(value: u64) -> u32 {
let bytes = value.to_le_bytes();
u32::from_le_bytes([bytes[0], bytes[1], bytes[2], bytes[3]])
}
fn write_varint(output: &mut Vec<u8>, mut value: u64) -> Result<(), Error> {
loop {
let mut byte = (value & 0x7f) as u8;
value >>= 7;
if value != 0 {
byte |= 0x80;
}
push(output, &[byte])?;
if value == 0 {
return Ok(());
}
}
}
fn write_length_delimited(output: &mut Vec<u8>, value: &[u8]) -> Result<(), Error> {
write_varint(output, value.len() as u64)?;
push(output, value)
}
fn push(output: &mut Vec<u8>, value: &[u8]) -> Result<(), Error> {
let length = output
.len()
.checked_add(value.len())
.ok_or(Error::Argument)?;
if length > MAX_BYTES {
return Err(Error::Argument);
}
output.extend_from_slice(value);
Ok(())
}
const fn parse_error(position: usize, context: &'static str) -> Error {
Error::Parse { position, context }
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn scalar_golden_bytes_match_the_private_reference_schema() {
assert_eq!(
serialize(OSD::Boolean(true), Some(false)).unwrap(),
[0x08, 0x01, 0x10, 0x01]
);
assert_eq!(
serialize(OSD::Integer(-1), Some(false)).unwrap(),
[0x08, 0x02, 0x18, 0x01]
);
assert_eq!(
serialize(OSD::String("A".into()), Some(false)).unwrap(),
[0x08, 0x04, 0x2a, 0x01, b'A']
);
let mut real = vec![0x08, 0x03, 0x21];
real.extend_from_slice(&1.5_f64.to_le_bytes());
assert_eq!(serialize(OSD::Real(1.5), Some(false)).unwrap(), real);
}
#[test]
fn composite_golden_bytes_and_header_are_stable() {
let value = OSD::Map(HashMap::from([("a".into(), OSD::Integer(1))]));
let fixture: Vec<u8> =
include_str!("../../../tests/fixtures/structured_data/protobuf_reference.hex")
.split_ascii_whitespace()
.map(|byte| u8::from_str_radix(byte, 16).unwrap())
.collect();
assert_eq!(serialize(value.clone(), Some(false)).unwrap(), fixture);
let encoded = serialize(value.clone(), None).unwrap();
assert!(encoded.starts_with(b"<? llsd/protobuf ?>\n"));
assert_eq!(deserialize_bytes(encoded.clone()).unwrap(), value);
let stream: Box<dyn ReadWrite + Send> = Box::new(std::io::Cursor::new(encoded));
assert_eq!(deserialize_stream(stream).unwrap(), value);
}
#[test]
fn every_osd_variant_round_trips_as_defined_by_schema() {
let values = [
OSD::Undefined,
OSD::Boolean(false),
OSD::Integer(i32::MIN),
OSD::Real(-12.5),
OSD::String(String::new()),
OSD::UUID(UUID::zero()),
OSD::Date(UNIX_EPOCH + Duration::from_secs(42)),
OSD::Uri(Uri("relative/path".into())),
OSD::Binary(vec![0, 255]),
OSD::Map(HashMap::new()),
OSD::Array(Vec::new()),
];
for value in values {
assert_eq!(
deserialize_bytes(serialize(value.clone(), Some(false)).unwrap()).unwrap(),
value
);
}
assert_eq!(
deserialize_bytes(serialize(OSD::LlsdXml("<raw />".into()), Some(false)).unwrap())
.unwrap(),
OSD::Undefined
);
}
#[test]
fn unknown_fields_are_skipped_and_malformed_lengths_are_positioned() {
let mut encoded = serialize(OSD::Boolean(true), Some(false)).unwrap();
encoded.extend_from_slice(&[0x78, 0x7f]);
assert_eq!(deserialize_bytes(encoded).unwrap(), OSD::Boolean(true));
for malformed in [
vec![0x08],
vec![0x08, 0x04, 0x2a, 0x7f],
vec![0x0b],
vec![0x08, 0x01, 0x15, 0, 0, 0, 0],
] {
assert!(matches!(
deserialize_bytes(malformed),
Err(Error::Parse { .. })
));
}
}
#[test]
fn nested_depth_is_bounded_before_recursive_descent() {
let mut value = OSD::Undefined;
for _ in 0..=OSD::DEFAULT_MAX_DEPTH {
value = OSD::Array(vec![value]);
}
assert_eq!(serialize(value, Some(false)), Err(Error::Argument));
}
}

75
tests/api-compile/Cargo.lock generated
View File

@@ -156,6 +156,12 @@ dependencies = [
"typenum",
]
[[package]]
name = "itoa"
version = "1.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
[[package]]
name = "js-sys"
version = "0.3.104"
@@ -245,6 +251,8 @@ version = "0.0.1"
dependencies = [
"base64",
"libremetaverse-types",
"serde",
"serde_json",
]
[[package]]
@@ -296,6 +304,12 @@ dependencies = [
"digest 0.10.7",
]
[[package]]
name = "memchr"
version = "2.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
[[package]]
name = "metacrate-api-compile"
version = "0.0.0"
@@ -357,6 +371,48 @@ version = "1.0.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f"
[[package]]
name = "serde"
version = "1.0.229"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba"
dependencies = [
"serde_core",
]
[[package]]
name = "serde_core"
version = "1.0.229"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48"
dependencies = [
"serde_derive",
]
[[package]]
name = "serde_derive"
version = "1.0.229"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
dependencies = [
"proc-macro2",
"quote",
"syn 3.0.3",
]
[[package]]
name = "serde_json"
version = "1.0.151"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14"
dependencies = [
"itoa",
"memchr",
"serde",
"serde_core",
"zmij",
]
[[package]]
name = "sha1"
version = "0.10.7"
@@ -396,6 +452,17 @@ dependencies = [
"unicode-ident",
]
[[package]]
name = "syn"
version = "3.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3"
dependencies = [
"proc-macro2",
"quote",
"unicode-ident",
]
[[package]]
name = "typenum"
version = "1.20.1"
@@ -457,7 +524,7 @@ dependencies = [
"bumpalo",
"proc-macro2",
"quote",
"syn",
"syn 2.0.119",
"wasm-bindgen-shared",
]
@@ -469,3 +536,9 @@ checksum = "7ef64dbcc55df09c7e5a46182d181c2cfa3e925f3da937ea764728b4bbb9dcbf"
dependencies = [
"unicode-ident",
]
[[package]]
name = "zmij"
version = "1.0.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"

1
tests/fixtures/structured_data/json_reference.json vendored Normal file
View File

@@ -0,0 +1 @@
{"array":[0,"x"],"false":false,"true":true,"zero":0}

1
tests/fixtures/structured_data/protobuf_reference.hex vendored Normal file
View File

@@ -0,0 +1 @@
08 09 52 09 0a 01 61 12 04 08 02 18 02

14
tests/red-suite-baseline.json
View File

@@ -63,6 +63,18 @@
"LibreMetaverse.Tests/NotationLLSDTests.cs::NotationSDTests.SerializeURI::test",
"LibreMetaverse.Tests/NotationLLSDTests.cs::NotationSDTests.SerializeUUID::test",
"LibreMetaverse.Tests/NotationLLSDTests.cs::NotationSDTests.SerializeUndef::test",
"LibreMetaverse.Tests/ProtobufTests.cs::ProtobufTests.AutoDetectProtobuf::test",
"LibreMetaverse.Tests/ProtobufTests.cs::ProtobufTests.SerializeArray::test",
"LibreMetaverse.Tests/ProtobufTests.cs::ProtobufTests.SerializeBinary::test",
"LibreMetaverse.Tests/ProtobufTests.cs::ProtobufTests.SerializeBoolean::test",
"LibreMetaverse.Tests/ProtobufTests.cs::ProtobufTests.SerializeDate::test",
"LibreMetaverse.Tests/ProtobufTests.cs::ProtobufTests.SerializeInteger::test",
"LibreMetaverse.Tests/ProtobufTests.cs::ProtobufTests.SerializeMap::test",
"LibreMetaverse.Tests/ProtobufTests.cs::ProtobufTests.SerializeNestedComposite::test",
"LibreMetaverse.Tests/ProtobufTests.cs::ProtobufTests.SerializeReal::test",
"LibreMetaverse.Tests/ProtobufTests.cs::ProtobufTests.SerializeString::test",
"LibreMetaverse.Tests/ProtobufTests.cs::ProtobufTests.SerializeUUID::test",
"LibreMetaverse.Tests/ProtobufTests.cs::ProtobufTests.SerializeUri::test",
"LibreMetaverse.Tests/TypeTests.cs::TypeTests.LLSDTerseParsing::test",
"LibreMetaverse.Tests/TypeTests.cs::TypeTests.Quaternions::test",
"LibreMetaverse.Tests/TypeTests.cs::TypeTests.TestMatrix::test",
@@ -86,5 +98,5 @@
"LibreMetaverse.Tests/XmlLLSDTests.cs::XmlSDTests.DeserializeUUID::test",
"LibreMetaverse.Tests/XmlLLSDTests.cs::XmlSDTests.DeserializeUndef::test"
],
"support_passes": 84
"support_passes": 93
}

14
tools/generate_api_shims.py
View File

@@ -91,6 +91,14 @@ NATIVE_MEMBER_BODIES = {
"crate::notation::deserialize_reader(reader)",
"M:LibreMetaverse.StructuredData.OSDParser.DeserializeLLSDNotation(System.String)":
"crate::notation::deserialize_string(notation_data)",
"M:LibreMetaverse.StructuredData.OSDParser.DeserializeLLSDProtobuf(System.Byte[])":
"crate::protobuf::deserialize_bytes(data)",
"M:LibreMetaverse.StructuredData.OSDParser.DeserializeLLSDProtobuf(System.IO.Stream)":
"crate::protobuf::deserialize_stream(stream)",
"M:LibreMetaverse.StructuredData.OSDParser.DeserializeJson(System.IO.Stream)":
"crate::json_codec::deserialize_stream(json)",
"M:LibreMetaverse.StructuredData.OSDParser.DeserializeJson(System.String)":
"crate::json_codec::deserialize_string(json)",
"M:LibreMetaverse.StructuredData.OSDParser.DeserializeLLSDXml(System.Byte[])":
"crate::xml_codec::deserialize_bytes(xml_data)",
"M:LibreMetaverse.StructuredData.OSDParser.DeserializeLLSDXml(System.IO.Stream)":
@@ -127,6 +135,8 @@ NATIVE_MEMBER_BODIES = {
"crate::binary::serialize_stream(data)",
"M:LibreMetaverse.StructuredData.OSDParser.SerializeLLSDBinaryStream(LibreMetaverse.StructuredData.OSD,System.Boolean)":
"crate::binary::serialize_stream_with_header(data, prepend_header)",
"M:LibreMetaverse.StructuredData.OSDParser.SerializeJsonString(LibreMetaverse.StructuredData.OSD,System.Boolean)":
"crate::json_codec::serialize(osd, preserve_defaults)",
"M:LibreMetaverse.StructuredData.OSDParser.SerializeLLSDInnerXmlString(LibreMetaverse.StructuredData.OSD)":
"crate::xml_codec::serialize_inner(data)",
"M:LibreMetaverse.StructuredData.OSDParser.SerializeLLSDNotation(LibreMetaverse.StructuredData.OSD)":
@@ -137,6 +147,8 @@ NATIVE_MEMBER_BODIES = {
"crate::notation::serialize_stream(osd)",
"M:LibreMetaverse.StructuredData.OSDParser.SerializeLLSDNotationStreamFormatted(LibreMetaverse.StructuredData.OSD)":
"crate::notation::serialize_stream_formatted(osd)",
"M:LibreMetaverse.StructuredData.OSDParser.SerializeLLSDProtobuf(LibreMetaverse.StructuredData.OSD,System.Boolean)":
"crate::protobuf::serialize(osd, prepend_header)",
"M:LibreMetaverse.StructuredData.OSDParser.SerializeLLSDXmlBytes(LibreMetaverse.StructuredData.OSD)":
"crate::xml_codec::serialize_bytes(data)",
"M:LibreMetaverse.StructuredData.OSDParser.SerializeLLSDXmlElement(System.Xml.XmlWriter,LibreMetaverse.StructuredData.OSD)":
@@ -988,7 +1000,7 @@ def coverage_report(catalog: dict, coverage: dict[str, tuple[int, int, bool]]) -
if item["doc_id"] in native_type_ids or member["doc_id"] in NATIVE_MEMBER_BODIES
}
if native_type_ids or native_member_ids:
if (len(native_type_ids), len(native_member_ids)) == (types, members):
if len(native_member_ids) == members:
status = (
f"native implementation: {len(native_type_ids):,} types / "
f"{len(native_member_ids):,} members; no generated shims remain"

2
tools/generate_rust_mapping.py
View File

@@ -982,7 +982,9 @@ def validate_generated_shims() -> None:
"crate::byte_order::",
"crate::binary::",
"crate::dispatch::",
"crate::json_codec::",
"crate::notation::",
"crate::protobuf::",
"crate::xml_codec::",
)
):