//! Private, bounded implementation of the `LibreMetaverse` OSD Protobuf schema. //! //! The pinned reference assigns fields 1 through 11 to the OSD type and value //! alternatives. Integers use `ZigZag` varints; real and date values are IEEE-754 //! fixed64 in little-endian Protobuf wire order. Unknown top-level and entry //! fields are skipped according to their wire type. Map ordering is not part of //! the schema, so this encoder sorts keys to make native output deterministic. #![allow(clippy::missing_errors_doc)] #![allow(clippy::needless_pass_by_value)] use crate::{Error, OSD}; use libremetaverse_types::UUID; use libremetaverse_types::compat::{ReadWrite, Uri}; use std::collections::HashMap; use std::io::Read as _; use std::time::{Duration, UNIX_EPOCH}; const HEADER: &[u8] = b""; const MAX_BYTES: usize = OSD::DEFAULT_MAX_BINARY_BYTES; const WIRE_VARINT: u8 = 0; const WIRE_FIXED64: u8 = 1; const WIRE_LENGTH: u8 = 2; const WIRE_FIXED32: u8 = 5; const FIELD_TYPE: u32 = 1; const FIELD_BOOLEAN: u32 = 2; const FIELD_INTEGER: u32 = 3; const FIELD_REAL: u32 = 4; const FIELD_STRING: u32 = 5; const FIELD_UUID: u32 = 6; const FIELD_DATE: u32 = 7; const FIELD_URI: u32 = 8; const FIELD_BINARY: u32 = 9; const FIELD_MAP_ENTRIES: u32 = 10; const FIELD_ARRAY_ELEMENTS: u32 = 11; pub(crate) fn deserialize_bytes(data: Vec) -> Result { if data.len() > MAX_BYTES { return Err(parse_error( 0, "Protobuf OSD input exceeds allocation limit", )); } let (payload, origin) = strip_header(&data); let mut budget = Budget::default(); let value = parse_value(payload, origin, 0, &mut budget)?; value.validate_limits( OSD::DEFAULT_MAX_DEPTH, OSD::DEFAULT_MAX_NODES, OSD::DEFAULT_MAX_BINARY_BYTES, )?; Ok(value) } pub(crate) fn deserialize_stream(mut stream: Box) -> Result { let mut data = Vec::new(); (&mut *stream) .take((MAX_BYTES + 1) as u64) .read_to_end(&mut data) .map_err(|_| Error::InvalidOperation)?; deserialize_bytes(data) } pub(crate) fn serialize(osd: OSD, prepend_header: Option) -> Result, Error> { osd.validate_limits( OSD::DEFAULT_MAX_DEPTH, OSD::DEFAULT_MAX_NODES, OSD::DEFAULT_MAX_BINARY_BYTES, )?; let mut output = Vec::new(); if prepend_header.unwrap_or(true) { push(&mut output, HEADER)?; push(&mut output, b"\n")?; } push(&mut output, &encode_value(&osd, 0)?)?; Ok(output) } fn strip_header(data: &[u8]) -> (&[u8], usize) { if let Some(rest) = data.strip_prefix(HEADER) { if let Some(rest) = rest.strip_prefix(b"\n") { (rest, HEADER.len() + 1) } else { (rest, HEADER.len()) } } else { (data, 0) } } #[derive(Default)] struct Budget { nodes: usize, allocated: usize, } impl Budget { fn node(&mut self, depth: usize, position: usize) -> Result<(), Error> { if depth > OSD::DEFAULT_MAX_DEPTH { return Err(parse_error(position, "Protobuf OSD nesting depth exceeded")); } self.nodes = self .nodes .checked_add(1) .ok_or_else(|| parse_error(position, "Protobuf OSD node overflow"))?; if self.nodes > OSD::DEFAULT_MAX_NODES { return Err(parse_error(position, "Protobuf OSD node limit exceeded")); } Ok(()) } fn allocate(&mut self, amount: usize, position: usize) -> Result<(), Error> { self.allocated = self .allocated .checked_add(amount) .ok_or_else(|| parse_error(position, "Protobuf OSD allocation overflow"))?; if self.allocated > MAX_BYTES { return Err(parse_error( position, "Protobuf OSD allocation limit exceeded", )); } Ok(()) } } enum Scalar { Boolean(bool), Integer(i32), Real(f64), String(String), Uuid(UUID), Date(std::time::SystemTime), Binary(Vec), } #[allow(clippy::too_many_lines)] fn parse_value( input: &[u8], origin: usize, depth: usize, budget: &mut Budget, ) -> Result { budget.node(depth, origin)?; let mut cursor = Cursor::new(input, origin); let mut type_code = 0_u64; let mut scalar = None; let mut map = None; let mut array = None; while !cursor.is_empty() { let tag_position = cursor.absolute_position(); let tag = cursor.varint()?; let field = u32::try_from(tag >> 3) .map_err(|_| parse_error(tag_position, "Protobuf OSD field number overflow"))?; let wire = (tag & 7) as u8; if field == 0 { return Err(parse_error( tag_position, "Protobuf OSD field number zero is invalid", )); } match field { FIELD_TYPE => { expect_wire(wire, WIRE_VARINT, tag_position, "OSD type")?; type_code = cursor.varint()?; } FIELD_BOOLEAN => { expect_wire(wire, WIRE_VARINT, tag_position, "boolean")?; scalar = Some(Scalar::Boolean(cursor.varint()? != 0)); } FIELD_INTEGER => { expect_wire(wire, WIRE_VARINT, tag_position, "integer")?; let encoded = cursor.varint()?; let magnitude = i64::try_from(encoded >> 1).map_err(|_| { parse_error(tag_position, "Protobuf OSD ZigZag integer overflow") })?; let sign = i64::try_from(encoded & 1) .map_err(|_| parse_error(tag_position, "Protobuf OSD ZigZag sign overflow"))?; let decoded = magnitude ^ -sign; let bytes = decoded.to_le_bytes(); scalar = Some(Scalar::Integer(i32::from_le_bytes([ bytes[0], bytes[1], bytes[2], bytes[3], ]))); } FIELD_REAL => { expect_wire(wire, WIRE_FIXED64, tag_position, "real")?; scalar = Some(Scalar::Real(cursor.fixed64()?)); } FIELD_STRING | FIELD_URI => { expect_wire(wire, WIRE_LENGTH, tag_position, "string/URI")?; let position = cursor.absolute_position(); let bytes = cursor.length_delimited()?; budget.allocate(bytes.len(), position)?; let value = std::str::from_utf8(bytes) .map_err(|_| parse_error(position, "Protobuf OSD string is not valid UTF-8"))?; scalar = Some(Scalar::String(value.to_owned())); } FIELD_UUID => { expect_wire(wire, WIRE_LENGTH, tag_position, "UUID")?; let position = cursor.absolute_position(); let bytes = cursor.length_delimited()?; if bytes.len() != 16 { return Err(parse_error( position, "Protobuf OSD UUID must contain 16 bytes", )); } scalar = Some(Scalar::Uuid( UUID::from_bytes(bytes.to_vec(), 0) .map_err(|_| parse_error(position, "invalid Protobuf OSD UUID"))?, )); } FIELD_DATE => { expect_wire(wire, WIRE_FIXED64, tag_position, "date")?; let seconds = cursor.fixed64()?; if !seconds.is_finite() { return Err(parse_error(tag_position, "invalid Protobuf OSD date")); } scalar = Some(Scalar::Date( UNIX_EPOCH + Duration::from_secs(u64::from(reference_u32_from_f64(seconds))), )); } FIELD_BINARY => { expect_wire(wire, WIRE_LENGTH, tag_position, "binary")?; let position = cursor.absolute_position(); let bytes = cursor.length_delimited()?; budget.allocate(bytes.len(), position)?; scalar = Some(Scalar::Binary(bytes.to_vec())); } FIELD_MAP_ENTRIES => { expect_wire(wire, WIRE_LENGTH, tag_position, "map entry")?; let entry = cursor.length_delimited()?; let entry_origin = cursor.absolute_position() - entry.len(); let values = map.get_or_insert_with(HashMap::new); parse_map_entry(entry, entry_origin, depth, budget, values)?; } FIELD_ARRAY_ELEMENTS => { expect_wire(wire, WIRE_LENGTH, tag_position, "array element")?; let element = cursor.length_delimited()?; let element_origin = cursor.absolute_position() - element.len(); budget.allocate(std::mem::size_of::(), element_origin)?; array.get_or_insert_with(Vec::new).push(parse_value( element, element_origin, depth + 1, budget, )?); } _ => cursor.skip(wire, tag_position)?, } } construct(type_code, scalar, map, array, origin) } fn construct( type_code: u64, scalar: Option, map: Option>, array: Option>, position: usize, ) -> Result { match type_code { 1 => match scalar { Some(Scalar::Boolean(value)) => Ok(OSD::Boolean(value)), None => Ok(OSD::Undefined), _ => Err(parse_error(position, "Protobuf OSD boolean field mismatch")), }, 2 => match scalar { Some(Scalar::Integer(value)) => Ok(OSD::Integer(value)), None => Ok(OSD::Undefined), _ => Err(parse_error(position, "Protobuf OSD integer field mismatch")), }, 3 => match scalar { Some(Scalar::Real(value)) => Ok(OSD::Real(value)), None => Ok(OSD::Undefined), _ => Err(parse_error(position, "Protobuf OSD real field mismatch")), }, 4 => match scalar { Some(Scalar::String(value)) => Ok(OSD::String(value)), None => Ok(OSD::String(String::new())), _ => Err(parse_error(position, "Protobuf OSD string field mismatch")), }, 5 => match scalar { Some(Scalar::Uuid(value)) => Ok(OSD::UUID(value)), None => Ok(OSD::UUID(UUID::zero())), _ => Err(parse_error(position, "Protobuf OSD UUID field mismatch")), }, 6 => match scalar { Some(Scalar::Date(value)) => Ok(OSD::Date(value)), None => Ok(OSD::Date(UNIX_EPOCH)), _ => Err(parse_error(position, "Protobuf OSD date field mismatch")), }, 7 => match scalar { Some(Scalar::String(value)) => Ok(OSD::Uri(Uri(value))), None => Ok(OSD::Uri(Uri(String::new()))), _ => Err(parse_error(position, "Protobuf OSD URI field mismatch")), }, 8 => match scalar { Some(Scalar::Binary(value)) => Ok(OSD::Binary(value)), None => Ok(OSD::Binary(Vec::new())), _ => Err(parse_error(position, "Protobuf OSD binary field mismatch")), }, 9 => Ok(OSD::Map(map.unwrap_or_default())), 10 => Ok(OSD::Array(array.unwrap_or_default())), _ => Ok(OSD::Undefined), } } fn parse_map_entry( input: &[u8], origin: usize, depth: usize, budget: &mut Budget, map: &mut HashMap, ) -> Result<(), Error> { let mut cursor = Cursor::new(input, origin); let mut key = None; let mut value = None; while !cursor.is_empty() { let tag_position = cursor.absolute_position(); let tag = cursor.varint()?; let field = u32::try_from(tag >> 3) .map_err(|_| parse_error(tag_position, "Protobuf map field number overflow"))?; let wire = (tag & 7) as u8; if field == 0 { return Err(parse_error( tag_position, "Protobuf map field number zero is invalid", )); } match field { 1 => { expect_wire(wire, WIRE_LENGTH, tag_position, "map key")?; let position = cursor.absolute_position(); let bytes = cursor.length_delimited()?; budget.allocate(bytes.len(), position)?; key = Some( std::str::from_utf8(bytes) .map_err(|_| parse_error(position, "Protobuf map key is not valid UTF-8"))? .to_owned(), ); } 2 => { expect_wire(wire, WIRE_LENGTH, tag_position, "map value")?; let bytes = cursor.length_delimited()?; let value_origin = cursor.absolute_position() - bytes.len(); value = Some(parse_value(bytes, value_origin, depth + 1, budget)?); } _ => cursor.skip(wire, tag_position)?, } } if let (Some(key), Some(value)) = (key, value) { budget.allocate(std::mem::size_of::<(String, OSD)>(), origin)?; map.insert(key, value); } Ok(()) } struct Cursor<'a> { input: &'a [u8], position: usize, origin: usize, } impl<'a> Cursor<'a> { const fn new(input: &'a [u8], origin: usize) -> Self { Self { input, position: 0, origin, } } fn varint(&mut self) -> Result { let start = self.absolute_position(); let mut value = 0_u64; for shift in (0..=63).step_by(7) { let byte = self.byte()?; if shift == 63 && byte > 1 { return Err(parse_error(start, "Protobuf OSD varint overflow")); } value |= u64::from(byte & 0x7f) << shift; if byte & 0x80 == 0 { return Ok(value); } } Err(parse_error(start, "Protobuf OSD varint exceeds ten bytes")) } fn fixed64(&mut self) -> Result { let bytes: [u8; 8] = self .bytes(8)? .try_into() .map_err(|_| parse_error(self.absolute_position(), "truncated fixed64"))?; Ok(f64::from_le_bytes(bytes)) } fn length_delimited(&mut self) -> Result<&'a [u8], Error> { let position = self.absolute_position(); let length = usize::try_from(self.varint()?) .map_err(|_| parse_error(position, "Protobuf OSD length overflow"))?; self.bytes(length) } fn skip(&mut self, wire: u8, position: usize) -> Result<(), Error> { match wire { WIRE_VARINT => { self.varint()?; } WIRE_FIXED64 => { self.bytes(8)?; } WIRE_LENGTH => { self.length_delimited()?; } WIRE_FIXED32 => { self.bytes(4)?; } _ => { return Err(parse_error(position, "unknown Protobuf OSD wire type")); } } Ok(()) } fn byte(&mut self) -> Result { let byte = self.input.get(self.position).copied().ok_or_else(|| { parse_error(self.absolute_position(), "truncated Protobuf OSD value") })?; self.position += 1; Ok(byte) } fn bytes(&mut self, length: usize) -> Result<&'a [u8], Error> { let end = self .position .checked_add(length) .ok_or_else(|| parse_error(self.absolute_position(), "Protobuf OSD length overflow"))?; let bytes = self .input .get(self.position..end) .ok_or_else(|| parse_error(self.absolute_position(), "truncated Protobuf OSD value"))?; self.position = end; Ok(bytes) } const fn is_empty(&self) -> bool { self.position == self.input.len() } const fn absolute_position(&self) -> usize { self.origin + self.position } } fn expect_wire( actual: u8, expected: u8, position: usize, _field: &'static str, ) -> Result<(), Error> { if actual == expected { Ok(()) } else { Err(parse_error(position, "incorrect Protobuf OSD wire type")) } } fn encode_value(value: &OSD, depth: usize) -> Result, Error> { if depth > OSD::DEFAULT_MAX_DEPTH { return Err(Error::Argument); } let mut output = Vec::new(); write_varint(&mut output, make_tag(FIELD_TYPE, WIRE_VARINT))?; write_varint(&mut output, value.type_() as u64)?; match value { OSD::Undefined | OSD::LlsdXml(_) => {} OSD::Boolean(value) => { write_varint(&mut output, make_tag(FIELD_BOOLEAN, WIRE_VARINT))?; write_varint(&mut output, u64::from(*value))?; } OSD::Integer(value) => { write_varint(&mut output, make_tag(FIELD_INTEGER, WIRE_VARINT))?; let bits = u32::from_ne_bytes(value.to_ne_bytes()); let sign = u32::from_ne_bytes((*value >> 31).to_ne_bytes()); write_varint(&mut output, u64::from(bits.wrapping_shl(1) ^ sign))?; } OSD::Real(value) => { write_varint(&mut output, make_tag(FIELD_REAL, WIRE_FIXED64))?; push(&mut output, &value.to_le_bytes())?; } OSD::String(value) => { write_varint(&mut output, make_tag(FIELD_STRING, WIRE_LENGTH))?; write_length_delimited(&mut output, value.as_bytes())?; } OSD::UUID(value) => { write_varint(&mut output, make_tag(FIELD_UUID, WIRE_LENGTH))?; write_length_delimited(&mut output, &value.get_bytes()?)?; } OSD::Date(value) => { write_varint(&mut output, make_tag(FIELD_DATE, WIRE_FIXED64))?; let seconds = match value.duration_since(UNIX_EPOCH) { Ok(duration) => low_u32(duration.as_secs()), Err(error) => low_u32(error.duration().as_secs()).wrapping_neg(), }; push(&mut output, &f64::from(seconds).to_le_bytes())?; } OSD::Uri(Uri(value)) => { write_varint(&mut output, make_tag(FIELD_URI, WIRE_LENGTH))?; write_length_delimited( &mut output, crate::model::format_uri_for_codec(value).as_bytes(), )?; } OSD::Binary(value) => { write_varint(&mut output, make_tag(FIELD_BINARY, WIRE_LENGTH))?; write_length_delimited(&mut output, value)?; } OSD::Map(values) => { let mut entries: Vec<_> = values.iter().collect(); entries.sort_unstable_by_key(|(key, _)| *key); for (key, value) in entries { write_varint(&mut output, make_tag(FIELD_MAP_ENTRIES, WIRE_LENGTH))?; let mut entry = Vec::new(); write_varint(&mut entry, make_tag(1, WIRE_LENGTH))?; write_length_delimited(&mut entry, key.as_bytes())?; write_varint(&mut entry, make_tag(2, WIRE_LENGTH))?; write_length_delimited(&mut entry, &encode_value(value, depth + 1)?)?; write_length_delimited(&mut output, &entry)?; } } OSD::Array(values) => { for value in values { write_varint(&mut output, make_tag(FIELD_ARRAY_ELEMENTS, WIRE_LENGTH))?; write_length_delimited(&mut output, &encode_value(value, depth + 1)?)?; } } } Ok(output) } const fn make_tag(field: u32, wire: u8) -> u64 { ((field as u64) << 3) | wire as u64 } #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] fn reference_u32_from_f64(value: f64) -> u32 { value as u32 } const fn low_u32(value: u64) -> u32 { let bytes = value.to_le_bytes(); u32::from_le_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]) } fn write_varint(output: &mut Vec, mut value: u64) -> Result<(), Error> { loop { let mut byte = (value & 0x7f) as u8; value >>= 7; if value != 0 { byte |= 0x80; } push(output, &[byte])?; if value == 0 { return Ok(()); } } } fn write_length_delimited(output: &mut Vec, value: &[u8]) -> Result<(), Error> { write_varint(output, value.len() as u64)?; push(output, value) } fn push(output: &mut Vec, value: &[u8]) -> Result<(), Error> { let length = output .len() .checked_add(value.len()) .ok_or(Error::Argument)?; if length > MAX_BYTES { return Err(Error::Argument); } output.extend_from_slice(value); Ok(()) } const fn parse_error(position: usize, context: &'static str) -> Error { Error::Parse { position, context } } #[cfg(test)] mod tests { use super::*; #[test] fn scalar_golden_bytes_match_the_private_reference_schema() { assert_eq!( serialize(OSD::Boolean(true), Some(false)).unwrap(), [0x08, 0x01, 0x10, 0x01] ); assert_eq!( serialize(OSD::Integer(-1), Some(false)).unwrap(), [0x08, 0x02, 0x18, 0x01] ); assert_eq!( serialize(OSD::String("A".into()), Some(false)).unwrap(), [0x08, 0x04, 0x2a, 0x01, b'A'] ); let mut real = vec![0x08, 0x03, 0x21]; real.extend_from_slice(&1.5_f64.to_le_bytes()); assert_eq!(serialize(OSD::Real(1.5), Some(false)).unwrap(), real); } #[test] fn composite_golden_bytes_and_header_are_stable() { let value = OSD::Map(HashMap::from([("a".into(), OSD::Integer(1))])); let fixture: Vec = include_str!("../../../tests/fixtures/structured_data/protobuf_reference.hex") .split_ascii_whitespace() .map(|byte| u8::from_str_radix(byte, 16).unwrap()) .collect(); assert_eq!(serialize(value.clone(), Some(false)).unwrap(), fixture); let encoded = serialize(value.clone(), None).unwrap(); assert!(encoded.starts_with(b"\n")); assert_eq!(deserialize_bytes(encoded.clone()).unwrap(), value); let stream: Box = Box::new(std::io::Cursor::new(encoded)); assert_eq!(deserialize_stream(stream).unwrap(), value); } #[test] fn every_osd_variant_round_trips_as_defined_by_schema() { let values = [ OSD::Undefined, OSD::Boolean(false), OSD::Integer(i32::MIN), OSD::Real(-12.5), OSD::String(String::new()), OSD::UUID(UUID::zero()), OSD::Date(UNIX_EPOCH + Duration::from_secs(42)), OSD::Uri(Uri("relative/path".into())), OSD::Binary(vec![0, 255]), OSD::Map(HashMap::new()), OSD::Array(Vec::new()), ]; for value in values { assert_eq!( deserialize_bytes(serialize(value.clone(), Some(false)).unwrap()).unwrap(), value ); } assert_eq!( deserialize_bytes(serialize(OSD::LlsdXml("".into()), Some(false)).unwrap()) .unwrap(), OSD::Undefined ); } #[test] fn unknown_fields_are_skipped_and_malformed_lengths_are_positioned() { let mut encoded = serialize(OSD::Boolean(true), Some(false)).unwrap(); encoded.extend_from_slice(&[0x78, 0x7f]); assert_eq!(deserialize_bytes(encoded).unwrap(), OSD::Boolean(true)); for malformed in [ vec![0x08], vec![0x08, 0x04, 0x2a, 0x7f], vec![0x0b], vec![0x08, 0x01, 0x15, 0, 0, 0, 0], ] { assert!(matches!( deserialize_bytes(malformed), Err(Error::Parse { .. }) )); } } #[test] fn nested_depth_is_bounded_before_recursive_descent() { let mut value = OSD::Undefined; for _ in 0..=OSD::DEFAULT_MAX_DEPTH { value = OSD::Array(vec![value]); } assert_eq!(serialize(value, Some(false)), Err(Error::Argument)); } }