#![allow(clippy::cast_precision_loss)] use std::alloc::System; use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::fmt::Write as _; use std::fs; use std::hint::black_box; use std::path::Path; use std::sync::Arc; use std::time::Instant; use libremetaverse::assets::AssetNotecard; use libremetaverse::imaging::Targa; use libremetaverse::import_export::{ModelFace, ModelPrim}; use libremetaverse::messages::linden::RemoteParcelRequestReply; use libremetaverse::packets::AgentPausePacket; use libremetaverse::rendering::{DetailLevel, Vertex}; use libremetaverse::{GridClient, Inventory, InventoryItem, Primitive, PrimitiveConstructionData}; use libremetaverse_imaging::{ManagedImage, ManagedImageImageChannels}; use libremetaverse_rendering_mesh_foundry::MeshFoundry; use libremetaverse_rendering_simple::SimpleRenderer; use libremetaverse_structured_data::{OSD, OSDParser}; use libremetaverse_types::{ HoleType, Matrix4, PCode, PathCurve, ProfileCurve, Quaternion, UUID, Vector2, Vector3, }; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; use stats_alloc::{INSTRUMENTED_SYSTEM, Region, Stats, StatsAlloc}; #[global_allocator] static ALLOCATOR: &StatsAlloc = &INSTRUMENTED_SYSTEM; const RUNS: usize = 7; const TOOL_SCHEMA: u32 = 1; const EXPECTED_REFERENCE: &str = "2aa70bb68513b39795da5d13c88f31b86e85a3ba"; type AppResult = Result; #[derive(Clone, Deserialize)] struct Fixture { schema: u32, reference_commit: String, uuid: String, uuid_math_iterations: usize, llsd_iterations: usize, packet_iterations: usize, asset_iterations: usize, image_iterations: usize, mesh_iterations: usize, inventory_iterations: usize, object_iterations: usize, rendering_iterations: usize, client_iterations: usize, warmup_divisor: usize, llsd_array_length: usize, packet_blocks: usize, image_width: i32, image_height: i32, mesh_side: usize, inventory_items: usize, object_count: usize, client_batch: usize, } #[derive(Serialize, Deserialize)] struct Environment { os: String, arch: String, logical_cpus: usize, toolchain: String, profile: String, } #[derive(Clone, Serialize, Deserialize)] struct Metrics { iterations: usize, elapsed_ns: u128, latency_ns: f64, throughput_per_second: f64, allocations: Option, bytes_allocated: usize, retained_bytes: i128, } #[derive(Serialize, Deserialize)] struct BenchmarkResult { name: String, category: String, unit: String, fixture: String, cold: Metrics, warm_samples: Vec, warm_median: Metrics, } #[derive(Serialize, Deserialize)] struct Report { schema: u32, runtime: String, reference_commit: String, fixture_hashes: BTreeMap, environment: Environment, results: Vec, } struct Context { fixture: Fixture, uuid: UUID, llsd: OSD, packet: Vec, notecard: Vec, image: Vec, mesh: Vec, } #[derive(Clone, Copy)] struct Workload { name: &'static str, category: &'static str, fixture: &'static str, iterations: fn(&Fixture) -> usize, run: fn(&Context, usize) -> AppResult, } fn main() { if let Err(error) = run_cli() { eprintln!("performance audit failed: {error}"); std::process::exit(1); } } fn run_cli() -> AppResult<()> { let args: Vec = std::env::args().skip(1).collect(); match args.first().map(String::as_str) { Some("fixtures") => { let root = value(&args, "--fixture-root").unwrap_or_else(|| "benchmarks/fixtures".into()); generate_fixtures(Path::new(&root)) } Some("run") => { let root = value(&args, "--fixture-root").unwrap_or_else(|| "benchmarks/fixtures".into()); let output = value(&args, "--output") .unwrap_or_else(|| "benchmarks/results/rust-linux-x86_64.json".into()); let report = run_benchmarks(Path::new(&root))?; write_json(Path::new(&output), &report) } Some("compare") => { let rust = required(&args, "--rust")?; let reference = required(&args, "--reference")?; let output = required(&args, "--output")?; compare_reports(Path::new(&rust), Path::new(&reference), Path::new(&output)) } Some("audit") => { let root = value(&args, "--fixture-root").unwrap_or_else(|| "benchmarks/fixtures".into()); let rust = required(&args, "--rust")?; let reference = required(&args, "--reference")?; let comparison = required(&args, "--comparison")?; audit( Path::new(&root), Path::new(&rust), Path::new(&reference), Path::new(&comparison), ) } _ => Err("usage: metacrate-performance [options]".into()), } } fn value(args: &[String], key: &str) -> Option { args.windows(2) .find(|pair| pair[0] == key) .map(|pair| pair[1].clone()) } fn required(args: &[String], key: &str) -> AppResult { value(args, key).ok_or_else(|| format!("missing {key}")) } fn fixture(path: &Path) -> AppResult { let bytes = fs::read(path).map_err(|error| format!("read {}: {error}", path.display()))?; let fixture: Fixture = serde_json::from_slice(&bytes) .map_err(|error| format!("parse {}: {error}", path.display()))?; if fixture.schema != 1 || fixture.reference_commit != EXPECTED_REFERENCE { return Err("fixture schema or pinned reference commit does not match policy".into()); } if fixture.warmup_divisor == 0 { return Err("warmup_divisor must be nonzero".into()); } Ok(fixture) } fn generate_fixtures(root: &Path) -> AppResult<()> { fs::create_dir_all(root).map_err(|error| error.to_string())?; let config = fixture(&root.join("workloads.json"))?; let mut image = ManagedImage::new( config.image_width, config.image_height, ManagedImageImageChannels( ManagedImageImageChannels::COLOR.0 | ManagedImageImageChannels::ALPHA.0, ), ) .map_err(debug)?; for index in 0..image.red.len() { image.red[index] = u8::try_from(index & 255).map_err(debug)?; image.green[index] = u8::try_from((index * 3) & 255).map_err(debug)?; image.blue[index] = u8::try_from((index * 7) & 255).map_err(debug)?; image.alpha[index] = 255; } fs::write(root.join("image.tga"), Targa::encode(image).map_err(debug)?) .map_err(|error| error.to_string())?; let mut model = ModelPrim::new().map_err(debug)?; let mut face = ModelFace::new().map_err(debug)?; let side = config.mesh_side; for y in 0..side { for x in 0..side { let u = x as f32 / (side - 1) as f32; let v = y as f32 / (side - 1) as f32; face.vertices.push(Vertex { position: Vector3 { x: u - 0.5, y: v - 0.5, z: (u * std::f32::consts::TAU).sin() * 0.05, }, normal: Vector3::unit_z(), tex_coord: Vector2 { x: u, y: v }, }); } } for y in 0..side - 1 { for x in 0..side - 1 { let a = u32::try_from(y * side + x).map_err(debug)?; let c = a + u32::try_from(side).map_err(debug)?; face.indices .extend_from_slice(&[a, a + 1, c, a + 1, c + 1, c]); } } model.faces.push(face); model.create_asset(UUID::zero()).map_err(debug)?; fs::write(root.join("mesh_asset.bin"), model.asset).map_err(|error| error.to_string())?; let mut packet = AgentPausePacket::new_with_constructor().map_err(debug)?; packet.agent_data.agent_id = UUID::new_with_string(config.uuid.clone()).map_err(debug)?; packet.agent_data.session_id = UUID::new_with_u_int64(0x1234).map_err(debug)?; packet.agent_data.serial_num = u32::try_from(config.packet_blocks).map_err(debug)?; fs::write( root.join("agent_pause.packet"), packet.to_bytes_with_method().map_err(debug)?, ) .map_err(|error| error.to_string())?; write_manifest(root) } fn write_manifest(root: &Path) -> AppResult<()> { let names = [ "workloads.json", "notecard.txt", "image.tga", "mesh_asset.bin", "agent_pause.packet", ]; let mut hashes = BTreeMap::new(); for name in names { hashes.insert(name.to_owned(), hash_file(&root.join(name))?); } write_json(&root.join("manifest.json"), &hashes) } fn context(root: &Path) -> AppResult { let fixture = fixture(&root.join("workloads.json"))?; let uuid = UUID::new_with_string(fixture.uuid.clone()).map_err(debug)?; let llsd = OSD::Map(HashMap::from([ ("agent_id".into(), OSD::UUID(uuid)), ("name".into(), OSD::String("MetaCrate benchmark".into())), ("active".into(), OSD::Boolean(true)), ("score".into(), OSD::Real(42.25)), ( "items".into(), OSD::Array( (0..fixture.llsd_array_length) .map(|value| OSD::Integer(i32::try_from(value).unwrap_or(i32::MAX))) .collect(), ), ), ])); Ok(Context { fixture, uuid, llsd, packet: read(root, "agent_pause.packet")?, notecard: read(root, "notecard.txt")?, image: read(root, "image.tga")?, mesh: read(root, "mesh_asset.bin")?, }) } fn read(root: &Path, name: &str) -> AppResult> { fs::read(root.join(name)).map_err(|error| format!("read {name}: {error}")) } fn run_benchmarks(root: &Path) -> AppResult { verify_manifest(root)?; let context = context(root)?; let mut results = Vec::new(); for workload in workloads() { let iterations = (workload.iterations)(&context.fixture); let cold = measure(&context, workload.run, 1)?; let warmup = (iterations / context.fixture.warmup_divisor).max(1); black_box((workload.run)(&context, warmup)?); let mut warm_samples = Vec::with_capacity(RUNS); for _ in 0..RUNS { warm_samples.push(measure(&context, workload.run, iterations)?); } let mut ordered = warm_samples.clone(); ordered.sort_by_key(|sample| sample.elapsed_ns); let warm_median = ordered[RUNS / 2].clone(); results.push(BenchmarkResult { name: workload.name.into(), category: workload.category.into(), unit: "operation".into(), fixture: workload.fixture.into(), cold, warm_samples, warm_median, }); } Ok(Report { schema: TOOL_SCHEMA, runtime: "rust-native".into(), reference_commit: context.fixture.reference_commit.clone(), fixture_hashes: manifest(root)?, environment: Environment { os: std::env::consts::OS.into(), arch: std::env::consts::ARCH.into(), logical_cpus: std::thread::available_parallelism().map_or(1, usize::from), toolchain: command_version("rustc", &["--version"]), profile: "benchmark (opt-level=1, debug=0, incremental=false)".into(), }, results, }) } fn measure( context: &Context, function: fn(&Context, usize) -> AppResult, iterations: usize, ) -> AppResult { let region = Region::new(ALLOCATOR); let started = Instant::now(); black_box(function(context, iterations)?); let elapsed = started.elapsed(); let stats = region.change(); Ok(metrics(iterations, elapsed.as_nanos(), stats)) } fn metrics(iterations: usize, elapsed_ns: u128, stats: Stats) -> Metrics { let seconds = elapsed_ns as f64 / 1_000_000_000.0; let retained = stats.bytes_allocated as i128 - stats.bytes_deallocated as i128 + stats.bytes_reallocated as i128; Metrics { iterations, elapsed_ns, latency_ns: elapsed_ns as f64 / iterations as f64, throughput_per_second: iterations as f64 / seconds, allocations: Some(stats.allocations + stats.reallocations), bytes_allocated: stats.bytes_allocated, retained_bytes: retained, } } fn workloads() -> [Workload; 14] { [ Workload { name: "uuid_math", category: "uuid/math", fixture: "workloads.json", iterations: |f| f.uuid_math_iterations, run: uuid_math, }, Workload { name: "llsd_xml", category: "LLSD", fixture: "workloads.json", iterations: |f| f.llsd_iterations, run: llsd_xml, }, Workload { name: "llsd_json", category: "LLSD", fixture: "workloads.json", iterations: |f| f.llsd_iterations, run: llsd_json, }, Workload { name: "llsd_binary", category: "LLSD", fixture: "workloads.json", iterations: |f| f.llsd_iterations, run: llsd_binary, }, Workload { name: "llsd_notation", category: "LLSD", fixture: "workloads.json", iterations: |f| f.llsd_iterations, run: llsd_notation, }, Workload { name: "llsd_protobuf", category: "LLSD", fixture: "workloads.json", iterations: |f| f.llsd_iterations, run: llsd_protobuf, }, Workload { name: "packet_codec", category: "packet codec", fixture: "agent_pause.packet", iterations: |f| f.packet_iterations, run: packet_codec, }, Workload { name: "asset_decode", category: "asset decode", fixture: "notecard.txt", iterations: |f| f.asset_iterations, run: asset_decode, }, Workload { name: "image_decode", category: "image decode", fixture: "image.tga", iterations: |f| f.image_iterations, run: image_decode, }, Workload { name: "mesh_decode", category: "mesh decode", fixture: "mesh_asset.bin", iterations: |f| f.mesh_iterations, run: mesh_decode, }, Workload { name: "inventory_update", category: "inventory update", fixture: "workloads.json", iterations: |f| f.inventory_iterations, run: inventory_update, }, Workload { name: "object_update", category: "object update", fixture: "workloads.json", iterations: |f| f.object_iterations, run: object_update, }, Workload { name: "rendering", category: "rendering", fixture: "workloads.json", iterations: |f| f.rendering_iterations, run: rendering, }, Workload { name: "client_throughput", category: "client throughput", fixture: "workloads.json", iterations: |f| f.client_iterations, run: client_throughput, }, ] } fn uuid_math(context: &Context, iterations: usize) -> AppResult { let rotation = Quaternion::create_from_eulers_with_single_single_single(0.25, 0.5, 0.75).map_err(debug)?; let mut matrix = Matrix4::create_translation(Vector3 { x: 10.0, y: 20.0, z: 30.0, }) .map_err(debug)?; let mut checksum = 0_u64; for index in 0..iterations { let parsed = UUID::new_with_string(context.fixture.uuid.clone()).map_err(debug)?; matrix = Matrix4::transform(matrix, rotation).map_err(debug)?; checksum ^= parsed.get_u_long().map_err(debug)? ^ u64::from(matrix.m41.to_bits()) ^ index as u64; } Ok(checksum) } macro_rules! llsd_workload { ($name:ident, $serialize:expr, $deserialize:expr) => { fn $name(context: &Context, iterations: usize) -> AppResult { let mut checksum = 0_u64; for _ in 0..iterations { let encoded = $serialize(&context.llsd).map_err(debug)?; let decoded = $deserialize(encoded).map_err(debug)?; checksum ^= u64::from(decoded.as_boolean().map_err(debug)?); } Ok(checksum) } }; } llsd_workload!( llsd_xml, |value: &OSD| OSDParser::serialize_llsd_xml_bytes(value.clone()), OSDParser::deserialize_llsd_xml_with_bytes ); llsd_workload!( llsd_binary, |value: &OSD| OSDParser::serialize_llsd_binary_with_osd(value.clone()), OSDParser::deserialize_llsd_binary_with_bytes ); llsd_workload!( llsd_protobuf, |value: &OSD| OSDParser::serialize_llsd_protobuf(value.clone(), Some(true)), OSDParser::deserialize_llsd_protobuf_with_bytes ); fn llsd_json(context: &Context, iterations: usize) -> AppResult { let mut checksum = 0; for _ in 0..iterations { let encoded = OSDParser::serialize_json_string(context.llsd.clone(), None).map_err(debug)?; let decoded = OSDParser::deserialize_json_with_string(encoded.clone()).map_err(debug)?; checksum ^= encoded.len() as u64 ^ u64::from(decoded.as_boolean().map_err(debug)?); } Ok(checksum) } fn llsd_notation(context: &Context, iterations: usize) -> AppResult { let mut checksum = 0; for _ in 0..iterations { let encoded = OSDParser::serialize_llsd_notation(context.llsd.clone()).map_err(debug)?; let decoded = OSDParser::deserialize_llsd_notation_with_string(encoded.clone()).map_err(debug)?; checksum ^= encoded.len() as u64 ^ u64::from(decoded.as_boolean().map_err(debug)?); } Ok(checksum) } fn packet_codec(context: &Context, iterations: usize) -> AppResult { let mut checksum = 0; for _ in 0..iterations { let mut start = 0; let packet = AgentPausePacket::new_with_bytes_int32(context.packet.clone(), &mut start) .map_err(debug)?; let encoded = packet.to_bytes_with_method().map_err(debug)?; checksum ^= encoded.len() as u64 ^ u64::from(packet.agent_data.serial_num); } Ok(checksum) } fn asset_decode(context: &Context, iterations: usize) -> AppResult { let mut checksum = 0; for _ in 0..iterations { let asset = AssetNotecard::new_with_uuid_bytes(context.uuid, context.notecard.clone()) .map_err(debug)?; if !asset.decode().map_err(debug)? { return Err("notecard decode returned false".into()); } checksum ^= asset.body_text.len() as u64; } Ok(checksum) } fn image_decode(context: &Context, iterations: usize) -> AppResult { let mut checksum = 0; for _ in 0..iterations { let image = Targa::decode_to_managed_image_with_bytes(&context.image).map_err(debug)?; checksum ^= image.red.len() as u64; } Ok(checksum) } fn mesh_decode(context: &Context, iterations: usize) -> AppResult { let renderer = MeshFoundry::new().map_err(debug)?; let mut checksum = 0; for _ in 0..iterations { let mesh = renderer .generate_faceted_mesh_mesh_with_primitive_bytes_detail_level( Primitive::new_with_constructor().map_err(debug)?, context.mesh.clone(), DetailLevel::Highest, ) .map_err(debug)? .ok_or_else(|| "mesh fixture returned no geometry".to_owned())?; checksum ^= mesh .faces .iter() .map(|face| face.vertices.len() + face.indices.len()) .sum::() as u64; } Ok(checksum) } fn inventory_update(context: &Context, iterations: usize) -> AppResult { let store = Inventory::new_with_grid_client_uuid( Arc::new(GridClient::new().map_err(debug)?), context.uuid, ) .map_err(debug)?; for index in 0..iterations { let id = UUID::new_with_u_int64((index % context.fixture.inventory_items) as u64 + 1) .map_err(debug)?; let mut item = InventoryItem::new_with_uuid(id).map_err(debug)?; item.base.set_name(format!("item-{index}")); store.update_node_for(&item).map_err(debug)?; } u64::try_from(store.count()).map_err(debug) } fn object_update(context: &Context, iterations: usize) -> AppResult { let mut objects = (0..context.fixture.object_count) .map(|index| { let mut primitive = Primitive::new_with_constructor().expect("bounded primitive"); primitive.local_id = u32::try_from(index).unwrap_or(u32::MAX); primitive }) .collect::>(); let mut checksum = 0; let object_count = objects.len(); for index in 0..iterations { let object = &mut objects[index % object_count]; object.position = Vector3 { x: index as f32, y: (index & 255) as f32, z: 21.0, }; object.rotation = Quaternion::create_from_axis_angle_with_vector3_single( Vector3::unit_z(), (index & 255) as f32 / 255.0, ) .map_err(debug)?; checksum ^= u64::from(object.local_id) ^ u64::from(object.position.x.to_bits()); } Ok(checksum) } fn rendering(_context: &Context, iterations: usize) -> AppResult { let renderer = SimpleRenderer::new().map_err(debug)?; let mut checksum = 0; for _ in 0..iterations { let mesh = renderer .generate_faceted_mesh(box_primitive()?, DetailLevel::High) .map_err(debug)?; checksum ^= mesh .faces .iter() .map(|face| face.indices.len()) .sum::() as u64; } Ok(checksum) } fn box_primitive() -> AppResult { let mut primitive = Primitive::new_with_constructor().map_err(debug)?; let mut data = PrimitiveConstructionData::new_with_constructor().map_err(debug)?; data.p_code = PCode::Prim; data.path_curve = PathCurve::Line; data.set_profile_curve_with_property(ProfileCurve::Square); data.path_scale_x = 1.0; data.path_scale_y = 1.0; data.path_begin = 0.0; data.path_end = 1.0; data.profile_begin = 0.0; data.profile_end = 1.0; data.profile_hollow = 0.0; data.set_profile_hole(HoleType::Same); data.path_revolutions = 1.0; primitive.prim_data = data; Ok(primitive) } fn client_throughput(context: &Context, iterations: usize) -> AppResult { let _client = GridClient::new().map_err(debug)?; let mut checksum = 0; for index in 0..iterations { for _ in 0..context.fixture.client_batch { let mut source = RemoteParcelRequestReply::new().map_err(debug)?; source.parcel_id = context.uuid; let map = source.serialize().map_err(debug)?; let mut target = RemoteParcelRequestReply::new().map_err(debug)?; target.deserialize(map).map_err(debug)?; checksum ^= target.parcel_id.get_u_long().map_err(debug)? ^ index as u64; } } Ok(checksum) } #[derive(Debug, PartialEq, Serialize, Deserialize)] struct ComparisonRow { name: String, rust_latency_ns: f64, reference_latency_ns: f64, latency_ratio: f64, rust_bytes_per_operation: f64, reference_bytes_per_operation: f64, allocation_ratio: f64, accepted: bool, criterion: String, } #[derive(Debug, PartialEq, Serialize, Deserialize)] struct Comparison { schema: u32, reference_commit: String, fixture_hashes: BTreeMap, criteria: Criteria, results: Vec, accepted: bool, } #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] struct Criteria { maximum_latency_ratio: f64, maximum_allocation_ratio: f64, material_latency_regression_ratio: f64, minimum_reference_latency_ns: f64, reviewed_exceptions: Vec, } #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] struct ReviewedException { workload: String, maximum_latency_ratio: f64, maximum_allocation_ratio: f64, rationale: String, } fn criteria() -> Criteria { serde_json::from_str(include_str!("../../../benchmarks/release-criteria.json")) .expect("committed release criteria must be valid") } fn compare_reports(rust_path: &Path, reference_path: &Path, output: &Path) -> AppResult<()> { let rust: Report = read_json(rust_path)?; let reference: Report = read_json(reference_path)?; validate_pair(&rust, &reference)?; write_json(output, &build_comparison(&rust, &reference)?) } fn build_comparison(rust: &Report, reference: &Report) -> AppResult { let policy = criteria(); let mut rows = Vec::new(); for rust_result in &rust.results { let reference_result = reference .results .iter() .find(|result| result.name == rust_result.name) .ok_or_else(|| format!("reference report is missing {}", rust_result.name))?; let latency_ratio = rust_result.warm_median.latency_ns / reference_result.warm_median.latency_ns; let rust_bytes = rust_result.warm_median.bytes_allocated as f64 / rust_result.warm_median.iterations as f64; let reference_bytes = reference_result.warm_median.bytes_allocated as f64 / reference_result.warm_median.iterations as f64; let allocation_ratio = if reference_bytes == 0.0 { if rust_bytes == 0.0 { 1.0 } else { f64::INFINITY } } else { rust_bytes / reference_bytes }; let exception = policy .reviewed_exceptions .iter() .find(|exception| exception.workload == rust_result.name); let maximum_latency = exception.map_or(policy.maximum_latency_ratio, |value| { value.maximum_latency_ratio }); let maximum_allocation = exception.map_or(policy.maximum_allocation_ratio, |value| { value.maximum_allocation_ratio }); let latency_noise_exempt = reference_result.warm_median.latency_ns < policy.minimum_reference_latency_ns; let material_latency_ok = latency_noise_exempt || latency_ratio <= policy.material_latency_regression_ratio || exception.is_some(); let latency_ok = (latency_noise_exempt || latency_ratio <= maximum_latency) && material_latency_ok; let allocation_ok = allocation_ratio <= maximum_allocation; rows.push(ComparisonRow { name: rust_result.name.clone(), rust_latency_ns: rust_result.warm_median.latency_ns, reference_latency_ns: reference_result.warm_median.latency_ns, latency_ratio, rust_bytes_per_operation: rust_bytes, reference_bytes_per_operation: reference_bytes, allocation_ratio, accepted: latency_ok && allocation_ok, criterion: if latency_ok && allocation_ok { exception.map_or_else( || "within default release thresholds".into(), |value| format!("reviewed exception: {}", value.rationale), ) } else { "material regression requires profiling evidence or an explicit reviewed exception" .into() }, }); } let accepted = rows.iter().all(|row| row.accepted); Ok(Comparison { schema: TOOL_SCHEMA, reference_commit: rust.reference_commit.clone(), fixture_hashes: rust.fixture_hashes.clone(), criteria: policy, results: rows, accepted, }) } fn audit( root: &Path, rust_path: &Path, reference_path: &Path, comparison_path: &Path, ) -> AppResult<()> { verify_manifest(root)?; let rust: Report = read_json(rust_path)?; let reference: Report = read_json(reference_path)?; let comparison: Comparison = read_json(comparison_path)?; validate_pair(&rust, &reference)?; let hashes = manifest(root)?; if rust.fixture_hashes != hashes || reference.fixture_hashes != hashes || comparison.fixture_hashes != hashes { return Err("one or more reports do not match the committed fixture hashes".into()); } let expected: Vec<_> = workloads().iter().map(|workload| workload.name).collect(); let expected_set = expected.iter().copied().collect::>(); for report in [&rust, &reference] { if report.schema != TOOL_SCHEMA || report.reference_commit != EXPECTED_REFERENCE { return Err("invalid report schema or reference commit".into()); } let reported = report .results .iter() .map(|result| result.name.as_str()) .collect::>(); if report.results.len() != expected.len() || reported != expected_set { return Err(format!( "{} report contains missing, duplicate, or unexpected workloads", report.runtime )); } for name in &expected { let result = report .results .iter() .find(|result| result.name == *name) .ok_or_else(|| format!("{} report missing {name}", report.runtime))?; if result.cold.iterations != 1 || result.warm_samples.len() != RUNS || result.warm_median.iterations == 0 { return Err(format!("{} has incomplete cold/warm evidence", result.name)); } } } let recomputed = build_comparison(&rust, &reference)?; let checked_bytes = fs::read(comparison_path) .map_err(|error| format!("read {}: {error}", comparison_path.display()))?; if checked_bytes != json_bytes(&recomputed)? { return Err("release comparison is stale; rerun the compare command".into()); } if !comparison.accepted { return Err("release comparison is incomplete or contains an unaccepted regression".into()); } println!( "performance audit passed: {} workloads, pinned reference {}, fixture hashes verified", expected.len(), EXPECTED_REFERENCE ); Ok(()) } fn validate_pair(rust: &Report, reference: &Report) -> AppResult<()> { if rust.runtime != "rust-native" || reference.runtime != "dotnet-reference" { return Err( "reports must be generated independently by rust-native and dotnet-reference".into(), ); } if rust.reference_commit != EXPECTED_REFERENCE || reference.reference_commit != EXPECTED_REFERENCE || rust.fixture_hashes != reference.fixture_hashes { return Err("reports use different fixtures or reference commits".into()); } if rust.environment.os != reference.environment.os || rust.environment.arch != reference.environment.arch { return Err("reports must be captured on the same OS and architecture".into()); } Ok(()) } fn verify_manifest(root: &Path) -> AppResult<()> { let expected = manifest(root)?; for (name, hash) in expected { let actual = hash_file(&root.join(&name))?; if actual != hash { return Err(format!("fixture hash mismatch for {name}")); } } Ok(()) } fn manifest(root: &Path) -> AppResult> { read_json(&root.join("manifest.json")) } fn hash_file(path: &Path) -> AppResult { let bytes = fs::read(path).map_err(|error| format!("read {}: {error}", path.display()))?; Ok(Sha256::digest(bytes) .iter() .fold(String::with_capacity(64), |mut output, byte| { write!(output, "{byte:02x}").expect("writing to a String is infallible"); output })) } fn read_json Deserialize<'de>>(path: &Path) -> AppResult { serde_json::from_slice( &fs::read(path).map_err(|error| format!("read {}: {error}", path.display()))?, ) .map_err(|error| format!("parse {}: {error}", path.display())) } fn write_json(path: &Path, value: &T) -> AppResult<()> { if let Some(parent) = path.parent() { fs::create_dir_all(parent).map_err(|error| error.to_string())?; } fs::write(path, json_bytes(value)?) .map_err(|error| format!("write {}: {error}", path.display())) } fn json_bytes(value: &T) -> AppResult> { let mut bytes = serde_json::to_vec_pretty(value).map_err(|error| error.to_string())?; bytes.push(b'\n'); Ok(bytes) } fn command_version(program: &str, args: &[&str]) -> String { std::process::Command::new(program) .args(args) .output() .ok() .filter(|output| output.status.success()) .map_or_else( || "unavailable".into(), |output| String::from_utf8_lossy(&output.stdout).trim().into(), ) } fn debug(error: impl std::fmt::Debug) -> String { format!("{error:?}") } #[cfg(test)] #[allow(clippy::float_cmp)] // Metrics use exact integer-derived values in these tests. mod tests { use super::*; #[test] fn workload_set_covers_every_issue_category() { let categories: Vec<_> = workloads() .iter() .map(|workload| workload.category) .collect(); for expected in [ "uuid/math", "LLSD", "packet codec", "asset decode", "image decode", "mesh decode", "inventory update", "object update", "rendering", "client throughput", ] { assert!(categories.contains(&expected), "missing {expected}"); } } #[test] fn metrics_separate_latency_throughput_allocations_and_retained_memory() { let result = metrics( 10, 1_000, Stats { allocations: 2, deallocations: 1, reallocations: 1, bytes_allocated: 100, bytes_deallocated: 40, bytes_reallocated: 20, }, ); assert_eq!(result.latency_ns, 100.0); assert_eq!(result.throughput_per_second, 10_000_000.0); assert_eq!(result.allocations, Some(3)); assert_eq!(result.retained_bytes, 80); } #[test] fn release_criteria_identify_material_regressions() { let policy = criteria(); assert_eq!(policy.maximum_latency_ratio, 2.0); assert!(policy.material_latency_regression_ratio < policy.maximum_latency_ratio); assert_eq!(policy.reviewed_exceptions.len(), 3); assert!( policy .reviewed_exceptions .iter() .any(|exception| exception.workload == "client_throughput") ); } }