Chili Palmer
c9a1170a27
Some checks failed
API and SemVer surface / api-surface (push) Failing after 1m34s
Native code generation / deterministic (push) Has been cancelled
Concurrency and resource soak audit / soak (push) Has been cancelled
Documentation / documentation (push) Has been cancelled
performance evidence / audit (push) Has been cancelled
First release candidate / non-fuzz-release-gate (push) Has been cancelled
Release platform and feature matrix / audit (push) Has been cancelled
Release platform and feature matrix / matrix (false, linux-stable-minimal, x86_64-unknown-linux-gnu, stable) (push) Has been cancelled
Release platform and feature matrix / matrix (false, macos-stable-portable, x86_64-apple-darwin, stable) (push) Has been cancelled
Release platform and feature matrix / matrix (false, windows-stable-portable, x86_64-pc-windows-gnu, stable) (push) Has been cancelled
Release platform and feature matrix / matrix (true, linux-msrv-portable, x86_64-unknown-linux-gnu, 1.96.0) (push) Has been cancelled
Release platform and feature matrix / matrix (true, linux-stable-default, x86_64-unknown-linux-gnu, stable) (push) Has been cancelled
Release platform and feature matrix / matrix (true, linux-stable-features, x86_64-unknown-linux-gnu, stable) (push) Has been cancelled
Release platform and feature matrix / matrix (true, linux-stable-release-surface, x86_64-unknown-linux-gnu, stable) (push) Has been cancelled
Dependency and supply-chain audit / audit (push) Has been cancelled
Native release artifact audit / audit (push) Has been cancelled
Imaging and meshing gate / native (push) Failing after 53s
JPEG 2000 feature / linux (push) Successful in 2m49s
Native Rust workspace compile / compile (push) Failing after 59s
Skia feature / linux (push) Has been cancelled
1031 lines
35 KiB
Rust
1031 lines
35 KiB
Rust
#![allow(clippy::cast_precision_loss)]
|
|
|
|
use std::alloc::System;
|
|
use std::collections::{BTreeMap, BTreeSet, HashMap};
|
|
use std::fmt::Write as _;
|
|
use std::fs;
|
|
use std::hint::black_box;
|
|
use std::path::Path;
|
|
use std::sync::Arc;
|
|
use std::time::Instant;
|
|
|
|
use libremetaverse::assets::AssetNotecard;
|
|
use libremetaverse::imaging::Targa;
|
|
use libremetaverse::import_export::{ModelFace, ModelPrim};
|
|
use libremetaverse::messages::linden::RemoteParcelRequestReply;
|
|
use libremetaverse::packets::AgentPausePacket;
|
|
use libremetaverse::rendering::{DetailLevel, Vertex};
|
|
use libremetaverse::{GridClient, Inventory, InventoryItem, Primitive, PrimitiveConstructionData};
|
|
use libremetaverse_imaging::{ManagedImage, ManagedImageImageChannels};
|
|
use libremetaverse_rendering_mesh_foundry::MeshFoundry;
|
|
use libremetaverse_rendering_simple::SimpleRenderer;
|
|
use libremetaverse_structured_data::{OSD, OSDParser};
|
|
use libremetaverse_types::{
|
|
HoleType, Matrix4, PCode, PathCurve, ProfileCurve, Quaternion, UUID, Vector2, Vector3,
|
|
};
|
|
use serde::{Deserialize, Serialize};
|
|
use sha2::{Digest, Sha256};
|
|
use stats_alloc::{INSTRUMENTED_SYSTEM, Region, Stats, StatsAlloc};
|
|
|
|
#[global_allocator]
|
|
static ALLOCATOR: &StatsAlloc<System> = &INSTRUMENTED_SYSTEM;
|
|
|
|
const RUNS: usize = 7;
|
|
const TOOL_SCHEMA: u32 = 1;
|
|
const EXPECTED_REFERENCE: &str = "2aa70bb68513b39795da5d13c88f31b86e85a3ba";
|
|
|
|
type AppResult<T> = Result<T, String>;
|
|
|
|
#[derive(Clone, Deserialize)]
|
|
struct Fixture {
|
|
schema: u32,
|
|
reference_commit: String,
|
|
uuid: String,
|
|
uuid_math_iterations: usize,
|
|
llsd_iterations: usize,
|
|
packet_iterations: usize,
|
|
asset_iterations: usize,
|
|
image_iterations: usize,
|
|
mesh_iterations: usize,
|
|
inventory_iterations: usize,
|
|
object_iterations: usize,
|
|
rendering_iterations: usize,
|
|
client_iterations: usize,
|
|
warmup_divisor: usize,
|
|
llsd_array_length: usize,
|
|
packet_blocks: usize,
|
|
image_width: i32,
|
|
image_height: i32,
|
|
mesh_side: usize,
|
|
inventory_items: usize,
|
|
object_count: usize,
|
|
client_batch: usize,
|
|
}
|
|
|
|
#[derive(Serialize, Deserialize)]
|
|
struct Environment {
|
|
os: String,
|
|
arch: String,
|
|
logical_cpus: usize,
|
|
toolchain: String,
|
|
profile: String,
|
|
}
|
|
|
|
#[derive(Clone, Serialize, Deserialize)]
|
|
struct Metrics {
|
|
iterations: usize,
|
|
elapsed_ns: u128,
|
|
latency_ns: f64,
|
|
throughput_per_second: f64,
|
|
allocations: Option<usize>,
|
|
bytes_allocated: usize,
|
|
retained_bytes: i128,
|
|
}
|
|
|
|
#[derive(Serialize, Deserialize)]
|
|
struct BenchmarkResult {
|
|
name: String,
|
|
category: String,
|
|
unit: String,
|
|
fixture: String,
|
|
cold: Metrics,
|
|
warm_samples: Vec<Metrics>,
|
|
warm_median: Metrics,
|
|
}
|
|
|
|
#[derive(Serialize, Deserialize)]
|
|
struct Report {
|
|
schema: u32,
|
|
runtime: String,
|
|
reference_commit: String,
|
|
fixture_hashes: BTreeMap<String, String>,
|
|
environment: Environment,
|
|
results: Vec<BenchmarkResult>,
|
|
}
|
|
|
|
struct Context {
|
|
fixture: Fixture,
|
|
uuid: UUID,
|
|
llsd: OSD,
|
|
packet: Vec<u8>,
|
|
notecard: Vec<u8>,
|
|
image: Vec<u8>,
|
|
mesh: Vec<u8>,
|
|
}
|
|
|
|
#[derive(Clone, Copy)]
|
|
struct Workload {
|
|
name: &'static str,
|
|
category: &'static str,
|
|
fixture: &'static str,
|
|
iterations: fn(&Fixture) -> usize,
|
|
run: fn(&Context, usize) -> AppResult<u64>,
|
|
}
|
|
|
|
fn main() {
|
|
if let Err(error) = run_cli() {
|
|
eprintln!("performance audit failed: {error}");
|
|
std::process::exit(1);
|
|
}
|
|
}
|
|
|
|
fn run_cli() -> AppResult<()> {
|
|
let args: Vec<String> = std::env::args().skip(1).collect();
|
|
match args.first().map(String::as_str) {
|
|
Some("fixtures") => {
|
|
let root =
|
|
value(&args, "--fixture-root").unwrap_or_else(|| "benchmarks/fixtures".into());
|
|
generate_fixtures(Path::new(&root))
|
|
}
|
|
Some("run") => {
|
|
let root =
|
|
value(&args, "--fixture-root").unwrap_or_else(|| "benchmarks/fixtures".into());
|
|
let output = value(&args, "--output")
|
|
.unwrap_or_else(|| "benchmarks/results/rust-linux-x86_64.json".into());
|
|
let report = run_benchmarks(Path::new(&root))?;
|
|
write_json(Path::new(&output), &report)
|
|
}
|
|
Some("compare") => {
|
|
let rust = required(&args, "--rust")?;
|
|
let reference = required(&args, "--reference")?;
|
|
let output = required(&args, "--output")?;
|
|
compare_reports(Path::new(&rust), Path::new(&reference), Path::new(&output))
|
|
}
|
|
Some("audit") => {
|
|
let root =
|
|
value(&args, "--fixture-root").unwrap_or_else(|| "benchmarks/fixtures".into());
|
|
let rust = required(&args, "--rust")?;
|
|
let reference = required(&args, "--reference")?;
|
|
let comparison = required(&args, "--comparison")?;
|
|
audit(
|
|
Path::new(&root),
|
|
Path::new(&rust),
|
|
Path::new(&reference),
|
|
Path::new(&comparison),
|
|
)
|
|
}
|
|
_ => Err("usage: metacrate-performance <fixtures|run|compare|audit> [options]".into()),
|
|
}
|
|
}
|
|
|
|
fn value(args: &[String], key: &str) -> Option<String> {
|
|
args.windows(2)
|
|
.find(|pair| pair[0] == key)
|
|
.map(|pair| pair[1].clone())
|
|
}
|
|
|
|
fn required(args: &[String], key: &str) -> AppResult<String> {
|
|
value(args, key).ok_or_else(|| format!("missing {key}"))
|
|
}
|
|
|
|
fn fixture(path: &Path) -> AppResult<Fixture> {
|
|
let bytes = fs::read(path).map_err(|error| format!("read {}: {error}", path.display()))?;
|
|
let fixture: Fixture = serde_json::from_slice(&bytes)
|
|
.map_err(|error| format!("parse {}: {error}", path.display()))?;
|
|
if fixture.schema != 1 || fixture.reference_commit != EXPECTED_REFERENCE {
|
|
return Err("fixture schema or pinned reference commit does not match policy".into());
|
|
}
|
|
if fixture.warmup_divisor == 0 {
|
|
return Err("warmup_divisor must be nonzero".into());
|
|
}
|
|
Ok(fixture)
|
|
}
|
|
|
|
fn generate_fixtures(root: &Path) -> AppResult<()> {
|
|
fs::create_dir_all(root).map_err(|error| error.to_string())?;
|
|
let config = fixture(&root.join("workloads.json"))?;
|
|
|
|
let mut image = ManagedImage::new(
|
|
config.image_width,
|
|
config.image_height,
|
|
ManagedImageImageChannels(
|
|
ManagedImageImageChannels::COLOR.0 | ManagedImageImageChannels::ALPHA.0,
|
|
),
|
|
)
|
|
.map_err(debug)?;
|
|
for index in 0..image.red.len() {
|
|
image.red[index] = u8::try_from(index & 255).map_err(debug)?;
|
|
image.green[index] = u8::try_from((index * 3) & 255).map_err(debug)?;
|
|
image.blue[index] = u8::try_from((index * 7) & 255).map_err(debug)?;
|
|
image.alpha[index] = 255;
|
|
}
|
|
fs::write(root.join("image.tga"), Targa::encode(image).map_err(debug)?)
|
|
.map_err(|error| error.to_string())?;
|
|
|
|
let mut model = ModelPrim::new().map_err(debug)?;
|
|
let mut face = ModelFace::new().map_err(debug)?;
|
|
let side = config.mesh_side;
|
|
for y in 0..side {
|
|
for x in 0..side {
|
|
let u = x as f32 / (side - 1) as f32;
|
|
let v = y as f32 / (side - 1) as f32;
|
|
face.vertices.push(Vertex {
|
|
position: Vector3 {
|
|
x: u - 0.5,
|
|
y: v - 0.5,
|
|
z: (u * std::f32::consts::TAU).sin() * 0.05,
|
|
},
|
|
normal: Vector3::unit_z(),
|
|
tex_coord: Vector2 { x: u, y: v },
|
|
});
|
|
}
|
|
}
|
|
for y in 0..side - 1 {
|
|
for x in 0..side - 1 {
|
|
let a = u32::try_from(y * side + x).map_err(debug)?;
|
|
let c = a + u32::try_from(side).map_err(debug)?;
|
|
face.indices
|
|
.extend_from_slice(&[a, a + 1, c, a + 1, c + 1, c]);
|
|
}
|
|
}
|
|
model.faces.push(face);
|
|
model.create_asset(UUID::zero()).map_err(debug)?;
|
|
fs::write(root.join("mesh_asset.bin"), model.asset).map_err(|error| error.to_string())?;
|
|
|
|
let mut packet = AgentPausePacket::new_with_constructor().map_err(debug)?;
|
|
packet.agent_data.agent_id = UUID::new_with_string(config.uuid.clone()).map_err(debug)?;
|
|
packet.agent_data.session_id = UUID::new_with_u_int64(0x1234).map_err(debug)?;
|
|
packet.agent_data.serial_num = u32::try_from(config.packet_blocks).map_err(debug)?;
|
|
fs::write(
|
|
root.join("agent_pause.packet"),
|
|
packet.to_bytes_with_method().map_err(debug)?,
|
|
)
|
|
.map_err(|error| error.to_string())?;
|
|
write_manifest(root)
|
|
}
|
|
|
|
fn write_manifest(root: &Path) -> AppResult<()> {
|
|
let names = [
|
|
"workloads.json",
|
|
"notecard.txt",
|
|
"image.tga",
|
|
"mesh_asset.bin",
|
|
"agent_pause.packet",
|
|
];
|
|
let mut hashes = BTreeMap::new();
|
|
for name in names {
|
|
hashes.insert(name.to_owned(), hash_file(&root.join(name))?);
|
|
}
|
|
write_json(&root.join("manifest.json"), &hashes)
|
|
}
|
|
|
|
fn context(root: &Path) -> AppResult<Context> {
|
|
let fixture = fixture(&root.join("workloads.json"))?;
|
|
let uuid = UUID::new_with_string(fixture.uuid.clone()).map_err(debug)?;
|
|
let llsd = OSD::Map(HashMap::from([
|
|
("agent_id".into(), OSD::UUID(uuid)),
|
|
("name".into(), OSD::String("MetaCrate benchmark".into())),
|
|
("active".into(), OSD::Boolean(true)),
|
|
("score".into(), OSD::Real(42.25)),
|
|
(
|
|
"items".into(),
|
|
OSD::Array(
|
|
(0..fixture.llsd_array_length)
|
|
.map(|value| OSD::Integer(i32::try_from(value).unwrap_or(i32::MAX)))
|
|
.collect(),
|
|
),
|
|
),
|
|
]));
|
|
Ok(Context {
|
|
fixture,
|
|
uuid,
|
|
llsd,
|
|
packet: read(root, "agent_pause.packet")?,
|
|
notecard: read(root, "notecard.txt")?,
|
|
image: read(root, "image.tga")?,
|
|
mesh: read(root, "mesh_asset.bin")?,
|
|
})
|
|
}
|
|
|
|
fn read(root: &Path, name: &str) -> AppResult<Vec<u8>> {
|
|
fs::read(root.join(name)).map_err(|error| format!("read {name}: {error}"))
|
|
}
|
|
|
|
fn run_benchmarks(root: &Path) -> AppResult<Report> {
|
|
verify_manifest(root)?;
|
|
let context = context(root)?;
|
|
let mut results = Vec::new();
|
|
for workload in workloads() {
|
|
let iterations = (workload.iterations)(&context.fixture);
|
|
let cold = measure(&context, workload.run, 1)?;
|
|
let warmup = (iterations / context.fixture.warmup_divisor).max(1);
|
|
black_box((workload.run)(&context, warmup)?);
|
|
let mut warm_samples = Vec::with_capacity(RUNS);
|
|
for _ in 0..RUNS {
|
|
warm_samples.push(measure(&context, workload.run, iterations)?);
|
|
}
|
|
let mut ordered = warm_samples.clone();
|
|
ordered.sort_by_key(|sample| sample.elapsed_ns);
|
|
let warm_median = ordered[RUNS / 2].clone();
|
|
results.push(BenchmarkResult {
|
|
name: workload.name.into(),
|
|
category: workload.category.into(),
|
|
unit: "operation".into(),
|
|
fixture: workload.fixture.into(),
|
|
cold,
|
|
warm_samples,
|
|
warm_median,
|
|
});
|
|
}
|
|
Ok(Report {
|
|
schema: TOOL_SCHEMA,
|
|
runtime: "rust-native".into(),
|
|
reference_commit: context.fixture.reference_commit.clone(),
|
|
fixture_hashes: manifest(root)?,
|
|
environment: Environment {
|
|
os: std::env::consts::OS.into(),
|
|
arch: std::env::consts::ARCH.into(),
|
|
logical_cpus: std::thread::available_parallelism().map_or(1, usize::from),
|
|
toolchain: command_version("rustc", &["--version"]),
|
|
profile: "benchmark (opt-level=1, debug=0, incremental=false)".into(),
|
|
},
|
|
results,
|
|
})
|
|
}
|
|
|
|
fn measure(
|
|
context: &Context,
|
|
function: fn(&Context, usize) -> AppResult<u64>,
|
|
iterations: usize,
|
|
) -> AppResult<Metrics> {
|
|
let region = Region::new(ALLOCATOR);
|
|
let started = Instant::now();
|
|
black_box(function(context, iterations)?);
|
|
let elapsed = started.elapsed();
|
|
let stats = region.change();
|
|
Ok(metrics(iterations, elapsed.as_nanos(), stats))
|
|
}
|
|
|
|
fn metrics(iterations: usize, elapsed_ns: u128, stats: Stats) -> Metrics {
|
|
let seconds = elapsed_ns as f64 / 1_000_000_000.0;
|
|
let retained = stats.bytes_allocated as i128 - stats.bytes_deallocated as i128
|
|
+ stats.bytes_reallocated as i128;
|
|
Metrics {
|
|
iterations,
|
|
elapsed_ns,
|
|
latency_ns: elapsed_ns as f64 / iterations as f64,
|
|
throughput_per_second: iterations as f64 / seconds,
|
|
allocations: Some(stats.allocations + stats.reallocations),
|
|
bytes_allocated: stats.bytes_allocated,
|
|
retained_bytes: retained,
|
|
}
|
|
}
|
|
|
|
fn workloads() -> [Workload; 14] {
|
|
[
|
|
Workload {
|
|
name: "uuid_math",
|
|
category: "uuid/math",
|
|
fixture: "workloads.json",
|
|
iterations: |f| f.uuid_math_iterations,
|
|
run: uuid_math,
|
|
},
|
|
Workload {
|
|
name: "llsd_xml",
|
|
category: "LLSD",
|
|
fixture: "workloads.json",
|
|
iterations: |f| f.llsd_iterations,
|
|
run: llsd_xml,
|
|
},
|
|
Workload {
|
|
name: "llsd_json",
|
|
category: "LLSD",
|
|
fixture: "workloads.json",
|
|
iterations: |f| f.llsd_iterations,
|
|
run: llsd_json,
|
|
},
|
|
Workload {
|
|
name: "llsd_binary",
|
|
category: "LLSD",
|
|
fixture: "workloads.json",
|
|
iterations: |f| f.llsd_iterations,
|
|
run: llsd_binary,
|
|
},
|
|
Workload {
|
|
name: "llsd_notation",
|
|
category: "LLSD",
|
|
fixture: "workloads.json",
|
|
iterations: |f| f.llsd_iterations,
|
|
run: llsd_notation,
|
|
},
|
|
Workload {
|
|
name: "llsd_protobuf",
|
|
category: "LLSD",
|
|
fixture: "workloads.json",
|
|
iterations: |f| f.llsd_iterations,
|
|
run: llsd_protobuf,
|
|
},
|
|
Workload {
|
|
name: "packet_codec",
|
|
category: "packet codec",
|
|
fixture: "agent_pause.packet",
|
|
iterations: |f| f.packet_iterations,
|
|
run: packet_codec,
|
|
},
|
|
Workload {
|
|
name: "asset_decode",
|
|
category: "asset decode",
|
|
fixture: "notecard.txt",
|
|
iterations: |f| f.asset_iterations,
|
|
run: asset_decode,
|
|
},
|
|
Workload {
|
|
name: "image_decode",
|
|
category: "image decode",
|
|
fixture: "image.tga",
|
|
iterations: |f| f.image_iterations,
|
|
run: image_decode,
|
|
},
|
|
Workload {
|
|
name: "mesh_decode",
|
|
category: "mesh decode",
|
|
fixture: "mesh_asset.bin",
|
|
iterations: |f| f.mesh_iterations,
|
|
run: mesh_decode,
|
|
},
|
|
Workload {
|
|
name: "inventory_update",
|
|
category: "inventory update",
|
|
fixture: "workloads.json",
|
|
iterations: |f| f.inventory_iterations,
|
|
run: inventory_update,
|
|
},
|
|
Workload {
|
|
name: "object_update",
|
|
category: "object update",
|
|
fixture: "workloads.json",
|
|
iterations: |f| f.object_iterations,
|
|
run: object_update,
|
|
},
|
|
Workload {
|
|
name: "rendering",
|
|
category: "rendering",
|
|
fixture: "workloads.json",
|
|
iterations: |f| f.rendering_iterations,
|
|
run: rendering,
|
|
},
|
|
Workload {
|
|
name: "client_throughput",
|
|
category: "client throughput",
|
|
fixture: "workloads.json",
|
|
iterations: |f| f.client_iterations,
|
|
run: client_throughput,
|
|
},
|
|
]
|
|
}
|
|
|
|
fn uuid_math(context: &Context, iterations: usize) -> AppResult<u64> {
|
|
let rotation =
|
|
Quaternion::create_from_eulers_with_single_single_single(0.25, 0.5, 0.75).map_err(debug)?;
|
|
let mut matrix = Matrix4::create_translation(Vector3 {
|
|
x: 10.0,
|
|
y: 20.0,
|
|
z: 30.0,
|
|
})
|
|
.map_err(debug)?;
|
|
let mut checksum = 0_u64;
|
|
for index in 0..iterations {
|
|
let parsed = UUID::new_with_string(context.fixture.uuid.clone()).map_err(debug)?;
|
|
matrix = Matrix4::transform(matrix, rotation).map_err(debug)?;
|
|
checksum ^=
|
|
parsed.get_u_long().map_err(debug)? ^ u64::from(matrix.m41.to_bits()) ^ index as u64;
|
|
}
|
|
Ok(checksum)
|
|
}
|
|
|
|
macro_rules! llsd_workload {
|
|
($name:ident, $serialize:expr, $deserialize:expr) => {
|
|
fn $name(context: &Context, iterations: usize) -> AppResult<u64> {
|
|
let mut checksum = 0_u64;
|
|
for _ in 0..iterations {
|
|
let encoded = $serialize(&context.llsd).map_err(debug)?;
|
|
let decoded = $deserialize(encoded).map_err(debug)?;
|
|
checksum ^= u64::from(decoded.as_boolean().map_err(debug)?);
|
|
}
|
|
Ok(checksum)
|
|
}
|
|
};
|
|
}
|
|
|
|
llsd_workload!(
|
|
llsd_xml,
|
|
|value: &OSD| OSDParser::serialize_llsd_xml_bytes(value.clone()),
|
|
OSDParser::deserialize_llsd_xml_with_bytes
|
|
);
|
|
llsd_workload!(
|
|
llsd_binary,
|
|
|value: &OSD| OSDParser::serialize_llsd_binary_with_osd(value.clone()),
|
|
OSDParser::deserialize_llsd_binary_with_bytes
|
|
);
|
|
llsd_workload!(
|
|
llsd_protobuf,
|
|
|value: &OSD| OSDParser::serialize_llsd_protobuf(value.clone(), Some(true)),
|
|
OSDParser::deserialize_llsd_protobuf_with_bytes
|
|
);
|
|
|
|
fn llsd_json(context: &Context, iterations: usize) -> AppResult<u64> {
|
|
let mut checksum = 0;
|
|
for _ in 0..iterations {
|
|
let encoded =
|
|
OSDParser::serialize_json_string(context.llsd.clone(), None).map_err(debug)?;
|
|
let decoded = OSDParser::deserialize_json_with_string(encoded.clone()).map_err(debug)?;
|
|
checksum ^= encoded.len() as u64 ^ u64::from(decoded.as_boolean().map_err(debug)?);
|
|
}
|
|
Ok(checksum)
|
|
}
|
|
|
|
fn llsd_notation(context: &Context, iterations: usize) -> AppResult<u64> {
|
|
let mut checksum = 0;
|
|
for _ in 0..iterations {
|
|
let encoded = OSDParser::serialize_llsd_notation(context.llsd.clone()).map_err(debug)?;
|
|
let decoded =
|
|
OSDParser::deserialize_llsd_notation_with_string(encoded.clone()).map_err(debug)?;
|
|
checksum ^= encoded.len() as u64 ^ u64::from(decoded.as_boolean().map_err(debug)?);
|
|
}
|
|
Ok(checksum)
|
|
}
|
|
|
|
fn packet_codec(context: &Context, iterations: usize) -> AppResult<u64> {
|
|
let mut checksum = 0;
|
|
for _ in 0..iterations {
|
|
let mut start = 0;
|
|
let packet = AgentPausePacket::new_with_bytes_int32(context.packet.clone(), &mut start)
|
|
.map_err(debug)?;
|
|
let encoded = packet.to_bytes_with_method().map_err(debug)?;
|
|
checksum ^= encoded.len() as u64 ^ u64::from(packet.agent_data.serial_num);
|
|
}
|
|
Ok(checksum)
|
|
}
|
|
|
|
fn asset_decode(context: &Context, iterations: usize) -> AppResult<u64> {
|
|
let mut checksum = 0;
|
|
for _ in 0..iterations {
|
|
let asset = AssetNotecard::new_with_uuid_bytes(context.uuid, context.notecard.clone())
|
|
.map_err(debug)?;
|
|
if !asset.decode().map_err(debug)? {
|
|
return Err("notecard decode returned false".into());
|
|
}
|
|
checksum ^= asset.body_text.len() as u64;
|
|
}
|
|
Ok(checksum)
|
|
}
|
|
|
|
fn image_decode(context: &Context, iterations: usize) -> AppResult<u64> {
|
|
let mut checksum = 0;
|
|
for _ in 0..iterations {
|
|
let image = Targa::decode_to_managed_image_with_bytes(&context.image).map_err(debug)?;
|
|
checksum ^= image.red.len() as u64;
|
|
}
|
|
Ok(checksum)
|
|
}
|
|
|
|
fn mesh_decode(context: &Context, iterations: usize) -> AppResult<u64> {
|
|
let renderer = MeshFoundry::new().map_err(debug)?;
|
|
let mut checksum = 0;
|
|
for _ in 0..iterations {
|
|
let mesh = renderer
|
|
.generate_faceted_mesh_mesh_with_primitive_bytes_detail_level(
|
|
Primitive::new_with_constructor().map_err(debug)?,
|
|
context.mesh.clone(),
|
|
DetailLevel::Highest,
|
|
)
|
|
.map_err(debug)?
|
|
.ok_or_else(|| "mesh fixture returned no geometry".to_owned())?;
|
|
checksum ^= mesh
|
|
.faces
|
|
.iter()
|
|
.map(|face| face.vertices.len() + face.indices.len())
|
|
.sum::<usize>() as u64;
|
|
}
|
|
Ok(checksum)
|
|
}
|
|
|
|
fn inventory_update(context: &Context, iterations: usize) -> AppResult<u64> {
|
|
let store = Inventory::new_with_grid_client_uuid(
|
|
Arc::new(GridClient::new().map_err(debug)?),
|
|
context.uuid,
|
|
)
|
|
.map_err(debug)?;
|
|
for index in 0..iterations {
|
|
let id = UUID::new_with_u_int64((index % context.fixture.inventory_items) as u64 + 1)
|
|
.map_err(debug)?;
|
|
let mut item = InventoryItem::new_with_uuid(id).map_err(debug)?;
|
|
item.base.set_name(format!("item-{index}"));
|
|
store.update_node_for(&item).map_err(debug)?;
|
|
}
|
|
u64::try_from(store.count()).map_err(debug)
|
|
}
|
|
|
|
fn object_update(context: &Context, iterations: usize) -> AppResult<u64> {
|
|
let mut objects = (0..context.fixture.object_count)
|
|
.map(|index| {
|
|
let mut primitive = Primitive::new_with_constructor().expect("bounded primitive");
|
|
primitive.local_id = u32::try_from(index).unwrap_or(u32::MAX);
|
|
primitive
|
|
})
|
|
.collect::<Vec<_>>();
|
|
let mut checksum = 0;
|
|
let object_count = objects.len();
|
|
for index in 0..iterations {
|
|
let object = &mut objects[index % object_count];
|
|
object.position = Vector3 {
|
|
x: index as f32,
|
|
y: (index & 255) as f32,
|
|
z: 21.0,
|
|
};
|
|
object.rotation = Quaternion::create_from_axis_angle_with_vector3_single(
|
|
Vector3::unit_z(),
|
|
(index & 255) as f32 / 255.0,
|
|
)
|
|
.map_err(debug)?;
|
|
checksum ^= u64::from(object.local_id) ^ u64::from(object.position.x.to_bits());
|
|
}
|
|
Ok(checksum)
|
|
}
|
|
|
|
fn rendering(_context: &Context, iterations: usize) -> AppResult<u64> {
|
|
let renderer = SimpleRenderer::new().map_err(debug)?;
|
|
let mut checksum = 0;
|
|
for _ in 0..iterations {
|
|
let mesh = renderer
|
|
.generate_faceted_mesh(box_primitive()?, DetailLevel::High)
|
|
.map_err(debug)?;
|
|
checksum ^= mesh
|
|
.faces
|
|
.iter()
|
|
.map(|face| face.indices.len())
|
|
.sum::<usize>() as u64;
|
|
}
|
|
Ok(checksum)
|
|
}
|
|
|
|
fn box_primitive() -> AppResult<Primitive> {
|
|
let mut primitive = Primitive::new_with_constructor().map_err(debug)?;
|
|
let mut data = PrimitiveConstructionData::new_with_constructor().map_err(debug)?;
|
|
data.p_code = PCode::Prim;
|
|
data.path_curve = PathCurve::Line;
|
|
data.set_profile_curve_with_property(ProfileCurve::Square);
|
|
data.path_scale_x = 1.0;
|
|
data.path_scale_y = 1.0;
|
|
data.path_begin = 0.0;
|
|
data.path_end = 1.0;
|
|
data.profile_begin = 0.0;
|
|
data.profile_end = 1.0;
|
|
data.profile_hollow = 0.0;
|
|
data.set_profile_hole(HoleType::Same);
|
|
data.path_revolutions = 1.0;
|
|
primitive.prim_data = data;
|
|
Ok(primitive)
|
|
}
|
|
|
|
fn client_throughput(context: &Context, iterations: usize) -> AppResult<u64> {
|
|
let _client = GridClient::new().map_err(debug)?;
|
|
let mut checksum = 0;
|
|
for index in 0..iterations {
|
|
for _ in 0..context.fixture.client_batch {
|
|
let mut source = RemoteParcelRequestReply::new().map_err(debug)?;
|
|
source.parcel_id = context.uuid;
|
|
let map = source.serialize().map_err(debug)?;
|
|
let mut target = RemoteParcelRequestReply::new().map_err(debug)?;
|
|
target.deserialize(map).map_err(debug)?;
|
|
checksum ^= target.parcel_id.get_u_long().map_err(debug)? ^ index as u64;
|
|
}
|
|
}
|
|
Ok(checksum)
|
|
}
|
|
|
|
#[derive(Debug, PartialEq, Serialize, Deserialize)]
|
|
struct ComparisonRow {
|
|
name: String,
|
|
rust_latency_ns: f64,
|
|
reference_latency_ns: f64,
|
|
latency_ratio: f64,
|
|
rust_bytes_per_operation: f64,
|
|
reference_bytes_per_operation: f64,
|
|
allocation_ratio: f64,
|
|
accepted: bool,
|
|
criterion: String,
|
|
}
|
|
|
|
#[derive(Debug, PartialEq, Serialize, Deserialize)]
|
|
struct Comparison {
|
|
schema: u32,
|
|
reference_commit: String,
|
|
fixture_hashes: BTreeMap<String, String>,
|
|
criteria: Criteria,
|
|
results: Vec<ComparisonRow>,
|
|
accepted: bool,
|
|
}
|
|
|
|
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
|
struct Criteria {
|
|
maximum_latency_ratio: f64,
|
|
maximum_allocation_ratio: f64,
|
|
material_latency_regression_ratio: f64,
|
|
minimum_reference_latency_ns: f64,
|
|
reviewed_exceptions: Vec<ReviewedException>,
|
|
}
|
|
|
|
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
|
struct ReviewedException {
|
|
workload: String,
|
|
maximum_latency_ratio: f64,
|
|
maximum_allocation_ratio: f64,
|
|
rationale: String,
|
|
}
|
|
|
|
fn criteria() -> Criteria {
|
|
serde_json::from_str(include_str!("../../../benchmarks/release-criteria.json"))
|
|
.expect("committed release criteria must be valid")
|
|
}
|
|
|
|
fn compare_reports(rust_path: &Path, reference_path: &Path, output: &Path) -> AppResult<()> {
|
|
let rust: Report = read_json(rust_path)?;
|
|
let reference: Report = read_json(reference_path)?;
|
|
validate_pair(&rust, &reference)?;
|
|
write_json(output, &build_comparison(&rust, &reference)?)
|
|
}
|
|
|
|
fn build_comparison(rust: &Report, reference: &Report) -> AppResult<Comparison> {
|
|
let policy = criteria();
|
|
let mut rows = Vec::new();
|
|
for rust_result in &rust.results {
|
|
let reference_result = reference
|
|
.results
|
|
.iter()
|
|
.find(|result| result.name == rust_result.name)
|
|
.ok_or_else(|| format!("reference report is missing {}", rust_result.name))?;
|
|
let latency_ratio =
|
|
rust_result.warm_median.latency_ns / reference_result.warm_median.latency_ns;
|
|
let rust_bytes = rust_result.warm_median.bytes_allocated as f64
|
|
/ rust_result.warm_median.iterations as f64;
|
|
let reference_bytes = reference_result.warm_median.bytes_allocated as f64
|
|
/ reference_result.warm_median.iterations as f64;
|
|
let allocation_ratio = if reference_bytes == 0.0 {
|
|
if rust_bytes == 0.0 {
|
|
1.0
|
|
} else {
|
|
f64::INFINITY
|
|
}
|
|
} else {
|
|
rust_bytes / reference_bytes
|
|
};
|
|
let exception = policy
|
|
.reviewed_exceptions
|
|
.iter()
|
|
.find(|exception| exception.workload == rust_result.name);
|
|
let maximum_latency = exception.map_or(policy.maximum_latency_ratio, |value| {
|
|
value.maximum_latency_ratio
|
|
});
|
|
let maximum_allocation = exception.map_or(policy.maximum_allocation_ratio, |value| {
|
|
value.maximum_allocation_ratio
|
|
});
|
|
let latency_noise_exempt =
|
|
reference_result.warm_median.latency_ns < policy.minimum_reference_latency_ns;
|
|
let material_latency_ok = latency_noise_exempt
|
|
|| latency_ratio <= policy.material_latency_regression_ratio
|
|
|| exception.is_some();
|
|
let latency_ok =
|
|
(latency_noise_exempt || latency_ratio <= maximum_latency) && material_latency_ok;
|
|
let allocation_ok = allocation_ratio <= maximum_allocation;
|
|
rows.push(ComparisonRow {
|
|
name: rust_result.name.clone(),
|
|
rust_latency_ns: rust_result.warm_median.latency_ns,
|
|
reference_latency_ns: reference_result.warm_median.latency_ns,
|
|
latency_ratio,
|
|
rust_bytes_per_operation: rust_bytes,
|
|
reference_bytes_per_operation: reference_bytes,
|
|
allocation_ratio,
|
|
accepted: latency_ok && allocation_ok,
|
|
criterion: if latency_ok && allocation_ok {
|
|
exception.map_or_else(
|
|
|| "within default release thresholds".into(),
|
|
|value| format!("reviewed exception: {}", value.rationale),
|
|
)
|
|
} else {
|
|
"material regression requires profiling evidence or an explicit reviewed exception"
|
|
.into()
|
|
},
|
|
});
|
|
}
|
|
let accepted = rows.iter().all(|row| row.accepted);
|
|
Ok(Comparison {
|
|
schema: TOOL_SCHEMA,
|
|
reference_commit: rust.reference_commit.clone(),
|
|
fixture_hashes: rust.fixture_hashes.clone(),
|
|
criteria: policy,
|
|
results: rows,
|
|
accepted,
|
|
})
|
|
}
|
|
|
|
fn audit(
|
|
root: &Path,
|
|
rust_path: &Path,
|
|
reference_path: &Path,
|
|
comparison_path: &Path,
|
|
) -> AppResult<()> {
|
|
verify_manifest(root)?;
|
|
let rust: Report = read_json(rust_path)?;
|
|
let reference: Report = read_json(reference_path)?;
|
|
let comparison: Comparison = read_json(comparison_path)?;
|
|
validate_pair(&rust, &reference)?;
|
|
let hashes = manifest(root)?;
|
|
if rust.fixture_hashes != hashes
|
|
|| reference.fixture_hashes != hashes
|
|
|| comparison.fixture_hashes != hashes
|
|
{
|
|
return Err("one or more reports do not match the committed fixture hashes".into());
|
|
}
|
|
let expected: Vec<_> = workloads().iter().map(|workload| workload.name).collect();
|
|
let expected_set = expected.iter().copied().collect::<BTreeSet<_>>();
|
|
for report in [&rust, &reference] {
|
|
if report.schema != TOOL_SCHEMA || report.reference_commit != EXPECTED_REFERENCE {
|
|
return Err("invalid report schema or reference commit".into());
|
|
}
|
|
let reported = report
|
|
.results
|
|
.iter()
|
|
.map(|result| result.name.as_str())
|
|
.collect::<BTreeSet<_>>();
|
|
if report.results.len() != expected.len() || reported != expected_set {
|
|
return Err(format!(
|
|
"{} report contains missing, duplicate, or unexpected workloads",
|
|
report.runtime
|
|
));
|
|
}
|
|
for name in &expected {
|
|
let result = report
|
|
.results
|
|
.iter()
|
|
.find(|result| result.name == *name)
|
|
.ok_or_else(|| format!("{} report missing {name}", report.runtime))?;
|
|
if result.cold.iterations != 1
|
|
|| result.warm_samples.len() != RUNS
|
|
|| result.warm_median.iterations == 0
|
|
{
|
|
return Err(format!("{} has incomplete cold/warm evidence", result.name));
|
|
}
|
|
}
|
|
}
|
|
let recomputed = build_comparison(&rust, &reference)?;
|
|
let checked_bytes = fs::read(comparison_path)
|
|
.map_err(|error| format!("read {}: {error}", comparison_path.display()))?;
|
|
if checked_bytes != json_bytes(&recomputed)? {
|
|
return Err("release comparison is stale; rerun the compare command".into());
|
|
}
|
|
if !comparison.accepted {
|
|
return Err("release comparison is incomplete or contains an unaccepted regression".into());
|
|
}
|
|
println!(
|
|
"performance audit passed: {} workloads, pinned reference {}, fixture hashes verified",
|
|
expected.len(),
|
|
EXPECTED_REFERENCE
|
|
);
|
|
Ok(())
|
|
}
|
|
|
|
fn validate_pair(rust: &Report, reference: &Report) -> AppResult<()> {
|
|
if rust.runtime != "rust-native" || reference.runtime != "dotnet-reference" {
|
|
return Err(
|
|
"reports must be generated independently by rust-native and dotnet-reference".into(),
|
|
);
|
|
}
|
|
if rust.reference_commit != EXPECTED_REFERENCE
|
|
|| reference.reference_commit != EXPECTED_REFERENCE
|
|
|| rust.fixture_hashes != reference.fixture_hashes
|
|
{
|
|
return Err("reports use different fixtures or reference commits".into());
|
|
}
|
|
if rust.environment.os != reference.environment.os
|
|
|| rust.environment.arch != reference.environment.arch
|
|
{
|
|
return Err("reports must be captured on the same OS and architecture".into());
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
fn verify_manifest(root: &Path) -> AppResult<()> {
|
|
let expected = manifest(root)?;
|
|
for (name, hash) in expected {
|
|
let actual = hash_file(&root.join(&name))?;
|
|
if actual != hash {
|
|
return Err(format!("fixture hash mismatch for {name}"));
|
|
}
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
fn manifest(root: &Path) -> AppResult<BTreeMap<String, String>> {
|
|
read_json(&root.join("manifest.json"))
|
|
}
|
|
|
|
fn hash_file(path: &Path) -> AppResult<String> {
|
|
let bytes = fs::read(path).map_err(|error| format!("read {}: {error}", path.display()))?;
|
|
Ok(Sha256::digest(bytes)
|
|
.iter()
|
|
.fold(String::with_capacity(64), |mut output, byte| {
|
|
write!(output, "{byte:02x}").expect("writing to a String is infallible");
|
|
output
|
|
}))
|
|
}
|
|
|
|
fn read_json<T: for<'de> Deserialize<'de>>(path: &Path) -> AppResult<T> {
|
|
serde_json::from_slice(
|
|
&fs::read(path).map_err(|error| format!("read {}: {error}", path.display()))?,
|
|
)
|
|
.map_err(|error| format!("parse {}: {error}", path.display()))
|
|
}
|
|
|
|
fn write_json<T: Serialize>(path: &Path, value: &T) -> AppResult<()> {
|
|
if let Some(parent) = path.parent() {
|
|
fs::create_dir_all(parent).map_err(|error| error.to_string())?;
|
|
}
|
|
fs::write(path, json_bytes(value)?)
|
|
.map_err(|error| format!("write {}: {error}", path.display()))
|
|
}
|
|
|
|
fn json_bytes<T: Serialize>(value: &T) -> AppResult<Vec<u8>> {
|
|
let mut bytes = serde_json::to_vec_pretty(value).map_err(|error| error.to_string())?;
|
|
bytes.push(b'\n');
|
|
Ok(bytes)
|
|
}
|
|
|
|
fn command_version(program: &str, args: &[&str]) -> String {
|
|
std::process::Command::new(program)
|
|
.args(args)
|
|
.output()
|
|
.ok()
|
|
.filter(|output| output.status.success())
|
|
.map_or_else(
|
|
|| "unavailable".into(),
|
|
|output| String::from_utf8_lossy(&output.stdout).trim().into(),
|
|
)
|
|
}
|
|
|
|
fn debug(error: impl std::fmt::Debug) -> String {
|
|
format!("{error:?}")
|
|
}
|
|
|
|
#[cfg(test)]
|
|
#[allow(clippy::float_cmp)] // Metrics use exact integer-derived values in these tests.
|
|
mod tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn workload_set_covers_every_issue_category() {
|
|
let categories: Vec<_> = workloads()
|
|
.iter()
|
|
.map(|workload| workload.category)
|
|
.collect();
|
|
for expected in [
|
|
"uuid/math",
|
|
"LLSD",
|
|
"packet codec",
|
|
"asset decode",
|
|
"image decode",
|
|
"mesh decode",
|
|
"inventory update",
|
|
"object update",
|
|
"rendering",
|
|
"client throughput",
|
|
] {
|
|
assert!(categories.contains(&expected), "missing {expected}");
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn metrics_separate_latency_throughput_allocations_and_retained_memory() {
|
|
let result = metrics(
|
|
10,
|
|
1_000,
|
|
Stats {
|
|
allocations: 2,
|
|
deallocations: 1,
|
|
reallocations: 1,
|
|
bytes_allocated: 100,
|
|
bytes_deallocated: 40,
|
|
bytes_reallocated: 20,
|
|
},
|
|
);
|
|
assert_eq!(result.latency_ns, 100.0);
|
|
assert_eq!(result.throughput_per_second, 10_000_000.0);
|
|
assert_eq!(result.allocations, Some(3));
|
|
assert_eq!(result.retained_bytes, 80);
|
|
}
|
|
|
|
#[test]
|
|
fn release_criteria_identify_material_regressions() {
|
|
let policy = criteria();
|
|
assert_eq!(policy.maximum_latency_ratio, 2.0);
|
|
assert!(policy.material_latency_regression_ratio < policy.maximum_latency_ratio);
|
|
assert_eq!(policy.reviewed_exceptions.len(), 3);
|
|
assert!(
|
|
policy
|
|
.reviewed_exceptions
|
|
.iter()
|
|
.any(|exception| exception.workload == "client_throughput")
|
|
);
|
|
}
|
|
}
|