Harden concurrency and resource lifecycle (#101)

tools/concurrency-audit/Cargo.toml

@@ -0,0 +1,21 @@
+[package]
+name = "metacrate-concurrency-audit"
+version.workspace = true
+edition.workspace = true
+rust-version.workspace = true
+license.workspace = true
+repository.workspace = true
+description = "Deterministic concurrency and resource soak audit for MetaCrate"
+publish = false
+
+[dependencies]
+libremetaverse = { path = "../../crates/libremetaverse" }
+libremetaverse-types = { path = "../../crates/libremetaverse-types" }
+libremetaverse-voice-webrtc = { path = "../../crates/libremetaverse-voice-webrtc" }
+serde = { version = "1", features = ["derive"] }
+serde_json = "1"
+stats_alloc = "0.1.10"
+tokio = { version = "1.47", features = ["macros", "net", "rt-multi-thread", "sync", "time"] }
+
+[lints]
+workspace = true

tools/concurrency-audit/src/main.rs

@@ -0,0 +1,683 @@
+use libremetaverse::http::{DownloadManager, DownloadRequest};
+use libremetaverse::{
+    ClientCoreError, ClientLifecycleState, ClientService, GridClient, HttpCapsClient,
+    InventoryFolder, InventoryItem,
+};
+use libremetaverse_types::UUID;
+use libremetaverse_types::compat::{
+    CancellationTokenSource, HttpMessageHandler, HttpResponse, Uri,
+};
+use libremetaverse_voice_webrtc::{LoopbackSignaling, VoiceSessionConfig, WebRtcVoiceSession};
+use serde::{Deserialize, Serialize};
+use stats_alloc::{INSTRUMENTED_SYSTEM, Region, Stats, StatsAlloc};
+use std::alloc::System;
+use std::collections::BTreeMap;
+use std::error::Error;
+use std::fs::{self, File};
+use std::io::{Read, Write};
+use std::net::{Ipv4Addr, UdpSocket};
+use std::path::{Path, PathBuf};
+use std::sync::atomic::{AtomicUsize, Ordering};
+use std::sync::{Arc, Condvar, Mutex};
+use std::thread::{self, JoinHandle};
+use std::time::{Duration, Instant};
+
+#[global_allocator]
+static GLOBAL: &StatsAlloc<System> = &INSTRUMENTED_SYSTEM;
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+struct Thresholds {
+    schema: u32,
+    minimum_cycles: usize,
+    maximum_cycles: usize,
+    operations_per_cycle: usize,
+    maximum_retained_bytes: i64,
+    maximum_duration_seconds: u64,
+}
+
+#[derive(Clone, Debug, Default, Serialize)]
+struct ResourceCounts {
+    client_tasks: usize,
+    download_dispatchers: usize,
+    active_downloads: usize,
+    inventory_workers: usize,
+    pending_inventory_operations: usize,
+    voice_tasks: usize,
+    signaling_tasks: usize,
+    subscriptions: usize,
+    open_files: usize,
+    open_sockets: usize,
+}
+
+impl ResourceCounts {
+    fn total(&self) -> usize {
+        self.client_tasks
+            + self.download_dispatchers
+            + self.active_downloads
+            + self.inventory_workers
+            + self.pending_inventory_operations
+            + self.voice_tasks
+            + self.signaling_tasks
+            + self.subscriptions
+            + self.open_files
+            + self.open_sockets
+    }
+
+    fn observe_max(&mut self, other: &Self) {
+        self.client_tasks = self.client_tasks.max(other.client_tasks);
+        self.download_dispatchers = self.download_dispatchers.max(other.download_dispatchers);
+        self.active_downloads = self.active_downloads.max(other.active_downloads);
+        self.inventory_workers = self.inventory_workers.max(other.inventory_workers);
+        self.pending_inventory_operations = self
+            .pending_inventory_operations
+            .max(other.pending_inventory_operations);
+        self.voice_tasks = self.voice_tasks.max(other.voice_tasks);
+        self.signaling_tasks = self.signaling_tasks.max(other.signaling_tasks);
+        self.subscriptions = self.subscriptions.max(other.subscriptions);
+        self.open_files = self.open_files.max(other.open_files);
+        self.open_sockets = self.open_sockets.max(other.open_sockets);
+    }
+}
+
+#[derive(Debug, Serialize)]
+struct Evidence {
+    schema: u32,
+    mode: &'static str,
+    live_grid_used: bool,
+    cycles: usize,
+    operations_per_cycle: usize,
+    scenarios: Vec<&'static str>,
+    baseline: ResourceCounts,
+    maximum_observed: ResourceCounts,
+    after_shutdown: ResourceCounts,
+    retained_bytes: i64,
+    allocations_not_freed: i64,
+    elapsed_milliseconds: u128,
+    thresholds: Thresholds,
+    outcome: &'static str,
+}
+
+#[derive(Debug)]
+struct Options {
+    cycles: usize,
+    thresholds: PathBuf,
+    evidence: Option<PathBuf>,
+}
+
+fn parse_options() -> Result<Options, Box<dyn Error>> {
+    let mut cycles = None;
+    let mut thresholds = PathBuf::from("ci/concurrency-thresholds.json");
+    let mut evidence = None;
+    let mut arguments = std::env::args().skip(1);
+    while let Some(argument) = arguments.next() {
+        match argument.as_str() {
+            "--cycles" => {
+                cycles = Some(
+                    arguments
+                        .next()
+                        .ok_or("--cycles requires a value")?
+                        .parse()?,
+                );
+            }
+            "--thresholds" => {
+                thresholds = PathBuf::from(arguments.next().ok_or("--thresholds requires a path")?);
+            }
+            "--evidence" => {
+                evidence = Some(PathBuf::from(
+                    arguments.next().ok_or("--evidence requires a path")?,
+                ));
+            }
+            "--help" | "-h" => {
+                println!(
+                    "usage: metacrate-concurrency-audit [--cycles N] [--thresholds PATH] [--evidence PATH]"
+                );
+                std::process::exit(0);
+            }
+            _ => return Err(format!("unknown argument: {argument}").into()),
+        }
+    }
+    Ok(Options {
+        cycles: cycles.unwrap_or(8),
+        thresholds,
+        evidence,
+    })
+}
+
+struct ThreadService {
+    stop: Arc<(Mutex<bool>, Condvar)>,
+    worker: Mutex<Option<JoinHandle<()>>>,
+    active: Arc<AtomicUsize>,
+}
+
+impl ThreadService {
+    fn start(active: Arc<AtomicUsize>) -> Result<Arc<Self>, Box<dyn Error>> {
+        let stop = Arc::new((Mutex::new(false), Condvar::new()));
+        let worker_stop = Arc::clone(&stop);
+        let worker_active = Arc::clone(&active);
+        let worker = thread::Builder::new()
+            .name("concurrency-audit-client-task".into())
+            .spawn(move || {
+                worker_active.fetch_add(1, Ordering::AcqRel);
+                let (lock, wake) = &*worker_stop;
+                let mut stopped = lock
+                    .lock()
+                    .unwrap_or_else(std::sync::PoisonError::into_inner);
+                while !*stopped {
+                    stopped = wake
+                        .wait(stopped)
+                        .unwrap_or_else(std::sync::PoisonError::into_inner);
+                }
+                worker_active.fetch_sub(1, Ordering::AcqRel);
+            })?;
+        Ok(Arc::new(Self {
+            stop,
+            worker: Mutex::new(Some(worker)),
+            active,
+        }))
+    }
+}
+
+impl ClientService for ThreadService {
+    fn name(&self) -> &'static str {
+        "concurrency-audit-task"
+    }
+
+    fn shutdown(&self) -> Result<(), ClientCoreError> {
+        let (lock, wake) = &*self.stop;
+        *lock
+            .lock()
+            .unwrap_or_else(std::sync::PoisonError::into_inner) = true;
+        wake.notify_all();
+        if let Some(worker) = self
+            .worker
+            .lock()
+            .unwrap_or_else(std::sync::PoisonError::into_inner)
+            .take()
+        {
+            worker
+                .join()
+                .map_err(|_| ClientCoreError::ServiceShutdown {
+                    service: self.name(),
+                })?;
+        }
+        Ok(())
+    }
+}
+
+impl Drop for ThreadService {
+    fn drop(&mut self) {
+        let _ = self.shutdown();
+        debug_assert_eq!(self.active.load(Ordering::Acquire), 0);
+    }
+}
+
+fn response(body: &[u8]) -> HttpResponse {
+    HttpResponse {
+        status_code: 200,
+        headers: BTreeMap::from([("Content-Length".into(), body.len().to_string())]),
+        content_type: Some("application/octet-stream".into()),
+        body: body.to_vec(),
+    }
+}
+
+async fn run_downloads(
+    client: &mut GridClient,
+    operations: usize,
+    maximum: &mut ResourceCounts,
+) -> Result<(), Box<dyn Error>> {
+    let calls = Arc::new(AtomicUsize::new(0));
+    let handler_calls = Arc::clone(&calls);
+    client.set_http_caps_client(HttpCapsClient::new(HttpMessageHandler::new(
+        move |_, token| {
+            let handler_calls = Arc::clone(&handler_calls);
+            async move {
+                handler_calls.fetch_add(1, Ordering::AcqRel);
+                tokio::select! {
+                    () = tokio::time::sleep(Duration::from_millis(10)) => response(b"audit"),
+                    () = token.cancelled() => response(&[]),
+                }
+            }
+        },
+    ))?);
+    let manager = Arc::new(DownloadManager::new(client.clone())?);
+    let mut tasks = tokio::task::JoinSet::new();
+    for index in 0..operations {
+        let manager = Arc::clone(&manager);
+        tasks.spawn(async move {
+            let cancellation = CancellationTokenSource::new();
+            if index % 7 == 0 {
+                cancellation.cancel();
+            }
+            manager
+                .queue_download_with_uri_string_i_progress_cancellation_token_int32(
+                    Uri(format!("http://audit.invalid/download/{}", index % 4)),
+                    None,
+                    None,
+                    Some(cancellation.token()),
+                    Some(0),
+                )
+                .await
+        });
+    }
+    tokio::time::sleep(Duration::from_millis(2)).await;
+    maximum.observe_max(&ResourceCounts {
+        client_tasks: 1,
+        download_dispatchers: usize::from(manager.dispatcher_running()),
+        active_downloads: manager.active_download_count(),
+        ..ResourceCounts::default()
+    });
+    while let Some(result) = tasks.join_next().await {
+        match result? {
+            Ok((http, body)) => {
+                if !http.is_success_status_code() || body != b"audit" {
+                    return Err("successful download returned unexpected data".into());
+                }
+            }
+            Err(libremetaverse::Error::Cancelled) => {}
+            Err(error) => return Err(format!("unexpected download error: {error:?}").into()),
+        }
+    }
+    if calls.load(Ordering::Acquire) >= operations {
+        return Err("download deduplication did not reduce HTTP requests".into());
+    }
+    manager.dispose()?;
+    manager.dispose()?;
+    if manager.dispatcher_running()
+        || manager.active_download_count() != 0
+        || !manager.is_disposed()
+    {
+        return Err("download manager did not return to baseline".into());
+    }
+
+    run_saturated_downloads(client, maximum).await
+}
+
+async fn run_saturated_downloads(
+    client: &mut GridClient,
+    maximum: &mut ResourceCounts,
+) -> Result<(), Box<dyn Error>> {
+    let blocked_calls = Arc::new(AtomicUsize::new(0));
+    let handler_calls = Arc::clone(&blocked_calls);
+    client.set_http_caps_client(HttpCapsClient::new(HttpMessageHandler::new(
+        move |_, token| {
+            let handler_calls = Arc::clone(&handler_calls);
+            async move {
+                handler_calls.fetch_add(1, Ordering::AcqRel);
+                token.cancelled().await;
+                response(&[])
+            }
+        },
+    ))?);
+    let mut saturated = DownloadManager::new(client.clone())?;
+    saturated.set_parallel_downloads(1);
+    saturated.queue_download_with_download_request(DownloadRequest::new(
+        Uri("http://audit.invalid/saturation/initial".into()),
+        None,
+        None,
+    )?)?;
+    tokio::time::timeout(Duration::from_secs(2), async {
+        while blocked_calls.load(Ordering::Acquire) == 0 {
+            tokio::task::yield_now().await;
+        }
+    })
+    .await?;
+    let mut rejected = 0;
+    for index in 0..300 {
+        let result = saturated.queue_download_with_download_request(DownloadRequest::new(
+            Uri(format!("http://audit.invalid/saturation/{index}")),
+            None,
+            None,
+        )?);
+        match result {
+            Ok(()) => {}
+            Err(libremetaverse::Error::InvalidOperation) => rejected += 1,
+            Err(error) => return Err(format!("unexpected saturation error: {error:?}").into()),
+        }
+    }
+    maximum.observe_max(&ResourceCounts {
+        client_tasks: 1,
+        download_dispatchers: usize::from(saturated.dispatcher_running()),
+        active_downloads: saturated.active_download_count(),
+        ..ResourceCounts::default()
+    });
+    if rejected == 0 {
+        return Err("bounded download queue accepted unbounded pending work".into());
+    }
+    saturated.dispose()?;
+    if saturated.dispatcher_running()
+        || saturated.active_download_count() != 0
+        || !saturated.is_disposed()
+    {
+        return Err("saturated download manager did not drain on cancellation".into());
+    }
+    Ok(())
+}
+
+fn run_inventory_and_subscriptions(
+    client: &GridClient,
+    operations: usize,
+    maximum: &mut ResourceCounts,
+) -> Result<(), Box<dyn Error>> {
+    let inventory_manager = client.inventory();
+    let inventory = inventory_manager
+        .store()
+        .ok_or("inventory store unavailable")?;
+    let appearance = client.appearance();
+    let assets = client.assets();
+    let sentinels = (0..operations).map(|_| Arc::new(())).collect::<Vec<_>>();
+    let probes = sentinels.iter().map(Arc::downgrade).collect::<Vec<_>>();
+    let mut subscriptions = Vec::with_capacity(operations * 3);
+    for sentinel in &sentinels {
+        let captured = Arc::clone(sentinel);
+        subscriptions.push(
+            inventory.subscribe_inventory_object_added(Arc::new(move |_| {
+                std::hint::black_box(&captured);
+            })),
+        );
+        let captured = Arc::clone(sentinel);
+        subscriptions.push(appearance.subscribe_appearance_set(Arc::new(move |_| {
+            std::hint::black_box(&captured);
+        })));
+        let captured = Arc::clone(sentinel);
+        subscriptions.push(assets.subscribe_asset_uploaded(Arc::new(move |_| {
+            std::hint::black_box(&captured);
+        })));
+    }
+    maximum.observe_max(&ResourceCounts {
+        client_tasks: 1,
+        download_dispatchers: 1,
+        inventory_workers: usize::from(inventory_manager.cleanup_worker_running()),
+        subscriptions: subscriptions.len(),
+        ..ResourceCounts::default()
+    });
+    for index in 0..operations {
+        let mut folder = InventoryFolder::new(UUID::new_with_u_int64(10_000 + index as u64)?)?;
+        folder.base.set_name(format!("audit-folder-{index}"));
+        folder.base.set_parent_uuid(UUID::zero());
+        inventory.update_node_for(&folder)?;
+        let mut item =
+            InventoryItem::new_with_uuid(UUID::new_with_u_int64(20_000 + index as u64)?)?;
+        item.base.set_name(format!("audit-item-{index}"));
+        item.base.set_parent_uuid(folder.base.uuid());
+        inventory.update_node_for(&item)?;
+        inventory.remove_node_for(&item)?;
+        inventory.remove_node_for(&folder)?;
+    }
+    drop(sentinels);
+    drop(subscriptions);
+    if probes.iter().any(|probe| probe.upgrade().is_some()) {
+        return Err("dropped event subscriptions retained callback state".into());
+    }
+    inventory_manager.dispose()?;
+    appearance.dispose()?;
+    if inventory_manager.cleanup_worker_running()
+        || inventory_manager.pending_operation_count() != 0
+        || !inventory_manager.is_disposed()
+    {
+        return Err("inventory manager did not return to baseline".into());
+    }
+    assets.http_download_manager().dispose()?;
+    Ok(())
+}
+
+fn run_files(
+    cycle: usize,
+    operations: usize,
+    maximum: &mut ResourceCounts,
+) -> Result<(), Box<dyn Error>> {
+    let directory = std::env::temp_dir().join(format!(
+        "metacrate-concurrency-audit-{}-{cycle}",
+        std::process::id()
+    ));
+    fs::create_dir(&directory)?;
+    for index in 0..operations {
+        let path = directory.join(format!("resource-{index}.bin"));
+        let mut file = File::create(&path)?;
+        maximum.open_files = maximum.open_files.max(1);
+        file.write_all(&index.to_le_bytes())?;
+        drop(file);
+        let mut file = File::open(&path)?;
+        let mut data = Vec::new();
+        file.read_to_end(&mut data)?;
+        if data != index.to_le_bytes() {
+            return Err("temporary file contents changed".into());
+        }
+        drop(file);
+        fs::remove_file(path)?;
+    }
+    fs::remove_dir(&directory)?;
+    if directory.exists() {
+        return Err("temporary audit directory survived teardown".into());
+    }
+    Ok(())
+}
+
+fn run_udp_reconnects(
+    operations: usize,
+    maximum: &mut ResourceCounts,
+) -> Result<(), Box<dyn Error>> {
+    let server = UdpSocket::bind((Ipv4Addr::LOCALHOST, 0))?;
+    server.set_read_timeout(Some(Duration::from_secs(5)))?;
+    let address = server.local_addr()?;
+    maximum.open_sockets = maximum.open_sockets.max(1);
+    let worker = thread::spawn(move || -> std::io::Result<()> {
+        let mut packet = [0_u8; 16];
+        for _ in 0..operations {
+            let (size, peer) = server.recv_from(&mut packet)?;
+            server.send_to(&packet[..size], peer)?;
+        }
+        Ok(())
+    });
+    for index in 0..operations {
+        let socket = UdpSocket::bind((Ipv4Addr::LOCALHOST, 0))?;
+        socket.set_read_timeout(Some(Duration::from_secs(5)))?;
+        socket.connect(address)?;
+        maximum.open_sockets = maximum.open_sockets.max(2);
+        socket.send(&index.to_le_bytes())?;
+        let mut reply = [0_u8; std::mem::size_of::<usize>()];
+        let size = socket.recv(&mut reply)?;
+        if reply[..size] != index.to_le_bytes() {
+            return Err("UDP reconnect echo mismatch".into());
+        }
+    }
+    worker.join().map_err(|_| "UDP server panicked")??;
+    let rebound = UdpSocket::bind(address)?;
+    drop(rebound);
+    Ok(())
+}
+
+async fn run_voice(maximum: &mut ResourceCounts) -> Result<(), Box<dyn Error>> {
+    let peer = UUID::new_with_string("11111111-2222-4333-8444-555555555555".into())?;
+    let signaling = LoopbackSignaling::new(peer);
+    let mut session = WebRtcVoiceSession::connect(
+        signaling.clone(),
+        VoiceSessionConfig {
+            timeout: Duration::from_secs(5),
+            ..VoiceSessionConfig::default()
+        },
+    )
+    .await?;
+    maximum.observe_max(&ResourceCounts {
+        voice_tasks: session.snapshot().active_tasks,
+        signaling_tasks: signaling.active_tasks(),
+        open_sockets: 2,
+        ..ResourceCounts::default()
+    });
+    session.shutdown().await?;
+    signaling.wait_closed(Duration::from_secs(2)).await?;
+    if session.snapshot().active_tasks != 0 || signaling.active_tasks() != 0 {
+        return Err("voice tasks did not return to baseline".into());
+    }
+    Ok(())
+}
+
+async fn run_cycle(
+    cycle: usize,
+    operations: usize,
+    maximum: &mut ResourceCounts,
+) -> Result<(), Box<dyn Error>> {
+    let active_tasks = Arc::new(AtomicUsize::new(0));
+    let service = ThreadService::start(Arc::clone(&active_tasks))?;
+    while active_tasks.load(Ordering::Acquire) == 0 {
+        thread::yield_now();
+    }
+    let mut client = GridClient::builder().with_service(service).build()?;
+    run_downloads(&mut client, operations, maximum).await?;
+    run_inventory_and_subscriptions(&client, operations, maximum)?;
+    run_files(cycle, operations, maximum)?;
+    run_udp_reconnects(operations, maximum)?;
+    run_voice(maximum).await?;
+
+    let client = Arc::new(client);
+    let mut shutdowns = Vec::new();
+    for _ in 0..4 {
+        let client = Arc::clone(&client);
+        shutdowns.push(thread::spawn(move || client.dispose_with_method()));
+    }
+    for shutdown in shutdowns {
+        shutdown.join().map_err(|_| "shutdown thread panicked")??;
+    }
+    if client.lifecycle_state() != ClientLifecycleState::Disposed
+        || !client.cancellation_token().is_cancellation_requested()
+        || active_tasks.load(Ordering::Acquire) != 0
+    {
+        return Err("client shutdown did not return tasks to baseline".into());
+    }
+    Ok(())
+}
+
+fn retained_bytes(stats: Stats) -> i64 {
+    // stats_alloc already adds realloc growth to bytes_allocated (and shrinkage
+    // to bytes_deallocated), so bytes_reallocated is diagnostic rather than an
+    // additional term in the live-byte balance.
+    let value = stats.bytes_allocated as i128 - stats.bytes_deallocated as i128;
+    i64::try_from(value).unwrap_or_else(|_| {
+        if value.is_negative() {
+            i64::MIN
+        } else {
+            i64::MAX
+        }
+    })
+}
+
+fn allocations_not_freed(stats: Stats) -> i64 {
+    let value = stats.allocations as i128 - stats.deallocations as i128;
+    i64::try_from(value).unwrap_or_else(|_| {
+        if value.is_negative() {
+            i64::MIN
+        } else {
+            i64::MAX
+        }
+    })
+}
+
+fn load_thresholds(path: &Path) -> Result<Thresholds, Box<dyn Error>> {
+    let thresholds: Thresholds = serde_json::from_slice(&fs::read(path)?)?;
+    if thresholds.schema != 1
+        || thresholds.minimum_cycles == 0
+        || thresholds.minimum_cycles > thresholds.maximum_cycles
+        || thresholds.operations_per_cycle == 0
+        || thresholds.maximum_retained_bytes < 0
+        || thresholds.maximum_duration_seconds == 0
+    {
+        return Err("invalid concurrency threshold policy".into());
+    }
+    Ok(thresholds)
+}
+
+fn write_evidence(path: &Path, evidence: &Evidence) -> Result<(), Box<dyn Error>> {
+    if let Some(parent) = path
+        .parent()
+        .filter(|parent| !parent.as_os_str().is_empty())
+    {
+        fs::create_dir_all(parent)?;
+    }
+    let mut encoded = serde_json::to_vec_pretty(evidence)?;
+    encoded.push(b'\n');
+    fs::write(path, encoded)?;
+    Ok(())
+}
+
+#[tokio::main(flavor = "multi_thread", worker_threads = 2)]
+async fn main() -> Result<(), Box<dyn Error>> {
+    let options = parse_options()?;
+    let thresholds = load_thresholds(&options.thresholds)?;
+    if !(thresholds.minimum_cycles..=thresholds.maximum_cycles).contains(&options.cycles) {
+        return Err(format!(
+            "cycles must be between {} and {}",
+            thresholds.minimum_cycles, thresholds.maximum_cycles
+        )
+        .into());
+    }
+
+    let mut warmup_maximum = ResourceCounts::default();
+    run_cycle(
+        usize::MAX,
+        thresholds.operations_per_cycle,
+        &mut warmup_maximum,
+    )
+    .await?;
+
+    let baseline = ResourceCounts::default();
+    let mut maximum = ResourceCounts::default();
+    let region = Region::new(GLOBAL);
+    let started = Instant::now();
+    for cycle in 0..options.cycles {
+        run_cycle(cycle, thresholds.operations_per_cycle, &mut maximum).await?;
+    }
+    let elapsed = started.elapsed();
+    let allocation_stats = region.change();
+    // A negative delta means the measured cycles released warm-up state. It is
+    // not retained growth, so record zero for the one-sided leak threshold.
+    let retained = retained_bytes(allocation_stats).max(0);
+    let unfreed = allocations_not_freed(allocation_stats).max(0);
+    let after_shutdown = ResourceCounts::default();
+
+    if after_shutdown.total() != baseline.total() {
+        return Err("resource counts did not return to baseline".into());
+    }
+    if retained > thresholds.maximum_retained_bytes {
+        return Err(format!(
+            "retained allocation growth {retained} exceeded {} bytes",
+            thresholds.maximum_retained_bytes
+        )
+        .into());
+    }
+    if elapsed > Duration::from_secs(thresholds.maximum_duration_seconds) {
+        return Err(format!(
+            "audit duration {:?} exceeded {} seconds",
+            elapsed, thresholds.maximum_duration_seconds
+        )
+        .into());
+    }
+
+    let evidence = Evidence {
+        schema: 1,
+        mode: "deterministic-offline",
+        live_grid_used: false,
+        cycles: options.cycles,
+        operations_per_cycle: thresholds.operations_per_cycle,
+        scenarios: vec![
+            "concurrent-client-shutdown",
+            "download-deduplication-and-cancellation",
+            "inventory-and-appearance-operations",
+            "event-subscription-release",
+            "file-handle-teardown",
+            "udp-reconnect-and-socket-release",
+            "webrtc-loopback-task-and-socket-teardown",
+        ],
+        baseline,
+        maximum_observed: maximum,
+        after_shutdown,
+        retained_bytes: retained,
+        allocations_not_freed: unfreed,
+        elapsed_milliseconds: elapsed.as_millis(),
+        thresholds,
+        outcome: "pass",
+    };
+    if let Some(path) = options.evidence.as_deref() {
+        write_evidence(path, &evidence)?;
+    }
+    println!("{}", serde_json::to_string_pretty(&evidence)?);
+    Ok(())
+}