Complete first release candidate audit (#107)

tools/ci-matrix/src/api_surface.rs

@@ -104,7 +104,7 @@ const REQUIRED_THREAD_SAFE_TRAITS: [(&str, &str); 6] = [
         "pub trait IRlvQueryCallbacks: Send + Sync",
     ),
     (
-        "crates/libremetaverse-voice-webrtc/src/generated.rs",
+        "crates/libremetaverse-voice-webrtc/src/compatibility.rs",
         "pub trait IVoiceLogger: Send + Sync",
     ),
 ];

tools/ci-matrix/src/lib.rs

@@ -16,12 +16,14 @@ mod artifact;
 mod dependency;
 mod documentation;
 mod provenance;
+mod release_candidate;
 
 pub use api_surface::{audit_api_surface, write_api_baseline};
 pub use artifact::audit_artifacts;
 pub use dependency::audit_dependencies;
 pub use documentation::{audit_documentation, write_documentation_report};
 pub use provenance::{audit_provenance, write_provenance_reports};
+pub use release_candidate::audit_release_candidate;
 
 pub const MATRIX_PATH: &str = "ci/release-matrix.json";
 const WORKFLOW_PATH: &str = ".gitea/workflows/release-matrix.yml";

tools/ci-matrix/src/main.rs

@@ -1,7 +1,7 @@
 use metacrate_ci_matrix::{
     audit, audit_api_surface, audit_artifacts, audit_dependencies, audit_documentation,
-    audit_provenance, load, run, workspace_root, write_api_baseline, write_documentation_report,
-    write_provenance_reports,
+    audit_provenance, audit_release_candidate, load, run, workspace_root, write_api_baseline,
+    write_documentation_report, write_provenance_reports,
 };
 use std::path::{Path, PathBuf};
 
@@ -53,6 +53,9 @@ fn execute() -> Result<(), Box<dyn std::error::Error>> {
         Some("artifact-audit") => {
             artifact_audit_command(&root, arguments)?;
         }
+        Some("release-candidate-audit") => {
+            release_candidate_audit_command(&root, arguments)?;
+        }
         Some("documentation-report") if arguments.next().is_none() => {
             write_documentation_report(&root)?;
             println!("documentation coverage report: updated");
@@ -98,7 +101,7 @@ fn execute() -> Result<(), Box<dyn std::error::Error>> {
         }
         _ => {
             return Err(
-                "usage: ci-matrix audit | run PROFILE --evidence FILE | dependency-audit --evidence FILE | artifact-audit --artifact-dir DIR --package-dir DIR --evidence FILE | documentation-report | documentation-audit --evidence FILE | api-baseline-write | api-audit --evidence FILE | provenance-report | provenance-audit --evidence FILE"
+                "usage: ci-matrix audit | run PROFILE --evidence FILE | dependency-audit --evidence FILE | artifact-audit --artifact-dir DIR --package-dir DIR --evidence FILE | release-candidate-audit --evidence FILE | documentation-report | documentation-audit --evidence FILE | api-baseline-write | api-audit --evidence FILE | provenance-report | provenance-audit --evidence FILE"
                     .into(),
             );
         }
@@ -106,6 +109,25 @@ fn execute() -> Result<(), Box<dyn std::error::Error>> {
     Ok(())
 }
 
+fn release_candidate_audit_command(
+    root: &Path,
+    mut arguments: impl Iterator<Item = String>,
+) -> Result<(), Box<dyn std::error::Error>> {
+    let flag = arguments
+        .next()
+        .ok_or("release-candidate-audit requires --evidence FILE")?;
+    let evidence = arguments
+        .next()
+        .ok_or("release-candidate-audit requires --evidence FILE")?;
+    if flag != "--evidence" || arguments.next().is_some() {
+        return Err("usage: ci-matrix release-candidate-audit --evidence FILE".into());
+    }
+    let evidence = absolute_or_rooted(root, &evidence);
+    audit_release_candidate(root, &evidence)?;
+    println!("first release candidate: ok ({})", evidence.display());
+    Ok(())
+}
+
 fn artifact_audit_command(
     root: &Path,
     mut arguments: impl Iterator<Item = String>,

tools/ci-matrix/src/release_candidate.rs

@@ -0,0 +1,710 @@
+//! Fail-closed first-release-candidate completeness and evidence audit.
+
+#![allow(clippy::too_many_lines)] // The gate records every invariant in one ordered audit.
+
+use super::{MatrixError, Result};
+use serde::{Deserialize, Serialize};
+use serde_json::Value;
+use sha2::{Digest, Sha256};
+use std::collections::{BTreeMap, BTreeSet};
+use std::ffi::OsStr;
+use std::fs::{self, OpenOptions};
+use std::io::Write as _;
+use std::path::{Path, PathBuf};
+use std::process::Command;
+use std::time::{SystemTime, UNIX_EPOCH};
+
+const POLICY_PATH: &str = "ci/release-candidate-policy.json";
+const PARITY_PATH: &str = "tests/upstream-tests.json";
+const ARTIFACT_POLICY_PATH: &str = "ci/artifact-policy.json";
+const FORBIDDEN_LIVE_PHRASES: [&str; 4] = [
+    "return;",
+    "skipping assertion",
+    "transient server issue",
+    "proceeding with tests",
+];
+const SMOKE_STAGES: [&str; 9] = [
+    "completeness",
+    "login",
+    "capabilities-simulator",
+    "im-chat",
+    "movement-teleport",
+    "inventory-folder",
+    "object-properties",
+    "asset-texture",
+    "logout",
+];
+
+#[derive(Deserialize)]
+struct Policy {
+    schema: u32,
+    release: String,
+    current_rust_prefix: String,
+    upstream_commit: String,
+    expected: Expected,
+    required_evidence: Vec<EvidenceRequirement>,
+    live_test_files: Vec<String>,
+    fake_grid_evidence: String,
+    live_grid_evidence: String,
+    release_notes: String,
+    changelog: String,
+    checksums: String,
+    signature_required: bool,
+    #[serde(rename = "signature_policy")]
+    authenticity_description: String,
+    deferred_by_user: Vec<String>,
+}
+
+#[derive(Deserialize)]
+struct Expected {
+    release_packages: usize,
+    shipped_binaries: usize,
+    mapped_public_types: u64,
+    mapped_public_members: u64,
+    parity_cases: u64,
+    translated_cases: u64,
+    live_cases: u64,
+    benchmark_cases: u64,
+    live_smoke_records: usize,
+}
+
+#[derive(Deserialize)]
+struct EvidenceRequirement {
+    id: String,
+    path: String,
+    assertions: Vec<JsonAssertion>,
+}
+
+#[derive(Deserialize)]
+struct JsonAssertion {
+    pointer: String,
+    equals: Value,
+}
+
+#[derive(Deserialize)]
+struct Metadata {
+    packages: Vec<MetadataPackage>,
+    workspace_members: Vec<String>,
+}
+
+#[derive(Deserialize)]
+struct MetadataPackage {
+    id: String,
+    name: String,
+    version: String,
+    manifest_path: PathBuf,
+}
+
+#[derive(Deserialize)]
+struct ParityLedger {
+    schema_version: u32,
+    upstream_commit: String,
+    expected_cases: u64,
+    report: ParityReport,
+}
+
+#[derive(Deserialize, Serialize)]
+struct ParityReport {
+    pending: u64,
+    translated: u64,
+    ignored_live: u64,
+    benchmark: u64,
+    drifted: u64,
+    unreviewed: u64,
+}
+
+#[derive(Serialize)]
+struct AuditEvidence {
+    schema: u32,
+    release: String,
+    source_commit: String,
+    rustc: String,
+    recorded_unix_seconds: u64,
+    package_count: usize,
+    binary_count: usize,
+    parity: ParityReport,
+    stub_findings: BTreeMap<String, usize>,
+    silent_live_skip_findings: BTreeMap<String, Vec<String>>,
+    required_evidence: Vec<VerifiedEvidence>,
+    fake_grid_records: usize,
+    live_grid_records: usize,
+    signature_required: bool,
+    signature_policy: String,
+    deferred_by_user: Vec<String>,
+    checks: Vec<Check>,
+    status: &'static str,
+}
+
+#[derive(Serialize)]
+struct VerifiedEvidence {
+    id: String,
+    path: String,
+    sha256: Option<String>,
+    valid: bool,
+}
+
+#[derive(Serialize)]
+struct Check {
+    id: &'static str,
+    passed: bool,
+    detail: String,
+}
+
+/// Audits the complete non-fuzz first-release-candidate gate.
+///
+/// The evidence file is written even when blockers are found so CI preserves a
+/// precise failure inventory. Existing evidence is never overwritten.
+///
+/// # Errors
+///
+/// Returns an error after recording evidence when any release invariant fails,
+/// or immediately for an invalid policy, unreadable input, or existing output.
+pub fn audit_release_candidate(root: &Path, evidence_path: &Path) -> Result<()> {
+    if evidence_path.exists() {
+        return Err(MatrixError::new(format!(
+            "{} already exists; preserve or remove it before rerunning the audit",
+            evidence_path.display()
+        )));
+    }
+    let policy: Policy = serde_json::from_slice(&fs::read(root.join(POLICY_PATH))?)?;
+    validate_policy(&policy)?;
+
+    let rustc = command_output(root, "rustc", &["--version"])?;
+    let source_commit = command_output(root, "git", &["rev-parse", "--verify", "HEAD"])?;
+    let metadata = cargo_metadata(root)?;
+    let (package_count, versions_ok) = release_versions(root, &metadata, &policy)?;
+    let artifact_policy: Value =
+        serde_json::from_slice(&fs::read(root.join(ARTIFACT_POLICY_PATH))?)?;
+    let binary_count = artifact_policy
+        .pointer("/binaries")
+        .and_then(Value::as_array)
+        .map_or(0, Vec::len);
+    let artifact_package_count = artifact_policy
+        .pointer("/release_packages")
+        .and_then(Value::as_array)
+        .map_or(0, Vec::len);
+
+    let parity: ParityLedger = serde_json::from_slice(&fs::read(root.join(PARITY_PATH))?)?;
+    let parity_ok = parity.schema_version == 2
+        && parity.upstream_commit == policy.upstream_commit
+        && parity.expected_cases == policy.expected.parity_cases
+        && parity.report.pending == 0
+        && parity.report.unreviewed == 0
+        && parity.report.drifted == 0
+        && parity.report.translated == policy.expected.translated_cases
+        && parity.report.ignored_live == policy.expected.live_cases
+        && parity.report.benchmark == policy.expected.benchmark_cases;
+
+    let stub_findings = scan_stub_findings(root)?;
+    let silent_live_skip_findings = scan_live_skip_findings(root, &policy.live_test_files)?;
+    let required_evidence = verify_evidence(root, &policy.required_evidence)?;
+    let fake_grid = validate_smoke_evidence(
+        &root.join(&policy.fake_grid_evidence),
+        policy.expected.live_smoke_records,
+        "fake",
+        &policy.release,
+    );
+    let live_grid = validate_smoke_evidence(
+        &root.join(&policy.live_grid_evidence),
+        policy.expected.live_smoke_records,
+        "open-sim-live",
+        &policy.release,
+    );
+    let release_notes_ok = release_text_has_version(root, &policy.release_notes, &policy.release);
+    let changelog_ok = release_text_has_version(root, &policy.changelog, &policy.release);
+    let checksums_ok = validate_checksums(root, &policy.checksums);
+    let workflows_ok = ubuntu_only_workflows(root)?;
+
+    let checks = vec![
+        Check {
+            id: "current-rust",
+            passed: rustc.starts_with(&policy.current_rust_prefix),
+            detail: rustc.clone(),
+        },
+        Check {
+            id: "workspace-version",
+            passed: versions_ok && package_count == policy.expected.release_packages,
+            detail: format!(
+                "{package_count} release packages at version {}",
+                policy.release
+            ),
+        },
+        Check {
+            id: "artifact-inventory",
+            passed: artifact_package_count == policy.expected.release_packages
+                && binary_count == policy.expected.shipped_binaries,
+            detail: format!("{artifact_package_count} packages; {binary_count} binaries"),
+        },
+        Check {
+            id: "api-and-test-parity",
+            passed: parity_ok,
+            detail: format!(
+                "{} cases; {} translated; {} live; {} benchmark; {} pending; {} unreviewed",
+                parity.expected_cases,
+                parity.report.translated,
+                parity.report.ignored_live,
+                parity.report.benchmark,
+                parity.report.pending,
+                parity.report.unreviewed
+            ),
+        },
+        Check {
+            id: "zero-unimplemented-shims",
+            passed: stub_findings.is_empty(),
+            detail: format!(
+                "{} callable failure markers",
+                stub_findings.values().sum::<usize>()
+            ),
+        },
+        Check {
+            id: "no-silent-live-skips",
+            passed: silent_live_skip_findings.is_empty(),
+            detail: format!(
+                "{} files with silent-skip paths",
+                silent_live_skip_findings.len()
+            ),
+        },
+        Check {
+            id: "prior-audit-evidence",
+            passed: required_evidence.iter().all(|item| item.valid),
+            detail: format!("{} required evidence files", required_evidence.len()),
+        },
+        Check {
+            id: "fake-grid-smoke",
+            passed: fake_grid.is_ok(),
+            detail: smoke_detail(&fake_grid),
+        },
+        Check {
+            id: "opensim-live-smoke",
+            passed: live_grid.is_ok(),
+            detail: smoke_detail(&live_grid),
+        },
+        Check {
+            id: "release-notes",
+            passed: release_notes_ok && changelog_ok,
+            detail: format!("release notes and changelog cover {}", policy.release),
+        },
+        Check {
+            id: "artifact-checksums",
+            passed: checksums_ok,
+            detail: policy.authenticity_description.clone(),
+        },
+        Check {
+            id: "ubuntu-only-gitea-actions",
+            passed: workflows_ok,
+            detail: "every Gitea Actions job uses ubuntu-latest".into(),
+        },
+    ];
+    let success = checks.iter().all(|check| check.passed);
+    let evidence = AuditEvidence {
+        schema: 1,
+        release: policy.release,
+        source_commit,
+        rustc,
+        recorded_unix_seconds: SystemTime::now()
+            .duration_since(UNIX_EPOCH)
+            .map_err(|_| MatrixError::new("system clock predates Unix epoch"))?
+            .as_secs(),
+        package_count,
+        binary_count,
+        parity: parity.report,
+        stub_findings,
+        silent_live_skip_findings,
+        required_evidence,
+        fake_grid_records: fake_grid.unwrap_or(0),
+        live_grid_records: live_grid.unwrap_or(0),
+        signature_required: policy.signature_required,
+        signature_policy: policy.authenticity_description,
+        deferred_by_user: policy.deferred_by_user,
+        checks,
+        status: if success { "ok" } else { "blocked" },
+    };
+    write_new_json(evidence_path, &evidence)?;
+    if success {
+        Ok(())
+    } else {
+        Err(MatrixError::new(format!(
+            "release candidate has {} failing non-fuzz gate(s); see {}",
+            evidence.checks.iter().filter(|check| !check.passed).count(),
+            evidence_path.display()
+        )))
+    }
+}
+
+fn validate_policy(policy: &Policy) -> Result<()> {
+    if policy.schema != 1
+        || policy.release.trim().is_empty()
+        || !policy.current_rust_prefix.starts_with("rustc 1.97.")
+        || policy.upstream_commit.len() != 40
+        || policy.required_evidence.is_empty()
+        || policy.live_test_files.is_empty()
+        || policy.authenticity_description.trim().is_empty()
+        || policy.deferred_by_user != ["fuzz-smoke"]
+        || policy.expected.mapped_public_types != 3_066
+        || policy.expected.mapped_public_members != 30_789
+    {
+        return Err(MatrixError::new("invalid release-candidate policy"));
+    }
+    let ids = policy
+        .required_evidence
+        .iter()
+        .map(|item| item.id.as_str())
+        .collect::<BTreeSet<_>>();
+    if ids.len() != policy.required_evidence.len()
+        || policy
+            .required_evidence
+            .iter()
+            .any(|item| item.assertions.is_empty())
+    {
+        return Err(MatrixError::new(
+            "duplicate or assertion-free release evidence policy",
+        ));
+    }
+    Ok(())
+}
+
+fn cargo_metadata(root: &Path) -> Result<Metadata> {
+    let output = Command::new("cargo")
+        .args(["metadata", "--format-version", "1", "--locked", "--no-deps"])
+        .current_dir(root)
+        .output()?;
+    if !output.status.success() {
+        return Err(MatrixError::new(format!(
+            "cargo metadata failed: {}",
+            String::from_utf8_lossy(&output.stderr).trim()
+        )));
+    }
+    Ok(serde_json::from_slice(&output.stdout)?)
+}
+
+fn release_versions(root: &Path, metadata: &Metadata, policy: &Policy) -> Result<(usize, bool)> {
+    let members = metadata
+        .workspace_members
+        .iter()
+        .map(String::as_str)
+        .collect::<BTreeSet<_>>();
+    let crates = root.canonicalize()?.join("crates");
+    let programs = root.canonicalize()?.join("programs/Cargo.toml");
+    let release = metadata
+        .packages
+        .iter()
+        .filter(|package| members.contains(package.id.as_str()))
+        .filter(|package| {
+            package
+                .manifest_path
+                .canonicalize()
+                .is_ok_and(|manifest| manifest.starts_with(&crates) || manifest == programs)
+        })
+        .collect::<Vec<_>>();
+    let versions_ok = release.iter().all(|package| {
+        package.version == policy.release && package.name.starts_with("libremetaverse")
+    });
+    Ok((release.len(), versions_ok))
+}
+
+fn scan_stub_findings(root: &Path) -> Result<BTreeMap<String, usize>> {
+    let mut findings = BTreeMap::new();
+    let crates = root.join("crates");
+    walk_rust(&crates, &mut |path, text| {
+        if path.ends_with("libremetaverse-types/src/shim.rs") {
+            return Ok(());
+        }
+        let count = text.lines().filter(|line| stub_markers(line) > 0).count();
+        if count > 0 {
+            findings.insert(relative(root, path)?, count);
+        }
+        Ok(())
+    })?;
+    Ok(findings)
+}
+
+fn stub_markers(text: &str) -> usize {
+    [
+        "not_implemented(",
+        "unimplemented_api!(",
+        "todo!(",
+        "unimplemented!(",
+    ]
+    .iter()
+    .filter(|marker| text.contains(**marker))
+    .count()
+}
+
+fn walk_rust(directory: &Path, visitor: &mut impl FnMut(&Path, &str) -> Result<()>) -> Result<()> {
+    let mut entries = fs::read_dir(directory)?.collect::<std::io::Result<Vec<_>>>()?;
+    entries.sort_by_key(std::fs::DirEntry::file_name);
+    for entry in entries {
+        let file_type = entry.file_type()?;
+        if file_type.is_symlink() {
+            return Err(MatrixError::new(format!(
+                "release source contains symlink {}",
+                entry.path().display()
+            )));
+        }
+        if file_type.is_dir() {
+            walk_rust(&entry.path(), visitor)?;
+        } else if file_type.is_file() && entry.path().extension() == Some(OsStr::new("rs")) {
+            visitor(&entry.path(), &fs::read_to_string(entry.path())?)?;
+        }
+    }
+    Ok(())
+}
+
+fn scan_live_skip_findings(root: &Path, paths: &[String]) -> Result<BTreeMap<String, Vec<String>>> {
+    let mut findings = BTreeMap::new();
+    for relative in paths {
+        let text = fs::read_to_string(root.join(relative))?;
+        let found = FORBIDDEN_LIVE_PHRASES
+            .iter()
+            .filter(|phrase| text.to_ascii_lowercase().contains(**phrase))
+            .map(|phrase| (*phrase).to_owned())
+            .collect::<Vec<_>>();
+        if !found.is_empty() {
+            findings.insert(relative.clone(), found);
+        }
+    }
+    Ok(findings)
+}
+
+fn verify_evidence(
+    root: &Path,
+    requirements: &[EvidenceRequirement],
+) -> Result<Vec<VerifiedEvidence>> {
+    requirements
+        .iter()
+        .map(|requirement| {
+            let path = root.join(&requirement.path);
+            let bytes = fs::read(&path).ok();
+            let value = bytes
+                .as_deref()
+                .and_then(|bytes| serde_json::from_slice::<Value>(bytes).ok());
+            let valid = value.as_ref().is_some_and(|value| {
+                requirement
+                    .assertions
+                    .iter()
+                    .all(|assertion| value.pointer(&assertion.pointer) == Some(&assertion.equals))
+            });
+            Ok(VerifiedEvidence {
+                id: requirement.id.clone(),
+                path: requirement.path.clone(),
+                sha256: bytes.as_deref().map(hash_bytes),
+                valid,
+            })
+        })
+        .collect()
+}
+
+fn validate_smoke_evidence(
+    path: &Path,
+    expected: usize,
+    expected_mode: &str,
+    release: &str,
+) -> Result<usize> {
+    let text = fs::read_to_string(path)?;
+    let mut stages = BTreeSet::new();
+    let mut count = 0;
+    for (index, line) in text.lines().enumerate() {
+        let value: Value = serde_json::from_str(line)?;
+        let schema = value.pointer("/schema").and_then(Value::as_u64);
+        let mode = value.pointer("/mode").and_then(Value::as_str);
+        let sequence = value.pointer("/sequence").and_then(Value::as_u64);
+        let status = value.pointer("/status").and_then(Value::as_str);
+        let stage = value.pointer("/stage").and_then(Value::as_str);
+        let program_version = value.pointer("/program_version").and_then(Value::as_str);
+        let rust_commit = value.pointer("/rust_commit").and_then(Value::as_str);
+        let recorded = value
+            .pointer("/recorded_unix_seconds")
+            .and_then(Value::as_u64);
+        let metrics = value.pointer("/metrics").and_then(Value::as_object);
+        let expected_stage = SMOKE_STAGES.get(index).copied();
+        let timestamp_valid = recorded.is_some_and(|seconds| {
+            seconds >= 1_577_836_800
+                && SystemTime::now()
+                    .duration_since(UNIX_EPOCH)
+                    .is_ok_and(|now| seconds <= now.as_secs().saturating_add(300))
+        });
+        let commit_valid = rust_commit.is_some_and(|commit| {
+            commit.len() == 40 && commit.bytes().all(|byte| byte.is_ascii_hexdigit())
+        });
+        let metrics_valid = metrics
+            .is_some_and(|metrics| !metrics.is_empty() && metrics.values().all(Value::is_u64));
+        if schema != Some(1)
+            || mode != Some(expected_mode)
+            || sequence != u64::try_from(index + 1).ok()
+            || status != Some("ok")
+            || stage != expected_stage
+            || program_version != Some(release)
+            || !timestamp_valid
+            || !commit_valid
+            || !metrics_valid
+            || !stages.insert(stage.unwrap_or_default().to_owned())
+            || line.contains("://")
+            || line.to_ascii_lowercase().contains("password")
+            || line.to_ascii_lowercase().contains("token")
+        {
+            return Err(MatrixError::new(format!(
+                "invalid sanitized smoke evidence {} at record {}",
+                path.display(),
+                index + 1
+            )));
+        }
+        count += 1;
+    }
+    if count != expected {
+        return Err(MatrixError::new(format!(
+            "{} has {count} smoke records, expected {expected}",
+            path.display()
+        )));
+    }
+    Ok(count)
+}
+
+fn smoke_detail(result: &Result<usize>) -> String {
+    match result {
+        Ok(count) => format!("{count} successful sanitized mode-bound records"),
+        Err(error) => error.to_string(),
+    }
+}
+
+fn release_text_has_version(root: &Path, relative: &str, release: &str) -> bool {
+    fs::read_to_string(root.join(relative)).is_ok_and(|text| text.contains(release))
+}
+
+fn validate_checksums(root: &Path, relative: &str) -> bool {
+    let Ok(text) = fs::read_to_string(root.join(relative)) else {
+        return false;
+    };
+    let mut count = 0;
+    for line in text.lines().filter(|line| !line.trim().is_empty()) {
+        let Some((hash, path)) = line.split_once("  ") else {
+            return false;
+        };
+        if hash.len() != 64 || !hash.bytes().all(|byte| byte.is_ascii_hexdigit()) {
+            return false;
+        }
+        let path = root.join(path);
+        let Ok(bytes) = fs::read(path) else {
+            return false;
+        };
+        if hash_bytes(&bytes) != hash {
+            return false;
+        }
+        count += 1;
+    }
+    count >= 6
+}
+
+fn ubuntu_only_workflows(root: &Path) -> Result<bool> {
+    let directory = root.join(".gitea/workflows");
+    for entry in fs::read_dir(directory)? {
+        let entry = entry?;
+        if entry.path().extension() != Some(OsStr::new("yml")) {
+            continue;
+        }
+        for line in fs::read_to_string(entry.path())?.lines() {
+            if let Some(runner) = line.trim().strip_prefix("runs-on:")
+                && runner.trim() != "ubuntu-latest"
+            {
+                return Ok(false);
+            }
+        }
+    }
+    Ok(true)
+}
+
+fn command_output(root: &Path, program: &str, args: &[&str]) -> Result<String> {
+    let output = Command::new(program)
+        .args(args)
+        .current_dir(root)
+        .output()?;
+    if !output.status.success() {
+        return Err(MatrixError::new(format!("{program} command failed")));
+    }
+    String::from_utf8(output.stdout)
+        .map(|output| output.trim().to_owned())
+        .map_err(|_| MatrixError::new(format!("{program} output was not UTF-8")))
+}
+
+fn relative(root: &Path, path: &Path) -> Result<String> {
+    path.strip_prefix(root)
+        .map_err(|_| MatrixError::new(format!("{} is outside the workspace", path.display())))?
+        .to_str()
+        .map(str::to_owned)
+        .ok_or_else(|| MatrixError::new(format!("{} is not UTF-8", path.display())))
+}
+
+fn hash_bytes(bytes: &[u8]) -> String {
+    Sha256::digest(bytes)
+        .iter()
+        .fold(String::with_capacity(64), |mut output, byte| {
+            use std::fmt::Write as _;
+            write!(output, "{byte:02x}").expect("writing to a String cannot fail");
+            output
+        })
+}
+
+fn write_new_json(path: &Path, value: &impl Serialize) -> Result<()> {
+    if let Some(parent) = path.parent() {
+        fs::create_dir_all(parent)?;
+    }
+    let mut file = OpenOptions::new().write(true).create_new(true).open(path)?;
+    serde_json::to_writer_pretty(&mut file, value)?;
+    file.write_all(b"\n")?;
+    file.sync_all()?;
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn stub_marker_scan_rejects_every_failure_construct() {
+        assert_eq!(stub_markers("value.not_implemented()"), 1);
+        assert_eq!(stub_markers("unimplemented_api!(\"M:x\")"), 1);
+        assert_eq!(stub_markers("todo!()"), 1);
+        assert_eq!(stub_markers("unimplemented!()"), 1);
+        assert_eq!(stub_markers("implemented()"), 0);
+    }
+
+    #[test]
+    fn checked_in_release_candidate_policy_is_well_formed() {
+        let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../..");
+        let policy: Policy =
+            serde_json::from_slice(&fs::read(root.join(POLICY_PATH)).unwrap()).unwrap();
+        validate_policy(&policy).unwrap();
+        assert_eq!(policy.expected.release_packages, 16);
+        assert_eq!(policy.expected.shipped_binaries, 10);
+    }
+
+    #[test]
+    fn smoke_evidence_is_bound_to_mode_stage_version_and_commit() {
+        use std::fmt::Write as _;
+
+        let directory =
+            std::env::temp_dir().join(format!("metacrate-rc-smoke-{}", std::process::id()));
+        fs::create_dir_all(&directory).unwrap();
+        let path = directory.join("evidence.jsonl");
+        let recorded = SystemTime::now()
+            .duration_since(UNIX_EPOCH)
+            .unwrap()
+            .as_secs();
+        let mut contents = String::new();
+        for (index, stage) in SMOKE_STAGES.iter().enumerate() {
+            writeln!(
+                contents,
+                "{{\"schema\":1,\"mode\":\"fake\",\"sequence\":{},\"recorded_unix_seconds\":{recorded},\"program_version\":\"0.0.1\",\"rust_commit\":\"0123456789abcdef0123456789abcdef01234567\",\"stage\":\"{stage}\",\"status\":\"ok\",\"metrics\":{{\"proved\":1}}}}",
+                index + 1
+            )
+            .unwrap();
+        }
+        fs::write(&path, &contents).unwrap();
+        assert_eq!(
+            validate_smoke_evidence(&path, 9, "fake", "0.0.1").unwrap(),
+            9
+        );
+        assert!(validate_smoke_evidence(&path, 9, "open-sim-live", "0.0.1").is_err());
+        fs::remove_file(path).unwrap();
+        fs::remove_dir(directory).unwrap();
+    }
+}