MetaCrate/tools/ci-matrix/src/release_candidate.rs

//! Fail-closed first-release-candidate completeness and evidence audit.

#![allow(clippy::too_many_lines)] // The gate records every invariant in one ordered audit.

use super::{MatrixError, Result};
use serde::{Deserialize, Serialize};
use serde_json::Value;
use sha2::{Digest, Sha256};
use std::collections::{BTreeMap, BTreeSet};
use std::ffi::OsStr;
use std::fs::{self, OpenOptions};
use std::io::Write as _;
use std::path::{Path, PathBuf};
use std::process::Command;
use std::time::{SystemTime, UNIX_EPOCH};

const POLICY_PATH: &str = "ci/release-candidate-policy.json";
const PARITY_PATH: &str = "tests/upstream-tests.json";
const ARTIFACT_POLICY_PATH: &str = "ci/artifact-policy.json";
const FORBIDDEN_LIVE_PHRASES: [&str; 4] = [
    "return;",
    "skipping assertion",
    "transient server issue",
    "proceeding with tests",
];
const SMOKE_STAGES: [&str; 9] = [
    "completeness",
    "login",
    "capabilities-simulator",
    "im-chat",
    "movement-teleport",
    "inventory-folder",
    "object-properties",
    "asset-texture",
    "logout",
];

#[derive(Deserialize)]
struct Policy {
    schema: u32,
    release: String,
    current_rust_prefix: String,
    upstream_commit: String,
    expected: Expected,
    required_evidence: Vec<EvidenceRequirement>,
    live_test_files: Vec<String>,
    fake_grid_evidence: String,
    live_grid_evidence: String,
    release_notes: String,
    changelog: String,
    checksums: String,
    signature_required: bool,
    #[serde(rename = "signature_policy")]
    authenticity_description: String,
    deferred_by_user: Vec<String>,
}

#[derive(Deserialize)]
struct Expected {
    release_packages: usize,
    shipped_binaries: usize,
    mapped_public_types: u64,
    mapped_public_members: u64,
    parity_cases: u64,
    translated_cases: u64,
    live_cases: u64,
    benchmark_cases: u64,
    live_smoke_records: usize,
}

#[derive(Deserialize)]
struct EvidenceRequirement {
    id: String,
    path: String,
    assertions: Vec<JsonAssertion>,
}

#[derive(Deserialize)]
struct JsonAssertion {
    pointer: String,
    equals: Value,
}

#[derive(Deserialize)]
struct Metadata {
    packages: Vec<MetadataPackage>,
    workspace_members: Vec<String>,
}

#[derive(Deserialize)]
struct MetadataPackage {
    id: String,
    name: String,
    version: String,
    manifest_path: PathBuf,
}

#[derive(Deserialize)]
struct ParityLedger {
    schema_version: u32,
    upstream_commit: String,
    expected_cases: u64,
    report: ParityReport,
}

#[derive(Deserialize, Serialize)]
struct ParityReport {
    pending: u64,
    translated: u64,
    ignored_live: u64,
    benchmark: u64,
    drifted: u64,
    unreviewed: u64,
}

#[derive(Serialize)]
struct AuditEvidence {
    schema: u32,
    release: String,
    source_commit: String,
    rustc: String,
    recorded_unix_seconds: u64,
    package_count: usize,
    binary_count: usize,
    parity: ParityReport,
    stub_findings: BTreeMap<String, usize>,
    silent_live_skip_findings: BTreeMap<String, Vec<String>>,
    required_evidence: Vec<VerifiedEvidence>,
    fake_grid_records: usize,
    live_grid_records: usize,
    signature_required: bool,
    signature_policy: String,
    deferred_by_user: Vec<String>,
    checks: Vec<Check>,
    status: &'static str,
}

#[derive(Serialize)]
struct VerifiedEvidence {
    id: String,
    path: String,
    sha256: Option<String>,
    valid: bool,
}

#[derive(Serialize)]
struct Check {
    id: &'static str,
    passed: bool,
    detail: String,
}

/// Audits the complete non-fuzz first-release-candidate gate.
///
/// The evidence file is written even when blockers are found so CI preserves a
/// precise failure inventory. Existing evidence is never overwritten.
///
/// # Errors
///
/// Returns an error after recording evidence when any release invariant fails,
/// or immediately for an invalid policy, unreadable input, or existing output.
pub fn audit_release_candidate(root: &Path, evidence_path: &Path) -> Result<()> {
    if evidence_path.exists() {
        return Err(MatrixError::new(format!(
            "{} already exists; preserve or remove it before rerunning the audit",
            evidence_path.display()
        )));
    }
    let policy: Policy = serde_json::from_slice(&fs::read(root.join(POLICY_PATH))?)?;
    validate_policy(&policy)?;

    let rustc = command_output(root, "rustc", &["--version"])?;
    let source_commit = command_output(root, "git", &["rev-parse", "--verify", "HEAD"])?;
    let metadata = cargo_metadata(root)?;
    let (package_count, versions_ok) = release_versions(root, &metadata, &policy)?;
    let artifact_policy: Value =
        serde_json::from_slice(&fs::read(root.join(ARTIFACT_POLICY_PATH))?)?;
    let binary_count = artifact_policy
        .pointer("/binaries")
        .and_then(Value::as_array)
        .map_or(0, Vec::len);
    let artifact_package_count = artifact_policy
        .pointer("/release_packages")
        .and_then(Value::as_array)
        .map_or(0, Vec::len);

    let parity: ParityLedger = serde_json::from_slice(&fs::read(root.join(PARITY_PATH))?)?;
    let parity_ok = parity.schema_version == 2
        && parity.upstream_commit == policy.upstream_commit
        && parity.expected_cases == policy.expected.parity_cases
        && parity.report.pending == 0
        && parity.report.unreviewed == 0
        && parity.report.drifted == 0
        && parity.report.translated == policy.expected.translated_cases
        && parity.report.ignored_live == policy.expected.live_cases
        && parity.report.benchmark == policy.expected.benchmark_cases;

    let stub_findings = scan_stub_findings(root)?;
    let silent_live_skip_findings = scan_live_skip_findings(root, &policy.live_test_files)?;
    let required_evidence = verify_evidence(root, &policy.required_evidence)?;
    let fake_grid = validate_smoke_evidence(
        &root.join(&policy.fake_grid_evidence),
        policy.expected.live_smoke_records,
        "fake",
        &policy.release,
    );
    let live_grid = validate_smoke_evidence(
        &root.join(&policy.live_grid_evidence),
        policy.expected.live_smoke_records,
        "open-sim-live",
        &policy.release,
    );
    let release_notes_ok = release_text_has_version(root, &policy.release_notes, &policy.release);
    let changelog_ok = release_text_has_version(root, &policy.changelog, &policy.release);
    let checksums_ok = validate_checksums(root, &policy.checksums);
    let workflows_ok = ubuntu_only_workflows(root)?;

    let checks = vec![
        Check {
            id: "current-rust",
            passed: rustc.starts_with(&policy.current_rust_prefix),
            detail: rustc.clone(),
        },
        Check {
            id: "workspace-version",
            passed: versions_ok && package_count == policy.expected.release_packages,
            detail: format!(
                "{package_count} release packages at version {}",
                policy.release
            ),
        },
        Check {
            id: "artifact-inventory",
            passed: artifact_package_count == policy.expected.release_packages
                && binary_count == policy.expected.shipped_binaries,
            detail: format!("{artifact_package_count} packages; {binary_count} binaries"),
        },
        Check {
            id: "api-and-test-parity",
            passed: parity_ok,
            detail: format!(
                "{} cases; {} translated; {} live; {} benchmark; {} pending; {} unreviewed",
                parity.expected_cases,
                parity.report.translated,
                parity.report.ignored_live,
                parity.report.benchmark,
                parity.report.pending,
                parity.report.unreviewed
            ),
        },
        Check {
            id: "zero-unimplemented-shims",
            passed: stub_findings.is_empty(),
            detail: format!(
                "{} callable failure markers",
                stub_findings.values().sum::<usize>()
            ),
        },
        Check {
            id: "no-silent-live-skips",
            passed: silent_live_skip_findings.is_empty(),
            detail: format!(
                "{} files with silent-skip paths",
                silent_live_skip_findings.len()
            ),
        },
        Check {
            id: "prior-audit-evidence",
            passed: required_evidence.iter().all(|item| item.valid),
            detail: format!("{} required evidence files", required_evidence.len()),
        },
        Check {
            id: "fake-grid-smoke",
            passed: fake_grid.is_ok(),
            detail: smoke_detail(&fake_grid),
        },
        Check {
            id: "opensim-live-smoke",
            passed: live_grid.is_ok(),
            detail: smoke_detail(&live_grid),
        },
        Check {
            id: "release-notes",
            passed: release_notes_ok && changelog_ok,
            detail: format!("release notes and changelog cover {}", policy.release),
        },
        Check {
            id: "artifact-checksums",
            passed: checksums_ok,
            detail: policy.authenticity_description.clone(),
        },
        Check {
            id: "ubuntu-only-gitea-actions",
            passed: workflows_ok,
            detail: "every Gitea Actions job uses ubuntu-latest".into(),
        },
    ];
    let success = checks.iter().all(|check| check.passed);
    let evidence = AuditEvidence {
        schema: 1,
        release: policy.release,
        source_commit,
        rustc,
        recorded_unix_seconds: SystemTime::now()
            .duration_since(UNIX_EPOCH)
            .map_err(|_| MatrixError::new("system clock predates Unix epoch"))?
            .as_secs(),
        package_count,
        binary_count,
        parity: parity.report,
        stub_findings,
        silent_live_skip_findings,
        required_evidence,
        fake_grid_records: fake_grid.unwrap_or(0),
        live_grid_records: live_grid.unwrap_or(0),
        signature_required: policy.signature_required,
        signature_policy: policy.authenticity_description,
        deferred_by_user: policy.deferred_by_user,
        checks,
        status: if success { "ok" } else { "blocked" },
    };
    write_new_json(evidence_path, &evidence)?;
    if success {
        Ok(())
    } else {
        Err(MatrixError::new(format!(
            "release candidate has {} failing non-fuzz gate(s); see {}",
            evidence.checks.iter().filter(|check| !check.passed).count(),
            evidence_path.display()
        )))
    }
}

fn validate_policy(policy: &Policy) -> Result<()> {
    if policy.schema != 1
        || policy.release.trim().is_empty()
        || !policy.current_rust_prefix.starts_with("rustc 1.97.")
        || policy.upstream_commit.len() != 40
        || policy.required_evidence.is_empty()
        || policy.live_test_files.is_empty()
        || policy.authenticity_description.trim().is_empty()
        || policy.deferred_by_user != ["fuzz-smoke"]
        || policy.expected.mapped_public_types != 3_066
        || policy.expected.mapped_public_members != 30_789
    {
        return Err(MatrixError::new("invalid release-candidate policy"));
    }
    let ids = policy
        .required_evidence
        .iter()
        .map(|item| item.id.as_str())
        .collect::<BTreeSet<_>>();
    if ids.len() != policy.required_evidence.len()
        || policy
            .required_evidence
            .iter()
            .any(|item| item.assertions.is_empty())
    {
        return Err(MatrixError::new(
            "duplicate or assertion-free release evidence policy",
        ));
    }
    Ok(())
}

fn cargo_metadata(root: &Path) -> Result<Metadata> {
    let output = Command::new("cargo")
        .args(["metadata", "--format-version", "1", "--locked", "--no-deps"])
        .current_dir(root)
        .output()?;
    if !output.status.success() {
        return Err(MatrixError::new(format!(
            "cargo metadata failed: {}",
            String::from_utf8_lossy(&output.stderr).trim()
        )));
    }
    Ok(serde_json::from_slice(&output.stdout)?)
}

fn release_versions(root: &Path, metadata: &Metadata, policy: &Policy) -> Result<(usize, bool)> {
    let members = metadata
        .workspace_members
        .iter()
        .map(String::as_str)
        .collect::<BTreeSet<_>>();
    let crates = root.canonicalize()?.join("crates");
    let programs = root.canonicalize()?.join("programs/Cargo.toml");
    let release = metadata
        .packages
        .iter()
        .filter(|package| members.contains(package.id.as_str()))
        .filter(|package| {
            package
                .manifest_path
                .canonicalize()
                .is_ok_and(|manifest| manifest.starts_with(&crates) || manifest == programs)
        })
        .collect::<Vec<_>>();
    let versions_ok = release.iter().all(|package| {
        package.version == policy.release && package.name.starts_with("libremetaverse")
    });
    Ok((release.len(), versions_ok))
}

fn scan_stub_findings(root: &Path) -> Result<BTreeMap<String, usize>> {
    let mut findings = BTreeMap::new();
    let crates = root.join("crates");
    walk_rust(&crates, &mut |path, text| {
        if path.ends_with("libremetaverse-types/src/shim.rs") {
            return Ok(());
        }
        let count = text.lines().filter(|line| stub_markers(line) > 0).count();
        if count > 0 {
            findings.insert(relative(root, path)?, count);
        }
        Ok(())
    })?;
    Ok(findings)
}

fn stub_markers(text: &str) -> usize {
    [
        "not_implemented(",
        "unimplemented_api!(",
        "todo!(",
        "unimplemented!(",
    ]
    .iter()
    .filter(|marker| text.contains(**marker))
    .count()
}

fn walk_rust(directory: &Path, visitor: &mut impl FnMut(&Path, &str) -> Result<()>) -> Result<()> {
    let mut entries = fs::read_dir(directory)?.collect::<std::io::Result<Vec<_>>>()?;
    entries.sort_by_key(std::fs::DirEntry::file_name);
    for entry in entries {
        let file_type = entry.file_type()?;
        if file_type.is_symlink() {
            return Err(MatrixError::new(format!(
                "release source contains symlink {}",
                entry.path().display()
            )));
        }
        if file_type.is_dir() {
            walk_rust(&entry.path(), visitor)?;
        } else if file_type.is_file() && entry.path().extension() == Some(OsStr::new("rs")) {
            visitor(&entry.path(), &fs::read_to_string(entry.path())?)?;
        }
    }
    Ok(())
}

fn scan_live_skip_findings(root: &Path, paths: &[String]) -> Result<BTreeMap<String, Vec<String>>> {
    let mut findings = BTreeMap::new();
    for relative in paths {
        let text = fs::read_to_string(root.join(relative))?;
        let found = FORBIDDEN_LIVE_PHRASES
            .iter()
            .filter(|phrase| text.to_ascii_lowercase().contains(**phrase))
            .map(|phrase| (*phrase).to_owned())
            .collect::<Vec<_>>();
        if !found.is_empty() {
            findings.insert(relative.clone(), found);
        }
    }
    Ok(findings)
}

fn verify_evidence(
    root: &Path,
    requirements: &[EvidenceRequirement],
) -> Result<Vec<VerifiedEvidence>> {
    requirements
        .iter()
        .map(|requirement| {
            let path = root.join(&requirement.path);
            let bytes = fs::read(&path).ok();
            let value = bytes
                .as_deref()
                .and_then(|bytes| serde_json::from_slice::<Value>(bytes).ok());
            let valid = value.as_ref().is_some_and(|value| {
                requirement
                    .assertions
                    .iter()
                    .all(|assertion| value.pointer(&assertion.pointer) == Some(&assertion.equals))
            });
            Ok(VerifiedEvidence {
                id: requirement.id.clone(),
                path: requirement.path.clone(),
                sha256: bytes.as_deref().map(hash_bytes),
                valid,
            })
        })
        .collect()
}

fn validate_smoke_evidence(
    path: &Path,
    expected: usize,
    expected_mode: &str,
    release: &str,
) -> Result<usize> {
    let text = fs::read_to_string(path)?;
    let mut stages = BTreeSet::new();
    let mut count = 0;
    for (index, line) in text.lines().enumerate() {
        let value: Value = serde_json::from_str(line)?;
        let schema = value.pointer("/schema").and_then(Value::as_u64);
        let mode = value.pointer("/mode").and_then(Value::as_str);
        let sequence = value.pointer("/sequence").and_then(Value::as_u64);
        let status = value.pointer("/status").and_then(Value::as_str);
        let stage = value.pointer("/stage").and_then(Value::as_str);
        let program_version = value.pointer("/program_version").and_then(Value::as_str);
        let rust_commit = value.pointer("/rust_commit").and_then(Value::as_str);
        let recorded = value
            .pointer("/recorded_unix_seconds")
            .and_then(Value::as_u64);
        let metrics = value.pointer("/metrics").and_then(Value::as_object);
        let expected_stage = SMOKE_STAGES.get(index).copied();
        let timestamp_valid = recorded.is_some_and(|seconds| {
            seconds >= 1_577_836_800
                && SystemTime::now()
                    .duration_since(UNIX_EPOCH)
                    .is_ok_and(|now| seconds <= now.as_secs().saturating_add(300))
        });
        let commit_valid = rust_commit.is_some_and(|commit| {
            commit.len() == 40 && commit.bytes().all(|byte| byte.is_ascii_hexdigit())
        });
        let metrics_valid = metrics
            .is_some_and(|metrics| !metrics.is_empty() && metrics.values().all(Value::is_u64));
        if schema != Some(1)
            || mode != Some(expected_mode)
            || sequence != u64::try_from(index + 1).ok()
            || status != Some("ok")
            || stage != expected_stage
            || program_version != Some(release)
            || !timestamp_valid
            || !commit_valid
            || !metrics_valid
            || !stages.insert(stage.unwrap_or_default().to_owned())
            || line.contains("://")
            || line.to_ascii_lowercase().contains("password")
            || line.to_ascii_lowercase().contains("token")
        {
            return Err(MatrixError::new(format!(
                "invalid sanitized smoke evidence {} at record {}",
                path.display(),
                index + 1
            )));
        }
        count += 1;
    }
    if count != expected {
        return Err(MatrixError::new(format!(
            "{} has {count} smoke records, expected {expected}",
            path.display()
        )));
    }
    Ok(count)
}

fn smoke_detail(result: &Result<usize>) -> String {
    match result {
        Ok(count) => format!("{count} successful sanitized mode-bound records"),
        Err(error) => error.to_string(),
    }
}

fn release_text_has_version(root: &Path, relative: &str, release: &str) -> bool {
    fs::read_to_string(root.join(relative)).is_ok_and(|text| text.contains(release))
}

fn validate_checksums(root: &Path, relative: &str) -> bool {
    let Ok(text) = fs::read_to_string(root.join(relative)) else {
        return false;
    };
    let mut count = 0;
    for line in text.lines().filter(|line| !line.trim().is_empty()) {
        let Some((hash, path)) = line.split_once("  ") else {
            return false;
        };
        if hash.len() != 64 || !hash.bytes().all(|byte| byte.is_ascii_hexdigit()) {
            return false;
        }
        let path = root.join(path);
        let Ok(bytes) = fs::read(path) else {
            return false;
        };
        if hash_bytes(&bytes) != hash {
            return false;
        }
        count += 1;
    }
    count >= 6
}

fn ubuntu_only_workflows(root: &Path) -> Result<bool> {
    let directory = root.join(".gitea/workflows");
    for entry in fs::read_dir(directory)? {
        let entry = entry?;
        if entry.path().extension() != Some(OsStr::new("yml")) {
            continue;
        }
        for line in fs::read_to_string(entry.path())?.lines() {
            if let Some(runner) = line.trim().strip_prefix("runs-on:")
                && runner.trim() != "ubuntu-latest"
            {
                return Ok(false);
            }
        }
    }
    Ok(true)
}

fn command_output(root: &Path, program: &str, args: &[&str]) -> Result<String> {
    let output = Command::new(program)
        .args(args)
        .current_dir(root)
        .output()?;
    if !output.status.success() {
        return Err(MatrixError::new(format!("{program} command failed")));
    }
    String::from_utf8(output.stdout)
        .map(|output| output.trim().to_owned())
        .map_err(|_| MatrixError::new(format!("{program} output was not UTF-8")))
}

fn relative(root: &Path, path: &Path) -> Result<String> {
    path.strip_prefix(root)
        .map_err(|_| MatrixError::new(format!("{} is outside the workspace", path.display())))?
        .to_str()
        .map(str::to_owned)
        .ok_or_else(|| MatrixError::new(format!("{} is not UTF-8", path.display())))
}

fn hash_bytes(bytes: &[u8]) -> String {
    Sha256::digest(bytes)
        .iter()
        .fold(String::with_capacity(64), |mut output, byte| {
            use std::fmt::Write as _;
            write!(output, "{byte:02x}").expect("writing to a String cannot fail");
            output
        })
}

fn write_new_json(path: &Path, value: &impl Serialize) -> Result<()> {
    if let Some(parent) = path.parent() {
        fs::create_dir_all(parent)?;
    }
    let mut file = OpenOptions::new().write(true).create_new(true).open(path)?;
    serde_json::to_writer_pretty(&mut file, value)?;
    file.write_all(b"\n")?;
    file.sync_all()?;
    Ok(())
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn stub_marker_scan_rejects_every_failure_construct() {
        assert_eq!(stub_markers("value.not_implemented()"), 1);
        assert_eq!(stub_markers("unimplemented_api!(\"M:x\")"), 1);
        assert_eq!(stub_markers("todo!()"), 1);
        assert_eq!(stub_markers("unimplemented!()"), 1);
        assert_eq!(stub_markers("implemented()"), 0);
    }

    #[test]
    fn checked_in_release_candidate_policy_is_well_formed() {
        let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../..");
        let policy: Policy =
            serde_json::from_slice(&fs::read(root.join(POLICY_PATH)).unwrap()).unwrap();
        validate_policy(&policy).unwrap();
        assert_eq!(policy.expected.release_packages, 16);
        assert_eq!(policy.expected.shipped_binaries, 10);
    }

    #[test]
    fn smoke_evidence_is_bound_to_mode_stage_version_and_commit() {
        use std::fmt::Write as _;

        let directory =
            std::env::temp_dir().join(format!("metacrate-rc-smoke-{}", std::process::id()));
        fs::create_dir_all(&directory).unwrap();
        let path = directory.join("evidence.jsonl");
        let recorded = SystemTime::now()
            .duration_since(UNIX_EPOCH)
            .unwrap()
            .as_secs();
        let mut contents = String::new();
        for (index, stage) in SMOKE_STAGES.iter().enumerate() {
            writeln!(
                contents,
                "{{\"schema\":1,\"mode\":\"fake\",\"sequence\":{},\"recorded_unix_seconds\":{recorded},\"program_version\":\"0.0.1\",\"rust_commit\":\"0123456789abcdef0123456789abcdef01234567\",\"stage\":\"{stage}\",\"status\":\"ok\",\"metrics\":{{\"proved\":1}}}}",
                index + 1
            )
            .unwrap();
        }
        fs::write(&path, &contents).unwrap();
        assert_eq!(
            validate_smoke_evidence(&path, 9, "fake", "0.0.1").unwrap(),
            9
        );
        assert!(validate_smoke_evidence(&path, 9, "open-sim-live", "0.0.1").is_err());
        fs::remove_file(path).unwrap();
        fs::remove_dir(directory).unwrap();
    }
}