MetaCrate/tools/ci-matrix/src/provenance.rs

use super::{MatrixError, Result};
use serde::{Deserialize, Serialize};
use serde_json::Value;
use sha2::{Digest as _, Sha256};
use std::collections::{BTreeMap, BTreeSet};
use std::ffi::OsStr;
use std::fmt::Write as _;
use std::fs::{self, OpenOptions};
use std::io::{Read as _, Write as _};
use std::path::{Component, Path, PathBuf};
use std::process::Command;
use std::time::{SystemTime, UNIX_EPOCH};

const POLICY_PATH: &str = "ci/provenance-policy.json";
const DEPENDENCY_MANIFEST_PATH: &str = "release/DEPENDENCY-LICENSES.json";
const THIRD_PARTY_NOTICE_PATH: &str = "release/THIRD-PARTY-NOTICES.md";
const NATIVE_NOTICE_PATH: &str = "release/NATIVE-LICENSES.md";
const DISTRIBUTION_MANIFEST_PATH: &str = "release/DISTRIBUTION-MANIFEST.json";
const GENERATED_RELEASE_PATHS: [&str; 4] = [
    DEPENDENCY_MANIFEST_PATH,
    THIRD_PARTY_NOTICE_PATH,
    NATIVE_NOTICE_PATH,
    DISTRIBUTION_MANIFEST_PATH,
];
const MATERIAL_ROOTS: [&str; 5] = [
    "codegen/inputs",
    "tests/fixtures",
    "fuzz/corpus",
    "benchmarks/fixtures",
    "crates/libremetaverse/assets",
];
const BUNDLED_EXTENSIONS: [&str; 18] = [
    "a", "animatn", "bmp", "bodypart", "clothing", "dll", "dylib", "gesture", "gif", "jpeg", "jpg",
    "llm", "ogg", "png", "so", "tga", "wav", "webp",
];

#[derive(Debug, Deserialize)]
struct ProvenancePolicy {
    schema: u32,
    upstream_repository: String,
    upstream_commit: String,
    project_license: String,
    project_license_path: String,
    project_license_sha256: String,
    upstream_license_path: String,
    upstream_license_sha256: String,
    cc_by_sa_legal_code_sha256: String,
    required_source_notices: Vec<String>,
    required_binary_notices: Vec<String>,
    materials: Vec<Material>,
    generated_outputs: Vec<GeneratedOutput>,
    provenance_ledgers: Vec<ProvenanceLedger>,
    native_components: Vec<NativeComponent>,
}

#[derive(Debug, Deserialize, Serialize)]
struct Material {
    path: String,
    sha256: String,
    kind: String,
    origin: String,
    license: String,
    distribution: String,
}

#[derive(Debug, Deserialize, Serialize)]
struct GeneratedOutput {
    path: String,
    generator: String,
    license: String,
}

#[derive(Debug, Deserialize, Serialize)]
struct ProvenanceLedger {
    path: String,
    kind: String,
    license: String,
}

#[derive(Debug, Deserialize, Serialize)]
struct NativeComponent {
    id: String,
    version: String,
    license: String,
    source: String,
    linkage: String,
    bundled: bool,
    obligation: String,
}

#[derive(Debug, Serialize)]
struct DependencyManifest {
    schema: u32,
    cargo_lock_sha256: String,
    packages: Vec<DependencyPackage>,
}

#[derive(Debug, Serialize)]
struct DependencyPackage {
    name: String,
    version: String,
    checksum: String,
    license: String,
    repository: Option<String>,
    notice_files: Vec<PackageNotice>,
}

#[derive(Clone, Debug, Serialize)]
struct PackageNotice {
    path: String,
    sha256: String,
}

struct NoticeText {
    packages: BTreeSet<String>,
    paths: BTreeSet<String>,
    contents: String,
}

#[derive(Debug, Serialize)]
struct DistributionManifest<'a> {
    schema: u32,
    cargo_lock_sha256: String,
    provenance_policy_sha256: String,
    upstream_repository: &'a str,
    upstream_commit: &'a str,
    source_files: Vec<DistributedFile>,
    generated_release_files: Vec<DistributedFile>,
    materials: &'a [Material],
    generated_outputs: &'a [GeneratedOutput],
    provenance_ledgers: &'a [ProvenanceLedger],
    required_source_notices: &'a [String],
    required_binary_notices: &'a [String],
    native_components: &'a [NativeComponent],
}

#[derive(Debug, Serialize)]
struct DistributedFile {
    path: String,
    bytes: u64,
    sha256: String,
}

struct Reports {
    dependency_manifest: Vec<u8>,
    third_party_notices: Vec<u8>,
    native_notices: Vec<u8>,
    distribution_manifest: Vec<u8>,
    source_file_count: usize,
    dependency_count: usize,
    dependency_notice_count: usize,
}

#[derive(Debug, Serialize)]
struct ProvenanceEvidence {
    schema: u32,
    recorded_unix_seconds: u64,
    upstream_commit: String,
    material_count: usize,
    generated_output_count: usize,
    provenance_ledger_count: usize,
    source_file_count: usize,
    dependency_count: usize,
    dependency_notice_count: usize,
    native_component_count: usize,
    dependency_manifest_sha256: String,
    third_party_notices_sha256: String,
    native_notices_sha256: String,
    distribution_manifest_sha256: String,
    unknown_materials: usize,
    unknown_bundled_assets: usize,
    status: &'static str,
}

/// Regenerates the locked dependency notices and distribution manifest.
///
/// # Errors
///
/// Returns an error when policy, Cargo metadata, package license files, source
/// inventory, or an output file cannot be read or validated.
pub fn write_provenance_reports(root: &Path) -> Result<()> {
    let policy = load_and_validate_policy(root)?;
    audit_materials(root, &policy)?;
    let reports = generate_reports(root, &policy)?;
    write_report(root, DEPENDENCY_MANIFEST_PATH, &reports.dependency_manifest)?;
    write_report(root, THIRD_PARTY_NOTICE_PATH, &reports.third_party_notices)?;
    write_report(root, NATIVE_NOTICE_PATH, &reports.native_notices)?;
    write_report(
        root,
        DISTRIBUTION_MANIFEST_PATH,
        &reports.distribution_manifest,
    )?;
    Ok(())
}

/// Audits all provenance inputs and checked-in release notices.
///
/// # Errors
///
/// Returns an error for unknown or changed assets, missing attribution,
/// dependency/license drift, stale generated reports, unsafe paths, bundled
/// binaries without provenance, or evidence I/O failures.
pub fn audit_provenance(root: &Path, evidence: &Path) -> Result<()> {
    if evidence.exists() {
        return Err(MatrixError::new(format!(
            "{} already exists; preserve or remove it before rerunning the audit",
            evidence.display()
        )));
    }
    let policy = load_and_validate_policy(root)?;
    audit_materials(root, &policy)?;
    let reports = generate_reports(root, &policy)?;
    compare_report(root, DEPENDENCY_MANIFEST_PATH, &reports.dependency_manifest)?;
    compare_report(root, THIRD_PARTY_NOTICE_PATH, &reports.third_party_notices)?;
    compare_report(root, NATIVE_NOTICE_PATH, &reports.native_notices)?;
    compare_report(
        root,
        DISTRIBUTION_MANIFEST_PATH,
        &reports.distribution_manifest,
    )?;

    let record = ProvenanceEvidence {
        schema: 1,
        recorded_unix_seconds: SystemTime::now()
            .duration_since(UNIX_EPOCH)
            .map_err(|_| MatrixError::new("system clock predates Unix epoch"))?
            .as_secs(),
        upstream_commit: policy.upstream_commit,
        material_count: policy.materials.len(),
        generated_output_count: policy.generated_outputs.len(),
        provenance_ledger_count: policy.provenance_ledgers.len(),
        source_file_count: reports.source_file_count,
        dependency_count: reports.dependency_count,
        dependency_notice_count: reports.dependency_notice_count,
        native_component_count: policy.native_components.len(),
        dependency_manifest_sha256: sha256(&reports.dependency_manifest),
        third_party_notices_sha256: sha256(&reports.third_party_notices),
        native_notices_sha256: sha256(&reports.native_notices),
        distribution_manifest_sha256: sha256(&reports.distribution_manifest),
        unknown_materials: 0,
        unknown_bundled_assets: 0,
        status: "ok",
    };
    write_new_json(evidence, &record)
}

fn load_and_validate_policy(root: &Path) -> Result<ProvenancePolicy> {
    let bytes = fs::read(root.join(POLICY_PATH))?;
    let policy: ProvenancePolicy = serde_json::from_slice(&bytes)?;
    if policy.schema != 1
        || policy.upstream_commit.len() != 40
        || !policy
            .upstream_commit
            .bytes()
            .all(|byte| byte.is_ascii_hexdigit())
        || policy.upstream_repository != "https://github.com/cinderblocks/libremetaverse"
        || policy.project_license != "BSD-3-Clause"
        || policy.upstream_license_sha256
            != "2e40a7bac96023b6299c9062195d927d8e53ca271931412fcdd0e1928e0a3b4e"
        || policy.cc_by_sa_legal_code_sha256
            != "075dad5e5fc96c27014fabc269f4f5732909cffd178a486f546d982b6cf86b74"
    {
        return Err(MatrixError::new("invalid provenance policy header"));
    }
    validate_relative(&policy.project_license_path)?;
    validate_relative(&policy.upstream_license_path)?;
    validate_hash(&policy.project_license_sha256)?;
    let license = fs::read(root.join(&policy.project_license_path))?;
    if sha256(&license) != policy.project_license_sha256 {
        return Err(MatrixError::new("project BSD license hash changed"));
    }
    let upstream_license = fs::read(root.join(&policy.upstream_license_path))?;
    let upstream_without_optional_newline = upstream_license
        .strip_suffix(b"\n")
        .unwrap_or(&upstream_license);
    if sha256(upstream_without_optional_newline) != policy.upstream_license_sha256 {
        return Err(MatrixError::new(
            "included LibreMetaverse BSD license differs from the pinned upstream text",
        ));
    }
    let license_text = String::from_utf8_lossy(&license);
    for required in [
        "Copyright (c) 2026, rfc1437",
        "Copyright (c) 2006-2016, openmetaverse.co",
        "Copyright (c) 2017-2025, Sjofn LLC",
        policy.upstream_commit.as_str(),
    ] {
        if !license_text.contains(required) {
            return Err(MatrixError::new(format!(
                "project BSD license is missing required attribution {required}"
            )));
        }
    }
    validate_notices(root, &policy)?;
    validate_material_policy(&policy)?;
    validate_generated_policy(root, &policy)?;
    validate_provenance_ledgers(root, &policy)?;
    validate_native_policy(&policy)?;
    Ok(policy)
}

fn validate_notices(root: &Path, policy: &ProvenancePolicy) -> Result<()> {
    let required = BTreeSet::from([
        "LICENSE.md",
        "NOTICE.md",
        "licenses/CC-BY-SA-3.0-NOTICE.md",
        "licenses/LibreMetaverse-BSD-3-Clause.txt",
        "release/NATIVE-LICENSES.md",
        "release/THIRD-PARTY-NOTICES.md",
    ]);
    let configured = policy
        .required_source_notices
        .iter()
        .map(String::as_str)
        .collect::<BTreeSet<_>>();
    if policy.required_source_notices != policy.required_binary_notices || configured != required {
        return Err(MatrixError::new(
            "source and binary distributions must carry the complete notice set",
        ));
    }
    let mut unique = BTreeSet::new();
    for notice in &policy.required_source_notices {
        validate_relative(notice)?;
        if !unique.insert(notice.as_str()) {
            return Err(MatrixError::new(format!(
                "duplicate required notice {notice}"
            )));
        }
        if !GENERATED_RELEASE_PATHS.contains(&notice.as_str()) && !root.join(notice).is_file() {
            return Err(MatrixError::new(format!(
                "required notice {notice} is missing"
            )));
        }
    }
    let notice = fs::read_to_string(root.join("NOTICE.md"))?;
    let cc_notice = fs::read_to_string(root.join("licenses/CC-BY-SA-3.0-NOTICE.md"))?;
    for required in [
        policy.upstream_commit.as_str(),
        "Linden Lab",
        "CC-BY-SA-3.0",
        "codegen/inputs/avatar_lad.xml",
        "No Linden textures, meshes, animations",
    ] {
        if !notice.contains(required) {
            return Err(MatrixError::new(format!("NOTICE.md is missing {required}")));
        }
    }
    if !cc_notice.contains("https://creativecommons.org/licenses/by-sa/3.0/legalcode")
        || !cc_notice.contains(&policy.cc_by_sa_legal_code_sha256)
    {
        return Err(MatrixError::new(
            "CC BY-SA notice is missing its legal-code URI or pinned hash",
        ));
    }
    Ok(())
}

fn validate_material_policy(policy: &ProvenancePolicy) -> Result<()> {
    let mut paths = BTreeSet::new();
    let mut linden_count = 0;
    for material in &policy.materials {
        validate_relative(&material.path)?;
        validate_hash(&material.sha256)?;
        if !paths.insert(material.path.as_str())
            || material.kind.trim().is_empty()
            || material.origin.trim().is_empty()
            || material.distribution.trim().is_empty()
            || !matches!(material.license.as_str(), "BSD-3-Clause" | "CC-BY-SA-3.0")
        {
            return Err(MatrixError::new(format!(
                "invalid or duplicate material {}",
                material.path
            )));
        }
        if material.kind == "linden-data" {
            linden_count += 1;
            if material.license != "CC-BY-SA-3.0"
                || !material.origin.starts_with("LibreMetaverse/linden/")
            {
                return Err(MatrixError::new(format!(
                    "Linden material {} must retain CC-BY-SA-3.0 provenance",
                    material.path
                )));
            }
        }
    }
    if linden_count != 7 {
        return Err(MatrixError::new(format!(
            "expected exactly seven required Linden data inputs, found {linden_count}"
        )));
    }
    Ok(())
}

fn validate_generated_policy(root: &Path, policy: &ProvenancePolicy) -> Result<()> {
    let mut paths = BTreeSet::new();
    for output in &policy.generated_outputs {
        validate_relative(&output.path)?;
        if !paths.insert(output.path.as_str())
            || output.generator.trim().is_empty()
            || !matches!(
                output.license.as_str(),
                "BSD-3-Clause" | "BSD-3-Clause AND CC-BY-SA-3.0"
            )
        {
            return Err(MatrixError::new(format!(
                "invalid generated-output provenance {}",
                output.path
            )));
        }
        let contents = fs::read_to_string(root.join(&output.path))?;
        if !contents.contains("generated")
            || (output.license.contains("CC-BY-SA-3.0") && !contents.contains("CC-BY-SA-3.0"))
        {
            return Err(MatrixError::new(format!(
                "generated output {} does not retain its provenance header",
                output.path
            )));
        }
    }
    Ok(())
}

fn validate_native_policy(policy: &ProvenancePolicy) -> Result<()> {
    let expected = BTreeSet::from([
        "alsa-lib",
        "libopus",
        "ogg-next",
        "openjpeg",
        "skia",
        "vorbis-aotuv-lancer",
    ]);
    let mut actual = BTreeSet::new();
    for component in &policy.native_components {
        if !actual.insert(component.id.as_str())
            || component.version.trim().is_empty()
            || component.license.trim().is_empty()
            || !component.source.starts_with("https://")
            || component.linkage.trim().is_empty()
            || component.obligation.len() < 40
        {
            return Err(MatrixError::new(format!(
                "native component {} has incomplete review metadata",
                component.id
            )));
        }
    }
    if actual != expected {
        return Err(MatrixError::new(format!(
            "native component policy mismatch; expected {expected:?}, found {actual:?}"
        )));
    }
    Ok(())
}

fn validate_provenance_ledgers(root: &Path, policy: &ProvenancePolicy) -> Result<()> {
    let expected = BTreeSet::from([
        "api/public-api.json",
        "programs/upstream-programs.json",
        "tests/upstream-tests.json",
    ]);
    let mut actual = BTreeSet::new();
    for ledger in &policy.provenance_ledgers {
        validate_relative(&ledger.path)?;
        if !actual.insert(ledger.path.as_str())
            || ledger.kind.trim().is_empty()
            || ledger.license != "BSD-3-Clause"
        {
            return Err(MatrixError::new(format!(
                "invalid provenance ledger {}",
                ledger.path
            )));
        }
        let value: Value = serde_json::from_slice(&fs::read(root.join(&ledger.path))?)?;
        let commit = value["upstream_commit"]
            .as_str()
            .or_else(|| value["upstream"]["commit"].as_str());
        if commit != Some(policy.upstream_commit.as_str()) {
            return Err(MatrixError::new(format!(
                "provenance ledger {} is not pinned to {}",
                ledger.path, policy.upstream_commit
            )));
        }
    }
    if actual != expected {
        return Err(MatrixError::new(format!(
            "source-derived provenance ledger mismatch; expected {expected:?}, found {actual:?}"
        )));
    }
    Ok(())
}

fn audit_materials(root: &Path, policy: &ProvenancePolicy) -> Result<()> {
    let expected = policy
        .materials
        .iter()
        .map(|material| (material.path.as_str(), material))
        .collect::<BTreeMap<_, _>>();
    let discovered = discover_materials(root)?;
    let expected_paths = expected.keys().copied().collect::<BTreeSet<_>>();
    let discovered_paths = discovered
        .iter()
        .map(String::as_str)
        .collect::<BTreeSet<_>>();
    if expected_paths != discovered_paths {
        return Err(MatrixError::new(format!(
            "material inventory mismatch; missing {:?}, unknown {:?}",
            expected_paths
                .difference(&discovered_paths)
                .collect::<Vec<_>>(),
            discovered_paths
                .difference(&expected_paths)
                .collect::<Vec<_>>()
        )));
    }
    for path in discovered {
        let bytes = fs::read(root.join(&path))?;
        if sha256(&bytes) != expected[path.as_str()].sha256 {
            return Err(MatrixError::new(format!("material hash changed: {path}")));
        }
    }
    audit_codegen_inventory(root, policy)?;
    audit_bundled_extensions(root, &expected_paths)?;
    Ok(())
}

fn discover_materials(root: &Path) -> Result<Vec<String>> {
    let mut paths = Vec::new();
    for relative in MATERIAL_ROOTS {
        walk_files(root, &root.join(relative), &mut paths)?;
    }
    paths.retain(|path| !path.ends_with("/README.md"));
    paths.sort();
    Ok(paths)
}

fn walk_files(root: &Path, directory: &Path, output: &mut Vec<String>) -> Result<()> {
    let mut entries = fs::read_dir(directory)?.collect::<std::io::Result<Vec<_>>>()?;
    entries.sort_by_key(std::fs::DirEntry::file_name);
    for entry in entries {
        let file_type = entry.file_type()?;
        if file_type.is_symlink() {
            return Err(MatrixError::new(format!(
                "provenance roots may not contain symlinks: {}",
                entry.path().display()
            )));
        }
        if file_type.is_dir() {
            walk_files(root, &entry.path(), output)?;
        } else if file_type.is_file() {
            output.push(relative_utf8(root, &entry.path())?);
        }
    }
    Ok(())
}

fn audit_codegen_inventory(root: &Path, policy: &ProvenancePolicy) -> Result<()> {
    let inventory: Value = serde_json::from_slice(&fs::read(root.join("codegen/sources.json"))?)?;
    let inputs = inventory["inputs"]
        .as_array()
        .ok_or_else(|| MatrixError::new("codegen source inventory has no inputs"))?;
    let materials = policy
        .materials
        .iter()
        .map(|material| (material.path.as_str(), material))
        .collect::<BTreeMap<_, _>>();
    for input in inputs {
        let path = json_string(input, "vendored_path")?;
        let material = materials.get(path).ok_or_else(|| {
            MatrixError::new(format!(
                "codegen input {path} is absent from provenance policy"
            ))
        })?;
        if json_string(input, "sha256")? != material.sha256
            || json_string(input, "license")? != material.license
            || json_string(input, "reference_path")? != material.origin
        {
            return Err(MatrixError::new(format!(
                "codegen and provenance inventories disagree for {path}"
            )));
        }
    }
    Ok(())
}

fn audit_bundled_extensions(root: &Path, materials: &BTreeSet<&str>) -> Result<()> {
    for path in source_paths(root)? {
        let extension = Path::new(&path)
            .extension()
            .and_then(OsStr::to_str)
            .map(str::to_ascii_lowercase);
        if extension
            .as_deref()
            .is_some_and(|value| BUNDLED_EXTENSIONS.contains(&value))
            && !materials.contains(path.as_str())
        {
            return Err(MatrixError::new(format!(
                "bundled binary/asset has no explicit provenance: {path}"
            )));
        }
    }
    Ok(())
}

fn generate_reports(root: &Path, policy: &ProvenancePolicy) -> Result<Reports> {
    let (dependency_manifest, third_party_notices, dependency_count, notice_count) =
        dependency_reports(root)?;
    let native_notices = native_notice(policy).into_bytes();
    let generated_release_files = vec![
        distributed_bytes(DEPENDENCY_MANIFEST_PATH, &dependency_manifest),
        distributed_bytes(THIRD_PARTY_NOTICE_PATH, &third_party_notices),
        distributed_bytes(NATIVE_NOTICE_PATH, &native_notices),
    ];
    let source_files = source_manifest(root)?;
    let cargo_lock_sha256 = sha256(&fs::read(root.join("Cargo.lock"))?);
    let provenance_policy_sha256 = sha256(&fs::read(root.join(POLICY_PATH))?);
    let distribution = DistributionManifest {
        schema: 1,
        cargo_lock_sha256,
        provenance_policy_sha256,
        upstream_repository: &policy.upstream_repository,
        upstream_commit: &policy.upstream_commit,
        source_files,
        generated_release_files,
        materials: &policy.materials,
        generated_outputs: &policy.generated_outputs,
        provenance_ledgers: &policy.provenance_ledgers,
        required_source_notices: &policy.required_source_notices,
        required_binary_notices: &policy.required_binary_notices,
        native_components: &policy.native_components,
    };
    validate_distribution_contract(&distribution)?;
    let source_file_count = distribution.source_files.len();
    Ok(Reports {
        dependency_manifest,
        third_party_notices,
        native_notices,
        distribution_manifest: pretty_json(&distribution)?,
        source_file_count,
        dependency_count,
        dependency_notice_count: notice_count,
    })
}

fn validate_distribution_contract(manifest: &DistributionManifest<'_>) -> Result<()> {
    let included = manifest
        .source_files
        .iter()
        .chain(&manifest.generated_release_files)
        .map(|file| file.path.as_str())
        .collect::<BTreeSet<_>>();
    for notice in manifest
        .required_source_notices
        .iter()
        .chain(manifest.required_binary_notices)
    {
        if !included.contains(notice.as_str()) {
            return Err(MatrixError::new(format!(
                "required distribution notice {notice} is absent from the release manifest"
            )));
        }
    }
    let locked = manifest
        .generated_release_files
        .iter()
        .map(|file| file.path.as_str())
        .collect::<BTreeSet<_>>();
    for required in [
        DEPENDENCY_MANIFEST_PATH,
        THIRD_PARTY_NOTICE_PATH,
        NATIVE_NOTICE_PATH,
    ] {
        if !locked.contains(required) {
            return Err(MatrixError::new(format!(
                "generated release artifact {required} is missing"
            )));
        }
    }
    Ok(())
}

fn dependency_reports(root: &Path) -> Result<(Vec<u8>, Vec<u8>, usize, usize)> {
    let metadata = cargo_metadata(root)?;
    let lock = fs::read_to_string(root.join("Cargo.lock"))?;
    let checksums = lock_checksums(&lock)?;
    let packages = metadata["packages"]
        .as_array()
        .ok_or_else(|| MatrixError::new("cargo metadata has no packages"))?;
    let mut records = Vec::new();
    let mut texts = BTreeMap::<String, NoticeText>::new();
    for package in packages {
        if package["source"].is_null() {
            continue;
        }
        let name = json_string(package, "name")?.to_owned();
        let version = json_string(package, "version")?.to_owned();
        let license = package["license"]
            .as_str()
            .filter(|value| !value.trim().is_empty())
            .ok_or_else(|| MatrixError::new(format!("{name} {version} has no license expression")))?
            .to_owned();
        let checksum = checksums
            .get(&(name.clone(), version.clone()))
            .ok_or_else(|| MatrixError::new(format!("{name} {version} has no locked checksum")))?
            .clone();
        let manifest = PathBuf::from(json_string(package, "manifest_path")?);
        let package_root = manifest
            .parent()
            .ok_or_else(|| MatrixError::new(format!("{name} manifest has no parent")))?;
        let mut notices = package_notices(package_root, package["license_file"].as_str())?;
        if notices.is_empty() {
            notices = sibling_package_notices(packages, package)?;
        }
        if !notices.iter().any(|(path, _)| !path.contains('/')) {
            notices.push((
                "generated-package-license-notice.txt".to_owned(),
                generated_package_notice(package, &license)?,
            ));
        }
        let package_id = format!("{name} {version}");
        let mut package_notice_records = Vec::new();
        for (path, contents) in notices {
            let hash = sha256(contents.as_bytes());
            let text = texts.entry(hash.clone()).or_insert_with(|| NoticeText {
                packages: BTreeSet::new(),
                paths: BTreeSet::new(),
                contents,
            });
            text.packages.insert(package_id.clone());
            text.paths.insert(path.clone());
            package_notice_records.push(PackageNotice { path, sha256: hash });
        }
        package_notice_records.sort_by(|left, right| left.path.cmp(&right.path));
        records.push(DependencyPackage {
            name,
            version,
            checksum,
            license,
            repository: package["repository"].as_str().map(ToOwned::to_owned),
            notice_files: package_notice_records,
        });
    }
    records.sort_by(|left, right| (&left.name, &left.version).cmp(&(&right.name, &right.version)));
    let manifest = DependencyManifest {
        schema: 1,
        cargo_lock_sha256: sha256(lock.as_bytes()),
        packages: records,
    };
    let count = manifest.packages.len();
    let notices = third_party_notice(&manifest, &texts);
    Ok((
        pretty_json(&manifest)?,
        notices.into_bytes(),
        count,
        texts.len(),
    ))
}

fn sibling_package_notices(packages: &[Value], package: &Value) -> Result<Vec<(String, String)>> {
    let repository = package["repository"].as_str().map(normalize_repository);
    let license = package["license"].as_str();
    let manifest = json_string(package, "manifest_path")?;
    let Some(repository) = repository else {
        return Ok(Vec::new());
    };
    let mut candidates = packages.iter().collect::<Vec<_>>();
    candidates.sort_by_key(|candidate| {
        (
            candidate["name"].as_str().unwrap_or_default(),
            candidate["version"].as_str().unwrap_or_default(),
        )
    });
    for candidate in candidates {
        if candidate["source"].is_null()
            || !candidate["license"]
                .as_str()
                .zip(license)
                .is_some_and(|(left, right)| equivalent_license_expression(left, right))
            || candidate["repository"]
                .as_str()
                .map(normalize_repository)
                .as_deref()
                != Some(repository.as_str())
        {
            continue;
        }
        let candidate_manifest = json_string(candidate, "manifest_path")?;
        if candidate_manifest == manifest {
            continue;
        }
        let root = Path::new(candidate_manifest)
            .parent()
            .ok_or_else(|| MatrixError::new("sibling package manifest has no parent"))?;
        let notices = package_notices(root, candidate["license_file"].as_str())?;
        if !notices.is_empty() {
            let package_name = json_string(candidate, "name")?;
            let package_version = json_string(candidate, "version")?;
            return Ok(notices
                .into_iter()
                .map(|(path, contents)| {
                    (
                        format!("repository-license/{package_name}-{package_version}/{path}"),
                        contents,
                    )
                })
                .collect());
        }
    }
    Ok(Vec::new())
}

fn normalize_repository(value: &str) -> String {
    value
        .trim_end_matches('/')
        .trim_end_matches(".git")
        .to_owned()
}

fn equivalent_license_expression(left: &str, right: &str) -> bool {
    fn identifiers(value: &str) -> BTreeSet<&str> {
        value
            .split(|character: char| character.is_ascii_whitespace() || "/()".contains(character))
            .filter(|token| !token.is_empty() && !matches!(*token, "AND" | "OR" | "WITH"))
            .collect()
    }
    identifiers(left) == identifiers(right)
}

fn generated_package_notice(package: &Value, license: &str) -> Result<String> {
    let name = json_string(package, "name")?;
    let version = json_string(package, "version")?;
    let repository = package["repository"].as_str().unwrap_or("not supplied");
    let authors = package["authors"]
        .as_array()
        .into_iter()
        .flatten()
        .filter_map(Value::as_str)
        .collect::<Vec<_>>();
    let holders = if authors.is_empty() {
        "not supplied in Cargo package metadata".to_owned()
    } else {
        authors.join(", ")
    };
    let links = license_identifiers(license)
        .into_iter()
        .map(|identifier| format!("https://spdx.org/licenses/{identifier}.html"))
        .collect::<Vec<_>>()
        .join("\n");
    Ok(format!(
        "Package: {name} {version}\nDeclared license: {license}\nAuthors/copyright attribution from package metadata: {holders}\nSource: {repository}\nSPDX license texts:\n{links}\n\nThe published crate archive contains no discoverable top-level license/notice file. This generated notice preserves all attribution and license metadata supplied by that archive instead of silently omitting the package.\n"
    ))
}

fn license_identifiers(value: &str) -> BTreeSet<&str> {
    value
        .split(|character: char| character.is_ascii_whitespace() || "/()".contains(character))
        .filter(|token| !token.is_empty() && !matches!(*token, "AND" | "OR" | "WITH"))
        .collect()
}

fn cargo_metadata(root: &Path) -> Result<Value> {
    let output = Command::new(super::cargo_program())
        .args([
            "metadata",
            "--locked",
            "--all-features",
            "--format-version",
            "1",
        ])
        .current_dir(root)
        .output()?;
    if !output.status.success() {
        return Err(MatrixError::new(format!(
            "cargo metadata failed during provenance audit: {}",
            String::from_utf8_lossy(&output.stderr).trim()
        )));
    }
    Ok(serde_json::from_slice(&output.stdout)?)
}

fn lock_checksums(lock: &str) -> Result<BTreeMap<(String, String), String>> {
    let mut result = BTreeMap::new();
    let mut package = BTreeMap::<String, String>::new();
    let mut in_package = false;
    for line in lock.lines().chain(std::iter::once("[[package]]")) {
        if line == "[[package]]" {
            if package
                .get("source")
                .is_some_and(|value| value.starts_with("registry+"))
            {
                let name = package
                    .remove("name")
                    .ok_or_else(|| MatrixError::new("locked registry package has no name"))?;
                let version = package
                    .remove("version")
                    .ok_or_else(|| MatrixError::new("locked registry package has no version"))?;
                let checksum = package
                    .remove("checksum")
                    .ok_or_else(|| MatrixError::new(format!("{name} {version} has no checksum")))?;
                result.insert((name, version), checksum);
            }
            package.clear();
            in_package = true;
        } else if in_package
            && let Some((key, value)) = line.split_once(" = ")
            && matches!(key, "name" | "version" | "source" | "checksum")
        {
            package.insert(key.to_owned(), parse_lock_string(value)?);
        }
    }
    Ok(result)
}

fn parse_lock_string(value: &str) -> Result<String> {
    serde_json::from_str(value).map_err(Into::into)
}

fn package_notices(root: &Path, explicit: Option<&str>) -> Result<Vec<(String, String)>> {
    if let Some(archive) = registry_archive(root).filter(|path| path.is_file()) {
        let package = root
            .file_name()
            .and_then(OsStr::to_str)
            .ok_or_else(|| MatrixError::new("registry package path has no UTF-8 name"))?;
        return package_notices_from_archive(&archive, package, explicit);
    }

    let mut candidates = Vec::new();
    collect_notice_files(root, &mut candidates)?;
    if let Some(explicit) = explicit {
        let path = root.join(explicit);
        if path.is_file() {
            candidates.push(path);
        }
    }
    candidates.sort();
    candidates.dedup();
    let mut notices = Vec::new();
    for path in candidates {
        let bytes = fs::read(&path)?;
        if bytes.len() > 2 * 1024 * 1024 {
            return Err(MatrixError::new(format!(
                "package notice is unexpectedly large: {}",
                path.display()
            )));
        }
        let contents = String::from_utf8(bytes).map_err(|_| {
            MatrixError::new(format!("package notice is not UTF-8: {}", path.display()))
        })?;
        notices.push((
            relative_utf8(root, &path)?,
            normalize_notice_text(&contents),
        ));
    }
    Ok(notices)
}

fn registry_archive(root: &Path) -> Option<PathBuf> {
    let package = root.file_name()?.to_str()?;
    let index = root.parent()?.file_name()?;
    let registry = root.parent()?.parent()?.parent()?;
    Some(
        registry
            .join("cache")
            .join(index)
            .join(format!("{package}.crate")),
    )
}

fn package_notices_from_archive(
    archive: &Path,
    package: &str,
    explicit: Option<&str>,
) -> Result<Vec<(String, String)>> {
    let decoder = flate2::read::GzDecoder::new(fs::File::open(archive)?);
    let mut archive = tar::Archive::new(decoder);
    let mut notices = Vec::new();
    for entry in archive.entries()? {
        let mut entry = entry?;
        if !entry.header().entry_type().is_file() {
            continue;
        }
        let path = entry.path()?;
        let mut components = path.components();
        if components.next() != Some(Component::Normal(OsStr::new(package))) {
            return Err(MatrixError::new(format!(
                "package archive entry is outside its {package} root: {}",
                path.display()
            )));
        }
        let relative = components.collect::<PathBuf>();
        let relative_text = relative
            .to_str()
            .ok_or_else(|| MatrixError::new("package archive contains a non-UTF-8 path"))?;
        validate_relative(relative_text)?;
        let depth = relative.components().count();
        let selected = explicit == Some(relative_text)
            || (depth <= 3
                && relative
                    .file_name()
                    .and_then(OsStr::to_str)
                    .is_some_and(is_notice_name));
        if !selected {
            continue;
        }
        let size = entry.header().size()?;
        if size > 2 * 1024 * 1024 {
            return Err(MatrixError::new(format!(
                "package notice is unexpectedly large: {relative_text}"
            )));
        }
        let mut contents = String::new();
        entry.read_to_string(&mut contents).map_err(|error| {
            MatrixError::new(format!(
                "package notice is not UTF-8 ({relative_text}): {error}"
            ))
        })?;
        notices.push((relative_text.to_owned(), normalize_notice_text(&contents)));
    }
    notices.sort_by(|left, right| left.0.cmp(&right.0));
    notices.dedup_by(|left, right| left.0 == right.0);
    Ok(notices)
}

fn is_notice_name(name: &str) -> bool {
    let name = name.to_ascii_lowercase();
    [
        "license",
        "licence",
        "copying",
        "notice",
        "copyright",
        "unlicense",
    ]
    .iter()
    .any(|prefix| name.starts_with(prefix))
}

fn collect_notice_files(directory: &Path, output: &mut Vec<PathBuf>) -> Result<()> {
    // Cargo build scripts may populate nested source directories (rust-skia
    // creates `skia/` in its registry checkout). Only package-root notices and
    // Cargo's explicit `license_file` are immutable package metadata.
    let mut entries = fs::read_dir(directory)?.collect::<std::io::Result<Vec<_>>>()?;
    entries.sort_by_key(std::fs::DirEntry::file_name);
    for entry in entries {
        let path = entry.path();
        let kind = entry.file_type()?;
        if kind.is_file() && is_notice_name(&entry.file_name().to_string_lossy()) {
            output.push(path);
        }
    }
    Ok(())
}

fn third_party_notice(
    manifest: &DependencyManifest,
    texts: &BTreeMap<String, NoticeText>,
) -> String {
    let mut output = String::from(
        "# Locked Rust dependency notices\n\n\
Generated by `metacrate-ci-matrix provenance-report` from `Cargo.lock` and \
`cargo metadata --locked --all-features`. Do not edit by hand. Package source \
checksums bind this inventory to crates.io archives; notice hashes bind every \
included license text.\n\n\
This document must accompany source and binary distributions. A package's \
presence here does not mean every target or feature links it.\n\n\
## Package inventory\n\n\
| Package | License expression | crates.io checksum | Notice hashes |\n\
| --- | --- | --- | --- |\n",
    );
    for package in &manifest.packages {
        let hashes = package
            .notice_files
            .iter()
            .map(|notice| format!("`{}`", notice.sha256))
            .collect::<Vec<_>>()
            .join("<br>");
        let _ = writeln!(
            output,
            "| `{} {}` | `{}` | `{}` | {} |",
            package.name, package.version, package.license, package.checksum, hashes
        );
    }
    output.push_str("\n## License and notice texts\n");
    for (hash, text) in texts {
        let packages = text.packages.iter().cloned().collect::<Vec<_>>().join(", ");
        let paths = text.paths.iter().cloned().collect::<Vec<_>>().join(", ");
        let _ = write!(
            output,
            "\n### `{hash}`\n\nPackages: {packages}<br>\nSource filenames: {paths}\n\n<pre>\n{}\n</pre>\n",
            html_escape(&text.contents)
        );
    }
    output
}

fn native_notice(policy: &ProvenancePolicy) -> String {
    let mut output = String::from(
        "# Native component license notices\n\n\
Generated by `metacrate-ci-matrix provenance-report` from \
`ci/provenance-policy.json`. Do not edit by hand. This inventory distinguishes \
system libraries MetaCrate does not bundle from native code that optional Rust \
features can compile or link into an artifact.\n\n\
| Component | Version | License | Linkage | Bundled by feature | Source |\n\
| --- | --- | --- | --- | --- | --- |\n",
    );
    for component in &policy.native_components {
        let _ = writeln!(
            output,
            "| {} | {} | `{}` | `{}` | {} | <{}> |",
            component.id,
            component.version,
            component.license,
            component.linkage,
            if component.bundled { "yes" } else { "no" },
            component.source
        );
    }
    output.push_str("\n## Distribution obligations\n");
    for component in &policy.native_components {
        let _ = writeln!(output, "\n- **{}:** {}", component.id, component.obligation);
    }
    output.push_str(
        "\nThe complete package-specific texts for Skia, Vorbis/aoTuV/Lancer, and \
libogg are included in `THIRD-PARTY-NOTICES.md` because those sources enter \
through locked crates. System OpenJPEG, Opus, and ALSA binaries are not \
redistributed by MetaCrate; a downstream distributor that supplies them must \
add the exact notices from the supplied native packages.\n",
    );
    output
}

fn source_manifest(root: &Path) -> Result<Vec<DistributedFile>> {
    let mut output = Vec::new();
    for path in source_paths(root)? {
        if GENERATED_RELEASE_PATHS.contains(&path.as_str())
            || path == "ci/evidence/provenance-audit.json"
        {
            continue;
        }
        let bytes = fs::read(root.join(&path))?;
        output.push(distributed_bytes(&path, &bytes));
    }
    output.sort_by(|left, right| left.path.cmp(&right.path));
    Ok(output)
}

fn source_paths(root: &Path) -> Result<Vec<String>> {
    let output = Command::new("git")
        .args([
            "ls-files",
            "-z",
            "--cached",
            "--others",
            "--exclude-standard",
        ])
        .current_dir(root)
        .output()?;
    if !output.status.success() {
        return Err(MatrixError::new(format!(
            "git ls-files failed during source manifest generation: {}",
            String::from_utf8_lossy(&output.stderr).trim()
        )));
    }
    let mut paths = output
        .stdout
        .split(|byte| *byte == 0)
        .filter(|value| !value.is_empty())
        .map(|value| {
            String::from_utf8(value.to_vec())
                .map_err(|_| MatrixError::new("source archive contains a non-UTF-8 path"))
        })
        .collect::<Result<Vec<_>>>()?;
    // `git ls-files --cached` retains index entries for working-tree deletions
    // until the consolidation commit is created. A source distribution always
    // describes files that actually exist, including during a pre-commit audit.
    paths.retain(|path| root.join(path).is_file());
    for path in &paths {
        validate_relative(path)?;
    }
    paths.sort();
    paths.dedup();
    Ok(paths)
}

fn distributed_bytes(path: &str, bytes: &[u8]) -> DistributedFile {
    DistributedFile {
        path: path.to_owned(),
        bytes: u64::try_from(bytes.len()).unwrap_or(u64::MAX),
        sha256: sha256(bytes),
    }
}

fn write_report(root: &Path, relative: &str, bytes: &[u8]) -> Result<()> {
    let path = root.join(relative);
    if let Some(parent) = path.parent() {
        fs::create_dir_all(parent)?;
    }
    fs::write(path, bytes)?;
    Ok(())
}

fn compare_report(root: &Path, relative: &str, expected: &[u8]) -> Result<()> {
    let path = root.join(relative);
    let actual = fs::read(&path).map_err(|error| {
        MatrixError::new(format!("generated report {}: {error}", path.display()))
    })?;
    if actual != expected {
        if let Some(directory) = std::env::var_os("METACRATE_PROVENANCE_DIAGNOSTICS_DIR") {
            let directory = PathBuf::from(directory);
            fs::create_dir_all(&directory)?;
            let name = path
                .file_name()
                .ok_or_else(|| MatrixError::new("generated report path has no file name"))?;
            fs::write(directory.join(name), expected)?;
        }
        return Err(MatrixError::new(format!(
            "{relative} is stale (checked-in sha256 {}, generated sha256 {}); run `cargo run --locked -p metacrate-ci-matrix -- provenance-report`",
            sha256(&actual),
            sha256(expected),
        )));
    }
    Ok(())
}

fn write_new_json<T: Serialize>(path: &Path, value: &T) -> Result<()> {
    if let Some(parent) = path.parent() {
        fs::create_dir_all(parent)?;
    }
    let mut file = OpenOptions::new().write(true).create_new(true).open(path)?;
    serde_json::to_writer_pretty(&mut file, value)?;
    file.write_all(b"\n")?;
    file.sync_all()?;
    Ok(())
}

fn pretty_json<T: Serialize>(value: &T) -> Result<Vec<u8>> {
    let mut bytes = serde_json::to_vec_pretty(value)?;
    bytes.push(b'\n');
    Ok(bytes)
}

fn html_escape(value: &str) -> String {
    value
        .replace('&', "&amp;")
        .replace('<', "&lt;")
        .replace('>', "&gt;")
}

fn normalize_notice_text(value: &str) -> String {
    let unix = value.replace("\r\n", "\n").replace('\r', "\n");
    let mut output = unix
        .lines()
        .map(str::trim_end)
        .collect::<Vec<_>>()
        .join("\n");
    output.push('\n');
    output
}

fn relative_utf8(root: &Path, path: &Path) -> Result<String> {
    path.strip_prefix(root)
        .map_err(|_| MatrixError::new(format!("{} is outside the workspace", path.display())))?
        .to_str()
        .map(|value| value.replace('\\', "/"))
        .ok_or_else(|| MatrixError::new(format!("{} is not a UTF-8 path", path.display())))
}

fn validate_relative(value: &str) -> Result<()> {
    let path = Path::new(value);
    if value.is_empty()
        || path.is_absolute()
        || path
            .components()
            .any(|component| !matches!(component, Component::Normal(_)))
    {
        return Err(MatrixError::new(format!("unsafe provenance path {value}")));
    }
    Ok(())
}

fn validate_hash(value: &str) -> Result<()> {
    if value.len() == 64 && value.bytes().all(|byte| byte.is_ascii_hexdigit()) {
        Ok(())
    } else {
        Err(MatrixError::new(format!("invalid SHA-256 {value}")))
    }
}

fn json_string<'a>(value: &'a Value, key: &str) -> Result<&'a str> {
    value[key]
        .as_str()
        .ok_or_else(|| MatrixError::new(format!("JSON field {key} is not a string")))
}

fn sha256(bytes: &[u8]) -> String {
    const HEX: &[u8; 16] = b"0123456789abcdef";
    let mut output = String::with_capacity(64);
    for byte in Sha256::digest(bytes) {
        output.push(char::from(HEX[usize::from(byte >> 4)]));
        output.push(char::from(HEX[usize::from(byte & 0x0f)]));
    }
    output
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn checked_in_provenance_reports_match_locked_inputs() {
        let root = super::super::workspace_root(Path::new(env!("CARGO_MANIFEST_DIR"))).unwrap();
        let evidence = std::env::temp_dir().join(format!(
            "metacrate-provenance-audit-{}.json",
            std::process::id()
        ));
        let _ = fs::remove_file(&evidence);
        audit_provenance(&root, &evidence).unwrap();
        let contents = fs::read_to_string(&evidence).unwrap();
        assert!(contents.contains("\"unknown_materials\": 0"));
        assert!(contents.contains("\"unknown_bundled_assets\": 0"));
        assert!(contents.contains("\"status\": \"ok\""));
        fs::remove_file(evidence).unwrap();
    }

    #[test]
    fn lock_parser_keeps_registry_checksums() {
        let lock = r#"version = 4

[[package]]
name = "demo"
version = "1.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "abcdef"

[[package]]
name = "local"
version = "0.0.1"
"#;
        assert_eq!(
            lock_checksums(lock).unwrap()[&("demo".to_owned(), "1.2.3".to_owned())],
            "abcdef"
        );
    }

    #[test]
    fn package_notice_scan_ignores_build_generated_subtrees() {
        let root =
            std::env::temp_dir().join(format!("metacrate-package-notices-{}", std::process::id()));
        let _ = fs::remove_dir_all(&root);
        fs::create_dir_all(root.join("generated")).unwrap();
        fs::write(root.join("LICENSE"), "package license\n").unwrap();
        fs::write(root.join("generated/LICENSE"), "generated license\n").unwrap();
        fs::write(root.join("generated/NOTICE.explicit"), "explicit notice\n").unwrap();

        let notices = package_notices(&root, Some("generated/NOTICE.explicit")).unwrap();
        assert_eq!(
            notices
                .iter()
                .map(|(path, _)| path.as_str())
                .collect::<Vec<_>>(),
            ["LICENSE", "generated/NOTICE.explicit"]
        );
        fs::remove_dir_all(root).unwrap();
    }

    #[test]
    fn package_notice_scan_preserves_nested_files_from_registry_archive() {
        let base =
            std::env::temp_dir().join(format!("metacrate-package-archive-{}", std::process::id()));
        let root = base.join("registry/src/index/demo-1.0.0");
        let archive_path = base.join("registry/cache/index/demo-1.0.0.crate");
        let _ = fs::remove_dir_all(&base);
        fs::create_dir_all(root.join("generated")).unwrap();
        fs::create_dir_all(archive_path.parent().unwrap()).unwrap();
        fs::write(root.join("generated/LICENSE"), "build output\n").unwrap();

        let encoder = flate2::write::GzEncoder::new(
            fs::File::create(&archive_path).unwrap(),
            flate2::Compression::default(),
        );
        let mut archive = tar::Builder::new(encoder);
        for (path, contents) in [
            ("demo-1.0.0/LICENSE", "package license\n"),
            ("demo-1.0.0/vendor/COPYING", "vendored license\n"),
            ("demo-1.0.0/docs/legal/terms.txt", "explicit terms\n"),
        ] {
            let mut header = tar::Header::new_gnu();
            header.set_size(contents.len() as u64);
            header.set_mode(0o644);
            header.set_cksum();
            archive
                .append_data(&mut header, path, contents.as_bytes())
                .unwrap();
        }
        archive.into_inner().unwrap().finish().unwrap();

        let notices = package_notices(&root, Some("docs/legal/terms.txt")).unwrap();
        assert_eq!(
            notices
                .iter()
                .map(|(path, _)| path.as_str())
                .collect::<Vec<_>>(),
            ["LICENSE", "docs/legal/terms.txt", "vendor/COPYING"]
        );
        fs::remove_dir_all(base).unwrap();
    }

    #[test]
    fn unsafe_paths_are_rejected() {
        for path in ["", "../escape", "a/../b", "/absolute"] {
            assert!(validate_relative(path).is_err(), "accepted {path}");
        }
        assert!(validate_relative("release/NOTICE.md").is_ok());
    }
}