Consolidate required CI gate (#115)

tools/ci-matrix/src/lib.rs

@@ -13,6 +13,7 @@ use std::time::{SystemTime, UNIX_EPOCH};
 
 mod api_surface;
 mod artifact;
+mod ci_gate;
 mod dependency;
 mod documentation;
 mod provenance;
@@ -20,13 +21,14 @@ mod release_candidate;
 
 pub use api_surface::{audit_api_surface, write_api_baseline};
 pub use artifact::audit_artifacts;
+pub use ci_gate::{audit_consolidated_ci, run_release_gate, run_required_gate};
 pub use dependency::audit_dependencies;
 pub use documentation::{audit_documentation, write_documentation_report};
 pub use provenance::{audit_provenance, write_provenance_reports};
 pub use release_candidate::audit_release_candidate;
 
 pub const MATRIX_PATH: &str = "ci/release-matrix.json";
-const WORKFLOW_PATH: &str = ".gitea/workflows/release-matrix.yml";
+const WORKFLOW_PATH: &str = ".gitea/workflows/release.yml";
 const REQUIRED_PROFILES: [&str; 7] = [
     "linux-msrv-portable",
     "linux-stable-default",
@@ -458,25 +460,16 @@ fn audit_cargo_features(root: &Path) -> Result<()> {
 
 fn audit_workflows(root: &Path, matrix: &ReleaseMatrix) -> Result<()> {
     let workflow = fs::read_to_string(root.join(WORKFLOW_PATH))?;
-    for profile in &matrix.profiles {
-        if !workflow.contains(&format!("profile: {}", profile.id)) {
-            return Err(MatrixError::new(format!(
-                "workflow does not schedule profile {}",
-                profile.id
-            )));
-        }
-    }
-    if !workflow.contains("~/.cargo/registry")
+    if matrix.profiles.is_empty()
+        || !workflow.contains("release-gate")
+        || !workflow.contains("~/.cargo/registry")
         || !workflow.contains("~/.cargo/git")
-        || workflow.lines().any(|line| {
-            let trimmed = line.trim();
-            trimmed == "target" || trimmed.starts_with("target/") || trimmed.contains("/target/")
-        })
     {
         return Err(MatrixError::new(
-            "workflow must cache Cargo downloads without caching build target directories",
+            "release workflow must invoke the Rust gate and cache Cargo downloads",
         ));
     }
+    audit_consolidated_ci(root)?;
     let workflows = root.join(".gitea/workflows");
     for entry in fs::read_dir(workflows)? {
         let path = entry?.path();

tools/ci-matrix/src/main.rs

@@ -1,7 +1,8 @@
 use metacrate_ci_matrix::{
-    audit, audit_api_surface, audit_artifacts, audit_dependencies, audit_documentation,
-    audit_provenance, audit_release_candidate, load, run, workspace_root, write_api_baseline,
-    write_documentation_report, write_provenance_reports,
+    audit, audit_api_surface, audit_artifacts, audit_consolidated_ci, audit_dependencies,
+    audit_documentation, audit_provenance, audit_release_candidate, load, run, run_release_gate,
+    run_required_gate, workspace_root, write_api_baseline, write_documentation_report,
+    write_provenance_reports,
 };
 use std::path::{Path, PathBuf};
 
@@ -12,12 +13,23 @@ fn main() {
     }
 }
 
+#[allow(clippy::too_many_lines)] // Flat CLI command routing is clearer than nested dispatch.
 fn execute() -> Result<(), Box<dyn std::error::Error>> {
     let current = std::env::current_dir()?;
     let root = workspace_root(&current).ok_or("could not locate the Cargo workspace")?;
     let matrix = load(&root)?;
     let mut arguments = std::env::args().skip(1);
     match arguments.next().as_deref() {
+        Some("ci-audit") if arguments.next().is_none() => {
+            audit_consolidated_ci(&root)?;
+            println!("consolidated CI coverage and workflow split: ok");
+        }
+        Some("required-gate") => {
+            gate_command(&root, arguments, "required", run_required_gate)?;
+        }
+        Some("release-gate") => {
+            gate_command(&root, arguments, "release", run_release_gate)?;
+        }
         Some("audit") if arguments.next().is_none() => {
             audit(&root, &matrix)?;
             println!(
@@ -101,7 +113,7 @@ fn execute() -> Result<(), Box<dyn std::error::Error>> {
         }
         _ => {
             return Err(
-                "usage: ci-matrix audit | run PROFILE --evidence FILE | dependency-audit --evidence FILE | artifact-audit --artifact-dir DIR --package-dir DIR --evidence FILE | release-candidate-audit --evidence FILE | documentation-report | documentation-audit --evidence FILE | api-baseline-write | api-audit --evidence FILE | provenance-report | provenance-audit --evidence FILE"
+                "usage: ci-matrix ci-audit | required-gate --evidence FILE | release-gate --evidence FILE | audit | run PROFILE --evidence FILE | dependency-audit --evidence FILE | artifact-audit --artifact-dir DIR --package-dir DIR --evidence FILE | release-candidate-audit --evidence FILE | documentation-report | documentation-audit --evidence FILE | api-baseline-write | api-audit --evidence FILE | provenance-report | provenance-audit --evidence FILE"
                     .into(),
             );
         }
@@ -109,6 +121,27 @@ fn execute() -> Result<(), Box<dyn std::error::Error>> {
     Ok(())
 }
 
+fn gate_command(
+    root: &Path,
+    mut arguments: impl Iterator<Item = String>,
+    gate: &str,
+    run_gate: fn(&Path, &Path) -> metacrate_ci_matrix::Result<()>,
+) -> Result<(), Box<dyn std::error::Error>> {
+    let flag = arguments
+        .next()
+        .ok_or_else(|| format!("{gate}-gate requires --evidence FILE"))?;
+    let evidence = arguments
+        .next()
+        .ok_or_else(|| format!("{gate}-gate requires --evidence FILE"))?;
+    if flag != "--evidence" || arguments.next().is_some() {
+        return Err(format!("usage: ci-matrix {gate}-gate --evidence FILE").into());
+    }
+    let evidence = absolute_or_rooted(root, &evidence);
+    run_gate(root, &evidence)?;
+    println!("{gate} CI gate: ok ({})", evidence.display());
+    Ok(())
+}
+
 fn release_candidate_audit_command(
     root: &Path,
     mut arguments: impl Iterator<Item = String>,

tools/ci-matrix/src/provenance.rs

@@ -6,7 +6,7 @@ use std::collections::{BTreeMap, BTreeSet};
 use std::ffi::OsStr;
 use std::fmt::Write as _;
 use std::fs::{self, OpenOptions};
-use std::io::Write as _;
+use std::io::{Read as _, Write as _};
 use std::path::{Component, Path, PathBuf};
 use std::process::Command;
 use std::time::{SystemTime, UNIX_EPOCH};
@@ -916,8 +916,16 @@ fn parse_lock_string(value: &str) -> Result<String> {
 }
 
 fn package_notices(root: &Path, explicit: Option<&str>) -> Result<Vec<(String, String)>> {
+    if let Some(archive) = registry_archive(root).filter(|path| path.is_file()) {
+        let package = root
+            .file_name()
+            .and_then(OsStr::to_str)
+            .ok_or_else(|| MatrixError::new("registry package path has no UTF-8 name"))?;
+        return package_notices_from_archive(&archive, package, explicit);
+    }
+
     let mut candidates = Vec::new();
-    collect_notice_files(root, 0, &mut candidates)?;
+    collect_notice_files(root, &mut candidates)?;
     if let Some(explicit) = explicit {
         let path = root.join(explicit);
         if path.is_file() {
@@ -946,31 +954,98 @@ fn package_notices(root: &Path, explicit: Option<&str>) -> Result<Vec<(String, S
     Ok(notices)
 }
 
-fn collect_notice_files(directory: &Path, depth: u8, output: &mut Vec<PathBuf>) -> Result<()> {
-    if depth > 2 {
-        return Ok(());
+fn registry_archive(root: &Path) -> Option<PathBuf> {
+    let package = root.file_name()?.to_str()?;
+    let index = root.parent()?.file_name()?;
+    let registry = root.parent()?.parent()?.parent()?;
+    Some(
+        registry
+            .join("cache")
+            .join(index)
+            .join(format!("{package}.crate")),
+    )
+}
+
+fn package_notices_from_archive(
+    archive: &Path,
+    package: &str,
+    explicit: Option<&str>,
+) -> Result<Vec<(String, String)>> {
+    let decoder = flate2::read::GzDecoder::new(fs::File::open(archive)?);
+    let mut archive = tar::Archive::new(decoder);
+    let mut notices = Vec::new();
+    for entry in archive.entries()? {
+        let mut entry = entry?;
+        if !entry.header().entry_type().is_file() {
+            continue;
+        }
+        let path = entry.path()?;
+        let mut components = path.components();
+        if components.next() != Some(Component::Normal(OsStr::new(package))) {
+            return Err(MatrixError::new(format!(
+                "package archive entry is outside its {package} root: {}",
+                path.display()
+            )));
+        }
+        let relative = components.collect::<PathBuf>();
+        let relative_text = relative
+            .to_str()
+            .ok_or_else(|| MatrixError::new("package archive contains a non-UTF-8 path"))?;
+        validate_relative(relative_text)?;
+        let depth = relative.components().count();
+        let selected = explicit == Some(relative_text)
+            || (depth <= 3
+                && relative
+                    .file_name()
+                    .and_then(OsStr::to_str)
+                    .is_some_and(is_notice_name));
+        if !selected {
+            continue;
+        }
+        let size = entry.header().size()?;
+        if size > 2 * 1024 * 1024 {
+            return Err(MatrixError::new(format!(
+                "package notice is unexpectedly large: {relative_text}"
+            )));
+        }
+        let mut contents = String::new();
+        entry.read_to_string(&mut contents).map_err(|error| {
+            MatrixError::new(format!(
+                "package notice is not UTF-8 ({relative_text}): {error}"
+            ))
+        })?;
+        notices.push((relative_text.to_owned(), normalize_notice_text(&contents)));
     }
+    notices.sort_by(|left, right| left.0.cmp(&right.0));
+    notices.dedup_by(|left, right| left.0 == right.0);
+    Ok(notices)
+}
+
+fn is_notice_name(name: &str) -> bool {
+    let name = name.to_ascii_lowercase();
+    [
+        "license",
+        "licence",
+        "copying",
+        "notice",
+        "copyright",
+        "unlicense",
+    ]
+    .iter()
+    .any(|prefix| name.starts_with(prefix))
+}
+
+fn collect_notice_files(directory: &Path, output: &mut Vec<PathBuf>) -> Result<()> {
+    // Cargo build scripts may populate nested source directories (rust-skia
+    // creates `skia/` in its registry checkout). Only package-root notices and
+    // Cargo's explicit `license_file` are immutable package metadata.
     let mut entries = fs::read_dir(directory)?.collect::<std::io::Result<Vec<_>>>()?;
     entries.sort_by_key(std::fs::DirEntry::file_name);
     for entry in entries {
         let path = entry.path();
         let kind = entry.file_type()?;
-        let name = entry.file_name().to_string_lossy().to_ascii_lowercase();
-        if kind.is_file()
-            && [
-                "license",
-                "licence",
-                "copying",
-                "notice",
-                "copyright",
-                "unlicense",
-            ]
-            .iter()
-            .any(|prefix| name.starts_with(prefix))
-        {
+        if kind.is_file() && is_notice_name(&entry.file_name().to_string_lossy()) {
             output.push(path);
-        } else if kind.is_dir() && depth < 2 {
-            collect_notice_files(&path, depth + 1, output)?;
         }
     }
     Ok(())
@@ -1095,6 +1170,10 @@ fn source_paths(root: &Path) -> Result<Vec<String>> {
                 .map_err(|_| MatrixError::new("source archive contains a non-UTF-8 path"))
         })
         .collect::<Result<Vec<_>>>()?;
+    // `git ls-files --cached` retains index entries for working-tree deletions
+    // until the consolidation commit is created. A source distribution always
+    // describes files that actually exist, including during a pre-commit audit.
+    paths.retain(|path| root.join(path).is_file());
     for path in &paths {
         validate_relative(path)?;
     }
@@ -1126,8 +1205,18 @@ fn compare_report(root: &Path, relative: &str, expected: &[u8]) -> Result<()> {
         MatrixError::new(format!("generated report {}: {error}", path.display()))
     })?;
     if actual != expected {
+        if let Some(directory) = std::env::var_os("METACRATE_PROVENANCE_DIAGNOSTICS_DIR") {
+            let directory = PathBuf::from(directory);
+            fs::create_dir_all(&directory)?;
+            let name = path
+                .file_name()
+                .ok_or_else(|| MatrixError::new("generated report path has no file name"))?;
+            fs::write(directory.join(name), expected)?;
+        }
         return Err(MatrixError::new(format!(
-            "{relative} is stale; run `cargo run --locked -p metacrate-ci-matrix -- provenance-report`"
+            "{relative} is stale (checked-in sha256 {}, generated sha256 {}); run `cargo run --locked -p metacrate-ci-matrix -- provenance-report`",
+            sha256(&actual),
+            sha256(expected),
         )));
     }
     Ok(())
@@ -1253,6 +1342,69 @@ version = "0.0.1"
         );
     }
 
+    #[test]
+    fn package_notice_scan_ignores_build_generated_subtrees() {
+        let root =
+            std::env::temp_dir().join(format!("metacrate-package-notices-{}", std::process::id()));
+        let _ = fs::remove_dir_all(&root);
+        fs::create_dir_all(root.join("generated")).unwrap();
+        fs::write(root.join("LICENSE"), "package license\n").unwrap();
+        fs::write(root.join("generated/LICENSE"), "generated license\n").unwrap();
+        fs::write(root.join("generated/NOTICE.explicit"), "explicit notice\n").unwrap();
+
+        let notices = package_notices(&root, Some("generated/NOTICE.explicit")).unwrap();
+        assert_eq!(
+            notices
+                .iter()
+                .map(|(path, _)| path.as_str())
+                .collect::<Vec<_>>(),
+            ["LICENSE", "generated/NOTICE.explicit"]
+        );
+        fs::remove_dir_all(root).unwrap();
+    }
+
+    #[test]
+    fn package_notice_scan_preserves_nested_files_from_registry_archive() {
+        let base =
+            std::env::temp_dir().join(format!("metacrate-package-archive-{}", std::process::id()));
+        let root = base.join("registry/src/index/demo-1.0.0");
+        let archive_path = base.join("registry/cache/index/demo-1.0.0.crate");
+        let _ = fs::remove_dir_all(&base);
+        fs::create_dir_all(root.join("generated")).unwrap();
+        fs::create_dir_all(archive_path.parent().unwrap()).unwrap();
+        fs::write(root.join("generated/LICENSE"), "build output\n").unwrap();
+
+        let encoder = flate2::write::GzEncoder::new(
+            fs::File::create(&archive_path).unwrap(),
+            flate2::Compression::default(),
+        );
+        let mut archive = tar::Builder::new(encoder);
+        for (path, contents) in [
+            ("demo-1.0.0/LICENSE", "package license\n"),
+            ("demo-1.0.0/vendor/COPYING", "vendored license\n"),
+            ("demo-1.0.0/docs/legal/terms.txt", "explicit terms\n"),
+        ] {
+            let mut header = tar::Header::new_gnu();
+            header.set_size(contents.len() as u64);
+            header.set_mode(0o644);
+            header.set_cksum();
+            archive
+                .append_data(&mut header, path, contents.as_bytes())
+                .unwrap();
+        }
+        archive.into_inner().unwrap().finish().unwrap();
+
+        let notices = package_notices(&root, Some("docs/legal/terms.txt")).unwrap();
+        assert_eq!(
+            notices
+                .iter()
+                .map(|(path, _)| path.as_str())
+                .collect::<Vec<_>>(),
+            ["LICENSE", "docs/legal/terms.txt", "vendor/COPYING"]
+        );
+        fs::remove_dir_all(base).unwrap();
+    }
+
     #[test]
     fn unsafe_paths_are_rejected() {
         for path in ["", "../escape", "a/../b", "/absolute"] {