Consolidate required CI gate (#115)

tools/ci-matrix/src/provenance.rs

@@ -6,7 +6,7 @@ use std::collections::{BTreeMap, BTreeSet};
 use std::ffi::OsStr;
 use std::fmt::Write as _;
 use std::fs::{self, OpenOptions};
-use std::io::Write as _;
+use std::io::{Read as _, Write as _};
 use std::path::{Component, Path, PathBuf};
 use std::process::Command;
 use std::time::{SystemTime, UNIX_EPOCH};
@@ -859,6 +859,20 @@ fn license_identifiers(value: &str) -> BTreeSet<&str> {
 }
 
 fn cargo_metadata(root: &Path) -> Result<Value> {
+    // Fetch without `--target` so Cargo materializes the immutable archives for
+    // every target represented by Cargo.lock. Reading notice files from those
+    // archives makes the report independent of the host architecture and of
+    // build-script changes to extracted registry sources.
+    let fetch = Command::new(super::cargo_program())
+        .args(["fetch", "--locked"])
+        .current_dir(root)
+        .output()?;
+    if !fetch.status.success() {
+        return Err(MatrixError::new(format!(
+            "cargo fetch failed during provenance audit: {}",
+            String::from_utf8_lossy(&fetch.stderr).trim()
+        )));
+    }
     let output = Command::new(super::cargo_program())
         .args([
             "metadata",
@@ -916,8 +930,22 @@ fn parse_lock_string(value: &str) -> Result<String> {
 }
 
 fn package_notices(root: &Path, explicit: Option<&str>) -> Result<Vec<(String, String)>> {
+    if let Some(archive) = registry_archive(root) {
+        if !archive.is_file() {
+            return Err(MatrixError::new(format!(
+                "registry archive is missing after cargo fetch: {}",
+                archive.display()
+            )));
+        }
+        let package = root
+            .file_name()
+            .and_then(OsStr::to_str)
+            .ok_or_else(|| MatrixError::new("registry package path has no UTF-8 name"))?;
+        return package_notices_from_archive(&archive, package, explicit);
+    }
+
     let mut candidates = Vec::new();
-    collect_notice_files(root, 0, &mut candidates)?;
+    collect_notice_files(root, &mut candidates)?;
     if let Some(explicit) = explicit {
         let path = root.join(explicit);
         if path.is_file() {
@@ -946,31 +974,105 @@ fn package_notices(root: &Path, explicit: Option<&str>) -> Result<Vec<(String, S
     Ok(notices)
 }
 
-fn collect_notice_files(directory: &Path, depth: u8, output: &mut Vec<PathBuf>) -> Result<()> {
-    if depth > 2 {
-        return Ok(());
+fn registry_archive(root: &Path) -> Option<PathBuf> {
+    let package = root.file_name()?.to_str()?;
+    let index = root.parent()?.file_name()?;
+    let source = root.parent()?.parent()?;
+    if source.file_name()? != OsStr::new("src") {
+        return None;
     }
+    let registry = source.parent()?;
+    if registry.file_name()? != OsStr::new("registry") {
+        return None;
+    }
+    Some(
+        registry
+            .join("cache")
+            .join(index)
+            .join(format!("{package}.crate")),
+    )
+}
+
+fn package_notices_from_archive(
+    archive: &Path,
+    package: &str,
+    explicit: Option<&str>,
+) -> Result<Vec<(String, String)>> {
+    let decoder = flate2::read::GzDecoder::new(fs::File::open(archive)?);
+    let mut archive = tar::Archive::new(decoder);
+    let mut notices = Vec::new();
+    for entry in archive.entries()? {
+        let mut entry = entry?;
+        if !entry.header().entry_type().is_file() {
+            continue;
+        }
+        let path = entry.path()?;
+        let mut components = path.components();
+        if components.next() != Some(Component::Normal(OsStr::new(package))) {
+            return Err(MatrixError::new(format!(
+                "package archive entry is outside its {package} root: {}",
+                path.display()
+            )));
+        }
+        let relative = components.collect::<PathBuf>();
+        let relative_text = relative
+            .to_str()
+            .ok_or_else(|| MatrixError::new("package archive contains a non-UTF-8 path"))?;
+        validate_relative(relative_text)?;
+        let depth = relative.components().count();
+        let selected = explicit == Some(relative_text)
+            || (depth <= 3
+                && relative
+                    .file_name()
+                    .and_then(OsStr::to_str)
+                    .is_some_and(is_notice_name));
+        if !selected {
+            continue;
+        }
+        let size = entry.header().size()?;
+        if size > 2 * 1024 * 1024 {
+            return Err(MatrixError::new(format!(
+                "package notice is unexpectedly large: {relative_text}"
+            )));
+        }
+        let mut contents = String::new();
+        entry.read_to_string(&mut contents).map_err(|error| {
+            MatrixError::new(format!(
+                "package notice is not UTF-8 ({relative_text}): {error}"
+            ))
+        })?;
+        notices.push((relative_text.to_owned(), normalize_notice_text(&contents)));
+    }
+    notices.sort_by(|left, right| left.0.cmp(&right.0));
+    notices.dedup_by(|left, right| left.0 == right.0);
+    Ok(notices)
+}
+
+fn is_notice_name(name: &str) -> bool {
+    let name = name.to_ascii_lowercase();
+    [
+        "license",
+        "licence",
+        "copying",
+        "notice",
+        "copyright",
+        "unlicense",
+    ]
+    .iter()
+    .any(|prefix| name.starts_with(prefix))
+}
+
+fn collect_notice_files(directory: &Path, output: &mut Vec<PathBuf>) -> Result<()> {
+    // Cargo build scripts may populate nested source directories (rust-skia
+    // creates `skia/` in its registry checkout). Only package-root notices and
+    // Cargo's explicit `license_file` are immutable package metadata.
     let mut entries = fs::read_dir(directory)?.collect::<std::io::Result<Vec<_>>>()?;
     entries.sort_by_key(std::fs::DirEntry::file_name);
     for entry in entries {
         let path = entry.path();
         let kind = entry.file_type()?;
-        let name = entry.file_name().to_string_lossy().to_ascii_lowercase();
-        if kind.is_file()
-            && [
-                "license",
-                "licence",
-                "copying",
-                "notice",
-                "copyright",
-                "unlicense",
-            ]
-            .iter()
-            .any(|prefix| name.starts_with(prefix))
-        {
+        if kind.is_file() && is_notice_name(&entry.file_name().to_string_lossy()) {
             output.push(path);
-        } else if kind.is_dir() && depth < 2 {
-            collect_notice_files(&path, depth + 1, output)?;
         }
     }
     Ok(())
@@ -1095,6 +1197,10 @@ fn source_paths(root: &Path) -> Result<Vec<String>> {
                 .map_err(|_| MatrixError::new("source archive contains a non-UTF-8 path"))
         })
         .collect::<Result<Vec<_>>>()?;
+    // `git ls-files --cached` retains index entries for working-tree deletions
+    // until the consolidation commit is created. A source distribution always
+    // describes files that actually exist, including during a pre-commit audit.
+    paths.retain(|path| root.join(path).is_file());
     for path in &paths {
         validate_relative(path)?;
     }
@@ -1126,8 +1232,18 @@ fn compare_report(root: &Path, relative: &str, expected: &[u8]) -> Result<()> {
         MatrixError::new(format!("generated report {}: {error}", path.display()))
     })?;
     if actual != expected {
+        if let Some(directory) = std::env::var_os("METACRATE_PROVENANCE_DIAGNOSTICS_DIR") {
+            let directory = PathBuf::from(directory);
+            fs::create_dir_all(&directory)?;
+            let name = path
+                .file_name()
+                .ok_or_else(|| MatrixError::new("generated report path has no file name"))?;
+            fs::write(directory.join(name), expected)?;
+        }
         return Err(MatrixError::new(format!(
-            "{relative} is stale; run `cargo run --locked -p metacrate-ci-matrix -- provenance-report`"
+            "{relative} is stale (checked-in sha256 {}, generated sha256 {}); run `cargo run --locked -p metacrate-ci-matrix -- provenance-report`",
+            sha256(&actual),
+            sha256(expected),
         )));
     }
     Ok(())
@@ -1253,6 +1369,106 @@ version = "0.0.1"
         );
     }
 
+    #[test]
+    fn package_notice_scan_ignores_build_generated_subtrees() {
+        let root =
+            std::env::temp_dir().join(format!("metacrate-package-notices-{}", std::process::id()));
+        let _ = fs::remove_dir_all(&root);
+        fs::create_dir_all(root.join("generated")).unwrap();
+        fs::write(root.join("LICENSE"), "package license\n").unwrap();
+        fs::write(root.join("generated/LICENSE"), "generated license\n").unwrap();
+        fs::write(root.join("generated/NOTICE.explicit"), "explicit notice\n").unwrap();
+
+        let notices = package_notices(&root, Some("generated/NOTICE.explicit")).unwrap();
+        assert_eq!(
+            notices
+                .iter()
+                .map(|(path, _)| path.as_str())
+                .collect::<Vec<_>>(),
+            ["LICENSE", "generated/NOTICE.explicit"]
+        );
+        fs::remove_dir_all(root).unwrap();
+    }
+
+    #[test]
+    fn package_notice_scan_preserves_nested_files_from_registry_archive() {
+        let base =
+            std::env::temp_dir().join(format!("metacrate-package-archive-{}", std::process::id()));
+        let root = base.join("registry/src/index/demo-1.0.0");
+        let archive_path = base.join("registry/cache/index/demo-1.0.0.crate");
+        let _ = fs::remove_dir_all(&base);
+        fs::create_dir_all(root.join("generated")).unwrap();
+        fs::create_dir_all(archive_path.parent().unwrap()).unwrap();
+        fs::write(root.join("generated/LICENSE"), "build output\n").unwrap();
+
+        let encoder = flate2::write::GzEncoder::new(
+            fs::File::create(&archive_path).unwrap(),
+            flate2::Compression::default(),
+        );
+        let mut archive = tar::Builder::new(encoder);
+        for (path, contents) in [
+            ("demo-1.0.0/LICENSE", "package license\n"),
+            ("demo-1.0.0/vendor/COPYING", "vendored license\n"),
+            ("demo-1.0.0/docs/legal/terms.txt", "explicit terms\n"),
+        ] {
+            let mut header = tar::Header::new_gnu();
+            header.set_size(contents.len() as u64);
+            header.set_mode(0o644);
+            header.set_cksum();
+            archive
+                .append_data(&mut header, path, contents.as_bytes())
+                .unwrap();
+        }
+        archive.into_inner().unwrap().finish().unwrap();
+
+        let notices = package_notices(&root, Some("docs/legal/terms.txt")).unwrap();
+        assert_eq!(
+            notices
+                .iter()
+                .map(|(path, _)| path.as_str())
+                .collect::<Vec<_>>(),
+            ["LICENSE", "docs/legal/terms.txt", "vendor/COPYING"]
+        );
+        fs::remove_dir_all(base).unwrap();
+    }
+
+    #[test]
+    fn registry_notice_scan_requires_the_fetched_archive() {
+        let base = std::env::temp_dir().join(format!(
+            "metacrate-missing-package-archive-{}",
+            std::process::id()
+        ));
+        let root = base.join("registry/src/index/demo-1.0.0");
+        let _ = fs::remove_dir_all(&base);
+        fs::create_dir_all(&root).unwrap();
+        fs::write(root.join("LICENSE"), "package license\n").unwrap();
+
+        let error = package_notices(&root, None).unwrap_err();
+        assert!(
+            error.to_string().contains("registry archive is missing"),
+            "unexpected error: {error}"
+        );
+        fs::remove_dir_all(base).unwrap();
+    }
+
+    #[test]
+    fn non_registry_sources_keep_the_extracted_notice_fallback() {
+        let root = std::env::temp_dir().join(format!(
+            "metacrate-git-package-notices-{}",
+            std::process::id()
+        ));
+        let _ = fs::remove_dir_all(&root);
+        fs::create_dir_all(&root).unwrap();
+        fs::write(root.join("LICENSE"), "package license\n").unwrap();
+
+        let notices = package_notices(&root, None).unwrap();
+        assert_eq!(
+            notices,
+            [("LICENSE".to_owned(), "package license\n".to_owned())]
+        );
+        fs::remove_dir_all(root).unwrap();
+    }
+
     #[test]
     fn unsafe_paths_are_rejected() {
         for path in ["", "../escape", "a/../b", "/absolute"] {