Consolidate required CI gate (#115)

tools/check_milestone_06.py

@@ -104,11 +104,7 @@ def check_benchmarks_and_ci() -> None:
         if not (crate / "benches" / f"{name}.rs").is_file():
             raise SystemExit(f"missing {name} benchmark source")
 
-    workflows = {
-        "imaging-meshing": ROOT / ".gitea" / "workflows" / "imaging-meshing.yml",
-        "jpeg2000": ROOT / ".gitea" / "workflows" / "jpeg2000.yml",
-        "skia": ROOT / ".gitea" / "workflows" / "skia.yml",
-    }
+    workflows = {"consolidated": ROOT / ".gitea" / "workflows" / "ci.yml"}
     for name, path in workflows.items():
         if not path.is_file():
             raise SystemExit(f"missing {name} native-feature workflow")
@@ -121,6 +117,10 @@ def check_benchmarks_and_ci() -> None:
                 f"{name} workflow must use only ubuntu-latest, found: "
                 + ", ".join(unsupported)
             )
+        text = path.read_text()
+        for marker in ("required-gate", "METACRATE_SKIA_ARCHIVE", "OPENJPEG_PREFIX"):
+            if marker not in text:
+                raise SystemExit(f"{name} workflow is missing {marker}")
 
 
 def main() -> None:

tools/check_milestone_10.py

@@ -227,12 +227,11 @@ def check_generation_and_ci() -> None:
     subprocess.run(
         ["python3", "tools/generate_lsl_tables.py", "--check"], cwd=ROOT, check=True
     )
-    workflow = ROOT / ".gitea" / "workflows" / "rust-workspace.yml"
+    workflow = ROOT / ".gitea" / "workflows" / "ci.yml"
     require_markers(
         workflow,
         (
-            "python3 tools/check_milestone_10.py",
-            "python3 tools/test_milestone_10.py",
+            "required-gate",
             "runs-on: ubuntu-latest",
         ),
     )

tools/check_milestone_10_issue_76.py

@@ -22,7 +22,7 @@ REFERENCE = (
     / "reference_pipeline.rs"
 )
 DOC = ROOT / "crates" / "libremetaverse-rendering-simple" / "README.md"
-WORKFLOW = ROOT / ".gitea" / "workflows" / "rust-workspace.yml"
+WORKFLOW = ROOT / ".gitea" / "workflows" / "ci.yml"
 STUB_RE = re.compile(r"\b(?:not_implemented|unimplemented_api)\b|\b(?:todo|unimplemented)!\s*\(")
 
 TYPE = "T:LibreMetaverse.Rendering.SimpleRenderer"
@@ -85,7 +85,7 @@ def main() -> None:
         "all four `DetailLevel`", "65,536", "Error::Rendering",
         "source primitive UUID", "MeshFoundry", "prim-local coordinates",
     ))
-    require_markers(WORKFLOW, ("python3 tools/check_milestone_10_issue_76.py",))
+    require_markers(WORKFLOW, ("required-gate",))
     print(
         "issue 76 audit: native prim/sculpt conversion, face metadata, texture mapping, "
         "bounded errors, deterministic evidence, dependency isolation, and docs are present"

tools/check_milestone_10_issue_77.py

@@ -19,7 +19,7 @@ BENCH = ROOT / "crates" / "libremetaverse-rendering-mesh-foundry" / "benches" /
 WORKSPACE = ROOT / "Cargo.toml"
 DOC = ROOT / "crates" / "libremetaverse-rendering-mesh-foundry" / "README.md"
 STRUCTURED = ROOT / "crates" / "libremetaverse-structured-data" / "src" / "lib.rs"
-WORKFLOW = ROOT / ".gitea" / "workflows" / "rust-workspace.yml"
+WORKFLOW = ROOT / ".gitea" / "workflows" / "ci.yml"
 STUB_RE = re.compile(r"\b(?:not_implemented|unimplemented_api)\b|\b(?:todo|unimplemented)!\s*\(")
 
 TYPE = "T:LibreMetaverse.Rendering.MeshFoundry"
@@ -98,7 +98,7 @@ def main() -> None:
         "exactly-four-influence", "XOR winding", "64 MiB", "65,536", "6,000,000",
         "source primitive UUID", "57,600-vertex",
     ))
-    require_markers(WORKFLOW, ("python3 tools/check_milestone_10_issue_77.py",))
+    require_markers(WORKFLOW, ("required-gate",))
     print(
         "issue 77 audit: native prim/sculpt/terrain and packed mesh pipelines, bounded LLSD/zlib, "
         "LOD, normals/tangents/UVs, materials, rigging, convex data, tests, benchmark, and docs are present"

tools/check_milestone_10_issue_78.py

@@ -15,7 +15,7 @@ GENERATED = ROOT / "crates" / "libremetaverse-rlv" / "src" / "generated.rs"
 TESTS = ROOT / "crates" / "libremetaverse-rlv" / "tests" / "protocol_parsing.rs"
 COMPAT = ROOT / "tests" / "compat" / "tests" / "rlv_common_semantics.rs"
 DOC = ROOT / "crates" / "libremetaverse-rlv" / "README.md"
-WORKFLOW = ROOT / ".gitea" / "workflows" / "rust-workspace.yml"
+WORKFLOW = ROOT / ".gitea" / "workflows" / "ci.yml"
 STUB_RE = re.compile(r"\b(?:not_implemented|unimplemented_api)\b|\b(?:todo|unimplemented)!\s*\(")
 
 TYPES = {
@@ -98,7 +98,7 @@ def main() -> None:
         "side-effect-free", "119 behavior", "56 pinned", "half-open byte span",
         "64 KiB", "128 comma-separated", "does not wait", "ubuntu-latest",
     ))
-    require_markers(WORKFLOW, ("python3 tools/check_milestone_10_issue_78.py",))
+    require_markers(WORKFLOW, ("required-gate",))
     print(
         "issue 78 audit: bounded pure RLV parsing, typed commands/queries/restrictions, "
         "exact aliases, positioned errors, mapped values, mutation evidence, and docs are present"

tools/check_milestone_10_issue_79.py

@@ -16,7 +16,7 @@ GENERATED = ROOT / "crates" / "libremetaverse-rlv" / "src" / "generated.rs"
 TESTS = ROOT / "crates" / "libremetaverse-rlv" / "tests" / "state_semantics.rs"
 COMPAT = ROOT / "tests" / "compat" / "tests" / "rlv_inventory_map_semantics.rs"
 DOC = ROOT / "crates" / "libremetaverse-rlv" / "README.md"
-WORKFLOW = ROOT / ".gitea" / "workflows" / "rust-workspace.yml"
+WORKFLOW = ROOT / ".gitea" / "workflows" / "ci.yml"
 CATALOG = ROOT / "api" / "public-api.json"
 STUB_RE = re.compile(r"\b(?:not_implemented|unimplemented_api)\b|\b(?:todo|unimplemented)!\s*\(")
 
@@ -111,7 +111,7 @@ def main() -> None:
         "after manager locks are released", "secure and explicit", "no network",
         "ubuntu-latest",
     ))
-    require_markers(WORKFLOW, ("python3 tools/check_milestone_10_issue_79.py",))
+    require_markers(WORKFLOW, ("required-gate",))
     print(
         "issue 79 audit: native restriction state, inventory maps, recursive locks, "
         "camera aggregation, blacklist, permission precedence, concurrency evidence, "

tools/check_milestone_10_issue_80.py

@@ -20,7 +20,7 @@ QUERY_TESTS = ROOT / "tests" / "compat" / "tests" / "rlv_query_basics_semantics.
 ACTION_TESTS = ROOT / "tests" / "compat" / "tests" / "rlv_attach_commands_semantics.rs"
 NOTIFY_TESTS = ROOT / "tests" / "compat" / "tests" / "rlv_notify_semantics.rs"
 DOC = ROOT / "crates" / "libremetaverse-rlv" / "README.md"
-WORKFLOW = ROOT / ".gitea" / "workflows" / "rust-workspace.yml"
+WORKFLOW = ROOT / ".gitea" / "workflows" / "ci.yml"
 CATALOG = ROOT / "api" / "public-api.json"
 STUB_RE = re.compile(r"\b(?:not_implemented|unimplemented_api)\b|\b(?:todo|unimplemented)!\s*\(")
 
@@ -136,7 +136,7 @@ def main() -> None:
         "after releasing every manager lock", "completed inventory and agent",
         "616", "ubuntu-latest",
     ))
-    require_markers(WORKFLOW, ("python3 tools/check_milestone_10_issue_80.py",))
+    require_markers(WORKFLOW, ("required-gate",))
     print(
         "issue 80 audit: native callbacks, typed actions and queries, bounded service "
         "dispatch, inventory/agent adapters, cancellation, post-lock notifications, "

tools/check_milestone_10_issue_81.py

@@ -18,7 +18,7 @@ COMPAT_TESTS = (
     ROOT / "crates" / "libremetaverse-lsl-tools" / "tests" / "lexer_compat.rs"
 )
 DOC = ROOT / "crates" / "libremetaverse-lsl-tools" / "README.md"
-WORKFLOW = ROOT / ".gitea" / "workflows" / "rust-workspace.yml"
+WORKFLOW = ROOT / ".gitea" / "workflows" / "ci.yml"
 CATALOG = ROOT / "api" / "public-api.json"
 STUB_RE = re.compile(
     r"\b(?:not_implemented|unimplemented_api)\b|\b(?:todo|unimplemented)!\s*\("
@@ -135,7 +135,7 @@ def main() -> None:
         "64 Mi UTF-16", "16 Mi units", "Lexer::iter", "46 focused",
         "No C#, .NET runtime", "ubuntu-latest",
     ))
-    require_markers(WORKFLOW, ("python3 tools/check_milestone_10_issue_81.py",))
+    require_markers(WORKFLOW, ("required-gate",))
     print(
         "issue 81 audit: 22 mapped native types and 188 members, bounded UTF-16 "
         "source handling, comments and line directives, Unicode categories, deterministic "

tools/check_milestone_10_issue_82.py

@@ -16,7 +16,7 @@ GENERATED = ROOT / "crates" / "libremetaverse-lsl-tools" / "src" / "generated.rs
 TESTS = ROOT / "crates" / "libremetaverse-lsl-tools" / "tests" / "parser_compat.rs"
 EXTENSION_TESTS = ROOT / "tests" / "compat" / "tests" / "extension_shims.rs"
 DOC = ROOT / "crates" / "libremetaverse-lsl-tools" / "README.md"
-WORKFLOW = ROOT / ".gitea" / "workflows" / "rust-workspace.yml"
+WORKFLOW = ROOT / ".gitea" / "workflows" / "ci.yml"
 CATALOG = ROOT / "api" / "public-api.json"
 STUB_RE = re.compile(
     r"\b(?:not_implemented|unimplemented_api)\b|\b(?:todo|unimplemented)!\s*\("
@@ -125,7 +125,7 @@ def main() -> None:
         "nonassociative", "1,048,576", "16,777,216", "46 focused",
         "issue 83", "ubuntu-latest",
     ))
-    require_markers(WORKFLOW, ("python3 tools/check_milestone_10_issue_82.py",))
+    require_markers(WORKFLOW, ("required-gate",))
     print(
         "issue 82 audit: 22 native mapped parser types and 215 members, deterministic "
         "LALR tables, precedence conflicts, bounded shifts/reductions, error-token "

tools/check_milestone_10_issue_83.py

@@ -19,7 +19,7 @@ GENERATOR = ROOT / "crates" / "libremetaverse-lsl-tools" / "src" / "generator.rs
 GENERATED = ROOT / "crates" / "libremetaverse-lsl-tools" / "src" / "generated.rs"
 TESTS = ROOT / "crates" / "libremetaverse-lsl-tools" / "tests" / "generator_compat.rs"
 DOC = ROOT / "crates" / "libremetaverse-lsl-tools" / "README.md"
-WORKFLOW = ROOT / ".gitea" / "workflows" / "rust-workspace.yml"
+WORKFLOW = ROOT / ".gitea" / "workflows" / "ci.yml"
 SCRIPT = ROOT / "tools" / "generate_lsl_tables.py"
 STUB_RE = re.compile(
     r"\b(?:not_implemented|unimplemented_api)\b|\b(?:todo|unimplemented)!\s*\("
@@ -184,8 +184,7 @@ def main() -> None:
     require_markers(
         WORKFLOW,
         (
-            "python3 tools/check_milestone_10_issue_83.py",
-            "python3 tools/generate_lsl_tables.py --check",
+            "required-gate",
         ),
     )
     if "runs-on: ubuntu-latest" not in workflow or re.search(

tools/ci-matrix/src/lib.rs

@@ -13,6 +13,7 @@ use std::time::{SystemTime, UNIX_EPOCH};
 
 mod api_surface;
 mod artifact;
+mod ci_gate;
 mod dependency;
 mod documentation;
 mod provenance;
@@ -20,13 +21,14 @@ mod release_candidate;
 
 pub use api_surface::{audit_api_surface, write_api_baseline};
 pub use artifact::audit_artifacts;
+pub use ci_gate::{audit_consolidated_ci, run_release_gate, run_required_gate};
 pub use dependency::audit_dependencies;
 pub use documentation::{audit_documentation, write_documentation_report};
 pub use provenance::{audit_provenance, write_provenance_reports};
 pub use release_candidate::audit_release_candidate;
 
 pub const MATRIX_PATH: &str = "ci/release-matrix.json";
-const WORKFLOW_PATH: &str = ".gitea/workflows/release-matrix.yml";
+const WORKFLOW_PATH: &str = ".gitea/workflows/release.yml";
 const REQUIRED_PROFILES: [&str; 7] = [
     "linux-msrv-portable",
     "linux-stable-default",
@@ -458,25 +460,16 @@ fn audit_cargo_features(root: &Path) -> Result<()> {
 
 fn audit_workflows(root: &Path, matrix: &ReleaseMatrix) -> Result<()> {
     let workflow = fs::read_to_string(root.join(WORKFLOW_PATH))?;
-    for profile in &matrix.profiles {
-        if !workflow.contains(&format!("profile: {}", profile.id)) {
-            return Err(MatrixError::new(format!(
-                "workflow does not schedule profile {}",
-                profile.id
-            )));
-        }
-    }
-    if !workflow.contains("~/.cargo/registry")
+    if matrix.profiles.is_empty()
+        || !workflow.contains("release-gate")
+        || !workflow.contains("~/.cargo/registry")
         || !workflow.contains("~/.cargo/git")
-        || workflow.lines().any(|line| {
-            let trimmed = line.trim();
-            trimmed == "target" || trimmed.starts_with("target/") || trimmed.contains("/target/")
-        })
     {
         return Err(MatrixError::new(
-            "workflow must cache Cargo downloads without caching build target directories",
+            "release workflow must invoke the Rust gate and cache Cargo downloads",
         ));
     }
+    audit_consolidated_ci(root)?;
     let workflows = root.join(".gitea/workflows");
     for entry in fs::read_dir(workflows)? {
         let path = entry?.path();

tools/ci-matrix/src/main.rs

@@ -1,7 +1,8 @@
 use metacrate_ci_matrix::{
-    audit, audit_api_surface, audit_artifacts, audit_dependencies, audit_documentation,
-    audit_provenance, audit_release_candidate, load, run, workspace_root, write_api_baseline,
-    write_documentation_report, write_provenance_reports,
+    audit, audit_api_surface, audit_artifacts, audit_consolidated_ci, audit_dependencies,
+    audit_documentation, audit_provenance, audit_release_candidate, load, run, run_release_gate,
+    run_required_gate, workspace_root, write_api_baseline, write_documentation_report,
+    write_provenance_reports,
 };
 use std::path::{Path, PathBuf};
 
@@ -12,12 +13,23 @@ fn main() {
     }
 }
 
+#[allow(clippy::too_many_lines)] // Flat CLI command routing is clearer than nested dispatch.
 fn execute() -> Result<(), Box<dyn std::error::Error>> {
     let current = std::env::current_dir()?;
     let root = workspace_root(&current).ok_or("could not locate the Cargo workspace")?;
     let matrix = load(&root)?;
     let mut arguments = std::env::args().skip(1);
     match arguments.next().as_deref() {
+        Some("ci-audit") if arguments.next().is_none() => {
+            audit_consolidated_ci(&root)?;
+            println!("consolidated CI coverage and workflow split: ok");
+        }
+        Some("required-gate") => {
+            gate_command(&root, arguments, "required", run_required_gate)?;
+        }
+        Some("release-gate") => {
+            gate_command(&root, arguments, "release", run_release_gate)?;
+        }
         Some("audit") if arguments.next().is_none() => {
             audit(&root, &matrix)?;
             println!(
@@ -101,7 +113,7 @@ fn execute() -> Result<(), Box<dyn std::error::Error>> {
         }
         _ => {
             return Err(
-                "usage: ci-matrix audit | run PROFILE --evidence FILE | dependency-audit --evidence FILE | artifact-audit --artifact-dir DIR --package-dir DIR --evidence FILE | release-candidate-audit --evidence FILE | documentation-report | documentation-audit --evidence FILE | api-baseline-write | api-audit --evidence FILE | provenance-report | provenance-audit --evidence FILE"
+                "usage: ci-matrix ci-audit | required-gate --evidence FILE | release-gate --evidence FILE | audit | run PROFILE --evidence FILE | dependency-audit --evidence FILE | artifact-audit --artifact-dir DIR --package-dir DIR --evidence FILE | release-candidate-audit --evidence FILE | documentation-report | documentation-audit --evidence FILE | api-baseline-write | api-audit --evidence FILE | provenance-report | provenance-audit --evidence FILE"
                     .into(),
             );
         }
@@ -109,6 +121,27 @@ fn execute() -> Result<(), Box<dyn std::error::Error>> {
     Ok(())
 }
 
+fn gate_command(
+    root: &Path,
+    mut arguments: impl Iterator<Item = String>,
+    gate: &str,
+    run_gate: fn(&Path, &Path) -> metacrate_ci_matrix::Result<()>,
+) -> Result<(), Box<dyn std::error::Error>> {
+    let flag = arguments
+        .next()
+        .ok_or_else(|| format!("{gate}-gate requires --evidence FILE"))?;
+    let evidence = arguments
+        .next()
+        .ok_or_else(|| format!("{gate}-gate requires --evidence FILE"))?;
+    if flag != "--evidence" || arguments.next().is_some() {
+        return Err(format!("usage: ci-matrix {gate}-gate --evidence FILE").into());
+    }
+    let evidence = absolute_or_rooted(root, &evidence);
+    run_gate(root, &evidence)?;
+    println!("{gate} CI gate: ok ({})", evidence.display());
+    Ok(())
+}
+
 fn release_candidate_audit_command(
     root: &Path,
     mut arguments: impl Iterator<Item = String>,

tools/ci-matrix/src/provenance.rs

@@ -917,7 +917,7 @@ fn parse_lock_string(value: &str) -> Result<String> {
 
 fn package_notices(root: &Path, explicit: Option<&str>) -> Result<Vec<(String, String)>> {
     let mut candidates = Vec::new();
-    collect_notice_files(root, 0, &mut candidates)?;
+    collect_notice_files(root, &mut candidates)?;
     if let Some(explicit) = explicit {
         let path = root.join(explicit);
         if path.is_file() {
@@ -946,10 +946,10 @@ fn package_notices(root: &Path, explicit: Option<&str>) -> Result<Vec<(String, S
     Ok(notices)
 }
 
-fn collect_notice_files(directory: &Path, depth: u8, output: &mut Vec<PathBuf>) -> Result<()> {
-    if depth > 2 {
-        return Ok(());
-    }
+fn collect_notice_files(directory: &Path, output: &mut Vec<PathBuf>) -> Result<()> {
+    // Cargo build scripts may populate nested source directories (rust-skia
+    // creates `skia/` in its registry checkout). Only package-root notices and
+    // Cargo's explicit `license_file` are immutable package metadata.
     let mut entries = fs::read_dir(directory)?.collect::<std::io::Result<Vec<_>>>()?;
     entries.sort_by_key(std::fs::DirEntry::file_name);
     for entry in entries {
@@ -969,8 +969,6 @@ fn collect_notice_files(directory: &Path, depth: u8, output: &mut Vec<PathBuf>)
             .any(|prefix| name.starts_with(prefix))
         {
             output.push(path);
-        } else if kind.is_dir() && depth < 2 {
-            collect_notice_files(&path, depth + 1, output)?;
         }
     }
     Ok(())
@@ -1095,6 +1093,10 @@ fn source_paths(root: &Path) -> Result<Vec<String>> {
                 .map_err(|_| MatrixError::new("source archive contains a non-UTF-8 path"))
         })
         .collect::<Result<Vec<_>>>()?;
+    // `git ls-files --cached` retains index entries for working-tree deletions
+    // until the consolidation commit is created. A source distribution always
+    // describes files that actually exist, including during a pre-commit audit.
+    paths.retain(|path| root.join(path).is_file());
     for path in &paths {
         validate_relative(path)?;
     }
@@ -1126,8 +1128,18 @@ fn compare_report(root: &Path, relative: &str, expected: &[u8]) -> Result<()> {
         MatrixError::new(format!("generated report {}: {error}", path.display()))
     })?;
     if actual != expected {
+        if let Some(directory) = std::env::var_os("METACRATE_PROVENANCE_DIAGNOSTICS_DIR") {
+            let directory = PathBuf::from(directory);
+            fs::create_dir_all(&directory)?;
+            let name = path
+                .file_name()
+                .ok_or_else(|| MatrixError::new("generated report path has no file name"))?;
+            fs::write(directory.join(name), expected)?;
+        }
         return Err(MatrixError::new(format!(
-            "{relative} is stale; run `cargo run --locked -p metacrate-ci-matrix -- provenance-report`"
+            "{relative} is stale (checked-in sha256 {}, generated sha256 {}); run `cargo run --locked -p metacrate-ci-matrix -- provenance-report`",
+            sha256(&actual),
+            sha256(expected),
         )));
     }
     Ok(())
@@ -1253,6 +1265,27 @@ version = "0.0.1"
         );
     }
 
+    #[test]
+    fn package_notice_scan_ignores_build_generated_subtrees() {
+        let root =
+            std::env::temp_dir().join(format!("metacrate-package-notices-{}", std::process::id()));
+        let _ = fs::remove_dir_all(&root);
+        fs::create_dir_all(root.join("generated")).unwrap();
+        fs::write(root.join("LICENSE"), "package license\n").unwrap();
+        fs::write(root.join("generated/LICENSE"), "generated license\n").unwrap();
+        fs::write(root.join("generated/NOTICE.explicit"), "explicit notice\n").unwrap();
+
+        let notices = package_notices(&root, Some("generated/NOTICE.explicit")).unwrap();
+        assert_eq!(
+            notices
+                .iter()
+                .map(|(path, _)| path.as_str())
+                .collect::<Vec<_>>(),
+            ["LICENSE", "generated/NOTICE.explicit"]
+        );
+        fs::remove_dir_all(root).unwrap();
+    }
+
     #[test]
     fn unsafe_paths_are_rejected() {
         for path in ["", "../escape", "a/../b", "/absolute"] {

tools/install_openjpeg_2_5_4.sh

@@ -13,6 +13,8 @@ fi
 install_prefix=$1
 pkg_config_path="$install_prefix/lib/pkgconfig"
 library_path="$install_prefix/lib"
+identity_path="$install_prefix/metacrate-openjpeg.identity"
+target_arch=$(uname -m)
 
 persist_actions_environment() {
     if [ -z "${GITHUB_ENV:-}" ]; then
@@ -33,6 +35,10 @@ persist_actions_environment() {
 }
 
 if [ -f "$pkg_config_path/libopenjp2.pc" ] &&
+    [ -f "$identity_path" ] &&
+    [ "$(sed -n '1p' "$identity_path")" = "version=$OPENJPEG_VERSION" ] &&
+    [ "$(sed -n '2p' "$identity_path")" = "commit=$OPENJPEG_COMMIT" ] &&
+    [ "$(sed -n '3p' "$identity_path")" = "architecture=$target_arch" ] &&
     PKG_CONFIG_PATH="$pkg_config_path" pkg-config --exact-version "$OPENJPEG_VERSION" libopenjp2
 then
     persist_actions_environment
@@ -64,5 +70,11 @@ cmake -S "$source_dir" -B "$build_dir" \
 cmake --build "$build_dir" --parallel
 cmake --install "$build_dir"
 
+{
+    printf 'version=%s\n' "$OPENJPEG_VERSION"
+    printf 'commit=%s\n' "$OPENJPEG_COMMIT"
+    printf 'architecture=%s\n' "$target_arch"
+} >"$identity_path"
+
 PKG_CONFIG_PATH="$pkg_config_path" pkg-config --exact-version "$OPENJPEG_VERSION" libopenjp2
 persist_actions_environment