Consolidate required CI gate (#115)
All checks were successful
CI / required (push) Successful in 3m55s
All checks were successful
CI / required (push) Successful in 3m55s
This commit is contained in:
@@ -104,11 +104,7 @@ def check_benchmarks_and_ci() -> None:
|
||||
if not (crate / "benches" / f"{name}.rs").is_file():
|
||||
raise SystemExit(f"missing {name} benchmark source")
|
||||
|
||||
workflows = {
|
||||
"imaging-meshing": ROOT / ".gitea" / "workflows" / "imaging-meshing.yml",
|
||||
"jpeg2000": ROOT / ".gitea" / "workflows" / "jpeg2000.yml",
|
||||
"skia": ROOT / ".gitea" / "workflows" / "skia.yml",
|
||||
}
|
||||
workflows = {"consolidated": ROOT / ".gitea" / "workflows" / "ci.yml"}
|
||||
for name, path in workflows.items():
|
||||
if not path.is_file():
|
||||
raise SystemExit(f"missing {name} native-feature workflow")
|
||||
@@ -121,6 +117,10 @@ def check_benchmarks_and_ci() -> None:
|
||||
f"{name} workflow must use only ubuntu-latest, found: "
|
||||
+ ", ".join(unsupported)
|
||||
)
|
||||
text = path.read_text()
|
||||
for marker in ("required-gate", "METACRATE_SKIA_ARCHIVE", "OPENJPEG_PREFIX"):
|
||||
if marker not in text:
|
||||
raise SystemExit(f"{name} workflow is missing {marker}")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
|
||||
@@ -227,12 +227,11 @@ def check_generation_and_ci() -> None:
|
||||
subprocess.run(
|
||||
["python3", "tools/generate_lsl_tables.py", "--check"], cwd=ROOT, check=True
|
||||
)
|
||||
workflow = ROOT / ".gitea" / "workflows" / "rust-workspace.yml"
|
||||
workflow = ROOT / ".gitea" / "workflows" / "ci.yml"
|
||||
require_markers(
|
||||
workflow,
|
||||
(
|
||||
"python3 tools/check_milestone_10.py",
|
||||
"python3 tools/test_milestone_10.py",
|
||||
"required-gate",
|
||||
"runs-on: ubuntu-latest",
|
||||
),
|
||||
)
|
||||
|
||||
@@ -22,7 +22,7 @@ REFERENCE = (
|
||||
/ "reference_pipeline.rs"
|
||||
)
|
||||
DOC = ROOT / "crates" / "libremetaverse-rendering-simple" / "README.md"
|
||||
WORKFLOW = ROOT / ".gitea" / "workflows" / "rust-workspace.yml"
|
||||
WORKFLOW = ROOT / ".gitea" / "workflows" / "ci.yml"
|
||||
STUB_RE = re.compile(r"\b(?:not_implemented|unimplemented_api)\b|\b(?:todo|unimplemented)!\s*\(")
|
||||
|
||||
TYPE = "T:LibreMetaverse.Rendering.SimpleRenderer"
|
||||
@@ -85,7 +85,7 @@ def main() -> None:
|
||||
"all four `DetailLevel`", "65,536", "Error::Rendering",
|
||||
"source primitive UUID", "MeshFoundry", "prim-local coordinates",
|
||||
))
|
||||
require_markers(WORKFLOW, ("python3 tools/check_milestone_10_issue_76.py",))
|
||||
require_markers(WORKFLOW, ("required-gate",))
|
||||
print(
|
||||
"issue 76 audit: native prim/sculpt conversion, face metadata, texture mapping, "
|
||||
"bounded errors, deterministic evidence, dependency isolation, and docs are present"
|
||||
|
||||
@@ -19,7 +19,7 @@ BENCH = ROOT / "crates" / "libremetaverse-rendering-mesh-foundry" / "benches" /
|
||||
WORKSPACE = ROOT / "Cargo.toml"
|
||||
DOC = ROOT / "crates" / "libremetaverse-rendering-mesh-foundry" / "README.md"
|
||||
STRUCTURED = ROOT / "crates" / "libremetaverse-structured-data" / "src" / "lib.rs"
|
||||
WORKFLOW = ROOT / ".gitea" / "workflows" / "rust-workspace.yml"
|
||||
WORKFLOW = ROOT / ".gitea" / "workflows" / "ci.yml"
|
||||
STUB_RE = re.compile(r"\b(?:not_implemented|unimplemented_api)\b|\b(?:todo|unimplemented)!\s*\(")
|
||||
|
||||
TYPE = "T:LibreMetaverse.Rendering.MeshFoundry"
|
||||
@@ -98,7 +98,7 @@ def main() -> None:
|
||||
"exactly-four-influence", "XOR winding", "64 MiB", "65,536", "6,000,000",
|
||||
"source primitive UUID", "57,600-vertex",
|
||||
))
|
||||
require_markers(WORKFLOW, ("python3 tools/check_milestone_10_issue_77.py",))
|
||||
require_markers(WORKFLOW, ("required-gate",))
|
||||
print(
|
||||
"issue 77 audit: native prim/sculpt/terrain and packed mesh pipelines, bounded LLSD/zlib, "
|
||||
"LOD, normals/tangents/UVs, materials, rigging, convex data, tests, benchmark, and docs are present"
|
||||
|
||||
@@ -15,7 +15,7 @@ GENERATED = ROOT / "crates" / "libremetaverse-rlv" / "src" / "generated.rs"
|
||||
TESTS = ROOT / "crates" / "libremetaverse-rlv" / "tests" / "protocol_parsing.rs"
|
||||
COMPAT = ROOT / "tests" / "compat" / "tests" / "rlv_common_semantics.rs"
|
||||
DOC = ROOT / "crates" / "libremetaverse-rlv" / "README.md"
|
||||
WORKFLOW = ROOT / ".gitea" / "workflows" / "rust-workspace.yml"
|
||||
WORKFLOW = ROOT / ".gitea" / "workflows" / "ci.yml"
|
||||
STUB_RE = re.compile(r"\b(?:not_implemented|unimplemented_api)\b|\b(?:todo|unimplemented)!\s*\(")
|
||||
|
||||
TYPES = {
|
||||
@@ -98,7 +98,7 @@ def main() -> None:
|
||||
"side-effect-free", "119 behavior", "56 pinned", "half-open byte span",
|
||||
"64 KiB", "128 comma-separated", "does not wait", "ubuntu-latest",
|
||||
))
|
||||
require_markers(WORKFLOW, ("python3 tools/check_milestone_10_issue_78.py",))
|
||||
require_markers(WORKFLOW, ("required-gate",))
|
||||
print(
|
||||
"issue 78 audit: bounded pure RLV parsing, typed commands/queries/restrictions, "
|
||||
"exact aliases, positioned errors, mapped values, mutation evidence, and docs are present"
|
||||
|
||||
@@ -16,7 +16,7 @@ GENERATED = ROOT / "crates" / "libremetaverse-rlv" / "src" / "generated.rs"
|
||||
TESTS = ROOT / "crates" / "libremetaverse-rlv" / "tests" / "state_semantics.rs"
|
||||
COMPAT = ROOT / "tests" / "compat" / "tests" / "rlv_inventory_map_semantics.rs"
|
||||
DOC = ROOT / "crates" / "libremetaverse-rlv" / "README.md"
|
||||
WORKFLOW = ROOT / ".gitea" / "workflows" / "rust-workspace.yml"
|
||||
WORKFLOW = ROOT / ".gitea" / "workflows" / "ci.yml"
|
||||
CATALOG = ROOT / "api" / "public-api.json"
|
||||
STUB_RE = re.compile(r"\b(?:not_implemented|unimplemented_api)\b|\b(?:todo|unimplemented)!\s*\(")
|
||||
|
||||
@@ -111,7 +111,7 @@ def main() -> None:
|
||||
"after manager locks are released", "secure and explicit", "no network",
|
||||
"ubuntu-latest",
|
||||
))
|
||||
require_markers(WORKFLOW, ("python3 tools/check_milestone_10_issue_79.py",))
|
||||
require_markers(WORKFLOW, ("required-gate",))
|
||||
print(
|
||||
"issue 79 audit: native restriction state, inventory maps, recursive locks, "
|
||||
"camera aggregation, blacklist, permission precedence, concurrency evidence, "
|
||||
|
||||
@@ -20,7 +20,7 @@ QUERY_TESTS = ROOT / "tests" / "compat" / "tests" / "rlv_query_basics_semantics.
|
||||
ACTION_TESTS = ROOT / "tests" / "compat" / "tests" / "rlv_attach_commands_semantics.rs"
|
||||
NOTIFY_TESTS = ROOT / "tests" / "compat" / "tests" / "rlv_notify_semantics.rs"
|
||||
DOC = ROOT / "crates" / "libremetaverse-rlv" / "README.md"
|
||||
WORKFLOW = ROOT / ".gitea" / "workflows" / "rust-workspace.yml"
|
||||
WORKFLOW = ROOT / ".gitea" / "workflows" / "ci.yml"
|
||||
CATALOG = ROOT / "api" / "public-api.json"
|
||||
STUB_RE = re.compile(r"\b(?:not_implemented|unimplemented_api)\b|\b(?:todo|unimplemented)!\s*\(")
|
||||
|
||||
@@ -136,7 +136,7 @@ def main() -> None:
|
||||
"after releasing every manager lock", "completed inventory and agent",
|
||||
"616", "ubuntu-latest",
|
||||
))
|
||||
require_markers(WORKFLOW, ("python3 tools/check_milestone_10_issue_80.py",))
|
||||
require_markers(WORKFLOW, ("required-gate",))
|
||||
print(
|
||||
"issue 80 audit: native callbacks, typed actions and queries, bounded service "
|
||||
"dispatch, inventory/agent adapters, cancellation, post-lock notifications, "
|
||||
|
||||
@@ -18,7 +18,7 @@ COMPAT_TESTS = (
|
||||
ROOT / "crates" / "libremetaverse-lsl-tools" / "tests" / "lexer_compat.rs"
|
||||
)
|
||||
DOC = ROOT / "crates" / "libremetaverse-lsl-tools" / "README.md"
|
||||
WORKFLOW = ROOT / ".gitea" / "workflows" / "rust-workspace.yml"
|
||||
WORKFLOW = ROOT / ".gitea" / "workflows" / "ci.yml"
|
||||
CATALOG = ROOT / "api" / "public-api.json"
|
||||
STUB_RE = re.compile(
|
||||
r"\b(?:not_implemented|unimplemented_api)\b|\b(?:todo|unimplemented)!\s*\("
|
||||
@@ -135,7 +135,7 @@ def main() -> None:
|
||||
"64 Mi UTF-16", "16 Mi units", "Lexer::iter", "46 focused",
|
||||
"No C#, .NET runtime", "ubuntu-latest",
|
||||
))
|
||||
require_markers(WORKFLOW, ("python3 tools/check_milestone_10_issue_81.py",))
|
||||
require_markers(WORKFLOW, ("required-gate",))
|
||||
print(
|
||||
"issue 81 audit: 22 mapped native types and 188 members, bounded UTF-16 "
|
||||
"source handling, comments and line directives, Unicode categories, deterministic "
|
||||
|
||||
@@ -16,7 +16,7 @@ GENERATED = ROOT / "crates" / "libremetaverse-lsl-tools" / "src" / "generated.rs
|
||||
TESTS = ROOT / "crates" / "libremetaverse-lsl-tools" / "tests" / "parser_compat.rs"
|
||||
EXTENSION_TESTS = ROOT / "tests" / "compat" / "tests" / "extension_shims.rs"
|
||||
DOC = ROOT / "crates" / "libremetaverse-lsl-tools" / "README.md"
|
||||
WORKFLOW = ROOT / ".gitea" / "workflows" / "rust-workspace.yml"
|
||||
WORKFLOW = ROOT / ".gitea" / "workflows" / "ci.yml"
|
||||
CATALOG = ROOT / "api" / "public-api.json"
|
||||
STUB_RE = re.compile(
|
||||
r"\b(?:not_implemented|unimplemented_api)\b|\b(?:todo|unimplemented)!\s*\("
|
||||
@@ -125,7 +125,7 @@ def main() -> None:
|
||||
"nonassociative", "1,048,576", "16,777,216", "46 focused",
|
||||
"issue 83", "ubuntu-latest",
|
||||
))
|
||||
require_markers(WORKFLOW, ("python3 tools/check_milestone_10_issue_82.py",))
|
||||
require_markers(WORKFLOW, ("required-gate",))
|
||||
print(
|
||||
"issue 82 audit: 22 native mapped parser types and 215 members, deterministic "
|
||||
"LALR tables, precedence conflicts, bounded shifts/reductions, error-token "
|
||||
|
||||
@@ -19,7 +19,7 @@ GENERATOR = ROOT / "crates" / "libremetaverse-lsl-tools" / "src" / "generator.rs
|
||||
GENERATED = ROOT / "crates" / "libremetaverse-lsl-tools" / "src" / "generated.rs"
|
||||
TESTS = ROOT / "crates" / "libremetaverse-lsl-tools" / "tests" / "generator_compat.rs"
|
||||
DOC = ROOT / "crates" / "libremetaverse-lsl-tools" / "README.md"
|
||||
WORKFLOW = ROOT / ".gitea" / "workflows" / "rust-workspace.yml"
|
||||
WORKFLOW = ROOT / ".gitea" / "workflows" / "ci.yml"
|
||||
SCRIPT = ROOT / "tools" / "generate_lsl_tables.py"
|
||||
STUB_RE = re.compile(
|
||||
r"\b(?:not_implemented|unimplemented_api)\b|\b(?:todo|unimplemented)!\s*\("
|
||||
@@ -184,8 +184,7 @@ def main() -> None:
|
||||
require_markers(
|
||||
WORKFLOW,
|
||||
(
|
||||
"python3 tools/check_milestone_10_issue_83.py",
|
||||
"python3 tools/generate_lsl_tables.py --check",
|
||||
"required-gate",
|
||||
),
|
||||
)
|
||||
if "runs-on: ubuntu-latest" not in workflow or re.search(
|
||||
|
||||
1424
tools/ci-matrix/src/ci_gate.rs
Normal file
1424
tools/ci-matrix/src/ci_gate.rs
Normal file
File diff suppressed because it is too large
Load Diff
@@ -13,6 +13,7 @@ use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
mod api_surface;
|
||||
mod artifact;
|
||||
mod ci_gate;
|
||||
mod dependency;
|
||||
mod documentation;
|
||||
mod provenance;
|
||||
@@ -20,13 +21,14 @@ mod release_candidate;
|
||||
|
||||
pub use api_surface::{audit_api_surface, write_api_baseline};
|
||||
pub use artifact::audit_artifacts;
|
||||
pub use ci_gate::{audit_consolidated_ci, run_release_gate, run_required_gate};
|
||||
pub use dependency::audit_dependencies;
|
||||
pub use documentation::{audit_documentation, write_documentation_report};
|
||||
pub use provenance::{audit_provenance, write_provenance_reports};
|
||||
pub use release_candidate::audit_release_candidate;
|
||||
|
||||
pub const MATRIX_PATH: &str = "ci/release-matrix.json";
|
||||
const WORKFLOW_PATH: &str = ".gitea/workflows/release-matrix.yml";
|
||||
const WORKFLOW_PATH: &str = ".gitea/workflows/release.yml";
|
||||
const REQUIRED_PROFILES: [&str; 7] = [
|
||||
"linux-msrv-portable",
|
||||
"linux-stable-default",
|
||||
@@ -458,25 +460,16 @@ fn audit_cargo_features(root: &Path) -> Result<()> {
|
||||
|
||||
fn audit_workflows(root: &Path, matrix: &ReleaseMatrix) -> Result<()> {
|
||||
let workflow = fs::read_to_string(root.join(WORKFLOW_PATH))?;
|
||||
for profile in &matrix.profiles {
|
||||
if !workflow.contains(&format!("profile: {}", profile.id)) {
|
||||
return Err(MatrixError::new(format!(
|
||||
"workflow does not schedule profile {}",
|
||||
profile.id
|
||||
)));
|
||||
}
|
||||
}
|
||||
if !workflow.contains("~/.cargo/registry")
|
||||
if matrix.profiles.is_empty()
|
||||
|| !workflow.contains("release-gate")
|
||||
|| !workflow.contains("~/.cargo/registry")
|
||||
|| !workflow.contains("~/.cargo/git")
|
||||
|| workflow.lines().any(|line| {
|
||||
let trimmed = line.trim();
|
||||
trimmed == "target" || trimmed.starts_with("target/") || trimmed.contains("/target/")
|
||||
})
|
||||
{
|
||||
return Err(MatrixError::new(
|
||||
"workflow must cache Cargo downloads without caching build target directories",
|
||||
"release workflow must invoke the Rust gate and cache Cargo downloads",
|
||||
));
|
||||
}
|
||||
audit_consolidated_ci(root)?;
|
||||
let workflows = root.join(".gitea/workflows");
|
||||
for entry in fs::read_dir(workflows)? {
|
||||
let path = entry?.path();
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
use metacrate_ci_matrix::{
|
||||
audit, audit_api_surface, audit_artifacts, audit_dependencies, audit_documentation,
|
||||
audit_provenance, audit_release_candidate, load, run, workspace_root, write_api_baseline,
|
||||
write_documentation_report, write_provenance_reports,
|
||||
audit, audit_api_surface, audit_artifacts, audit_consolidated_ci, audit_dependencies,
|
||||
audit_documentation, audit_provenance, audit_release_candidate, load, run, run_release_gate,
|
||||
run_required_gate, workspace_root, write_api_baseline, write_documentation_report,
|
||||
write_provenance_reports,
|
||||
};
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
@@ -12,12 +13,23 @@ fn main() {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_lines)] // Flat CLI command routing is clearer than nested dispatch.
|
||||
fn execute() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let current = std::env::current_dir()?;
|
||||
let root = workspace_root(¤t).ok_or("could not locate the Cargo workspace")?;
|
||||
let matrix = load(&root)?;
|
||||
let mut arguments = std::env::args().skip(1);
|
||||
match arguments.next().as_deref() {
|
||||
Some("ci-audit") if arguments.next().is_none() => {
|
||||
audit_consolidated_ci(&root)?;
|
||||
println!("consolidated CI coverage and workflow split: ok");
|
||||
}
|
||||
Some("required-gate") => {
|
||||
gate_command(&root, arguments, "required", run_required_gate)?;
|
||||
}
|
||||
Some("release-gate") => {
|
||||
gate_command(&root, arguments, "release", run_release_gate)?;
|
||||
}
|
||||
Some("audit") if arguments.next().is_none() => {
|
||||
audit(&root, &matrix)?;
|
||||
println!(
|
||||
@@ -101,7 +113,7 @@ fn execute() -> Result<(), Box<dyn std::error::Error>> {
|
||||
}
|
||||
_ => {
|
||||
return Err(
|
||||
"usage: ci-matrix audit | run PROFILE --evidence FILE | dependency-audit --evidence FILE | artifact-audit --artifact-dir DIR --package-dir DIR --evidence FILE | release-candidate-audit --evidence FILE | documentation-report | documentation-audit --evidence FILE | api-baseline-write | api-audit --evidence FILE | provenance-report | provenance-audit --evidence FILE"
|
||||
"usage: ci-matrix ci-audit | required-gate --evidence FILE | release-gate --evidence FILE | audit | run PROFILE --evidence FILE | dependency-audit --evidence FILE | artifact-audit --artifact-dir DIR --package-dir DIR --evidence FILE | release-candidate-audit --evidence FILE | documentation-report | documentation-audit --evidence FILE | api-baseline-write | api-audit --evidence FILE | provenance-report | provenance-audit --evidence FILE"
|
||||
.into(),
|
||||
);
|
||||
}
|
||||
@@ -109,6 +121,27 @@ fn execute() -> Result<(), Box<dyn std::error::Error>> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn gate_command(
|
||||
root: &Path,
|
||||
mut arguments: impl Iterator<Item = String>,
|
||||
gate: &str,
|
||||
run_gate: fn(&Path, &Path) -> metacrate_ci_matrix::Result<()>,
|
||||
) -> Result<(), Box<dyn std::error::Error>> {
|
||||
let flag = arguments
|
||||
.next()
|
||||
.ok_or_else(|| format!("{gate}-gate requires --evidence FILE"))?;
|
||||
let evidence = arguments
|
||||
.next()
|
||||
.ok_or_else(|| format!("{gate}-gate requires --evidence FILE"))?;
|
||||
if flag != "--evidence" || arguments.next().is_some() {
|
||||
return Err(format!("usage: ci-matrix {gate}-gate --evidence FILE").into());
|
||||
}
|
||||
let evidence = absolute_or_rooted(root, &evidence);
|
||||
run_gate(root, &evidence)?;
|
||||
println!("{gate} CI gate: ok ({})", evidence.display());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn release_candidate_audit_command(
|
||||
root: &Path,
|
||||
mut arguments: impl Iterator<Item = String>,
|
||||
|
||||
@@ -6,7 +6,7 @@ use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::ffi::OsStr;
|
||||
use std::fmt::Write as _;
|
||||
use std::fs::{self, OpenOptions};
|
||||
use std::io::Write as _;
|
||||
use std::io::{Read as _, Write as _};
|
||||
use std::path::{Component, Path, PathBuf};
|
||||
use std::process::Command;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
@@ -859,6 +859,20 @@ fn license_identifiers(value: &str) -> BTreeSet<&str> {
|
||||
}
|
||||
|
||||
fn cargo_metadata(root: &Path) -> Result<Value> {
|
||||
// Fetch without `--target` so Cargo materializes the immutable archives for
|
||||
// every target represented by Cargo.lock. Reading notice files from those
|
||||
// archives makes the report independent of the host architecture and of
|
||||
// build-script changes to extracted registry sources.
|
||||
let fetch = Command::new(super::cargo_program())
|
||||
.args(["fetch", "--locked"])
|
||||
.current_dir(root)
|
||||
.output()?;
|
||||
if !fetch.status.success() {
|
||||
return Err(MatrixError::new(format!(
|
||||
"cargo fetch failed during provenance audit: {}",
|
||||
String::from_utf8_lossy(&fetch.stderr).trim()
|
||||
)));
|
||||
}
|
||||
let output = Command::new(super::cargo_program())
|
||||
.args([
|
||||
"metadata",
|
||||
@@ -916,8 +930,22 @@ fn parse_lock_string(value: &str) -> Result<String> {
|
||||
}
|
||||
|
||||
fn package_notices(root: &Path, explicit: Option<&str>) -> Result<Vec<(String, String)>> {
|
||||
if let Some(archive) = registry_archive(root) {
|
||||
if !archive.is_file() {
|
||||
return Err(MatrixError::new(format!(
|
||||
"registry archive is missing after cargo fetch: {}",
|
||||
archive.display()
|
||||
)));
|
||||
}
|
||||
let package = root
|
||||
.file_name()
|
||||
.and_then(OsStr::to_str)
|
||||
.ok_or_else(|| MatrixError::new("registry package path has no UTF-8 name"))?;
|
||||
return package_notices_from_archive(&archive, package, explicit);
|
||||
}
|
||||
|
||||
let mut candidates = Vec::new();
|
||||
collect_notice_files(root, 0, &mut candidates)?;
|
||||
collect_notice_files(root, &mut candidates)?;
|
||||
if let Some(explicit) = explicit {
|
||||
let path = root.join(explicit);
|
||||
if path.is_file() {
|
||||
@@ -946,31 +974,105 @@ fn package_notices(root: &Path, explicit: Option<&str>) -> Result<Vec<(String, S
|
||||
Ok(notices)
|
||||
}
|
||||
|
||||
fn collect_notice_files(directory: &Path, depth: u8, output: &mut Vec<PathBuf>) -> Result<()> {
|
||||
if depth > 2 {
|
||||
return Ok(());
|
||||
fn registry_archive(root: &Path) -> Option<PathBuf> {
|
||||
let package = root.file_name()?.to_str()?;
|
||||
let index = root.parent()?.file_name()?;
|
||||
let source = root.parent()?.parent()?;
|
||||
if source.file_name()? != OsStr::new("src") {
|
||||
return None;
|
||||
}
|
||||
let registry = source.parent()?;
|
||||
if registry.file_name()? != OsStr::new("registry") {
|
||||
return None;
|
||||
}
|
||||
Some(
|
||||
registry
|
||||
.join("cache")
|
||||
.join(index)
|
||||
.join(format!("{package}.crate")),
|
||||
)
|
||||
}
|
||||
|
||||
fn package_notices_from_archive(
|
||||
archive: &Path,
|
||||
package: &str,
|
||||
explicit: Option<&str>,
|
||||
) -> Result<Vec<(String, String)>> {
|
||||
let decoder = flate2::read::GzDecoder::new(fs::File::open(archive)?);
|
||||
let mut archive = tar::Archive::new(decoder);
|
||||
let mut notices = Vec::new();
|
||||
for entry in archive.entries()? {
|
||||
let mut entry = entry?;
|
||||
if !entry.header().entry_type().is_file() {
|
||||
continue;
|
||||
}
|
||||
let path = entry.path()?;
|
||||
let mut components = path.components();
|
||||
if components.next() != Some(Component::Normal(OsStr::new(package))) {
|
||||
return Err(MatrixError::new(format!(
|
||||
"package archive entry is outside its {package} root: {}",
|
||||
path.display()
|
||||
)));
|
||||
}
|
||||
let relative = components.collect::<PathBuf>();
|
||||
let relative_text = relative
|
||||
.to_str()
|
||||
.ok_or_else(|| MatrixError::new("package archive contains a non-UTF-8 path"))?;
|
||||
validate_relative(relative_text)?;
|
||||
let depth = relative.components().count();
|
||||
let selected = explicit == Some(relative_text)
|
||||
|| (depth <= 3
|
||||
&& relative
|
||||
.file_name()
|
||||
.and_then(OsStr::to_str)
|
||||
.is_some_and(is_notice_name));
|
||||
if !selected {
|
||||
continue;
|
||||
}
|
||||
let size = entry.header().size()?;
|
||||
if size > 2 * 1024 * 1024 {
|
||||
return Err(MatrixError::new(format!(
|
||||
"package notice is unexpectedly large: {relative_text}"
|
||||
)));
|
||||
}
|
||||
let mut contents = String::new();
|
||||
entry.read_to_string(&mut contents).map_err(|error| {
|
||||
MatrixError::new(format!(
|
||||
"package notice is not UTF-8 ({relative_text}): {error}"
|
||||
))
|
||||
})?;
|
||||
notices.push((relative_text.to_owned(), normalize_notice_text(&contents)));
|
||||
}
|
||||
notices.sort_by(|left, right| left.0.cmp(&right.0));
|
||||
notices.dedup_by(|left, right| left.0 == right.0);
|
||||
Ok(notices)
|
||||
}
|
||||
|
||||
fn is_notice_name(name: &str) -> bool {
|
||||
let name = name.to_ascii_lowercase();
|
||||
[
|
||||
"license",
|
||||
"licence",
|
||||
"copying",
|
||||
"notice",
|
||||
"copyright",
|
||||
"unlicense",
|
||||
]
|
||||
.iter()
|
||||
.any(|prefix| name.starts_with(prefix))
|
||||
}
|
||||
|
||||
fn collect_notice_files(directory: &Path, output: &mut Vec<PathBuf>) -> Result<()> {
|
||||
// Cargo build scripts may populate nested source directories (rust-skia
|
||||
// creates `skia/` in its registry checkout). Only package-root notices and
|
||||
// Cargo's explicit `license_file` are immutable package metadata.
|
||||
let mut entries = fs::read_dir(directory)?.collect::<std::io::Result<Vec<_>>>()?;
|
||||
entries.sort_by_key(std::fs::DirEntry::file_name);
|
||||
for entry in entries {
|
||||
let path = entry.path();
|
||||
let kind = entry.file_type()?;
|
||||
let name = entry.file_name().to_string_lossy().to_ascii_lowercase();
|
||||
if kind.is_file()
|
||||
&& [
|
||||
"license",
|
||||
"licence",
|
||||
"copying",
|
||||
"notice",
|
||||
"copyright",
|
||||
"unlicense",
|
||||
]
|
||||
.iter()
|
||||
.any(|prefix| name.starts_with(prefix))
|
||||
{
|
||||
if kind.is_file() && is_notice_name(&entry.file_name().to_string_lossy()) {
|
||||
output.push(path);
|
||||
} else if kind.is_dir() && depth < 2 {
|
||||
collect_notice_files(&path, depth + 1, output)?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
@@ -1095,6 +1197,10 @@ fn source_paths(root: &Path) -> Result<Vec<String>> {
|
||||
.map_err(|_| MatrixError::new("source archive contains a non-UTF-8 path"))
|
||||
})
|
||||
.collect::<Result<Vec<_>>>()?;
|
||||
// `git ls-files --cached` retains index entries for working-tree deletions
|
||||
// until the consolidation commit is created. A source distribution always
|
||||
// describes files that actually exist, including during a pre-commit audit.
|
||||
paths.retain(|path| root.join(path).is_file());
|
||||
for path in &paths {
|
||||
validate_relative(path)?;
|
||||
}
|
||||
@@ -1126,8 +1232,18 @@ fn compare_report(root: &Path, relative: &str, expected: &[u8]) -> Result<()> {
|
||||
MatrixError::new(format!("generated report {}: {error}", path.display()))
|
||||
})?;
|
||||
if actual != expected {
|
||||
if let Some(directory) = std::env::var_os("METACRATE_PROVENANCE_DIAGNOSTICS_DIR") {
|
||||
let directory = PathBuf::from(directory);
|
||||
fs::create_dir_all(&directory)?;
|
||||
let name = path
|
||||
.file_name()
|
||||
.ok_or_else(|| MatrixError::new("generated report path has no file name"))?;
|
||||
fs::write(directory.join(name), expected)?;
|
||||
}
|
||||
return Err(MatrixError::new(format!(
|
||||
"{relative} is stale; run `cargo run --locked -p metacrate-ci-matrix -- provenance-report`"
|
||||
"{relative} is stale (checked-in sha256 {}, generated sha256 {}); run `cargo run --locked -p metacrate-ci-matrix -- provenance-report`",
|
||||
sha256(&actual),
|
||||
sha256(expected),
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
@@ -1253,6 +1369,106 @@ version = "0.0.1"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn package_notice_scan_ignores_build_generated_subtrees() {
|
||||
let root =
|
||||
std::env::temp_dir().join(format!("metacrate-package-notices-{}", std::process::id()));
|
||||
let _ = fs::remove_dir_all(&root);
|
||||
fs::create_dir_all(root.join("generated")).unwrap();
|
||||
fs::write(root.join("LICENSE"), "package license\n").unwrap();
|
||||
fs::write(root.join("generated/LICENSE"), "generated license\n").unwrap();
|
||||
fs::write(root.join("generated/NOTICE.explicit"), "explicit notice\n").unwrap();
|
||||
|
||||
let notices = package_notices(&root, Some("generated/NOTICE.explicit")).unwrap();
|
||||
assert_eq!(
|
||||
notices
|
||||
.iter()
|
||||
.map(|(path, _)| path.as_str())
|
||||
.collect::<Vec<_>>(),
|
||||
["LICENSE", "generated/NOTICE.explicit"]
|
||||
);
|
||||
fs::remove_dir_all(root).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn package_notice_scan_preserves_nested_files_from_registry_archive() {
|
||||
let base =
|
||||
std::env::temp_dir().join(format!("metacrate-package-archive-{}", std::process::id()));
|
||||
let root = base.join("registry/src/index/demo-1.0.0");
|
||||
let archive_path = base.join("registry/cache/index/demo-1.0.0.crate");
|
||||
let _ = fs::remove_dir_all(&base);
|
||||
fs::create_dir_all(root.join("generated")).unwrap();
|
||||
fs::create_dir_all(archive_path.parent().unwrap()).unwrap();
|
||||
fs::write(root.join("generated/LICENSE"), "build output\n").unwrap();
|
||||
|
||||
let encoder = flate2::write::GzEncoder::new(
|
||||
fs::File::create(&archive_path).unwrap(),
|
||||
flate2::Compression::default(),
|
||||
);
|
||||
let mut archive = tar::Builder::new(encoder);
|
||||
for (path, contents) in [
|
||||
("demo-1.0.0/LICENSE", "package license\n"),
|
||||
("demo-1.0.0/vendor/COPYING", "vendored license\n"),
|
||||
("demo-1.0.0/docs/legal/terms.txt", "explicit terms\n"),
|
||||
] {
|
||||
let mut header = tar::Header::new_gnu();
|
||||
header.set_size(contents.len() as u64);
|
||||
header.set_mode(0o644);
|
||||
header.set_cksum();
|
||||
archive
|
||||
.append_data(&mut header, path, contents.as_bytes())
|
||||
.unwrap();
|
||||
}
|
||||
archive.into_inner().unwrap().finish().unwrap();
|
||||
|
||||
let notices = package_notices(&root, Some("docs/legal/terms.txt")).unwrap();
|
||||
assert_eq!(
|
||||
notices
|
||||
.iter()
|
||||
.map(|(path, _)| path.as_str())
|
||||
.collect::<Vec<_>>(),
|
||||
["LICENSE", "docs/legal/terms.txt", "vendor/COPYING"]
|
||||
);
|
||||
fs::remove_dir_all(base).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn registry_notice_scan_requires_the_fetched_archive() {
|
||||
let base = std::env::temp_dir().join(format!(
|
||||
"metacrate-missing-package-archive-{}",
|
||||
std::process::id()
|
||||
));
|
||||
let root = base.join("registry/src/index/demo-1.0.0");
|
||||
let _ = fs::remove_dir_all(&base);
|
||||
fs::create_dir_all(&root).unwrap();
|
||||
fs::write(root.join("LICENSE"), "package license\n").unwrap();
|
||||
|
||||
let error = package_notices(&root, None).unwrap_err();
|
||||
assert!(
|
||||
error.to_string().contains("registry archive is missing"),
|
||||
"unexpected error: {error}"
|
||||
);
|
||||
fs::remove_dir_all(base).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn non_registry_sources_keep_the_extracted_notice_fallback() {
|
||||
let root = std::env::temp_dir().join(format!(
|
||||
"metacrate-git-package-notices-{}",
|
||||
std::process::id()
|
||||
));
|
||||
let _ = fs::remove_dir_all(&root);
|
||||
fs::create_dir_all(&root).unwrap();
|
||||
fs::write(root.join("LICENSE"), "package license\n").unwrap();
|
||||
|
||||
let notices = package_notices(&root, None).unwrap();
|
||||
assert_eq!(
|
||||
notices,
|
||||
[("LICENSE".to_owned(), "package license\n".to_owned())]
|
||||
);
|
||||
fs::remove_dir_all(root).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unsafe_paths_are_rejected() {
|
||||
for path in ["", "../escape", "a/../b", "/absolute"] {
|
||||
|
||||
@@ -13,6 +13,8 @@ fi
|
||||
install_prefix=$1
|
||||
pkg_config_path="$install_prefix/lib/pkgconfig"
|
||||
library_path="$install_prefix/lib"
|
||||
identity_path="$install_prefix/metacrate-openjpeg.identity"
|
||||
target_arch=$(uname -m)
|
||||
|
||||
persist_actions_environment() {
|
||||
if [ -z "${GITHUB_ENV:-}" ]; then
|
||||
@@ -33,6 +35,10 @@ persist_actions_environment() {
|
||||
}
|
||||
|
||||
if [ -f "$pkg_config_path/libopenjp2.pc" ] &&
|
||||
[ -f "$identity_path" ] &&
|
||||
[ "$(sed -n '1p' "$identity_path")" = "version=$OPENJPEG_VERSION" ] &&
|
||||
[ "$(sed -n '2p' "$identity_path")" = "commit=$OPENJPEG_COMMIT" ] &&
|
||||
[ "$(sed -n '3p' "$identity_path")" = "architecture=$target_arch" ] &&
|
||||
PKG_CONFIG_PATH="$pkg_config_path" pkg-config --exact-version "$OPENJPEG_VERSION" libopenjp2
|
||||
then
|
||||
persist_actions_environment
|
||||
@@ -64,5 +70,11 @@ cmake -S "$source_dir" -B "$build_dir" \
|
||||
cmake --build "$build_dir" --parallel
|
||||
cmake --install "$build_dir"
|
||||
|
||||
{
|
||||
printf 'version=%s\n' "$OPENJPEG_VERSION"
|
||||
printf 'commit=%s\n' "$OPENJPEG_COMMIT"
|
||||
printf 'architecture=%s\n' "$target_arch"
|
||||
} >"$identity_path"
|
||||
|
||||
PKG_CONFIG_PATH="$pkg_config_path" pkg-config --exact-version "$OPENJPEG_VERSION" libopenjp2
|
||||
persist_actions_environment
|
||||
|
||||
164
tools/normalize_git_mtimes.py
Normal file
164
tools/normalize_git_mtimes.py
Normal file
@@ -0,0 +1,164 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Give tracked files stable mtimes derived from their immutable Git blobs."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path, PurePath
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
|
||||
|
||||
WINDOWS_TICK_NANOSECONDS = 100
|
||||
|
||||
|
||||
def tracked_entries(root: Path) -> list[tuple[bytes, Path]]:
|
||||
output = subprocess.run(
|
||||
["git", "ls-files", "--stage", "-z"],
|
||||
cwd=root,
|
||||
check=True,
|
||||
stdout=subprocess.PIPE,
|
||||
).stdout
|
||||
entries: list[tuple[bytes, Path]] = []
|
||||
for record in output.split(b"\0"):
|
||||
if not record:
|
||||
continue
|
||||
metadata, separator, raw_path = record.partition(b"\t")
|
||||
if not separator:
|
||||
raise RuntimeError("git ls-files produced a record without a path")
|
||||
mode, _object_id, stage = metadata.split(b" ")
|
||||
if stage != b"0":
|
||||
raise RuntimeError(
|
||||
f"cannot normalize an unmerged index entry: {os.fsdecode(raw_path)}"
|
||||
)
|
||||
relative = PurePath(os.fsdecode(raw_path))
|
||||
if relative.is_absolute() or ".." in relative.parts:
|
||||
raise RuntimeError(f"unsafe tracked path: {relative}")
|
||||
entries.append((mode, root.joinpath(*relative.parts)))
|
||||
return entries
|
||||
|
||||
|
||||
def content_identity(mode: bytes, path: Path) -> str:
|
||||
digest = hashlib.sha256(mode + b"\0")
|
||||
if path.is_symlink():
|
||||
digest.update(os.fsencode(os.readlink(path)))
|
||||
else:
|
||||
with path.open("rb") as source:
|
||||
while chunk := source.read(1024 * 1024):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def default_state_path(root: Path) -> Path:
|
||||
return root / "target" / ".metacrate-mtimes.json"
|
||||
|
||||
|
||||
def read_state(path: Path) -> dict[str, dict[str, int | str]]:
|
||||
if not path.is_file():
|
||||
return {}
|
||||
value = json.loads(path.read_text(encoding="utf-8"))
|
||||
if value.get("schema") != 1 or not isinstance(value.get("files"), dict):
|
||||
raise RuntimeError(f"invalid tracked-mtime state: {path}")
|
||||
return value["files"]
|
||||
|
||||
|
||||
def write_state(path: Path, files: dict[str, dict[str, int | str]]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp")
|
||||
temporary.write_text(
|
||||
json.dumps({"schema": 1, "files": files}, indent=2, sort_keys=True) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
os.replace(temporary, path)
|
||||
|
||||
|
||||
def normalize(root: Path, state_path: Path, check: bool) -> int:
|
||||
previous = read_state(state_path)
|
||||
current: dict[str, dict[str, int | str]] = {}
|
||||
changed = 0
|
||||
prior_max = max(
|
||||
(int(value["mtime_ns"]) for value in previous.values()), default=0
|
||||
)
|
||||
wall_time = time.time_ns() // WINDOWS_TICK_NANOSECONDS * WINDOWS_TICK_NANOSECONDS
|
||||
now = max(wall_time, prior_max + WINDOWS_TICK_NANOSECONDS)
|
||||
for mode, path in tracked_entries(root):
|
||||
if not (path.is_file() or path.is_symlink()):
|
||||
continue
|
||||
relative = path.relative_to(root).as_posix()
|
||||
identity = content_identity(mode, path)
|
||||
prior = previous.get(relative)
|
||||
expected = (
|
||||
int(prior["mtime_ns"])
|
||||
if prior is not None and prior.get("identity") == identity
|
||||
else now
|
||||
)
|
||||
current[relative] = {"identity": identity, "mtime_ns": expected}
|
||||
actual = os.stat(path, follow_symlinks=False).st_mtime_ns
|
||||
if actual == expected:
|
||||
continue
|
||||
if check:
|
||||
raise RuntimeError(f"tracked mtime is not normalized: {path}")
|
||||
os.utime(path, ns=(expected, expected), follow_symlinks=False)
|
||||
changed += 1
|
||||
if not check:
|
||||
write_state(state_path, current)
|
||||
return changed
|
||||
|
||||
|
||||
def self_test() -> None:
|
||||
with tempfile.TemporaryDirectory(prefix="metacrate-mtimes-") as directory:
|
||||
root = Path(directory)
|
||||
subprocess.run(["git", "init", "--quiet"], cwd=root, check=True)
|
||||
subprocess.run(
|
||||
["git", "config", "user.email", "ci@example.invalid"], cwd=root, check=True
|
||||
)
|
||||
subprocess.run(
|
||||
["git", "config", "user.name", "MetaCrate CI"], cwd=root, check=True
|
||||
)
|
||||
first = root / "first.txt"
|
||||
second = root / "second.txt"
|
||||
first.write_text("first\n", encoding="utf-8")
|
||||
second.write_text("second\n", encoding="utf-8")
|
||||
subprocess.run(["git", "add", "."], cwd=root, check=True)
|
||||
subprocess.run(["git", "commit", "--quiet", "-m", "fixture"], cwd=root, check=True)
|
||||
state = root / "state.json"
|
||||
|
||||
assert normalize(root, state, False) == 2
|
||||
assert normalize(root, state, True) == 0
|
||||
first_mtime = first.stat().st_mtime_ns
|
||||
second_mtime = second.stat().st_mtime_ns
|
||||
|
||||
first.write_text("changed\n", encoding="utf-8")
|
||||
assert normalize(root, state, False) == 1
|
||||
assert first.stat().st_mtime_ns > first_mtime
|
||||
assert second.stat().st_mtime_ns == second_mtime
|
||||
assert normalize(root, state, True) == 0
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--check", action="store_true")
|
||||
parser.add_argument("--self-test", action="store_true")
|
||||
parser.add_argument("--root", type=Path, default=Path.cwd())
|
||||
parser.add_argument("--state", type=Path)
|
||||
args = parser.parse_args()
|
||||
if args.self_test:
|
||||
self_test()
|
||||
print("tracked Git mtime self-test: ok")
|
||||
return 0
|
||||
root = args.root.resolve()
|
||||
state_path = args.state or default_state_path(root)
|
||||
if not state_path.is_absolute():
|
||||
state_path = root / state_path
|
||||
changed = normalize(root, state_path, args.check)
|
||||
verb = "verified" if args.check else "normalized"
|
||||
print(f"{verb} tracked Git mtimes ({changed} changed)")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user