fix(ci): serialize and prune target caches

2026-08-13 12:02:33 +00:00
parent d8beeab98d
commit 1098b4d543
3 changed files with 49 additions and 28 deletions
--- a/.gitea/workflows/ci.yml
+++ b/.gitea/workflows/ci.yml
@@ -12,8 +12,10 @@ on:
        type: boolean

 concurrency:
-  group: ci-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
+  # The runner-persistent Cargo cache is shared across refs. Queue complete CI
+  # runs instead of trying to kill one while it owns or mutates that cache.
+  group: ci-${{ github.workflow }}
+  cancel-in-progress: false

 jobs:
  rust-skia:
@@ -76,17 +78,29 @@ jobs:
          openjpeg_prefix="${{ runner.tool_cache }}/metacrate/native/openjpeg-2.5.4-${{ runner.arch }}"
          skia_dir="${{ runner.tool_cache }}/metacrate/native/skia-0.99.0-aarch64-unknown-linux-gnu"
          skia_archive="$skia_dir/skia-binaries-a25a0fdb7d90429aa2d1-aarch64-unknown-linux-gnu-jpegd-jpege-pdf-svg-textlayout-vulkan-webpd-webpe.tar.gz"
-          if ! test -d "$cargo_target"; then
-            previous_target="$(find "$cargo_cache_root" -mindepth 1 -maxdepth 1 -type d -name 'target-rust-1.97.1-aarch64-unknown-linux-gnu-*' -printf '%T@ %p\n' | sort -nr | head -n 1 | cut -d ' ' -f 2-)"
-            if test -n "$previous_target"; then
-              cp -al "$previous_target" "$cargo_target"
-            fi
-          fi
+          mkdir -p "$CARGO_HOME" "$cargo_cache_root" "$skia_dir"
+          (
+            flock --exclusive 9
+
            if test "${{ github.event_name }}" = workflow_dispatch && test "${{ inputs.cold_cache }}" = true; then
              rm -rf -- "$CARGO_HOME/registry/cache" "$CARGO_HOME/registry/index" "$CARGO_HOME/git/db" "$openjpeg_prefix" "$skia_dir"
              rm -f -- "$CARGO_HOME/bin/cargo-deny" "$CARGO_HOME/bin/cargo-machete"
            fi
-          mkdir -p "$CARGO_HOME" "$cargo_target" "$skia_dir"
+            if ! test -d "$cargo_target"; then
+              previous_target="$(find "$cargo_cache_root" -mindepth 1 -maxdepth 1 -type d -name 'target-rust-1.97.1-aarch64-unknown-linux-gnu-*' -printf '%T@ %p\n' | sort -nr | head -n 1 | cut -d ' ' -f 2-)"
+              if test -n "$previous_target"; then
+                cp -al "$previous_target" "$cargo_target"
+                rm -f -- "$cargo_target/.metacrate-ready"
+              else
+                mkdir -p "$cargo_target"
+              fi
+            fi
+            while IFS= read -r -d '' stale_target; do
+              if test "$stale_target" != "$cargo_target"; then
+                rm -rf -- "$stale_target"
+              fi
+            done < <(find "$cargo_cache_root" -mindepth 1 -maxdepth 1 -type d -name 'target-rust-1.97.1-aarch64-unknown-linux-gnu-*' -print0)
+          ) 9>"$cargo_cache_root/required-target.lock"
          test -f "$cargo_target/.metacrate-ready" && cargo_hit=true || cargo_hit=false
          test -d "$CARGO_HOME/registry/cache" && dependency_hit=true || dependency_hit=false
          test -f "$openjpeg_prefix/metacrate-openjpeg.identity" && test -f "$skia_archive" && native_hit=true || native_hit=false
@@ -101,12 +115,6 @@ jobs:
          echo "METACRATE_NATIVE_CACHE_HIT=$native_hit" >> "$GITHUB_ENV"
          echo "METACRATE_TOOLS_CACHE_HIT=$tools_hit" >> "$GITHUB_ENV"

-      - name: Normalize tracked mtimes for reusable Cargo fingerprints
-        run: |
-          python3 tools/normalize_git_mtimes.py --self-test
-          python3 tools/normalize_git_mtimes.py --state "$CARGO_TARGET_DIR/.metacrate-mtimes.json"
-          python3 tools/normalize_git_mtimes.py --state "$CARGO_TARGET_DIR/.metacrate-mtimes.json" --check
-
      - name: Install the single native prerequisite set
        run: |
          sudo apt-get update
@@ -139,11 +147,19 @@ jobs:
          fi
          echo 'dd127f458a5e67a79f3936a8aa19f822fe90a1d6a11b50b5f84df2b0519d909c  '"$METACRATE_SKIA_ARCHIVE" | sha256sum --check

-      - name: Run the authoritative code-ready graph
-        run: cargo run --locked -p metacrate-ci-matrix -- required-gate --evidence artifacts/ci/required-gate.json
-
-      - name: Mark the validated cache generation ready
-        run: touch "$CARGO_TARGET_DIR/.metacrate-ready"
+      - name: Run the authoritative code-ready graph with exclusive cache ownership
+        run: |
+          cargo_cache_root="${{ runner.tool_cache }}/metacrate/cargo"
+          cache_lock="$cargo_cache_root/required-target.lock"
+          (
+            flock --exclusive 9
+            rm -f -- "$CARGO_TARGET_DIR/.metacrate-ready"
+            python3 tools/normalize_git_mtimes.py --self-test
+            python3 tools/normalize_git_mtimes.py --state "$CARGO_TARGET_DIR/.metacrate-mtimes.json"
+            python3 tools/normalize_git_mtimes.py --state "$CARGO_TARGET_DIR/.metacrate-mtimes.json" --check
+            cargo run --locked -p metacrate-ci-matrix -- required-gate --evidence artifacts/ci/required-gate.json
+            touch "$CARGO_TARGET_DIR/.metacrate-ready"
+          ) 9>"$cache_lock"

      - name: Upload stage timing and audit evidence
        if: always()
--- a/release/DISTRIBUTION-MANIFEST.json
+++ b/release/DISTRIBUTION-MANIFEST.json
@@ -17,8 +17,8 @@
    },
    {
      "path": ".gitea/workflows/ci.yml",
-      "bytes": 7875,
-      "sha256": "6bf276d38174ef88546eff5d992181e65535baf02c5be9126b625d9a69e7ea50"
+      "bytes": 8705,
+      "sha256": "e259db6e8c619decc4c87e41d66bd16e1ee359d2485199f4de9eedad97abc170"
    },
    {
      "path": ".gitea/workflows/release.yml",
@@ -2637,8 +2637,8 @@
    },
    {
      "path": "tools/ci-matrix/src/ci_gate.rs",
-      "bytes": 45664,
-      "sha256": "d1ebbd96de67078ce01ec0f9a42a6f823ced7946a10b31d97b0f0229ac6ae588"
+      "bytes": 45878,
+      "sha256": "e445e48b7fd7b1e37dbe7d6347c52324bffeea234d5be08ce08fe93068b8eccf"
    },
    {
      "path": "tools/ci-matrix/src/dependency.rs",
--- a/tools/ci-matrix/src/ci_gate.rs
+++ b/tools/ci-matrix/src/ci_gate.rs
@@ -1067,9 +1067,14 @@ fn validate_workflows(root: &Path, manifest: &CoverageManifest) -> Result<()> {
        "push:",
        "pull_request:",
        "timeout-minutes: 30",
-        "cancel-in-progress: true",
+        "group: ci-${{ github.workflow }}",
+        "cancel-in-progress: false",
        "runs-on: ubuntu-latest",
        "required-gate",
+        "flock --exclusive 9",
+        "rm -f -- \"$cargo_target/.metacrate-ready\"",
+        "rm -f -- \"$CARGO_TARGET_DIR/.metacrate-ready\"",
+        "stale_target",
        "python3 tools/normalize_git_mtimes.py --state",
        ".metacrate-mtimes.json\" --check",
        "'**/Cargo.toml'",