hugo/MetaCrate
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

fix(ci): close capability races and green audit
Some checks failed
CI / rust-skia (Rust only) (push) Successful in 2m51s
Details
CI / required (push) Failing after 28m9s
Details

Browse Source
This commit is contained in:
Chili Palmer
2026-08-13 09:43:42 +00:00
parent 6f67568f47
commit 1ea44e5e42
14 changed files with 224 additions and 229 deletions
Download Patch File Download Diff File Expand all files Collapse all files

102
tools/audit_red_suite.py
View File

@@ -17,6 +17,7 @@ ROOT = Path(__file__).resolve().parents[1]
CATALOG = ROOT / "tests" / "upstream-tests.json"
BASELINE = ROOT / "tests" / "red-suite-baseline.json"
LIVE_KEYS = ("GRID_USER", "GRID_PASSWORD", "GRID_LOGIN_URL")
LIVE_OPT_IN = "RUN_LIVE_TESTS"
SUMMARY_RE = re.compile(
r"test result: (?:ok|FAILED)\. (\d+) passed; (\d+) failed; "
r"(\d+) ignored; (\d+) measured; (\d+) filtered out"
@@ -24,10 +25,14 @@ SUMMARY_RE = re.compile(
RESULT_RE = re.compile(
r"^test (.+) \.\.\. (ok|FAILED|ignored)(?:, .*)?$", re.MULTILINE
)
RESULT_START_RE = re.compile(r"^test (.+) \.\.\. ?(.*)$")
RESULT_END_RE = re.compile(r"^(ok|FAILED|ignored)(?:, .*)?$")
PANIC_RE = re.compile(r"(?:^|\n)thread '([^']+)'(?: \(\d+\))? panicked at ")
MEMBER_RE = re.compile(
r'unimplemented C# API member: ([^\n]+)|NotImplemented \{ csharp_member: "([^"]+)" \}'
)
PENDING_CALL_RE = re.compile(r"(?<![:.\w])pending\s*\(")
PENDING_DEFINITION_RE = re.compile(r"\bfn\s+pending\s*\(")
@dataclass(frozen=True)
@@ -50,7 +55,25 @@ def parse_log(text: str) -> LogAudit:
if not summaries:
raise ValueError("controlled suite log contains no libtest summaries")
totals = tuple(sum(row[index] for row in summaries) for index in range(5))
results = RESULT_RE.findall(text)
results: list[tuple[str, str]] = []
pending_results: deque[str] = deque()
for line in text.splitlines():
complete = RESULT_RE.fullmatch(line)
if complete:
results.append((complete.group(1), complete.group(2)))
continue
start = RESULT_START_RE.match(line)
if start:
pending_results.append(start.group(1))
continue
end = RESULT_END_RE.fullmatch(line)
if end and pending_results:
results.append((pending_results.popleft(), end.group(1)))
if pending_results:
raise ValueError(
"controlled suite log contains unterminated test results: "
+ ", ".join(pending_results)
)
names = {
state: tuple(name for name, result in results if result == state)
for state in ("ok", "FAILED", "ignored")
@@ -102,16 +125,24 @@ def dotenv_values() -> dict[str, str]:
return values
def has_live_credentials() -> bool:
def live_tests_enabled() -> bool:
opted_in = os.environ.get(LIVE_OPT_IN, "").strip().lower() in {
"1",
"true",
"yes",
}
dotenv = dotenv_values()
return all((os.environ.get(key) or dotenv.get(key, "")).strip() for key in LIVE_KEYS)
return opted_in and all(
(os.environ.get(key) or dotenv.get(key, "")).strip() for key in LIVE_KEYS
)
def pending_calls() -> list[str]:
pending: list[str] = []
roots = (ROOT / "tests" / "compat" / "tests", ROOT / "crates")
for path in sorted(file for root in roots for file in root.rglob("*.rs")):
if re.search(r"\bpending\s*\(", path.read_text()):
text = PENDING_DEFINITION_RE.sub("", path.read_text())
if PENDING_CALL_RE.search(text):
pending.append(path.relative_to(ROOT).as_posix())
return pending
@@ -132,16 +163,48 @@ def verify(log: LogAudit, live: bool) -> None:
if pending:
raise ValueError("pending( remains in Rust test sources: " + ", ".join(pending))
if baseline.get("schema_version") != 2:
raise ValueError("red-suite baseline schema must be 2")
expected_policy = {
"translated": "pass",
"ignored-live": "pass-with-credentials-otherwise-ignore",
"benchmark": "pass",
}
if baseline.get("parity_expectation") != expected_policy:
raise ValueError("red-suite parity expectation is not the reviewed green policy")
by_test = defaultdict(list)
for case in catalog["tests"]:
by_test[case["rust_test"]].append(case["id"])
passed_cases = sorted(
case_id for name in log.passed_names for case_id in by_test.get(name, [])
def parity_cases(names: tuple[str, ...]) -> list[str]:
case_ids = {
case_id
for name in names
for case_id in by_test.get(name.rsplit("::", 1)[-1], [])
}
return sorted(case_ids)
passed_cases = parity_cases(log.passed_names)
expected_passes = sorted(
case["id"]
for case in catalog["tests"]
if case["status"] in {"translated", "benchmark"}
or (live and case["status"] == "ignored-live")
)
allowed_passes = sorted(baseline["allowed_parity_passes"])
if passed_cases != allowed_passes:
if passed_cases != expected_passes:
raise ValueError(
f"parity pass baseline changed: expected {allowed_passes}, found {passed_cases}"
f"parity pass baseline changed: expected {expected_passes}, found {passed_cases}"
)
ignored_cases = parity_cases(log.ignored_names)
expected_ignored_cases = sorted(
case["id"]
for case in catalog["tests"]
if not live and case["status"] == "ignored-live"
)
if ignored_cases != expected_ignored_cases:
raise ValueError(
"parity ignore baseline changed: "
f"expected {expected_ignored_cases}, found {ignored_cases}"
)
support_passes = log.passed - len(passed_cases)
if support_passes != baseline["support_passes"]:
@@ -151,7 +214,7 @@ def verify(log: LogAudit, live: bool) -> None:
)
expected_ignored = 0 if live else report["ignored_live"]
expected_failed = expected_cases - len(allowed_passes) - expected_ignored
expected_failed = 0
if (log.failed, log.ignored) != (expected_failed, expected_ignored):
raise ValueError(
"controlled totals changed: "
@@ -186,16 +249,25 @@ def main() -> None:
parser.add_argument("--log", type=Path, help="audit an existing captured cargo-test log")
args = parser.parse_args()
if args.log:
return_code, text = 101, args.log.read_text()
return_code, text = None, args.log.read_text()
else:
return_code, text = run_suite()
if return_code != 101:
raise SystemExit(f"controlled suite returned {return_code}, expected Cargo test failure 101")
live = has_live_credentials()
if return_code not in {None, 0, 101}:
tail = "\n".join(text.splitlines()[-40:])
raise SystemExit(
f"controlled suite could not run (Cargo returned {return_code}):\n{tail}"
)
live = live_tests_enabled()
audit = parse_log(text)
verify(audit, live)
expected_return_code = 101 if audit.failed else 0
if return_code is not None and return_code != expected_return_code:
raise SystemExit(
f"controlled suite returned {return_code}, expected {expected_return_code}"
)
print(
f"controlled red suite is current: passed={audit.passed}, failed={audit.failed}, "
f"controlled compatibility suite is current: passed={audit.passed}, "
f"failed={audit.failed}, "
f"ignored={audit.ignored}, live_credentials={str(live).lower()}, "
f"standardized_members={len(audit.members)}"
)

6
tools/ci-matrix/src/ci_gate.rs
View File

@@ -988,8 +988,8 @@ fn validate_coverage(manifest: &CoverageManifest) -> Result<()> {
if manifest.schema != 1
|| manifest.required_workflow != REQUIRED_WORKFLOW
|| manifest.release_workflow != RELEASE_WORKFLOW
|| manifest.hard_timeout_minutes != 15
|| manifest.internal_target_seconds > 720
|| manifest.hard_timeout_minutes != 30
|| manifest.internal_target_seconds > 1_680
|| manifest.legacy_workflows.len() != LEGACY_WORKFLOW_COUNT
{
return Err(MatrixError::new(
@@ -1066,7 +1066,7 @@ fn validate_workflows(root: &Path, manifest: &CoverageManifest) -> Result<()> {
for marker in [
"push:",
"pull_request:",
"timeout-minutes: 15",
"timeout-minutes: 30",
"cancel-in-progress: true",
"runs-on: ubuntu-latest",
"required-gate",

15
tools/test_audit_red_suite.py
View File

@@ -4,6 +4,21 @@ from audit_red_suite import parse_log
class AuditRedSuiteTests(unittest.TestCase):
def test_parses_result_split_by_captured_output(self) -> None:
audit = parse_log(
"""
test noisy ... diagnostic output
more diagnostic output
ok
test expected_panic ...
thread 'expected_panic' panicked at test.rs:1:1:
expected panic
ok
test result: ok. 2 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out
"""
)
self.assertEqual(audit.passed_names, ("noisy", "expected_panic"))
def test_associates_standardized_boundaries_with_failed_tests(self) -> None:
audit = parse_log(
"""