fix(ci): close capability races and green audit

tools/audit_red_suite.py

@@ -17,6 +17,7 @@ ROOT = Path(__file__).resolve().parents[1]
 CATALOG = ROOT / "tests" / "upstream-tests.json"
 BASELINE = ROOT / "tests" / "red-suite-baseline.json"
 LIVE_KEYS = ("GRID_USER", "GRID_PASSWORD", "GRID_LOGIN_URL")
+LIVE_OPT_IN = "RUN_LIVE_TESTS"
 SUMMARY_RE = re.compile(
     r"test result: (?:ok|FAILED)\. (\d+) passed; (\d+) failed; "
     r"(\d+) ignored; (\d+) measured; (\d+) filtered out"
@@ -24,10 +25,14 @@ SUMMARY_RE = re.compile(
 RESULT_RE = re.compile(
     r"^test (.+) \.\.\. (ok|FAILED|ignored)(?:, .*)?$", re.MULTILINE
 )
+RESULT_START_RE = re.compile(r"^test (.+) \.\.\. ?(.*)$")
+RESULT_END_RE = re.compile(r"^(ok|FAILED|ignored)(?:, .*)?$")
 PANIC_RE = re.compile(r"(?:^|\n)thread '([^']+)'(?: \(\d+\))? panicked at ")
 MEMBER_RE = re.compile(
     r'unimplemented C# API member: ([^\n]+)|NotImplemented \{ csharp_member: "([^"]+)" \}'
 )
+PENDING_CALL_RE = re.compile(r"(?<![:.\w])pending\s*\(")
+PENDING_DEFINITION_RE = re.compile(r"\bfn\s+pending\s*\(")
 
 
 @dataclass(frozen=True)
@@ -50,7 +55,25 @@ def parse_log(text: str) -> LogAudit:
     if not summaries:
         raise ValueError("controlled suite log contains no libtest summaries")
     totals = tuple(sum(row[index] for row in summaries) for index in range(5))
-    results = RESULT_RE.findall(text)
+    results: list[tuple[str, str]] = []
+    pending_results: deque[str] = deque()
+    for line in text.splitlines():
+        complete = RESULT_RE.fullmatch(line)
+        if complete:
+            results.append((complete.group(1), complete.group(2)))
+            continue
+        start = RESULT_START_RE.match(line)
+        if start:
+            pending_results.append(start.group(1))
+            continue
+        end = RESULT_END_RE.fullmatch(line)
+        if end and pending_results:
+            results.append((pending_results.popleft(), end.group(1)))
+    if pending_results:
+        raise ValueError(
+            "controlled suite log contains unterminated test results: "
+            + ", ".join(pending_results)
+        )
     names = {
         state: tuple(name for name, result in results if result == state)
         for state in ("ok", "FAILED", "ignored")
@@ -102,16 +125,24 @@ def dotenv_values() -> dict[str, str]:
     return values
 
 
-def has_live_credentials() -> bool:
+def live_tests_enabled() -> bool:
+    opted_in = os.environ.get(LIVE_OPT_IN, "").strip().lower() in {
+        "1",
+        "true",
+        "yes",
+    }
     dotenv = dotenv_values()
-    return all((os.environ.get(key) or dotenv.get(key, "")).strip() for key in LIVE_KEYS)
+    return opted_in and all(
+        (os.environ.get(key) or dotenv.get(key, "")).strip() for key in LIVE_KEYS
+    )
 
 
 def pending_calls() -> list[str]:
     pending: list[str] = []
     roots = (ROOT / "tests" / "compat" / "tests", ROOT / "crates")
     for path in sorted(file for root in roots for file in root.rglob("*.rs")):
-        if re.search(r"\bpending\s*\(", path.read_text()):
+        text = PENDING_DEFINITION_RE.sub("", path.read_text())
+        if PENDING_CALL_RE.search(text):
             pending.append(path.relative_to(ROOT).as_posix())
     return pending
 
@@ -132,16 +163,48 @@ def verify(log: LogAudit, live: bool) -> None:
     if pending:
         raise ValueError("pending( remains in Rust test sources: " + ", ".join(pending))
 
+    if baseline.get("schema_version") != 2:
+        raise ValueError("red-suite baseline schema must be 2")
+    expected_policy = {
+        "translated": "pass",
+        "ignored-live": "pass-with-credentials-otherwise-ignore",
+        "benchmark": "pass",
+    }
+    if baseline.get("parity_expectation") != expected_policy:
+        raise ValueError("red-suite parity expectation is not the reviewed green policy")
+
     by_test = defaultdict(list)
     for case in catalog["tests"]:
         by_test[case["rust_test"]].append(case["id"])
-    passed_cases = sorted(
-        case_id for name in log.passed_names for case_id in by_test.get(name, [])
+    def parity_cases(names: tuple[str, ...]) -> list[str]:
+        case_ids = {
+            case_id
+            for name in names
+            for case_id in by_test.get(name.rsplit("::", 1)[-1], [])
+        }
+        return sorted(case_ids)
+
+    passed_cases = parity_cases(log.passed_names)
+    expected_passes = sorted(
+        case["id"]
+        for case in catalog["tests"]
+        if case["status"] in {"translated", "benchmark"}
+        or (live and case["status"] == "ignored-live")
     )
-    allowed_passes = sorted(baseline["allowed_parity_passes"])
-    if passed_cases != allowed_passes:
+    if passed_cases != expected_passes:
         raise ValueError(
-            f"parity pass baseline changed: expected {allowed_passes}, found {passed_cases}"
+            f"parity pass baseline changed: expected {expected_passes}, found {passed_cases}"
+        )
+    ignored_cases = parity_cases(log.ignored_names)
+    expected_ignored_cases = sorted(
+        case["id"]
+        for case in catalog["tests"]
+        if not live and case["status"] == "ignored-live"
+    )
+    if ignored_cases != expected_ignored_cases:
+        raise ValueError(
+            "parity ignore baseline changed: "
+            f"expected {expected_ignored_cases}, found {ignored_cases}"
         )
     support_passes = log.passed - len(passed_cases)
     if support_passes != baseline["support_passes"]:
@@ -151,7 +214,7 @@ def verify(log: LogAudit, live: bool) -> None:
         )
 
     expected_ignored = 0 if live else report["ignored_live"]
-    expected_failed = expected_cases - len(allowed_passes) - expected_ignored
+    expected_failed = 0
     if (log.failed, log.ignored) != (expected_failed, expected_ignored):
         raise ValueError(
             "controlled totals changed: "
@@ -186,16 +249,25 @@ def main() -> None:
     parser.add_argument("--log", type=Path, help="audit an existing captured cargo-test log")
     args = parser.parse_args()
     if args.log:
-        return_code, text = 101, args.log.read_text()
+        return_code, text = None, args.log.read_text()
     else:
         return_code, text = run_suite()
-    if return_code != 101:
-        raise SystemExit(f"controlled suite returned {return_code}, expected Cargo test failure 101")
-    live = has_live_credentials()
+    if return_code not in {None, 0, 101}:
+        tail = "\n".join(text.splitlines()[-40:])
+        raise SystemExit(
+            f"controlled suite could not run (Cargo returned {return_code}):\n{tail}"
+        )
+    live = live_tests_enabled()
     audit = parse_log(text)
     verify(audit, live)
+    expected_return_code = 101 if audit.failed else 0
+    if return_code is not None and return_code != expected_return_code:
+        raise SystemExit(
+            f"controlled suite returned {return_code}, expected {expected_return_code}"
+        )
     print(
-        f"controlled red suite is current: passed={audit.passed}, failed={audit.failed}, "
+        f"controlled compatibility suite is current: passed={audit.passed}, "
+        f"failed={audit.failed}, "
         f"ignored={audit.ignored}, live_credentials={str(live).lower()}, "
         f"standardized_members={len(audit.members)}"
     )