Add durable NUnit parity harness

tools/generate_surface.py

@@ -13,7 +13,9 @@ import hashlib
 import json
 import keyword
 import re
-from collections import defaultdict
+import subprocess
+import tempfile
+from collections import Counter, defaultdict
 from dataclasses import dataclass, field
 from pathlib import Path
 
@@ -51,13 +53,29 @@ TYPE_RE = re.compile(
 )
 NAMESPACE_RE = re.compile(r"^\s*namespace\s+([A-Za-z_][A-Za-z0-9_.]*)", re.MULTILINE)
 TEST_ATTR_RE = re.compile(r"\[(Test|TestCase)(?:\((.*?)\))?\]", re.DOTALL)
+CATEGORY_RE = re.compile(r'\[Category\("([^"]+)"\)\]')
 METHOD_RE = re.compile(
     r"\b(?:public|internal)\s+(?:static\s+)?(?:async\s+)?"
     r"(?:void|Task(?:\s*<[^>]+>)?|ValueTask(?:\s*<[^>]+>)?)\s+"
     r"([A-Za-z_][A-Za-z0-9_]*)\s*\(",
     re.DOTALL,
 )
-CLASS_RE = re.compile(r"\b(?:public|internal)\s+(?:sealed\s+|partial\s+)?class\s+([A-Za-z_][A-Za-z0-9_]*)")
+CLASS_RE = re.compile(
+    r"\b(?:(?:public|internal|private|protected|sealed|partial|abstract|static)\s+)*"
+    r"class\s+([A-Za-z_][A-Za-z0-9_]*)"
+)
+PARITY_MARKER_RE = re.compile(
+    r"^// parity-case: (?P<id>\S+) (?P<body_sha256>[0-9a-f]{64}) "
+    r"(?P<status>pending|translated|ignored-live|benchmark)$",
+    re.MULTILINE,
+)
+RUST_TEST_RE = re.compile(r"(?:#\[[^\]]+\]\s*)*fn\s+([A-Za-z_][A-Za-z0-9_]*)\s*\(", re.MULTILINE)
+EXPECTED_TESTS = 1295
+PARITY_FILES = (
+    Path("tests/compat/tests/generated_parity.rs"),
+    Path("tests/PARITY.md"),
+    Path("tests/upstream-tests.json"),
+)
 
 
 def snake(name: str) -> str:
@@ -200,9 +218,28 @@ def generate_apis(upstream: Path, output: Path) -> tuple[int, int]:
     return total_types, total_members
 
 
-def enclosing_class(text: str, position: int) -> str:
-    matches = [m for m in CLASS_RE.finditer(text) if m.start() <= position]
-    return matches[-1].group(1) if matches else "UnknownFixture"
+def enclosing_class_match(text: str, position: int) -> re.Match[str] | None:
+    containing: list[re.Match[str]] = []
+    for match in CLASS_RE.finditer(text, 0, position):
+        brace = text.find("{", match.end())
+        if brace == -1 or brace >= position:
+            continue
+        depth = 0
+        for index in range(brace, len(text)):
+            if text[index] == "{":
+                depth += 1
+            elif text[index] == "}":
+                depth -= 1
+                if depth == 0:
+                    if position < index:
+                        containing.append(match)
+                    break
+    return containing[-1] if containing else None
+
+
+def attributes_before(text: str, position: int) -> str:
+    match = re.search(r"((?:\s*\[[^\]]+\]\s*)+)$", text[:position], re.DOTALL)
+    return match.group(1) if match else ""
 
 
 def method_body_hash(text: str, method_start: int) -> str:
@@ -227,13 +264,41 @@ def method_body_hash(text: str, method_start: int) -> str:
     return hashlib.sha256(body.encode()).hexdigest()
 
 
-def generate_tests(upstream: Path, output: Path) -> int:
+def stable_case_id(source: str, fixture: str, method: str, attribute: str, parameters: str | None) -> str:
+    test_id = f"{source}::{fixture}.{method}"
+    if attribute == "Test":
+        return f"{test_id}::test"
+    normalized = " ".join((parameters or "").split())
+    digest = hashlib.sha256(normalized.encode()).hexdigest()[:16]
+    return f"{test_id}::case:{digest}"
+
+
+def helper_types(roots: tuple[Path, ...], upstream: Path) -> dict[str, str]:
+    helpers: dict[str, str] = {}
+    for root in roots:
+        for path in source_files(root):
+            text = path.read_text(encoding="utf-8-sig")
+            if TEST_ATTR_RE.search(text):
+                continue
+            relative = path.relative_to(upstream).as_posix()
+            for match in CLASS_RE.finditer(text):
+                helpers.setdefault(match.group(1), relative)
+    return helpers
+
+
+def extract_tests(upstream: Path) -> list[dict[str, object]]:
     roots = (upstream / "LibreMetaverse.Tests", upstream / "LibreMetaverse.Rendering.Tests")
+    helpers = helper_types(roots, upstream)
     tests: list[dict[str, object]] = []
     for root in roots:
         for path in source_files(root):
             text = path.read_text(encoding="utf-8-sig")
             relative = path.relative_to(upstream).as_posix()
+            dependencies = sorted(
+                helper_path
+                for helper, helper_path in helpers.items()
+                if helper_path != relative and re.search(rf"\b{re.escape(helper)}\b", text)
+            )
             attrs = list(TEST_ATTR_RE.finditer(text))
             for attr_index, attr in enumerate(attrs):
                 next_attr = attrs[attr_index + 1].start() if attr_index + 1 < len(attrs) else len(text)
@@ -243,60 +308,279 @@ def generate_tests(upstream: Path, output: Path) -> int:
                 if method is None:
                     raise RuntimeError(f"No test method after {relative}:{text.count(chr(10), 0, attr.start()) + 1}")
                 name = method.group(1)
-                fixture = enclosing_class(text, method.start())
+                class_match = enclosing_class_match(text, method.start())
+                fixture = class_match.group(1) if class_match else "UnknownFixture"
                 attr_text = " ".join(attr.group(0).split())
+                parameters = " ".join((attr.group(2) or "").split()) or None
                 line = text.count("\n", 0, attr.start()) + 1
+                category_text = text[attr.start() : method.start()]
+                if class_match:
+                    category_text += attributes_before(text, class_match.start())
+                categories = sorted(set(CATEGORY_RE.findall(category_text)))
                 tests.append(
                     {
+                        "id": stable_case_id(relative, fixture, name, attr.group(1), parameters),
+                        "csharp_test_id": f"{relative}::{fixture}.{name}",
+                        "parameter_case": parameters,
                         "source": relative,
                         "line": line,
                         "fixture": fixture,
                         "method": name,
                         "attribute": attr_text,
                         "body_sha256": method_body_hash(text, method.start()),
+                        "categories": categories,
+                        "fixture_dependencies": dependencies,
                     }
                 )
-    seen: defaultdict[str, int] = defaultdict(int)
+    ids = [str(test["id"]) for test in tests]
+    duplicates = sorted(case_id for case_id, count in Counter(ids).items() if count > 1)
+    if duplicates:
+        raise RuntimeError("Duplicate stable NUnit case IDs: " + ", ".join(duplicates))
+    return tests
+
+
+def reviewed_tests(root: Path) -> dict[str, dict[str, object]]:
+    reviews: dict[str, dict[str, object]] = {}
+    tests_root = root / "tests" / "compat" / "tests"
+    for path in sorted(tests_root.rglob("*.rs")):
+        if path.name == "generated_parity.rs":
+            continue
+        text = path.read_text()
+        for marker in PARITY_MARKER_RE.finditer(text):
+            status = marker.group("status")
+            if status == "pending":
+                raise RuntimeError(f"Pending parity marker must stay generated: {path}:{text.count(chr(10), 0, marker.start()) + 1}")
+            next_marker = PARITY_MARKER_RE.search(text, marker.end())
+            rust_test = RUST_TEST_RE.search(text, marker.end(), next_marker.start() if next_marker else len(text))
+            if rust_test is None:
+                raise RuntimeError(f"Parity marker has no following Rust test: {path}:{text.count(chr(10), 0, marker.start()) + 1}")
+            case_id = marker.group("id")
+            if case_id in reviews:
+                raise RuntimeError(f"Duplicate reviewed parity case: {case_id}")
+            reviews[case_id] = {
+                "body_sha256": marker.group("body_sha256"),
+                "status": status,
+                "rust_file": path.relative_to(root).as_posix(),
+                "rust_line": text.count("\n", 0, marker.start()) + 1,
+                "rust_test": rust_test.group(1),
+            }
+    return reviews
+
+
+def render_parity_report(tests: list[dict[str, object]]) -> str:
+    status_counts = {status: sum(test["status"] == status for test in tests) for status in ("pending", "translated", "ignored-live", "benchmark", "drifted")}
+    live_candidates = sum("RequiresLiveServer" in test["categories"] for test in tests)
+    benchmark_candidates = sum("Benchmark" in test["categories"] for test in tests)
+    lines = [
+        "# NUnit to Rust parity ledger",
+        "",
+        f"Generated from LibreMetaverse `{UPSTREAM_COMMIT}`. Stable IDs identify one NUnit `[Test]` or `[TestCase]` invocation; the body hash covers the original C# method declaration and body.",
+        "",
+        "## Status",
+        "",
+        f"- Total: **{len(tests):,}**",
+        f"- Pending/unreviewed: **{status_counts['pending']:,}**",
+        f"- Translated/reviewed: **{status_counts['translated']:,}**",
+        f"- Ignored live/reviewed: **{status_counts['ignored-live']:,}** ({live_candidates:,} upstream live candidates)",
+        f"- Benchmarks/reviewed: **{status_counts['benchmark']:,}** ({benchmark_candidates:,} upstream benchmark candidates)",
+        f"- Drifted: **{status_counts['drifted']:,}**",
+        "",
+        "Reviewed Rust tests live outside `generated_parity.rs` and carry a `parity-case` marker. Regeneration preserves those files and fails if their source body hash drifts.",
+        "",
+        "| Stable case ID | C# test | Parameter case | Source | Categories | Fixtures | Rust location | Status | Body SHA-256 |",
+        "|---|---|---|---|---|---|---|---|---|",
+    ]
+    for test in tests:
+        parameter = str(test["parameter_case"] or "").replace("|", "&#124;")
+        categories = ", ".join(test["categories"])
+        fixtures = ", ".join(test["fixture_dependencies"])
+        rust_location = f"{test['rust_file']}:{test['rust_line']} (`{test['rust_test']}`)"
+        lines.append(
+            f"| `{test['id']}` | `{test['fixture']}.{test['method']}` | `{parameter}` | `{test['source']}:{test['line']}` | `{categories}` | `{fixtures}` | `{rust_location}` | `{test['status']}` | `{test['body_sha256']}` |"
+        )
+    return "\n".join(lines) + "\n"
+
+
+def generate_tests(upstream: Path, output: Path, review_root: Path | None = None) -> int:
+    tests = extract_tests(upstream)
+    reviews = reviewed_tests(review_root or output)
+    tests_by_id = {str(test["id"]): test for test in tests}
+    stale = sorted(set(reviews) - set(tests_by_id))
+    if stale:
+        raise RuntimeError("Reviewed cases missing from pinned upstream: " + ", ".join(stale))
+    drifted = sorted(
+        case_id
+        for case_id, review in reviews.items()
+        if review["body_sha256"] != tests_by_id[case_id]["body_sha256"]
+    )
+    if drifted:
+        raise RuntimeError("Reviewed cases have drifted C# bodies: " + ", ".join(drifted))
+
     rust_lines = [
         "// @generated by tools/generate_surface.py; do not edit by hand.",
         f"// Source: LibreMetaverse {UPSTREAM_COMMIT}",
         "use libremetaverse_compat_tests::pending;",
         "",
     ]
-    parity_lines = [
-        "# NUnit to Rust parity ledger",
-        "",
-        f"Generated from LibreMetaverse `{UPSTREAM_COMMIT}`. Each row is one NUnit `[Test]` or `[TestCase]` invocation. The body hash covers the original C# method declaration and body.",
-        "",
-        "| Rust test | C# test | Source | Attribute | Body SHA-256 |",
-        "|---|---|---|---|---|",
-    ]
     for test in tests:
+        review = reviews.get(str(test["id"]))
+        if review:
+            test.update(review)
+            test["semantic_review"] = "reviewed"
+            continue
         base = snake(f"{test['fixture']}_{test['method']}")
-        seen[base] += 1
-        rust_name = base if seen[base] == 1 else f"{base}_case_{seen[base]}"
-        test["rust_test"] = rust_name
+        suffix = hashlib.sha256(str(test["id"]).encode()).hexdigest()[:12]
+        rust_name = f"{base[:64].rstrip('_')}_{suffix}"
+        test.update(
+            {
+                "rust_file": "tests/compat/tests/generated_parity.rs",
+                "rust_line": len(rust_lines) + 1,
+                "rust_test": rust_name,
+                "status": "pending",
+                "semantic_review": "unreviewed",
+            }
+        )
         identity = f"{test['fixture']}.{test['method']}"
         rust_lines.extend(
             [
+                f"// parity-case: {test['id']} {test['body_sha256']} pending",
                 "#[test]",
                 f"fn {rust_name}() {{",
-                f"    pending({json.dumps(test['source'])}, {test['line']}, {json.dumps(identity)}, {json.dumps(test['attribute'])}, {json.dumps(test['body_sha256'])});",
+                "    pending(",
+                f"        {json.dumps(test['id'])},",
+                f"        {json.dumps(test['source'])},",
+                f"        {test['line']},",
+                f"        {json.dumps(identity)},",
+                f"        {json.dumps(test['attribute'])},",
+                f"        {json.dumps(test['body_sha256'])},",
+                "    );",
                 "}",
                 "",
             ]
         )
-        parity_lines.append(
-            f"| `{rust_name}` | `{identity}` | `{test['source']}:{test['line']}` | `{str(test['attribute']).replace('|', '&#124;')}` | `{test['body_sha256']}` |"
-        )
     generated_dir = output / "tests" / "compat" / "tests"
     generated_dir.mkdir(parents=True, exist_ok=True)
     (generated_dir / "generated_parity.rs").write_text("\n".join(rust_lines))
-    (output / "tests" / "PARITY.md").write_text("\n".join(parity_lines) + "\n")
-    (output / "tests" / "upstream-tests.json").write_text(json.dumps({"upstream_commit": UPSTREAM_COMMIT, "tests": tests}, indent=2) + "\n")
+    (output / "tests" / "PARITY.md").write_text(render_parity_report(tests))
+    report = {
+        "pending": sum(test["status"] == "pending" for test in tests),
+        "translated": sum(test["status"] == "translated" for test in tests),
+        "ignored_live": sum(test["status"] == "ignored-live" for test in tests),
+        "benchmark": sum(test["status"] == "benchmark" for test in tests),
+        "drifted": 0,
+        "unreviewed": sum(test["semantic_review"] == "unreviewed" for test in tests),
+    }
+    catalog = {
+        "schema_version": 1,
+        "upstream_commit": UPSTREAM_COMMIT,
+        "expected_cases": EXPECTED_TESTS,
+        "report": report,
+        "tests": tests,
+    }
+    (output / "tests" / "upstream-tests.json").write_text(json.dumps(catalog, indent=2) + "\n")
     return len(tests)
 
 
+def scan_parity_markers(root: Path) -> dict[str, list[dict[str, object]]]:
+    markers: defaultdict[str, list[dict[str, object]]] = defaultdict(list)
+    for path in sorted((root / "tests" / "compat" / "tests").rglob("*.rs")):
+        text = path.read_text()
+        for marker in PARITY_MARKER_RE.finditer(text):
+            next_marker = PARITY_MARKER_RE.search(text, marker.end())
+            rust_test = RUST_TEST_RE.search(text, marker.end(), next_marker.start() if next_marker else len(text))
+            if rust_test is None:
+                raise RuntimeError(f"Parity marker has no following Rust test: {path}:{text.count(chr(10), 0, marker.start()) + 1}")
+            markers[marker.group("id")].append(
+                {
+                    "body_sha256": marker.group("body_sha256"),
+                    "status": marker.group("status"),
+                    "rust_file": path.relative_to(root).as_posix(),
+                    "rust_line": text.count("\n", 0, marker.start()) + 1,
+                    "rust_test": rust_test.group(1),
+                }
+            )
+    return dict(markers)
+
+
+def check_test_parity(root: Path, require_reviewed: bool = False) -> dict[str, int]:
+    catalog_path = root / "tests" / "upstream-tests.json"
+    catalog = json.loads(catalog_path.read_text())
+    if catalog.get("schema_version") != 1:
+        raise RuntimeError("Unsupported test parity catalog schema")
+    tests = catalog.get("tests", [])
+    expected = catalog.get("expected_cases")
+    if expected != EXPECTED_TESTS or len(tests) != EXPECTED_TESTS:
+        raise RuntimeError(f"Expected {EXPECTED_TESTS} cataloged NUnit invocations, found {len(tests)}")
+    if catalog.get("upstream_commit") != UPSTREAM_COMMIT:
+        raise RuntimeError("Test parity catalog targets the wrong upstream commit")
+
+    ids = [test["id"] for test in tests]
+    duplicate_ids = sorted(case_id for case_id, count in Counter(ids).items() if count > 1)
+    if duplicate_ids:
+        raise RuntimeError("Duplicate catalog case IDs: " + ", ".join(duplicate_ids))
+    by_id = {test["id"]: test for test in tests}
+    markers = scan_parity_markers(root)
+    missing = sorted(set(by_id) - set(markers))
+    stale = sorted(set(markers) - set(by_id))
+    duplicates = sorted(case_id for case_id, entries in markers.items() if len(entries) != 1)
+    if missing:
+        raise RuntimeError("Missing Rust parity cases: " + ", ".join(missing))
+    if stale:
+        raise RuntimeError("Stale Rust parity cases: " + ", ".join(stale))
+    if duplicates:
+        raise RuntimeError("Duplicate Rust parity cases: " + ", ".join(duplicates))
+
+    drifted: list[str] = []
+    mismatched: list[str] = []
+    for case_id, test in by_id.items():
+        marker = markers[case_id][0]
+        if marker["body_sha256"] != test["body_sha256"]:
+            drifted.append(case_id)
+        for key in ("status", "rust_file", "rust_line", "rust_test"):
+            if marker[key] != test[key]:
+                mismatched.append(f"{case_id}:{key}")
+    if drifted:
+        raise RuntimeError("Drifted reviewed C# test bodies: " + ", ".join(drifted))
+    if mismatched:
+        raise RuntimeError("Stale parity catalog metadata: " + ", ".join(mismatched))
+
+    report = {
+        "pending": sum(test["status"] == "pending" for test in tests),
+        "translated": sum(test["status"] == "translated" for test in tests),
+        "ignored_live": sum(test["status"] == "ignored-live" for test in tests),
+        "benchmark": sum(test["status"] == "benchmark" for test in tests),
+        "drifted": 0,
+        "unreviewed": sum(test["semantic_review"] != "reviewed" for test in tests),
+    }
+    if catalog.get("report") != report:
+        raise RuntimeError("Stale parity summary report")
+    if require_reviewed and report["unreviewed"]:
+        raise RuntimeError(f"{report['unreviewed']} NUnit invocations remain semantically unreviewed")
+    return report
+
+
+def verify_upstream(upstream: Path) -> None:
+    commit = subprocess.run(
+        ["git", "-C", str(upstream), "rev-parse", "HEAD"],
+        check=True,
+        capture_output=True,
+        text=True,
+    ).stdout.strip()
+    if commit != UPSTREAM_COMMIT:
+        raise RuntimeError(f"Expected LibreMetaverse {UPSTREAM_COMMIT}, found {commit}")
+
+
+def check_test_regeneration(upstream: Path, output: Path) -> None:
+    with tempfile.TemporaryDirectory() as temporary:
+        generated = Path(temporary)
+        count = generate_tests(upstream, generated, review_root=output)
+        if count != EXPECTED_TESTS:
+            raise RuntimeError(f"Expected {EXPECTED_TESTS} NUnit invocations at {UPSTREAM_COMMIT}, found {count}")
+        stale = [str(path) for path in PARITY_FILES if not (output / path).exists() or (output / path).read_bytes() != (generated / path).read_bytes()]
+        if stale:
+            raise RuntimeError("Stale generated test parity files: " + ", ".join(stale))
+
+
 def generate_program_manifest(upstream: Path, output: Path) -> int:
     roots = [
         upstream / "Programs" / "VivoxTest",
@@ -331,14 +615,28 @@ def main() -> None:
     parser = argparse.ArgumentParser()
     parser.add_argument("--upstream", type=Path, default=Path("../libremetaverse"))
     parser.add_argument("--output", type=Path, default=Path("."))
+    parser.add_argument("--tests-only", action="store_true")
+    parser.add_argument("--check", action="store_true")
     args = parser.parse_args()
     upstream = args.upstream.resolve()
     output = args.output.resolve()
+    verify_upstream(upstream)
+    if args.check:
+        check_test_regeneration(upstream, output)
+        report = check_test_parity(output)
+        print(f"test parity is current: {report}")
+        return
+    if args.tests_only:
+        tests = generate_tests(upstream, output)
+        if tests != EXPECTED_TESTS:
+            raise RuntimeError(f"Expected {EXPECTED_TESTS} NUnit invocations at {UPSTREAM_COMMIT}, found {tests}")
+        print(f"generated {tests} test parity cases")
+        return
     types, members = generate_apis(upstream, output)
     tests = generate_tests(upstream, output)
     programs = generate_program_manifest(upstream, output)
-    if tests != 1295:
-        raise RuntimeError(f"Expected 1295 NUnit invocations at {UPSTREAM_COMMIT}, found {tests}")
+    if tests != EXPECTED_TESTS:
+        raise RuntimeError(f"Expected {EXPECTED_TESTS} NUnit invocations at {UPSTREAM_COMMIT}, found {tests}")
     print(f"generated {types} type declarations, {members} public declaration lines, {tests} tests, and {programs} program manifests")