Generate native LSL parser tables (#83)

tools/check_milestone_10_issue_83.py

@@ -0,0 +1,204 @@
+#!/usr/bin/env python3
+"""Audit issue 83's deterministic native LSL parser/token generation."""
+
+from __future__ import annotations
+
+import json
+import re
+from pathlib import Path
+
+import generate_api_shims
+import generate_lsl_tables
+
+
+ROOT = Path(__file__).resolve().parents[1]
+CATALOG = ROOT / "api" / "public-api.json"
+INPUT = ROOT / "codegen" / "inputs" / "lsl_tools_grammar.json"
+TABLES = ROOT / "crates" / "libremetaverse-lsl-tools" / "src" / "generated_tables.rs"
+GENERATOR = ROOT / "crates" / "libremetaverse-lsl-tools" / "src" / "generator.rs"
+GENERATED = ROOT / "crates" / "libremetaverse-lsl-tools" / "src" / "generated.rs"
+TESTS = ROOT / "crates" / "libremetaverse-lsl-tools" / "tests" / "generator_compat.rs"
+DOC = ROOT / "crates" / "libremetaverse-lsl-tools" / "README.md"
+WORKFLOW = ROOT / ".gitea" / "workflows" / "rust-workspace.yml"
+SCRIPT = ROOT / "tools" / "generate_lsl_tables.py"
+STUB_RE = re.compile(
+    r"\b(?:not_implemented|unimplemented_api)\b|\b(?:todo|unimplemented)!\s*\("
+)
+
+
+def owned_types() -> dict[str, str]:
+    return {
+        doc_id: declaration
+        for doc_id, declaration in generate_api_shims.NATIVE_TYPES.items()
+        if declaration.startswith("crate::generated_tables::")
+        or declaration.startswith("crate::generator::")
+    }
+
+
+def catalog_counts(doc_ids: set[str]) -> tuple[int, int]:
+    catalog = json.loads(CATALOG.read_text())
+    assembly = next(
+        value
+        for value in catalog["assemblies"]
+        if value["identity"]["name"] == "LibreMetaverse.LslTools"
+    )
+    found: dict[str, int] = {}
+
+    def visit(value: object) -> None:
+        if isinstance(value, dict):
+            doc_id = value.get("doc_id")
+            if doc_id in doc_ids and str(doc_id).startswith("T:"):
+                found[str(doc_id)] = len(value.get("members", []))
+            for child in value.values():
+                visit(child)
+        elif isinstance(value, list):
+            for child in value:
+                visit(child)
+
+    visit(assembly["types"])
+    if set(found) != doc_ids:
+        missing = sorted(doc_ids - set(found))
+        raise SystemExit(f"issue 83 catalog types are missing: {missing}")
+    return len(found), sum(found.values())
+
+
+def generated_type_block(text: str, doc_id: str) -> str:
+    marker = f"/// C# type: `{doc_id}`."
+    start = text.find(marker)
+    if start < 0:
+        raise SystemExit(f"generated declaration is missing for {doc_id}")
+    next_type = text.find("/// C# type:", start + len(marker))
+    return text[start:] if next_type < 0 else text[start:next_type]
+
+
+def require_markers(path: Path, markers: tuple[str, ...]) -> None:
+    text = path.read_text()
+    missing = [marker for marker in markers if marker not in text]
+    if missing:
+        raise SystemExit(f"{path.name}: audit evidence missing: " + ", ".join(missing))
+
+
+def main() -> None:
+    types = owned_types()
+    type_count, member_count = catalog_counts(set(types))
+    if (type_count, member_count) != (120, 365):
+        raise SystemExit(
+            f"issue 83 expected 120 mapped types and 365 members, found "
+            f"{type_count} and {member_count}"
+        )
+
+    data = json.loads(INPUT.read_text())
+    if data["source"].get("license") != "BSD-3-Clause":
+        raise SystemExit("issue 83 reviewed grammar input has no BSD-3-Clause provenance")
+    if len(data["tokens"]) != 14 or len(data["nonterminals"]) != 8:
+        raise SystemExit("issue 83 grammar must contain 14 tokens and 8 nonterminals")
+    if len(data["productions"]) != 25:
+        raise SystemExit("issue 83 grammar must contain all 25 reviewed productions")
+    first = generate_lsl_tables.render(data).encode()
+    second = generate_lsl_tables.render(json.loads(INPUT.read_text())).encode()
+    if first != second or first != TABLES.read_bytes():
+        raise SystemExit("two clean LSL generations were not byte-identical")
+
+    for path in (TABLES, GENERATOR):
+        source = path.read_text()
+        if STUB_RE.search(source):
+            raise SystemExit(f"issue 83 owned Rust stubs remain in {path.name}")
+        if re.search(r"unsafe\s*\{|unsafe\s+impl|target_os\s*=\s*\"macos\"", source):
+            raise SystemExit(f"{path.name} contains unsafe or macOS-only code")
+    script = SCRIPT.read_text()
+    if re.search(r"\b(?:dotnet|csc|mcs)\b", script, re.IGNORECASE):
+        raise SystemExit("LSL regeneration depends on a C#/.NET process")
+
+    generated = GENERATED.read_text()
+    for doc_id in types:
+        if STUB_RE.search(generated_type_block(generated, doc_id)):
+            raise SystemExit(f"issue 83 generated stubs remain for {doc_id}")
+
+    require_markers(
+        TABLES,
+        (
+            "LSL_GENERATOR_SOURCE_COMMIT",
+            "LSL_GENERATOR_SOURCE_PARSER",
+            "LSL_GENERATOR_SOURCE_LEXER",
+            "LSL_GENERATOR_SOURCE_LICENSE",
+            "LSL_GENERATOR_SERIALIZATION_VERSION",
+            "GENERATED_PARSER_DATA",
+            "pub fn generated_parser()",
+            "pub fn generated_lexer_with_handler",
+            "pub struct GeneratedLexerToken",
+            "pub struct GeneratedParserSymbol",
+            "pub struct yycs0syntax",
+            "pub struct yycs0tokens",
+            "pub struct cs0syntax",
+            "pub struct cs0tokens",
+        ),
+    )
+    require_markers(
+        GENERATOR,
+        (
+            "pub struct Serialiser",
+            "pub struct GenBase",
+            "pub struct SymbolsGen",
+            "pub struct TokensGen",
+            "pub struct Regex",
+            "pub struct Nfa",
+            "pub struct Sfactory",
+            "pub struct Tfactory",
+            "pub struct TokClassDef",
+            "pub struct ObjectListOListEnumerator",
+            "pub struct AddToFunc",
+            "MAX_GENERATED_DFA_STATES",
+            "MAX_REGEX_UNITS",
+            "System.Char",
+        ),
+    )
+    require_markers(
+        TESTS,
+        (
+            "generated_lexer_preserves_tokens_reserved_words_and_eof",
+            "generated_parser_accepts_the_reviewed_class_body_grammar",
+            "parser_and_lexer_emission_is_byte_deterministic",
+            "generated_token_classes_have_live_identity_and_text",
+            "regex_and_nfa_build_a_functional_native_dfa",
+            "serialiser_round_trips_deterministically_with_versioning",
+            "compatibility_enumerator_and_generator_state_are_live",
+            "symbol_and_token_factories_execute_registered_rust_closures",
+            "symbols_paths_and_token_class_definitions_keep_native_state",
+        ),
+    )
+    if TESTS.read_text().count("#[test]") != 9:
+        raise SystemExit("issue 83 expected 9 focused generator fixtures")
+    require_markers(
+        DOC,
+        (
+            "Checked-in generator contract",
+            "25-production grammar",
+            "84 generated syntax",
+            "13 generated token classes",
+            "python3 tools/generate_lsl_tables.py --check",
+            "46 focused",
+            "ubuntu-latest",
+        ),
+    )
+    workflow = WORKFLOW.read_text()
+    require_markers(
+        WORKFLOW,
+        (
+            "python3 tools/check_milestone_10_issue_83.py",
+            "python3 tools/generate_lsl_tables.py --check",
+        ),
+    )
+    if "runs-on: ubuntu-latest" not in workflow or re.search(
+        r"runs-on:\s*(?:macos|windows)", workflow, re.IGNORECASE
+    ):
+        raise SystemExit("issue 83 workflow is not ubuntu-only")
+
+    print(
+        "issue 83 audit: 120 native mapped generator/table types and 365 members, "
+        "14 tokens, 8 nonterminals, 25 productions, two byte-identical generations, "
+        "9 focused fixtures, migration docs, and ubuntu-only CI are present"
+    )
+
+
+if __name__ == "__main__":
+    main()