MetaCrate/tools/check_milestone_10_issue_83.py

#!/usr/bin/env python3
"""Audit issue 83's deterministic native LSL parser/token generation."""

from __future__ import annotations

import json
import re
from pathlib import Path

import generate_api_shims
import generate_lsl_tables


ROOT = Path(__file__).resolve().parents[1]
CATALOG = ROOT / "api" / "public-api.json"
INPUT = ROOT / "codegen" / "inputs" / "lsl_tools_grammar.json"
TABLES = ROOT / "crates" / "libremetaverse-lsl-tools" / "src" / "generated_tables.rs"
GENERATOR = ROOT / "crates" / "libremetaverse-lsl-tools" / "src" / "generator.rs"
GENERATED = ROOT / "crates" / "libremetaverse-lsl-tools" / "src" / "generated.rs"
TESTS = ROOT / "crates" / "libremetaverse-lsl-tools" / "tests" / "generator_compat.rs"
DOC = ROOT / "crates" / "libremetaverse-lsl-tools" / "README.md"
WORKFLOW = ROOT / ".gitea" / "workflows" / "rust-workspace.yml"
SCRIPT = ROOT / "tools" / "generate_lsl_tables.py"
STUB_RE = re.compile(
    r"\b(?:not_implemented|unimplemented_api)\b|\b(?:todo|unimplemented)!\s*\("
)


def owned_types() -> dict[str, str]:
    return {
        doc_id: declaration
        for doc_id, declaration in generate_api_shims.NATIVE_TYPES.items()
        if declaration.startswith("crate::generated_tables::")
        or declaration.startswith("crate::generator::")
    }


def catalog_counts(doc_ids: set[str]) -> tuple[int, int]:
    catalog = json.loads(CATALOG.read_text())
    assembly = next(
        value
        for value in catalog["assemblies"]
        if value["identity"]["name"] == "LibreMetaverse.LslTools"
    )
    found: dict[str, int] = {}

    def visit(value: object) -> None:
        if isinstance(value, dict):
            doc_id = value.get("doc_id")
            if doc_id in doc_ids and str(doc_id).startswith("T:"):
                found[str(doc_id)] = len(value.get("members", []))
            for child in value.values():
                visit(child)
        elif isinstance(value, list):
            for child in value:
                visit(child)

    visit(assembly["types"])
    if set(found) != doc_ids:
        missing = sorted(doc_ids - set(found))
        raise SystemExit(f"issue 83 catalog types are missing: {missing}")
    return len(found), sum(found.values())


def generated_type_block(text: str, doc_id: str) -> str:
    marker = f"/// C# type: `{doc_id}`."
    start = text.find(marker)
    if start < 0:
        raise SystemExit(f"generated declaration is missing for {doc_id}")
    next_type = text.find("/// C# type:", start + len(marker))
    return text[start:] if next_type < 0 else text[start:next_type]


def require_markers(path: Path, markers: tuple[str, ...]) -> None:
    text = path.read_text()
    missing = [marker for marker in markers if marker not in text]
    if missing:
        raise SystemExit(f"{path.name}: audit evidence missing: " + ", ".join(missing))


def main() -> None:
    types = owned_types()
    type_count, member_count = catalog_counts(set(types))
    if (type_count, member_count) != (120, 365):
        raise SystemExit(
            f"issue 83 expected 120 mapped types and 365 members, found "
            f"{type_count} and {member_count}"
        )

    data = json.loads(INPUT.read_text())
    if data["source"].get("license") != "BSD-3-Clause":
        raise SystemExit("issue 83 reviewed grammar input has no BSD-3-Clause provenance")
    if len(data["tokens"]) != 14 or len(data["nonterminals"]) != 8:
        raise SystemExit("issue 83 grammar must contain 14 tokens and 8 nonterminals")
    if len(data["productions"]) != 25:
        raise SystemExit("issue 83 grammar must contain all 25 reviewed productions")
    first = generate_lsl_tables.render(data).encode()
    second = generate_lsl_tables.render(json.loads(INPUT.read_text())).encode()
    if first != second or first != TABLES.read_bytes():
        raise SystemExit("two clean LSL generations were not byte-identical")

    for path in (TABLES, GENERATOR):
        source = path.read_text()
        if STUB_RE.search(source):
            raise SystemExit(f"issue 83 owned Rust stubs remain in {path.name}")
        if re.search(r"unsafe\s*\{|unsafe\s+impl|target_os\s*=\s*\"macos\"", source):
            raise SystemExit(f"{path.name} contains unsafe or macOS-only code")
    script = SCRIPT.read_text()
    if re.search(r"\b(?:dotnet|csc|mcs)\b", script, re.IGNORECASE):
        raise SystemExit("LSL regeneration depends on a C#/.NET process")

    generated = GENERATED.read_text()
    for doc_id in types:
        if STUB_RE.search(generated_type_block(generated, doc_id)):
            raise SystemExit(f"issue 83 generated stubs remain for {doc_id}")

    require_markers(
        TABLES,
        (
            "LSL_GENERATOR_SOURCE_COMMIT",
            "LSL_GENERATOR_SOURCE_PARSER",
            "LSL_GENERATOR_SOURCE_LEXER",
            "LSL_GENERATOR_SOURCE_LICENSE",
            "LSL_GENERATOR_SERIALIZATION_VERSION",
            "GENERATED_PARSER_DATA",
            "pub fn generated_parser()",
            "pub fn generated_lexer_with_handler",
            "pub struct GeneratedLexerToken",
            "pub struct GeneratedParserSymbol",
            "pub struct yycs0syntax",
            "pub struct yycs0tokens",
            "pub struct cs0syntax",
            "pub struct cs0tokens",
        ),
    )
    require_markers(
        GENERATOR,
        (
            "pub struct Serialiser",
            "pub struct GenBase",
            "pub struct SymbolsGen",
            "pub struct TokensGen",
            "pub struct Regex",
            "pub struct Nfa",
            "pub struct Sfactory",
            "pub struct Tfactory",
            "pub struct TokClassDef",
            "pub struct ObjectListOListEnumerator",
            "pub struct AddToFunc",
            "MAX_GENERATED_DFA_STATES",
            "MAX_REGEX_UNITS",
            "System.Char",
        ),
    )
    require_markers(
        TESTS,
        (
            "generated_lexer_preserves_tokens_reserved_words_and_eof",
            "generated_parser_accepts_the_reviewed_class_body_grammar",
            "parser_and_lexer_emission_is_byte_deterministic",
            "generated_token_classes_have_live_identity_and_text",
            "regex_and_nfa_build_a_functional_native_dfa",
            "serialiser_round_trips_deterministically_with_versioning",
            "compatibility_enumerator_and_generator_state_are_live",
            "symbol_and_token_factories_execute_registered_rust_closures",
            "symbols_paths_and_token_class_definitions_keep_native_state",
        ),
    )
    if TESTS.read_text().count("#[test]") != 9:
        raise SystemExit("issue 83 expected 9 focused generator fixtures")
    require_markers(
        DOC,
        (
            "Checked-in generator contract",
            "25-production grammar",
            "84 generated syntax",
            "13 generated token classes",
            "python3 tools/generate_lsl_tables.py --check",
            "46 focused",
            "ubuntu-latest",
        ),
    )
    workflow = WORKFLOW.read_text()
    require_markers(
        WORKFLOW,
        (
            "python3 tools/check_milestone_10_issue_83.py",
            "python3 tools/generate_lsl_tables.py --check",
        ),
    )
    if "runs-on: ubuntu-latest" not in workflow or re.search(
        r"runs-on:\s*(?:macos|windows)", workflow, re.IGNORECASE
    ):
        raise SystemExit("issue 83 workflow is not ubuntu-only")

    print(
        "issue 83 audit: 120 native mapped generator/table types and 365 members, "
        "14 tokens, 8 nonterminals, 25 productions, two byte-identical generations, "
        "9 focused fixtures, migration docs, and ubuntu-only CI are present"
    )


if __name__ == "__main__":
    main()