MetaCrate/tools/check_milestone_10_issue_81.py

#!/usr/bin/env python3
"""Audit issue 81's native lexer, token, source, and diagnostic boundary."""

from __future__ import annotations

import json
import re
from pathlib import Path

import generate_api_shims


ROOT = Path(__file__).resolve().parents[1]
SOURCE = ROOT / "crates" / "libremetaverse-lsl-tools" / "src" / "lexer.rs"
GENERATED = ROOT / "crates" / "libremetaverse-lsl-tools" / "src" / "generated.rs"
UNIT_TESTS = SOURCE
COMPAT_TESTS = (
    ROOT / "crates" / "libremetaverse-lsl-tools" / "tests" / "lexer_compat.rs"
)
DOC = ROOT / "crates" / "libremetaverse-lsl-tools" / "README.md"
WORKFLOW = ROOT / ".gitea" / "workflows" / "rust-workspace.yml"
CATALOG = ROOT / "api" / "public-api.json"
STUB_RE = re.compile(
    r"\b(?:not_implemented|unimplemented_api)\b|\b(?:todo|unimplemented)!\s*\("
)

TYPES = {
    "T:LibreMetaverse.LslTools.CSToolsException": "crate::lexer::CSToolsException",
    "T:LibreMetaverse.LslTools.CSToolsFatalException":
        "crate::lexer::CSToolsFatalException",
    "T:LibreMetaverse.LslTools.CSToolsStopException":
        "crate::lexer::CSToolsStopException",
    "T:LibreMetaverse.LslTools.CatTest": "crate::lexer::CatTest",
    "T:LibreMetaverse.LslTools.Charset": "crate::lexer::Charset",
    "T:LibreMetaverse.LslTools.CommentList": "crate::lexer::CommentList",
    "T:LibreMetaverse.LslTools.CsReader": "crate::lexer::CsReader",
    "T:LibreMetaverse.LslTools.Dfa": "crate::lexer::Dfa",
    "T:LibreMetaverse.LslTools.Dfa.Action": "crate::lexer::DfaAction",
    "T:LibreMetaverse.LslTools.EOF": "crate::lexer::EOF",
    "T:LibreMetaverse.LslTools.ErrorHandler": "crate::lexer::ErrorHandler",
    "T:LibreMetaverse.LslTools.Lexer": "crate::lexer::Lexer",
    "T:LibreMetaverse.LslTools.Lexer._Enumerator": "crate::lexer::LexerEnumerator",
    "T:LibreMetaverse.LslTools.LineList": "crate::lexer::LineList",
    "T:LibreMetaverse.LslTools.LineManager": "crate::lexer::LineManager",
    "T:LibreMetaverse.LslTools.Null": "crate::lexer::Null",
    "T:LibreMetaverse.LslTools.ObjectList": "crate::lexer::ObjectList",
    "T:LibreMetaverse.LslTools.ResWds": "crate::lexer::ResWds",
    "T:LibreMetaverse.LslTools.SYMBOL": "crate::lexer::SYMBOL",
    "T:LibreMetaverse.LslTools.SourceLineInfo": "crate::lexer::SourceLineInfo",
    "T:LibreMetaverse.LslTools.TOKEN": "crate::lexer::TOKEN",
    "T:LibreMetaverse.LslTools.YyLexer": "crate::lexer::YyLexer",
}


def require_markers(path: Path, markers: tuple[str, ...]) -> None:
    text = path.read_text()
    missing = [marker for marker in markers if marker not in text]
    if missing:
        raise SystemExit(f"{path.name}: audit evidence missing: " + ", ".join(missing))


def catalog_members() -> set[str]:
    catalog = json.loads(CATALOG.read_text())
    assembly = next(
        value for value in catalog["assemblies"]
        if value["identity"]["name"] == "LibreMetaverse.LslTools"
    )
    return {
        member["doc_id"]
        for api_type in assembly["types"]
        if api_type["doc_id"] in TYPES
        for member in api_type["members"]
    }


def generated_type_block(text: str, doc_id: str) -> str:
    marker = f"/// C# type: `{doc_id}`."
    start = text.find(marker)
    if start < 0:
        raise SystemExit(f"generated declaration is missing for {doc_id}")
    next_type = text.find("/// C# type:", start + len(marker))
    return text[start:] if next_type < 0 else text[start:next_type]


def main() -> None:
    for api_type, declaration in TYPES.items():
        if generate_api_shims.NATIVE_TYPES.get(api_type) != declaration:
            raise SystemExit(f"issue 81 native type mapping is missing for {api_type}")
    members = catalog_members()
    if len(members) != 188:
        raise SystemExit(f"issue 81 expected 188 mapped members, found {len(members)}")

    source = SOURCE.read_text()
    if STUB_RE.search(source):
        raise SystemExit("issue 81 owned Rust stubs remain in lexer.rs")
    if "Ok(o)" in source or "TokenDefinition::new(name, 1)" in source:
        raise SystemExit("issue 81 contains a silent compatibility no-op or fabricated token")
    if re.search(r"unsafe\s*\{|unsafe\s+impl|target_os\s*=\s*\"macos\"", source):
        raise SystemExit("lexer.rs contains an unsafe or macOS-only implementation")
    generated = GENERATED.read_text()
    for api_type in TYPES:
        if STUB_RE.search(generated_type_block(generated, api_type)):
            raise SystemExit(f"issue 81 owned generated stubs remain for {api_type}")

    require_markers(SOURCE, (
        "MAX_SOURCE_UNITS", "MAX_TOKEN_UNITS", "pub enum DotNetUnicodeCategory",
        "pub enum InputEncoding", "pub struct CatTest", "pub struct Charset",
        "pub struct CsReader", "fn filter_source", "fn parse_line_directive",
        "pub struct LineManager", "pub struct SourceLineInfo",
        "pub enum DiagnosticCategory", "pub struct Diagnostic",
        "pub struct ErrorHandler", "pub enum CharacterMatcher",
        "pub struct DfaState", "pub struct Dfa", "fn longest_match",
        "pub struct ResWds", "pub struct YyLexer", "pub struct SYMBOL",
        "pub struct TOKEN", "pub struct EOF", "pub struct Lexer",
        "pub struct LexerIterator", "impl Iterator for LexerIterator",
        "lookup_parser_entry", "emit_dfa", "record_diagnostic",
        "unicode_general_category", "String::from_utf16_lossy",
    ))
    require_markers(COMPAT_TESTS, (
        "token_fixture_preserves_names_text_numbers_positions_and_eof",
        "invalid_input_reports_stable_category_location_and_text",
        "source_reader_removes_both_comment_forms_and_keeps_newlines",
        "utf16_input_decoding_and_surrogate_validation_are_deterministic",
        "category_predicates_match_dotnet_values_and_groups",
        "dfa_uses_maximum_munch_and_exposes_action_number",
        "deterministic_table_output_is_independent_of_hash_iteration",
        "compatibility_enumerator_resets_to_the_first_token",
        "error_handler_counts_reports_and_honors_throw_mode",
    ))
    test_count = UNIT_TESTS.read_text().count("#[test]") + COMPAT_TESTS.read_text().count("#[test]")
    if test_count != 27:
        raise SystemExit(f"issue 81 expected 27 focused fixtures, found {test_count}")
    require_markers(DOC, (
        "UTF-16 code-unit offsets", "maximum munch", "structured `Diagnostic`",
        "64 Mi UTF-16", "16 Mi units", "Lexer::iter", "27 focused",
        "No C#, .NET runtime", "ubuntu-latest",
    ))
    require_markers(WORKFLOW, ("python3 tools/check_milestone_10_issue_81.py",))
    print(
        "issue 81 audit: 22 mapped native types and 188 members, bounded UTF-16 "
        "source handling, comments and line directives, Unicode categories, deterministic "
        "DFA/token behavior, reserved words, EOF, diagnostics, iterators, parser-table "
        "lookup, 27 focused fixtures, API documentation, and ubuntu-only CI are present"
    )


if __name__ == "__main__":
    main()