Implement native LSL lexer runtime (#81)

tools/check_milestone_10_issue_81.py

@@ -0,0 +1,148 @@
+#!/usr/bin/env python3
+"""Audit issue 81's native lexer, token, source, and diagnostic boundary."""
+
+from __future__ import annotations
+
+import json
+import re
+from pathlib import Path
+
+import generate_api_shims
+
+
+ROOT = Path(__file__).resolve().parents[1]
+SOURCE = ROOT / "crates" / "libremetaverse-lsl-tools" / "src" / "lexer.rs"
+GENERATED = ROOT / "crates" / "libremetaverse-lsl-tools" / "src" / "generated.rs"
+UNIT_TESTS = SOURCE
+COMPAT_TESTS = (
+    ROOT / "crates" / "libremetaverse-lsl-tools" / "tests" / "lexer_compat.rs"
+)
+DOC = ROOT / "crates" / "libremetaverse-lsl-tools" / "README.md"
+WORKFLOW = ROOT / ".gitea" / "workflows" / "rust-workspace.yml"
+CATALOG = ROOT / "api" / "public-api.json"
+STUB_RE = re.compile(
+    r"\b(?:not_implemented|unimplemented_api)\b|\b(?:todo|unimplemented)!\s*\("
+)
+
+TYPES = {
+    "T:LibreMetaverse.LslTools.CSToolsException": "crate::lexer::CSToolsException",
+    "T:LibreMetaverse.LslTools.CSToolsFatalException":
+        "crate::lexer::CSToolsFatalException",
+    "T:LibreMetaverse.LslTools.CSToolsStopException":
+        "crate::lexer::CSToolsStopException",
+    "T:LibreMetaverse.LslTools.CatTest": "crate::lexer::CatTest",
+    "T:LibreMetaverse.LslTools.Charset": "crate::lexer::Charset",
+    "T:LibreMetaverse.LslTools.CommentList": "crate::lexer::CommentList",
+    "T:LibreMetaverse.LslTools.CsReader": "crate::lexer::CsReader",
+    "T:LibreMetaverse.LslTools.Dfa": "crate::lexer::Dfa",
+    "T:LibreMetaverse.LslTools.Dfa.Action": "crate::lexer::DfaAction",
+    "T:LibreMetaverse.LslTools.EOF": "crate::lexer::EOF",
+    "T:LibreMetaverse.LslTools.ErrorHandler": "crate::lexer::ErrorHandler",
+    "T:LibreMetaverse.LslTools.Lexer": "crate::lexer::Lexer",
+    "T:LibreMetaverse.LslTools.Lexer._Enumerator": "crate::lexer::LexerEnumerator",
+    "T:LibreMetaverse.LslTools.LineList": "crate::lexer::LineList",
+    "T:LibreMetaverse.LslTools.LineManager": "crate::lexer::LineManager",
+    "T:LibreMetaverse.LslTools.Null": "crate::lexer::Null",
+    "T:LibreMetaverse.LslTools.ObjectList": "crate::lexer::ObjectList",
+    "T:LibreMetaverse.LslTools.ResWds": "crate::lexer::ResWds",
+    "T:LibreMetaverse.LslTools.SYMBOL": "crate::lexer::SYMBOL",
+    "T:LibreMetaverse.LslTools.SourceLineInfo": "crate::lexer::SourceLineInfo",
+    "T:LibreMetaverse.LslTools.TOKEN": "crate::lexer::TOKEN",
+    "T:LibreMetaverse.LslTools.YyLexer": "crate::lexer::YyLexer",
+}
+
+
+def require_markers(path: Path, markers: tuple[str, ...]) -> None:
+    text = path.read_text()
+    missing = [marker for marker in markers if marker not in text]
+    if missing:
+        raise SystemExit(f"{path.name}: audit evidence missing: " + ", ".join(missing))
+
+
+def catalog_members() -> set[str]:
+    catalog = json.loads(CATALOG.read_text())
+    assembly = next(
+        value for value in catalog["assemblies"]
+        if value["identity"]["name"] == "LibreMetaverse.LslTools"
+    )
+    return {
+        member["doc_id"]
+        for api_type in assembly["types"]
+        if api_type["doc_id"] in TYPES
+        for member in api_type["members"]
+    }
+
+
+def generated_type_block(text: str, doc_id: str) -> str:
+    marker = f"/// C# type: `{doc_id}`."
+    start = text.find(marker)
+    if start < 0:
+        raise SystemExit(f"generated declaration is missing for {doc_id}")
+    next_type = text.find("/// C# type:", start + len(marker))
+    return text[start:] if next_type < 0 else text[start:next_type]
+
+
+def main() -> None:
+    for api_type, declaration in TYPES.items():
+        if generate_api_shims.NATIVE_TYPES.get(api_type) != declaration:
+            raise SystemExit(f"issue 81 native type mapping is missing for {api_type}")
+    members = catalog_members()
+    if len(members) != 188:
+        raise SystemExit(f"issue 81 expected 188 mapped members, found {len(members)}")
+
+    source = SOURCE.read_text()
+    if STUB_RE.search(source):
+        raise SystemExit("issue 81 owned Rust stubs remain in lexer.rs")
+    if "Ok(o)" in source or "TokenDefinition::new(name, 1)" in source:
+        raise SystemExit("issue 81 contains a silent compatibility no-op or fabricated token")
+    if re.search(r"unsafe\s*\{|unsafe\s+impl|target_os\s*=\s*\"macos\"", source):
+        raise SystemExit("lexer.rs contains an unsafe or macOS-only implementation")
+    generated = GENERATED.read_text()
+    for api_type in TYPES:
+        if STUB_RE.search(generated_type_block(generated, api_type)):
+            raise SystemExit(f"issue 81 owned generated stubs remain for {api_type}")
+
+    require_markers(SOURCE, (
+        "MAX_SOURCE_UNITS", "MAX_TOKEN_UNITS", "pub enum DotNetUnicodeCategory",
+        "pub enum InputEncoding", "pub struct CatTest", "pub struct Charset",
+        "pub struct CsReader", "fn filter_source", "fn parse_line_directive",
+        "pub struct LineManager", "pub struct SourceLineInfo",
+        "pub enum DiagnosticCategory", "pub struct Diagnostic",
+        "pub struct ErrorHandler", "pub enum CharacterMatcher",
+        "pub struct DfaState", "pub struct Dfa", "fn longest_match",
+        "pub struct ResWds", "pub struct YyLexer", "pub struct SYMBOL",
+        "pub struct TOKEN", "pub struct EOF", "pub struct Lexer",
+        "pub struct LexerIterator", "impl Iterator for LexerIterator",
+        "lookup_parser_entry", "emit_dfa", "record_diagnostic",
+        "unicode_general_category", "String::from_utf16_lossy",
+    ))
+    require_markers(COMPAT_TESTS, (
+        "token_fixture_preserves_names_text_numbers_positions_and_eof",
+        "invalid_input_reports_stable_category_location_and_text",
+        "source_reader_removes_both_comment_forms_and_keeps_newlines",
+        "utf16_input_decoding_and_surrogate_validation_are_deterministic",
+        "category_predicates_match_dotnet_values_and_groups",
+        "dfa_uses_maximum_munch_and_exposes_action_number",
+        "deterministic_table_output_is_independent_of_hash_iteration",
+        "compatibility_enumerator_resets_to_the_first_token",
+        "error_handler_counts_reports_and_honors_throw_mode",
+    ))
+    test_count = UNIT_TESTS.read_text().count("#[test]") + COMPAT_TESTS.read_text().count("#[test]")
+    if test_count != 27:
+        raise SystemExit(f"issue 81 expected 27 focused fixtures, found {test_count}")
+    require_markers(DOC, (
+        "UTF-16 code-unit offsets", "maximum munch", "structured `Diagnostic`",
+        "64 Mi UTF-16", "16 Mi units", "Lexer::iter", "27 focused",
+        "No C#, .NET runtime", "ubuntu-latest",
+    ))
+    require_markers(WORKFLOW, ("python3 tools/check_milestone_10_issue_81.py",))
+    print(
+        "issue 81 audit: 22 mapped native types and 188 members, bounded UTF-16 "
+        "source handling, comments and line directives, Unicode categories, deterministic "
+        "DFA/token behavior, reserved words, EOF, diagnostics, iterators, parser-table "
+        "lookup, 27 focused fixtures, API documentation, and ubuntu-only CI are present"
+    )
+
+
+if __name__ == "__main__":
+    main()

tools/generate_api_shims.py

@@ -38,6 +38,28 @@ TARGETS = {
 # implementations. The generated module keeps catalog markers and re-exports
 # the hand-written type so coverage remains deterministic.
 NATIVE_TYPES = {
+    "T:LibreMetaverse.LslTools.CSToolsException": "crate::lexer::CSToolsException",
+    "T:LibreMetaverse.LslTools.CSToolsFatalException": "crate::lexer::CSToolsFatalException",
+    "T:LibreMetaverse.LslTools.CSToolsStopException": "crate::lexer::CSToolsStopException",
+    "T:LibreMetaverse.LslTools.CatTest": "crate::lexer::CatTest",
+    "T:LibreMetaverse.LslTools.Charset": "crate::lexer::Charset",
+    "T:LibreMetaverse.LslTools.CommentList": "crate::lexer::CommentList",
+    "T:LibreMetaverse.LslTools.CsReader": "crate::lexer::CsReader",
+    "T:LibreMetaverse.LslTools.Dfa": "crate::lexer::Dfa",
+    "T:LibreMetaverse.LslTools.Dfa.Action": "crate::lexer::DfaAction",
+    "T:LibreMetaverse.LslTools.EOF": "crate::lexer::EOF",
+    "T:LibreMetaverse.LslTools.ErrorHandler": "crate::lexer::ErrorHandler",
+    "T:LibreMetaverse.LslTools.Lexer": "crate::lexer::Lexer",
+    "T:LibreMetaverse.LslTools.Lexer._Enumerator": "crate::lexer::LexerEnumerator",
+    "T:LibreMetaverse.LslTools.LineList": "crate::lexer::LineList",
+    "T:LibreMetaverse.LslTools.LineManager": "crate::lexer::LineManager",
+    "T:LibreMetaverse.LslTools.Null": "crate::lexer::Null",
+    "T:LibreMetaverse.LslTools.ObjectList": "crate::lexer::ObjectList",
+    "T:LibreMetaverse.LslTools.ResWds": "crate::lexer::ResWds",
+    "T:LibreMetaverse.LslTools.SYMBOL": "crate::lexer::SYMBOL",
+    "T:LibreMetaverse.LslTools.SourceLineInfo": "crate::lexer::SourceLineInfo",
+    "T:LibreMetaverse.LslTools.TOKEN": "crate::lexer::TOKEN",
+    "T:LibreMetaverse.LslTools.YyLexer": "crate::lexer::YyLexer",
     "T:LibreMetaverse.RLV.AttachmentRequest": "crate::service::AttachmentRequest",
     "T:LibreMetaverse.RLV.RlvActionCallbacksDefault": "crate::service::RlvActionCallbacksDefault",
     "T:LibreMetaverse.RLV.RlvCallbacksDefault": "crate::service::RlvCallbacksDefault",