#!/usr/bin/env python3 """Audit issue 81's native lexer, token, source, and diagnostic boundary.""" from __future__ import annotations import json import re from pathlib import Path import generate_api_shims ROOT = Path(__file__).resolve().parents[1] SOURCE = ROOT / "crates" / "metacrate-lsl-tools" / "src" / "lexer.rs" GENERATED = ROOT / "crates" / "metacrate-lsl-tools" / "src" / "generated.rs" UNIT_TESTS = SOURCE COMPAT_TESTS = ( ROOT / "crates" / "metacrate-lsl-tools" / "tests" / "lexer_compat.rs" ) DOC = ROOT / "crates" / "metacrate-lsl-tools" / "README.md" WORKFLOW = ROOT / ".gitea" / "workflows" / "ci.yml" CATALOG = ROOT / "api" / "public-api.json" STUB_RE = re.compile( r"\b(?:not_implemented|unimplemented_api)\b|\b(?:todo|unimplemented)!\s*\(" ) TYPES = { "T:LibreMetaverse.LslTools.CSToolsException": "crate::lexer::CSToolsException", "T:LibreMetaverse.LslTools.CSToolsFatalException": "crate::lexer::CSToolsFatalException", "T:LibreMetaverse.LslTools.CSToolsStopException": "crate::lexer::CSToolsStopException", "T:LibreMetaverse.LslTools.CatTest": "crate::lexer::CatTest", "T:LibreMetaverse.LslTools.Charset": "crate::lexer::Charset", "T:LibreMetaverse.LslTools.CommentList": "crate::lexer::CommentList", "T:LibreMetaverse.LslTools.CsReader": "crate::lexer::CsReader", "T:LibreMetaverse.LslTools.Dfa": "crate::lexer::Dfa", "T:LibreMetaverse.LslTools.Dfa.Action": "crate::lexer::DfaAction", "T:LibreMetaverse.LslTools.EOF": "crate::lexer::EOF", "T:LibreMetaverse.LslTools.ErrorHandler": "crate::lexer::ErrorHandler", "T:LibreMetaverse.LslTools.Lexer": "crate::lexer::Lexer", "T:LibreMetaverse.LslTools.Lexer._Enumerator": "crate::lexer::LexerEnumerator", "T:LibreMetaverse.LslTools.LineList": "crate::lexer::LineList", "T:LibreMetaverse.LslTools.LineManager": "crate::lexer::LineManager", "T:LibreMetaverse.LslTools.Null": "crate::lexer::Null", "T:LibreMetaverse.LslTools.ObjectList": "crate::lexer::ObjectList", "T:LibreMetaverse.LslTools.ResWds": "crate::lexer::ResWds", "T:LibreMetaverse.LslTools.SYMBOL": "crate::lexer::SYMBOL", "T:LibreMetaverse.LslTools.SourceLineInfo": "crate::lexer::SourceLineInfo", "T:LibreMetaverse.LslTools.TOKEN": "crate::lexer::TOKEN", "T:LibreMetaverse.LslTools.YyLexer": "crate::lexer::YyLexer", } def require_markers(path: Path, markers: tuple[str, ...]) -> None: text = path.read_text() missing = [marker for marker in markers if marker not in text] if missing: raise SystemExit(f"{path.name}: audit evidence missing: " + ", ".join(missing)) def catalog_members() -> set[str]: catalog = json.loads(CATALOG.read_text()) assembly = next( value for value in catalog["assemblies"] if value["identity"]["name"] == "LibreMetaverse.LslTools" ) return { member["doc_id"] for api_type in assembly["types"] if api_type["doc_id"] in TYPES for member in api_type["members"] } def generated_type_block(text: str, doc_id: str) -> str: marker = f"/// C# type: `{doc_id}`." start = text.find(marker) if start < 0: raise SystemExit(f"generated declaration is missing for {doc_id}") next_type = text.find("/// C# type:", start + len(marker)) return text[start:] if next_type < 0 else text[start:next_type] def main() -> None: for api_type, declaration in TYPES.items(): if generate_api_shims.NATIVE_TYPES.get(api_type) != declaration: raise SystemExit(f"issue 81 native type mapping is missing for {api_type}") members = catalog_members() if len(members) != 188: raise SystemExit(f"issue 81 expected 188 mapped members, found {len(members)}") source = SOURCE.read_text() if STUB_RE.search(source): raise SystemExit("issue 81 owned Rust stubs remain in lexer.rs") if "Ok(o)" in source or "TokenDefinition::new(name, 1)" in source: raise SystemExit("issue 81 contains a silent compatibility no-op or fabricated token") if re.search(r"unsafe\s*\{|unsafe\s+impl|target_os\s*=\s*\"macos\"", source): raise SystemExit("lexer.rs contains an unsafe or macOS-only implementation") generated = GENERATED.read_text() for api_type in TYPES: if STUB_RE.search(generated_type_block(generated, api_type)): raise SystemExit(f"issue 81 owned generated stubs remain for {api_type}") require_markers(SOURCE, ( "MAX_SOURCE_UNITS", "MAX_TOKEN_UNITS", "pub enum DotNetUnicodeCategory", "pub enum InputEncoding", "pub struct CatTest", "pub struct Charset", "pub struct CsReader", "fn filter_source", "fn parse_line_directive", "pub struct LineManager", "pub struct SourceLineInfo", "pub enum DiagnosticCategory", "pub struct Diagnostic", "pub struct ErrorHandler", "pub enum CharacterMatcher", "pub struct DfaState", "pub struct Dfa", "fn longest_match", "pub struct ResWds", "pub struct YyLexer", "pub struct SYMBOL", "pub struct TOKEN", "pub struct EOF", "pub struct Lexer", "pub struct LexerIterator", "impl Iterator for LexerIterator", "lookup_parser_entry", "emit_dfa", "record_diagnostic", "unicode_general_category", "String::from_utf16_lossy", )) require_markers(COMPAT_TESTS, ( "token_fixture_preserves_names_text_numbers_positions_and_eof", "invalid_input_reports_stable_category_location_and_text", "source_reader_removes_both_comment_forms_and_keeps_newlines", "utf16_input_decoding_and_surrogate_validation_are_deterministic", "category_predicates_match_dotnet_values_and_groups", "dfa_uses_maximum_munch_and_exposes_action_number", "deterministic_table_output_is_independent_of_hash_iteration", "compatibility_enumerator_resets_to_the_first_token", "error_handler_counts_reports_and_honors_throw_mode", )) test_count = UNIT_TESTS.read_text().count("#[test]") + COMPAT_TESTS.read_text().count("#[test]") if test_count != 27: raise SystemExit(f"issue 81 expected 27 focused fixtures, found {test_count}") require_markers(DOC, ( "UTF-16 code-unit offsets", "maximum munch", "structured `Diagnostic`", "64 Mi UTF-16", "16 Mi units", "Lexer::iter", "46 focused", "No C#, .NET runtime", "ubuntu-latest", )) require_markers(WORKFLOW, ("required-gate",)) print( "issue 81 audit: 22 mapped native types and 188 members, bounded UTF-16 " "source handling, comments and line directives, Unicode categories, deterministic " "DFA/token behavior, reserved words, EOF, diagnostics, iterators, parser-table " "lookup, 27 focused fixtures, API documentation, and ubuntu-only CI are present" ) if __name__ == "__main__": main()