Implement native LSL lexer runtime (#81)

.gitea/workflows/rust-workspace.yml

@@ -12,6 +12,7 @@ on:
       - "tools/check_milestone_10_issue_78.py"
       - "tools/check_milestone_10_issue_79.py"
       - "tools/check_milestone_10_issue_80.py"
+      - "tools/check_milestone_10_issue_81.py"
       - "**/*.rs"
       - "**/Cargo.toml"
       - "Cargo.lock"
@@ -26,6 +27,7 @@ on:
       - "tools/check_milestone_10_issue_78.py"
       - "tools/check_milestone_10_issue_79.py"
       - "tools/check_milestone_10_issue_80.py"
+      - "tools/check_milestone_10_issue_81.py"
       - "**/*.rs"
       - "**/Cargo.toml"
       - "Cargo.lock"
@@ -66,6 +68,7 @@ jobs:
           python3 tools/check_milestone_10_issue_78.py
           python3 tools/check_milestone_10_issue_79.py
           python3 tools/check_milestone_10_issue_80.py
+          python3 tools/check_milestone_10_issue_81.py
       - name: Test the complete native world milestone
         run: python3 tools/test_milestone_09.py
       - name: Compile every workspace target with bounded memory

Cargo.lock

@@ -924,6 +924,7 @@ name = "libremetaverse-lsl-tools"
 version = "0.0.1"
 dependencies = [
  "libremetaverse-types",
+ "unicode-general-category",
 ]
 
 [[package]]
@@ -2213,6 +2214,12 @@ version = "1.20.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20"
 
+[[package]]
+name = "unicode-general-category"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b993bddc193ae5bd0d623b49ec06ac3e9312875fdae725a975c51db1cc1677f"
+
 [[package]]
 name = "unicode-ident"
 version = "1.0.24"

api/SHIM-COVERAGE.md

@@ -7,7 +7,7 @@ Generated by `python3 tools/generate_api_shims.py`; do not edit by hand.
 | `LibreMetaverse` | 2,711 | 27,281 | native implementation: 401 types / 17,025 members; remaining surface is callable failure-only shims |
 | `LibreMetaverse.Imaging.Abstractions` | 3 | 20 | native implementation: 3 types / 20 members; no generated shims remain |
 | `LibreMetaverse.Imaging.Skia` | 1 | 3 | native implementation: 1 type / 3 members; no generated shims remain |
-| `LibreMetaverse.LslTools` | 164 | 768 | callable failure-only shim |
+| `LibreMetaverse.LslTools` | 164 | 768 | native implementation: 22 types / 188 members; remaining surface is callable failure-only shims |
 | `LibreMetaverse.PrimMesher` | 17 | 207 | native implementation: 15 types / 200 members; remaining surface is callable failure-only shims |
 | `LibreMetaverse.RLV` | 28 | 499 | native implementation: 17 types / 200 members; remaining surface is callable failure-only shims |
 | `LibreMetaverse.Rendering.MeshFoundry` | 1 | 14 | native implementation: 1 type / 14 members; no generated shims remain |

crates/libremetaverse-lsl-tools/Cargo.toml

@@ -5,10 +5,11 @@ edition.workspace = true
 rust-version.workspace = true
 license.workspace = true
 repository.workspace = true
-description = "LSL parser-tool shims for the MetaCrate LibreMetaverse rewrite"
+description = "Native LSL lexer and parser tooling for the MetaCrate LibreMetaverse rewrite"
 
 [dependencies]
 libremetaverse-types = { path = "../libremetaverse-types" }
+unicode-general-category = "1.1"
 
 [lints]
 workspace = true

crates/libremetaverse-lsl-tools/README.md

@@ -0,0 +1,94 @@
+# Native LSL lexer and parser tools
+
+`libremetaverse-lsl-tools` is the native Rust replacement for the lexer and
+parser-generator runtime in the pinned `LibreMetaverse.LslTools` assembly. The
+issue 81 boundary supplies source reading, comments, Unicode character sets,
+deterministic DFA execution, reserved words, terminal symbols, EOF, source
+locations, diagnostics, and Rust iterators. Grammar productions, reductions,
+and recovery are added by issue 82; deterministic generated grammar and token
+tables are added by issue 83.
+
+## Source and position contract
+
+All public positions are UTF-16 code-unit offsets, matching the original
+`System.Char` API. They are not UTF-8 byte offsets. `SourceLineInfo` reports a
+one-based line and character position while retaining the filtered-line bounds
+and the raw character position before removed comments. Non-BMP characters
+therefore occupy two positions, exactly as they do in C#.
+
+`CsReader` accepts strings, files, or explicitly encoded byte slices. It
+normalizes CRLF and lone CR to LF, removes `//` and `/* ... */` comments while
+preserving their newline structure, tracks the removed UTF-16 lengths for raw
+columns, and applies `#line N "file"` directives. Unterminated block comments,
+invalid UTF-8/UTF-16, unpaired surrogates, trailing UTF-16 bytes, and non-ASCII
+bytes in ASCII mode return positioned errors instead of replacement text.
+
+The portable encoding set is UTF-8, UTF-16LE, UTF-16BE, ASCII, and ASCIICAPS.
+UTF-7 and platform code pages are deliberately not delegated to host APIs, so
+Linux and Windows produce the same result. A source is bounded to 64 Mi UTF-16
+units and a token to 16 Mi units before unbounded allocation or matching can
+occur.
+
+## Lexer table contract
+
+`Dfa` is a validated deterministic table of `DfaState` values and
+`CharacterMatcher` transitions. Matchers support exact UTF-16 values, ranges,
+sets, all .NET Unicode general categories, category unions, any character, and
+EOF. The runtime applies maximum munch, preserves rule order for overlapping
+transitions, rejects empty non-EOF matches, and exposes a deterministic textual
+table representation through `YyLexer::emit_dfa`.
+
+Accepting states carry a `TokenDefinition`, action number, optional reserved
+word table, and one of four actions: emit, skip, emit and change start
+condition, or skip and change start condition. Reserved words are exact by
+default and use Unicode uppercase only when the table requests the reference
+`U { ... }` behavior. `TOKEN` preserves token name, number, lexeme, semantic
+value, half-open UTF-16 span, and source location. Configured EOF is emitted
+once at the end of the filtered buffer.
+
+The old C# enumerators remain callable as compatibility adapters. New Rust code
+should use `Lexer::iter` or `Lexer::next_token`; both stop deterministically
+after a diagnostic and never yield a fabricated token. Token and symbol
+`Pass` members perform real lookup against the mapped parser symbol/literal
+tables and populate the supplied parser-entry priority. Missing or malformed
+table data is an explicit positioned error.
+
+## Diagnostics and migration
+
+`ErrorHandler` collects structured `Diagnostic` values. Each diagnostic has a
+stable numeric code, `DiagnosticCategory`, severity, message, input fragment,
+and `SourceLineInfo`. Invalid characters and start conditions are recorded at
+the offending UTF-16 position. Compatibility exception constructors retain
+their original number, input, symbol/token location, handled state, and fatal
+or stop behavior. A handler configured to throw increments its counter and
+returns immediately without reporting, matching the reference order.
+
+The principal API mapping is:
+
+| C# concept | Native Rust API |
+| --- | --- |
+| `CsReader`, `LineManager`, `SourceLineInfo` | Same mapped names, UTF-16-safe source model |
+| `Dfa`, `Dfa.Action`, `YyLexer` | `Dfa`, `DfaAction`, `YyLexer`, plus typed state builders |
+| `SYMBOL`, `TOKEN`, `EOF`, `Null` | Same mapped names with owned Rust values |
+| `Lexer._Enumerator` | `LexerEnumerator`; prefer `LexerIterator` |
+| `Charset`, `CatTest` | Same mapped names plus `DotNetUnicodeCategory` |
+| `CSToolsException`, `ErrorHandler` | Same mapped names plus structured `Diagnostic` |
+
+No C#, .NET runtime, dynamically loaded class, macOS-only API, platform code
+page, or runtime source generation is used by this boundary.
+
+## Reproducible verification
+
+Run the issue-owned gates with one build job:
+
+```sh
+CARGO_BUILD_JOBS=1 cargo test -p libremetaverse-lsl-tools --locked
+CARGO_BUILD_JOBS=1 cargo check --manifest-path tests/api-compile/Cargo.toml --locked
+CARGO_BUILD_JOBS=1 cargo clippy -p libremetaverse-lsl-tools --all-targets --locked -- -D warnings
+RUSTDOCFLAGS='-D warnings' CARGO_BUILD_JOBS=1 cargo doc -p libremetaverse-lsl-tools --no-deps --locked
+python3 tools/check_milestone_10_issue_81.py
+python3 tools/generate_api_shims.py --check
+```
+
+The package contains 27 focused native and compatibility fixtures. The Gitea
+workflow runs the audit and workspace compile on `ubuntu-latest` only.

crates/libremetaverse-lsl-tools/src/lib.rs

@@ -3,6 +3,12 @@
 extern crate self as libremetaverse_lsl_tools;
 
 mod generated;
+mod lexer;
 
 pub use generated::*;
+pub use lexer::{
+    CharacterMatcher, DfaAccept, DfaState, Diagnostic, DiagnosticCategory, DiagnosticSeverity,
+    DotNetUnicodeCategory, InputEncoding, LexerAction, LexerIterator, MAX_SOURCE_UNITS,
+    MAX_TOKEN_UNITS, TokenDefinition, UnicodeClass,
+};
 pub use libremetaverse_types::Error;

crates/libremetaverse-lsl-tools/tests/lexer_compat.rs

@@ -0,0 +1,391 @@
+//! Focused compatibility fixtures translated from the pinned lexer runtime.
+
+use std::collections::BTreeSet;
+use std::sync::{Arc, Mutex};
+
+use libremetaverse_lsl_tools::{
+    CSToolsException, CharacterMatcher, CsReader, Dfa, DfaAccept, DfaState, DiagnosticCategory,
+    DotNetUnicodeCategory, Error, ErrorHandler, InputEncoding, Lexer, LexerAction, LineManager,
+    ObjectList, ResWds, SourceLineInfo, TOKEN, TokenDefinition, YyLexer,
+};
+use libremetaverse_types::compat::{Object, UnicodeCategory, Utf16CodeUnit};
+
+fn token(name: &str, number: i32) -> TokenDefinition {
+    TokenDefinition::new(name, number).expect("valid definition")
+}
+
+fn accept(name: &str, number: i32, action: LexerAction, reserved_words: Option<&str>) -> DfaAccept {
+    DfaAccept {
+        token: token(name, number),
+        action,
+        action_number: number,
+        reserved_words: reserved_words.map(ToOwned::to_owned),
+    }
+}
+
+fn language_table() -> YyLexer {
+    let letters = vec![
+        DotNetUnicodeCategory::UppercaseLetter,
+        DotNetUnicodeCategory::LowercaseLetter,
+        DotNetUnicodeCategory::TitlecaseLetter,
+        DotNetUnicodeCategory::ModifierLetter,
+        DotNetUnicodeCategory::OtherLetter,
+    ];
+    let whitespace = vec![
+        DotNetUnicodeCategory::Control,
+        DotNetUnicodeCategory::SpaceSeparator,
+        DotNetUnicodeCategory::LineSeparator,
+        DotNetUnicodeCategory::ParagraphSeparator,
+    ];
+    let states = vec![
+        DfaState::default()
+            .transition(CharacterMatcher::Categories(letters.clone()), 1)
+            .transition(CharacterMatcher::Exact(u16::from(b'_')), 1)
+            .transition(
+                CharacterMatcher::Category(DotNetUnicodeCategory::DecimalDigitNumber),
+                2,
+            )
+            .transition(CharacterMatcher::Categories(whitespace.clone()), 3)
+            .transition(CharacterMatcher::Exact(u16::from(b'+')), 4),
+        DfaState::default()
+            .transition(CharacterMatcher::Categories(letters), 1)
+            .transition(CharacterMatcher::Exact(u16::from(b'_')), 1)
+            .transition(
+                CharacterMatcher::Category(DotNetUnicodeCategory::DecimalDigitNumber),
+                1,
+            )
+            .accepting(accept("ID", 3, LexerAction::Emit, Some("keywords"))),
+        DfaState::default()
+            .transition(
+                CharacterMatcher::Category(DotNetUnicodeCategory::DecimalDigitNumber),
+                2,
+            )
+            .accepting(accept("INTEGER", 4, LexerAction::Emit, None)),
+        DfaState::default()
+            .transition(CharacterMatcher::Categories(whitespace), 3)
+            .accepting(accept("WS", 5, LexerAction::Skip, None)),
+        DfaState::default().accepting(accept("PLUS", 6, LexerAction::Emit, None)),
+    ];
+    let mut table = YyLexer::new(ErrorHandler::default()).expect("table");
+    table.using_eof = true;
+    table
+        .set_start_dfa("YYINITIAL", Dfa::from_states(states, 0).expect("dfa"))
+        .expect("start state");
+    table
+        .set_reserved_words(
+            "keywords",
+            ResWds::from_pairs(
+                [("if", token("IF", 20)), ("while", token("WHILE", 21))],
+                false,
+            )
+            .expect("reserved words"),
+        )
+        .expect("reserved table");
+    table
+}
+
+fn lexer(source: &str) -> Lexer {
+    let mut lexer = Lexer::new(language_table()).expect("lexer");
+    lexer
+        .start_with_string(source.to_owned())
+        .expect("valid source");
+    lexer
+}
+
+#[test]
+fn token_fixture_preserves_names_text_numbers_positions_and_eof() {
+    let mut lexer = lexer("if café_2 + 19");
+    let mut tokens = Vec::new();
+    while let Some(value) = lexer.next_token().expect("tokenization") {
+        tokens.push(value);
+    }
+    let actual = tokens
+        .iter()
+        .map(|token| {
+            (
+                token.yyname(),
+                token.yytext(),
+                token.yynum(),
+                token.pos,
+                token.end,
+            )
+        })
+        .collect::<Vec<_>>();
+    assert_eq!(
+        actual,
+        [
+            ("IF".to_owned(), "if".to_owned(), 20, 0, 2),
+            ("ID".to_owned(), "café_2".to_owned(), 3, 3, 9),
+            ("PLUS".to_owned(), "+".to_owned(), 6, 10, 11),
+            ("INTEGER".to_owned(), "19".to_owned(), 4, 12, 14),
+            ("EOF".to_owned(), "EOF".to_owned(), 2, 15, 15),
+        ]
+    );
+}
+
+#[test]
+fn reserved_words_are_exact_unless_the_table_requests_uppercase() {
+    let mut lexer = lexer("IF if");
+    assert_eq!(lexer.next().expect("identifier").yyname(), "ID");
+    assert_eq!(lexer.next().expect("keyword").yyname(), "IF");
+
+    let words = ResWds::from_pairs([("while", token("WHILE", 21))], true).expect("words");
+    let mut value = TOKEN::new_with_lexer_string(lexer.clone(), "WhIlE".to_owned()).expect("token");
+    words.check(lexer, &mut value).expect("check");
+    assert_eq!((value.yyname(), value.yynum()), ("WHILE".to_owned(), 21));
+}
+
+#[test]
+fn invalid_input_reports_stable_category_location_and_text() {
+    let mut lexer = lexer("ok @ nope");
+    assert_eq!(lexer.next().expect("first").yytext(), "ok");
+    assert_eq!(
+        lexer.next().expect_err("invalid character"),
+        Error::Parse {
+            position: 3,
+            context: "input does not match any lexer rule"
+        }
+    );
+    let diagnostic = &lexer.diagnostics()[0];
+    assert_eq!(diagnostic.category, DiagnosticCategory::InvalidCharacter);
+    assert_eq!(diagnostic.location.line_number, 1);
+    assert_eq!(diagnostic.location.raw_char_position, 3);
+    assert_eq!(diagnostic.input, "@");
+}
+
+#[test]
+fn unknown_start_condition_reports_invalid_state_without_changing_state() {
+    let mut lexer = lexer("value");
+    assert_eq!(
+        lexer.yy_begin("MISSING".to_owned()),
+        Err(Error::Parse {
+            position: 0,
+            context: "unknown lexer start condition"
+        })
+    );
+    assert_eq!(lexer.m_state, "YYINITIAL");
+    assert_eq!(
+        lexer.diagnostics()[0].category,
+        DiagnosticCategory::InvalidState
+    );
+}
+
+#[test]
+fn source_reader_removes_both_comment_forms_and_keeps_newlines() {
+    let mut reader = CsReader::new_with_string("a/*x\ny*/b//z\nc".to_owned()).expect("reader");
+    assert_eq!(reader.read_line().expect("line"), "a");
+    assert_eq!(reader.read_line().expect("line"), "b");
+    assert_eq!(reader.read_line().expect("line"), "c");
+    assert!(reader.eof().expect("eof"));
+}
+
+#[test]
+fn source_reader_normalizes_crlf_and_lone_cr() {
+    let mut reader = CsReader::new_with_string("a\r\nb\rc\n".to_owned()).expect("reader");
+    assert_eq!(reader.read_line().expect("line"), "a");
+    assert_eq!(reader.read_line().expect("line"), "b");
+    assert_eq!(reader.read_line().expect("line"), "c");
+}
+
+#[test]
+fn source_reader_rejects_unterminated_block_comment_at_eof() {
+    assert_eq!(
+        CsReader::new_with_string("before /* never closed".to_owned()).expect_err("invalid"),
+        Error::Parse {
+            position: 22,
+            context: "unterminated block comment"
+        }
+    );
+}
+
+#[test]
+fn line_directive_updates_file_and_logical_line() {
+    let reader =
+        CsReader::new_with_string("#line 700 \"table.lex\"\nword\n".to_owned()).expect("reader");
+    assert_eq!(reader.fname, "table.lex");
+    let info = SourceLineInfo::new_with_line_manager_int32(reader.lm, 1).expect("location");
+    assert_eq!(info.line_number, 700);
+}
+
+#[test]
+fn utf16_input_decoding_and_surrogate_validation_are_deterministic() {
+    let mut reader = CsReader::from_bytes(
+        &[0xFF, 0xFE, b'A', 0, 0x3D, 0xD8, 0, 0xDE],
+        InputEncoding::Utf16Le,
+        "source.lex",
+    )
+    .expect("utf16");
+    assert_eq!(reader.read_line().expect("line"), "A😀");
+    let mut bom_override =
+        CsReader::from_bytes(&[0xFE, 0xFF, 0, b'B'], InputEncoding::Utf16Le, "source.lex")
+            .expect("BOM override");
+    assert_eq!(bom_override.read_line().expect("line"), "B");
+    assert!(matches!(
+        CsReader::from_bytes(&[0x00, 0xD8], InputEncoding::Utf16Le, "bad"),
+        Err(Error::Parse {
+            position: 0,
+            context: "source contains an unpaired UTF-16 surrogate"
+        })
+    ));
+}
+
+#[test]
+fn ascii_input_rejects_non_ascii_instead_of_lossily_replacing_it() {
+    assert!(matches!(
+        CsReader::from_bytes(&[b'a', 0x80], InputEncoding::Ascii, "bad"),
+        Err(Error::Parse {
+            position: 1,
+            context: "source contains a non-ASCII byte"
+        })
+    ));
+}
+
+#[test]
+fn category_predicates_match_dotnet_values_and_groups() {
+    let mut table = YyLexer::new(ErrorHandler::default()).expect("table");
+    let punctuation = table.get_test("Punctuation").expect("punctuation");
+    let whitespace = table.get_test("WhiteSpace").expect("whitespace");
+    let states = vec![
+        DfaState::default()
+            .transition(punctuation, 1)
+            .transition(whitespace, 2),
+        DfaState::default().accepting(accept("PUNCT", 1, LexerAction::Emit, None)),
+        DfaState::default().accepting(accept("SPACE", 2, LexerAction::Emit, None)),
+    ];
+    let dfa = Dfa::from_states(states, 0).expect("dfa");
+    let mut action = -1;
+    assert_eq!(dfa.match_("—".to_owned(), 0, &mut action), Ok(1));
+    assert_eq!(dfa.match_("\t".to_owned(), 0, &mut action), Ok(1));
+    assert_eq!(dfa.match_("\0".to_owned(), 0, &mut action), Ok(-1));
+    assert!(
+        libremetaverse_lsl_tools::CatTest::new(UnicodeCategory(16))
+            .expect("surrogate")
+            .test(Utf16CodeUnit(0xD800))
+            .expect("test")
+    );
+    assert_eq!(table.get_test("MissingClass"), Err(Error::Argument));
+    assert_eq!(
+        table.erh.diagnostics()[0].category,
+        DiagnosticCategory::UnknownCharacterSet
+    );
+}
+
+#[test]
+fn dfa_uses_maximum_munch_and_exposes_action_number() {
+    let states = vec![
+        DfaState::default().transition(CharacterMatcher::Exact(b'a'.into()), 1),
+        DfaState::default()
+            .transition(CharacterMatcher::Exact(b'b'.into()), 2)
+            .accepting(accept("A", 1, LexerAction::Emit, None)),
+        DfaState::default().accepting(accept("AB", 2, LexerAction::Emit, None)),
+    ];
+    let dfa = Dfa::from_states(states, 0).expect("dfa");
+    let mut action = -1;
+    assert_eq!(dfa.match_("abc".to_owned(), 0, &mut action), Ok(2));
+    assert_eq!(action, 2);
+}
+
+#[test]
+fn deterministic_table_output_is_independent_of_hash_iteration() {
+    let table = language_table();
+    let first = Arc::new(Mutex::new(Vec::new()));
+    table
+        .emit_dfa(Box::new(SharedWriter(first.clone())))
+        .expect("emit first");
+    let second = Arc::new(Mutex::new(Vec::new()));
+    table
+        .emit_dfa(Box::new(SharedWriter(second.clone())))
+        .expect("emit second");
+    let first = first.lock().expect("first output").clone();
+    let second = second.lock().expect("second output").clone();
+    assert_eq!(first, second);
+    assert!(
+        String::from_utf8(first)
+            .expect("utf8")
+            .starts_with("LSLLEXER 1\n")
+    );
+}
+
+struct SharedWriter(Arc<Mutex<Vec<u8>>>);
+
+impl std::io::Write for SharedWriter {
+    fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {
+        self.0
+            .lock()
+            .map_err(|_| std::io::Error::other("poisoned output"))?
+            .extend_from_slice(bytes);
+        Ok(bytes.len())
+    }
+
+    fn flush(&mut self) -> std::io::Result<()> {
+        Ok(())
+    }
+}
+
+#[test]
+fn compatibility_enumerator_resets_to_the_first_token() {
+    let lexer = lexer("one two");
+    let mut enumerator = lexer.get_enumerator().expect("enumerator");
+    assert!(enumerator.move_next().expect("first"));
+    assert_eq!(enumerator.current().yytext(), "one");
+    assert!(enumerator.move_next().expect("second"));
+    assert_eq!(enumerator.current().yytext(), "two");
+    enumerator.reset().expect("reset");
+    assert!(enumerator.move_next().expect("first again"));
+    assert_eq!(enumerator.current().yytext(), "one");
+}
+
+#[test]
+fn error_handler_counts_reports_and_honors_throw_mode() {
+    let error =
+        CSToolsException::new_with_int32_string(43, "bad encoding".to_owned()).expect("diagnostic");
+    let mut collecting = ErrorHandler::new_with_constructor().expect("handler");
+    collecting.error(error.clone()).expect("reported");
+    assert_eq!(collecting.counter, 1);
+    assert_eq!(
+        collecting.diagnostics()[0].category,
+        DiagnosticCategory::Encoding
+    );
+
+    let mut throwing = ErrorHandler::new_with_boolean(true).expect("handler");
+    assert_eq!(throwing.error(error), Err(Error::InvalidOperation));
+    assert_eq!(throwing.counter, 1);
+    assert!(throwing.diagnostics().is_empty());
+}
+
+#[test]
+fn line_manager_backtracking_removes_future_lines() {
+    let mut manager = LineManager::new().expect("manager");
+    manager.newline(4).expect("line 2");
+    let first_on_second =
+        SourceLineInfo::new_with_line_manager_int32(manager.clone(), 4).expect("location");
+    assert_eq!(first_on_second.raw_char_position, 1);
+    manager.newline(8).expect("line 3");
+    manager.backto(6).expect("rewind");
+    assert_eq!(manager.lines, 2);
+    assert_eq!(manager.end, 8);
+}
+
+#[test]
+fn object_list_preserves_push_add_top_pop_and_index_order() {
+    let mut values = ObjectList::new().expect("list");
+    values.add(Object::Integer(2)).expect("add");
+    values.push(Object::Integer(1)).expect("push");
+    assert_eq!(values.count(), 2);
+    assert_eq!(values.top(), Object::Integer(1));
+    assert_eq!(values.item(1), Object::Integer(2));
+    assert_eq!(values.pop(), Ok(Object::Integer(1)));
+}
+
+#[test]
+fn matcher_set_is_value_based_and_bounded() {
+    let matcher = CharacterMatcher::Set(BTreeSet::from([b'x'.into(), b'y'.into()]));
+    let states = vec![
+        DfaState::default().transition(matcher, 1),
+        DfaState::default().accepting(accept("XY", 1, LexerAction::Emit, None)),
+    ];
+    let dfa = Dfa::from_states(states, 0).expect("dfa");
+    let mut action = 0;
+    assert_eq!(dfa.match_("y".to_owned(), 0, &mut action), Ok(1));
+    assert_eq!(dfa.match_("z".to_owned(), 0, &mut action), Ok(-1));
+}

crates/libremetaverse-types/src/compat.rs

@@ -269,7 +269,7 @@ pub struct ArrayList(pub Vec<Object>);
 #[derive(Clone, Debug, Eq, PartialEq)]
 pub struct Array(pub Vec<Object>);
 
-#[derive(Clone, Debug, Eq, PartialEq)]
+#[derive(Clone, Debug, Default, Eq, PartialEq)]
 pub struct Hashtable(pub std::collections::HashMap<Object, Object>);
 
 #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
@@ -1307,8 +1307,10 @@ impl<'a, T> IntoIterator for &'a ImmutableList<T> {
 
 pub struct ICollection;
 
+#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
 pub struct UnicodeCategory(pub i32);
 
+#[derive(Clone, Copy, Debug, Default, Eq, Hash, PartialEq)]
 pub struct TextEncoding;
 
 pub struct SocketException;

tests/api-compile/Cargo.lock

@@ -783,6 +783,7 @@ name = "libremetaverse-lsl-tools"
 version = "0.0.1"
 dependencies = [
  "libremetaverse-types",
+ "unicode-general-category",
 ]
 
 [[package]]
@@ -1921,6 +1922,12 @@ version = "1.20.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20"
 
+[[package]]
+name = "unicode-general-category"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b993bddc193ae5bd0d623b49ec06ac3e9312875fdae725a975c51db1cc1677f"
+
 [[package]]
 name = "unicode-ident"
 version = "1.0.24"

tools/check_milestone_10_issue_81.py

@@ -0,0 +1,148 @@
+#!/usr/bin/env python3
+"""Audit issue 81's native lexer, token, source, and diagnostic boundary."""
+
+from __future__ import annotations
+
+import json
+import re
+from pathlib import Path
+
+import generate_api_shims
+
+
+ROOT = Path(__file__).resolve().parents[1]
+SOURCE = ROOT / "crates" / "libremetaverse-lsl-tools" / "src" / "lexer.rs"
+GENERATED = ROOT / "crates" / "libremetaverse-lsl-tools" / "src" / "generated.rs"
+UNIT_TESTS = SOURCE
+COMPAT_TESTS = (
+    ROOT / "crates" / "libremetaverse-lsl-tools" / "tests" / "lexer_compat.rs"
+)
+DOC = ROOT / "crates" / "libremetaverse-lsl-tools" / "README.md"
+WORKFLOW = ROOT / ".gitea" / "workflows" / "rust-workspace.yml"
+CATALOG = ROOT / "api" / "public-api.json"
+STUB_RE = re.compile(
+    r"\b(?:not_implemented|unimplemented_api)\b|\b(?:todo|unimplemented)!\s*\("
+)
+
+TYPES = {
+    "T:LibreMetaverse.LslTools.CSToolsException": "crate::lexer::CSToolsException",
+    "T:LibreMetaverse.LslTools.CSToolsFatalException":
+        "crate::lexer::CSToolsFatalException",
+    "T:LibreMetaverse.LslTools.CSToolsStopException":
+        "crate::lexer::CSToolsStopException",
+    "T:LibreMetaverse.LslTools.CatTest": "crate::lexer::CatTest",
+    "T:LibreMetaverse.LslTools.Charset": "crate::lexer::Charset",
+    "T:LibreMetaverse.LslTools.CommentList": "crate::lexer::CommentList",
+    "T:LibreMetaverse.LslTools.CsReader": "crate::lexer::CsReader",
+    "T:LibreMetaverse.LslTools.Dfa": "crate::lexer::Dfa",
+    "T:LibreMetaverse.LslTools.Dfa.Action": "crate::lexer::DfaAction",
+    "T:LibreMetaverse.LslTools.EOF": "crate::lexer::EOF",
+    "T:LibreMetaverse.LslTools.ErrorHandler": "crate::lexer::ErrorHandler",
+    "T:LibreMetaverse.LslTools.Lexer": "crate::lexer::Lexer",
+    "T:LibreMetaverse.LslTools.Lexer._Enumerator": "crate::lexer::LexerEnumerator",
+    "T:LibreMetaverse.LslTools.LineList": "crate::lexer::LineList",
+    "T:LibreMetaverse.LslTools.LineManager": "crate::lexer::LineManager",
+    "T:LibreMetaverse.LslTools.Null": "crate::lexer::Null",
+    "T:LibreMetaverse.LslTools.ObjectList": "crate::lexer::ObjectList",
+    "T:LibreMetaverse.LslTools.ResWds": "crate::lexer::ResWds",
+    "T:LibreMetaverse.LslTools.SYMBOL": "crate::lexer::SYMBOL",
+    "T:LibreMetaverse.LslTools.SourceLineInfo": "crate::lexer::SourceLineInfo",
+    "T:LibreMetaverse.LslTools.TOKEN": "crate::lexer::TOKEN",
+    "T:LibreMetaverse.LslTools.YyLexer": "crate::lexer::YyLexer",
+}
+
+
+def require_markers(path: Path, markers: tuple[str, ...]) -> None:
+    text = path.read_text()
+    missing = [marker for marker in markers if marker not in text]
+    if missing:
+        raise SystemExit(f"{path.name}: audit evidence missing: " + ", ".join(missing))
+
+
+def catalog_members() -> set[str]:
+    catalog = json.loads(CATALOG.read_text())
+    assembly = next(
+        value for value in catalog["assemblies"]
+        if value["identity"]["name"] == "LibreMetaverse.LslTools"
+    )
+    return {
+        member["doc_id"]
+        for api_type in assembly["types"]
+        if api_type["doc_id"] in TYPES
+        for member in api_type["members"]
+    }
+
+
+def generated_type_block(text: str, doc_id: str) -> str:
+    marker = f"/// C# type: `{doc_id}`."
+    start = text.find(marker)
+    if start < 0:
+        raise SystemExit(f"generated declaration is missing for {doc_id}")
+    next_type = text.find("/// C# type:", start + len(marker))
+    return text[start:] if next_type < 0 else text[start:next_type]
+
+
+def main() -> None:
+    for api_type, declaration in TYPES.items():
+        if generate_api_shims.NATIVE_TYPES.get(api_type) != declaration:
+            raise SystemExit(f"issue 81 native type mapping is missing for {api_type}")
+    members = catalog_members()
+    if len(members) != 188:
+        raise SystemExit(f"issue 81 expected 188 mapped members, found {len(members)}")
+
+    source = SOURCE.read_text()
+    if STUB_RE.search(source):
+        raise SystemExit("issue 81 owned Rust stubs remain in lexer.rs")
+    if "Ok(o)" in source or "TokenDefinition::new(name, 1)" in source:
+        raise SystemExit("issue 81 contains a silent compatibility no-op or fabricated token")
+    if re.search(r"unsafe\s*\{|unsafe\s+impl|target_os\s*=\s*\"macos\"", source):
+        raise SystemExit("lexer.rs contains an unsafe or macOS-only implementation")
+    generated = GENERATED.read_text()
+    for api_type in TYPES:
+        if STUB_RE.search(generated_type_block(generated, api_type)):
+            raise SystemExit(f"issue 81 owned generated stubs remain for {api_type}")
+
+    require_markers(SOURCE, (
+        "MAX_SOURCE_UNITS", "MAX_TOKEN_UNITS", "pub enum DotNetUnicodeCategory",
+        "pub enum InputEncoding", "pub struct CatTest", "pub struct Charset",
+        "pub struct CsReader", "fn filter_source", "fn parse_line_directive",
+        "pub struct LineManager", "pub struct SourceLineInfo",
+        "pub enum DiagnosticCategory", "pub struct Diagnostic",
+        "pub struct ErrorHandler", "pub enum CharacterMatcher",
+        "pub struct DfaState", "pub struct Dfa", "fn longest_match",
+        "pub struct ResWds", "pub struct YyLexer", "pub struct SYMBOL",
+        "pub struct TOKEN", "pub struct EOF", "pub struct Lexer",
+        "pub struct LexerIterator", "impl Iterator for LexerIterator",
+        "lookup_parser_entry", "emit_dfa", "record_diagnostic",
+        "unicode_general_category", "String::from_utf16_lossy",
+    ))
+    require_markers(COMPAT_TESTS, (
+        "token_fixture_preserves_names_text_numbers_positions_and_eof",
+        "invalid_input_reports_stable_category_location_and_text",
+        "source_reader_removes_both_comment_forms_and_keeps_newlines",
+        "utf16_input_decoding_and_surrogate_validation_are_deterministic",
+        "category_predicates_match_dotnet_values_and_groups",
+        "dfa_uses_maximum_munch_and_exposes_action_number",
+        "deterministic_table_output_is_independent_of_hash_iteration",
+        "compatibility_enumerator_resets_to_the_first_token",
+        "error_handler_counts_reports_and_honors_throw_mode",
+    ))
+    test_count = UNIT_TESTS.read_text().count("#[test]") + COMPAT_TESTS.read_text().count("#[test]")
+    if test_count != 27:
+        raise SystemExit(f"issue 81 expected 27 focused fixtures, found {test_count}")
+    require_markers(DOC, (
+        "UTF-16 code-unit offsets", "maximum munch", "structured `Diagnostic`",
+        "64 Mi UTF-16", "16 Mi units", "Lexer::iter", "27 focused",
+        "No C#, .NET runtime", "ubuntu-latest",
+    ))
+    require_markers(WORKFLOW, ("python3 tools/check_milestone_10_issue_81.py",))
+    print(
+        "issue 81 audit: 22 mapped native types and 188 members, bounded UTF-16 "
+        "source handling, comments and line directives, Unicode categories, deterministic "
+        "DFA/token behavior, reserved words, EOF, diagnostics, iterators, parser-table "
+        "lookup, 27 focused fixtures, API documentation, and ubuntu-only CI are present"
+    )
+
+
+if __name__ == "__main__":
+    main()

tools/generate_api_shims.py

@@ -38,6 +38,28 @@ TARGETS = {
 # implementations. The generated module keeps catalog markers and re-exports
 # the hand-written type so coverage remains deterministic.
 NATIVE_TYPES = {
+    "T:LibreMetaverse.LslTools.CSToolsException": "crate::lexer::CSToolsException",
+    "T:LibreMetaverse.LslTools.CSToolsFatalException": "crate::lexer::CSToolsFatalException",
+    "T:LibreMetaverse.LslTools.CSToolsStopException": "crate::lexer::CSToolsStopException",
+    "T:LibreMetaverse.LslTools.CatTest": "crate::lexer::CatTest",
+    "T:LibreMetaverse.LslTools.Charset": "crate::lexer::Charset",
+    "T:LibreMetaverse.LslTools.CommentList": "crate::lexer::CommentList",
+    "T:LibreMetaverse.LslTools.CsReader": "crate::lexer::CsReader",
+    "T:LibreMetaverse.LslTools.Dfa": "crate::lexer::Dfa",
+    "T:LibreMetaverse.LslTools.Dfa.Action": "crate::lexer::DfaAction",
+    "T:LibreMetaverse.LslTools.EOF": "crate::lexer::EOF",
+    "T:LibreMetaverse.LslTools.ErrorHandler": "crate::lexer::ErrorHandler",
+    "T:LibreMetaverse.LslTools.Lexer": "crate::lexer::Lexer",
+    "T:LibreMetaverse.LslTools.Lexer._Enumerator": "crate::lexer::LexerEnumerator",
+    "T:LibreMetaverse.LslTools.LineList": "crate::lexer::LineList",
+    "T:LibreMetaverse.LslTools.LineManager": "crate::lexer::LineManager",
+    "T:LibreMetaverse.LslTools.Null": "crate::lexer::Null",
+    "T:LibreMetaverse.LslTools.ObjectList": "crate::lexer::ObjectList",
+    "T:LibreMetaverse.LslTools.ResWds": "crate::lexer::ResWds",
+    "T:LibreMetaverse.LslTools.SYMBOL": "crate::lexer::SYMBOL",
+    "T:LibreMetaverse.LslTools.SourceLineInfo": "crate::lexer::SourceLineInfo",
+    "T:LibreMetaverse.LslTools.TOKEN": "crate::lexer::TOKEN",
+    "T:LibreMetaverse.LslTools.YyLexer": "crate::lexer::YyLexer",
     "T:LibreMetaverse.RLV.AttachmentRequest": "crate::service::AttachmentRequest",
     "T:LibreMetaverse.RLV.RlvActionCallbacksDefault": "crate::service::RlvActionCallbacksDefault",
     "T:LibreMetaverse.RLV.RlvCallbacksDefault": "crate::service::RlvCallbacksDefault",