Generate native LSL parser tables (#83)

crates/libremetaverse-lsl-tools/Cargo.toml

@@ -10,6 +10,7 @@ description = "Native LSL lexer and parser tooling for the MetaCrate LibreMetave
 [dependencies]
 libremetaverse-types = { path = "../libremetaverse-types" }
 unicode-general-category = "1.1"
+regex = "1"
 
 [lints]
 workspace = true

crates/libremetaverse-lsl-tools/README.md

@@ -6,8 +6,8 @@ issue 81 boundary supplies source reading, comments, Unicode character sets,
 deterministic DFA execution, reserved words, terminal symbols, EOF, source
 locations, diagnostics, and Rust iterators. The issue 82 boundary supplies
 grammar productions, parser tables, precedence, reductions, and recovery.
-Deterministic checked-in generated grammar and token tables are added by issue
-83.
+The issue 83 boundary adds deterministic checked-in grammar/token tables and
+native replacements for the retired parser-generator object model.
 
 ## Source and position contract
 
@@ -81,9 +81,35 @@ recovery attempts.
 Compatibility `SymbolSet` clones share deterministic FIRST/FOLLOW membership,
 and cloned `CSymbol` values share registered production metadata. This retains
 the observable reference behavior of the C# grammar model without unsafe code.
-Parser-source serialization and the old `Builder`-driven generated-output
-members return `InvalidOperation` explicitly until issue 83 supplies the
-checked-in generator; they never report a false success.
+Legacy parser-runtime serialization members that cannot faithfully round-trip
+the richer native LALR machine remain explicit `InvalidOperation` results.
+Callers should use the stable `YyParser::emit` and `YyLexer::emit_dfa` formats,
+or regenerate the canonical checked-in table module described below.
+
+## Checked-in generator contract
+
+[`codegen/inputs/lsl_tools_grammar.json`](../../codegen/inputs/lsl_tools_grammar.json)
+is the reviewed source of truth reconstructed from the pinned 4.5
+`yycs0syntax.cs` and `yycs0tokens.cs` tables at upstream commit
+`2aa70bb68513b39795da5d13c88f31b86e85a3ba`. It records every token,
+nonterminal, production, and semantic-action class. The portable Python
+generator uses the standard library plus the workspace Rust formatter and emits
+`src/generated_tables.rs`; it does not invoke C#, a C# source generator, or a
+.NET runtime.
+
+Regenerate with `python3 tools/generate_lsl_tables.py`. Use
+`python3 tools/generate_lsl_tables.py --check` in reviews and CI. The check
+renders the complete output in memory and compares bytes, so repeated runs are
+independent of hash iteration, locale, host operating system, and timestamps.
+The checked-in module supplies the canonical 25-production grammar, a Unicode
+lexer with reserved `base`, `this`, and `new` tokens, the 84 generated syntax
+classes, the 13 generated token classes, and the four `cs0` runtime wrappers.
+
+The old `GenBase`, `SymbolsGen`, `TokensGen`, `Regex`, NFA, delegate, factory,
+and 4.5 `Serialiser` APIs are live Rust compatibility adapters. New code should
+prefer `Grammar`, typed `DfaState` builders, `generated_parser`, and
+`generated_lexer`; these avoid CLR-style method pointers and class-shaped
+generated inheritance while retaining deterministic mapped behavior.
 
 ## Diagnostics and migration
 
@@ -108,6 +134,9 @@ The principal API mapping is:
 | `CSymbol`, `SymbolSet`, `Production`, `Precedence` | Same mapped names plus typed `Grammar` builders |
 | `YyParser`, parser entries | Deterministic native tables and conflict records |
 | `Parser`, `ParseStackEntry`, `Error`, `recoveredError` | Bounded parsing, typed trees, and recovery |
+| `yycs0syntax`, `yycs0tokens` | `generated_parser`, `generated_lexer`, checked-in native types |
+| `GenBase`, `SymbolsGen`, `TokensGen`, `Regex`, `Nfa` | Portable Rust generator adapters; prefer typed grammar/DFA builders |
+| `Serialiser` | Deterministic 4.5-compatible integer stream for supported mapped values |
 
 No C#, .NET runtime, dynamically loaded class, macOS-only API, platform code
 page, or runtime source generation is used by this boundary.
@@ -123,8 +152,10 @@ CARGO_BUILD_JOBS=1 cargo clippy -p libremetaverse-lsl-tools --all-targets --lock
 RUSTDOCFLAGS='-D warnings' CARGO_BUILD_JOBS=1 cargo doc -p libremetaverse-lsl-tools --no-deps --locked
 python3 tools/check_milestone_10_issue_81.py
 python3 tools/check_milestone_10_issue_82.py
+python3 tools/check_milestone_10_issue_83.py
+python3 tools/generate_lsl_tables.py --check
 python3 tools/generate_api_shims.py --check
 ```
 
-The package contains 37 focused native and compatibility fixtures. The Gitea
+The package contains 46 focused native and compatibility fixtures. The Gitea
 workflow runs the audit and workspace compile on `ubuntu-latest` only.

crates/libremetaverse-lsl-tools/src/generated_tables.rs

@@ -0,0 +1,857 @@
+// @generated by tools/generate_lsl_tables.py; do not edit by hand.
+#![allow(non_camel_case_types)]
+#![allow(non_snake_case)]
+#![allow(clippy::missing_errors_doc)]
+#![allow(clippy::must_use_candidate)]
+
+use std::ops::{Deref, DerefMut};
+
+use libremetaverse_types::compat::{Object, UnicodeCategory};
+
+use crate::{
+    CharacterMatcher, Dfa, DfaAccept, DfaState, Error, ErrorHandler, Grammar, Lexer, LexerAction,
+    Parser, ResWds, SYMBOL, TOKEN, TokenDefinition, UnicodeClass, YyLexer, YyParser,
+};
+
+pub const LSL_GENERATOR_SOURCE_COMMIT: &str = "2aa70bb68513b39795da5d13c88f31b86e85a3ba";
+pub const LSL_GENERATOR_SOURCE_PARSER: &str = "LibreMetaverse.LslTools/YYClass/yycs0syntax.cs";
+pub const LSL_GENERATOR_SOURCE_LEXER: &str = "LibreMetaverse.LslTools/YYClass/yycs0tokens.cs";
+pub const LSL_GENERATOR_SOURCE_LICENSE: &str = "BSD-3-Clause";
+pub const LSL_GENERATOR_SERIALIZATION_VERSION: &str = "4.5";
+pub const GENERATED_PARSER_DATA: &[i32] = &[
+    1, 19, 2, 22, 2, 1, 3, 3, 1, 4, 4, 1, 4, 5, 1, 3, 6, 1, 2, 7, 1, 3, 8, 1, 5, 9, 1, 9, 10, 1, 6,
+    11, 1, 6, 12, 1, 6, 13, 1, 6, 14, 1, 6, 15, 1, 6, 16, 0, 6, 17, 0, 5, 18, 0, 4, 19, 0, 9, 20,
+    0, 4, 21, 0, 4, 22, 0, 8, 23, 0, 4, 25, 1, 19, 3, 16, 20, 16, 2, 16, 0, 3, 16, 2, 16, 18, 4,
+    16, 2, 16, 21, 5, 17, 0, 6, 17, 2, 17, 18, 7, 20, 5, 23, 12, 17, 13, 22, 8, 21, 4, 23, 12, 17,
+    13, 9, 22, 0, 10, 22, 5, 8, 3, 12, 17, 13, 11, 22, 5, 8, 4, 12, 17, 13, 12, 23, 1, 6, 13, 23,
+    4, 6, 14, 17, 15, 14, 18, 1, 6, 15, 18, 1, 7, 16, 18, 1, 9, 17, 18, 1, 3, 18, 18, 1, 4, 19, 18,
+    4, 4, 14, 17, 15, 20, 18, 1, 8, 21, 18, 2, 5, 23, 22, 18, 2, 5, 6, 23, 18, 3, 12, 17, 13, 24,
+    18, 3, 10, 16, 11, 25, 18, 3, 14, 17, 15,
+];
+
+pub fn generated_symbol_name(number: i32) -> Option<&'static str> {
+    match number {
+        2 => Some("EOF"),
+        3 => Some("BASE"),
+        4 => Some("THIS"),
+        5 => Some("NEW"),
+        6 => Some("ID"),
+        7 => Some("ANY"),
+        8 => Some("COLON"),
+        9 => Some("SEMICOLON"),
+        10 => Some("LBRACE"),
+        11 => Some("RBRACE"),
+        12 => Some("LPAREN"),
+        13 => Some("RPAREN"),
+        14 => Some("LBRACK"),
+        15 => Some("RBRACK"),
+        16 => Some("GStuff"),
+        17 => Some("Stuff"),
+        18 => Some("Item"),
+        19 => Some("ClassBody"),
+        20 => Some("Cons"),
+        21 => Some("Call"),
+        22 => Some("BaseCall"),
+        23 => Some("Name"),
+        _ => None,
+    }
+}
+
+pub fn generated_parser() -> Result<YyParser, Error> {
+    let mut grammar = Grammar::new(19, 2)?;
+    grammar.add_symbol("EOF", 2, true)?;
+    grammar.add_symbol("BASE", 3, true)?;
+    grammar.add_symbol("THIS", 4, true)?;
+    grammar.add_symbol("NEW", 5, true)?;
+    grammar.add_symbol("ID", 6, true)?;
+    grammar.add_symbol("ANY", 7, true)?;
+    grammar.add_symbol("COLON", 8, true)?;
+    grammar.add_symbol("SEMICOLON", 9, true)?;
+    grammar.add_symbol("LBRACE", 10, true)?;
+    grammar.add_symbol("RBRACE", 11, true)?;
+    grammar.add_symbol("LPAREN", 12, true)?;
+    grammar.add_symbol("RPAREN", 13, true)?;
+    grammar.add_symbol("LBRACK", 14, true)?;
+    grammar.add_symbol("RBRACK", 15, true)?;
+    grammar.add_symbol("GStuff", 16, false)?;
+    grammar.add_symbol("Stuff", 17, false)?;
+    grammar.add_symbol("Item", 18, false)?;
+    grammar.add_symbol("ClassBody", 19, false)?;
+    grammar.add_symbol("Cons", 20, false)?;
+    grammar.add_symbol("Call", 21, false)?;
+    grammar.add_symbol("BaseCall", 22, false)?;
+    grammar.add_symbol("Name", 23, false)?;
+    grammar.add_production(19, vec![16, 20, 16])?;
+    grammar.add_production(16, vec![])?;
+    grammar.add_production(16, vec![16, 18])?;
+    grammar.add_production(16, vec![16, 21])?;
+    grammar.add_production(17, vec![])?;
+    grammar.add_production(17, vec![17, 18])?;
+    grammar.add_production(20, vec![23, 12, 17, 13, 22])?;
+    grammar.add_production(21, vec![23, 12, 17, 13])?;
+    grammar.add_production(22, vec![])?;
+    grammar.add_production(22, vec![8, 3, 12, 17, 13])?;
+    grammar.add_production(22, vec![8, 4, 12, 17, 13])?;
+    grammar.add_production(23, vec![6])?;
+    grammar.add_production(23, vec![6, 14, 17, 15])?;
+    grammar.add_production(18, vec![6])?;
+    grammar.add_production(18, vec![7])?;
+    grammar.add_production(18, vec![9])?;
+    grammar.add_production(18, vec![3])?;
+    grammar.add_production(18, vec![4])?;
+    grammar.add_production(18, vec![4, 14, 17, 15])?;
+    grammar.add_production(18, vec![8])?;
+    grammar.add_production(18, vec![5, 23])?;
+    grammar.add_production(18, vec![5, 6])?;
+    grammar.add_production(18, vec![12, 17, 13])?;
+    grammar.add_production(18, vec![10, 16, 11])?;
+    grammar.add_production(18, vec![14, 17, 15])?;
+    let mut parser = grammar.build()?;
+    parser.arr = GENERATED_PARSER_DATA.to_vec();
+    Ok(parser)
+}
+
+fn accept(name: &str, number: i32, action: LexerAction, action_number: i32) -> DfaAccept {
+    DfaAccept {
+        token: TokenDefinition {
+            name: name.to_owned(),
+            number,
+        },
+        action,
+        action_number,
+        reserved_words: (name == "ID").then(|| "ID".to_owned()),
+    }
+}
+
+pub fn generated_lexer_with_handler(error_handler: ErrorHandler) -> Result<YyLexer, Error> {
+    let punctuation = [
+        (58, "COLON", 8),
+        (59, "SEMICOLON", 9),
+        (123, "LBRACE", 10),
+        (125, "RBRACE", 11),
+        (40, "LPAREN", 12),
+        (41, "RPAREN", 13),
+        (91, "LBRACK", 14),
+        (93, "RBRACK", 15),
+    ];
+    let punctuation_start = 3usize;
+    let any_state = punctuation_start + punctuation.len();
+    let mut states = vec![DfaState::default(); any_state + 1];
+    states[0] = states[0]
+        .clone()
+        .transition(CharacterMatcher::UnicodeClass(UnicodeClass::WhiteSpace), 1)
+        .transition(CharacterMatcher::UnicodeClass(UnicodeClass::Letter), 2)
+        .transition(CharacterMatcher::Exact(u16::from(b'_')), 2);
+    for (offset, (unit, _, _)) in punctuation.iter().enumerate() {
+        states[0]
+            .transitions
+            .push((CharacterMatcher::Exact(*unit), punctuation_start + offset));
+    }
+    states[0]
+        .transitions
+        .push((CharacterMatcher::Any, any_state));
+    states[1] = states[1]
+        .clone()
+        .transition(CharacterMatcher::UnicodeClass(UnicodeClass::WhiteSpace), 1)
+        .accepting(accept("ANY", 7, LexerAction::Skip, -1));
+    states[2] = states[2]
+        .clone()
+        .transition(CharacterMatcher::UnicodeClass(UnicodeClass::Letter), 2)
+        .transition(CharacterMatcher::UnicodeClass(UnicodeClass::Number), 2)
+        .transition(CharacterMatcher::Exact(u16::from(b'_')), 2)
+        .accepting(accept("ID", 6, LexerAction::Emit, 0));
+    for (offset, (_, name, number)) in punctuation.iter().enumerate() {
+        states[punctuation_start + offset] =
+            DfaState::default().accepting(accept(name, *number, LexerAction::Emit, *number));
+    }
+    states[any_state] = DfaState::default().accepting(accept("ANY", 7, LexerAction::Emit, 1));
+    let mut lexer = YyLexer::new(error_handler)?;
+    lexer.set_start_dfa("YYINITIAL", Dfa::from_states(states, 0)?)?;
+    lexer.set_reserved_words(
+        "ID",
+        ResWds::from_pairs(
+            [
+                ("base", TokenDefinition::new("BASE", 3)?),
+                ("this", TokenDefinition::new("THIS", 4)?),
+                ("new", TokenDefinition::new("NEW", 5)?),
+            ],
+            false,
+        )?,
+    )?;
+    lexer.using_eof = true;
+    for (name, number) in [
+        ("EOF", 2),
+        ("BASE", 3),
+        ("THIS", 4),
+        ("NEW", 5),
+        ("ID", 6),
+        ("ANY", 7),
+        ("COLON", 8),
+        ("SEMICOLON", 9),
+        ("LBRACE", 10),
+        ("RBRACE", 11),
+        ("LPAREN", 12),
+        ("RPAREN", 13),
+        ("LBRACK", 14),
+        ("RBRACK", 15),
+    ] {
+        lexer
+            .tokens
+            .0
+            .insert(Object::String(name.to_owned()), Object::Integer(number));
+    }
+    let _ = lexer.using_cat(UnicodeCategory(0))?;
+    Ok(lexer)
+}
+
+pub fn generated_lexer() -> Result<YyLexer, Error> {
+    generated_lexer_with_handler(ErrorHandler::new_with_boolean(false)?)
+}
+
+#[derive(Clone, Debug)]
+pub struct GeneratedLexerToken<const NUMBER: i32>(TOKEN);
+
+impl<const NUMBER: i32> GeneratedLexerToken<NUMBER> {
+    pub fn new(lexer: Lexer) -> Result<Self, Error> {
+        let name = generated_symbol_name(NUMBER).ok_or(Error::Argument)?;
+        Ok(Self(TOKEN::generated_with_lexer(lexer, name, NUMBER)?))
+    }
+    pub fn yyname(&self) -> String {
+        self.0.yyname()
+    }
+    pub const fn yynum(&self) -> i32 {
+        NUMBER
+    }
+    pub fn yytext(&self) -> String {
+        self.0.yytext()
+    }
+}
+
+impl<const NUMBER: i32> Deref for GeneratedLexerToken<NUMBER> {
+    type Target = TOKEN;
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}
+impl<const NUMBER: i32> DerefMut for GeneratedLexerToken<NUMBER> {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.0
+    }
+}
+pub type BASE = GeneratedLexerToken<3>;
+pub type THIS = GeneratedLexerToken<4>;
+pub type NEW = GeneratedLexerToken<5>;
+pub type ID = GeneratedLexerToken<6>;
+pub type ANY = GeneratedLexerToken<7>;
+pub type COLON = GeneratedLexerToken<8>;
+pub type SEMICOLON = GeneratedLexerToken<9>;
+pub type LBRACE = GeneratedLexerToken<10>;
+pub type RBRACE = GeneratedLexerToken<11>;
+pub type LPAREN = GeneratedLexerToken<12>;
+pub type RPAREN = GeneratedLexerToken<13>;
+pub type LBRACK = GeneratedLexerToken<14>;
+pub type RBRACK = GeneratedLexerToken<15>;
+
+fn generated_parser_identity(generated: i32) -> Option<(&'static str, i32)> {
+    match generated {
+        27..=35 => Some(("GStuff", 16)),
+        36..=41 => Some(("Stuff", 17)),
+        63..=98 => Some(("Item", 18)),
+        24..=26 => Some(("ClassBody", 19)),
+        42..=44 => Some(("Cons", 20)),
+        45..=47 => Some(("Call", 21)),
+        48..=56 => Some(("BaseCall", 22)),
+        57..=62 => Some(("Name", 23)),
+        _ => None,
+    }
+}
+
+fn stack_text(parser: &Parser, index: i32) -> Result<String, Error> {
+    Ok(parser.stack_at(index)?.m_value.raw_text())
+}
+
+fn generated_semantic_text(generated: i32, parser: &Parser) -> Result<String, Error> {
+    Ok(match generated {
+        26 => stack_text(parser, 1)?,
+        32 | 35 | 41 => format!("{}{}", stack_text(parser, 1)?, stack_text(parser, 0)?),
+        44 => format!(
+            "{}({}){}",
+            stack_text(parser, 4)?.trim(),
+            stack_text(parser, 2)?,
+            stack_text(parser, 0)?
+        ),
+        47 => format!(
+            "{}({})",
+            stack_text(parser, 3)?.trim(),
+            stack_text(parser, 1)?
+        ),
+        53 => format!("base{}", stack_text(parser, 1)?),
+        56 => format!("this{}", stack_text(parser, 1)?),
+        59 => format!(" {} ", stack_text(parser, 0)?),
+        62 => format!("{}[{}]", stack_text(parser, 3)?, stack_text(parser, 1)?),
+        65 | 68 => stack_text(parser, 0)?,
+        71 => ";\n".to_owned(),
+        74 => " base ".to_owned(),
+        77 => " this ".to_owned(),
+        80 => format!(" this[{}]", stack_text(parser, 1)?),
+        83 => ":".to_owned(),
+        86 | 89 => format!(" new {}", stack_text(parser, 0)?),
+        92 => format!("({})", stack_text(parser, 1)?),
+        95 => format!("{{{}}}\n", stack_text(parser, 1)?),
+        98 => format!("[{}]", stack_text(parser, 1)?),
+        _ => String::new(),
+    })
+}
+
+#[derive(Clone, Debug)]
+pub struct GeneratedParserSymbol<const GENERATED: i32>(TOKEN);
+
+impl<const GENERATED: i32> GeneratedParserSymbol<GENERATED> {
+    pub fn new(parser: Parser) -> Result<Self, Error> {
+        let (name, number) = generated_parser_identity(GENERATED).ok_or(Error::Argument)?;
+        let text = generated_semantic_text(GENERATED, &parser)?;
+        Ok(Self(TOKEN::generated_with_parser(
+            parser, name, number, text,
+        )?))
+    }
+    pub fn yyname(&self) -> String {
+        self.0.yyname()
+    }
+    pub fn yynum(&self) -> i32 {
+        self.0.yynum()
+    }
+    pub fn yytext(&self) -> String {
+        self.0.yytext()
+    }
+}
+
+impl<const GENERATED: i32> Deref for GeneratedParserSymbol<GENERATED> {
+    type Target = TOKEN;
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}
+impl<const GENERATED: i32> DerefMut for GeneratedParserSymbol<GENERATED> {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.0
+    }
+}
+pub type ClassBody_1 = GeneratedParserSymbol<24>;
+pub type ClassBody_2 = GeneratedParserSymbol<25>;
+pub type ClassBody_2_1 = GeneratedParserSymbol<26>;
+pub type GStuff_1 = GeneratedParserSymbol<27>;
+pub type GStuff_2 = GeneratedParserSymbol<28>;
+pub type GStuff_2_1 = GeneratedParserSymbol<29>;
+pub type GStuff_3 = GeneratedParserSymbol<30>;
+pub type GStuff_4 = GeneratedParserSymbol<31>;
+pub type GStuff_4_1 = GeneratedParserSymbol<32>;
+pub type GStuff_5 = GeneratedParserSymbol<33>;
+pub type GStuff_6 = GeneratedParserSymbol<34>;
+pub type GStuff_6_1 = GeneratedParserSymbol<35>;
+pub type Stuff_1 = GeneratedParserSymbol<36>;
+pub type Stuff_2 = GeneratedParserSymbol<37>;
+pub type Stuff_2_1 = GeneratedParserSymbol<38>;
+pub type Stuff_3 = GeneratedParserSymbol<39>;
+pub type Stuff_4 = GeneratedParserSymbol<40>;
+pub type Stuff_4_1 = GeneratedParserSymbol<41>;
+pub type Cons_1 = GeneratedParserSymbol<42>;
+pub type Cons_2 = GeneratedParserSymbol<43>;
+pub type Cons_2_1 = GeneratedParserSymbol<44>;
+pub type Call_1 = GeneratedParserSymbol<45>;
+pub type Call_2 = GeneratedParserSymbol<46>;
+pub type Call_2_1 = GeneratedParserSymbol<47>;
+pub type BaseCall_1 = GeneratedParserSymbol<48>;
+pub type BaseCall_2 = GeneratedParserSymbol<49>;
+pub type BaseCall_2_1 = GeneratedParserSymbol<50>;
+pub type BaseCall_3 = GeneratedParserSymbol<51>;
+pub type BaseCall_4 = GeneratedParserSymbol<52>;
+pub type BaseCall_4_1 = GeneratedParserSymbol<53>;
+pub type BaseCall_5 = GeneratedParserSymbol<54>;
+pub type BaseCall_6 = GeneratedParserSymbol<55>;
+pub type BaseCall_6_1 = GeneratedParserSymbol<56>;
+pub type Name_1 = GeneratedParserSymbol<57>;
+pub type Name_2 = GeneratedParserSymbol<58>;
+pub type Name_2_1 = GeneratedParserSymbol<59>;
+pub type Name_3 = GeneratedParserSymbol<60>;
+pub type Name_4 = GeneratedParserSymbol<61>;
+pub type Name_4_1 = GeneratedParserSymbol<62>;
+pub type Item_1 = GeneratedParserSymbol<63>;
+pub type Item_2 = GeneratedParserSymbol<64>;
+pub type Item_2_1 = GeneratedParserSymbol<65>;
+pub type Item_3 = GeneratedParserSymbol<66>;
+pub type Item_4 = GeneratedParserSymbol<67>;
+pub type Item_4_1 = GeneratedParserSymbol<68>;
+pub type Item_5 = GeneratedParserSymbol<69>;
+pub type Item_6 = GeneratedParserSymbol<70>;
+pub type Item_6_1 = GeneratedParserSymbol<71>;
+pub type Item_7 = GeneratedParserSymbol<72>;
+pub type Item_8 = GeneratedParserSymbol<73>;
+pub type Item_8_1 = GeneratedParserSymbol<74>;
+pub type Item_9 = GeneratedParserSymbol<75>;
+pub type Item_10 = GeneratedParserSymbol<76>;
+pub type Item_10_1 = GeneratedParserSymbol<77>;
+pub type Item_11 = GeneratedParserSymbol<78>;
+pub type Item_12 = GeneratedParserSymbol<79>;
+pub type Item_12_1 = GeneratedParserSymbol<80>;
+pub type Item_13 = GeneratedParserSymbol<81>;
+pub type Item_14 = GeneratedParserSymbol<82>;
+pub type Item_14_1 = GeneratedParserSymbol<83>;
+pub type Item_15 = GeneratedParserSymbol<84>;
+pub type Item_16 = GeneratedParserSymbol<85>;
+pub type Item_16_1 = GeneratedParserSymbol<86>;
+pub type Item_17 = GeneratedParserSymbol<87>;
+pub type Item_18 = GeneratedParserSymbol<88>;
+pub type Item_18_1 = GeneratedParserSymbol<89>;
+pub type Item_19 = GeneratedParserSymbol<90>;
+pub type Item_20 = GeneratedParserSymbol<91>;
+pub type Item_20_1 = GeneratedParserSymbol<92>;
+pub type Item_21 = GeneratedParserSymbol<93>;
+pub type Item_22 = GeneratedParserSymbol<94>;
+pub type Item_22_1 = GeneratedParserSymbol<95>;
+pub type Item_23 = GeneratedParserSymbol<96>;
+pub type Item_24 = GeneratedParserSymbol<97>;
+pub type Item_24_1 = GeneratedParserSymbol<98>;
+pub type GStuff = GeneratedParserSymbol<27>;
+pub type Stuff = GeneratedParserSymbol<36>;
+pub type Item = GeneratedParserSymbol<63>;
+pub type ClassBody = GeneratedParserSymbol<24>;
+pub type Cons = GeneratedParserSymbol<42>;
+pub type Call = GeneratedParserSymbol<45>;
+pub type BaseCall = GeneratedParserSymbol<48>;
+pub type Name = GeneratedParserSymbol<57>;
+
+#[derive(Clone, Debug)]
+pub struct yycs0syntax(pub YyParser);
+impl yycs0syntax {
+    pub fn new() -> Result<Self, Error> {
+        Ok(Self(generated_parser()?))
+    }
+    pub fn action(&self, parser: Parser, symbol: SYMBOL, action: i32) -> Result<Object, Error> {
+        self.0.action(parser, symbol, action)
+    }
+    pub fn class_body_1_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(ClassBody_1::new(parser)?))
+    }
+    pub fn class_body_2_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(ClassBody_2::new(parser)?))
+    }
+    pub fn class_body_2_1_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(ClassBody_2_1::new(parser)?))
+    }
+    pub fn g_stuff_1_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(GStuff_1::new(parser)?))
+    }
+    pub fn g_stuff_2_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(GStuff_2::new(parser)?))
+    }
+    pub fn g_stuff_2_1_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(GStuff_2_1::new(parser)?))
+    }
+    pub fn g_stuff_3_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(GStuff_3::new(parser)?))
+    }
+    pub fn g_stuff_4_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(GStuff_4::new(parser)?))
+    }
+    pub fn g_stuff_4_1_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(GStuff_4_1::new(parser)?))
+    }
+    pub fn g_stuff_5_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(GStuff_5::new(parser)?))
+    }
+    pub fn g_stuff_6_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(GStuff_6::new(parser)?))
+    }
+    pub fn g_stuff_6_1_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(GStuff_6_1::new(parser)?))
+    }
+    pub fn stuff_1_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(Stuff_1::new(parser)?))
+    }
+    pub fn stuff_2_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(Stuff_2::new(parser)?))
+    }
+    pub fn stuff_2_1_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(Stuff_2_1::new(parser)?))
+    }
+    pub fn stuff_3_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(Stuff_3::new(parser)?))
+    }
+    pub fn stuff_4_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(Stuff_4::new(parser)?))
+    }
+    pub fn stuff_4_1_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(Stuff_4_1::new(parser)?))
+    }
+    pub fn cons_1_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(Cons_1::new(parser)?))
+    }
+    pub fn cons_2_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(Cons_2::new(parser)?))
+    }
+    pub fn cons_2_1_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(Cons_2_1::new(parser)?))
+    }
+    pub fn call_1_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(Call_1::new(parser)?))
+    }
+    pub fn call_2_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(Call_2::new(parser)?))
+    }
+    pub fn call_2_1_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(Call_2_1::new(parser)?))
+    }
+    pub fn base_call_1_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(BaseCall_1::new(parser)?))
+    }
+    pub fn base_call_2_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(BaseCall_2::new(parser)?))
+    }
+    pub fn base_call_2_1_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(BaseCall_2_1::new(parser)?))
+    }
+    pub fn base_call_3_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(BaseCall_3::new(parser)?))
+    }
+    pub fn base_call_4_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(BaseCall_4::new(parser)?))
+    }
+    pub fn base_call_4_1_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(BaseCall_4_1::new(parser)?))
+    }
+    pub fn base_call_5_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(BaseCall_5::new(parser)?))
+    }
+    pub fn base_call_6_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(BaseCall_6::new(parser)?))
+    }
+    pub fn base_call_6_1_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(BaseCall_6_1::new(parser)?))
+    }
+    pub fn name_1_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(Name_1::new(parser)?))
+    }
+    pub fn name_2_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(Name_2::new(parser)?))
+    }
+    pub fn name_2_1_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(Name_2_1::new(parser)?))
+    }
+    pub fn name_3_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(Name_3::new(parser)?))
+    }
+    pub fn name_4_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(Name_4::new(parser)?))
+    }
+    pub fn name_4_1_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(Name_4_1::new(parser)?))
+    }
+    pub fn item_1_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(Item_1::new(parser)?))
+    }
+    pub fn item_2_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(Item_2::new(parser)?))
+    }
+    pub fn item_2_1_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(Item_2_1::new(parser)?))
+    }
+    pub fn item_3_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(Item_3::new(parser)?))
+    }
+    pub fn item_4_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(Item_4::new(parser)?))
+    }
+    pub fn item_4_1_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(Item_4_1::new(parser)?))
+    }
+    pub fn item_5_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(Item_5::new(parser)?))
+    }
+    pub fn item_6_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(Item_6::new(parser)?))
+    }
+    pub fn item_6_1_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(Item_6_1::new(parser)?))
+    }
+    pub fn item_7_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(Item_7::new(parser)?))
+    }
+    pub fn item_8_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(Item_8::new(parser)?))
+    }
+    pub fn item_8_1_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(Item_8_1::new(parser)?))
+    }
+    pub fn item_9_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(Item_9::new(parser)?))
+    }
+    pub fn item_10_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(Item_10::new(parser)?))
+    }
+    pub fn item_10_1_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(Item_10_1::new(parser)?))
+    }
+    pub fn item_11_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(Item_11::new(parser)?))
+    }
+    pub fn item_12_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(Item_12::new(parser)?))
+    }
+    pub fn item_12_1_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(Item_12_1::new(parser)?))
+    }
+    pub fn item_13_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(Item_13::new(parser)?))
+    }
+    pub fn item_14_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(Item_14::new(parser)?))
+    }
+    pub fn item_14_1_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(Item_14_1::new(parser)?))
+    }
+    pub fn item_15_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(Item_15::new(parser)?))
+    }
+    pub fn item_16_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(Item_16::new(parser)?))
+    }
+    pub fn item_16_1_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(Item_16_1::new(parser)?))
+    }
+    pub fn item_17_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(Item_17::new(parser)?))
+    }
+    pub fn item_18_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(Item_18::new(parser)?))
+    }
+    pub fn item_18_1_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(Item_18_1::new(parser)?))
+    }
+    pub fn item_19_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(Item_19::new(parser)?))
+    }
+    pub fn item_20_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(Item_20::new(parser)?))
+    }
+    pub fn item_20_1_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(Item_20_1::new(parser)?))
+    }
+    pub fn item_21_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(Item_21::new(parser)?))
+    }
+    pub fn item_22_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(Item_22::new(parser)?))
+    }
+    pub fn item_22_1_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(Item_22_1::new(parser)?))
+    }
+    pub fn item_23_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(Item_23::new(parser)?))
+    }
+    pub fn item_24_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(Item_24::new(parser)?))
+    }
+    pub fn item_24_1_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(Item_24_1::new(parser)?))
+    }
+    pub fn g_stuff_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(GStuff::new(parser)?))
+    }
+    pub fn stuff_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(Stuff::new(parser)?))
+    }
+    pub fn item_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(Item::new(parser)?))
+    }
+    pub fn class_body_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(ClassBody::new(parser)?))
+    }
+    pub fn cons_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(Cons::new(parser)?))
+    }
+    pub fn call_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(Call::new(parser)?))
+    }
+    pub fn base_call_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(BaseCall::new(parser)?))
+    }
+    pub fn name_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(Name::new(parser)?))
+    }
+    pub fn error_factory(parser: Parser) -> Result<Object, Error> {
+        Ok(Object::opaque(crate::LslError::new_with_parser(parser)?))
+    }
+}
+impl Deref for yycs0syntax {
+    type Target = YyParser;
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}
+impl DerefMut for yycs0syntax {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.0
+    }
+}
+
+#[derive(Clone, Debug)]
+pub struct yycs0tokens(pub YyLexer);
+impl yycs0tokens {
+    pub fn new(error_handler: ErrorHandler) -> Result<Self, Error> {
+        Ok(Self(generated_lexer_with_handler(error_handler)?))
+    }
+    pub fn base_factory(lexer: Lexer) -> Result<Object, Error> {
+        Ok(Object::opaque(BASE::new(lexer)?))
+    }
+    pub fn this_factory(lexer: Lexer) -> Result<Object, Error> {
+        Ok(Object::opaque(THIS::new(lexer)?))
+    }
+    pub fn new_factory(lexer: Lexer) -> Result<Object, Error> {
+        Ok(Object::opaque(NEW::new(lexer)?))
+    }
+    pub fn id_factory(lexer: Lexer) -> Result<Object, Error> {
+        Ok(Object::opaque(ID::new(lexer)?))
+    }
+    pub fn any_factory(lexer: Lexer) -> Result<Object, Error> {
+        Ok(Object::opaque(ANY::new(lexer)?))
+    }
+    pub fn colon_factory(lexer: Lexer) -> Result<Object, Error> {
+        Ok(Object::opaque(COLON::new(lexer)?))
+    }
+    pub fn semicolon_factory(lexer: Lexer) -> Result<Object, Error> {
+        Ok(Object::opaque(SEMICOLON::new(lexer)?))
+    }
+    pub fn lbrace_factory(lexer: Lexer) -> Result<Object, Error> {
+        Ok(Object::opaque(LBRACE::new(lexer)?))
+    }
+    pub fn rbrace_factory(lexer: Lexer) -> Result<Object, Error> {
+        Ok(Object::opaque(RBRACE::new(lexer)?))
+    }
+    pub fn lparen_factory(lexer: Lexer) -> Result<Object, Error> {
+        Ok(Object::opaque(LPAREN::new(lexer)?))
+    }
+    pub fn rparen_factory(lexer: Lexer) -> Result<Object, Error> {
+        Ok(Object::opaque(RPAREN::new(lexer)?))
+    }
+    pub fn lbrack_factory(lexer: Lexer) -> Result<Object, Error> {
+        Ok(Object::opaque(LBRACK::new(lexer)?))
+    }
+    pub fn rbrack_factory(lexer: Lexer) -> Result<Object, Error> {
+        Ok(Object::opaque(RBRACK::new(lexer)?))
+    }
+    pub fn old_action(
+        &self,
+        lexer: Lexer,
+        yytext: &mut String,
+        action: i32,
+        reject: &mut bool,
+    ) -> Result<TOKEN, Error> {
+        if matches!(action, 8 | 55 | 74) {
+            match action {
+                8 => "yym.yy_begin",
+                55 => "((cs0tokens)yym)",
+                _ => "((cs0syntax)yyq)",
+            }
+            .clone_into(yytext);
+            return TOKEN::generated_with_lexer(lexer, "ANY", 7);
+        }
+        *reject = true;
+        Err(Error::InvalidOperation)
+    }
+}
+impl Deref for yycs0tokens {
+    type Target = YyLexer;
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}
+impl DerefMut for yycs0tokens {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.0
+    }
+}
+
+#[derive(Clone, Debug)]
+pub struct cs0tokens {
+    pub out: String,
+    pub lexer: Lexer,
+}
+impl cs0tokens {
+    pub fn new() -> Result<Self, Error> {
+        Self::new_with_yy_lexer(generated_lexer()?)
+    }
+    pub fn new_with_constructor() -> Result<Self, Error> {
+        Self::new()
+    }
+    pub fn new_with_error_handler(handler: ErrorHandler) -> Result<Self, Error> {
+        Self::new_with_yy_lexer(generated_lexer_with_handler(handler)?)
+    }
+    pub fn new_with_yy_lexer(tokens: YyLexer) -> Result<Self, Error> {
+        Ok(Self {
+            out: String::new(),
+            lexer: Lexer::new(tokens)?,
+        })
+    }
+}
+impl Deref for cs0tokens {
+    type Target = Lexer;
+    fn deref(&self) -> &Self::Target {
+        &self.lexer
+    }
+}
+impl DerefMut for cs0tokens {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.lexer
+    }
+}
+
+#[derive(Clone, Debug)]
+pub struct cs0syntax {
+    pub out: String,
+    pub cls: String,
+    pub par: String,
+    pub ctx: String,
+    pub defconseen: bool,
+    pub parser: Parser,
+}
+impl cs0syntax {
+    pub fn new() -> Result<Self, Error> {
+        Self::new_with_yy_parser(generated_parser()?)
+    }
+    pub fn new_with_constructor() -> Result<Self, Error> {
+        Self::new()
+    }
+    pub fn new_with_yy_parser(symbols: YyParser) -> Result<Self, Error> {
+        let lexer = Lexer::new(generated_lexer()?)?;
+        Self::from_parts(symbols, lexer)
+    }
+    pub fn new_with_yy_parser_error_handler(
+        mut symbols: YyParser,
+        handler: ErrorHandler,
+    ) -> Result<Self, Error> {
+        symbols.erh = handler.clone();
+        let lexer = Lexer::new(generated_lexer_with_handler(handler)?)?;
+        Self::from_parts(symbols, lexer)
+    }
+    fn from_parts(symbols: YyParser, lexer: Lexer) -> Result<Self, Error> {
+        Ok(Self {
+            out: String::new(),
+            cls: String::new(),
+            par: String::new(),
+            ctx: String::new(),
+            defconseen: false,
+            parser: Parser::new(symbols, lexer)?,
+        })
+    }
+}
+impl Deref for cs0syntax {
+    type Target = Parser;
+    fn deref(&self) -> &Self::Target {
+        &self.parser
+    }
+}
+impl DerefMut for cs0syntax {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.parser
+    }
+}

crates/libremetaverse-lsl-tools/src/lexer.rs

@@ -1483,6 +1483,21 @@ impl DfaAction {
 }
 
 /// Runtime deterministic finite automaton.
+#[derive(Clone, Debug)]
+struct RegexDfa {
+    source: String,
+    compiled: regex::Regex,
+    accept: DfaAccept,
+}
+
+impl PartialEq for RegexDfa {
+    fn eq(&self, other: &Self) -> bool {
+        self.source == other.source && self.accept == other.accept
+    }
+}
+
+impl Eq for RegexDfa {}
+
 #[derive(Clone, Debug, Eq, PartialEq)]
 pub struct Dfa {
     pub m_actions: Vec<DfaAction>,
@@ -1491,6 +1506,7 @@ pub struct Dfa {
     pub m_tok_class: String,
     states: Vec<DfaState>,
     start: usize,
+    regex: Option<RegexDfa>,
 }
 
 impl Default for Dfa {
@@ -1502,6 +1518,7 @@ impl Default for Dfa {
             m_tok_class: String::new(),
             states: vec![DfaState::default()],
             start: 0,
+            regex: None,
         }
     }
 }
@@ -1525,17 +1542,54 @@ impl Dfa {
         })
     }
 
-    pub fn new_with_nfa(_nfa: Nfa) -> Result<Self, Error> {
-        // The NFA-to-DFA generator is issue #83; the runtime never manufactures
-        // fake transitions from an opaque generated NFA.
-        Err(Error::InvalidOperation)
+    pub fn new_with_nfa(nfa: Nfa) -> Result<Self, Error> {
+        let name = if nfa.m_end.m_s_terminal.is_empty() {
+            "ANY".to_owned()
+        } else {
+            nfa.m_end.m_s_terminal.clone()
+        };
+        let accept = DfaAccept {
+            token: TokenDefinition::new(name, 7)?,
+            action: LexerAction::Emit,
+            action_number: 0,
+            reserved_words: None,
+        };
+        if let Some(source) = nfa.regex_source()? {
+            let compiled =
+                regex::Regex::new(&format!("^(?:{source})")).map_err(|_| Error::Parse {
+                    position: 0,
+                    context: "invalid NFA regular expression",
+                })?;
+            return Ok(Self {
+                regex: Some(RegexDfa {
+                    source,
+                    compiled,
+                    accept,
+                }),
+                ..Self::default()
+            });
+        }
+        Self::from_states(nfa.deterministic_states(accept)?, 0)
     }
 
     pub fn new_with_tokens_gen(_tokens: TokensGen) -> Result<Self, Error> {
-        Err(Error::InvalidOperation)
+        // The C# constructor creates the initial, unpopulated determinisation
+        // node. NFA-based construction is exposed separately by
+        // `new_with_nfa`; returning an empty native state here preserves that
+        // useful distinction without manufacturing an NFA transition.
+        Ok(Self::default())
     }
 
     fn longest_match(&self, units: &[u16], offset: usize) -> Option<(usize, DfaAccept)> {
+        if let Some(regex) = &self.regex {
+            let text = String::from_utf16(units.get(offset..)?).ok()?;
+            let found = regex.compiled.find(&text)?;
+            if found.start() != 0 || found.end() == 0 {
+                return None;
+            }
+            let length = text[..found.end()].encode_utf16().count();
+            return Some((length, regex.accept.clone()));
+        }
         let mut state_index = self.start;
         let mut cursor = offset;
         let mut accepted = self.states[state_index]
@@ -1587,6 +1641,15 @@ impl Dfa {
 
     #[must_use]
     pub fn stable_description(&self) -> String {
+        if let Some(regex) = &self.regex {
+            return format!(
+                "regex {:?} accept {} {} {}",
+                regex.source,
+                regex.accept.token.number,
+                regex.accept.token.name,
+                regex.accept.action_number
+            );
+        }
         let mut lines = Vec::new();
         for (state_index, state) in self.states.iter().enumerate() {
             if let Some(accept) = &state.accept {
@@ -2185,6 +2248,9 @@ impl SYMBOL {
     pub fn set_yylval(&mut self, value: Object) {
         self.m_dollar = value;
     }
+    pub(crate) fn raw_text(&self) -> String {
+        self.text.clone()
+    }
     #[must_use]
     pub fn yyname(&self) -> String {
         self.name.clone()
@@ -2215,6 +2281,30 @@ pub struct TOKEN {
 }
 
 impl TOKEN {
+    pub(crate) fn generated_with_lexer(
+        lexer: Lexer,
+        name: &str,
+        number: i32,
+    ) -> Result<Self, Error> {
+        let mut token = Self::new_with_lexer(lexer)?;
+        name.clone_into(&mut token.name);
+        token.number = number;
+        Ok(token)
+    }
+
+    pub(crate) fn generated_with_parser(
+        parser: Parser,
+        name: &str,
+        number: i32,
+        text: String,
+    ) -> Result<Self, Error> {
+        let mut token = Self::new_with_parser(parser)?;
+        name.clone_into(&mut token.name);
+        token.number = number;
+        token.text = text;
+        Ok(token)
+    }
+
     pub fn new_with_lexer(lexer: Lexer) -> Result<Self, Error> {
         let text = lexer.yytext.clone();
         Self::new_with_lexer_string(lexer, text)

crates/libremetaverse-lsl-tools/src/lib.rs

@@ -3,10 +3,13 @@
 extern crate self as libremetaverse_lsl_tools;
 
 mod generated;
+mod generated_tables;
+mod generator;
 mod lexer;
 mod parser;
 
 pub use generated::*;
+pub use generated_tables::*;
 pub use lexer::{
     CharacterMatcher, DfaAccept, DfaState, Diagnostic, DiagnosticCategory, DiagnosticSeverity,
     DotNetUnicodeCategory, InputEncoding, LexerAction, LexerIterator, MAX_SOURCE_UNITS,

crates/libremetaverse-lsl-tools/src/parser.rs

@@ -1006,6 +1006,10 @@ impl SymbolSet {
         }
     }
 
+    pub(crate) fn compatibility_empty() -> Self {
+        Self::empty()
+    }
+
     /// Creates an empty set independent of generator state.
     #[must_use]
     pub fn native() -> Self {
@@ -1646,6 +1650,10 @@ impl ParseState {
         }
     }
 
+    pub(crate) fn compatibility_numbered(state: i32) -> Self {
+        Self::numbered(state)
+    }
+
     pub fn new(symbols: SymbolsGen, accessing: CSymbol) -> Result<Self, Error> {
         Ok(Self {
             m_accessing_symbol: accessing,

crates/libremetaverse-lsl-tools/tests/generator_compat.rs

@@ -0,0 +1,341 @@
+//! Focused compatibility fixtures for issue #83's checked-in LSL generator.
+
+use std::collections::HashMap;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::{Arc, Mutex};
+
+use libremetaverse_lsl_tools::{
+    BASE, CSymbol, CSymbolSymType, Dfa, ErrorHandler, GENERATED_PARSER_DATA, ID,
+    LSL_GENERATOR_SERIALIZATION_VERSION, Lexer, Nfa, NfaNode, ObjectList,
+    ObjectListOListEnumerator, Path, Regex, SCreator, Serialiser, Sfactory, SymbolType, SymbolsGen,
+    TCreator, Tfactory, TokClassDef, TokensGen, generated_lexer, generated_parser,
+};
+use libremetaverse_types::compat::{Object, Utf16CodeUnit};
+
+struct SharedWriter(Arc<Mutex<Vec<u8>>>);
+
+impl std::io::Write for SharedWriter {
+    fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {
+        self.0
+            .lock()
+            .map_err(|_| std::io::Error::other("poisoned output"))?
+            .extend_from_slice(bytes);
+        Ok(bytes.len())
+    }
+
+    fn flush(&mut self) -> std::io::Result<()> {
+        Ok(())
+    }
+}
+
+fn capture<F>(function: F) -> Vec<u8>
+where
+    F: FnOnce(Box<dyn std::io::Write + Send>),
+{
+    let output = Arc::new(Mutex::new(Vec::new()));
+    function(Box::new(SharedWriter(output.clone())));
+    Arc::try_unwrap(output)
+        .expect("single output owner")
+        .into_inner()
+        .expect("output")
+}
+
+#[test]
+fn generated_lexer_preserves_tokens_reserved_words_and_eof() {
+    let mut lexer = Lexer::new(generated_lexer().expect("generated lexer")).expect("runtime");
+    lexer
+        .start_with_string("Widget(value;):base(new Child[])".to_owned())
+        .expect("source");
+    let mut tokens = Vec::new();
+    while let Some(token) = lexer.next_token().expect("lexing") {
+        tokens.push((token.yyname(), token.yytext(), token.yynum()));
+    }
+    assert_eq!(
+        tokens,
+        [
+            ("ID".to_owned(), "Widget".to_owned(), 6),
+            ("LPAREN".to_owned(), "(".to_owned(), 12),
+            ("ID".to_owned(), "value".to_owned(), 6),
+            ("SEMICOLON".to_owned(), ";".to_owned(), 9),
+            ("RPAREN".to_owned(), ")".to_owned(), 13),
+            ("COLON".to_owned(), ":".to_owned(), 8),
+            ("BASE".to_owned(), "base".to_owned(), 3),
+            ("LPAREN".to_owned(), "(".to_owned(), 12),
+            ("NEW".to_owned(), "new".to_owned(), 5),
+            ("ID".to_owned(), "Child".to_owned(), 6),
+            ("LBRACK".to_owned(), "[".to_owned(), 14),
+            ("RBRACK".to_owned(), "]".to_owned(), 15),
+            ("RPAREN".to_owned(), ")".to_owned(), 13),
+            ("EOF".to_owned(), "EOF".to_owned(), 2),
+        ]
+    );
+}
+
+#[test]
+fn generated_parser_accepts_the_reviewed_class_body_grammar() {
+    let table = generated_parser().expect("generated parser");
+    assert_eq!(table.arr, GENERATED_PARSER_DATA);
+    let lexer = Lexer::new(generated_lexer().expect("generated lexer")).expect("runtime");
+    let mut parser = libremetaverse_lsl_tools::Parser::new(table, lexer).expect("parser");
+    let result = parser
+        .parse_with_string("Widget(value;):base(new Child[])".to_owned())
+        .expect("class body");
+    assert_eq!(result.yyname(), "ClassBody");
+    assert_eq!(result.yynum(), 19);
+}
+
+#[test]
+fn parser_and_lexer_emission_is_byte_deterministic() {
+    let first_parser = capture(|output| {
+        generated_parser()
+            .expect("parser")
+            .emit(output)
+            .expect("emit");
+    });
+    let second_parser = capture(|output| {
+        generated_parser()
+            .expect("parser")
+            .emit(output)
+            .expect("emit");
+    });
+    assert_eq!(first_parser, second_parser);
+
+    let first_lexer = capture(|output| {
+        generated_lexer()
+            .expect("lexer")
+            .emit_dfa(output)
+            .expect("emit");
+    });
+    let second_lexer = capture(|output| {
+        generated_lexer()
+            .expect("lexer")
+            .emit_dfa(output)
+            .expect("emit");
+    });
+    assert_eq!(first_lexer, second_lexer);
+}
+
+#[test]
+fn generated_token_classes_have_live_identity_and_text() {
+    let mut lexer = Lexer::new(generated_lexer().expect("lexer")).expect("runtime");
+    lexer
+        .start_with_string("base name".to_owned())
+        .expect("source");
+    let base = lexer.next_token().expect("lexing").expect("base");
+    let mapped_base = BASE::new(lexer.clone()).expect("mapped base");
+    assert_eq!((base.yyname(), base.yynum()), ("BASE".to_owned(), 3));
+    assert_eq!(
+        (mapped_base.yyname(), mapped_base.yynum()),
+        ("BASE".to_owned(), 3)
+    );
+    let _ = lexer.next_token().expect("lexing").expect("name");
+    let mapped_id = ID::new(lexer).expect("mapped id");
+    assert_eq!(
+        (mapped_id.yyname(), mapped_id.yynum()),
+        ("ID".to_owned(), 6)
+    );
+}
+
+#[test]
+fn regex_and_nfa_build_a_functional_native_dfa() {
+    let handler = ErrorHandler::default();
+    let tokens = TokensGen::new(handler.clone()).expect("generator");
+    let regex = Regex::new(tokens.clone(), 0, "[A-Z][a-z]+".to_owned()).expect("regex");
+    assert!(
+        !regex
+            .match__with_char(Utf16CodeUnit(u16::from(b'Q')))
+            .expect("single-unit mismatch")
+    );
+    assert!(
+        !regex
+            .match__with_char(Utf16CodeUnit(u16::from(b'q')))
+            .expect("mismatch")
+    );
+    assert_eq!(
+        regex
+            .match__with_string("Zebra".to_owned())
+            .expect("prefix"),
+        5
+    );
+
+    let mut nfa = Nfa::new_with_tokens_gen_regex(tokens, regex).expect("nfa");
+    nfa.m_end.m_s_terminal = "CAPITAL".to_owned();
+    let dfa = Dfa::new_with_nfa(nfa).expect("dfa");
+    let mut table = libremetaverse_lsl_tools::YyLexer::new(handler).expect("table");
+    table.set_start_dfa("YYINITIAL", dfa).expect("start");
+    table.using_eof = true;
+    let mut lexer = Lexer::new(table).expect("runtime");
+    lexer.start_with_string("Rust".to_owned()).expect("source");
+    let token = lexer.next_token().expect("lexing").expect("token");
+    assert_eq!(
+        (token.yyname(), token.yytext()),
+        ("CAPITAL".to_owned(), "Rust".to_owned())
+    );
+
+    let tokens = TokensGen::new(ErrorHandler::default()).expect("manual generator");
+    let mut manual = Nfa::new_with_tokens_gen(tokens.clone()).expect("manual nfa");
+    manual.m_end.m_s_terminal = "PAIR".to_owned();
+    let middle = NfaNode::new(tokens).expect("middle state");
+    manual
+        .start
+        .add_arc(Utf16CodeUnit(u16::from(b'a')), middle.clone())
+        .expect("first arc");
+    middle
+        .add_arc(Utf16CodeUnit(u16::from(b'b')), manual.m_end.clone())
+        .expect("second arc");
+    let dfa = Dfa::new_with_nfa(manual).expect("determinised nfa");
+    let mut action = -1;
+    assert_eq!(
+        dfa.match_("ab".to_owned(), 0, &mut action).expect("pair"),
+        2
+    );
+}
+
+#[test]
+fn serialiser_round_trips_deterministically_with_versioning() {
+    let value = Object::Map(HashMap::from([
+        ("b".to_owned(), Object::Integer(7)),
+        (
+            "a".to_owned(),
+            Object::Array(vec![Object::Boolean(true), Object::String("λ".to_owned())]),
+        ),
+    ]));
+    let encoded = capture(|output| {
+        let serialiser = Serialiser::new_with_text_writer(output).expect("writer");
+        serialiser.version_check().expect("version");
+        serialiser.serialise(value.clone()).expect("serialise");
+    });
+    let encoded_again = capture(|output| {
+        let serialiser = Serialiser::new_with_text_writer(output).expect("writer");
+        serialiser.version_check().expect("version");
+        serialiser.serialise(value.clone()).expect("serialise");
+    });
+    assert_eq!(encoded, encoded_again);
+    let integers = String::from_utf8(encoded)
+        .expect("ASCII integers")
+        .split(',')
+        .filter(|value| !value.trim().is_empty())
+        .map(|value| value.trim().parse::<i32>().expect("integer"))
+        .collect::<Vec<_>>();
+    let decoder = Serialiser::new_with_int32_array(integers).expect("reader");
+    decoder
+        .version_check()
+        .expect(LSL_GENERATOR_SERIALIZATION_VERSION);
+    assert_eq!(decoder.deserialise().expect("deserialise"), value);
+}
+
+#[test]
+fn compatibility_enumerator_and_generator_state_are_live() {
+    let mut values = ObjectList::new().expect("list");
+    values.add(Object::Integer(1)).expect("first");
+    values
+        .add(Object::String("two".to_owned()))
+        .expect("second");
+    let enumerator = ObjectListOListEnumerator::new(values).expect("enumerator");
+    assert_eq!(enumerator.current(), Object::Undefined);
+    assert!(enumerator.move_next().expect("first"));
+    assert_eq!(enumerator.current(), Object::Integer(1));
+    assert!(enumerator.move_next().expect("second"));
+    assert_eq!(enumerator.current(), Object::String("two".to_owned()));
+    assert!(!enumerator.move_next().expect("end"));
+    enumerator.reset().expect("reset");
+    assert!(enumerator.move_next().expect("first again"));
+
+    let generator = TokensGen::new(ErrorHandler::default()).expect("generator");
+    assert_eq!(generator.new_state().expect("state one"), 1);
+    assert_eq!(generator.new_state().expect("state two"), 2);
+    assert_eq!(
+        generator
+            .fix_actions("yybegin(); yyl".to_owned())
+            .expect("rewritten action"),
+        "yym.yy_begin(); ((tokens)yym)"
+    );
+    assert!(Dfa::new_with_tokens_gen(generator).is_ok());
+}
+
+#[test]
+fn symbol_and_token_factories_execute_registered_rust_closures() {
+    let parser = libremetaverse_lsl_tools::Parser::new(
+        generated_parser().expect("parser table"),
+        Lexer::new(generated_lexer().expect("lexer table")).expect("lexer"),
+    )
+    .expect("parser");
+    let symbol_creator = SCreator::from_fn(|_| Ok(Object::String("symbol".to_owned())));
+    let registration = Sfactory::new(
+        generated_parser().expect("parser table"),
+        "GeneratedSymbol".to_owned(),
+        symbol_creator,
+    )
+    .expect("symbol registration");
+    assert_eq!(registration.name(), "GeneratedSymbol");
+    assert_eq!(
+        Sfactory::create("GeneratedSymbol_Derived".to_owned(), parser).expect("symbol factory"),
+        Object::String("symbol".to_owned())
+    );
+
+    let lexer = Lexer::new(generated_lexer().expect("lexer table")).expect("lexer");
+    let token_creator = TCreator::from_fn(|_| Ok(Object::Integer(83)));
+    let registration = Tfactory::new(
+        generated_lexer().expect("lexer table"),
+        "GeneratedToken".to_owned(),
+        token_creator,
+    )
+    .expect("token registration");
+    assert_eq!(registration.name(), "GeneratedToken");
+    assert_eq!(
+        Tfactory::create("GeneratedToken_Derived".to_owned(), lexer).expect("token factory"),
+        Object::Integer(83)
+    );
+
+    let callback_ran = Arc::new(AtomicBool::new(false));
+    let callback_state = callback_ran.clone();
+    let async_creator = TCreator::from_fn(|_| Ok(Object::String("async".to_owned())));
+    let result = async_creator
+        .begin_invoke(
+            Lexer::new(generated_lexer().expect("lexer table")).expect("lexer"),
+            Box::new(move |_| callback_state.store(true, Ordering::SeqCst)),
+            Object::Undefined,
+        )
+        .expect("begin invoke");
+    assert!(callback_ran.load(Ordering::SeqCst));
+    assert_eq!(
+        async_creator.end_invoke(result).expect("end invoke"),
+        Object::String("async".to_owned())
+    );
+}
+
+#[test]
+fn symbols_paths_and_token_class_definitions_keep_native_state() {
+    let symbols = SymbolsGen::new(ErrorHandler::default()).expect("symbols generator");
+    let declared = SymbolType::new_with_symbols_gen_string_boolean(
+        symbols.clone(),
+        "SemanticValue".to_owned(),
+        true,
+    )
+    .expect("symbol type");
+    assert_eq!(
+        declared
+            .find("SemanticValue".to_owned())
+            .expect("lookup")
+            .name,
+        "SemanticValue"
+    );
+
+    let identifier = CSymbol::native("ID", 6, CSymbolSymType::Terminal).expect("symbol");
+    let path = Path::new_with_c_symbol_array(vec![identifier.clone()]).expect("path");
+    assert_eq!(path.spelling()[0].m_yynum, 6);
+    assert!(!path.valid);
+    assert_eq!(path.top().m_state, 0);
+
+    let output = Arc::new(Mutex::new(Vec::new()));
+    let generator = libremetaverse_lsl_tools::GenBase::new(
+        ErrorHandler::default(),
+        Box::new(SharedWriter(output)),
+    )
+    .expect("generator base");
+    let definition = TokClassDef::new(generator, "IDENTIFIER".to_owned(), "TOKEN".to_owned())
+        .expect("token class");
+    assert_eq!(definition.m_name, "IDENTIFIER");
+    assert_eq!(definition.m_ref_token, "TOKEN");
+    assert_eq!(definition.m_yynum, 3);
+}