Generate native LSL parser tables (#83)

tools/check_milestone_10_issue_83.py

@@ -0,0 +1,204 @@
+#!/usr/bin/env python3
+"""Audit issue 83's deterministic native LSL parser/token generation."""
+
+from __future__ import annotations
+
+import json
+import re
+from pathlib import Path
+
+import generate_api_shims
+import generate_lsl_tables
+
+
+ROOT = Path(__file__).resolve().parents[1]
+CATALOG = ROOT / "api" / "public-api.json"
+INPUT = ROOT / "codegen" / "inputs" / "lsl_tools_grammar.json"
+TABLES = ROOT / "crates" / "libremetaverse-lsl-tools" / "src" / "generated_tables.rs"
+GENERATOR = ROOT / "crates" / "libremetaverse-lsl-tools" / "src" / "generator.rs"
+GENERATED = ROOT / "crates" / "libremetaverse-lsl-tools" / "src" / "generated.rs"
+TESTS = ROOT / "crates" / "libremetaverse-lsl-tools" / "tests" / "generator_compat.rs"
+DOC = ROOT / "crates" / "libremetaverse-lsl-tools" / "README.md"
+WORKFLOW = ROOT / ".gitea" / "workflows" / "rust-workspace.yml"
+SCRIPT = ROOT / "tools" / "generate_lsl_tables.py"
+STUB_RE = re.compile(
+    r"\b(?:not_implemented|unimplemented_api)\b|\b(?:todo|unimplemented)!\s*\("
+)
+
+
+def owned_types() -> dict[str, str]:
+    return {
+        doc_id: declaration
+        for doc_id, declaration in generate_api_shims.NATIVE_TYPES.items()
+        if declaration.startswith("crate::generated_tables::")
+        or declaration.startswith("crate::generator::")
+    }
+
+
+def catalog_counts(doc_ids: set[str]) -> tuple[int, int]:
+    catalog = json.loads(CATALOG.read_text())
+    assembly = next(
+        value
+        for value in catalog["assemblies"]
+        if value["identity"]["name"] == "LibreMetaverse.LslTools"
+    )
+    found: dict[str, int] = {}
+
+    def visit(value: object) -> None:
+        if isinstance(value, dict):
+            doc_id = value.get("doc_id")
+            if doc_id in doc_ids and str(doc_id).startswith("T:"):
+                found[str(doc_id)] = len(value.get("members", []))
+            for child in value.values():
+                visit(child)
+        elif isinstance(value, list):
+            for child in value:
+                visit(child)
+
+    visit(assembly["types"])
+    if set(found) != doc_ids:
+        missing = sorted(doc_ids - set(found))
+        raise SystemExit(f"issue 83 catalog types are missing: {missing}")
+    return len(found), sum(found.values())
+
+
+def generated_type_block(text: str, doc_id: str) -> str:
+    marker = f"/// C# type: `{doc_id}`."
+    start = text.find(marker)
+    if start < 0:
+        raise SystemExit(f"generated declaration is missing for {doc_id}")
+    next_type = text.find("/// C# type:", start + len(marker))
+    return text[start:] if next_type < 0 else text[start:next_type]
+
+
+def require_markers(path: Path, markers: tuple[str, ...]) -> None:
+    text = path.read_text()
+    missing = [marker for marker in markers if marker not in text]
+    if missing:
+        raise SystemExit(f"{path.name}: audit evidence missing: " + ", ".join(missing))
+
+
+def main() -> None:
+    types = owned_types()
+    type_count, member_count = catalog_counts(set(types))
+    if (type_count, member_count) != (120, 365):
+        raise SystemExit(
+            f"issue 83 expected 120 mapped types and 365 members, found "
+            f"{type_count} and {member_count}"
+        )
+
+    data = json.loads(INPUT.read_text())
+    if data["source"].get("license") != "BSD-3-Clause":
+        raise SystemExit("issue 83 reviewed grammar input has no BSD-3-Clause provenance")
+    if len(data["tokens"]) != 14 or len(data["nonterminals"]) != 8:
+        raise SystemExit("issue 83 grammar must contain 14 tokens and 8 nonterminals")
+    if len(data["productions"]) != 25:
+        raise SystemExit("issue 83 grammar must contain all 25 reviewed productions")
+    first = generate_lsl_tables.render(data).encode()
+    second = generate_lsl_tables.render(json.loads(INPUT.read_text())).encode()
+    if first != second or first != TABLES.read_bytes():
+        raise SystemExit("two clean LSL generations were not byte-identical")
+
+    for path in (TABLES, GENERATOR):
+        source = path.read_text()
+        if STUB_RE.search(source):
+            raise SystemExit(f"issue 83 owned Rust stubs remain in {path.name}")
+        if re.search(r"unsafe\s*\{|unsafe\s+impl|target_os\s*=\s*\"macos\"", source):
+            raise SystemExit(f"{path.name} contains unsafe or macOS-only code")
+    script = SCRIPT.read_text()
+    if re.search(r"\b(?:dotnet|csc|mcs)\b", script, re.IGNORECASE):
+        raise SystemExit("LSL regeneration depends on a C#/.NET process")
+
+    generated = GENERATED.read_text()
+    for doc_id in types:
+        if STUB_RE.search(generated_type_block(generated, doc_id)):
+            raise SystemExit(f"issue 83 generated stubs remain for {doc_id}")
+
+    require_markers(
+        TABLES,
+        (
+            "LSL_GENERATOR_SOURCE_COMMIT",
+            "LSL_GENERATOR_SOURCE_PARSER",
+            "LSL_GENERATOR_SOURCE_LEXER",
+            "LSL_GENERATOR_SOURCE_LICENSE",
+            "LSL_GENERATOR_SERIALIZATION_VERSION",
+            "GENERATED_PARSER_DATA",
+            "pub fn generated_parser()",
+            "pub fn generated_lexer_with_handler",
+            "pub struct GeneratedLexerToken",
+            "pub struct GeneratedParserSymbol",
+            "pub struct yycs0syntax",
+            "pub struct yycs0tokens",
+            "pub struct cs0syntax",
+            "pub struct cs0tokens",
+        ),
+    )
+    require_markers(
+        GENERATOR,
+        (
+            "pub struct Serialiser",
+            "pub struct GenBase",
+            "pub struct SymbolsGen",
+            "pub struct TokensGen",
+            "pub struct Regex",
+            "pub struct Nfa",
+            "pub struct Sfactory",
+            "pub struct Tfactory",
+            "pub struct TokClassDef",
+            "pub struct ObjectListOListEnumerator",
+            "pub struct AddToFunc",
+            "MAX_GENERATED_DFA_STATES",
+            "MAX_REGEX_UNITS",
+            "System.Char",
+        ),
+    )
+    require_markers(
+        TESTS,
+        (
+            "generated_lexer_preserves_tokens_reserved_words_and_eof",
+            "generated_parser_accepts_the_reviewed_class_body_grammar",
+            "parser_and_lexer_emission_is_byte_deterministic",
+            "generated_token_classes_have_live_identity_and_text",
+            "regex_and_nfa_build_a_functional_native_dfa",
+            "serialiser_round_trips_deterministically_with_versioning",
+            "compatibility_enumerator_and_generator_state_are_live",
+            "symbol_and_token_factories_execute_registered_rust_closures",
+            "symbols_paths_and_token_class_definitions_keep_native_state",
+        ),
+    )
+    if TESTS.read_text().count("#[test]") != 9:
+        raise SystemExit("issue 83 expected 9 focused generator fixtures")
+    require_markers(
+        DOC,
+        (
+            "Checked-in generator contract",
+            "25-production grammar",
+            "84 generated syntax",
+            "13 generated token classes",
+            "python3 tools/generate_lsl_tables.py --check",
+            "46 focused",
+            "ubuntu-latest",
+        ),
+    )
+    workflow = WORKFLOW.read_text()
+    require_markers(
+        WORKFLOW,
+        (
+            "python3 tools/check_milestone_10_issue_83.py",
+            "python3 tools/generate_lsl_tables.py --check",
+        ),
+    )
+    if "runs-on: ubuntu-latest" not in workflow or re.search(
+        r"runs-on:\s*(?:macos|windows)", workflow, re.IGNORECASE
+    ):
+        raise SystemExit("issue 83 workflow is not ubuntu-only")
+
+    print(
+        "issue 83 audit: 120 native mapped generator/table types and 365 members, "
+        "14 tokens, 8 nonterminals, 25 productions, two byte-identical generations, "
+        "9 focused fixtures, migration docs, and ubuntu-only CI are present"
+    )
+
+
+if __name__ == "__main__":
+    main()

tools/generate_api_shims.py

@@ -511,6 +511,39 @@ NATIVE_TYPES = {
     "T:LibreMetaverse.Rendering.LindenSkeleton": "crate::skeleton::LindenSkeleton",
 }
 
+# The cs0 parser/token classes are generated from the reviewed Rust grammar
+# input. Keep this list algorithmic so adding an alternative cannot leave a
+# hand-maintained compatibility shell behind.
+_LSL_GENERATED_TYPES = [
+    "ANY", "BASE", "COLON", "ID", "LBRACE", "LBRACK", "LPAREN", "NEW",
+    "RBRACE", "RBRACK", "RPAREN", "SEMICOLON", "THIS",
+]
+for _lhs, _alternatives in (
+    ("ClassBody", 1), ("GStuff", 3), ("Stuff", 2), ("Cons", 1),
+    ("Call", 1), ("BaseCall", 3), ("Name", 2), ("Item", 12),
+):
+    _LSL_GENERATED_TYPES.append(_lhs)
+    for _alternative in range(1, _alternatives + 1):
+        _LSL_GENERATED_TYPES.extend((
+            f"{_lhs}_{_alternative * 2 - 1}",
+            f"{_lhs}_{_alternative * 2}",
+            f"{_lhs}_{_alternative * 2}_1",
+        ))
+_LSL_GENERATED_TYPES.extend(("cs0syntax", "cs0tokens", "yycs0syntax", "yycs0tokens"))
+NATIVE_TYPES.update({
+    f"T:LibreMetaverse.LslTools.{name}": f"crate::generated_tables::{name}"
+    for name in _LSL_GENERATED_TYPES
+})
+for _name in (
+    "AddToFunc", "Builder", "Func", "GenBase", "LNode", "Nfa", "NfaNode",
+    "Path", "Regex", "Relation", "SCreator", "Serialiser", "Sfactory",
+    "SymbolType", "SymbolsGen", "TCreator", "Tfactory", "TokClassDef", "TokensGen",
+):
+    NATIVE_TYPES[f"T:LibreMetaverse.LslTools.{_name}"] = f"crate::generator::{_name}"
+NATIVE_TYPES["T:LibreMetaverse.LslTools.ObjectList.OListEnumerator"] = (
+    "crate::generator::ObjectListOListEnumerator"
+)
+
 # Native declarations whose cataloged method implementations are still emitted
 # below. This is used for static namespace types such as OSDParser: the type is
 # hand-written, while its fixed public methods remain generator-audited.

tools/generate_lsl_tables.py

@@ -0,0 +1,389 @@
+#!/usr/bin/env python3
+"""Generate native Rust LSL parser/token types from the reviewed grammar."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import subprocess
+from pathlib import Path
+
+import generate_rust_mapping as mapping
+
+
+ROOT = Path(__file__).resolve().parents[1]
+INPUT = ROOT / "codegen" / "inputs" / "lsl_tools_grammar.json"
+OUTPUT = (
+    ROOT / "crates" / "libremetaverse-lsl-tools" / "src" / "generated_tables.rs"
+)
+
+
+def rust_string(value: str) -> str:
+    return json.dumps(value, ensure_ascii=True)
+
+
+def format_rust(source: str) -> str:
+    """Apply the workspace's canonical Rust formatting deterministically."""
+    result = subprocess.run(
+        ["rustfmt", "--edition", "2024", "--emit", "stdout"],
+        input=source,
+        text=True,
+        capture_output=True,
+        check=False,
+    )
+    if result.returncode != 0:
+        raise SystemExit(f"rustfmt rejected generated LSL Rust:\n{result.stderr}")
+    return result.stdout
+
+
+def semantic_expression(action: str) -> str:
+    expressions = {
+        "ClassBody_2_1": "stack_text(parser, 1)?",
+        "GStuff_2_1": "String::new()",
+        "GStuff_4_1": "format!(\"{}{}\", stack_text(parser, 1)?, stack_text(parser, 0)?)",
+        "GStuff_6_1": "format!(\"{}{}\", stack_text(parser, 1)?, stack_text(parser, 0)?)",
+        "Stuff_2_1": "String::new()",
+        "Stuff_4_1": "format!(\"{}{}\", stack_text(parser, 1)?, stack_text(parser, 0)?)",
+        "Cons_2_1": "format!(\"{}({}){}\", stack_text(parser, 4)?.trim(), stack_text(parser, 2)?, stack_text(parser, 0)?)",
+        "Call_2_1": "format!(\"{}({})\", stack_text(parser, 3)?.trim(), stack_text(parser, 1)?)",
+        "BaseCall_2_1": "String::new()",
+        "BaseCall_4_1": "format!(\"base{}\", stack_text(parser, 1)?)",
+        "BaseCall_6_1": "format!(\"this{}\", stack_text(parser, 1)?)",
+        "Name_2_1": "format!(\" {} \", stack_text(parser, 0)?)",
+        "Name_4_1": "format!(\"{}[{}]\", stack_text(parser, 3)?, stack_text(parser, 1)?)",
+        "Item_2_1": "stack_text(parser, 0)?",
+        "Item_4_1": "stack_text(parser, 0)?",
+        "Item_6_1": "\";\\n\".to_owned()",
+        "Item_8_1": "\" base \".to_owned()",
+        "Item_10_1": "\" this \".to_owned()",
+        "Item_12_1": "format!(\" this[{}]\", stack_text(parser, 1)?)",
+        "Item_14_1": "\":\".to_owned()",
+        "Item_16_1": "format!(\" new {}\", stack_text(parser, 0)?)",
+        "Item_18_1": "format!(\" new {}\", stack_text(parser, 0)?)",
+        "Item_20_1": "format!(\"({})\", stack_text(parser, 1)?)",
+        "Item_22_1": "format!(\"{{{}}}\\n\", stack_text(parser, 1)?)",
+        "Item_24_1": "format!(\"[{}]\", stack_text(parser, 1)?)",
+    }
+    return expressions[action]
+
+
+def render(data: dict) -> str:
+    tokens = data["tokens"]
+    nonterminals = data["nonterminals"]
+    numbers = {name: number for name, number, *_ in tokens}
+    numbers.update({name: number for name, number in nonterminals})
+
+    generated = []
+    next_number = 24
+    alternatives: dict[str, int] = {}
+    for _, lhs, _, action in data["productions"]:
+        alternative = alternatives.get(lhs, 0) + 1
+        alternatives[lhs] = alternative
+        for name in (f"{lhs}_{alternative * 2 - 1}", f"{lhs}_{alternative * 2}", action):
+            generated.append((name, next_number, lhs, numbers[lhs]))
+            next_number += 1
+    assert next_number == 99
+
+    flat = [data["schema"], data["start"], data["eof"], len(tokens) + len(nonterminals)]
+    for name, number, *_ in tokens:
+        flat.extend((number, 1, len(name)))
+    for name, number in nonterminals:
+        flat.extend((number, 0, len(name)))
+    flat.append(len(data["productions"]))
+    for production, lhs, rhs, _ in data["productions"]:
+        flat.extend((production, numbers[lhs], len(rhs), *(numbers[name] for name in rhs)))
+
+    lines = [
+        "// @generated by tools/generate_lsl_tables.py; do not edit by hand.",
+        "#![allow(non_camel_case_types)]",
+        "#![allow(non_snake_case)]",
+        "#![allow(clippy::missing_errors_doc)]",
+        "#![allow(clippy::must_use_candidate)]",
+        "",
+        "use std::ops::{Deref, DerefMut};",
+        "",
+        "use libremetaverse_types::compat::{Object, UnicodeCategory};",
+        "",
+        "use crate::{",
+        "    CharacterMatcher, Dfa, DfaAccept, DfaState, Error, ErrorHandler, Grammar,",
+        "    Lexer, LexerAction, Parser, ResWds, SYMBOL, TOKEN, TokenDefinition,",
+        "    UnicodeClass, YyLexer, YyParser,",
+        "};",
+        "",
+        f"pub const LSL_GENERATOR_SOURCE_COMMIT: &str = {rust_string(data['source']['commit'])};",
+        f"pub const LSL_GENERATOR_SOURCE_PARSER: &str = {rust_string(data['source']['parser'])};",
+        f"pub const LSL_GENERATOR_SOURCE_LEXER: &str = {rust_string(data['source']['lexer'])};",
+        f"pub const LSL_GENERATOR_SOURCE_LICENSE: &str = {rust_string(data['source']['license'])};",
+        f"pub const LSL_GENERATOR_SERIALIZATION_VERSION: &str = {rust_string(data['source']['serialization_version'])};",
+        "pub const GENERATED_PARSER_DATA: &[i32] = &[",
+    ]
+    for index in range(0, len(flat), 16):
+        lines.append("    " + ", ".join(map(str, flat[index:index + 16])) + ",")
+    lines.extend(["];"])
+
+    lines.extend([
+        "",
+        "pub fn generated_symbol_name(number: i32) -> Option<&'static str> {",
+        "    match number {",
+    ])
+    for name, number, *_ in tokens:
+        lines.append(f"        {number} => Some({rust_string(name)}),")
+    for name, number in nonterminals:
+        lines.append(f"        {number} => Some({rust_string(name)}),")
+    lines.extend(["        _ => None,", "    }", "}"])
+
+    lines.extend([
+        "",
+        "pub fn generated_parser() -> Result<YyParser, Error> {",
+        f"    let mut grammar = Grammar::new({data['start']}, {data['eof']})?;",
+    ])
+    for name, number, *_ in tokens:
+        lines.append(f"    grammar.add_symbol({rust_string(name)}, {number}, true)?;")
+    for name, number in nonterminals:
+        lines.append(f"    grammar.add_symbol({rust_string(name)}, {number}, false)?;")
+    for _, lhs, rhs, _ in data["productions"]:
+        rhs_numbers = ", ".join(str(numbers[name]) for name in rhs)
+        lines.append(f"    grammar.add_production({numbers[lhs]}, vec![{rhs_numbers}])?;")
+    lines.extend([
+        "    let mut parser = grammar.build()?;",
+        "    parser.arr = GENERATED_PARSER_DATA.to_vec();",
+        "    Ok(parser)",
+        "}",
+        "",
+        "fn accept(name: &str, number: i32, action: LexerAction, action_number: i32) -> DfaAccept {",
+        "    DfaAccept {",
+        "        token: TokenDefinition { name: name.to_owned(), number },",
+        "        action,",
+        "        action_number,",
+        "        reserved_words: (name == \"ID\").then(|| \"ID\".to_owned()),",
+        "    }",
+        "}",
+        "",
+        "pub fn generated_lexer_with_handler(error_handler: ErrorHandler) -> Result<YyLexer, Error> {",
+        "    let punctuation = [",
+    ])
+    punctuation = [(name, number, literal) for name, number, literal in tokens if literal and len(literal) == 1]
+    for name, number, literal in punctuation:
+        lines.append(f"        ({ord(literal)}, {rust_string(name)}, {number}),")
+    lines.extend([
+        "    ];",
+        "    let punctuation_start = 3usize;",
+        "    let any_state = punctuation_start + punctuation.len();",
+        "    let mut states = vec![DfaState::default(); any_state + 1];",
+        "    states[0] = states[0].clone()",
+        "        .transition(CharacterMatcher::UnicodeClass(UnicodeClass::WhiteSpace), 1)",
+        "        .transition(CharacterMatcher::UnicodeClass(UnicodeClass::Letter), 2)",
+        "        .transition(CharacterMatcher::Exact(u16::from(b'_')), 2);",
+        "    for (offset, (unit, _, _)) in punctuation.iter().enumerate() {",
+        "        states[0].transitions.push((CharacterMatcher::Exact(*unit), punctuation_start + offset));",
+        "    }",
+        "    states[0].transitions.push((CharacterMatcher::Any, any_state));",
+        "    states[1] = states[1].clone()",
+        "        .transition(CharacterMatcher::UnicodeClass(UnicodeClass::WhiteSpace), 1)",
+        "        .accepting(accept(\"ANY\", 7, LexerAction::Skip, -1));",
+        "    states[2] = states[2].clone()",
+        "        .transition(CharacterMatcher::UnicodeClass(UnicodeClass::Letter), 2)",
+        "        .transition(CharacterMatcher::UnicodeClass(UnicodeClass::Number), 2)",
+        "        .transition(CharacterMatcher::Exact(u16::from(b'_')), 2)",
+        "        .accepting(accept(\"ID\", 6, LexerAction::Emit, 0));",
+        "    for (offset, (_, name, number)) in punctuation.iter().enumerate() {",
+        "        states[punctuation_start + offset] = DfaState::default()",
+        "            .accepting(accept(name, *number, LexerAction::Emit, *number));",
+        "    }",
+        "    states[any_state] = DfaState::default()",
+        "        .accepting(accept(\"ANY\", 7, LexerAction::Emit, 1));",
+        "    let mut lexer = YyLexer::new(error_handler)?;",
+        "    lexer.set_start_dfa(\"YYINITIAL\", Dfa::from_states(states, 0)?)?;",
+        "    lexer.set_reserved_words(\"ID\", ResWds::from_pairs([",
+        "        (\"base\", TokenDefinition::new(\"BASE\", 3)?),",
+        "        (\"this\", TokenDefinition::new(\"THIS\", 4)?),",
+        "        (\"new\", TokenDefinition::new(\"NEW\", 5)?),",
+        "    ], false)?)?;",
+        "    lexer.using_eof = true;",
+        "    for (name, number) in [",
+    ])
+    for name, number, *_ in tokens:
+        lines.append(f"        ({rust_string(name)}, {number}),")
+    lines.extend([
+        "    ] {",
+        "        lexer.tokens.0.insert(Object::String(name.to_owned()), Object::Integer(number));",
+        "    }",
+        "    let _ = lexer.using_cat(UnicodeCategory(0))?;",
+        "    Ok(lexer)",
+        "}",
+        "",
+        "pub fn generated_lexer() -> Result<YyLexer, Error> {",
+        "    generated_lexer_with_handler(ErrorHandler::new_with_boolean(false)?)",
+        "}",
+        "",
+        "#[derive(Clone, Debug)]",
+        "pub struct GeneratedLexerToken<const NUMBER: i32>(TOKEN);",
+        "",
+        "impl<const NUMBER: i32> GeneratedLexerToken<NUMBER> {",
+        "    pub fn new(lexer: Lexer) -> Result<Self, Error> {",
+        "        let name = generated_symbol_name(NUMBER).ok_or(Error::Argument)?;",
+        "        Ok(Self(TOKEN::generated_with_lexer(lexer, name, NUMBER)?))",
+        "    }",
+        "    pub fn yyname(&self) -> String { self.0.yyname() }",
+        "    pub const fn yynum(&self) -> i32 { NUMBER }",
+        "    pub fn yytext(&self) -> String { self.0.yytext() }",
+        "}",
+        "",
+        "impl<const NUMBER: i32> Deref for GeneratedLexerToken<NUMBER> {",
+        "    type Target = TOKEN;",
+        "    fn deref(&self) -> &Self::Target { &self.0 }",
+        "}",
+        "impl<const NUMBER: i32> DerefMut for GeneratedLexerToken<NUMBER> {",
+        "    fn deref_mut(&mut self) -> &mut Self::Target { &mut self.0 }",
+        "}",
+    ])
+    for name, number, *_ in tokens:
+        if name != "EOF":
+            lines.append(f"pub type {name} = GeneratedLexerToken<{number}>;")
+
+    lines.extend([
+        "",
+        "fn generated_parser_identity(generated: i32) -> Option<(&'static str, i32)> {",
+        "    match generated {",
+    ])
+    for lhs, number in nonterminals:
+        identities = [internal for _, internal, owner, _ in generated if owner == lhs]
+        pattern = (
+            str(identities[0])
+            if len(identities) == 1
+            else f"{identities[0]}..={identities[-1]}"
+        )
+        lines.append(f"        {pattern} => Some(({rust_string(lhs)}, {number})),")
+    lines.extend(["        _ => None,", "    }", "}"])
+    lines.extend([
+        "",
+        "fn stack_text(parser: &Parser, index: i32) -> Result<String, Error> {",
+        "    Ok(parser.stack_at(index)?.m_value.raw_text())",
+        "}",
+        "",
+        "fn generated_semantic_text(generated: i32, parser: &Parser) -> Result<String, Error> {",
+        "    Ok(match generated {",
+    ])
+    action_ids = {name: internal for name, internal, _, _ in generated}
+    expressions: dict[str, list[int]] = {}
+    for _, _, _, action in data["productions"]:
+        expression = semantic_expression(action)
+        if expression != "String::new()":
+            expressions.setdefault(expression, []).append(action_ids[action])
+    for expression, action_numbers in expressions.items():
+        pattern = " | ".join(str(number) for number in action_numbers)
+        lines.append(f"        {pattern} => {expression},")
+    lines.extend(["        _ => String::new(),", "    })", "}"])
+    lines.extend([
+        "",
+        "#[derive(Clone, Debug)]",
+        "pub struct GeneratedParserSymbol<const GENERATED: i32>(TOKEN);",
+        "",
+        "impl<const GENERATED: i32> GeneratedParserSymbol<GENERATED> {",
+        "    pub fn new(parser: Parser) -> Result<Self, Error> {",
+        "        let (name, number) = generated_parser_identity(GENERATED).ok_or(Error::Argument)?;",
+        "        let text = generated_semantic_text(GENERATED, &parser)?;",
+        "        Ok(Self(TOKEN::generated_with_parser(parser, name, number, text)?))",
+        "    }",
+        "    pub fn yyname(&self) -> String { self.0.yyname() }",
+        "    pub fn yynum(&self) -> i32 { self.0.yynum() }",
+        "    pub fn yytext(&self) -> String { self.0.yytext() }",
+        "}",
+        "",
+        "impl<const GENERATED: i32> Deref for GeneratedParserSymbol<GENERATED> {",
+        "    type Target = TOKEN;",
+        "    fn deref(&self) -> &Self::Target { &self.0 }",
+        "}",
+        "impl<const GENERATED: i32> DerefMut for GeneratedParserSymbol<GENERATED> {",
+        "    fn deref_mut(&mut self) -> &mut Self::Target { &mut self.0 }",
+        "}",
+    ])
+    base_ids = {}
+    for name, internal, lhs, _ in generated:
+        base_ids.setdefault(lhs, internal)
+        lines.append(f"pub type {name} = GeneratedParserSymbol<{internal}>;")
+    for name, _ in nonterminals:
+        lines.append(f"pub type {name} = GeneratedParserSymbol<{base_ids[name]}>;")
+
+    lines.extend([
+        "",
+        "#[derive(Clone, Debug)]",
+        "pub struct yycs0syntax(pub YyParser);",
+        "impl yycs0syntax {",
+        "    pub fn new() -> Result<Self, Error> { Ok(Self(generated_parser()?)) }",
+        "    pub fn action(&self, parser: Parser, symbol: SYMBOL, action: i32) -> Result<Object, Error> { self.0.action(parser, symbol, action) }",
+    ])
+    for name, internal, _, _ in generated:
+        factory = mapping.snake(f"{name}_factory")
+        lines.append(f"    pub fn {factory}(parser: Parser) -> Result<Object, Error> {{ Ok(Object::opaque({name}::new(parser)?)) }}")
+    for name, _ in nonterminals:
+        factory = mapping.snake(f"{name}_factory")
+        lines.append(f"    pub fn {factory}(parser: Parser) -> Result<Object, Error> {{ Ok(Object::opaque({name}::new(parser)?)) }}")
+    lines.extend([
+        "    pub fn error_factory(parser: Parser) -> Result<Object, Error> { Ok(Object::opaque(crate::LslError::new_with_parser(parser)?)) }",
+        "}",
+        "impl Deref for yycs0syntax { type Target = YyParser; fn deref(&self) -> &Self::Target { &self.0 } }",
+        "impl DerefMut for yycs0syntax { fn deref_mut(&mut self) -> &mut Self::Target { &mut self.0 } }",
+        "",
+        "#[derive(Clone, Debug)]",
+        "pub struct yycs0tokens(pub YyLexer);",
+        "impl yycs0tokens {",
+        "    pub fn new(error_handler: ErrorHandler) -> Result<Self, Error> { Ok(Self(generated_lexer_with_handler(error_handler)?)) }",
+    ])
+    for name, number, *_ in tokens:
+        if name != "EOF":
+            factory = mapping.snake(f"{name}_factory")
+            lines.append(f"    pub fn {factory}(lexer: Lexer) -> Result<Object, Error> {{ Ok(Object::opaque({name}::new(lexer)?)) }}")
+    lines.extend([
+        "    pub fn old_action(&self, lexer: Lexer, yytext: &mut String, action: i32, reject: &mut bool) -> Result<TOKEN, Error> {",
+        "        if matches!(action, 8 | 55 | 74) { match action { 8 => \"yym.yy_begin\", 55 => \"((cs0tokens)yym)\", _ => \"((cs0syntax)yyq)\" }.clone_into(yytext); return TOKEN::generated_with_lexer(lexer, \"ANY\", 7); }",
+        "        *reject = true;",
+        "        Err(Error::InvalidOperation)",
+        "    }",
+        "}",
+        "impl Deref for yycs0tokens { type Target = YyLexer; fn deref(&self) -> &Self::Target { &self.0 } }",
+        "impl DerefMut for yycs0tokens { fn deref_mut(&mut self) -> &mut Self::Target { &mut self.0 } }",
+        "",
+        "#[derive(Clone, Debug)]",
+        "pub struct cs0tokens { pub out: String, pub lexer: Lexer }",
+        "impl cs0tokens {",
+        "    pub fn new() -> Result<Self, Error> { Self::new_with_yy_lexer(generated_lexer()?) }",
+        "    pub fn new_with_constructor() -> Result<Self, Error> { Self::new() }",
+        "    pub fn new_with_error_handler(handler: ErrorHandler) -> Result<Self, Error> { Self::new_with_yy_lexer(generated_lexer_with_handler(handler)?) }",
+        "    pub fn new_with_yy_lexer(tokens: YyLexer) -> Result<Self, Error> { Ok(Self { out: String::new(), lexer: Lexer::new(tokens)? }) }",
+        "}",
+        "impl Deref for cs0tokens { type Target = Lexer; fn deref(&self) -> &Self::Target { &self.lexer } }",
+        "impl DerefMut for cs0tokens { fn deref_mut(&mut self) -> &mut Self::Target { &mut self.lexer } }",
+        "",
+        "#[derive(Clone, Debug)]",
+        "pub struct cs0syntax { pub out: String, pub cls: String, pub par: String, pub ctx: String, pub defconseen: bool, pub parser: Parser }",
+        "impl cs0syntax {",
+        "    pub fn new() -> Result<Self, Error> { Self::new_with_yy_parser(generated_parser()?) }",
+        "    pub fn new_with_constructor() -> Result<Self, Error> { Self::new() }",
+        "    pub fn new_with_yy_parser(symbols: YyParser) -> Result<Self, Error> { let lexer = Lexer::new(generated_lexer()?)?; Self::from_parts(symbols, lexer) }",
+        "    pub fn new_with_yy_parser_error_handler(mut symbols: YyParser, handler: ErrorHandler) -> Result<Self, Error> { symbols.erh = handler.clone(); let lexer = Lexer::new(generated_lexer_with_handler(handler)?)?; Self::from_parts(symbols, lexer) }",
+        "    fn from_parts(symbols: YyParser, lexer: Lexer) -> Result<Self, Error> { Ok(Self { out: String::new(), cls: String::new(), par: String::new(), ctx: String::new(), defconseen: false, parser: Parser::new(symbols, lexer)? }) }",
+        "}",
+        "impl Deref for cs0syntax { type Target = Parser; fn deref(&self) -> &Self::Target { &self.parser } }",
+        "impl DerefMut for cs0syntax { fn deref_mut(&mut self) -> &mut Self::Target { &mut self.parser } }",
+        "",
+    ])
+    return format_rust("\n".join(lines))
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--check", action="store_true")
+    args = parser.parse_args()
+    data = json.loads(INPUT.read_text())
+    output = render(data)
+    if args.check:
+        if not OUTPUT.exists() or OUTPUT.read_text() != output:
+            raise SystemExit(f"generated LSL table is stale: {OUTPUT}")
+        print("LSL parser/token generation is byte-identical")
+        return
+    OUTPUT.write_text(output)
+
+
+if __name__ == "__main__":
+    main()