MetaCrate/tools/generate_lsl_tables.py

#!/usr/bin/env python3
"""Generate native Rust LSL parser/token types from the reviewed grammar."""

from __future__ import annotations

import argparse
import json
import subprocess
from pathlib import Path

import generate_rust_mapping as mapping


ROOT = Path(__file__).resolve().parents[1]
INPUT = ROOT / "codegen" / "inputs" / "lsl_tools_grammar.json"
OUTPUT = (
    ROOT / "crates" / "libremetaverse-lsl-tools" / "src" / "generated_tables.rs"
)


def rust_string(value: str) -> str:
    return json.dumps(value, ensure_ascii=True)


def format_rust(source: str) -> str:
    """Apply the workspace's canonical Rust formatting deterministically."""
    result = subprocess.run(
        ["rustfmt", "--edition", "2024", "--emit", "stdout"],
        input=source,
        text=True,
        capture_output=True,
        check=False,
    )
    if result.returncode != 0:
        raise SystemExit(f"rustfmt rejected generated LSL Rust:\n{result.stderr}")
    return result.stdout


def semantic_expression(action: str) -> str:
    expressions = {
        "ClassBody_2_1": "stack_text(parser, 1)?",
        "GStuff_2_1": "String::new()",
        "GStuff_4_1": "format!(\"{}{}\", stack_text(parser, 1)?, stack_text(parser, 0)?)",
        "GStuff_6_1": "format!(\"{}{}\", stack_text(parser, 1)?, stack_text(parser, 0)?)",
        "Stuff_2_1": "String::new()",
        "Stuff_4_1": "format!(\"{}{}\", stack_text(parser, 1)?, stack_text(parser, 0)?)",
        "Cons_2_1": "format!(\"{}({}){}\", stack_text(parser, 4)?.trim(), stack_text(parser, 2)?, stack_text(parser, 0)?)",
        "Call_2_1": "format!(\"{}({})\", stack_text(parser, 3)?.trim(), stack_text(parser, 1)?)",
        "BaseCall_2_1": "String::new()",
        "BaseCall_4_1": "format!(\"base{}\", stack_text(parser, 1)?)",
        "BaseCall_6_1": "format!(\"this{}\", stack_text(parser, 1)?)",
        "Name_2_1": "format!(\" {} \", stack_text(parser, 0)?)",
        "Name_4_1": "format!(\"{}[{}]\", stack_text(parser, 3)?, stack_text(parser, 1)?)",
        "Item_2_1": "stack_text(parser, 0)?",
        "Item_4_1": "stack_text(parser, 0)?",
        "Item_6_1": "\";\\n\".to_owned()",
        "Item_8_1": "\" base \".to_owned()",
        "Item_10_1": "\" this \".to_owned()",
        "Item_12_1": "format!(\" this[{}]\", stack_text(parser, 1)?)",
        "Item_14_1": "\":\".to_owned()",
        "Item_16_1": "format!(\" new {}\", stack_text(parser, 0)?)",
        "Item_18_1": "format!(\" new {}\", stack_text(parser, 0)?)",
        "Item_20_1": "format!(\"({})\", stack_text(parser, 1)?)",
        "Item_22_1": "format!(\"{{{}}}\\n\", stack_text(parser, 1)?)",
        "Item_24_1": "format!(\"[{}]\", stack_text(parser, 1)?)",
    }
    return expressions[action]


def render(data: dict) -> str:
    tokens = data["tokens"]
    nonterminals = data["nonterminals"]
    numbers = {name: number for name, number, *_ in tokens}
    numbers.update({name: number for name, number in nonterminals})

    generated = []
    next_number = 24
    alternatives: dict[str, int] = {}
    for _, lhs, _, action in data["productions"]:
        alternative = alternatives.get(lhs, 0) + 1
        alternatives[lhs] = alternative
        for name in (f"{lhs}_{alternative * 2 - 1}", f"{lhs}_{alternative * 2}", action):
            generated.append((name, next_number, lhs, numbers[lhs]))
            next_number += 1
    assert next_number == 99

    flat = [data["schema"], data["start"], data["eof"], len(tokens) + len(nonterminals)]
    for name, number, *_ in tokens:
        flat.extend((number, 1, len(name)))
    for name, number in nonterminals:
        flat.extend((number, 0, len(name)))
    flat.append(len(data["productions"]))
    for production, lhs, rhs, _ in data["productions"]:
        flat.extend((production, numbers[lhs], len(rhs), *(numbers[name] for name in rhs)))

    lines = [
        "// @generated by tools/generate_lsl_tables.py; do not edit by hand.",
        "#![allow(non_camel_case_types)]",
        "#![allow(non_snake_case)]",
        "#![allow(clippy::missing_errors_doc)]",
        "#![allow(clippy::must_use_candidate)]",
        "",
        "use std::ops::{Deref, DerefMut};",
        "",
        "use libremetaverse_types::compat::{Object, UnicodeCategory};",
        "",
        "use crate::{",
        "    CharacterMatcher, Dfa, DfaAccept, DfaState, Error, ErrorHandler, Grammar,",
        "    Lexer, LexerAction, Parser, ResWds, SYMBOL, TOKEN, TokenDefinition,",
        "    UnicodeClass, YyLexer, YyParser,",
        "};",
        "",
        f"pub const LSL_GENERATOR_SOURCE_COMMIT: &str = {rust_string(data['source']['commit'])};",
        f"pub const LSL_GENERATOR_SOURCE_PARSER: &str = {rust_string(data['source']['parser'])};",
        f"pub const LSL_GENERATOR_SOURCE_LEXER: &str = {rust_string(data['source']['lexer'])};",
        f"pub const LSL_GENERATOR_SOURCE_LICENSE: &str = {rust_string(data['source']['license'])};",
        f"pub const LSL_GENERATOR_SERIALIZATION_VERSION: &str = {rust_string(data['source']['serialization_version'])};",
        "pub const GENERATED_PARSER_DATA: &[i32] = &[",
    ]
    for index in range(0, len(flat), 16):
        lines.append("    " + ", ".join(map(str, flat[index:index + 16])) + ",")
    lines.extend(["];"])

    lines.extend([
        "",
        "pub fn generated_symbol_name(number: i32) -> Option<&'static str> {",
        "    match number {",
    ])
    for name, number, *_ in tokens:
        lines.append(f"        {number} => Some({rust_string(name)}),")
    for name, number in nonterminals:
        lines.append(f"        {number} => Some({rust_string(name)}),")
    lines.extend(["        _ => None,", "    }", "}"])

    lines.extend([
        "",
        "pub fn generated_parser() -> Result<YyParser, Error> {",
        f"    let mut grammar = Grammar::new({data['start']}, {data['eof']})?;",
    ])
    for name, number, *_ in tokens:
        lines.append(f"    grammar.add_symbol({rust_string(name)}, {number}, true)?;")
    for name, number in nonterminals:
        lines.append(f"    grammar.add_symbol({rust_string(name)}, {number}, false)?;")
    for _, lhs, rhs, _ in data["productions"]:
        rhs_numbers = ", ".join(str(numbers[name]) for name in rhs)
        lines.append(f"    grammar.add_production({numbers[lhs]}, vec![{rhs_numbers}])?;")
    lines.extend([
        "    let mut parser = grammar.build()?;",
        "    parser.arr = GENERATED_PARSER_DATA.to_vec();",
        "    Ok(parser)",
        "}",
        "",
        "fn accept(name: &str, number: i32, action: LexerAction, action_number: i32) -> DfaAccept {",
        "    DfaAccept {",
        "        token: TokenDefinition { name: name.to_owned(), number },",
        "        action,",
        "        action_number,",
        "        reserved_words: (name == \"ID\").then(|| \"ID\".to_owned()),",
        "    }",
        "}",
        "",
        "pub fn generated_lexer_with_handler(error_handler: ErrorHandler) -> Result<YyLexer, Error> {",
        "    let punctuation = [",
    ])
    punctuation = [(name, number, literal) for name, number, literal in tokens if literal and len(literal) == 1]
    for name, number, literal in punctuation:
        lines.append(f"        ({ord(literal)}, {rust_string(name)}, {number}),")
    lines.extend([
        "    ];",
        "    let punctuation_start = 3usize;",
        "    let any_state = punctuation_start + punctuation.len();",
        "    let mut states = vec![DfaState::default(); any_state + 1];",
        "    states[0] = states[0].clone()",
        "        .transition(CharacterMatcher::UnicodeClass(UnicodeClass::WhiteSpace), 1)",
        "        .transition(CharacterMatcher::UnicodeClass(UnicodeClass::Letter), 2)",
        "        .transition(CharacterMatcher::Exact(u16::from(b'_')), 2);",
        "    for (offset, (unit, _, _)) in punctuation.iter().enumerate() {",
        "        states[0].transitions.push((CharacterMatcher::Exact(*unit), punctuation_start + offset));",
        "    }",
        "    states[0].transitions.push((CharacterMatcher::Any, any_state));",
        "    states[1] = states[1].clone()",
        "        .transition(CharacterMatcher::UnicodeClass(UnicodeClass::WhiteSpace), 1)",
        "        .accepting(accept(\"ANY\", 7, LexerAction::Skip, -1));",
        "    states[2] = states[2].clone()",
        "        .transition(CharacterMatcher::UnicodeClass(UnicodeClass::Letter), 2)",
        "        .transition(CharacterMatcher::UnicodeClass(UnicodeClass::Number), 2)",
        "        .transition(CharacterMatcher::Exact(u16::from(b'_')), 2)",
        "        .accepting(accept(\"ID\", 6, LexerAction::Emit, 0));",
        "    for (offset, (_, name, number)) in punctuation.iter().enumerate() {",
        "        states[punctuation_start + offset] = DfaState::default()",
        "            .accepting(accept(name, *number, LexerAction::Emit, *number));",
        "    }",
        "    states[any_state] = DfaState::default()",
        "        .accepting(accept(\"ANY\", 7, LexerAction::Emit, 1));",
        "    let mut lexer = YyLexer::new(error_handler)?;",
        "    lexer.set_start_dfa(\"YYINITIAL\", Dfa::from_states(states, 0)?)?;",
        "    lexer.set_reserved_words(\"ID\", ResWds::from_pairs([",
        "        (\"base\", TokenDefinition::new(\"BASE\", 3)?),",
        "        (\"this\", TokenDefinition::new(\"THIS\", 4)?),",
        "        (\"new\", TokenDefinition::new(\"NEW\", 5)?),",
        "    ], false)?)?;",
        "    lexer.using_eof = true;",
        "    for (name, number) in [",
    ])
    for name, number, *_ in tokens:
        lines.append(f"        ({rust_string(name)}, {number}),")
    lines.extend([
        "    ] {",
        "        lexer.tokens.0.insert(Object::String(name.to_owned()), Object::Integer(number));",
        "    }",
        "    let _ = lexer.using_cat(UnicodeCategory(0))?;",
        "    Ok(lexer)",
        "}",
        "",
        "pub fn generated_lexer() -> Result<YyLexer, Error> {",
        "    generated_lexer_with_handler(ErrorHandler::new_with_boolean(false)?)",
        "}",
        "",
        "#[derive(Clone, Debug)]",
        "pub struct GeneratedLexerToken<const NUMBER: i32>(TOKEN);",
        "",
        "impl<const NUMBER: i32> GeneratedLexerToken<NUMBER> {",
        "    pub fn new(lexer: Lexer) -> Result<Self, Error> {",
        "        let name = generated_symbol_name(NUMBER).ok_or(Error::Argument)?;",
        "        Ok(Self(TOKEN::generated_with_lexer(lexer, name, NUMBER)?))",
        "    }",
        "    pub fn yyname(&self) -> String { self.0.yyname() }",
        "    pub const fn yynum(&self) -> i32 { NUMBER }",
        "    pub fn yytext(&self) -> String { self.0.yytext() }",
        "}",
        "",
        "impl<const NUMBER: i32> Deref for GeneratedLexerToken<NUMBER> {",
        "    type Target = TOKEN;",
        "    fn deref(&self) -> &Self::Target { &self.0 }",
        "}",
        "impl<const NUMBER: i32> DerefMut for GeneratedLexerToken<NUMBER> {",
        "    fn deref_mut(&mut self) -> &mut Self::Target { &mut self.0 }",
        "}",
    ])
    for name, number, *_ in tokens:
        if name != "EOF":
            lines.append(f"pub type {name} = GeneratedLexerToken<{number}>;")

    lines.extend([
        "",
        "fn generated_parser_identity(generated: i32) -> Option<(&'static str, i32)> {",
        "    match generated {",
    ])
    for lhs, number in nonterminals:
        identities = [internal for _, internal, owner, _ in generated if owner == lhs]
        pattern = (
            str(identities[0])
            if len(identities) == 1
            else f"{identities[0]}..={identities[-1]}"
        )
        lines.append(f"        {pattern} => Some(({rust_string(lhs)}, {number})),")
    lines.extend(["        _ => None,", "    }", "}"])
    lines.extend([
        "",
        "fn stack_text(parser: &Parser, index: i32) -> Result<String, Error> {",
        "    Ok(parser.stack_at(index)?.m_value.raw_text())",
        "}",
        "",
        "fn generated_semantic_text(generated: i32, parser: &Parser) -> Result<String, Error> {",
        "    Ok(match generated {",
    ])
    action_ids = {name: internal for name, internal, _, _ in generated}
    expressions: dict[str, list[int]] = {}
    for _, _, _, action in data["productions"]:
        expression = semantic_expression(action)
        if expression != "String::new()":
            expressions.setdefault(expression, []).append(action_ids[action])
    for expression, action_numbers in expressions.items():
        pattern = " | ".join(str(number) for number in action_numbers)
        lines.append(f"        {pattern} => {expression},")
    lines.extend(["        _ => String::new(),", "    })", "}"])
    lines.extend([
        "",
        "#[derive(Clone, Debug)]",
        "pub struct GeneratedParserSymbol<const GENERATED: i32>(TOKEN);",
        "",
        "impl<const GENERATED: i32> GeneratedParserSymbol<GENERATED> {",
        "    pub fn new(parser: Parser) -> Result<Self, Error> {",
        "        let (name, number) = generated_parser_identity(GENERATED).ok_or(Error::Argument)?;",
        "        let text = generated_semantic_text(GENERATED, &parser)?;",
        "        Ok(Self(TOKEN::generated_with_parser(parser, name, number, text)?))",
        "    }",
        "    pub fn yyname(&self) -> String { self.0.yyname() }",
        "    pub fn yynum(&self) -> i32 { self.0.yynum() }",
        "    pub fn yytext(&self) -> String { self.0.yytext() }",
        "}",
        "",
        "impl<const GENERATED: i32> Deref for GeneratedParserSymbol<GENERATED> {",
        "    type Target = TOKEN;",
        "    fn deref(&self) -> &Self::Target { &self.0 }",
        "}",
        "impl<const GENERATED: i32> DerefMut for GeneratedParserSymbol<GENERATED> {",
        "    fn deref_mut(&mut self) -> &mut Self::Target { &mut self.0 }",
        "}",
    ])
    base_ids = {}
    for name, internal, lhs, _ in generated:
        base_ids.setdefault(lhs, internal)
        lines.append(f"pub type {name} = GeneratedParserSymbol<{internal}>;")
    for name, _ in nonterminals:
        lines.append(f"pub type {name} = GeneratedParserSymbol<{base_ids[name]}>;")

    lines.extend([
        "",
        "#[derive(Clone, Debug)]",
        "pub struct yycs0syntax(pub YyParser);",
        "impl yycs0syntax {",
        "    pub fn new() -> Result<Self, Error> { Ok(Self(generated_parser()?)) }",
        "    pub fn action(&self, parser: Parser, symbol: SYMBOL, action: i32) -> Result<Object, Error> { self.0.action(parser, symbol, action) }",
    ])
    for name, internal, _, _ in generated:
        factory = mapping.snake(f"{name}_factory")
        lines.append(f"    pub fn {factory}(parser: Parser) -> Result<Object, Error> {{ Ok(Object::opaque({name}::new(parser)?)) }}")
    for name, _ in nonterminals:
        factory = mapping.snake(f"{name}_factory")
        lines.append(f"    pub fn {factory}(parser: Parser) -> Result<Object, Error> {{ Ok(Object::opaque({name}::new(parser)?)) }}")
    lines.extend([
        "    pub fn error_factory(parser: Parser) -> Result<Object, Error> { Ok(Object::opaque(crate::LslError::new_with_parser(parser)?)) }",
        "}",
        "impl Deref for yycs0syntax { type Target = YyParser; fn deref(&self) -> &Self::Target { &self.0 } }",
        "impl DerefMut for yycs0syntax { fn deref_mut(&mut self) -> &mut Self::Target { &mut self.0 } }",
        "",
        "#[derive(Clone, Debug)]",
        "pub struct yycs0tokens(pub YyLexer);",
        "impl yycs0tokens {",
        "    pub fn new(error_handler: ErrorHandler) -> Result<Self, Error> { Ok(Self(generated_lexer_with_handler(error_handler)?)) }",
    ])
    for name, number, *_ in tokens:
        if name != "EOF":
            factory = mapping.snake(f"{name}_factory")
            lines.append(f"    pub fn {factory}(lexer: Lexer) -> Result<Object, Error> {{ Ok(Object::opaque({name}::new(lexer)?)) }}")
    lines.extend([
        "    pub fn old_action(&self, lexer: Lexer, yytext: &mut String, action: i32, reject: &mut bool) -> Result<TOKEN, Error> {",
        "        if matches!(action, 8 | 55 | 74) { match action { 8 => \"yym.yy_begin\", 55 => \"((cs0tokens)yym)\", _ => \"((cs0syntax)yyq)\" }.clone_into(yytext); return TOKEN::generated_with_lexer(lexer, \"ANY\", 7); }",
        "        *reject = true;",
        "        Err(Error::InvalidOperation)",
        "    }",
        "}",
        "impl Deref for yycs0tokens { type Target = YyLexer; fn deref(&self) -> &Self::Target { &self.0 } }",
        "impl DerefMut for yycs0tokens { fn deref_mut(&mut self) -> &mut Self::Target { &mut self.0 } }",
        "",
        "#[derive(Clone, Debug)]",
        "pub struct cs0tokens { pub out: String, pub lexer: Lexer }",
        "impl cs0tokens {",
        "    pub fn new() -> Result<Self, Error> { Self::new_with_yy_lexer(generated_lexer()?) }",
        "    pub fn new_with_constructor() -> Result<Self, Error> { Self::new() }",
        "    pub fn new_with_error_handler(handler: ErrorHandler) -> Result<Self, Error> { Self::new_with_yy_lexer(generated_lexer_with_handler(handler)?) }",
        "    pub fn new_with_yy_lexer(tokens: YyLexer) -> Result<Self, Error> { Ok(Self { out: String::new(), lexer: Lexer::new(tokens)? }) }",
        "}",
        "impl Deref for cs0tokens { type Target = Lexer; fn deref(&self) -> &Self::Target { &self.lexer } }",
        "impl DerefMut for cs0tokens { fn deref_mut(&mut self) -> &mut Self::Target { &mut self.lexer } }",
        "",
        "#[derive(Clone, Debug)]",
        "pub struct cs0syntax { pub out: String, pub cls: String, pub par: String, pub ctx: String, pub defconseen: bool, pub parser: Parser }",
        "impl cs0syntax {",
        "    pub fn new() -> Result<Self, Error> { Self::new_with_yy_parser(generated_parser()?) }",
        "    pub fn new_with_constructor() -> Result<Self, Error> { Self::new() }",
        "    pub fn new_with_yy_parser(symbols: YyParser) -> Result<Self, Error> { let lexer = Lexer::new(generated_lexer()?)?; Self::from_parts(symbols, lexer) }",
        "    pub fn new_with_yy_parser_error_handler(mut symbols: YyParser, handler: ErrorHandler) -> Result<Self, Error> { symbols.erh = handler.clone(); let lexer = Lexer::new(generated_lexer_with_handler(handler)?)?; Self::from_parts(symbols, lexer) }",
        "    fn from_parts(symbols: YyParser, lexer: Lexer) -> Result<Self, Error> { Ok(Self { out: String::new(), cls: String::new(), par: String::new(), ctx: String::new(), defconseen: false, parser: Parser::new(symbols, lexer)? }) }",
        "}",
        "impl Deref for cs0syntax { type Target = Parser; fn deref(&self) -> &Self::Target { &self.parser } }",
        "impl DerefMut for cs0syntax { fn deref_mut(&mut self) -> &mut Self::Target { &mut self.parser } }",
        "",
    ])
    return format_rust("\n".join(lines))


def main() -> None:
    parser = argparse.ArgumentParser()
    parser.add_argument("--check", action="store_true")
    args = parser.parse_args()
    data = json.loads(INPUT.read_text())
    output = render(data)
    if args.check:
        if not OUTPUT.exists() or OUTPUT.read_text() != output:
            raise SystemExit(f"generated LSL table is stale: {OUTPUT}")
        print("LSL parser/token generation is byte-identical")
        return
    OUTPUT.write_text(output)


if __name__ == "__main__":
    main()