#!/usr/bin/env python3 """Generate native Rust LSL parser/token types from the reviewed grammar.""" from __future__ import annotations import argparse import json import subprocess from pathlib import Path import generate_rust_mapping as mapping ROOT = Path(__file__).resolve().parents[1] INPUT = ROOT / "codegen" / "inputs" / "lsl_tools_grammar.json" OUTPUT = ( ROOT / "crates" / "metacrate-lsl-tools" / "src" / "generated_tables.rs" ) def rust_string(value: str) -> str: return json.dumps(value, ensure_ascii=True) def format_rust(source: str) -> str: """Apply the workspace's canonical Rust formatting deterministically.""" result = subprocess.run( ["rustfmt", "--edition", "2024", "--emit", "stdout"], input=source, text=True, capture_output=True, check=False, ) if result.returncode != 0: raise SystemExit(f"rustfmt rejected generated LSL Rust:\n{result.stderr}") return result.stdout def semantic_expression(action: str) -> str: expressions = { "ClassBody_2_1": "stack_text(parser, 1)?", "GStuff_2_1": "String::new()", "GStuff_4_1": "format!(\"{}{}\", stack_text(parser, 1)?, stack_text(parser, 0)?)", "GStuff_6_1": "format!(\"{}{}\", stack_text(parser, 1)?, stack_text(parser, 0)?)", "Stuff_2_1": "String::new()", "Stuff_4_1": "format!(\"{}{}\", stack_text(parser, 1)?, stack_text(parser, 0)?)", "Cons_2_1": "format!(\"{}({}){}\", stack_text(parser, 4)?.trim(), stack_text(parser, 2)?, stack_text(parser, 0)?)", "Call_2_1": "format!(\"{}({})\", stack_text(parser, 3)?.trim(), stack_text(parser, 1)?)", "BaseCall_2_1": "String::new()", "BaseCall_4_1": "format!(\"base{}\", stack_text(parser, 1)?)", "BaseCall_6_1": "format!(\"this{}\", stack_text(parser, 1)?)", "Name_2_1": "format!(\" {} \", stack_text(parser, 0)?)", "Name_4_1": "format!(\"{}[{}]\", stack_text(parser, 3)?, stack_text(parser, 1)?)", "Item_2_1": "stack_text(parser, 0)?", "Item_4_1": "stack_text(parser, 0)?", "Item_6_1": "\";\\n\".to_owned()", "Item_8_1": "\" base \".to_owned()", "Item_10_1": "\" this \".to_owned()", "Item_12_1": "format!(\" this[{}]\", stack_text(parser, 1)?)", "Item_14_1": "\":\".to_owned()", "Item_16_1": "format!(\" new {}\", stack_text(parser, 0)?)", "Item_18_1": "format!(\" new {}\", stack_text(parser, 0)?)", "Item_20_1": "format!(\"({})\", stack_text(parser, 1)?)", "Item_22_1": "format!(\"{{{}}}\\n\", stack_text(parser, 1)?)", "Item_24_1": "format!(\"[{}]\", stack_text(parser, 1)?)", } return expressions[action] def render(data: dict) -> str: tokens = data["tokens"] nonterminals = data["nonterminals"] numbers = {name: number for name, number, *_ in tokens} numbers.update({name: number for name, number in nonterminals}) generated = [] next_number = 24 alternatives: dict[str, int] = {} for _, lhs, _, action in data["productions"]: alternative = alternatives.get(lhs, 0) + 1 alternatives[lhs] = alternative for name in (f"{lhs}_{alternative * 2 - 1}", f"{lhs}_{alternative * 2}", action): generated.append((name, next_number, lhs, numbers[lhs])) next_number += 1 assert next_number == 99 flat = [data["schema"], data["start"], data["eof"], len(tokens) + len(nonterminals)] for name, number, *_ in tokens: flat.extend((number, 1, len(name))) for name, number in nonterminals: flat.extend((number, 0, len(name))) flat.append(len(data["productions"])) for production, lhs, rhs, _ in data["productions"]: flat.extend((production, numbers[lhs], len(rhs), *(numbers[name] for name in rhs))) lines = [ "// @generated by tools/generate_lsl_tables.py; do not edit by hand.", "#![allow(non_camel_case_types)]", "#![allow(non_snake_case)]", "#![allow(clippy::missing_errors_doc)]", "#![allow(clippy::must_use_candidate)]", "", "use std::ops::{Deref, DerefMut};", "", "use libremetaverse_types::compat::{Object, UnicodeCategory};", "", "use crate::{", " CharacterMatcher, Dfa, DfaAccept, DfaState, Error, ErrorHandler, Grammar,", " Lexer, LexerAction, Parser, ResWds, SYMBOL, TOKEN, TokenDefinition,", " UnicodeClass, YyLexer, YyParser,", "};", "", f"pub const LSL_GENERATOR_SOURCE_COMMIT: &str = {rust_string(data['source']['commit'])};", f"pub const LSL_GENERATOR_SOURCE_PARSER: &str = {rust_string(data['source']['parser'])};", f"pub const LSL_GENERATOR_SOURCE_LEXER: &str = {rust_string(data['source']['lexer'])};", f"pub const LSL_GENERATOR_SOURCE_LICENSE: &str = {rust_string(data['source']['license'])};", f"pub const LSL_GENERATOR_SERIALIZATION_VERSION: &str = {rust_string(data['source']['serialization_version'])};", "pub const GENERATED_PARSER_DATA: &[i32] = &[", ] for index in range(0, len(flat), 16): lines.append(" " + ", ".join(map(str, flat[index:index + 16])) + ",") lines.extend(["];"]) lines.extend([ "", "pub fn generated_symbol_name(number: i32) -> Option<&'static str> {", " match number {", ]) for name, number, *_ in tokens: lines.append(f" {number} => Some({rust_string(name)}),") for name, number in nonterminals: lines.append(f" {number} => Some({rust_string(name)}),") lines.extend([" _ => None,", " }", "}"]) lines.extend([ "", "pub fn generated_parser() -> Result {", f" let mut grammar = Grammar::new({data['start']}, {data['eof']})?;", ]) for name, number, *_ in tokens: lines.append(f" grammar.add_symbol({rust_string(name)}, {number}, true)?;") for name, number in nonterminals: lines.append(f" grammar.add_symbol({rust_string(name)}, {number}, false)?;") for _, lhs, rhs, _ in data["productions"]: rhs_numbers = ", ".join(str(numbers[name]) for name in rhs) lines.append(f" grammar.add_production({numbers[lhs]}, vec![{rhs_numbers}])?;") lines.extend([ " let mut parser = grammar.build()?;", " parser.arr = GENERATED_PARSER_DATA.to_vec();", " Ok(parser)", "}", "", "fn accept(name: &str, number: i32, action: LexerAction, action_number: i32) -> DfaAccept {", " DfaAccept {", " token: TokenDefinition { name: name.to_owned(), number },", " action,", " action_number,", " reserved_words: (name == \"ID\").then(|| \"ID\".to_owned()),", " }", "}", "", "pub fn generated_lexer_with_handler(error_handler: ErrorHandler) -> Result {", " let punctuation = [", ]) punctuation = [(name, number, literal) for name, number, literal in tokens if literal and len(literal) == 1] for name, number, literal in punctuation: lines.append(f" ({ord(literal)}, {rust_string(name)}, {number}),") lines.extend([ " ];", " let punctuation_start = 3usize;", " let any_state = punctuation_start + punctuation.len();", " let mut states = vec![DfaState::default(); any_state + 1];", " states[0] = states[0].clone()", " .transition(CharacterMatcher::UnicodeClass(UnicodeClass::WhiteSpace), 1)", " .transition(CharacterMatcher::UnicodeClass(UnicodeClass::Letter), 2)", " .transition(CharacterMatcher::Exact(u16::from(b'_')), 2);", " for (offset, (unit, _, _)) in punctuation.iter().enumerate() {", " states[0].transitions.push((CharacterMatcher::Exact(*unit), punctuation_start + offset));", " }", " states[0].transitions.push((CharacterMatcher::Any, any_state));", " states[1] = states[1].clone()", " .transition(CharacterMatcher::UnicodeClass(UnicodeClass::WhiteSpace), 1)", " .accepting(accept(\"ANY\", 7, LexerAction::Skip, -1));", " states[2] = states[2].clone()", " .transition(CharacterMatcher::UnicodeClass(UnicodeClass::Letter), 2)", " .transition(CharacterMatcher::UnicodeClass(UnicodeClass::Number), 2)", " .transition(CharacterMatcher::Exact(u16::from(b'_')), 2)", " .accepting(accept(\"ID\", 6, LexerAction::Emit, 0));", " for (offset, (_, name, number)) in punctuation.iter().enumerate() {", " states[punctuation_start + offset] = DfaState::default()", " .accepting(accept(name, *number, LexerAction::Emit, *number));", " }", " states[any_state] = DfaState::default()", " .accepting(accept(\"ANY\", 7, LexerAction::Emit, 1));", " let mut lexer = YyLexer::new(error_handler)?;", " lexer.set_start_dfa(\"YYINITIAL\", Dfa::from_states(states, 0)?)?;", " lexer.set_reserved_words(\"ID\", ResWds::from_pairs([", " (\"base\", TokenDefinition::new(\"BASE\", 3)?),", " (\"this\", TokenDefinition::new(\"THIS\", 4)?),", " (\"new\", TokenDefinition::new(\"NEW\", 5)?),", " ], false)?)?;", " lexer.using_eof = true;", " for (name, number) in [", ]) for name, number, *_ in tokens: lines.append(f" ({rust_string(name)}, {number}),") lines.extend([ " ] {", " lexer.tokens.0.insert(Object::String(name.to_owned()), Object::Integer(number));", " }", " let _ = lexer.using_cat(UnicodeCategory(0))?;", " Ok(lexer)", "}", "", "pub fn generated_lexer() -> Result {", " generated_lexer_with_handler(ErrorHandler::new_with_boolean(false)?)", "}", "", "#[derive(Clone, Debug)]", "pub struct GeneratedLexerToken(TOKEN);", "", "impl GeneratedLexerToken {", " pub fn new(lexer: Lexer) -> Result {", " let name = generated_symbol_name(NUMBER).ok_or(Error::Argument)?;", " Ok(Self(TOKEN::generated_with_lexer(lexer, name, NUMBER)?))", " }", " pub fn yyname(&self) -> String { self.0.yyname() }", " pub const fn yynum(&self) -> i32 { NUMBER }", " pub fn yytext(&self) -> String { self.0.yytext() }", "}", "", "impl Deref for GeneratedLexerToken {", " type Target = TOKEN;", " fn deref(&self) -> &Self::Target { &self.0 }", "}", "impl DerefMut for GeneratedLexerToken {", " fn deref_mut(&mut self) -> &mut Self::Target { &mut self.0 }", "}", ]) for name, number, *_ in tokens: if name != "EOF": lines.append(f"pub type {name} = GeneratedLexerToken<{number}>;") lines.extend([ "", "fn generated_parser_identity(generated: i32) -> Option<(&'static str, i32)> {", " match generated {", ]) for lhs, number in nonterminals: identities = [internal for _, internal, owner, _ in generated if owner == lhs] pattern = ( str(identities[0]) if len(identities) == 1 else f"{identities[0]}..={identities[-1]}" ) lines.append(f" {pattern} => Some(({rust_string(lhs)}, {number})),") lines.extend([" _ => None,", " }", "}"]) lines.extend([ "", "fn stack_text(parser: &Parser, index: i32) -> Result {", " Ok(parser.stack_at(index)?.m_value.raw_text())", "}", "", "fn generated_semantic_text(generated: i32, parser: &Parser) -> Result {", " Ok(match generated {", ]) action_ids = {name: internal for name, internal, _, _ in generated} expressions: dict[str, list[int]] = {} for _, _, _, action in data["productions"]: expression = semantic_expression(action) if expression != "String::new()": expressions.setdefault(expression, []).append(action_ids[action]) for expression, action_numbers in expressions.items(): pattern = " | ".join(str(number) for number in action_numbers) lines.append(f" {pattern} => {expression},") lines.extend([" _ => String::new(),", " })", "}"]) lines.extend([ "", "#[derive(Clone, Debug)]", "pub struct GeneratedParserSymbol(TOKEN);", "", "impl GeneratedParserSymbol {", " pub fn new(parser: Parser) -> Result {", " let (name, number) = generated_parser_identity(GENERATED).ok_or(Error::Argument)?;", " let text = generated_semantic_text(GENERATED, &parser)?;", " Ok(Self(TOKEN::generated_with_parser(parser, name, number, text)?))", " }", " pub fn yyname(&self) -> String { self.0.yyname() }", " pub fn yynum(&self) -> i32 { self.0.yynum() }", " pub fn yytext(&self) -> String { self.0.yytext() }", "}", "", "impl Deref for GeneratedParserSymbol {", " type Target = TOKEN;", " fn deref(&self) -> &Self::Target { &self.0 }", "}", "impl DerefMut for GeneratedParserSymbol {", " fn deref_mut(&mut self) -> &mut Self::Target { &mut self.0 }", "}", ]) base_ids = {} for name, internal, lhs, _ in generated: base_ids.setdefault(lhs, internal) lines.append(f"pub type {name} = GeneratedParserSymbol<{internal}>;") for name, _ in nonterminals: lines.append(f"pub type {name} = GeneratedParserSymbol<{base_ids[name]}>;") lines.extend([ "", "#[derive(Clone, Debug)]", "pub struct yycs0syntax(pub YyParser);", "impl yycs0syntax {", " pub fn new() -> Result { Ok(Self(generated_parser()?)) }", " pub fn action(&self, parser: Parser, symbol: SYMBOL, action: i32) -> Result { self.0.action(parser, symbol, action) }", ]) for name, internal, _, _ in generated: factory = mapping.snake(f"{name}_factory") lines.append(f" pub fn {factory}(parser: Parser) -> Result {{ Ok(Object::opaque({name}::new(parser)?)) }}") for name, _ in nonterminals: factory = mapping.snake(f"{name}_factory") lines.append(f" pub fn {factory}(parser: Parser) -> Result {{ Ok(Object::opaque({name}::new(parser)?)) }}") lines.extend([ " pub fn error_factory(parser: Parser) -> Result { Ok(Object::opaque(crate::LslError::new_with_parser(parser)?)) }", "}", "impl Deref for yycs0syntax { type Target = YyParser; fn deref(&self) -> &Self::Target { &self.0 } }", "impl DerefMut for yycs0syntax { fn deref_mut(&mut self) -> &mut Self::Target { &mut self.0 } }", "", "#[derive(Clone, Debug)]", "pub struct yycs0tokens(pub YyLexer);", "impl yycs0tokens {", " pub fn new(error_handler: ErrorHandler) -> Result { Ok(Self(generated_lexer_with_handler(error_handler)?)) }", ]) for name, number, *_ in tokens: if name != "EOF": factory = mapping.snake(f"{name}_factory") lines.append(f" pub fn {factory}(lexer: Lexer) -> Result {{ Ok(Object::opaque({name}::new(lexer)?)) }}") lines.extend([ " pub fn old_action(&self, lexer: Lexer, yytext: &mut String, action: i32, reject: &mut bool) -> Result {", " if matches!(action, 8 | 55 | 74) { match action { 8 => \"yym.yy_begin\", 55 => \"((cs0tokens)yym)\", _ => \"((cs0syntax)yyq)\" }.clone_into(yytext); return TOKEN::generated_with_lexer(lexer, \"ANY\", 7); }", " *reject = true;", " Err(Error::InvalidOperation)", " }", "}", "impl Deref for yycs0tokens { type Target = YyLexer; fn deref(&self) -> &Self::Target { &self.0 } }", "impl DerefMut for yycs0tokens { fn deref_mut(&mut self) -> &mut Self::Target { &mut self.0 } }", "", "#[derive(Clone, Debug)]", "pub struct cs0tokens { pub out: String, pub lexer: Lexer }", "impl cs0tokens {", " pub fn new() -> Result { Self::new_with_yy_lexer(generated_lexer()?) }", " pub fn new_with_constructor() -> Result { Self::new() }", " pub fn new_with_error_handler(handler: ErrorHandler) -> Result { Self::new_with_yy_lexer(generated_lexer_with_handler(handler)?) }", " pub fn new_with_yy_lexer(tokens: YyLexer) -> Result { Ok(Self { out: String::new(), lexer: Lexer::new(tokens)? }) }", "}", "impl Deref for cs0tokens { type Target = Lexer; fn deref(&self) -> &Self::Target { &self.lexer } }", "impl DerefMut for cs0tokens { fn deref_mut(&mut self) -> &mut Self::Target { &mut self.lexer } }", "", "#[derive(Clone, Debug)]", "pub struct cs0syntax { pub out: String, pub cls: String, pub par: String, pub ctx: String, pub defconseen: bool, pub parser: Parser }", "impl cs0syntax {", " pub fn new() -> Result { Self::new_with_yy_parser(generated_parser()?) }", " pub fn new_with_constructor() -> Result { Self::new() }", " pub fn new_with_yy_parser(symbols: YyParser) -> Result { let lexer = Lexer::new(generated_lexer()?)?; Self::from_parts(symbols, lexer) }", " pub fn new_with_yy_parser_error_handler(mut symbols: YyParser, handler: ErrorHandler) -> Result { symbols.erh = handler.clone(); let lexer = Lexer::new(generated_lexer_with_handler(handler)?)?; Self::from_parts(symbols, lexer) }", " fn from_parts(symbols: YyParser, lexer: Lexer) -> Result { Ok(Self { out: String::new(), cls: String::new(), par: String::new(), ctx: String::new(), defconseen: false, parser: Parser::new(symbols, lexer)? }) }", "}", "impl Deref for cs0syntax { type Target = Parser; fn deref(&self) -> &Self::Target { &self.parser } }", "impl DerefMut for cs0syntax { fn deref_mut(&mut self) -> &mut Self::Target { &mut self.parser } }", "", ]) return format_rust("\n".join(lines)) def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("--check", action="store_true") args = parser.parse_args() data = json.loads(INPUT.read_text()) output = render(data) if args.check: if not OUTPUT.exists() or OUTPUT.read_text() != output: raise SystemExit(f"generated LSL table is stale: {OUTPUT}") print("LSL parser/token generation is byte-identical") return OUTPUT.write_text(output) if __name__ == "__main__": main()