390 lines
19 KiB
Python
390 lines
19 KiB
Python
#!/usr/bin/env python3
|
|
"""Generate native Rust LSL parser/token types from the reviewed grammar."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import subprocess
|
|
from pathlib import Path
|
|
|
|
import generate_rust_mapping as mapping
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
INPUT = ROOT / "codegen" / "inputs" / "lsl_tools_grammar.json"
|
|
OUTPUT = (
|
|
ROOT / "crates" / "metacrate-lsl-tools" / "src" / "generated_tables.rs"
|
|
)
|
|
|
|
|
|
def rust_string(value: str) -> str:
|
|
return json.dumps(value, ensure_ascii=True)
|
|
|
|
|
|
def format_rust(source: str) -> str:
|
|
"""Apply the workspace's canonical Rust formatting deterministically."""
|
|
result = subprocess.run(
|
|
["rustfmt", "--edition", "2024", "--emit", "stdout"],
|
|
input=source,
|
|
text=True,
|
|
capture_output=True,
|
|
check=False,
|
|
)
|
|
if result.returncode != 0:
|
|
raise SystemExit(f"rustfmt rejected generated LSL Rust:\n{result.stderr}")
|
|
return result.stdout
|
|
|
|
|
|
def semantic_expression(action: str) -> str:
|
|
expressions = {
|
|
"ClassBody_2_1": "stack_text(parser, 1)?",
|
|
"GStuff_2_1": "String::new()",
|
|
"GStuff_4_1": "format!(\"{}{}\", stack_text(parser, 1)?, stack_text(parser, 0)?)",
|
|
"GStuff_6_1": "format!(\"{}{}\", stack_text(parser, 1)?, stack_text(parser, 0)?)",
|
|
"Stuff_2_1": "String::new()",
|
|
"Stuff_4_1": "format!(\"{}{}\", stack_text(parser, 1)?, stack_text(parser, 0)?)",
|
|
"Cons_2_1": "format!(\"{}({}){}\", stack_text(parser, 4)?.trim(), stack_text(parser, 2)?, stack_text(parser, 0)?)",
|
|
"Call_2_1": "format!(\"{}({})\", stack_text(parser, 3)?.trim(), stack_text(parser, 1)?)",
|
|
"BaseCall_2_1": "String::new()",
|
|
"BaseCall_4_1": "format!(\"base{}\", stack_text(parser, 1)?)",
|
|
"BaseCall_6_1": "format!(\"this{}\", stack_text(parser, 1)?)",
|
|
"Name_2_1": "format!(\" {} \", stack_text(parser, 0)?)",
|
|
"Name_4_1": "format!(\"{}[{}]\", stack_text(parser, 3)?, stack_text(parser, 1)?)",
|
|
"Item_2_1": "stack_text(parser, 0)?",
|
|
"Item_4_1": "stack_text(parser, 0)?",
|
|
"Item_6_1": "\";\\n\".to_owned()",
|
|
"Item_8_1": "\" base \".to_owned()",
|
|
"Item_10_1": "\" this \".to_owned()",
|
|
"Item_12_1": "format!(\" this[{}]\", stack_text(parser, 1)?)",
|
|
"Item_14_1": "\":\".to_owned()",
|
|
"Item_16_1": "format!(\" new {}\", stack_text(parser, 0)?)",
|
|
"Item_18_1": "format!(\" new {}\", stack_text(parser, 0)?)",
|
|
"Item_20_1": "format!(\"({})\", stack_text(parser, 1)?)",
|
|
"Item_22_1": "format!(\"{{{}}}\\n\", stack_text(parser, 1)?)",
|
|
"Item_24_1": "format!(\"[{}]\", stack_text(parser, 1)?)",
|
|
}
|
|
return expressions[action]
|
|
|
|
|
|
def render(data: dict) -> str:
|
|
tokens = data["tokens"]
|
|
nonterminals = data["nonterminals"]
|
|
numbers = {name: number for name, number, *_ in tokens}
|
|
numbers.update({name: number for name, number in nonterminals})
|
|
|
|
generated = []
|
|
next_number = 24
|
|
alternatives: dict[str, int] = {}
|
|
for _, lhs, _, action in data["productions"]:
|
|
alternative = alternatives.get(lhs, 0) + 1
|
|
alternatives[lhs] = alternative
|
|
for name in (f"{lhs}_{alternative * 2 - 1}", f"{lhs}_{alternative * 2}", action):
|
|
generated.append((name, next_number, lhs, numbers[lhs]))
|
|
next_number += 1
|
|
assert next_number == 99
|
|
|
|
flat = [data["schema"], data["start"], data["eof"], len(tokens) + len(nonterminals)]
|
|
for name, number, *_ in tokens:
|
|
flat.extend((number, 1, len(name)))
|
|
for name, number in nonterminals:
|
|
flat.extend((number, 0, len(name)))
|
|
flat.append(len(data["productions"]))
|
|
for production, lhs, rhs, _ in data["productions"]:
|
|
flat.extend((production, numbers[lhs], len(rhs), *(numbers[name] for name in rhs)))
|
|
|
|
lines = [
|
|
"// @generated by tools/generate_lsl_tables.py; do not edit by hand.",
|
|
"#![allow(non_camel_case_types)]",
|
|
"#![allow(non_snake_case)]",
|
|
"#![allow(clippy::missing_errors_doc)]",
|
|
"#![allow(clippy::must_use_candidate)]",
|
|
"",
|
|
"use std::ops::{Deref, DerefMut};",
|
|
"",
|
|
"use libremetaverse_types::compat::{Object, UnicodeCategory};",
|
|
"",
|
|
"use crate::{",
|
|
" CharacterMatcher, Dfa, DfaAccept, DfaState, Error, ErrorHandler, Grammar,",
|
|
" Lexer, LexerAction, Parser, ResWds, SYMBOL, TOKEN, TokenDefinition,",
|
|
" UnicodeClass, YyLexer, YyParser,",
|
|
"};",
|
|
"",
|
|
f"pub const LSL_GENERATOR_SOURCE_COMMIT: &str = {rust_string(data['source']['commit'])};",
|
|
f"pub const LSL_GENERATOR_SOURCE_PARSER: &str = {rust_string(data['source']['parser'])};",
|
|
f"pub const LSL_GENERATOR_SOURCE_LEXER: &str = {rust_string(data['source']['lexer'])};",
|
|
f"pub const LSL_GENERATOR_SOURCE_LICENSE: &str = {rust_string(data['source']['license'])};",
|
|
f"pub const LSL_GENERATOR_SERIALIZATION_VERSION: &str = {rust_string(data['source']['serialization_version'])};",
|
|
"pub const GENERATED_PARSER_DATA: &[i32] = &[",
|
|
]
|
|
for index in range(0, len(flat), 16):
|
|
lines.append(" " + ", ".join(map(str, flat[index:index + 16])) + ",")
|
|
lines.extend(["];"])
|
|
|
|
lines.extend([
|
|
"",
|
|
"pub fn generated_symbol_name(number: i32) -> Option<&'static str> {",
|
|
" match number {",
|
|
])
|
|
for name, number, *_ in tokens:
|
|
lines.append(f" {number} => Some({rust_string(name)}),")
|
|
for name, number in nonterminals:
|
|
lines.append(f" {number} => Some({rust_string(name)}),")
|
|
lines.extend([" _ => None,", " }", "}"])
|
|
|
|
lines.extend([
|
|
"",
|
|
"pub fn generated_parser() -> Result<YyParser, Error> {",
|
|
f" let mut grammar = Grammar::new({data['start']}, {data['eof']})?;",
|
|
])
|
|
for name, number, *_ in tokens:
|
|
lines.append(f" grammar.add_symbol({rust_string(name)}, {number}, true)?;")
|
|
for name, number in nonterminals:
|
|
lines.append(f" grammar.add_symbol({rust_string(name)}, {number}, false)?;")
|
|
for _, lhs, rhs, _ in data["productions"]:
|
|
rhs_numbers = ", ".join(str(numbers[name]) for name in rhs)
|
|
lines.append(f" grammar.add_production({numbers[lhs]}, vec![{rhs_numbers}])?;")
|
|
lines.extend([
|
|
" let mut parser = grammar.build()?;",
|
|
" parser.arr = GENERATED_PARSER_DATA.to_vec();",
|
|
" Ok(parser)",
|
|
"}",
|
|
"",
|
|
"fn accept(name: &str, number: i32, action: LexerAction, action_number: i32) -> DfaAccept {",
|
|
" DfaAccept {",
|
|
" token: TokenDefinition { name: name.to_owned(), number },",
|
|
" action,",
|
|
" action_number,",
|
|
" reserved_words: (name == \"ID\").then(|| \"ID\".to_owned()),",
|
|
" }",
|
|
"}",
|
|
"",
|
|
"pub fn generated_lexer_with_handler(error_handler: ErrorHandler) -> Result<YyLexer, Error> {",
|
|
" let punctuation = [",
|
|
])
|
|
punctuation = [(name, number, literal) for name, number, literal in tokens if literal and len(literal) == 1]
|
|
for name, number, literal in punctuation:
|
|
lines.append(f" ({ord(literal)}, {rust_string(name)}, {number}),")
|
|
lines.extend([
|
|
" ];",
|
|
" let punctuation_start = 3usize;",
|
|
" let any_state = punctuation_start + punctuation.len();",
|
|
" let mut states = vec![DfaState::default(); any_state + 1];",
|
|
" states[0] = states[0].clone()",
|
|
" .transition(CharacterMatcher::UnicodeClass(UnicodeClass::WhiteSpace), 1)",
|
|
" .transition(CharacterMatcher::UnicodeClass(UnicodeClass::Letter), 2)",
|
|
" .transition(CharacterMatcher::Exact(u16::from(b'_')), 2);",
|
|
" for (offset, (unit, _, _)) in punctuation.iter().enumerate() {",
|
|
" states[0].transitions.push((CharacterMatcher::Exact(*unit), punctuation_start + offset));",
|
|
" }",
|
|
" states[0].transitions.push((CharacterMatcher::Any, any_state));",
|
|
" states[1] = states[1].clone()",
|
|
" .transition(CharacterMatcher::UnicodeClass(UnicodeClass::WhiteSpace), 1)",
|
|
" .accepting(accept(\"ANY\", 7, LexerAction::Skip, -1));",
|
|
" states[2] = states[2].clone()",
|
|
" .transition(CharacterMatcher::UnicodeClass(UnicodeClass::Letter), 2)",
|
|
" .transition(CharacterMatcher::UnicodeClass(UnicodeClass::Number), 2)",
|
|
" .transition(CharacterMatcher::Exact(u16::from(b'_')), 2)",
|
|
" .accepting(accept(\"ID\", 6, LexerAction::Emit, 0));",
|
|
" for (offset, (_, name, number)) in punctuation.iter().enumerate() {",
|
|
" states[punctuation_start + offset] = DfaState::default()",
|
|
" .accepting(accept(name, *number, LexerAction::Emit, *number));",
|
|
" }",
|
|
" states[any_state] = DfaState::default()",
|
|
" .accepting(accept(\"ANY\", 7, LexerAction::Emit, 1));",
|
|
" let mut lexer = YyLexer::new(error_handler)?;",
|
|
" lexer.set_start_dfa(\"YYINITIAL\", Dfa::from_states(states, 0)?)?;",
|
|
" lexer.set_reserved_words(\"ID\", ResWds::from_pairs([",
|
|
" (\"base\", TokenDefinition::new(\"BASE\", 3)?),",
|
|
" (\"this\", TokenDefinition::new(\"THIS\", 4)?),",
|
|
" (\"new\", TokenDefinition::new(\"NEW\", 5)?),",
|
|
" ], false)?)?;",
|
|
" lexer.using_eof = true;",
|
|
" for (name, number) in [",
|
|
])
|
|
for name, number, *_ in tokens:
|
|
lines.append(f" ({rust_string(name)}, {number}),")
|
|
lines.extend([
|
|
" ] {",
|
|
" lexer.tokens.0.insert(Object::String(name.to_owned()), Object::Integer(number));",
|
|
" }",
|
|
" let _ = lexer.using_cat(UnicodeCategory(0))?;",
|
|
" Ok(lexer)",
|
|
"}",
|
|
"",
|
|
"pub fn generated_lexer() -> Result<YyLexer, Error> {",
|
|
" generated_lexer_with_handler(ErrorHandler::new_with_boolean(false)?)",
|
|
"}",
|
|
"",
|
|
"#[derive(Clone, Debug)]",
|
|
"pub struct GeneratedLexerToken<const NUMBER: i32>(TOKEN);",
|
|
"",
|
|
"impl<const NUMBER: i32> GeneratedLexerToken<NUMBER> {",
|
|
" pub fn new(lexer: Lexer) -> Result<Self, Error> {",
|
|
" let name = generated_symbol_name(NUMBER).ok_or(Error::Argument)?;",
|
|
" Ok(Self(TOKEN::generated_with_lexer(lexer, name, NUMBER)?))",
|
|
" }",
|
|
" pub fn yyname(&self) -> String { self.0.yyname() }",
|
|
" pub const fn yynum(&self) -> i32 { NUMBER }",
|
|
" pub fn yytext(&self) -> String { self.0.yytext() }",
|
|
"}",
|
|
"",
|
|
"impl<const NUMBER: i32> Deref for GeneratedLexerToken<NUMBER> {",
|
|
" type Target = TOKEN;",
|
|
" fn deref(&self) -> &Self::Target { &self.0 }",
|
|
"}",
|
|
"impl<const NUMBER: i32> DerefMut for GeneratedLexerToken<NUMBER> {",
|
|
" fn deref_mut(&mut self) -> &mut Self::Target { &mut self.0 }",
|
|
"}",
|
|
])
|
|
for name, number, *_ in tokens:
|
|
if name != "EOF":
|
|
lines.append(f"pub type {name} = GeneratedLexerToken<{number}>;")
|
|
|
|
lines.extend([
|
|
"",
|
|
"fn generated_parser_identity(generated: i32) -> Option<(&'static str, i32)> {",
|
|
" match generated {",
|
|
])
|
|
for lhs, number in nonterminals:
|
|
identities = [internal for _, internal, owner, _ in generated if owner == lhs]
|
|
pattern = (
|
|
str(identities[0])
|
|
if len(identities) == 1
|
|
else f"{identities[0]}..={identities[-1]}"
|
|
)
|
|
lines.append(f" {pattern} => Some(({rust_string(lhs)}, {number})),")
|
|
lines.extend([" _ => None,", " }", "}"])
|
|
lines.extend([
|
|
"",
|
|
"fn stack_text(parser: &Parser, index: i32) -> Result<String, Error> {",
|
|
" Ok(parser.stack_at(index)?.m_value.raw_text())",
|
|
"}",
|
|
"",
|
|
"fn generated_semantic_text(generated: i32, parser: &Parser) -> Result<String, Error> {",
|
|
" Ok(match generated {",
|
|
])
|
|
action_ids = {name: internal for name, internal, _, _ in generated}
|
|
expressions: dict[str, list[int]] = {}
|
|
for _, _, _, action in data["productions"]:
|
|
expression = semantic_expression(action)
|
|
if expression != "String::new()":
|
|
expressions.setdefault(expression, []).append(action_ids[action])
|
|
for expression, action_numbers in expressions.items():
|
|
pattern = " | ".join(str(number) for number in action_numbers)
|
|
lines.append(f" {pattern} => {expression},")
|
|
lines.extend([" _ => String::new(),", " })", "}"])
|
|
lines.extend([
|
|
"",
|
|
"#[derive(Clone, Debug)]",
|
|
"pub struct GeneratedParserSymbol<const GENERATED: i32>(TOKEN);",
|
|
"",
|
|
"impl<const GENERATED: i32> GeneratedParserSymbol<GENERATED> {",
|
|
" pub fn new(parser: Parser) -> Result<Self, Error> {",
|
|
" let (name, number) = generated_parser_identity(GENERATED).ok_or(Error::Argument)?;",
|
|
" let text = generated_semantic_text(GENERATED, &parser)?;",
|
|
" Ok(Self(TOKEN::generated_with_parser(parser, name, number, text)?))",
|
|
" }",
|
|
" pub fn yyname(&self) -> String { self.0.yyname() }",
|
|
" pub fn yynum(&self) -> i32 { self.0.yynum() }",
|
|
" pub fn yytext(&self) -> String { self.0.yytext() }",
|
|
"}",
|
|
"",
|
|
"impl<const GENERATED: i32> Deref for GeneratedParserSymbol<GENERATED> {",
|
|
" type Target = TOKEN;",
|
|
" fn deref(&self) -> &Self::Target { &self.0 }",
|
|
"}",
|
|
"impl<const GENERATED: i32> DerefMut for GeneratedParserSymbol<GENERATED> {",
|
|
" fn deref_mut(&mut self) -> &mut Self::Target { &mut self.0 }",
|
|
"}",
|
|
])
|
|
base_ids = {}
|
|
for name, internal, lhs, _ in generated:
|
|
base_ids.setdefault(lhs, internal)
|
|
lines.append(f"pub type {name} = GeneratedParserSymbol<{internal}>;")
|
|
for name, _ in nonterminals:
|
|
lines.append(f"pub type {name} = GeneratedParserSymbol<{base_ids[name]}>;")
|
|
|
|
lines.extend([
|
|
"",
|
|
"#[derive(Clone, Debug)]",
|
|
"pub struct yycs0syntax(pub YyParser);",
|
|
"impl yycs0syntax {",
|
|
" pub fn new() -> Result<Self, Error> { Ok(Self(generated_parser()?)) }",
|
|
" pub fn action(&self, parser: Parser, symbol: SYMBOL, action: i32) -> Result<Object, Error> { self.0.action(parser, symbol, action) }",
|
|
])
|
|
for name, internal, _, _ in generated:
|
|
factory = mapping.snake(f"{name}_factory")
|
|
lines.append(f" pub fn {factory}(parser: Parser) -> Result<Object, Error> {{ Ok(Object::opaque({name}::new(parser)?)) }}")
|
|
for name, _ in nonterminals:
|
|
factory = mapping.snake(f"{name}_factory")
|
|
lines.append(f" pub fn {factory}(parser: Parser) -> Result<Object, Error> {{ Ok(Object::opaque({name}::new(parser)?)) }}")
|
|
lines.extend([
|
|
" pub fn error_factory(parser: Parser) -> Result<Object, Error> { Ok(Object::opaque(crate::LslError::new_with_parser(parser)?)) }",
|
|
"}",
|
|
"impl Deref for yycs0syntax { type Target = YyParser; fn deref(&self) -> &Self::Target { &self.0 } }",
|
|
"impl DerefMut for yycs0syntax { fn deref_mut(&mut self) -> &mut Self::Target { &mut self.0 } }",
|
|
"",
|
|
"#[derive(Clone, Debug)]",
|
|
"pub struct yycs0tokens(pub YyLexer);",
|
|
"impl yycs0tokens {",
|
|
" pub fn new(error_handler: ErrorHandler) -> Result<Self, Error> { Ok(Self(generated_lexer_with_handler(error_handler)?)) }",
|
|
])
|
|
for name, number, *_ in tokens:
|
|
if name != "EOF":
|
|
factory = mapping.snake(f"{name}_factory")
|
|
lines.append(f" pub fn {factory}(lexer: Lexer) -> Result<Object, Error> {{ Ok(Object::opaque({name}::new(lexer)?)) }}")
|
|
lines.extend([
|
|
" pub fn old_action(&self, lexer: Lexer, yytext: &mut String, action: i32, reject: &mut bool) -> Result<TOKEN, Error> {",
|
|
" if matches!(action, 8 | 55 | 74) { match action { 8 => \"yym.yy_begin\", 55 => \"((cs0tokens)yym)\", _ => \"((cs0syntax)yyq)\" }.clone_into(yytext); return TOKEN::generated_with_lexer(lexer, \"ANY\", 7); }",
|
|
" *reject = true;",
|
|
" Err(Error::InvalidOperation)",
|
|
" }",
|
|
"}",
|
|
"impl Deref for yycs0tokens { type Target = YyLexer; fn deref(&self) -> &Self::Target { &self.0 } }",
|
|
"impl DerefMut for yycs0tokens { fn deref_mut(&mut self) -> &mut Self::Target { &mut self.0 } }",
|
|
"",
|
|
"#[derive(Clone, Debug)]",
|
|
"pub struct cs0tokens { pub out: String, pub lexer: Lexer }",
|
|
"impl cs0tokens {",
|
|
" pub fn new() -> Result<Self, Error> { Self::new_with_yy_lexer(generated_lexer()?) }",
|
|
" pub fn new_with_constructor() -> Result<Self, Error> { Self::new() }",
|
|
" pub fn new_with_error_handler(handler: ErrorHandler) -> Result<Self, Error> { Self::new_with_yy_lexer(generated_lexer_with_handler(handler)?) }",
|
|
" pub fn new_with_yy_lexer(tokens: YyLexer) -> Result<Self, Error> { Ok(Self { out: String::new(), lexer: Lexer::new(tokens)? }) }",
|
|
"}",
|
|
"impl Deref for cs0tokens { type Target = Lexer; fn deref(&self) -> &Self::Target { &self.lexer } }",
|
|
"impl DerefMut for cs0tokens { fn deref_mut(&mut self) -> &mut Self::Target { &mut self.lexer } }",
|
|
"",
|
|
"#[derive(Clone, Debug)]",
|
|
"pub struct cs0syntax { pub out: String, pub cls: String, pub par: String, pub ctx: String, pub defconseen: bool, pub parser: Parser }",
|
|
"impl cs0syntax {",
|
|
" pub fn new() -> Result<Self, Error> { Self::new_with_yy_parser(generated_parser()?) }",
|
|
" pub fn new_with_constructor() -> Result<Self, Error> { Self::new() }",
|
|
" pub fn new_with_yy_parser(symbols: YyParser) -> Result<Self, Error> { let lexer = Lexer::new(generated_lexer()?)?; Self::from_parts(symbols, lexer) }",
|
|
" pub fn new_with_yy_parser_error_handler(mut symbols: YyParser, handler: ErrorHandler) -> Result<Self, Error> { symbols.erh = handler.clone(); let lexer = Lexer::new(generated_lexer_with_handler(handler)?)?; Self::from_parts(symbols, lexer) }",
|
|
" fn from_parts(symbols: YyParser, lexer: Lexer) -> Result<Self, Error> { Ok(Self { out: String::new(), cls: String::new(), par: String::new(), ctx: String::new(), defconseen: false, parser: Parser::new(symbols, lexer)? }) }",
|
|
"}",
|
|
"impl Deref for cs0syntax { type Target = Parser; fn deref(&self) -> &Self::Target { &self.parser } }",
|
|
"impl DerefMut for cs0syntax { fn deref_mut(&mut self) -> &mut Self::Target { &mut self.parser } }",
|
|
"",
|
|
])
|
|
return format_rust("\n".join(lines))
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--check", action="store_true")
|
|
args = parser.parse_args()
|
|
data = json.loads(INPUT.read_text())
|
|
output = render(data)
|
|
if args.check:
|
|
if not OUTPUT.exists() or OUTPUT.read_text() != output:
|
|
raise SystemExit(f"generated LSL table is stale: {OUTPUT}")
|
|
print("LSL parser/token generation is byte-identical")
|
|
return
|
|
OUTPUT.write_text(output)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|