hugo/MetaCrate
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

Generate native LSL parser tables (#83)
Some checks failed
Native code generation / deterministic (push) Failing after 2m36s
Details
Imaging and meshing gate / native (push) Successful in 5m29s
Details
JPEG 2000 feature / linux (push) Successful in 2m46s
Details
Native Rust workspace compile / compile (push) Failing after 1m58s
Details
Skia feature / linux (push) Successful in 31m14s
Details

Browse Source
This commit is contained in:
Chili Palmer
2026-08-11 03:36:14 +00:00
parent 656d837778
commit 2558e2f49f
17 changed files with 3905 additions and 2700 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
.gitea/workflows/rust-workspace.yml
View File

@@ -14,6 +14,9 @@ on:
- "tools/check_milestone_10_issue_80.py" - "tools/check_milestone_10_issue_80.py"
- "tools/check_milestone_10_issue_81.py" - "tools/check_milestone_10_issue_81.py"
- "tools/check_milestone_10_issue_82.py" - "tools/check_milestone_10_issue_82.py"
- "tools/check_milestone_10_issue_83.py"
- "tools/generate_lsl_tables.py"
- "codegen/inputs/lsl_tools_grammar.json"
- "**/*.rs" - "**/*.rs"
- "**/Cargo.toml" - "**/Cargo.toml"
- "Cargo.lock" - "Cargo.lock"
@@ -30,6 +33,9 @@ on:
- "tools/check_milestone_10_issue_80.py" - "tools/check_milestone_10_issue_80.py"
- "tools/check_milestone_10_issue_81.py" - "tools/check_milestone_10_issue_81.py"
- "tools/check_milestone_10_issue_82.py" - "tools/check_milestone_10_issue_82.py"
- "tools/check_milestone_10_issue_83.py"
- "tools/generate_lsl_tables.py"
- "codegen/inputs/lsl_tools_grammar.json"
- "**/*.rs" - "**/*.rs"
- "**/Cargo.toml" - "**/Cargo.toml"
- "Cargo.lock" - "Cargo.lock"
@@ -72,6 +78,8 @@ jobs:
python3 tools/check_milestone_10_issue_80.py python3 tools/check_milestone_10_issue_80.py
python3 tools/check_milestone_10_issue_81.py python3 tools/check_milestone_10_issue_81.py
python3 tools/check_milestone_10_issue_82.py python3 tools/check_milestone_10_issue_82.py
python3 tools/check_milestone_10_issue_83.py
python3 tools/generate_lsl_tables.py --check
- name: Test the complete native world milestone - name: Test the complete native world milestone
run: python3 tools/test_milestone_09.py run: python3 tools/test_milestone_09.py
- name: Compile every workspace target with bounded memory - name: Compile every workspace target with bounded memory

1
Cargo.lock generated
View File

@@ -924,6 +924,7 @@ name = "libremetaverse-lsl-tools"
version = "0.0.1" version = "0.0.1"
dependencies = [ dependencies = [
"libremetaverse-types", "libremetaverse-types",
"regex",
"unicode-general-category", "unicode-general-category",
] ]

2
api/SHIM-COVERAGE.md
View File

@@ -7,7 +7,7 @@ Generated by `python3 tools/generate_api_shims.py`; do not edit by hand.
| `LibreMetaverse` | 2,711 | 27,281 | native implementation: 401 types / 17,025 members; remaining surface is callable failure-only shims | | `LibreMetaverse` | 2,711 | 27,281 | native implementation: 401 types / 17,025 members; remaining surface is callable failure-only shims |
| `LibreMetaverse.Imaging.Abstractions` | 3 | 20 | native implementation: 3 types / 20 members; no generated shims remain | | `LibreMetaverse.Imaging.Abstractions` | 3 | 20 | native implementation: 3 types / 20 members; no generated shims remain |
| `LibreMetaverse.Imaging.Skia` | 1 | 3 | native implementation: 1 type / 3 members; no generated shims remain | | `LibreMetaverse.Imaging.Skia` | 1 | 3 | native implementation: 1 type / 3 members; no generated shims remain |
| `LibreMetaverse.LslTools` | 164 | 768 | native implementation: 44 types / 403 members; remaining surface is callable failure-only shims | | `LibreMetaverse.LslTools` | 164 | 768 | native implementation: 164 types / 768 members; no generated shims remain |
| `LibreMetaverse.PrimMesher` | 17 | 207 | native implementation: 15 types / 200 members; remaining surface is callable failure-only shims | | `LibreMetaverse.PrimMesher` | 17 | 207 | native implementation: 15 types / 200 members; remaining surface is callable failure-only shims |
| `LibreMetaverse.RLV` | 28 | 499 | native implementation: 17 types / 200 members; remaining surface is callable failure-only shims | | `LibreMetaverse.RLV` | 28 | 499 | native implementation: 17 types / 200 members; remaining surface is callable failure-only shims |
| `LibreMetaverse.Rendering.MeshFoundry` | 1 | 14 | native implementation: 1 type / 14 members; no generated shims remain | | `LibreMetaverse.Rendering.MeshFoundry` | 1 | 14 | native implementation: 1 type / 14 members; no generated shims remain |

66
codegen/inputs/lsl_tools_grammar.json Normal file
View File

@@ -0,0 +1,66 @@
{
"schema": 1,
"source": {
"repository": "https://github.com/cinderblocks/libremetaverse",
"commit": "2aa70bb68513b39795da5d13c88f31b86e85a3ba",
"parser": "LibreMetaverse.LslTools/YYClass/yycs0syntax.cs",
"lexer": "LibreMetaverse.LslTools/YYClass/yycs0tokens.cs",
"license": "BSD-3-Clause",
"serialization_version": "4.5"
},
"start": 19,
"eof": 2,
"tokens": [
["EOF", 2, null],
["BASE", 3, "base"],
["THIS", 4, "this"],
["NEW", 5, "new"],
["ID", 6, null],
["ANY", 7, null],
["COLON", 8, ":"],
["SEMICOLON", 9, ";"],
["LBRACE", 10, "{"],
["RBRACE", 11, "}"],
["LPAREN", 12, "("],
["RPAREN", 13, ")"],
["LBRACK", 14, "["],
["RBRACK", 15, "]"]
],
"nonterminals": [
["GStuff", 16],
["Stuff", 17],
["Item", 18],
["ClassBody", 19],
["Cons", 20],
["Call", 21],
["BaseCall", 22],
["Name", 23]
],
"productions": [
[1, "ClassBody", ["GStuff", "Cons", "GStuff"], "ClassBody_2_1"],
[2, "GStuff", [], "GStuff_2_1"],
[3, "GStuff", ["GStuff", "Item"], "GStuff_4_1"],
[4, "GStuff", ["GStuff", "Call"], "GStuff_6_1"],
[5, "Stuff", [], "Stuff_2_1"],
[6, "Stuff", ["Stuff", "Item"], "Stuff_4_1"],
[7, "Cons", ["Name", "LPAREN", "Stuff", "RPAREN", "BaseCall"], "Cons_2_1"],
[8, "Call", ["Name", "LPAREN", "Stuff", "RPAREN"], "Call_2_1"],
[9, "BaseCall", [], "BaseCall_2_1"],
[10, "BaseCall", ["COLON", "BASE", "LPAREN", "Stuff", "RPAREN"], "BaseCall_4_1"],
[11, "BaseCall", ["COLON", "THIS", "LPAREN", "Stuff", "RPAREN"], "BaseCall_6_1"],
[12, "Name", ["ID"], "Name_2_1"],
[13, "Name", ["ID", "LBRACK", "Stuff", "RBRACK"], "Name_4_1"],
[14, "Item", ["ID"], "Item_2_1"],
[15, "Item", ["ANY"], "Item_4_1"],
[16, "Item", ["SEMICOLON"], "Item_6_1"],
[17, "Item", ["BASE"], "Item_8_1"],
[18, "Item", ["THIS"], "Item_10_1"],
[19, "Item", ["THIS", "LBRACK", "Stuff", "RBRACK"], "Item_12_1"],
[20, "Item", ["COLON"], "Item_14_1"],
[21, "Item", ["NEW", "Name"], "Item_16_1"],
[22, "Item", ["NEW", "ID"], "Item_18_1"],
[23, "Item", ["LPAREN", "Stuff", "RPAREN"], "Item_20_1"],
[24, "Item", ["LBRACE", "GStuff", "RBRACE"], "Item_22_1"],
[25, "Item", ["LBRACK", "Stuff", "RBRACK"], "Item_24_1"]
]
}

1
crates/libremetaverse-lsl-tools/Cargo.toml
View File

@@ -10,6 +10,7 @@ description = "Native LSL lexer and parser tooling for the MetaCrate LibreMetave
[dependencies] [dependencies]
libremetaverse-types = { path = "../libremetaverse-types" } libremetaverse-types = { path = "../libremetaverse-types" }
unicode-general-category = "1.1" unicode-general-category = "1.1"
regex = "1"
[lints] [lints]
workspace = true workspace = true

43
crates/libremetaverse-lsl-tools/README.md
View File

@@ -6,8 +6,8 @@ issue 81 boundary supplies source reading, comments, Unicode character sets,
deterministic DFA execution, reserved words, terminal symbols, EOF, source deterministic DFA execution, reserved words, terminal symbols, EOF, source
locations, diagnostics, and Rust iterators. The issue 82 boundary supplies locations, diagnostics, and Rust iterators. The issue 82 boundary supplies
grammar productions, parser tables, precedence, reductions, and recovery. grammar productions, parser tables, precedence, reductions, and recovery.
Deterministic checked-in generated grammar and token tables are added by issue The issue 83 boundary adds deterministic checked-in grammar/token tables and
83. native replacements for the retired parser-generator object model.
## Source and position contract ## Source and position contract
@@ -81,9 +81,35 @@ recovery attempts.
Compatibility `SymbolSet` clones share deterministic FIRST/FOLLOW membership, Compatibility `SymbolSet` clones share deterministic FIRST/FOLLOW membership,
and cloned `CSymbol` values share registered production metadata. This retains and cloned `CSymbol` values share registered production metadata. This retains
the observable reference behavior of the C# grammar model without unsafe code. the observable reference behavior of the C# grammar model without unsafe code.
Parser-source serialization and the old `Builder`-driven generated-output Legacy parser-runtime serialization members that cannot faithfully round-trip
members return `InvalidOperation` explicitly until issue 83 supplies the the richer native LALR machine remain explicit `InvalidOperation` results.
checked-in generator; they never report a false success. Callers should use the stable `YyParser::emit` and `YyLexer::emit_dfa` formats,
or regenerate the canonical checked-in table module described below.
## Checked-in generator contract
[`codegen/inputs/lsl_tools_grammar.json`](../../codegen/inputs/lsl_tools_grammar.json)
is the reviewed source of truth reconstructed from the pinned 4.5
`yycs0syntax.cs` and `yycs0tokens.cs` tables at upstream commit
`2aa70bb68513b39795da5d13c88f31b86e85a3ba`. It records every token,
nonterminal, production, and semantic-action class. The portable Python
generator uses the standard library plus the workspace Rust formatter and emits
`src/generated_tables.rs`; it does not invoke C#, a C# source generator, or a
.NET runtime.
Regenerate with `python3 tools/generate_lsl_tables.py`. Use
`python3 tools/generate_lsl_tables.py --check` in reviews and CI. The check
renders the complete output in memory and compares bytes, so repeated runs are
independent of hash iteration, locale, host operating system, and timestamps.
The checked-in module supplies the canonical 25-production grammar, a Unicode
lexer with reserved `base`, `this`, and `new` tokens, the 84 generated syntax
classes, the 13 generated token classes, and the four `cs0` runtime wrappers.
The old `GenBase`, `SymbolsGen`, `TokensGen`, `Regex`, NFA, delegate, factory,
and 4.5 `Serialiser` APIs are live Rust compatibility adapters. New code should
prefer `Grammar`, typed `DfaState` builders, `generated_parser`, and
`generated_lexer`; these avoid CLR-style method pointers and class-shaped
generated inheritance while retaining deterministic mapped behavior.
## Diagnostics and migration ## Diagnostics and migration
@@ -108,6 +134,9 @@ The principal API mapping is:
| `CSymbol`, `SymbolSet`, `Production`, `Precedence` | Same mapped names plus typed `Grammar` builders | | `CSymbol`, `SymbolSet`, `Production`, `Precedence` | Same mapped names plus typed `Grammar` builders |
| `YyParser`, parser entries | Deterministic native tables and conflict records | | `YyParser`, parser entries | Deterministic native tables and conflict records |
| `Parser`, `ParseStackEntry`, `Error`, `recoveredError` | Bounded parsing, typed trees, and recovery | | `Parser`, `ParseStackEntry`, `Error`, `recoveredError` | Bounded parsing, typed trees, and recovery |
| `yycs0syntax`, `yycs0tokens` | `generated_parser`, `generated_lexer`, checked-in native types |
| `GenBase`, `SymbolsGen`, `TokensGen`, `Regex`, `Nfa` | Portable Rust generator adapters; prefer typed grammar/DFA builders |
| `Serialiser` | Deterministic 4.5-compatible integer stream for supported mapped values |
No C#, .NET runtime, dynamically loaded class, macOS-only API, platform code No C#, .NET runtime, dynamically loaded class, macOS-only API, platform code
page, or runtime source generation is used by this boundary. page, or runtime source generation is used by this boundary.
@@ -123,8 +152,10 @@ CARGO_BUILD_JOBS=1 cargo clippy -p libremetaverse-lsl-tools --all-targets --lock
RUSTDOCFLAGS='-D warnings' CARGO_BUILD_JOBS=1 cargo doc -p libremetaverse-lsl-tools --no-deps --locked RUSTDOCFLAGS='-D warnings' CARGO_BUILD_JOBS=1 cargo doc -p libremetaverse-lsl-tools --no-deps --locked
python3 tools/check_milestone_10_issue_81.py python3 tools/check_milestone_10_issue_81.py
python3 tools/check_milestone_10_issue_82.py python3 tools/check_milestone_10_issue_82.py
python3 tools/check_milestone_10_issue_83.py
python3 tools/generate_lsl_tables.py --check
python3 tools/generate_api_shims.py --check python3 tools/generate_api_shims.py --check
``` ```
The package contains 37 focused native and compatibility fixtures. The Gitea The package contains 46 focused native and compatibility fixtures. The Gitea
workflow runs the audit and workspace compile on `ubuntu-latest` only. workflow runs the audit and workspace compile on `ubuntu-latest` only.

3173
crates/libremetaverse-lsl-tools/src/generated.rs
View File

File diff suppressed because it is too large Load Diff

857
crates/libremetaverse-lsl-tools/src/generated_tables.rs Normal file
View File

@@ -0,0 +1,857 @@
// @generated by tools/generate_lsl_tables.py; do not edit by hand.
#![allow(non_camel_case_types)]
#![allow(non_snake_case)]
#![allow(clippy::missing_errors_doc)]
#![allow(clippy::must_use_candidate)]
use std::ops::{Deref, DerefMut};
use libremetaverse_types::compat::{Object, UnicodeCategory};
use crate::{
CharacterMatcher, Dfa, DfaAccept, DfaState, Error, ErrorHandler, Grammar, Lexer, LexerAction,
Parser, ResWds, SYMBOL, TOKEN, TokenDefinition, UnicodeClass, YyLexer, YyParser,
};
pub const LSL_GENERATOR_SOURCE_COMMIT: &str = "2aa70bb68513b39795da5d13c88f31b86e85a3ba";
pub const LSL_GENERATOR_SOURCE_PARSER: &str = "LibreMetaverse.LslTools/YYClass/yycs0syntax.cs";
pub const LSL_GENERATOR_SOURCE_LEXER: &str = "LibreMetaverse.LslTools/YYClass/yycs0tokens.cs";
pub const LSL_GENERATOR_SOURCE_LICENSE: &str = "BSD-3-Clause";
pub const LSL_GENERATOR_SERIALIZATION_VERSION: &str = "4.5";
pub const GENERATED_PARSER_DATA: &[i32] = &[
1, 19, 2, 22, 2, 1, 3, 3, 1, 4, 4, 1, 4, 5, 1, 3, 6, 1, 2, 7, 1, 3, 8, 1, 5, 9, 1, 9, 10, 1, 6,
11, 1, 6, 12, 1, 6, 13, 1, 6, 14, 1, 6, 15, 1, 6, 16, 0, 6, 17, 0, 5, 18, 0, 4, 19, 0, 9, 20,
0, 4, 21, 0, 4, 22, 0, 8, 23, 0, 4, 25, 1, 19, 3, 16, 20, 16, 2, 16, 0, 3, 16, 2, 16, 18, 4,
16, 2, 16, 21, 5, 17, 0, 6, 17, 2, 17, 18, 7, 20, 5, 23, 12, 17, 13, 22, 8, 21, 4, 23, 12, 17,
13, 9, 22, 0, 10, 22, 5, 8, 3, 12, 17, 13, 11, 22, 5, 8, 4, 12, 17, 13, 12, 23, 1, 6, 13, 23,
4, 6, 14, 17, 15, 14, 18, 1, 6, 15, 18, 1, 7, 16, 18, 1, 9, 17, 18, 1, 3, 18, 18, 1, 4, 19, 18,
4, 4, 14, 17, 15, 20, 18, 1, 8, 21, 18, 2, 5, 23, 22, 18, 2, 5, 6, 23, 18, 3, 12, 17, 13, 24,
18, 3, 10, 16, 11, 25, 18, 3, 14, 17, 15,
];
pub fn generated_symbol_name(number: i32) -> Option<&'static str> {
match number {
2 => Some("EOF"),
3 => Some("BASE"),
4 => Some("THIS"),
5 => Some("NEW"),
6 => Some("ID"),
7 => Some("ANY"),
8 => Some("COLON"),
9 => Some("SEMICOLON"),
10 => Some("LBRACE"),
11 => Some("RBRACE"),
12 => Some("LPAREN"),
13 => Some("RPAREN"),
14 => Some("LBRACK"),
15 => Some("RBRACK"),
16 => Some("GStuff"),
17 => Some("Stuff"),
18 => Some("Item"),
19 => Some("ClassBody"),
20 => Some("Cons"),
21 => Some("Call"),
22 => Some("BaseCall"),
23 => Some("Name"),
_ => None,
}
}
pub fn generated_parser() -> Result<YyParser, Error> {
let mut grammar = Grammar::new(19, 2)?;
grammar.add_symbol("EOF", 2, true)?;
grammar.add_symbol("BASE", 3, true)?;
grammar.add_symbol("THIS", 4, true)?;
grammar.add_symbol("NEW", 5, true)?;
grammar.add_symbol("ID", 6, true)?;
grammar.add_symbol("ANY", 7, true)?;
grammar.add_symbol("COLON", 8, true)?;
grammar.add_symbol("SEMICOLON", 9, true)?;
grammar.add_symbol("LBRACE", 10, true)?;
grammar.add_symbol("RBRACE", 11, true)?;
grammar.add_symbol("LPAREN", 12, true)?;
grammar.add_symbol("RPAREN", 13, true)?;
grammar.add_symbol("LBRACK", 14, true)?;
grammar.add_symbol("RBRACK", 15, true)?;
grammar.add_symbol("GStuff", 16, false)?;
grammar.add_symbol("Stuff", 17, false)?;
grammar.add_symbol("Item", 18, false)?;
grammar.add_symbol("ClassBody", 19, false)?;
grammar.add_symbol("Cons", 20, false)?;
grammar.add_symbol("Call", 21, false)?;
grammar.add_symbol("BaseCall", 22, false)?;
grammar.add_symbol("Name", 23, false)?;
grammar.add_production(19, vec![16, 20, 16])?;
grammar.add_production(16, vec![])?;
grammar.add_production(16, vec![16, 18])?;
grammar.add_production(16, vec![16, 21])?;
grammar.add_production(17, vec![])?;
grammar.add_production(17, vec![17, 18])?;
grammar.add_production(20, vec![23, 12, 17, 13, 22])?;
grammar.add_production(21, vec![23, 12, 17, 13])?;
grammar.add_production(22, vec![])?;
grammar.add_production(22, vec![8, 3, 12, 17, 13])?;
grammar.add_production(22, vec![8, 4, 12, 17, 13])?;
grammar.add_production(23, vec![6])?;
grammar.add_production(23, vec![6, 14, 17, 15])?;
grammar.add_production(18, vec![6])?;
grammar.add_production(18, vec![7])?;
grammar.add_production(18, vec![9])?;
grammar.add_production(18, vec![3])?;
grammar.add_production(18, vec![4])?;
grammar.add_production(18, vec![4, 14, 17, 15])?;
grammar.add_production(18, vec![8])?;
grammar.add_production(18, vec![5, 23])?;
grammar.add_production(18, vec![5, 6])?;
grammar.add_production(18, vec![12, 17, 13])?;
grammar.add_production(18, vec![10, 16, 11])?;
grammar.add_production(18, vec![14, 17, 15])?;
let mut parser = grammar.build()?;
parser.arr = GENERATED_PARSER_DATA.to_vec();
Ok(parser)
}
fn accept(name: &str, number: i32, action: LexerAction, action_number: i32) -> DfaAccept {
DfaAccept {
token: TokenDefinition {
name: name.to_owned(),
number,
},
action,
action_number,
reserved_words: (name == "ID").then(|| "ID".to_owned()),
}
}
pub fn generated_lexer_with_handler(error_handler: ErrorHandler) -> Result<YyLexer, Error> {
let punctuation = [
(58, "COLON", 8),
(59, "SEMICOLON", 9),
(123, "LBRACE", 10),
(125, "RBRACE", 11),
(40, "LPAREN", 12),
(41, "RPAREN", 13),
(91, "LBRACK", 14),
(93, "RBRACK", 15),
];
let punctuation_start = 3usize;
let any_state = punctuation_start + punctuation.len();
let mut states = vec![DfaState::default(); any_state + 1];
states[0] = states[0]
.clone()
.transition(CharacterMatcher::UnicodeClass(UnicodeClass::WhiteSpace), 1)
.transition(CharacterMatcher::UnicodeClass(UnicodeClass::Letter), 2)
.transition(CharacterMatcher::Exact(u16::from(b'_')), 2);
for (offset, (unit, _, _)) in punctuation.iter().enumerate() {
states[0]
.transitions
.push((CharacterMatcher::Exact(*unit), punctuation_start + offset));
}
states[0]
.transitions
.push((CharacterMatcher::Any, any_state));
states[1] = states[1]
.clone()
.transition(CharacterMatcher::UnicodeClass(UnicodeClass::WhiteSpace), 1)
.accepting(accept("ANY", 7, LexerAction::Skip, -1));
states[2] = states[2]
.clone()
.transition(CharacterMatcher::UnicodeClass(UnicodeClass::Letter), 2)
.transition(CharacterMatcher::UnicodeClass(UnicodeClass::Number), 2)
.transition(CharacterMatcher::Exact(u16::from(b'_')), 2)
.accepting(accept("ID", 6, LexerAction::Emit, 0));
for (offset, (_, name, number)) in punctuation.iter().enumerate() {
states[punctuation_start + offset] =
DfaState::default().accepting(accept(name, *number, LexerAction::Emit, *number));
}
states[any_state] = DfaState::default().accepting(accept("ANY", 7, LexerAction::Emit, 1));
let mut lexer = YyLexer::new(error_handler)?;
lexer.set_start_dfa("YYINITIAL", Dfa::from_states(states, 0)?)?;
lexer.set_reserved_words(
"ID",
ResWds::from_pairs(
[
("base", TokenDefinition::new("BASE", 3)?),
("this", TokenDefinition::new("THIS", 4)?),
("new", TokenDefinition::new("NEW", 5)?),
],
false,
)?,
)?;
lexer.using_eof = true;
for (name, number) in [
("EOF", 2),
("BASE", 3),
("THIS", 4),
("NEW", 5),
("ID", 6),
("ANY", 7),
("COLON", 8),
("SEMICOLON", 9),
("LBRACE", 10),
("RBRACE", 11),
("LPAREN", 12),
("RPAREN", 13),
("LBRACK", 14),
("RBRACK", 15),
] {
lexer
.tokens
.0
.insert(Object::String(name.to_owned()), Object::Integer(number));
}
let _ = lexer.using_cat(UnicodeCategory(0))?;
Ok(lexer)
}
pub fn generated_lexer() -> Result<YyLexer, Error> {
generated_lexer_with_handler(ErrorHandler::new_with_boolean(false)?)
}
#[derive(Clone, Debug)]
pub struct GeneratedLexerToken<const NUMBER: i32>(TOKEN);
impl<const NUMBER: i32> GeneratedLexerToken<NUMBER> {
pub fn new(lexer: Lexer) -> Result<Self, Error> {
let name = generated_symbol_name(NUMBER).ok_or(Error::Argument)?;
Ok(Self(TOKEN::generated_with_lexer(lexer, name, NUMBER)?))
}
pub fn yyname(&self) -> String {
self.0.yyname()
}
pub const fn yynum(&self) -> i32 {
NUMBER
}
pub fn yytext(&self) -> String {
self.0.yytext()
}
}
impl<const NUMBER: i32> Deref for GeneratedLexerToken<NUMBER> {
type Target = TOKEN;
fn deref(&self) -> &Self::Target {
&self.0
}
}
impl<const NUMBER: i32> DerefMut for GeneratedLexerToken<NUMBER> {
fn deref_mut(&mut self) -> &mut Self::Target {
&mut self.0
}
}
pub type BASE = GeneratedLexerToken<3>;
pub type THIS = GeneratedLexerToken<4>;
pub type NEW = GeneratedLexerToken<5>;
pub type ID = GeneratedLexerToken<6>;
pub type ANY = GeneratedLexerToken<7>;
pub type COLON = GeneratedLexerToken<8>;
pub type SEMICOLON = GeneratedLexerToken<9>;
pub type LBRACE = GeneratedLexerToken<10>;
pub type RBRACE = GeneratedLexerToken<11>;
pub type LPAREN = GeneratedLexerToken<12>;
pub type RPAREN = GeneratedLexerToken<13>;
pub type LBRACK = GeneratedLexerToken<14>;
pub type RBRACK = GeneratedLexerToken<15>;
fn generated_parser_identity(generated: i32) -> Option<(&'static str, i32)> {
match generated {
27..=35 => Some(("GStuff", 16)),
36..=41 => Some(("Stuff", 17)),
63..=98 => Some(("Item", 18)),
24..=26 => Some(("ClassBody", 19)),
42..=44 => Some(("Cons", 20)),
45..=47 => Some(("Call", 21)),
48..=56 => Some(("BaseCall", 22)),
57..=62 => Some(("Name", 23)),
_ => None,
}
}
fn stack_text(parser: &Parser, index: i32) -> Result<String, Error> {
Ok(parser.stack_at(index)?.m_value.raw_text())
}
fn generated_semantic_text(generated: i32, parser: &Parser) -> Result<String, Error> {
Ok(match generated {
26 => stack_text(parser, 1)?,
32 | 35 | 41 => format!("{}{}", stack_text(parser, 1)?, stack_text(parser, 0)?),
44 => format!(
"{}({}){}",
stack_text(parser, 4)?.trim(),
stack_text(parser, 2)?,
stack_text(parser, 0)?
),
47 => format!(
"{}({})",
stack_text(parser, 3)?.trim(),
stack_text(parser, 1)?
),
53 => format!("base{}", stack_text(parser, 1)?),
56 => format!("this{}", stack_text(parser, 1)?),
59 => format!(" {} ", stack_text(parser, 0)?),
62 => format!("{}[{}]", stack_text(parser, 3)?, stack_text(parser, 1)?),
65 | 68 => stack_text(parser, 0)?,
71 => ";\n".to_owned(),
74 => " base ".to_owned(),
77 => " this ".to_owned(),
80 => format!(" this[{}]", stack_text(parser, 1)?),
83 => ":".to_owned(),
86 | 89 => format!(" new {}", stack_text(parser, 0)?),
92 => format!("({})", stack_text(parser, 1)?),
95 => format!("{{{}}}\n", stack_text(parser, 1)?),
98 => format!("[{}]", stack_text(parser, 1)?),
_ => String::new(),
})
}
#[derive(Clone, Debug)]
pub struct GeneratedParserSymbol<const GENERATED: i32>(TOKEN);
impl<const GENERATED: i32> GeneratedParserSymbol<GENERATED> {
pub fn new(parser: Parser) -> Result<Self, Error> {
let (name, number) = generated_parser_identity(GENERATED).ok_or(Error::Argument)?;
let text = generated_semantic_text(GENERATED, &parser)?;
Ok(Self(TOKEN::generated_with_parser(
parser, name, number, text,
)?))
}
pub fn yyname(&self) -> String {
self.0.yyname()
}
pub fn yynum(&self) -> i32 {
self.0.yynum()
}
pub fn yytext(&self) -> String {
self.0.yytext()
}
}
impl<const GENERATED: i32> Deref for GeneratedParserSymbol<GENERATED> {
type Target = TOKEN;
fn deref(&self) -> &Self::Target {
&self.0
}
}
impl<const GENERATED: i32> DerefMut for GeneratedParserSymbol<GENERATED> {
fn deref_mut(&mut self) -> &mut Self::Target {
&mut self.0
}
}
pub type ClassBody_1 = GeneratedParserSymbol<24>;
pub type ClassBody_2 = GeneratedParserSymbol<25>;
pub type ClassBody_2_1 = GeneratedParserSymbol<26>;
pub type GStuff_1 = GeneratedParserSymbol<27>;
pub type GStuff_2 = GeneratedParserSymbol<28>;
pub type GStuff_2_1 = GeneratedParserSymbol<29>;
pub type GStuff_3 = GeneratedParserSymbol<30>;
pub type GStuff_4 = GeneratedParserSymbol<31>;
pub type GStuff_4_1 = GeneratedParserSymbol<32>;
pub type GStuff_5 = GeneratedParserSymbol<33>;
pub type GStuff_6 = GeneratedParserSymbol<34>;
pub type GStuff_6_1 = GeneratedParserSymbol<35>;
pub type Stuff_1 = GeneratedParserSymbol<36>;
pub type Stuff_2 = GeneratedParserSymbol<37>;
pub type Stuff_2_1 = GeneratedParserSymbol<38>;
pub type Stuff_3 = GeneratedParserSymbol<39>;
pub type Stuff_4 = GeneratedParserSymbol<40>;
pub type Stuff_4_1 = GeneratedParserSymbol<41>;
pub type Cons_1 = GeneratedParserSymbol<42>;
pub type Cons_2 = GeneratedParserSymbol<43>;
pub type Cons_2_1 = GeneratedParserSymbol<44>;
pub type Call_1 = GeneratedParserSymbol<45>;
pub type Call_2 = GeneratedParserSymbol<46>;
pub type Call_2_1 = GeneratedParserSymbol<47>;
pub type BaseCall_1 = GeneratedParserSymbol<48>;
pub type BaseCall_2 = GeneratedParserSymbol<49>;
pub type BaseCall_2_1 = GeneratedParserSymbol<50>;
pub type BaseCall_3 = GeneratedParserSymbol<51>;
pub type BaseCall_4 = GeneratedParserSymbol<52>;
pub type BaseCall_4_1 = GeneratedParserSymbol<53>;
pub type BaseCall_5 = GeneratedParserSymbol<54>;
pub type BaseCall_6 = GeneratedParserSymbol<55>;
pub type BaseCall_6_1 = GeneratedParserSymbol<56>;
pub type Name_1 = GeneratedParserSymbol<57>;
pub type Name_2 = GeneratedParserSymbol<58>;
pub type Name_2_1 = GeneratedParserSymbol<59>;
pub type Name_3 = GeneratedParserSymbol<60>;
pub type Name_4 = GeneratedParserSymbol<61>;
pub type Name_4_1 = GeneratedParserSymbol<62>;
pub type Item_1 = GeneratedParserSymbol<63>;
pub type Item_2 = GeneratedParserSymbol<64>;
pub type Item_2_1 = GeneratedParserSymbol<65>;
pub type Item_3 = GeneratedParserSymbol<66>;
pub type Item_4 = GeneratedParserSymbol<67>;
pub type Item_4_1 = GeneratedParserSymbol<68>;
pub type Item_5 = GeneratedParserSymbol<69>;
pub type Item_6 = GeneratedParserSymbol<70>;
pub type Item_6_1 = GeneratedParserSymbol<71>;
pub type Item_7 = GeneratedParserSymbol<72>;
pub type Item_8 = GeneratedParserSymbol<73>;
pub type Item_8_1 = GeneratedParserSymbol<74>;
pub type Item_9 = GeneratedParserSymbol<75>;
pub type Item_10 = GeneratedParserSymbol<76>;
pub type Item_10_1 = GeneratedParserSymbol<77>;
pub type Item_11 = GeneratedParserSymbol<78>;
pub type Item_12 = GeneratedParserSymbol<79>;
pub type Item_12_1 = GeneratedParserSymbol<80>;
pub type Item_13 = GeneratedParserSymbol<81>;
pub type Item_14 = GeneratedParserSymbol<82>;
pub type Item_14_1 = GeneratedParserSymbol<83>;
pub type Item_15 = GeneratedParserSymbol<84>;
pub type Item_16 = GeneratedParserSymbol<85>;
pub type Item_16_1 = GeneratedParserSymbol<86>;
pub type Item_17 = GeneratedParserSymbol<87>;
pub type Item_18 = GeneratedParserSymbol<88>;
pub type Item_18_1 = GeneratedParserSymbol<89>;
pub type Item_19 = GeneratedParserSymbol<90>;
pub type Item_20 = GeneratedParserSymbol<91>;
pub type Item_20_1 = GeneratedParserSymbol<92>;
pub type Item_21 = GeneratedParserSymbol<93>;
pub type Item_22 = GeneratedParserSymbol<94>;
pub type Item_22_1 = GeneratedParserSymbol<95>;
pub type Item_23 = GeneratedParserSymbol<96>;
pub type Item_24 = GeneratedParserSymbol<97>;
pub type Item_24_1 = GeneratedParserSymbol<98>;
pub type GStuff = GeneratedParserSymbol<27>;
pub type Stuff = GeneratedParserSymbol<36>;
pub type Item = GeneratedParserSymbol<63>;
pub type ClassBody = GeneratedParserSymbol<24>;
pub type Cons = GeneratedParserSymbol<42>;
pub type Call = GeneratedParserSymbol<45>;
pub type BaseCall = GeneratedParserSymbol<48>;
pub type Name = GeneratedParserSymbol<57>;
#[derive(Clone, Debug)]
pub struct yycs0syntax(pub YyParser);
impl yycs0syntax {
pub fn new() -> Result<Self, Error> {
Ok(Self(generated_parser()?))
}
pub fn action(&self, parser: Parser, symbol: SYMBOL, action: i32) -> Result<Object, Error> {
self.0.action(parser, symbol, action)
}
pub fn class_body_1_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(ClassBody_1::new(parser)?))
}
pub fn class_body_2_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(ClassBody_2::new(parser)?))
}
pub fn class_body_2_1_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(ClassBody_2_1::new(parser)?))
}
pub fn g_stuff_1_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(GStuff_1::new(parser)?))
}
pub fn g_stuff_2_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(GStuff_2::new(parser)?))
}
pub fn g_stuff_2_1_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(GStuff_2_1::new(parser)?))
}
pub fn g_stuff_3_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(GStuff_3::new(parser)?))
}
pub fn g_stuff_4_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(GStuff_4::new(parser)?))
}
pub fn g_stuff_4_1_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(GStuff_4_1::new(parser)?))
}
pub fn g_stuff_5_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(GStuff_5::new(parser)?))
}
pub fn g_stuff_6_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(GStuff_6::new(parser)?))
}
pub fn g_stuff_6_1_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(GStuff_6_1::new(parser)?))
}
pub fn stuff_1_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(Stuff_1::new(parser)?))
}
pub fn stuff_2_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(Stuff_2::new(parser)?))
}
pub fn stuff_2_1_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(Stuff_2_1::new(parser)?))
}
pub fn stuff_3_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(Stuff_3::new(parser)?))
}
pub fn stuff_4_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(Stuff_4::new(parser)?))
}
pub fn stuff_4_1_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(Stuff_4_1::new(parser)?))
}
pub fn cons_1_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(Cons_1::new(parser)?))
}
pub fn cons_2_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(Cons_2::new(parser)?))
}
pub fn cons_2_1_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(Cons_2_1::new(parser)?))
}
pub fn call_1_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(Call_1::new(parser)?))
}
pub fn call_2_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(Call_2::new(parser)?))
}
pub fn call_2_1_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(Call_2_1::new(parser)?))
}
pub fn base_call_1_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(BaseCall_1::new(parser)?))
}
pub fn base_call_2_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(BaseCall_2::new(parser)?))
}
pub fn base_call_2_1_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(BaseCall_2_1::new(parser)?))
}
pub fn base_call_3_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(BaseCall_3::new(parser)?))
}
pub fn base_call_4_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(BaseCall_4::new(parser)?))
}
pub fn base_call_4_1_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(BaseCall_4_1::new(parser)?))
}
pub fn base_call_5_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(BaseCall_5::new(parser)?))
}
pub fn base_call_6_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(BaseCall_6::new(parser)?))
}
pub fn base_call_6_1_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(BaseCall_6_1::new(parser)?))
}
pub fn name_1_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(Name_1::new(parser)?))
}
pub fn name_2_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(Name_2::new(parser)?))
}
pub fn name_2_1_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(Name_2_1::new(parser)?))
}
pub fn name_3_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(Name_3::new(parser)?))
}
pub fn name_4_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(Name_4::new(parser)?))
}
pub fn name_4_1_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(Name_4_1::new(parser)?))
}
pub fn item_1_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(Item_1::new(parser)?))
}
pub fn item_2_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(Item_2::new(parser)?))
}
pub fn item_2_1_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(Item_2_1::new(parser)?))
}
pub fn item_3_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(Item_3::new(parser)?))
}
pub fn item_4_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(Item_4::new(parser)?))
}
pub fn item_4_1_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(Item_4_1::new(parser)?))
}
pub fn item_5_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(Item_5::new(parser)?))
}
pub fn item_6_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(Item_6::new(parser)?))
}
pub fn item_6_1_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(Item_6_1::new(parser)?))
}
pub fn item_7_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(Item_7::new(parser)?))
}
pub fn item_8_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(Item_8::new(parser)?))
}
pub fn item_8_1_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(Item_8_1::new(parser)?))
}
pub fn item_9_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(Item_9::new(parser)?))
}
pub fn item_10_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(Item_10::new(parser)?))
}
pub fn item_10_1_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(Item_10_1::new(parser)?))
}
pub fn item_11_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(Item_11::new(parser)?))
}
pub fn item_12_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(Item_12::new(parser)?))
}
pub fn item_12_1_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(Item_12_1::new(parser)?))
}
pub fn item_13_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(Item_13::new(parser)?))
}
pub fn item_14_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(Item_14::new(parser)?))
}
pub fn item_14_1_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(Item_14_1::new(parser)?))
}
pub fn item_15_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(Item_15::new(parser)?))
}
pub fn item_16_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(Item_16::new(parser)?))
}
pub fn item_16_1_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(Item_16_1::new(parser)?))
}
pub fn item_17_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(Item_17::new(parser)?))
}
pub fn item_18_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(Item_18::new(parser)?))
}
pub fn item_18_1_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(Item_18_1::new(parser)?))
}
pub fn item_19_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(Item_19::new(parser)?))
}
pub fn item_20_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(Item_20::new(parser)?))
}
pub fn item_20_1_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(Item_20_1::new(parser)?))
}
pub fn item_21_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(Item_21::new(parser)?))
}
pub fn item_22_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(Item_22::new(parser)?))
}
pub fn item_22_1_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(Item_22_1::new(parser)?))
}
pub fn item_23_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(Item_23::new(parser)?))
}
pub fn item_24_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(Item_24::new(parser)?))
}
pub fn item_24_1_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(Item_24_1::new(parser)?))
}
pub fn g_stuff_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(GStuff::new(parser)?))
}
pub fn stuff_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(Stuff::new(parser)?))
}
pub fn item_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(Item::new(parser)?))
}
pub fn class_body_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(ClassBody::new(parser)?))
}
pub fn cons_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(Cons::new(parser)?))
}
pub fn call_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(Call::new(parser)?))
}
pub fn base_call_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(BaseCall::new(parser)?))
}
pub fn name_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(Name::new(parser)?))
}
pub fn error_factory(parser: Parser) -> Result<Object, Error> {
Ok(Object::opaque(crate::LslError::new_with_parser(parser)?))
}
}
impl Deref for yycs0syntax {
type Target = YyParser;
fn deref(&self) -> &Self::Target {
&self.0
}
}
impl DerefMut for yycs0syntax {
fn deref_mut(&mut self) -> &mut Self::Target {
&mut self.0
}
}
#[derive(Clone, Debug)]
pub struct yycs0tokens(pub YyLexer);
impl yycs0tokens {
pub fn new(error_handler: ErrorHandler) -> Result<Self, Error> {
Ok(Self(generated_lexer_with_handler(error_handler)?))
}
pub fn base_factory(lexer: Lexer) -> Result<Object, Error> {
Ok(Object::opaque(BASE::new(lexer)?))
}
pub fn this_factory(lexer: Lexer) -> Result<Object, Error> {
Ok(Object::opaque(THIS::new(lexer)?))
}
pub fn new_factory(lexer: Lexer) -> Result<Object, Error> {
Ok(Object::opaque(NEW::new(lexer)?))
}
pub fn id_factory(lexer: Lexer) -> Result<Object, Error> {
Ok(Object::opaque(ID::new(lexer)?))
}
pub fn any_factory(lexer: Lexer) -> Result<Object, Error> {
Ok(Object::opaque(ANY::new(lexer)?))
}
pub fn colon_factory(lexer: Lexer) -> Result<Object, Error> {
Ok(Object::opaque(COLON::new(lexer)?))
}
pub fn semicolon_factory(lexer: Lexer) -> Result<Object, Error> {
Ok(Object::opaque(SEMICOLON::new(lexer)?))
}
pub fn lbrace_factory(lexer: Lexer) -> Result<Object, Error> {
Ok(Object::opaque(LBRACE::new(lexer)?))
}
pub fn rbrace_factory(lexer: Lexer) -> Result<Object, Error> {
Ok(Object::opaque(RBRACE::new(lexer)?))
}
pub fn lparen_factory(lexer: Lexer) -> Result<Object, Error> {
Ok(Object::opaque(LPAREN::new(lexer)?))
}
pub fn rparen_factory(lexer: Lexer) -> Result<Object, Error> {
Ok(Object::opaque(RPAREN::new(lexer)?))
}
pub fn lbrack_factory(lexer: Lexer) -> Result<Object, Error> {
Ok(Object::opaque(LBRACK::new(lexer)?))
}
pub fn rbrack_factory(lexer: Lexer) -> Result<Object, Error> {
Ok(Object::opaque(RBRACK::new(lexer)?))
}
pub fn old_action(
&self,
lexer: Lexer,
yytext: &mut String,
action: i32,
reject: &mut bool,
) -> Result<TOKEN, Error> {
if matches!(action, 8 | 55 | 74) {
match action {
8 => "yym.yy_begin",
55 => "((cs0tokens)yym)",
_ => "((cs0syntax)yyq)",
}
.clone_into(yytext);
return TOKEN::generated_with_lexer(lexer, "ANY", 7);
}
*reject = true;
Err(Error::InvalidOperation)
}
}
impl Deref for yycs0tokens {
type Target = YyLexer;
fn deref(&self) -> &Self::Target {
&self.0
}
}
impl DerefMut for yycs0tokens {
fn deref_mut(&mut self) -> &mut Self::Target {
&mut self.0
}
}
#[derive(Clone, Debug)]
pub struct cs0tokens {
pub out: String,
pub lexer: Lexer,
}
impl cs0tokens {
pub fn new() -> Result<Self, Error> {
Self::new_with_yy_lexer(generated_lexer()?)
}
pub fn new_with_constructor() -> Result<Self, Error> {
Self::new()
}
pub fn new_with_error_handler(handler: ErrorHandler) -> Result<Self, Error> {
Self::new_with_yy_lexer(generated_lexer_with_handler(handler)?)
}
pub fn new_with_yy_lexer(tokens: YyLexer) -> Result<Self, Error> {
Ok(Self {
out: String::new(),
lexer: Lexer::new(tokens)?,
})
}
}
impl Deref for cs0tokens {
type Target = Lexer;
fn deref(&self) -> &Self::Target {
&self.lexer
}
}
impl DerefMut for cs0tokens {
fn deref_mut(&mut self) -> &mut Self::Target {
&mut self.lexer
}
}
#[derive(Clone, Debug)]
pub struct cs0syntax {
pub out: String,
pub cls: String,
pub par: String,
pub ctx: String,
pub defconseen: bool,
pub parser: Parser,
}
impl cs0syntax {
pub fn new() -> Result<Self, Error> {
Self::new_with_yy_parser(generated_parser()?)
}
pub fn new_with_constructor() -> Result<Self, Error> {
Self::new()
}
pub fn new_with_yy_parser(symbols: YyParser) -> Result<Self, Error> {
let lexer = Lexer::new(generated_lexer()?)?;
Self::from_parts(symbols, lexer)
}
pub fn new_with_yy_parser_error_handler(
mut symbols: YyParser,
handler: ErrorHandler,
) -> Result<Self, Error> {
symbols.erh = handler.clone();
let lexer = Lexer::new(generated_lexer_with_handler(handler)?)?;
Self::from_parts(symbols, lexer)
}
fn from_parts(symbols: YyParser, lexer: Lexer) -> Result<Self, Error> {
Ok(Self {
out: String::new(),
cls: String::new(),
par: String::new(),
ctx: String::new(),
defconseen: false,
parser: Parser::new(symbols, lexer)?,
})
}
}
impl Deref for cs0syntax {
type Target = Parser;
fn deref(&self) -> &Self::Target {
&self.parser
}
}
impl DerefMut for cs0syntax {
fn deref_mut(&mut self) -> &mut Self::Target {
&mut self.parser
}
}

1337
crates/libremetaverse-lsl-tools/src/generator.rs Normal file
View File

File diff suppressed because it is too large Load Diff

100
crates/libremetaverse-lsl-tools/src/lexer.rs
View File

@@ -1483,6 +1483,21 @@ impl DfaAction {
} }
/// Runtime deterministic finite automaton. /// Runtime deterministic finite automaton.
#[derive(Clone, Debug)]
struct RegexDfa {
source: String,
compiled: regex::Regex,
accept: DfaAccept,
}
impl PartialEq for RegexDfa {
fn eq(&self, other: &Self) -> bool {
self.source == other.source && self.accept == other.accept
}
}
impl Eq for RegexDfa {}
#[derive(Clone, Debug, Eq, PartialEq)] #[derive(Clone, Debug, Eq, PartialEq)]
pub struct Dfa { pub struct Dfa {
pub m_actions: Vec<DfaAction>, pub m_actions: Vec<DfaAction>,
@@ -1491,6 +1506,7 @@ pub struct Dfa {
pub m_tok_class: String, pub m_tok_class: String,
states: Vec<DfaState>, states: Vec<DfaState>,
start: usize, start: usize,
regex: Option<RegexDfa>,
} }
impl Default for Dfa { impl Default for Dfa {
@@ -1502,6 +1518,7 @@ impl Default for Dfa {
m_tok_class: String::new(), m_tok_class: String::new(),
states: vec![DfaState::default()], states: vec![DfaState::default()],
start: 0, start: 0,
regex: None,
} }
} }
} }
@@ -1525,17 +1542,54 @@ impl Dfa {
}) })
} }
pub fn new_with_nfa(_nfa: Nfa) -> Result<Self, Error> { pub fn new_with_nfa(nfa: Nfa) -> Result<Self, Error> {
// The NFA-to-DFA generator is issue #83; the runtime never manufactures let name = if nfa.m_end.m_s_terminal.is_empty() {
// fake transitions from an opaque generated NFA. "ANY".to_owned()
Err(Error::InvalidOperation) } else {
nfa.m_end.m_s_terminal.clone()
};
let accept = DfaAccept {
token: TokenDefinition::new(name, 7)?,
action: LexerAction::Emit,
action_number: 0,
reserved_words: None,
};
if let Some(source) = nfa.regex_source()? {
let compiled =
regex::Regex::new(&format!("^(?:{source})")).map_err(|_| Error::Parse {
position: 0,
context: "invalid NFA regular expression",
})?;
return Ok(Self {
regex: Some(RegexDfa {
source,
compiled,
accept,
}),
..Self::default()
});
}
Self::from_states(nfa.deterministic_states(accept)?, 0)
} }
pub fn new_with_tokens_gen(_tokens: TokensGen) -> Result<Self, Error> { pub fn new_with_tokens_gen(_tokens: TokensGen) -> Result<Self, Error> {
Err(Error::InvalidOperation) // The C# constructor creates the initial, unpopulated determinisation
// node. NFA-based construction is exposed separately by
// `new_with_nfa`; returning an empty native state here preserves that
// useful distinction without manufacturing an NFA transition.
Ok(Self::default())
} }
fn longest_match(&self, units: &[u16], offset: usize) -> Option<(usize, DfaAccept)> { fn longest_match(&self, units: &[u16], offset: usize) -> Option<(usize, DfaAccept)> {
if let Some(regex) = &self.regex {
let text = String::from_utf16(units.get(offset..)?).ok()?;
let found = regex.compiled.find(&text)?;
if found.start() != 0 || found.end() == 0 {
return None;
}
let length = text[..found.end()].encode_utf16().count();
return Some((length, regex.accept.clone()));
}
let mut state_index = self.start; let mut state_index = self.start;
let mut cursor = offset; let mut cursor = offset;
let mut accepted = self.states[state_index] let mut accepted = self.states[state_index]
@@ -1587,6 +1641,15 @@ impl Dfa {
#[must_use] #[must_use]
pub fn stable_description(&self) -> String { pub fn stable_description(&self) -> String {
if let Some(regex) = &self.regex {
return format!(
"regex {:?} accept {} {} {}",
regex.source,
regex.accept.token.number,
regex.accept.token.name,
regex.accept.action_number
);
}
let mut lines = Vec::new(); let mut lines = Vec::new();
for (state_index, state) in self.states.iter().enumerate() { for (state_index, state) in self.states.iter().enumerate() {
if let Some(accept) = &state.accept { if let Some(accept) = &state.accept {
@@ -2185,6 +2248,9 @@ impl SYMBOL {
pub fn set_yylval(&mut self, value: Object) { pub fn set_yylval(&mut self, value: Object) {
self.m_dollar = value; self.m_dollar = value;
} }
pub(crate) fn raw_text(&self) -> String {
self.text.clone()
}
#[must_use] #[must_use]
pub fn yyname(&self) -> String { pub fn yyname(&self) -> String {
self.name.clone() self.name.clone()
@@ -2215,6 +2281,30 @@ pub struct TOKEN {
} }
impl TOKEN { impl TOKEN {
pub(crate) fn generated_with_lexer(
lexer: Lexer,
name: &str,
number: i32,
) -> Result<Self, Error> {
let mut token = Self::new_with_lexer(lexer)?;
name.clone_into(&mut token.name);
token.number = number;
Ok(token)
}
pub(crate) fn generated_with_parser(
parser: Parser,
name: &str,
number: i32,
text: String,
) -> Result<Self, Error> {
let mut token = Self::new_with_parser(parser)?;
name.clone_into(&mut token.name);
token.number = number;
token.text = text;
Ok(token)
}
pub fn new_with_lexer(lexer: Lexer) -> Result<Self, Error> { pub fn new_with_lexer(lexer: Lexer) -> Result<Self, Error> {
let text = lexer.yytext.clone(); let text = lexer.yytext.clone();
Self::new_with_lexer_string(lexer, text) Self::new_with_lexer_string(lexer, text)

3
crates/libremetaverse-lsl-tools/src/lib.rs
View File

@@ -3,10 +3,13 @@
extern crate self as libremetaverse_lsl_tools; extern crate self as libremetaverse_lsl_tools;
mod generated; mod generated;
mod generated_tables;
mod generator;
mod lexer; mod lexer;
mod parser; mod parser;
pub use generated::*; pub use generated::*;
pub use generated_tables::*;
pub use lexer::{ pub use lexer::{
CharacterMatcher, DfaAccept, DfaState, Diagnostic, DiagnosticCategory, DiagnosticSeverity, CharacterMatcher, DfaAccept, DfaState, Diagnostic, DiagnosticCategory, DiagnosticSeverity,
DotNetUnicodeCategory, InputEncoding, LexerAction, LexerIterator, MAX_SOURCE_UNITS, DotNetUnicodeCategory, InputEncoding, LexerAction, LexerIterator, MAX_SOURCE_UNITS,

8
crates/libremetaverse-lsl-tools/src/parser.rs
View File

@@ -1006,6 +1006,10 @@ impl SymbolSet {
} }
} }
pub(crate) fn compatibility_empty() -> Self {
Self::empty()
}
/// Creates an empty set independent of generator state. /// Creates an empty set independent of generator state.
#[must_use] #[must_use]
pub fn native() -> Self { pub fn native() -> Self {
@@ -1646,6 +1650,10 @@ impl ParseState {
} }
} }
pub(crate) fn compatibility_numbered(state: i32) -> Self {
Self::numbered(state)
}
pub fn new(symbols: SymbolsGen, accessing: CSymbol) -> Result<Self, Error> { pub fn new(symbols: SymbolsGen, accessing: CSymbol) -> Result<Self, Error> {
Ok(Self { Ok(Self {
m_accessing_symbol: accessing, m_accessing_symbol: accessing,

341
crates/libremetaverse-lsl-tools/tests/generator_compat.rs Normal file
View File

@@ -0,0 +1,341 @@
//! Focused compatibility fixtures for issue #83's checked-in LSL generator.
use std::collections::HashMap;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex};
use libremetaverse_lsl_tools::{
BASE, CSymbol, CSymbolSymType, Dfa, ErrorHandler, GENERATED_PARSER_DATA, ID,
LSL_GENERATOR_SERIALIZATION_VERSION, Lexer, Nfa, NfaNode, ObjectList,
ObjectListOListEnumerator, Path, Regex, SCreator, Serialiser, Sfactory, SymbolType, SymbolsGen,
TCreator, Tfactory, TokClassDef, TokensGen, generated_lexer, generated_parser,
};
use libremetaverse_types::compat::{Object, Utf16CodeUnit};
struct SharedWriter(Arc<Mutex<Vec<u8>>>);
impl std::io::Write for SharedWriter {
fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {
self.0
.lock()
.map_err(|_| std::io::Error::other("poisoned output"))?
.extend_from_slice(bytes);
Ok(bytes.len())
}
fn flush(&mut self) -> std::io::Result<()> {
Ok(())
}
}
fn capture<F>(function: F) -> Vec<u8>
where
F: FnOnce(Box<dyn std::io::Write + Send>),
{
let output = Arc::new(Mutex::new(Vec::new()));
function(Box::new(SharedWriter(output.clone())));
Arc::try_unwrap(output)
.expect("single output owner")
.into_inner()
.expect("output")
}
#[test]
fn generated_lexer_preserves_tokens_reserved_words_and_eof() {
let mut lexer = Lexer::new(generated_lexer().expect("generated lexer")).expect("runtime");
lexer
.start_with_string("Widget(value;):base(new Child[])".to_owned())
.expect("source");
let mut tokens = Vec::new();
while let Some(token) = lexer.next_token().expect("lexing") {
tokens.push((token.yyname(), token.yytext(), token.yynum()));
}
assert_eq!(
tokens,
[
("ID".to_owned(), "Widget".to_owned(), 6),
("LPAREN".to_owned(), "(".to_owned(), 12),
("ID".to_owned(), "value".to_owned(), 6),
("SEMICOLON".to_owned(), ";".to_owned(), 9),
("RPAREN".to_owned(), ")".to_owned(), 13),
("COLON".to_owned(), ":".to_owned(), 8),
("BASE".to_owned(), "base".to_owned(), 3),
("LPAREN".to_owned(), "(".to_owned(), 12),
("NEW".to_owned(), "new".to_owned(), 5),
("ID".to_owned(), "Child".to_owned(), 6),
("LBRACK".to_owned(), "[".to_owned(), 14),
("RBRACK".to_owned(), "]".to_owned(), 15),
("RPAREN".to_owned(), ")".to_owned(), 13),
("EOF".to_owned(), "EOF".to_owned(), 2),
]
);
}
#[test]
fn generated_parser_accepts_the_reviewed_class_body_grammar() {
let table = generated_parser().expect("generated parser");
assert_eq!(table.arr, GENERATED_PARSER_DATA);
let lexer = Lexer::new(generated_lexer().expect("generated lexer")).expect("runtime");
let mut parser = libremetaverse_lsl_tools::Parser::new(table, lexer).expect("parser");
let result = parser
.parse_with_string("Widget(value;):base(new Child[])".to_owned())
.expect("class body");
assert_eq!(result.yyname(), "ClassBody");
assert_eq!(result.yynum(), 19);
}
#[test]
fn parser_and_lexer_emission_is_byte_deterministic() {
let first_parser = capture(|output| {
generated_parser()
.expect("parser")
.emit(output)
.expect("emit");
});
let second_parser = capture(|output| {
generated_parser()
.expect("parser")
.emit(output)
.expect("emit");
});
assert_eq!(first_parser, second_parser);
let first_lexer = capture(|output| {
generated_lexer()
.expect("lexer")
.emit_dfa(output)
.expect("emit");
});
let second_lexer = capture(|output| {
generated_lexer()
.expect("lexer")
.emit_dfa(output)
.expect("emit");
});
assert_eq!(first_lexer, second_lexer);
}
#[test]
fn generated_token_classes_have_live_identity_and_text() {
let mut lexer = Lexer::new(generated_lexer().expect("lexer")).expect("runtime");
lexer
.start_with_string("base name".to_owned())
.expect("source");
let base = lexer.next_token().expect("lexing").expect("base");
let mapped_base = BASE::new(lexer.clone()).expect("mapped base");
assert_eq!((base.yyname(), base.yynum()), ("BASE".to_owned(), 3));
assert_eq!(
(mapped_base.yyname(), mapped_base.yynum()),
("BASE".to_owned(), 3)
);
let _ = lexer.next_token().expect("lexing").expect("name");
let mapped_id = ID::new(lexer).expect("mapped id");
assert_eq!(
(mapped_id.yyname(), mapped_id.yynum()),
("ID".to_owned(), 6)
);
}
#[test]
fn regex_and_nfa_build_a_functional_native_dfa() {
let handler = ErrorHandler::default();
let tokens = TokensGen::new(handler.clone()).expect("generator");
let regex = Regex::new(tokens.clone(), 0, "[A-Z][a-z]+".to_owned()).expect("regex");
assert!(
!regex
.match__with_char(Utf16CodeUnit(u16::from(b'Q')))
.expect("single-unit mismatch")
);
assert!(
!regex
.match__with_char(Utf16CodeUnit(u16::from(b'q')))
.expect("mismatch")
);
assert_eq!(
regex
.match__with_string("Zebra".to_owned())
.expect("prefix"),
5
);
let mut nfa = Nfa::new_with_tokens_gen_regex(tokens, regex).expect("nfa");
nfa.m_end.m_s_terminal = "CAPITAL".to_owned();
let dfa = Dfa::new_with_nfa(nfa).expect("dfa");
let mut table = libremetaverse_lsl_tools::YyLexer::new(handler).expect("table");
table.set_start_dfa("YYINITIAL", dfa).expect("start");
table.using_eof = true;
let mut lexer = Lexer::new(table).expect("runtime");
lexer.start_with_string("Rust".to_owned()).expect("source");
let token = lexer.next_token().expect("lexing").expect("token");
assert_eq!(
(token.yyname(), token.yytext()),
("CAPITAL".to_owned(), "Rust".to_owned())
);
let tokens = TokensGen::new(ErrorHandler::default()).expect("manual generator");
let mut manual = Nfa::new_with_tokens_gen(tokens.clone()).expect("manual nfa");
manual.m_end.m_s_terminal = "PAIR".to_owned();
let middle = NfaNode::new(tokens).expect("middle state");
manual
.start
.add_arc(Utf16CodeUnit(u16::from(b'a')), middle.clone())
.expect("first arc");
middle
.add_arc(Utf16CodeUnit(u16::from(b'b')), manual.m_end.clone())
.expect("second arc");
let dfa = Dfa::new_with_nfa(manual).expect("determinised nfa");
let mut action = -1;
assert_eq!(
dfa.match_("ab".to_owned(), 0, &mut action).expect("pair"),
2
);
}
#[test]
fn serialiser_round_trips_deterministically_with_versioning() {
let value = Object::Map(HashMap::from([
("b".to_owned(), Object::Integer(7)),
(
"a".to_owned(),
Object::Array(vec![Object::Boolean(true), Object::String("λ".to_owned())]),
),
]));
let encoded = capture(|output| {
let serialiser = Serialiser::new_with_text_writer(output).expect("writer");
serialiser.version_check().expect("version");
serialiser.serialise(value.clone()).expect("serialise");
});
let encoded_again = capture(|output| {
let serialiser = Serialiser::new_with_text_writer(output).expect("writer");
serialiser.version_check().expect("version");
serialiser.serialise(value.clone()).expect("serialise");
});
assert_eq!(encoded, encoded_again);
let integers = String::from_utf8(encoded)
.expect("ASCII integers")
.split(',')
.filter(|value| !value.trim().is_empty())
.map(|value| value.trim().parse::<i32>().expect("integer"))
.collect::<Vec<_>>();
let decoder = Serialiser::new_with_int32_array(integers).expect("reader");
decoder
.version_check()
.expect(LSL_GENERATOR_SERIALIZATION_VERSION);
assert_eq!(decoder.deserialise().expect("deserialise"), value);
}
#[test]
fn compatibility_enumerator_and_generator_state_are_live() {
let mut values = ObjectList::new().expect("list");
values.add(Object::Integer(1)).expect("first");
values
.add(Object::String("two".to_owned()))
.expect("second");
let enumerator = ObjectListOListEnumerator::new(values).expect("enumerator");
assert_eq!(enumerator.current(), Object::Undefined);
assert!(enumerator.move_next().expect("first"));
assert_eq!(enumerator.current(), Object::Integer(1));
assert!(enumerator.move_next().expect("second"));
assert_eq!(enumerator.current(), Object::String("two".to_owned()));
assert!(!enumerator.move_next().expect("end"));
enumerator.reset().expect("reset");
assert!(enumerator.move_next().expect("first again"));
let generator = TokensGen::new(ErrorHandler::default()).expect("generator");
assert_eq!(generator.new_state().expect("state one"), 1);
assert_eq!(generator.new_state().expect("state two"), 2);
assert_eq!(
generator
.fix_actions("yybegin(); yyl".to_owned())
.expect("rewritten action"),
"yym.yy_begin(); ((tokens)yym)"
);
assert!(Dfa::new_with_tokens_gen(generator).is_ok());
}
#[test]
fn symbol_and_token_factories_execute_registered_rust_closures() {
let parser = libremetaverse_lsl_tools::Parser::new(
generated_parser().expect("parser table"),
Lexer::new(generated_lexer().expect("lexer table")).expect("lexer"),
)
.expect("parser");
let symbol_creator = SCreator::from_fn(|_| Ok(Object::String("symbol".to_owned())));
let registration = Sfactory::new(
generated_parser().expect("parser table"),
"GeneratedSymbol".to_owned(),
symbol_creator,
)
.expect("symbol registration");
assert_eq!(registration.name(), "GeneratedSymbol");
assert_eq!(
Sfactory::create("GeneratedSymbol_Derived".to_owned(), parser).expect("symbol factory"),
Object::String("symbol".to_owned())
);
let lexer = Lexer::new(generated_lexer().expect("lexer table")).expect("lexer");
let token_creator = TCreator::from_fn(|_| Ok(Object::Integer(83)));
let registration = Tfactory::new(
generated_lexer().expect("lexer table"),
"GeneratedToken".to_owned(),
token_creator,
)
.expect("token registration");
assert_eq!(registration.name(), "GeneratedToken");
assert_eq!(
Tfactory::create("GeneratedToken_Derived".to_owned(), lexer).expect("token factory"),
Object::Integer(83)
);
let callback_ran = Arc::new(AtomicBool::new(false));
let callback_state = callback_ran.clone();
let async_creator = TCreator::from_fn(|_| Ok(Object::String("async".to_owned())));
let result = async_creator
.begin_invoke(
Lexer::new(generated_lexer().expect("lexer table")).expect("lexer"),
Box::new(move |_| callback_state.store(true, Ordering::SeqCst)),
Object::Undefined,
)
.expect("begin invoke");
assert!(callback_ran.load(Ordering::SeqCst));
assert_eq!(
async_creator.end_invoke(result).expect("end invoke"),
Object::String("async".to_owned())
);
}
#[test]
fn symbols_paths_and_token_class_definitions_keep_native_state() {
let symbols = SymbolsGen::new(ErrorHandler::default()).expect("symbols generator");
let declared = SymbolType::new_with_symbols_gen_string_boolean(
symbols.clone(),
"SemanticValue".to_owned(),
true,
)
.expect("symbol type");
assert_eq!(
declared
.find("SemanticValue".to_owned())
.expect("lookup")
.name,
"SemanticValue"
);
let identifier = CSymbol::native("ID", 6, CSymbolSymType::Terminal).expect("symbol");
let path = Path::new_with_c_symbol_array(vec![identifier.clone()]).expect("path");
assert_eq!(path.spelling()[0].m_yynum, 6);
assert!(!path.valid);
assert_eq!(path.top().m_state, 0);
let output = Arc::new(Mutex::new(Vec::new()));
let generator = libremetaverse_lsl_tools::GenBase::new(
ErrorHandler::default(),
Box::new(SharedWriter(output)),
)
.expect("generator base");
let definition = TokClassDef::new(generator, "IDENTIFIER".to_owned(), "TOKEN".to_owned())
.expect("token class");
assert_eq!(definition.m_name, "IDENTIFIER");
assert_eq!(definition.m_ref_token, "TOKEN");
assert_eq!(definition.m_yynum, 3);
}

39
tests/api-compile/Cargo.lock generated
View File

@@ -8,6 +8,15 @@ version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
[[package]]
name = "aho-corasick"
version = "1.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba"
dependencies = [
"memchr",
]
[[package]] [[package]]
name = "android_system_properties" name = "android_system_properties"
version = "0.1.6" version = "0.1.6"
@@ -783,6 +792,7 @@ name = "libremetaverse-lsl-tools"
version = "0.0.1" version = "0.0.1"
dependencies = [ dependencies = [
"libremetaverse-types", "libremetaverse-types",
"regex",
"unicode-general-category", "unicode-general-category",
] ]
@@ -1336,6 +1346,35 @@ dependencies = [
"rand_core", "rand_core",
] ]
[[package]]
name = "regex"
version = "1.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d"
dependencies = [
"aho-corasick",
"memchr",
"regex-automata",
"regex-syntax",
]
[[package]]
name = "regex-automata"
version = "0.4.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2"
dependencies = [
"aho-corasick",
"memchr",
"regex-syntax",
]
[[package]]
name = "regex-syntax"
version = "0.8.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4"
[[package]] [[package]]
name = "reqwest" name = "reqwest"
version = "0.13.4" version = "0.13.4"

204
tools/check_milestone_10_issue_83.py Normal file
View File

@@ -0,0 +1,204 @@
#!/usr/bin/env python3
"""Audit issue 83's deterministic native LSL parser/token generation."""
from __future__ import annotations
import json
import re
from pathlib import Path
import generate_api_shims
import generate_lsl_tables
ROOT = Path(__file__).resolve().parents[1]
CATALOG = ROOT / "api" / "public-api.json"
INPUT = ROOT / "codegen" / "inputs" / "lsl_tools_grammar.json"
TABLES = ROOT / "crates" / "libremetaverse-lsl-tools" / "src" / "generated_tables.rs"
GENERATOR = ROOT / "crates" / "libremetaverse-lsl-tools" / "src" / "generator.rs"
GENERATED = ROOT / "crates" / "libremetaverse-lsl-tools" / "src" / "generated.rs"
TESTS = ROOT / "crates" / "libremetaverse-lsl-tools" / "tests" / "generator_compat.rs"
DOC = ROOT / "crates" / "libremetaverse-lsl-tools" / "README.md"
WORKFLOW = ROOT / ".gitea" / "workflows" / "rust-workspace.yml"
SCRIPT = ROOT / "tools" / "generate_lsl_tables.py"
STUB_RE = re.compile(
r"\b(?:not_implemented|unimplemented_api)\b|\b(?:todo|unimplemented)!\s*\("
)
def owned_types() -> dict[str, str]:
return {
doc_id: declaration
for doc_id, declaration in generate_api_shims.NATIVE_TYPES.items()
if declaration.startswith("crate::generated_tables::")
or declaration.startswith("crate::generator::")
}
def catalog_counts(doc_ids: set[str]) -> tuple[int, int]:
catalog = json.loads(CATALOG.read_text())
assembly = next(
value
for value in catalog["assemblies"]
if value["identity"]["name"] == "LibreMetaverse.LslTools"
)
found: dict[str, int] = {}
def visit(value: object) -> None:
if isinstance(value, dict):
doc_id = value.get("doc_id")
if doc_id in doc_ids and str(doc_id).startswith("T:"):
found[str(doc_id)] = len(value.get("members", []))
for child in value.values():
visit(child)
elif isinstance(value, list):
for child in value:
visit(child)
visit(assembly["types"])
if set(found) != doc_ids:
missing = sorted(doc_ids - set(found))
raise SystemExit(f"issue 83 catalog types are missing: {missing}")
return len(found), sum(found.values())
def generated_type_block(text: str, doc_id: str) -> str:
marker = f"/// C# type: `{doc_id}`."
start = text.find(marker)
if start < 0:
raise SystemExit(f"generated declaration is missing for {doc_id}")
next_type = text.find("/// C# type:", start + len(marker))
return text[start:] if next_type < 0 else text[start:next_type]
def require_markers(path: Path, markers: tuple[str, ...]) -> None:
text = path.read_text()
missing = [marker for marker in markers if marker not in text]
if missing:
raise SystemExit(f"{path.name}: audit evidence missing: " + ", ".join(missing))
def main() -> None:
types = owned_types()
type_count, member_count = catalog_counts(set(types))
if (type_count, member_count) != (120, 365):
raise SystemExit(
f"issue 83 expected 120 mapped types and 365 members, found "
f"{type_count} and {member_count}"
)
data = json.loads(INPUT.read_text())
if data["source"].get("license") != "BSD-3-Clause":
raise SystemExit("issue 83 reviewed grammar input has no BSD-3-Clause provenance")
if len(data["tokens"]) != 14 or len(data["nonterminals"]) != 8:
raise SystemExit("issue 83 grammar must contain 14 tokens and 8 nonterminals")
if len(data["productions"]) != 25:
raise SystemExit("issue 83 grammar must contain all 25 reviewed productions")
first = generate_lsl_tables.render(data).encode()
second = generate_lsl_tables.render(json.loads(INPUT.read_text())).encode()
if first != second or first != TABLES.read_bytes():
raise SystemExit("two clean LSL generations were not byte-identical")
for path in (TABLES, GENERATOR):
source = path.read_text()
if STUB_RE.search(source):
raise SystemExit(f"issue 83 owned Rust stubs remain in {path.name}")
if re.search(r"unsafe\s*\{|unsafe\s+impl|target_os\s*=\s*\"macos\"", source):
raise SystemExit(f"{path.name} contains unsafe or macOS-only code")
script = SCRIPT.read_text()
if re.search(r"\b(?:dotnet|csc|mcs)\b", script, re.IGNORECASE):
raise SystemExit("LSL regeneration depends on a C#/.NET process")
generated = GENERATED.read_text()
for doc_id in types:
if STUB_RE.search(generated_type_block(generated, doc_id)):
raise SystemExit(f"issue 83 generated stubs remain for {doc_id}")
require_markers(
TABLES,
(
"LSL_GENERATOR_SOURCE_COMMIT",
"LSL_GENERATOR_SOURCE_PARSER",
"LSL_GENERATOR_SOURCE_LEXER",
"LSL_GENERATOR_SOURCE_LICENSE",
"LSL_GENERATOR_SERIALIZATION_VERSION",
"GENERATED_PARSER_DATA",
"pub fn generated_parser()",
"pub fn generated_lexer_with_handler",
"pub struct GeneratedLexerToken",
"pub struct GeneratedParserSymbol",
"pub struct yycs0syntax",
"pub struct yycs0tokens",
"pub struct cs0syntax",
"pub struct cs0tokens",
),
)
require_markers(
GENERATOR,
(
"pub struct Serialiser",
"pub struct GenBase",
"pub struct SymbolsGen",
"pub struct TokensGen",
"pub struct Regex",
"pub struct Nfa",
"pub struct Sfactory",
"pub struct Tfactory",
"pub struct TokClassDef",
"pub struct ObjectListOListEnumerator",
"pub struct AddToFunc",
"MAX_GENERATED_DFA_STATES",
"MAX_REGEX_UNITS",
"System.Char",
),
)
require_markers(
TESTS,
(
"generated_lexer_preserves_tokens_reserved_words_and_eof",
"generated_parser_accepts_the_reviewed_class_body_grammar",
"parser_and_lexer_emission_is_byte_deterministic",
"generated_token_classes_have_live_identity_and_text",
"regex_and_nfa_build_a_functional_native_dfa",
"serialiser_round_trips_deterministically_with_versioning",
"compatibility_enumerator_and_generator_state_are_live",
"symbol_and_token_factories_execute_registered_rust_closures",
"symbols_paths_and_token_class_definitions_keep_native_state",
),
)
if TESTS.read_text().count("#[test]") != 9:
raise SystemExit("issue 83 expected 9 focused generator fixtures")
require_markers(
DOC,
(
"Checked-in generator contract",
"25-production grammar",
"84 generated syntax",
"13 generated token classes",
"python3 tools/generate_lsl_tables.py --check",
"46 focused",
"ubuntu-latest",
),
)
workflow = WORKFLOW.read_text()
require_markers(
WORKFLOW,
(
"python3 tools/check_milestone_10_issue_83.py",
"python3 tools/generate_lsl_tables.py --check",
),
)
if "runs-on: ubuntu-latest" not in workflow or re.search(
r"runs-on:\s*(?:macos|windows)", workflow, re.IGNORECASE
):
raise SystemExit("issue 83 workflow is not ubuntu-only")
print(
"issue 83 audit: 120 native mapped generator/table types and 365 members, "
"14 tokens, 8 nonterminals, 25 productions, two byte-identical generations, "
"9 focused fixtures, migration docs, and ubuntu-only CI are present"
)
if __name__ == "__main__":
main()

33
tools/generate_api_shims.py
View File

@@ -511,6 +511,39 @@ NATIVE_TYPES = {
"T:LibreMetaverse.Rendering.LindenSkeleton": "crate::skeleton::LindenSkeleton", "T:LibreMetaverse.Rendering.LindenSkeleton": "crate::skeleton::LindenSkeleton",
} }
# The cs0 parser/token classes are generated from the reviewed Rust grammar
# input. Keep this list algorithmic so adding an alternative cannot leave a
# hand-maintained compatibility shell behind.
_LSL_GENERATED_TYPES = [
"ANY", "BASE", "COLON", "ID", "LBRACE", "LBRACK", "LPAREN", "NEW",
"RBRACE", "RBRACK", "RPAREN", "SEMICOLON", "THIS",
]
for _lhs, _alternatives in (
("ClassBody", 1), ("GStuff", 3), ("Stuff", 2), ("Cons", 1),
("Call", 1), ("BaseCall", 3), ("Name", 2), ("Item", 12),
):
_LSL_GENERATED_TYPES.append(_lhs)
for _alternative in range(1, _alternatives + 1):
_LSL_GENERATED_TYPES.extend((
f"{_lhs}_{_alternative * 2 - 1}",
f"{_lhs}_{_alternative * 2}",
f"{_lhs}_{_alternative * 2}_1",
))
_LSL_GENERATED_TYPES.extend(("cs0syntax", "cs0tokens", "yycs0syntax", "yycs0tokens"))
NATIVE_TYPES.update({
f"T:LibreMetaverse.LslTools.{name}": f"crate::generated_tables::{name}"
for name in _LSL_GENERATED_TYPES
})
for _name in (
"AddToFunc", "Builder", "Func", "GenBase", "LNode", "Nfa", "NfaNode",
"Path", "Regex", "Relation", "SCreator", "Serialiser", "Sfactory",
"SymbolType", "SymbolsGen", "TCreator", "Tfactory", "TokClassDef", "TokensGen",
):
NATIVE_TYPES[f"T:LibreMetaverse.LslTools.{_name}"] = f"crate::generator::{_name}"
NATIVE_TYPES["T:LibreMetaverse.LslTools.ObjectList.OListEnumerator"] = (
"crate::generator::ObjectListOListEnumerator"
)
# Native declarations whose cataloged method implementations are still emitted # Native declarations whose cataloged method implementations are still emitted
# below. This is used for static namespace types such as OSDParser: the type is # below. This is used for static namespace types such as OSDParser: the type is
# hand-written, while its fixed public methods remain generator-audited. # hand-written, while its fixed public methods remain generator-audited.

389
tools/generate_lsl_tables.py Normal file
View File

@@ -0,0 +1,389 @@
#!/usr/bin/env python3
"""Generate native Rust LSL parser/token types from the reviewed grammar."""
from __future__ import annotations
import argparse
import json
import subprocess
from pathlib import Path
import generate_rust_mapping as mapping
ROOT = Path(__file__).resolve().parents[1]
INPUT = ROOT / "codegen" / "inputs" / "lsl_tools_grammar.json"
OUTPUT = (
ROOT / "crates" / "libremetaverse-lsl-tools" / "src" / "generated_tables.rs"
)
def rust_string(value: str) -> str:
return json.dumps(value, ensure_ascii=True)
def format_rust(source: str) -> str:
"""Apply the workspace's canonical Rust formatting deterministically."""
result = subprocess.run(
["rustfmt", "--edition", "2024", "--emit", "stdout"],
input=source,
text=True,
capture_output=True,
check=False,
)
if result.returncode != 0:
raise SystemExit(f"rustfmt rejected generated LSL Rust:\n{result.stderr}")
return result.stdout
def semantic_expression(action: str) -> str:
expressions = {
"ClassBody_2_1": "stack_text(parser, 1)?",
"GStuff_2_1": "String::new()",
"GStuff_4_1": "format!(\"{}{}\", stack_text(parser, 1)?, stack_text(parser, 0)?)",
"GStuff_6_1": "format!(\"{}{}\", stack_text(parser, 1)?, stack_text(parser, 0)?)",
"Stuff_2_1": "String::new()",
"Stuff_4_1": "format!(\"{}{}\", stack_text(parser, 1)?, stack_text(parser, 0)?)",
"Cons_2_1": "format!(\"{}({}){}\", stack_text(parser, 4)?.trim(), stack_text(parser, 2)?, stack_text(parser, 0)?)",
"Call_2_1": "format!(\"{}({})\", stack_text(parser, 3)?.trim(), stack_text(parser, 1)?)",
"BaseCall_2_1": "String::new()",
"BaseCall_4_1": "format!(\"base{}\", stack_text(parser, 1)?)",
"BaseCall_6_1": "format!(\"this{}\", stack_text(parser, 1)?)",
"Name_2_1": "format!(\" {} \", stack_text(parser, 0)?)",
"Name_4_1": "format!(\"{}[{}]\", stack_text(parser, 3)?, stack_text(parser, 1)?)",
"Item_2_1": "stack_text(parser, 0)?",
"Item_4_1": "stack_text(parser, 0)?",
"Item_6_1": "\";\\n\".to_owned()",
"Item_8_1": "\" base \".to_owned()",
"Item_10_1": "\" this \".to_owned()",
"Item_12_1": "format!(\" this[{}]\", stack_text(parser, 1)?)",
"Item_14_1": "\":\".to_owned()",
"Item_16_1": "format!(\" new {}\", stack_text(parser, 0)?)",
"Item_18_1": "format!(\" new {}\", stack_text(parser, 0)?)",
"Item_20_1": "format!(\"({})\", stack_text(parser, 1)?)",
"Item_22_1": "format!(\"{{{}}}\\n\", stack_text(parser, 1)?)",
"Item_24_1": "format!(\"[{}]\", stack_text(parser, 1)?)",
}
return expressions[action]
def render(data: dict) -> str:
tokens = data["tokens"]
nonterminals = data["nonterminals"]
numbers = {name: number for name, number, *_ in tokens}
numbers.update({name: number for name, number in nonterminals})
generated = []
next_number = 24
alternatives: dict[str, int] = {}
for _, lhs, _, action in data["productions"]:
alternative = alternatives.get(lhs, 0) + 1
alternatives[lhs] = alternative
for name in (f"{lhs}_{alternative * 2 - 1}", f"{lhs}_{alternative * 2}", action):
generated.append((name, next_number, lhs, numbers[lhs]))
next_number += 1
assert next_number == 99
flat = [data["schema"], data["start"], data["eof"], len(tokens) + len(nonterminals)]
for name, number, *_ in tokens:
flat.extend((number, 1, len(name)))
for name, number in nonterminals:
flat.extend((number, 0, len(name)))
flat.append(len(data["productions"]))
for production, lhs, rhs, _ in data["productions"]:
flat.extend((production, numbers[lhs], len(rhs), *(numbers[name] for name in rhs)))
lines = [
"// @generated by tools/generate_lsl_tables.py; do not edit by hand.",
"#![allow(non_camel_case_types)]",
"#![allow(non_snake_case)]",
"#![allow(clippy::missing_errors_doc)]",
"#![allow(clippy::must_use_candidate)]",
"",
"use std::ops::{Deref, DerefMut};",
"",
"use libremetaverse_types::compat::{Object, UnicodeCategory};",
"",
"use crate::{",
" CharacterMatcher, Dfa, DfaAccept, DfaState, Error, ErrorHandler, Grammar,",
" Lexer, LexerAction, Parser, ResWds, SYMBOL, TOKEN, TokenDefinition,",
" UnicodeClass, YyLexer, YyParser,",
"};",
"",
f"pub const LSL_GENERATOR_SOURCE_COMMIT: &str = {rust_string(data['source']['commit'])};",
f"pub const LSL_GENERATOR_SOURCE_PARSER: &str = {rust_string(data['source']['parser'])};",
f"pub const LSL_GENERATOR_SOURCE_LEXER: &str = {rust_string(data['source']['lexer'])};",
f"pub const LSL_GENERATOR_SOURCE_LICENSE: &str = {rust_string(data['source']['license'])};",
f"pub const LSL_GENERATOR_SERIALIZATION_VERSION: &str = {rust_string(data['source']['serialization_version'])};",
"pub const GENERATED_PARSER_DATA: &[i32] = &[",
]
for index in range(0, len(flat), 16):
lines.append(" " + ", ".join(map(str, flat[index:index + 16])) + ",")
lines.extend(["];"])
lines.extend([
"",
"pub fn generated_symbol_name(number: i32) -> Option<&'static str> {",
" match number {",
])
for name, number, *_ in tokens:
lines.append(f" {number} => Some({rust_string(name)}),")
for name, number in nonterminals:
lines.append(f" {number} => Some({rust_string(name)}),")
lines.extend([" _ => None,", " }", "}"])
lines.extend([
"",
"pub fn generated_parser() -> Result<YyParser, Error> {",
f" let mut grammar = Grammar::new({data['start']}, {data['eof']})?;",
])
for name, number, *_ in tokens:
lines.append(f" grammar.add_symbol({rust_string(name)}, {number}, true)?;")
for name, number in nonterminals:
lines.append(f" grammar.add_symbol({rust_string(name)}, {number}, false)?;")
for _, lhs, rhs, _ in data["productions"]:
rhs_numbers = ", ".join(str(numbers[name]) for name in rhs)
lines.append(f" grammar.add_production({numbers[lhs]}, vec![{rhs_numbers}])?;")
lines.extend([
" let mut parser = grammar.build()?;",
" parser.arr = GENERATED_PARSER_DATA.to_vec();",
" Ok(parser)",
"}",
"",
"fn accept(name: &str, number: i32, action: LexerAction, action_number: i32) -> DfaAccept {",
" DfaAccept {",
" token: TokenDefinition { name: name.to_owned(), number },",
" action,",
" action_number,",
" reserved_words: (name == \"ID\").then(|| \"ID\".to_owned()),",
" }",
"}",
"",
"pub fn generated_lexer_with_handler(error_handler: ErrorHandler) -> Result<YyLexer, Error> {",
" let punctuation = [",
])
punctuation = [(name, number, literal) for name, number, literal in tokens if literal and len(literal) == 1]
for name, number, literal in punctuation:
lines.append(f" ({ord(literal)}, {rust_string(name)}, {number}),")
lines.extend([
" ];",
" let punctuation_start = 3usize;",
" let any_state = punctuation_start + punctuation.len();",
" let mut states = vec![DfaState::default(); any_state + 1];",
" states[0] = states[0].clone()",
" .transition(CharacterMatcher::UnicodeClass(UnicodeClass::WhiteSpace), 1)",
" .transition(CharacterMatcher::UnicodeClass(UnicodeClass::Letter), 2)",
" .transition(CharacterMatcher::Exact(u16::from(b'_')), 2);",
" for (offset, (unit, _, _)) in punctuation.iter().enumerate() {",
" states[0].transitions.push((CharacterMatcher::Exact(*unit), punctuation_start + offset));",
" }",
" states[0].transitions.push((CharacterMatcher::Any, any_state));",
" states[1] = states[1].clone()",
" .transition(CharacterMatcher::UnicodeClass(UnicodeClass::WhiteSpace), 1)",
" .accepting(accept(\"ANY\", 7, LexerAction::Skip, -1));",
" states[2] = states[2].clone()",
" .transition(CharacterMatcher::UnicodeClass(UnicodeClass::Letter), 2)",
" .transition(CharacterMatcher::UnicodeClass(UnicodeClass::Number), 2)",
" .transition(CharacterMatcher::Exact(u16::from(b'_')), 2)",
" .accepting(accept(\"ID\", 6, LexerAction::Emit, 0));",
" for (offset, (_, name, number)) in punctuation.iter().enumerate() {",
" states[punctuation_start + offset] = DfaState::default()",
" .accepting(accept(name, *number, LexerAction::Emit, *number));",
" }",
" states[any_state] = DfaState::default()",
" .accepting(accept(\"ANY\", 7, LexerAction::Emit, 1));",
" let mut lexer = YyLexer::new(error_handler)?;",
" lexer.set_start_dfa(\"YYINITIAL\", Dfa::from_states(states, 0)?)?;",
" lexer.set_reserved_words(\"ID\", ResWds::from_pairs([",
" (\"base\", TokenDefinition::new(\"BASE\", 3)?),",
" (\"this\", TokenDefinition::new(\"THIS\", 4)?),",
" (\"new\", TokenDefinition::new(\"NEW\", 5)?),",
" ], false)?)?;",
" lexer.using_eof = true;",
" for (name, number) in [",
])
for name, number, *_ in tokens:
lines.append(f" ({rust_string(name)}, {number}),")
lines.extend([
" ] {",
" lexer.tokens.0.insert(Object::String(name.to_owned()), Object::Integer(number));",
" }",
" let _ = lexer.using_cat(UnicodeCategory(0))?;",
" Ok(lexer)",
"}",
"",
"pub fn generated_lexer() -> Result<YyLexer, Error> {",
" generated_lexer_with_handler(ErrorHandler::new_with_boolean(false)?)",
"}",
"",
"#[derive(Clone, Debug)]",
"pub struct GeneratedLexerToken<const NUMBER: i32>(TOKEN);",
"",
"impl<const NUMBER: i32> GeneratedLexerToken<NUMBER> {",
" pub fn new(lexer: Lexer) -> Result<Self, Error> {",
" let name = generated_symbol_name(NUMBER).ok_or(Error::Argument)?;",
" Ok(Self(TOKEN::generated_with_lexer(lexer, name, NUMBER)?))",
" }",
" pub fn yyname(&self) -> String { self.0.yyname() }",
" pub const fn yynum(&self) -> i32 { NUMBER }",
" pub fn yytext(&self) -> String { self.0.yytext() }",
"}",
"",
"impl<const NUMBER: i32> Deref for GeneratedLexerToken<NUMBER> {",
" type Target = TOKEN;",
" fn deref(&self) -> &Self::Target { &self.0 }",
"}",
"impl<const NUMBER: i32> DerefMut for GeneratedLexerToken<NUMBER> {",
" fn deref_mut(&mut self) -> &mut Self::Target { &mut self.0 }",
"}",
])
for name, number, *_ in tokens:
if name != "EOF":
lines.append(f"pub type {name} = GeneratedLexerToken<{number}>;")
lines.extend([
"",
"fn generated_parser_identity(generated: i32) -> Option<(&'static str, i32)> {",
" match generated {",
])
for lhs, number in nonterminals:
identities = [internal for _, internal, owner, _ in generated if owner == lhs]
pattern = (
str(identities[0])
if len(identities) == 1
else f"{identities[0]}..={identities[-1]}"
)
lines.append(f" {pattern} => Some(({rust_string(lhs)}, {number})),")
lines.extend([" _ => None,", " }", "}"])
lines.extend([
"",
"fn stack_text(parser: &Parser, index: i32) -> Result<String, Error> {",
" Ok(parser.stack_at(index)?.m_value.raw_text())",
"}",
"",
"fn generated_semantic_text(generated: i32, parser: &Parser) -> Result<String, Error> {",
" Ok(match generated {",
])
action_ids = {name: internal for name, internal, _, _ in generated}
expressions: dict[str, list[int]] = {}
for _, _, _, action in data["productions"]:
expression = semantic_expression(action)
if expression != "String::new()":
expressions.setdefault(expression, []).append(action_ids[action])
for expression, action_numbers in expressions.items():
pattern = " | ".join(str(number) for number in action_numbers)
lines.append(f" {pattern} => {expression},")
lines.extend([" _ => String::new(),", " })", "}"])
lines.extend([
"",
"#[derive(Clone, Debug)]",
"pub struct GeneratedParserSymbol<const GENERATED: i32>(TOKEN);",
"",
"impl<const GENERATED: i32> GeneratedParserSymbol<GENERATED> {",
" pub fn new(parser: Parser) -> Result<Self, Error> {",
" let (name, number) = generated_parser_identity(GENERATED).ok_or(Error::Argument)?;",
" let text = generated_semantic_text(GENERATED, &parser)?;",
" Ok(Self(TOKEN::generated_with_parser(parser, name, number, text)?))",
" }",
" pub fn yyname(&self) -> String { self.0.yyname() }",
" pub fn yynum(&self) -> i32 { self.0.yynum() }",
" pub fn yytext(&self) -> String { self.0.yytext() }",
"}",
"",
"impl<const GENERATED: i32> Deref for GeneratedParserSymbol<GENERATED> {",
" type Target = TOKEN;",
" fn deref(&self) -> &Self::Target { &self.0 }",
"}",
"impl<const GENERATED: i32> DerefMut for GeneratedParserSymbol<GENERATED> {",
" fn deref_mut(&mut self) -> &mut Self::Target { &mut self.0 }",
"}",
])
base_ids = {}
for name, internal, lhs, _ in generated:
base_ids.setdefault(lhs, internal)
lines.append(f"pub type {name} = GeneratedParserSymbol<{internal}>;")
for name, _ in nonterminals:
lines.append(f"pub type {name} = GeneratedParserSymbol<{base_ids[name]}>;")
lines.extend([
"",
"#[derive(Clone, Debug)]",
"pub struct yycs0syntax(pub YyParser);",
"impl yycs0syntax {",
" pub fn new() -> Result<Self, Error> { Ok(Self(generated_parser()?)) }",
" pub fn action(&self, parser: Parser, symbol: SYMBOL, action: i32) -> Result<Object, Error> { self.0.action(parser, symbol, action) }",
])
for name, internal, _, _ in generated:
factory = mapping.snake(f"{name}_factory")
lines.append(f" pub fn {factory}(parser: Parser) -> Result<Object, Error> {{ Ok(Object::opaque({name}::new(parser)?)) }}")
for name, _ in nonterminals:
factory = mapping.snake(f"{name}_factory")
lines.append(f" pub fn {factory}(parser: Parser) -> Result<Object, Error> {{ Ok(Object::opaque({name}::new(parser)?)) }}")
lines.extend([
" pub fn error_factory(parser: Parser) -> Result<Object, Error> { Ok(Object::opaque(crate::LslError::new_with_parser(parser)?)) }",
"}",
"impl Deref for yycs0syntax { type Target = YyParser; fn deref(&self) -> &Self::Target { &self.0 } }",
"impl DerefMut for yycs0syntax { fn deref_mut(&mut self) -> &mut Self::Target { &mut self.0 } }",
"",
"#[derive(Clone, Debug)]",
"pub struct yycs0tokens(pub YyLexer);",
"impl yycs0tokens {",
" pub fn new(error_handler: ErrorHandler) -> Result<Self, Error> { Ok(Self(generated_lexer_with_handler(error_handler)?)) }",
])
for name, number, *_ in tokens:
if name != "EOF":
factory = mapping.snake(f"{name}_factory")
lines.append(f" pub fn {factory}(lexer: Lexer) -> Result<Object, Error> {{ Ok(Object::opaque({name}::new(lexer)?)) }}")
lines.extend([
" pub fn old_action(&self, lexer: Lexer, yytext: &mut String, action: i32, reject: &mut bool) -> Result<TOKEN, Error> {",
" if matches!(action, 8 | 55 | 74) { match action { 8 => \"yym.yy_begin\", 55 => \"((cs0tokens)yym)\", _ => \"((cs0syntax)yyq)\" }.clone_into(yytext); return TOKEN::generated_with_lexer(lexer, \"ANY\", 7); }",
" *reject = true;",
" Err(Error::InvalidOperation)",
" }",
"}",
"impl Deref for yycs0tokens { type Target = YyLexer; fn deref(&self) -> &Self::Target { &self.0 } }",
"impl DerefMut for yycs0tokens { fn deref_mut(&mut self) -> &mut Self::Target { &mut self.0 } }",
"",
"#[derive(Clone, Debug)]",
"pub struct cs0tokens { pub out: String, pub lexer: Lexer }",
"impl cs0tokens {",
" pub fn new() -> Result<Self, Error> { Self::new_with_yy_lexer(generated_lexer()?) }",
" pub fn new_with_constructor() -> Result<Self, Error> { Self::new() }",
" pub fn new_with_error_handler(handler: ErrorHandler) -> Result<Self, Error> { Self::new_with_yy_lexer(generated_lexer_with_handler(handler)?) }",
" pub fn new_with_yy_lexer(tokens: YyLexer) -> Result<Self, Error> { Ok(Self { out: String::new(), lexer: Lexer::new(tokens)? }) }",
"}",
"impl Deref for cs0tokens { type Target = Lexer; fn deref(&self) -> &Self::Target { &self.lexer } }",
"impl DerefMut for cs0tokens { fn deref_mut(&mut self) -> &mut Self::Target { &mut self.lexer } }",
"",
"#[derive(Clone, Debug)]",
"pub struct cs0syntax { pub out: String, pub cls: String, pub par: String, pub ctx: String, pub defconseen: bool, pub parser: Parser }",
"impl cs0syntax {",
" pub fn new() -> Result<Self, Error> { Self::new_with_yy_parser(generated_parser()?) }",
" pub fn new_with_constructor() -> Result<Self, Error> { Self::new() }",
" pub fn new_with_yy_parser(symbols: YyParser) -> Result<Self, Error> { let lexer = Lexer::new(generated_lexer()?)?; Self::from_parts(symbols, lexer) }",
" pub fn new_with_yy_parser_error_handler(mut symbols: YyParser, handler: ErrorHandler) -> Result<Self, Error> { symbols.erh = handler.clone(); let lexer = Lexer::new(generated_lexer_with_handler(handler)?)?; Self::from_parts(symbols, lexer) }",
" fn from_parts(symbols: YyParser, lexer: Lexer) -> Result<Self, Error> { Ok(Self { out: String::new(), cls: String::new(), par: String::new(), ctx: String::new(), defconseen: false, parser: Parser::new(symbols, lexer)? }) }",
"}",
"impl Deref for cs0syntax { type Target = Parser; fn deref(&self) -> &Self::Target { &self.parser } }",
"impl DerefMut for cs0syntax { fn deref_mut(&mut self) -> &mut Self::Target { &mut self.parser } }",
"",
])
return format_rust("\n".join(lines))
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--check", action="store_true")
args = parser.parse_args()
data = json.loads(INPUT.read_text())
output = render(data)
if args.check:
if not OUTPUT.exists() or OUTPUT.read_text() != output:
raise SystemExit(f"generated LSL table is stale: {OUTPUT}")
print("LSL parser/token generation is byte-identical")
return
OUTPUT.write_text(output)
if __name__ == "__main__":
main()