diff --git a/.gitea/workflows/rust-workspace.yml b/.gitea/workflows/rust-workspace.yml index 40c31c3..ca42b65 100644 --- a/.gitea/workflows/rust-workspace.yml +++ b/.gitea/workflows/rust-workspace.yml @@ -12,6 +12,7 @@ on: - "tools/check_milestone_10_issue_78.py" - "tools/check_milestone_10_issue_79.py" - "tools/check_milestone_10_issue_80.py" + - "tools/check_milestone_10_issue_81.py" - "**/*.rs" - "**/Cargo.toml" - "Cargo.lock" @@ -26,6 +27,7 @@ on: - "tools/check_milestone_10_issue_78.py" - "tools/check_milestone_10_issue_79.py" - "tools/check_milestone_10_issue_80.py" + - "tools/check_milestone_10_issue_81.py" - "**/*.rs" - "**/Cargo.toml" - "Cargo.lock" @@ -66,6 +68,7 @@ jobs: python3 tools/check_milestone_10_issue_78.py python3 tools/check_milestone_10_issue_79.py python3 tools/check_milestone_10_issue_80.py + python3 tools/check_milestone_10_issue_81.py - name: Test the complete native world milestone run: python3 tools/test_milestone_09.py - name: Compile every workspace target with bounded memory diff --git a/Cargo.lock b/Cargo.lock index 8a452b7..04514bd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -924,6 +924,7 @@ name = "libremetaverse-lsl-tools" version = "0.0.1" dependencies = [ "libremetaverse-types", + "unicode-general-category", ] [[package]] @@ -2213,6 +2214,12 @@ version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" +[[package]] +name = "unicode-general-category" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b993bddc193ae5bd0d623b49ec06ac3e9312875fdae725a975c51db1cc1677f" + [[package]] name = "unicode-ident" version = "1.0.24" diff --git a/api/SHIM-COVERAGE.md b/api/SHIM-COVERAGE.md index a33fa5e..befe03e 100644 --- a/api/SHIM-COVERAGE.md +++ b/api/SHIM-COVERAGE.md @@ -7,7 +7,7 @@ Generated by `python3 tools/generate_api_shims.py`; do not edit by hand. | `LibreMetaverse` | 2,711 | 27,281 | native implementation: 401 types / 17,025 members; remaining surface is callable failure-only shims | | `LibreMetaverse.Imaging.Abstractions` | 3 | 20 | native implementation: 3 types / 20 members; no generated shims remain | | `LibreMetaverse.Imaging.Skia` | 1 | 3 | native implementation: 1 type / 3 members; no generated shims remain | -| `LibreMetaverse.LslTools` | 164 | 768 | callable failure-only shim | +| `LibreMetaverse.LslTools` | 164 | 768 | native implementation: 22 types / 188 members; remaining surface is callable failure-only shims | | `LibreMetaverse.PrimMesher` | 17 | 207 | native implementation: 15 types / 200 members; remaining surface is callable failure-only shims | | `LibreMetaverse.RLV` | 28 | 499 | native implementation: 17 types / 200 members; remaining surface is callable failure-only shims | | `LibreMetaverse.Rendering.MeshFoundry` | 1 | 14 | native implementation: 1 type / 14 members; no generated shims remain | diff --git a/crates/libremetaverse-lsl-tools/Cargo.toml b/crates/libremetaverse-lsl-tools/Cargo.toml index f0753c2..59a1bad 100644 --- a/crates/libremetaverse-lsl-tools/Cargo.toml +++ b/crates/libremetaverse-lsl-tools/Cargo.toml @@ -5,10 +5,11 @@ edition.workspace = true rust-version.workspace = true license.workspace = true repository.workspace = true -description = "LSL parser-tool shims for the MetaCrate LibreMetaverse rewrite" +description = "Native LSL lexer and parser tooling for the MetaCrate LibreMetaverse rewrite" [dependencies] libremetaverse-types = { path = "../libremetaverse-types" } +unicode-general-category = "1.1" [lints] workspace = true diff --git a/crates/libremetaverse-lsl-tools/README.md b/crates/libremetaverse-lsl-tools/README.md new file mode 100644 index 0000000..6f7ae4a --- /dev/null +++ b/crates/libremetaverse-lsl-tools/README.md @@ -0,0 +1,94 @@ +# Native LSL lexer and parser tools + +`libremetaverse-lsl-tools` is the native Rust replacement for the lexer and +parser-generator runtime in the pinned `LibreMetaverse.LslTools` assembly. The +issue 81 boundary supplies source reading, comments, Unicode character sets, +deterministic DFA execution, reserved words, terminal symbols, EOF, source +locations, diagnostics, and Rust iterators. Grammar productions, reductions, +and recovery are added by issue 82; deterministic generated grammar and token +tables are added by issue 83. + +## Source and position contract + +All public positions are UTF-16 code-unit offsets, matching the original +`System.Char` API. They are not UTF-8 byte offsets. `SourceLineInfo` reports a +one-based line and character position while retaining the filtered-line bounds +and the raw character position before removed comments. Non-BMP characters +therefore occupy two positions, exactly as they do in C#. + +`CsReader` accepts strings, files, or explicitly encoded byte slices. It +normalizes CRLF and lone CR to LF, removes `//` and `/* ... */` comments while +preserving their newline structure, tracks the removed UTF-16 lengths for raw +columns, and applies `#line N "file"` directives. Unterminated block comments, +invalid UTF-8/UTF-16, unpaired surrogates, trailing UTF-16 bytes, and non-ASCII +bytes in ASCII mode return positioned errors instead of replacement text. + +The portable encoding set is UTF-8, UTF-16LE, UTF-16BE, ASCII, and ASCIICAPS. +UTF-7 and platform code pages are deliberately not delegated to host APIs, so +Linux and Windows produce the same result. A source is bounded to 64 Mi UTF-16 +units and a token to 16 Mi units before unbounded allocation or matching can +occur. + +## Lexer table contract + +`Dfa` is a validated deterministic table of `DfaState` values and +`CharacterMatcher` transitions. Matchers support exact UTF-16 values, ranges, +sets, all .NET Unicode general categories, category unions, any character, and +EOF. The runtime applies maximum munch, preserves rule order for overlapping +transitions, rejects empty non-EOF matches, and exposes a deterministic textual +table representation through `YyLexer::emit_dfa`. + +Accepting states carry a `TokenDefinition`, action number, optional reserved +word table, and one of four actions: emit, skip, emit and change start +condition, or skip and change start condition. Reserved words are exact by +default and use Unicode uppercase only when the table requests the reference +`U { ... }` behavior. `TOKEN` preserves token name, number, lexeme, semantic +value, half-open UTF-16 span, and source location. Configured EOF is emitted +once at the end of the filtered buffer. + +The old C# enumerators remain callable as compatibility adapters. New Rust code +should use `Lexer::iter` or `Lexer::next_token`; both stop deterministically +after a diagnostic and never yield a fabricated token. Token and symbol +`Pass` members perform real lookup against the mapped parser symbol/literal +tables and populate the supplied parser-entry priority. Missing or malformed +table data is an explicit positioned error. + +## Diagnostics and migration + +`ErrorHandler` collects structured `Diagnostic` values. Each diagnostic has a +stable numeric code, `DiagnosticCategory`, severity, message, input fragment, +and `SourceLineInfo`. Invalid characters and start conditions are recorded at +the offending UTF-16 position. Compatibility exception constructors retain +their original number, input, symbol/token location, handled state, and fatal +or stop behavior. A handler configured to throw increments its counter and +returns immediately without reporting, matching the reference order. + +The principal API mapping is: + +| C# concept | Native Rust API | +| --- | --- | +| `CsReader`, `LineManager`, `SourceLineInfo` | Same mapped names, UTF-16-safe source model | +| `Dfa`, `Dfa.Action`, `YyLexer` | `Dfa`, `DfaAction`, `YyLexer`, plus typed state builders | +| `SYMBOL`, `TOKEN`, `EOF`, `Null` | Same mapped names with owned Rust values | +| `Lexer._Enumerator` | `LexerEnumerator`; prefer `LexerIterator` | +| `Charset`, `CatTest` | Same mapped names plus `DotNetUnicodeCategory` | +| `CSToolsException`, `ErrorHandler` | Same mapped names plus structured `Diagnostic` | + +No C#, .NET runtime, dynamically loaded class, macOS-only API, platform code +page, or runtime source generation is used by this boundary. + +## Reproducible verification + +Run the issue-owned gates with one build job: + +```sh +CARGO_BUILD_JOBS=1 cargo test -p libremetaverse-lsl-tools --locked +CARGO_BUILD_JOBS=1 cargo check --manifest-path tests/api-compile/Cargo.toml --locked +CARGO_BUILD_JOBS=1 cargo clippy -p libremetaverse-lsl-tools --all-targets --locked -- -D warnings +RUSTDOCFLAGS='-D warnings' CARGO_BUILD_JOBS=1 cargo doc -p libremetaverse-lsl-tools --no-deps --locked +python3 tools/check_milestone_10_issue_81.py +python3 tools/generate_api_shims.py --check +``` + +The package contains 27 focused native and compatibility fixtures. The Gitea +workflow runs the audit and workspace compile on `ubuntu-latest` only. diff --git a/crates/libremetaverse-lsl-tools/src/generated.rs b/crates/libremetaverse-lsl-tools/src/generated.rs index 017553d..a154e71 100644 --- a/crates/libremetaverse-lsl-tools/src/generated.rs +++ b/crates/libremetaverse-lsl-tools/src/generated.rs @@ -266,248 +266,41 @@ impl COLON { } /// C# type: `T:LibreMetaverse.LslTools.CSToolsException`. -pub struct CSToolsException { - /// C# member: `F:LibreMetaverse.LslTools.CSToolsException.handled`. - pub handled: bool, - /// C# member: `F:LibreMetaverse.LslTools.CSToolsException.nExceptionNumber`. - pub n_exception_number: i32, - /// C# member: `F:LibreMetaverse.LslTools.CSToolsException.sInput`. - pub s_input: String, - /// C# member: `F:LibreMetaverse.LslTools.CSToolsException.slInfo`. - pub sl_info: libremetaverse_lsl_tools::SourceLineInfo, - /// C# member: `F:LibreMetaverse.LslTools.CSToolsException.sym`. - pub sym: libremetaverse_lsl_tools::SYMBOL, -} -impl CSToolsException { - /// C# member: `M:LibreMetaverse.LslTools.CSToolsException.#ctor(System.Int32,LibreMetaverse.LslTools.Lexer,System.Int32,System.String,System.String)`. - pub fn new_with_int32_lexer_int32_string_string( - en: i32, - yl: libremetaverse_lsl_tools::Lexer, - p: i32, - y: String, - s: String, - ) -> Result { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.CSToolsException.#ctor(System.Int32,LibreMetaverse.LslTools.Lexer,System.Int32,System.String,System.String)", - ) - } - /// C# member: `M:LibreMetaverse.LslTools.CSToolsException.#ctor(System.Int32,LibreMetaverse.LslTools.Lexer,System.String)`. - pub fn new_with_int32_lexer_string( - n: i32, - yl: libremetaverse_lsl_tools::Lexer, - s: String, - ) -> Result { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.CSToolsException.#ctor(System.Int32,LibreMetaverse.LslTools.Lexer,System.String)", - ) - } - /// C# member: `M:LibreMetaverse.LslTools.CSToolsException.#ctor(System.Int32,LibreMetaverse.LslTools.Lexer,System.String,System.String)`. - pub fn new_with_int32_lexer_string_string( - n: i32, - yl: libremetaverse_lsl_tools::Lexer, - yy: String, - s: String, - ) -> Result { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.CSToolsException.#ctor(System.Int32,LibreMetaverse.LslTools.Lexer,System.String,System.String)", - ) - } - /// C# member: `M:LibreMetaverse.LslTools.CSToolsException.#ctor(System.Int32,LibreMetaverse.LslTools.SYMBOL,System.String)`. - pub fn new_with_int32_symbol_string( - n: i32, - t: libremetaverse_lsl_tools::SYMBOL, - s: String, - ) -> Result { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.CSToolsException.#ctor(System.Int32,LibreMetaverse.LslTools.SYMBOL,System.String)", - ) - } - /// C# member: `M:LibreMetaverse.LslTools.CSToolsException.#ctor(System.Int32,LibreMetaverse.LslTools.SourceLineInfo,System.String,System.String)`. - pub fn new_with_int32_source_line_info_string_string( - en: i32, - s: libremetaverse_lsl_tools::SourceLineInfo, - y: String, - m: String, - ) -> Result { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.CSToolsException.#ctor(System.Int32,LibreMetaverse.LslTools.SourceLineInfo,System.String,System.String)", - ) - } - /// C# member: `M:LibreMetaverse.LslTools.CSToolsException.#ctor(System.Int32,LibreMetaverse.LslTools.TOKEN,System.String)`. - pub fn new_with_int32_token_string( - n: i32, - t: libremetaverse_lsl_tools::TOKEN, - s: String, - ) -> Result { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.CSToolsException.#ctor(System.Int32,LibreMetaverse.LslTools.TOKEN,System.String)", - ) - } - /// C# member: `M:LibreMetaverse.LslTools.CSToolsException.#ctor(System.Int32,System.String)`. - pub fn new_with_int32_string(n: i32, s: String) -> Result { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.CSToolsException.#ctor(System.Int32,System.String)", - ) - } - /// C# member: `M:LibreMetaverse.LslTools.CSToolsException.Handle(LibreMetaverse.LslTools.ErrorHandler)`. - pub fn handle(&self, erh: libremetaverse_lsl_tools::ErrorHandler) -> Result<(), crate::Error> { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.CSToolsException.Handle(LibreMetaverse.LslTools.ErrorHandler)", - ) - } -} +/// C# member: `F:LibreMetaverse.LslTools.CSToolsException.handled`. +/// C# member: `F:LibreMetaverse.LslTools.CSToolsException.nExceptionNumber`. +/// C# member: `F:LibreMetaverse.LslTools.CSToolsException.sInput`. +/// C# member: `F:LibreMetaverse.LslTools.CSToolsException.slInfo`. +/// C# member: `F:LibreMetaverse.LslTools.CSToolsException.sym`. +/// C# member: `M:LibreMetaverse.LslTools.CSToolsException.#ctor(System.Int32,LibreMetaverse.LslTools.Lexer,System.Int32,System.String,System.String)`. +/// C# member: `M:LibreMetaverse.LslTools.CSToolsException.#ctor(System.Int32,LibreMetaverse.LslTools.Lexer,System.String)`. +/// C# member: `M:LibreMetaverse.LslTools.CSToolsException.#ctor(System.Int32,LibreMetaverse.LslTools.Lexer,System.String,System.String)`. +/// C# member: `M:LibreMetaverse.LslTools.CSToolsException.#ctor(System.Int32,LibreMetaverse.LslTools.SYMBOL,System.String)`. +/// C# member: `M:LibreMetaverse.LslTools.CSToolsException.#ctor(System.Int32,LibreMetaverse.LslTools.SourceLineInfo,System.String,System.String)`. +/// C# member: `M:LibreMetaverse.LslTools.CSToolsException.#ctor(System.Int32,LibreMetaverse.LslTools.TOKEN,System.String)`. +/// C# member: `M:LibreMetaverse.LslTools.CSToolsException.#ctor(System.Int32,System.String)`. +/// C# member: `M:LibreMetaverse.LslTools.CSToolsException.Handle(LibreMetaverse.LslTools.ErrorHandler)`. +pub use crate::lexer::CSToolsException; /// C# type: `T:LibreMetaverse.LslTools.CSToolsFatalException`. -pub struct CSToolsFatalException; -impl CSToolsFatalException { - /// C# member: `M:LibreMetaverse.LslTools.CSToolsFatalException.#ctor(System.Int32,LibreMetaverse.LslTools.Lexer,System.Int32,System.String,System.String)`. - pub fn new_with_int32_lexer_int32_string_string( - n: i32, - yl: libremetaverse_lsl_tools::Lexer, - p: i32, - y: String, - s: String, - ) -> Result { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.CSToolsFatalException.#ctor(System.Int32,LibreMetaverse.LslTools.Lexer,System.Int32,System.String,System.String)", - ) - } - /// C# member: `M:LibreMetaverse.LslTools.CSToolsFatalException.#ctor(System.Int32,LibreMetaverse.LslTools.Lexer,System.String)`. - pub fn new_with_int32_lexer_string( - n: i32, - yl: libremetaverse_lsl_tools::Lexer, - s: String, - ) -> Result { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.CSToolsFatalException.#ctor(System.Int32,LibreMetaverse.LslTools.Lexer,System.String)", - ) - } - /// C# member: `M:LibreMetaverse.LslTools.CSToolsFatalException.#ctor(System.Int32,LibreMetaverse.LslTools.Lexer,System.String,System.String)`. - pub fn new_with_int32_lexer_string_string( - n: i32, - yl: libremetaverse_lsl_tools::Lexer, - yy: String, - s: String, - ) -> Result { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.CSToolsFatalException.#ctor(System.Int32,LibreMetaverse.LslTools.Lexer,System.String,System.String)", - ) - } - /// C# member: `M:LibreMetaverse.LslTools.CSToolsFatalException.#ctor(System.Int32,LibreMetaverse.LslTools.SourceLineInfo,System.String,System.String)`. - pub fn new_with_int32_source_line_info_string_string( - en: i32, - s: libremetaverse_lsl_tools::SourceLineInfo, - y: String, - m: String, - ) -> Result { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.CSToolsFatalException.#ctor(System.Int32,LibreMetaverse.LslTools.SourceLineInfo,System.String,System.String)", - ) - } - /// C# member: `M:LibreMetaverse.LslTools.CSToolsFatalException.#ctor(System.Int32,LibreMetaverse.LslTools.TOKEN,System.String)`. - pub fn new_with_int32_token_string( - n: i32, - t: libremetaverse_lsl_tools::TOKEN, - s: String, - ) -> Result { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.CSToolsFatalException.#ctor(System.Int32,LibreMetaverse.LslTools.TOKEN,System.String)", - ) - } - /// C# member: `M:LibreMetaverse.LslTools.CSToolsFatalException.#ctor(System.Int32,System.String)`. - pub fn new_with_int32_string(n: i32, s: String) -> Result { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.CSToolsFatalException.#ctor(System.Int32,System.String)", - ) - } - /// C# member: `M:LibreMetaverse.LslTools.CSToolsFatalException.Handle(LibreMetaverse.LslTools.ErrorHandler)`. - pub fn handle(&self, erh: libremetaverse_lsl_tools::ErrorHandler) -> Result<(), crate::Error> { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.CSToolsFatalException.Handle(LibreMetaverse.LslTools.ErrorHandler)", - ) - } -} +/// C# member: `M:LibreMetaverse.LslTools.CSToolsFatalException.#ctor(System.Int32,LibreMetaverse.LslTools.Lexer,System.Int32,System.String,System.String)`. +/// C# member: `M:LibreMetaverse.LslTools.CSToolsFatalException.#ctor(System.Int32,LibreMetaverse.LslTools.Lexer,System.String)`. +/// C# member: `M:LibreMetaverse.LslTools.CSToolsFatalException.#ctor(System.Int32,LibreMetaverse.LslTools.Lexer,System.String,System.String)`. +/// C# member: `M:LibreMetaverse.LslTools.CSToolsFatalException.#ctor(System.Int32,LibreMetaverse.LslTools.SourceLineInfo,System.String,System.String)`. +/// C# member: `M:LibreMetaverse.LslTools.CSToolsFatalException.#ctor(System.Int32,LibreMetaverse.LslTools.TOKEN,System.String)`. +/// C# member: `M:LibreMetaverse.LslTools.CSToolsFatalException.#ctor(System.Int32,System.String)`. +/// C# member: `M:LibreMetaverse.LslTools.CSToolsFatalException.Handle(LibreMetaverse.LslTools.ErrorHandler)`. +pub use crate::lexer::CSToolsFatalException; /// C# type: `T:LibreMetaverse.LslTools.CSToolsStopException`. -pub struct CSToolsStopException; -impl CSToolsStopException { - /// C# member: `M:LibreMetaverse.LslTools.CSToolsStopException.#ctor(System.Int32,LibreMetaverse.LslTools.Lexer,System.Int32,System.String,System.String)`. - pub fn new_with_int32_lexer_int32_string_string( - n: i32, - yl: libremetaverse_lsl_tools::Lexer, - p: i32, - y: String, - s: String, - ) -> Result { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.CSToolsStopException.#ctor(System.Int32,LibreMetaverse.LslTools.Lexer,System.Int32,System.String,System.String)", - ) - } - /// C# member: `M:LibreMetaverse.LslTools.CSToolsStopException.#ctor(System.Int32,LibreMetaverse.LslTools.Lexer,System.String)`. - pub fn new_with_int32_lexer_string( - n: i32, - yl: libremetaverse_lsl_tools::Lexer, - s: String, - ) -> Result { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.CSToolsStopException.#ctor(System.Int32,LibreMetaverse.LslTools.Lexer,System.String)", - ) - } - /// C# member: `M:LibreMetaverse.LslTools.CSToolsStopException.#ctor(System.Int32,LibreMetaverse.LslTools.Lexer,System.String,System.String)`. - pub fn new_with_int32_lexer_string_string( - n: i32, - yl: libremetaverse_lsl_tools::Lexer, - yy: String, - s: String, - ) -> Result { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.CSToolsStopException.#ctor(System.Int32,LibreMetaverse.LslTools.Lexer,System.String,System.String)", - ) - } - /// C# member: `M:LibreMetaverse.LslTools.CSToolsStopException.#ctor(System.Int32,LibreMetaverse.LslTools.SYMBOL,System.String)`. - pub fn new_with_int32_symbol_string( - n: i32, - t: libremetaverse_lsl_tools::SYMBOL, - s: String, - ) -> Result { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.CSToolsStopException.#ctor(System.Int32,LibreMetaverse.LslTools.SYMBOL,System.String)", - ) - } - /// C# member: `M:LibreMetaverse.LslTools.CSToolsStopException.#ctor(System.Int32,LibreMetaverse.LslTools.SourceLineInfo,System.String,System.String)`. - pub fn new_with_int32_source_line_info_string_string( - en: i32, - s: libremetaverse_lsl_tools::SourceLineInfo, - y: String, - m: String, - ) -> Result { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.CSToolsStopException.#ctor(System.Int32,LibreMetaverse.LslTools.SourceLineInfo,System.String,System.String)", - ) - } - /// C# member: `M:LibreMetaverse.LslTools.CSToolsStopException.#ctor(System.Int32,LibreMetaverse.LslTools.TOKEN,System.String)`. - pub fn new_with_int32_token_string( - n: i32, - t: libremetaverse_lsl_tools::TOKEN, - s: String, - ) -> Result { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.CSToolsStopException.#ctor(System.Int32,LibreMetaverse.LslTools.TOKEN,System.String)", - ) - } - /// C# member: `M:LibreMetaverse.LslTools.CSToolsStopException.#ctor(System.Int32,System.String)`. - pub fn new_with_int32_string(n: i32, s: String) -> Result { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.CSToolsStopException.#ctor(System.Int32,System.String)", - ) - } - /// C# member: `M:LibreMetaverse.LslTools.CSToolsStopException.Handle(LibreMetaverse.LslTools.ErrorHandler)`. - pub fn handle(&self, erh: libremetaverse_lsl_tools::ErrorHandler) -> Result<(), crate::Error> { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.CSToolsStopException.Handle(LibreMetaverse.LslTools.ErrorHandler)", - ) - } -} +/// C# member: `M:LibreMetaverse.LslTools.CSToolsStopException.#ctor(System.Int32,LibreMetaverse.LslTools.Lexer,System.Int32,System.String,System.String)`. +/// C# member: `M:LibreMetaverse.LslTools.CSToolsStopException.#ctor(System.Int32,LibreMetaverse.LslTools.Lexer,System.String)`. +/// C# member: `M:LibreMetaverse.LslTools.CSToolsStopException.#ctor(System.Int32,LibreMetaverse.LslTools.Lexer,System.String,System.String)`. +/// C# member: `M:LibreMetaverse.LslTools.CSToolsStopException.#ctor(System.Int32,LibreMetaverse.LslTools.SYMBOL,System.String)`. +/// C# member: `M:LibreMetaverse.LslTools.CSToolsStopException.#ctor(System.Int32,LibreMetaverse.LslTools.SourceLineInfo,System.String,System.String)`. +/// C# member: `M:LibreMetaverse.LslTools.CSToolsStopException.#ctor(System.Int32,LibreMetaverse.LslTools.TOKEN,System.String)`. +/// C# member: `M:LibreMetaverse.LslTools.CSToolsStopException.#ctor(System.Int32,System.String)`. +/// C# member: `M:LibreMetaverse.LslTools.CSToolsStopException.Handle(LibreMetaverse.LslTools.ErrorHandler)`. +pub use crate::lexer::CSToolsStopException; /// C# type: `T:LibreMetaverse.LslTools.CSymbol`. pub struct CSymbol { @@ -686,46 +479,14 @@ impl Call_2_1 { } /// C# type: `T:LibreMetaverse.LslTools.CatTest`. -pub struct CatTest; -impl CatTest { - /// C# member: `M:LibreMetaverse.LslTools.CatTest.#ctor(System.Globalization.UnicodeCategory)`. - pub fn new(c: libremetaverse_types::compat::UnicodeCategory) -> Result { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.CatTest.#ctor(System.Globalization.UnicodeCategory)", - ) - } - /// C# member: `M:LibreMetaverse.LslTools.CatTest.Test(System.Char)`. - pub fn test( - &self, - ch: libremetaverse_types::compat::Utf16CodeUnit, - ) -> Result { - libremetaverse_types::not_implemented("M:LibreMetaverse.LslTools.CatTest.Test(System.Char)") - } -} +/// C# member: `M:LibreMetaverse.LslTools.CatTest.#ctor(System.Globalization.UnicodeCategory)`. +/// C# member: `M:LibreMetaverse.LslTools.CatTest.Test(System.Char)`. +pub use crate::lexer::CatTest; /// C# type: `T:LibreMetaverse.LslTools.Charset`. -pub struct Charset; -impl Charset { - /// C# member: `M:LibreMetaverse.LslTools.Charset.GetEncoding(System.String,System.Boolean@,LibreMetaverse.LslTools.ErrorHandler)`. - pub fn get_encoding( - enc: String, - toupper: &mut bool, - erh: libremetaverse_lsl_tools::ErrorHandler, - ) -> Result { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.Charset.GetEncoding(System.String,System.Boolean@,LibreMetaverse.LslTools.ErrorHandler)", - ) - } - /// C# member: `M:LibreMetaverse.LslTools.Charset.Serialise(System.Object,LibreMetaverse.LslTools.Serialiser)`. - pub fn serialise( - o: libremetaverse_types::compat::Object, - s: libremetaverse_lsl_tools::Serialiser, - ) -> Result { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.Charset.Serialise(System.Object,LibreMetaverse.LslTools.Serialiser)", - ) - } -} +/// C# member: `M:LibreMetaverse.LslTools.Charset.GetEncoding(System.String,System.Boolean@,LibreMetaverse.LslTools.ErrorHandler)`. +/// C# member: `M:LibreMetaverse.LslTools.Charset.Serialise(System.Object,LibreMetaverse.LslTools.Serialiser)`. +pub use crate::lexer::Charset; /// C# type: `T:LibreMetaverse.LslTools.ClassBody`. pub struct ClassBody; @@ -780,26 +541,11 @@ impl ClassBody_2_1 { } /// C# type: `T:LibreMetaverse.LslTools.CommentList`. -pub struct CommentList { - /// C# member: `F:LibreMetaverse.LslTools.CommentList.len`. - pub len: i32, - /// C# member: `F:LibreMetaverse.LslTools.CommentList.spos`. - pub spos: i32, - /// C# member: `F:LibreMetaverse.LslTools.CommentList.tail`. - pub tail: Box, -} -impl CommentList { - /// C# member: `M:LibreMetaverse.LslTools.CommentList.#ctor(System.Int32,System.Int32,LibreMetaverse.LslTools.CommentList)`. - pub fn new( - st: i32, - ln: i32, - t: libremetaverse_lsl_tools::CommentList, - ) -> Result { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.CommentList.#ctor(System.Int32,System.Int32,LibreMetaverse.LslTools.CommentList)", - ) - } -} +/// C# member: `F:LibreMetaverse.LslTools.CommentList.len`. +/// C# member: `F:LibreMetaverse.LslTools.CommentList.spos`. +/// C# member: `F:LibreMetaverse.LslTools.CommentList.tail`. +/// C# member: `M:LibreMetaverse.LslTools.CommentList.#ctor(System.Int32,System.Int32,LibreMetaverse.LslTools.CommentList)`. +pub use crate::lexer::CommentList; /// C# type: `T:LibreMetaverse.LslTools.Cons`. pub struct Cons; @@ -854,178 +600,44 @@ impl Cons_2_1 { } /// C# type: `T:LibreMetaverse.LslTools.CsReader`. -pub struct CsReader { - /// C# member: `F:LibreMetaverse.LslTools.CsReader.fname`. - pub fname: String, - /// C# member: `F:LibreMetaverse.LslTools.CsReader.lm`. - pub lm: libremetaverse_lsl_tools::LineManager, -} -impl CsReader { - /// C# member: `M:LibreMetaverse.LslTools.CsReader.#ctor(LibreMetaverse.LslTools.CsReader,System.Text.Encoding)`. - pub fn new_with_cs_reader_encoding( - inf: libremetaverse_lsl_tools::CsReader, - enc: libremetaverse_types::compat::TextEncoding, - ) -> Result { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.CsReader.#ctor(LibreMetaverse.LslTools.CsReader,System.Text.Encoding)", - ) - } - /// C# member: `M:LibreMetaverse.LslTools.CsReader.#ctor(System.String)`. - pub fn new_with_string(data: String) -> Result { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.CsReader.#ctor(System.String)", - ) - } - /// C# member: `M:LibreMetaverse.LslTools.CsReader.#ctor(System.String,System.Text.Encoding)`. - pub fn new_with_string_encoding( - file_name: String, - enc: libremetaverse_types::compat::TextEncoding, - ) -> Result { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.CsReader.#ctor(System.String,System.Text.Encoding)", - ) - } - /// C# member: `M:LibreMetaverse.LslTools.CsReader.Eof`. - pub fn eof(&self) -> Result { - libremetaverse_types::not_implemented("M:LibreMetaverse.LslTools.CsReader.Eof") - } - /// C# member: `M:LibreMetaverse.LslTools.CsReader.Read`. - pub fn read_with_method(&self) -> Result { - libremetaverse_types::not_implemented("M:LibreMetaverse.LslTools.CsReader.Read") - } - /// C# member: `M:LibreMetaverse.LslTools.CsReader.Read(System.Char[],System.Int32,System.Int32)`. - pub fn read_with_char_array_int32_int32( - &self, - arr: Vec, - offset: i32, - count: i32, - ) -> Result { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.CsReader.Read(System.Char[],System.Int32,System.Int32)", - ) - } - /// C# member: `M:LibreMetaverse.LslTools.CsReader.ReadLine`. - pub fn read_line(&self) -> Result { - libremetaverse_types::not_implemented("M:LibreMetaverse.LslTools.CsReader.ReadLine") - } -} +/// C# member: `F:LibreMetaverse.LslTools.CsReader.fname`. +/// C# member: `F:LibreMetaverse.LslTools.CsReader.lm`. +/// C# member: `M:LibreMetaverse.LslTools.CsReader.#ctor(LibreMetaverse.LslTools.CsReader,System.Text.Encoding)`. +/// C# member: `M:LibreMetaverse.LslTools.CsReader.#ctor(System.String)`. +/// C# member: `M:LibreMetaverse.LslTools.CsReader.#ctor(System.String,System.Text.Encoding)`. +/// C# member: `M:LibreMetaverse.LslTools.CsReader.Eof`. +/// C# member: `M:LibreMetaverse.LslTools.CsReader.Read`. +/// C# member: `M:LibreMetaverse.LslTools.CsReader.Read(System.Char[],System.Int32,System.Int32)`. +/// C# member: `M:LibreMetaverse.LslTools.CsReader.ReadLine`. +pub use crate::lexer::CsReader; /// C# type: `T:LibreMetaverse.LslTools.Dfa`. -pub struct Dfa { - /// C# member: `F:LibreMetaverse.LslTools.Dfa.m_actions`. - pub m_actions: libremetaverse_lsl_tools::DfaAction, - /// C# member: `F:LibreMetaverse.LslTools.Dfa.m_map`. - pub m_map: libremetaverse_types::compat::Hashtable, - /// C# member: `F:LibreMetaverse.LslTools.Dfa.m_reswds`. - pub m_reswds: i32, - /// C# member: `F:LibreMetaverse.LslTools.Dfa.m_tokClass`. - pub m_tok_class: String, -} -impl Dfa { - /// C# member: `M:LibreMetaverse.LslTools.Dfa.#ctor(LibreMetaverse.LslTools.Nfa)`. - pub fn new_with_nfa(nfa: libremetaverse_lsl_tools::Nfa) -> Result { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.Dfa.#ctor(LibreMetaverse.LslTools.Nfa)", - ) - } - /// C# member: `M:LibreMetaverse.LslTools.Dfa.#ctor(LibreMetaverse.LslTools.TokensGen)`. - pub fn new_with_tokens_gen( - tks: libremetaverse_lsl_tools::TokensGen, - ) -> Result { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.Dfa.#ctor(LibreMetaverse.LslTools.TokensGen)", - ) - } - /// C# member: `M:LibreMetaverse.LslTools.Dfa.Match(System.String,System.Int32,System.Int32@)`. - pub fn match_(&self, str: String, ix: i32, action: &mut i32) -> Result { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.Dfa.Match(System.String,System.Int32,System.Int32@)", - ) - } - /// C# member: `M:LibreMetaverse.LslTools.Dfa.Print`. - pub fn print(&self) -> Result<(), crate::Error> { - libremetaverse_types::not_implemented("M:LibreMetaverse.LslTools.Dfa.Print") - } - /// C# member: `M:LibreMetaverse.LslTools.Dfa.Serialise(System.Object,LibreMetaverse.LslTools.Serialiser)`. - pub fn serialise( - o: libremetaverse_types::compat::Object, - s: libremetaverse_lsl_tools::Serialiser, - ) -> Result { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.Dfa.Serialise(System.Object,LibreMetaverse.LslTools.Serialiser)", - ) - } - /// C# member: `M:LibreMetaverse.LslTools.Dfa.SetTokens(LibreMetaverse.LslTools.YyLexer,System.Collections.Hashtable)`. - pub fn set_tokens( - tks: libremetaverse_lsl_tools::YyLexer, - h: libremetaverse_types::compat::Hashtable, - ) -> Result<(), crate::Error> { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.Dfa.SetTokens(LibreMetaverse.LslTools.YyLexer,System.Collections.Hashtable)", - ) - } -} +/// C# member: `F:LibreMetaverse.LslTools.Dfa.m_actions`. +/// C# member: `F:LibreMetaverse.LslTools.Dfa.m_map`. +/// C# member: `F:LibreMetaverse.LslTools.Dfa.m_reswds`. +/// C# member: `F:LibreMetaverse.LslTools.Dfa.m_tokClass`. +/// C# member: `M:LibreMetaverse.LslTools.Dfa.#ctor(LibreMetaverse.LslTools.Nfa)`. +/// C# member: `M:LibreMetaverse.LslTools.Dfa.#ctor(LibreMetaverse.LslTools.TokensGen)`. +/// C# member: `M:LibreMetaverse.LslTools.Dfa.Match(System.String,System.Int32,System.Int32@)`. +/// C# member: `M:LibreMetaverse.LslTools.Dfa.Print`. +/// C# member: `M:LibreMetaverse.LslTools.Dfa.Serialise(System.Object,LibreMetaverse.LslTools.Serialiser)`. +/// C# member: `M:LibreMetaverse.LslTools.Dfa.SetTokens(LibreMetaverse.LslTools.YyLexer,System.Collections.Hashtable)`. +pub use crate::lexer::Dfa; /// C# type: `T:LibreMetaverse.LslTools.Dfa.Action`. -pub struct DfaAction { - /// C# member: `F:LibreMetaverse.LslTools.Dfa.Action.a_act`. - pub a_act: i32, - /// C# member: `F:LibreMetaverse.LslTools.Dfa.Action.a_next`. - pub a_next: Box, -} -impl DfaAction { - /// C# member: `M:LibreMetaverse.LslTools.Dfa.Action.#ctor(System.Int32,LibreMetaverse.LslTools.Dfa.Action)`. - pub fn new(act: i32, next: libremetaverse_lsl_tools::DfaAction) -> Result { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.Dfa.Action.#ctor(System.Int32,LibreMetaverse.LslTools.Dfa.Action)", - ) - } - /// C# member: `M:LibreMetaverse.LslTools.Dfa.Action.Serialise(System.Object,LibreMetaverse.LslTools.Serialiser)`. - pub fn serialise( - o: libremetaverse_types::compat::Object, - s: libremetaverse_lsl_tools::Serialiser, - ) -> Result { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.Dfa.Action.Serialise(System.Object,LibreMetaverse.LslTools.Serialiser)", - ) - } -} +/// C# member: `F:LibreMetaverse.LslTools.Dfa.Action.a_act`. +/// C# member: `F:LibreMetaverse.LslTools.Dfa.Action.a_next`. +/// C# member: `M:LibreMetaverse.LslTools.Dfa.Action.#ctor(System.Int32,LibreMetaverse.LslTools.Dfa.Action)`. +/// C# member: `M:LibreMetaverse.LslTools.Dfa.Action.Serialise(System.Object,LibreMetaverse.LslTools.Serialiser)`. +pub use crate::lexer::DfaAction; /// C# type: `T:LibreMetaverse.LslTools.EOF`. -pub struct EOF; -impl EOF { - /// C# member: `M:LibreMetaverse.LslTools.EOF.#ctor(LibreMetaverse.LslTools.Lexer)`. - pub fn new_with_lexer(yyl: libremetaverse_lsl_tools::Lexer) -> Result { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.EOF.#ctor(LibreMetaverse.LslTools.Lexer)", - ) - } - /// C# member: `M:LibreMetaverse.LslTools.EOF.#ctor(LibreMetaverse.LslTools.SymbolsGen)`. - pub fn new_with_symbols_gen( - yyp: libremetaverse_lsl_tools::SymbolsGen, - ) -> Result { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.EOF.#ctor(LibreMetaverse.LslTools.SymbolsGen)", - ) - } - /// C# member: `M:LibreMetaverse.LslTools.EOF.Serialise(System.Object,LibreMetaverse.LslTools.Serialiser)`. - pub fn serialise( - o: libremetaverse_types::compat::Object, - s: libremetaverse_lsl_tools::Serialiser, - ) -> Result { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.EOF.Serialise(System.Object,LibreMetaverse.LslTools.Serialiser)", - ) - } - /// C# member: `P:LibreMetaverse.LslTools.EOF.yyname`. - pub fn yyname(&self) -> String { - libremetaverse_types::unimplemented_api!("P:LibreMetaverse.LslTools.EOF.yyname") - } - /// C# member: `P:LibreMetaverse.LslTools.EOF.yynum`. - pub fn yynum(&self) -> i32 { - libremetaverse_types::unimplemented_api!("P:LibreMetaverse.LslTools.EOF.yynum") - } -} +/// C# member: `M:LibreMetaverse.LslTools.EOF.#ctor(LibreMetaverse.LslTools.Lexer)`. +/// C# member: `M:LibreMetaverse.LslTools.EOF.#ctor(LibreMetaverse.LslTools.SymbolsGen)`. +/// C# member: `M:LibreMetaverse.LslTools.EOF.Serialise(System.Object,LibreMetaverse.LslTools.Serialiser)`. +/// C# member: `P:LibreMetaverse.LslTools.EOF.yyname`. +/// C# member: `P:LibreMetaverse.LslTools.EOF.yynum`. +pub use crate::lexer::EOF; /// C# type: `T:LibreMetaverse.LslTools.Error`. pub struct LslError { @@ -1061,39 +673,13 @@ impl LslError { } /// C# type: `T:LibreMetaverse.LslTools.ErrorHandler`. -pub struct ErrorHandler { - /// C# member: `F:LibreMetaverse.LslTools.ErrorHandler.counter`. - pub counter: i32, - /// C# member: `F:LibreMetaverse.LslTools.ErrorHandler.throwExceptions`. - pub throw_exceptions: bool, -} -impl ErrorHandler { - /// C# member: `M:LibreMetaverse.LslTools.ErrorHandler.#ctor`. - pub fn new_with_constructor() -> Result { - libremetaverse_types::not_implemented("M:LibreMetaverse.LslTools.ErrorHandler.#ctor") - } - /// C# member: `M:LibreMetaverse.LslTools.ErrorHandler.#ctor(System.Boolean)`. - pub fn new_with_boolean(ee: bool) -> Result { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.ErrorHandler.#ctor(System.Boolean)", - ) - } - /// C# member: `M:LibreMetaverse.LslTools.ErrorHandler.Error(LibreMetaverse.LslTools.CSToolsException)`. - pub fn error(&self, e: libremetaverse_lsl_tools::CSToolsException) -> Result<(), crate::Error> { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.ErrorHandler.Error(LibreMetaverse.LslTools.CSToolsException)", - ) - } - /// C# member: `M:LibreMetaverse.LslTools.ErrorHandler.Report(LibreMetaverse.LslTools.CSToolsException)`. - pub fn report( - &self, - e: libremetaverse_lsl_tools::CSToolsException, - ) -> Result<(), crate::Error> { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.ErrorHandler.Report(LibreMetaverse.LslTools.CSToolsException)", - ) - } -} +/// C# member: `F:LibreMetaverse.LslTools.ErrorHandler.counter`. +/// C# member: `F:LibreMetaverse.LslTools.ErrorHandler.throwExceptions`. +/// C# member: `M:LibreMetaverse.LslTools.ErrorHandler.#ctor`. +/// C# member: `M:LibreMetaverse.LslTools.ErrorHandler.#ctor(System.Boolean)`. +/// C# member: `M:LibreMetaverse.LslTools.ErrorHandler.Error(LibreMetaverse.LslTools.CSToolsException)`. +/// C# member: `M:LibreMetaverse.LslTools.ErrorHandler.Report(LibreMetaverse.LslTools.CSToolsException)`. +pub use crate::lexer::ErrorHandler; /// C# type: `T:LibreMetaverse.LslTools.Func`. pub struct Func; @@ -1834,217 +1420,55 @@ impl LPAREN { } /// C# type: `T:LibreMetaverse.LslTools.Lexer`. -pub struct Lexer { - /// C# member: `F:LibreMetaverse.LslTools.Lexer.m_buf`. - pub m_buf: String, - /// C# member: `F:LibreMetaverse.LslTools.Lexer.m_debug`. - pub m_debug: bool, - /// C# member: `F:LibreMetaverse.LslTools.Lexer.m_pch`. - pub m_pch: i32, - /// C# member: `F:LibreMetaverse.LslTools.Lexer.m_state`. - pub m_state: String, - /// C# member: `F:LibreMetaverse.LslTools.Lexer.yytext`. - pub yytext: String, -} -impl Lexer { - /// C# member: `M:LibreMetaverse.LslTools.Lexer.#ctor(LibreMetaverse.LslTools.YyLexer)`. - pub fn new(tks: libremetaverse_lsl_tools::YyLexer) -> Result { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.Lexer.#ctor(LibreMetaverse.LslTools.YyLexer)", - ) - } - /// C# member: `M:LibreMetaverse.LslTools.Lexer.Advance`. - pub fn advance(&self) -> Result<(), crate::Error> { - libremetaverse_types::not_implemented("M:LibreMetaverse.LslTools.Lexer.Advance") - } - /// C# member: `M:LibreMetaverse.LslTools.Lexer.GetChar`. - pub fn get_char(&self) -> Result { - libremetaverse_types::not_implemented("M:LibreMetaverse.LslTools.Lexer.GetChar") - } - /// C# member: `M:LibreMetaverse.LslTools.Lexer.GetEnumerator`. - pub fn get_enumerator( - &self, - ) -> Result { - libremetaverse_types::not_implemented("M:LibreMetaverse.LslTools.Lexer.GetEnumerator") - } - /// C# member: `M:LibreMetaverse.LslTools.Lexer.Next`. - pub fn next(&self) -> Result { - libremetaverse_types::not_implemented("M:LibreMetaverse.LslTools.Lexer.Next") - } - /// C# member: `M:LibreMetaverse.LslTools.Lexer.PeekChar`. - pub fn peek_char(&self) -> Result { - libremetaverse_types::not_implemented("M:LibreMetaverse.LslTools.Lexer.PeekChar") - } - /// C# member: `M:LibreMetaverse.LslTools.Lexer.Reset`. - pub fn reset(&self) -> Result<(), crate::Error> { - libremetaverse_types::not_implemented("M:LibreMetaverse.LslTools.Lexer.Reset") - } - /// C# member: `M:LibreMetaverse.LslTools.Lexer.Saypos(System.Int32)`. - pub fn saypos(&self, pos: i32) -> Result { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.Lexer.Saypos(System.Int32)", - ) - } - /// C# member: `M:LibreMetaverse.LslTools.Lexer.Start(LibreMetaverse.LslTools.CsReader)`. - pub fn start_with_cs_reader( - &self, - in_file: libremetaverse_lsl_tools::CsReader, - ) -> Result<(), crate::Error> { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.Lexer.Start(LibreMetaverse.LslTools.CsReader)", - ) - } - /// C# member: `M:LibreMetaverse.LslTools.Lexer.Start(System.IO.StreamReader)`. - pub fn start_with_stream_reader( - &self, - in_file: libremetaverse_types::compat::StreamReader, - ) -> Result<(), crate::Error> { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.Lexer.Start(System.IO.StreamReader)", - ) - } - /// C# member: `M:LibreMetaverse.LslTools.Lexer.Start(System.String)`. - pub fn start_with_string(&self, buf: String) -> Result<(), crate::Error> { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.Lexer.Start(System.String)", - ) - } - /// C# member: `M:LibreMetaverse.LslTools.Lexer.UnGetChar`. - pub fn un_get_char(&self) -> Result<(), crate::Error> { - libremetaverse_types::not_implemented("M:LibreMetaverse.LslTools.Lexer.UnGetChar") - } - /// C# member: `M:LibreMetaverse.LslTools.Lexer.sourceLine(LibreMetaverse.LslTools.SourceLineInfo)`. - pub fn source_line( - &self, - s: libremetaverse_lsl_tools::SourceLineInfo, - ) -> Result { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.Lexer.sourceLine(LibreMetaverse.LslTools.SourceLineInfo)", - ) - } - /// C# member: `M:LibreMetaverse.LslTools.Lexer.sourceLineInfo(System.Int32)`. - pub fn source_line_info( - &self, - pos: i32, - ) -> Result { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.Lexer.sourceLineInfo(System.Int32)", - ) - } - /// C# member: `M:LibreMetaverse.LslTools.Lexer.yy_begin(System.String)`. - pub fn yy_begin(&self, newstate: String) -> Result<(), crate::Error> { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.Lexer.yy_begin(System.String)", - ) - } - /// C# member: `P:LibreMetaverse.LslTools.Lexer.m_start`. - pub fn m_start(&self) -> libremetaverse_lsl_tools::Dfa { - libremetaverse_types::unimplemented_api!("P:LibreMetaverse.LslTools.Lexer.m_start") - } - /// C# member: `P:LibreMetaverse.LslTools.Lexer.tokens`. - pub fn tokens(&self) -> libremetaverse_lsl_tools::YyLexer { - libremetaverse_types::unimplemented_api!("P:LibreMetaverse.LslTools.Lexer.tokens") - } - /// Setter for C# member: `P:LibreMetaverse.LslTools.Lexer.tokens`. - pub fn set_tokens(&mut self, value: libremetaverse_lsl_tools::YyLexer) { - libremetaverse_types::unimplemented_api!("P:LibreMetaverse.LslTools.Lexer.tokens") - } - /// C# member: `P:LibreMetaverse.LslTools.Lexer.yypos`. - pub fn yypos(&self) -> i32 { - libremetaverse_types::unimplemented_api!("P:LibreMetaverse.LslTools.Lexer.yypos") - } -} +/// C# member: `F:LibreMetaverse.LslTools.Lexer.m_buf`. +/// C# member: `F:LibreMetaverse.LslTools.Lexer.m_debug`. +/// C# member: `F:LibreMetaverse.LslTools.Lexer.m_pch`. +/// C# member: `F:LibreMetaverse.LslTools.Lexer.m_state`. +/// C# member: `F:LibreMetaverse.LslTools.Lexer.yytext`. +/// C# member: `M:LibreMetaverse.LslTools.Lexer.#ctor(LibreMetaverse.LslTools.YyLexer)`. +/// C# member: `M:LibreMetaverse.LslTools.Lexer.Advance`. +/// C# member: `M:LibreMetaverse.LslTools.Lexer.GetChar`. +/// C# member: `M:LibreMetaverse.LslTools.Lexer.GetEnumerator`. +/// C# member: `M:LibreMetaverse.LslTools.Lexer.Next`. +/// C# member: `M:LibreMetaverse.LslTools.Lexer.PeekChar`. +/// C# member: `M:LibreMetaverse.LslTools.Lexer.Reset`. +/// C# member: `M:LibreMetaverse.LslTools.Lexer.Saypos(System.Int32)`. +/// C# member: `M:LibreMetaverse.LslTools.Lexer.Start(LibreMetaverse.LslTools.CsReader)`. +/// C# member: `M:LibreMetaverse.LslTools.Lexer.Start(System.IO.StreamReader)`. +/// C# member: `M:LibreMetaverse.LslTools.Lexer.Start(System.String)`. +/// C# member: `M:LibreMetaverse.LslTools.Lexer.UnGetChar`. +/// C# member: `M:LibreMetaverse.LslTools.Lexer.sourceLine(LibreMetaverse.LslTools.SourceLineInfo)`. +/// C# member: `M:LibreMetaverse.LslTools.Lexer.sourceLineInfo(System.Int32)`. +/// C# member: `M:LibreMetaverse.LslTools.Lexer.yy_begin(System.String)`. +/// C# member: `P:LibreMetaverse.LslTools.Lexer.m_start`. +/// C# member: `P:LibreMetaverse.LslTools.Lexer.tokens`. +/// C# member: `P:LibreMetaverse.LslTools.Lexer.yypos`. +pub use crate::lexer::Lexer; /// C# type: `T:LibreMetaverse.LslTools.Lexer._Enumerator`. -pub struct LexerEnumerator; -impl LexerEnumerator { - /// C# member: `M:LibreMetaverse.LslTools.Lexer._Enumerator.#ctor(LibreMetaverse.LslTools.Lexer)`. - pub fn new(x: libremetaverse_lsl_tools::Lexer) -> Result { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.Lexer._Enumerator.#ctor(LibreMetaverse.LslTools.Lexer)", - ) - } - /// C# member: `M:LibreMetaverse.LslTools.Lexer._Enumerator.MoveNext`. - pub fn move_next(&self) -> Result { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.Lexer._Enumerator.MoveNext", - ) - } - /// C# member: `M:LibreMetaverse.LslTools.Lexer._Enumerator.Reset`. - pub fn reset(&self) -> Result<(), crate::Error> { - libremetaverse_types::not_implemented("M:LibreMetaverse.LslTools.Lexer._Enumerator.Reset") - } - /// C# member: `P:LibreMetaverse.LslTools.Lexer._Enumerator.Current`. - pub fn current(&self) -> libremetaverse_lsl_tools::TOKEN { - libremetaverse_types::unimplemented_api!( - "P:LibreMetaverse.LslTools.Lexer._Enumerator.Current" - ) - } - /// Setter for C# member: `P:LibreMetaverse.LslTools.Lexer._Enumerator.Current`. - pub fn set_current(&mut self, value: libremetaverse_lsl_tools::TOKEN) { - libremetaverse_types::unimplemented_api!( - "P:LibreMetaverse.LslTools.Lexer._Enumerator.Current" - ) - } -} +/// C# member: `M:LibreMetaverse.LslTools.Lexer._Enumerator.#ctor(LibreMetaverse.LslTools.Lexer)`. +/// C# member: `M:LibreMetaverse.LslTools.Lexer._Enumerator.MoveNext`. +/// C# member: `M:LibreMetaverse.LslTools.Lexer._Enumerator.Reset`. +/// C# member: `P:LibreMetaverse.LslTools.Lexer._Enumerator.Current`. +pub use crate::lexer::LexerEnumerator; /// C# type: `T:LibreMetaverse.LslTools.LineList`. -pub struct LineList { - /// C# member: `F:LibreMetaverse.LslTools.LineList.comments`. - pub comments: libremetaverse_lsl_tools::CommentList, - /// C# member: `F:LibreMetaverse.LslTools.LineList.head`. - pub head: i32, - /// C# member: `F:LibreMetaverse.LslTools.LineList.tail`. - pub tail: Box, -} -impl LineList { - /// C# member: `M:LibreMetaverse.LslTools.LineList.#ctor(System.Int32,LibreMetaverse.LslTools.LineList)`. - pub fn new(h: i32, t: libremetaverse_lsl_tools::LineList) -> Result { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.LineList.#ctor(System.Int32,LibreMetaverse.LslTools.LineList)", - ) - } - /// C# member: `M:LibreMetaverse.LslTools.LineList.getpos(System.Int32)`. - pub fn getpos(&self, pos: i32) -> Result { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.LineList.getpos(System.Int32)", - ) - } -} +/// C# member: `F:LibreMetaverse.LslTools.LineList.comments`. +/// C# member: `F:LibreMetaverse.LslTools.LineList.head`. +/// C# member: `F:LibreMetaverse.LslTools.LineList.tail`. +/// C# member: `M:LibreMetaverse.LslTools.LineList.#ctor(System.Int32,LibreMetaverse.LslTools.LineList)`. +/// C# member: `M:LibreMetaverse.LslTools.LineList.getpos(System.Int32)`. +pub use crate::lexer::LineList; /// C# type: `T:LibreMetaverse.LslTools.LineManager`. -pub struct LineManager { - /// C# member: `F:LibreMetaverse.LslTools.LineManager.end`. - pub end: i32, - /// C# member: `F:LibreMetaverse.LslTools.LineManager.lines`. - pub lines: i32, - /// C# member: `F:LibreMetaverse.LslTools.LineManager.list`. - pub list: libremetaverse_lsl_tools::LineList, -} -impl LineManager { - /// C# member: `M:LibreMetaverse.LslTools.LineManager.#ctor`. - pub fn new() -> Result { - libremetaverse_types::not_implemented("M:LibreMetaverse.LslTools.LineManager.#ctor") - } - /// C# member: `M:LibreMetaverse.LslTools.LineManager.backto(System.Int32)`. - pub fn backto(&self, pos: i32) -> Result<(), crate::Error> { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.LineManager.backto(System.Int32)", - ) - } - /// C# member: `M:LibreMetaverse.LslTools.LineManager.comment(System.Int32,System.Int32)`. - pub fn comment(&self, pos: i32, len: i32) -> Result<(), crate::Error> { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.LineManager.comment(System.Int32,System.Int32)", - ) - } - /// C# member: `M:LibreMetaverse.LslTools.LineManager.newline(System.Int32)`. - pub fn newline(&self, pos: i32) -> Result<(), crate::Error> { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.LineManager.newline(System.Int32)", - ) - } -} +/// C# member: `F:LibreMetaverse.LslTools.LineManager.end`. +/// C# member: `F:LibreMetaverse.LslTools.LineManager.lines`. +/// C# member: `F:LibreMetaverse.LslTools.LineManager.list`. +/// C# member: `M:LibreMetaverse.LslTools.LineManager.#ctor`. +/// C# member: `M:LibreMetaverse.LslTools.LineManager.backto(System.Int32)`. +/// C# member: `M:LibreMetaverse.LslTools.LineManager.comment(System.Int32,System.Int32)`. +/// C# member: `M:LibreMetaverse.LslTools.LineManager.newline(System.Int32)`. +pub use crate::lexer::LineManager; /// C# type: `T:LibreMetaverse.LslTools.Literal`. pub struct Literal; @@ -2277,80 +1701,21 @@ impl NfaNode { } /// C# type: `T:LibreMetaverse.LslTools.Null`. -pub struct Null; -impl Null { - /// C# member: `M:LibreMetaverse.LslTools.Null.#ctor(LibreMetaverse.LslTools.Lexer,System.String)`. - pub fn new_with_lexer_string( - yyl: libremetaverse_lsl_tools::Lexer, - proxy: String, - ) -> Result { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.Null.#ctor(LibreMetaverse.LslTools.Lexer,System.String)", - ) - } - /// C# member: `M:LibreMetaverse.LslTools.Null.#ctor(LibreMetaverse.LslTools.Parser,System.String)`. - pub fn new_with_parser_string( - yyp: libremetaverse_lsl_tools::Parser, - proxy: String, - ) -> Result { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.Null.#ctor(LibreMetaverse.LslTools.Parser,System.String)", - ) - } - /// C# member: `P:LibreMetaverse.LslTools.Null.yyname`. - pub fn yyname(&self) -> String { - libremetaverse_types::unimplemented_api!("P:LibreMetaverse.LslTools.Null.yyname") - } -} +/// C# member: `M:LibreMetaverse.LslTools.Null.#ctor(LibreMetaverse.LslTools.Lexer,System.String)`. +/// C# member: `M:LibreMetaverse.LslTools.Null.#ctor(LibreMetaverse.LslTools.Parser,System.String)`. +/// C# member: `P:LibreMetaverse.LslTools.Null.yyname`. +pub use crate::lexer::Null; /// C# type: `T:LibreMetaverse.LslTools.ObjectList`. -pub struct ObjectList; -impl ObjectList { - /// C# member: `M:LibreMetaverse.LslTools.ObjectList.#ctor`. - pub fn new() -> Result { - libremetaverse_types::not_implemented("M:LibreMetaverse.LslTools.ObjectList.#ctor") - } - /// C# member: `M:LibreMetaverse.LslTools.ObjectList.Add(System.Object)`. - pub fn add(&self, o: libremetaverse_types::compat::Object) -> Result<(), crate::Error> { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.ObjectList.Add(System.Object)", - ) - } - /// C# member: `M:LibreMetaverse.LslTools.ObjectList.GetEnumerator`. - pub fn get_enumerator( - &self, - ) -> Result { - libremetaverse_types::not_implemented("M:LibreMetaverse.LslTools.ObjectList.GetEnumerator") - } - /// C# member: `M:LibreMetaverse.LslTools.ObjectList.Pop`. - pub fn pop(&self) -> Result { - libremetaverse_types::not_implemented("M:LibreMetaverse.LslTools.ObjectList.Pop") - } - /// C# member: `M:LibreMetaverse.LslTools.ObjectList.Push(System.Object)`. - pub fn push(&self, o: libremetaverse_types::compat::Object) -> Result<(), crate::Error> { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.ObjectList.Push(System.Object)", - ) - } - /// C# member: `P:LibreMetaverse.LslTools.ObjectList.Count`. - pub fn count(&self) -> i32 { - libremetaverse_types::unimplemented_api!("P:LibreMetaverse.LslTools.ObjectList.Count") - } - /// Setter for C# member: `P:LibreMetaverse.LslTools.ObjectList.Count`. - pub fn set_count(&mut self, value: i32) { - libremetaverse_types::unimplemented_api!("P:LibreMetaverse.LslTools.ObjectList.Count") - } - /// C# member: `P:LibreMetaverse.LslTools.ObjectList.Item(System.Int32)`. - pub fn item(&self, ix: i32) -> libremetaverse_types::compat::Object { - libremetaverse_types::unimplemented_api!( - "P:LibreMetaverse.LslTools.ObjectList.Item(System.Int32)" - ) - } - /// C# member: `P:LibreMetaverse.LslTools.ObjectList.Top`. - pub fn top(&self) -> libremetaverse_types::compat::Object { - libremetaverse_types::unimplemented_api!("P:LibreMetaverse.LslTools.ObjectList.Top") - } -} +/// C# member: `M:LibreMetaverse.LslTools.ObjectList.#ctor`. +/// C# member: `M:LibreMetaverse.LslTools.ObjectList.Add(System.Object)`. +/// C# member: `M:LibreMetaverse.LslTools.ObjectList.GetEnumerator`. +/// C# member: `M:LibreMetaverse.LslTools.ObjectList.Pop`. +/// C# member: `M:LibreMetaverse.LslTools.ObjectList.Push(System.Object)`. +/// C# member: `P:LibreMetaverse.LslTools.ObjectList.Count`. +/// C# member: `P:LibreMetaverse.LslTools.ObjectList.Item(System.Int32)`. +/// C# member: `P:LibreMetaverse.LslTools.ObjectList.Top`. +pub use crate::lexer::ObjectList; /// C# type: `T:LibreMetaverse.LslTools.ObjectList.OListEnumerator`. pub struct ObjectListOListEnumerator; @@ -3249,46 +2614,13 @@ impl Relation { } /// C# type: `T:LibreMetaverse.LslTools.ResWds`. -pub struct ResWds { - /// C# member: `F:LibreMetaverse.LslTools.ResWds.m_upper`. - pub m_upper: bool, - /// C# member: `F:LibreMetaverse.LslTools.ResWds.m_wds`. - pub m_wds: libremetaverse_types::compat::Hashtable, -} -impl ResWds { - /// C# member: `M:LibreMetaverse.LslTools.ResWds.#ctor`. - pub fn new_with_constructor() -> Result { - libremetaverse_types::not_implemented("M:LibreMetaverse.LslTools.ResWds.#ctor") - } - /// C# member: `M:LibreMetaverse.LslTools.ResWds.Check(LibreMetaverse.LslTools.Lexer,LibreMetaverse.LslTools.TOKEN@)`. - pub fn check( - &self, - yyl: libremetaverse_lsl_tools::Lexer, - tok: &mut libremetaverse_lsl_tools::TOKEN, - ) -> Result<(), crate::Error> { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.ResWds.Check(LibreMetaverse.LslTools.Lexer,LibreMetaverse.LslTools.TOKEN@)", - ) - } - /// C# member: `M:LibreMetaverse.LslTools.ResWds.New(LibreMetaverse.LslTools.TokensGen,System.String)`. - pub fn new_with_tokens_gen_string( - tks: libremetaverse_lsl_tools::TokensGen, - str: String, - ) -> Result { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.ResWds.New(LibreMetaverse.LslTools.TokensGen,System.String)", - ) - } - /// C# member: `M:LibreMetaverse.LslTools.ResWds.Serialise(System.Object,LibreMetaverse.LslTools.Serialiser)`. - pub fn serialise( - o: libremetaverse_types::compat::Object, - s: libremetaverse_lsl_tools::Serialiser, - ) -> Result { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.ResWds.Serialise(System.Object,LibreMetaverse.LslTools.Serialiser)", - ) - } -} +/// C# member: `F:LibreMetaverse.LslTools.ResWds.m_upper`. +/// C# member: `F:LibreMetaverse.LslTools.ResWds.m_wds`. +/// C# member: `M:LibreMetaverse.LslTools.ResWds.#ctor`. +/// C# member: `M:LibreMetaverse.LslTools.ResWds.Check(LibreMetaverse.LslTools.Lexer,LibreMetaverse.LslTools.TOKEN@)`. +/// C# member: `M:LibreMetaverse.LslTools.ResWds.New(LibreMetaverse.LslTools.TokensGen,System.String)`. +/// C# member: `M:LibreMetaverse.LslTools.ResWds.Serialise(System.Object,LibreMetaverse.LslTools.Serialiser)`. +pub use crate::lexer::ResWds; /// C# type: `T:LibreMetaverse.LslTools.SCreator`. pub struct SCreator; @@ -3353,111 +2685,30 @@ impl SEMICOLON { } /// C# type: `T:LibreMetaverse.LslTools.SYMBOL`. -pub struct SYMBOL { - /// C# member: `F:LibreMetaverse.LslTools.SYMBOL.kids`. - pub kids: libremetaverse_lsl_tools::ObjectList, - /// C# member: `F:LibreMetaverse.LslTools.SYMBOL.m_dollar`. - pub m_dollar: libremetaverse_types::compat::Object, - /// C# member: `F:LibreMetaverse.LslTools.SYMBOL.pos`. - pub pos_with_field: i32, - /// C# member: `F:LibreMetaverse.LslTools.SYMBOL.yylx`. - pub yylx: libremetaverse_lsl_tools::Lexer, - /// C# member: `F:LibreMetaverse.LslTools.SYMBOL.yyps`. - pub yyps: libremetaverse_lsl_tools::Parser, -} -impl SYMBOL { - /// C# member: `M:LibreMetaverse.LslTools.SYMBOL.#ctor(LibreMetaverse.LslTools.Lexer)`. - pub fn new_with_lexer(yyl: libremetaverse_lsl_tools::Lexer) -> Result { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.SYMBOL.#ctor(LibreMetaverse.LslTools.Lexer)", - ) - } - /// C# member: `M:LibreMetaverse.LslTools.SYMBOL.#ctor(LibreMetaverse.LslTools.Parser)`. - pub fn new_with_parser(yyp: libremetaverse_lsl_tools::Parser) -> Result { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.SYMBOL.#ctor(LibreMetaverse.LslTools.Parser)", - ) - } - /// C# member: `M:LibreMetaverse.LslTools.SYMBOL.ConcreteSyntaxTree`. - pub fn concrete_syntax_tree(&self) -> Result<(), crate::Error> { - libremetaverse_types::not_implemented("M:LibreMetaverse.LslTools.SYMBOL.ConcreteSyntaxTree") - } - /// C# member: `M:LibreMetaverse.LslTools.SYMBOL.IsAction`. - pub fn is_action(&self) -> Result { - libremetaverse_types::not_implemented("M:LibreMetaverse.LslTools.SYMBOL.IsAction") - } - /// C# member: `M:LibreMetaverse.LslTools.SYMBOL.IsCSymbol`. - pub fn is_c_symbol(&self) -> Result { - libremetaverse_types::not_implemented("M:LibreMetaverse.LslTools.SYMBOL.IsCSymbol") - } - /// C# member: `M:LibreMetaverse.LslTools.SYMBOL.IsTerminal`. - pub fn is_terminal(&self) -> Result { - libremetaverse_types::not_implemented("M:LibreMetaverse.LslTools.SYMBOL.IsTerminal") - } - /// C# member: `M:LibreMetaverse.LslTools.SYMBOL.Matches(System.String)`. - pub fn matches(&self, s: String) -> Result { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.SYMBOL.Matches(System.String)", - ) - } - /// C# member: `M:LibreMetaverse.LslTools.SYMBOL.Pass(LibreMetaverse.LslTools.YyParser,System.Int32,LibreMetaverse.LslTools.ParserEntry@)`. - pub fn pass_( - &self, - syms: libremetaverse_lsl_tools::YyParser, - snum: i32, - entry: &mut libremetaverse_lsl_tools::ParserEntry, - ) -> Result { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.SYMBOL.Pass(LibreMetaverse.LslTools.YyParser,System.Int32,LibreMetaverse.LslTools.ParserEntry@)", - ) - } - /// C# member: `M:LibreMetaverse.LslTools.SYMBOL.Print`. - pub fn print(&self) -> Result<(), crate::Error> { - libremetaverse_types::not_implemented("M:LibreMetaverse.LslTools.SYMBOL.Print") - } - /// C# member: `M:LibreMetaverse.LslTools.SYMBOL.ToString`. - pub fn to_string(&self) -> String { - libremetaverse_types::unimplemented_api!("M:LibreMetaverse.LslTools.SYMBOL.ToString") - } - /// C# member: `M:LibreMetaverse.LslTools.SYMBOL.op_Implicit(LibreMetaverse.LslTools.SYMBOL)~System.Int32`. - pub fn from(s: libremetaverse_lsl_tools::SYMBOL) -> i32 { - libremetaverse_types::unimplemented_api!( - "M:LibreMetaverse.LslTools.SYMBOL.op_Implicit(LibreMetaverse.LslTools.SYMBOL)~System.Int32" - ) - } - /// C# member: `P:LibreMetaverse.LslTools.SYMBOL.Line`. - pub fn line(&self) -> i32 { - libremetaverse_types::unimplemented_api!("P:LibreMetaverse.LslTools.SYMBOL.Line") - } - /// C# member: `P:LibreMetaverse.LslTools.SYMBOL.Pos`. - pub fn pos_with_property(&self) -> String { - libremetaverse_types::unimplemented_api!("P:LibreMetaverse.LslTools.SYMBOL.Pos") - } - /// C# member: `P:LibreMetaverse.LslTools.SYMBOL.Position`. - pub fn position(&self) -> i32 { - libremetaverse_types::unimplemented_api!("P:LibreMetaverse.LslTools.SYMBOL.Position") - } - /// C# member: `P:LibreMetaverse.LslTools.SYMBOL.yyact`. - pub fn yyact(&self) -> libremetaverse_lsl_tools::YyParser { - libremetaverse_types::unimplemented_api!("P:LibreMetaverse.LslTools.SYMBOL.yyact") - } - /// C# member: `P:LibreMetaverse.LslTools.SYMBOL.yylval`. - pub fn yylval(&self) -> libremetaverse_types::compat::Object { - libremetaverse_types::unimplemented_api!("P:LibreMetaverse.LslTools.SYMBOL.yylval") - } - /// Setter for C# member: `P:LibreMetaverse.LslTools.SYMBOL.yylval`. - pub fn set_yylval(&mut self, value: libremetaverse_types::compat::Object) { - libremetaverse_types::unimplemented_api!("P:LibreMetaverse.LslTools.SYMBOL.yylval") - } - /// C# member: `P:LibreMetaverse.LslTools.SYMBOL.yyname`. - pub fn yyname(&self) -> String { - libremetaverse_types::unimplemented_api!("P:LibreMetaverse.LslTools.SYMBOL.yyname") - } - /// C# member: `P:LibreMetaverse.LslTools.SYMBOL.yynum`. - pub fn yynum(&self) -> i32 { - libremetaverse_types::unimplemented_api!("P:LibreMetaverse.LslTools.SYMBOL.yynum") - } -} +/// C# member: `F:LibreMetaverse.LslTools.SYMBOL.kids`. +/// C# member: `F:LibreMetaverse.LslTools.SYMBOL.m_dollar`. +/// C# member: `F:LibreMetaverse.LslTools.SYMBOL.pos`. +/// C# member: `F:LibreMetaverse.LslTools.SYMBOL.yylx`. +/// C# member: `F:LibreMetaverse.LslTools.SYMBOL.yyps`. +/// C# member: `M:LibreMetaverse.LslTools.SYMBOL.#ctor(LibreMetaverse.LslTools.Lexer)`. +/// C# member: `M:LibreMetaverse.LslTools.SYMBOL.#ctor(LibreMetaverse.LslTools.Parser)`. +/// C# member: `M:LibreMetaverse.LslTools.SYMBOL.ConcreteSyntaxTree`. +/// C# member: `M:LibreMetaverse.LslTools.SYMBOL.IsAction`. +/// C# member: `M:LibreMetaverse.LslTools.SYMBOL.IsCSymbol`. +/// C# member: `M:LibreMetaverse.LslTools.SYMBOL.IsTerminal`. +/// C# member: `M:LibreMetaverse.LslTools.SYMBOL.Matches(System.String)`. +/// C# member: `M:LibreMetaverse.LslTools.SYMBOL.Pass(LibreMetaverse.LslTools.YyParser,System.Int32,LibreMetaverse.LslTools.ParserEntry@)`. +/// C# member: `M:LibreMetaverse.LslTools.SYMBOL.Print`. +/// C# member: `M:LibreMetaverse.LslTools.SYMBOL.ToString`. +/// C# member: `M:LibreMetaverse.LslTools.SYMBOL.op_Implicit(LibreMetaverse.LslTools.SYMBOL)~System.Int32`. +/// C# member: `P:LibreMetaverse.LslTools.SYMBOL.Line`. +/// C# member: `P:LibreMetaverse.LslTools.SYMBOL.Pos`. +/// C# member: `P:LibreMetaverse.LslTools.SYMBOL.Position`. +/// C# member: `P:LibreMetaverse.LslTools.SYMBOL.yyact`. +/// C# member: `P:LibreMetaverse.LslTools.SYMBOL.yylval`. +/// C# member: `P:LibreMetaverse.LslTools.SYMBOL.yyname`. +/// C# member: `P:LibreMetaverse.LslTools.SYMBOL.yynum`. +pub use crate::lexer::SYMBOL; /// C# type: `T:LibreMetaverse.LslTools.Serialiser`. pub struct Serialiser; @@ -3529,58 +2780,18 @@ impl Sfactory { } /// C# type: `T:LibreMetaverse.LslTools.SourceLineInfo`. -pub struct SourceLineInfo { - /// C# member: `F:LibreMetaverse.LslTools.SourceLineInfo.charPosition`. - pub char_position: i32, - /// C# member: `F:LibreMetaverse.LslTools.SourceLineInfo.endOfLine`. - pub end_of_line: i32, - /// C# member: `F:LibreMetaverse.LslTools.SourceLineInfo.lineNumber`. - pub line_number: i32, - /// C# member: `F:LibreMetaverse.LslTools.SourceLineInfo.lxr`. - pub lxr: libremetaverse_lsl_tools::Lexer, - /// C# member: `F:LibreMetaverse.LslTools.SourceLineInfo.rawCharPosition`. - pub raw_char_position: i32, - /// C# member: `F:LibreMetaverse.LslTools.SourceLineInfo.startOfLine`. - pub start_of_line: i32, -} -impl SourceLineInfo { - /// C# member: `M:LibreMetaverse.LslTools.SourceLineInfo.#ctor(LibreMetaverse.LslTools.Lexer,System.Int32)`. - pub fn new_with_lexer_int32( - lx: libremetaverse_lsl_tools::Lexer, - pos: i32, - ) -> Result { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.SourceLineInfo.#ctor(LibreMetaverse.LslTools.Lexer,System.Int32)", - ) - } - /// C# member: `M:LibreMetaverse.LslTools.SourceLineInfo.#ctor(LibreMetaverse.LslTools.LineManager,System.Int32)`. - pub fn new_with_line_manager_int32( - lm: libremetaverse_lsl_tools::LineManager, - pos: i32, - ) -> Result { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.SourceLineInfo.#ctor(LibreMetaverse.LslTools.LineManager,System.Int32)", - ) - } - /// C# member: `M:LibreMetaverse.LslTools.SourceLineInfo.#ctor(System.Int32)`. - pub fn new_with_int32(pos: i32) -> Result { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.SourceLineInfo.#ctor(System.Int32)", - ) - } - /// C# member: `M:LibreMetaverse.LslTools.SourceLineInfo.ToString`. - pub fn to_string(&self) -> String { - libremetaverse_types::unimplemented_api!( - "M:LibreMetaverse.LslTools.SourceLineInfo.ToString" - ) - } - /// C# member: `P:LibreMetaverse.LslTools.SourceLineInfo.sourceLine`. - pub fn source_line(&self) -> String { - libremetaverse_types::unimplemented_api!( - "P:LibreMetaverse.LslTools.SourceLineInfo.sourceLine" - ) - } -} +/// C# member: `F:LibreMetaverse.LslTools.SourceLineInfo.charPosition`. +/// C# member: `F:LibreMetaverse.LslTools.SourceLineInfo.endOfLine`. +/// C# member: `F:LibreMetaverse.LslTools.SourceLineInfo.lineNumber`. +/// C# member: `F:LibreMetaverse.LslTools.SourceLineInfo.lxr`. +/// C# member: `F:LibreMetaverse.LslTools.SourceLineInfo.rawCharPosition`. +/// C# member: `F:LibreMetaverse.LslTools.SourceLineInfo.startOfLine`. +/// C# member: `M:LibreMetaverse.LslTools.SourceLineInfo.#ctor(LibreMetaverse.LslTools.Lexer,System.Int32)`. +/// C# member: `M:LibreMetaverse.LslTools.SourceLineInfo.#ctor(LibreMetaverse.LslTools.LineManager,System.Int32)`. +/// C# member: `M:LibreMetaverse.LslTools.SourceLineInfo.#ctor(System.Int32)`. +/// C# member: `M:LibreMetaverse.LslTools.SourceLineInfo.ToString`. +/// C# member: `P:LibreMetaverse.LslTools.SourceLineInfo.sourceLine`. +pub use crate::lexer::SourceLineInfo; /// C# type: `T:LibreMetaverse.LslTools.Stuff`. pub struct Stuff; @@ -3930,75 +3141,18 @@ impl THIS { } /// C# type: `T:LibreMetaverse.LslTools.TOKEN`. -pub struct TOKEN; -impl TOKEN { - /// C# member: `M:LibreMetaverse.LslTools.TOKEN.#ctor(LibreMetaverse.LslTools.Lexer)`. - pub fn new_with_lexer(yyl: libremetaverse_lsl_tools::Lexer) -> Result { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.TOKEN.#ctor(LibreMetaverse.LslTools.Lexer)", - ) - } - /// C# member: `M:LibreMetaverse.LslTools.TOKEN.#ctor(LibreMetaverse.LslTools.Lexer,System.String)`. - pub fn new_with_lexer_string( - yyl: libremetaverse_lsl_tools::Lexer, - s: String, - ) -> Result { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.TOKEN.#ctor(LibreMetaverse.LslTools.Lexer,System.String)", - ) - } - /// C# member: `M:LibreMetaverse.LslTools.TOKEN.#ctor(LibreMetaverse.LslTools.Parser)`. - pub fn new_with_parser(yyp: libremetaverse_lsl_tools::Parser) -> Result { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.TOKEN.#ctor(LibreMetaverse.LslTools.Parser)", - ) - } - /// C# member: `M:LibreMetaverse.LslTools.TOKEN.IsTerminal`. - pub fn is_terminal(&self) -> Result { - libremetaverse_types::not_implemented("M:LibreMetaverse.LslTools.TOKEN.IsTerminal") - } - /// C# member: `M:LibreMetaverse.LslTools.TOKEN.Matches(System.String)`. - pub fn matches(&self, s: String) -> Result { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.TOKEN.Matches(System.String)", - ) - } - /// C# member: `M:LibreMetaverse.LslTools.TOKEN.Pass(LibreMetaverse.LslTools.YyParser,System.Int32,LibreMetaverse.LslTools.ParserEntry@)`. - pub fn pass_( - &self, - syms: libremetaverse_lsl_tools::YyParser, - snum: i32, - entry: &mut libremetaverse_lsl_tools::ParserEntry, - ) -> Result { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.TOKEN.Pass(LibreMetaverse.LslTools.YyParser,System.Int32,LibreMetaverse.LslTools.ParserEntry@)", - ) - } - /// C# member: `M:LibreMetaverse.LslTools.TOKEN.Print`. - pub fn print(&self) -> Result<(), crate::Error> { - libremetaverse_types::not_implemented("M:LibreMetaverse.LslTools.TOKEN.Print") - } - /// C# member: `M:LibreMetaverse.LslTools.TOKEN.ToString`. - pub fn to_string(&self) -> String { - libremetaverse_types::unimplemented_api!("M:LibreMetaverse.LslTools.TOKEN.ToString") - } - /// C# member: `P:LibreMetaverse.LslTools.TOKEN.yyname`. - pub fn yyname(&self) -> String { - libremetaverse_types::unimplemented_api!("P:LibreMetaverse.LslTools.TOKEN.yyname") - } - /// C# member: `P:LibreMetaverse.LslTools.TOKEN.yynum`. - pub fn yynum(&self) -> i32 { - libremetaverse_types::unimplemented_api!("P:LibreMetaverse.LslTools.TOKEN.yynum") - } - /// C# member: `P:LibreMetaverse.LslTools.TOKEN.yytext`. - pub fn yytext(&self) -> String { - libremetaverse_types::unimplemented_api!("P:LibreMetaverse.LslTools.TOKEN.yytext") - } - /// Setter for C# member: `P:LibreMetaverse.LslTools.TOKEN.yytext`. - pub fn set_yytext(&mut self, value: String) { - libremetaverse_types::unimplemented_api!("P:LibreMetaverse.LslTools.TOKEN.yytext") - } -} +/// C# member: `M:LibreMetaverse.LslTools.TOKEN.#ctor(LibreMetaverse.LslTools.Lexer)`. +/// C# member: `M:LibreMetaverse.LslTools.TOKEN.#ctor(LibreMetaverse.LslTools.Lexer,System.String)`. +/// C# member: `M:LibreMetaverse.LslTools.TOKEN.#ctor(LibreMetaverse.LslTools.Parser)`. +/// C# member: `M:LibreMetaverse.LslTools.TOKEN.IsTerminal`. +/// C# member: `M:LibreMetaverse.LslTools.TOKEN.Matches(System.String)`. +/// C# member: `M:LibreMetaverse.LslTools.TOKEN.Pass(LibreMetaverse.LslTools.YyParser,System.Int32,LibreMetaverse.LslTools.ParserEntry@)`. +/// C# member: `M:LibreMetaverse.LslTools.TOKEN.Print`. +/// C# member: `M:LibreMetaverse.LslTools.TOKEN.ToString`. +/// C# member: `P:LibreMetaverse.LslTools.TOKEN.yyname`. +/// C# member: `P:LibreMetaverse.LslTools.TOKEN.yynum`. +/// C# member: `P:LibreMetaverse.LslTools.TOKEN.yytext`. +pub use crate::lexer::TOKEN; /// C# type: `T:LibreMetaverse.LslTools.Tfactory`. pub struct Tfactory; @@ -4221,77 +3375,24 @@ impl Transition { } /// C# type: `T:LibreMetaverse.LslTools.YyLexer`. -pub struct YyLexer { - /// C# member: `F:LibreMetaverse.LslTools.YyLexer.cats`. - pub cats: libremetaverse_types::compat::Hashtable, - /// C# member: `F:LibreMetaverse.LslTools.YyLexer.erh`. - pub erh: libremetaverse_lsl_tools::ErrorHandler, - /// C# member: `F:LibreMetaverse.LslTools.YyLexer.m_encoding`. - pub m_encoding: libremetaverse_types::compat::TextEncoding, - /// C# member: `F:LibreMetaverse.LslTools.YyLexer.m_gencat`. - pub m_gencat: libremetaverse_types::compat::UnicodeCategory, - /// C# member: `F:LibreMetaverse.LslTools.YyLexer.reswds`. - pub reswds: libremetaverse_types::compat::Hashtable, - /// C# member: `F:LibreMetaverse.LslTools.YyLexer.starts`. - pub starts: libremetaverse_types::compat::Hashtable, - /// C# member: `F:LibreMetaverse.LslTools.YyLexer.tokens`. - pub tokens: libremetaverse_types::compat::Hashtable, - /// C# member: `F:LibreMetaverse.LslTools.YyLexer.toupper`. - pub toupper: bool, - /// C# member: `F:LibreMetaverse.LslTools.YyLexer.types`. - pub types: libremetaverse_types::compat::Hashtable, - /// C# member: `F:LibreMetaverse.LslTools.YyLexer.usingEOF`. - pub using_eof: bool, -} -impl YyLexer { - /// C# member: `M:LibreMetaverse.LslTools.YyLexer.#ctor(LibreMetaverse.LslTools.ErrorHandler)`. - pub fn new(eh: libremetaverse_lsl_tools::ErrorHandler) -> Result { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.YyLexer.#ctor(LibreMetaverse.LslTools.ErrorHandler)", - ) - } - /// C# member: `M:LibreMetaverse.LslTools.YyLexer.EmitDfa(System.IO.TextWriter)`. - pub fn emit_dfa(&self, out_file: Box) -> Result<(), crate::Error> { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.YyLexer.EmitDfa(System.IO.TextWriter)", - ) - } - /// C# member: `M:LibreMetaverse.LslTools.YyLexer.GetDfa`. - pub fn get_dfa(&self) -> Result<(), crate::Error> { - libremetaverse_types::not_implemented("M:LibreMetaverse.LslTools.YyLexer.GetDfa") - } - /// C# member: `M:LibreMetaverse.LslTools.YyLexer.GetEnumerator`. - pub fn get_enumerator( - &self, - ) -> Result { - libremetaverse_types::not_implemented("M:LibreMetaverse.LslTools.YyLexer.GetEnumerator") - } - /// C# member: `M:LibreMetaverse.LslTools.YyLexer.OldAction(LibreMetaverse.LslTools.Lexer,System.String@,System.Int32,System.Boolean@)`. - pub fn old_action( - &self, - yyl: libremetaverse_lsl_tools::Lexer, - yytext: &mut String, - action: i32, - reject: &mut bool, - ) -> Result { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.YyLexer.OldAction(LibreMetaverse.LslTools.Lexer,System.String@,System.Int32,System.Boolean@)", - ) - } - /// C# member: `M:LibreMetaverse.LslTools.YyLexer.UsingCat(System.Globalization.UnicodeCategory)`. - pub fn using_cat( - &self, - cat: libremetaverse_types::compat::UnicodeCategory, - ) -> Result { - libremetaverse_types::not_implemented( - "M:LibreMetaverse.LslTools.YyLexer.UsingCat(System.Globalization.UnicodeCategory)", - ) - } - /// C# member: `P:LibreMetaverse.LslTools.YyLexer.InputEncoding`. - pub fn set_input_encoding(&mut self, value: String) { - libremetaverse_types::unimplemented_api!("P:LibreMetaverse.LslTools.YyLexer.InputEncoding") - } -} +/// C# member: `F:LibreMetaverse.LslTools.YyLexer.cats`. +/// C# member: `F:LibreMetaverse.LslTools.YyLexer.erh`. +/// C# member: `F:LibreMetaverse.LslTools.YyLexer.m_encoding`. +/// C# member: `F:LibreMetaverse.LslTools.YyLexer.m_gencat`. +/// C# member: `F:LibreMetaverse.LslTools.YyLexer.reswds`. +/// C# member: `F:LibreMetaverse.LslTools.YyLexer.starts`. +/// C# member: `F:LibreMetaverse.LslTools.YyLexer.tokens`. +/// C# member: `F:LibreMetaverse.LslTools.YyLexer.toupper`. +/// C# member: `F:LibreMetaverse.LslTools.YyLexer.types`. +/// C# member: `F:LibreMetaverse.LslTools.YyLexer.usingEOF`. +/// C# member: `M:LibreMetaverse.LslTools.YyLexer.#ctor(LibreMetaverse.LslTools.ErrorHandler)`. +/// C# member: `M:LibreMetaverse.LslTools.YyLexer.EmitDfa(System.IO.TextWriter)`. +/// C# member: `M:LibreMetaverse.LslTools.YyLexer.GetDfa`. +/// C# member: `M:LibreMetaverse.LslTools.YyLexer.GetEnumerator`. +/// C# member: `M:LibreMetaverse.LslTools.YyLexer.OldAction(LibreMetaverse.LslTools.Lexer,System.String@,System.Int32,System.Boolean@)`. +/// C# member: `M:LibreMetaverse.LslTools.YyLexer.UsingCat(System.Globalization.UnicodeCategory)`. +/// C# member: `P:LibreMetaverse.LslTools.YyLexer.InputEncoding`. +pub use crate::lexer::YyLexer; /// C# type: `T:LibreMetaverse.LslTools.YyParser`. pub struct YyParser { diff --git a/crates/libremetaverse-lsl-tools/src/lexer.rs b/crates/libremetaverse-lsl-tools/src/lexer.rs new file mode 100644 index 0000000..1495568 --- /dev/null +++ b/crates/libremetaverse-lsl-tools/src/lexer.rs @@ -0,0 +1,3023 @@ +//! Native lexer runtime and source diagnostics. +//! +//! Positions in this module are UTF-16 code-unit offsets. That is deliberate: +//! the original C# API exposes `System.Char` positions, and using byte offsets +//! would move every diagnostic after a non-ASCII character. + +#![allow(clippy::inherent_to_string_shadow_display)] +#![allow(clippy::missing_errors_doc)] +#![allow(clippy::missing_panics_doc)] +#![allow(clippy::must_use_candidate)] +#![allow(clippy::needless_pass_by_value)] +#![allow(clippy::should_implement_trait)] +#![allow(clippy::too_many_lines)] + +use std::collections::{BTreeMap, BTreeSet, VecDeque}; +use std::fmt; +use std::io::Write; + +use libremetaverse_types::compat::{ + Hashtable, IEnumerator, Object, StreamReader, TextEncoding, UnicodeCategory, Utf16CodeUnit, +}; +use unicode_general_category::{GeneralCategory, get_general_category}; + +use crate::{Error, Nfa, Parser, ParserEntry, Serialiser, SymbolsGen, TokensGen, YyParser}; + +/// Largest source accepted by the compatibility reader (64 Mi UTF-16 units). +pub const MAX_SOURCE_UNITS: usize = 64 * 1024 * 1024; +/// Largest individual token accepted by the lexer (16 Mi UTF-16 units). +pub const MAX_TOKEN_UNITS: usize = 16 * 1024 * 1024; + +fn index(value: i32) -> Result { + usize::try_from(value).map_err(|_| Error::IndexOutOfRange) +} + +fn i32_len(value: usize) -> Result { + i32::try_from(value).map_err(|_| Error::IndexOutOfRange) +} + +fn units_to_string(units: &[u16]) -> String { + String::from_utf16_lossy(units) +} + +fn utf16_len(value: &str) -> usize { + value.encode_utf16().count() +} + +/// .NET-compatible Unicode general categories. +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +#[repr(i32)] +pub enum DotNetUnicodeCategory { + UppercaseLetter = 0, + LowercaseLetter = 1, + TitlecaseLetter = 2, + ModifierLetter = 3, + OtherLetter = 4, + NonSpacingMark = 5, + SpacingCombiningMark = 6, + EnclosingMark = 7, + DecimalDigitNumber = 8, + LetterNumber = 9, + OtherNumber = 10, + SpaceSeparator = 11, + LineSeparator = 12, + ParagraphSeparator = 13, + Control = 14, + Format = 15, + Surrogate = 16, + PrivateUse = 17, + ConnectorPunctuation = 18, + DashPunctuation = 19, + OpenPunctuation = 20, + ClosePunctuation = 21, + InitialQuotePunctuation = 22, + FinalQuotePunctuation = 23, + OtherPunctuation = 24, + MathSymbol = 25, + CurrencySymbol = 26, + ModifierSymbol = 27, + OtherSymbol = 28, + OtherNotAssigned = 29, +} + +impl DotNetUnicodeCategory { + fn from_compat(value: UnicodeCategory) -> Result { + Self::try_from(value.0) + } + + fn of_unit(unit: u16) -> Self { + let Some(ch) = char::from_u32(u32::from(unit)) else { + return Self::Surrogate; + }; + match get_general_category(ch) { + GeneralCategory::UppercaseLetter => Self::UppercaseLetter, + GeneralCategory::LowercaseLetter => Self::LowercaseLetter, + GeneralCategory::TitlecaseLetter => Self::TitlecaseLetter, + GeneralCategory::ModifierLetter => Self::ModifierLetter, + GeneralCategory::OtherLetter => Self::OtherLetter, + GeneralCategory::NonspacingMark => Self::NonSpacingMark, + GeneralCategory::SpacingMark => Self::SpacingCombiningMark, + GeneralCategory::EnclosingMark => Self::EnclosingMark, + GeneralCategory::DecimalNumber => Self::DecimalDigitNumber, + GeneralCategory::LetterNumber => Self::LetterNumber, + GeneralCategory::OtherNumber => Self::OtherNumber, + GeneralCategory::SpaceSeparator => Self::SpaceSeparator, + GeneralCategory::LineSeparator => Self::LineSeparator, + GeneralCategory::ParagraphSeparator => Self::ParagraphSeparator, + GeneralCategory::Control => Self::Control, + GeneralCategory::Format => Self::Format, + GeneralCategory::Surrogate => Self::Surrogate, + GeneralCategory::PrivateUse => Self::PrivateUse, + GeneralCategory::ConnectorPunctuation => Self::ConnectorPunctuation, + GeneralCategory::DashPunctuation => Self::DashPunctuation, + GeneralCategory::OpenPunctuation => Self::OpenPunctuation, + GeneralCategory::ClosePunctuation => Self::ClosePunctuation, + GeneralCategory::InitialPunctuation => Self::InitialQuotePunctuation, + GeneralCategory::FinalPunctuation => Self::FinalQuotePunctuation, + GeneralCategory::OtherPunctuation => Self::OtherPunctuation, + GeneralCategory::MathSymbol => Self::MathSymbol, + GeneralCategory::CurrencySymbol => Self::CurrencySymbol, + GeneralCategory::ModifierSymbol => Self::ModifierSymbol, + GeneralCategory::OtherSymbol => Self::OtherSymbol, + _ => Self::OtherNotAssigned, + } + } + + fn name(self) -> &'static str { + match self { + Self::UppercaseLetter => "UppercaseLetter", + Self::LowercaseLetter => "LowercaseLetter", + Self::TitlecaseLetter => "TitlecaseLetter", + Self::ModifierLetter => "ModifierLetter", + Self::OtherLetter => "OtherLetter", + Self::NonSpacingMark => "NonSpacingMark", + Self::SpacingCombiningMark => "SpacingCombiningMark", + Self::EnclosingMark => "EnclosingMark", + Self::DecimalDigitNumber => "DecimalDigitNumber", + Self::LetterNumber => "LetterNumber", + Self::OtherNumber => "OtherNumber", + Self::SpaceSeparator => "SpaceSeparator", + Self::LineSeparator => "LineSeparator", + Self::ParagraphSeparator => "ParagraphSeparator", + Self::Control => "Control", + Self::Format => "Format", + Self::Surrogate => "Surrogate", + Self::PrivateUse => "PrivateUse", + Self::ConnectorPunctuation => "ConnectorPunctuation", + Self::DashPunctuation => "DashPunctuation", + Self::OpenPunctuation => "OpenPunctuation", + Self::ClosePunctuation => "ClosePunctuation", + Self::InitialQuotePunctuation => "InitialQuotePunctuation", + Self::FinalQuotePunctuation => "FinalQuotePunctuation", + Self::OtherPunctuation => "OtherPunctuation", + Self::MathSymbol => "MathSymbol", + Self::CurrencySymbol => "CurrencySymbol", + Self::ModifierSymbol => "ModifierSymbol", + Self::OtherSymbol => "OtherSymbol", + Self::OtherNotAssigned => "OtherNotAssigned", + } + } +} + +impl TryFrom for DotNetUnicodeCategory { + type Error = Error; + + fn try_from(value: i32) -> Result { + Ok(match value { + 0 => Self::UppercaseLetter, + 1 => Self::LowercaseLetter, + 2 => Self::TitlecaseLetter, + 3 => Self::ModifierLetter, + 4 => Self::OtherLetter, + 5 => Self::NonSpacingMark, + 6 => Self::SpacingCombiningMark, + 7 => Self::EnclosingMark, + 8 => Self::DecimalDigitNumber, + 9 => Self::LetterNumber, + 10 => Self::OtherNumber, + 11 => Self::SpaceSeparator, + 12 => Self::LineSeparator, + 13 => Self::ParagraphSeparator, + 14 => Self::Control, + 15 => Self::Format, + 16 => Self::Surrogate, + 17 => Self::PrivateUse, + 18 => Self::ConnectorPunctuation, + 19 => Self::DashPunctuation, + 20 => Self::OpenPunctuation, + 21 => Self::ClosePunctuation, + 22 => Self::InitialQuotePunctuation, + 23 => Self::FinalQuotePunctuation, + 24 => Self::OtherPunctuation, + 25 => Self::MathSymbol, + 26 => Self::CurrencySymbol, + 27 => Self::ModifierSymbol, + 28 => Self::OtherSymbol, + 29 => Self::OtherNotAssigned, + _ => return Err(Error::Argument), + }) + } +} + +/// Character-category predicate corresponding to the C# `CatTest` delegate. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct CatTest { + category: DotNetUnicodeCategory, +} + +impl CatTest { + /// Creates a category predicate. + pub fn new(category: UnicodeCategory) -> Result { + Ok(Self { + category: DotNetUnicodeCategory::from_compat(category)?, + }) + } + + /// Tests one UTF-16 code unit using .NET category numbering. + pub fn test(&self, ch: Utf16CodeUnit) -> Result { + Ok(DotNetUnicodeCategory::of_unit(ch.0) == self.category) + } +} + +/// Input encodings supported by lexer source files. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub enum InputEncoding { + Ascii, + AsciiUpper, + #[default] + Utf8, + Utf16Le, + Utf16Be, +} + +impl InputEncoding { + /// Resolves the names accepted by the C# runtime. + pub fn parse(name: &str) -> Result { + match name.trim().to_ascii_uppercase().as_str() { + "" | "UTF8" | "UTF-8" | "65001" => Ok(Self::Utf8), + "ASCII" | "US-ASCII" | "20127" => Ok(Self::Ascii), + "ASCIICAPS" => Ok(Self::AsciiUpper), + "UNICODE" | "UTF16" | "UTF-16" | "UTF-16LE" | "1200" => Ok(Self::Utf16Le), + "UTF-16BE" | "1201" => Ok(Self::Utf16Be), + // UTF-7 is intentionally rejected: it is obsolete and unsafe, and + // modern .NET also disables it unless explicitly enabled. + _ => Err(Error::Argument), + } + } + + fn decode(self, bytes: &[u8]) -> Result { + let bytes = match bytes.strip_prefix(&[0xEF, 0xBB, 0xBF]) { + Some(rest) => rest, + None => bytes, + }; + match self { + Self::Utf8 => String::from_utf8(bytes.to_vec()).map_err(|error| Error::Parse { + position: error.utf8_error().valid_up_to(), + context: "source is not valid UTF-8", + }), + Self::Ascii | Self::AsciiUpper => { + if let Some(position) = bytes.iter().position(|byte| !byte.is_ascii()) { + return Err(Error::Parse { + position, + context: "source contains a non-ASCII byte", + }); + } + let value = String::from_utf8(bytes.to_vec()).map_err(|_| Error::Argument)?; + Ok(if self == Self::AsciiUpper { + value.to_ascii_uppercase() + } else { + value + }) + } + Self::Utf16Le | Self::Utf16Be => { + let (encoding, bytes) = if let Some(bytes) = bytes.strip_prefix(&[0xFF, 0xFE]) { + (Self::Utf16Le, bytes) + } else if let Some(bytes) = bytes.strip_prefix(&[0xFE, 0xFF]) { + (Self::Utf16Be, bytes) + } else { + (self, bytes) + }; + if bytes.len() % 2 != 0 { + return Err(Error::Parse { + position: bytes.len() - 1, + context: "UTF-16 source has a trailing byte", + }); + } + let units = bytes + .chunks_exact(2) + .map(|pair| match encoding { + Self::Utf16Le => u16::from_le_bytes([pair[0], pair[1]]), + Self::Utf16Be => u16::from_be_bytes([pair[0], pair[1]]), + _ => unreachable!(), + }) + .collect::>(); + String::from_utf16(&units).map_err(|_| Error::Parse { + position: first_unpaired_surrogate(&units), + context: "source contains an unpaired UTF-16 surrogate", + }) + } + } + } +} + +fn first_unpaired_surrogate(units: &[u16]) -> usize { + let mut index = 0; + while index < units.len() { + match units[index] { + 0xD800..=0xDBFF + if units + .get(index + 1) + .is_some_and(|unit| (0xDC00..=0xDFFF).contains(unit)) => + { + index += 2; + } + 0xD800..=0xDFFF => return index, + _ => index += 1, + } + } + units.len() +} + +/// Unicode character set used by generated lexer transitions. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct Charset { + pub(crate) category: DotNetUnicodeCategory, + pub(crate) generic: u16, + pub(crate) explicit: BTreeSet, +} + +impl Charset { + fn for_category(category: DotNetUnicodeCategory) -> Self { + let generic = (0..=u16::MAX) + .find(|unit| DotNetUnicodeCategory::of_unit(*unit) == category) + .unwrap_or(0); + Self { + category, + generic, + explicit: BTreeSet::from([generic]), + } + } + + /// Resolves a source encoding and updates the C# uppercase compatibility flag. + pub fn get_encoding( + enc: String, + toupper: &mut bool, + mut erh: ErrorHandler, + ) -> Result { + if let Ok(encoding) = InputEncoding::parse(&enc) { + *toupper = encoding == InputEncoding::AsciiUpper; + Ok(TextEncoding) + } else { + let diagnostic = CSToolsException::new_with_int32_string( + 43, + format!("Warning: Encoding {enc} unknown: ignored"), + )?; + erh.error(diagnostic)?; + *toupper = false; + Ok(TextEncoding) + } + } + + /// Rejects the generator serializer boundary until its native table codec is + /// available. Lexer tables use [`YyLexer::emit_dfa`] in this runtime. + pub fn serialise(o: Object, _s: Serialiser) -> Result { + let _ = o; + Err(Error::InvalidOperation) + } + + /// Tests whether a UTF-16 unit belongs to this set. + #[must_use] + pub fn contains(&self, unit: u16) -> bool { + self.explicit.contains(&unit) || DotNetUnicodeCategory::of_unit(unit) == self.category + } +} + +/// One removed comment segment, measured in UTF-16 code units. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct CommentList { + pub spos: i32, + pub len: i32, + pub tail: Option>, +} + +impl CommentList { + pub fn new(st: i32, ln: i32, tail: CommentList) -> Result { + if st < 0 || ln < 0 { + return Err(Error::Argument); + } + Ok(Self { + spos: st, + len: ln, + tail: Some(Box::new(tail)), + }) + } +} + +/// Linked-list compatibility view of a source line. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct LineList { + pub head: i32, + pub comments: Option>, + pub tail: Option>, +} + +impl LineList { + pub fn new(head: i32, tail: LineList) -> Result { + if head < 0 { + return Err(Error::Argument); + } + Ok(Self { + head, + comments: None, + tail: Some(Box::new(tail)), + }) + } + + fn first(head: i32) -> Self { + Self { + head, + comments: None, + tail: None, + } + } + + pub fn getpos(&self, pos: i32) -> Result { + if pos < self.head { + return Err(Error::IndexOutOfRange); + } + let mut result = pos - self.head; + let mut comment = self.comments.as_deref(); + while let Some(item) = comment { + if pos > item.spos { + result = result.checked_add(item.len).ok_or(Error::IndexOutOfRange)?; + } + comment = item.tail.as_deref(); + } + Ok(result) + } +} + +/// Tracks filtered and raw line positions. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct LineManager { + pub end: i32, + pub lines: i32, + pub list: Option>, + starts: Vec, + logical_lines: Vec, +} + +impl Default for LineManager { + fn default() -> Self { + Self { + end: 0, + lines: 1, + list: None, + starts: vec![0], + logical_lines: vec![1], + } + } +} + +impl LineManager { + pub fn new() -> Result { + Ok(Self::default()) + } + + pub fn newline(&mut self, pos: i32) -> Result<(), Error> { + self.backto(pos)?; + self.lines = self.lines.checked_add(1).ok_or(Error::IndexOutOfRange)?; + self.starts.push(pos); + self.logical_lines.push(self.lines); + let previous = self.list.take(); + self.list = Some(Box::new(LineList { + head: pos, + comments: None, + tail: previous, + })); + Ok(()) + } + + pub fn backto(&mut self, pos: i32) -> Result<(), Error> { + if pos < 0 { + return Err(Error::Argument); + } + self.end = self.end.max(pos); + while self.starts.len() > 1 && self.starts.last().is_some_and(|start| *start >= pos) { + self.starts.pop(); + self.logical_lines.pop(); + self.lines -= 1; + self.list = self.list.take().and_then(|line| line.tail); + } + Ok(()) + } + + pub fn comment(&mut self, pos: i32, len: i32) -> Result<(), Error> { + if pos < 0 || len < 0 { + return Err(Error::Argument); + } + self.end = self.end.max(pos); + if self.list.is_none() { + self.list = Some(Box::new(LineList::first(0))); + self.lines = 1; + } + if let Some(line) = self.list.as_mut() { + let tail = line.comments.take(); + line.comments = Some(Box::new(CommentList { + spos: pos, + len, + tail, + })); + } + Ok(()) + } + + fn set_logical_line(&mut self, line: i32) -> Result<(), Error> { + if line < 1 { + return Err(Error::Argument); + } + self.lines = line - 1; + if let Some(last) = self.logical_lines.last_mut() { + *last = line - 1; + } + Ok(()) + } + + fn locate(&self, pos: i32) -> Result<(usize, i32, i32, i32), Error> { + if pos < 0 || pos > self.end { + return Err(Error::IndexOutOfRange); + } + let line_index = self + .starts + .partition_point(|start| *start <= pos) + .saturating_sub(1); + let start = self.starts[line_index]; + let end = self.starts.get(line_index + 1).copied().unwrap_or(self.end); + let line = self + .logical_lines + .get(line_index) + .copied() + .unwrap_or_else(|| i32::try_from(line_index + 1).unwrap_or(i32::MAX)); + Ok((line_index, line, start, end)) + } +} + +/// Source location compatible with `SourceLineInfo`. +#[derive(Clone, Debug)] +pub struct SourceLineInfo { + pub char_position: i32, + pub end_of_line: i32, + pub line_number: i32, + pub lxr: Option>, + pub raw_char_position: i32, + pub start_of_line: i32, + source_snapshot: String, +} + +impl SourceLineInfo { + pub fn new_with_int32(pos: i32) -> Result { + if pos < 0 { + return Err(Error::Argument); + } + Ok(Self { + char_position: 1, + end_of_line: pos, + line_number: 1, + lxr: None, + raw_char_position: pos, + start_of_line: 0, + source_snapshot: String::new(), + }) + } + + pub fn new_with_line_manager_int32(manager: LineManager, pos: i32) -> Result { + Self::from_manager(&manager, pos, String::new()) + } + + pub fn new_with_lexer_int32(lexer: Lexer, pos: i32) -> Result { + Self::from_manager(&lexer.m_line_manager, pos, lexer.m_buf.clone()) + } + + fn from_manager(manager: &LineManager, pos: i32, source: String) -> Result { + let (line_index, line_number, start, end) = manager.locate(pos)?; + let mut raw = pos - start + i32::from(line_index > 0); + let mut line = manager.list.as_deref(); + while let Some(candidate) = line { + if candidate.head == start { + raw = candidate + .getpos(pos)? + .checked_add(i32::from(line_index > 0)) + .ok_or(Error::IndexOutOfRange)?; + break; + } + line = candidate.tail.as_deref(); + } + Ok(Self { + char_position: pos - start + 1, + end_of_line: end, + line_number, + lxr: None, + raw_char_position: raw, + start_of_line: start, + source_snapshot: source, + }) + } + + #[must_use] + pub fn source_line(&self) -> String { + if self.source_snapshot.is_empty() { + return String::new(); + } + let units = self.source_snapshot.encode_utf16().collect::>(); + let start = usize::try_from(self.start_of_line) + .unwrap_or(0) + .min(units.len()); + let end = usize::try_from(self.end_of_line) + .unwrap_or(units.len()) + .min(units.len()); + units_to_string(&units[start..end]) + .trim_end_matches('\n') + .to_owned() + } + + #[must_use] + pub fn to_string(&self) -> String { + format!("Line {}, char {}", self.line_number, self.raw_char_position) + } +} + +impl fmt::Display for SourceLineInfo { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(&self.to_string()) + } +} + +/// Stable diagnostic categories emitted by the lexer runtime. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum DiagnosticCategory { + Encoding, + InvalidCharacter, + InvalidState, + Source, + TokenTooLong, + UnknownCharacterSet, + UnexpectedEof, +} + +/// Diagnostic severity. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum DiagnosticSeverity { + Warning, + Error, + Fatal, + Stop, +} + +/// Structured diagnostic suitable for deterministic fixture assertions. +#[derive(Clone, Debug)] +pub struct Diagnostic { + pub code: i32, + pub category: DiagnosticCategory, + pub severity: DiagnosticSeverity, + pub message: String, + pub location: SourceLineInfo, + pub input: String, +} + +/// C# compatibility exception carrying a structured native diagnostic. +#[derive(Debug)] +pub struct CSToolsException { + pub handled: bool, + pub n_exception_number: i32, + pub s_input: String, + pub sl_info: SourceLineInfo, + pub sym: Option>, + pub message: String, + severity: DiagnosticSeverity, + category: DiagnosticCategory, +} + +impl Clone for CSToolsException { + fn clone(&self) -> Self { + Self { + handled: self.handled, + n_exception_number: self.n_exception_number, + s_input: self.s_input.clone(), + sl_info: self.sl_info.clone(), + sym: None, + message: self.message.clone(), + severity: self.severity, + category: self.category, + } + } +} + +impl CSToolsException { + fn category_for_code(number: i32) -> DiagnosticCategory { + match number { + 2 => DiagnosticCategory::InvalidState, + 24 => DiagnosticCategory::UnknownCharacterSet, + 43 => DiagnosticCategory::Encoding, + _ => DiagnosticCategory::Source, + } + } + + fn create( + number: i32, + location: SourceLineInfo, + input: String, + message: String, + severity: DiagnosticSeverity, + category: DiagnosticCategory, + ) -> Self { + Self { + handled: false, + n_exception_number: number, + s_input: input, + sl_info: location, + sym: None, + message, + severity, + category, + } + } + + pub fn new_with_int32_string(number: i32, message: String) -> Result { + Ok(Self::create( + number, + SourceLineInfo::new_with_int32(0)?, + String::new(), + message, + DiagnosticSeverity::Error, + Self::category_for_code(number), + )) + } + + pub fn new_with_int32_source_line_info_string_string( + number: i32, + location: SourceLineInfo, + input: String, + message: String, + ) -> Result { + Ok(Self::create( + number, + location, + input, + message, + DiagnosticSeverity::Error, + Self::category_for_code(number), + )) + } + + pub fn new_with_int32_lexer_int32_string_string( + number: i32, + lexer: Lexer, + pos: i32, + input: String, + message: String, + ) -> Result { + let location = lexer.source_line_info(pos)?; + Ok(Self::create( + number, + location, + input, + message, + DiagnosticSeverity::Error, + Self::category_for_code(number), + )) + } + + pub fn new_with_int32_lexer_string( + number: i32, + lexer: Lexer, + message: String, + ) -> Result { + let pos = lexer.yypos(); + Self::new_with_int32_lexer_int32_string_string(number, lexer, pos, String::new(), message) + } + + pub fn new_with_int32_lexer_string_string( + number: i32, + lexer: Lexer, + input: String, + message: String, + ) -> Result { + let pos = lexer.yypos(); + Self::new_with_int32_lexer_int32_string_string(number, lexer, pos, input, message) + } + + pub fn new_with_int32_symbol_string( + number: i32, + symbol: SYMBOL, + message: String, + ) -> Result { + let location = symbol.location()?; + let mut result = Self::create( + number, + location, + symbol.yyname(), + message, + DiagnosticSeverity::Error, + Self::category_for_code(number), + ); + result.sym = Some(Box::new(symbol)); + Ok(result) + } + + pub fn new_with_int32_token_string( + number: i32, + token: TOKEN, + message: String, + ) -> Result { + Ok(Self::create( + number, + token.location()?, + token.yytext(), + message, + DiagnosticSeverity::Error, + Self::category_for_code(number), + )) + } + + pub fn handle(&mut self, handler: ErrorHandler) -> Result<(), Error> { + if handler.throw_exceptions { + return Err(Error::InvalidOperation); + } + self.handled = true; + Ok(()) + } + + fn diagnostic(&self) -> Diagnostic { + Diagnostic { + code: self.n_exception_number, + category: self.category, + severity: self.severity, + message: self.message.clone(), + location: self.sl_info.clone(), + input: self.s_input.clone(), + } + } +} + +impl fmt::Display for CSToolsException { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(formatter, "{}: {}", self.sl_info, self.message) + } +} + +impl std::error::Error for CSToolsException {} + +macro_rules! exception_wrapper { + ($name:ident, $severity:expr) => { + #[derive(Clone, Debug)] + pub struct $name(pub CSToolsException); + + impl $name { + pub fn new_with_int32_string(number: i32, message: String) -> Result { + let mut inner = CSToolsException::new_with_int32_string(number, message)?; + inner.severity = $severity; + Ok(Self(inner)) + } + + pub fn new_with_int32_source_line_info_string_string( + number: i32, + location: SourceLineInfo, + input: String, + message: String, + ) -> Result { + let mut inner = CSToolsException::new_with_int32_source_line_info_string_string( + number, location, input, message, + )?; + inner.severity = $severity; + Ok(Self(inner)) + } + + pub fn new_with_int32_lexer_int32_string_string( + number: i32, + lexer: Lexer, + pos: i32, + input: String, + message: String, + ) -> Result { + let mut inner = CSToolsException::new_with_int32_lexer_int32_string_string( + number, lexer, pos, input, message, + )?; + inner.severity = $severity; + Ok(Self(inner)) + } + + pub fn new_with_int32_lexer_string( + number: i32, + lexer: Lexer, + message: String, + ) -> Result { + let mut inner = + CSToolsException::new_with_int32_lexer_string(number, lexer, message)?; + inner.severity = $severity; + Ok(Self(inner)) + } + + pub fn new_with_int32_lexer_string_string( + number: i32, + lexer: Lexer, + input: String, + message: String, + ) -> Result { + let mut inner = CSToolsException::new_with_int32_lexer_string_string( + number, lexer, input, message, + )?; + inner.severity = $severity; + Ok(Self(inner)) + } + + pub fn new_with_int32_token_string( + number: i32, + token: TOKEN, + message: String, + ) -> Result { + let mut inner = + CSToolsException::new_with_int32_token_string(number, token, message)?; + inner.severity = $severity; + Ok(Self(inner)) + } + + pub fn new_with_int32_symbol_string( + number: i32, + symbol: SYMBOL, + message: String, + ) -> Result { + let mut inner = + CSToolsException::new_with_int32_symbol_string(number, symbol, message)?; + inner.severity = $severity; + Ok(Self(inner)) + } + + pub fn handle(&mut self, _handler: ErrorHandler) -> Result<(), Error> { + Err(Error::InvalidOperation) + } + } + }; +} + +exception_wrapper!(CSToolsFatalException, DiagnosticSeverity::Fatal); +exception_wrapper!(CSToolsStopException, DiagnosticSeverity::Stop); + +/// Collecting error handler. No output is written implicitly. +#[derive(Clone, Debug, Default)] +pub struct ErrorHandler { + pub counter: i32, + pub throw_exceptions: bool, + diagnostics: Vec, +} + +impl ErrorHandler { + pub fn new_with_constructor() -> Result { + Ok(Self::default()) + } + + pub fn new_with_boolean(throw_exceptions: bool) -> Result { + Ok(Self { + throw_exceptions, + ..Self::default() + }) + } + + pub fn error(&mut self, mut error: CSToolsException) -> Result<(), Error> { + self.counter = self.counter.checked_add(1).ok_or(Error::IndexOutOfRange)?; + if self.throw_exceptions { + return Err(Error::InvalidOperation); + } + self.report(error.clone())?; + error.handled = true; + Ok(()) + } + + pub fn report(&mut self, error: CSToolsException) -> Result<(), Error> { + self.diagnostics.push(error.diagnostic()); + Ok(()) + } + + #[must_use] + pub fn diagnostics(&self) -> &[Diagnostic] { + &self.diagnostics + } + + pub fn take_diagnostics(&mut self) -> Vec { + std::mem::take(&mut self.diagnostics) + } + + fn push(&mut self, diagnostic: Diagnostic) -> Result<(), Error> { + self.counter = self.counter.checked_add(1).ok_or(Error::IndexOutOfRange)?; + if self.throw_exceptions { + return Err(Error::InvalidOperation); + } + self.diagnostics.push(diagnostic); + Ok(()) + } +} + +/// Comment-filtering source reader corresponding to the C# `CsReader`. +#[derive(Clone, Debug)] +pub struct CsReader { + pub fname: String, + pub lm: LineManager, + units: Vec, + cursor: usize, +} + +impl CsReader { + pub fn new_with_string(data: String) -> Result { + Self::from_source(data, String::new()) + } + + pub fn new_with_string_encoding( + file_name: String, + _encoding: TextEncoding, + ) -> Result { + let bytes = std::fs::read(&file_name).map_err(|_| Error::Argument)?; + let encoding = if bytes.starts_with(&[0xFF, 0xFE]) { + InputEncoding::Utf16Le + } else if bytes.starts_with(&[0xFE, 0xFF]) { + InputEncoding::Utf16Be + } else { + InputEncoding::Utf8 + }; + Self::from_source(encoding.decode(&bytes)?, file_name) + } + + pub fn new_with_cs_reader_encoding( + reader: CsReader, + _encoding: TextEncoding, + ) -> Result { + Ok(reader) + } + + /// Creates a reader from encoded bytes without a platform-specific API. + pub fn from_bytes( + bytes: &[u8], + encoding: InputEncoding, + file_name: impl Into, + ) -> Result { + Self::from_source(encoding.decode(bytes)?, file_name.into()) + } + + fn from_source(source: String, file_name: String) -> Result { + let input = source.encode_utf16().collect::>(); + if input.len() > MAX_SOURCE_UNITS { + return Err(Error::Argument); + } + let (units, lm, directive_name) = filter_source(&input)?; + Ok(Self { + fname: directive_name.unwrap_or(file_name), + lm, + units, + cursor: 0, + }) + } + + pub fn eof(&self) -> Result { + Ok(self.cursor >= self.units.len()) + } + + pub fn read_with_method(&mut self) -> Result { + let Some(unit) = self.units.get(self.cursor).copied() else { + return Ok(-1); + }; + self.cursor += 1; + Ok(i32::from(unit)) + } + + pub fn read_with_char_array_int32_int32( + &mut self, + mut destination: Vec, + offset: i32, + count: i32, + ) -> Result { + let offset = index(offset)?; + let count = index(count)?; + let end = offset.checked_add(count).ok_or(Error::IndexOutOfRange)?; + if end > destination.len() { + return Err(Error::IndexOutOfRange); + } + let copied = self.read_into(&mut destination[offset..end]); + i32_len(copied) + } + + pub fn read_into(&mut self, destination: &mut [Utf16CodeUnit]) -> usize { + let count = destination + .len() + .min(self.units.len().saturating_sub(self.cursor)); + for (slot, unit) in destination + .iter_mut() + .zip(&self.units[self.cursor..self.cursor + count]) + { + *slot = Utf16CodeUnit(*unit); + } + self.cursor += count; + count + } + + pub fn read_line(&mut self) -> Result { + if self.cursor >= self.units.len() { + return Ok(String::new()); + } + let start = self.cursor; + while self.cursor < self.units.len() && self.units[self.cursor] != b'\n'.into() { + self.cursor += 1; + } + let line = units_to_string(&self.units[start..self.cursor]); + if self.cursor < self.units.len() { + self.cursor += 1; + } + Ok(line) + } + + fn into_parts(self) -> (String, Vec, LineManager) { + (self.fname, self.units, self.lm) + } +} + +fn filter_source(input: &[u16]) -> Result<(Vec, LineManager, Option), Error> { + #[derive(Clone, Copy)] + enum State { + Normal, + Slash, + LineComment { removed: usize }, + BlockComment { removed: usize }, + BlockStar { removed: usize }, + } + + let mut output = Vec::with_capacity(input.len()); + let mut lm = LineManager::default(); + let mut state = State::Normal; + let mut at_line_start = true; + let mut directive_name = None; + let mut i = 0; + while i < input.len() { + let unit = input[i]; + let normalized = if unit == b'\r'.into() { + if input.get(i + 1) == Some(&u16::from(b'\n')) { + i += 1; + } + u16::from(b'\n') + } else { + unit + }; + + if matches!(state, State::Normal) && at_line_start && normalized == u16::from(b'#') { + let start = i; + let mut end = i; + while end < input.len() + && input[end] != u16::from(b'\n') + && input[end] != u16::from(b'\r') + { + end += 1; + } + let directive = units_to_string(&input[start..end]); + if let Some((line, name)) = parse_line_directive(&directive) { + if let Some(name) = name { + directive_name = Some(name); + } + let removed = end - start; + lm.comment(i32_len(output.len())?, i32_len(removed)?)?; + lm.set_logical_line(line)?; + i = end; + continue; + } + } + + match state { + State::Normal => { + if normalized == u16::from(b'/') { + state = State::Slash; + } else { + output.push(normalized); + if normalized == u16::from(b'\n') { + lm.newline(i32_len(output.len())?)?; + at_line_start = true; + } else { + at_line_start = false; + } + } + } + State::Slash => { + if normalized == u16::from(b'/') { + state = State::LineComment { removed: 2 }; + } else if normalized == u16::from(b'*') { + state = State::BlockComment { removed: 2 }; + } else { + output.push(u16::from(b'/')); + output.push(normalized); + at_line_start = false; + state = State::Normal; + } + } + State::LineComment { removed } => { + if normalized == u16::from(b'\n') { + lm.comment(i32_len(output.len())?, i32_len(removed)?)?; + output.push(normalized); + lm.newline(i32_len(output.len())?)?; + at_line_start = true; + state = State::Normal; + } else { + state = State::LineComment { + removed: removed + 1, + }; + } + } + State::BlockComment { removed } => { + if normalized == u16::from(b'*') { + state = State::BlockStar { + removed: removed + 1, + }; + } else if normalized == u16::from(b'\n') { + lm.comment(i32_len(output.len())?, i32_len(removed)?)?; + output.push(normalized); + lm.newline(i32_len(output.len())?)?; + at_line_start = true; + state = State::BlockComment { removed: 0 }; + } else { + state = State::BlockComment { + removed: removed + 1, + }; + } + } + State::BlockStar { removed } => { + if normalized == u16::from(b'/') { + lm.comment(i32_len(output.len())?, i32_len(removed + 1)?)?; + state = State::Normal; + } else if normalized == u16::from(b'*') { + state = State::BlockStar { + removed: removed + 1, + }; + } else if normalized == u16::from(b'\n') { + lm.comment(i32_len(output.len())?, i32_len(removed)?)?; + output.push(normalized); + lm.newline(i32_len(output.len())?)?; + at_line_start = true; + state = State::BlockComment { removed: 0 }; + } else { + state = State::BlockComment { + removed: removed + 1, + }; + } + } + } + i += 1; + } + + match state { + State::Normal => {} + State::Slash => output.push(u16::from(b'/')), + State::LineComment { removed } => { + lm.comment(i32_len(output.len())?, i32_len(removed)?)?; + } + State::BlockComment { .. } | State::BlockStar { .. } => { + return Err(Error::Parse { + position: input.len(), + context: "unterminated block comment", + }); + } + } + lm.end = i32_len(output.len())?; + Ok((output, lm, directive_name)) +} + +fn parse_line_directive(value: &str) -> Option<(i32, Option)> { + let value = value.strip_prefix('#')?.trim_start(); + let value = value.strip_prefix("line")?.trim_start(); + let (number, rest) = value.split_once(char::is_whitespace).unwrap_or((value, "")); + let line = number.parse::().ok()?; + if line < 1 { + return None; + } + let rest = rest.trim(); + let name = rest + .strip_prefix('"') + .and_then(|quoted| quoted.strip_suffix('"')) + .map(ToOwned::to_owned); + Some((line, name)) +} + +/// A character predicate on one UTF-16 code unit. +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum CharacterMatcher { + Any, + Category(DotNetUnicodeCategory), + Categories(Vec), + UnicodeClass(UnicodeClass), + Eof, + Exact(u16), + Range { first: u16, last: u16 }, + Set(BTreeSet), +} + +impl CharacterMatcher { + fn matches(&self, value: Option) -> bool { + match (self, value) { + (Self::Eof, None) | (Self::Any, Some(_)) => true, + (Self::Category(category), Some(unit)) => { + DotNetUnicodeCategory::of_unit(unit) == *category + } + (Self::Categories(categories), Some(unit)) => { + categories.contains(&DotNetUnicodeCategory::of_unit(unit)) + } + (Self::UnicodeClass(class), Some(unit)) => class.matches(unit), + (Self::Exact(expected), Some(unit)) => unit == *expected, + (Self::Range { first, last }, Some(unit)) => (*first..=*last).contains(&unit), + (Self::Set(values), Some(unit)) => values.contains(&unit), + _ => false, + } + } + + fn stable_text(&self) -> String { + match self { + Self::Any => "any".to_owned(), + Self::Category(category) => format!("category:{}", category.name()), + Self::Categories(categories) => format!( + "categories:{}", + categories + .iter() + .map(|category| category.name()) + .collect::>() + .join(",") + ), + Self::UnicodeClass(class) => format!("class:{class:?}"), + Self::Eof => "eof".to_owned(), + Self::Exact(unit) => format!("exact:{unit:04X}"), + Self::Range { first, last } => format!("range:{first:04X}-{last:04X}"), + Self::Set(values) => { + let values = values + .iter() + .map(|value| format!("{value:04X}")) + .collect::>(); + format!("set:{}", values.join(",")) + } + } + } +} + +/// Named character predicates accepted by the original lexer specification. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum UnicodeClass { + Symbol, + Punctuation, + Separator, + WhiteSpace, + Number, + Digit, + Letter, + Lower, + Upper, +} + +impl UnicodeClass { + fn matches(self, unit: u16) -> bool { + let category = DotNetUnicodeCategory::of_unit(unit); + match self { + Self::Symbol => matches!( + category, + DotNetUnicodeCategory::MathSymbol + | DotNetUnicodeCategory::CurrencySymbol + | DotNetUnicodeCategory::ModifierSymbol + | DotNetUnicodeCategory::OtherSymbol + ), + Self::Punctuation => matches!( + category, + DotNetUnicodeCategory::ConnectorPunctuation + | DotNetUnicodeCategory::DashPunctuation + | DotNetUnicodeCategory::OpenPunctuation + | DotNetUnicodeCategory::ClosePunctuation + | DotNetUnicodeCategory::InitialQuotePunctuation + | DotNetUnicodeCategory::FinalQuotePunctuation + | DotNetUnicodeCategory::OtherPunctuation + ), + Self::Separator => matches!( + category, + DotNetUnicodeCategory::SpaceSeparator + | DotNetUnicodeCategory::LineSeparator + | DotNetUnicodeCategory::ParagraphSeparator + ), + Self::WhiteSpace => { + matches!(unit, 0x0009..=0x000D | 0x0085) || Self::Separator.matches(unit) + } + Self::Number => matches!( + category, + DotNetUnicodeCategory::DecimalDigitNumber + | DotNetUnicodeCategory::LetterNumber + | DotNetUnicodeCategory::OtherNumber + ), + Self::Digit => category == DotNetUnicodeCategory::DecimalDigitNumber, + Self::Letter => matches!( + category, + DotNetUnicodeCategory::UppercaseLetter + | DotNetUnicodeCategory::LowercaseLetter + | DotNetUnicodeCategory::TitlecaseLetter + | DotNetUnicodeCategory::ModifierLetter + | DotNetUnicodeCategory::OtherLetter + ), + Self::Lower => category == DotNetUnicodeCategory::LowercaseLetter, + Self::Upper => category == DotNetUnicodeCategory::UppercaseLetter, + } + } +} + +/// Lexer action attached to an accepting DFA state. +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum LexerAction { + Emit, + EmitAndBegin(String), + Skip, + SkipAndBegin(String), +} + +/// Token metadata stored in a deterministic lexer table. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct TokenDefinition { + pub name: String, + pub number: i32, +} + +impl TokenDefinition { + pub fn new(name: impl Into, number: i32) -> Result { + let name = name.into(); + if name.is_empty() || number < 0 { + return Err(Error::Argument); + } + Ok(Self { name, number }) + } +} + +/// Accepting metadata for a DFA state. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct DfaAccept { + pub token: TokenDefinition, + pub action: LexerAction, + pub action_number: i32, + pub reserved_words: Option, +} + +/// One deterministic automaton state. +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub struct DfaState { + pub transitions: Vec<(CharacterMatcher, usize)>, + pub accept: Option, +} + +impl DfaState { + #[must_use] + pub fn transition(mut self, matcher: CharacterMatcher, target: usize) -> Self { + self.transitions.push((matcher, target)); + self + } + + #[must_use] + pub fn accepting(mut self, accept: DfaAccept) -> Self { + self.accept = Some(accept); + self + } +} + +/// Compatibility linked action. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct DfaAction { + pub a_act: i32, + pub a_next: Option>, +} + +impl DfaAction { + pub fn new(action: i32, next: DfaAction) -> Result { + Ok(Self { + a_act: action, + a_next: Some(Box::new(next)), + }) + } + + pub fn serialise(o: Object, _s: Serialiser) -> Result { + let _ = o; + Err(Error::InvalidOperation) + } +} + +/// Runtime deterministic finite automaton. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct Dfa { + pub m_actions: Vec, + pub m_map: Hashtable, + pub m_reswds: i32, + pub m_tok_class: String, + states: Vec, + start: usize, +} + +impl Default for Dfa { + fn default() -> Self { + Self { + m_actions: Vec::new(), + m_map: Hashtable::default(), + m_reswds: -1, + m_tok_class: String::new(), + states: vec![DfaState::default()], + start: 0, + } + } +} + +impl Dfa { + pub fn from_states(states: Vec, start: usize) -> Result { + if states.is_empty() || start >= states.len() { + return Err(Error::Argument); + } + if states + .iter() + .flat_map(|state| &state.transitions) + .any(|(_, target)| *target >= states.len()) + { + return Err(Error::Argument); + } + Ok(Self { + states, + start, + ..Self::default() + }) + } + + pub fn new_with_nfa(_nfa: Nfa) -> Result { + // The NFA-to-DFA generator is issue #83; the runtime never manufactures + // fake transitions from an opaque generated NFA. + Err(Error::InvalidOperation) + } + + pub fn new_with_tokens_gen(_tokens: TokensGen) -> Result { + Err(Error::InvalidOperation) + } + + fn longest_match(&self, units: &[u16], offset: usize) -> Option<(usize, DfaAccept)> { + let mut state_index = self.start; + let mut cursor = offset; + let mut accepted = self.states[state_index] + .accept + .clone() + .map(|accept| (0, accept)); + loop { + let value = units.get(cursor).copied(); + let next = self.states[state_index] + .transitions + .iter() + .find(|(matcher, _)| matcher.matches(value)) + .map(|(_, target)| *target); + let Some(next) = next else { + break; + }; + // EOF transitions do not consume a code unit. + if value.is_some() { + cursor += 1; + } + state_index = next; + if let Some(accept) = self.states[state_index].accept.clone() { + accepted = Some((cursor - offset, accept)); + } + if value.is_none() { + break; + } + } + accepted + } + + pub fn match_(&self, value: String, offset: i32, action: &mut i32) -> Result { + let units = value.encode_utf16().collect::>(); + let offset = index(offset)?; + if offset > units.len() { + return Err(Error::IndexOutOfRange); + } + let Some((length, accept)) = self.longest_match(&units, offset) else { + return Ok(-1); + }; + *action = accept.action_number; + i32_len(length) + } + + pub fn print(&self) -> Result<(), Error> { + println!("{}", self.stable_description()); + Ok(()) + } + + #[must_use] + pub fn stable_description(&self) -> String { + let mut lines = Vec::new(); + for (state_index, state) in self.states.iter().enumerate() { + if let Some(accept) = &state.accept { + lines.push(format!( + "state {state_index} accept {} {} {}", + accept.token.number, accept.token.name, accept.action_number + )); + } + for (matcher, target) in &state.transitions { + lines.push(format!( + "state {state_index} {} -> {target}", + matcher.stable_text() + )); + } + } + lines.join("\n") + } + + pub fn serialise(o: Object, _s: Serialiser) -> Result { + let _ = o; + Err(Error::InvalidOperation) + } + + pub fn set_tokens(_tokens: YyLexer, _starts: Hashtable) -> Result<(), Error> { + Err(Error::InvalidOperation) + } +} + +/// Reserved-word remapping for a token class. +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub struct ResWds { + pub m_wds: BTreeMap, + pub m_upper: bool, +} + +impl ResWds { + pub fn new_with_constructor() -> Result { + Ok(Self::default()) + } + + pub fn new_with_tokens_gen_string( + tokens: TokensGen, + specification: String, + ) -> Result { + let mut specification = specification.trim(); + let uppercase = specification.starts_with('U'); + if uppercase { + specification = specification[1..].trim_start(); + } + let Some(body) = specification + .strip_prefix('{') + .and_then(|value| value.strip_suffix('}')) + else { + return Err(Error::Parse { + position: 0, + context: "bad ResWds element", + }); + }; + let mut pairs = Vec::new(); + for item in body.split(',') { + let mut fields = item.split_whitespace(); + let Some(word) = fields.next() else { + continue; + }; + let name = fields.next().unwrap_or(word); + if fields.next().is_some() { + return Err(Error::Parse { + position: specification.find(item).unwrap_or(0), + context: "reserved word has too many fields", + }); + } + let key = Object::String(name.to_owned()); + let Some(value) = tokens.m_tokens.tokens.0.get(&key) else { + // Creating a missing token class mutates generator state and is + // deliberately owned by issue #83. Never invent its number. + return Err(Error::InvalidOperation); + }; + let (token_name, token_number) = match value { + Object::Integer(number) => (name.to_owned(), *number), + Object::Map(metadata) => { + let token_name = match metadata.get("name") { + Some(Object::String(value)) => value.clone(), + None => name.to_owned(), + _ => return Err(Error::InvalidOperation), + }; + let Some(Object::Integer(number)) = metadata.get("number") else { + return Err(Error::InvalidOperation); + }; + (token_name, *number) + } + _ => return Err(Error::InvalidOperation), + }; + pairs.push(( + word.to_owned(), + TokenDefinition::new(token_name, token_number)?, + )); + } + Self::from_pairs(pairs, uppercase) + } + + pub fn from_pairs(pairs: I, uppercase: bool) -> Result + where + I: IntoIterator, + K: Into, + { + let mut words = BTreeMap::new(); + for (word, token) in pairs { + let word = word.into(); + if word.is_empty() { + return Err(Error::Argument); + } + let key = if uppercase { word.to_uppercase() } else { word }; + words.insert(key, token); + } + Ok(Self { + m_wds: words, + m_upper: uppercase, + }) + } + + fn remap(&self, text: &str) -> Option { + let key = if self.m_upper { + text.to_uppercase() + } else { + text.to_owned() + }; + self.m_wds.get(&key).cloned() + } + + pub fn check(&self, _lexer: Lexer, token: &mut TOKEN) -> Result<(), Error> { + if let Some(definition) = self.remap(&token.text) { + token.name = definition.name; + token.number = definition.number; + } + Ok(()) + } + + pub fn serialise(o: Object, _s: Serialiser) -> Result { + let _ = o; + Err(Error::InvalidOperation) + } +} + +/// Lexer table/configuration corresponding to `YyLexer`. +#[derive(Clone, Debug)] +pub struct YyLexer { + pub cats: Hashtable, + pub erh: ErrorHandler, + pub m_encoding: TextEncoding, + pub m_gencat: UnicodeCategory, + pub reswds: Hashtable, + pub starts: Hashtable, + pub tokens: Hashtable, + pub toupper: bool, + pub types: Hashtable, + pub using_eof: bool, + encoding: InputEncoding, + categories: BTreeMap, + start_dfas: BTreeMap, + reserved: BTreeMap, +} + +impl YyLexer { + pub fn new(error_handler: ErrorHandler) -> Result { + let default_category = DotNetUnicodeCategory::OtherPunctuation; + let mut categories = BTreeMap::new(); + categories.insert(default_category, Charset::for_category(default_category)); + Ok(Self { + cats: Hashtable::default(), + erh: error_handler, + m_encoding: TextEncoding, + m_gencat: UnicodeCategory(default_category as i32), + reswds: Hashtable::default(), + starts: Hashtable::default(), + tokens: Hashtable::default(), + toupper: false, + types: Hashtable::default(), + using_eof: false, + encoding: InputEncoding::Utf8, + categories, + start_dfas: BTreeMap::new(), + reserved: BTreeMap::new(), + }) + } + + pub fn set_start_dfa(&mut self, state: impl Into, dfa: Dfa) -> Result<(), Error> { + let state = state.into(); + if state.is_empty() { + return Err(Error::Argument); + } + self.start_dfas.insert(state, dfa); + Ok(()) + } + + pub fn set_reserved_words( + &mut self, + name: impl Into, + words: ResWds, + ) -> Result<(), Error> { + let name = name.into(); + if name.is_empty() { + return Err(Error::Argument); + } + self.reserved.insert(name, words); + Ok(()) + } + + pub fn set_input_encoding(&mut self, value: String) { + if let Ok(encoding) = InputEncoding::parse(&value) { + self.toupper = encoding == InputEncoding::AsciiUpper; + self.encoding = encoding; + } else if let Ok(error) = CSToolsException::new_with_int32_string( + 43, + format!("Warning: Encoding {value} unknown: ignored"), + ) { + let _ = self.erh.error(error); + } + } + + #[must_use] + pub const fn input_encoding(&self) -> InputEncoding { + self.encoding + } + + pub fn using_cat(&mut self, category: UnicodeCategory) -> Result { + let category = DotNetUnicodeCategory::from_compat(category)?; + Ok(self + .categories + .entry(category) + .or_insert_with(|| Charset::for_category(category)) + .clone()) + } + + pub fn get_test(&mut self, name: &str) -> Result { + if let Some(category) = (0..=29) + .filter_map(|value| DotNetUnicodeCategory::try_from(value).ok()) + .find(|category| category.name() == name) + { + let _ = self.using_cat(UnicodeCategory(category as i32))?; + return Ok(CharacterMatcher::Category(category)); + } + let (class, categories): (UnicodeClass, &[DotNetUnicodeCategory]) = match name { + "Symbol" => ( + UnicodeClass::Symbol, + &[ + DotNetUnicodeCategory::MathSymbol, + DotNetUnicodeCategory::CurrencySymbol, + DotNetUnicodeCategory::ModifierSymbol, + DotNetUnicodeCategory::OtherSymbol, + ], + ), + "Punctuation" => ( + UnicodeClass::Punctuation, + &[ + DotNetUnicodeCategory::ConnectorPunctuation, + DotNetUnicodeCategory::DashPunctuation, + DotNetUnicodeCategory::OpenPunctuation, + DotNetUnicodeCategory::ClosePunctuation, + DotNetUnicodeCategory::InitialQuotePunctuation, + DotNetUnicodeCategory::FinalQuotePunctuation, + DotNetUnicodeCategory::OtherPunctuation, + ], + ), + "Separator" => ( + UnicodeClass::Separator, + &[ + DotNetUnicodeCategory::SpaceSeparator, + DotNetUnicodeCategory::LineSeparator, + DotNetUnicodeCategory::ParagraphSeparator, + ], + ), + "Number" => ( + UnicodeClass::Number, + &[ + DotNetUnicodeCategory::DecimalDigitNumber, + DotNetUnicodeCategory::LetterNumber, + DotNetUnicodeCategory::OtherNumber, + ], + ), + "Letter" => ( + UnicodeClass::Letter, + &[ + DotNetUnicodeCategory::UppercaseLetter, + DotNetUnicodeCategory::LowercaseLetter, + DotNetUnicodeCategory::TitlecaseLetter, + DotNetUnicodeCategory::ModifierLetter, + DotNetUnicodeCategory::OtherLetter, + ], + ), + "Digit" => ( + UnicodeClass::Digit, + &[DotNetUnicodeCategory::DecimalDigitNumber], + ), + "Lower" => ( + UnicodeClass::Lower, + &[DotNetUnicodeCategory::LowercaseLetter], + ), + "Upper" => ( + UnicodeClass::Upper, + &[DotNetUnicodeCategory::UppercaseLetter], + ), + "EOF" => { + self.using_eof = true; + return Ok(CharacterMatcher::Eof); + } + "WhiteSpace" => ( + UnicodeClass::WhiteSpace, + &[ + DotNetUnicodeCategory::Control, + DotNetUnicodeCategory::SpaceSeparator, + DotNetUnicodeCategory::LineSeparator, + DotNetUnicodeCategory::ParagraphSeparator, + ], + ), + _ => { + if let Ok(error) = + CSToolsException::new_with_int32_string(24, format!("No such Charset {name}")) + { + self.erh.error(error)?; + } + return Err(Error::Argument); + } + }; + for category in categories { + let _ = self.using_cat(UnicodeCategory(*category as i32))?; + } + Ok(CharacterMatcher::UnicodeClass(class)) + } + + #[must_use] + pub fn category_group(name: &str) -> Option<&'static [DotNetUnicodeCategory]> { + match name { + "WhiteSpace" => Some(&[ + DotNetUnicodeCategory::Control, + DotNetUnicodeCategory::SpaceSeparator, + DotNetUnicodeCategory::LineSeparator, + DotNetUnicodeCategory::ParagraphSeparator, + ]), + "Letter" => Some(&[ + DotNetUnicodeCategory::UppercaseLetter, + DotNetUnicodeCategory::LowercaseLetter, + DotNetUnicodeCategory::TitlecaseLetter, + DotNetUnicodeCategory::ModifierLetter, + DotNetUnicodeCategory::OtherLetter, + ]), + "Number" => Some(&[ + DotNetUnicodeCategory::DecimalDigitNumber, + DotNetUnicodeCategory::LetterNumber, + DotNetUnicodeCategory::OtherNumber, + ]), + _ => None, + } + } + + pub fn get_dfa(&self) -> Result<(), Error> { + if self.start_dfas.is_empty() { + return Err(Error::InvalidOperation); + } + Ok(()) + } + + pub fn emit_dfa(&self, mut output: Box) -> Result<(), Error> { + writeln!(output, "LSLLEXER 1").map_err(|_| Error::InvalidOperation)?; + writeln!(output, "encoding {:?}", self.encoding).map_err(|_| Error::InvalidOperation)?; + writeln!(output, "toupper {}", self.toupper).map_err(|_| Error::InvalidOperation)?; + writeln!(output, "eof {}", self.using_eof).map_err(|_| Error::InvalidOperation)?; + for (name, dfa) in &self.start_dfas { + writeln!(output, "start {name}").map_err(|_| Error::InvalidOperation)?; + writeln!(output, "{}", dfa.stable_description()) + .map_err(|_| Error::InvalidOperation)?; + } + for (name, words) in &self.reserved { + for (word, token) in &words.m_wds { + writeln!( + output, + "reserved {name} {word} {} {}", + token.number, token.name + ) + .map_err(|_| Error::InvalidOperation)?; + } + } + Ok(()) + } + + pub fn get_enumerator(&self) -> Result { + Ok(IEnumerator) + } + + #[allow(clippy::ptr_arg)] + pub fn old_action( + &self, + lexer: Lexer, + yytext: &mut String, + action: i32, + reject: &mut bool, + ) -> Result { + *reject = false; + let end = lexer.m_pch; + let length = i32_len(utf16_len(yytext))?; + let mut token = TOKEN::new_with_lexer_string(lexer, yytext.clone())?; + token.number = action; + token.pos = end.checked_sub(length).ok_or(Error::IndexOutOfRange)?; + token.end = end; + Ok(token) + } + + fn dfa(&self, state: &str) -> Option<&Dfa> { + self.start_dfas.get(state) + } + + fn reserved_word(&self, name: &str, text: &str) -> Option { + self.reserved.get(name).and_then(|words| words.remap(text)) + } +} + +/// Base parser symbol with source location and semantic value. +#[allow(clippy::upper_case_acronyms)] +pub struct SYMBOL { + pub kids: ObjectList, + pub m_dollar: Object, + pub pos_with_field: i32, + pub yylx: Option>, + pub yyps: Option>, +} + +impl fmt::Debug for SYMBOL { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("SYMBOL") + .field("kids", &self.kids) + .field("m_dollar", &self.m_dollar) + .field("pos", &self.pos_with_field) + .field("has_lexer", &self.yylx.is_some()) + .field("has_parser", &self.yyps.is_some()) + .finish() + } +} + +impl SYMBOL { + pub fn new_with_lexer(lexer: Lexer) -> Result { + Ok(Self { + kids: ObjectList::new()?, + m_dollar: Object::Undefined, + pos_with_field: lexer.yypos(), + yylx: Some(Box::new(lexer)), + yyps: None, + }) + } + + pub fn new_with_parser(parser: Parser) -> Result { + Ok(Self { + kids: ObjectList::new()?, + m_dollar: Object::Undefined, + pos_with_field: 0, + yylx: None, + yyps: Some(Box::new(parser)), + }) + } + + fn location(&self) -> Result { + self.yylx + .as_deref() + .ok_or(Error::InvalidOperation)? + .source_line_info(self.pos_with_field) + } + + pub fn concrete_syntax_tree(&self) -> Result<(), Error> { + println!("{}", self.concrete_syntax_tree_text()); + Ok(()) + } + + #[must_use] + pub fn concrete_syntax_tree_text(&self) -> String { + let mut output = self.yyname(); + for child in self.kids.iter() { + output.push('\n'); + let _ = std::fmt::write(&mut output, format_args!(" {child:?}")); + } + output + } + + pub fn is_action(&self) -> Result { + Ok(false) + } + pub fn is_c_symbol(&self) -> Result { + Ok(false) + } + pub fn is_terminal(&self) -> Result { + Ok(false) + } + pub fn matches(&self, _value: String) -> Result { + Ok(false) + } + + pub fn pass_( + &self, + symbols: YyParser, + state: i32, + entry: &mut ParserEntry, + ) -> Result { + lookup_parser_entry(&symbols, self.yynum(), state, entry) + } + + pub fn print(&self) -> Result<(), Error> { + println!("{}", self.to_string()); + Ok(()) + } + #[must_use] + pub fn to_string(&self) -> String { + self.yyname() + } + + pub fn from(symbol: SYMBOL) -> i32 { + match symbol.m_dollar { + Object::Integer(value) => value, + _ => panic!("SYMBOL semantic value is not a System.Int32"), + } + } + + #[must_use] + pub fn line(&self) -> i32 { + self.location().map_or(0, |value| value.line_number) + } + #[must_use] + pub fn pos_with_property(&self) -> String { + self.location() + .map_or_else(|_| String::new(), |value| value.to_string()) + } + #[must_use] + pub fn position(&self) -> i32 { + self.location().map_or(0, |value| value.raw_char_position) + } + + pub fn yyact(&self) -> Result<&YyParser, Error> { + self.yyps + .as_deref() + .map(|parser| &parser.m_symbols) + .ok_or(Error::InvalidOperation) + } + + #[must_use] + pub fn yylval(&self) -> Object { + self.m_dollar.clone() + } + pub fn set_yylval(&mut self, value: Object) { + self.m_dollar = value; + } + #[must_use] + pub fn yyname(&self) -> String { + "SYMBOL".to_owned() + } + #[must_use] + pub const fn yynum(&self) -> i32 { + 0 + } +} + +impl fmt::Display for SYMBOL { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(&self.yyname()) + } +} + +/// Terminal token emitted by [`Lexer`]. +#[allow(clippy::upper_case_acronyms)] +#[derive(Clone, Debug)] +pub struct TOKEN { + text: String, + name: String, + number: i32, + pub pos: i32, + pub end: i32, + value: Object, + source: Option>, +} + +impl TOKEN { + pub fn new_with_lexer(lexer: Lexer) -> Result { + let text = lexer.yytext.clone(); + Self::new_with_lexer_string(lexer, text) + } + + pub fn new_with_lexer_string(lexer: Lexer, text: String) -> Result { + let pos = lexer.yypos(); + let end = pos + .checked_add(i32_len(utf16_len(&text))?) + .ok_or(Error::IndexOutOfRange)?; + Ok(Self { + text, + name: "TOKEN".to_owned(), + number: 1, + pos, + end, + value: Object::Undefined, + source: Some(Box::new(lexer)), + }) + } + + pub fn new_with_parser(_parser: Parser) -> Result { + Ok(Self { + text: String::new(), + name: "TOKEN".to_owned(), + number: 1, + pos: 0, + end: 0, + value: Object::Undefined, + source: None, + }) + } + + fn emitted( + lexer: &Lexer, + definition: TokenDefinition, + text: String, + pos: usize, + end: usize, + ) -> Result { + Ok(Self { + text, + name: definition.name, + number: definition.number, + pos: i32_len(pos)?, + end: i32_len(end)?, + value: Object::Undefined, + source: Some(Box::new(lexer.clone())), + }) + } + + fn eof(lexer: &Lexer) -> Result { + let pos = lexer.units.len(); + Self::emitted( + lexer, + TokenDefinition { + name: "EOF".to_owned(), + number: 2, + }, + "EOF".to_owned(), + pos, + pos, + ) + } + + fn location(&self) -> Result { + self.source + .as_deref() + .ok_or(Error::InvalidOperation)? + .source_line_info(self.pos) + } + + pub fn is_terminal(&self) -> Result { + Ok(true) + } + pub fn matches(&self, value: String) -> Result { + Ok(self.text == value) + } + + pub fn pass_( + &self, + symbols: YyParser, + state: i32, + entry: &mut ParserEntry, + ) -> Result { + let number = match symbols.literals.0.get(&Object::String(self.text.clone())) { + Some(Object::Integer(number)) => *number, + _ => self.number, + }; + lookup_parser_entry(&symbols, number, state, entry) + } + + pub fn print(&self) -> Result<(), Error> { + println!("{}", self.to_string()); + Ok(()) + } + #[must_use] + pub fn to_string(&self) -> String { + format!("{}<{}>", self.name, self.text) + } + #[must_use] + pub fn yyname(&self) -> String { + self.name.clone() + } + #[must_use] + pub const fn yynum(&self) -> i32 { + self.number + } + #[must_use] + pub fn yytext(&self) -> String { + self.text.clone() + } + pub fn set_yytext(&mut self, value: String) { + self.text = value; + } + #[must_use] + pub fn yylval(&self) -> Object { + self.value.clone() + } + pub fn set_yylval(&mut self, value: Object) { + self.value = value; + } +} + +impl fmt::Display for TOKEN { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(&self.to_string()) + } +} + +fn lookup_parser_entry( + symbols: &YyParser, + number: i32, + state: i32, + entry: &mut ParserEntry, +) -> Result { + let Some(info) = symbols.symbol_info.0.get(&Object::Integer(number)) else { + return Err(Error::Parse { + position: 0, + context: "parser does not recognize token or symbol", + }); + }; + let Object::Map(states) = info else { + return Err(Error::Parse { + position: 0, + context: "parser symbol information is malformed", + }); + }; + let Some(value) = states.get(&state.to_string()) else { + return Ok(false); + }; + entry.m_priority = match value { + Object::Integer(priority) => *priority, + Object::Map(metadata) => match metadata.get("priority") { + Some(Object::Integer(priority)) => *priority, + _ => 0, + }, + _ => { + return Err(Error::Parse { + position: 0, + context: "parser entry is malformed", + }); + } + }; + Ok(true) +} + +/// End-of-input terminal. +#[allow(clippy::upper_case_acronyms)] +#[derive(Clone, Debug)] +pub struct EOF(pub TOKEN); + +impl EOF { + pub fn new_with_lexer(lexer: Lexer) -> Result { + Ok(Self(TOKEN::eof(&lexer)?)) + } + + pub fn new_with_symbols_gen(_symbols: SymbolsGen) -> Result { + Ok(Self(TOKEN { + text: "EOF".to_owned(), + name: "EOF".to_owned(), + number: 2, + pos: 0, + end: 0, + value: Object::Undefined, + source: None, + })) + } + + pub fn serialise(o: Object, _s: Serialiser) -> Result { + let _ = o; + Err(Error::InvalidOperation) + } + #[must_use] + pub fn yyname(&self) -> String { + "EOF".to_owned() + } + #[must_use] + pub const fn yynum(&self) -> i32 { + 2 + } +} + +/// A proxy token used by generated grammars for nullable productions. +#[derive(Clone, Debug)] +pub struct Null { + proxy: String, +} + +impl Null { + pub fn new_with_lexer_string(_lexer: Lexer, proxy: String) -> Result { + if proxy.is_empty() { + return Err(Error::Argument); + } + Ok(Self { proxy }) + } + + pub fn new_with_parser_string(_parser: Parser, proxy: String) -> Result { + if proxy.is_empty() { + return Err(Error::Argument); + } + Ok(Self { proxy }) + } + + #[must_use] + pub fn yyname(&self) -> String { + self.proxy.clone() + } +} + +/// Rust replacement for the C# cons-list, backed by contiguous storage. +#[derive(Debug, Default)] +pub struct ObjectList { + values: VecDeque, + count_override: Option, +} + +impl ObjectList { + pub fn new() -> Result { + Ok(Self::default()) + } + pub fn add(&mut self, value: Object) -> Result<(), Error> { + self.values.push_back(value); + Ok(()) + } + pub fn push(&mut self, value: Object) -> Result<(), Error> { + self.values.push_front(value); + Ok(()) + } + pub fn pop(&mut self) -> Result { + self.values.pop_front().ok_or(Error::InvalidOperation) + } + pub fn top(&self) -> Object { + self.values.front().cloned().unwrap_or(Object::Undefined) + } + pub fn get_enumerator(&self) -> Result { + Ok(IEnumerator) + } + #[must_use] + pub fn count(&self) -> i32 { + self.count_override + .unwrap_or_else(|| i32::try_from(self.values.len()).unwrap_or(i32::MAX)) + } + pub fn set_count(&mut self, value: i32) { + let Ok(value) = usize::try_from(value) else { + return; + }; + self.values.resize(value, Object::Undefined); + self.count_override = None; + } + #[must_use] + pub fn item(&self, item_index: i32) -> Object { + index(item_index) + .ok() + .and_then(|index| self.values.get(index)) + .cloned() + .unwrap_or(Object::Undefined) + } + pub fn set_item(&mut self, index_value: i32, value: Object) { + if let Ok(index) = index(index_value) + && let Some(slot) = self.values.get_mut(index) + { + *slot = value; + } + } + pub fn iter(&self) -> impl Iterator { + self.values.iter() + } +} + +/// Runtime lexer over one immutable filtered source buffer. +#[derive(Clone, Debug)] +pub struct Lexer { + pub m_buf: String, + pub m_debug: bool, + pub m_pch: i32, + pub m_state: String, + pub yytext: String, + pub m_line_manager: LineManager, + tokens: YyLexer, + units: Vec, + cursor: usize, + start_match: usize, + eof_emitted: bool, +} + +impl Lexer { + pub fn new(tokens: YyLexer) -> Result { + Ok(Self { + m_buf: String::new(), + m_debug: false, + m_pch: 0, + m_state: "YYINITIAL".to_owned(), + yytext: String::new(), + m_line_manager: LineManager::default(), + tokens, + units: Vec::new(), + cursor: 0, + start_match: 0, + eof_emitted: false, + }) + } + + pub fn start_with_string(&mut self, source: String) -> Result<(), Error> { + self.start_with_cs_reader(CsReader::new_with_string(format!("{source}\n"))?) + } + + pub fn start_with_stream_reader(&mut self, reader: StreamReader) -> Result<(), Error> { + let source = self.tokens.encoding.decode(&reader.0)?; + self.start_with_cs_reader(CsReader::new_with_string(source)?) + } + + pub fn start_with_cs_reader(&mut self, reader: CsReader) -> Result<(), Error> { + let (_, units, manager) = reader.into_parts(); + if units.len() > MAX_SOURCE_UNITS { + return Err(Error::Argument); + } + self.m_buf = units_to_string(&units); + self.units = units; + self.m_line_manager = manager; + self.reset()?; + Ok(()) + } + + pub fn reset(&mut self) -> Result<(), Error> { + self.cursor = 0; + self.start_match = 0; + self.m_pch = 0; + self.yytext.clear(); + self.eof_emitted = false; + Ok(()) + } + + pub fn yy_begin(&mut self, state: String) -> Result<(), Error> { + if self.tokens.dfa(&state).is_none() { + self.record_diagnostic( + 2, + DiagnosticCategory::InvalidState, + "unknown lexer start condition", + self.cursor, + )?; + return Err(Error::Parse { + position: self.cursor, + context: "unknown lexer start condition", + }); + } + self.m_state = state; + Ok(()) + } + + pub fn next_token(&mut self) -> Result, Error> { + loop { + if self.cursor >= self.units.len() { + if self.tokens.using_eof && !self.eof_emitted { + self.eof_emitted = true; + "EOF".clone_into(&mut self.yytext); + return Ok(Some(TOKEN::eof(self)?)); + } + return Ok(None); + } + let matched = self + .tokens + .dfa(&self.m_state) + .map(|dfa| dfa.longest_match(&self.units, self.cursor)); + let Some(matched) = matched else { + self.record_diagnostic( + 2, + DiagnosticCategory::InvalidState, + "lexer start condition has no DFA", + self.cursor, + )?; + return Err(Error::Parse { + position: self.cursor, + context: "lexer start condition has no DFA", + }); + }; + let Some((length, accept)) = matched else { + let position = self.cursor; + self.cursor += 1; + self.m_pch = i32_len(self.cursor)?; + self.record_diagnostic( + 1, + DiagnosticCategory::InvalidCharacter, + "input does not match any lexer rule", + position, + )?; + return Err(Error::Parse { + position, + context: "input does not match any lexer rule", + }); + }; + if length == 0 { + self.record_diagnostic( + 2, + DiagnosticCategory::InvalidState, + "non-EOF lexer rule accepted an empty token", + self.cursor, + )?; + return Err(Error::Parse { + position: self.cursor, + context: "non-EOF lexer rule accepted an empty token", + }); + } + if length > MAX_TOKEN_UNITS { + self.record_diagnostic( + 3, + DiagnosticCategory::TokenTooLong, + "token exceeds the configured size bound", + self.cursor, + )?; + return Err(Error::Parse { + position: self.cursor, + context: "token exceeds the configured size bound", + }); + } + self.start_match = self.cursor; + let end = self.cursor + length; + let mut text = units_to_string(&self.units[self.cursor..end]); + if self.tokens.toupper { + text = text.to_uppercase(); + } + self.cursor = end; + self.m_pch = i32_len(end)?; + self.yytext.clone_from(&text); + let mut definition = accept.token; + if let Some(table) = accept.reserved_words.as_deref() + && let Some(reserved) = self.tokens.reserved_word(table, &text) + { + definition = reserved; + } + let should_emit = match accept.action { + LexerAction::Emit => true, + LexerAction::Skip => false, + LexerAction::EmitAndBegin(state) => { + self.yy_begin(state)?; + true + } + LexerAction::SkipAndBegin(state) => { + self.yy_begin(state)?; + false + } + }; + if should_emit { + return Ok(Some(TOKEN::emitted( + self, + definition, + text, + self.start_match, + end, + )?)); + } + } + } + + pub fn next(&mut self) -> Result { + self.next_token()?.ok_or(Error::InvalidOperation) + } + + pub fn get_enumerator(&self) -> Result { + LexerEnumerator::new(self.clone()) + } + + #[must_use] + pub fn iter(&self) -> LexerIterator { + LexerIterator { + lexer: self.clone(), + finished: false, + } + } + + pub fn peek_char(&self) -> Result { + Ok(Utf16CodeUnit( + self.units + .get(self.cursor) + .copied() + .unwrap_or(if self.tokens.using_eof { u16::MAX } else { 0 }), + )) + } + + pub fn get_char(&mut self) -> Result { + let value = self.peek_char()?.0; + if self.cursor < self.units.len() { + self.cursor += 1; + self.m_pch = i32_len(self.cursor)?; + } + Ok(i32::from(value)) + } + + pub fn advance(&mut self) -> Result<(), Error> { + let _ = self.get_char()?; + Ok(()) + } + + pub fn un_get_char(&mut self) -> Result<(), Error> { + if self.cursor == 0 { + return Err(Error::InvalidOperation); + } + self.cursor -= 1; + self.m_pch = i32_len(self.cursor)?; + Ok(()) + } + + pub fn source_line_info(&self, pos: i32) -> Result { + SourceLineInfo::from_manager(&self.m_line_manager, pos, self.m_buf.clone()) + } + + pub fn source_line(&self, info: SourceLineInfo) -> Result { + Ok(info.source_line()) + } + + pub fn saypos(&self, pos: i32) -> Result { + Ok(self.source_line_info(pos)?.to_string()) + } + + fn record_diagnostic( + &mut self, + code: i32, + category: DiagnosticCategory, + message: &str, + position: usize, + ) -> Result<(), Error> { + let location = self.source_line_info(i32_len(position)?)?; + let input = self + .units + .get(position) + .map(|unit| units_to_string(std::slice::from_ref(unit))) + .unwrap_or_default(); + self.tokens.erh.push(Diagnostic { + code, + category, + severity: DiagnosticSeverity::Error, + message: message.to_owned(), + location, + input, + }) + } + + #[must_use] + pub fn diagnostics(&self) -> &[Diagnostic] { + self.tokens.erh.diagnostics() + } + + #[must_use] + pub fn m_start(&self) -> Dfa { + self.tokens.dfa(&self.m_state).cloned().unwrap_or_default() + } + #[must_use] + pub fn tokens(&self) -> YyLexer { + self.tokens.clone() + } + pub fn set_tokens(&mut self, value: YyLexer) { + self.tokens = value; + } + #[must_use] + pub fn yypos(&self) -> i32 { + self.m_pch + } +} + +/// Idiomatic Rust iterator over lexer results. +pub struct LexerIterator { + lexer: Lexer, + finished: bool, +} + +impl Iterator for LexerIterator { + type Item = Result; + + fn next(&mut self) -> Option { + if self.finished { + return None; + } + match self.lexer.next_token() { + Ok(Some(token)) => Some(Ok(token)), + Ok(None) => { + self.finished = true; + None + } + Err(error) => { + self.finished = true; + Some(Err(error)) + } + } + } +} + +impl IntoIterator for &Lexer { + type Item = Result; + type IntoIter = LexerIterator; + + fn into_iter(self) -> Self::IntoIter { + self.iter() + } +} + +/// Compatibility enumerator backed by the same bounded lexer iterator. +#[derive(Clone, Debug)] +pub struct LexerEnumerator { + lexer: Lexer, + initial: Lexer, + current: Option, +} + +impl LexerEnumerator { + pub fn new(lexer: Lexer) -> Result { + Ok(Self { + initial: lexer.clone(), + lexer, + current: None, + }) + } + + pub fn move_next(&mut self) -> Result { + self.current = self.lexer.next_token()?; + Ok(self.current.is_some()) + } + + pub fn reset(&mut self) -> Result<(), Error> { + self.lexer = self.initial.clone(); + self.current = None; + Ok(()) + } + + #[must_use] + pub fn current(&self) -> TOKEN { + self.current + .clone() + .expect("Current is unavailable before MoveNext") + } + + pub fn set_current(&mut self, value: TOKEN) { + self.current = Some(value); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn definition(name: &str, number: i32) -> TokenDefinition { + TokenDefinition::new(name, number).expect("valid token definition") + } + + fn accept(name: &str, number: i32, action: LexerAction) -> DfaAccept { + DfaAccept { + token: definition(name, number), + action, + action_number: 0, + reserved_words: None, + } + } + + fn sample_lexer() -> Lexer { + let mut table = YyLexer::new(ErrorHandler::default()).expect("table"); + table.using_eof = true; + let states = vec![ + DfaState::default() + .transition( + CharacterMatcher::Range { + first: b'a'.into(), + last: b'z'.into(), + }, + 1, + ) + .transition(CharacterMatcher::Exact(b' '.into()), 2) + .transition(CharacterMatcher::Exact(b'\n'.into()), 2), + DfaState::default() + .transition( + CharacterMatcher::Range { + first: b'a'.into(), + last: b'z'.into(), + }, + 1, + ) + .accepting(accept("ID", 3, LexerAction::Emit)), + DfaState::default() + .transition(CharacterMatcher::Exact(b' '.into()), 2) + .transition(CharacterMatcher::Exact(b'\n'.into()), 2) + .accepting(accept("WS", 4, LexerAction::Skip)), + ]; + table + .set_start_dfa("YYINITIAL", Dfa::from_states(states, 0).expect("dfa")) + .expect("start"); + let mut lexer = Lexer::new(table).expect("lexer"); + lexer + .start_with_string("alpha beta".to_owned()) + .expect("source"); + lexer + } + + #[test] + fn lexer_uses_maximum_munch_skips_and_emits_eof() { + let mut lexer = sample_lexer(); + let first = lexer.next().expect("first"); + let second = lexer.next().expect("second"); + let eof = lexer.next().expect("eof"); + assert_eq!( + (first.yyname(), first.yytext(), first.pos, first.end), + ("ID".to_owned(), "alpha".to_owned(), 0, 5) + ); + assert_eq!( + (second.yyname(), second.yytext(), second.pos, second.end), + ("ID".to_owned(), "beta".to_owned(), 6, 10) + ); + assert_eq!( + (eof.yyname(), eof.yynum(), eof.pos), + ("EOF".to_owned(), 2, 11) + ); + assert!(lexer.next_token().expect("done").is_none()); + } + + #[test] + fn comments_are_removed_but_raw_columns_are_preserved() { + let reader = + CsReader::new_with_string("ab/*xyz*/cd\n// comment\nef".to_owned()).expect("reader"); + assert_eq!(units_to_string(&reader.units), "abcd\n\nef"); + let first = SourceLineInfo::from_manager(&reader.lm, 2, units_to_string(&reader.units)) + .expect("location"); + let after = SourceLineInfo::from_manager(&reader.lm, 3, units_to_string(&reader.units)) + .expect("location"); + assert_eq!(first.raw_char_position, 2); + assert_eq!(after.raw_char_position, 10); + let third_line = + SourceLineInfo::from_manager(&reader.lm, 6, units_to_string(&reader.units)) + .expect("location"); + assert_eq!(third_line.line_number, 3); + } + + #[test] + fn utf16_positions_do_not_turn_into_utf8_byte_offsets() { + let reader = CsReader::new_with_string("a😀b\n".to_owned()).expect("reader"); + assert_eq!(reader.units.len(), 5); + let info = SourceLineInfo::from_manager(&reader.lm, 4, units_to_string(&reader.units)) + .expect("location"); + assert_eq!(info.char_position, 5); + } + + #[test] + fn line_directives_change_logical_line_and_file() { + let reader = CsReader::new_with_string("#line 42 \"generated.cs\"\nvalue\n".to_owned()) + .expect("reader"); + assert_eq!(reader.fname, "generated.cs"); + let info = SourceLineInfo::from_manager(&reader.lm, 1, units_to_string(&reader.units)) + .expect("location"); + assert_eq!(info.line_number, 42); + } + + #[test] + fn unterminated_block_comment_has_stable_category_and_position() { + assert_eq!( + CsReader::new_with_string("a /* nope".to_owned()).expect_err("invalid"), + Error::Parse { + position: 9, + context: "unterminated block comment" + } + ); + } + + #[test] + fn unicode_categories_match_dotnet_numbering() { + assert!( + CatTest::new(UnicodeCategory(0)) + .expect("category") + .test(Utf16CodeUnit('A' as u16)) + .expect("test") + ); + assert!( + CatTest::new(UnicodeCategory(25)) + .expect("category") + .test(Utf16CodeUnit('+' as u16)) + .expect("test") + ); + assert!( + CatTest::new(UnicodeCategory(16)) + .expect("category") + .test(Utf16CodeUnit(0xD800)) + .expect("test") + ); + } + + #[test] + fn reserved_words_remap_only_the_configured_token() { + let mut lexer = sample_lexer(); + lexer + .tokens + .set_reserved_words( + "keywords", + ResWds::from_pairs([("alpha", definition("ALPHA", 9))], false).expect("words"), + ) + .expect("table"); + if let Some(dfa) = lexer.tokens.start_dfas.get_mut("YYINITIAL") { + dfa.states[1] + .accept + .as_mut() + .expect("accept") + .reserved_words = Some("keywords".to_owned()); + } + let first = lexer.next().expect("token"); + assert_eq!((first.yyname(), first.yynum()), ("ALPHA".to_owned(), 9)); + } + + #[test] + fn encoded_input_is_validated_without_platform_code_pages() { + assert_eq!( + InputEncoding::Ascii.decode(&[0x80]), + Err(Error::Parse { + position: 0, + context: "source contains a non-ASCII byte" + }) + ); + assert_eq!( + InputEncoding::Utf16Le.decode(&[b'a', 0, b'b']), + Err(Error::Parse { + position: 2, + context: "UTF-16 source has a trailing byte" + }) + ); + } + + #[test] + fn lexer_iterator_stops_after_a_diagnostic() { + let mut lexer = sample_lexer(); + lexer + .start_with_string("alpha ! beta".to_owned()) + .expect("source"); + let mut iter = lexer.iter(); + assert_eq!( + iter.next().expect("first").expect("token").yytext(), + "alpha" + ); + assert!(matches!( + iter.next().expect("diagnostic"), + Err(Error::Parse { + position: 6, + context: "input does not match any lexer rule" + }) + )); + assert!(iter.next().is_none()); + } +} diff --git a/crates/libremetaverse-lsl-tools/src/lib.rs b/crates/libremetaverse-lsl-tools/src/lib.rs index c8e29eb..f4614b4 100644 --- a/crates/libremetaverse-lsl-tools/src/lib.rs +++ b/crates/libremetaverse-lsl-tools/src/lib.rs @@ -3,6 +3,12 @@ extern crate self as libremetaverse_lsl_tools; mod generated; +mod lexer; pub use generated::*; +pub use lexer::{ + CharacterMatcher, DfaAccept, DfaState, Diagnostic, DiagnosticCategory, DiagnosticSeverity, + DotNetUnicodeCategory, InputEncoding, LexerAction, LexerIterator, MAX_SOURCE_UNITS, + MAX_TOKEN_UNITS, TokenDefinition, UnicodeClass, +}; pub use libremetaverse_types::Error; diff --git a/crates/libremetaverse-lsl-tools/tests/lexer_compat.rs b/crates/libremetaverse-lsl-tools/tests/lexer_compat.rs new file mode 100644 index 0000000..dc2811d --- /dev/null +++ b/crates/libremetaverse-lsl-tools/tests/lexer_compat.rs @@ -0,0 +1,391 @@ +//! Focused compatibility fixtures translated from the pinned lexer runtime. + +use std::collections::BTreeSet; +use std::sync::{Arc, Mutex}; + +use libremetaverse_lsl_tools::{ + CSToolsException, CharacterMatcher, CsReader, Dfa, DfaAccept, DfaState, DiagnosticCategory, + DotNetUnicodeCategory, Error, ErrorHandler, InputEncoding, Lexer, LexerAction, LineManager, + ObjectList, ResWds, SourceLineInfo, TOKEN, TokenDefinition, YyLexer, +}; +use libremetaverse_types::compat::{Object, UnicodeCategory, Utf16CodeUnit}; + +fn token(name: &str, number: i32) -> TokenDefinition { + TokenDefinition::new(name, number).expect("valid definition") +} + +fn accept(name: &str, number: i32, action: LexerAction, reserved_words: Option<&str>) -> DfaAccept { + DfaAccept { + token: token(name, number), + action, + action_number: number, + reserved_words: reserved_words.map(ToOwned::to_owned), + } +} + +fn language_table() -> YyLexer { + let letters = vec![ + DotNetUnicodeCategory::UppercaseLetter, + DotNetUnicodeCategory::LowercaseLetter, + DotNetUnicodeCategory::TitlecaseLetter, + DotNetUnicodeCategory::ModifierLetter, + DotNetUnicodeCategory::OtherLetter, + ]; + let whitespace = vec![ + DotNetUnicodeCategory::Control, + DotNetUnicodeCategory::SpaceSeparator, + DotNetUnicodeCategory::LineSeparator, + DotNetUnicodeCategory::ParagraphSeparator, + ]; + let states = vec![ + DfaState::default() + .transition(CharacterMatcher::Categories(letters.clone()), 1) + .transition(CharacterMatcher::Exact(u16::from(b'_')), 1) + .transition( + CharacterMatcher::Category(DotNetUnicodeCategory::DecimalDigitNumber), + 2, + ) + .transition(CharacterMatcher::Categories(whitespace.clone()), 3) + .transition(CharacterMatcher::Exact(u16::from(b'+')), 4), + DfaState::default() + .transition(CharacterMatcher::Categories(letters), 1) + .transition(CharacterMatcher::Exact(u16::from(b'_')), 1) + .transition( + CharacterMatcher::Category(DotNetUnicodeCategory::DecimalDigitNumber), + 1, + ) + .accepting(accept("ID", 3, LexerAction::Emit, Some("keywords"))), + DfaState::default() + .transition( + CharacterMatcher::Category(DotNetUnicodeCategory::DecimalDigitNumber), + 2, + ) + .accepting(accept("INTEGER", 4, LexerAction::Emit, None)), + DfaState::default() + .transition(CharacterMatcher::Categories(whitespace), 3) + .accepting(accept("WS", 5, LexerAction::Skip, None)), + DfaState::default().accepting(accept("PLUS", 6, LexerAction::Emit, None)), + ]; + let mut table = YyLexer::new(ErrorHandler::default()).expect("table"); + table.using_eof = true; + table + .set_start_dfa("YYINITIAL", Dfa::from_states(states, 0).expect("dfa")) + .expect("start state"); + table + .set_reserved_words( + "keywords", + ResWds::from_pairs( + [("if", token("IF", 20)), ("while", token("WHILE", 21))], + false, + ) + .expect("reserved words"), + ) + .expect("reserved table"); + table +} + +fn lexer(source: &str) -> Lexer { + let mut lexer = Lexer::new(language_table()).expect("lexer"); + lexer + .start_with_string(source.to_owned()) + .expect("valid source"); + lexer +} + +#[test] +fn token_fixture_preserves_names_text_numbers_positions_and_eof() { + let mut lexer = lexer("if café_2 + 19"); + let mut tokens = Vec::new(); + while let Some(value) = lexer.next_token().expect("tokenization") { + tokens.push(value); + } + let actual = tokens + .iter() + .map(|token| { + ( + token.yyname(), + token.yytext(), + token.yynum(), + token.pos, + token.end, + ) + }) + .collect::>(); + assert_eq!( + actual, + [ + ("IF".to_owned(), "if".to_owned(), 20, 0, 2), + ("ID".to_owned(), "café_2".to_owned(), 3, 3, 9), + ("PLUS".to_owned(), "+".to_owned(), 6, 10, 11), + ("INTEGER".to_owned(), "19".to_owned(), 4, 12, 14), + ("EOF".to_owned(), "EOF".to_owned(), 2, 15, 15), + ] + ); +} + +#[test] +fn reserved_words_are_exact_unless_the_table_requests_uppercase() { + let mut lexer = lexer("IF if"); + assert_eq!(lexer.next().expect("identifier").yyname(), "ID"); + assert_eq!(lexer.next().expect("keyword").yyname(), "IF"); + + let words = ResWds::from_pairs([("while", token("WHILE", 21))], true).expect("words"); + let mut value = TOKEN::new_with_lexer_string(lexer.clone(), "WhIlE".to_owned()).expect("token"); + words.check(lexer, &mut value).expect("check"); + assert_eq!((value.yyname(), value.yynum()), ("WHILE".to_owned(), 21)); +} + +#[test] +fn invalid_input_reports_stable_category_location_and_text() { + let mut lexer = lexer("ok @ nope"); + assert_eq!(lexer.next().expect("first").yytext(), "ok"); + assert_eq!( + lexer.next().expect_err("invalid character"), + Error::Parse { + position: 3, + context: "input does not match any lexer rule" + } + ); + let diagnostic = &lexer.diagnostics()[0]; + assert_eq!(diagnostic.category, DiagnosticCategory::InvalidCharacter); + assert_eq!(diagnostic.location.line_number, 1); + assert_eq!(diagnostic.location.raw_char_position, 3); + assert_eq!(diagnostic.input, "@"); +} + +#[test] +fn unknown_start_condition_reports_invalid_state_without_changing_state() { + let mut lexer = lexer("value"); + assert_eq!( + lexer.yy_begin("MISSING".to_owned()), + Err(Error::Parse { + position: 0, + context: "unknown lexer start condition" + }) + ); + assert_eq!(lexer.m_state, "YYINITIAL"); + assert_eq!( + lexer.diagnostics()[0].category, + DiagnosticCategory::InvalidState + ); +} + +#[test] +fn source_reader_removes_both_comment_forms_and_keeps_newlines() { + let mut reader = CsReader::new_with_string("a/*x\ny*/b//z\nc".to_owned()).expect("reader"); + assert_eq!(reader.read_line().expect("line"), "a"); + assert_eq!(reader.read_line().expect("line"), "b"); + assert_eq!(reader.read_line().expect("line"), "c"); + assert!(reader.eof().expect("eof")); +} + +#[test] +fn source_reader_normalizes_crlf_and_lone_cr() { + let mut reader = CsReader::new_with_string("a\r\nb\rc\n".to_owned()).expect("reader"); + assert_eq!(reader.read_line().expect("line"), "a"); + assert_eq!(reader.read_line().expect("line"), "b"); + assert_eq!(reader.read_line().expect("line"), "c"); +} + +#[test] +fn source_reader_rejects_unterminated_block_comment_at_eof() { + assert_eq!( + CsReader::new_with_string("before /* never closed".to_owned()).expect_err("invalid"), + Error::Parse { + position: 22, + context: "unterminated block comment" + } + ); +} + +#[test] +fn line_directive_updates_file_and_logical_line() { + let reader = + CsReader::new_with_string("#line 700 \"table.lex\"\nword\n".to_owned()).expect("reader"); + assert_eq!(reader.fname, "table.lex"); + let info = SourceLineInfo::new_with_line_manager_int32(reader.lm, 1).expect("location"); + assert_eq!(info.line_number, 700); +} + +#[test] +fn utf16_input_decoding_and_surrogate_validation_are_deterministic() { + let mut reader = CsReader::from_bytes( + &[0xFF, 0xFE, b'A', 0, 0x3D, 0xD8, 0, 0xDE], + InputEncoding::Utf16Le, + "source.lex", + ) + .expect("utf16"); + assert_eq!(reader.read_line().expect("line"), "A😀"); + let mut bom_override = + CsReader::from_bytes(&[0xFE, 0xFF, 0, b'B'], InputEncoding::Utf16Le, "source.lex") + .expect("BOM override"); + assert_eq!(bom_override.read_line().expect("line"), "B"); + assert!(matches!( + CsReader::from_bytes(&[0x00, 0xD8], InputEncoding::Utf16Le, "bad"), + Err(Error::Parse { + position: 0, + context: "source contains an unpaired UTF-16 surrogate" + }) + )); +} + +#[test] +fn ascii_input_rejects_non_ascii_instead_of_lossily_replacing_it() { + assert!(matches!( + CsReader::from_bytes(&[b'a', 0x80], InputEncoding::Ascii, "bad"), + Err(Error::Parse { + position: 1, + context: "source contains a non-ASCII byte" + }) + )); +} + +#[test] +fn category_predicates_match_dotnet_values_and_groups() { + let mut table = YyLexer::new(ErrorHandler::default()).expect("table"); + let punctuation = table.get_test("Punctuation").expect("punctuation"); + let whitespace = table.get_test("WhiteSpace").expect("whitespace"); + let states = vec![ + DfaState::default() + .transition(punctuation, 1) + .transition(whitespace, 2), + DfaState::default().accepting(accept("PUNCT", 1, LexerAction::Emit, None)), + DfaState::default().accepting(accept("SPACE", 2, LexerAction::Emit, None)), + ]; + let dfa = Dfa::from_states(states, 0).expect("dfa"); + let mut action = -1; + assert_eq!(dfa.match_("—".to_owned(), 0, &mut action), Ok(1)); + assert_eq!(dfa.match_("\t".to_owned(), 0, &mut action), Ok(1)); + assert_eq!(dfa.match_("\0".to_owned(), 0, &mut action), Ok(-1)); + assert!( + libremetaverse_lsl_tools::CatTest::new(UnicodeCategory(16)) + .expect("surrogate") + .test(Utf16CodeUnit(0xD800)) + .expect("test") + ); + assert_eq!(table.get_test("MissingClass"), Err(Error::Argument)); + assert_eq!( + table.erh.diagnostics()[0].category, + DiagnosticCategory::UnknownCharacterSet + ); +} + +#[test] +fn dfa_uses_maximum_munch_and_exposes_action_number() { + let states = vec![ + DfaState::default().transition(CharacterMatcher::Exact(b'a'.into()), 1), + DfaState::default() + .transition(CharacterMatcher::Exact(b'b'.into()), 2) + .accepting(accept("A", 1, LexerAction::Emit, None)), + DfaState::default().accepting(accept("AB", 2, LexerAction::Emit, None)), + ]; + let dfa = Dfa::from_states(states, 0).expect("dfa"); + let mut action = -1; + assert_eq!(dfa.match_("abc".to_owned(), 0, &mut action), Ok(2)); + assert_eq!(action, 2); +} + +#[test] +fn deterministic_table_output_is_independent_of_hash_iteration() { + let table = language_table(); + let first = Arc::new(Mutex::new(Vec::new())); + table + .emit_dfa(Box::new(SharedWriter(first.clone()))) + .expect("emit first"); + let second = Arc::new(Mutex::new(Vec::new())); + table + .emit_dfa(Box::new(SharedWriter(second.clone()))) + .expect("emit second"); + let first = first.lock().expect("first output").clone(); + let second = second.lock().expect("second output").clone(); + assert_eq!(first, second); + assert!( + String::from_utf8(first) + .expect("utf8") + .starts_with("LSLLEXER 1\n") + ); +} + +struct SharedWriter(Arc>>); + +impl std::io::Write for SharedWriter { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + self.0 + .lock() + .map_err(|_| std::io::Error::other("poisoned output"))? + .extend_from_slice(bytes); + Ok(bytes.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +#[test] +fn compatibility_enumerator_resets_to_the_first_token() { + let lexer = lexer("one two"); + let mut enumerator = lexer.get_enumerator().expect("enumerator"); + assert!(enumerator.move_next().expect("first")); + assert_eq!(enumerator.current().yytext(), "one"); + assert!(enumerator.move_next().expect("second")); + assert_eq!(enumerator.current().yytext(), "two"); + enumerator.reset().expect("reset"); + assert!(enumerator.move_next().expect("first again")); + assert_eq!(enumerator.current().yytext(), "one"); +} + +#[test] +fn error_handler_counts_reports_and_honors_throw_mode() { + let error = + CSToolsException::new_with_int32_string(43, "bad encoding".to_owned()).expect("diagnostic"); + let mut collecting = ErrorHandler::new_with_constructor().expect("handler"); + collecting.error(error.clone()).expect("reported"); + assert_eq!(collecting.counter, 1); + assert_eq!( + collecting.diagnostics()[0].category, + DiagnosticCategory::Encoding + ); + + let mut throwing = ErrorHandler::new_with_boolean(true).expect("handler"); + assert_eq!(throwing.error(error), Err(Error::InvalidOperation)); + assert_eq!(throwing.counter, 1); + assert!(throwing.diagnostics().is_empty()); +} + +#[test] +fn line_manager_backtracking_removes_future_lines() { + let mut manager = LineManager::new().expect("manager"); + manager.newline(4).expect("line 2"); + let first_on_second = + SourceLineInfo::new_with_line_manager_int32(manager.clone(), 4).expect("location"); + assert_eq!(first_on_second.raw_char_position, 1); + manager.newline(8).expect("line 3"); + manager.backto(6).expect("rewind"); + assert_eq!(manager.lines, 2); + assert_eq!(manager.end, 8); +} + +#[test] +fn object_list_preserves_push_add_top_pop_and_index_order() { + let mut values = ObjectList::new().expect("list"); + values.add(Object::Integer(2)).expect("add"); + values.push(Object::Integer(1)).expect("push"); + assert_eq!(values.count(), 2); + assert_eq!(values.top(), Object::Integer(1)); + assert_eq!(values.item(1), Object::Integer(2)); + assert_eq!(values.pop(), Ok(Object::Integer(1))); +} + +#[test] +fn matcher_set_is_value_based_and_bounded() { + let matcher = CharacterMatcher::Set(BTreeSet::from([b'x'.into(), b'y'.into()])); + let states = vec![ + DfaState::default().transition(matcher, 1), + DfaState::default().accepting(accept("XY", 1, LexerAction::Emit, None)), + ]; + let dfa = Dfa::from_states(states, 0).expect("dfa"); + let mut action = 0; + assert_eq!(dfa.match_("y".to_owned(), 0, &mut action), Ok(1)); + assert_eq!(dfa.match_("z".to_owned(), 0, &mut action), Ok(-1)); +} diff --git a/crates/libremetaverse-types/src/compat.rs b/crates/libremetaverse-types/src/compat.rs index 05f314a..376fa45 100644 --- a/crates/libremetaverse-types/src/compat.rs +++ b/crates/libremetaverse-types/src/compat.rs @@ -269,7 +269,7 @@ pub struct ArrayList(pub Vec); #[derive(Clone, Debug, Eq, PartialEq)] pub struct Array(pub Vec); -#[derive(Clone, Debug, Eq, PartialEq)] +#[derive(Clone, Debug, Default, Eq, PartialEq)] pub struct Hashtable(pub std::collections::HashMap); #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] @@ -1307,8 +1307,10 @@ impl<'a, T> IntoIterator for &'a ImmutableList { pub struct ICollection; +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] pub struct UnicodeCategory(pub i32); +#[derive(Clone, Copy, Debug, Default, Eq, Hash, PartialEq)] pub struct TextEncoding; pub struct SocketException; diff --git a/tests/api-compile/Cargo.lock b/tests/api-compile/Cargo.lock index 531009e..c3fad8c 100644 --- a/tests/api-compile/Cargo.lock +++ b/tests/api-compile/Cargo.lock @@ -783,6 +783,7 @@ name = "libremetaverse-lsl-tools" version = "0.0.1" dependencies = [ "libremetaverse-types", + "unicode-general-category", ] [[package]] @@ -1921,6 +1922,12 @@ version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" +[[package]] +name = "unicode-general-category" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b993bddc193ae5bd0d623b49ec06ac3e9312875fdae725a975c51db1cc1677f" + [[package]] name = "unicode-ident" version = "1.0.24" diff --git a/tools/check_milestone_10_issue_81.py b/tools/check_milestone_10_issue_81.py new file mode 100644 index 0000000..d2d6c43 --- /dev/null +++ b/tools/check_milestone_10_issue_81.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python3 +"""Audit issue 81's native lexer, token, source, and diagnostic boundary.""" + +from __future__ import annotations + +import json +import re +from pathlib import Path + +import generate_api_shims + + +ROOT = Path(__file__).resolve().parents[1] +SOURCE = ROOT / "crates" / "libremetaverse-lsl-tools" / "src" / "lexer.rs" +GENERATED = ROOT / "crates" / "libremetaverse-lsl-tools" / "src" / "generated.rs" +UNIT_TESTS = SOURCE +COMPAT_TESTS = ( + ROOT / "crates" / "libremetaverse-lsl-tools" / "tests" / "lexer_compat.rs" +) +DOC = ROOT / "crates" / "libremetaverse-lsl-tools" / "README.md" +WORKFLOW = ROOT / ".gitea" / "workflows" / "rust-workspace.yml" +CATALOG = ROOT / "api" / "public-api.json" +STUB_RE = re.compile( + r"\b(?:not_implemented|unimplemented_api)\b|\b(?:todo|unimplemented)!\s*\(" +) + +TYPES = { + "T:LibreMetaverse.LslTools.CSToolsException": "crate::lexer::CSToolsException", + "T:LibreMetaverse.LslTools.CSToolsFatalException": + "crate::lexer::CSToolsFatalException", + "T:LibreMetaverse.LslTools.CSToolsStopException": + "crate::lexer::CSToolsStopException", + "T:LibreMetaverse.LslTools.CatTest": "crate::lexer::CatTest", + "T:LibreMetaverse.LslTools.Charset": "crate::lexer::Charset", + "T:LibreMetaverse.LslTools.CommentList": "crate::lexer::CommentList", + "T:LibreMetaverse.LslTools.CsReader": "crate::lexer::CsReader", + "T:LibreMetaverse.LslTools.Dfa": "crate::lexer::Dfa", + "T:LibreMetaverse.LslTools.Dfa.Action": "crate::lexer::DfaAction", + "T:LibreMetaverse.LslTools.EOF": "crate::lexer::EOF", + "T:LibreMetaverse.LslTools.ErrorHandler": "crate::lexer::ErrorHandler", + "T:LibreMetaverse.LslTools.Lexer": "crate::lexer::Lexer", + "T:LibreMetaverse.LslTools.Lexer._Enumerator": "crate::lexer::LexerEnumerator", + "T:LibreMetaverse.LslTools.LineList": "crate::lexer::LineList", + "T:LibreMetaverse.LslTools.LineManager": "crate::lexer::LineManager", + "T:LibreMetaverse.LslTools.Null": "crate::lexer::Null", + "T:LibreMetaverse.LslTools.ObjectList": "crate::lexer::ObjectList", + "T:LibreMetaverse.LslTools.ResWds": "crate::lexer::ResWds", + "T:LibreMetaverse.LslTools.SYMBOL": "crate::lexer::SYMBOL", + "T:LibreMetaverse.LslTools.SourceLineInfo": "crate::lexer::SourceLineInfo", + "T:LibreMetaverse.LslTools.TOKEN": "crate::lexer::TOKEN", + "T:LibreMetaverse.LslTools.YyLexer": "crate::lexer::YyLexer", +} + + +def require_markers(path: Path, markers: tuple[str, ...]) -> None: + text = path.read_text() + missing = [marker for marker in markers if marker not in text] + if missing: + raise SystemExit(f"{path.name}: audit evidence missing: " + ", ".join(missing)) + + +def catalog_members() -> set[str]: + catalog = json.loads(CATALOG.read_text()) + assembly = next( + value for value in catalog["assemblies"] + if value["identity"]["name"] == "LibreMetaverse.LslTools" + ) + return { + member["doc_id"] + for api_type in assembly["types"] + if api_type["doc_id"] in TYPES + for member in api_type["members"] + } + + +def generated_type_block(text: str, doc_id: str) -> str: + marker = f"/// C# type: `{doc_id}`." + start = text.find(marker) + if start < 0: + raise SystemExit(f"generated declaration is missing for {doc_id}") + next_type = text.find("/// C# type:", start + len(marker)) + return text[start:] if next_type < 0 else text[start:next_type] + + +def main() -> None: + for api_type, declaration in TYPES.items(): + if generate_api_shims.NATIVE_TYPES.get(api_type) != declaration: + raise SystemExit(f"issue 81 native type mapping is missing for {api_type}") + members = catalog_members() + if len(members) != 188: + raise SystemExit(f"issue 81 expected 188 mapped members, found {len(members)}") + + source = SOURCE.read_text() + if STUB_RE.search(source): + raise SystemExit("issue 81 owned Rust stubs remain in lexer.rs") + if "Ok(o)" in source or "TokenDefinition::new(name, 1)" in source: + raise SystemExit("issue 81 contains a silent compatibility no-op or fabricated token") + if re.search(r"unsafe\s*\{|unsafe\s+impl|target_os\s*=\s*\"macos\"", source): + raise SystemExit("lexer.rs contains an unsafe or macOS-only implementation") + generated = GENERATED.read_text() + for api_type in TYPES: + if STUB_RE.search(generated_type_block(generated, api_type)): + raise SystemExit(f"issue 81 owned generated stubs remain for {api_type}") + + require_markers(SOURCE, ( + "MAX_SOURCE_UNITS", "MAX_TOKEN_UNITS", "pub enum DotNetUnicodeCategory", + "pub enum InputEncoding", "pub struct CatTest", "pub struct Charset", + "pub struct CsReader", "fn filter_source", "fn parse_line_directive", + "pub struct LineManager", "pub struct SourceLineInfo", + "pub enum DiagnosticCategory", "pub struct Diagnostic", + "pub struct ErrorHandler", "pub enum CharacterMatcher", + "pub struct DfaState", "pub struct Dfa", "fn longest_match", + "pub struct ResWds", "pub struct YyLexer", "pub struct SYMBOL", + "pub struct TOKEN", "pub struct EOF", "pub struct Lexer", + "pub struct LexerIterator", "impl Iterator for LexerIterator", + "lookup_parser_entry", "emit_dfa", "record_diagnostic", + "unicode_general_category", "String::from_utf16_lossy", + )) + require_markers(COMPAT_TESTS, ( + "token_fixture_preserves_names_text_numbers_positions_and_eof", + "invalid_input_reports_stable_category_location_and_text", + "source_reader_removes_both_comment_forms_and_keeps_newlines", + "utf16_input_decoding_and_surrogate_validation_are_deterministic", + "category_predicates_match_dotnet_values_and_groups", + "dfa_uses_maximum_munch_and_exposes_action_number", + "deterministic_table_output_is_independent_of_hash_iteration", + "compatibility_enumerator_resets_to_the_first_token", + "error_handler_counts_reports_and_honors_throw_mode", + )) + test_count = UNIT_TESTS.read_text().count("#[test]") + COMPAT_TESTS.read_text().count("#[test]") + if test_count != 27: + raise SystemExit(f"issue 81 expected 27 focused fixtures, found {test_count}") + require_markers(DOC, ( + "UTF-16 code-unit offsets", "maximum munch", "structured `Diagnostic`", + "64 Mi UTF-16", "16 Mi units", "Lexer::iter", "27 focused", + "No C#, .NET runtime", "ubuntu-latest", + )) + require_markers(WORKFLOW, ("python3 tools/check_milestone_10_issue_81.py",)) + print( + "issue 81 audit: 22 mapped native types and 188 members, bounded UTF-16 " + "source handling, comments and line directives, Unicode categories, deterministic " + "DFA/token behavior, reserved words, EOF, diagnostics, iterators, parser-table " + "lookup, 27 focused fixtures, API documentation, and ubuntu-only CI are present" + ) + + +if __name__ == "__main__": + main() diff --git a/tools/generate_api_shims.py b/tools/generate_api_shims.py index d8f7042..96d34cb 100644 --- a/tools/generate_api_shims.py +++ b/tools/generate_api_shims.py @@ -38,6 +38,28 @@ TARGETS = { # implementations. The generated module keeps catalog markers and re-exports # the hand-written type so coverage remains deterministic. NATIVE_TYPES = { + "T:LibreMetaverse.LslTools.CSToolsException": "crate::lexer::CSToolsException", + "T:LibreMetaverse.LslTools.CSToolsFatalException": "crate::lexer::CSToolsFatalException", + "T:LibreMetaverse.LslTools.CSToolsStopException": "crate::lexer::CSToolsStopException", + "T:LibreMetaverse.LslTools.CatTest": "crate::lexer::CatTest", + "T:LibreMetaverse.LslTools.Charset": "crate::lexer::Charset", + "T:LibreMetaverse.LslTools.CommentList": "crate::lexer::CommentList", + "T:LibreMetaverse.LslTools.CsReader": "crate::lexer::CsReader", + "T:LibreMetaverse.LslTools.Dfa": "crate::lexer::Dfa", + "T:LibreMetaverse.LslTools.Dfa.Action": "crate::lexer::DfaAction", + "T:LibreMetaverse.LslTools.EOF": "crate::lexer::EOF", + "T:LibreMetaverse.LslTools.ErrorHandler": "crate::lexer::ErrorHandler", + "T:LibreMetaverse.LslTools.Lexer": "crate::lexer::Lexer", + "T:LibreMetaverse.LslTools.Lexer._Enumerator": "crate::lexer::LexerEnumerator", + "T:LibreMetaverse.LslTools.LineList": "crate::lexer::LineList", + "T:LibreMetaverse.LslTools.LineManager": "crate::lexer::LineManager", + "T:LibreMetaverse.LslTools.Null": "crate::lexer::Null", + "T:LibreMetaverse.LslTools.ObjectList": "crate::lexer::ObjectList", + "T:LibreMetaverse.LslTools.ResWds": "crate::lexer::ResWds", + "T:LibreMetaverse.LslTools.SYMBOL": "crate::lexer::SYMBOL", + "T:LibreMetaverse.LslTools.SourceLineInfo": "crate::lexer::SourceLineInfo", + "T:LibreMetaverse.LslTools.TOKEN": "crate::lexer::TOKEN", + "T:LibreMetaverse.LslTools.YyLexer": "crate::lexer::YyLexer", "T:LibreMetaverse.RLV.AttachmentRequest": "crate::service::AttachmentRequest", "T:LibreMetaverse.RLV.RlvActionCallbacksDefault": "crate::service::RlvActionCallbacksDefault", "T:LibreMetaverse.RLV.RlvCallbacksDefault": "crate::service::RlvCallbacksDefault",