MetaCrate/crates/libremetaverse-lsl-tools/src/lexer.rs

//! Native lexer runtime and source diagnostics.
//!
//! Positions in this module are UTF-16 code-unit offsets. That is deliberate:
//! the original C# API exposes `System.Char` positions, and using byte offsets
//! would move every diagnostic after a non-ASCII character.

#![allow(clippy::inherent_to_string_shadow_display)]
#![allow(clippy::missing_errors_doc)]
#![allow(clippy::missing_panics_doc)]
#![allow(clippy::must_use_candidate)]
#![allow(clippy::needless_pass_by_value)]
#![allow(clippy::should_implement_trait)]
#![allow(clippy::too_many_lines)]

use std::collections::{BTreeMap, BTreeSet, VecDeque};
use std::fmt;
use std::io::Write;

use libremetaverse_types::compat::{
    Hashtable, IEnumerator, Object, StreamReader, TextEncoding, UnicodeCategory, Utf16CodeUnit,
};
use unicode_general_category::{GeneralCategory, get_general_category};

use crate::{Error, Nfa, Parser, ParserEntry, Serialiser, SymbolsGen, TokensGen, YyParser};

/// Largest source accepted by the compatibility reader (64 Mi UTF-16 units).
pub const MAX_SOURCE_UNITS: usize = 64 * 1024 * 1024;
/// Largest individual token accepted by the lexer (16 Mi UTF-16 units).
pub const MAX_TOKEN_UNITS: usize = 16 * 1024 * 1024;

fn index(value: i32) -> Result<usize, Error> {
    usize::try_from(value).map_err(|_| Error::IndexOutOfRange)
}

fn i32_len(value: usize) -> Result<i32, Error> {
    i32::try_from(value).map_err(|_| Error::IndexOutOfRange)
}

fn units_to_string(units: &[u16]) -> String {
    String::from_utf16_lossy(units)
}

fn utf16_len(value: &str) -> usize {
    value.encode_utf16().count()
}

/// .NET-compatible Unicode general categories.
#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
#[repr(i32)]
pub enum DotNetUnicodeCategory {
    UppercaseLetter = 0,
    LowercaseLetter = 1,
    TitlecaseLetter = 2,
    ModifierLetter = 3,
    OtherLetter = 4,
    NonSpacingMark = 5,
    SpacingCombiningMark = 6,
    EnclosingMark = 7,
    DecimalDigitNumber = 8,
    LetterNumber = 9,
    OtherNumber = 10,
    SpaceSeparator = 11,
    LineSeparator = 12,
    ParagraphSeparator = 13,
    Control = 14,
    Format = 15,
    Surrogate = 16,
    PrivateUse = 17,
    ConnectorPunctuation = 18,
    DashPunctuation = 19,
    OpenPunctuation = 20,
    ClosePunctuation = 21,
    InitialQuotePunctuation = 22,
    FinalQuotePunctuation = 23,
    OtherPunctuation = 24,
    MathSymbol = 25,
    CurrencySymbol = 26,
    ModifierSymbol = 27,
    OtherSymbol = 28,
    OtherNotAssigned = 29,
}

impl DotNetUnicodeCategory {
    fn from_compat(value: UnicodeCategory) -> Result<Self, Error> {
        Self::try_from(value.0)
    }

    fn of_unit(unit: u16) -> Self {
        let Some(ch) = char::from_u32(u32::from(unit)) else {
            return Self::Surrogate;
        };
        match get_general_category(ch) {
            GeneralCategory::UppercaseLetter => Self::UppercaseLetter,
            GeneralCategory::LowercaseLetter => Self::LowercaseLetter,
            GeneralCategory::TitlecaseLetter => Self::TitlecaseLetter,
            GeneralCategory::ModifierLetter => Self::ModifierLetter,
            GeneralCategory::OtherLetter => Self::OtherLetter,
            GeneralCategory::NonspacingMark => Self::NonSpacingMark,
            GeneralCategory::SpacingMark => Self::SpacingCombiningMark,
            GeneralCategory::EnclosingMark => Self::EnclosingMark,
            GeneralCategory::DecimalNumber => Self::DecimalDigitNumber,
            GeneralCategory::LetterNumber => Self::LetterNumber,
            GeneralCategory::OtherNumber => Self::OtherNumber,
            GeneralCategory::SpaceSeparator => Self::SpaceSeparator,
            GeneralCategory::LineSeparator => Self::LineSeparator,
            GeneralCategory::ParagraphSeparator => Self::ParagraphSeparator,
            GeneralCategory::Control => Self::Control,
            GeneralCategory::Format => Self::Format,
            GeneralCategory::Surrogate => Self::Surrogate,
            GeneralCategory::PrivateUse => Self::PrivateUse,
            GeneralCategory::ConnectorPunctuation => Self::ConnectorPunctuation,
            GeneralCategory::DashPunctuation => Self::DashPunctuation,
            GeneralCategory::OpenPunctuation => Self::OpenPunctuation,
            GeneralCategory::ClosePunctuation => Self::ClosePunctuation,
            GeneralCategory::InitialPunctuation => Self::InitialQuotePunctuation,
            GeneralCategory::FinalPunctuation => Self::FinalQuotePunctuation,
            GeneralCategory::OtherPunctuation => Self::OtherPunctuation,
            GeneralCategory::MathSymbol => Self::MathSymbol,
            GeneralCategory::CurrencySymbol => Self::CurrencySymbol,
            GeneralCategory::ModifierSymbol => Self::ModifierSymbol,
            GeneralCategory::OtherSymbol => Self::OtherSymbol,
            _ => Self::OtherNotAssigned,
        }
    }

    fn name(self) -> &'static str {
        match self {
            Self::UppercaseLetter => "UppercaseLetter",
            Self::LowercaseLetter => "LowercaseLetter",
            Self::TitlecaseLetter => "TitlecaseLetter",
            Self::ModifierLetter => "ModifierLetter",
            Self::OtherLetter => "OtherLetter",
            Self::NonSpacingMark => "NonSpacingMark",
            Self::SpacingCombiningMark => "SpacingCombiningMark",
            Self::EnclosingMark => "EnclosingMark",
            Self::DecimalDigitNumber => "DecimalDigitNumber",
            Self::LetterNumber => "LetterNumber",
            Self::OtherNumber => "OtherNumber",
            Self::SpaceSeparator => "SpaceSeparator",
            Self::LineSeparator => "LineSeparator",
            Self::ParagraphSeparator => "ParagraphSeparator",
            Self::Control => "Control",
            Self::Format => "Format",
            Self::Surrogate => "Surrogate",
            Self::PrivateUse => "PrivateUse",
            Self::ConnectorPunctuation => "ConnectorPunctuation",
            Self::DashPunctuation => "DashPunctuation",
            Self::OpenPunctuation => "OpenPunctuation",
            Self::ClosePunctuation => "ClosePunctuation",
            Self::InitialQuotePunctuation => "InitialQuotePunctuation",
            Self::FinalQuotePunctuation => "FinalQuotePunctuation",
            Self::OtherPunctuation => "OtherPunctuation",
            Self::MathSymbol => "MathSymbol",
            Self::CurrencySymbol => "CurrencySymbol",
            Self::ModifierSymbol => "ModifierSymbol",
            Self::OtherSymbol => "OtherSymbol",
            Self::OtherNotAssigned => "OtherNotAssigned",
        }
    }
}

impl TryFrom<i32> for DotNetUnicodeCategory {
    type Error = Error;

    fn try_from(value: i32) -> Result<Self, Self::Error> {
        Ok(match value {
            0 => Self::UppercaseLetter,
            1 => Self::LowercaseLetter,
            2 => Self::TitlecaseLetter,
            3 => Self::ModifierLetter,
            4 => Self::OtherLetter,
            5 => Self::NonSpacingMark,
            6 => Self::SpacingCombiningMark,
            7 => Self::EnclosingMark,
            8 => Self::DecimalDigitNumber,
            9 => Self::LetterNumber,
            10 => Self::OtherNumber,
            11 => Self::SpaceSeparator,
            12 => Self::LineSeparator,
            13 => Self::ParagraphSeparator,
            14 => Self::Control,
            15 => Self::Format,
            16 => Self::Surrogate,
            17 => Self::PrivateUse,
            18 => Self::ConnectorPunctuation,
            19 => Self::DashPunctuation,
            20 => Self::OpenPunctuation,
            21 => Self::ClosePunctuation,
            22 => Self::InitialQuotePunctuation,
            23 => Self::FinalQuotePunctuation,
            24 => Self::OtherPunctuation,
            25 => Self::MathSymbol,
            26 => Self::CurrencySymbol,
            27 => Self::ModifierSymbol,
            28 => Self::OtherSymbol,
            29 => Self::OtherNotAssigned,
            _ => return Err(Error::Argument),
        })
    }
}

/// Character-category predicate corresponding to the C# `CatTest` delegate.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub struct CatTest {
    category: DotNetUnicodeCategory,
}

impl CatTest {
    /// Creates a category predicate.
    pub fn new(category: UnicodeCategory) -> Result<Self, Error> {
        Ok(Self {
            category: DotNetUnicodeCategory::from_compat(category)?,
        })
    }

    /// Tests one UTF-16 code unit using .NET category numbering.
    pub fn test(&self, ch: Utf16CodeUnit) -> Result<bool, Error> {
        Ok(DotNetUnicodeCategory::of_unit(ch.0) == self.category)
    }
}

/// Input encodings supported by lexer source files.
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
pub enum InputEncoding {
    Ascii,
    AsciiUpper,
    #[default]
    Utf8,
    Utf16Le,
    Utf16Be,
}

impl InputEncoding {
    /// Resolves the names accepted by the C# runtime.
    pub fn parse(name: &str) -> Result<Self, Error> {
        match name.trim().to_ascii_uppercase().as_str() {
            "" | "UTF8" | "UTF-8" | "65001" => Ok(Self::Utf8),
            "ASCII" | "US-ASCII" | "20127" => Ok(Self::Ascii),
            "ASCIICAPS" => Ok(Self::AsciiUpper),
            "UNICODE" | "UTF16" | "UTF-16" | "UTF-16LE" | "1200" => Ok(Self::Utf16Le),
            "UTF-16BE" | "1201" => Ok(Self::Utf16Be),
            // UTF-7 is intentionally rejected: it is obsolete and unsafe, and
            // modern .NET also disables it unless explicitly enabled.
            _ => Err(Error::Argument),
        }
    }

    fn decode(self, bytes: &[u8]) -> Result<String, Error> {
        let bytes = match bytes.strip_prefix(&[0xEF, 0xBB, 0xBF]) {
            Some(rest) => rest,
            None => bytes,
        };
        match self {
            Self::Utf8 => String::from_utf8(bytes.to_vec()).map_err(|error| Error::Parse {
                position: error.utf8_error().valid_up_to(),
                context: "source is not valid UTF-8",
            }),
            Self::Ascii | Self::AsciiUpper => {
                if let Some(position) = bytes.iter().position(|byte| !byte.is_ascii()) {
                    return Err(Error::Parse {
                        position,
                        context: "source contains a non-ASCII byte",
                    });
                }
                let value = String::from_utf8(bytes.to_vec()).map_err(|_| Error::Argument)?;
                Ok(if self == Self::AsciiUpper {
                    value.to_ascii_uppercase()
                } else {
                    value
                })
            }
            Self::Utf16Le | Self::Utf16Be => {
                let (encoding, bytes) = if let Some(bytes) = bytes.strip_prefix(&[0xFF, 0xFE]) {
                    (Self::Utf16Le, bytes)
                } else if let Some(bytes) = bytes.strip_prefix(&[0xFE, 0xFF]) {
                    (Self::Utf16Be, bytes)
                } else {
                    (self, bytes)
                };
                if bytes.len() % 2 != 0 {
                    return Err(Error::Parse {
                        position: bytes.len() - 1,
                        context: "UTF-16 source has a trailing byte",
                    });
                }
                let units = bytes
                    .chunks_exact(2)
                    .map(|pair| match encoding {
                        Self::Utf16Le => u16::from_le_bytes([pair[0], pair[1]]),
                        Self::Utf16Be => u16::from_be_bytes([pair[0], pair[1]]),
                        _ => unreachable!(),
                    })
                    .collect::<Vec<_>>();
                String::from_utf16(&units).map_err(|_| Error::Parse {
                    position: first_unpaired_surrogate(&units),
                    context: "source contains an unpaired UTF-16 surrogate",
                })
            }
        }
    }
}

fn first_unpaired_surrogate(units: &[u16]) -> usize {
    let mut index = 0;
    while index < units.len() {
        match units[index] {
            0xD800..=0xDBFF
                if units
                    .get(index + 1)
                    .is_some_and(|unit| (0xDC00..=0xDFFF).contains(unit)) =>
            {
                index += 2;
            }
            0xD800..=0xDFFF => return index,
            _ => index += 1,
        }
    }
    units.len()
}

/// Unicode character set used by generated lexer transitions.
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct Charset {
    pub(crate) category: DotNetUnicodeCategory,
    pub(crate) generic: u16,
    pub(crate) explicit: BTreeSet<u16>,
}

impl Charset {
    fn for_category(category: DotNetUnicodeCategory) -> Self {
        let generic = (0..=u16::MAX)
            .find(|unit| DotNetUnicodeCategory::of_unit(*unit) == category)
            .unwrap_or(0);
        Self {
            category,
            generic,
            explicit: BTreeSet::from([generic]),
        }
    }

    /// Resolves a source encoding and updates the C# uppercase compatibility flag.
    pub fn get_encoding(
        enc: String,
        toupper: &mut bool,
        mut erh: ErrorHandler,
    ) -> Result<TextEncoding, Error> {
        if let Ok(encoding) = InputEncoding::parse(&enc) {
            *toupper = encoding == InputEncoding::AsciiUpper;
            Ok(TextEncoding)
        } else {
            let diagnostic = CSToolsException::new_with_int32_string(
                43,
                format!("Warning: Encoding {enc} unknown: ignored"),
            )?;
            erh.error(diagnostic)?;
            *toupper = false;
            Ok(TextEncoding)
        }
    }

    /// Rejects the generator serializer boundary until its native table codec is
    /// available. Lexer tables use [`YyLexer::emit_dfa`] in this runtime.
    pub fn serialise(o: Object, _s: Serialiser) -> Result<Object, Error> {
        let _ = o;
        Err(Error::InvalidOperation)
    }

    /// Tests whether a UTF-16 unit belongs to this set.
    #[must_use]
    pub fn contains(&self, unit: u16) -> bool {
        self.explicit.contains(&unit) || DotNetUnicodeCategory::of_unit(unit) == self.category
    }
}

/// One removed comment segment, measured in UTF-16 code units.
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct CommentList {
    pub spos: i32,
    pub len: i32,
    pub tail: Option<Box<CommentList>>,
}

impl CommentList {
    pub fn new(st: i32, ln: i32, tail: CommentList) -> Result<Self, Error> {
        if st < 0 || ln < 0 {
            return Err(Error::Argument);
        }
        Ok(Self {
            spos: st,
            len: ln,
            tail: Some(Box::new(tail)),
        })
    }
}

/// Linked-list compatibility view of a source line.
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct LineList {
    pub head: i32,
    pub comments: Option<Box<CommentList>>,
    pub tail: Option<Box<LineList>>,
}

impl LineList {
    pub fn new(head: i32, tail: LineList) -> Result<Self, Error> {
        if head < 0 {
            return Err(Error::Argument);
        }
        Ok(Self {
            head,
            comments: None,
            tail: Some(Box::new(tail)),
        })
    }

    fn first(head: i32) -> Self {
        Self {
            head,
            comments: None,
            tail: None,
        }
    }

    pub fn getpos(&self, pos: i32) -> Result<i32, Error> {
        if pos < self.head {
            return Err(Error::IndexOutOfRange);
        }
        let mut result = pos - self.head;
        let mut comment = self.comments.as_deref();
        while let Some(item) = comment {
            if pos > item.spos {
                result = result.checked_add(item.len).ok_or(Error::IndexOutOfRange)?;
            }
            comment = item.tail.as_deref();
        }
        Ok(result)
    }
}

/// Tracks filtered and raw line positions.
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct LineManager {
    pub end: i32,
    pub lines: i32,
    pub list: Option<Box<LineList>>,
    starts: Vec<i32>,
    logical_lines: Vec<i32>,
}

impl Default for LineManager {
    fn default() -> Self {
        Self {
            end: 0,
            lines: 1,
            list: None,
            starts: vec![0],
            logical_lines: vec![1],
        }
    }
}

impl LineManager {
    pub fn new() -> Result<Self, Error> {
        Ok(Self::default())
    }

    pub fn newline(&mut self, pos: i32) -> Result<(), Error> {
        self.backto(pos)?;
        self.lines = self.lines.checked_add(1).ok_or(Error::IndexOutOfRange)?;
        self.starts.push(pos);
        self.logical_lines.push(self.lines);
        let previous = self.list.take();
        self.list = Some(Box::new(LineList {
            head: pos,
            comments: None,
            tail: previous,
        }));
        Ok(())
    }

    pub fn backto(&mut self, pos: i32) -> Result<(), Error> {
        if pos < 0 {
            return Err(Error::Argument);
        }
        self.end = self.end.max(pos);
        while self.starts.len() > 1 && self.starts.last().is_some_and(|start| *start >= pos) {
            self.starts.pop();
            self.logical_lines.pop();
            self.lines -= 1;
            self.list = self.list.take().and_then(|line| line.tail);
        }
        Ok(())
    }

    pub fn comment(&mut self, pos: i32, len: i32) -> Result<(), Error> {
        if pos < 0 || len < 0 {
            return Err(Error::Argument);
        }
        self.end = self.end.max(pos);
        if self.list.is_none() {
            self.list = Some(Box::new(LineList::first(0)));
            self.lines = 1;
        }
        if let Some(line) = self.list.as_mut() {
            let tail = line.comments.take();
            line.comments = Some(Box::new(CommentList {
                spos: pos,
                len,
                tail,
            }));
        }
        Ok(())
    }

    fn set_logical_line(&mut self, line: i32) -> Result<(), Error> {
        if line < 1 {
            return Err(Error::Argument);
        }
        self.lines = line - 1;
        if let Some(last) = self.logical_lines.last_mut() {
            *last = line - 1;
        }
        Ok(())
    }

    fn locate(&self, pos: i32) -> Result<(usize, i32, i32, i32), Error> {
        if pos < 0 || pos > self.end {
            return Err(Error::IndexOutOfRange);
        }
        let line_index = self
            .starts
            .partition_point(|start| *start <= pos)
            .saturating_sub(1);
        let start = self.starts[line_index];
        let end = self.starts.get(line_index + 1).copied().unwrap_or(self.end);
        let line = self
            .logical_lines
            .get(line_index)
            .copied()
            .unwrap_or_else(|| i32::try_from(line_index + 1).unwrap_or(i32::MAX));
        Ok((line_index, line, start, end))
    }
}

/// Source location compatible with `SourceLineInfo`.
#[derive(Clone, Debug, Default)]
pub struct SourceLineInfo {
    pub char_position: i32,
    pub end_of_line: i32,
    pub line_number: i32,
    pub lxr: Option<Box<Lexer>>,
    pub raw_char_position: i32,
    pub start_of_line: i32,
    source_snapshot: String,
}

impl SourceLineInfo {
    pub fn new_with_int32(pos: i32) -> Result<Self, Error> {
        if pos < 0 {
            return Err(Error::Argument);
        }
        Ok(Self {
            char_position: 1,
            end_of_line: pos,
            line_number: 1,
            lxr: None,
            raw_char_position: pos,
            start_of_line: 0,
            source_snapshot: String::new(),
        })
    }

    pub fn new_with_line_manager_int32(manager: LineManager, pos: i32) -> Result<Self, Error> {
        Self::from_manager(&manager, pos, String::new())
    }

    pub fn new_with_lexer_int32(lexer: Lexer, pos: i32) -> Result<Self, Error> {
        Self::from_manager(&lexer.m_line_manager, pos, lexer.m_buf.clone())
    }

    fn from_manager(manager: &LineManager, pos: i32, source: String) -> Result<Self, Error> {
        let (line_index, line_number, start, end) = manager.locate(pos)?;
        let mut raw = pos - start + i32::from(line_index > 0);
        let mut line = manager.list.as_deref();
        while let Some(candidate) = line {
            if candidate.head == start {
                raw = candidate
                    .getpos(pos)?
                    .checked_add(i32::from(line_index > 0))
                    .ok_or(Error::IndexOutOfRange)?;
                break;
            }
            line = candidate.tail.as_deref();
        }
        Ok(Self {
            char_position: pos - start + 1,
            end_of_line: end,
            line_number,
            lxr: None,
            raw_char_position: raw,
            start_of_line: start,
            source_snapshot: source,
        })
    }

    #[must_use]
    pub fn source_line(&self) -> String {
        if self.source_snapshot.is_empty() {
            return String::new();
        }
        let units = self.source_snapshot.encode_utf16().collect::<Vec<_>>();
        let start = usize::try_from(self.start_of_line)
            .unwrap_or(0)
            .min(units.len());
        let end = usize::try_from(self.end_of_line)
            .unwrap_or(units.len())
            .min(units.len());
        units_to_string(&units[start..end])
            .trim_end_matches('\n')
            .to_owned()
    }

    #[must_use]
    pub fn to_string(&self) -> String {
        format!("Line {}, char {}", self.line_number, self.raw_char_position)
    }
}

impl fmt::Display for SourceLineInfo {
    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
        formatter.write_str(&self.to_string())
    }
}

/// Stable diagnostic categories emitted by the lexer runtime.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum DiagnosticCategory {
    Encoding,
    InvalidCharacter,
    InvalidState,
    ParserRecovery,
    ParserStackLimit,
    Source,
    Syntax,
    TokenTooLong,
    UnknownCharacterSet,
    UnexpectedEof,
}

/// Diagnostic severity.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum DiagnosticSeverity {
    Warning,
    Error,
    Fatal,
    Stop,
}

/// Structured diagnostic suitable for deterministic fixture assertions.
#[derive(Clone, Debug)]
pub struct Diagnostic {
    pub code: i32,
    pub category: DiagnosticCategory,
    pub severity: DiagnosticSeverity,
    pub message: String,
    pub location: SourceLineInfo,
    pub input: String,
}

/// C# compatibility exception carrying a structured native diagnostic.
#[derive(Debug)]
pub struct CSToolsException {
    pub handled: bool,
    pub n_exception_number: i32,
    pub s_input: String,
    pub sl_info: SourceLineInfo,
    pub sym: Option<Box<SYMBOL>>,
    pub message: String,
    severity: DiagnosticSeverity,
    category: DiagnosticCategory,
}

impl Clone for CSToolsException {
    fn clone(&self) -> Self {
        Self {
            handled: self.handled,
            n_exception_number: self.n_exception_number,
            s_input: self.s_input.clone(),
            sl_info: self.sl_info.clone(),
            sym: None,
            message: self.message.clone(),
            severity: self.severity,
            category: self.category,
        }
    }
}

impl CSToolsException {
    fn category_for_code(number: i32) -> DiagnosticCategory {
        match number {
            2 => DiagnosticCategory::InvalidState,
            24 => DiagnosticCategory::UnknownCharacterSet,
            43 => DiagnosticCategory::Encoding,
            _ => DiagnosticCategory::Source,
        }
    }

    fn create(
        number: i32,
        location: SourceLineInfo,
        input: String,
        message: String,
        severity: DiagnosticSeverity,
        category: DiagnosticCategory,
    ) -> Self {
        Self {
            handled: false,
            n_exception_number: number,
            s_input: input,
            sl_info: location,
            sym: None,
            message,
            severity,
            category,
        }
    }

    pub fn new_with_int32_string(number: i32, message: String) -> Result<Self, Error> {
        Ok(Self::create(
            number,
            SourceLineInfo::new_with_int32(0)?,
            String::new(),
            message,
            DiagnosticSeverity::Error,
            Self::category_for_code(number),
        ))
    }

    pub fn new_with_int32_source_line_info_string_string(
        number: i32,
        location: SourceLineInfo,
        input: String,
        message: String,
    ) -> Result<Self, Error> {
        Ok(Self::create(
            number,
            location,
            input,
            message,
            DiagnosticSeverity::Error,
            Self::category_for_code(number),
        ))
    }

    pub fn new_with_int32_lexer_int32_string_string(
        number: i32,
        lexer: Lexer,
        pos: i32,
        input: String,
        message: String,
    ) -> Result<Self, Error> {
        let location = lexer.source_line_info(pos)?;
        Ok(Self::create(
            number,
            location,
            input,
            message,
            DiagnosticSeverity::Error,
            Self::category_for_code(number),
        ))
    }

    pub fn new_with_int32_lexer_string(
        number: i32,
        lexer: Lexer,
        message: String,
    ) -> Result<Self, Error> {
        let pos = lexer.yypos();
        Self::new_with_int32_lexer_int32_string_string(number, lexer, pos, String::new(), message)
    }

    pub fn new_with_int32_lexer_string_string(
        number: i32,
        lexer: Lexer,
        input: String,
        message: String,
    ) -> Result<Self, Error> {
        let pos = lexer.yypos();
        Self::new_with_int32_lexer_int32_string_string(number, lexer, pos, input, message)
    }

    pub fn new_with_int32_symbol_string(
        number: i32,
        symbol: SYMBOL,
        message: String,
    ) -> Result<Self, Error> {
        let location = symbol.location()?;
        let mut result = Self::create(
            number,
            location,
            symbol.yyname(),
            message,
            DiagnosticSeverity::Error,
            Self::category_for_code(number),
        );
        result.sym = Some(Box::new(symbol));
        Ok(result)
    }

    pub fn new_with_int32_token_string(
        number: i32,
        token: TOKEN,
        message: String,
    ) -> Result<Self, Error> {
        Ok(Self::create(
            number,
            token.location()?,
            token.yytext(),
            message,
            DiagnosticSeverity::Error,
            Self::category_for_code(number),
        ))
    }

    pub fn handle(&mut self, handler: ErrorHandler) -> Result<(), Error> {
        if handler.throw_exceptions {
            return Err(Error::InvalidOperation);
        }
        self.handled = true;
        Ok(())
    }

    fn diagnostic(&self) -> Diagnostic {
        Diagnostic {
            code: self.n_exception_number,
            category: self.category,
            severity: self.severity,
            message: self.message.clone(),
            location: self.sl_info.clone(),
            input: self.s_input.clone(),
        }
    }
}

impl fmt::Display for CSToolsException {
    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
        write!(formatter, "{}: {}", self.sl_info, self.message)
    }
}

impl std::error::Error for CSToolsException {}

macro_rules! exception_wrapper {
    ($name:ident, $severity:expr) => {
        #[derive(Clone, Debug)]
        pub struct $name(pub CSToolsException);

        impl $name {
            pub fn new_with_int32_string(number: i32, message: String) -> Result<Self, Error> {
                let mut inner = CSToolsException::new_with_int32_string(number, message)?;
                inner.severity = $severity;
                Ok(Self(inner))
            }

            pub fn new_with_int32_source_line_info_string_string(
                number: i32,
                location: SourceLineInfo,
                input: String,
                message: String,
            ) -> Result<Self, Error> {
                let mut inner = CSToolsException::new_with_int32_source_line_info_string_string(
                    number, location, input, message,
                )?;
                inner.severity = $severity;
                Ok(Self(inner))
            }

            pub fn new_with_int32_lexer_int32_string_string(
                number: i32,
                lexer: Lexer,
                pos: i32,
                input: String,
                message: String,
            ) -> Result<Self, Error> {
                let mut inner = CSToolsException::new_with_int32_lexer_int32_string_string(
                    number, lexer, pos, input, message,
                )?;
                inner.severity = $severity;
                Ok(Self(inner))
            }

            pub fn new_with_int32_lexer_string(
                number: i32,
                lexer: Lexer,
                message: String,
            ) -> Result<Self, Error> {
                let mut inner =
                    CSToolsException::new_with_int32_lexer_string(number, lexer, message)?;
                inner.severity = $severity;
                Ok(Self(inner))
            }

            pub fn new_with_int32_lexer_string_string(
                number: i32,
                lexer: Lexer,
                input: String,
                message: String,
            ) -> Result<Self, Error> {
                let mut inner = CSToolsException::new_with_int32_lexer_string_string(
                    number, lexer, input, message,
                )?;
                inner.severity = $severity;
                Ok(Self(inner))
            }

            pub fn new_with_int32_token_string(
                number: i32,
                token: TOKEN,
                message: String,
            ) -> Result<Self, Error> {
                let mut inner =
                    CSToolsException::new_with_int32_token_string(number, token, message)?;
                inner.severity = $severity;
                Ok(Self(inner))
            }

            pub fn new_with_int32_symbol_string(
                number: i32,
                symbol: SYMBOL,
                message: String,
            ) -> Result<Self, Error> {
                let mut inner =
                    CSToolsException::new_with_int32_symbol_string(number, symbol, message)?;
                inner.severity = $severity;
                Ok(Self(inner))
            }

            pub fn handle(&mut self, _handler: ErrorHandler) -> Result<(), Error> {
                Err(Error::InvalidOperation)
            }
        }
    };
}

exception_wrapper!(CSToolsFatalException, DiagnosticSeverity::Fatal);
exception_wrapper!(CSToolsStopException, DiagnosticSeverity::Stop);

/// Collecting error handler. No output is written implicitly.
#[derive(Clone, Debug, Default)]
pub struct ErrorHandler {
    pub counter: i32,
    pub throw_exceptions: bool,
    diagnostics: Vec<Diagnostic>,
}

impl ErrorHandler {
    pub fn new_with_constructor() -> Result<Self, Error> {
        Ok(Self::default())
    }

    pub fn new_with_boolean(throw_exceptions: bool) -> Result<Self, Error> {
        Ok(Self {
            throw_exceptions,
            ..Self::default()
        })
    }

    pub fn error(&mut self, mut error: CSToolsException) -> Result<(), Error> {
        self.counter = self.counter.checked_add(1).ok_or(Error::IndexOutOfRange)?;
        if self.throw_exceptions {
            return Err(Error::InvalidOperation);
        }
        self.report(error.clone())?;
        error.handled = true;
        Ok(())
    }

    pub fn report(&mut self, error: CSToolsException) -> Result<(), Error> {
        self.diagnostics.push(error.diagnostic());
        Ok(())
    }

    #[must_use]
    pub fn diagnostics(&self) -> &[Diagnostic] {
        &self.diagnostics
    }

    pub fn take_diagnostics(&mut self) -> Vec<Diagnostic> {
        std::mem::take(&mut self.diagnostics)
    }

    pub(crate) fn push(&mut self, diagnostic: Diagnostic) -> Result<(), Error> {
        self.counter = self.counter.checked_add(1).ok_or(Error::IndexOutOfRange)?;
        if self.throw_exceptions {
            return Err(Error::InvalidOperation);
        }
        self.diagnostics.push(diagnostic);
        Ok(())
    }
}

/// Comment-filtering source reader corresponding to the C# `CsReader`.
#[derive(Clone, Debug)]
pub struct CsReader {
    pub fname: String,
    pub lm: LineManager,
    units: Vec<u16>,
    cursor: usize,
}

impl CsReader {
    pub fn new_with_string(data: String) -> Result<Self, Error> {
        Self::from_source(data, String::new())
    }

    pub fn new_with_string_encoding(
        file_name: String,
        _encoding: TextEncoding,
    ) -> Result<Self, Error> {
        let bytes = std::fs::read(&file_name).map_err(|_| Error::Argument)?;
        let encoding = if bytes.starts_with(&[0xFF, 0xFE]) {
            InputEncoding::Utf16Le
        } else if bytes.starts_with(&[0xFE, 0xFF]) {
            InputEncoding::Utf16Be
        } else {
            InputEncoding::Utf8
        };
        Self::from_source(encoding.decode(&bytes)?, file_name)
    }

    pub fn new_with_cs_reader_encoding(
        reader: CsReader,
        _encoding: TextEncoding,
    ) -> Result<Self, Error> {
        Ok(reader)
    }

    /// Creates a reader from encoded bytes without a platform-specific API.
    pub fn from_bytes(
        bytes: &[u8],
        encoding: InputEncoding,
        file_name: impl Into<String>,
    ) -> Result<Self, Error> {
        Self::from_source(encoding.decode(bytes)?, file_name.into())
    }

    fn from_source(source: String, file_name: String) -> Result<Self, Error> {
        let input = source.encode_utf16().collect::<Vec<_>>();
        if input.len() > MAX_SOURCE_UNITS {
            return Err(Error::Argument);
        }
        let (units, lm, directive_name) = filter_source(&input)?;
        Ok(Self {
            fname: directive_name.unwrap_or(file_name),
            lm,
            units,
            cursor: 0,
        })
    }

    pub fn eof(&self) -> Result<bool, Error> {
        Ok(self.cursor >= self.units.len())
    }

    pub fn read_with_method(&mut self) -> Result<i32, Error> {
        let Some(unit) = self.units.get(self.cursor).copied() else {
            return Ok(-1);
        };
        self.cursor += 1;
        Ok(i32::from(unit))
    }

    pub fn read_with_char_array_int32_int32(
        &mut self,
        mut destination: Vec<Utf16CodeUnit>,
        offset: i32,
        count: i32,
    ) -> Result<i32, Error> {
        let offset = index(offset)?;
        let count = index(count)?;
        let end = offset.checked_add(count).ok_or(Error::IndexOutOfRange)?;
        if end > destination.len() {
            return Err(Error::IndexOutOfRange);
        }
        let copied = self.read_into(&mut destination[offset..end]);
        i32_len(copied)
    }

    pub fn read_into(&mut self, destination: &mut [Utf16CodeUnit]) -> usize {
        let count = destination
            .len()
            .min(self.units.len().saturating_sub(self.cursor));
        for (slot, unit) in destination
            .iter_mut()
            .zip(&self.units[self.cursor..self.cursor + count])
        {
            *slot = Utf16CodeUnit(*unit);
        }
        self.cursor += count;
        count
    }

    pub fn read_line(&mut self) -> Result<String, Error> {
        if self.cursor >= self.units.len() {
            return Ok(String::new());
        }
        let start = self.cursor;
        while self.cursor < self.units.len() && self.units[self.cursor] != b'\n'.into() {
            self.cursor += 1;
        }
        let line = units_to_string(&self.units[start..self.cursor]);
        if self.cursor < self.units.len() {
            self.cursor += 1;
        }
        Ok(line)
    }

    fn into_parts(self) -> (String, Vec<u16>, LineManager) {
        (self.fname, self.units, self.lm)
    }
}

fn filter_source(input: &[u16]) -> Result<(Vec<u16>, LineManager, Option<String>), Error> {
    #[derive(Clone, Copy)]
    enum State {
        Normal,
        Slash,
        LineComment { removed: usize },
        BlockComment { removed: usize },
        BlockStar { removed: usize },
    }

    let mut output = Vec::with_capacity(input.len());
    let mut lm = LineManager::default();
    let mut state = State::Normal;
    let mut at_line_start = true;
    let mut directive_name = None;
    let mut i = 0;
    while i < input.len() {
        let unit = input[i];
        let normalized = if unit == b'\r'.into() {
            if input.get(i + 1) == Some(&u16::from(b'\n')) {
                i += 1;
            }
            u16::from(b'\n')
        } else {
            unit
        };

        if matches!(state, State::Normal) && at_line_start && normalized == u16::from(b'#') {
            let start = i;
            let mut end = i;
            while end < input.len()
                && input[end] != u16::from(b'\n')
                && input[end] != u16::from(b'\r')
            {
                end += 1;
            }
            let directive = units_to_string(&input[start..end]);
            if let Some((line, name)) = parse_line_directive(&directive) {
                if let Some(name) = name {
                    directive_name = Some(name);
                }
                let removed = end - start;
                lm.comment(i32_len(output.len())?, i32_len(removed)?)?;
                lm.set_logical_line(line)?;
                i = end;
                continue;
            }
        }

        match state {
            State::Normal => {
                if normalized == u16::from(b'/') {
                    state = State::Slash;
                } else {
                    output.push(normalized);
                    if normalized == u16::from(b'\n') {
                        lm.newline(i32_len(output.len())?)?;
                        at_line_start = true;
                    } else {
                        at_line_start = false;
                    }
                }
            }
            State::Slash => {
                if normalized == u16::from(b'/') {
                    state = State::LineComment { removed: 2 };
                } else if normalized == u16::from(b'*') {
                    state = State::BlockComment { removed: 2 };
                } else {
                    output.push(u16::from(b'/'));
                    output.push(normalized);
                    at_line_start = false;
                    state = State::Normal;
                }
            }
            State::LineComment { removed } => {
                if normalized == u16::from(b'\n') {
                    lm.comment(i32_len(output.len())?, i32_len(removed)?)?;
                    output.push(normalized);
                    lm.newline(i32_len(output.len())?)?;
                    at_line_start = true;
                    state = State::Normal;
                } else {
                    state = State::LineComment {
                        removed: removed + 1,
                    };
                }
            }
            State::BlockComment { removed } => {
                if normalized == u16::from(b'*') {
                    state = State::BlockStar {
                        removed: removed + 1,
                    };
                } else if normalized == u16::from(b'\n') {
                    lm.comment(i32_len(output.len())?, i32_len(removed)?)?;
                    output.push(normalized);
                    lm.newline(i32_len(output.len())?)?;
                    at_line_start = true;
                    state = State::BlockComment { removed: 0 };
                } else {
                    state = State::BlockComment {
                        removed: removed + 1,
                    };
                }
            }
            State::BlockStar { removed } => {
                if normalized == u16::from(b'/') {
                    lm.comment(i32_len(output.len())?, i32_len(removed + 1)?)?;
                    state = State::Normal;
                } else if normalized == u16::from(b'*') {
                    state = State::BlockStar {
                        removed: removed + 1,
                    };
                } else if normalized == u16::from(b'\n') {
                    lm.comment(i32_len(output.len())?, i32_len(removed)?)?;
                    output.push(normalized);
                    lm.newline(i32_len(output.len())?)?;
                    at_line_start = true;
                    state = State::BlockComment { removed: 0 };
                } else {
                    state = State::BlockComment {
                        removed: removed + 1,
                    };
                }
            }
        }
        i += 1;
    }

    match state {
        State::Normal => {}
        State::Slash => output.push(u16::from(b'/')),
        State::LineComment { removed } => {
            lm.comment(i32_len(output.len())?, i32_len(removed)?)?;
        }
        State::BlockComment { .. } | State::BlockStar { .. } => {
            return Err(Error::Parse {
                position: input.len(),
                context: "unterminated block comment",
            });
        }
    }
    lm.end = i32_len(output.len())?;
    Ok((output, lm, directive_name))
}

fn parse_line_directive(value: &str) -> Option<(i32, Option<String>)> {
    let value = value.strip_prefix('#')?.trim_start();
    let value = value.strip_prefix("line")?.trim_start();
    let (number, rest) = value.split_once(char::is_whitespace).unwrap_or((value, ""));
    let line = number.parse::<i32>().ok()?;
    if line < 1 {
        return None;
    }
    let rest = rest.trim();
    let name = rest
        .strip_prefix('"')
        .and_then(|quoted| quoted.strip_suffix('"'))
        .map(ToOwned::to_owned);
    Some((line, name))
}

/// A character predicate on one UTF-16 code unit.
#[derive(Clone, Debug, Eq, PartialEq)]
pub enum CharacterMatcher {
    Any,
    Category(DotNetUnicodeCategory),
    Categories(Vec<DotNetUnicodeCategory>),
    UnicodeClass(UnicodeClass),
    Eof,
    Exact(u16),
    Range { first: u16, last: u16 },
    Set(BTreeSet<u16>),
}

impl CharacterMatcher {
    fn matches(&self, value: Option<u16>) -> bool {
        match (self, value) {
            (Self::Eof, None) | (Self::Any, Some(_)) => true,
            (Self::Category(category), Some(unit)) => {
                DotNetUnicodeCategory::of_unit(unit) == *category
            }
            (Self::Categories(categories), Some(unit)) => {
                categories.contains(&DotNetUnicodeCategory::of_unit(unit))
            }
            (Self::UnicodeClass(class), Some(unit)) => class.matches(unit),
            (Self::Exact(expected), Some(unit)) => unit == *expected,
            (Self::Range { first, last }, Some(unit)) => (*first..=*last).contains(&unit),
            (Self::Set(values), Some(unit)) => values.contains(&unit),
            _ => false,
        }
    }

    fn stable_text(&self) -> String {
        match self {
            Self::Any => "any".to_owned(),
            Self::Category(category) => format!("category:{}", category.name()),
            Self::Categories(categories) => format!(
                "categories:{}",
                categories
                    .iter()
                    .map(|category| category.name())
                    .collect::<Vec<_>>()
                    .join(",")
            ),
            Self::UnicodeClass(class) => format!("class:{class:?}"),
            Self::Eof => "eof".to_owned(),
            Self::Exact(unit) => format!("exact:{unit:04X}"),
            Self::Range { first, last } => format!("range:{first:04X}-{last:04X}"),
            Self::Set(values) => {
                let values = values
                    .iter()
                    .map(|value| format!("{value:04X}"))
                    .collect::<Vec<_>>();
                format!("set:{}", values.join(","))
            }
        }
    }
}

/// Named character predicates accepted by the original lexer specification.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum UnicodeClass {
    Symbol,
    Punctuation,
    Separator,
    WhiteSpace,
    Number,
    Digit,
    Letter,
    Lower,
    Upper,
}

impl UnicodeClass {
    fn matches(self, unit: u16) -> bool {
        let category = DotNetUnicodeCategory::of_unit(unit);
        match self {
            Self::Symbol => matches!(
                category,
                DotNetUnicodeCategory::MathSymbol
                    | DotNetUnicodeCategory::CurrencySymbol
                    | DotNetUnicodeCategory::ModifierSymbol
                    | DotNetUnicodeCategory::OtherSymbol
            ),
            Self::Punctuation => matches!(
                category,
                DotNetUnicodeCategory::ConnectorPunctuation
                    | DotNetUnicodeCategory::DashPunctuation
                    | DotNetUnicodeCategory::OpenPunctuation
                    | DotNetUnicodeCategory::ClosePunctuation
                    | DotNetUnicodeCategory::InitialQuotePunctuation
                    | DotNetUnicodeCategory::FinalQuotePunctuation
                    | DotNetUnicodeCategory::OtherPunctuation
            ),
            Self::Separator => matches!(
                category,
                DotNetUnicodeCategory::SpaceSeparator
                    | DotNetUnicodeCategory::LineSeparator
                    | DotNetUnicodeCategory::ParagraphSeparator
            ),
            Self::WhiteSpace => {
                matches!(unit, 0x0009..=0x000D | 0x0085) || Self::Separator.matches(unit)
            }
            Self::Number => matches!(
                category,
                DotNetUnicodeCategory::DecimalDigitNumber
                    | DotNetUnicodeCategory::LetterNumber
                    | DotNetUnicodeCategory::OtherNumber
            ),
            Self::Digit => category == DotNetUnicodeCategory::DecimalDigitNumber,
            Self::Letter => matches!(
                category,
                DotNetUnicodeCategory::UppercaseLetter
                    | DotNetUnicodeCategory::LowercaseLetter
                    | DotNetUnicodeCategory::TitlecaseLetter
                    | DotNetUnicodeCategory::ModifierLetter
                    | DotNetUnicodeCategory::OtherLetter
            ),
            Self::Lower => category == DotNetUnicodeCategory::LowercaseLetter,
            Self::Upper => category == DotNetUnicodeCategory::UppercaseLetter,
        }
    }
}

/// Lexer action attached to an accepting DFA state.
#[derive(Clone, Debug, Eq, PartialEq)]
pub enum LexerAction {
    Emit,
    EmitAndBegin(String),
    Skip,
    SkipAndBegin(String),
}

/// Token metadata stored in a deterministic lexer table.
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct TokenDefinition {
    pub name: String,
    pub number: i32,
}

impl TokenDefinition {
    pub fn new(name: impl Into<String>, number: i32) -> Result<Self, Error> {
        let name = name.into();
        if name.is_empty() || number < 0 {
            return Err(Error::Argument);
        }
        Ok(Self { name, number })
    }
}

/// Accepting metadata for a DFA state.
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct DfaAccept {
    pub token: TokenDefinition,
    pub action: LexerAction,
    pub action_number: i32,
    pub reserved_words: Option<String>,
}

/// One deterministic automaton state.
#[derive(Clone, Debug, Default, Eq, PartialEq)]
pub struct DfaState {
    pub transitions: Vec<(CharacterMatcher, usize)>,
    pub accept: Option<DfaAccept>,
}

impl DfaState {
    #[must_use]
    pub fn transition(mut self, matcher: CharacterMatcher, target: usize) -> Self {
        self.transitions.push((matcher, target));
        self
    }

    #[must_use]
    pub fn accepting(mut self, accept: DfaAccept) -> Self {
        self.accept = Some(accept);
        self
    }
}

/// Compatibility linked action.
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct DfaAction {
    pub a_act: i32,
    pub a_next: Option<Box<DfaAction>>,
}

impl DfaAction {
    pub fn new(action: i32, next: DfaAction) -> Result<Self, Error> {
        Ok(Self {
            a_act: action,
            a_next: Some(Box::new(next)),
        })
    }

    pub fn serialise(o: Object, _s: Serialiser) -> Result<Object, Error> {
        let _ = o;
        Err(Error::InvalidOperation)
    }
}

/// Runtime deterministic finite automaton.
#[derive(Clone, Debug)]
struct RegexDfa {
    source: String,
    compiled: regex::Regex,
    accept: DfaAccept,
}

impl PartialEq for RegexDfa {
    fn eq(&self, other: &Self) -> bool {
        self.source == other.source && self.accept == other.accept
    }
}

impl Eq for RegexDfa {}

#[derive(Clone, Debug, Eq, PartialEq)]
pub struct Dfa {
    pub m_actions: Vec<DfaAction>,
    pub m_map: Hashtable,
    pub m_reswds: i32,
    pub m_tok_class: String,
    states: Vec<DfaState>,
    start: usize,
    regex: Option<RegexDfa>,
}

impl Default for Dfa {
    fn default() -> Self {
        Self {
            m_actions: Vec::new(),
            m_map: Hashtable::default(),
            m_reswds: -1,
            m_tok_class: String::new(),
            states: vec![DfaState::default()],
            start: 0,
            regex: None,
        }
    }
}

impl Dfa {
    pub fn from_states(states: Vec<DfaState>, start: usize) -> Result<Self, Error> {
        if states.is_empty() || start >= states.len() {
            return Err(Error::Argument);
        }
        if states
            .iter()
            .flat_map(|state| &state.transitions)
            .any(|(_, target)| *target >= states.len())
        {
            return Err(Error::Argument);
        }
        Ok(Self {
            states,
            start,
            ..Self::default()
        })
    }

    pub fn new_with_nfa(nfa: Nfa) -> Result<Self, Error> {
        let name = if nfa.m_end.m_s_terminal.is_empty() {
            "ANY".to_owned()
        } else {
            nfa.m_end.m_s_terminal.clone()
        };
        let accept = DfaAccept {
            token: TokenDefinition::new(name, 7)?,
            action: LexerAction::Emit,
            action_number: 0,
            reserved_words: None,
        };
        if let Some(source) = nfa.regex_source()? {
            let compiled =
                regex::Regex::new(&format!("^(?:{source})")).map_err(|_| Error::Parse {
                    position: 0,
                    context: "invalid NFA regular expression",
                })?;
            return Ok(Self {
                regex: Some(RegexDfa {
                    source,
                    compiled,
                    accept,
                }),
                ..Self::default()
            });
        }
        Self::from_states(nfa.deterministic_states(accept)?, 0)
    }

    pub fn new_with_tokens_gen(_tokens: TokensGen) -> Result<Self, Error> {
        // The C# constructor creates the initial, unpopulated determinisation
        // node. NFA-based construction is exposed separately by
        // `new_with_nfa`; returning an empty native state here preserves that
        // useful distinction without manufacturing an NFA transition.
        Ok(Self::default())
    }

    fn longest_match(&self, units: &[u16], offset: usize) -> Option<(usize, DfaAccept)> {
        if let Some(regex) = &self.regex {
            let text = String::from_utf16(units.get(offset..)?).ok()?;
            let found = regex.compiled.find(&text)?;
            if found.start() != 0 || found.end() == 0 {
                return None;
            }
            let length = text[..found.end()].encode_utf16().count();
            return Some((length, regex.accept.clone()));
        }
        let mut state_index = self.start;
        let mut cursor = offset;
        let mut accepted = self.states[state_index]
            .accept
            .clone()
            .map(|accept| (0, accept));
        loop {
            let value = units.get(cursor).copied();
            let next = self.states[state_index]
                .transitions
                .iter()
                .find(|(matcher, _)| matcher.matches(value))
                .map(|(_, target)| *target);
            let Some(next) = next else {
                break;
            };
            // EOF transitions do not consume a code unit.
            if value.is_some() {
                cursor += 1;
            }
            state_index = next;
            if let Some(accept) = self.states[state_index].accept.clone() {
                accepted = Some((cursor - offset, accept));
            }
            if value.is_none() {
                break;
            }
        }
        accepted
    }

    pub fn match_(&self, value: String, offset: i32, action: &mut i32) -> Result<i32, Error> {
        let units = value.encode_utf16().collect::<Vec<_>>();
        let offset = index(offset)?;
        if offset > units.len() {
            return Err(Error::IndexOutOfRange);
        }
        let Some((length, accept)) = self.longest_match(&units, offset) else {
            return Ok(-1);
        };
        *action = accept.action_number;
        i32_len(length)
    }

    pub fn print(&self) -> Result<(), Error> {
        println!("{}", self.stable_description());
        Ok(())
    }

    #[must_use]
    pub fn stable_description(&self) -> String {
        if let Some(regex) = &self.regex {
            return format!(
                "regex {:?} accept {} {} {}",
                regex.source,
                regex.accept.token.number,
                regex.accept.token.name,
                regex.accept.action_number
            );
        }
        let mut lines = Vec::new();
        for (state_index, state) in self.states.iter().enumerate() {
            if let Some(accept) = &state.accept {
                lines.push(format!(
                    "state {state_index} accept {} {} {}",
                    accept.token.number, accept.token.name, accept.action_number
                ));
            }
            for (matcher, target) in &state.transitions {
                lines.push(format!(
                    "state {state_index} {} -> {target}",
                    matcher.stable_text()
                ));
            }
        }
        lines.join("\n")
    }

    pub fn serialise(o: Object, _s: Serialiser) -> Result<Object, Error> {
        let _ = o;
        Err(Error::InvalidOperation)
    }

    pub fn set_tokens(_tokens: YyLexer, _starts: Hashtable) -> Result<(), Error> {
        Err(Error::InvalidOperation)
    }
}

/// Reserved-word remapping for a token class.
#[derive(Clone, Debug, Default, Eq, PartialEq)]
pub struct ResWds {
    pub m_wds: BTreeMap<String, TokenDefinition>,
    pub m_upper: bool,
}

impl ResWds {
    pub fn new_with_constructor() -> Result<Self, Error> {
        Ok(Self::default())
    }

    pub fn new_with_tokens_gen_string(
        tokens: TokensGen,
        specification: String,
    ) -> Result<Self, Error> {
        let mut specification = specification.trim();
        let uppercase = specification.starts_with('U');
        if uppercase {
            specification = specification[1..].trim_start();
        }
        let Some(body) = specification
            .strip_prefix('{')
            .and_then(|value| value.strip_suffix('}'))
        else {
            return Err(Error::Parse {
                position: 0,
                context: "bad ResWds element",
            });
        };
        let mut pairs = Vec::new();
        for item in body.split(',') {
            let mut fields = item.split_whitespace();
            let Some(word) = fields.next() else {
                continue;
            };
            let name = fields.next().unwrap_or(word);
            if fields.next().is_some() {
                return Err(Error::Parse {
                    position: specification.find(item).unwrap_or(0),
                    context: "reserved word has too many fields",
                });
            }
            let key = Object::String(name.to_owned());
            let Some(value) = tokens.m_tokens.tokens.0.get(&key) else {
                // Creating a missing token class mutates generator state and is
                // deliberately owned by issue #83. Never invent its number.
                return Err(Error::InvalidOperation);
            };
            let (token_name, token_number) = match value {
                Object::Integer(number) => (name.to_owned(), *number),
                Object::Map(metadata) => {
                    let token_name = match metadata.get("name") {
                        Some(Object::String(value)) => value.clone(),
                        None => name.to_owned(),
                        _ => return Err(Error::InvalidOperation),
                    };
                    let Some(Object::Integer(number)) = metadata.get("number") else {
                        return Err(Error::InvalidOperation);
                    };
                    (token_name, *number)
                }
                _ => return Err(Error::InvalidOperation),
            };
            pairs.push((
                word.to_owned(),
                TokenDefinition::new(token_name, token_number)?,
            ));
        }
        Self::from_pairs(pairs, uppercase)
    }

    pub fn from_pairs<I, K>(pairs: I, uppercase: bool) -> Result<Self, Error>
    where
        I: IntoIterator<Item = (K, TokenDefinition)>,
        K: Into<String>,
    {
        let mut words = BTreeMap::new();
        for (word, token) in pairs {
            let word = word.into();
            if word.is_empty() {
                return Err(Error::Argument);
            }
            let key = if uppercase { word.to_uppercase() } else { word };
            words.insert(key, token);
        }
        Ok(Self {
            m_wds: words,
            m_upper: uppercase,
        })
    }

    fn remap(&self, text: &str) -> Option<TokenDefinition> {
        let key = if self.m_upper {
            text.to_uppercase()
        } else {
            text.to_owned()
        };
        self.m_wds.get(&key).cloned()
    }

    pub fn check(&self, _lexer: Lexer, token: &mut TOKEN) -> Result<(), Error> {
        if let Some(definition) = self.remap(&token.text) {
            token.name = definition.name;
            token.number = definition.number;
        }
        Ok(())
    }

    pub fn serialise(o: Object, _s: Serialiser) -> Result<Object, Error> {
        let _ = o;
        Err(Error::InvalidOperation)
    }
}

/// Lexer table/configuration corresponding to `YyLexer`.
#[derive(Clone, Debug)]
pub struct YyLexer {
    pub cats: Hashtable,
    pub erh: ErrorHandler,
    pub m_encoding: TextEncoding,
    pub m_gencat: UnicodeCategory,
    pub reswds: Hashtable,
    pub starts: Hashtable,
    pub tokens: Hashtable,
    pub toupper: bool,
    pub types: Hashtable,
    pub using_eof: bool,
    encoding: InputEncoding,
    categories: BTreeMap<DotNetUnicodeCategory, Charset>,
    start_dfas: BTreeMap<String, Dfa>,
    reserved: BTreeMap<String, ResWds>,
}

impl YyLexer {
    pub fn new(error_handler: ErrorHandler) -> Result<Self, Error> {
        let default_category = DotNetUnicodeCategory::OtherPunctuation;
        let mut categories = BTreeMap::new();
        categories.insert(default_category, Charset::for_category(default_category));
        Ok(Self {
            cats: Hashtable::default(),
            erh: error_handler,
            m_encoding: TextEncoding,
            m_gencat: UnicodeCategory(default_category as i32),
            reswds: Hashtable::default(),
            starts: Hashtable::default(),
            tokens: Hashtable::default(),
            toupper: false,
            types: Hashtable::default(),
            using_eof: false,
            encoding: InputEncoding::Utf8,
            categories,
            start_dfas: BTreeMap::new(),
            reserved: BTreeMap::new(),
        })
    }

    pub fn set_start_dfa(&mut self, state: impl Into<String>, dfa: Dfa) -> Result<(), Error> {
        let state = state.into();
        if state.is_empty() {
            return Err(Error::Argument);
        }
        self.start_dfas.insert(state, dfa);
        Ok(())
    }

    pub fn set_reserved_words(
        &mut self,
        name: impl Into<String>,
        words: ResWds,
    ) -> Result<(), Error> {
        let name = name.into();
        if name.is_empty() {
            return Err(Error::Argument);
        }
        self.reserved.insert(name, words);
        Ok(())
    }

    pub fn set_input_encoding(&mut self, value: String) {
        if let Ok(encoding) = InputEncoding::parse(&value) {
            self.toupper = encoding == InputEncoding::AsciiUpper;
            self.encoding = encoding;
        } else if let Ok(error) = CSToolsException::new_with_int32_string(
            43,
            format!("Warning: Encoding {value} unknown: ignored"),
        ) {
            let _ = self.erh.error(error);
        }
    }

    #[must_use]
    pub const fn input_encoding(&self) -> InputEncoding {
        self.encoding
    }

    pub fn using_cat(&mut self, category: UnicodeCategory) -> Result<Charset, Error> {
        let category = DotNetUnicodeCategory::from_compat(category)?;
        Ok(self
            .categories
            .entry(category)
            .or_insert_with(|| Charset::for_category(category))
            .clone())
    }

    pub fn get_test(&mut self, name: &str) -> Result<CharacterMatcher, Error> {
        if let Some(category) = (0..=29)
            .filter_map(|value| DotNetUnicodeCategory::try_from(value).ok())
            .find(|category| category.name() == name)
        {
            let _ = self.using_cat(UnicodeCategory(category as i32))?;
            return Ok(CharacterMatcher::Category(category));
        }
        let (class, categories): (UnicodeClass, &[DotNetUnicodeCategory]) = match name {
            "Symbol" => (
                UnicodeClass::Symbol,
                &[
                    DotNetUnicodeCategory::MathSymbol,
                    DotNetUnicodeCategory::CurrencySymbol,
                    DotNetUnicodeCategory::ModifierSymbol,
                    DotNetUnicodeCategory::OtherSymbol,
                ],
            ),
            "Punctuation" => (
                UnicodeClass::Punctuation,
                &[
                    DotNetUnicodeCategory::ConnectorPunctuation,
                    DotNetUnicodeCategory::DashPunctuation,
                    DotNetUnicodeCategory::OpenPunctuation,
                    DotNetUnicodeCategory::ClosePunctuation,
                    DotNetUnicodeCategory::InitialQuotePunctuation,
                    DotNetUnicodeCategory::FinalQuotePunctuation,
                    DotNetUnicodeCategory::OtherPunctuation,
                ],
            ),
            "Separator" => (
                UnicodeClass::Separator,
                &[
                    DotNetUnicodeCategory::SpaceSeparator,
                    DotNetUnicodeCategory::LineSeparator,
                    DotNetUnicodeCategory::ParagraphSeparator,
                ],
            ),
            "Number" => (
                UnicodeClass::Number,
                &[
                    DotNetUnicodeCategory::DecimalDigitNumber,
                    DotNetUnicodeCategory::LetterNumber,
                    DotNetUnicodeCategory::OtherNumber,
                ],
            ),
            "Letter" => (
                UnicodeClass::Letter,
                &[
                    DotNetUnicodeCategory::UppercaseLetter,
                    DotNetUnicodeCategory::LowercaseLetter,
                    DotNetUnicodeCategory::TitlecaseLetter,
                    DotNetUnicodeCategory::ModifierLetter,
                    DotNetUnicodeCategory::OtherLetter,
                ],
            ),
            "Digit" => (
                UnicodeClass::Digit,
                &[DotNetUnicodeCategory::DecimalDigitNumber],
            ),
            "Lower" => (
                UnicodeClass::Lower,
                &[DotNetUnicodeCategory::LowercaseLetter],
            ),
            "Upper" => (
                UnicodeClass::Upper,
                &[DotNetUnicodeCategory::UppercaseLetter],
            ),
            "EOF" => {
                self.using_eof = true;
                return Ok(CharacterMatcher::Eof);
            }
            "WhiteSpace" => (
                UnicodeClass::WhiteSpace,
                &[
                    DotNetUnicodeCategory::Control,
                    DotNetUnicodeCategory::SpaceSeparator,
                    DotNetUnicodeCategory::LineSeparator,
                    DotNetUnicodeCategory::ParagraphSeparator,
                ],
            ),
            _ => {
                if let Ok(error) =
                    CSToolsException::new_with_int32_string(24, format!("No such Charset {name}"))
                {
                    self.erh.error(error)?;
                }
                return Err(Error::Argument);
            }
        };
        for category in categories {
            let _ = self.using_cat(UnicodeCategory(*category as i32))?;
        }
        Ok(CharacterMatcher::UnicodeClass(class))
    }

    #[must_use]
    pub fn category_group(name: &str) -> Option<&'static [DotNetUnicodeCategory]> {
        match name {
            "WhiteSpace" => Some(&[
                DotNetUnicodeCategory::Control,
                DotNetUnicodeCategory::SpaceSeparator,
                DotNetUnicodeCategory::LineSeparator,
                DotNetUnicodeCategory::ParagraphSeparator,
            ]),
            "Letter" => Some(&[
                DotNetUnicodeCategory::UppercaseLetter,
                DotNetUnicodeCategory::LowercaseLetter,
                DotNetUnicodeCategory::TitlecaseLetter,
                DotNetUnicodeCategory::ModifierLetter,
                DotNetUnicodeCategory::OtherLetter,
            ]),
            "Number" => Some(&[
                DotNetUnicodeCategory::DecimalDigitNumber,
                DotNetUnicodeCategory::LetterNumber,
                DotNetUnicodeCategory::OtherNumber,
            ]),
            _ => None,
        }
    }

    pub fn get_dfa(&self) -> Result<(), Error> {
        if self.start_dfas.is_empty() {
            return Err(Error::InvalidOperation);
        }
        Ok(())
    }

    pub fn emit_dfa(&self, mut output: Box<dyn Write + Send>) -> Result<(), Error> {
        writeln!(output, "LSLLEXER 1").map_err(|_| Error::InvalidOperation)?;
        writeln!(output, "encoding {:?}", self.encoding).map_err(|_| Error::InvalidOperation)?;
        writeln!(output, "toupper {}", self.toupper).map_err(|_| Error::InvalidOperation)?;
        writeln!(output, "eof {}", self.using_eof).map_err(|_| Error::InvalidOperation)?;
        for (name, dfa) in &self.start_dfas {
            writeln!(output, "start {name}").map_err(|_| Error::InvalidOperation)?;
            writeln!(output, "{}", dfa.stable_description())
                .map_err(|_| Error::InvalidOperation)?;
        }
        for (name, words) in &self.reserved {
            for (word, token) in &words.m_wds {
                writeln!(
                    output,
                    "reserved {name} {word} {} {}",
                    token.number, token.name
                )
                .map_err(|_| Error::InvalidOperation)?;
            }
        }
        Ok(())
    }

    pub fn get_enumerator(&self) -> Result<IEnumerator, Error> {
        Ok(IEnumerator)
    }

    #[allow(clippy::ptr_arg)]
    pub fn old_action(
        &self,
        lexer: Lexer,
        yytext: &mut String,
        action: i32,
        reject: &mut bool,
    ) -> Result<TOKEN, Error> {
        *reject = false;
        let end = lexer.m_pch;
        let length = i32_len(utf16_len(yytext))?;
        let mut token = TOKEN::new_with_lexer_string(lexer, yytext.clone())?;
        token.number = action;
        token.pos = end.checked_sub(length).ok_or(Error::IndexOutOfRange)?;
        token.end = end;
        Ok(token)
    }

    fn dfa(&self, state: &str) -> Option<&Dfa> {
        self.start_dfas.get(state)
    }

    fn reserved_word(&self, name: &str, text: &str) -> Option<TokenDefinition> {
        self.reserved.get(name).and_then(|words| words.remap(text))
    }
}

/// Base parser symbol with source location and semantic value.
#[allow(clippy::upper_case_acronyms)]
pub struct SYMBOL {
    pub kids: ObjectList,
    pub m_dollar: Object,
    pub pos_with_field: i32,
    pub yylx: Option<Box<Lexer>>,
    pub yyps: Option<Box<Parser>>,
    name: String,
    number: i32,
    text: String,
}

impl Clone for SYMBOL {
    fn clone(&self) -> Self {
        Self {
            kids: self.kids.clone(),
            m_dollar: self.m_dollar.clone(),
            pos_with_field: self.pos_with_field,
            yylx: self.yylx.clone(),
            yyps: self.yyps.clone(),
            name: self.name.clone(),
            number: self.number,
            text: self.text.clone(),
        }
    }
}

impl fmt::Debug for SYMBOL {
    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
        formatter
            .debug_struct("SYMBOL")
            .field("kids", &self.kids)
            .field("m_dollar", &self.m_dollar)
            .field("pos", &self.pos_with_field)
            .field("has_lexer", &self.yylx.is_some())
            .field("has_parser", &self.yyps.is_some())
            .field("name", &self.name)
            .field("number", &self.number)
            .field("text", &self.text)
            .finish()
    }
}

impl SYMBOL {
    pub fn new_with_lexer(lexer: Lexer) -> Result<Self, Error> {
        Ok(Self {
            kids: ObjectList::new()?,
            m_dollar: Object::Undefined,
            pos_with_field: lexer.yypos(),
            yylx: Some(Box::new(lexer)),
            yyps: None,
            name: "SYMBOL".to_owned(),
            number: 0,
            text: String::new(),
        })
    }

    pub fn new_with_parser(parser: Parser) -> Result<Self, Error> {
        Ok(Self {
            kids: ObjectList::new()?,
            m_dollar: Object::Undefined,
            pos_with_field: 0,
            yylx: None,
            yyps: Some(Box::new(parser)),
            name: "SYMBOL".to_owned(),
            number: 0,
            text: String::new(),
        })
    }

    pub(crate) fn parser_symbol(
        name: String,
        number: i32,
        text: String,
        pos: i32,
        value: Object,
        kids: ObjectList,
    ) -> Self {
        Self {
            kids,
            m_dollar: value,
            pos_with_field: pos,
            yylx: None,
            yyps: None,
            name,
            number,
            text,
        }
    }

    fn location(&self) -> Result<SourceLineInfo, Error> {
        self.yylx
            .as_deref()
            .ok_or(Error::InvalidOperation)?
            .source_line_info(self.pos_with_field)
    }

    pub fn concrete_syntax_tree(&self) -> Result<(), Error> {
        println!("{}", self.concrete_syntax_tree_text());
        Ok(())
    }

    #[must_use]
    pub fn concrete_syntax_tree_text(&self) -> String {
        let mut output = self.yyname();
        for child in self.kids.iter() {
            output.push('\n');
            let _ = std::fmt::write(&mut output, format_args!("  {child:?}"));
        }
        output
    }

    pub fn is_action(&self) -> Result<bool, Error> {
        Ok(false)
    }
    pub fn is_c_symbol(&self) -> Result<bool, Error> {
        Ok(false)
    }
    pub fn is_terminal(&self) -> Result<bool, Error> {
        Ok(false)
    }
    pub fn matches(&self, value: String) -> Result<bool, Error> {
        Ok(self.text == value)
    }

    pub fn pass_(
        &self,
        symbols: YyParser,
        state: i32,
        entry: &mut ParserEntry,
    ) -> Result<bool, Error> {
        lookup_parser_entry(&symbols, self.yynum(), state, entry)
    }

    pub fn print(&self) -> Result<(), Error> {
        println!("{}", self.to_string());
        Ok(())
    }
    #[must_use]
    pub fn to_string(&self) -> String {
        if self.text.is_empty() {
            self.yyname()
        } else {
            format!("{}<{}>", self.yyname(), self.text)
        }
    }

    pub fn from(symbol: SYMBOL) -> i32 {
        match symbol.m_dollar {
            Object::Integer(value) => value,
            _ => panic!("SYMBOL semantic value is not a System.Int32"),
        }
    }

    #[must_use]
    pub fn line(&self) -> i32 {
        self.location().map_or(0, |value| value.line_number)
    }
    #[must_use]
    pub fn pos_with_property(&self) -> String {
        self.location()
            .map_or_else(|_| String::new(), |value| value.to_string())
    }
    #[must_use]
    pub fn position(&self) -> i32 {
        self.location().map_or(0, |value| value.raw_char_position)
    }

    pub fn yyact(&self) -> Result<&YyParser, Error> {
        self.yyps
            .as_deref()
            .map(|parser| &parser.m_symbols)
            .ok_or(Error::InvalidOperation)
    }

    #[must_use]
    pub fn yylval(&self) -> Object {
        self.m_dollar.clone()
    }
    pub fn set_yylval(&mut self, value: Object) {
        self.m_dollar = value;
    }
    pub(crate) fn raw_text(&self) -> String {
        self.text.clone()
    }
    #[must_use]
    pub fn yyname(&self) -> String {
        self.name.clone()
    }
    #[must_use]
    pub const fn yynum(&self) -> i32 {
        self.number
    }
}

impl fmt::Display for SYMBOL {
    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
        formatter.write_str(&self.yyname())
    }
}

/// Terminal token emitted by [`Lexer`].
#[allow(clippy::upper_case_acronyms)]
#[derive(Clone, Debug)]
pub struct TOKEN {
    text: String,
    name: String,
    number: i32,
    pub pos: i32,
    pub end: i32,
    value: Object,
    source: Option<Box<Lexer>>,
}

impl TOKEN {
    pub(crate) fn generated_with_lexer(
        lexer: Lexer,
        name: &str,
        number: i32,
    ) -> Result<Self, Error> {
        let mut token = Self::new_with_lexer(lexer)?;
        name.clone_into(&mut token.name);
        token.number = number;
        Ok(token)
    }

    pub(crate) fn generated_with_parser(
        parser: Parser,
        name: &str,
        number: i32,
        text: String,
    ) -> Result<Self, Error> {
        let mut token = Self::new_with_parser(parser)?;
        name.clone_into(&mut token.name);
        token.number = number;
        token.text = text;
        Ok(token)
    }

    pub fn new_with_lexer(lexer: Lexer) -> Result<Self, Error> {
        let text = lexer.yytext.clone();
        Self::new_with_lexer_string(lexer, text)
    }

    pub fn new_with_lexer_string(lexer: Lexer, text: String) -> Result<Self, Error> {
        let pos = lexer.yypos();
        let end = pos
            .checked_add(i32_len(utf16_len(&text))?)
            .ok_or(Error::IndexOutOfRange)?;
        Ok(Self {
            text,
            name: "TOKEN".to_owned(),
            number: 1,
            pos,
            end,
            value: Object::Undefined,
            source: Some(Box::new(lexer)),
        })
    }

    pub fn new_with_parser(_parser: Parser) -> Result<Self, Error> {
        Ok(Self {
            text: String::new(),
            name: "TOKEN".to_owned(),
            number: 1,
            pos: 0,
            end: 0,
            value: Object::Undefined,
            source: None,
        })
    }

    fn emitted(
        lexer: &Lexer,
        definition: TokenDefinition,
        text: String,
        pos: usize,
        end: usize,
    ) -> Result<Self, Error> {
        Ok(Self {
            text,
            name: definition.name,
            number: definition.number,
            pos: i32_len(pos)?,
            end: i32_len(end)?,
            value: Object::Undefined,
            source: Some(Box::new(lexer.clone())),
        })
    }

    fn eof(lexer: &Lexer) -> Result<Self, Error> {
        let pos = lexer.units.len();
        Self::emitted(
            lexer,
            TokenDefinition {
                name: "EOF".to_owned(),
                number: 2,
            },
            "EOF".to_owned(),
            pos,
            pos,
        )
    }

    fn location(&self) -> Result<SourceLineInfo, Error> {
        self.source
            .as_deref()
            .ok_or(Error::InvalidOperation)?
            .source_line_info(self.pos)
    }

    pub fn is_terminal(&self) -> Result<bool, Error> {
        Ok(true)
    }
    pub fn matches(&self, value: String) -> Result<bool, Error> {
        Ok(self.text == value)
    }

    pub fn pass_(
        &self,
        symbols: YyParser,
        state: i32,
        entry: &mut ParserEntry,
    ) -> Result<bool, Error> {
        let number = match symbols.literals.0.get(&Object::String(self.text.clone())) {
            Some(Object::Integer(number)) => *number,
            _ => self.number,
        };
        lookup_parser_entry(&symbols, number, state, entry)
    }

    pub fn print(&self) -> Result<(), Error> {
        println!("{}", self.to_string());
        Ok(())
    }
    #[must_use]
    pub fn to_string(&self) -> String {
        format!("{}<{}>", self.name, self.text)
    }
    #[must_use]
    pub fn yyname(&self) -> String {
        self.name.clone()
    }
    #[must_use]
    pub const fn yynum(&self) -> i32 {
        self.number
    }
    #[must_use]
    pub fn yytext(&self) -> String {
        self.text.clone()
    }
    pub fn set_yytext(&mut self, value: String) {
        self.text = value;
    }
    #[must_use]
    pub fn yylval(&self) -> Object {
        self.value.clone()
    }
    pub fn set_yylval(&mut self, value: Object) {
        self.value = value;
    }
}

impl fmt::Display for TOKEN {
    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
        formatter.write_str(&self.to_string())
    }
}

fn lookup_parser_entry(
    symbols: &YyParser,
    number: i32,
    state: i32,
    entry: &mut ParserEntry,
) -> Result<bool, Error> {
    if let Some(native) = symbols.lookup_entry(number, state)? {
        *entry = native;
        return Ok(true);
    }
    let Some(info) = symbols.symbol_info.0.get(&Object::Integer(number)) else {
        return Err(Error::Parse {
            position: 0,
            context: "parser does not recognize token or symbol",
        });
    };
    let Object::Map(states) = info else {
        return Err(Error::Parse {
            position: 0,
            context: "parser symbol information is malformed",
        });
    };
    let Some(value) = states.get(&state.to_string()) else {
        return Ok(false);
    };
    entry.m_priority = match value {
        Object::Integer(priority) => *priority,
        Object::Map(metadata) => match metadata.get("priority") {
            Some(Object::Integer(priority)) => *priority,
            _ => 0,
        },
        _ => {
            return Err(Error::Parse {
                position: 0,
                context: "parser entry is malformed",
            });
        }
    };
    Ok(true)
}

/// End-of-input terminal.
#[allow(clippy::upper_case_acronyms)]
#[derive(Clone, Debug)]
pub struct EOF(pub TOKEN);

impl EOF {
    pub fn new_with_lexer(lexer: Lexer) -> Result<Self, Error> {
        Ok(Self(TOKEN::eof(&lexer)?))
    }

    pub fn new_with_symbols_gen(_symbols: SymbolsGen) -> Result<Self, Error> {
        Ok(Self(TOKEN {
            text: "EOF".to_owned(),
            name: "EOF".to_owned(),
            number: 2,
            pos: 0,
            end: 0,
            value: Object::Undefined,
            source: None,
        }))
    }

    pub fn serialise(o: Object, _s: Serialiser) -> Result<Object, Error> {
        let _ = o;
        Err(Error::InvalidOperation)
    }
    #[must_use]
    pub fn yyname(&self) -> String {
        "EOF".to_owned()
    }
    #[must_use]
    pub const fn yynum(&self) -> i32 {
        2
    }
}

/// A proxy token used by generated grammars for nullable productions.
#[derive(Clone, Debug)]
pub struct Null {
    proxy: String,
}

impl Null {
    pub fn new_with_lexer_string(_lexer: Lexer, proxy: String) -> Result<Self, Error> {
        if proxy.is_empty() {
            return Err(Error::Argument);
        }
        Ok(Self { proxy })
    }

    pub fn new_with_parser_string(_parser: Parser, proxy: String) -> Result<Self, Error> {
        if proxy.is_empty() {
            return Err(Error::Argument);
        }
        Ok(Self { proxy })
    }

    #[must_use]
    pub fn yyname(&self) -> String {
        self.proxy.clone()
    }
}

/// Rust replacement for the C# cons-list, backed by contiguous storage.
#[derive(Clone, Debug, Default)]
pub struct ObjectList {
    values: VecDeque<Object>,
    count_override: Option<i32>,
}

impl ObjectList {
    pub fn new() -> Result<Self, Error> {
        Ok(Self::default())
    }
    pub fn add(&mut self, value: Object) -> Result<(), Error> {
        self.values.push_back(value);
        Ok(())
    }
    pub fn push(&mut self, value: Object) -> Result<(), Error> {
        self.values.push_front(value);
        Ok(())
    }
    pub fn pop(&mut self) -> Result<Object, Error> {
        self.values.pop_front().ok_or(Error::InvalidOperation)
    }
    pub fn top(&self) -> Object {
        self.values.front().cloned().unwrap_or(Object::Undefined)
    }
    pub fn get_enumerator(&self) -> Result<IEnumerator, Error> {
        Ok(IEnumerator)
    }
    #[must_use]
    pub fn count(&self) -> i32 {
        self.count_override
            .unwrap_or_else(|| i32::try_from(self.values.len()).unwrap_or(i32::MAX))
    }
    pub fn set_count(&mut self, value: i32) {
        let Ok(value) = usize::try_from(value) else {
            return;
        };
        self.values.resize(value, Object::Undefined);
        self.count_override = None;
    }
    #[must_use]
    pub fn item(&self, item_index: i32) -> Object {
        index(item_index)
            .ok()
            .and_then(|index| self.values.get(index))
            .cloned()
            .unwrap_or(Object::Undefined)
    }
    pub fn set_item(&mut self, index_value: i32, value: Object) {
        if let Ok(index) = index(index_value)
            && let Some(slot) = self.values.get_mut(index)
        {
            *slot = value;
        }
    }
    pub fn iter(&self) -> impl Iterator<Item = &Object> {
        self.values.iter()
    }
}

/// Runtime lexer over one immutable filtered source buffer.
#[derive(Clone, Debug)]
pub struct Lexer {
    pub m_buf: String,
    pub m_debug: bool,
    pub m_pch: i32,
    pub m_state: String,
    pub yytext: String,
    pub m_line_manager: LineManager,
    tokens: YyLexer,
    units: Vec<u16>,
    cursor: usize,
    start_match: usize,
    eof_emitted: bool,
}

impl Lexer {
    pub fn new(tokens: YyLexer) -> Result<Self, Error> {
        Ok(Self {
            m_buf: String::new(),
            m_debug: false,
            m_pch: 0,
            m_state: "YYINITIAL".to_owned(),
            yytext: String::new(),
            m_line_manager: LineManager::default(),
            tokens,
            units: Vec::new(),
            cursor: 0,
            start_match: 0,
            eof_emitted: false,
        })
    }

    pub fn start_with_string(&mut self, source: String) -> Result<(), Error> {
        self.start_with_cs_reader(CsReader::new_with_string(format!("{source}\n"))?)
    }

    pub fn start_with_stream_reader(&mut self, reader: StreamReader) -> Result<(), Error> {
        let source = self.tokens.encoding.decode(&reader.0)?;
        self.start_with_cs_reader(CsReader::new_with_string(source)?)
    }

    pub fn start_with_cs_reader(&mut self, reader: CsReader) -> Result<(), Error> {
        let (_, units, manager) = reader.into_parts();
        if units.len() > MAX_SOURCE_UNITS {
            return Err(Error::Argument);
        }
        self.m_buf = units_to_string(&units);
        self.units = units;
        self.m_line_manager = manager;
        self.reset()?;
        Ok(())
    }

    pub fn reset(&mut self) -> Result<(), Error> {
        self.cursor = 0;
        self.start_match = 0;
        self.m_pch = 0;
        self.yytext.clear();
        self.eof_emitted = false;
        Ok(())
    }

    pub fn yy_begin(&mut self, state: String) -> Result<(), Error> {
        if self.tokens.dfa(&state).is_none() {
            self.record_diagnostic(
                2,
                DiagnosticCategory::InvalidState,
                "unknown lexer start condition",
                self.cursor,
            )?;
            return Err(Error::Parse {
                position: self.cursor,
                context: "unknown lexer start condition",
            });
        }
        self.m_state = state;
        Ok(())
    }

    pub fn next_token(&mut self) -> Result<Option<TOKEN>, Error> {
        loop {
            if self.cursor >= self.units.len() {
                if self.tokens.using_eof && !self.eof_emitted {
                    self.eof_emitted = true;
                    "EOF".clone_into(&mut self.yytext);
                    return Ok(Some(TOKEN::eof(self)?));
                }
                return Ok(None);
            }
            let matched = self
                .tokens
                .dfa(&self.m_state)
                .map(|dfa| dfa.longest_match(&self.units, self.cursor));
            let Some(matched) = matched else {
                self.record_diagnostic(
                    2,
                    DiagnosticCategory::InvalidState,
                    "lexer start condition has no DFA",
                    self.cursor,
                )?;
                return Err(Error::Parse {
                    position: self.cursor,
                    context: "lexer start condition has no DFA",
                });
            };
            let Some((length, accept)) = matched else {
                let position = self.cursor;
                self.cursor += 1;
                self.m_pch = i32_len(self.cursor)?;
                self.record_diagnostic(
                    1,
                    DiagnosticCategory::InvalidCharacter,
                    "input does not match any lexer rule",
                    position,
                )?;
                return Err(Error::Parse {
                    position,
                    context: "input does not match any lexer rule",
                });
            };
            if length == 0 {
                self.record_diagnostic(
                    2,
                    DiagnosticCategory::InvalidState,
                    "non-EOF lexer rule accepted an empty token",
                    self.cursor,
                )?;
                return Err(Error::Parse {
                    position: self.cursor,
                    context: "non-EOF lexer rule accepted an empty token",
                });
            }
            if length > MAX_TOKEN_UNITS {
                self.record_diagnostic(
                    3,
                    DiagnosticCategory::TokenTooLong,
                    "token exceeds the configured size bound",
                    self.cursor,
                )?;
                return Err(Error::Parse {
                    position: self.cursor,
                    context: "token exceeds the configured size bound",
                });
            }
            self.start_match = self.cursor;
            let end = self.cursor + length;
            let mut text = units_to_string(&self.units[self.cursor..end]);
            if self.tokens.toupper {
                text = text.to_uppercase();
            }
            self.cursor = end;
            self.m_pch = i32_len(end)?;
            self.yytext.clone_from(&text);
            let mut definition = accept.token;
            if let Some(table) = accept.reserved_words.as_deref()
                && let Some(reserved) = self.tokens.reserved_word(table, &text)
            {
                definition = reserved;
            }
            let should_emit = match accept.action {
                LexerAction::Emit => true,
                LexerAction::Skip => false,
                LexerAction::EmitAndBegin(state) => {
                    self.yy_begin(state)?;
                    true
                }
                LexerAction::SkipAndBegin(state) => {
                    self.yy_begin(state)?;
                    false
                }
            };
            if should_emit {
                return Ok(Some(TOKEN::emitted(
                    self,
                    definition,
                    text,
                    self.start_match,
                    end,
                )?));
            }
        }
    }

    pub fn next(&mut self) -> Result<TOKEN, Error> {
        self.next_token()?.ok_or(Error::InvalidOperation)
    }

    pub fn get_enumerator(&self) -> Result<LexerEnumerator, Error> {
        LexerEnumerator::new(self.clone())
    }

    #[must_use]
    pub fn iter(&self) -> LexerIterator {
        LexerIterator {
            lexer: self.clone(),
            finished: false,
        }
    }

    pub fn peek_char(&self) -> Result<Utf16CodeUnit, Error> {
        Ok(Utf16CodeUnit(
            self.units
                .get(self.cursor)
                .copied()
                .unwrap_or(if self.tokens.using_eof { u16::MAX } else { 0 }),
        ))
    }

    pub fn get_char(&mut self) -> Result<i32, Error> {
        let value = self.peek_char()?.0;
        if self.cursor < self.units.len() {
            self.cursor += 1;
            self.m_pch = i32_len(self.cursor)?;
        }
        Ok(i32::from(value))
    }

    pub fn advance(&mut self) -> Result<(), Error> {
        let _ = self.get_char()?;
        Ok(())
    }

    pub fn un_get_char(&mut self) -> Result<(), Error> {
        if self.cursor == 0 {
            return Err(Error::InvalidOperation);
        }
        self.cursor -= 1;
        self.m_pch = i32_len(self.cursor)?;
        Ok(())
    }

    pub fn source_line_info(&self, pos: i32) -> Result<SourceLineInfo, Error> {
        SourceLineInfo::from_manager(&self.m_line_manager, pos, self.m_buf.clone())
    }

    pub fn source_line(&self, info: SourceLineInfo) -> Result<String, Error> {
        Ok(info.source_line())
    }

    pub fn saypos(&self, pos: i32) -> Result<String, Error> {
        Ok(self.source_line_info(pos)?.to_string())
    }

    fn record_diagnostic(
        &mut self,
        code: i32,
        category: DiagnosticCategory,
        message: &str,
        position: usize,
    ) -> Result<(), Error> {
        let location = self.source_line_info(i32_len(position)?)?;
        let input = self
            .units
            .get(position)
            .map(|unit| units_to_string(std::slice::from_ref(unit)))
            .unwrap_or_default();
        self.tokens.erh.push(Diagnostic {
            code,
            category,
            severity: DiagnosticSeverity::Error,
            message: message.to_owned(),
            location,
            input,
        })
    }

    #[must_use]
    pub fn diagnostics(&self) -> &[Diagnostic] {
        self.tokens.erh.diagnostics()
    }

    #[must_use]
    pub fn m_start(&self) -> Dfa {
        self.tokens.dfa(&self.m_state).cloned().unwrap_or_default()
    }
    #[must_use]
    pub fn tokens(&self) -> YyLexer {
        self.tokens.clone()
    }
    pub fn set_tokens(&mut self, value: YyLexer) {
        self.tokens = value;
    }
    #[must_use]
    pub fn yypos(&self) -> i32 {
        self.m_pch
    }
}

/// Idiomatic Rust iterator over lexer results.
pub struct LexerIterator {
    lexer: Lexer,
    finished: bool,
}

impl Iterator for LexerIterator {
    type Item = Result<TOKEN, Error>;

    fn next(&mut self) -> Option<Self::Item> {
        if self.finished {
            return None;
        }
        match self.lexer.next_token() {
            Ok(Some(token)) => Some(Ok(token)),
            Ok(None) => {
                self.finished = true;
                None
            }
            Err(error) => {
                self.finished = true;
                Some(Err(error))
            }
        }
    }
}

impl IntoIterator for &Lexer {
    type Item = Result<TOKEN, Error>;
    type IntoIter = LexerIterator;

    fn into_iter(self) -> Self::IntoIter {
        self.iter()
    }
}

/// Compatibility enumerator backed by the same bounded lexer iterator.
#[derive(Clone, Debug)]
pub struct LexerEnumerator {
    lexer: Lexer,
    initial: Lexer,
    current: Option<TOKEN>,
}

impl LexerEnumerator {
    pub fn new(lexer: Lexer) -> Result<Self, Error> {
        Ok(Self {
            initial: lexer.clone(),
            lexer,
            current: None,
        })
    }

    pub fn move_next(&mut self) -> Result<bool, Error> {
        self.current = self.lexer.next_token()?;
        Ok(self.current.is_some())
    }

    pub fn reset(&mut self) -> Result<(), Error> {
        self.lexer = self.initial.clone();
        self.current = None;
        Ok(())
    }

    #[must_use]
    pub fn current(&self) -> TOKEN {
        self.current
            .clone()
            .expect("Current is unavailable before MoveNext")
    }

    pub fn set_current(&mut self, value: TOKEN) {
        self.current = Some(value);
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    fn definition(name: &str, number: i32) -> TokenDefinition {
        TokenDefinition::new(name, number).expect("valid token definition")
    }

    fn accept(name: &str, number: i32, action: LexerAction) -> DfaAccept {
        DfaAccept {
            token: definition(name, number),
            action,
            action_number: 0,
            reserved_words: None,
        }
    }

    fn sample_lexer() -> Lexer {
        let mut table = YyLexer::new(ErrorHandler::default()).expect("table");
        table.using_eof = true;
        let states = vec![
            DfaState::default()
                .transition(
                    CharacterMatcher::Range {
                        first: b'a'.into(),
                        last: b'z'.into(),
                    },
                    1,
                )
                .transition(CharacterMatcher::Exact(b' '.into()), 2)
                .transition(CharacterMatcher::Exact(b'\n'.into()), 2),
            DfaState::default()
                .transition(
                    CharacterMatcher::Range {
                        first: b'a'.into(),
                        last: b'z'.into(),
                    },
                    1,
                )
                .accepting(accept("ID", 3, LexerAction::Emit)),
            DfaState::default()
                .transition(CharacterMatcher::Exact(b' '.into()), 2)
                .transition(CharacterMatcher::Exact(b'\n'.into()), 2)
                .accepting(accept("WS", 4, LexerAction::Skip)),
        ];
        table
            .set_start_dfa("YYINITIAL", Dfa::from_states(states, 0).expect("dfa"))
            .expect("start");
        let mut lexer = Lexer::new(table).expect("lexer");
        lexer
            .start_with_string("alpha beta".to_owned())
            .expect("source");
        lexer
    }

    #[test]
    fn lexer_uses_maximum_munch_skips_and_emits_eof() {
        let mut lexer = sample_lexer();
        let first = lexer.next().expect("first");
        let second = lexer.next().expect("second");
        let eof = lexer.next().expect("eof");
        assert_eq!(
            (first.yyname(), first.yytext(), first.pos, first.end),
            ("ID".to_owned(), "alpha".to_owned(), 0, 5)
        );
        assert_eq!(
            (second.yyname(), second.yytext(), second.pos, second.end),
            ("ID".to_owned(), "beta".to_owned(), 6, 10)
        );
        assert_eq!(
            (eof.yyname(), eof.yynum(), eof.pos),
            ("EOF".to_owned(), 2, 11)
        );
        assert!(lexer.next_token().expect("done").is_none());
    }

    #[test]
    fn comments_are_removed_but_raw_columns_are_preserved() {
        let reader =
            CsReader::new_with_string("ab/*xyz*/cd\n// comment\nef".to_owned()).expect("reader");
        assert_eq!(units_to_string(&reader.units), "abcd\n\nef");
        let first = SourceLineInfo::from_manager(&reader.lm, 2, units_to_string(&reader.units))
            .expect("location");
        let after = SourceLineInfo::from_manager(&reader.lm, 3, units_to_string(&reader.units))
            .expect("location");
        assert_eq!(first.raw_char_position, 2);
        assert_eq!(after.raw_char_position, 10);
        let third_line =
            SourceLineInfo::from_manager(&reader.lm, 6, units_to_string(&reader.units))
                .expect("location");
        assert_eq!(third_line.line_number, 3);
    }

    #[test]
    fn utf16_positions_do_not_turn_into_utf8_byte_offsets() {
        let reader = CsReader::new_with_string("a😀b\n".to_owned()).expect("reader");
        assert_eq!(reader.units.len(), 5);
        let info = SourceLineInfo::from_manager(&reader.lm, 4, units_to_string(&reader.units))
            .expect("location");
        assert_eq!(info.char_position, 5);
    }

    #[test]
    fn line_directives_change_logical_line_and_file() {
        let reader = CsReader::new_with_string("#line 42 \"generated.cs\"\nvalue\n".to_owned())
            .expect("reader");
        assert_eq!(reader.fname, "generated.cs");
        let info = SourceLineInfo::from_manager(&reader.lm, 1, units_to_string(&reader.units))
            .expect("location");
        assert_eq!(info.line_number, 42);
    }

    #[test]
    fn unterminated_block_comment_has_stable_category_and_position() {
        assert_eq!(
            CsReader::new_with_string("a /* nope".to_owned()).expect_err("invalid"),
            Error::Parse {
                position: 9,
                context: "unterminated block comment"
            }
        );
    }

    #[test]
    fn unicode_categories_match_dotnet_numbering() {
        assert!(
            CatTest::new(UnicodeCategory(0))
                .expect("category")
                .test(Utf16CodeUnit('A' as u16))
                .expect("test")
        );
        assert!(
            CatTest::new(UnicodeCategory(25))
                .expect("category")
                .test(Utf16CodeUnit('+' as u16))
                .expect("test")
        );
        assert!(
            CatTest::new(UnicodeCategory(16))
                .expect("category")
                .test(Utf16CodeUnit(0xD800))
                .expect("test")
        );
    }

    #[test]
    fn reserved_words_remap_only_the_configured_token() {
        let mut lexer = sample_lexer();
        lexer
            .tokens
            .set_reserved_words(
                "keywords",
                ResWds::from_pairs([("alpha", definition("ALPHA", 9))], false).expect("words"),
            )
            .expect("table");
        if let Some(dfa) = lexer.tokens.start_dfas.get_mut("YYINITIAL") {
            dfa.states[1]
                .accept
                .as_mut()
                .expect("accept")
                .reserved_words = Some("keywords".to_owned());
        }
        let first = lexer.next().expect("token");
        assert_eq!((first.yyname(), first.yynum()), ("ALPHA".to_owned(), 9));
    }

    #[test]
    fn encoded_input_is_validated_without_platform_code_pages() {
        assert_eq!(
            InputEncoding::Ascii.decode(&[0x80]),
            Err(Error::Parse {
                position: 0,
                context: "source contains a non-ASCII byte"
            })
        );
        assert_eq!(
            InputEncoding::Utf16Le.decode(&[b'a', 0, b'b']),
            Err(Error::Parse {
                position: 2,
                context: "UTF-16 source has a trailing byte"
            })
        );
    }

    #[test]
    fn lexer_iterator_stops_after_a_diagnostic() {
        let mut lexer = sample_lexer();
        lexer
            .start_with_string("alpha ! beta".to_owned())
            .expect("source");
        let mut iter = lexer.iter();
        assert_eq!(
            iter.next().expect("first").expect("token").yytext(),
            "alpha"
        );
        assert!(matches!(
            iter.next().expect("diagnostic"),
            Err(Error::Parse {
                position: 6,
                context: "input does not match any lexer rule"
            })
        ));
        assert!(iter.next().is_none());
    }
}