MetaCrate/crates/metacrate-lsl-tools/tests/lexer_compat.rs

//! Focused compatibility fixtures translated from the pinned lexer runtime.

use std::collections::BTreeSet;
use std::sync::{Arc, Mutex};

use libremetaverse_types::compat::{Object, UnicodeCategory, Utf16CodeUnit};
use metacrate_lsl_tools::{
    CSToolsException, CharacterMatcher, CsReader, Dfa, DfaAccept, DfaState, DiagnosticCategory,
    DotNetUnicodeCategory, Error, ErrorHandler, InputEncoding, Lexer, LexerAction, LineManager,
    ObjectList, ResWds, SourceLineInfo, TOKEN, TokenDefinition, YyLexer,
};

fn token(name: &str, number: i32) -> TokenDefinition {
    TokenDefinition::new(name, number).expect("valid definition")
}

fn accept(name: &str, number: i32, action: LexerAction, reserved_words: Option<&str>) -> DfaAccept {
    DfaAccept {
        token: token(name, number),
        action,
        action_number: number,
        reserved_words: reserved_words.map(ToOwned::to_owned),
    }
}

fn language_table() -> YyLexer {
    let letters = vec![
        DotNetUnicodeCategory::UppercaseLetter,
        DotNetUnicodeCategory::LowercaseLetter,
        DotNetUnicodeCategory::TitlecaseLetter,
        DotNetUnicodeCategory::ModifierLetter,
        DotNetUnicodeCategory::OtherLetter,
    ];
    let whitespace = vec![
        DotNetUnicodeCategory::Control,
        DotNetUnicodeCategory::SpaceSeparator,
        DotNetUnicodeCategory::LineSeparator,
        DotNetUnicodeCategory::ParagraphSeparator,
    ];
    let states = vec![
        DfaState::default()
            .transition(CharacterMatcher::Categories(letters.clone()), 1)
            .transition(CharacterMatcher::Exact(u16::from(b'_')), 1)
            .transition(
                CharacterMatcher::Category(DotNetUnicodeCategory::DecimalDigitNumber),
                2,
            )
            .transition(CharacterMatcher::Categories(whitespace.clone()), 3)
            .transition(CharacterMatcher::Exact(u16::from(b'+')), 4),
        DfaState::default()
            .transition(CharacterMatcher::Categories(letters), 1)
            .transition(CharacterMatcher::Exact(u16::from(b'_')), 1)
            .transition(
                CharacterMatcher::Category(DotNetUnicodeCategory::DecimalDigitNumber),
                1,
            )
            .accepting(accept("ID", 3, LexerAction::Emit, Some("keywords"))),
        DfaState::default()
            .transition(
                CharacterMatcher::Category(DotNetUnicodeCategory::DecimalDigitNumber),
                2,
            )
            .accepting(accept("INTEGER", 4, LexerAction::Emit, None)),
        DfaState::default()
            .transition(CharacterMatcher::Categories(whitespace), 3)
            .accepting(accept("WS", 5, LexerAction::Skip, None)),
        DfaState::default().accepting(accept("PLUS", 6, LexerAction::Emit, None)),
    ];
    let mut table = YyLexer::new(ErrorHandler::default()).expect("table");
    table.using_eof = true;
    table
        .set_start_dfa("YYINITIAL", Dfa::from_states(states, 0).expect("dfa"))
        .expect("start state");
    table
        .set_reserved_words(
            "keywords",
            ResWds::from_pairs(
                [("if", token("IF", 20)), ("while", token("WHILE", 21))],
                false,
            )
            .expect("reserved words"),
        )
        .expect("reserved table");
    table
}

fn lexer(source: &str) -> Lexer {
    let mut lexer = Lexer::new(language_table()).expect("lexer");
    lexer
        .start_with_string(source.to_owned())
        .expect("valid source");
    lexer
}

#[test]
fn token_fixture_preserves_names_text_numbers_positions_and_eof() {
    let mut lexer = lexer("if café_2 + 19");
    let mut tokens = Vec::new();
    while let Some(value) = lexer.next_token().expect("tokenization") {
        tokens.push(value);
    }
    let actual = tokens
        .iter()
        .map(|token| {
            (
                token.yyname(),
                token.yytext(),
                token.yynum(),
                token.pos,
                token.end,
            )
        })
        .collect::<Vec<_>>();
    assert_eq!(
        actual,
        [
            ("IF".to_owned(), "if".to_owned(), 20, 0, 2),
            ("ID".to_owned(), "café_2".to_owned(), 3, 3, 9),
            ("PLUS".to_owned(), "+".to_owned(), 6, 10, 11),
            ("INTEGER".to_owned(), "19".to_owned(), 4, 12, 14),
            ("EOF".to_owned(), "EOF".to_owned(), 2, 15, 15),
        ]
    );
}

#[test]
fn reserved_words_are_exact_unless_the_table_requests_uppercase() {
    let mut lexer = lexer("IF if");
    assert_eq!(lexer.next().expect("identifier").yyname(), "ID");
    assert_eq!(lexer.next().expect("keyword").yyname(), "IF");

    let words = ResWds::from_pairs([("while", token("WHILE", 21))], true).expect("words");
    let mut value = TOKEN::new_with_lexer_string(lexer.clone(), "WhIlE".to_owned()).expect("token");
    words.check(lexer, &mut value).expect("check");
    assert_eq!((value.yyname(), value.yynum()), ("WHILE".to_owned(), 21));
}

#[test]
fn invalid_input_reports_stable_category_location_and_text() {
    let mut lexer = lexer("ok @ nope");
    assert_eq!(lexer.next().expect("first").yytext(), "ok");
    assert_eq!(
        lexer.next().expect_err("invalid character"),
        Error::Parse {
            position: 3,
            context: "input does not match any lexer rule"
        }
    );
    let diagnostic = &lexer.diagnostics()[0];
    assert_eq!(diagnostic.category, DiagnosticCategory::InvalidCharacter);
    assert_eq!(diagnostic.location.line_number, 1);
    assert_eq!(diagnostic.location.raw_char_position, 3);
    assert_eq!(diagnostic.input, "@");
}

#[test]
fn unknown_start_condition_reports_invalid_state_without_changing_state() {
    let mut lexer = lexer("value");
    assert_eq!(
        lexer.yy_begin("MISSING".to_owned()),
        Err(Error::Parse {
            position: 0,
            context: "unknown lexer start condition"
        })
    );
    assert_eq!(lexer.m_state, "YYINITIAL");
    assert_eq!(
        lexer.diagnostics()[0].category,
        DiagnosticCategory::InvalidState
    );
}

#[test]
fn source_reader_removes_both_comment_forms_and_keeps_newlines() {
    let mut reader = CsReader::new_with_string("a/*x\ny*/b//z\nc".to_owned()).expect("reader");
    assert_eq!(reader.read_line().expect("line"), "a");
    assert_eq!(reader.read_line().expect("line"), "b");
    assert_eq!(reader.read_line().expect("line"), "c");
    assert!(reader.eof().expect("eof"));
}

#[test]
fn source_reader_normalizes_crlf_and_lone_cr() {
    let mut reader = CsReader::new_with_string("a\r\nb\rc\n".to_owned()).expect("reader");
    assert_eq!(reader.read_line().expect("line"), "a");
    assert_eq!(reader.read_line().expect("line"), "b");
    assert_eq!(reader.read_line().expect("line"), "c");
}

#[test]
fn source_reader_rejects_unterminated_block_comment_at_eof() {
    assert_eq!(
        CsReader::new_with_string("before /* never closed".to_owned()).expect_err("invalid"),
        Error::Parse {
            position: 22,
            context: "unterminated block comment"
        }
    );
}

#[test]
fn line_directive_updates_file_and_logical_line() {
    let reader =
        CsReader::new_with_string("#line 700 \"table.lex\"\nword\n".to_owned()).expect("reader");
    assert_eq!(reader.fname, "table.lex");
    let info = SourceLineInfo::new_with_line_manager_int32(reader.lm, 1).expect("location");
    assert_eq!(info.line_number, 700);
}

#[test]
fn utf16_input_decoding_and_surrogate_validation_are_deterministic() {
    let mut reader = CsReader::from_bytes(
        &[0xFF, 0xFE, b'A', 0, 0x3D, 0xD8, 0, 0xDE],
        InputEncoding::Utf16Le,
        "source.lex",
    )
    .expect("utf16");
    assert_eq!(reader.read_line().expect("line"), "A😀");
    let mut bom_override =
        CsReader::from_bytes(&[0xFE, 0xFF, 0, b'B'], InputEncoding::Utf16Le, "source.lex")
            .expect("BOM override");
    assert_eq!(bom_override.read_line().expect("line"), "B");
    assert!(matches!(
        CsReader::from_bytes(&[0x00, 0xD8], InputEncoding::Utf16Le, "bad"),
        Err(Error::Parse {
            position: 0,
            context: "source contains an unpaired UTF-16 surrogate"
        })
    ));
}

#[test]
fn ascii_input_rejects_non_ascii_instead_of_lossily_replacing_it() {
    assert!(matches!(
        CsReader::from_bytes(&[b'a', 0x80], InputEncoding::Ascii, "bad"),
        Err(Error::Parse {
            position: 1,
            context: "source contains a non-ASCII byte"
        })
    ));
}

#[test]
fn category_predicates_match_dotnet_values_and_groups() {
    let mut table = YyLexer::new(ErrorHandler::default()).expect("table");
    let punctuation = table.get_test("Punctuation").expect("punctuation");
    let whitespace = table.get_test("WhiteSpace").expect("whitespace");
    let states = vec![
        DfaState::default()
            .transition(punctuation, 1)
            .transition(whitespace, 2),
        DfaState::default().accepting(accept("PUNCT", 1, LexerAction::Emit, None)),
        DfaState::default().accepting(accept("SPACE", 2, LexerAction::Emit, None)),
    ];
    let dfa = Dfa::from_states(states, 0).expect("dfa");
    let mut action = -1;
    assert_eq!(dfa.match_("—".to_owned(), 0, &mut action), Ok(1));
    assert_eq!(dfa.match_("\t".to_owned(), 0, &mut action), Ok(1));
    assert_eq!(dfa.match_("\0".to_owned(), 0, &mut action), Ok(-1));
    assert!(
        metacrate_lsl_tools::CatTest::new(UnicodeCategory(16))
            .expect("surrogate")
            .test(Utf16CodeUnit(0xD800))
            .expect("test")
    );
    assert_eq!(table.get_test("MissingClass"), Err(Error::Argument));
    assert_eq!(
        table.erh.diagnostics()[0].category,
        DiagnosticCategory::UnknownCharacterSet
    );
}

#[test]
fn dfa_uses_maximum_munch_and_exposes_action_number() {
    let states = vec![
        DfaState::default().transition(CharacterMatcher::Exact(b'a'.into()), 1),
        DfaState::default()
            .transition(CharacterMatcher::Exact(b'b'.into()), 2)
            .accepting(accept("A", 1, LexerAction::Emit, None)),
        DfaState::default().accepting(accept("AB", 2, LexerAction::Emit, None)),
    ];
    let dfa = Dfa::from_states(states, 0).expect("dfa");
    let mut action = -1;
    assert_eq!(dfa.match_("abc".to_owned(), 0, &mut action), Ok(2));
    assert_eq!(action, 2);
}

#[test]
fn deterministic_table_output_is_independent_of_hash_iteration() {
    let table = language_table();
    let first = Arc::new(Mutex::new(Vec::new()));
    table
        .emit_dfa(Box::new(SharedWriter(first.clone())))
        .expect("emit first");
    let second = Arc::new(Mutex::new(Vec::new()));
    table
        .emit_dfa(Box::new(SharedWriter(second.clone())))
        .expect("emit second");
    let first = first.lock().expect("first output").clone();
    let second = second.lock().expect("second output").clone();
    assert_eq!(first, second);
    assert!(
        String::from_utf8(first)
            .expect("utf8")
            .starts_with("LSLLEXER 1\n")
    );
}

struct SharedWriter(Arc<Mutex<Vec<u8>>>);

impl std::io::Write for SharedWriter {
    fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {
        self.0
            .lock()
            .map_err(|_| std::io::Error::other("poisoned output"))?
            .extend_from_slice(bytes);
        Ok(bytes.len())
    }

    fn flush(&mut self) -> std::io::Result<()> {
        Ok(())
    }
}

#[test]
fn compatibility_enumerator_resets_to_the_first_token() {
    let lexer = lexer("one two");
    let mut enumerator = lexer.get_enumerator().expect("enumerator");
    assert!(enumerator.move_next().expect("first"));
    assert_eq!(enumerator.current().yytext(), "one");
    assert!(enumerator.move_next().expect("second"));
    assert_eq!(enumerator.current().yytext(), "two");
    enumerator.reset().expect("reset");
    assert!(enumerator.move_next().expect("first again"));
    assert_eq!(enumerator.current().yytext(), "one");
}

#[test]
fn error_handler_counts_reports_and_honors_throw_mode() {
    let error =
        CSToolsException::new_with_int32_string(43, "bad encoding".to_owned()).expect("diagnostic");
    let mut collecting = ErrorHandler::new_with_constructor().expect("handler");
    collecting.error(error.clone()).expect("reported");
    assert_eq!(collecting.counter, 1);
    assert_eq!(
        collecting.diagnostics()[0].category,
        DiagnosticCategory::Encoding
    );

    let mut throwing = ErrorHandler::new_with_boolean(true).expect("handler");
    assert_eq!(throwing.error(error), Err(Error::InvalidOperation));
    assert_eq!(throwing.counter, 1);
    assert!(throwing.diagnostics().is_empty());
}

#[test]
fn line_manager_backtracking_removes_future_lines() {
    let mut manager = LineManager::new().expect("manager");
    manager.newline(4).expect("line 2");
    let first_on_second =
        SourceLineInfo::new_with_line_manager_int32(manager.clone(), 4).expect("location");
    assert_eq!(first_on_second.raw_char_position, 1);
    manager.newline(8).expect("line 3");
    manager.backto(6).expect("rewind");
    assert_eq!(manager.lines, 2);
    assert_eq!(manager.end, 8);
}

#[test]
fn object_list_preserves_push_add_top_pop_and_index_order() {
    let mut values = ObjectList::new().expect("list");
    values.add(Object::Integer(2)).expect("add");
    values.push(Object::Integer(1)).expect("push");
    assert_eq!(values.count(), 2);
    assert_eq!(values.top(), Object::Integer(1));
    assert_eq!(values.item(1), Object::Integer(2));
    assert_eq!(values.pop(), Ok(Object::Integer(1)));
}

#[test]
fn matcher_set_is_value_based_and_bounded() {
    let matcher = CharacterMatcher::Set(BTreeSet::from([b'x'.into(), b'y'.into()]));
    let states = vec![
        DfaState::default().transition(matcher, 1),
        DfaState::default().accepting(accept("XY", 1, LexerAction::Emit, None)),
    ];
    let dfa = Dfa::from_states(states, 0).expect("dfa");
    let mut action = 0;
    assert_eq!(dfa.match_("y".to_owned(), 0, &mut action), Ok(1));
    assert_eq!(dfa.match_("z".to_owned(), 0, &mut action), Ok(-1));
}