//! Focused compatibility fixtures translated from the pinned lexer runtime. use std::collections::BTreeSet; use std::sync::{Arc, Mutex}; use libremetaverse_lsl_tools::{ CSToolsException, CharacterMatcher, CsReader, Dfa, DfaAccept, DfaState, DiagnosticCategory, DotNetUnicodeCategory, Error, ErrorHandler, InputEncoding, Lexer, LexerAction, LineManager, ObjectList, ResWds, SourceLineInfo, TOKEN, TokenDefinition, YyLexer, }; use libremetaverse_types::compat::{Object, UnicodeCategory, Utf16CodeUnit}; fn token(name: &str, number: i32) -> TokenDefinition { TokenDefinition::new(name, number).expect("valid definition") } fn accept(name: &str, number: i32, action: LexerAction, reserved_words: Option<&str>) -> DfaAccept { DfaAccept { token: token(name, number), action, action_number: number, reserved_words: reserved_words.map(ToOwned::to_owned), } } fn language_table() -> YyLexer { let letters = vec![ DotNetUnicodeCategory::UppercaseLetter, DotNetUnicodeCategory::LowercaseLetter, DotNetUnicodeCategory::TitlecaseLetter, DotNetUnicodeCategory::ModifierLetter, DotNetUnicodeCategory::OtherLetter, ]; let whitespace = vec![ DotNetUnicodeCategory::Control, DotNetUnicodeCategory::SpaceSeparator, DotNetUnicodeCategory::LineSeparator, DotNetUnicodeCategory::ParagraphSeparator, ]; let states = vec![ DfaState::default() .transition(CharacterMatcher::Categories(letters.clone()), 1) .transition(CharacterMatcher::Exact(u16::from(b'_')), 1) .transition( CharacterMatcher::Category(DotNetUnicodeCategory::DecimalDigitNumber), 2, ) .transition(CharacterMatcher::Categories(whitespace.clone()), 3) .transition(CharacterMatcher::Exact(u16::from(b'+')), 4), DfaState::default() .transition(CharacterMatcher::Categories(letters), 1) .transition(CharacterMatcher::Exact(u16::from(b'_')), 1) .transition( CharacterMatcher::Category(DotNetUnicodeCategory::DecimalDigitNumber), 1, ) .accepting(accept("ID", 3, LexerAction::Emit, Some("keywords"))), DfaState::default() .transition( CharacterMatcher::Category(DotNetUnicodeCategory::DecimalDigitNumber), 2, ) .accepting(accept("INTEGER", 4, LexerAction::Emit, None)), DfaState::default() .transition(CharacterMatcher::Categories(whitespace), 3) .accepting(accept("WS", 5, LexerAction::Skip, None)), DfaState::default().accepting(accept("PLUS", 6, LexerAction::Emit, None)), ]; let mut table = YyLexer::new(ErrorHandler::default()).expect("table"); table.using_eof = true; table .set_start_dfa("YYINITIAL", Dfa::from_states(states, 0).expect("dfa")) .expect("start state"); table .set_reserved_words( "keywords", ResWds::from_pairs( [("if", token("IF", 20)), ("while", token("WHILE", 21))], false, ) .expect("reserved words"), ) .expect("reserved table"); table } fn lexer(source: &str) -> Lexer { let mut lexer = Lexer::new(language_table()).expect("lexer"); lexer .start_with_string(source.to_owned()) .expect("valid source"); lexer } #[test] fn token_fixture_preserves_names_text_numbers_positions_and_eof() { let mut lexer = lexer("if café_2 + 19"); let mut tokens = Vec::new(); while let Some(value) = lexer.next_token().expect("tokenization") { tokens.push(value); } let actual = tokens .iter() .map(|token| { ( token.yyname(), token.yytext(), token.yynum(), token.pos, token.end, ) }) .collect::>(); assert_eq!( actual, [ ("IF".to_owned(), "if".to_owned(), 20, 0, 2), ("ID".to_owned(), "café_2".to_owned(), 3, 3, 9), ("PLUS".to_owned(), "+".to_owned(), 6, 10, 11), ("INTEGER".to_owned(), "19".to_owned(), 4, 12, 14), ("EOF".to_owned(), "EOF".to_owned(), 2, 15, 15), ] ); } #[test] fn reserved_words_are_exact_unless_the_table_requests_uppercase() { let mut lexer = lexer("IF if"); assert_eq!(lexer.next().expect("identifier").yyname(), "ID"); assert_eq!(lexer.next().expect("keyword").yyname(), "IF"); let words = ResWds::from_pairs([("while", token("WHILE", 21))], true).expect("words"); let mut value = TOKEN::new_with_lexer_string(lexer.clone(), "WhIlE".to_owned()).expect("token"); words.check(lexer, &mut value).expect("check"); assert_eq!((value.yyname(), value.yynum()), ("WHILE".to_owned(), 21)); } #[test] fn invalid_input_reports_stable_category_location_and_text() { let mut lexer = lexer("ok @ nope"); assert_eq!(lexer.next().expect("first").yytext(), "ok"); assert_eq!( lexer.next().expect_err("invalid character"), Error::Parse { position: 3, context: "input does not match any lexer rule" } ); let diagnostic = &lexer.diagnostics()[0]; assert_eq!(diagnostic.category, DiagnosticCategory::InvalidCharacter); assert_eq!(diagnostic.location.line_number, 1); assert_eq!(diagnostic.location.raw_char_position, 3); assert_eq!(diagnostic.input, "@"); } #[test] fn unknown_start_condition_reports_invalid_state_without_changing_state() { let mut lexer = lexer("value"); assert_eq!( lexer.yy_begin("MISSING".to_owned()), Err(Error::Parse { position: 0, context: "unknown lexer start condition" }) ); assert_eq!(lexer.m_state, "YYINITIAL"); assert_eq!( lexer.diagnostics()[0].category, DiagnosticCategory::InvalidState ); } #[test] fn source_reader_removes_both_comment_forms_and_keeps_newlines() { let mut reader = CsReader::new_with_string("a/*x\ny*/b//z\nc".to_owned()).expect("reader"); assert_eq!(reader.read_line().expect("line"), "a"); assert_eq!(reader.read_line().expect("line"), "b"); assert_eq!(reader.read_line().expect("line"), "c"); assert!(reader.eof().expect("eof")); } #[test] fn source_reader_normalizes_crlf_and_lone_cr() { let mut reader = CsReader::new_with_string("a\r\nb\rc\n".to_owned()).expect("reader"); assert_eq!(reader.read_line().expect("line"), "a"); assert_eq!(reader.read_line().expect("line"), "b"); assert_eq!(reader.read_line().expect("line"), "c"); } #[test] fn source_reader_rejects_unterminated_block_comment_at_eof() { assert_eq!( CsReader::new_with_string("before /* never closed".to_owned()).expect_err("invalid"), Error::Parse { position: 22, context: "unterminated block comment" } ); } #[test] fn line_directive_updates_file_and_logical_line() { let reader = CsReader::new_with_string("#line 700 \"table.lex\"\nword\n".to_owned()).expect("reader"); assert_eq!(reader.fname, "table.lex"); let info = SourceLineInfo::new_with_line_manager_int32(reader.lm, 1).expect("location"); assert_eq!(info.line_number, 700); } #[test] fn utf16_input_decoding_and_surrogate_validation_are_deterministic() { let mut reader = CsReader::from_bytes( &[0xFF, 0xFE, b'A', 0, 0x3D, 0xD8, 0, 0xDE], InputEncoding::Utf16Le, "source.lex", ) .expect("utf16"); assert_eq!(reader.read_line().expect("line"), "A😀"); let mut bom_override = CsReader::from_bytes(&[0xFE, 0xFF, 0, b'B'], InputEncoding::Utf16Le, "source.lex") .expect("BOM override"); assert_eq!(bom_override.read_line().expect("line"), "B"); assert!(matches!( CsReader::from_bytes(&[0x00, 0xD8], InputEncoding::Utf16Le, "bad"), Err(Error::Parse { position: 0, context: "source contains an unpaired UTF-16 surrogate" }) )); } #[test] fn ascii_input_rejects_non_ascii_instead_of_lossily_replacing_it() { assert!(matches!( CsReader::from_bytes(&[b'a', 0x80], InputEncoding::Ascii, "bad"), Err(Error::Parse { position: 1, context: "source contains a non-ASCII byte" }) )); } #[test] fn category_predicates_match_dotnet_values_and_groups() { let mut table = YyLexer::new(ErrorHandler::default()).expect("table"); let punctuation = table.get_test("Punctuation").expect("punctuation"); let whitespace = table.get_test("WhiteSpace").expect("whitespace"); let states = vec![ DfaState::default() .transition(punctuation, 1) .transition(whitespace, 2), DfaState::default().accepting(accept("PUNCT", 1, LexerAction::Emit, None)), DfaState::default().accepting(accept("SPACE", 2, LexerAction::Emit, None)), ]; let dfa = Dfa::from_states(states, 0).expect("dfa"); let mut action = -1; assert_eq!(dfa.match_("—".to_owned(), 0, &mut action), Ok(1)); assert_eq!(dfa.match_("\t".to_owned(), 0, &mut action), Ok(1)); assert_eq!(dfa.match_("\0".to_owned(), 0, &mut action), Ok(-1)); assert!( libremetaverse_lsl_tools::CatTest::new(UnicodeCategory(16)) .expect("surrogate") .test(Utf16CodeUnit(0xD800)) .expect("test") ); assert_eq!(table.get_test("MissingClass"), Err(Error::Argument)); assert_eq!( table.erh.diagnostics()[0].category, DiagnosticCategory::UnknownCharacterSet ); } #[test] fn dfa_uses_maximum_munch_and_exposes_action_number() { let states = vec![ DfaState::default().transition(CharacterMatcher::Exact(b'a'.into()), 1), DfaState::default() .transition(CharacterMatcher::Exact(b'b'.into()), 2) .accepting(accept("A", 1, LexerAction::Emit, None)), DfaState::default().accepting(accept("AB", 2, LexerAction::Emit, None)), ]; let dfa = Dfa::from_states(states, 0).expect("dfa"); let mut action = -1; assert_eq!(dfa.match_("abc".to_owned(), 0, &mut action), Ok(2)); assert_eq!(action, 2); } #[test] fn deterministic_table_output_is_independent_of_hash_iteration() { let table = language_table(); let first = Arc::new(Mutex::new(Vec::new())); table .emit_dfa(Box::new(SharedWriter(first.clone()))) .expect("emit first"); let second = Arc::new(Mutex::new(Vec::new())); table .emit_dfa(Box::new(SharedWriter(second.clone()))) .expect("emit second"); let first = first.lock().expect("first output").clone(); let second = second.lock().expect("second output").clone(); assert_eq!(first, second); assert!( String::from_utf8(first) .expect("utf8") .starts_with("LSLLEXER 1\n") ); } struct SharedWriter(Arc>>); impl std::io::Write for SharedWriter { fn write(&mut self, bytes: &[u8]) -> std::io::Result { self.0 .lock() .map_err(|_| std::io::Error::other("poisoned output"))? .extend_from_slice(bytes); Ok(bytes.len()) } fn flush(&mut self) -> std::io::Result<()> { Ok(()) } } #[test] fn compatibility_enumerator_resets_to_the_first_token() { let lexer = lexer("one two"); let mut enumerator = lexer.get_enumerator().expect("enumerator"); assert!(enumerator.move_next().expect("first")); assert_eq!(enumerator.current().yytext(), "one"); assert!(enumerator.move_next().expect("second")); assert_eq!(enumerator.current().yytext(), "two"); enumerator.reset().expect("reset"); assert!(enumerator.move_next().expect("first again")); assert_eq!(enumerator.current().yytext(), "one"); } #[test] fn error_handler_counts_reports_and_honors_throw_mode() { let error = CSToolsException::new_with_int32_string(43, "bad encoding".to_owned()).expect("diagnostic"); let mut collecting = ErrorHandler::new_with_constructor().expect("handler"); collecting.error(error.clone()).expect("reported"); assert_eq!(collecting.counter, 1); assert_eq!( collecting.diagnostics()[0].category, DiagnosticCategory::Encoding ); let mut throwing = ErrorHandler::new_with_boolean(true).expect("handler"); assert_eq!(throwing.error(error), Err(Error::InvalidOperation)); assert_eq!(throwing.counter, 1); assert!(throwing.diagnostics().is_empty()); } #[test] fn line_manager_backtracking_removes_future_lines() { let mut manager = LineManager::new().expect("manager"); manager.newline(4).expect("line 2"); let first_on_second = SourceLineInfo::new_with_line_manager_int32(manager.clone(), 4).expect("location"); assert_eq!(first_on_second.raw_char_position, 1); manager.newline(8).expect("line 3"); manager.backto(6).expect("rewind"); assert_eq!(manager.lines, 2); assert_eq!(manager.end, 8); } #[test] fn object_list_preserves_push_add_top_pop_and_index_order() { let mut values = ObjectList::new().expect("list"); values.add(Object::Integer(2)).expect("add"); values.push(Object::Integer(1)).expect("push"); assert_eq!(values.count(), 2); assert_eq!(values.top(), Object::Integer(1)); assert_eq!(values.item(1), Object::Integer(2)); assert_eq!(values.pop(), Ok(Object::Integer(1))); } #[test] fn matcher_set_is_value_based_and_bounded() { let matcher = CharacterMatcher::Set(BTreeSet::from([b'x'.into(), b'y'.into()])); let states = vec![ DfaState::default().transition(matcher, 1), DfaState::default().accepting(accept("XY", 1, LexerAction::Emit, None)), ]; let dfa = Dfa::from_states(states, 0).expect("dfa"); let mut action = 0; assert_eq!(dfa.match_("y".to_owned(), 0, &mut action), Ok(1)); assert_eq!(dfa.match_("z".to_owned(), 0, &mut action), Ok(-1)); }