//! Focused compatibility fixtures for the native issue #82 parser boundary. use std::sync::{Arc, Mutex}; use libremetaverse_lsl_tools::{ Associativity, CSymbol, CSymbolSymType, CharacterMatcher, Dfa, DfaAccept, DfaState, DiagnosticCategory, Error, ErrorHandler, Grammar, Lexer, LexerAction, LslError, ParseTree, Parser, ParserEntry, Precedence, PrecedencePrecType, Production, ResWds, SymbolSet, TokenDefinition, YyLexer, }; const EOF: i32 = 2; const ID: i32 = 3; const INTEGER: i32 = 4; const PLUS: i32 = 5; const STAR: i32 = 6; const MINUS: i32 = 7; const LPAREN: i32 = 8; const RPAREN: i32 = 9; const SEMICOLON: i32 = 10; const COMMA: i32 = 11; const STRING: i32 = 12; const LBRACE: i32 = 13; const RBRACE: i32 = 14; const EQUAL: i32 = 15; const DEFAULT: i32 = 20; const STATE_ENTRY: i32 = 21; fn definition(name: &str, number: i32) -> TokenDefinition { TokenDefinition::new(name, number).expect("valid token") } fn accept(name: &str, number: i32, action: LexerAction) -> DfaAccept { DfaAccept { token: definition(name, number), action, action_number: number, reserved_words: (number == ID).then(|| "keywords".to_owned()), } } fn parser_lexer(source: &str) -> Lexer { use libremetaverse_lsl_tools::DotNetUnicodeCategory as Category; let letters = vec![ Category::UppercaseLetter, Category::LowercaseLetter, Category::TitlecaseLetter, Category::ModifierLetter, Category::OtherLetter, ]; let whitespace = vec![ Category::Control, Category::SpaceSeparator, Category::LineSeparator, Category::ParagraphSeparator, ]; let states = vec![ DfaState::default() .transition(CharacterMatcher::Categories(letters.clone()), 1) .transition(CharacterMatcher::Exact(b'_'.into()), 1) .transition(CharacterMatcher::Category(Category::DecimalDigitNumber), 2) .transition(CharacterMatcher::Categories(whitespace.clone()), 3) .transition(CharacterMatcher::Exact(b'+'.into()), 4) .transition(CharacterMatcher::Exact(b'*'.into()), 5) .transition(CharacterMatcher::Exact(b'-'.into()), 6) .transition(CharacterMatcher::Exact(b'('.into()), 7) .transition(CharacterMatcher::Exact(b')'.into()), 8) .transition(CharacterMatcher::Exact(b';'.into()), 9) .transition(CharacterMatcher::Exact(b','.into()), 10) .transition(CharacterMatcher::Exact(b'{'.into()), 11) .transition(CharacterMatcher::Exact(b'}'.into()), 12) .transition(CharacterMatcher::Exact(b'"'.into()), 13) .transition(CharacterMatcher::Exact(b'='.into()), 15), DfaState::default() .transition(CharacterMatcher::Categories(letters.clone()), 1) .transition(CharacterMatcher::Exact(b'_'.into()), 1) .transition(CharacterMatcher::Category(Category::DecimalDigitNumber), 1) .accepting(accept("ID", ID, LexerAction::Emit)), DfaState::default() .transition(CharacterMatcher::Category(Category::DecimalDigitNumber), 2) .accepting(accept("INTEGER", INTEGER, LexerAction::Emit)), DfaState::default() .transition(CharacterMatcher::Categories(whitespace), 3) .accepting(accept("WS", 30, LexerAction::Skip)), DfaState::default().accepting(accept("PLUS", PLUS, LexerAction::Emit)), DfaState::default().accepting(accept("STAR", STAR, LexerAction::Emit)), DfaState::default().accepting(accept("MINUS", MINUS, LexerAction::Emit)), DfaState::default().accepting(accept("LPAREN", LPAREN, LexerAction::Emit)), DfaState::default().accepting(accept("RPAREN", RPAREN, LexerAction::Emit)), DfaState::default().accepting(accept("SEMICOLON", SEMICOLON, LexerAction::Emit)), DfaState::default().accepting(accept("COMMA", COMMA, LexerAction::Emit)), DfaState::default().accepting(accept("LBRACE", LBRACE, LexerAction::Emit)), DfaState::default().accepting(accept("RBRACE", RBRACE, LexerAction::Emit)), DfaState::default() .transition(CharacterMatcher::Categories(letters), 13) .transition(CharacterMatcher::Exact(b' '.into()), 13) .transition(CharacterMatcher::Exact(b'"'.into()), 14), DfaState::default().accepting(accept("STRING", STRING, LexerAction::Emit)), DfaState::default().accepting(accept("EQUAL", EQUAL, LexerAction::Emit)), ]; let mut table = YyLexer::new(ErrorHandler::default()).expect("lexer table"); table.using_eof = true; table .set_start_dfa("YYINITIAL", Dfa::from_states(states, 0).expect("dfa")) .expect("start state"); table .set_reserved_words( "keywords", ResWds::from_pairs( [ ("default", definition("DEFAULT", DEFAULT)), ("state_entry", definition("STATE_ENTRY", STATE_ENTRY)), ], false, ) .expect("reserved words"), ) .expect("reserved table"); let mut lexer = Lexer::new(table).expect("lexer"); lexer.start_with_string(source.to_owned()).expect("source"); lexer } fn add_common_terminals(grammar: &mut Grammar) { for (name, number) in [ ("EOF", EOF), ("ID", ID), ("INTEGER", INTEGER), ("PLUS", PLUS), ("STAR", STAR), ("MINUS", MINUS), ("LPAREN", LPAREN), ("RPAREN", RPAREN), ("SEMICOLON", SEMICOLON), ("COMMA", COMMA), ("STRING", STRING), ("LBRACE", LBRACE), ("RBRACE", RBRACE), ("EQUAL", EQUAL), ("DEFAULT", DEFAULT), ("STATE_ENTRY", STATE_ENTRY), ("Error", 0), ] { grammar.add_symbol(name, number, true).expect("terminal"); } } fn expression_parser(source: &str) -> Parser { const EXPRESSION: i32 = 100; let mut grammar = Grammar::new(EXPRESSION, EOF).expect("grammar"); add_common_terminals(&mut grammar); grammar .add_symbol("Expression", EXPRESSION, false) .expect("nonterminal"); grammar .set_precedence(PLUS, 1, Associativity::Left) .expect("plus precedence"); grammar .set_precedence(STAR, 2, Associativity::Left) .expect("star precedence"); grammar .set_precedence(MINUS, 3, Associativity::Right) .expect("minus precedence"); grammar .add_production(EXPRESSION, vec![EXPRESSION, PLUS, EXPRESSION]) .expect("plus"); grammar .add_production(EXPRESSION, vec![EXPRESSION, STAR, EXPRESSION]) .expect("multiply"); grammar .add_production_with_precedence(EXPRESSION, vec![MINUS, EXPRESSION], Some(MINUS)) .expect("unary"); grammar .add_production(EXPRESSION, vec![LPAREN, EXPRESSION, RPAREN]) .expect("parentheses"); grammar .add_production(EXPRESSION, vec![ID]) .expect("identifier"); Parser::new(grammar.build().expect("table"), parser_lexer(source)).expect("parser") } fn tree(symbol: &libremetaverse_lsl_tools::SYMBOL) -> &ParseTree { symbol .m_dollar .downcast_ref::() .expect("parse tree") } #[test] fn precedence_and_associativity_choose_the_reference_tree() { let mut parser = expression_parser("a + b * c + d"); let result = parser .parse_with_string("a + b * c + d".to_owned()) .expect("parse"); let root = tree(&result); assert_eq!(root.name, "Expression"); assert_eq!(root.children[1].number, PLUS); assert_eq!(root.children[0].children[1].number, PLUS); assert_eq!(root.children[0].children[2].children[1].number, STAR); assert!(!parser.m_symbols.conflicts().is_empty()); } #[test] fn right_associative_unary_precedence_is_deterministic() { let mut parser = expression_parser("- - a"); let result = parser.parse_with_string("- - a".to_owned()).expect("parse"); let root = tree(&result); assert_eq!(root.children[0].number, MINUS); assert_eq!(root.children[1].children[0].number, MINUS); } #[test] fn nonassociative_conflict_rejects_a_chained_operator() { const EXPRESSION: i32 = 100; let mut grammar = Grammar::new(EXPRESSION, EOF).expect("grammar"); add_common_terminals(&mut grammar); grammar .add_symbol("Expression", EXPRESSION, false) .expect("nonterminal"); grammar .set_precedence(PLUS, 1, Associativity::Nonassoc) .expect("precedence"); grammar .add_production(EXPRESSION, vec![EXPRESSION, PLUS, EXPRESSION]) .expect("binary"); grammar .add_production(EXPRESSION, vec![ID]) .expect("identifier"); let mut parser = Parser::new(grammar.build().expect("table"), parser_lexer("a+a+a")).expect("parser"); assert!(matches!( parser.parse_with_string("a+a+a".to_owned()), Err(Error::Parse { context: "syntax error", .. }) )); assert_eq!( parser.diagnostics().expect("diagnostics")[0].category, DiagnosticCategory::Syntax ); } #[test] fn empty_production_accepts_an_empty_input() { const START: i32 = 100; const ITEMS: i32 = 101; let mut grammar = Grammar::new(START, EOF).expect("grammar"); add_common_terminals(&mut grammar); grammar.add_symbol("Start", START, false).expect("start"); grammar.add_symbol("Items", ITEMS, false).expect("items"); grammar.add_production(START, vec![ITEMS]).expect("root"); grammar.add_production(ITEMS, vec![]).expect("empty"); let mut parser = Parser::new(grammar.build().expect("table"), parser_lexer("")).expect("parser"); let result = parser.parse_with_string(String::new()).expect("parse"); assert_eq!(tree(&result).children[0].name, "Items"); } #[test] fn lalr_lookaheads_avoid_the_classic_slr_assignment_conflict() { const START: i32 = 100; const LEFT: i32 = 101; const RIGHT: i32 = 102; let mut grammar = Grammar::new(START, EOF).expect("grammar"); add_common_terminals(&mut grammar); for (name, number) in [("Start", START), ("Left", LEFT), ("Right", RIGHT)] { grammar .add_symbol(name, number, false) .expect("nonterminal"); } grammar .add_production(START, vec![LEFT, EQUAL, RIGHT]) .expect("assignment"); grammar.add_production(START, vec![RIGHT]).expect("value"); grammar .add_production(LEFT, vec![STAR, RIGHT]) .expect("dereference"); grammar.add_production(LEFT, vec![ID]).expect("name"); grammar.add_production(RIGHT, vec![LEFT]).expect("right"); let table = grammar.build().expect("LALR table"); assert!(table.conflicts().is_empty()); for source in ["a = b", "* a = b"] { let mut parser = Parser::new(table.clone(), parser_lexer(source)).expect("parser"); let result = parser .parse_with_string(source.to_owned()) .expect("assignment parse"); assert_eq!(tree(&result).name, "Start"); } } #[test] fn error_token_recovery_discards_input_and_returns_recovered_tree() { const START: i32 = 100; const LIST: i32 = 101; const STATEMENT: i32 = 102; let mut grammar = Grammar::new(START, EOF).expect("grammar"); add_common_terminals(&mut grammar); for (name, number) in [("Start", START), ("List", LIST), ("Statement", STATEMENT)] { grammar .add_symbol(name, number, false) .expect("nonterminal"); } grammar.add_production(START, vec![LIST]).expect("start"); grammar .add_production(LIST, vec![LIST, STATEMENT]) .expect("list"); grammar .add_production(LIST, vec![STATEMENT]) .expect("single"); grammar .add_production(STATEMENT, vec![ID, SEMICOLON]) .expect("statement"); grammar .add_production(STATEMENT, vec![0, ID, SEMICOLON]) .expect("recovery"); let mut parser = Parser::new(grammar.build().expect("table"), parser_lexer("a; + b;")).expect("parser"); let result = parser .parse_with_string("a; + b;".to_owned()) .expect("recovered parse"); assert_eq!(result.yyname(), "recoveredError"); let diagnostics = parser.diagnostics().expect("diagnostics"); assert_eq!(diagnostics[0].category, DiagnosticCategory::Syntax); assert_eq!(diagnostics[1].category, DiagnosticCategory::ParserRecovery); } #[test] fn representative_lsl_script_parses_to_expected_model() { const SCRIPT: i32 = 100; const EVENT: i32 = 101; const CALL: i32 = 102; let source = r#"default { state_entry() { llSay(0, "hello"); } }"#; let mut grammar = Grammar::new(SCRIPT, EOF).expect("grammar"); add_common_terminals(&mut grammar); for (name, number) in [("Script", SCRIPT), ("Event", EVENT), ("Call", CALL)] { grammar .add_symbol(name, number, false) .expect("nonterminal"); } grammar .add_production(SCRIPT, vec![DEFAULT, LBRACE, EVENT, RBRACE]) .expect("script"); grammar .add_production( EVENT, vec![STATE_ENTRY, LPAREN, RPAREN, LBRACE, CALL, SEMICOLON, RBRACE], ) .expect("event"); grammar .add_production(CALL, vec![ID, LPAREN, INTEGER, COMMA, STRING, RPAREN]) .expect("call"); let mut parser = Parser::new(grammar.build().expect("table"), parser_lexer(source)).expect("parser"); let result = parser .parse_with_string(source.to_owned()) .expect("LSL source"); let root = tree(&result); assert_eq!(root.name, "Script"); assert_eq!(root.children[2].name, "Event"); assert_eq!(root.children[2].children[4].name, "Call"); assert_eq!( root.children[2].children[4].children[0].text.as_deref(), Some("llSay") ); } #[test] fn parser_table_emission_is_byte_deterministic() { let parser = expression_parser("a"); let first = Arc::new(Mutex::new(Vec::new())); parser .m_symbols .emit(Box::new(SharedWriter(first.clone()))) .expect("first"); let second = Arc::new(Mutex::new(Vec::new())); parser .m_symbols .emit(Box::new(SharedWriter(second.clone()))) .expect("second"); assert_eq!( first.lock().expect("first output").as_slice(), second.lock().expect("second output").as_slice() ); } struct SharedWriter(Arc>>); impl std::io::Write for SharedWriter { fn write(&mut self, bytes: &[u8]) -> std::io::Result { self.0 .lock() .map_err(|_| std::io::Error::other("poisoned output"))? .extend_from_slice(bytes); Ok(bytes.len()) } fn flush(&mut self) -> std::io::Result<()> { Ok(()) } } #[test] fn malformed_corpus_terminates_without_panics_or_unbounded_growth() { for source in ["+", "a +", "((a", "a * * b", "-", "a )", "@"] { let mut parser = expression_parser(source); let _ = parser.parse_with_string(source.to_owned()); assert!(parser.diagnostics().expect("diagnostics").len() <= 2); } } #[test] fn mapped_symbol_set_production_precedence_and_entry_apis_are_live() { let terminal = CSymbol::native("PLUS", PLUS, CSymbolSymType::Terminal).expect("terminal"); let nonterminal = CSymbol::native("Expression", 100, CSymbolSymType::Nonterminal).expect("nonterminal"); let mut set = SymbolSet::native(); assert!(set.check_in(terminal.clone()).expect("insert")); assert!(!set.check_in(terminal.clone()).expect("deduplicate")); assert!(set.contains(terminal.clone()).expect("contains")); assert!(terminal.matches("PLUS".to_owned()).expect("symbol text")); let mut followed = nonterminal.clone(); assert!( followed .add_follow(terminal.m_first.clone()) .expect("first follow insertion") ); assert_eq!(nonterminal.m_follow.count(), 1); let nullable = CSymbol::native("Optional", 101, CSymbolSymType::Nonterminal).expect("nullable symbol"); let _empty = Production::native(8, nullable.clone(), vec![]).expect("empty production"); assert!(nullable.is_nullable().expect("registered empty production")); let mut production = Production::native(7, nonterminal.clone(), vec![]).expect("production"); production.add_to_rhs(nonterminal.clone()).expect("lhs"); production.add_to_rhs(terminal.clone()).expect("operator"); assert_eq!(production.prefix(1).expect("prefix")[0].m_yynum, 100); let target = CSymbol::native("Target", 102, CSymbolSymType::Nonterminal).expect("target"); production .add_first(target.clone(), 1) .expect("FIRST/FOLLOW propagation"); assert!( target .m_follow .contains(terminal.clone()) .expect("shared follow mutation") ); let precedence = Precedence::first(PrecedencePrecType::Left, 5).expect("precedence"); assert_eq!( Precedence::check_with_precedence_prec_type_int32(precedence, PrecedencePrecType::Left, 0,), Ok(5) ); let parser = expression_parser("a"); let mut lexer = parser.m_lexer.clone(); let symbol = lexer.next_token().expect("lexing").expect("token"); let mut entry = ParserEntry::default(); assert_eq!( symbol.pass_(parser.m_symbols.clone(), 0, &mut entry), Ok(true) ); assert_eq!(entry.str(), "shift 1"); let lsl_error = LslError { state: 4, sym: libremetaverse_lsl_tools::SYMBOL::new_with_lexer(parser.m_lexer.clone()) .expect("symbol"), }; assert!(lsl_error.to_string().contains("state 4")); }