475 lines
17 KiB
Rust
475 lines
17 KiB
Rust
//! Focused compatibility fixtures for the native issue #82 parser boundary.
|
|
|
|
use std::sync::{Arc, Mutex};
|
|
|
|
use libremetaverse_lsl_tools::{
|
|
Associativity, CSymbol, CSymbolSymType, CharacterMatcher, Dfa, DfaAccept, DfaState,
|
|
DiagnosticCategory, Error, ErrorHandler, Grammar, Lexer, LexerAction, LslError, ParseTree,
|
|
Parser, ParserEntry, Precedence, PrecedencePrecType, Production, ResWds, SymbolSet,
|
|
TokenDefinition, YyLexer,
|
|
};
|
|
|
|
const EOF: i32 = 2;
|
|
const ID: i32 = 3;
|
|
const INTEGER: i32 = 4;
|
|
const PLUS: i32 = 5;
|
|
const STAR: i32 = 6;
|
|
const MINUS: i32 = 7;
|
|
const LPAREN: i32 = 8;
|
|
const RPAREN: i32 = 9;
|
|
const SEMICOLON: i32 = 10;
|
|
const COMMA: i32 = 11;
|
|
const STRING: i32 = 12;
|
|
const LBRACE: i32 = 13;
|
|
const RBRACE: i32 = 14;
|
|
const EQUAL: i32 = 15;
|
|
const DEFAULT: i32 = 20;
|
|
const STATE_ENTRY: i32 = 21;
|
|
|
|
fn definition(name: &str, number: i32) -> TokenDefinition {
|
|
TokenDefinition::new(name, number).expect("valid token")
|
|
}
|
|
|
|
fn accept(name: &str, number: i32, action: LexerAction) -> DfaAccept {
|
|
DfaAccept {
|
|
token: definition(name, number),
|
|
action,
|
|
action_number: number,
|
|
reserved_words: (number == ID).then(|| "keywords".to_owned()),
|
|
}
|
|
}
|
|
|
|
fn parser_lexer(source: &str) -> Lexer {
|
|
use libremetaverse_lsl_tools::DotNetUnicodeCategory as Category;
|
|
|
|
let letters = vec![
|
|
Category::UppercaseLetter,
|
|
Category::LowercaseLetter,
|
|
Category::TitlecaseLetter,
|
|
Category::ModifierLetter,
|
|
Category::OtherLetter,
|
|
];
|
|
let whitespace = vec![
|
|
Category::Control,
|
|
Category::SpaceSeparator,
|
|
Category::LineSeparator,
|
|
Category::ParagraphSeparator,
|
|
];
|
|
let states = vec![
|
|
DfaState::default()
|
|
.transition(CharacterMatcher::Categories(letters.clone()), 1)
|
|
.transition(CharacterMatcher::Exact(b'_'.into()), 1)
|
|
.transition(CharacterMatcher::Category(Category::DecimalDigitNumber), 2)
|
|
.transition(CharacterMatcher::Categories(whitespace.clone()), 3)
|
|
.transition(CharacterMatcher::Exact(b'+'.into()), 4)
|
|
.transition(CharacterMatcher::Exact(b'*'.into()), 5)
|
|
.transition(CharacterMatcher::Exact(b'-'.into()), 6)
|
|
.transition(CharacterMatcher::Exact(b'('.into()), 7)
|
|
.transition(CharacterMatcher::Exact(b')'.into()), 8)
|
|
.transition(CharacterMatcher::Exact(b';'.into()), 9)
|
|
.transition(CharacterMatcher::Exact(b','.into()), 10)
|
|
.transition(CharacterMatcher::Exact(b'{'.into()), 11)
|
|
.transition(CharacterMatcher::Exact(b'}'.into()), 12)
|
|
.transition(CharacterMatcher::Exact(b'"'.into()), 13)
|
|
.transition(CharacterMatcher::Exact(b'='.into()), 15),
|
|
DfaState::default()
|
|
.transition(CharacterMatcher::Categories(letters.clone()), 1)
|
|
.transition(CharacterMatcher::Exact(b'_'.into()), 1)
|
|
.transition(CharacterMatcher::Category(Category::DecimalDigitNumber), 1)
|
|
.accepting(accept("ID", ID, LexerAction::Emit)),
|
|
DfaState::default()
|
|
.transition(CharacterMatcher::Category(Category::DecimalDigitNumber), 2)
|
|
.accepting(accept("INTEGER", INTEGER, LexerAction::Emit)),
|
|
DfaState::default()
|
|
.transition(CharacterMatcher::Categories(whitespace), 3)
|
|
.accepting(accept("WS", 30, LexerAction::Skip)),
|
|
DfaState::default().accepting(accept("PLUS", PLUS, LexerAction::Emit)),
|
|
DfaState::default().accepting(accept("STAR", STAR, LexerAction::Emit)),
|
|
DfaState::default().accepting(accept("MINUS", MINUS, LexerAction::Emit)),
|
|
DfaState::default().accepting(accept("LPAREN", LPAREN, LexerAction::Emit)),
|
|
DfaState::default().accepting(accept("RPAREN", RPAREN, LexerAction::Emit)),
|
|
DfaState::default().accepting(accept("SEMICOLON", SEMICOLON, LexerAction::Emit)),
|
|
DfaState::default().accepting(accept("COMMA", COMMA, LexerAction::Emit)),
|
|
DfaState::default().accepting(accept("LBRACE", LBRACE, LexerAction::Emit)),
|
|
DfaState::default().accepting(accept("RBRACE", RBRACE, LexerAction::Emit)),
|
|
DfaState::default()
|
|
.transition(CharacterMatcher::Categories(letters), 13)
|
|
.transition(CharacterMatcher::Exact(b' '.into()), 13)
|
|
.transition(CharacterMatcher::Exact(b'"'.into()), 14),
|
|
DfaState::default().accepting(accept("STRING", STRING, LexerAction::Emit)),
|
|
DfaState::default().accepting(accept("EQUAL", EQUAL, LexerAction::Emit)),
|
|
];
|
|
let mut table = YyLexer::new(ErrorHandler::default()).expect("lexer table");
|
|
table.using_eof = true;
|
|
table
|
|
.set_start_dfa("YYINITIAL", Dfa::from_states(states, 0).expect("dfa"))
|
|
.expect("start state");
|
|
table
|
|
.set_reserved_words(
|
|
"keywords",
|
|
ResWds::from_pairs(
|
|
[
|
|
("default", definition("DEFAULT", DEFAULT)),
|
|
("state_entry", definition("STATE_ENTRY", STATE_ENTRY)),
|
|
],
|
|
false,
|
|
)
|
|
.expect("reserved words"),
|
|
)
|
|
.expect("reserved table");
|
|
let mut lexer = Lexer::new(table).expect("lexer");
|
|
lexer.start_with_string(source.to_owned()).expect("source");
|
|
lexer
|
|
}
|
|
|
|
fn add_common_terminals(grammar: &mut Grammar) {
|
|
for (name, number) in [
|
|
("EOF", EOF),
|
|
("ID", ID),
|
|
("INTEGER", INTEGER),
|
|
("PLUS", PLUS),
|
|
("STAR", STAR),
|
|
("MINUS", MINUS),
|
|
("LPAREN", LPAREN),
|
|
("RPAREN", RPAREN),
|
|
("SEMICOLON", SEMICOLON),
|
|
("COMMA", COMMA),
|
|
("STRING", STRING),
|
|
("LBRACE", LBRACE),
|
|
("RBRACE", RBRACE),
|
|
("EQUAL", EQUAL),
|
|
("DEFAULT", DEFAULT),
|
|
("STATE_ENTRY", STATE_ENTRY),
|
|
("Error", 0),
|
|
] {
|
|
grammar.add_symbol(name, number, true).expect("terminal");
|
|
}
|
|
}
|
|
|
|
fn expression_parser(source: &str) -> Parser {
|
|
const EXPRESSION: i32 = 100;
|
|
let mut grammar = Grammar::new(EXPRESSION, EOF).expect("grammar");
|
|
add_common_terminals(&mut grammar);
|
|
grammar
|
|
.add_symbol("Expression", EXPRESSION, false)
|
|
.expect("nonterminal");
|
|
grammar
|
|
.set_precedence(PLUS, 1, Associativity::Left)
|
|
.expect("plus precedence");
|
|
grammar
|
|
.set_precedence(STAR, 2, Associativity::Left)
|
|
.expect("star precedence");
|
|
grammar
|
|
.set_precedence(MINUS, 3, Associativity::Right)
|
|
.expect("minus precedence");
|
|
grammar
|
|
.add_production(EXPRESSION, vec![EXPRESSION, PLUS, EXPRESSION])
|
|
.expect("plus");
|
|
grammar
|
|
.add_production(EXPRESSION, vec![EXPRESSION, STAR, EXPRESSION])
|
|
.expect("multiply");
|
|
grammar
|
|
.add_production_with_precedence(EXPRESSION, vec![MINUS, EXPRESSION], Some(MINUS))
|
|
.expect("unary");
|
|
grammar
|
|
.add_production(EXPRESSION, vec![LPAREN, EXPRESSION, RPAREN])
|
|
.expect("parentheses");
|
|
grammar
|
|
.add_production(EXPRESSION, vec![ID])
|
|
.expect("identifier");
|
|
Parser::new(grammar.build().expect("table"), parser_lexer(source)).expect("parser")
|
|
}
|
|
|
|
fn tree(symbol: &libremetaverse_lsl_tools::SYMBOL) -> &ParseTree {
|
|
symbol
|
|
.m_dollar
|
|
.downcast_ref::<ParseTree>()
|
|
.expect("parse tree")
|
|
}
|
|
|
|
#[test]
|
|
fn precedence_and_associativity_choose_the_reference_tree() {
|
|
let mut parser = expression_parser("a + b * c + d");
|
|
let result = parser
|
|
.parse_with_string("a + b * c + d".to_owned())
|
|
.expect("parse");
|
|
let root = tree(&result);
|
|
assert_eq!(root.name, "Expression");
|
|
assert_eq!(root.children[1].number, PLUS);
|
|
assert_eq!(root.children[0].children[1].number, PLUS);
|
|
assert_eq!(root.children[0].children[2].children[1].number, STAR);
|
|
assert!(!parser.m_symbols.conflicts().is_empty());
|
|
}
|
|
|
|
#[test]
|
|
fn right_associative_unary_precedence_is_deterministic() {
|
|
let mut parser = expression_parser("- - a");
|
|
let result = parser.parse_with_string("- - a".to_owned()).expect("parse");
|
|
let root = tree(&result);
|
|
assert_eq!(root.children[0].number, MINUS);
|
|
assert_eq!(root.children[1].children[0].number, MINUS);
|
|
}
|
|
|
|
#[test]
|
|
fn nonassociative_conflict_rejects_a_chained_operator() {
|
|
const EXPRESSION: i32 = 100;
|
|
let mut grammar = Grammar::new(EXPRESSION, EOF).expect("grammar");
|
|
add_common_terminals(&mut grammar);
|
|
grammar
|
|
.add_symbol("Expression", EXPRESSION, false)
|
|
.expect("nonterminal");
|
|
grammar
|
|
.set_precedence(PLUS, 1, Associativity::Nonassoc)
|
|
.expect("precedence");
|
|
grammar
|
|
.add_production(EXPRESSION, vec![EXPRESSION, PLUS, EXPRESSION])
|
|
.expect("binary");
|
|
grammar
|
|
.add_production(EXPRESSION, vec![ID])
|
|
.expect("identifier");
|
|
let mut parser =
|
|
Parser::new(grammar.build().expect("table"), parser_lexer("a+a+a")).expect("parser");
|
|
assert!(matches!(
|
|
parser.parse_with_string("a+a+a".to_owned()),
|
|
Err(Error::Parse {
|
|
context: "syntax error",
|
|
..
|
|
})
|
|
));
|
|
assert_eq!(
|
|
parser.diagnostics().expect("diagnostics")[0].category,
|
|
DiagnosticCategory::Syntax
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn empty_production_accepts_an_empty_input() {
|
|
const START: i32 = 100;
|
|
const ITEMS: i32 = 101;
|
|
let mut grammar = Grammar::new(START, EOF).expect("grammar");
|
|
add_common_terminals(&mut grammar);
|
|
grammar.add_symbol("Start", START, false).expect("start");
|
|
grammar.add_symbol("Items", ITEMS, false).expect("items");
|
|
grammar.add_production(START, vec![ITEMS]).expect("root");
|
|
grammar.add_production(ITEMS, vec![]).expect("empty");
|
|
let mut parser =
|
|
Parser::new(grammar.build().expect("table"), parser_lexer("")).expect("parser");
|
|
let result = parser.parse_with_string(String::new()).expect("parse");
|
|
assert_eq!(tree(&result).children[0].name, "Items");
|
|
}
|
|
|
|
#[test]
|
|
fn lalr_lookaheads_avoid_the_classic_slr_assignment_conflict() {
|
|
const START: i32 = 100;
|
|
const LEFT: i32 = 101;
|
|
const RIGHT: i32 = 102;
|
|
let mut grammar = Grammar::new(START, EOF).expect("grammar");
|
|
add_common_terminals(&mut grammar);
|
|
for (name, number) in [("Start", START), ("Left", LEFT), ("Right", RIGHT)] {
|
|
grammar
|
|
.add_symbol(name, number, false)
|
|
.expect("nonterminal");
|
|
}
|
|
grammar
|
|
.add_production(START, vec![LEFT, EQUAL, RIGHT])
|
|
.expect("assignment");
|
|
grammar.add_production(START, vec![RIGHT]).expect("value");
|
|
grammar
|
|
.add_production(LEFT, vec![STAR, RIGHT])
|
|
.expect("dereference");
|
|
grammar.add_production(LEFT, vec![ID]).expect("name");
|
|
grammar.add_production(RIGHT, vec![LEFT]).expect("right");
|
|
let table = grammar.build().expect("LALR table");
|
|
assert!(table.conflicts().is_empty());
|
|
for source in ["a = b", "* a = b"] {
|
|
let mut parser = Parser::new(table.clone(), parser_lexer(source)).expect("parser");
|
|
let result = parser
|
|
.parse_with_string(source.to_owned())
|
|
.expect("assignment parse");
|
|
assert_eq!(tree(&result).name, "Start");
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn error_token_recovery_discards_input_and_returns_recovered_tree() {
|
|
const START: i32 = 100;
|
|
const LIST: i32 = 101;
|
|
const STATEMENT: i32 = 102;
|
|
let mut grammar = Grammar::new(START, EOF).expect("grammar");
|
|
add_common_terminals(&mut grammar);
|
|
for (name, number) in [("Start", START), ("List", LIST), ("Statement", STATEMENT)] {
|
|
grammar
|
|
.add_symbol(name, number, false)
|
|
.expect("nonterminal");
|
|
}
|
|
grammar.add_production(START, vec![LIST]).expect("start");
|
|
grammar
|
|
.add_production(LIST, vec![LIST, STATEMENT])
|
|
.expect("list");
|
|
grammar
|
|
.add_production(LIST, vec![STATEMENT])
|
|
.expect("single");
|
|
grammar
|
|
.add_production(STATEMENT, vec![ID, SEMICOLON])
|
|
.expect("statement");
|
|
grammar
|
|
.add_production(STATEMENT, vec![0, ID, SEMICOLON])
|
|
.expect("recovery");
|
|
let mut parser =
|
|
Parser::new(grammar.build().expect("table"), parser_lexer("a; + b;")).expect("parser");
|
|
let result = parser
|
|
.parse_with_string("a; + b;".to_owned())
|
|
.expect("recovered parse");
|
|
assert_eq!(result.yyname(), "recoveredError");
|
|
let diagnostics = parser.diagnostics().expect("diagnostics");
|
|
assert_eq!(diagnostics[0].category, DiagnosticCategory::Syntax);
|
|
assert_eq!(diagnostics[1].category, DiagnosticCategory::ParserRecovery);
|
|
}
|
|
|
|
#[test]
|
|
fn representative_lsl_script_parses_to_expected_model() {
|
|
const SCRIPT: i32 = 100;
|
|
const EVENT: i32 = 101;
|
|
const CALL: i32 = 102;
|
|
let source = r#"default { state_entry() { llSay(0, "hello"); } }"#;
|
|
let mut grammar = Grammar::new(SCRIPT, EOF).expect("grammar");
|
|
add_common_terminals(&mut grammar);
|
|
for (name, number) in [("Script", SCRIPT), ("Event", EVENT), ("Call", CALL)] {
|
|
grammar
|
|
.add_symbol(name, number, false)
|
|
.expect("nonterminal");
|
|
}
|
|
grammar
|
|
.add_production(SCRIPT, vec![DEFAULT, LBRACE, EVENT, RBRACE])
|
|
.expect("script");
|
|
grammar
|
|
.add_production(
|
|
EVENT,
|
|
vec![STATE_ENTRY, LPAREN, RPAREN, LBRACE, CALL, SEMICOLON, RBRACE],
|
|
)
|
|
.expect("event");
|
|
grammar
|
|
.add_production(CALL, vec![ID, LPAREN, INTEGER, COMMA, STRING, RPAREN])
|
|
.expect("call");
|
|
let mut parser =
|
|
Parser::new(grammar.build().expect("table"), parser_lexer(source)).expect("parser");
|
|
let result = parser
|
|
.parse_with_string(source.to_owned())
|
|
.expect("LSL source");
|
|
let root = tree(&result);
|
|
assert_eq!(root.name, "Script");
|
|
assert_eq!(root.children[2].name, "Event");
|
|
assert_eq!(root.children[2].children[4].name, "Call");
|
|
assert_eq!(
|
|
root.children[2].children[4].children[0].text.as_deref(),
|
|
Some("llSay")
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn parser_table_emission_is_byte_deterministic() {
|
|
let parser = expression_parser("a");
|
|
let first = Arc::new(Mutex::new(Vec::new()));
|
|
parser
|
|
.m_symbols
|
|
.emit(Box::new(SharedWriter(first.clone())))
|
|
.expect("first");
|
|
let second = Arc::new(Mutex::new(Vec::new()));
|
|
parser
|
|
.m_symbols
|
|
.emit(Box::new(SharedWriter(second.clone())))
|
|
.expect("second");
|
|
assert_eq!(
|
|
first.lock().expect("first output").as_slice(),
|
|
second.lock().expect("second output").as_slice()
|
|
);
|
|
}
|
|
|
|
struct SharedWriter(Arc<Mutex<Vec<u8>>>);
|
|
|
|
impl std::io::Write for SharedWriter {
|
|
fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {
|
|
self.0
|
|
.lock()
|
|
.map_err(|_| std::io::Error::other("poisoned output"))?
|
|
.extend_from_slice(bytes);
|
|
Ok(bytes.len())
|
|
}
|
|
|
|
fn flush(&mut self) -> std::io::Result<()> {
|
|
Ok(())
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn malformed_corpus_terminates_without_panics_or_unbounded_growth() {
|
|
for source in ["+", "a +", "((a", "a * * b", "-", "a )", "@"] {
|
|
let mut parser = expression_parser(source);
|
|
let _ = parser.parse_with_string(source.to_owned());
|
|
assert!(parser.diagnostics().expect("diagnostics").len() <= 2);
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn mapped_symbol_set_production_precedence_and_entry_apis_are_live() {
|
|
let terminal = CSymbol::native("PLUS", PLUS, CSymbolSymType::Terminal).expect("terminal");
|
|
let nonterminal =
|
|
CSymbol::native("Expression", 100, CSymbolSymType::Nonterminal).expect("nonterminal");
|
|
let mut set = SymbolSet::native();
|
|
assert!(set.check_in(terminal.clone()).expect("insert"));
|
|
assert!(!set.check_in(terminal.clone()).expect("deduplicate"));
|
|
assert!(set.contains(terminal.clone()).expect("contains"));
|
|
assert!(terminal.matches("PLUS".to_owned()).expect("symbol text"));
|
|
|
|
let mut followed = nonterminal.clone();
|
|
assert!(
|
|
followed
|
|
.add_follow(terminal.m_first.clone())
|
|
.expect("first follow insertion")
|
|
);
|
|
assert_eq!(nonterminal.m_follow.count(), 1);
|
|
|
|
let nullable =
|
|
CSymbol::native("Optional", 101, CSymbolSymType::Nonterminal).expect("nullable symbol");
|
|
let _empty = Production::native(8, nullable.clone(), vec![]).expect("empty production");
|
|
assert!(nullable.is_nullable().expect("registered empty production"));
|
|
|
|
let mut production = Production::native(7, nonterminal.clone(), vec![]).expect("production");
|
|
production.add_to_rhs(nonterminal.clone()).expect("lhs");
|
|
production.add_to_rhs(terminal.clone()).expect("operator");
|
|
assert_eq!(production.prefix(1).expect("prefix")[0].m_yynum, 100);
|
|
let target = CSymbol::native("Target", 102, CSymbolSymType::Nonterminal).expect("target");
|
|
production
|
|
.add_first(target.clone(), 1)
|
|
.expect("FIRST/FOLLOW propagation");
|
|
assert!(
|
|
target
|
|
.m_follow
|
|
.contains(terminal.clone())
|
|
.expect("shared follow mutation")
|
|
);
|
|
|
|
let precedence = Precedence::first(PrecedencePrecType::Left, 5).expect("precedence");
|
|
assert_eq!(
|
|
Precedence::check_with_precedence_prec_type_int32(precedence, PrecedencePrecType::Left, 0,),
|
|
Ok(5)
|
|
);
|
|
|
|
let parser = expression_parser("a");
|
|
let mut lexer = parser.m_lexer.clone();
|
|
let symbol = lexer.next_token().expect("lexing").expect("token");
|
|
let mut entry = ParserEntry::default();
|
|
assert_eq!(
|
|
symbol.pass_(parser.m_symbols.clone(), 0, &mut entry),
|
|
Ok(true)
|
|
);
|
|
assert_eq!(entry.str(), "shift 1");
|
|
|
|
let lsl_error = LslError {
|
|
state: 4,
|
|
sym: libremetaverse_lsl_tools::SYMBOL::new_with_lexer(parser.m_lexer.clone())
|
|
.expect("symbol"),
|
|
};
|
|
assert!(lsl_error.to_string().contains("state 4"));
|
|
}
|