//! Focused compatibility fixtures for issue #83's checked-in LSL generator. use std::collections::HashMap; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; use libremetaverse_lsl_tools::{ BASE, CSymbol, CSymbolSymType, Dfa, ErrorHandler, GENERATED_PARSER_DATA, ID, LSL_GENERATOR_SERIALIZATION_VERSION, Lexer, Nfa, NfaNode, ObjectList, ObjectListOListEnumerator, Path, Regex, SCreator, Serialiser, Sfactory, SymbolType, SymbolsGen, TCreator, Tfactory, TokClassDef, TokensGen, generated_lexer, generated_parser, }; use libremetaverse_types::compat::{Object, Utf16CodeUnit}; struct SharedWriter(Arc>>); impl std::io::Write for SharedWriter { fn write(&mut self, bytes: &[u8]) -> std::io::Result { self.0 .lock() .map_err(|_| std::io::Error::other("poisoned output"))? .extend_from_slice(bytes); Ok(bytes.len()) } fn flush(&mut self) -> std::io::Result<()> { Ok(()) } } fn capture(function: F) -> Vec where F: FnOnce(Box), { let output = Arc::new(Mutex::new(Vec::new())); function(Box::new(SharedWriter(output.clone()))); Arc::try_unwrap(output) .expect("single output owner") .into_inner() .expect("output") } #[test] fn generated_lexer_preserves_tokens_reserved_words_and_eof() { let mut lexer = Lexer::new(generated_lexer().expect("generated lexer")).expect("runtime"); lexer .start_with_string("Widget(value;):base(new Child[])".to_owned()) .expect("source"); let mut tokens = Vec::new(); while let Some(token) = lexer.next_token().expect("lexing") { tokens.push((token.yyname(), token.yytext(), token.yynum())); } assert_eq!( tokens, [ ("ID".to_owned(), "Widget".to_owned(), 6), ("LPAREN".to_owned(), "(".to_owned(), 12), ("ID".to_owned(), "value".to_owned(), 6), ("SEMICOLON".to_owned(), ";".to_owned(), 9), ("RPAREN".to_owned(), ")".to_owned(), 13), ("COLON".to_owned(), ":".to_owned(), 8), ("BASE".to_owned(), "base".to_owned(), 3), ("LPAREN".to_owned(), "(".to_owned(), 12), ("NEW".to_owned(), "new".to_owned(), 5), ("ID".to_owned(), "Child".to_owned(), 6), ("LBRACK".to_owned(), "[".to_owned(), 14), ("RBRACK".to_owned(), "]".to_owned(), 15), ("RPAREN".to_owned(), ")".to_owned(), 13), ("EOF".to_owned(), "EOF".to_owned(), 2), ] ); } #[test] fn generated_parser_accepts_the_reviewed_class_body_grammar() { let table = generated_parser().expect("generated parser"); assert_eq!(table.arr, GENERATED_PARSER_DATA); let lexer = Lexer::new(generated_lexer().expect("generated lexer")).expect("runtime"); let mut parser = libremetaverse_lsl_tools::Parser::new(table, lexer).expect("parser"); let result = parser .parse_with_string("Widget(value;):base(new Child[])".to_owned()) .expect("class body"); assert_eq!(result.yyname(), "ClassBody"); assert_eq!(result.yynum(), 19); } #[test] fn parser_and_lexer_emission_is_byte_deterministic() { let first_parser = capture(|output| { generated_parser() .expect("parser") .emit(output) .expect("emit"); }); let second_parser = capture(|output| { generated_parser() .expect("parser") .emit(output) .expect("emit"); }); assert_eq!(first_parser, second_parser); let first_lexer = capture(|output| { generated_lexer() .expect("lexer") .emit_dfa(output) .expect("emit"); }); let second_lexer = capture(|output| { generated_lexer() .expect("lexer") .emit_dfa(output) .expect("emit"); }); assert_eq!(first_lexer, second_lexer); } #[test] fn generated_token_classes_have_live_identity_and_text() { let mut lexer = Lexer::new(generated_lexer().expect("lexer")).expect("runtime"); lexer .start_with_string("base name".to_owned()) .expect("source"); let base = lexer.next_token().expect("lexing").expect("base"); let mapped_base = BASE::new(lexer.clone()).expect("mapped base"); assert_eq!((base.yyname(), base.yynum()), ("BASE".to_owned(), 3)); assert_eq!( (mapped_base.yyname(), mapped_base.yynum()), ("BASE".to_owned(), 3) ); let _ = lexer.next_token().expect("lexing").expect("name"); let mapped_id = ID::new(lexer).expect("mapped id"); assert_eq!( (mapped_id.yyname(), mapped_id.yynum()), ("ID".to_owned(), 6) ); } #[test] fn regex_and_nfa_build_a_functional_native_dfa() { let handler = ErrorHandler::default(); let tokens = TokensGen::new(handler.clone()).expect("generator"); let regex = Regex::new(tokens.clone(), 0, "[A-Z][a-z]+".to_owned()).expect("regex"); assert!( !regex .match__with_char(Utf16CodeUnit(u16::from(b'Q'))) .expect("single-unit mismatch") ); assert!( !regex .match__with_char(Utf16CodeUnit(u16::from(b'q'))) .expect("mismatch") ); assert_eq!( regex .match__with_string("Zebra".to_owned()) .expect("prefix"), 5 ); let mut nfa = Nfa::new_with_tokens_gen_regex(tokens, regex).expect("nfa"); nfa.m_end.m_s_terminal = "CAPITAL".to_owned(); let dfa = Dfa::new_with_nfa(nfa).expect("dfa"); let mut table = libremetaverse_lsl_tools::YyLexer::new(handler).expect("table"); table.set_start_dfa("YYINITIAL", dfa).expect("start"); table.using_eof = true; let mut lexer = Lexer::new(table).expect("runtime"); lexer.start_with_string("Rust".to_owned()).expect("source"); let token = lexer.next_token().expect("lexing").expect("token"); assert_eq!( (token.yyname(), token.yytext()), ("CAPITAL".to_owned(), "Rust".to_owned()) ); let tokens = TokensGen::new(ErrorHandler::default()).expect("manual generator"); let mut manual = Nfa::new_with_tokens_gen(tokens.clone()).expect("manual nfa"); manual.m_end.m_s_terminal = "PAIR".to_owned(); let middle = NfaNode::new(tokens).expect("middle state"); manual .start .add_arc(Utf16CodeUnit(u16::from(b'a')), middle.clone()) .expect("first arc"); middle .add_arc(Utf16CodeUnit(u16::from(b'b')), manual.m_end.clone()) .expect("second arc"); let dfa = Dfa::new_with_nfa(manual).expect("determinised nfa"); let mut action = -1; assert_eq!( dfa.match_("ab".to_owned(), 0, &mut action).expect("pair"), 2 ); } #[test] fn serialiser_round_trips_deterministically_with_versioning() { let value = Object::Map(HashMap::from([ ("b".to_owned(), Object::Integer(7)), ( "a".to_owned(), Object::Array(vec![Object::Boolean(true), Object::String("λ".to_owned())]), ), ])); let encoded = capture(|output| { let serialiser = Serialiser::new_with_text_writer(output).expect("writer"); serialiser.version_check().expect("version"); serialiser.serialise(value.clone()).expect("serialise"); }); let encoded_again = capture(|output| { let serialiser = Serialiser::new_with_text_writer(output).expect("writer"); serialiser.version_check().expect("version"); serialiser.serialise(value.clone()).expect("serialise"); }); assert_eq!(encoded, encoded_again); let integers = String::from_utf8(encoded) .expect("ASCII integers") .split(',') .filter(|value| !value.trim().is_empty()) .map(|value| value.trim().parse::().expect("integer")) .collect::>(); let decoder = Serialiser::new_with_int32_array(integers).expect("reader"); decoder .version_check() .expect(LSL_GENERATOR_SERIALIZATION_VERSION); assert_eq!(decoder.deserialise().expect("deserialise"), value); } #[test] fn compatibility_enumerator_and_generator_state_are_live() { let mut values = ObjectList::new().expect("list"); values.add(Object::Integer(1)).expect("first"); values .add(Object::String("two".to_owned())) .expect("second"); let enumerator = ObjectListOListEnumerator::new(values).expect("enumerator"); assert_eq!(enumerator.current(), Object::Undefined); assert!(enumerator.move_next().expect("first")); assert_eq!(enumerator.current(), Object::Integer(1)); assert!(enumerator.move_next().expect("second")); assert_eq!(enumerator.current(), Object::String("two".to_owned())); assert!(!enumerator.move_next().expect("end")); enumerator.reset().expect("reset"); assert!(enumerator.move_next().expect("first again")); let generator = TokensGen::new(ErrorHandler::default()).expect("generator"); assert_eq!(generator.new_state().expect("state one"), 1); assert_eq!(generator.new_state().expect("state two"), 2); assert_eq!( generator .fix_actions("yybegin(); yyl".to_owned()) .expect("rewritten action"), "yym.yy_begin(); ((tokens)yym)" ); assert!(Dfa::new_with_tokens_gen(generator).is_ok()); } #[test] fn symbol_and_token_factories_execute_registered_rust_closures() { let parser = libremetaverse_lsl_tools::Parser::new( generated_parser().expect("parser table"), Lexer::new(generated_lexer().expect("lexer table")).expect("lexer"), ) .expect("parser"); let symbol_creator = SCreator::from_fn(|_| Ok(Object::String("symbol".to_owned()))); let registration = Sfactory::new( generated_parser().expect("parser table"), "GeneratedSymbol".to_owned(), symbol_creator, ) .expect("symbol registration"); assert_eq!(registration.name(), "GeneratedSymbol"); assert_eq!( Sfactory::create("GeneratedSymbol_Derived".to_owned(), parser).expect("symbol factory"), Object::String("symbol".to_owned()) ); let lexer = Lexer::new(generated_lexer().expect("lexer table")).expect("lexer"); let token_creator = TCreator::from_fn(|_| Ok(Object::Integer(83))); let registration = Tfactory::new( generated_lexer().expect("lexer table"), "GeneratedToken".to_owned(), token_creator, ) .expect("token registration"); assert_eq!(registration.name(), "GeneratedToken"); assert_eq!( Tfactory::create("GeneratedToken_Derived".to_owned(), lexer).expect("token factory"), Object::Integer(83) ); let callback_ran = Arc::new(AtomicBool::new(false)); let callback_state = callback_ran.clone(); let async_creator = TCreator::from_fn(|_| Ok(Object::String("async".to_owned()))); let result = async_creator .begin_invoke( Lexer::new(generated_lexer().expect("lexer table")).expect("lexer"), Box::new(move |_| callback_state.store(true, Ordering::SeqCst)), Object::Undefined, ) .expect("begin invoke"); assert!(callback_ran.load(Ordering::SeqCst)); assert_eq!( async_creator.end_invoke(result).expect("end invoke"), Object::String("async".to_owned()) ); } #[test] fn symbols_paths_and_token_class_definitions_keep_native_state() { let symbols = SymbolsGen::new(ErrorHandler::default()).expect("symbols generator"); let declared = SymbolType::new_with_symbols_gen_string_boolean( symbols.clone(), "SemanticValue".to_owned(), true, ) .expect("symbol type"); assert_eq!( declared .find("SemanticValue".to_owned()) .expect("lookup") .name, "SemanticValue" ); let identifier = CSymbol::native("ID", 6, CSymbolSymType::Terminal).expect("symbol"); let path = Path::new_with_c_symbol_array(vec![identifier.clone()]).expect("path"); assert_eq!(path.spelling()[0].m_yynum, 6); assert!(!path.valid); assert_eq!(path.top().m_state, 0); let output = Arc::new(Mutex::new(Vec::new())); let generator = libremetaverse_lsl_tools::GenBase::new( ErrorHandler::default(), Box::new(SharedWriter(output)), ) .expect("generator base"); let definition = TokClassDef::new(generator, "IDENTIFIER".to_owned(), "TOKEN".to_owned()) .expect("token class"); assert_eq!(definition.m_name, "IDENTIFIER"); assert_eq!(definition.m_ref_token, "TOKEN"); assert_eq!(definition.m_yynum, 3); }