MetaCrate/crates/libremetaverse-lsl-tools/tests/generator_compat.rs

//! Focused compatibility fixtures for issue #83's checked-in LSL generator.

use std::collections::HashMap;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex};

use libremetaverse_lsl_tools::{
    BASE, CSymbol, CSymbolSymType, Dfa, ErrorHandler, GENERATED_PARSER_DATA, ID,
    LSL_GENERATOR_SERIALIZATION_VERSION, Lexer, Nfa, NfaNode, ObjectList,
    ObjectListOListEnumerator, Path, Regex, SCreator, Serialiser, Sfactory, SymbolType, SymbolsGen,
    TCreator, Tfactory, TokClassDef, TokensGen, generated_lexer, generated_parser,
};
use libremetaverse_types::compat::{Object, Utf16CodeUnit};

struct SharedWriter(Arc<Mutex<Vec<u8>>>);

impl std::io::Write for SharedWriter {
    fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {
        self.0
            .lock()
            .map_err(|_| std::io::Error::other("poisoned output"))?
            .extend_from_slice(bytes);
        Ok(bytes.len())
    }

    fn flush(&mut self) -> std::io::Result<()> {
        Ok(())
    }
}

fn capture<F>(function: F) -> Vec<u8>
where
    F: FnOnce(Box<dyn std::io::Write + Send>),
{
    let output = Arc::new(Mutex::new(Vec::new()));
    function(Box::new(SharedWriter(output.clone())));
    Arc::try_unwrap(output)
        .expect("single output owner")
        .into_inner()
        .expect("output")
}

#[test]
fn generated_lexer_preserves_tokens_reserved_words_and_eof() {
    let mut lexer = Lexer::new(generated_lexer().expect("generated lexer")).expect("runtime");
    lexer
        .start_with_string("Widget(value;):base(new Child[])".to_owned())
        .expect("source");
    let mut tokens = Vec::new();
    while let Some(token) = lexer.next_token().expect("lexing") {
        tokens.push((token.yyname(), token.yytext(), token.yynum()));
    }
    assert_eq!(
        tokens,
        [
            ("ID".to_owned(), "Widget".to_owned(), 6),
            ("LPAREN".to_owned(), "(".to_owned(), 12),
            ("ID".to_owned(), "value".to_owned(), 6),
            ("SEMICOLON".to_owned(), ";".to_owned(), 9),
            ("RPAREN".to_owned(), ")".to_owned(), 13),
            ("COLON".to_owned(), ":".to_owned(), 8),
            ("BASE".to_owned(), "base".to_owned(), 3),
            ("LPAREN".to_owned(), "(".to_owned(), 12),
            ("NEW".to_owned(), "new".to_owned(), 5),
            ("ID".to_owned(), "Child".to_owned(), 6),
            ("LBRACK".to_owned(), "[".to_owned(), 14),
            ("RBRACK".to_owned(), "]".to_owned(), 15),
            ("RPAREN".to_owned(), ")".to_owned(), 13),
            ("EOF".to_owned(), "EOF".to_owned(), 2),
        ]
    );
}

#[test]
fn generated_parser_accepts_the_reviewed_class_body_grammar() {
    let table = generated_parser().expect("generated parser");
    assert_eq!(table.arr, GENERATED_PARSER_DATA);
    let lexer = Lexer::new(generated_lexer().expect("generated lexer")).expect("runtime");
    let mut parser = libremetaverse_lsl_tools::Parser::new(table, lexer).expect("parser");
    let result = parser
        .parse_with_string("Widget(value;):base(new Child[])".to_owned())
        .expect("class body");
    assert_eq!(result.yyname(), "ClassBody");
    assert_eq!(result.yynum(), 19);
}

#[test]
fn parser_and_lexer_emission_is_byte_deterministic() {
    let first_parser = capture(|output| {
        generated_parser()
            .expect("parser")
            .emit(output)
            .expect("emit");
    });
    let second_parser = capture(|output| {
        generated_parser()
            .expect("parser")
            .emit(output)
            .expect("emit");
    });
    assert_eq!(first_parser, second_parser);

    let first_lexer = capture(|output| {
        generated_lexer()
            .expect("lexer")
            .emit_dfa(output)
            .expect("emit");
    });
    let second_lexer = capture(|output| {
        generated_lexer()
            .expect("lexer")
            .emit_dfa(output)
            .expect("emit");
    });
    assert_eq!(first_lexer, second_lexer);
}

#[test]
fn generated_token_classes_have_live_identity_and_text() {
    let mut lexer = Lexer::new(generated_lexer().expect("lexer")).expect("runtime");
    lexer
        .start_with_string("base name".to_owned())
        .expect("source");
    let base = lexer.next_token().expect("lexing").expect("base");
    let mapped_base = BASE::new(lexer.clone()).expect("mapped base");
    assert_eq!((base.yyname(), base.yynum()), ("BASE".to_owned(), 3));
    assert_eq!(
        (mapped_base.yyname(), mapped_base.yynum()),
        ("BASE".to_owned(), 3)
    );
    let _ = lexer.next_token().expect("lexing").expect("name");
    let mapped_id = ID::new(lexer).expect("mapped id");
    assert_eq!(
        (mapped_id.yyname(), mapped_id.yynum()),
        ("ID".to_owned(), 6)
    );
}

#[test]
fn regex_and_nfa_build_a_functional_native_dfa() {
    let handler = ErrorHandler::default();
    let tokens = TokensGen::new(handler.clone()).expect("generator");
    let regex = Regex::new(tokens.clone(), 0, "[A-Z][a-z]+".to_owned()).expect("regex");
    assert!(
        !regex
            .match__with_char(Utf16CodeUnit(u16::from(b'Q')))
            .expect("single-unit mismatch")
    );
    assert!(
        !regex
            .match__with_char(Utf16CodeUnit(u16::from(b'q')))
            .expect("mismatch")
    );
    assert_eq!(
        regex
            .match__with_string("Zebra".to_owned())
            .expect("prefix"),
        5
    );

    let mut nfa = Nfa::new_with_tokens_gen_regex(tokens, regex).expect("nfa");
    nfa.m_end.m_s_terminal = "CAPITAL".to_owned();
    let dfa = Dfa::new_with_nfa(nfa).expect("dfa");
    let mut table = libremetaverse_lsl_tools::YyLexer::new(handler).expect("table");
    table.set_start_dfa("YYINITIAL", dfa).expect("start");
    table.using_eof = true;
    let mut lexer = Lexer::new(table).expect("runtime");
    lexer.start_with_string("Rust".to_owned()).expect("source");
    let token = lexer.next_token().expect("lexing").expect("token");
    assert_eq!(
        (token.yyname(), token.yytext()),
        ("CAPITAL".to_owned(), "Rust".to_owned())
    );

    let tokens = TokensGen::new(ErrorHandler::default()).expect("manual generator");
    let mut manual = Nfa::new_with_tokens_gen(tokens.clone()).expect("manual nfa");
    manual.m_end.m_s_terminal = "PAIR".to_owned();
    let middle = NfaNode::new(tokens).expect("middle state");
    manual
        .start
        .add_arc(Utf16CodeUnit(u16::from(b'a')), middle.clone())
        .expect("first arc");
    middle
        .add_arc(Utf16CodeUnit(u16::from(b'b')), manual.m_end.clone())
        .expect("second arc");
    let dfa = Dfa::new_with_nfa(manual).expect("determinised nfa");
    let mut action = -1;
    assert_eq!(
        dfa.match_("ab".to_owned(), 0, &mut action).expect("pair"),
        2
    );
}

#[test]
fn serialiser_round_trips_deterministically_with_versioning() {
    let value = Object::Map(HashMap::from([
        ("b".to_owned(), Object::Integer(7)),
        (
            "a".to_owned(),
            Object::Array(vec![Object::Boolean(true), Object::String("λ".to_owned())]),
        ),
    ]));
    let encoded = capture(|output| {
        let serialiser = Serialiser::new_with_text_writer(output).expect("writer");
        serialiser.version_check().expect("version");
        serialiser.serialise(value.clone()).expect("serialise");
    });
    let encoded_again = capture(|output| {
        let serialiser = Serialiser::new_with_text_writer(output).expect("writer");
        serialiser.version_check().expect("version");
        serialiser.serialise(value.clone()).expect("serialise");
    });
    assert_eq!(encoded, encoded_again);
    let integers = String::from_utf8(encoded)
        .expect("ASCII integers")
        .split(',')
        .filter(|value| !value.trim().is_empty())
        .map(|value| value.trim().parse::<i32>().expect("integer"))
        .collect::<Vec<_>>();
    let decoder = Serialiser::new_with_int32_array(integers).expect("reader");
    decoder
        .version_check()
        .expect(LSL_GENERATOR_SERIALIZATION_VERSION);
    assert_eq!(decoder.deserialise().expect("deserialise"), value);
}

#[test]
fn compatibility_enumerator_and_generator_state_are_live() {
    let mut values = ObjectList::new().expect("list");
    values.add(Object::Integer(1)).expect("first");
    values
        .add(Object::String("two".to_owned()))
        .expect("second");
    let enumerator = ObjectListOListEnumerator::new(values).expect("enumerator");
    assert_eq!(enumerator.current(), Object::Undefined);
    assert!(enumerator.move_next().expect("first"));
    assert_eq!(enumerator.current(), Object::Integer(1));
    assert!(enumerator.move_next().expect("second"));
    assert_eq!(enumerator.current(), Object::String("two".to_owned()));
    assert!(!enumerator.move_next().expect("end"));
    enumerator.reset().expect("reset");
    assert!(enumerator.move_next().expect("first again"));

    let generator = TokensGen::new(ErrorHandler::default()).expect("generator");
    assert_eq!(generator.new_state().expect("state one"), 1);
    assert_eq!(generator.new_state().expect("state two"), 2);
    assert_eq!(
        generator
            .fix_actions("yybegin(); yyl".to_owned())
            .expect("rewritten action"),
        "yym.yy_begin(); ((tokens)yym)"
    );
    assert!(Dfa::new_with_tokens_gen(generator).is_ok());
}

#[test]
fn symbol_and_token_factories_execute_registered_rust_closures() {
    let parser = libremetaverse_lsl_tools::Parser::new(
        generated_parser().expect("parser table"),
        Lexer::new(generated_lexer().expect("lexer table")).expect("lexer"),
    )
    .expect("parser");
    let symbol_creator = SCreator::from_fn(|_| Ok(Object::String("symbol".to_owned())));
    let registration = Sfactory::new(
        generated_parser().expect("parser table"),
        "GeneratedSymbol".to_owned(),
        symbol_creator,
    )
    .expect("symbol registration");
    assert_eq!(registration.name(), "GeneratedSymbol");
    assert_eq!(
        Sfactory::create("GeneratedSymbol_Derived".to_owned(), parser).expect("symbol factory"),
        Object::String("symbol".to_owned())
    );

    let lexer = Lexer::new(generated_lexer().expect("lexer table")).expect("lexer");
    let token_creator = TCreator::from_fn(|_| Ok(Object::Integer(83)));
    let registration = Tfactory::new(
        generated_lexer().expect("lexer table"),
        "GeneratedToken".to_owned(),
        token_creator,
    )
    .expect("token registration");
    assert_eq!(registration.name(), "GeneratedToken");
    assert_eq!(
        Tfactory::create("GeneratedToken_Derived".to_owned(), lexer).expect("token factory"),
        Object::Integer(83)
    );

    let callback_ran = Arc::new(AtomicBool::new(false));
    let callback_state = callback_ran.clone();
    let async_creator = TCreator::from_fn(|_| Ok(Object::String("async".to_owned())));
    let result = async_creator
        .begin_invoke(
            Lexer::new(generated_lexer().expect("lexer table")).expect("lexer"),
            Box::new(move |_| callback_state.store(true, Ordering::SeqCst)),
            Object::Undefined,
        )
        .expect("begin invoke");
    assert!(callback_ran.load(Ordering::SeqCst));
    assert_eq!(
        async_creator.end_invoke(result).expect("end invoke"),
        Object::String("async".to_owned())
    );
}

#[test]
fn symbols_paths_and_token_class_definitions_keep_native_state() {
    let symbols = SymbolsGen::new(ErrorHandler::default()).expect("symbols generator");
    let declared = SymbolType::new_with_symbols_gen_string_boolean(
        symbols.clone(),
        "SemanticValue".to_owned(),
        true,
    )
    .expect("symbol type");
    assert_eq!(
        declared
            .find("SemanticValue".to_owned())
            .expect("lookup")
            .name,
        "SemanticValue"
    );

    let identifier = CSymbol::native("ID", 6, CSymbolSymType::Terminal).expect("symbol");
    let path = Path::new_with_c_symbol_array(vec![identifier.clone()]).expect("path");
    assert_eq!(path.spelling()[0].m_yynum, 6);
    assert!(!path.valid);
    assert_eq!(path.top().m_state, 0);

    let output = Arc::new(Mutex::new(Vec::new()));
    let generator = libremetaverse_lsl_tools::GenBase::new(
        ErrorHandler::default(),
        Box::new(SharedWriter(output)),
    )
    .expect("generator base");
    let definition = TokClassDef::new(generator, "IDENTIFIER".to_owned(), "TOKEN".to_owned())
        .expect("token class");
    assert_eq!(definition.m_name, "IDENTIFIER");
    assert_eq!(definition.m_ref_token, "TOKEN");
    assert_eq!(definition.m_yynum, 3);
}