Generate native LSL parser tables (#83)

crates/libremetaverse-lsl-tools/src/lexer.rs

@@ -1483,6 +1483,21 @@ impl DfaAction {
 }
 
 /// Runtime deterministic finite automaton.
+#[derive(Clone, Debug)]
+struct RegexDfa {
+    source: String,
+    compiled: regex::Regex,
+    accept: DfaAccept,
+}
+
+impl PartialEq for RegexDfa {
+    fn eq(&self, other: &Self) -> bool {
+        self.source == other.source && self.accept == other.accept
+    }
+}
+
+impl Eq for RegexDfa {}
+
 #[derive(Clone, Debug, Eq, PartialEq)]
 pub struct Dfa {
     pub m_actions: Vec<DfaAction>,
@@ -1491,6 +1506,7 @@ pub struct Dfa {
     pub m_tok_class: String,
     states: Vec<DfaState>,
     start: usize,
+    regex: Option<RegexDfa>,
 }
 
 impl Default for Dfa {
@@ -1502,6 +1518,7 @@ impl Default for Dfa {
             m_tok_class: String::new(),
             states: vec![DfaState::default()],
             start: 0,
+            regex: None,
         }
     }
 }
@@ -1525,17 +1542,54 @@ impl Dfa {
         })
     }
 
-    pub fn new_with_nfa(_nfa: Nfa) -> Result<Self, Error> {
-        // The NFA-to-DFA generator is issue #83; the runtime never manufactures
-        // fake transitions from an opaque generated NFA.
-        Err(Error::InvalidOperation)
+    pub fn new_with_nfa(nfa: Nfa) -> Result<Self, Error> {
+        let name = if nfa.m_end.m_s_terminal.is_empty() {
+            "ANY".to_owned()
+        } else {
+            nfa.m_end.m_s_terminal.clone()
+        };
+        let accept = DfaAccept {
+            token: TokenDefinition::new(name, 7)?,
+            action: LexerAction::Emit,
+            action_number: 0,
+            reserved_words: None,
+        };
+        if let Some(source) = nfa.regex_source()? {
+            let compiled =
+                regex::Regex::new(&format!("^(?:{source})")).map_err(|_| Error::Parse {
+                    position: 0,
+                    context: "invalid NFA regular expression",
+                })?;
+            return Ok(Self {
+                regex: Some(RegexDfa {
+                    source,
+                    compiled,
+                    accept,
+                }),
+                ..Self::default()
+            });
+        }
+        Self::from_states(nfa.deterministic_states(accept)?, 0)
     }
 
     pub fn new_with_tokens_gen(_tokens: TokensGen) -> Result<Self, Error> {
-        Err(Error::InvalidOperation)
+        // The C# constructor creates the initial, unpopulated determinisation
+        // node. NFA-based construction is exposed separately by
+        // `new_with_nfa`; returning an empty native state here preserves that
+        // useful distinction without manufacturing an NFA transition.
+        Ok(Self::default())
     }
 
     fn longest_match(&self, units: &[u16], offset: usize) -> Option<(usize, DfaAccept)> {
+        if let Some(regex) = &self.regex {
+            let text = String::from_utf16(units.get(offset..)?).ok()?;
+            let found = regex.compiled.find(&text)?;
+            if found.start() != 0 || found.end() == 0 {
+                return None;
+            }
+            let length = text[..found.end()].encode_utf16().count();
+            return Some((length, regex.accept.clone()));
+        }
         let mut state_index = self.start;
         let mut cursor = offset;
         let mut accepted = self.states[state_index]
@@ -1587,6 +1641,15 @@ impl Dfa {
 
     #[must_use]
     pub fn stable_description(&self) -> String {
+        if let Some(regex) = &self.regex {
+            return format!(
+                "regex {:?} accept {} {} {}",
+                regex.source,
+                regex.accept.token.number,
+                regex.accept.token.name,
+                regex.accept.action_number
+            );
+        }
         let mut lines = Vec::new();
         for (state_index, state) in self.states.iter().enumerate() {
             if let Some(accept) = &state.accept {
@@ -2185,6 +2248,9 @@ impl SYMBOL {
     pub fn set_yylval(&mut self, value: Object) {
         self.m_dollar = value;
     }
+    pub(crate) fn raw_text(&self) -> String {
+        self.text.clone()
+    }
     #[must_use]
     pub fn yyname(&self) -> String {
         self.name.clone()
@@ -2215,6 +2281,30 @@ pub struct TOKEN {
 }
 
 impl TOKEN {
+    pub(crate) fn generated_with_lexer(
+        lexer: Lexer,
+        name: &str,
+        number: i32,
+    ) -> Result<Self, Error> {
+        let mut token = Self::new_with_lexer(lexer)?;
+        name.clone_into(&mut token.name);
+        token.number = number;
+        Ok(token)
+    }
+
+    pub(crate) fn generated_with_parser(
+        parser: Parser,
+        name: &str,
+        number: i32,
+        text: String,
+    ) -> Result<Self, Error> {
+        let mut token = Self::new_with_parser(parser)?;
+        name.clone_into(&mut token.name);
+        token.number = number;
+        token.text = text;
+        Ok(token)
+    }
+
     pub fn new_with_lexer(lexer: Lexer) -> Result<Self, Error> {
         let text = lexer.yytext.clone();
         Self::new_with_lexer_string(lexer, text)