Match Qwen Unicode number token boundaries

src/engine/tokenizer.rs

@@ -326,7 +326,7 @@ impl Tokenizer {
         match self.family {
             ModelFamily::DeepSeek => self.tokenize_joyai(text, &mut output),
             ModelFamily::Glm => self.tokenize_gpt4(text, &mut output, 3),
-            ModelFamily::Qwen => self.tokenize_gpt4(text, &mut output, 1),
+            ModelFamily::Qwen => self.tokenize_qwen(text, &mut output),
         }
         output
     }
@@ -374,7 +374,7 @@ impl Tokenizer {
         match self.family {
             ModelFamily::DeepSeek => self.tokenize_joyai(text, output),
             ModelFamily::Glm => self.tokenize_gpt4(text, output, 3),
-            ModelFamily::Qwen => self.tokenize_gpt4(text, output, 1),
+            ModelFamily::Qwen => self.tokenize_qwen(text, output),
         }
     }
 
@@ -771,6 +771,12 @@ impl Tokenizer {
         }
     }
 
+    fn tokenize_qwen(&self, text: &str, output: &mut Vec<i32>) {
+        for piece in qwen_pieces(text) {
+            self.emit_piece(piece.as_bytes(), output);
+        }
+    }
+
     fn tokenize_gpt4(&self, text: &str, output: &mut Vec<i32>, max_digits: usize) {
         let mut position = 0;
         while position < text.len() {
@@ -861,6 +867,39 @@ impl Tokenizer {
     }
 }
 
+/// The trained Qwen3 Split expression. Rust regex supports its Unicode
+/// categories but not the penultimate `\s+(?!\S)` alternative: before a
+/// non-whitespace character that branch backtracks by one whitespace scalar,
+/// leaving it for the next word/punctuation match. At EOF it keeps the run.
+fn qwen_pieces(text: &str) -> impl Iterator<Item = &str> {
+    static SPLIT: std::sync::LazyLock<regex::Regex> = std::sync::LazyLock::new(|| {
+        regex::Regex::new(
+            r"(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?[\p{L}\p{M}]+|\p{N}| ?[^\s\p{L}\p{M}\p{N}]+[\r\n]*|\s*[\r\n]+|(?P<whitespace>\s+)",
+        ).expect("valid Qwen3 pretokenizer expression")
+    });
+    let mut position = 0;
+    std::iter::from_fn(move || {
+        if position == text.len() {
+            return None;
+        }
+        let captures = SPLIT
+            .captures_at(text, position)
+            .expect("Qwen3 covers every Unicode scalar");
+        let matched = captures.get(0).unwrap();
+        debug_assert_eq!(matched.start(), position);
+        let mut end = matched.end();
+        if captures.name("whitespace").is_some() && end < text.len() {
+            let last = matched.as_str().char_indices().next_back().unwrap().0;
+            if last != 0 {
+                end = position + last;
+            }
+        }
+        let piece = &text[position..end];
+        position = end;
+        Some(piece)
+    })
+}
+
 fn byte_encode(bytes: &[u8]) -> String {
     bytes
         .iter()
@@ -1343,3 +1382,7 @@ mod tests {
         );
     }
 }
+
+#[cfg(test)]
+#[path = "tokenizer_issue111.rs"]
+mod issue111;