Apply Qwen NFC normalization before token splitting

src/engine/tokenizer.rs

@@ -8,6 +8,7 @@ use serde::Deserialize;
 use std::collections::HashMap;
 use std::fs;
 use std::path::Path;
+use unicode_normalization_alignments::UnicodeNormalization;
 
 const HIGH_REASONING_PREFIX: &str = "Reasoning Effort: Absolute maximum with no shortcuts permitted.\n\
 You MUST be very thorough in your thinking and comprehensively decompose the problem to resolve the root cause, rigorously stress-testing your logic against all potential paths, edge cases, and adversarial scenarios.\n\
@@ -54,10 +55,18 @@ pub(super) struct Tokenizer {
 
 #[derive(Deserialize)]
 struct QwenTokenizerFile {
+    normalizer: QwenNormalizer,
     added_tokens: Vec<QwenAddedToken>,
     model: QwenBpe,
 }
 
+#[derive(Deserialize)]
+#[serde(tag = "type")]
+enum QwenNormalizer {
+    #[serde(rename = "NFC")]
+    Nfc,
+}
+
 #[derive(Deserialize)]
 struct QwenAddedToken {
     id: usize,
@@ -190,6 +199,7 @@ impl Tokenizer {
             &fs::read(path).map_err(|error| format!("{}: {error}", path.display()))?,
         )
         .map_err(|error| format!("{}: {error}", path.display()))?;
+        let QwenNormalizer::Nfc = file.normalizer;
         if file.model.kind != "BPE" || file.model.byte_fallback {
             return Err("Qwen tokenizer must use BPE without byte fallback".into());
         }
@@ -772,7 +782,10 @@ impl Tokenizer {
     }
 
     fn tokenize_qwen(&self, text: &str, output: &mut Vec<i32>) {
-        for piece in qwen_pieces(text) {
+        // Added tokens are extracted first by tokenize_rendered. Normalize each
+        // remaining span before the trained split and BPE, as in tokenizers.
+        let normalized = qwen_normalize(text);
+        for piece in qwen_pieces(&normalized) {
             self.emit_piece(piece.as_bytes(), output);
         }
     }
@@ -867,6 +880,21 @@ impl Tokenizer {
     }
 }
 
+fn qwen_normalize(text: &str) -> std::borrow::Cow<'_, str> {
+    if text.is_ascii() {
+        text.into()
+    } else {
+        text.nfc()
+            .map(|(character, _)| character)
+            .collect::<String>()
+            .into()
+    }
+}
+
+pub(super) fn qwen_normalization_changes(text: &str) -> bool {
+    qwen_normalize(text) != text
+}
+
 /// The trained Qwen3 Split expression. Rust regex supports its Unicode
 /// categories but not the penultimate `\s+(?!\S)` alternative: before a
 /// non-whitespace character that branch backtracks by one whitespace scalar,
@@ -1214,7 +1242,7 @@ mod tests {
                 let rendered = render_text_prompt(
                     ModelRef::Gguf(&model),
                     &bootstrap,
-                    conversation_tag("", settings.reasoning_mode, &[]),
+                    conversation_tag(model.shape.family, "", settings.reasoning_mode, &[]),
                     &messages,
                     &settings,
                 );
@@ -1386,3 +1414,7 @@ mod tests {
 #[cfg(test)]
 #[path = "tokenizer_issue111.rs"]
 mod issue111;
+
+#[cfg(test)]
+#[path = "tokenizer_issue116.rs"]
+mod issue116;