Match Qwen Unicode number token boundaries

src/engine/metal/qwen_mtplx/issue103_analysis.rs

@@ -332,7 +332,10 @@ fn issue103_tokenizer_receipts() {
         "{}",
         serde_json::json!({"case":"tokenizer_summary","mismatches":mismatches})
     );
-    assert_eq!(mismatches, 5, "diagnosis changed; inspect exact token IDs");
+    assert_eq!(
+        mismatches, 0,
+        "Qwen IDs must match the trained pretokenizer"
+    );
 }
 
 #[test]

src/engine/metal/qwen_mtplx/issue111_product.rs

@@ -0,0 +1,126 @@
+//! Token-exact cache compatibility across the Qwen split correction.
+use super::*;
+use crate::settings::{
+    GenerationPreferences, ReasoningMode, RuntimePreferences, SpeculativePreferences,
+    effective_settings,
+};
+use std::path::PathBuf;
+
+#[test]
+#[ignore = "installed model; serial external memory supervisor required"]
+fn issue111_token_prefix_cache() {
+    let root = PathBuf::from(std::env::var_os("DS4_QWEN38_ARTIFACTS").unwrap());
+    let oracle = PathBuf::from(std::env::var_os("DS4_ISSUE111_ORACLE").unwrap());
+    let directory = PathBuf::from(std::env::var_os("DS4_ISSUE111_DIR").unwrap());
+    std::fs::create_dir_all(&directory).unwrap();
+    let requests: serde_json::Value =
+        serde_json::from_slice(&std::fs::read(oracle.join("requests.json")).unwrap()).unwrap();
+    let legacy: Vec<Vec<i32>> =
+        serde_json::from_slice(&std::fs::read(oracle.join("legacy-requests.json")).unwrap())
+            .unwrap();
+    let case = &requests[1];
+    let ids: Vec<i32> = serde_json::from_value(case["ids"].clone()).unwrap();
+    let settings = effective_settings(
+        crate::model::ModelChoice::Qwen38FlashNext,
+        &GenerationPreferences {
+            context_tokens: 8192,
+            max_generated_tokens: 2,
+            system_prompt: String::new(),
+            temperature: Some(0.),
+            top_p: Some(0.95),
+            min_p: Some(0.),
+            seed: Some(1),
+            reasoning_mode: ReasoningMode::Low,
+        },
+        &RuntimePreferences {
+            speculative: SpeculativePreferences {
+                glm_mtp: true,
+                ..Default::default()
+            },
+            ..Default::default()
+        },
+        root.parent().unwrap(),
+    )
+    .unwrap();
+    let messages = case["messages"]
+        .as_array()
+        .unwrap()
+        .iter()
+        .map(|m| ChatTurn {
+            user: m["role"] == "user",
+            tool: false,
+            system: false,
+            skip_previous_eos: false,
+            reasoning: m["reasoning_content"].as_str().map(str::to_owned),
+            reasoning_complete: true,
+            content: m["content"].as_str().unwrap().into(),
+        })
+        .collect::<Vec<_>>();
+    let mut owner = QwenExecutor::open(&settings.engine).unwrap();
+    owner.policy.store_min_suffix = Some(0);
+    assert_eq!(
+        owner
+            .metadata
+            .tokenizer
+            .encode_conversation("", &messages, ReasoningMode::Low),
+        ids
+    );
+    assert_ne!(legacy[1], ids);
+    owner.prefill(&legacy[1], |_| true).unwrap();
+    owner
+        .save_checkpoint(&directory.join("old-prefix.bin"), [111; 32], &mut |_| {})
+        .unwrap();
+    owner.reset().unwrap();
+    assert!(
+        owner
+            .load_checkpoint(&directory.join("old-prefix.bin"), &mut |_| {})
+            .unwrap()
+    );
+    assert_eq!(owner.tokens(), legacy[1]);
+    assert_eq!(owner.align_prompt(&ids).unwrap(), 0);
+    assert!(owner.live.current.is_none());
+    let metrics = Metrics::new(&directory.join("metrics"));
+    let run = |owner: &mut QwenExecutor, messages: &[ChatTurn]| {
+        let (out, complete) = owner
+            .generate(
+                messages,
+                &settings.turn,
+                &AtomicBool::new(false),
+                &metrics,
+                |_, _, _| {},
+                |_, _| {},
+            )
+            .unwrap();
+        assert!(complete);
+        out
+    };
+    let cold = run(&mut owner, &messages);
+    assert_eq!(cold.cached_tokens, 0);
+    assert_eq!(cold.prompt_tokens as usize, ids.len());
+    assert!(owner.tokens().starts_with(&ids));
+    owner
+        .save_checkpoint(&directory.join("new-prefix.bin"), [111; 32], &mut |_| {})
+        .unwrap();
+    let mut next = messages.clone();
+    next.push(cold.message.clone());
+    next.push(ChatTurn {
+        content: "Continue with Ⅷ and ².".into(),
+        ..messages[0].clone()
+    });
+    let warm = run(&mut owner, &next);
+    assert!(warm.cached_tokens >= cold.prompt_tokens);
+    let expected = super::issue107_product::state(&owner);
+    owner.reset().unwrap();
+    assert!(
+        owner
+            .load_checkpoint(&directory.join("new-prefix.bin"), &mut |_| {})
+            .unwrap()
+    );
+    let reopened = run(&mut owner, &next);
+    assert_eq!(warm.cached_tokens, reopened.cached_tokens);
+    assert_eq!(expected, super::issue107_product::state(&owner));
+    println!(
+        "{}",
+        serde_json::json!({"case":"token_cache", "old_tokens":legacy[1].len(),"new_tokens":ids.len(),"old_prefix_miss":true,"warm_cached":warm.cached_tokens,"disk_cached":reopened.cached_tokens,"warm_disk_state_exact":true})
+    );
+}

src/engine/metal/qwen_mtplx/owner.rs

@@ -1076,3 +1076,7 @@ mod issue107_product;
 #[cfg(test)]
 #[path = "issue109_product.rs"]
 mod issue109_product;
+
+#[cfg(test)]
+#[path = "issue111_product.rs"]
+mod issue111_product;

src/engine/tokenizer.rs

@@ -326,7 +326,7 @@ impl Tokenizer {
         match self.family {
             ModelFamily::DeepSeek => self.tokenize_joyai(text, &mut output),
             ModelFamily::Glm => self.tokenize_gpt4(text, &mut output, 3),
-            ModelFamily::Qwen => self.tokenize_gpt4(text, &mut output, 1),
+            ModelFamily::Qwen => self.tokenize_qwen(text, &mut output),
         }
         output
     }
@@ -374,7 +374,7 @@ impl Tokenizer {
         match self.family {
             ModelFamily::DeepSeek => self.tokenize_joyai(text, output),
             ModelFamily::Glm => self.tokenize_gpt4(text, output, 3),
-            ModelFamily::Qwen => self.tokenize_gpt4(text, output, 1),
+            ModelFamily::Qwen => self.tokenize_qwen(text, output),
         }
     }
 
@@ -771,6 +771,12 @@ impl Tokenizer {
         }
     }
 
+    fn tokenize_qwen(&self, text: &str, output: &mut Vec<i32>) {
+        for piece in qwen_pieces(text) {
+            self.emit_piece(piece.as_bytes(), output);
+        }
+    }
+
     fn tokenize_gpt4(&self, text: &str, output: &mut Vec<i32>, max_digits: usize) {
         let mut position = 0;
         while position < text.len() {
@@ -861,6 +867,39 @@ impl Tokenizer {
     }
 }
 
+/// The trained Qwen3 Split expression. Rust regex supports its Unicode
+/// categories but not the penultimate `\s+(?!\S)` alternative: before a
+/// non-whitespace character that branch backtracks by one whitespace scalar,
+/// leaving it for the next word/punctuation match. At EOF it keeps the run.
+fn qwen_pieces(text: &str) -> impl Iterator<Item = &str> {
+    static SPLIT: std::sync::LazyLock<regex::Regex> = std::sync::LazyLock::new(|| {
+        regex::Regex::new(
+            r"(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?[\p{L}\p{M}]+|\p{N}| ?[^\s\p{L}\p{M}\p{N}]+[\r\n]*|\s*[\r\n]+|(?P<whitespace>\s+)",
+        ).expect("valid Qwen3 pretokenizer expression")
+    });
+    let mut position = 0;
+    std::iter::from_fn(move || {
+        if position == text.len() {
+            return None;
+        }
+        let captures = SPLIT
+            .captures_at(text, position)
+            .expect("Qwen3 covers every Unicode scalar");
+        let matched = captures.get(0).unwrap();
+        debug_assert_eq!(matched.start(), position);
+        let mut end = matched.end();
+        if captures.name("whitespace").is_some() && end < text.len() {
+            let last = matched.as_str().char_indices().next_back().unwrap().0;
+            if last != 0 {
+                end = position + last;
+            }
+        }
+        let piece = &text[position..end];
+        position = end;
+        Some(piece)
+    })
+}
+
 fn byte_encode(bytes: &[u8]) -> String {
     bytes
         .iter()
@@ -1343,3 +1382,7 @@ mod tests {
         );
     }
 }
+
+#[cfg(test)]
+#[path = "tokenizer_issue111.rs"]
+mod issue111;

src/engine/tokenizer_issue111.rs

@@ -0,0 +1,154 @@
+use super::*;
+use sha2::{Digest, Sha256};
+
+fn oracle() -> std::path::PathBuf {
+    std::path::PathBuf::from(std::env::var_os("DS4_ISSUE111_ORACLE").unwrap())
+}
+
+#[test]
+#[ignore = "CPU-only; installed DS4 GGUFs and exact oracle IDs"]
+fn issue111_ds4_families() {
+    let expected: serde_json::Value =
+        serde_json::from_slice(&fs::read(oracle().join("ds4.json")).unwrap()).unwrap();
+    for family in expected["families"].as_array().unwrap() {
+        let choice = if family["family"] == "glm" {
+            crate::model::ModelChoice::Glm53Flash
+        } else {
+            crate::model::ModelChoice::DeepSeekV4Flash0731
+        };
+        let model =
+            super::super::Model::open_main(Path::new(family["model"].as_str().unwrap()), choice)
+                .unwrap();
+        for case in family["cases"].as_array().unwrap() {
+            let text = case["text"].as_str().unwrap();
+            for ids in [
+                model.tokenizer.tokenize(text),
+                model.tokenizer.tokenize_rendered(text),
+            ] {
+                assert_eq!(
+                    serde_json::json!(ids),
+                    case["ids"],
+                    "{} {text:?}",
+                    family["family"]
+                );
+            }
+        }
+        println!(
+            "{}",
+            serde_json::json!({"case":"ds4_control", "family":family["family"], "cases":family["cases"].as_array().unwrap().len(), "plain_rendered_exact":true})
+        );
+    }
+}
+
+#[test]
+fn qwen_split_boundaries() {
+    for (text, expected) in [
+        (" ① Ⅷ ² 𝟘", vec![" ", "①", " ", "Ⅷ", " ", "²", " ", "𝟘"]),
+        ("a\u{301}  \t b   ", vec!["a\u{301}", "  \t", " b", "   "]),
+        (
+            "I'm WE'RE\r\n  x",
+            vec!["I", "'m", " WE", "'RE", "\r\n", " ", " x"],
+        ),
+        ("́ १́b", vec!["́", " ", "१", "́b"]),
+    ] {
+        assert_eq!(qwen_pieces(text).collect::<Vec<_>>(), expected, "{text:?}");
+    }
+}
+
+#[test]
+#[ignore = "CPU-only; exhaustive pinned Qwen split receipt"]
+fn issue111_every_unicode_scalar_split() {
+    let expected: serde_json::Value =
+        serde_json::from_slice(&fs::read(oracle().join("scalar-split.json")).unwrap()).unwrap();
+    let mut text = String::new();
+    for c in (0..=0x10ffff).filter_map(char::from_u32) {
+        text.push('A');
+        text.push(c);
+        text.push_str("A ");
+    }
+    let mut hash = Sha256::new();
+    let mut count = 0;
+    for piece in qwen_pieces(&text) {
+        hash.update((piece.len() as u32).to_le_bytes());
+        hash.update(piece.as_bytes());
+        count += 1;
+    }
+    let hash = hash
+        .finalize()
+        .iter()
+        .map(|b| format!("{b:02x}"))
+        .collect::<String>();
+    assert_eq!(count, expected["pieces"].as_u64().unwrap());
+    assert_eq!(hash, expected["sha256"].as_str().unwrap());
+    println!(
+        "{}",
+        serde_json::json!({"case":"unicode_split","scalars":expected["scalars"],"pieces":count,"sha256":hash,"exact":true})
+    );
+}
+
+#[test]
+#[ignore = "CPU-only; installed Qwen tokenizer and pinned exact IDs"]
+fn issue111_qwen_tokens_and_requests() {
+    let root = std::path::PathBuf::from(std::env::var_os("DS4_QWEN38_ARTIFACTS").unwrap());
+    let tokenizer = Tokenizer::load_qwen(&root.join("tokenizer.json")).unwrap();
+    let expected: serde_json::Value =
+        serde_json::from_slice(&fs::read(oracle().join("qwen.json")).unwrap()).unwrap();
+    for case in expected["cases"].as_array().unwrap() {
+        let text = case["text"].as_str().unwrap();
+        assert_eq!(
+            serde_json::json!(qwen_pieces(text).collect::<Vec<_>>()),
+            case["pieces"],
+            "split {text:?}"
+        );
+        for ids in [tokenizer.tokenize(text), tokenizer.tokenize_rendered(text)] {
+            assert_eq!(serde_json::json!(ids), case["ids"], "{text:?}");
+            let decoded = ids
+                .iter()
+                .flat_map(|&id| tokenizer.token_bytes(id).unwrap())
+                .collect::<Vec<_>>();
+            assert_eq!(decoded, text.as_bytes());
+        }
+    }
+    let requests: serde_json::Value =
+        serde_json::from_slice(&fs::read(oracle().join("requests.json")).unwrap()).unwrap();
+    for case in requests.as_array().unwrap() {
+        let effort = match case["effort"].as_str().unwrap() {
+            "direct" => ReasoningMode::Direct,
+            "low" => ReasoningMode::Low,
+            "medium" => ReasoningMode::Medium,
+            "xhigh" => ReasoningMode::XHigh,
+            _ => unreachable!(),
+        };
+        let messages = case["messages"]
+            .as_array()
+            .unwrap()
+            .iter()
+            .map(|m| ChatTurn {
+                user: m["role"] == "user",
+                tool: false,
+                system: false,
+                skip_previous_eos: false,
+                reasoning: m["reasoning_content"].as_str().map(str::to_owned),
+                reasoning_complete: true,
+                content: m["content"].as_str().unwrap().to_owned(),
+            })
+            .collect::<Vec<_>>();
+        let ids = tokenizer.encode_conversation("", &messages, effort);
+        assert_eq!(serde_json::json!(ids), case["ids"], "{}", case["effort"]);
+        assert_eq!(
+            serde_json::json!(tokenizer.tokenize_rendered(case["rendered"].as_str().unwrap())),
+            case["ids"]
+        );
+        let mut resumed = tokenizer.encode_history("", &messages[..2], effort);
+        resumed.extend(tokenizer.encode_continuation(&messages[2].content, effort, false));
+        assert_eq!(resumed, ids);
+        println!(
+            "{}",
+            serde_json::json!({"case":"request","effort":case["effort"],"context_tokens":ids.len(),"exact":true})
+        );
+    }
+    println!(
+        "{}",
+        serde_json::json!({"case":"tokenizer","number_count":expected["number_count"],"cases":expected["cases"].as_array().unwrap().len(),"plain_rendered_roundtrip_exact":true})
+    );
+}