Match Qwen Unicode number token boundaries

src/engine/metal/qwen_mtplx/issue111_product.rs

@@ -0,0 +1,126 @@
+//! Token-exact cache compatibility across the Qwen split correction.
+use super::*;
+use crate::settings::{
+    GenerationPreferences, ReasoningMode, RuntimePreferences, SpeculativePreferences,
+    effective_settings,
+};
+use std::path::PathBuf;
+
+#[test]
+#[ignore = "installed model; serial external memory supervisor required"]
+fn issue111_token_prefix_cache() {
+    let root = PathBuf::from(std::env::var_os("DS4_QWEN38_ARTIFACTS").unwrap());
+    let oracle = PathBuf::from(std::env::var_os("DS4_ISSUE111_ORACLE").unwrap());
+    let directory = PathBuf::from(std::env::var_os("DS4_ISSUE111_DIR").unwrap());
+    std::fs::create_dir_all(&directory).unwrap();
+    let requests: serde_json::Value =
+        serde_json::from_slice(&std::fs::read(oracle.join("requests.json")).unwrap()).unwrap();
+    let legacy: Vec<Vec<i32>> =
+        serde_json::from_slice(&std::fs::read(oracle.join("legacy-requests.json")).unwrap())
+            .unwrap();
+    let case = &requests[1];
+    let ids: Vec<i32> = serde_json::from_value(case["ids"].clone()).unwrap();
+    let settings = effective_settings(
+        crate::model::ModelChoice::Qwen38FlashNext,
+        &GenerationPreferences {
+            context_tokens: 8192,
+            max_generated_tokens: 2,
+            system_prompt: String::new(),
+            temperature: Some(0.),
+            top_p: Some(0.95),
+            min_p: Some(0.),
+            seed: Some(1),
+            reasoning_mode: ReasoningMode::Low,
+        },
+        &RuntimePreferences {
+            speculative: SpeculativePreferences {
+                glm_mtp: true,
+                ..Default::default()
+            },
+            ..Default::default()
+        },
+        root.parent().unwrap(),
+    )
+    .unwrap();
+    let messages = case["messages"]
+        .as_array()
+        .unwrap()
+        .iter()
+        .map(|m| ChatTurn {
+            user: m["role"] == "user",
+            tool: false,
+            system: false,
+            skip_previous_eos: false,
+            reasoning: m["reasoning_content"].as_str().map(str::to_owned),
+            reasoning_complete: true,
+            content: m["content"].as_str().unwrap().into(),
+        })
+        .collect::<Vec<_>>();
+    let mut owner = QwenExecutor::open(&settings.engine).unwrap();
+    owner.policy.store_min_suffix = Some(0);
+    assert_eq!(
+        owner
+            .metadata
+            .tokenizer
+            .encode_conversation("", &messages, ReasoningMode::Low),
+        ids
+    );
+    assert_ne!(legacy[1], ids);
+    owner.prefill(&legacy[1], |_| true).unwrap();
+    owner
+        .save_checkpoint(&directory.join("old-prefix.bin"), [111; 32], &mut |_| {})
+        .unwrap();
+    owner.reset().unwrap();
+    assert!(
+        owner
+            .load_checkpoint(&directory.join("old-prefix.bin"), &mut |_| {})
+            .unwrap()
+    );
+    assert_eq!(owner.tokens(), legacy[1]);
+    assert_eq!(owner.align_prompt(&ids).unwrap(), 0);
+    assert!(owner.live.current.is_none());
+    let metrics = Metrics::new(&directory.join("metrics"));
+    let run = |owner: &mut QwenExecutor, messages: &[ChatTurn]| {
+        let (out, complete) = owner
+            .generate(
+                messages,
+                &settings.turn,
+                &AtomicBool::new(false),
+                &metrics,
+                |_, _, _| {},
+                |_, _| {},
+            )
+            .unwrap();
+        assert!(complete);
+        out
+    };
+    let cold = run(&mut owner, &messages);
+    assert_eq!(cold.cached_tokens, 0);
+    assert_eq!(cold.prompt_tokens as usize, ids.len());
+    assert!(owner.tokens().starts_with(&ids));
+    owner
+        .save_checkpoint(&directory.join("new-prefix.bin"), [111; 32], &mut |_| {})
+        .unwrap();
+    let mut next = messages.clone();
+    next.push(cold.message.clone());
+    next.push(ChatTurn {
+        content: "Continue with Ⅷ and ².".into(),
+        ..messages[0].clone()
+    });
+    let warm = run(&mut owner, &next);
+    assert!(warm.cached_tokens >= cold.prompt_tokens);
+    let expected = super::issue107_product::state(&owner);
+    owner.reset().unwrap();
+    assert!(
+        owner
+            .load_checkpoint(&directory.join("new-prefix.bin"), &mut |_| {})
+            .unwrap()
+    );
+    let reopened = run(&mut owner, &next);
+    assert_eq!(warm.cached_tokens, reopened.cached_tokens);
+    assert_eq!(expected, super::issue107_product::state(&owner));
+    println!(
+        "{}",
+        serde_json::json!({"case":"token_cache", "old_tokens":legacy[1].len(),"new_tokens":ids.len(),"old_prefix_miss":true,"warm_cached":warm.cached_tokens,"disk_cached":reopened.cached_tokens,"warm_disk_state_exact":true})
+    );
+}