Align DeepSeek and GLM execution with DS4

src/engine/tokenizer.rs

@@ -1092,6 +1092,108 @@ fn unicode_punctuation(cp: u32) -> bool {
 mod tests {
     use super::*;
 
+    #[test]
+    #[ignore = "CPU-only; requires DS4SERVER_CHAT_REFERENCE and its installed GGUF"]
+    fn ds4_chat_matches_original_session_tokens() {
+        use super::super::{Model, ModelRef, conversation_tag, render_text_prompt};
+        use crate::model::ModelChoice;
+        let reference = std::env::var_os("DS4SERVER_CHAT_REFERENCE").unwrap();
+        let fixture: serde_json::Value =
+            serde_json::from_slice(&fs::read(reference).unwrap()).unwrap();
+        let choice = match fixture["family"].as_str().unwrap() {
+            "deepseek" => ModelChoice::DeepSeekV4Flash0731,
+            "glm" => ModelChoice::Glm53Flash,
+            other => panic!("unsupported reference family: {other}"),
+        };
+        let model =
+            Model::open_main(Path::new(fixture["model"].as_str().unwrap()), choice).unwrap();
+        let tokenizer = &model.tokenizer;
+        let cases = fixture["cases"].as_array().unwrap();
+        assert_eq!(cases.len(), 3);
+        let mut previous = Vec::new();
+        for (index, case) in cases.iter().enumerate() {
+            let prompt = case["prompt"].as_str().unwrap();
+            let expected: Vec<i32> = serde_json::from_value(case["tokens"].clone()).unwrap();
+            let actual = if index == 0 {
+                tokenizer.encode_chat("", prompt, ReasoningMode::Low)
+            } else {
+                let mut tokens = previous;
+                tokens.extend(tokenizer.encode_continuation(prompt, ReasoningMode::Low, false));
+                tokens
+            };
+            if actual != expected {
+                let first = actual
+                    .iter()
+                    .zip(&expected)
+                    .position(|(a, b)| a != b)
+                    .unwrap_or(actual.len().min(expected.len()));
+                let context = |tokens: &[i32]| {
+                    tokens[first.saturating_sub(4)..tokens.len().min(first + 8)]
+                        .iter()
+                        .map(|&id| {
+                            (
+                                id,
+                                String::from_utf8_lossy(&tokenizer.token_bytes(id).unwrap())
+                                    .into_owned(),
+                            )
+                        })
+                        .collect::<Vec<_>>()
+                };
+                panic!(
+                    "turn {} first mismatch {first}, lengths {}/{}; Rust {:?}; DS4 {:?}",
+                    index + 1,
+                    actual.len(),
+                    expected.len(),
+                    context(&actual),
+                    context(&expected)
+                );
+            }
+            if index == 0 {
+                let settings = crate::settings::TurnSettings {
+                    kv_cache: crate::settings::KvCachePreferences::default().settings(),
+                    context_tokens: 32768,
+                    max_generated_tokens: i32::MAX,
+                    system_prompt: String::new(),
+                    temperature: 0.6,
+                    top_p: 0.95,
+                    min_p: 0.0,
+                    top_k: 0,
+                    stops: vec![],
+                    seed: Some(42),
+                    reasoning_mode: ReasoningMode::Low,
+                };
+                let messages = [ChatTurn {
+                    user: true,
+                    tool: false,
+                    system: false,
+                    skip_previous_eos: false,
+                    reasoning: None,
+                    reasoning_complete: true,
+                    content: prompt.to_owned(),
+                }];
+                let bootstrap = model.render_history("", &[], settings.reasoning_mode);
+                let rendered = render_text_prompt(
+                    ModelRef::Gguf(&model),
+                    &bootstrap,
+                    conversation_tag("", settings.reasoning_mode, &[]),
+                    &messages,
+                    &settings,
+                );
+                assert!(
+                    rendered == expected,
+                    "shared bootstrap renderer differs: {}/{} tokens; starts {:?}/{:?}",
+                    rendered.len(),
+                    expected.len(),
+                    &rendered[..4],
+                    &expected[..4]
+                );
+            }
+            previous = expected;
+            let reply: Vec<i32> = serde_json::from_value(case["reply_tokens"].clone()).unwrap();
+            previous.extend(reply);
+        }
+    }
+
     #[test]
     #[ignore = "requires local tokenizer and tools/qwen-chat-reference.py output; no inference"]
     fn qwen_plain_chat_matches_mtplx_server_tokens() {