Save inference parity implementation and evaluation harness

src/engine/tokenizer.rs

@@ -426,7 +426,7 @@ impl Tokenizer {
     ) -> Vec<i32> {
         if self.family == ModelFamily::Qwen {
             return self.encode_qwen_messages(
-                system_prompt,
+                Some(system_prompt),
                 messages,
                 reasoning,
                 continue_assistant,
@@ -523,7 +523,7 @@ impl Tokenizer {
 
     fn encode_qwen_messages(
         &self,
-        system_prompt: &str,
+        system_prompt: Option<&str>,
         messages: &[ChatTurn],
         reasoning: ReasoningMode,
         continue_assistant: bool,
@@ -541,8 +541,10 @@ impl Tokenizer {
             }
         };
         let mut rendered = String::new();
-        let system_prompt = system_prompt.trim();
-        if !instruction.is_empty() || !system_prompt.is_empty() {
+        // A checkpoint continuation already contains the initial system message.
+        if let Some(system_prompt) = system_prompt.map(str::trim)
+            && (!instruction.is_empty() || !system_prompt.is_empty())
+        {
             rendered.push_str("<|im_start|>system\n");
             if !instruction.is_empty() {
                 rendered.push_str(instruction);
@@ -608,7 +610,7 @@ impl Tokenizer {
                 reasoning_complete: true,
                 content: prompt.to_owned(),
             }];
-            return self.encode_qwen_messages("", &messages, reasoning, true);
+            return self.encode_qwen_messages(None, &messages, reasoning, true);
         }
         let mut output = Vec::new();
         if self.family == ModelFamily::DeepSeek && !skip_previous_eos {
@@ -1090,6 +1092,107 @@ fn unicode_punctuation(cp: u32) -> bool {
 mod tests {
     use super::*;
 
+    #[test]
+    #[ignore = "requires local tokenizer and tools/qwen-chat-reference.py output; no inference"]
+    fn qwen_plain_chat_matches_mtplx_server_tokens() {
+        let root = std::env::var_os("DS4SERVER_QWEN38_SOURCE").unwrap();
+        let reference = std::env::var_os("DS4SERVER_QWEN38_CHAT_REFERENCE").unwrap();
+        let fixture: serde_json::Value =
+            serde_json::from_slice(&fs::read(reference).unwrap()).unwrap();
+        let tokenizer = Tokenizer::load_qwen(&Path::new(&root).join("tokenizer.json")).unwrap();
+        let cases = fixture["cases"].as_array().unwrap();
+        assert_eq!(cases.len(), 3);
+        for (index, case) in cases.iter().enumerate() {
+            let messages = case["messages"]
+                .as_array()
+                .unwrap()
+                .iter()
+                .map(|message| ChatTurn {
+                    user: message["role"] == "user",
+                    tool: false,
+                    system: false,
+                    skip_previous_eos: false,
+                    reasoning: message["reasoning_content"].as_str().map(str::to_owned),
+                    reasoning_complete: true,
+                    content: message["content"].as_str().unwrap().to_owned(),
+                })
+                .collect::<Vec<_>>();
+            let actual = tokenizer.encode_conversation("", &messages, ReasoningMode::Low);
+            let rendered = actual
+                .iter()
+                .flat_map(|&id| tokenizer.token_bytes(id).unwrap())
+                .collect::<Vec<_>>();
+            assert_eq!(
+                rendered,
+                case["rendered"].as_str().unwrap().as_bytes(),
+                "turn {} rendered bytes",
+                index + 1
+            );
+            let expected: Vec<i32> = serde_json::from_value(case["token_ids"].clone()).unwrap();
+            assert!(
+                actual == expected,
+                "turn {} token mismatch: actual={}, reference={}, first_difference={:?}",
+                index + 1,
+                actual.len(),
+                expected.len(),
+                actual.iter().zip(&expected).position(|(a, b)| a != b)
+            );
+            let mut resumed =
+                tokenizer.encode_history("", &messages[..messages.len() - 1], ReasoningMode::Low);
+            resumed.extend(tokenizer.encode_continuation(
+                &messages.last().unwrap().content,
+                ReasoningMode::Low,
+                false,
+            ));
+            assert!(resumed == expected, "turn {} checkpoint tokens", index + 1);
+        }
+    }
+
+    #[test]
+    #[ignore = "requires the installed Qwen tokenizer, but no GPU or model weights"]
+    fn qwen_checkpoint_continuation_matches_full_conversation() {
+        let root = std::env::var_os("DS4SERVER_QWEN38_SOURCE")
+            .expect("set DS4SERVER_QWEN38_SOURCE to the installed artifact directory");
+        let tokenizer = Tokenizer::load_qwen(&Path::new(&root).join("tokenizer.json")).unwrap();
+        let user = ChatTurn {
+            user: true,
+            tool: false,
+            system: false,
+            skip_previous_eos: false,
+            reasoning: None,
+            reasoning_complete: true,
+            content: "ping".into(),
+        };
+        let assistant = ChatTurn {
+            user: false,
+            reasoning: Some("Brief thought.".into()),
+            content: "pong".into(),
+            ..user.clone()
+        };
+        for reasoning in [
+            ReasoningMode::Direct,
+            ReasoningMode::Low,
+            ReasoningMode::Medium,
+            ReasoningMode::XHigh,
+        ] {
+            for mut history in [vec![], vec![user.clone(), assistant.clone()]] {
+                let mut resumed =
+                    tokenizer.encode_history("System instruction", &history, reasoning);
+                resumed.extend(tokenizer.encode_continuation("Next question", reasoning, false));
+                history.push(ChatTurn {
+                    content: "Next question".into(),
+                    ..user.clone()
+                });
+                assert_eq!(
+                    resumed,
+                    tokenizer.encode_conversation("System instruction", &history, reasoning),
+                    "checkpoint/full prompt mismatch: {reasoning:?}, {} messages",
+                    history.len(),
+                );
+            }
+        }
+    }
+
     #[test]
     fn character_helpers_are_total_at_every_byte_offset() {
         let text = "Aé中😊";