Match Qwen reasoning history across effort changes

src/engine/tokenizer.rs

@@ -538,6 +538,39 @@ impl Tokenizer {
         reasoning: ReasoningMode,
         continue_assistant: bool,
     ) -> Vec<i32> {
+        let messages = qwen_history(messages, reasoning);
+        let rendered =
+            Self::render_qwen_messages(system_prompt, &messages, reasoning, continue_assistant);
+        // MTPLX preserves the generation-time BPE boundary in thinking
+        // histories, including the two separate newlines of an empty thought.
+        // Direct mode tokenizes the closed scaffold together.
+        if reasoning != ReasoningMode::Direct
+            && messages
+                .iter()
+                .any(|message| !message.user && !message.tool && !message.system)
+        {
+            let mut output = Vec::new();
+            let mut start = 0;
+            for (at, marker) in rendered.match_indices(QWEN_ASSISTANT_THINK_PROMPT) {
+                let end = at + marker.len();
+                if end < rendered.len() {
+                    output.extend(self.tokenize_rendered(&rendered[start..end]));
+                    start = end;
+                }
+            }
+            output.extend(self.tokenize_rendered(&rendered[start..]));
+            output
+        } else {
+            self.tokenize_rendered(&rendered)
+        }
+    }
+
+    fn render_qwen_messages(
+        system_prompt: Option<&str>,
+        messages: &[ChatTurn],
+        reasoning: ReasoningMode,
+        continue_assistant: bool,
+    ) -> String {
         let instruction = match reasoning {
             ReasoningMode::Direct | ReasoningMode::Medium => "",
             ReasoningMode::Low => {
@@ -587,7 +620,11 @@ impl Tokenizer {
                 rendered.push_str("<|im_end|>\n");
             } else {
                 rendered.push_str("<|im_start|>assistant\n<think>\n");
-                if let Some(thinking) = &message.reasoning {
+                // MTPLX's preserve-echo policy excludes reasoning_content
+                // when enable_thinking is false, retaining the empty scaffold.
+                if reasoning != ReasoningMode::Direct
+                    && let Some(thinking) = &message.reasoning
+                {
                     rendered.push_str(thinking.trim());
                 }
                 rendered.push_str("\n</think>\n\n");
@@ -601,7 +638,7 @@ impl Tokenizer {
                 rendered.push_str("\n</think>\n\n");
             }
         }
-        self.tokenize_rendered(&rendered)
+        rendered
     }
 
     pub(super) fn encode_continuation(
@@ -895,6 +932,48 @@ pub(super) fn qwen_normalization_changes(text: &str) -> bool {
     qwen_normalize(text) != text
 }
 
+pub(super) const QWEN_ASSISTANT_THINK_PROMPT: &str = "<|im_start|>assistant\n<think>\n";
+
+pub(super) fn qwen_void_assistant(message: &ChatTurn) -> bool {
+    !message.user && !message.tool && !message.system && message.content.trim().is_empty()
+}
+
+fn qwen_history(messages: &[ChatTurn], mode: ReasoningMode) -> Vec<ChatTurn> {
+    // MTPLX drops void assistants, then merges adjacent user/assistant roles.
+    // The server also drops reasoning-only assistants. Keep the
+    // first assistant's reasoning when merging; trim user text after merging.
+    let mut normalized: Vec<ChatTurn> = Vec::with_capacity(messages.len());
+    for message in messages {
+        if qwen_void_assistant(message) {
+            continue;
+        }
+        let mut message = message.clone();
+        if !message.user && !message.tool && !message.system {
+            message.content = message.content.trim().to_owned();
+            if mode == ReasoningMode::Direct {
+                message.reasoning = None;
+            }
+        }
+        if let Some(previous) = normalized.last_mut()
+            && !previous.tool
+            && !previous.system
+            && !message.tool
+            && !message.system
+            && previous.user == message.user
+        {
+            if !message.content.is_empty() {
+                if !previous.content.is_empty() {
+                    previous.content.push_str("\n\n");
+                }
+                previous.content.push_str(&message.content);
+            }
+        } else {
+            normalized.push(message);
+        }
+    }
+    normalized
+}
+
 /// The trained Qwen3 Split expression. Rust regex supports its Unicode
 /// categories but not the penultimate `\s+(?!\S)` alternative: before a
 /// non-whitespace character that branch backtracks by one whitespace scalar,
@@ -1418,3 +1497,7 @@ mod issue111;
 #[cfg(test)]
 #[path = "tokenizer_issue116.rs"]
 mod issue116;
+
+#[cfg(test)]
+#[path = "tokenizer_issue117.rs"]
+mod issue117;