Make compaction transitions recoverable

src/engine/tokenizer.rs

@@ -210,6 +210,25 @@ impl Tokenizer {
         system_prompt: &str,
         messages: &[ChatTurn],
         reasoning: ReasoningMode,
+    ) -> Vec<i32> {
+        self.encode_messages(system_prompt, messages, reasoning, true)
+    }
+
+    pub(super) fn encode_history(
+        &self,
+        system_prompt: &str,
+        messages: &[ChatTurn],
+        reasoning: ReasoningMode,
+    ) -> Vec<i32> {
+        self.encode_messages(system_prompt, messages, reasoning, false)
+    }
+
+    fn encode_messages(
+        &self,
+        system_prompt: &str,
+        messages: &[ChatTurn],
+        reasoning: ReasoningMode,
+        continue_assistant: bool,
     ) -> Vec<i32> {
         let mut output = vec![self.bos];
         if self.family == ModelFamily::Glm && self.sop >= 0 {
@@ -235,7 +254,7 @@ impl Tokenizer {
             }
             output.extend(self.tokenize_rendered(system_prompt));
         }
-        for message in messages {
+        for (index, message) in messages.iter().enumerate() {
             if message.tool {
                 if self.family == ModelFamily::Glm {
                     output.push(self.observation);
@@ -281,17 +300,22 @@ impl Tokenizer {
                 }
             }
             output.extend(self.tokenize_rendered(&message.content));
-            if !message.user && self.family == ModelFamily::DeepSeek {
+            if !message.user
+                && self.family == ModelFamily::DeepSeek
+                && (continue_assistant || index + 1 < messages.len())
+            {
                 output.push(self.eos);
             }
         }
-        output.push(self.assistant);
-        if reasoning != ReasoningMode::Direct {
-            output.push(self.think_start);
-        } else if self.family == ModelFamily::Glm {
-            output.extend([self.think_start, self.think_end]);
-        } else {
-            output.push(self.think_end);
+        if continue_assistant {
+            output.push(self.assistant);
+            if reasoning != ReasoningMode::Direct {
+                output.push(self.think_start);
+            } else if self.family == ModelFamily::Glm {
+                output.extend([self.think_start, self.think_end]);
+            } else {
+                output.push(self.think_end);
+            }
         }
         output
     }