fix: utf-8 was decoded before complete text, so emojis broke

src/engine.rs

@@ -559,6 +559,7 @@ impl Generator {
         };
         let mut emitted_reasoning = 0;
         let mut emitted_content = 0;
+        let mut pending_utf8 = Vec::new();
         let prompt_tokens = tokens.len();
         let suffix = &tokens[reused..];
         let completed = if (reused == 0 && tokens.len() > 1) || suffix.len() >= 4 {
@@ -602,6 +603,7 @@ impl Generator {
             .min((max_context - self.executor.position() as usize) as i32)
         {
             if cancelled.load(Ordering::Relaxed) {
+                append_generated_bytes(&mut generated, reasoning, &mut pending_utf8, &[], true);
                 flush_generated(
                     &mut generated,
                     &mut emitted_reasoning,
@@ -635,6 +637,7 @@ impl Generator {
                 .model()
                 .is_stop_token_for_reasoning(token, settings.reasoning_mode)
             {
+                append_generated_bytes(&mut generated, reasoning, &mut pending_utf8, &[], true);
                 flush_generated(
                     &mut generated,
                     &mut emitted_reasoning,
@@ -656,9 +659,11 @@ impl Generator {
                 ));
             }
             if self.executor.model().is_think_start_token(token) {
+                append_generated_bytes(&mut generated, reasoning, &mut pending_utf8, &[], true);
                 reasoning = true;
                 generated.reasoning.get_or_insert_default();
             } else if self.executor.model().is_think_end_token(token) {
+                append_generated_bytes(&mut generated, reasoning, &mut pending_utf8, &[], true);
                 if let Some(reasoning_text) = &mut generated.reasoning
                     && emit_safe_text(
                         reasoning_text,
@@ -686,10 +691,9 @@ impl Generator {
                 generated.reasoning_complete = true;
                 emit(false, String::new());
             } else if let Some(bytes) = self.executor.model().token_bytes(token) {
-                let content = String::from_utf8_lossy(&bytes).into_owned();
+                append_generated_bytes(&mut generated, reasoning, &mut pending_utf8, &bytes, false);
                 let stopped = if reasoning {
                     let text = generated.reasoning.get_or_insert_default();
-                    text.push_str(&content);
                     emit_safe_text(
                         text,
                         &mut emitted_reasoning,
@@ -700,7 +704,6 @@ impl Generator {
                     )
                 } else {
                     generated.reasoning_complete = true;
-                    generated.content.push_str(&content);
                     emit_safe_text(
                         &mut generated.content,
                         &mut emitted_content,
@@ -736,6 +739,7 @@ impl Generator {
                 ),
             );
         }
+        append_generated_bytes(&mut generated, reasoning, &mut pending_utf8, &[], true);
         flush_generated(
             &mut generated,
             &mut emitted_reasoning,
@@ -758,6 +762,48 @@ impl Generator {
     }
 }
 
+#[cfg(target_os = "macos")]
+fn append_generated_bytes(
+    generated: &mut ChatTurn,
+    reasoning: bool,
+    pending: &mut Vec<u8>,
+    bytes: &[u8],
+    final_flush: bool,
+) {
+    let text = if reasoning {
+        generated.reasoning.get_or_insert_default()
+    } else {
+        &mut generated.content
+    };
+    pending.extend_from_slice(bytes);
+    loop {
+        match std::str::from_utf8(pending) {
+            Ok(valid) => {
+                text.push_str(valid);
+                pending.clear();
+                return;
+            }
+            Err(error) => {
+                let valid = error.valid_up_to();
+                text.push_str(std::str::from_utf8(&pending[..valid]).unwrap());
+                pending.drain(..valid);
+                match error.error_len() {
+                    Some(length) => {
+                        text.push('\u{fffd}');
+                        pending.drain(..length);
+                    }
+                    None if final_flush => {
+                        text.push_str(&String::from_utf8_lossy(pending));
+                        pending.clear();
+                        return;
+                    }
+                    None => return,
+                }
+            }
+        }
+    }
+}
+
 #[cfg(target_os = "macos")]
 fn flush_generated(
     generated: &mut ChatTurn,
@@ -994,6 +1040,25 @@ mod sampling_tests {
         assert_eq!(chunks, ["hello "]);
     }
 
+    #[test]
+    fn split_utf8_token_bytes_are_joined_before_decoding() {
+        let mut generated = ChatTurn {
+            user: false,
+            skip_previous_eos: false,
+            reasoning: None,
+            reasoning_complete: true,
+            content: String::new(),
+        };
+        let mut pending = Vec::new();
+
+        append_generated_bytes(&mut generated, false, &mut pending, &[0xf0, 0x9f], false);
+        assert!(generated.content.is_empty());
+        append_generated_bytes(&mut generated, false, &mut pending, &[0x98, 0x8a], false);
+
+        assert_eq!(generated.content, "😊");
+        assert!(pending.is_empty());
+    }
+
     #[test]
     fn checkpoint_tag_covers_the_canonical_chat_state() {
         let messages = [ChatTurn {