feat: first cut at actual token generation and model loading

src/engine/tokenizer.rs

@@ -1,5 +1,5 @@
-use super::ModelFamily;
 use super::gguf::Gguf;
+use super::{ChatTurn, ModelFamily};
 use crate::settings::ReasoningMode;
 use std::collections::HashMap;
 
@@ -131,6 +131,23 @@ impl Tokenizer {
         system_prompt: &str,
         prompt: &str,
         reasoning: ReasoningMode,
+    ) -> Vec<i32> {
+        self.encode_conversation(
+            system_prompt,
+            &[ChatTurn {
+                user: true,
+                reasoning: false,
+                content: prompt.to_owned(),
+            }],
+            reasoning,
+        )
+    }
+
+    pub(super) fn encode_conversation(
+        &self,
+        system_prompt: &str,
+        messages: &[ChatTurn],
+        reasoning: ReasoningMode,
     ) -> Vec<i32> {
         let mut output = vec![self.bos];
         if self.family == ModelFamily::Glm && self.sop >= 0 {
@@ -156,8 +173,23 @@ impl Tokenizer {
             }
             output.extend(self.tokenize(system_prompt));
         }
-        output.push(self.user);
-        output.extend(self.tokenize(prompt));
+        for message in messages {
+            output.push(if message.user {
+                self.user
+            } else {
+                self.assistant
+            });
+            if !message.user {
+                if message.reasoning {
+                    output.push(self.think_start);
+                } else if self.family == ModelFamily::Glm {
+                    output.extend([self.think_start, self.think_end]);
+                } else {
+                    output.push(self.think_end);
+                }
+            }
+            output.extend(self.tokenize(&message.content));
+        }
         output.push(self.assistant);
         if reasoning != ReasoningMode::Direct {
             output.push(self.think_start);