feat: first cut at actual token generation and model loading

src/app.rs

@@ -3,6 +3,8 @@ mod view;
 pub(crate) use view::app_theme;
 
 use crate::database::{AppPreferences, Database, ProjectWithSessions};
+#[cfg(target_os = "macos")]
+use crate::engine::{ChatTurn, Generator};
 use crate::model::{self, DownloadOutcome, DownloadProgress, ManagedArtifactId, ModelChoice};
 use crate::settings::{
     DiagnosticPreferences, ExecutionPreferences, GIB, GenerationPreferences, ReasoningMode,
@@ -398,9 +400,46 @@ pub(crate) struct App {
     pending_project_path: Option<PathBuf>,
     project_name_input: String,
     model_download: ModelDownload,
+    pub(super) composer: String,
+    pub(super) conversation: Vec<ChatMessage>,
+    pub(super) generating: bool,
+    #[cfg(target_os = "macos")]
+    generation_worker: Option<GenerationWorker>,
     error: Option<String>,
 }
 
+#[derive(Clone, Debug)]
+pub(super) struct ChatMessage {
+    pub(super) user: bool,
+    reasoning: bool,
+    pub(super) content: String,
+}
+
+#[cfg(target_os = "macos")]
+struct GenerationWorker {
+    commands: mpsc::Sender<GenerationCommand>,
+    events: mpsc::Receiver<GenerationEvent>,
+    cancel: Option<Arc<AtomicBool>>,
+}
+
+#[cfg(target_os = "macos")]
+enum GenerationCommand {
+    Generate {
+        engine: crate::settings::EngineSettings,
+        turn: crate::settings::TurnSettings,
+        messages: Vec<ChatTurn>,
+        idle_timeout: Duration,
+        cancel: Arc<AtomicBool>,
+    },
+}
+
+#[cfg(target_os = "macos")]
+enum GenerationEvent {
+    Loading,
+    Chunk(String),
+    Finished(Result<(), String>),
+}
+
 #[derive(Debug)]
 pub(super) enum ModelDownload {
     Idle,
@@ -479,6 +518,10 @@ pub(crate) enum Message {
     CancelDeleteArtifact,
     StopModelDownload,
     DownloadProgressTick,
+    ComposerChanged(String),
+    SubmitPrompt,
+    StopGeneration,
+    GenerationTick,
     ChooseProjectFolder,
     ProjectFolderPicked(Option<PathBuf>),
     ProjectNameChanged(String),
@@ -519,6 +562,11 @@ impl App {
                         pending_project_path: None,
                         project_name_input: String::new(),
                         model_download: ModelDownload::Idle,
+                        composer: String::new(),
+                        conversation: Vec::new(),
+                        generating: false,
+                        #[cfg(target_os = "macos")]
+                        generation_worker: None,
                         error: None,
                     }
                 }
@@ -550,6 +598,11 @@ impl App {
             pending_project_path: None,
             project_name_input: String::new(),
             model_download: ModelDownload::Idle,
+            composer: String::new(),
+            conversation: Vec::new(),
+            generating: false,
+            #[cfg(target_os = "macos")]
+            generation_worker: None,
             error: Some(format!("Could not open the project database: {error}")),
         }
     }
@@ -816,6 +869,19 @@ impl App {
                 }
             }
             Message::DownloadProgressTick => self.update_download_progress(),
+            Message::ComposerChanged(value) => self.composer = value,
+            Message::SubmitPrompt => self.start_generation(),
+            Message::StopGeneration => {
+                #[cfg(target_os = "macos")]
+                if let Some(cancel) = self
+                    .generation_worker
+                    .as_ref()
+                    .and_then(|worker| worker.cancel.as_ref())
+                {
+                    cancel.store(true, Ordering::Relaxed);
+                }
+            }
+            Message::GenerationTick => self.poll_generation(),
             Message::ChooseProjectFolder => {
                 self.choosing_folder = true;
                 return Task::perform(
@@ -843,11 +909,21 @@ impl App {
                 self.error = None;
             }
             Message::SelectProject(project_id) => {
+                if self.generating {
+                    self.error =
+                        Some("Stop the active generation before changing sessions.".into());
+                    return Task::none();
+                }
                 self.selected_project = Some(project_id);
                 self.selected_session = None;
                 self.error = None;
             }
             Message::DeleteProject(project_id) => {
+                if self.generating && self.selected_project == Some(project_id) {
+                    self.error =
+                        Some("Stop the active generation before deleting its project.".into());
+                    return Task::none();
+                }
                 if let Some(database) = &mut self.database {
                     match database.delete_project(project_id) {
                         Ok(()) => {
@@ -863,11 +939,25 @@ impl App {
             }
             Message::CreateSession => self.create_session(),
             Message::SelectSession(project_id, session_id) => {
+                if self.generating && self.selected_session != Some(session_id) {
+                    self.error =
+                        Some("Stop the active generation before changing sessions.".into());
+                    return Task::none();
+                }
+                if self.selected_session != Some(session_id) {
+                    self.conversation.clear();
+                    self.composer.clear();
+                }
                 self.selected_project = Some(project_id);
                 self.selected_session = Some(session_id);
                 self.error = None;
             }
             Message::DeleteSession(session_id) => {
+                if self.generating && self.selected_session == Some(session_id) {
+                    self.error =
+                        Some("Stop the active generation before deleting its session.".into());
+                    return Task::none();
+                }
                 if let Some(database) = &mut self.database {
                     match database.delete_session(session_id) {
                         Ok(()) => {
@@ -905,6 +995,11 @@ impl App {
                 iced::time::every(Duration::from_secs(1)).map(|_| Message::DownloadProgressTick),
             );
         }
+        if self.generating {
+            subscriptions.push(
+                iced::time::every(Duration::from_millis(50)).map(|_| Message::GenerationTick),
+            );
+        }
         Subscription::batch(subscriptions)
     }
 
@@ -1108,6 +1203,10 @@ impl App {
     }
 
     fn create_session(&mut self) {
+        if self.generating {
+            self.error = Some("Stop the active generation before creating a session.".into());
+            return;
+        }
         let Some(project_id) = self.selected_project else {
             self.error = Some("Select a project first.".into());
             return;
@@ -1205,6 +1304,196 @@ impl App {
             }
         }
     }
+
+    fn start_generation(&mut self) {
+        if self.generating || self.selected_session.is_none() {
+            return;
+        }
+        let prompt = self.composer.trim().to_owned();
+        if prompt.is_empty() {
+            return;
+        }
+        let model = match ModelChoice::from_id(&self.preferences.selected_model) {
+            Some(model) => model,
+            None => {
+                self.error = Some("The selected model is not supported.".into());
+                return;
+            }
+        };
+        let effective = self.preferences.generation().and_then(|generation| {
+            self.preferences.runtime().and_then(|runtime| {
+                crate::settings::effective_settings(model, &generation, &runtime, &models_path())
+            })
+        });
+        let effective = match effective {
+            Ok(settings) => settings,
+            Err(error) => {
+                self.error = Some(error);
+                return;
+            }
+        };
+        let assistant_reasoning = effective.turn.reasoning_mode != ReasoningMode::Direct;
+        #[cfg(target_os = "macos")]
+        let mut messages = self
+            .conversation
+            .iter()
+            .map(|message| ChatTurn {
+                user: message.user,
+                reasoning: message.reasoning,
+                content: message.content.clone(),
+            })
+            .collect::<Vec<_>>();
+        #[cfg(target_os = "macos")]
+        messages.push(ChatTurn {
+            user: true,
+            reasoning: false,
+            content: prompt.clone(),
+        });
+
+        #[cfg(target_os = "macos")]
+        {
+            if self.generation_worker.is_none() {
+                match spawn_generation_worker() {
+                    Ok(worker) => self.generation_worker = Some(worker),
+                    Err(error) => {
+                        self.error = Some(error);
+                        return;
+                    }
+                }
+            }
+            let cancel = Arc::new(AtomicBool::new(false));
+            let idle_timeout =
+                Duration::from_secs(self.preferences.idle_timeout_minutes.max(1) as u64 * 60);
+            let command = GenerationCommand::Generate {
+                engine: effective.engine,
+                turn: effective.turn,
+                messages,
+                idle_timeout,
+                cancel: Arc::clone(&cancel),
+            };
+            let worker = self.generation_worker.as_mut().expect("worker was created");
+            if worker.commands.send(command).is_err() {
+                self.generation_worker = None;
+                self.error = Some("The local generation worker stopped unexpectedly.".into());
+                return;
+            }
+            worker.cancel = Some(cancel);
+        }
+        #[cfg(not(target_os = "macos"))]
+        {
+            let _ = effective;
+            self.error = Some("Local Metal generation requires macOS.".into());
+            return;
+        }
+        self.composer.clear();
+        self.conversation.push(ChatMessage {
+            user: true,
+            reasoning: false,
+            content: prompt,
+        });
+        self.conversation.push(ChatMessage {
+            user: false,
+            reasoning: assistant_reasoning,
+            content: String::new(),
+        });
+        self.generating = true;
+        self.error = None;
+    }
+
+    fn poll_generation(&mut self) {
+        #[cfg(target_os = "macos")]
+        let Some(worker) = &mut self.generation_worker else {
+            self.generating = false;
+            return;
+        };
+        #[cfg(target_os = "macos")]
+        loop {
+            match worker.events.try_recv() {
+                Ok(GenerationEvent::Loading) => {}
+                Ok(GenerationEvent::Chunk(chunk)) => {
+                    if let Some(message) = self.conversation.last_mut()
+                        && !message.user
+                    {
+                        message.content.push_str(&chunk);
+                    }
+                }
+                Ok(GenerationEvent::Finished(result)) => {
+                    self.generating = false;
+                    worker.cancel = None;
+                    if let Err(error) = result {
+                        self.error = Some(error);
+                    }
+                    break;
+                }
+                Err(TryRecvError::Empty) => break,
+                Err(TryRecvError::Disconnected) => {
+                    self.generating = false;
+                    self.generation_worker = None;
+                    self.error = Some("The local generation worker stopped unexpectedly.".into());
+                    break;
+                }
+            }
+        }
+    }
+}
+
+#[cfg(target_os = "macos")]
+fn spawn_generation_worker() -> Result<GenerationWorker, String> {
+    let (command_sender, command_receiver) = mpsc::channel();
+    let (event_sender, event_receiver) = mpsc::channel();
+    thread::Builder::new()
+        .name("local-generation".into())
+        .spawn(move || {
+            let mut loaded = None::<(crate::settings::EngineSettings, Generator)>;
+            let mut last_used = Instant::now();
+            let mut idle_timeout = Duration::from_secs(15 * 60);
+            loop {
+                match command_receiver.recv_timeout(Duration::from_secs(1)) {
+                    Ok(GenerationCommand::Generate {
+                        engine,
+                        turn,
+                        messages,
+                        idle_timeout: requested_timeout,
+                        cancel,
+                    }) => {
+                        idle_timeout = requested_timeout;
+                        if loaded
+                            .as_ref()
+                            .is_none_or(|(current, _)| current != &engine)
+                        {
+                            let _ = event_sender.send(GenerationEvent::Loading);
+                            loaded = match Generator::open(&engine) {
+                                Ok(generator) => Some((engine.clone(), generator)),
+                                Err(error) => {
+                                    let _ =
+                                        event_sender.send(GenerationEvent::Finished(Err(error)));
+                                    None
+                                }
+                            };
+                        }
+                        if let Some((_, generator)) = &mut loaded {
+                            let result = generator.generate(&messages, &turn, &cancel, |chunk| {
+                                let _ = event_sender.send(GenerationEvent::Chunk(chunk));
+                            });
+                            let _ = event_sender.send(GenerationEvent::Finished(result));
+                            last_used = Instant::now();
+                        }
+                    }
+                    Err(mpsc::RecvTimeoutError::Timeout) => {
+                        if loaded.is_some() && last_used.elapsed() >= idle_timeout {
+                            loaded = None;
+                        }
+                    }
+                    Err(mpsc::RecvTimeoutError::Disconnected) => break,
+                }
+            }
+        })
+        .map_err(|error| format!("Could not start local generation: {error}"))?;
+    Ok(GenerationWorker {
+        commands: command_sender,
+        events: event_receiver,
+        cancel: None,
+    })
 }
 
 impl Drop for App {
@@ -1212,6 +1501,14 @@ impl Drop for App {
         if let ModelDownload::Active(download) = &self.model_download {
             download.cancel.store(true, Ordering::Relaxed);
         }
+        #[cfg(target_os = "macos")]
+        if let Some(cancel) = self
+            .generation_worker
+            .as_ref()
+            .and_then(|worker| worker.cancel.as_ref())
+        {
+            cancel.store(true, Ordering::Relaxed);
+        }
     }
 }