feat: first cut at openai compatible server

src/runtime.rs

@@ -0,0 +1,151 @@
+use crate::engine::{ChatTurn, GenerationOutput, Generator};
+use crate::settings::{EngineSettings, TurnSettings};
+use std::path::PathBuf;
+use std::sync::Arc;
+use std::sync::atomic::AtomicBool;
+use std::sync::mpsc::{self, Receiver, Sender};
+use std::thread;
+use std::time::{Duration, Instant};
+
+#[derive(Clone)]
+pub(crate) struct GenerationService {
+    commands: Sender<Command>,
+}
+
+pub(crate) struct ActiveGeneration {
+    pub(crate) events: Receiver<GenerationEvent>,
+    pub(crate) cancel: Arc<AtomicBool>,
+}
+
+pub(crate) enum CheckpointTarget {
+    Local(PathBuf),
+    Transient(PathBuf),
+}
+
+pub(crate) enum GenerationEvent {
+    Loading,
+    Chunk {
+        reasoning: bool,
+        content: String,
+    },
+    Context {
+        used: u32,
+        limit: u32,
+        tokens_per_second: Option<f32>,
+    },
+    Finished(Result<GenerationOutput, String>),
+}
+
+struct Command {
+    engine: EngineSettings,
+    turn: TurnSettings,
+    messages: Vec<ChatTurn>,
+    checkpoint: CheckpointTarget,
+    idle_timeout: Duration,
+    cancel: Arc<AtomicBool>,
+    events: Sender<GenerationEvent>,
+}
+
+impl GenerationService {
+    pub(crate) fn spawn() -> Result<Self, String> {
+        let (commands, receiver) = mpsc::channel::<Command>();
+        thread::Builder::new()
+            .name("model-runtime".into())
+            .spawn(move || run(receiver))
+            .map_err(|error| format!("Could not start the model runtime: {error}"))?;
+        Ok(Self { commands })
+    }
+
+    pub(crate) fn generate(
+        &self,
+        engine: EngineSettings,
+        turn: TurnSettings,
+        messages: Vec<ChatTurn>,
+        checkpoint: CheckpointTarget,
+        idle_timeout: Duration,
+    ) -> Result<ActiveGeneration, String> {
+        let cancel = Arc::new(AtomicBool::new(false));
+        let (events, receiver) = mpsc::channel();
+        self.commands
+            .send(Command {
+                engine,
+                turn,
+                messages,
+                checkpoint,
+                idle_timeout,
+                cancel: Arc::clone(&cancel),
+                events,
+            })
+            .map_err(|_| "The model runtime stopped unexpectedly.".to_owned())?;
+        Ok(ActiveGeneration {
+            events: receiver,
+            cancel,
+        })
+    }
+}
+
+fn run(commands: Receiver<Command>) {
+    let mut loaded = None::<(EngineSettings, Generator)>;
+    let mut last_used = Instant::now();
+    let mut idle_timeout = Duration::from_secs(15 * 60);
+    loop {
+        match commands.recv_timeout(Duration::from_secs(1)) {
+            Ok(command) => {
+                idle_timeout = command.idle_timeout;
+                if loaded
+                    .as_ref()
+                    .is_none_or(|(settings, _)| settings != &command.engine)
+                {
+                    let _ = command.events.send(GenerationEvent::Loading);
+                    loaded = match Generator::open(&command.engine) {
+                        Ok(generator) => Some((command.engine.clone(), generator)),
+                        Err(error) => {
+                            let _ = command.events.send(GenerationEvent::Finished(Err(error)));
+                            None
+                        }
+                    };
+                }
+                if let Some((_, generator)) = &mut loaded {
+                    let mut emit = |reasoning, content| {
+                        let _ = command
+                            .events
+                            .send(GenerationEvent::Chunk { reasoning, content });
+                    };
+                    let mut progress = |used, limit, tokens_per_second| {
+                        let _ = command.events.send(GenerationEvent::Context {
+                            used,
+                            limit,
+                            tokens_per_second,
+                        });
+                    };
+                    let result = match command.checkpoint {
+                        CheckpointTarget::Local(path) => generator.generate(
+                            &path,
+                            &command.messages,
+                            &command.turn,
+                            &command.cancel,
+                            &mut emit,
+                            &mut progress,
+                        ),
+                        CheckpointTarget::Transient(directory) => generator.generate_transient(
+                            &directory,
+                            &command.messages,
+                            &command.turn,
+                            &command.cancel,
+                            &mut emit,
+                            &mut progress,
+                        ),
+                    };
+                    let _ = command.events.send(GenerationEvent::Finished(result));
+                    last_used = Instant::now();
+                }
+            }
+            Err(mpsc::RecvTimeoutError::Timeout) => {
+                if loaded.is_some() && last_used.elapsed() >= idle_timeout {
+                    loaded = None;
+                }
+            }
+            Err(mpsc::RecvTimeoutError::Disconnected) => break,
+        }
+    }
+}