feat: first cut at openai compatible server

Cargo.lock

@@ -1150,6 +1150,8 @@ dependencies = [
  "muda",
  "png",
  "rfd",
+ "serde",
+ "serde_json",
  "sha2",
  "ureq",
 ]
@@ -4037,6 +4039,7 @@ version = "1.0.151"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14"
 dependencies = [
+ "indexmap",
  "itoa",
  "memchr",
  "serde",

Cargo.toml

@@ -14,10 +14,12 @@ cc = "1.3.0"
 [dependencies]
 diesel = { version = "2.3.11", features = ["sqlite", "returning_clauses_for_sqlite_3_35", "64-column-tables"] }
 diesel_migrations = "2.3.2"
-iced = { version = "0.13.1", features = ["highlighter", "markdown", "svg", "tokio"] }
+iced = { version = "0.13.1", features = ["advanced", "highlighter", "markdown", "svg", "tokio"] }
 memmap2 = "0.9.11"
 png = "0.17.16"
 rfd = "0.15.4"
+serde = { version = "1.0.228", features = ["derive"] }
+serde_json = { version = "1.0.149", features = ["preserve_order"] }
 sha2 = "0.11.0"
 ureq = { version = "3.3.0", default-features = false, features = ["rustls"] }
 

PLAN.md

@@ -13,11 +13,11 @@ sessions, tools, and turn orchestration.
 | Area | Status | Available now | Still open |
 | --- | --- | --- | --- |
 | App shell | Implemented | Native multi-window Iced app, Codex-inspired layout, native Application/Edit/Window menus, Preferences panel, Model Manager, and streaming composer | Native app release packaging |
-| Projects and sessions | Implemented with transcripts | Native folder picker, project-name dialog, create/select/delete projects and sessions, and durable structured chat history | KV payloads, rename/archive, and model binding |
+| Projects and sessions | Implemented with transcripts and KV resume | Native folder picker, project-name dialog, create/select/delete projects and sessions, durable structured chat history, and per-session KV checkpoints | Rename/archive and model binding |
-| Persistence | Implemented for metadata, preferences, and chat | SQLite in Application Support, Diesel ORM, embedded migrations, constrained defaults, streamed message updates, and reopen tests | Downloaded-artifact metadata and session-runtime migrations |
+| Persistence | Implemented for local chat | SQLite in Application Support, Diesel ORM, embedded migrations, constrained defaults, streamed message updates, reopen tests, context/speed state, and local KV rehydration | Downloaded-artifact metadata and session-runtime migrations; HTTP chats deliberately never enter SQLite |
-| Model runtime | DeepSeek Flash vertical implemented | Rust-owned mmap/model/session graph, reused Metal kernels, 2K compressed-attention generation, DS4 sampling, lazy background loading, cancellation, and idle unloading | Ratio-4 sparse indexer beyond 2K, DSpark/SSD/steering, GLM/Pro, KV checkpoint persistence, and shared acquisition for HTTP |
+| Model runtime | DeepSeek Flash vertical implemented | Rust-owned mmap/model/session graph, local Metal kernels, full configured sparse context, DS4 sampling, one process-wide UI/HTTP owner, cancellation, idle unloading, prefix continuation, and KV checkpoint save/load | DSpark/SSD/steering and GLM/Pro execution |
-| Local endpoint | Not started | Nothing listening | OpenAI-compatible HTTP and streaming surfaces |
+| Local endpoint | Chat Completions vertical implemented | Automatic configurable localhost listener (port 4000 by default), verified-on-disk model discovery, transient requests, non-streaming and SSE Chat Completions, reasoning, usage, sampling, stops, cancellation, tools, and exact in-process tool replay | GUI status and enable/disable controls, Responses, legacy Completions, Anthropic Messages, full `ds4_server.c` fixture parity, and cross-restart exact tool replay |
-| Agent | Basic local chat implemented | Multi-turn role rendering, structured reasoning disclosure, streamed DeepSeek Flash text, composer send/stop, and durable transcript rehydration | Tools, approvals, compaction, statistics, and KV-backed resume |
+| Agent | Solid local chat implemented | Multi-turn role rendering, structured reasoning disclosure, Markdown, automatic scrolling, context/speed display, durable transcript rehydration, and KV-backed resume | Tools, approvals, compaction, and richer statistics |
 | A2UI | Future extension | bDS2 provides the reference structured render-tool contract | Native inline surfaces for local chat after core chat and tools are stable |
 | Dev brain | Not started | No vault access | Obsidian vault selection, durable access, and agent knowledge/memory tools |
 | Release | Not started | Development binary builds and tests | `.app` packaging, signing, entitlements, and notarization |
@@ -28,8 +28,9 @@ without blocking the UI. Local chat now lazily acquires the Rust DeepSeek Flash
 executor, streams generated text, supports cancellation and multi-turn role
 replay, persists reasoning and answers while they stream, rehydrates selected
 sessions, and unloads model resources after the configured idle timeout. The
-current Rust graph supports DS4 compressed attention through 2K context; the sparse ratio-4 indexer,
+current Rust graph supports the full configured context with ratio-4 sparse
-KV checkpoints, HTTP service, and agent tools remain open.
+attention and durable local KV checkpoints. The shared HTTP Chat Completions
+service is available; the remaining protocol routes and agent tools remain open.
 
 ## Phase 0 — project and session shell
 
@@ -49,16 +50,16 @@ Exit criterion: restart the app and see the same project/session tree.
 ## Phase 1 — model library and on-demand loading
 
 Status: **partially implemented**. The typed preference/download/intake path and
-a DeepSeek Flash Rust/Metal vertical are implemented, including compressed KV
+a DeepSeek Flash Rust/Metal vertical are implemented, including full configured
-through 2K context, DS4 sampling, a single background owner, cancellation, and
+context, ratio-4 sparse indexed attention, prefix continuation, KV checkpoint
-idle unload. Sparse indexed attention, persistent KV/session sync, other model
+save/load, DS4 sampling, cancellation, idle unload, and shared HTTP acquisition.
-families, and optional execution modes remain open.
+Other model families and optional execution modes remain open.
 
 1. **Implemented:** the typed model/runtime preference contract described below
    is complete before chat, the HTTP endpoint, or the engine lifecycle.
-2. **Partially implemented:** mmap-backed loading, tokenizer/prompt rendering,
+2. **Implemented for DeepSeek Flash:** mmap-backed loading, tokenizer/prompt
-   Rust-owned Flash session creation/evaluation/sampling, and model ownership
+   rendering, Rust-owned session creation/evaluation/sampling, prefix sync, and
-   are complete. Prefix sync plus KV save/load remain.
+   KV save/load are complete.
 3. **Implemented:** retain the tested model restrictions from DwarfStar. Main
    and DSpark artifacts reject invalid GGUF versions, catalog mismatches, and
    incompatible metadata, tensor shapes, offsets, or quantization before
@@ -66,8 +67,11 @@ families, and optional execution modes remain open.
 4. **Partially implemented:** reuse the existing `.metal` kernels and preserve mmap-backed resident
    loading plus the explicit SSD-streaming path. Keep Objective-C only at the
    Metal interop boundary.
-5. **Implemented for local chat:** keep one engine resident at most, matching DwarfStar's instance-lock and
+5. Keep exactly one model resident process-wide. Local chat and every endpoint
-   memory assumptions.
+   request must use the same owner and queue; model selection may replace the
+   resident model only after active work reaches a safe boundary. Multiple
+   simultaneously loaded models are out of scope because DS4Server targets
+   models large enough to consume most available memory.
 6. **Implemented for local chat:** move engine work off the UI thread and support cooperative cancellation at
    the safe session boundaries already defined by `ds4_session_sync`.
 
@@ -92,21 +96,25 @@ to Rust while preserving its semantics and on-disk compatibility:
   checkpoints, model/quantization/context/payload-ABI compatibility checks,
   tool-call replay metadata, disk-budget enforcement, hit tracking, and
   eviction behavior.
-- Use one process-wide Rust session registry and KV checkpoint store for both
+- Use one process-wide Rust model owner and KV checkpoint implementation for
-  local chat and the HTTP server. A remote request bound to an application
+  local chat and the HTTP server. Model access is serialized so two callers
-  session resolves to the same logical DS4 session as the GUI and mutations are
+  never mutate the single resident inference graph concurrently.
-  serialized so two callers never write one live timeline concurrently.
+- Application projects and sessions belong exclusively to local chat. HTTP
-  Stateless HTTP requests still participate in the same compatible-prefix
+  requests are transient and must never create projects, sessions, messages,
-  checkpoint pool, so a prefix produced locally can be reused remotely and
+  transcript rows, request-identity mappings, or any other durable chat record.
-  vice versa.
+  The outside client owns conversation identity and resends the history needed
-- Store that shared checkpoint pool under Application Support in
+  for each request, exactly as `ds4_server.c` expects.
-  `kv-cache/`, using the DS4 key/header/payload rules. Session rows hold only an
+- Transient HTTP requests may read and write engine-owned KV cache files for
-  optional reference to their current checkpoint; the large opaque payload is
+  compatible-prefix reuse. Those opaque cache files are the only server-side
-  not copied into separate GUI and HTTP caches.
+  persistence permitted for external conversations and do not establish an
-- Sharing does not merge unrelated conversations or expose one session's
+  application session or recoverable transcript.
-  transcript to another. It shares the DS4 implementation, checkpoint pool, and
+- Store engine-owned checkpoints under Application Support in `kv-cache/`.
-  explicitly bound logical session; each unrelated live conversation keeps its
+  Local chats use their application session ID; HTTP uses content-addressed
-  own mutable session state.
+  entries under `kv-cache/http/`. The large opaque payload is never stored in
+  SQLite, and HTTP cache entries do not create application sessions.
+- Cache sharing does not merge conversations or expose transcript text. It
+  shares only the DS4 implementation and compatible opaque prefix checkpoints;
+  the request body remains the source of truth for external conversation state.
 - Engine unload must first leave every reusable live timeline in a valid
   persisted checkpoint. Reload restores a compatible checkpoint immediately;
   if model identity, quantization, context size, payload ABI, or rendered
@@ -140,6 +148,8 @@ to Rust while preserving its semantics and on-disk compatibility:
 - Let the user enable or disable DSpark for the selected model. Disable the
   control for models that do not support DSpark.
 - Let the user configure the inactivity timeout; default to 10 minutes.
+- Let the user configure the localhost endpoint port; default to 4000 and apply
+  a changed port when preferences are saved.
 - Persist preferences in the application SQLite database. Changing preferences
   never loads a model by itself.
 
@@ -258,43 +268,58 @@ engine lifecycle.
 
 ## Phase 2 — OpenAI-compatible local endpoint
 
-Status: **open; not started**.
+Status: **partially implemented**. The shared single-model runtime,
+verified-model discovery, configurable localhost port, and the first
+`ds4_server.c`-compatible Chat Completions vertical are implemented. The
+remaining protocol routes plus GUI status and enable/disable controls remain
+open.
 
-1. Add a localhost-only HTTP service whose lifecycle is independent of the
+1. **Partially implemented:** add a localhost-only HTTP service whose lifecycle is independent of the
    engine. It can listen while the model is unloaded and acquires the configured
-   model only for inference. Start and stop it from the GUI; never expose a LAN
+   model only for inference. It listens on configurable port 4000 by default;
-   listener without an explicit setting.
+   start/stop controls remain open. Never expose a LAN listener without an
-2. Implement the DwarfStar compatibility surface in this order:
+   explicit setting.
+2. Implement the DwarfStar compatibility surface from `ds4_server.c` in this
+   order, preserving its request parsing, prompt formatting, reasoning fields,
+   streaming events, errors, usage accounting, stop behavior, tool replay, and
+   cancellation semantics rather than inventing an app-specific protocol:
    - `GET /v1/models`
    - `POST /v1/chat/completions`
    - `POST /v1/responses`
    - `POST /v1/completions`
-   `GET /v1/models` reports the configured model without forcing it to load.
+   `GET /v1/models` scans managed artifacts without forcing a model to load and
+   reports only main model files that are fully downloaded and have a matching
+   on-disk verification marker. Missing, partial, unverified, or checksum-invalid
+   artifacts are never advertised.
 3. Support streaming SSE, reasoning output, usage accounting, cancellation,
    sampling parameters, tool schemas, and tool choice.
 4. Port exact sampled tool-call replay and deterministic canonicalization from
    `ds4_server.c` so a client's normalized JSON does not destroy KV-prefix
    reuse.
-5. Route API conversations through the shared session registry and KV store
+5. Route API work through the single process-wide model owner and engine-owned
-   defined in Phase 1. Responses `conversation` bindings and any other
+   KV store defined in Phase 1. External requests are always stateless at the
-   persistent request identity map to the same application session used by the
+   application layer: never write their messages to SQLite and never bind a
-   GUI; stateless requests use shared DS4 prefix lookup. Persist/restore only
+   Responses `conversation`, request identifier, or client-supplied session key
-   through the engine-owned serialization format. Add resident batching only
+   to a local project/session. The client must replay its protocol history;
-   after single-session and cross-surface correctness are established.
+   DS4-compatible prefix lookup may accelerate that replay using opaque KV cache
+   files. Add resident batching only if one-model serialized execution is later
+   proven insufficient.
 6. Display endpoint address, model, active requests, token rates, and errors in
    the GUI.
 
 Exit criterion: the upstream DwarfStar server tests pass against the app for
 non-streaming and streaming Chat Completions and Responses calls, including a
-multi-turn tool call. A compatible prefix created by local chat must register a
+multi-turn tool call. Verified installed models are listed without loading one;
-remote cache hit, a remote checkpoint must be reusable by local chat, and a
+missing/unverified models are absent; local and HTTP generations cannot load two
-GUI/HTTP race on one bound session must serialize without corrupting its token
+models concurrently; external calls leave no project, session, message, or
-or KV frontier.
+transcript rows behind; and compatible KV cache files remain reusable.
 
 ## Phase 3 — durable agent sessions
 
-Status: **partially implemented**. Project/session metadata and structured
+Status: **partially implemented for local chat only**. Project/session metadata,
-transcripts are durable; model binding and KV payload persistence remain open.
+structured transcripts, context statistics, and per-session KV checkpoints are
+durable. Model binding and archive behavior remain open. This phase never
+applies to external endpoint conversations.
 
 Extend each metadata-only session with transcript and shared-checkpoint
 metadata:
@@ -306,14 +331,14 @@ Application Support/DS4Server.rfc1437.de/
     main.gguf
     dspark.gguf       # only when enabled
   kv-cache/
-    <ds4-cache-key>.kv
+    <local-session-id>.bin
+    http/<conversation-tag>.bin
 ```
 
-- **Partially implemented:** migrated message tables persist user text,
+- **Implemented:** migrated message tables persist local user text, reasoning
-  reasoning state, and assistant output while generation streams. Add the
+  state, assistant output, context usage, and generation speed while generation
-  optional current-checkpoint key to SQLite. Keep large engine-owned KV
+  streams. Local session KV payloads remain engine-owned files written with
-  payloads only in the process-wide Phase 1 store, written with atomic
+  atomic replacement; SQLite never stores opaque tensor data.
-  replacement; do not introduce separate agent-only or HTTP-only cache formats.
 - Record model identity, context size, rendered token history, title,
   timestamps, and working directory.
 - Restore compatible KV immediately. If KV is absent or incompatible, rebuild
@@ -365,6 +390,11 @@ ephemeral presentation preferences are the only exception.
 
 ### Native macOS menus (required with chat)
 
+Status: **partially implemented**. Application, Edit, and Window menus are
+installed; Cut, Copy, Paste, and Select All now bridge into the focused Iced
+text field. File/View/Help menus, dynamic enabled state, the remaining Edit
+actions, and selectable transcript text remain open.
+
 The application must install and maintain a complete native macOS menu bar;
 the application-name menu alone is not sufficient. Use platform menu roles,
 ordering, selectors, and standard key equivalents rather than drawing menus
@@ -493,11 +523,13 @@ tests, and release packaging remain open.
 
 1. Finish the Flash session core as one larger slice: port the ratio-4 indexer
    and indexed attention for full configured context, then add prefix sync and
-   compatible KV checkpoint save/load.
+   compatible KV checkpoint save/load. **Implemented.**
 2. Put the OpenAI-compatible endpoint on the same engine lifecycle so local
-   chats and HTTP requests cannot load duplicate engines.
+   chats and HTTP requests cannot load duplicate models. Keep HTTP conversation
-3. Add transcript/KV persistence, then connect the existing conversation UI and
+   state transient except for opaque KV cache files. **Chat Completions vertical
-   finally port the agent tools.
+   implemented; remaining reference routes are open.**
+3. Local transcript/KV persistence and the conversation UI are implemented;
+   port the agent tools after the shared endpoint lifecycle is stable.
 4. Add the opt-in Dev Brain tool after local file/search boundaries and agent
    approvals are stable.
 5. Add bDS2-style A2UI render tools and native inline surfaces after the core

README.md

@@ -4,11 +4,15 @@ DS4Server is a native macOS coding-agent application that rewrites the
 DwarfStar (`ds4`) inference engine in Rust. It uses Rust and Iced and will combine local model loading, an
 OpenAI-compatible localhost endpoint, and project-scoped agent chat in one app.
 
+DS4Server vendors and adapts the Metal kernels and Objective-C Metal glue from
+[DwarfStar (`ds4`)](https://github.com/antirez/ds4). Their copyright and license
+notices are retained in [`native/metal/LICENSE`](native/metal/LICENSE).
+
 The current milestone provides a Codex-inspired project/session layout. A native
 macOS folder picker selects each workspace, then the app asks for its display
 name. Projects, sessions, and model preferences are persisted through Diesel in
 SQLite. Open Preferences with `Command-,` to configure model, generation,
-runtime, and idle-unload settings. The separate Model Manager
+runtime, local endpoint, and idle-unload settings. The separate Model Manager
 (`Shift-Command-M`)
 lists local main and DSpark artifacts, their on-disk sizes and state, and lets
 you download, resume, validate, or delete them. Rust-native background work
@@ -21,9 +25,17 @@ The selected DeepSeek V4 Flash model can run directly from a project session.
 The model, KV/compressor state, 43-layer graph, sampling, and lifecycle are
 owned by Rust; a fixed snapshot of the Objective-C Metal boundary and unchanged
 Metal kernels is vendored and built inside this repository. Tokens stream into the chat UI, Stop cancels generation,
-follow-up turns replay their role history, and the model unloads after the idle
+follow-up turns reuse durable transcript and KV state, and the model unloads
-timeout. The current graph supports 2K context while the sparse long-context
+after the idle timeout. The graph uses the full configured context with the
-indexer and persistent transcript/KV restore remain the next engine slice.
+ratio-4 sparse indexer.
+
+The app also listens on `127.0.0.1:4000` by default for `GET /v1/models` and
+`POST /v1/chat/completions`; the port is configurable in Preferences. The
+endpoint and local chat share the single model owner. External conversations
+are client-managed and never enter the project, session, message, or transcript
+database; only opaque content-addressed KV cache files are retained. Model
+discovery advertises only supported main artifacts that are fully downloaded
+and verified on disk.
 
 ```sh
 cargo install cargo-packager --locked --version 0.11.8
@@ -35,6 +47,7 @@ State is stored at:
 
 ```text
 ~/Library/Application Support/DS4Server.rfc1437.de/data.sqlite3
+~/Library/Application Support/DS4Server.rfc1437.de/kv-cache/
 ```
 
 Deleting a project or session removes only DS4Server metadata. It never deletes

migrations/20260725000000_add_endpoint_port/down.sql

@@ -0,0 +1 @@
+ALTER TABLE preferences DROP COLUMN endpoint_port;

migrations/20260725000000_add_endpoint_port/up.sql

@@ -0,0 +1,2 @@
+ALTER TABLE preferences ADD COLUMN endpoint_port INTEGER NOT NULL DEFAULT 4000
+    CHECK (endpoint_port BETWEEN 1 AND 65535);

src/app.rs

@@ -4,8 +4,10 @@ pub(crate) use view::app_theme;
 
 use crate::database::{AppPreferences, Database, ProjectWithSessions, StoredMessage};
 #[cfg(target_os = "macos")]
-use crate::engine::{ChatTurn, Generator};
+use crate::engine::ChatTurn;
 use crate::model::{self, DownloadOutcome, DownloadProgress, ManagedArtifactId, ModelChoice};
+#[cfg(target_os = "macos")]
+use crate::runtime::{ActiveGeneration, CheckpointTarget, GenerationEvent, GenerationService};
 use crate::settings::{
     DiagnosticPreferences, ExecutionPreferences, GIB, GenerationPreferences, ReasoningMode,
     RuntimePreferences, SpeculativePreferences, SsdPreferences, SteeringPreferences,
@@ -16,9 +18,9 @@ use iced::{Size, Subscription, Task, keyboard, window};
 use rfd::AsyncFileDialog;
 use std::fs;
 use std::path::PathBuf;
-use std::sync::Arc;
 use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
 use std::sync::mpsc::{self, TryRecvError};
+use std::sync::{Arc, RwLock};
 use std::thread;
 use std::time::{Duration, Instant};
 
@@ -29,6 +31,7 @@ struct PreferenceDraft {
     model: ModelChoice,
     dspark_enabled: bool,
     idle_timeout_minutes: String,
+    endpoint_port: String,
     context_tokens: String,
     max_generated_tokens: String,
     system_prompt: String,
@@ -73,6 +76,7 @@ impl PreferenceDraft {
             model,
             dspark_enabled: speculative.dspark_enabled,
             idle_timeout_minutes: preferences.idle_timeout_minutes.to_string(),
+            endpoint_port: preferences.endpoint_port.to_string(),
             context_tokens: generation.context_tokens.to_string(),
             max_generated_tokens: generation.max_generated_tokens.to_string(),
             system_prompt: generation.system_prompt,
@@ -168,6 +172,7 @@ impl PreferenceDraft {
         self.model = ModelChoice::default();
         self.dspark_enabled = false;
         self.idle_timeout_minutes = "10".into();
+        self.endpoint_port = "4000".into();
         self.context_tokens = defaults.context_tokens.to_string();
         self.max_generated_tokens = defaults.max_generated_tokens.to_string();
         self.system_prompt = defaults.system_prompt;
@@ -389,6 +394,8 @@ pub(crate) struct App {
     pub(super) pending_model_delete: Option<ManagedArtifactId>,
     #[cfg(target_os = "macos")]
     _native_menu: Option<crate::native_menu::NativeMenu>,
+    #[cfg(target_os = "macos")]
+    pub(super) native_edit_commands: crate::native_edit::EditCommandQueue,
     database: Option<Database>,
     projects: Vec<ProjectWithSessions>,
     preferences: AppPreferences,
@@ -408,7 +415,13 @@ pub(crate) struct App {
     pub(super) context_limit: u32,
     pub(super) tokens_per_second: Option<f32>,
     #[cfg(target_os = "macos")]
-    generation_worker: Option<GenerationWorker>,
+    generation_service: Option<GenerationService>,
+    #[cfg(target_os = "macos")]
+    active_generation: Option<ActiveGeneration>,
+    #[cfg(target_os = "macos")]
+    runtime_preferences: Arc<RwLock<AppPreferences>>,
+    #[cfg(target_os = "macos")]
+    _endpoint: Option<crate::server::ServerHandle>,
     error: Option<String>,
 }
 
@@ -461,40 +474,6 @@ impl From<StoredMessage> for ChatMessage {
     }
 }
 
-#[cfg(target_os = "macos")]
-struct GenerationWorker {
-    commands: mpsc::Sender<GenerationCommand>,
-    events: mpsc::Receiver<GenerationEvent>,
-    cancel: Option<Arc<AtomicBool>>,
-}
-
-#[cfg(target_os = "macos")]
-enum GenerationCommand {
-    Generate {
-        engine: crate::settings::EngineSettings,
-        turn: crate::settings::TurnSettings,
-        messages: Vec<ChatTurn>,
-        checkpoint: PathBuf,
-        idle_timeout: Duration,
-        cancel: Arc<AtomicBool>,
-    },
-}
-
-#[cfg(target_os = "macos")]
-enum GenerationEvent {
-    Loading,
-    Chunk {
-        reasoning: bool,
-        content: String,
-    },
-    Context {
-        used: u32,
-        limit: u32,
-        tokens_per_second: Option<f32>,
-    },
-    Finished(Result<(), String>),
-}
-
 #[derive(Debug)]
 pub(super) enum ModelDownload {
     Idle,
@@ -526,6 +505,8 @@ pub(super) struct ActiveDownload {
 #[derive(Debug, Clone)]
 pub(crate) enum Message {
     Noop,
+    #[cfg(target_os = "macos")]
+    NativeEdit(crate::native_edit::EditCommand),
     OpenPreferences,
     OpenModelManager,
     ModelManagerOpened(window::Id),
@@ -535,6 +516,7 @@ pub(crate) enum Message {
     PreferenceModelChanged(ModelChoice),
     PreferenceDsparkChanged(bool),
     PreferenceTimeoutChanged(String),
+    PreferenceEndpointPortChanged(String),
     PreferenceContextChanged(String),
     PreferenceMaxTokensChanged(String),
     PreferenceSystemPromptChanged(String),
@@ -602,12 +584,17 @@ impl App {
                         Err(error) => return Self::failed(error, main_window),
                     };
                     let context_limit = preferences.context_tokens.max(0) as u32;
+                    #[cfg(target_os = "macos")]
+                    let (runtime_preferences, generation_service, endpoint, service_error) =
+                        spawn_services(&preferences);
                     Self {
                         main_window,
                         model_manager_window: None,
                         pending_model_delete: None,
                         #[cfg(target_os = "macos")]
                         _native_menu: None,
+                        #[cfg(target_os = "macos")]
+                        native_edit_commands: crate::native_edit::command_queue(),
                         database: Some(database),
                         projects,
                         preferences,
@@ -627,8 +614,23 @@ impl App {
                         context_limit,
                         tokens_per_second: None,
                         #[cfg(target_os = "macos")]
-                        generation_worker: None,
+                        generation_service,
-                        error: None,
+                        #[cfg(target_os = "macos")]
+                        active_generation: None,
+                        #[cfg(target_os = "macos")]
+                        runtime_preferences,
+                        #[cfg(target_os = "macos")]
+                        _endpoint: endpoint,
+                        error: {
+                            #[cfg(target_os = "macos")]
+                            {
+                                service_error
+                            }
+                            #[cfg(not(target_os = "macos"))]
+                            {
+                                None
+                            }
+                        },
                     }
                 }
                 (Err(error), _) | (_, Err(error)) => Self::failed(error, main_window),
@@ -642,12 +644,21 @@ impl App {
         let preference_draft = PreferenceDraft::from_saved(&preferences)
             .expect("default preferences must use a supported model");
         let context_limit = preferences.context_tokens.max(0) as u32;
+        #[cfg(target_os = "macos")]
+        let (runtime_preferences, generation_service, endpoint, service_error) =
+            spawn_services(&preferences);
+        #[cfg(target_os = "macos")]
+        let startup_error = service_error;
+        #[cfg(not(target_os = "macos"))]
+        let startup_error = None::<String>;
         Self {
             main_window,
             model_manager_window: None,
             pending_model_delete: None,
             #[cfg(target_os = "macos")]
             _native_menu: None,
+            #[cfg(target_os = "macos")]
+            native_edit_commands: crate::native_edit::command_queue(),
             database: None,
             projects: Vec::new(),
             preferences,
@@ -667,14 +678,29 @@ impl App {
             context_limit,
             tokens_per_second: None,
             #[cfg(target_os = "macos")]
-            generation_worker: None,
+            generation_service,
-            error: Some(format!("Could not open the project database: {error}")),
+            #[cfg(target_os = "macos")]
+            active_generation: None,
+            #[cfg(target_os = "macos")]
+            runtime_preferences,
+            #[cfg(target_os = "macos")]
+            _endpoint: endpoint,
+            error: Some(match startup_error {
+                Some(service_error) => {
+                    format!("Could not open the project database: {error}. {service_error}")
+                }
+                None => format!("Could not open the project database: {error}"),
+            }),
         }
     }
 
     pub(crate) fn update(&mut self, message: Message) -> Task<Message> {
         match message {
             Message::Noop => {}
+            #[cfg(target_os = "macos")]
+            Message::NativeEdit(command) => {
+                crate::native_edit::queue_command(&self.native_edit_commands, command)
+            }
             Message::OpenPreferences => self.open_preferences(),
             Message::OpenModelManager => return self.open_model_manager(),
             Message::ModelManagerOpened(id) => {
@@ -749,6 +775,10 @@ impl App {
                 self.preference_draft.idle_timeout_minutes = value;
                 self.preference_error = None;
             }
+            Message::PreferenceEndpointPortChanged(value) => {
+                self.preference_draft.endpoint_port = value;
+                self.preference_error = None;
+            }
             Message::PreferenceContextChanged(value) => {
                 self.preference_draft.context_tokens = value;
                 self.preference_error = None;
@@ -953,14 +983,11 @@ impl App {
                 self.start_generation();
                 return scroll_chat_to_end();
             }
-            Message::StopGeneration => {
+            Message::StopGeneration =>
+            {
                 #[cfg(target_os = "macos")]
-                if let Some(cancel) = self
+                if let Some(active) = &self.active_generation {
-                    .generation_worker
+                    active.cancel.store(true, Ordering::Relaxed);
-                    .as_ref()
-                    .and_then(|worker| worker.cancel.as_ref())
-                {
-                    cancel.store(true, Ordering::Relaxed);
                 }
             }
             Message::GenerationTick => {
@@ -1133,6 +1160,9 @@ impl App {
                 Some(crate::native_menu::NativeMenuEvent::ModelManager) => {
                     Message::OpenModelManager
                 }
+                Some(crate::native_menu::NativeMenuEvent::Edit(command)) => {
+                    Message::NativeEdit(command)
+                }
                 None => Message::Noop,
             }
         }));
@@ -1255,6 +1285,14 @@ impl App {
             self.preference_error = Some("Idle timeout must be between 1 and 1440 minutes.".into());
             return;
         }
+        let Ok(endpoint_port) = self.preference_draft.endpoint_port.trim().parse::<u16>() else {
+            self.preference_error = Some("Endpoint port must be a whole number.".into());
+            return;
+        };
+        if endpoint_port == 0 {
+            self.preference_error = Some("Endpoint port must be between 1 and 65535.".into());
+            return;
+        }
         let generation = match self.preference_draft.generation() {
             Ok(generation) => generation,
             Err(error) => {
@@ -1275,12 +1313,50 @@ impl App {
             self.preference_error = Some(error);
             return;
         }
+        #[cfg(target_os = "macos")]
+        let pending_endpoint = if self.preferences.endpoint_port != i32::from(endpoint_port)
+            || self._endpoint.is_none()
+        {
+            let Some(generation) = &self.generation_service else {
+                self.preference_error = Some("The model runtime is unavailable.".into());
+                return;
+            };
+            match crate::server::ServerHandle::spawn(
+                generation.clone(),
+                Arc::clone(&self.runtime_preferences),
+                models_path(),
+                application_support_path().join("kv-cache").join("http"),
+                endpoint_port,
+            ) {
+                Ok(endpoint) => Some(endpoint),
+                Err(error) => {
+                    self.preference_error = Some(error);
+                    return;
+                }
+            }
+        } else {
+            None
+        };
         let Some(database) = &mut self.database else {
             return;
         };
-        match database.update_preferences(model.id(), idle_timeout_minutes, &generation, &runtime) {
+        match database.update_preferences(
+            model.id(),
+            idle_timeout_minutes,
+            i32::from(endpoint_port),
+            &generation,
+            &runtime,
+        ) {
             Ok(preferences) => {
                 self.preferences = preferences;
+                #[cfg(target_os = "macos")]
+                if let Ok(mut runtime_preferences) = self.runtime_preferences.write() {
+                    *runtime_preferences = self.preferences.clone();
+                }
+                #[cfg(target_os = "macos")]
+                if let Some(endpoint) = pending_endpoint {
+                    self._endpoint = Some(endpoint);
+                }
                 self.preference_draft = PreferenceDraft::from_saved(&self.preferences)
                     .expect("the saved model was selected from the supported catalog");
                 self.preferences_open = false;
@@ -1508,15 +1584,10 @@ impl App {
 
         #[cfg(target_os = "macos")]
         {
-            if self.generation_worker.is_none() {
+            let Some(service) = &self.generation_service else {
-                match spawn_generation_worker() {
+                self.error = Some("The model runtime is unavailable.".into());
-                    Ok(worker) => self.generation_worker = Some(worker),
+                return;
-                    Err(error) => {
+            };
-                        self.error = Some(error);
-                        return;
-                    }
-                }
-            }
             let Some(database) = &mut self.database else {
                 return;
             };
@@ -1527,24 +1598,22 @@ impl App {
                     return;
                 }
             };
-            let cancel = Arc::new(AtomicBool::new(false));
             let idle_timeout =
                 Duration::from_secs(self.preferences.idle_timeout_minutes.max(1) as u64 * 60);
-            let command = GenerationCommand::Generate {
+            self.active_generation = match service.generate(
-                engine: effective.engine,
+                effective.engine,
-                turn: effective.turn,
+                effective.turn,
                 messages,
-                checkpoint: session_checkpoint_path(session_id),
+                CheckpointTarget::Local(session_checkpoint_path(session_id)),
                 idle_timeout,
-                cancel: Arc::clone(&cancel),
+            ) {
+                Ok(active) => Some(active),
+                Err(error) => {
+                    self.generation_service = None;
+                    self.error = Some(error);
+                    return;
+                }
             };
-            let worker = self.generation_worker.as_mut().expect("worker was created");
-            if worker.commands.send(command).is_err() {
-                self.generation_worker = None;
-                self.error = Some("The local generation worker stopped unexpectedly.".into());
-                return;
-            }
-            worker.cancel = Some(cancel);
             let user = ChatMessage::from(saved.0);
             let mut assistant = ChatMessage::from(saved.1);
             assistant.reasoning_open = assistant_reasoning;
@@ -1565,7 +1634,7 @@ impl App {
 
     fn poll_generation(&mut self) -> bool {
         #[cfg(target_os = "macos")]
-        let Some(worker) = &mut self.generation_worker else {
+        let Some(active) = &mut self.active_generation else {
             self.generating = false;
             return false;
         };
@@ -1575,7 +1644,7 @@ impl App {
         let mut context_changed = false;
         #[cfg(target_os = "macos")]
         loop {
-            match worker.events.try_recv() {
+            match active.events.try_recv() {
                 Ok(GenerationEvent::Loading) => {}
                 Ok(GenerationEvent::Chunk { reasoning, content }) => {
                     if let Some(message) = self.conversation.last_mut()
@@ -1597,17 +1666,17 @@ impl App {
                 }
                 Ok(GenerationEvent::Finished(result)) => {
                     self.generating = false;
-                    worker.cancel = None;
                     if let Err(error) = result {
                         self.error = Some(error);
                     }
+                    self.active_generation = None;
                     break;
                 }
                 Err(TryRecvError::Empty) => break,
                 Err(TryRecvError::Disconnected) => {
                     self.generating = false;
-                    self.generation_worker = None;
+                    self.active_generation = None;
-                    self.error = Some("The local generation worker stopped unexpectedly.".into());
+                    self.error = Some("The model runtime stopped unexpectedly.".into());
                     break;
                 }
             }
@@ -1627,12 +1696,8 @@ impl App {
                 &message.content,
             )
         {
-            if let Some(cancel) = self
+            if let Some(active) = &self.active_generation {
-                .generation_worker
+                active.cancel.store(true, Ordering::Relaxed);
-                .as_ref()
-                .and_then(|worker| worker.cancel.as_ref())
-            {
-                cancel.store(true, Ordering::Relaxed);
             }
             self.error = Some(format!("Could not save generated chat text: {error}"));
         }
@@ -1667,77 +1732,30 @@ impl App {
 }
 
 #[cfg(target_os = "macos")]
-fn spawn_generation_worker() -> Result<GenerationWorker, String> {
+fn spawn_services(
-    let (command_sender, command_receiver) = mpsc::channel();
+    preferences: &AppPreferences,
-    let (event_sender, event_receiver) = mpsc::channel();
+) -> (
-    thread::Builder::new()
+    Arc<RwLock<AppPreferences>>,
-        .name("local-generation".into())
+    Option<GenerationService>,
-        .spawn(move || {
+    Option<crate::server::ServerHandle>,
-            let mut loaded = None::<(crate::settings::EngineSettings, Generator)>;
+    Option<String>,
-            let mut last_used = Instant::now();
+) {
-            let mut idle_timeout = Duration::from_secs(15 * 60);
+    let runtime_preferences = Arc::new(RwLock::new(preferences.clone()));
-            loop {
+    let generation = match GenerationService::spawn() {
-                match command_receiver.recv_timeout(Duration::from_secs(1)) {
+        Ok(generation) => generation,
-                    Ok(GenerationCommand::Generate {
+        Err(error) => return (runtime_preferences, None, None, Some(error)),
-                        engine,
+    };
-                        turn,
+    let endpoint = crate::server::ServerHandle::spawn(
-                        messages,
+        generation.clone(),
-                        checkpoint,
+        Arc::clone(&runtime_preferences),
-                        idle_timeout: requested_timeout,
+        models_path(),
-                        cancel,
+        application_support_path().join("kv-cache").join("http"),
-                    }) => {
+        u16::try_from(preferences.endpoint_port).unwrap_or(4000),
-                        idle_timeout = requested_timeout;
+    );
-                        if loaded
+    match endpoint {
-                            .as_ref()
+        Ok(endpoint) => (runtime_preferences, Some(generation), Some(endpoint), None),
-                            .is_none_or(|(current, _)| current != &engine)
+        Err(error) => (runtime_preferences, Some(generation), None, Some(error)),
-                        {
+    }
-                            let _ = event_sender.send(GenerationEvent::Loading);
-                            loaded = match Generator::open(&engine) {
-                                Ok(generator) => Some((engine.clone(), generator)),
-                                Err(error) => {
-                                    let _ =
-                                        event_sender.send(GenerationEvent::Finished(Err(error)));
-                                    None
-                                }
-                            };
-                        }
-                        if let Some((_, generator)) = &mut loaded {
-                            let result = generator.generate(
-                                &checkpoint,
-                                &messages,
-                                &turn,
-                                &cancel,
-                                |reasoning, content| {
-                                    let _ = event_sender
-                                        .send(GenerationEvent::Chunk { reasoning, content });
-                                },
-                                |used, limit, tokens_per_second| {
-                                    let _ = event_sender.send(GenerationEvent::Context {
-                                        used,
-                                        limit,
-                                        tokens_per_second,
-                                    });
-                                },
-                            );
-                            let _ = event_sender.send(GenerationEvent::Finished(result));
-                            last_used = Instant::now();
-                        }
-                    }
-                    Err(mpsc::RecvTimeoutError::Timeout) => {
-                        if loaded.is_some() && last_used.elapsed() >= idle_timeout {
-                            loaded = None;
-                        }
-                    }
-                    Err(mpsc::RecvTimeoutError::Disconnected) => break,
-                }
-            }
-        })
-        .map_err(|error| format!("Could not start local generation: {error}"))?;
-    Ok(GenerationWorker {
-        commands: command_sender,
-        events: event_receiver,
-        cancel: None,
-    })
 }
 
 impl Drop for App {
@@ -1746,12 +1764,8 @@ impl Drop for App {
             download.cancel.store(true, Ordering::Relaxed);
         }
         #[cfg(target_os = "macos")]
-        if let Some(cancel) = self
+        if let Some(active) = &self.active_generation {
-            .generation_worker
+            active.cancel.store(true, Ordering::Relaxed);
-            .as_ref()
-            .and_then(|worker| worker.cancel.as_ref())
-        {
-            cancel.store(true, Ordering::Relaxed);
         }
     }
 }

src/app/view.rs

@@ -31,7 +31,12 @@ impl App {
         if self.model_manager_window == Some(id) {
             self.model_manager()
         } else {
-            self.main_view()
+            let content = self.main_view();
+            #[cfg(target_os = "macos")]
+            return crate::native_edit::native_edit(content, self.native_edit_commands.clone())
+                .into();
+            #[cfg(not(target_os = "macos"))]
+            content
         }
     }
 
@@ -457,286 +462,331 @@ impl App {
                 dspark_confidence.on_input(Message::PreferenceDsparkConfidenceChanged);
         }
 
-        let mut fields = column![
+        let model_group = preference_group(
-            text("MODEL").size(11),
+            "MODEL & LIFECYCLE",
-            pick_list(
+            column![
-                &MODEL_CHOICES[..],
-                Some(self.preference_draft.model),
-                Message::PreferenceModelChanged,
-            )
-            .width(Length::Fill),
-            text(format!(
-                "Main: {}{}",
-                engine.map_or_else(
-                    || "Invalid settings".to_owned(),
-                    |engine| engine.artifacts.model.display().to_string(),
-                ),
-                engine
-                    .and_then(|engine| engine.artifacts.mtp.as_ref())
-                    .map_or_else(String::new, |path| format!(" • support: {}", path.display())),
-            ))
-            .size(12),
-            Space::with_height(8),
-            text("EXECUTION").size(11),
-            preference_input_row(
-                "CPU helper threads",
-                text_input("Automatic", &self.preference_draft.cpu_threads)
-                    .on_input(Message::PreferenceCpuThreadsChanged),
-            ),
-            preference_input_row("GPU power percent", power),
-            preference_input_row("Prefill chunk", prefill),
-            checkbox("Prefer exact quality kernels", self.preference_draft.quality)
-                .on_toggle(Message::PreferenceQualityChanged),
-            checkbox("Warm mapped weights at load time", self.preference_draft.warm_weights)
-                .on_toggle(Message::PreferenceWarmWeightsChanged),
-            text(if self.preference_draft.model == ModelChoice::Glm52 {
-                "GLM 5.2 uses full GPU power and selects prefill chunks automatically."
-            } else {
-                "Blank numeric values preserve DS4's automatic engine behavior."
-            })
-            .size(12),
-            text(engine.as_ref().map_or_else(
-                || "Effective execution settings will appear after valid values are entered."
-                    .to_owned(),
-                |engine| {
-                    let settings = engine.execution;
-                    format!(
-                    "Metal engine: threads {} • power {}% • prefill {} • quality {} • warm weights {}",
-                    if settings.cpu_threads == 0 { "auto".to_owned() } else { settings.cpu_threads.to_string() },
-                    if settings.power_percent == 0 { 100 } else { settings.power_percent },
-                    if settings.prefill_chunk == 0 { "auto".to_owned() } else { settings.prefill_chunk.to_string() },
-                    if settings.quality { "on" } else { "off" },
-                    if settings.warm_weights { "on" } else { "off" },
-                )},
-            ))
-            .size(12),
-            Space::with_height(8),
-            text("SPECULATIVE DECODING").size(11),
-            preference_input_row(
-                "MTP draft tokens",
-                text_input("1", &self.preference_draft.mtp_draft_tokens)
-                    .on_input(Message::PreferenceMtpDraftChanged),
-            ),
-            preference_input_row(
-                "MTP verifier margin",
-                text_input("3", &self.preference_draft.mtp_margin)
-                    .on_input(Message::PreferenceMtpMarginChanged),
-            ),
-            checkbox("Enable integrated GLM MTP", self.preference_draft.glm_mtp)
-                .on_toggle_maybe(glm_mtp_toggle),
-            checkbox(
-                "Log GLM MTP timing counters",
-                self.preference_draft.glm_mtp_timing,
-            )
-            .on_toggle_maybe(glm_mtp_timing_toggle),
-            dspark,
-            preference_input_row("DSpark confidence threshold", dspark_confidence),
-            checkbox(
-                "DSpark target-only decode",
-                self.preference_draft.dspark_strict,
-            )
-            .on_toggle_maybe(dspark_strict_toggle),
-            text(if self.preference_draft.model.supports_dspark() {
-                "DSpark uses the managed support artifact; entering a threshold or enabling strict mode also enables DSpark."
-            } else if self.preference_draft.model == ModelChoice::Glm52 {
-                "GLM MTP is integrated; DSpark is unavailable for this model."
-            } else {
-                "No managed MTP support artifact is available for this model."
-            })
-            .size(12),
-            text(engine.as_ref().map_or_else(
-                || "Effective speculative settings will appear after valid values are entered."
-                    .to_owned(),
-                |engine| {
-                    let settings = engine.speculative;
-                    format!(
-                    "Engine: MTP draft {} • margin {} • GLM MTP {} • timing {} • DSpark {} • confidence {}{} • target-only {}",
-                    settings.mtp_draft_tokens,
-                    settings.mtp_margin,
-                    if settings.glm_mtp { "on" } else { "off" },
-                    if settings.glm_mtp_timing { "on" } else { "off" },
-                    if settings.dspark { "on" } else { "off" },
-                    settings.dspark_confidence_threshold,
-                    if settings.dspark_confidence_threshold_set { " explicit" } else { " default" },
-                    if settings.dspark_strict { "on" } else { "off" },
-                )},
-            ))
-            .size(12),
-            Space::with_height(8),
-            text("SSD STREAMING").size(11),
-            checkbox("Enable SSD-backed model streaming", self.preference_draft.ssd_streaming)
-                .on_toggle(Message::PreferenceSsdChanged),
-            checkbox("Skip automatic expert preload", self.preference_draft.ssd_streaming_cold)
-                .on_toggle(Message::PreferenceSsdColdChanged),
-            preference_input_row(
-                "Expert cache count or GiB",
-                text_input("Automatic, 128, or 64GB", &self.preference_draft.ssd_cache)
-                    .on_input(Message::PreferenceSsdCacheChanged),
-            ),
-            preference_input_row("Fully resident GLM layers", ssd_full_layers),
-            preference_input_row(
-                "Explicit expert preload count",
-                text_input("Automatic", &self.preference_draft.ssd_preload_experts)
-                    .on_input(Message::PreferenceSsdPreloadChanged),
-            ),
-            text("A blank full-layer value is automatic; an explicit 0 disables fully resident GLM layers. SSD streaming and DSpark are mutually exclusive.")
-                .size(12),
-            text(engine.as_ref().map_or_else(
-                || "Effective SSD settings will appear after valid values are entered."
-                    .to_owned(),
-                |engine| {
-                    let settings = engine.ssd;
-                    let cache = if settings.cache_bytes > 0 {
-                        format!("{} GiB", settings.cache_bytes / GIB)
-                    } else if settings.cache_experts > 0 {
-                        format!("{} experts", settings.cache_experts)
-                    } else {
-                        "auto".to_owned()
-                    };
-                    format!(
-                        "Engine: streaming {} • cold {} • cache {} • full layers {}{} • preload {}",
-                        if settings.enabled { "on" } else { "off" },
-                        if settings.cold { "on" } else { "off" },
-                        cache,
-                        settings.full_layers,
-                        if settings.full_layers_set { " explicit" } else { " auto" },
-                        if settings.preload_experts == 0 { "auto".to_owned() } else { settings.preload_experts.to_string() },
-                    )
-                },
-            ))
-            .size(12),
-            Space::with_height(8),
-            text("DIRECTIONAL STEERING").size(11),
-            text("Direction-vector file").size(13),
-            steering_file.padding(9),
-            preference_input_row("FFN scale", steering_ffn),
-            preference_input_row("Attention scale", steering_attn),
-            text(if self.preference_draft.model == ModelChoice::Glm52 {
-                "Directional steering is not supported for GLM 5.2."
-            } else {
-                "With a file and no explicit scale, DS4 defaults the FFN scale to 1. Scales accept -100 through 100."
-            })
-            .size(12),
-            text(engine.as_ref().map_or_else(
-                || "Effective steering settings will appear after valid values are entered."
-                    .to_owned(),
-                |engine| format!(
-                    "Engine: file {} • FFN scale {} • attention scale {}",
-                    if engine.steering.file.is_some() { "set" } else { "off" },
-                    engine.steering.ffn_scale,
-                    engine.steering.attention_scale,
-                ),
-            ))
-            .size(12),
-            Space::with_height(8),
-            text("CAPACITY").size(11),
-            preference_input_row(
-                "Context tokens",
-                text_input("32768", &self.preference_draft.context_tokens)
-                    .on_input(Message::PreferenceContextChanged),
-            ),
-            preference_input_row(
-                "Maximum generated tokens",
-                text_input("50000", &self.preference_draft.max_generated_tokens)
-                    .on_input(Message::PreferenceMaxTokensChanged),
-            ),
-            text("System prompt").size(13),
-            text_input(
-                "You are a helpful assistant",
-                &self.preference_draft.system_prompt,
-            )
-            .on_input(Message::PreferenceSystemPromptChanged)
-            .padding(9),
-            Space::with_height(8),
-            text("SAMPLING AND REASONING").size(11),
-            preference_input_row(
-                "Temperature",
-                text_input("DS4 default", &self.preference_draft.temperature)
-                    .on_input(Message::PreferenceTemperatureChanged),
-            ),
-            preference_input_row(
-                "Top-p",
-                text_input("DS4 default", &self.preference_draft.top_p)
-                    .on_input(Message::PreferenceTopPChanged),
-            ),
-            preference_input_row(
-                "Min-p",
-                text_input("DS4 default", &self.preference_draft.min_p)
-                    .on_input(Message::PreferenceMinPChanged),
-            ),
-            preference_input_row(
-                "Seed",
-                text_input("Random", &self.preference_draft.seed)
-                    .on_input(Message::PreferenceSeedChanged),
-            ),
-            row![
-                text("Reasoning").size(13).width(Length::Fill),
                 pick_list(
-                    &REASONING_MODES[..],
+                    &MODEL_CHOICES[..],
-                    Some(self.preference_draft.reasoning_mode),
+                    Some(self.preference_draft.model),
-                    Message::PreferenceReasoningChanged,
+                    Message::PreferenceModelChanged,
                 )
-                .width(240),
+                .width(Length::Fill),
-            ]
+                text(format!(
-            .spacing(12)
+                    "Main: {}{}",
-            .align_y(Alignment::Center),
+                    engine.map_or_else(
-            text("Blank sampling values retain DS4's model-family defaults. Think Max needs at least 393216 context tokens.")
+                        || "Invalid settings".to_owned(),
+                        |engine| engine.artifacts.model.display().to_string(),
+                    ),
+                    engine
+                        .and_then(|engine| engine.artifacts.mtp.as_ref())
+                        .map_or_else(String::new, |path| format!(
+                            " • support: {}",
+                            path.display()
+                        )),
+                ))
                 .size(12),
-            text(turn.map_or_else(
+                row![
-                || "Effective settings will appear after valid values are entered.".to_owned(),
+                    text_input("10", &self.preference_draft.idle_timeout_minutes)
-                |settings| format!(
+                        .on_input(Message::PreferenceTimeoutChanged)
-                    "Effective: {} context • {} max • temp {} • top-p {} • min-p {} • seed {} • {} • system prompt {}",
+                        .width(90)
-                    settings.context_tokens,
+                        .padding(9),
-                    settings.max_generated_tokens,
+                    text("minutes before unloading the model").size(13),
-                    settings.temperature,
+                ]
-                    settings.top_p,
+                .spacing(10)
-                    settings.min_p,
+                .align_y(Alignment::Center),
-                    settings.seed.map_or_else(|| "random".to_owned(), |seed| seed.to_string()),
+                text("Enter a whole number from 1 to 1440.").size(12),
-                    settings.reasoning_mode,
-                    if settings.system_prompt.is_empty() { "off" } else { "on" },
-                ),
-            ))
-            .size(12),
-            Space::with_height(8),
-            text("ADVANCED DIAGNOSTICS").size(11),
-            preference_input_row(
-                "Simulated used memory (GiB)",
-                text_input("Disabled", &self.preference_draft.simulated_used_memory_gib)
-                    .on_input(Message::PreferenceSimulatedMemoryChanged),
-            ),
-            text("Routed expert profile output").size(13),
-            text_input("Output file path", &self.preference_draft.expert_profile_path)
-                .on_input(Message::PreferenceExpertProfileChanged)
-                .padding(9),
-            text(engine.as_ref().map_or_else(
-                || "Effective diagnostic settings will appear after valid values are entered."
-                    .to_owned(),
-                |engine| format!(
-                    "{} load: simulated memory {} • expert profile {}",
-                    engine.model,
-                    if engine.diagnostics.simulated_used_memory_bytes == 0 {
-                        "off".to_owned()
-                    } else {
-                        format!("{} GiB", engine.diagnostics.simulated_used_memory_bytes / GIB)
-                    },
-                    if engine.diagnostics.expert_profile_path.is_some() { "set" } else { "off" },
-                ),
-            ))
-            .size(12),
-            Space::with_height(8),
-            text("INACTIVITY").size(11),
-            row![
-                text_input("10", &self.preference_draft.idle_timeout_minutes)
-                    .on_input(Message::PreferenceTimeoutChanged)
-                    .width(90)
-                    .padding(9),
-                text("minutes before unloading the model").size(13),
             ]
-            .spacing(10)
+            .spacing(10),
-            .align_y(Alignment::Center),
+        );
-            text("Enter a whole number from 1 to 1440.").size(12),
+        let endpoint_group = preference_group(
+            "LOCAL ENDPOINT",
+            column![
+                preference_input_row(
+                    "Port",
+                    text_input("4000", &self.preference_draft.endpoint_port)
+                        .on_input(Message::PreferenceEndpointPortChanged),
+                ),
+                text("Listens on 127.0.0.1. Saving a changed port restarts the local endpoint.")
+                    .size(12),
+            ]
+            .spacing(10),
+        );
+        let generation_group = preference_group(
+            "GENERATION",
+            column![
+                preference_input_row(
+                    "Context tokens",
+                    text_input("32768", &self.preference_draft.context_tokens)
+                        .on_input(Message::PreferenceContextChanged),
+                ),
+                preference_input_row(
+                    "Maximum generated tokens",
+                    text_input("50000", &self.preference_draft.max_generated_tokens)
+                        .on_input(Message::PreferenceMaxTokensChanged),
+                ),
+                text("System prompt").size(13),
+                text_input(
+                    "You are a helpful assistant",
+                    &self.preference_draft.system_prompt,
+                )
+                .on_input(Message::PreferenceSystemPromptChanged)
+                .padding(9),
+                Space::with_height(4),
+                text("SAMPLING & REASONING").size(11).color(muted_text()),
+                preference_input_row(
+                    "Temperature",
+                    text_input("DS4 default", &self.preference_draft.temperature)
+                        .on_input(Message::PreferenceTemperatureChanged),
+                ),
+                preference_input_row(
+                    "Top-p",
+                    text_input("DS4 default", &self.preference_draft.top_p)
+                        .on_input(Message::PreferenceTopPChanged),
+                ),
+                preference_input_row(
+                    "Min-p",
+                    text_input("DS4 default", &self.preference_draft.min_p)
+                        .on_input(Message::PreferenceMinPChanged),
+                ),
+                preference_input_row(
+                    "Seed",
+                    text_input("Random", &self.preference_draft.seed)
+                        .on_input(Message::PreferenceSeedChanged),
+                ),
+                row![
+                    text("Reasoning").size(13).width(Length::Fill),
+                    pick_list(
+                        &REASONING_MODES[..],
+                        Some(self.preference_draft.reasoning_mode),
+                        Message::PreferenceReasoningChanged,
+                    )
+                    .width(240),
+                ]
+                .spacing(12)
+                .align_y(Alignment::Center),
+                text("Blank sampling values retain DS4's model-family defaults. Think Max needs at least 393216 context tokens.")
+                    .size(12),
+                text(turn.map_or_else(
+                    || "Effective settings will appear after valid values are entered.".to_owned(),
+                    |settings| format!(
+                        "Effective: {} context • {} max • temp {} • top-p {} • min-p {} • seed {} • {} • system prompt {}",
+                        settings.context_tokens,
+                        settings.max_generated_tokens,
+                        settings.temperature,
+                        settings.top_p,
+                        settings.min_p,
+                        settings.seed.map_or_else(|| "random".to_owned(), |seed| seed.to_string()),
+                        settings.reasoning_mode,
+                        if settings.system_prompt.is_empty() { "off" } else { "on" },
+                    ),
+                ))
+                .size(12),
+            ]
+            .spacing(10),
+        );
+        let execution_group = preference_group(
+            "EXECUTION",
+            column![
+                preference_input_row(
+                    "CPU helper threads",
+                    text_input("Automatic", &self.preference_draft.cpu_threads)
+                        .on_input(Message::PreferenceCpuThreadsChanged),
+                ),
+                preference_input_row("GPU power percent", power),
+                preference_input_row("Prefill chunk", prefill),
+                checkbox("Prefer exact quality kernels", self.preference_draft.quality)
+                    .on_toggle(Message::PreferenceQualityChanged),
+                checkbox("Warm mapped weights at load time", self.preference_draft.warm_weights)
+                    .on_toggle(Message::PreferenceWarmWeightsChanged),
+                text(if self.preference_draft.model == ModelChoice::Glm52 {
+                    "GLM 5.2 uses full GPU power and selects prefill chunks automatically."
+                } else {
+                    "Blank numeric values preserve DS4's automatic engine behavior."
+                })
+                .size(12),
+                text(engine.as_ref().map_or_else(
+                    || "Effective execution settings will appear after valid values are entered."
+                        .to_owned(),
+                    |engine| {
+                        let settings = engine.execution;
+                        format!(
+                            "Metal engine: threads {} • power {}% • prefill {} • quality {} • warm weights {}",
+                            if settings.cpu_threads == 0 { "auto".to_owned() } else { settings.cpu_threads.to_string() },
+                            if settings.power_percent == 0 { 100 } else { settings.power_percent },
+                            if settings.prefill_chunk == 0 { "auto".to_owned() } else { settings.prefill_chunk.to_string() },
+                            if settings.quality { "on" } else { "off" },
+                            if settings.warm_weights { "on" } else { "off" },
+                        )
+                    },
+                ))
+                .size(12),
+            ]
+            .spacing(10),
+        );
+        let acceleration_group = preference_group(
+            "ACCELERATION & MEMORY",
+            column![
+                text("SPECULATIVE DECODING").size(11).color(muted_text()),
+                preference_input_row(
+                    "MTP draft tokens",
+                    text_input("1", &self.preference_draft.mtp_draft_tokens)
+                        .on_input(Message::PreferenceMtpDraftChanged),
+                ),
+                preference_input_row(
+                    "MTP verifier margin",
+                    text_input("3", &self.preference_draft.mtp_margin)
+                        .on_input(Message::PreferenceMtpMarginChanged),
+                ),
+                checkbox("Enable integrated GLM MTP", self.preference_draft.glm_mtp)
+                    .on_toggle_maybe(glm_mtp_toggle),
+                checkbox(
+                    "Log GLM MTP timing counters",
+                    self.preference_draft.glm_mtp_timing,
+                )
+                .on_toggle_maybe(glm_mtp_timing_toggle),
+                dspark,
+                preference_input_row("DSpark confidence threshold", dspark_confidence),
+                checkbox(
+                    "DSpark target-only decode",
+                    self.preference_draft.dspark_strict,
+                )
+                .on_toggle_maybe(dspark_strict_toggle),
+                text(if self.preference_draft.model.supports_dspark() {
+                    "DSpark uses the managed support artifact; entering a threshold or enabling strict mode also enables DSpark."
+                } else if self.preference_draft.model == ModelChoice::Glm52 {
+                    "GLM MTP is integrated; DSpark is unavailable for this model."
+                } else {
+                    "No managed MTP support artifact is available for this model."
+                })
+                .size(12),
+                text(engine.as_ref().map_or_else(
+                    || "Effective speculative settings will appear after valid values are entered."
+                        .to_owned(),
+                    |engine| {
+                        let settings = engine.speculative;
+                        format!(
+                            "Engine: MTP draft {} • margin {} • GLM MTP {} • timing {} • DSpark {} • confidence {}{} • target-only {}",
+                            settings.mtp_draft_tokens,
+                            settings.mtp_margin,
+                            if settings.glm_mtp { "on" } else { "off" },
+                            if settings.glm_mtp_timing { "on" } else { "off" },
+                            if settings.dspark { "on" } else { "off" },
+                            settings.dspark_confidence_threshold,
+                            if settings.dspark_confidence_threshold_set { " explicit" } else { " default" },
+                            if settings.dspark_strict { "on" } else { "off" },
+                        )
+                    },
+                ))
+                .size(12),
+                Space::with_height(6),
+                text("SSD STREAMING").size(11).color(muted_text()),
+                checkbox("Enable SSD-backed model streaming", self.preference_draft.ssd_streaming)
+                    .on_toggle(Message::PreferenceSsdChanged),
+                checkbox("Skip automatic expert preload", self.preference_draft.ssd_streaming_cold)
+                    .on_toggle(Message::PreferenceSsdColdChanged),
+                preference_input_row(
+                    "Expert cache count or GiB",
+                    text_input("Automatic, 128, or 64GB", &self.preference_draft.ssd_cache)
+                        .on_input(Message::PreferenceSsdCacheChanged),
+                ),
+                preference_input_row("Fully resident GLM layers", ssd_full_layers),
+                preference_input_row(
+                    "Explicit expert preload count",
+                    text_input("Automatic", &self.preference_draft.ssd_preload_experts)
+                        .on_input(Message::PreferenceSsdPreloadChanged),
+                ),
+                text("A blank full-layer value is automatic; an explicit 0 disables fully resident GLM layers. SSD streaming and DSpark are mutually exclusive.")
+                    .size(12),
+                text(engine.as_ref().map_or_else(
+                    || "Effective SSD settings will appear after valid values are entered."
+                        .to_owned(),
+                    |engine| {
+                        let settings = engine.ssd;
+                        let cache = if settings.cache_bytes > 0 {
+                            format!("{} GiB", settings.cache_bytes / GIB)
+                        } else if settings.cache_experts > 0 {
+                            format!("{} experts", settings.cache_experts)
+                        } else {
+                            "auto".to_owned()
+                        };
+                        format!(
+                            "Engine: streaming {} • cold {} • cache {} • full layers {}{} • preload {}",
+                            if settings.enabled { "on" } else { "off" },
+                            if settings.cold { "on" } else { "off" },
+                            cache,
+                            settings.full_layers,
+                            if settings.full_layers_set { " explicit" } else { " auto" },
+                            if settings.preload_experts == 0 { "auto".to_owned() } else { settings.preload_experts.to_string() },
+                        )
+                    },
+                ))
+                .size(12),
+            ]
+            .spacing(10),
+        );
+        let steering_group = preference_group(
+            "STEERING & DIAGNOSTICS",
+            column![
+                text("DIRECTIONAL STEERING").size(11).color(muted_text()),
+                text("Direction-vector file").size(13),
+                steering_file.padding(9),
+                preference_input_row("FFN scale", steering_ffn),
+                preference_input_row("Attention scale", steering_attn),
+                text(if self.preference_draft.model == ModelChoice::Glm52 {
+                    "Directional steering is not supported for GLM 5.2."
+                } else {
+                    "With a file and no explicit scale, DS4 defaults the FFN scale to 1. Scales accept -100 through 100."
+                })
+                .size(12),
+                text(engine.as_ref().map_or_else(
+                    || "Effective steering settings will appear after valid values are entered."
+                        .to_owned(),
+                    |engine| format!(
+                        "Engine: file {} • FFN scale {} • attention scale {}",
+                        if engine.steering.file.is_some() { "set" } else { "off" },
+                        engine.steering.ffn_scale,
+                        engine.steering.attention_scale,
+                    ),
+                ))
+                .size(12),
+                Space::with_height(6),
+                text("ADVANCED DIAGNOSTICS").size(11).color(muted_text()),
+                preference_input_row(
+                    "Simulated used memory (GiB)",
+                    text_input("Disabled", &self.preference_draft.simulated_used_memory_gib)
+                        .on_input(Message::PreferenceSimulatedMemoryChanged),
+                ),
+                text("Routed expert profile output").size(13),
+                text_input("Output file path", &self.preference_draft.expert_profile_path)
+                    .on_input(Message::PreferenceExpertProfileChanged)
+                    .padding(9),
+                text(engine.as_ref().map_or_else(
+                    || "Effective diagnostic settings will appear after valid values are entered."
+                        .to_owned(),
+                    |engine| format!(
+                        "{} load: simulated memory {} • expert profile {}",
+                        engine.model,
+                        if engine.diagnostics.simulated_used_memory_bytes == 0 {
+                            "off".to_owned()
+                        } else {
+                            format!("{} GiB", engine.diagnostics.simulated_used_memory_bytes / GIB)
+                        },
+                        if engine.diagnostics.expert_profile_path.is_some() { "set" } else { "off" },
+                    ),
+                ))
+                .size(12),
+            ]
+            .spacing(10),
+        );
+        let mut fields = column![
+            model_group,
+            endpoint_group,
+            generation_group,
+            execution_group,
+            acceleration_group,
+            steering_group,
         ]
-        .spacing(10);
+        .spacing(12);
 
         if let Some(error) = &self.preference_error {
             fields = fields.push(text(error).style(iced::widget::text::danger));
@@ -971,6 +1021,17 @@ fn preference_input_row<'a>(
     .into()
 }
 
+fn preference_group<'a>(
+    title: &'a str,
+    content: impl Into<Element<'a, Message>>,
+) -> Element<'a, Message> {
+    container(column![text(title).size(11).color(muted_text()), content.into(),].spacing(10))
+        .width(Length::Fill)
+        .padding(14)
+        .style(preference_group_style)
+        .into()
+}
+
 fn download_status_bar(download: &ActiveDownload) -> Element<'_, Message> {
     let progress = &download.progress;
     let percent = progress.fraction() * 100.0;
@@ -1201,6 +1262,18 @@ fn overview_style(_: &Theme) -> container::Style {
     }
 }
 
+fn preference_group_style(_: &Theme) -> container::Style {
+    container::Style {
+        background: Some(Background::Color(Color::from_rgb8(38, 38, 40))),
+        border: Border {
+            color: Color::from_rgb8(58, 58, 61),
+            width: 1.0,
+            radius: 14.0.into(),
+        },
+        ..container::Style::default()
+    }
+}
+
 fn chat_message_style(theme: &Theme, user: bool) -> container::Style {
     if !user {
         return overview_style(theme);
@@ -1247,6 +1320,10 @@ mod tests {
             chat_message_style(&theme, true).background,
             chat_message_style(&theme, false).background
         );
+        assert_eq!(
+            preference_group_style(&theme).background,
+            Some(Background::Color(Color::from_rgb8(38, 38, 40)))
+        );
     }
 
     #[test]

src/database.rs

@@ -51,6 +51,7 @@ pub struct AppPreferences {
     pub directional_steering_attn: Option<f32>,
     pub simulated_used_memory_gib: Option<i64>,
     pub expert_profile_path: Option<String>,
+    pub endpoint_port: i32,
 }
 
 impl Default for AppPreferences {
@@ -90,6 +91,7 @@ impl Default for AppPreferences {
             directional_steering_attn: None,
             simulated_used_memory_gib: None,
             expert_profile_path: None,
+            endpoint_port: 4000,
         }
     }
 }
@@ -234,6 +236,7 @@ struct PreferenceChanges<'a> {
     directional_steering_attn: Option<f32>,
     simulated_used_memory_gib: Option<i64>,
     expert_profile_path: Option<&'a str>,
+    endpoint_port: i32,
 }
 
 #[derive(Clone, Debug, Identifiable, Queryable, Selectable)]
@@ -360,6 +363,7 @@ impl Database {
         &mut self,
         selected_model: &str,
         idle_timeout_minutes: i32,
+        endpoint_port: i32,
         generation: &GenerationPreferences,
         runtime: &RuntimePreferences,
     ) -> Result<AppPreferences, String> {
@@ -367,6 +371,9 @@ impl Database {
         let model = crate::model::ModelChoice::from_id(selected_model)
             .ok_or_else(|| format!("Unsupported model: {selected_model}"))?;
         runtime.validate(model)?;
+        if !(1..=65_535).contains(&endpoint_port) {
+            return Err("Endpoint port must be between 1 and 65535.".into());
+        }
         let execution = &runtime.execution;
         let speculative = &runtime.speculative;
         let (ssd_cache_experts, ssd_cache_gib) = match runtime.ssd.cache {
@@ -416,6 +423,7 @@ impl Database {
                     .simulated_used_memory_gib
                     .map(|value| value as i64),
                 expert_profile_path: runtime.diagnostics.expert_profile_path.as_deref(),
+                endpoint_port,
             })
             .returning(AppPreferences::as_returning())
             .get_result(&mut self.connection)
@@ -572,6 +580,7 @@ mod tests {
         assert_eq!(preferences.selected_model, "deepseek-v4-flash");
         assert!(!preferences.dspark_enabled);
         assert_eq!(preferences.idle_timeout_minutes, 10);
+        assert_eq!(preferences.endpoint_port, 4000);
         let generation = GenerationPreferences::default();
         let runtime = RuntimePreferences::default();
         assert!(
@@ -579,6 +588,7 @@ mod tests {
                 .update_preferences(
                     "glm-5.2",
                     30,
+                    4000,
                     &generation,
                     &RuntimePreferences {
                         speculative: SpeculativePreferences {
@@ -592,7 +602,7 @@ mod tests {
         );
         assert!(
             database
-                .update_preferences("deepseek-v4-flash", 0, &generation, &runtime,)
+                .update_preferences("deepseek-v4-flash", 0, 4000, &generation, &runtime,)
                 .is_err()
         );
         let generation = GenerationPreferences {
@@ -630,7 +640,7 @@ mod tests {
             ..RuntimePreferences::default()
         };
         database
-            .update_preferences("glm-5.2", 30, &generation, &runtime)
+            .update_preferences("glm-5.2", 30, 4567, &generation, &runtime)
             .unwrap();
 
         let project = database.create_project("DS4", "/tmp/ds4").unwrap();
@@ -652,6 +662,7 @@ mod tests {
         let preferences = reopened.load_preferences().unwrap();
         assert_eq!(preferences.selected_model, "glm-5.2");
         assert_eq!(preferences.idle_timeout_minutes, 30);
+        assert_eq!(preferences.endpoint_port, 4567);
         assert_eq!(preferences.generation().unwrap(), generation);
         assert_eq!(preferences.runtime().unwrap(), runtime);
         drop(reopened);

src/engine.rs

@@ -305,6 +305,14 @@ pub(crate) struct ChatTurn {
     pub(crate) content: String,
 }
 
+pub(crate) struct GenerationOutput {
+    pub(crate) message: ChatTurn,
+    pub(crate) prompt_tokens: u32,
+    pub(crate) cached_tokens: u32,
+    pub(crate) completion_tokens: u32,
+    pub(crate) finish_reason: &'static str,
+}
+
 #[cfg(target_os = "macos")]
 impl Generator {
     pub(crate) fn open(settings: &EngineSettings) -> Result<Self, String> {
@@ -337,12 +345,12 @@ impl Generator {
         cancelled: &AtomicBool,
         mut emit: impl FnMut(bool, String),
         mut progress: impl FnMut(u32, u32, Option<f32>),
-    ) -> Result<(), String> {
+    ) -> Result<GenerationOutput, String> {
         self.select_checkpoint(checkpoint)?;
-        let (generated, prompt_complete) =
+        let (output, prompt_complete) =
             self.generate_inner(messages, settings, cancelled, &mut emit, &mut progress)?;
         let mut completed = messages.to_vec();
-        completed.push(generated);
+        completed.push(output.message.clone());
         self.executor.save_checkpoint(
             checkpoint,
             if prompt_complete {
@@ -350,7 +358,57 @@ impl Generator {
             } else {
                 [0; 32]
             },
-        )
+        )?;
+        Ok(output)
+    }
+
+    pub(crate) fn generate_transient(
+        &mut self,
+        directory: &Path,
+        messages: &[ChatTurn],
+        settings: &TurnSettings,
+        cancelled: &AtomicBool,
+        mut emit: impl FnMut(bool, String),
+        mut progress: impl FnMut(u32, u32, Option<f32>),
+    ) -> Result<GenerationOutput, String> {
+        let history = messages
+            .split_last()
+            .map_or(messages, |(_, history)| history);
+        let history_tag =
+            conversation_tag(&settings.system_prompt, settings.reasoning_mode, history);
+        let checkpoint = directory.join(format!("{}.bin", hex_tag(history_tag)));
+        if self.executor.checkpoint_tag() != history_tag {
+            self.select_checkpoint(&checkpoint)?;
+            if self.executor.checkpoint_tag() != history_tag {
+                self.executor.reset()?;
+                self.checkpoint = None;
+                let _ = std::fs::remove_file(&checkpoint);
+            }
+        }
+
+        let (output, prompt_complete) =
+            self.generate_inner(messages, settings, cancelled, &mut emit, &mut progress)?;
+        let mut completed = messages.to_vec();
+        completed.push(output.message.clone());
+        let completed_tag =
+            conversation_tag(&settings.system_prompt, settings.reasoning_mode, &completed);
+        std::fs::create_dir_all(directory).map_err(|error| {
+            format!(
+                "Could not create transient KV cache directory {}: {error}",
+                directory.display()
+            )
+        })?;
+        let completed_checkpoint = directory.join(format!("{}.bin", hex_tag(completed_tag)));
+        self.executor.save_checkpoint(
+            &completed_checkpoint,
+            if prompt_complete {
+                completed_tag
+            } else {
+                [0; 32]
+            },
+        )?;
+        self.checkpoint = Some(completed_checkpoint);
+        Ok(output)
     }
 
     fn select_checkpoint(&mut self, checkpoint: &Path) -> Result<(), String> {
@@ -373,7 +431,7 @@ impl Generator {
         cancelled: &AtomicBool,
         emit: &mut impl FnMut(bool, String),
         progress: &mut impl FnMut(u32, u32, Option<f32>),
-    ) -> Result<(ChatTurn, bool), String> {
+    ) -> Result<(GenerationOutput, bool), String> {
         let tokens = match messages.split_last() {
             Some((latest, history))
                 if latest.user
@@ -418,10 +476,21 @@ impl Generator {
             reasoning_complete: !reasoning,
             content: String::new(),
         };
+        let mut emitted_reasoning = 0;
+        let mut emitted_content = 0;
         let prompt_tokens = tokens.len();
         for (index, token) in tokens.into_iter().enumerate().skip(reused) {
             if cancelled.load(Ordering::Relaxed) {
-                return Ok((generated, false));
+                return Ok((
+                    GenerationOutput {
+                        message: generated,
+                        prompt_tokens: prompt_tokens as u32,
+                        cached_tokens: reused as u32,
+                        completion_tokens: 0,
+                        finish_reason: "stop",
+                    },
+                    false,
+                ));
             }
             self.executor.eval(token)?;
             if (index + 1).is_multiple_of(16) || index + 1 == prompt_tokens {
@@ -436,13 +505,30 @@ impl Generator {
             .min((max_context - self.executor.position() as usize) as i32)
         {
             if cancelled.load(Ordering::Relaxed) {
-                return Ok((generated, true));
+                flush_generated(
+                    &mut generated,
+                    &mut emitted_reasoning,
+                    &mut emitted_content,
+                    &settings.stops,
+                    emit,
+                );
+                return Ok((
+                    GenerationOutput {
+                        message: generated,
+                        prompt_tokens: prompt_tokens as u32,
+                        cached_tokens: reused as u32,
+                        completion_tokens: generated_tokens,
+                        finish_reason: "stop",
+                    },
+                    true,
+                ));
             }
             let token = sample(
                 self.executor.logits(),
                 settings.temperature,
                 settings.top_p,
                 settings.min_p,
+                settings.top_k,
                 &mut rng,
             );
             if self
@@ -450,27 +536,89 @@ impl Generator {
                 .model()
                 .is_stop_token_for_reasoning(token, settings.reasoning_mode)
             {
-                return Ok((generated, true));
+                flush_generated(
+                    &mut generated,
+                    &mut emitted_reasoning,
+                    &mut emitted_content,
+                    &settings.stops,
+                    emit,
+                );
+                return Ok((
+                    GenerationOutput {
+                        message: generated,
+                        prompt_tokens: prompt_tokens as u32,
+                        cached_tokens: reused as u32,
+                        completion_tokens: generated_tokens,
+                        finish_reason: "stop",
+                    },
+                    true,
+                ));
             }
             if self.executor.model().is_think_start_token(token) {
                 reasoning = true;
                 generated.reasoning.get_or_insert_default();
             } else if self.executor.model().is_think_end_token(token) {
+                if let Some(reasoning_text) = &mut generated.reasoning
+                    && emit_safe_text(
+                        reasoning_text,
+                        &mut emitted_reasoning,
+                        &settings.stops,
+                        true,
+                        true,
+                        emit,
+                    )
+                {
+                    return Ok((
+                        GenerationOutput {
+                            message: generated,
+                            prompt_tokens: prompt_tokens as u32,
+                            cached_tokens: reused as u32,
+                            completion_tokens: generated_tokens + 1,
+                            finish_reason: "stop",
+                        },
+                        false,
+                    ));
+                }
                 reasoning = false;
                 generated.reasoning_complete = true;
                 emit(false, String::new());
             } else if let Some(bytes) = self.executor.model().token_bytes(token) {
                 let content = String::from_utf8_lossy(&bytes).into_owned();
-                if reasoning {
+                let stopped = if reasoning {
-                    generated
+                    let text = generated.reasoning.get_or_insert_default();
-                        .reasoning
+                    text.push_str(&content);
-                        .get_or_insert_default()
+                    emit_safe_text(
-                        .push_str(&content);
+                        text,
+                        &mut emitted_reasoning,
+                        &settings.stops,
+                        false,
+                        true,
+                        emit,
+                    )
                 } else {
                     generated.reasoning_complete = true;
                     generated.content.push_str(&content);
+                    emit_safe_text(
+                        &mut generated.content,
+                        &mut emitted_content,
+                        &settings.stops,
+                        false,
+                        false,
+                        emit,
+                    )
+                };
+                if stopped {
+                    return Ok((
+                        GenerationOutput {
+                            message: generated,
+                            prompt_tokens: prompt_tokens as u32,
+                            cached_tokens: reused as u32,
+                            completion_tokens: generated_tokens + 1,
+                            finish_reason: "stop",
+                        },
+                        false,
+                    ));
                 }
-                emit(reasoning, content);
             }
             self.executor.eval(token)?;
             generated_tokens += 1;
@@ -483,10 +631,103 @@ impl Generator {
                 ),
             );
         }
-        Ok((generated, true))
+        flush_generated(
+            &mut generated,
+            &mut emitted_reasoning,
+            &mut emitted_content,
+            &settings.stops,
+            emit,
+        );
+        Ok((
+            GenerationOutput {
+                message: generated,
+                prompt_tokens: prompt_tokens as u32,
+                cached_tokens: reused as u32,
+                completion_tokens: generated_tokens,
+                finish_reason: "length",
+            },
+            true,
+        ))
     }
 }
 
+#[cfg(target_os = "macos")]
+fn flush_generated(
+    generated: &mut ChatTurn,
+    emitted_reasoning: &mut usize,
+    emitted_content: &mut usize,
+    stops: &[String],
+    emit: &mut impl FnMut(bool, String),
+) {
+    if let Some(reasoning) = &mut generated.reasoning {
+        let _ = emit_safe_text(reasoning, emitted_reasoning, stops, true, true, emit);
+    }
+    let _ = emit_safe_text(
+        &mut generated.content,
+        emitted_content,
+        stops,
+        true,
+        false,
+        emit,
+    );
+}
+
+#[cfg(target_os = "macos")]
+fn emit_safe_text(
+    text: &mut String,
+    emitted: &mut usize,
+    stops: &[String],
+    final_flush: bool,
+    reasoning: bool,
+    emit: &mut impl FnMut(bool, String),
+) -> bool {
+    let stop = stops
+        .iter()
+        .filter_map(|stop| {
+            text[*emitted..]
+                .find(stop)
+                .map(|position| *emitted + position)
+        })
+        .min();
+    if let Some(stop) = stop {
+        if stop > *emitted {
+            emit(reasoning, text[*emitted..stop].to_owned());
+        }
+        text.truncate(stop);
+        *emitted = stop;
+        return true;
+    }
+    let hold = if final_flush {
+        0
+    } else {
+        stops
+            .iter()
+            .map(|stop| stop.len().saturating_sub(1))
+            .max()
+            .unwrap_or(0)
+    };
+    let mut safe = text.len().saturating_sub(hold);
+    while safe > *emitted && !text.is_char_boundary(safe) {
+        safe -= 1;
+    }
+    if safe > *emitted {
+        emit(reasoning, text[*emitted..safe].to_owned());
+        *emitted = safe;
+    }
+    false
+}
+
+#[cfg(target_os = "macos")]
+fn hex_tag(tag: [u8; 32]) -> String {
+    const HEX: &[u8; 16] = b"0123456789abcdef";
+    let mut output = String::with_capacity(64);
+    for byte in tag {
+        output.push(HEX[(byte >> 4) as usize] as char);
+        output.push(HEX[(byte & 0x0f) as usize] as char);
+    }
+    output
+}
+
 #[cfg(target_os = "macos")]
 fn conversation_tag(system: &str, reasoning: ReasoningMode, messages: &[ChatTurn]) -> [u8; 32] {
     fn text(hasher: &mut Sha256, value: &str) {
@@ -518,7 +759,14 @@ fn conversation_tag(system: &str, reasoning: ReasoningMode, messages: &[ChatTurn
 }
 
 #[cfg(target_os = "macos")]
-fn sample(logits: &[f32], temperature: f32, top_p: f32, min_p: f32, rng: &mut Rng) -> i32 {
+fn sample(
+    logits: &[f32],
+    temperature: f32,
+    top_p: f32,
+    min_p: f32,
+    top_k: i32,
+    rng: &mut Rng,
+) -> i32 {
     if temperature <= 0.0 {
         return logits
             .iter()
@@ -554,12 +802,16 @@ fn sample(logits: &[f32], temperature: f32, top_p: f32, min_p: f32, rng: &mut Rn
             .max_by(|a, b| a.1.total_cmp(b.1))
             .map_or(0, |(index, _)| index as i32);
     }
-    if top_p < 1.0 {
+    if top_p < 1.0 || top_k > 0 {
         probabilities.sort_unstable_by(|a, b| b.1.total_cmp(&a.1).then_with(|| a.0.cmp(&b.0)));
-        let total: f32 = logits
+        if top_k > 0 {
+            probabilities.truncate(probabilities.len().min(top_k as usize));
+        }
+    }
+    if top_p < 1.0 {
+        let total: f32 = probabilities
             .iter()
-            .filter(|logit| logit.is_finite())
+            .map(|(_, probability)| probability)
-            .map(|logit| ((*logit - maximum) / temperature).exp())
             .sum();
         let mut kept = 0.0;
         let count = probabilities
@@ -612,7 +864,27 @@ mod sampling_tests {
     #[test]
     fn zero_temperature_is_greedy() {
         let mut rng = Rng::new(1);
-        assert_eq!(sample(&[1.0, 4.0, 2.0], 0.0, 1.0, 0.0, &mut rng), 1);
+        assert_eq!(sample(&[1.0, 4.0, 2.0], 0.0, 1.0, 0.0, 0, &mut rng), 1);
+    }
+
+    #[test]
+    fn top_k_and_stream_stops_are_applied_before_output() {
+        let mut rng = Rng::new(1);
+        assert_eq!(sample(&[1.0, 4.0, 2.0], 1.0, 1.0, 0.0, 1, &mut rng), 1);
+
+        let mut text = "hello STOP hidden".to_owned();
+        let mut emitted = 0;
+        let mut chunks = Vec::new();
+        assert!(emit_safe_text(
+            &mut text,
+            &mut emitted,
+            &["STOP".into()],
+            false,
+            false,
+            &mut |_, chunk| chunks.push(chunk),
+        ));
+        assert_eq!(text, "hello ");
+        assert_eq!(chunks, ["hello "]);
     }
 
     #[test]

src/main.rs

@@ -3,8 +3,14 @@ mod database;
 mod engine;
 mod model;
 #[cfg(target_os = "macos")]
+mod native_edit;
+#[cfg(target_os = "macos")]
 mod native_menu;
+#[cfg(target_os = "macos")]
+mod runtime;
 mod schema;
+#[cfg(target_os = "macos")]
+mod server;
 mod settings;
 
 use app::{App, Message, app_icon, app_theme};

src/model.rs

@@ -327,6 +327,13 @@ pub(crate) fn managed_artifacts(models_path: &Path) -> Vec<ManagedArtifact> {
         .collect()
 }
 
+pub(crate) fn installed_models(models_path: &Path) -> Vec<ModelChoice> {
+    MODEL_CHOICES
+        .into_iter()
+        .filter(|model| model.main_artifact().is_installed(*model, models_path))
+        .collect()
+}
+
 pub(crate) fn artifact_download_progress(
     id: ManagedArtifactId,
     models_path: &Path,

src/native_edit.rs

@@ -0,0 +1,315 @@
+use std::collections::VecDeque;
+use std::sync::{Arc, Mutex};
+
+use iced::advanced::layout;
+use iced::advanced::overlay;
+use iced::advanced::renderer;
+use iced::advanced::widget::{Operation, Tree, tree};
+use iced::advanced::{Clipboard, Layout, Shell, Widget};
+use iced::event;
+use iced::keyboard::{self, Key, Location, Modifiers, key};
+use iced::mouse;
+use iced::{Element, Event, Length, Rectangle, Size, Vector};
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub(crate) enum EditCommand {
+    Undo,
+    Redo,
+    Cut,
+    Copy,
+    Paste,
+    SelectAll,
+}
+
+pub(crate) type EditCommandQueue = Arc<Mutex<VecDeque<EditCommand>>>;
+
+pub(crate) fn command_queue() -> EditCommandQueue {
+    Arc::new(Mutex::new(VecDeque::new()))
+}
+
+pub(crate) fn queue_command(queue: &EditCommandQueue, command: EditCommand) {
+    queue
+        .lock()
+        .unwrap_or_else(|poisoned| poisoned.into_inner())
+        .push_back(command);
+}
+
+fn pop_command(queue: &EditCommandQueue) -> Option<EditCommand> {
+    queue
+        .lock()
+        .unwrap_or_else(|poisoned| poisoned.into_inner())
+        .pop_front()
+}
+
+/// Replays native Edit menu actions through the focused Iced widget.
+pub(crate) struct NativeEdit<'a, Message, Theme = iced::Theme, Renderer = iced::Renderer> {
+    content: Element<'a, Message, Theme, Renderer>,
+    commands: EditCommandQueue,
+}
+
+impl<'a, Message, Theme, Renderer> NativeEdit<'a, Message, Theme, Renderer> {
+    fn new(
+        content: impl Into<Element<'a, Message, Theme, Renderer>>,
+        commands: EditCommandQueue,
+    ) -> Self {
+        Self {
+            content: content.into(),
+            commands,
+        }
+    }
+}
+
+#[derive(Default)]
+struct State;
+
+fn modifier_sync_event(event: &Event) -> Option<Event> {
+    let Event::Keyboard(keyboard::Event::KeyPressed { modifiers, .. }) = event else {
+        return None;
+    };
+    Some(Event::Keyboard(keyboard::Event::ModifiersChanged(
+        *modifiers,
+    )))
+}
+
+fn command_events(command: EditCommand) -> [Event; 4] {
+    let (character, modified_character, physical_key, modifiers) = match command {
+        EditCommand::Undo => ("z", "z", key::Code::KeyZ, Modifiers::COMMAND),
+        EditCommand::Redo => (
+            "z",
+            "Z",
+            key::Code::KeyZ,
+            Modifiers::COMMAND | Modifiers::SHIFT,
+        ),
+        EditCommand::Cut => ("x", "x", key::Code::KeyX, Modifiers::COMMAND),
+        EditCommand::Copy => ("c", "c", key::Code::KeyC, Modifiers::COMMAND),
+        EditCommand::Paste => ("v", "v", key::Code::KeyV, Modifiers::COMMAND),
+        EditCommand::SelectAll => ("a", "a", key::Code::KeyA, Modifiers::COMMAND),
+    };
+    let key = Key::Character(character.into());
+    [
+        Event::Keyboard(keyboard::Event::ModifiersChanged(modifiers)),
+        Event::Keyboard(keyboard::Event::KeyPressed {
+            key: key.clone(),
+            modified_key: Key::Character(modified_character.into()),
+            physical_key: key::Physical::Code(physical_key),
+            location: Location::Standard,
+            modifiers,
+            text: None,
+        }),
+        Event::Keyboard(keyboard::Event::KeyReleased {
+            key,
+            location: Location::Standard,
+            modifiers,
+        }),
+        Event::Keyboard(keyboard::Event::ModifiersChanged(Modifiers::default())),
+    ]
+}
+
+impl<'a, Message, Theme, Renderer> Widget<Message, Theme, Renderer>
+    for NativeEdit<'a, Message, Theme, Renderer>
+where
+    Renderer: renderer::Renderer,
+{
+    fn tag(&self) -> tree::Tag {
+        tree::Tag::of::<State>()
+    }
+
+    fn state(&self) -> tree::State {
+        tree::State::new(State)
+    }
+
+    fn children(&self) -> Vec<Tree> {
+        vec![Tree::new(&self.content)]
+    }
+
+    fn diff(&self, tree: &mut Tree) {
+        tree.diff_children(std::slice::from_ref(&self.content));
+    }
+
+    fn size(&self) -> Size<Length> {
+        self.content.as_widget().size()
+    }
+
+    fn layout(
+        &self,
+        tree: &mut Tree,
+        renderer: &Renderer,
+        limits: &layout::Limits,
+    ) -> layout::Node {
+        self.content
+            .as_widget()
+            .layout(&mut tree.children[0], renderer, limits)
+    }
+
+    fn operate(
+        &self,
+        tree: &mut Tree,
+        layout: Layout<'_>,
+        renderer: &Renderer,
+        operation: &mut dyn Operation,
+    ) {
+        self.content
+            .as_widget()
+            .operate(&mut tree.children[0], layout, renderer, operation);
+    }
+
+    fn on_event(
+        &mut self,
+        tree: &mut Tree,
+        event: Event,
+        layout: Layout<'_>,
+        cursor: mouse::Cursor,
+        renderer: &Renderer,
+        clipboard: &mut dyn Clipboard,
+        shell: &mut Shell<'_, Message>,
+        viewport: &Rectangle,
+    ) -> event::Status {
+        let mut status = event::Status::Ignored;
+        if matches!(
+            event,
+            Event::Window(iced::window::Event::RedrawRequested(_))
+        ) {
+            while let Some(command) = pop_command(&self.commands) {
+                for command_event in command_events(command) {
+                    if self.content.as_widget_mut().on_event(
+                        &mut tree.children[0],
+                        command_event,
+                        layout,
+                        cursor,
+                        renderer,
+                        clipboard,
+                        shell,
+                        viewport,
+                    ) == event::Status::Captured
+                    {
+                        status = event::Status::Captured;
+                    }
+                }
+            }
+        }
+
+        if let Some(sync_event) = modifier_sync_event(&event) {
+            let _ = self.content.as_widget_mut().on_event(
+                &mut tree.children[0],
+                sync_event,
+                layout,
+                cursor,
+                renderer,
+                clipboard,
+                shell,
+                viewport,
+            );
+        }
+
+        if self.content.as_widget_mut().on_event(
+            &mut tree.children[0],
+            event,
+            layout,
+            cursor,
+            renderer,
+            clipboard,
+            shell,
+            viewport,
+        ) == event::Status::Captured
+        {
+            event::Status::Captured
+        } else {
+            status
+        }
+    }
+
+    fn mouse_interaction(
+        &self,
+        tree: &Tree,
+        layout: Layout<'_>,
+        cursor: mouse::Cursor,
+        viewport: &Rectangle,
+        renderer: &Renderer,
+    ) -> mouse::Interaction {
+        self.content.as_widget().mouse_interaction(
+            &tree.children[0],
+            layout,
+            cursor,
+            viewport,
+            renderer,
+        )
+    }
+
+    fn draw(
+        &self,
+        tree: &Tree,
+        renderer: &mut Renderer,
+        theme: &Theme,
+        style: &renderer::Style,
+        layout: Layout<'_>,
+        cursor: mouse::Cursor,
+        viewport: &Rectangle,
+    ) {
+        self.content.as_widget().draw(
+            &tree.children[0],
+            renderer,
+            theme,
+            style,
+            layout,
+            cursor,
+            viewport,
+        );
+    }
+
+    fn overlay<'b>(
+        &'b mut self,
+        tree: &'b mut Tree,
+        layout: Layout<'_>,
+        renderer: &Renderer,
+        translation: Vector,
+    ) -> Option<overlay::Element<'b, Message, Theme, Renderer>> {
+        self.content
+            .as_widget_mut()
+            .overlay(&mut tree.children[0], layout, renderer, translation)
+    }
+}
+
+impl<'a, Message, Theme, Renderer> From<NativeEdit<'a, Message, Theme, Renderer>>
+    for Element<'a, Message, Theme, Renderer>
+where
+    Message: 'a,
+    Theme: 'a,
+    Renderer: 'a + renderer::Renderer,
+{
+    fn from(bridge: NativeEdit<'a, Message, Theme, Renderer>) -> Self {
+        Element::new(bridge)
+    }
+}
+
+pub(crate) fn native_edit<'a, Message, Theme, Renderer>(
+    content: impl Into<Element<'a, Message, Theme, Renderer>>,
+    commands: EditCommandQueue,
+) -> NativeEdit<'a, Message, Theme, Renderer> {
+    NativeEdit::new(content, commands)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn paste_replays_a_complete_command_shortcut() {
+        let events = command_events(EditCommand::Paste);
+        assert!(matches!(
+            events[0],
+            Event::Keyboard(keyboard::Event::ModifiersChanged(Modifiers::COMMAND))
+        ));
+        assert!(matches!(
+            events[1],
+            Event::Keyboard(keyboard::Event::KeyPressed {
+                key: Key::Character(ref key),
+                modifiers: Modifiers::COMMAND,
+                ..
+            }) if key == "v"
+        ));
+        assert!(matches!(
+            events[3],
+            Event::Keyboard(keyboard::Event::ModifiersChanged(modifiers)) if modifiers.is_empty()
+        ));
+    }
+}

src/native_menu.rs

@@ -1,8 +1,16 @@
-use muda::accelerator::{Accelerator, Code, Modifiers};
+use muda::accelerator::{Accelerator, CMD_OR_CTRL, Code, Modifiers};
 use muda::{Menu, MenuEvent, MenuItem, PredefinedMenuItem, Submenu};
 
+use crate::native_edit::EditCommand;
+
 const PREFERENCES: &str = "preferences";
 const MODEL_MANAGER: &str = "model-manager";
+const UNDO: &str = "undo";
+const REDO: &str = "redo";
+const CUT: &str = "cut";
+const COPY: &str = "copy";
+const PASTE: &str = "paste";
+const SELECT_ALL: &str = "select-all";
 
 pub(crate) struct NativeMenu {
     _menu: Menu,
@@ -12,6 +20,7 @@ pub(crate) struct NativeMenu {
 pub(crate) enum NativeMenuEvent {
     Preferences,
     ModelManager,
+    Edit(EditCommand),
 }
 
 pub(crate) fn install() -> Result<NativeMenu, String> {
@@ -34,6 +43,12 @@ pub(crate) fn install() -> Result<NativeMenu, String> {
             Code::KeyM,
         )),
     );
+    let undo = edit_item(UNDO, "Undo", Code::KeyZ, None);
+    let redo = edit_item(REDO, "Redo", Code::KeyZ, Some(Modifiers::SHIFT));
+    let cut = edit_item(CUT, "Cut", Code::KeyX, None);
+    let copy = edit_item(COPY, "Copy", Code::KeyC, None);
+    let paste = edit_item(PASTE, "Paste", Code::KeyV, None);
+    let select_all = edit_item(SELECT_ALL, "Select All", Code::KeyA, None);
 
     application
         .append_items(&[
@@ -51,13 +66,13 @@ pub(crate) fn install() -> Result<NativeMenu, String> {
         ])
         .map_err(|error| error.to_string())?;
     edit.append_items(&[
-        &PredefinedMenuItem::undo(None),
+        &undo,
-        &PredefinedMenuItem::redo(None),
+        &redo,
         &PredefinedMenuItem::separator(),
-        &PredefinedMenuItem::cut(None),
+        &cut,
-        &PredefinedMenuItem::copy(None),
+        &copy,
-        &PredefinedMenuItem::paste(None),
+        &paste,
-        &PredefinedMenuItem::select_all(None),
+        &select_all,
     ])
     .map_err(|error| error.to_string())?;
     window
@@ -79,12 +94,47 @@ pub(crate) fn install() -> Result<NativeMenu, String> {
 
 pub(crate) fn next_event() -> Option<NativeMenuEvent> {
     while let Ok(event) = MenuEvent::receiver().try_recv() {
-        if event.id == PREFERENCES {
+        let event = match event.id.0.as_str() {
-            return Some(NativeMenuEvent::Preferences);
+            PREFERENCES => NativeMenuEvent::Preferences,
-        }
+            MODEL_MANAGER => NativeMenuEvent::ModelManager,
-        if event.id == MODEL_MANAGER {
+            UNDO => NativeMenuEvent::Edit(EditCommand::Undo),
-            return Some(NativeMenuEvent::ModelManager);
+            REDO => NativeMenuEvent::Edit(EditCommand::Redo),
-        }
+            CUT => NativeMenuEvent::Edit(EditCommand::Cut),
+            COPY => NativeMenuEvent::Edit(EditCommand::Copy),
+            PASTE => NativeMenuEvent::Edit(EditCommand::Paste),
+            SELECT_ALL => NativeMenuEvent::Edit(EditCommand::SelectAll),
+            _ => continue,
+        };
+        return Some(event);
     }
     None
 }
+
+fn edit_item(
+    id: &'static str,
+    label: &'static str,
+    code: Code,
+    extra_modifier: Option<Modifiers>,
+) -> MenuItem {
+    MenuItem::with_id(
+        id,
+        label,
+        true,
+        Some(Accelerator::new(
+            Some(CMD_OR_CTRL | extra_modifier.unwrap_or_else(Modifiers::empty)),
+            code,
+        )),
+    )
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn edit_items_are_custom_menu_events() {
+        let paste = edit_item(PASTE, "Paste", Code::KeyV, None);
+        assert_eq!(paste.id().0, PASTE);
+        assert!(paste.is_enabled());
+    }
+}

src/runtime.rs

@@ -0,0 +1,151 @@
+use crate::engine::{ChatTurn, GenerationOutput, Generator};
+use crate::settings::{EngineSettings, TurnSettings};
+use std::path::PathBuf;
+use std::sync::Arc;
+use std::sync::atomic::AtomicBool;
+use std::sync::mpsc::{self, Receiver, Sender};
+use std::thread;
+use std::time::{Duration, Instant};
+
+#[derive(Clone)]
+pub(crate) struct GenerationService {
+    commands: Sender<Command>,
+}
+
+pub(crate) struct ActiveGeneration {
+    pub(crate) events: Receiver<GenerationEvent>,
+    pub(crate) cancel: Arc<AtomicBool>,
+}
+
+pub(crate) enum CheckpointTarget {
+    Local(PathBuf),
+    Transient(PathBuf),
+}
+
+pub(crate) enum GenerationEvent {
+    Loading,
+    Chunk {
+        reasoning: bool,
+        content: String,
+    },
+    Context {
+        used: u32,
+        limit: u32,
+        tokens_per_second: Option<f32>,
+    },
+    Finished(Result<GenerationOutput, String>),
+}
+
+struct Command {
+    engine: EngineSettings,
+    turn: TurnSettings,
+    messages: Vec<ChatTurn>,
+    checkpoint: CheckpointTarget,
+    idle_timeout: Duration,
+    cancel: Arc<AtomicBool>,
+    events: Sender<GenerationEvent>,
+}
+
+impl GenerationService {
+    pub(crate) fn spawn() -> Result<Self, String> {
+        let (commands, receiver) = mpsc::channel::<Command>();
+        thread::Builder::new()
+            .name("model-runtime".into())
+            .spawn(move || run(receiver))
+            .map_err(|error| format!("Could not start the model runtime: {error}"))?;
+        Ok(Self { commands })
+    }
+
+    pub(crate) fn generate(
+        &self,
+        engine: EngineSettings,
+        turn: TurnSettings,
+        messages: Vec<ChatTurn>,
+        checkpoint: CheckpointTarget,
+        idle_timeout: Duration,
+    ) -> Result<ActiveGeneration, String> {
+        let cancel = Arc::new(AtomicBool::new(false));
+        let (events, receiver) = mpsc::channel();
+        self.commands
+            .send(Command {
+                engine,
+                turn,
+                messages,
+                checkpoint,
+                idle_timeout,
+                cancel: Arc::clone(&cancel),
+                events,
+            })
+            .map_err(|_| "The model runtime stopped unexpectedly.".to_owned())?;
+        Ok(ActiveGeneration {
+            events: receiver,
+            cancel,
+        })
+    }
+}
+
+fn run(commands: Receiver<Command>) {
+    let mut loaded = None::<(EngineSettings, Generator)>;
+    let mut last_used = Instant::now();
+    let mut idle_timeout = Duration::from_secs(15 * 60);
+    loop {
+        match commands.recv_timeout(Duration::from_secs(1)) {
+            Ok(command) => {
+                idle_timeout = command.idle_timeout;
+                if loaded
+                    .as_ref()
+                    .is_none_or(|(settings, _)| settings != &command.engine)
+                {
+                    let _ = command.events.send(GenerationEvent::Loading);
+                    loaded = match Generator::open(&command.engine) {
+                        Ok(generator) => Some((command.engine.clone(), generator)),
+                        Err(error) => {
+                            let _ = command.events.send(GenerationEvent::Finished(Err(error)));
+                            None
+                        }
+                    };
+                }
+                if let Some((_, generator)) = &mut loaded {
+                    let mut emit = |reasoning, content| {
+                        let _ = command
+                            .events
+                            .send(GenerationEvent::Chunk { reasoning, content });
+                    };
+                    let mut progress = |used, limit, tokens_per_second| {
+                        let _ = command.events.send(GenerationEvent::Context {
+                            used,
+                            limit,
+                            tokens_per_second,
+                        });
+                    };
+                    let result = match command.checkpoint {
+                        CheckpointTarget::Local(path) => generator.generate(
+                            &path,
+                            &command.messages,
+                            &command.turn,
+                            &command.cancel,
+                            &mut emit,
+                            &mut progress,
+                        ),
+                        CheckpointTarget::Transient(directory) => generator.generate_transient(
+                            &directory,
+                            &command.messages,
+                            &command.turn,
+                            &command.cancel,
+                            &mut emit,
+                            &mut progress,
+                        ),
+                    };
+                    let _ = command.events.send(GenerationEvent::Finished(result));
+                    last_used = Instant::now();
+                }
+            }
+            Err(mpsc::RecvTimeoutError::Timeout) => {
+                if loaded.is_some() && last_used.elapsed() >= idle_timeout {
+                    loaded = None;
+                }
+            }
+            Err(mpsc::RecvTimeoutError::Disconnected) => break,
+        }
+    }
+}

src/schema.rs

@@ -53,6 +53,7 @@ diesel::table! {
         directional_steering_attn -> Nullable<Float>,
         simulated_used_memory_gib -> Nullable<BigInt>,
         expert_profile_path -> Nullable<Text>,
+        endpoint_port -> Integer,
     }
 }
 

src/settings.rs

@@ -443,6 +443,8 @@ impl GenerationPreferences {
             temperature: self.temperature.unwrap_or(1.0),
             top_p: self.top_p.unwrap_or(if glm { 0.95 } else { 1.0 }),
             min_p: self.min_p.unwrap_or(if glm { 0.0 } else { 0.05 }),
+            top_k: 0,
+            stops: Vec::new(),
             seed: self.seed,
             reasoning_mode: if self.reasoning_mode == ReasoningMode::Max
                 && self.context_tokens < THINK_MAX_MIN_CONTEXT
@@ -489,6 +491,8 @@ pub(crate) struct TurnSettings {
     pub(crate) temperature: f32,
     pub(crate) top_p: f32,
     pub(crate) min_p: f32,
+    pub(crate) top_k: i32,
+    pub(crate) stops: Vec<String>,
     pub(crate) seed: Option<u64>,
     pub(crate) reasoning_mode: ReasoningMode,
 }