hugo/DS4Server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: first cut at openai compatible server

Browse Source
This commit is contained in:
Georg Bauer
2026-07-24 21:12:48 +02:00
parent bbbe65a75f
commit d554b77b9d
18 changed files with 2612 additions and 535 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
Cargo.lock generated
View File

@@ -1150,6 +1150,8 @@ dependencies = [
"muda",
"png",
"rfd",
"serde",
"serde_json",
"sha2",
"ureq",
]
@@ -4037,6 +4039,7 @@ version = "1.0.151"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14"
dependencies = [
"indexmap",
"itoa",
"memchr",
"serde",

4
Cargo.toml
View File

@@ -14,10 +14,12 @@ cc = "1.3.0"
[dependencies]
diesel = { version = "2.3.11", features = ["sqlite", "returning_clauses_for_sqlite_3_35", "64-column-tables"] }
diesel_migrations = "2.3.2"
iced = { version = "0.13.1", features = ["highlighter", "markdown", "svg", "tokio"] }
iced = { version = "0.13.1", features = ["advanced", "highlighter", "markdown", "svg", "tokio"] }
memmap2 = "0.9.11"
png = "0.17.16"
rfd = "0.15.4"
serde = { version = "1.0.228", features = ["derive"] }
serde_json = { version = "1.0.149", features = ["preserve_order"] }
sha2 = "0.11.0"
ureq = { version = "3.3.0", default-features = false, features = ["rustls"] }

150
PLAN.md
View File

@@ -13,11 +13,11 @@ sessions, tools, and turn orchestration.
| Area | Status | Available now | Still open |
| --- | --- | --- | --- |
| App shell | Implemented | Native multi-window Iced app, Codex-inspired layout, native Application/Edit/Window menus, Preferences panel, Model Manager, and streaming composer | Native app release packaging |
| Projects and sessions | Implemented with transcripts | Native folder picker, project-name dialog, create/select/delete projects and sessions, and durable structured chat history | KV payloads, rename/archive, and model binding |
| Persistence | Implemented for metadata, preferences, and chat | SQLite in Application Support, Diesel ORM, embedded migrations, constrained defaults, streamed message updates, and reopen tests | Downloaded-artifact metadata and session-runtime migrations |
| Model runtime | DeepSeek Flash vertical implemented | Rust-owned mmap/model/session graph, reused Metal kernels, 2K compressed-attention generation, DS4 sampling, lazy background loading, cancellation, and idle unloading | Ratio-4 sparse indexer beyond 2K, DSpark/SSD/steering, GLM/Pro, KV checkpoint persistence, and shared acquisition for HTTP |
| Local endpoint | Not started | Nothing listening | OpenAI-compatible HTTP and streaming surfaces |
| Agent | Basic local chat implemented | Multi-turn role rendering, structured reasoning disclosure, streamed DeepSeek Flash text, composer send/stop, and durable transcript rehydration | Tools, approvals, compaction, statistics, and KV-backed resume |
| Projects and sessions | Implemented with transcripts and KV resume | Native folder picker, project-name dialog, create/select/delete projects and sessions, durable structured chat history, and per-session KV checkpoints | Rename/archive and model binding |
| Persistence | Implemented for local chat | SQLite in Application Support, Diesel ORM, embedded migrations, constrained defaults, streamed message updates, reopen tests, context/speed state, and local KV rehydration | Downloaded-artifact metadata and session-runtime migrations; HTTP chats deliberately never enter SQLite |
| Model runtime | DeepSeek Flash vertical implemented | Rust-owned mmap/model/session graph, local Metal kernels, full configured sparse context, DS4 sampling, one process-wide UI/HTTP owner, cancellation, idle unloading, prefix continuation, and KV checkpoint save/load | DSpark/SSD/steering and GLM/Pro execution |
| Local endpoint | Chat Completions vertical implemented | Automatic configurable localhost listener (port 4000 by default), verified-on-disk model discovery, transient requests, non-streaming and SSE Chat Completions, reasoning, usage, sampling, stops, cancellation, tools, and exact in-process tool replay | GUI status and enable/disable controls, Responses, legacy Completions, Anthropic Messages, full `ds4_server.c` fixture parity, and cross-restart exact tool replay |
| Agent | Solid local chat implemented | Multi-turn role rendering, structured reasoning disclosure, Markdown, automatic scrolling, context/speed display, durable transcript rehydration, and KV-backed resume | Tools, approvals, compaction, and richer statistics |
| A2UI | Future extension | bDS2 provides the reference structured render-tool contract | Native inline surfaces for local chat after core chat and tools are stable |
| Dev brain | Not started | No vault access | Obsidian vault selection, durable access, and agent knowledge/memory tools |
| Release | Not started | Development binary builds and tests | `.app` packaging, signing, entitlements, and notarization |
@@ -28,8 +28,9 @@ without blocking the UI. Local chat now lazily acquires the Rust DeepSeek Flash
executor, streams generated text, supports cancellation and multi-turn role
replay, persists reasoning and answers while they stream, rehydrates selected
sessions, and unloads model resources after the configured idle timeout. The
current Rust graph supports DS4 compressed attention through 2K context; the sparse ratio-4 indexer,
KV checkpoints, HTTP service, and agent tools remain open.
current Rust graph supports the full configured context with ratio-4 sparse
attention and durable local KV checkpoints. The shared HTTP Chat Completions
service is available; the remaining protocol routes and agent tools remain open.
## Phase 0 — project and session shell
@@ -49,16 +50,16 @@ Exit criterion: restart the app and see the same project/session tree.
## Phase 1 — model library and on-demand loading
Status: **partially implemented**. The typed preference/download/intake path and
a DeepSeek Flash Rust/Metal vertical are implemented, including compressed KV
through 2K context, DS4 sampling, a single background owner, cancellation, and
idle unload. Sparse indexed attention, persistent KV/session sync, other model
families, and optional execution modes remain open.
a DeepSeek Flash Rust/Metal vertical are implemented, including full configured
context, ratio-4 sparse indexed attention, prefix continuation, KV checkpoint
save/load, DS4 sampling, cancellation, idle unload, and shared HTTP acquisition.
Other model families and optional execution modes remain open.
1. **Implemented:** the typed model/runtime preference contract described below
is complete before chat, the HTTP endpoint, or the engine lifecycle.
2. **Partially implemented:** mmap-backed loading, tokenizer/prompt rendering,
Rust-owned Flash session creation/evaluation/sampling, and model ownership
are complete. Prefix sync plus KV save/load remain.
2. **Implemented for DeepSeek Flash:** mmap-backed loading, tokenizer/prompt
rendering, Rust-owned session creation/evaluation/sampling, prefix sync, and
KV save/load are complete.
3. **Implemented:** retain the tested model restrictions from DwarfStar. Main
and DSpark artifacts reject invalid GGUF versions, catalog mismatches, and
incompatible metadata, tensor shapes, offsets, or quantization before
@@ -66,8 +67,11 @@ families, and optional execution modes remain open.
4. **Partially implemented:** reuse the existing `.metal` kernels and preserve mmap-backed resident
loading plus the explicit SSD-streaming path. Keep Objective-C only at the
Metal interop boundary.
5. **Implemented for local chat:** keep one engine resident at most, matching DwarfStar's instance-lock and
memory assumptions.
5. Keep exactly one model resident process-wide. Local chat and every endpoint
request must use the same owner and queue; model selection may replace the
resident model only after active work reaches a safe boundary. Multiple
simultaneously loaded models are out of scope because DS4Server targets
models large enough to consume most available memory.
6. **Implemented for local chat:** move engine work off the UI thread and support cooperative cancellation at
the safe session boundaries already defined by `ds4_session_sync`.
@@ -92,21 +96,25 @@ to Rust while preserving its semantics and on-disk compatibility:
checkpoints, model/quantization/context/payload-ABI compatibility checks,
tool-call replay metadata, disk-budget enforcement, hit tracking, and
eviction behavior.
- Use one process-wide Rust session registry and KV checkpoint store for both
local chat and the HTTP server. A remote request bound to an application
session resolves to the same logical DS4 session as the GUI and mutations are
serialized so two callers never write one live timeline concurrently.
Stateless HTTP requests still participate in the same compatible-prefix
checkpoint pool, so a prefix produced locally can be reused remotely and
vice versa.
- Store that shared checkpoint pool under Application Support in
`kv-cache/`, using the DS4 key/header/payload rules. Session rows hold only an
optional reference to their current checkpoint; the large opaque payload is
not copied into separate GUI and HTTP caches.
- Sharing does not merge unrelated conversations or expose one session's
transcript to another. It shares the DS4 implementation, checkpoint pool, and
explicitly bound logical session; each unrelated live conversation keeps its
own mutable session state.
- Use one process-wide Rust model owner and KV checkpoint implementation for
local chat and the HTTP server. Model access is serialized so two callers
never mutate the single resident inference graph concurrently.
- Application projects and sessions belong exclusively to local chat. HTTP
requests are transient and must never create projects, sessions, messages,
transcript rows, request-identity mappings, or any other durable chat record.
The outside client owns conversation identity and resends the history needed
for each request, exactly as `ds4_server.c` expects.
- Transient HTTP requests may read and write engine-owned KV cache files for
compatible-prefix reuse. Those opaque cache files are the only server-side
persistence permitted for external conversations and do not establish an
application session or recoverable transcript.
- Store engine-owned checkpoints under Application Support in `kv-cache/`.
Local chats use their application session ID; HTTP uses content-addressed
entries under `kv-cache/http/`. The large opaque payload is never stored in
SQLite, and HTTP cache entries do not create application sessions.
- Cache sharing does not merge conversations or expose transcript text. It
shares only the DS4 implementation and compatible opaque prefix checkpoints;
the request body remains the source of truth for external conversation state.
- Engine unload must first leave every reusable live timeline in a valid
persisted checkpoint. Reload restores a compatible checkpoint immediately;
if model identity, quantization, context size, payload ABI, or rendered
@@ -140,6 +148,8 @@ to Rust while preserving its semantics and on-disk compatibility:
- Let the user enable or disable DSpark for the selected model. Disable the
control for models that do not support DSpark.
- Let the user configure the inactivity timeout; default to 10 minutes.
- Let the user configure the localhost endpoint port; default to 4000 and apply
a changed port when preferences are saved.
- Persist preferences in the application SQLite database. Changing preferences
never loads a model by itself.
@@ -258,43 +268,58 @@ engine lifecycle.
## Phase 2 — OpenAI-compatible local endpoint
Status: **open; not started**.
Status: **partially implemented**. The shared single-model runtime,
verified-model discovery, configurable localhost port, and the first
`ds4_server.c`-compatible Chat Completions vertical are implemented. The
remaining protocol routes plus GUI status and enable/disable controls remain
open.
1. Add a localhost-only HTTP service whose lifecycle is independent of the
1. **Partially implemented:** add a localhost-only HTTP service whose lifecycle is independent of the
engine. It can listen while the model is unloaded and acquires the configured
model only for inference. Start and stop it from the GUI; never expose a LAN
listener without an explicit setting.
2. Implement the DwarfStar compatibility surface in this order:
model only for inference. It listens on configurable port 4000 by default;
start/stop controls remain open. Never expose a LAN listener without an
explicit setting.
2. Implement the DwarfStar compatibility surface from `ds4_server.c` in this
order, preserving its request parsing, prompt formatting, reasoning fields,
streaming events, errors, usage accounting, stop behavior, tool replay, and
cancellation semantics rather than inventing an app-specific protocol:
- `GET /v1/models`
- `POST /v1/chat/completions`
- `POST /v1/responses`
- `POST /v1/completions`
`GET /v1/models` reports the configured model without forcing it to load.
`GET /v1/models` scans managed artifacts without forcing a model to load and
reports only main model files that are fully downloaded and have a matching
on-disk verification marker. Missing, partial, unverified, or checksum-invalid
artifacts are never advertised.
3. Support streaming SSE, reasoning output, usage accounting, cancellation,
sampling parameters, tool schemas, and tool choice.
4. Port exact sampled tool-call replay and deterministic canonicalization from
`ds4_server.c` so a client's normalized JSON does not destroy KV-prefix
reuse.
5. Route API conversations through the shared session registry and KV store
defined in Phase 1. Responses `conversation` bindings and any other
persistent request identity map to the same application session used by the
GUI; stateless requests use shared DS4 prefix lookup. Persist/restore only
through the engine-owned serialization format. Add resident batching only
after single-session and cross-surface correctness are established.
5. Route API work through the single process-wide model owner and engine-owned
KV store defined in Phase 1. External requests are always stateless at the
application layer: never write their messages to SQLite and never bind a
Responses `conversation`, request identifier, or client-supplied session key
to a local project/session. The client must replay its protocol history;
DS4-compatible prefix lookup may accelerate that replay using opaque KV cache
files. Add resident batching only if one-model serialized execution is later
proven insufficient.
6. Display endpoint address, model, active requests, token rates, and errors in
the GUI.
Exit criterion: the upstream DwarfStar server tests pass against the app for
non-streaming and streaming Chat Completions and Responses calls, including a
multi-turn tool call. A compatible prefix created by local chat must register a
remote cache hit, a remote checkpoint must be reusable by local chat, and a
GUI/HTTP race on one bound session must serialize without corrupting its token
or KV frontier.
multi-turn tool call. Verified installed models are listed without loading one;
missing/unverified models are absent; local and HTTP generations cannot load two
models concurrently; external calls leave no project, session, message, or
transcript rows behind; and compatible KV cache files remain reusable.
## Phase 3 — durable agent sessions
Status: **partially implemented**. Project/session metadata and structured
transcripts are durable; model binding and KV payload persistence remain open.
Status: **partially implemented for local chat only**. Project/session metadata,
structured transcripts, context statistics, and per-session KV checkpoints are
durable. Model binding and archive behavior remain open. This phase never
applies to external endpoint conversations.
Extend each metadata-only session with transcript and shared-checkpoint
metadata:
@@ -306,14 +331,14 @@ Application Support/DS4Server.rfc1437.de/
main.gguf
dspark.gguf # only when enabled
kv-cache/
<ds4-cache-key>.kv
<local-session-id>.bin
http/<conversation-tag>.bin
```
- **Partially implemented:** migrated message tables persist user text,
reasoning state, and assistant output while generation streams. Add the
optional current-checkpoint key to SQLite. Keep large engine-owned KV
payloads only in the process-wide Phase 1 store, written with atomic
replacement; do not introduce separate agent-only or HTTP-only cache formats.
- **Implemented:** migrated message tables persist local user text, reasoning
state, assistant output, context usage, and generation speed while generation
streams. Local session KV payloads remain engine-owned files written with
atomic replacement; SQLite never stores opaque tensor data.
- Record model identity, context size, rendered token history, title,
timestamps, and working directory.
- Restore compatible KV immediately. If KV is absent or incompatible, rebuild
@@ -365,6 +390,11 @@ ephemeral presentation preferences are the only exception.
### Native macOS menus (required with chat)
Status: **partially implemented**. Application, Edit, and Window menus are
installed; Cut, Copy, Paste, and Select All now bridge into the focused Iced
text field. File/View/Help menus, dynamic enabled state, the remaining Edit
actions, and selectable transcript text remain open.
The application must install and maintain a complete native macOS menu bar;
the application-name menu alone is not sufficient. Use platform menu roles,
ordering, selectors, and standard key equivalents rather than drawing menus
@@ -493,11 +523,13 @@ tests, and release packaging remain open.
1. Finish the Flash session core as one larger slice: port the ratio-4 indexer
and indexed attention for full configured context, then add prefix sync and
compatible KV checkpoint save/load.
compatible KV checkpoint save/load. **Implemented.**
2. Put the OpenAI-compatible endpoint on the same engine lifecycle so local
chats and HTTP requests cannot load duplicate engines.
3. Add transcript/KV persistence, then connect the existing conversation UI and
finally port the agent tools.
chats and HTTP requests cannot load duplicate models. Keep HTTP conversation
state transient except for opaque KV cache files. **Chat Completions vertical
implemented; remaining reference routes are open.**
3. Local transcript/KV persistence and the conversation UI are implemented;
port the agent tools after the shared endpoint lifecycle is stable.
4. Add the opt-in Dev Brain tool after local file/search boundaries and agent
approvals are stable.
5. Add bDS2-style A2UI render tools and native inline surfaces after the core

21
README.md
View File

@@ -4,11 +4,15 @@ DS4Server is a native macOS coding-agent application that rewrites the
DwarfStar (`ds4`) inference engine in Rust. It uses Rust and Iced and will combine local model loading, an
OpenAI-compatible localhost endpoint, and project-scoped agent chat in one app.
DS4Server vendors and adapts the Metal kernels and Objective-C Metal glue from
[DwarfStar (`ds4`)](https://github.com/antirez/ds4). Their copyright and license
notices are retained in [`native/metal/LICENSE`](native/metal/LICENSE).
The current milestone provides a Codex-inspired project/session layout. A native
macOS folder picker selects each workspace, then the app asks for its display
name. Projects, sessions, and model preferences are persisted through Diesel in
SQLite. Open Preferences with `Command-,` to configure model, generation,
runtime, and idle-unload settings. The separate Model Manager
runtime, local endpoint, and idle-unload settings. The separate Model Manager
(`Shift-Command-M`)
lists local main and DSpark artifacts, their on-disk sizes and state, and lets
you download, resume, validate, or delete them. Rust-native background work
@@ -21,9 +25,17 @@ The selected DeepSeek V4 Flash model can run directly from a project session.
The model, KV/compressor state, 43-layer graph, sampling, and lifecycle are
owned by Rust; a fixed snapshot of the Objective-C Metal boundary and unchanged
Metal kernels is vendored and built inside this repository. Tokens stream into the chat UI, Stop cancels generation,
follow-up turns replay their role history, and the model unloads after the idle
timeout. The current graph supports 2K context while the sparse long-context
indexer and persistent transcript/KV restore remain the next engine slice.
follow-up turns reuse durable transcript and KV state, and the model unloads
after the idle timeout. The graph uses the full configured context with the
ratio-4 sparse indexer.
The app also listens on `127.0.0.1:4000` by default for `GET /v1/models` and
`POST /v1/chat/completions`; the port is configurable in Preferences. The
endpoint and local chat share the single model owner. External conversations
are client-managed and never enter the project, session, message, or transcript
database; only opaque content-addressed KV cache files are retained. Model
discovery advertises only supported main artifacts that are fully downloaded
and verified on disk.
```sh
cargo install cargo-packager --locked --version 0.11.8
@@ -35,6 +47,7 @@ State is stored at:
```text
~/Library/Application Support/DS4Server.rfc1437.de/data.sqlite3
~/Library/Application Support/DS4Server.rfc1437.de/kv-cache/
```
Deleting a project or session removes only DS4Server metadata. It never deletes

1
migrations/20260725000000_add_endpoint_port/down.sql Normal file
View File

@@ -0,0 +1 @@
ALTER TABLE preferences DROP COLUMN endpoint_port;

2
migrations/20260725000000_add_endpoint_port/up.sql Normal file
View File

@@ -0,0 +1,2 @@
ALTER TABLE preferences ADD COLUMN endpoint_port INTEGER NOT NULL DEFAULT 4000
CHECK (endpoint_port BETWEEN 1 AND 65535);

332
src/app.rs
View File

@@ -4,8 +4,10 @@ pub(crate) use view::app_theme;
use crate::database::{AppPreferences, Database, ProjectWithSessions, StoredMessage};
#[cfg(target_os = "macos")]
use crate::engine::{ChatTurn, Generator};
use crate::engine::ChatTurn;
use crate::model::{self, DownloadOutcome, DownloadProgress, ManagedArtifactId, ModelChoice};
#[cfg(target_os = "macos")]
use crate::runtime::{ActiveGeneration, CheckpointTarget, GenerationEvent, GenerationService};
use crate::settings::{
DiagnosticPreferences, ExecutionPreferences, GIB, GenerationPreferences, ReasoningMode,
RuntimePreferences, SpeculativePreferences, SsdPreferences, SteeringPreferences,
@@ -16,9 +18,9 @@ use iced::{Size, Subscription, Task, keyboard, window};
use rfd::AsyncFileDialog;
use std::fs;
use std::path::PathBuf;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::sync::mpsc::{self, TryRecvError};
use std::sync::{Arc, RwLock};
use std::thread;
use std::time::{Duration, Instant};
@@ -29,6 +31,7 @@ struct PreferenceDraft {
model: ModelChoice,
dspark_enabled: bool,
idle_timeout_minutes: String,
endpoint_port: String,
context_tokens: String,
max_generated_tokens: String,
system_prompt: String,
@@ -73,6 +76,7 @@ impl PreferenceDraft {
model,
dspark_enabled: speculative.dspark_enabled,
idle_timeout_minutes: preferences.idle_timeout_minutes.to_string(),
endpoint_port: preferences.endpoint_port.to_string(),
context_tokens: generation.context_tokens.to_string(),
max_generated_tokens: generation.max_generated_tokens.to_string(),
system_prompt: generation.system_prompt,
@@ -168,6 +172,7 @@ impl PreferenceDraft {
self.model = ModelChoice::default();
self.dspark_enabled = false;
self.idle_timeout_minutes = "10".into();
self.endpoint_port = "4000".into();
self.context_tokens = defaults.context_tokens.to_string();
self.max_generated_tokens = defaults.max_generated_tokens.to_string();
self.system_prompt = defaults.system_prompt;
@@ -389,6 +394,8 @@ pub(crate) struct App {
pub(super) pending_model_delete: Option<ManagedArtifactId>,
#[cfg(target_os = "macos")]
_native_menu: Option<crate::native_menu::NativeMenu>,
#[cfg(target_os = "macos")]
pub(super) native_edit_commands: crate::native_edit::EditCommandQueue,
database: Option<Database>,
projects: Vec<ProjectWithSessions>,
preferences: AppPreferences,
@@ -408,7 +415,13 @@ pub(crate) struct App {
pub(super) context_limit: u32,
pub(super) tokens_per_second: Option<f32>,
#[cfg(target_os = "macos")]
generation_worker: Option<GenerationWorker>,
generation_service: Option<GenerationService>,
#[cfg(target_os = "macos")]
active_generation: Option<ActiveGeneration>,
#[cfg(target_os = "macos")]
runtime_preferences: Arc<RwLock<AppPreferences>>,
#[cfg(target_os = "macos")]
_endpoint: Option<crate::server::ServerHandle>,
error: Option<String>,
}
@@ -461,40 +474,6 @@ impl From<StoredMessage> for ChatMessage {
}
}
#[cfg(target_os = "macos")]
struct GenerationWorker {
commands: mpsc::Sender<GenerationCommand>,
events: mpsc::Receiver<GenerationEvent>,
cancel: Option<Arc<AtomicBool>>,
}
#[cfg(target_os = "macos")]
enum GenerationCommand {
Generate {
engine: crate::settings::EngineSettings,
turn: crate::settings::TurnSettings,
messages: Vec<ChatTurn>,
checkpoint: PathBuf,
idle_timeout: Duration,
cancel: Arc<AtomicBool>,
},
}
#[cfg(target_os = "macos")]
enum GenerationEvent {
Loading,
Chunk {
reasoning: bool,
content: String,
},
Context {
used: u32,
limit: u32,
tokens_per_second: Option<f32>,
},
Finished(Result<(), String>),
}
#[derive(Debug)]
pub(super) enum ModelDownload {
Idle,
@@ -526,6 +505,8 @@ pub(super) struct ActiveDownload {
#[derive(Debug, Clone)]
pub(crate) enum Message {
Noop,
#[cfg(target_os = "macos")]
NativeEdit(crate::native_edit::EditCommand),
OpenPreferences,
OpenModelManager,
ModelManagerOpened(window::Id),
@@ -535,6 +516,7 @@ pub(crate) enum Message {
PreferenceModelChanged(ModelChoice),
PreferenceDsparkChanged(bool),
PreferenceTimeoutChanged(String),
PreferenceEndpointPortChanged(String),
PreferenceContextChanged(String),
PreferenceMaxTokensChanged(String),
PreferenceSystemPromptChanged(String),
@@ -602,12 +584,17 @@ impl App {
Err(error) => return Self::failed(error, main_window),
};
let context_limit = preferences.context_tokens.max(0) as u32;
#[cfg(target_os = "macos")]
let (runtime_preferences, generation_service, endpoint, service_error) =
spawn_services(&preferences);
Self {
main_window,
model_manager_window: None,
pending_model_delete: None,
#[cfg(target_os = "macos")]
_native_menu: None,
#[cfg(target_os = "macos")]
native_edit_commands: crate::native_edit::command_queue(),
database: Some(database),
projects,
preferences,
@@ -627,8 +614,23 @@ impl App {
context_limit,
tokens_per_second: None,
#[cfg(target_os = "macos")]
generation_worker: None,
error: None,
generation_service,
#[cfg(target_os = "macos")]
active_generation: None,
#[cfg(target_os = "macos")]
runtime_preferences,
#[cfg(target_os = "macos")]
_endpoint: endpoint,
error: {
#[cfg(target_os = "macos")]
{
service_error
}
#[cfg(not(target_os = "macos"))]
{
None
}
},
}
}
(Err(error), _) | (_, Err(error)) => Self::failed(error, main_window),
@@ -642,12 +644,21 @@ impl App {
let preference_draft = PreferenceDraft::from_saved(&preferences)
.expect("default preferences must use a supported model");
let context_limit = preferences.context_tokens.max(0) as u32;
#[cfg(target_os = "macos")]
let (runtime_preferences, generation_service, endpoint, service_error) =
spawn_services(&preferences);
#[cfg(target_os = "macos")]
let startup_error = service_error;
#[cfg(not(target_os = "macos"))]
let startup_error = None::<String>;
Self {
main_window,
model_manager_window: None,
pending_model_delete: None,
#[cfg(target_os = "macos")]
_native_menu: None,
#[cfg(target_os = "macos")]
native_edit_commands: crate::native_edit::command_queue(),
database: None,
projects: Vec::new(),
preferences,
@@ -667,14 +678,29 @@ impl App {
context_limit,
tokens_per_second: None,
#[cfg(target_os = "macos")]
generation_worker: None,
error: Some(format!("Could not open the project database: {error}")),
generation_service,
#[cfg(target_os = "macos")]
active_generation: None,
#[cfg(target_os = "macos")]
runtime_preferences,
#[cfg(target_os = "macos")]
_endpoint: endpoint,
error: Some(match startup_error {
Some(service_error) => {
format!("Could not open the project database: {error}. {service_error}")
}
None => format!("Could not open the project database: {error}"),
}),
}
}
pub(crate) fn update(&mut self, message: Message) -> Task<Message> {
match message {
Message::Noop => {}
#[cfg(target_os = "macos")]
Message::NativeEdit(command) => {
crate::native_edit::queue_command(&self.native_edit_commands, command)
}
Message::OpenPreferences => self.open_preferences(),
Message::OpenModelManager => return self.open_model_manager(),
Message::ModelManagerOpened(id) => {
@@ -749,6 +775,10 @@ impl App {
self.preference_draft.idle_timeout_minutes = value;
self.preference_error = None;
}
Message::PreferenceEndpointPortChanged(value) => {
self.preference_draft.endpoint_port = value;
self.preference_error = None;
}
Message::PreferenceContextChanged(value) => {
self.preference_draft.context_tokens = value;
self.preference_error = None;
@@ -953,14 +983,11 @@ impl App {
self.start_generation();
return scroll_chat_to_end();
}
Message::StopGeneration => {
Message::StopGeneration =>
{
#[cfg(target_os = "macos")]
if let Some(cancel) = self
.generation_worker
.as_ref()
.and_then(|worker| worker.cancel.as_ref())
{
cancel.store(true, Ordering::Relaxed);
if let Some(active) = &self.active_generation {
active.cancel.store(true, Ordering::Relaxed);
}
}
Message::GenerationTick => {
@@ -1133,6 +1160,9 @@ impl App {
Some(crate::native_menu::NativeMenuEvent::ModelManager) => {
Message::OpenModelManager
}
Some(crate::native_menu::NativeMenuEvent::Edit(command)) => {
Message::NativeEdit(command)
}
None => Message::Noop,
}
}));
@@ -1255,6 +1285,14 @@ impl App {
self.preference_error = Some("Idle timeout must be between 1 and 1440 minutes.".into());
return;
}
let Ok(endpoint_port) = self.preference_draft.endpoint_port.trim().parse::<u16>() else {
self.preference_error = Some("Endpoint port must be a whole number.".into());
return;
};
if endpoint_port == 0 {
self.preference_error = Some("Endpoint port must be between 1 and 65535.".into());
return;
}
let generation = match self.preference_draft.generation() {
Ok(generation) => generation,
Err(error) => {
@@ -1275,12 +1313,50 @@ impl App {
self.preference_error = Some(error);
return;
}
#[cfg(target_os = "macos")]
let pending_endpoint = if self.preferences.endpoint_port != i32::from(endpoint_port)
|| self._endpoint.is_none()
{
let Some(generation) = &self.generation_service else {
self.preference_error = Some("The model runtime is unavailable.".into());
return;
};
match crate::server::ServerHandle::spawn(
generation.clone(),
Arc::clone(&self.runtime_preferences),
models_path(),
application_support_path().join("kv-cache").join("http"),
endpoint_port,
) {
Ok(endpoint) => Some(endpoint),
Err(error) => {
self.preference_error = Some(error);
return;
}
}
} else {
None
};
let Some(database) = &mut self.database else {
return;
};
match database.update_preferences(model.id(), idle_timeout_minutes, &generation, &runtime) {
match database.update_preferences(
model.id(),
idle_timeout_minutes,
i32::from(endpoint_port),
&generation,
&runtime,
) {
Ok(preferences) => {
self.preferences = preferences;
#[cfg(target_os = "macos")]
if let Ok(mut runtime_preferences) = self.runtime_preferences.write() {
*runtime_preferences = self.preferences.clone();
}
#[cfg(target_os = "macos")]
if let Some(endpoint) = pending_endpoint {
self._endpoint = Some(endpoint);
}
self.preference_draft = PreferenceDraft::from_saved(&self.preferences)
.expect("the saved model was selected from the supported catalog");
self.preferences_open = false;
@@ -1508,15 +1584,10 @@ impl App {
#[cfg(target_os = "macos")]
{
if self.generation_worker.is_none() {
match spawn_generation_worker() {
Ok(worker) => self.generation_worker = Some(worker),
Err(error) => {
self.error = Some(error);
return;
}
}
}
let Some(service) = &self.generation_service else {
self.error = Some("The model runtime is unavailable.".into());
return;
};
let Some(database) = &mut self.database else {
return;
};
@@ -1527,24 +1598,22 @@ impl App {
return;
}
};
let cancel = Arc::new(AtomicBool::new(false));
let idle_timeout =
Duration::from_secs(self.preferences.idle_timeout_minutes.max(1) as u64 * 60);
let command = GenerationCommand::Generate {
engine: effective.engine,
turn: effective.turn,
self.active_generation = match service.generate(
effective.engine,
effective.turn,
messages,
checkpoint: session_checkpoint_path(session_id),
CheckpointTarget::Local(session_checkpoint_path(session_id)),
idle_timeout,
cancel: Arc::clone(&cancel),
) {
Ok(active) => Some(active),
Err(error) => {
self.generation_service = None;
self.error = Some(error);
return;
}
};
let worker = self.generation_worker.as_mut().expect("worker was created");
if worker.commands.send(command).is_err() {
self.generation_worker = None;
self.error = Some("The local generation worker stopped unexpectedly.".into());
return;
}
worker.cancel = Some(cancel);
let user = ChatMessage::from(saved.0);
let mut assistant = ChatMessage::from(saved.1);
assistant.reasoning_open = assistant_reasoning;
@@ -1565,7 +1634,7 @@ impl App {
fn poll_generation(&mut self) -> bool {
#[cfg(target_os = "macos")]
let Some(worker) = &mut self.generation_worker else {
let Some(active) = &mut self.active_generation else {
self.generating = false;
return false;
};
@@ -1575,7 +1644,7 @@ impl App {
let mut context_changed = false;
#[cfg(target_os = "macos")]
loop {
match worker.events.try_recv() {
match active.events.try_recv() {
Ok(GenerationEvent::Loading) => {}
Ok(GenerationEvent::Chunk { reasoning, content }) => {
if let Some(message) = self.conversation.last_mut()
@@ -1597,17 +1666,17 @@ impl App {
}
Ok(GenerationEvent::Finished(result)) => {
self.generating = false;
worker.cancel = None;
if let Err(error) = result {
self.error = Some(error);
}
self.active_generation = None;
break;
}
Err(TryRecvError::Empty) => break,
Err(TryRecvError::Disconnected) => {
self.generating = false;
self.generation_worker = None;
self.error = Some("The local generation worker stopped unexpectedly.".into());
self.active_generation = None;
self.error = Some("The model runtime stopped unexpectedly.".into());
break;
}
}
@@ -1627,12 +1696,8 @@ impl App {
&message.content,
)
{
if let Some(cancel) = self
.generation_worker
.as_ref()
.and_then(|worker| worker.cancel.as_ref())
{
cancel.store(true, Ordering::Relaxed);
if let Some(active) = &self.active_generation {
active.cancel.store(true, Ordering::Relaxed);
}
self.error = Some(format!("Could not save generated chat text: {error}"));
}
@@ -1667,77 +1732,30 @@ impl App {
}
#[cfg(target_os = "macos")]
fn spawn_generation_worker() -> Result<GenerationWorker, String> {
let (command_sender, command_receiver) = mpsc::channel();
let (event_sender, event_receiver) = mpsc::channel();
thread::Builder::new()
.name("local-generation".into())
.spawn(move || {
let mut loaded = None::<(crate::settings::EngineSettings, Generator)>;
let mut last_used = Instant::now();
let mut idle_timeout = Duration::from_secs(15 * 60);
loop {
match command_receiver.recv_timeout(Duration::from_secs(1)) {
Ok(GenerationCommand::Generate {
engine,
turn,
messages,
checkpoint,
idle_timeout: requested_timeout,
cancel,
}) => {
idle_timeout = requested_timeout;
if loaded
.as_ref()
.is_none_or(|(current, _)| current != &engine)
{
let _ = event_sender.send(GenerationEvent::Loading);
loaded = match Generator::open(&engine) {
Ok(generator) => Some((engine.clone(), generator)),
Err(error) => {
let _ =
event_sender.send(GenerationEvent::Finished(Err(error)));
None
}
};
}
if let Some((_, generator)) = &mut loaded {
let result = generator.generate(
&checkpoint,
&messages,
&turn,
&cancel,
|reasoning, content| {
let _ = event_sender
.send(GenerationEvent::Chunk { reasoning, content });
},
|used, limit, tokens_per_second| {
let _ = event_sender.send(GenerationEvent::Context {
used,
limit,
tokens_per_second,
});
},
);
let _ = event_sender.send(GenerationEvent::Finished(result));
last_used = Instant::now();
}
}
Err(mpsc::RecvTimeoutError::Timeout) => {
if loaded.is_some() && last_used.elapsed() >= idle_timeout {
loaded = None;
}
}
Err(mpsc::RecvTimeoutError::Disconnected) => break,
}
}
})
.map_err(|error| format!("Could not start local generation: {error}"))?;
Ok(GenerationWorker {
commands: command_sender,
events: event_receiver,
cancel: None,
})
fn spawn_services(
preferences: &AppPreferences,
) -> (
Arc<RwLock<AppPreferences>>,
Option<GenerationService>,
Option<crate::server::ServerHandle>,
Option<String>,
) {
let runtime_preferences = Arc::new(RwLock::new(preferences.clone()));
let generation = match GenerationService::spawn() {
Ok(generation) => generation,
Err(error) => return (runtime_preferences, None, None, Some(error)),
};
let endpoint = crate::server::ServerHandle::spawn(
generation.clone(),
Arc::clone(&runtime_preferences),
models_path(),
application_support_path().join("kv-cache").join("http"),
u16::try_from(preferences.endpoint_port).unwrap_or(4000),
);
match endpoint {
Ok(endpoint) => (runtime_preferences, Some(generation), Some(endpoint), None),
Err(error) => (runtime_preferences, Some(generation), None, Some(error)),
}
}
impl Drop for App {
@@ -1746,12 +1764,8 @@ impl Drop for App {
download.cancel.store(true, Ordering::Relaxed);
}
#[cfg(target_os = "macos")]
if let Some(cancel) = self
.generation_worker
.as_ref()
.and_then(|worker| worker.cancel.as_ref())
{
cancel.store(true, Ordering::Relaxed);
if let Some(active) = &self.active_generation {
active.cancel.store(true, Ordering::Relaxed);
}
}
}

629
src/app/view.rs
View File

@@ -31,7 +31,12 @@ impl App {
if self.model_manager_window == Some(id) {
self.model_manager()
} else {
self.main_view()
let content = self.main_view();
#[cfg(target_os = "macos")]
return crate::native_edit::native_edit(content, self.native_edit_commands.clone())
.into();
#[cfg(not(target_os = "macos"))]
content
}
}
@@ -457,286 +462,331 @@ impl App {
dspark_confidence.on_input(Message::PreferenceDsparkConfidenceChanged);
}
let mut fields = column![
text("MODEL").size(11),
pick_list(
&MODEL_CHOICES[..],
Some(self.preference_draft.model),
Message::PreferenceModelChanged,
)
.width(Length::Fill),
text(format!(
"Main: {}{}",
engine.map_or_else(
|| "Invalid settings".to_owned(),
|engine| engine.artifacts.model.display().to_string(),
),
engine
.and_then(|engine| engine.artifacts.mtp.as_ref())
.map_or_else(String::new, |path| format!(" • support: {}", path.display())),
))
.size(12),
Space::with_height(8),
text("EXECUTION").size(11),
preference_input_row(
"CPU helper threads",
text_input("Automatic", &self.preference_draft.cpu_threads)
.on_input(Message::PreferenceCpuThreadsChanged),
),
preference_input_row("GPU power percent", power),
preference_input_row("Prefill chunk", prefill),
checkbox("Prefer exact quality kernels", self.preference_draft.quality)
.on_toggle(Message::PreferenceQualityChanged),
checkbox("Warm mapped weights at load time", self.preference_draft.warm_weights)
.on_toggle(Message::PreferenceWarmWeightsChanged),
text(if self.preference_draft.model == ModelChoice::Glm52 {
"GLM 5.2 uses full GPU power and selects prefill chunks automatically."
} else {
"Blank numeric values preserve DS4's automatic engine behavior."
})
.size(12),
text(engine.as_ref().map_or_else(
|| "Effective execution settings will appear after valid values are entered."
.to_owned(),
|engine| {
let settings = engine.execution;
format!(
"Metal engine: threads {} • power {}% • prefill {} • quality {} • warm weights {}",
if settings.cpu_threads == 0 { "auto".to_owned() } else { settings.cpu_threads.to_string() },
if settings.power_percent == 0 { 100 } else { settings.power_percent },
if settings.prefill_chunk == 0 { "auto".to_owned() } else { settings.prefill_chunk.to_string() },
if settings.quality { "on" } else { "off" },
if settings.warm_weights { "on" } else { "off" },
)},
))
.size(12),
Space::with_height(8),
text("SPECULATIVE DECODING").size(11),
preference_input_row(
"MTP draft tokens",
text_input("1", &self.preference_draft.mtp_draft_tokens)
.on_input(Message::PreferenceMtpDraftChanged),
),
preference_input_row(
"MTP verifier margin",
text_input("3", &self.preference_draft.mtp_margin)
.on_input(Message::PreferenceMtpMarginChanged),
),
checkbox("Enable integrated GLM MTP", self.preference_draft.glm_mtp)
.on_toggle_maybe(glm_mtp_toggle),
checkbox(
"Log GLM MTP timing counters",
self.preference_draft.glm_mtp_timing,
)
.on_toggle_maybe(glm_mtp_timing_toggle),
dspark,
preference_input_row("DSpark confidence threshold", dspark_confidence),
checkbox(
"DSpark target-only decode",
self.preference_draft.dspark_strict,
)
.on_toggle_maybe(dspark_strict_toggle),
text(if self.preference_draft.model.supports_dspark() {
"DSpark uses the managed support artifact; entering a threshold or enabling strict mode also enables DSpark."
} else if self.preference_draft.model == ModelChoice::Glm52 {
"GLM MTP is integrated; DSpark is unavailable for this model."
} else {
"No managed MTP support artifact is available for this model."
})
.size(12),
text(engine.as_ref().map_or_else(
|| "Effective speculative settings will appear after valid values are entered."
.to_owned(),
|engine| {
let settings = engine.speculative;
format!(
"Engine: MTP draft {} • margin {} • GLM MTP {} • timing {} • DSpark {} • confidence {}{} • target-only {}",
settings.mtp_draft_tokens,
settings.mtp_margin,
if settings.glm_mtp { "on" } else { "off" },
if settings.glm_mtp_timing { "on" } else { "off" },
if settings.dspark { "on" } else { "off" },
settings.dspark_confidence_threshold,
if settings.dspark_confidence_threshold_set { " explicit" } else { " default" },
if settings.dspark_strict { "on" } else { "off" },
)},
))
.size(12),
Space::with_height(8),
text("SSD STREAMING").size(11),
checkbox("Enable SSD-backed model streaming", self.preference_draft.ssd_streaming)
.on_toggle(Message::PreferenceSsdChanged),
checkbox("Skip automatic expert preload", self.preference_draft.ssd_streaming_cold)
.on_toggle(Message::PreferenceSsdColdChanged),
preference_input_row(
"Expert cache count or GiB",
text_input("Automatic, 128, or 64GB", &self.preference_draft.ssd_cache)
.on_input(Message::PreferenceSsdCacheChanged),
),
preference_input_row("Fully resident GLM layers", ssd_full_layers),
preference_input_row(
"Explicit expert preload count",
text_input("Automatic", &self.preference_draft.ssd_preload_experts)
.on_input(Message::PreferenceSsdPreloadChanged),
),
text("A blank full-layer value is automatic; an explicit 0 disables fully resident GLM layers. SSD streaming and DSpark are mutually exclusive.")
.size(12),
text(engine.as_ref().map_or_else(
|| "Effective SSD settings will appear after valid values are entered."
.to_owned(),
|engine| {
let settings = engine.ssd;
let cache = if settings.cache_bytes > 0 {
format!("{} GiB", settings.cache_bytes / GIB)
} else if settings.cache_experts > 0 {
format!("{} experts", settings.cache_experts)
} else {
"auto".to_owned()
};
format!(
"Engine: streaming {} • cold {} • cache {} • full layers {}{} • preload {}",
if settings.enabled { "on" } else { "off" },
if settings.cold { "on" } else { "off" },
cache,
settings.full_layers,
if settings.full_layers_set { " explicit" } else { " auto" },
if settings.preload_experts == 0 { "auto".to_owned() } else { settings.preload_experts.to_string() },
)
},
))
.size(12),
Space::with_height(8),
text("DIRECTIONAL STEERING").size(11),
text("Direction-vector file").size(13),
steering_file.padding(9),
preference_input_row("FFN scale", steering_ffn),
preference_input_row("Attention scale", steering_attn),
text(if self.preference_draft.model == ModelChoice::Glm52 {
"Directional steering is not supported for GLM 5.2."
} else {
"With a file and no explicit scale, DS4 defaults the FFN scale to 1. Scales accept -100 through 100."
})
.size(12),
text(engine.as_ref().map_or_else(
|| "Effective steering settings will appear after valid values are entered."
.to_owned(),
|engine| format!(
"Engine: file {} • FFN scale {} • attention scale {}",
if engine.steering.file.is_some() { "set" } else { "off" },
engine.steering.ffn_scale,
engine.steering.attention_scale,
),
))
.size(12),
Space::with_height(8),
text("CAPACITY").size(11),
preference_input_row(
"Context tokens",
text_input("32768", &self.preference_draft.context_tokens)
.on_input(Message::PreferenceContextChanged),
),
preference_input_row(
"Maximum generated tokens",
text_input("50000", &self.preference_draft.max_generated_tokens)
.on_input(Message::PreferenceMaxTokensChanged),
),
text("System prompt").size(13),
text_input(
"You are a helpful assistant",
&self.preference_draft.system_prompt,
)
.on_input(Message::PreferenceSystemPromptChanged)
.padding(9),
Space::with_height(8),
text("SAMPLING AND REASONING").size(11),
preference_input_row(
"Temperature",
text_input("DS4 default", &self.preference_draft.temperature)
.on_input(Message::PreferenceTemperatureChanged),
),
preference_input_row(
"Top-p",
text_input("DS4 default", &self.preference_draft.top_p)
.on_input(Message::PreferenceTopPChanged),
),
preference_input_row(
"Min-p",
text_input("DS4 default", &self.preference_draft.min_p)
.on_input(Message::PreferenceMinPChanged),
),
preference_input_row(
"Seed",
text_input("Random", &self.preference_draft.seed)
.on_input(Message::PreferenceSeedChanged),
),
row![
text("Reasoning").size(13).width(Length::Fill),
let model_group = preference_group(
"MODEL & LIFECYCLE",
column![
pick_list(
&REASONING_MODES[..],
Some(self.preference_draft.reasoning_mode),
Message::PreferenceReasoningChanged,
&MODEL_CHOICES[..],
Some(self.preference_draft.model),
Message::PreferenceModelChanged,
)
.width(240),
]
.spacing(12)
.align_y(Alignment::Center),
text("Blank sampling values retain DS4's model-family defaults. Think Max needs at least 393216 context tokens.")
.width(Length::Fill),
text(format!(
"Main: {}{}",
engine.map_or_else(
|| "Invalid settings".to_owned(),
|engine| engine.artifacts.model.display().to_string(),
),
engine
.and_then(|engine| engine.artifacts.mtp.as_ref())
.map_or_else(String::new, |path| format!(
" • support: {}",
path.display()
)),
))
.size(12),
text(turn.map_or_else(
|| "Effective settings will appear after valid values are entered.".to_owned(),
|settings| format!(
"Effective: {} context • {} max • temp {} • top-p {} • min-p {} • seed {} • {} • system prompt {}",
settings.context_tokens,
settings.max_generated_tokens,
settings.temperature,
settings.top_p,
settings.min_p,
settings.seed.map_or_else(|| "random".to_owned(), |seed| seed.to_string()),
settings.reasoning_mode,
if settings.system_prompt.is_empty() { "off" } else { "on" },
),
))
.size(12),
Space::with_height(8),
text("ADVANCED DIAGNOSTICS").size(11),
preference_input_row(
"Simulated used memory (GiB)",
text_input("Disabled", &self.preference_draft.simulated_used_memory_gib)
.on_input(Message::PreferenceSimulatedMemoryChanged),
),
text("Routed expert profile output").size(13),
text_input("Output file path", &self.preference_draft.expert_profile_path)
.on_input(Message::PreferenceExpertProfileChanged)
.padding(9),
text(engine.as_ref().map_or_else(
|| "Effective diagnostic settings will appear after valid values are entered."
.to_owned(),
|engine| format!(
"{} load: simulated memory {} • expert profile {}",
engine.model,
if engine.diagnostics.simulated_used_memory_bytes == 0 {
"off".to_owned()
} else {
format!("{} GiB", engine.diagnostics.simulated_used_memory_bytes / GIB)
},
if engine.diagnostics.expert_profile_path.is_some() { "set" } else { "off" },
),
))
.size(12),
Space::with_height(8),
text("INACTIVITY").size(11),
row![
text_input("10", &self.preference_draft.idle_timeout_minutes)
.on_input(Message::PreferenceTimeoutChanged)
.width(90)
.padding(9),
text("minutes before unloading the model").size(13),
row![
text_input("10", &self.preference_draft.idle_timeout_minutes)
.on_input(Message::PreferenceTimeoutChanged)
.width(90)
.padding(9),
text("minutes before unloading the model").size(13),
]
.spacing(10)
.align_y(Alignment::Center),
text("Enter a whole number from 1 to 1440.").size(12),
]
.spacing(10)
.align_y(Alignment::Center),
text("Enter a whole number from 1 to 1440.").size(12),
.spacing(10),
);
let endpoint_group = preference_group(
"LOCAL ENDPOINT",
column![
preference_input_row(
"Port",
text_input("4000", &self.preference_draft.endpoint_port)
.on_input(Message::PreferenceEndpointPortChanged),
),
text("Listens on 127.0.0.1. Saving a changed port restarts the local endpoint.")
.size(12),
]
.spacing(10),
);
let generation_group = preference_group(
"GENERATION",
column![
preference_input_row(
"Context tokens",
text_input("32768", &self.preference_draft.context_tokens)
.on_input(Message::PreferenceContextChanged),
),
preference_input_row(
"Maximum generated tokens",
text_input("50000", &self.preference_draft.max_generated_tokens)
.on_input(Message::PreferenceMaxTokensChanged),
),
text("System prompt").size(13),
text_input(
"You are a helpful assistant",
&self.preference_draft.system_prompt,
)
.on_input(Message::PreferenceSystemPromptChanged)
.padding(9),
Space::with_height(4),
text("SAMPLING & REASONING").size(11).color(muted_text()),
preference_input_row(
"Temperature",
text_input("DS4 default", &self.preference_draft.temperature)
.on_input(Message::PreferenceTemperatureChanged),
),
preference_input_row(
"Top-p",
text_input("DS4 default", &self.preference_draft.top_p)
.on_input(Message::PreferenceTopPChanged),
),
preference_input_row(
"Min-p",
text_input("DS4 default", &self.preference_draft.min_p)
.on_input(Message::PreferenceMinPChanged),
),
preference_input_row(
"Seed",
text_input("Random", &self.preference_draft.seed)
.on_input(Message::PreferenceSeedChanged),
),
row![
text("Reasoning").size(13).width(Length::Fill),
pick_list(
&REASONING_MODES[..],
Some(self.preference_draft.reasoning_mode),
Message::PreferenceReasoningChanged,
)
.width(240),
]
.spacing(12)
.align_y(Alignment::Center),
text("Blank sampling values retain DS4's model-family defaults. Think Max needs at least 393216 context tokens.")
.size(12),
text(turn.map_or_else(
|| "Effective settings will appear after valid values are entered.".to_owned(),
|settings| format!(
"Effective: {} context • {} max • temp {} • top-p {} • min-p {} • seed {} • {} • system prompt {}",
settings.context_tokens,
settings.max_generated_tokens,
settings.temperature,
settings.top_p,
settings.min_p,
settings.seed.map_or_else(|| "random".to_owned(), |seed| seed.to_string()),
settings.reasoning_mode,
if settings.system_prompt.is_empty() { "off" } else { "on" },
),
))
.size(12),
]
.spacing(10),
);
let execution_group = preference_group(
"EXECUTION",
column![
preference_input_row(
"CPU helper threads",
text_input("Automatic", &self.preference_draft.cpu_threads)
.on_input(Message::PreferenceCpuThreadsChanged),
),
preference_input_row("GPU power percent", power),
preference_input_row("Prefill chunk", prefill),
checkbox("Prefer exact quality kernels", self.preference_draft.quality)
.on_toggle(Message::PreferenceQualityChanged),
checkbox("Warm mapped weights at load time", self.preference_draft.warm_weights)
.on_toggle(Message::PreferenceWarmWeightsChanged),
text(if self.preference_draft.model == ModelChoice::Glm52 {
"GLM 5.2 uses full GPU power and selects prefill chunks automatically."
} else {
"Blank numeric values preserve DS4's automatic engine behavior."
})
.size(12),
text(engine.as_ref().map_or_else(
|| "Effective execution settings will appear after valid values are entered."
.to_owned(),
|engine| {
let settings = engine.execution;
format!(
"Metal engine: threads {} • power {}% • prefill {} • quality {} • warm weights {}",
if settings.cpu_threads == 0 { "auto".to_owned() } else { settings.cpu_threads.to_string() },
if settings.power_percent == 0 { 100 } else { settings.power_percent },
if settings.prefill_chunk == 0 { "auto".to_owned() } else { settings.prefill_chunk.to_string() },
if settings.quality { "on" } else { "off" },
if settings.warm_weights { "on" } else { "off" },
)
},
))
.size(12),
]
.spacing(10),
);
let acceleration_group = preference_group(
"ACCELERATION & MEMORY",
column![
text("SPECULATIVE DECODING").size(11).color(muted_text()),
preference_input_row(
"MTP draft tokens",
text_input("1", &self.preference_draft.mtp_draft_tokens)
.on_input(Message::PreferenceMtpDraftChanged),
),
preference_input_row(
"MTP verifier margin",
text_input("3", &self.preference_draft.mtp_margin)
.on_input(Message::PreferenceMtpMarginChanged),
),
checkbox("Enable integrated GLM MTP", self.preference_draft.glm_mtp)
.on_toggle_maybe(glm_mtp_toggle),
checkbox(
"Log GLM MTP timing counters",
self.preference_draft.glm_mtp_timing,
)
.on_toggle_maybe(glm_mtp_timing_toggle),
dspark,
preference_input_row("DSpark confidence threshold", dspark_confidence),
checkbox(
"DSpark target-only decode",
self.preference_draft.dspark_strict,
)
.on_toggle_maybe(dspark_strict_toggle),
text(if self.preference_draft.model.supports_dspark() {
"DSpark uses the managed support artifact; entering a threshold or enabling strict mode also enables DSpark."
} else if self.preference_draft.model == ModelChoice::Glm52 {
"GLM MTP is integrated; DSpark is unavailable for this model."
} else {
"No managed MTP support artifact is available for this model."
})
.size(12),
text(engine.as_ref().map_or_else(
|| "Effective speculative settings will appear after valid values are entered."
.to_owned(),
|engine| {
let settings = engine.speculative;
format!(
"Engine: MTP draft {} • margin {} • GLM MTP {} • timing {} • DSpark {} • confidence {}{} • target-only {}",
settings.mtp_draft_tokens,
settings.mtp_margin,
if settings.glm_mtp { "on" } else { "off" },
if settings.glm_mtp_timing { "on" } else { "off" },
if settings.dspark { "on" } else { "off" },
settings.dspark_confidence_threshold,
if settings.dspark_confidence_threshold_set { " explicit" } else { " default" },
if settings.dspark_strict { "on" } else { "off" },
)
},
))
.size(12),
Space::with_height(6),
text("SSD STREAMING").size(11).color(muted_text()),
checkbox("Enable SSD-backed model streaming", self.preference_draft.ssd_streaming)
.on_toggle(Message::PreferenceSsdChanged),
checkbox("Skip automatic expert preload", self.preference_draft.ssd_streaming_cold)
.on_toggle(Message::PreferenceSsdColdChanged),
preference_input_row(
"Expert cache count or GiB",
text_input("Automatic, 128, or 64GB", &self.preference_draft.ssd_cache)
.on_input(Message::PreferenceSsdCacheChanged),
),
preference_input_row("Fully resident GLM layers", ssd_full_layers),
preference_input_row(
"Explicit expert preload count",
text_input("Automatic", &self.preference_draft.ssd_preload_experts)
.on_input(Message::PreferenceSsdPreloadChanged),
),
text("A blank full-layer value is automatic; an explicit 0 disables fully resident GLM layers. SSD streaming and DSpark are mutually exclusive.")
.size(12),
text(engine.as_ref().map_or_else(
|| "Effective SSD settings will appear after valid values are entered."
.to_owned(),
|engine| {
let settings = engine.ssd;
let cache = if settings.cache_bytes > 0 {
format!("{} GiB", settings.cache_bytes / GIB)
} else if settings.cache_experts > 0 {
format!("{} experts", settings.cache_experts)
} else {
"auto".to_owned()
};
format!(
"Engine: streaming {} • cold {} • cache {} • full layers {}{} • preload {}",
if settings.enabled { "on" } else { "off" },
if settings.cold { "on" } else { "off" },
cache,
settings.full_layers,
if settings.full_layers_set { " explicit" } else { " auto" },
if settings.preload_experts == 0 { "auto".to_owned() } else { settings.preload_experts.to_string() },
)
},
))
.size(12),
]
.spacing(10),
);
let steering_group = preference_group(
"STEERING & DIAGNOSTICS",
column![
text("DIRECTIONAL STEERING").size(11).color(muted_text()),
text("Direction-vector file").size(13),
steering_file.padding(9),
preference_input_row("FFN scale", steering_ffn),
preference_input_row("Attention scale", steering_attn),
text(if self.preference_draft.model == ModelChoice::Glm52 {
"Directional steering is not supported for GLM 5.2."
} else {
"With a file and no explicit scale, DS4 defaults the FFN scale to 1. Scales accept -100 through 100."
})
.size(12),
text(engine.as_ref().map_or_else(
|| "Effective steering settings will appear after valid values are entered."
.to_owned(),
|engine| format!(
"Engine: file {} • FFN scale {} • attention scale {}",
if engine.steering.file.is_some() { "set" } else { "off" },
engine.steering.ffn_scale,
engine.steering.attention_scale,
),
))
.size(12),
Space::with_height(6),
text("ADVANCED DIAGNOSTICS").size(11).color(muted_text()),
preference_input_row(
"Simulated used memory (GiB)",
text_input("Disabled", &self.preference_draft.simulated_used_memory_gib)
.on_input(Message::PreferenceSimulatedMemoryChanged),
),
text("Routed expert profile output").size(13),
text_input("Output file path", &self.preference_draft.expert_profile_path)
.on_input(Message::PreferenceExpertProfileChanged)
.padding(9),
text(engine.as_ref().map_or_else(
|| "Effective diagnostic settings will appear after valid values are entered."
.to_owned(),
|engine| format!(
"{} load: simulated memory {} • expert profile {}",
engine.model,
if engine.diagnostics.simulated_used_memory_bytes == 0 {
"off".to_owned()
} else {
format!("{} GiB", engine.diagnostics.simulated_used_memory_bytes / GIB)
},
if engine.diagnostics.expert_profile_path.is_some() { "set" } else { "off" },
),
))
.size(12),
]
.spacing(10),
);
let mut fields = column![
model_group,
endpoint_group,
generation_group,
execution_group,
acceleration_group,
steering_group,
]
.spacing(10);
.spacing(12);
if let Some(error) = &self.preference_error {
fields = fields.push(text(error).style(iced::widget::text::danger));
@@ -971,6 +1021,17 @@ fn preference_input_row<'a>(
.into()
}
fn preference_group<'a>(
title: &'a str,
content: impl Into<Element<'a, Message>>,
) -> Element<'a, Message> {
container(column![text(title).size(11).color(muted_text()), content.into(),].spacing(10))
.width(Length::Fill)
.padding(14)
.style(preference_group_style)
.into()
}
fn download_status_bar(download: &ActiveDownload) -> Element<'_, Message> {
let progress = &download.progress;
let percent = progress.fraction() * 100.0;
@@ -1201,6 +1262,18 @@ fn overview_style(_: &Theme) -> container::Style {
}
}
fn preference_group_style(_: &Theme) -> container::Style {
container::Style {
background: Some(Background::Color(Color::from_rgb8(38, 38, 40))),
border: Border {
color: Color::from_rgb8(58, 58, 61),
width: 1.0,
radius: 14.0.into(),
},
..container::Style::default()
}
}
fn chat_message_style(theme: &Theme, user: bool) -> container::Style {
if !user {
return overview_style(theme);
@@ -1247,6 +1320,10 @@ mod tests {
chat_message_style(&theme, true).background,
chat_message_style(&theme, false).background
);
assert_eq!(
preference_group_style(&theme).background,
Some(Background::Color(Color::from_rgb8(38, 38, 40)))
);
}
#[test]

15
src/database.rs
View File

@@ -51,6 +51,7 @@ pub struct AppPreferences {
pub directional_steering_attn: Option<f32>,
pub simulated_used_memory_gib: Option<i64>,
pub expert_profile_path: Option<String>,
pub endpoint_port: i32,
}
impl Default for AppPreferences {
@@ -90,6 +91,7 @@ impl Default for AppPreferences {
directional_steering_attn: None,
simulated_used_memory_gib: None,
expert_profile_path: None,
endpoint_port: 4000,
}
}
}
@@ -234,6 +236,7 @@ struct PreferenceChanges<'a> {
directional_steering_attn: Option<f32>,
simulated_used_memory_gib: Option<i64>,
expert_profile_path: Option<&'a str>,
endpoint_port: i32,
}
#[derive(Clone, Debug, Identifiable, Queryable, Selectable)]
@@ -360,6 +363,7 @@ impl Database {
&mut self,
selected_model: &str,
idle_timeout_minutes: i32,
endpoint_port: i32,
generation: &GenerationPreferences,
runtime: &RuntimePreferences,
) -> Result<AppPreferences, String> {
@@ -367,6 +371,9 @@ impl Database {
let model = crate::model::ModelChoice::from_id(selected_model)
.ok_or_else(|| format!("Unsupported model: {selected_model}"))?;
runtime.validate(model)?;
if !(1..=65_535).contains(&endpoint_port) {
return Err("Endpoint port must be between 1 and 65535.".into());
}
let execution = &runtime.execution;
let speculative = &runtime.speculative;
let (ssd_cache_experts, ssd_cache_gib) = match runtime.ssd.cache {
@@ -416,6 +423,7 @@ impl Database {
.simulated_used_memory_gib
.map(|value| value as i64),
expert_profile_path: runtime.diagnostics.expert_profile_path.as_deref(),
endpoint_port,
})
.returning(AppPreferences::as_returning())
.get_result(&mut self.connection)
@@ -572,6 +580,7 @@ mod tests {
assert_eq!(preferences.selected_model, "deepseek-v4-flash");
assert!(!preferences.dspark_enabled);
assert_eq!(preferences.idle_timeout_minutes, 10);
assert_eq!(preferences.endpoint_port, 4000);
let generation = GenerationPreferences::default();
let runtime = RuntimePreferences::default();
assert!(
@@ -579,6 +588,7 @@ mod tests {
.update_preferences(
"glm-5.2",
30,
4000,
&generation,
&RuntimePreferences {
speculative: SpeculativePreferences {
@@ -592,7 +602,7 @@ mod tests {
);
assert!(
database
.update_preferences("deepseek-v4-flash", 0, &generation, &runtime,)
.update_preferences("deepseek-v4-flash", 0, 4000, &generation, &runtime,)
.is_err()
);
let generation = GenerationPreferences {
@@ -630,7 +640,7 @@ mod tests {
..RuntimePreferences::default()
};
database
.update_preferences("glm-5.2", 30, &generation, &runtime)
.update_preferences("glm-5.2", 30, 4567, &generation, &runtime)
.unwrap();
let project = database.create_project("DS4", "/tmp/ds4").unwrap();
@@ -652,6 +662,7 @@ mod tests {
let preferences = reopened.load_preferences().unwrap();
assert_eq!(preferences.selected_model, "glm-5.2");
assert_eq!(preferences.idle_timeout_minutes, 30);
assert_eq!(preferences.endpoint_port, 4567);
assert_eq!(preferences.generation().unwrap(), generation);
assert_eq!(preferences.runtime().unwrap(), runtime);
drop(reopened);

314
src/engine.rs
View File

@@ -305,6 +305,14 @@ pub(crate) struct ChatTurn {
pub(crate) content: String,
}
pub(crate) struct GenerationOutput {
pub(crate) message: ChatTurn,
pub(crate) prompt_tokens: u32,
pub(crate) cached_tokens: u32,
pub(crate) completion_tokens: u32,
pub(crate) finish_reason: &'static str,
}
#[cfg(target_os = "macos")]
impl Generator {
pub(crate) fn open(settings: &EngineSettings) -> Result<Self, String> {
@@ -337,12 +345,12 @@ impl Generator {
cancelled: &AtomicBool,
mut emit: impl FnMut(bool, String),
mut progress: impl FnMut(u32, u32, Option<f32>),
) -> Result<(), String> {
) -> Result<GenerationOutput, String> {
self.select_checkpoint(checkpoint)?;
let (generated, prompt_complete) =
let (output, prompt_complete) =
self.generate_inner(messages, settings, cancelled, &mut emit, &mut progress)?;
let mut completed = messages.to_vec();
completed.push(generated);
completed.push(output.message.clone());
self.executor.save_checkpoint(
checkpoint,
if prompt_complete {
@@ -350,7 +358,57 @@ impl Generator {
} else {
[0; 32]
},
)
)?;
Ok(output)
}
pub(crate) fn generate_transient(
&mut self,
directory: &Path,
messages: &[ChatTurn],
settings: &TurnSettings,
cancelled: &AtomicBool,
mut emit: impl FnMut(bool, String),
mut progress: impl FnMut(u32, u32, Option<f32>),
) -> Result<GenerationOutput, String> {
let history = messages
.split_last()
.map_or(messages, |(_, history)| history);
let history_tag =
conversation_tag(&settings.system_prompt, settings.reasoning_mode, history);
let checkpoint = directory.join(format!("{}.bin", hex_tag(history_tag)));
if self.executor.checkpoint_tag() != history_tag {
self.select_checkpoint(&checkpoint)?;
if self.executor.checkpoint_tag() != history_tag {
self.executor.reset()?;
self.checkpoint = None;
let _ = std::fs::remove_file(&checkpoint);
}
}
let (output, prompt_complete) =
self.generate_inner(messages, settings, cancelled, &mut emit, &mut progress)?;
let mut completed = messages.to_vec();
completed.push(output.message.clone());
let completed_tag =
conversation_tag(&settings.system_prompt, settings.reasoning_mode, &completed);
std::fs::create_dir_all(directory).map_err(|error| {
format!(
"Could not create transient KV cache directory {}: {error}",
directory.display()
)
})?;
let completed_checkpoint = directory.join(format!("{}.bin", hex_tag(completed_tag)));
self.executor.save_checkpoint(
&completed_checkpoint,
if prompt_complete {
completed_tag
} else {
[0; 32]
},
)?;
self.checkpoint = Some(completed_checkpoint);
Ok(output)
}
fn select_checkpoint(&mut self, checkpoint: &Path) -> Result<(), String> {
@@ -373,7 +431,7 @@ impl Generator {
cancelled: &AtomicBool,
emit: &mut impl FnMut(bool, String),
progress: &mut impl FnMut(u32, u32, Option<f32>),
) -> Result<(ChatTurn, bool), String> {
) -> Result<(GenerationOutput, bool), String> {
let tokens = match messages.split_last() {
Some((latest, history))
if latest.user
@@ -418,10 +476,21 @@ impl Generator {
reasoning_complete: !reasoning,
content: String::new(),
};
let mut emitted_reasoning = 0;
let mut emitted_content = 0;
let prompt_tokens = tokens.len();
for (index, token) in tokens.into_iter().enumerate().skip(reused) {
if cancelled.load(Ordering::Relaxed) {
return Ok((generated, false));
return Ok((
GenerationOutput {
message: generated,
prompt_tokens: prompt_tokens as u32,
cached_tokens: reused as u32,
completion_tokens: 0,
finish_reason: "stop",
},
false,
));
}
self.executor.eval(token)?;
if (index + 1).is_multiple_of(16) || index + 1 == prompt_tokens {
@@ -436,13 +505,30 @@ impl Generator {
.min((max_context - self.executor.position() as usize) as i32)
{
if cancelled.load(Ordering::Relaxed) {
return Ok((generated, true));
flush_generated(
&mut generated,
&mut emitted_reasoning,
&mut emitted_content,
&settings.stops,
emit,
);
return Ok((
GenerationOutput {
message: generated,
prompt_tokens: prompt_tokens as u32,
cached_tokens: reused as u32,
completion_tokens: generated_tokens,
finish_reason: "stop",
},
true,
));
}
let token = sample(
self.executor.logits(),
settings.temperature,
settings.top_p,
settings.min_p,
settings.top_k,
&mut rng,
);
if self
@@ -450,27 +536,89 @@ impl Generator {
.model()
.is_stop_token_for_reasoning(token, settings.reasoning_mode)
{
return Ok((generated, true));
flush_generated(
&mut generated,
&mut emitted_reasoning,
&mut emitted_content,
&settings.stops,
emit,
);
return Ok((
GenerationOutput {
message: generated,
prompt_tokens: prompt_tokens as u32,
cached_tokens: reused as u32,
completion_tokens: generated_tokens,
finish_reason: "stop",
},
true,
));
}
if self.executor.model().is_think_start_token(token) {
reasoning = true;
generated.reasoning.get_or_insert_default();
} else if self.executor.model().is_think_end_token(token) {
if let Some(reasoning_text) = &mut generated.reasoning
&& emit_safe_text(
reasoning_text,
&mut emitted_reasoning,
&settings.stops,
true,
true,
emit,
)
{
return Ok((
GenerationOutput {
message: generated,
prompt_tokens: prompt_tokens as u32,
cached_tokens: reused as u32,
completion_tokens: generated_tokens + 1,
finish_reason: "stop",
},
false,
));
}
reasoning = false;
generated.reasoning_complete = true;
emit(false, String::new());
} else if let Some(bytes) = self.executor.model().token_bytes(token) {
let content = String::from_utf8_lossy(&bytes).into_owned();
if reasoning {
generated
.reasoning
.get_or_insert_default()
.push_str(&content);
let stopped = if reasoning {
let text = generated.reasoning.get_or_insert_default();
text.push_str(&content);
emit_safe_text(
text,
&mut emitted_reasoning,
&settings.stops,
false,
true,
emit,
)
} else {
generated.reasoning_complete = true;
generated.content.push_str(&content);
emit_safe_text(
&mut generated.content,
&mut emitted_content,
&settings.stops,
false,
false,
emit,
)
};
if stopped {
return Ok((
GenerationOutput {
message: generated,
prompt_tokens: prompt_tokens as u32,
cached_tokens: reused as u32,
completion_tokens: generated_tokens + 1,
finish_reason: "stop",
},
false,
));
}
emit(reasoning, content);
}
self.executor.eval(token)?;
generated_tokens += 1;
@@ -483,10 +631,103 @@ impl Generator {
),
);
}
Ok((generated, true))
flush_generated(
&mut generated,
&mut emitted_reasoning,
&mut emitted_content,
&settings.stops,
emit,
);
Ok((
GenerationOutput {
message: generated,
prompt_tokens: prompt_tokens as u32,
cached_tokens: reused as u32,
completion_tokens: generated_tokens,
finish_reason: "length",
},
true,
))
}
}
#[cfg(target_os = "macos")]
fn flush_generated(
generated: &mut ChatTurn,
emitted_reasoning: &mut usize,
emitted_content: &mut usize,
stops: &[String],
emit: &mut impl FnMut(bool, String),
) {
if let Some(reasoning) = &mut generated.reasoning {
let _ = emit_safe_text(reasoning, emitted_reasoning, stops, true, true, emit);
}
let _ = emit_safe_text(
&mut generated.content,
emitted_content,
stops,
true,
false,
emit,
);
}
#[cfg(target_os = "macos")]
fn emit_safe_text(
text: &mut String,
emitted: &mut usize,
stops: &[String],
final_flush: bool,
reasoning: bool,
emit: &mut impl FnMut(bool, String),
) -> bool {
let stop = stops
.iter()
.filter_map(|stop| {
text[*emitted..]
.find(stop)
.map(|position| *emitted + position)
})
.min();
if let Some(stop) = stop {
if stop > *emitted {
emit(reasoning, text[*emitted..stop].to_owned());
}
text.truncate(stop);
*emitted = stop;
return true;
}
let hold = if final_flush {
0
} else {
stops
.iter()
.map(|stop| stop.len().saturating_sub(1))
.max()
.unwrap_or(0)
};
let mut safe = text.len().saturating_sub(hold);
while safe > *emitted && !text.is_char_boundary(safe) {
safe -= 1;
}
if safe > *emitted {
emit(reasoning, text[*emitted..safe].to_owned());
*emitted = safe;
}
false
}
#[cfg(target_os = "macos")]
fn hex_tag(tag: [u8; 32]) -> String {
const HEX: &[u8; 16] = b"0123456789abcdef";
let mut output = String::with_capacity(64);
for byte in tag {
output.push(HEX[(byte >> 4) as usize] as char);
output.push(HEX[(byte & 0x0f) as usize] as char);
}
output
}
#[cfg(target_os = "macos")]
fn conversation_tag(system: &str, reasoning: ReasoningMode, messages: &[ChatTurn]) -> [u8; 32] {
fn text(hasher: &mut Sha256, value: &str) {
@@ -518,7 +759,14 @@ fn conversation_tag(system: &str, reasoning: ReasoningMode, messages: &[ChatTurn
}
#[cfg(target_os = "macos")]
fn sample(logits: &[f32], temperature: f32, top_p: f32, min_p: f32, rng: &mut Rng) -> i32 {
fn sample(
logits: &[f32],
temperature: f32,
top_p: f32,
min_p: f32,
top_k: i32,
rng: &mut Rng,
) -> i32 {
if temperature <= 0.0 {
return logits
.iter()
@@ -554,12 +802,16 @@ fn sample(logits: &[f32], temperature: f32, top_p: f32, min_p: f32, rng: &mut Rn
.max_by(|a, b| a.1.total_cmp(b.1))
.map_or(0, |(index, _)| index as i32);
}
if top_p < 1.0 {
if top_p < 1.0 || top_k > 0 {
probabilities.sort_unstable_by(|a, b| b.1.total_cmp(&a.1).then_with(|| a.0.cmp(&b.0)));
let total: f32 = logits
if top_k > 0 {
probabilities.truncate(probabilities.len().min(top_k as usize));
}
}
if top_p < 1.0 {
let total: f32 = probabilities
.iter()
.filter(|logit| logit.is_finite())
.map(|logit| ((*logit - maximum) / temperature).exp())
.map(|(_, probability)| probability)
.sum();
let mut kept = 0.0;
let count = probabilities
@@ -612,7 +864,27 @@ mod sampling_tests {
#[test]
fn zero_temperature_is_greedy() {
let mut rng = Rng::new(1);
assert_eq!(sample(&[1.0, 4.0, 2.0], 0.0, 1.0, 0.0, &mut rng), 1);
assert_eq!(sample(&[1.0, 4.0, 2.0], 0.0, 1.0, 0.0, 0, &mut rng), 1);
}
#[test]
fn top_k_and_stream_stops_are_applied_before_output() {
let mut rng = Rng::new(1);
assert_eq!(sample(&[1.0, 4.0, 2.0], 1.0, 1.0, 0.0, 1, &mut rng), 1);
let mut text = "hello STOP hidden".to_owned();
let mut emitted = 0;
let mut chunks = Vec::new();
assert!(emit_safe_text(
&mut text,
&mut emitted,
&["STOP".into()],
false,
false,
&mut |_, chunk| chunks.push(chunk),
));
assert_eq!(text, "hello ");
assert_eq!(chunks, ["hello "]);
}
#[test]

6
src/main.rs
View File

@@ -3,8 +3,14 @@ mod database;
mod engine;
mod model;
#[cfg(target_os = "macos")]
mod native_edit;
#[cfg(target_os = "macos")]
mod native_menu;
#[cfg(target_os = "macos")]
mod runtime;
mod schema;
#[cfg(target_os = "macos")]
mod server;
mod settings;
use app::{App, Message, app_icon, app_theme};

7
src/model.rs
View File

@@ -327,6 +327,13 @@ pub(crate) fn managed_artifacts(models_path: &Path) -> Vec<ManagedArtifact> {
.collect()
}
pub(crate) fn installed_models(models_path: &Path) -> Vec<ModelChoice> {
MODEL_CHOICES
.into_iter()
.filter(|model| model.main_artifact().is_installed(*model, models_path))
.collect()
}
pub(crate) fn artifact_download_progress(
id: ManagedArtifactId,
models_path: &Path,

315
src/native_edit.rs Normal file
View File

@@ -0,0 +1,315 @@
use std::collections::VecDeque;
use std::sync::{Arc, Mutex};
use iced::advanced::layout;
use iced::advanced::overlay;
use iced::advanced::renderer;
use iced::advanced::widget::{Operation, Tree, tree};
use iced::advanced::{Clipboard, Layout, Shell, Widget};
use iced::event;
use iced::keyboard::{self, Key, Location, Modifiers, key};
use iced::mouse;
use iced::{Element, Event, Length, Rectangle, Size, Vector};
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum EditCommand {
Undo,
Redo,
Cut,
Copy,
Paste,
SelectAll,
}
pub(crate) type EditCommandQueue = Arc<Mutex<VecDeque<EditCommand>>>;
pub(crate) fn command_queue() -> EditCommandQueue {
Arc::new(Mutex::new(VecDeque::new()))
}
pub(crate) fn queue_command(queue: &EditCommandQueue, command: EditCommand) {
queue
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.push_back(command);
}
fn pop_command(queue: &EditCommandQueue) -> Option<EditCommand> {
queue
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.pop_front()
}
/// Replays native Edit menu actions through the focused Iced widget.
pub(crate) struct NativeEdit<'a, Message, Theme = iced::Theme, Renderer = iced::Renderer> {
content: Element<'a, Message, Theme, Renderer>,
commands: EditCommandQueue,
}
impl<'a, Message, Theme, Renderer> NativeEdit<'a, Message, Theme, Renderer> {
fn new(
content: impl Into<Element<'a, Message, Theme, Renderer>>,
commands: EditCommandQueue,
) -> Self {
Self {
content: content.into(),
commands,
}
}
}
#[derive(Default)]
struct State;
fn modifier_sync_event(event: &Event) -> Option<Event> {
let Event::Keyboard(keyboard::Event::KeyPressed { modifiers, .. }) = event else {
return None;
};
Some(Event::Keyboard(keyboard::Event::ModifiersChanged(
*modifiers,
)))
}
fn command_events(command: EditCommand) -> [Event; 4] {
let (character, modified_character, physical_key, modifiers) = match command {
EditCommand::Undo => ("z", "z", key::Code::KeyZ, Modifiers::COMMAND),
EditCommand::Redo => (
"z",
"Z",
key::Code::KeyZ,
Modifiers::COMMAND | Modifiers::SHIFT,
),
EditCommand::Cut => ("x", "x", key::Code::KeyX, Modifiers::COMMAND),
EditCommand::Copy => ("c", "c", key::Code::KeyC, Modifiers::COMMAND),
EditCommand::Paste => ("v", "v", key::Code::KeyV, Modifiers::COMMAND),
EditCommand::SelectAll => ("a", "a", key::Code::KeyA, Modifiers::COMMAND),
};
let key = Key::Character(character.into());
[
Event::Keyboard(keyboard::Event::ModifiersChanged(modifiers)),
Event::Keyboard(keyboard::Event::KeyPressed {
key: key.clone(),
modified_key: Key::Character(modified_character.into()),
physical_key: key::Physical::Code(physical_key),
location: Location::Standard,
modifiers,
text: None,
}),
Event::Keyboard(keyboard::Event::KeyReleased {
key,
location: Location::Standard,
modifiers,
}),
Event::Keyboard(keyboard::Event::ModifiersChanged(Modifiers::default())),
]
}
impl<'a, Message, Theme, Renderer> Widget<Message, Theme, Renderer>
for NativeEdit<'a, Message, Theme, Renderer>
where
Renderer: renderer::Renderer,
{
fn tag(&self) -> tree::Tag {
tree::Tag::of::<State>()
}
fn state(&self) -> tree::State {
tree::State::new(State)
}
fn children(&self) -> Vec<Tree> {
vec![Tree::new(&self.content)]
}
fn diff(&self, tree: &mut Tree) {
tree.diff_children(std::slice::from_ref(&self.content));
}
fn size(&self) -> Size<Length> {
self.content.as_widget().size()
}
fn layout(
&self,
tree: &mut Tree,
renderer: &Renderer,
limits: &layout::Limits,
) -> layout::Node {
self.content
.as_widget()
.layout(&mut tree.children[0], renderer, limits)
}
fn operate(
&self,
tree: &mut Tree,
layout: Layout<'_>,
renderer: &Renderer,
operation: &mut dyn Operation,
) {
self.content
.as_widget()
.operate(&mut tree.children[0], layout, renderer, operation);
}
fn on_event(
&mut self,
tree: &mut Tree,
event: Event,
layout: Layout<'_>,
cursor: mouse::Cursor,
renderer: &Renderer,
clipboard: &mut dyn Clipboard,
shell: &mut Shell<'_, Message>,
viewport: &Rectangle,
) -> event::Status {
let mut status = event::Status::Ignored;
if matches!(
event,
Event::Window(iced::window::Event::RedrawRequested(_))
) {
while let Some(command) = pop_command(&self.commands) {
for command_event in command_events(command) {
if self.content.as_widget_mut().on_event(
&mut tree.children[0],
command_event,
layout,
cursor,
renderer,
clipboard,
shell,
viewport,
) == event::Status::Captured
{
status = event::Status::Captured;
}
}
}
}
if let Some(sync_event) = modifier_sync_event(&event) {
let _ = self.content.as_widget_mut().on_event(
&mut tree.children[0],
sync_event,
layout,
cursor,
renderer,
clipboard,
shell,
viewport,
);
}
if self.content.as_widget_mut().on_event(
&mut tree.children[0],
event,
layout,
cursor,
renderer,
clipboard,
shell,
viewport,
) == event::Status::Captured
{
event::Status::Captured
} else {
status
}
}
fn mouse_interaction(
&self,
tree: &Tree,
layout: Layout<'_>,
cursor: mouse::Cursor,
viewport: &Rectangle,
renderer: &Renderer,
) -> mouse::Interaction {
self.content.as_widget().mouse_interaction(
&tree.children[0],
layout,
cursor,
viewport,
renderer,
)
}
fn draw(
&self,
tree: &Tree,
renderer: &mut Renderer,
theme: &Theme,
style: &renderer::Style,
layout: Layout<'_>,
cursor: mouse::Cursor,
viewport: &Rectangle,
) {
self.content.as_widget().draw(
&tree.children[0],
renderer,
theme,
style,
layout,
cursor,
viewport,
);
}
fn overlay<'b>(
&'b mut self,
tree: &'b mut Tree,
layout: Layout<'_>,
renderer: &Renderer,
translation: Vector,
) -> Option<overlay::Element<'b, Message, Theme, Renderer>> {
self.content
.as_widget_mut()
.overlay(&mut tree.children[0], layout, renderer, translation)
}
}
impl<'a, Message, Theme, Renderer> From<NativeEdit<'a, Message, Theme, Renderer>>
for Element<'a, Message, Theme, Renderer>
where
Message: 'a,
Theme: 'a,
Renderer: 'a + renderer::Renderer,
{
fn from(bridge: NativeEdit<'a, Message, Theme, Renderer>) -> Self {
Element::new(bridge)
}
}
pub(crate) fn native_edit<'a, Message, Theme, Renderer>(
content: impl Into<Element<'a, Message, Theme, Renderer>>,
commands: EditCommandQueue,
) -> NativeEdit<'a, Message, Theme, Renderer> {
NativeEdit::new(content, commands)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn paste_replays_a_complete_command_shortcut() {
let events = command_events(EditCommand::Paste);
assert!(matches!(
events[0],
Event::Keyboard(keyboard::Event::ModifiersChanged(Modifiers::COMMAND))
));
assert!(matches!(
events[1],
Event::Keyboard(keyboard::Event::KeyPressed {
key: Key::Character(ref key),
modifiers: Modifiers::COMMAND,
..
}) if key == "v"
));
assert!(matches!(
events[3],
Event::Keyboard(keyboard::Event::ModifiersChanged(modifiers)) if modifiers.is_empty()
));
}
}

76
src/native_menu.rs
View File

@@ -1,8 +1,16 @@
use muda::accelerator::{Accelerator, Code, Modifiers};
use muda::accelerator::{Accelerator, CMD_OR_CTRL, Code, Modifiers};
use muda::{Menu, MenuEvent, MenuItem, PredefinedMenuItem, Submenu};
use crate::native_edit::EditCommand;
const PREFERENCES: &str = "preferences";
const MODEL_MANAGER: &str = "model-manager";
const UNDO: &str = "undo";
const REDO: &str = "redo";
const CUT: &str = "cut";
const COPY: &str = "copy";
const PASTE: &str = "paste";
const SELECT_ALL: &str = "select-all";
pub(crate) struct NativeMenu {
_menu: Menu,
@@ -12,6 +20,7 @@ pub(crate) struct NativeMenu {
pub(crate) enum NativeMenuEvent {
Preferences,
ModelManager,
Edit(EditCommand),
}
pub(crate) fn install() -> Result<NativeMenu, String> {
@@ -34,6 +43,12 @@ pub(crate) fn install() -> Result<NativeMenu, String> {
Code::KeyM,
)),
);
let undo = edit_item(UNDO, "Undo", Code::KeyZ, None);
let redo = edit_item(REDO, "Redo", Code::KeyZ, Some(Modifiers::SHIFT));
let cut = edit_item(CUT, "Cut", Code::KeyX, None);
let copy = edit_item(COPY, "Copy", Code::KeyC, None);
let paste = edit_item(PASTE, "Paste", Code::KeyV, None);
let select_all = edit_item(SELECT_ALL, "Select All", Code::KeyA, None);
application
.append_items(&[
@@ -51,13 +66,13 @@ pub(crate) fn install() -> Result<NativeMenu, String> {
])
.map_err(|error| error.to_string())?;
edit.append_items(&[
&PredefinedMenuItem::undo(None),
&PredefinedMenuItem::redo(None),
&undo,
&redo,
&PredefinedMenuItem::separator(),
&PredefinedMenuItem::cut(None),
&PredefinedMenuItem::copy(None),
&PredefinedMenuItem::paste(None),
&PredefinedMenuItem::select_all(None),
&cut,
&copy,
&paste,
&select_all,
])
.map_err(|error| error.to_string())?;
window
@@ -79,12 +94,47 @@ pub(crate) fn install() -> Result<NativeMenu, String> {
pub(crate) fn next_event() -> Option<NativeMenuEvent> {
while let Ok(event) = MenuEvent::receiver().try_recv() {
if event.id == PREFERENCES {
return Some(NativeMenuEvent::Preferences);
}
if event.id == MODEL_MANAGER {
return Some(NativeMenuEvent::ModelManager);
}
let event = match event.id.0.as_str() {
PREFERENCES => NativeMenuEvent::Preferences,
MODEL_MANAGER => NativeMenuEvent::ModelManager,
UNDO => NativeMenuEvent::Edit(EditCommand::Undo),
REDO => NativeMenuEvent::Edit(EditCommand::Redo),
CUT => NativeMenuEvent::Edit(EditCommand::Cut),
COPY => NativeMenuEvent::Edit(EditCommand::Copy),
PASTE => NativeMenuEvent::Edit(EditCommand::Paste),
SELECT_ALL => NativeMenuEvent::Edit(EditCommand::SelectAll),
_ => continue,
};
return Some(event);
}
None
}
fn edit_item(
id: &'static str,
label: &'static str,
code: Code,
extra_modifier: Option<Modifiers>,
) -> MenuItem {
MenuItem::with_id(
id,
label,
true,
Some(Accelerator::new(
Some(CMD_OR_CTRL | extra_modifier.unwrap_or_else(Modifiers::empty)),
code,
)),
)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn edit_items_are_custom_menu_events() {
let paste = edit_item(PASTE, "Paste", Code::KeyV, None);
assert_eq!(paste.id().0, PASTE);
assert!(paste.is_enabled());
}
}

151
src/runtime.rs Normal file
View File

@@ -0,0 +1,151 @@
use crate::engine::{ChatTurn, GenerationOutput, Generator};
use crate::settings::{EngineSettings, TurnSettings};
use std::path::PathBuf;
use std::sync::Arc;
use std::sync::atomic::AtomicBool;
use std::sync::mpsc::{self, Receiver, Sender};
use std::thread;
use std::time::{Duration, Instant};
#[derive(Clone)]
pub(crate) struct GenerationService {
commands: Sender<Command>,
}
pub(crate) struct ActiveGeneration {
pub(crate) events: Receiver<GenerationEvent>,
pub(crate) cancel: Arc<AtomicBool>,
}
pub(crate) enum CheckpointTarget {
Local(PathBuf),
Transient(PathBuf),
}
pub(crate) enum GenerationEvent {
Loading,
Chunk {
reasoning: bool,
content: String,
},
Context {
used: u32,
limit: u32,
tokens_per_second: Option<f32>,
},
Finished(Result<GenerationOutput, String>),
}
struct Command {
engine: EngineSettings,
turn: TurnSettings,
messages: Vec<ChatTurn>,
checkpoint: CheckpointTarget,
idle_timeout: Duration,
cancel: Arc<AtomicBool>,
events: Sender<GenerationEvent>,
}
impl GenerationService {
pub(crate) fn spawn() -> Result<Self, String> {
let (commands, receiver) = mpsc::channel::<Command>();
thread::Builder::new()
.name("model-runtime".into())
.spawn(move || run(receiver))
.map_err(|error| format!("Could not start the model runtime: {error}"))?;
Ok(Self { commands })
}
pub(crate) fn generate(
&self,
engine: EngineSettings,
turn: TurnSettings,
messages: Vec<ChatTurn>,
checkpoint: CheckpointTarget,
idle_timeout: Duration,
) -> Result<ActiveGeneration, String> {
let cancel = Arc::new(AtomicBool::new(false));
let (events, receiver) = mpsc::channel();
self.commands
.send(Command {
engine,
turn,
messages,
checkpoint,
idle_timeout,
cancel: Arc::clone(&cancel),
events,
})
.map_err(|_| "The model runtime stopped unexpectedly.".to_owned())?;
Ok(ActiveGeneration {
events: receiver,
cancel,
})
}
}
fn run(commands: Receiver<Command>) {
let mut loaded = None::<(EngineSettings, Generator)>;
let mut last_used = Instant::now();
let mut idle_timeout = Duration::from_secs(15 * 60);
loop {
match commands.recv_timeout(Duration::from_secs(1)) {
Ok(command) => {
idle_timeout = command.idle_timeout;
if loaded
.as_ref()
.is_none_or(|(settings, _)| settings != &command.engine)
{
let _ = command.events.send(GenerationEvent::Loading);
loaded = match Generator::open(&command.engine) {
Ok(generator) => Some((command.engine.clone(), generator)),
Err(error) => {
let _ = command.events.send(GenerationEvent::Finished(Err(error)));
None
}
};
}
if let Some((_, generator)) = &mut loaded {
let mut emit = |reasoning, content| {
let _ = command
.events
.send(GenerationEvent::Chunk { reasoning, content });
};
let mut progress = |used, limit, tokens_per_second| {
let _ = command.events.send(GenerationEvent::Context {
used,
limit,
tokens_per_second,
});
};
let result = match command.checkpoint {
CheckpointTarget::Local(path) => generator.generate(
&path,
&command.messages,
&command.turn,
&command.cancel,
&mut emit,
&mut progress,
),
CheckpointTarget::Transient(directory) => generator.generate_transient(
&directory,
&command.messages,
&command.turn,
&command.cancel,
&mut emit,
&mut progress,
),
};
let _ = command.events.send(GenerationEvent::Finished(result));
last_used = Instant::now();
}
}
Err(mpsc::RecvTimeoutError::Timeout) => {
if loaded.is_some() && last_used.elapsed() >= idle_timeout {
loaded = None;
}
}
Err(mpsc::RecvTimeoutError::Disconnected) => break,
}
}
}

1
src/schema.rs
View File

@@ -53,6 +53,7 @@ diesel::table! {
directional_steering_attn -> Nullable<Float>,
simulated_used_memory_gib -> Nullable<BigInt>,
expert_profile_path -> Nullable<Text>,
endpoint_port -> Integer,
}
}

1116
src/server.rs Normal file
View File

File diff suppressed because it is too large Load Diff

4
src/settings.rs
View File

@@ -443,6 +443,8 @@ impl GenerationPreferences {
temperature: self.temperature.unwrap_or(1.0),
top_p: self.top_p.unwrap_or(if glm { 0.95 } else { 1.0 }),
min_p: self.min_p.unwrap_or(if glm { 0.0 } else { 0.05 }),
top_k: 0,
stops: Vec::new(),
seed: self.seed,
reasoning_mode: if self.reasoning_mode == ReasoningMode::Max
&& self.context_tokens < THINK_MAX_MIN_CONTEXT
@@ -489,6 +491,8 @@ pub(crate) struct TurnSettings {
pub(crate) temperature: f32,
pub(crate) top_p: f32,
pub(crate) min_p: f32,
pub(crate) top_k: i32,
pub(crate) stops: Vec<String>,
pub(crate) seed: Option<u64>,
pub(crate) reasoning_mode: ReasoningMode,
}