hugo/DS4Server
1
0
Fork 0
Code Issues 8 Pull Requests Actions Packages Projects Releases Wiki Activity

feat: first cut at openai compatible server

Browse Source
This commit is contained in:
Georg Bauer
2026-07-24 21:12:48 +02:00
parent bbbe65a75f
commit d554b77b9d
18 changed files with 2612 additions and 535 deletions
Download Patch File Download Diff File Expand all files Collapse all files

332
src/app.rs
View File

@@ -4,8 +4,10 @@ pub(crate) use view::app_theme;
use crate::database::{AppPreferences, Database, ProjectWithSessions, StoredMessage};
#[cfg(target_os = "macos")]
use crate::engine::{ChatTurn, Generator};
use crate::engine::ChatTurn;
use crate::model::{self, DownloadOutcome, DownloadProgress, ManagedArtifactId, ModelChoice};
#[cfg(target_os = "macos")]
use crate::runtime::{ActiveGeneration, CheckpointTarget, GenerationEvent, GenerationService};
use crate::settings::{
DiagnosticPreferences, ExecutionPreferences, GIB, GenerationPreferences, ReasoningMode,
RuntimePreferences, SpeculativePreferences, SsdPreferences, SteeringPreferences,
@@ -16,9 +18,9 @@ use iced::{Size, Subscription, Task, keyboard, window};
use rfd::AsyncFileDialog;
use std::fs;
use std::path::PathBuf;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::sync::mpsc::{self, TryRecvError};
use std::sync::{Arc, RwLock};
use std::thread;
use std::time::{Duration, Instant};
@@ -29,6 +31,7 @@ struct PreferenceDraft {
model: ModelChoice,
dspark_enabled: bool,
idle_timeout_minutes: String,
endpoint_port: String,
context_tokens: String,
max_generated_tokens: String,
system_prompt: String,
@@ -73,6 +76,7 @@ impl PreferenceDraft {
model,
dspark_enabled: speculative.dspark_enabled,
idle_timeout_minutes: preferences.idle_timeout_minutes.to_string(),
endpoint_port: preferences.endpoint_port.to_string(),
context_tokens: generation.context_tokens.to_string(),
max_generated_tokens: generation.max_generated_tokens.to_string(),
system_prompt: generation.system_prompt,
@@ -168,6 +172,7 @@ impl PreferenceDraft {
self.model = ModelChoice::default();
self.dspark_enabled = false;
self.idle_timeout_minutes = "10".into();
self.endpoint_port = "4000".into();
self.context_tokens = defaults.context_tokens.to_string();
self.max_generated_tokens = defaults.max_generated_tokens.to_string();
self.system_prompt = defaults.system_prompt;
@@ -389,6 +394,8 @@ pub(crate) struct App {
pub(super) pending_model_delete: Option<ManagedArtifactId>,
#[cfg(target_os = "macos")]
_native_menu: Option<crate::native_menu::NativeMenu>,
#[cfg(target_os = "macos")]
pub(super) native_edit_commands: crate::native_edit::EditCommandQueue,
database: Option<Database>,
projects: Vec<ProjectWithSessions>,
preferences: AppPreferences,
@@ -408,7 +415,13 @@ pub(crate) struct App {
pub(super) context_limit: u32,
pub(super) tokens_per_second: Option<f32>,
#[cfg(target_os = "macos")]
generation_worker: Option<GenerationWorker>,
generation_service: Option<GenerationService>,
#[cfg(target_os = "macos")]
active_generation: Option<ActiveGeneration>,
#[cfg(target_os = "macos")]
runtime_preferences: Arc<RwLock<AppPreferences>>,
#[cfg(target_os = "macos")]
_endpoint: Option<crate::server::ServerHandle>,
error: Option<String>,
}
@@ -461,40 +474,6 @@ impl From<StoredMessage> for ChatMessage {
}
}
#[cfg(target_os = "macos")]
struct GenerationWorker {
commands: mpsc::Sender<GenerationCommand>,
events: mpsc::Receiver<GenerationEvent>,
cancel: Option<Arc<AtomicBool>>,
}
#[cfg(target_os = "macos")]
enum GenerationCommand {
Generate {
engine: crate::settings::EngineSettings,
turn: crate::settings::TurnSettings,
messages: Vec<ChatTurn>,
checkpoint: PathBuf,
idle_timeout: Duration,
cancel: Arc<AtomicBool>,
},
}
#[cfg(target_os = "macos")]
enum GenerationEvent {
Loading,
Chunk {
reasoning: bool,
content: String,
},
Context {
used: u32,
limit: u32,
tokens_per_second: Option<f32>,
},
Finished(Result<(), String>),
}
#[derive(Debug)]
pub(super) enum ModelDownload {
Idle,
@@ -526,6 +505,8 @@ pub(super) struct ActiveDownload {
#[derive(Debug, Clone)]
pub(crate) enum Message {
Noop,
#[cfg(target_os = "macos")]
NativeEdit(crate::native_edit::EditCommand),
OpenPreferences,
OpenModelManager,
ModelManagerOpened(window::Id),
@@ -535,6 +516,7 @@ pub(crate) enum Message {
PreferenceModelChanged(ModelChoice),
PreferenceDsparkChanged(bool),
PreferenceTimeoutChanged(String),
PreferenceEndpointPortChanged(String),
PreferenceContextChanged(String),
PreferenceMaxTokensChanged(String),
PreferenceSystemPromptChanged(String),
@@ -602,12 +584,17 @@ impl App {
Err(error) => return Self::failed(error, main_window),
};
let context_limit = preferences.context_tokens.max(0) as u32;
#[cfg(target_os = "macos")]
let (runtime_preferences, generation_service, endpoint, service_error) =
spawn_services(&preferences);
Self {
main_window,
model_manager_window: None,
pending_model_delete: None,
#[cfg(target_os = "macos")]
_native_menu: None,
#[cfg(target_os = "macos")]
native_edit_commands: crate::native_edit::command_queue(),
database: Some(database),
projects,
preferences,
@@ -627,8 +614,23 @@ impl App {
context_limit,
tokens_per_second: None,
#[cfg(target_os = "macos")]
generation_worker: None,
error: None,
generation_service,
#[cfg(target_os = "macos")]
active_generation: None,
#[cfg(target_os = "macos")]
runtime_preferences,
#[cfg(target_os = "macos")]
_endpoint: endpoint,
error: {
#[cfg(target_os = "macos")]
{
service_error
}
#[cfg(not(target_os = "macos"))]
{
None
}
},
}
}
(Err(error), _) | (_, Err(error)) => Self::failed(error, main_window),
@@ -642,12 +644,21 @@ impl App {
let preference_draft = PreferenceDraft::from_saved(&preferences)
.expect("default preferences must use a supported model");
let context_limit = preferences.context_tokens.max(0) as u32;
#[cfg(target_os = "macos")]
let (runtime_preferences, generation_service, endpoint, service_error) =
spawn_services(&preferences);
#[cfg(target_os = "macos")]
let startup_error = service_error;
#[cfg(not(target_os = "macos"))]
let startup_error = None::<String>;
Self {
main_window,
model_manager_window: None,
pending_model_delete: None,
#[cfg(target_os = "macos")]
_native_menu: None,
#[cfg(target_os = "macos")]
native_edit_commands: crate::native_edit::command_queue(),
database: None,
projects: Vec::new(),
preferences,
@@ -667,14 +678,29 @@ impl App {
context_limit,
tokens_per_second: None,
#[cfg(target_os = "macos")]
generation_worker: None,
error: Some(format!("Could not open the project database: {error}")),
generation_service,
#[cfg(target_os = "macos")]
active_generation: None,
#[cfg(target_os = "macos")]
runtime_preferences,
#[cfg(target_os = "macos")]
_endpoint: endpoint,
error: Some(match startup_error {
Some(service_error) => {
format!("Could not open the project database: {error}. {service_error}")
}
None => format!("Could not open the project database: {error}"),
}),
}
}
pub(crate) fn update(&mut self, message: Message) -> Task<Message> {
match message {
Message::Noop => {}
#[cfg(target_os = "macos")]
Message::NativeEdit(command) => {
crate::native_edit::queue_command(&self.native_edit_commands, command)
}
Message::OpenPreferences => self.open_preferences(),
Message::OpenModelManager => return self.open_model_manager(),
Message::ModelManagerOpened(id) => {
@@ -749,6 +775,10 @@ impl App {
self.preference_draft.idle_timeout_minutes = value;
self.preference_error = None;
}
Message::PreferenceEndpointPortChanged(value) => {
self.preference_draft.endpoint_port = value;
self.preference_error = None;
}
Message::PreferenceContextChanged(value) => {
self.preference_draft.context_tokens = value;
self.preference_error = None;
@@ -953,14 +983,11 @@ impl App {
self.start_generation();
return scroll_chat_to_end();
}
Message::StopGeneration => {
Message::StopGeneration =>
{
#[cfg(target_os = "macos")]
if let Some(cancel) = self
.generation_worker
.as_ref()
.and_then(|worker| worker.cancel.as_ref())
{
cancel.store(true, Ordering::Relaxed);
if let Some(active) = &self.active_generation {
active.cancel.store(true, Ordering::Relaxed);
}
}
Message::GenerationTick => {
@@ -1133,6 +1160,9 @@ impl App {
Some(crate::native_menu::NativeMenuEvent::ModelManager) => {
Message::OpenModelManager
}
Some(crate::native_menu::NativeMenuEvent::Edit(command)) => {
Message::NativeEdit(command)
}
None => Message::Noop,
}
}));
@@ -1255,6 +1285,14 @@ impl App {
self.preference_error = Some("Idle timeout must be between 1 and 1440 minutes.".into());
return;
}
let Ok(endpoint_port) = self.preference_draft.endpoint_port.trim().parse::<u16>() else {
self.preference_error = Some("Endpoint port must be a whole number.".into());
return;
};
if endpoint_port == 0 {
self.preference_error = Some("Endpoint port must be between 1 and 65535.".into());
return;
}
let generation = match self.preference_draft.generation() {
Ok(generation) => generation,
Err(error) => {
@@ -1275,12 +1313,50 @@ impl App {
self.preference_error = Some(error);
return;
}
#[cfg(target_os = "macos")]
let pending_endpoint = if self.preferences.endpoint_port != i32::from(endpoint_port)
|| self._endpoint.is_none()
{
let Some(generation) = &self.generation_service else {
self.preference_error = Some("The model runtime is unavailable.".into());
return;
};
match crate::server::ServerHandle::spawn(
generation.clone(),
Arc::clone(&self.runtime_preferences),
models_path(),
application_support_path().join("kv-cache").join("http"),
endpoint_port,
) {
Ok(endpoint) => Some(endpoint),
Err(error) => {
self.preference_error = Some(error);
return;
}
}
} else {
None
};
let Some(database) = &mut self.database else {
return;
};
match database.update_preferences(model.id(), idle_timeout_minutes, &generation, &runtime) {
match database.update_preferences(
model.id(),
idle_timeout_minutes,
i32::from(endpoint_port),
&generation,
&runtime,
) {
Ok(preferences) => {
self.preferences = preferences;
#[cfg(target_os = "macos")]
if let Ok(mut runtime_preferences) = self.runtime_preferences.write() {
*runtime_preferences = self.preferences.clone();
}
#[cfg(target_os = "macos")]
if let Some(endpoint) = pending_endpoint {
self._endpoint = Some(endpoint);
}
self.preference_draft = PreferenceDraft::from_saved(&self.preferences)
.expect("the saved model was selected from the supported catalog");
self.preferences_open = false;
@@ -1508,15 +1584,10 @@ impl App {
#[cfg(target_os = "macos")]
{
if self.generation_worker.is_none() {
match spawn_generation_worker() {
Ok(worker) => self.generation_worker = Some(worker),
Err(error) => {
self.error = Some(error);
return;
}
}
}
let Some(service) = &self.generation_service else {
self.error = Some("The model runtime is unavailable.".into());
return;
};
let Some(database) = &mut self.database else {
return;
};
@@ -1527,24 +1598,22 @@ impl App {
return;
}
};
let cancel = Arc::new(AtomicBool::new(false));
let idle_timeout =
Duration::from_secs(self.preferences.idle_timeout_minutes.max(1) as u64 * 60);
let command = GenerationCommand::Generate {
engine: effective.engine,
turn: effective.turn,
self.active_generation = match service.generate(
effective.engine,
effective.turn,
messages,
checkpoint: session_checkpoint_path(session_id),
CheckpointTarget::Local(session_checkpoint_path(session_id)),
idle_timeout,
cancel: Arc::clone(&cancel),
) {
Ok(active) => Some(active),
Err(error) => {
self.generation_service = None;
self.error = Some(error);
return;
}
};
let worker = self.generation_worker.as_mut().expect("worker was created");
if worker.commands.send(command).is_err() {
self.generation_worker = None;
self.error = Some("The local generation worker stopped unexpectedly.".into());
return;
}
worker.cancel = Some(cancel);
let user = ChatMessage::from(saved.0);
let mut assistant = ChatMessage::from(saved.1);
assistant.reasoning_open = assistant_reasoning;
@@ -1565,7 +1634,7 @@ impl App {
fn poll_generation(&mut self) -> bool {
#[cfg(target_os = "macos")]
let Some(worker) = &mut self.generation_worker else {
let Some(active) = &mut self.active_generation else {
self.generating = false;
return false;
};
@@ -1575,7 +1644,7 @@ impl App {
let mut context_changed = false;
#[cfg(target_os = "macos")]
loop {
match worker.events.try_recv() {
match active.events.try_recv() {
Ok(GenerationEvent::Loading) => {}
Ok(GenerationEvent::Chunk { reasoning, content }) => {
if let Some(message) = self.conversation.last_mut()
@@ -1597,17 +1666,17 @@ impl App {
}
Ok(GenerationEvent::Finished(result)) => {
self.generating = false;
worker.cancel = None;
if let Err(error) = result {
self.error = Some(error);
}
self.active_generation = None;
break;
}
Err(TryRecvError::Empty) => break,
Err(TryRecvError::Disconnected) => {
self.generating = false;
self.generation_worker = None;
self.error = Some("The local generation worker stopped unexpectedly.".into());
self.active_generation = None;
self.error = Some("The model runtime stopped unexpectedly.".into());
break;
}
}
@@ -1627,12 +1696,8 @@ impl App {
&message.content,
)
{
if let Some(cancel) = self
.generation_worker
.as_ref()
.and_then(|worker| worker.cancel.as_ref())
{
cancel.store(true, Ordering::Relaxed);
if let Some(active) = &self.active_generation {
active.cancel.store(true, Ordering::Relaxed);
}
self.error = Some(format!("Could not save generated chat text: {error}"));
}
@@ -1667,77 +1732,30 @@ impl App {
}
#[cfg(target_os = "macos")]
fn spawn_generation_worker() -> Result<GenerationWorker, String> {
let (command_sender, command_receiver) = mpsc::channel();
let (event_sender, event_receiver) = mpsc::channel();
thread::Builder::new()
.name("local-generation".into())
.spawn(move || {
let mut loaded = None::<(crate::settings::EngineSettings, Generator)>;
let mut last_used = Instant::now();
let mut idle_timeout = Duration::from_secs(15 * 60);
loop {
match command_receiver.recv_timeout(Duration::from_secs(1)) {
Ok(GenerationCommand::Generate {
engine,
turn,
messages,
checkpoint,
idle_timeout: requested_timeout,
cancel,
}) => {
idle_timeout = requested_timeout;
if loaded
.as_ref()
.is_none_or(|(current, _)| current != &engine)
{
let _ = event_sender.send(GenerationEvent::Loading);
loaded = match Generator::open(&engine) {
Ok(generator) => Some((engine.clone(), generator)),
Err(error) => {
let _ =
event_sender.send(GenerationEvent::Finished(Err(error)));
None
}
};
}
if let Some((_, generator)) = &mut loaded {
let result = generator.generate(
&checkpoint,
&messages,
&turn,
&cancel,
|reasoning, content| {
let _ = event_sender
.send(GenerationEvent::Chunk { reasoning, content });
},
|used, limit, tokens_per_second| {
let _ = event_sender.send(GenerationEvent::Context {
used,
limit,
tokens_per_second,
});
},
);
let _ = event_sender.send(GenerationEvent::Finished(result));
last_used = Instant::now();
}
}
Err(mpsc::RecvTimeoutError::Timeout) => {
if loaded.is_some() && last_used.elapsed() >= idle_timeout {
loaded = None;
}
}
Err(mpsc::RecvTimeoutError::Disconnected) => break,
}
}
})
.map_err(|error| format!("Could not start local generation: {error}"))?;
Ok(GenerationWorker {
commands: command_sender,
events: event_receiver,
cancel: None,
})
fn spawn_services(
preferences: &AppPreferences,
) -> (
Arc<RwLock<AppPreferences>>,
Option<GenerationService>,
Option<crate::server::ServerHandle>,
Option<String>,
) {
let runtime_preferences = Arc::new(RwLock::new(preferences.clone()));
let generation = match GenerationService::spawn() {
Ok(generation) => generation,
Err(error) => return (runtime_preferences, None, None, Some(error)),
};
let endpoint = crate::server::ServerHandle::spawn(
generation.clone(),
Arc::clone(&runtime_preferences),
models_path(),
application_support_path().join("kv-cache").join("http"),
u16::try_from(preferences.endpoint_port).unwrap_or(4000),
);
match endpoint {
Ok(endpoint) => (runtime_preferences, Some(generation), Some(endpoint), None),
Err(error) => (runtime_preferences, Some(generation), None, Some(error)),
}
}
impl Drop for App {
@@ -1746,12 +1764,8 @@ impl Drop for App {
download.cancel.store(true, Ordering::Relaxed);
}
#[cfg(target_os = "macos")]
if let Some(cancel) = self
.generation_worker
.as_ref()
.and_then(|worker| worker.cancel.as_ref())
{
cancel.store(true, Ordering::Relaxed);
if let Some(active) = &self.active_generation {
active.cancel.store(true, Ordering::Relaxed);
}
}
}

629
src/app/view.rs
View File

@@ -31,7 +31,12 @@ impl App {
if self.model_manager_window == Some(id) {
self.model_manager()
} else {
self.main_view()
let content = self.main_view();
#[cfg(target_os = "macos")]
return crate::native_edit::native_edit(content, self.native_edit_commands.clone())
.into();
#[cfg(not(target_os = "macos"))]
content
}
}
@@ -457,286 +462,331 @@ impl App {
dspark_confidence.on_input(Message::PreferenceDsparkConfidenceChanged);
}
let mut fields = column![
text("MODEL").size(11),
pick_list(
&MODEL_CHOICES[..],
Some(self.preference_draft.model),
Message::PreferenceModelChanged,
)
.width(Length::Fill),
text(format!(
"Main: {}{}",
engine.map_or_else(
|| "Invalid settings".to_owned(),
|engine| engine.artifacts.model.display().to_string(),
),
engine
.and_then(|engine| engine.artifacts.mtp.as_ref())
.map_or_else(String::new, |path| format!(" • support: {}", path.display())),
))
.size(12),
Space::with_height(8),
text("EXECUTION").size(11),
preference_input_row(
"CPU helper threads",
text_input("Automatic", &self.preference_draft.cpu_threads)
.on_input(Message::PreferenceCpuThreadsChanged),
),
preference_input_row("GPU power percent", power),
preference_input_row("Prefill chunk", prefill),
checkbox("Prefer exact quality kernels", self.preference_draft.quality)
.on_toggle(Message::PreferenceQualityChanged),
checkbox("Warm mapped weights at load time", self.preference_draft.warm_weights)
.on_toggle(Message::PreferenceWarmWeightsChanged),
text(if self.preference_draft.model == ModelChoice::Glm52 {
"GLM 5.2 uses full GPU power and selects prefill chunks automatically."
} else {
"Blank numeric values preserve DS4's automatic engine behavior."
})
.size(12),
text(engine.as_ref().map_or_else(
|| "Effective execution settings will appear after valid values are entered."
.to_owned(),
|engine| {
let settings = engine.execution;
format!(
"Metal engine: threads {} • power {}% • prefill {} • quality {} • warm weights {}",
if settings.cpu_threads == 0 { "auto".to_owned() } else { settings.cpu_threads.to_string() },
if settings.power_percent == 0 { 100 } else { settings.power_percent },
if settings.prefill_chunk == 0 { "auto".to_owned() } else { settings.prefill_chunk.to_string() },
if settings.quality { "on" } else { "off" },
if settings.warm_weights { "on" } else { "off" },
)},
))
.size(12),
Space::with_height(8),
text("SPECULATIVE DECODING").size(11),
preference_input_row(
"MTP draft tokens",
text_input("1", &self.preference_draft.mtp_draft_tokens)
.on_input(Message::PreferenceMtpDraftChanged),
),
preference_input_row(
"MTP verifier margin",
text_input("3", &self.preference_draft.mtp_margin)
.on_input(Message::PreferenceMtpMarginChanged),
),
checkbox("Enable integrated GLM MTP", self.preference_draft.glm_mtp)
.on_toggle_maybe(glm_mtp_toggle),
checkbox(
"Log GLM MTP timing counters",
self.preference_draft.glm_mtp_timing,
)
.on_toggle_maybe(glm_mtp_timing_toggle),
dspark,
preference_input_row("DSpark confidence threshold", dspark_confidence),
checkbox(
"DSpark target-only decode",
self.preference_draft.dspark_strict,
)
.on_toggle_maybe(dspark_strict_toggle),
text(if self.preference_draft.model.supports_dspark() {
"DSpark uses the managed support artifact; entering a threshold or enabling strict mode also enables DSpark."
} else if self.preference_draft.model == ModelChoice::Glm52 {
"GLM MTP is integrated; DSpark is unavailable for this model."
} else {
"No managed MTP support artifact is available for this model."
})
.size(12),
text(engine.as_ref().map_or_else(
|| "Effective speculative settings will appear after valid values are entered."
.to_owned(),
|engine| {
let settings = engine.speculative;
format!(
"Engine: MTP draft {} • margin {} • GLM MTP {} • timing {} • DSpark {} • confidence {}{} • target-only {}",
settings.mtp_draft_tokens,
settings.mtp_margin,
if settings.glm_mtp { "on" } else { "off" },
if settings.glm_mtp_timing { "on" } else { "off" },
if settings.dspark { "on" } else { "off" },
settings.dspark_confidence_threshold,
if settings.dspark_confidence_threshold_set { " explicit" } else { " default" },
if settings.dspark_strict { "on" } else { "off" },
)},
))
.size(12),
Space::with_height(8),
text("SSD STREAMING").size(11),
checkbox("Enable SSD-backed model streaming", self.preference_draft.ssd_streaming)
.on_toggle(Message::PreferenceSsdChanged),
checkbox("Skip automatic expert preload", self.preference_draft.ssd_streaming_cold)
.on_toggle(Message::PreferenceSsdColdChanged),
preference_input_row(
"Expert cache count or GiB",
text_input("Automatic, 128, or 64GB", &self.preference_draft.ssd_cache)
.on_input(Message::PreferenceSsdCacheChanged),
),
preference_input_row("Fully resident GLM layers", ssd_full_layers),
preference_input_row(
"Explicit expert preload count",
text_input("Automatic", &self.preference_draft.ssd_preload_experts)
.on_input(Message::PreferenceSsdPreloadChanged),
),
text("A blank full-layer value is automatic; an explicit 0 disables fully resident GLM layers. SSD streaming and DSpark are mutually exclusive.")
.size(12),
text(engine.as_ref().map_or_else(
|| "Effective SSD settings will appear after valid values are entered."
.to_owned(),
|engine| {
let settings = engine.ssd;
let cache = if settings.cache_bytes > 0 {
format!("{} GiB", settings.cache_bytes / GIB)
} else if settings.cache_experts > 0 {
format!("{} experts", settings.cache_experts)
} else {
"auto".to_owned()
};
format!(
"Engine: streaming {} • cold {} • cache {} • full layers {}{} • preload {}",
if settings.enabled { "on" } else { "off" },
if settings.cold { "on" } else { "off" },
cache,
settings.full_layers,
if settings.full_layers_set { " explicit" } else { " auto" },
if settings.preload_experts == 0 { "auto".to_owned() } else { settings.preload_experts.to_string() },
)
},
))
.size(12),
Space::with_height(8),
text("DIRECTIONAL STEERING").size(11),
text("Direction-vector file").size(13),
steering_file.padding(9),
preference_input_row("FFN scale", steering_ffn),
preference_input_row("Attention scale", steering_attn),
text(if self.preference_draft.model == ModelChoice::Glm52 {
"Directional steering is not supported for GLM 5.2."
} else {
"With a file and no explicit scale, DS4 defaults the FFN scale to 1. Scales accept -100 through 100."
})
.size(12),
text(engine.as_ref().map_or_else(
|| "Effective steering settings will appear after valid values are entered."
.to_owned(),
|engine| format!(
"Engine: file {} • FFN scale {} • attention scale {}",
if engine.steering.file.is_some() { "set" } else { "off" },
engine.steering.ffn_scale,
engine.steering.attention_scale,
),
))
.size(12),
Space::with_height(8),
text("CAPACITY").size(11),
preference_input_row(
"Context tokens",
text_input("32768", &self.preference_draft.context_tokens)
.on_input(Message::PreferenceContextChanged),
),
preference_input_row(
"Maximum generated tokens",
text_input("50000", &self.preference_draft.max_generated_tokens)
.on_input(Message::PreferenceMaxTokensChanged),
),
text("System prompt").size(13),
text_input(
"You are a helpful assistant",
&self.preference_draft.system_prompt,
)
.on_input(Message::PreferenceSystemPromptChanged)
.padding(9),
Space::with_height(8),
text("SAMPLING AND REASONING").size(11),
preference_input_row(
"Temperature",
text_input("DS4 default", &self.preference_draft.temperature)
.on_input(Message::PreferenceTemperatureChanged),
),
preference_input_row(
"Top-p",
text_input("DS4 default", &self.preference_draft.top_p)
.on_input(Message::PreferenceTopPChanged),
),
preference_input_row(
"Min-p",
text_input("DS4 default", &self.preference_draft.min_p)
.on_input(Message::PreferenceMinPChanged),
),
preference_input_row(
"Seed",
text_input("Random", &self.preference_draft.seed)
.on_input(Message::PreferenceSeedChanged),
),
row![
text("Reasoning").size(13).width(Length::Fill),
let model_group = preference_group(
"MODEL & LIFECYCLE",
column![
pick_list(
&REASONING_MODES[..],
Some(self.preference_draft.reasoning_mode),
Message::PreferenceReasoningChanged,
&MODEL_CHOICES[..],
Some(self.preference_draft.model),
Message::PreferenceModelChanged,
)
.width(240),
]
.spacing(12)
.align_y(Alignment::Center),
text("Blank sampling values retain DS4's model-family defaults. Think Max needs at least 393216 context tokens.")
.width(Length::Fill),
text(format!(
"Main: {}{}",
engine.map_or_else(
|| "Invalid settings".to_owned(),
|engine| engine.artifacts.model.display().to_string(),
),
engine
.and_then(|engine| engine.artifacts.mtp.as_ref())
.map_or_else(String::new, |path| format!(
" • support: {}",
path.display()
)),
))
.size(12),
text(turn.map_or_else(
|| "Effective settings will appear after valid values are entered.".to_owned(),
|settings| format!(
"Effective: {} context • {} max • temp {} • top-p {} • min-p {} • seed {} • {} • system prompt {}",
settings.context_tokens,
settings.max_generated_tokens,
settings.temperature,
settings.top_p,
settings.min_p,
settings.seed.map_or_else(|| "random".to_owned(), |seed| seed.to_string()),
settings.reasoning_mode,
if settings.system_prompt.is_empty() { "off" } else { "on" },
),
))
.size(12),
Space::with_height(8),
text("ADVANCED DIAGNOSTICS").size(11),
preference_input_row(
"Simulated used memory (GiB)",
text_input("Disabled", &self.preference_draft.simulated_used_memory_gib)
.on_input(Message::PreferenceSimulatedMemoryChanged),
),
text("Routed expert profile output").size(13),
text_input("Output file path", &self.preference_draft.expert_profile_path)
.on_input(Message::PreferenceExpertProfileChanged)
.padding(9),
text(engine.as_ref().map_or_else(
|| "Effective diagnostic settings will appear after valid values are entered."
.to_owned(),
|engine| format!(
"{} load: simulated memory {} • expert profile {}",
engine.model,
if engine.diagnostics.simulated_used_memory_bytes == 0 {
"off".to_owned()
} else {
format!("{} GiB", engine.diagnostics.simulated_used_memory_bytes / GIB)
},
if engine.diagnostics.expert_profile_path.is_some() { "set" } else { "off" },
),
))
.size(12),
Space::with_height(8),
text("INACTIVITY").size(11),
row![
text_input("10", &self.preference_draft.idle_timeout_minutes)
.on_input(Message::PreferenceTimeoutChanged)
.width(90)
.padding(9),
text("minutes before unloading the model").size(13),
row![
text_input("10", &self.preference_draft.idle_timeout_minutes)
.on_input(Message::PreferenceTimeoutChanged)
.width(90)
.padding(9),
text("minutes before unloading the model").size(13),
]
.spacing(10)
.align_y(Alignment::Center),
text("Enter a whole number from 1 to 1440.").size(12),
]
.spacing(10)
.align_y(Alignment::Center),
text("Enter a whole number from 1 to 1440.").size(12),
.spacing(10),
);
let endpoint_group = preference_group(
"LOCAL ENDPOINT",
column![
preference_input_row(
"Port",
text_input("4000", &self.preference_draft.endpoint_port)
.on_input(Message::PreferenceEndpointPortChanged),
),
text("Listens on 127.0.0.1. Saving a changed port restarts the local endpoint.")
.size(12),
]
.spacing(10),
);
let generation_group = preference_group(
"GENERATION",
column![
preference_input_row(
"Context tokens",
text_input("32768", &self.preference_draft.context_tokens)
.on_input(Message::PreferenceContextChanged),
),
preference_input_row(
"Maximum generated tokens",
text_input("50000", &self.preference_draft.max_generated_tokens)
.on_input(Message::PreferenceMaxTokensChanged),
),
text("System prompt").size(13),
text_input(
"You are a helpful assistant",
&self.preference_draft.system_prompt,
)
.on_input(Message::PreferenceSystemPromptChanged)
.padding(9),
Space::with_height(4),
text("SAMPLING & REASONING").size(11).color(muted_text()),
preference_input_row(
"Temperature",
text_input("DS4 default", &self.preference_draft.temperature)
.on_input(Message::PreferenceTemperatureChanged),
),
preference_input_row(
"Top-p",
text_input("DS4 default", &self.preference_draft.top_p)
.on_input(Message::PreferenceTopPChanged),
),
preference_input_row(
"Min-p",
text_input("DS4 default", &self.preference_draft.min_p)
.on_input(Message::PreferenceMinPChanged),
),
preference_input_row(
"Seed",
text_input("Random", &self.preference_draft.seed)
.on_input(Message::PreferenceSeedChanged),
),
row![
text("Reasoning").size(13).width(Length::Fill),
pick_list(
&REASONING_MODES[..],
Some(self.preference_draft.reasoning_mode),
Message::PreferenceReasoningChanged,
)
.width(240),
]
.spacing(12)
.align_y(Alignment::Center),
text("Blank sampling values retain DS4's model-family defaults. Think Max needs at least 393216 context tokens.")
.size(12),
text(turn.map_or_else(
|| "Effective settings will appear after valid values are entered.".to_owned(),
|settings| format!(
"Effective: {} context • {} max • temp {} • top-p {} • min-p {} • seed {} • {} • system prompt {}",
settings.context_tokens,
settings.max_generated_tokens,
settings.temperature,
settings.top_p,
settings.min_p,
settings.seed.map_or_else(|| "random".to_owned(), |seed| seed.to_string()),
settings.reasoning_mode,
if settings.system_prompt.is_empty() { "off" } else { "on" },
),
))
.size(12),
]
.spacing(10),
);
let execution_group = preference_group(
"EXECUTION",
column![
preference_input_row(
"CPU helper threads",
text_input("Automatic", &self.preference_draft.cpu_threads)
.on_input(Message::PreferenceCpuThreadsChanged),
),
preference_input_row("GPU power percent", power),
preference_input_row("Prefill chunk", prefill),
checkbox("Prefer exact quality kernels", self.preference_draft.quality)
.on_toggle(Message::PreferenceQualityChanged),
checkbox("Warm mapped weights at load time", self.preference_draft.warm_weights)
.on_toggle(Message::PreferenceWarmWeightsChanged),
text(if self.preference_draft.model == ModelChoice::Glm52 {
"GLM 5.2 uses full GPU power and selects prefill chunks automatically."
} else {
"Blank numeric values preserve DS4's automatic engine behavior."
})
.size(12),
text(engine.as_ref().map_or_else(
|| "Effective execution settings will appear after valid values are entered."
.to_owned(),
|engine| {
let settings = engine.execution;
format!(
"Metal engine: threads {} • power {}% • prefill {} • quality {} • warm weights {}",
if settings.cpu_threads == 0 { "auto".to_owned() } else { settings.cpu_threads.to_string() },
if settings.power_percent == 0 { 100 } else { settings.power_percent },
if settings.prefill_chunk == 0 { "auto".to_owned() } else { settings.prefill_chunk.to_string() },
if settings.quality { "on" } else { "off" },
if settings.warm_weights { "on" } else { "off" },
)
},
))
.size(12),
]
.spacing(10),
);
let acceleration_group = preference_group(
"ACCELERATION & MEMORY",
column![
text("SPECULATIVE DECODING").size(11).color(muted_text()),
preference_input_row(
"MTP draft tokens",
text_input("1", &self.preference_draft.mtp_draft_tokens)
.on_input(Message::PreferenceMtpDraftChanged),
),
preference_input_row(
"MTP verifier margin",
text_input("3", &self.preference_draft.mtp_margin)
.on_input(Message::PreferenceMtpMarginChanged),
),
checkbox("Enable integrated GLM MTP", self.preference_draft.glm_mtp)
.on_toggle_maybe(glm_mtp_toggle),
checkbox(
"Log GLM MTP timing counters",
self.preference_draft.glm_mtp_timing,
)
.on_toggle_maybe(glm_mtp_timing_toggle),
dspark,
preference_input_row("DSpark confidence threshold", dspark_confidence),
checkbox(
"DSpark target-only decode",
self.preference_draft.dspark_strict,
)
.on_toggle_maybe(dspark_strict_toggle),
text(if self.preference_draft.model.supports_dspark() {
"DSpark uses the managed support artifact; entering a threshold or enabling strict mode also enables DSpark."
} else if self.preference_draft.model == ModelChoice::Glm52 {
"GLM MTP is integrated; DSpark is unavailable for this model."
} else {
"No managed MTP support artifact is available for this model."
})
.size(12),
text(engine.as_ref().map_or_else(
|| "Effective speculative settings will appear after valid values are entered."
.to_owned(),
|engine| {
let settings = engine.speculative;
format!(
"Engine: MTP draft {} • margin {} • GLM MTP {} • timing {} • DSpark {} • confidence {}{} • target-only {}",
settings.mtp_draft_tokens,
settings.mtp_margin,
if settings.glm_mtp { "on" } else { "off" },
if settings.glm_mtp_timing { "on" } else { "off" },
if settings.dspark { "on" } else { "off" },
settings.dspark_confidence_threshold,
if settings.dspark_confidence_threshold_set { " explicit" } else { " default" },
if settings.dspark_strict { "on" } else { "off" },
)
},
))
.size(12),
Space::with_height(6),
text("SSD STREAMING").size(11).color(muted_text()),
checkbox("Enable SSD-backed model streaming", self.preference_draft.ssd_streaming)
.on_toggle(Message::PreferenceSsdChanged),
checkbox("Skip automatic expert preload", self.preference_draft.ssd_streaming_cold)
.on_toggle(Message::PreferenceSsdColdChanged),
preference_input_row(
"Expert cache count or GiB",
text_input("Automatic, 128, or 64GB", &self.preference_draft.ssd_cache)
.on_input(Message::PreferenceSsdCacheChanged),
),
preference_input_row("Fully resident GLM layers", ssd_full_layers),
preference_input_row(
"Explicit expert preload count",
text_input("Automatic", &self.preference_draft.ssd_preload_experts)
.on_input(Message::PreferenceSsdPreloadChanged),
),
text("A blank full-layer value is automatic; an explicit 0 disables fully resident GLM layers. SSD streaming and DSpark are mutually exclusive.")
.size(12),
text(engine.as_ref().map_or_else(
|| "Effective SSD settings will appear after valid values are entered."
.to_owned(),
|engine| {
let settings = engine.ssd;
let cache = if settings.cache_bytes > 0 {
format!("{} GiB", settings.cache_bytes / GIB)
} else if settings.cache_experts > 0 {
format!("{} experts", settings.cache_experts)
} else {
"auto".to_owned()
};
format!(
"Engine: streaming {} • cold {} • cache {} • full layers {}{} • preload {}",
if settings.enabled { "on" } else { "off" },
if settings.cold { "on" } else { "off" },
cache,
settings.full_layers,
if settings.full_layers_set { " explicit" } else { " auto" },
if settings.preload_experts == 0 { "auto".to_owned() } else { settings.preload_experts.to_string() },
)
},
))
.size(12),
]
.spacing(10),
);
let steering_group = preference_group(
"STEERING & DIAGNOSTICS",
column![
text("DIRECTIONAL STEERING").size(11).color(muted_text()),
text("Direction-vector file").size(13),
steering_file.padding(9),
preference_input_row("FFN scale", steering_ffn),
preference_input_row("Attention scale", steering_attn),
text(if self.preference_draft.model == ModelChoice::Glm52 {
"Directional steering is not supported for GLM 5.2."
} else {
"With a file and no explicit scale, DS4 defaults the FFN scale to 1. Scales accept -100 through 100."
})
.size(12),
text(engine.as_ref().map_or_else(
|| "Effective steering settings will appear after valid values are entered."
.to_owned(),
|engine| format!(
"Engine: file {} • FFN scale {} • attention scale {}",
if engine.steering.file.is_some() { "set" } else { "off" },
engine.steering.ffn_scale,
engine.steering.attention_scale,
),
))
.size(12),
Space::with_height(6),
text("ADVANCED DIAGNOSTICS").size(11).color(muted_text()),
preference_input_row(
"Simulated used memory (GiB)",
text_input("Disabled", &self.preference_draft.simulated_used_memory_gib)
.on_input(Message::PreferenceSimulatedMemoryChanged),
),
text("Routed expert profile output").size(13),
text_input("Output file path", &self.preference_draft.expert_profile_path)
.on_input(Message::PreferenceExpertProfileChanged)
.padding(9),
text(engine.as_ref().map_or_else(
|| "Effective diagnostic settings will appear after valid values are entered."
.to_owned(),
|engine| format!(
"{} load: simulated memory {} • expert profile {}",
engine.model,
if engine.diagnostics.simulated_used_memory_bytes == 0 {
"off".to_owned()
} else {
format!("{} GiB", engine.diagnostics.simulated_used_memory_bytes / GIB)
},
if engine.diagnostics.expert_profile_path.is_some() { "set" } else { "off" },
),
))
.size(12),
]
.spacing(10),
);
let mut fields = column![
model_group,
endpoint_group,
generation_group,
execution_group,
acceleration_group,
steering_group,
]
.spacing(10);
.spacing(12);
if let Some(error) = &self.preference_error {
fields = fields.push(text(error).style(iced::widget::text::danger));
@@ -971,6 +1021,17 @@ fn preference_input_row<'a>(
.into()
}
fn preference_group<'a>(
title: &'a str,
content: impl Into<Element<'a, Message>>,
) -> Element<'a, Message> {
container(column![text(title).size(11).color(muted_text()), content.into(),].spacing(10))
.width(Length::Fill)
.padding(14)
.style(preference_group_style)
.into()
}
fn download_status_bar(download: &ActiveDownload) -> Element<'_, Message> {
let progress = &download.progress;
let percent = progress.fraction() * 100.0;
@@ -1201,6 +1262,18 @@ fn overview_style(_: &Theme) -> container::Style {
}
}
fn preference_group_style(_: &Theme) -> container::Style {
container::Style {
background: Some(Background::Color(Color::from_rgb8(38, 38, 40))),
border: Border {
color: Color::from_rgb8(58, 58, 61),
width: 1.0,
radius: 14.0.into(),
},
..container::Style::default()
}
}
fn chat_message_style(theme: &Theme, user: bool) -> container::Style {
if !user {
return overview_style(theme);
@@ -1247,6 +1320,10 @@ mod tests {
chat_message_style(&theme, true).background,
chat_message_style(&theme, false).background
);
assert_eq!(
preference_group_style(&theme).background,
Some(Background::Color(Color::from_rgb8(38, 38, 40)))
);
}
#[test]

15
src/database.rs
View File

@@ -51,6 +51,7 @@ pub struct AppPreferences {
pub directional_steering_attn: Option<f32>,
pub simulated_used_memory_gib: Option<i64>,
pub expert_profile_path: Option<String>,
pub endpoint_port: i32,
}
impl Default for AppPreferences {
@@ -90,6 +91,7 @@ impl Default for AppPreferences {
directional_steering_attn: None,
simulated_used_memory_gib: None,
expert_profile_path: None,
endpoint_port: 4000,
}
}
}
@@ -234,6 +236,7 @@ struct PreferenceChanges<'a> {
directional_steering_attn: Option<f32>,
simulated_used_memory_gib: Option<i64>,
expert_profile_path: Option<&'a str>,
endpoint_port: i32,
}
#[derive(Clone, Debug, Identifiable, Queryable, Selectable)]
@@ -360,6 +363,7 @@ impl Database {
&mut self,
selected_model: &str,
idle_timeout_minutes: i32,
endpoint_port: i32,
generation: &GenerationPreferences,
runtime: &RuntimePreferences,
) -> Result<AppPreferences, String> {
@@ -367,6 +371,9 @@ impl Database {
let model = crate::model::ModelChoice::from_id(selected_model)
.ok_or_else(|| format!("Unsupported model: {selected_model}"))?;
runtime.validate(model)?;
if !(1..=65_535).contains(&endpoint_port) {
return Err("Endpoint port must be between 1 and 65535.".into());
}
let execution = &runtime.execution;
let speculative = &runtime.speculative;
let (ssd_cache_experts, ssd_cache_gib) = match runtime.ssd.cache {
@@ -416,6 +423,7 @@ impl Database {
.simulated_used_memory_gib
.map(|value| value as i64),
expert_profile_path: runtime.diagnostics.expert_profile_path.as_deref(),
endpoint_port,
})
.returning(AppPreferences::as_returning())
.get_result(&mut self.connection)
@@ -572,6 +580,7 @@ mod tests {
assert_eq!(preferences.selected_model, "deepseek-v4-flash");
assert!(!preferences.dspark_enabled);
assert_eq!(preferences.idle_timeout_minutes, 10);
assert_eq!(preferences.endpoint_port, 4000);
let generation = GenerationPreferences::default();
let runtime = RuntimePreferences::default();
assert!(
@@ -579,6 +588,7 @@ mod tests {
.update_preferences(
"glm-5.2",
30,
4000,
&generation,
&RuntimePreferences {
speculative: SpeculativePreferences {
@@ -592,7 +602,7 @@ mod tests {
);
assert!(
database
.update_preferences("deepseek-v4-flash", 0, &generation, &runtime,)
.update_preferences("deepseek-v4-flash", 0, 4000, &generation, &runtime,)
.is_err()
);
let generation = GenerationPreferences {
@@ -630,7 +640,7 @@ mod tests {
..RuntimePreferences::default()
};
database
.update_preferences("glm-5.2", 30, &generation, &runtime)
.update_preferences("glm-5.2", 30, 4567, &generation, &runtime)
.unwrap();
let project = database.create_project("DS4", "/tmp/ds4").unwrap();
@@ -652,6 +662,7 @@ mod tests {
let preferences = reopened.load_preferences().unwrap();
assert_eq!(preferences.selected_model, "glm-5.2");
assert_eq!(preferences.idle_timeout_minutes, 30);
assert_eq!(preferences.endpoint_port, 4567);
assert_eq!(preferences.generation().unwrap(), generation);
assert_eq!(preferences.runtime().unwrap(), runtime);
drop(reopened);

314
src/engine.rs
View File

@@ -305,6 +305,14 @@ pub(crate) struct ChatTurn {
pub(crate) content: String,
}
pub(crate) struct GenerationOutput {
pub(crate) message: ChatTurn,
pub(crate) prompt_tokens: u32,
pub(crate) cached_tokens: u32,
pub(crate) completion_tokens: u32,
pub(crate) finish_reason: &'static str,
}
#[cfg(target_os = "macos")]
impl Generator {
pub(crate) fn open(settings: &EngineSettings) -> Result<Self, String> {
@@ -337,12 +345,12 @@ impl Generator {
cancelled: &AtomicBool,
mut emit: impl FnMut(bool, String),
mut progress: impl FnMut(u32, u32, Option<f32>),
) -> Result<(), String> {
) -> Result<GenerationOutput, String> {
self.select_checkpoint(checkpoint)?;
let (generated, prompt_complete) =
let (output, prompt_complete) =
self.generate_inner(messages, settings, cancelled, &mut emit, &mut progress)?;
let mut completed = messages.to_vec();
completed.push(generated);
completed.push(output.message.clone());
self.executor.save_checkpoint(
checkpoint,
if prompt_complete {
@@ -350,7 +358,57 @@ impl Generator {
} else {
[0; 32]
},
)
)?;
Ok(output)
}
pub(crate) fn generate_transient(
&mut self,
directory: &Path,
messages: &[ChatTurn],
settings: &TurnSettings,
cancelled: &AtomicBool,
mut emit: impl FnMut(bool, String),
mut progress: impl FnMut(u32, u32, Option<f32>),
) -> Result<GenerationOutput, String> {
let history = messages
.split_last()
.map_or(messages, |(_, history)| history);
let history_tag =
conversation_tag(&settings.system_prompt, settings.reasoning_mode, history);
let checkpoint = directory.join(format!("{}.bin", hex_tag(history_tag)));
if self.executor.checkpoint_tag() != history_tag {
self.select_checkpoint(&checkpoint)?;
if self.executor.checkpoint_tag() != history_tag {
self.executor.reset()?;
self.checkpoint = None;
let _ = std::fs::remove_file(&checkpoint);
}
}
let (output, prompt_complete) =
self.generate_inner(messages, settings, cancelled, &mut emit, &mut progress)?;
let mut completed = messages.to_vec();
completed.push(output.message.clone());
let completed_tag =
conversation_tag(&settings.system_prompt, settings.reasoning_mode, &completed);
std::fs::create_dir_all(directory).map_err(|error| {
format!(
"Could not create transient KV cache directory {}: {error}",
directory.display()
)
})?;
let completed_checkpoint = directory.join(format!("{}.bin", hex_tag(completed_tag)));
self.executor.save_checkpoint(
&completed_checkpoint,
if prompt_complete {
completed_tag
} else {
[0; 32]
},
)?;
self.checkpoint = Some(completed_checkpoint);
Ok(output)
}
fn select_checkpoint(&mut self, checkpoint: &Path) -> Result<(), String> {
@@ -373,7 +431,7 @@ impl Generator {
cancelled: &AtomicBool,
emit: &mut impl FnMut(bool, String),
progress: &mut impl FnMut(u32, u32, Option<f32>),
) -> Result<(ChatTurn, bool), String> {
) -> Result<(GenerationOutput, bool), String> {
let tokens = match messages.split_last() {
Some((latest, history))
if latest.user
@@ -418,10 +476,21 @@ impl Generator {
reasoning_complete: !reasoning,
content: String::new(),
};
let mut emitted_reasoning = 0;
let mut emitted_content = 0;
let prompt_tokens = tokens.len();
for (index, token) in tokens.into_iter().enumerate().skip(reused) {
if cancelled.load(Ordering::Relaxed) {
return Ok((generated, false));
return Ok((
GenerationOutput {
message: generated,
prompt_tokens: prompt_tokens as u32,
cached_tokens: reused as u32,
completion_tokens: 0,
finish_reason: "stop",
},
false,
));
}
self.executor.eval(token)?;
if (index + 1).is_multiple_of(16) || index + 1 == prompt_tokens {
@@ -436,13 +505,30 @@ impl Generator {
.min((max_context - self.executor.position() as usize) as i32)
{
if cancelled.load(Ordering::Relaxed) {
return Ok((generated, true));
flush_generated(
&mut generated,
&mut emitted_reasoning,
&mut emitted_content,
&settings.stops,
emit,
);
return Ok((
GenerationOutput {
message: generated,
prompt_tokens: prompt_tokens as u32,
cached_tokens: reused as u32,
completion_tokens: generated_tokens,
finish_reason: "stop",
},
true,
));
}
let token = sample(
self.executor.logits(),
settings.temperature,
settings.top_p,
settings.min_p,
settings.top_k,
&mut rng,
);
if self
@@ -450,27 +536,89 @@ impl Generator {
.model()
.is_stop_token_for_reasoning(token, settings.reasoning_mode)
{
return Ok((generated, true));
flush_generated(
&mut generated,
&mut emitted_reasoning,
&mut emitted_content,
&settings.stops,
emit,
);
return Ok((
GenerationOutput {
message: generated,
prompt_tokens: prompt_tokens as u32,
cached_tokens: reused as u32,
completion_tokens: generated_tokens,
finish_reason: "stop",
},
true,
));
}
if self.executor.model().is_think_start_token(token) {
reasoning = true;
generated.reasoning.get_or_insert_default();
} else if self.executor.model().is_think_end_token(token) {
if let Some(reasoning_text) = &mut generated.reasoning
&& emit_safe_text(
reasoning_text,
&mut emitted_reasoning,
&settings.stops,
true,
true,
emit,
)
{
return Ok((
GenerationOutput {
message: generated,
prompt_tokens: prompt_tokens as u32,
cached_tokens: reused as u32,
completion_tokens: generated_tokens + 1,
finish_reason: "stop",
},
false,
));
}
reasoning = false;
generated.reasoning_complete = true;
emit(false, String::new());
} else if let Some(bytes) = self.executor.model().token_bytes(token) {
let content = String::from_utf8_lossy(&bytes).into_owned();
if reasoning {
generated
.reasoning
.get_or_insert_default()
.push_str(&content);
let stopped = if reasoning {
let text = generated.reasoning.get_or_insert_default();
text.push_str(&content);
emit_safe_text(
text,
&mut emitted_reasoning,
&settings.stops,
false,
true,
emit,
)
} else {
generated.reasoning_complete = true;
generated.content.push_str(&content);
emit_safe_text(
&mut generated.content,
&mut emitted_content,
&settings.stops,
false,
false,
emit,
)
};
if stopped {
return Ok((
GenerationOutput {
message: generated,
prompt_tokens: prompt_tokens as u32,
cached_tokens: reused as u32,
completion_tokens: generated_tokens + 1,
finish_reason: "stop",
},
false,
));
}
emit(reasoning, content);
}
self.executor.eval(token)?;
generated_tokens += 1;
@@ -483,10 +631,103 @@ impl Generator {
),
);
}
Ok((generated, true))
flush_generated(
&mut generated,
&mut emitted_reasoning,
&mut emitted_content,
&settings.stops,
emit,
);
Ok((
GenerationOutput {
message: generated,
prompt_tokens: prompt_tokens as u32,
cached_tokens: reused as u32,
completion_tokens: generated_tokens,
finish_reason: "length",
},
true,
))
}
}
#[cfg(target_os = "macos")]
fn flush_generated(
generated: &mut ChatTurn,
emitted_reasoning: &mut usize,
emitted_content: &mut usize,
stops: &[String],
emit: &mut impl FnMut(bool, String),
) {
if let Some(reasoning) = &mut generated.reasoning {
let _ = emit_safe_text(reasoning, emitted_reasoning, stops, true, true, emit);
}
let _ = emit_safe_text(
&mut generated.content,
emitted_content,
stops,
true,
false,
emit,
);
}
#[cfg(target_os = "macos")]
fn emit_safe_text(
text: &mut String,
emitted: &mut usize,
stops: &[String],
final_flush: bool,
reasoning: bool,
emit: &mut impl FnMut(bool, String),
) -> bool {
let stop = stops
.iter()
.filter_map(|stop| {
text[*emitted..]
.find(stop)
.map(|position| *emitted + position)
})
.min();
if let Some(stop) = stop {
if stop > *emitted {
emit(reasoning, text[*emitted..stop].to_owned());
}
text.truncate(stop);
*emitted = stop;
return true;
}
let hold = if final_flush {
0
} else {
stops
.iter()
.map(|stop| stop.len().saturating_sub(1))
.max()
.unwrap_or(0)
};
let mut safe = text.len().saturating_sub(hold);
while safe > *emitted && !text.is_char_boundary(safe) {
safe -= 1;
}
if safe > *emitted {
emit(reasoning, text[*emitted..safe].to_owned());
*emitted = safe;
}
false
}
#[cfg(target_os = "macos")]
fn hex_tag(tag: [u8; 32]) -> String {
const HEX: &[u8; 16] = b"0123456789abcdef";
let mut output = String::with_capacity(64);
for byte in tag {
output.push(HEX[(byte >> 4) as usize] as char);
output.push(HEX[(byte & 0x0f) as usize] as char);
}
output
}
#[cfg(target_os = "macos")]
fn conversation_tag(system: &str, reasoning: ReasoningMode, messages: &[ChatTurn]) -> [u8; 32] {
fn text(hasher: &mut Sha256, value: &str) {
@@ -518,7 +759,14 @@ fn conversation_tag(system: &str, reasoning: ReasoningMode, messages: &[ChatTurn
}
#[cfg(target_os = "macos")]
fn sample(logits: &[f32], temperature: f32, top_p: f32, min_p: f32, rng: &mut Rng) -> i32 {
fn sample(
logits: &[f32],
temperature: f32,
top_p: f32,
min_p: f32,
top_k: i32,
rng: &mut Rng,
) -> i32 {
if temperature <= 0.0 {
return logits
.iter()
@@ -554,12 +802,16 @@ fn sample(logits: &[f32], temperature: f32, top_p: f32, min_p: f32, rng: &mut Rn
.max_by(|a, b| a.1.total_cmp(b.1))
.map_or(0, |(index, _)| index as i32);
}
if top_p < 1.0 {
if top_p < 1.0 || top_k > 0 {
probabilities.sort_unstable_by(|a, b| b.1.total_cmp(&a.1).then_with(|| a.0.cmp(&b.0)));
let total: f32 = logits
if top_k > 0 {
probabilities.truncate(probabilities.len().min(top_k as usize));
}
}
if top_p < 1.0 {
let total: f32 = probabilities
.iter()
.filter(|logit| logit.is_finite())
.map(|logit| ((*logit - maximum) / temperature).exp())
.map(|(_, probability)| probability)
.sum();
let mut kept = 0.0;
let count = probabilities
@@ -612,7 +864,27 @@ mod sampling_tests {
#[test]
fn zero_temperature_is_greedy() {
let mut rng = Rng::new(1);
assert_eq!(sample(&[1.0, 4.0, 2.0], 0.0, 1.0, 0.0, &mut rng), 1);
assert_eq!(sample(&[1.0, 4.0, 2.0], 0.0, 1.0, 0.0, 0, &mut rng), 1);
}
#[test]
fn top_k_and_stream_stops_are_applied_before_output() {
let mut rng = Rng::new(1);
assert_eq!(sample(&[1.0, 4.0, 2.0], 1.0, 1.0, 0.0, 1, &mut rng), 1);
let mut text = "hello STOP hidden".to_owned();
let mut emitted = 0;
let mut chunks = Vec::new();
assert!(emit_safe_text(
&mut text,
&mut emitted,
&["STOP".into()],
false,
false,
&mut |_, chunk| chunks.push(chunk),
));
assert_eq!(text, "hello ");
assert_eq!(chunks, ["hello "]);
}
#[test]

6
src/main.rs
View File

@@ -3,8 +3,14 @@ mod database;
mod engine;
mod model;
#[cfg(target_os = "macos")]
mod native_edit;
#[cfg(target_os = "macos")]
mod native_menu;
#[cfg(target_os = "macos")]
mod runtime;
mod schema;
#[cfg(target_os = "macos")]
mod server;
mod settings;
use app::{App, Message, app_icon, app_theme};

7
src/model.rs
View File

@@ -327,6 +327,13 @@ pub(crate) fn managed_artifacts(models_path: &Path) -> Vec<ManagedArtifact> {
.collect()
}
pub(crate) fn installed_models(models_path: &Path) -> Vec<ModelChoice> {
MODEL_CHOICES
.into_iter()
.filter(|model| model.main_artifact().is_installed(*model, models_path))
.collect()
}
pub(crate) fn artifact_download_progress(
id: ManagedArtifactId,
models_path: &Path,

315
src/native_edit.rs Normal file
View File

@@ -0,0 +1,315 @@
use std::collections::VecDeque;
use std::sync::{Arc, Mutex};
use iced::advanced::layout;
use iced::advanced::overlay;
use iced::advanced::renderer;
use iced::advanced::widget::{Operation, Tree, tree};
use iced::advanced::{Clipboard, Layout, Shell, Widget};
use iced::event;
use iced::keyboard::{self, Key, Location, Modifiers, key};
use iced::mouse;
use iced::{Element, Event, Length, Rectangle, Size, Vector};
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum EditCommand {
Undo,
Redo,
Cut,
Copy,
Paste,
SelectAll,
}
pub(crate) type EditCommandQueue = Arc<Mutex<VecDeque<EditCommand>>>;
pub(crate) fn command_queue() -> EditCommandQueue {
Arc::new(Mutex::new(VecDeque::new()))
}
pub(crate) fn queue_command(queue: &EditCommandQueue, command: EditCommand) {
queue
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.push_back(command);
}
fn pop_command(queue: &EditCommandQueue) -> Option<EditCommand> {
queue
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.pop_front()
}
/// Replays native Edit menu actions through the focused Iced widget.
pub(crate) struct NativeEdit<'a, Message, Theme = iced::Theme, Renderer = iced::Renderer> {
content: Element<'a, Message, Theme, Renderer>,
commands: EditCommandQueue,
}
impl<'a, Message, Theme, Renderer> NativeEdit<'a, Message, Theme, Renderer> {
fn new(
content: impl Into<Element<'a, Message, Theme, Renderer>>,
commands: EditCommandQueue,
) -> Self {
Self {
content: content.into(),
commands,
}
}
}
#[derive(Default)]
struct State;
fn modifier_sync_event(event: &Event) -> Option<Event> {
let Event::Keyboard(keyboard::Event::KeyPressed { modifiers, .. }) = event else {
return None;
};
Some(Event::Keyboard(keyboard::Event::ModifiersChanged(
*modifiers,
)))
}
fn command_events(command: EditCommand) -> [Event; 4] {
let (character, modified_character, physical_key, modifiers) = match command {
EditCommand::Undo => ("z", "z", key::Code::KeyZ, Modifiers::COMMAND),
EditCommand::Redo => (
"z",
"Z",
key::Code::KeyZ,
Modifiers::COMMAND | Modifiers::SHIFT,
),
EditCommand::Cut => ("x", "x", key::Code::KeyX, Modifiers::COMMAND),
EditCommand::Copy => ("c", "c", key::Code::KeyC, Modifiers::COMMAND),
EditCommand::Paste => ("v", "v", key::Code::KeyV, Modifiers::COMMAND),
EditCommand::SelectAll => ("a", "a", key::Code::KeyA, Modifiers::COMMAND),
};
let key = Key::Character(character.into());
[
Event::Keyboard(keyboard::Event::ModifiersChanged(modifiers)),
Event::Keyboard(keyboard::Event::KeyPressed {
key: key.clone(),
modified_key: Key::Character(modified_character.into()),
physical_key: key::Physical::Code(physical_key),
location: Location::Standard,
modifiers,
text: None,
}),
Event::Keyboard(keyboard::Event::KeyReleased {
key,
location: Location::Standard,
modifiers,
}),
Event::Keyboard(keyboard::Event::ModifiersChanged(Modifiers::default())),
]
}
impl<'a, Message, Theme, Renderer> Widget<Message, Theme, Renderer>
for NativeEdit<'a, Message, Theme, Renderer>
where
Renderer: renderer::Renderer,
{
fn tag(&self) -> tree::Tag {
tree::Tag::of::<State>()
}
fn state(&self) -> tree::State {
tree::State::new(State)
}
fn children(&self) -> Vec<Tree> {
vec![Tree::new(&self.content)]
}
fn diff(&self, tree: &mut Tree) {
tree.diff_children(std::slice::from_ref(&self.content));
}
fn size(&self) -> Size<Length> {
self.content.as_widget().size()
}
fn layout(
&self,
tree: &mut Tree,
renderer: &Renderer,
limits: &layout::Limits,
) -> layout::Node {
self.content
.as_widget()
.layout(&mut tree.children[0], renderer, limits)
}
fn operate(
&self,
tree: &mut Tree,
layout: Layout<'_>,
renderer: &Renderer,
operation: &mut dyn Operation,
) {
self.content
.as_widget()
.operate(&mut tree.children[0], layout, renderer, operation);
}
fn on_event(
&mut self,
tree: &mut Tree,
event: Event,
layout: Layout<'_>,
cursor: mouse::Cursor,
renderer: &Renderer,
clipboard: &mut dyn Clipboard,
shell: &mut Shell<'_, Message>,
viewport: &Rectangle,
) -> event::Status {
let mut status = event::Status::Ignored;
if matches!(
event,
Event::Window(iced::window::Event::RedrawRequested(_))
) {
while let Some(command) = pop_command(&self.commands) {
for command_event in command_events(command) {
if self.content.as_widget_mut().on_event(
&mut tree.children[0],
command_event,
layout,
cursor,
renderer,
clipboard,
shell,
viewport,
) == event::Status::Captured
{
status = event::Status::Captured;
}
}
}
}
if let Some(sync_event) = modifier_sync_event(&event) {
let _ = self.content.as_widget_mut().on_event(
&mut tree.children[0],
sync_event,
layout,
cursor,
renderer,
clipboard,
shell,
viewport,
);
}
if self.content.as_widget_mut().on_event(
&mut tree.children[0],
event,
layout,
cursor,
renderer,
clipboard,
shell,
viewport,
) == event::Status::Captured
{
event::Status::Captured
} else {
status
}
}
fn mouse_interaction(
&self,
tree: &Tree,
layout: Layout<'_>,
cursor: mouse::Cursor,
viewport: &Rectangle,
renderer: &Renderer,
) -> mouse::Interaction {
self.content.as_widget().mouse_interaction(
&tree.children[0],
layout,
cursor,
viewport,
renderer,
)
}
fn draw(
&self,
tree: &Tree,
renderer: &mut Renderer,
theme: &Theme,
style: &renderer::Style,
layout: Layout<'_>,
cursor: mouse::Cursor,
viewport: &Rectangle,
) {
self.content.as_widget().draw(
&tree.children[0],
renderer,
theme,
style,
layout,
cursor,
viewport,
);
}
fn overlay<'b>(
&'b mut self,
tree: &'b mut Tree,
layout: Layout<'_>,
renderer: &Renderer,
translation: Vector,
) -> Option<overlay::Element<'b, Message, Theme, Renderer>> {
self.content
.as_widget_mut()
.overlay(&mut tree.children[0], layout, renderer, translation)
}
}
impl<'a, Message, Theme, Renderer> From<NativeEdit<'a, Message, Theme, Renderer>>
for Element<'a, Message, Theme, Renderer>
where
Message: 'a,
Theme: 'a,
Renderer: 'a + renderer::Renderer,
{
fn from(bridge: NativeEdit<'a, Message, Theme, Renderer>) -> Self {
Element::new(bridge)
}
}
pub(crate) fn native_edit<'a, Message, Theme, Renderer>(
content: impl Into<Element<'a, Message, Theme, Renderer>>,
commands: EditCommandQueue,
) -> NativeEdit<'a, Message, Theme, Renderer> {
NativeEdit::new(content, commands)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn paste_replays_a_complete_command_shortcut() {
let events = command_events(EditCommand::Paste);
assert!(matches!(
events[0],
Event::Keyboard(keyboard::Event::ModifiersChanged(Modifiers::COMMAND))
));
assert!(matches!(
events[1],
Event::Keyboard(keyboard::Event::KeyPressed {
key: Key::Character(ref key),
modifiers: Modifiers::COMMAND,
..
}) if key == "v"
));
assert!(matches!(
events[3],
Event::Keyboard(keyboard::Event::ModifiersChanged(modifiers)) if modifiers.is_empty()
));
}
}

76
src/native_menu.rs
View File

@@ -1,8 +1,16 @@
use muda::accelerator::{Accelerator, Code, Modifiers};
use muda::accelerator::{Accelerator, CMD_OR_CTRL, Code, Modifiers};
use muda::{Menu, MenuEvent, MenuItem, PredefinedMenuItem, Submenu};
use crate::native_edit::EditCommand;
const PREFERENCES: &str = "preferences";
const MODEL_MANAGER: &str = "model-manager";
const UNDO: &str = "undo";
const REDO: &str = "redo";
const CUT: &str = "cut";
const COPY: &str = "copy";
const PASTE: &str = "paste";
const SELECT_ALL: &str = "select-all";
pub(crate) struct NativeMenu {
_menu: Menu,
@@ -12,6 +20,7 @@ pub(crate) struct NativeMenu {
pub(crate) enum NativeMenuEvent {
Preferences,
ModelManager,
Edit(EditCommand),
}
pub(crate) fn install() -> Result<NativeMenu, String> {
@@ -34,6 +43,12 @@ pub(crate) fn install() -> Result<NativeMenu, String> {
Code::KeyM,
)),
);
let undo = edit_item(UNDO, "Undo", Code::KeyZ, None);
let redo = edit_item(REDO, "Redo", Code::KeyZ, Some(Modifiers::SHIFT));
let cut = edit_item(CUT, "Cut", Code::KeyX, None);
let copy = edit_item(COPY, "Copy", Code::KeyC, None);
let paste = edit_item(PASTE, "Paste", Code::KeyV, None);
let select_all = edit_item(SELECT_ALL, "Select All", Code::KeyA, None);
application
.append_items(&[
@@ -51,13 +66,13 @@ pub(crate) fn install() -> Result<NativeMenu, String> {
])
.map_err(|error| error.to_string())?;
edit.append_items(&[
&PredefinedMenuItem::undo(None),
&PredefinedMenuItem::redo(None),
&undo,
&redo,
&PredefinedMenuItem::separator(),
&PredefinedMenuItem::cut(None),
&PredefinedMenuItem::copy(None),
&PredefinedMenuItem::paste(None),
&PredefinedMenuItem::select_all(None),
&cut,
&copy,
&paste,
&select_all,
])
.map_err(|error| error.to_string())?;
window
@@ -79,12 +94,47 @@ pub(crate) fn install() -> Result<NativeMenu, String> {
pub(crate) fn next_event() -> Option<NativeMenuEvent> {
while let Ok(event) = MenuEvent::receiver().try_recv() {
if event.id == PREFERENCES {
return Some(NativeMenuEvent::Preferences);
}
if event.id == MODEL_MANAGER {
return Some(NativeMenuEvent::ModelManager);
}
let event = match event.id.0.as_str() {
PREFERENCES => NativeMenuEvent::Preferences,
MODEL_MANAGER => NativeMenuEvent::ModelManager,
UNDO => NativeMenuEvent::Edit(EditCommand::Undo),
REDO => NativeMenuEvent::Edit(EditCommand::Redo),
CUT => NativeMenuEvent::Edit(EditCommand::Cut),
COPY => NativeMenuEvent::Edit(EditCommand::Copy),
PASTE => NativeMenuEvent::Edit(EditCommand::Paste),
SELECT_ALL => NativeMenuEvent::Edit(EditCommand::SelectAll),
_ => continue,
};
return Some(event);
}
None
}
fn edit_item(
id: &'static str,
label: &'static str,
code: Code,
extra_modifier: Option<Modifiers>,
) -> MenuItem {
MenuItem::with_id(
id,
label,
true,
Some(Accelerator::new(
Some(CMD_OR_CTRL | extra_modifier.unwrap_or_else(Modifiers::empty)),
code,
)),
)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn edit_items_are_custom_menu_events() {
let paste = edit_item(PASTE, "Paste", Code::KeyV, None);
assert_eq!(paste.id().0, PASTE);
assert!(paste.is_enabled());
}
}

151
src/runtime.rs Normal file
View File

@@ -0,0 +1,151 @@
use crate::engine::{ChatTurn, GenerationOutput, Generator};
use crate::settings::{EngineSettings, TurnSettings};
use std::path::PathBuf;
use std::sync::Arc;
use std::sync::atomic::AtomicBool;
use std::sync::mpsc::{self, Receiver, Sender};
use std::thread;
use std::time::{Duration, Instant};
#[derive(Clone)]
pub(crate) struct GenerationService {
commands: Sender<Command>,
}
pub(crate) struct ActiveGeneration {
pub(crate) events: Receiver<GenerationEvent>,
pub(crate) cancel: Arc<AtomicBool>,
}
pub(crate) enum CheckpointTarget {
Local(PathBuf),
Transient(PathBuf),
}
pub(crate) enum GenerationEvent {
Loading,
Chunk {
reasoning: bool,
content: String,
},
Context {
used: u32,
limit: u32,
tokens_per_second: Option<f32>,
},
Finished(Result<GenerationOutput, String>),
}
struct Command {
engine: EngineSettings,
turn: TurnSettings,
messages: Vec<ChatTurn>,
checkpoint: CheckpointTarget,
idle_timeout: Duration,
cancel: Arc<AtomicBool>,
events: Sender<GenerationEvent>,
}
impl GenerationService {
pub(crate) fn spawn() -> Result<Self, String> {
let (commands, receiver) = mpsc::channel::<Command>();
thread::Builder::new()
.name("model-runtime".into())
.spawn(move || run(receiver))
.map_err(|error| format!("Could not start the model runtime: {error}"))?;
Ok(Self { commands })
}
pub(crate) fn generate(
&self,
engine: EngineSettings,
turn: TurnSettings,
messages: Vec<ChatTurn>,
checkpoint: CheckpointTarget,
idle_timeout: Duration,
) -> Result<ActiveGeneration, String> {
let cancel = Arc::new(AtomicBool::new(false));
let (events, receiver) = mpsc::channel();
self.commands
.send(Command {
engine,
turn,
messages,
checkpoint,
idle_timeout,
cancel: Arc::clone(&cancel),
events,
})
.map_err(|_| "The model runtime stopped unexpectedly.".to_owned())?;
Ok(ActiveGeneration {
events: receiver,
cancel,
})
}
}
fn run(commands: Receiver<Command>) {
let mut loaded = None::<(EngineSettings, Generator)>;
let mut last_used = Instant::now();
let mut idle_timeout = Duration::from_secs(15 * 60);
loop {
match commands.recv_timeout(Duration::from_secs(1)) {
Ok(command) => {
idle_timeout = command.idle_timeout;
if loaded
.as_ref()
.is_none_or(|(settings, _)| settings != &command.engine)
{
let _ = command.events.send(GenerationEvent::Loading);
loaded = match Generator::open(&command.engine) {
Ok(generator) => Some((command.engine.clone(), generator)),
Err(error) => {
let _ = command.events.send(GenerationEvent::Finished(Err(error)));
None
}
};
}
if let Some((_, generator)) = &mut loaded {
let mut emit = |reasoning, content| {
let _ = command
.events
.send(GenerationEvent::Chunk { reasoning, content });
};
let mut progress = |used, limit, tokens_per_second| {
let _ = command.events.send(GenerationEvent::Context {
used,
limit,
tokens_per_second,
});
};
let result = match command.checkpoint {
CheckpointTarget::Local(path) => generator.generate(
&path,
&command.messages,
&command.turn,
&command.cancel,
&mut emit,
&mut progress,
),
CheckpointTarget::Transient(directory) => generator.generate_transient(
&directory,
&command.messages,
&command.turn,
&command.cancel,
&mut emit,
&mut progress,
),
};
let _ = command.events.send(GenerationEvent::Finished(result));
last_used = Instant::now();
}
}
Err(mpsc::RecvTimeoutError::Timeout) => {
if loaded.is_some() && last_used.elapsed() >= idle_timeout {
loaded = None;
}
}
Err(mpsc::RecvTimeoutError::Disconnected) => break,
}
}
}

1
src/schema.rs
View File

@@ -53,6 +53,7 @@ diesel::table! {
directional_steering_attn -> Nullable<Float>,
simulated_used_memory_gib -> Nullable<BigInt>,
expert_profile_path -> Nullable<Text>,
endpoint_port -> Integer,
}
}

1116
src/server.rs Normal file
View File

File diff suppressed because it is too large Load Diff

4
src/settings.rs
View File

@@ -443,6 +443,8 @@ impl GenerationPreferences {
temperature: self.temperature.unwrap_or(1.0),
top_p: self.top_p.unwrap_or(if glm { 0.95 } else { 1.0 }),
min_p: self.min_p.unwrap_or(if glm { 0.0 } else { 0.05 }),
top_k: 0,
stops: Vec::new(),
seed: self.seed,
reasoning_mode: if self.reasoning_mode == ReasoningMode::Max
&& self.context_tokens < THINK_MAX_MIN_CONTEXT
@@ -489,6 +491,8 @@ pub(crate) struct TurnSettings {
pub(crate) temperature: f32,
pub(crate) top_p: f32,
pub(crate) min_p: f32,
pub(crate) top_k: i32,
pub(crate) stops: Vec<String>,
pub(crate) seed: Option<u64>,
pub(crate) reasoning_mode: ReasoningMode,
}