diff --git a/Cargo.toml b/Cargo.toml index 1eeb3a7..3ef1d66 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ license = "MIT" publish = false [dependencies] -diesel = { version = "2.3.11", features = ["sqlite", "returning_clauses_for_sqlite_3_35"] } +diesel = { version = "2.3.11", features = ["sqlite", "returning_clauses_for_sqlite_3_35", "64-column-tables"] } diesel_migrations = "2.3.2" iced = { version = "0.13.1", features = ["svg", "tokio"] } png = "0.17.16" diff --git a/PLAN.md b/PLAN.md index 3a54e08..e72e057 100644 --- a/PLAN.md +++ b/PLAN.md @@ -15,7 +15,7 @@ sessions, tools, and turn orchestration. | App shell | Implemented | Native multi-window Iced app, Codex-inspired layout, native Application/Edit/Window menus, Preferences panel, and Model Manager window | Functional chat content and native app packaging | | Projects and sessions | Implemented for metadata | Native folder picker, project-name dialog, create/select/delete projects and sessions | Transcripts, KV payloads, rename/archive, and model binding | | Persistence | Implemented for metadata and preferences | SQLite in Application Support, Diesel ORM, embedded migrations, constrained defaults and CRUD tests | Messages, downloaded-artifact metadata, and session-runtime migrations | -| Model runtime | Download workflow implemented | Persisted model choice, DSpark toggle, idle timeout, and a Model Manager for background download, validation, deletion, progress, ETA, stop, and restart-resume | Loading, inference, idle unloading, and Metal integration | +| Model runtime | Preferences and downloads implemented | Complete typed DS4 preference contract plus a Model Manager for background download, validation, deletion, progress, ETA, stop, and restart-resume | Loading, inference, idle unloading, and Metal integration | | Local endpoint | Not started | Nothing listening | OpenAI-compatible HTTP and streaming surfaces | | Agent | UI shell only | Empty conversation pane and composer placeholder | Messages, generation, tools, approvals, compaction, and cancellation | | A2UI | Future extension | bDS2 provides the reference structured render-tool contract | Native inline surfaces for local chat after core chat and tools are stable | @@ -45,12 +45,12 @@ Exit criterion: restart the app and see the same project/session tree. ## Phase 1 — model library and on-demand loading -Status: **partially implemented**. Typed generation and execution preferences -and the complete background download workflow are implemented; the remaining -DS4 runtime preferences, loading, inference, and idle unloading remain open. +Status: **partially implemented**. The typed DS4 preference contract and the +complete background download workflow are implemented; loading, inference, +and idle unloading remain open. -1. Complete the typed model/runtime preference contract described below before - connecting chat, the HTTP endpoint, or the engine lifecycle. +1. **Implemented:** the typed model/runtime preference contract described below + is complete before chat, the HTTP endpoint, or the engine lifecycle. 2. Port the narrow GGUF loader, tokenizer, prompt renderer, and Metal session API behind a small Rust `Engine` surface modeled on `ds4.h`: `open`, `close`, `create_session`, `sync`, `sample`, and KV save/load. @@ -136,7 +136,7 @@ to Rust while preserving its semantics and on-disk compatibility: - Persist preferences in the application SQLite database. Changing preferences never loads a model by itself. -### DS4 model and generation preferences (required before chat) +### DS4 model and generation preferences (implemented) The Preferences panel must expose every persistent model, runtime, and generation parameter accepted by the `ds4` command line. `../ds4/ds4_cli.c`, diff --git a/README.md b/README.md index 4280929..c9aa2da 100644 --- a/README.md +++ b/README.md @@ -7,8 +7,9 @@ OpenAI-compatible localhost endpoint, and project-scoped agent chat in one app. The current milestone provides a Codex-inspired project/session layout. A native macOS folder picker selects each workspace, then the app asks for its display name. Projects, sessions, and model preferences are persisted through Diesel in -SQLite. Open Preferences with `Command-,` to select the model, toggle DSpark, -and configure the model idle timeout. The separate Model Manager (`Shift-Command-M`) +SQLite. Open Preferences with `Command-,` to configure model, generation, +runtime, and idle-unload settings. The separate Model Manager +(`Shift-Command-M`) lists local main and DSpark artifacts, their on-disk sizes and state, and lets you download, resume, validate, or delete them. Rust-native background work shows live byte progress, speed, and ETA in Model Manager and the app status diff --git a/migrations/20260724220000_add_runtime_preferences/down.sql b/migrations/20260724220000_add_runtime_preferences/down.sql new file mode 100644 index 0000000..38d1279 --- /dev/null +++ b/migrations/20260724220000_add_runtime_preferences/down.sql @@ -0,0 +1,17 @@ +ALTER TABLE preferences DROP COLUMN expert_profile_path; +ALTER TABLE preferences DROP COLUMN simulated_used_memory_gib; +ALTER TABLE preferences DROP COLUMN directional_steering_attn; +ALTER TABLE preferences DROP COLUMN directional_steering_ffn; +ALTER TABLE preferences DROP COLUMN directional_steering_file; +ALTER TABLE preferences DROP COLUMN ssd_preload_experts; +ALTER TABLE preferences DROP COLUMN ssd_full_layers; +ALTER TABLE preferences DROP COLUMN ssd_cache_gib; +ALTER TABLE preferences DROP COLUMN ssd_cache_experts; +ALTER TABLE preferences DROP COLUMN ssd_streaming_cold; +ALTER TABLE preferences DROP COLUMN ssd_streaming; +ALTER TABLE preferences DROP COLUMN dspark_strict; +ALTER TABLE preferences DROP COLUMN dspark_confidence_threshold; +ALTER TABLE preferences DROP COLUMN glm_mtp_timing; +ALTER TABLE preferences DROP COLUMN glm_mtp; +ALTER TABLE preferences DROP COLUMN mtp_margin; +ALTER TABLE preferences DROP COLUMN mtp_draft_tokens; diff --git a/migrations/20260724220000_add_runtime_preferences/up.sql b/migrations/20260724220000_add_runtime_preferences/up.sql new file mode 100644 index 0000000..0f92c5f --- /dev/null +++ b/migrations/20260724220000_add_runtime_preferences/up.sql @@ -0,0 +1,35 @@ +ALTER TABLE preferences ADD COLUMN mtp_draft_tokens INTEGER NOT NULL DEFAULT 1 + CHECK (mtp_draft_tokens > 0); +ALTER TABLE preferences ADD COLUMN mtp_margin REAL NOT NULL DEFAULT 3 + CHECK (mtp_margin BETWEEN 0 AND 1000); +ALTER TABLE preferences ADD COLUMN glm_mtp BOOLEAN NOT NULL DEFAULT 0 + CHECK (glm_mtp IN (0, 1)); +ALTER TABLE preferences ADD COLUMN glm_mtp_timing BOOLEAN NOT NULL DEFAULT 0 + CHECK (glm_mtp_timing IN (0, 1)); +ALTER TABLE preferences ADD COLUMN dspark_confidence_threshold REAL + CHECK (dspark_confidence_threshold IS NULL OR dspark_confidence_threshold BETWEEN 0 AND 1); +ALTER TABLE preferences ADD COLUMN dspark_strict BOOLEAN NOT NULL DEFAULT 0 + CHECK (dspark_strict IN (0, 1)); + +ALTER TABLE preferences ADD COLUMN ssd_streaming BOOLEAN NOT NULL DEFAULT 0 + CHECK (ssd_streaming IN (0, 1)); +ALTER TABLE preferences ADD COLUMN ssd_streaming_cold BOOLEAN NOT NULL DEFAULT 0 + CHECK (ssd_streaming_cold IN (0, 1)); +ALTER TABLE preferences ADD COLUMN ssd_cache_experts BIGINT + CHECK (ssd_cache_experts IS NULL OR ssd_cache_experts > 0); +ALTER TABLE preferences ADD COLUMN ssd_cache_gib BIGINT + CHECK (ssd_cache_gib IS NULL OR ssd_cache_gib > 0); +ALTER TABLE preferences ADD COLUMN ssd_full_layers INTEGER + CHECK (ssd_full_layers IS NULL OR ssd_full_layers >= 0); +ALTER TABLE preferences ADD COLUMN ssd_preload_experts INTEGER + CHECK (ssd_preload_experts IS NULL OR ssd_preload_experts > 0); + +ALTER TABLE preferences ADD COLUMN directional_steering_file TEXT; +ALTER TABLE preferences ADD COLUMN directional_steering_ffn REAL + CHECK (directional_steering_ffn IS NULL OR directional_steering_ffn BETWEEN -100 AND 100); +ALTER TABLE preferences ADD COLUMN directional_steering_attn REAL + CHECK (directional_steering_attn IS NULL OR directional_steering_attn BETWEEN -100 AND 100); + +ALTER TABLE preferences ADD COLUMN simulated_used_memory_gib BIGINT + CHECK (simulated_used_memory_gib IS NULL OR simulated_used_memory_gib > 0); +ALTER TABLE preferences ADD COLUMN expert_profile_path TEXT; diff --git a/src/app.rs b/src/app.rs index d7dcd54..0987424 100644 --- a/src/app.rs +++ b/src/app.rs @@ -4,7 +4,11 @@ pub(crate) use view::app_theme; use crate::database::{AppPreferences, Database, ProjectWithSessions}; use crate::model::{self, DownloadOutcome, DownloadProgress, ManagedArtifactId, ModelChoice}; -use crate::settings::{ExecutionPreferences, GenerationPreferences, ReasoningMode}; +use crate::settings::{ + DiagnosticPreferences, ExecutionPreferences, GIB, GenerationPreferences, ReasoningMode, + RuntimePreferences, SpeculativePreferences, SsdPreferences, SteeringPreferences, + StreamingCacheBudget, +}; use iced::{Size, Subscription, Task, keyboard, window}; use rfd::AsyncFileDialog; use std::fs; @@ -35,6 +39,22 @@ struct PreferenceDraft { prefill_chunk: String, quality: bool, warm_weights: bool, + mtp_draft_tokens: String, + mtp_margin: String, + glm_mtp: bool, + glm_mtp_timing: bool, + dspark_confidence_threshold: String, + dspark_strict: bool, + ssd_streaming: bool, + ssd_streaming_cold: bool, + ssd_cache: String, + ssd_full_layers: String, + ssd_preload_experts: String, + directional_steering_file: String, + directional_steering_ffn: String, + directional_steering_attn: String, + simulated_used_memory_gib: String, + expert_profile_path: String, } impl PreferenceDraft { @@ -42,11 +62,13 @@ impl PreferenceDraft { let model = ModelChoice::from_id(&preferences.selected_model) .ok_or_else(|| format!("Unsupported model: {}", preferences.selected_model))?; let generation = preferences.generation()?; - let execution = preferences.execution()?; - execution.validate(model)?; + let runtime = preferences.runtime()?; + runtime.engine_settings(model)?; + let execution = &runtime.execution; + let speculative = &runtime.speculative; Ok(Self { model, - dspark_enabled: model.supports_dspark() && preferences.dspark_enabled, + dspark_enabled: speculative.dspark_enabled, idle_timeout_minutes: preferences.idle_timeout_minutes.to_string(), context_tokens: generation.context_tokens.to_string(), max_generated_tokens: generation.max_generated_tokens.to_string(), @@ -75,6 +97,45 @@ impl PreferenceDraft { .map_or_else(String::new, |value| value.to_string()), quality: execution.quality, warm_weights: execution.warm_weights, + mtp_draft_tokens: speculative.mtp_draft_tokens.to_string(), + mtp_margin: speculative.mtp_margin.to_string(), + glm_mtp: speculative.glm_mtp, + glm_mtp_timing: speculative.glm_mtp_timing, + dspark_confidence_threshold: speculative + .dspark_confidence_threshold + .map_or_else(String::new, |value| value.to_string()), + dspark_strict: speculative.dspark_strict, + ssd_streaming: runtime.ssd.enabled, + ssd_streaming_cold: runtime.ssd.cold, + ssd_cache: runtime + .ssd + .cache + .map_or_else(String::new, |cache| match cache { + StreamingCacheBudget::Experts(experts) => experts.to_string(), + StreamingCacheBudget::Gib(gib) => format!("{gib}GB"), + }), + ssd_full_layers: runtime + .ssd + .full_layers + .map_or_else(String::new, |value| value.to_string()), + ssd_preload_experts: runtime + .ssd + .preload_experts + .map_or_else(String::new, |value| value.to_string()), + directional_steering_file: runtime.steering.file.unwrap_or_default(), + directional_steering_ffn: runtime + .steering + .ffn_scale + .map_or_else(String::new, |value| value.to_string()), + directional_steering_attn: runtime + .steering + .attention_scale + .map_or_else(String::new, |value| value.to_string()), + simulated_used_memory_gib: runtime + .diagnostics + .simulated_used_memory_gib + .map_or_else(String::new, |value| value.to_string()), + expert_profile_path: runtime.diagnostics.expert_profile_path.unwrap_or_default(), }) } @@ -99,6 +160,8 @@ impl PreferenceDraft { fn reset(&mut self) { let defaults = GenerationPreferences::default(); let execution = ExecutionPreferences::default(); + let speculative = SpeculativePreferences::default(); + let ssd = SsdPreferences::default(); self.model = ModelChoice::default(); self.dspark_enabled = false; self.idle_timeout_minutes = "10".into(); @@ -115,6 +178,22 @@ impl PreferenceDraft { self.prefill_chunk.clear(); self.quality = execution.quality; self.warm_weights = execution.warm_weights; + self.mtp_draft_tokens = speculative.mtp_draft_tokens.to_string(); + self.mtp_margin = speculative.mtp_margin.to_string(); + self.glm_mtp = speculative.glm_mtp; + self.glm_mtp_timing = speculative.glm_mtp_timing; + self.dspark_confidence_threshold.clear(); + self.dspark_strict = speculative.dspark_strict; + self.ssd_streaming = ssd.enabled; + self.ssd_streaming_cold = ssd.cold; + self.ssd_cache.clear(); + self.ssd_full_layers.clear(); + self.ssd_preload_experts.clear(); + self.directional_steering_file.clear(); + self.directional_steering_ffn.clear(); + self.directional_steering_attn.clear(); + self.simulated_used_memory_gib.clear(); + self.expert_profile_path.clear(); } fn execution(&self) -> Result { @@ -126,6 +205,56 @@ impl PreferenceDraft { warm_weights: self.warm_weights, }) } + + fn speculative(&self) -> Result { + Ok(SpeculativePreferences { + mtp_draft_tokens: parse_positive_i32("MTP draft tokens", &self.mtp_draft_tokens)?, + mtp_margin: parse_f32("MTP margin", &self.mtp_margin)?, + glm_mtp: self.glm_mtp, + glm_mtp_timing: self.glm_mtp_timing, + dspark_enabled: self.dspark_enabled, + dspark_confidence_threshold: parse_optional_f32( + "DSpark confidence", + &self.dspark_confidence_threshold, + )?, + dspark_strict: self.dspark_strict, + }) + } + + fn runtime(&self) -> Result { + Ok(RuntimePreferences { + execution: self.execution()?, + speculative: self.speculative()?, + ssd: SsdPreferences { + enabled: self.ssd_streaming, + cold: self.ssd_streaming_cold, + cache: parse_streaming_cache(&self.ssd_cache)?, + full_layers: parse_optional_u32("SSD full-layer count", &self.ssd_full_layers)?, + preload_experts: parse_optional_u32( + "SSD preload experts", + &self.ssd_preload_experts, + )?, + }, + steering: SteeringPreferences { + file: optional_text(&self.directional_steering_file), + ffn_scale: parse_optional_f32( + "Directional FFN scale", + &self.directional_steering_ffn, + )?, + attention_scale: parse_optional_f32( + "Directional attention scale", + &self.directional_steering_attn, + )?, + }, + diagnostics: DiagnosticPreferences { + simulated_used_memory_gib: parse_optional_gib( + "Simulated used memory", + &self.simulated_used_memory_gib, + )?, + expert_profile_path: optional_text(&self.expert_profile_path), + }, + }) + } } fn parse_positive_i32(name: &str, value: &str) -> Result { @@ -149,6 +278,13 @@ fn parse_optional_f32(name: &str, value: &str) -> Result, String> { } } +fn parse_f32(name: &str, value: &str) -> Result { + value + .trim() + .parse() + .map_err(|_| format!("{name} must be a number.")) +} + fn parse_optional_u64(name: &str, value: &str) -> Result, String> { let value = value.trim(); if value.is_empty() { @@ -184,6 +320,66 @@ fn parse_optional_number( } } +fn parse_streaming_cache(value: &str) -> Result, String> { + let value = value.trim(); + if value.is_empty() { + return Ok(None); + } + if value.len() > 2 + && value + .get(value.len() - 2..) + .is_some_and(|suffix| suffix.eq_ignore_ascii_case("gb")) + { + return parse_gib("SSD cache budget", value) + .map(|gib| Some(StreamingCacheBudget::Gib(gib))); + } + if !value.chars().all(|character| character.is_ascii_digit()) { + return Err( + "SSD cache budget must be a positive expert count or whole GiB value such as 64GB." + .into(), + ); + } + value + .parse::() + .ok() + .filter(|value| *value > 0) + .map(StreamingCacheBudget::Experts) + .map(Some) + .ok_or_else(|| { + "SSD cache budget must be a positive expert count or whole GiB value such as 64GB." + .into() + }) +} + +fn parse_optional_gib(name: &str, value: &str) -> Result, String> { + let value = value.trim(); + if value.is_empty() { + Ok(None) + } else { + parse_gib(name, value).map(Some) + } +} + +fn parse_gib(name: &str, value: &str) -> Result { + let value = value + .get(value.len().saturating_sub(2)..) + .filter(|suffix| suffix.eq_ignore_ascii_case("gb")) + .map_or(value, |_| &value[..value.len() - 2]); + if !value.chars().all(|character| character.is_ascii_digit()) { + return Err(format!("{name} must be a positive whole GiB value.")); + } + value + .parse::() + .ok() + .filter(|value| *value > 0 && *value <= u64::MAX / GIB) + .ok_or_else(|| format!("{name} must be a positive whole GiB value.")) +} + +fn optional_text(value: &str) -> Option { + let value = value.trim(); + (!value.is_empty()).then(|| value.to_owned()) +} + pub(crate) struct App { main_window: window::Id, pub(super) model_manager_window: Option, @@ -258,6 +454,22 @@ pub(crate) enum Message { PreferencePrefillChunkChanged(String), PreferenceQualityChanged(bool), PreferenceWarmWeightsChanged(bool), + PreferenceMtpDraftChanged(String), + PreferenceMtpMarginChanged(String), + PreferenceGlmMtpChanged(bool), + PreferenceGlmMtpTimingChanged(bool), + PreferenceDsparkConfidenceChanged(String), + PreferenceDsparkStrictChanged(bool), + PreferenceSsdChanged(bool), + PreferenceSsdColdChanged(bool), + PreferenceSsdCacheChanged(String), + PreferenceSsdFullLayersChanged(String), + PreferenceSsdPreloadChanged(String), + PreferenceSteeringFileChanged(String), + PreferenceSteeringFfnChanged(String), + PreferenceSteeringAttnChanged(String), + PreferenceSimulatedMemoryChanged(String), + PreferenceExpertProfileChanged(String), ResetPreferences, SavePreferences, DownloadArtifact(ManagedArtifactId), @@ -388,16 +600,31 @@ impl App { self.preference_draft.model = model; if !model.supports_dspark() { self.preference_draft.dspark_enabled = false; + self.preference_draft.dspark_confidence_threshold.clear(); + self.preference_draft.dspark_strict = false; } if model == ModelChoice::Glm52 { self.preference_draft.power_percent.clear(); self.preference_draft.prefill_chunk.clear(); + self.preference_draft.directional_steering_file.clear(); + self.preference_draft.directional_steering_ffn.clear(); + self.preference_draft.directional_steering_attn.clear(); + } else { + self.preference_draft.glm_mtp = false; + self.preference_draft.glm_mtp_timing = false; + self.preference_draft.ssd_full_layers.clear(); } self.preference_error = None; } Message::PreferenceDsparkChanged(enabled) => { self.preference_draft.dspark_enabled = self.preference_draft.model.supports_dspark() && enabled; + if !self.preference_draft.dspark_enabled { + self.preference_draft.dspark_confidence_threshold.clear(); + self.preference_draft.dspark_strict = false; + } else { + self.preference_draft.ssd_streaming = false; + } self.preference_error = None; } Message::PreferenceTimeoutChanged(value) => { @@ -456,6 +683,106 @@ impl App { self.preference_draft.warm_weights = value; self.preference_error = None; } + Message::PreferenceMtpDraftChanged(value) => { + self.preference_draft.mtp_draft_tokens = value; + self.preference_error = None; + } + Message::PreferenceMtpMarginChanged(value) => { + self.preference_draft.mtp_margin = value; + self.preference_error = None; + } + Message::PreferenceGlmMtpChanged(value) => { + self.preference_draft.glm_mtp = + self.preference_draft.model == ModelChoice::Glm52 && value; + if !self.preference_draft.glm_mtp { + self.preference_draft.glm_mtp_timing = false; + } + self.preference_error = None; + } + Message::PreferenceGlmMtpTimingChanged(value) => { + self.preference_draft.glm_mtp_timing = + self.preference_draft.model == ModelChoice::Glm52 && value; + if self.preference_draft.glm_mtp_timing { + self.preference_draft.glm_mtp = true; + } + self.preference_error = None; + } + Message::PreferenceDsparkConfidenceChanged(value) => { + self.preference_draft.dspark_confidence_threshold = value; + if self.preference_draft.model.supports_dspark() + && !self + .preference_draft + .dspark_confidence_threshold + .trim() + .is_empty() + { + self.preference_draft.dspark_enabled = true; + self.preference_draft.ssd_streaming = false; + } + self.preference_error = None; + } + Message::PreferenceDsparkStrictChanged(value) => { + self.preference_draft.dspark_strict = + self.preference_draft.model.supports_dspark() && value; + if self.preference_draft.dspark_strict { + self.preference_draft.dspark_enabled = true; + self.preference_draft.ssd_streaming = false; + } + self.preference_error = None; + } + Message::PreferenceSsdChanged(value) => { + self.preference_draft.ssd_streaming = value; + if value { + self.preference_draft.dspark_enabled = false; + self.preference_draft.dspark_confidence_threshold.clear(); + self.preference_draft.dspark_strict = false; + } + self.preference_error = None; + } + Message::PreferenceSsdColdChanged(value) => { + self.preference_draft.ssd_streaming_cold = value; + self.preference_error = None; + } + Message::PreferenceSsdCacheChanged(value) => { + self.preference_draft.ssd_cache = value; + self.preference_error = None; + } + Message::PreferenceSsdFullLayersChanged(value) => { + if self.preference_draft.model == ModelChoice::Glm52 { + self.preference_draft.ssd_full_layers = value; + } + self.preference_error = None; + } + Message::PreferenceSsdPreloadChanged(value) => { + self.preference_draft.ssd_preload_experts = value; + self.preference_error = None; + } + Message::PreferenceSteeringFileChanged(value) => { + if self.preference_draft.model != ModelChoice::Glm52 { + self.preference_draft.directional_steering_file = value; + } + self.preference_error = None; + } + Message::PreferenceSteeringFfnChanged(value) => { + if self.preference_draft.model != ModelChoice::Glm52 { + self.preference_draft.directional_steering_ffn = value; + } + self.preference_error = None; + } + Message::PreferenceSteeringAttnChanged(value) => { + if self.preference_draft.model != ModelChoice::Glm52 { + self.preference_draft.directional_steering_attn = value; + } + self.preference_error = None; + } + Message::PreferenceSimulatedMemoryChanged(value) => { + self.preference_draft.simulated_used_memory_gib = value; + self.preference_error = None; + } + Message::PreferenceExpertProfileChanged(value) => { + self.preference_draft.expert_profile_path = value; + self.preference_error = None; + } Message::ResetPreferences => { self.preference_draft.reset(); self.preference_error = None; @@ -696,28 +1023,21 @@ impl App { }; let model = self.preference_draft.model; - let execution = match self.preference_draft.execution() { - Ok(execution) => execution, + let runtime = match self.preference_draft.runtime() { + Ok(runtime) => runtime, Err(error) => { self.preference_error = Some(error); return; } }; - if let Err(error) = execution.validate(model) { + if let Err(error) = runtime.engine_settings(model) { self.preference_error = Some(error); return; } - let dspark_enabled = model.supports_dspark() && self.preference_draft.dspark_enabled; let Some(database) = &mut self.database else { return; }; - match database.update_preferences( - model.id(), - dspark_enabled, - idle_timeout_minutes, - &generation, - &execution, - ) { + match database.update_preferences(model.id(), idle_timeout_minutes, &generation, &runtime) { Ok(preferences) => { self.preferences = preferences; self.preference_draft = PreferenceDraft::from_saved(&self.preferences) @@ -957,4 +1277,18 @@ mod tests { assert!(!ModelChoice::DeepSeekV4Pro.supports_dspark()); assert!(!ModelChoice::Glm52.supports_dspark()); } + + #[test] + fn ds4_gib_and_streaming_cache_inputs_are_typed() { + assert_eq!( + parse_streaming_cache("128").unwrap(), + Some(StreamingCacheBudget::Experts(128)) + ); + assert_eq!( + parse_streaming_cache("64gB").unwrap(), + Some(StreamingCacheBudget::Gib(64)) + ); + assert!(parse_streaming_cache("1.5GB").is_err()); + assert_eq!(parse_optional_gib("Memory", "8GB").unwrap(), Some(8)); + } } diff --git a/src/app/view.rs b/src/app/view.rs index 217c3e9..7b95948 100644 --- a/src/app/view.rs +++ b/src/app/view.rs @@ -3,7 +3,7 @@ use crate::database::{ProjectWithSessions, Session}; use crate::model::{ self, DownloadPhase, MODEL_CHOICES, ManagedArtifact, ManagedArtifactState, ModelChoice, }; -use crate::settings::REASONING_MODES; +use crate::settings::{GIB, REASONING_MODES}; use iced::theme::{Palette, palette}; use iced::widget::{ Button, Space, Svg, button, checkbox, column, container, horizontal_rule, opaque, pick_list, @@ -300,21 +300,58 @@ impl App { self.preference_draft.dspark_enabled, ) .on_toggle_maybe(dspark_toggle); + let glm_mtp_toggle: Option Message> = (self.preference_draft.model + == ModelChoice::Glm52) + .then_some(Message::PreferenceGlmMtpChanged); + let glm_mtp_timing_toggle: Option Message> = (self.preference_draft.model + == ModelChoice::Glm52) + .then_some(Message::PreferenceGlmMtpTimingChanged); + let dspark_strict_toggle: Option Message> = self + .preference_draft + .model + .supports_dspark() + .then_some(Message::PreferenceDsparkStrictChanged); let effective = self .preference_draft .generation() .ok() - .map(|generation| generation.effective(self.preference_draft.model)); - let execution = self + .map(|generation| generation.turn_settings(self.preference_draft.model)); + let engine = self .preference_draft - .execution() + .runtime() .ok() - .map(|execution| execution.engine_settings()); + .and_then(|runtime| runtime.engine_settings(self.preference_draft.model).ok()); + let artifacts = model::engine_artifacts( + self.preference_draft.model, + self.preference_draft.dspark_enabled, + &models_path(), + ); let mut power = text_input("100", &self.preference_draft.power_percent); let mut prefill = text_input("Automatic", &self.preference_draft.prefill_chunk); + let mut ssd_full_layers = text_input("Automatic", &self.preference_draft.ssd_full_layers); + let mut steering_file = text_input( + "Direction-vector file path", + &self.preference_draft.directional_steering_file, + ); + let mut steering_ffn = + text_input("Automatic", &self.preference_draft.directional_steering_ffn); + let mut steering_attn = text_input("0", &self.preference_draft.directional_steering_attn); + let mut dspark_confidence = text_input( + "0.9 (DS4 default)", + &self.preference_draft.dspark_confidence_threshold, + ); if self.preference_draft.model != ModelChoice::Glm52 { power = power.on_input(Message::PreferencePowerChanged); prefill = prefill.on_input(Message::PreferencePrefillChunkChanged); + steering_file = steering_file.on_input(Message::PreferenceSteeringFileChanged); + steering_ffn = steering_ffn.on_input(Message::PreferenceSteeringFfnChanged); + steering_attn = steering_attn.on_input(Message::PreferenceSteeringAttnChanged); + } else { + ssd_full_layers = ssd_full_layers.on_input(Message::PreferenceSsdFullLayersChanged); + } + if self.preference_draft.model.supports_dspark() { + dspark_confidence = + dspark_confidence.on_input(Message::PreferenceDsparkConfidenceChanged); } let mut fields = column![ @@ -325,12 +362,14 @@ impl App { Message::PreferenceModelChanged, ) .width(Length::Fill), - dspark, - text(if self.preference_draft.model.supports_dspark() { - "DSpark support is used only when enabled. Downloads are managed in Model Manager." - } else { - "DSpark is not available for the selected model." - }) + text(format!( + "Main: {}{}", + artifacts.model.display(), + artifacts + .mtp + .as_ref() + .map_or_else(String::new, |path| format!(" • support: {}", path.display())), + )) .size(12), Space::with_height(8), text("EXECUTION").size(11), @@ -351,16 +390,136 @@ impl App { "Blank numeric values preserve DS4's automatic engine behavior." }) .size(12), - text(execution.map_or_else( + text(engine.as_ref().map_or_else( || "Effective execution settings will appear after valid values are entered." .to_owned(), - |settings| format!( + |engine| { + let settings = engine.execution; + format!( "Metal engine: threads {} • power {}% • prefill {} • quality {} • warm weights {}", if settings.cpu_threads == 0 { "auto".to_owned() } else { settings.cpu_threads.to_string() }, if settings.power_percent == 0 { 100 } else { settings.power_percent }, if settings.prefill_chunk == 0 { "auto".to_owned() } else { settings.prefill_chunk.to_string() }, if settings.quality { "on" } else { "off" }, if settings.warm_weights { "on" } else { "off" }, + )}, + )) + .size(12), + Space::with_height(8), + text("SPECULATIVE DECODING").size(11), + preference_input_row( + "MTP draft tokens", + text_input("1", &self.preference_draft.mtp_draft_tokens) + .on_input(Message::PreferenceMtpDraftChanged), + ), + preference_input_row( + "MTP verifier margin", + text_input("3", &self.preference_draft.mtp_margin) + .on_input(Message::PreferenceMtpMarginChanged), + ), + checkbox("Enable integrated GLM MTP", self.preference_draft.glm_mtp) + .on_toggle_maybe(glm_mtp_toggle), + checkbox( + "Log GLM MTP timing counters", + self.preference_draft.glm_mtp_timing, + ) + .on_toggle_maybe(glm_mtp_timing_toggle), + dspark, + preference_input_row("DSpark confidence threshold", dspark_confidence), + checkbox( + "DSpark target-only decode", + self.preference_draft.dspark_strict, + ) + .on_toggle_maybe(dspark_strict_toggle), + text(if self.preference_draft.model.supports_dspark() { + "DSpark uses the managed support artifact; entering a threshold or enabling strict mode also enables DSpark." + } else if self.preference_draft.model == ModelChoice::Glm52 { + "GLM MTP is integrated; DSpark is unavailable for this model." + } else { + "No managed MTP support artifact is available for this model." + }) + .size(12), + text(engine.as_ref().map_or_else( + || "Effective speculative settings will appear after valid values are entered." + .to_owned(), + |engine| { + let settings = engine.speculative; + format!( + "Engine: MTP draft {} • margin {} • GLM MTP {} • timing {} • DSpark {} • confidence {}{} • target-only {}", + settings.mtp_draft_tokens, + settings.mtp_margin, + if settings.glm_mtp { "on" } else { "off" }, + if settings.glm_mtp_timing { "on" } else { "off" }, + if settings.dspark { "on" } else { "off" }, + settings.dspark_confidence_threshold, + if settings.dspark_confidence_threshold_set { " explicit" } else { " default" }, + if settings.dspark_strict { "on" } else { "off" }, + )}, + )) + .size(12), + Space::with_height(8), + text("SSD STREAMING").size(11), + checkbox("Enable SSD-backed model streaming", self.preference_draft.ssd_streaming) + .on_toggle(Message::PreferenceSsdChanged), + checkbox("Skip automatic expert preload", self.preference_draft.ssd_streaming_cold) + .on_toggle(Message::PreferenceSsdColdChanged), + preference_input_row( + "Expert cache count or GiB", + text_input("Automatic, 128, or 64GB", &self.preference_draft.ssd_cache) + .on_input(Message::PreferenceSsdCacheChanged), + ), + preference_input_row("Fully resident GLM layers", ssd_full_layers), + preference_input_row( + "Explicit expert preload count", + text_input("Automatic", &self.preference_draft.ssd_preload_experts) + .on_input(Message::PreferenceSsdPreloadChanged), + ), + text("A blank full-layer value is automatic; an explicit 0 disables fully resident GLM layers. SSD streaming and DSpark are mutually exclusive.") + .size(12), + text(engine.as_ref().map_or_else( + || "Effective SSD settings will appear after valid values are entered." + .to_owned(), + |engine| { + let settings = engine.ssd; + let cache = if settings.cache_bytes > 0 { + format!("{} GiB", settings.cache_bytes / GIB) + } else if settings.cache_experts > 0 { + format!("{} experts", settings.cache_experts) + } else { + "auto".to_owned() + }; + format!( + "Engine: streaming {} • cold {} • cache {} • full layers {}{} • preload {}", + if settings.enabled { "on" } else { "off" }, + if settings.cold { "on" } else { "off" }, + cache, + settings.full_layers, + if settings.full_layers_set { " explicit" } else { " auto" }, + if settings.preload_experts == 0 { "auto".to_owned() } else { settings.preload_experts.to_string() }, + ) + }, + )) + .size(12), + Space::with_height(8), + text("DIRECTIONAL STEERING").size(11), + text("Direction-vector file").size(13), + steering_file.padding(9), + preference_input_row("FFN scale", steering_ffn), + preference_input_row("Attention scale", steering_attn), + text(if self.preference_draft.model == ModelChoice::Glm52 { + "Directional steering is not supported for GLM 5.2." + } else { + "With a file and no explicit scale, DS4 defaults the FFN scale to 1. Scales accept -100 through 100." + }) + .size(12), + text(engine.as_ref().map_or_else( + || "Effective steering settings will appear after valid values are entered." + .to_owned(), + |engine| format!( + "Engine: file {} • FFN scale {} • attention scale {}", + if engine.steering.file.is_some() { "set" } else { "off" }, + engine.steering.ffn_scale, + engine.steering.attention_scale, ), )) .size(12), @@ -434,6 +593,32 @@ impl App { )) .size(12), Space::with_height(8), + text("ADVANCED DIAGNOSTICS").size(11), + preference_input_row( + "Simulated used memory (GiB)", + text_input("Disabled", &self.preference_draft.simulated_used_memory_gib) + .on_input(Message::PreferenceSimulatedMemoryChanged), + ), + text("Routed expert profile output").size(13), + text_input("Output file path", &self.preference_draft.expert_profile_path) + .on_input(Message::PreferenceExpertProfileChanged) + .padding(9), + text(engine.as_ref().map_or_else( + || "Effective diagnostic settings will appear after valid values are entered." + .to_owned(), + |engine| format!( + "{} load: simulated memory {} • expert profile {}", + engine.model, + if engine.diagnostics.simulated_used_memory_bytes == 0 { + "off".to_owned() + } else { + format!("{} GiB", engine.diagnostics.simulated_used_memory_bytes / GIB) + }, + if engine.diagnostics.expert_profile_path.is_some() { "set" } else { "off" }, + ), + )) + .size(12), + Space::with_height(8), text("INACTIVITY").size(11), row![ text_input("10", &self.preference_draft.idle_timeout_minutes) diff --git a/src/database.rs b/src/database.rs index 4d5ead9..44dfcfa 100644 --- a/src/database.rs +++ b/src/database.rs @@ -5,7 +5,11 @@ use std::fs; use std::path::Path; use crate::schema::{preferences, projects, sessions}; -use crate::settings::{ExecutionPreferences, GenerationPreferences, ReasoningMode}; +use crate::settings::{ + DiagnosticPreferences, ExecutionPreferences, GenerationPreferences, ReasoningMode, + RuntimePreferences, SpeculativePreferences, SsdPreferences, SteeringPreferences, + StreamingCacheBudget, +}; pub const MIGRATIONS: EmbeddedMigrations = embed_migrations!("migrations"); @@ -30,6 +34,23 @@ pub struct AppPreferences { pub prefill_chunk: Option, pub quality: bool, pub warm_weights: bool, + pub mtp_draft_tokens: i32, + pub mtp_margin: f32, + pub glm_mtp: bool, + pub glm_mtp_timing: bool, + pub dspark_confidence_threshold: Option, + pub dspark_strict: bool, + pub ssd_streaming: bool, + pub ssd_streaming_cold: bool, + pub ssd_cache_experts: Option, + pub ssd_cache_gib: Option, + pub ssd_full_layers: Option, + pub ssd_preload_experts: Option, + pub directional_steering_file: Option, + pub directional_steering_ffn: Option, + pub directional_steering_attn: Option, + pub simulated_used_memory_gib: Option, + pub expert_profile_path: Option, } impl Default for AppPreferences { @@ -52,6 +73,23 @@ impl Default for AppPreferences { prefill_chunk: None, quality: false, warm_weights: false, + mtp_draft_tokens: 1, + mtp_margin: 3.0, + glm_mtp: false, + glm_mtp_timing: false, + dspark_confidence_threshold: None, + dspark_strict: false, + ssd_streaming: false, + ssd_streaming_cold: false, + ssd_cache_experts: None, + ssd_cache_gib: None, + ssd_full_layers: None, + ssd_preload_experts: None, + directional_steering_file: None, + directional_steering_ffn: None, + directional_steering_attn: None, + simulated_used_memory_gib: None, + expert_profile_path: None, } } } @@ -99,6 +137,65 @@ impl AppPreferences { warm_weights: self.warm_weights, }) } + + pub(crate) fn speculative(&self) -> SpeculativePreferences { + SpeculativePreferences { + mtp_draft_tokens: self.mtp_draft_tokens, + mtp_margin: self.mtp_margin, + glm_mtp: self.glm_mtp, + glm_mtp_timing: self.glm_mtp_timing, + dspark_enabled: self.dspark_enabled, + dspark_confidence_threshold: self.dspark_confidence_threshold, + dspark_strict: self.dspark_strict, + } + } + + pub(crate) fn runtime(&self) -> Result { + let cache = match (self.ssd_cache_experts, self.ssd_cache_gib) { + (None, None) => None, + (Some(experts), None) => Some(StreamingCacheBudget::Experts( + u32::try_from(experts).map_err(|_| "Saved SSD expert count is invalid.")?, + )), + (None, Some(gib)) => Some(StreamingCacheBudget::Gib( + u64::try_from(gib).map_err(|_| "Saved SSD cache budget is invalid.")?, + )), + (Some(_), Some(_)) => { + return Err("Saved SSD cache budget has both count and GiB values.".into()); + } + }; + Ok(RuntimePreferences { + execution: self.execution()?, + speculative: self.speculative(), + ssd: SsdPreferences { + enabled: self.ssd_streaming, + cold: self.ssd_streaming_cold, + cache, + full_layers: self + .ssd_full_layers + .map(u32::try_from) + .transpose() + .map_err(|_| "Saved SSD full-layer count is invalid.")?, + preload_experts: self + .ssd_preload_experts + .map(u32::try_from) + .transpose() + .map_err(|_| "Saved SSD preload count is invalid.")?, + }, + steering: SteeringPreferences { + file: self.directional_steering_file.clone(), + ffn_scale: self.directional_steering_ffn, + attention_scale: self.directional_steering_attn, + }, + diagnostics: DiagnosticPreferences { + simulated_used_memory_gib: self + .simulated_used_memory_gib + .map(u64::try_from) + .transpose() + .map_err(|_| "Saved simulated memory is invalid.")?, + expert_profile_path: self.expert_profile_path.clone(), + }, + }) + } } #[derive(AsChangeset)] @@ -120,6 +217,23 @@ struct PreferenceChanges<'a> { prefill_chunk: Option, quality: bool, warm_weights: bool, + mtp_draft_tokens: i32, + mtp_margin: f32, + glm_mtp: bool, + glm_mtp_timing: bool, + dspark_confidence_threshold: Option, + dspark_strict: bool, + ssd_streaming: bool, + ssd_streaming_cold: bool, + ssd_cache_experts: Option, + ssd_cache_gib: Option, + ssd_full_layers: Option, + ssd_preload_experts: Option, + directional_steering_file: Option<&'a str>, + directional_steering_ffn: Option, + directional_steering_attn: Option, + simulated_used_memory_gib: Option, + expert_profile_path: Option<&'a str>, } #[derive(Clone, Debug, Identifiable, Queryable, Selectable)] @@ -220,20 +334,29 @@ impl Database { pub fn update_preferences( &mut self, selected_model: &str, - dspark_enabled: bool, idle_timeout_minutes: i32, generation: &GenerationPreferences, - execution: &ExecutionPreferences, + runtime: &RuntimePreferences, ) -> Result { generation.validate()?; let model = crate::model::ModelChoice::from_id(selected_model) .ok_or_else(|| format!("Unsupported model: {selected_model}"))?; - execution.validate(model)?; + runtime.engine_settings(model)?; + let execution = &runtime.execution; + let speculative = &runtime.speculative; + let (ssd_cache_experts, ssd_cache_gib) = match runtime.ssd.cache { + None => (None, None), + Some(StreamingCacheBudget::Experts(experts)) => (Some(i64::from(experts)), None), + Some(StreamingCacheBudget::Gib(gib)) => ( + None, + Some(i64::try_from(gib).map_err(|_| "SSD cache budget is too large.")?), + ), + }; let seed = generation.seed.map(|seed| seed.to_string()); diesel::update(preferences::table.find(1)) .set(PreferenceChanges { selected_model, - dspark_enabled, + dspark_enabled: speculative.dspark_enabled, idle_timeout_minutes, context_tokens: generation.context_tokens, max_generated_tokens: generation.max_generated_tokens, @@ -248,6 +371,26 @@ impl Database { prefill_chunk: execution.prefill_chunk.map(|value| value as i32), quality: execution.quality, warm_weights: execution.warm_weights, + mtp_draft_tokens: speculative.mtp_draft_tokens, + mtp_margin: speculative.mtp_margin, + glm_mtp: speculative.glm_mtp, + glm_mtp_timing: speculative.glm_mtp_timing, + dspark_confidence_threshold: speculative.dspark_confidence_threshold, + dspark_strict: speculative.dspark_strict, + ssd_streaming: runtime.ssd.enabled, + ssd_streaming_cold: runtime.ssd.cold, + ssd_cache_experts, + ssd_cache_gib, + ssd_full_layers: runtime.ssd.full_layers.map(|value| value as i32), + ssd_preload_experts: runtime.ssd.preload_experts.map(|value| value as i32), + directional_steering_file: runtime.steering.file.as_deref(), + directional_steering_ffn: runtime.steering.ffn_scale, + directional_steering_attn: runtime.steering.attention_scale, + simulated_used_memory_gib: runtime + .diagnostics + .simulated_used_memory_gib + .map(|value| value as i64), + expert_profile_path: runtime.diagnostics.expert_profile_path.as_deref(), }) .returning(AppPreferences::as_returning()) .get_result(&mut self.connection) @@ -308,15 +451,26 @@ mod tests { assert!(!preferences.dspark_enabled); assert_eq!(preferences.idle_timeout_minutes, 10); let generation = GenerationPreferences::default(); - let execution = ExecutionPreferences::default(); + let runtime = RuntimePreferences::default(); assert!( database - .update_preferences("glm-5.2", true, 30, &generation, &execution) + .update_preferences( + "glm-5.2", + 30, + &generation, + &RuntimePreferences { + speculative: SpeculativePreferences { + dspark_enabled: true, + ..runtime.speculative.clone() + }, + ..runtime.clone() + } + ) .is_err() ); assert!( database - .update_preferences("deepseek-v4-flash", false, 0, &generation, &execution) + .update_preferences("deepseek-v4-flash", 0, &generation, &runtime,) .is_err() ); let generation = GenerationPreferences { @@ -328,10 +482,33 @@ mod tests { let execution = ExecutionPreferences { cpu_threads: Some(8), quality: true, - ..execution + ..runtime.execution + }; + let speculative = SpeculativePreferences { + glm_mtp: true, + glm_mtp_timing: true, + mtp_draft_tokens: 4, + mtp_margin: 2.5, + ..runtime.speculative + }; + let runtime = RuntimePreferences { + execution, + speculative, + ssd: SsdPreferences { + enabled: true, + cache: Some(StreamingCacheBudget::Gib(64)), + full_layers: Some(0), + preload_experts: Some(32), + ..SsdPreferences::default() + }, + diagnostics: DiagnosticPreferences { + simulated_used_memory_gib: Some(8), + expert_profile_path: Some("/tmp/experts.json".into()), + }, + ..RuntimePreferences::default() }; database - .update_preferences("glm-5.2", false, 30, &generation, &execution) + .update_preferences("glm-5.2", 30, &generation, &runtime) .unwrap(); let project = database.create_project("DS4", "/tmp/ds4").unwrap(); @@ -354,7 +531,7 @@ mod tests { assert_eq!(preferences.selected_model, "glm-5.2"); assert_eq!(preferences.idle_timeout_minutes, 30); assert_eq!(preferences.generation().unwrap(), generation); - assert_eq!(preferences.execution().unwrap(), execution); + assert_eq!(preferences.runtime().unwrap(), runtime); drop(reopened); fs::remove_file(path).unwrap(); } diff --git a/src/model.rs b/src/model.rs index 0c27386..524520b 100644 --- a/src/model.rs +++ b/src/model.rs @@ -75,7 +75,6 @@ impl ModelChoice { self == Self::DeepSeekV4Flash } - #[cfg(test)] fn main_artifact(self) -> &'static Artifact { match self { Self::DeepSeekV4Flash => &FLASH, @@ -95,6 +94,24 @@ impl ModelChoice { } } +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct EngineArtifacts { + pub(crate) model: PathBuf, + pub(crate) mtp: Option, +} + +pub(crate) fn engine_artifacts( + model: ModelChoice, + dspark_enabled: bool, + models_path: &Path, +) -> EngineArtifacts { + EngineArtifacts { + model: model.main_artifact().path(model, models_path), + mtp: (model.supports_dspark() && dspark_enabled) + .then(|| FLASH_DSPARK.path(model, models_path)), + } +} + #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub(crate) enum ManagedArtifactId { DeepSeekV4Flash, @@ -663,6 +680,15 @@ mod tests { .unwrap() .as_nanos(); let models_path = std::env::temp_dir().join(format!("ds4-server-models-{id}")); + let engine = engine_artifacts(ModelChoice::DeepSeekV4Flash, true, &models_path); + assert_eq!( + engine.model.file_name(), + Some(std::ffi::OsStr::new(FLASH.file_name)) + ); + assert_eq!( + engine.mtp.as_deref().and_then(Path::file_name), + Some(std::ffi::OsStr::new(FLASH_DSPARK.file_name)) + ); let empty = Artifact { label: "empty", file_name: "empty", diff --git a/src/schema.rs b/src/schema.rs index ab161ef..4e58900 100644 --- a/src/schema.rs +++ b/src/schema.rs @@ -25,6 +25,23 @@ diesel::table! { prefill_chunk -> Nullable, quality -> Bool, warm_weights -> Bool, + mtp_draft_tokens -> Integer, + mtp_margin -> Float, + glm_mtp -> Bool, + glm_mtp_timing -> Bool, + dspark_confidence_threshold -> Nullable, + dspark_strict -> Bool, + ssd_streaming -> Bool, + ssd_streaming_cold -> Bool, + ssd_cache_experts -> Nullable, + ssd_cache_gib -> Nullable, + ssd_full_layers -> Nullable, + ssd_preload_experts -> Nullable, + directional_steering_file -> Nullable, + directional_steering_ffn -> Nullable, + directional_steering_attn -> Nullable, + simulated_used_memory_gib -> Nullable, + expert_profile_path -> Nullable, } } diff --git a/src/settings.rs b/src/settings.rs index c6c43cc..6349acc 100644 --- a/src/settings.rs +++ b/src/settings.rs @@ -8,6 +8,274 @@ pub(crate) const REASONING_MODES: [ReasoningMode; 3] = [ ]; const THINK_MAX_MIN_CONTEXT: i32 = 393_216; const MAX_CPU_THREADS: u32 = 32; +const MAX_MTP_DRAFT_TOKENS: i32 = 16; +pub(crate) const GIB: u64 = 1024 * 1024 * 1024; + +#[derive(Clone, Debug, PartialEq)] +pub(crate) struct SpeculativePreferences { + pub(crate) mtp_draft_tokens: i32, + pub(crate) mtp_margin: f32, + pub(crate) glm_mtp: bool, + pub(crate) glm_mtp_timing: bool, + pub(crate) dspark_enabled: bool, + pub(crate) dspark_confidence_threshold: Option, + pub(crate) dspark_strict: bool, +} + +impl Default for SpeculativePreferences { + fn default() -> Self { + Self { + mtp_draft_tokens: 1, + mtp_margin: 3.0, + glm_mtp: false, + glm_mtp_timing: false, + dspark_enabled: false, + dspark_confidence_threshold: None, + dspark_strict: false, + } + } +} + +impl SpeculativePreferences { + pub(crate) fn validate(&self, model: ModelChoice) -> Result<(), String> { + if self.mtp_draft_tokens <= 0 { + return Err("MTP draft tokens must be a positive whole number.".into()); + } + validate_float("MTP margin", self.mtp_margin, 0.0, 1000.0)?; + if self.glm_mtp_timing && !self.glm_mtp { + return Err("GLM MTP timing requires GLM MTP.".into()); + } + if model != ModelChoice::Glm52 && (self.glm_mtp || self.glm_mtp_timing) { + return Err("GLM MTP is available only for GLM 5.2.".into()); + } + if self.dspark_enabled && !model.supports_dspark() { + return Err("DSpark is not available for the selected model.".into()); + } + if (self.dspark_confidence_threshold.is_some() || self.dspark_strict) + && !self.dspark_enabled + { + return Err("DSpark tuning requires DSpark to be enabled.".into()); + } + if let Some(threshold) = self.dspark_confidence_threshold { + validate_float("DSpark confidence", threshold, 0.0, 1.0)?; + } + Ok(()) + } + + pub(crate) fn engine_settings(&self) -> EngineSpeculativeSettings { + EngineSpeculativeSettings { + mtp_draft_tokens: self.mtp_draft_tokens.min(MAX_MTP_DRAFT_TOKENS), + mtp_margin: self.mtp_margin, + glm_mtp: self.glm_mtp, + glm_mtp_timing: self.glm_mtp_timing, + dspark: self.dspark_enabled, + dspark_confidence_threshold: self.dspark_confidence_threshold.unwrap_or(0.9), + dspark_confidence_threshold_set: self.dspark_confidence_threshold.is_some(), + dspark_strict: self.dspark_strict, + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq)] +pub(crate) struct EngineSpeculativeSettings { + pub(crate) mtp_draft_tokens: i32, + pub(crate) mtp_margin: f32, + pub(crate) glm_mtp: bool, + pub(crate) glm_mtp_timing: bool, + pub(crate) dspark: bool, + pub(crate) dspark_confidence_threshold: f32, + pub(crate) dspark_confidence_threshold_set: bool, + pub(crate) dspark_strict: bool, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum StreamingCacheBudget { + Experts(u32), + Gib(u64), +} + +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub(crate) struct SsdPreferences { + pub(crate) enabled: bool, + pub(crate) cold: bool, + pub(crate) cache: Option, + pub(crate) full_layers: Option, + pub(crate) preload_experts: Option, +} + +impl SsdPreferences { + pub(crate) fn validate(&self, model: ModelChoice) -> Result<(), String> { + if self.preload_experts == Some(0) { + return Err("SSD preload experts must be a positive whole number.".into()); + } + if self + .preload_experts + .is_some_and(|experts| experts > i32::MAX as u32) + { + return Err("SSD preload experts is too large.".into()); + } + if self + .full_layers + .is_some_and(|layers| layers > i32::MAX as u32) + { + return Err("SSD full-layer count is too large.".into()); + } + if self.full_layers.is_some_and(|layers| layers > 0) && model != ModelChoice::Glm52 { + return Err("Fully resident SSD layers are available only for GLM 5.2.".into()); + } + if let Some(StreamingCacheBudget::Gib(gib)) = self.cache { + validate_gib("SSD cache budget", gib)?; + } + Ok(()) + } + + pub(crate) fn engine_settings(&self) -> EngineSsdSettings { + let (cache_experts, cache_bytes) = match self.cache { + None => (0, 0), + Some(StreamingCacheBudget::Experts(experts)) => (experts, 0), + Some(StreamingCacheBudget::Gib(gib)) => (0, gib * GIB), + }; + EngineSsdSettings { + enabled: self.enabled, + cold: self.cold, + cache_experts, + cache_bytes, + full_layers: self.full_layers.unwrap_or(0), + full_layers_set: self.full_layers.is_some(), + preload_experts: self.preload_experts.unwrap_or(0), + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) struct EngineSsdSettings { + pub(crate) enabled: bool, + pub(crate) cold: bool, + pub(crate) cache_experts: u32, + pub(crate) cache_bytes: u64, + pub(crate) full_layers: u32, + pub(crate) full_layers_set: bool, + pub(crate) preload_experts: u32, +} + +#[derive(Clone, Debug, Default, PartialEq)] +pub(crate) struct SteeringPreferences { + pub(crate) file: Option, + pub(crate) ffn_scale: Option, + pub(crate) attention_scale: Option, +} + +impl SteeringPreferences { + pub(crate) fn validate(&self, model: ModelChoice) -> Result<(), String> { + validate_optional_float("Directional FFN scale", self.ffn_scale, -100.0, 100.0)?; + validate_optional_float( + "Directional attention scale", + self.attention_scale, + -100.0, + 100.0, + )?; + if self.file.is_some() && model == ModelChoice::Glm52 { + return Err("Directional steering is not supported for GLM 5.2.".into()); + } + if self.file.is_none() + && (self.ffn_scale.is_some_and(|scale| scale != 0.0) + || self.attention_scale.is_some_and(|scale| scale != 0.0)) + { + return Err("Directional steering scales require a direction-vector file.".into()); + } + Ok(()) + } + + pub(crate) fn engine_settings(&self) -> EngineSteeringSettings { + let no_explicit_scale = self.ffn_scale.is_none() && self.attention_scale.is_none(); + EngineSteeringSettings { + file: self.file.clone(), + ffn_scale: self + .ffn_scale + .unwrap_or(if self.file.is_some() && no_explicit_scale { + 1.0 + } else { + 0.0 + }), + attention_scale: self.attention_scale.unwrap_or(0.0), + } + } +} + +#[derive(Clone, Debug, PartialEq)] +pub(crate) struct EngineSteeringSettings { + pub(crate) file: Option, + pub(crate) ffn_scale: f32, + pub(crate) attention_scale: f32, +} + +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub(crate) struct DiagnosticPreferences { + pub(crate) simulated_used_memory_gib: Option, + pub(crate) expert_profile_path: Option, +} + +impl DiagnosticPreferences { + pub(crate) fn validate(&self) -> Result<(), String> { + if let Some(gib) = self.simulated_used_memory_gib { + validate_gib("Simulated used memory", gib)?; + } + Ok(()) + } + + pub(crate) fn engine_settings(&self) -> EngineDiagnosticSettings { + EngineDiagnosticSettings { + simulated_used_memory_bytes: self.simulated_used_memory_gib.unwrap_or(0) * GIB, + expert_profile_path: self.expert_profile_path.clone(), + } + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct EngineDiagnosticSettings { + pub(crate) simulated_used_memory_bytes: u64, + pub(crate) expert_profile_path: Option, +} + +#[derive(Clone, Debug, Default, PartialEq)] +pub(crate) struct RuntimePreferences { + pub(crate) execution: ExecutionPreferences, + pub(crate) speculative: SpeculativePreferences, + pub(crate) ssd: SsdPreferences, + pub(crate) steering: SteeringPreferences, + pub(crate) diagnostics: DiagnosticPreferences, +} + +impl RuntimePreferences { + pub(crate) fn engine_settings(&self, model: ModelChoice) -> Result { + self.execution.validate(model)?; + self.speculative.validate(model)?; + self.ssd.validate(model)?; + self.steering.validate(model)?; + self.diagnostics.validate()?; + if self.ssd.enabled && self.speculative.dspark_enabled { + return Err("SSD streaming is not compatible with DSpark support.".into()); + } + Ok(EngineSettings { + model, + execution: self.execution.engine_settings(), + speculative: self.speculative.engine_settings(), + ssd: self.ssd.engine_settings(), + steering: self.steering.engine_settings(), + diagnostics: self.diagnostics.engine_settings(), + }) + } +} + +#[derive(Clone, Debug, PartialEq)] +pub(crate) struct EngineSettings { + pub(crate) model: ModelChoice, + pub(crate) execution: EngineExecutionSettings, + pub(crate) speculative: EngineSpeculativeSettings, + pub(crate) ssd: EngineSsdSettings, + pub(crate) steering: EngineSteeringSettings, + pub(crate) diagnostics: EngineDiagnosticSettings, +} #[derive(Clone, Debug, Default, Eq, PartialEq)] pub(crate) struct ExecutionPreferences { @@ -151,9 +419,9 @@ impl GenerationPreferences { Ok(()) } - pub(crate) fn effective(&self, model: ModelChoice) -> EffectiveGenerationSettings { + pub(crate) fn turn_settings(&self, model: ModelChoice) -> TurnSettings { let glm = model == ModelChoice::Glm52; - EffectiveGenerationSettings { + TurnSettings { context_tokens: self.context_tokens, max_generated_tokens: self.max_generated_tokens, system_prompt: self.system_prompt.clone(), @@ -178,14 +446,28 @@ fn validate_optional_float( minimum: f32, maximum: f32, ) -> Result<(), String> { - if value.is_some_and(|value| !value.is_finite() || !(minimum..=maximum).contains(&value)) { + if let Some(value) = value { + validate_float(name, value, minimum, maximum)?; + } + Ok(()) +} + +fn validate_float(name: &str, value: f32, minimum: f32, maximum: f32) -> Result<(), String> { + if !value.is_finite() || !(minimum..=maximum).contains(&value) { return Err(format!("{name} must be between {minimum} and {maximum}.")); } Ok(()) } +fn validate_gib(name: &str, gib: u64) -> Result<(), String> { + if gib == 0 || gib > u64::MAX / GIB { + return Err(format!("{name} must be a positive whole GiB value.")); + } + Ok(()) +} + #[derive(Clone, Debug, PartialEq)] -pub(crate) struct EffectiveGenerationSettings { +pub(crate) struct TurnSettings { pub(crate) context_tokens: i32, pub(crate) max_generated_tokens: i32, pub(crate) system_prompt: String, @@ -199,17 +481,18 @@ pub(crate) struct EffectiveGenerationSettings { #[cfg(test)] mod tests { use super::*; + use std::collections::BTreeSet; #[test] fn effective_settings_preserve_unset_model_defaults() { let defaults = GenerationPreferences::default(); - let deepseek = defaults.effective(ModelChoice::DeepSeekV4Flash); + let deepseek = defaults.turn_settings(ModelChoice::DeepSeekV4Flash); assert_eq!( (deepseek.temperature, deepseek.top_p, deepseek.min_p), (1.0, 1.0, 0.05) ); - let glm = defaults.effective(ModelChoice::Glm52); + let glm = defaults.turn_settings(ModelChoice::Glm52); assert_eq!((glm.temperature, glm.top_p, glm.min_p), (1.0, 0.95, 0.0)); let explicit = GenerationPreferences { @@ -218,7 +501,7 @@ mod tests { reasoning_mode: ReasoningMode::Max, ..defaults }; - let effective = explicit.effective(ModelChoice::Glm52); + let effective = explicit.turn_settings(ModelChoice::Glm52); assert_eq!((effective.top_p, effective.min_p), (0.4, 0.2)); assert_eq!(effective.reasoning_mode, ReasoningMode::High); } @@ -245,4 +528,189 @@ mod tests { assert!(tuned.validate(ModelChoice::Glm52).is_err()); assert_eq!(tuned.engine_settings().cpu_threads, MAX_CPU_THREADS); } + + #[test] + fn speculative_settings_match_ds4_defaults_and_dependencies() { + let defaults = SpeculativePreferences::default(); + let engine = defaults.engine_settings(); + assert_eq!((engine.mtp_draft_tokens, engine.mtp_margin), (1, 3.0)); + assert_eq!(engine.dspark_confidence_threshold, 0.9); + assert!(!engine.dspark_confidence_threshold_set); + + let tuned = SpeculativePreferences { + mtp_draft_tokens: 20, + dspark_enabled: true, + dspark_confidence_threshold: Some(0.7), + dspark_strict: true, + ..defaults + }; + assert!(tuned.validate(ModelChoice::DeepSeekV4Flash).is_ok()); + assert!(tuned.validate(ModelChoice::Glm52).is_err()); + assert_eq!(tuned.engine_settings().mtp_draft_tokens, 16); + + let glm = SpeculativePreferences { + glm_mtp: true, + glm_mtp_timing: true, + ..SpeculativePreferences::default() + }; + assert!(glm.validate(ModelChoice::Glm52).is_ok()); + assert!(glm.validate(ModelChoice::DeepSeekV4Pro).is_err()); + } + + #[test] + fn runtime_settings_preserve_automatic_values_and_cross_field_rules() { + let runtime = RuntimePreferences { + ssd: SsdPreferences { + enabled: true, + cache: Some(StreamingCacheBudget::Gib(64)), + full_layers: Some(0), + ..SsdPreferences::default() + }, + steering: SteeringPreferences { + file: Some("direction.gguf".into()), + ..SteeringPreferences::default() + }, + diagnostics: DiagnosticPreferences { + simulated_used_memory_gib: Some(8), + ..DiagnosticPreferences::default() + }, + ..RuntimePreferences::default() + }; + let engine = runtime + .engine_settings(ModelChoice::DeepSeekV4Flash) + .unwrap(); + assert_eq!(engine.ssd.cache_bytes, 64 * GIB); + assert!(engine.ssd.full_layers_set); + assert_eq!(engine.ssd.full_layers, 0); + assert_eq!(engine.steering.ffn_scale, 1.0); + assert_eq!(engine.diagnostics.simulated_used_memory_bytes, 8 * GIB); + + let incompatible = RuntimePreferences { + speculative: SpeculativePreferences { + dspark_enabled: true, + ..SpeculativePreferences::default() + }, + ..runtime + }; + assert!( + incompatible + .engine_settings(ModelChoice::DeepSeekV4Flash) + .is_err() + ); + } + + #[test] + fn every_ds4_cli_option_is_mapped_or_explicitly_not_a_preference() { + const PREFERENCES: &[&str] = &[ + "--ctx", + "--dir-steering-attn", + "--dir-steering-ffn", + "--dir-steering-file", + "--dspark", + "--dspark-confidence", + "--dspark-strict", + "--expert-profile", + "--glm-mtp", + "--glm-mtp-timing", + "--min-p", + "--model", + "--mtp", + "--mtp-draft", + "--mtp-margin", + "--nothink", + "--power", + "--prefill-chunk", + "--quality", + "--seed", + "--simulate-used-memory", + "--ssd-streaming", + "--ssd-streaming-cache-experts", + "--ssd-streaming-cold", + "--ssd-streaming-full-layers", + "--ssd-streaming-preload-experts", + "--system", + "--temp", + "--think", + "--think-max", + "--threads", + "--tokens", + "--top-p", + "--warm-weights", + ]; + const NON_PREFERENCES: &[&str] = &[ + "--backend", + "--coordinator", + "--cpu", + "--cuda", + "--cuda-tensor-parallel", + "--debug", + "--debug-hash", + "--decode-consistency", + "--dist-activation-bits", + "--dist-prefill-chunk", + "--dist-prefill-window", + "--dist-replay-check", + "--dump-logits", + "--dump-logprobs", + "--dump-tokens", + "--first-token-test", + "--gpu-devices", + "--gpu-vram", + "--head-test", + "--help", + "--imatrix-dataset", + "--imatrix-max-prompts", + "--imatrix-max-tokens", + "--imatrix-out", + "--inspect", + "--layers", + "--listen", + "--logprobs-top-k", + "--metal", + "--metal-graph-full-test", + "--metal-graph-generate", + "--metal-graph-prompt-test", + "--metal-graph-test", + "--perplexity-file", + "--prompt", + "--prompt-file", + "--raw", + "--raw-prompt", + "--rdma-device", + "--rdma-gid-index", + "--rocm", + "--role", + "--server", + "--tensor-parallel", + "--tensor-parallel-token-prefill", + "--transport", + ]; + let source = [ + include_str!("../../ds4/ds4_cli.c"), + include_str!("../../ds4/ds4_tp.c"), + include_str!("../../ds4/ds4_distributed.c"), + ] + .join("\n"); + let actual = option_branches(&source); + let classified = PREFERENCES + .iter() + .chain(NON_PREFERENCES) + .copied() + .collect::>(); + assert_eq!(actual, classified); + assert!( + PREFERENCES + .iter() + .all(|option| !NON_PREFERENCES.contains(option)) + ); + } + + fn option_branches(source: &str) -> BTreeSet<&str> { + source + .split("!strcmp(arg, \"") + .skip(1) + .filter_map(|tail| tail.split('"').next()) + .filter(|option| option.starts_with("--")) + .collect() + } }