use crate::model::{self, EngineArtifacts, ModelChoice}; use std::fmt; use std::path::Path; pub(crate) const REASONING_MODES: [ReasoningMode; 3] = [ ReasoningMode::High, ReasoningMode::Max, ReasoningMode::Direct, ]; const THINK_MAX_MIN_CONTEXT: i32 = 393_216; const MAX_CPU_THREADS: u32 = 32; const MAX_MTP_DRAFT_TOKENS: i32 = 16; pub(crate) const GIB: u64 = 1024 * 1024 * 1024; #[derive(Clone, Debug, PartialEq)] pub(crate) struct SpeculativePreferences { pub(crate) mtp_draft_tokens: i32, pub(crate) mtp_margin: f32, pub(crate) glm_mtp: bool, pub(crate) glm_mtp_timing: bool, pub(crate) dspark_enabled: bool, pub(crate) dspark_confidence_threshold: Option, pub(crate) dspark_strict: bool, } impl Default for SpeculativePreferences { fn default() -> Self { Self { mtp_draft_tokens: 1, mtp_margin: 3.0, glm_mtp: false, glm_mtp_timing: false, dspark_enabled: false, dspark_confidence_threshold: None, dspark_strict: false, } } } impl SpeculativePreferences { pub(crate) fn validate(&self, model: ModelChoice) -> Result<(), String> { if self.mtp_draft_tokens <= 0 { return Err("MTP draft tokens must be a positive whole number.".into()); } validate_float("MTP margin", self.mtp_margin, 0.0, 1000.0)?; if self.glm_mtp_timing && !self.glm_mtp { return Err("GLM MTP timing requires GLM MTP.".into()); } if model != ModelChoice::Glm52 && (self.glm_mtp || self.glm_mtp_timing) { return Err("GLM MTP is available only for GLM 5.2.".into()); } if self.dspark_enabled && !model.supports_dspark() { return Err("DSpark is not available for the selected model.".into()); } if (self.dspark_confidence_threshold.is_some() || self.dspark_strict) && !self.dspark_enabled { return Err("DSpark tuning requires DSpark to be enabled.".into()); } if let Some(threshold) = self.dspark_confidence_threshold { validate_float("DSpark confidence", threshold, 0.0, 1.0)?; } Ok(()) } pub(crate) fn engine_settings(&self) -> EngineSpeculativeSettings { EngineSpeculativeSettings { mtp_draft_tokens: self.mtp_draft_tokens.min(MAX_MTP_DRAFT_TOKENS), mtp_margin: self.mtp_margin, glm_mtp: self.glm_mtp, glm_mtp_timing: self.glm_mtp_timing, dspark: self.dspark_enabled, dspark_confidence_threshold: self.dspark_confidence_threshold.unwrap_or(0.9), dspark_confidence_threshold_set: self.dspark_confidence_threshold.is_some(), dspark_strict: self.dspark_strict, } } } #[derive(Clone, Copy, Debug, PartialEq)] pub(crate) struct EngineSpeculativeSettings { pub(crate) mtp_draft_tokens: i32, pub(crate) mtp_margin: f32, pub(crate) glm_mtp: bool, pub(crate) glm_mtp_timing: bool, pub(crate) dspark: bool, pub(crate) dspark_confidence_threshold: f32, pub(crate) dspark_confidence_threshold_set: bool, pub(crate) dspark_strict: bool, } #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub(crate) enum StreamingCacheBudget { Experts(u32), Gib(u64), } #[derive(Clone, Debug, Default, Eq, PartialEq)] pub(crate) struct SsdPreferences { pub(crate) enabled: bool, pub(crate) cold: bool, pub(crate) cache: Option, pub(crate) full_layers: Option, pub(crate) preload_experts: Option, } impl SsdPreferences { pub(crate) fn validate(&self, model: ModelChoice) -> Result<(), String> { if self.preload_experts == Some(0) { return Err("SSD preload experts must be a positive whole number.".into()); } if self .preload_experts .is_some_and(|experts| experts > i32::MAX as u32) { return Err("SSD preload experts is too large.".into()); } if self .full_layers .is_some_and(|layers| layers > i32::MAX as u32) { return Err("SSD full-layer count is too large.".into()); } if self.full_layers.is_some_and(|layers| layers > 0) && model != ModelChoice::Glm52 { return Err("Fully resident SSD layers are available only for GLM 5.2.".into()); } if let Some(StreamingCacheBudget::Gib(gib)) = self.cache { validate_gib("SSD cache budget", gib)?; } Ok(()) } pub(crate) fn engine_settings(&self) -> EngineSsdSettings { let (cache_experts, cache_bytes) = match self.cache { None => (0, 0), Some(StreamingCacheBudget::Experts(experts)) => (experts, 0), Some(StreamingCacheBudget::Gib(gib)) => (0, gib * GIB), }; EngineSsdSettings { enabled: self.enabled, cold: self.cold, cache_experts, cache_bytes, full_layers: self.full_layers.unwrap_or(0), full_layers_set: self.full_layers.is_some(), preload_experts: self.preload_experts.unwrap_or(0), } } } #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub(crate) struct EngineSsdSettings { pub(crate) enabled: bool, pub(crate) cold: bool, pub(crate) cache_experts: u32, pub(crate) cache_bytes: u64, pub(crate) full_layers: u32, pub(crate) full_layers_set: bool, pub(crate) preload_experts: u32, } #[derive(Clone, Debug, Default, PartialEq)] pub(crate) struct SteeringPreferences { pub(crate) file: Option, pub(crate) ffn_scale: Option, pub(crate) attention_scale: Option, } impl SteeringPreferences { pub(crate) fn validate(&self, model: ModelChoice) -> Result<(), String> { validate_optional_float("Directional FFN scale", self.ffn_scale, -100.0, 100.0)?; validate_optional_float( "Directional attention scale", self.attention_scale, -100.0, 100.0, )?; if self.file.is_some() && model == ModelChoice::Glm52 { return Err("Directional steering is not supported for GLM 5.2.".into()); } if self.file.is_none() && (self.ffn_scale.is_some_and(|scale| scale != 0.0) || self.attention_scale.is_some_and(|scale| scale != 0.0)) { return Err("Directional steering scales require a direction-vector file.".into()); } Ok(()) } pub(crate) fn engine_settings(&self) -> EngineSteeringSettings { let no_explicit_scale = self.ffn_scale.is_none() && self.attention_scale.is_none(); EngineSteeringSettings { file: self.file.clone(), ffn_scale: self .ffn_scale .unwrap_or(if self.file.is_some() && no_explicit_scale { 1.0 } else { 0.0 }), attention_scale: self.attention_scale.unwrap_or(0.0), } } } #[derive(Clone, Debug, PartialEq)] pub(crate) struct EngineSteeringSettings { pub(crate) file: Option, pub(crate) ffn_scale: f32, pub(crate) attention_scale: f32, } #[derive(Clone, Debug, Default, Eq, PartialEq)] pub(crate) struct DiagnosticPreferences { pub(crate) simulated_used_memory_gib: Option, pub(crate) expert_profile_path: Option, } impl DiagnosticPreferences { pub(crate) fn validate(&self) -> Result<(), String> { if let Some(gib) = self.simulated_used_memory_gib { validate_gib("Simulated used memory", gib)?; } Ok(()) } pub(crate) fn engine_settings(&self) -> EngineDiagnosticSettings { EngineDiagnosticSettings { simulated_used_memory_bytes: self.simulated_used_memory_gib.unwrap_or(0) * GIB, expert_profile_path: self.expert_profile_path.clone(), } } } #[derive(Clone, Debug, Eq, PartialEq)] pub(crate) struct EngineDiagnosticSettings { pub(crate) simulated_used_memory_bytes: u64, pub(crate) expert_profile_path: Option, } #[derive(Clone, Debug, Default, PartialEq)] pub(crate) struct RuntimePreferences { pub(crate) execution: ExecutionPreferences, pub(crate) speculative: SpeculativePreferences, pub(crate) ssd: SsdPreferences, pub(crate) steering: SteeringPreferences, pub(crate) diagnostics: DiagnosticPreferences, } impl RuntimePreferences { pub(crate) fn validate(&self, model: ModelChoice) -> Result<(), String> { self.execution.validate(model)?; self.speculative.validate(model)?; self.ssd.validate(model)?; self.steering.validate(model)?; self.diagnostics.validate()?; if self.ssd.enabled && self.speculative.dspark_enabled { return Err("SSD streaming is not compatible with DSpark support.".into()); } Ok(()) } fn engine_settings( &self, model: ModelChoice, context_tokens: i32, models_path: &Path, ) -> Result { self.validate(model)?; Ok(EngineSettings { model, artifacts: model::engine_artifacts(model, self.speculative.dspark_enabled, models_path), context_tokens, execution: self.execution.engine_settings(), speculative: self.speculative.engine_settings(), ssd: self.ssd.engine_settings(), steering: self.steering.engine_settings(), diagnostics: self.diagnostics.engine_settings(), }) } } #[derive(Clone, Debug, PartialEq)] pub(crate) struct EngineSettings { pub(crate) model: ModelChoice, pub(crate) artifacts: EngineArtifacts, pub(crate) context_tokens: i32, pub(crate) execution: EngineExecutionSettings, pub(crate) speculative: EngineSpeculativeSettings, pub(crate) ssd: EngineSsdSettings, pub(crate) steering: EngineSteeringSettings, pub(crate) diagnostics: EngineDiagnosticSettings, } #[derive(Clone, Debug, Default, Eq, PartialEq)] pub(crate) struct ExecutionPreferences { pub(crate) cpu_threads: Option, pub(crate) power_percent: Option, pub(crate) prefill_chunk: Option, pub(crate) quality: bool, pub(crate) warm_weights: bool, } impl ExecutionPreferences { pub(crate) fn validate(&self, model: ModelChoice) -> Result<(), String> { if self.cpu_threads == Some(0) { return Err("CPU helper threads must be a positive whole number.".into()); } if self .cpu_threads .is_some_and(|threads| threads > i32::MAX as u32) { return Err("CPU helper threads is too large.".into()); } if self .power_percent .is_some_and(|power| !(1..=100).contains(&power)) { return Err("GPU power must be between 1 and 100 percent.".into()); } if self.prefill_chunk == Some(0) { return Err("Prefill chunk must be a positive whole number.".into()); } if self .prefill_chunk .is_some_and(|chunk| chunk > i32::MAX as u32) { return Err("Prefill chunk is too large.".into()); } if model == ModelChoice::Glm52 { if self.power_percent.is_some_and(|power| power != 100) { return Err("GLM 5.2 currently requires 100% GPU power.".into()); } if self.prefill_chunk.is_some() { return Err("GLM 5.2 selects its prefill chunk automatically.".into()); } } Ok(()) } pub(crate) fn engine_settings(&self) -> EngineExecutionSettings { EngineExecutionSettings { cpu_threads: self.cpu_threads.unwrap_or(0).min(MAX_CPU_THREADS), power_percent: self.power_percent.unwrap_or(0), prefill_chunk: self.prefill_chunk.unwrap_or(0), quality: self.quality, warm_weights: self.warm_weights, } } } #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub(crate) struct EngineExecutionSettings { pub(crate) cpu_threads: u32, pub(crate) power_percent: u8, pub(crate) prefill_chunk: u32, pub(crate) quality: bool, pub(crate) warm_weights: bool, } #[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] pub(crate) enum ReasoningMode { Direct, #[default] High, Max, } impl ReasoningMode { pub(crate) fn id(self) -> &'static str { match self { Self::Direct => "none", Self::High => "high", Self::Max => "max", } } pub(crate) fn from_id(id: &str) -> Option { REASONING_MODES.into_iter().find(|mode| mode.id() == id) } } impl fmt::Display for ReasoningMode { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { formatter.write_str(match self { Self::Direct => "Direct (no thinking)", Self::High => "Thinking", Self::Max => "Think Max", }) } } #[derive(Clone, Debug, PartialEq)] pub(crate) struct GenerationPreferences { pub(crate) context_tokens: i32, pub(crate) max_generated_tokens: i32, pub(crate) system_prompt: String, pub(crate) temperature: Option, pub(crate) top_p: Option, pub(crate) min_p: Option, pub(crate) seed: Option, pub(crate) reasoning_mode: ReasoningMode, } impl Default for GenerationPreferences { fn default() -> Self { Self { context_tokens: 32_768, max_generated_tokens: 50_000, system_prompt: "You are a helpful assistant".into(), temperature: None, top_p: None, min_p: None, seed: None, reasoning_mode: ReasoningMode::High, } } } impl GenerationPreferences { pub(crate) fn validate(&self) -> Result<(), String> { if self.context_tokens <= 0 { return Err("Context tokens must be a positive whole number.".into()); } if self.max_generated_tokens <= 0 { return Err("Maximum generated tokens must be a positive whole number.".into()); } validate_optional_float("Temperature", self.temperature, 0.0, 100.0)?; validate_optional_float("Top-p", self.top_p, 0.0, 1.0)?; validate_optional_float("Min-p", self.min_p, 0.0, 1.0)?; if self.seed == Some(0) { return Err("Seed must be a positive whole number.".into()); } Ok(()) } pub(crate) fn turn_settings(&self, model: ModelChoice) -> TurnSettings { let glm = model == ModelChoice::Glm52; TurnSettings { context_tokens: self.context_tokens, max_generated_tokens: self.max_generated_tokens, system_prompt: self.system_prompt.clone(), temperature: self.temperature.unwrap_or(1.0), top_p: self.top_p.unwrap_or(if glm { 0.95 } else { 1.0 }), min_p: self.min_p.unwrap_or(if glm { 0.0 } else { 0.05 }), top_k: 0, stops: Vec::new(), seed: self.seed, reasoning_mode: if self.reasoning_mode == ReasoningMode::Max && self.context_tokens < THINK_MAX_MIN_CONTEXT { ReasoningMode::High } else { self.reasoning_mode }, } } } fn validate_optional_float( name: &str, value: Option, minimum: f32, maximum: f32, ) -> Result<(), String> { if let Some(value) = value { validate_float(name, value, minimum, maximum)?; } Ok(()) } fn validate_float(name: &str, value: f32, minimum: f32, maximum: f32) -> Result<(), String> { if !value.is_finite() || !(minimum..=maximum).contains(&value) { return Err(format!("{name} must be between {minimum} and {maximum}.")); } Ok(()) } fn validate_gib(name: &str, gib: u64) -> Result<(), String> { if gib == 0 || gib > u64::MAX / GIB { return Err(format!("{name} must be a positive whole GiB value.")); } Ok(()) } #[derive(Clone, Debug, PartialEq)] pub(crate) struct TurnSettings { pub(crate) context_tokens: i32, pub(crate) max_generated_tokens: i32, pub(crate) system_prompt: String, pub(crate) temperature: f32, pub(crate) top_p: f32, pub(crate) min_p: f32, pub(crate) top_k: i32, pub(crate) stops: Vec, pub(crate) seed: Option, pub(crate) reasoning_mode: ReasoningMode, } #[derive(Clone, Debug, PartialEq)] pub(crate) struct EffectiveSettings { pub(crate) engine: EngineSettings, pub(crate) turn: TurnSettings, } pub(crate) fn effective_settings( model: ModelChoice, generation: &GenerationPreferences, runtime: &RuntimePreferences, models_path: &Path, ) -> Result { generation.validate()?; Ok(EffectiveSettings { engine: runtime.engine_settings(model, generation.context_tokens, models_path)?, turn: generation.turn_settings(model), }) } #[cfg(test)] mod tests { use super::*; use std::collections::BTreeSet; #[test] fn effective_settings_preserve_unset_model_defaults() { let defaults = GenerationPreferences::default(); let deepseek = defaults.turn_settings(ModelChoice::DeepSeekV4Flash); assert_eq!( (deepseek.temperature, deepseek.top_p, deepseek.min_p), (1.0, 1.0, 0.05) ); let glm = defaults.turn_settings(ModelChoice::Glm52); assert_eq!((glm.temperature, glm.top_p, glm.min_p), (1.0, 0.95, 0.0)); let explicit = GenerationPreferences { top_p: Some(0.4), min_p: Some(0.2), reasoning_mode: ReasoningMode::Max, ..defaults }; let effective = explicit.turn_settings(ModelChoice::Glm52); assert_eq!((effective.top_p, effective.min_p), (0.4, 0.2)); assert_eq!(effective.reasoning_mode, ReasoningMode::High); } #[test] fn execution_settings_keep_ds4_automatic_values_and_glm_rules() { let defaults = ExecutionPreferences::default().engine_settings(); assert_eq!( ( defaults.cpu_threads, defaults.power_percent, defaults.prefill_chunk ), (0, 0, 0) ); let tuned = ExecutionPreferences { cpu_threads: Some(100), power_percent: Some(50), prefill_chunk: Some(4096), ..ExecutionPreferences::default() }; assert!(tuned.validate(ModelChoice::DeepSeekV4Flash).is_ok()); assert!(tuned.validate(ModelChoice::Glm52).is_err()); assert_eq!(tuned.engine_settings().cpu_threads, MAX_CPU_THREADS); } #[test] fn speculative_settings_match_ds4_defaults_and_dependencies() { let defaults = SpeculativePreferences::default(); let engine = defaults.engine_settings(); assert_eq!((engine.mtp_draft_tokens, engine.mtp_margin), (1, 3.0)); assert_eq!(engine.dspark_confidence_threshold, 0.9); assert!(!engine.dspark_confidence_threshold_set); let tuned = SpeculativePreferences { mtp_draft_tokens: 20, dspark_enabled: true, dspark_confidence_threshold: Some(0.7), dspark_strict: true, ..defaults }; assert!(tuned.validate(ModelChoice::DeepSeekV4Flash).is_ok()); assert!(tuned.validate(ModelChoice::Glm52).is_err()); assert_eq!(tuned.engine_settings().mtp_draft_tokens, 16); let glm = SpeculativePreferences { glm_mtp: true, glm_mtp_timing: true, ..SpeculativePreferences::default() }; assert!(glm.validate(ModelChoice::Glm52).is_ok()); assert!(glm.validate(ModelChoice::DeepSeekV4Pro).is_err()); } #[test] fn runtime_settings_preserve_automatic_values_and_cross_field_rules() { let runtime = RuntimePreferences { ssd: SsdPreferences { enabled: true, cache: Some(StreamingCacheBudget::Gib(64)), full_layers: Some(0), ..SsdPreferences::default() }, steering: SteeringPreferences { file: Some("direction.gguf".into()), ..SteeringPreferences::default() }, diagnostics: DiagnosticPreferences { simulated_used_memory_gib: Some(8), ..DiagnosticPreferences::default() }, ..RuntimePreferences::default() }; let effective = effective_settings( ModelChoice::DeepSeekV4Flash, &GenerationPreferences::default(), &runtime, Path::new("/models"), ) .unwrap(); let engine = effective.engine; assert_eq!(engine.context_tokens, 32_768); assert!(engine.artifacts.mtp.is_none()); assert_eq!(engine.ssd.cache_bytes, 64 * GIB); assert!(engine.ssd.full_layers_set); assert_eq!(engine.ssd.full_layers, 0); assert_eq!(engine.steering.ffn_scale, 1.0); assert_eq!(engine.diagnostics.simulated_used_memory_bytes, 8 * GIB); let incompatible = RuntimePreferences { speculative: SpeculativePreferences { dspark_enabled: true, ..SpeculativePreferences::default() }, ..runtime }; assert!(incompatible.validate(ModelChoice::DeepSeekV4Flash).is_err()); } #[test] fn effective_settings_build_one_engine_and_turn_configuration() { let generation = GenerationPreferences { context_tokens: 65_536, temperature: Some(0.25), ..GenerationPreferences::default() }; let runtime = RuntimePreferences { speculative: SpeculativePreferences { dspark_enabled: true, ..SpeculativePreferences::default() }, ..RuntimePreferences::default() }; let effective = effective_settings( ModelChoice::DeepSeekV4Flash, &generation, &runtime, Path::new("/models"), ) .unwrap(); assert_eq!(effective.engine.context_tokens, 65_536); assert_eq!( effective.engine.artifacts.model.parent(), Some(Path::new("/models/deepseek-v4-flash")) ); assert!(effective.engine.artifacts.mtp.is_some()); assert_eq!(effective.turn.temperature, 0.25); } #[test] fn captured_ds4_cli_options_are_uniquely_classified() { const PREFERENCES: &[&str] = &[ "--ctx", "--dir-steering-attn", "--dir-steering-ffn", "--dir-steering-file", "--dspark", "--dspark-confidence", "--dspark-strict", "--expert-profile", "--glm-mtp", "--glm-mtp-timing", "--min-p", "--model", "--mtp", "--mtp-draft", "--mtp-margin", "--nothink", "--power", "--prefill-chunk", "--quality", "--seed", "--simulate-used-memory", "--ssd-streaming", "--ssd-streaming-cache-experts", "--ssd-streaming-cold", "--ssd-streaming-full-layers", "--ssd-streaming-preload-experts", "--system", "--temp", "--think", "--think-max", "--threads", "--tokens", "--top-p", "--warm-weights", ]; const NON_PREFERENCES: &[&str] = &[ "--backend", "--coordinator", "--cpu", "--cuda", "--cuda-tensor-parallel", "--debug", "--debug-hash", "--decode-consistency", "--dist-activation-bits", "--dist-prefill-chunk", "--dist-prefill-window", "--dist-replay-check", "--dump-logits", "--dump-logprobs", "--dump-tokens", "--first-token-test", "--gpu-devices", "--gpu-vram", "--head-test", "--help", "--imatrix-dataset", "--imatrix-max-prompts", "--imatrix-max-tokens", "--imatrix-out", "--inspect", "--layers", "--listen", "--logprobs-top-k", "--metal", "--metal-graph-full-test", "--metal-graph-generate", "--metal-graph-prompt-test", "--metal-graph-test", "--perplexity-file", "--prompt", "--prompt-file", "--raw", "--raw-prompt", "--rdma-device", "--rdma-gid-index", "--rocm", "--role", "--server", "--tensor-parallel", "--tensor-parallel-token-prefill", "--transport", ]; let classified = PREFERENCES .iter() .chain(NON_PREFERENCES) .copied() .collect::>(); assert_eq!(classified.len(), PREFERENCES.len() + NON_PREFERENCES.len()); assert!( PREFERENCES .iter() .all(|option| !NON_PREFERENCES.contains(option)) ); } }