449 lines
16 KiB
Rust
449 lines
16 KiB
Rust
use super::*;
|
|
|
|
#[derive(Clone)]
|
|
pub(super) struct PreferenceDraft {
|
|
pub(super) model: ModelChoice,
|
|
pub(super) dspark_enabled: bool,
|
|
pub(super) idle_timeout_minutes: String,
|
|
pub(super) endpoint_port: String,
|
|
pub(super) endpoint_enabled: bool,
|
|
pub(super) endpoint_cors: bool,
|
|
pub(super) context_tokens: String,
|
|
pub(super) max_generated_tokens: String,
|
|
pub(super) system_prompt: String,
|
|
pub(super) temperature: String,
|
|
pub(super) top_p: String,
|
|
pub(super) min_p: String,
|
|
pub(super) seed: String,
|
|
pub(super) reasoning_mode: ReasoningMode,
|
|
pub(super) cpu_threads: String,
|
|
pub(super) power_percent: String,
|
|
pub(super) prefill_chunk: String,
|
|
pub(super) quality: bool,
|
|
pub(super) warm_weights: bool,
|
|
pub(super) mtp_draft_tokens: String,
|
|
pub(super) mtp_margin: String,
|
|
pub(super) glm_mtp: bool,
|
|
pub(super) glm_mtp_timing: bool,
|
|
pub(super) dspark_confidence_threshold: String,
|
|
pub(super) dspark_strict: bool,
|
|
pub(super) ssd_streaming: bool,
|
|
pub(super) ssd_streaming_cold: bool,
|
|
pub(super) ssd_cache: String,
|
|
pub(super) ssd_full_layers: String,
|
|
pub(super) ssd_preload_experts: String,
|
|
pub(super) directional_steering_file: String,
|
|
pub(super) directional_steering_ffn: String,
|
|
pub(super) directional_steering_attn: String,
|
|
pub(super) simulated_used_memory_gib: String,
|
|
pub(super) expert_profile_path: String,
|
|
pub(super) kv_budget_gib: String,
|
|
pub(super) kv_min_tokens: String,
|
|
pub(super) kv_cold_max_tokens: String,
|
|
pub(super) kv_continued_interval_tokens: String,
|
|
}
|
|
|
|
impl PreferenceDraft {
|
|
pub(super) fn from_saved(config: &Config) -> Self {
|
|
let generation = &config.generation;
|
|
let runtime = &config.runtime;
|
|
let execution = &runtime.execution;
|
|
let speculative = &runtime.speculative;
|
|
Self {
|
|
model: config.model,
|
|
dspark_enabled: speculative.dspark_enabled,
|
|
idle_timeout_minutes: config.idle_timeout_minutes.to_string(),
|
|
endpoint_port: config.endpoint.port.to_string(),
|
|
endpoint_enabled: config.endpoint.enabled,
|
|
endpoint_cors: config.endpoint.cors,
|
|
context_tokens: generation.context_tokens.to_string(),
|
|
max_generated_tokens: generation.max_generated_tokens.to_string(),
|
|
system_prompt: generation.system_prompt.clone(),
|
|
temperature: optional_string(generation.temperature),
|
|
top_p: optional_string(generation.top_p),
|
|
min_p: optional_string(generation.min_p),
|
|
seed: optional_string(generation.seed),
|
|
reasoning_mode: generation.reasoning_mode,
|
|
cpu_threads: optional_string(execution.cpu_threads),
|
|
power_percent: optional_string(execution.power_percent),
|
|
prefill_chunk: optional_string(execution.prefill_chunk),
|
|
quality: execution.quality,
|
|
warm_weights: execution.warm_weights,
|
|
mtp_draft_tokens: speculative.mtp_draft_tokens.to_string(),
|
|
mtp_margin: speculative.mtp_margin.to_string(),
|
|
glm_mtp: speculative.glm_mtp,
|
|
glm_mtp_timing: speculative.glm_mtp_timing,
|
|
dspark_confidence_threshold: optional_string(speculative.dspark_confidence_threshold),
|
|
dspark_strict: speculative.dspark_strict,
|
|
ssd_streaming: runtime.ssd.enabled,
|
|
ssd_streaming_cold: runtime.ssd.cold,
|
|
ssd_cache: optional_string(runtime.ssd.cache),
|
|
ssd_full_layers: optional_string(runtime.ssd.full_layers),
|
|
ssd_preload_experts: optional_string(runtime.ssd.preload_experts),
|
|
directional_steering_file: runtime.steering.file.clone().unwrap_or_default(),
|
|
directional_steering_ffn: optional_string(runtime.steering.ffn_scale),
|
|
directional_steering_attn: optional_string(runtime.steering.attention_scale),
|
|
simulated_used_memory_gib: optional_string(
|
|
runtime.diagnostics.simulated_used_memory_gib,
|
|
),
|
|
expert_profile_path: runtime
|
|
.diagnostics
|
|
.expert_profile_path
|
|
.clone()
|
|
.unwrap_or_default(),
|
|
kv_budget_gib: optional_string(runtime.kv_cache.budget_gib),
|
|
kv_min_tokens: optional_string(runtime.kv_cache.min_tokens),
|
|
kv_cold_max_tokens: optional_string(runtime.kv_cache.cold_max_tokens),
|
|
kv_continued_interval_tokens: optional_string(
|
|
runtime.kv_cache.continued_interval_tokens,
|
|
),
|
|
}
|
|
}
|
|
|
|
pub(super) fn generation(&self) -> Result<GenerationPreferences, String> {
|
|
let preferences = GenerationPreferences {
|
|
context_tokens: parse_positive_i32("Context tokens", &self.context_tokens)?,
|
|
max_generated_tokens: parse_positive_i32(
|
|
"Maximum generated tokens",
|
|
&self.max_generated_tokens,
|
|
)?,
|
|
system_prompt: self.system_prompt.clone(),
|
|
temperature: parse_optional_f32("Temperature", &self.temperature)?,
|
|
top_p: parse_optional_f32("Top-p", &self.top_p)?,
|
|
min_p: parse_optional_f32("Min-p", &self.min_p)?,
|
|
seed: parse_optional_u64("Seed", &self.seed)?,
|
|
reasoning_mode: self.reasoning_mode,
|
|
};
|
|
preferences.validate()?;
|
|
Ok(preferences)
|
|
}
|
|
|
|
pub(super) fn reset(&mut self) {
|
|
*self = Self::from_saved(&Config::default());
|
|
}
|
|
|
|
pub(super) fn execution(&self) -> Result<ExecutionPreferences, String> {
|
|
Ok(ExecutionPreferences {
|
|
cpu_threads: parse_optional_u32("CPU helper threads", &self.cpu_threads)?,
|
|
power_percent: parse_optional_u8("GPU power", &self.power_percent)?,
|
|
prefill_chunk: parse_optional_u32("Prefill chunk", &self.prefill_chunk)?,
|
|
quality: self.quality,
|
|
warm_weights: self.warm_weights,
|
|
})
|
|
}
|
|
|
|
pub(super) fn speculative(&self) -> Result<SpeculativePreferences, String> {
|
|
Ok(SpeculativePreferences {
|
|
mtp_draft_tokens: parse_positive_i32("MTP draft tokens", &self.mtp_draft_tokens)?,
|
|
mtp_margin: parse_f32("MTP margin", &self.mtp_margin)?,
|
|
glm_mtp: self.glm_mtp,
|
|
glm_mtp_timing: self.glm_mtp_timing,
|
|
dspark_enabled: self.dspark_enabled,
|
|
dspark_confidence_threshold: parse_optional_f32(
|
|
"DSpark confidence",
|
|
&self.dspark_confidence_threshold,
|
|
)?,
|
|
dspark_strict: self.dspark_strict,
|
|
})
|
|
}
|
|
|
|
pub(super) fn runtime(&self) -> Result<RuntimePreferences, String> {
|
|
Ok(RuntimePreferences {
|
|
execution: self.execution()?,
|
|
speculative: self.speculative()?,
|
|
ssd: SsdPreferences {
|
|
enabled: self.ssd_streaming,
|
|
cold: self.ssd_streaming_cold,
|
|
cache: parse_streaming_cache(&self.ssd_cache)?,
|
|
full_layers: parse_optional_u32("SSD full-layer count", &self.ssd_full_layers)?,
|
|
preload_experts: parse_optional_u32(
|
|
"SSD preload experts",
|
|
&self.ssd_preload_experts,
|
|
)?,
|
|
},
|
|
steering: SteeringPreferences {
|
|
file: optional_text(&self.directional_steering_file),
|
|
ffn_scale: parse_optional_f32(
|
|
"Directional FFN scale",
|
|
&self.directional_steering_ffn,
|
|
)?,
|
|
attention_scale: parse_optional_f32(
|
|
"Directional attention scale",
|
|
&self.directional_steering_attn,
|
|
)?,
|
|
},
|
|
diagnostics: DiagnosticPreferences {
|
|
simulated_used_memory_gib: parse_optional_gib(
|
|
"Simulated used memory",
|
|
&self.simulated_used_memory_gib,
|
|
)?,
|
|
expert_profile_path: optional_text(&self.expert_profile_path),
|
|
},
|
|
kv_cache: KvCachePreferences {
|
|
budget_gib: parse_optional_gib("KV cache budget", &self.kv_budget_gib)?,
|
|
min_tokens: parse_optional_u32("KV cache minimum tokens", &self.kv_min_tokens)?,
|
|
cold_max_tokens: parse_optional_u32(
|
|
"KV cache cold maximum",
|
|
&self.kv_cold_max_tokens,
|
|
)?,
|
|
continued_interval_tokens: parse_optional_u32(
|
|
"KV cache continued interval",
|
|
&self.kv_continued_interval_tokens,
|
|
)?,
|
|
},
|
|
})
|
|
}
|
|
}
|
|
|
|
fn parse_positive_i32(name: &str, value: &str) -> Result<i32, String> {
|
|
value
|
|
.trim()
|
|
.parse::<i32>()
|
|
.ok()
|
|
.filter(|value| *value > 0)
|
|
.ok_or_else(|| format!("{name} must be a positive whole number."))
|
|
}
|
|
|
|
fn parse_optional_f32(name: &str, value: &str) -> Result<Option<f32>, String> {
|
|
let value = value.trim();
|
|
if value.is_empty() {
|
|
Ok(None)
|
|
} else {
|
|
value
|
|
.parse()
|
|
.map(Some)
|
|
.map_err(|_| format!("{name} must be a number or left blank for the DS4 default."))
|
|
}
|
|
}
|
|
|
|
fn parse_f32(name: &str, value: &str) -> Result<f32, String> {
|
|
value
|
|
.trim()
|
|
.parse()
|
|
.map_err(|_| format!("{name} must be a number."))
|
|
}
|
|
|
|
fn parse_optional_u64(name: &str, value: &str) -> Result<Option<u64>, String> {
|
|
let value = value.trim();
|
|
if value.is_empty() {
|
|
Ok(None)
|
|
} else {
|
|
value
|
|
.parse()
|
|
.map(Some)
|
|
.map_err(|_| format!("{name} must be a positive whole number or left blank."))
|
|
}
|
|
}
|
|
|
|
fn parse_optional_u32(name: &str, value: &str) -> Result<Option<u32>, String> {
|
|
parse_optional_number(name, value)
|
|
}
|
|
|
|
fn parse_optional_u8(name: &str, value: &str) -> Result<Option<u8>, String> {
|
|
parse_optional_number(name, value)
|
|
}
|
|
|
|
fn parse_optional_number<T: std::str::FromStr>(
|
|
name: &str,
|
|
value: &str,
|
|
) -> Result<Option<T>, String> {
|
|
let value = value.trim();
|
|
if value.is_empty() {
|
|
Ok(None)
|
|
} else {
|
|
value
|
|
.parse()
|
|
.map(Some)
|
|
.map_err(|_| format!("{name} must be a positive whole number or left blank."))
|
|
}
|
|
}
|
|
|
|
pub(super) fn parse_streaming_cache(value: &str) -> Result<Option<StreamingCacheBudget>, String> {
|
|
let value = value.trim();
|
|
if value.is_empty() {
|
|
return Ok(None);
|
|
}
|
|
value.parse().map(Some)
|
|
}
|
|
|
|
pub(super) fn parse_optional_gib(name: &str, value: &str) -> Result<Option<u64>, String> {
|
|
let value = value.trim();
|
|
if value.is_empty() {
|
|
Ok(None)
|
|
} else {
|
|
parse_gib(name, value).map(Some)
|
|
}
|
|
}
|
|
|
|
fn parse_gib(name: &str, value: &str) -> Result<u64, String> {
|
|
let value = value
|
|
.get(value.len().saturating_sub(2)..)
|
|
.filter(|suffix| suffix.eq_ignore_ascii_case("gb"))
|
|
.map_or(value, |_| &value[..value.len() - 2]);
|
|
if !value.chars().all(|character| character.is_ascii_digit()) {
|
|
return Err(format!("{name} must be a positive whole GiB value."));
|
|
}
|
|
value
|
|
.parse::<u64>()
|
|
.ok()
|
|
.filter(|value| *value > 0 && *value <= u64::MAX / GIB)
|
|
.ok_or_else(|| format!("{name} must be a positive whole GiB value."))
|
|
}
|
|
|
|
fn optional_text(value: &str) -> Option<String> {
|
|
let value = value.trim();
|
|
(!value.is_empty()).then(|| value.to_owned())
|
|
}
|
|
|
|
/// An unset preference shows as an empty field.
|
|
fn optional_string<T: ToString>(value: Option<T>) -> String {
|
|
value.map_or_else(String::new, |value| value.to_string())
|
|
}
|
|
|
|
fn update_runtime_config(runtime_config: &RwLock<Config>, config: &Config) {
|
|
*runtime_config
|
|
.write()
|
|
.unwrap_or_else(|poisoned| poisoned.into_inner()) = config.clone();
|
|
}
|
|
|
|
impl App {
|
|
pub(super) fn open_preferences(&mut self) {
|
|
if self.database.is_none() || self.pending_project_path.is_some() || self.choosing_folder {
|
|
return;
|
|
}
|
|
self.preference_draft = PreferenceDraft::from_saved(&self.config);
|
|
self.preference_error = None;
|
|
self.preferences_open = true;
|
|
}
|
|
|
|
pub(super) fn save_preferences(&mut self) {
|
|
let Ok(idle_timeout_minutes) = self
|
|
.preference_draft
|
|
.idle_timeout_minutes
|
|
.trim()
|
|
.parse::<i32>()
|
|
else {
|
|
self.preference_error = Some("Idle timeout must be a whole number.".into());
|
|
return;
|
|
};
|
|
let Ok(endpoint_port) = self.preference_draft.endpoint_port.trim().parse::<u16>() else {
|
|
self.preference_error = Some("Endpoint port must be a whole number.".into());
|
|
return;
|
|
};
|
|
let generation = match self.preference_draft.generation() {
|
|
Ok(generation) => generation,
|
|
Err(error) => {
|
|
self.preference_error = Some(error);
|
|
return;
|
|
}
|
|
};
|
|
let runtime = match self.preference_draft.runtime() {
|
|
Ok(runtime) => runtime,
|
|
Err(error) => {
|
|
self.preference_error = Some(error);
|
|
return;
|
|
}
|
|
};
|
|
let config = Config {
|
|
model: self.preference_draft.model,
|
|
idle_timeout_minutes,
|
|
endpoint: EndpointConfig {
|
|
port: i32::from(endpoint_port),
|
|
enabled: self.preference_draft.endpoint_enabled,
|
|
cors: self.preference_draft.endpoint_cors,
|
|
},
|
|
generation,
|
|
runtime,
|
|
interface: self.config.interface.clone(),
|
|
};
|
|
if let Err(error) = config.validate() {
|
|
self.preference_error = Some(error);
|
|
return;
|
|
}
|
|
#[cfg(target_os = "macos")]
|
|
let endpoint_changed = self.config.endpoint != config.endpoint;
|
|
#[cfg(target_os = "macos")]
|
|
if endpoint_changed
|
|
&& self.config.endpoint.port == config.endpoint.port
|
|
&& self._endpoint.is_some()
|
|
{
|
|
self._endpoint = None;
|
|
}
|
|
let pending_endpoint =
|
|
if config.endpoint.enabled && (endpoint_changed || self._endpoint.is_none()) {
|
|
let Some(generation) = &self.generation_service else {
|
|
self.preference_error = Some("The model runtime is unavailable.".into());
|
|
return;
|
|
};
|
|
match crate::server::ServerHandle::spawn(
|
|
generation.clone(),
|
|
Arc::clone(&self.runtime_config),
|
|
models_path(),
|
|
application_support_path().join("kv-cache").join("http"),
|
|
endpoint_port,
|
|
config.endpoint.cors,
|
|
Arc::clone(&self.metrics),
|
|
) {
|
|
Ok(endpoint) => Some(endpoint),
|
|
Err(error) => {
|
|
self.preference_error = Some(error);
|
|
return;
|
|
}
|
|
}
|
|
} else {
|
|
None
|
|
};
|
|
if let Err(error) = config.save(&config_path()) {
|
|
self.preference_error = Some(error);
|
|
return;
|
|
}
|
|
self.config = config;
|
|
#[cfg(target_os = "macos")]
|
|
update_runtime_config(&self.runtime_config, &self.config);
|
|
#[cfg(target_os = "macos")]
|
|
if endpoint_changed {
|
|
self._endpoint = pending_endpoint;
|
|
} else if let Some(endpoint) = pending_endpoint {
|
|
self._endpoint = Some(endpoint);
|
|
}
|
|
self.preference_draft = PreferenceDraft::from_saved(&self.config);
|
|
self.preferences_open = false;
|
|
self.preference_error = None;
|
|
self.error = None;
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn runtime_config_updates_after_lock_poisoning() {
|
|
let runtime = Arc::new(RwLock::new(Config::default()));
|
|
let poisoned = Arc::clone(&runtime);
|
|
let _ = std::thread::spawn(move || {
|
|
let _guard = poisoned.write().unwrap();
|
|
panic!("poison configuration lock");
|
|
})
|
|
.join();
|
|
let config = Config {
|
|
endpoint: EndpointConfig {
|
|
port: 4567,
|
|
..EndpointConfig::default()
|
|
},
|
|
..Config::default()
|
|
};
|
|
|
|
update_runtime_config(&runtime, &config);
|
|
|
|
assert_eq!(
|
|
runtime
|
|
.read()
|
|
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
|
.endpoint
|
|
.port,
|
|
4567
|
|
);
|
|
}
|
|
}
|