514 lines
19 KiB
Rust
514 lines
19 KiB
Rust
use super::*;
|
|
|
|
#[derive(Clone)]
|
|
pub(super) struct PreferenceDraft {
|
|
pub(super) model: ModelChoice,
|
|
pub(super) dspark_enabled: bool,
|
|
pub(super) idle_timeout_minutes: String,
|
|
pub(super) endpoint_port: String,
|
|
pub(super) context_tokens: String,
|
|
pub(super) max_generated_tokens: String,
|
|
pub(super) system_prompt: String,
|
|
pub(super) temperature: String,
|
|
pub(super) top_p: String,
|
|
pub(super) min_p: String,
|
|
pub(super) seed: String,
|
|
pub(super) reasoning_mode: ReasoningMode,
|
|
pub(super) cpu_threads: String,
|
|
pub(super) power_percent: String,
|
|
pub(super) prefill_chunk: String,
|
|
pub(super) quality: bool,
|
|
pub(super) warm_weights: bool,
|
|
pub(super) mtp_draft_tokens: String,
|
|
pub(super) mtp_margin: String,
|
|
pub(super) glm_mtp: bool,
|
|
pub(super) glm_mtp_timing: bool,
|
|
pub(super) dspark_confidence_threshold: String,
|
|
pub(super) dspark_strict: bool,
|
|
pub(super) ssd_streaming: bool,
|
|
pub(super) ssd_streaming_cold: bool,
|
|
pub(super) ssd_cache: String,
|
|
pub(super) ssd_full_layers: String,
|
|
pub(super) ssd_preload_experts: String,
|
|
pub(super) directional_steering_file: String,
|
|
pub(super) directional_steering_ffn: String,
|
|
pub(super) directional_steering_attn: String,
|
|
pub(super) simulated_used_memory_gib: String,
|
|
pub(super) expert_profile_path: String,
|
|
}
|
|
|
|
impl PreferenceDraft {
|
|
pub(super) fn from_saved(preferences: &AppPreferences) -> Result<Self, String> {
|
|
let model = ModelChoice::from_id(&preferences.selected_model)
|
|
.ok_or_else(|| format!("Unsupported model: {}", preferences.selected_model))?;
|
|
let generation = preferences.generation()?;
|
|
let runtime = preferences.runtime()?;
|
|
runtime.validate(model)?;
|
|
let execution = &runtime.execution;
|
|
let speculative = &runtime.speculative;
|
|
Ok(Self {
|
|
model,
|
|
dspark_enabled: speculative.dspark_enabled,
|
|
idle_timeout_minutes: preferences.idle_timeout_minutes.to_string(),
|
|
endpoint_port: preferences.endpoint_port.to_string(),
|
|
context_tokens: generation.context_tokens.to_string(),
|
|
max_generated_tokens: generation.max_generated_tokens.to_string(),
|
|
system_prompt: generation.system_prompt,
|
|
temperature: generation
|
|
.temperature
|
|
.map_or_else(String::new, |value| value.to_string()),
|
|
top_p: generation
|
|
.top_p
|
|
.map_or_else(String::new, |value| value.to_string()),
|
|
min_p: generation
|
|
.min_p
|
|
.map_or_else(String::new, |value| value.to_string()),
|
|
seed: generation
|
|
.seed
|
|
.map_or_else(String::new, |value| value.to_string()),
|
|
reasoning_mode: generation.reasoning_mode,
|
|
cpu_threads: execution
|
|
.cpu_threads
|
|
.map_or_else(String::new, |value| value.to_string()),
|
|
power_percent: execution
|
|
.power_percent
|
|
.map_or_else(String::new, |value| value.to_string()),
|
|
prefill_chunk: execution
|
|
.prefill_chunk
|
|
.map_or_else(String::new, |value| value.to_string()),
|
|
quality: execution.quality,
|
|
warm_weights: execution.warm_weights,
|
|
mtp_draft_tokens: speculative.mtp_draft_tokens.to_string(),
|
|
mtp_margin: speculative.mtp_margin.to_string(),
|
|
glm_mtp: speculative.glm_mtp,
|
|
glm_mtp_timing: speculative.glm_mtp_timing,
|
|
dspark_confidence_threshold: speculative
|
|
.dspark_confidence_threshold
|
|
.map_or_else(String::new, |value| value.to_string()),
|
|
dspark_strict: speculative.dspark_strict,
|
|
ssd_streaming: runtime.ssd.enabled,
|
|
ssd_streaming_cold: runtime.ssd.cold,
|
|
ssd_cache: runtime
|
|
.ssd
|
|
.cache
|
|
.map_or_else(String::new, |cache| match cache {
|
|
StreamingCacheBudget::Experts(experts) => experts.to_string(),
|
|
StreamingCacheBudget::Gib(gib) => format!("{gib}GB"),
|
|
}),
|
|
ssd_full_layers: runtime
|
|
.ssd
|
|
.full_layers
|
|
.map_or_else(String::new, |value| value.to_string()),
|
|
ssd_preload_experts: runtime
|
|
.ssd
|
|
.preload_experts
|
|
.map_or_else(String::new, |value| value.to_string()),
|
|
directional_steering_file: runtime.steering.file.unwrap_or_default(),
|
|
directional_steering_ffn: runtime
|
|
.steering
|
|
.ffn_scale
|
|
.map_or_else(String::new, |value| value.to_string()),
|
|
directional_steering_attn: runtime
|
|
.steering
|
|
.attention_scale
|
|
.map_or_else(String::new, |value| value.to_string()),
|
|
simulated_used_memory_gib: runtime
|
|
.diagnostics
|
|
.simulated_used_memory_gib
|
|
.map_or_else(String::new, |value| value.to_string()),
|
|
expert_profile_path: runtime.diagnostics.expert_profile_path.unwrap_or_default(),
|
|
})
|
|
}
|
|
|
|
pub(super) fn generation(&self) -> Result<GenerationPreferences, String> {
|
|
let preferences = GenerationPreferences {
|
|
context_tokens: parse_positive_i32("Context tokens", &self.context_tokens)?,
|
|
max_generated_tokens: parse_positive_i32(
|
|
"Maximum generated tokens",
|
|
&self.max_generated_tokens,
|
|
)?,
|
|
system_prompt: self.system_prompt.clone(),
|
|
temperature: parse_optional_f32("Temperature", &self.temperature)?,
|
|
top_p: parse_optional_f32("Top-p", &self.top_p)?,
|
|
min_p: parse_optional_f32("Min-p", &self.min_p)?,
|
|
seed: parse_optional_u64("Seed", &self.seed)?,
|
|
reasoning_mode: self.reasoning_mode,
|
|
};
|
|
preferences.validate()?;
|
|
Ok(preferences)
|
|
}
|
|
|
|
pub(super) fn reset(&mut self) {
|
|
let defaults = GenerationPreferences::default();
|
|
let execution = ExecutionPreferences::default();
|
|
let speculative = SpeculativePreferences::default();
|
|
let ssd = SsdPreferences::default();
|
|
self.model = ModelChoice::default();
|
|
self.dspark_enabled = false;
|
|
self.idle_timeout_minutes = "10".into();
|
|
self.endpoint_port = "4000".into();
|
|
self.context_tokens = defaults.context_tokens.to_string();
|
|
self.max_generated_tokens = defaults.max_generated_tokens.to_string();
|
|
self.system_prompt = defaults.system_prompt;
|
|
self.temperature.clear();
|
|
self.top_p.clear();
|
|
self.min_p.clear();
|
|
self.seed.clear();
|
|
self.reasoning_mode = defaults.reasoning_mode;
|
|
self.cpu_threads.clear();
|
|
self.power_percent.clear();
|
|
self.prefill_chunk.clear();
|
|
self.quality = execution.quality;
|
|
self.warm_weights = execution.warm_weights;
|
|
self.mtp_draft_tokens = speculative.mtp_draft_tokens.to_string();
|
|
self.mtp_margin = speculative.mtp_margin.to_string();
|
|
self.glm_mtp = speculative.glm_mtp;
|
|
self.glm_mtp_timing = speculative.glm_mtp_timing;
|
|
self.dspark_confidence_threshold.clear();
|
|
self.dspark_strict = speculative.dspark_strict;
|
|
self.ssd_streaming = ssd.enabled;
|
|
self.ssd_streaming_cold = ssd.cold;
|
|
self.ssd_cache.clear();
|
|
self.ssd_full_layers.clear();
|
|
self.ssd_preload_experts.clear();
|
|
self.directional_steering_file.clear();
|
|
self.directional_steering_ffn.clear();
|
|
self.directional_steering_attn.clear();
|
|
self.simulated_used_memory_gib.clear();
|
|
self.expert_profile_path.clear();
|
|
}
|
|
|
|
pub(super) fn execution(&self) -> Result<ExecutionPreferences, String> {
|
|
Ok(ExecutionPreferences {
|
|
cpu_threads: parse_optional_u32("CPU helper threads", &self.cpu_threads)?,
|
|
power_percent: parse_optional_u8("GPU power", &self.power_percent)?,
|
|
prefill_chunk: parse_optional_u32("Prefill chunk", &self.prefill_chunk)?,
|
|
quality: self.quality,
|
|
warm_weights: self.warm_weights,
|
|
})
|
|
}
|
|
|
|
pub(super) fn speculative(&self) -> Result<SpeculativePreferences, String> {
|
|
Ok(SpeculativePreferences {
|
|
mtp_draft_tokens: parse_positive_i32("MTP draft tokens", &self.mtp_draft_tokens)?,
|
|
mtp_margin: parse_f32("MTP margin", &self.mtp_margin)?,
|
|
glm_mtp: self.glm_mtp,
|
|
glm_mtp_timing: self.glm_mtp_timing,
|
|
dspark_enabled: self.dspark_enabled,
|
|
dspark_confidence_threshold: parse_optional_f32(
|
|
"DSpark confidence",
|
|
&self.dspark_confidence_threshold,
|
|
)?,
|
|
dspark_strict: self.dspark_strict,
|
|
})
|
|
}
|
|
|
|
pub(super) fn runtime(&self) -> Result<RuntimePreferences, String> {
|
|
Ok(RuntimePreferences {
|
|
execution: self.execution()?,
|
|
speculative: self.speculative()?,
|
|
ssd: SsdPreferences {
|
|
enabled: self.ssd_streaming,
|
|
cold: self.ssd_streaming_cold,
|
|
cache: parse_streaming_cache(&self.ssd_cache)?,
|
|
full_layers: parse_optional_u32("SSD full-layer count", &self.ssd_full_layers)?,
|
|
preload_experts: parse_optional_u32(
|
|
"SSD preload experts",
|
|
&self.ssd_preload_experts,
|
|
)?,
|
|
},
|
|
steering: SteeringPreferences {
|
|
file: optional_text(&self.directional_steering_file),
|
|
ffn_scale: parse_optional_f32(
|
|
"Directional FFN scale",
|
|
&self.directional_steering_ffn,
|
|
)?,
|
|
attention_scale: parse_optional_f32(
|
|
"Directional attention scale",
|
|
&self.directional_steering_attn,
|
|
)?,
|
|
},
|
|
diagnostics: DiagnosticPreferences {
|
|
simulated_used_memory_gib: parse_optional_gib(
|
|
"Simulated used memory",
|
|
&self.simulated_used_memory_gib,
|
|
)?,
|
|
expert_profile_path: optional_text(&self.expert_profile_path),
|
|
},
|
|
})
|
|
}
|
|
}
|
|
|
|
fn parse_positive_i32(name: &str, value: &str) -> Result<i32, String> {
|
|
value
|
|
.trim()
|
|
.parse::<i32>()
|
|
.ok()
|
|
.filter(|value| *value > 0)
|
|
.ok_or_else(|| format!("{name} must be a positive whole number."))
|
|
}
|
|
|
|
fn parse_optional_f32(name: &str, value: &str) -> Result<Option<f32>, String> {
|
|
let value = value.trim();
|
|
if value.is_empty() {
|
|
Ok(None)
|
|
} else {
|
|
value
|
|
.parse()
|
|
.map(Some)
|
|
.map_err(|_| format!("{name} must be a number or left blank for the DS4 default."))
|
|
}
|
|
}
|
|
|
|
fn parse_f32(name: &str, value: &str) -> Result<f32, String> {
|
|
value
|
|
.trim()
|
|
.parse()
|
|
.map_err(|_| format!("{name} must be a number."))
|
|
}
|
|
|
|
fn parse_optional_u64(name: &str, value: &str) -> Result<Option<u64>, String> {
|
|
let value = value.trim();
|
|
if value.is_empty() {
|
|
Ok(None)
|
|
} else {
|
|
value
|
|
.parse()
|
|
.map(Some)
|
|
.map_err(|_| format!("{name} must be a positive whole number or left blank."))
|
|
}
|
|
}
|
|
|
|
fn parse_optional_u32(name: &str, value: &str) -> Result<Option<u32>, String> {
|
|
parse_optional_number(name, value)
|
|
}
|
|
|
|
fn parse_optional_u8(name: &str, value: &str) -> Result<Option<u8>, String> {
|
|
parse_optional_number(name, value)
|
|
}
|
|
|
|
fn parse_optional_number<T: std::str::FromStr>(
|
|
name: &str,
|
|
value: &str,
|
|
) -> Result<Option<T>, String> {
|
|
let value = value.trim();
|
|
if value.is_empty() {
|
|
Ok(None)
|
|
} else {
|
|
value
|
|
.parse()
|
|
.map(Some)
|
|
.map_err(|_| format!("{name} must be a positive whole number or left blank."))
|
|
}
|
|
}
|
|
|
|
pub(super) fn parse_streaming_cache(value: &str) -> Result<Option<StreamingCacheBudget>, String> {
|
|
let value = value.trim();
|
|
if value.is_empty() {
|
|
return Ok(None);
|
|
}
|
|
if value.len() > 2
|
|
&& value
|
|
.get(value.len() - 2..)
|
|
.is_some_and(|suffix| suffix.eq_ignore_ascii_case("gb"))
|
|
{
|
|
return parse_gib("SSD cache budget", value)
|
|
.map(|gib| Some(StreamingCacheBudget::Gib(gib)));
|
|
}
|
|
if !value.chars().all(|character| character.is_ascii_digit()) {
|
|
return Err(
|
|
"SSD cache budget must be a positive expert count or whole GiB value such as 64GB."
|
|
.into(),
|
|
);
|
|
}
|
|
value
|
|
.parse::<u32>()
|
|
.ok()
|
|
.filter(|value| *value > 0)
|
|
.map(StreamingCacheBudget::Experts)
|
|
.map(Some)
|
|
.ok_or_else(|| {
|
|
"SSD cache budget must be a positive expert count or whole GiB value such as 64GB."
|
|
.into()
|
|
})
|
|
}
|
|
|
|
pub(super) fn parse_optional_gib(name: &str, value: &str) -> Result<Option<u64>, String> {
|
|
let value = value.trim();
|
|
if value.is_empty() {
|
|
Ok(None)
|
|
} else {
|
|
parse_gib(name, value).map(Some)
|
|
}
|
|
}
|
|
|
|
fn parse_gib(name: &str, value: &str) -> Result<u64, String> {
|
|
let value = value
|
|
.get(value.len().saturating_sub(2)..)
|
|
.filter(|suffix| suffix.eq_ignore_ascii_case("gb"))
|
|
.map_or(value, |_| &value[..value.len() - 2]);
|
|
if !value.chars().all(|character| character.is_ascii_digit()) {
|
|
return Err(format!("{name} must be a positive whole GiB value."));
|
|
}
|
|
value
|
|
.parse::<u64>()
|
|
.ok()
|
|
.filter(|value| *value > 0 && *value <= u64::MAX / GIB)
|
|
.ok_or_else(|| format!("{name} must be a positive whole GiB value."))
|
|
}
|
|
|
|
fn optional_text(value: &str) -> Option<String> {
|
|
let value = value.trim();
|
|
(!value.is_empty()).then(|| value.to_owned())
|
|
}
|
|
|
|
fn update_runtime_preferences(
|
|
runtime_preferences: &RwLock<AppPreferences>,
|
|
preferences: &AppPreferences,
|
|
) {
|
|
*runtime_preferences
|
|
.write()
|
|
.unwrap_or_else(|poisoned| poisoned.into_inner()) = preferences.clone();
|
|
}
|
|
|
|
impl App {
|
|
pub(super) fn open_preferences(&mut self) {
|
|
if self.database.is_none() || self.pending_project_path.is_some() || self.choosing_folder {
|
|
return;
|
|
}
|
|
match PreferenceDraft::from_saved(&self.preferences) {
|
|
Ok(draft) => {
|
|
self.preference_draft = draft;
|
|
self.preference_error = None;
|
|
self.preferences_open = true;
|
|
}
|
|
Err(error) => self.error = Some(error),
|
|
}
|
|
}
|
|
|
|
pub(super) fn save_preferences(&mut self) {
|
|
let Ok(idle_timeout_minutes) = self
|
|
.preference_draft
|
|
.idle_timeout_minutes
|
|
.trim()
|
|
.parse::<i32>()
|
|
else {
|
|
self.preference_error = Some("Idle timeout must be a whole number.".into());
|
|
return;
|
|
};
|
|
if !(1..=1440).contains(&idle_timeout_minutes) {
|
|
self.preference_error = Some("Idle timeout must be between 1 and 1440 minutes.".into());
|
|
return;
|
|
}
|
|
let Ok(endpoint_port) = self.preference_draft.endpoint_port.trim().parse::<u16>() else {
|
|
self.preference_error = Some("Endpoint port must be a whole number.".into());
|
|
return;
|
|
};
|
|
if endpoint_port == 0 {
|
|
self.preference_error = Some("Endpoint port must be between 1 and 65535.".into());
|
|
return;
|
|
}
|
|
let generation = match self.preference_draft.generation() {
|
|
Ok(generation) => generation,
|
|
Err(error) => {
|
|
self.preference_error = Some(error);
|
|
return;
|
|
}
|
|
};
|
|
|
|
let model = self.preference_draft.model;
|
|
let runtime = match self.preference_draft.runtime() {
|
|
Ok(runtime) => runtime,
|
|
Err(error) => {
|
|
self.preference_error = Some(error);
|
|
return;
|
|
}
|
|
};
|
|
if let Err(error) = runtime.validate(model) {
|
|
self.preference_error = Some(error);
|
|
return;
|
|
}
|
|
#[cfg(target_os = "macos")]
|
|
let pending_endpoint = if self.preferences.endpoint_port != i32::from(endpoint_port)
|
|
|| self._endpoint.is_none()
|
|
{
|
|
let Some(generation) = &self.generation_service else {
|
|
self.preference_error = Some("The model runtime is unavailable.".into());
|
|
return;
|
|
};
|
|
match crate::server::ServerHandle::spawn(
|
|
generation.clone(),
|
|
Arc::clone(&self.runtime_preferences),
|
|
models_path(),
|
|
application_support_path().join("kv-cache").join("http"),
|
|
endpoint_port,
|
|
Arc::clone(&self.metrics),
|
|
) {
|
|
Ok(endpoint) => Some(endpoint),
|
|
Err(error) => {
|
|
self.preference_error = Some(error);
|
|
return;
|
|
}
|
|
}
|
|
} else {
|
|
None
|
|
};
|
|
let Some(database) = &mut self.database else {
|
|
return;
|
|
};
|
|
match database.update_preferences(
|
|
model.id(),
|
|
idle_timeout_minutes,
|
|
i32::from(endpoint_port),
|
|
&generation,
|
|
&runtime,
|
|
) {
|
|
Ok(preferences) => {
|
|
self.preferences = preferences;
|
|
#[cfg(target_os = "macos")]
|
|
update_runtime_preferences(&self.runtime_preferences, &self.preferences);
|
|
#[cfg(target_os = "macos")]
|
|
if let Some(endpoint) = pending_endpoint {
|
|
self._endpoint = Some(endpoint);
|
|
}
|
|
self.preference_draft = PreferenceDraft::from_saved(&self.preferences)
|
|
.expect("the saved model was selected from the supported catalog");
|
|
self.preferences_open = false;
|
|
self.preference_error = None;
|
|
self.error = None;
|
|
}
|
|
Err(error) => self.preference_error = Some(error),
|
|
}
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn runtime_preferences_update_after_lock_poisoning() {
|
|
let runtime = Arc::new(RwLock::new(AppPreferences::default()));
|
|
let poisoned = Arc::clone(&runtime);
|
|
let _ = std::thread::spawn(move || {
|
|
let _guard = poisoned.write().unwrap();
|
|
panic!("poison preference lock");
|
|
})
|
|
.join();
|
|
let preferences = AppPreferences {
|
|
endpoint_port: 4567,
|
|
..AppPreferences::default()
|
|
};
|
|
|
|
update_runtime_preferences(&runtime, &preferences);
|
|
|
|
assert_eq!(
|
|
runtime
|
|
.read()
|
|
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
|
.endpoint_port,
|
|
4567
|
|
);
|
|
}
|
|
}
|