986 lines
33 KiB
Rust
986 lines
33 KiB
Rust
use crate::model::{self, EngineArtifacts, ModelChoice};
|
|
use serde::{Deserialize, Serialize};
|
|
use std::fmt;
|
|
use std::path::Path;
|
|
|
|
pub(crate) const REASONING_MODES: [ReasoningMode; 3] = [
|
|
ReasoningMode::High,
|
|
ReasoningMode::Max,
|
|
ReasoningMode::Direct,
|
|
];
|
|
const THINK_MAX_MIN_CONTEXT: i32 = 393_216;
|
|
const MAX_CPU_THREADS: u32 = 32;
|
|
const MAX_MTP_DRAFT_TOKENS: i32 = 16;
|
|
pub(crate) const GIB: u64 = 1024 * 1024 * 1024;
|
|
/// DS4 disk KV cache defaults, from `ds4_kvstore.h` and `--kv-disk-space-mb`.
|
|
pub(crate) const DEFAULT_KV_BUDGET_GIB: u64 = 4;
|
|
const DEFAULT_KV_MIN_TOKENS: u32 = 512;
|
|
const DEFAULT_KV_COLD_MAX_TOKENS: u32 = 30_000;
|
|
const DEFAULT_KV_CONTINUED_INTERVAL_TOKENS: u32 = 10_000;
|
|
|
|
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
|
|
#[serde(default, deny_unknown_fields)]
|
|
pub(crate) struct SpeculativePreferences {
|
|
pub(crate) mtp_draft_tokens: i32,
|
|
pub(crate) mtp_margin: f32,
|
|
pub(crate) legacy_mtp_enabled: bool,
|
|
pub(crate) glm_mtp: bool,
|
|
pub(crate) glm_mtp_timing: bool,
|
|
pub(crate) dspark_enabled: bool,
|
|
pub(crate) dspark_confidence_threshold: Option<f32>,
|
|
pub(crate) dspark_strict: bool,
|
|
}
|
|
|
|
impl Default for SpeculativePreferences {
|
|
fn default() -> Self {
|
|
Self {
|
|
mtp_draft_tokens: 1,
|
|
mtp_margin: 3.0,
|
|
legacy_mtp_enabled: false,
|
|
glm_mtp: false,
|
|
glm_mtp_timing: false,
|
|
dspark_enabled: false,
|
|
dspark_confidence_threshold: None,
|
|
dspark_strict: false,
|
|
}
|
|
}
|
|
}
|
|
|
|
impl SpeculativePreferences {
|
|
pub(crate) fn validate(&self, model: ModelChoice) -> Result<(), String> {
|
|
if self.mtp_draft_tokens <= 0 {
|
|
return Err("MTP draft tokens must be a positive whole number.".into());
|
|
}
|
|
validate_float("MTP margin", self.mtp_margin, 0.0, 1000.0)?;
|
|
if self.glm_mtp_timing && !self.glm_mtp {
|
|
return Err("GLM MTP timing requires GLM MTP.".into());
|
|
}
|
|
if model != ModelChoice::Glm52 && (self.glm_mtp || self.glm_mtp_timing) {
|
|
return Err("GLM MTP is available only for GLM 5.2.".into());
|
|
}
|
|
if self.dspark_enabled && !model.supports_dspark() {
|
|
return Err("DSpark is not available for the selected model.".into());
|
|
}
|
|
if self.legacy_mtp_enabled && !model.supports_dspark() {
|
|
return Err("Legacy MTP is not available for the selected model.".into());
|
|
}
|
|
if self.legacy_mtp_enabled && self.dspark_enabled {
|
|
return Err("Legacy MTP and DSpark use different support artifacts.".into());
|
|
}
|
|
if (self.dspark_confidence_threshold.is_some() || self.dspark_strict)
|
|
&& !self.dspark_enabled
|
|
{
|
|
return Err("DSpark tuning requires DSpark to be enabled.".into());
|
|
}
|
|
if let Some(threshold) = self.dspark_confidence_threshold {
|
|
validate_float("DSpark confidence", threshold, 0.0, 1.0)?;
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
pub(crate) fn engine_settings(&self) -> EngineSpeculativeSettings {
|
|
EngineSpeculativeSettings {
|
|
mtp_draft_tokens: self.mtp_draft_tokens.min(MAX_MTP_DRAFT_TOKENS),
|
|
mtp_margin: self.mtp_margin,
|
|
glm_mtp: self.glm_mtp,
|
|
glm_mtp_timing: self.glm_mtp_timing,
|
|
dspark: self.dspark_enabled,
|
|
dspark_confidence_threshold: self.dspark_confidence_threshold.unwrap_or(0.9),
|
|
dspark_confidence_threshold_set: self.dspark_confidence_threshold.is_some(),
|
|
dspark_strict: self.dspark_strict,
|
|
}
|
|
}
|
|
}
|
|
|
|
#[derive(Clone, Copy, Debug, PartialEq)]
|
|
pub(crate) struct EngineSpeculativeSettings {
|
|
pub(crate) mtp_draft_tokens: i32,
|
|
pub(crate) mtp_margin: f32,
|
|
pub(crate) glm_mtp: bool,
|
|
pub(crate) glm_mtp_timing: bool,
|
|
pub(crate) dspark: bool,
|
|
pub(crate) dspark_confidence_threshold: f32,
|
|
pub(crate) dspark_confidence_threshold_set: bool,
|
|
pub(crate) dspark_strict: bool,
|
|
}
|
|
|
|
/// An expert count, or a whole GiB budget. Written as `4` or `64GB`, the same
|
|
/// spelling the preferences window accepts.
|
|
#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
|
|
#[serde(into = "String", try_from = "String")]
|
|
pub(crate) enum StreamingCacheBudget {
|
|
Experts(u32),
|
|
Gib(u64),
|
|
}
|
|
|
|
impl fmt::Display for StreamingCacheBudget {
|
|
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
|
match self {
|
|
Self::Experts(experts) => write!(formatter, "{experts}"),
|
|
Self::Gib(gib) => write!(formatter, "{gib}GB"),
|
|
}
|
|
}
|
|
}
|
|
|
|
impl From<StreamingCacheBudget> for String {
|
|
fn from(budget: StreamingCacheBudget) -> Self {
|
|
budget.to_string()
|
|
}
|
|
}
|
|
|
|
impl TryFrom<String> for StreamingCacheBudget {
|
|
type Error = String;
|
|
|
|
fn try_from(value: String) -> Result<Self, Self::Error> {
|
|
value.parse()
|
|
}
|
|
}
|
|
|
|
impl std::str::FromStr for StreamingCacheBudget {
|
|
type Err = String;
|
|
|
|
fn from_str(value: &str) -> Result<Self, Self::Err> {
|
|
let value = value.trim();
|
|
let gib = value
|
|
.get(value.len().saturating_sub(2)..)
|
|
.is_some_and(|suffix| suffix.eq_ignore_ascii_case("gb"));
|
|
let digits = if gib {
|
|
&value[..value.len() - 2]
|
|
} else {
|
|
value
|
|
};
|
|
let number = (!digits.is_empty() && digits.bytes().all(|byte| byte.is_ascii_digit()))
|
|
.then(|| digits.parse::<u64>().ok())
|
|
.flatten()
|
|
.filter(|number| *number > 0);
|
|
match number {
|
|
Some(number) if gib && number <= u64::MAX / GIB => Ok(Self::Gib(number)),
|
|
Some(number) if !gib && number <= u64::from(u32::MAX) => {
|
|
Ok(Self::Experts(number as u32))
|
|
}
|
|
_ => Err(
|
|
"SSD cache budget must be a positive expert count or whole GiB value such as 64GB."
|
|
.into(),
|
|
),
|
|
}
|
|
}
|
|
}
|
|
|
|
#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
|
|
#[serde(default, deny_unknown_fields)]
|
|
pub(crate) struct SsdPreferences {
|
|
pub(crate) enabled: bool,
|
|
pub(crate) cold: bool,
|
|
pub(crate) cache: Option<StreamingCacheBudget>,
|
|
pub(crate) full_layers: Option<u32>,
|
|
pub(crate) preload_experts: Option<u32>,
|
|
}
|
|
|
|
impl SsdPreferences {
|
|
pub(crate) fn validate(&self, model: ModelChoice) -> Result<(), String> {
|
|
if self.preload_experts == Some(0) {
|
|
return Err("SSD preload experts must be a positive whole number.".into());
|
|
}
|
|
if self
|
|
.preload_experts
|
|
.is_some_and(|experts| experts > i32::MAX as u32)
|
|
{
|
|
return Err("SSD preload experts is too large.".into());
|
|
}
|
|
if self
|
|
.full_layers
|
|
.is_some_and(|layers| layers > i32::MAX as u32)
|
|
{
|
|
return Err("SSD full-layer count is too large.".into());
|
|
}
|
|
if self.full_layers.is_some_and(|layers| layers > 0) && model != ModelChoice::Glm52 {
|
|
return Err("Fully resident SSD layers are available only for GLM 5.2.".into());
|
|
}
|
|
if let Some(StreamingCacheBudget::Gib(gib)) = self.cache {
|
|
validate_gib("SSD cache budget", gib)?;
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
pub(crate) fn engine_settings(&self) -> EngineSsdSettings {
|
|
let (cache_experts, cache_bytes) = match self.cache {
|
|
None => (0, 0),
|
|
Some(StreamingCacheBudget::Experts(experts)) => (experts, 0),
|
|
Some(StreamingCacheBudget::Gib(gib)) => (0, gib * GIB),
|
|
};
|
|
EngineSsdSettings {
|
|
enabled: self.enabled,
|
|
cold: self.cold,
|
|
cache_experts,
|
|
cache_bytes,
|
|
full_layers: self.full_layers.unwrap_or(0),
|
|
full_layers_set: self.full_layers.is_some(),
|
|
preload_experts: self.preload_experts.unwrap_or(0),
|
|
}
|
|
}
|
|
}
|
|
|
|
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
|
pub(crate) struct EngineSsdSettings {
|
|
pub(crate) enabled: bool,
|
|
pub(crate) cold: bool,
|
|
pub(crate) cache_experts: u32,
|
|
pub(crate) cache_bytes: u64,
|
|
pub(crate) full_layers: u32,
|
|
pub(crate) full_layers_set: bool,
|
|
pub(crate) preload_experts: u32,
|
|
}
|
|
|
|
/// Disk KV cache management, mirroring ds4's `--kv-disk-space-mb` budget and
|
|
/// its store-side gates. Unset values keep the DS4 defaults.
|
|
#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
|
|
#[serde(default, deny_unknown_fields)]
|
|
pub(crate) struct KvCachePreferences {
|
|
pub(crate) budget_gib: Option<u64>,
|
|
pub(crate) min_tokens: Option<u32>,
|
|
pub(crate) cold_max_tokens: Option<u32>,
|
|
pub(crate) continued_interval_tokens: Option<u32>,
|
|
}
|
|
|
|
impl KvCachePreferences {
|
|
pub(crate) fn validate(&self) -> Result<(), String> {
|
|
if let Some(gib) = self.budget_gib {
|
|
validate_gib("KV cache budget", gib)?;
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
pub(crate) fn settings(&self) -> KvCacheSettings {
|
|
KvCacheSettings {
|
|
budget_bytes: self.budget_gib.unwrap_or(DEFAULT_KV_BUDGET_GIB) * GIB,
|
|
min_tokens: self.min_tokens.unwrap_or(DEFAULT_KV_MIN_TOKENS),
|
|
cold_max_tokens: self.cold_max_tokens.unwrap_or(DEFAULT_KV_COLD_MAX_TOKENS),
|
|
continued_interval_tokens: self
|
|
.continued_interval_tokens
|
|
.unwrap_or(DEFAULT_KV_CONTINUED_INTERVAL_TOKENS),
|
|
}
|
|
}
|
|
}
|
|
|
|
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
|
pub(crate) struct KvCacheSettings {
|
|
pub(crate) budget_bytes: u64,
|
|
pub(crate) min_tokens: u32,
|
|
/// Largest cold (first-prompt) checkpoint that may be stored. 0 stores none.
|
|
pub(crate) cold_max_tokens: u32,
|
|
/// Token distance between continued checkpoints. 0 stores none.
|
|
pub(crate) continued_interval_tokens: u32,
|
|
}
|
|
|
|
impl KvCacheSettings {
|
|
/// ds4's store gate: too short, too large a cold prompt, or too close to the
|
|
/// previous continued frontier all skip the store.
|
|
pub(crate) fn stores(&self, tokens: u32, cold: bool, last_store_tokens: u32) -> bool {
|
|
if tokens < self.min_tokens {
|
|
return false;
|
|
}
|
|
if cold {
|
|
return self.cold_max_tokens > 0 && tokens <= self.cold_max_tokens;
|
|
}
|
|
self.continued_interval_tokens > 0
|
|
&& tokens >= last_store_tokens.saturating_add(self.continued_interval_tokens)
|
|
}
|
|
}
|
|
|
|
#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)]
|
|
#[serde(default, deny_unknown_fields)]
|
|
pub(crate) struct SteeringPreferences {
|
|
pub(crate) file: Option<String>,
|
|
pub(crate) ffn_scale: Option<f32>,
|
|
pub(crate) attention_scale: Option<f32>,
|
|
}
|
|
|
|
impl SteeringPreferences {
|
|
pub(crate) fn validate(&self, model: ModelChoice) -> Result<(), String> {
|
|
validate_optional_float("Directional FFN scale", self.ffn_scale, -100.0, 100.0)?;
|
|
validate_optional_float(
|
|
"Directional attention scale",
|
|
self.attention_scale,
|
|
-100.0,
|
|
100.0,
|
|
)?;
|
|
if self.file.is_some() && model == ModelChoice::Glm52 {
|
|
return Err("Directional steering is not supported for GLM 5.2.".into());
|
|
}
|
|
if self.file.is_none()
|
|
&& (self.ffn_scale.is_some_and(|scale| scale != 0.0)
|
|
|| self.attention_scale.is_some_and(|scale| scale != 0.0))
|
|
{
|
|
return Err("Directional steering scales require a direction-vector file.".into());
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
pub(crate) fn engine_settings(&self) -> EngineSteeringSettings {
|
|
let no_explicit_scale = self.ffn_scale.is_none() && self.attention_scale.is_none();
|
|
EngineSteeringSettings {
|
|
file: self.file.clone(),
|
|
ffn_scale: self
|
|
.ffn_scale
|
|
.unwrap_or(if self.file.is_some() && no_explicit_scale {
|
|
1.0
|
|
} else {
|
|
0.0
|
|
}),
|
|
attention_scale: self.attention_scale.unwrap_or(0.0),
|
|
}
|
|
}
|
|
}
|
|
|
|
#[derive(Clone, Debug, PartialEq)]
|
|
pub(crate) struct EngineSteeringSettings {
|
|
pub(crate) file: Option<String>,
|
|
pub(crate) ffn_scale: f32,
|
|
pub(crate) attention_scale: f32,
|
|
}
|
|
|
|
#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
|
|
#[serde(default, deny_unknown_fields)]
|
|
pub(crate) struct DiagnosticPreferences {
|
|
pub(crate) simulated_used_memory_gib: Option<u64>,
|
|
pub(crate) expert_profile_path: Option<String>,
|
|
}
|
|
|
|
impl DiagnosticPreferences {
|
|
pub(crate) fn validate(&self) -> Result<(), String> {
|
|
if let Some(gib) = self.simulated_used_memory_gib {
|
|
validate_gib("Simulated used memory", gib)?;
|
|
}
|
|
if self
|
|
.expert_profile_path
|
|
.as_deref()
|
|
.is_some_and(|path| path.trim().is_empty())
|
|
{
|
|
return Err("Expert profile path cannot be empty.".into());
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
pub(crate) fn engine_settings(&self) -> EngineDiagnosticSettings {
|
|
EngineDiagnosticSettings {
|
|
simulated_used_memory_bytes: self.simulated_used_memory_gib.unwrap_or(0) * GIB,
|
|
expert_profile_path: self.expert_profile_path.clone(),
|
|
}
|
|
}
|
|
}
|
|
|
|
#[derive(Clone, Debug, Eq, PartialEq)]
|
|
pub(crate) struct EngineDiagnosticSettings {
|
|
pub(crate) simulated_used_memory_bytes: u64,
|
|
pub(crate) expert_profile_path: Option<String>,
|
|
}
|
|
|
|
#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)]
|
|
#[serde(default, deny_unknown_fields)]
|
|
pub(crate) struct RuntimePreferences {
|
|
pub(crate) execution: ExecutionPreferences,
|
|
pub(crate) speculative: SpeculativePreferences,
|
|
pub(crate) ssd: SsdPreferences,
|
|
pub(crate) steering: SteeringPreferences,
|
|
pub(crate) diagnostics: DiagnosticPreferences,
|
|
pub(crate) kv_cache: KvCachePreferences,
|
|
}
|
|
|
|
impl RuntimePreferences {
|
|
pub(crate) fn validate(&self, model: ModelChoice) -> Result<(), String> {
|
|
self.execution.validate(model)?;
|
|
self.speculative.validate(model)?;
|
|
self.ssd.validate(model)?;
|
|
self.steering.validate(model)?;
|
|
self.diagnostics.validate()?;
|
|
self.kv_cache.validate()?;
|
|
Ok(())
|
|
}
|
|
|
|
fn engine_settings(
|
|
&self,
|
|
model: ModelChoice,
|
|
context_tokens: i32,
|
|
models_path: &Path,
|
|
) -> Result<EngineSettings, String> {
|
|
self.validate(model)?;
|
|
Ok(EngineSettings {
|
|
model,
|
|
artifacts: model::engine_artifacts(
|
|
model,
|
|
self.speculative.legacy_mtp_enabled,
|
|
self.speculative.dspark_enabled,
|
|
models_path,
|
|
),
|
|
context_tokens,
|
|
execution: self.execution.engine_settings(),
|
|
speculative: self.speculative.engine_settings(),
|
|
ssd: self.ssd.engine_settings(),
|
|
steering: self.steering.engine_settings(),
|
|
diagnostics: self.diagnostics.engine_settings(),
|
|
})
|
|
}
|
|
}
|
|
|
|
#[derive(Clone, Debug, PartialEq)]
|
|
pub(crate) struct EngineSettings {
|
|
pub(crate) model: ModelChoice,
|
|
pub(crate) artifacts: EngineArtifacts,
|
|
pub(crate) context_tokens: i32,
|
|
pub(crate) execution: EngineExecutionSettings,
|
|
pub(crate) speculative: EngineSpeculativeSettings,
|
|
pub(crate) ssd: EngineSsdSettings,
|
|
pub(crate) steering: EngineSteeringSettings,
|
|
pub(crate) diagnostics: EngineDiagnosticSettings,
|
|
}
|
|
|
|
#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
|
|
#[serde(default, deny_unknown_fields)]
|
|
pub(crate) struct ExecutionPreferences {
|
|
pub(crate) cpu_threads: Option<u32>,
|
|
pub(crate) power_percent: Option<u8>,
|
|
pub(crate) prefill_chunk: Option<u32>,
|
|
pub(crate) quality: bool,
|
|
pub(crate) warm_weights: bool,
|
|
}
|
|
|
|
impl ExecutionPreferences {
|
|
pub(crate) fn validate(&self, model: ModelChoice) -> Result<(), String> {
|
|
if self.cpu_threads == Some(0) {
|
|
return Err("CPU helper threads must be a positive whole number.".into());
|
|
}
|
|
if self
|
|
.cpu_threads
|
|
.is_some_and(|threads| threads > i32::MAX as u32)
|
|
{
|
|
return Err("CPU helper threads is too large.".into());
|
|
}
|
|
if self.cpu_threads.is_some() {
|
|
return Err("CPU helper threads do not apply to the Rust Metal executor.".into());
|
|
}
|
|
if self
|
|
.power_percent
|
|
.is_some_and(|power| !(1..=100).contains(&power))
|
|
{
|
|
return Err("GPU power must be between 1 and 100 percent.".into());
|
|
}
|
|
if self.prefill_chunk == Some(0) {
|
|
return Err("Prefill chunk must be a positive whole number.".into());
|
|
}
|
|
if self
|
|
.prefill_chunk
|
|
.is_some_and(|chunk| chunk > i32::MAX as u32)
|
|
{
|
|
return Err("Prefill chunk is too large.".into());
|
|
}
|
|
if model == ModelChoice::Glm52 {
|
|
if self.power_percent.is_some_and(|power| power != 100) {
|
|
return Err("GLM 5.2 currently requires 100% GPU power.".into());
|
|
}
|
|
if self.prefill_chunk.is_some() {
|
|
return Err("GLM 5.2 selects its prefill chunk automatically.".into());
|
|
}
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
pub(crate) fn engine_settings(&self) -> EngineExecutionSettings {
|
|
EngineExecutionSettings {
|
|
cpu_threads: self.cpu_threads.unwrap_or(0).min(MAX_CPU_THREADS),
|
|
power_percent: self.power_percent.unwrap_or(0),
|
|
prefill_chunk: self.prefill_chunk.unwrap_or(0),
|
|
quality: self.quality,
|
|
warm_weights: self.warm_weights,
|
|
}
|
|
}
|
|
}
|
|
|
|
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
|
pub(crate) struct EngineExecutionSettings {
|
|
pub(crate) cpu_threads: u32,
|
|
pub(crate) power_percent: u8,
|
|
pub(crate) prefill_chunk: u32,
|
|
pub(crate) quality: bool,
|
|
pub(crate) warm_weights: bool,
|
|
}
|
|
|
|
#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
|
|
#[serde(rename_all = "lowercase")]
|
|
pub(crate) enum ReasoningMode {
|
|
#[serde(rename = "none")]
|
|
Direct,
|
|
#[default]
|
|
High,
|
|
Max,
|
|
}
|
|
|
|
impl fmt::Display for ReasoningMode {
|
|
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
|
formatter.write_str(match self {
|
|
Self::Direct => "Direct (no thinking)",
|
|
Self::High => "Thinking",
|
|
Self::Max => "Think Max",
|
|
})
|
|
}
|
|
}
|
|
|
|
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
|
|
#[serde(default, deny_unknown_fields)]
|
|
pub(crate) struct GenerationPreferences {
|
|
pub(crate) context_tokens: i32,
|
|
pub(crate) max_generated_tokens: i32,
|
|
pub(crate) system_prompt: String,
|
|
pub(crate) temperature: Option<f32>,
|
|
pub(crate) top_p: Option<f32>,
|
|
pub(crate) min_p: Option<f32>,
|
|
pub(crate) seed: Option<u64>,
|
|
pub(crate) reasoning_mode: ReasoningMode,
|
|
}
|
|
|
|
impl Default for GenerationPreferences {
|
|
fn default() -> Self {
|
|
Self {
|
|
context_tokens: 32_768,
|
|
max_generated_tokens: 50_000,
|
|
system_prompt: "You are a helpful assistant".into(),
|
|
temperature: None,
|
|
top_p: None,
|
|
min_p: None,
|
|
seed: None,
|
|
reasoning_mode: ReasoningMode::High,
|
|
}
|
|
}
|
|
}
|
|
|
|
impl GenerationPreferences {
|
|
pub(crate) fn validate(&self) -> Result<(), String> {
|
|
if self.context_tokens <= 0 {
|
|
return Err("Context tokens must be a positive whole number.".into());
|
|
}
|
|
if self.max_generated_tokens <= 0 {
|
|
return Err("Maximum generated tokens must be a positive whole number.".into());
|
|
}
|
|
validate_optional_float("Temperature", self.temperature, 0.0, 100.0)?;
|
|
validate_optional_float("Top-p", self.top_p, 0.0, 1.0)?;
|
|
validate_optional_float("Min-p", self.min_p, 0.0, 1.0)?;
|
|
if self.seed == Some(0) {
|
|
return Err("Seed must be a positive whole number.".into());
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
pub(crate) fn turn_settings(
|
|
&self,
|
|
model: ModelChoice,
|
|
kv_cache: KvCacheSettings,
|
|
) -> TurnSettings {
|
|
let glm = model == ModelChoice::Glm52;
|
|
TurnSettings {
|
|
kv_cache,
|
|
context_tokens: self.context_tokens,
|
|
max_generated_tokens: self.max_generated_tokens,
|
|
system_prompt: self.system_prompt.clone(),
|
|
temperature: self.temperature.unwrap_or(1.0),
|
|
top_p: self.top_p.unwrap_or(if glm { 0.95 } else { 1.0 }),
|
|
min_p: self.min_p.unwrap_or(if glm { 0.0 } else { 0.05 }),
|
|
top_k: 0,
|
|
stops: Vec::new(),
|
|
seed: self.seed,
|
|
reasoning_mode: if self.reasoning_mode == ReasoningMode::Max
|
|
&& self.context_tokens < THINK_MAX_MIN_CONTEXT
|
|
{
|
|
ReasoningMode::High
|
|
} else {
|
|
self.reasoning_mode
|
|
},
|
|
}
|
|
}
|
|
}
|
|
|
|
fn validate_optional_float(
|
|
name: &str,
|
|
value: Option<f32>,
|
|
minimum: f32,
|
|
maximum: f32,
|
|
) -> Result<(), String> {
|
|
if let Some(value) = value {
|
|
validate_float(name, value, minimum, maximum)?;
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
fn validate_float(name: &str, value: f32, minimum: f32, maximum: f32) -> Result<(), String> {
|
|
if !value.is_finite() || !(minimum..=maximum).contains(&value) {
|
|
return Err(format!("{name} must be between {minimum} and {maximum}."));
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
fn validate_gib(name: &str, gib: u64) -> Result<(), String> {
|
|
if gib == 0 || gib > u64::MAX / GIB {
|
|
return Err(format!("{name} must be a positive whole GiB value."));
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
#[derive(Clone, Debug, PartialEq)]
|
|
pub(crate) struct TurnSettings {
|
|
/// Disc cache policy for this request. Kept out of [`EngineSettings`] so
|
|
/// changing a cache knob never reloads the model.
|
|
pub(crate) kv_cache: KvCacheSettings,
|
|
pub(crate) context_tokens: i32,
|
|
pub(crate) max_generated_tokens: i32,
|
|
pub(crate) system_prompt: String,
|
|
pub(crate) temperature: f32,
|
|
pub(crate) top_p: f32,
|
|
pub(crate) min_p: f32,
|
|
pub(crate) top_k: i32,
|
|
pub(crate) stops: Vec<String>,
|
|
pub(crate) seed: Option<u64>,
|
|
pub(crate) reasoning_mode: ReasoningMode,
|
|
}
|
|
|
|
#[derive(Clone, Debug, PartialEq)]
|
|
pub(crate) struct EffectiveSettings {
|
|
pub(crate) engine: EngineSettings,
|
|
pub(crate) turn: TurnSettings,
|
|
}
|
|
|
|
pub(crate) fn effective_settings(
|
|
model: ModelChoice,
|
|
generation: &GenerationPreferences,
|
|
runtime: &RuntimePreferences,
|
|
models_path: &Path,
|
|
) -> Result<EffectiveSettings, String> {
|
|
generation.validate()?;
|
|
Ok(EffectiveSettings {
|
|
engine: runtime.engine_settings(model, generation.context_tokens, models_path)?,
|
|
turn: generation.turn_settings(model, runtime.kv_cache.settings()),
|
|
})
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
use std::collections::BTreeSet;
|
|
|
|
#[test]
|
|
fn effective_settings_preserve_unset_model_defaults() {
|
|
let defaults = GenerationPreferences::default();
|
|
let cache = KvCachePreferences::default().settings();
|
|
let deepseek = defaults.turn_settings(ModelChoice::DeepSeekV4Flash, cache);
|
|
assert_eq!(
|
|
(deepseek.temperature, deepseek.top_p, deepseek.min_p),
|
|
(1.0, 1.0, 0.05)
|
|
);
|
|
|
|
let glm = defaults.turn_settings(ModelChoice::Glm52, cache);
|
|
assert_eq!((glm.temperature, glm.top_p, glm.min_p), (1.0, 0.95, 0.0));
|
|
|
|
let explicit = GenerationPreferences {
|
|
top_p: Some(0.4),
|
|
min_p: Some(0.2),
|
|
reasoning_mode: ReasoningMode::Max,
|
|
..defaults
|
|
};
|
|
let effective = explicit.turn_settings(ModelChoice::Glm52, cache);
|
|
assert_eq!((effective.top_p, effective.min_p), (0.4, 0.2));
|
|
assert_eq!(effective.reasoning_mode, ReasoningMode::High);
|
|
}
|
|
|
|
#[test]
|
|
fn execution_settings_keep_ds4_automatic_values_and_glm_rules() {
|
|
let defaults = ExecutionPreferences::default().engine_settings();
|
|
assert_eq!(
|
|
(
|
|
defaults.cpu_threads,
|
|
defaults.power_percent,
|
|
defaults.prefill_chunk
|
|
),
|
|
(0, 0, 0)
|
|
);
|
|
|
|
let unsupported_threads = ExecutionPreferences {
|
|
cpu_threads: Some(100),
|
|
..ExecutionPreferences::default()
|
|
};
|
|
assert!(
|
|
unsupported_threads
|
|
.validate(ModelChoice::DeepSeekV4Flash)
|
|
.is_err()
|
|
);
|
|
|
|
let tuned = ExecutionPreferences {
|
|
power_percent: Some(50),
|
|
prefill_chunk: Some(4096),
|
|
..ExecutionPreferences::default()
|
|
};
|
|
assert!(tuned.validate(ModelChoice::DeepSeekV4Flash).is_ok());
|
|
assert!(tuned.validate(ModelChoice::Glm52).is_err());
|
|
}
|
|
|
|
#[test]
|
|
fn speculative_settings_match_ds4_defaults_and_dependencies() {
|
|
let defaults = SpeculativePreferences::default();
|
|
let engine = defaults.engine_settings();
|
|
assert_eq!((engine.mtp_draft_tokens, engine.mtp_margin), (1, 3.0));
|
|
assert_eq!(engine.dspark_confidence_threshold, 0.9);
|
|
assert!(!engine.dspark_confidence_threshold_set);
|
|
|
|
let tuned = SpeculativePreferences {
|
|
mtp_draft_tokens: 20,
|
|
dspark_enabled: true,
|
|
dspark_confidence_threshold: Some(0.7),
|
|
dspark_strict: true,
|
|
..defaults
|
|
};
|
|
assert!(tuned.validate(ModelChoice::DeepSeekV4Flash).is_ok());
|
|
assert!(tuned.validate(ModelChoice::Glm52).is_err());
|
|
assert_eq!(tuned.engine_settings().mtp_draft_tokens, 16);
|
|
|
|
let glm = SpeculativePreferences {
|
|
glm_mtp: true,
|
|
glm_mtp_timing: true,
|
|
..SpeculativePreferences::default()
|
|
};
|
|
assert!(glm.validate(ModelChoice::Glm52).is_ok());
|
|
assert!(glm.validate(ModelChoice::DeepSeekV4Pro).is_err());
|
|
|
|
let legacy = SpeculativePreferences {
|
|
legacy_mtp_enabled: true,
|
|
..SpeculativePreferences::default()
|
|
};
|
|
assert!(legacy.validate(ModelChoice::DeepSeekV4Flash).is_ok());
|
|
assert!(legacy.validate(ModelChoice::DeepSeekV4Pro).is_err());
|
|
assert!(
|
|
SpeculativePreferences {
|
|
dspark_enabled: true,
|
|
..legacy
|
|
}
|
|
.validate(ModelChoice::DeepSeekV4Flash)
|
|
.is_err()
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn kv_cache_gates_follow_the_ds4_store_rules() {
|
|
let defaults = KvCachePreferences::default().settings();
|
|
assert_eq!(defaults.budget_bytes, DEFAULT_KV_BUDGET_GIB * GIB);
|
|
|
|
// Too short to be worth a checkpoint at all.
|
|
assert!(!defaults.stores(511, true, 0));
|
|
assert!(defaults.stores(512, true, 0));
|
|
// Cold prompts above the cold maximum are skipped, continued ones are
|
|
// spaced by the interval regardless of length.
|
|
assert!(!defaults.stores(30_001, true, 0));
|
|
assert!(defaults.stores(30_001, false, 20_000));
|
|
assert!(!defaults.stores(29_999, false, 20_000));
|
|
|
|
let off = KvCachePreferences {
|
|
cold_max_tokens: Some(0),
|
|
continued_interval_tokens: Some(0),
|
|
..KvCachePreferences::default()
|
|
}
|
|
.settings();
|
|
assert!(!off.stores(1_000, true, 0));
|
|
assert!(!off.stores(1_000_000, false, 0));
|
|
|
|
assert!(
|
|
KvCachePreferences {
|
|
budget_gib: Some(0),
|
|
..KvCachePreferences::default()
|
|
}
|
|
.validate()
|
|
.is_err()
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn runtime_settings_preserve_automatic_values_and_cross_field_rules() {
|
|
let runtime = RuntimePreferences {
|
|
ssd: SsdPreferences {
|
|
enabled: true,
|
|
cache: Some(StreamingCacheBudget::Gib(64)),
|
|
full_layers: Some(0),
|
|
..SsdPreferences::default()
|
|
},
|
|
steering: SteeringPreferences {
|
|
file: Some("direction.gguf".into()),
|
|
..SteeringPreferences::default()
|
|
},
|
|
diagnostics: DiagnosticPreferences {
|
|
simulated_used_memory_gib: Some(8),
|
|
..DiagnosticPreferences::default()
|
|
},
|
|
..RuntimePreferences::default()
|
|
};
|
|
let effective = effective_settings(
|
|
ModelChoice::DeepSeekV4Flash,
|
|
&GenerationPreferences::default(),
|
|
&runtime,
|
|
Path::new("/models"),
|
|
)
|
|
.unwrap();
|
|
let engine = effective.engine;
|
|
assert_eq!(engine.context_tokens, 32_768);
|
|
assert!(engine.artifacts.mtp.is_none());
|
|
assert_eq!(engine.ssd.cache_bytes, 64 * GIB);
|
|
assert!(engine.ssd.full_layers_set);
|
|
assert_eq!(engine.ssd.full_layers, 0);
|
|
assert_eq!(engine.steering.ffn_scale, 1.0);
|
|
assert_eq!(engine.diagnostics.simulated_used_memory_bytes, 8 * GIB);
|
|
|
|
let combined = RuntimePreferences {
|
|
speculative: SpeculativePreferences {
|
|
dspark_enabled: true,
|
|
..SpeculativePreferences::default()
|
|
},
|
|
..runtime
|
|
};
|
|
assert!(combined.validate(ModelChoice::DeepSeekV4Flash).is_ok());
|
|
}
|
|
|
|
#[test]
|
|
fn effective_settings_build_one_engine_and_turn_configuration() {
|
|
let generation = GenerationPreferences {
|
|
context_tokens: 65_536,
|
|
temperature: Some(0.25),
|
|
..GenerationPreferences::default()
|
|
};
|
|
let runtime = RuntimePreferences {
|
|
speculative: SpeculativePreferences {
|
|
dspark_enabled: true,
|
|
..SpeculativePreferences::default()
|
|
},
|
|
..RuntimePreferences::default()
|
|
};
|
|
let effective = effective_settings(
|
|
ModelChoice::DeepSeekV4Flash,
|
|
&generation,
|
|
&runtime,
|
|
Path::new("/models"),
|
|
)
|
|
.unwrap();
|
|
|
|
assert_eq!(effective.engine.context_tokens, 65_536);
|
|
assert_eq!(
|
|
effective.engine.artifacts.model.parent(),
|
|
Some(Path::new("/models/deepseek-v4-flash"))
|
|
);
|
|
assert!(effective.engine.artifacts.mtp.is_some());
|
|
assert_eq!(effective.turn.temperature, 0.25);
|
|
}
|
|
|
|
#[test]
|
|
fn captured_ds4_cli_options_are_uniquely_classified() {
|
|
const PREFERENCES: &[&str] = &[
|
|
"--ctx",
|
|
"--dir-steering-attn",
|
|
"--dir-steering-ffn",
|
|
"--dir-steering-file",
|
|
"--dspark",
|
|
"--dspark-confidence",
|
|
"--dspark-strict",
|
|
"--expert-profile",
|
|
"--glm-mtp",
|
|
"--glm-mtp-timing",
|
|
"--kv-cache-cold-max-tokens",
|
|
"--kv-cache-continued-interval-tokens",
|
|
"--kv-cache-min-tokens",
|
|
"--kv-disk-space-mb",
|
|
"--min-p",
|
|
"--model",
|
|
"--mtp",
|
|
"--mtp-draft",
|
|
"--mtp-margin",
|
|
"--nothink",
|
|
"--power",
|
|
"--prefill-chunk",
|
|
"--quality",
|
|
"--seed",
|
|
"--simulate-used-memory",
|
|
"--ssd-streaming",
|
|
"--ssd-streaming-cache-experts",
|
|
"--ssd-streaming-cold",
|
|
"--ssd-streaming-full-layers",
|
|
"--ssd-streaming-preload-experts",
|
|
"--system",
|
|
"--temp",
|
|
"--think",
|
|
"--think-max",
|
|
"--threads",
|
|
"--tokens",
|
|
"--top-p",
|
|
"--warm-weights",
|
|
];
|
|
const NON_PREFERENCES: &[&str] = &[
|
|
"--backend",
|
|
"--coordinator",
|
|
"--cpu",
|
|
"--cuda",
|
|
"--cuda-tensor-parallel",
|
|
"--debug",
|
|
"--debug-hash",
|
|
"--decode-consistency",
|
|
"--dist-activation-bits",
|
|
"--dist-prefill-chunk",
|
|
"--dist-prefill-window",
|
|
"--dist-replay-check",
|
|
"--dump-logits",
|
|
"--dump-logprobs",
|
|
"--dump-tokens",
|
|
"--first-token-test",
|
|
"--gpu-devices",
|
|
"--gpu-vram",
|
|
"--head-test",
|
|
"--help",
|
|
"--imatrix-dataset",
|
|
"--imatrix-max-prompts",
|
|
"--imatrix-max-tokens",
|
|
"--imatrix-out",
|
|
"--inspect",
|
|
// Token-boundary trimming belongs to ds4's token-prefix lookup; our
|
|
// store keys on the rendered conversation instead.
|
|
"--kv-cache-boundary-align-tokens",
|
|
"--kv-cache-boundary-trim-tokens",
|
|
// Checkpoints already bind the model file identity, so a different
|
|
// quantization can never load.
|
|
"--kv-cache-reject-different-quant",
|
|
// The cache directory is fixed inside the application container.
|
|
"--kv-disk-dir",
|
|
"--layers",
|
|
"--listen",
|
|
"--logprobs-top-k",
|
|
"--metal",
|
|
"--metal-graph-full-test",
|
|
"--metal-graph-generate",
|
|
"--metal-graph-prompt-test",
|
|
"--metal-graph-test",
|
|
"--perplexity-file",
|
|
"--prompt",
|
|
"--prompt-file",
|
|
"--raw",
|
|
"--raw-prompt",
|
|
"--rdma-device",
|
|
"--rdma-gid-index",
|
|
"--rocm",
|
|
"--role",
|
|
"--server",
|
|
"--tensor-parallel",
|
|
"--tensor-parallel-token-prefill",
|
|
"--transport",
|
|
];
|
|
let classified = PREFERENCES
|
|
.iter()
|
|
.chain(NON_PREFERENCES)
|
|
.copied()
|
|
.collect::<BTreeSet<_>>();
|
|
assert_eq!(classified.len(), PREFERENCES.len() + NON_PREFERENCES.len());
|
|
assert!(
|
|
PREFERENCES
|
|
.iter()
|
|
.all(|option| !NON_PREFERENCES.contains(option))
|
|
);
|
|
}
|
|
}
|