Integrate DS4 execution parity in Rust
This commit is contained in:
@@ -23,6 +23,7 @@ const DEFAULT_KV_CONTINUED_INTERVAL_TOKENS: u32 = 10_000;
|
||||
pub(crate) struct SpeculativePreferences {
|
||||
pub(crate) mtp_draft_tokens: i32,
|
||||
pub(crate) mtp_margin: f32,
|
||||
pub(crate) legacy_mtp_enabled: bool,
|
||||
pub(crate) glm_mtp: bool,
|
||||
pub(crate) glm_mtp_timing: bool,
|
||||
pub(crate) dspark_enabled: bool,
|
||||
@@ -35,6 +36,7 @@ impl Default for SpeculativePreferences {
|
||||
Self {
|
||||
mtp_draft_tokens: 1,
|
||||
mtp_margin: 3.0,
|
||||
legacy_mtp_enabled: false,
|
||||
glm_mtp: false,
|
||||
glm_mtp_timing: false,
|
||||
dspark_enabled: false,
|
||||
@@ -59,6 +61,12 @@ impl SpeculativePreferences {
|
||||
if self.dspark_enabled && !model.supports_dspark() {
|
||||
return Err("DSpark is not available for the selected model.".into());
|
||||
}
|
||||
if self.legacy_mtp_enabled && !model.supports_dspark() {
|
||||
return Err("Legacy MTP is not available for the selected model.".into());
|
||||
}
|
||||
if self.legacy_mtp_enabled && self.dspark_enabled {
|
||||
return Err("Legacy MTP and DSpark use different support artifacts.".into());
|
||||
}
|
||||
if (self.dspark_confidence_threshold.is_some() || self.dspark_strict)
|
||||
&& !self.dspark_enabled
|
||||
{
|
||||
@@ -343,6 +351,9 @@ impl DiagnosticPreferences {
|
||||
if let Some(gib) = self.simulated_used_memory_gib {
|
||||
validate_gib("Simulated used memory", gib)?;
|
||||
}
|
||||
if self.expert_profile_path.is_some() {
|
||||
return Err("Expert profiling is not available in the Rust Metal executor yet.".into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -379,9 +390,6 @@ impl RuntimePreferences {
|
||||
self.steering.validate(model)?;
|
||||
self.diagnostics.validate()?;
|
||||
self.kv_cache.validate()?;
|
||||
if self.ssd.enabled && self.speculative.dspark_enabled {
|
||||
return Err("SSD streaming is not compatible with DSpark support.".into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -394,7 +402,12 @@ impl RuntimePreferences {
|
||||
self.validate(model)?;
|
||||
Ok(EngineSettings {
|
||||
model,
|
||||
artifacts: model::engine_artifacts(model, self.speculative.dspark_enabled, models_path),
|
||||
artifacts: model::engine_artifacts(
|
||||
model,
|
||||
self.speculative.legacy_mtp_enabled,
|
||||
self.speculative.dspark_enabled,
|
||||
models_path,
|
||||
),
|
||||
context_tokens,
|
||||
execution: self.execution.engine_settings(),
|
||||
speculative: self.speculative.engine_settings(),
|
||||
@@ -438,6 +451,9 @@ impl ExecutionPreferences {
|
||||
{
|
||||
return Err("CPU helper threads is too large.".into());
|
||||
}
|
||||
if self.cpu_threads.is_some() {
|
||||
return Err("CPU helper threads do not apply to the Rust Metal executor.".into());
|
||||
}
|
||||
if self
|
||||
.power_percent
|
||||
.is_some_and(|power| !(1..=100).contains(&power))
|
||||
@@ -680,15 +696,23 @@ mod tests {
|
||||
(0, 0, 0)
|
||||
);
|
||||
|
||||
let tuned = ExecutionPreferences {
|
||||
let unsupported_threads = ExecutionPreferences {
|
||||
cpu_threads: Some(100),
|
||||
..ExecutionPreferences::default()
|
||||
};
|
||||
assert!(
|
||||
unsupported_threads
|
||||
.validate(ModelChoice::DeepSeekV4Flash)
|
||||
.is_err()
|
||||
);
|
||||
|
||||
let tuned = ExecutionPreferences {
|
||||
power_percent: Some(50),
|
||||
prefill_chunk: Some(4096),
|
||||
..ExecutionPreferences::default()
|
||||
};
|
||||
assert!(tuned.validate(ModelChoice::DeepSeekV4Flash).is_ok());
|
||||
assert!(tuned.validate(ModelChoice::Glm52).is_err());
|
||||
assert_eq!(tuned.engine_settings().cpu_threads, MAX_CPU_THREADS);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -717,6 +741,21 @@ mod tests {
|
||||
};
|
||||
assert!(glm.validate(ModelChoice::Glm52).is_ok());
|
||||
assert!(glm.validate(ModelChoice::DeepSeekV4Pro).is_err());
|
||||
|
||||
let legacy = SpeculativePreferences {
|
||||
legacy_mtp_enabled: true,
|
||||
..SpeculativePreferences::default()
|
||||
};
|
||||
assert!(legacy.validate(ModelChoice::DeepSeekV4Flash).is_ok());
|
||||
assert!(legacy.validate(ModelChoice::DeepSeekV4Pro).is_err());
|
||||
assert!(
|
||||
SpeculativePreferences {
|
||||
dspark_enabled: true,
|
||||
..legacy
|
||||
}
|
||||
.validate(ModelChoice::DeepSeekV4Flash)
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -787,14 +826,14 @@ mod tests {
|
||||
assert_eq!(engine.steering.ffn_scale, 1.0);
|
||||
assert_eq!(engine.diagnostics.simulated_used_memory_bytes, 8 * GIB);
|
||||
|
||||
let incompatible = RuntimePreferences {
|
||||
let combined = RuntimePreferences {
|
||||
speculative: SpeculativePreferences {
|
||||
dspark_enabled: true,
|
||||
..SpeculativePreferences::default()
|
||||
},
|
||||
..runtime
|
||||
};
|
||||
assert!(incompatible.validate(ModelChoice::DeepSeekV4Flash).is_err());
|
||||
assert!(combined.validate(ModelChoice::DeepSeekV4Flash).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
Reference in New Issue
Block a user