hugo/DS4Server
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

Load GLM vision weights on demand

Browse Source
This commit is contained in:
Georg Bauer
2026-09-01 20:08:36 +02:00
parent 46d6a976a5
commit ee515ee824
10 changed files with 293 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
src/app.rs
View File

@@ -449,6 +449,7 @@ pub(crate) enum Message {
PreferenceWarmWeightsChanged(bool),
PreferenceGlmMtpChanged(bool),
PreferenceGlmMtpTimingChanged(bool),
PreferenceKeepVisionLoadedChanged(bool),
PreferenceDsparkConfidenceChanged(String),
PreferenceDsparkStrictChanged(bool),
PreferenceDsparkExactSamplingChanged(bool),

14
src/app/preferences.rs
View File

@@ -40,6 +40,7 @@ pub(super) struct PreferenceDraft {
pub(super) warm_weights: bool,
pub(super) glm_mtp: bool,
pub(super) glm_mtp_timing: bool,
pub(super) keep_vision_loaded: bool,
pub(super) dspark_confidence_threshold: String,
pub(super) dspark_strict: bool,
pub(super) dspark_exact_sampling: bool,
@@ -104,6 +105,7 @@ impl PreferenceDraft {
warm_weights: execution.warm_weights,
glm_mtp: speculative.glm_mtp,
glm_mtp_timing: speculative.glm_mtp_timing,
keep_vision_loaded: speculative.keep_vision_loaded,
dspark_confidence_threshold: optional_string(speculative.dspark_confidence_threshold),
dspark_strict: speculative.dspark_strict,
dspark_exact_sampling: speculative.dspark_exact_sampling,
@@ -213,6 +215,7 @@ impl PreferenceDraft {
Ok(SpeculativePreferences {
glm_mtp: self.glm_mtp,
glm_mtp_timing: self.glm_mtp_timing,
keep_vision_loaded: self.keep_vision_loaded,
dspark_enabled: self.dspark_enabled,
dspark_confidence_threshold: parse_optional_f32(
"DSpark confidence",
@@ -256,6 +259,7 @@ impl PreferenceDraft {
self.dspark_enabled = speculative.dspark_enabled;
self.glm_mtp = speculative.glm_mtp;
self.glm_mtp_timing = speculative.glm_mtp_timing;
self.keep_vision_loaded = speculative.keep_vision_loaded;
self.dspark_confidence_threshold = optional_string(speculative.dspark_confidence_threshold);
self.dspark_strict = speculative.dspark_strict;
self.dspark_exact_sampling = speculative.dspark_exact_sampling;
@@ -870,6 +874,11 @@ impl App {
}
self.preference_error = None;
}
Message::PreferenceKeepVisionLoadedChanged(value) => {
self.preference_draft.keep_vision_loaded =
self.preference_draft.acceleration_model == ModelChoice::Glm53Flash && value;
self.preference_error = None;
}
Message::PreferenceDsparkConfidenceChanged(value) => {
self.preference_draft.dspark_confidence_threshold = value;
if self.preference_draft.acceleration_model.supports_dspark()
@@ -1013,10 +1022,15 @@ mod tests {
draft.select_acceleration(ModelChoice::Glm52).unwrap();
assert!(!draft.ssd_streaming);
draft.select_acceleration(ModelChoice::Glm53Flash).unwrap();
draft.keep_vision_loaded = true;
draft
.select_acceleration(ModelChoice::DeepSeekV4Flash0731)
.unwrap();
assert!(draft.ssd_streaming);
assert!(!draft.keep_vision_loaded);
draft.select_acceleration(ModelChoice::Glm53Flash).unwrap();
assert!(draft.keep_vision_loaded);
}
#[test]

9
src/app/view/preferences.rs
View File

@@ -24,6 +24,9 @@ impl App {
.acceleration_model
.supports_glm_mtp()
.then_some(Message::PreferenceGlmMtpTimingChanged);
let keep_vision_loaded_toggle: Option<fn(bool) -> Message> =
(self.preference_draft.acceleration_model == ModelChoice::Glm53Flash)
.then_some(Message::PreferenceKeepVisionLoadedChanged);
let dspark_strict_toggle: Option<fn(bool) -> Message> = self
.preference_draft
.acceleration_model
@@ -615,6 +618,12 @@ impl App {
.on_toggle_maybe(glm_mtp_timing_toggle),
"Records per-stage timings of the speculative path to the log, to show where the acceleration actually goes. A diagnostic aid that costs a little throughput.",
),
hint(
toggle(self.preference_draft.keep_vision_loaded)
.label("Keep GLM 5.3 vision weights loaded")
.on_toggle_maybe(keep_vision_loaded_toggle),
"Keeps the vision encoder mapped between image turns for lower image latency. Off releases it after encoding all images in a turn, leaving more memory for long text contexts.",
),
dspark,
preference_input_row(
"DSpark confidence threshold",

33
src/engine.rs
View File

@@ -312,7 +312,7 @@ impl Model {
if let Some(path) = &settings.artifacts.vision {
validate_vision_artifact(path)?;
let vision = Gguf::open(path)?;
if settings.execution.warm_weights {
if settings.execution.warm_weights && settings.speculative.keep_vision_loaded {
vision.warm()?;
}
model.vision = Some(vision);
@@ -1166,20 +1166,18 @@ impl Generator {
reasoning: ReasoningMode,
) -> Result<(Vec<i32>, VisionOverlays), String> {
let mut rendered = messages.to_vec();
let mut embeddings = Vec::new();
let mut images = Vec::new();
let mut total_images = 0_usize;
let mut total_bytes = 0_usize;
for message in &mut rendered {
for message in &rendered {
if !message.content.contains(VISION_DATA_START) {
continue;
}
if !message.user && !message.tool {
return Err("vision input is allowed only in user or tool messages".into());
}
let mut content = String::with_capacity(message.content.len());
let mut rest = message.content.as_str();
while let Some(start) = rest.find(VISION_DATA_START) {
content.push_str(&rest[..start]);
let encoded = &rest[start + VISION_DATA_START.len()..];
let end = encoded
.find(VISION_DATA_END)
@@ -1202,7 +1200,27 @@ impl Generator {
if total_bytes > 64 * 1024 * 1024 {
return Err("image inputs exceed the 64 MiB request limit".into());
}
let embedding = self.executor.encode_vision(&bytes)?;
images.push(bytes);
rest = &encoded[end + VISION_DATA_END.len()..];
}
}
let mut encoded_images = self.executor.encode_visions(&images)?.into_iter();
let mut embeddings = Vec::with_capacity(images.len());
for message in &mut rendered {
if !message.content.contains(VISION_DATA_START) {
continue;
}
let mut content = String::with_capacity(message.content.len());
let mut rest = message.content.as_str();
while let Some(start) = rest.find(VISION_DATA_START) {
content.push_str(&rest[..start]);
let encoded = &rest[start + VISION_DATA_START.len()..];
let end = encoded
.find(VISION_DATA_END)
.ok_or("unterminated image input")?;
let embedding = encoded_images
.next()
.ok_or("vision encoder returned too few embeddings")?;
content.push_str(VISION_TOKEN_START);
content.push_str(&embedding.tokens.to_string());
content.push_str(VISION_TOKEN_END);
@@ -1212,6 +1230,9 @@ impl Generator {
content.push_str(rest);
message.content = content;
}
if encoded_images.next().is_some() {
return Err("vision encoder returned too many embeddings".into());
}
let tokens = self
.executor
.model()

19
src/engine/metal.rs
View File

@@ -3056,7 +3056,7 @@ impl DeepSeekExecutor {
} else {
resident_deepseek_admission_bytes(&model, context, prefill_chunk)?
};
let context_handle = Context::open(&model, quality, ssd.enabled, admission, spans)?;
let context_handle = Context::open(&model, quality, ssd.enabled, admission, spans, false)?;
let mxfp4_decode_fast_lookup =
mxfp4_decode_fast_lookup_allowed(&model, &weights, quality, ssd.enabled);
let steering = Steering::load(&model, steering)?;
@@ -4387,6 +4387,7 @@ impl Executor {
EngineSpeculativeSettings {
glm_mtp: false,
glm_mtp_timing: false,
keep_vision_loaded: false,
dspark: false,
dspark_confidence_threshold: 0.9,
dspark_confidence_threshold_set: false,
@@ -4490,9 +4491,12 @@ impl Executor {
}
}
pub(super) fn encode_vision(&self, encoded: &[u8]) -> Result<vision::VisionEmbedding, String> {
pub(super) fn encode_visions(
&self,
encoded: &[Vec<u8>],
) -> Result<Vec<vision::VisionEmbedding>, String> {
match self {
Self::Glm(executor) => executor.encode_vision(encoded),
Self::Glm(executor) => executor.encode_visions(encoded),
Self::DeepSeek(_) => Err("vision input requires GLM 5.3 Flash".into()),
}
}
@@ -8526,6 +8530,7 @@ mod tests {
EngineSpeculativeSettings {
glm_mtp: false,
glm_mtp_timing: false,
keep_vision_loaded: false,
dspark,
dspark_confidence_threshold: confidence.unwrap_or(0.8),
dspark_confidence_threshold_set: confidence.is_some(),
@@ -8714,6 +8719,7 @@ mod tests {
EngineSpeculativeSettings {
glm_mtp: false,
glm_mtp_timing: false,
keep_vision_loaded: false,
dspark: false,
dspark_confidence_threshold: 0.9,
dspark_confidence_threshold_set: false,
@@ -8799,6 +8805,7 @@ mod tests {
EngineSpeculativeSettings {
glm_mtp: false,
glm_mtp_timing: false,
keep_vision_loaded: false,
dspark: true,
dspark_confidence_threshold: 0.9,
dspark_confidence_threshold_set: false,
@@ -8945,6 +8952,7 @@ mod tests {
EngineSpeculativeSettings {
glm_mtp: false,
glm_mtp_timing: false,
keep_vision_loaded: false,
dspark: true,
dspark_confidence_threshold: 0.6,
dspark_confidence_threshold_set: false,
@@ -9057,6 +9065,7 @@ mod tests {
let speculative = EngineSpeculativeSettings {
glm_mtp: false,
glm_mtp_timing: false,
keep_vision_loaded: false,
dspark: false,
dspark_confidence_threshold: 0.9,
dspark_confidence_threshold_set: false,
@@ -9149,6 +9158,7 @@ mod tests {
EngineSpeculativeSettings {
glm_mtp: false,
glm_mtp_timing: false,
keep_vision_loaded: false,
dspark: false,
dspark_confidence_threshold: 0.9,
dspark_confidence_threshold_set: false,
@@ -9242,6 +9252,7 @@ mod tests {
EngineSpeculativeSettings {
glm_mtp: false,
glm_mtp_timing: false,
keep_vision_loaded: false,
dspark: false,
dspark_confidence_threshold: 0.9,
dspark_confidence_threshold_set: false,
@@ -9325,6 +9336,7 @@ mod tests {
EngineSpeculativeSettings {
glm_mtp: false,
glm_mtp_timing: false,
keep_vision_loaded: false,
dspark: false,
dspark_confidence_threshold: 0.9,
dspark_confidence_threshold_set: false,
@@ -9399,6 +9411,7 @@ mod tests {
EngineSpeculativeSettings {
glm_mtp: false,
glm_mtp_timing: false,
keep_vision_loaded: false,
dspark: false,
dspark_confidence_threshold: 0.9,
dspark_confidence_threshold_set: false,

85
src/engine/metal/glm.rs
View File

@@ -650,6 +650,7 @@ pub(in crate::engine) struct GlmExecutor {
model_identity: [u8; 32],
streaming_spans: Option<Vec<(u64, u64)>>,
vision: Option<VisionEncoder>,
keep_vision_loaded: bool,
vision_overlays: Vec<VisionOverlay>,
_context: Context,
model: Model,
@@ -680,6 +681,7 @@ impl GlmExecutor {
EngineSpeculativeSettings {
glm_mtp: false,
glm_mtp_timing: false,
keep_vision_loaded: false,
dspark: false,
dspark_confidence_threshold: 0.9,
dspark_confidence_threshold_set: false,
@@ -713,7 +715,13 @@ impl GlmExecutor {
let weights = GlmWeights::bind(&model)?;
let streaming = glm_streaming_plan(&model, &weights, ssd)?;
let effective_ssd = streaming.map_or(ssd, |plan| plan.settings);
let admission = admission_bytes(&model, &weights, context, streaming.as_ref())?;
let admission = admission_bytes(
&model,
&weights,
context,
streaming.as_ref(),
speculative.keep_vision_loaded,
)?;
let model_spans = streaming
.as_ref()
.map(|plan| glm_streaming_model_spans(&model, &weights, plan))
@@ -727,6 +735,7 @@ impl GlmExecutor {
effective_ssd.enabled,
admission,
context_spans,
speculative.keep_vision_loaded,
)?;
let model_spans = model_spans.map(|(spans, _)| spans);
configure_streaming(&model, &weights, streaming.as_ref())?;
@@ -826,13 +835,17 @@ impl GlmExecutor {
model_identity,
streaming_spans: model_spans,
vision,
keep_vision_loaded: speculative.keep_vision_loaded,
vision_overlays: Vec::new(),
_context: context_handle,
model,
})
}
pub(super) fn encode_vision(&self, encoded: &[u8]) -> Result<VisionEmbedding, String> {
pub(super) fn encode_visions(
&self,
encoded: &[Vec<u8>],
) -> Result<Vec<VisionEmbedding>, String> {
let encoder = self
.vision
.as_ref()
@@ -842,7 +855,38 @@ impl GlmExecutor {
.vision
.as_ref()
.ok_or("GLM 5.3 vision GGUF is not loaded")?;
encoder.encode(vision, encoded)
if !self.keep_vision_loaded {
call(
unsafe {
ds4_gpu_set_transient_model_map_range(
vision.map_ptr().cast(),
vision.len(),
vision.data_offset(),
vision.len() - vision.data_offset(),
vision.max_tensor_bytes(),
)
},
"mapping GLM 5.3 vision weights",
)?;
}
let result = encoded
.iter()
.map(|image| encoder.encode(vision, image))
.collect::<Result<Vec<_>, _>>();
let released = if self.keep_vision_loaded {
Ok(())
} else {
call(
unsafe {
ds4_gpu_release_transient_model_map(vision.map_ptr().cast(), vision.len())
},
"releasing GLM 5.3 vision weights",
)
};
match (result, released) {
(Ok(embeddings), Ok(())) => Ok(embeddings),
(Err(error), _) | (Ok(_), Err(error)) => Err(error),
}
}
pub(super) fn set_vision_overlays(
@@ -5659,6 +5703,7 @@ fn admission_bytes(
weights: &GlmWeights,
context: u32,
streaming: Option<&GlmStreamingPlan>,
keep_vision_loaded: bool,
) -> Result<u64, String> {
let shape = model.shape;
let normal_layers = u64::from(shape.layers - shape.nextn);
@@ -5712,12 +5757,14 @@ fn admission_bytes(
} else {
model.main.len() - model.main.data_offset()
};
let resident = resident.saturating_add(
let resident = resident.saturating_add(if keep_vision_loaded {
model
.vision
.as_ref()
.map_or(0, |vision| vision.len() - vision.data_offset()),
);
.map_or(0, |vision| vision.len() - vision.data_offset())
} else {
0
});
let cache = streaming.map_or(0, |plan| plan.planned_expert_bytes);
let scratch = glm_batch_scratch_bytes(shape, context.min(INDEXED_PREFILL_CHUNK), context)?
.max(512 * 1024 * 1024);
@@ -6303,6 +6350,7 @@ mod tests {
EngineSpeculativeSettings {
glm_mtp: mtp,
glm_mtp_timing: std::env::var_os("DS4SERVER_GLM53_MTP_TIMING").is_some(),
keep_vision_loaded: false,
dspark: false,
dspark_confidence_threshold: 0.9,
dspark_confidence_threshold_set: false,
@@ -6409,6 +6457,7 @@ mod tests {
let mut model = Model::open_main(&model_path, ModelChoice::Glm53Flash).unwrap();
let eos = model.eos_token();
model.vision = Some(crate::engine::gguf::Gguf::open(&vision_path).unwrap());
let keep_vision_loaded = std::env::var_os("DS4SERVER_GLM53_KEEP_VISION_LOADED").is_some();
let mut executor = GlmExecutor::open_profile(
model,
4_096,
@@ -6425,6 +6474,7 @@ mod tests {
EngineSpeculativeSettings {
glm_mtp: false,
glm_mtp_timing: false,
keep_vision_loaded,
dspark: false,
dspark_confidence_threshold: 0.9,
dspark_confidence_threshold_set: false,
@@ -6439,12 +6489,32 @@ mod tests {
None,
)
.unwrap();
let vision = executor.model().vision.as_ref().unwrap();
assert_eq!(
unsafe {
super::super::gpu::ds4_gpu_model_map_active(vision.map_ptr().cast(), vision.len())
!= 0
},
keep_vision_loaded,
);
let encoded = std::fs::read(image_path).unwrap();
let mut runs = Vec::new();
let mut embedding = None;
for _ in 0..3 {
let started = Instant::now();
embedding = Some(executor.encode_vision(&encoded).unwrap());
embedding = executor
.encode_visions(std::slice::from_ref(&encoded))
.unwrap()
.pop();
assert_eq!(
unsafe {
super::super::gpu::ds4_gpu_model_map_active(
vision.map_ptr().cast(),
vision.len(),
) != 0
},
keep_vision_loaded,
);
runs.push(started.elapsed().as_secs_f64());
}
runs.sort_by(f64::total_cmp);
@@ -6766,6 +6836,7 @@ mod tests {
EngineSpeculativeSettings {
glm_mtp: enabled,
glm_mtp_timing: false,
keep_vision_loaded: false,
dspark: false,
dspark_confidence_threshold: 0.9,
dspark_confidence_threshold_set: false,

16
src/engine/metal/gpu.rs
View File

@@ -81,6 +81,19 @@ unsafe extern "C" {
map_size: u64,
max_tensor_bytes: u64,
) -> i32;
pub(super) fn ds4_gpu_set_transient_model_map_range(
model_map: *const c_void,
model_size: u64,
map_offset: u64,
map_size: u64,
max_tensor_bytes: u64,
) -> i32;
pub(super) fn ds4_gpu_release_transient_model_map(
model_map: *const c_void,
model_size: u64,
) -> i32;
#[cfg(test)]
pub(super) fn ds4_gpu_model_map_active(model_map: *const c_void, model_size: u64) -> i32;
pub(super) fn ds4_gpu_set_model_map_spans(
model_map: *const c_void,
model_size: u64,
@@ -1801,6 +1814,7 @@ impl Context {
ssd_streaming: bool,
admission_bytes: u64,
model_spans: Option<(&[(u64, u64)], u64)>,
map_vision: bool,
) -> Result<Self, String> {
check(unsafe { ds4_gpu_init() }, "Metal initialization")?;
unsafe {
@@ -1862,7 +1876,7 @@ impl Context {
return Err(error);
}
}
if let Some(vision) = &model.vision {
if map_vision && let Some(vision) = &model.vision {
let mapped = unsafe {
ds4_gpu_set_model_map_range(
vision.map_ptr().cast(),

14
src/settings.rs
View File

@@ -23,6 +23,7 @@ const DEFAULT_KV_CONTINUED_INTERVAL_TOKENS: u32 = 10_000;
pub(crate) struct SpeculativePreferences {
pub(crate) glm_mtp: bool,
pub(crate) glm_mtp_timing: bool,
pub(crate) keep_vision_loaded: bool,
pub(crate) dspark_enabled: bool,
pub(crate) dspark_confidence_threshold: Option<f32>,
pub(crate) dspark_strict: bool,
@@ -37,6 +38,9 @@ impl SpeculativePreferences {
if !model.supports_glm_mtp() && (self.glm_mtp || self.glm_mtp_timing) {
return Err("GLM MTP is available only for GLM models.".into());
}
if self.keep_vision_loaded && model != ModelChoice::Glm53Flash {
return Err("Persistent vision weights are available only for GLM 5.3 Flash.".into());
}
if self.dspark_enabled && !model.supports_dspark() {
return Err("DSpark is not available for the selected model.".into());
}
@@ -57,6 +61,7 @@ impl SpeculativePreferences {
EngineSpeculativeSettings {
glm_mtp: self.glm_mtp,
glm_mtp_timing: self.glm_mtp_timing,
keep_vision_loaded: self.keep_vision_loaded,
dspark: self.dspark_enabled,
dspark_confidence_threshold: self.dspark_confidence_threshold.unwrap_or(0.8),
dspark_confidence_threshold_set: self.dspark_confidence_threshold.is_some(),
@@ -70,6 +75,7 @@ impl SpeculativePreferences {
pub(crate) struct EngineSpeculativeSettings {
pub(crate) glm_mtp: bool,
pub(crate) glm_mtp_timing: bool,
pub(crate) keep_vision_loaded: bool,
pub(crate) dspark: bool,
pub(crate) dspark_confidence_threshold: f32,
pub(crate) dspark_confidence_threshold_set: bool,
@@ -772,6 +778,14 @@ mod tests {
assert!(glm.validate(ModelChoice::Glm52).is_ok());
assert!(glm.validate(ModelChoice::DeepSeekV4Pro).is_err());
let persistent_vision = SpeculativePreferences {
keep_vision_loaded: true,
..SpeculativePreferences::default()
};
assert!(persistent_vision.validate(ModelChoice::Glm53Flash).is_ok());
assert!(persistent_vision.validate(ModelChoice::Glm52).is_err());
assert!(persistent_vision.engine_settings().keep_vision_loaded);
assert!(
SpeculativePreferences {
dspark_exact_sampling: true,