hugo/DS4Server
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

Load GLM vision weights on demand

Browse Source
This commit is contained in:
Georg Bauer
2026-09-01 20:08:36 +02:00
parent 46d6a976a5
commit ee515ee824
10 changed files with 293 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
src/engine.rs
View File

@@ -312,7 +312,7 @@ impl Model {
if let Some(path) = &settings.artifacts.vision {
validate_vision_artifact(path)?;
let vision = Gguf::open(path)?;
if settings.execution.warm_weights {
if settings.execution.warm_weights && settings.speculative.keep_vision_loaded {
vision.warm()?;
}
model.vision = Some(vision);
@@ -1166,20 +1166,18 @@ impl Generator {
reasoning: ReasoningMode,
) -> Result<(Vec<i32>, VisionOverlays), String> {
let mut rendered = messages.to_vec();
let mut embeddings = Vec::new();
let mut images = Vec::new();
let mut total_images = 0_usize;
let mut total_bytes = 0_usize;
for message in &mut rendered {
for message in &rendered {
if !message.content.contains(VISION_DATA_START) {
continue;
}
if !message.user && !message.tool {
return Err("vision input is allowed only in user or tool messages".into());
}
let mut content = String::with_capacity(message.content.len());
let mut rest = message.content.as_str();
while let Some(start) = rest.find(VISION_DATA_START) {
content.push_str(&rest[..start]);
let encoded = &rest[start + VISION_DATA_START.len()..];
let end = encoded
.find(VISION_DATA_END)
@@ -1202,7 +1200,27 @@ impl Generator {
if total_bytes > 64 * 1024 * 1024 {
return Err("image inputs exceed the 64 MiB request limit".into());
}
let embedding = self.executor.encode_vision(&bytes)?;
images.push(bytes);
rest = &encoded[end + VISION_DATA_END.len()..];
}
}
let mut encoded_images = self.executor.encode_visions(&images)?.into_iter();
let mut embeddings = Vec::with_capacity(images.len());
for message in &mut rendered {
if !message.content.contains(VISION_DATA_START) {
continue;
}
let mut content = String::with_capacity(message.content.len());
let mut rest = message.content.as_str();
while let Some(start) = rest.find(VISION_DATA_START) {
content.push_str(&rest[..start]);
let encoded = &rest[start + VISION_DATA_START.len()..];
let end = encoded
.find(VISION_DATA_END)
.ok_or("unterminated image input")?;
let embedding = encoded_images
.next()
.ok_or("vision encoder returned too few embeddings")?;
content.push_str(VISION_TOKEN_START);
content.push_str(&embedding.tokens.to_string());
content.push_str(VISION_TOKEN_END);
@@ -1212,6 +1230,9 @@ impl Generator {
content.push_str(rest);
message.content = content;
}
if encoded_images.next().is_some() {
return Err("vision encoder returned too many embeddings".into());
}
let tokens = self
.executor
.model()