fix: ssd streaming and deepseek fixes

This commit is contained in:
Georg Bauer
2026-07-26 21:50:59 +02:00
parent 0d80c217c4
commit 47b405e72b
6 changed files with 457 additions and 84 deletions

View File

@@ -2725,6 +2725,7 @@ fn glm_streaming_plan(
let max_layer = weights
.layers
.iter()
.chain(weights.nextn.iter())
.filter_map(|layer| layer.sparse)
.map(|weights| {
sparse_expert_bytes(weights, shape.experts)
@@ -3256,7 +3257,7 @@ mod tests {
#[test]
#[ignore = "requires the 197 GiB GLM 5.2 checkpoint and Apple Metal"]
fn resident_and_streamed_glm_match_the_short_code_fixture() {
fn resident_and_streamed_glm_match_ds4_decode_oracles() {
let path = std::env::var("DS4_GLM_MODEL").unwrap_or_else(|_| {
"../ds4/models/GLM-5.2-UD-IQ2_XXS_RoutedIQ2XXS_blk78Q2K.gguf".into()
});
@@ -3290,6 +3291,9 @@ mod tests {
}
Err(error) => panic!("GLM executor failed to open: {error}"),
};
if streamed {
assert_eq!(executor.ssd_cache_experts, 671);
}
assert_eq!(executor.prefill(&tokens, |_| true).unwrap(), tokens.len());
let token = executor
.logits()
@@ -3302,6 +3306,23 @@ mod tests {
executor.model().token_bytes(token).as_deref(),
Some(b");\n".as_slice())
);
let greeting = executor.model().render_prompt(
"You are a helpful assistant",
"Write one short greeting.",
ReasoningMode::Direct,
);
executor.reset().unwrap();
assert_eq!(
executor.prefill(&greeting, |_| true).unwrap(),
greeting.len()
);
let mut generated = Vec::new();
for _ in 0..4 {
let token = argmax(executor.logits());
generated.push(token);
executor.eval(token).unwrap();
}
assert_eq!(generated, [9703, 0, 2585, 646]);
}
}