fix: ssd streaming and deepseek fixes
This commit is contained in:
@@ -2725,6 +2725,7 @@ fn glm_streaming_plan(
|
||||
let max_layer = weights
|
||||
.layers
|
||||
.iter()
|
||||
.chain(weights.nextn.iter())
|
||||
.filter_map(|layer| layer.sparse)
|
||||
.map(|weights| {
|
||||
sparse_expert_bytes(weights, shape.experts)
|
||||
@@ -3256,7 +3257,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
#[ignore = "requires the 197 GiB GLM 5.2 checkpoint and Apple Metal"]
|
||||
fn resident_and_streamed_glm_match_the_short_code_fixture() {
|
||||
fn resident_and_streamed_glm_match_ds4_decode_oracles() {
|
||||
let path = std::env::var("DS4_GLM_MODEL").unwrap_or_else(|_| {
|
||||
"../ds4/models/GLM-5.2-UD-IQ2_XXS_RoutedIQ2XXS_blk78Q2K.gguf".into()
|
||||
});
|
||||
@@ -3290,6 +3291,9 @@ mod tests {
|
||||
}
|
||||
Err(error) => panic!("GLM executor failed to open: {error}"),
|
||||
};
|
||||
if streamed {
|
||||
assert_eq!(executor.ssd_cache_experts, 671);
|
||||
}
|
||||
assert_eq!(executor.prefill(&tokens, |_| true).unwrap(), tokens.len());
|
||||
let token = executor
|
||||
.logits()
|
||||
@@ -3302,6 +3306,23 @@ mod tests {
|
||||
executor.model().token_bytes(token).as_deref(),
|
||||
Some(b");\n".as_slice())
|
||||
);
|
||||
let greeting = executor.model().render_prompt(
|
||||
"You are a helpful assistant",
|
||||
"Write one short greeting.",
|
||||
ReasoningMode::Direct,
|
||||
);
|
||||
executor.reset().unwrap();
|
||||
assert_eq!(
|
||||
executor.prefill(&greeting, |_| true).unwrap(),
|
||||
greeting.len()
|
||||
);
|
||||
let mut generated = Vec::new();
|
||||
for _ in 0..4 {
|
||||
let token = argmax(executor.logits());
|
||||
generated.push(token);
|
||||
executor.eval(token).unwrap();
|
||||
}
|
||||
assert_eq!(generated, [9703, 0, 2585, 646]);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -74,6 +74,13 @@ unsafe extern "C" {
|
||||
) -> i32;
|
||||
pub(super) fn ds4_gpu_stream_expert_cache_note_service_thread();
|
||||
pub(super) fn ds4_gpu_stream_expert_cache_reset_route_hotness();
|
||||
pub(super) fn ds4_gpu_stream_prefill_batch_selected_addr_enabled(
|
||||
n_tokens: u32,
|
||||
n_total_expert: u32,
|
||||
n_expert: u32,
|
||||
gate_type: u32,
|
||||
down_type: u32,
|
||||
) -> i32;
|
||||
pub(super) fn ds4_gpu_glm_stream_expert_cache_begin_selected_load_tensor(
|
||||
table: *const StreamExpertTable,
|
||||
selected: *const GpuTensor,
|
||||
|
||||
Reference in New Issue
Block a user