Add end-to-end MXFP4 Metal support

This commit is contained in:
Georg Bauer
2026-08-30 20:42:23 +02:00
parent 3977261bd3
commit 28231e1faf
7 changed files with 514 additions and 124 deletions

View File

@@ -434,13 +434,17 @@ impl GlmExecutor {
.as_ref()
.map(|plan| glm_streaming_model_spans(&model, &weights, plan))
.transpose()?;
let context_spans = model_spans
.as_ref()
.map(|(spans, max_tensor_bytes)| (spans.as_slice(), *max_tensor_bytes));
let context_handle = Context::open(
&model,
quality,
effective_ssd.enabled,
admission,
model_spans.as_deref(),
context_spans,
)?;
let model_spans = model_spans.map(|(spans, _)| spans);
configure_streaming(&model, &weights, streaming.as_ref())?;
let scratch = GlmScratch::allocate(&model, context)?;
let caches = (0..weights.layers.len())
@@ -2851,7 +2855,7 @@ fn glm_streaming_model_spans(
model: &Model,
weights: &GlmWeights,
plan: &GlmStreamingPlan,
) -> Result<Vec<(u64, u64)>, String> {
) -> Result<(Vec<(u64, u64)>, u64), String> {
let full_before = model
.shape
.leading_dense
@@ -2890,6 +2894,11 @@ fn glm_streaming_model_spans(
})
.map(|(_, tensor)| (tensor.offset, tensor.bytes))
.collect::<Vec<_>>();
let max_tensor_bytes = spans
.iter()
.map(|(_, bytes)| *bytes)
.max()
.ok_or("GLM SSD streaming found no resident model tensors")?;
spans.sort_unstable_by_key(|span| span.0);
let mut merged: Vec<(u64, u64)> = Vec::new();
for (offset, bytes) in spans {
@@ -2903,10 +2912,7 @@ fn glm_streaming_model_spans(
}
merged.push((offset, bytes));
}
if merged.is_empty() {
return Err("GLM SSD streaming found no resident model tensors".into());
}
Ok(merged)
Ok((merged, max_tensor_bytes))
}
fn glm_layer_model_spans(model: &Model, layer: u32) -> Result<Vec<(u64, u64)>, String> {