Add end-to-end MXFP4 Metal support
This commit is contained in:
@@ -434,13 +434,17 @@ impl GlmExecutor {
|
||||
.as_ref()
|
||||
.map(|plan| glm_streaming_model_spans(&model, &weights, plan))
|
||||
.transpose()?;
|
||||
let context_spans = model_spans
|
||||
.as_ref()
|
||||
.map(|(spans, max_tensor_bytes)| (spans.as_slice(), *max_tensor_bytes));
|
||||
let context_handle = Context::open(
|
||||
&model,
|
||||
quality,
|
||||
effective_ssd.enabled,
|
||||
admission,
|
||||
model_spans.as_deref(),
|
||||
context_spans,
|
||||
)?;
|
||||
let model_spans = model_spans.map(|(spans, _)| spans);
|
||||
configure_streaming(&model, &weights, streaming.as_ref())?;
|
||||
let scratch = GlmScratch::allocate(&model, context)?;
|
||||
let caches = (0..weights.layers.len())
|
||||
@@ -2851,7 +2855,7 @@ fn glm_streaming_model_spans(
|
||||
model: &Model,
|
||||
weights: &GlmWeights,
|
||||
plan: &GlmStreamingPlan,
|
||||
) -> Result<Vec<(u64, u64)>, String> {
|
||||
) -> Result<(Vec<(u64, u64)>, u64), String> {
|
||||
let full_before = model
|
||||
.shape
|
||||
.leading_dense
|
||||
@@ -2890,6 +2894,11 @@ fn glm_streaming_model_spans(
|
||||
})
|
||||
.map(|(_, tensor)| (tensor.offset, tensor.bytes))
|
||||
.collect::<Vec<_>>();
|
||||
let max_tensor_bytes = spans
|
||||
.iter()
|
||||
.map(|(_, bytes)| *bytes)
|
||||
.max()
|
||||
.ok_or("GLM SSD streaming found no resident model tensors")?;
|
||||
spans.sort_unstable_by_key(|span| span.0);
|
||||
let mut merged: Vec<(u64, u64)> = Vec::new();
|
||||
for (offset, bytes) in spans {
|
||||
@@ -2903,10 +2912,7 @@ fn glm_streaming_model_spans(
|
||||
}
|
||||
merged.push((offset, bytes));
|
||||
}
|
||||
if merged.is_empty() {
|
||||
return Err("GLM SSD streaming found no resident model tensors".into());
|
||||
}
|
||||
Ok(merged)
|
||||
Ok((merged, max_tensor_bytes))
|
||||
}
|
||||
|
||||
fn glm_layer_model_spans(model: &Model, layer: u32) -> Result<Vec<(u64, u64)>, String> {
|
||||
|
||||
Reference in New Issue
Block a user