Add native Qwen MTP speculation

This commit is contained in:
Georg Bauer
2026-09-03 22:38:52 +02:00
parent 714b39f7f3
commit bf82df77cb
14 changed files with 1302 additions and 78 deletions

View File

@@ -657,7 +657,7 @@ fn precision_assignments(
reject_unassigned_quantized(&core, &assignments)?;
let mtp = tensor_headers(manifest, source, Some("mtp"), complete)?;
add_inferred_sidecar(&mtp, 64, &mut assignments)?;
add_inferred_sidecar(&mtp, &mut assignments)?;
let ple = tensor_headers(manifest, source, Some("ple"), complete)?;
let ple_group = config
.pointer("/mtplx_recipe/ngram/group_size")
@@ -711,7 +711,6 @@ fn validate_unquantized(base: &str, tensors: &BTreeMap<String, Tensor>) -> Resul
fn add_inferred_sidecar(
tensors: &BTreeMap<String, Tensor>,
group_size: u64,
assignments: &mut BTreeMap<String, Quantization>,
) -> Result<(), String> {
for (name, tensor) in tensors {
@@ -732,6 +731,11 @@ fn add_inferred_sidecar(
.shape
.last()
.ok_or_else(|| format!("{base}.scales has no dimensions"))?;
let group_size = if base.contains(".switch_mlp.") {
32
} else {
64
};
let pack = groups
.checked_mul(group_size)
.and_then(|columns| columns.checked_div(packed))