Add native Qwen MTP speculation
This commit is contained in:
@@ -657,7 +657,7 @@ fn precision_assignments(
|
||||
reject_unassigned_quantized(&core, &assignments)?;
|
||||
|
||||
let mtp = tensor_headers(manifest, source, Some("mtp"), complete)?;
|
||||
add_inferred_sidecar(&mtp, 64, &mut assignments)?;
|
||||
add_inferred_sidecar(&mtp, &mut assignments)?;
|
||||
let ple = tensor_headers(manifest, source, Some("ple"), complete)?;
|
||||
let ple_group = config
|
||||
.pointer("/mtplx_recipe/ngram/group_size")
|
||||
@@ -711,7 +711,6 @@ fn validate_unquantized(base: &str, tensors: &BTreeMap<String, Tensor>) -> Resul
|
||||
|
||||
fn add_inferred_sidecar(
|
||||
tensors: &BTreeMap<String, Tensor>,
|
||||
group_size: u64,
|
||||
assignments: &mut BTreeMap<String, Quantization>,
|
||||
) -> Result<(), String> {
|
||||
for (name, tensor) in tensors {
|
||||
@@ -732,6 +731,11 @@ fn add_inferred_sidecar(
|
||||
.shape
|
||||
.last()
|
||||
.ok_or_else(|| format!("{base}.scales has no dimensions"))?;
|
||||
let group_size = if base.contains(".switch_mlp.") {
|
||||
32
|
||||
} else {
|
||||
64
|
||||
};
|
||||
let pack = groups
|
||||
.checked_mul(group_size)
|
||||
.and_then(|columns| columns.checked_div(packed))
|
||||
|
||||
Reference in New Issue
Block a user