hugo/DS4Server
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

Add GLM 5.3 Flash support

Browse Source
This commit is contained in:
Georg Bauer
2026-09-01 19:06:20 +02:00
parent 9a33c61ea6
commit 46d6a976a5
31 changed files with 7806 additions and 447 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/engine/gguf.rs
View File

@@ -19,6 +19,7 @@ pub(super) const Q5_K: u32 = 13;
pub(super) const Q6_K: u32 = 14;
pub(super) const IQ2_XXS: u32 = 16;
pub(super) const I32: u32 = 26;
pub(super) const BF16: u32 = 30;
pub(super) const MXFP4: u32 = 39;
#[derive(Clone, Debug)]
@@ -488,6 +489,7 @@ impl<'a> Cursor<'a> {
| "deepseek4.dspark.target_layer_ids"
| "deepseek4.dspark_target_layer_ids"
| "dspark.target_layer_ids"
| "glm5-next.layer_types"
);
if !keep {
for _ in 0..len {

37
src/engine/metal.rs
View File

@@ -3,12 +3,14 @@ mod glm;
mod gpu;
mod hotlist;
mod profile;
mod vision;
pub(super) use vision::VisionEmbedding;
use glm::GlmExecutor;
use gpu::*;
use profile::ExpertProfile;
use super::gguf::{F16, F32, Gguf, IQ2_XXS, MXFP4, Q2_K, Q4_K, Q8_0, Tensor as GgufTensor};
use super::gguf::{BF16, F16, F32, Gguf, IQ2_XXS, MXFP4, Q2_K, Q4_K, Q8_0, Tensor as GgufTensor};
use super::validation::{DsparkConfig, SupportKind, dspark_config};
use super::{Model, ModelFamily, Rng, exact_delta_sample};
use crate::model::ModelChoice;
@@ -42,7 +44,7 @@ fn environment_present(name: &CStr) -> bool {
!unsafe { getenv(name.as_ptr()) }.is_null()
}
const SOURCES: [(&str, &str); 19] = [
const SOURCES: [(&str, &str); 22] = [
("DS4_METAL_FLASH_ATTN_SOURCE", "flash_attn.metal"),
("DS4_METAL_DENSE_SOURCE", "dense.metal"),
("DS4_METAL_MOE_SOURCE", "moe.metal"),
@@ -62,6 +64,9 @@ const SOURCES: [(&str, &str); 19] = [
("DS4_METAL_NORM_SOURCE", "norm.metal"),
("DS4_METAL_BIN_SOURCE", "bin.metal"),
("DS4_METAL_SET_ROWS_SOURCE", "set_rows.metal"),
("DS4_METAL_GLM53_BF16_SOURCE", "glm53_bf16.metal"),
("DS4_METAL_GLM53_VISION_SOURCE", "glm53_vision.metal"),
("DS4_METAL_GLM53_KDA_SOURCE", "glm53_kda.metal"),
];
// The Metal boundary uses this only to decide whether diagnostic logs get ANSI
@@ -1593,7 +1598,7 @@ impl Dspark {
}
}
struct Steering {
pub(super) struct Steering {
directions: Buffer,
attention_scale: f32,
ffn_scale: f32,
@@ -1615,6 +1620,7 @@ impl Steering {
let expected = model
.shape
.layers
.saturating_sub(model.shape.nextn)
.checked_mul(model.shape.embd as u32)
.and_then(|values| values.checked_mul(4))
.ok_or("directional steering size overflow")? as usize;
@@ -2168,7 +2174,9 @@ impl SsdPlan {
let hotlist = match model.shape.model {
ModelChoice::DeepSeekV4Flash0731 => hotlist::FLASH,
ModelChoice::DeepSeekV4Pro => hotlist::PRO,
ModelChoice::Glm52 => unreachable!("GLM uses its dedicated executor"),
ModelChoice::Glm52 | ModelChoice::Glm53Flash => {
unreachable!("GLM uses its dedicated executor")
}
};
for &(layer, expert) in hotlist {
if loaded == self.preload_experts {
@@ -4435,6 +4443,7 @@ impl Executor {
quality,
ssd,
speculative,
steering,
expert_profile_path,
)
.map(Box::new)
@@ -4481,6 +4490,24 @@ impl Executor {
}
}
pub(super) fn encode_vision(&self, encoded: &[u8]) -> Result<vision::VisionEmbedding, String> {
match self {
Self::Glm(executor) => executor.encode_vision(encoded),
Self::DeepSeek(_) => Err("vision input requires GLM 5.3 Flash".into()),
}
}
pub(super) fn set_vision_overlays(
&mut self,
overlays: Vec<(u32, vision::VisionEmbedding)>,
) -> Result<(), String> {
match self {
Self::Glm(executor) => executor.set_vision_overlays(overlays),
Self::DeepSeek(_) if overlays.is_empty() => Ok(()),
Self::DeepSeek(_) => Err("vision input requires GLM 5.3 Flash".into()),
}
}
pub(super) fn eval_speculative_greedy(
&mut self,
token: i32,
@@ -7648,7 +7675,7 @@ fn compression_ratio(shape: super::Shape, layer: u32) -> u32 {
}
crate::model::ModelChoice::DeepSeekV4Flash0731
| crate::model::ModelChoice::DeepSeekV4Pro => 128,
crate::model::ModelChoice::Glm52 => 0,
crate::model::ModelChoice::Glm52 | crate::model::ModelChoice::Glm53Flash => 0,
}
}

3721
src/engine/metal/glm.rs
View File

File diff suppressed because it is too large Load Diff

279
src/engine/metal/gpu.rs
View File

@@ -5,6 +5,42 @@ pub(super) struct GpuTensor {
_private: [u8; 0],
}
#[derive(Clone, Copy, Default)]
#[repr(C)]
pub(super) struct Glm53VisionLayerWeights {
pub(super) norm1: u64,
pub(super) qkv_weight: u64,
pub(super) qkv_bias: u64,
pub(super) q_norm: u64,
pub(super) k_norm: u64,
pub(super) attn_proj_weight: u64,
pub(super) attn_proj_bias: u64,
pub(super) norm2: u64,
pub(super) gate_weight: u64,
pub(super) gate_bias: u64,
pub(super) up_weight: u64,
pub(super) up_bias: u64,
pub(super) down_weight: u64,
pub(super) down_bias: u64,
}
#[derive(Clone, Copy, Default)]
#[repr(C)]
pub(super) struct Glm53VisionWeights {
pub(super) patch_weight: u64,
pub(super) patch_bias: u64,
pub(super) post_norm: u64,
pub(super) downsample_weight: u64,
pub(super) downsample_bias: u64,
pub(super) merger_proj: u64,
pub(super) merger_norm: u64,
pub(super) merger_norm_bias: u64,
pub(super) merger_gate: u64,
pub(super) merger_up: u64,
pub(super) merger_down: u64,
pub(super) layer: [Glm53VisionLayerWeights; 24],
}
#[repr(C)]
pub(super) struct StreamExpertTable {
pub(super) model_map: *const c_void,
@@ -93,6 +129,12 @@ unsafe extern "C" {
count: u32,
) -> i32;
pub(super) fn ds4_gpu_flush_commands() -> i32;
pub(super) fn ds4_gpu_flush_encoder() -> i32;
pub(super) fn ds4_gpu_argmax_tensor(
out: *mut GpuTensor,
logits: *const GpuTensor,
vocab: u32,
) -> i32;
pub(super) fn ds4_gpu_device_is_pre_m5_apple_silicon() -> i32;
pub(super) fn ds4_gpu_device_is_m5_apple_silicon() -> i32;
#[cfg(test)]
@@ -227,6 +269,65 @@ unsafe extern "C" {
embd: u32,
hc: u32,
) -> i32;
pub(super) fn ds4_gpu_repeat_hc_rows_tensor(
out: *mut GpuTensor,
x: *const GpuTensor,
rows: u32,
embd: u32,
hc: u32,
) -> i32;
pub(super) fn ds4_gpu_glm53_embedding_bf16(
out: *mut GpuTensor,
map: *const c_void,
size: u64,
weight: u64,
tokens: *const GpuTensor,
rows: u32,
embd: u32,
vocab: u32,
) -> i32;
pub(super) fn ds4_gpu_glm53_vision_encode(
out: *mut f32,
patches: *const f32,
grid_h: u32,
grid_w: u32,
map: *const c_void,
size: u64,
weights: *const Glm53VisionWeights,
) -> i32;
pub(super) fn ds4_gpu_glm53_scatter_image_hc(
hc: *mut GpuTensor,
image: *const GpuTensor,
dst_row: u32,
image_row: u32,
rows: u32,
total_rows: u32,
embd: u32,
hc_count: u32,
) -> i32;
pub(super) fn ds4_gpu_glm53_matmul_bf16(
out: *mut GpuTensor,
map: *const c_void,
size: u64,
weight: u64,
input: u32,
output: u32,
x: *const GpuTensor,
rows: u32,
) -> i32;
pub(super) fn ds4_gpu_glm53_matmul_bf16_qkv(
q: *mut GpuTensor,
k: *mut GpuTensor,
v: *mut GpuTensor,
map: *const c_void,
size: u64,
q_weight: u64,
k_weight: u64,
v_weight: u64,
input: u32,
output: u32,
x: *const GpuTensor,
) -> i32;
pub(super) fn ds4_gpu_attention_noncausal_raw_batch_heads_tensor(
out: *mut GpuTensor,
map: *const c_void,
@@ -417,6 +518,35 @@ unsafe extern "C" {
beta_slow: f32,
cache_f16: bool,
) -> i32;
pub(super) fn ds4_gpu_glm53_indexer_pool_update_tensor(
cache: *mut GpuTensor,
tail_k: *mut GpuTensor,
tail_gate: *mut GpuTensor,
raw_k: *const GpuTensor,
gate: *const GpuTensor,
map: *const c_void,
size: u64,
norm_weight: u64,
norm_bias: u64,
ape: u64,
pos: u32,
rows: u32,
cache_cap: u32,
head_dim: u32,
pool_size: u32,
eps: f32,
cache_f16: bool,
) -> i32;
pub(super) fn ds4_gpu_glm53_expand_pool_selection_tensor(
selected: *mut GpuTensor,
pools: *const GpuTensor,
rows: u32,
pos: u32,
selected_pools: u32,
top_k: u32,
pool_size: u32,
width: u32,
) -> i32;
pub(super) fn ds4_gpu_glm_fill_selected_range_tensor(
selected: *mut GpuTensor,
count: u32,
@@ -467,6 +597,66 @@ unsafe extern "C" {
scale: f32,
cache_f16: bool,
) -> i32;
pub(super) fn ds4_gpu_glm53_indexer_scores_batch_tensor(
scores: *mut GpuTensor,
q: *const GpuTensor,
weights: *const GpuTensor,
cache: *const GpuTensor,
visible: u32,
rows: u32,
pos: u32,
pool_size: u32,
heads: u32,
head_dim: u32,
scale: f32,
cache_f16: bool,
) -> i32;
pub(super) fn ds4_gpu_glm53_kda_decode(
out: *mut GpuTensor,
conv: *mut GpuTensor,
recurrent: *mut GpuTensor,
q: *const GpuTensor,
k: *const GpuTensor,
v: *const GpuTensor,
gate: *const GpuTensor,
beta: *const GpuTensor,
output_gate: *const GpuTensor,
map: *const c_void,
size: u64,
q_conv: u64,
k_conv: u64,
v_conv: u64,
a_log: u64,
dt_bias: u64,
output_norm: u64,
heads: u32,
rows: u32,
gate_lower_bound: f32,
eps: f32,
) -> i32;
pub(super) fn ds4_gpu_glm53_kda_prefill(
out: *mut GpuTensor,
conv: *mut GpuTensor,
recurrent: *mut GpuTensor,
q: *mut GpuTensor,
k: *mut GpuTensor,
v: *mut GpuTensor,
gate: *mut GpuTensor,
beta: *const GpuTensor,
output_gate: *const GpuTensor,
map: *const c_void,
size: u64,
q_conv: u64,
k_conv: u64,
v_conv: u64,
a_log: u64,
dt_bias: u64,
output_norm: u64,
heads: u32,
rows: u32,
gate_lower_bound: f32,
eps: f32,
) -> i32;
pub(super) fn ds4_gpu_glm_qk_lowrank_typed_tensor(
out: *mut GpuTensor,
q: *const GpuTensor,
@@ -554,6 +744,19 @@ unsafe extern "C" {
beta_fast: f32,
beta_slow: f32,
) -> i32;
pub(super) fn ds4_gpu_glm_attention_dense_compact_lora_causal_tensor(
out: *mut GpuTensor,
qk_low: *const GpuTensor,
kv_cache: *const GpuTensor,
q_row0: u32,
rows: u32,
selected: u32,
cache_cap: u32,
cache_f16: bool,
heads: u32,
kv_lora: u32,
q_nope: u32,
) -> i32;
pub(super) fn ds4_gpu_glm_attention_indexed_batch_lora_valid_tensor(
out: *mut GpuTensor,
q: *const GpuTensor,
@@ -777,6 +980,20 @@ unsafe extern "C" {
eps: f32,
norm_eps: f32,
) -> i32;
pub(super) fn ds4_gpu_hc_split_weighted_sum_tensor(
out: *mut GpuTensor,
split: *mut GpuTensor,
mix: *const GpuTensor,
residual: *const GpuTensor,
map: *const c_void,
size: u64,
scale: u64,
base: u64,
embd: u32,
hc: u32,
iterations: u32,
eps: f32,
) -> i32;
pub(super) fn ds4_gpu_hc_rms_norm_mix_f16_available() -> i32;
pub(super) fn ds4_gpu_hc_rms_norm_mix_f16_tensor(
out: *mut GpuTensor,
@@ -1359,6 +1576,15 @@ unsafe extern "C" {
embd: u32,
hc: u32,
) -> i32;
pub(super) fn ds4_gpu_hc_expand_tensor(
out: *mut GpuTensor,
block: *const GpuTensor,
residual: *const GpuTensor,
post: *const GpuTensor,
combine: *const GpuTensor,
embd: u32,
hc: u32,
) -> i32;
pub(super) fn ds4_gpu_hc_expand_add_split_tensor(
out: *mut GpuTensor,
block: *const GpuTensor,
@@ -1463,6 +1689,44 @@ unsafe extern "C" {
x: *const GpuTensor,
clamp: f32,
) -> i32;
pub(super) fn ds4_gpu_shared_mid_swiglu_q8_0_tensor(
mid: *mut GpuTensor,
map: *const c_void,
size: u64,
gate_weight: u64,
up_weight: u64,
input: u64,
output: u64,
x: *const GpuTensor,
clamp: f32,
) -> i32;
pub(super) fn ds4_gpu_shared_gate_up_swiglu_q8_0_model_view_tensor(
gate: *mut GpuTensor,
up: *mut GpuTensor,
mid: *mut GpuTensor,
map: *const c_void,
size: u64,
gate_weight: u64,
up_weight: u64,
input: u64,
output: u64,
x: *const GpuTensor,
clamp: f32,
) -> i32;
pub(super) fn ds4_gpu_shared_gate_up_swiglu_q8_0_rows_tensor(
gate: *mut GpuTensor,
up: *mut GpuTensor,
mid: *mut GpuTensor,
map: *const c_void,
size: u64,
gate_weight: u64,
up_weight: u64,
input: u64,
output: u64,
x: *const GpuTensor,
rows: u64,
clamp: f32,
) -> i32;
pub(super) fn ds4_gpu_router_shared_gate_up_q8_0_tensor(
router_logits: *mut GpuTensor,
gate: *mut GpuTensor,
@@ -1598,6 +1862,21 @@ impl Context {
return Err(error);
}
}
if let Some(vision) = &model.vision {
let mapped = unsafe {
ds4_gpu_set_model_map_range(
vision.map_ptr().cast(),
vision.len(),
vision.data_offset(),
vision.len() - vision.data_offset(),
vision.max_tensor_bytes(),
)
};
if let Err(error) = check(mapped, "vision-model mapping") {
unsafe { ds4_gpu_cleanup() };
return Err(error);
}
}
unsafe { ds4_gpu_set_quality(quality) };
let model_file = File::open(model.main.path()).map_err(|error| {
unsafe { ds4_gpu_cleanup() };

311
src/engine/metal/vision.rs Normal file
View File

@@ -0,0 +1,311 @@
use super::gpu::{Glm53VisionLayerWeights, Glm53VisionWeights, ds4_gpu_glm53_vision_encode};
use super::{Gguf, call};
use image::{DynamicImage, ImageDecoder, ImageReader};
use std::io::Cursor;
const EMBEDDING: usize = 4096;
const PATCH: u32 = 14;
const MERGE: u32 = 2;
const MIN_TOKENS: u32 = 16;
const MAX_TOKENS: u32 = 8000;
pub(super) struct VisionEncoder {
weights: Glm53VisionWeights,
}
pub(in crate::engine) struct VisionEmbedding {
pub(in crate::engine) values: Vec<f32>,
pub(in crate::engine) tokens: u32,
pub(in crate::engine) width: u32,
pub(in crate::engine) height: u32,
pub(in crate::engine) content_width: u32,
pub(in crate::engine) content_height: u32,
}
impl VisionEncoder {
pub(super) fn bind(model: &Gguf) -> Result<Self, String> {
let offset = |name: &str| model.tensor(name).map(|tensor| tensor.offset);
let mut weights = Glm53VisionWeights {
patch_weight: offset("model.visual.patch_embed.proj.weight")?,
patch_bias: offset("model.visual.patch_embed.proj.bias")?,
post_norm: offset("model.visual.post_layernorm.weight")?,
downsample_weight: offset("model.visual.downsample.weight")?,
downsample_bias: offset("model.visual.downsample.bias")?,
merger_proj: offset("model.visual.merger.proj.weight")?,
merger_norm: offset("model.visual.merger.post_projection_norm.weight")?,
merger_norm_bias: offset("model.visual.merger.post_projection_norm.bias")?,
merger_gate: offset("model.visual.merger.gate_proj.weight")?,
merger_up: offset("model.visual.merger.up_proj.weight")?,
merger_down: offset("model.visual.merger.down_proj.weight")?,
..Glm53VisionWeights::default()
};
for (layer, target) in weights.layer.iter_mut().enumerate() {
let name = |suffix: &str| format!("model.visual.blocks.{layer}.{suffix}");
*target = Glm53VisionLayerWeights {
norm1: offset(&name("norm1.weight"))?,
qkv_weight: offset(&name("attn.qkv.weight"))?,
qkv_bias: offset(&name("attn.qkv.bias"))?,
q_norm: offset(&name("attn.q_norm.weight"))?,
k_norm: offset(&name("attn.k_norm.weight"))?,
attn_proj_weight: offset(&name("attn.proj.weight"))?,
attn_proj_bias: offset(&name("attn.proj.bias"))?,
norm2: offset(&name("norm2.weight"))?,
gate_weight: offset(&name("mlp.gate_proj.weight"))?,
gate_bias: offset(&name("mlp.gate_proj.bias"))?,
up_weight: offset(&name("mlp.up_proj.weight"))?,
up_bias: offset(&name("mlp.up_proj.bias"))?,
down_weight: offset(&name("mlp.down_proj.weight"))?,
down_bias: offset(&name("mlp.down_proj.bias"))?,
};
}
Ok(Self { weights })
}
pub(super) fn encode(&self, model: &Gguf, encoded: &[u8]) -> Result<VisionEmbedding, String> {
if encoded.is_empty() || encoded.len() > 64 * 1024 * 1024 {
return Err("image is empty or exceeds the 64 MiB encoded limit".into());
}
let reader = ImageReader::new(Cursor::new(encoded))
.with_guessed_format()
.map_err(|error| error.to_string())?;
let mut decoder = reader.into_decoder().map_err(|error| error.to_string())?;
let orientation = decoder.orientation().map_err(|error| error.to_string())?;
let mut image = DynamicImage::from_decoder(decoder).map_err(|error| error.to_string())?;
image.apply_orientation(orientation);
let rgb = image.into_rgb8();
let (width, height) = rgb.dimensions();
if width == 0
|| height == 0
|| width > 16_384
|| height > 16_384
|| u64::from(width) * u64::from(height) > 64 * 1024 * 1024
{
return Err("image dimensions exceed the GLM 5.3 vision limits".into());
}
let patches = preprocess(rgb.as_raw(), width, height)?;
let tokens = patches.grid_height * patches.grid_width / 4;
let mut values = vec![0.0_f32; tokens as usize * EMBEDDING];
call(
unsafe {
ds4_gpu_glm53_vision_encode(
values.as_mut_ptr(),
patches.values.as_ptr(),
patches.grid_height,
patches.grid_width,
model.map_ptr().cast(),
model.len(),
&self.weights,
)
},
"encoding a GLM 5.3 image",
)?;
Ok(VisionEmbedding {
values,
tokens,
width,
height,
content_width: patches.content_width,
content_height: patches.content_height,
})
}
}
struct Patches {
values: Vec<f32>,
content_width: u32,
content_height: u32,
grid_width: u32,
grid_height: u32,
}
fn preprocess(rgb: &[u8], width: u32, height: u32) -> Result<Patches, String> {
const MEAN: [f32; 3] = [0.48145466, 0.4578275, 0.40821073];
const STDDEV: [f32; 3] = [0.26862954, 0.261_302_6, 0.275_777_1];
let (target_height, target_width) = smart_resize(height, width)?;
let mut scale = (target_height as f64 / height as f64).min(target_width as f64 / width as f64);
if 2 * u64::from(height) * u64::from(width) >= 2 * 28 * 28 * MIN_TOKENS as u64 && scale > 1.0 {
scale = 1.0;
}
let content_height = ((height as f64 * scale).floor() as u32).clamp(1, target_height);
let content_width = ((width as f64 * scale).floor() as u32).clamp(1, target_width);
let mut canvas = vec![0.0_f32; target_height as usize * target_width as usize * 3];
if content_width == width && content_height == height {
for y in 0..height {
for x in 0..width {
let source = (y as usize * width as usize + x as usize) * 3;
let target = (y as usize * target_width as usize + x as usize) * 3;
for channel in 0..3 {
canvas[target + channel] = f32::from(rgb[source + channel]);
}
}
}
} else {
resize_bicubic(
rgb,
width,
height,
&mut canvas,
content_width,
content_height,
target_width,
);
}
for y in 0..target_height {
for x in 0..target_width {
let pixel = (y as usize * target_width as usize + x as usize) * 3;
for channel in 0..3 {
let value = if x < content_width && y < content_height {
canvas[pixel + channel]
} else {
0.0
};
canvas[pixel + channel] = (value / 255.0 - MEAN[channel]) / STDDEV[channel];
}
}
}
let grid_height = target_height / PATCH;
let grid_width = target_width / PATCH;
let patch_values = grid_height as usize * grid_width as usize * 3 * 2 * 14 * 14;
let mut values = Vec::with_capacity(patch_values);
for block_y in 0..grid_height / MERGE {
for block_x in 0..grid_width / MERGE {
for merge_y in 0..MERGE {
for merge_x in 0..MERGE {
let patch_y = block_y * MERGE + merge_y;
let patch_x = block_x * MERGE + merge_x;
for channel in 0..3 {
for _ in 0..2 {
for y in 0..PATCH {
for x in 0..PATCH {
let pixel = ((patch_y * PATCH + y) as usize
* target_width as usize
+ (patch_x * PATCH + x) as usize)
* 3;
values.push(canvas[pixel + channel]);
}
}
}
}
}
}
}
}
if values.len() != patch_values {
return Err("internal GLM 5.3 vision patch layout mismatch".into());
}
Ok(Patches {
values,
content_width,
content_height,
grid_width,
grid_height,
})
}
fn smart_resize(height: u32, width: u32) -> Result<(u32, u32), String> {
let factor = 28_u32;
let align = |value: u32| value.div_ceil(factor) * factor;
let pixels_per_token = 2_u64 * factor as u64 * factor as u64;
let min_pixels = MIN_TOKENS as u64 * pixels_per_token;
let max_pixels = MAX_TOKENS as u64 * pixels_per_token;
let mut aligned_height = align(height);
let mut aligned_width = align(width);
let mut budget = 2_u64 * aligned_height as u64 * aligned_width as u64;
if budget < min_pixels {
let scale = (min_pixels as f64 / (2.0 * height as f64 * width as f64)).sqrt();
aligned_height = align((height as f64 * scale).ceil() as u32);
aligned_width = align((width as f64 * scale).ceil() as u32);
budget = 2_u64 * aligned_height as u64 * aligned_width as u64;
}
if budget > max_pixels {
let (mut low, mut high) = (1_u32, height);
aligned_height = factor;
aligned_width = factor;
while low <= high {
let content_height = low + (high - low) / 2;
let content_width =
((width as f64 * content_height as f64 / height as f64).floor() as u32).max(1);
let candidate_height = align(content_height);
let candidate_width = align(content_width);
if 2_u64 * candidate_height as u64 * candidate_width as u64 <= max_pixels {
aligned_height = candidate_height;
aligned_width = candidate_width;
low = content_height + 1;
} else {
high = content_height - 1;
}
}
}
Ok((aligned_height, aligned_width))
}
#[allow(clippy::too_many_arguments)]
fn resize_bicubic(
source: &[u8],
source_width: u32,
source_height: u32,
target: &mut [f32],
target_width: u32,
target_height: u32,
target_stride: u32,
) {
let scale_x = source_width as f64 / target_width as f64;
let scale_y = source_height as f64 / target_height as f64;
let filter_x = if scale_x >= 1.0 { 1.0 / scale_x } else { 1.0 };
let filter_y = if scale_y >= 1.0 { 1.0 / scale_y } else { 1.0 };
let support_x = if scale_x >= 1.0 { 2.0 * scale_x } else { 2.0 };
let support_y = if scale_y >= 1.0 { 2.0 * scale_y } else { 2.0 };
for dy in 0..target_height {
let center_y = scale_y * (dy as f64 + 0.5);
let y0 = (center_y - support_y + 0.5).max(0.0) as u32;
let y1 = (center_y + support_y + 0.5).min(source_height as f64) as u32;
for dx in 0..target_width {
let center_x = scale_x * (dx as f64 + 0.5);
let x0 = (center_x - support_x + 0.5).max(0.0) as u32;
let x1 = (center_x + support_x + 0.5).min(source_width as f64) as u32;
let mut sum = [0.0; 3];
let mut weight_sum = 0.0;
for iy in y0..y1 {
let wy = cubic((iy as f64 + 0.5 - center_y) * filter_y);
for ix in x0..x1 {
let weight = wy * cubic((ix as f64 + 0.5 - center_x) * filter_x);
let pixel = (iy as usize * source_width as usize + ix as usize) * 3;
for channel in 0..3 {
sum[channel] += source[pixel + channel] as f64 * weight;
}
weight_sum += weight;
}
}
let pixel = (dy as usize * target_stride as usize + dx as usize) * 3;
for channel in 0..3 {
target[pixel + channel] =
(sum[channel] / weight_sum).round().clamp(0.0, 255.0) as f32;
}
}
}
}
fn cubic(mut x: f64) -> f64 {
const A: f64 = -0.5;
x = x.abs();
if x < 1.0 {
((A + 2.0) * x - (A + 3.0)) * x * x + 1.0
} else if x < 2.0 {
((A * x - 5.0 * A) * x + 8.0 * A) * x - 4.0 * A
} else {
0.0
}
}
#[cfg(test)]
mod tests {
use super::smart_resize;
#[test]
fn glm53_resize_matches_reference_token_grids() {
assert_eq!(smart_resize(28, 28), Ok((112, 112)));
assert_eq!(smart_resize(1024, 1024), Ok((1036, 1036)));
assert_eq!(smart_resize(1080, 1920), Ok((1092, 1932)));
}
}

21
src/engine/tokenizer.rs
View File

@@ -1,5 +1,8 @@
use super::gguf::Gguf;
use super::{ChatTurn, ModelFamily};
use super::{
ChatTurn, ModelFamily, VISION_END_TOKEN, VISION_IMAGE_TOKEN, VISION_START_TOKEN,
VISION_TOKEN_END, VISION_TOKEN_START,
};
use crate::settings::ReasoningMode;
use std::collections::HashMap;
@@ -160,6 +163,22 @@ impl Tokenizer {
let mut span = 0;
let mut position = 0;
while position < bytes.len() {
if bytes[position..].starts_with(VISION_TOKEN_START.as_bytes())
&& let Some(relative_end) =
text[position + VISION_TOKEN_START.len()..].find(VISION_TOKEN_END)
{
let count_start = position + VISION_TOKEN_START.len();
let count_end = count_start + relative_end;
if let Ok(count) = text[count_start..count_end].parse::<usize>() {
self.tokenize_plain(&text[span..position], &mut output);
output.push(VISION_START_TOKEN);
output.extend(std::iter::repeat_n(VISION_IMAGE_TOKEN, count));
output.push(VISION_END_TOKEN);
position = count_end + VISION_TOKEN_END.len();
span = position;
continue;
}
}
let special = self
.rendered_specials
.iter()

196
src/engine/validation.rs
View File

@@ -9,7 +9,7 @@ pub(crate) fn validate_model_artifact(
let model = Gguf::open(path)?;
let shape = match expected {
ModelChoice::DeepSeekV4Flash0731 => FLASH_0731,
ModelChoice::DeepSeekV4Pro | ModelChoice::Glm52 => {
ModelChoice::DeepSeekV4Pro | ModelChoice::Glm52 | ModelChoice::Glm53Flash => {
return Err(format!("{expected} does not use an external support GGUF"));
}
};
@@ -35,6 +35,79 @@ pub(crate) fn validate_model_artifact(
}
}
pub(crate) fn validate_vision_artifact(path: &Path) -> Result<(), String> {
let model = Gguf::open(path)?;
if model.bytes("general.architecture")? != b"glm5-next-vision" {
return Err("vision GGUF architecture is not glm5-next-vision".into());
}
if model.tensors.len() != 347 {
return Err(format!(
"vision GGUF has {} tensors, expected 347",
model.tensors.len()
));
}
for (key, expected) in [
("block_count", 24),
("embedding_length", 1024),
("feed_forward_length", 4096),
("attention.head_count", 16),
("projection_length", 4096),
("projection.feed_forward_length", 10_240),
("patch_size", 14),
("temporal_patch_size", 2),
("spatial_merge_size", 2),
("image_token_id", VISION_IMAGE_TOKEN as u64),
("image_start_token_id", VISION_START_TOKEN as u64),
("image_end_token_id", VISION_END_TOKEN as u64),
] {
expect_u64(&model, &format!("glm5-next-vision.{key}"), expected)?;
}
let bf16 = &[BF16];
for (name, dims) in [
(
"model.visual.patch_embed.proj.weight",
vec![14, 14, 2, 3, 1024],
),
("model.visual.patch_embed.proj.bias", vec![1024]),
("model.visual.post_layernorm.weight", vec![1024]),
("model.visual.downsample.weight", vec![2, 2, 1024, 4096]),
("model.visual.downsample.bias", vec![4096]),
("model.visual.merger.proj.weight", vec![4096, 4096]),
(
"model.visual.merger.post_projection_norm.weight",
vec![4096],
),
("model.visual.merger.post_projection_norm.bias", vec![4096]),
("model.visual.merger.gate_proj.weight", vec![4096, 10_240]),
("model.visual.merger.up_proj.weight", vec![4096, 10_240]),
("model.visual.merger.down_proj.weight", vec![10_240, 4096]),
] {
expect(&model, name, bf16, &dims)?;
}
for layer in 0..24 {
let name = |suffix: &str| format!("model.visual.blocks.{layer}.{suffix}");
for (suffix, dims) in [
("norm1.weight", vec![1024]),
("attn.qkv.weight", vec![1024, 3072]),
("attn.qkv.bias", vec![3072]),
("attn.q_norm.weight", vec![64]),
("attn.k_norm.weight", vec![64]),
("attn.proj.weight", vec![1024, 1024]),
("attn.proj.bias", vec![1024]),
("norm2.weight", vec![1024]),
("mlp.gate_proj.weight", vec![1024, 4096]),
("mlp.gate_proj.bias", vec![4096]),
("mlp.up_proj.weight", vec![1024, 4096]),
("mlp.up_proj.bias", vec![4096]),
("mlp.down_proj.weight", vec![4096, 1024]),
("mlp.down_proj.bias", vec![1024]),
] {
expect(&model, &name(suffix), bf16, &dims)?;
}
}
Ok(())
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub(super) enum SupportKind {
DSpark,
@@ -116,14 +189,11 @@ pub(super) fn validate_support(model: &Gguf, shape: &Shape) -> Result<SupportKin
}
pub(super) fn validate_main(model: &Gguf, expected: ModelChoice) -> Result<Shape, String> {
let family = if model.bytes("general.architecture").ok() == Some(b"glm-dsa") {
ModelFamily::Glm
} else {
ModelFamily::DeepSeek
};
let shape = match family {
ModelFamily::Glm => GLM,
ModelFamily::DeepSeek => match (model.u32("deepseek4.block_count")?, expected) {
let architecture = model.bytes("general.architecture")?;
let shape = match architecture {
b"glm-dsa" => GLM,
b"glm5-next" => GLM53_FLASH,
_ => match (model.u32("deepseek4.block_count")?, expected) {
(43, ModelChoice::DeepSeekV4Flash0731) => FLASH_0731,
(43, _) => FLASH_0731,
(61, _) => PRO,
@@ -143,6 +213,9 @@ pub(super) fn validate_main(model: &Gguf, expected: ModelChoice) -> Result<Shape
}
fn validate_metadata(model: &Gguf, shape: &Shape) -> Result<(), String> {
if shape.model == ModelChoice::Glm53Flash {
return validate_glm53_metadata(model, shape);
}
let prefix = if shape.family == ModelFamily::Glm {
"glm-dsa"
} else {
@@ -266,13 +339,102 @@ fn validate_metadata(model: &Gguf, shape: &Shape) -> Result<(), String> {
Ok(())
}
fn validate_glm53_metadata(model: &Gguf, shape: &Shape) -> Result<(), String> {
let prefix = "glm5-next";
for (key, expected) in [
("block_count", u64::from(shape.layers)),
("trunk_block_count", u64::from(shape.layers - shape.nextn)),
("nextn_predict_layers", u64::from(shape.nextn)),
("context_length", shape.original_context),
("embedding_length", shape.embd),
("vocab_size", shape.vocab),
("feed_forward_length", shape.ff_dense),
("expert_feed_forward_length", shape.ff_expert),
("expert_count", shape.experts),
("expert_used_count", shape.experts_used),
("expert_shared_count", shape.expert_shared),
("leading_dense_block_count", u64::from(shape.leading_dense)),
("attention.head_count", shape.heads),
("attention.key_length", shape.key_mla),
("attention.value_length", shape.value_mla),
("attention.q_lora_rank", shape.lora_q),
("attention.kv_lora_rank", shape.kv_lora),
("attention.rope_dimension_count", shape.rot),
("attention.indexer.head_count", shape.indexer_heads),
("attention.indexer.key_length", shape.indexer_head_dim),
("attention.indexer.top_k", shape.indexer_top_k),
("attention.indexer.pool_size", 4),
("linear_attention.head_count", 64),
("linear_attention.head_dimension", 128),
("linear_attention.conv_kernel", 4),
("hyper_connection.count", shape.hc),
("hyper_connection.sinkhorn_iterations", shape.hc_sinkhorn),
] {
expect_u64(model, &format!("{prefix}.{key}"), expected)?;
}
for (key, expected) in [
("expert_weights_scale", shape.expert_weight_scale),
("swiglu_limit", shape.swiglu_clamp),
("attention.layer_norm_rms_epsilon", shape.rms_epsilon),
("linear_attention.gate_lower_bound", -5.0),
("hyper_connection.epsilon", shape.hc_epsilon),
] {
expect_float(model, &format!("{prefix}.{key}"), expected)?;
}
if !model.boolean("glm5-next.expert_weights_norm")? {
return Err("glm5-next.expert_weights_norm must be true".into());
}
let layer_types = model.u32s("glm5-next.layer_types")?;
if layer_types.len() != shape.layers as usize {
return Err("glm5-next.layer_types must contain one entry per layer".into());
}
for (layer, &kind) in layer_types.iter().enumerate() {
let expected =
u32::from(layer + shape.nextn as usize >= shape.layers as usize || layer % 4 == 3);
if kind != expected {
return Err(format!(
"unexpected GLM 5.3 attention type at layer {layer}"
));
}
}
Ok(())
}
fn validate_tensors(model: &Gguf, shape: &Shape) -> Result<(), String> {
match shape.family {
ModelFamily::DeepSeek => validate_deepseek_tensors(model, shape),
ModelFamily::Glm if shape.model == ModelChoice::Glm53Flash => {
validate_glm53_tensors(model, shape)
}
ModelFamily::Glm => validate_glm_tensors(model, shape),
}
}
fn validate_glm53_tensors(model: &Gguf, shape: &Shape) -> Result<(), String> {
expect(
model,
"token_embd.weight",
DENSE,
&[shape.embd, shape.vocab],
)?;
expect(model, "output_norm.weight", &[F32], &[shape.embd])?;
expect(model, "output.weight", DENSE, &[shape.embd, shape.vocab])?;
expect(model, "blk.0.kda_q.weight", DENSE, &[shape.embd, 8192])?;
expect(
model,
"blk.3.attn_q_a.weight",
DENSE,
&[shape.embd, shape.lora_q],
)?;
expect(
model,
"blk.45.nextn.eh_proj.weight",
DENSE,
&[2 * shape.embd, shape.embd],
)?;
Ok(())
}
fn validate_deepseek_tensors(model: &Gguf, shape: &Shape) -> Result<(), String> {
let hc_dim = shape.embd * shape.hc;
let hc_mix = 2 * shape.hc + shape.hc * shape.hc;
@@ -902,7 +1064,7 @@ fn compression_ratio(shape: &Shape, layer: u32) -> u32 {
4
}
ModelChoice::DeepSeekV4Flash0731 | ModelChoice::DeepSeekV4Pro => 128,
ModelChoice::Glm52 => 0,
ModelChoice::Glm52 | ModelChoice::Glm53Flash => 0,
}
}
@@ -1157,4 +1319,18 @@ mod tests {
validate_model_artifact(&path, ModelChoice::DeepSeekV4Flash0731, true).unwrap();
}
}
#[test]
fn configured_glm53_vision_fixture_passes_the_exact_layout() {
if let Some(path) = std::env::var_os("DS4SERVER_GLM53_VISION") {
validate_vision_artifact(Path::new(&path)).unwrap();
}
}
#[test]
fn configured_glm53_main_fixture_passes_the_exact_layout() {
if let Some(path) = std::env::var_os("DS4SERVER_GLM53_MODEL") {
validate_model_artifact(Path::new(&path), ModelChoice::Glm53Flash, false).unwrap();
}
}
}