hugo/DS4Server
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

Add GLM 5.3 Flash support

Browse Source
This commit is contained in:
Georg Bauer
2026-09-01 19:06:20 +02:00
parent 9a33c61ea6
commit 46d6a976a5
31 changed files with 7806 additions and 447 deletions
Download Patch File Download Diff File Expand all files Collapse all files

223
native/metal/ds4_gpu.h
View File

@@ -118,6 +118,11 @@ int ds4_gpu_build_derived_artifacts(const void *model_map, uint64_t model_size,
int ds4_gpu_model_range_replaced(const void *model_map, uint64_t offset,
uint64_t bytes);
int ds4_gpu_set_model_map_range(const void *model_map, uint64_t model_size, uint64_t map_offset, uint64_t map_size, uint64_t max_tensor_bytes);
/* Add a secondary GGUF mapping without replacing the primary model mapping. */
int ds4_gpu_set_aux_model_map_range(const void *model_map,
uint64_t model_size,
uint64_t map_offset,
uint64_t map_size);
int ds4_gpu_set_model_map_spans(const void *model_map, uint64_t model_size, const uint64_t *offsets, const uint64_t *sizes, uint32_t count, uint64_t max_tensor_bytes);
int ds4_gpu_cache_model_range(const void *model_map, uint64_t model_size, uint64_t offset, uint64_t bytes, const char *label);
int ds4_gpu_cache_q8_f16_range(const void *model_map, uint64_t model_size, uint64_t offset, uint64_t bytes, uint64_t in_dim, uint64_t out_dim, const char *label);
@@ -668,6 +673,17 @@ int ds4_gpu_matmul_q8_0_pair_tensor(
const ds4_gpu_tensor *x,
uint64_t n_tok);
int ds4_gpu_matmul_q4_K_pair_decode_tensor(
ds4_gpu_tensor *out0,
ds4_gpu_tensor *out1,
const void *model_map,
uint64_t model_size,
uint64_t weight0_offset,
uint64_t weight1_offset,
uint64_t in_dim,
uint64_t out_dim,
const ds4_gpu_tensor *x);
/* Multi-row decode projections that preserve the one-row reduction order. */
int ds4_gpu_matmul_q8_0_decode_rows_exact_tensor(
ds4_gpu_tensor *out,
@@ -1276,6 +1292,37 @@ int ds4_gpu_glm_store_indexer_k_tensor(
float beta_slow,
bool cache_f16);
/* GLM-5.3 pools four normalized indexer keys with a learned, per-channel
* softmax. Partial pools are retained in tail_k/tail_gate across calls. */
int ds4_gpu_glm53_indexer_pool_update_tensor(
ds4_gpu_tensor *pool_cache,
ds4_gpu_tensor *tail_k,
ds4_gpu_tensor *tail_gate,
const ds4_gpu_tensor *raw_k,
const ds4_gpu_tensor *gate,
const void *model_map,
uint64_t model_size,
uint64_t norm_weight_offset,
uint64_t norm_bias_offset,
uint64_t ape_offset,
uint32_t pos0,
uint32_t n_tokens,
uint32_t cache_cap,
uint32_t head_dim,
uint32_t pool_size,
float eps,
bool cache_f16);
int ds4_gpu_glm53_expand_pool_selection_tensor(
ds4_gpu_tensor *raw_selected,
const ds4_gpu_tensor *pool_selected,
uint32_t n_tokens,
uint32_t pos0,
uint32_t selected_pools,
uint32_t index_topk,
uint32_t pool_size,
uint32_t output_width);
int ds4_gpu_glm_build_kv_cache_tensor(
ds4_gpu_tensor *key_cache,
ds4_gpu_tensor *value_cache,
@@ -1388,6 +1435,20 @@ int ds4_gpu_glm_indexer_scores_batch_tensor(
float scale,
bool cache_f16);
int ds4_gpu_glm53_indexer_scores_batch_tensor(
ds4_gpu_tensor *scores,
const ds4_gpu_tensor *q,
const ds4_gpu_tensor *weights,
const ds4_gpu_tensor *indexer_key_cache,
uint32_t n_rows,
uint32_t n_tokens,
uint32_t pos0,
uint32_t pool_size,
uint32_t n_head,
uint32_t head_dim,
float scale,
bool cache_f16);
int ds4_gpu_glm_qk_lowrank_q8_0_tensor(
ds4_gpu_tensor *qk_low,
const ds4_gpu_tensor *q,
@@ -1698,6 +1759,21 @@ int ds4_gpu_glm_attention_indexed_batch_lora_causal_tensor(
float beta_fast,
float beta_slow);
/* Dense causal MLA over the shared compact latent cache. qk_low and lora_out
* are [token, head, kv_lora_dim]; the F16 cache is shared by all heads. */
int ds4_gpu_glm_attention_dense_compact_lora_causal_tensor(
ds4_gpu_tensor *lora_out,
const ds4_gpu_tensor *qk_low,
const ds4_gpu_tensor *kv_lora_cache,
uint32_t q_row0,
uint32_t n_q,
uint32_t n_kv,
uint32_t cache_cap,
bool cache_f16,
uint32_t n_head,
uint32_t kv_lora_dim,
uint32_t qk_dim);
int ds4_gpu_glm_attention_indexed_batch_lora_valid_tensor(
ds4_gpu_tensor *lora_out,
const ds4_gpu_tensor *q,
@@ -2366,6 +2442,7 @@ int ds4_gpu_glm_routed_moe_one_tensor(
const ds4_gpu_tensor *weights,
uint32_t n_total_expert,
uint32_t n_expert,
float swiglu_clamp,
uint32_t layer_index,
const ds4_gpu_tensor *x,
bool force_resident);
@@ -2394,6 +2471,7 @@ int ds4_gpu_glm_routed_moe_batch_tensor(
const ds4_gpu_tensor *weights,
uint32_t n_total_expert,
uint32_t n_expert,
float swiglu_clamp,
uint32_t layer_index,
const ds4_gpu_tensor *x,
uint32_t n_tokens,
@@ -2424,6 +2502,7 @@ int ds4_gpu_glm_routed_moe_batch_direct_scalar_q4_tensor(
const ds4_gpu_tensor *weights,
uint32_t n_total_expert,
uint32_t n_expert,
float swiglu_clamp,
uint32_t layer_index,
const ds4_gpu_tensor *x,
uint32_t n_tokens,
@@ -2431,6 +2510,10 @@ int ds4_gpu_glm_routed_moe_batch_direct_scalar_q4_tensor(
int ds4_gpu_routed_moe_set_selected_override(const int32_t *selected, uint32_t n_selected);
void ds4_gpu_set_glm_mtp_verify_mode(bool enabled);
#ifdef DS4_ROCM_BUILD
int ds4_gpu_dspark_gfx1151_fast_path(void);
void ds4_gpu_set_dspark_verify_mode(bool enabled);
#endif
int ds4_gpu_matmul_q8_0_kslice_hc_expand_add_tensor(
ds4_gpu_tensor *out_hc,
@@ -2850,6 +2933,146 @@ int ds4_gpu_matmul_q8_0_hc_expand_tensor(
uint32_t n_embd,
uint32_t n_hc);
int ds4_gpu_glm53_embedding_bf16(
ds4_gpu_tensor *out,
const void *model_map,
uint64_t model_size,
uint64_t weight_offset,
const ds4_gpu_tensor *token_ids,
uint32_t n_tokens,
uint32_t n_embd,
uint32_t n_vocab);
int ds4_gpu_glm53_matmul_bf16(
ds4_gpu_tensor *out,
const void *model_map,
uint64_t model_size,
uint64_t weight_offset,
uint32_t in_dim,
uint32_t out_dim,
const ds4_gpu_tensor *x,
uint32_t n_rows);
int ds4_gpu_glm53_matmul_bf16_qkv(
ds4_gpu_tensor *out_q,
ds4_gpu_tensor *out_k,
ds4_gpu_tensor *out_v,
const void *model_map,
uint64_t model_size,
uint64_t weight_q_offset,
uint64_t weight_k_offset,
uint64_t weight_v_offset,
uint32_t in_dim,
uint32_t out_dim,
const ds4_gpu_tensor *x);
#ifndef DS4_GLM53_VISION_TYPES_DEFINED
#define DS4_GLM53_VISION_TYPES_DEFINED
#define DS4_GLM53_VISION_LAYERS 24u
typedef struct {
uint64_t norm1;
uint64_t qkv_weight;
uint64_t qkv_bias;
uint64_t q_norm;
uint64_t k_norm;
uint64_t attn_proj_weight;
uint64_t attn_proj_bias;
uint64_t norm2;
uint64_t gate_weight;
uint64_t gate_bias;
uint64_t up_weight;
uint64_t up_bias;
uint64_t down_weight;
uint64_t down_bias;
} ds4_glm53_vision_layer_weights;
typedef struct {
uint64_t patch_weight;
uint64_t patch_bias;
uint64_t post_norm;
uint64_t downsample_weight;
uint64_t downsample_bias;
uint64_t merger_proj;
uint64_t merger_norm;
uint64_t merger_norm_bias;
uint64_t merger_gate;
uint64_t merger_up;
uint64_t merger_down;
ds4_glm53_vision_layer_weights layer[DS4_GLM53_VISION_LAYERS];
} ds4_glm53_vision_weights;
#endif
/* Encode normalized, block-major image patches into 4096-wide language-model
* embeddings. GPU implementations keep every intermediate on device. */
int ds4_gpu_glm53_vision_encode(
float *out,
const float *patches,
uint32_t grid_h,
uint32_t grid_w,
const void *model_map,
uint64_t model_size,
const ds4_glm53_vision_weights *weights);
/* Replace token rows with projected image embeddings and repeat each row into
* every GLM hyperconnection stream. Must be called in an active command batch. */
int ds4_gpu_glm53_scatter_image_hc(
ds4_gpu_tensor *hc,
const ds4_gpu_tensor *image,
uint32_t dst_row,
uint32_t image_row,
uint32_t rows,
uint32_t total_rows,
uint32_t n_embd,
uint32_t n_hc);
/* GLM-5.3 Kimi Delta Attention. Recurrent and convolution state stay FP32. */
int ds4_gpu_glm53_kda_decode(
ds4_gpu_tensor *out,
ds4_gpu_tensor *conv_state,
ds4_gpu_tensor *recurrent_state,
const ds4_gpu_tensor *q,
const ds4_gpu_tensor *k,
const ds4_gpu_tensor *v,
const ds4_gpu_tensor *raw_gate,
const ds4_gpu_tensor *raw_beta,
const ds4_gpu_tensor *output_gate,
const void *model_map,
uint64_t model_size,
uint64_t q_conv_offset,
uint64_t k_conv_offset,
uint64_t v_conv_offset,
uint64_t a_log_offset,
uint64_t dt_bias_offset,
uint64_t output_norm_offset,
uint32_t n_heads,
uint32_t n_rows,
float gate_lower_bound,
float norm_eps);
int ds4_gpu_glm53_kda_prefill(
ds4_gpu_tensor *out,
ds4_gpu_tensor *conv_state,
ds4_gpu_tensor *recurrent_state,
ds4_gpu_tensor *q,
ds4_gpu_tensor *k,
ds4_gpu_tensor *v,
ds4_gpu_tensor *raw_gate,
const ds4_gpu_tensor *raw_beta,
const ds4_gpu_tensor *output_gate,
const void *model_map,
uint64_t model_size,
uint64_t q_conv_offset,
uint64_t k_conv_offset,
uint64_t v_conv_offset,
uint64_t a_log_offset,
uint64_t dt_bias_offset,
uint64_t output_norm_offset,
uint32_t n_heads,
uint32_t n_tokens,
float gate_lower_bound,
float norm_eps);
/* Decode-island CUDA graph capture (CUDA backend; Metal/ROCm/CPU stub it
* out and stay eager). Design ported from the Entrpi/ds4 batched-serving
* fork's per-layer decode graph capture. The key identifies a captured