hugo/DS4Server
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

Add GLM 5.3 Flash support

Browse Source
This commit is contained in:
Georg Bauer
2026-09-01 19:06:20 +02:00
parent 9a33c61ea6
commit 46d6a976a5
31 changed files with 7806 additions and 447 deletions
Download Patch File Download Diff File Expand all files Collapse all files

230
src/engine.rs
View File

@@ -12,8 +12,10 @@ use crate::model::{ModelChoice, validate_engine_artifacts};
#[cfg(target_os = "macos")]
use crate::settings::TurnSettings;
use crate::settings::{EngineSettings, ReasoningMode};
#[cfg(target_os = "macos")]
use base64::Engine as _;
use gguf::{
F16, F32, Gguf, I32, IQ2_XXS, MXFP4, Q2_K, Q4_0, Q4_K, Q5_K, Q6_K, Q8_0, Tensor, Value,
BF16, F16, F32, Gguf, I32, IQ2_XXS, MXFP4, Q2_K, Q4_0, Q4_K, Q5_K, Q6_K, Q8_0, Tensor, Value,
};
#[cfg(target_os = "macos")]
use kvstore::{KvStore, StoreReason};
@@ -32,6 +34,19 @@ use std::time::Instant;
use tokenizer::Tokenizer;
use validation::{SupportKind, validate_main, validate_support};
const VISION_DATA_START: &str = "<|ds4server_image_data|>";
const VISION_DATA_END: &str = "<|/ds4server_image_data|>";
const VISION_TOKEN_START: &str = "\u{fdd0}ds4-image:";
const VISION_TOKEN_END: &str = "\u{fdd1}";
const VISION_IMAGE_TOKEN: i32 = 154_854;
const VISION_START_TOKEN: i32 = 154_830;
const VISION_END_TOKEN: i32 = 154_831;
type VisionOverlays = Vec<(u32, metal::VisionEmbedding)>;
pub(crate) fn vision_data_marker(uri: &str) -> String {
format!("{VISION_DATA_START}{uri}{VISION_DATA_END}")
}
#[cfg(target_os = "macos")]
unsafe extern "C" {
fn mmap(
@@ -47,14 +62,14 @@ unsafe extern "C" {
fn munmap(address: *mut std::ffi::c_void, length: usize) -> i32;
}
pub(crate) use validation::validate_model_artifact;
pub(crate) use validation::{validate_model_artifact, validate_vision_artifact};
#[cfg(target_os = "macos")]
pub(crate) use kvstore::sweep_unreachable as sweep_transient_cache;
#[cfg(target_os = "macos")]
pub(crate) use metal::configure_sources as configure_metal_sources;
const DENSE: &[u32] = &[Q8_0, Q4_K, Q4_0];
const DENSE: &[u32] = &[BF16, Q8_0, Q4_K, Q4_0];
const ROUTED: &[u32] = &[Q8_0, IQ2_XXS, Q2_K, Q4_K, Q5_K, Q6_K, MXFP4];
const PLAIN: &[u32] = &[F16, F32];
const DSPARK_DENSE: &[u32] = &[F16, F32, Q8_0];
@@ -211,9 +226,53 @@ const GLM: Shape = Shape {
original_context: 1_048_576,
};
const GLM53_FLASH: Shape = Shape {
model: ModelChoice::Glm53Flash,
family: ModelFamily::Glm,
layers: 46,
embd: 4096,
vocab: 154_880,
heads: 64,
head_kv: 1,
head_dim: 512,
value_dim: 256,
rot: 0,
out_groups: 0,
lora_q: 1536,
lora_o: 0,
experts: 288,
experts_used: 8,
expert_shared: 1,
ff_expert: 2048,
ff_dense: 12_288,
hash_layers: 0,
sliding_window: 0,
indexer_heads: 32,
indexer_head_dim: 128,
indexer_top_k: 2048,
hc: 4,
hc_sinkhorn: 20,
nextn: 1,
leading_dense: 3,
kv_lora: 512,
key_mla: 256,
value_mla: 256,
rms_epsilon: 1.0e-5,
hc_epsilon: 1.0e-6,
expert_weight_scale: 2.5,
swiglu_clamp: 10.0,
rope_base: 0.0,
rope_scale: 0.0,
rope_beta_fast: 0.0,
rope_beta_slow: 0.0,
compress_rope_base: 0.0,
original_context: 1_048_576,
};
pub(crate) struct Model {
main: Gguf,
support: Option<Gguf>,
vision: Option<Gguf>,
support_kind: Option<SupportKind>,
shape: Shape,
tokenizer: Tokenizer,
@@ -226,6 +285,7 @@ pub(crate) struct ModelSummary {
pub(crate) tensor_count: usize,
pub(crate) vocabulary_size: usize,
pub(crate) support_loaded: bool,
pub(crate) vision_loaded: bool,
}
impl Model {
@@ -249,6 +309,14 @@ impl Model {
model.support = Some(support);
model.support_kind = Some(kind);
}
if let Some(path) = &settings.artifacts.vision {
validate_vision_artifact(path)?;
let vision = Gguf::open(path)?;
if settings.execution.warm_weights {
vision.warm()?;
}
model.vision = Some(vision);
}
Ok(model)
}
@@ -266,6 +334,7 @@ impl Model {
Ok(Self {
main,
support: None,
vision: None,
support_kind: None,
shape,
tokenizer,
@@ -275,14 +344,21 @@ impl Model {
pub(crate) fn summary(&self) -> ModelSummary {
ModelSummary {
model: self.shape.model,
mapped_bytes: self.main.len() + self.support.as_ref().map_or(0, Gguf::len),
mapped_bytes: self.main.len()
+ self.support.as_ref().map_or(0, Gguf::len)
+ self.vision.as_ref().map_or(0, Gguf::len),
tensor_count: self.main.tensors.len()
+ self
.support
.as_ref()
.map_or(0, |support| support.tensors.len()),
.map_or(0, |support| support.tensors.len())
+ self
.vision
.as_ref()
.map_or(0, |vision| vision.tensors.len()),
vocabulary_size: self.tokenizer.vocab_size(),
support_loaded: self.support.is_some(),
vision_loaded: self.vision.is_some(),
}
}
@@ -293,6 +369,9 @@ impl Model {
if let Some(support) = &self.support {
hash.update(support.checkpoint_identity());
}
if let Some(vision) = &self.vision {
hash.update(vision.checkpoint_identity());
}
hash.finalize().into()
}
@@ -732,7 +811,7 @@ impl Generator {
}
phase("Updating system prompt cache…");
let completed = self.prefill_suffix(&tokens, reused, cancelled, progress)?;
let completed = self.prefill_suffix(&tokens, reused, false, cancelled, progress)?;
if completed != tokens.len() - reused {
return Err("generation cancelled while updating the system prompt cache".into());
}
@@ -1055,11 +1134,12 @@ impl Generator {
&mut self,
tokens: &[i32],
reused: usize,
has_vision: bool,
cancelled: &AtomicBool,
progress: &mut impl FnMut(u32, u32, Option<f32>),
) -> Result<usize, String> {
let suffix = &tokens[reused..];
if (reused == 0 && tokens.len() > 1) || suffix.len() >= 4 {
if has_vision || (reused == 0 && tokens.len() > 1) || suffix.len() >= 4 {
let context = self.executor.context();
self.executor.prefill(suffix, |used| {
progress(used, context, None);
@@ -1079,6 +1159,87 @@ impl Generator {
}
}
fn render_multimodal_conversation(
&mut self,
system: &str,
messages: &[ChatTurn],
reasoning: ReasoningMode,
) -> Result<(Vec<i32>, VisionOverlays), String> {
let mut rendered = messages.to_vec();
let mut embeddings = Vec::new();
let mut total_images = 0_usize;
let mut total_bytes = 0_usize;
for message in &mut rendered {
if !message.content.contains(VISION_DATA_START) {
continue;
}
if !message.user && !message.tool {
return Err("vision input is allowed only in user or tool messages".into());
}
let mut content = String::with_capacity(message.content.len());
let mut rest = message.content.as_str();
while let Some(start) = rest.find(VISION_DATA_START) {
content.push_str(&rest[..start]);
let encoded = &rest[start + VISION_DATA_START.len()..];
let end = encoded
.find(VISION_DATA_END)
.ok_or("unterminated image input")?;
let uri = &encoded[..end];
let payload = uri
.strip_prefix("data:image/png;base64,")
.or_else(|| uri.strip_prefix("data:image/jpeg;base64,"))
.ok_or("image input must be an inline PNG or JPEG data URI")?;
total_images += 1;
if total_images > 16 {
return Err("a request may contain at most 16 images".into());
}
let bytes = base64::engine::general_purpose::STANDARD
.decode(payload)
.map_err(|_| "image data URI contains invalid base64")?;
total_bytes = total_bytes
.checked_add(bytes.len())
.ok_or("image input size overflow")?;
if total_bytes > 64 * 1024 * 1024 {
return Err("image inputs exceed the 64 MiB request limit".into());
}
let embedding = self.executor.encode_vision(&bytes)?;
content.push_str(VISION_TOKEN_START);
content.push_str(&embedding.tokens.to_string());
content.push_str(VISION_TOKEN_END);
embeddings.push(embedding);
rest = &encoded[end + VISION_DATA_END.len()..];
}
content.push_str(rest);
message.content = content;
}
let tokens = self
.executor
.model()
.render_conversation(system, &rendered, reasoning);
let mut overlays = Vec::with_capacity(embeddings.len());
let mut cursor = 0_usize;
for embedding in embeddings {
let count = embedding.tokens as usize;
let relative = tokens[cursor..]
.windows(count + 2)
.position(|window| {
window[0] == VISION_START_TOKEN
&& window[count + 1] == VISION_END_TOKEN
&& window[1..count + 1]
.iter()
.all(|token| *token == VISION_IMAGE_TOKEN)
})
.ok_or("rendered prompt lost an image placeholder")?;
let start = cursor + relative + 1;
overlays.push((
u32::try_from(start).map_err(|_| "image prompt position overflow")?,
embedding,
));
cursor = start + count + 1;
}
Ok((tokens, overlays))
}
fn generate_inner(
&mut self,
messages: &[ChatTurn],
@@ -1087,29 +1248,41 @@ impl Generator {
emit: &mut impl FnMut(bool, String),
progress: &mut impl FnMut(u32, u32, Option<f32>),
) -> Result<(GenerationOutput, bool), String> {
let tokens = match messages.split_last() {
Some((latest, history))
if latest.user
&& self.executor.checkpoint_tag()
== conversation_tag(
&settings.system_prompt,
settings.reasoning_mode,
history,
) =>
{
let mut tokens = self.executor.tokens().to_vec();
tokens.extend(self.executor.model().render_continuation(
&latest.content,
settings.reasoning_mode,
latest.skip_previous_eos,
));
tokens
}
_ => self.executor.model().render_conversation(
let has_vision = messages
.iter()
.any(|message| message.content.contains(VISION_DATA_START));
let (tokens, overlays) = if has_vision {
self.render_multimodal_conversation(
&settings.system_prompt,
messages,
settings.reasoning_mode,
),
)?
} else {
let tokens = match messages.split_last() {
Some((latest, history))
if latest.user
&& self.executor.checkpoint_tag()
== conversation_tag(
&settings.system_prompt,
settings.reasoning_mode,
history,
) =>
{
let mut tokens = self.executor.tokens().to_vec();
tokens.extend(self.executor.model().render_continuation(
&latest.content,
settings.reasoning_mode,
latest.skip_previous_eos,
));
tokens
}
_ => self.executor.model().render_conversation(
&settings.system_prompt,
messages,
settings.reasoning_mode,
),
};
(tokens, Vec::new())
};
if tokens.is_empty() {
return Err("the rendered prompt is empty".into());
@@ -1122,6 +1295,7 @@ impl Generator {
));
}
let reused = self.executor.align_prompt(&tokens)?;
self.executor.set_vision_overlays(overlays)?;
self.metrics.kv_prefix_reused(reused);
progress(self.executor.position(), self.executor.context(), None);
let mut rng = Rng::new(settings.seed.unwrap_or(0x4453_3453_4552_5645));
@@ -1140,7 +1314,7 @@ impl Generator {
let mut pending_utf8 = Vec::new();
let prompt_tokens = tokens.len();
let suffix = &tokens[reused..];
let completed = self.prefill_suffix(&tokens, reused, cancelled, progress)?;
let completed = self.prefill_suffix(&tokens, reused, has_vision, cancelled, progress)?;
self.publish_execution_stats();
if completed != suffix.len() {
return Ok((