hugo/DS4Server
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

Add GLM 5.3 Flash support

Browse Source
This commit is contained in:
Georg Bauer
2026-09-01 19:06:20 +02:00
parent 9a33c61ea6
commit 46d6a976a5
31 changed files with 7806 additions and 447 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/agent.rs
View File

@@ -3612,7 +3612,7 @@ pub(crate) fn parse_tool_calls(
model: ModelChoice,
text: &str,
) -> Result<(String, Vec<ToolCall>), String> {
let (content, calls) = if model == ModelChoice::Glm52 {
let (content, calls) = if model.is_glm() {
parse_glm_calls(text)?
} else {
crate::dsml::parse_tool_calls(text)?
@@ -3643,7 +3643,7 @@ fn system_prompt_with_tools(
ralph_child: bool,
) -> String {
let schemas = tool_schemas(dev_brain, ralph_child);
let tools = if model == ModelChoice::Glm52 {
let tools = if model.is_glm() {
format!(
"You are a coding agent running in a local workspace. Use tools for local file and system work. Avoid printing large file contents or code blocks as answers; edit files with tools, then summarize briefly.\n\n# Tools\n\n<tools>\n{schemas}\n</tools>\n\nFor a function call, output exactly: <tool_call>function-name<arg_key>key</arg_key><arg_value>value</arg_value></tool_call>\nTool calls are not allowed inside <think></think>. Pass numbers and booleans as JSON primitives, not quoted strings. When a tool fails validation or execution, use its code, field, expected, and received feedback to correct the next call. Preserve the current system configuration unless the user explicitly asks otherwise."
)

64
src/app.rs
View File

@@ -35,6 +35,7 @@ use crate::settings::{
ReasoningMode, RuntimePreferences, SpeculativePreferences, SsdPreferences, SteeringPreferences,
StreamingCacheBudget,
};
use base64::Engine as _;
use iced::widget::{markdown, scrollable, text_editor};
use iced::{Size, Subscription, Task, keyboard, mouse, window};
use rfd::AsyncFileDialog;
@@ -115,6 +116,7 @@ pub(crate) struct App {
project_name_input: String,
model_download: ModelDownload,
pub(super) composer: text_editor::Content,
pub(super) pending_vision_image: Option<PendingVisionImage>,
pub(super) queued_inputs: VecDeque<String>,
pub(super) conversation: Vec<ChatMessage>,
/// Follow appended chat content until the user scrolls away from the tail.
@@ -191,6 +193,7 @@ struct ChatSnapshot {
selected_session: Option<i32>,
permission_mode: PermissionMode,
composer: text_editor::Content,
pending_vision_image: Option<PendingVisionImage>,
queued_inputs: VecDeque<String>,
conversation: Vec<ChatMessage>,
chat_follow_tail: bool,
@@ -231,6 +234,11 @@ struct ChatSnapshot {
skip_compaction_once: bool,
}
pub(super) struct PendingVisionImage {
pub(super) name: String,
marker: String,
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub(super) struct ProjectChoice {
id: i32,
@@ -480,6 +488,9 @@ pub(crate) enum Message {
StopModelDownload,
DownloadProgressTick,
ComposerAction(text_editor::Action),
ChooseVisionImage,
VisionImagePicked(Result<Option<(String, Vec<u8>)>, String>),
RemoveVisionImage,
TranscriptAction(usize, text_editor::Action),
ToggleReasoning(usize),
OpenLink(markdown::Uri),
@@ -641,6 +652,7 @@ impl App {
project_name_input: String::new(),
model_download: ModelDownload::Idle,
composer: text_editor::Content::new(),
pending_vision_image: None,
queued_inputs: VecDeque::new(),
conversation: Vec::new(),
chat_follow_tail: true,
@@ -805,6 +817,7 @@ impl App {
project_name_input: String::new(),
model_download: ModelDownload::Idle,
composer: text_editor::Content::new(),
pending_vision_image: None,
queued_inputs: VecDeque::new(),
conversation: Vec::new(),
chat_follow_tail: true,
@@ -882,6 +895,7 @@ impl App {
selected_session: self.selected_session.take(),
permission_mode: self.permission_mode,
composer: std::mem::take(&mut self.composer),
pending_vision_image: self.pending_vision_image.take(),
queued_inputs: std::mem::take(&mut self.queued_inputs),
conversation: std::mem::take(&mut self.conversation),
chat_follow_tail: std::mem::replace(&mut self.chat_follow_tail, true),
@@ -932,6 +946,7 @@ impl App {
self.selected_session = snapshot.selected_session;
self.permission_mode = snapshot.permission_mode;
self.composer = snapshot.composer;
self.pending_vision_image = snapshot.pending_vision_image;
self.queued_inputs = snapshot.queued_inputs;
self.conversation = snapshot.conversation;
self.chat_follow_tail = snapshot.chat_follow_tail;
@@ -1290,6 +1305,55 @@ impl App {
}
Message::MetricsTick => self.sample_metrics(),
Message::ComposerAction(action) => self.composer.perform(action),
Message::ChooseVisionImage => {
if self.config.model != ModelChoice::Glm53Flash || self.generating {
return Task::none();
}
return Task::perform(
async {
let file = AsyncFileDialog::new()
.set_title("Attach an image")
.add_filter("Image", &["png", "jpg", "jpeg"])
.pick_file()
.await;
let Some(file) = file else {
return Ok::<_, String>(None);
};
let name = file.file_name();
let bytes = file.read().await;
Ok::<_, String>(Some((name, bytes)))
},
Message::VisionImagePicked,
);
}
Message::VisionImagePicked(result) => match result {
Ok(Some((name, bytes))) => {
if bytes.is_empty() || bytes.len() > 64 * 1024 * 1024 {
self.error = Some("Image must be between 1 byte and 64 MiB.".into());
} else {
let mime = match image::guess_format(&bytes) {
Ok(image::ImageFormat::Png) => Some("image/png"),
Ok(image::ImageFormat::Jpeg) => Some("image/jpeg"),
_ => None,
};
if let Some(mime) = mime {
let encoded = base64::engine::general_purpose::STANDARD.encode(bytes);
self.pending_vision_image = Some(PendingVisionImage {
name,
marker: crate::engine::vision_data_marker(&format!(
"data:{mime};base64,{encoded}"
)),
});
self.error = None;
} else {
self.error = Some("Only PNG and JPEG images are supported.".into());
}
}
}
Ok(None) => {}
Err(error) => self.error = Some(error),
},
Message::RemoveVisionImage => self.pending_vision_image = None,
Message::TranscriptAction(index, action) => {
if !action.is_edit()
&& let Some(message) = self.conversation.get_mut(index)

49
src/app/generation.rs
View File

@@ -517,7 +517,10 @@ impl App {
if self.selected_project.is_none() {
return;
}
let prompt = self.composer.text().trim().to_owned();
let mut prompt = self.composer.text().trim().to_owned();
if prompt.is_empty() && self.pending_vision_image.is_some() {
prompt = "Describe this image.".into();
}
if prompt.is_empty() {
return;
}
@@ -659,13 +662,20 @@ impl App {
#[cfg(target_os = "macos")]
let assistant_reasoning = effective.turn.reasoning_mode != ReasoningMode::Direct;
#[cfg(target_os = "macos")]
let (visible_prompt, base_model_prompt) = vision_prompts(
&prompt,
self.pending_vision_image
.as_ref()
.map(|image| (image.name.as_str(), image.marker.as_str())),
);
#[cfg(target_os = "macos")]
let model_prompt = if self.config.a2ui_enabled {
format!(
"{prompt}\n\nA2UI client metadata:\n{}",
"{base_model_prompt}\n\nA2UI client metadata:\n{}",
self.a2ui.client_metadata()
)
} else {
prompt.clone()
base_model_prompt
};
#[cfg(target_os = "macos")]
let mut injected_system = self.workspace_instruction_messages(&[], opening_turn);
@@ -750,8 +760,8 @@ impl App {
};
let mut saved = match database.start_chat_turn(
session_id,
&prompt,
(model_prompt != prompt).then_some(model_prompt.as_str()),
&visible_prompt,
(model_prompt != visible_prompt).then_some(model_prompt.as_str()),
&injected_system,
assistant_reasoning,
) {
@@ -811,6 +821,7 @@ impl App {
}
assistant.reasoning_open = assistant_reasoning;
self.composer = text_editor::Content::new();
self.pending_vision_image = None;
self.conversation.push(user);
self.conversation.push(assistant);
self.generating = true;
@@ -2459,6 +2470,18 @@ impl App {
}
}
fn vision_prompts(prompt: &str, image: Option<(&str, &str)>) -> (String, String) {
image.map_or_else(
|| (prompt.to_owned(), prompt.to_owned()),
|(name, marker)| {
(
format!("[Image: {name}]\n{prompt}"),
format!("{marker}\n{prompt}"),
)
},
)
}
/// Reduces a model reply to a single sidebar-sized line, or `None` if nothing
/// usable came back.
pub(super) fn session_title(reply: &str) -> Option<String> {
@@ -2485,7 +2508,7 @@ mod tests {
ChatMessage, TOOL_PROTOCOL_CORRECTION, TurnSummary, chat_turn, compacted_context_start,
correction_already_sent, extension_context_visibility, has_chat_after_last_compaction,
has_misplaced_tool_call, is_empty_response, promote_legacy_turn_summaries, queued_prompt,
sync_a2ui_message, title_context,
sync_a2ui_message, title_context, vision_prompts,
};
use crate::engine::ChatTurn;
use crate::model::ModelChoice;
@@ -2553,6 +2576,20 @@ mod tests {
assert_eq!(user.content, "visible question");
}
#[test]
fn attached_image_marker_is_model_only() {
let (visible, model) = vision_prompts(
"Describe this.",
Some(("diagram.png", "<|vision_start|>payload<|vision_end|>")),
);
assert_eq!(visible, "[Image: diagram.png]\nDescribe this.");
assert_eq!(
model,
"<|vision_start|>payload<|vision_end|>\nDescribe this."
);
}
#[test]
fn legacy_generation_rows_collapse_into_their_user_turn() {
let mut user = assistant(None, "question");

13
src/app/preferences.rs
View File

@@ -593,6 +593,9 @@ impl App {
return;
}
self.config = config;
if self.config.model != ModelChoice::Glm53Flash {
self.pending_vision_image = None;
}
self.context_limit = self.config.active_generation().context_tokens.max(0) as u32;
#[cfg(target_os = "macos")]
{
@@ -653,9 +656,11 @@ impl App {
self.preference_draft.default_reasoning_mode = mode;
self.preference_draft.load_generation(model, mode);
self.preference_draft.load_acceleration(model);
if model == ModelChoice::Glm52 {
if model.is_glm() {
self.preference_draft.power_percent.clear();
self.preference_draft.prefill_chunk.clear();
}
if model == ModelChoice::Glm52 {
self.preference_draft.directional_steering_file.clear();
self.preference_draft.directional_steering_ffn.clear();
self.preference_draft.directional_steering_attn.clear();
@@ -851,7 +856,7 @@ impl App {
}
Message::PreferenceGlmMtpChanged(value) => {
self.preference_draft.glm_mtp =
self.preference_draft.acceleration_model == ModelChoice::Glm52 && value;
self.preference_draft.acceleration_model.supports_glm_mtp() && value;
if !self.preference_draft.glm_mtp {
self.preference_draft.glm_mtp_timing = false;
}
@@ -859,7 +864,7 @@ impl App {
}
Message::PreferenceGlmMtpTimingChanged(value) => {
self.preference_draft.glm_mtp_timing =
self.preference_draft.acceleration_model == ModelChoice::Glm52 && value;
self.preference_draft.acceleration_model.supports_glm_mtp() && value;
if self.preference_draft.glm_mtp_timing {
self.preference_draft.glm_mtp = true;
}
@@ -907,7 +912,7 @@ impl App {
self.preference_error = None;
}
Message::PreferenceSsdFullLayersChanged(value) => {
if self.preference_draft.acceleration_model == ModelChoice::Glm52 {
if self.preference_draft.acceleration_model.is_glm() {
self.preference_draft.ssd_full_layers = value;
}
self.preference_error = None;

41
src/app/view/chat.rs
View File

@@ -250,7 +250,8 @@ impl App {
.padding(8)
.style(stop_button_style)
.on_press(Message::StopGeneration)
} else if self.composer.text().trim().is_empty() {
} else if self.composer.text().trim().is_empty() && self.pending_vision_image.is_none()
{
action_button(icon(ICON_SEND, 18)).padding(8)
} else {
action_button(icon(ICON_SEND, 18))
@@ -263,6 +264,20 @@ impl App {
self.context_used.min(self.context_limit) as f32 / self.context_limit as f32
};
let mut composer_content = column![composer].spacing(6);
if let Some(image) = &self.pending_vision_image {
composer_content = composer_content.push(
row![
icon(ICON_PAPERCLIP, 14),
text(format!("Image · {}", image.name)).size(12),
button(text("Remove").size(11))
.padding(0)
.style(button::text)
.on_press(Message::RemoveVisionImage),
]
.spacing(7)
.align_y(Alignment::Center),
);
}
for queued in &self.queued_inputs {
let mut queued = queued.replace('\n', " ");
if queued.chars().count() > 120 {
@@ -322,10 +337,32 @@ impl App {
.padding([2, 6])
.into()
};
let vision_ready = self.config.model == ModelChoice::Glm53Flash
&& model::engine_artifacts(ModelChoice::Glm53Flash, false, &models_path())
.vision
.is_some();
let attach = if vision_ready && !self.generating && self.pending_vision_image.is_none()
{
action_button(icon(ICON_PAPERCLIP, 19))
.padding(4)
.on_press(Message::ChooseVisionImage)
} else {
action_button(icon(ICON_PAPERCLIP, 19)).padding(4)
};
composer_content =
composer_content.push(
row![
icon(ICON_PAPERCLIP, 19),
tooltip(
attach,
container(text(if vision_ready {
"Attach PNG or JPEG"
} else {
"GLM 5.3 Flash vision sidecar is not ready"
}))
.padding(8)
.style(preference_group_style),
tooltip::Position::Top,
),
tooltip(
context_pie(context_fraction, 19),
container(

36
src/app/view/preferences.rs
View File

@@ -14,12 +14,16 @@ impl App {
.on_toggle_maybe(dspark_toggle),
"Speculative decoding with the managed DSpark draft artifact: a small model proposes tokens that the main model verifies in one pass. Usually a large speedup; the target model may also stream routed experts from SSD.",
);
let glm_mtp_toggle: Option<fn(bool) -> Message> =
(self.preference_draft.acceleration_model == ModelChoice::Glm52)
.then_some(Message::PreferenceGlmMtpChanged);
let glm_mtp_timing_toggle: Option<fn(bool) -> Message> =
(self.preference_draft.acceleration_model == ModelChoice::Glm52)
.then_some(Message::PreferenceGlmMtpTimingChanged);
let glm_mtp_toggle: Option<fn(bool) -> Message> = self
.preference_draft
.acceleration_model
.supports_glm_mtp()
.then_some(Message::PreferenceGlmMtpChanged);
let glm_mtp_timing_toggle: Option<fn(bool) -> Message> = self
.preference_draft
.acceleration_model
.supports_glm_mtp()
.then_some(Message::PreferenceGlmMtpTimingChanged);
let dspark_strict_toggle: Option<fn(bool) -> Message> = self
.preference_draft
.acceleration_model
@@ -71,14 +75,16 @@ impl App {
"0.8 (DeepSeek V4 Flash default)",
&self.preference_draft.dspark_confidence_threshold,
);
if self.preference_draft.model != ModelChoice::Glm52 {
if !self.preference_draft.model.is_glm() {
power = power.on_input(Message::PreferencePowerChanged);
prefill = prefill.on_input(Message::PreferencePrefillChunkChanged);
}
if self.preference_draft.model != ModelChoice::Glm52 {
steering_file = steering_file.on_input(Message::PreferenceSteeringFileChanged);
steering_ffn = steering_ffn.on_input(Message::PreferenceSteeringFfnChanged);
steering_attn = steering_attn.on_input(Message::PreferenceSteeringAttnChanged);
}
if self.preference_draft.acceleration_model == ModelChoice::Glm52 {
if self.preference_draft.acceleration_model.is_glm() {
ssd_full_layers = ssd_full_layers.on_input(Message::PreferenceSsdFullLayersChanged);
}
if self.preference_draft.acceleration_model.supports_dspark() {
@@ -467,6 +473,12 @@ impl App {
text_input("32768", &self.preference_draft.context_tokens)
.on_input(Message::PreferenceContextChanged),
),
text(if self.preference_draft.generation_model == ModelChoice::Glm53Flash {
"GLM 5.3 Flash: 32768 is the recommended default on this 128 GB machine; 50000 is the validated extended-session target."
} else {
""
})
.size(12),
preference_input_row(
"Maximum generated tokens",
"Hard stop for a single reply, counted from the first generated token. It bounds runaway answers and reasoning loops; it does not reserve memory.",
@@ -550,8 +562,8 @@ impl App {
.on_toggle(Message::PreferenceWarmWeightsChanged),
"Reads every mapped weight page once at load, so the first reply is not interrupted by page faults from disk. Loading takes longer and memory pressure rises immediately.",
),
text(if self.preference_draft.model == ModelChoice::Glm52 {
"GLM 5.2 uses full GPU power and selects prefill chunks automatically."
text(if self.preference_draft.model.is_glm() {
"GLM uses full GPU power and selects prefill chunks automatically."
} else {
"Blank numeric values preserve DS4's automatic engine behavior."
})
@@ -595,7 +607,7 @@ impl App {
toggle(self.preference_draft.glm_mtp)
.label("Enable integrated GLM MTP")
.on_toggle_maybe(glm_mtp_toggle),
"Uses the prediction head built into GLM 5.2 for speculative decoding, so no separate draft model is loaded. Available for GLM 5.2 only.",
"Uses the prediction head built into GLM for speculative decoding, so no separate draft model is loaded.",
),
hint(
toggle(self.preference_draft.glm_mtp_timing)
@@ -623,7 +635,7 @@ impl App {
),
text(if self.preference_draft.acceleration_model.supports_dspark() {
"DeepSeek V4 Flash 0731 uses its managed DSpark support artifact."
} else if self.preference_draft.acceleration_model == ModelChoice::Glm52 {
} else if self.preference_draft.acceleration_model.supports_glm_mtp() {
"GLM MTP is integrated; DSpark is unavailable for this model."
} else {
"No speculative-decoding support is available for this model."

230
src/engine.rs
View File

@@ -12,8 +12,10 @@ use crate::model::{ModelChoice, validate_engine_artifacts};
#[cfg(target_os = "macos")]
use crate::settings::TurnSettings;
use crate::settings::{EngineSettings, ReasoningMode};
#[cfg(target_os = "macos")]
use base64::Engine as _;
use gguf::{
F16, F32, Gguf, I32, IQ2_XXS, MXFP4, Q2_K, Q4_0, Q4_K, Q5_K, Q6_K, Q8_0, Tensor, Value,
BF16, F16, F32, Gguf, I32, IQ2_XXS, MXFP4, Q2_K, Q4_0, Q4_K, Q5_K, Q6_K, Q8_0, Tensor, Value,
};
#[cfg(target_os = "macos")]
use kvstore::{KvStore, StoreReason};
@@ -32,6 +34,19 @@ use std::time::Instant;
use tokenizer::Tokenizer;
use validation::{SupportKind, validate_main, validate_support};
const VISION_DATA_START: &str = "<|ds4server_image_data|>";
const VISION_DATA_END: &str = "<|/ds4server_image_data|>";
const VISION_TOKEN_START: &str = "\u{fdd0}ds4-image:";
const VISION_TOKEN_END: &str = "\u{fdd1}";
const VISION_IMAGE_TOKEN: i32 = 154_854;
const VISION_START_TOKEN: i32 = 154_830;
const VISION_END_TOKEN: i32 = 154_831;
type VisionOverlays = Vec<(u32, metal::VisionEmbedding)>;
pub(crate) fn vision_data_marker(uri: &str) -> String {
format!("{VISION_DATA_START}{uri}{VISION_DATA_END}")
}
#[cfg(target_os = "macos")]
unsafe extern "C" {
fn mmap(
@@ -47,14 +62,14 @@ unsafe extern "C" {
fn munmap(address: *mut std::ffi::c_void, length: usize) -> i32;
}
pub(crate) use validation::validate_model_artifact;
pub(crate) use validation::{validate_model_artifact, validate_vision_artifact};
#[cfg(target_os = "macos")]
pub(crate) use kvstore::sweep_unreachable as sweep_transient_cache;
#[cfg(target_os = "macos")]
pub(crate) use metal::configure_sources as configure_metal_sources;
const DENSE: &[u32] = &[Q8_0, Q4_K, Q4_0];
const DENSE: &[u32] = &[BF16, Q8_0, Q4_K, Q4_0];
const ROUTED: &[u32] = &[Q8_0, IQ2_XXS, Q2_K, Q4_K, Q5_K, Q6_K, MXFP4];
const PLAIN: &[u32] = &[F16, F32];
const DSPARK_DENSE: &[u32] = &[F16, F32, Q8_0];
@@ -211,9 +226,53 @@ const GLM: Shape = Shape {
original_context: 1_048_576,
};
const GLM53_FLASH: Shape = Shape {
model: ModelChoice::Glm53Flash,
family: ModelFamily::Glm,
layers: 46,
embd: 4096,
vocab: 154_880,
heads: 64,
head_kv: 1,
head_dim: 512,
value_dim: 256,
rot: 0,
out_groups: 0,
lora_q: 1536,
lora_o: 0,
experts: 288,
experts_used: 8,
expert_shared: 1,
ff_expert: 2048,
ff_dense: 12_288,
hash_layers: 0,
sliding_window: 0,
indexer_heads: 32,
indexer_head_dim: 128,
indexer_top_k: 2048,
hc: 4,
hc_sinkhorn: 20,
nextn: 1,
leading_dense: 3,
kv_lora: 512,
key_mla: 256,
value_mla: 256,
rms_epsilon: 1.0e-5,
hc_epsilon: 1.0e-6,
expert_weight_scale: 2.5,
swiglu_clamp: 10.0,
rope_base: 0.0,
rope_scale: 0.0,
rope_beta_fast: 0.0,
rope_beta_slow: 0.0,
compress_rope_base: 0.0,
original_context: 1_048_576,
};
pub(crate) struct Model {
main: Gguf,
support: Option<Gguf>,
vision: Option<Gguf>,
support_kind: Option<SupportKind>,
shape: Shape,
tokenizer: Tokenizer,
@@ -226,6 +285,7 @@ pub(crate) struct ModelSummary {
pub(crate) tensor_count: usize,
pub(crate) vocabulary_size: usize,
pub(crate) support_loaded: bool,
pub(crate) vision_loaded: bool,
}
impl Model {
@@ -249,6 +309,14 @@ impl Model {
model.support = Some(support);
model.support_kind = Some(kind);
}
if let Some(path) = &settings.artifacts.vision {
validate_vision_artifact(path)?;
let vision = Gguf::open(path)?;
if settings.execution.warm_weights {
vision.warm()?;
}
model.vision = Some(vision);
}
Ok(model)
}
@@ -266,6 +334,7 @@ impl Model {
Ok(Self {
main,
support: None,
vision: None,
support_kind: None,
shape,
tokenizer,
@@ -275,14 +344,21 @@ impl Model {
pub(crate) fn summary(&self) -> ModelSummary {
ModelSummary {
model: self.shape.model,
mapped_bytes: self.main.len() + self.support.as_ref().map_or(0, Gguf::len),
mapped_bytes: self.main.len()
+ self.support.as_ref().map_or(0, Gguf::len)
+ self.vision.as_ref().map_or(0, Gguf::len),
tensor_count: self.main.tensors.len()
+ self
.support
.as_ref()
.map_or(0, |support| support.tensors.len()),
.map_or(0, |support| support.tensors.len())
+ self
.vision
.as_ref()
.map_or(0, |vision| vision.tensors.len()),
vocabulary_size: self.tokenizer.vocab_size(),
support_loaded: self.support.is_some(),
vision_loaded: self.vision.is_some(),
}
}
@@ -293,6 +369,9 @@ impl Model {
if let Some(support) = &self.support {
hash.update(support.checkpoint_identity());
}
if let Some(vision) = &self.vision {
hash.update(vision.checkpoint_identity());
}
hash.finalize().into()
}
@@ -732,7 +811,7 @@ impl Generator {
}
phase("Updating system prompt cache…");
let completed = self.prefill_suffix(&tokens, reused, cancelled, progress)?;
let completed = self.prefill_suffix(&tokens, reused, false, cancelled, progress)?;
if completed != tokens.len() - reused {
return Err("generation cancelled while updating the system prompt cache".into());
}
@@ -1055,11 +1134,12 @@ impl Generator {
&mut self,
tokens: &[i32],
reused: usize,
has_vision: bool,
cancelled: &AtomicBool,
progress: &mut impl FnMut(u32, u32, Option<f32>),
) -> Result<usize, String> {
let suffix = &tokens[reused..];
if (reused == 0 && tokens.len() > 1) || suffix.len() >= 4 {
if has_vision || (reused == 0 && tokens.len() > 1) || suffix.len() >= 4 {
let context = self.executor.context();
self.executor.prefill(suffix, |used| {
progress(used, context, None);
@@ -1079,6 +1159,87 @@ impl Generator {
}
}
fn render_multimodal_conversation(
&mut self,
system: &str,
messages: &[ChatTurn],
reasoning: ReasoningMode,
) -> Result<(Vec<i32>, VisionOverlays), String> {
let mut rendered = messages.to_vec();
let mut embeddings = Vec::new();
let mut total_images = 0_usize;
let mut total_bytes = 0_usize;
for message in &mut rendered {
if !message.content.contains(VISION_DATA_START) {
continue;
}
if !message.user && !message.tool {
return Err("vision input is allowed only in user or tool messages".into());
}
let mut content = String::with_capacity(message.content.len());
let mut rest = message.content.as_str();
while let Some(start) = rest.find(VISION_DATA_START) {
content.push_str(&rest[..start]);
let encoded = &rest[start + VISION_DATA_START.len()..];
let end = encoded
.find(VISION_DATA_END)
.ok_or("unterminated image input")?;
let uri = &encoded[..end];
let payload = uri
.strip_prefix("data:image/png;base64,")
.or_else(|| uri.strip_prefix("data:image/jpeg;base64,"))
.ok_or("image input must be an inline PNG or JPEG data URI")?;
total_images += 1;
if total_images > 16 {
return Err("a request may contain at most 16 images".into());
}
let bytes = base64::engine::general_purpose::STANDARD
.decode(payload)
.map_err(|_| "image data URI contains invalid base64")?;
total_bytes = total_bytes
.checked_add(bytes.len())
.ok_or("image input size overflow")?;
if total_bytes > 64 * 1024 * 1024 {
return Err("image inputs exceed the 64 MiB request limit".into());
}
let embedding = self.executor.encode_vision(&bytes)?;
content.push_str(VISION_TOKEN_START);
content.push_str(&embedding.tokens.to_string());
content.push_str(VISION_TOKEN_END);
embeddings.push(embedding);
rest = &encoded[end + VISION_DATA_END.len()..];
}
content.push_str(rest);
message.content = content;
}
let tokens = self
.executor
.model()
.render_conversation(system, &rendered, reasoning);
let mut overlays = Vec::with_capacity(embeddings.len());
let mut cursor = 0_usize;
for embedding in embeddings {
let count = embedding.tokens as usize;
let relative = tokens[cursor..]
.windows(count + 2)
.position(|window| {
window[0] == VISION_START_TOKEN
&& window[count + 1] == VISION_END_TOKEN
&& window[1..count + 1]
.iter()
.all(|token| *token == VISION_IMAGE_TOKEN)
})
.ok_or("rendered prompt lost an image placeholder")?;
let start = cursor + relative + 1;
overlays.push((
u32::try_from(start).map_err(|_| "image prompt position overflow")?,
embedding,
));
cursor = start + count + 1;
}
Ok((tokens, overlays))
}
fn generate_inner(
&mut self,
messages: &[ChatTurn],
@@ -1087,29 +1248,41 @@ impl Generator {
emit: &mut impl FnMut(bool, String),
progress: &mut impl FnMut(u32, u32, Option<f32>),
) -> Result<(GenerationOutput, bool), String> {
let tokens = match messages.split_last() {
Some((latest, history))
if latest.user
&& self.executor.checkpoint_tag()
== conversation_tag(
&settings.system_prompt,
settings.reasoning_mode,
history,
) =>
{
let mut tokens = self.executor.tokens().to_vec();
tokens.extend(self.executor.model().render_continuation(
&latest.content,
settings.reasoning_mode,
latest.skip_previous_eos,
));
tokens
}
_ => self.executor.model().render_conversation(
let has_vision = messages
.iter()
.any(|message| message.content.contains(VISION_DATA_START));
let (tokens, overlays) = if has_vision {
self.render_multimodal_conversation(
&settings.system_prompt,
messages,
settings.reasoning_mode,
),
)?
} else {
let tokens = match messages.split_last() {
Some((latest, history))
if latest.user
&& self.executor.checkpoint_tag()
== conversation_tag(
&settings.system_prompt,
settings.reasoning_mode,
history,
) =>
{
let mut tokens = self.executor.tokens().to_vec();
tokens.extend(self.executor.model().render_continuation(
&latest.content,
settings.reasoning_mode,
latest.skip_previous_eos,
));
tokens
}
_ => self.executor.model().render_conversation(
&settings.system_prompt,
messages,
settings.reasoning_mode,
),
};
(tokens, Vec::new())
};
if tokens.is_empty() {
return Err("the rendered prompt is empty".into());
@@ -1122,6 +1295,7 @@ impl Generator {
));
}
let reused = self.executor.align_prompt(&tokens)?;
self.executor.set_vision_overlays(overlays)?;
self.metrics.kv_prefix_reused(reused);
progress(self.executor.position(), self.executor.context(), None);
let mut rng = Rng::new(settings.seed.unwrap_or(0x4453_3453_4552_5645));
@@ -1140,7 +1314,7 @@ impl Generator {
let mut pending_utf8 = Vec::new();
let prompt_tokens = tokens.len();
let suffix = &tokens[reused..];
let completed = self.prefill_suffix(&tokens, reused, cancelled, progress)?;
let completed = self.prefill_suffix(&tokens, reused, has_vision, cancelled, progress)?;
self.publish_execution_stats();
if completed != suffix.len() {
return Ok((

2
src/engine/gguf.rs
View File

@@ -19,6 +19,7 @@ pub(super) const Q5_K: u32 = 13;
pub(super) const Q6_K: u32 = 14;
pub(super) const IQ2_XXS: u32 = 16;
pub(super) const I32: u32 = 26;
pub(super) const BF16: u32 = 30;
pub(super) const MXFP4: u32 = 39;
#[derive(Clone, Debug)]
@@ -488,6 +489,7 @@ impl<'a> Cursor<'a> {
| "deepseek4.dspark.target_layer_ids"
| "deepseek4.dspark_target_layer_ids"
| "dspark.target_layer_ids"
| "glm5-next.layer_types"
);
if !keep {
for _ in 0..len {

37
src/engine/metal.rs
View File

@@ -3,12 +3,14 @@ mod glm;
mod gpu;
mod hotlist;
mod profile;
mod vision;
pub(super) use vision::VisionEmbedding;
use glm::GlmExecutor;
use gpu::*;
use profile::ExpertProfile;
use super::gguf::{F16, F32, Gguf, IQ2_XXS, MXFP4, Q2_K, Q4_K, Q8_0, Tensor as GgufTensor};
use super::gguf::{BF16, F16, F32, Gguf, IQ2_XXS, MXFP4, Q2_K, Q4_K, Q8_0, Tensor as GgufTensor};
use super::validation::{DsparkConfig, SupportKind, dspark_config};
use super::{Model, ModelFamily, Rng, exact_delta_sample};
use crate::model::ModelChoice;
@@ -42,7 +44,7 @@ fn environment_present(name: &CStr) -> bool {
!unsafe { getenv(name.as_ptr()) }.is_null()
}
const SOURCES: [(&str, &str); 19] = [
const SOURCES: [(&str, &str); 22] = [
("DS4_METAL_FLASH_ATTN_SOURCE", "flash_attn.metal"),
("DS4_METAL_DENSE_SOURCE", "dense.metal"),
("DS4_METAL_MOE_SOURCE", "moe.metal"),
@@ -62,6 +64,9 @@ const SOURCES: [(&str, &str); 19] = [
("DS4_METAL_NORM_SOURCE", "norm.metal"),
("DS4_METAL_BIN_SOURCE", "bin.metal"),
("DS4_METAL_SET_ROWS_SOURCE", "set_rows.metal"),
("DS4_METAL_GLM53_BF16_SOURCE", "glm53_bf16.metal"),
("DS4_METAL_GLM53_VISION_SOURCE", "glm53_vision.metal"),
("DS4_METAL_GLM53_KDA_SOURCE", "glm53_kda.metal"),
];
// The Metal boundary uses this only to decide whether diagnostic logs get ANSI
@@ -1593,7 +1598,7 @@ impl Dspark {
}
}
struct Steering {
pub(super) struct Steering {
directions: Buffer,
attention_scale: f32,
ffn_scale: f32,
@@ -1615,6 +1620,7 @@ impl Steering {
let expected = model
.shape
.layers
.saturating_sub(model.shape.nextn)
.checked_mul(model.shape.embd as u32)
.and_then(|values| values.checked_mul(4))
.ok_or("directional steering size overflow")? as usize;
@@ -2168,7 +2174,9 @@ impl SsdPlan {
let hotlist = match model.shape.model {
ModelChoice::DeepSeekV4Flash0731 => hotlist::FLASH,
ModelChoice::DeepSeekV4Pro => hotlist::PRO,
ModelChoice::Glm52 => unreachable!("GLM uses its dedicated executor"),
ModelChoice::Glm52 | ModelChoice::Glm53Flash => {
unreachable!("GLM uses its dedicated executor")
}
};
for &(layer, expert) in hotlist {
if loaded == self.preload_experts {
@@ -4435,6 +4443,7 @@ impl Executor {
quality,
ssd,
speculative,
steering,
expert_profile_path,
)
.map(Box::new)
@@ -4481,6 +4490,24 @@ impl Executor {
}
}
pub(super) fn encode_vision(&self, encoded: &[u8]) -> Result<vision::VisionEmbedding, String> {
match self {
Self::Glm(executor) => executor.encode_vision(encoded),
Self::DeepSeek(_) => Err("vision input requires GLM 5.3 Flash".into()),
}
}
pub(super) fn set_vision_overlays(
&mut self,
overlays: Vec<(u32, vision::VisionEmbedding)>,
) -> Result<(), String> {
match self {
Self::Glm(executor) => executor.set_vision_overlays(overlays),
Self::DeepSeek(_) if overlays.is_empty() => Ok(()),
Self::DeepSeek(_) => Err("vision input requires GLM 5.3 Flash".into()),
}
}
pub(super) fn eval_speculative_greedy(
&mut self,
token: i32,
@@ -7648,7 +7675,7 @@ fn compression_ratio(shape: super::Shape, layer: u32) -> u32 {
}
crate::model::ModelChoice::DeepSeekV4Flash0731
| crate::model::ModelChoice::DeepSeekV4Pro => 128,
crate::model::ModelChoice::Glm52 => 0,
crate::model::ModelChoice::Glm52 | crate::model::ModelChoice::Glm53Flash => 0,
}
}

3721
src/engine/metal/glm.rs
View File

File diff suppressed because it is too large Load Diff

279
src/engine/metal/gpu.rs
View File

@@ -5,6 +5,42 @@ pub(super) struct GpuTensor {
_private: [u8; 0],
}
#[derive(Clone, Copy, Default)]
#[repr(C)]
pub(super) struct Glm53VisionLayerWeights {
pub(super) norm1: u64,
pub(super) qkv_weight: u64,
pub(super) qkv_bias: u64,
pub(super) q_norm: u64,
pub(super) k_norm: u64,
pub(super) attn_proj_weight: u64,
pub(super) attn_proj_bias: u64,
pub(super) norm2: u64,
pub(super) gate_weight: u64,
pub(super) gate_bias: u64,
pub(super) up_weight: u64,
pub(super) up_bias: u64,
pub(super) down_weight: u64,
pub(super) down_bias: u64,
}
#[derive(Clone, Copy, Default)]
#[repr(C)]
pub(super) struct Glm53VisionWeights {
pub(super) patch_weight: u64,
pub(super) patch_bias: u64,
pub(super) post_norm: u64,
pub(super) downsample_weight: u64,
pub(super) downsample_bias: u64,
pub(super) merger_proj: u64,
pub(super) merger_norm: u64,
pub(super) merger_norm_bias: u64,
pub(super) merger_gate: u64,
pub(super) merger_up: u64,
pub(super) merger_down: u64,
pub(super) layer: [Glm53VisionLayerWeights; 24],
}
#[repr(C)]
pub(super) struct StreamExpertTable {
pub(super) model_map: *const c_void,
@@ -93,6 +129,12 @@ unsafe extern "C" {
count: u32,
) -> i32;
pub(super) fn ds4_gpu_flush_commands() -> i32;
pub(super) fn ds4_gpu_flush_encoder() -> i32;
pub(super) fn ds4_gpu_argmax_tensor(
out: *mut GpuTensor,
logits: *const GpuTensor,
vocab: u32,
) -> i32;
pub(super) fn ds4_gpu_device_is_pre_m5_apple_silicon() -> i32;
pub(super) fn ds4_gpu_device_is_m5_apple_silicon() -> i32;
#[cfg(test)]
@@ -227,6 +269,65 @@ unsafe extern "C" {
embd: u32,
hc: u32,
) -> i32;
pub(super) fn ds4_gpu_repeat_hc_rows_tensor(
out: *mut GpuTensor,
x: *const GpuTensor,
rows: u32,
embd: u32,
hc: u32,
) -> i32;
pub(super) fn ds4_gpu_glm53_embedding_bf16(
out: *mut GpuTensor,
map: *const c_void,
size: u64,
weight: u64,
tokens: *const GpuTensor,
rows: u32,
embd: u32,
vocab: u32,
) -> i32;
pub(super) fn ds4_gpu_glm53_vision_encode(
out: *mut f32,
patches: *const f32,
grid_h: u32,
grid_w: u32,
map: *const c_void,
size: u64,
weights: *const Glm53VisionWeights,
) -> i32;
pub(super) fn ds4_gpu_glm53_scatter_image_hc(
hc: *mut GpuTensor,
image: *const GpuTensor,
dst_row: u32,
image_row: u32,
rows: u32,
total_rows: u32,
embd: u32,
hc_count: u32,
) -> i32;
pub(super) fn ds4_gpu_glm53_matmul_bf16(
out: *mut GpuTensor,
map: *const c_void,
size: u64,
weight: u64,
input: u32,
output: u32,
x: *const GpuTensor,
rows: u32,
) -> i32;
pub(super) fn ds4_gpu_glm53_matmul_bf16_qkv(
q: *mut GpuTensor,
k: *mut GpuTensor,
v: *mut GpuTensor,
map: *const c_void,
size: u64,
q_weight: u64,
k_weight: u64,
v_weight: u64,
input: u32,
output: u32,
x: *const GpuTensor,
) -> i32;
pub(super) fn ds4_gpu_attention_noncausal_raw_batch_heads_tensor(
out: *mut GpuTensor,
map: *const c_void,
@@ -417,6 +518,35 @@ unsafe extern "C" {
beta_slow: f32,
cache_f16: bool,
) -> i32;
pub(super) fn ds4_gpu_glm53_indexer_pool_update_tensor(
cache: *mut GpuTensor,
tail_k: *mut GpuTensor,
tail_gate: *mut GpuTensor,
raw_k: *const GpuTensor,
gate: *const GpuTensor,
map: *const c_void,
size: u64,
norm_weight: u64,
norm_bias: u64,
ape: u64,
pos: u32,
rows: u32,
cache_cap: u32,
head_dim: u32,
pool_size: u32,
eps: f32,
cache_f16: bool,
) -> i32;
pub(super) fn ds4_gpu_glm53_expand_pool_selection_tensor(
selected: *mut GpuTensor,
pools: *const GpuTensor,
rows: u32,
pos: u32,
selected_pools: u32,
top_k: u32,
pool_size: u32,
width: u32,
) -> i32;
pub(super) fn ds4_gpu_glm_fill_selected_range_tensor(
selected: *mut GpuTensor,
count: u32,
@@ -467,6 +597,66 @@ unsafe extern "C" {
scale: f32,
cache_f16: bool,
) -> i32;
pub(super) fn ds4_gpu_glm53_indexer_scores_batch_tensor(
scores: *mut GpuTensor,
q: *const GpuTensor,
weights: *const GpuTensor,
cache: *const GpuTensor,
visible: u32,
rows: u32,
pos: u32,
pool_size: u32,
heads: u32,
head_dim: u32,
scale: f32,
cache_f16: bool,
) -> i32;
pub(super) fn ds4_gpu_glm53_kda_decode(
out: *mut GpuTensor,
conv: *mut GpuTensor,
recurrent: *mut GpuTensor,
q: *const GpuTensor,
k: *const GpuTensor,
v: *const GpuTensor,
gate: *const GpuTensor,
beta: *const GpuTensor,
output_gate: *const GpuTensor,
map: *const c_void,
size: u64,
q_conv: u64,
k_conv: u64,
v_conv: u64,
a_log: u64,
dt_bias: u64,
output_norm: u64,
heads: u32,
rows: u32,
gate_lower_bound: f32,
eps: f32,
) -> i32;
pub(super) fn ds4_gpu_glm53_kda_prefill(
out: *mut GpuTensor,
conv: *mut GpuTensor,
recurrent: *mut GpuTensor,
q: *mut GpuTensor,
k: *mut GpuTensor,
v: *mut GpuTensor,
gate: *mut GpuTensor,
beta: *const GpuTensor,
output_gate: *const GpuTensor,
map: *const c_void,
size: u64,
q_conv: u64,
k_conv: u64,
v_conv: u64,
a_log: u64,
dt_bias: u64,
output_norm: u64,
heads: u32,
rows: u32,
gate_lower_bound: f32,
eps: f32,
) -> i32;
pub(super) fn ds4_gpu_glm_qk_lowrank_typed_tensor(
out: *mut GpuTensor,
q: *const GpuTensor,
@@ -554,6 +744,19 @@ unsafe extern "C" {
beta_fast: f32,
beta_slow: f32,
) -> i32;
pub(super) fn ds4_gpu_glm_attention_dense_compact_lora_causal_tensor(
out: *mut GpuTensor,
qk_low: *const GpuTensor,
kv_cache: *const GpuTensor,
q_row0: u32,
rows: u32,
selected: u32,
cache_cap: u32,
cache_f16: bool,
heads: u32,
kv_lora: u32,
q_nope: u32,
) -> i32;
pub(super) fn ds4_gpu_glm_attention_indexed_batch_lora_valid_tensor(
out: *mut GpuTensor,
q: *const GpuTensor,
@@ -777,6 +980,20 @@ unsafe extern "C" {
eps: f32,
norm_eps: f32,
) -> i32;
pub(super) fn ds4_gpu_hc_split_weighted_sum_tensor(
out: *mut GpuTensor,
split: *mut GpuTensor,
mix: *const GpuTensor,
residual: *const GpuTensor,
map: *const c_void,
size: u64,
scale: u64,
base: u64,
embd: u32,
hc: u32,
iterations: u32,
eps: f32,
) -> i32;
pub(super) fn ds4_gpu_hc_rms_norm_mix_f16_available() -> i32;
pub(super) fn ds4_gpu_hc_rms_norm_mix_f16_tensor(
out: *mut GpuTensor,
@@ -1359,6 +1576,15 @@ unsafe extern "C" {
embd: u32,
hc: u32,
) -> i32;
pub(super) fn ds4_gpu_hc_expand_tensor(
out: *mut GpuTensor,
block: *const GpuTensor,
residual: *const GpuTensor,
post: *const GpuTensor,
combine: *const GpuTensor,
embd: u32,
hc: u32,
) -> i32;
pub(super) fn ds4_gpu_hc_expand_add_split_tensor(
out: *mut GpuTensor,
block: *const GpuTensor,
@@ -1463,6 +1689,44 @@ unsafe extern "C" {
x: *const GpuTensor,
clamp: f32,
) -> i32;
pub(super) fn ds4_gpu_shared_mid_swiglu_q8_0_tensor(
mid: *mut GpuTensor,
map: *const c_void,
size: u64,
gate_weight: u64,
up_weight: u64,
input: u64,
output: u64,
x: *const GpuTensor,
clamp: f32,
) -> i32;
pub(super) fn ds4_gpu_shared_gate_up_swiglu_q8_0_model_view_tensor(
gate: *mut GpuTensor,
up: *mut GpuTensor,
mid: *mut GpuTensor,
map: *const c_void,
size: u64,
gate_weight: u64,
up_weight: u64,
input: u64,
output: u64,
x: *const GpuTensor,
clamp: f32,
) -> i32;
pub(super) fn ds4_gpu_shared_gate_up_swiglu_q8_0_rows_tensor(
gate: *mut GpuTensor,
up: *mut GpuTensor,
mid: *mut GpuTensor,
map: *const c_void,
size: u64,
gate_weight: u64,
up_weight: u64,
input: u64,
output: u64,
x: *const GpuTensor,
rows: u64,
clamp: f32,
) -> i32;
pub(super) fn ds4_gpu_router_shared_gate_up_q8_0_tensor(
router_logits: *mut GpuTensor,
gate: *mut GpuTensor,
@@ -1598,6 +1862,21 @@ impl Context {
return Err(error);
}
}
if let Some(vision) = &model.vision {
let mapped = unsafe {
ds4_gpu_set_model_map_range(
vision.map_ptr().cast(),
vision.len(),
vision.data_offset(),
vision.len() - vision.data_offset(),
vision.max_tensor_bytes(),
)
};
if let Err(error) = check(mapped, "vision-model mapping") {
unsafe { ds4_gpu_cleanup() };
return Err(error);
}
}
unsafe { ds4_gpu_set_quality(quality) };
let model_file = File::open(model.main.path()).map_err(|error| {
unsafe { ds4_gpu_cleanup() };

311
src/engine/metal/vision.rs Normal file
View File

@@ -0,0 +1,311 @@
use super::gpu::{Glm53VisionLayerWeights, Glm53VisionWeights, ds4_gpu_glm53_vision_encode};
use super::{Gguf, call};
use image::{DynamicImage, ImageDecoder, ImageReader};
use std::io::Cursor;
const EMBEDDING: usize = 4096;
const PATCH: u32 = 14;
const MERGE: u32 = 2;
const MIN_TOKENS: u32 = 16;
const MAX_TOKENS: u32 = 8000;
pub(super) struct VisionEncoder {
weights: Glm53VisionWeights,
}
pub(in crate::engine) struct VisionEmbedding {
pub(in crate::engine) values: Vec<f32>,
pub(in crate::engine) tokens: u32,
pub(in crate::engine) width: u32,
pub(in crate::engine) height: u32,
pub(in crate::engine) content_width: u32,
pub(in crate::engine) content_height: u32,
}
impl VisionEncoder {
pub(super) fn bind(model: &Gguf) -> Result<Self, String> {
let offset = |name: &str| model.tensor(name).map(|tensor| tensor.offset);
let mut weights = Glm53VisionWeights {
patch_weight: offset("model.visual.patch_embed.proj.weight")?,
patch_bias: offset("model.visual.patch_embed.proj.bias")?,
post_norm: offset("model.visual.post_layernorm.weight")?,
downsample_weight: offset("model.visual.downsample.weight")?,
downsample_bias: offset("model.visual.downsample.bias")?,
merger_proj: offset("model.visual.merger.proj.weight")?,
merger_norm: offset("model.visual.merger.post_projection_norm.weight")?,
merger_norm_bias: offset("model.visual.merger.post_projection_norm.bias")?,
merger_gate: offset("model.visual.merger.gate_proj.weight")?,
merger_up: offset("model.visual.merger.up_proj.weight")?,
merger_down: offset("model.visual.merger.down_proj.weight")?,
..Glm53VisionWeights::default()
};
for (layer, target) in weights.layer.iter_mut().enumerate() {
let name = |suffix: &str| format!("model.visual.blocks.{layer}.{suffix}");
*target = Glm53VisionLayerWeights {
norm1: offset(&name("norm1.weight"))?,
qkv_weight: offset(&name("attn.qkv.weight"))?,
qkv_bias: offset(&name("attn.qkv.bias"))?,
q_norm: offset(&name("attn.q_norm.weight"))?,
k_norm: offset(&name("attn.k_norm.weight"))?,
attn_proj_weight: offset(&name("attn.proj.weight"))?,
attn_proj_bias: offset(&name("attn.proj.bias"))?,
norm2: offset(&name("norm2.weight"))?,
gate_weight: offset(&name("mlp.gate_proj.weight"))?,
gate_bias: offset(&name("mlp.gate_proj.bias"))?,
up_weight: offset(&name("mlp.up_proj.weight"))?,
up_bias: offset(&name("mlp.up_proj.bias"))?,
down_weight: offset(&name("mlp.down_proj.weight"))?,
down_bias: offset(&name("mlp.down_proj.bias"))?,
};
}
Ok(Self { weights })
}
pub(super) fn encode(&self, model: &Gguf, encoded: &[u8]) -> Result<VisionEmbedding, String> {
if encoded.is_empty() || encoded.len() > 64 * 1024 * 1024 {
return Err("image is empty or exceeds the 64 MiB encoded limit".into());
}
let reader = ImageReader::new(Cursor::new(encoded))
.with_guessed_format()
.map_err(|error| error.to_string())?;
let mut decoder = reader.into_decoder().map_err(|error| error.to_string())?;
let orientation = decoder.orientation().map_err(|error| error.to_string())?;
let mut image = DynamicImage::from_decoder(decoder).map_err(|error| error.to_string())?;
image.apply_orientation(orientation);
let rgb = image.into_rgb8();
let (width, height) = rgb.dimensions();
if width == 0
|| height == 0
|| width > 16_384
|| height > 16_384
|| u64::from(width) * u64::from(height) > 64 * 1024 * 1024
{
return Err("image dimensions exceed the GLM 5.3 vision limits".into());
}
let patches = preprocess(rgb.as_raw(), width, height)?;
let tokens = patches.grid_height * patches.grid_width / 4;
let mut values = vec![0.0_f32; tokens as usize * EMBEDDING];
call(
unsafe {
ds4_gpu_glm53_vision_encode(
values.as_mut_ptr(),
patches.values.as_ptr(),
patches.grid_height,
patches.grid_width,
model.map_ptr().cast(),
model.len(),
&self.weights,
)
},
"encoding a GLM 5.3 image",
)?;
Ok(VisionEmbedding {
values,
tokens,
width,
height,
content_width: patches.content_width,
content_height: patches.content_height,
})
}
}
struct Patches {
values: Vec<f32>,
content_width: u32,
content_height: u32,
grid_width: u32,
grid_height: u32,
}
fn preprocess(rgb: &[u8], width: u32, height: u32) -> Result<Patches, String> {
const MEAN: [f32; 3] = [0.48145466, 0.4578275, 0.40821073];
const STDDEV: [f32; 3] = [0.26862954, 0.261_302_6, 0.275_777_1];
let (target_height, target_width) = smart_resize(height, width)?;
let mut scale = (target_height as f64 / height as f64).min(target_width as f64 / width as f64);
if 2 * u64::from(height) * u64::from(width) >= 2 * 28 * 28 * MIN_TOKENS as u64 && scale > 1.0 {
scale = 1.0;
}
let content_height = ((height as f64 * scale).floor() as u32).clamp(1, target_height);
let content_width = ((width as f64 * scale).floor() as u32).clamp(1, target_width);
let mut canvas = vec![0.0_f32; target_height as usize * target_width as usize * 3];
if content_width == width && content_height == height {
for y in 0..height {
for x in 0..width {
let source = (y as usize * width as usize + x as usize) * 3;
let target = (y as usize * target_width as usize + x as usize) * 3;
for channel in 0..3 {
canvas[target + channel] = f32::from(rgb[source + channel]);
}
}
}
} else {
resize_bicubic(
rgb,
width,
height,
&mut canvas,
content_width,
content_height,
target_width,
);
}
for y in 0..target_height {
for x in 0..target_width {
let pixel = (y as usize * target_width as usize + x as usize) * 3;
for channel in 0..3 {
let value = if x < content_width && y < content_height {
canvas[pixel + channel]
} else {
0.0
};
canvas[pixel + channel] = (value / 255.0 - MEAN[channel]) / STDDEV[channel];
}
}
}
let grid_height = target_height / PATCH;
let grid_width = target_width / PATCH;
let patch_values = grid_height as usize * grid_width as usize * 3 * 2 * 14 * 14;
let mut values = Vec::with_capacity(patch_values);
for block_y in 0..grid_height / MERGE {
for block_x in 0..grid_width / MERGE {
for merge_y in 0..MERGE {
for merge_x in 0..MERGE {
let patch_y = block_y * MERGE + merge_y;
let patch_x = block_x * MERGE + merge_x;
for channel in 0..3 {
for _ in 0..2 {
for y in 0..PATCH {
for x in 0..PATCH {
let pixel = ((patch_y * PATCH + y) as usize
* target_width as usize
+ (patch_x * PATCH + x) as usize)
* 3;
values.push(canvas[pixel + channel]);
}
}
}
}
}
}
}
}
if values.len() != patch_values {
return Err("internal GLM 5.3 vision patch layout mismatch".into());
}
Ok(Patches {
values,
content_width,
content_height,
grid_width,
grid_height,
})
}
fn smart_resize(height: u32, width: u32) -> Result<(u32, u32), String> {
let factor = 28_u32;
let align = |value: u32| value.div_ceil(factor) * factor;
let pixels_per_token = 2_u64 * factor as u64 * factor as u64;
let min_pixels = MIN_TOKENS as u64 * pixels_per_token;
let max_pixels = MAX_TOKENS as u64 * pixels_per_token;
let mut aligned_height = align(height);
let mut aligned_width = align(width);
let mut budget = 2_u64 * aligned_height as u64 * aligned_width as u64;
if budget < min_pixels {
let scale = (min_pixels as f64 / (2.0 * height as f64 * width as f64)).sqrt();
aligned_height = align((height as f64 * scale).ceil() as u32);
aligned_width = align((width as f64 * scale).ceil() as u32);
budget = 2_u64 * aligned_height as u64 * aligned_width as u64;
}
if budget > max_pixels {
let (mut low, mut high) = (1_u32, height);
aligned_height = factor;
aligned_width = factor;
while low <= high {
let content_height = low + (high - low) / 2;
let content_width =
((width as f64 * content_height as f64 / height as f64).floor() as u32).max(1);
let candidate_height = align(content_height);
let candidate_width = align(content_width);
if 2_u64 * candidate_height as u64 * candidate_width as u64 <= max_pixels {
aligned_height = candidate_height;
aligned_width = candidate_width;
low = content_height + 1;
} else {
high = content_height - 1;
}
}
}
Ok((aligned_height, aligned_width))
}
#[allow(clippy::too_many_arguments)]
fn resize_bicubic(
source: &[u8],
source_width: u32,
source_height: u32,
target: &mut [f32],
target_width: u32,
target_height: u32,
target_stride: u32,
) {
let scale_x = source_width as f64 / target_width as f64;
let scale_y = source_height as f64 / target_height as f64;
let filter_x = if scale_x >= 1.0 { 1.0 / scale_x } else { 1.0 };
let filter_y = if scale_y >= 1.0 { 1.0 / scale_y } else { 1.0 };
let support_x = if scale_x >= 1.0 { 2.0 * scale_x } else { 2.0 };
let support_y = if scale_y >= 1.0 { 2.0 * scale_y } else { 2.0 };
for dy in 0..target_height {
let center_y = scale_y * (dy as f64 + 0.5);
let y0 = (center_y - support_y + 0.5).max(0.0) as u32;
let y1 = (center_y + support_y + 0.5).min(source_height as f64) as u32;
for dx in 0..target_width {
let center_x = scale_x * (dx as f64 + 0.5);
let x0 = (center_x - support_x + 0.5).max(0.0) as u32;
let x1 = (center_x + support_x + 0.5).min(source_width as f64) as u32;
let mut sum = [0.0; 3];
let mut weight_sum = 0.0;
for iy in y0..y1 {
let wy = cubic((iy as f64 + 0.5 - center_y) * filter_y);
for ix in x0..x1 {
let weight = wy * cubic((ix as f64 + 0.5 - center_x) * filter_x);
let pixel = (iy as usize * source_width as usize + ix as usize) * 3;
for channel in 0..3 {
sum[channel] += source[pixel + channel] as f64 * weight;
}
weight_sum += weight;
}
}
let pixel = (dy as usize * target_stride as usize + dx as usize) * 3;
for channel in 0..3 {
target[pixel + channel] =
(sum[channel] / weight_sum).round().clamp(0.0, 255.0) as f32;
}
}
}
}
fn cubic(mut x: f64) -> f64 {
const A: f64 = -0.5;
x = x.abs();
if x < 1.0 {
((A + 2.0) * x - (A + 3.0)) * x * x + 1.0
} else if x < 2.0 {
((A * x - 5.0 * A) * x + 8.0 * A) * x - 4.0 * A
} else {
0.0
}
}
#[cfg(test)]
mod tests {
use super::smart_resize;
#[test]
fn glm53_resize_matches_reference_token_grids() {
assert_eq!(smart_resize(28, 28), Ok((112, 112)));
assert_eq!(smart_resize(1024, 1024), Ok((1036, 1036)));
assert_eq!(smart_resize(1080, 1920), Ok((1092, 1932)));
}
}

21
src/engine/tokenizer.rs
View File

@@ -1,5 +1,8 @@
use super::gguf::Gguf;
use super::{ChatTurn, ModelFamily};
use super::{
ChatTurn, ModelFamily, VISION_END_TOKEN, VISION_IMAGE_TOKEN, VISION_START_TOKEN,
VISION_TOKEN_END, VISION_TOKEN_START,
};
use crate::settings::ReasoningMode;
use std::collections::HashMap;
@@ -160,6 +163,22 @@ impl Tokenizer {
let mut span = 0;
let mut position = 0;
while position < bytes.len() {
if bytes[position..].starts_with(VISION_TOKEN_START.as_bytes())
&& let Some(relative_end) =
text[position + VISION_TOKEN_START.len()..].find(VISION_TOKEN_END)
{
let count_start = position + VISION_TOKEN_START.len();
let count_end = count_start + relative_end;
if let Ok(count) = text[count_start..count_end].parse::<usize>() {
self.tokenize_plain(&text[span..position], &mut output);
output.push(VISION_START_TOKEN);
output.extend(std::iter::repeat_n(VISION_IMAGE_TOKEN, count));
output.push(VISION_END_TOKEN);
position = count_end + VISION_TOKEN_END.len();
span = position;
continue;
}
}
let special = self
.rendered_specials
.iter()

196
src/engine/validation.rs
View File

@@ -9,7 +9,7 @@ pub(crate) fn validate_model_artifact(
let model = Gguf::open(path)?;
let shape = match expected {
ModelChoice::DeepSeekV4Flash0731 => FLASH_0731,
ModelChoice::DeepSeekV4Pro | ModelChoice::Glm52 => {
ModelChoice::DeepSeekV4Pro | ModelChoice::Glm52 | ModelChoice::Glm53Flash => {
return Err(format!("{expected} does not use an external support GGUF"));
}
};
@@ -35,6 +35,79 @@ pub(crate) fn validate_model_artifact(
}
}
pub(crate) fn validate_vision_artifact(path: &Path) -> Result<(), String> {
let model = Gguf::open(path)?;
if model.bytes("general.architecture")? != b"glm5-next-vision" {
return Err("vision GGUF architecture is not glm5-next-vision".into());
}
if model.tensors.len() != 347 {
return Err(format!(
"vision GGUF has {} tensors, expected 347",
model.tensors.len()
));
}
for (key, expected) in [
("block_count", 24),
("embedding_length", 1024),
("feed_forward_length", 4096),
("attention.head_count", 16),
("projection_length", 4096),
("projection.feed_forward_length", 10_240),
("patch_size", 14),
("temporal_patch_size", 2),
("spatial_merge_size", 2),
("image_token_id", VISION_IMAGE_TOKEN as u64),
("image_start_token_id", VISION_START_TOKEN as u64),
("image_end_token_id", VISION_END_TOKEN as u64),
] {
expect_u64(&model, &format!("glm5-next-vision.{key}"), expected)?;
}
let bf16 = &[BF16];
for (name, dims) in [
(
"model.visual.patch_embed.proj.weight",
vec![14, 14, 2, 3, 1024],
),
("model.visual.patch_embed.proj.bias", vec![1024]),
("model.visual.post_layernorm.weight", vec![1024]),
("model.visual.downsample.weight", vec![2, 2, 1024, 4096]),
("model.visual.downsample.bias", vec![4096]),
("model.visual.merger.proj.weight", vec![4096, 4096]),
(
"model.visual.merger.post_projection_norm.weight",
vec![4096],
),
("model.visual.merger.post_projection_norm.bias", vec![4096]),
("model.visual.merger.gate_proj.weight", vec![4096, 10_240]),
("model.visual.merger.up_proj.weight", vec![4096, 10_240]),
("model.visual.merger.down_proj.weight", vec![10_240, 4096]),
] {
expect(&model, name, bf16, &dims)?;
}
for layer in 0..24 {
let name = |suffix: &str| format!("model.visual.blocks.{layer}.{suffix}");
for (suffix, dims) in [
("norm1.weight", vec![1024]),
("attn.qkv.weight", vec![1024, 3072]),
("attn.qkv.bias", vec![3072]),
("attn.q_norm.weight", vec![64]),
("attn.k_norm.weight", vec![64]),
("attn.proj.weight", vec![1024, 1024]),
("attn.proj.bias", vec![1024]),
("norm2.weight", vec![1024]),
("mlp.gate_proj.weight", vec![1024, 4096]),
("mlp.gate_proj.bias", vec![4096]),
("mlp.up_proj.weight", vec![1024, 4096]),
("mlp.up_proj.bias", vec![4096]),
("mlp.down_proj.weight", vec![4096, 1024]),
("mlp.down_proj.bias", vec![1024]),
] {
expect(&model, &name(suffix), bf16, &dims)?;
}
}
Ok(())
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub(super) enum SupportKind {
DSpark,
@@ -116,14 +189,11 @@ pub(super) fn validate_support(model: &Gguf, shape: &Shape) -> Result<SupportKin
}
pub(super) fn validate_main(model: &Gguf, expected: ModelChoice) -> Result<Shape, String> {
let family = if model.bytes("general.architecture").ok() == Some(b"glm-dsa") {
ModelFamily::Glm
} else {
ModelFamily::DeepSeek
};
let shape = match family {
ModelFamily::Glm => GLM,
ModelFamily::DeepSeek => match (model.u32("deepseek4.block_count")?, expected) {
let architecture = model.bytes("general.architecture")?;
let shape = match architecture {
b"glm-dsa" => GLM,
b"glm5-next" => GLM53_FLASH,
_ => match (model.u32("deepseek4.block_count")?, expected) {
(43, ModelChoice::DeepSeekV4Flash0731) => FLASH_0731,
(43, _) => FLASH_0731,
(61, _) => PRO,
@@ -143,6 +213,9 @@ pub(super) fn validate_main(model: &Gguf, expected: ModelChoice) -> Result<Shape
}
fn validate_metadata(model: &Gguf, shape: &Shape) -> Result<(), String> {
if shape.model == ModelChoice::Glm53Flash {
return validate_glm53_metadata(model, shape);
}
let prefix = if shape.family == ModelFamily::Glm {
"glm-dsa"
} else {
@@ -266,13 +339,102 @@ fn validate_metadata(model: &Gguf, shape: &Shape) -> Result<(), String> {
Ok(())
}
fn validate_glm53_metadata(model: &Gguf, shape: &Shape) -> Result<(), String> {
let prefix = "glm5-next";
for (key, expected) in [
("block_count", u64::from(shape.layers)),
("trunk_block_count", u64::from(shape.layers - shape.nextn)),
("nextn_predict_layers", u64::from(shape.nextn)),
("context_length", shape.original_context),
("embedding_length", shape.embd),
("vocab_size", shape.vocab),
("feed_forward_length", shape.ff_dense),
("expert_feed_forward_length", shape.ff_expert),
("expert_count", shape.experts),
("expert_used_count", shape.experts_used),
("expert_shared_count", shape.expert_shared),
("leading_dense_block_count", u64::from(shape.leading_dense)),
("attention.head_count", shape.heads),
("attention.key_length", shape.key_mla),
("attention.value_length", shape.value_mla),
("attention.q_lora_rank", shape.lora_q),
("attention.kv_lora_rank", shape.kv_lora),
("attention.rope_dimension_count", shape.rot),
("attention.indexer.head_count", shape.indexer_heads),
("attention.indexer.key_length", shape.indexer_head_dim),
("attention.indexer.top_k", shape.indexer_top_k),
("attention.indexer.pool_size", 4),
("linear_attention.head_count", 64),
("linear_attention.head_dimension", 128),
("linear_attention.conv_kernel", 4),
("hyper_connection.count", shape.hc),
("hyper_connection.sinkhorn_iterations", shape.hc_sinkhorn),
] {
expect_u64(model, &format!("{prefix}.{key}"), expected)?;
}
for (key, expected) in [
("expert_weights_scale", shape.expert_weight_scale),
("swiglu_limit", shape.swiglu_clamp),
("attention.layer_norm_rms_epsilon", shape.rms_epsilon),
("linear_attention.gate_lower_bound", -5.0),
("hyper_connection.epsilon", shape.hc_epsilon),
] {
expect_float(model, &format!("{prefix}.{key}"), expected)?;
}
if !model.boolean("glm5-next.expert_weights_norm")? {
return Err("glm5-next.expert_weights_norm must be true".into());
}
let layer_types = model.u32s("glm5-next.layer_types")?;
if layer_types.len() != shape.layers as usize {
return Err("glm5-next.layer_types must contain one entry per layer".into());
}
for (layer, &kind) in layer_types.iter().enumerate() {
let expected =
u32::from(layer + shape.nextn as usize >= shape.layers as usize || layer % 4 == 3);
if kind != expected {
return Err(format!(
"unexpected GLM 5.3 attention type at layer {layer}"
));
}
}
Ok(())
}
fn validate_tensors(model: &Gguf, shape: &Shape) -> Result<(), String> {
match shape.family {
ModelFamily::DeepSeek => validate_deepseek_tensors(model, shape),
ModelFamily::Glm if shape.model == ModelChoice::Glm53Flash => {
validate_glm53_tensors(model, shape)
}
ModelFamily::Glm => validate_glm_tensors(model, shape),
}
}
fn validate_glm53_tensors(model: &Gguf, shape: &Shape) -> Result<(), String> {
expect(
model,
"token_embd.weight",
DENSE,
&[shape.embd, shape.vocab],
)?;
expect(model, "output_norm.weight", &[F32], &[shape.embd])?;
expect(model, "output.weight", DENSE, &[shape.embd, shape.vocab])?;
expect(model, "blk.0.kda_q.weight", DENSE, &[shape.embd, 8192])?;
expect(
model,
"blk.3.attn_q_a.weight",
DENSE,
&[shape.embd, shape.lora_q],
)?;
expect(
model,
"blk.45.nextn.eh_proj.weight",
DENSE,
&[2 * shape.embd, shape.embd],
)?;
Ok(())
}
fn validate_deepseek_tensors(model: &Gguf, shape: &Shape) -> Result<(), String> {
let hc_dim = shape.embd * shape.hc;
let hc_mix = 2 * shape.hc + shape.hc * shape.hc;
@@ -902,7 +1064,7 @@ fn compression_ratio(shape: &Shape, layer: u32) -> u32 {
4
}
ModelChoice::DeepSeekV4Flash0731 | ModelChoice::DeepSeekV4Pro => 128,
ModelChoice::Glm52 => 0,
ModelChoice::Glm52 | ModelChoice::Glm53Flash => 0,
}
}
@@ -1157,4 +1319,18 @@ mod tests {
validate_model_artifact(&path, ModelChoice::DeepSeekV4Flash0731, true).unwrap();
}
}
#[test]
fn configured_glm53_vision_fixture_passes_the_exact_layout() {
if let Some(path) = std::env::var_os("DS4SERVER_GLM53_VISION") {
validate_vision_artifact(Path::new(&path)).unwrap();
}
}
#[test]
fn configured_glm53_main_fixture_passes_the_exact_layout() {
if let Some(path) = std::env::var_os("DS4SERVER_GLM53_MODEL") {
validate_model_artifact(Path::new(&path), ModelChoice::Glm53Flash, false).unwrap();
}
}
}

2
src/metrics.rs
View File

@@ -847,6 +847,7 @@ fn model_code(model: ModelChoice) -> u8 {
ModelChoice::DeepSeekV4Pro => 2,
ModelChoice::Glm52 => 3,
ModelChoice::DeepSeekV4Flash0731 => 4,
ModelChoice::Glm53Flash => 5,
}
}
@@ -856,6 +857,7 @@ fn model_name(value: u8) -> &'static str {
2 => "DeepSeek V4 Pro",
3 => "GLM 5.2",
4 => "DeepSeek V4 Flash 0731",
5 => "GLM 5.3 Flash Q2",
_ => "No model loaded",
}
}

69
src/model.rs
View File

@@ -9,20 +9,24 @@ use std::fmt;
use std::fs;
use std::path::{Path, PathBuf};
pub(crate) const MODEL_CHOICES: [ModelChoice; 3] = [
pub(crate) const MODEL_CHOICES: [ModelChoice; 4] = [
ModelChoice::DeepSeekV4Flash0731,
ModelChoice::DeepSeekV4Pro,
ModelChoice::Glm52,
ModelChoice::Glm53Flash,
];
pub(crate) const MANAGED_ARTIFACTS: [ManagedArtifactId; 4] = [
pub(crate) const MANAGED_ARTIFACTS: [ManagedArtifactId; 6] = [
ManagedArtifactId::DeepSeekV4Flash0731,
ManagedArtifactId::DeepSeekV4Flash0731Dspark,
ManagedArtifactId::DeepSeekV4Pro,
ManagedArtifactId::Glm52,
ManagedArtifactId::Glm53Flash,
ManagedArtifactId::Glm53FlashVision,
];
const DEEPSEEK_REPOSITORY: &str = "antirez/deepseek-v4-gguf";
const GLM_REPOSITORY: &str = "antirez/glm-5.2-gguf";
const GLM53_REPOSITORY: &str = "antirez/glm-5.3-flash-gguf";
const FLASH_0731: Artifact = Artifact {
label: "DeepSeek V4 Flash 0731 model",
@@ -31,6 +35,7 @@ const FLASH_0731: Artifact = Artifact {
size: 86_720_111_488,
sha256: "ca22ae2f838e14077c22bc1c1417b71b45b5e5a3687bd96c2ac6e17fdb6261c0",
support: Some(false),
vision: false,
};
const FLASH_0731_DSPARK: Artifact = Artifact {
label: "DeepSeek V4 Flash 0731 DSpark support",
@@ -39,6 +44,7 @@ const FLASH_0731_DSPARK: Artifact = Artifact {
size: 5_989_114_272,
sha256: "7e319924541db3f7a163ed7e11d7532a70d48228ab59d36cb81e1d4511885360",
support: Some(true),
vision: false,
};
const PRO: Artifact = Artifact {
label: "DeepSeek V4 Pro 0813 model",
@@ -47,6 +53,7 @@ const PRO: Artifact = Artifact {
size: 464_627_334_560,
sha256: "c4d997ab9894b6c78b759f7869fe1726b6314b6515f6ff82607df3797c5eb193",
support: Some(false),
vision: false,
};
const GLM: Artifact = Artifact {
label: "GLM 5.2 model",
@@ -55,6 +62,25 @@ const GLM: Artifact = Artifact {
size: 211_075_856_448,
sha256: "a49de64c5020432bdae23de36a423a9660a5621bc0db8d12b66bd8814b07fea0",
support: Some(false),
vision: false,
};
const GLM53_FLASH: Artifact = Artifact {
label: "GLM 5.3 Flash Q2 model",
file_name: "GLM-5.3-Flash-Q2.gguf",
repository: GLM53_REPOSITORY,
size: 96_505_816_384,
sha256: "e81fd6241c6e55a64e1e14e47a3eab61a173fa8d7e4b5c1d1848827119705b32",
support: Some(false),
vision: false,
};
const GLM53_FLASH_VISION: Artifact = Artifact {
label: "GLM 5.3 Flash vision encoder",
file_name: "GLM-5.3-Flash-Vision-Encoder.gguf",
repository: GLM53_REPOSITORY,
size: 1_127_280_960,
sha256: "ae23e14c6979e889051b2e4a39351abcdafb161e18e606fae4d8c40095a4bf3a",
support: None,
vision: true,
};
#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, Ord, PartialEq, PartialOrd, Serialize)]
@@ -66,6 +92,8 @@ pub(crate) enum ModelChoice {
DeepSeekV4Pro,
#[serde(rename = "glm-5.2")]
Glm52,
#[serde(rename = "glm-5.3-flash")]
Glm53Flash,
}
impl ModelChoice {
@@ -74,6 +102,7 @@ impl ModelChoice {
Self::DeepSeekV4Flash0731 => "deepseek-v4-flash-0731",
Self::DeepSeekV4Pro => "deepseek-v4-pro",
Self::Glm52 => "glm-5.2",
Self::Glm53Flash => "glm-5.3-flash",
}
}
@@ -85,26 +114,40 @@ impl ModelChoice {
self == Self::DeepSeekV4Flash0731
}
pub(crate) fn is_glm(self) -> bool {
matches!(self, Self::Glm52 | Self::Glm53Flash)
}
pub(crate) fn supports_glm_mtp(self) -> bool {
self.is_glm()
}
fn main_artifact(self) -> &'static Artifact {
match self {
Self::DeepSeekV4Flash0731 => &FLASH_0731,
Self::DeepSeekV4Pro => &PRO,
Self::Glm52 => &GLM,
Self::Glm53Flash => &GLM53_FLASH,
}
}
fn dspark_artifact(self) -> Option<&'static Artifact> {
match self {
Self::DeepSeekV4Flash0731 => Some(&FLASH_0731_DSPARK),
Self::DeepSeekV4Pro | Self::Glm52 => None,
Self::DeepSeekV4Pro | Self::Glm52 | Self::Glm53Flash => None,
}
}
fn vision_artifact(self) -> Option<&'static Artifact> {
(self == Self::Glm53Flash).then_some(&GLM53_FLASH_VISION)
}
#[cfg(test)]
fn artifacts(self, dspark_enabled: bool) -> impl Iterator<Item = &'static Artifact> {
[
Some(self.main_artifact()),
dspark_enabled.then(|| self.dspark_artifact()).flatten(),
self.vision_artifact(),
]
.into_iter()
.flatten()
@@ -115,6 +158,7 @@ impl ModelChoice {
pub(crate) struct EngineArtifacts {
pub(crate) model: PathBuf,
pub(crate) support: Option<PathBuf>,
pub(crate) vision: Option<PathBuf>,
}
pub(crate) fn engine_artifacts(
@@ -131,6 +175,11 @@ pub(crate) fn engine_artifacts(
} else {
None
},
vision: model.vision_artifact().and_then(|artifact| {
artifact
.is_installed(model, models_path)
.then(|| artifact.path(model, models_path))
}),
}
}
@@ -158,7 +207,14 @@ pub(crate) fn validate_engine_artifacts(
"{} is not compatible with the selected {model} checkpoint",
path.display()
)),
}?;
if let Some(path) = artifacts.vision.as_deref() {
model
.vision_artifact()
.ok_or_else(|| format!("{} is not compatible with {model}", path.display()))?
.validate_installed_path(path)?;
}
Ok(())
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
@@ -167,6 +223,8 @@ pub(crate) enum ManagedArtifactId {
DeepSeekV4Flash0731Dspark,
DeepSeekV4Pro,
Glm52,
Glm53Flash,
Glm53FlashVision,
}
impl ManagedArtifactId {
@@ -177,6 +235,7 @@ impl ManagedArtifactId {
}
Self::DeepSeekV4Pro => ModelChoice::DeepSeekV4Pro,
Self::Glm52 => ModelChoice::Glm52,
Self::Glm53Flash | Self::Glm53FlashVision => ModelChoice::Glm53Flash,
}
}
@@ -186,6 +245,8 @@ impl ManagedArtifactId {
Self::DeepSeekV4Flash0731Dspark => &FLASH_0731_DSPARK,
Self::DeepSeekV4Pro => &PRO,
Self::Glm52 => &GLM,
Self::Glm53Flash => &GLM53_FLASH,
Self::Glm53FlashVision => &GLM53_FLASH_VISION,
}
}
}
@@ -280,6 +341,7 @@ impl fmt::Display for ModelChoice {
Self::DeepSeekV4Flash0731 => "DeepSeek V4 Flash 0731",
Self::DeepSeekV4Pro => "DeepSeek V4 Pro 0813",
Self::Glm52 => "GLM 5.2",
Self::Glm53Flash => "GLM 5.3 Flash Q2",
})
}
}
@@ -291,6 +353,7 @@ struct Artifact {
size: u64,
sha256: &'static str,
support: Option<bool>,
vision: bool,
}
impl Artifact {

14
src/model/transfer.rs
View File

@@ -205,7 +205,9 @@ fn verify(
path.display()
));
}
if let Some(support) = artifact.support {
if artifact.vision {
crate::engine::validate_vision_artifact(path)?;
} else if let Some(support) = artifact.support {
crate::engine::validate_model_artifact(path, model, support)?;
}
Ok(DownloadOutcome::Complete)
@@ -330,8 +332,8 @@ mod tests {
Some(ModelChoice::DeepSeekV4Flash0731)
);
assert!(ModelChoice::from_id("unknown").is_none());
assert_eq!(MODEL_CHOICES.len(), 3);
assert_eq!(MANAGED_ARTIFACTS.len(), 4);
assert_eq!(MODEL_CHOICES.len(), 4);
assert_eq!(MANAGED_ARTIFACTS.len(), 6);
assert_eq!(ModelChoice::Glm52.main_artifact().size, 211_075_856_448);
assert_eq!(
ModelChoice::DeepSeekV4Flash0731.main_artifact().size,
@@ -339,6 +341,7 @@ mod tests {
);
assert_eq!(ModelChoice::DeepSeekV4Flash0731.artifacts(true).count(), 2);
assert_eq!(ModelChoice::Glm52.artifacts(true).count(), 1);
assert_eq!(ModelChoice::Glm53Flash.artifacts(true).count(), 2);
let id = SystemTime::now()
.duration_since(UNIX_EPOCH)
@@ -366,6 +369,7 @@ mod tests {
size: 0,
sha256: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
support: None,
vision: false,
};
let partial = empty.partial_path(ModelChoice::DeepSeekV4Flash0731, &models_path);
fs::create_dir_all(partial.parent().unwrap()).unwrap();
@@ -393,6 +397,7 @@ mod tests {
&EngineArtifacts {
model: installed.clone(),
support: None,
vision: None,
},
)
.is_err()
@@ -404,6 +409,7 @@ mod tests {
&EngineArtifacts {
model: installed,
support: None,
vision: None,
},
)
.is_err()
@@ -428,6 +434,7 @@ mod tests {
size: 3,
sha256: "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad",
support: None,
vision: false,
};
let verified_bytes = AtomicU64::new(999);
@@ -593,6 +600,7 @@ mod tests {
size: 10,
sha256: "unused",
support: None,
vision: false,
};
let partial = artifact.partial_path(ModelChoice::DeepSeekV4Flash0731, &directory);
fs::create_dir_all(partial.parent().unwrap()).unwrap();

13
src/server.rs
View File

@@ -554,6 +554,12 @@ fn model_alias(id: &str) -> Option<ModelChoice> {
| "zai/glm-5.2"
| "zai/glm-5.2-chat"
| "zai/glm-5.2-reasoner" => Some(ModelChoice::Glm52),
"glm-5.3-flash"
| "glm-5.3-flash-chat"
| "glm-5.3-flash-reasoner"
| "zai/glm-5.3-flash"
| "zai/glm-5.3-flash-chat"
| "zai/glm-5.3-flash-reasoner" => Some(ModelChoice::Glm53Flash),
_ => ModelChoice::from_id(id),
}
}
@@ -900,6 +906,13 @@ mod tests {
content_text(&json!([{"type": "text", "text": "one"}, " two"])),
"one two"
);
assert_eq!(
content_text(&json!([{
"type": "image_url",
"image_url": {"url": "data:image/png;base64,YQ=="}
}])),
"<|ds4server_image_data|>data:image/png;base64,YQ==<|/ds4server_image_data|>"
);
}
#[test]

42
src/server/request.rs
View File

@@ -54,6 +54,23 @@ pub(super) fn anthropic_request(value: Value) -> Result<ChatRequest, (u16, Strin
for block in blocks {
match block.get("type").and_then(Value::as_str).unwrap_or("text") {
"text" => text.push_str(block.get("text").and_then(Value::as_str).unwrap_or("")),
"image" => {
let source = block.get("source").and_then(Value::as_object);
let media = source
.and_then(|source| source.get("media_type"))
.and_then(Value::as_str)
.unwrap_or("");
let data = source
.and_then(|source| source.get("data"))
.and_then(Value::as_str)
.unwrap_or("");
if !matches!(media, "image/png" | "image/jpeg") || data.is_empty() {
return Err((400, "image input must be inline PNG or JPEG base64".into()));
}
text.push_str(&crate::engine::vision_data_marker(&format!(
"data:{media};base64,{data}"
)));
}
"thinking" | "redacted_thinking" => reasoning.push_str(
block
.get("thinking")
@@ -382,6 +399,24 @@ fn responses_content_text(value: &Value) -> Result<String, (u16, String)> {
text.push_str(value);
}
}
Value::Object(part)
if part.get("type").and_then(Value::as_str) == Some("input_image") =>
{
let url = part
.get("image_url")
.or_else(|| part.get("url"))
.and_then(Value::as_str)
.ok_or_else(|| (400, "invalid image input".into()))?;
if !url.starts_with("data:image/png;base64,")
&& !url.starts_with("data:image/jpeg;base64,")
{
return Err((
400,
"image input must be an inline PNG or JPEG data URI".into(),
));
}
text.push_str(&crate::engine::vision_data_marker(url));
}
_ => return Err((400, "invalid JSON request".into())),
}
}
@@ -574,7 +609,12 @@ fn request_reasoning(
|| (explicit_thinking.is_none()
&& matches!(
model_id,
"deepseek-chat" | "glm-5.2-chat" | "glm-5.2-no-think" | "glm-5.2-nothink"
"deepseek-chat"
| "glm-5.2-chat"
| "glm-5.2-no-think"
| "glm-5.2-nothink"
| "glm-5.3-flash-chat"
| "zai/glm-5.3-flash-chat"
))
{
reasoning = ReasoningMode::Direct;

39
src/server/tools.rs
View File

@@ -365,6 +365,7 @@ pub(super) fn render_messages(
let mut turns = Vec::<ChatTurn>::new();
for message in messages {
validate_inline_images(&message.content)?;
let content = content_text(&message.content);
match message.role.as_str() {
"system" | "developer" => {
@@ -426,6 +427,30 @@ pub(super) fn render_messages(
Ok((system, turns))
}
fn validate_inline_images(value: &Value) -> Result<(), (u16, String)> {
let Some(parts) = value.as_array() else {
return Ok(());
};
for part in parts {
let Some(image) = part.get("image_url") else {
continue;
};
let url = image
.get("url")
.or(Some(image))
.and_then(Value::as_str)
.ok_or_else(|| (400, "invalid image input".into()))?;
if !url.starts_with("data:image/png;base64,") && !url.starts_with("data:image/jpeg;base64,")
{
return Err((
400,
"image input must be an inline PNG or JPEG data URI".into(),
));
}
}
Ok(())
}
pub(super) fn validate_tool_results(
state: &State,
messages: &[ApiMessage],
@@ -779,8 +804,18 @@ pub(super) fn content_text(value: &Value) -> String {
Value::Array(parts) => parts
.iter()
.filter_map(|part| match part {
Value::String(text) => Some(text.as_str()),
Value::Object(object) => object.get("text").and_then(Value::as_str),
Value::String(text) => Some(text.clone()),
Value::Object(object) => object
.get("text")
.and_then(Value::as_str)
.map(str::to_owned)
.or_else(|| {
let url = object
.get("image_url")
.and_then(|value| value.get("url").or(Some(value)))
.and_then(Value::as_str)?;
Some(crate::engine::vision_data_marker(url))
}),
_ => None,
})
.collect(),

16
src/settings.rs
View File

@@ -34,8 +34,8 @@ impl SpeculativePreferences {
if self.glm_mtp_timing && !self.glm_mtp {
return Err("GLM MTP timing requires GLM MTP.".into());
}
if model != ModelChoice::Glm52 && (self.glm_mtp || self.glm_mtp_timing) {
return Err("GLM MTP is available only for GLM 5.2.".into());
if !model.supports_glm_mtp() && (self.glm_mtp || self.glm_mtp_timing) {
return Err("GLM MTP is available only for GLM models.".into());
}
if self.dspark_enabled && !model.supports_dspark() {
return Err("DSpark is not available for the selected model.".into());
@@ -166,8 +166,8 @@ impl SsdPreferences {
{
return Err("SSD full-layer count is too large.".into());
}
if self.full_layers.is_some_and(|layers| layers > 0) && model != ModelChoice::Glm52 {
return Err("Fully resident SSD layers are available only for GLM 5.2.".into());
if self.full_layers.is_some_and(|layers| layers > 0) && !model.is_glm() {
return Err("Fully resident SSD layers are available only for GLM models.".into());
}
if let Some(StreamingCacheBudget::Gib(gib)) = self.cache {
validate_gib("SSD cache budget", gib)?;
@@ -443,12 +443,12 @@ impl ExecutionPreferences {
{
return Err("Prefill chunk is too large.".into());
}
if model == ModelChoice::Glm52 {
if model.is_glm() {
if self.power_percent.is_some_and(|power| power != 100) {
return Err("GLM 5.2 currently requires 100% GPU power.".into());
return Err("GLM currently requires 100% GPU power.".into());
}
if self.prefill_chunk.is_some() {
return Err("GLM 5.2 selects its prefill chunk automatically.".into());
return Err("GLM selects its prefill chunk automatically.".into());
}
}
Ok(())
@@ -569,7 +569,7 @@ impl GenerationPreferences {
model: ModelChoice,
kv_cache: KvCacheSettings,
) -> TurnSettings {
let glm = model == ModelChoice::Glm52;
let glm = model.is_glm();
TurnSettings {
kv_cache,
context_tokens: self.context_tokens,