hugo/DS4Server
1
0
Fork 0
Code Issues 8 Pull Requests Actions Packages Projects Releases Wiki Activity

feat: first cut at openai compatible server

Browse Source
This commit is contained in:
Georg Bauer
2026-07-24 21:12:48 +02:00
parent bbbe65a75f
commit d554b77b9d
18 changed files with 2612 additions and 535 deletions
Download Patch File Download Diff File Expand all files Collapse all files

629
src/app/view.rs
View File

@@ -31,7 +31,12 @@ impl App {
if self.model_manager_window == Some(id) {
self.model_manager()
} else {
self.main_view()
let content = self.main_view();
#[cfg(target_os = "macos")]
return crate::native_edit::native_edit(content, self.native_edit_commands.clone())
.into();
#[cfg(not(target_os = "macos"))]
content
}
}
@@ -457,286 +462,331 @@ impl App {
dspark_confidence.on_input(Message::PreferenceDsparkConfidenceChanged);
}
let mut fields = column![
text("MODEL").size(11),
pick_list(
&MODEL_CHOICES[..],
Some(self.preference_draft.model),
Message::PreferenceModelChanged,
)
.width(Length::Fill),
text(format!(
"Main: {}{}",
engine.map_or_else(
|| "Invalid settings".to_owned(),
|engine| engine.artifacts.model.display().to_string(),
),
engine
.and_then(|engine| engine.artifacts.mtp.as_ref())
.map_or_else(String::new, |path| format!(" • support: {}", path.display())),
))
.size(12),
Space::with_height(8),
text("EXECUTION").size(11),
preference_input_row(
"CPU helper threads",
text_input("Automatic", &self.preference_draft.cpu_threads)
.on_input(Message::PreferenceCpuThreadsChanged),
),
preference_input_row("GPU power percent", power),
preference_input_row("Prefill chunk", prefill),
checkbox("Prefer exact quality kernels", self.preference_draft.quality)
.on_toggle(Message::PreferenceQualityChanged),
checkbox("Warm mapped weights at load time", self.preference_draft.warm_weights)
.on_toggle(Message::PreferenceWarmWeightsChanged),
text(if self.preference_draft.model == ModelChoice::Glm52 {
"GLM 5.2 uses full GPU power and selects prefill chunks automatically."
} else {
"Blank numeric values preserve DS4's automatic engine behavior."
})
.size(12),
text(engine.as_ref().map_or_else(
|| "Effective execution settings will appear after valid values are entered."
.to_owned(),
|engine| {
let settings = engine.execution;
format!(
"Metal engine: threads {} • power {}% • prefill {} • quality {} • warm weights {}",
if settings.cpu_threads == 0 { "auto".to_owned() } else { settings.cpu_threads.to_string() },
if settings.power_percent == 0 { 100 } else { settings.power_percent },
if settings.prefill_chunk == 0 { "auto".to_owned() } else { settings.prefill_chunk.to_string() },
if settings.quality { "on" } else { "off" },
if settings.warm_weights { "on" } else { "off" },
)},
))
.size(12),
Space::with_height(8),
text("SPECULATIVE DECODING").size(11),
preference_input_row(
"MTP draft tokens",
text_input("1", &self.preference_draft.mtp_draft_tokens)
.on_input(Message::PreferenceMtpDraftChanged),
),
preference_input_row(
"MTP verifier margin",
text_input("3", &self.preference_draft.mtp_margin)
.on_input(Message::PreferenceMtpMarginChanged),
),
checkbox("Enable integrated GLM MTP", self.preference_draft.glm_mtp)
.on_toggle_maybe(glm_mtp_toggle),
checkbox(
"Log GLM MTP timing counters",
self.preference_draft.glm_mtp_timing,
)
.on_toggle_maybe(glm_mtp_timing_toggle),
dspark,
preference_input_row("DSpark confidence threshold", dspark_confidence),
checkbox(
"DSpark target-only decode",
self.preference_draft.dspark_strict,
)
.on_toggle_maybe(dspark_strict_toggle),
text(if self.preference_draft.model.supports_dspark() {
"DSpark uses the managed support artifact; entering a threshold or enabling strict mode also enables DSpark."
} else if self.preference_draft.model == ModelChoice::Glm52 {
"GLM MTP is integrated; DSpark is unavailable for this model."
} else {
"No managed MTP support artifact is available for this model."
})
.size(12),
text(engine.as_ref().map_or_else(
|| "Effective speculative settings will appear after valid values are entered."
.to_owned(),
|engine| {
let settings = engine.speculative;
format!(
"Engine: MTP draft {} • margin {} • GLM MTP {} • timing {} • DSpark {} • confidence {}{} • target-only {}",
settings.mtp_draft_tokens,
settings.mtp_margin,
if settings.glm_mtp { "on" } else { "off" },
if settings.glm_mtp_timing { "on" } else { "off" },
if settings.dspark { "on" } else { "off" },
settings.dspark_confidence_threshold,
if settings.dspark_confidence_threshold_set { " explicit" } else { " default" },
if settings.dspark_strict { "on" } else { "off" },
)},
))
.size(12),
Space::with_height(8),
text("SSD STREAMING").size(11),
checkbox("Enable SSD-backed model streaming", self.preference_draft.ssd_streaming)
.on_toggle(Message::PreferenceSsdChanged),
checkbox("Skip automatic expert preload", self.preference_draft.ssd_streaming_cold)
.on_toggle(Message::PreferenceSsdColdChanged),
preference_input_row(
"Expert cache count or GiB",
text_input("Automatic, 128, or 64GB", &self.preference_draft.ssd_cache)
.on_input(Message::PreferenceSsdCacheChanged),
),
preference_input_row("Fully resident GLM layers", ssd_full_layers),
preference_input_row(
"Explicit expert preload count",
text_input("Automatic", &self.preference_draft.ssd_preload_experts)
.on_input(Message::PreferenceSsdPreloadChanged),
),
text("A blank full-layer value is automatic; an explicit 0 disables fully resident GLM layers. SSD streaming and DSpark are mutually exclusive.")
.size(12),
text(engine.as_ref().map_or_else(
|| "Effective SSD settings will appear after valid values are entered."
.to_owned(),
|engine| {
let settings = engine.ssd;
let cache = if settings.cache_bytes > 0 {
format!("{} GiB", settings.cache_bytes / GIB)
} else if settings.cache_experts > 0 {
format!("{} experts", settings.cache_experts)
} else {
"auto".to_owned()
};
format!(
"Engine: streaming {} • cold {} • cache {} • full layers {}{} • preload {}",
if settings.enabled { "on" } else { "off" },
if settings.cold { "on" } else { "off" },
cache,
settings.full_layers,
if settings.full_layers_set { " explicit" } else { " auto" },
if settings.preload_experts == 0 { "auto".to_owned() } else { settings.preload_experts.to_string() },
)
},
))
.size(12),
Space::with_height(8),
text("DIRECTIONAL STEERING").size(11),
text("Direction-vector file").size(13),
steering_file.padding(9),
preference_input_row("FFN scale", steering_ffn),
preference_input_row("Attention scale", steering_attn),
text(if self.preference_draft.model == ModelChoice::Glm52 {
"Directional steering is not supported for GLM 5.2."
} else {
"With a file and no explicit scale, DS4 defaults the FFN scale to 1. Scales accept -100 through 100."
})
.size(12),
text(engine.as_ref().map_or_else(
|| "Effective steering settings will appear after valid values are entered."
.to_owned(),
|engine| format!(
"Engine: file {} • FFN scale {} • attention scale {}",
if engine.steering.file.is_some() { "set" } else { "off" },
engine.steering.ffn_scale,
engine.steering.attention_scale,
),
))
.size(12),
Space::with_height(8),
text("CAPACITY").size(11),
preference_input_row(
"Context tokens",
text_input("32768", &self.preference_draft.context_tokens)
.on_input(Message::PreferenceContextChanged),
),
preference_input_row(
"Maximum generated tokens",
text_input("50000", &self.preference_draft.max_generated_tokens)
.on_input(Message::PreferenceMaxTokensChanged),
),
text("System prompt").size(13),
text_input(
"You are a helpful assistant",
&self.preference_draft.system_prompt,
)
.on_input(Message::PreferenceSystemPromptChanged)
.padding(9),
Space::with_height(8),
text("SAMPLING AND REASONING").size(11),
preference_input_row(
"Temperature",
text_input("DS4 default", &self.preference_draft.temperature)
.on_input(Message::PreferenceTemperatureChanged),
),
preference_input_row(
"Top-p",
text_input("DS4 default", &self.preference_draft.top_p)
.on_input(Message::PreferenceTopPChanged),
),
preference_input_row(
"Min-p",
text_input("DS4 default", &self.preference_draft.min_p)
.on_input(Message::PreferenceMinPChanged),
),
preference_input_row(
"Seed",
text_input("Random", &self.preference_draft.seed)
.on_input(Message::PreferenceSeedChanged),
),
row![
text("Reasoning").size(13).width(Length::Fill),
let model_group = preference_group(
"MODEL & LIFECYCLE",
column![
pick_list(
&REASONING_MODES[..],
Some(self.preference_draft.reasoning_mode),
Message::PreferenceReasoningChanged,
&MODEL_CHOICES[..],
Some(self.preference_draft.model),
Message::PreferenceModelChanged,
)
.width(240),
]
.spacing(12)
.align_y(Alignment::Center),
text("Blank sampling values retain DS4's model-family defaults. Think Max needs at least 393216 context tokens.")
.width(Length::Fill),
text(format!(
"Main: {}{}",
engine.map_or_else(
|| "Invalid settings".to_owned(),
|engine| engine.artifacts.model.display().to_string(),
),
engine
.and_then(|engine| engine.artifacts.mtp.as_ref())
.map_or_else(String::new, |path| format!(
" • support: {}",
path.display()
)),
))
.size(12),
text(turn.map_or_else(
|| "Effective settings will appear after valid values are entered.".to_owned(),
|settings| format!(
"Effective: {} context • {} max • temp {} • top-p {} • min-p {} • seed {} • {} • system prompt {}",
settings.context_tokens,
settings.max_generated_tokens,
settings.temperature,
settings.top_p,
settings.min_p,
settings.seed.map_or_else(|| "random".to_owned(), |seed| seed.to_string()),
settings.reasoning_mode,
if settings.system_prompt.is_empty() { "off" } else { "on" },
),
))
.size(12),
Space::with_height(8),
text("ADVANCED DIAGNOSTICS").size(11),
preference_input_row(
"Simulated used memory (GiB)",
text_input("Disabled", &self.preference_draft.simulated_used_memory_gib)
.on_input(Message::PreferenceSimulatedMemoryChanged),
),
text("Routed expert profile output").size(13),
text_input("Output file path", &self.preference_draft.expert_profile_path)
.on_input(Message::PreferenceExpertProfileChanged)
.padding(9),
text(engine.as_ref().map_or_else(
|| "Effective diagnostic settings will appear after valid values are entered."
.to_owned(),
|engine| format!(
"{} load: simulated memory {} • expert profile {}",
engine.model,
if engine.diagnostics.simulated_used_memory_bytes == 0 {
"off".to_owned()
} else {
format!("{} GiB", engine.diagnostics.simulated_used_memory_bytes / GIB)
},
if engine.diagnostics.expert_profile_path.is_some() { "set" } else { "off" },
),
))
.size(12),
Space::with_height(8),
text("INACTIVITY").size(11),
row![
text_input("10", &self.preference_draft.idle_timeout_minutes)
.on_input(Message::PreferenceTimeoutChanged)
.width(90)
.padding(9),
text("minutes before unloading the model").size(13),
row![
text_input("10", &self.preference_draft.idle_timeout_minutes)
.on_input(Message::PreferenceTimeoutChanged)
.width(90)
.padding(9),
text("minutes before unloading the model").size(13),
]
.spacing(10)
.align_y(Alignment::Center),
text("Enter a whole number from 1 to 1440.").size(12),
]
.spacing(10)
.align_y(Alignment::Center),
text("Enter a whole number from 1 to 1440.").size(12),
.spacing(10),
);
let endpoint_group = preference_group(
"LOCAL ENDPOINT",
column![
preference_input_row(
"Port",
text_input("4000", &self.preference_draft.endpoint_port)
.on_input(Message::PreferenceEndpointPortChanged),
),
text("Listens on 127.0.0.1. Saving a changed port restarts the local endpoint.")
.size(12),
]
.spacing(10),
);
let generation_group = preference_group(
"GENERATION",
column![
preference_input_row(
"Context tokens",
text_input("32768", &self.preference_draft.context_tokens)
.on_input(Message::PreferenceContextChanged),
),
preference_input_row(
"Maximum generated tokens",
text_input("50000", &self.preference_draft.max_generated_tokens)
.on_input(Message::PreferenceMaxTokensChanged),
),
text("System prompt").size(13),
text_input(
"You are a helpful assistant",
&self.preference_draft.system_prompt,
)
.on_input(Message::PreferenceSystemPromptChanged)
.padding(9),
Space::with_height(4),
text("SAMPLING & REASONING").size(11).color(muted_text()),
preference_input_row(
"Temperature",
text_input("DS4 default", &self.preference_draft.temperature)
.on_input(Message::PreferenceTemperatureChanged),
),
preference_input_row(
"Top-p",
text_input("DS4 default", &self.preference_draft.top_p)
.on_input(Message::PreferenceTopPChanged),
),
preference_input_row(
"Min-p",
text_input("DS4 default", &self.preference_draft.min_p)
.on_input(Message::PreferenceMinPChanged),
),
preference_input_row(
"Seed",
text_input("Random", &self.preference_draft.seed)
.on_input(Message::PreferenceSeedChanged),
),
row![
text("Reasoning").size(13).width(Length::Fill),
pick_list(
&REASONING_MODES[..],
Some(self.preference_draft.reasoning_mode),
Message::PreferenceReasoningChanged,
)
.width(240),
]
.spacing(12)
.align_y(Alignment::Center),
text("Blank sampling values retain DS4's model-family defaults. Think Max needs at least 393216 context tokens.")
.size(12),
text(turn.map_or_else(
|| "Effective settings will appear after valid values are entered.".to_owned(),
|settings| format!(
"Effective: {} context • {} max • temp {} • top-p {} • min-p {} • seed {} • {} • system prompt {}",
settings.context_tokens,
settings.max_generated_tokens,
settings.temperature,
settings.top_p,
settings.min_p,
settings.seed.map_or_else(|| "random".to_owned(), |seed| seed.to_string()),
settings.reasoning_mode,
if settings.system_prompt.is_empty() { "off" } else { "on" },
),
))
.size(12),
]
.spacing(10),
);
let execution_group = preference_group(
"EXECUTION",
column![
preference_input_row(
"CPU helper threads",
text_input("Automatic", &self.preference_draft.cpu_threads)
.on_input(Message::PreferenceCpuThreadsChanged),
),
preference_input_row("GPU power percent", power),
preference_input_row("Prefill chunk", prefill),
checkbox("Prefer exact quality kernels", self.preference_draft.quality)
.on_toggle(Message::PreferenceQualityChanged),
checkbox("Warm mapped weights at load time", self.preference_draft.warm_weights)
.on_toggle(Message::PreferenceWarmWeightsChanged),
text(if self.preference_draft.model == ModelChoice::Glm52 {
"GLM 5.2 uses full GPU power and selects prefill chunks automatically."
} else {
"Blank numeric values preserve DS4's automatic engine behavior."
})
.size(12),
text(engine.as_ref().map_or_else(
|| "Effective execution settings will appear after valid values are entered."
.to_owned(),
|engine| {
let settings = engine.execution;
format!(
"Metal engine: threads {} • power {}% • prefill {} • quality {} • warm weights {}",
if settings.cpu_threads == 0 { "auto".to_owned() } else { settings.cpu_threads.to_string() },
if settings.power_percent == 0 { 100 } else { settings.power_percent },
if settings.prefill_chunk == 0 { "auto".to_owned() } else { settings.prefill_chunk.to_string() },
if settings.quality { "on" } else { "off" },
if settings.warm_weights { "on" } else { "off" },
)
},
))
.size(12),
]
.spacing(10),
);
let acceleration_group = preference_group(
"ACCELERATION & MEMORY",
column![
text("SPECULATIVE DECODING").size(11).color(muted_text()),
preference_input_row(
"MTP draft tokens",
text_input("1", &self.preference_draft.mtp_draft_tokens)
.on_input(Message::PreferenceMtpDraftChanged),
),
preference_input_row(
"MTP verifier margin",
text_input("3", &self.preference_draft.mtp_margin)
.on_input(Message::PreferenceMtpMarginChanged),
),
checkbox("Enable integrated GLM MTP", self.preference_draft.glm_mtp)
.on_toggle_maybe(glm_mtp_toggle),
checkbox(
"Log GLM MTP timing counters",
self.preference_draft.glm_mtp_timing,
)
.on_toggle_maybe(glm_mtp_timing_toggle),
dspark,
preference_input_row("DSpark confidence threshold", dspark_confidence),
checkbox(
"DSpark target-only decode",
self.preference_draft.dspark_strict,
)
.on_toggle_maybe(dspark_strict_toggle),
text(if self.preference_draft.model.supports_dspark() {
"DSpark uses the managed support artifact; entering a threshold or enabling strict mode also enables DSpark."
} else if self.preference_draft.model == ModelChoice::Glm52 {
"GLM MTP is integrated; DSpark is unavailable for this model."
} else {
"No managed MTP support artifact is available for this model."
})
.size(12),
text(engine.as_ref().map_or_else(
|| "Effective speculative settings will appear after valid values are entered."
.to_owned(),
|engine| {
let settings = engine.speculative;
format!(
"Engine: MTP draft {} • margin {} • GLM MTP {} • timing {} • DSpark {} • confidence {}{} • target-only {}",
settings.mtp_draft_tokens,
settings.mtp_margin,
if settings.glm_mtp { "on" } else { "off" },
if settings.glm_mtp_timing { "on" } else { "off" },
if settings.dspark { "on" } else { "off" },
settings.dspark_confidence_threshold,
if settings.dspark_confidence_threshold_set { " explicit" } else { " default" },
if settings.dspark_strict { "on" } else { "off" },
)
},
))
.size(12),
Space::with_height(6),
text("SSD STREAMING").size(11).color(muted_text()),
checkbox("Enable SSD-backed model streaming", self.preference_draft.ssd_streaming)
.on_toggle(Message::PreferenceSsdChanged),
checkbox("Skip automatic expert preload", self.preference_draft.ssd_streaming_cold)
.on_toggle(Message::PreferenceSsdColdChanged),
preference_input_row(
"Expert cache count or GiB",
text_input("Automatic, 128, or 64GB", &self.preference_draft.ssd_cache)
.on_input(Message::PreferenceSsdCacheChanged),
),
preference_input_row("Fully resident GLM layers", ssd_full_layers),
preference_input_row(
"Explicit expert preload count",
text_input("Automatic", &self.preference_draft.ssd_preload_experts)
.on_input(Message::PreferenceSsdPreloadChanged),
),
text("A blank full-layer value is automatic; an explicit 0 disables fully resident GLM layers. SSD streaming and DSpark are mutually exclusive.")
.size(12),
text(engine.as_ref().map_or_else(
|| "Effective SSD settings will appear after valid values are entered."
.to_owned(),
|engine| {
let settings = engine.ssd;
let cache = if settings.cache_bytes > 0 {
format!("{} GiB", settings.cache_bytes / GIB)
} else if settings.cache_experts > 0 {
format!("{} experts", settings.cache_experts)
} else {
"auto".to_owned()
};
format!(
"Engine: streaming {} • cold {} • cache {} • full layers {}{} • preload {}",
if settings.enabled { "on" } else { "off" },
if settings.cold { "on" } else { "off" },
cache,
settings.full_layers,
if settings.full_layers_set { " explicit" } else { " auto" },
if settings.preload_experts == 0 { "auto".to_owned() } else { settings.preload_experts.to_string() },
)
},
))
.size(12),
]
.spacing(10),
);
let steering_group = preference_group(
"STEERING & DIAGNOSTICS",
column![
text("DIRECTIONAL STEERING").size(11).color(muted_text()),
text("Direction-vector file").size(13),
steering_file.padding(9),
preference_input_row("FFN scale", steering_ffn),
preference_input_row("Attention scale", steering_attn),
text(if self.preference_draft.model == ModelChoice::Glm52 {
"Directional steering is not supported for GLM 5.2."
} else {
"With a file and no explicit scale, DS4 defaults the FFN scale to 1. Scales accept -100 through 100."
})
.size(12),
text(engine.as_ref().map_or_else(
|| "Effective steering settings will appear after valid values are entered."
.to_owned(),
|engine| format!(
"Engine: file {} • FFN scale {} • attention scale {}",
if engine.steering.file.is_some() { "set" } else { "off" },
engine.steering.ffn_scale,
engine.steering.attention_scale,
),
))
.size(12),
Space::with_height(6),
text("ADVANCED DIAGNOSTICS").size(11).color(muted_text()),
preference_input_row(
"Simulated used memory (GiB)",
text_input("Disabled", &self.preference_draft.simulated_used_memory_gib)
.on_input(Message::PreferenceSimulatedMemoryChanged),
),
text("Routed expert profile output").size(13),
text_input("Output file path", &self.preference_draft.expert_profile_path)
.on_input(Message::PreferenceExpertProfileChanged)
.padding(9),
text(engine.as_ref().map_or_else(
|| "Effective diagnostic settings will appear after valid values are entered."
.to_owned(),
|engine| format!(
"{} load: simulated memory {} • expert profile {}",
engine.model,
if engine.diagnostics.simulated_used_memory_bytes == 0 {
"off".to_owned()
} else {
format!("{} GiB", engine.diagnostics.simulated_used_memory_bytes / GIB)
},
if engine.diagnostics.expert_profile_path.is_some() { "set" } else { "off" },
),
))
.size(12),
]
.spacing(10),
);
let mut fields = column![
model_group,
endpoint_group,
generation_group,
execution_group,
acceleration_group,
steering_group,
]
.spacing(10);
.spacing(12);
if let Some(error) = &self.preference_error {
fields = fields.push(text(error).style(iced::widget::text::danger));
@@ -971,6 +1021,17 @@ fn preference_input_row<'a>(
.into()
}
fn preference_group<'a>(
title: &'a str,
content: impl Into<Element<'a, Message>>,
) -> Element<'a, Message> {
container(column![text(title).size(11).color(muted_text()), content.into(),].spacing(10))
.width(Length::Fill)
.padding(14)
.style(preference_group_style)
.into()
}
fn download_status_bar(download: &ActiveDownload) -> Element<'_, Message> {
let progress = &download.progress;
let percent = progress.fraction() * 100.0;
@@ -1201,6 +1262,18 @@ fn overview_style(_: &Theme) -> container::Style {
}
}
fn preference_group_style(_: &Theme) -> container::Style {
container::Style {
background: Some(Background::Color(Color::from_rgb8(38, 38, 40))),
border: Border {
color: Color::from_rgb8(58, 58, 61),
width: 1.0,
radius: 14.0.into(),
},
..container::Style::default()
}
}
fn chat_message_style(theme: &Theme, user: bool) -> container::Style {
if !user {
return overview_style(theme);
@@ -1247,6 +1320,10 @@ mod tests {
chat_message_style(&theme, true).background,
chat_message_style(&theme, false).background
);
assert_eq!(
preference_group_style(&theme).background,
Some(Background::Color(Color::from_rgb8(38, 38, 40)))
);
}
#[test]