hugo/DS4Server
1
0
Fork 0
Code Issues 7 Pull Requests Actions 1 Packages Projects Releases Wiki Activity

feat: first cut at openai compatible server

Browse Source
This commit is contained in:
Georg Bauer
2026-07-24 21:12:48 +02:00
parent bbbe65a75f
commit d554b77b9d
18 changed files with 2612 additions and 535 deletions
Download Patch File Download Diff File Expand all files Collapse all files

332
src/app.rs
View File

@@ -4,8 +4,10 @@ pub(crate) use view::app_theme;
use crate::database::{AppPreferences, Database, ProjectWithSessions, StoredMessage};
#[cfg(target_os = "macos")]
use crate::engine::{ChatTurn, Generator};
use crate::engine::ChatTurn;
use crate::model::{self, DownloadOutcome, DownloadProgress, ManagedArtifactId, ModelChoice};
#[cfg(target_os = "macos")]
use crate::runtime::{ActiveGeneration, CheckpointTarget, GenerationEvent, GenerationService};
use crate::settings::{
DiagnosticPreferences, ExecutionPreferences, GIB, GenerationPreferences, ReasoningMode,
RuntimePreferences, SpeculativePreferences, SsdPreferences, SteeringPreferences,
@@ -16,9 +18,9 @@ use iced::{Size, Subscription, Task, keyboard, window};
use rfd::AsyncFileDialog;
use std::fs;
use std::path::PathBuf;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::sync::mpsc::{self, TryRecvError};
use std::sync::{Arc, RwLock};
use std::thread;
use std::time::{Duration, Instant};
@@ -29,6 +31,7 @@ struct PreferenceDraft {
model: ModelChoice,
dspark_enabled: bool,
idle_timeout_minutes: String,
endpoint_port: String,
context_tokens: String,
max_generated_tokens: String,
system_prompt: String,
@@ -73,6 +76,7 @@ impl PreferenceDraft {
model,
dspark_enabled: speculative.dspark_enabled,
idle_timeout_minutes: preferences.idle_timeout_minutes.to_string(),
endpoint_port: preferences.endpoint_port.to_string(),
context_tokens: generation.context_tokens.to_string(),
max_generated_tokens: generation.max_generated_tokens.to_string(),
system_prompt: generation.system_prompt,
@@ -168,6 +172,7 @@ impl PreferenceDraft {
self.model = ModelChoice::default();
self.dspark_enabled = false;
self.idle_timeout_minutes = "10".into();
self.endpoint_port = "4000".into();
self.context_tokens = defaults.context_tokens.to_string();
self.max_generated_tokens = defaults.max_generated_tokens.to_string();
self.system_prompt = defaults.system_prompt;
@@ -389,6 +394,8 @@ pub(crate) struct App {
pub(super) pending_model_delete: Option<ManagedArtifactId>,
#[cfg(target_os = "macos")]
_native_menu: Option<crate::native_menu::NativeMenu>,
#[cfg(target_os = "macos")]
pub(super) native_edit_commands: crate::native_edit::EditCommandQueue,
database: Option<Database>,
projects: Vec<ProjectWithSessions>,
preferences: AppPreferences,
@@ -408,7 +415,13 @@ pub(crate) struct App {
pub(super) context_limit: u32,
pub(super) tokens_per_second: Option<f32>,
#[cfg(target_os = "macos")]
generation_worker: Option<GenerationWorker>,
generation_service: Option<GenerationService>,
#[cfg(target_os = "macos")]
active_generation: Option<ActiveGeneration>,
#[cfg(target_os = "macos")]
runtime_preferences: Arc<RwLock<AppPreferences>>,
#[cfg(target_os = "macos")]
_endpoint: Option<crate::server::ServerHandle>,
error: Option<String>,
}
@@ -461,40 +474,6 @@ impl From<StoredMessage> for ChatMessage {
}
}
#[cfg(target_os = "macos")]
struct GenerationWorker {
commands: mpsc::Sender<GenerationCommand>,
events: mpsc::Receiver<GenerationEvent>,
cancel: Option<Arc<AtomicBool>>,
}
#[cfg(target_os = "macos")]
enum GenerationCommand {
Generate {
engine: crate::settings::EngineSettings,
turn: crate::settings::TurnSettings,
messages: Vec<ChatTurn>,
checkpoint: PathBuf,
idle_timeout: Duration,
cancel: Arc<AtomicBool>,
},
}
#[cfg(target_os = "macos")]
enum GenerationEvent {
Loading,
Chunk {
reasoning: bool,
content: String,
},
Context {
used: u32,
limit: u32,
tokens_per_second: Option<f32>,
},
Finished(Result<(), String>),
}
#[derive(Debug)]
pub(super) enum ModelDownload {
Idle,
@@ -526,6 +505,8 @@ pub(super) struct ActiveDownload {
#[derive(Debug, Clone)]
pub(crate) enum Message {
Noop,
#[cfg(target_os = "macos")]
NativeEdit(crate::native_edit::EditCommand),
OpenPreferences,
OpenModelManager,
ModelManagerOpened(window::Id),
@@ -535,6 +516,7 @@ pub(crate) enum Message {
PreferenceModelChanged(ModelChoice),
PreferenceDsparkChanged(bool),
PreferenceTimeoutChanged(String),
PreferenceEndpointPortChanged(String),
PreferenceContextChanged(String),
PreferenceMaxTokensChanged(String),
PreferenceSystemPromptChanged(String),
@@ -602,12 +584,17 @@ impl App {
Err(error) => return Self::failed(error, main_window),
};
let context_limit = preferences.context_tokens.max(0) as u32;
#[cfg(target_os = "macos")]
let (runtime_preferences, generation_service, endpoint, service_error) =
spawn_services(&preferences);
Self {
main_window,
model_manager_window: None,
pending_model_delete: None,
#[cfg(target_os = "macos")]
_native_menu: None,
#[cfg(target_os = "macos")]
native_edit_commands: crate::native_edit::command_queue(),
database: Some(database),
projects,
preferences,
@@ -627,8 +614,23 @@ impl App {
context_limit,
tokens_per_second: None,
#[cfg(target_os = "macos")]
generation_worker: None,
error: None,
generation_service,
#[cfg(target_os = "macos")]
active_generation: None,
#[cfg(target_os = "macos")]
runtime_preferences,
#[cfg(target_os = "macos")]
_endpoint: endpoint,
error: {
#[cfg(target_os = "macos")]
{
service_error
}
#[cfg(not(target_os = "macos"))]
{
None
}
},
}
}
(Err(error), _) | (_, Err(error)) => Self::failed(error, main_window),
@@ -642,12 +644,21 @@ impl App {
let preference_draft = PreferenceDraft::from_saved(&preferences)
.expect("default preferences must use a supported model");
let context_limit = preferences.context_tokens.max(0) as u32;
#[cfg(target_os = "macos")]
let (runtime_preferences, generation_service, endpoint, service_error) =
spawn_services(&preferences);
#[cfg(target_os = "macos")]
let startup_error = service_error;
#[cfg(not(target_os = "macos"))]
let startup_error = None::<String>;
Self {
main_window,
model_manager_window: None,
pending_model_delete: None,
#[cfg(target_os = "macos")]
_native_menu: None,
#[cfg(target_os = "macos")]
native_edit_commands: crate::native_edit::command_queue(),
database: None,
projects: Vec::new(),
preferences,
@@ -667,14 +678,29 @@ impl App {
context_limit,
tokens_per_second: None,
#[cfg(target_os = "macos")]
generation_worker: None,
error: Some(format!("Could not open the project database: {error}")),
generation_service,
#[cfg(target_os = "macos")]
active_generation: None,
#[cfg(target_os = "macos")]
runtime_preferences,
#[cfg(target_os = "macos")]
_endpoint: endpoint,
error: Some(match startup_error {
Some(service_error) => {
format!("Could not open the project database: {error}. {service_error}")
}
None => format!("Could not open the project database: {error}"),
}),
}
}
pub(crate) fn update(&mut self, message: Message) -> Task<Message> {
match message {
Message::Noop => {}
#[cfg(target_os = "macos")]
Message::NativeEdit(command) => {
crate::native_edit::queue_command(&self.native_edit_commands, command)
}
Message::OpenPreferences => self.open_preferences(),
Message::OpenModelManager => return self.open_model_manager(),
Message::ModelManagerOpened(id) => {
@@ -749,6 +775,10 @@ impl App {
self.preference_draft.idle_timeout_minutes = value;
self.preference_error = None;
}
Message::PreferenceEndpointPortChanged(value) => {
self.preference_draft.endpoint_port = value;
self.preference_error = None;
}
Message::PreferenceContextChanged(value) => {
self.preference_draft.context_tokens = value;
self.preference_error = None;
@@ -953,14 +983,11 @@ impl App {
self.start_generation();
return scroll_chat_to_end();
}
Message::StopGeneration => {
Message::StopGeneration =>
{
#[cfg(target_os = "macos")]
if let Some(cancel) = self
.generation_worker
.as_ref()
.and_then(|worker| worker.cancel.as_ref())
{
cancel.store(true, Ordering::Relaxed);
if let Some(active) = &self.active_generation {
active.cancel.store(true, Ordering::Relaxed);
}
}
Message::GenerationTick => {
@@ -1133,6 +1160,9 @@ impl App {
Some(crate::native_menu::NativeMenuEvent::ModelManager) => {
Message::OpenModelManager
}
Some(crate::native_menu::NativeMenuEvent::Edit(command)) => {
Message::NativeEdit(command)
}
None => Message::Noop,
}
}));
@@ -1255,6 +1285,14 @@ impl App {
self.preference_error = Some("Idle timeout must be between 1 and 1440 minutes.".into());
return;
}
let Ok(endpoint_port) = self.preference_draft.endpoint_port.trim().parse::<u16>() else {
self.preference_error = Some("Endpoint port must be a whole number.".into());
return;
};
if endpoint_port == 0 {
self.preference_error = Some("Endpoint port must be between 1 and 65535.".into());
return;
}
let generation = match self.preference_draft.generation() {
Ok(generation) => generation,
Err(error) => {
@@ -1275,12 +1313,50 @@ impl App {
self.preference_error = Some(error);
return;
}
#[cfg(target_os = "macos")]
let pending_endpoint = if self.preferences.endpoint_port != i32::from(endpoint_port)
|| self._endpoint.is_none()
{
let Some(generation) = &self.generation_service else {
self.preference_error = Some("The model runtime is unavailable.".into());
return;
};
match crate::server::ServerHandle::spawn(
generation.clone(),
Arc::clone(&self.runtime_preferences),
models_path(),
application_support_path().join("kv-cache").join("http"),
endpoint_port,
) {
Ok(endpoint) => Some(endpoint),
Err(error) => {
self.preference_error = Some(error);
return;
}
}
} else {
None
};
let Some(database) = &mut self.database else {
return;
};
match database.update_preferences(model.id(), idle_timeout_minutes, &generation, &runtime) {
match database.update_preferences(
model.id(),
idle_timeout_minutes,
i32::from(endpoint_port),
&generation,
&runtime,
) {
Ok(preferences) => {
self.preferences = preferences;
#[cfg(target_os = "macos")]
if let Ok(mut runtime_preferences) = self.runtime_preferences.write() {
*runtime_preferences = self.preferences.clone();
}
#[cfg(target_os = "macos")]
if let Some(endpoint) = pending_endpoint {
self._endpoint = Some(endpoint);
}
self.preference_draft = PreferenceDraft::from_saved(&self.preferences)
.expect("the saved model was selected from the supported catalog");
self.preferences_open = false;
@@ -1508,15 +1584,10 @@ impl App {
#[cfg(target_os = "macos")]
{
if self.generation_worker.is_none() {
match spawn_generation_worker() {
Ok(worker) => self.generation_worker = Some(worker),
Err(error) => {
self.error = Some(error);
return;
}
}
}
let Some(service) = &self.generation_service else {
self.error = Some("The model runtime is unavailable.".into());
return;
};
let Some(database) = &mut self.database else {
return;
};
@@ -1527,24 +1598,22 @@ impl App {
return;
}
};
let cancel = Arc::new(AtomicBool::new(false));
let idle_timeout =
Duration::from_secs(self.preferences.idle_timeout_minutes.max(1) as u64 * 60);
let command = GenerationCommand::Generate {
engine: effective.engine,
turn: effective.turn,
self.active_generation = match service.generate(
effective.engine,
effective.turn,
messages,
checkpoint: session_checkpoint_path(session_id),
CheckpointTarget::Local(session_checkpoint_path(session_id)),
idle_timeout,
cancel: Arc::clone(&cancel),
) {
Ok(active) => Some(active),
Err(error) => {
self.generation_service = None;
self.error = Some(error);
return;
}
};
let worker = self.generation_worker.as_mut().expect("worker was created");
if worker.commands.send(command).is_err() {
self.generation_worker = None;
self.error = Some("The local generation worker stopped unexpectedly.".into());
return;
}
worker.cancel = Some(cancel);
let user = ChatMessage::from(saved.0);
let mut assistant = ChatMessage::from(saved.1);
assistant.reasoning_open = assistant_reasoning;
@@ -1565,7 +1634,7 @@ impl App {
fn poll_generation(&mut self) -> bool {
#[cfg(target_os = "macos")]
let Some(worker) = &mut self.generation_worker else {
let Some(active) = &mut self.active_generation else {
self.generating = false;
return false;
};
@@ -1575,7 +1644,7 @@ impl App {
let mut context_changed = false;
#[cfg(target_os = "macos")]
loop {
match worker.events.try_recv() {
match active.events.try_recv() {
Ok(GenerationEvent::Loading) => {}
Ok(GenerationEvent::Chunk { reasoning, content }) => {
if let Some(message) = self.conversation.last_mut()
@@ -1597,17 +1666,17 @@ impl App {
}
Ok(GenerationEvent::Finished(result)) => {
self.generating = false;
worker.cancel = None;
if let Err(error) = result {
self.error = Some(error);
}
self.active_generation = None;
break;
}
Err(TryRecvError::Empty) => break,
Err(TryRecvError::Disconnected) => {
self.generating = false;
self.generation_worker = None;
self.error = Some("The local generation worker stopped unexpectedly.".into());
self.active_generation = None;
self.error = Some("The model runtime stopped unexpectedly.".into());
break;
}
}
@@ -1627,12 +1696,8 @@ impl App {
&message.content,
)
{
if let Some(cancel) = self
.generation_worker
.as_ref()
.and_then(|worker| worker.cancel.as_ref())
{
cancel.store(true, Ordering::Relaxed);
if let Some(active) = &self.active_generation {
active.cancel.store(true, Ordering::Relaxed);
}
self.error = Some(format!("Could not save generated chat text: {error}"));
}
@@ -1667,77 +1732,30 @@ impl App {
}
#[cfg(target_os = "macos")]
fn spawn_generation_worker() -> Result<GenerationWorker, String> {
let (command_sender, command_receiver) = mpsc::channel();
let (event_sender, event_receiver) = mpsc::channel();
thread::Builder::new()
.name("local-generation".into())
.spawn(move || {
let mut loaded = None::<(crate::settings::EngineSettings, Generator)>;
let mut last_used = Instant::now();
let mut idle_timeout = Duration::from_secs(15 * 60);
loop {
match command_receiver.recv_timeout(Duration::from_secs(1)) {
Ok(GenerationCommand::Generate {
engine,
turn,
messages,
checkpoint,
idle_timeout: requested_timeout,
cancel,
}) => {
idle_timeout = requested_timeout;
if loaded
.as_ref()
.is_none_or(|(current, _)| current != &engine)
{
let _ = event_sender.send(GenerationEvent::Loading);
loaded = match Generator::open(&engine) {
Ok(generator) => Some((engine.clone(), generator)),
Err(error) => {
let _ =
event_sender.send(GenerationEvent::Finished(Err(error)));
None
}
};
}
if let Some((_, generator)) = &mut loaded {
let result = generator.generate(
&checkpoint,
&messages,
&turn,
&cancel,
|reasoning, content| {
let _ = event_sender
.send(GenerationEvent::Chunk { reasoning, content });
},
|used, limit, tokens_per_second| {
let _ = event_sender.send(GenerationEvent::Context {
used,
limit,
tokens_per_second,
});
},
);
let _ = event_sender.send(GenerationEvent::Finished(result));
last_used = Instant::now();
}
}
Err(mpsc::RecvTimeoutError::Timeout) => {
if loaded.is_some() && last_used.elapsed() >= idle_timeout {
loaded = None;
}
}
Err(mpsc::RecvTimeoutError::Disconnected) => break,
}
}
})
.map_err(|error| format!("Could not start local generation: {error}"))?;
Ok(GenerationWorker {
commands: command_sender,
events: event_receiver,
cancel: None,
})
fn spawn_services(
preferences: &AppPreferences,
) -> (
Arc<RwLock<AppPreferences>>,
Option<GenerationService>,
Option<crate::server::ServerHandle>,
Option<String>,
) {
let runtime_preferences = Arc::new(RwLock::new(preferences.clone()));
let generation = match GenerationService::spawn() {
Ok(generation) => generation,
Err(error) => return (runtime_preferences, None, None, Some(error)),
};
let endpoint = crate::server::ServerHandle::spawn(
generation.clone(),
Arc::clone(&runtime_preferences),
models_path(),
application_support_path().join("kv-cache").join("http"),
u16::try_from(preferences.endpoint_port).unwrap_or(4000),
);
match endpoint {
Ok(endpoint) => (runtime_preferences, Some(generation), Some(endpoint), None),
Err(error) => (runtime_preferences, Some(generation), None, Some(error)),
}
}
impl Drop for App {
@@ -1746,12 +1764,8 @@ impl Drop for App {
download.cancel.store(true, Ordering::Relaxed);
}
#[cfg(target_os = "macos")]
if let Some(cancel) = self
.generation_worker
.as_ref()
.and_then(|worker| worker.cancel.as_ref())
{
cancel.store(true, Ordering::Relaxed);
if let Some(active) = &self.active_generation {
active.cancel.store(true, Ordering::Relaxed);
}
}
}