hugo/DS4Server
1
0
Fork 0
Code Issues 7 Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Implement batched server parity and observability

Browse Source
This commit is contained in:
Georg Bauer
2026-07-25 08:32:35 +02:00
parent d554b77b9d
commit 2fbe605b5d
15 changed files with 6478 additions and 266 deletions
Download Patch File Download Diff File Expand all files Collapse all files

105
src/app.rs
View File

@@ -5,6 +5,7 @@ pub(crate) use view::app_theme;
use crate::database::{AppPreferences, Database, ProjectWithSessions, StoredMessage};
#[cfg(target_os = "macos")]
use crate::engine::ChatTurn;
use crate::metrics::{Metrics, MetricsSnapshot};
use crate::model::{self, DownloadOutcome, DownloadProgress, ManagedArtifactId, ModelChoice};
#[cfg(target_os = "macos")]
use crate::runtime::{ActiveGeneration, CheckpointTarget, GenerationEvent, GenerationService};
@@ -16,6 +17,7 @@ use crate::settings::{
use iced::widget::{markdown, scrollable};
use iced::{Size, Subscription, Task, keyboard, window};
use rfd::AsyncFileDialog;
use std::collections::VecDeque;
use std::fs;
use std::path::PathBuf;
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
@@ -25,6 +27,7 @@ use std::thread;
use std::time::{Duration, Instant};
const APP_ID: &str = "DS4Server.rfc1437.de";
const METRICS_SAMPLE_INTERVAL: Duration = Duration::from_millis(200);
#[derive(Clone)]
struct PreferenceDraft {
@@ -414,6 +417,13 @@ pub(crate) struct App {
pub(super) context_used: u32,
pub(super) context_limit: u32,
pub(super) tokens_per_second: Option<f32>,
pub(super) detail_tab: DetailTab,
metrics: Arc<Metrics>,
pub(super) metrics_snapshot: MetricsSnapshot,
pub(super) metrics_history: VecDeque<MetricsPoint>,
last_http_requests: u64,
last_kv_read_bytes: u64,
last_kv_write_bytes: u64,
#[cfg(target_os = "macos")]
generation_service: Option<GenerationService>,
#[cfg(target_os = "macos")]
@@ -425,6 +435,22 @@ pub(crate) struct App {
error: Option<String>,
}
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
pub(super) enum DetailTab {
#[default]
Chat,
Stats,
}
#[derive(Clone, Copy, Debug, Default)]
pub(super) struct MetricsPoint {
pub(super) decode_tokens_per_second: f32,
pub(super) prefill_tokens_per_second: f32,
pub(super) http_requests_per_second: f32,
pub(super) kv_read_bytes_per_second: f32,
pub(super) kv_write_bytes_per_second: f32,
}
#[derive(Clone, Debug)]
pub(super) struct ChatMessage {
id: i32,
@@ -571,6 +597,9 @@ pub(crate) enum Message {
CreateSession,
SelectSession(i32, i32),
DeleteSession(i32),
ShowChat,
ShowStats,
MetricsTick,
}
impl App {
@@ -584,9 +613,12 @@ impl App {
Err(error) => return Self::failed(error, main_window),
};
let context_limit = preferences.context_tokens.max(0) as u32;
let metrics =
Arc::new(Metrics::new(&application_support_path().join("kv-cache")));
let metrics_snapshot = metrics.snapshot();
#[cfg(target_os = "macos")]
let (runtime_preferences, generation_service, endpoint, service_error) =
spawn_services(&preferences);
spawn_services(&preferences, Arc::clone(&metrics));
Self {
main_window,
model_manager_window: None,
@@ -613,6 +645,13 @@ impl App {
context_used: 0,
context_limit,
tokens_per_second: None,
detail_tab: DetailTab::Chat,
metrics,
metrics_snapshot,
metrics_history: VecDeque::with_capacity(120),
last_http_requests: 0,
last_kv_read_bytes: 0,
last_kv_write_bytes: 0,
#[cfg(target_os = "macos")]
generation_service,
#[cfg(target_os = "macos")]
@@ -644,9 +683,11 @@ impl App {
let preference_draft = PreferenceDraft::from_saved(&preferences)
.expect("default preferences must use a supported model");
let context_limit = preferences.context_tokens.max(0) as u32;
let metrics = Arc::new(Metrics::new(&application_support_path().join("kv-cache")));
let metrics_snapshot = metrics.snapshot();
#[cfg(target_os = "macos")]
let (runtime_preferences, generation_service, endpoint, service_error) =
spawn_services(&preferences);
spawn_services(&preferences, Arc::clone(&metrics));
#[cfg(target_os = "macos")]
let startup_error = service_error;
#[cfg(not(target_os = "macos"))]
@@ -677,6 +718,13 @@ impl App {
context_used: 0,
context_limit,
tokens_per_second: None,
detail_tab: DetailTab::Chat,
metrics,
metrics_snapshot,
metrics_history: VecDeque::with_capacity(120),
last_http_requests: 0,
last_kv_read_bytes: 0,
last_kv_write_bytes: 0,
#[cfg(target_os = "macos")]
generation_service,
#[cfg(target_os = "macos")]
@@ -740,6 +788,9 @@ impl App {
self.error = None;
}
}
Message::ShowChat => self.detail_tab = DetailTab::Chat,
Message::ShowStats => self.detail_tab = DetailTab::Stats,
Message::MetricsTick => self.sample_metrics(),
Message::PreferenceModelChanged(model) => {
self.preference_draft.model = model;
if !model.supports_dspark() {
@@ -1153,6 +1204,8 @@ impl App {
window::close_requests().map(Message::WindowClosed),
window::close_events().map(Message::WindowClosed),
];
subscriptions
.push(iced::time::every(METRICS_SAMPLE_INTERVAL).map(|_| Message::MetricsTick));
#[cfg(target_os = "macos")]
subscriptions.push(iced::time::every(Duration::from_millis(50)).map(|_| {
match crate::native_menu::next_event() {
@@ -1327,6 +1380,7 @@ impl App {
models_path(),
application_support_path().join("kv-cache").join("http"),
endpoint_port,
Arc::clone(&self.metrics),
) {
Ok(endpoint) => Some(endpoint),
Err(error) => {
@@ -1569,6 +1623,7 @@ impl App {
.iter()
.map(|message| ChatTurn {
user: message.user,
skip_previous_eos: false,
reasoning: message.reasoning.clone(),
reasoning_complete: message.reasoning_complete,
content: message.content.clone(),
@@ -1577,6 +1632,7 @@ impl App {
#[cfg(target_os = "macos")]
messages.push(ChatTurn {
user: true,
skip_previous_eos: false,
reasoning: None,
reasoning_complete: true,
content: prompt.clone(),
@@ -1729,11 +1785,53 @@ impl App {
#[cfg(not(target_os = "macos"))]
false
}
fn sample_metrics(&mut self) {
let snapshot = self.metrics.snapshot();
let http_requests_per_second = snapshot
.http_requests
.saturating_sub(self.last_http_requests) as f32
/ METRICS_SAMPLE_INTERVAL.as_secs_f32();
let kv_read_bytes_per_second = snapshot
.kv_read_bytes
.saturating_sub(self.last_kv_read_bytes) as f32
/ METRICS_SAMPLE_INTERVAL.as_secs_f32();
let kv_write_bytes_per_second = snapshot
.kv_write_bytes
.saturating_sub(self.last_kv_write_bytes)
as f32
/ METRICS_SAMPLE_INTERVAL.as_secs_f32();
self.last_http_requests = snapshot.http_requests;
self.last_kv_read_bytes = snapshot.kv_read_bytes;
self.last_kv_write_bytes = snapshot.kv_write_bytes;
self.metrics_history.push_back(MetricsPoint {
decode_tokens_per_second: if snapshot.phase == crate::metrics::RuntimePhase::Generating
{
snapshot.decode_tokens_per_second
} else {
0.0
},
prefill_tokens_per_second: if snapshot.phase == crate::metrics::RuntimePhase::Prefilling
{
snapshot.prefill_tokens_per_second
} else {
0.0
},
http_requests_per_second,
kv_read_bytes_per_second,
kv_write_bytes_per_second,
});
if self.metrics_history.len() > 120 {
self.metrics_history.pop_front();
}
self.metrics_snapshot = snapshot;
}
}
#[cfg(target_os = "macos")]
fn spawn_services(
preferences: &AppPreferences,
metrics: Arc<Metrics>,
) -> (
Arc<RwLock<AppPreferences>>,
Option<GenerationService>,
@@ -1741,7 +1839,7 @@ fn spawn_services(
Option<String>,
) {
let runtime_preferences = Arc::new(RwLock::new(preferences.clone()));
let generation = match GenerationService::spawn() {
let generation = match GenerationService::spawn(Arc::clone(&metrics)) {
Ok(generation) => generation,
Err(error) => return (runtime_preferences, None, None, Some(error)),
};
@@ -1751,6 +1849,7 @@ fn spawn_services(
models_path(),
application_support_path().join("kv-cache").join("http"),
u16::try_from(preferences.endpoint_port).unwrap_or(4000),
metrics,
);
match endpoint {
Ok(endpoint) => (runtime_preferences, Some(generation), Some(endpoint), None),