hugo/DS4Server
1
0
Fork 0
Code Issues 7 Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Manage the KV cache disc usage from preferences and stats

Browse Source 
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Georg Bauer
2026-07-26 09:10:18 +02:00
parent b182e7007c
commit 63e2a74ad5
14 changed files with 794 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
src/app/preferences.rs
View File

@@ -37,6 +37,10 @@ pub(super) struct PreferenceDraft {
pub(super) directional_steering_attn: String,
pub(super) simulated_used_memory_gib: String,
pub(super) expert_profile_path: String,
pub(super) kv_budget_gib: String,
pub(super) kv_min_tokens: String,
pub(super) kv_cold_max_tokens: String,
pub(super) kv_continued_interval_tokens: String,
}
impl PreferenceDraft {
@@ -121,6 +125,22 @@ impl PreferenceDraft {
.simulated_used_memory_gib
.map_or_else(String::new, |value| value.to_string()),
expert_profile_path: runtime.diagnostics.expert_profile_path.unwrap_or_default(),
kv_budget_gib: runtime
.kv_cache
.budget_gib
.map_or_else(String::new, |value| value.to_string()),
kv_min_tokens: runtime
.kv_cache
.min_tokens
.map_or_else(String::new, |value| value.to_string()),
kv_cold_max_tokens: runtime
.kv_cache
.cold_max_tokens
.map_or_else(String::new, |value| value.to_string()),
kv_continued_interval_tokens: runtime
.kv_cache
.continued_interval_tokens
.map_or_else(String::new, |value| value.to_string()),
})
}
@@ -182,6 +202,10 @@ impl PreferenceDraft {
self.directional_steering_attn.clear();
self.simulated_used_memory_gib.clear();
self.expert_profile_path.clear();
self.kv_budget_gib.clear();
self.kv_min_tokens.clear();
self.kv_cold_max_tokens.clear();
self.kv_continued_interval_tokens.clear();
}
pub(super) fn execution(&self) -> Result<ExecutionPreferences, String> {
@@ -241,6 +265,18 @@ impl PreferenceDraft {
)?,
expert_profile_path: optional_text(&self.expert_profile_path),
},
kv_cache: KvCachePreferences {
budget_gib: parse_optional_gib("KV cache budget", &self.kv_budget_gib)?,
min_tokens: parse_optional_u32("KV cache minimum tokens", &self.kv_min_tokens)?,
cold_max_tokens: parse_optional_u32(
"KV cache cold maximum",
&self.kv_cold_max_tokens,
)?,
continued_interval_tokens: parse_optional_u32(
"KV cache continued interval",
&self.kv_continued_interval_tokens,
)?,
},
})
}
}

37
src/app/projects.rs
View File

@@ -132,7 +132,12 @@ impl App {
pub(super) fn reload_projects(&mut self) {
if let Some(database) = &mut self.database {
match database.load_projects() {
Ok(projects) => self.projects = projects,
Ok(projects) => {
self.projects = projects;
if super::sweep_orphan_checkpoints(&super::kv_cache_path(), &self.projects) {
self.finish_cache_change();
}
}
Err(error) => self.error = Some(error),
}
}
@@ -190,6 +195,36 @@ mod tests {
assert_eq!(draft_title(&projects, 99), "Session 1");
}
#[test]
fn sweeping_removes_checkpoints_of_sessions_that_no_longer_exist() {
let directory =
std::env::temp_dir().join(format!("ds4-server-sweep-{}", std::process::id()));
std::fs::create_dir_all(&directory).unwrap();
for name in ["1.bin", "2.bin", "notes.bin", "7.bin"] {
std::fs::write(directory.join(name), b"payload").unwrap();
}
let projects = vec![ProjectWithSessions {
project: project(1, "First"),
sessions: vec![session(1, 1), session(2, 1)],
}];
assert!(super::super::sweep_orphan_checkpoints(
&directory, &projects
));
assert!(directory.join("1.bin").exists());
assert!(directory.join("2.bin").exists());
// Not a session checkpoint, so not ours to delete.
assert!(directory.join("notes.bin").exists());
assert!(!directory.join("7.bin").exists());
// Nothing left to sweep on the next pass.
assert!(!super::super::sweep_orphan_checkpoints(
&directory, &projects
));
std::fs::remove_dir_all(directory).unwrap();
}
fn project(id: i32, name: &str) -> Project {
Project {
id,

51
src/app/view/preferences.rs
View File

@@ -479,12 +479,63 @@ impl App {
]
.spacing(10),
);
let kv_cache_group = preference_group(
"KV CACHE",
column![
preference_input_row(
"Disc budget (GiB)",
"How much disc the reusable prompt checkpoints of the endpoint and one-shot requests may occupy. When a new checkpoint does not fit, the least valuable older ones are deleted; they cost only a prefill to rebuild. Blank uses DS4's 4 GiB.",
text_input("4", &self.preference_draft.kv_budget_gib)
.on_input(Message::PreferenceKvBudgetChanged),
),
preference_input_row(
"Minimum tokens",
"Shortest prompt worth keeping a checkpoint for. Below this the prefill is cheaper than the disc traffic of storing and loading it.",
text_input("512", &self.preference_draft.kv_min_tokens)
.on_input(Message::PreferenceKvMinTokensChanged),
),
preference_input_row(
"Cold maximum tokens",
"Largest first prompt of a conversation that is still stored. Very long cold prompts write huge checkpoints that are rarely asked for a second time; 0 stops storing cold prompts entirely.",
text_input("30000", &self.preference_draft.kv_cold_max_tokens)
.on_input(Message::PreferenceKvColdMaxChanged),
),
preference_input_row(
"Continued interval tokens",
"How far a conversation has to grow before the next checkpoint of it is written. Wider spacing writes less and keeps fewer near-identical copies; 0 stores no continued checkpoints at all.",
text_input("10000", &self.preference_draft.kv_continued_interval_tokens)
.on_input(Message::PreferenceKvContinuedIntervalChanged),
),
text("Session checkpoints are not covered by the budget: they belong to their session and go away with it. Blank values keep the DS4 defaults.")
.size(12),
text(
self.preference_draft
.runtime()
.map_or_else(
|_| "Effective KV cache settings will appear after valid values are entered.".to_owned(),
|runtime| {
let settings = runtime.kv_cache.settings();
format!(
"Transient store: budget {} GiB • minimum {} • cold max {} • continued every {}",
settings.budget_bytes / GIB,
settings.min_tokens,
if settings.cold_max_tokens == 0 { "off".to_owned() } else { settings.cold_max_tokens.to_string() },
if settings.continued_interval_tokens == 0 { "off".to_owned() } else { settings.continued_interval_tokens.to_string() },
)
},
)
)
.size(12),
]
.spacing(10),
);
let mut fields = column![
model_group,
endpoint_group,
generation_group,
execution_group,
acceleration_group,
kv_cache_group,
steering_group,
]
.spacing(12);

133
src/app/view/stats.rs
View File

@@ -319,6 +319,7 @@ impl App {
.spacing(9)
.into(),
);
let disc = stats_panel("KV CACHE DISC USAGE", self.cache_explorer());
let server = stats_panel(
"LOCAL SERVER",
column![
@@ -353,6 +354,7 @@ impl App {
throughput,
kv_io,
requests,
disc,
row![model, runtime].spacing(10),
row![cache, server].spacing(10),
text("Counters are published by the runtime with relaxed atomics and sampled by the UI every 200 ms.")
@@ -368,4 +370,135 @@ impl App {
.height(Length::Fill)
.into()
}
/// Disc usage of both cache buckets, split by age. The bar is the age
/// distribution of what is stored; the budget governs the transient store
/// only and is reported on its own row.
fn cache_explorer(&self) -> Element<'_, Message> {
let usage = &self.kv_cache_report;
let total = usage.total_bytes();
let capacity = total.max(1);
let mut bar = row![].height(14).spacing(2);
let mut legend = column![].spacing(6);
for (index, (label, _)) in crate::metrics::CACHE_AGE_BUCKETS.iter().enumerate() {
let bytes = usage.age_bytes[index];
if bytes == 0 {
continue;
}
let color = AGE_COLORS[index];
bar = bar.push(
container(Space::new(Length::Fill, Length::Fill))
.width(Length::FillPortion(portion(bytes, capacity)))
.style(move |_| chart_bar_style(color)),
);
legend = legend.push(
row![
container(Space::new(9, 9)).style(move |_| chart_bar_style(color)),
text(*label).size(12).color(muted_text()),
Space::with_width(Length::Fill),
text(format!(
"{} · {:.0}%",
format_bytes(bytes),
bytes as f32 / total.max(1) as f32 * 100.0
))
.size(12),
]
.spacing(8)
.align_y(Alignment::Center),
);
}
if total == 0 {
bar = bar.push(
container(Space::new(Length::Fill, Length::Fill))
.style(|_| chart_bar_style(muted_text().scale_alpha(0.25))),
);
}
let mut rows = column![].spacing(7);
for entry in &usage.entries {
let path = entry.path.clone();
rows = rows.push(
row![
text(entry.name.clone()).size(12),
Space::with_width(Length::Fill),
text(format!(
"{} · {} old",
format_bytes(entry.bytes),
format_duration(entry.age_seconds as f64)
))
.size(12)
.color(muted_text()),
action_button(text("Discard").size(12))
.on_press(Message::DiscardCacheEntry(path)),
]
.spacing(10)
.align_y(Alignment::Center),
);
}
column![
metric_row(
"Stored",
format!(
"{} · {} files",
format_bytes(total),
usage.session_files + usage.transient_files
),
),
container(bar).height(14).width(Length::Fill),
legend,
metric_row(
"Sessions",
format!(
"{} · {} files · not evicted",
format_bytes(usage.session_bytes),
usage.session_files
),
),
metric_row(
"Transient",
format!(
"{} of {} budget · {} files · {:.0}% full",
format_bytes(usage.transient_bytes),
format_bytes(usage.budget_bytes),
usage.transient_files,
usage.transient_bytes as f32 / usage.budget_bytes.max(1) as f32 * 100.0,
),
),
Space::with_height(2),
text(if usage.entries.is_empty() {
"No checkpoints stored yet."
} else {
"OLDEST CHECKPOINTS"
})
.size(10)
.color(muted_text()),
rows,
row![
text("Discarding a checkpoint costs one prefill to rebuild it.")
.size(11)
.color(muted_text()),
Space::with_width(Length::Fill),
action_button(text("Clear transient cache").size(12))
.on_press(Message::ClearTransientCache),
]
.spacing(10)
.align_y(Alignment::Center),
]
.spacing(9)
.into()
}
}
/// Age buckets from green (fresh) to grey (past every hit half-life).
const AGE_COLORS: [Color; 5] = [
Color::from_rgb(0.28, 0.69, 0.44),
Color::from_rgb(0.33, 0.67, 1.0),
Color::from_rgb(0.62, 0.47, 1.0),
Color::from_rgb(0.94, 0.71, 0.27),
Color::from_rgb(0.45, 0.45, 0.48),
];
fn portion(bytes: u64, capacity: u64) -> u16 {
((bytes.saturating_mul(1000) / capacity.max(1)) as u16).max(1)
}