Manage the KV cache disc usage from preferences and stats
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Georg Bauer
@@ -479,12 +479,63 @@ impl App {
|
||||
]
|
||||
.spacing(10),
|
||||
);
|
||||
let kv_cache_group = preference_group(
|
||||
"KV CACHE",
|
||||
column![
|
||||
preference_input_row(
|
||||
"Disc budget (GiB)",
|
||||
"How much disc the reusable prompt checkpoints of the endpoint and one-shot requests may occupy. When a new checkpoint does not fit, the least valuable older ones are deleted; they cost only a prefill to rebuild. Blank uses DS4's 4 GiB.",
|
||||
text_input("4", &self.preference_draft.kv_budget_gib)
|
||||
.on_input(Message::PreferenceKvBudgetChanged),
|
||||
),
|
||||
preference_input_row(
|
||||
"Minimum tokens",
|
||||
"Shortest prompt worth keeping a checkpoint for. Below this the prefill is cheaper than the disc traffic of storing and loading it.",
|
||||
text_input("512", &self.preference_draft.kv_min_tokens)
|
||||
.on_input(Message::PreferenceKvMinTokensChanged),
|
||||
),
|
||||
preference_input_row(
|
||||
"Cold maximum tokens",
|
||||
"Largest first prompt of a conversation that is still stored. Very long cold prompts write huge checkpoints that are rarely asked for a second time; 0 stops storing cold prompts entirely.",
|
||||
text_input("30000", &self.preference_draft.kv_cold_max_tokens)
|
||||
.on_input(Message::PreferenceKvColdMaxChanged),
|
||||
),
|
||||
preference_input_row(
|
||||
"Continued interval tokens",
|
||||
"How far a conversation has to grow before the next checkpoint of it is written. Wider spacing writes less and keeps fewer near-identical copies; 0 stores no continued checkpoints at all.",
|
||||
text_input("10000", &self.preference_draft.kv_continued_interval_tokens)
|
||||
.on_input(Message::PreferenceKvContinuedIntervalChanged),
|
||||
),
|
||||
text("Session checkpoints are not covered by the budget: they belong to their session and go away with it. Blank values keep the DS4 defaults.")
|
||||
.size(12),
|
||||
text(
|
||||
self.preference_draft
|
||||
.runtime()
|
||||
.map_or_else(
|
||||
|_| "Effective KV cache settings will appear after valid values are entered.".to_owned(),
|
||||
|runtime| {
|
||||
let settings = runtime.kv_cache.settings();
|
||||
format!(
|
||||
"Transient store: budget {} GiB • minimum {} • cold max {} • continued every {}",
|
||||
settings.budget_bytes / GIB,
|
||||
settings.min_tokens,
|
||||
if settings.cold_max_tokens == 0 { "off".to_owned() } else { settings.cold_max_tokens.to_string() },
|
||||
if settings.continued_interval_tokens == 0 { "off".to_owned() } else { settings.continued_interval_tokens.to_string() },
|
||||
)
|
||||
},
|
||||
)
|
||||
)
|
||||
.size(12),
|
||||
]
|
||||
.spacing(10),
|
||||
);
|
||||
let mut fields = column![
|
||||
model_group,
|
||||
endpoint_group,
|
||||
generation_group,
|
||||
execution_group,
|
||||
acceleration_group,
|
||||
kv_cache_group,
|
||||
steering_group,
|
||||
]
|
||||
.spacing(12);
|
||||
|
||||
@@ -319,6 +319,7 @@ impl App {
|
||||
.spacing(9)
|
||||
.into(),
|
||||
);
|
||||
let disc = stats_panel("KV CACHE DISC USAGE", self.cache_explorer());
|
||||
let server = stats_panel(
|
||||
"LOCAL SERVER",
|
||||
column![
|
||||
@@ -353,6 +354,7 @@ impl App {
|
||||
throughput,
|
||||
kv_io,
|
||||
requests,
|
||||
disc,
|
||||
row![model, runtime].spacing(10),
|
||||
row![cache, server].spacing(10),
|
||||
text("Counters are published by the runtime with relaxed atomics and sampled by the UI every 200 ms.")
|
||||
@@ -368,4 +370,135 @@ impl App {
|
||||
.height(Length::Fill)
|
||||
.into()
|
||||
}
|
||||
|
||||
/// Disc usage of both cache buckets, split by age. The bar is the age
|
||||
/// distribution of what is stored; the budget governs the transient store
|
||||
/// only and is reported on its own row.
|
||||
fn cache_explorer(&self) -> Element<'_, Message> {
|
||||
let usage = &self.kv_cache_report;
|
||||
let total = usage.total_bytes();
|
||||
let capacity = total.max(1);
|
||||
let mut bar = row![].height(14).spacing(2);
|
||||
let mut legend = column![].spacing(6);
|
||||
for (index, (label, _)) in crate::metrics::CACHE_AGE_BUCKETS.iter().enumerate() {
|
||||
let bytes = usage.age_bytes[index];
|
||||
if bytes == 0 {
|
||||
continue;
|
||||
}
|
||||
let color = AGE_COLORS[index];
|
||||
bar = bar.push(
|
||||
container(Space::new(Length::Fill, Length::Fill))
|
||||
.width(Length::FillPortion(portion(bytes, capacity)))
|
||||
.style(move |_| chart_bar_style(color)),
|
||||
);
|
||||
legend = legend.push(
|
||||
row![
|
||||
container(Space::new(9, 9)).style(move |_| chart_bar_style(color)),
|
||||
text(*label).size(12).color(muted_text()),
|
||||
Space::with_width(Length::Fill),
|
||||
text(format!(
|
||||
"{} · {:.0}%",
|
||||
format_bytes(bytes),
|
||||
bytes as f32 / total.max(1) as f32 * 100.0
|
||||
))
|
||||
.size(12),
|
||||
]
|
||||
.spacing(8)
|
||||
.align_y(Alignment::Center),
|
||||
);
|
||||
}
|
||||
if total == 0 {
|
||||
bar = bar.push(
|
||||
container(Space::new(Length::Fill, Length::Fill))
|
||||
.style(|_| chart_bar_style(muted_text().scale_alpha(0.25))),
|
||||
);
|
||||
}
|
||||
|
||||
let mut rows = column![].spacing(7);
|
||||
for entry in &usage.entries {
|
||||
let path = entry.path.clone();
|
||||
rows = rows.push(
|
||||
row![
|
||||
text(entry.name.clone()).size(12),
|
||||
Space::with_width(Length::Fill),
|
||||
text(format!(
|
||||
"{} · {} old",
|
||||
format_bytes(entry.bytes),
|
||||
format_duration(entry.age_seconds as f64)
|
||||
))
|
||||
.size(12)
|
||||
.color(muted_text()),
|
||||
action_button(text("Discard").size(12))
|
||||
.on_press(Message::DiscardCacheEntry(path)),
|
||||
]
|
||||
.spacing(10)
|
||||
.align_y(Alignment::Center),
|
||||
);
|
||||
}
|
||||
|
||||
column![
|
||||
metric_row(
|
||||
"Stored",
|
||||
format!(
|
||||
"{} · {} files",
|
||||
format_bytes(total),
|
||||
usage.session_files + usage.transient_files
|
||||
),
|
||||
),
|
||||
container(bar).height(14).width(Length::Fill),
|
||||
legend,
|
||||
metric_row(
|
||||
"Sessions",
|
||||
format!(
|
||||
"{} · {} files · not evicted",
|
||||
format_bytes(usage.session_bytes),
|
||||
usage.session_files
|
||||
),
|
||||
),
|
||||
metric_row(
|
||||
"Transient",
|
||||
format!(
|
||||
"{} of {} budget · {} files · {:.0}% full",
|
||||
format_bytes(usage.transient_bytes),
|
||||
format_bytes(usage.budget_bytes),
|
||||
usage.transient_files,
|
||||
usage.transient_bytes as f32 / usage.budget_bytes.max(1) as f32 * 100.0,
|
||||
),
|
||||
),
|
||||
Space::with_height(2),
|
||||
text(if usage.entries.is_empty() {
|
||||
"No checkpoints stored yet."
|
||||
} else {
|
||||
"OLDEST CHECKPOINTS"
|
||||
})
|
||||
.size(10)
|
||||
.color(muted_text()),
|
||||
rows,
|
||||
row![
|
||||
text("Discarding a checkpoint costs one prefill to rebuild it.")
|
||||
.size(11)
|
||||
.color(muted_text()),
|
||||
Space::with_width(Length::Fill),
|
||||
action_button(text("Clear transient cache").size(12))
|
||||
.on_press(Message::ClearTransientCache),
|
||||
]
|
||||
.spacing(10)
|
||||
.align_y(Alignment::Center),
|
||||
]
|
||||
.spacing(9)
|
||||
.into()
|
||||
}
|
||||
}
|
||||
|
||||
/// Age buckets from green (fresh) to grey (past every hit half-life).
|
||||
const AGE_COLORS: [Color; 5] = [
|
||||
Color::from_rgb(0.28, 0.69, 0.44),
|
||||
Color::from_rgb(0.33, 0.67, 1.0),
|
||||
Color::from_rgb(0.62, 0.47, 1.0),
|
||||
Color::from_rgb(0.94, 0.71, 0.27),
|
||||
Color::from_rgb(0.45, 0.45, 0.48),
|
||||
];
|
||||
|
||||
fn portion(bytes: u64, capacity: u64) -> u16 {
|
||||
((bytes.saturating_mul(1000) / capacity.max(1)) as u16).max(1)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user