fix: ssd streaming and deepseek fixes

This commit is contained in:
Georg Bauer
2026-07-26 21:50:59 +02:00
parent 0d80c217c4
commit 47b405e72b
6 changed files with 457 additions and 84 deletions

View File

@@ -376,6 +376,14 @@ struct ResidentSlot {
last_store_tokens: u32,
}
#[cfg(target_os = "macos")]
impl Drop for Generator {
fn drop(&mut self) {
// Match ds4: every resident session graph must die before Metal cleanup.
self.resident_sessions.clear();
}
}
#[cfg(target_os = "macos")]
struct SimulatedMemory {
address: std::ptr::NonNull<std::ffi::c_void>,
@@ -524,9 +532,10 @@ impl Generator {
resident_active: None,
resident_limit: std::env::var("DS4_RESIDENT_SESSIONS")
.ok()
.and_then(|value| value.parse().ok())
.and_then(|value| value.parse::<usize>().ok())
.filter(|limit| *limit > 0)
.unwrap_or(4),
.unwrap_or(1)
.saturating_sub(1),
})
}
@@ -908,21 +917,23 @@ impl Generator {
.map_or((None, 0), |slot| (Some(slot.state), slot.last_store_tokens));
self.executor.swap_resident_state(&mut incoming_state)?;
if let (Some(previous), Some(outgoing)) = (self.resident_active.take(), incoming_state) {
// ponytail: four resident sessions bound memory; raise
// DS4_RESIDENT_SESSIONS when the machine can hold more KV state.
if self.resident_sessions.len() >= self.resident_limit {
let evicted = self.resident_sessions.keys().next().cloned();
if let Some(evicted) = evicted {
self.resident_sessions.remove(&evicted);
// DS4 defaults to one active session; opt into its batched-server
// behavior with DS4_RESIDENT_SESSIONS when memory permits.
if self.resident_limit != 0 {
if self.resident_sessions.len() >= self.resident_limit {
let evicted = self.resident_sessions.keys().next().cloned();
if let Some(evicted) = evicted {
self.resident_sessions.remove(&evicted);
}
}
self.resident_sessions.insert(
previous,
ResidentSlot {
state: outgoing,
last_store_tokens: self.last_store_tokens,
},
);
}
self.resident_sessions.insert(
previous,
ResidentSlot {
state: outgoing,
last_store_tokens: self.last_store_tokens,
},
);
}
self.resident_active = Some(key);
self.checkpoint = None;