fix: ssd streaming and deepseek fixes
This commit is contained in:
@@ -376,6 +376,14 @@ struct ResidentSlot {
|
||||
last_store_tokens: u32,
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
impl Drop for Generator {
|
||||
fn drop(&mut self) {
|
||||
// Match ds4: every resident session graph must die before Metal cleanup.
|
||||
self.resident_sessions.clear();
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
struct SimulatedMemory {
|
||||
address: std::ptr::NonNull<std::ffi::c_void>,
|
||||
@@ -524,9 +532,10 @@ impl Generator {
|
||||
resident_active: None,
|
||||
resident_limit: std::env::var("DS4_RESIDENT_SESSIONS")
|
||||
.ok()
|
||||
.and_then(|value| value.parse().ok())
|
||||
.and_then(|value| value.parse::<usize>().ok())
|
||||
.filter(|limit| *limit > 0)
|
||||
.unwrap_or(4),
|
||||
.unwrap_or(1)
|
||||
.saturating_sub(1),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -908,21 +917,23 @@ impl Generator {
|
||||
.map_or((None, 0), |slot| (Some(slot.state), slot.last_store_tokens));
|
||||
self.executor.swap_resident_state(&mut incoming_state)?;
|
||||
if let (Some(previous), Some(outgoing)) = (self.resident_active.take(), incoming_state) {
|
||||
// ponytail: four resident sessions bound memory; raise
|
||||
// DS4_RESIDENT_SESSIONS when the machine can hold more KV state.
|
||||
if self.resident_sessions.len() >= self.resident_limit {
|
||||
let evicted = self.resident_sessions.keys().next().cloned();
|
||||
if let Some(evicted) = evicted {
|
||||
self.resident_sessions.remove(&evicted);
|
||||
// DS4 defaults to one active session; opt into its batched-server
|
||||
// behavior with DS4_RESIDENT_SESSIONS when memory permits.
|
||||
if self.resident_limit != 0 {
|
||||
if self.resident_sessions.len() >= self.resident_limit {
|
||||
let evicted = self.resident_sessions.keys().next().cloned();
|
||||
if let Some(evicted) = evicted {
|
||||
self.resident_sessions.remove(&evicted);
|
||||
}
|
||||
}
|
||||
self.resident_sessions.insert(
|
||||
previous,
|
||||
ResidentSlot {
|
||||
state: outgoing,
|
||||
last_store_tokens: self.last_store_tokens,
|
||||
},
|
||||
);
|
||||
}
|
||||
self.resident_sessions.insert(
|
||||
previous,
|
||||
ResidentSlot {
|
||||
state: outgoing,
|
||||
last_store_tokens: self.last_store_tokens,
|
||||
},
|
||||
);
|
||||
}
|
||||
self.resident_active = Some(key);
|
||||
self.checkpoint = None;
|
||||
|
||||
Reference in New Issue
Block a user