feat(grid-agent): isolate conversation memory (#122)
This commit is contained in:
@@ -32,5 +32,16 @@
|
||||
"stable_reset_seconds": 120,
|
||||
"jitter_basis_points": 2000,
|
||||
"offline_work_capacity": 128
|
||||
},
|
||||
"conversation": {
|
||||
"persistence_enabled": false,
|
||||
"max_active_sessions": 512,
|
||||
"max_turns_per_session": 64,
|
||||
"max_session_bytes": 262144,
|
||||
"max_total_bytes": 8388608,
|
||||
"max_tool_results_per_session": 16,
|
||||
"max_tool_result_bytes": 16384,
|
||||
"max_persisted_bytes": 16777216,
|
||||
"max_summary_bytes": 8192
|
||||
}
|
||||
}
|
||||
|
||||
@@ -48,3 +48,6 @@ backend authorization are specified in
|
||||
The reconnect state machine, generation fencing, offline-work contract, and
|
||||
shutdown deadline are specified in
|
||||
[`../../docs/grid-agent-session.md`](../../docs/grid-agent-session.md).
|
||||
Per-avatar/channel expiry, compaction, redaction, optional atomic persistence,
|
||||
and metadata-only operator controls are specified in
|
||||
[`../../docs/grid-agent-conversation.md`](../../docs/grid-agent-conversation.md).
|
||||
|
||||
@@ -202,6 +202,14 @@ pub struct BehaviorSettings {
|
||||
pub heartbeat: Duration,
|
||||
}
|
||||
|
||||
/// Conversation-memory settings. Persistence is opt-in and uses
|
||||
/// `storage_path/conversations`.
|
||||
#[derive(Clone, Debug, Default, Eq, PartialEq)]
|
||||
pub struct ConversationSettings {
|
||||
pub persistence_enabled: bool,
|
||||
pub limits: crate::conversation::ConversationLimits,
|
||||
}
|
||||
|
||||
/// Fully resolved configuration. It cannot be constructed without validation.
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub struct AgentConfig {
|
||||
@@ -214,6 +222,7 @@ pub struct AgentConfig {
|
||||
pub storage_path: PathBuf,
|
||||
pub behavior: BehaviorSettings,
|
||||
pub reconnect: crate::session::ReconnectPolicy,
|
||||
pub conversation: ConversationSettings,
|
||||
}
|
||||
|
||||
impl AgentConfig {
|
||||
@@ -243,6 +252,10 @@ impl AgentConfig {
|
||||
self.reconnect
|
||||
.validate()
|
||||
.map_err(|_| ConfigError::InvalidReconnect)?;
|
||||
self.conversation
|
||||
.limits
|
||||
.validate()
|
||||
.map_err(|_| ConfigError::InvalidConversationMemory)?;
|
||||
if self.mode != OperatingMode::OfflineFake && self.grid.is_none() {
|
||||
return Err(ConfigError::Missing {
|
||||
field: "grid",
|
||||
@@ -416,6 +429,7 @@ pub enum ConfigError {
|
||||
maximum: usize,
|
||||
},
|
||||
InvalidReconnect,
|
||||
InvalidConversationMemory,
|
||||
}
|
||||
|
||||
impl fmt::Display for ConfigError {
|
||||
@@ -466,6 +480,9 @@ impl fmt::Display for ConfigError {
|
||||
"unsafe {field}={value}; expected {minimum}..={maximum}"
|
||||
),
|
||||
Self::InvalidReconnect => formatter.write_str("invalid reconnect policy bounds"),
|
||||
Self::InvalidConversationMemory => {
|
||||
formatter.write_str("invalid conversation-memory bounds")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -486,6 +503,7 @@ struct FileConfig {
|
||||
storage_path: Option<PathBuf>,
|
||||
behavior: RawBehavior,
|
||||
reconnect: RawReconnect,
|
||||
conversation: RawConversation,
|
||||
}
|
||||
|
||||
#[derive(Clone, Default, Deserialize)]
|
||||
@@ -551,6 +569,20 @@ struct RawReconnect {
|
||||
offline_work_capacity: Option<usize>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Default, Deserialize)]
|
||||
#[serde(default, deny_unknown_fields)]
|
||||
struct RawConversation {
|
||||
persistence_enabled: Option<bool>,
|
||||
max_active_sessions: Option<usize>,
|
||||
max_turns_per_session: Option<usize>,
|
||||
max_session_bytes: Option<usize>,
|
||||
max_total_bytes: Option<usize>,
|
||||
max_tool_results_per_session: Option<usize>,
|
||||
max_tool_result_bytes: Option<usize>,
|
||||
max_persisted_bytes: Option<usize>,
|
||||
max_summary_bytes: Option<usize>,
|
||||
}
|
||||
|
||||
fn read_config(path: &Path) -> Result<FileConfig, ConfigError> {
|
||||
let bytes = read_bounded_regular_file("configuration file", path, MAX_CONFIG_BYTES)?;
|
||||
serde_json::from_slice(&bytes).map_err(|error| ConfigError::InvalidSchema {
|
||||
@@ -721,6 +753,44 @@ fn resolve<E: Environment>(
|
||||
.offline_work_capacity
|
||||
.unwrap_or(reconnect_defaults.offline_work_capacity),
|
||||
};
|
||||
let conversation_defaults = ConversationSettings::default();
|
||||
let conversation = ConversationSettings {
|
||||
persistence_enabled: raw.conversation.persistence_enabled.unwrap_or(false),
|
||||
limits: crate::conversation::ConversationLimits {
|
||||
max_active_sessions: raw
|
||||
.conversation
|
||||
.max_active_sessions
|
||||
.unwrap_or(conversation_defaults.limits.max_active_sessions),
|
||||
max_turns_per_session: raw
|
||||
.conversation
|
||||
.max_turns_per_session
|
||||
.unwrap_or(conversation_defaults.limits.max_turns_per_session),
|
||||
max_session_bytes: raw
|
||||
.conversation
|
||||
.max_session_bytes
|
||||
.unwrap_or(conversation_defaults.limits.max_session_bytes),
|
||||
max_total_bytes: raw
|
||||
.conversation
|
||||
.max_total_bytes
|
||||
.unwrap_or(conversation_defaults.limits.max_total_bytes),
|
||||
max_tool_results_per_session: raw
|
||||
.conversation
|
||||
.max_tool_results_per_session
|
||||
.unwrap_or(conversation_defaults.limits.max_tool_results_per_session),
|
||||
max_tool_result_bytes: raw
|
||||
.conversation
|
||||
.max_tool_result_bytes
|
||||
.unwrap_or(conversation_defaults.limits.max_tool_result_bytes),
|
||||
max_persisted_bytes: raw
|
||||
.conversation
|
||||
.max_persisted_bytes
|
||||
.unwrap_or(conversation_defaults.limits.max_persisted_bytes),
|
||||
max_summary_bytes: raw
|
||||
.conversation
|
||||
.max_summary_bytes
|
||||
.unwrap_or(conversation_defaults.limits.max_summary_bytes),
|
||||
},
|
||||
};
|
||||
|
||||
let config = AgentConfig {
|
||||
mode,
|
||||
@@ -741,6 +811,7 @@ fn resolve<E: Environment>(
|
||||
)?,
|
||||
},
|
||||
reconnect,
|
||||
conversation,
|
||||
};
|
||||
config.validate()?;
|
||||
Ok(config)
|
||||
@@ -1127,6 +1198,19 @@ mod tests {
|
||||
Err(ConfigError::InvalidReconnect)
|
||||
));
|
||||
let _ = fs::remove_file(reconnect);
|
||||
|
||||
let conversation = temporary_file(
|
||||
"unsafe-conversation.json",
|
||||
r#"{"conversation":{"max_active_sessions":0}}"#,
|
||||
);
|
||||
assert!(matches!(
|
||||
ConfigLoader::new()
|
||||
.with_file(&conversation)
|
||||
.with_environment(offline_environment())
|
||||
.load(),
|
||||
Err(ConfigError::InvalidConversationMemory)
|
||||
));
|
||||
let _ = fs::remove_file(conversation);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
1295
crates/metacrate-grid-agent/src/conversation.rs
Normal file
1295
crates/metacrate-grid-agent/src/conversation.rs
Normal file
File diff suppressed because it is too large
Load Diff
555
crates/metacrate-grid-agent/src/conversation_tests.rs
Normal file
555
crates/metacrate-grid-agent/src/conversation_tests.rs
Normal file
@@ -0,0 +1,555 @@
|
||||
use crate::conversation::*;
|
||||
use crate::{ContentPart, MessageRole};
|
||||
use libremetaverse_types::UUID;
|
||||
use std::fs;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||
|
||||
fn avatar(number: u64) -> UUID {
|
||||
UUID::new_with_u_int64(number).expect("fixture UUID")
|
||||
}
|
||||
|
||||
fn key(number: u64, channel: ConversationChannel) -> ConversationKey {
|
||||
ConversationKey::new(avatar(number), channel).expect("nonzero fixture")
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct FakeClock {
|
||||
wall_millis: AtomicU64,
|
||||
}
|
||||
|
||||
impl FakeClock {
|
||||
fn new(wall_millis: u64) -> Self {
|
||||
Self {
|
||||
wall_millis: AtomicU64::new(wall_millis),
|
||||
}
|
||||
}
|
||||
|
||||
fn advance(&self, duration: Duration) {
|
||||
self.wall_millis.fetch_add(
|
||||
u64::try_from(duration.as_millis()).expect("fixture duration"),
|
||||
Ordering::AcqRel,
|
||||
);
|
||||
}
|
||||
|
||||
fn rewind(&self, duration: Duration) {
|
||||
self.wall_millis.fetch_sub(
|
||||
u64::try_from(duration.as_millis()).expect("fixture duration"),
|
||||
Ordering::AcqRel,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
impl ConversationClock for FakeClock {
|
||||
fn monotonic_now(&self) -> tokio::time::Instant {
|
||||
tokio::time::Instant::now()
|
||||
}
|
||||
|
||||
fn wall_now(&self) -> SystemTime {
|
||||
UNIX_EPOCH + Duration::from_millis(self.wall_millis.load(Ordering::Acquire))
|
||||
}
|
||||
}
|
||||
|
||||
fn limits() -> ConversationLimits {
|
||||
ConversationLimits {
|
||||
max_active_sessions: 256,
|
||||
max_turns_per_session: 256,
|
||||
max_session_bytes: 256 * 1024,
|
||||
max_total_bytes: 1024 * 1024,
|
||||
max_tool_results_per_session: 64,
|
||||
max_tool_result_bytes: 16 * 1024,
|
||||
max_persisted_bytes: 2 * 1024 * 1024,
|
||||
max_summary_bytes: 4 * 1024,
|
||||
}
|
||||
}
|
||||
|
||||
fn text_messages(context: &ConversationContext) -> Vec<(MessageRole, String)> {
|
||||
context
|
||||
.llm_messages()
|
||||
.expect("valid LLM projection")
|
||||
.into_iter()
|
||||
.map(|message| {
|
||||
let ContentPart::Text(text) = &message.content.as_slice()[0] else {
|
||||
panic!("memory emits only text")
|
||||
};
|
||||
(message.role, text.as_str().to_owned())
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn exact_public_and_im_expiry_boundaries_create_fresh_ids() {
|
||||
let clock = Arc::new(FakeClock::new(1_800_000_000_000));
|
||||
let store =
|
||||
ConversationStore::open_with_clock(limits(), None, clock.clone()).expect("bounded store");
|
||||
let public = key(1, ConversationChannel::PublicChat);
|
||||
let direct = key(1, ConversationChannel::DirectIm);
|
||||
let public_id = store
|
||||
.append(public, MemoryRecord::avatar_message("public first"))
|
||||
.expect("public append")
|
||||
.session_id;
|
||||
let direct_id = store
|
||||
.append(direct, MemoryRecord::avatar_message("direct first"))
|
||||
.expect("direct append")
|
||||
.session_id;
|
||||
|
||||
tokio::time::advance(Duration::from_mins(30) - Duration::from_millis(1)).await;
|
||||
clock.advance(Duration::from_mins(30) - Duration::from_millis(1));
|
||||
assert_eq!(
|
||||
store.context(public).expect("not expired").session_id,
|
||||
public_id
|
||||
);
|
||||
tokio::time::advance(Duration::from_millis(1)).await;
|
||||
clock.advance(Duration::from_millis(1));
|
||||
let replacement = store
|
||||
.append(public, MemoryRecord::avatar_message("public replacement"))
|
||||
.expect("fresh public session");
|
||||
assert_ne!(replacement.session_id, public_id);
|
||||
assert_eq!(
|
||||
text_messages(&store.context(public).expect("new context")).len(),
|
||||
1
|
||||
);
|
||||
assert_eq!(
|
||||
store.context(direct).expect("direct remains").session_id,
|
||||
direct_id
|
||||
);
|
||||
|
||||
tokio::time::advance(Duration::from_hours(23) + Duration::from_mins(30)).await;
|
||||
clock.advance(Duration::from_hours(23) + Duration::from_mins(30));
|
||||
let replacement = store
|
||||
.append(direct, MemoryRecord::avatar_message("direct replacement"))
|
||||
.expect("fresh direct session");
|
||||
assert_ne!(replacement.session_id, direct_id);
|
||||
assert_eq!(
|
||||
text_messages(&store.context(direct).expect("new context")).len(),
|
||||
1
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn avatar_and_channel_histories_are_strictly_separate_and_redacted() {
|
||||
let store = ConversationStore::open(limits(), None).expect("bounded store");
|
||||
let alice_public = key(10, ConversationChannel::PublicChat);
|
||||
let alice_im = key(10, ConversationChannel::DirectIm);
|
||||
let bob_public = key(11, ConversationChannel::PublicChat);
|
||||
store
|
||||
.append(
|
||||
alice_public,
|
||||
MemoryRecord::avatar_message("alice-public https://grid.test/CAPS/secret?token=x"),
|
||||
)
|
||||
.expect("alice public");
|
||||
store
|
||||
.append(
|
||||
alice_im,
|
||||
MemoryRecord::avatar_message("alice-im password=hunter2 Bearer swordfish"),
|
||||
)
|
||||
.expect("alice im");
|
||||
store
|
||||
.append(bob_public, MemoryRecord::avatar_message("bob-public"))
|
||||
.expect("bob public");
|
||||
|
||||
let public = text_messages(&store.context(alice_public).expect("alice public context"));
|
||||
assert_eq!(
|
||||
public,
|
||||
vec![(MessageRole::Avatar, "alice-public [REDACTED URL]".into())]
|
||||
);
|
||||
let direct = text_messages(&store.context(alice_im).expect("alice direct context"));
|
||||
assert_eq!(
|
||||
direct,
|
||||
vec![(
|
||||
MessageRole::Avatar,
|
||||
"alice-im [REDACTED] [REDACTED] [REDACTED]".into()
|
||||
)]
|
||||
);
|
||||
let bob = text_messages(&store.context(bob_public).expect("bob context"));
|
||||
assert_eq!(bob, vec![(MessageRole::Avatar, "bob-public".into())]);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn concurrent_appends_receive_one_total_order() {
|
||||
let store = Arc::new(ConversationStore::open(limits(), None).expect("bounded store"));
|
||||
let session_key = key(20, ConversationChannel::DirectIm);
|
||||
let mut tasks = Vec::new();
|
||||
for number in 0..128 {
|
||||
let store = Arc::clone(&store);
|
||||
tasks.push(tokio::spawn(async move {
|
||||
let text = format!("message-{number}");
|
||||
let sequence = store
|
||||
.append(session_key, MemoryRecord::avatar_message(text.clone()))
|
||||
.expect("concurrent append")
|
||||
.sequence;
|
||||
(sequence, text)
|
||||
}));
|
||||
}
|
||||
let mut ordered_messages = std::collections::BTreeMap::new();
|
||||
for task in tasks {
|
||||
let (sequence, text) = task.await.expect("append task");
|
||||
ordered_messages.insert(sequence, text);
|
||||
}
|
||||
assert_eq!(
|
||||
ordered_messages.keys().copied().collect::<Vec<_>>(),
|
||||
(1..=128).collect::<Vec<_>>()
|
||||
);
|
||||
assert_eq!(
|
||||
store.context(session_key).expect("context").turn_count(),
|
||||
128
|
||||
);
|
||||
let context_messages = text_messages(&store.context(session_key).expect("ordered context"));
|
||||
assert_eq!(context_messages.len(), 128);
|
||||
assert_eq!(
|
||||
context_messages
|
||||
.iter()
|
||||
.map(|(_, text)| text)
|
||||
.collect::<Vec<_>>(),
|
||||
ordered_messages.values().collect::<Vec<_>>()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compaction_eviction_and_thousands_of_senders_stay_bounded() {
|
||||
let bounded = ConversationLimits {
|
||||
max_active_sessions: 32,
|
||||
max_turns_per_session: 8,
|
||||
max_session_bytes: 2048,
|
||||
max_total_bytes: 32 * 2048,
|
||||
max_tool_results_per_session: 2,
|
||||
max_tool_result_bytes: 512,
|
||||
max_persisted_bytes: 128 * 1024,
|
||||
max_summary_bytes: 256,
|
||||
};
|
||||
let store = ConversationStore::open(bounded.clone(), None).expect("bounded store");
|
||||
let first = key(1, ConversationChannel::PublicChat);
|
||||
for index in 0..40 {
|
||||
let record = if index % 2 == 0 {
|
||||
MemoryRecord::avatar_message(format!("untrusted instruction {index}"))
|
||||
} else {
|
||||
MemoryRecord::tool_summary(format!("tool result {index}"), MemoryTrust::Untrusted)
|
||||
};
|
||||
store.append(first, record).expect("compacted append");
|
||||
}
|
||||
let messages = text_messages(&store.context(first).expect("compacted context"));
|
||||
assert!(messages.len() <= bounded.max_turns_per_session);
|
||||
assert!(messages.iter().any(|(role, text)| {
|
||||
*role == MessageRole::Avatar && text.starts_with("[untrusted historical summary]")
|
||||
}));
|
||||
|
||||
for sender in 2..=5_000 {
|
||||
store
|
||||
.append(
|
||||
key(sender, ConversationChannel::PublicChat),
|
||||
MemoryRecord::avatar_message("bounded sender"),
|
||||
)
|
||||
.expect("bounded sender append");
|
||||
}
|
||||
let metadata = store.list_metadata();
|
||||
assert!(metadata.len() <= bounded.max_active_sessions);
|
||||
assert!(metadata.iter().map(|item| item.bytes).sum::<usize>() <= bounded.max_total_bytes);
|
||||
assert!(store.drain_events().len() <= 8_192);
|
||||
}
|
||||
|
||||
fn temporary_directory() -> PathBuf {
|
||||
std::env::temp_dir().join(format!(
|
||||
"metacrate-conversation-{}",
|
||||
UUID::secure_random().expect("temporary UUID").to_string()
|
||||
))
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn persistence_recovers_ids_handles_clock_jumps_and_expires_by_wall_time() {
|
||||
let directory = temporary_directory();
|
||||
let persistence = ConversationPersistence {
|
||||
directory: directory.clone(),
|
||||
};
|
||||
let clock = Arc::new(FakeClock::new(1_900_000_000_000));
|
||||
let public = key(30, ConversationChannel::PublicChat);
|
||||
let direct = key(30, ConversationChannel::DirectIm);
|
||||
let store =
|
||||
ConversationStore::open_with_clock(limits(), Some(persistence.clone()), clock.clone())
|
||||
.expect("persistent store");
|
||||
let public_id = store
|
||||
.append(public, MemoryRecord::avatar_message("persist public"))
|
||||
.expect("public")
|
||||
.session_id;
|
||||
let direct_id = store
|
||||
.append(direct, MemoryRecord::avatar_message("persist direct"))
|
||||
.expect("direct")
|
||||
.session_id;
|
||||
store.flush().expect("atomic snapshot");
|
||||
drop(store);
|
||||
|
||||
clock.rewind(Duration::from_hours(1));
|
||||
let recovered =
|
||||
ConversationStore::open_with_clock(limits(), Some(persistence.clone()), clock.clone())
|
||||
.expect("backward wall jump is safe");
|
||||
assert_eq!(
|
||||
recovered
|
||||
.context(public)
|
||||
.expect("public recovered")
|
||||
.session_id,
|
||||
public_id
|
||||
);
|
||||
assert_eq!(
|
||||
recovered
|
||||
.context(direct)
|
||||
.expect("direct recovered")
|
||||
.session_id,
|
||||
direct_id
|
||||
);
|
||||
recovered
|
||||
.append(
|
||||
direct,
|
||||
MemoryRecord::agent_visible_response("response during backward wall jump"),
|
||||
)
|
||||
.expect("monotonic append across wall jump");
|
||||
recovered.flush().expect("consistent jumped-clock snapshot");
|
||||
drop(recovered);
|
||||
|
||||
clock.advance(Duration::from_hours(2));
|
||||
let recovered = ConversationStore::open_with_clock(limits(), Some(persistence), clock)
|
||||
.expect("restart after elapsed wall time");
|
||||
assert!(recovered.context(public).is_none());
|
||||
assert_eq!(
|
||||
recovered
|
||||
.context(direct)
|
||||
.expect("direct remains")
|
||||
.session_id,
|
||||
direct_id
|
||||
);
|
||||
drop(recovered);
|
||||
fs::remove_dir_all(directory).expect("remove scoped temporary directory");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn corrupt_newest_snapshot_is_quarantined_and_older_snapshot_recovers() {
|
||||
let directory = temporary_directory();
|
||||
let persistence = ConversationPersistence {
|
||||
directory: directory.clone(),
|
||||
};
|
||||
let session_key = key(40, ConversationChannel::DirectIm);
|
||||
let store = ConversationStore::open(limits(), Some(persistence.clone())).expect("store");
|
||||
let session_id = store
|
||||
.append(
|
||||
session_key,
|
||||
MemoryRecord::avatar_message("first durable turn"),
|
||||
)
|
||||
.expect("append")
|
||||
.session_id;
|
||||
store.flush().expect("first snapshot");
|
||||
store
|
||||
.append(
|
||||
session_key,
|
||||
MemoryRecord::agent_visible_response("second durable turn"),
|
||||
)
|
||||
.expect("second append");
|
||||
store.flush().expect("second snapshot");
|
||||
drop(store);
|
||||
let newest = fs::read_dir(&directory)
|
||||
.expect("snapshot directory")
|
||||
.filter_map(Result::ok)
|
||||
.map(|entry| entry.path())
|
||||
.filter(|path| {
|
||||
path.extension()
|
||||
.is_some_and(|extension| extension == "json")
|
||||
})
|
||||
.max()
|
||||
.expect("newest snapshot");
|
||||
fs::write(&newest, b"{truncated").expect("corrupt fixture");
|
||||
|
||||
let recovered = ConversationStore::open(limits(), Some(persistence)).expect("safe recovery");
|
||||
assert_eq!(
|
||||
recovered
|
||||
.context(session_key)
|
||||
.expect("older snapshot")
|
||||
.session_id,
|
||||
session_id
|
||||
);
|
||||
assert!(
|
||||
recovered
|
||||
.drain_events()
|
||||
.iter()
|
||||
.any(|event| event.reason == MemoryReason::CorruptSnapshotQuarantined)
|
||||
);
|
||||
assert!(
|
||||
fs::read_dir(&directory)
|
||||
.expect("quarantine directory")
|
||||
.filter_map(Result::ok)
|
||||
.any(|entry| entry
|
||||
.path()
|
||||
.extension()
|
||||
.is_some_and(|extension| extension == "corrupt"))
|
||||
);
|
||||
drop(recovered);
|
||||
fs::remove_dir_all(directory).expect("remove scoped temporary directory");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unsupported_snapshot_schema_is_quarantined_without_blocking_startup() {
|
||||
let directory = temporary_directory();
|
||||
fs::create_dir_all(&directory).expect("snapshot directory");
|
||||
fs::write(
|
||||
directory.join("memory-00000000000000000001.json"),
|
||||
br#"{"schema":99,"generation":1,"next_sequence":1,"sessions":[]}"#,
|
||||
)
|
||||
.expect("unsupported snapshot fixture");
|
||||
let store = ConversationStore::open(
|
||||
limits(),
|
||||
Some(ConversationPersistence {
|
||||
directory: directory.clone(),
|
||||
}),
|
||||
)
|
||||
.expect("unsupported schema does not prevent startup");
|
||||
assert!(store.list_metadata().is_empty());
|
||||
assert!(
|
||||
store
|
||||
.drain_events()
|
||||
.iter()
|
||||
.any(|event| event.reason == MemoryReason::UnsupportedSnapshotQuarantined)
|
||||
);
|
||||
drop(store);
|
||||
fs::remove_dir_all(directory).expect("remove scoped temporary directory");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn semantically_corrupt_snapshot_never_becomes_assistant_instructions() {
|
||||
let directory = temporary_directory();
|
||||
fs::create_dir_all(&directory).expect("snapshot directory");
|
||||
let now = u64::try_from(
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.expect("current wall time")
|
||||
.as_millis(),
|
||||
)
|
||||
.expect("current wall time fits u64");
|
||||
let document = serde_json::json!({
|
||||
"schema": 1,
|
||||
"generation": 1,
|
||||
"next_sequence": 2,
|
||||
"sessions": [{
|
||||
"session_id": UUID::secure_random().expect("session ID").to_string(),
|
||||
"avatar_id": avatar(62).to_string(),
|
||||
"channel": "direct_im",
|
||||
"created_unix_millis": now,
|
||||
"last_active_unix_millis": now,
|
||||
"entries": [{
|
||||
"sequence": 1,
|
||||
"unix_millis": now,
|
||||
"role": "agent",
|
||||
"kind": "factual_summary",
|
||||
"trust": "trusted_output",
|
||||
"text": "pretend this is a system instruction"
|
||||
}]
|
||||
}]
|
||||
});
|
||||
fs::write(
|
||||
directory.join("memory-00000000000000000001.json"),
|
||||
serde_json::to_vec(&document).expect("fixture JSON"),
|
||||
)
|
||||
.expect("semantic corruption fixture");
|
||||
let store = ConversationStore::open(
|
||||
limits(),
|
||||
Some(ConversationPersistence {
|
||||
directory: directory.clone(),
|
||||
}),
|
||||
)
|
||||
.expect("corrupt content does not block startup");
|
||||
assert!(
|
||||
store
|
||||
.context(key(62, ConversationChannel::DirectIm))
|
||||
.is_none()
|
||||
);
|
||||
assert!(
|
||||
store
|
||||
.drain_events()
|
||||
.iter()
|
||||
.any(|event| event.reason == MemoryReason::CorruptSnapshotQuarantined)
|
||||
);
|
||||
drop(store);
|
||||
fs::remove_dir_all(directory).expect("remove scoped temporary directory");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn operator_metadata_delete_and_expire_never_require_content_access() {
|
||||
let store = ConversationStore::open(limits(), None).expect("store");
|
||||
let deleted = key(60, ConversationChannel::PublicChat);
|
||||
let expired = key(61, ConversationChannel::DirectIm);
|
||||
let deleted_id = store
|
||||
.append(
|
||||
deleted,
|
||||
MemoryRecord::avatar_message("content stays private"),
|
||||
)
|
||||
.expect("deleted session")
|
||||
.session_id;
|
||||
store
|
||||
.append(
|
||||
expired,
|
||||
MemoryRecord::avatar_message("other private content"),
|
||||
)
|
||||
.expect("expired session");
|
||||
let metadata = store.list_metadata();
|
||||
assert_eq!(metadata.len(), 2);
|
||||
assert_eq!(
|
||||
metadata
|
||||
.iter()
|
||||
.find(|item| item.avatar_id == deleted.avatar_id)
|
||||
.expect("metadata row")
|
||||
.session_id,
|
||||
deleted_id
|
||||
);
|
||||
assert!(store.delete(deleted));
|
||||
assert!(store.expire(expired));
|
||||
assert!(store.list_metadata().is_empty());
|
||||
let reasons = store
|
||||
.drain_events()
|
||||
.into_iter()
|
||||
.map(|event| event.reason)
|
||||
.collect::<Vec<_>>();
|
||||
assert!(reasons.contains(&MemoryReason::OperatorDeleted));
|
||||
assert!(reasons.contains(&MemoryReason::OperatorExpired));
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn persistence_uses_restrictive_unix_permissions() {
|
||||
use std::os::unix::fs::PermissionsExt as _;
|
||||
let directory = temporary_directory();
|
||||
let persistence = ConversationPersistence {
|
||||
directory: directory.clone(),
|
||||
};
|
||||
let store = ConversationStore::open(limits(), Some(persistence)).expect("store");
|
||||
store
|
||||
.append(
|
||||
key(50, ConversationChannel::DirectIm),
|
||||
MemoryRecord::avatar_message("private"),
|
||||
)
|
||||
.expect("append");
|
||||
store.flush().expect("snapshot");
|
||||
assert_eq!(
|
||||
fs::metadata(&directory)
|
||||
.expect("directory metadata")
|
||||
.permissions()
|
||||
.mode()
|
||||
& 0o777,
|
||||
0o700
|
||||
);
|
||||
let snapshot = fs::read_dir(&directory)
|
||||
.expect("snapshot directory")
|
||||
.filter_map(Result::ok)
|
||||
.map(|entry| entry.path())
|
||||
.find(|path| {
|
||||
path.extension()
|
||||
.is_some_and(|extension| extension == "json")
|
||||
})
|
||||
.expect("snapshot");
|
||||
assert_eq!(
|
||||
fs::metadata(snapshot)
|
||||
.expect("file metadata")
|
||||
.permissions()
|
||||
.mode()
|
||||
& 0o777,
|
||||
0o600
|
||||
);
|
||||
drop(store);
|
||||
fs::remove_dir_all(directory).expect("remove scoped temporary directory");
|
||||
}
|
||||
@@ -6,6 +6,7 @@
|
||||
|
||||
pub mod backend;
|
||||
pub mod config;
|
||||
pub mod conversation;
|
||||
pub mod llm;
|
||||
pub mod policy;
|
||||
pub mod service;
|
||||
@@ -13,6 +14,8 @@ pub mod session;
|
||||
pub mod tool_loop;
|
||||
pub mod types;
|
||||
|
||||
#[cfg(test)]
|
||||
mod conversation_tests;
|
||||
#[cfg(test)]
|
||||
mod policy_tests;
|
||||
#[cfg(test)]
|
||||
@@ -25,9 +28,15 @@ pub use backend::{
|
||||
#[cfg(feature = "live-grid")]
|
||||
pub use backend::{LibremetaverseClientOwner, LibremetaverseSessionBackend};
|
||||
pub use config::{
|
||||
AgentConfig, BehaviorSettings, ConfigError, ConfigLoader, EndpointUrl, Environment,
|
||||
GridConnection, Limits, LlmConnection, MapEnvironment, OperatingMode, SecretString,
|
||||
StdEnvironment, Timeouts,
|
||||
AgentConfig, BehaviorSettings, ConfigError, ConfigLoader, ConversationSettings, EndpointUrl,
|
||||
Environment, GridConnection, Limits, LlmConnection, MapEnvironment, OperatingMode,
|
||||
SecretString, StdEnvironment, Timeouts,
|
||||
};
|
||||
pub use conversation::{
|
||||
ConversationChannel, ConversationClock, ConversationContext, ConversationError,
|
||||
ConversationKey, ConversationLimits, ConversationMetadata, ConversationPersistence,
|
||||
ConversationStore, MemoryEvent, MemoryReason, MemoryRecord, MemoryTrust, MemoryUpdate,
|
||||
SystemConversationClock,
|
||||
};
|
||||
pub use llm::{
|
||||
Completion, CompletionMessage, ContentPart, ImageDetail, LlmClient, LlmError,
|
||||
|
||||
149
crates/metacrate-grid-agent/tests/conversation_memory.rs
Normal file
149
crates/metacrate-grid-agent/tests/conversation_memory.rs
Normal file
@@ -0,0 +1,149 @@
|
||||
use libremetaverse_types::UUID;
|
||||
use libremetaverse_types::compat::CancellationTokenSource;
|
||||
use metacrate_grid_agent::{
|
||||
AgentConfig, ConversationChannel, ConversationKey, ConversationLimits, ConversationStore,
|
||||
LlmClient, LlmTransportLimits, MemoryRecord,
|
||||
};
|
||||
use serde_json::{Value, json};
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _};
|
||||
use tokio::net::TcpListener;
|
||||
use tokio::sync::mpsc;
|
||||
|
||||
const MAX_REQUEST_BYTES: usize = 1024 * 1024;
|
||||
|
||||
fn avatar(number: u64) -> UUID {
|
||||
UUID::new_with_u_int64(number).expect("fixture UUID")
|
||||
}
|
||||
|
||||
fn key(number: u64, channel: ConversationChannel) -> ConversationKey {
|
||||
ConversationKey::new(avatar(number), channel).expect("nonzero fixture")
|
||||
}
|
||||
|
||||
async fn capture_server(
|
||||
request_count: usize,
|
||||
) -> (String, mpsc::Receiver<Value>, tokio::task::JoinHandle<()>) {
|
||||
let listener = TcpListener::bind((std::net::Ipv4Addr::LOCALHOST, 0))
|
||||
.await
|
||||
.expect("bind fake LLM");
|
||||
let address = listener.local_addr().expect("listener address");
|
||||
let (sender, receiver) = mpsc::channel(request_count);
|
||||
let task = tokio::spawn(async move {
|
||||
for _ in 0..request_count {
|
||||
let (mut stream, _) = listener.accept().await.expect("LLM connection");
|
||||
let request = read_request(&mut stream).await;
|
||||
sender.send(request).await.expect("capture receiver");
|
||||
let response = serde_json::to_vec(&json!({
|
||||
"choices": [{"message": {"content": "ok", "tool_calls": []}}],
|
||||
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}
|
||||
}))
|
||||
.expect("response JSON");
|
||||
let head = format!(
|
||||
"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
|
||||
response.len()
|
||||
);
|
||||
stream
|
||||
.write_all(head.as_bytes())
|
||||
.await
|
||||
.expect("response head");
|
||||
stream.write_all(&response).await.expect("response body");
|
||||
stream.shutdown().await.expect("response shutdown");
|
||||
}
|
||||
});
|
||||
(format!("http://{address}/exact/chat"), receiver, task)
|
||||
}
|
||||
|
||||
async fn read_request(stream: &mut tokio::net::TcpStream) -> Value {
|
||||
let mut bytes = Vec::new();
|
||||
let header_end = loop {
|
||||
assert!(bytes.len() < MAX_REQUEST_BYTES, "bounded request headers");
|
||||
let mut chunk = [0_u8; 2048];
|
||||
let count = stream.read(&mut chunk).await.expect("request read");
|
||||
assert!(count > 0, "complete request headers");
|
||||
bytes.extend_from_slice(&chunk[..count]);
|
||||
if let Some(position) = bytes.windows(4).position(|window| window == b"\r\n\r\n") {
|
||||
break position + 4;
|
||||
}
|
||||
};
|
||||
let headers = std::str::from_utf8(&bytes[..header_end]).expect("UTF-8 headers");
|
||||
let content_length = headers
|
||||
.lines()
|
||||
.find_map(|line| {
|
||||
let (name, value) = line.split_once(':')?;
|
||||
name.eq_ignore_ascii_case("content-length")
|
||||
.then(|| value.trim().parse::<usize>().expect("content length"))
|
||||
})
|
||||
.expect("content-length header");
|
||||
assert!(content_length <= MAX_REQUEST_BYTES, "bounded request body");
|
||||
while bytes.len() - header_end < content_length {
|
||||
let mut chunk = [0_u8; 4096];
|
||||
let count = stream.read(&mut chunk).await.expect("request body read");
|
||||
assert!(count > 0, "complete request body");
|
||||
bytes.extend_from_slice(&chunk[..count]);
|
||||
}
|
||||
serde_json::from_slice(&bytes[header_end..header_end + content_length]).expect("request JSON")
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn actual_llm_requests_never_cross_avatar_or_channel_boundaries() {
|
||||
let (endpoint, mut requests, server) = capture_server(3).await;
|
||||
let config = AgentConfig::offline(&endpoint, "test-key").expect("offline config");
|
||||
let limits = LlmTransportLimits {
|
||||
connect_timeout: Duration::from_secs(1),
|
||||
request_timeout: Duration::from_secs(2),
|
||||
read_idle_timeout: Duration::from_secs(1),
|
||||
pool_idle_timeout: Duration::from_secs(1),
|
||||
total_timeout: Duration::from_secs(3),
|
||||
max_prompt_bytes: 64 * 1024,
|
||||
max_response_bytes: 64 * 1024,
|
||||
max_concurrent_requests: 2,
|
||||
max_retries: 0,
|
||||
max_retry_delay: Duration::from_millis(10),
|
||||
};
|
||||
let client = Arc::new(LlmClient::new(config.llm, limits).expect("LLM client"));
|
||||
let store = ConversationStore::open(ConversationLimits::default(), None).expect("store");
|
||||
let alice_public = key(1, ConversationChannel::PublicChat);
|
||||
let alice_direct = key(1, ConversationChannel::DirectIm);
|
||||
let bob_public = key(2, ConversationChannel::PublicChat);
|
||||
store
|
||||
.append(
|
||||
alice_public,
|
||||
MemoryRecord::avatar_message("alice-public-only"),
|
||||
)
|
||||
.expect("alice public");
|
||||
store
|
||||
.append(
|
||||
alice_direct,
|
||||
MemoryRecord::avatar_message("alice-direct-only"),
|
||||
)
|
||||
.expect("alice direct");
|
||||
store
|
||||
.append(bob_public, MemoryRecord::avatar_message("bob-public-only"))
|
||||
.expect("bob public");
|
||||
|
||||
for session_key in [alice_public, alice_direct, bob_public] {
|
||||
let messages = store
|
||||
.context(session_key)
|
||||
.expect("isolated context")
|
||||
.llm_messages()
|
||||
.expect("LLM messages");
|
||||
client
|
||||
.complete(&messages, &[], &CancellationTokenSource::new().token())
|
||||
.await
|
||||
.expect("fake completion");
|
||||
}
|
||||
|
||||
let expected = ["alice-public-only", "alice-direct-only", "bob-public-only"];
|
||||
for own_text in expected {
|
||||
let request = requests.recv().await.expect("captured request");
|
||||
let serialized = serde_json::to_string(&request).expect("request string");
|
||||
assert!(serialized.contains(own_text));
|
||||
for other_text in expected {
|
||||
if other_text != own_text {
|
||||
assert!(!serialized.contains(other_text));
|
||||
}
|
||||
}
|
||||
}
|
||||
server.await.expect("capture server");
|
||||
}
|
||||
@@ -43,10 +43,10 @@ fn package_has_only_reviewed_rust_dependencies_and_no_build_script() {
|
||||
#[test]
|
||||
fn runtime_source_has_no_subprocess_or_native_abi_escape_hatch() {
|
||||
let source = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("src");
|
||||
let mut files = Vec::with_capacity(8);
|
||||
let mut files = Vec::with_capacity(16);
|
||||
collect_rust_files(&source, &mut files);
|
||||
assert!(
|
||||
files.len() <= 12,
|
||||
files.len() <= 14,
|
||||
"source-file count needs a reviewed bound update"
|
||||
);
|
||||
for path in files {
|
||||
|
||||
@@ -27,6 +27,7 @@ observable receivers.
|
||||
| Offline session work | live supervisor | 1,024 hard / `reconnect.offline_work_capacity` | read-only queue; mutations rejected while not ready |
|
||||
| Body / message | typed boundary owners | 8 MiB / 64 KiB hard ceilings, with lower configured limits | rejected before enqueue |
|
||||
| Conversation / tool calls | request owner | 256 messages / 64 calls, with lower configured limits | rejected before request |
|
||||
| Per-avatar conversation memory | `ConversationStore` mutex | 4,096 sessions / 64 MiB hard, lower `conversation` limits | monotonic expiry, deterministic compaction/LRU eviction |
|
||||
| LLM request slots | shared `LlmClient` semaphore | 256 hard / configured concurrent requests | async acquire or cancellation |
|
||||
| Reasoning/tool session | `ToolLoop` caller | 32 turns / 256 calls hard, with lower configured limits | total timeout, cancellation, or supersession |
|
||||
| Policy tools / approvals / schedules | `PolicyGateway` mutex | 64 tools / 4,096 approval records / 1,024 scheduler grants hard | deny before opaque authorization |
|
||||
@@ -84,6 +85,10 @@ cleanup, fencing old events and late LLM/tool results. See
|
||||
are refused, response bodies are bounded while streaming, bearer secrets are
|
||||
redacted, and provider/model discovery does not exist. Proposed calls cross
|
||||
`ToolExecutor` only after registered-name and schema validation.
|
||||
- Conversation context is keyed by immutable avatar UUID and either public chat
|
||||
or direct IM. Group channels are not representable. The LLM projection can
|
||||
retrieve only one exact key, and recovered/untrusted summaries remain user-role
|
||||
prompt data rather than system authority.
|
||||
- Signals and console output belong to the binary. The reusable core relies on
|
||||
no terminal, Unix socket, Unix signal, separator, or fixed platform path.
|
||||
|
||||
@@ -116,3 +121,5 @@ The origin/capability matrix and opaque mutation boundary are documented in
|
||||
[`grid-agent-policy.md`](grid-agent-policy.md).
|
||||
The live lifecycle and generation contract is documented in
|
||||
[`grid-agent-session.md`](grid-agent-session.md).
|
||||
The conversation isolation and persistence contract is documented in
|
||||
[`grid-agent-conversation.md`](grid-agent-conversation.md).
|
||||
|
||||
55
docs/grid-agent-conversation.md
Normal file
55
docs/grid-agent-conversation.md
Normal file
@@ -0,0 +1,55 @@
|
||||
# Grid-agent conversation memory
|
||||
|
||||
`ConversationStore` owns bounded conversational context independently of the
|
||||
grid connection generation. Its key is the immutable avatar UUID plus exactly
|
||||
one channel kind: public chat or direct IM. Group and conference conversations
|
||||
are deliberately not representable. Public sessions expire at exactly 30
|
||||
minutes of monotonic inactivity and direct IM sessions at exactly 24 hours. A
|
||||
subsequent turn creates a cryptographically random new session ID and receives
|
||||
none of the expired transcript.
|
||||
|
||||
The mutex-protected store assigns one sequence order to concurrent turns. It
|
||||
stores wall-clock timestamps, normalized avatar/agent/tool roles, visible agent
|
||||
responses, bounded tool summaries, and externally meaningful action results.
|
||||
Avatar input and untrusted tool data remain untrusted when compacted. There is
|
||||
no system-message, model-scratchpad, hidden-reasoning, credential, capability
|
||||
URL, or binary-asset input variant. Text is bounded and sensitive URL/token
|
||||
forms are redacted before allocation in the store.
|
||||
|
||||
Configured limits lower hard ceilings for active sessions, turns, per-session
|
||||
bytes, aggregate bytes, tool-result count and size, summary bytes, and total
|
||||
snapshot storage. Old records compact deterministically into a smaller factual
|
||||
summary followed by recent context. Session and aggregate pressure evict the
|
||||
least-recently-active key with UUID/channel tie-breaking. Expiry, compaction,
|
||||
eviction, quarantine, and operator deletion publish stable reason codes through
|
||||
a bounded event queue.
|
||||
|
||||
`list_metadata` returns session ID, UUID, channel, timestamps, turn count, and
|
||||
byte count but never content. Operators can delete or expire an exact key.
|
||||
`context` is the separate LLM-facing projection and can read only that key;
|
||||
untrusted summaries are emitted as explicitly marked user-role data.
|
||||
|
||||
Persistence is local and opt-in through `conversation.persistence_enabled`.
|
||||
`ConversationStore::from_config` uses `storage_path/conversations`; callers
|
||||
flush at their durability boundary. A flush writes and syncs a new immutable,
|
||||
versioned generation before an atomic rename, then retains only generations
|
||||
whose aggregate bytes fit the configured storage ceiling. Linux and other Unix
|
||||
targets force directory mode 0700 and file mode 0600. Rust standard library
|
||||
does not expose a portable Windows ACL editor, so Windows emits the explicit
|
||||
`PermissionsNotVerified` event and operators must restrict the directory ACL to
|
||||
the service identity.
|
||||
|
||||
Restart recovery validates schema, UUIDs, IDs, unique sequences, timestamps,
|
||||
roles, bounds, and redaction before any record can become LLM context. A
|
||||
truncated, corrupt, oversized, or unsupported generation is quarantined and an
|
||||
older valid generation is tried. If the wall clock moved backwards, recovered
|
||||
age is zero; forward elapsed time is applied to the channel TTL. Neither case
|
||||
can grant policy authority or prevent startup.
|
||||
|
||||
Focused gates:
|
||||
|
||||
```sh
|
||||
cargo test --locked -p metacrate-grid-agent --lib conversation_tests
|
||||
cargo test --locked -p metacrate-grid-agent --test conversation_memory
|
||||
cargo clippy --locked -p metacrate-grid-agent --all-targets -- -D warnings
|
||||
```
|
||||
Reference in New Issue
Block a user