Add deterministic agent protocol harness (#133)
This commit is contained in:
337
crates/metacrate-grid-agent/tests/deterministic_harness.rs
Normal file
337
crates/metacrate-grid-agent/tests/deterministic_harness.rs
Normal file
@@ -0,0 +1,337 @@
|
||||
use libremetaverse_types::{UUID, compat::CancellationToken};
|
||||
use metacrate_grid_agent::testing::{
|
||||
FakeGrid, FakeLlmStep, GridScriptStep, HarnessMetrics, HarnessRandom, HarnessScript,
|
||||
adversarial_corpus,
|
||||
};
|
||||
use metacrate_grid_agent::{
|
||||
BehaviorRandom, BuildGrid, BuildPrim, BuildShape, CompletionMessage, GridSessionBackend,
|
||||
LandmarkGrid, LlmClient, LlmError, LlmTransportLimits, MessageRole, ReconnectPolicy,
|
||||
RoamingRandom, ScriptInventory, SessionState, SessionSupervisor,
|
||||
};
|
||||
use std::collections::BTreeSet;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
fn uuid(value: u32) -> UUID {
|
||||
UUID::new_with_string(format!("00000000-0000-4000-8000-{value:012}")).expect("fixture UUID")
|
||||
}
|
||||
|
||||
async fn settle() {
|
||||
for _ in 0..32 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn scripted_lifecycle_reconnects_with_generation_fencing_and_no_resources() {
|
||||
let grid = FakeGrid::scripted([
|
||||
GridScriptStep::Ready,
|
||||
GridScriptStep::MaintenanceDisconnect,
|
||||
GridScriptStep::Ready,
|
||||
GridScriptStep::Hold,
|
||||
]);
|
||||
let backend: Arc<dyn GridSessionBackend> = Arc::new(grid.clone());
|
||||
let mut handle = SessionSupervisor::new(
|
||||
backend,
|
||||
ReconnectPolicy {
|
||||
jitter_basis_points: 0,
|
||||
instance_seed: 73,
|
||||
..ReconnectPolicy::default()
|
||||
},
|
||||
16,
|
||||
64,
|
||||
)
|
||||
.expect("supervisor")
|
||||
.start();
|
||||
settle().await;
|
||||
assert_eq!(handle.state(), SessionState::Backoff);
|
||||
let stale = handle.status().generation;
|
||||
tokio::time::advance(Duration::from_secs(1)).await;
|
||||
settle().await;
|
||||
assert_eq!(handle.state(), SessionState::Online);
|
||||
assert!(handle.status().generation > stale);
|
||||
assert!(!handle.accepts_result(stale));
|
||||
handle.shutdown().await.expect("shutdown");
|
||||
assert_eq!(grid.active_sessions(), 0);
|
||||
assert_eq!(grid.max_sessions(), 1);
|
||||
assert!(grid.reproduction(73).contains("seed=73"));
|
||||
let operations = grid
|
||||
.evidence()
|
||||
.into_iter()
|
||||
.map(|event| event.operation)
|
||||
.collect::<BTreeSet<_>>();
|
||||
assert!(operations.contains("grid.login"));
|
||||
assert!(operations.contains("grid.logout"));
|
||||
assert!(operations.contains("audit.flush"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn every_mutating_fake_boundary_emits_hashed_protocol_evidence() {
|
||||
let grid = FakeGrid::scripted([]);
|
||||
let token = CancellationToken::default();
|
||||
let prim = BuildPrim {
|
||||
id: "root".into(),
|
||||
parent: None,
|
||||
shape: BuildShape::Box,
|
||||
position_millimeters: [1_000, 2_000, 3_000],
|
||||
scale_millimeters: [500, 500, 500],
|
||||
rotation_degrees: [0.0; 3],
|
||||
color_rgba: [1.0; 4],
|
||||
material: "wood".into(),
|
||||
texture_inventory_id: None,
|
||||
name: "fixture".into(),
|
||||
description: "fixture".into(),
|
||||
script: Some("default { state_entry() {} }".into()),
|
||||
};
|
||||
BuildGrid::validate_land(&grid, "region", &[prim.position_millimeters], token.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
let receipt = BuildGrid::create_prim(&grid, "tx-1", &prim, token.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
BuildGrid::configure_prim(&grid, &receipt, &prim, token.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
BuildGrid::insert_script(
|
||||
&grid,
|
||||
&receipt,
|
||||
prim.script.as_deref().unwrap(),
|
||||
token.clone(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
BuildGrid::confirm_prim(&grid, &receipt, token.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
BuildGrid::delete_owned_prim(&grid, &receipt, token.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
LandmarkGrid::accept_offer(&grid, "offer-1", token.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
LandmarkGrid::verify_landmark(&grid, uuid(1), uuid(2), 9, token.clone())
|
||||
.await
|
||||
.unwrap()
|
||||
);
|
||||
assert!(
|
||||
LandmarkGrid::teleport_landmark(&grid, uuid(2), token.clone())
|
||||
.await
|
||||
.unwrap()
|
||||
);
|
||||
let inventory = ScriptInventory::create_full_permission(
|
||||
&grid,
|
||||
metacrate_grid_agent::GeneratedScript {
|
||||
name: "safe script".into(),
|
||||
description: "generated".into(),
|
||||
source: "default { state_entry() {} }".into(),
|
||||
},
|
||||
token.clone(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
ScriptInventory::give_to(&grid, inventory, uuid(3), token)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let evidence = grid.evidence();
|
||||
let operations = evidence
|
||||
.iter()
|
||||
.map(|event| event.operation.as_str())
|
||||
.collect::<BTreeSet<_>>();
|
||||
for required in [
|
||||
"build.validate_land",
|
||||
"build.create",
|
||||
"build.configure",
|
||||
"build.insert_script",
|
||||
"build.confirm",
|
||||
"build.delete",
|
||||
"landmark.accept",
|
||||
"landmark.verify",
|
||||
"landmark.teleport",
|
||||
"inventory.create_script",
|
||||
"inventory.give_script",
|
||||
] {
|
||||
assert!(
|
||||
operations.contains(required),
|
||||
"missing protocol evidence {required}"
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
evidence
|
||||
.iter()
|
||||
.all(|event| event.payload_sha256.len() == 64)
|
||||
);
|
||||
assert!(!format!("{evidence:?}").contains("state_entry"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[allow(clippy::too_many_lines)]
|
||||
fn corpus_randomness_scripts_and_load_metrics_are_reproducible_and_redacted() {
|
||||
let corpus = adversarial_corpus();
|
||||
assert_eq!(corpus.len(), 17);
|
||||
let fields = corpus
|
||||
.iter()
|
||||
.map(|case| case.field)
|
||||
.collect::<BTreeSet<_>>();
|
||||
for required in [
|
||||
"public_chat",
|
||||
"instant_message",
|
||||
"object.name",
|
||||
"avatar.name",
|
||||
"approval_id",
|
||||
"generation",
|
||||
"journal",
|
||||
"all_logs",
|
||||
] {
|
||||
assert!(fields.contains(required));
|
||||
}
|
||||
let debug = format!("{corpus:?}");
|
||||
assert!(!debug.contains("METACRATE_SECRET_CANARY_9f31"));
|
||||
assert!(!debug.contains("ignore policy"));
|
||||
let first = HarnessRandom::new(99);
|
||||
let second = HarnessRandom::new(99);
|
||||
assert_eq!(
|
||||
BehaviorRandom::next_u64(&first),
|
||||
BehaviorRandom::next_u64(&second)
|
||||
);
|
||||
assert_eq!(
|
||||
RoamingRandom::interval_seconds(&first, 10, 20),
|
||||
RoamingRandom::interval_seconds(&second, 10, 20)
|
||||
);
|
||||
let script = HarnessScript {
|
||||
seed: 99,
|
||||
grid: vec![
|
||||
GridScriptStep::PublicChat {
|
||||
avatar_id: uuid(1).to_string(),
|
||||
text: "hello".into(),
|
||||
},
|
||||
GridScriptStep::InstantMessage {
|
||||
avatar_id: uuid(2).to_string(),
|
||||
text: "private".into(),
|
||||
},
|
||||
GridScriptStep::InventoryOffer {
|
||||
avatar_id: uuid(2).to_string(),
|
||||
inventory_id: uuid(3).to_string(),
|
||||
kind: "landmark".into(),
|
||||
},
|
||||
GridScriptStep::AvatarUpdate {
|
||||
avatar_id: uuid(1).to_string(),
|
||||
position: [1, 2, 3],
|
||||
},
|
||||
GridScriptStep::ObjectUpdate {
|
||||
object_id: uuid(4).to_string(),
|
||||
local_id: 7,
|
||||
position: [4, 5, 6],
|
||||
},
|
||||
GridScriptStep::MovementReceipt {
|
||||
operation: "walk".into(),
|
||||
},
|
||||
GridScriptStep::TeleportReceipt {
|
||||
region_id: uuid(5).to_string(),
|
||||
},
|
||||
GridScriptStep::ObjectCreated {
|
||||
transaction_id: "tx".into(),
|
||||
object_id: uuid(6).to_string(),
|
||||
},
|
||||
GridScriptStep::CapabilityReply {
|
||||
capability: "InventoryAPIv3".into(),
|
||||
status: 200,
|
||||
body: "SECRET_CANARY".into(),
|
||||
},
|
||||
GridScriptStep::DelayTicks { ticks: 2 },
|
||||
GridScriptStep::Duplicate {
|
||||
correlation_id: "duplicate".into(),
|
||||
},
|
||||
GridScriptStep::MalformedPacket,
|
||||
GridScriptStep::PartialFailure {
|
||||
operation: "build.configure".into(),
|
||||
},
|
||||
],
|
||||
llm: vec![
|
||||
FakeLlmStep::Text { body: "ok".into() },
|
||||
FakeLlmStep::ToolCalls {
|
||||
body: serde_json::json!({"choices":[]}),
|
||||
},
|
||||
FakeLlmStep::Malformed,
|
||||
FakeLlmStep::Oversized { bytes: 4096 },
|
||||
FakeLlmStep::RateLimited,
|
||||
FakeLlmStep::SlowStream { chunks: 3 },
|
||||
FakeLlmStep::Disconnect,
|
||||
FakeLlmStep::MultimodalAccepted {
|
||||
body: "seen".into(),
|
||||
},
|
||||
FakeLlmStep::MultimodalRejected,
|
||||
],
|
||||
};
|
||||
let script_debug = format!("{script:?}");
|
||||
assert!(script_debug.contains("script_sha256"));
|
||||
assert!(!script_debug.contains("SECRET_CANARY"));
|
||||
let mut metrics = HarnessMetrics::default();
|
||||
metrics.observe_queue(8);
|
||||
metrics.observe_queue(3);
|
||||
metrics.observe_memory(4096);
|
||||
metrics.observe_response_latency(7);
|
||||
metrics.observe_control_latency(3);
|
||||
metrics.record_dropped_events(2);
|
||||
assert_eq!(metrics.queue_depth_high_water, 8);
|
||||
assert_eq!(metrics.memory_bytes_high_water, 4096);
|
||||
assert_eq!(metrics.response_latency_ticks, 7);
|
||||
assert_eq!(metrics.control_latency_ticks, 3);
|
||||
assert_eq!(metrics.dropped_events, 2);
|
||||
metrics.assert_drained().unwrap();
|
||||
}
|
||||
|
||||
#[cfg(any(unix, windows))]
|
||||
#[tokio::test]
|
||||
async fn local_openai_peer_records_only_redacted_schema_and_scripts_multimodal_rejection() {
|
||||
use metacrate_grid_agent::testing::FakeOpenAiEndpoint;
|
||||
use metacrate_grid_agent::{BoundedText, BoundedVec, ContentPart, ImageDetail};
|
||||
let endpoint = FakeOpenAiEndpoint::start(vec![FakeLlmStep::MultimodalRejected])
|
||||
.await
|
||||
.expect("loopback endpoint");
|
||||
let config = metacrate_grid_agent::AgentConfig::offline(endpoint.url(), "SECRET_CANARY")
|
||||
.expect("offline config");
|
||||
let client = LlmClient::new(
|
||||
config.llm,
|
||||
LlmTransportLimits {
|
||||
max_retries: 3,
|
||||
..LlmTransportLimits::default()
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
let message = CompletionMessage {
|
||||
role: MessageRole::Avatar,
|
||||
content: BoundedVec::try_from_vec(
|
||||
"content",
|
||||
vec![ContentPart::Image {
|
||||
url: BoundedText::new("url", "data:image/png;base64,aW1hZ2U=").unwrap(),
|
||||
detail: ImageDetail::Low,
|
||||
}],
|
||||
)
|
||||
.unwrap(),
|
||||
tool_call_id: None,
|
||||
proposed_calls: BoundedVec::new(),
|
||||
};
|
||||
assert_eq!(
|
||||
client
|
||||
.complete(&[message], &[], &CancellationToken::default())
|
||||
.await
|
||||
.unwrap_err(),
|
||||
LlmError::MultimodalUnsupported
|
||||
);
|
||||
tokio::task::yield_now().await;
|
||||
let requests = endpoint.requests();
|
||||
assert_eq!(requests.len(), 1, "large request is never retried");
|
||||
assert!(requests[0].has_image);
|
||||
assert_eq!(requests[0].body_sha256.len(), 64);
|
||||
assert!(!format!("{requests:?}").contains("SECRET_CANARY"));
|
||||
let reproduction = endpoint.reproduction(99);
|
||||
assert!(reproduction.contains("seed=99"));
|
||||
assert!(!reproduction.contains("SECRET_CANARY"));
|
||||
for _ in 0..8 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
assert_eq!(endpoint.active_sockets(), 0);
|
||||
drop(endpoint);
|
||||
}
|
||||
Reference in New Issue
Block a user