feat(grid-agent): supervise grid sessions (#121)
Some checks failed
CI / rust-skia (Rust only) (push) Successful in 2m47s
CI / required (push) Failing after 2m54s

This commit is contained in:
2026-08-17 21:56:15 +00:00
parent e3b9d575f9
commit 3553c83ffa
13 changed files with 2377 additions and 36 deletions

View File

@@ -18,6 +18,9 @@ sha2 = "0.11"
tokio = { version = "1.53.1", features = ["macros", "rt", "sync", "time"] }
url = "2.5.8"
[dev-dependencies]
tokio = { version = "1.53.1", features = ["test-util"] }
[target.'cfg(any(unix, windows))'.dependencies]
tokio = { version = "1.53.1", features = ["io-util", "net", "rt-multi-thread", "signal"] }

View File

@@ -6,9 +6,10 @@ OpenSim grid agent. It contains a reusable library and the
offline: it publishes a deterministic ready event, accepts control commands,
and shuts down both owned tasks without contacting a grid or LLM. The library
also provides the bounded exact-endpoint LLM transport and tool loop for live
adapters. The `live-grid` feature exposes the side-effect-free owner for the existing
`libremetaverse::GridClient`; later live adapters must extend that manager graph
instead of adding a protocol client.
adapters. The `live-grid` feature exposes the owner for the existing
`libremetaverse::GridClient`. Its supervised session adapter reuses that
client's native `NetworkManager` for login, event-queue readiness, disconnect
notifications, and logout instead of adding a protocol client.
The LLM connection identity has exactly two resolved fields:
`llm.endpoint_url` and `llm.api_key`. The endpoint is used exactly as supplied;
@@ -44,3 +45,6 @@ compatibility envelope, retry rules, and tool-loop safety contract. The central
origin matrix, approval binding, budgets, prompt-data boundary, and opaque
backend authorization are specified in
[`../../docs/grid-agent-policy.md`](../../docs/grid-agent-policy.md).
The reconnect state machine, generation fencing, offline-work contract, and
shutdown deadline are specified in
[`../../docs/grid-agent-session.md`](../../docs/grid-agent-session.md).

View File

@@ -7,6 +7,10 @@ use std::error::Error;
use std::fmt;
use std::future::Future;
use std::pin::Pin;
#[cfg(feature = "live-grid")]
use std::sync::Arc;
#[cfg(feature = "live-grid")]
use std::sync::atomic::{AtomicBool, Ordering};
use tokio::sync::mpsc;
pub type BackendFuture<'a, T> = Pin<Box<dyn Future<Output = T> + Send + 'a>>;
@@ -82,11 +86,45 @@ impl OfflineGridBackend {
/// Live-feature composition owner that guarantees production adapters reuse
/// the existing `libremetaverse` manager/client graph.
#[cfg(feature = "live-grid")]
#[derive(Debug)]
pub struct LibremetaverseClientOwner {
client: libremetaverse::GridClient,
}
#[cfg(feature = "live-grid")]
impl fmt::Debug for LibremetaverseClientOwner {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("LibremetaverseClientOwner")
.field("client", &"[NATIVE CLIENT REDACTED]")
.finish()
}
}
/// Feature-gated session adapter that reuses the existing native
/// `NetworkManager` login, event, disconnect, and logout lifecycle.
#[cfg(feature = "live-grid")]
#[derive(Clone)]
pub struct LibremetaverseSessionBackend {
network: libremetaverse::NetworkManager,
login_url: String,
first_name: String,
last_name: String,
password: crate::config::SecretString,
}
#[cfg(feature = "live-grid")]
impl fmt::Debug for LibremetaverseSessionBackend {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("LibremetaverseSessionBackend")
.field("transport_connected", &self.network.native_connected())
.field("login_url", &"[REDACTED ENDPOINT]")
.field("avatar", &"[REDACTED IDENTITY]")
.field("password", &self.password)
.finish_non_exhaustive()
}
}
#[cfg(feature = "live-grid")]
impl LibremetaverseClientOwner {
/// Builds the shared client composition root without starting login or I/O.
@@ -107,6 +145,213 @@ impl LibremetaverseClientOwner {
pub const fn client(&self) -> &libremetaverse::GridClient {
&self.client
}
/// Creates the supervised live-session adapter without logging in.
pub fn session_backend(
&self,
connection: crate::config::GridConnection,
) -> Result<LibremetaverseSessionBackend, BackendError> {
let avatar = connection.avatar_name.trim();
let (first_name, last_name) = avatar
.split_once(char::is_whitespace)
.map_or((avatar, "Resident"), |(first, last)| (first, last.trim()));
if first_name.is_empty() || last_name.is_empty() {
return Err(BackendError::Configuration {
component: "grid avatar identity",
});
}
Ok(LibremetaverseSessionBackend {
network: self.client.network(),
login_url: connection.login_url.expose_url().to_owned(),
first_name: first_name.to_owned(),
last_name: last_name.to_owned(),
password: connection.password,
})
}
}
#[cfg(feature = "live-grid")]
struct LibremetaverseSession {
generation: u64,
network: libremetaverse::NetworkManager,
signals: mpsc::Receiver<crate::session::SessionSignal>,
subscriptions: Vec<libremetaverse_types::compat::Subscription>,
}
#[cfg(feature = "live-grid")]
impl crate::session::GridSession for LibremetaverseSession {
fn generation(&self) -> u64 {
self.generation
}
fn next_signal(
&mut self,
cancellation: CancellationToken,
) -> crate::session::SessionFuture<'_, crate::session::SessionSignal> {
Box::pin(async move {
tokio::select! {
signal = self.signals.recv() => signal.unwrap_or_else(|| {
crate::session::SessionSignal::Disconnected(
crate::session::SessionFailure::new(
crate::session::SessionFailureKind::TransientTransport,
),
)
}),
() = cancellation.cancelled() => crate::session::SessionSignal::Disconnected(
crate::session::SessionFailure::new(
crate::session::SessionFailureKind::TransientTransport,
),
),
}
})
}
fn logout(
mut self: Box<Self>,
cancellation: CancellationToken,
) -> crate::session::SessionFuture<'static, Result<(), crate::session::SessionFailure>> {
Box::pin(async move {
self.subscriptions.clear();
self.network
.native_logout_async(Some(cancellation))
.await
.map_err(|_| {
crate::session::SessionFailure::new(
crate::session::SessionFailureKind::TransientTransport,
)
})
})
}
}
#[cfg(feature = "live-grid")]
impl crate::session::GridSessionBackend for LibremetaverseSessionBackend {
fn login(
&self,
generation: u64,
cancellation: CancellationToken,
) -> crate::session::SessionFuture<
'_,
Result<Box<dyn crate::session::GridSession>, crate::session::SessionFailure>,
> {
Box::pin(async move {
let mut params = self
.network
.native_default_login_params(
self.first_name.clone(),
self.last_name.clone(),
self.password.expose_secret().to_owned(),
"MetaCrate".to_owned(),
env!("CARGO_PKG_VERSION").to_owned(),
)
.map_err(|_| {
crate::session::SessionFailure::new(
crate::session::SessionFailureKind::InvalidConfiguration,
)
})?;
params.uri.clone_from(&self.login_url);
params.start = "last".to_owned();
let logged_in = self
.network
.native_login(params, Some(cancellation))
.await
.map_err(|_| {
crate::session::SessionFailure::new(
crate::session::SessionFailureKind::TransientTransport,
)
})?;
if !logged_in {
return Err(classify_native_login_failure(
&self.network.native_login_error_key(),
));
}
// Native login has already installed the current simulator and
// parsed inventory state. The existing GridClient managers retain
// world/inventory and movement-worker ownership; this generation
// adds only one disconnect and one readiness subscription.
let (sender, signals) = mpsc::channel(8);
let disconnected_sender = sender.clone();
let disconnected = self
.network
.native_subscribe_disconnected(Arc::new(move |event| {
let kind = match event.reason() {
libremetaverse::NetworkManagerDisconnectType::NetworkTimeout => {
crate::session::SessionFailureKind::TransientTransport
}
libremetaverse::NetworkManagerDisconnectType::ServerInitiated => {
crate::session::SessionFailureKind::Kicked
}
libremetaverse::NetworkManagerDisconnectType::SimShutdown => {
crate::session::SessionFailureKind::Maintenance
}
libremetaverse::NetworkManagerDisconnectType::ClientInitiated => {
crate::session::SessionFailureKind::TransientTransport
}
};
let _ =
disconnected_sender.try_send(crate::session::SessionSignal::Disconnected(
crate::session::SessionFailure::new(kind),
));
}));
let ready_sender = sender.clone();
let ready_once = Arc::new(AtomicBool::new(false));
let callback_ready = Arc::clone(&ready_once);
let ready = self
.network
.native_subscribe_event_queue_running(Arc::new(move |_| {
if !callback_ready.swap(true, Ordering::AcqRel) {
let _ = ready_sender.try_send(crate::session::SessionSignal::Ready);
}
}));
if !self.network.native_connected() {
let _ = sender.try_send(crate::session::SessionSignal::Disconnected(
crate::session::SessionFailure::new(
crate::session::SessionFailureKind::TransientTransport,
),
));
} else if self
.network
.native_current_sim()
.and_then(|simulator| simulator.native_is_event_queue_running(None).ok())
.unwrap_or(false)
{
if !ready_once.swap(true, Ordering::AcqRel) {
let _ = sender.try_send(crate::session::SessionSignal::Ready);
}
}
let session: Box<dyn crate::session::GridSession> = Box::new(LibremetaverseSession {
generation,
network: self.network.clone(),
signals,
subscriptions: vec![disconnected, ready],
});
Ok(session)
})
}
fn flush_audit(
&self,
_cancellation: CancellationToken,
) -> crate::session::SessionFuture<'_, Result<(), crate::session::SessionFailure>> {
Box::pin(async { Ok(()) })
}
}
#[cfg(feature = "live-grid")]
fn classify_native_login_failure(error_key: &str) -> crate::session::SessionFailure {
let normalized = error_key.trim().to_ascii_lowercase();
let kind = if matches!(
normalized.as_str(),
"key" | "password" | "credential" | "account" | "username" | "user"
) {
crate::session::SessionFailureKind::InvalidCredentials
} else if normalized == "canceled" {
crate::session::SessionFailureKind::TransientTransport
} else {
crate::session::SessionFailureKind::ServerFailure
};
crate::session::SessionFailure::new(kind)
}
impl GridBackend for OfflineGridBackend {

View File

@@ -213,6 +213,7 @@ pub struct AgentConfig {
pub limits: Limits,
pub storage_path: PathBuf,
pub behavior: BehaviorSettings,
pub reconnect: crate::session::ReconnectPolicy,
}
impl AgentConfig {
@@ -239,6 +240,9 @@ impl AgentConfig {
/// Returns the first invalid field without performing I/O.
pub fn validate(&self) -> Result<(), ConfigError> {
validate_limits(&self.limits)?;
self.reconnect
.validate()
.map_err(|_| ConfigError::InvalidReconnect)?;
if self.mode != OperatingMode::OfflineFake && self.grid.is_none() {
return Err(ConfigError::Missing {
field: "grid",
@@ -411,6 +415,7 @@ pub enum ConfigError {
minimum: usize,
maximum: usize,
},
InvalidReconnect,
}
impl fmt::Display for ConfigError {
@@ -460,6 +465,7 @@ impl fmt::Display for ConfigError {
formatter,
"unsafe {field}={value}; expected {minimum}..={maximum}"
),
Self::InvalidReconnect => formatter.write_str("invalid reconnect policy bounds"),
}
}
}
@@ -479,6 +485,7 @@ struct FileConfig {
limits: RawLimits,
storage_path: Option<PathBuf>,
behavior: RawBehavior,
reconnect: RawReconnect,
}
#[derive(Clone, Default, Deserialize)]
@@ -534,6 +541,16 @@ struct RawBehavior {
heartbeat_seconds: Option<u64>,
}
#[derive(Clone, Default, Deserialize)]
#[serde(default, deny_unknown_fields)]
struct RawReconnect {
initial_delay_milliseconds: Option<u64>,
maximum_delay_seconds: Option<u64>,
stable_reset_seconds: Option<u64>,
jitter_basis_points: Option<u16>,
offline_work_capacity: Option<usize>,
}
fn read_config(path: &Path) -> Result<FileConfig, ConfigError> {
let bytes = read_bounded_regular_file("configuration file", path, MAX_CONFIG_BYTES)?;
serde_json::from_slice(&bytes).map_err(|error| ConfigError::InvalidSchema {
@@ -666,31 +683,51 @@ fn resolve<E: Environment>(
};
validate_limits(&limits)?;
let timeouts = Timeouts {
startup: checked_duration(
"timeouts.startup_seconds",
raw.timeouts.startup.unwrap_or(30),
1,
300,
)?,
shutdown: checked_duration(
"timeouts.shutdown_seconds",
raw.timeouts.shutdown.unwrap_or(10),
1,
60,
)?,
request: checked_duration(
"timeouts.request_seconds",
raw.timeouts.request.unwrap_or(60),
1,
300,
)?,
};
let reconnect_defaults = crate::session::ReconnectPolicy::default();
let reconnect = crate::session::ReconnectPolicy {
initial_delay: Duration::from_millis(
raw.reconnect.initial_delay_milliseconds.unwrap_or(1_000),
),
maximum_delay: Duration::from_secs(raw.reconnect.maximum_delay_seconds.unwrap_or(60)),
stable_reset_after: Duration::from_secs(raw.reconnect.stable_reset_seconds.unwrap_or(120)),
shutdown_deadline: timeouts.shutdown,
jitter_basis_points: raw
.reconnect
.jitter_basis_points
.unwrap_or(reconnect_defaults.jitter_basis_points),
instance_seed: reconnect_defaults.instance_seed,
offline_work_capacity: raw
.reconnect
.offline_work_capacity
.unwrap_or(reconnect_defaults.offline_work_capacity),
};
let config = AgentConfig {
mode,
llm,
grid,
authorized_avatar_uuids,
timeouts: Timeouts {
startup: checked_duration(
"timeouts.startup_seconds",
raw.timeouts.startup.unwrap_or(30),
1,
300,
)?,
shutdown: checked_duration(
"timeouts.shutdown_seconds",
raw.timeouts.shutdown.unwrap_or(10),
1,
60,
)?,
request: checked_duration(
"timeouts.request_seconds",
raw.timeouts.request.unwrap_or(60),
1,
300,
)?,
},
timeouts,
limits,
storage_path: raw
.storage_path
@@ -703,6 +740,7 @@ fn resolve<E: Environment>(
300,
)?,
},
reconnect,
};
config.validate()?;
Ok(config)
@@ -1076,6 +1114,19 @@ mod tests {
Err(ConfigError::UnsafeLimit { .. })
));
let _ = fs::remove_file(path);
let reconnect = temporary_file(
"unsafe-reconnect.json",
r#"{"reconnect":{"initial_delay_milliseconds":0}}"#,
);
assert!(matches!(
ConfigLoader::new()
.with_file(&reconnect)
.with_environment(offline_environment())
.load(),
Err(ConfigError::InvalidReconnect)
));
let _ = fs::remove_file(reconnect);
}
#[test]

View File

@@ -9,18 +9,21 @@ pub mod config;
pub mod llm;
pub mod policy;
pub mod service;
pub mod session;
pub mod tool_loop;
pub mod types;
#[cfg(test)]
mod policy_tests;
#[cfg(test)]
mod session_tests;
#[cfg(feature = "live-grid")]
pub use backend::LibremetaverseClientOwner;
pub use backend::{
AuthorizedToolBackend, BackendError, BackendFuture, GridBackend, OfflineGridBackend,
WorldMutator,
};
#[cfg(feature = "live-grid")]
pub use backend::{LibremetaverseClientOwner, LibremetaverseSessionBackend};
pub use config::{
AgentConfig, BehaviorSettings, ConfigError, ConfigLoader, EndpointUrl, Environment,
GridConnection, Limits, LlmConnection, MapEnvironment, OperatingMode, SecretString,
@@ -39,6 +42,12 @@ pub use policy::{
ResourceCost, ResourceEstimator, Risk, SchedulerGrantId, UntrustedData, UntrustedSource,
};
pub use service::{AgentService, ServiceError, ServiceHandle, ServiceState};
pub use session::{
GridSession, GridSessionBackend, ReconnectPolicy, SessionControl, SessionFailure,
SessionFailureKind, SessionFuture, SessionObservation, SessionSignal, SessionState,
SessionStatus, SessionSupervisor, SessionSupervisorError, SessionSupervisorHandle, SessionWork,
WorkDisposition, WorkKind,
};
pub use tool_loop::{
HistorySummarizer, SessionGeneration, ToolExecution, ToolExecutor, ToolFuture, ToolLoop,
ToolLoopError, ToolLoopLimits, ToolLoopOutcome,

View File

@@ -4,6 +4,8 @@ use metacrate_grid_agent::{
use std::error::Error;
use std::fmt;
use std::path::PathBuf;
#[cfg(feature = "live-grid")]
use std::sync::Arc;
const MAX_ARGUMENTS: usize = 8;
@@ -84,9 +86,11 @@ async fn main() -> Result<(), Box<dyn Error>> {
return Ok(());
}
if config.mode != OperatingMode::OfflineFake {
#[cfg(feature = "live-grid")]
return run_live(config, options.run_once).await;
#[cfg(not(feature = "live-grid"))]
return Err(CliError(
"this architecture issue starts only the offline backend; live login is owned by a later milestone issue"
.into(),
"live grid mode requires rebuilding with --features live-grid".into(),
)
.into());
}
@@ -133,3 +137,101 @@ async fn main() -> Result<(), Box<dyn Error>> {
println!("grid agent stopped cleanly");
Ok(())
}
#[cfg(feature = "live-grid")]
async fn run_live(
config: metacrate_grid_agent::AgentConfig,
run_once: bool,
) -> Result<(), Box<dyn Error>> {
use metacrate_grid_agent::{
GridSessionBackend, LibremetaverseClientOwner, SessionObservation, SessionState,
SessionSupervisor,
};
let connection = config.grid.clone().ok_or_else(|| {
CliError("validated live configuration did not contain a grid connection".into())
})?;
let owner = LibremetaverseClientOwner::new()?;
let backend = owner.session_backend(connection)?;
let erased: Arc<dyn GridSessionBackend> = Arc::new(backend);
let mut handle = SessionSupervisor::new(
erased,
config.reconnect,
config.limits.control_queue,
config.limits.observable_queue,
)?
.start();
if run_once {
let readiness = tokio::time::timeout(config.timeouts.startup, async {
loop {
match handle.next_observation().await {
Some(SessionObservation::Transition { status, .. }) if status.agent_ready => {
return Ok::<(), CliError>(());
}
Some(SessionObservation::Transition {
status:
metacrate_grid_agent::SessionStatus {
state: SessionState::AuthenticationBlocked,
..
},
..
}) => {
return Err(CliError(
"grid authentication/configuration requires operator action".into(),
));
}
Some(_) => {}
None => {
return Err(CliError(
"session supervisor stopped before readiness".into(),
));
}
}
}
})
.await;
let readiness = match readiness {
Ok(result) => result,
Err(_) => Err(CliError("timed out waiting for full grid readiness".into())),
};
if let Err(error) = readiness {
handle.shutdown().await?;
return Err(error.into());
}
handle.shutdown().await?;
println!("grid agent completed one supervised login/logout cycle");
return Ok(());
}
println!("grid agent session supervisor started; press Ctrl-C to stop");
let mut signal_error = None;
loop {
tokio::select! {
signal = tokio::signal::ctrl_c() => {
if let Err(error) = signal {
signal_error = Some(error);
}
break;
}
event = handle.next_observation() => {
let Some(event) = event else { break; };
if let SessionObservation::Transition { status, reason, retry_in } = event {
println!(
"grid session state={} generation={} transport_connected={} agent_ready={} reason={reason:?} retry_in={retry_in:?}",
status.state.as_str(),
status.generation,
status.transport_connected,
status.agent_ready,
);
}
}
}
}
handle.shutdown().await?;
if let Some(error) = signal_error {
return Err(error.into());
}
println!("grid agent stopped cleanly");
Ok(())
}

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,526 @@
use crate::session::*;
use libremetaverse_types::compat::CancellationToken;
use std::collections::VecDeque;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::{Arc, Mutex};
use std::time::Duration;
#[derive(Clone)]
enum LoginPlan {
Failure {
after: Duration,
failure: SessionFailure,
},
Success {
after: Duration,
signals: VecDeque<(Duration, SessionSignal)>,
},
}
#[derive(Default)]
struct FakeStats {
logins: AtomicUsize,
logouts: AtomicUsize,
flushes: AtomicUsize,
active_sessions: AtomicUsize,
active_callbacks: AtomicUsize,
active_workers: AtomicUsize,
active_sockets: AtomicUsize,
max_sessions: AtomicUsize,
max_callbacks: AtomicUsize,
max_workers: AtomicUsize,
max_sockets: AtomicUsize,
}
impl FakeStats {
fn acquire(&self) {
update_max(&self.active_sessions, &self.max_sessions, 1);
update_max(&self.active_callbacks, &self.max_callbacks, 4);
update_max(&self.active_workers, &self.max_workers, 2);
update_max(&self.active_sockets, &self.max_sockets, 1);
}
fn release(&self) {
self.active_sessions.fetch_sub(1, Ordering::AcqRel);
self.active_callbacks.fetch_sub(4, Ordering::AcqRel);
self.active_workers.fetch_sub(2, Ordering::AcqRel);
self.active_sockets.fetch_sub(1, Ordering::AcqRel);
}
}
fn update_max(active: &AtomicUsize, maximum: &AtomicUsize, amount: usize) {
let next = active.fetch_add(amount, Ordering::AcqRel) + amount;
maximum.fetch_max(next, Ordering::AcqRel);
}
struct FakeBackend {
plans: Mutex<VecDeque<LoginPlan>>,
stats: Arc<FakeStats>,
}
impl FakeBackend {
fn new(plans: impl IntoIterator<Item = LoginPlan>) -> (Arc<Self>, Arc<FakeStats>) {
let stats = Arc::new(FakeStats::default());
(
Arc::new(Self {
plans: Mutex::new(plans.into_iter().collect()),
stats: Arc::clone(&stats),
}),
stats,
)
}
}
impl GridSessionBackend for FakeBackend {
fn login(
&self,
generation: u64,
cancellation: CancellationToken,
) -> SessionFuture<'_, Result<Box<dyn GridSession>, SessionFailure>> {
self.stats.logins.fetch_add(1, Ordering::AcqRel);
let plan = self
.plans
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.pop_front()
.unwrap_or(LoginPlan::Success {
after: Duration::ZERO,
signals: VecDeque::from([(Duration::ZERO, SessionSignal::Ready)]),
});
let stats = Arc::clone(&self.stats);
Box::pin(async move {
let after = match &plan {
LoginPlan::Failure { after, .. } | LoginPlan::Success { after, .. } => *after,
};
tokio::select! {
() = tokio::time::sleep(after) => {}
() = cancellation.cancelled() => {
return Err(SessionFailure::new(SessionFailureKind::TransientTransport));
}
}
match plan {
LoginPlan::Failure { failure, .. } => Err(failure),
LoginPlan::Success { signals, .. } => {
stats.acquire();
let session: Box<dyn GridSession> = Box::new(FakeSession {
generation,
signals,
stats,
released: false,
});
Ok(session)
}
}
})
}
fn flush_audit(
&self,
_cancellation: CancellationToken,
) -> SessionFuture<'_, Result<(), SessionFailure>> {
self.stats.flushes.fetch_add(1, Ordering::AcqRel);
Box::pin(async { Ok(()) })
}
}
struct FakeSession {
generation: u64,
signals: VecDeque<(Duration, SessionSignal)>,
stats: Arc<FakeStats>,
released: bool,
}
impl FakeSession {
fn release(&mut self) {
if !self.released {
self.released = true;
self.stats.release();
}
}
}
impl GridSession for FakeSession {
fn generation(&self) -> u64 {
self.generation
}
fn next_signal(&mut self, cancellation: CancellationToken) -> SessionFuture<'_, SessionSignal> {
let next = self.signals.pop_front();
Box::pin(async move {
if let Some((after, signal)) = next {
tokio::select! {
() = tokio::time::sleep(after) => signal,
() = cancellation.cancelled() => SessionSignal::Disconnected(
SessionFailure::new(SessionFailureKind::TransientTransport)
),
}
} else {
cancellation.cancelled().await;
SessionSignal::Disconnected(SessionFailure::new(
SessionFailureKind::TransientTransport,
))
}
})
}
fn logout(
mut self: Box<Self>,
_cancellation: CancellationToken,
) -> SessionFuture<'static, Result<(), SessionFailure>> {
self.stats.logouts.fetch_add(1, Ordering::AcqRel);
self.release();
Box::pin(async { Ok(()) })
}
}
impl Drop for FakeSession {
fn drop(&mut self) {
self.release();
}
}
fn ready_then(after: Duration, failure: SessionFailureKind) -> LoginPlan {
LoginPlan::Success {
after: Duration::ZERO,
signals: VecDeque::from([
(Duration::ZERO, SessionSignal::Ready),
(
after,
SessionSignal::Disconnected(SessionFailure::new(failure)),
),
]),
}
}
fn test_policy() -> ReconnectPolicy {
ReconnectPolicy {
initial_delay: Duration::from_secs(1),
maximum_delay: Duration::from_secs(8),
stable_reset_after: Duration::from_secs(10),
shutdown_deadline: Duration::from_secs(2),
jitter_basis_points: 0,
instance_seed: 7,
offline_work_capacity: 2,
}
}
#[test]
fn retry_hints_are_minimums_and_all_delays_remain_capped() {
let mut policy = test_policy();
policy.jitter_basis_points = 5_000;
for seed in 0..128 {
policy.instance_seed = seed;
let hinted = Duration::from_secs(6);
let delay = policy.retry_delay(1, Some(hinted));
assert!(delay >= hinted);
assert!(delay <= policy.maximum_delay);
assert!(policy.retry_delay(64, None) <= policy.maximum_delay);
}
}
fn start(plans: impl IntoIterator<Item = LoginPlan>) -> (SessionSupervisorHandle, Arc<FakeStats>) {
let (backend, stats) = FakeBackend::new(plans);
let erased: Arc<dyn GridSessionBackend> = backend;
let handle = SessionSupervisor::new(erased, test_policy(), 32, 256)
.expect("supervisor")
.start();
(handle, stats)
}
async fn settle() {
for _ in 0..20 {
tokio::task::yield_now().await;
}
}
async fn wait_state(handle: &SessionSupervisorHandle, expected: SessionState) {
for _ in 0..100 {
if handle.state() == expected {
return;
}
tokio::task::yield_now().await;
}
panic!("expected {expected:?}, got {:?}", handle.status());
}
#[tokio::test(start_paused = true)]
async fn paused_time_classifies_failures_backoff_flapping_and_stable_reset() {
let plans = [
LoginPlan::Failure {
after: Duration::ZERO,
failure: SessionFailure::new(SessionFailureKind::TransientTransport),
},
LoginPlan::Failure {
after: Duration::ZERO,
failure: SessionFailure::with_retry_after(
SessionFailureKind::Maintenance,
Duration::from_secs(5),
),
},
ready_then(Duration::from_secs(1), SessionFailureKind::Kicked),
ready_then(
Duration::from_secs(11),
SessionFailureKind::SimulatorDisconnected,
),
LoginPlan::Success {
after: Duration::ZERO,
signals: VecDeque::from([(Duration::ZERO, SessionSignal::Ready)]),
},
];
let (mut handle, stats) = start(plans);
wait_state(&handle, SessionState::Backoff).await;
assert_eq!(handle.status().consecutive_failures, 1);
tokio::time::advance(Duration::from_secs(1)).await;
settle().await;
wait_state(&handle, SessionState::Backoff).await;
assert_eq!(handle.status().consecutive_failures, 2);
tokio::time::advance(Duration::from_secs(5)).await;
settle().await;
wait_state(&handle, SessionState::Online).await;
let first_online = handle.status().generation;
tokio::time::advance(Duration::from_secs(1)).await;
settle().await;
wait_state(&handle, SessionState::Backoff).await;
assert!(!handle.accepts_result(first_online));
tokio::time::advance(Duration::from_secs(4)).await;
settle().await;
wait_state(&handle, SessionState::Online).await;
tokio::time::advance(Duration::from_secs(11)).await;
settle().await;
wait_state(&handle, SessionState::Backoff).await;
assert_eq!(handle.status().consecutive_failures, 1);
tokio::time::advance(Duration::from_secs(1)).await;
settle().await;
wait_state(&handle, SessionState::Online).await;
handle.shutdown().await.expect("shutdown");
assert_eq!(stats.flushes.load(Ordering::Acquire), 1);
}
#[tokio::test(start_paused = true)]
async fn invalid_credentials_and_configuration_block_until_operator_reconnect() {
for failure in [
SessionFailureKind::InvalidCredentials,
SessionFailureKind::InvalidConfiguration,
] {
let plans = [
LoginPlan::Failure {
after: Duration::ZERO,
failure: SessionFailure::new(failure),
},
LoginPlan::Success {
after: Duration::ZERO,
signals: VecDeque::from([(Duration::ZERO, SessionSignal::Ready)]),
},
];
let (mut handle, stats) = start(plans);
wait_state(&handle, SessionState::AuthenticationBlocked).await;
tokio::time::advance(Duration::from_hours(1)).await;
settle().await;
assert_eq!(stats.logins.load(Ordering::Acquire), 1);
handle
.control(SessionControl::ForceReconnect)
.await
.expect("force reconnect");
settle().await;
wait_state(&handle, SessionState::Online).await;
handle.shutdown().await.expect("shutdown");
}
}
#[tokio::test(start_paused = true)]
async fn manual_pause_logout_reconnect_and_offline_work_are_predictable() {
let (mut handle, stats) = start([LoginPlan::Success {
after: Duration::ZERO,
signals: VecDeque::from([(Duration::ZERO, SessionSignal::Ready)]),
}]);
wait_state(&handle, SessionState::Online).await;
let generation = handle.status().generation;
let token = handle
.generation_token(generation)
.expect("generation token");
handle.control(SessionControl::Pause).await.expect("pause");
settle().await;
wait_state(&handle, SessionState::Paused).await;
assert!(token.is_cancellation_requested());
assert!(!handle.accepts_result(generation));
assert_eq!(
handle
.submit(SessionWork::new("mutate", WorkKind::IdempotentMutation).expect("work"))
.await
.expect("decision"),
WorkDisposition::RejectedOfflineMutation
);
assert_eq!(
handle
.submit(SessionWork::new("read", WorkKind::ReadOnly).expect("work"))
.await
.expect("decision"),
WorkDisposition::Queued
);
handle
.control(SessionControl::Resume)
.await
.expect("resume");
settle().await;
wait_state(&handle, SessionState::Online).await;
assert_ne!(handle.status().generation, generation);
assert_eq!(
handle
.submit(SessionWork::new("read", WorkKind::ReadOnly).expect("work"))
.await
.expect("dedupe decision"),
WorkDisposition::RejectedDuplicate
);
let before_force = handle.status().generation;
handle
.control(SessionControl::ForceReconnect)
.await
.expect("active reconnect");
settle().await;
wait_state(&handle, SessionState::Online).await;
assert_ne!(handle.status().generation, before_force);
handle
.control(SessionControl::Logout)
.await
.expect("logout");
settle().await;
wait_state(&handle, SessionState::Stopped).await;
handle
.control(SessionControl::ForceReconnect)
.await
.expect("reconnect");
settle().await;
wait_state(&handle, SessionState::Online).await;
handle.shutdown().await.expect("shutdown");
assert_eq!(stats.active_sessions.load(Ordering::Acquire), 0);
}
#[tokio::test(start_paused = true)]
async fn transport_connection_is_distinct_from_full_agent_readiness() {
let (mut handle, _) = start([LoginPlan::Success {
after: Duration::ZERO,
signals: VecDeque::from([(Duration::from_secs(10), SessionSignal::Ready)]),
}]);
wait_state(&handle, SessionState::Degraded).await;
let degraded = handle.status();
assert!(degraded.transport_connected);
assert!(!degraded.agent_ready);
assert!(!handle.accepts_result(degraded.generation));
tokio::time::advance(Duration::from_secs(10)).await;
settle().await;
wait_state(&handle, SessionState::Online).await;
assert!(handle.status().agent_ready);
handle.shutdown().await.expect("shutdown");
}
#[tokio::test(start_paused = true)]
async fn shutdown_covers_degraded_authentication_blocked_stopped_and_online_states() {
let cases = [
(
LoginPlan::Success {
after: Duration::ZERO,
signals: VecDeque::from([(Duration::from_hours(1), SessionSignal::Ready)]),
},
SessionState::Degraded,
),
(
LoginPlan::Failure {
after: Duration::ZERO,
failure: SessionFailure::new(SessionFailureKind::InvalidCredentials),
},
SessionState::AuthenticationBlocked,
),
(
LoginPlan::Success {
after: Duration::ZERO,
signals: VecDeque::from([(Duration::ZERO, SessionSignal::Ready)]),
},
SessionState::Online,
),
];
for (plan, expected) in cases {
let (mut handle, stats) = start([plan]);
wait_state(&handle, expected).await;
if expected == SessionState::Online {
handle
.control(SessionControl::Logout)
.await
.expect("logout");
settle().await;
wait_state(&handle, SessionState::Stopped).await;
}
handle.shutdown().await.expect("shutdown");
assert_eq!(stats.active_sessions.load(Ordering::Acquire), 0);
assert_eq!(stats.flushes.load(Ordering::Acquire), 1);
}
}
#[tokio::test(start_paused = true)]
async fn fake_grid_reconnects_keep_exactly_one_resource_set_and_no_stale_execution() {
let plans = [
ready_then(Duration::from_secs(1), SessionFailureKind::Maintenance),
ready_then(Duration::from_secs(1), SessionFailureKind::Kicked),
LoginPlan::Success {
after: Duration::ZERO,
signals: VecDeque::from([(Duration::ZERO, SessionSignal::Ready)]),
},
];
let (mut handle, stats) = start(plans);
wait_state(&handle, SessionState::Online).await;
let stale = handle.status().generation;
tokio::time::advance(Duration::from_secs(1)).await;
settle().await;
tokio::time::advance(Duration::from_secs(1)).await;
settle().await;
wait_state(&handle, SessionState::Online).await;
tokio::time::advance(Duration::from_secs(1)).await;
settle().await;
tokio::time::advance(Duration::from_secs(2)).await;
settle().await;
wait_state(&handle, SessionState::Online).await;
assert!(!handle.accepts_result(stale));
assert!(handle.accepts_result(handle.status().generation));
assert_eq!(stats.max_sessions.load(Ordering::Acquire), 1);
assert_eq!(stats.max_callbacks.load(Ordering::Acquire), 4);
assert_eq!(stats.max_workers.load(Ordering::Acquire), 2);
assert_eq!(stats.max_sockets.load(Ordering::Acquire), 1);
handle.shutdown().await.expect("shutdown");
assert_eq!(stats.active_sessions.load(Ordering::Acquire), 0);
assert_eq!(stats.active_callbacks.load(Ordering::Acquire), 0);
assert_eq!(stats.active_workers.load(Ordering::Acquire), 0);
assert_eq!(stats.active_sockets.load(Ordering::Acquire), 0);
assert_eq!(stats.logouts.load(Ordering::Acquire), 3);
assert_eq!(stats.flushes.load(Ordering::Acquire), 1);
}
#[tokio::test(start_paused = true)]
async fn shutdown_is_cancellation_driven_during_connect_backoff_paused_and_online() {
let scenarios = [
LoginPlan::Success {
after: Duration::from_hours(1),
signals: VecDeque::new(),
},
LoginPlan::Failure {
after: Duration::ZERO,
failure: SessionFailure::new(SessionFailureKind::TransientTransport),
},
LoginPlan::Success {
after: Duration::ZERO,
signals: VecDeque::from([(Duration::ZERO, SessionSignal::Ready)]),
},
];
for (index, plan) in scenarios.into_iter().enumerate() {
let (mut handle, stats) = start([plan]);
settle().await;
if index == 2 {
wait_state(&handle, SessionState::Online).await;
handle.control(SessionControl::Pause).await.expect("pause");
settle().await;
wait_state(&handle, SessionState::Paused).await;
}
handle.shutdown().await.expect("bounded shutdown");
assert_eq!(handle.state(), SessionState::Stopped);
assert_eq!(stats.active_sessions.load(Ordering::Acquire), 0);
assert_eq!(stats.flushes.load(Ordering::Acquire), 1);
}
}

View File

@@ -46,7 +46,7 @@ fn runtime_source_has_no_subprocess_or_native_abi_escape_hatch() {
let mut files = Vec::with_capacity(8);
collect_rust_files(&source, &mut files);
assert!(
files.len() <= 10,
files.len() <= 12,
"source-file count needs a reviewed bound update"
);
for path in files {
@@ -86,6 +86,25 @@ fn world_backend_requires_the_opaque_policy_authorization() {
assert!(!backend.contains("decision: PolicyDecision"));
}
#[test]
fn live_session_adapter_reuses_only_the_native_network_manager_lifecycle() {
let root = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
let backend = fs::read_to_string(root.join("src/backend.rs")).expect("backend source");
for required in [
".native_default_login_params(",
".native_login(",
".native_subscribe_disconnected(",
".native_subscribe_event_queue_running(",
".native_logout_async(",
] {
assert!(
backend.contains(required),
"missing native lifecycle {required}"
);
}
assert!(!backend.contains("reqwest"));
}
fn collect_rust_files(directory: &Path, output: &mut Vec<PathBuf>) {
for entry in fs::read_dir(directory).expect("read source directory") {
let path = entry.expect("source entry").path();

View File

@@ -0,0 +1,227 @@
use libremetaverse_types::compat::CancellationToken;
use metacrate_grid_agent::{
GridSession, GridSessionBackend, ReconnectPolicy, SessionFailure, SessionFailureKind,
SessionFuture, SessionSignal, SessionState, SessionSupervisor, SessionSupervisorHandle,
SessionWork, WorkDisposition, WorkKind,
};
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::time::Duration;
#[derive(Default)]
struct ResourceStats {
attempts: AtomicUsize,
logouts: AtomicUsize,
flushes: AtomicUsize,
sessions: AtomicUsize,
callbacks: AtomicUsize,
workers: AtomicUsize,
sockets: AtomicUsize,
max_sessions: AtomicUsize,
max_callbacks: AtomicUsize,
max_workers: AtomicUsize,
max_sockets: AtomicUsize,
}
impl ResourceStats {
fn acquire(&self) {
acquire(&self.sessions, &self.max_sessions, 1);
acquire(&self.callbacks, &self.max_callbacks, 3);
acquire(&self.workers, &self.max_workers, 2);
acquire(&self.sockets, &self.max_sockets, 1);
}
fn release(&self) {
self.sessions.fetch_sub(1, Ordering::AcqRel);
self.callbacks.fetch_sub(3, Ordering::AcqRel);
self.workers.fetch_sub(2, Ordering::AcqRel);
self.sockets.fetch_sub(1, Ordering::AcqRel);
}
}
fn acquire(active: &AtomicUsize, maximum: &AtomicUsize, amount: usize) {
let next = active.fetch_add(amount, Ordering::AcqRel) + amount;
maximum.fetch_max(next, Ordering::AcqRel);
}
struct ReconnectingFakeGrid {
stats: Arc<ResourceStats>,
}
impl GridSessionBackend for ReconnectingFakeGrid {
fn login(
&self,
generation: u64,
_cancellation: CancellationToken,
) -> SessionFuture<'_, Result<Box<dyn GridSession>, SessionFailure>> {
let attempt = self.stats.attempts.fetch_add(1, Ordering::AcqRel) + 1;
self.stats.acquire();
let stats = Arc::clone(&self.stats);
Box::pin(async move {
let session: Box<dyn GridSession> = Box::new(FakeConnectedSession {
generation,
attempt,
step: 0,
stats,
released: false,
});
Ok(session)
})
}
fn flush_audit(
&self,
_cancellation: CancellationToken,
) -> SessionFuture<'_, Result<(), SessionFailure>> {
self.stats.flushes.fetch_add(1, Ordering::AcqRel);
Box::pin(async { Ok(()) })
}
}
struct FakeConnectedSession {
generation: u64,
attempt: usize,
step: u8,
stats: Arc<ResourceStats>,
released: bool,
}
impl FakeConnectedSession {
fn release(&mut self) {
if !self.released {
self.released = true;
self.stats.release();
}
}
}
impl GridSession for FakeConnectedSession {
fn generation(&self) -> u64 {
self.generation
}
fn next_signal(&mut self, cancellation: CancellationToken) -> SessionFuture<'_, SessionSignal> {
let attempt = self.attempt;
let step = self.step;
self.step = self.step.saturating_add(1);
Box::pin(async move {
if step == 0 {
return SessionSignal::Ready;
}
if attempt <= 2 {
tokio::select! {
() = tokio::time::sleep(Duration::from_secs(1)) => {
let kind = if attempt == 1 {
SessionFailureKind::Maintenance
} else {
SessionFailureKind::Kicked
};
SessionSignal::Disconnected(SessionFailure::new(kind))
}
() = cancellation.cancelled() => SessionSignal::Disconnected(
SessionFailure::new(SessionFailureKind::TransientTransport),
),
}
} else {
cancellation.cancelled().await;
SessionSignal::Disconnected(SessionFailure::new(
SessionFailureKind::TransientTransport,
))
}
})
}
fn logout(
mut self: Box<Self>,
_cancellation: CancellationToken,
) -> SessionFuture<'static, Result<(), SessionFailure>> {
self.stats.logouts.fetch_add(1, Ordering::AcqRel);
self.release();
Box::pin(async { Ok(()) })
}
}
impl Drop for FakeConnectedSession {
fn drop(&mut self) {
self.release();
}
}
async fn settle() {
for _ in 0..30 {
tokio::task::yield_now().await;
}
}
async fn wait_online(handle: &SessionSupervisorHandle) {
for _ in 0..100 {
if handle.state() == SessionState::Online {
return;
}
tokio::task::yield_now().await;
}
panic!("fake grid did not become ready: {:?}", handle.status());
}
#[tokio::test(start_paused = true)]
async fn repeated_reconnects_have_one_resource_set_no_replay_and_no_shutdown_leaks() {
let stats = Arc::new(ResourceStats::default());
let backend: Arc<dyn GridSessionBackend> = Arc::new(ReconnectingFakeGrid {
stats: Arc::clone(&stats),
});
let policy = ReconnectPolicy {
initial_delay: Duration::from_secs(1),
maximum_delay: Duration::from_secs(4),
stable_reset_after: Duration::from_secs(30),
shutdown_deadline: Duration::from_secs(3),
jitter_basis_points: 0,
instance_seed: 1,
offline_work_capacity: 8,
};
let mut handle = SessionSupervisor::new(backend, policy, 32, 256)
.expect("supervisor")
.start();
wait_online(&handle).await;
let stale_generation = handle.status().generation;
assert_eq!(
handle
.submit(SessionWork::new("response-1", WorkKind::ReadOnly).expect("work"))
.await
.expect("accepted"),
WorkDisposition::Accepted {
generation: stale_generation
}
);
tokio::time::advance(Duration::from_secs(1)).await;
settle().await;
tokio::time::advance(Duration::from_secs(1)).await;
settle().await;
wait_online(&handle).await;
tokio::time::advance(Duration::from_secs(1)).await;
settle().await;
tokio::time::advance(Duration::from_secs(2)).await;
settle().await;
wait_online(&handle).await;
assert!(!handle.accepts_result(stale_generation));
assert_eq!(
handle
.submit(SessionWork::new("response-1", WorkKind::ReadOnly).expect("work"))
.await
.expect("dedupe"),
WorkDisposition::RejectedDuplicate
);
assert_eq!(stats.max_sessions.load(Ordering::Acquire), 1);
assert_eq!(stats.max_callbacks.load(Ordering::Acquire), 3);
assert_eq!(stats.max_workers.load(Ordering::Acquire), 2);
assert_eq!(stats.max_sockets.load(Ordering::Acquire), 1);
handle.shutdown().await.expect("shutdown");
assert_eq!(stats.sessions.load(Ordering::Acquire), 0);
assert_eq!(stats.callbacks.load(Ordering::Acquire), 0);
assert_eq!(stats.workers.load(Ordering::Acquire), 0);
assert_eq!(stats.sockets.load(Ordering::Acquire), 0);
assert_eq!(stats.logouts.load(Ordering::Acquire), 3);
assert_eq!(stats.flushes.load(Ordering::Acquire), 1);
}