feat(grid-agent): supervise grid sessions (#121)
Some checks failed
CI / rust-skia (Rust only) (push) Successful in 2m47s
CI / required (push) Failing after 2m54s

This commit is contained in:
2026-08-17 21:56:15 +00:00
parent e3b9d575f9
commit 3553c83ffa
13 changed files with 2377 additions and 36 deletions

View File

@@ -25,5 +25,12 @@
"storage_path": "data/grid-agent",
"behavior": {
"heartbeat_seconds": 30
},
"reconnect": {
"initial_delay_milliseconds": 1000,
"maximum_delay_seconds": 60,
"stable_reset_seconds": 120,
"jitter_basis_points": 2000,
"offline_work_capacity": 128
}
}

View File

@@ -18,6 +18,9 @@ sha2 = "0.11"
tokio = { version = "1.53.1", features = ["macros", "rt", "sync", "time"] }
url = "2.5.8"
[dev-dependencies]
tokio = { version = "1.53.1", features = ["test-util"] }
[target.'cfg(any(unix, windows))'.dependencies]
tokio = { version = "1.53.1", features = ["io-util", "net", "rt-multi-thread", "signal"] }

View File

@@ -6,9 +6,10 @@ OpenSim grid agent. It contains a reusable library and the
offline: it publishes a deterministic ready event, accepts control commands,
and shuts down both owned tasks without contacting a grid or LLM. The library
also provides the bounded exact-endpoint LLM transport and tool loop for live
adapters. The `live-grid` feature exposes the side-effect-free owner for the existing
`libremetaverse::GridClient`; later live adapters must extend that manager graph
instead of adding a protocol client.
adapters. The `live-grid` feature exposes the owner for the existing
`libremetaverse::GridClient`. Its supervised session adapter reuses that
client's native `NetworkManager` for login, event-queue readiness, disconnect
notifications, and logout instead of adding a protocol client.
The LLM connection identity has exactly two resolved fields:
`llm.endpoint_url` and `llm.api_key`. The endpoint is used exactly as supplied;
@@ -44,3 +45,6 @@ compatibility envelope, retry rules, and tool-loop safety contract. The central
origin matrix, approval binding, budgets, prompt-data boundary, and opaque
backend authorization are specified in
[`../../docs/grid-agent-policy.md`](../../docs/grid-agent-policy.md).
The reconnect state machine, generation fencing, offline-work contract, and
shutdown deadline are specified in
[`../../docs/grid-agent-session.md`](../../docs/grid-agent-session.md).

View File

@@ -7,6 +7,10 @@ use std::error::Error;
use std::fmt;
use std::future::Future;
use std::pin::Pin;
#[cfg(feature = "live-grid")]
use std::sync::Arc;
#[cfg(feature = "live-grid")]
use std::sync::atomic::{AtomicBool, Ordering};
use tokio::sync::mpsc;
pub type BackendFuture<'a, T> = Pin<Box<dyn Future<Output = T> + Send + 'a>>;
@@ -82,11 +86,45 @@ impl OfflineGridBackend {
/// Live-feature composition owner that guarantees production adapters reuse
/// the existing `libremetaverse` manager/client graph.
#[cfg(feature = "live-grid")]
#[derive(Debug)]
pub struct LibremetaverseClientOwner {
client: libremetaverse::GridClient,
}
#[cfg(feature = "live-grid")]
impl fmt::Debug for LibremetaverseClientOwner {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("LibremetaverseClientOwner")
.field("client", &"[NATIVE CLIENT REDACTED]")
.finish()
}
}
/// Feature-gated session adapter that reuses the existing native
/// `NetworkManager` login, event, disconnect, and logout lifecycle.
#[cfg(feature = "live-grid")]
#[derive(Clone)]
pub struct LibremetaverseSessionBackend {
network: libremetaverse::NetworkManager,
login_url: String,
first_name: String,
last_name: String,
password: crate::config::SecretString,
}
#[cfg(feature = "live-grid")]
impl fmt::Debug for LibremetaverseSessionBackend {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("LibremetaverseSessionBackend")
.field("transport_connected", &self.network.native_connected())
.field("login_url", &"[REDACTED ENDPOINT]")
.field("avatar", &"[REDACTED IDENTITY]")
.field("password", &self.password)
.finish_non_exhaustive()
}
}
#[cfg(feature = "live-grid")]
impl LibremetaverseClientOwner {
/// Builds the shared client composition root without starting login or I/O.
@@ -107,6 +145,213 @@ impl LibremetaverseClientOwner {
pub const fn client(&self) -> &libremetaverse::GridClient {
&self.client
}
/// Creates the supervised live-session adapter without logging in.
pub fn session_backend(
&self,
connection: crate::config::GridConnection,
) -> Result<LibremetaverseSessionBackend, BackendError> {
let avatar = connection.avatar_name.trim();
let (first_name, last_name) = avatar
.split_once(char::is_whitespace)
.map_or((avatar, "Resident"), |(first, last)| (first, last.trim()));
if first_name.is_empty() || last_name.is_empty() {
return Err(BackendError::Configuration {
component: "grid avatar identity",
});
}
Ok(LibremetaverseSessionBackend {
network: self.client.network(),
login_url: connection.login_url.expose_url().to_owned(),
first_name: first_name.to_owned(),
last_name: last_name.to_owned(),
password: connection.password,
})
}
}
#[cfg(feature = "live-grid")]
struct LibremetaverseSession {
generation: u64,
network: libremetaverse::NetworkManager,
signals: mpsc::Receiver<crate::session::SessionSignal>,
subscriptions: Vec<libremetaverse_types::compat::Subscription>,
}
#[cfg(feature = "live-grid")]
impl crate::session::GridSession for LibremetaverseSession {
fn generation(&self) -> u64 {
self.generation
}
fn next_signal(
&mut self,
cancellation: CancellationToken,
) -> crate::session::SessionFuture<'_, crate::session::SessionSignal> {
Box::pin(async move {
tokio::select! {
signal = self.signals.recv() => signal.unwrap_or_else(|| {
crate::session::SessionSignal::Disconnected(
crate::session::SessionFailure::new(
crate::session::SessionFailureKind::TransientTransport,
),
)
}),
() = cancellation.cancelled() => crate::session::SessionSignal::Disconnected(
crate::session::SessionFailure::new(
crate::session::SessionFailureKind::TransientTransport,
),
),
}
})
}
fn logout(
mut self: Box<Self>,
cancellation: CancellationToken,
) -> crate::session::SessionFuture<'static, Result<(), crate::session::SessionFailure>> {
Box::pin(async move {
self.subscriptions.clear();
self.network
.native_logout_async(Some(cancellation))
.await
.map_err(|_| {
crate::session::SessionFailure::new(
crate::session::SessionFailureKind::TransientTransport,
)
})
})
}
}
#[cfg(feature = "live-grid")]
impl crate::session::GridSessionBackend for LibremetaverseSessionBackend {
fn login(
&self,
generation: u64,
cancellation: CancellationToken,
) -> crate::session::SessionFuture<
'_,
Result<Box<dyn crate::session::GridSession>, crate::session::SessionFailure>,
> {
Box::pin(async move {
let mut params = self
.network
.native_default_login_params(
self.first_name.clone(),
self.last_name.clone(),
self.password.expose_secret().to_owned(),
"MetaCrate".to_owned(),
env!("CARGO_PKG_VERSION").to_owned(),
)
.map_err(|_| {
crate::session::SessionFailure::new(
crate::session::SessionFailureKind::InvalidConfiguration,
)
})?;
params.uri.clone_from(&self.login_url);
params.start = "last".to_owned();
let logged_in = self
.network
.native_login(params, Some(cancellation))
.await
.map_err(|_| {
crate::session::SessionFailure::new(
crate::session::SessionFailureKind::TransientTransport,
)
})?;
if !logged_in {
return Err(classify_native_login_failure(
&self.network.native_login_error_key(),
));
}
// Native login has already installed the current simulator and
// parsed inventory state. The existing GridClient managers retain
// world/inventory and movement-worker ownership; this generation
// adds only one disconnect and one readiness subscription.
let (sender, signals) = mpsc::channel(8);
let disconnected_sender = sender.clone();
let disconnected = self
.network
.native_subscribe_disconnected(Arc::new(move |event| {
let kind = match event.reason() {
libremetaverse::NetworkManagerDisconnectType::NetworkTimeout => {
crate::session::SessionFailureKind::TransientTransport
}
libremetaverse::NetworkManagerDisconnectType::ServerInitiated => {
crate::session::SessionFailureKind::Kicked
}
libremetaverse::NetworkManagerDisconnectType::SimShutdown => {
crate::session::SessionFailureKind::Maintenance
}
libremetaverse::NetworkManagerDisconnectType::ClientInitiated => {
crate::session::SessionFailureKind::TransientTransport
}
};
let _ =
disconnected_sender.try_send(crate::session::SessionSignal::Disconnected(
crate::session::SessionFailure::new(kind),
));
}));
let ready_sender = sender.clone();
let ready_once = Arc::new(AtomicBool::new(false));
let callback_ready = Arc::clone(&ready_once);
let ready = self
.network
.native_subscribe_event_queue_running(Arc::new(move |_| {
if !callback_ready.swap(true, Ordering::AcqRel) {
let _ = ready_sender.try_send(crate::session::SessionSignal::Ready);
}
}));
if !self.network.native_connected() {
let _ = sender.try_send(crate::session::SessionSignal::Disconnected(
crate::session::SessionFailure::new(
crate::session::SessionFailureKind::TransientTransport,
),
));
} else if self
.network
.native_current_sim()
.and_then(|simulator| simulator.native_is_event_queue_running(None).ok())
.unwrap_or(false)
{
if !ready_once.swap(true, Ordering::AcqRel) {
let _ = sender.try_send(crate::session::SessionSignal::Ready);
}
}
let session: Box<dyn crate::session::GridSession> = Box::new(LibremetaverseSession {
generation,
network: self.network.clone(),
signals,
subscriptions: vec![disconnected, ready],
});
Ok(session)
})
}
fn flush_audit(
&self,
_cancellation: CancellationToken,
) -> crate::session::SessionFuture<'_, Result<(), crate::session::SessionFailure>> {
Box::pin(async { Ok(()) })
}
}
#[cfg(feature = "live-grid")]
fn classify_native_login_failure(error_key: &str) -> crate::session::SessionFailure {
let normalized = error_key.trim().to_ascii_lowercase();
let kind = if matches!(
normalized.as_str(),
"key" | "password" | "credential" | "account" | "username" | "user"
) {
crate::session::SessionFailureKind::InvalidCredentials
} else if normalized == "canceled" {
crate::session::SessionFailureKind::TransientTransport
} else {
crate::session::SessionFailureKind::ServerFailure
};
crate::session::SessionFailure::new(kind)
}
impl GridBackend for OfflineGridBackend {

View File

@@ -213,6 +213,7 @@ pub struct AgentConfig {
pub limits: Limits,
pub storage_path: PathBuf,
pub behavior: BehaviorSettings,
pub reconnect: crate::session::ReconnectPolicy,
}
impl AgentConfig {
@@ -239,6 +240,9 @@ impl AgentConfig {
/// Returns the first invalid field without performing I/O.
pub fn validate(&self) -> Result<(), ConfigError> {
validate_limits(&self.limits)?;
self.reconnect
.validate()
.map_err(|_| ConfigError::InvalidReconnect)?;
if self.mode != OperatingMode::OfflineFake && self.grid.is_none() {
return Err(ConfigError::Missing {
field: "grid",
@@ -411,6 +415,7 @@ pub enum ConfigError {
minimum: usize,
maximum: usize,
},
InvalidReconnect,
}
impl fmt::Display for ConfigError {
@@ -460,6 +465,7 @@ impl fmt::Display for ConfigError {
formatter,
"unsafe {field}={value}; expected {minimum}..={maximum}"
),
Self::InvalidReconnect => formatter.write_str("invalid reconnect policy bounds"),
}
}
}
@@ -479,6 +485,7 @@ struct FileConfig {
limits: RawLimits,
storage_path: Option<PathBuf>,
behavior: RawBehavior,
reconnect: RawReconnect,
}
#[derive(Clone, Default, Deserialize)]
@@ -534,6 +541,16 @@ struct RawBehavior {
heartbeat_seconds: Option<u64>,
}
#[derive(Clone, Default, Deserialize)]
#[serde(default, deny_unknown_fields)]
struct RawReconnect {
initial_delay_milliseconds: Option<u64>,
maximum_delay_seconds: Option<u64>,
stable_reset_seconds: Option<u64>,
jitter_basis_points: Option<u16>,
offline_work_capacity: Option<usize>,
}
fn read_config(path: &Path) -> Result<FileConfig, ConfigError> {
let bytes = read_bounded_regular_file("configuration file", path, MAX_CONFIG_BYTES)?;
serde_json::from_slice(&bytes).map_err(|error| ConfigError::InvalidSchema {
@@ -666,31 +683,51 @@ fn resolve<E: Environment>(
};
validate_limits(&limits)?;
let timeouts = Timeouts {
startup: checked_duration(
"timeouts.startup_seconds",
raw.timeouts.startup.unwrap_or(30),
1,
300,
)?,
shutdown: checked_duration(
"timeouts.shutdown_seconds",
raw.timeouts.shutdown.unwrap_or(10),
1,
60,
)?,
request: checked_duration(
"timeouts.request_seconds",
raw.timeouts.request.unwrap_or(60),
1,
300,
)?,
};
let reconnect_defaults = crate::session::ReconnectPolicy::default();
let reconnect = crate::session::ReconnectPolicy {
initial_delay: Duration::from_millis(
raw.reconnect.initial_delay_milliseconds.unwrap_or(1_000),
),
maximum_delay: Duration::from_secs(raw.reconnect.maximum_delay_seconds.unwrap_or(60)),
stable_reset_after: Duration::from_secs(raw.reconnect.stable_reset_seconds.unwrap_or(120)),
shutdown_deadline: timeouts.shutdown,
jitter_basis_points: raw
.reconnect
.jitter_basis_points
.unwrap_or(reconnect_defaults.jitter_basis_points),
instance_seed: reconnect_defaults.instance_seed,
offline_work_capacity: raw
.reconnect
.offline_work_capacity
.unwrap_or(reconnect_defaults.offline_work_capacity),
};
let config = AgentConfig {
mode,
llm,
grid,
authorized_avatar_uuids,
timeouts: Timeouts {
startup: checked_duration(
"timeouts.startup_seconds",
raw.timeouts.startup.unwrap_or(30),
1,
300,
)?,
shutdown: checked_duration(
"timeouts.shutdown_seconds",
raw.timeouts.shutdown.unwrap_or(10),
1,
60,
)?,
request: checked_duration(
"timeouts.request_seconds",
raw.timeouts.request.unwrap_or(60),
1,
300,
)?,
},
timeouts,
limits,
storage_path: raw
.storage_path
@@ -703,6 +740,7 @@ fn resolve<E: Environment>(
300,
)?,
},
reconnect,
};
config.validate()?;
Ok(config)
@@ -1076,6 +1114,19 @@ mod tests {
Err(ConfigError::UnsafeLimit { .. })
));
let _ = fs::remove_file(path);
let reconnect = temporary_file(
"unsafe-reconnect.json",
r#"{"reconnect":{"initial_delay_milliseconds":0}}"#,
);
assert!(matches!(
ConfigLoader::new()
.with_file(&reconnect)
.with_environment(offline_environment())
.load(),
Err(ConfigError::InvalidReconnect)
));
let _ = fs::remove_file(reconnect);
}
#[test]

View File

@@ -9,18 +9,21 @@ pub mod config;
pub mod llm;
pub mod policy;
pub mod service;
pub mod session;
pub mod tool_loop;
pub mod types;
#[cfg(test)]
mod policy_tests;
#[cfg(test)]
mod session_tests;
#[cfg(feature = "live-grid")]
pub use backend::LibremetaverseClientOwner;
pub use backend::{
AuthorizedToolBackend, BackendError, BackendFuture, GridBackend, OfflineGridBackend,
WorldMutator,
};
#[cfg(feature = "live-grid")]
pub use backend::{LibremetaverseClientOwner, LibremetaverseSessionBackend};
pub use config::{
AgentConfig, BehaviorSettings, ConfigError, ConfigLoader, EndpointUrl, Environment,
GridConnection, Limits, LlmConnection, MapEnvironment, OperatingMode, SecretString,
@@ -39,6 +42,12 @@ pub use policy::{
ResourceCost, ResourceEstimator, Risk, SchedulerGrantId, UntrustedData, UntrustedSource,
};
pub use service::{AgentService, ServiceError, ServiceHandle, ServiceState};
pub use session::{
GridSession, GridSessionBackend, ReconnectPolicy, SessionControl, SessionFailure,
SessionFailureKind, SessionFuture, SessionObservation, SessionSignal, SessionState,
SessionStatus, SessionSupervisor, SessionSupervisorError, SessionSupervisorHandle, SessionWork,
WorkDisposition, WorkKind,
};
pub use tool_loop::{
HistorySummarizer, SessionGeneration, ToolExecution, ToolExecutor, ToolFuture, ToolLoop,
ToolLoopError, ToolLoopLimits, ToolLoopOutcome,

View File

@@ -4,6 +4,8 @@ use metacrate_grid_agent::{
use std::error::Error;
use std::fmt;
use std::path::PathBuf;
#[cfg(feature = "live-grid")]
use std::sync::Arc;
const MAX_ARGUMENTS: usize = 8;
@@ -84,9 +86,11 @@ async fn main() -> Result<(), Box<dyn Error>> {
return Ok(());
}
if config.mode != OperatingMode::OfflineFake {
#[cfg(feature = "live-grid")]
return run_live(config, options.run_once).await;
#[cfg(not(feature = "live-grid"))]
return Err(CliError(
"this architecture issue starts only the offline backend; live login is owned by a later milestone issue"
.into(),
"live grid mode requires rebuilding with --features live-grid".into(),
)
.into());
}
@@ -133,3 +137,101 @@ async fn main() -> Result<(), Box<dyn Error>> {
println!("grid agent stopped cleanly");
Ok(())
}
#[cfg(feature = "live-grid")]
async fn run_live(
config: metacrate_grid_agent::AgentConfig,
run_once: bool,
) -> Result<(), Box<dyn Error>> {
use metacrate_grid_agent::{
GridSessionBackend, LibremetaverseClientOwner, SessionObservation, SessionState,
SessionSupervisor,
};
let connection = config.grid.clone().ok_or_else(|| {
CliError("validated live configuration did not contain a grid connection".into())
})?;
let owner = LibremetaverseClientOwner::new()?;
let backend = owner.session_backend(connection)?;
let erased: Arc<dyn GridSessionBackend> = Arc::new(backend);
let mut handle = SessionSupervisor::new(
erased,
config.reconnect,
config.limits.control_queue,
config.limits.observable_queue,
)?
.start();
if run_once {
let readiness = tokio::time::timeout(config.timeouts.startup, async {
loop {
match handle.next_observation().await {
Some(SessionObservation::Transition { status, .. }) if status.agent_ready => {
return Ok::<(), CliError>(());
}
Some(SessionObservation::Transition {
status:
metacrate_grid_agent::SessionStatus {
state: SessionState::AuthenticationBlocked,
..
},
..
}) => {
return Err(CliError(
"grid authentication/configuration requires operator action".into(),
));
}
Some(_) => {}
None => {
return Err(CliError(
"session supervisor stopped before readiness".into(),
));
}
}
}
})
.await;
let readiness = match readiness {
Ok(result) => result,
Err(_) => Err(CliError("timed out waiting for full grid readiness".into())),
};
if let Err(error) = readiness {
handle.shutdown().await?;
return Err(error.into());
}
handle.shutdown().await?;
println!("grid agent completed one supervised login/logout cycle");
return Ok(());
}
println!("grid agent session supervisor started; press Ctrl-C to stop");
let mut signal_error = None;
loop {
tokio::select! {
signal = tokio::signal::ctrl_c() => {
if let Err(error) = signal {
signal_error = Some(error);
}
break;
}
event = handle.next_observation() => {
let Some(event) = event else { break; };
if let SessionObservation::Transition { status, reason, retry_in } = event {
println!(
"grid session state={} generation={} transport_connected={} agent_ready={} reason={reason:?} retry_in={retry_in:?}",
status.state.as_str(),
status.generation,
status.transport_connected,
status.agent_ready,
);
}
}
}
}
handle.shutdown().await?;
if let Some(error) = signal_error {
return Err(error.into());
}
println!("grid agent stopped cleanly");
Ok(())
}

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,526 @@
use crate::session::*;
use libremetaverse_types::compat::CancellationToken;
use std::collections::VecDeque;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::{Arc, Mutex};
use std::time::Duration;
#[derive(Clone)]
enum LoginPlan {
Failure {
after: Duration,
failure: SessionFailure,
},
Success {
after: Duration,
signals: VecDeque<(Duration, SessionSignal)>,
},
}
#[derive(Default)]
struct FakeStats {
logins: AtomicUsize,
logouts: AtomicUsize,
flushes: AtomicUsize,
active_sessions: AtomicUsize,
active_callbacks: AtomicUsize,
active_workers: AtomicUsize,
active_sockets: AtomicUsize,
max_sessions: AtomicUsize,
max_callbacks: AtomicUsize,
max_workers: AtomicUsize,
max_sockets: AtomicUsize,
}
impl FakeStats {
fn acquire(&self) {
update_max(&self.active_sessions, &self.max_sessions, 1);
update_max(&self.active_callbacks, &self.max_callbacks, 4);
update_max(&self.active_workers, &self.max_workers, 2);
update_max(&self.active_sockets, &self.max_sockets, 1);
}
fn release(&self) {
self.active_sessions.fetch_sub(1, Ordering::AcqRel);
self.active_callbacks.fetch_sub(4, Ordering::AcqRel);
self.active_workers.fetch_sub(2, Ordering::AcqRel);
self.active_sockets.fetch_sub(1, Ordering::AcqRel);
}
}
fn update_max(active: &AtomicUsize, maximum: &AtomicUsize, amount: usize) {
let next = active.fetch_add(amount, Ordering::AcqRel) + amount;
maximum.fetch_max(next, Ordering::AcqRel);
}
struct FakeBackend {
plans: Mutex<VecDeque<LoginPlan>>,
stats: Arc<FakeStats>,
}
impl FakeBackend {
fn new(plans: impl IntoIterator<Item = LoginPlan>) -> (Arc<Self>, Arc<FakeStats>) {
let stats = Arc::new(FakeStats::default());
(
Arc::new(Self {
plans: Mutex::new(plans.into_iter().collect()),
stats: Arc::clone(&stats),
}),
stats,
)
}
}
impl GridSessionBackend for FakeBackend {
fn login(
&self,
generation: u64,
cancellation: CancellationToken,
) -> SessionFuture<'_, Result<Box<dyn GridSession>, SessionFailure>> {
self.stats.logins.fetch_add(1, Ordering::AcqRel);
let plan = self
.plans
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.pop_front()
.unwrap_or(LoginPlan::Success {
after: Duration::ZERO,
signals: VecDeque::from([(Duration::ZERO, SessionSignal::Ready)]),
});
let stats = Arc::clone(&self.stats);
Box::pin(async move {
let after = match &plan {
LoginPlan::Failure { after, .. } | LoginPlan::Success { after, .. } => *after,
};
tokio::select! {
() = tokio::time::sleep(after) => {}
() = cancellation.cancelled() => {
return Err(SessionFailure::new(SessionFailureKind::TransientTransport));
}
}
match plan {
LoginPlan::Failure { failure, .. } => Err(failure),
LoginPlan::Success { signals, .. } => {
stats.acquire();
let session: Box<dyn GridSession> = Box::new(FakeSession {
generation,
signals,
stats,
released: false,
});
Ok(session)
}
}
})
}
fn flush_audit(
&self,
_cancellation: CancellationToken,
) -> SessionFuture<'_, Result<(), SessionFailure>> {
self.stats.flushes.fetch_add(1, Ordering::AcqRel);
Box::pin(async { Ok(()) })
}
}
struct FakeSession {
generation: u64,
signals: VecDeque<(Duration, SessionSignal)>,
stats: Arc<FakeStats>,
released: bool,
}
impl FakeSession {
fn release(&mut self) {
if !self.released {
self.released = true;
self.stats.release();
}
}
}
impl GridSession for FakeSession {
fn generation(&self) -> u64 {
self.generation
}
fn next_signal(&mut self, cancellation: CancellationToken) -> SessionFuture<'_, SessionSignal> {
let next = self.signals.pop_front();
Box::pin(async move {
if let Some((after, signal)) = next {
tokio::select! {
() = tokio::time::sleep(after) => signal,
() = cancellation.cancelled() => SessionSignal::Disconnected(
SessionFailure::new(SessionFailureKind::TransientTransport)
),
}
} else {
cancellation.cancelled().await;
SessionSignal::Disconnected(SessionFailure::new(
SessionFailureKind::TransientTransport,
))
}
})
}
fn logout(
mut self: Box<Self>,
_cancellation: CancellationToken,
) -> SessionFuture<'static, Result<(), SessionFailure>> {
self.stats.logouts.fetch_add(1, Ordering::AcqRel);
self.release();
Box::pin(async { Ok(()) })
}
}
impl Drop for FakeSession {
fn drop(&mut self) {
self.release();
}
}
fn ready_then(after: Duration, failure: SessionFailureKind) -> LoginPlan {
LoginPlan::Success {
after: Duration::ZERO,
signals: VecDeque::from([
(Duration::ZERO, SessionSignal::Ready),
(
after,
SessionSignal::Disconnected(SessionFailure::new(failure)),
),
]),
}
}
fn test_policy() -> ReconnectPolicy {
ReconnectPolicy {
initial_delay: Duration::from_secs(1),
maximum_delay: Duration::from_secs(8),
stable_reset_after: Duration::from_secs(10),
shutdown_deadline: Duration::from_secs(2),
jitter_basis_points: 0,
instance_seed: 7,
offline_work_capacity: 2,
}
}
#[test]
fn retry_hints_are_minimums_and_all_delays_remain_capped() {
let mut policy = test_policy();
policy.jitter_basis_points = 5_000;
for seed in 0..128 {
policy.instance_seed = seed;
let hinted = Duration::from_secs(6);
let delay = policy.retry_delay(1, Some(hinted));
assert!(delay >= hinted);
assert!(delay <= policy.maximum_delay);
assert!(policy.retry_delay(64, None) <= policy.maximum_delay);
}
}
fn start(plans: impl IntoIterator<Item = LoginPlan>) -> (SessionSupervisorHandle, Arc<FakeStats>) {
let (backend, stats) = FakeBackend::new(plans);
let erased: Arc<dyn GridSessionBackend> = backend;
let handle = SessionSupervisor::new(erased, test_policy(), 32, 256)
.expect("supervisor")
.start();
(handle, stats)
}
async fn settle() {
for _ in 0..20 {
tokio::task::yield_now().await;
}
}
async fn wait_state(handle: &SessionSupervisorHandle, expected: SessionState) {
for _ in 0..100 {
if handle.state() == expected {
return;
}
tokio::task::yield_now().await;
}
panic!("expected {expected:?}, got {:?}", handle.status());
}
#[tokio::test(start_paused = true)]
async fn paused_time_classifies_failures_backoff_flapping_and_stable_reset() {
let plans = [
LoginPlan::Failure {
after: Duration::ZERO,
failure: SessionFailure::new(SessionFailureKind::TransientTransport),
},
LoginPlan::Failure {
after: Duration::ZERO,
failure: SessionFailure::with_retry_after(
SessionFailureKind::Maintenance,
Duration::from_secs(5),
),
},
ready_then(Duration::from_secs(1), SessionFailureKind::Kicked),
ready_then(
Duration::from_secs(11),
SessionFailureKind::SimulatorDisconnected,
),
LoginPlan::Success {
after: Duration::ZERO,
signals: VecDeque::from([(Duration::ZERO, SessionSignal::Ready)]),
},
];
let (mut handle, stats) = start(plans);
wait_state(&handle, SessionState::Backoff).await;
assert_eq!(handle.status().consecutive_failures, 1);
tokio::time::advance(Duration::from_secs(1)).await;
settle().await;
wait_state(&handle, SessionState::Backoff).await;
assert_eq!(handle.status().consecutive_failures, 2);
tokio::time::advance(Duration::from_secs(5)).await;
settle().await;
wait_state(&handle, SessionState::Online).await;
let first_online = handle.status().generation;
tokio::time::advance(Duration::from_secs(1)).await;
settle().await;
wait_state(&handle, SessionState::Backoff).await;
assert!(!handle.accepts_result(first_online));
tokio::time::advance(Duration::from_secs(4)).await;
settle().await;
wait_state(&handle, SessionState::Online).await;
tokio::time::advance(Duration::from_secs(11)).await;
settle().await;
wait_state(&handle, SessionState::Backoff).await;
assert_eq!(handle.status().consecutive_failures, 1);
tokio::time::advance(Duration::from_secs(1)).await;
settle().await;
wait_state(&handle, SessionState::Online).await;
handle.shutdown().await.expect("shutdown");
assert_eq!(stats.flushes.load(Ordering::Acquire), 1);
}
#[tokio::test(start_paused = true)]
async fn invalid_credentials_and_configuration_block_until_operator_reconnect() {
for failure in [
SessionFailureKind::InvalidCredentials,
SessionFailureKind::InvalidConfiguration,
] {
let plans = [
LoginPlan::Failure {
after: Duration::ZERO,
failure: SessionFailure::new(failure),
},
LoginPlan::Success {
after: Duration::ZERO,
signals: VecDeque::from([(Duration::ZERO, SessionSignal::Ready)]),
},
];
let (mut handle, stats) = start(plans);
wait_state(&handle, SessionState::AuthenticationBlocked).await;
tokio::time::advance(Duration::from_hours(1)).await;
settle().await;
assert_eq!(stats.logins.load(Ordering::Acquire), 1);
handle
.control(SessionControl::ForceReconnect)
.await
.expect("force reconnect");
settle().await;
wait_state(&handle, SessionState::Online).await;
handle.shutdown().await.expect("shutdown");
}
}
#[tokio::test(start_paused = true)]
async fn manual_pause_logout_reconnect_and_offline_work_are_predictable() {
let (mut handle, stats) = start([LoginPlan::Success {
after: Duration::ZERO,
signals: VecDeque::from([(Duration::ZERO, SessionSignal::Ready)]),
}]);
wait_state(&handle, SessionState::Online).await;
let generation = handle.status().generation;
let token = handle
.generation_token(generation)
.expect("generation token");
handle.control(SessionControl::Pause).await.expect("pause");
settle().await;
wait_state(&handle, SessionState::Paused).await;
assert!(token.is_cancellation_requested());
assert!(!handle.accepts_result(generation));
assert_eq!(
handle
.submit(SessionWork::new("mutate", WorkKind::IdempotentMutation).expect("work"))
.await
.expect("decision"),
WorkDisposition::RejectedOfflineMutation
);
assert_eq!(
handle
.submit(SessionWork::new("read", WorkKind::ReadOnly).expect("work"))
.await
.expect("decision"),
WorkDisposition::Queued
);
handle
.control(SessionControl::Resume)
.await
.expect("resume");
settle().await;
wait_state(&handle, SessionState::Online).await;
assert_ne!(handle.status().generation, generation);
assert_eq!(
handle
.submit(SessionWork::new("read", WorkKind::ReadOnly).expect("work"))
.await
.expect("dedupe decision"),
WorkDisposition::RejectedDuplicate
);
let before_force = handle.status().generation;
handle
.control(SessionControl::ForceReconnect)
.await
.expect("active reconnect");
settle().await;
wait_state(&handle, SessionState::Online).await;
assert_ne!(handle.status().generation, before_force);
handle
.control(SessionControl::Logout)
.await
.expect("logout");
settle().await;
wait_state(&handle, SessionState::Stopped).await;
handle
.control(SessionControl::ForceReconnect)
.await
.expect("reconnect");
settle().await;
wait_state(&handle, SessionState::Online).await;
handle.shutdown().await.expect("shutdown");
assert_eq!(stats.active_sessions.load(Ordering::Acquire), 0);
}
#[tokio::test(start_paused = true)]
async fn transport_connection_is_distinct_from_full_agent_readiness() {
let (mut handle, _) = start([LoginPlan::Success {
after: Duration::ZERO,
signals: VecDeque::from([(Duration::from_secs(10), SessionSignal::Ready)]),
}]);
wait_state(&handle, SessionState::Degraded).await;
let degraded = handle.status();
assert!(degraded.transport_connected);
assert!(!degraded.agent_ready);
assert!(!handle.accepts_result(degraded.generation));
tokio::time::advance(Duration::from_secs(10)).await;
settle().await;
wait_state(&handle, SessionState::Online).await;
assert!(handle.status().agent_ready);
handle.shutdown().await.expect("shutdown");
}
#[tokio::test(start_paused = true)]
async fn shutdown_covers_degraded_authentication_blocked_stopped_and_online_states() {
let cases = [
(
LoginPlan::Success {
after: Duration::ZERO,
signals: VecDeque::from([(Duration::from_hours(1), SessionSignal::Ready)]),
},
SessionState::Degraded,
),
(
LoginPlan::Failure {
after: Duration::ZERO,
failure: SessionFailure::new(SessionFailureKind::InvalidCredentials),
},
SessionState::AuthenticationBlocked,
),
(
LoginPlan::Success {
after: Duration::ZERO,
signals: VecDeque::from([(Duration::ZERO, SessionSignal::Ready)]),
},
SessionState::Online,
),
];
for (plan, expected) in cases {
let (mut handle, stats) = start([plan]);
wait_state(&handle, expected).await;
if expected == SessionState::Online {
handle
.control(SessionControl::Logout)
.await
.expect("logout");
settle().await;
wait_state(&handle, SessionState::Stopped).await;
}
handle.shutdown().await.expect("shutdown");
assert_eq!(stats.active_sessions.load(Ordering::Acquire), 0);
assert_eq!(stats.flushes.load(Ordering::Acquire), 1);
}
}
#[tokio::test(start_paused = true)]
async fn fake_grid_reconnects_keep_exactly_one_resource_set_and_no_stale_execution() {
let plans = [
ready_then(Duration::from_secs(1), SessionFailureKind::Maintenance),
ready_then(Duration::from_secs(1), SessionFailureKind::Kicked),
LoginPlan::Success {
after: Duration::ZERO,
signals: VecDeque::from([(Duration::ZERO, SessionSignal::Ready)]),
},
];
let (mut handle, stats) = start(plans);
wait_state(&handle, SessionState::Online).await;
let stale = handle.status().generation;
tokio::time::advance(Duration::from_secs(1)).await;
settle().await;
tokio::time::advance(Duration::from_secs(1)).await;
settle().await;
wait_state(&handle, SessionState::Online).await;
tokio::time::advance(Duration::from_secs(1)).await;
settle().await;
tokio::time::advance(Duration::from_secs(2)).await;
settle().await;
wait_state(&handle, SessionState::Online).await;
assert!(!handle.accepts_result(stale));
assert!(handle.accepts_result(handle.status().generation));
assert_eq!(stats.max_sessions.load(Ordering::Acquire), 1);
assert_eq!(stats.max_callbacks.load(Ordering::Acquire), 4);
assert_eq!(stats.max_workers.load(Ordering::Acquire), 2);
assert_eq!(stats.max_sockets.load(Ordering::Acquire), 1);
handle.shutdown().await.expect("shutdown");
assert_eq!(stats.active_sessions.load(Ordering::Acquire), 0);
assert_eq!(stats.active_callbacks.load(Ordering::Acquire), 0);
assert_eq!(stats.active_workers.load(Ordering::Acquire), 0);
assert_eq!(stats.active_sockets.load(Ordering::Acquire), 0);
assert_eq!(stats.logouts.load(Ordering::Acquire), 3);
assert_eq!(stats.flushes.load(Ordering::Acquire), 1);
}
#[tokio::test(start_paused = true)]
async fn shutdown_is_cancellation_driven_during_connect_backoff_paused_and_online() {
let scenarios = [
LoginPlan::Success {
after: Duration::from_hours(1),
signals: VecDeque::new(),
},
LoginPlan::Failure {
after: Duration::ZERO,
failure: SessionFailure::new(SessionFailureKind::TransientTransport),
},
LoginPlan::Success {
after: Duration::ZERO,
signals: VecDeque::from([(Duration::ZERO, SessionSignal::Ready)]),
},
];
for (index, plan) in scenarios.into_iter().enumerate() {
let (mut handle, stats) = start([plan]);
settle().await;
if index == 2 {
wait_state(&handle, SessionState::Online).await;
handle.control(SessionControl::Pause).await.expect("pause");
settle().await;
wait_state(&handle, SessionState::Paused).await;
}
handle.shutdown().await.expect("bounded shutdown");
assert_eq!(handle.state(), SessionState::Stopped);
assert_eq!(stats.active_sessions.load(Ordering::Acquire), 0);
assert_eq!(stats.flushes.load(Ordering::Acquire), 1);
}
}

View File

@@ -46,7 +46,7 @@ fn runtime_source_has_no_subprocess_or_native_abi_escape_hatch() {
let mut files = Vec::with_capacity(8);
collect_rust_files(&source, &mut files);
assert!(
files.len() <= 10,
files.len() <= 12,
"source-file count needs a reviewed bound update"
);
for path in files {
@@ -86,6 +86,25 @@ fn world_backend_requires_the_opaque_policy_authorization() {
assert!(!backend.contains("decision: PolicyDecision"));
}
#[test]
fn live_session_adapter_reuses_only_the_native_network_manager_lifecycle() {
let root = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
let backend = fs::read_to_string(root.join("src/backend.rs")).expect("backend source");
for required in [
".native_default_login_params(",
".native_login(",
".native_subscribe_disconnected(",
".native_subscribe_event_queue_running(",
".native_logout_async(",
] {
assert!(
backend.contains(required),
"missing native lifecycle {required}"
);
}
assert!(!backend.contains("reqwest"));
}
fn collect_rust_files(directory: &Path, output: &mut Vec<PathBuf>) {
for entry in fs::read_dir(directory).expect("read source directory") {
let path = entry.expect("source entry").path();

View File

@@ -0,0 +1,227 @@
use libremetaverse_types::compat::CancellationToken;
use metacrate_grid_agent::{
GridSession, GridSessionBackend, ReconnectPolicy, SessionFailure, SessionFailureKind,
SessionFuture, SessionSignal, SessionState, SessionSupervisor, SessionSupervisorHandle,
SessionWork, WorkDisposition, WorkKind,
};
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::time::Duration;
#[derive(Default)]
struct ResourceStats {
attempts: AtomicUsize,
logouts: AtomicUsize,
flushes: AtomicUsize,
sessions: AtomicUsize,
callbacks: AtomicUsize,
workers: AtomicUsize,
sockets: AtomicUsize,
max_sessions: AtomicUsize,
max_callbacks: AtomicUsize,
max_workers: AtomicUsize,
max_sockets: AtomicUsize,
}
impl ResourceStats {
fn acquire(&self) {
acquire(&self.sessions, &self.max_sessions, 1);
acquire(&self.callbacks, &self.max_callbacks, 3);
acquire(&self.workers, &self.max_workers, 2);
acquire(&self.sockets, &self.max_sockets, 1);
}
fn release(&self) {
self.sessions.fetch_sub(1, Ordering::AcqRel);
self.callbacks.fetch_sub(3, Ordering::AcqRel);
self.workers.fetch_sub(2, Ordering::AcqRel);
self.sockets.fetch_sub(1, Ordering::AcqRel);
}
}
fn acquire(active: &AtomicUsize, maximum: &AtomicUsize, amount: usize) {
let next = active.fetch_add(amount, Ordering::AcqRel) + amount;
maximum.fetch_max(next, Ordering::AcqRel);
}
struct ReconnectingFakeGrid {
stats: Arc<ResourceStats>,
}
impl GridSessionBackend for ReconnectingFakeGrid {
fn login(
&self,
generation: u64,
_cancellation: CancellationToken,
) -> SessionFuture<'_, Result<Box<dyn GridSession>, SessionFailure>> {
let attempt = self.stats.attempts.fetch_add(1, Ordering::AcqRel) + 1;
self.stats.acquire();
let stats = Arc::clone(&self.stats);
Box::pin(async move {
let session: Box<dyn GridSession> = Box::new(FakeConnectedSession {
generation,
attempt,
step: 0,
stats,
released: false,
});
Ok(session)
})
}
fn flush_audit(
&self,
_cancellation: CancellationToken,
) -> SessionFuture<'_, Result<(), SessionFailure>> {
self.stats.flushes.fetch_add(1, Ordering::AcqRel);
Box::pin(async { Ok(()) })
}
}
struct FakeConnectedSession {
generation: u64,
attempt: usize,
step: u8,
stats: Arc<ResourceStats>,
released: bool,
}
impl FakeConnectedSession {
fn release(&mut self) {
if !self.released {
self.released = true;
self.stats.release();
}
}
}
impl GridSession for FakeConnectedSession {
fn generation(&self) -> u64 {
self.generation
}
fn next_signal(&mut self, cancellation: CancellationToken) -> SessionFuture<'_, SessionSignal> {
let attempt = self.attempt;
let step = self.step;
self.step = self.step.saturating_add(1);
Box::pin(async move {
if step == 0 {
return SessionSignal::Ready;
}
if attempt <= 2 {
tokio::select! {
() = tokio::time::sleep(Duration::from_secs(1)) => {
let kind = if attempt == 1 {
SessionFailureKind::Maintenance
} else {
SessionFailureKind::Kicked
};
SessionSignal::Disconnected(SessionFailure::new(kind))
}
() = cancellation.cancelled() => SessionSignal::Disconnected(
SessionFailure::new(SessionFailureKind::TransientTransport),
),
}
} else {
cancellation.cancelled().await;
SessionSignal::Disconnected(SessionFailure::new(
SessionFailureKind::TransientTransport,
))
}
})
}
fn logout(
mut self: Box<Self>,
_cancellation: CancellationToken,
) -> SessionFuture<'static, Result<(), SessionFailure>> {
self.stats.logouts.fetch_add(1, Ordering::AcqRel);
self.release();
Box::pin(async { Ok(()) })
}
}
impl Drop for FakeConnectedSession {
fn drop(&mut self) {
self.release();
}
}
async fn settle() {
for _ in 0..30 {
tokio::task::yield_now().await;
}
}
async fn wait_online(handle: &SessionSupervisorHandle) {
for _ in 0..100 {
if handle.state() == SessionState::Online {
return;
}
tokio::task::yield_now().await;
}
panic!("fake grid did not become ready: {:?}", handle.status());
}
#[tokio::test(start_paused = true)]
async fn repeated_reconnects_have_one_resource_set_no_replay_and_no_shutdown_leaks() {
let stats = Arc::new(ResourceStats::default());
let backend: Arc<dyn GridSessionBackend> = Arc::new(ReconnectingFakeGrid {
stats: Arc::clone(&stats),
});
let policy = ReconnectPolicy {
initial_delay: Duration::from_secs(1),
maximum_delay: Duration::from_secs(4),
stable_reset_after: Duration::from_secs(30),
shutdown_deadline: Duration::from_secs(3),
jitter_basis_points: 0,
instance_seed: 1,
offline_work_capacity: 8,
};
let mut handle = SessionSupervisor::new(backend, policy, 32, 256)
.expect("supervisor")
.start();
wait_online(&handle).await;
let stale_generation = handle.status().generation;
assert_eq!(
handle
.submit(SessionWork::new("response-1", WorkKind::ReadOnly).expect("work"))
.await
.expect("accepted"),
WorkDisposition::Accepted {
generation: stale_generation
}
);
tokio::time::advance(Duration::from_secs(1)).await;
settle().await;
tokio::time::advance(Duration::from_secs(1)).await;
settle().await;
wait_online(&handle).await;
tokio::time::advance(Duration::from_secs(1)).await;
settle().await;
tokio::time::advance(Duration::from_secs(2)).await;
settle().await;
wait_online(&handle).await;
assert!(!handle.accepts_result(stale_generation));
assert_eq!(
handle
.submit(SessionWork::new("response-1", WorkKind::ReadOnly).expect("work"))
.await
.expect("dedupe"),
WorkDisposition::RejectedDuplicate
);
assert_eq!(stats.max_sessions.load(Ordering::Acquire), 1);
assert_eq!(stats.max_callbacks.load(Ordering::Acquire), 3);
assert_eq!(stats.max_workers.load(Ordering::Acquire), 2);
assert_eq!(stats.max_sockets.load(Ordering::Acquire), 1);
handle.shutdown().await.expect("shutdown");
assert_eq!(stats.sessions.load(Ordering::Acquire), 0);
assert_eq!(stats.callbacks.load(Ordering::Acquire), 0);
assert_eq!(stats.workers.load(Ordering::Acquire), 0);
assert_eq!(stats.sockets.load(Ordering::Acquire), 0);
assert_eq!(stats.logouts.load(Ordering::Acquire), 3);
assert_eq!(stats.flushes.load(Ordering::Acquire), 1);
}

View File

@@ -10,18 +10,21 @@ platform-specific core path.
## Ownership and bounds
`AgentService::start` is the sole task-creation point. It validates the complete
configuration before allocating channels, calling a backend, or permitting
network access. `ServiceHandle` then exclusively owns cancellation, both join
handles, the control sender, and observable receiver.
`AgentService::start` owns offline task creation, while
`SessionSupervisor::start` owns the live lifecycle task. Configuration is
validated before either allocates channels, calls a backend, or permits network
access. Their handles exclusively own cancellation, join handles, controls, and
observable receivers.
| Resource | Owner | Hard/configured bound | Backpressure/termination |
| --- | --- | --- | --- |
| Grid-event queue | coordinator receives; backend sends | 8,192 / `grid_event_queue` | async send or cancellation |
| Control queue | coordinator receives; handle sends | 256 / `control_queue` | async send; closed after stop |
| Observable queue | handle receives; coordinator/backend send | 8,192 / `observable_queue` | async send or cancellation |
| Backend task | `ServiceHandle.tasks[0]` | exactly one | shared cancellation token, joined first |
| Backend task | `ServiceHandle.tasks[0]` | exactly one in offline mode | shared cancellation token, joined first |
| Coordinator task | `ServiceHandle.tasks[1]` | exactly one | shared cancellation token, joined second |
| Live session supervisor | `SessionSupervisorHandle` | one owner; one active generation/session | generation cancellation, exact-once logout, bounded join |
| Offline session work | live supervisor | 1,024 hard / `reconnect.offline_work_capacity` | read-only queue; mutations rejected while not ready |
| Body / message | typed boundary owners | 8 MiB / 64 KiB hard ceilings, with lower configured limits | rejected before enqueue |
| Conversation / tool calls | request owner | 256 messages / 64 calls, with lower configured limits | rejected before request |
| LLM request slots | shared `LlmClient` semaphore | 256 hard / configured concurrent requests | async acquire or cancellation |
@@ -36,7 +39,7 @@ type. Dynamic configuration can lower these absolute ceilings but cannot raise
them. Backend error text is not forwarded; observations publish a fixed bounded
diagnostic.
## State machine and shutdown
## State machines and shutdown
The explicit service states are `starting -> running <-> paused -> stopping ->
stopped`, with `failed` reserved for task failure. The offline backend publishes
@@ -54,6 +57,13 @@ feature-enabled live adapter drops its
`LibremetaverseClientOwner` last, invoking the existing client ownership
shutdown.
Live modes use `stopped`, `connecting`, `degraded` (transport connected but not
fully ready), `online`, `backoff`, `authentication-blocked`, `paused`, and
`shutting-down`. Every attempt rotates a generation cancellation token.
Disconnect, pause, logout, force reconnect, or shutdown invalidates it before
cleanup, fencing old events and late LLM/tool results. See
[`grid-agent-session.md`](grid-agent-session.md).
## Trust boundaries
- JSON configuration and environment text are untrusted. Unknown fields,
@@ -83,7 +93,7 @@ shutdown.
metacrate-grid-agent binary (portable Ctrl-C + config path)
|
v
AgentService -> bounded Tokio channels/tasks -> injected GridBackend
offline AgentService -> bounded Tokio channels/tasks -> injected GridBackend
| |
v v
typed config/events/policy boundaries live-grid feature boundary
@@ -104,3 +114,5 @@ The precise LLM compatibility and cancellation contract is documented in
[`grid-agent-llm.md`](grid-agent-llm.md).
The origin/capability matrix and opaque mutation boundary are documented in
[`grid-agent-policy.md`](grid-agent-policy.md).
The live lifecycle and generation contract is documented in
[`grid-agent-session.md`](grid-agent-session.md).

View File

@@ -0,0 +1,75 @@
# Grid-session supervision
Live integrated and split modes run through `SessionSupervisor`. Its injected
`GridSessionBackend` makes the lifecycle deterministic under a fake grid while
the `live-grid` adapter reuses `libremetaverse::NetworkManager` for native
login, event-queue readiness, disconnect callbacks, and logout.
The native login does not yield a session until the login response has populated
the existing inventory skeleton and a current simulator/UDP circuit exists.
The adapter then waits for that simulator's event queue before publishing full
readiness. Existing `GridClient` inventory/world managers and its single
movement-update owner remain the composition owners; the adapter never creates
duplicates. Only its generation-scoped readiness/disconnect subscriptions are
recreated, and their RAII guards are dropped before logout.
## States and reasons
| State | Transport connected | Agent ready | Exit condition |
| --- | --- | --- | --- |
| `stopped` | no | no | startup, resume, or force reconnect |
| `connecting` | no | no | login result, operator control, or cancellation |
| `degraded` | yes | no | readiness, disconnect, or operator control |
| `online` | yes | yes | readiness loss, disconnect, or operator control |
| `backoff` | no | no | cancellation-driven timer or operator control |
| `authentication-blocked` | no | no | resume/force reconnect or shutdown |
| `paused` | no | no | resume/force reconnect, logout, or shutdown |
| `shutting-down` | no | no | session logout, audit flush, and joined owner task |
Every transition carries a stable `SessionReason`; raw login responses, server
messages, credentials, capability URLs, and session tokens cannot enter the
failure or observation types. `SessionStatus` reports transport connectivity
and complete readiness separately.
Invalid credentials and invalid local login configuration stop automatic
retry. Transport failures, maintenance, kicks, simulator disconnects, and
server failures retry with exponential backoff. `reconnect.maximum_delay_seconds`
is a hard cap. A server retry hint is a minimum up to that cap. Per-instance
jitter is bounded by `jitter_basis_points`, preventing synchronized reconnects,
and a connection that remains up for `stable_reset_seconds` resets the failure
streak.
Backoff defaults are one second initially, 60 seconds maximum, 20 percent
jitter, and a 120-second stable reset window. All waits use Tokio timers inside
`select!` with cancellation/control; there are no blocking sleeps.
## Generation and work safety
Each attempt receives a monotonically changing generation and cancellation
token. Session-scoped subscriptions, readiness ownership, and workers belong to
the returned `GridSession` and are closed once by its consuming `logout` method.
Late inference/tool results are accepted only when `accepts_result` sees their
exact generation in fully ready state.
While not ready, bounded read-only work may queue. Mutation work—including
nominally idempotent mutation—is rejected, because the supervisor cannot prove
that a previous policy authorization remains valid. Work IDs are deduplicated
across reconnects with a bounded recent-ID set, so reconnect never silently
duplicates an outbound response. Queued reads are released once into the new
generation.
## Shutdown
Shutdown first rejects new work and invalidates the generation token, which
cancels inference, tools, and scheduled consumers sharing it. It then consumes
the active session for one logout attempt, waits within
`timeouts.shutdown_seconds`, calls the backend's bounded audit-flush hook, and
joins the supervisor owner task. A late task is aborted and awaited by the
handle, so no task or socket is detached.
Focused deterministic verification:
```sh
cargo test --locked -p metacrate-grid-agent --lib session_tests
cargo clippy --locked -p metacrate-grid-agent --all-targets -- -D warnings
```