use crate::session::*; use libremetaverse_types::compat::CancellationToken; use std::collections::VecDeque; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, Mutex}; use std::time::Duration; #[derive(Clone)] enum LoginPlan { Failure { after: Duration, failure: SessionFailure, }, Success { after: Duration, signals: VecDeque<(Duration, SessionSignal)>, }, } #[derive(Default)] struct FakeStats { logins: AtomicUsize, logouts: AtomicUsize, flushes: AtomicUsize, active_sessions: AtomicUsize, active_callbacks: AtomicUsize, active_workers: AtomicUsize, active_sockets: AtomicUsize, max_sessions: AtomicUsize, max_callbacks: AtomicUsize, max_workers: AtomicUsize, max_sockets: AtomicUsize, } impl FakeStats { fn acquire(&self) { update_max(&self.active_sessions, &self.max_sessions, 1); update_max(&self.active_callbacks, &self.max_callbacks, 4); update_max(&self.active_workers, &self.max_workers, 2); update_max(&self.active_sockets, &self.max_sockets, 1); } fn release(&self) { self.active_sessions.fetch_sub(1, Ordering::AcqRel); self.active_callbacks.fetch_sub(4, Ordering::AcqRel); self.active_workers.fetch_sub(2, Ordering::AcqRel); self.active_sockets.fetch_sub(1, Ordering::AcqRel); } } fn update_max(active: &AtomicUsize, maximum: &AtomicUsize, amount: usize) { let next = active.fetch_add(amount, Ordering::AcqRel) + amount; maximum.fetch_max(next, Ordering::AcqRel); } struct FakeBackend { plans: Mutex>, stats: Arc, } impl FakeBackend { fn new(plans: impl IntoIterator) -> (Arc, Arc) { let stats = Arc::new(FakeStats::default()); ( Arc::new(Self { plans: Mutex::new(plans.into_iter().collect()), stats: Arc::clone(&stats), }), stats, ) } } impl GridSessionBackend for FakeBackend { fn login( &self, generation: u64, cancellation: CancellationToken, ) -> SessionFuture<'_, Result, SessionFailure>> { self.stats.logins.fetch_add(1, Ordering::AcqRel); let plan = self .plans .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) .pop_front() .unwrap_or(LoginPlan::Success { after: Duration::ZERO, signals: VecDeque::from([(Duration::ZERO, SessionSignal::Ready)]), }); let stats = Arc::clone(&self.stats); Box::pin(async move { let after = match &plan { LoginPlan::Failure { after, .. } | LoginPlan::Success { after, .. } => *after, }; tokio::select! { () = tokio::time::sleep(after) => {} () = cancellation.cancelled() => { return Err(SessionFailure::new(SessionFailureKind::TransientTransport)); } } match plan { LoginPlan::Failure { failure, .. } => Err(failure), LoginPlan::Success { signals, .. } => { stats.acquire(); let session: Box = Box::new(FakeSession { generation, signals, stats, released: false, }); Ok(session) } } }) } fn flush_audit( &self, _cancellation: CancellationToken, ) -> SessionFuture<'_, Result<(), SessionFailure>> { self.stats.flushes.fetch_add(1, Ordering::AcqRel); Box::pin(async { Ok(()) }) } } struct FakeSession { generation: u64, signals: VecDeque<(Duration, SessionSignal)>, stats: Arc, released: bool, } impl FakeSession { fn release(&mut self) { if !self.released { self.released = true; self.stats.release(); } } } impl GridSession for FakeSession { fn generation(&self) -> u64 { self.generation } fn next_signal(&mut self, cancellation: CancellationToken) -> SessionFuture<'_, SessionSignal> { let next = self.signals.pop_front(); Box::pin(async move { if let Some((after, signal)) = next { tokio::select! { () = tokio::time::sleep(after) => signal, () = cancellation.cancelled() => SessionSignal::Disconnected( SessionFailure::new(SessionFailureKind::TransientTransport) ), } } else { cancellation.cancelled().await; SessionSignal::Disconnected(SessionFailure::new( SessionFailureKind::TransientTransport, )) } }) } fn logout( mut self: Box, _cancellation: CancellationToken, ) -> SessionFuture<'static, Result<(), SessionFailure>> { self.stats.logouts.fetch_add(1, Ordering::AcqRel); self.release(); Box::pin(async { Ok(()) }) } } impl Drop for FakeSession { fn drop(&mut self) { self.release(); } } fn ready_then(after: Duration, failure: SessionFailureKind) -> LoginPlan { LoginPlan::Success { after: Duration::ZERO, signals: VecDeque::from([ (Duration::ZERO, SessionSignal::Ready), ( after, SessionSignal::Disconnected(SessionFailure::new(failure)), ), ]), } } fn test_policy() -> ReconnectPolicy { ReconnectPolicy { initial_delay: Duration::from_secs(1), maximum_delay: Duration::from_secs(8), readiness_timeout: Duration::from_secs(30), stable_reset_after: Duration::from_secs(10), shutdown_deadline: Duration::from_secs(2), jitter_basis_points: 0, instance_seed: 7, offline_work_capacity: 2, } } #[test] fn retry_hints_are_minimums_and_all_delays_remain_capped() { let mut policy = test_policy(); policy.jitter_basis_points = 5_000; for seed in 0..128 { policy.instance_seed = seed; let hinted = Duration::from_secs(6); let delay = policy.retry_delay(1, Some(hinted)); assert!(delay >= hinted); assert!(delay <= policy.maximum_delay); assert!(policy.retry_delay(64, None) <= policy.maximum_delay); } } fn start(plans: impl IntoIterator) -> (SessionSupervisorHandle, Arc) { let (backend, stats) = FakeBackend::new(plans); let erased: Arc = backend; let handle = SessionSupervisor::new(erased, test_policy(), 32, 256) .expect("supervisor") .start(); (handle, stats) } async fn settle() { for _ in 0..20 { tokio::task::yield_now().await; } } async fn wait_state(handle: &SessionSupervisorHandle, expected: SessionState) { for _ in 0..100 { if handle.state() == expected { return; } tokio::task::yield_now().await; } panic!("expected {expected:?}, got {:?}", handle.status()); } #[tokio::test(start_paused = true)] async fn paused_time_classifies_failures_backoff_flapping_and_stable_reset() { let plans = [ LoginPlan::Failure { after: Duration::ZERO, failure: SessionFailure::new(SessionFailureKind::TransientTransport), }, LoginPlan::Failure { after: Duration::ZERO, failure: SessionFailure::with_retry_after( SessionFailureKind::Maintenance, Duration::from_secs(5), ), }, ready_then(Duration::from_secs(1), SessionFailureKind::Kicked), ready_then( Duration::from_secs(11), SessionFailureKind::SimulatorDisconnected, ), LoginPlan::Success { after: Duration::ZERO, signals: VecDeque::from([(Duration::ZERO, SessionSignal::Ready)]), }, ]; let (mut handle, stats) = start(plans); wait_state(&handle, SessionState::Backoff).await; assert_eq!(handle.status().consecutive_failures, 1); tokio::time::advance(Duration::from_secs(1)).await; settle().await; wait_state(&handle, SessionState::Backoff).await; assert_eq!(handle.status().consecutive_failures, 2); tokio::time::advance(Duration::from_secs(5)).await; settle().await; wait_state(&handle, SessionState::Online).await; let first_online = handle.status().generation; tokio::time::advance(Duration::from_secs(1)).await; settle().await; wait_state(&handle, SessionState::Backoff).await; assert!(!handle.accepts_result(first_online)); tokio::time::advance(Duration::from_secs(4)).await; settle().await; wait_state(&handle, SessionState::Online).await; tokio::time::advance(Duration::from_secs(11)).await; settle().await; wait_state(&handle, SessionState::Backoff).await; assert_eq!(handle.status().consecutive_failures, 1); tokio::time::advance(Duration::from_secs(1)).await; settle().await; wait_state(&handle, SessionState::Online).await; handle.shutdown().await.expect("shutdown"); assert_eq!(stats.flushes.load(Ordering::Acquire), 1); } #[tokio::test(start_paused = true)] async fn invalid_credentials_and_configuration_block_until_operator_reconnect() { for failure in [ SessionFailureKind::InvalidCredentials, SessionFailureKind::InvalidConfiguration, ] { let plans = [ LoginPlan::Failure { after: Duration::ZERO, failure: SessionFailure::new(failure), }, LoginPlan::Success { after: Duration::ZERO, signals: VecDeque::from([(Duration::ZERO, SessionSignal::Ready)]), }, ]; let (mut handle, stats) = start(plans); wait_state(&handle, SessionState::AuthenticationBlocked).await; tokio::time::advance(Duration::from_hours(1)).await; settle().await; assert_eq!(stats.logins.load(Ordering::Acquire), 1); handle .control(SessionControl::ForceReconnect) .await .expect("force reconnect"); settle().await; wait_state(&handle, SessionState::Online).await; handle.shutdown().await.expect("shutdown"); } } #[tokio::test(start_paused = true)] async fn manual_pause_logout_reconnect_and_offline_work_are_predictable() { let (mut handle, stats) = start([LoginPlan::Success { after: Duration::ZERO, signals: VecDeque::from([(Duration::ZERO, SessionSignal::Ready)]), }]); wait_state(&handle, SessionState::Online).await; let generation = handle.status().generation; let token = handle .generation_token(generation) .expect("generation token"); handle.control(SessionControl::Pause).await.expect("pause"); settle().await; wait_state(&handle, SessionState::Paused).await; assert!(token.is_cancellation_requested()); assert!(!handle.accepts_result(generation)); assert_eq!( handle .submit(SessionWork::new("mutate", WorkKind::IdempotentMutation).expect("work")) .await .expect("decision"), WorkDisposition::RejectedOfflineMutation ); assert_eq!( handle .submit(SessionWork::new("read", WorkKind::ReadOnly).expect("work")) .await .expect("decision"), WorkDisposition::Queued ); handle .control(SessionControl::Resume) .await .expect("resume"); settle().await; wait_state(&handle, SessionState::Online).await; assert_ne!(handle.status().generation, generation); assert_eq!( handle .submit(SessionWork::new("read", WorkKind::ReadOnly).expect("work")) .await .expect("dedupe decision"), WorkDisposition::RejectedDuplicate ); let before_force = handle.status().generation; handle .control(SessionControl::ForceReconnect) .await .expect("active reconnect"); settle().await; wait_state(&handle, SessionState::Online).await; assert_ne!(handle.status().generation, before_force); handle .control(SessionControl::Logout) .await .expect("logout"); settle().await; wait_state(&handle, SessionState::Stopped).await; handle .control(SessionControl::ForceReconnect) .await .expect("reconnect"); settle().await; wait_state(&handle, SessionState::Online).await; handle.shutdown().await.expect("shutdown"); assert_eq!(stats.active_sessions.load(Ordering::Acquire), 0); } #[tokio::test(start_paused = true)] async fn transport_connection_is_distinct_from_full_agent_readiness() { let (mut handle, _) = start([LoginPlan::Success { after: Duration::ZERO, signals: VecDeque::from([(Duration::from_secs(10), SessionSignal::Ready)]), }]); wait_state(&handle, SessionState::Degraded).await; let degraded = handle.status(); assert!(degraded.transport_connected); assert!(!degraded.agent_ready); assert!(!handle.accepts_result(degraded.generation)); tokio::time::advance(Duration::from_secs(10)).await; settle().await; wait_state(&handle, SessionState::Online).await; assert!(handle.status().agent_ready); handle.shutdown().await.expect("shutdown"); } #[tokio::test(start_paused = true)] async fn readiness_timeout_logs_out_and_retries_instead_of_staying_degraded() { let plans = [ LoginPlan::Success { after: Duration::ZERO, signals: VecDeque::new(), }, LoginPlan::Success { after: Duration::ZERO, signals: VecDeque::from([(Duration::ZERO, SessionSignal::Ready)]), }, ]; let (backend, stats) = FakeBackend::new(plans); let mut policy = test_policy(); policy.readiness_timeout = Duration::from_secs(5); let erased: Arc = backend; let mut handle = SessionSupervisor::new(erased, policy, 32, 256) .expect("supervisor") .start(); wait_state(&handle, SessionState::Degraded).await; tokio::time::advance(Duration::from_secs(5)).await; settle().await; wait_state(&handle, SessionState::Backoff).await; assert_eq!(stats.logouts.load(Ordering::Acquire), 1); tokio::time::advance(Duration::from_secs(1)).await; settle().await; wait_state(&handle, SessionState::Online).await; handle.shutdown().await.expect("shutdown"); assert_eq!(stats.active_sessions.load(Ordering::Acquire), 0); } #[tokio::test(start_paused = true)] async fn shutdown_covers_degraded_authentication_blocked_stopped_and_online_states() { let cases = [ ( LoginPlan::Success { after: Duration::ZERO, signals: VecDeque::from([(Duration::from_hours(1), SessionSignal::Ready)]), }, SessionState::Degraded, ), ( LoginPlan::Failure { after: Duration::ZERO, failure: SessionFailure::new(SessionFailureKind::InvalidCredentials), }, SessionState::AuthenticationBlocked, ), ( LoginPlan::Success { after: Duration::ZERO, signals: VecDeque::from([(Duration::ZERO, SessionSignal::Ready)]), }, SessionState::Online, ), ]; for (plan, expected) in cases { let (mut handle, stats) = start([plan]); wait_state(&handle, expected).await; if expected == SessionState::Online { handle .control(SessionControl::Logout) .await .expect("logout"); settle().await; wait_state(&handle, SessionState::Stopped).await; } handle.shutdown().await.expect("shutdown"); assert_eq!(stats.active_sessions.load(Ordering::Acquire), 0); assert_eq!(stats.flushes.load(Ordering::Acquire), 1); } } #[tokio::test(start_paused = true)] async fn fake_grid_reconnects_keep_exactly_one_resource_set_and_no_stale_execution() { let plans = [ ready_then(Duration::from_secs(1), SessionFailureKind::Maintenance), ready_then(Duration::from_secs(1), SessionFailureKind::Kicked), LoginPlan::Success { after: Duration::ZERO, signals: VecDeque::from([(Duration::ZERO, SessionSignal::Ready)]), }, ]; let (mut handle, stats) = start(plans); wait_state(&handle, SessionState::Online).await; let stale = handle.status().generation; tokio::time::advance(Duration::from_secs(1)).await; settle().await; tokio::time::advance(Duration::from_secs(1)).await; settle().await; wait_state(&handle, SessionState::Online).await; tokio::time::advance(Duration::from_secs(1)).await; settle().await; tokio::time::advance(Duration::from_secs(2)).await; settle().await; wait_state(&handle, SessionState::Online).await; assert!(!handle.accepts_result(stale)); assert!(handle.accepts_result(handle.status().generation)); assert_eq!(stats.max_sessions.load(Ordering::Acquire), 1); assert_eq!(stats.max_callbacks.load(Ordering::Acquire), 4); assert_eq!(stats.max_workers.load(Ordering::Acquire), 2); assert_eq!(stats.max_sockets.load(Ordering::Acquire), 1); handle.shutdown().await.expect("shutdown"); assert_eq!(stats.active_sessions.load(Ordering::Acquire), 0); assert_eq!(stats.active_callbacks.load(Ordering::Acquire), 0); assert_eq!(stats.active_workers.load(Ordering::Acquire), 0); assert_eq!(stats.active_sockets.load(Ordering::Acquire), 0); assert_eq!(stats.logouts.load(Ordering::Acquire), 3); assert_eq!(stats.flushes.load(Ordering::Acquire), 1); } #[tokio::test(start_paused = true)] async fn shutdown_is_cancellation_driven_during_connect_backoff_paused_and_online() { let scenarios = [ LoginPlan::Success { after: Duration::from_hours(1), signals: VecDeque::new(), }, LoginPlan::Failure { after: Duration::ZERO, failure: SessionFailure::new(SessionFailureKind::TransientTransport), }, LoginPlan::Success { after: Duration::ZERO, signals: VecDeque::from([(Duration::ZERO, SessionSignal::Ready)]), }, ]; for (index, plan) in scenarios.into_iter().enumerate() { let (mut handle, stats) = start([plan]); settle().await; if index == 2 { wait_state(&handle, SessionState::Online).await; handle.control(SessionControl::Pause).await.expect("pause"); settle().await; wait_state(&handle, SessionState::Paused).await; } handle.shutdown().await.expect("bounded shutdown"); assert_eq!(handle.state(), SessionState::Stopped); assert_eq!(stats.active_sessions.load(Ordering::Acquire), 0); assert_eq!(stats.flushes.load(Ordering::Acquire), 1); } }