feat(grid-agent): supervise grid sessions (#121)
This commit is contained in:
526
crates/metacrate-grid-agent/src/session_tests.rs
Normal file
526
crates/metacrate-grid-agent/src/session_tests.rs
Normal file
@@ -0,0 +1,526 @@
|
||||
use crate::session::*;
|
||||
use libremetaverse_types::compat::CancellationToken;
|
||||
use std::collections::VecDeque;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::Duration;
|
||||
|
||||
#[derive(Clone)]
|
||||
enum LoginPlan {
|
||||
Failure {
|
||||
after: Duration,
|
||||
failure: SessionFailure,
|
||||
},
|
||||
Success {
|
||||
after: Duration,
|
||||
signals: VecDeque<(Duration, SessionSignal)>,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct FakeStats {
|
||||
logins: AtomicUsize,
|
||||
logouts: AtomicUsize,
|
||||
flushes: AtomicUsize,
|
||||
active_sessions: AtomicUsize,
|
||||
active_callbacks: AtomicUsize,
|
||||
active_workers: AtomicUsize,
|
||||
active_sockets: AtomicUsize,
|
||||
max_sessions: AtomicUsize,
|
||||
max_callbacks: AtomicUsize,
|
||||
max_workers: AtomicUsize,
|
||||
max_sockets: AtomicUsize,
|
||||
}
|
||||
|
||||
impl FakeStats {
|
||||
fn acquire(&self) {
|
||||
update_max(&self.active_sessions, &self.max_sessions, 1);
|
||||
update_max(&self.active_callbacks, &self.max_callbacks, 4);
|
||||
update_max(&self.active_workers, &self.max_workers, 2);
|
||||
update_max(&self.active_sockets, &self.max_sockets, 1);
|
||||
}
|
||||
|
||||
fn release(&self) {
|
||||
self.active_sessions.fetch_sub(1, Ordering::AcqRel);
|
||||
self.active_callbacks.fetch_sub(4, Ordering::AcqRel);
|
||||
self.active_workers.fetch_sub(2, Ordering::AcqRel);
|
||||
self.active_sockets.fetch_sub(1, Ordering::AcqRel);
|
||||
}
|
||||
}
|
||||
|
||||
fn update_max(active: &AtomicUsize, maximum: &AtomicUsize, amount: usize) {
|
||||
let next = active.fetch_add(amount, Ordering::AcqRel) + amount;
|
||||
maximum.fetch_max(next, Ordering::AcqRel);
|
||||
}
|
||||
|
||||
struct FakeBackend {
|
||||
plans: Mutex<VecDeque<LoginPlan>>,
|
||||
stats: Arc<FakeStats>,
|
||||
}
|
||||
|
||||
impl FakeBackend {
|
||||
fn new(plans: impl IntoIterator<Item = LoginPlan>) -> (Arc<Self>, Arc<FakeStats>) {
|
||||
let stats = Arc::new(FakeStats::default());
|
||||
(
|
||||
Arc::new(Self {
|
||||
plans: Mutex::new(plans.into_iter().collect()),
|
||||
stats: Arc::clone(&stats),
|
||||
}),
|
||||
stats,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
impl GridSessionBackend for FakeBackend {
|
||||
fn login(
|
||||
&self,
|
||||
generation: u64,
|
||||
cancellation: CancellationToken,
|
||||
) -> SessionFuture<'_, Result<Box<dyn GridSession>, SessionFailure>> {
|
||||
self.stats.logins.fetch_add(1, Ordering::AcqRel);
|
||||
let plan = self
|
||||
.plans
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.pop_front()
|
||||
.unwrap_or(LoginPlan::Success {
|
||||
after: Duration::ZERO,
|
||||
signals: VecDeque::from([(Duration::ZERO, SessionSignal::Ready)]),
|
||||
});
|
||||
let stats = Arc::clone(&self.stats);
|
||||
Box::pin(async move {
|
||||
let after = match &plan {
|
||||
LoginPlan::Failure { after, .. } | LoginPlan::Success { after, .. } => *after,
|
||||
};
|
||||
tokio::select! {
|
||||
() = tokio::time::sleep(after) => {}
|
||||
() = cancellation.cancelled() => {
|
||||
return Err(SessionFailure::new(SessionFailureKind::TransientTransport));
|
||||
}
|
||||
}
|
||||
match plan {
|
||||
LoginPlan::Failure { failure, .. } => Err(failure),
|
||||
LoginPlan::Success { signals, .. } => {
|
||||
stats.acquire();
|
||||
let session: Box<dyn GridSession> = Box::new(FakeSession {
|
||||
generation,
|
||||
signals,
|
||||
stats,
|
||||
released: false,
|
||||
});
|
||||
Ok(session)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
fn flush_audit(
|
||||
&self,
|
||||
_cancellation: CancellationToken,
|
||||
) -> SessionFuture<'_, Result<(), SessionFailure>> {
|
||||
self.stats.flushes.fetch_add(1, Ordering::AcqRel);
|
||||
Box::pin(async { Ok(()) })
|
||||
}
|
||||
}
|
||||
|
||||
struct FakeSession {
|
||||
generation: u64,
|
||||
signals: VecDeque<(Duration, SessionSignal)>,
|
||||
stats: Arc<FakeStats>,
|
||||
released: bool,
|
||||
}
|
||||
|
||||
impl FakeSession {
|
||||
fn release(&mut self) {
|
||||
if !self.released {
|
||||
self.released = true;
|
||||
self.stats.release();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl GridSession for FakeSession {
|
||||
fn generation(&self) -> u64 {
|
||||
self.generation
|
||||
}
|
||||
|
||||
fn next_signal(&mut self, cancellation: CancellationToken) -> SessionFuture<'_, SessionSignal> {
|
||||
let next = self.signals.pop_front();
|
||||
Box::pin(async move {
|
||||
if let Some((after, signal)) = next {
|
||||
tokio::select! {
|
||||
() = tokio::time::sleep(after) => signal,
|
||||
() = cancellation.cancelled() => SessionSignal::Disconnected(
|
||||
SessionFailure::new(SessionFailureKind::TransientTransport)
|
||||
),
|
||||
}
|
||||
} else {
|
||||
cancellation.cancelled().await;
|
||||
SessionSignal::Disconnected(SessionFailure::new(
|
||||
SessionFailureKind::TransientTransport,
|
||||
))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
fn logout(
|
||||
mut self: Box<Self>,
|
||||
_cancellation: CancellationToken,
|
||||
) -> SessionFuture<'static, Result<(), SessionFailure>> {
|
||||
self.stats.logouts.fetch_add(1, Ordering::AcqRel);
|
||||
self.release();
|
||||
Box::pin(async { Ok(()) })
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for FakeSession {
|
||||
fn drop(&mut self) {
|
||||
self.release();
|
||||
}
|
||||
}
|
||||
|
||||
fn ready_then(after: Duration, failure: SessionFailureKind) -> LoginPlan {
|
||||
LoginPlan::Success {
|
||||
after: Duration::ZERO,
|
||||
signals: VecDeque::from([
|
||||
(Duration::ZERO, SessionSignal::Ready),
|
||||
(
|
||||
after,
|
||||
SessionSignal::Disconnected(SessionFailure::new(failure)),
|
||||
),
|
||||
]),
|
||||
}
|
||||
}
|
||||
|
||||
fn test_policy() -> ReconnectPolicy {
|
||||
ReconnectPolicy {
|
||||
initial_delay: Duration::from_secs(1),
|
||||
maximum_delay: Duration::from_secs(8),
|
||||
stable_reset_after: Duration::from_secs(10),
|
||||
shutdown_deadline: Duration::from_secs(2),
|
||||
jitter_basis_points: 0,
|
||||
instance_seed: 7,
|
||||
offline_work_capacity: 2,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn retry_hints_are_minimums_and_all_delays_remain_capped() {
|
||||
let mut policy = test_policy();
|
||||
policy.jitter_basis_points = 5_000;
|
||||
for seed in 0..128 {
|
||||
policy.instance_seed = seed;
|
||||
let hinted = Duration::from_secs(6);
|
||||
let delay = policy.retry_delay(1, Some(hinted));
|
||||
assert!(delay >= hinted);
|
||||
assert!(delay <= policy.maximum_delay);
|
||||
assert!(policy.retry_delay(64, None) <= policy.maximum_delay);
|
||||
}
|
||||
}
|
||||
|
||||
fn start(plans: impl IntoIterator<Item = LoginPlan>) -> (SessionSupervisorHandle, Arc<FakeStats>) {
|
||||
let (backend, stats) = FakeBackend::new(plans);
|
||||
let erased: Arc<dyn GridSessionBackend> = backend;
|
||||
let handle = SessionSupervisor::new(erased, test_policy(), 32, 256)
|
||||
.expect("supervisor")
|
||||
.start();
|
||||
(handle, stats)
|
||||
}
|
||||
|
||||
async fn settle() {
|
||||
for _ in 0..20 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
}
|
||||
|
||||
async fn wait_state(handle: &SessionSupervisorHandle, expected: SessionState) {
|
||||
for _ in 0..100 {
|
||||
if handle.state() == expected {
|
||||
return;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
panic!("expected {expected:?}, got {:?}", handle.status());
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn paused_time_classifies_failures_backoff_flapping_and_stable_reset() {
|
||||
let plans = [
|
||||
LoginPlan::Failure {
|
||||
after: Duration::ZERO,
|
||||
failure: SessionFailure::new(SessionFailureKind::TransientTransport),
|
||||
},
|
||||
LoginPlan::Failure {
|
||||
after: Duration::ZERO,
|
||||
failure: SessionFailure::with_retry_after(
|
||||
SessionFailureKind::Maintenance,
|
||||
Duration::from_secs(5),
|
||||
),
|
||||
},
|
||||
ready_then(Duration::from_secs(1), SessionFailureKind::Kicked),
|
||||
ready_then(
|
||||
Duration::from_secs(11),
|
||||
SessionFailureKind::SimulatorDisconnected,
|
||||
),
|
||||
LoginPlan::Success {
|
||||
after: Duration::ZERO,
|
||||
signals: VecDeque::from([(Duration::ZERO, SessionSignal::Ready)]),
|
||||
},
|
||||
];
|
||||
let (mut handle, stats) = start(plans);
|
||||
wait_state(&handle, SessionState::Backoff).await;
|
||||
assert_eq!(handle.status().consecutive_failures, 1);
|
||||
tokio::time::advance(Duration::from_secs(1)).await;
|
||||
settle().await;
|
||||
wait_state(&handle, SessionState::Backoff).await;
|
||||
assert_eq!(handle.status().consecutive_failures, 2);
|
||||
tokio::time::advance(Duration::from_secs(5)).await;
|
||||
settle().await;
|
||||
wait_state(&handle, SessionState::Online).await;
|
||||
let first_online = handle.status().generation;
|
||||
tokio::time::advance(Duration::from_secs(1)).await;
|
||||
settle().await;
|
||||
wait_state(&handle, SessionState::Backoff).await;
|
||||
assert!(!handle.accepts_result(first_online));
|
||||
tokio::time::advance(Duration::from_secs(4)).await;
|
||||
settle().await;
|
||||
wait_state(&handle, SessionState::Online).await;
|
||||
tokio::time::advance(Duration::from_secs(11)).await;
|
||||
settle().await;
|
||||
wait_state(&handle, SessionState::Backoff).await;
|
||||
assert_eq!(handle.status().consecutive_failures, 1);
|
||||
tokio::time::advance(Duration::from_secs(1)).await;
|
||||
settle().await;
|
||||
wait_state(&handle, SessionState::Online).await;
|
||||
handle.shutdown().await.expect("shutdown");
|
||||
assert_eq!(stats.flushes.load(Ordering::Acquire), 1);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn invalid_credentials_and_configuration_block_until_operator_reconnect() {
|
||||
for failure in [
|
||||
SessionFailureKind::InvalidCredentials,
|
||||
SessionFailureKind::InvalidConfiguration,
|
||||
] {
|
||||
let plans = [
|
||||
LoginPlan::Failure {
|
||||
after: Duration::ZERO,
|
||||
failure: SessionFailure::new(failure),
|
||||
},
|
||||
LoginPlan::Success {
|
||||
after: Duration::ZERO,
|
||||
signals: VecDeque::from([(Duration::ZERO, SessionSignal::Ready)]),
|
||||
},
|
||||
];
|
||||
let (mut handle, stats) = start(plans);
|
||||
wait_state(&handle, SessionState::AuthenticationBlocked).await;
|
||||
tokio::time::advance(Duration::from_hours(1)).await;
|
||||
settle().await;
|
||||
assert_eq!(stats.logins.load(Ordering::Acquire), 1);
|
||||
handle
|
||||
.control(SessionControl::ForceReconnect)
|
||||
.await
|
||||
.expect("force reconnect");
|
||||
settle().await;
|
||||
wait_state(&handle, SessionState::Online).await;
|
||||
handle.shutdown().await.expect("shutdown");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn manual_pause_logout_reconnect_and_offline_work_are_predictable() {
|
||||
let (mut handle, stats) = start([LoginPlan::Success {
|
||||
after: Duration::ZERO,
|
||||
signals: VecDeque::from([(Duration::ZERO, SessionSignal::Ready)]),
|
||||
}]);
|
||||
wait_state(&handle, SessionState::Online).await;
|
||||
let generation = handle.status().generation;
|
||||
let token = handle
|
||||
.generation_token(generation)
|
||||
.expect("generation token");
|
||||
handle.control(SessionControl::Pause).await.expect("pause");
|
||||
settle().await;
|
||||
wait_state(&handle, SessionState::Paused).await;
|
||||
assert!(token.is_cancellation_requested());
|
||||
assert!(!handle.accepts_result(generation));
|
||||
assert_eq!(
|
||||
handle
|
||||
.submit(SessionWork::new("mutate", WorkKind::IdempotentMutation).expect("work"))
|
||||
.await
|
||||
.expect("decision"),
|
||||
WorkDisposition::RejectedOfflineMutation
|
||||
);
|
||||
assert_eq!(
|
||||
handle
|
||||
.submit(SessionWork::new("read", WorkKind::ReadOnly).expect("work"))
|
||||
.await
|
||||
.expect("decision"),
|
||||
WorkDisposition::Queued
|
||||
);
|
||||
handle
|
||||
.control(SessionControl::Resume)
|
||||
.await
|
||||
.expect("resume");
|
||||
settle().await;
|
||||
wait_state(&handle, SessionState::Online).await;
|
||||
assert_ne!(handle.status().generation, generation);
|
||||
assert_eq!(
|
||||
handle
|
||||
.submit(SessionWork::new("read", WorkKind::ReadOnly).expect("work"))
|
||||
.await
|
||||
.expect("dedupe decision"),
|
||||
WorkDisposition::RejectedDuplicate
|
||||
);
|
||||
let before_force = handle.status().generation;
|
||||
handle
|
||||
.control(SessionControl::ForceReconnect)
|
||||
.await
|
||||
.expect("active reconnect");
|
||||
settle().await;
|
||||
wait_state(&handle, SessionState::Online).await;
|
||||
assert_ne!(handle.status().generation, before_force);
|
||||
handle
|
||||
.control(SessionControl::Logout)
|
||||
.await
|
||||
.expect("logout");
|
||||
settle().await;
|
||||
wait_state(&handle, SessionState::Stopped).await;
|
||||
handle
|
||||
.control(SessionControl::ForceReconnect)
|
||||
.await
|
||||
.expect("reconnect");
|
||||
settle().await;
|
||||
wait_state(&handle, SessionState::Online).await;
|
||||
handle.shutdown().await.expect("shutdown");
|
||||
assert_eq!(stats.active_sessions.load(Ordering::Acquire), 0);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn transport_connection_is_distinct_from_full_agent_readiness() {
|
||||
let (mut handle, _) = start([LoginPlan::Success {
|
||||
after: Duration::ZERO,
|
||||
signals: VecDeque::from([(Duration::from_secs(10), SessionSignal::Ready)]),
|
||||
}]);
|
||||
wait_state(&handle, SessionState::Degraded).await;
|
||||
let degraded = handle.status();
|
||||
assert!(degraded.transport_connected);
|
||||
assert!(!degraded.agent_ready);
|
||||
assert!(!handle.accepts_result(degraded.generation));
|
||||
tokio::time::advance(Duration::from_secs(10)).await;
|
||||
settle().await;
|
||||
wait_state(&handle, SessionState::Online).await;
|
||||
assert!(handle.status().agent_ready);
|
||||
handle.shutdown().await.expect("shutdown");
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn shutdown_covers_degraded_authentication_blocked_stopped_and_online_states() {
|
||||
let cases = [
|
||||
(
|
||||
LoginPlan::Success {
|
||||
after: Duration::ZERO,
|
||||
signals: VecDeque::from([(Duration::from_hours(1), SessionSignal::Ready)]),
|
||||
},
|
||||
SessionState::Degraded,
|
||||
),
|
||||
(
|
||||
LoginPlan::Failure {
|
||||
after: Duration::ZERO,
|
||||
failure: SessionFailure::new(SessionFailureKind::InvalidCredentials),
|
||||
},
|
||||
SessionState::AuthenticationBlocked,
|
||||
),
|
||||
(
|
||||
LoginPlan::Success {
|
||||
after: Duration::ZERO,
|
||||
signals: VecDeque::from([(Duration::ZERO, SessionSignal::Ready)]),
|
||||
},
|
||||
SessionState::Online,
|
||||
),
|
||||
];
|
||||
for (plan, expected) in cases {
|
||||
let (mut handle, stats) = start([plan]);
|
||||
wait_state(&handle, expected).await;
|
||||
if expected == SessionState::Online {
|
||||
handle
|
||||
.control(SessionControl::Logout)
|
||||
.await
|
||||
.expect("logout");
|
||||
settle().await;
|
||||
wait_state(&handle, SessionState::Stopped).await;
|
||||
}
|
||||
handle.shutdown().await.expect("shutdown");
|
||||
assert_eq!(stats.active_sessions.load(Ordering::Acquire), 0);
|
||||
assert_eq!(stats.flushes.load(Ordering::Acquire), 1);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn fake_grid_reconnects_keep_exactly_one_resource_set_and_no_stale_execution() {
|
||||
let plans = [
|
||||
ready_then(Duration::from_secs(1), SessionFailureKind::Maintenance),
|
||||
ready_then(Duration::from_secs(1), SessionFailureKind::Kicked),
|
||||
LoginPlan::Success {
|
||||
after: Duration::ZERO,
|
||||
signals: VecDeque::from([(Duration::ZERO, SessionSignal::Ready)]),
|
||||
},
|
||||
];
|
||||
let (mut handle, stats) = start(plans);
|
||||
wait_state(&handle, SessionState::Online).await;
|
||||
let stale = handle.status().generation;
|
||||
tokio::time::advance(Duration::from_secs(1)).await;
|
||||
settle().await;
|
||||
tokio::time::advance(Duration::from_secs(1)).await;
|
||||
settle().await;
|
||||
wait_state(&handle, SessionState::Online).await;
|
||||
tokio::time::advance(Duration::from_secs(1)).await;
|
||||
settle().await;
|
||||
tokio::time::advance(Duration::from_secs(2)).await;
|
||||
settle().await;
|
||||
wait_state(&handle, SessionState::Online).await;
|
||||
assert!(!handle.accepts_result(stale));
|
||||
assert!(handle.accepts_result(handle.status().generation));
|
||||
assert_eq!(stats.max_sessions.load(Ordering::Acquire), 1);
|
||||
assert_eq!(stats.max_callbacks.load(Ordering::Acquire), 4);
|
||||
assert_eq!(stats.max_workers.load(Ordering::Acquire), 2);
|
||||
assert_eq!(stats.max_sockets.load(Ordering::Acquire), 1);
|
||||
handle.shutdown().await.expect("shutdown");
|
||||
assert_eq!(stats.active_sessions.load(Ordering::Acquire), 0);
|
||||
assert_eq!(stats.active_callbacks.load(Ordering::Acquire), 0);
|
||||
assert_eq!(stats.active_workers.load(Ordering::Acquire), 0);
|
||||
assert_eq!(stats.active_sockets.load(Ordering::Acquire), 0);
|
||||
assert_eq!(stats.logouts.load(Ordering::Acquire), 3);
|
||||
assert_eq!(stats.flushes.load(Ordering::Acquire), 1);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn shutdown_is_cancellation_driven_during_connect_backoff_paused_and_online() {
|
||||
let scenarios = [
|
||||
LoginPlan::Success {
|
||||
after: Duration::from_hours(1),
|
||||
signals: VecDeque::new(),
|
||||
},
|
||||
LoginPlan::Failure {
|
||||
after: Duration::ZERO,
|
||||
failure: SessionFailure::new(SessionFailureKind::TransientTransport),
|
||||
},
|
||||
LoginPlan::Success {
|
||||
after: Duration::ZERO,
|
||||
signals: VecDeque::from([(Duration::ZERO, SessionSignal::Ready)]),
|
||||
},
|
||||
];
|
||||
for (index, plan) in scenarios.into_iter().enumerate() {
|
||||
let (mut handle, stats) = start([plan]);
|
||||
settle().await;
|
||||
if index == 2 {
|
||||
wait_state(&handle, SessionState::Online).await;
|
||||
handle.control(SessionControl::Pause).await.expect("pause");
|
||||
settle().await;
|
||||
wait_state(&handle, SessionState::Paused).await;
|
||||
}
|
||||
handle.shutdown().await.expect("bounded shutdown");
|
||||
assert_eq!(handle.state(), SessionState::Stopped);
|
||||
assert_eq!(stats.active_sessions.load(Ordering::Acquire), 0);
|
||||
assert_eq!(stats.flushes.load(Ordering::Acquire), 1);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user