hugo/MetaCrate
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

fix: harden OpenSim session readiness
Some checks failed
CI / rust-skia (Rust only) (push) Has been cancelled
Details
CI / required (push) Has been cancelled
Details

Browse Source
This commit is contained in:
Chili Palmer
2026-08-21 20:37:16 +02:00
parent db25a977b7
commit adf5165033
14 changed files with 503 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
crates/metacrate-grid-agent/src/acceptance.rs
View File

@@ -448,6 +448,7 @@ pub async fn run_deterministic_acceptance(
ReconnectPolicy {
initial_delay: Duration::from_millis(10),
maximum_delay: Duration::from_millis(20),
readiness_timeout: Duration::from_secs(1),
stable_reset_after: Duration::from_secs(1),
shutdown_deadline: Duration::from_secs(1),
jitter_basis_points: 0,

102
crates/metacrate-grid-agent/src/backend.rs
View File

@@ -929,35 +929,43 @@ impl crate::session::GridSessionBackend for LibremetaverseSessionBackend {
.delivery_generation
.write()
.unwrap_or_else(std::sync::PoisonError::into_inner) = None;
let mut params = self
.network
.native_default_login_params(
self.first_name.clone(),
self.last_name.clone(),
self.password.expose_secret().to_owned(),
"MetaCrate".to_owned(),
env!("CARGO_PKG_VERSION").to_owned(),
)
.map_err(|_| {
crate::session::SessionFailure::new(
crate::session::SessionFailureKind::InvalidConfiguration,
let mut start = "last";
loop {
let mut params = self
.network
.native_default_login_params(
self.first_name.clone(),
self.last_name.clone(),
self.password.expose_secret().to_owned(),
"MetaCrate".to_owned(),
env!("CARGO_PKG_VERSION").to_owned(),
)
})?;
params.uri.clone_from(&self.login_url);
"last".clone_into(&mut params.start);
let logged_in = self
.network
.native_login(params, Some(cancellation))
.await
.map_err(|_| {
crate::session::SessionFailure::new(
crate::session::SessionFailureKind::TransientTransport,
)
})?;
if !logged_in {
return Err(classify_native_login_failure(
&self.network.native_login_error_key(),
));
.map_err(|_| {
crate::session::SessionFailure::new(
crate::session::SessionFailureKind::InvalidConfiguration,
)
})?;
params.uri.clone_from(&self.login_url);
start.clone_into(&mut params.start);
let logged_in = self
.network
.native_login(params, Some(cancellation.clone()))
.await
.map_err(|_| {
crate::session::SessionFailure::new(
crate::session::SessionFailureKind::TransientTransport,
)
})?;
if logged_in {
break;
}
let error_key = self.network.native_login_error_key();
let message = self.network.native_login_message();
if start == "last" && native_last_location_unavailable(&message) {
start = "home";
continue;
}
return Err(classify_native_login_failure(&error_key, &message));
}
// Native login has already installed the current simulator and
@@ -1252,9 +1260,16 @@ fn native_delivery_id(prefix: &str, fields: &[&str]) -> String {
}
#[cfg(feature = "live-grid")]
fn classify_native_login_failure(error_key: &str) -> crate::session::SessionFailure {
fn classify_native_login_failure(error_key: &str, message: &str) -> crate::session::SessionFailure {
let normalized = error_key.trim().to_ascii_lowercase();
let kind = if matches!(
let kind = if native_last_location_unavailable(message)
|| message
.trim()
.to_ascii_lowercase()
.contains("already logged in")
{
crate::session::SessionFailureKind::ServerFailure
} else if matches!(
normalized.as_str(),
"key" | "password" | "credential" | "account" | "username" | "user"
) {
@@ -1267,6 +1282,13 @@ fn classify_native_login_failure(error_key: &str) -> crate::session::SessionFail
crate::session::SessionFailure::new(kind)
}
#[cfg(feature = "live-grid")]
fn native_last_location_unavailable(message: &str) -> bool {
let message = message.trim().to_ascii_lowercase();
message.contains("failed to verify user presence")
|| message.contains("access denied to region")
}
impl GridBackend for OfflineGridBackend {
fn name(&self) -> &'static str {
"offline-fake"
@@ -1315,4 +1337,24 @@ mod tests {
cancellation.cancel();
run.await.expect("clean cancellation");
}
#[cfg(feature = "live-grid")]
#[test]
fn stale_presence_is_retryable_without_masking_bad_credentials() {
assert!(native_last_location_unavailable(
"Failed to verify user presence in the grid, access denied to region",
));
assert_eq!(
classify_native_login_failure(
"account",
"Failed to verify user presence in the grid, access denied to region",
)
.kind(),
crate::session::SessionFailureKind::ServerFailure,
);
assert_eq!(
classify_native_login_failure("account", "Invalid credentials").kind(),
crate::session::SessionFailureKind::InvalidCredentials,
);
}
}

9
crates/metacrate-grid-agent/src/config.rs
View File

@@ -1120,6 +1120,7 @@ struct RawBehavior {
struct RawReconnect {
initial_delay_milliseconds: Option<u64>,
maximum_delay_seconds: Option<u64>,
readiness_timeout_seconds: Option<u64>,
stable_reset_seconds: Option<u64>,
jitter_basis_points: Option<u16>,
offline_work_capacity: Option<usize>,
@@ -1318,7 +1319,7 @@ fn resolve<E: Environment>(
let timeouts = Timeouts {
startup: checked_duration(
"timeouts.startup_seconds",
raw.timeouts.startup.unwrap_or(30),
raw.timeouts.startup.unwrap_or(120),
1,
300,
)?,
@@ -1341,6 +1342,11 @@ fn resolve<E: Environment>(
raw.reconnect.initial_delay_milliseconds.unwrap_or(1_000),
),
maximum_delay: Duration::from_secs(raw.reconnect.maximum_delay_seconds.unwrap_or(60)),
readiness_timeout: Duration::from_secs(
raw.reconnect
.readiness_timeout_seconds
.unwrap_or(reconnect_defaults.readiness_timeout.as_secs()),
),
stable_reset_after: Duration::from_secs(raw.reconnect.stable_reset_seconds.unwrap_or(120)),
shutdown_deadline: timeouts.shutdown,
jitter_basis_points: raw
@@ -1952,6 +1958,7 @@ mod tests {
.load()
.unwrap();
assert_eq!(config.storage_path, PathBuf::from("operator-data"));
assert_eq!(config.timeouts.startup, Duration::from_mins(2));
let paths = PlatformPaths::from_environment(&environment);
assert!(paths.config_file.ends_with("config.yml"));
assert!(paths.data_directory.ends_with("grid-agent"));

37
crates/metacrate-grid-agent/src/session.rs
View File

@@ -20,6 +20,7 @@ const MAX_OFFLINE_WORK: usize = 1_024;
const MAX_SEEN_WORK: usize = 4_096;
const MAX_BACKOFF: Duration = Duration::from_hours(1);
const MAX_STABLE_RESET: Duration = Duration::from_hours(24);
const MAX_READINESS_TIMEOUT: Duration = Duration::from_mins(10);
const MAX_SHUTDOWN: Duration = Duration::from_mins(1);
pub type SessionFuture<'a, T> = Pin<Box<dyn Future<Output = T> + Send + 'a>>;
@@ -201,6 +202,7 @@ pub trait GridSessionBackend: Send + Sync + 'static {
pub struct ReconnectPolicy {
pub initial_delay: Duration,
pub maximum_delay: Duration,
pub readiness_timeout: Duration,
pub stable_reset_after: Duration,
pub shutdown_deadline: Duration,
pub jitter_basis_points: u16,
@@ -213,6 +215,7 @@ impl Default for ReconnectPolicy {
Self {
initial_delay: Duration::from_secs(1),
maximum_delay: Duration::from_mins(1),
readiness_timeout: Duration::from_secs(30),
stable_reset_after: Duration::from_mins(2),
shutdown_deadline: Duration::from_secs(10),
jitter_basis_points: 2_000,
@@ -237,6 +240,8 @@ impl ReconnectPolicy {
if self.initial_delay.is_zero()
|| self.initial_delay > self.maximum_delay
|| self.maximum_delay > MAX_BACKOFF
|| self.readiness_timeout.is_zero()
|| self.readiness_timeout > MAX_READINESS_TIMEOUT
|| self.stable_reset_after.is_zero()
|| self.stable_reset_after > MAX_STABLE_RESET
|| self.shutdown_deadline.is_zero()
@@ -608,6 +613,7 @@ enum LoginEvent {
enum ActiveEvent {
Cancelled,
Command(Option<Command>),
ReadinessTimedOut,
Signal(SessionSignal),
}
@@ -635,6 +641,7 @@ impl Runtime {
let mut desired = DesiredState::Running;
let mut session: Option<Box<dyn GridSession>> = None;
let mut ready = false;
let mut readiness_deadline = None;
let mut stable_since = None;
loop {
@@ -711,6 +718,8 @@ impl Runtime {
Ok(connected) if connected.generation() == generation => {
session = Some(connected);
ready = false;
readiness_deadline =
Some(Instant::now() + self.policy.readiness_timeout);
stable_since = None;
self.transition(
SessionState::Degraded,
@@ -756,9 +765,18 @@ impl Runtime {
};
let signal = active.next_signal(token);
tokio::pin!(signal);
let readiness = async {
if let Some(deadline) = readiness_deadline {
tokio::time::sleep_until(deadline).await;
} else {
std::future::pending::<()>().await;
}
};
tokio::pin!(readiness);
tokio::select! {
() = self.cancellation.token().cancelled() => ActiveEvent::Cancelled,
command = self.commands.recv() => ActiveEvent::Command(command),
() = &mut readiness => ActiveEvent::ReadinessTimedOut,
next = &mut signal => ActiveEvent::Signal(next),
}
};
@@ -766,6 +784,21 @@ impl Runtime {
ActiveEvent::Cancelled => {
desired = DesiredState::Shutdown;
}
ActiveEvent::ReadinessTimedOut => {
ready = false;
readiness_deadline = None;
stable_since = None;
self.fence.invalidate();
if let Some(unready) = session.take() {
self.close_session(unready).await;
}
self.failures = self.failures.saturating_add(1);
desired = self
.backoff(SessionFailure::new(
SessionFailureKind::TransientTransport,
))
.await;
}
ActiveEvent::Command(command) => match command {
Some(Command::Submit(work, response)) => {
let disposition = self.handle_work(work.clone(), ready, generation);
@@ -796,6 +829,7 @@ impl Runtime {
ActiveEvent::Signal(next) => match next {
SessionSignal::Ready => {
ready = true;
readiness_deadline = None;
stable_since.get_or_insert_with(Instant::now);
self.transition(
SessionState::Online,
@@ -807,6 +841,8 @@ impl Runtime {
}
SessionSignal::Degraded => {
ready = false;
readiness_deadline =
Some(Instant::now() + self.policy.readiness_timeout);
stable_since = None;
self.transition(
SessionState::Degraded,
@@ -846,6 +882,7 @@ impl Runtime {
self.close_session(active).await;
}
ready = false;
readiness_deadline = None;
stable_since = None;
}
}

33
crates/metacrate-grid-agent/src/session_tests.rs
View File

@@ -196,6 +196,7 @@ fn test_policy() -> ReconnectPolicy {
ReconnectPolicy {
initial_delay: Duration::from_secs(1),
maximum_delay: Duration::from_secs(8),
readiness_timeout: Duration::from_secs(30),
stable_reset_after: Duration::from_secs(10),
shutdown_deadline: Duration::from_secs(2),
jitter_basis_points: 0,
@@ -413,6 +414,38 @@ async fn transport_connection_is_distinct_from_full_agent_readiness() {
handle.shutdown().await.expect("shutdown");
}
#[tokio::test(start_paused = true)]
async fn readiness_timeout_logs_out_and_retries_instead_of_staying_degraded() {
let plans = [
LoginPlan::Success {
after: Duration::ZERO,
signals: VecDeque::new(),
},
LoginPlan::Success {
after: Duration::ZERO,
signals: VecDeque::from([(Duration::ZERO, SessionSignal::Ready)]),
},
];
let (backend, stats) = FakeBackend::new(plans);
let mut policy = test_policy();
policy.readiness_timeout = Duration::from_secs(5);
let erased: Arc<dyn GridSessionBackend> = backend;
let mut handle = SessionSupervisor::new(erased, policy, 32, 256)
.expect("supervisor")
.start();
wait_state(&handle, SessionState::Degraded).await;
tokio::time::advance(Duration::from_secs(5)).await;
settle().await;
wait_state(&handle, SessionState::Backoff).await;
assert_eq!(stats.logouts.load(Ordering::Acquire), 1);
tokio::time::advance(Duration::from_secs(1)).await;
settle().await;
wait_state(&handle, SessionState::Online).await;
handle.shutdown().await.expect("shutdown");
assert_eq!(stats.active_sessions.load(Ordering::Acquire), 0);
}
#[tokio::test(start_paused = true)]
async fn shutdown_covers_degraded_authentication_blocked_stopped_and_online_states() {
let cases = [

1
crates/metacrate-grid-agent/tests/session_supervisor.rs
View File

@@ -172,6 +172,7 @@ async fn repeated_reconnects_have_one_resource_set_no_replay_and_no_shutdown_lea
let policy = ReconnectPolicy {
initial_delay: Duration::from_secs(1),
maximum_delay: Duration::from_secs(4),
readiness_timeout: Duration::from_secs(30),
stable_reset_after: Duration::from_secs(30),
shutdown_deadline: Duration::from_secs(3),
jitter_basis_points: 0,