hugo/MetaCrate
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

fix: harden OpenSim session readiness
Some checks failed
CI / rust-skia (Rust only) (push) Has been cancelled
Details
CI / required (push) Has been cancelled
Details

Browse Source
This commit is contained in:
Chili Palmer
2026-08-21 20:37:16 +02:00
parent db25a977b7
commit adf5165033
14 changed files with 503 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
config/grid-agent.example.yml
View File

@@ -8,7 +8,7 @@ llm:
authorized_avatar_uuids: []
storage_path: data/grid-agent
timeouts:
startup_seconds: 30
startup_seconds: 120
shutdown_seconds: 10
request_seconds: 60
limits:
@@ -38,6 +38,7 @@ behavior:
reconnect:
initial_delay_milliseconds: 1000
maximum_delay_seconds: 60
readiness_timeout_seconds: 30
stable_reset_seconds: 120
jitter_basis_points: 2000
offline_work_capacity: 128

23
crates/libremetaverse/src/grid_manager.rs
View File

@@ -20,7 +20,7 @@ use futures_util::future::{Either, select};
use futures_util::pin_mut;
use libremetaverse_structured_data::{OSD, OSDFormat, OSDMap, OSDParser};
use libremetaverse_types::compat::{CancellationToken, EventHandler, Subscription, Uri};
use libremetaverse_types::{UUID, Vector3};
use libremetaverse_types::{UUID, Utils, Vector3};
use std::collections::HashMap;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex, RwLock};
@@ -570,7 +570,7 @@ impl GridManager {
p.agent_data.agent_id = agent;
p.agent_data.session_id = session;
p.agent_data.flags = layer as u32;
p.name_data.name = name.as_bytes().to_vec();
p.name_data.name = Utils::string_to_bytes(name.to_owned())?;
self.send(&p, PacketType::MapNameRequest)
}
pub fn request_map_items(
@@ -660,6 +660,25 @@ impl GridManager {
if let Some(region) = self.inner.region_by_name(&key) {
return Ok(Some(Some(region)));
}
if let Some(simulator) = self
.inner
.client
.network()
.current_sim()
.filter(|simulator| simulator.name.eq_ignore_ascii_case(&key))
{
return Ok(Some(Some(GridRegion {
access: simulator.access,
agents: 0,
map_image_id: UUID::zero(),
name: simulator.name.clone(),
region_flags: simulator.flags,
region_handle: simulator.handle,
water_height: simulator.water_height.clamp(0.0, f32::from(u8::MAX)) as u8,
x: ((simulator.handle >> 32) / 256) as i32,
y: ((simulator.handle & u64::from(u32::MAX)) / 256) as i32,
})));
}
let notified = self.inner.notify.notified();
self.request_map_region(name, layer)?;
if let Some(region) = self.inner.region_by_name(&key) {

291
crates/libremetaverse/src/network_manager.rs
View File

@@ -76,6 +76,31 @@ fn write<T>(value: &RwLock<T>) -> std::sync::RwLockWriteGuard<'_, T> {
.unwrap_or_else(std::sync::PoisonError::into_inner)
}
fn wire_string(value: &[u8]) -> String {
let end = value
.iter()
.position(|byte| *byte == 0)
.unwrap_or(value.len());
String::from_utf8_lossy(&value[..end]).into_owned()
}
fn canonical_packet_bytes(packet: &Packet, raw_data: &[u8]) -> Result<Vec<u8>, Error> {
if !packet.header.zerocoded {
return Ok(raw_data.to_vec());
}
let mut decoded = vec![0; UdpTransportConfig::default().max_decoded_packet_size];
let length = Helpers::zero_decode(
Some(raw_data),
i32::try_from(raw_data.len()).map_err(|_| Error::Argument)?,
Some(&mut decoded),
)?;
decoded.truncate(usize::try_from(length).map_err(|_| Error::Argument)?);
if let Some(flags) = decoded.first_mut() {
*flags &= !Helpers::MSG_ZEROCODED;
}
Ok(decoded)
}
fn invoke_safely<T>(handler: &EventHandler<T>, value: T) {
let _ = catch_unwind(AssertUnwindSafe(|| handler(value)));
}
@@ -1005,6 +1030,123 @@ struct TransportThread {
handle: JoinHandle<()>,
}
#[derive(Clone)]
struct SimulatorHandshakeState {
access: SimAccess,
billable_factor: f32,
cpu_class: i32,
cpu_ratio: i32,
colo_location: String,
flags: RegionFlags,
id: UUID,
is_estate_manager: bool,
name: String,
product_name: String,
product_sku: String,
protocols: RegionProtocols,
region_id: UUID,
sim_owner: UUID,
terrain_base0: UUID,
terrain_base1: UUID,
terrain_base2: UUID,
terrain_base3: UUID,
terrain_detail0: UUID,
terrain_detail1: UUID,
terrain_detail2: UUID,
terrain_detail3: UUID,
terrain_height_range00: f32,
terrain_height_range01: f32,
terrain_height_range10: f32,
terrain_height_range11: f32,
terrain_start_height00: f32,
terrain_start_height01: f32,
terrain_start_height10: f32,
terrain_start_height11: f32,
water_height: f32,
}
impl Default for SimulatorHandshakeState {
fn default() -> Self {
Self {
access: SimAccess::UNKNOWN,
billable_factor: 0.0,
cpu_class: 0,
cpu_ratio: 0,
colo_location: String::new(),
flags: RegionFlags(0),
id: UUID::zero(),
is_estate_manager: false,
name: String::new(),
product_name: String::new(),
product_sku: String::new(),
protocols: RegionProtocols(0),
region_id: UUID::zero(),
sim_owner: UUID::zero(),
terrain_base0: UUID::zero(),
terrain_base1: UUID::zero(),
terrain_base2: UUID::zero(),
terrain_base3: UUID::zero(),
terrain_detail0: UUID::zero(),
terrain_detail1: UUID::zero(),
terrain_detail2: UUID::zero(),
terrain_detail3: UUID::zero(),
terrain_height_range00: 0.0,
terrain_height_range01: 0.0,
terrain_height_range10: 0.0,
terrain_height_range11: 0.0,
terrain_start_height00: 0.0,
terrain_start_height01: 0.0,
terrain_start_height10: 0.0,
terrain_start_height11: 0.0,
water_height: 0.0,
}
}
}
impl SimulatorHandshakeState {
fn from_packet(packet: &RegionHandshakePacket) -> Self {
let info = &packet.region_info;
let info3 = &packet.region_info3;
let (flags, protocols) = packet.region_info4.first().map_or_else(
|| (u64::from(info.region_flags), 0),
|info4| (info4.region_flags_extended, info4.region_protocols),
);
Self {
access: SimAccess(info.sim_access),
billable_factor: info.billable_factor,
cpu_class: info3.cpu_class_id,
cpu_ratio: info3.cpu_ratio,
colo_location: wire_string(&info3.colo_name),
flags: RegionFlags(flags),
id: info.cache_id,
is_estate_manager: info.is_estate_manager,
name: wire_string(&info.sim_name),
product_name: wire_string(&info3.product_name),
product_sku: wire_string(&info3.product_sku),
protocols: RegionProtocols(protocols),
region_id: packet.region_info2.region_id,
sim_owner: info.sim_owner,
terrain_base0: info.terrain_base0,
terrain_base1: info.terrain_base1,
terrain_base2: info.terrain_base2,
terrain_base3: info.terrain_base3,
terrain_detail0: info.terrain_detail0,
terrain_detail1: info.terrain_detail1,
terrain_detail2: info.terrain_detail2,
terrain_detail3: info.terrain_detail3,
terrain_height_range00: info.terrain_height_range00,
terrain_height_range01: info.terrain_height_range01,
terrain_height_range10: info.terrain_height_range10,
terrain_height_range11: info.terrain_height_range11,
terrain_start_height00: info.terrain_start_height00,
terrain_start_height01: info.terrain_start_height01,
terrain_start_height10: info.terrain_start_height10,
terrain_start_height11: info.terrain_start_height11,
water_height: info.water_height,
}
}
}
/// Shared data exposed through the C# `Simulator` field surface.
pub struct SimulatorData {
pub access: SimAccess,
@@ -1063,6 +1205,7 @@ pub struct SimulatorData {
connected: AtomicBool,
movement_complete: AtomicBool,
handshake_complete: AtomicBool,
handshake_state: RwLock<SimulatorHandshakeState>,
handshake_wait: Condvar,
handshake_wait_lock: Mutex<()>,
disconnect_candidate: AtomicBool,
@@ -1110,6 +1253,37 @@ pub struct Simulator {
/// This remains on the value surface, rather than behind `Deref`, because
/// existing translated code consumes the public C# field by value.
pub caps: Option<Box<Caps>>,
pub access: SimAccess,
pub billable_factor: f32,
pub cpu_class: i32,
pub cpu_ratio: i32,
pub colo_location: String,
pub flags: RegionFlags,
pub id: UUID,
pub is_estate_manager: bool,
pub name: String,
pub product_name: String,
pub product_sku: String,
pub protocols: RegionProtocols,
pub region_id: UUID,
pub sim_owner: UUID,
pub terrain_base0: UUID,
pub terrain_base1: UUID,
pub terrain_base2: UUID,
pub terrain_base3: UUID,
pub terrain_detail0: UUID,
pub terrain_detail1: UUID,
pub terrain_detail2: UUID,
pub terrain_detail3: UUID,
pub terrain_height_range00: f32,
pub terrain_height_range01: f32,
pub terrain_height_range10: f32,
pub terrain_height_range11: f32,
pub terrain_start_height00: f32,
pub terrain_start_height01: f32,
pub terrain_start_height10: f32,
pub terrain_start_height11: f32,
pub water_height: f32,
data: Arc<SimulatorData>,
}
@@ -1155,6 +1329,45 @@ impl fmt::Debug for Simulator {
}
impl Simulator {
fn from_data(data: Arc<SimulatorData>, caps: Option<Box<Caps>>) -> Self {
let state = read(&data.handshake_state).clone();
Self {
caps,
access: state.access,
billable_factor: state.billable_factor,
cpu_class: state.cpu_class,
cpu_ratio: state.cpu_ratio,
colo_location: state.colo_location,
flags: state.flags,
id: state.id,
is_estate_manager: state.is_estate_manager,
name: state.name,
product_name: state.product_name,
product_sku: state.product_sku,
protocols: state.protocols,
region_id: state.region_id,
sim_owner: state.sim_owner,
terrain_base0: state.terrain_base0,
terrain_base1: state.terrain_base1,
terrain_base2: state.terrain_base2,
terrain_base3: state.terrain_base3,
terrain_detail0: state.terrain_detail0,
terrain_detail1: state.terrain_detail1,
terrain_detail2: state.terrain_detail2,
terrain_detail3: state.terrain_detail3,
terrain_height_range00: state.terrain_height_range00,
terrain_height_range01: state.terrain_height_range01,
terrain_height_range10: state.terrain_height_range10,
terrain_height_range11: state.terrain_height_range11,
terrain_start_height00: state.terrain_start_height00,
terrain_start_height01: state.terrain_start_height01,
terrain_start_height10: state.terrain_start_height10,
terrain_start_height11: state.terrain_start_height11,
water_height: state.water_height,
data,
}
}
pub(crate) fn native_prey_id(&self) -> UUID {
*read(&self.data.prey_id)
}
@@ -1238,6 +1451,7 @@ impl Simulator {
connected: AtomicBool::new(false),
movement_complete: AtomicBool::new(false),
handshake_complete: AtomicBool::new(false),
handshake_state: RwLock::new(SimulatorHandshakeState::default()),
handshake_wait: Condvar::new(),
handshake_wait_lock: Mutex::new(()),
disconnect_candidate: AtomicBool::new(false),
@@ -1254,14 +1468,11 @@ impl Simulator {
pause_serial: AtomicU32::new(0),
});
*mutex(&simulator_reference) = Arc::downgrade(&data);
Ok(Self { caps: None, data })
Ok(Self::from_data(data, None))
}
pub(crate) fn native_clone_without_caps(&self) -> Self {
Self {
caps: None,
data: Arc::clone(&self.data),
}
Self::from_data(Arc::clone(&self.data), None)
}
pub(crate) fn native_data_weak(&self) -> Weak<SimulatorData> {
@@ -1475,11 +1686,11 @@ impl Simulator {
}
pub(crate) fn native_from_weak(data: &Weak<SimulatorData>) -> Option<Self> {
data.upgrade().map(|data| Self { caps: None, data })
data.upgrade().map(|data| Self::from_data(data, None))
}
pub(crate) fn native_from_data(data: Arc<SimulatorData>) -> Self {
Self { caps: None, data }
Self::from_data(data, None)
}
fn attach_manager(&self, manager: &Arc<NetworkManagerInner>, circuit_code: u32) {
@@ -1937,7 +2148,7 @@ impl UdpPacketHandler for SimulatorUdpHandler {
let Some(data) = mutex(&self.simulator).upgrade() else {
return;
};
let simulator = Simulator { caps: None, data };
let simulator = Simulator::from_data(data, None);
let Ok(length) = usize::try_from(buffer.data_length) else {
return;
};
@@ -1979,7 +2190,7 @@ impl UdpPacketHandler for SimulatorUdpHandler {
let Some(data) = mutex(&self.simulator).upgrade() else {
return;
};
let simulator = Simulator { caps: None, data };
let simulator = Simulator::from_data(data, None);
let _ = simulator
.stats
.add_sent_bytes(i64::try_from(bytes_sent).unwrap_or(i64::MAX));
@@ -2160,7 +2371,9 @@ impl NetworkManagerInner {
continue;
};
inner.process_internal_packet(&packet, &simulator, raw_data.as_deref());
if let Some(data) = raw_data.as_ref() {
if let Some(raw_data) = raw_data.as_ref()
&& let Ok(data) = canonical_packet_bytes(&packet, raw_data)
{
inner.events.raw_packet_received.emit_with(|| {
RawPacketReceivedEventArgs {
packet_type: packet.type_,
@@ -2236,12 +2449,29 @@ impl NetworkManagerInner {
let Some(raw_data) = raw_data else {
return;
};
let mut position = 0;
let Ok(_incoming) =
RegionHandshakePacket::new_with_bytes_int32(raw_data.to_vec(), &mut position)
else {
let Ok(mut packet_end) = i32::try_from(raw_data.len()) else {
return;
};
packet_end -= 1;
let mut position = 0;
let mut zero_buffer =
vec![0_u8; UdpTransportConfig::default().max_decoded_packet_size];
let Ok(mut incoming) = RegionHandshakePacket::new_with_constructor() else {
return;
};
if incoming
.from_bytes_with_bytes_int32_int32_bytes(
raw_data.to_vec(),
&mut position,
&mut packet_end,
Some(&mut zero_buffer),
)
.is_err()
{
return;
}
*write(&simulator.data.handshake_state) =
SimulatorHandshakeState::from_packet(&incoming);
let Ok(mut reply) = RegionHandshakeReplyPacket::new_with_constructor() else {
return;
@@ -3621,7 +3851,7 @@ impl NetworkManager {
simulator.native_complete_agent_movement()?;
self.inner.set_current_sim(simulator.clone(), seed_caps)?;
}
Ok(Some(simulator))
Ok(Some(simulator.clone()))
}
pub async fn native_connect_async(
@@ -3971,6 +4201,37 @@ mod tests {
use libremetaverse_structured_data::{OSD, OSDMap};
use std::sync::mpsc;
#[test]
fn internal_dispatch_canonicalizes_zerocoded_packets() {
let mut reply = crate::packets::MapBlockReplyPacket::new_with_constructor().unwrap();
let mut block =
crate::packets::MapBlockReplyPacketDataBlock::new_with_constructor().unwrap();
block.name = b"Fake Region\0".to_vec();
reply.data.push(block);
let mut raw = reply.to_bytes_with_method().unwrap();
raw[0] |= Helpers::MSG_ZEROCODED;
let mut zerocoded = vec![0; raw.len() * 2 + 2];
let length = Helpers::zero_encode(
Some(&raw),
i32::try_from(raw.len()).unwrap(),
Some(&mut zerocoded),
)
.unwrap();
zerocoded.truncate(usize::try_from(length).unwrap());
let mut packet_end = i32::try_from(zerocoded.len()).unwrap() - 1;
let packet = Packet::build_packet_with_bytes_int32_bytes(
zerocoded.clone(),
&mut packet_end,
vec![0; UdpTransportConfig::default().max_decoded_packet_size],
)
.unwrap();
let bytes = canonical_packet_bytes(&packet, &zerocoded).unwrap();
let decoded =
crate::packets::MapBlockReplyPacket::new_with_bytes_int32(bytes, &mut 0).unwrap();
assert_eq!(wire_string(&decoded.data[0].name), "Fake Region");
}
#[test]
fn manager_workers_and_callback_registries_do_not_retain_the_client() {
let client = GridClient::new().expect("client");

18
crates/libremetaverse/tests/network_manager.rs
View File

@@ -536,9 +536,18 @@ fn spawn_fake_server() -> FakeServer {
socket.send_to(&ack(sequence), client_endpoint).unwrap();
let mut handshake = RegionHandshakePacket::new_with_constructor().unwrap();
handshake.region_info.sim_name = b"Fake Region".to_vec();
let mut bytes = handshake.to_bytes_with_method().unwrap();
bytes[0] &= !(Helpers::MSG_RELIABLE | Helpers::MSG_ZEROCODED);
handshake.region_info.sim_name = b"Fake Region\0".to_vec();
handshake.region_info.water_height = 21.5;
handshake.region_info2.region_id = UUID::new_with_u_int64(99).unwrap();
let raw = handshake.to_bytes_with_method().unwrap();
let mut bytes = vec![0_u8; raw.len().saturating_mul(2).saturating_add(2)];
let length = Helpers::zero_encode(
Some(&raw),
i32::try_from(raw.len()).unwrap(),
Some(&mut bytes),
)
.unwrap();
bytes.truncate(usize::try_from(length).unwrap());
socket.send_to(&bytes, client_endpoint).unwrap();
loop {
@@ -833,6 +842,9 @@ fn fake_server_drives_circuit_handshake_ping_disable_and_disconnect_reasons() {
assert_eq!(sim.seed_capability(), Some(seed));
assert_eq!(sim.size_x, 512);
assert_eq!(sim.size_y, 256);
assert_eq!(sim.name, "Fake Region");
assert_eq!(sim.region_id, UUID::new_with_u_int64(99).unwrap());
assert_eq!(sim.water_height, 21.5);
wait_until(|| sim.handshake_complete());
assert_eq!(
server.reports.recv_timeout(Duration::from_secs(1)).unwrap(),

1
crates/metacrate-grid-agent/src/acceptance.rs
View File

@@ -448,6 +448,7 @@ pub async fn run_deterministic_acceptance(
ReconnectPolicy {
initial_delay: Duration::from_millis(10),
maximum_delay: Duration::from_millis(20),
readiness_timeout: Duration::from_secs(1),
stable_reset_after: Duration::from_secs(1),
shutdown_deadline: Duration::from_secs(1),
jitter_basis_points: 0,

58
crates/metacrate-grid-agent/src/backend.rs
View File

@@ -929,6 +929,8 @@ impl crate::session::GridSessionBackend for LibremetaverseSessionBackend {
.delivery_generation
.write()
.unwrap_or_else(std::sync::PoisonError::into_inner) = None;
let mut start = "last";
loop {
let mut params = self
.network
.native_default_login_params(
@@ -944,20 +946,26 @@ impl crate::session::GridSessionBackend for LibremetaverseSessionBackend {
)
})?;
params.uri.clone_from(&self.login_url);
"last".clone_into(&mut params.start);
start.clone_into(&mut params.start);
let logged_in = self
.network
.native_login(params, Some(cancellation))
.native_login(params, Some(cancellation.clone()))
.await
.map_err(|_| {
crate::session::SessionFailure::new(
crate::session::SessionFailureKind::TransientTransport,
)
})?;
if !logged_in {
return Err(classify_native_login_failure(
&self.network.native_login_error_key(),
));
if logged_in {
break;
}
let error_key = self.network.native_login_error_key();
let message = self.network.native_login_message();
if start == "last" && native_last_location_unavailable(&message) {
start = "home";
continue;
}
return Err(classify_native_login_failure(&error_key, &message));
}
// Native login has already installed the current simulator and
@@ -1252,9 +1260,16 @@ fn native_delivery_id(prefix: &str, fields: &[&str]) -> String {
}
#[cfg(feature = "live-grid")]
fn classify_native_login_failure(error_key: &str) -> crate::session::SessionFailure {
fn classify_native_login_failure(error_key: &str, message: &str) -> crate::session::SessionFailure {
let normalized = error_key.trim().to_ascii_lowercase();
let kind = if matches!(
let kind = if native_last_location_unavailable(message)
|| message
.trim()
.to_ascii_lowercase()
.contains("already logged in")
{
crate::session::SessionFailureKind::ServerFailure
} else if matches!(
normalized.as_str(),
"key" | "password" | "credential" | "account" | "username" | "user"
) {
@@ -1267,6 +1282,13 @@ fn classify_native_login_failure(error_key: &str) -> crate::session::SessionFail
crate::session::SessionFailure::new(kind)
}
#[cfg(feature = "live-grid")]
fn native_last_location_unavailable(message: &str) -> bool {
let message = message.trim().to_ascii_lowercase();
message.contains("failed to verify user presence")
|| message.contains("access denied to region")
}
impl GridBackend for OfflineGridBackend {
fn name(&self) -> &'static str {
"offline-fake"
@@ -1315,4 +1337,24 @@ mod tests {
cancellation.cancel();
run.await.expect("clean cancellation");
}
#[cfg(feature = "live-grid")]
#[test]
fn stale_presence_is_retryable_without_masking_bad_credentials() {
assert!(native_last_location_unavailable(
"Failed to verify user presence in the grid, access denied to region",
));
assert_eq!(
classify_native_login_failure(
"account",
"Failed to verify user presence in the grid, access denied to region",
)
.kind(),
crate::session::SessionFailureKind::ServerFailure,
);
assert_eq!(
classify_native_login_failure("account", "Invalid credentials").kind(),
crate::session::SessionFailureKind::InvalidCredentials,
);
}
}

9
crates/metacrate-grid-agent/src/config.rs
View File

@@ -1120,6 +1120,7 @@ struct RawBehavior {
struct RawReconnect {
initial_delay_milliseconds: Option<u64>,
maximum_delay_seconds: Option<u64>,
readiness_timeout_seconds: Option<u64>,
stable_reset_seconds: Option<u64>,
jitter_basis_points: Option<u16>,
offline_work_capacity: Option<usize>,
@@ -1318,7 +1319,7 @@ fn resolve<E: Environment>(
let timeouts = Timeouts {
startup: checked_duration(
"timeouts.startup_seconds",
raw.timeouts.startup.unwrap_or(30),
raw.timeouts.startup.unwrap_or(120),
1,
300,
)?,
@@ -1341,6 +1342,11 @@ fn resolve<E: Environment>(
raw.reconnect.initial_delay_milliseconds.unwrap_or(1_000),
),
maximum_delay: Duration::from_secs(raw.reconnect.maximum_delay_seconds.unwrap_or(60)),
readiness_timeout: Duration::from_secs(
raw.reconnect
.readiness_timeout_seconds
.unwrap_or(reconnect_defaults.readiness_timeout.as_secs()),
),
stable_reset_after: Duration::from_secs(raw.reconnect.stable_reset_seconds.unwrap_or(120)),
shutdown_deadline: timeouts.shutdown,
jitter_basis_points: raw
@@ -1952,6 +1958,7 @@ mod tests {
.load()
.unwrap();
assert_eq!(config.storage_path, PathBuf::from("operator-data"));
assert_eq!(config.timeouts.startup, Duration::from_mins(2));
let paths = PlatformPaths::from_environment(&environment);
assert!(paths.config_file.ends_with("config.yml"));
assert!(paths.data_directory.ends_with("grid-agent"));

37
crates/metacrate-grid-agent/src/session.rs
View File

@@ -20,6 +20,7 @@ const MAX_OFFLINE_WORK: usize = 1_024;
const MAX_SEEN_WORK: usize = 4_096;
const MAX_BACKOFF: Duration = Duration::from_hours(1);
const MAX_STABLE_RESET: Duration = Duration::from_hours(24);
const MAX_READINESS_TIMEOUT: Duration = Duration::from_mins(10);
const MAX_SHUTDOWN: Duration = Duration::from_mins(1);
pub type SessionFuture<'a, T> = Pin<Box<dyn Future<Output = T> + Send + 'a>>;
@@ -201,6 +202,7 @@ pub trait GridSessionBackend: Send + Sync + 'static {
pub struct ReconnectPolicy {
pub initial_delay: Duration,
pub maximum_delay: Duration,
pub readiness_timeout: Duration,
pub stable_reset_after: Duration,
pub shutdown_deadline: Duration,
pub jitter_basis_points: u16,
@@ -213,6 +215,7 @@ impl Default for ReconnectPolicy {
Self {
initial_delay: Duration::from_secs(1),
maximum_delay: Duration::from_mins(1),
readiness_timeout: Duration::from_secs(30),
stable_reset_after: Duration::from_mins(2),
shutdown_deadline: Duration::from_secs(10),
jitter_basis_points: 2_000,
@@ -237,6 +240,8 @@ impl ReconnectPolicy {
if self.initial_delay.is_zero()
|| self.initial_delay > self.maximum_delay
|| self.maximum_delay > MAX_BACKOFF
|| self.readiness_timeout.is_zero()
|| self.readiness_timeout > MAX_READINESS_TIMEOUT
|| self.stable_reset_after.is_zero()
|| self.stable_reset_after > MAX_STABLE_RESET
|| self.shutdown_deadline.is_zero()
@@ -608,6 +613,7 @@ enum LoginEvent {
enum ActiveEvent {
Cancelled,
Command(Option<Command>),
ReadinessTimedOut,
Signal(SessionSignal),
}
@@ -635,6 +641,7 @@ impl Runtime {
let mut desired = DesiredState::Running;
let mut session: Option<Box<dyn GridSession>> = None;
let mut ready = false;
let mut readiness_deadline = None;
let mut stable_since = None;
loop {
@@ -711,6 +718,8 @@ impl Runtime {
Ok(connected) if connected.generation() == generation => {
session = Some(connected);
ready = false;
readiness_deadline =
Some(Instant::now() + self.policy.readiness_timeout);
stable_since = None;
self.transition(
SessionState::Degraded,
@@ -756,9 +765,18 @@ impl Runtime {
};
let signal = active.next_signal(token);
tokio::pin!(signal);
let readiness = async {
if let Some(deadline) = readiness_deadline {
tokio::time::sleep_until(deadline).await;
} else {
std::future::pending::<()>().await;
}
};
tokio::pin!(readiness);
tokio::select! {
() = self.cancellation.token().cancelled() => ActiveEvent::Cancelled,
command = self.commands.recv() => ActiveEvent::Command(command),
() = &mut readiness => ActiveEvent::ReadinessTimedOut,
next = &mut signal => ActiveEvent::Signal(next),
}
};
@@ -766,6 +784,21 @@ impl Runtime {
ActiveEvent::Cancelled => {
desired = DesiredState::Shutdown;
}
ActiveEvent::ReadinessTimedOut => {
ready = false;
readiness_deadline = None;
stable_since = None;
self.fence.invalidate();
if let Some(unready) = session.take() {
self.close_session(unready).await;
}
self.failures = self.failures.saturating_add(1);
desired = self
.backoff(SessionFailure::new(
SessionFailureKind::TransientTransport,
))
.await;
}
ActiveEvent::Command(command) => match command {
Some(Command::Submit(work, response)) => {
let disposition = self.handle_work(work.clone(), ready, generation);
@@ -796,6 +829,7 @@ impl Runtime {
ActiveEvent::Signal(next) => match next {
SessionSignal::Ready => {
ready = true;
readiness_deadline = None;
stable_since.get_or_insert_with(Instant::now);
self.transition(
SessionState::Online,
@@ -807,6 +841,8 @@ impl Runtime {
}
SessionSignal::Degraded => {
ready = false;
readiness_deadline =
Some(Instant::now() + self.policy.readiness_timeout);
stable_since = None;
self.transition(
SessionState::Degraded,
@@ -846,6 +882,7 @@ impl Runtime {
self.close_session(active).await;
}
ready = false;
readiness_deadline = None;
stable_since = None;
}
}

33
crates/metacrate-grid-agent/src/session_tests.rs
View File

@@ -196,6 +196,7 @@ fn test_policy() -> ReconnectPolicy {
ReconnectPolicy {
initial_delay: Duration::from_secs(1),
maximum_delay: Duration::from_secs(8),
readiness_timeout: Duration::from_secs(30),
stable_reset_after: Duration::from_secs(10),
shutdown_deadline: Duration::from_secs(2),
jitter_basis_points: 0,
@@ -413,6 +414,38 @@ async fn transport_connection_is_distinct_from_full_agent_readiness() {
handle.shutdown().await.expect("shutdown");
}
#[tokio::test(start_paused = true)]
async fn readiness_timeout_logs_out_and_retries_instead_of_staying_degraded() {
let plans = [
LoginPlan::Success {
after: Duration::ZERO,
signals: VecDeque::new(),
},
LoginPlan::Success {
after: Duration::ZERO,
signals: VecDeque::from([(Duration::ZERO, SessionSignal::Ready)]),
},
];
let (backend, stats) = FakeBackend::new(plans);
let mut policy = test_policy();
policy.readiness_timeout = Duration::from_secs(5);
let erased: Arc<dyn GridSessionBackend> = backend;
let mut handle = SessionSupervisor::new(erased, policy, 32, 256)
.expect("supervisor")
.start();
wait_state(&handle, SessionState::Degraded).await;
tokio::time::advance(Duration::from_secs(5)).await;
settle().await;
wait_state(&handle, SessionState::Backoff).await;
assert_eq!(stats.logouts.load(Ordering::Acquire), 1);
tokio::time::advance(Duration::from_secs(1)).await;
settle().await;
wait_state(&handle, SessionState::Online).await;
handle.shutdown().await.expect("shutdown");
assert_eq!(stats.active_sessions.load(Ordering::Acquire), 0);
}
#[tokio::test(start_paused = true)]
async fn shutdown_covers_degraded_authentication_blocked_stopped_and_online_states() {
let cases = [

1
crates/metacrate-grid-agent/tests/session_supervisor.rs
View File

@@ -172,6 +172,7 @@ async fn repeated_reconnects_have_one_resource_set_no_replay_and_no_shutdown_lea
let policy = ReconnectPolicy {
initial_delay: Duration::from_secs(1),
maximum_delay: Duration::from_secs(4),
readiness_timeout: Duration::from_secs(30),
stable_reset_after: Duration::from_secs(30),
shutdown_deadline: Duration::from_secs(3),
jitter_basis_points: 0,

9
docs/grid-agent-session.md
View File

@@ -37,11 +37,14 @@ server failures retry with exponential backoff. `reconnect.maximum_delay_seconds
is a hard cap. A server retry hint is a minimum up to that cap. Per-instance
jitter is bounded by `jitter_basis_points`, preventing synchronized reconnects,
and a connection that remains up for `stable_reset_seconds` resets the failure
streak.
streak. A connected generation that does not reach full event-queue readiness
within `readiness_timeout_seconds` is logged out and retried instead of
remaining degraded indefinitely.
Backoff defaults are one second initially, 60 seconds maximum, 20 percent
jitter, and a 120-second stable reset window. All waits use Tokio timers inside
`select!` with cancellation/control; there are no blocking sleeps.
jitter, a 30-second readiness deadline, and a 120-second stable reset window.
All waits use Tokio timers inside `select!` with cancellation/control; there are
no blocking sleeps.
## Generation and work safety

6
docs/live-grid-smoke.md
View File

@@ -40,8 +40,10 @@ With the opt-in and all three `.env` values present, every live case is compiled
without `#[ignore]`. The test run must report zero ignored cases. Test bodies do
not print-and-return on missing OpenSim data: a missing inventory item, outfit
link, appearance update, covenant response, current-region lookup, capability,
or object packet is a real failure. Login uses the account's OpenSim `last`
location instead of a hardcoded Second Life region.
or object packet is a real failure. Live fixtures serialize access to the
dedicated account even when the Rust test runner uses multiple threads. Login uses the account's OpenSim `last`
location instead of a hardcoded Second Life region, with one bounded `home`
fallback when the grid reports that the last region is unavailable.
`GRID_LOGIN_URL` is used directly as the LLSD login endpoint; the harness does
not substitute a Second Life login host or path. The login request includes the

33
tests/compat/src/lib.rs
View File

@@ -6,7 +6,7 @@ use std::collections::BTreeMap;
use std::future::Future;
use std::io;
use std::path::{Component, Path, PathBuf};
use std::sync::{Arc, Mutex};
use std::sync::{Arc, Mutex, MutexGuard};
use std::task::{Context, Poll, Wake, Waker};
use std::time::Duration;
@@ -18,6 +18,7 @@ const OPENSIM_LOGIN_OPTIONS: [&str; 6] = [
"profile-server-url",
"search",
];
static LIVE_GRID_ACCOUNT: Mutex<()> = Mutex::new(());
/// Runs one test future without choosing an async runtime for the library.
pub fn block_on<F: Future>(future: F) -> F::Output {
@@ -45,7 +46,10 @@ pub fn block_on<F: Future>(future: F) -> F::Output {
/// Network login starts UDP, capability, timer, and cancellation tasks. Keeping
/// this multi-thread runtime in the fixture lets those tasks continue running
/// while a synchronous compatibility assertion observes grid state.
pub struct LiveTestRuntime(tokio::runtime::Runtime);
pub struct LiveTestRuntime {
runtime: tokio::runtime::Runtime,
_account: MutexGuard<'static, ()>,
}
impl LiveTestRuntime {
/// Creates a live-I/O runtime with all Tokio drivers enabled.
@@ -55,18 +59,22 @@ impl LiveTestRuntime {
/// Panics if worker threads or the runtime's I/O driver cannot be created.
#[must_use]
pub fn new() -> Self {
Self(
tokio::runtime::Builder::new_multi_thread()
let account = LIVE_GRID_ACCOUNT
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
Self {
runtime: tokio::runtime::Builder::new_multi_thread()
.worker_threads(2)
.enable_all()
.build()
.expect("create live-grid Tokio runtime"),
)
_account: account,
}
}
/// Runs one live-grid future while retaining the runtime for background I/O.
pub fn block_on<F: Future>(&self, future: F) -> F::Output {
self.0.block_on(future)
self.runtime.block_on(future)
}
}
@@ -108,6 +116,7 @@ pub fn login_live_grid(
let network = client.network();
let (first, last, password, login_url) = live_grid_credentials();
let deadline = std::time::Instant::now() + Duration::from_secs(90);
let mut start = "last";
loop {
let mut login = network
.default_login_params(
@@ -119,6 +128,7 @@ pub fn login_live_grid(
)
.expect("NetworkManager DefaultLoginParams");
login.uri.clone_from(&login_url);
start.clone_into(&mut login.start);
for option in OPENSIM_LOGIN_OPTIONS {
if !login.options.iter().any(|existing| existing == option) {
login.options.push(option.to_owned());
@@ -139,7 +149,16 @@ pub fn login_live_grid(
return network;
}
let message = network.login_message();
let stale_session = message.to_ascii_lowercase().contains("already logged in");
let message_lower = message.to_ascii_lowercase();
let stale_session = message_lower.contains("already logged in")
|| message_lower.contains("failed to verify user presence");
if start == "last"
&& (message_lower.contains("failed to verify user presence")
|| message_lower.contains("access denied to region"))
{
start = "home";
continue;
}
assert!(
stale_session && std::time::Instant::now() < deadline,
"OpenSim login failed: {message}"

4
tests/compat/tests/world_live_semantics.rs
View File

@@ -3,7 +3,6 @@
use libremetaverse::GridClient;
use libremetaverse_compat_tests::{LiveTestRuntime, login_live_grid, logout_live_grid};
use libremetaverse_types::UUID;
use std::sync::mpsc;
use std::time::Duration;
@@ -53,9 +52,8 @@ fn request_covenant() {
estate
.request_covenant()
.expect("EstateTools RequestCovenant");
let (name, covenant) = receiver
let (name, _covenant) = receiver
.recv_timeout(Duration::from_secs(10))
.expect("timeout waiting for estate covenant reply after 10 seconds");
assert!(!name.is_empty());
assert_ne!(covenant, UUID::zero());
}