fix: harden OpenSim session readiness
This commit is contained in:
@@ -20,7 +20,7 @@ use futures_util::future::{Either, select};
|
||||
use futures_util::pin_mut;
|
||||
use libremetaverse_structured_data::{OSD, OSDFormat, OSDMap, OSDParser};
|
||||
use libremetaverse_types::compat::{CancellationToken, EventHandler, Subscription, Uri};
|
||||
use libremetaverse_types::{UUID, Vector3};
|
||||
use libremetaverse_types::{UUID, Utils, Vector3};
|
||||
use std::collections::HashMap;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::{Arc, Mutex, RwLock};
|
||||
@@ -570,7 +570,7 @@ impl GridManager {
|
||||
p.agent_data.agent_id = agent;
|
||||
p.agent_data.session_id = session;
|
||||
p.agent_data.flags = layer as u32;
|
||||
p.name_data.name = name.as_bytes().to_vec();
|
||||
p.name_data.name = Utils::string_to_bytes(name.to_owned())?;
|
||||
self.send(&p, PacketType::MapNameRequest)
|
||||
}
|
||||
pub fn request_map_items(
|
||||
@@ -660,6 +660,25 @@ impl GridManager {
|
||||
if let Some(region) = self.inner.region_by_name(&key) {
|
||||
return Ok(Some(Some(region)));
|
||||
}
|
||||
if let Some(simulator) = self
|
||||
.inner
|
||||
.client
|
||||
.network()
|
||||
.current_sim()
|
||||
.filter(|simulator| simulator.name.eq_ignore_ascii_case(&key))
|
||||
{
|
||||
return Ok(Some(Some(GridRegion {
|
||||
access: simulator.access,
|
||||
agents: 0,
|
||||
map_image_id: UUID::zero(),
|
||||
name: simulator.name.clone(),
|
||||
region_flags: simulator.flags,
|
||||
region_handle: simulator.handle,
|
||||
water_height: simulator.water_height.clamp(0.0, f32::from(u8::MAX)) as u8,
|
||||
x: ((simulator.handle >> 32) / 256) as i32,
|
||||
y: ((simulator.handle & u64::from(u32::MAX)) / 256) as i32,
|
||||
})));
|
||||
}
|
||||
let notified = self.inner.notify.notified();
|
||||
self.request_map_region(name, layer)?;
|
||||
if let Some(region) = self.inner.region_by_name(&key) {
|
||||
|
||||
@@ -76,6 +76,31 @@ fn write<T>(value: &RwLock<T>) -> std::sync::RwLockWriteGuard<'_, T> {
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
}
|
||||
|
||||
fn wire_string(value: &[u8]) -> String {
|
||||
let end = value
|
||||
.iter()
|
||||
.position(|byte| *byte == 0)
|
||||
.unwrap_or(value.len());
|
||||
String::from_utf8_lossy(&value[..end]).into_owned()
|
||||
}
|
||||
|
||||
fn canonical_packet_bytes(packet: &Packet, raw_data: &[u8]) -> Result<Vec<u8>, Error> {
|
||||
if !packet.header.zerocoded {
|
||||
return Ok(raw_data.to_vec());
|
||||
}
|
||||
let mut decoded = vec![0; UdpTransportConfig::default().max_decoded_packet_size];
|
||||
let length = Helpers::zero_decode(
|
||||
Some(raw_data),
|
||||
i32::try_from(raw_data.len()).map_err(|_| Error::Argument)?,
|
||||
Some(&mut decoded),
|
||||
)?;
|
||||
decoded.truncate(usize::try_from(length).map_err(|_| Error::Argument)?);
|
||||
if let Some(flags) = decoded.first_mut() {
|
||||
*flags &= !Helpers::MSG_ZEROCODED;
|
||||
}
|
||||
Ok(decoded)
|
||||
}
|
||||
|
||||
fn invoke_safely<T>(handler: &EventHandler<T>, value: T) {
|
||||
let _ = catch_unwind(AssertUnwindSafe(|| handler(value)));
|
||||
}
|
||||
@@ -1005,6 +1030,123 @@ struct TransportThread {
|
||||
handle: JoinHandle<()>,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct SimulatorHandshakeState {
|
||||
access: SimAccess,
|
||||
billable_factor: f32,
|
||||
cpu_class: i32,
|
||||
cpu_ratio: i32,
|
||||
colo_location: String,
|
||||
flags: RegionFlags,
|
||||
id: UUID,
|
||||
is_estate_manager: bool,
|
||||
name: String,
|
||||
product_name: String,
|
||||
product_sku: String,
|
||||
protocols: RegionProtocols,
|
||||
region_id: UUID,
|
||||
sim_owner: UUID,
|
||||
terrain_base0: UUID,
|
||||
terrain_base1: UUID,
|
||||
terrain_base2: UUID,
|
||||
terrain_base3: UUID,
|
||||
terrain_detail0: UUID,
|
||||
terrain_detail1: UUID,
|
||||
terrain_detail2: UUID,
|
||||
terrain_detail3: UUID,
|
||||
terrain_height_range00: f32,
|
||||
terrain_height_range01: f32,
|
||||
terrain_height_range10: f32,
|
||||
terrain_height_range11: f32,
|
||||
terrain_start_height00: f32,
|
||||
terrain_start_height01: f32,
|
||||
terrain_start_height10: f32,
|
||||
terrain_start_height11: f32,
|
||||
water_height: f32,
|
||||
}
|
||||
|
||||
impl Default for SimulatorHandshakeState {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
access: SimAccess::UNKNOWN,
|
||||
billable_factor: 0.0,
|
||||
cpu_class: 0,
|
||||
cpu_ratio: 0,
|
||||
colo_location: String::new(),
|
||||
flags: RegionFlags(0),
|
||||
id: UUID::zero(),
|
||||
is_estate_manager: false,
|
||||
name: String::new(),
|
||||
product_name: String::new(),
|
||||
product_sku: String::new(),
|
||||
protocols: RegionProtocols(0),
|
||||
region_id: UUID::zero(),
|
||||
sim_owner: UUID::zero(),
|
||||
terrain_base0: UUID::zero(),
|
||||
terrain_base1: UUID::zero(),
|
||||
terrain_base2: UUID::zero(),
|
||||
terrain_base3: UUID::zero(),
|
||||
terrain_detail0: UUID::zero(),
|
||||
terrain_detail1: UUID::zero(),
|
||||
terrain_detail2: UUID::zero(),
|
||||
terrain_detail3: UUID::zero(),
|
||||
terrain_height_range00: 0.0,
|
||||
terrain_height_range01: 0.0,
|
||||
terrain_height_range10: 0.0,
|
||||
terrain_height_range11: 0.0,
|
||||
terrain_start_height00: 0.0,
|
||||
terrain_start_height01: 0.0,
|
||||
terrain_start_height10: 0.0,
|
||||
terrain_start_height11: 0.0,
|
||||
water_height: 0.0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl SimulatorHandshakeState {
|
||||
fn from_packet(packet: &RegionHandshakePacket) -> Self {
|
||||
let info = &packet.region_info;
|
||||
let info3 = &packet.region_info3;
|
||||
let (flags, protocols) = packet.region_info4.first().map_or_else(
|
||||
|| (u64::from(info.region_flags), 0),
|
||||
|info4| (info4.region_flags_extended, info4.region_protocols),
|
||||
);
|
||||
Self {
|
||||
access: SimAccess(info.sim_access),
|
||||
billable_factor: info.billable_factor,
|
||||
cpu_class: info3.cpu_class_id,
|
||||
cpu_ratio: info3.cpu_ratio,
|
||||
colo_location: wire_string(&info3.colo_name),
|
||||
flags: RegionFlags(flags),
|
||||
id: info.cache_id,
|
||||
is_estate_manager: info.is_estate_manager,
|
||||
name: wire_string(&info.sim_name),
|
||||
product_name: wire_string(&info3.product_name),
|
||||
product_sku: wire_string(&info3.product_sku),
|
||||
protocols: RegionProtocols(protocols),
|
||||
region_id: packet.region_info2.region_id,
|
||||
sim_owner: info.sim_owner,
|
||||
terrain_base0: info.terrain_base0,
|
||||
terrain_base1: info.terrain_base1,
|
||||
terrain_base2: info.terrain_base2,
|
||||
terrain_base3: info.terrain_base3,
|
||||
terrain_detail0: info.terrain_detail0,
|
||||
terrain_detail1: info.terrain_detail1,
|
||||
terrain_detail2: info.terrain_detail2,
|
||||
terrain_detail3: info.terrain_detail3,
|
||||
terrain_height_range00: info.terrain_height_range00,
|
||||
terrain_height_range01: info.terrain_height_range01,
|
||||
terrain_height_range10: info.terrain_height_range10,
|
||||
terrain_height_range11: info.terrain_height_range11,
|
||||
terrain_start_height00: info.terrain_start_height00,
|
||||
terrain_start_height01: info.terrain_start_height01,
|
||||
terrain_start_height10: info.terrain_start_height10,
|
||||
terrain_start_height11: info.terrain_start_height11,
|
||||
water_height: info.water_height,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Shared data exposed through the C# `Simulator` field surface.
|
||||
pub struct SimulatorData {
|
||||
pub access: SimAccess,
|
||||
@@ -1063,6 +1205,7 @@ pub struct SimulatorData {
|
||||
connected: AtomicBool,
|
||||
movement_complete: AtomicBool,
|
||||
handshake_complete: AtomicBool,
|
||||
handshake_state: RwLock<SimulatorHandshakeState>,
|
||||
handshake_wait: Condvar,
|
||||
handshake_wait_lock: Mutex<()>,
|
||||
disconnect_candidate: AtomicBool,
|
||||
@@ -1110,6 +1253,37 @@ pub struct Simulator {
|
||||
/// This remains on the value surface, rather than behind `Deref`, because
|
||||
/// existing translated code consumes the public C# field by value.
|
||||
pub caps: Option<Box<Caps>>,
|
||||
pub access: SimAccess,
|
||||
pub billable_factor: f32,
|
||||
pub cpu_class: i32,
|
||||
pub cpu_ratio: i32,
|
||||
pub colo_location: String,
|
||||
pub flags: RegionFlags,
|
||||
pub id: UUID,
|
||||
pub is_estate_manager: bool,
|
||||
pub name: String,
|
||||
pub product_name: String,
|
||||
pub product_sku: String,
|
||||
pub protocols: RegionProtocols,
|
||||
pub region_id: UUID,
|
||||
pub sim_owner: UUID,
|
||||
pub terrain_base0: UUID,
|
||||
pub terrain_base1: UUID,
|
||||
pub terrain_base2: UUID,
|
||||
pub terrain_base3: UUID,
|
||||
pub terrain_detail0: UUID,
|
||||
pub terrain_detail1: UUID,
|
||||
pub terrain_detail2: UUID,
|
||||
pub terrain_detail3: UUID,
|
||||
pub terrain_height_range00: f32,
|
||||
pub terrain_height_range01: f32,
|
||||
pub terrain_height_range10: f32,
|
||||
pub terrain_height_range11: f32,
|
||||
pub terrain_start_height00: f32,
|
||||
pub terrain_start_height01: f32,
|
||||
pub terrain_start_height10: f32,
|
||||
pub terrain_start_height11: f32,
|
||||
pub water_height: f32,
|
||||
data: Arc<SimulatorData>,
|
||||
}
|
||||
|
||||
@@ -1155,6 +1329,45 @@ impl fmt::Debug for Simulator {
|
||||
}
|
||||
|
||||
impl Simulator {
|
||||
fn from_data(data: Arc<SimulatorData>, caps: Option<Box<Caps>>) -> Self {
|
||||
let state = read(&data.handshake_state).clone();
|
||||
Self {
|
||||
caps,
|
||||
access: state.access,
|
||||
billable_factor: state.billable_factor,
|
||||
cpu_class: state.cpu_class,
|
||||
cpu_ratio: state.cpu_ratio,
|
||||
colo_location: state.colo_location,
|
||||
flags: state.flags,
|
||||
id: state.id,
|
||||
is_estate_manager: state.is_estate_manager,
|
||||
name: state.name,
|
||||
product_name: state.product_name,
|
||||
product_sku: state.product_sku,
|
||||
protocols: state.protocols,
|
||||
region_id: state.region_id,
|
||||
sim_owner: state.sim_owner,
|
||||
terrain_base0: state.terrain_base0,
|
||||
terrain_base1: state.terrain_base1,
|
||||
terrain_base2: state.terrain_base2,
|
||||
terrain_base3: state.terrain_base3,
|
||||
terrain_detail0: state.terrain_detail0,
|
||||
terrain_detail1: state.terrain_detail1,
|
||||
terrain_detail2: state.terrain_detail2,
|
||||
terrain_detail3: state.terrain_detail3,
|
||||
terrain_height_range00: state.terrain_height_range00,
|
||||
terrain_height_range01: state.terrain_height_range01,
|
||||
terrain_height_range10: state.terrain_height_range10,
|
||||
terrain_height_range11: state.terrain_height_range11,
|
||||
terrain_start_height00: state.terrain_start_height00,
|
||||
terrain_start_height01: state.terrain_start_height01,
|
||||
terrain_start_height10: state.terrain_start_height10,
|
||||
terrain_start_height11: state.terrain_start_height11,
|
||||
water_height: state.water_height,
|
||||
data,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn native_prey_id(&self) -> UUID {
|
||||
*read(&self.data.prey_id)
|
||||
}
|
||||
@@ -1238,6 +1451,7 @@ impl Simulator {
|
||||
connected: AtomicBool::new(false),
|
||||
movement_complete: AtomicBool::new(false),
|
||||
handshake_complete: AtomicBool::new(false),
|
||||
handshake_state: RwLock::new(SimulatorHandshakeState::default()),
|
||||
handshake_wait: Condvar::new(),
|
||||
handshake_wait_lock: Mutex::new(()),
|
||||
disconnect_candidate: AtomicBool::new(false),
|
||||
@@ -1254,14 +1468,11 @@ impl Simulator {
|
||||
pause_serial: AtomicU32::new(0),
|
||||
});
|
||||
*mutex(&simulator_reference) = Arc::downgrade(&data);
|
||||
Ok(Self { caps: None, data })
|
||||
Ok(Self::from_data(data, None))
|
||||
}
|
||||
|
||||
pub(crate) fn native_clone_without_caps(&self) -> Self {
|
||||
Self {
|
||||
caps: None,
|
||||
data: Arc::clone(&self.data),
|
||||
}
|
||||
Self::from_data(Arc::clone(&self.data), None)
|
||||
}
|
||||
|
||||
pub(crate) fn native_data_weak(&self) -> Weak<SimulatorData> {
|
||||
@@ -1475,11 +1686,11 @@ impl Simulator {
|
||||
}
|
||||
|
||||
pub(crate) fn native_from_weak(data: &Weak<SimulatorData>) -> Option<Self> {
|
||||
data.upgrade().map(|data| Self { caps: None, data })
|
||||
data.upgrade().map(|data| Self::from_data(data, None))
|
||||
}
|
||||
|
||||
pub(crate) fn native_from_data(data: Arc<SimulatorData>) -> Self {
|
||||
Self { caps: None, data }
|
||||
Self::from_data(data, None)
|
||||
}
|
||||
|
||||
fn attach_manager(&self, manager: &Arc<NetworkManagerInner>, circuit_code: u32) {
|
||||
@@ -1937,7 +2148,7 @@ impl UdpPacketHandler for SimulatorUdpHandler {
|
||||
let Some(data) = mutex(&self.simulator).upgrade() else {
|
||||
return;
|
||||
};
|
||||
let simulator = Simulator { caps: None, data };
|
||||
let simulator = Simulator::from_data(data, None);
|
||||
let Ok(length) = usize::try_from(buffer.data_length) else {
|
||||
return;
|
||||
};
|
||||
@@ -1979,7 +2190,7 @@ impl UdpPacketHandler for SimulatorUdpHandler {
|
||||
let Some(data) = mutex(&self.simulator).upgrade() else {
|
||||
return;
|
||||
};
|
||||
let simulator = Simulator { caps: None, data };
|
||||
let simulator = Simulator::from_data(data, None);
|
||||
let _ = simulator
|
||||
.stats
|
||||
.add_sent_bytes(i64::try_from(bytes_sent).unwrap_or(i64::MAX));
|
||||
@@ -2160,7 +2371,9 @@ impl NetworkManagerInner {
|
||||
continue;
|
||||
};
|
||||
inner.process_internal_packet(&packet, &simulator, raw_data.as_deref());
|
||||
if let Some(data) = raw_data.as_ref() {
|
||||
if let Some(raw_data) = raw_data.as_ref()
|
||||
&& let Ok(data) = canonical_packet_bytes(&packet, raw_data)
|
||||
{
|
||||
inner.events.raw_packet_received.emit_with(|| {
|
||||
RawPacketReceivedEventArgs {
|
||||
packet_type: packet.type_,
|
||||
@@ -2236,12 +2449,29 @@ impl NetworkManagerInner {
|
||||
let Some(raw_data) = raw_data else {
|
||||
return;
|
||||
};
|
||||
let mut position = 0;
|
||||
let Ok(_incoming) =
|
||||
RegionHandshakePacket::new_with_bytes_int32(raw_data.to_vec(), &mut position)
|
||||
else {
|
||||
let Ok(mut packet_end) = i32::try_from(raw_data.len()) else {
|
||||
return;
|
||||
};
|
||||
packet_end -= 1;
|
||||
let mut position = 0;
|
||||
let mut zero_buffer =
|
||||
vec![0_u8; UdpTransportConfig::default().max_decoded_packet_size];
|
||||
let Ok(mut incoming) = RegionHandshakePacket::new_with_constructor() else {
|
||||
return;
|
||||
};
|
||||
if incoming
|
||||
.from_bytes_with_bytes_int32_int32_bytes(
|
||||
raw_data.to_vec(),
|
||||
&mut position,
|
||||
&mut packet_end,
|
||||
Some(&mut zero_buffer),
|
||||
)
|
||||
.is_err()
|
||||
{
|
||||
return;
|
||||
}
|
||||
*write(&simulator.data.handshake_state) =
|
||||
SimulatorHandshakeState::from_packet(&incoming);
|
||||
|
||||
let Ok(mut reply) = RegionHandshakeReplyPacket::new_with_constructor() else {
|
||||
return;
|
||||
@@ -3621,7 +3851,7 @@ impl NetworkManager {
|
||||
simulator.native_complete_agent_movement()?;
|
||||
self.inner.set_current_sim(simulator.clone(), seed_caps)?;
|
||||
}
|
||||
Ok(Some(simulator))
|
||||
Ok(Some(simulator.clone()))
|
||||
}
|
||||
|
||||
pub async fn native_connect_async(
|
||||
@@ -3971,6 +4201,37 @@ mod tests {
|
||||
use libremetaverse_structured_data::{OSD, OSDMap};
|
||||
use std::sync::mpsc;
|
||||
|
||||
#[test]
|
||||
fn internal_dispatch_canonicalizes_zerocoded_packets() {
|
||||
let mut reply = crate::packets::MapBlockReplyPacket::new_with_constructor().unwrap();
|
||||
let mut block =
|
||||
crate::packets::MapBlockReplyPacketDataBlock::new_with_constructor().unwrap();
|
||||
block.name = b"Fake Region\0".to_vec();
|
||||
reply.data.push(block);
|
||||
let mut raw = reply.to_bytes_with_method().unwrap();
|
||||
raw[0] |= Helpers::MSG_ZEROCODED;
|
||||
let mut zerocoded = vec![0; raw.len() * 2 + 2];
|
||||
let length = Helpers::zero_encode(
|
||||
Some(&raw),
|
||||
i32::try_from(raw.len()).unwrap(),
|
||||
Some(&mut zerocoded),
|
||||
)
|
||||
.unwrap();
|
||||
zerocoded.truncate(usize::try_from(length).unwrap());
|
||||
let mut packet_end = i32::try_from(zerocoded.len()).unwrap() - 1;
|
||||
let packet = Packet::build_packet_with_bytes_int32_bytes(
|
||||
zerocoded.clone(),
|
||||
&mut packet_end,
|
||||
vec![0; UdpTransportConfig::default().max_decoded_packet_size],
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let bytes = canonical_packet_bytes(&packet, &zerocoded).unwrap();
|
||||
let decoded =
|
||||
crate::packets::MapBlockReplyPacket::new_with_bytes_int32(bytes, &mut 0).unwrap();
|
||||
assert_eq!(wire_string(&decoded.data[0].name), "Fake Region");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn manager_workers_and_callback_registries_do_not_retain_the_client() {
|
||||
let client = GridClient::new().expect("client");
|
||||
|
||||
@@ -536,9 +536,18 @@ fn spawn_fake_server() -> FakeServer {
|
||||
socket.send_to(&ack(sequence), client_endpoint).unwrap();
|
||||
|
||||
let mut handshake = RegionHandshakePacket::new_with_constructor().unwrap();
|
||||
handshake.region_info.sim_name = b"Fake Region".to_vec();
|
||||
let mut bytes = handshake.to_bytes_with_method().unwrap();
|
||||
bytes[0] &= !(Helpers::MSG_RELIABLE | Helpers::MSG_ZEROCODED);
|
||||
handshake.region_info.sim_name = b"Fake Region\0".to_vec();
|
||||
handshake.region_info.water_height = 21.5;
|
||||
handshake.region_info2.region_id = UUID::new_with_u_int64(99).unwrap();
|
||||
let raw = handshake.to_bytes_with_method().unwrap();
|
||||
let mut bytes = vec![0_u8; raw.len().saturating_mul(2).saturating_add(2)];
|
||||
let length = Helpers::zero_encode(
|
||||
Some(&raw),
|
||||
i32::try_from(raw.len()).unwrap(),
|
||||
Some(&mut bytes),
|
||||
)
|
||||
.unwrap();
|
||||
bytes.truncate(usize::try_from(length).unwrap());
|
||||
socket.send_to(&bytes, client_endpoint).unwrap();
|
||||
|
||||
loop {
|
||||
@@ -833,6 +842,9 @@ fn fake_server_drives_circuit_handshake_ping_disable_and_disconnect_reasons() {
|
||||
assert_eq!(sim.seed_capability(), Some(seed));
|
||||
assert_eq!(sim.size_x, 512);
|
||||
assert_eq!(sim.size_y, 256);
|
||||
assert_eq!(sim.name, "Fake Region");
|
||||
assert_eq!(sim.region_id, UUID::new_with_u_int64(99).unwrap());
|
||||
assert_eq!(sim.water_height, 21.5);
|
||||
wait_until(|| sim.handshake_complete());
|
||||
assert_eq!(
|
||||
server.reports.recv_timeout(Duration::from_secs(1)).unwrap(),
|
||||
|
||||
@@ -448,6 +448,7 @@ pub async fn run_deterministic_acceptance(
|
||||
ReconnectPolicy {
|
||||
initial_delay: Duration::from_millis(10),
|
||||
maximum_delay: Duration::from_millis(20),
|
||||
readiness_timeout: Duration::from_secs(1),
|
||||
stable_reset_after: Duration::from_secs(1),
|
||||
shutdown_deadline: Duration::from_secs(1),
|
||||
jitter_basis_points: 0,
|
||||
|
||||
@@ -929,35 +929,43 @@ impl crate::session::GridSessionBackend for LibremetaverseSessionBackend {
|
||||
.delivery_generation
|
||||
.write()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner) = None;
|
||||
let mut params = self
|
||||
.network
|
||||
.native_default_login_params(
|
||||
self.first_name.clone(),
|
||||
self.last_name.clone(),
|
||||
self.password.expose_secret().to_owned(),
|
||||
"MetaCrate".to_owned(),
|
||||
env!("CARGO_PKG_VERSION").to_owned(),
|
||||
)
|
||||
.map_err(|_| {
|
||||
crate::session::SessionFailure::new(
|
||||
crate::session::SessionFailureKind::InvalidConfiguration,
|
||||
let mut start = "last";
|
||||
loop {
|
||||
let mut params = self
|
||||
.network
|
||||
.native_default_login_params(
|
||||
self.first_name.clone(),
|
||||
self.last_name.clone(),
|
||||
self.password.expose_secret().to_owned(),
|
||||
"MetaCrate".to_owned(),
|
||||
env!("CARGO_PKG_VERSION").to_owned(),
|
||||
)
|
||||
})?;
|
||||
params.uri.clone_from(&self.login_url);
|
||||
"last".clone_into(&mut params.start);
|
||||
let logged_in = self
|
||||
.network
|
||||
.native_login(params, Some(cancellation))
|
||||
.await
|
||||
.map_err(|_| {
|
||||
crate::session::SessionFailure::new(
|
||||
crate::session::SessionFailureKind::TransientTransport,
|
||||
)
|
||||
})?;
|
||||
if !logged_in {
|
||||
return Err(classify_native_login_failure(
|
||||
&self.network.native_login_error_key(),
|
||||
));
|
||||
.map_err(|_| {
|
||||
crate::session::SessionFailure::new(
|
||||
crate::session::SessionFailureKind::InvalidConfiguration,
|
||||
)
|
||||
})?;
|
||||
params.uri.clone_from(&self.login_url);
|
||||
start.clone_into(&mut params.start);
|
||||
let logged_in = self
|
||||
.network
|
||||
.native_login(params, Some(cancellation.clone()))
|
||||
.await
|
||||
.map_err(|_| {
|
||||
crate::session::SessionFailure::new(
|
||||
crate::session::SessionFailureKind::TransientTransport,
|
||||
)
|
||||
})?;
|
||||
if logged_in {
|
||||
break;
|
||||
}
|
||||
let error_key = self.network.native_login_error_key();
|
||||
let message = self.network.native_login_message();
|
||||
if start == "last" && native_last_location_unavailable(&message) {
|
||||
start = "home";
|
||||
continue;
|
||||
}
|
||||
return Err(classify_native_login_failure(&error_key, &message));
|
||||
}
|
||||
|
||||
// Native login has already installed the current simulator and
|
||||
@@ -1252,9 +1260,16 @@ fn native_delivery_id(prefix: &str, fields: &[&str]) -> String {
|
||||
}
|
||||
|
||||
#[cfg(feature = "live-grid")]
|
||||
fn classify_native_login_failure(error_key: &str) -> crate::session::SessionFailure {
|
||||
fn classify_native_login_failure(error_key: &str, message: &str) -> crate::session::SessionFailure {
|
||||
let normalized = error_key.trim().to_ascii_lowercase();
|
||||
let kind = if matches!(
|
||||
let kind = if native_last_location_unavailable(message)
|
||||
|| message
|
||||
.trim()
|
||||
.to_ascii_lowercase()
|
||||
.contains("already logged in")
|
||||
{
|
||||
crate::session::SessionFailureKind::ServerFailure
|
||||
} else if matches!(
|
||||
normalized.as_str(),
|
||||
"key" | "password" | "credential" | "account" | "username" | "user"
|
||||
) {
|
||||
@@ -1267,6 +1282,13 @@ fn classify_native_login_failure(error_key: &str) -> crate::session::SessionFail
|
||||
crate::session::SessionFailure::new(kind)
|
||||
}
|
||||
|
||||
#[cfg(feature = "live-grid")]
|
||||
fn native_last_location_unavailable(message: &str) -> bool {
|
||||
let message = message.trim().to_ascii_lowercase();
|
||||
message.contains("failed to verify user presence")
|
||||
|| message.contains("access denied to region")
|
||||
}
|
||||
|
||||
impl GridBackend for OfflineGridBackend {
|
||||
fn name(&self) -> &'static str {
|
||||
"offline-fake"
|
||||
@@ -1315,4 +1337,24 @@ mod tests {
|
||||
cancellation.cancel();
|
||||
run.await.expect("clean cancellation");
|
||||
}
|
||||
|
||||
#[cfg(feature = "live-grid")]
|
||||
#[test]
|
||||
fn stale_presence_is_retryable_without_masking_bad_credentials() {
|
||||
assert!(native_last_location_unavailable(
|
||||
"Failed to verify user presence in the grid, access denied to region",
|
||||
));
|
||||
assert_eq!(
|
||||
classify_native_login_failure(
|
||||
"account",
|
||||
"Failed to verify user presence in the grid, access denied to region",
|
||||
)
|
||||
.kind(),
|
||||
crate::session::SessionFailureKind::ServerFailure,
|
||||
);
|
||||
assert_eq!(
|
||||
classify_native_login_failure("account", "Invalid credentials").kind(),
|
||||
crate::session::SessionFailureKind::InvalidCredentials,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1120,6 +1120,7 @@ struct RawBehavior {
|
||||
struct RawReconnect {
|
||||
initial_delay_milliseconds: Option<u64>,
|
||||
maximum_delay_seconds: Option<u64>,
|
||||
readiness_timeout_seconds: Option<u64>,
|
||||
stable_reset_seconds: Option<u64>,
|
||||
jitter_basis_points: Option<u16>,
|
||||
offline_work_capacity: Option<usize>,
|
||||
@@ -1318,7 +1319,7 @@ fn resolve<E: Environment>(
|
||||
let timeouts = Timeouts {
|
||||
startup: checked_duration(
|
||||
"timeouts.startup_seconds",
|
||||
raw.timeouts.startup.unwrap_or(30),
|
||||
raw.timeouts.startup.unwrap_or(120),
|
||||
1,
|
||||
300,
|
||||
)?,
|
||||
@@ -1341,6 +1342,11 @@ fn resolve<E: Environment>(
|
||||
raw.reconnect.initial_delay_milliseconds.unwrap_or(1_000),
|
||||
),
|
||||
maximum_delay: Duration::from_secs(raw.reconnect.maximum_delay_seconds.unwrap_or(60)),
|
||||
readiness_timeout: Duration::from_secs(
|
||||
raw.reconnect
|
||||
.readiness_timeout_seconds
|
||||
.unwrap_or(reconnect_defaults.readiness_timeout.as_secs()),
|
||||
),
|
||||
stable_reset_after: Duration::from_secs(raw.reconnect.stable_reset_seconds.unwrap_or(120)),
|
||||
shutdown_deadline: timeouts.shutdown,
|
||||
jitter_basis_points: raw
|
||||
@@ -1952,6 +1958,7 @@ mod tests {
|
||||
.load()
|
||||
.unwrap();
|
||||
assert_eq!(config.storage_path, PathBuf::from("operator-data"));
|
||||
assert_eq!(config.timeouts.startup, Duration::from_mins(2));
|
||||
let paths = PlatformPaths::from_environment(&environment);
|
||||
assert!(paths.config_file.ends_with("config.yml"));
|
||||
assert!(paths.data_directory.ends_with("grid-agent"));
|
||||
|
||||
@@ -20,6 +20,7 @@ const MAX_OFFLINE_WORK: usize = 1_024;
|
||||
const MAX_SEEN_WORK: usize = 4_096;
|
||||
const MAX_BACKOFF: Duration = Duration::from_hours(1);
|
||||
const MAX_STABLE_RESET: Duration = Duration::from_hours(24);
|
||||
const MAX_READINESS_TIMEOUT: Duration = Duration::from_mins(10);
|
||||
const MAX_SHUTDOWN: Duration = Duration::from_mins(1);
|
||||
|
||||
pub type SessionFuture<'a, T> = Pin<Box<dyn Future<Output = T> + Send + 'a>>;
|
||||
@@ -201,6 +202,7 @@ pub trait GridSessionBackend: Send + Sync + 'static {
|
||||
pub struct ReconnectPolicy {
|
||||
pub initial_delay: Duration,
|
||||
pub maximum_delay: Duration,
|
||||
pub readiness_timeout: Duration,
|
||||
pub stable_reset_after: Duration,
|
||||
pub shutdown_deadline: Duration,
|
||||
pub jitter_basis_points: u16,
|
||||
@@ -213,6 +215,7 @@ impl Default for ReconnectPolicy {
|
||||
Self {
|
||||
initial_delay: Duration::from_secs(1),
|
||||
maximum_delay: Duration::from_mins(1),
|
||||
readiness_timeout: Duration::from_secs(30),
|
||||
stable_reset_after: Duration::from_mins(2),
|
||||
shutdown_deadline: Duration::from_secs(10),
|
||||
jitter_basis_points: 2_000,
|
||||
@@ -237,6 +240,8 @@ impl ReconnectPolicy {
|
||||
if self.initial_delay.is_zero()
|
||||
|| self.initial_delay > self.maximum_delay
|
||||
|| self.maximum_delay > MAX_BACKOFF
|
||||
|| self.readiness_timeout.is_zero()
|
||||
|| self.readiness_timeout > MAX_READINESS_TIMEOUT
|
||||
|| self.stable_reset_after.is_zero()
|
||||
|| self.stable_reset_after > MAX_STABLE_RESET
|
||||
|| self.shutdown_deadline.is_zero()
|
||||
@@ -608,6 +613,7 @@ enum LoginEvent {
|
||||
enum ActiveEvent {
|
||||
Cancelled,
|
||||
Command(Option<Command>),
|
||||
ReadinessTimedOut,
|
||||
Signal(SessionSignal),
|
||||
}
|
||||
|
||||
@@ -635,6 +641,7 @@ impl Runtime {
|
||||
let mut desired = DesiredState::Running;
|
||||
let mut session: Option<Box<dyn GridSession>> = None;
|
||||
let mut ready = false;
|
||||
let mut readiness_deadline = None;
|
||||
let mut stable_since = None;
|
||||
|
||||
loop {
|
||||
@@ -711,6 +718,8 @@ impl Runtime {
|
||||
Ok(connected) if connected.generation() == generation => {
|
||||
session = Some(connected);
|
||||
ready = false;
|
||||
readiness_deadline =
|
||||
Some(Instant::now() + self.policy.readiness_timeout);
|
||||
stable_since = None;
|
||||
self.transition(
|
||||
SessionState::Degraded,
|
||||
@@ -756,9 +765,18 @@ impl Runtime {
|
||||
};
|
||||
let signal = active.next_signal(token);
|
||||
tokio::pin!(signal);
|
||||
let readiness = async {
|
||||
if let Some(deadline) = readiness_deadline {
|
||||
tokio::time::sleep_until(deadline).await;
|
||||
} else {
|
||||
std::future::pending::<()>().await;
|
||||
}
|
||||
};
|
||||
tokio::pin!(readiness);
|
||||
tokio::select! {
|
||||
() = self.cancellation.token().cancelled() => ActiveEvent::Cancelled,
|
||||
command = self.commands.recv() => ActiveEvent::Command(command),
|
||||
() = &mut readiness => ActiveEvent::ReadinessTimedOut,
|
||||
next = &mut signal => ActiveEvent::Signal(next),
|
||||
}
|
||||
};
|
||||
@@ -766,6 +784,21 @@ impl Runtime {
|
||||
ActiveEvent::Cancelled => {
|
||||
desired = DesiredState::Shutdown;
|
||||
}
|
||||
ActiveEvent::ReadinessTimedOut => {
|
||||
ready = false;
|
||||
readiness_deadline = None;
|
||||
stable_since = None;
|
||||
self.fence.invalidate();
|
||||
if let Some(unready) = session.take() {
|
||||
self.close_session(unready).await;
|
||||
}
|
||||
self.failures = self.failures.saturating_add(1);
|
||||
desired = self
|
||||
.backoff(SessionFailure::new(
|
||||
SessionFailureKind::TransientTransport,
|
||||
))
|
||||
.await;
|
||||
}
|
||||
ActiveEvent::Command(command) => match command {
|
||||
Some(Command::Submit(work, response)) => {
|
||||
let disposition = self.handle_work(work.clone(), ready, generation);
|
||||
@@ -796,6 +829,7 @@ impl Runtime {
|
||||
ActiveEvent::Signal(next) => match next {
|
||||
SessionSignal::Ready => {
|
||||
ready = true;
|
||||
readiness_deadline = None;
|
||||
stable_since.get_or_insert_with(Instant::now);
|
||||
self.transition(
|
||||
SessionState::Online,
|
||||
@@ -807,6 +841,8 @@ impl Runtime {
|
||||
}
|
||||
SessionSignal::Degraded => {
|
||||
ready = false;
|
||||
readiness_deadline =
|
||||
Some(Instant::now() + self.policy.readiness_timeout);
|
||||
stable_since = None;
|
||||
self.transition(
|
||||
SessionState::Degraded,
|
||||
@@ -846,6 +882,7 @@ impl Runtime {
|
||||
self.close_session(active).await;
|
||||
}
|
||||
ready = false;
|
||||
readiness_deadline = None;
|
||||
stable_since = None;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -196,6 +196,7 @@ fn test_policy() -> ReconnectPolicy {
|
||||
ReconnectPolicy {
|
||||
initial_delay: Duration::from_secs(1),
|
||||
maximum_delay: Duration::from_secs(8),
|
||||
readiness_timeout: Duration::from_secs(30),
|
||||
stable_reset_after: Duration::from_secs(10),
|
||||
shutdown_deadline: Duration::from_secs(2),
|
||||
jitter_basis_points: 0,
|
||||
@@ -413,6 +414,38 @@ async fn transport_connection_is_distinct_from_full_agent_readiness() {
|
||||
handle.shutdown().await.expect("shutdown");
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn readiness_timeout_logs_out_and_retries_instead_of_staying_degraded() {
|
||||
let plans = [
|
||||
LoginPlan::Success {
|
||||
after: Duration::ZERO,
|
||||
signals: VecDeque::new(),
|
||||
},
|
||||
LoginPlan::Success {
|
||||
after: Duration::ZERO,
|
||||
signals: VecDeque::from([(Duration::ZERO, SessionSignal::Ready)]),
|
||||
},
|
||||
];
|
||||
let (backend, stats) = FakeBackend::new(plans);
|
||||
let mut policy = test_policy();
|
||||
policy.readiness_timeout = Duration::from_secs(5);
|
||||
let erased: Arc<dyn GridSessionBackend> = backend;
|
||||
let mut handle = SessionSupervisor::new(erased, policy, 32, 256)
|
||||
.expect("supervisor")
|
||||
.start();
|
||||
|
||||
wait_state(&handle, SessionState::Degraded).await;
|
||||
tokio::time::advance(Duration::from_secs(5)).await;
|
||||
settle().await;
|
||||
wait_state(&handle, SessionState::Backoff).await;
|
||||
assert_eq!(stats.logouts.load(Ordering::Acquire), 1);
|
||||
tokio::time::advance(Duration::from_secs(1)).await;
|
||||
settle().await;
|
||||
wait_state(&handle, SessionState::Online).await;
|
||||
handle.shutdown().await.expect("shutdown");
|
||||
assert_eq!(stats.active_sessions.load(Ordering::Acquire), 0);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn shutdown_covers_degraded_authentication_blocked_stopped_and_online_states() {
|
||||
let cases = [
|
||||
|
||||
@@ -172,6 +172,7 @@ async fn repeated_reconnects_have_one_resource_set_no_replay_and_no_shutdown_lea
|
||||
let policy = ReconnectPolicy {
|
||||
initial_delay: Duration::from_secs(1),
|
||||
maximum_delay: Duration::from_secs(4),
|
||||
readiness_timeout: Duration::from_secs(30),
|
||||
stable_reset_after: Duration::from_secs(30),
|
||||
shutdown_deadline: Duration::from_secs(3),
|
||||
jitter_basis_points: 0,
|
||||
|
||||
Reference in New Issue
Block a user