Implement network manager simulator lifecycle (#53)

crates/libremetaverse/tests/network_manager.rs

@@ -0,0 +1,899 @@
+use libremetaverse::interfaces::IMessage;
+use libremetaverse::messages::linden::{
+    EnableSimulatorMessage, EnableSimulatorMessageSimulatorInfoBlock,
+};
+use libremetaverse::packets::{
+    DisableSimulatorPacket, Packet, PacketAckPacket, PacketAckPacketPacketsBlock, PacketType,
+    RegionHandshakePacket, RegionHandshakeReplyPacket, StartPingCheckPacket, UseCircuitCodePacket,
+};
+use libremetaverse::{
+    CapsEventDictionary, CapsEventQueueCallback, GridClient, Helpers, NetworkManager,
+    NetworkManagerDisconnectType, PacketEventDictionary, Simulator,
+};
+use libremetaverse_types::compat::{EventHandler, Uri};
+use std::net::{IpAddr, Ipv4Addr, SocketAddr, UdpSocket};
+use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
+use std::sync::mpsc::{Receiver, Sender, channel};
+use std::sync::{Arc, Barrier, Mutex, Weak};
+use std::thread::{self, JoinHandle};
+use std::time::{Duration, Instant};
+
+fn block_on<F: std::future::Future>(future: F) -> F::Output {
+    use std::task::{Context, Poll, Wake, Waker};
+
+    struct ThreadWake(thread::Thread);
+
+    impl Wake for ThreadWake {
+        fn wake(self: Arc<Self>) {
+            self.0.unpark();
+        }
+    }
+
+    let waker = Waker::from(Arc::new(ThreadWake(thread::current())));
+    let mut context = Context::from_waker(&waker);
+    let mut future = std::pin::pin!(future);
+    loop {
+        match future.as_mut().poll(&mut context) {
+            Poll::Ready(output) => return output,
+            Poll::Pending => thread::park(),
+        }
+    }
+}
+
+fn loopback(port: u16) -> SocketAddr {
+    SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), port)
+}
+
+fn simulator(client: &GridClient) -> Simulator {
+    Simulator::new(
+        client.clone(),
+        loopback(13000),
+        0x1000,
+        Some(256),
+        Some(256),
+    )
+    .expect("simulator")
+}
+
+fn base_packet(packet_type: PacketType) -> Packet {
+    Packet::build_packet_with_packet_type(packet_type).expect("base packet")
+}
+
+fn wait_until(mut condition: impl FnMut() -> bool) {
+    let deadline = Instant::now() + Duration::from_secs(2);
+    while !condition() {
+        assert!(Instant::now() < deadline, "condition timed out");
+        thread::sleep(Duration::from_millis(2));
+    }
+}
+
+#[test]
+fn packet_callbacks_preserve_filtering_order_async_policy_and_reentrancy() {
+    let client = GridClient::new().expect("client");
+    let sim = simulator(&client);
+    let events = Arc::new(PacketEventDictionary::new(client).expect("packet events"));
+    let order = Arc::new(Mutex::new(Vec::new()));
+
+    let default_order = Arc::clone(&order);
+    events
+        .register_event(
+            PacketType::Default,
+            Arc::new(move |_| default_order.lock().unwrap().push("default")),
+            false,
+        )
+        .unwrap();
+    let specific_order = Arc::clone(&order);
+    events
+        .register_event(
+            PacketType::StartPingCheck,
+            Arc::new(move |_| specific_order.lock().unwrap().push("specific")),
+            false,
+        )
+        .unwrap();
+
+    events
+        .invoke_raise_event(
+            PacketType::StartPingCheck,
+            base_packet(PacketType::StartPingCheck),
+            sim.clone(),
+        )
+        .unwrap();
+    assert_eq!(*order.lock().unwrap(), ["default", "specific"]);
+
+    order.lock().unwrap().clear();
+    events
+        .invoke_raise_event(
+            PacketType::CompletePingCheck,
+            base_packet(PacketType::CompletePingCheck),
+            sim.clone(),
+        )
+        .unwrap();
+    assert_eq!(*order.lock().unwrap(), ["default"]);
+
+    let reentrant_count = Arc::new(AtomicUsize::new(0));
+    let weak_events: Weak<PacketEventDictionary> = Arc::downgrade(&events);
+    let count = Arc::clone(&reentrant_count);
+    let reentrant: EventHandler<_> = Arc::new(move |_| {
+        count.fetch_add(1, Ordering::Relaxed);
+        if let Some(events) = weak_events.upgrade() {
+            events
+                .register_event(PacketType::UseCircuitCode, Arc::new(|_| {}), false)
+                .unwrap();
+        }
+    });
+    events
+        .register_event(PacketType::UseCircuitCode, reentrant, false)
+        .unwrap();
+    events
+        .invoke_raise_event(
+            PacketType::UseCircuitCode,
+            base_packet(PacketType::UseCircuitCode),
+            sim.clone(),
+        )
+        .unwrap();
+    assert_eq!(reentrant_count.load(Ordering::Relaxed), 1);
+
+    let async_events = PacketEventDictionary::new(sim.client.clone()).unwrap();
+    let removed_async: EventHandler<_> = Arc::new(|_| {});
+    async_events
+        .register_event(PacketType::StartPingCheck, Arc::clone(&removed_async), true)
+        .unwrap();
+    let caller_thread = thread::current().id();
+    let (thread_sender, thread_receiver) = channel();
+    async_events
+        .register_event(
+            PacketType::StartPingCheck,
+            Arc::new(move |_| thread_sender.send(thread::current().id()).unwrap()),
+            false,
+        )
+        .unwrap();
+    async_events
+        .unregister_event(PacketType::StartPingCheck, removed_async)
+        .unwrap();
+    async_events
+        .invoke_raise_event(
+            PacketType::StartPingCheck,
+            base_packet(PacketType::StartPingCheck),
+            sim,
+        )
+        .unwrap();
+    assert_ne!(
+        thread_receiver
+            .recv_timeout(Duration::from_secs(1))
+            .unwrap(),
+        caller_thread,
+        "C# keeps a packet type asynchronous after its first async registration"
+    );
+}
+
+#[test]
+fn synchronous_specific_dispatch_suppresses_default_async_chain_like_reference() {
+    let mixed_events = PacketEventDictionary::new(GridClient::new().unwrap()).unwrap();
+    let (default_sender, default_receiver) = channel();
+    mixed_events
+        .register_event(
+            PacketType::Default,
+            Arc::new(move |_| default_sender.send(()).unwrap()),
+            true,
+        )
+        .unwrap();
+    let specific_calls = Arc::new(AtomicUsize::new(0));
+    let calls = Arc::clone(&specific_calls);
+    mixed_events
+        .register_event(
+            PacketType::StartPingCheck,
+            Arc::new(move |_| {
+                calls.fetch_add(1, Ordering::Relaxed);
+            }),
+            false,
+        )
+        .unwrap();
+    mixed_events
+        .invoke_raise_event(
+            PacketType::StartPingCheck,
+            base_packet(PacketType::StartPingCheck),
+            simulator(&mixed_events.client),
+        )
+        .unwrap();
+    assert_eq!(specific_calls.load(Ordering::Relaxed), 1);
+    assert!(
+        default_receiver
+            .recv_timeout(Duration::from_millis(50))
+            .is_err(),
+        "C# returns after a synchronous specific chain without queuing the default async chain"
+    );
+}
+
+struct TestCapsMessage;
+impl IMessage for TestCapsMessage {}
+
+#[test]
+fn caps_callbacks_are_ordered_and_subscription_guards_unregister() {
+    let client = GridClient::new().expect("client");
+    let sim = simulator(&client);
+    let events = CapsEventDictionary::new(client).expect("CAPS events");
+    let order = Arc::new(Mutex::new(Vec::new()));
+
+    let default_order = Arc::clone(&order);
+    let default = CapsEventQueueCallback::from_handler(move |_, _, _| {
+        default_order.lock().unwrap().push("default");
+    });
+    let specific_order = Arc::clone(&order);
+    let specific = CapsEventQueueCallback::from_handler(move |_, _, _| {
+        specific_order.lock().unwrap().push("specific");
+    });
+    let default_guard = events.subscribe(String::new(), default);
+    let specific_guard = events.subscribe("EnableSimulator".into(), specific);
+
+    events.invoke_raise_event("EnableSimulator", &TestCapsMessage, sim.clone());
+    assert_eq!(*order.lock().unwrap(), ["default", "specific"]);
+
+    drop(specific_guard);
+    order.lock().unwrap().clear();
+    events.invoke_raise_event("EnableSimulator", &TestCapsMessage, sim);
+    assert_eq!(*order.lock().unwrap(), ["default"]);
+    drop(default_guard);
+}
+
+#[test]
+fn concurrent_registration_and_removal_is_deterministic() {
+    const CALLBACKS: usize = 16;
+    let client = GridClient::new().expect("client");
+    let sim = simulator(&client);
+    let events = Arc::new(PacketEventDictionary::new(client).unwrap());
+    let barrier = Arc::new(Barrier::new(CALLBACKS + 1));
+    let calls = Arc::new(AtomicUsize::new(0));
+    let (guard_sender, guard_receiver) = channel();
+
+    thread::scope(|scope| {
+        for _ in 0..CALLBACKS {
+            let events = Arc::clone(&events);
+            let barrier = Arc::clone(&barrier);
+            let calls = Arc::clone(&calls);
+            let guard_sender = guard_sender.clone();
+            scope.spawn(move || {
+                let guard = events.subscribe(
+                    PacketType::StartPingCheck,
+                    Arc::new(move |_| {
+                        calls.fetch_add(1, Ordering::Relaxed);
+                    }),
+                    false,
+                );
+                guard_sender.send(guard).unwrap();
+                barrier.wait();
+            });
+        }
+        barrier.wait();
+    });
+    drop(guard_sender);
+    let guards: Vec<_> = guard_receiver.into_iter().collect();
+
+    events
+        .invoke_raise_event(
+            PacketType::StartPingCheck,
+            base_packet(PacketType::StartPingCheck),
+            sim.clone(),
+        )
+        .unwrap();
+    assert_eq!(calls.load(Ordering::Relaxed), CALLBACKS);
+
+    drop(guards);
+    calls.store(0, Ordering::Relaxed);
+    events
+        .invoke_raise_event(
+            PacketType::StartPingCheck,
+            base_packet(PacketType::StartPingCheck),
+            sim,
+        )
+        .unwrap();
+    assert_eq!(calls.load(Ordering::Relaxed), 0);
+}
+
+#[test]
+fn sim_connecting_can_cancel_without_leaking_collection_state() {
+    let client = GridClient::new().expect("client");
+    let manager = NetworkManager::new(client).expect("manager");
+    let guard = manager.subscribe_sim_connecting(Arc::new(|mut args| args.set_cancel(true)));
+
+    let connected = manager
+        .connect_with_ip_end_point_u_int64_boolean_uri_u_int32_u_int32(
+            loopback(9),
+            0x2000,
+            true,
+            Some(Uri("http://caps.invalid/seed".into())),
+            512,
+            256,
+        )
+        .unwrap();
+
+    assert!(connected.is_none());
+    assert!(manager.simulators.is_empty());
+    assert!(
+        manager.connected(),
+        "C# marks the manager connected before raising the cancellable event"
+    );
+    assert!(manager.current_sim().is_none());
+    drop(guard);
+}
+
+enum ServerCommand {
+    Ping,
+    Disable,
+    Stop,
+}
+
+#[derive(Debug, Eq, PartialEq)]
+enum ServerReport {
+    Ack(u32),
+    Circuit(u32),
+    HandshakeReply(u32),
+    Ping(u8),
+}
+
+struct FakeServer {
+    endpoint: SocketAddr,
+    commands: Sender<ServerCommand>,
+    reports: Receiver<ServerReport>,
+    handle: Option<JoinHandle<()>>,
+}
+
+impl Drop for FakeServer {
+    fn drop(&mut self) {
+        let _ = self.commands.send(ServerCommand::Stop);
+        if let Some(handle) = self.handle.take() {
+            let _ = handle.join();
+        }
+    }
+}
+
+fn packet_type(bytes: &[u8]) -> Option<PacketType> {
+    let mut packet_end = i32::try_from(bytes.len()).ok()?.checked_sub(1)?;
+    Packet::build_packet_with_bytes_int32_bytes(bytes.to_vec(), &mut packet_end, vec![0; 8192])
+        .ok()
+        .map(|packet| packet.type_)
+}
+
+fn ack(sequence: u32) -> Vec<u8> {
+    let mut packet = PacketAckPacket::new_with_constructor().unwrap();
+    packet.packets = vec![PacketAckPacketPacketsBlock { id: sequence }];
+    let mut bytes = packet.to_bytes_with_method().unwrap();
+    bytes[0] &= !(Helpers::MSG_RELIABLE | Helpers::MSG_ZEROCODED);
+    bytes
+}
+
+fn serve_ping_command(
+    socket: &UdpSocket,
+    client_endpoint: SocketAddr,
+    report_sender: &Sender<ServerReport>,
+    buffer: &mut [u8],
+) {
+    let mut ping = StartPingCheckPacket::new_with_constructor().unwrap();
+    ping.ping_id.ping_id = 42;
+    ping.ping_id.oldest_unacked = 1;
+    let mut bytes = ping.to_bytes_with_method().unwrap();
+    bytes[0] &= !Helpers::MSG_ZEROCODED;
+    bytes[0] |= Helpers::MSG_RELIABLE;
+    bytes[1..5].copy_from_slice(&99_u32.to_be_bytes());
+    socket.send_to(&bytes, client_endpoint).unwrap();
+
+    loop {
+        let (length, _) = socket.recv_from(buffer).expect("ping response");
+        match packet_type(&buffer[..length]) {
+            Some(PacketType::PacketAck) => {
+                let mut position = 0;
+                let response =
+                    PacketAckPacket::new_with_bytes_int32(buffer[..length].to_vec(), &mut position)
+                        .unwrap();
+                assert_eq!(response.packets.len(), 1);
+                report_sender
+                    .send(ServerReport::Ack(response.packets[0].id))
+                    .unwrap();
+            }
+            Some(PacketType::CompletePingCheck) => {
+                let mut position = 0;
+                let response =
+                    libremetaverse::packets::CompletePingCheckPacket::new_with_bytes_int32(
+                        buffer[..length].to_vec(),
+                        &mut position,
+                    )
+                    .unwrap();
+                report_sender
+                    .send(ServerReport::Ping(response.ping_id.ping_id))
+                    .unwrap();
+                break;
+            }
+            _ => {}
+        }
+    }
+}
+
+fn spawn_fake_server() -> FakeServer {
+    let socket = UdpSocket::bind(loopback(0)).expect("fake UDP server");
+    socket
+        .set_read_timeout(Some(Duration::from_secs(2)))
+        .unwrap();
+    let endpoint = socket.local_addr().unwrap();
+    let (command_sender, command_receiver) = channel();
+    let (report_sender, report_receiver) = channel();
+    let handle = thread::spawn(move || {
+        let mut buffer = [0_u8; 4096];
+        let (length, client_endpoint) = socket.recv_from(&mut buffer).expect("UseCircuitCode");
+        assert_eq!(
+            packet_type(&buffer[..length]),
+            Some(PacketType::UseCircuitCode)
+        );
+        let mut position = 0;
+        let use_circuit =
+            UseCircuitCodePacket::new_with_bytes_int32(buffer[..length].to_vec(), &mut position)
+                .expect("decode UseCircuitCode");
+        report_sender
+            .send(ServerReport::Circuit(use_circuit.circuit_code.code))
+            .unwrap();
+        let sequence = u32::from_be_bytes(buffer[1..5].try_into().unwrap());
+        socket.send_to(&ack(sequence), client_endpoint).unwrap();
+
+        let mut handshake = RegionHandshakePacket::new_with_constructor().unwrap();
+        handshake.region_info.sim_name = b"Fake Region".to_vec();
+        let mut bytes = handshake.to_bytes_with_method().unwrap();
+        bytes[0] &= !(Helpers::MSG_RELIABLE | Helpers::MSG_ZEROCODED);
+        socket.send_to(&bytes, client_endpoint).unwrap();
+
+        loop {
+            let (length, _) = socket.recv_from(&mut buffer).expect("handshake reply");
+            let received_type = packet_type(&buffer[..length]);
+            if received_type == Some(PacketType::RegionHandshakeReply) {
+                let mut position = 0;
+                let mut packet_end = i32::try_from(length).unwrap() - 1;
+                let mut zero_buffer = vec![0_u8; 8192];
+                let mut response = RegionHandshakeReplyPacket::new_with_constructor().unwrap();
+                response
+                    .from_bytes_with_bytes_int32_int32_bytes(
+                        buffer[..length].to_vec(),
+                        &mut position,
+                        &mut packet_end,
+                        Some(&mut zero_buffer),
+                    )
+                    .unwrap();
+                report_sender
+                    .send(ServerReport::HandshakeReply(response.region_info.flags))
+                    .unwrap();
+                break;
+            }
+        }
+
+        while let Ok(command) = command_receiver.recv() {
+            match command {
+                ServerCommand::Ping => {
+                    serve_ping_command(&socket, client_endpoint, &report_sender, &mut buffer);
+                }
+                ServerCommand::Disable => {
+                    let disable = DisableSimulatorPacket::new_with_constructor().unwrap();
+                    let mut bytes = disable.to_bytes_with_method().unwrap();
+                    bytes[0] &= !(Helpers::MSG_RELIABLE | Helpers::MSG_ZEROCODED);
+                    socket.send_to(&bytes, client_endpoint).unwrap();
+                }
+                ServerCommand::Stop => break,
+            }
+        }
+    });
+    FakeServer {
+        endpoint,
+        commands: command_sender,
+        reports: report_receiver,
+        handle: Some(handle),
+    }
+}
+
+fn spawn_ack_only_server() -> (SocketAddr, JoinHandle<()>) {
+    let socket = UdpSocket::bind(loopback(0)).expect("ACK-only UDP server");
+    socket
+        .set_read_timeout(Some(Duration::from_secs(2)))
+        .unwrap();
+    let endpoint = socket.local_addr().unwrap();
+    let handle = thread::spawn(move || {
+        let mut buffer = [0_u8; 4096];
+        let (length, client_endpoint) = socket.recv_from(&mut buffer).expect("UseCircuitCode");
+        assert_eq!(
+            packet_type(&buffer[..length]),
+            Some(PacketType::UseCircuitCode)
+        );
+        let sequence = u32::from_be_bytes(buffer[1..5].try_into().unwrap());
+        socket.send_to(&ack(sequence), client_endpoint).unwrap();
+    });
+    (endpoint, handle)
+}
+
+fn assert_ping_exchange(server: &FakeServer, packet_receiver: &Receiver<PacketType>) {
+    server.commands.send(ServerCommand::Ping).unwrap();
+    assert_eq!(
+        packet_receiver
+            .recv_timeout(Duration::from_secs(1))
+            .unwrap(),
+        PacketType::StartPingCheck
+    );
+    assert_eq!(
+        server.reports.recv_timeout(Duration::from_secs(1)).unwrap(),
+        ServerReport::Ack(99)
+    );
+    assert_eq!(
+        server.reports.recv_timeout(Duration::from_secs(1)).unwrap(),
+        ServerReport::Ping(42)
+    );
+}
+
+fn assert_reuse_preserves_connected_override(
+    manager: &mut NetworkManager,
+    server: &FakeServer,
+    simulator: &Simulator,
+) {
+    manager.set_connected(false);
+    let reused = manager
+        .connect_with_ip_end_point_u_int64_boolean_uri_u_int32_u_int32(
+            server.endpoint,
+            simulator.handle,
+            false,
+            None,
+            simulator.size_x,
+            simulator.size_y,
+        )
+        .unwrap()
+        .unwrap();
+    assert_eq!(reused, simulator.clone());
+    assert!(
+        !manager.connected(),
+        "C# does not alter manager Connected when reusing a connected simulator"
+    );
+    manager.set_connected(true);
+}
+
+#[test]
+fn fake_server_drives_circuit_handshake_ping_disable_and_disconnect_reasons() {
+    let server = spawn_fake_server();
+    let client = GridClient::new().expect("client");
+    let mut manager = NetworkManager::new(client).expect("manager");
+    manager.set_circuit_code(0x1122_3344);
+    let (changed_sender, changed_receiver) = channel();
+    let (connected_sender, connected_receiver) = channel();
+    let (packet_sender, packet_receiver) = channel();
+    let (sim_disconnected_sender, sim_disconnected_receiver) = channel();
+    let (disconnected_sender, disconnected_receiver) = channel();
+    let guards = [
+        manager.subscribe_sim_changed(Arc::new(move |args| {
+            changed_sender
+                .send(args.previous_simulator().is_none())
+                .unwrap();
+        })),
+        manager.subscribe_sim_connected(Arc::new(move |args| {
+            connected_sender.send(args.simulator().handle).unwrap();
+        })),
+        manager.subscribe_sim_disconnected(Arc::new(move |args| {
+            sim_disconnected_sender
+                .send((args.reason(), args.was_connected()))
+                .unwrap();
+        })),
+        manager.subscribe_disconnected(Arc::new(move |args| {
+            disconnected_sender.send(args.reason()).unwrap();
+        })),
+    ];
+    manager
+        .register_callback_with_packet_type_event_handler_boolean(
+            PacketType::StartPingCheck,
+            Arc::new(move |args| packet_sender.send(args.packet().type_).unwrap()),
+            false,
+        )
+        .unwrap();
+
+    let seed = Uri("http://caps.invalid/seed".into());
+    let sim = manager
+        .connect_with_ip_end_point_u_int64_boolean_uri_u_int32_u_int32(
+            server.endpoint,
+            0x1234_0000,
+            true,
+            Some(seed.clone()),
+            512,
+            256,
+        )
+        .unwrap()
+        .expect("connected simulator");
+
+    assert_eq!(
+        server.reports.recv_timeout(Duration::from_secs(1)).unwrap(),
+        ServerReport::Circuit(0x1122_3344)
+    );
+    assert!(
+        changed_receiver
+            .recv_timeout(Duration::from_secs(1))
+            .unwrap()
+    );
+    assert_eq!(
+        connected_receiver
+            .recv_timeout(Duration::from_secs(1))
+            .unwrap(),
+        sim.handle
+    );
+    assert_eq!(manager.current_sim(), Some(sim.clone()));
+    assert_eq!(sim.seed_capability(), Some(seed));
+    assert_eq!(sim.size_x, 512);
+    assert_eq!(sim.size_y, 256);
+    wait_until(|| sim.handshake_complete());
+    assert_eq!(
+        server.reports.recv_timeout(Duration::from_secs(1)).unwrap(),
+        ServerReport::HandshakeReply(0x1 | 0x2 | 0x4)
+    );
+
+    assert_reuse_preserves_connected_override(&mut manager, &server, &sim);
+
+    assert_ping_exchange(&server, &packet_receiver);
+
+    server.commands.send(ServerCommand::Disable).unwrap();
+    assert_eq!(
+        sim_disconnected_receiver
+            .recv_timeout(Duration::from_secs(1))
+            .unwrap(),
+        (NetworkManagerDisconnectType::NetworkTimeout, true)
+    );
+    assert_eq!(
+        disconnected_receiver
+            .recv_timeout(Duration::from_secs(1))
+            .unwrap(),
+        NetworkManagerDisconnectType::SimShutdown
+    );
+    assert!(manager.simulators.is_empty());
+    assert!(manager.current_sim().is_none());
+    assert!(!manager.connected());
+    drop(guards);
+}
+
+#[test]
+fn concurrent_disconnect_emits_each_transition_once() {
+    const CALLERS: usize = 12;
+    let server = spawn_fake_server();
+    let client = GridClient::new().expect("client");
+    let mut manager = NetworkManager::new(client).expect("manager");
+    manager.set_circuit_code(7);
+    let sim_disconnects = Arc::new(AtomicUsize::new(0));
+    let global_disconnects = Arc::new(AtomicUsize::new(0));
+    let sim_count = Arc::clone(&sim_disconnects);
+    let sim_guard = manager.subscribe_sim_disconnected(Arc::new(move |args| {
+        assert!(args.was_connected());
+        sim_count.fetch_add(1, Ordering::Relaxed);
+    }));
+    let global_count = Arc::clone(&global_disconnects);
+    let global_guard = manager.subscribe_disconnected(Arc::new(move |_| {
+        global_count.fetch_add(1, Ordering::Relaxed);
+    }));
+    let sim = manager
+        .connect_with_ip_end_point_u_int64_boolean_uri_u_int32_u_int32(
+            server.endpoint,
+            9,
+            true,
+            None,
+            256,
+            256,
+        )
+        .unwrap()
+        .unwrap();
+    let manager = Arc::new(manager);
+    let barrier = Arc::new(Barrier::new(CALLERS));
+
+    thread::scope(|scope| {
+        for _ in 0..CALLERS {
+            let manager = Arc::clone(&manager);
+            let simulator = sim.clone();
+            let barrier = Arc::clone(&barrier);
+            scope.spawn(move || {
+                barrier.wait();
+                manager.disconnect_sim(simulator, false).unwrap();
+            });
+        }
+    });
+
+    assert_eq!(sim_disconnects.load(Ordering::Relaxed), 1);
+    assert_eq!(global_disconnects.load(Ordering::Relaxed), 1);
+    assert!(manager.simulators.is_empty());
+    assert!(!manager.connected());
+    drop(sim_guard);
+    drop(global_guard);
+}
+
+#[test]
+fn sim_disconnected_callback_can_reenter_disconnect_without_deadlock() {
+    let server = spawn_fake_server();
+    let client = GridClient::new().expect("client");
+    let mut manager = NetworkManager::new(client).expect("manager");
+    manager.set_circuit_code(8);
+    let simulator = manager
+        .connect_with_ip_end_point_u_int64_boolean_uri_u_int32_u_int32(
+            server.endpoint,
+            10,
+            true,
+            None,
+            256,
+            256,
+        )
+        .unwrap()
+        .unwrap();
+    let manager = Arc::new(manager);
+    let weak_manager = Arc::downgrade(&manager);
+    let callback_completed = Arc::new(AtomicBool::new(false));
+    let completed = Arc::clone(&callback_completed);
+    let guard = manager.subscribe_sim_disconnected(Arc::new(move |args| {
+        let manager = weak_manager.upgrade().expect("manager during callback");
+        manager
+            .disconnect_sim(args.simulator(), false)
+            .expect("reentrant disconnect");
+        completed.store(true, Ordering::Release);
+    }));
+
+    manager.disconnect_sim(simulator, false).unwrap();
+
+    assert!(callback_completed.load(Ordering::Acquire));
+    assert!(manager.simulators.is_empty());
+    assert!(!manager.connected());
+    drop(guard);
+}
+
+#[test]
+fn keepalive_requires_two_silent_intervals_before_network_timeout() {
+    let server = spawn_fake_server();
+    let client = GridClient::new().expect("client");
+    let mut manager = NetworkManager::new(client).expect("manager");
+    manager.set_circuit_code(11);
+    let (disconnected_sender, disconnected_receiver) = channel();
+    let guard = manager.subscribe_disconnected(Arc::new(move |args| {
+        disconnected_sender.send(args.reason()).unwrap();
+    }));
+    let sim = manager
+        .connect_with_ip_end_point_u_int64_boolean_uri_u_int32_u_int32(
+            server.endpoint,
+            12,
+            true,
+            None,
+            256,
+            256,
+        )
+        .unwrap()
+        .unwrap();
+    wait_until(|| sim.handshake_complete());
+
+    manager.poll_keepalive();
+    assert!(manager.connected());
+    assert!(
+        disconnected_receiver
+            .recv_timeout(Duration::from_millis(50))
+            .is_err()
+    );
+
+    manager.poll_keepalive();
+    assert_eq!(
+        disconnected_receiver
+            .recv_timeout(Duration::from_secs(1))
+            .unwrap(),
+        NetworkManagerDisconnectType::NetworkTimeout
+    );
+    assert!(!manager.connected());
+    assert!(manager.current_sim().is_none());
+    drop(guard);
+}
+
+#[test]
+fn enable_simulator_caps_event_adds_each_new_region_once() {
+    let first_server = spawn_fake_server();
+    let second_server = spawn_fake_server();
+    let mut client = GridClient::new().expect("client");
+    client.settings().agent_settings_mut().multiple_sims = true;
+    let mut manager = NetworkManager::new(client).expect("manager");
+    manager.set_circuit_code(17);
+    let current = manager
+        .connect_with_ip_end_point_u_int64_boolean_uri_u_int32_u_int32(
+            first_server.endpoint,
+            18,
+            true,
+            None,
+            256,
+            256,
+        )
+        .unwrap()
+        .unwrap();
+
+    let mut message = EnableSimulatorMessage::new().unwrap();
+    let mut info = EnableSimulatorMessageSimulatorInfoBlock::new().unwrap();
+    info.ip = second_server.endpoint.ip();
+    info.port = i32::from(second_server.endpoint.port());
+    info.region_handle = 19;
+    info.region_size_x = 1024;
+    info.region_size_y = 512;
+    message.simulators = vec![info];
+
+    manager.dispatch_caps_event("EnableSimulator", &message, current.clone());
+    manager.dispatch_caps_event("EnableSimulator", &message, current);
+
+    assert_eq!(manager.simulators.len(), 2);
+    let added = manager
+        .find_simulator_with_ip_end_point(second_server.endpoint)
+        .unwrap()
+        .expect("enabled simulator");
+    assert_eq!(added.handle, 19);
+    assert_eq!(added.size_x, 1024);
+    assert_eq!(added.size_y, 512);
+    assert_eq!(
+        second_server
+            .reports
+            .recv_timeout(Duration::from_secs(1))
+            .unwrap(),
+        ServerReport::Circuit(17)
+    );
+    assert_eq!(
+        second_server
+            .reports
+            .recv_timeout(Duration::from_secs(1))
+            .unwrap(),
+        ServerReport::HandshakeReply(0x1 | 0x2 | 0x4)
+    );
+
+    manager
+        .shutdown_with_disconnect_type(NetworkManagerDisconnectType::ClientInitiated)
+        .unwrap();
+}
+
+#[test]
+fn async_connect_and_shutdown_do_not_require_an_ambient_tokio_runtime() {
+    let server = spawn_fake_server();
+    let client = GridClient::new().expect("client");
+    let mut manager = NetworkManager::new(client).expect("manager");
+    manager.set_circuit_code(23);
+
+    let simulator = block_on(
+        manager.connect_with_ip_end_point_u_int64_boolean_uri_u_int32_u_int32_20e51cc0(
+            server.endpoint,
+            24,
+            true,
+            None,
+            256,
+            256,
+        ),
+    )
+    .unwrap()
+    .expect("async connection");
+
+    assert!(simulator.connected());
+    assert_eq!(manager.current_sim(), Some(simulator));
+    block_on(manager.shutdown_with_disconnect_type_string_7641243c(
+        NetworkManagerDisconnectType::ClientInitiated,
+        "ClientInitiated".into(),
+    ))
+    .unwrap();
+    assert!(!manager.connected());
+}
+
+#[test]
+fn handshake_timeout_preserves_reference_connect_result_and_shutdown_tears_down_current_sim() {
+    let (endpoint, server) = spawn_ack_only_server();
+    let mut client = GridClient::new().expect("client");
+    client.settings().timing().login_timeout = 25;
+    let mut manager = NetworkManager::new(client).expect("manager");
+    manager.set_circuit_code(25);
+    let sim_disconnects = Arc::new(AtomicUsize::new(0));
+    let disconnect_count = Arc::clone(&sim_disconnects);
+    let guard = manager.subscribe_sim_disconnected(Arc::new(move |_| {
+        disconnect_count.fetch_add(1, Ordering::Relaxed);
+    }));
+
+    let simulator = manager
+        .connect_with_ip_end_point_u_int64_boolean_uri_u_int32_u_int32(
+            endpoint, 26, true, None, 256, 256,
+        )
+        .unwrap()
+        .expect("C# returns the UDP-connected simulator after handshake timeout");
+    server.join().unwrap();
+
+    assert!(!simulator.handshake_complete());
+    assert!(manager.simulators.is_empty());
+    assert_eq!(manager.current_sim(), Some(simulator.clone()));
+    manager
+        .shutdown_with_disconnect_type(NetworkManagerDisconnectType::ClientInitiated)
+        .unwrap();
+    assert!(!simulator.connected());
+    assert_eq!(sim_disconnects.load(Ordering::Relaxed), 1);
+    drop(guard);
+}