Implement native UDP transport reliability (#52)

crates/libremetaverse/tests/udp_transport.rs

@@ -0,0 +1,511 @@
+use libremetaverse::packets::{
+    AgentThrottlePacket, CompletePingCheckPacket, Packet, PacketAckPacket,
+    PacketAckPacketPacketsBlock, PacketType, UseCircuitCodePacket,
+};
+use libremetaverse::{
+    AgentThrottle, AgentThrottleSender, GridClient, Helpers, IncomingPacketIDCollection, Simulator,
+    UDPBase, UDPPacketBuffer, UdpPacketHandler, UdpTransportConfig, UdpTransportError,
+};
+use libremetaverse_types::Error;
+use libremetaverse_types::compat::{Array, CancellationToken, Object};
+use std::net::{IpAddr, Ipv4Addr, SocketAddr};
+use std::sync::{Arc, Mutex};
+use std::time::Duration;
+use tokio::net::UdpSocket;
+
+#[derive(Default)]
+struct RecordingHandler {
+    received: Mutex<Vec<Vec<u8>>>,
+    sent: Mutex<Vec<Vec<u8>>>,
+    drops: Mutex<usize>,
+}
+
+impl UdpPacketHandler for RecordingHandler {
+    fn packet_received(&self, buffer: UDPPacketBuffer) {
+        self.received
+            .lock()
+            .unwrap_or_else(std::sync::PoisonError::into_inner)
+            .push(buffer.data[..usize::try_from(buffer.data_length).unwrap()].to_vec());
+    }
+
+    fn packet_sent(&self, buffer: UDPPacketBuffer, bytes_sent: usize) {
+        self.sent
+            .lock()
+            .unwrap_or_else(std::sync::PoisonError::into_inner)
+            .push(buffer.data[..bytes_sent].to_vec());
+    }
+
+    fn packet_dropped(&self) {
+        *self
+            .drops
+            .lock()
+            .unwrap_or_else(std::sync::PoisonError::into_inner) += 1;
+    }
+}
+
+fn loopback() -> SocketAddr {
+    SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0)
+}
+
+fn reliable_packet() -> Vec<u8> {
+    let mut packet = UseCircuitCodePacket::new_with_constructor().expect("packet constructor");
+    packet.circuit_code.code = 0x1122_3344;
+    packet.to_bytes_with_method().expect("packet bytes")
+}
+
+fn packet_ack(sequence: u32) -> Vec<u8> {
+    let mut packet = PacketAckPacket::new_with_constructor().expect("ACK constructor");
+    packet.packets = vec![PacketAckPacketPacketsBlock { id: sequence }];
+    let mut bytes = packet.to_bytes_with_method().expect("ACK bytes");
+    bytes[0] &= !Helpers::MSG_RELIABLE;
+    bytes
+}
+
+fn complete_ping_packet() -> Vec<u8> {
+    let mut packet = CompletePingCheckPacket::new_with_constructor().expect("ping constructor");
+    packet.ping_id.ping_id = 7;
+    packet.to_bytes_with_method().expect("ping bytes")
+}
+
+async fn wait_until(mut condition: impl FnMut() -> bool) {
+    tokio::time::timeout(Duration::from_secs(2), async {
+        while !condition() {
+            tokio::task::yield_now().await;
+            tokio::time::sleep(Duration::from_millis(1)).await;
+        }
+    })
+    .await
+    .expect("condition timed out");
+}
+
+#[test]
+fn packet_buffer_constructors_and_copy_match_the_reference() {
+    let endpoint = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 13000);
+    let mut packet = UDPPacketBuffer::new_with_ip_end_point_int32(endpoint, 4).unwrap();
+    assert_eq!(packet.data, vec![0; 4]);
+    assert_eq!(packet.data_length, 0);
+
+    packet
+        .copy_from_with_array(Array(vec![Object::Bytes(vec![1, 2, 3, 4])]))
+        .unwrap();
+    assert_eq!(packet.data, vec![1, 2, 3, 4]);
+    assert_eq!(packet.data_length, 0);
+    assert_eq!(
+        packet.copy_from_slice(&[1, 2, 3, 4, 5]),
+        Err(Error::IndexOutOfRange)
+    );
+    packet.reset_endpoint().unwrap();
+    assert_eq!(packet.remote_end_point, "0.0.0.0:0".parse().unwrap());
+
+    let borrowed = vec![9, 8, 7];
+    let packet = UDPPacketBuffer::new_with_ip_end_point_bytes(endpoint, borrowed).unwrap();
+    assert_eq!(packet.data, vec![9, 8, 7]);
+    assert_eq!(packet.data_length, 0);
+}
+
+#[test]
+fn incoming_packet_archive_is_bounded_and_rejects_duplicates() {
+    let archive = IncomingPacketIDCollection::new(3).unwrap();
+    assert!(archive.try_enqueue(1));
+    assert!(!archive.try_enqueue(1));
+    assert!(archive.try_enqueue(2));
+    // The C# ring keeps one slot empty, so adding the third item evicts 1.
+    assert!(archive.try_enqueue(3));
+    assert!(archive.try_enqueue(1));
+    assert_eq!(
+        IncomingPacketIDCollection::new(0).unwrap_err(),
+        Error::Argument
+    );
+}
+
+#[test]
+fn agent_throttle_clamps_and_round_trips_golden_little_endian_values() {
+    let throttle = AgentThrottle::default();
+    assert_eq!(throttle.resend(), 150_000.0);
+    assert_eq!(throttle.land(), 170_000.0);
+    assert_eq!(throttle.wind(), 34_000.0);
+    assert_eq!(throttle.cloud(), 34_000.0);
+    assert_eq!(throttle.task(), 360_448.0);
+    assert_eq!(throttle.texture(), 360_448.0);
+    assert_eq!(throttle.asset(), 220_000.0);
+    assert_eq!(throttle.total(), 1_328_896.0);
+
+    let bytes = throttle.to_bytes().unwrap();
+    assert_eq!(bytes.len(), 28);
+    let decoded = AgentThrottle::new_with_bytes_int32(bytes, 0).unwrap();
+    assert_eq!(decoded.to_bytes().unwrap(), throttle.to_bytes().unwrap());
+
+    let mut limits = AgentThrottle::default();
+    limits.set_total(-1.0);
+    assert_eq!(limits.resend(), 10_000.0);
+    assert_eq!(limits.task(), 4_000.0);
+    assert_eq!(limits.land(), 0.0);
+}
+
+#[derive(Default)]
+struct RecordingThrottleSender(Mutex<Vec<Vec<u8>>>);
+
+impl AgentThrottleSender for RecordingThrottleSender {
+    fn send_throttle(
+        &self,
+        throttle_bytes: &[u8],
+        _simulator: Option<&Simulator>,
+    ) -> Result<(), Error> {
+        self.0.lock().unwrap().push(throttle_bytes.to_vec());
+        Ok(())
+    }
+}
+
+#[test]
+fn mapped_agent_throttle_set_uses_the_client_network_binding() {
+    let sender = Arc::new(RecordingThrottleSender::default());
+    let mut client = GridClient::new().unwrap();
+    client.set_agent_throttle_sender(sender.clone()).unwrap();
+    let throttle = AgentThrottle::new_with_grid_client(client).unwrap();
+    throttle.set_with_method().unwrap();
+    assert_eq!(
+        sender.0.lock().unwrap().as_slice(),
+        &[throttle.to_bytes().unwrap()]
+    );
+}
+
+#[test]
+fn construction_is_runtime_neutral_and_start_requires_an_injected_runtime() {
+    let transport = UDPBase::client_with_defaults("127.0.0.1:13000".parse().unwrap()).unwrap();
+    assert!(!transport.is_running());
+    assert_eq!(transport.start(), Err(Error::InvalidOperation));
+    assert!(!transport.is_running());
+}
+
+#[tokio::test(flavor = "current_thread")]
+async fn bounded_command_queue_reports_backpressure_without_unbounded_buffering() {
+    let server = UdpSocket::bind(loopback()).await.unwrap();
+    let config = UdpTransportConfig {
+        command_queue_capacity: 1,
+        ..UdpTransportConfig::default()
+    };
+    let transport = UDPBase::client(
+        server.local_addr().unwrap(),
+        config,
+        Arc::new(RecordingHandler::default()),
+        CancellationToken::default(),
+    )
+    .unwrap();
+    transport.start().unwrap();
+
+    // A current-thread runtime does not poll spawned tasks until this test
+    // yields, making the capacity-one boundary deterministic.
+    transport
+        .try_send_packet(
+            complete_ping_packet(),
+            server.local_addr().unwrap(),
+            PacketType::CompletePingCheck,
+            false,
+        )
+        .unwrap();
+    assert!(matches!(
+        transport.try_send_packet(
+            complete_ping_packet(),
+            server.local_addr().unwrap(),
+            PacketType::CompletePingCheck,
+            false,
+        ),
+        Err(UdpTransportError::Backpressure)
+    ));
+    assert_eq!(transport.stats().dropped_send_queue, 1);
+    transport.stop_async().await;
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn local_socket_send_assigns_sequence_and_ack_stops_resends() {
+    let server = UdpSocket::bind(loopback()).await.unwrap();
+    let server_address = server.local_addr().unwrap();
+    let handler = Arc::new(RecordingHandler::default());
+    let config = UdpTransportConfig {
+        resend_timeout: Duration::from_millis(60),
+        network_tick_interval: Duration::from_millis(10),
+        ..UdpTransportConfig::default()
+    };
+    let transport = UDPBase::client(
+        server_address,
+        config,
+        handler.clone(),
+        CancellationToken::default(),
+    )
+    .unwrap();
+    transport.start().unwrap();
+
+    let sequence = transport
+        .send_packet(
+            reliable_packet(),
+            PacketType::UseCircuitCode,
+            false,
+            CancellationToken::default(),
+        )
+        .await
+        .unwrap();
+    assert_eq!(sequence, 1);
+    let mut receive = [0_u8; 4096];
+    let (length, client_address) =
+        tokio::time::timeout(Duration::from_secs(2), server.recv_from(&mut receive))
+            .await
+            .unwrap()
+            .unwrap();
+    assert_eq!(u32::from_be_bytes(receive[1..5].try_into().unwrap()), 1);
+    assert_ne!(receive[0] & Helpers::MSG_RELIABLE, 0);
+    server
+        .send_to(&packet_ack(sequence), client_address)
+        .await
+        .unwrap();
+    wait_until(|| transport.stats().acknowledgements_received == 1).await;
+    tokio::time::sleep(Duration::from_millis(100)).await;
+    assert_eq!(transport.stats().resent_datagrams, 0);
+    assert_eq!(handler.sent.lock().unwrap().len(), 1);
+    assert_eq!(length, handler.sent.lock().unwrap()[0].len());
+
+    transport.stop_async().await;
+    assert!(!transport.is_running());
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn reliable_packets_resend_with_same_sequence_then_expire() {
+    let server = UdpSocket::bind(loopback()).await.unwrap();
+    let config = UdpTransportConfig {
+        resend_timeout: Duration::from_millis(20),
+        network_tick_interval: Duration::from_millis(5),
+        max_resend_count: 2,
+        ..UdpTransportConfig::default()
+    };
+    let transport = UDPBase::client(
+        server.local_addr().unwrap(),
+        config,
+        Arc::new(RecordingHandler::default()),
+        CancellationToken::default(),
+    )
+    .unwrap();
+    transport.start().unwrap();
+    transport
+        .send_packet(
+            reliable_packet(),
+            PacketType::UseCircuitCode,
+            false,
+            CancellationToken::default(),
+        )
+        .await
+        .unwrap();
+
+    let mut datagrams = Vec::new();
+    let mut receive = [0_u8; 4096];
+    for _ in 0..3 {
+        let (length, _) =
+            tokio::time::timeout(Duration::from_secs(1), server.recv_from(&mut receive))
+                .await
+                .unwrap()
+                .unwrap();
+        datagrams.push(receive[..length].to_vec());
+    }
+    assert_eq!(
+        datagrams
+            .iter()
+            .map(|bytes| u32::from_be_bytes(bytes[1..5].try_into().unwrap()))
+            .collect::<Vec<_>>(),
+        vec![1, 1, 1]
+    );
+    assert_eq!(datagrams[0][0] & Helpers::MSG_RESENT, 0);
+    assert_ne!(datagrams[1][0] & Helpers::MSG_RESENT, 0);
+    wait_until(|| transport.stats().failed_resends == 1).await;
+    assert_eq!(transport.stats().resent_datagrams, 2);
+    transport.stop_async().await;
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn incoming_reliable_packets_are_acked_and_duplicates_are_not_dispatched() {
+    let server = UdpSocket::bind(loopback()).await.unwrap();
+    let handler = Arc::new(RecordingHandler::default());
+    let config = UdpTransportConfig {
+        max_pending_acks: 2,
+        network_tick_interval: Duration::from_secs(1),
+        ..UdpTransportConfig::default()
+    };
+    let transport = UDPBase::client(
+        server.local_addr().unwrap(),
+        config,
+        handler.clone(),
+        CancellationToken::default(),
+    )
+    .unwrap();
+    transport.start().unwrap();
+    let client_address = transport.local_address().unwrap();
+
+    let mut first = reliable_packet();
+    first[1..5].copy_from_slice(&10_u32.to_be_bytes());
+    let mut second = reliable_packet();
+    second[1..5].copy_from_slice(&12_u32.to_be_bytes());
+    let mut late = reliable_packet();
+    late[1..5].copy_from_slice(&11_u32.to_be_bytes());
+    server.send_to(&first, client_address).await.unwrap();
+    server.send_to(&first, client_address).await.unwrap();
+    server.send_to(&second, client_address).await.unwrap();
+    server.send_to(&late, client_address).await.unwrap();
+
+    let mut receive = [0_u8; 4096];
+    let mut ack_groups = Vec::new();
+    for _ in 0..2 {
+        let (length, _) =
+            tokio::time::timeout(Duration::from_secs(2), server.recv_from(&mut receive))
+                .await
+                .unwrap()
+                .unwrap();
+        let mut position = 0;
+        let ack = PacketAckPacket::new_with_bytes_int32(receive[..length].to_vec(), &mut position)
+            .unwrap();
+        ack_groups.push(ack.packets.iter().map(|block| block.id).collect::<Vec<_>>());
+    }
+    assert_eq!(ack_groups, vec![vec![10, 10], vec![12, 11]]);
+    wait_until(|| handler.received.lock().unwrap().len() == 3).await;
+    assert_eq!(transport.stats().duplicate_datagrams, 1);
+    assert_eq!(transport.stats().sequence_gaps, 1);
+    assert_eq!(transport.stats().out_of_order_datagrams, 1);
+    assert_eq!(transport.stats().acknowledgements_sent, 4);
+    transport.stop_async().await;
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn pending_ack_is_piggybacked_before_the_mtu_boundary() {
+    let server = UdpSocket::bind(loopback()).await.unwrap();
+    let handler = Arc::new(RecordingHandler::default());
+    let config = UdpTransportConfig {
+        network_tick_interval: Duration::from_secs(10),
+        ..UdpTransportConfig::default()
+    };
+    let transport = UDPBase::client(
+        server.local_addr().unwrap(),
+        config,
+        handler.clone(),
+        CancellationToken::default(),
+    )
+    .unwrap();
+    transport.start().unwrap();
+
+    let mut reliable = reliable_packet();
+    reliable[1..5].copy_from_slice(&20_u32.to_be_bytes());
+    server
+        .send_to(&reliable, transport.local_address().unwrap())
+        .await
+        .unwrap();
+    wait_until(|| handler.received.lock().unwrap().len() == 1).await;
+
+    transport
+        .send_packet(
+            complete_ping_packet(),
+            PacketType::CompletePingCheck,
+            false,
+            CancellationToken::default(),
+        )
+        .await
+        .unwrap();
+    let mut receive = [0_u8; 4096];
+    let (length, _) = server.recv_from(&mut receive).await.unwrap();
+    assert_ne!(receive[0] & Helpers::MSG_APPENDED_ACKS, 0);
+    let mut end = i32::try_from(length).unwrap() - 1;
+    let decoded = Packet::build_packet_with_bytes_int32_bytes(
+        receive[..length].to_vec(),
+        &mut end,
+        vec![0; 8192],
+    )
+    .unwrap();
+    assert_eq!(decoded.type_, PacketType::CompletePingCheck);
+    assert_eq!(decoded.header.ack_list, Some(vec![20]));
+    transport.stop_async().await;
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn zerocoding_mtu_malformed_input_and_cancellation_are_bounded() {
+    let server = UdpSocket::bind(loopback()).await.unwrap();
+    let handler = Arc::new(RecordingHandler::default());
+    let transport = UDPBase::client(
+        server.local_addr().unwrap(),
+        UdpTransportConfig::default(),
+        handler.clone(),
+        CancellationToken::default(),
+    )
+    .unwrap();
+    transport.start().unwrap();
+    let client_address = transport.local_address().unwrap();
+
+    let mut packet = AgentThrottlePacket::new_with_constructor().unwrap();
+    packet.throttle.throttles = vec![0; 28];
+    let raw = packet.to_bytes_with_method().unwrap();
+    transport
+        .send_packet(
+            raw.clone(),
+            PacketType::AgentThrottle,
+            true,
+            CancellationToken::default(),
+        )
+        .await
+        .unwrap();
+    let mut receive = [0_u8; 4096];
+    let (length, _) = server.recv_from(&mut receive).await.unwrap();
+    assert_ne!(receive[0] & Helpers::MSG_ZEROCODED, 0);
+    assert!(length < raw.len());
+    let mut end = i32::try_from(length).unwrap() - 1;
+    Packet::build_packet_with_bytes_int32_bytes(
+        receive[..length].to_vec(),
+        &mut end,
+        vec![0; 8192],
+    )
+    .unwrap();
+
+    assert_eq!(
+        transport
+            .send_packet(
+                vec![0; 1201],
+                PacketType::AgentThrottle,
+                false,
+                CancellationToken::default(),
+            )
+            .await,
+        Err(UdpTransportError::MtuExceeded)
+    );
+
+    for malformed in [vec![], vec![0x40], vec![0x40, 0, 0, 0, 1, 0, 0xfe]] {
+        server.send_to(&malformed, client_address).await.unwrap();
+    }
+    wait_until(|| transport.stats().malformed_datagrams >= 3).await;
+    assert!(handler.received.lock().unwrap().is_empty());
+
+    transport.stop_async().await;
+    assert!(!transport.is_running());
+    assert_eq!(
+        transport
+            .send_packet(
+                reliable_packet(),
+                PacketType::UseCircuitCode,
+                false,
+                CancellationToken::default(),
+            )
+            .await,
+        Err(UdpTransportError::NotRunning)
+    );
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn dropping_the_last_transport_handle_releases_its_socket_tasks() {
+    let server = UdpSocket::bind(loopback()).await.unwrap();
+    let transport = UDPBase::client_with_defaults(server.local_addr().unwrap()).unwrap();
+    transport.start().unwrap();
+    let local_address = transport.local_address().unwrap();
+    drop(transport);
+
+    tokio::time::timeout(Duration::from_secs(2), async {
+        loop {
+            match UdpSocket::bind(local_address).await {
+                Ok(rebound) => break rebound,
+                Err(_) => tokio::time::sleep(Duration::from_millis(1)).await,
+            }
+        }
+    })
+    .await
+    .expect("transport tasks retained the UDP socket after drop");
+}