From f6f5abe4643b9eeb2a0034c297ffac5a303b98e8 Mon Sep 17 00:00:00 2001 From: Chili Palmer Date: Sat, 22 Aug 2026 13:45:31 +0200 Subject: [PATCH] Fix extended-region scene rendering and delivery --- AGENTS.md | 1 + Cargo.lock | 9 +- Cargo.toml | 2 + crates/libremetaverse/src/network_manager.rs | 119 +- crates/libremetaverse/src/terrain_manager.rs | 24 +- .../libremetaverse/tests/network_manager.rs | 80 +- crates/metacrate-grid-agent/Cargo.toml | 4 +- crates/metacrate-grid-agent/src/backend.rs | 79 +- crates/metacrate-grid-agent/src/vision.rs | 1013 +++++++++-------- .../metacrate-grid-agent/src/vision_tests.rs | 97 +- .../tests/dependency_policy.rs | 7 +- crates/metacrate-rendering-wgpu/Cargo.toml | 19 + crates/metacrate-rendering-wgpu/src/lib.rs | 426 +++++++ programs/src/test_client.rs | 2 +- 14 files changed, 1349 insertions(+), 533 deletions(-) create mode 100644 crates/metacrate-rendering-wgpu/Cargo.toml create mode 100644 crates/metacrate-rendering-wgpu/src/lib.rs diff --git a/AGENTS.md b/AGENTS.md index 62832d6..dc08e0e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,2 +1,3 @@ All code and API mappings must be cross-platform; do not use macOS-only APIs unless equivalent Linux and Windows implementations are provided. Gitea Actions workflows must target only `ubuntu-latest`; do not add macOS or Windows Gitea jobs. +Crates named `libremetaverse-*` are restricted to compatibility mappings of the C# LibreMetaverse libraries. New MetaCrate-native functionality must use a `metacrate-*` crate name; the reusable wgpu renderer is `metacrate-rendering-wgpu`. diff --git a/Cargo.lock b/Cargo.lock index f0512fb..8dccba6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2535,6 +2535,7 @@ dependencies = [ "libremetaverse-types", "mentra", "metacrate-lsl-tools", + "metacrate-rendering-wgpu", "rustls", "serde", "serde_json", @@ -2544,7 +2545,6 @@ dependencies = [ "tokio-rustls", "unicode-width 0.2.2", "url", - "wgpu", ] [[package]] @@ -2572,6 +2572,13 @@ dependencies = [ "stats_alloc", ] +[[package]] +name = "metacrate-rendering-wgpu" +version = "0.0.1" +dependencies = [ + "wgpu", +] + [[package]] name = "minimal-lexical" version = "0.2.1" diff --git a/Cargo.toml b/Cargo.toml index 6ad944a..d3b9092 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,6 +11,7 @@ members = [ "crates/libremetaverse-rendering-simple", "crates/libremetaverse-rendering-mesh-foundry", "crates/metacrate-lsl-tools", + "crates/metacrate-rendering-wgpu", "crates/libremetaverse-rlv", "crates/libremetaverse-utilities", "crates/libremetaverse-voice-vivox", @@ -36,6 +37,7 @@ default-members = [ "crates/libremetaverse-rendering-simple", "crates/libremetaverse-rendering-mesh-foundry", "crates/metacrate-lsl-tools", + "crates/metacrate-rendering-wgpu", "crates/libremetaverse-rlv", "crates/libremetaverse-utilities", "crates/libremetaverse-voice-vivox", diff --git a/crates/libremetaverse/src/network_manager.rs b/crates/libremetaverse/src/network_manager.rs index 2b2185c..55004e6 100644 --- a/crates/libremetaverse/src/network_manager.rs +++ b/crates/libremetaverse/src/network_manager.rs @@ -14,12 +14,15 @@ use crate::interfaces::IMessage; use crate::packets::{ - AgentPausePacket, AgentResumePacket, CloseCircuitPacket, CompleteAgentMovementPacket, - CompletePingCheckPacket, GenericStreamingMessagePacket, KickUserPacket, LogoutReplyPacket, - LogoutRequestPacket, Packet, PacketType, RegionHandshakePacket, RegionHandshakeReplyPacket, - SimStatsPacket, StartPingCheckPacket, UseCircuitCodePacket, + AgentPausePacket, AgentResumePacket, AgentThrottlePacket, CloseCircuitPacket, + CompleteAgentMovementPacket, CompletePingCheckPacket, GenericStreamingMessagePacket, + KickUserPacket, LogoutReplyPacket, LogoutRequestPacket, Packet, PacketType, + RegionHandshakePacket, RegionHandshakeReplyPacket, SimStatsPacket, StartPingCheckPacket, + UseCircuitCodePacket, +}; +use crate::udp_transport::{ + AgentThrottleSender, UDPBase, UDPPacketBuffer, UdpPacketHandler, UdpTransportConfig, }; -use crate::udp_transport::{UDPBase, UDPPacketBuffer, UdpPacketHandler, UdpTransportConfig}; use crate::{ AccountLevelBenefits, Avatar, Caps, Error, GenericStreamingMethod, GridClient, Helpers, LoginCredential, LoginParams, LoginProgressEventArgs, LoginResponseData, LoginStatus, @@ -1509,17 +1512,10 @@ impl Simulator { patch: TerrainPatch, large_region: bool, ) -> Result<(), Error> { - let (width, height) = if large_region { - ( - usize::try_from(self.size_x.max(16) / 16).map_err(|_| Error::Argument)?, - usize::try_from(self.size_y.max(16) / 16).map_err(|_| Error::Argument)?, - ) - } else { - (16, 16) - }; + let (width, height) = if large_region { (256, 256) } else { (16, 16) }; let x = usize::try_from(patch.x).map_err(|_| Error::Argument)?; let y = usize::try_from(patch.y).map_err(|_| Error::Argument)?; - if x >= width || y >= height || width > 256 || height > 256 { + if x >= width || y >= height { return Err(Error::Argument); } let index = y @@ -1528,6 +1524,23 @@ impl Simulator { .ok_or(Error::Argument)?; let patch_count = width.checked_mul(height).ok_or(Error::Argument)?; let mut terrain = write(&self.terrain); + if !terrain.is_empty() && terrain.len() != patch_count { + let previous = std::mem::take(&mut *terrain); + terrain.resize_with(patch_count, || TerrainPatch { + data: Vec::new(), + x: 0, + y: 0, + }); + for old in previous.into_iter().filter(|old| !old.data.is_empty()) { + let (Ok(old_x), Ok(old_y)) = (usize::try_from(old.x), usize::try_from(old.y)) + else { + continue; + }; + if old_x < width && old_y < height { + terrain[old_y * width + old_x] = old; + } + } + } terrain.resize_with(patch_count, || TerrainPatch { data: Vec::new(), x: 0, @@ -1622,14 +1635,21 @@ impl Simulator { let (Ok(x), Ok(y)) = (u32::try_from(x), u32::try_from(y)) else { return Ok(false); }; - if x >= self.size_x || y >= self.size_y { + let terrain = read(&self.terrain); + let per_edge = if terrain.len() > 256 { + 256 + } else { + usize::try_from(self.size_x.max(16) / 16).map_err(|_| Error::Argument)? + }; + if usize::try_from(x / 16).map_err(|_| Error::Argument)? >= per_edge + || usize::try_from(y / 16).map_err(|_| Error::Argument)? + >= terrain.len().div_ceil(per_edge) + { return Ok(false); } - let per_edge = usize::try_from(self.size_x.max(16) / 16).map_err(|_| Error::Argument)?; let patch_x = usize::try_from(x / 16).map_err(|_| Error::Argument)?; let patch_y = usize::try_from(y / 16).map_err(|_| Error::Argument)?; let point = usize::try_from((y % 16) * 16 + (x % 16)).map_err(|_| Error::Argument)?; - let terrain = read(&self.terrain); let Some(patch) = terrain.get(patch_y * per_edge + patch_x) else { return Ok(false); }; @@ -2927,6 +2947,36 @@ impl fmt::Debug for NetworkManager { } } +struct NetworkAgentThrottleSender { + inner: Weak, +} + +impl AgentThrottleSender for NetworkAgentThrottleSender { + fn send_throttle( + &self, + throttle_bytes: &[u8], + simulator: Option<&Simulator>, + ) -> Result<(), Error> { + if throttle_bytes.len() != 28 { + return Err(Error::Argument); + } + let inner = self.inner.upgrade().ok_or(Error::InvalidOperation)?; + let manager = NetworkManager::native_from_inner(inner); + let simulator = simulator + .cloned() + .or_else(|| manager.native_current_sim()) + .ok_or(Error::InvalidOperation)?; + let mut packet = AgentThrottlePacket::new_with_constructor()?; + packet.agent_data.agent_id = manager.native_agent_id(); + packet.agent_data.session_id = manager.native_session_id(); + packet.agent_data.circuit_code = manager.native_circuit_code(); + packet.throttle.throttles = throttle_bytes.to_vec(); + let data = packet.to_bytes_with_method()?; + let length = i32::try_from(data.len()).map_err(|_| Error::Argument)?; + simulator.native_send_packet_data(data, length, PacketType::AgentThrottle, true) + } +} + impl NetworkManager { pub(crate) fn native_from_inner(inner: Arc) -> Self { Self { @@ -3019,11 +3069,18 @@ impl NetworkManager { } }), )?; - Ok(Self { + let manager = Self { login_response_data: Some(login_response), simulators, inner, - }) + }; + let mut client = manager.inner.client.clone(); + client + .set_agent_throttle_sender(Arc::new(NetworkAgentThrottleSender { + inner: Arc::downgrade(&manager.inner), + })) + .map_err(|_| Error::InvalidOperation)?; + Ok(manager) } pub fn native_subscribe_login_progress( @@ -4324,4 +4381,28 @@ mod tests { ("UpdateAgentLanguage".to_owned(), true, 11) ); } + + #[test] + fn extended_terrain_patch_is_stored_outside_the_legacy_region_extent() { + let client = GridClient::new().expect("client"); + let simulator = + Simulator::new(client, "127.0.0.1:14002".parse().unwrap(), 12, None, None).unwrap(); + simulator + .native_store_terrain_patch( + TerrainPatch { + data: vec![27.0; 256], + x: 42, + y: 33, + }, + true, + ) + .unwrap(); + let mut height = 0.0; + assert!( + simulator + .native_terrain_height_at_point(675, 538, &mut height) + .unwrap() + ); + assert!((height - 27.0).abs() < f32::EPSILON); + } } diff --git a/crates/libremetaverse/src/terrain_manager.rs b/crates/libremetaverse/src/terrain_manager.rs index fb10218..244c633 100644 --- a/crates/libremetaverse/src/terrain_manager.rs +++ b/crates/libremetaverse/src/terrain_manager.rs @@ -428,7 +428,7 @@ mod tests { } #[test] - fn rectangular_large_region_uses_independent_patch_dimensions() { + fn large_region_packets_can_extend_stale_declared_dimensions() { let client = GridClient::new().unwrap(); let sim = Simulator::new( client, @@ -450,16 +450,16 @@ mod tests { let mut height = 0.0; assert!(sim.terrain_height_at_point(511, 255, &mut height).unwrap()); assert_eq!(height, 41.5); - assert!( - sim.native_store_terrain_patch( - TerrainPatch { - data: vec![0.0; 256], - x: 0, - y: 16, - }, - true, - ) - .is_err() - ); + sim.native_store_terrain_patch( + TerrainPatch { + data: vec![42.5; 256], + x: 0, + y: 16, + }, + true, + ) + .unwrap(); + assert!(sim.terrain_height_at_point(0, 256, &mut height).unwrap()); + assert_eq!(height, 42.5); } } diff --git a/crates/libremetaverse/tests/network_manager.rs b/crates/libremetaverse/tests/network_manager.rs index d1eab97..70ca153 100644 --- a/crates/libremetaverse/tests/network_manager.rs +++ b/crates/libremetaverse/tests/network_manager.rs @@ -9,15 +9,16 @@ use libremetaverse::messages::linden::{ EnableSimulatorMessage, EnableSimulatorMessageSimulatorInfoBlock, }; use libremetaverse::packets::{ - CompleteAgentMovementPacket, DisableSimulatorPacket, LogoutReplyPacket, + AgentThrottlePacket, CompleteAgentMovementPacket, DisableSimulatorPacket, LogoutReplyPacket, LogoutReplyPacketInventoryDataBlock, LogoutRequestPacket, Packet, PacketAckPacket, PacketAckPacketPacketsBlock, PacketType, RegionHandshakePacket, RegionHandshakeReplyPacket, SimStatsPacket, SimStatsPacketStatBlock, StartPingCheckPacket, UseCircuitCodePacket, }; use libremetaverse::{ - CapsEventDictionary, CapsEventQueueCallback, GridClient, Helpers, HttpCapsClient, - InventoryItem, LoginState, LoginStatus, NetworkManager, NetworkManagerDisconnectType, - NetworkManagerLoginResponseCallback, PacketEventDictionary, Simulator, + AgentThrottle, CapsEventDictionary, CapsEventQueueCallback, GridClient, Helpers, + HttpCapsClient, InventoryItem, LoginState, LoginStatus, NetworkManager, + NetworkManagerDisconnectType, NetworkManagerLoginResponseCallback, PacketEventDictionary, + Simulator, }; use libremetaverse_structured_data::{OSD, OSDMap, OSDParser}; use libremetaverse_types::compat::{ @@ -417,6 +418,7 @@ enum ServerCommand { Disable, ExpectEconomy, Handoff, + ExpectThrottle, ExpectLogout, LogoutReply(Vec), Stop, @@ -431,6 +433,7 @@ enum ServerReport { Logout(UUID, UUID), Movement(u32), Ping(u8), + Throttle(u32, Vec), } struct FakeServer { @@ -650,6 +653,35 @@ fn spawn_fake_server() -> FakeServer { break; } } + ServerCommand::ExpectThrottle => loop { + let (length, _) = socket.recv_from(&mut buffer).expect("AgentThrottle"); + if packet_type(&buffer[..length]) != Some(PacketType::AgentThrottle) { + continue; + } + let mut bytes = buffer[..length].to_vec(); + if bytes[0] & Helpers::MSG_ZEROCODED != 0 { + let mut decoded = vec![0; 8192]; + let decoded_length = Helpers::zero_decode( + Some(&bytes), + i32::try_from(bytes.len()).unwrap(), + Some(&mut decoded), + ) + .unwrap(); + decoded.truncate(usize::try_from(decoded_length).unwrap()); + decoded[0] &= !Helpers::MSG_ZEROCODED; + bytes = decoded; + } + let mut position = 0; + let packet = + AgentThrottlePacket::new_with_bytes_int32(bytes, &mut position).unwrap(); + report_sender + .send(ServerReport::Throttle( + packet.agent_data.circuit_code, + packet.throttle.throttles, + )) + .unwrap(); + break; + }, ServerCommand::ExpectLogout => loop { let (length, _) = socket.recv_from(&mut buffer).expect("LogoutRequest"); if packet_type(&buffer[..length]) != Some(PacketType::LogoutRequest) { @@ -772,6 +804,46 @@ fn assert_reuse_preserves_connected_override( manager.set_connected(true); } +#[test] +fn grid_client_throttle_sender_writes_agent_throttle_to_the_simulator() { + let _network_guard = network_test_guard(); + let server = spawn_fake_server(); + let client = GridClient::new().expect("client"); + let mut manager = NetworkManager::new(client.clone()).expect("manager"); + manager.set_circuit_code(0x4455_6677); + let simulator = manager + .connect_with_ip_end_point_u_int64_boolean_uri_u_int32_u_int32( + server.endpoint, + 0x1234, + true, + None, + 256, + 256, + ) + .unwrap() + .expect("simulator"); + assert_eq!( + server.reports.recv_timeout(Duration::from_secs(1)).unwrap(), + ServerReport::Circuit(0x4455_6677) + ); + assert_eq!( + server.reports.recv_timeout(Duration::from_secs(1)).unwrap(), + ServerReport::HandshakeReply(0x1 | 0x2 | 0x4) + ); + + let throttle = AgentThrottle::new_with_grid_client(client).unwrap(); + server.commands.send(ServerCommand::ExpectThrottle).unwrap(); + throttle.set_with_simulator(Some(simulator)).unwrap(); + assert_eq!( + server.reports.recv_timeout(Duration::from_secs(1)).unwrap(), + ServerReport::Throttle(0x4455_6677, throttle.to_bytes().unwrap()) + ); + + manager + .shutdown_with_disconnect_type(NetworkManagerDisconnectType::ClientInitiated) + .unwrap(); +} + #[test] fn fake_server_drives_circuit_handshake_ping_disable_and_disconnect_reasons() { let _network_guard = network_test_guard(); diff --git a/crates/metacrate-grid-agent/Cargo.toml b/crates/metacrate-grid-agent/Cargo.toml index 6849eb0..822605d 100644 --- a/crates/metacrate-grid-agent/Cargo.toml +++ b/crates/metacrate-grid-agent/Cargo.toml @@ -17,6 +17,7 @@ libremetaverse-imaging = { version = "0.0.1", path = "../libremetaverse-imaging" libremetaverse-rendering-simple = { version = "0.0.1", path = "../libremetaverse-rendering-simple", optional = true } libremetaverse-types = { version = "0.0.1", path = "../libremetaverse-types" } metacrate-lsl-tools = { version = "0.0.1", path = "../metacrate-lsl-tools" } +metacrate-rendering-wgpu = { version = "0.0.1", path = "../metacrate-rendering-wgpu" } mentra = { version = "0.18.3", default-features = false } jpeg-encoder = "0.6.1" rustls = { version = "0.23.43", default-features = false, features = ["aws_lc_rs", "std", "tls12"] } @@ -28,7 +29,6 @@ tokio = { version = "1.53.1", features = ["macros", "rt", "sync", "time"] } tokio-rustls = { version = "0.26.4", default-features = false, features = ["aws_lc_rs", "tls12"] } url = "2.5.8" unicode-width = "0.2" -wgpu = { version = "30.0.1", default-features = false, features = ["std", "dx12", "metal", "gles", "vulkan", "wgsl"], optional = true } [dev-dependencies] jpeg-decoder = { version = "0.3.2", default-features = false } @@ -39,7 +39,7 @@ tokio = { version = "1.53.1", features = ["io-util", "net", "rt-multi-thread", " [features] default = [] -live-grid = ["dep:libremetaverse", "dep:libremetaverse-imaging", "dep:libremetaverse-rendering-simple", "dep:wgpu"] +live-grid = ["dep:libremetaverse", "dep:libremetaverse-imaging", "dep:libremetaverse-rendering-simple", "metacrate-rendering-wgpu/wgpu"] [lints] workspace = true diff --git a/crates/metacrate-grid-agent/src/backend.rs b/crates/metacrate-grid-agent/src/backend.rs index 099331c..5b95df5 100644 --- a/crates/metacrate-grid-agent/src/backend.rs +++ b/crates/metacrate-grid-agent/src/backend.rs @@ -94,6 +94,7 @@ impl OfflineGridBackend { #[cfg(feature = "live-grid")] pub struct LibremetaverseClientOwner { client: libremetaverse::GridClient, + network: libremetaverse::NetworkManager, agent: Arc, delivery_generation: Arc>>, } @@ -132,6 +133,7 @@ impl fmt::Debug for LibremetaverseClientOwner { pub struct LibremetaverseSessionBackend { network: libremetaverse::NetworkManager, agent: Arc, + throttle: libremetaverse::AgentThrottle, login_url: String, first_name: String, last_name: String, @@ -168,7 +170,11 @@ impl LibremetaverseClientOwner { .map_err(|_| BackendError::Configuration { component: "libremetaverse client defaults", })?; + // The compatibility client caches its network manager weakly. Retain the + // composition root before managers register packet subscriptions on it. + let network = client.network(); let _grid = client.grid(); + let _terrain = client.terrain(); let agent = Arc::new( libremetaverse::AgentManager::new(Some(Arc::new(client.clone()))).map_err(|_| { BackendError::Configuration { @@ -178,6 +184,7 @@ impl LibremetaverseClientOwner { ); Ok(Self { client, + network, agent, delivery_generation: Arc::new(std::sync::RwLock::new(None)), }) @@ -273,8 +280,12 @@ impl LibremetaverseClientOwner { }); } Ok(LibremetaverseSessionBackend { - network: self.client.network(), + network: self.network.clone(), agent: Arc::clone(&self.agent), + throttle: libremetaverse::AgentThrottle::new_with_grid_client(self.client.clone()) + .map_err(|_| BackendError::Configuration { + component: "libremetaverse agent throttle", + })?, login_url: connection.login_url.expose_url().to_owned(), first_name: first_name.to_owned(), last_name: last_name.to_owned(), @@ -794,30 +805,7 @@ impl crate::behavior::EmbodimentSink for LibremetaverseEmbodimentSink { return Err(crate::behavior::BehaviorError::Cancelled); } self.ensure_generation(generation)?; - let avatar = self.agent.sim_position(); - let forward = libremetaverse_types::Vector3::mul_with_vector3_quaternion( - libremetaverse_types::Vector3::unit_x(), - self.agent.sim_rotation(), - ); - let position = libremetaverse_types::Vector3 { - x: avatar.x, - y: avatar.y, - z: avatar.z + 1.6, - }; - let target = libremetaverse_types::Vector3 { - x: position.x + forward.x * 4.0, - y: position.y + forward.y * 4.0, - z: position.z + forward.z * 4.0, - }; - self.agent - .movement - .camera - .look_at_with_vector3_vector3(position, target) - .and_then(|()| { - self.agent - .movement - .set_fov_vertical_angle(60.0_f32.to_radians()) - }) + native_reset_camera(&self.agent) .and_then(|()| self.agent.movement.send_update_with_boolean(Some(true))) .map_err(|_| crate::behavior::BehaviorError::NativeOperation) }) @@ -847,6 +835,30 @@ fn native_position(value: libremetaverse_types::Vector3) -> crate::perception::W } } +#[cfg(feature = "live-grid")] +fn native_reset_camera(agent: &libremetaverse::AgentManager) -> Result<(), libremetaverse::Error> { + let avatar = agent.sim_position(); + let forward = libremetaverse_types::Vector3::mul_with_vector3_quaternion( + libremetaverse_types::Vector3::unit_x(), + agent.sim_rotation(), + ); + let position = libremetaverse_types::Vector3 { + x: avatar.x, + y: avatar.y, + z: avatar.z + 1.6, + }; + let target = libremetaverse_types::Vector3 { + x: position.x + forward.x * 4.0, + y: position.y + forward.y * 4.0, + z: position.z + forward.z * 4.0, + }; + agent + .movement + .camera + .look_at_with_vector3_vector3(position, target)?; + agent.movement.set_fov_vertical_angle(60.0_f32.to_radians()) +} + #[cfg(feature = "live-grid")] fn native_inventory_metadata( client: &libremetaverse::GridClient, @@ -1089,6 +1101,23 @@ impl crate::session::GridSessionBackend for LibremetaverseSessionBackend { ) })?; if logged_in { + let simulator = self.network.current_sim().ok_or_else(|| { + crate::session::SessionFailure::new( + crate::session::SessionFailureKind::TransientTransport, + ) + })?; + self.throttle + .set_with_simulator(Some(simulator)) + .map_err(|_| { + crate::session::SessionFailure::new( + crate::session::SessionFailureKind::TransientTransport, + ) + })?; + native_reset_camera(&self.agent).map_err(|_| { + crate::session::SessionFailure::new( + crate::session::SessionFailureKind::TransientTransport, + ) + })?; break; } let error_key = self.network.native_login_error_key(); diff --git a/crates/metacrate-grid-agent/src/vision.rs b/crates/metacrate-grid-agent/src/vision.rs index 11fd592..059b85c 100644 --- a/crates/metacrate-grid-agent/src/vision.rs +++ b/crates/metacrate-grid-agent/src/vision.rs @@ -23,13 +23,7 @@ use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; use tokio::sync::Semaphore; -#[derive(Clone, Copy, Debug, PartialEq)] -pub struct CameraPose { - pub position: [f32; 3], - pub forward: [f32; 3], - pub up: [f32; 3], - pub vertical_fov_degrees: f32, -} +pub use metacrate_rendering_wgpu::{Camera as CameraPose, Triangle as SceneTriangle}; #[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)] #[serde(rename_all = "snake_case")] @@ -40,12 +34,6 @@ pub enum SceneEntityKind { Environment, } -#[derive(Clone, Debug, PartialEq)] -pub struct SceneTriangle { - pub vertices: [[f32; 3]; 3], - pub color_srgb: [u8; 4], -} - #[derive(Clone, Debug, PartialEq)] pub struct SceneEntity { pub id: UUID, @@ -96,16 +84,16 @@ pub struct VisionLimits { impl Default for VisionLimits { fn default() -> Self { Self { - width: 320, - height: 180, + width: 640, + height: 360, max_entities: 256, max_triangles: 32_768, - max_texture_fetches: 8, - max_texture_bytes: 8 * 1024 * 1024, - max_decode_pixels: 4_194_304, + max_texture_fetches: 128, + max_texture_bytes: 64 * 1024 * 1024, + max_decode_pixels: 16_777_216, max_jpeg_bytes: 512 * 1024, max_concurrent_captures: 1, - capture_timeout: Duration::from_secs(10), + capture_timeout: Duration::from_secs(30), minimum_interval: Duration::from_secs(5), } } @@ -117,9 +105,9 @@ impl VisionLimits { && self.width as usize * self.height as usize <= 307_200 && (1..=512).contains(&self.max_entities) && (1..=65_536).contains(&self.max_triangles) - && self.max_texture_fetches <= 64 - && self.max_texture_bytes <= 16 * 1024 * 1024 - && self.max_decode_pixels <= 8_388_608 + && self.max_texture_fetches <= 128 + && self.max_texture_bytes <= 64 * 1024 * 1024 + && self.max_decode_pixels <= 16_777_216 && (4_096..=1024 * 1024).contains(&self.max_jpeg_bytes) && (1..=4).contains(&self.max_concurrent_captures) && !self.capture_timeout.is_zero() @@ -221,7 +209,7 @@ pub struct VisionService { #[cfg(feature = "live-grid")] gpu_preferred: bool, #[cfg(feature = "live-grid")] - gpu: tokio::sync::OnceCell>>, + gpu: tokio::sync::OnceCell>>, } impl VisionService { @@ -395,15 +383,58 @@ impl VisionService { if self.gpu_preferred { let renderer = self .gpu - .get_or_init(|| async { WgpuRenderer::new().await.ok().map(Arc::new) }) + .get_or_init(|| async { + metacrate_rendering_wgpu::Renderer::new() + .await + .ok() + .map(Arc::new) + }) .await .clone(); if let Some(renderer) = renderer { + let mut triangles = scene + .entities + .iter() + .flat_map(|entity| { + entity + .triangles + .iter() + .cloned() + .map(move |triangle| (entity.id.to_string(), triangle)) + }) + .collect::>(); + triangles.sort_by(|left, right| left.0.cmp(&right.0)); + let triangles = triangles + .into_iter() + .map(|(_, triangle)| triangle) + .collect::>(); let gpu_scene = scene.clone(); let limits = self.limits; if let Ok(Ok(capture)) = tokio::task::spawn_blocking(move || { renderer - .render(&gpu_scene, limits) + .render( + gpu_scene.camera, + &triangles, + environment_color(&gpu_scene), + metacrate_rendering_wgpu::RenderLimits { + width: limits.width, + height: limits.height, + max_triangles: limits.max_triangles, + far_distance: 512, + }, + ) + .map_err(|error| match error { + metacrate_rendering_wgpu::RenderError::InvalidScene => { + VisionError::InvalidScene + } + metacrate_rendering_wgpu::RenderError::ResourceLimit => { + VisionError::ResourceLimit + } + metacrate_rendering_wgpu::RenderError::TimedOut => { + VisionError::TimedOut + } + _ => VisionError::Encode, + }) .and_then(|rgba| finish_capture(&gpu_scene, limits, &rgba)) }) .await @@ -794,7 +825,13 @@ fn raster_triangle( let index = y as usize * width as usize + x as usize; if z < depth[index] { depth[index] = z; - rgba[index * 4..index * 4 + 4].copy_from_slice(&triangle.color_srgb); + for channel in 0..4 { + rgba[index * 4 + channel] = (w0 + * f32::from(triangle.colors_srgb[0][channel]) + + w1 * f32::from(triangle.colors_srgb[1][channel]) + + w2 * f32::from(triangle.colors_srgb[2][channel])) + .round() as u8; + } } } } @@ -817,315 +854,6 @@ fn environment_color(scene: &SceneSnapshot) -> [u8; 4] { ] } -#[cfg(feature = "live-grid")] -pub(crate) struct WgpuRenderer { - device: wgpu::Device, - queue: wgpu::Queue, - pipeline: wgpu::RenderPipeline, -} - -#[cfg(feature = "live-grid")] -const WGPU_VERTEX_ATTRIBUTES: [wgpu::VertexAttribute; 2] = - wgpu::vertex_attr_array![0 => Float32x3, 1 => Float32x4]; - -#[cfg(feature = "live-grid")] -impl WgpuRenderer { - pub(crate) async fn new() -> Result { - let instance = wgpu::Instance::new(wgpu::InstanceDescriptor::new_without_display_handle()); - let adapter = instance - .request_adapter(&wgpu::RequestAdapterOptions { - power_preference: wgpu::PowerPreference::LowPower, - ..Default::default() - }) - .await - .map_err(|_| VisionError::Encode)?; - let (device, queue) = adapter - .request_device(&wgpu::DeviceDescriptor { - label: Some("metacrate-agent-vision"), - required_features: wgpu::Features::empty(), - required_limits: wgpu::Limits::downlevel_defaults(), - memory_hints: wgpu::MemoryHints::MemoryUsage, - ..Default::default() - }) - .await - .map_err(|_| VisionError::Encode)?; - let shader = device.create_shader_module(wgpu::ShaderModuleDescriptor { - label: Some("metacrate-agent-vision-shader"), - source: wgpu::ShaderSource::Wgsl( - r" -struct VertexInput { - @location(0) position: vec3, - @location(1) color: vec4, -}; -struct VertexOutput { - @builtin(position) position: vec4, - @location(0) color: vec4, -}; -@vertex -fn vs_main(input: VertexInput) -> VertexOutput { - var output: VertexOutput; - output.position = vec4(input.position, 1.0); - output.color = input.color; - return output; -} -@fragment -fn fs_main(input: VertexOutput) -> @location(0) vec4 { - return input.color; -} -" - .into(), - ), - }); - let layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { - label: Some("metacrate-agent-vision-layout"), - bind_group_layouts: &[], - immediate_size: 0, - }); - let pipeline = device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { - label: Some("metacrate-agent-vision-pipeline"), - layout: Some(&layout), - vertex: wgpu::VertexState { - module: &shader, - entry_point: Some("vs_main"), - compilation_options: wgpu::PipelineCompilationOptions::default(), - buffers: &[Some(wgpu::VertexBufferLayout { - array_stride: 7 * size_of::() as u64, - step_mode: wgpu::VertexStepMode::Vertex, - attributes: &WGPU_VERTEX_ATTRIBUTES, - })], - }, - primitive: wgpu::PrimitiveState { - topology: wgpu::PrimitiveTopology::TriangleList, - cull_mode: None, - ..Default::default() - }, - depth_stencil: Some(wgpu::DepthStencilState { - format: wgpu::TextureFormat::Depth32Float, - depth_write_enabled: Some(true), - depth_compare: Some(wgpu::CompareFunction::Less), - stencil: wgpu::StencilState::default(), - bias: wgpu::DepthBiasState::default(), - }), - multisample: wgpu::MultisampleState::default(), - fragment: Some(wgpu::FragmentState { - module: &shader, - entry_point: Some("fs_main"), - compilation_options: wgpu::PipelineCompilationOptions::default(), - targets: &[Some(wgpu::ColorTargetState { - format: wgpu::TextureFormat::Rgba8UnormSrgb, - blend: None, - write_mask: wgpu::ColorWrites::ALL, - })], - }), - multiview_mask: None, - cache: None, - }); - Ok(Self { - device, - queue, - pipeline, - }) - } - - #[allow(clippy::too_many_lines)] - pub(crate) fn render( - &self, - scene: &SceneSnapshot, - limits: VisionLimits, - ) -> Result, VisionError> { - validate_scene(scene, limits)?; - let (vertices, vertex_count) = gpu_vertices(scene, limits)?; - let vertex_buffer = self.device.create_buffer(&wgpu::BufferDescriptor { - label: Some("metacrate-agent-vision-vertices"), - size: vertices.len().max(4) as u64, - usage: wgpu::BufferUsages::VERTEX | wgpu::BufferUsages::COPY_DST, - mapped_at_creation: false, - }); - if !vertices.is_empty() { - self.queue.write_buffer(&vertex_buffer, 0, &vertices); - } - let extent = wgpu::Extent3d { - width: limits.width, - height: limits.height, - depth_or_array_layers: 1, - }; - let color = self.device.create_texture(&wgpu::TextureDescriptor { - label: Some("metacrate-agent-vision-color"), - size: extent, - mip_level_count: 1, - sample_count: 1, - dimension: wgpu::TextureDimension::D2, - format: wgpu::TextureFormat::Rgba8UnormSrgb, - usage: wgpu::TextureUsages::RENDER_ATTACHMENT | wgpu::TextureUsages::COPY_SRC, - view_formats: &[], - }); - let depth = self.device.create_texture(&wgpu::TextureDescriptor { - label: Some("metacrate-agent-vision-depth"), - size: extent, - mip_level_count: 1, - sample_count: 1, - dimension: wgpu::TextureDimension::D2, - format: wgpu::TextureFormat::Depth32Float, - usage: wgpu::TextureUsages::RENDER_ATTACHMENT, - view_formats: &[], - }); - let row_bytes = limits - .width - .checked_mul(4) - .ok_or(VisionError::ResourceLimit)?; - let padded_row_bytes = row_bytes.div_ceil(wgpu::COPY_BYTES_PER_ROW_ALIGNMENT) - * wgpu::COPY_BYTES_PER_ROW_ALIGNMENT; - let readback_size = u64::from(padded_row_bytes) - .checked_mul(u64::from(limits.height)) - .ok_or(VisionError::ResourceLimit)?; - let readback = self.device.create_buffer(&wgpu::BufferDescriptor { - label: Some("metacrate-agent-vision-readback"), - size: readback_size, - usage: wgpu::BufferUsages::COPY_DST | wgpu::BufferUsages::MAP_READ, - mapped_at_creation: false, - }); - let color_view = color.create_view(&wgpu::TextureViewDescriptor::default()); - let depth_view = depth.create_view(&wgpu::TextureViewDescriptor::default()); - let sky = environment_color(scene); - let mut encoder = self - .device - .create_command_encoder(&wgpu::CommandEncoderDescriptor { - label: Some("metacrate-agent-vision-commands"), - }); - { - let mut pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { - label: Some("metacrate-agent-vision-pass"), - color_attachments: &[Some(wgpu::RenderPassColorAttachment { - view: &color_view, - depth_slice: None, - resolve_target: None, - ops: wgpu::Operations { - load: wgpu::LoadOp::Clear(wgpu::Color { - r: f64::from(sky[0]) / 255.0, - g: f64::from(sky[1]) / 255.0, - b: f64::from(sky[2]) / 255.0, - a: 1.0, - }), - store: wgpu::StoreOp::Store, - }, - })], - depth_stencil_attachment: Some(wgpu::RenderPassDepthStencilAttachment { - view: &depth_view, - depth_ops: Some(wgpu::Operations { - load: wgpu::LoadOp::Clear(1.0), - store: wgpu::StoreOp::Discard, - }), - stencil_ops: None, - }), - ..Default::default() - }); - if vertex_count != 0 { - pass.set_pipeline(&self.pipeline); - pass.set_vertex_buffer(0, vertex_buffer.slice(..)); - pass.draw(0..vertex_count, 0..1); - } - } - encoder.copy_texture_to_buffer( - wgpu::TexelCopyTextureInfo { - texture: &color, - mip_level: 0, - origin: wgpu::Origin3d::ZERO, - aspect: wgpu::TextureAspect::All, - }, - wgpu::TexelCopyBufferInfo { - buffer: &readback, - layout: wgpu::TexelCopyBufferLayout { - offset: 0, - bytes_per_row: Some(padded_row_bytes), - rows_per_image: Some(limits.height), - }, - }, - extent, - ); - let submission = self.queue.submit([encoder.finish()]); - let slice = readback.slice(..); - let (sender, receiver) = std::sync::mpsc::sync_channel(1); - slice.map_async(wgpu::MapMode::Read, move |result| { - let _ = sender.send(result); - }); - self.device - .poll(wgpu::PollType::Wait { - submission_index: Some(submission), - timeout: Some(Duration::from_secs(5)), - }) - .map_err(|_| VisionError::TimedOut)?; - receiver - .recv_timeout(Duration::from_secs(1)) - .map_err(|_| VisionError::TimedOut)? - .map_err(|_| VisionError::Encode)?; - let mapped = slice.get_mapped_range().map_err(|_| VisionError::Encode)?; - let row_bytes = row_bytes as usize; - let padded_row_bytes = padded_row_bytes as usize; - let mut rgba = Vec::with_capacity(row_bytes * limits.height as usize); - for row in mapped - .chunks_exact(padded_row_bytes) - .take(limits.height as usize) - { - rgba.extend_from_slice(&row[..row_bytes]); - } - drop(mapped); - readback.unmap(); - Ok(rgba) - } -} - -#[cfg(feature = "live-grid")] -fn gpu_vertices( - scene: &SceneSnapshot, - limits: VisionLimits, -) -> Result<(Vec, u32), VisionError> { - let basis = camera_basis(scene.camera)?; - let aspect = limits.width as f32 / limits.height as f32; - let focal = 1.0 / (scene.camera.vertical_fov_degrees.to_radians() / 2.0).tan(); - let mut triangles = scene - .entities - .iter() - .flat_map(|entity| { - entity - .triangles - .iter() - .map(move |triangle| (entity, triangle)) - }) - .collect::>(); - triangles.sort_by_key(|(entity, _)| entity.id.to_string()); - let mut bytes = Vec::with_capacity(triangles.len() * 3 * 7 * size_of::()); - let mut count = 0u32; - for (_, triangle) in triangles { - let projected = triangle.vertices.map(|world| { - let delta = [ - world[0] - scene.camera.position[0], - world[1] - scene.camera.position[1], - world[2] - scene.camera.position[2], - ]; - let z = dot(delta, basis.2); - [ - dot(delta, basis.0) * focal / aspect / z, - dot(delta, basis.1) * focal / z, - (z / 512.0).clamp(0.0, 1.0), - z, - ] - }); - if projected.iter().any(|vertex| vertex[3] <= 0.05) { - continue; - } - let color = triangle - .color_srgb - .map(|channel| f32::from(channel) / 255.0); - for vertex in projected { - for value in vertex[..3].iter().chain(color.iter()) { - bytes.extend_from_slice(&value.to_ne_bytes()); - } - count = count.checked_add(1).ok_or(VisionError::ResourceLimit)?; - } - } - Ok((bytes, count)) -} - fn encode_jpeg( width: u32, height: u32, @@ -1161,9 +889,16 @@ fn scene_summary(scene: &SceneSnapshot) -> String { .iter() .filter(|e| e.kind == SceneEntityKind::Resident) .count(); + let triangles = scene + .entities + .iter() + .map(|entity| entity.triangles.len()) + .sum::(); format!( - "Reconstructed viewport of region {:?}: {objects} tracked objects and {residents} privacy-marked residents. generation={} observed_unix_millis={}. completeness: objects_truncated={}, avatars_truncated={}, textures_missing={}, terrain_available={}. This is reconstructed from available scene data, not a viewer framebuffer.", + "Reconstructed viewport of region {:?}: {objects} tracked objects, {residents} privacy-marked residents, and {triangles} triangles. camera_position={:?}, camera_forward={:?}. generation={} observed_unix_millis={}. completeness: objects_truncated={}, avatars_truncated={}, textures_missing={}, terrain_available={}. This is reconstructed from available scene data, not a viewer framebuffer.", sanitize(&scene.region_name), + scene.camera.position, + scene.camera.forward, scene.generation, scene.observed_unix_millis, scene.completeness.objects_truncated, @@ -1217,88 +952,203 @@ impl SceneSource for LibremetaverseSceneSource { .network() .current_sim() .ok_or(VisionError::StaleGeneration)?; + let scene_deadline = Instant::now() + Duration::from_secs(5); + loop { + let terrain_ready = simulator + .terrain + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .iter() + .any(|patch| patch.data.len() == 256); + if terrain_ready { + break; + } + if Instant::now() >= scene_deadline { + return Err(VisionError::TimedOut); + } + tokio::select! { + () = cancellation.cancelled() => return Err(VisionError::Cancelled), + () = tokio::time::sleep(Duration::from_millis(100)) => {} + } + } let camera = &self.agent.movement.camera; let position = camera.position(); - let forward = camera.at_axis(); + let forward = native_camera_forward(camera); let up = camera.up_axis(); let entity_limit = self.limits.max_entities.saturating_sub(1); - let prims = simulator - .objects_primitives - .read() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .values() - .filter(|prim| !prim.is_attachment) - .take(entity_limit) - .cloned() - .collect::>(); + let mut prims = { + let primitive_cache = simulator + .objects_primitives + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let mut prims = primitive_cache + .values() + .filter(|prim| !prim.is_attachment) + .cloned() + .collect::>(); + for prim in &mut prims { + if let Some(parent) = primitive_cache.get(&prim.parent_id) { + prim.position = libremetaverse_types::Vector3::add_with_vector3_vector3( + parent.position, + libremetaverse_types::Vector3::mul_with_vector3_quaternion( + prim.position, + parent.rotation, + ), + ) + .unwrap_or(prim.position); + prim.rotation = + libremetaverse_types::Quaternion::mul_with_quaternion_quaternion( + parent.rotation, + prim.rotation, + ); + prim.parent_id = 0; + } + } + prims + }; + let camera_position = [position.x, position.y, position.z]; + let camera_forward = [forward.x, forward.y, forward.z]; + // ponytail: center-plane culling is the bounded broad phase; use + // object bounds when the scene cache exposes reliable world AABBs. + prims.retain(|prim| { + scene_center_in_front(prim.position, camera_position, camera_forward) + }); + prims.sort_by(|left, right| { + scene_distance_score(left.position, camera_position, camera_forward).total_cmp( + &scene_distance_score(right.position, camera_position, camera_forward), + ) + }); + let objects_truncated = prims.len() > entity_limit; + prims.truncate(entity_limit); let remaining = entity_limit.saturating_sub(prims.len()); - let avatars = simulator - .objects_avatars - .read() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .values() - .take(remaining) - .cloned() - .collect::>(); - let objects_truncated = simulator - .objects_primitives - .read() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .len() - > prims.len(); - let avatars_truncated = simulator - .objects_avatars - .read() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .len() - > avatars.len(); + let (mut avatars, avatars_truncated) = { + let avatar_cache = simulator + .objects_avatars + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner); + ( + avatar_cache.values().cloned().collect::>(), + avatar_cache.len() > remaining, + ) + }; + avatars.sort_by(|left, right| { + scene_distance_score(left.position, camera_position, camera_forward).total_cmp( + &scene_distance_score(right.position, camera_position, camera_forward), + ) + }); + avatars.truncate(remaining); let region_id = simulator.region_id; let region_name = simulator.name.clone(); let water = simulator.water_height; let region_size = (simulator.size_x as f32, simulator.size_y as f32); - let texture_ids = prims - .iter() - .filter_map(|prim| { - prim.textures - .as_ref()? + let terrain = simulator + .terrain + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone(); + let terrain_available = terrain.iter().any(|patch| patch.data.len() == 256); + let terrain_texture_ids = [ + simulator.terrain_detail0, + simulator.terrain_detail1, + simulator.terrain_detail2, + simulator.terrain_detail3, + ]; + let mut texture_ids = terrain_texture_ids.to_vec(); + for prim in &prims { + if let Some(textures) = &prim.textures { + let faces = textures .default_texture - .as_ref() - .map(libremetaverse::PrimitiveTextureEntryFace::texture_id) - }) + .iter() + .chain(textures.face_textures.iter().flatten()); + for face in faces { + let id = face.texture_id(); + if id != UUID::zero() && !texture_ids.contains(&id) { + texture_ids.push(id); + } + } + } + } + let mesh_ids = prims + .iter() + .filter_map(|prim| prim.sculpt.as_ref()) + .filter(|sculpt| sculpt.type_() == libremetaverse_types::SculptType::Mesh) + .map(|sculpt| sculpt.sculpt_texture) .filter(|id| *id != UUID::zero()) .fold(Vec::new(), |mut ids, id| { - if !ids.contains(&id) && ids.len() < self.limits.max_texture_fetches { + if !ids.contains(&id) && ids.len() < self.limits.max_texture_fetches / 2 { ids.push(id); } ids }); + texture_ids.retain(|id| *id != UUID::zero()); + texture_ids.truncate( + self.limits + .max_texture_fetches + .saturating_sub(mesh_ids.len()), + ); let assets = self.client.assets(); let texture_fetches = texture_ids.len(); let mut encoded_textures = Vec::new(); + let mut encoded_meshes = Vec::new(); let mut texture_bytes = 0usize; + let mut pending = tokio::task::JoinSet::new(); for id in texture_ids { + let assets = assets.clone(); + let token = cancellation.clone(); + pending.spawn(async move { + let asset = tokio::time::timeout( + Duration::from_secs(5), + assets.request_asset_with_uuid_asset_type_boolean_cancellation_token( + id, + libremetaverse_types::AssetType::Texture, + false, + Some(token), + ), + ) + .await + .ok() + .and_then(Result::ok) + .flatten(); + (false, id, asset) + }); + } + for id in mesh_ids { + let assets = assets.clone(); + let token = cancellation.clone(); + pending.spawn(async move { + let asset = tokio::time::timeout( + Duration::from_secs(5), + assets.request_asset_with_uuid_asset_type_boolean_cancellation_token( + id, + libremetaverse_types::AssetType::Mesh, + false, + Some(token), + ), + ) + .await + .ok() + .and_then(Result::ok) + .flatten(); + (true, id, asset) + }); + } + while let Some(result) = pending.join_next().await { if cancellation.is_cancellation_requested() { + pending.abort_all(); return Err(VisionError::Cancelled); } - let asset = tokio::time::timeout( - Duration::from_millis(250), - assets.request_asset_with_uuid_asset_type_boolean_cancellation_token( - id, - libremetaverse_types::AssetType::Texture, - false, - Some(cancellation.clone()), - ), - ) - .await - .ok() - .and_then(Result::ok) - .flatten(); - if let Some(asset) = asset - && texture_bytes.saturating_add(asset.asset_data.len()) - <= self.limits.max_texture_bytes + let Ok((mesh, id, Some(asset))) = result else { + continue; + }; + if texture_bytes.saturating_add(asset.asset_data.len()) + <= self.limits.max_texture_bytes { texture_bytes += asset.asset_data.len(); - encoded_textures.push((id, asset.asset_data)); + if mesh { + encoded_meshes.push((id, asset.asset_data)); + } else { + encoded_textures.push((id, asset.asset_data)); + } } } let maximum = self.limits.max_triangles; @@ -1310,8 +1160,11 @@ impl SceneSource for LibremetaverseSceneSource { avatars, water, region_size, + &terrain, + terrain_texture_ids, maximum, encoded_textures, + encoded_meshes, (decode_bytes, decode_pixels), ) }) @@ -1348,7 +1201,7 @@ impl SceneSource for LibremetaverseSceneSource { objects_truncated, avatars_truncated, textures_missing, - terrain_available: true, + terrain_available, }, texture_fetches, texture_bytes, @@ -1359,18 +1212,68 @@ impl SceneSource for LibremetaverseSceneSource { } #[cfg(feature = "live-grid")] -#[allow(clippy::too_many_lines)] +pub(crate) fn native_camera_forward( + camera: &libremetaverse::AgentManagerAgentMovementAgentCamera, +) -> libremetaverse_types::Vector3 { + // LibreMetaverse's AgentCamera wrapper swaps its frame X/Y axes when it + // writes AgentUpdate; LeftAxis is therefore the actual look direction. + camera.left_axis() +} + +#[cfg(feature = "live-grid")] +pub(crate) fn scene_center_in_front( + position: libremetaverse_types::Vector3, + camera: [f32; 3], + forward: [f32; 3], +) -> bool { + dot( + [ + position.x - camera[0], + position.y - camera[1], + position.z - camera[2], + ], + forward, + ) > 0.0 +} + +#[cfg(feature = "live-grid")] +fn scene_distance_score( + position: libremetaverse_types::Vector3, + camera: [f32; 3], + forward: [f32; 3], +) -> f32 { + let delta = [ + position.x - camera[0], + position.y - camera[1], + position.z - camera[2], + ]; + let distance = dot(delta, delta); + if dot(delta, forward) > 0.0 { + distance + } else { + distance + 1_000_000.0 + } +} + +#[cfg(feature = "live-grid")] +#[allow(clippy::too_many_arguments, clippy::too_many_lines)] // Single blocking scene-conversion boundary. fn native_entities( prims: Vec, avatars: Vec, water: f32, region_size: (f32, f32), + terrain: &[libremetaverse::TerrainPatch], + terrain_texture_ids: [UUID; 4], maximum: usize, encoded_textures: Vec<(UUID, Vec)>, + encoded_meshes: Vec<(UUID, Vec)>, decode_limits: (usize, usize), ) -> Result<(Vec, usize), VisionError> { - let (texture_colors, pixels_consumed) = - decode_texture_colors(encoded_textures, decode_limits.0, decode_limits.1); + let (textures, pixels_consumed) = + decode_textures(encoded_textures, decode_limits.0, decode_limits.1); + let mesh_assets = encoded_meshes + .into_iter() + .collect::>(); let renderer = libremetaverse_rendering_simple::SimpleRenderer::new() .map_err(|_| VisionError::InvalidScene)?; let mut entities = Vec::new(); @@ -1380,35 +1283,52 @@ fn native_entities( || "tracked object".to_owned(), |properties| properties.name.clone(), ); - let texture_id = prim - .textures - .as_ref() - .and_then(|textures| textures.default_texture.as_ref()) - .map(libremetaverse::PrimitiveTextureEntryFace::texture_id); - let texture_color = texture_id.and_then(|id| texture_colors.get(&id).copied()); - let texture_available = texture_color.is_some(); - let Ok(mesh) = renderer - .generate_faceted_mesh(prim.clone(), libremetaverse::rendering::DetailLevel::Low) - else { - continue; + let mesh = if let Some(sculpt) = &prim.sculpt + && sculpt.type_() == libremetaverse_types::SculptType::Mesh + { + let Some(bytes) = mesh_assets.get(&sculpt.sculpt_texture) else { + continue; + }; + let Ok(asset) = libremetaverse::assets::AssetMesh::new_with_uuid_bytes( + sculpt.sculpt_texture, + bytes.clone(), + ) else { + continue; + }; + let mut decoded = None; + if !libremetaverse::rendering::FacetedMesh::try_decode_from_asset( + prim.clone(), + asset, + libremetaverse::rendering::DetailLevel::Low, + &mut decoded, + ) { + continue; + } + let Some(mesh) = decoded else { continue }; + mesh + } else { + let Ok(mesh) = renderer + .generate_faceted_mesh(prim.clone(), libremetaverse::rendering::DetailLevel::Low) + else { + continue; + }; + mesh }; let mut output = Vec::new(); + let mut texture_available = false; for face in mesh.faces { let rgba = face.texture_face.rgba(); - let mut color = [ + let tint = [ unit_byte(rgba.r), unit_byte(rgba.g), unit_byte(rgba.b), unit_byte(rgba.a), ]; - if let Some(texture) = texture_color { - for channel in 0..3 { - color[channel] = - ((u16::from(color[channel]) * u16::from(texture[channel])) / 255) as u8; - } - } + let texture = textures.get(&face.texture_face.texture_id()); + texture_available |= texture.is_some(); 'triangle: for indices in face.indices.chunks_exact(3) { let mut vertices = [[0.0; 3]; 3]; + let mut colors = [tint; 3]; for (index, source) in indices.iter().enumerate() { let Some(vertex) = face.vertices.get(*source as usize) else { continue 'triangle; @@ -1425,17 +1345,25 @@ fn native_entities( rotated.y + prim.position.y, rotated.z + prim.position.z, ]; + if let Some(texture) = texture { + colors[index] = multiply_color( + tint, + texture.sample(vertex.tex_coord.x, vertex.tex_coord.y), + ); + } } if vertices.iter().flatten().any(|value| !value.is_finite()) { continue; } - triangles = triangles.checked_add(1).ok_or(VisionError::ResourceLimit)?; - if triangles > maximum { - return Err(VisionError::ResourceLimit); + let shade = triangle_shade(vertices); + colors = colors.map(|color| shade_color(color, shade)); + if triangles >= maximum / 2 { + break; } + triangles += 1; output.push(SceneTriangle { vertices, - color_srgb: color, + colors_srgb: colors, }); } } @@ -1448,19 +1376,22 @@ fn native_entities( }); } for avatar in avatars { + if triangles.saturating_add(2) > maximum / 2 { + break; + } let p = avatar.position; let half = 0.3; let bottom = p.z; let top = p.z + 1.8; let front = p.y; - let triangles = vec![ + let avatar_triangles = vec![ SceneTriangle { vertices: [ [p.x - half, front, bottom], [p.x + half, front, bottom], [p.x + half, front, top], ], - color_srgb: [80, 140, 220, 255], + colors_srgb: [[80, 140, 220, 255]; 3], }, SceneTriangle { vertices: [ @@ -1468,9 +1399,10 @@ fn native_entities( [p.x + half, front, top], [p.x - half, front, top], ], - color_srgb: [80, 140, 220, 255], + colors_srgb: [[80, 140, 220, 255]; 3], }, ]; + triangles += 2; entities.push(SceneEntity { id: avatar.id, kind: SceneEntityKind::Resident, @@ -1478,43 +1410,63 @@ fn native_entities( "resident-{}", crate::observability::pseudonymous_identifier(&avatar.id.to_string()) ), - triangles, + triangles: avatar_triangles, texture_available: false, }); } - entities.push(SceneEntity { - id: UUID::zero(), - kind: SceneEntityKind::Terrain, - display_name: "bounded terrain/water plane".into(), - triangles: vec![ - SceneTriangle { - vertices: [ - [0.0, 0.0, water], - [region_size.0, 0.0, water], - [region_size.0, region_size.1, water], - ], - color_srgb: [55, 95, 80, 255], - }, - SceneTriangle { - vertices: [ - [0.0, 0.0, water], - [region_size.0, region_size.1, water], - [0.0, region_size.1, water], - ], - color_srgb: [55, 95, 80, 255], - }, - ], - texture_available: false, - }); + let terrain_triangles = terrain_triangles( + terrain, + region_size, + water, + &textures, + terrain_texture_ids, + maximum.saturating_sub(triangles), + ); + if !terrain_triangles.is_empty() { + entities.push(SceneEntity { + id: UUID::zero(), + kind: SceneEntityKind::Terrain, + display_name: "simulator terrain".into(), + triangles: terrain_triangles, + texture_available: terrain_texture_ids + .iter() + .any(|id| textures.contains_key(id)), + }); + } Ok((entities, pixels_consumed)) } #[cfg(feature = "live-grid")] -fn decode_texture_colors( +struct DecodedTexture { + width: usize, + height: usize, + red: Vec, + green: Vec, + blue: Vec, + alpha: Vec, +} + +#[cfg(feature = "live-grid")] +impl DecodedTexture { + fn sample(&self, u: f32, v: f32) -> [u8; 4] { + let x = (u.rem_euclid(1.0) * self.width as f32) as usize % self.width; + let y = ((1.0 - v).rem_euclid(1.0) * self.height as f32) as usize % self.height; + let index = y * self.width + x; + [ + self.red[index], + self.green.get(index).copied().unwrap_or(self.red[index]), + self.blue.get(index).copied().unwrap_or(self.red[index]), + self.alpha.get(index).copied().unwrap_or(255), + ] + } +} + +#[cfg(feature = "live-grid")] +fn decode_textures( textures: Vec<(UUID, Vec)>, max_bytes: usize, max_pixels: usize, -) -> (std::collections::BTreeMap, usize) { +) -> (std::collections::BTreeMap, usize) { let mut output = std::collections::BTreeMap::new(); let mut remaining_pixels = max_pixels; for (id, bytes) in textures { @@ -1538,26 +1490,165 @@ fn decode_texture_colors( if pixels == 0 || pixels > remaining_pixels || image.red.len() != pixels { continue; } - let average = |plane: &[u8], fallback: u8| -> u8 { - if plane.len() != pixels { - return fallback; - } - let sum = plane.iter().map(|value| u64::from(*value)).sum::(); - u8::try_from(sum / pixels as u64).unwrap_or(fallback) - }; output.insert( id, - [ - average(&image.red, 255), - average(&image.green, 255), - average(&image.blue, 255), - average(&image.alpha, 255), - ], + DecodedTexture { + width, + height, + red: image.red, + green: image.green, + blue: image.blue, + alpha: image.alpha, + }, ); remaining_pixels -= pixels; } (output, max_pixels - remaining_pixels) } + +#[cfg(feature = "live-grid")] +fn multiply_color(tint: [u8; 4], texture: [u8; 4]) -> [u8; 4] { + std::array::from_fn(|channel| { + ((u16::from(tint[channel]) * u16::from(texture[channel])) / 255) as u8 + }) +} + +#[cfg(feature = "live-grid")] +pub(crate) fn terrain_height( + terrain: &[libremetaverse::TerrainPatch], + patch_width: usize, + x: usize, + y: usize, +) -> Option { + let patch = terrain.get((y / 16) * patch_width + x / 16)?; + patch.data.get((y % 16) * 16 + x % 16).copied() +} + +#[cfg(feature = "live-grid")] +fn terrain_triangles( + terrain: &[libremetaverse::TerrainPatch], + region_size: (f32, f32), + water: f32, + textures: &std::collections::BTreeMap, + texture_ids: [UUID; 4], + maximum: usize, +) -> Vec { + let (width, height, patch_width) = terrain_layout(terrain, region_size); + if maximum < 2 || width < 2 || height < 2 { + return Vec::new(); + } + let heights = terrain + .iter() + .flat_map(|patch| patch.data.iter().copied()) + .filter(|value| value.is_finite()) + .collect::>(); + let Some((&minimum, &maximum_height)) = heights + .iter() + .min_by(|a, b| a.total_cmp(b)) + .zip(heights.iter().max_by(|a, b| a.total_cmp(b))) + else { + return Vec::new(); + }; + let mut step = 4usize; + while width.div_ceil(step) * height.div_ceil(step) * 2 > maximum { + step *= 2; + } + let color = |x: usize, y: usize, z: f32| { + let range = (maximum_height - minimum).max(1.0); + let level = ((z - minimum) / range).clamp(0.0, 0.999); + let texture = textures.get(&texture_ids[(level * 4.0) as usize]); + if let Some(texture) = texture { + texture.sample(x as f32 / 128.0, y as f32 / 128.0) + } else if z <= water { + [45, 90, 125, 255] + } else { + let light = unit_byte(0.65 + level * 0.3); + [light / 2, light, light / 3, 255] + } + }; + let mut output = Vec::new(); + for y in (0..height.saturating_sub(1)).step_by(step) { + for x in (0..width.saturating_sub(1)).step_by(step) { + if output.len().saturating_add(2) > maximum { + return output; + } + let x1 = (x + step).min(width - 1); + let y1 = (y + step).min(height - 1); + let Some(h00) = terrain_height(terrain, patch_width, x, y) else { + continue; + }; + let Some(h10) = terrain_height(terrain, patch_width, x1, y) else { + continue; + }; + let Some(h11) = terrain_height(terrain, patch_width, x1, y1) else { + continue; + }; + let Some(h01) = terrain_height(terrain, patch_width, x, y1) else { + continue; + }; + let p00 = [x as f32, y as f32, h00]; + let p10 = [x1 as f32, y as f32, h10]; + let p11 = [x1 as f32, y1 as f32, h11]; + let p01 = [x as f32, y1 as f32, h01]; + let first_vertices = [p00, p10, p11]; + let second_vertices = [p00, p11, p01]; + output.push(SceneTriangle { + vertices: first_vertices, + colors_srgb: [color(x, y, h00), color(x1, y, h10), color(x1, y1, h11)] + .map(|color| shade_color(color, triangle_shade(first_vertices))), + }); + output.push(SceneTriangle { + vertices: second_vertices, + colors_srgb: [color(x, y, h00), color(x1, y1, h11), color(x, y1, h01)] + .map(|color| shade_color(color, triangle_shade(second_vertices))), + }); + } + } + output +} + +#[cfg(feature = "live-grid")] +pub(crate) fn triangle_shade(vertices: [[f32; 3]; 3]) -> f32 { + let edge_a = std::array::from_fn(|axis| vertices[1][axis] - vertices[0][axis]); + let edge_b = std::array::from_fn(|axis| vertices[2][axis] - vertices[0][axis]); + normalize(cross(edge_a, edge_b)).map_or(0.6, |normal| { + 0.35 + 0.65 * dot(normal, [-0.3, -0.4, 0.866]).abs() + }) +} + +#[cfg(feature = "live-grid")] +pub(crate) fn shade_color(mut color: [u8; 4], shade: f32) -> [u8; 4] { + for channel in &mut color[..3] { + *channel = (f32::from(*channel) * shade.clamp(0.25, 1.0)).round() as u8; + } + color +} + +#[cfg(feature = "live-grid")] +pub(crate) fn terrain_layout( + terrain: &[libremetaverse::TerrainPatch], + declared_size: (f32, f32), +) -> (usize, usize, usize) { + let extended = terrain.len() > 256; + let patch_width = if extended { + 256 + } else { + (declared_size.0.max(16.0) as usize).div_ceil(16) + }; + let (received_width, received_height) = terrain + .iter() + .filter(|patch| !patch.data.is_empty()) + .fold((0usize, 0usize), |(width, height), patch| { + let x = usize::try_from(patch.x).unwrap_or_default(); + let y = usize::try_from(patch.y).unwrap_or_default(); + (width.max((x + 1) * 16), height.max((y + 1) * 16)) + }); + ( + (declared_size.0.max(0.0) as usize).max(received_width), + (declared_size.1.max(0.0) as usize).max(received_height), + patch_width, + ) +} #[cfg(feature = "live-grid")] fn unit_byte(value: f32) -> u8 { (value.clamp(0.0, 1.0) * 255.0).round() as u8 diff --git a/crates/metacrate-grid-agent/src/vision_tests.rs b/crates/metacrate-grid-agent/src/vision_tests.rs index 4f3a7f8..5c361c0 100644 --- a/crates/metacrate-grid-agent/src/vision_tests.rs +++ b/crates/metacrate-grid-agent/src/vision_tests.rs @@ -35,9 +35,76 @@ fn uuid(value: u32) -> UUID { fn triangle(z: f32, color: [u8; 4]) -> SceneTriangle { SceneTriangle { vertices: [[-1.0, -1.0, z], [1.0, -1.0, z], [0.0, 1.0, z]], - color_srgb: color, + colors_srgb: [color; 3], } } + +#[cfg(feature = "live-grid")] +#[test] +fn extended_terrain_layout_uses_received_patch_coordinates() { + let mut terrain = (0..65_536) + .map(|_| libremetaverse::TerrainPatch { + data: Vec::new(), + x: 0, + y: 0, + }) + .collect::>(); + terrain[33 * 256 + 42] = libremetaverse::TerrainPatch { + data: vec![20.0; 256], + x: 42, + y: 33, + }; + assert_eq!(terrain_layout(&terrain, (256.0, 256.0)), (688, 544, 256)); + assert_eq!(terrain_height(&terrain, 256, 675, 538), Some(20.0)); +} + +#[cfg(feature = "live-grid")] +#[test] +fn renderer_uses_the_camera_axis_that_points_at_the_requested_target() { + let camera = libremetaverse::AgentManagerAgentMovementAgentCamera::new().unwrap(); + camera + .look_at_with_vector3_vector3( + libremetaverse_types::Vector3::zero(), + libremetaverse_types::Vector3::unit_y(), + ) + .unwrap(); + assert_eq!( + native_camera_forward(&camera), + libremetaverse_types::Vector3::unit_y() + ); +} + +#[cfg(feature = "live-grid")] +#[test] +fn scene_faces_are_shaded_without_changing_alpha() { + let shade = triangle_shade([[0.0, 0.0, 0.0], [1.0, 0.0, 0.0], [0.0, 1.0, 0.0]]); + let color = shade_color([200, 160, 120, 77], shade); + assert!(color[0] < 200); + assert_eq!(color[3], 77); +} + +#[cfg(feature = "live-grid")] +#[test] +fn scene_broad_phase_rejects_object_centers_behind_the_camera() { + assert!(scene_center_in_front( + libremetaverse_types::Vector3 { + x: 4.0, + y: 0.0, + z: 0.0 + }, + [0.0; 3], + [1.0, 0.0, 0.0] + )); + assert!(!scene_center_in_front( + libremetaverse_types::Vector3 { + x: -4.0, + y: 0.0, + z: 0.0 + }, + [0.0; 3], + [1.0, 0.0, 0.0] + )); +} fn scene() -> SceneSnapshot { SceneSnapshot { generation: 7, @@ -123,7 +190,7 @@ async fn golden_scene_is_deterministic_depth_ordered_and_privacy_marked() { #[cfg(feature = "live-grid")] #[tokio::test(flavor = "multi_thread")] async fn offscreen_wgpu_renders_depth_ordered_scene_when_an_adapter_is_available() { - let Ok(renderer) = WgpuRenderer::new().await else { + let Ok(renderer) = metacrate_rendering_wgpu::Renderer::new().await else { return; }; let limits = VisionLimits { @@ -132,10 +199,28 @@ async fn offscreen_wgpu_renders_depth_ordered_scene_when_an_adapter_is_available minimum_interval: std::time::Duration::ZERO, ..VisionLimits::default() }; - let rgba = tokio::task::spawn_blocking(move || renderer.render(&scene(), limits)) - .await - .expect("render worker") - .expect("offscreen render"); + let scene = scene(); + let triangles = scene + .entities + .iter() + .flat_map(|entity| entity.triangles.iter().cloned()) + .collect::>(); + let rgba = tokio::task::spawn_blocking(move || { + renderer.render( + scene.camera, + &triangles, + [72, 96, 120, 255], + metacrate_rendering_wgpu::RenderLimits { + width: limits.width, + height: limits.height, + max_triangles: limits.max_triangles, + far_distance: 512, + }, + ) + }) + .await + .expect("render worker") + .expect("offscreen render"); assert_eq!(rgba.len(), 64 * 64 * 4); let center = (32 * 64 + 32) * 4; assert!(rgba[center + 1] > 180 && rgba[center] < 80 && rgba[center + 2] < 80); diff --git a/crates/metacrate-grid-agent/tests/dependency_policy.rs b/crates/metacrate-grid-agent/tests/dependency_policy.rs index cff148c..a7bf4d6 100644 --- a/crates/metacrate-grid-agent/tests/dependency_policy.rs +++ b/crates/metacrate-grid-agent/tests/dependency_policy.rs @@ -10,6 +10,7 @@ const ALLOWED_DEPENDENCIES: [&str; 20] = [ "libremetaverse-imaging", "libremetaverse-rendering-simple", "metacrate-lsl-tools", + "metacrate-rendering-wgpu", "mentra", "jpeg-encoder", "libremetaverse-types", @@ -22,7 +23,6 @@ const ALLOWED_DEPENDENCIES: [&str; 20] = [ "tokio-rustls", "url", "unicode-width", - "wgpu", ]; #[test] @@ -101,7 +101,10 @@ fn vision_path_selects_only_reviewed_rust_renderers_and_codec() { .expect("simple renderer manifest"); assert!(!renderer.contains("imaging-skia")); assert!(!renderer.contains("openjpeg")); - assert!(manifest.contains("wgpu = { version = \"30.0.1\"")); + assert!(manifest.contains("metacrate-rendering-wgpu =")); + let renderer = fs::read_to_string(root.join("../metacrate-rendering-wgpu/Cargo.toml")) + .expect("wgpu renderer manifest"); + assert!(renderer.contains("wgpu = { version = \"30.0.1\"")); } #[test] diff --git a/crates/metacrate-rendering-wgpu/Cargo.toml b/crates/metacrate-rendering-wgpu/Cargo.toml new file mode 100644 index 0000000..18ab200 --- /dev/null +++ b/crates/metacrate-rendering-wgpu/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "metacrate-rendering-wgpu" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +description = "Reusable headless wgpu scene renderer and projection camera" +publish = false + +[dependencies] +wgpu = { version = "30.0.1", default-features = false, features = ["std", "dx12", "metal", "gles", "vulkan", "wgsl"], optional = true } + +[features] +default = [] +wgpu = ["dep:wgpu"] + +[lints] +workspace = true diff --git a/crates/metacrate-rendering-wgpu/src/lib.rs b/crates/metacrate-rendering-wgpu/src/lib.rs new file mode 100644 index 0000000..7aaad78 --- /dev/null +++ b/crates/metacrate-rendering-wgpu/src/lib.rs @@ -0,0 +1,426 @@ +//! Cross-platform projection camera and headless `wgpu` triangle renderer. + +#![allow(clippy::missing_errors_doc)] + +type CameraBasis = ([f32; 3], [f32; 3], [f32; 3]); + +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct Camera { + pub position: [f32; 3], + pub forward: [f32; 3], + pub up: [f32; 3], + pub vertical_fov_degrees: f32, +} + +impl Camera { + pub fn look_at( + position: [f32; 3], + target: [f32; 3], + up: [f32; 3], + vertical_fov_degrees: f32, + ) -> Result { + let forward = normalize([ + target[0] - position[0], + target[1] - position[1], + target[2] - position[2], + ])?; + let camera = Self { + position, + forward, + up: normalize(up)?, + vertical_fov_degrees, + }; + camera.basis()?; + Ok(camera) + } + + fn basis(self) -> Result { + if !(10.0..=140.0).contains(&self.vertical_fov_degrees) + || self + .position + .iter() + .chain(self.forward.iter()) + .chain(self.up.iter()) + .any(|value| !value.is_finite()) + { + return Err(RenderError::InvalidScene); + } + let forward = normalize(self.forward)?; + let right = normalize(cross(forward, self.up))?; + Ok((right, cross(right, forward), forward)) + } +} + +#[derive(Clone, Debug, PartialEq)] +pub struct Triangle { + pub vertices: [[f32; 3]; 3], + pub colors_srgb: [[u8; 4]; 3], +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct RenderLimits { + pub width: u32, + pub height: u32, + pub max_triangles: usize, + pub far_distance: u32, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RenderError { + AdapterUnavailable, + InvalidScene, + ResourceLimit, + TimedOut, + Readback, +} + +impl std::fmt::Display for RenderError { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str(match self { + Self::AdapterUnavailable => "wgpu adapter is unavailable", + Self::InvalidScene => "render scene is invalid", + Self::ResourceLimit => "render resource limit exceeded", + Self::TimedOut => "wgpu render timed out", + Self::Readback => "wgpu readback failed", + }) + } +} + +impl std::error::Error for RenderError {} + +fn normalize(value: [f32; 3]) -> Result<[f32; 3], RenderError> { + let length = dot(value, value).sqrt(); + if length < 0.0001 { + return Err(RenderError::InvalidScene); + } + Ok([value[0] / length, value[1] / length, value[2] / length]) +} + +fn dot(left: [f32; 3], right: [f32; 3]) -> f32 { + left[0] * right[0] + left[1] * right[1] + left[2] * right[2] +} + +fn cross(left: [f32; 3], right: [f32; 3]) -> [f32; 3] { + [ + left[1] * right[2] - left[2] * right[1], + left[2] * right[0] - left[0] * right[2], + left[0] * right[1] - left[1] * right[0], + ] +} + +#[cfg(feature = "wgpu")] +pub struct Renderer { + device: wgpu::Device, + queue: wgpu::Queue, + pipeline: wgpu::RenderPipeline, +} + +#[cfg(feature = "wgpu")] +const VERTEX_ATTRIBUTES: [wgpu::VertexAttribute; 2] = + wgpu::vertex_attr_array![0 => Float32x3, 1 => Float32x4]; + +#[cfg(feature = "wgpu")] +impl Renderer { + pub async fn new() -> Result { + let instance = wgpu::Instance::new(wgpu::InstanceDescriptor::new_without_display_handle()); + let adapter = instance + .request_adapter(&wgpu::RequestAdapterOptions { + power_preference: wgpu::PowerPreference::LowPower, + ..Default::default() + }) + .await + .map_err(|_| RenderError::AdapterUnavailable)?; + let (device, queue) = adapter + .request_device(&wgpu::DeviceDescriptor { + label: Some("metacrate-rendering-wgpu"), + required_features: wgpu::Features::empty(), + required_limits: wgpu::Limits::downlevel_defaults(), + memory_hints: wgpu::MemoryHints::MemoryUsage, + ..Default::default() + }) + .await + .map_err(|_| RenderError::AdapterUnavailable)?; + let shader = device.create_shader_module(wgpu::ShaderModuleDescriptor { + label: Some("metacrate-rendering-wgpu-shader"), + source: wgpu::ShaderSource::Wgsl( + r" +struct VertexInput { @location(0) position: vec3, @location(1) color: vec4, }; +struct VertexOutput { @builtin(position) position: vec4, @location(0) color: vec4, }; +@vertex fn vs_main(input: VertexInput) -> VertexOutput { + var output: VertexOutput; + output.position = vec4(input.position, 1.0); + output.color = input.color; + return output; +} +@fragment fn fs_main(input: VertexOutput) -> @location(0) vec4 { return input.color; } +" + .into(), + ), + }); + let layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { + label: Some("metacrate-rendering-wgpu-layout"), + bind_group_layouts: &[], + immediate_size: 0, + }); + let pipeline = device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { + label: Some("metacrate-rendering-wgpu-pipeline"), + layout: Some(&layout), + vertex: wgpu::VertexState { + module: &shader, + entry_point: Some("vs_main"), + compilation_options: wgpu::PipelineCompilationOptions::default(), + buffers: &[Some(wgpu::VertexBufferLayout { + array_stride: 7 * std::mem::size_of::() as u64, + step_mode: wgpu::VertexStepMode::Vertex, + attributes: &VERTEX_ATTRIBUTES, + })], + }, + primitive: wgpu::PrimitiveState { + topology: wgpu::PrimitiveTopology::TriangleList, + cull_mode: None, + ..Default::default() + }, + depth_stencil: Some(wgpu::DepthStencilState { + format: wgpu::TextureFormat::Depth32Float, + depth_write_enabled: Some(true), + depth_compare: Some(wgpu::CompareFunction::Less), + stencil: wgpu::StencilState::default(), + bias: wgpu::DepthBiasState::default(), + }), + multisample: wgpu::MultisampleState::default(), + fragment: Some(wgpu::FragmentState { + module: &shader, + entry_point: Some("fs_main"), + compilation_options: wgpu::PipelineCompilationOptions::default(), + targets: &[Some(wgpu::ColorTargetState { + format: wgpu::TextureFormat::Rgba8UnormSrgb, + blend: None, + write_mask: wgpu::ColorWrites::ALL, + })], + }), + multiview_mask: None, + cache: None, + }); + Ok(Self { + device, + queue, + pipeline, + }) + } + + #[allow(clippy::too_many_lines)] + pub fn render( + &self, + camera: Camera, + triangles: &[Triangle], + background_srgb: [u8; 4], + limits: RenderLimits, + ) -> Result, RenderError> { + let (vertices, vertex_count) = gpu_vertices(camera, triangles, limits)?; + let vertex_buffer = self.device.create_buffer(&wgpu::BufferDescriptor { + label: Some("metacrate-rendering-wgpu-vertices"), + size: vertices.len().max(4) as u64, + usage: wgpu::BufferUsages::VERTEX | wgpu::BufferUsages::COPY_DST, + mapped_at_creation: false, + }); + if !vertices.is_empty() { + self.queue.write_buffer(&vertex_buffer, 0, &vertices); + } + let extent = wgpu::Extent3d { + width: limits.width, + height: limits.height, + depth_or_array_layers: 1, + }; + let color = self.device.create_texture(&wgpu::TextureDescriptor { + label: Some("metacrate-rendering-wgpu-color"), + size: extent, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: wgpu::TextureFormat::Rgba8UnormSrgb, + usage: wgpu::TextureUsages::RENDER_ATTACHMENT | wgpu::TextureUsages::COPY_SRC, + view_formats: &[], + }); + let depth = self.device.create_texture(&wgpu::TextureDescriptor { + label: Some("metacrate-rendering-wgpu-depth"), + size: extent, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: wgpu::TextureFormat::Depth32Float, + usage: wgpu::TextureUsages::RENDER_ATTACHMENT, + view_formats: &[], + }); + let row_bytes = limits + .width + .checked_mul(4) + .ok_or(RenderError::ResourceLimit)?; + let padded_row_bytes = row_bytes.div_ceil(wgpu::COPY_BYTES_PER_ROW_ALIGNMENT) + * wgpu::COPY_BYTES_PER_ROW_ALIGNMENT; + let readback = self.device.create_buffer(&wgpu::BufferDescriptor { + label: Some("metacrate-rendering-wgpu-readback"), + size: u64::from(padded_row_bytes) + .checked_mul(u64::from(limits.height)) + .ok_or(RenderError::ResourceLimit)?, + usage: wgpu::BufferUsages::COPY_DST | wgpu::BufferUsages::MAP_READ, + mapped_at_creation: false, + }); + let color_view = color.create_view(&wgpu::TextureViewDescriptor::default()); + let depth_view = depth.create_view(&wgpu::TextureViewDescriptor::default()); + let mut encoder = self + .device + .create_command_encoder(&wgpu::CommandEncoderDescriptor { + label: Some("metacrate-rendering-wgpu-commands"), + }); + { + let mut pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("metacrate-rendering-wgpu-pass"), + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view: &color_view, + depth_slice: None, + resolve_target: None, + ops: wgpu::Operations { + load: wgpu::LoadOp::Clear(wgpu::Color { + r: f64::from(background_srgb[0]) / 255.0, + g: f64::from(background_srgb[1]) / 255.0, + b: f64::from(background_srgb[2]) / 255.0, + a: f64::from(background_srgb[3]) / 255.0, + }), + store: wgpu::StoreOp::Store, + }, + })], + depth_stencil_attachment: Some(wgpu::RenderPassDepthStencilAttachment { + view: &depth_view, + depth_ops: Some(wgpu::Operations { + load: wgpu::LoadOp::Clear(1.0), + store: wgpu::StoreOp::Discard, + }), + stencil_ops: None, + }), + ..Default::default() + }); + if vertex_count != 0 { + pass.set_pipeline(&self.pipeline); + pass.set_vertex_buffer(0, vertex_buffer.slice(..)); + pass.draw(0..vertex_count, 0..1); + } + } + encoder.copy_texture_to_buffer( + wgpu::TexelCopyTextureInfo { + texture: &color, + mip_level: 0, + origin: wgpu::Origin3d::ZERO, + aspect: wgpu::TextureAspect::All, + }, + wgpu::TexelCopyBufferInfo { + buffer: &readback, + layout: wgpu::TexelCopyBufferLayout { + offset: 0, + bytes_per_row: Some(padded_row_bytes), + rows_per_image: Some(limits.height), + }, + }, + extent, + ); + let submission = self.queue.submit([encoder.finish()]); + let slice = readback.slice(..); + let (sender, receiver) = std::sync::mpsc::sync_channel(1); + slice.map_async(wgpu::MapMode::Read, move |result| { + let _ = sender.send(result); + }); + self.device + .poll(wgpu::PollType::Wait { + submission_index: Some(submission), + timeout: Some(std::time::Duration::from_secs(5)), + }) + .map_err(|_| RenderError::TimedOut)?; + receiver + .recv_timeout(std::time::Duration::from_secs(1)) + .map_err(|_| RenderError::TimedOut)? + .map_err(|_| RenderError::Readback)?; + let mapped = slice + .get_mapped_range() + .map_err(|_| RenderError::Readback)?; + let mut rgba = Vec::with_capacity(row_bytes as usize * limits.height as usize); + for row in mapped + .chunks_exact(padded_row_bytes as usize) + .take(limits.height as usize) + { + rgba.extend_from_slice(&row[..row_bytes as usize]); + } + drop(mapped); + readback.unmap(); + Ok(rgba) + } +} + +#[cfg(feature = "wgpu")] +#[allow(clippy::cast_precision_loss)] // Render dimensions and distance are bounded well below f32's exact integer range. +fn gpu_vertices( + camera: Camera, + triangles: &[Triangle], + limits: RenderLimits, +) -> Result<(Vec, u32), RenderError> { + if limits.width == 0 + || limits.height == 0 + || limits.far_distance == 0 + || triangles.len() > limits.max_triangles + || triangles + .iter() + .flat_map(|triangle| triangle.vertices.iter().flatten()) + .any(|value| !value.is_finite()) + { + return Err(RenderError::InvalidScene); + } + let basis = camera.basis()?; + let aspect = limits.width as f32 / limits.height as f32; + let focal = 1.0 / (camera.vertical_fov_degrees.to_radians() / 2.0).tan(); + let mut bytes = Vec::with_capacity(triangles.len() * 3 * 7 * std::mem::size_of::()); + let mut count = 0u32; + for triangle in triangles { + let projected = triangle.vertices.map(|world| { + let delta = [ + world[0] - camera.position[0], + world[1] - camera.position[1], + world[2] - camera.position[2], + ]; + let z = dot(delta, basis.2); + [ + dot(delta, basis.0) * focal / aspect / z, + dot(delta, basis.1) * focal / z, + (z / limits.far_distance as f32).clamp(0.0, 1.0), + z, + ] + }); + if projected.iter().any(|vertex| vertex[3] <= 0.05) { + continue; + } + for (vertex, color) in projected.into_iter().zip(triangle.colors_srgb) { + let color = color.map(|channel| f32::from(channel) / 255.0); + for value in vertex[..3].iter().chain(color.iter()) { + bytes.extend_from_slice(&value.to_ne_bytes()); + } + count = count.checked_add(1).ok_or(RenderError::ResourceLimit)?; + } + } + Ok((bytes, count)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn look_at_builds_a_valid_projection_camera() { + let camera = + Camera::look_at([1.0, 2.0, 3.0], [5.0, 2.0, 3.0], [0.0, 0.0, 1.0], 60.0).unwrap(); + assert!( + camera + .forward + .into_iter() + .zip([1.0, 0.0, 0.0]) + .all(|(actual, expected)| (actual - expected).abs() < f32::EPSILON) + ); + assert!(Camera::look_at([0.0; 3], [0.0; 3], [0.0, 0.0, 1.0], 60.0).is_err()); + } +} diff --git a/programs/src/test_client.rs b/programs/src/test_client.rs index ebb9ff6..e3dd83a 100644 --- a/programs/src/test_client.rs +++ b/programs/src/test_client.rs @@ -39,7 +39,7 @@ const MAX_INPUT_LINES: usize = 4096; const MAX_CLIENTS: usize = 64; const MAX_COMMAND_BYTES: usize = 4096; const MAX_MESSAGE_BYTES: usize = 1023; -const EVENT_QUEUE_CAPACITY: usize = 512; +const EVENT_QUEUE_CAPACITY: usize = 16_384; const MAX_PACKET_LOG_COUNT: usize = 10_000; const MAX_PACKET_LOG_BYTES: u64 = 16 * 1024 * 1024; const MAX_LOADED_COMMANDS: usize = 128;