From c101007362acac05495d263ee955bef92632d52b Mon Sep 17 00:00:00 2001 From: Chili Palmer Date: Sat, 22 Aug 2026 11:36:34 +0200 Subject: [PATCH] Add headless wgpu vision and camera controls --- Cargo.lock | 460 +++++++++++++++++- crates/libremetaverse/src/agent_movement.rs | 40 +- crates/metacrate-grid-agent/Cargo.toml | 3 +- crates/metacrate-grid-agent/src/backend.rs | 76 +++ crates/metacrate-grid-agent/src/behavior.rs | 132 ++++- .../src/behavior_tests.rs | 77 ++- .../src/control_runtime.rs | 2 +- .../metacrate-grid-agent/src/interaction.rs | 17 +- .../src/interaction_tests.rs | 20 + crates/metacrate-grid-agent/src/lib.rs | 6 +- crates/metacrate-grid-agent/src/main.rs | 11 +- crates/metacrate-grid-agent/src/vision.rs | 378 +++++++++++++- .../metacrate-grid-agent/src/vision_tests.rs | 23 +- .../tests/dependency_policy.rs | 13 +- docs/grid-agent-acceptance.md | 5 +- docs/grid-agent-architecture.md | 8 +- docs/grid-agent-llm.md | 22 +- docs/grid-agent-operations.md | 11 +- docs/grid-agent-release-evidence.md | 2 +- 19 files changed, 1250 insertions(+), 56 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d4f7606..f0512fb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -164,6 +164,15 @@ version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" +[[package]] +name = "ash" +version = "0.38.0+1.3.281" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bb44936d800fea8f016d7f2311c6a4f97aebd5dc86f09906139ec848cf3a46f" +dependencies = [ + "libloading", +] + [[package]] name = "asn1-rs" version = "0.7.2" @@ -303,6 +312,15 @@ dependencies = [ "bit-vec 0.8.0", ] +[[package]] +name = "bit-set" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2f926cc3060f09db9ebc5b52823d85268d24bb917e472c0c4bea35780a7d" +dependencies = [ + "bit-vec 0.9.1", +] + [[package]] name = "bit-vec" version = "0.8.0" @@ -568,6 +586,15 @@ dependencies = [ "unicode-width 0.1.14", ] +[[package]] +name = "codespan-reporting" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af491d569909a7e4dee0ad7db7f5341fef5c614d5b8ec8cf765732aba3cff681" +dependencies = [ + "unicode-width 0.2.2", +] + [[package]] name = "color_quant" version = "1.1.0" @@ -765,6 +792,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + [[package]] name = "crypto-bigint" version = "0.5.5" @@ -1015,6 +1048,15 @@ dependencies = [ "syn 3.0.3", ] +[[package]] +name = "dlib" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab8ecd87370524b461f8557c119c405552c396ed91fc0a8eec68679eab26f94a" +dependencies = [ + "libloading", +] + [[package]] name = "document-features" version = "0.2.12" @@ -1379,6 +1421,17 @@ dependencies = [ "weezl", ] +[[package]] +name = "gl_generator" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a95dfc23a2b4a9a2f5ab41d194f8bfda3cabec42af4e39f08c339eb2a0c124d" +dependencies = [ + "khronos_api", + "log", + "xml-rs", +] + [[package]] name = "glam" version = "0.29.3" @@ -1397,6 +1450,27 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9985c9503b412198aa4197559e9a318524ebc4519c229bfa05a535828c950b9d" +[[package]] +name = "glow" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29038e1c483364cc6bb3cf78feee1816002e127c331a1eec55a4d202b9e1adb5" +dependencies = [ + "js-sys", + "slotmap", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "glutin_wgl_sys" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c4ee00b289aba7a9e5306d57c2d05499b2e5dc427f84ac708bd2c090212cf3e" +dependencies = [ + "gl_generator", +] + [[package]] name = "goblin" version = "0.10.5" @@ -1408,6 +1482,20 @@ dependencies = [ "scroll", ] +[[package]] +name = "gpu-allocator" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51255ea7cfaadb6c5f1528d43e92a82acb2b96c43365989a28b2d44ee38f8795" +dependencies = [ + "ash", + "hashbrown 0.16.1", + "log", + "presser", + "thiserror 2.0.20", + "windows", +] + [[package]] name = "group" version = "0.13.0" @@ -1419,6 +1507,18 @@ dependencies = [ "subtle", ] +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "num-traits", + "zerocopy", +] + [[package]] name = "hash32" version = "0.3.1" @@ -1444,6 +1544,9 @@ name = "hashbrown" version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "foldhash", +] [[package]] name = "hashlink" @@ -1974,6 +2077,23 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "khronos-egl" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6aae1df220ece3c0ada96b8153459b67eebe9ae9212258bb0134ae60416fdf76" +dependencies = [ + "libc", + "libloading", + "pkg-config", +] + +[[package]] +name = "khronos_api" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2db585e1d738fc771bf08a151420d3ed193d9d895a36df7f6f8a9456b911ddc" + [[package]] name = "lazy_static" version = "1.5.0" @@ -2424,6 +2544,7 @@ dependencies = [ "tokio-rustls", "unicode-width 0.2.2", "url", + "wgpu", ] [[package]] @@ -2496,20 +2617,58 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "364f94bc34f61332abebe8cad6f6cd82a5b65cff22c828d05d0968911462ca4f" dependencies = [ "arrayvec", - "bit-set", + "bit-set 0.8.0", "bitflags 2.13.1", "cfg_aliases 0.1.1", - "codespan-reporting", + "codespan-reporting 0.11.1", "hexf-parse", "indexmap", "log", "rustc-hash 1.1.0", - "spirv", + "spirv 0.3.0+sdk-1.3.268.0", "termcolor", "thiserror 1.0.69", "unicode-xid", ] +[[package]] +name = "naga" +version = "30.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a616d2fb8c89516ac2723a581f69d6c18576046bed761bd6b305e5618e6ae130" +dependencies = [ + "arrayvec", + "bit-set 0.10.0", + "bitflags 2.13.1", + "cfg-if", + "cfg_aliases 0.2.2", + "codespan-reporting 0.13.1", + "half", + "hashbrown 0.17.1", + "indexmap", + "libm", + "log", + "naga-types", + "num-traits", + "once_cell", + "rustc-hash 1.1.0", + "spirv 0.4.0+sdk-1.4.341.0", + "thiserror 2.0.20", + "unicode-ident", +] + +[[package]] +name = "naga-types" +version = "30.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "590afbf58a6f4f62873cd5cff4468061844bafa1cdf399cc954537c22d768d49" +dependencies = [ + "hashbrown 0.17.1", + "indexmap", + "rustc-hash 1.1.0", + "thiserror 2.0.20", +] + [[package]] name = "ndk" version = "0.9.0" @@ -2818,6 +2977,18 @@ dependencies = [ "objc2-core-foundation", ] +[[package]] +name = "objc2-metal" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0125f776a10d00af4152d74616409f0d4a2053a6f57fa5b7d6aa2854ac04794" +dependencies = [ + "bitflags 2.13.1", + "block2", + "objc2", + "objc2-foundation", +] + [[package]] name = "objc2-quartz-core" version = "0.3.2" @@ -2827,7 +2998,9 @@ dependencies = [ "bitflags 2.13.1", "objc2", "objc2-core-foundation", + "objc2-core-graphics", "objc2-foundation", + "objc2-metal", ] [[package]] @@ -2909,6 +3082,15 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" +[[package]] +name = "ordered-float" +version = "5.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c7c9e0d9b23589f26070720bac724174bfec1083e82f7854cdd0267518343c0" +dependencies = [ + "num-traits", +] + [[package]] name = "os_info" version = "3.15.0" @@ -3050,6 +3232,21 @@ dependencies = [ "universal-hash", ] +[[package]] +name = "portable-atomic" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" + +[[package]] +name = "portable-atomic-util" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" +dependencies = [ + "portable-atomic", +] + [[package]] name = "potential_utf" version = "0.1.5" @@ -3074,6 +3271,12 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "presser" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8cf8e6a8aa66ce33f63993ffc4ea4271eb5b0530a9002db8455ea6050c77bfa" + [[package]] name = "prettyplease" version = "0.2.37" @@ -3111,6 +3314,12 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "profiling" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d595e54a326bc53c1c197b32d295e14b169e3cfeaa8dc82b529f947fba6bcf5" + [[package]] name = "pxfm" version = "0.1.30" @@ -3265,6 +3474,30 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "range-alloc" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca45419789ae5a7899559e9512e58ca889e41f04f1f2445e9f4b290ceccd1d08" + +[[package]] +name = "raw-window-handle" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20675572f6f24e9e76ef639bc5552774ed45f1c30e2951e1e99c59888861c539" + +[[package]] +name = "raw-window-metal" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40d213455a5f1dc59214213c7330e074ddf8114c9a42411eb890c767357ce135" +dependencies = [ + "objc2", + "objc2-core-foundation", + "objc2-foundation", + "objc2-quartz-core", +] + [[package]] name = "rayon" version = "1.12.0" @@ -3347,6 +3580,12 @@ version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" +[[package]] +name = "renderdoc-sys" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19b30a45b0cd0bcca8037f3d0dc3421eaf95327a17cad11964fb8179b4fc4832" + [[package]] name = "reqwest" version = "0.12.28" @@ -3985,7 +4224,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f4b7ea0fc7fa887d5875e49809eabc3cbd9e6a0247930a9f6456ad4c59cedfb1" dependencies = [ "bitflags 2.13.1", - "naga", + "naga 23.1.0", "skia-rs-core", "skia-rs-path", "thiserror 2.0.20", @@ -4032,6 +4271,15 @@ version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" +[[package]] +name = "slotmap" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdd58c3c93c3d278ca835519292445cb4b0d4dc59ccfdf7ceadaab3f8aeb4038" +dependencies = [ + "version_check", +] + [[package]] name = "smallvec" version = "1.15.2" @@ -4069,6 +4317,15 @@ dependencies = [ "bitflags 2.13.1", ] +[[package]] +name = "spirv" +version = "0.4.0+sdk-1.4.341.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9571ea910ebd84c86af4b3ed27f9dbdc6ad06f17c5f96146b2b671e2976744f" +dependencies = [ + "bitflags 2.13.1", +] + [[package]] name = "spki" version = "0.7.3" @@ -4097,6 +4354,12 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + [[package]] name = "stats_alloc" version = "0.1.10" @@ -4795,6 +5058,18 @@ dependencies = [ "web-sys", ] +[[package]] +name = "wayland-sys" +version = "0.31.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8eab23fefc9e41f8e841df4a9c707e8a8c4ed26e944ef69297184de2785e3be" +dependencies = [ + "dlib", + "log", + "once_cell", + "pkg-config", +] + [[package]] name = "web-sys" version = "0.3.104" @@ -4839,6 +5114,177 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88" +[[package]] +name = "wgpu" +version = "30.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "527ccdf43dd5b2e8676eed9984ce00e2bbb0a1b85b70c1969dcb6cd2eb55ab9e" +dependencies = [ + "arrayvec", + "bitflags 2.13.1", + "bytemuck", + "cfg-if", + "cfg_aliases 0.2.2", + "document-features", + "hashbrown 0.17.1", + "js-sys", + "log", + "portable-atomic", + "profiling", + "raw-window-handle", + "smallvec", + "static_assertions", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "wgpu-core", + "wgpu-hal", + "wgpu-types", +] + +[[package]] +name = "wgpu-core" +version = "30.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14c018fce9b6270aa203c2fdd56f3cce996713534bd757e4ea58c8560b121f14" +dependencies = [ + "arrayvec", + "bit-set 0.10.0", + "bit-vec 0.9.1", + "bitflags 2.13.1", + "bytemuck", + "cfg_aliases 0.2.2", + "document-features", + "hashbrown 0.17.1", + "indexmap", + "log", + "naga 30.0.1", + "naga-types", + "once_cell", + "parking_lot", + "portable-atomic", + "profiling", + "raw-window-handle", + "rustc-hash 1.1.0", + "smallvec", + "thiserror 2.0.20", + "wgpu-core-deps-apple", + "wgpu-core-deps-emscripten", + "wgpu-core-deps-windows-linux-android", + "wgpu-hal", + "wgpu-naga-bridge", + "wgpu-types", +] + +[[package]] +name = "wgpu-core-deps-apple" +version = "30.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "061f3d319a40d39d00b1ecc2c33b89fe21d4e6fe01859df3500a3a8ecccd6b68" +dependencies = [ + "wgpu-hal", +] + +[[package]] +name = "wgpu-core-deps-emscripten" +version = "30.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98b86cf4abf524a902dd35f18ca6a3f08fc2ae9847c8f10b48e30491b1f0b86" +dependencies = [ + "wgpu-hal", +] + +[[package]] +name = "wgpu-core-deps-windows-linux-android" +version = "30.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7586165fd5f6d881cb9ce4bb71f40d6caab2c0f1837e3fc1d9788a197fb6004f" +dependencies = [ + "wgpu-hal", +] + +[[package]] +name = "wgpu-hal" +version = "30.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6b7fb58561a792bc237628ba0792e332de418fefe145f13b5ed8201e6d52f58" +dependencies = [ + "android_system_properties", + "arrayvec", + "ash", + "bit-set 0.10.0", + "bitflags 2.13.1", + "block2", + "bytemuck", + "cfg-if", + "cfg_aliases 0.2.2", + "glow", + "glutin_wgl_sys", + "gpu-allocator", + "hashbrown 0.17.1", + "js-sys", + "khronos-egl", + "libc", + "libloading", + "log", + "naga 30.0.1", + "naga-types", + "ndk-sys", + "objc2", + "objc2-core-foundation", + "objc2-core-graphics", + "objc2-foundation", + "objc2-metal", + "objc2-quartz-core", + "once_cell", + "ordered-float", + "parking_lot", + "portable-atomic", + "portable-atomic-util", + "profiling", + "range-alloc", + "raw-window-handle", + "raw-window-metal", + "renderdoc-sys", + "smallvec", + "static_assertions", + "thiserror 2.0.20", + "wasm-bindgen", + "wayland-sys", + "web-sys", + "wgpu-naga-bridge", + "wgpu-types", + "windows", + "windows-core", + "windows-result", +] + +[[package]] +name = "wgpu-naga-bridge" +version = "30.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f62e73117bb7a62bfd9c5a5841438a823f6566c6442a808ee269d2d055c081" +dependencies = [ + "naga 30.0.1", + "wgpu-types", +] + +[[package]] +name = "wgpu-types" +version = "30.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99dad6f1fbdbbdb4c278a6508b059d44688f5cebddf78d005a46a31340269286" +dependencies = [ + "bitflags 2.13.1", + "bytemuck", + "js-sys", + "log", + "naga-types", + "raw-window-handle", + "static_assertions", + "web-sys", +] + [[package]] name = "widestring" version = "1.2.1" @@ -5140,6 +5586,12 @@ dependencies = [ "rustix", ] +[[package]] +name = "xml-rs" +version = "0.8.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e450f9b2ed1dff33c94c12589a87338689467b9c4f5d8a5710bd09a847d2c8a7" + [[package]] name = "yasna" version = "0.6.0" diff --git a/crates/libremetaverse/src/agent_movement.rs b/crates/libremetaverse/src/agent_movement.rs index dd589d9..57a8c5b 100644 --- a/crates/libremetaverse/src/agent_movement.rs +++ b/crates/libremetaverse/src/agent_movement.rs @@ -399,6 +399,7 @@ impl CameraFrame { pub struct AgentManagerAgentMovementAgentCamera { pub far: f32, frame: Arc>, + vertical_fov_radians: Arc, } impl fmt::Debug for AgentManagerAgentMovementAgentCamera { @@ -406,6 +407,7 @@ impl fmt::Debug for AgentManagerAgentMovementAgentCamera { formatter .debug_struct("AgentCamera") .field("far", &self.far) + .field("vertical_fov_radians", &self.vertical_fov_angle()) .field("frame", &*mutex(&self.frame)) .finish() } @@ -416,11 +418,20 @@ impl AgentManagerAgentMovementAgentCamera { Ok(Self { far: 128.0, frame: Arc::new(Mutex::new(CameraFrame::default())), + vertical_fov_radians: Arc::new(AtomicU32::new(60.0_f32.to_radians().to_bits())), }) } - fn from_frame(frame: Arc>, far: f32) -> Self { - Self { far, frame } + fn from_frame( + frame: Arc>, + vertical_fov_radians: Arc, + far: f32, + ) -> Self { + Self { + far, + frame, + vertical_fov_radians, + } } pub fn look_at_with_vector3_vector3( @@ -584,6 +595,21 @@ impl AgentManagerAgentMovementAgentCamera { mutex(&self.frame).z_axis = value; } } + + #[must_use] + pub fn vertical_fov_angle(&self) -> f32 { + f32::from_bits(self.vertical_fov_radians.load(Ordering::Acquire)) + } + + pub fn set_vertical_fov_angle(&self, value: f32) -> Result<(), Error> { + if !value.is_finite() || !(10.0_f32.to_radians()..=140.0_f32.to_radians()).contains(&value) + { + return Err(Error::Argument); + } + self.vertical_fov_radians + .store(value.to_bits(), Ordering::Release); + Ok(()) + } } #[derive(Clone, Copy, Debug, PartialEq)] @@ -733,6 +759,7 @@ pub(crate) struct AgentMovementRuntime { auto_reset_controls: AtomicBool, update_interval: AtomicI32, camera_frame: Arc>, + vertical_fov_radians: Arc, last_update: Mutex, kinematics: RwLock, teleport: Mutex, @@ -805,6 +832,7 @@ impl AgentMovementRuntime { auto_reset_controls: AtomicBool::new(false), update_interval: AtomicI32::new(interval), camera_frame: Arc::new(Mutex::new(CameraFrame::default())), + vertical_fov_radians: Arc::new(AtomicU32::new(60.0_f32.to_radians().to_bits())), last_update: Mutex::new(LastUpdate::default()), kinematics: RwLock::new(Kinematics::default()), teleport: Mutex::new(TeleportState::default()), @@ -986,7 +1014,11 @@ impl AgentMovementRuntime { } fn camera(&self, far: f32) -> AgentManagerAgentMovementAgentCamera { - AgentManagerAgentMovementAgentCamera::from_frame(Arc::clone(&self.camera_frame), far) + AgentManagerAgentMovementAgentCamera::from_frame( + Arc::clone(&self.camera_frame), + Arc::clone(&self.vertical_fov_radians), + far, + ) } fn send_update( @@ -2099,6 +2131,7 @@ impl AgentManagerAgentMovement { } pub fn set_fov_vertical_angle(&self, angle: f32) -> Result<(), Error> { + self.camera.set_vertical_fov_angle(angle)?; let (agent_id, session_id) = self.runtime.ids(); let mut packet = AgentFOVPacket::new_with_constructor()?; packet.agent_data.agent_id = agent_id; @@ -3167,6 +3200,7 @@ mod tests { assert!(run.agent_data.always_run); manager.movement.set_fov_vertical_angle(1.25).unwrap(); + assert_eq!(manager.movement.camera.vertical_fov_angle(), 1.25); let fov = decode_packet::(&grid.receive(PacketType::AgentFOV)).unwrap(); assert_eq!(fov.fov_block.vertical_angle, 1.25); diff --git a/crates/metacrate-grid-agent/Cargo.toml b/crates/metacrate-grid-agent/Cargo.toml index 5099a4f..6849eb0 100644 --- a/crates/metacrate-grid-agent/Cargo.toml +++ b/crates/metacrate-grid-agent/Cargo.toml @@ -28,6 +28,7 @@ tokio = { version = "1.53.1", features = ["macros", "rt", "sync", "time"] } tokio-rustls = { version = "0.26.4", default-features = false, features = ["aws_lc_rs", "tls12"] } url = "2.5.8" unicode-width = "0.2" +wgpu = { version = "30.0.1", default-features = false, features = ["std", "dx12", "metal", "gles", "vulkan", "wgsl"], optional = true } [dev-dependencies] jpeg-decoder = { version = "0.3.2", default-features = false } @@ -38,7 +39,7 @@ tokio = { version = "1.53.1", features = ["io-util", "net", "rt-multi-thread", " [features] default = [] -live-grid = ["dep:libremetaverse", "dep:libremetaverse-imaging", "dep:libremetaverse-rendering-simple"] +live-grid = ["dep:libremetaverse", "dep:libremetaverse-imaging", "dep:libremetaverse-rendering-simple", "dep:wgpu"] [lints] workspace = true diff --git a/crates/metacrate-grid-agent/src/backend.rs b/crates/metacrate-grid-agent/src/backend.rs index 7b8dd9e..099331c 100644 --- a/crates/metacrate-grid-agent/src/backend.rs +++ b/crates/metacrate-grid-agent/src/backend.rs @@ -746,6 +746,82 @@ impl crate::behavior::EmbodimentSink for LibremetaverseEmbodimentSink { .map_err(|_| crate::behavior::BehaviorError::NativeOperation) }) } + + fn set_camera( + &self, + generation: u64, + position: crate::perception::WorldPosition, + target: crate::perception::WorldPosition, + vertical_fov_degrees: f64, + cancellation: CancellationToken, + ) -> crate::behavior::EmbodimentFuture<'_, ()> { + Box::pin(async move { + if cancellation.is_cancellation_requested() { + return Err(crate::behavior::BehaviorError::Cancelled); + } + self.ensure_generation(generation)?; + self.agent + .movement + .camera + .look_at_with_vector3_vector3( + libremetaverse_types::Vector3 { + x: position.x as f32, + y: position.y as f32, + z: position.z as f32, + }, + libremetaverse_types::Vector3 { + x: target.x as f32, + y: target.y as f32, + z: target.z as f32, + }, + ) + .map_err(|_| crate::behavior::BehaviorError::NativeOperation)?; + self.agent + .movement + .set_fov_vertical_angle((vertical_fov_degrees as f32).to_radians()) + .and_then(|()| self.agent.movement.send_update_with_boolean(Some(true))) + .map_err(|_| crate::behavior::BehaviorError::NativeOperation) + }) + } + + fn reset_camera( + &self, + generation: u64, + cancellation: CancellationToken, + ) -> crate::behavior::EmbodimentFuture<'_, ()> { + Box::pin(async move { + if cancellation.is_cancellation_requested() { + return Err(crate::behavior::BehaviorError::Cancelled); + } + self.ensure_generation(generation)?; + let avatar = self.agent.sim_position(); + let forward = libremetaverse_types::Vector3::mul_with_vector3_quaternion( + libremetaverse_types::Vector3::unit_x(), + self.agent.sim_rotation(), + ); + let position = libremetaverse_types::Vector3 { + x: avatar.x, + y: avatar.y, + z: avatar.z + 1.6, + }; + let target = libremetaverse_types::Vector3 { + x: position.x + forward.x * 4.0, + y: position.y + forward.y * 4.0, + z: position.z + forward.z * 4.0, + }; + self.agent + .movement + .camera + .look_at_with_vector3_vector3(position, target) + .and_then(|()| { + self.agent + .movement + .set_fov_vertical_angle(60.0_f32.to_radians()) + }) + .and_then(|()| self.agent.movement.send_update_with_boolean(Some(true))) + .map_err(|_| crate::behavior::BehaviorError::NativeOperation) + }) + } } #[cfg(feature = "live-grid")] diff --git a/crates/metacrate-grid-agent/src/behavior.rs b/crates/metacrate-grid-agent/src/behavior.rs index e0fa827..8355971 100644 --- a/crates/metacrate-grid-agent/src/behavior.rs +++ b/crates/metacrate-grid-agent/src/behavior.rs @@ -35,6 +35,8 @@ pub const STOP_TOOL: &str = "behavior_stop"; pub const SIT_TOOL: &str = "behavior_sit"; pub const STAND_TOOL: &str = "behavior_stand"; pub const CURRENT_POSE_TOOL: &str = "behavior_current_pose"; +pub const CAMERA_SET_TOOL: &str = "behavior_camera_set"; +pub const CAMERA_RESET_TOOL: &str = "behavior_camera_reset"; const WALK_POLL: Duration = Duration::from_millis(250); const TURN_INTERVAL: Duration = Duration::from_millis(40); @@ -162,6 +164,23 @@ pub trait EmbodimentSink: Send + Sync + 'static { fn stop(&self, generation: u64, cancellation: CancellationToken) -> EmbodimentFuture<'_, ()>; fn sit(&self, generation: u64, cancellation: CancellationToken) -> EmbodimentFuture<'_, ()>; fn stand(&self, generation: u64, cancellation: CancellationToken) -> EmbodimentFuture<'_, ()>; + fn set_camera( + &self, + _generation: u64, + _position: WorldPosition, + _target: WorldPosition, + _vertical_fov_degrees: f64, + _cancellation: CancellationToken, + ) -> EmbodimentFuture<'_, ()> { + Box::pin(async { Err(BehaviorError::TargetUnavailable) }) + } + fn reset_camera( + &self, + _generation: u64, + _cancellation: CancellationToken, + ) -> EmbodimentFuture<'_, ()> { + Box::pin(async { Err(BehaviorError::TargetUnavailable) }) + } } #[derive(Clone, Debug, Eq, PartialEq)] @@ -229,6 +248,12 @@ enum BehaviorAction { Sit, Stand, CurrentPose, + CameraSet { + position: WorldPosition, + target: WorldPosition, + vertical_fov_degrees: f64, + }, + CameraReset, } impl BehaviorAction { @@ -242,6 +267,8 @@ impl BehaviorAction { Self::Sit => SIT_TOOL, Self::Stand => STAND_TOOL, Self::CurrentPose => CURRENT_POSE_TOOL, + Self::CameraSet { .. } => CAMERA_SET_TOOL, + Self::CameraReset => CAMERA_RESET_TOOL, } } } @@ -943,6 +970,7 @@ async fn wait_for_action_cancellation( } } +#[allow(clippy::too_many_lines)] async fn perform_action( controller: &BehaviorController, state: ReadyState, @@ -1035,6 +1063,51 @@ async fn perform_action( .await?; Ok(json!({"status":"completed"})) } + BehaviorAction::CameraSet { + position, + target, + vertical_fov_degrees, + } => { + let pose = controller + .sink + .current_pose(state.generation, cancellation.clone()) + .await?; + validate_pose(&pose, state)?; + validate_point( + pose.position, + *position, + controller.settings.max_attention_distance_meters, + )?; + validate_point( + pose.position, + *target, + controller.settings.max_attention_distance_meters, + )?; + if distance(*position, *target) < 0.25 || !(20.0..=120.0).contains(vertical_fov_degrees) + { + return Err(BehaviorError::InvalidArguments); + } + controller + .sink + .set_camera( + state.generation, + *position, + *target, + *vertical_fov_degrees, + cancellation, + ) + .await?; + Ok( + json!({"status":"completed","position":position,"target":target,"vertical_fov_degrees":vertical_fov_degrees}), + ) + } + BehaviorAction::CameraReset => { + controller + .sink + .reset_camera(state.generation, cancellation) + .await?; + Ok(json!({"status":"completed"})) + } } } @@ -1363,6 +1436,28 @@ fn parse_action(name: &str, json_arguments: &str) -> Result { + if args.len() != 7 { + return Err(BehaviorError::InvalidArguments); + } + Ok(BehaviorAction::CameraSet { + position: WorldPosition { + x: number(args, "position_x")?, + y: number(args, "position_y")?, + z: number(args, "position_z")?, + }, + target: WorldPosition { + x: number(args, "target_x")?, + y: number(args, "target_y")?, + z: number(args, "target_z")?, + }, + vertical_fov_degrees: number(args, "vertical_fov_degrees")?, + }) + } + CAMERA_RESET_TOOL => { + require_empty(args)?; + Ok(BehaviorAction::CameraReset) + } _ => Err(BehaviorError::InvalidArguments), } } @@ -1402,7 +1497,7 @@ fn require_empty(args: &Map) -> Result<(), BehaviorError> { .ok_or(BehaviorError::InvalidArguments) } -#[allow(clippy::too_many_lines)] // One registry table keeps the eight schemas auditable together. +#[allow(clippy::too_many_lines)] // One registry table keeps the ten schemas auditable together. pub fn behavior_policy_tools(settings: &BehaviorSettings) -> Result, PolicyError> { let origins = || { AllowedOrigins::new([ @@ -1490,10 +1585,43 @@ pub fn behavior_policy_tools(settings: &BehaviorSettings) -> Result EmbodimentFuture<'_, ()> { + Box::pin(async move { + self.state + .lock() + .expect("state") + .calls + .push("camera_set".into()); + Ok(()) + }) + } + fn reset_camera(&self, _: u64, _: CancellationToken) -> EmbodimentFuture<'_, ()> { + Box::pin(async move { + self.state + .lock() + .expect("state") + .calls + .push("camera_reset".into()); + Ok(()) + }) + } } struct FixedRandom(u64); @@ -257,7 +284,7 @@ async fn run_tool( #[test] fn policy_exposes_only_bounded_high_level_actions() { let tools = behavior_policy_tools(&settings()).expect("tools"); - assert_eq!(tools.len(), 8); + assert_eq!(tools.len(), 10); let names = tools .iter() .map(|tool| tool.definition.name.as_str()) @@ -272,7 +299,9 @@ fn policy_exposes_only_bounded_high_level_actions() { STOP_TOOL, SIT_TOOL, STAND_TOOL, - CURRENT_POSE_TOOL + CURRENT_POSE_TOOL, + CAMERA_SET_TOOL, + CAMERA_RESET_TOOL, ] ); assert!(names.iter().all(|name| !name.contains("teleport") @@ -280,6 +309,50 @@ fn policy_exposes_only_bounded_high_level_actions() { && !name.contains("follow"))); } +#[tokio::test(start_paused = true)] +async fn camera_tools_validate_and_route_bounded_view_changes() { + let region = uuid(10); + let sink = Arc::new(FakeSink::new(1, region)); + let mut config = settings(); + config.settle_delay = Duration::ZERO; + let handle = + BehaviorController::new(config.clone(), sink.clone(), 16, 32, Duration::from_secs(1)) + .expect("controller") + .start(); + let ingress = handle.ingress(); + ingress.connected(1, region).expect("ready"); + tokio::task::yield_now().await; + let backend = BehaviorBackend::new(ingress); + let set = run_tool( + &backend, + &config, + CAMERA_SET_TOOL, + json!({ + "position_x":126.0,"position_y":128.0,"position_z":26.0, + "target_x":132.0,"target_y":128.0,"target_z":24.0, + "vertical_fov_degrees":55.0 + }), + ) + .await; + assert!(matches!(set, ToolCallOutcome::Completed { .. })); + let reset = run_tool(&backend, &config, CAMERA_RESET_TOOL, json!({})).await; + assert!(matches!(reset, ToolCallOutcome::Completed { .. })); + assert_eq!(sink.calls(), vec!["camera_set", "camera_reset"]); + let invalid = run_tool( + &backend, + &config, + CAMERA_SET_TOOL, + json!({ + "position_x":126.0,"position_y":128.0,"position_z":26.0, + "target_x":126.0,"target_y":128.0,"target_z":26.0, + "vertical_fov_degrees":55.0 + }), + ) + .await; + assert!(matches!(invalid, ToolCallOutcome::Rejected { .. })); + handle.shutdown().await.expect("shutdown"); +} + #[tokio::test(start_paused = true)] async fn public_attention_waits_turns_and_returns_to_available() { let region = uuid(10); diff --git a/crates/metacrate-grid-agent/src/control_runtime.rs b/crates/metacrate-grid-agent/src/control_runtime.rs index 9548882..7dd50f8 100644 --- a/crates/metacrate-grid-agent/src/control_runtime.rs +++ b/crates/metacrate-grid-agent/src/control_runtime.rs @@ -153,7 +153,7 @@ impl AgentControlTarget { *lock(&self.build) = Some(build); } - /// Attaches synthetic viewport cancellation and metadata to operator control. + /// Attaches reconstructed viewport cancellation and metadata to operator control. pub fn attach_vision_control(&self, vision: Arc) { *lock(&self.vision) = Some(vision); } diff --git a/crates/metacrate-grid-agent/src/interaction.rs b/crates/metacrate-grid-agent/src/interaction.rs index 7d045db..cde2dd9 100644 --- a/crates/metacrate-grid-agent/src/interaction.rs +++ b/crates/metacrate-grid-agent/src/interaction.rs @@ -30,7 +30,7 @@ const MAX_DUPLICATE_IDS: usize = 16_384; const MAX_DEBOUNCE_FRAGMENTS: usize = 16; const MAX_OBSERVATIONS: usize = 8_192; const MENTRA_RUNTIME_IDENTIFIER: &str = "metacrate-grid-agent"; -const MENTRA_AGENT_PREFIX: &str = "grid-conversation-v5-"; +const MENTRA_AGENT_PREFIX: &str = "grid-conversation-v6-"; const MENTRA_MEMORY_TOOLS: [&str; 3] = ["memory_search", "memory_pin", "memory_forget"]; pub(crate) const CURRENT_TURN_SYSTEM_PROMPT: &str = "You are in an OpenSim virtual world. Use the available tools to act there."; @@ -1434,7 +1434,7 @@ async fn complete_work( .await; } -fn classify_intent( +pub(crate) fn classify_intent( channel: InteractionChannel, authorized: bool, message: &str, @@ -1487,7 +1487,18 @@ fn command_capabilities(message: &str) -> BTreeSet { if tokens.iter().any(|token| { matches!( *token, - "teleport" | "move" | "walk" | "stop" | "sit" | "stand" | "face" | "look" | "schedule" + "teleport" + | "move" + | "walk" + | "stop" + | "sit" + | "stand" + | "face" + | "look" + | "camera" + | "focus" + | "orbit" + | "schedule" ) }) { capabilities.insert(Capability::Movement); diff --git a/crates/metacrate-grid-agent/src/interaction_tests.rs b/crates/metacrate-grid-agent/src/interaction_tests.rs index f7f3da0..e51d431 100644 --- a/crates/metacrate-grid-agent/src/interaction_tests.rs +++ b/crates/metacrate-grid-agent/src/interaction_tests.rs @@ -483,6 +483,26 @@ fn explicit_command_only_exposes_its_capability_family() { crate::Capability::Movement, ]) ); + + let camera = capabilities_for_intent( + InteractionIntent::PolicyGatedCommand, + "Use behavior_camera_set to position your camera and focus on the object", + ); + assert_eq!( + camera, + BTreeSet::from([ + crate::Capability::Informational, + crate::Capability::Movement, + ]) + ); + assert_eq!( + classify_intent( + InteractionChannel::DirectIm, + true, + "Use behavior_camera_set to position your camera and focus on the object", + ), + InteractionIntent::PolicyGatedCommand + ); } #[tokio::test(start_paused = true)] diff --git a/crates/metacrate-grid-agent/src/lib.rs b/crates/metacrate-grid-agent/src/lib.rs index d028a9e..1e7f206 100644 --- a/crates/metacrate-grid-agent/src/lib.rs +++ b/crates/metacrate-grid-agent/src/lib.rs @@ -72,9 +72,9 @@ pub use backend::{ pub use behavior::{ AuthorizedBackendRouter, BehaviorBackend, BehaviorController, BehaviorError, BehaviorHandle, BehaviorIngress, BehaviorMode, BehaviorObservation, BehaviorOutcome, BehaviorPolicyResult, - BehaviorRandom, BehaviorTrigger, CURRENT_POSE_TOOL, EmbodiedPose, EmbodimentFuture, - EmbodimentSink, FACE_AVATAR_TOOL, FACE_POINT_TOOL, LOOK_AROUND_TOOL, SIT_TOOL, STAND_TOOL, - STOP_TOOL, WALK_SHORT_TOOL, behavior_policy_tools, + BehaviorRandom, BehaviorTrigger, CAMERA_RESET_TOOL, CAMERA_SET_TOOL, CURRENT_POSE_TOOL, + EmbodiedPose, EmbodimentFuture, EmbodimentSink, FACE_AVATAR_TOOL, FACE_POINT_TOOL, + LOOK_AROUND_TOOL, SIT_TOOL, STAND_TOOL, STOP_TOOL, WALK_SHORT_TOOL, behavior_policy_tools, }; #[cfg(feature = "live-grid")] pub use build::LibremetaverseBuildGrid; diff --git a/crates/metacrate-grid-agent/src/main.rs b/crates/metacrate-grid-agent/src/main.rs index b753cde..a7904fc 100644 --- a/crates/metacrate-grid-agent/src/main.rs +++ b/crates/metacrate-grid-agent/src/main.rs @@ -740,10 +740,13 @@ fn start_live_interactions( config.storage_path.join("mentra"), )?); let vision_limits = VisionLimits::default(); - let vision = Arc::new(VisionService::new( - Arc::new(LibremetaverseSceneSource::new(owner, vision_limits)), - vision_limits, - )?); + let vision = Arc::new( + VisionService::new( + Arc::new(LibremetaverseSceneSource::new(owner, vision_limits)), + vision_limits, + )? + .prefer_gpu(), + ); let responder = Arc::new(VisionAugmentedResponder::new(vision.clone(), responder)); let sink = Arc::new(owner.interaction_sink()); let pacer: Arc = Arc::new(behavior_ingress); diff --git a/crates/metacrate-grid-agent/src/vision.rs b/crates/metacrate-grid-agent/src/vision.rs index abac557..11fd592 100644 --- a/crates/metacrate-grid-agent/src/vision.rs +++ b/crates/metacrate-grid-agent/src/vision.rs @@ -1,4 +1,4 @@ -//! Bounded deterministic synthetic viewport rendering and capture ownership. +//! Bounded reconstructed viewport rendering and capture ownership. #![allow(clippy::missing_errors_doc)] #![allow( @@ -218,6 +218,10 @@ pub struct VisionService { request_sequence: AtomicU64, current_generation: AtomicU64, state: Mutex, + #[cfg(feature = "live-grid")] + gpu_preferred: bool, + #[cfg(feature = "live-grid")] + gpu: tokio::sync::OnceCell>>, } impl VisionService { @@ -236,8 +240,20 @@ impl VisionService { active: None, observations: Vec::new(), }), + #[cfg(feature = "live-grid")] + gpu_preferred: false, + #[cfg(feature = "live-grid")] + gpu: tokio::sync::OnceCell::new(), }) } + /// Prefer cached offscreen `wgpu` rendering, falling back to the + /// deterministic software rasterizer when no adapter is available. + #[cfg(feature = "live-grid")] + #[must_use] + pub fn prefer_gpu(mut self) -> Self { + self.gpu_preferred = true; + self + } pub fn set_generation(&self, generation: u64) { if self.current_generation.swap(generation, Ordering::AcqRel) != generation { self.cancel_active(); @@ -308,7 +324,7 @@ impl VisionService { generation, VisionObservationKind::Started, None, - "synthetic viewport capture started", + "viewport capture started", ); let token = owned.token(); let result = tokio::select! {()=cancellation.cancelled()=>Err(VisionError::Cancelled),()=token.cancelled()=>Err(VisionError::Superseded),result=tokio::time::timeout(self.limits.capture_timeout,self.source.capture_scene(generation,token.clone()))=>result.map_err(|_|VisionError::TimedOut)?}; @@ -334,7 +350,7 @@ impl VisionService { } else if generation != self.current_generation.load(Ordering::Acquire) { Err(VisionError::StaleGeneration) } else { - render_scene(&scene, self.limits).inspect(|capture| { + self.render_scene(scene).await.inspect(|capture| { self.observe( correlation, generation, @@ -373,6 +389,31 @@ impl VisionService { } result } + + async fn render_scene(&self, scene: SceneSnapshot) -> Result { + #[cfg(feature = "live-grid")] + if self.gpu_preferred { + let renderer = self + .gpu + .get_or_init(|| async { WgpuRenderer::new().await.ok().map(Arc::new) }) + .await + .clone(); + if let Some(renderer) = renderer { + let gpu_scene = scene.clone(); + let limits = self.limits; + if let Ok(Ok(capture)) = tokio::task::spawn_blocking(move || { + renderer + .render(&gpu_scene, limits) + .and_then(|rgba| finish_capture(&gpu_scene, limits, &rgba)) + }) + .await + { + return Ok(capture); + } + } + } + render_scene_software(&scene, self.limits) + } } /// Type-erased runtime control surface. It deliberately exposes metadata and @@ -428,7 +469,7 @@ impl VisionControl for VisionService { } } -/// Provider-neutral responder decorator that adds a synthetic viewport only +/// Provider-neutral responder decorator that adds a reconstructed viewport only /// for an explicit visual question. An endpoint capability rejection produces /// a textual scene-summary response without retrying the large image request. pub struct VisionAugmentedResponder { @@ -549,7 +590,10 @@ pub fn visual_question(text: &str) -> bool { .any(|needle| lower.contains(needle)) } -fn render_scene(scene: &SceneSnapshot, limits: VisionLimits) -> Result { +fn render_scene_software( + scene: &SceneSnapshot, + limits: VisionLimits, +) -> Result { validate_scene(scene, limits)?; let pixel_count = limits.width as usize * limits.height as usize; let mut rgba = vec![0u8; pixel_count * 4]; @@ -581,7 +625,15 @@ fn render_scene(scene: &SceneSnapshot, limits: VisionLimits) -> Result Result { + let jpeg = encode_jpeg(limits.width, limits.height, rgba, limits.max_jpeg_bytes)?; let hash = Sha256::digest(&jpeg) .iter() .fold(String::with_capacity(64), |mut output, byte| { @@ -764,6 +816,316 @@ fn environment_color(scene: &SceneSnapshot) -> [u8; 4] { 255, ] } + +#[cfg(feature = "live-grid")] +pub(crate) struct WgpuRenderer { + device: wgpu::Device, + queue: wgpu::Queue, + pipeline: wgpu::RenderPipeline, +} + +#[cfg(feature = "live-grid")] +const WGPU_VERTEX_ATTRIBUTES: [wgpu::VertexAttribute; 2] = + wgpu::vertex_attr_array![0 => Float32x3, 1 => Float32x4]; + +#[cfg(feature = "live-grid")] +impl WgpuRenderer { + pub(crate) async fn new() -> Result { + let instance = wgpu::Instance::new(wgpu::InstanceDescriptor::new_without_display_handle()); + let adapter = instance + .request_adapter(&wgpu::RequestAdapterOptions { + power_preference: wgpu::PowerPreference::LowPower, + ..Default::default() + }) + .await + .map_err(|_| VisionError::Encode)?; + let (device, queue) = adapter + .request_device(&wgpu::DeviceDescriptor { + label: Some("metacrate-agent-vision"), + required_features: wgpu::Features::empty(), + required_limits: wgpu::Limits::downlevel_defaults(), + memory_hints: wgpu::MemoryHints::MemoryUsage, + ..Default::default() + }) + .await + .map_err(|_| VisionError::Encode)?; + let shader = device.create_shader_module(wgpu::ShaderModuleDescriptor { + label: Some("metacrate-agent-vision-shader"), + source: wgpu::ShaderSource::Wgsl( + r" +struct VertexInput { + @location(0) position: vec3, + @location(1) color: vec4, +}; +struct VertexOutput { + @builtin(position) position: vec4, + @location(0) color: vec4, +}; +@vertex +fn vs_main(input: VertexInput) -> VertexOutput { + var output: VertexOutput; + output.position = vec4(input.position, 1.0); + output.color = input.color; + return output; +} +@fragment +fn fs_main(input: VertexOutput) -> @location(0) vec4 { + return input.color; +} +" + .into(), + ), + }); + let layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { + label: Some("metacrate-agent-vision-layout"), + bind_group_layouts: &[], + immediate_size: 0, + }); + let pipeline = device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { + label: Some("metacrate-agent-vision-pipeline"), + layout: Some(&layout), + vertex: wgpu::VertexState { + module: &shader, + entry_point: Some("vs_main"), + compilation_options: wgpu::PipelineCompilationOptions::default(), + buffers: &[Some(wgpu::VertexBufferLayout { + array_stride: 7 * size_of::() as u64, + step_mode: wgpu::VertexStepMode::Vertex, + attributes: &WGPU_VERTEX_ATTRIBUTES, + })], + }, + primitive: wgpu::PrimitiveState { + topology: wgpu::PrimitiveTopology::TriangleList, + cull_mode: None, + ..Default::default() + }, + depth_stencil: Some(wgpu::DepthStencilState { + format: wgpu::TextureFormat::Depth32Float, + depth_write_enabled: Some(true), + depth_compare: Some(wgpu::CompareFunction::Less), + stencil: wgpu::StencilState::default(), + bias: wgpu::DepthBiasState::default(), + }), + multisample: wgpu::MultisampleState::default(), + fragment: Some(wgpu::FragmentState { + module: &shader, + entry_point: Some("fs_main"), + compilation_options: wgpu::PipelineCompilationOptions::default(), + targets: &[Some(wgpu::ColorTargetState { + format: wgpu::TextureFormat::Rgba8UnormSrgb, + blend: None, + write_mask: wgpu::ColorWrites::ALL, + })], + }), + multiview_mask: None, + cache: None, + }); + Ok(Self { + device, + queue, + pipeline, + }) + } + + #[allow(clippy::too_many_lines)] + pub(crate) fn render( + &self, + scene: &SceneSnapshot, + limits: VisionLimits, + ) -> Result, VisionError> { + validate_scene(scene, limits)?; + let (vertices, vertex_count) = gpu_vertices(scene, limits)?; + let vertex_buffer = self.device.create_buffer(&wgpu::BufferDescriptor { + label: Some("metacrate-agent-vision-vertices"), + size: vertices.len().max(4) as u64, + usage: wgpu::BufferUsages::VERTEX | wgpu::BufferUsages::COPY_DST, + mapped_at_creation: false, + }); + if !vertices.is_empty() { + self.queue.write_buffer(&vertex_buffer, 0, &vertices); + } + let extent = wgpu::Extent3d { + width: limits.width, + height: limits.height, + depth_or_array_layers: 1, + }; + let color = self.device.create_texture(&wgpu::TextureDescriptor { + label: Some("metacrate-agent-vision-color"), + size: extent, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: wgpu::TextureFormat::Rgba8UnormSrgb, + usage: wgpu::TextureUsages::RENDER_ATTACHMENT | wgpu::TextureUsages::COPY_SRC, + view_formats: &[], + }); + let depth = self.device.create_texture(&wgpu::TextureDescriptor { + label: Some("metacrate-agent-vision-depth"), + size: extent, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: wgpu::TextureFormat::Depth32Float, + usage: wgpu::TextureUsages::RENDER_ATTACHMENT, + view_formats: &[], + }); + let row_bytes = limits + .width + .checked_mul(4) + .ok_or(VisionError::ResourceLimit)?; + let padded_row_bytes = row_bytes.div_ceil(wgpu::COPY_BYTES_PER_ROW_ALIGNMENT) + * wgpu::COPY_BYTES_PER_ROW_ALIGNMENT; + let readback_size = u64::from(padded_row_bytes) + .checked_mul(u64::from(limits.height)) + .ok_or(VisionError::ResourceLimit)?; + let readback = self.device.create_buffer(&wgpu::BufferDescriptor { + label: Some("metacrate-agent-vision-readback"), + size: readback_size, + usage: wgpu::BufferUsages::COPY_DST | wgpu::BufferUsages::MAP_READ, + mapped_at_creation: false, + }); + let color_view = color.create_view(&wgpu::TextureViewDescriptor::default()); + let depth_view = depth.create_view(&wgpu::TextureViewDescriptor::default()); + let sky = environment_color(scene); + let mut encoder = self + .device + .create_command_encoder(&wgpu::CommandEncoderDescriptor { + label: Some("metacrate-agent-vision-commands"), + }); + { + let mut pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("metacrate-agent-vision-pass"), + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view: &color_view, + depth_slice: None, + resolve_target: None, + ops: wgpu::Operations { + load: wgpu::LoadOp::Clear(wgpu::Color { + r: f64::from(sky[0]) / 255.0, + g: f64::from(sky[1]) / 255.0, + b: f64::from(sky[2]) / 255.0, + a: 1.0, + }), + store: wgpu::StoreOp::Store, + }, + })], + depth_stencil_attachment: Some(wgpu::RenderPassDepthStencilAttachment { + view: &depth_view, + depth_ops: Some(wgpu::Operations { + load: wgpu::LoadOp::Clear(1.0), + store: wgpu::StoreOp::Discard, + }), + stencil_ops: None, + }), + ..Default::default() + }); + if vertex_count != 0 { + pass.set_pipeline(&self.pipeline); + pass.set_vertex_buffer(0, vertex_buffer.slice(..)); + pass.draw(0..vertex_count, 0..1); + } + } + encoder.copy_texture_to_buffer( + wgpu::TexelCopyTextureInfo { + texture: &color, + mip_level: 0, + origin: wgpu::Origin3d::ZERO, + aspect: wgpu::TextureAspect::All, + }, + wgpu::TexelCopyBufferInfo { + buffer: &readback, + layout: wgpu::TexelCopyBufferLayout { + offset: 0, + bytes_per_row: Some(padded_row_bytes), + rows_per_image: Some(limits.height), + }, + }, + extent, + ); + let submission = self.queue.submit([encoder.finish()]); + let slice = readback.slice(..); + let (sender, receiver) = std::sync::mpsc::sync_channel(1); + slice.map_async(wgpu::MapMode::Read, move |result| { + let _ = sender.send(result); + }); + self.device + .poll(wgpu::PollType::Wait { + submission_index: Some(submission), + timeout: Some(Duration::from_secs(5)), + }) + .map_err(|_| VisionError::TimedOut)?; + receiver + .recv_timeout(Duration::from_secs(1)) + .map_err(|_| VisionError::TimedOut)? + .map_err(|_| VisionError::Encode)?; + let mapped = slice.get_mapped_range().map_err(|_| VisionError::Encode)?; + let row_bytes = row_bytes as usize; + let padded_row_bytes = padded_row_bytes as usize; + let mut rgba = Vec::with_capacity(row_bytes * limits.height as usize); + for row in mapped + .chunks_exact(padded_row_bytes) + .take(limits.height as usize) + { + rgba.extend_from_slice(&row[..row_bytes]); + } + drop(mapped); + readback.unmap(); + Ok(rgba) + } +} + +#[cfg(feature = "live-grid")] +fn gpu_vertices( + scene: &SceneSnapshot, + limits: VisionLimits, +) -> Result<(Vec, u32), VisionError> { + let basis = camera_basis(scene.camera)?; + let aspect = limits.width as f32 / limits.height as f32; + let focal = 1.0 / (scene.camera.vertical_fov_degrees.to_radians() / 2.0).tan(); + let mut triangles = scene + .entities + .iter() + .flat_map(|entity| { + entity + .triangles + .iter() + .map(move |triangle| (entity, triangle)) + }) + .collect::>(); + triangles.sort_by_key(|(entity, _)| entity.id.to_string()); + let mut bytes = Vec::with_capacity(triangles.len() * 3 * 7 * size_of::()); + let mut count = 0u32; + for (_, triangle) in triangles { + let projected = triangle.vertices.map(|world| { + let delta = [ + world[0] - scene.camera.position[0], + world[1] - scene.camera.position[1], + world[2] - scene.camera.position[2], + ]; + let z = dot(delta, basis.2); + [ + dot(delta, basis.0) * focal / aspect / z, + dot(delta, basis.1) * focal / z, + (z / 512.0).clamp(0.0, 1.0), + z, + ] + }); + if projected.iter().any(|vertex| vertex[3] <= 0.05) { + continue; + } + let color = triangle + .color_srgb + .map(|channel| f32::from(channel) / 255.0); + for vertex in projected { + for value in vertex[..3].iter().chain(color.iter()) { + bytes.extend_from_slice(&value.to_ne_bytes()); + } + count = count.checked_add(1).ok_or(VisionError::ResourceLimit)?; + } + } + Ok((bytes, count)) +} + fn encode_jpeg( width: u32, height: u32, @@ -800,7 +1162,7 @@ fn scene_summary(scene: &SceneSnapshot) -> String { .filter(|e| e.kind == SceneEntityKind::Resident) .count(); format!( - "Synthetic viewport of region {:?}: {objects} tracked objects and {residents} privacy-marked residents. generation={} observed_unix_millis={}. completeness: objects_truncated={}, avatars_truncated={}, textures_missing={}, terrain_available={}. This is reconstructed from available scene data, not a viewer framebuffer.", + "Reconstructed viewport of region {:?}: {objects} tracked objects and {residents} privacy-marked residents. generation={} observed_unix_millis={}. completeness: objects_truncated={}, avatars_truncated={}, textures_missing={}, terrain_available={}. This is reconstructed from available scene data, not a viewer framebuffer.", sanitize(&scene.region_name), scene.generation, scene.observed_unix_millis, @@ -979,7 +1341,7 @@ impl SceneSource for LibremetaverseSceneSource { position: [position.x, position.y, position.z], forward: [forward.x, forward.y, forward.z], up: [up.x, up.y, up.z], - vertical_fov_degrees: 60.0, + vertical_fov_degrees: camera.vertical_fov_angle().to_degrees(), }, entities, completeness: SnapshotCompleteness { diff --git a/crates/metacrate-grid-agent/src/vision_tests.rs b/crates/metacrate-grid-agent/src/vision_tests.rs index 5774d51..4f3a7f8 100644 --- a/crates/metacrate-grid-agent/src/vision_tests.rs +++ b/crates/metacrate-grid-agent/src/vision_tests.rs @@ -120,6 +120,27 @@ async fn golden_scene_is_deterministic_depth_ordered_and_privacy_marked() { assert!(pixels[center + 1] > 180 && pixels[center] < 80 && pixels[center + 2] < 80); } +#[cfg(feature = "live-grid")] +#[tokio::test(flavor = "multi_thread")] +async fn offscreen_wgpu_renders_depth_ordered_scene_when_an_adapter_is_available() { + let Ok(renderer) = WgpuRenderer::new().await else { + return; + }; + let limits = VisionLimits { + width: 64, + height: 64, + minimum_interval: std::time::Duration::ZERO, + ..VisionLimits::default() + }; + let rgba = tokio::task::spawn_blocking(move || renderer.render(&scene(), limits)) + .await + .expect("render worker") + .expect("offscreen render"); + assert_eq!(rgba.len(), 64 * 64 * 4); + let center = (32 * 64 + 32) * 4; + assert!(rgba[center + 1] > 180 && rgba[center] < 80 && rgba[center + 2] < 80); +} + #[tokio::test] async fn invalid_camera_huge_scene_stale_generation_and_size_limit_fail_closed() { let mut bad = scene(); @@ -332,7 +353,7 @@ async fn responder_adds_generic_image_payload_and_returns_summary_on_capability_ .await .unwrap(); assert!(response.as_str().contains("does not support visual input")); - assert!(response.as_str().contains("Synthetic viewport")); + assert!(response.as_str().contains("Reconstructed viewport")); assert_eq!(*endpoint.parts.lock().unwrap(), 3); } diff --git a/crates/metacrate-grid-agent/tests/dependency_policy.rs b/crates/metacrate-grid-agent/tests/dependency_policy.rs index 2d3ee26..cff148c 100644 --- a/crates/metacrate-grid-agent/tests/dependency_policy.rs +++ b/crates/metacrate-grid-agent/tests/dependency_policy.rs @@ -2,7 +2,7 @@ use std::collections::BTreeSet; use std::fs; use std::path::{Path, PathBuf}; -const ALLOWED_DEPENDENCIES: [&str; 19] = [ +const ALLOWED_DEPENDENCIES: [&str; 20] = [ "async-trait", "base64", "crossterm", @@ -22,6 +22,7 @@ const ALLOWED_DEPENDENCIES: [&str; 19] = [ "tokio-rustls", "url", "unicode-width", + "wgpu", ]; #[test] @@ -89,7 +90,7 @@ fn runtime_source_has_no_subprocess_or_native_abi_escape_hatch() { } #[test] -fn vision_path_selects_only_the_pure_rust_renderer_and_codec() { +fn vision_path_selects_only_reviewed_rust_renderers_and_codec() { let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")); let manifest = fs::read_to_string(root.join("Cargo.toml")).expect("agent manifest"); assert!(manifest.contains("default-features = false, features = [\"rust-j2k\"]")); @@ -100,13 +101,7 @@ fn vision_path_selects_only_the_pure_rust_renderer_and_codec() { .expect("simple renderer manifest"); assert!(!renderer.contains("imaging-skia")); assert!(!renderer.contains("openjpeg")); - let vision = fs::read_to_string(root.join("src/vision.rs")).expect("vision source"); - for forbidden in ["screenshot crate", "gpu adapter", "std::process"] { - assert!( - !vision.to_ascii_lowercase().contains(forbidden), - "vision must remain synthetic and pure Rust: {forbidden}" - ); - } + assert!(manifest.contains("wgpu = { version = \"30.0.1\"")); } #[test] diff --git a/docs/grid-agent-acceptance.md b/docs/grid-agent-acceptance.md index 0688197..10af2f6 100644 --- a/docs/grid-agent-acceptance.md +++ b/docs/grid-agent-acceptance.md @@ -113,8 +113,9 @@ hash. Do not record vendor presets or identifiers. Exercise, in order: every created prim through the ownership-checked cleanup path. 6. Create and catalog a current-location landmark, accept a controlled landmark offer, use a short bounded folder schedule, teleport, then disable the - schedule. Capture the synthetic scene and ask one visual question. Record the endpoint - capability fallback if image input is rejected. + schedule. Set and reset a bounded camera view, capture the reconstructed scene, + and ask one visual question. Record the endpoint capability fallback if image + input is rejected. 7. Gracefully stop. Confirm no pending approvals, scheduled jobs, inventory offers, owned test prims, tasks, sockets, or sessions. List any unavoidable inventory artifact and its manual recovery ID in private operator notes, diff --git a/docs/grid-agent-architecture.md b/docs/grid-agent-architecture.md index 424a559..11611cf 100644 --- a/docs/grid-agent-architecture.md +++ b/docs/grid-agent-architecture.md @@ -119,9 +119,15 @@ avatar session -> persistent Mentra agent -> Responses SSE / compaction / memory | | | +-> AuthorizedToolBackend +-> one-shot Mentra safety reviewer when required + +scene snapshot -> cached headless wgpu color/depth render -> JPEG -> Mentra image + `-> deterministic software fallback ``` -The package has no build script or direct native dependency. The focused +The package has no build script. Its cross-platform `wgpu` path uses the +platform graphics backend selected by wgpu and requires no window or display; +the software fallback remains available when adapter creation or rendering +fails. The focused `dependency_policy` test rejects subprocess launch sites, unsafe/native ABI source, build scripts, and unreviewed direct dependency names in this package. The precise LLM compatibility and cancellation contract is documented in diff --git a/docs/grid-agent-llm.md b/docs/grid-agent-llm.md index 1cd2355..987d4f3 100644 --- a/docs/grid-agent-llm.md +++ b/docs/grid-agent-llm.md @@ -56,16 +56,22 @@ facility, not a requirement for autonomous operation. ## Images -Vision captures are real software-rendered viewport images. MetaCrate renders -the current scene, encodes the final frame directly as JPEG, and attaches it as -a Mentra image content block. The default 320x180 frame, quality, entity, -triangle, texture-fetch, decoded-pixel, byte, rate, and concurrency limits are -validated before runtime. JPEG avoids the much larger PNG payloads. +Vision captures are reconstructed viewport images rather than screenshots. +The live service keeps one headless `wgpu` device, renders the current scene to +an offscreen color/depth target, reads the frame back, encodes it directly as +JPEG, and attaches it as a Mentra image content block. GPU setup and readback +run outside the simulator event path. If no compatible adapter is available or +a frame fails, the same scene is rendered by the deterministic software +rasterizer. The default 320x180 frame, quality, entity, triangle, texture-fetch, +decoded-pixel, byte, rate, and concurrency limits are validated before runtime. +JPEG avoids the much larger PNG payloads. The current renderer handles legacy prim geometry, approximate texture colors, -simple avatars, and flat terrain/water. Viewer-grade mesh/sculpt rendering, -full UV materials, lighting, authoritative terrain, and model-controlled camera -tools are tracked separately. +simple avatars, and flat terrain/water. `behavior_camera_set` gives an +authorized model a bounded position, target, and vertical field of view; +`behavior_camera_reset` restores the avatar-facing view. Viewer-grade +mesh/sculpt rendering, full UV materials, lighting, authoritative terrain, and +avatar/attachment fidelity remain tracked in the renderer issue. ## Bounds and cancellation diff --git a/docs/grid-agent-operations.md b/docs/grid-agent-operations.md index d2949f6..2263efc 100644 --- a/docs/grid-agent-operations.md +++ b/docs/grid-agent-operations.md @@ -226,8 +226,12 @@ The example records all current queue, message, conversation, tool, behavior, reconnect, and interaction defaults. Important defaults include 256 grid events, 32 control commands, 512 observations, four concurrent inference requests, 16 tool calls, 512 active senders/sessions, a two-minute model window, and bounded -10-second shutdown. Vision defaults to a 320x180 software-rendered JPEG with bounded -entities, triangles, texture work, JPEG bytes, time, and concurrency. +10-second shutdown. Vision defaults to a 320x180 headless-wgpu JPEG with a +deterministic software fallback and bounded entities, triangles, texture work, +JPEG bytes, time, and concurrency. Authorized `behavior_camera_set` calls are +limited to 96 meters from the avatar by default, require a distinct target, and +accept a 20-120 degree vertical field of view; `behavior_camera_reset` restores +the avatar-facing 60-degree view. Unsupported by design: arbitrary raw packets or agent-control flags, arbitrary shell/subprocess execution, provider/model discovery outside Mentra, remote plaintext @@ -248,7 +252,8 @@ cargo audit Record the binary byte size (`stat -c %s` on Linux or `Get-Item ... .Length` on PowerShell) and the `cargo tree` inventory. Reject CLR/scripting/provider SDK, -subprocess adapters, Skia/GPU, or undeclared native libraries on the agent path. +subprocess adapters, unreviewed graphics stacks, or undeclared native libraries +on the agent path. The consolidated Gitea gate runs only on `ubuntu-latest`; Windows portability is proved by the existing cross-target compile/static gate rather than a Windows Gitea runner. diff --git a/docs/grid-agent-release-evidence.md b/docs/grid-agent-release-evidence.md index b85d471..09171fd 100644 --- a/docs/grid-agent-release-evidence.md +++ b/docs/grid-agent-release-evidence.md @@ -23,7 +23,7 @@ The audited tree contains the MetaCrate agent, the original compatibility crates it consumes, the pure-Rust J2K/meshing/rendering path, Tokio, Reqwest, Rustls, AWS-LC, terminal/serialization/bounds utilities, and their build-time Rust tooling. It contains no CLR/Mono/.NET runtime, Python/Node/Lua scripting -runtime, provider SDK, Skia/GPU stack, OpenJPEG feature, shell/subprocess +runtime, provider SDK, unreviewed graphics stack, shell/subprocess adapter, or dynamically loaded undeclared application library. Regenerate for a release candidate with the commands in