Add headless wgpu vision and camera controls
Some checks failed
CI / rust-skia (Rust only) (push) Has been cancelled
CI / required (push) Has been cancelled

This commit is contained in:
2026-08-22 11:36:34 +02:00
parent 0dd2ca5824
commit c101007362
19 changed files with 1250 additions and 56 deletions

460
Cargo.lock generated
View File

@@ -164,6 +164,15 @@ version = "0.7.8"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56"
[[package]]
name = "ash"
version = "0.38.0+1.3.281"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0bb44936d800fea8f016d7f2311c6a4f97aebd5dc86f09906139ec848cf3a46f"
dependencies = [
"libloading",
]
[[package]] [[package]]
name = "asn1-rs" name = "asn1-rs"
version = "0.7.2" version = "0.7.2"
@@ -303,6 +312,15 @@ dependencies = [
"bit-vec 0.8.0", "bit-vec 0.8.0",
] ]
[[package]]
name = "bit-set"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09ec2f926cc3060f09db9ebc5b52823d85268d24bb917e472c0c4bea35780a7d"
dependencies = [
"bit-vec 0.9.1",
]
[[package]] [[package]]
name = "bit-vec" name = "bit-vec"
version = "0.8.0" version = "0.8.0"
@@ -568,6 +586,15 @@ dependencies = [
"unicode-width 0.1.14", "unicode-width 0.1.14",
] ]
[[package]]
name = "codespan-reporting"
version = "0.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "af491d569909a7e4dee0ad7db7f5341fef5c614d5b8ec8cf765732aba3cff681"
dependencies = [
"unicode-width 0.2.2",
]
[[package]] [[package]]
name = "color_quant" name = "color_quant"
version = "1.1.0" version = "1.1.0"
@@ -765,6 +792,12 @@ dependencies = [
"winapi", "winapi",
] ]
[[package]]
name = "crunchy"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5"
[[package]] [[package]]
name = "crypto-bigint" name = "crypto-bigint"
version = "0.5.5" version = "0.5.5"
@@ -1015,6 +1048,15 @@ dependencies = [
"syn 3.0.3", "syn 3.0.3",
] ]
[[package]]
name = "dlib"
version = "0.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ab8ecd87370524b461f8557c119c405552c396ed91fc0a8eec68679eab26f94a"
dependencies = [
"libloading",
]
[[package]] [[package]]
name = "document-features" name = "document-features"
version = "0.2.12" version = "0.2.12"
@@ -1379,6 +1421,17 @@ dependencies = [
"weezl", "weezl",
] ]
[[package]]
name = "gl_generator"
version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1a95dfc23a2b4a9a2f5ab41d194f8bfda3cabec42af4e39f08c339eb2a0c124d"
dependencies = [
"khronos_api",
"log",
"xml-rs",
]
[[package]] [[package]]
name = "glam" name = "glam"
version = "0.29.3" version = "0.29.3"
@@ -1397,6 +1450,27 @@ version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9985c9503b412198aa4197559e9a318524ebc4519c229bfa05a535828c950b9d" checksum = "9985c9503b412198aa4197559e9a318524ebc4519c229bfa05a535828c950b9d"
[[package]]
name = "glow"
version = "0.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "29038e1c483364cc6bb3cf78feee1816002e127c331a1eec55a4d202b9e1adb5"
dependencies = [
"js-sys",
"slotmap",
"wasm-bindgen",
"web-sys",
]
[[package]]
name = "glutin_wgl_sys"
version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2c4ee00b289aba7a9e5306d57c2d05499b2e5dc427f84ac708bd2c090212cf3e"
dependencies = [
"gl_generator",
]
[[package]] [[package]]
name = "goblin" name = "goblin"
version = "0.10.5" version = "0.10.5"
@@ -1408,6 +1482,20 @@ dependencies = [
"scroll", "scroll",
] ]
[[package]]
name = "gpu-allocator"
version = "0.28.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "51255ea7cfaadb6c5f1528d43e92a82acb2b96c43365989a28b2d44ee38f8795"
dependencies = [
"ash",
"hashbrown 0.16.1",
"log",
"presser",
"thiserror 2.0.20",
"windows",
]
[[package]] [[package]]
name = "group" name = "group"
version = "0.13.0" version = "0.13.0"
@@ -1419,6 +1507,18 @@ dependencies = [
"subtle", "subtle",
] ]
[[package]]
name = "half"
version = "2.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b"
dependencies = [
"cfg-if",
"crunchy",
"num-traits",
"zerocopy",
]
[[package]] [[package]]
name = "hash32" name = "hash32"
version = "0.3.1" version = "0.3.1"
@@ -1444,6 +1544,9 @@ name = "hashbrown"
version = "0.17.1" version = "0.17.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a"
dependencies = [
"foldhash",
]
[[package]] [[package]]
name = "hashlink" name = "hashlink"
@@ -1974,6 +2077,23 @@ dependencies = [
"wasm-bindgen", "wasm-bindgen",
] ]
[[package]]
name = "khronos-egl"
version = "6.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6aae1df220ece3c0ada96b8153459b67eebe9ae9212258bb0134ae60416fdf76"
dependencies = [
"libc",
"libloading",
"pkg-config",
]
[[package]]
name = "khronos_api"
version = "3.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e2db585e1d738fc771bf08a151420d3ed193d9d895a36df7f6f8a9456b911ddc"
[[package]] [[package]]
name = "lazy_static" name = "lazy_static"
version = "1.5.0" version = "1.5.0"
@@ -2424,6 +2544,7 @@ dependencies = [
"tokio-rustls", "tokio-rustls",
"unicode-width 0.2.2", "unicode-width 0.2.2",
"url", "url",
"wgpu",
] ]
[[package]] [[package]]
@@ -2496,20 +2617,58 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "364f94bc34f61332abebe8cad6f6cd82a5b65cff22c828d05d0968911462ca4f" checksum = "364f94bc34f61332abebe8cad6f6cd82a5b65cff22c828d05d0968911462ca4f"
dependencies = [ dependencies = [
"arrayvec", "arrayvec",
"bit-set", "bit-set 0.8.0",
"bitflags 2.13.1", "bitflags 2.13.1",
"cfg_aliases 0.1.1", "cfg_aliases 0.1.1",
"codespan-reporting", "codespan-reporting 0.11.1",
"hexf-parse", "hexf-parse",
"indexmap", "indexmap",
"log", "log",
"rustc-hash 1.1.0", "rustc-hash 1.1.0",
"spirv", "spirv 0.3.0+sdk-1.3.268.0",
"termcolor", "termcolor",
"thiserror 1.0.69", "thiserror 1.0.69",
"unicode-xid", "unicode-xid",
] ]
[[package]]
name = "naga"
version = "30.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a616d2fb8c89516ac2723a581f69d6c18576046bed761bd6b305e5618e6ae130"
dependencies = [
"arrayvec",
"bit-set 0.10.0",
"bitflags 2.13.1",
"cfg-if",
"cfg_aliases 0.2.2",
"codespan-reporting 0.13.1",
"half",
"hashbrown 0.17.1",
"indexmap",
"libm",
"log",
"naga-types",
"num-traits",
"once_cell",
"rustc-hash 1.1.0",
"spirv 0.4.0+sdk-1.4.341.0",
"thiserror 2.0.20",
"unicode-ident",
]
[[package]]
name = "naga-types"
version = "30.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "590afbf58a6f4f62873cd5cff4468061844bafa1cdf399cc954537c22d768d49"
dependencies = [
"hashbrown 0.17.1",
"indexmap",
"rustc-hash 1.1.0",
"thiserror 2.0.20",
]
[[package]] [[package]]
name = "ndk" name = "ndk"
version = "0.9.0" version = "0.9.0"
@@ -2818,6 +2977,18 @@ dependencies = [
"objc2-core-foundation", "objc2-core-foundation",
] ]
[[package]]
name = "objc2-metal"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a0125f776a10d00af4152d74616409f0d4a2053a6f57fa5b7d6aa2854ac04794"
dependencies = [
"bitflags 2.13.1",
"block2",
"objc2",
"objc2-foundation",
]
[[package]] [[package]]
name = "objc2-quartz-core" name = "objc2-quartz-core"
version = "0.3.2" version = "0.3.2"
@@ -2827,7 +2998,9 @@ dependencies = [
"bitflags 2.13.1", "bitflags 2.13.1",
"objc2", "objc2",
"objc2-core-foundation", "objc2-core-foundation",
"objc2-core-graphics",
"objc2-foundation", "objc2-foundation",
"objc2-metal",
] ]
[[package]] [[package]]
@@ -2909,6 +3082,15 @@ version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d"
[[package]]
name = "ordered-float"
version = "5.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8c7c9e0d9b23589f26070720bac724174bfec1083e82f7854cdd0267518343c0"
dependencies = [
"num-traits",
]
[[package]] [[package]]
name = "os_info" name = "os_info"
version = "3.15.0" version = "3.15.0"
@@ -3050,6 +3232,21 @@ dependencies = [
"universal-hash", "universal-hash",
] ]
[[package]]
name = "portable-atomic"
version = "1.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85"
[[package]]
name = "portable-atomic-util"
version = "0.2.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618"
dependencies = [
"portable-atomic",
]
[[package]] [[package]]
name = "potential_utf" name = "potential_utf"
version = "0.1.5" version = "0.1.5"
@@ -3074,6 +3271,12 @@ dependencies = [
"zerocopy", "zerocopy",
] ]
[[package]]
name = "presser"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e8cf8e6a8aa66ce33f63993ffc4ea4271eb5b0530a9002db8455ea6050c77bfa"
[[package]] [[package]]
name = "prettyplease" name = "prettyplease"
version = "0.2.37" version = "0.2.37"
@@ -3111,6 +3314,12 @@ dependencies = [
"unicode-ident", "unicode-ident",
] ]
[[package]]
name = "profiling"
version = "1.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3d595e54a326bc53c1c197b32d295e14b169e3cfeaa8dc82b529f947fba6bcf5"
[[package]] [[package]]
name = "pxfm" name = "pxfm"
version = "0.1.30" version = "0.1.30"
@@ -3265,6 +3474,30 @@ dependencies = [
"rand_core 0.10.1", "rand_core 0.10.1",
] ]
[[package]]
name = "range-alloc"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ca45419789ae5a7899559e9512e58ca889e41f04f1f2445e9f4b290ceccd1d08"
[[package]]
name = "raw-window-handle"
version = "0.6.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "20675572f6f24e9e76ef639bc5552774ed45f1c30e2951e1e99c59888861c539"
[[package]]
name = "raw-window-metal"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "40d213455a5f1dc59214213c7330e074ddf8114c9a42411eb890c767357ce135"
dependencies = [
"objc2",
"objc2-core-foundation",
"objc2-foundation",
"objc2-quartz-core",
]
[[package]] [[package]]
name = "rayon" name = "rayon"
version = "1.12.0" version = "1.12.0"
@@ -3347,6 +3580,12 @@ version = "0.8.11"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4"
[[package]]
name = "renderdoc-sys"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "19b30a45b0cd0bcca8037f3d0dc3421eaf95327a17cad11964fb8179b4fc4832"
[[package]] [[package]]
name = "reqwest" name = "reqwest"
version = "0.12.28" version = "0.12.28"
@@ -3985,7 +4224,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f4b7ea0fc7fa887d5875e49809eabc3cbd9e6a0247930a9f6456ad4c59cedfb1" checksum = "f4b7ea0fc7fa887d5875e49809eabc3cbd9e6a0247930a9f6456ad4c59cedfb1"
dependencies = [ dependencies = [
"bitflags 2.13.1", "bitflags 2.13.1",
"naga", "naga 23.1.0",
"skia-rs-core", "skia-rs-core",
"skia-rs-path", "skia-rs-path",
"thiserror 2.0.20", "thiserror 2.0.20",
@@ -4032,6 +4271,15 @@ version = "0.4.12"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5"
[[package]]
name = "slotmap"
version = "1.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bdd58c3c93c3d278ca835519292445cb4b0d4dc59ccfdf7ceadaab3f8aeb4038"
dependencies = [
"version_check",
]
[[package]] [[package]]
name = "smallvec" name = "smallvec"
version = "1.15.2" version = "1.15.2"
@@ -4069,6 +4317,15 @@ dependencies = [
"bitflags 2.13.1", "bitflags 2.13.1",
] ]
[[package]]
name = "spirv"
version = "0.4.0+sdk-1.4.341.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d9571ea910ebd84c86af4b3ed27f9dbdc6ad06f17c5f96146b2b671e2976744f"
dependencies = [
"bitflags 2.13.1",
]
[[package]] [[package]]
name = "spki" name = "spki"
version = "0.7.3" version = "0.7.3"
@@ -4097,6 +4354,12 @@ version = "1.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596"
[[package]]
name = "static_assertions"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
[[package]] [[package]]
name = "stats_alloc" name = "stats_alloc"
version = "0.1.10" version = "0.1.10"
@@ -4795,6 +5058,18 @@ dependencies = [
"web-sys", "web-sys",
] ]
[[package]]
name = "wayland-sys"
version = "0.31.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d8eab23fefc9e41f8e841df4a9c707e8a8c4ed26e944ef69297184de2785e3be"
dependencies = [
"dlib",
"log",
"once_cell",
"pkg-config",
]
[[package]] [[package]]
name = "web-sys" name = "web-sys"
version = "0.3.104" version = "0.3.104"
@@ -4839,6 +5114,177 @@ version = "0.1.12"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88" checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88"
[[package]]
name = "wgpu"
version = "30.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "527ccdf43dd5b2e8676eed9984ce00e2bbb0a1b85b70c1969dcb6cd2eb55ab9e"
dependencies = [
"arrayvec",
"bitflags 2.13.1",
"bytemuck",
"cfg-if",
"cfg_aliases 0.2.2",
"document-features",
"hashbrown 0.17.1",
"js-sys",
"log",
"portable-atomic",
"profiling",
"raw-window-handle",
"smallvec",
"static_assertions",
"wasm-bindgen",
"wasm-bindgen-futures",
"web-sys",
"wgpu-core",
"wgpu-hal",
"wgpu-types",
]
[[package]]
name = "wgpu-core"
version = "30.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "14c018fce9b6270aa203c2fdd56f3cce996713534bd757e4ea58c8560b121f14"
dependencies = [
"arrayvec",
"bit-set 0.10.0",
"bit-vec 0.9.1",
"bitflags 2.13.1",
"bytemuck",
"cfg_aliases 0.2.2",
"document-features",
"hashbrown 0.17.1",
"indexmap",
"log",
"naga 30.0.1",
"naga-types",
"once_cell",
"parking_lot",
"portable-atomic",
"profiling",
"raw-window-handle",
"rustc-hash 1.1.0",
"smallvec",
"thiserror 2.0.20",
"wgpu-core-deps-apple",
"wgpu-core-deps-emscripten",
"wgpu-core-deps-windows-linux-android",
"wgpu-hal",
"wgpu-naga-bridge",
"wgpu-types",
]
[[package]]
name = "wgpu-core-deps-apple"
version = "30.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "061f3d319a40d39d00b1ecc2c33b89fe21d4e6fe01859df3500a3a8ecccd6b68"
dependencies = [
"wgpu-hal",
]
[[package]]
name = "wgpu-core-deps-emscripten"
version = "30.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d98b86cf4abf524a902dd35f18ca6a3f08fc2ae9847c8f10b48e30491b1f0b86"
dependencies = [
"wgpu-hal",
]
[[package]]
name = "wgpu-core-deps-windows-linux-android"
version = "30.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7586165fd5f6d881cb9ce4bb71f40d6caab2c0f1837e3fc1d9788a197fb6004f"
dependencies = [
"wgpu-hal",
]
[[package]]
name = "wgpu-hal"
version = "30.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6b7fb58561a792bc237628ba0792e332de418fefe145f13b5ed8201e6d52f58"
dependencies = [
"android_system_properties",
"arrayvec",
"ash",
"bit-set 0.10.0",
"bitflags 2.13.1",
"block2",
"bytemuck",
"cfg-if",
"cfg_aliases 0.2.2",
"glow",
"glutin_wgl_sys",
"gpu-allocator",
"hashbrown 0.17.1",
"js-sys",
"khronos-egl",
"libc",
"libloading",
"log",
"naga 30.0.1",
"naga-types",
"ndk-sys",
"objc2",
"objc2-core-foundation",
"objc2-core-graphics",
"objc2-foundation",
"objc2-metal",
"objc2-quartz-core",
"once_cell",
"ordered-float",
"parking_lot",
"portable-atomic",
"portable-atomic-util",
"profiling",
"range-alloc",
"raw-window-handle",
"raw-window-metal",
"renderdoc-sys",
"smallvec",
"static_assertions",
"thiserror 2.0.20",
"wasm-bindgen",
"wayland-sys",
"web-sys",
"wgpu-naga-bridge",
"wgpu-types",
"windows",
"windows-core",
"windows-result",
]
[[package]]
name = "wgpu-naga-bridge"
version = "30.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d2f62e73117bb7a62bfd9c5a5841438a823f6566c6442a808ee269d2d055c081"
dependencies = [
"naga 30.0.1",
"wgpu-types",
]
[[package]]
name = "wgpu-types"
version = "30.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "99dad6f1fbdbbdb4c278a6508b059d44688f5cebddf78d005a46a31340269286"
dependencies = [
"bitflags 2.13.1",
"bytemuck",
"js-sys",
"log",
"naga-types",
"raw-window-handle",
"static_assertions",
"web-sys",
]
[[package]] [[package]]
name = "widestring" name = "widestring"
version = "1.2.1" version = "1.2.1"
@@ -5140,6 +5586,12 @@ dependencies = [
"rustix", "rustix",
] ]
[[package]]
name = "xml-rs"
version = "0.8.29"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e450f9b2ed1dff33c94c12589a87338689467b9c4f5d8a5710bd09a847d2c8a7"
[[package]] [[package]]
name = "yasna" name = "yasna"
version = "0.6.0" version = "0.6.0"

View File

@@ -399,6 +399,7 @@ impl CameraFrame {
pub struct AgentManagerAgentMovementAgentCamera { pub struct AgentManagerAgentMovementAgentCamera {
pub far: f32, pub far: f32,
frame: Arc<Mutex<CameraFrame>>, frame: Arc<Mutex<CameraFrame>>,
vertical_fov_radians: Arc<AtomicU32>,
} }
impl fmt::Debug for AgentManagerAgentMovementAgentCamera { impl fmt::Debug for AgentManagerAgentMovementAgentCamera {
@@ -406,6 +407,7 @@ impl fmt::Debug for AgentManagerAgentMovementAgentCamera {
formatter formatter
.debug_struct("AgentCamera") .debug_struct("AgentCamera")
.field("far", &self.far) .field("far", &self.far)
.field("vertical_fov_radians", &self.vertical_fov_angle())
.field("frame", &*mutex(&self.frame)) .field("frame", &*mutex(&self.frame))
.finish() .finish()
} }
@@ -416,11 +418,20 @@ impl AgentManagerAgentMovementAgentCamera {
Ok(Self { Ok(Self {
far: 128.0, far: 128.0,
frame: Arc::new(Mutex::new(CameraFrame::default())), frame: Arc::new(Mutex::new(CameraFrame::default())),
vertical_fov_radians: Arc::new(AtomicU32::new(60.0_f32.to_radians().to_bits())),
}) })
} }
fn from_frame(frame: Arc<Mutex<CameraFrame>>, far: f32) -> Self { fn from_frame(
Self { far, frame } frame: Arc<Mutex<CameraFrame>>,
vertical_fov_radians: Arc<AtomicU32>,
far: f32,
) -> Self {
Self {
far,
frame,
vertical_fov_radians,
}
} }
pub fn look_at_with_vector3_vector3( pub fn look_at_with_vector3_vector3(
@@ -584,6 +595,21 @@ impl AgentManagerAgentMovementAgentCamera {
mutex(&self.frame).z_axis = value; mutex(&self.frame).z_axis = value;
} }
} }
#[must_use]
pub fn vertical_fov_angle(&self) -> f32 {
f32::from_bits(self.vertical_fov_radians.load(Ordering::Acquire))
}
pub fn set_vertical_fov_angle(&self, value: f32) -> Result<(), Error> {
if !value.is_finite() || !(10.0_f32.to_radians()..=140.0_f32.to_radians()).contains(&value)
{
return Err(Error::Argument);
}
self.vertical_fov_radians
.store(value.to_bits(), Ordering::Release);
Ok(())
}
} }
#[derive(Clone, Copy, Debug, PartialEq)] #[derive(Clone, Copy, Debug, PartialEq)]
@@ -733,6 +759,7 @@ pub(crate) struct AgentMovementRuntime {
auto_reset_controls: AtomicBool, auto_reset_controls: AtomicBool,
update_interval: AtomicI32, update_interval: AtomicI32,
camera_frame: Arc<Mutex<CameraFrame>>, camera_frame: Arc<Mutex<CameraFrame>>,
vertical_fov_radians: Arc<AtomicU32>,
last_update: Mutex<LastUpdate>, last_update: Mutex<LastUpdate>,
kinematics: RwLock<Kinematics>, kinematics: RwLock<Kinematics>,
teleport: Mutex<TeleportState>, teleport: Mutex<TeleportState>,
@@ -805,6 +832,7 @@ impl AgentMovementRuntime {
auto_reset_controls: AtomicBool::new(false), auto_reset_controls: AtomicBool::new(false),
update_interval: AtomicI32::new(interval), update_interval: AtomicI32::new(interval),
camera_frame: Arc::new(Mutex::new(CameraFrame::default())), camera_frame: Arc::new(Mutex::new(CameraFrame::default())),
vertical_fov_radians: Arc::new(AtomicU32::new(60.0_f32.to_radians().to_bits())),
last_update: Mutex::new(LastUpdate::default()), last_update: Mutex::new(LastUpdate::default()),
kinematics: RwLock::new(Kinematics::default()), kinematics: RwLock::new(Kinematics::default()),
teleport: Mutex::new(TeleportState::default()), teleport: Mutex::new(TeleportState::default()),
@@ -986,7 +1014,11 @@ impl AgentMovementRuntime {
} }
fn camera(&self, far: f32) -> AgentManagerAgentMovementAgentCamera { fn camera(&self, far: f32) -> AgentManagerAgentMovementAgentCamera {
AgentManagerAgentMovementAgentCamera::from_frame(Arc::clone(&self.camera_frame), far) AgentManagerAgentMovementAgentCamera::from_frame(
Arc::clone(&self.camera_frame),
Arc::clone(&self.vertical_fov_radians),
far,
)
} }
fn send_update( fn send_update(
@@ -2099,6 +2131,7 @@ impl AgentManagerAgentMovement {
} }
pub fn set_fov_vertical_angle(&self, angle: f32) -> Result<(), Error> { pub fn set_fov_vertical_angle(&self, angle: f32) -> Result<(), Error> {
self.camera.set_vertical_fov_angle(angle)?;
let (agent_id, session_id) = self.runtime.ids(); let (agent_id, session_id) = self.runtime.ids();
let mut packet = AgentFOVPacket::new_with_constructor()?; let mut packet = AgentFOVPacket::new_with_constructor()?;
packet.agent_data.agent_id = agent_id; packet.agent_data.agent_id = agent_id;
@@ -3167,6 +3200,7 @@ mod tests {
assert!(run.agent_data.always_run); assert!(run.agent_data.always_run);
manager.movement.set_fov_vertical_angle(1.25).unwrap(); manager.movement.set_fov_vertical_angle(1.25).unwrap();
assert_eq!(manager.movement.camera.vertical_fov_angle(), 1.25);
let fov = decode_packet::<AgentFOVPacket>(&grid.receive(PacketType::AgentFOV)).unwrap(); let fov = decode_packet::<AgentFOVPacket>(&grid.receive(PacketType::AgentFOV)).unwrap();
assert_eq!(fov.fov_block.vertical_angle, 1.25); assert_eq!(fov.fov_block.vertical_angle, 1.25);

View File

@@ -28,6 +28,7 @@ tokio = { version = "1.53.1", features = ["macros", "rt", "sync", "time"] }
tokio-rustls = { version = "0.26.4", default-features = false, features = ["aws_lc_rs", "tls12"] } tokio-rustls = { version = "0.26.4", default-features = false, features = ["aws_lc_rs", "tls12"] }
url = "2.5.8" url = "2.5.8"
unicode-width = "0.2" unicode-width = "0.2"
wgpu = { version = "30.0.1", default-features = false, features = ["std", "dx12", "metal", "gles", "vulkan", "wgsl"], optional = true }
[dev-dependencies] [dev-dependencies]
jpeg-decoder = { version = "0.3.2", default-features = false } jpeg-decoder = { version = "0.3.2", default-features = false }
@@ -38,7 +39,7 @@ tokio = { version = "1.53.1", features = ["io-util", "net", "rt-multi-thread", "
[features] [features]
default = [] default = []
live-grid = ["dep:libremetaverse", "dep:libremetaverse-imaging", "dep:libremetaverse-rendering-simple"] live-grid = ["dep:libremetaverse", "dep:libremetaverse-imaging", "dep:libremetaverse-rendering-simple", "dep:wgpu"]
[lints] [lints]
workspace = true workspace = true

View File

@@ -746,6 +746,82 @@ impl crate::behavior::EmbodimentSink for LibremetaverseEmbodimentSink {
.map_err(|_| crate::behavior::BehaviorError::NativeOperation) .map_err(|_| crate::behavior::BehaviorError::NativeOperation)
}) })
} }
fn set_camera(
&self,
generation: u64,
position: crate::perception::WorldPosition,
target: crate::perception::WorldPosition,
vertical_fov_degrees: f64,
cancellation: CancellationToken,
) -> crate::behavior::EmbodimentFuture<'_, ()> {
Box::pin(async move {
if cancellation.is_cancellation_requested() {
return Err(crate::behavior::BehaviorError::Cancelled);
}
self.ensure_generation(generation)?;
self.agent
.movement
.camera
.look_at_with_vector3_vector3(
libremetaverse_types::Vector3 {
x: position.x as f32,
y: position.y as f32,
z: position.z as f32,
},
libremetaverse_types::Vector3 {
x: target.x as f32,
y: target.y as f32,
z: target.z as f32,
},
)
.map_err(|_| crate::behavior::BehaviorError::NativeOperation)?;
self.agent
.movement
.set_fov_vertical_angle((vertical_fov_degrees as f32).to_radians())
.and_then(|()| self.agent.movement.send_update_with_boolean(Some(true)))
.map_err(|_| crate::behavior::BehaviorError::NativeOperation)
})
}
fn reset_camera(
&self,
generation: u64,
cancellation: CancellationToken,
) -> crate::behavior::EmbodimentFuture<'_, ()> {
Box::pin(async move {
if cancellation.is_cancellation_requested() {
return Err(crate::behavior::BehaviorError::Cancelled);
}
self.ensure_generation(generation)?;
let avatar = self.agent.sim_position();
let forward = libremetaverse_types::Vector3::mul_with_vector3_quaternion(
libremetaverse_types::Vector3::unit_x(),
self.agent.sim_rotation(),
);
let position = libremetaverse_types::Vector3 {
x: avatar.x,
y: avatar.y,
z: avatar.z + 1.6,
};
let target = libremetaverse_types::Vector3 {
x: position.x + forward.x * 4.0,
y: position.y + forward.y * 4.0,
z: position.z + forward.z * 4.0,
};
self.agent
.movement
.camera
.look_at_with_vector3_vector3(position, target)
.and_then(|()| {
self.agent
.movement
.set_fov_vertical_angle(60.0_f32.to_radians())
})
.and_then(|()| self.agent.movement.send_update_with_boolean(Some(true)))
.map_err(|_| crate::behavior::BehaviorError::NativeOperation)
})
}
} }
#[cfg(feature = "live-grid")] #[cfg(feature = "live-grid")]

View File

@@ -35,6 +35,8 @@ pub const STOP_TOOL: &str = "behavior_stop";
pub const SIT_TOOL: &str = "behavior_sit"; pub const SIT_TOOL: &str = "behavior_sit";
pub const STAND_TOOL: &str = "behavior_stand"; pub const STAND_TOOL: &str = "behavior_stand";
pub const CURRENT_POSE_TOOL: &str = "behavior_current_pose"; pub const CURRENT_POSE_TOOL: &str = "behavior_current_pose";
pub const CAMERA_SET_TOOL: &str = "behavior_camera_set";
pub const CAMERA_RESET_TOOL: &str = "behavior_camera_reset";
const WALK_POLL: Duration = Duration::from_millis(250); const WALK_POLL: Duration = Duration::from_millis(250);
const TURN_INTERVAL: Duration = Duration::from_millis(40); const TURN_INTERVAL: Duration = Duration::from_millis(40);
@@ -162,6 +164,23 @@ pub trait EmbodimentSink: Send + Sync + 'static {
fn stop(&self, generation: u64, cancellation: CancellationToken) -> EmbodimentFuture<'_, ()>; fn stop(&self, generation: u64, cancellation: CancellationToken) -> EmbodimentFuture<'_, ()>;
fn sit(&self, generation: u64, cancellation: CancellationToken) -> EmbodimentFuture<'_, ()>; fn sit(&self, generation: u64, cancellation: CancellationToken) -> EmbodimentFuture<'_, ()>;
fn stand(&self, generation: u64, cancellation: CancellationToken) -> EmbodimentFuture<'_, ()>; fn stand(&self, generation: u64, cancellation: CancellationToken) -> EmbodimentFuture<'_, ()>;
fn set_camera(
&self,
_generation: u64,
_position: WorldPosition,
_target: WorldPosition,
_vertical_fov_degrees: f64,
_cancellation: CancellationToken,
) -> EmbodimentFuture<'_, ()> {
Box::pin(async { Err(BehaviorError::TargetUnavailable) })
}
fn reset_camera(
&self,
_generation: u64,
_cancellation: CancellationToken,
) -> EmbodimentFuture<'_, ()> {
Box::pin(async { Err(BehaviorError::TargetUnavailable) })
}
} }
#[derive(Clone, Debug, Eq, PartialEq)] #[derive(Clone, Debug, Eq, PartialEq)]
@@ -229,6 +248,12 @@ enum BehaviorAction {
Sit, Sit,
Stand, Stand,
CurrentPose, CurrentPose,
CameraSet {
position: WorldPosition,
target: WorldPosition,
vertical_fov_degrees: f64,
},
CameraReset,
} }
impl BehaviorAction { impl BehaviorAction {
@@ -242,6 +267,8 @@ impl BehaviorAction {
Self::Sit => SIT_TOOL, Self::Sit => SIT_TOOL,
Self::Stand => STAND_TOOL, Self::Stand => STAND_TOOL,
Self::CurrentPose => CURRENT_POSE_TOOL, Self::CurrentPose => CURRENT_POSE_TOOL,
Self::CameraSet { .. } => CAMERA_SET_TOOL,
Self::CameraReset => CAMERA_RESET_TOOL,
} }
} }
} }
@@ -943,6 +970,7 @@ async fn wait_for_action_cancellation(
} }
} }
#[allow(clippy::too_many_lines)]
async fn perform_action( async fn perform_action(
controller: &BehaviorController, controller: &BehaviorController,
state: ReadyState, state: ReadyState,
@@ -1035,6 +1063,51 @@ async fn perform_action(
.await?; .await?;
Ok(json!({"status":"completed"})) Ok(json!({"status":"completed"}))
} }
BehaviorAction::CameraSet {
position,
target,
vertical_fov_degrees,
} => {
let pose = controller
.sink
.current_pose(state.generation, cancellation.clone())
.await?;
validate_pose(&pose, state)?;
validate_point(
pose.position,
*position,
controller.settings.max_attention_distance_meters,
)?;
validate_point(
pose.position,
*target,
controller.settings.max_attention_distance_meters,
)?;
if distance(*position, *target) < 0.25 || !(20.0..=120.0).contains(vertical_fov_degrees)
{
return Err(BehaviorError::InvalidArguments);
}
controller
.sink
.set_camera(
state.generation,
*position,
*target,
*vertical_fov_degrees,
cancellation,
)
.await?;
Ok(
json!({"status":"completed","position":position,"target":target,"vertical_fov_degrees":vertical_fov_degrees}),
)
}
BehaviorAction::CameraReset => {
controller
.sink
.reset_camera(state.generation, cancellation)
.await?;
Ok(json!({"status":"completed"}))
}
} }
} }
@@ -1363,6 +1436,28 @@ fn parse_action(name: &str, json_arguments: &str) -> Result<BehaviorAction, Beha
require_empty(args)?; require_empty(args)?;
Ok(BehaviorAction::CurrentPose) Ok(BehaviorAction::CurrentPose)
} }
CAMERA_SET_TOOL => {
if args.len() != 7 {
return Err(BehaviorError::InvalidArguments);
}
Ok(BehaviorAction::CameraSet {
position: WorldPosition {
x: number(args, "position_x")?,
y: number(args, "position_y")?,
z: number(args, "position_z")?,
},
target: WorldPosition {
x: number(args, "target_x")?,
y: number(args, "target_y")?,
z: number(args, "target_z")?,
},
vertical_fov_degrees: number(args, "vertical_fov_degrees")?,
})
}
CAMERA_RESET_TOOL => {
require_empty(args)?;
Ok(BehaviorAction::CameraReset)
}
_ => Err(BehaviorError::InvalidArguments), _ => Err(BehaviorError::InvalidArguments),
} }
} }
@@ -1402,7 +1497,7 @@ fn require_empty(args: &Map<String, Value>) -> Result<(), BehaviorError> {
.ok_or(BehaviorError::InvalidArguments) .ok_or(BehaviorError::InvalidArguments)
} }
#[allow(clippy::too_many_lines)] // One registry table keeps the eight schemas auditable together. #[allow(clippy::too_many_lines)] // One registry table keeps the ten schemas auditable together.
pub fn behavior_policy_tools(settings: &BehaviorSettings) -> Result<Vec<PolicyTool>, PolicyError> { pub fn behavior_policy_tools(settings: &BehaviorSettings) -> Result<Vec<PolicyTool>, PolicyError> {
let origins = || { let origins = || {
AllowedOrigins::new([ AllowedOrigins::new([
@@ -1490,10 +1585,43 @@ pub fn behavior_policy_tools(settings: &BehaviorSettings) -> Result<Vec<PolicyTo
( (
CURRENT_POSE_TOOL, CURRENT_POSE_TOOL,
"Read current embodied pose and session provenance.", "Read current embodied pose and session provenance.",
empty, empty.clone(),
false, false,
0, 0,
), ),
(
CAMERA_SET_TOOL,
"Place the camera near the avatar and look at a point. Use this to focus or orbit objects; coordinates are current-region meters.",
object(
&[
("position_x", ToolSchema::Number),
("position_y", ToolSchema::Number),
("position_z", ToolSchema::Number),
("target_x", ToolSchema::Number),
("target_y", ToolSchema::Number),
("target_z", ToolSchema::Number),
("vertical_fov_degrees", ToolSchema::Number),
],
&[
"position_x",
"position_y",
"position_z",
"target_x",
"target_y",
"target_z",
"vertical_fov_degrees",
],
),
true,
0,
),
(
CAMERA_RESET_TOOL,
"Reset the camera to the avatar position and facing direction.",
empty,
true,
0,
),
]; ];
specs specs
.into_iter() .into_iter()

View File

@@ -198,6 +198,33 @@ impl EmbodimentSink for FakeSink {
Ok(()) Ok(())
}) })
} }
fn set_camera(
&self,
_: u64,
_: WorldPosition,
_: WorldPosition,
_: f64,
_: CancellationToken,
) -> EmbodimentFuture<'_, ()> {
Box::pin(async move {
self.state
.lock()
.expect("state")
.calls
.push("camera_set".into());
Ok(())
})
}
fn reset_camera(&self, _: u64, _: CancellationToken) -> EmbodimentFuture<'_, ()> {
Box::pin(async move {
self.state
.lock()
.expect("state")
.calls
.push("camera_reset".into());
Ok(())
})
}
} }
struct FixedRandom(u64); struct FixedRandom(u64);
@@ -257,7 +284,7 @@ async fn run_tool(
#[test] #[test]
fn policy_exposes_only_bounded_high_level_actions() { fn policy_exposes_only_bounded_high_level_actions() {
let tools = behavior_policy_tools(&settings()).expect("tools"); let tools = behavior_policy_tools(&settings()).expect("tools");
assert_eq!(tools.len(), 8); assert_eq!(tools.len(), 10);
let names = tools let names = tools
.iter() .iter()
.map(|tool| tool.definition.name.as_str()) .map(|tool| tool.definition.name.as_str())
@@ -272,7 +299,9 @@ fn policy_exposes_only_bounded_high_level_actions() {
STOP_TOOL, STOP_TOOL,
SIT_TOOL, SIT_TOOL,
STAND_TOOL, STAND_TOOL,
CURRENT_POSE_TOOL CURRENT_POSE_TOOL,
CAMERA_SET_TOOL,
CAMERA_RESET_TOOL,
] ]
); );
assert!(names.iter().all(|name| !name.contains("teleport") assert!(names.iter().all(|name| !name.contains("teleport")
@@ -280,6 +309,50 @@ fn policy_exposes_only_bounded_high_level_actions() {
&& !name.contains("follow"))); && !name.contains("follow")));
} }
#[tokio::test(start_paused = true)]
async fn camera_tools_validate_and_route_bounded_view_changes() {
let region = uuid(10);
let sink = Arc::new(FakeSink::new(1, region));
let mut config = settings();
config.settle_delay = Duration::ZERO;
let handle =
BehaviorController::new(config.clone(), sink.clone(), 16, 32, Duration::from_secs(1))
.expect("controller")
.start();
let ingress = handle.ingress();
ingress.connected(1, region).expect("ready");
tokio::task::yield_now().await;
let backend = BehaviorBackend::new(ingress);
let set = run_tool(
&backend,
&config,
CAMERA_SET_TOOL,
json!({
"position_x":126.0,"position_y":128.0,"position_z":26.0,
"target_x":132.0,"target_y":128.0,"target_z":24.0,
"vertical_fov_degrees":55.0
}),
)
.await;
assert!(matches!(set, ToolCallOutcome::Completed { .. }));
let reset = run_tool(&backend, &config, CAMERA_RESET_TOOL, json!({})).await;
assert!(matches!(reset, ToolCallOutcome::Completed { .. }));
assert_eq!(sink.calls(), vec!["camera_set", "camera_reset"]);
let invalid = run_tool(
&backend,
&config,
CAMERA_SET_TOOL,
json!({
"position_x":126.0,"position_y":128.0,"position_z":26.0,
"target_x":126.0,"target_y":128.0,"target_z":26.0,
"vertical_fov_degrees":55.0
}),
)
.await;
assert!(matches!(invalid, ToolCallOutcome::Rejected { .. }));
handle.shutdown().await.expect("shutdown");
}
#[tokio::test(start_paused = true)] #[tokio::test(start_paused = true)]
async fn public_attention_waits_turns_and_returns_to_available() { async fn public_attention_waits_turns_and_returns_to_available() {
let region = uuid(10); let region = uuid(10);

View File

@@ -153,7 +153,7 @@ impl AgentControlTarget {
*lock(&self.build) = Some(build); *lock(&self.build) = Some(build);
} }
/// Attaches synthetic viewport cancellation and metadata to operator control. /// Attaches reconstructed viewport cancellation and metadata to operator control.
pub fn attach_vision_control(&self, vision: Arc<dyn crate::vision::VisionControl>) { pub fn attach_vision_control(&self, vision: Arc<dyn crate::vision::VisionControl>) {
*lock(&self.vision) = Some(vision); *lock(&self.vision) = Some(vision);
} }

View File

@@ -30,7 +30,7 @@ const MAX_DUPLICATE_IDS: usize = 16_384;
const MAX_DEBOUNCE_FRAGMENTS: usize = 16; const MAX_DEBOUNCE_FRAGMENTS: usize = 16;
const MAX_OBSERVATIONS: usize = 8_192; const MAX_OBSERVATIONS: usize = 8_192;
const MENTRA_RUNTIME_IDENTIFIER: &str = "metacrate-grid-agent"; const MENTRA_RUNTIME_IDENTIFIER: &str = "metacrate-grid-agent";
const MENTRA_AGENT_PREFIX: &str = "grid-conversation-v5-"; const MENTRA_AGENT_PREFIX: &str = "grid-conversation-v6-";
const MENTRA_MEMORY_TOOLS: [&str; 3] = ["memory_search", "memory_pin", "memory_forget"]; const MENTRA_MEMORY_TOOLS: [&str; 3] = ["memory_search", "memory_pin", "memory_forget"];
pub(crate) const CURRENT_TURN_SYSTEM_PROMPT: &str = pub(crate) const CURRENT_TURN_SYSTEM_PROMPT: &str =
"You are in an OpenSim virtual world. Use the available tools to act there."; "You are in an OpenSim virtual world. Use the available tools to act there.";
@@ -1434,7 +1434,7 @@ async fn complete_work(
.await; .await;
} }
fn classify_intent( pub(crate) fn classify_intent(
channel: InteractionChannel, channel: InteractionChannel,
authorized: bool, authorized: bool,
message: &str, message: &str,
@@ -1487,7 +1487,18 @@ fn command_capabilities(message: &str) -> BTreeSet<crate::policy::Capability> {
if tokens.iter().any(|token| { if tokens.iter().any(|token| {
matches!( matches!(
*token, *token,
"teleport" | "move" | "walk" | "stop" | "sit" | "stand" | "face" | "look" | "schedule" "teleport"
| "move"
| "walk"
| "stop"
| "sit"
| "stand"
| "face"
| "look"
| "camera"
| "focus"
| "orbit"
| "schedule"
) )
}) { }) {
capabilities.insert(Capability::Movement); capabilities.insert(Capability::Movement);

View File

@@ -483,6 +483,26 @@ fn explicit_command_only_exposes_its_capability_family() {
crate::Capability::Movement, crate::Capability::Movement,
]) ])
); );
let camera = capabilities_for_intent(
InteractionIntent::PolicyGatedCommand,
"Use behavior_camera_set to position your camera and focus on the object",
);
assert_eq!(
camera,
BTreeSet::from([
crate::Capability::Informational,
crate::Capability::Movement,
])
);
assert_eq!(
classify_intent(
InteractionChannel::DirectIm,
true,
"Use behavior_camera_set to position your camera and focus on the object",
),
InteractionIntent::PolicyGatedCommand
);
} }
#[tokio::test(start_paused = true)] #[tokio::test(start_paused = true)]

View File

@@ -72,9 +72,9 @@ pub use backend::{
pub use behavior::{ pub use behavior::{
AuthorizedBackendRouter, BehaviorBackend, BehaviorController, BehaviorError, BehaviorHandle, AuthorizedBackendRouter, BehaviorBackend, BehaviorController, BehaviorError, BehaviorHandle,
BehaviorIngress, BehaviorMode, BehaviorObservation, BehaviorOutcome, BehaviorPolicyResult, BehaviorIngress, BehaviorMode, BehaviorObservation, BehaviorOutcome, BehaviorPolicyResult,
BehaviorRandom, BehaviorTrigger, CURRENT_POSE_TOOL, EmbodiedPose, EmbodimentFuture, BehaviorRandom, BehaviorTrigger, CAMERA_RESET_TOOL, CAMERA_SET_TOOL, CURRENT_POSE_TOOL,
EmbodimentSink, FACE_AVATAR_TOOL, FACE_POINT_TOOL, LOOK_AROUND_TOOL, SIT_TOOL, STAND_TOOL, EmbodiedPose, EmbodimentFuture, EmbodimentSink, FACE_AVATAR_TOOL, FACE_POINT_TOOL,
STOP_TOOL, WALK_SHORT_TOOL, behavior_policy_tools, LOOK_AROUND_TOOL, SIT_TOOL, STAND_TOOL, STOP_TOOL, WALK_SHORT_TOOL, behavior_policy_tools,
}; };
#[cfg(feature = "live-grid")] #[cfg(feature = "live-grid")]
pub use build::LibremetaverseBuildGrid; pub use build::LibremetaverseBuildGrid;

View File

@@ -740,10 +740,13 @@ fn start_live_interactions(
config.storage_path.join("mentra"), config.storage_path.join("mentra"),
)?); )?);
let vision_limits = VisionLimits::default(); let vision_limits = VisionLimits::default();
let vision = Arc::new(VisionService::new( let vision = Arc::new(
Arc::new(LibremetaverseSceneSource::new(owner, vision_limits)), VisionService::new(
vision_limits, Arc::new(LibremetaverseSceneSource::new(owner, vision_limits)),
)?); vision_limits,
)?
.prefer_gpu(),
);
let responder = Arc::new(VisionAugmentedResponder::new(vision.clone(), responder)); let responder = Arc::new(VisionAugmentedResponder::new(vision.clone(), responder));
let sink = Arc::new(owner.interaction_sink()); let sink = Arc::new(owner.interaction_sink());
let pacer: Arc<dyn metacrate_grid_agent::ResponsePacer> = Arc::new(behavior_ingress); let pacer: Arc<dyn metacrate_grid_agent::ResponsePacer> = Arc::new(behavior_ingress);

View File

@@ -1,4 +1,4 @@
//! Bounded deterministic synthetic viewport rendering and capture ownership. //! Bounded reconstructed viewport rendering and capture ownership.
#![allow(clippy::missing_errors_doc)] #![allow(clippy::missing_errors_doc)]
#![allow( #![allow(
@@ -218,6 +218,10 @@ pub struct VisionService<S: SceneSource> {
request_sequence: AtomicU64, request_sequence: AtomicU64,
current_generation: AtomicU64, current_generation: AtomicU64,
state: Mutex<VisionState>, state: Mutex<VisionState>,
#[cfg(feature = "live-grid")]
gpu_preferred: bool,
#[cfg(feature = "live-grid")]
gpu: tokio::sync::OnceCell<Option<Arc<WgpuRenderer>>>,
} }
impl<S: SceneSource> VisionService<S> { impl<S: SceneSource> VisionService<S> {
@@ -236,8 +240,20 @@ impl<S: SceneSource> VisionService<S> {
active: None, active: None,
observations: Vec::new(), observations: Vec::new(),
}), }),
#[cfg(feature = "live-grid")]
gpu_preferred: false,
#[cfg(feature = "live-grid")]
gpu: tokio::sync::OnceCell::new(),
}) })
} }
/// Prefer cached offscreen `wgpu` rendering, falling back to the
/// deterministic software rasterizer when no adapter is available.
#[cfg(feature = "live-grid")]
#[must_use]
pub fn prefer_gpu(mut self) -> Self {
self.gpu_preferred = true;
self
}
pub fn set_generation(&self, generation: u64) { pub fn set_generation(&self, generation: u64) {
if self.current_generation.swap(generation, Ordering::AcqRel) != generation { if self.current_generation.swap(generation, Ordering::AcqRel) != generation {
self.cancel_active(); self.cancel_active();
@@ -308,7 +324,7 @@ impl<S: SceneSource> VisionService<S> {
generation, generation,
VisionObservationKind::Started, VisionObservationKind::Started,
None, None,
"synthetic viewport capture started", "viewport capture started",
); );
let token = owned.token(); let token = owned.token();
let result = tokio::select! {()=cancellation.cancelled()=>Err(VisionError::Cancelled),()=token.cancelled()=>Err(VisionError::Superseded),result=tokio::time::timeout(self.limits.capture_timeout,self.source.capture_scene(generation,token.clone()))=>result.map_err(|_|VisionError::TimedOut)?}; let result = tokio::select! {()=cancellation.cancelled()=>Err(VisionError::Cancelled),()=token.cancelled()=>Err(VisionError::Superseded),result=tokio::time::timeout(self.limits.capture_timeout,self.source.capture_scene(generation,token.clone()))=>result.map_err(|_|VisionError::TimedOut)?};
@@ -334,7 +350,7 @@ impl<S: SceneSource> VisionService<S> {
} else if generation != self.current_generation.load(Ordering::Acquire) { } else if generation != self.current_generation.load(Ordering::Acquire) {
Err(VisionError::StaleGeneration) Err(VisionError::StaleGeneration)
} else { } else {
render_scene(&scene, self.limits).inspect(|capture| { self.render_scene(scene).await.inspect(|capture| {
self.observe( self.observe(
correlation, correlation,
generation, generation,
@@ -373,6 +389,31 @@ impl<S: SceneSource> VisionService<S> {
} }
result result
} }
async fn render_scene(&self, scene: SceneSnapshot) -> Result<VisionCapture, VisionError> {
#[cfg(feature = "live-grid")]
if self.gpu_preferred {
let renderer = self
.gpu
.get_or_init(|| async { WgpuRenderer::new().await.ok().map(Arc::new) })
.await
.clone();
if let Some(renderer) = renderer {
let gpu_scene = scene.clone();
let limits = self.limits;
if let Ok(Ok(capture)) = tokio::task::spawn_blocking(move || {
renderer
.render(&gpu_scene, limits)
.and_then(|rgba| finish_capture(&gpu_scene, limits, &rgba))
})
.await
{
return Ok(capture);
}
}
}
render_scene_software(&scene, self.limits)
}
} }
/// Type-erased runtime control surface. It deliberately exposes metadata and /// Type-erased runtime control surface. It deliberately exposes metadata and
@@ -428,7 +469,7 @@ impl<S: SceneSource> VisionControl for VisionService<S> {
} }
} }
/// Provider-neutral responder decorator that adds a synthetic viewport only /// Provider-neutral responder decorator that adds a reconstructed viewport only
/// for an explicit visual question. An endpoint capability rejection produces /// for an explicit visual question. An endpoint capability rejection produces
/// a textual scene-summary response without retrying the large image request. /// a textual scene-summary response without retrying the large image request.
pub struct VisionAugmentedResponder<S: SceneSource, R: crate::interaction::InteractionResponder> { pub struct VisionAugmentedResponder<S: SceneSource, R: crate::interaction::InteractionResponder> {
@@ -549,7 +590,10 @@ pub fn visual_question(text: &str) -> bool {
.any(|needle| lower.contains(needle)) .any(|needle| lower.contains(needle))
} }
fn render_scene(scene: &SceneSnapshot, limits: VisionLimits) -> Result<VisionCapture, VisionError> { fn render_scene_software(
scene: &SceneSnapshot,
limits: VisionLimits,
) -> Result<VisionCapture, VisionError> {
validate_scene(scene, limits)?; validate_scene(scene, limits)?;
let pixel_count = limits.width as usize * limits.height as usize; let pixel_count = limits.width as usize * limits.height as usize;
let mut rgba = vec![0u8; pixel_count * 4]; let mut rgba = vec![0u8; pixel_count * 4];
@@ -581,7 +625,15 @@ fn render_scene(scene: &SceneSnapshot, limits: VisionLimits) -> Result<VisionCap
triangle, triangle,
); );
} }
let jpeg = encode_jpeg(limits.width, limits.height, &rgba, limits.max_jpeg_bytes)?; finish_capture(scene, limits, &rgba)
}
fn finish_capture(
scene: &SceneSnapshot,
limits: VisionLimits,
rgba: &[u8],
) -> Result<VisionCapture, VisionError> {
let jpeg = encode_jpeg(limits.width, limits.height, rgba, limits.max_jpeg_bytes)?;
let hash = Sha256::digest(&jpeg) let hash = Sha256::digest(&jpeg)
.iter() .iter()
.fold(String::with_capacity(64), |mut output, byte| { .fold(String::with_capacity(64), |mut output, byte| {
@@ -764,6 +816,316 @@ fn environment_color(scene: &SceneSnapshot) -> [u8; 4] {
255, 255,
] ]
} }
#[cfg(feature = "live-grid")]
pub(crate) struct WgpuRenderer {
device: wgpu::Device,
queue: wgpu::Queue,
pipeline: wgpu::RenderPipeline,
}
#[cfg(feature = "live-grid")]
const WGPU_VERTEX_ATTRIBUTES: [wgpu::VertexAttribute; 2] =
wgpu::vertex_attr_array![0 => Float32x3, 1 => Float32x4];
#[cfg(feature = "live-grid")]
impl WgpuRenderer {
pub(crate) async fn new() -> Result<Self, VisionError> {
let instance = wgpu::Instance::new(wgpu::InstanceDescriptor::new_without_display_handle());
let adapter = instance
.request_adapter(&wgpu::RequestAdapterOptions {
power_preference: wgpu::PowerPreference::LowPower,
..Default::default()
})
.await
.map_err(|_| VisionError::Encode)?;
let (device, queue) = adapter
.request_device(&wgpu::DeviceDescriptor {
label: Some("metacrate-agent-vision"),
required_features: wgpu::Features::empty(),
required_limits: wgpu::Limits::downlevel_defaults(),
memory_hints: wgpu::MemoryHints::MemoryUsage,
..Default::default()
})
.await
.map_err(|_| VisionError::Encode)?;
let shader = device.create_shader_module(wgpu::ShaderModuleDescriptor {
label: Some("metacrate-agent-vision-shader"),
source: wgpu::ShaderSource::Wgsl(
r"
struct VertexInput {
@location(0) position: vec3<f32>,
@location(1) color: vec4<f32>,
};
struct VertexOutput {
@builtin(position) position: vec4<f32>,
@location(0) color: vec4<f32>,
};
@vertex
fn vs_main(input: VertexInput) -> VertexOutput {
var output: VertexOutput;
output.position = vec4<f32>(input.position, 1.0);
output.color = input.color;
return output;
}
@fragment
fn fs_main(input: VertexOutput) -> @location(0) vec4<f32> {
return input.color;
}
"
.into(),
),
});
let layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor {
label: Some("metacrate-agent-vision-layout"),
bind_group_layouts: &[],
immediate_size: 0,
});
let pipeline = device.create_render_pipeline(&wgpu::RenderPipelineDescriptor {
label: Some("metacrate-agent-vision-pipeline"),
layout: Some(&layout),
vertex: wgpu::VertexState {
module: &shader,
entry_point: Some("vs_main"),
compilation_options: wgpu::PipelineCompilationOptions::default(),
buffers: &[Some(wgpu::VertexBufferLayout {
array_stride: 7 * size_of::<f32>() as u64,
step_mode: wgpu::VertexStepMode::Vertex,
attributes: &WGPU_VERTEX_ATTRIBUTES,
})],
},
primitive: wgpu::PrimitiveState {
topology: wgpu::PrimitiveTopology::TriangleList,
cull_mode: None,
..Default::default()
},
depth_stencil: Some(wgpu::DepthStencilState {
format: wgpu::TextureFormat::Depth32Float,
depth_write_enabled: Some(true),
depth_compare: Some(wgpu::CompareFunction::Less),
stencil: wgpu::StencilState::default(),
bias: wgpu::DepthBiasState::default(),
}),
multisample: wgpu::MultisampleState::default(),
fragment: Some(wgpu::FragmentState {
module: &shader,
entry_point: Some("fs_main"),
compilation_options: wgpu::PipelineCompilationOptions::default(),
targets: &[Some(wgpu::ColorTargetState {
format: wgpu::TextureFormat::Rgba8UnormSrgb,
blend: None,
write_mask: wgpu::ColorWrites::ALL,
})],
}),
multiview_mask: None,
cache: None,
});
Ok(Self {
device,
queue,
pipeline,
})
}
#[allow(clippy::too_many_lines)]
pub(crate) fn render(
&self,
scene: &SceneSnapshot,
limits: VisionLimits,
) -> Result<Vec<u8>, VisionError> {
validate_scene(scene, limits)?;
let (vertices, vertex_count) = gpu_vertices(scene, limits)?;
let vertex_buffer = self.device.create_buffer(&wgpu::BufferDescriptor {
label: Some("metacrate-agent-vision-vertices"),
size: vertices.len().max(4) as u64,
usage: wgpu::BufferUsages::VERTEX | wgpu::BufferUsages::COPY_DST,
mapped_at_creation: false,
});
if !vertices.is_empty() {
self.queue.write_buffer(&vertex_buffer, 0, &vertices);
}
let extent = wgpu::Extent3d {
width: limits.width,
height: limits.height,
depth_or_array_layers: 1,
};
let color = self.device.create_texture(&wgpu::TextureDescriptor {
label: Some("metacrate-agent-vision-color"),
size: extent,
mip_level_count: 1,
sample_count: 1,
dimension: wgpu::TextureDimension::D2,
format: wgpu::TextureFormat::Rgba8UnormSrgb,
usage: wgpu::TextureUsages::RENDER_ATTACHMENT | wgpu::TextureUsages::COPY_SRC,
view_formats: &[],
});
let depth = self.device.create_texture(&wgpu::TextureDescriptor {
label: Some("metacrate-agent-vision-depth"),
size: extent,
mip_level_count: 1,
sample_count: 1,
dimension: wgpu::TextureDimension::D2,
format: wgpu::TextureFormat::Depth32Float,
usage: wgpu::TextureUsages::RENDER_ATTACHMENT,
view_formats: &[],
});
let row_bytes = limits
.width
.checked_mul(4)
.ok_or(VisionError::ResourceLimit)?;
let padded_row_bytes = row_bytes.div_ceil(wgpu::COPY_BYTES_PER_ROW_ALIGNMENT)
* wgpu::COPY_BYTES_PER_ROW_ALIGNMENT;
let readback_size = u64::from(padded_row_bytes)
.checked_mul(u64::from(limits.height))
.ok_or(VisionError::ResourceLimit)?;
let readback = self.device.create_buffer(&wgpu::BufferDescriptor {
label: Some("metacrate-agent-vision-readback"),
size: readback_size,
usage: wgpu::BufferUsages::COPY_DST | wgpu::BufferUsages::MAP_READ,
mapped_at_creation: false,
});
let color_view = color.create_view(&wgpu::TextureViewDescriptor::default());
let depth_view = depth.create_view(&wgpu::TextureViewDescriptor::default());
let sky = environment_color(scene);
let mut encoder = self
.device
.create_command_encoder(&wgpu::CommandEncoderDescriptor {
label: Some("metacrate-agent-vision-commands"),
});
{
let mut pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor {
label: Some("metacrate-agent-vision-pass"),
color_attachments: &[Some(wgpu::RenderPassColorAttachment {
view: &color_view,
depth_slice: None,
resolve_target: None,
ops: wgpu::Operations {
load: wgpu::LoadOp::Clear(wgpu::Color {
r: f64::from(sky[0]) / 255.0,
g: f64::from(sky[1]) / 255.0,
b: f64::from(sky[2]) / 255.0,
a: 1.0,
}),
store: wgpu::StoreOp::Store,
},
})],
depth_stencil_attachment: Some(wgpu::RenderPassDepthStencilAttachment {
view: &depth_view,
depth_ops: Some(wgpu::Operations {
load: wgpu::LoadOp::Clear(1.0),
store: wgpu::StoreOp::Discard,
}),
stencil_ops: None,
}),
..Default::default()
});
if vertex_count != 0 {
pass.set_pipeline(&self.pipeline);
pass.set_vertex_buffer(0, vertex_buffer.slice(..));
pass.draw(0..vertex_count, 0..1);
}
}
encoder.copy_texture_to_buffer(
wgpu::TexelCopyTextureInfo {
texture: &color,
mip_level: 0,
origin: wgpu::Origin3d::ZERO,
aspect: wgpu::TextureAspect::All,
},
wgpu::TexelCopyBufferInfo {
buffer: &readback,
layout: wgpu::TexelCopyBufferLayout {
offset: 0,
bytes_per_row: Some(padded_row_bytes),
rows_per_image: Some(limits.height),
},
},
extent,
);
let submission = self.queue.submit([encoder.finish()]);
let slice = readback.slice(..);
let (sender, receiver) = std::sync::mpsc::sync_channel(1);
slice.map_async(wgpu::MapMode::Read, move |result| {
let _ = sender.send(result);
});
self.device
.poll(wgpu::PollType::Wait {
submission_index: Some(submission),
timeout: Some(Duration::from_secs(5)),
})
.map_err(|_| VisionError::TimedOut)?;
receiver
.recv_timeout(Duration::from_secs(1))
.map_err(|_| VisionError::TimedOut)?
.map_err(|_| VisionError::Encode)?;
let mapped = slice.get_mapped_range().map_err(|_| VisionError::Encode)?;
let row_bytes = row_bytes as usize;
let padded_row_bytes = padded_row_bytes as usize;
let mut rgba = Vec::with_capacity(row_bytes * limits.height as usize);
for row in mapped
.chunks_exact(padded_row_bytes)
.take(limits.height as usize)
{
rgba.extend_from_slice(&row[..row_bytes]);
}
drop(mapped);
readback.unmap();
Ok(rgba)
}
}
#[cfg(feature = "live-grid")]
fn gpu_vertices(
scene: &SceneSnapshot,
limits: VisionLimits,
) -> Result<(Vec<u8>, u32), VisionError> {
let basis = camera_basis(scene.camera)?;
let aspect = limits.width as f32 / limits.height as f32;
let focal = 1.0 / (scene.camera.vertical_fov_degrees.to_radians() / 2.0).tan();
let mut triangles = scene
.entities
.iter()
.flat_map(|entity| {
entity
.triangles
.iter()
.map(move |triangle| (entity, triangle))
})
.collect::<Vec<_>>();
triangles.sort_by_key(|(entity, _)| entity.id.to_string());
let mut bytes = Vec::with_capacity(triangles.len() * 3 * 7 * size_of::<f32>());
let mut count = 0u32;
for (_, triangle) in triangles {
let projected = triangle.vertices.map(|world| {
let delta = [
world[0] - scene.camera.position[0],
world[1] - scene.camera.position[1],
world[2] - scene.camera.position[2],
];
let z = dot(delta, basis.2);
[
dot(delta, basis.0) * focal / aspect / z,
dot(delta, basis.1) * focal / z,
(z / 512.0).clamp(0.0, 1.0),
z,
]
});
if projected.iter().any(|vertex| vertex[3] <= 0.05) {
continue;
}
let color = triangle
.color_srgb
.map(|channel| f32::from(channel) / 255.0);
for vertex in projected {
for value in vertex[..3].iter().chain(color.iter()) {
bytes.extend_from_slice(&value.to_ne_bytes());
}
count = count.checked_add(1).ok_or(VisionError::ResourceLimit)?;
}
}
Ok((bytes, count))
}
fn encode_jpeg( fn encode_jpeg(
width: u32, width: u32,
height: u32, height: u32,
@@ -800,7 +1162,7 @@ fn scene_summary(scene: &SceneSnapshot) -> String {
.filter(|e| e.kind == SceneEntityKind::Resident) .filter(|e| e.kind == SceneEntityKind::Resident)
.count(); .count();
format!( format!(
"Synthetic viewport of region {:?}: {objects} tracked objects and {residents} privacy-marked residents. generation={} observed_unix_millis={}. completeness: objects_truncated={}, avatars_truncated={}, textures_missing={}, terrain_available={}. This is reconstructed from available scene data, not a viewer framebuffer.", "Reconstructed viewport of region {:?}: {objects} tracked objects and {residents} privacy-marked residents. generation={} observed_unix_millis={}. completeness: objects_truncated={}, avatars_truncated={}, textures_missing={}, terrain_available={}. This is reconstructed from available scene data, not a viewer framebuffer.",
sanitize(&scene.region_name), sanitize(&scene.region_name),
scene.generation, scene.generation,
scene.observed_unix_millis, scene.observed_unix_millis,
@@ -979,7 +1341,7 @@ impl SceneSource for LibremetaverseSceneSource {
position: [position.x, position.y, position.z], position: [position.x, position.y, position.z],
forward: [forward.x, forward.y, forward.z], forward: [forward.x, forward.y, forward.z],
up: [up.x, up.y, up.z], up: [up.x, up.y, up.z],
vertical_fov_degrees: 60.0, vertical_fov_degrees: camera.vertical_fov_angle().to_degrees(),
}, },
entities, entities,
completeness: SnapshotCompleteness { completeness: SnapshotCompleteness {

View File

@@ -120,6 +120,27 @@ async fn golden_scene_is_deterministic_depth_ordered_and_privacy_marked() {
assert!(pixels[center + 1] > 180 && pixels[center] < 80 && pixels[center + 2] < 80); assert!(pixels[center + 1] > 180 && pixels[center] < 80 && pixels[center + 2] < 80);
} }
#[cfg(feature = "live-grid")]
#[tokio::test(flavor = "multi_thread")]
async fn offscreen_wgpu_renders_depth_ordered_scene_when_an_adapter_is_available() {
let Ok(renderer) = WgpuRenderer::new().await else {
return;
};
let limits = VisionLimits {
width: 64,
height: 64,
minimum_interval: std::time::Duration::ZERO,
..VisionLimits::default()
};
let rgba = tokio::task::spawn_blocking(move || renderer.render(&scene(), limits))
.await
.expect("render worker")
.expect("offscreen render");
assert_eq!(rgba.len(), 64 * 64 * 4);
let center = (32 * 64 + 32) * 4;
assert!(rgba[center + 1] > 180 && rgba[center] < 80 && rgba[center + 2] < 80);
}
#[tokio::test] #[tokio::test]
async fn invalid_camera_huge_scene_stale_generation_and_size_limit_fail_closed() { async fn invalid_camera_huge_scene_stale_generation_and_size_limit_fail_closed() {
let mut bad = scene(); let mut bad = scene();
@@ -332,7 +353,7 @@ async fn responder_adds_generic_image_payload_and_returns_summary_on_capability_
.await .await
.unwrap(); .unwrap();
assert!(response.as_str().contains("does not support visual input")); assert!(response.as_str().contains("does not support visual input"));
assert!(response.as_str().contains("Synthetic viewport")); assert!(response.as_str().contains("Reconstructed viewport"));
assert_eq!(*endpoint.parts.lock().unwrap(), 3); assert_eq!(*endpoint.parts.lock().unwrap(), 3);
} }

View File

@@ -2,7 +2,7 @@ use std::collections::BTreeSet;
use std::fs; use std::fs;
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
const ALLOWED_DEPENDENCIES: [&str; 19] = [ const ALLOWED_DEPENDENCIES: [&str; 20] = [
"async-trait", "async-trait",
"base64", "base64",
"crossterm", "crossterm",
@@ -22,6 +22,7 @@ const ALLOWED_DEPENDENCIES: [&str; 19] = [
"tokio-rustls", "tokio-rustls",
"url", "url",
"unicode-width", "unicode-width",
"wgpu",
]; ];
#[test] #[test]
@@ -89,7 +90,7 @@ fn runtime_source_has_no_subprocess_or_native_abi_escape_hatch() {
} }
#[test] #[test]
fn vision_path_selects_only_the_pure_rust_renderer_and_codec() { fn vision_path_selects_only_reviewed_rust_renderers_and_codec() {
let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")); let root = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
let manifest = fs::read_to_string(root.join("Cargo.toml")).expect("agent manifest"); let manifest = fs::read_to_string(root.join("Cargo.toml")).expect("agent manifest");
assert!(manifest.contains("default-features = false, features = [\"rust-j2k\"]")); assert!(manifest.contains("default-features = false, features = [\"rust-j2k\"]"));
@@ -100,13 +101,7 @@ fn vision_path_selects_only_the_pure_rust_renderer_and_codec() {
.expect("simple renderer manifest"); .expect("simple renderer manifest");
assert!(!renderer.contains("imaging-skia")); assert!(!renderer.contains("imaging-skia"));
assert!(!renderer.contains("openjpeg")); assert!(!renderer.contains("openjpeg"));
let vision = fs::read_to_string(root.join("src/vision.rs")).expect("vision source"); assert!(manifest.contains("wgpu = { version = \"30.0.1\""));
for forbidden in ["screenshot crate", "gpu adapter", "std::process"] {
assert!(
!vision.to_ascii_lowercase().contains(forbidden),
"vision must remain synthetic and pure Rust: {forbidden}"
);
}
} }
#[test] #[test]

View File

@@ -113,8 +113,9 @@ hash. Do not record vendor presets or identifiers. Exercise, in order:
every created prim through the ownership-checked cleanup path. every created prim through the ownership-checked cleanup path.
6. Create and catalog a current-location landmark, accept a controlled landmark 6. Create and catalog a current-location landmark, accept a controlled landmark
offer, use a short bounded folder schedule, teleport, then disable the offer, use a short bounded folder schedule, teleport, then disable the
schedule. Capture the synthetic scene and ask one visual question. Record the endpoint schedule. Set and reset a bounded camera view, capture the reconstructed scene,
capability fallback if image input is rejected. and ask one visual question. Record the endpoint capability fallback if image
input is rejected.
7. Gracefully stop. Confirm no pending approvals, scheduled jobs, inventory 7. Gracefully stop. Confirm no pending approvals, scheduled jobs, inventory
offers, owned test prims, tasks, sockets, or sessions. List any unavoidable offers, owned test prims, tasks, sockets, or sessions. List any unavoidable
inventory artifact and its manual recovery ID in private operator notes, inventory artifact and its manual recovery ID in private operator notes,

View File

@@ -119,9 +119,15 @@ avatar session -> persistent Mentra agent -> Responses SSE / compaction / memory
| | | |
| +-> AuthorizedToolBackend | +-> AuthorizedToolBackend
+-> one-shot Mentra safety reviewer when required +-> one-shot Mentra safety reviewer when required
scene snapshot -> cached headless wgpu color/depth render -> JPEG -> Mentra image
`-> deterministic software fallback
``` ```
The package has no build script or direct native dependency. The focused The package has no build script. Its cross-platform `wgpu` path uses the
platform graphics backend selected by wgpu and requires no window or display;
the software fallback remains available when adapter creation or rendering
fails. The focused
`dependency_policy` test rejects subprocess launch sites, unsafe/native ABI `dependency_policy` test rejects subprocess launch sites, unsafe/native ABI
source, build scripts, and unreviewed direct dependency names in this package. source, build scripts, and unreviewed direct dependency names in this package.
The precise LLM compatibility and cancellation contract is documented in The precise LLM compatibility and cancellation contract is documented in

View File

@@ -56,16 +56,22 @@ facility, not a requirement for autonomous operation.
## Images ## Images
Vision captures are real software-rendered viewport images. MetaCrate renders Vision captures are reconstructed viewport images rather than screenshots.
the current scene, encodes the final frame directly as JPEG, and attaches it as The live service keeps one headless `wgpu` device, renders the current scene to
a Mentra image content block. The default 320x180 frame, quality, entity, an offscreen color/depth target, reads the frame back, encodes it directly as
triangle, texture-fetch, decoded-pixel, byte, rate, and concurrency limits are JPEG, and attaches it as a Mentra image content block. GPU setup and readback
validated before runtime. JPEG avoids the much larger PNG payloads. run outside the simulator event path. If no compatible adapter is available or
a frame fails, the same scene is rendered by the deterministic software
rasterizer. The default 320x180 frame, quality, entity, triangle, texture-fetch,
decoded-pixel, byte, rate, and concurrency limits are validated before runtime.
JPEG avoids the much larger PNG payloads.
The current renderer handles legacy prim geometry, approximate texture colors, The current renderer handles legacy prim geometry, approximate texture colors,
simple avatars, and flat terrain/water. Viewer-grade mesh/sculpt rendering, simple avatars, and flat terrain/water. `behavior_camera_set` gives an
full UV materials, lighting, authoritative terrain, and model-controlled camera authorized model a bounded position, target, and vertical field of view;
tools are tracked separately. `behavior_camera_reset` restores the avatar-facing view. Viewer-grade
mesh/sculpt rendering, full UV materials, lighting, authoritative terrain, and
avatar/attachment fidelity remain tracked in the renderer issue.
## Bounds and cancellation ## Bounds and cancellation

View File

@@ -226,8 +226,12 @@ The example records all current queue, message, conversation, tool, behavior,
reconnect, and interaction defaults. Important defaults include 256 grid events, reconnect, and interaction defaults. Important defaults include 256 grid events,
32 control commands, 512 observations, four concurrent inference requests, 32 control commands, 512 observations, four concurrent inference requests,
16 tool calls, 512 active senders/sessions, a two-minute model window, and bounded 16 tool calls, 512 active senders/sessions, a two-minute model window, and bounded
10-second shutdown. Vision defaults to a 320x180 software-rendered JPEG with bounded 10-second shutdown. Vision defaults to a 320x180 headless-wgpu JPEG with a
entities, triangles, texture work, JPEG bytes, time, and concurrency. deterministic software fallback and bounded entities, triangles, texture work,
JPEG bytes, time, and concurrency. Authorized `behavior_camera_set` calls are
limited to 96 meters from the avatar by default, require a distinct target, and
accept a 20-120 degree vertical field of view; `behavior_camera_reset` restores
the avatar-facing 60-degree view.
Unsupported by design: arbitrary raw packets or agent-control flags, arbitrary Unsupported by design: arbitrary raw packets or agent-control flags, arbitrary
shell/subprocess execution, provider/model discovery outside Mentra, remote plaintext shell/subprocess execution, provider/model discovery outside Mentra, remote plaintext
@@ -248,7 +252,8 @@ cargo audit
Record the binary byte size (`stat -c %s` on Linux or `Get-Item ... .Length` on Record the binary byte size (`stat -c %s` on Linux or `Get-Item ... .Length` on
PowerShell) and the `cargo tree` inventory. Reject CLR/scripting/provider SDK, PowerShell) and the `cargo tree` inventory. Reject CLR/scripting/provider SDK,
subprocess adapters, Skia/GPU, or undeclared native libraries on the agent path. subprocess adapters, unreviewed graphics stacks, or undeclared native libraries
on the agent path.
The consolidated Gitea gate runs only on `ubuntu-latest`; Windows portability is The consolidated Gitea gate runs only on `ubuntu-latest`; Windows portability is
proved by the existing cross-target compile/static gate rather than a Windows proved by the existing cross-target compile/static gate rather than a Windows
Gitea runner. Gitea runner.

View File

@@ -23,7 +23,7 @@ The audited tree contains the MetaCrate agent, the original compatibility
crates it consumes, the pure-Rust J2K/meshing/rendering path, Tokio, Reqwest, crates it consumes, the pure-Rust J2K/meshing/rendering path, Tokio, Reqwest,
Rustls, AWS-LC, terminal/serialization/bounds utilities, and their build-time Rustls, AWS-LC, terminal/serialization/bounds utilities, and their build-time
Rust tooling. It contains no CLR/Mono/.NET runtime, Python/Node/Lua scripting Rust tooling. It contains no CLR/Mono/.NET runtime, Python/Node/Lua scripting
runtime, provider SDK, Skia/GPU stack, OpenJPEG feature, shell/subprocess runtime, provider SDK, unreviewed graphics stack, shell/subprocess
adapter, or dynamically loaded undeclared application library. adapter, or dynamically loaded undeclared application library.
Regenerate for a release candidate with the commands in Regenerate for a release candidate with the commands in