Add headless wgpu vision and camera controls
This commit is contained in:
460
Cargo.lock
generated
460
Cargo.lock
generated
@@ -164,6 +164,15 @@ version = "0.7.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56"
|
||||
|
||||
[[package]]
|
||||
name = "ash"
|
||||
version = "0.38.0+1.3.281"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0bb44936d800fea8f016d7f2311c6a4f97aebd5dc86f09906139ec848cf3a46f"
|
||||
dependencies = [
|
||||
"libloading",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "asn1-rs"
|
||||
version = "0.7.2"
|
||||
@@ -303,6 +312,15 @@ dependencies = [
|
||||
"bit-vec 0.8.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bit-set"
|
||||
version = "0.10.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "09ec2f926cc3060f09db9ebc5b52823d85268d24bb917e472c0c4bea35780a7d"
|
||||
dependencies = [
|
||||
"bit-vec 0.9.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bit-vec"
|
||||
version = "0.8.0"
|
||||
@@ -568,6 +586,15 @@ dependencies = [
|
||||
"unicode-width 0.1.14",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "codespan-reporting"
|
||||
version = "0.13.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "af491d569909a7e4dee0ad7db7f5341fef5c614d5b8ec8cf765732aba3cff681"
|
||||
dependencies = [
|
||||
"unicode-width 0.2.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "color_quant"
|
||||
version = "1.1.0"
|
||||
@@ -765,6 +792,12 @@ dependencies = [
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crunchy"
|
||||
version = "0.2.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5"
|
||||
|
||||
[[package]]
|
||||
name = "crypto-bigint"
|
||||
version = "0.5.5"
|
||||
@@ -1015,6 +1048,15 @@ dependencies = [
|
||||
"syn 3.0.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dlib"
|
||||
version = "0.5.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ab8ecd87370524b461f8557c119c405552c396ed91fc0a8eec68679eab26f94a"
|
||||
dependencies = [
|
||||
"libloading",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "document-features"
|
||||
version = "0.2.12"
|
||||
@@ -1379,6 +1421,17 @@ dependencies = [
|
||||
"weezl",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "gl_generator"
|
||||
version = "0.14.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1a95dfc23a2b4a9a2f5ab41d194f8bfda3cabec42af4e39f08c339eb2a0c124d"
|
||||
dependencies = [
|
||||
"khronos_api",
|
||||
"log",
|
||||
"xml-rs",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "glam"
|
||||
version = "0.29.3"
|
||||
@@ -1397,6 +1450,27 @@ version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9985c9503b412198aa4197559e9a318524ebc4519c229bfa05a535828c950b9d"
|
||||
|
||||
[[package]]
|
||||
name = "glow"
|
||||
version = "0.17.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "29038e1c483364cc6bb3cf78feee1816002e127c331a1eec55a4d202b9e1adb5"
|
||||
dependencies = [
|
||||
"js-sys",
|
||||
"slotmap",
|
||||
"wasm-bindgen",
|
||||
"web-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "glutin_wgl_sys"
|
||||
version = "0.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2c4ee00b289aba7a9e5306d57c2d05499b2e5dc427f84ac708bd2c090212cf3e"
|
||||
dependencies = [
|
||||
"gl_generator",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "goblin"
|
||||
version = "0.10.5"
|
||||
@@ -1408,6 +1482,20 @@ dependencies = [
|
||||
"scroll",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "gpu-allocator"
|
||||
version = "0.28.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "51255ea7cfaadb6c5f1528d43e92a82acb2b96c43365989a28b2d44ee38f8795"
|
||||
dependencies = [
|
||||
"ash",
|
||||
"hashbrown 0.16.1",
|
||||
"log",
|
||||
"presser",
|
||||
"thiserror 2.0.20",
|
||||
"windows",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "group"
|
||||
version = "0.13.0"
|
||||
@@ -1419,6 +1507,18 @@ dependencies = [
|
||||
"subtle",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "half"
|
||||
version = "2.7.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"crunchy",
|
||||
"num-traits",
|
||||
"zerocopy",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hash32"
|
||||
version = "0.3.1"
|
||||
@@ -1444,6 +1544,9 @@ name = "hashbrown"
|
||||
version = "0.17.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a"
|
||||
dependencies = [
|
||||
"foldhash",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hashlink"
|
||||
@@ -1974,6 +2077,23 @@ dependencies = [
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "khronos-egl"
|
||||
version = "6.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6aae1df220ece3c0ada96b8153459b67eebe9ae9212258bb0134ae60416fdf76"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"libloading",
|
||||
"pkg-config",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "khronos_api"
|
||||
version = "3.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e2db585e1d738fc771bf08a151420d3ed193d9d895a36df7f6f8a9456b911ddc"
|
||||
|
||||
[[package]]
|
||||
name = "lazy_static"
|
||||
version = "1.5.0"
|
||||
@@ -2424,6 +2544,7 @@ dependencies = [
|
||||
"tokio-rustls",
|
||||
"unicode-width 0.2.2",
|
||||
"url",
|
||||
"wgpu",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2496,20 +2617,58 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "364f94bc34f61332abebe8cad6f6cd82a5b65cff22c828d05d0968911462ca4f"
|
||||
dependencies = [
|
||||
"arrayvec",
|
||||
"bit-set",
|
||||
"bit-set 0.8.0",
|
||||
"bitflags 2.13.1",
|
||||
"cfg_aliases 0.1.1",
|
||||
"codespan-reporting",
|
||||
"codespan-reporting 0.11.1",
|
||||
"hexf-parse",
|
||||
"indexmap",
|
||||
"log",
|
||||
"rustc-hash 1.1.0",
|
||||
"spirv",
|
||||
"spirv 0.3.0+sdk-1.3.268.0",
|
||||
"termcolor",
|
||||
"thiserror 1.0.69",
|
||||
"unicode-xid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "naga"
|
||||
version = "30.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a616d2fb8c89516ac2723a581f69d6c18576046bed761bd6b305e5618e6ae130"
|
||||
dependencies = [
|
||||
"arrayvec",
|
||||
"bit-set 0.10.0",
|
||||
"bitflags 2.13.1",
|
||||
"cfg-if",
|
||||
"cfg_aliases 0.2.2",
|
||||
"codespan-reporting 0.13.1",
|
||||
"half",
|
||||
"hashbrown 0.17.1",
|
||||
"indexmap",
|
||||
"libm",
|
||||
"log",
|
||||
"naga-types",
|
||||
"num-traits",
|
||||
"once_cell",
|
||||
"rustc-hash 1.1.0",
|
||||
"spirv 0.4.0+sdk-1.4.341.0",
|
||||
"thiserror 2.0.20",
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "naga-types"
|
||||
version = "30.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "590afbf58a6f4f62873cd5cff4468061844bafa1cdf399cc954537c22d768d49"
|
||||
dependencies = [
|
||||
"hashbrown 0.17.1",
|
||||
"indexmap",
|
||||
"rustc-hash 1.1.0",
|
||||
"thiserror 2.0.20",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ndk"
|
||||
version = "0.9.0"
|
||||
@@ -2818,6 +2977,18 @@ dependencies = [
|
||||
"objc2-core-foundation",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "objc2-metal"
|
||||
version = "0.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a0125f776a10d00af4152d74616409f0d4a2053a6f57fa5b7d6aa2854ac04794"
|
||||
dependencies = [
|
||||
"bitflags 2.13.1",
|
||||
"block2",
|
||||
"objc2",
|
||||
"objc2-foundation",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "objc2-quartz-core"
|
||||
version = "0.3.2"
|
||||
@@ -2827,7 +2998,9 @@ dependencies = [
|
||||
"bitflags 2.13.1",
|
||||
"objc2",
|
||||
"objc2-core-foundation",
|
||||
"objc2-core-graphics",
|
||||
"objc2-foundation",
|
||||
"objc2-metal",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2909,6 +3082,15 @@ version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d"
|
||||
|
||||
[[package]]
|
||||
name = "ordered-float"
|
||||
version = "5.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8c7c9e0d9b23589f26070720bac724174bfec1083e82f7854cdd0267518343c0"
|
||||
dependencies = [
|
||||
"num-traits",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "os_info"
|
||||
version = "3.15.0"
|
||||
@@ -3050,6 +3232,21 @@ dependencies = [
|
||||
"universal-hash",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "portable-atomic"
|
||||
version = "1.15.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85"
|
||||
|
||||
[[package]]
|
||||
name = "portable-atomic-util"
|
||||
version = "0.2.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618"
|
||||
dependencies = [
|
||||
"portable-atomic",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "potential_utf"
|
||||
version = "0.1.5"
|
||||
@@ -3074,6 +3271,12 @@ dependencies = [
|
||||
"zerocopy",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "presser"
|
||||
version = "0.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e8cf8e6a8aa66ce33f63993ffc4ea4271eb5b0530a9002db8455ea6050c77bfa"
|
||||
|
||||
[[package]]
|
||||
name = "prettyplease"
|
||||
version = "0.2.37"
|
||||
@@ -3111,6 +3314,12 @@ dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "profiling"
|
||||
version = "1.0.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3d595e54a326bc53c1c197b32d295e14b169e3cfeaa8dc82b529f947fba6bcf5"
|
||||
|
||||
[[package]]
|
||||
name = "pxfm"
|
||||
version = "0.1.30"
|
||||
@@ -3265,6 +3474,30 @@ dependencies = [
|
||||
"rand_core 0.10.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "range-alloc"
|
||||
version = "0.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ca45419789ae5a7899559e9512e58ca889e41f04f1f2445e9f4b290ceccd1d08"
|
||||
|
||||
[[package]]
|
||||
name = "raw-window-handle"
|
||||
version = "0.6.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "20675572f6f24e9e76ef639bc5552774ed45f1c30e2951e1e99c59888861c539"
|
||||
|
||||
[[package]]
|
||||
name = "raw-window-metal"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "40d213455a5f1dc59214213c7330e074ddf8114c9a42411eb890c767357ce135"
|
||||
dependencies = [
|
||||
"objc2",
|
||||
"objc2-core-foundation",
|
||||
"objc2-foundation",
|
||||
"objc2-quartz-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rayon"
|
||||
version = "1.12.0"
|
||||
@@ -3347,6 +3580,12 @@ version = "0.8.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4"
|
||||
|
||||
[[package]]
|
||||
name = "renderdoc-sys"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "19b30a45b0cd0bcca8037f3d0dc3421eaf95327a17cad11964fb8179b4fc4832"
|
||||
|
||||
[[package]]
|
||||
name = "reqwest"
|
||||
version = "0.12.28"
|
||||
@@ -3985,7 +4224,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f4b7ea0fc7fa887d5875e49809eabc3cbd9e6a0247930a9f6456ad4c59cedfb1"
|
||||
dependencies = [
|
||||
"bitflags 2.13.1",
|
||||
"naga",
|
||||
"naga 23.1.0",
|
||||
"skia-rs-core",
|
||||
"skia-rs-path",
|
||||
"thiserror 2.0.20",
|
||||
@@ -4032,6 +4271,15 @@ version = "0.4.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5"
|
||||
|
||||
[[package]]
|
||||
name = "slotmap"
|
||||
version = "1.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bdd58c3c93c3d278ca835519292445cb4b0d4dc59ccfdf7ceadaab3f8aeb4038"
|
||||
dependencies = [
|
||||
"version_check",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "smallvec"
|
||||
version = "1.15.2"
|
||||
@@ -4069,6 +4317,15 @@ dependencies = [
|
||||
"bitflags 2.13.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "spirv"
|
||||
version = "0.4.0+sdk-1.4.341.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d9571ea910ebd84c86af4b3ed27f9dbdc6ad06f17c5f96146b2b671e2976744f"
|
||||
dependencies = [
|
||||
"bitflags 2.13.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "spki"
|
||||
version = "0.7.3"
|
||||
@@ -4097,6 +4354,12 @@ version = "1.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596"
|
||||
|
||||
[[package]]
|
||||
name = "static_assertions"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
|
||||
|
||||
[[package]]
|
||||
name = "stats_alloc"
|
||||
version = "0.1.10"
|
||||
@@ -4795,6 +5058,18 @@ dependencies = [
|
||||
"web-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wayland-sys"
|
||||
version = "0.31.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d8eab23fefc9e41f8e841df4a9c707e8a8c4ed26e944ef69297184de2785e3be"
|
||||
dependencies = [
|
||||
"dlib",
|
||||
"log",
|
||||
"once_cell",
|
||||
"pkg-config",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "web-sys"
|
||||
version = "0.3.104"
|
||||
@@ -4839,6 +5114,177 @@ version = "0.1.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88"
|
||||
|
||||
[[package]]
|
||||
name = "wgpu"
|
||||
version = "30.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "527ccdf43dd5b2e8676eed9984ce00e2bbb0a1b85b70c1969dcb6cd2eb55ab9e"
|
||||
dependencies = [
|
||||
"arrayvec",
|
||||
"bitflags 2.13.1",
|
||||
"bytemuck",
|
||||
"cfg-if",
|
||||
"cfg_aliases 0.2.2",
|
||||
"document-features",
|
||||
"hashbrown 0.17.1",
|
||||
"js-sys",
|
||||
"log",
|
||||
"portable-atomic",
|
||||
"profiling",
|
||||
"raw-window-handle",
|
||||
"smallvec",
|
||||
"static_assertions",
|
||||
"wasm-bindgen",
|
||||
"wasm-bindgen-futures",
|
||||
"web-sys",
|
||||
"wgpu-core",
|
||||
"wgpu-hal",
|
||||
"wgpu-types",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wgpu-core"
|
||||
version = "30.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "14c018fce9b6270aa203c2fdd56f3cce996713534bd757e4ea58c8560b121f14"
|
||||
dependencies = [
|
||||
"arrayvec",
|
||||
"bit-set 0.10.0",
|
||||
"bit-vec 0.9.1",
|
||||
"bitflags 2.13.1",
|
||||
"bytemuck",
|
||||
"cfg_aliases 0.2.2",
|
||||
"document-features",
|
||||
"hashbrown 0.17.1",
|
||||
"indexmap",
|
||||
"log",
|
||||
"naga 30.0.1",
|
||||
"naga-types",
|
||||
"once_cell",
|
||||
"parking_lot",
|
||||
"portable-atomic",
|
||||
"profiling",
|
||||
"raw-window-handle",
|
||||
"rustc-hash 1.1.0",
|
||||
"smallvec",
|
||||
"thiserror 2.0.20",
|
||||
"wgpu-core-deps-apple",
|
||||
"wgpu-core-deps-emscripten",
|
||||
"wgpu-core-deps-windows-linux-android",
|
||||
"wgpu-hal",
|
||||
"wgpu-naga-bridge",
|
||||
"wgpu-types",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wgpu-core-deps-apple"
|
||||
version = "30.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "061f3d319a40d39d00b1ecc2c33b89fe21d4e6fe01859df3500a3a8ecccd6b68"
|
||||
dependencies = [
|
||||
"wgpu-hal",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wgpu-core-deps-emscripten"
|
||||
version = "30.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d98b86cf4abf524a902dd35f18ca6a3f08fc2ae9847c8f10b48e30491b1f0b86"
|
||||
dependencies = [
|
||||
"wgpu-hal",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wgpu-core-deps-windows-linux-android"
|
||||
version = "30.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7586165fd5f6d881cb9ce4bb71f40d6caab2c0f1837e3fc1d9788a197fb6004f"
|
||||
dependencies = [
|
||||
"wgpu-hal",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wgpu-hal"
|
||||
version = "30.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b6b7fb58561a792bc237628ba0792e332de418fefe145f13b5ed8201e6d52f58"
|
||||
dependencies = [
|
||||
"android_system_properties",
|
||||
"arrayvec",
|
||||
"ash",
|
||||
"bit-set 0.10.0",
|
||||
"bitflags 2.13.1",
|
||||
"block2",
|
||||
"bytemuck",
|
||||
"cfg-if",
|
||||
"cfg_aliases 0.2.2",
|
||||
"glow",
|
||||
"glutin_wgl_sys",
|
||||
"gpu-allocator",
|
||||
"hashbrown 0.17.1",
|
||||
"js-sys",
|
||||
"khronos-egl",
|
||||
"libc",
|
||||
"libloading",
|
||||
"log",
|
||||
"naga 30.0.1",
|
||||
"naga-types",
|
||||
"ndk-sys",
|
||||
"objc2",
|
||||
"objc2-core-foundation",
|
||||
"objc2-core-graphics",
|
||||
"objc2-foundation",
|
||||
"objc2-metal",
|
||||
"objc2-quartz-core",
|
||||
"once_cell",
|
||||
"ordered-float",
|
||||
"parking_lot",
|
||||
"portable-atomic",
|
||||
"portable-atomic-util",
|
||||
"profiling",
|
||||
"range-alloc",
|
||||
"raw-window-handle",
|
||||
"raw-window-metal",
|
||||
"renderdoc-sys",
|
||||
"smallvec",
|
||||
"static_assertions",
|
||||
"thiserror 2.0.20",
|
||||
"wasm-bindgen",
|
||||
"wayland-sys",
|
||||
"web-sys",
|
||||
"wgpu-naga-bridge",
|
||||
"wgpu-types",
|
||||
"windows",
|
||||
"windows-core",
|
||||
"windows-result",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wgpu-naga-bridge"
|
||||
version = "30.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d2f62e73117bb7a62bfd9c5a5841438a823f6566c6442a808ee269d2d055c081"
|
||||
dependencies = [
|
||||
"naga 30.0.1",
|
||||
"wgpu-types",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wgpu-types"
|
||||
version = "30.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "99dad6f1fbdbbdb4c278a6508b059d44688f5cebddf78d005a46a31340269286"
|
||||
dependencies = [
|
||||
"bitflags 2.13.1",
|
||||
"bytemuck",
|
||||
"js-sys",
|
||||
"log",
|
||||
"naga-types",
|
||||
"raw-window-handle",
|
||||
"static_assertions",
|
||||
"web-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "widestring"
|
||||
version = "1.2.1"
|
||||
@@ -5140,6 +5586,12 @@ dependencies = [
|
||||
"rustix",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "xml-rs"
|
||||
version = "0.8.29"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e450f9b2ed1dff33c94c12589a87338689467b9c4f5d8a5710bd09a847d2c8a7"
|
||||
|
||||
[[package]]
|
||||
name = "yasna"
|
||||
version = "0.6.0"
|
||||
|
||||
@@ -399,6 +399,7 @@ impl CameraFrame {
|
||||
pub struct AgentManagerAgentMovementAgentCamera {
|
||||
pub far: f32,
|
||||
frame: Arc<Mutex<CameraFrame>>,
|
||||
vertical_fov_radians: Arc<AtomicU32>,
|
||||
}
|
||||
|
||||
impl fmt::Debug for AgentManagerAgentMovementAgentCamera {
|
||||
@@ -406,6 +407,7 @@ impl fmt::Debug for AgentManagerAgentMovementAgentCamera {
|
||||
formatter
|
||||
.debug_struct("AgentCamera")
|
||||
.field("far", &self.far)
|
||||
.field("vertical_fov_radians", &self.vertical_fov_angle())
|
||||
.field("frame", &*mutex(&self.frame))
|
||||
.finish()
|
||||
}
|
||||
@@ -416,11 +418,20 @@ impl AgentManagerAgentMovementAgentCamera {
|
||||
Ok(Self {
|
||||
far: 128.0,
|
||||
frame: Arc::new(Mutex::new(CameraFrame::default())),
|
||||
vertical_fov_radians: Arc::new(AtomicU32::new(60.0_f32.to_radians().to_bits())),
|
||||
})
|
||||
}
|
||||
|
||||
fn from_frame(frame: Arc<Mutex<CameraFrame>>, far: f32) -> Self {
|
||||
Self { far, frame }
|
||||
fn from_frame(
|
||||
frame: Arc<Mutex<CameraFrame>>,
|
||||
vertical_fov_radians: Arc<AtomicU32>,
|
||||
far: f32,
|
||||
) -> Self {
|
||||
Self {
|
||||
far,
|
||||
frame,
|
||||
vertical_fov_radians,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn look_at_with_vector3_vector3(
|
||||
@@ -584,6 +595,21 @@ impl AgentManagerAgentMovementAgentCamera {
|
||||
mutex(&self.frame).z_axis = value;
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn vertical_fov_angle(&self) -> f32 {
|
||||
f32::from_bits(self.vertical_fov_radians.load(Ordering::Acquire))
|
||||
}
|
||||
|
||||
pub fn set_vertical_fov_angle(&self, value: f32) -> Result<(), Error> {
|
||||
if !value.is_finite() || !(10.0_f32.to_radians()..=140.0_f32.to_radians()).contains(&value)
|
||||
{
|
||||
return Err(Error::Argument);
|
||||
}
|
||||
self.vertical_fov_radians
|
||||
.store(value.to_bits(), Ordering::Release);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq)]
|
||||
@@ -733,6 +759,7 @@ pub(crate) struct AgentMovementRuntime {
|
||||
auto_reset_controls: AtomicBool,
|
||||
update_interval: AtomicI32,
|
||||
camera_frame: Arc<Mutex<CameraFrame>>,
|
||||
vertical_fov_radians: Arc<AtomicU32>,
|
||||
last_update: Mutex<LastUpdate>,
|
||||
kinematics: RwLock<Kinematics>,
|
||||
teleport: Mutex<TeleportState>,
|
||||
@@ -805,6 +832,7 @@ impl AgentMovementRuntime {
|
||||
auto_reset_controls: AtomicBool::new(false),
|
||||
update_interval: AtomicI32::new(interval),
|
||||
camera_frame: Arc::new(Mutex::new(CameraFrame::default())),
|
||||
vertical_fov_radians: Arc::new(AtomicU32::new(60.0_f32.to_radians().to_bits())),
|
||||
last_update: Mutex::new(LastUpdate::default()),
|
||||
kinematics: RwLock::new(Kinematics::default()),
|
||||
teleport: Mutex::new(TeleportState::default()),
|
||||
@@ -986,7 +1014,11 @@ impl AgentMovementRuntime {
|
||||
}
|
||||
|
||||
fn camera(&self, far: f32) -> AgentManagerAgentMovementAgentCamera {
|
||||
AgentManagerAgentMovementAgentCamera::from_frame(Arc::clone(&self.camera_frame), far)
|
||||
AgentManagerAgentMovementAgentCamera::from_frame(
|
||||
Arc::clone(&self.camera_frame),
|
||||
Arc::clone(&self.vertical_fov_radians),
|
||||
far,
|
||||
)
|
||||
}
|
||||
|
||||
fn send_update(
|
||||
@@ -2099,6 +2131,7 @@ impl AgentManagerAgentMovement {
|
||||
}
|
||||
|
||||
pub fn set_fov_vertical_angle(&self, angle: f32) -> Result<(), Error> {
|
||||
self.camera.set_vertical_fov_angle(angle)?;
|
||||
let (agent_id, session_id) = self.runtime.ids();
|
||||
let mut packet = AgentFOVPacket::new_with_constructor()?;
|
||||
packet.agent_data.agent_id = agent_id;
|
||||
@@ -3167,6 +3200,7 @@ mod tests {
|
||||
assert!(run.agent_data.always_run);
|
||||
|
||||
manager.movement.set_fov_vertical_angle(1.25).unwrap();
|
||||
assert_eq!(manager.movement.camera.vertical_fov_angle(), 1.25);
|
||||
let fov = decode_packet::<AgentFOVPacket>(&grid.receive(PacketType::AgentFOV)).unwrap();
|
||||
assert_eq!(fov.fov_block.vertical_angle, 1.25);
|
||||
|
||||
|
||||
@@ -28,6 +28,7 @@ tokio = { version = "1.53.1", features = ["macros", "rt", "sync", "time"] }
|
||||
tokio-rustls = { version = "0.26.4", default-features = false, features = ["aws_lc_rs", "tls12"] }
|
||||
url = "2.5.8"
|
||||
unicode-width = "0.2"
|
||||
wgpu = { version = "30.0.1", default-features = false, features = ["std", "dx12", "metal", "gles", "vulkan", "wgsl"], optional = true }
|
||||
|
||||
[dev-dependencies]
|
||||
jpeg-decoder = { version = "0.3.2", default-features = false }
|
||||
@@ -38,7 +39,7 @@ tokio = { version = "1.53.1", features = ["io-util", "net", "rt-multi-thread", "
|
||||
|
||||
[features]
|
||||
default = []
|
||||
live-grid = ["dep:libremetaverse", "dep:libremetaverse-imaging", "dep:libremetaverse-rendering-simple"]
|
||||
live-grid = ["dep:libremetaverse", "dep:libremetaverse-imaging", "dep:libremetaverse-rendering-simple", "dep:wgpu"]
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
@@ -746,6 +746,82 @@ impl crate::behavior::EmbodimentSink for LibremetaverseEmbodimentSink {
|
||||
.map_err(|_| crate::behavior::BehaviorError::NativeOperation)
|
||||
})
|
||||
}
|
||||
|
||||
fn set_camera(
|
||||
&self,
|
||||
generation: u64,
|
||||
position: crate::perception::WorldPosition,
|
||||
target: crate::perception::WorldPosition,
|
||||
vertical_fov_degrees: f64,
|
||||
cancellation: CancellationToken,
|
||||
) -> crate::behavior::EmbodimentFuture<'_, ()> {
|
||||
Box::pin(async move {
|
||||
if cancellation.is_cancellation_requested() {
|
||||
return Err(crate::behavior::BehaviorError::Cancelled);
|
||||
}
|
||||
self.ensure_generation(generation)?;
|
||||
self.agent
|
||||
.movement
|
||||
.camera
|
||||
.look_at_with_vector3_vector3(
|
||||
libremetaverse_types::Vector3 {
|
||||
x: position.x as f32,
|
||||
y: position.y as f32,
|
||||
z: position.z as f32,
|
||||
},
|
||||
libremetaverse_types::Vector3 {
|
||||
x: target.x as f32,
|
||||
y: target.y as f32,
|
||||
z: target.z as f32,
|
||||
},
|
||||
)
|
||||
.map_err(|_| crate::behavior::BehaviorError::NativeOperation)?;
|
||||
self.agent
|
||||
.movement
|
||||
.set_fov_vertical_angle((vertical_fov_degrees as f32).to_radians())
|
||||
.and_then(|()| self.agent.movement.send_update_with_boolean(Some(true)))
|
||||
.map_err(|_| crate::behavior::BehaviorError::NativeOperation)
|
||||
})
|
||||
}
|
||||
|
||||
fn reset_camera(
|
||||
&self,
|
||||
generation: u64,
|
||||
cancellation: CancellationToken,
|
||||
) -> crate::behavior::EmbodimentFuture<'_, ()> {
|
||||
Box::pin(async move {
|
||||
if cancellation.is_cancellation_requested() {
|
||||
return Err(crate::behavior::BehaviorError::Cancelled);
|
||||
}
|
||||
self.ensure_generation(generation)?;
|
||||
let avatar = self.agent.sim_position();
|
||||
let forward = libremetaverse_types::Vector3::mul_with_vector3_quaternion(
|
||||
libremetaverse_types::Vector3::unit_x(),
|
||||
self.agent.sim_rotation(),
|
||||
);
|
||||
let position = libremetaverse_types::Vector3 {
|
||||
x: avatar.x,
|
||||
y: avatar.y,
|
||||
z: avatar.z + 1.6,
|
||||
};
|
||||
let target = libremetaverse_types::Vector3 {
|
||||
x: position.x + forward.x * 4.0,
|
||||
y: position.y + forward.y * 4.0,
|
||||
z: position.z + forward.z * 4.0,
|
||||
};
|
||||
self.agent
|
||||
.movement
|
||||
.camera
|
||||
.look_at_with_vector3_vector3(position, target)
|
||||
.and_then(|()| {
|
||||
self.agent
|
||||
.movement
|
||||
.set_fov_vertical_angle(60.0_f32.to_radians())
|
||||
})
|
||||
.and_then(|()| self.agent.movement.send_update_with_boolean(Some(true)))
|
||||
.map_err(|_| crate::behavior::BehaviorError::NativeOperation)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "live-grid")]
|
||||
|
||||
@@ -35,6 +35,8 @@ pub const STOP_TOOL: &str = "behavior_stop";
|
||||
pub const SIT_TOOL: &str = "behavior_sit";
|
||||
pub const STAND_TOOL: &str = "behavior_stand";
|
||||
pub const CURRENT_POSE_TOOL: &str = "behavior_current_pose";
|
||||
pub const CAMERA_SET_TOOL: &str = "behavior_camera_set";
|
||||
pub const CAMERA_RESET_TOOL: &str = "behavior_camera_reset";
|
||||
|
||||
const WALK_POLL: Duration = Duration::from_millis(250);
|
||||
const TURN_INTERVAL: Duration = Duration::from_millis(40);
|
||||
@@ -162,6 +164,23 @@ pub trait EmbodimentSink: Send + Sync + 'static {
|
||||
fn stop(&self, generation: u64, cancellation: CancellationToken) -> EmbodimentFuture<'_, ()>;
|
||||
fn sit(&self, generation: u64, cancellation: CancellationToken) -> EmbodimentFuture<'_, ()>;
|
||||
fn stand(&self, generation: u64, cancellation: CancellationToken) -> EmbodimentFuture<'_, ()>;
|
||||
fn set_camera(
|
||||
&self,
|
||||
_generation: u64,
|
||||
_position: WorldPosition,
|
||||
_target: WorldPosition,
|
||||
_vertical_fov_degrees: f64,
|
||||
_cancellation: CancellationToken,
|
||||
) -> EmbodimentFuture<'_, ()> {
|
||||
Box::pin(async { Err(BehaviorError::TargetUnavailable) })
|
||||
}
|
||||
fn reset_camera(
|
||||
&self,
|
||||
_generation: u64,
|
||||
_cancellation: CancellationToken,
|
||||
) -> EmbodimentFuture<'_, ()> {
|
||||
Box::pin(async { Err(BehaviorError::TargetUnavailable) })
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
@@ -229,6 +248,12 @@ enum BehaviorAction {
|
||||
Sit,
|
||||
Stand,
|
||||
CurrentPose,
|
||||
CameraSet {
|
||||
position: WorldPosition,
|
||||
target: WorldPosition,
|
||||
vertical_fov_degrees: f64,
|
||||
},
|
||||
CameraReset,
|
||||
}
|
||||
|
||||
impl BehaviorAction {
|
||||
@@ -242,6 +267,8 @@ impl BehaviorAction {
|
||||
Self::Sit => SIT_TOOL,
|
||||
Self::Stand => STAND_TOOL,
|
||||
Self::CurrentPose => CURRENT_POSE_TOOL,
|
||||
Self::CameraSet { .. } => CAMERA_SET_TOOL,
|
||||
Self::CameraReset => CAMERA_RESET_TOOL,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -943,6 +970,7 @@ async fn wait_for_action_cancellation(
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_lines)]
|
||||
async fn perform_action(
|
||||
controller: &BehaviorController,
|
||||
state: ReadyState,
|
||||
@@ -1035,6 +1063,51 @@ async fn perform_action(
|
||||
.await?;
|
||||
Ok(json!({"status":"completed"}))
|
||||
}
|
||||
BehaviorAction::CameraSet {
|
||||
position,
|
||||
target,
|
||||
vertical_fov_degrees,
|
||||
} => {
|
||||
let pose = controller
|
||||
.sink
|
||||
.current_pose(state.generation, cancellation.clone())
|
||||
.await?;
|
||||
validate_pose(&pose, state)?;
|
||||
validate_point(
|
||||
pose.position,
|
||||
*position,
|
||||
controller.settings.max_attention_distance_meters,
|
||||
)?;
|
||||
validate_point(
|
||||
pose.position,
|
||||
*target,
|
||||
controller.settings.max_attention_distance_meters,
|
||||
)?;
|
||||
if distance(*position, *target) < 0.25 || !(20.0..=120.0).contains(vertical_fov_degrees)
|
||||
{
|
||||
return Err(BehaviorError::InvalidArguments);
|
||||
}
|
||||
controller
|
||||
.sink
|
||||
.set_camera(
|
||||
state.generation,
|
||||
*position,
|
||||
*target,
|
||||
*vertical_fov_degrees,
|
||||
cancellation,
|
||||
)
|
||||
.await?;
|
||||
Ok(
|
||||
json!({"status":"completed","position":position,"target":target,"vertical_fov_degrees":vertical_fov_degrees}),
|
||||
)
|
||||
}
|
||||
BehaviorAction::CameraReset => {
|
||||
controller
|
||||
.sink
|
||||
.reset_camera(state.generation, cancellation)
|
||||
.await?;
|
||||
Ok(json!({"status":"completed"}))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1363,6 +1436,28 @@ fn parse_action(name: &str, json_arguments: &str) -> Result<BehaviorAction, Beha
|
||||
require_empty(args)?;
|
||||
Ok(BehaviorAction::CurrentPose)
|
||||
}
|
||||
CAMERA_SET_TOOL => {
|
||||
if args.len() != 7 {
|
||||
return Err(BehaviorError::InvalidArguments);
|
||||
}
|
||||
Ok(BehaviorAction::CameraSet {
|
||||
position: WorldPosition {
|
||||
x: number(args, "position_x")?,
|
||||
y: number(args, "position_y")?,
|
||||
z: number(args, "position_z")?,
|
||||
},
|
||||
target: WorldPosition {
|
||||
x: number(args, "target_x")?,
|
||||
y: number(args, "target_y")?,
|
||||
z: number(args, "target_z")?,
|
||||
},
|
||||
vertical_fov_degrees: number(args, "vertical_fov_degrees")?,
|
||||
})
|
||||
}
|
||||
CAMERA_RESET_TOOL => {
|
||||
require_empty(args)?;
|
||||
Ok(BehaviorAction::CameraReset)
|
||||
}
|
||||
_ => Err(BehaviorError::InvalidArguments),
|
||||
}
|
||||
}
|
||||
@@ -1402,7 +1497,7 @@ fn require_empty(args: &Map<String, Value>) -> Result<(), BehaviorError> {
|
||||
.ok_or(BehaviorError::InvalidArguments)
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_lines)] // One registry table keeps the eight schemas auditable together.
|
||||
#[allow(clippy::too_many_lines)] // One registry table keeps the ten schemas auditable together.
|
||||
pub fn behavior_policy_tools(settings: &BehaviorSettings) -> Result<Vec<PolicyTool>, PolicyError> {
|
||||
let origins = || {
|
||||
AllowedOrigins::new([
|
||||
@@ -1490,10 +1585,43 @@ pub fn behavior_policy_tools(settings: &BehaviorSettings) -> Result<Vec<PolicyTo
|
||||
(
|
||||
CURRENT_POSE_TOOL,
|
||||
"Read current embodied pose and session provenance.",
|
||||
empty,
|
||||
empty.clone(),
|
||||
false,
|
||||
0,
|
||||
),
|
||||
(
|
||||
CAMERA_SET_TOOL,
|
||||
"Place the camera near the avatar and look at a point. Use this to focus or orbit objects; coordinates are current-region meters.",
|
||||
object(
|
||||
&[
|
||||
("position_x", ToolSchema::Number),
|
||||
("position_y", ToolSchema::Number),
|
||||
("position_z", ToolSchema::Number),
|
||||
("target_x", ToolSchema::Number),
|
||||
("target_y", ToolSchema::Number),
|
||||
("target_z", ToolSchema::Number),
|
||||
("vertical_fov_degrees", ToolSchema::Number),
|
||||
],
|
||||
&[
|
||||
"position_x",
|
||||
"position_y",
|
||||
"position_z",
|
||||
"target_x",
|
||||
"target_y",
|
||||
"target_z",
|
||||
"vertical_fov_degrees",
|
||||
],
|
||||
),
|
||||
true,
|
||||
0,
|
||||
),
|
||||
(
|
||||
CAMERA_RESET_TOOL,
|
||||
"Reset the camera to the avatar position and facing direction.",
|
||||
empty,
|
||||
true,
|
||||
0,
|
||||
),
|
||||
];
|
||||
specs
|
||||
.into_iter()
|
||||
|
||||
@@ -198,6 +198,33 @@ impl EmbodimentSink for FakeSink {
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
fn set_camera(
|
||||
&self,
|
||||
_: u64,
|
||||
_: WorldPosition,
|
||||
_: WorldPosition,
|
||||
_: f64,
|
||||
_: CancellationToken,
|
||||
) -> EmbodimentFuture<'_, ()> {
|
||||
Box::pin(async move {
|
||||
self.state
|
||||
.lock()
|
||||
.expect("state")
|
||||
.calls
|
||||
.push("camera_set".into());
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
fn reset_camera(&self, _: u64, _: CancellationToken) -> EmbodimentFuture<'_, ()> {
|
||||
Box::pin(async move {
|
||||
self.state
|
||||
.lock()
|
||||
.expect("state")
|
||||
.calls
|
||||
.push("camera_reset".into());
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
struct FixedRandom(u64);
|
||||
@@ -257,7 +284,7 @@ async fn run_tool(
|
||||
#[test]
|
||||
fn policy_exposes_only_bounded_high_level_actions() {
|
||||
let tools = behavior_policy_tools(&settings()).expect("tools");
|
||||
assert_eq!(tools.len(), 8);
|
||||
assert_eq!(tools.len(), 10);
|
||||
let names = tools
|
||||
.iter()
|
||||
.map(|tool| tool.definition.name.as_str())
|
||||
@@ -272,7 +299,9 @@ fn policy_exposes_only_bounded_high_level_actions() {
|
||||
STOP_TOOL,
|
||||
SIT_TOOL,
|
||||
STAND_TOOL,
|
||||
CURRENT_POSE_TOOL
|
||||
CURRENT_POSE_TOOL,
|
||||
CAMERA_SET_TOOL,
|
||||
CAMERA_RESET_TOOL,
|
||||
]
|
||||
);
|
||||
assert!(names.iter().all(|name| !name.contains("teleport")
|
||||
@@ -280,6 +309,50 @@ fn policy_exposes_only_bounded_high_level_actions() {
|
||||
&& !name.contains("follow")));
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn camera_tools_validate_and_route_bounded_view_changes() {
|
||||
let region = uuid(10);
|
||||
let sink = Arc::new(FakeSink::new(1, region));
|
||||
let mut config = settings();
|
||||
config.settle_delay = Duration::ZERO;
|
||||
let handle =
|
||||
BehaviorController::new(config.clone(), sink.clone(), 16, 32, Duration::from_secs(1))
|
||||
.expect("controller")
|
||||
.start();
|
||||
let ingress = handle.ingress();
|
||||
ingress.connected(1, region).expect("ready");
|
||||
tokio::task::yield_now().await;
|
||||
let backend = BehaviorBackend::new(ingress);
|
||||
let set = run_tool(
|
||||
&backend,
|
||||
&config,
|
||||
CAMERA_SET_TOOL,
|
||||
json!({
|
||||
"position_x":126.0,"position_y":128.0,"position_z":26.0,
|
||||
"target_x":132.0,"target_y":128.0,"target_z":24.0,
|
||||
"vertical_fov_degrees":55.0
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
assert!(matches!(set, ToolCallOutcome::Completed { .. }));
|
||||
let reset = run_tool(&backend, &config, CAMERA_RESET_TOOL, json!({})).await;
|
||||
assert!(matches!(reset, ToolCallOutcome::Completed { .. }));
|
||||
assert_eq!(sink.calls(), vec!["camera_set", "camera_reset"]);
|
||||
let invalid = run_tool(
|
||||
&backend,
|
||||
&config,
|
||||
CAMERA_SET_TOOL,
|
||||
json!({
|
||||
"position_x":126.0,"position_y":128.0,"position_z":26.0,
|
||||
"target_x":126.0,"target_y":128.0,"target_z":26.0,
|
||||
"vertical_fov_degrees":55.0
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
assert!(matches!(invalid, ToolCallOutcome::Rejected { .. }));
|
||||
handle.shutdown().await.expect("shutdown");
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn public_attention_waits_turns_and_returns_to_available() {
|
||||
let region = uuid(10);
|
||||
|
||||
@@ -153,7 +153,7 @@ impl AgentControlTarget {
|
||||
*lock(&self.build) = Some(build);
|
||||
}
|
||||
|
||||
/// Attaches synthetic viewport cancellation and metadata to operator control.
|
||||
/// Attaches reconstructed viewport cancellation and metadata to operator control.
|
||||
pub fn attach_vision_control(&self, vision: Arc<dyn crate::vision::VisionControl>) {
|
||||
*lock(&self.vision) = Some(vision);
|
||||
}
|
||||
|
||||
@@ -30,7 +30,7 @@ const MAX_DUPLICATE_IDS: usize = 16_384;
|
||||
const MAX_DEBOUNCE_FRAGMENTS: usize = 16;
|
||||
const MAX_OBSERVATIONS: usize = 8_192;
|
||||
const MENTRA_RUNTIME_IDENTIFIER: &str = "metacrate-grid-agent";
|
||||
const MENTRA_AGENT_PREFIX: &str = "grid-conversation-v5-";
|
||||
const MENTRA_AGENT_PREFIX: &str = "grid-conversation-v6-";
|
||||
const MENTRA_MEMORY_TOOLS: [&str; 3] = ["memory_search", "memory_pin", "memory_forget"];
|
||||
pub(crate) const CURRENT_TURN_SYSTEM_PROMPT: &str =
|
||||
"You are in an OpenSim virtual world. Use the available tools to act there.";
|
||||
@@ -1434,7 +1434,7 @@ async fn complete_work(
|
||||
.await;
|
||||
}
|
||||
|
||||
fn classify_intent(
|
||||
pub(crate) fn classify_intent(
|
||||
channel: InteractionChannel,
|
||||
authorized: bool,
|
||||
message: &str,
|
||||
@@ -1487,7 +1487,18 @@ fn command_capabilities(message: &str) -> BTreeSet<crate::policy::Capability> {
|
||||
if tokens.iter().any(|token| {
|
||||
matches!(
|
||||
*token,
|
||||
"teleport" | "move" | "walk" | "stop" | "sit" | "stand" | "face" | "look" | "schedule"
|
||||
"teleport"
|
||||
| "move"
|
||||
| "walk"
|
||||
| "stop"
|
||||
| "sit"
|
||||
| "stand"
|
||||
| "face"
|
||||
| "look"
|
||||
| "camera"
|
||||
| "focus"
|
||||
| "orbit"
|
||||
| "schedule"
|
||||
)
|
||||
}) {
|
||||
capabilities.insert(Capability::Movement);
|
||||
|
||||
@@ -483,6 +483,26 @@ fn explicit_command_only_exposes_its_capability_family() {
|
||||
crate::Capability::Movement,
|
||||
])
|
||||
);
|
||||
|
||||
let camera = capabilities_for_intent(
|
||||
InteractionIntent::PolicyGatedCommand,
|
||||
"Use behavior_camera_set to position your camera and focus on the object",
|
||||
);
|
||||
assert_eq!(
|
||||
camera,
|
||||
BTreeSet::from([
|
||||
crate::Capability::Informational,
|
||||
crate::Capability::Movement,
|
||||
])
|
||||
);
|
||||
assert_eq!(
|
||||
classify_intent(
|
||||
InteractionChannel::DirectIm,
|
||||
true,
|
||||
"Use behavior_camera_set to position your camera and focus on the object",
|
||||
),
|
||||
InteractionIntent::PolicyGatedCommand
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
|
||||
@@ -72,9 +72,9 @@ pub use backend::{
|
||||
pub use behavior::{
|
||||
AuthorizedBackendRouter, BehaviorBackend, BehaviorController, BehaviorError, BehaviorHandle,
|
||||
BehaviorIngress, BehaviorMode, BehaviorObservation, BehaviorOutcome, BehaviorPolicyResult,
|
||||
BehaviorRandom, BehaviorTrigger, CURRENT_POSE_TOOL, EmbodiedPose, EmbodimentFuture,
|
||||
EmbodimentSink, FACE_AVATAR_TOOL, FACE_POINT_TOOL, LOOK_AROUND_TOOL, SIT_TOOL, STAND_TOOL,
|
||||
STOP_TOOL, WALK_SHORT_TOOL, behavior_policy_tools,
|
||||
BehaviorRandom, BehaviorTrigger, CAMERA_RESET_TOOL, CAMERA_SET_TOOL, CURRENT_POSE_TOOL,
|
||||
EmbodiedPose, EmbodimentFuture, EmbodimentSink, FACE_AVATAR_TOOL, FACE_POINT_TOOL,
|
||||
LOOK_AROUND_TOOL, SIT_TOOL, STAND_TOOL, STOP_TOOL, WALK_SHORT_TOOL, behavior_policy_tools,
|
||||
};
|
||||
#[cfg(feature = "live-grid")]
|
||||
pub use build::LibremetaverseBuildGrid;
|
||||
|
||||
@@ -740,10 +740,13 @@ fn start_live_interactions(
|
||||
config.storage_path.join("mentra"),
|
||||
)?);
|
||||
let vision_limits = VisionLimits::default();
|
||||
let vision = Arc::new(VisionService::new(
|
||||
Arc::new(LibremetaverseSceneSource::new(owner, vision_limits)),
|
||||
vision_limits,
|
||||
)?);
|
||||
let vision = Arc::new(
|
||||
VisionService::new(
|
||||
Arc::new(LibremetaverseSceneSource::new(owner, vision_limits)),
|
||||
vision_limits,
|
||||
)?
|
||||
.prefer_gpu(),
|
||||
);
|
||||
let responder = Arc::new(VisionAugmentedResponder::new(vision.clone(), responder));
|
||||
let sink = Arc::new(owner.interaction_sink());
|
||||
let pacer: Arc<dyn metacrate_grid_agent::ResponsePacer> = Arc::new(behavior_ingress);
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
//! Bounded deterministic synthetic viewport rendering and capture ownership.
|
||||
//! Bounded reconstructed viewport rendering and capture ownership.
|
||||
|
||||
#![allow(clippy::missing_errors_doc)]
|
||||
#![allow(
|
||||
@@ -218,6 +218,10 @@ pub struct VisionService<S: SceneSource> {
|
||||
request_sequence: AtomicU64,
|
||||
current_generation: AtomicU64,
|
||||
state: Mutex<VisionState>,
|
||||
#[cfg(feature = "live-grid")]
|
||||
gpu_preferred: bool,
|
||||
#[cfg(feature = "live-grid")]
|
||||
gpu: tokio::sync::OnceCell<Option<Arc<WgpuRenderer>>>,
|
||||
}
|
||||
|
||||
impl<S: SceneSource> VisionService<S> {
|
||||
@@ -236,8 +240,20 @@ impl<S: SceneSource> VisionService<S> {
|
||||
active: None,
|
||||
observations: Vec::new(),
|
||||
}),
|
||||
#[cfg(feature = "live-grid")]
|
||||
gpu_preferred: false,
|
||||
#[cfg(feature = "live-grid")]
|
||||
gpu: tokio::sync::OnceCell::new(),
|
||||
})
|
||||
}
|
||||
/// Prefer cached offscreen `wgpu` rendering, falling back to the
|
||||
/// deterministic software rasterizer when no adapter is available.
|
||||
#[cfg(feature = "live-grid")]
|
||||
#[must_use]
|
||||
pub fn prefer_gpu(mut self) -> Self {
|
||||
self.gpu_preferred = true;
|
||||
self
|
||||
}
|
||||
pub fn set_generation(&self, generation: u64) {
|
||||
if self.current_generation.swap(generation, Ordering::AcqRel) != generation {
|
||||
self.cancel_active();
|
||||
@@ -308,7 +324,7 @@ impl<S: SceneSource> VisionService<S> {
|
||||
generation,
|
||||
VisionObservationKind::Started,
|
||||
None,
|
||||
"synthetic viewport capture started",
|
||||
"viewport capture started",
|
||||
);
|
||||
let token = owned.token();
|
||||
let result = tokio::select! {()=cancellation.cancelled()=>Err(VisionError::Cancelled),()=token.cancelled()=>Err(VisionError::Superseded),result=tokio::time::timeout(self.limits.capture_timeout,self.source.capture_scene(generation,token.clone()))=>result.map_err(|_|VisionError::TimedOut)?};
|
||||
@@ -334,7 +350,7 @@ impl<S: SceneSource> VisionService<S> {
|
||||
} else if generation != self.current_generation.load(Ordering::Acquire) {
|
||||
Err(VisionError::StaleGeneration)
|
||||
} else {
|
||||
render_scene(&scene, self.limits).inspect(|capture| {
|
||||
self.render_scene(scene).await.inspect(|capture| {
|
||||
self.observe(
|
||||
correlation,
|
||||
generation,
|
||||
@@ -373,6 +389,31 @@ impl<S: SceneSource> VisionService<S> {
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
async fn render_scene(&self, scene: SceneSnapshot) -> Result<VisionCapture, VisionError> {
|
||||
#[cfg(feature = "live-grid")]
|
||||
if self.gpu_preferred {
|
||||
let renderer = self
|
||||
.gpu
|
||||
.get_or_init(|| async { WgpuRenderer::new().await.ok().map(Arc::new) })
|
||||
.await
|
||||
.clone();
|
||||
if let Some(renderer) = renderer {
|
||||
let gpu_scene = scene.clone();
|
||||
let limits = self.limits;
|
||||
if let Ok(Ok(capture)) = tokio::task::spawn_blocking(move || {
|
||||
renderer
|
||||
.render(&gpu_scene, limits)
|
||||
.and_then(|rgba| finish_capture(&gpu_scene, limits, &rgba))
|
||||
})
|
||||
.await
|
||||
{
|
||||
return Ok(capture);
|
||||
}
|
||||
}
|
||||
}
|
||||
render_scene_software(&scene, self.limits)
|
||||
}
|
||||
}
|
||||
|
||||
/// Type-erased runtime control surface. It deliberately exposes metadata and
|
||||
@@ -428,7 +469,7 @@ impl<S: SceneSource> VisionControl for VisionService<S> {
|
||||
}
|
||||
}
|
||||
|
||||
/// Provider-neutral responder decorator that adds a synthetic viewport only
|
||||
/// Provider-neutral responder decorator that adds a reconstructed viewport only
|
||||
/// for an explicit visual question. An endpoint capability rejection produces
|
||||
/// a textual scene-summary response without retrying the large image request.
|
||||
pub struct VisionAugmentedResponder<S: SceneSource, R: crate::interaction::InteractionResponder> {
|
||||
@@ -549,7 +590,10 @@ pub fn visual_question(text: &str) -> bool {
|
||||
.any(|needle| lower.contains(needle))
|
||||
}
|
||||
|
||||
fn render_scene(scene: &SceneSnapshot, limits: VisionLimits) -> Result<VisionCapture, VisionError> {
|
||||
fn render_scene_software(
|
||||
scene: &SceneSnapshot,
|
||||
limits: VisionLimits,
|
||||
) -> Result<VisionCapture, VisionError> {
|
||||
validate_scene(scene, limits)?;
|
||||
let pixel_count = limits.width as usize * limits.height as usize;
|
||||
let mut rgba = vec![0u8; pixel_count * 4];
|
||||
@@ -581,7 +625,15 @@ fn render_scene(scene: &SceneSnapshot, limits: VisionLimits) -> Result<VisionCap
|
||||
triangle,
|
||||
);
|
||||
}
|
||||
let jpeg = encode_jpeg(limits.width, limits.height, &rgba, limits.max_jpeg_bytes)?;
|
||||
finish_capture(scene, limits, &rgba)
|
||||
}
|
||||
|
||||
fn finish_capture(
|
||||
scene: &SceneSnapshot,
|
||||
limits: VisionLimits,
|
||||
rgba: &[u8],
|
||||
) -> Result<VisionCapture, VisionError> {
|
||||
let jpeg = encode_jpeg(limits.width, limits.height, rgba, limits.max_jpeg_bytes)?;
|
||||
let hash = Sha256::digest(&jpeg)
|
||||
.iter()
|
||||
.fold(String::with_capacity(64), |mut output, byte| {
|
||||
@@ -764,6 +816,316 @@ fn environment_color(scene: &SceneSnapshot) -> [u8; 4] {
|
||||
255,
|
||||
]
|
||||
}
|
||||
|
||||
#[cfg(feature = "live-grid")]
|
||||
pub(crate) struct WgpuRenderer {
|
||||
device: wgpu::Device,
|
||||
queue: wgpu::Queue,
|
||||
pipeline: wgpu::RenderPipeline,
|
||||
}
|
||||
|
||||
#[cfg(feature = "live-grid")]
|
||||
const WGPU_VERTEX_ATTRIBUTES: [wgpu::VertexAttribute; 2] =
|
||||
wgpu::vertex_attr_array![0 => Float32x3, 1 => Float32x4];
|
||||
|
||||
#[cfg(feature = "live-grid")]
|
||||
impl WgpuRenderer {
|
||||
pub(crate) async fn new() -> Result<Self, VisionError> {
|
||||
let instance = wgpu::Instance::new(wgpu::InstanceDescriptor::new_without_display_handle());
|
||||
let adapter = instance
|
||||
.request_adapter(&wgpu::RequestAdapterOptions {
|
||||
power_preference: wgpu::PowerPreference::LowPower,
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.map_err(|_| VisionError::Encode)?;
|
||||
let (device, queue) = adapter
|
||||
.request_device(&wgpu::DeviceDescriptor {
|
||||
label: Some("metacrate-agent-vision"),
|
||||
required_features: wgpu::Features::empty(),
|
||||
required_limits: wgpu::Limits::downlevel_defaults(),
|
||||
memory_hints: wgpu::MemoryHints::MemoryUsage,
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.map_err(|_| VisionError::Encode)?;
|
||||
let shader = device.create_shader_module(wgpu::ShaderModuleDescriptor {
|
||||
label: Some("metacrate-agent-vision-shader"),
|
||||
source: wgpu::ShaderSource::Wgsl(
|
||||
r"
|
||||
struct VertexInput {
|
||||
@location(0) position: vec3<f32>,
|
||||
@location(1) color: vec4<f32>,
|
||||
};
|
||||
struct VertexOutput {
|
||||
@builtin(position) position: vec4<f32>,
|
||||
@location(0) color: vec4<f32>,
|
||||
};
|
||||
@vertex
|
||||
fn vs_main(input: VertexInput) -> VertexOutput {
|
||||
var output: VertexOutput;
|
||||
output.position = vec4<f32>(input.position, 1.0);
|
||||
output.color = input.color;
|
||||
return output;
|
||||
}
|
||||
@fragment
|
||||
fn fs_main(input: VertexOutput) -> @location(0) vec4<f32> {
|
||||
return input.color;
|
||||
}
|
||||
"
|
||||
.into(),
|
||||
),
|
||||
});
|
||||
let layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor {
|
||||
label: Some("metacrate-agent-vision-layout"),
|
||||
bind_group_layouts: &[],
|
||||
immediate_size: 0,
|
||||
});
|
||||
let pipeline = device.create_render_pipeline(&wgpu::RenderPipelineDescriptor {
|
||||
label: Some("metacrate-agent-vision-pipeline"),
|
||||
layout: Some(&layout),
|
||||
vertex: wgpu::VertexState {
|
||||
module: &shader,
|
||||
entry_point: Some("vs_main"),
|
||||
compilation_options: wgpu::PipelineCompilationOptions::default(),
|
||||
buffers: &[Some(wgpu::VertexBufferLayout {
|
||||
array_stride: 7 * size_of::<f32>() as u64,
|
||||
step_mode: wgpu::VertexStepMode::Vertex,
|
||||
attributes: &WGPU_VERTEX_ATTRIBUTES,
|
||||
})],
|
||||
},
|
||||
primitive: wgpu::PrimitiveState {
|
||||
topology: wgpu::PrimitiveTopology::TriangleList,
|
||||
cull_mode: None,
|
||||
..Default::default()
|
||||
},
|
||||
depth_stencil: Some(wgpu::DepthStencilState {
|
||||
format: wgpu::TextureFormat::Depth32Float,
|
||||
depth_write_enabled: Some(true),
|
||||
depth_compare: Some(wgpu::CompareFunction::Less),
|
||||
stencil: wgpu::StencilState::default(),
|
||||
bias: wgpu::DepthBiasState::default(),
|
||||
}),
|
||||
multisample: wgpu::MultisampleState::default(),
|
||||
fragment: Some(wgpu::FragmentState {
|
||||
module: &shader,
|
||||
entry_point: Some("fs_main"),
|
||||
compilation_options: wgpu::PipelineCompilationOptions::default(),
|
||||
targets: &[Some(wgpu::ColorTargetState {
|
||||
format: wgpu::TextureFormat::Rgba8UnormSrgb,
|
||||
blend: None,
|
||||
write_mask: wgpu::ColorWrites::ALL,
|
||||
})],
|
||||
}),
|
||||
multiview_mask: None,
|
||||
cache: None,
|
||||
});
|
||||
Ok(Self {
|
||||
device,
|
||||
queue,
|
||||
pipeline,
|
||||
})
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_lines)]
|
||||
pub(crate) fn render(
|
||||
&self,
|
||||
scene: &SceneSnapshot,
|
||||
limits: VisionLimits,
|
||||
) -> Result<Vec<u8>, VisionError> {
|
||||
validate_scene(scene, limits)?;
|
||||
let (vertices, vertex_count) = gpu_vertices(scene, limits)?;
|
||||
let vertex_buffer = self.device.create_buffer(&wgpu::BufferDescriptor {
|
||||
label: Some("metacrate-agent-vision-vertices"),
|
||||
size: vertices.len().max(4) as u64,
|
||||
usage: wgpu::BufferUsages::VERTEX | wgpu::BufferUsages::COPY_DST,
|
||||
mapped_at_creation: false,
|
||||
});
|
||||
if !vertices.is_empty() {
|
||||
self.queue.write_buffer(&vertex_buffer, 0, &vertices);
|
||||
}
|
||||
let extent = wgpu::Extent3d {
|
||||
width: limits.width,
|
||||
height: limits.height,
|
||||
depth_or_array_layers: 1,
|
||||
};
|
||||
let color = self.device.create_texture(&wgpu::TextureDescriptor {
|
||||
label: Some("metacrate-agent-vision-color"),
|
||||
size: extent,
|
||||
mip_level_count: 1,
|
||||
sample_count: 1,
|
||||
dimension: wgpu::TextureDimension::D2,
|
||||
format: wgpu::TextureFormat::Rgba8UnormSrgb,
|
||||
usage: wgpu::TextureUsages::RENDER_ATTACHMENT | wgpu::TextureUsages::COPY_SRC,
|
||||
view_formats: &[],
|
||||
});
|
||||
let depth = self.device.create_texture(&wgpu::TextureDescriptor {
|
||||
label: Some("metacrate-agent-vision-depth"),
|
||||
size: extent,
|
||||
mip_level_count: 1,
|
||||
sample_count: 1,
|
||||
dimension: wgpu::TextureDimension::D2,
|
||||
format: wgpu::TextureFormat::Depth32Float,
|
||||
usage: wgpu::TextureUsages::RENDER_ATTACHMENT,
|
||||
view_formats: &[],
|
||||
});
|
||||
let row_bytes = limits
|
||||
.width
|
||||
.checked_mul(4)
|
||||
.ok_or(VisionError::ResourceLimit)?;
|
||||
let padded_row_bytes = row_bytes.div_ceil(wgpu::COPY_BYTES_PER_ROW_ALIGNMENT)
|
||||
* wgpu::COPY_BYTES_PER_ROW_ALIGNMENT;
|
||||
let readback_size = u64::from(padded_row_bytes)
|
||||
.checked_mul(u64::from(limits.height))
|
||||
.ok_or(VisionError::ResourceLimit)?;
|
||||
let readback = self.device.create_buffer(&wgpu::BufferDescriptor {
|
||||
label: Some("metacrate-agent-vision-readback"),
|
||||
size: readback_size,
|
||||
usage: wgpu::BufferUsages::COPY_DST | wgpu::BufferUsages::MAP_READ,
|
||||
mapped_at_creation: false,
|
||||
});
|
||||
let color_view = color.create_view(&wgpu::TextureViewDescriptor::default());
|
||||
let depth_view = depth.create_view(&wgpu::TextureViewDescriptor::default());
|
||||
let sky = environment_color(scene);
|
||||
let mut encoder = self
|
||||
.device
|
||||
.create_command_encoder(&wgpu::CommandEncoderDescriptor {
|
||||
label: Some("metacrate-agent-vision-commands"),
|
||||
});
|
||||
{
|
||||
let mut pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor {
|
||||
label: Some("metacrate-agent-vision-pass"),
|
||||
color_attachments: &[Some(wgpu::RenderPassColorAttachment {
|
||||
view: &color_view,
|
||||
depth_slice: None,
|
||||
resolve_target: None,
|
||||
ops: wgpu::Operations {
|
||||
load: wgpu::LoadOp::Clear(wgpu::Color {
|
||||
r: f64::from(sky[0]) / 255.0,
|
||||
g: f64::from(sky[1]) / 255.0,
|
||||
b: f64::from(sky[2]) / 255.0,
|
||||
a: 1.0,
|
||||
}),
|
||||
store: wgpu::StoreOp::Store,
|
||||
},
|
||||
})],
|
||||
depth_stencil_attachment: Some(wgpu::RenderPassDepthStencilAttachment {
|
||||
view: &depth_view,
|
||||
depth_ops: Some(wgpu::Operations {
|
||||
load: wgpu::LoadOp::Clear(1.0),
|
||||
store: wgpu::StoreOp::Discard,
|
||||
}),
|
||||
stencil_ops: None,
|
||||
}),
|
||||
..Default::default()
|
||||
});
|
||||
if vertex_count != 0 {
|
||||
pass.set_pipeline(&self.pipeline);
|
||||
pass.set_vertex_buffer(0, vertex_buffer.slice(..));
|
||||
pass.draw(0..vertex_count, 0..1);
|
||||
}
|
||||
}
|
||||
encoder.copy_texture_to_buffer(
|
||||
wgpu::TexelCopyTextureInfo {
|
||||
texture: &color,
|
||||
mip_level: 0,
|
||||
origin: wgpu::Origin3d::ZERO,
|
||||
aspect: wgpu::TextureAspect::All,
|
||||
},
|
||||
wgpu::TexelCopyBufferInfo {
|
||||
buffer: &readback,
|
||||
layout: wgpu::TexelCopyBufferLayout {
|
||||
offset: 0,
|
||||
bytes_per_row: Some(padded_row_bytes),
|
||||
rows_per_image: Some(limits.height),
|
||||
},
|
||||
},
|
||||
extent,
|
||||
);
|
||||
let submission = self.queue.submit([encoder.finish()]);
|
||||
let slice = readback.slice(..);
|
||||
let (sender, receiver) = std::sync::mpsc::sync_channel(1);
|
||||
slice.map_async(wgpu::MapMode::Read, move |result| {
|
||||
let _ = sender.send(result);
|
||||
});
|
||||
self.device
|
||||
.poll(wgpu::PollType::Wait {
|
||||
submission_index: Some(submission),
|
||||
timeout: Some(Duration::from_secs(5)),
|
||||
})
|
||||
.map_err(|_| VisionError::TimedOut)?;
|
||||
receiver
|
||||
.recv_timeout(Duration::from_secs(1))
|
||||
.map_err(|_| VisionError::TimedOut)?
|
||||
.map_err(|_| VisionError::Encode)?;
|
||||
let mapped = slice.get_mapped_range().map_err(|_| VisionError::Encode)?;
|
||||
let row_bytes = row_bytes as usize;
|
||||
let padded_row_bytes = padded_row_bytes as usize;
|
||||
let mut rgba = Vec::with_capacity(row_bytes * limits.height as usize);
|
||||
for row in mapped
|
||||
.chunks_exact(padded_row_bytes)
|
||||
.take(limits.height as usize)
|
||||
{
|
||||
rgba.extend_from_slice(&row[..row_bytes]);
|
||||
}
|
||||
drop(mapped);
|
||||
readback.unmap();
|
||||
Ok(rgba)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "live-grid")]
|
||||
fn gpu_vertices(
|
||||
scene: &SceneSnapshot,
|
||||
limits: VisionLimits,
|
||||
) -> Result<(Vec<u8>, u32), VisionError> {
|
||||
let basis = camera_basis(scene.camera)?;
|
||||
let aspect = limits.width as f32 / limits.height as f32;
|
||||
let focal = 1.0 / (scene.camera.vertical_fov_degrees.to_radians() / 2.0).tan();
|
||||
let mut triangles = scene
|
||||
.entities
|
||||
.iter()
|
||||
.flat_map(|entity| {
|
||||
entity
|
||||
.triangles
|
||||
.iter()
|
||||
.map(move |triangle| (entity, triangle))
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
triangles.sort_by_key(|(entity, _)| entity.id.to_string());
|
||||
let mut bytes = Vec::with_capacity(triangles.len() * 3 * 7 * size_of::<f32>());
|
||||
let mut count = 0u32;
|
||||
for (_, triangle) in triangles {
|
||||
let projected = triangle.vertices.map(|world| {
|
||||
let delta = [
|
||||
world[0] - scene.camera.position[0],
|
||||
world[1] - scene.camera.position[1],
|
||||
world[2] - scene.camera.position[2],
|
||||
];
|
||||
let z = dot(delta, basis.2);
|
||||
[
|
||||
dot(delta, basis.0) * focal / aspect / z,
|
||||
dot(delta, basis.1) * focal / z,
|
||||
(z / 512.0).clamp(0.0, 1.0),
|
||||
z,
|
||||
]
|
||||
});
|
||||
if projected.iter().any(|vertex| vertex[3] <= 0.05) {
|
||||
continue;
|
||||
}
|
||||
let color = triangle
|
||||
.color_srgb
|
||||
.map(|channel| f32::from(channel) / 255.0);
|
||||
for vertex in projected {
|
||||
for value in vertex[..3].iter().chain(color.iter()) {
|
||||
bytes.extend_from_slice(&value.to_ne_bytes());
|
||||
}
|
||||
count = count.checked_add(1).ok_or(VisionError::ResourceLimit)?;
|
||||
}
|
||||
}
|
||||
Ok((bytes, count))
|
||||
}
|
||||
|
||||
fn encode_jpeg(
|
||||
width: u32,
|
||||
height: u32,
|
||||
@@ -800,7 +1162,7 @@ fn scene_summary(scene: &SceneSnapshot) -> String {
|
||||
.filter(|e| e.kind == SceneEntityKind::Resident)
|
||||
.count();
|
||||
format!(
|
||||
"Synthetic viewport of region {:?}: {objects} tracked objects and {residents} privacy-marked residents. generation={} observed_unix_millis={}. completeness: objects_truncated={}, avatars_truncated={}, textures_missing={}, terrain_available={}. This is reconstructed from available scene data, not a viewer framebuffer.",
|
||||
"Reconstructed viewport of region {:?}: {objects} tracked objects and {residents} privacy-marked residents. generation={} observed_unix_millis={}. completeness: objects_truncated={}, avatars_truncated={}, textures_missing={}, terrain_available={}. This is reconstructed from available scene data, not a viewer framebuffer.",
|
||||
sanitize(&scene.region_name),
|
||||
scene.generation,
|
||||
scene.observed_unix_millis,
|
||||
@@ -979,7 +1341,7 @@ impl SceneSource for LibremetaverseSceneSource {
|
||||
position: [position.x, position.y, position.z],
|
||||
forward: [forward.x, forward.y, forward.z],
|
||||
up: [up.x, up.y, up.z],
|
||||
vertical_fov_degrees: 60.0,
|
||||
vertical_fov_degrees: camera.vertical_fov_angle().to_degrees(),
|
||||
},
|
||||
entities,
|
||||
completeness: SnapshotCompleteness {
|
||||
|
||||
@@ -120,6 +120,27 @@ async fn golden_scene_is_deterministic_depth_ordered_and_privacy_marked() {
|
||||
assert!(pixels[center + 1] > 180 && pixels[center] < 80 && pixels[center + 2] < 80);
|
||||
}
|
||||
|
||||
#[cfg(feature = "live-grid")]
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn offscreen_wgpu_renders_depth_ordered_scene_when_an_adapter_is_available() {
|
||||
let Ok(renderer) = WgpuRenderer::new().await else {
|
||||
return;
|
||||
};
|
||||
let limits = VisionLimits {
|
||||
width: 64,
|
||||
height: 64,
|
||||
minimum_interval: std::time::Duration::ZERO,
|
||||
..VisionLimits::default()
|
||||
};
|
||||
let rgba = tokio::task::spawn_blocking(move || renderer.render(&scene(), limits))
|
||||
.await
|
||||
.expect("render worker")
|
||||
.expect("offscreen render");
|
||||
assert_eq!(rgba.len(), 64 * 64 * 4);
|
||||
let center = (32 * 64 + 32) * 4;
|
||||
assert!(rgba[center + 1] > 180 && rgba[center] < 80 && rgba[center + 2] < 80);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn invalid_camera_huge_scene_stale_generation_and_size_limit_fail_closed() {
|
||||
let mut bad = scene();
|
||||
@@ -332,7 +353,7 @@ async fn responder_adds_generic_image_payload_and_returns_summary_on_capability_
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(response.as_str().contains("does not support visual input"));
|
||||
assert!(response.as_str().contains("Synthetic viewport"));
|
||||
assert!(response.as_str().contains("Reconstructed viewport"));
|
||||
assert_eq!(*endpoint.parts.lock().unwrap(), 3);
|
||||
}
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@ use std::collections::BTreeSet;
|
||||
use std::fs;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
const ALLOWED_DEPENDENCIES: [&str; 19] = [
|
||||
const ALLOWED_DEPENDENCIES: [&str; 20] = [
|
||||
"async-trait",
|
||||
"base64",
|
||||
"crossterm",
|
||||
@@ -22,6 +22,7 @@ const ALLOWED_DEPENDENCIES: [&str; 19] = [
|
||||
"tokio-rustls",
|
||||
"url",
|
||||
"unicode-width",
|
||||
"wgpu",
|
||||
];
|
||||
|
||||
#[test]
|
||||
@@ -89,7 +90,7 @@ fn runtime_source_has_no_subprocess_or_native_abi_escape_hatch() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn vision_path_selects_only_the_pure_rust_renderer_and_codec() {
|
||||
fn vision_path_selects_only_reviewed_rust_renderers_and_codec() {
|
||||
let root = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
|
||||
let manifest = fs::read_to_string(root.join("Cargo.toml")).expect("agent manifest");
|
||||
assert!(manifest.contains("default-features = false, features = [\"rust-j2k\"]"));
|
||||
@@ -100,13 +101,7 @@ fn vision_path_selects_only_the_pure_rust_renderer_and_codec() {
|
||||
.expect("simple renderer manifest");
|
||||
assert!(!renderer.contains("imaging-skia"));
|
||||
assert!(!renderer.contains("openjpeg"));
|
||||
let vision = fs::read_to_string(root.join("src/vision.rs")).expect("vision source");
|
||||
for forbidden in ["screenshot crate", "gpu adapter", "std::process"] {
|
||||
assert!(
|
||||
!vision.to_ascii_lowercase().contains(forbidden),
|
||||
"vision must remain synthetic and pure Rust: {forbidden}"
|
||||
);
|
||||
}
|
||||
assert!(manifest.contains("wgpu = { version = \"30.0.1\""));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -113,8 +113,9 @@ hash. Do not record vendor presets or identifiers. Exercise, in order:
|
||||
every created prim through the ownership-checked cleanup path.
|
||||
6. Create and catalog a current-location landmark, accept a controlled landmark
|
||||
offer, use a short bounded folder schedule, teleport, then disable the
|
||||
schedule. Capture the synthetic scene and ask one visual question. Record the endpoint
|
||||
capability fallback if image input is rejected.
|
||||
schedule. Set and reset a bounded camera view, capture the reconstructed scene,
|
||||
and ask one visual question. Record the endpoint capability fallback if image
|
||||
input is rejected.
|
||||
7. Gracefully stop. Confirm no pending approvals, scheduled jobs, inventory
|
||||
offers, owned test prims, tasks, sockets, or sessions. List any unavoidable
|
||||
inventory artifact and its manual recovery ID in private operator notes,
|
||||
|
||||
@@ -119,9 +119,15 @@ avatar session -> persistent Mentra agent -> Responses SSE / compaction / memory
|
||||
| |
|
||||
| +-> AuthorizedToolBackend
|
||||
+-> one-shot Mentra safety reviewer when required
|
||||
|
||||
scene snapshot -> cached headless wgpu color/depth render -> JPEG -> Mentra image
|
||||
`-> deterministic software fallback
|
||||
```
|
||||
|
||||
The package has no build script or direct native dependency. The focused
|
||||
The package has no build script. Its cross-platform `wgpu` path uses the
|
||||
platform graphics backend selected by wgpu and requires no window or display;
|
||||
the software fallback remains available when adapter creation or rendering
|
||||
fails. The focused
|
||||
`dependency_policy` test rejects subprocess launch sites, unsafe/native ABI
|
||||
source, build scripts, and unreviewed direct dependency names in this package.
|
||||
The precise LLM compatibility and cancellation contract is documented in
|
||||
|
||||
@@ -56,16 +56,22 @@ facility, not a requirement for autonomous operation.
|
||||
|
||||
## Images
|
||||
|
||||
Vision captures are real software-rendered viewport images. MetaCrate renders
|
||||
the current scene, encodes the final frame directly as JPEG, and attaches it as
|
||||
a Mentra image content block. The default 320x180 frame, quality, entity,
|
||||
triangle, texture-fetch, decoded-pixel, byte, rate, and concurrency limits are
|
||||
validated before runtime. JPEG avoids the much larger PNG payloads.
|
||||
Vision captures are reconstructed viewport images rather than screenshots.
|
||||
The live service keeps one headless `wgpu` device, renders the current scene to
|
||||
an offscreen color/depth target, reads the frame back, encodes it directly as
|
||||
JPEG, and attaches it as a Mentra image content block. GPU setup and readback
|
||||
run outside the simulator event path. If no compatible adapter is available or
|
||||
a frame fails, the same scene is rendered by the deterministic software
|
||||
rasterizer. The default 320x180 frame, quality, entity, triangle, texture-fetch,
|
||||
decoded-pixel, byte, rate, and concurrency limits are validated before runtime.
|
||||
JPEG avoids the much larger PNG payloads.
|
||||
|
||||
The current renderer handles legacy prim geometry, approximate texture colors,
|
||||
simple avatars, and flat terrain/water. Viewer-grade mesh/sculpt rendering,
|
||||
full UV materials, lighting, authoritative terrain, and model-controlled camera
|
||||
tools are tracked separately.
|
||||
simple avatars, and flat terrain/water. `behavior_camera_set` gives an
|
||||
authorized model a bounded position, target, and vertical field of view;
|
||||
`behavior_camera_reset` restores the avatar-facing view. Viewer-grade
|
||||
mesh/sculpt rendering, full UV materials, lighting, authoritative terrain, and
|
||||
avatar/attachment fidelity remain tracked in the renderer issue.
|
||||
|
||||
## Bounds and cancellation
|
||||
|
||||
|
||||
@@ -226,8 +226,12 @@ The example records all current queue, message, conversation, tool, behavior,
|
||||
reconnect, and interaction defaults. Important defaults include 256 grid events,
|
||||
32 control commands, 512 observations, four concurrent inference requests,
|
||||
16 tool calls, 512 active senders/sessions, a two-minute model window, and bounded
|
||||
10-second shutdown. Vision defaults to a 320x180 software-rendered JPEG with bounded
|
||||
entities, triangles, texture work, JPEG bytes, time, and concurrency.
|
||||
10-second shutdown. Vision defaults to a 320x180 headless-wgpu JPEG with a
|
||||
deterministic software fallback and bounded entities, triangles, texture work,
|
||||
JPEG bytes, time, and concurrency. Authorized `behavior_camera_set` calls are
|
||||
limited to 96 meters from the avatar by default, require a distinct target, and
|
||||
accept a 20-120 degree vertical field of view; `behavior_camera_reset` restores
|
||||
the avatar-facing 60-degree view.
|
||||
|
||||
Unsupported by design: arbitrary raw packets or agent-control flags, arbitrary
|
||||
shell/subprocess execution, provider/model discovery outside Mentra, remote plaintext
|
||||
@@ -248,7 +252,8 @@ cargo audit
|
||||
|
||||
Record the binary byte size (`stat -c %s` on Linux or `Get-Item ... .Length` on
|
||||
PowerShell) and the `cargo tree` inventory. Reject CLR/scripting/provider SDK,
|
||||
subprocess adapters, Skia/GPU, or undeclared native libraries on the agent path.
|
||||
subprocess adapters, unreviewed graphics stacks, or undeclared native libraries
|
||||
on the agent path.
|
||||
The consolidated Gitea gate runs only on `ubuntu-latest`; Windows portability is
|
||||
proved by the existing cross-target compile/static gate rather than a Windows
|
||||
Gitea runner.
|
||||
|
||||
@@ -23,7 +23,7 @@ The audited tree contains the MetaCrate agent, the original compatibility
|
||||
crates it consumes, the pure-Rust J2K/meshing/rendering path, Tokio, Reqwest,
|
||||
Rustls, AWS-LC, terminal/serialization/bounds utilities, and their build-time
|
||||
Rust tooling. It contains no CLR/Mono/.NET runtime, Python/Node/Lua scripting
|
||||
runtime, provider SDK, Skia/GPU stack, OpenJPEG feature, shell/subprocess
|
||||
runtime, provider SDK, unreviewed graphics stack, shell/subprocess
|
||||
adapter, or dynamically loaded undeclared application library.
|
||||
|
||||
Regenerate for a release candidate with the commands in
|
||||
|
||||
Reference in New Issue
Block a user