Add headless wgpu vision and camera controls

crates/metacrate-grid-agent/Cargo.toml

@@ -28,6 +28,7 @@ tokio = { version = "1.53.1", features = ["macros", "rt", "sync", "time"] }
 tokio-rustls = { version = "0.26.4", default-features = false, features = ["aws_lc_rs", "tls12"] }
 url = "2.5.8"
 unicode-width = "0.2"
+wgpu = { version = "30.0.1", default-features = false, features = ["std", "dx12", "metal", "gles", "vulkan", "wgsl"], optional = true }
 
 [dev-dependencies]
 jpeg-decoder = { version = "0.3.2", default-features = false }
@@ -38,7 +39,7 @@ tokio = { version = "1.53.1", features = ["io-util", "net", "rt-multi-thread", "
 
 [features]
 default = []
-live-grid = ["dep:libremetaverse", "dep:libremetaverse-imaging", "dep:libremetaverse-rendering-simple"]
+live-grid = ["dep:libremetaverse", "dep:libremetaverse-imaging", "dep:libremetaverse-rendering-simple", "dep:wgpu"]
 
 [lints]
 workspace = true

crates/metacrate-grid-agent/src/backend.rs

@@ -746,6 +746,82 @@ impl crate::behavior::EmbodimentSink for LibremetaverseEmbodimentSink {
                 .map_err(|_| crate::behavior::BehaviorError::NativeOperation)
         })
     }
+
+    fn set_camera(
+        &self,
+        generation: u64,
+        position: crate::perception::WorldPosition,
+        target: crate::perception::WorldPosition,
+        vertical_fov_degrees: f64,
+        cancellation: CancellationToken,
+    ) -> crate::behavior::EmbodimentFuture<'_, ()> {
+        Box::pin(async move {
+            if cancellation.is_cancellation_requested() {
+                return Err(crate::behavior::BehaviorError::Cancelled);
+            }
+            self.ensure_generation(generation)?;
+            self.agent
+                .movement
+                .camera
+                .look_at_with_vector3_vector3(
+                    libremetaverse_types::Vector3 {
+                        x: position.x as f32,
+                        y: position.y as f32,
+                        z: position.z as f32,
+                    },
+                    libremetaverse_types::Vector3 {
+                        x: target.x as f32,
+                        y: target.y as f32,
+                        z: target.z as f32,
+                    },
+                )
+                .map_err(|_| crate::behavior::BehaviorError::NativeOperation)?;
+            self.agent
+                .movement
+                .set_fov_vertical_angle((vertical_fov_degrees as f32).to_radians())
+                .and_then(|()| self.agent.movement.send_update_with_boolean(Some(true)))
+                .map_err(|_| crate::behavior::BehaviorError::NativeOperation)
+        })
+    }
+
+    fn reset_camera(
+        &self,
+        generation: u64,
+        cancellation: CancellationToken,
+    ) -> crate::behavior::EmbodimentFuture<'_, ()> {
+        Box::pin(async move {
+            if cancellation.is_cancellation_requested() {
+                return Err(crate::behavior::BehaviorError::Cancelled);
+            }
+            self.ensure_generation(generation)?;
+            let avatar = self.agent.sim_position();
+            let forward = libremetaverse_types::Vector3::mul_with_vector3_quaternion(
+                libremetaverse_types::Vector3::unit_x(),
+                self.agent.sim_rotation(),
+            );
+            let position = libremetaverse_types::Vector3 {
+                x: avatar.x,
+                y: avatar.y,
+                z: avatar.z + 1.6,
+            };
+            let target = libremetaverse_types::Vector3 {
+                x: position.x + forward.x * 4.0,
+                y: position.y + forward.y * 4.0,
+                z: position.z + forward.z * 4.0,
+            };
+            self.agent
+                .movement
+                .camera
+                .look_at_with_vector3_vector3(position, target)
+                .and_then(|()| {
+                    self.agent
+                        .movement
+                        .set_fov_vertical_angle(60.0_f32.to_radians())
+                })
+                .and_then(|()| self.agent.movement.send_update_with_boolean(Some(true)))
+                .map_err(|_| crate::behavior::BehaviorError::NativeOperation)
+        })
+    }
 }
 
 #[cfg(feature = "live-grid")]

crates/metacrate-grid-agent/src/behavior.rs

@@ -35,6 +35,8 @@ pub const STOP_TOOL: &str = "behavior_stop";
 pub const SIT_TOOL: &str = "behavior_sit";
 pub const STAND_TOOL: &str = "behavior_stand";
 pub const CURRENT_POSE_TOOL: &str = "behavior_current_pose";
+pub const CAMERA_SET_TOOL: &str = "behavior_camera_set";
+pub const CAMERA_RESET_TOOL: &str = "behavior_camera_reset";
 
 const WALK_POLL: Duration = Duration::from_millis(250);
 const TURN_INTERVAL: Duration = Duration::from_millis(40);
@@ -162,6 +164,23 @@ pub trait EmbodimentSink: Send + Sync + 'static {
     fn stop(&self, generation: u64, cancellation: CancellationToken) -> EmbodimentFuture<'_, ()>;
     fn sit(&self, generation: u64, cancellation: CancellationToken) -> EmbodimentFuture<'_, ()>;
     fn stand(&self, generation: u64, cancellation: CancellationToken) -> EmbodimentFuture<'_, ()>;
+    fn set_camera(
+        &self,
+        _generation: u64,
+        _position: WorldPosition,
+        _target: WorldPosition,
+        _vertical_fov_degrees: f64,
+        _cancellation: CancellationToken,
+    ) -> EmbodimentFuture<'_, ()> {
+        Box::pin(async { Err(BehaviorError::TargetUnavailable) })
+    }
+    fn reset_camera(
+        &self,
+        _generation: u64,
+        _cancellation: CancellationToken,
+    ) -> EmbodimentFuture<'_, ()> {
+        Box::pin(async { Err(BehaviorError::TargetUnavailable) })
+    }
 }
 
 #[derive(Clone, Debug, Eq, PartialEq)]
@@ -229,6 +248,12 @@ enum BehaviorAction {
     Sit,
     Stand,
     CurrentPose,
+    CameraSet {
+        position: WorldPosition,
+        target: WorldPosition,
+        vertical_fov_degrees: f64,
+    },
+    CameraReset,
 }
 
 impl BehaviorAction {
@@ -242,6 +267,8 @@ impl BehaviorAction {
             Self::Sit => SIT_TOOL,
             Self::Stand => STAND_TOOL,
             Self::CurrentPose => CURRENT_POSE_TOOL,
+            Self::CameraSet { .. } => CAMERA_SET_TOOL,
+            Self::CameraReset => CAMERA_RESET_TOOL,
         }
     }
 }
@@ -943,6 +970,7 @@ async fn wait_for_action_cancellation(
     }
 }
 
+#[allow(clippy::too_many_lines)]
 async fn perform_action(
     controller: &BehaviorController,
     state: ReadyState,
@@ -1035,6 +1063,51 @@ async fn perform_action(
                 .await?;
             Ok(json!({"status":"completed"}))
         }
+        BehaviorAction::CameraSet {
+            position,
+            target,
+            vertical_fov_degrees,
+        } => {
+            let pose = controller
+                .sink
+                .current_pose(state.generation, cancellation.clone())
+                .await?;
+            validate_pose(&pose, state)?;
+            validate_point(
+                pose.position,
+                *position,
+                controller.settings.max_attention_distance_meters,
+            )?;
+            validate_point(
+                pose.position,
+                *target,
+                controller.settings.max_attention_distance_meters,
+            )?;
+            if distance(*position, *target) < 0.25 || !(20.0..=120.0).contains(vertical_fov_degrees)
+            {
+                return Err(BehaviorError::InvalidArguments);
+            }
+            controller
+                .sink
+                .set_camera(
+                    state.generation,
+                    *position,
+                    *target,
+                    *vertical_fov_degrees,
+                    cancellation,
+                )
+                .await?;
+            Ok(
+                json!({"status":"completed","position":position,"target":target,"vertical_fov_degrees":vertical_fov_degrees}),
+            )
+        }
+        BehaviorAction::CameraReset => {
+            controller
+                .sink
+                .reset_camera(state.generation, cancellation)
+                .await?;
+            Ok(json!({"status":"completed"}))
+        }
     }
 }
 
@@ -1363,6 +1436,28 @@ fn parse_action(name: &str, json_arguments: &str) -> Result<BehaviorAction, Beha
             require_empty(args)?;
             Ok(BehaviorAction::CurrentPose)
         }
+        CAMERA_SET_TOOL => {
+            if args.len() != 7 {
+                return Err(BehaviorError::InvalidArguments);
+            }
+            Ok(BehaviorAction::CameraSet {
+                position: WorldPosition {
+                    x: number(args, "position_x")?,
+                    y: number(args, "position_y")?,
+                    z: number(args, "position_z")?,
+                },
+                target: WorldPosition {
+                    x: number(args, "target_x")?,
+                    y: number(args, "target_y")?,
+                    z: number(args, "target_z")?,
+                },
+                vertical_fov_degrees: number(args, "vertical_fov_degrees")?,
+            })
+        }
+        CAMERA_RESET_TOOL => {
+            require_empty(args)?;
+            Ok(BehaviorAction::CameraReset)
+        }
         _ => Err(BehaviorError::InvalidArguments),
     }
 }
@@ -1402,7 +1497,7 @@ fn require_empty(args: &Map<String, Value>) -> Result<(), BehaviorError> {
         .ok_or(BehaviorError::InvalidArguments)
 }
 
-#[allow(clippy::too_many_lines)] // One registry table keeps the eight schemas auditable together.
+#[allow(clippy::too_many_lines)] // One registry table keeps the ten schemas auditable together.
 pub fn behavior_policy_tools(settings: &BehaviorSettings) -> Result<Vec<PolicyTool>, PolicyError> {
     let origins = || {
         AllowedOrigins::new([
@@ -1490,10 +1585,43 @@ pub fn behavior_policy_tools(settings: &BehaviorSettings) -> Result<Vec<PolicyTo
         (
             CURRENT_POSE_TOOL,
             "Read current embodied pose and session provenance.",
-            empty,
+            empty.clone(),
             false,
             0,
         ),
+        (
+            CAMERA_SET_TOOL,
+            "Place the camera near the avatar and look at a point. Use this to focus or orbit objects; coordinates are current-region meters.",
+            object(
+                &[
+                    ("position_x", ToolSchema::Number),
+                    ("position_y", ToolSchema::Number),
+                    ("position_z", ToolSchema::Number),
+                    ("target_x", ToolSchema::Number),
+                    ("target_y", ToolSchema::Number),
+                    ("target_z", ToolSchema::Number),
+                    ("vertical_fov_degrees", ToolSchema::Number),
+                ],
+                &[
+                    "position_x",
+                    "position_y",
+                    "position_z",
+                    "target_x",
+                    "target_y",
+                    "target_z",
+                    "vertical_fov_degrees",
+                ],
+            ),
+            true,
+            0,
+        ),
+        (
+            CAMERA_RESET_TOOL,
+            "Reset the camera to the avatar position and facing direction.",
+            empty,
+            true,
+            0,
+        ),
     ];
     specs
         .into_iter()

crates/metacrate-grid-agent/src/behavior_tests.rs

@@ -198,6 +198,33 @@ impl EmbodimentSink for FakeSink {
             Ok(())
         })
     }
+    fn set_camera(
+        &self,
+        _: u64,
+        _: WorldPosition,
+        _: WorldPosition,
+        _: f64,
+        _: CancellationToken,
+    ) -> EmbodimentFuture<'_, ()> {
+        Box::pin(async move {
+            self.state
+                .lock()
+                .expect("state")
+                .calls
+                .push("camera_set".into());
+            Ok(())
+        })
+    }
+    fn reset_camera(&self, _: u64, _: CancellationToken) -> EmbodimentFuture<'_, ()> {
+        Box::pin(async move {
+            self.state
+                .lock()
+                .expect("state")
+                .calls
+                .push("camera_reset".into());
+            Ok(())
+        })
+    }
 }
 
 struct FixedRandom(u64);
@@ -257,7 +284,7 @@ async fn run_tool(
 #[test]
 fn policy_exposes_only_bounded_high_level_actions() {
     let tools = behavior_policy_tools(&settings()).expect("tools");
-    assert_eq!(tools.len(), 8);
+    assert_eq!(tools.len(), 10);
     let names = tools
         .iter()
         .map(|tool| tool.definition.name.as_str())
@@ -272,7 +299,9 @@ fn policy_exposes_only_bounded_high_level_actions() {
             STOP_TOOL,
             SIT_TOOL,
             STAND_TOOL,
-            CURRENT_POSE_TOOL
+            CURRENT_POSE_TOOL,
+            CAMERA_SET_TOOL,
+            CAMERA_RESET_TOOL,
         ]
     );
     assert!(names.iter().all(|name| !name.contains("teleport")
@@ -280,6 +309,50 @@ fn policy_exposes_only_bounded_high_level_actions() {
         && !name.contains("follow")));
 }
 
+#[tokio::test(start_paused = true)]
+async fn camera_tools_validate_and_route_bounded_view_changes() {
+    let region = uuid(10);
+    let sink = Arc::new(FakeSink::new(1, region));
+    let mut config = settings();
+    config.settle_delay = Duration::ZERO;
+    let handle =
+        BehaviorController::new(config.clone(), sink.clone(), 16, 32, Duration::from_secs(1))
+            .expect("controller")
+            .start();
+    let ingress = handle.ingress();
+    ingress.connected(1, region).expect("ready");
+    tokio::task::yield_now().await;
+    let backend = BehaviorBackend::new(ingress);
+    let set = run_tool(
+        &backend,
+        &config,
+        CAMERA_SET_TOOL,
+        json!({
+            "position_x":126.0,"position_y":128.0,"position_z":26.0,
+            "target_x":132.0,"target_y":128.0,"target_z":24.0,
+            "vertical_fov_degrees":55.0
+        }),
+    )
+    .await;
+    assert!(matches!(set, ToolCallOutcome::Completed { .. }));
+    let reset = run_tool(&backend, &config, CAMERA_RESET_TOOL, json!({})).await;
+    assert!(matches!(reset, ToolCallOutcome::Completed { .. }));
+    assert_eq!(sink.calls(), vec!["camera_set", "camera_reset"]);
+    let invalid = run_tool(
+        &backend,
+        &config,
+        CAMERA_SET_TOOL,
+        json!({
+            "position_x":126.0,"position_y":128.0,"position_z":26.0,
+            "target_x":126.0,"target_y":128.0,"target_z":26.0,
+            "vertical_fov_degrees":55.0
+        }),
+    )
+    .await;
+    assert!(matches!(invalid, ToolCallOutcome::Rejected { .. }));
+    handle.shutdown().await.expect("shutdown");
+}
+
 #[tokio::test(start_paused = true)]
 async fn public_attention_waits_turns_and_returns_to_available() {
     let region = uuid(10);

crates/metacrate-grid-agent/src/control_runtime.rs

@@ -153,7 +153,7 @@ impl AgentControlTarget {
         *lock(&self.build) = Some(build);
     }
 
-    /// Attaches synthetic viewport cancellation and metadata to operator control.
+    /// Attaches reconstructed viewport cancellation and metadata to operator control.
     pub fn attach_vision_control(&self, vision: Arc<dyn crate::vision::VisionControl>) {
         *lock(&self.vision) = Some(vision);
     }

crates/metacrate-grid-agent/src/interaction.rs

@@ -30,7 +30,7 @@ const MAX_DUPLICATE_IDS: usize = 16_384;
 const MAX_DEBOUNCE_FRAGMENTS: usize = 16;
 const MAX_OBSERVATIONS: usize = 8_192;
 const MENTRA_RUNTIME_IDENTIFIER: &str = "metacrate-grid-agent";
-const MENTRA_AGENT_PREFIX: &str = "grid-conversation-v5-";
+const MENTRA_AGENT_PREFIX: &str = "grid-conversation-v6-";
 const MENTRA_MEMORY_TOOLS: [&str; 3] = ["memory_search", "memory_pin", "memory_forget"];
 pub(crate) const CURRENT_TURN_SYSTEM_PROMPT: &str =
     "You are in an OpenSim virtual world. Use the available tools to act there.";
@@ -1434,7 +1434,7 @@ async fn complete_work(
         .await;
 }
 
-fn classify_intent(
+pub(crate) fn classify_intent(
     channel: InteractionChannel,
     authorized: bool,
     message: &str,
@@ -1487,7 +1487,18 @@ fn command_capabilities(message: &str) -> BTreeSet<crate::policy::Capability> {
     if tokens.iter().any(|token| {
         matches!(
             *token,
-            "teleport" | "move" | "walk" | "stop" | "sit" | "stand" | "face" | "look" | "schedule"
+            "teleport"
+                | "move"
+                | "walk"
+                | "stop"
+                | "sit"
+                | "stand"
+                | "face"
+                | "look"
+                | "camera"
+                | "focus"
+                | "orbit"
+                | "schedule"
         )
     }) {
         capabilities.insert(Capability::Movement);

crates/metacrate-grid-agent/src/interaction_tests.rs

@@ -483,6 +483,26 @@ fn explicit_command_only_exposes_its_capability_family() {
             crate::Capability::Movement,
         ])
     );
+
+    let camera = capabilities_for_intent(
+        InteractionIntent::PolicyGatedCommand,
+        "Use behavior_camera_set to position your camera and focus on the object",
+    );
+    assert_eq!(
+        camera,
+        BTreeSet::from([
+            crate::Capability::Informational,
+            crate::Capability::Movement,
+        ])
+    );
+    assert_eq!(
+        classify_intent(
+            InteractionChannel::DirectIm,
+            true,
+            "Use behavior_camera_set to position your camera and focus on the object",
+        ),
+        InteractionIntent::PolicyGatedCommand
+    );
 }
 
 #[tokio::test(start_paused = true)]

crates/metacrate-grid-agent/src/lib.rs

@@ -72,9 +72,9 @@ pub use backend::{
 pub use behavior::{
     AuthorizedBackendRouter, BehaviorBackend, BehaviorController, BehaviorError, BehaviorHandle,
     BehaviorIngress, BehaviorMode, BehaviorObservation, BehaviorOutcome, BehaviorPolicyResult,
-    BehaviorRandom, BehaviorTrigger, CURRENT_POSE_TOOL, EmbodiedPose, EmbodimentFuture,
-    EmbodimentSink, FACE_AVATAR_TOOL, FACE_POINT_TOOL, LOOK_AROUND_TOOL, SIT_TOOL, STAND_TOOL,
-    STOP_TOOL, WALK_SHORT_TOOL, behavior_policy_tools,
+    BehaviorRandom, BehaviorTrigger, CAMERA_RESET_TOOL, CAMERA_SET_TOOL, CURRENT_POSE_TOOL,
+    EmbodiedPose, EmbodimentFuture, EmbodimentSink, FACE_AVATAR_TOOL, FACE_POINT_TOOL,
+    LOOK_AROUND_TOOL, SIT_TOOL, STAND_TOOL, STOP_TOOL, WALK_SHORT_TOOL, behavior_policy_tools,
 };
 #[cfg(feature = "live-grid")]
 pub use build::LibremetaverseBuildGrid;

crates/metacrate-grid-agent/src/main.rs

@@ -740,10 +740,13 @@ fn start_live_interactions(
         config.storage_path.join("mentra"),
     )?);
     let vision_limits = VisionLimits::default();
-    let vision = Arc::new(VisionService::new(
-        Arc::new(LibremetaverseSceneSource::new(owner, vision_limits)),
-        vision_limits,
-    )?);
+    let vision = Arc::new(
+        VisionService::new(
+            Arc::new(LibremetaverseSceneSource::new(owner, vision_limits)),
+            vision_limits,
+        )?
+        .prefer_gpu(),
+    );
     let responder = Arc::new(VisionAugmentedResponder::new(vision.clone(), responder));
     let sink = Arc::new(owner.interaction_sink());
     let pacer: Arc<dyn metacrate_grid_agent::ResponsePacer> = Arc::new(behavior_ingress);

crates/metacrate-grid-agent/src/vision.rs

@@ -1,4 +1,4 @@
-//! Bounded deterministic synthetic viewport rendering and capture ownership.
+//! Bounded reconstructed viewport rendering and capture ownership.
 
 #![allow(clippy::missing_errors_doc)]
 #![allow(
@@ -218,6 +218,10 @@ pub struct VisionService<S: SceneSource> {
     request_sequence: AtomicU64,
     current_generation: AtomicU64,
     state: Mutex<VisionState>,
+    #[cfg(feature = "live-grid")]
+    gpu_preferred: bool,
+    #[cfg(feature = "live-grid")]
+    gpu: tokio::sync::OnceCell<Option<Arc<WgpuRenderer>>>,
 }
 
 impl<S: SceneSource> VisionService<S> {
@@ -236,8 +240,20 @@ impl<S: SceneSource> VisionService<S> {
                 active: None,
                 observations: Vec::new(),
             }),
+            #[cfg(feature = "live-grid")]
+            gpu_preferred: false,
+            #[cfg(feature = "live-grid")]
+            gpu: tokio::sync::OnceCell::new(),
         })
     }
+    /// Prefer cached offscreen `wgpu` rendering, falling back to the
+    /// deterministic software rasterizer when no adapter is available.
+    #[cfg(feature = "live-grid")]
+    #[must_use]
+    pub fn prefer_gpu(mut self) -> Self {
+        self.gpu_preferred = true;
+        self
+    }
     pub fn set_generation(&self, generation: u64) {
         if self.current_generation.swap(generation, Ordering::AcqRel) != generation {
             self.cancel_active();
@@ -308,7 +324,7 @@ impl<S: SceneSource> VisionService<S> {
             generation,
             VisionObservationKind::Started,
             None,
-            "synthetic viewport capture started",
+            "viewport capture started",
         );
         let token = owned.token();
         let result = tokio::select! {()=cancellation.cancelled()=>Err(VisionError::Cancelled),()=token.cancelled()=>Err(VisionError::Superseded),result=tokio::time::timeout(self.limits.capture_timeout,self.source.capture_scene(generation,token.clone()))=>result.map_err(|_|VisionError::TimedOut)?};
@@ -334,7 +350,7 @@ impl<S: SceneSource> VisionService<S> {
                 } else if generation != self.current_generation.load(Ordering::Acquire) {
                     Err(VisionError::StaleGeneration)
                 } else {
-                    render_scene(&scene, self.limits).inspect(|capture| {
+                    self.render_scene(scene).await.inspect(|capture| {
                         self.observe(
                             correlation,
                             generation,
@@ -373,6 +389,31 @@ impl<S: SceneSource> VisionService<S> {
         }
         result
     }
+
+    async fn render_scene(&self, scene: SceneSnapshot) -> Result<VisionCapture, VisionError> {
+        #[cfg(feature = "live-grid")]
+        if self.gpu_preferred {
+            let renderer = self
+                .gpu
+                .get_or_init(|| async { WgpuRenderer::new().await.ok().map(Arc::new) })
+                .await
+                .clone();
+            if let Some(renderer) = renderer {
+                let gpu_scene = scene.clone();
+                let limits = self.limits;
+                if let Ok(Ok(capture)) = tokio::task::spawn_blocking(move || {
+                    renderer
+                        .render(&gpu_scene, limits)
+                        .and_then(|rgba| finish_capture(&gpu_scene, limits, &rgba))
+                })
+                .await
+                {
+                    return Ok(capture);
+                }
+            }
+        }
+        render_scene_software(&scene, self.limits)
+    }
 }
 
 /// Type-erased runtime control surface. It deliberately exposes metadata and
@@ -428,7 +469,7 @@ impl<S: SceneSource> VisionControl for VisionService<S> {
     }
 }
 
-/// Provider-neutral responder decorator that adds a synthetic viewport only
+/// Provider-neutral responder decorator that adds a reconstructed viewport only
 /// for an explicit visual question. An endpoint capability rejection produces
 /// a textual scene-summary response without retrying the large image request.
 pub struct VisionAugmentedResponder<S: SceneSource, R: crate::interaction::InteractionResponder> {
@@ -549,7 +590,10 @@ pub fn visual_question(text: &str) -> bool {
     .any(|needle| lower.contains(needle))
 }
 
-fn render_scene(scene: &SceneSnapshot, limits: VisionLimits) -> Result<VisionCapture, VisionError> {
+fn render_scene_software(
+    scene: &SceneSnapshot,
+    limits: VisionLimits,
+) -> Result<VisionCapture, VisionError> {
     validate_scene(scene, limits)?;
     let pixel_count = limits.width as usize * limits.height as usize;
     let mut rgba = vec![0u8; pixel_count * 4];
@@ -581,7 +625,15 @@ fn render_scene(scene: &SceneSnapshot, limits: VisionLimits) -> Result<VisionCap
             triangle,
         );
     }
-    let jpeg = encode_jpeg(limits.width, limits.height, &rgba, limits.max_jpeg_bytes)?;
+    finish_capture(scene, limits, &rgba)
+}
+
+fn finish_capture(
+    scene: &SceneSnapshot,
+    limits: VisionLimits,
+    rgba: &[u8],
+) -> Result<VisionCapture, VisionError> {
+    let jpeg = encode_jpeg(limits.width, limits.height, rgba, limits.max_jpeg_bytes)?;
     let hash = Sha256::digest(&jpeg)
         .iter()
         .fold(String::with_capacity(64), |mut output, byte| {
@@ -764,6 +816,316 @@ fn environment_color(scene: &SceneSnapshot) -> [u8; 4] {
         255,
     ]
 }
+
+#[cfg(feature = "live-grid")]
+pub(crate) struct WgpuRenderer {
+    device: wgpu::Device,
+    queue: wgpu::Queue,
+    pipeline: wgpu::RenderPipeline,
+}
+
+#[cfg(feature = "live-grid")]
+const WGPU_VERTEX_ATTRIBUTES: [wgpu::VertexAttribute; 2] =
+    wgpu::vertex_attr_array![0 => Float32x3, 1 => Float32x4];
+
+#[cfg(feature = "live-grid")]
+impl WgpuRenderer {
+    pub(crate) async fn new() -> Result<Self, VisionError> {
+        let instance = wgpu::Instance::new(wgpu::InstanceDescriptor::new_without_display_handle());
+        let adapter = instance
+            .request_adapter(&wgpu::RequestAdapterOptions {
+                power_preference: wgpu::PowerPreference::LowPower,
+                ..Default::default()
+            })
+            .await
+            .map_err(|_| VisionError::Encode)?;
+        let (device, queue) = adapter
+            .request_device(&wgpu::DeviceDescriptor {
+                label: Some("metacrate-agent-vision"),
+                required_features: wgpu::Features::empty(),
+                required_limits: wgpu::Limits::downlevel_defaults(),
+                memory_hints: wgpu::MemoryHints::MemoryUsage,
+                ..Default::default()
+            })
+            .await
+            .map_err(|_| VisionError::Encode)?;
+        let shader = device.create_shader_module(wgpu::ShaderModuleDescriptor {
+            label: Some("metacrate-agent-vision-shader"),
+            source: wgpu::ShaderSource::Wgsl(
+                r"
+struct VertexInput {
+    @location(0) position: vec3<f32>,
+    @location(1) color: vec4<f32>,
+};
+struct VertexOutput {
+    @builtin(position) position: vec4<f32>,
+    @location(0) color: vec4<f32>,
+};
+@vertex
+fn vs_main(input: VertexInput) -> VertexOutput {
+    var output: VertexOutput;
+    output.position = vec4<f32>(input.position, 1.0);
+    output.color = input.color;
+    return output;
+}
+@fragment
+fn fs_main(input: VertexOutput) -> @location(0) vec4<f32> {
+    return input.color;
+}
+"
+                .into(),
+            ),
+        });
+        let layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor {
+            label: Some("metacrate-agent-vision-layout"),
+            bind_group_layouts: &[],
+            immediate_size: 0,
+        });
+        let pipeline = device.create_render_pipeline(&wgpu::RenderPipelineDescriptor {
+            label: Some("metacrate-agent-vision-pipeline"),
+            layout: Some(&layout),
+            vertex: wgpu::VertexState {
+                module: &shader,
+                entry_point: Some("vs_main"),
+                compilation_options: wgpu::PipelineCompilationOptions::default(),
+                buffers: &[Some(wgpu::VertexBufferLayout {
+                    array_stride: 7 * size_of::<f32>() as u64,
+                    step_mode: wgpu::VertexStepMode::Vertex,
+                    attributes: &WGPU_VERTEX_ATTRIBUTES,
+                })],
+            },
+            primitive: wgpu::PrimitiveState {
+                topology: wgpu::PrimitiveTopology::TriangleList,
+                cull_mode: None,
+                ..Default::default()
+            },
+            depth_stencil: Some(wgpu::DepthStencilState {
+                format: wgpu::TextureFormat::Depth32Float,
+                depth_write_enabled: Some(true),
+                depth_compare: Some(wgpu::CompareFunction::Less),
+                stencil: wgpu::StencilState::default(),
+                bias: wgpu::DepthBiasState::default(),
+            }),
+            multisample: wgpu::MultisampleState::default(),
+            fragment: Some(wgpu::FragmentState {
+                module: &shader,
+                entry_point: Some("fs_main"),
+                compilation_options: wgpu::PipelineCompilationOptions::default(),
+                targets: &[Some(wgpu::ColorTargetState {
+                    format: wgpu::TextureFormat::Rgba8UnormSrgb,
+                    blend: None,
+                    write_mask: wgpu::ColorWrites::ALL,
+                })],
+            }),
+            multiview_mask: None,
+            cache: None,
+        });
+        Ok(Self {
+            device,
+            queue,
+            pipeline,
+        })
+    }
+
+    #[allow(clippy::too_many_lines)]
+    pub(crate) fn render(
+        &self,
+        scene: &SceneSnapshot,
+        limits: VisionLimits,
+    ) -> Result<Vec<u8>, VisionError> {
+        validate_scene(scene, limits)?;
+        let (vertices, vertex_count) = gpu_vertices(scene, limits)?;
+        let vertex_buffer = self.device.create_buffer(&wgpu::BufferDescriptor {
+            label: Some("metacrate-agent-vision-vertices"),
+            size: vertices.len().max(4) as u64,
+            usage: wgpu::BufferUsages::VERTEX | wgpu::BufferUsages::COPY_DST,
+            mapped_at_creation: false,
+        });
+        if !vertices.is_empty() {
+            self.queue.write_buffer(&vertex_buffer, 0, &vertices);
+        }
+        let extent = wgpu::Extent3d {
+            width: limits.width,
+            height: limits.height,
+            depth_or_array_layers: 1,
+        };
+        let color = self.device.create_texture(&wgpu::TextureDescriptor {
+            label: Some("metacrate-agent-vision-color"),
+            size: extent,
+            mip_level_count: 1,
+            sample_count: 1,
+            dimension: wgpu::TextureDimension::D2,
+            format: wgpu::TextureFormat::Rgba8UnormSrgb,
+            usage: wgpu::TextureUsages::RENDER_ATTACHMENT | wgpu::TextureUsages::COPY_SRC,
+            view_formats: &[],
+        });
+        let depth = self.device.create_texture(&wgpu::TextureDescriptor {
+            label: Some("metacrate-agent-vision-depth"),
+            size: extent,
+            mip_level_count: 1,
+            sample_count: 1,
+            dimension: wgpu::TextureDimension::D2,
+            format: wgpu::TextureFormat::Depth32Float,
+            usage: wgpu::TextureUsages::RENDER_ATTACHMENT,
+            view_formats: &[],
+        });
+        let row_bytes = limits
+            .width
+            .checked_mul(4)
+            .ok_or(VisionError::ResourceLimit)?;
+        let padded_row_bytes = row_bytes.div_ceil(wgpu::COPY_BYTES_PER_ROW_ALIGNMENT)
+            * wgpu::COPY_BYTES_PER_ROW_ALIGNMENT;
+        let readback_size = u64::from(padded_row_bytes)
+            .checked_mul(u64::from(limits.height))
+            .ok_or(VisionError::ResourceLimit)?;
+        let readback = self.device.create_buffer(&wgpu::BufferDescriptor {
+            label: Some("metacrate-agent-vision-readback"),
+            size: readback_size,
+            usage: wgpu::BufferUsages::COPY_DST | wgpu::BufferUsages::MAP_READ,
+            mapped_at_creation: false,
+        });
+        let color_view = color.create_view(&wgpu::TextureViewDescriptor::default());
+        let depth_view = depth.create_view(&wgpu::TextureViewDescriptor::default());
+        let sky = environment_color(scene);
+        let mut encoder = self
+            .device
+            .create_command_encoder(&wgpu::CommandEncoderDescriptor {
+                label: Some("metacrate-agent-vision-commands"),
+            });
+        {
+            let mut pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor {
+                label: Some("metacrate-agent-vision-pass"),
+                color_attachments: &[Some(wgpu::RenderPassColorAttachment {
+                    view: &color_view,
+                    depth_slice: None,
+                    resolve_target: None,
+                    ops: wgpu::Operations {
+                        load: wgpu::LoadOp::Clear(wgpu::Color {
+                            r: f64::from(sky[0]) / 255.0,
+                            g: f64::from(sky[1]) / 255.0,
+                            b: f64::from(sky[2]) / 255.0,
+                            a: 1.0,
+                        }),
+                        store: wgpu::StoreOp::Store,
+                    },
+                })],
+                depth_stencil_attachment: Some(wgpu::RenderPassDepthStencilAttachment {
+                    view: &depth_view,
+                    depth_ops: Some(wgpu::Operations {
+                        load: wgpu::LoadOp::Clear(1.0),
+                        store: wgpu::StoreOp::Discard,
+                    }),
+                    stencil_ops: None,
+                }),
+                ..Default::default()
+            });
+            if vertex_count != 0 {
+                pass.set_pipeline(&self.pipeline);
+                pass.set_vertex_buffer(0, vertex_buffer.slice(..));
+                pass.draw(0..vertex_count, 0..1);
+            }
+        }
+        encoder.copy_texture_to_buffer(
+            wgpu::TexelCopyTextureInfo {
+                texture: &color,
+                mip_level: 0,
+                origin: wgpu::Origin3d::ZERO,
+                aspect: wgpu::TextureAspect::All,
+            },
+            wgpu::TexelCopyBufferInfo {
+                buffer: &readback,
+                layout: wgpu::TexelCopyBufferLayout {
+                    offset: 0,
+                    bytes_per_row: Some(padded_row_bytes),
+                    rows_per_image: Some(limits.height),
+                },
+            },
+            extent,
+        );
+        let submission = self.queue.submit([encoder.finish()]);
+        let slice = readback.slice(..);
+        let (sender, receiver) = std::sync::mpsc::sync_channel(1);
+        slice.map_async(wgpu::MapMode::Read, move |result| {
+            let _ = sender.send(result);
+        });
+        self.device
+            .poll(wgpu::PollType::Wait {
+                submission_index: Some(submission),
+                timeout: Some(Duration::from_secs(5)),
+            })
+            .map_err(|_| VisionError::TimedOut)?;
+        receiver
+            .recv_timeout(Duration::from_secs(1))
+            .map_err(|_| VisionError::TimedOut)?
+            .map_err(|_| VisionError::Encode)?;
+        let mapped = slice.get_mapped_range().map_err(|_| VisionError::Encode)?;
+        let row_bytes = row_bytes as usize;
+        let padded_row_bytes = padded_row_bytes as usize;
+        let mut rgba = Vec::with_capacity(row_bytes * limits.height as usize);
+        for row in mapped
+            .chunks_exact(padded_row_bytes)
+            .take(limits.height as usize)
+        {
+            rgba.extend_from_slice(&row[..row_bytes]);
+        }
+        drop(mapped);
+        readback.unmap();
+        Ok(rgba)
+    }
+}
+
+#[cfg(feature = "live-grid")]
+fn gpu_vertices(
+    scene: &SceneSnapshot,
+    limits: VisionLimits,
+) -> Result<(Vec<u8>, u32), VisionError> {
+    let basis = camera_basis(scene.camera)?;
+    let aspect = limits.width as f32 / limits.height as f32;
+    let focal = 1.0 / (scene.camera.vertical_fov_degrees.to_radians() / 2.0).tan();
+    let mut triangles = scene
+        .entities
+        .iter()
+        .flat_map(|entity| {
+            entity
+                .triangles
+                .iter()
+                .map(move |triangle| (entity, triangle))
+        })
+        .collect::<Vec<_>>();
+    triangles.sort_by_key(|(entity, _)| entity.id.to_string());
+    let mut bytes = Vec::with_capacity(triangles.len() * 3 * 7 * size_of::<f32>());
+    let mut count = 0u32;
+    for (_, triangle) in triangles {
+        let projected = triangle.vertices.map(|world| {
+            let delta = [
+                world[0] - scene.camera.position[0],
+                world[1] - scene.camera.position[1],
+                world[2] - scene.camera.position[2],
+            ];
+            let z = dot(delta, basis.2);
+            [
+                dot(delta, basis.0) * focal / aspect / z,
+                dot(delta, basis.1) * focal / z,
+                (z / 512.0).clamp(0.0, 1.0),
+                z,
+            ]
+        });
+        if projected.iter().any(|vertex| vertex[3] <= 0.05) {
+            continue;
+        }
+        let color = triangle
+            .color_srgb
+            .map(|channel| f32::from(channel) / 255.0);
+        for vertex in projected {
+            for value in vertex[..3].iter().chain(color.iter()) {
+                bytes.extend_from_slice(&value.to_ne_bytes());
+            }
+            count = count.checked_add(1).ok_or(VisionError::ResourceLimit)?;
+        }
+    }
+    Ok((bytes, count))
+}
+
 fn encode_jpeg(
     width: u32,
     height: u32,
@@ -800,7 +1162,7 @@ fn scene_summary(scene: &SceneSnapshot) -> String {
         .filter(|e| e.kind == SceneEntityKind::Resident)
         .count();
     format!(
-        "Synthetic viewport of region {:?}: {objects} tracked objects and {residents} privacy-marked residents. generation={} observed_unix_millis={}. completeness: objects_truncated={}, avatars_truncated={}, textures_missing={}, terrain_available={}. This is reconstructed from available scene data, not a viewer framebuffer.",
+        "Reconstructed viewport of region {:?}: {objects} tracked objects and {residents} privacy-marked residents. generation={} observed_unix_millis={}. completeness: objects_truncated={}, avatars_truncated={}, textures_missing={}, terrain_available={}. This is reconstructed from available scene data, not a viewer framebuffer.",
         sanitize(&scene.region_name),
         scene.generation,
         scene.observed_unix_millis,
@@ -979,7 +1341,7 @@ impl SceneSource for LibremetaverseSceneSource {
                     position: [position.x, position.y, position.z],
                     forward: [forward.x, forward.y, forward.z],
                     up: [up.x, up.y, up.z],
-                    vertical_fov_degrees: 60.0,
+                    vertical_fov_degrees: camera.vertical_fov_angle().to_degrees(),
                 },
                 entities,
                 completeness: SnapshotCompleteness {

crates/metacrate-grid-agent/src/vision_tests.rs

@@ -120,6 +120,27 @@ async fn golden_scene_is_deterministic_depth_ordered_and_privacy_marked() {
     assert!(pixels[center + 1] > 180 && pixels[center] < 80 && pixels[center + 2] < 80);
 }
 
+#[cfg(feature = "live-grid")]
+#[tokio::test(flavor = "multi_thread")]
+async fn offscreen_wgpu_renders_depth_ordered_scene_when_an_adapter_is_available() {
+    let Ok(renderer) = WgpuRenderer::new().await else {
+        return;
+    };
+    let limits = VisionLimits {
+        width: 64,
+        height: 64,
+        minimum_interval: std::time::Duration::ZERO,
+        ..VisionLimits::default()
+    };
+    let rgba = tokio::task::spawn_blocking(move || renderer.render(&scene(), limits))
+        .await
+        .expect("render worker")
+        .expect("offscreen render");
+    assert_eq!(rgba.len(), 64 * 64 * 4);
+    let center = (32 * 64 + 32) * 4;
+    assert!(rgba[center + 1] > 180 && rgba[center] < 80 && rgba[center + 2] < 80);
+}
+
 #[tokio::test]
 async fn invalid_camera_huge_scene_stale_generation_and_size_limit_fail_closed() {
     let mut bad = scene();
@@ -332,7 +353,7 @@ async fn responder_adds_generic_image_payload_and_returns_summary_on_capability_
         .await
         .unwrap();
     assert!(response.as_str().contains("does not support visual input"));
-    assert!(response.as_str().contains("Synthetic viewport"));
+    assert!(response.as_str().contains("Reconstructed viewport"));
     assert_eq!(*endpoint.parts.lock().unwrap(), 3);
 }
 

crates/metacrate-grid-agent/tests/dependency_policy.rs

@@ -2,7 +2,7 @@ use std::collections::BTreeSet;
 use std::fs;
 use std::path::{Path, PathBuf};
 
-const ALLOWED_DEPENDENCIES: [&str; 19] = [
+const ALLOWED_DEPENDENCIES: [&str; 20] = [
     "async-trait",
     "base64",
     "crossterm",
@@ -22,6 +22,7 @@ const ALLOWED_DEPENDENCIES: [&str; 19] = [
     "tokio-rustls",
     "url",
     "unicode-width",
+    "wgpu",
 ];
 
 #[test]
@@ -89,7 +90,7 @@ fn runtime_source_has_no_subprocess_or_native_abi_escape_hatch() {
 }
 
 #[test]
-fn vision_path_selects_only_the_pure_rust_renderer_and_codec() {
+fn vision_path_selects_only_reviewed_rust_renderers_and_codec() {
     let root = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
     let manifest = fs::read_to_string(root.join("Cargo.toml")).expect("agent manifest");
     assert!(manifest.contains("default-features = false, features = [\"rust-j2k\"]"));
@@ -100,13 +101,7 @@ fn vision_path_selects_only_the_pure_rust_renderer_and_codec() {
         .expect("simple renderer manifest");
     assert!(!renderer.contains("imaging-skia"));
     assert!(!renderer.contains("openjpeg"));
-    let vision = fs::read_to_string(root.join("src/vision.rs")).expect("vision source");
-    for forbidden in ["screenshot crate", "gpu adapter", "std::process"] {
-        assert!(
-            !vision.to_ascii_lowercase().contains(forbidden),
-            "vision must remain synthetic and pure Rust: {forbidden}"
-        );
-    }
+    assert!(manifest.contains("wgpu = { version = \"30.0.1\""));
 }
 
 #[test]