Add headless wgpu vision and camera controls

crates/metacrate-grid-agent/src/behavior.rs

@@ -35,6 +35,8 @@ pub const STOP_TOOL: &str = "behavior_stop";
 pub const SIT_TOOL: &str = "behavior_sit";
 pub const STAND_TOOL: &str = "behavior_stand";
 pub const CURRENT_POSE_TOOL: &str = "behavior_current_pose";
+pub const CAMERA_SET_TOOL: &str = "behavior_camera_set";
+pub const CAMERA_RESET_TOOL: &str = "behavior_camera_reset";
 
 const WALK_POLL: Duration = Duration::from_millis(250);
 const TURN_INTERVAL: Duration = Duration::from_millis(40);
@@ -162,6 +164,23 @@ pub trait EmbodimentSink: Send + Sync + 'static {
     fn stop(&self, generation: u64, cancellation: CancellationToken) -> EmbodimentFuture<'_, ()>;
     fn sit(&self, generation: u64, cancellation: CancellationToken) -> EmbodimentFuture<'_, ()>;
     fn stand(&self, generation: u64, cancellation: CancellationToken) -> EmbodimentFuture<'_, ()>;
+    fn set_camera(
+        &self,
+        _generation: u64,
+        _position: WorldPosition,
+        _target: WorldPosition,
+        _vertical_fov_degrees: f64,
+        _cancellation: CancellationToken,
+    ) -> EmbodimentFuture<'_, ()> {
+        Box::pin(async { Err(BehaviorError::TargetUnavailable) })
+    }
+    fn reset_camera(
+        &self,
+        _generation: u64,
+        _cancellation: CancellationToken,
+    ) -> EmbodimentFuture<'_, ()> {
+        Box::pin(async { Err(BehaviorError::TargetUnavailable) })
+    }
 }
 
 #[derive(Clone, Debug, Eq, PartialEq)]
@@ -229,6 +248,12 @@ enum BehaviorAction {
     Sit,
     Stand,
     CurrentPose,
+    CameraSet {
+        position: WorldPosition,
+        target: WorldPosition,
+        vertical_fov_degrees: f64,
+    },
+    CameraReset,
 }
 
 impl BehaviorAction {
@@ -242,6 +267,8 @@ impl BehaviorAction {
             Self::Sit => SIT_TOOL,
             Self::Stand => STAND_TOOL,
             Self::CurrentPose => CURRENT_POSE_TOOL,
+            Self::CameraSet { .. } => CAMERA_SET_TOOL,
+            Self::CameraReset => CAMERA_RESET_TOOL,
         }
     }
 }
@@ -943,6 +970,7 @@ async fn wait_for_action_cancellation(
     }
 }
 
+#[allow(clippy::too_many_lines)]
 async fn perform_action(
     controller: &BehaviorController,
     state: ReadyState,
@@ -1035,6 +1063,51 @@ async fn perform_action(
                 .await?;
             Ok(json!({"status":"completed"}))
         }
+        BehaviorAction::CameraSet {
+            position,
+            target,
+            vertical_fov_degrees,
+        } => {
+            let pose = controller
+                .sink
+                .current_pose(state.generation, cancellation.clone())
+                .await?;
+            validate_pose(&pose, state)?;
+            validate_point(
+                pose.position,
+                *position,
+                controller.settings.max_attention_distance_meters,
+            )?;
+            validate_point(
+                pose.position,
+                *target,
+                controller.settings.max_attention_distance_meters,
+            )?;
+            if distance(*position, *target) < 0.25 || !(20.0..=120.0).contains(vertical_fov_degrees)
+            {
+                return Err(BehaviorError::InvalidArguments);
+            }
+            controller
+                .sink
+                .set_camera(
+                    state.generation,
+                    *position,
+                    *target,
+                    *vertical_fov_degrees,
+                    cancellation,
+                )
+                .await?;
+            Ok(
+                json!({"status":"completed","position":position,"target":target,"vertical_fov_degrees":vertical_fov_degrees}),
+            )
+        }
+        BehaviorAction::CameraReset => {
+            controller
+                .sink
+                .reset_camera(state.generation, cancellation)
+                .await?;
+            Ok(json!({"status":"completed"}))
+        }
     }
 }
 
@@ -1363,6 +1436,28 @@ fn parse_action(name: &str, json_arguments: &str) -> Result<BehaviorAction, Beha
             require_empty(args)?;
             Ok(BehaviorAction::CurrentPose)
         }
+        CAMERA_SET_TOOL => {
+            if args.len() != 7 {
+                return Err(BehaviorError::InvalidArguments);
+            }
+            Ok(BehaviorAction::CameraSet {
+                position: WorldPosition {
+                    x: number(args, "position_x")?,
+                    y: number(args, "position_y")?,
+                    z: number(args, "position_z")?,
+                },
+                target: WorldPosition {
+                    x: number(args, "target_x")?,
+                    y: number(args, "target_y")?,
+                    z: number(args, "target_z")?,
+                },
+                vertical_fov_degrees: number(args, "vertical_fov_degrees")?,
+            })
+        }
+        CAMERA_RESET_TOOL => {
+            require_empty(args)?;
+            Ok(BehaviorAction::CameraReset)
+        }
         _ => Err(BehaviorError::InvalidArguments),
     }
 }
@@ -1402,7 +1497,7 @@ fn require_empty(args: &Map<String, Value>) -> Result<(), BehaviorError> {
         .ok_or(BehaviorError::InvalidArguments)
 }
 
-#[allow(clippy::too_many_lines)] // One registry table keeps the eight schemas auditable together.
+#[allow(clippy::too_many_lines)] // One registry table keeps the ten schemas auditable together.
 pub fn behavior_policy_tools(settings: &BehaviorSettings) -> Result<Vec<PolicyTool>, PolicyError> {
     let origins = || {
         AllowedOrigins::new([
@@ -1490,10 +1585,43 @@ pub fn behavior_policy_tools(settings: &BehaviorSettings) -> Result<Vec<PolicyTo
         (
             CURRENT_POSE_TOOL,
             "Read current embodied pose and session provenance.",
-            empty,
+            empty.clone(),
             false,
             0,
         ),
+        (
+            CAMERA_SET_TOOL,
+            "Place the camera near the avatar and look at a point. Use this to focus or orbit objects; coordinates are current-region meters.",
+            object(
+                &[
+                    ("position_x", ToolSchema::Number),
+                    ("position_y", ToolSchema::Number),
+                    ("position_z", ToolSchema::Number),
+                    ("target_x", ToolSchema::Number),
+                    ("target_y", ToolSchema::Number),
+                    ("target_z", ToolSchema::Number),
+                    ("vertical_fov_degrees", ToolSchema::Number),
+                ],
+                &[
+                    "position_x",
+                    "position_y",
+                    "position_z",
+                    "target_x",
+                    "target_y",
+                    "target_z",
+                    "vertical_fov_degrees",
+                ],
+            ),
+            true,
+            0,
+        ),
+        (
+            CAMERA_RESET_TOOL,
+            "Reset the camera to the avatar position and facing direction.",
+            empty,
+            true,
+            0,
+        ),
     ];
     specs
         .into_iter()