feat(grid-agent): add portable control plane (#126)

crates/metacrate-grid-agent/src/behavior.rs

@@ -19,8 +19,8 @@ use serde_json::{Map, Value, json};
 use std::collections::{BTreeMap, BTreeSet};
 use std::error::Error;
 use std::fmt;
-use std::sync::Arc;
 use std::sync::atomic::{AtomicU64, Ordering};
+use std::sync::{Arc, Mutex, MutexGuard};
 use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
 use tokio::sync::{mpsc, oneshot, watch};
 use tokio::task::JoinHandle;
@@ -36,6 +36,7 @@ pub const CURRENT_POSE_TOOL: &str = "behavior_current_pose";
 
 const WALK_POLL: Duration = Duration::from_millis(250);
 const TURN_INTERVAL: Duration = Duration::from_millis(40);
+const MAX_ACTION_ID_BYTES: usize = 96;
 const MAX_TURN_STEP_DEGREES: f64 = 30.0;
 const ARRIVAL_METERS: f64 = 0.5;
 const STUCK_PROGRESS_METERS: f64 = 0.05;
@@ -196,6 +197,7 @@ pub enum BehaviorObservation {
         trigger: BehaviorTrigger,
     },
     Action {
+        action_id: String,
         generation: Option<u64>,
         region_id: Option<UUID>,
         action: String,
@@ -243,15 +245,23 @@ impl BehaviorAction {
 }
 
 struct ActionRequest {
+    action_id: String,
     action: BehaviorAction,
     trigger: BehaviorTrigger,
     policy: BehaviorPolicyResult,
     reply: oneshot::Sender<Result<Value, BehaviorError>>,
 }
 
+#[derive(Default)]
+struct ActionRegistry {
+    active: BTreeSet<String>,
+    cancelled: BTreeSet<String>,
+}
+
 enum Command {
     Action(ActionRequest),
     Attention {
+        action_id: String,
         delivery_id: String,
         avatar_id: UUID,
         reply: oneshot::Sender<Result<(), BehaviorError>>,
@@ -266,6 +276,10 @@ pub struct BehaviorIngress {
     ready: watch::Sender<Option<ReadyState>>,
     paused: watch::Sender<bool>,
     emergency: watch::Sender<bool>,
+    action_cancel: watch::Sender<u64>,
+    actions: Arc<Mutex<ActionRegistry>>,
+    next_action: Arc<AtomicU64>,
+    action_capacity: usize,
 }
 
 impl fmt::Debug for BehaviorIngress {
@@ -313,6 +327,28 @@ impl BehaviorIngress {
         let _ = self.emergency.send(false);
     }
 
+    /// Cancels one exact queued or executing behavior action ID.
+    pub fn cancel_action(&self, action_id: &str) -> Result<bool, BehaviorError> {
+        if !valid_action_id(action_id) {
+            return Err(BehaviorError::InvalidArguments);
+        }
+        let cancelled = {
+            let mut actions = lock_actions(&self.actions);
+            if actions.active.contains(action_id) {
+                actions.cancelled.insert(action_id.to_owned());
+                true
+            } else {
+                false
+            }
+        };
+        if cancelled {
+            self.action_cancel.send_modify(|generation| {
+                *generation = generation.saturating_add(1);
+            });
+        }
+        Ok(cancelled)
+    }
+
     /// Marks a separately authorized scheduler task as roaming. The controller
     /// itself never invents a roaming route or exposes follow/wander behavior.
     pub fn set_roaming(&self, roaming: bool) -> Result<(), BehaviorError> {
@@ -323,20 +359,30 @@ impl BehaviorIngress {
 
     async fn request(
         &self,
+        action_id: String,
         action: BehaviorAction,
         trigger: BehaviorTrigger,
         policy: BehaviorPolicyResult,
     ) -> Result<Value, BehaviorError> {
+        if !self.register_action(&action_id) {
+            return Err(BehaviorError::InvalidArguments);
+        }
         let (reply, receive) = oneshot::channel();
-        self.commands
+        if self
+            .commands
             .send(Command::Action(ActionRequest {
+                action_id: action_id.clone(),
                 action,
                 trigger,
                 policy,
                 reply,
             }))
             .await
-            .map_err(|_| BehaviorError::QueueClosed)?;
+            .is_err()
+        {
+            self.unregister_action(&action_id);
+            return Err(BehaviorError::QueueClosed);
+        }
         receive.await.map_err(|_| BehaviorError::QueueClosed)?
     }
 
@@ -345,17 +391,44 @@ impl BehaviorIngress {
         delivery_id: String,
         avatar_id: UUID,
     ) -> Result<(), BehaviorError> {
+        let action_id = format!(
+            "attention-{}",
+            self.next_action.fetch_add(1, Ordering::Relaxed)
+        );
+        if !self.register_action(&action_id) {
+            return Err(BehaviorError::InvalidArguments);
+        }
         let (reply, receive) = oneshot::channel();
-        self.commands
+        if self
+            .commands
             .send(Command::Attention {
+                action_id: action_id.clone(),
                 delivery_id,
                 avatar_id,
                 reply,
             })
             .await
-            .map_err(|_| BehaviorError::QueueClosed)?;
+            .is_err()
+        {
+            self.unregister_action(&action_id);
+            return Err(BehaviorError::QueueClosed);
+        }
         receive.await.map_err(|_| BehaviorError::QueueClosed)?
     }
+
+    fn register_action(&self, action_id: &str) -> bool {
+        if !valid_action_id(action_id) {
+            return false;
+        }
+        let mut actions = lock_actions(&self.actions);
+        actions.active.len() < self.action_capacity && actions.active.insert(action_id.to_owned())
+    }
+
+    fn unregister_action(&self, action_id: &str) {
+        let mut actions = lock_actions(&self.actions);
+        actions.active.remove(action_id);
+        actions.cancelled.remove(action_id);
+    }
 }
 
 impl ResponsePacer for BehaviorIngress {
@@ -485,12 +558,18 @@ impl BehaviorController {
         let (ready, ready_rx) = watch::channel(None);
         let (paused, paused_rx) = watch::channel(false);
         let (emergency, emergency_rx) = watch::channel(false);
+        let (action_cancel, action_cancel_rx) = watch::channel(0_u64);
+        let actions = Arc::new(Mutex::new(ActionRegistry::default()));
         let (observations, observation_rx) = mpsc::channel(self.observation_capacity);
         let ingress = BehaviorIngress {
             commands,
             ready,
             paused,
             emergency,
+            action_cancel,
+            actions: actions.clone(),
+            next_action: Arc::new(AtomicU64::new(1)),
+            action_capacity: self.queue_capacity,
         };
         let shutdown_timeout = self.shutdown_timeout;
         let task = tokio::spawn(run_actor(
@@ -499,6 +578,8 @@ impl BehaviorController {
             ready_rx,
             paused_rx,
             emergency_rx,
+            action_cancel_rx,
+            actions,
             observations,
         ));
         BehaviorHandle {
@@ -547,13 +628,15 @@ impl BehaviorRandom for SystemBehaviorRandom {
     }
 }
 
-#[allow(clippy::too_many_lines)] // Lifecycle, preemption, idle, and commands share one ordered owner.
+#[allow(clippy::too_many_arguments, clippy::too_many_lines)] // One ordered lifecycle owner.
 async fn run_actor(
     controller: BehaviorController,
     mut commands: mpsc::Receiver<Command>,
     mut ready: watch::Receiver<Option<ReadyState>>,
     mut paused: watch::Receiver<bool>,
     mut emergency: watch::Receiver<bool>,
+    action_cancel: watch::Receiver<u64>,
+    actions: Arc<Mutex<ActionRegistry>>,
     observations: mpsc::Sender<BehaviorObservation>,
 ) {
     let mut mode = BehaviorMode::Offline;
@@ -616,8 +699,16 @@ async fn run_actor(
             () = &mut idle, if controller.settings.idle_look_enabled => {
                 if mode == BehaviorMode::Available {
                     let (reply, _) = oneshot::channel();
-                    let request = ActionRequest { action: BehaviorAction::LookAround, trigger: BehaviorTrigger::IdleTimer, policy: BehaviorPolicyResult::InternalIdle, reply };
-                    execute_request(&controller, request, &mut mode, &mut last_action, &ready, &paused, &emergency, &observations).await;
+                    let action_id = format!("idle-{}", unix_millis_now());
+                    let registered = {
+                        let mut registry = lock_actions(&actions);
+                        registry.active.len() < controller.queue_capacity
+                            && registry.active.insert(action_id.clone())
+                    };
+                    if registered {
+                        let request = ActionRequest { action_id, action: BehaviorAction::LookAround, trigger: BehaviorTrigger::IdleTimer, policy: BehaviorPolicyResult::InternalIdle, reply };
+                        execute_request(&controller, request, &mut mode, &mut last_action, &ready, &paused, &emergency, &action_cancel, &actions, &observations).await;
+                    }
                 }
                 idle.as_mut().reset(tokio::time::Instant::now() + controller.settings.idle_interval);
             }
@@ -642,9 +733,10 @@ async fn run_actor(
                         };
                         transition(&observations, &mut mode, next, BehaviorTrigger::IdleTimer).await;
                     }
-                    Command::Action(request) => execute_request(&controller, request, &mut mode, &mut last_action, &ready, &paused, &emergency, &observations).await,
-                    Command::Attention { delivery_id, avatar_id, reply } => {
+                    Command::Action(request) => execute_request(&controller, request, &mut mode, &mut last_action, &ready, &paused, &emergency, &action_cancel, &actions, &observations).await,
+                    Command::Attention { action_id, delivery_id, avatar_id, reply } => {
                         let request = ActionRequest {
+                            action_id,
                             action: BehaviorAction::FaceAvatar(avatar_id),
                             trigger: BehaviorTrigger::PublicResponse { delivery_id, avatar_id },
                             policy: BehaviorPolicyResult::BuiltInAttention,
@@ -652,7 +744,7 @@ async fn run_actor(
                         };
                         let delay = random_duration(&controller, controller.settings.response_delay_min, controller.settings.response_delay_max);
                         tokio::time::sleep(delay).await;
-                        execute_request(&controller, request, &mut mode, &mut last_action, &ready, &paused, &emergency, &observations).await;
+                        execute_request(&controller, request, &mut mode, &mut last_action, &ready, &paused, &emergency, &action_cancel, &actions, &observations).await;
                         if mode == BehaviorMode::Engaged {
                             tokio::time::sleep(controller.settings.attention_dwell).await;
                             if !*paused.borrow() && ready.borrow().is_some() { transition(&observations, &mut mode, BehaviorMode::Available, BehaviorTrigger::IdleTimer).await; }
@@ -702,12 +794,17 @@ async fn execute_request(
     ready: &watch::Receiver<Option<ReadyState>>,
     paused: &watch::Receiver<bool>,
     emergency: &watch::Receiver<bool>,
+    action_cancel: &watch::Receiver<u64>,
+    actions: &Arc<Mutex<ActionRegistry>>,
     observations: &mpsc::Sender<BehaviorObservation>,
 ) {
     let started = Instant::now();
+    let action_id = request.action_id.clone();
     let action_name = request.action.name().to_owned();
     let state = *ready.borrow();
-    let mut result = if *emergency.borrow() {
+    let mut result = if take_action_cancellation(actions, &action_id) {
+        Err(BehaviorError::Cancelled)
+    } else if *emergency.borrow() {
         Err(BehaviorError::EmergencyStopped)
     } else if *paused.borrow() {
         Err(BehaviorError::Paused)
@@ -737,11 +834,15 @@ async fn execute_request(
         let mut action_ready = ready.clone();
         let mut action_paused = paused.clone();
         let mut action_emergency = emergency.clone();
+        let action_cancel = action_cancel.clone();
+        let action_registry = actions.clone();
+        let cancelled_id = action_id.clone();
         result = tokio::select! {
             value = tokio::time::timeout(controller.settings.action_timeout, perform_action(controller, state, &request.action)) => value.unwrap_or(Err(BehaviorError::TimedOut)),
             _ = action_ready.changed() => Err(BehaviorError::Cancelled),
             _ = action_paused.changed() => Err(BehaviorError::Paused),
             _ = action_emergency.changed() => Err(BehaviorError::EmergencyStopped),
+            () = wait_for_action_cancellation(action_cancel, action_registry, cancelled_id) => Err(BehaviorError::Cancelled),
         };
         if result.is_err() && matches!(request.action, BehaviorAction::WalkShort { .. }) {
             let _ = controller
@@ -811,6 +912,7 @@ async fn execute_request(
     };
     let _ = observations
         .send(BehaviorObservation::Action {
+            action_id: action_id.clone(),
             generation: state.map(|value| value.generation),
             region_id: state.map(|value| value.region_id),
             action: action_name,
@@ -821,6 +923,22 @@ async fn execute_request(
         })
         .await;
     let _ = request.reply.send(result);
+    unregister_action(actions, &action_id);
+}
+
+async fn wait_for_action_cancellation(
+    mut signal: watch::Receiver<u64>,
+    actions: Arc<Mutex<ActionRegistry>>,
+    action_id: String,
+) {
+    loop {
+        if take_action_cancellation(&actions, &action_id) {
+            return;
+        }
+        if signal.changed().await.is_err() {
+            std::future::pending::<()>().await;
+        }
+    }
 }
 
 async fn perform_action(
@@ -1086,6 +1204,38 @@ fn random_duration(
     minimum + Duration::from_millis(u64::try_from(offset).unwrap_or(u64::MAX))
 }
 
+fn valid_action_id(value: &str) -> bool {
+    !value.is_empty()
+        && value.len() <= MAX_ACTION_ID_BYTES
+        && value.chars().all(|character| {
+            character.is_ascii_alphanumeric() || matches!(character, '-' | '_' | '.' | ':')
+        })
+}
+
+fn lock_actions(value: &Mutex<ActionRegistry>) -> MutexGuard<'_, ActionRegistry> {
+    value
+        .lock()
+        .unwrap_or_else(std::sync::PoisonError::into_inner)
+}
+
+fn take_action_cancellation(actions: &Mutex<ActionRegistry>, action_id: &str) -> bool {
+    lock_actions(actions).cancelled.remove(action_id)
+}
+
+fn unregister_action(actions: &Mutex<ActionRegistry>, action_id: &str) {
+    let mut actions = lock_actions(actions);
+    actions.active.remove(action_id);
+    actions.cancelled.remove(action_id);
+}
+
+fn unix_millis_now() -> u64 {
+    SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .map_or(0, |duration| {
+            u64::try_from(duration.as_millis()).unwrap_or(u64::MAX)
+        })
+}
+
 #[derive(Clone)]
 pub struct BehaviorBackend {
     ingress: BehaviorIngress,
@@ -1136,7 +1286,7 @@ impl AuthorizedToolBackend for BehaviorBackend {
             let result = match parsed {
                 Ok(behavior_action) => tokio::select! {
                     () = cancellation.cancelled() => Err(BehaviorError::Cancelled),
-                    value = self.ingress.request(behavior_action, BehaviorTrigger::AuthorizedTool { authorization_id, tool: action.call().name.as_str().to_owned() }, BehaviorPolicyResult::Authorized(authorization_id)) => value,
+                    value = self.ingress.request(call_id.as_str().to_owned(), behavior_action, BehaviorTrigger::AuthorizedTool { authorization_id, tool: action.call().name.as_str().to_owned() }, BehaviorPolicyResult::Authorized(authorization_id)) => value,
                 },
                 Err(error) => Err(error),
             };