feat(grid-agent): add portable control plane (#126)
This commit is contained in:
2
Cargo.lock
generated
2
Cargo.lock
generated
@@ -2201,10 +2201,12 @@ dependencies = [
|
|||||||
"libremetaverse",
|
"libremetaverse",
|
||||||
"libremetaverse-types",
|
"libremetaverse-types",
|
||||||
"reqwest",
|
"reqwest",
|
||||||
|
"rustls",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
"sha2 0.11.0",
|
"sha2 0.11.0",
|
||||||
"tokio",
|
"tokio",
|
||||||
|
"tokio-rustls",
|
||||||
"url",
|
"url",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|||||||
@@ -12,10 +12,12 @@ publish = false
|
|||||||
libremetaverse = { version = "0.0.1", path = "../libremetaverse", default-features = false, optional = true }
|
libremetaverse = { version = "0.0.1", path = "../libremetaverse", default-features = false, optional = true }
|
||||||
libremetaverse-types = { version = "0.0.1", path = "../libremetaverse-types" }
|
libremetaverse-types = { version = "0.0.1", path = "../libremetaverse-types" }
|
||||||
reqwest = { version = "0.13.4", default-features = false, features = ["rustls"] }
|
reqwest = { version = "0.13.4", default-features = false, features = ["rustls"] }
|
||||||
|
rustls = { version = "0.23.43", default-features = false, features = ["aws_lc_rs", "std", "tls12"] }
|
||||||
serde = { version = "1", features = ["derive"] }
|
serde = { version = "1", features = ["derive"] }
|
||||||
serde_json = "1"
|
serde_json = "1"
|
||||||
sha2 = "0.11"
|
sha2 = "0.11"
|
||||||
tokio = { version = "1.53.1", features = ["macros", "rt", "sync", "time"] }
|
tokio = { version = "1.53.1", features = ["macros", "rt", "sync", "time"] }
|
||||||
|
tokio-rustls = { version = "0.26.4", default-features = false, features = ["aws_lc_rs", "tls12"] }
|
||||||
url = "2.5.8"
|
url = "2.5.8"
|
||||||
|
|
||||||
[dev-dependencies]
|
[dev-dependencies]
|
||||||
|
|||||||
@@ -22,7 +22,10 @@ optional JSON file, its referenced secret files, then environment values (an
|
|||||||
environment-referenced secret file is below a direct environment secret).
|
environment-referenced secret file is below a direct environment secret).
|
||||||
Supported secret environment variables are
|
Supported secret environment variables are
|
||||||
`METACRATE_AGENT_LLM_API_KEY[_FILE]` and
|
`METACRATE_AGENT_LLM_API_KEY[_FILE]` and
|
||||||
`METACRATE_AGENT_GRID_PASSWORD[_FILE]`. Secret files must be bounded regular,
|
`METACRATE_AGENT_GRID_PASSWORD[_FILE]`. Split control uses the separate
|
||||||
|
`METACRATE_AGENT_CONTROL_OPERATOR_TOKEN[_FILE]` and optional
|
||||||
|
`METACRATE_AGENT_CONTROL_OBSERVER_TOKEN[_FILE]`, plus
|
||||||
|
`METACRATE_AGENT_CONTROL_LISTEN`. Secret files must be bounded regular,
|
||||||
non-symlink UTF-8 files containing one line. Operators must restrict their OS
|
non-symlink UTF-8 files containing one line. Operators must restrict their OS
|
||||||
ACLs to the service identity; the core uses only portable `std::fs` checks and
|
ACLs to the service identity; the core uses only portable `std::fs` checks and
|
||||||
does not assume Unix permission bits.
|
does not assume Unix permission bits.
|
||||||
@@ -51,3 +54,6 @@ shutdown deadline are specified in
|
|||||||
Per-avatar/channel expiry, compaction, redaction, optional atomic persistence,
|
Per-avatar/channel expiry, compaction, redaction, optional atomic persistence,
|
||||||
and metadata-only operator controls are specified in
|
and metadata-only operator controls are specified in
|
||||||
[`../../docs/grid-agent-conversation.md`](../../docs/grid-agent-conversation.md).
|
[`../../docs/grid-agent-conversation.md`](../../docs/grid-agent-conversation.md).
|
||||||
|
The versioned integrated/TCP protocol, roles, framing, bounds, TLS remote-mode
|
||||||
|
requirements, events, and management methods are specified in
|
||||||
|
[`../../docs/grid-agent-control-plane.md`](../../docs/grid-agent-control-plane.md).
|
||||||
|
|||||||
@@ -19,8 +19,8 @@ use serde_json::{Map, Value, json};
|
|||||||
use std::collections::{BTreeMap, BTreeSet};
|
use std::collections::{BTreeMap, BTreeSet};
|
||||||
use std::error::Error;
|
use std::error::Error;
|
||||||
use std::fmt;
|
use std::fmt;
|
||||||
use std::sync::Arc;
|
|
||||||
use std::sync::atomic::{AtomicU64, Ordering};
|
use std::sync::atomic::{AtomicU64, Ordering};
|
||||||
|
use std::sync::{Arc, Mutex, MutexGuard};
|
||||||
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
|
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
|
||||||
use tokio::sync::{mpsc, oneshot, watch};
|
use tokio::sync::{mpsc, oneshot, watch};
|
||||||
use tokio::task::JoinHandle;
|
use tokio::task::JoinHandle;
|
||||||
@@ -36,6 +36,7 @@ pub const CURRENT_POSE_TOOL: &str = "behavior_current_pose";
|
|||||||
|
|
||||||
const WALK_POLL: Duration = Duration::from_millis(250);
|
const WALK_POLL: Duration = Duration::from_millis(250);
|
||||||
const TURN_INTERVAL: Duration = Duration::from_millis(40);
|
const TURN_INTERVAL: Duration = Duration::from_millis(40);
|
||||||
|
const MAX_ACTION_ID_BYTES: usize = 96;
|
||||||
const MAX_TURN_STEP_DEGREES: f64 = 30.0;
|
const MAX_TURN_STEP_DEGREES: f64 = 30.0;
|
||||||
const ARRIVAL_METERS: f64 = 0.5;
|
const ARRIVAL_METERS: f64 = 0.5;
|
||||||
const STUCK_PROGRESS_METERS: f64 = 0.05;
|
const STUCK_PROGRESS_METERS: f64 = 0.05;
|
||||||
@@ -196,6 +197,7 @@ pub enum BehaviorObservation {
|
|||||||
trigger: BehaviorTrigger,
|
trigger: BehaviorTrigger,
|
||||||
},
|
},
|
||||||
Action {
|
Action {
|
||||||
|
action_id: String,
|
||||||
generation: Option<u64>,
|
generation: Option<u64>,
|
||||||
region_id: Option<UUID>,
|
region_id: Option<UUID>,
|
||||||
action: String,
|
action: String,
|
||||||
@@ -243,15 +245,23 @@ impl BehaviorAction {
|
|||||||
}
|
}
|
||||||
|
|
||||||
struct ActionRequest {
|
struct ActionRequest {
|
||||||
|
action_id: String,
|
||||||
action: BehaviorAction,
|
action: BehaviorAction,
|
||||||
trigger: BehaviorTrigger,
|
trigger: BehaviorTrigger,
|
||||||
policy: BehaviorPolicyResult,
|
policy: BehaviorPolicyResult,
|
||||||
reply: oneshot::Sender<Result<Value, BehaviorError>>,
|
reply: oneshot::Sender<Result<Value, BehaviorError>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Default)]
|
||||||
|
struct ActionRegistry {
|
||||||
|
active: BTreeSet<String>,
|
||||||
|
cancelled: BTreeSet<String>,
|
||||||
|
}
|
||||||
|
|
||||||
enum Command {
|
enum Command {
|
||||||
Action(ActionRequest),
|
Action(ActionRequest),
|
||||||
Attention {
|
Attention {
|
||||||
|
action_id: String,
|
||||||
delivery_id: String,
|
delivery_id: String,
|
||||||
avatar_id: UUID,
|
avatar_id: UUID,
|
||||||
reply: oneshot::Sender<Result<(), BehaviorError>>,
|
reply: oneshot::Sender<Result<(), BehaviorError>>,
|
||||||
@@ -266,6 +276,10 @@ pub struct BehaviorIngress {
|
|||||||
ready: watch::Sender<Option<ReadyState>>,
|
ready: watch::Sender<Option<ReadyState>>,
|
||||||
paused: watch::Sender<bool>,
|
paused: watch::Sender<bool>,
|
||||||
emergency: watch::Sender<bool>,
|
emergency: watch::Sender<bool>,
|
||||||
|
action_cancel: watch::Sender<u64>,
|
||||||
|
actions: Arc<Mutex<ActionRegistry>>,
|
||||||
|
next_action: Arc<AtomicU64>,
|
||||||
|
action_capacity: usize,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl fmt::Debug for BehaviorIngress {
|
impl fmt::Debug for BehaviorIngress {
|
||||||
@@ -313,6 +327,28 @@ impl BehaviorIngress {
|
|||||||
let _ = self.emergency.send(false);
|
let _ = self.emergency.send(false);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Cancels one exact queued or executing behavior action ID.
|
||||||
|
pub fn cancel_action(&self, action_id: &str) -> Result<bool, BehaviorError> {
|
||||||
|
if !valid_action_id(action_id) {
|
||||||
|
return Err(BehaviorError::InvalidArguments);
|
||||||
|
}
|
||||||
|
let cancelled = {
|
||||||
|
let mut actions = lock_actions(&self.actions);
|
||||||
|
if actions.active.contains(action_id) {
|
||||||
|
actions.cancelled.insert(action_id.to_owned());
|
||||||
|
true
|
||||||
|
} else {
|
||||||
|
false
|
||||||
|
}
|
||||||
|
};
|
||||||
|
if cancelled {
|
||||||
|
self.action_cancel.send_modify(|generation| {
|
||||||
|
*generation = generation.saturating_add(1);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
Ok(cancelled)
|
||||||
|
}
|
||||||
|
|
||||||
/// Marks a separately authorized scheduler task as roaming. The controller
|
/// Marks a separately authorized scheduler task as roaming. The controller
|
||||||
/// itself never invents a roaming route or exposes follow/wander behavior.
|
/// itself never invents a roaming route or exposes follow/wander behavior.
|
||||||
pub fn set_roaming(&self, roaming: bool) -> Result<(), BehaviorError> {
|
pub fn set_roaming(&self, roaming: bool) -> Result<(), BehaviorError> {
|
||||||
@@ -323,20 +359,30 @@ impl BehaviorIngress {
|
|||||||
|
|
||||||
async fn request(
|
async fn request(
|
||||||
&self,
|
&self,
|
||||||
|
action_id: String,
|
||||||
action: BehaviorAction,
|
action: BehaviorAction,
|
||||||
trigger: BehaviorTrigger,
|
trigger: BehaviorTrigger,
|
||||||
policy: BehaviorPolicyResult,
|
policy: BehaviorPolicyResult,
|
||||||
) -> Result<Value, BehaviorError> {
|
) -> Result<Value, BehaviorError> {
|
||||||
|
if !self.register_action(&action_id) {
|
||||||
|
return Err(BehaviorError::InvalidArguments);
|
||||||
|
}
|
||||||
let (reply, receive) = oneshot::channel();
|
let (reply, receive) = oneshot::channel();
|
||||||
self.commands
|
if self
|
||||||
|
.commands
|
||||||
.send(Command::Action(ActionRequest {
|
.send(Command::Action(ActionRequest {
|
||||||
|
action_id: action_id.clone(),
|
||||||
action,
|
action,
|
||||||
trigger,
|
trigger,
|
||||||
policy,
|
policy,
|
||||||
reply,
|
reply,
|
||||||
}))
|
}))
|
||||||
.await
|
.await
|
||||||
.map_err(|_| BehaviorError::QueueClosed)?;
|
.is_err()
|
||||||
|
{
|
||||||
|
self.unregister_action(&action_id);
|
||||||
|
return Err(BehaviorError::QueueClosed);
|
||||||
|
}
|
||||||
receive.await.map_err(|_| BehaviorError::QueueClosed)?
|
receive.await.map_err(|_| BehaviorError::QueueClosed)?
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -345,17 +391,44 @@ impl BehaviorIngress {
|
|||||||
delivery_id: String,
|
delivery_id: String,
|
||||||
avatar_id: UUID,
|
avatar_id: UUID,
|
||||||
) -> Result<(), BehaviorError> {
|
) -> Result<(), BehaviorError> {
|
||||||
|
let action_id = format!(
|
||||||
|
"attention-{}",
|
||||||
|
self.next_action.fetch_add(1, Ordering::Relaxed)
|
||||||
|
);
|
||||||
|
if !self.register_action(&action_id) {
|
||||||
|
return Err(BehaviorError::InvalidArguments);
|
||||||
|
}
|
||||||
let (reply, receive) = oneshot::channel();
|
let (reply, receive) = oneshot::channel();
|
||||||
self.commands
|
if self
|
||||||
|
.commands
|
||||||
.send(Command::Attention {
|
.send(Command::Attention {
|
||||||
|
action_id: action_id.clone(),
|
||||||
delivery_id,
|
delivery_id,
|
||||||
avatar_id,
|
avatar_id,
|
||||||
reply,
|
reply,
|
||||||
})
|
})
|
||||||
.await
|
.await
|
||||||
.map_err(|_| BehaviorError::QueueClosed)?;
|
.is_err()
|
||||||
|
{
|
||||||
|
self.unregister_action(&action_id);
|
||||||
|
return Err(BehaviorError::QueueClosed);
|
||||||
|
}
|
||||||
receive.await.map_err(|_| BehaviorError::QueueClosed)?
|
receive.await.map_err(|_| BehaviorError::QueueClosed)?
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn register_action(&self, action_id: &str) -> bool {
|
||||||
|
if !valid_action_id(action_id) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
let mut actions = lock_actions(&self.actions);
|
||||||
|
actions.active.len() < self.action_capacity && actions.active.insert(action_id.to_owned())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn unregister_action(&self, action_id: &str) {
|
||||||
|
let mut actions = lock_actions(&self.actions);
|
||||||
|
actions.active.remove(action_id);
|
||||||
|
actions.cancelled.remove(action_id);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl ResponsePacer for BehaviorIngress {
|
impl ResponsePacer for BehaviorIngress {
|
||||||
@@ -485,12 +558,18 @@ impl BehaviorController {
|
|||||||
let (ready, ready_rx) = watch::channel(None);
|
let (ready, ready_rx) = watch::channel(None);
|
||||||
let (paused, paused_rx) = watch::channel(false);
|
let (paused, paused_rx) = watch::channel(false);
|
||||||
let (emergency, emergency_rx) = watch::channel(false);
|
let (emergency, emergency_rx) = watch::channel(false);
|
||||||
|
let (action_cancel, action_cancel_rx) = watch::channel(0_u64);
|
||||||
|
let actions = Arc::new(Mutex::new(ActionRegistry::default()));
|
||||||
let (observations, observation_rx) = mpsc::channel(self.observation_capacity);
|
let (observations, observation_rx) = mpsc::channel(self.observation_capacity);
|
||||||
let ingress = BehaviorIngress {
|
let ingress = BehaviorIngress {
|
||||||
commands,
|
commands,
|
||||||
ready,
|
ready,
|
||||||
paused,
|
paused,
|
||||||
emergency,
|
emergency,
|
||||||
|
action_cancel,
|
||||||
|
actions: actions.clone(),
|
||||||
|
next_action: Arc::new(AtomicU64::new(1)),
|
||||||
|
action_capacity: self.queue_capacity,
|
||||||
};
|
};
|
||||||
let shutdown_timeout = self.shutdown_timeout;
|
let shutdown_timeout = self.shutdown_timeout;
|
||||||
let task = tokio::spawn(run_actor(
|
let task = tokio::spawn(run_actor(
|
||||||
@@ -499,6 +578,8 @@ impl BehaviorController {
|
|||||||
ready_rx,
|
ready_rx,
|
||||||
paused_rx,
|
paused_rx,
|
||||||
emergency_rx,
|
emergency_rx,
|
||||||
|
action_cancel_rx,
|
||||||
|
actions,
|
||||||
observations,
|
observations,
|
||||||
));
|
));
|
||||||
BehaviorHandle {
|
BehaviorHandle {
|
||||||
@@ -547,13 +628,15 @@ impl BehaviorRandom for SystemBehaviorRandom {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(clippy::too_many_lines)] // Lifecycle, preemption, idle, and commands share one ordered owner.
|
#[allow(clippy::too_many_arguments, clippy::too_many_lines)] // One ordered lifecycle owner.
|
||||||
async fn run_actor(
|
async fn run_actor(
|
||||||
controller: BehaviorController,
|
controller: BehaviorController,
|
||||||
mut commands: mpsc::Receiver<Command>,
|
mut commands: mpsc::Receiver<Command>,
|
||||||
mut ready: watch::Receiver<Option<ReadyState>>,
|
mut ready: watch::Receiver<Option<ReadyState>>,
|
||||||
mut paused: watch::Receiver<bool>,
|
mut paused: watch::Receiver<bool>,
|
||||||
mut emergency: watch::Receiver<bool>,
|
mut emergency: watch::Receiver<bool>,
|
||||||
|
action_cancel: watch::Receiver<u64>,
|
||||||
|
actions: Arc<Mutex<ActionRegistry>>,
|
||||||
observations: mpsc::Sender<BehaviorObservation>,
|
observations: mpsc::Sender<BehaviorObservation>,
|
||||||
) {
|
) {
|
||||||
let mut mode = BehaviorMode::Offline;
|
let mut mode = BehaviorMode::Offline;
|
||||||
@@ -616,8 +699,16 @@ async fn run_actor(
|
|||||||
() = &mut idle, if controller.settings.idle_look_enabled => {
|
() = &mut idle, if controller.settings.idle_look_enabled => {
|
||||||
if mode == BehaviorMode::Available {
|
if mode == BehaviorMode::Available {
|
||||||
let (reply, _) = oneshot::channel();
|
let (reply, _) = oneshot::channel();
|
||||||
let request = ActionRequest { action: BehaviorAction::LookAround, trigger: BehaviorTrigger::IdleTimer, policy: BehaviorPolicyResult::InternalIdle, reply };
|
let action_id = format!("idle-{}", unix_millis_now());
|
||||||
execute_request(&controller, request, &mut mode, &mut last_action, &ready, &paused, &emergency, &observations).await;
|
let registered = {
|
||||||
|
let mut registry = lock_actions(&actions);
|
||||||
|
registry.active.len() < controller.queue_capacity
|
||||||
|
&& registry.active.insert(action_id.clone())
|
||||||
|
};
|
||||||
|
if registered {
|
||||||
|
let request = ActionRequest { action_id, action: BehaviorAction::LookAround, trigger: BehaviorTrigger::IdleTimer, policy: BehaviorPolicyResult::InternalIdle, reply };
|
||||||
|
execute_request(&controller, request, &mut mode, &mut last_action, &ready, &paused, &emergency, &action_cancel, &actions, &observations).await;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
idle.as_mut().reset(tokio::time::Instant::now() + controller.settings.idle_interval);
|
idle.as_mut().reset(tokio::time::Instant::now() + controller.settings.idle_interval);
|
||||||
}
|
}
|
||||||
@@ -642,9 +733,10 @@ async fn run_actor(
|
|||||||
};
|
};
|
||||||
transition(&observations, &mut mode, next, BehaviorTrigger::IdleTimer).await;
|
transition(&observations, &mut mode, next, BehaviorTrigger::IdleTimer).await;
|
||||||
}
|
}
|
||||||
Command::Action(request) => execute_request(&controller, request, &mut mode, &mut last_action, &ready, &paused, &emergency, &observations).await,
|
Command::Action(request) => execute_request(&controller, request, &mut mode, &mut last_action, &ready, &paused, &emergency, &action_cancel, &actions, &observations).await,
|
||||||
Command::Attention { delivery_id, avatar_id, reply } => {
|
Command::Attention { action_id, delivery_id, avatar_id, reply } => {
|
||||||
let request = ActionRequest {
|
let request = ActionRequest {
|
||||||
|
action_id,
|
||||||
action: BehaviorAction::FaceAvatar(avatar_id),
|
action: BehaviorAction::FaceAvatar(avatar_id),
|
||||||
trigger: BehaviorTrigger::PublicResponse { delivery_id, avatar_id },
|
trigger: BehaviorTrigger::PublicResponse { delivery_id, avatar_id },
|
||||||
policy: BehaviorPolicyResult::BuiltInAttention,
|
policy: BehaviorPolicyResult::BuiltInAttention,
|
||||||
@@ -652,7 +744,7 @@ async fn run_actor(
|
|||||||
};
|
};
|
||||||
let delay = random_duration(&controller, controller.settings.response_delay_min, controller.settings.response_delay_max);
|
let delay = random_duration(&controller, controller.settings.response_delay_min, controller.settings.response_delay_max);
|
||||||
tokio::time::sleep(delay).await;
|
tokio::time::sleep(delay).await;
|
||||||
execute_request(&controller, request, &mut mode, &mut last_action, &ready, &paused, &emergency, &observations).await;
|
execute_request(&controller, request, &mut mode, &mut last_action, &ready, &paused, &emergency, &action_cancel, &actions, &observations).await;
|
||||||
if mode == BehaviorMode::Engaged {
|
if mode == BehaviorMode::Engaged {
|
||||||
tokio::time::sleep(controller.settings.attention_dwell).await;
|
tokio::time::sleep(controller.settings.attention_dwell).await;
|
||||||
if !*paused.borrow() && ready.borrow().is_some() { transition(&observations, &mut mode, BehaviorMode::Available, BehaviorTrigger::IdleTimer).await; }
|
if !*paused.borrow() && ready.borrow().is_some() { transition(&observations, &mut mode, BehaviorMode::Available, BehaviorTrigger::IdleTimer).await; }
|
||||||
@@ -702,12 +794,17 @@ async fn execute_request(
|
|||||||
ready: &watch::Receiver<Option<ReadyState>>,
|
ready: &watch::Receiver<Option<ReadyState>>,
|
||||||
paused: &watch::Receiver<bool>,
|
paused: &watch::Receiver<bool>,
|
||||||
emergency: &watch::Receiver<bool>,
|
emergency: &watch::Receiver<bool>,
|
||||||
|
action_cancel: &watch::Receiver<u64>,
|
||||||
|
actions: &Arc<Mutex<ActionRegistry>>,
|
||||||
observations: &mpsc::Sender<BehaviorObservation>,
|
observations: &mpsc::Sender<BehaviorObservation>,
|
||||||
) {
|
) {
|
||||||
let started = Instant::now();
|
let started = Instant::now();
|
||||||
|
let action_id = request.action_id.clone();
|
||||||
let action_name = request.action.name().to_owned();
|
let action_name = request.action.name().to_owned();
|
||||||
let state = *ready.borrow();
|
let state = *ready.borrow();
|
||||||
let mut result = if *emergency.borrow() {
|
let mut result = if take_action_cancellation(actions, &action_id) {
|
||||||
|
Err(BehaviorError::Cancelled)
|
||||||
|
} else if *emergency.borrow() {
|
||||||
Err(BehaviorError::EmergencyStopped)
|
Err(BehaviorError::EmergencyStopped)
|
||||||
} else if *paused.borrow() {
|
} else if *paused.borrow() {
|
||||||
Err(BehaviorError::Paused)
|
Err(BehaviorError::Paused)
|
||||||
@@ -737,11 +834,15 @@ async fn execute_request(
|
|||||||
let mut action_ready = ready.clone();
|
let mut action_ready = ready.clone();
|
||||||
let mut action_paused = paused.clone();
|
let mut action_paused = paused.clone();
|
||||||
let mut action_emergency = emergency.clone();
|
let mut action_emergency = emergency.clone();
|
||||||
|
let action_cancel = action_cancel.clone();
|
||||||
|
let action_registry = actions.clone();
|
||||||
|
let cancelled_id = action_id.clone();
|
||||||
result = tokio::select! {
|
result = tokio::select! {
|
||||||
value = tokio::time::timeout(controller.settings.action_timeout, perform_action(controller, state, &request.action)) => value.unwrap_or(Err(BehaviorError::TimedOut)),
|
value = tokio::time::timeout(controller.settings.action_timeout, perform_action(controller, state, &request.action)) => value.unwrap_or(Err(BehaviorError::TimedOut)),
|
||||||
_ = action_ready.changed() => Err(BehaviorError::Cancelled),
|
_ = action_ready.changed() => Err(BehaviorError::Cancelled),
|
||||||
_ = action_paused.changed() => Err(BehaviorError::Paused),
|
_ = action_paused.changed() => Err(BehaviorError::Paused),
|
||||||
_ = action_emergency.changed() => Err(BehaviorError::EmergencyStopped),
|
_ = action_emergency.changed() => Err(BehaviorError::EmergencyStopped),
|
||||||
|
() = wait_for_action_cancellation(action_cancel, action_registry, cancelled_id) => Err(BehaviorError::Cancelled),
|
||||||
};
|
};
|
||||||
if result.is_err() && matches!(request.action, BehaviorAction::WalkShort { .. }) {
|
if result.is_err() && matches!(request.action, BehaviorAction::WalkShort { .. }) {
|
||||||
let _ = controller
|
let _ = controller
|
||||||
@@ -811,6 +912,7 @@ async fn execute_request(
|
|||||||
};
|
};
|
||||||
let _ = observations
|
let _ = observations
|
||||||
.send(BehaviorObservation::Action {
|
.send(BehaviorObservation::Action {
|
||||||
|
action_id: action_id.clone(),
|
||||||
generation: state.map(|value| value.generation),
|
generation: state.map(|value| value.generation),
|
||||||
region_id: state.map(|value| value.region_id),
|
region_id: state.map(|value| value.region_id),
|
||||||
action: action_name,
|
action: action_name,
|
||||||
@@ -821,6 +923,22 @@ async fn execute_request(
|
|||||||
})
|
})
|
||||||
.await;
|
.await;
|
||||||
let _ = request.reply.send(result);
|
let _ = request.reply.send(result);
|
||||||
|
unregister_action(actions, &action_id);
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn wait_for_action_cancellation(
|
||||||
|
mut signal: watch::Receiver<u64>,
|
||||||
|
actions: Arc<Mutex<ActionRegistry>>,
|
||||||
|
action_id: String,
|
||||||
|
) {
|
||||||
|
loop {
|
||||||
|
if take_action_cancellation(&actions, &action_id) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if signal.changed().await.is_err() {
|
||||||
|
std::future::pending::<()>().await;
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn perform_action(
|
async fn perform_action(
|
||||||
@@ -1086,6 +1204,38 @@ fn random_duration(
|
|||||||
minimum + Duration::from_millis(u64::try_from(offset).unwrap_or(u64::MAX))
|
minimum + Duration::from_millis(u64::try_from(offset).unwrap_or(u64::MAX))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn valid_action_id(value: &str) -> bool {
|
||||||
|
!value.is_empty()
|
||||||
|
&& value.len() <= MAX_ACTION_ID_BYTES
|
||||||
|
&& value.chars().all(|character| {
|
||||||
|
character.is_ascii_alphanumeric() || matches!(character, '-' | '_' | '.' | ':')
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn lock_actions(value: &Mutex<ActionRegistry>) -> MutexGuard<'_, ActionRegistry> {
|
||||||
|
value
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn take_action_cancellation(actions: &Mutex<ActionRegistry>, action_id: &str) -> bool {
|
||||||
|
lock_actions(actions).cancelled.remove(action_id)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn unregister_action(actions: &Mutex<ActionRegistry>, action_id: &str) {
|
||||||
|
let mut actions = lock_actions(actions);
|
||||||
|
actions.active.remove(action_id);
|
||||||
|
actions.cancelled.remove(action_id);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn unix_millis_now() -> u64 {
|
||||||
|
SystemTime::now()
|
||||||
|
.duration_since(UNIX_EPOCH)
|
||||||
|
.map_or(0, |duration| {
|
||||||
|
u64::try_from(duration.as_millis()).unwrap_or(u64::MAX)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
pub struct BehaviorBackend {
|
pub struct BehaviorBackend {
|
||||||
ingress: BehaviorIngress,
|
ingress: BehaviorIngress,
|
||||||
@@ -1136,7 +1286,7 @@ impl AuthorizedToolBackend for BehaviorBackend {
|
|||||||
let result = match parsed {
|
let result = match parsed {
|
||||||
Ok(behavior_action) => tokio::select! {
|
Ok(behavior_action) => tokio::select! {
|
||||||
() = cancellation.cancelled() => Err(BehaviorError::Cancelled),
|
() = cancellation.cancelled() => Err(BehaviorError::Cancelled),
|
||||||
value = self.ingress.request(behavior_action, BehaviorTrigger::AuthorizedTool { authorization_id, tool: action.call().name.as_str().to_owned() }, BehaviorPolicyResult::Authorized(authorization_id)) => value,
|
value = self.ingress.request(call_id.as_str().to_owned(), behavior_action, BehaviorTrigger::AuthorizedTool { authorization_id, tool: action.call().name.as_str().to_owned() }, BehaviorPolicyResult::Authorized(authorization_id)) => value,
|
||||||
},
|
},
|
||||||
Err(error) => Err(error),
|
Err(error) => Err(error),
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -453,6 +453,48 @@ async fn pause_preempts_stuck_walk_and_blocks_all_motion() {
|
|||||||
handle.shutdown().await.expect("shutdown");
|
handle.shutdown().await.expect("shutdown");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test(start_paused = true)]
|
||||||
|
async fn exact_action_id_cancels_only_the_matching_pending_action() {
|
||||||
|
let region = uuid(10);
|
||||||
|
let sink = Arc::new(FakeSink::new(1, region));
|
||||||
|
sink.set_stuck(true);
|
||||||
|
let mut config = settings();
|
||||||
|
config.settle_delay = Duration::ZERO;
|
||||||
|
let handle = BehaviorController::new(config.clone(), sink, 16, 32, Duration::from_secs(1))
|
||||||
|
.expect("controller")
|
||||||
|
.start();
|
||||||
|
let ingress = handle.ingress();
|
||||||
|
ingress.connected(1, region).expect("ready");
|
||||||
|
tokio::task::yield_now().await;
|
||||||
|
let backend = BehaviorBackend::new(ingress.clone());
|
||||||
|
let task = tokio::spawn({
|
||||||
|
let backend = backend.clone();
|
||||||
|
let config = config.clone();
|
||||||
|
async move {
|
||||||
|
run_tool(
|
||||||
|
&backend,
|
||||||
|
&config,
|
||||||
|
WALK_SHORT_TOOL,
|
||||||
|
json!({"heading_degrees":90.0,"distance_meters":2.0}),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
});
|
||||||
|
tokio::time::advance(Duration::from_millis(300)).await;
|
||||||
|
assert!(!ingress.cancel_action("unrelated-action").expect("cancel"));
|
||||||
|
assert!(ingress.cancel_action("behavior-call").expect("cancel"));
|
||||||
|
tokio::task::yield_now().await;
|
||||||
|
assert!(matches!(
|
||||||
|
task.await.expect("task"),
|
||||||
|
ToolCallOutcome::Rejected { .. }
|
||||||
|
));
|
||||||
|
assert!(matches!(
|
||||||
|
run_tool(&backend, &config, SIT_TOOL, json!({})).await,
|
||||||
|
ToolCallOutcome::Completed { .. }
|
||||||
|
));
|
||||||
|
handle.shutdown().await.expect("shutdown");
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test(start_paused = true)]
|
#[tokio::test(start_paused = true)]
|
||||||
async fn offline_and_emergency_stop_emit_zero_motion() {
|
async fn offline_and_emergency_stop_emit_zero_motion() {
|
||||||
let region = uuid(10);
|
let region = uuid(10);
|
||||||
|
|||||||
@@ -7,12 +7,15 @@ use std::collections::{BTreeMap, BTreeSet};
|
|||||||
use std::error::Error;
|
use std::error::Error;
|
||||||
use std::fmt;
|
use std::fmt;
|
||||||
use std::fs;
|
use std::fs;
|
||||||
|
use std::net::{IpAddr, Ipv4Addr, SocketAddr};
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
|
use std::sync::Arc;
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
use url::Url;
|
use url::Url;
|
||||||
|
|
||||||
const MAX_CONFIG_BYTES: u64 = 64 * 1024;
|
const MAX_CONFIG_BYTES: u64 = 64 * 1024;
|
||||||
const MAX_SECRET_BYTES: u64 = 16 * 1024;
|
const MAX_SECRET_BYTES: u64 = 16 * 1024;
|
||||||
|
const MAX_TLS_DER_BYTES: u64 = 1024 * 1024;
|
||||||
const MAX_AUTHORIZED_AVATARS: usize = 1_024;
|
const MAX_AUTHORIZED_AVATARS: usize = 1_024;
|
||||||
const MAX_QUEUE_CAPACITY: usize = 8_192;
|
const MAX_QUEUE_CAPACITY: usize = 8_192;
|
||||||
const MAX_BACKGROUND_TASKS: usize = 2;
|
const MAX_BACKGROUND_TASKS: usize = 2;
|
||||||
@@ -27,6 +30,11 @@ const ENV_GRID_AVATAR_NAME: &str = "METACRATE_AGENT_GRID_AVATAR_NAME";
|
|||||||
const ENV_GRID_PASSWORD: &str = "METACRATE_AGENT_GRID_PASSWORD";
|
const ENV_GRID_PASSWORD: &str = "METACRATE_AGENT_GRID_PASSWORD";
|
||||||
const ENV_GRID_PASSWORD_FILE: &str = "METACRATE_AGENT_GRID_PASSWORD_FILE";
|
const ENV_GRID_PASSWORD_FILE: &str = "METACRATE_AGENT_GRID_PASSWORD_FILE";
|
||||||
const ENV_AUTHORIZED_AVATARS: &str = "METACRATE_AGENT_AUTHORIZED_AVATAR_UUIDS";
|
const ENV_AUTHORIZED_AVATARS: &str = "METACRATE_AGENT_AUTHORIZED_AVATAR_UUIDS";
|
||||||
|
const ENV_CONTROL_LISTEN: &str = "METACRATE_AGENT_CONTROL_LISTEN";
|
||||||
|
const ENV_CONTROL_OPERATOR_TOKEN: &str = "METACRATE_AGENT_CONTROL_OPERATOR_TOKEN";
|
||||||
|
const ENV_CONTROL_OPERATOR_TOKEN_FILE: &str = "METACRATE_AGENT_CONTROL_OPERATOR_TOKEN_FILE";
|
||||||
|
const ENV_CONTROL_OBSERVER_TOKEN: &str = "METACRATE_AGENT_CONTROL_OBSERVER_TOKEN";
|
||||||
|
const ENV_CONTROL_OBSERVER_TOKEN_FILE: &str = "METACRATE_AGENT_CONTROL_OBSERVER_TOKEN_FILE";
|
||||||
|
|
||||||
/// Wrapper that never reveals its contents through `Debug` or `Display` and
|
/// Wrapper that never reveals its contents through `Debug` or `Display` and
|
||||||
/// deliberately does not implement serialization.
|
/// deliberately does not implement serialization.
|
||||||
@@ -247,6 +255,66 @@ pub struct ConversationSettings {
|
|||||||
pub limits: crate::conversation::ConversationLimits,
|
pub limits: crate::conversation::ConversationLimits,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Portable control-plane settings. Plain TCP is loopback-only; a remote
|
||||||
|
/// listener is admitted only with an explicit Rustls certificate and key.
|
||||||
|
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||||
|
pub struct ControlSettings {
|
||||||
|
pub listen: SocketAddr,
|
||||||
|
pub operator_token: Option<SecretString>,
|
||||||
|
pub observer_token: Option<SecretString>,
|
||||||
|
pub limits: crate::control_plane::ControlLimits,
|
||||||
|
pub remote_tls: Option<RemoteTlsSettings>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||||
|
pub struct RemoteTlsSettings {
|
||||||
|
pub certificate_der: PathBuf,
|
||||||
|
pub private_key_der: PathBuf,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl RemoteTlsSettings {
|
||||||
|
/// Loads bounded, regular DER files into a portable Rustls server config.
|
||||||
|
///
|
||||||
|
/// # Errors
|
||||||
|
///
|
||||||
|
/// Returns a configuration error when either file is unavailable, unsafe,
|
||||||
|
/// oversized, malformed, or does not form a valid certificate/key pair.
|
||||||
|
pub fn server_config(&self) -> Result<Arc<rustls::ServerConfig>, ConfigError> {
|
||||||
|
let certificate = read_bounded_regular_file(
|
||||||
|
"control.remote_tls_certificate_der",
|
||||||
|
&self.certificate_der,
|
||||||
|
MAX_TLS_DER_BYTES,
|
||||||
|
)?;
|
||||||
|
let private_key = read_bounded_regular_file(
|
||||||
|
"control.remote_tls_private_key_der",
|
||||||
|
&self.private_key_der,
|
||||||
|
MAX_TLS_DER_BYTES,
|
||||||
|
)?;
|
||||||
|
let private_key = rustls::pki_types::PrivateKeyDer::try_from(private_key)
|
||||||
|
.map_err(|_| ConfigError::InvalidControl)?;
|
||||||
|
let config = rustls::ServerConfig::builder()
|
||||||
|
.with_no_client_auth()
|
||||||
|
.with_single_cert(
|
||||||
|
vec![rustls::pki_types::CertificateDer::from(certificate)],
|
||||||
|
private_key,
|
||||||
|
)
|
||||||
|
.map_err(|_| ConfigError::InvalidControl)?;
|
||||||
|
Ok(Arc::new(config))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for ControlSettings {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self {
|
||||||
|
listen: SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 7943),
|
||||||
|
operator_token: None,
|
||||||
|
observer_token: None,
|
||||||
|
limits: crate::control_plane::ControlLimits::default(),
|
||||||
|
remote_tls: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Fully resolved configuration. It cannot be constructed without validation.
|
/// Fully resolved configuration. It cannot be constructed without validation.
|
||||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||||
pub struct AgentConfig {
|
pub struct AgentConfig {
|
||||||
@@ -261,6 +329,7 @@ pub struct AgentConfig {
|
|||||||
pub reconnect: crate::session::ReconnectPolicy,
|
pub reconnect: crate::session::ReconnectPolicy,
|
||||||
pub conversation: ConversationSettings,
|
pub conversation: ConversationSettings,
|
||||||
pub interaction: crate::interaction::InteractionSettings,
|
pub interaction: crate::interaction::InteractionSettings,
|
||||||
|
pub control: ControlSettings,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl AgentConfig {
|
impl AgentConfig {
|
||||||
@@ -300,6 +369,45 @@ impl AgentConfig {
|
|||||||
if !self.behavior.is_valid() {
|
if !self.behavior.is_valid() {
|
||||||
return Err(ConfigError::InvalidBehavior);
|
return Err(ConfigError::InvalidBehavior);
|
||||||
}
|
}
|
||||||
|
if (!self.control.listen.ip().is_loopback() && self.control.remote_tls.is_none())
|
||||||
|
|| !self.control.limits.is_valid()
|
||||||
|
{
|
||||||
|
return Err(ConfigError::InvalidControl);
|
||||||
|
}
|
||||||
|
if self.mode == OperatingMode::SplitService && self.control.operator_token.is_none() {
|
||||||
|
return Err(ConfigError::Missing {
|
||||||
|
field: "control.operator_token",
|
||||||
|
required_for: "split service mode",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if let Some(operator) = &self.control.operator_token
|
||||||
|
&& (self
|
||||||
|
.control
|
||||||
|
.observer_token
|
||||||
|
.as_ref()
|
||||||
|
.is_some_and(|observer| observer == operator)
|
||||||
|
|| operator == &self.llm.api_key
|
||||||
|
|| self
|
||||||
|
.grid
|
||||||
|
.as_ref()
|
||||||
|
.is_some_and(|grid| operator == &grid.password))
|
||||||
|
{
|
||||||
|
return Err(ConfigError::InvalidControl);
|
||||||
|
}
|
||||||
|
if self
|
||||||
|
.control
|
||||||
|
.observer_token
|
||||||
|
.as_ref()
|
||||||
|
.is_some_and(|observer| {
|
||||||
|
observer == &self.llm.api_key
|
||||||
|
|| self
|
||||||
|
.grid
|
||||||
|
.as_ref()
|
||||||
|
.is_some_and(|grid| observer == &grid.password)
|
||||||
|
})
|
||||||
|
{
|
||||||
|
return Err(ConfigError::InvalidControl);
|
||||||
|
}
|
||||||
if self.mode != OperatingMode::OfflineFake && self.grid.is_none() {
|
if self.mode != OperatingMode::OfflineFake && self.grid.is_none() {
|
||||||
return Err(ConfigError::Missing {
|
return Err(ConfigError::Missing {
|
||||||
field: "grid",
|
field: "grid",
|
||||||
@@ -476,6 +584,7 @@ pub enum ConfigError {
|
|||||||
InvalidConversationMemory,
|
InvalidConversationMemory,
|
||||||
InvalidInteraction,
|
InvalidInteraction,
|
||||||
InvalidBehavior,
|
InvalidBehavior,
|
||||||
|
InvalidControl,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl fmt::Display for ConfigError {
|
impl fmt::Display for ConfigError {
|
||||||
@@ -531,6 +640,9 @@ impl fmt::Display for ConfigError {
|
|||||||
}
|
}
|
||||||
Self::InvalidInteraction => formatter.write_str("invalid interaction bounds"),
|
Self::InvalidInteraction => formatter.write_str("invalid interaction bounds"),
|
||||||
Self::InvalidBehavior => formatter.write_str("invalid embodied behavior bounds"),
|
Self::InvalidBehavior => formatter.write_str("invalid embodied behavior bounds"),
|
||||||
|
Self::InvalidControl => formatter.write_str(
|
||||||
|
"invalid control plane: use distinct non-credential tokens and loopback TCP or an explicit TLS remote listener",
|
||||||
|
),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -553,6 +665,7 @@ struct FileConfig {
|
|||||||
reconnect: RawReconnect,
|
reconnect: RawReconnect,
|
||||||
conversation: RawConversation,
|
conversation: RawConversation,
|
||||||
interaction: RawInteraction,
|
interaction: RawInteraction,
|
||||||
|
control: RawControl,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone, Default, Deserialize)]
|
#[derive(Clone, Default, Deserialize)]
|
||||||
@@ -575,6 +688,18 @@ struct RawGrid {
|
|||||||
struct RawSecretFiles {
|
struct RawSecretFiles {
|
||||||
llm_api_key: Option<PathBuf>,
|
llm_api_key: Option<PathBuf>,
|
||||||
grid_password: Option<PathBuf>,
|
grid_password: Option<PathBuf>,
|
||||||
|
control_operator_token: Option<PathBuf>,
|
||||||
|
control_observer_token: Option<PathBuf>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Default, Deserialize)]
|
||||||
|
#[serde(default, deny_unknown_fields)]
|
||||||
|
struct RawControl {
|
||||||
|
listen: Option<String>,
|
||||||
|
operator_token: Option<String>,
|
||||||
|
observer_token: Option<String>,
|
||||||
|
remote_tls_certificate_der: Option<PathBuf>,
|
||||||
|
remote_tls_private_key_der: Option<PathBuf>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone, Default, Deserialize)]
|
#[derive(Clone, Default, Deserialize)]
|
||||||
@@ -932,6 +1057,57 @@ fn resolve<E: Environment>(
|
|||||||
.max_debounce_fragments
|
.max_debounce_fragments
|
||||||
.unwrap_or(interaction_defaults.max_debounce_fragments),
|
.unwrap_or(interaction_defaults.max_debounce_fragments),
|
||||||
};
|
};
|
||||||
|
let control_defaults = ControlSettings::default();
|
||||||
|
let listen_text = environment
|
||||||
|
.get(ENV_CONTROL_LISTEN)
|
||||||
|
.or(raw.control.listen)
|
||||||
|
.unwrap_or_else(|| control_defaults.listen.to_string());
|
||||||
|
let listen = listen_text
|
||||||
|
.parse::<SocketAddr>()
|
||||||
|
.map_err(|_| ConfigError::InvalidControl)?;
|
||||||
|
let operator_token = optional_secret_from_layers(
|
||||||
|
environment.get(ENV_CONTROL_OPERATOR_TOKEN),
|
||||||
|
environment
|
||||||
|
.get(ENV_CONTROL_OPERATOR_TOKEN_FILE)
|
||||||
|
.map(PathBuf::from),
|
||||||
|
raw.control.operator_token,
|
||||||
|
raw.secret_files.control_operator_token,
|
||||||
|
base,
|
||||||
|
"control.operator_token",
|
||||||
|
)?;
|
||||||
|
let observer_token = optional_secret_from_layers(
|
||||||
|
environment.get(ENV_CONTROL_OBSERVER_TOKEN),
|
||||||
|
environment
|
||||||
|
.get(ENV_CONTROL_OBSERVER_TOKEN_FILE)
|
||||||
|
.map(PathBuf::from),
|
||||||
|
raw.control.observer_token,
|
||||||
|
raw.secret_files.control_observer_token,
|
||||||
|
base,
|
||||||
|
"control.observer_token",
|
||||||
|
)?;
|
||||||
|
let remote_tls = match (
|
||||||
|
raw.control.remote_tls_certificate_der,
|
||||||
|
raw.control.remote_tls_private_key_der,
|
||||||
|
) {
|
||||||
|
(None, None) => None,
|
||||||
|
(Some(certificate_der), Some(private_key_der)) => Some(RemoteTlsSettings {
|
||||||
|
certificate_der: resolve_path(base, &certificate_der),
|
||||||
|
private_key_der: resolve_path(base, &private_key_der),
|
||||||
|
}),
|
||||||
|
_ => return Err(ConfigError::InvalidControl),
|
||||||
|
};
|
||||||
|
let control = ControlSettings {
|
||||||
|
listen,
|
||||||
|
operator_token,
|
||||||
|
observer_token,
|
||||||
|
limits: crate::control_plane::ControlLimits {
|
||||||
|
command_queue: limits.control_queue,
|
||||||
|
request_timeout: timeouts.request.min(Duration::from_mins(2)),
|
||||||
|
write_timeout: timeouts.shutdown.min(Duration::from_secs(30)),
|
||||||
|
..crate::control_plane::ControlLimits::default()
|
||||||
|
},
|
||||||
|
remote_tls,
|
||||||
|
};
|
||||||
|
|
||||||
let config = AgentConfig {
|
let config = AgentConfig {
|
||||||
mode,
|
mode,
|
||||||
@@ -996,6 +1172,7 @@ fn resolve<E: Environment>(
|
|||||||
reconnect,
|
reconnect,
|
||||||
conversation,
|
conversation,
|
||||||
interaction,
|
interaction,
|
||||||
|
control,
|
||||||
};
|
};
|
||||||
config.validate()?;
|
config.validate()?;
|
||||||
Ok(config)
|
Ok(config)
|
||||||
@@ -1064,6 +1241,28 @@ fn secret_from_layers(
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn optional_secret_from_layers(
|
||||||
|
environment_value: Option<String>,
|
||||||
|
environment_file: Option<PathBuf>,
|
||||||
|
file_value: Option<String>,
|
||||||
|
file_path: Option<PathBuf>,
|
||||||
|
base: &Path,
|
||||||
|
field: &'static str,
|
||||||
|
) -> Result<Option<SecretString>, ConfigError> {
|
||||||
|
if let Some(value) = environment_value {
|
||||||
|
return SecretString::new(field, value).map(Some);
|
||||||
|
}
|
||||||
|
if let Some(path) = environment_file {
|
||||||
|
return read_secret(field, &resolve_path(base, &path)).map(Some);
|
||||||
|
}
|
||||||
|
if let Some(path) = file_path {
|
||||||
|
return read_secret(field, &resolve_path(base, &path)).map(Some);
|
||||||
|
}
|
||||||
|
file_value
|
||||||
|
.map(|value| SecretString::new(field, value))
|
||||||
|
.transpose()
|
||||||
|
}
|
||||||
|
|
||||||
fn resolve_path(base: &Path, path: &Path) -> PathBuf {
|
fn resolve_path(base: &Path, path: &Path) -> PathBuf {
|
||||||
if path.is_absolute() {
|
if path.is_absolute() {
|
||||||
path.to_owned()
|
path.to_owned()
|
||||||
@@ -1437,6 +1636,50 @@ mod tests {
|
|||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn split_control_requires_distinct_tokens_and_tls_for_remote_bind() {
|
||||||
|
let split = MapEnvironment::from_pairs([
|
||||||
|
(ENV_SPLIT, "true"),
|
||||||
|
(ENV_LLM_ENDPOINT, "https://llm.invalid/chat"),
|
||||||
|
(ENV_LLM_API_KEY, "llm-key"),
|
||||||
|
(ENV_GRID_LOGIN_URL, "https://grid.invalid/login"),
|
||||||
|
(ENV_GRID_AVATAR_NAME, "Control Agent"),
|
||||||
|
(ENV_GRID_PASSWORD, "grid-password"),
|
||||||
|
(ENV_CONTROL_OPERATOR_TOKEN, "operator-capability"),
|
||||||
|
(ENV_CONTROL_OBSERVER_TOKEN, "observer-capability"),
|
||||||
|
]);
|
||||||
|
let config = ConfigLoader::new()
|
||||||
|
.with_environment(split.clone())
|
||||||
|
.load()
|
||||||
|
.expect("bounded loopback split control");
|
||||||
|
assert_eq!(config.mode, OperatingMode::SplitService);
|
||||||
|
let diagnostic = format!("{config:?}");
|
||||||
|
for secret in [
|
||||||
|
"operator-capability",
|
||||||
|
"observer-capability",
|
||||||
|
"llm-key",
|
||||||
|
"grid-password",
|
||||||
|
] {
|
||||||
|
assert!(!diagnostic.contains(secret));
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut reused = split.clone();
|
||||||
|
reused.insert(ENV_CONTROL_OPERATOR_TOKEN, "llm-key");
|
||||||
|
assert_eq!(
|
||||||
|
ConfigLoader::new().with_environment(reused).load(),
|
||||||
|
Err(ConfigError::InvalidControl)
|
||||||
|
);
|
||||||
|
|
||||||
|
let mut remote_plaintext = split;
|
||||||
|
remote_plaintext.insert(ENV_CONTROL_LISTEN, "0.0.0.0:7943");
|
||||||
|
assert_eq!(
|
||||||
|
ConfigLoader::new()
|
||||||
|
.with_environment(remote_plaintext)
|
||||||
|
.load(),
|
||||||
|
Err(ConfigError::InvalidControl)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn secret_file_is_bounded_regular_and_trailing_newline_is_removed() {
|
fn secret_file_is_bounded_regular_and_trailing_newline_is_removed() {
|
||||||
let secret = temporary_file("api-key", "secret-from-file\n");
|
let secret = temporary_file("api-key", "secret-from-file\n");
|
||||||
|
|||||||
1780
crates/metacrate-grid-agent/src/control_plane.rs
Normal file
1780
crates/metacrate-grid-agent/src/control_plane.rs
Normal file
File diff suppressed because it is too large
Load Diff
746
crates/metacrate-grid-agent/src/control_plane_tests.rs
Normal file
746
crates/metacrate-grid-agent/src/control_plane_tests.rs
Normal file
@@ -0,0 +1,746 @@
|
|||||||
|
use crate::config::SecretString;
|
||||||
|
use crate::control_plane::*;
|
||||||
|
use libremetaverse_types::compat::CancellationToken;
|
||||||
|
use serde_json::json;
|
||||||
|
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
||||||
|
use std::sync::{Arc, Mutex};
|
||||||
|
use std::time::Duration;
|
||||||
|
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||||
|
use tokio::net::TcpStream;
|
||||||
|
|
||||||
|
#[derive(Default)]
|
||||||
|
struct FakeTarget {
|
||||||
|
mutations: Mutex<Vec<String>>,
|
||||||
|
shutdown: AtomicBool,
|
||||||
|
active: AtomicUsize,
|
||||||
|
maximum_active: AtomicUsize,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl FakeTarget {
|
||||||
|
fn mutation_count(&self) -> usize {
|
||||||
|
self.mutations.lock().expect("mutations").len()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ControlTarget for FakeTarget {
|
||||||
|
#[allow(clippy::too_many_lines)] // Exhaustive fake covers every response family.
|
||||||
|
fn execute(
|
||||||
|
&self,
|
||||||
|
context: ControlContext,
|
||||||
|
request: ControlRequest,
|
||||||
|
cancellation: CancellationToken,
|
||||||
|
) -> ControlFuture<'_> {
|
||||||
|
Box::pin(async move {
|
||||||
|
let active = self.active.fetch_add(1, Ordering::AcqRel) + 1;
|
||||||
|
self.maximum_active.fetch_max(active, Ordering::AcqRel);
|
||||||
|
let result = match request {
|
||||||
|
ControlRequest::Health => Ok(ControlPayload::Health(HealthView {
|
||||||
|
service_state: "running".into(),
|
||||||
|
ready: true,
|
||||||
|
uptime_seconds: 42,
|
||||||
|
protocol_version: CONTROL_PROTOCOL_VERSION,
|
||||||
|
})),
|
||||||
|
ControlRequest::Runtime => Ok(ControlPayload::Runtime(RuntimeView {
|
||||||
|
grid_state: "online".into(),
|
||||||
|
generation: 7,
|
||||||
|
transport_connected: true,
|
||||||
|
agent_ready: true,
|
||||||
|
region_id: Some("00000000-0000-4000-8000-000000000001".into()),
|
||||||
|
region_name: Some("Safe Region".into()),
|
||||||
|
position: Some([128.0, 128.0, 24.0]),
|
||||||
|
behavior_mode: "available".into(),
|
||||||
|
control_queue_used: 1,
|
||||||
|
control_queue_capacity: 64,
|
||||||
|
budget_tool_calls_used: 3,
|
||||||
|
budget_movement_millimeters_used: 2_000,
|
||||||
|
})),
|
||||||
|
ControlRequest::ListSessions { page } => Ok(ControlPayload::Sessions(page_values(
|
||||||
|
&page,
|
||||||
|
(0..7)
|
||||||
|
.map(|index| SessionMetadataView {
|
||||||
|
session_id: format!("session-{index}"),
|
||||||
|
avatar_id: format!("00000000-0000-4000-8000-{index:012x}"),
|
||||||
|
channel: "direct_im".into(),
|
||||||
|
created_unix_millis: 10,
|
||||||
|
last_active_unix_millis: 20,
|
||||||
|
turns: 2,
|
||||||
|
bytes: 32,
|
||||||
|
})
|
||||||
|
.collect(),
|
||||||
|
))),
|
||||||
|
ControlRequest::ListScheduledJobs { page } => {
|
||||||
|
Ok(ControlPayload::ScheduledJobs(page_values(
|
||||||
|
&page,
|
||||||
|
vec![ScheduledJobView {
|
||||||
|
job_id: "roam-1".into(),
|
||||||
|
kind: "landmark_roam".into(),
|
||||||
|
enabled: true,
|
||||||
|
next_run_unix_millis: Some(50),
|
||||||
|
}],
|
||||||
|
)))
|
||||||
|
}
|
||||||
|
ControlRequest::ListPendingApprovals { page } => {
|
||||||
|
Ok(ControlPayload::PendingApprovals(page_values(
|
||||||
|
&page,
|
||||||
|
vec![PendingApprovalView {
|
||||||
|
approval_id: 1,
|
||||||
|
tool: "build".into(),
|
||||||
|
principal: "operator:local".into(),
|
||||||
|
expires_unix_seconds: 99,
|
||||||
|
movement_millimeters: 0,
|
||||||
|
inventory_operations: 0,
|
||||||
|
build_prims: 1,
|
||||||
|
}],
|
||||||
|
)))
|
||||||
|
}
|
||||||
|
ControlRequest::ListAuditEvents { page } => {
|
||||||
|
Ok(ControlPayload::AuditEvents(page_values(
|
||||||
|
&page,
|
||||||
|
vec![AuditEventView {
|
||||||
|
sequence: 1,
|
||||||
|
unix_millis: 1,
|
||||||
|
principal: "operator:local".into(),
|
||||||
|
operation: "pause_autonomy".into(),
|
||||||
|
outcome: "completed".into(),
|
||||||
|
authorization_id: None,
|
||||||
|
}],
|
||||||
|
)))
|
||||||
|
}
|
||||||
|
ControlRequest::InjectOperatorMessage { message } if message == "wait" => {
|
||||||
|
cancellation.cancelled().await;
|
||||||
|
Err(ControlError {
|
||||||
|
code: ControlErrorCode::Cancelled,
|
||||||
|
message: "operation cancelled",
|
||||||
|
retryable: false,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
ControlRequest::GracefulShutdown => {
|
||||||
|
self.shutdown.store(true, Ordering::Release);
|
||||||
|
self.mutations
|
||||||
|
.lock()
|
||||||
|
.expect("mutations")
|
||||||
|
.push("graceful_shutdown".into());
|
||||||
|
Ok(ControlPayload::Completed)
|
||||||
|
}
|
||||||
|
other => {
|
||||||
|
assert_eq!(context.role, ControlRole::Operator);
|
||||||
|
self.mutations
|
||||||
|
.lock()
|
||||||
|
.expect("mutations")
|
||||||
|
.push(format!("{other:?}"));
|
||||||
|
Ok(ControlPayload::Completed)
|
||||||
|
}
|
||||||
|
};
|
||||||
|
self.active.fetch_sub(1, Ordering::AcqRel);
|
||||||
|
result
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn page_values<T>(page: &PageRequest, values: Vec<T>) -> Page<T> {
|
||||||
|
let start = usize::try_from(page.cursor.unwrap_or(0))
|
||||||
|
.expect("cursor")
|
||||||
|
.min(values.len());
|
||||||
|
let end = start
|
||||||
|
.saturating_add(usize::from(page.limit))
|
||||||
|
.min(values.len());
|
||||||
|
let next_cursor = (end < values.len()).then(|| u64::try_from(end).expect("cursor"));
|
||||||
|
Page {
|
||||||
|
items: values.into_iter().skip(start).take(end - start).collect(),
|
||||||
|
next_cursor,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn token(value: &str) -> SecretString {
|
||||||
|
SecretString::new("control.test_token", value).expect("token")
|
||||||
|
}
|
||||||
|
|
||||||
|
fn limits() -> ControlLimits {
|
||||||
|
ControlLimits {
|
||||||
|
request_timeout: Duration::from_secs(5),
|
||||||
|
idle_timeout: Duration::from_secs(5),
|
||||||
|
write_timeout: Duration::from_secs(1),
|
||||||
|
..ControlLimits::default()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn plane(target: Arc<FakeTarget>, configured: ControlLimits) -> Arc<ControlPlane> {
|
||||||
|
ControlPlane::new(
|
||||||
|
target,
|
||||||
|
token("operator-secret"),
|
||||||
|
Some(token("observer-secret")),
|
||||||
|
configured,
|
||||||
|
)
|
||||||
|
.expect("plane")
|
||||||
|
}
|
||||||
|
|
||||||
|
fn request(id: &str, request: ControlRequest) -> ControlRequestEnvelope {
|
||||||
|
ControlRequestEnvelope {
|
||||||
|
version: CONTROL_PROTOCOL_VERSION,
|
||||||
|
request_id: id.into(),
|
||||||
|
request,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
enum Client {
|
||||||
|
InProcess(InProcessControlClient),
|
||||||
|
Tcp(TcpControlClient),
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Client {
|
||||||
|
async fn request(&self, envelope: ControlRequestEnvelope) -> ControlResponseEnvelope {
|
||||||
|
match self {
|
||||||
|
Self::InProcess(client) => client.request(envelope).await,
|
||||||
|
Self::Tcp(client) => client.request(envelope).await.expect("TCP response"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
enum Transport {
|
||||||
|
InProcess,
|
||||||
|
Tcp,
|
||||||
|
}
|
||||||
|
|
||||||
|
struct Fixture {
|
||||||
|
client: Client,
|
||||||
|
plane: Arc<ControlPlane>,
|
||||||
|
target: Arc<FakeTarget>,
|
||||||
|
server: Option<TcpControlServer>,
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn fixture(transport: Transport, role: ControlRole, configured: ControlLimits) -> Fixture {
|
||||||
|
let target = Arc::new(FakeTarget::default());
|
||||||
|
let plane = plane(target.clone(), configured);
|
||||||
|
let value = if role == ControlRole::Operator {
|
||||||
|
"operator-secret"
|
||||||
|
} else {
|
||||||
|
"observer-secret"
|
||||||
|
};
|
||||||
|
match transport {
|
||||||
|
Transport::InProcess => Fixture {
|
||||||
|
client: Client::InProcess(plane.connect(value).expect("connect")),
|
||||||
|
plane,
|
||||||
|
target,
|
||||||
|
server: None,
|
||||||
|
},
|
||||||
|
Transport::Tcp => {
|
||||||
|
let server = TcpControlServer::bind(
|
||||||
|
plane.clone(),
|
||||||
|
TcpControlConfig {
|
||||||
|
listen: "127.0.0.1:0".parse().expect("address"),
|
||||||
|
limits: configured,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("bind");
|
||||||
|
let client = TcpControlClient::connect(server.local_addr(), value, configured)
|
||||||
|
.await
|
||||||
|
.expect("connect");
|
||||||
|
Fixture {
|
||||||
|
client: Client::Tcp(client),
|
||||||
|
plane,
|
||||||
|
target,
|
||||||
|
server: Some(server),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn close(fixture: Fixture) {
|
||||||
|
drop(fixture.client);
|
||||||
|
if let Some(server) = fixture.server {
|
||||||
|
server.shutdown().await.expect("shutdown");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn conformance(transport: Transport) {
|
||||||
|
let fixture = fixture(transport, ControlRole::Operator, limits()).await;
|
||||||
|
let health = fixture
|
||||||
|
.client
|
||||||
|
.request(request("health-1", ControlRequest::Health))
|
||||||
|
.await;
|
||||||
|
assert_eq!(health.version, CONTROL_PROTOCOL_VERSION);
|
||||||
|
assert!(matches!(
|
||||||
|
health.result,
|
||||||
|
Ok(ControlPayload::Health(HealthView { ready: true, .. }))
|
||||||
|
));
|
||||||
|
let page = fixture
|
||||||
|
.client
|
||||||
|
.request(request(
|
||||||
|
"sessions-1",
|
||||||
|
ControlRequest::ListSessions {
|
||||||
|
page: PageRequest {
|
||||||
|
cursor: Some(2),
|
||||||
|
limit: 3,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
))
|
||||||
|
.await;
|
||||||
|
let Ok(ControlPayload::Sessions(page)) = page.result else {
|
||||||
|
panic!("session page")
|
||||||
|
};
|
||||||
|
assert_eq!(page.items.len(), 3);
|
||||||
|
assert_eq!(page.items[0].session_id, "session-2");
|
||||||
|
assert_eq!(page.next_cursor, Some(5));
|
||||||
|
assert!(
|
||||||
|
fixture
|
||||||
|
.client
|
||||||
|
.request(request("pause-1", ControlRequest::PauseAutonomy))
|
||||||
|
.await
|
||||||
|
.result
|
||||||
|
.is_ok()
|
||||||
|
);
|
||||||
|
let replay = fixture
|
||||||
|
.client
|
||||||
|
.request(request("pause-1", ControlRequest::ResumeAutonomy))
|
||||||
|
.await;
|
||||||
|
assert_eq!(
|
||||||
|
replay.result.expect_err("replay").code,
|
||||||
|
ControlErrorCode::Replay
|
||||||
|
);
|
||||||
|
assert_eq!(fixture.target.mutation_count(), 1);
|
||||||
|
let shutdown = fixture
|
||||||
|
.client
|
||||||
|
.request(request("shutdown-1", ControlRequest::GracefulShutdown))
|
||||||
|
.await;
|
||||||
|
assert!(shutdown.result.is_ok());
|
||||||
|
assert!(fixture.target.shutdown.load(Ordering::Acquire));
|
||||||
|
close(fixture).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn same_conformance_suite_runs_in_process() {
|
||||||
|
conformance(Transport::InProcess).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn same_conformance_suite_runs_over_loopback_tcp() {
|
||||||
|
conformance(Transport::Tcp).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn observer_is_read_only_and_tokens_are_redacted() {
|
||||||
|
let fixture = fixture(Transport::InProcess, ControlRole::Observer, limits()).await;
|
||||||
|
assert!(matches!(
|
||||||
|
fixture
|
||||||
|
.client
|
||||||
|
.request(request("read", ControlRequest::Runtime))
|
||||||
|
.await
|
||||||
|
.result,
|
||||||
|
Ok(ControlPayload::Runtime(_))
|
||||||
|
));
|
||||||
|
let denied = fixture
|
||||||
|
.client
|
||||||
|
.request(request("mutate", ControlRequest::PauseAutonomy))
|
||||||
|
.await;
|
||||||
|
assert_eq!(
|
||||||
|
denied.result.expect_err("denied").code,
|
||||||
|
ControlErrorCode::PermissionDenied
|
||||||
|
);
|
||||||
|
assert_eq!(fixture.target.mutation_count(), 0);
|
||||||
|
assert!(!format!("{:?}", fixture.plane).contains("operator-secret"));
|
||||||
|
let Err(error) = fixture.plane.connect("wrong") else {
|
||||||
|
panic!("bad token was accepted")
|
||||||
|
};
|
||||||
|
assert_eq!(error.code, ControlErrorCode::AuthenticationFailed);
|
||||||
|
close(fixture).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn version_bounds_and_malformed_requests_are_typed() {
|
||||||
|
let fixture = fixture(Transport::InProcess, ControlRole::Operator, limits()).await;
|
||||||
|
let mut wrong = request("version", ControlRequest::Health);
|
||||||
|
wrong.version = 99;
|
||||||
|
assert_eq!(
|
||||||
|
fixture
|
||||||
|
.client
|
||||||
|
.request(wrong)
|
||||||
|
.await
|
||||||
|
.result
|
||||||
|
.expect_err("version")
|
||||||
|
.code,
|
||||||
|
ControlErrorCode::VersionMismatch
|
||||||
|
);
|
||||||
|
let invalid = fixture
|
||||||
|
.client
|
||||||
|
.request(request(
|
||||||
|
"page",
|
||||||
|
ControlRequest::ListSessions {
|
||||||
|
page: PageRequest {
|
||||||
|
cursor: None,
|
||||||
|
limit: 101,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
))
|
||||||
|
.await;
|
||||||
|
assert_eq!(
|
||||||
|
invalid.result.expect_err("page").code,
|
||||||
|
ControlErrorCode::InvalidRequest
|
||||||
|
);
|
||||||
|
let secret_payload = fixture
|
||||||
|
.client
|
||||||
|
.request(request(
|
||||||
|
"audit",
|
||||||
|
ControlRequest::ListAuditEvents {
|
||||||
|
page: PageRequest::default(),
|
||||||
|
},
|
||||||
|
))
|
||||||
|
.await;
|
||||||
|
let rendered = serde_json::to_string(&secret_payload).expect("JSON");
|
||||||
|
for secret in ["operator-secret", "observer-secret", "Bearer ", "CAPS/"] {
|
||||||
|
assert!(!rendered.contains(secret));
|
||||||
|
}
|
||||||
|
close(fixture).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn cancellation_is_per_request_and_concurrent_operators_stay_responsive() {
|
||||||
|
for transport in [Transport::InProcess, Transport::Tcp] {
|
||||||
|
let fixture = fixture(transport, ControlRole::Operator, limits()).await;
|
||||||
|
let slow = match &fixture.client {
|
||||||
|
Client::InProcess(value) => Client::InProcess(value.clone()),
|
||||||
|
Client::Tcp(value) => Client::Tcp(value.clone()),
|
||||||
|
};
|
||||||
|
let task = tokio::spawn(async move {
|
||||||
|
slow.request(request(
|
||||||
|
"slow",
|
||||||
|
ControlRequest::InjectOperatorMessage {
|
||||||
|
message: "wait".into(),
|
||||||
|
},
|
||||||
|
))
|
||||||
|
.await
|
||||||
|
});
|
||||||
|
tokio::time::timeout(Duration::from_secs(1), async {
|
||||||
|
while fixture.target.active.load(Ordering::Acquire) == 0 {
|
||||||
|
tokio::task::yield_now().await;
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("slow request became active");
|
||||||
|
let concurrent_operator = match &fixture.server {
|
||||||
|
Some(server) => Client::Tcp(
|
||||||
|
TcpControlClient::connect(server.local_addr(), "operator-secret", limits())
|
||||||
|
.await
|
||||||
|
.expect("second TCP operator"),
|
||||||
|
),
|
||||||
|
None => Client::InProcess(
|
||||||
|
fixture
|
||||||
|
.plane
|
||||||
|
.connect("operator-secret")
|
||||||
|
.expect("second in-process operator"),
|
||||||
|
),
|
||||||
|
};
|
||||||
|
let health = concurrent_operator
|
||||||
|
.request(request("health-during", ControlRequest::Health))
|
||||||
|
.await;
|
||||||
|
assert!(health.result.is_ok());
|
||||||
|
let cancelled = fixture
|
||||||
|
.client
|
||||||
|
.request(request(
|
||||||
|
"cancel",
|
||||||
|
ControlRequest::CancelRequest {
|
||||||
|
target_request_id: "slow".into(),
|
||||||
|
},
|
||||||
|
))
|
||||||
|
.await;
|
||||||
|
assert!(matches!(
|
||||||
|
cancelled.result,
|
||||||
|
Ok(ControlPayload::Cancelled { .. })
|
||||||
|
));
|
||||||
|
let slow = task.await.expect("slow task");
|
||||||
|
assert_eq!(
|
||||||
|
slow.result.expect_err("cancelled").code,
|
||||||
|
ControlErrorCode::Cancelled
|
||||||
|
);
|
||||||
|
assert!(fixture.target.maximum_active.load(Ordering::Acquire) >= 2);
|
||||||
|
drop(concurrent_operator);
|
||||||
|
close(fixture).await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn event_stream_resubscribes_with_explicit_gap_and_slow_consumers_are_cut_off() {
|
||||||
|
let mut configured = limits();
|
||||||
|
configured.event_queue = 2;
|
||||||
|
configured.event_history = 3;
|
||||||
|
let fixture = fixture(Transport::InProcess, ControlRole::Operator, configured).await;
|
||||||
|
let mut in_subscription = None;
|
||||||
|
if let Client::InProcess(client) = &fixture.client {
|
||||||
|
let (_, subscription) = client
|
||||||
|
.subscribe(request(
|
||||||
|
"subscribe",
|
||||||
|
ControlRequest::SubscribeEvents {
|
||||||
|
after_sequence: Some(0),
|
||||||
|
},
|
||||||
|
))
|
||||||
|
.await
|
||||||
|
.expect("subscribe");
|
||||||
|
in_subscription = Some(subscription);
|
||||||
|
}
|
||||||
|
for index in 0..10 {
|
||||||
|
fixture.plane.publish(ControlEventKind::StateChanged {
|
||||||
|
component: "grid".into(),
|
||||||
|
state: format!("state-{index}"),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
assert_eq!(fixture.plane.retained_event_count(), 3);
|
||||||
|
let first = in_subscription
|
||||||
|
.as_mut()
|
||||||
|
.expect("subscription")
|
||||||
|
.recv()
|
||||||
|
.await
|
||||||
|
.expect("buffered");
|
||||||
|
assert_eq!(first.sequence, 1);
|
||||||
|
drop(in_subscription);
|
||||||
|
let client = fixture.plane.connect("operator-secret").expect("reconnect");
|
||||||
|
let (_, mut resumed) = client
|
||||||
|
.subscribe(request(
|
||||||
|
"resubscribe",
|
||||||
|
ControlRequest::SubscribeEvents {
|
||||||
|
after_sequence: Some(1),
|
||||||
|
},
|
||||||
|
))
|
||||||
|
.await
|
||||||
|
.expect("resubscribe");
|
||||||
|
let gap = resumed.recv().await.expect("gap");
|
||||||
|
assert!(matches!(
|
||||||
|
gap.event,
|
||||||
|
ControlEventKind::Gap {
|
||||||
|
first_available: 9,
|
||||||
|
last_missed: 8
|
||||||
|
}
|
||||||
|
));
|
||||||
|
close(fixture).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn zero_client_event_flood_remains_bounded_and_headless() {
|
||||||
|
let target = Arc::new(FakeTarget::default());
|
||||||
|
let mut configured = limits();
|
||||||
|
configured.event_history = 4;
|
||||||
|
let plane = plane(target, configured);
|
||||||
|
assert_eq!(plane.active_connections(), 0);
|
||||||
|
for index in 0..10_000 {
|
||||||
|
plane.publish(ControlEventKind::StateChanged {
|
||||||
|
component: "load".into(),
|
||||||
|
state: index.to_string(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
assert_eq!(plane.retained_event_count(), 4);
|
||||||
|
assert_eq!(plane.active_connections(), 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn subscriptions_have_a_hard_global_bound_and_release_on_drop() {
|
||||||
|
let mut configured = limits();
|
||||||
|
configured.max_subscriptions = 2;
|
||||||
|
let target = Arc::new(FakeTarget::default());
|
||||||
|
let plane = plane(target, configured);
|
||||||
|
let client = plane.connect("operator-secret").expect("client");
|
||||||
|
let (_, first) = client
|
||||||
|
.subscribe(request(
|
||||||
|
"subscribe-1",
|
||||||
|
ControlRequest::SubscribeEvents {
|
||||||
|
after_sequence: None,
|
||||||
|
},
|
||||||
|
))
|
||||||
|
.await
|
||||||
|
.expect("first subscription");
|
||||||
|
let (_, second) = client
|
||||||
|
.subscribe(request(
|
||||||
|
"subscribe-2",
|
||||||
|
ControlRequest::SubscribeEvents {
|
||||||
|
after_sequence: None,
|
||||||
|
},
|
||||||
|
))
|
||||||
|
.await
|
||||||
|
.expect("second subscription");
|
||||||
|
let error = client
|
||||||
|
.subscribe(request(
|
||||||
|
"subscribe-3",
|
||||||
|
ControlRequest::SubscribeEvents {
|
||||||
|
after_sequence: None,
|
||||||
|
},
|
||||||
|
))
|
||||||
|
.await
|
||||||
|
.err()
|
||||||
|
.expect("subscription bound");
|
||||||
|
assert_eq!(error.code, ControlErrorCode::Busy);
|
||||||
|
drop(first);
|
||||||
|
client
|
||||||
|
.subscribe(request(
|
||||||
|
"subscribe-4",
|
||||||
|
ControlRequest::SubscribeEvents {
|
||||||
|
after_sequence: None,
|
||||||
|
},
|
||||||
|
))
|
||||||
|
.await
|
||||||
|
.expect("released subscription");
|
||||||
|
drop(second);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
#[allow(clippy::too_many_lines)] // One raw-transport matrix shares one listener.
|
||||||
|
async fn tcp_rejects_bad_tokens_oversized_and_partial_frames() {
|
||||||
|
let configured = limits();
|
||||||
|
let target = Arc::new(FakeTarget::default());
|
||||||
|
let plane = plane(target, configured);
|
||||||
|
let server = TcpControlServer::bind(
|
||||||
|
plane.clone(),
|
||||||
|
TcpControlConfig {
|
||||||
|
listen: "127.0.0.1:0".parse().expect("address"),
|
||||||
|
limits: configured,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("server");
|
||||||
|
let Err(error) = TcpControlClient::connect(server.local_addr(), "wrong", configured).await
|
||||||
|
else {
|
||||||
|
panic!("bad token was accepted")
|
||||||
|
};
|
||||||
|
assert_eq!(error.code, ControlErrorCode::AuthenticationFailed);
|
||||||
|
|
||||||
|
let mut wrong_version = TcpStream::connect(server.local_addr())
|
||||||
|
.await
|
||||||
|
.expect("version connect");
|
||||||
|
let hello = serde_json::to_vec(&json!({
|
||||||
|
"frame":"hello",
|
||||||
|
"body":{"version":99,"token":"operator-secret"}
|
||||||
|
}))
|
||||||
|
.expect("hello JSON");
|
||||||
|
wrong_version
|
||||||
|
.write_all(
|
||||||
|
&u32::try_from(hello.len())
|
||||||
|
.expect("frame length")
|
||||||
|
.to_be_bytes(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("version header");
|
||||||
|
wrong_version.write_all(&hello).await.expect("version body");
|
||||||
|
let mut response_header = [0_u8; 4];
|
||||||
|
wrong_version
|
||||||
|
.read_exact(&mut response_header)
|
||||||
|
.await
|
||||||
|
.expect("version response header");
|
||||||
|
let mut response =
|
||||||
|
vec![0_u8; usize::try_from(u32::from_be_bytes(response_header)).expect("response length")];
|
||||||
|
wrong_version
|
||||||
|
.read_exact(&mut response)
|
||||||
|
.await
|
||||||
|
.expect("version response");
|
||||||
|
let response: serde_json::Value = serde_json::from_slice(&response).expect("version JSON");
|
||||||
|
assert_eq!(response["body"]["error"]["code"], json!("version_mismatch"));
|
||||||
|
|
||||||
|
let subscriber = TcpControlClient::connect(server.local_addr(), "operator-secret", configured)
|
||||||
|
.await
|
||||||
|
.expect("subscriber");
|
||||||
|
let subscription_response = subscriber
|
||||||
|
.request(request(
|
||||||
|
"events",
|
||||||
|
ControlRequest::SubscribeEvents {
|
||||||
|
after_sequence: None,
|
||||||
|
},
|
||||||
|
))
|
||||||
|
.await
|
||||||
|
.expect("subscription response");
|
||||||
|
assert!(matches!(
|
||||||
|
subscription_response.result,
|
||||||
|
Ok(ControlPayload::Subscribed { .. })
|
||||||
|
));
|
||||||
|
plane.publish(ControlEventKind::StateChanged {
|
||||||
|
component: "session".into(),
|
||||||
|
state: "online".into(),
|
||||||
|
});
|
||||||
|
let event = tokio::time::timeout(Duration::from_secs(1), subscriber.next_event())
|
||||||
|
.await
|
||||||
|
.expect("event deadline")
|
||||||
|
.expect("event");
|
||||||
|
assert!(matches!(event.event, ControlEventKind::StateChanged { .. }));
|
||||||
|
let last_sequence = event.sequence;
|
||||||
|
drop(subscriber);
|
||||||
|
plane.publish(ControlEventKind::StateChanged {
|
||||||
|
component: "session".into(),
|
||||||
|
state: "reconnecting".into(),
|
||||||
|
});
|
||||||
|
let reconnected = TcpControlClient::connect(server.local_addr(), "operator-secret", configured)
|
||||||
|
.await
|
||||||
|
.expect("reconnected subscriber");
|
||||||
|
reconnected
|
||||||
|
.request(request(
|
||||||
|
"events-resumed",
|
||||||
|
ControlRequest::SubscribeEvents {
|
||||||
|
after_sequence: Some(last_sequence),
|
||||||
|
},
|
||||||
|
))
|
||||||
|
.await
|
||||||
|
.expect("resubscription response");
|
||||||
|
let replayed = tokio::time::timeout(Duration::from_secs(1), reconnected.next_event())
|
||||||
|
.await
|
||||||
|
.expect("replay deadline")
|
||||||
|
.expect("replayed event");
|
||||||
|
assert_eq!(replayed.sequence, last_sequence.saturating_add(1));
|
||||||
|
let mut oversized = TcpStream::connect(server.local_addr())
|
||||||
|
.await
|
||||||
|
.expect("raw connect");
|
||||||
|
oversized
|
||||||
|
.write_all(&(u32::try_from(configured.max_frame_bytes).expect("bound") + 1).to_be_bytes())
|
||||||
|
.await
|
||||||
|
.expect("header");
|
||||||
|
let mut byte = [0_u8; 1];
|
||||||
|
assert_eq!(
|
||||||
|
tokio::time::timeout(Duration::from_secs(1), oversized.read(&mut byte))
|
||||||
|
.await
|
||||||
|
.expect("closed")
|
||||||
|
.expect("read"),
|
||||||
|
0
|
||||||
|
);
|
||||||
|
let mut partial = TcpStream::connect(server.local_addr())
|
||||||
|
.await
|
||||||
|
.expect("raw connect");
|
||||||
|
partial
|
||||||
|
.write_all(&100_u32.to_be_bytes())
|
||||||
|
.await
|
||||||
|
.expect("header");
|
||||||
|
partial.write_all(b"{}").await.expect("partial");
|
||||||
|
partial.shutdown().await.expect("close");
|
||||||
|
drop(reconnected);
|
||||||
|
server.shutdown().await.expect("shutdown");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn plaintext_server_rejects_non_loopback_before_binding() {
|
||||||
|
let configured = limits();
|
||||||
|
let target = Arc::new(FakeTarget::default());
|
||||||
|
let plane = plane(target, configured);
|
||||||
|
let Err(error) = TcpControlServer::bind(
|
||||||
|
plane,
|
||||||
|
TcpControlConfig {
|
||||||
|
listen: "0.0.0.0:7943".parse().expect("address"),
|
||||||
|
limits: configured,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
else {
|
||||||
|
panic!("plaintext remote listener was accepted")
|
||||||
|
};
|
||||||
|
assert_eq!(error.code, ControlErrorCode::InvalidRequest);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn plain_tcp_is_loopback_only_and_protocol_json_is_stable() {
|
||||||
|
let configured = limits();
|
||||||
|
assert!(
|
||||||
|
!TcpControlConfig {
|
||||||
|
listen: "0.0.0.0:7943".parse().expect("address"),
|
||||||
|
limits: configured
|
||||||
|
}
|
||||||
|
.listen
|
||||||
|
.ip()
|
||||||
|
.is_loopback()
|
||||||
|
);
|
||||||
|
let encoded = serde_json::to_value(request("health", ControlRequest::Health)).expect("JSON");
|
||||||
|
assert_eq!(
|
||||||
|
encoded,
|
||||||
|
json!({"version":1,"request_id":"health","request":{"method":"health"}})
|
||||||
|
);
|
||||||
|
}
|
||||||
522
crates/metacrate-grid-agent/src/control_runtime.rs
Normal file
522
crates/metacrate-grid-agent/src/control_runtime.rs
Normal file
@@ -0,0 +1,522 @@
|
|||||||
|
//! Production management target backed by the agent's bounded runtime stores.
|
||||||
|
|
||||||
|
#![allow(clippy::missing_errors_doc)]
|
||||||
|
|
||||||
|
use crate::behavior::{BehaviorIngress, BehaviorMode};
|
||||||
|
use crate::control_plane::{
|
||||||
|
AuditEventView, CONTROL_PROTOCOL_VERSION, ControlContext, ControlError, ControlErrorCode,
|
||||||
|
ControlFuture, ControlPayload, ControlRequest, ControlTarget, ConversationChannelView,
|
||||||
|
HealthView, Page, PageRequest, PendingApprovalView, RuntimeView, ScheduledJobView,
|
||||||
|
SessionMetadataView,
|
||||||
|
};
|
||||||
|
use crate::conversation::{ConversationChannel, ConversationKey, ConversationStore};
|
||||||
|
use crate::policy::{
|
||||||
|
ApprovalId, AuthenticatedPrincipal, MemoryPolicyAudit, PolicyFinalOutcome, PolicyGateway,
|
||||||
|
PolicyReasonCode,
|
||||||
|
};
|
||||||
|
use crate::session::{SessionState, SessionStatus};
|
||||||
|
use libremetaverse_types::compat::CancellationToken;
|
||||||
|
use std::collections::BTreeMap;
|
||||||
|
use std::fmt;
|
||||||
|
use std::sync::atomic::{AtomicU64, Ordering};
|
||||||
|
use std::sync::{Arc, Mutex, MutexGuard};
|
||||||
|
use std::time::{Instant, SystemTime, UNIX_EPOCH};
|
||||||
|
use tokio::sync::mpsc;
|
||||||
|
|
||||||
|
const ROAMING_JOB_ID: &str = "default-roaming";
|
||||||
|
|
||||||
|
/// Commands whose ownership remains with the service lifecycle loop.
|
||||||
|
pub enum RuntimeControlCommand {
|
||||||
|
Pause,
|
||||||
|
Resume,
|
||||||
|
ForceReconnect,
|
||||||
|
OperatorMessage { message: String },
|
||||||
|
GracefulShutdown,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl fmt::Debug for RuntimeControlCommand {
|
||||||
|
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||||
|
match self {
|
||||||
|
Self::OperatorMessage { message } => formatter
|
||||||
|
.debug_struct("OperatorMessage")
|
||||||
|
.field("message_bytes", &message.len())
|
||||||
|
.finish(),
|
||||||
|
Self::Pause => formatter.write_str("Pause"),
|
||||||
|
Self::Resume => formatter.write_str("Resume"),
|
||||||
|
Self::ForceReconnect => formatter.write_str("ForceReconnect"),
|
||||||
|
Self::GracefulShutdown => formatter.write_str("GracefulShutdown"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug)]
|
||||||
|
struct RuntimeState {
|
||||||
|
session: SessionStatus,
|
||||||
|
behavior: BehaviorMode,
|
||||||
|
service_state: &'static str,
|
||||||
|
region_id: Option<String>,
|
||||||
|
region_name: Option<String>,
|
||||||
|
position: Option<[f64; 3]>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for RuntimeState {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self {
|
||||||
|
session: SessionStatus::default(),
|
||||||
|
behavior: BehaviorMode::Offline,
|
||||||
|
service_state: "starting",
|
||||||
|
region_id: None,
|
||||||
|
region_name: None,
|
||||||
|
position: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Real control target shared by integrated and split transports.
|
||||||
|
pub struct AgentControlTarget {
|
||||||
|
started: Instant,
|
||||||
|
state: Mutex<RuntimeState>,
|
||||||
|
conversations: Arc<ConversationStore>,
|
||||||
|
policy: Arc<PolicyGateway>,
|
||||||
|
audit: Arc<MemoryPolicyAudit>,
|
||||||
|
behavior: BehaviorIngress,
|
||||||
|
commands: mpsc::Sender<RuntimeControlCommand>,
|
||||||
|
command_capacity: usize,
|
||||||
|
jobs: Mutex<BTreeMap<String, bool>>,
|
||||||
|
next_operation: AtomicU64,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl fmt::Debug for AgentControlTarget {
|
||||||
|
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||||
|
formatter
|
||||||
|
.debug_struct("AgentControlTarget")
|
||||||
|
.field("state", &lock(&self.state))
|
||||||
|
.field("command_capacity", &self.command_capacity)
|
||||||
|
.finish_non_exhaustive()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl AgentControlTarget {
|
||||||
|
pub fn new(
|
||||||
|
conversations: Arc<ConversationStore>,
|
||||||
|
policy: Arc<PolicyGateway>,
|
||||||
|
audit: Arc<MemoryPolicyAudit>,
|
||||||
|
behavior: BehaviorIngress,
|
||||||
|
command_capacity: usize,
|
||||||
|
) -> Result<(Arc<Self>, mpsc::Receiver<RuntimeControlCommand>), ControlError> {
|
||||||
|
if command_capacity == 0 || command_capacity > 8_192 {
|
||||||
|
return Err(control_error(
|
||||||
|
ControlErrorCode::InvalidRequest,
|
||||||
|
"invalid runtime control queue capacity",
|
||||||
|
false,
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let (commands, receiver) = mpsc::channel(command_capacity);
|
||||||
|
let mut jobs = BTreeMap::new();
|
||||||
|
jobs.insert(ROAMING_JOB_ID.to_owned(), false);
|
||||||
|
Ok((
|
||||||
|
Arc::new(Self {
|
||||||
|
started: Instant::now(),
|
||||||
|
state: Mutex::new(RuntimeState::default()),
|
||||||
|
conversations,
|
||||||
|
policy,
|
||||||
|
audit,
|
||||||
|
behavior,
|
||||||
|
commands,
|
||||||
|
command_capacity,
|
||||||
|
jobs: Mutex::new(jobs),
|
||||||
|
next_operation: AtomicU64::new(1),
|
||||||
|
}),
|
||||||
|
receiver,
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn update_session(&self, session: SessionStatus) {
|
||||||
|
let mut state = lock(&self.state);
|
||||||
|
state.session = session;
|
||||||
|
state.service_state = if session.state == SessionState::ShuttingDown {
|
||||||
|
"stopping"
|
||||||
|
} else {
|
||||||
|
"running"
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn update_behavior(&self, behavior: BehaviorMode) {
|
||||||
|
lock(&self.state).behavior = behavior;
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn update_region(
|
||||||
|
&self,
|
||||||
|
region_id: Option<String>,
|
||||||
|
region_name: Option<String>,
|
||||||
|
position: Option<[f64; 3]>,
|
||||||
|
) {
|
||||||
|
let mut state = lock(&self.state);
|
||||||
|
state.region_id = region_id.filter(|value| value.len() <= 64);
|
||||||
|
state.region_name = region_name.filter(|value| value.len() <= 256);
|
||||||
|
state.position =
|
||||||
|
position.filter(|value| value.iter().all(|component| component.is_finite()));
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn mark_stopping(&self) {
|
||||||
|
lock(&self.state).service_state = "stopping";
|
||||||
|
}
|
||||||
|
|
||||||
|
fn enqueue(
|
||||||
|
&self,
|
||||||
|
operation: &'static str,
|
||||||
|
command: RuntimeControlCommand,
|
||||||
|
) -> Result<ControlPayload, ControlError> {
|
||||||
|
self.commands
|
||||||
|
.try_send(command)
|
||||||
|
.map_err(|error| match error {
|
||||||
|
mpsc::error::TrySendError::Full(_) => control_error(
|
||||||
|
ControlErrorCode::Backpressure,
|
||||||
|
"runtime control queue is full",
|
||||||
|
true,
|
||||||
|
),
|
||||||
|
mpsc::error::TrySendError::Closed(_) => control_error(
|
||||||
|
ControlErrorCode::TransportClosed,
|
||||||
|
"runtime control queue is closed",
|
||||||
|
true,
|
||||||
|
),
|
||||||
|
})?;
|
||||||
|
Ok(ControlPayload::Accepted {
|
||||||
|
operation_id: format!(
|
||||||
|
"{operation}-{}",
|
||||||
|
self.next_operation.fetch_add(1, Ordering::Relaxed)
|
||||||
|
),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
#[allow(clippy::too_many_lines)] // Exhaustive protocol-to-runtime mapping stays auditable.
|
||||||
|
fn execute_now(
|
||||||
|
&self,
|
||||||
|
context: &ControlContext,
|
||||||
|
request: ControlRequest,
|
||||||
|
) -> Result<ControlPayload, ControlError> {
|
||||||
|
match request {
|
||||||
|
ControlRequest::Health => {
|
||||||
|
let state = lock(&self.state);
|
||||||
|
Ok(ControlPayload::Health(HealthView {
|
||||||
|
service_state: state.service_state.to_owned(),
|
||||||
|
ready: state.session.agent_ready,
|
||||||
|
uptime_seconds: self.started.elapsed().as_secs(),
|
||||||
|
protocol_version: CONTROL_PROTOCOL_VERSION,
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
ControlRequest::Runtime => {
|
||||||
|
let state = lock(&self.state).clone();
|
||||||
|
let usage = self.policy.global_budget_usage();
|
||||||
|
Ok(ControlPayload::Runtime(RuntimeView {
|
||||||
|
grid_state: state.session.state.as_str().to_owned(),
|
||||||
|
generation: state.session.generation,
|
||||||
|
transport_connected: state.session.transport_connected,
|
||||||
|
agent_ready: state.session.agent_ready,
|
||||||
|
region_id: state.region_id,
|
||||||
|
region_name: state.region_name,
|
||||||
|
position: state.position,
|
||||||
|
behavior_mode: behavior_name(state.behavior).to_owned(),
|
||||||
|
control_queue_used: self
|
||||||
|
.command_capacity
|
||||||
|
.saturating_sub(self.commands.capacity()),
|
||||||
|
control_queue_capacity: self.command_capacity,
|
||||||
|
budget_tool_calls_used: usage.tool_calls,
|
||||||
|
budget_movement_millimeters_used: usage.movement_millimeters,
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
ControlRequest::ListSessions { page } => {
|
||||||
|
let values = self
|
||||||
|
.conversations
|
||||||
|
.list_metadata()
|
||||||
|
.into_iter()
|
||||||
|
.map(|metadata| SessionMetadataView {
|
||||||
|
session_id: metadata.session_id.as_str().to_owned(),
|
||||||
|
avatar_id: metadata.avatar_id.to_string(),
|
||||||
|
channel: match metadata.channel {
|
||||||
|
ConversationChannel::PublicChat => "public_chat",
|
||||||
|
ConversationChannel::DirectIm => "direct_im",
|
||||||
|
}
|
||||||
|
.to_owned(),
|
||||||
|
created_unix_millis: metadata.created_unix_millis,
|
||||||
|
last_active_unix_millis: metadata.last_active_unix_millis,
|
||||||
|
turns: metadata.turns,
|
||||||
|
bytes: metadata.bytes,
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
Ok(ControlPayload::Sessions(page_values(&page, values)))
|
||||||
|
}
|
||||||
|
ControlRequest::ListScheduledJobs { page } => {
|
||||||
|
let values = lock(&self.jobs)
|
||||||
|
.iter()
|
||||||
|
.map(|(job_id, enabled)| ScheduledJobView {
|
||||||
|
job_id: job_id.clone(),
|
||||||
|
kind: "bounded_roaming".to_owned(),
|
||||||
|
enabled: *enabled,
|
||||||
|
next_run_unix_millis: None,
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
Ok(ControlPayload::ScheduledJobs(page_values(&page, values)))
|
||||||
|
}
|
||||||
|
ControlRequest::ListPendingApprovals { page } => {
|
||||||
|
let values = self
|
||||||
|
.policy
|
||||||
|
.pending_approvals(unix_seconds())
|
||||||
|
.into_iter()
|
||||||
|
.map(|approval| PendingApprovalView {
|
||||||
|
approval_id: approval.id.get(),
|
||||||
|
tool: approval.tool,
|
||||||
|
principal: approval.principal,
|
||||||
|
expires_unix_seconds: approval.expires_at,
|
||||||
|
movement_millimeters: approval.cost.movement_millimeters,
|
||||||
|
inventory_operations: approval.cost.inventory_operations,
|
||||||
|
build_prims: approval.cost.build_prims,
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
Ok(ControlPayload::PendingApprovals(page_values(&page, values)))
|
||||||
|
}
|
||||||
|
ControlRequest::ListAuditEvents { page } => {
|
||||||
|
let values = self
|
||||||
|
.audit
|
||||||
|
.snapshot()
|
||||||
|
.into_iter()
|
||||||
|
.enumerate()
|
||||||
|
.map(|(index, record)| AuditEventView {
|
||||||
|
sequence: u64::try_from(index).unwrap_or(u64::MAX).saturating_add(1),
|
||||||
|
unix_millis: record.recorded_unix_millis,
|
||||||
|
principal: record.principal.as_str().to_owned(),
|
||||||
|
operation: record.tool.as_str().to_owned(),
|
||||||
|
outcome: policy_outcome_name(record.final_outcome).to_owned(),
|
||||||
|
authorization_id: record.authorization_id,
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
Ok(ControlPayload::AuditEvents(page_values(&page, values)))
|
||||||
|
}
|
||||||
|
ControlRequest::PauseAutonomy => {
|
||||||
|
let response = self.enqueue("pause", RuntimeControlCommand::Pause)?;
|
||||||
|
self.behavior.pause();
|
||||||
|
Ok(response)
|
||||||
|
}
|
||||||
|
ControlRequest::ResumeAutonomy => {
|
||||||
|
let response = self.enqueue("resume", RuntimeControlCommand::Resume)?;
|
||||||
|
self.behavior.resume();
|
||||||
|
Ok(response)
|
||||||
|
}
|
||||||
|
ControlRequest::CancelAction { action_id } => {
|
||||||
|
let cancelled = self.behavior.cancel_action(&action_id).map_err(|_| {
|
||||||
|
control_error(ControlErrorCode::InvalidRequest, "invalid action ID", false)
|
||||||
|
})?;
|
||||||
|
if !cancelled {
|
||||||
|
return Err(control_error(
|
||||||
|
ControlErrorCode::NotFound,
|
||||||
|
"active behavior action not found",
|
||||||
|
false,
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Ok(ControlPayload::Completed)
|
||||||
|
}
|
||||||
|
ControlRequest::DecideApproval {
|
||||||
|
approval_id,
|
||||||
|
approve,
|
||||||
|
} => {
|
||||||
|
let id = ApprovalId::from_raw(approval_id).ok_or_else(|| {
|
||||||
|
control_error(
|
||||||
|
ControlErrorCode::InvalidRequest,
|
||||||
|
"invalid approval ID",
|
||||||
|
false,
|
||||||
|
)
|
||||||
|
})?;
|
||||||
|
let principal =
|
||||||
|
AuthenticatedPrincipal::from_authenticated_control(context.principal.clone())
|
||||||
|
.map_err(|_| {
|
||||||
|
control_error(
|
||||||
|
ControlErrorCode::PermissionDenied,
|
||||||
|
"invalid operator principal",
|
||||||
|
false,
|
||||||
|
)
|
||||||
|
})?;
|
||||||
|
let reason = if approve {
|
||||||
|
self.policy.grant_approval(id, &principal, unix_seconds())
|
||||||
|
} else {
|
||||||
|
self.policy.deny_approval(id, &principal, unix_seconds())
|
||||||
|
}
|
||||||
|
.map_err(|_| {
|
||||||
|
control_error(
|
||||||
|
ControlErrorCode::Internal,
|
||||||
|
"approval decision failed",
|
||||||
|
false,
|
||||||
|
)
|
||||||
|
})?;
|
||||||
|
match reason {
|
||||||
|
PolicyReasonCode::Allowed | PolicyReasonCode::ApprovalNotGranted => {
|
||||||
|
Ok(ControlPayload::Completed)
|
||||||
|
}
|
||||||
|
PolicyReasonCode::ApprovalUnknown => Err(control_error(
|
||||||
|
ControlErrorCode::NotFound,
|
||||||
|
"approval not found",
|
||||||
|
false,
|
||||||
|
)),
|
||||||
|
PolicyReasonCode::ApprovalExpired | PolicyReasonCode::ApprovalReplayed => {
|
||||||
|
Err(control_error(
|
||||||
|
ControlErrorCode::Conflict,
|
||||||
|
"approval is no longer pending",
|
||||||
|
false,
|
||||||
|
))
|
||||||
|
}
|
||||||
|
_ => Err(control_error(
|
||||||
|
ControlErrorCode::Conflict,
|
||||||
|
"approval decision rejected",
|
||||||
|
false,
|
||||||
|
)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
ControlRequest::ForceReconnect => {
|
||||||
|
self.enqueue("reconnect", RuntimeControlCommand::ForceReconnect)
|
||||||
|
}
|
||||||
|
ControlRequest::ExpireConversation { avatar_id, channel } => {
|
||||||
|
let avatar_id = libremetaverse_types::UUID::parse(avatar_id).map_err(|_| {
|
||||||
|
control_error(ControlErrorCode::InvalidRequest, "invalid avatar ID", false)
|
||||||
|
})?;
|
||||||
|
let channel = match channel {
|
||||||
|
ConversationChannelView::PublicChat => ConversationChannel::PublicChat,
|
||||||
|
ConversationChannelView::DirectIm => ConversationChannel::DirectIm,
|
||||||
|
};
|
||||||
|
let key = ConversationKey::new(avatar_id, channel).map_err(|_| {
|
||||||
|
control_error(
|
||||||
|
ControlErrorCode::InvalidRequest,
|
||||||
|
"invalid conversation key",
|
||||||
|
false,
|
||||||
|
)
|
||||||
|
})?;
|
||||||
|
if self.conversations.expire(key) {
|
||||||
|
Ok(ControlPayload::Completed)
|
||||||
|
} else {
|
||||||
|
Err(control_error(
|
||||||
|
ControlErrorCode::NotFound,
|
||||||
|
"conversation not found",
|
||||||
|
false,
|
||||||
|
))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
ControlRequest::SetRoamingJob { job_id, enabled } => {
|
||||||
|
let mut jobs = lock(&self.jobs);
|
||||||
|
let Some(current) = jobs.get_mut(&job_id) else {
|
||||||
|
return Err(control_error(
|
||||||
|
ControlErrorCode::NotFound,
|
||||||
|
"roaming job not found",
|
||||||
|
false,
|
||||||
|
));
|
||||||
|
};
|
||||||
|
self.behavior.set_roaming(enabled).map_err(|_| {
|
||||||
|
control_error(
|
||||||
|
ControlErrorCode::Backpressure,
|
||||||
|
"behavior queue unavailable",
|
||||||
|
true,
|
||||||
|
)
|
||||||
|
})?;
|
||||||
|
*current = enabled;
|
||||||
|
Ok(ControlPayload::Completed)
|
||||||
|
}
|
||||||
|
ControlRequest::InjectOperatorMessage { message } => self.enqueue(
|
||||||
|
"operator-message",
|
||||||
|
RuntimeControlCommand::OperatorMessage { message },
|
||||||
|
),
|
||||||
|
ControlRequest::GracefulShutdown => {
|
||||||
|
let response = self.enqueue("shutdown", RuntimeControlCommand::GracefulShutdown)?;
|
||||||
|
self.mark_stopping();
|
||||||
|
Ok(response)
|
||||||
|
}
|
||||||
|
ControlRequest::SubscribeEvents { .. } | ControlRequest::CancelRequest { .. } => {
|
||||||
|
Err(control_error(
|
||||||
|
ControlErrorCode::Internal,
|
||||||
|
"transport request reached runtime",
|
||||||
|
false,
|
||||||
|
))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ControlTarget for AgentControlTarget {
|
||||||
|
fn execute(
|
||||||
|
&self,
|
||||||
|
context: ControlContext,
|
||||||
|
request: ControlRequest,
|
||||||
|
cancellation: CancellationToken,
|
||||||
|
) -> ControlFuture<'_> {
|
||||||
|
Box::pin(async move {
|
||||||
|
if cancellation.is_cancellation_requested() {
|
||||||
|
return Err(control_error(
|
||||||
|
ControlErrorCode::Cancelled,
|
||||||
|
"control request cancelled",
|
||||||
|
false,
|
||||||
|
));
|
||||||
|
}
|
||||||
|
self.execute_now(&context, request)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn page_values<T>(request: &PageRequest, values: Vec<T>) -> Page<T> {
|
||||||
|
let start = usize::try_from(request.cursor.unwrap_or(0))
|
||||||
|
.unwrap_or(usize::MAX)
|
||||||
|
.min(values.len());
|
||||||
|
let end = start
|
||||||
|
.saturating_add(usize::from(request.limit))
|
||||||
|
.min(values.len());
|
||||||
|
let next_cursor = (end < values.len()).then(|| u64::try_from(end).unwrap_or(u64::MAX));
|
||||||
|
Page {
|
||||||
|
items: values.into_iter().skip(start).take(end - start).collect(),
|
||||||
|
next_cursor,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const fn behavior_name(mode: BehaviorMode) -> &'static str {
|
||||||
|
match mode {
|
||||||
|
BehaviorMode::Offline => "offline",
|
||||||
|
BehaviorMode::Settling => "settling",
|
||||||
|
BehaviorMode::Available => "available",
|
||||||
|
BehaviorMode::Engaged => "engaged",
|
||||||
|
BehaviorMode::Executing => "executing",
|
||||||
|
BehaviorMode::Roaming => "roaming",
|
||||||
|
BehaviorMode::Paused => "paused",
|
||||||
|
BehaviorMode::Recovering => "recovering",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const fn policy_outcome_name(outcome: PolicyFinalOutcome) -> &'static str {
|
||||||
|
match outcome {
|
||||||
|
PolicyFinalOutcome::Denied => "denied",
|
||||||
|
PolicyFinalOutcome::ApprovalRequired => "approval_required",
|
||||||
|
PolicyFinalOutcome::ApprovalGranted => "approval_granted",
|
||||||
|
PolicyFinalOutcome::Authorized => "authorized",
|
||||||
|
PolicyFinalOutcome::Completed => "completed",
|
||||||
|
PolicyFinalOutcome::Rejected => "rejected",
|
||||||
|
PolicyFinalOutcome::Failed => "failed",
|
||||||
|
PolicyFinalOutcome::AmbiguousMutation => "ambiguous_mutation",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn unix_seconds() -> u64 {
|
||||||
|
SystemTime::now()
|
||||||
|
.duration_since(UNIX_EPOCH)
|
||||||
|
.map_or(0, |duration| duration.as_secs())
|
||||||
|
}
|
||||||
|
|
||||||
|
const fn control_error(
|
||||||
|
code: ControlErrorCode,
|
||||||
|
message: &'static str,
|
||||||
|
retryable: bool,
|
||||||
|
) -> ControlError {
|
||||||
|
ControlError {
|
||||||
|
code,
|
||||||
|
message,
|
||||||
|
retryable,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn lock<T>(value: &Mutex<T>) -> MutexGuard<'_, T> {
|
||||||
|
value
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||||
|
}
|
||||||
167
crates/metacrate-grid-agent/src/control_runtime_tests.rs
Normal file
167
crates/metacrate-grid-agent/src/control_runtime_tests.rs
Normal file
@@ -0,0 +1,167 @@
|
|||||||
|
use crate::behavior::{BehaviorController, BehaviorError, EmbodimentFuture, EmbodimentSink};
|
||||||
|
use crate::control_plane::{
|
||||||
|
CONTROL_PROTOCOL_VERSION, ControlLimits, ControlPayload, ControlPlane, ControlRequest,
|
||||||
|
ControlRequestEnvelope,
|
||||||
|
};
|
||||||
|
use crate::control_runtime::{AgentControlTarget, RuntimeControlCommand};
|
||||||
|
use crate::conversation::{ConversationLimits, ConversationStore};
|
||||||
|
use crate::perception::WorldPosition;
|
||||||
|
use crate::policy::{MemoryPolicyAudit, PolicyAuditSink, PolicyGateway, PolicyLimits};
|
||||||
|
use crate::{AgentConfig, EmbodiedPose, SecretString};
|
||||||
|
use libremetaverse_types::UUID;
|
||||||
|
use libremetaverse_types::compat::CancellationToken;
|
||||||
|
use std::collections::BTreeSet;
|
||||||
|
use std::sync::Arc;
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
|
struct Sink;
|
||||||
|
|
||||||
|
impl EmbodimentSink for Sink {
|
||||||
|
fn current_pose(
|
||||||
|
&self,
|
||||||
|
generation: u64,
|
||||||
|
_cancellation: CancellationToken,
|
||||||
|
) -> EmbodimentFuture<'_, EmbodiedPose> {
|
||||||
|
Box::pin(async move {
|
||||||
|
Ok(EmbodiedPose {
|
||||||
|
generation,
|
||||||
|
region_id: UUID::zero(),
|
||||||
|
position: WorldPosition {
|
||||||
|
x: 128.0,
|
||||||
|
y: 128.0,
|
||||||
|
z: 24.0,
|
||||||
|
},
|
||||||
|
heading_degrees: 0.0,
|
||||||
|
sitting: false,
|
||||||
|
})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn resolve_avatar(
|
||||||
|
&self,
|
||||||
|
_generation: u64,
|
||||||
|
_avatar_id: UUID,
|
||||||
|
_cancellation: CancellationToken,
|
||||||
|
) -> EmbodimentFuture<'_, WorldPosition> {
|
||||||
|
Box::pin(async { Err(BehaviorError::TargetUnavailable) })
|
||||||
|
}
|
||||||
|
|
||||||
|
fn face_point(
|
||||||
|
&self,
|
||||||
|
_generation: u64,
|
||||||
|
_point: WorldPosition,
|
||||||
|
_cancellation: CancellationToken,
|
||||||
|
) -> EmbodimentFuture<'_, ()> {
|
||||||
|
Box::pin(async { Ok(()) })
|
||||||
|
}
|
||||||
|
|
||||||
|
fn begin_walk(
|
||||||
|
&self,
|
||||||
|
_generation: u64,
|
||||||
|
_point: WorldPosition,
|
||||||
|
_cancellation: CancellationToken,
|
||||||
|
) -> EmbodimentFuture<'_, ()> {
|
||||||
|
Box::pin(async { Ok(()) })
|
||||||
|
}
|
||||||
|
|
||||||
|
fn validate_walk_target(
|
||||||
|
&self,
|
||||||
|
_generation: u64,
|
||||||
|
_point: WorldPosition,
|
||||||
|
_cancellation: CancellationToken,
|
||||||
|
) -> EmbodimentFuture<'_, ()> {
|
||||||
|
Box::pin(async { Ok(()) })
|
||||||
|
}
|
||||||
|
|
||||||
|
fn stop(&self, _generation: u64, _cancellation: CancellationToken) -> EmbodimentFuture<'_, ()> {
|
||||||
|
Box::pin(async { Ok(()) })
|
||||||
|
}
|
||||||
|
|
||||||
|
fn sit(&self, _generation: u64, _cancellation: CancellationToken) -> EmbodimentFuture<'_, ()> {
|
||||||
|
Box::pin(async { Ok(()) })
|
||||||
|
}
|
||||||
|
|
||||||
|
fn stand(
|
||||||
|
&self,
|
||||||
|
_generation: u64,
|
||||||
|
_cancellation: CancellationToken,
|
||||||
|
) -> EmbodimentFuture<'_, ()> {
|
||||||
|
Box::pin(async { Ok(()) })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn envelope(id: &str, request: ControlRequest) -> ControlRequestEnvelope {
|
||||||
|
ControlRequestEnvelope {
|
||||||
|
version: CONTROL_PROTOCOL_VERSION,
|
||||||
|
request_id: id.to_owned(),
|
||||||
|
request,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn production_target_projects_state_and_routes_real_mutations() {
|
||||||
|
let settings = AgentConfig::offline("https://llm.invalid/chat", "llm-key")
|
||||||
|
.expect("offline config")
|
||||||
|
.behavior;
|
||||||
|
let behavior = BehaviorController::new(settings, Arc::new(Sink), 8, 8, Duration::from_secs(1))
|
||||||
|
.expect("behavior")
|
||||||
|
.start();
|
||||||
|
let audit = Arc::new(MemoryPolicyAudit::new(32).expect("audit"));
|
||||||
|
let audit_sink: Arc<dyn PolicyAuditSink> = audit.clone();
|
||||||
|
let policy = Arc::new(
|
||||||
|
PolicyGateway::new(
|
||||||
|
BTreeSet::new(),
|
||||||
|
Vec::new(),
|
||||||
|
PolicyLimits::default(),
|
||||||
|
audit_sink,
|
||||||
|
)
|
||||||
|
.expect("policy"),
|
||||||
|
);
|
||||||
|
let conversations = Arc::new(
|
||||||
|
ConversationStore::open(ConversationLimits::default(), None).expect("conversations"),
|
||||||
|
);
|
||||||
|
let (target, mut commands) =
|
||||||
|
AgentControlTarget::new(conversations, policy, audit, behavior.ingress(), 8)
|
||||||
|
.expect("target");
|
||||||
|
target.update_region(
|
||||||
|
Some("00000000-0000-4000-8000-000000000001".into()),
|
||||||
|
Some("Test Region".into()),
|
||||||
|
Some([128.0, 128.0, 24.0]),
|
||||||
|
);
|
||||||
|
let plane = ControlPlane::new(
|
||||||
|
target,
|
||||||
|
SecretString::new("test.operator", "operator-token").expect("token"),
|
||||||
|
Some(SecretString::new("test.observer", "observer-token").expect("token")),
|
||||||
|
ControlLimits::default(),
|
||||||
|
)
|
||||||
|
.expect("plane");
|
||||||
|
let operator = plane.connect("operator-token").expect("operator");
|
||||||
|
let runtime = operator
|
||||||
|
.request(envelope("runtime", ControlRequest::Runtime))
|
||||||
|
.await;
|
||||||
|
let Ok(ControlPayload::Runtime(runtime)) = runtime.result else {
|
||||||
|
panic!("runtime projection")
|
||||||
|
};
|
||||||
|
assert_eq!(runtime.region_name.as_deref(), Some("Test Region"));
|
||||||
|
assert_eq!(runtime.control_queue_capacity, 8);
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
operator
|
||||||
|
.request(envelope("pause", ControlRequest::PauseAutonomy))
|
||||||
|
.await
|
||||||
|
.result
|
||||||
|
.is_ok()
|
||||||
|
);
|
||||||
|
assert!(matches!(
|
||||||
|
commands.recv().await,
|
||||||
|
Some(RuntimeControlCommand::Pause)
|
||||||
|
));
|
||||||
|
|
||||||
|
let observer = plane.connect("observer-token").expect("observer");
|
||||||
|
let denied = observer
|
||||||
|
.request(envelope("shutdown", ControlRequest::GracefulShutdown))
|
||||||
|
.await;
|
||||||
|
assert!(denied.result.is_err());
|
||||||
|
assert!(commands.try_recv().is_err());
|
||||||
|
behavior.shutdown().await.expect("shutdown behavior");
|
||||||
|
}
|
||||||
@@ -7,6 +7,8 @@
|
|||||||
pub mod backend;
|
pub mod backend;
|
||||||
pub mod behavior;
|
pub mod behavior;
|
||||||
pub mod config;
|
pub mod config;
|
||||||
|
pub mod control_plane;
|
||||||
|
pub mod control_runtime;
|
||||||
pub mod conversation;
|
pub mod conversation;
|
||||||
pub mod interaction;
|
pub mod interaction;
|
||||||
pub mod llm;
|
pub mod llm;
|
||||||
@@ -20,6 +22,10 @@ pub mod types;
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod behavior_tests;
|
mod behavior_tests;
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
|
mod control_plane_tests;
|
||||||
|
#[cfg(test)]
|
||||||
|
mod control_runtime_tests;
|
||||||
|
#[cfg(test)]
|
||||||
mod conversation_tests;
|
mod conversation_tests;
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod interaction_tests;
|
mod interaction_tests;
|
||||||
@@ -47,10 +53,19 @@ pub use behavior::{
|
|||||||
STOP_TOOL, WALK_SHORT_TOOL, behavior_policy_tools,
|
STOP_TOOL, WALK_SHORT_TOOL, behavior_policy_tools,
|
||||||
};
|
};
|
||||||
pub use config::{
|
pub use config::{
|
||||||
AgentConfig, BehaviorSettings, ConfigError, ConfigLoader, ConversationSettings, EndpointUrl,
|
AgentConfig, BehaviorSettings, ConfigError, ConfigLoader, ControlSettings,
|
||||||
Environment, GridConnection, Limits, LlmConnection, MapEnvironment, OperatingMode,
|
ConversationSettings, EndpointUrl, Environment, GridConnection, Limits, LlmConnection,
|
||||||
SecretString, StdEnvironment, Timeouts,
|
MapEnvironment, OperatingMode, RemoteTlsSettings, SecretString, StdEnvironment, Timeouts,
|
||||||
};
|
};
|
||||||
|
pub use control_plane::{
|
||||||
|
AuditEventView, CONTROL_PROTOCOL_VERSION, ControlContext, ControlError, ControlErrorBody,
|
||||||
|
ControlErrorCode, ControlEvent, ControlEventKind, ControlFuture, ControlLimits, ControlPayload,
|
||||||
|
ControlPlane, ControlRequest, ControlRequestEnvelope, ControlResponseEnvelope, ControlRole,
|
||||||
|
ControlSubscription, ControlTarget, ConversationChannelView, HealthView,
|
||||||
|
InProcessControlClient, Page, PageRequest, PendingApprovalView, RuntimeView, ScheduledJobView,
|
||||||
|
SessionMetadataView, TcpControlClient, TcpControlConfig, TcpControlServer,
|
||||||
|
};
|
||||||
|
pub use control_runtime::{AgentControlTarget, RuntimeControlCommand};
|
||||||
pub use conversation::{
|
pub use conversation::{
|
||||||
ConversationChannel, ConversationClock, ConversationContext, ConversationError,
|
ConversationChannel, ConversationClock, ConversationContext, ConversationError,
|
||||||
ConversationKey, ConversationLimits, ConversationMetadata, ConversationPersistence,
|
ConversationKey, ConversationLimits, ConversationMetadata, ConversationPersistence,
|
||||||
@@ -81,10 +96,11 @@ pub use perception::{
|
|||||||
pub use policy::{
|
pub use policy::{
|
||||||
ActionOrigin, AllowedOrigins, ApprovalId, ApprovalRule, AuthenticatedPrincipal,
|
ActionOrigin, AllowedOrigins, ApprovalId, ApprovalRule, AuthenticatedPrincipal,
|
||||||
AuthorizedAction, BudgetLimits, Capability, FixedCost, Idempotency, MemoryPolicyAudit,
|
AuthorizedAction, BudgetLimits, Capability, FixedCost, Idempotency, MemoryPolicyAudit,
|
||||||
OriginClass, PolicyAuditError, PolicyAuditRecord, PolicyAuditSink, PolicyDisposition,
|
OriginClass, PendingApprovalMetadata, PolicyAuditError, PolicyAuditRecord, PolicyAuditSink,
|
||||||
PolicyError, PolicyEvaluation, PolicyFinalOutcome, PolicyGateway, PolicyLimits,
|
PolicyDisposition, PolicyError, PolicyEvaluation, PolicyFinalOutcome, PolicyGateway,
|
||||||
PolicyReasonCode, PolicyRequestContext, PolicySnapshot, PolicyTool, PolicyToolExecutor,
|
PolicyLimits, PolicyReasonCode, PolicyRequestContext, PolicySnapshot, PolicyTool,
|
||||||
ResourceCost, ResourceEstimator, Risk, SchedulerGrantId, UntrustedData, UntrustedSource,
|
PolicyToolExecutor, ResourceCost, ResourceEstimator, Risk, SchedulerGrantId, UntrustedData,
|
||||||
|
UntrustedSource,
|
||||||
};
|
};
|
||||||
pub use service::{AgentService, ServiceError, ServiceHandle, ServiceState};
|
pub use service::{AgentService, ServiceError, ServiceHandle, ServiceState};
|
||||||
pub use session::{
|
pub use session::{
|
||||||
|
|||||||
@@ -147,8 +147,10 @@ async fn run_live(
|
|||||||
run_once: bool,
|
run_once: bool,
|
||||||
) -> Result<(), Box<dyn Error>> {
|
) -> Result<(), Box<dyn Error>> {
|
||||||
use metacrate_grid_agent::{
|
use metacrate_grid_agent::{
|
||||||
GridSessionBackend, LibremetaverseClientOwner, SessionObservation, SessionState,
|
AgentControlTarget, BehaviorObservation, ControlEventKind, ControlPlane, ControlTarget,
|
||||||
SessionSupervisor,
|
GridSessionBackend, LibremetaverseClientOwner, OperatingMode, RuntimeControlCommand,
|
||||||
|
SessionControl, SessionObservation, SessionState, SessionSupervisor, TcpControlConfig,
|
||||||
|
TcpControlServer,
|
||||||
};
|
};
|
||||||
|
|
||||||
let connection = config.grid.clone().ok_or_else(|| {
|
let connection = config.grid.clone().ok_or_else(|| {
|
||||||
@@ -230,8 +232,56 @@ async fn run_live(
|
|||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
let (control_target, mut control_commands) = AgentControlTarget::new(
|
||||||
|
live.conversations.clone(),
|
||||||
|
live.policy.clone(),
|
||||||
|
live.audit.clone(),
|
||||||
|
live.behavior.ingress(),
|
||||||
|
config.limits.control_queue,
|
||||||
|
)?;
|
||||||
|
control_target.update_session(handle.status());
|
||||||
|
let erased_target: Arc<dyn ControlTarget> = control_target.clone();
|
||||||
|
let (control_plane, _integrated_client, control_server) = match config.mode {
|
||||||
|
OperatingMode::Integrated => {
|
||||||
|
let (plane, client) = ControlPlane::integrated(erased_target, config.control.limits)?;
|
||||||
|
(plane, Some(client), None)
|
||||||
|
}
|
||||||
|
OperatingMode::SplitService => {
|
||||||
|
let operator = config.control.operator_token.clone().ok_or_else(|| {
|
||||||
|
CliError("validated split configuration omitted its operator token".into())
|
||||||
|
})?;
|
||||||
|
let plane = ControlPlane::new(
|
||||||
|
erased_target,
|
||||||
|
operator,
|
||||||
|
config.control.observer_token.clone(),
|
||||||
|
config.control.limits,
|
||||||
|
)?;
|
||||||
|
let transport = TcpControlConfig {
|
||||||
|
listen: config.control.listen,
|
||||||
|
limits: config.control.limits,
|
||||||
|
};
|
||||||
|
let server = if let Some(tls) = &config.control.remote_tls {
|
||||||
|
eprintln!(
|
||||||
|
"WARNING: remote control is enabled with TLS; protect operator tokens and certificate keys"
|
||||||
|
);
|
||||||
|
TcpControlServer::bind_tls(plane.clone(), transport, tls.server_config()?).await?
|
||||||
|
} else {
|
||||||
|
TcpControlServer::bind(plane.clone(), transport).await?
|
||||||
|
};
|
||||||
|
println!(
|
||||||
|
"grid agent control plane listening on {}",
|
||||||
|
server.local_addr()
|
||||||
|
);
|
||||||
|
(plane, None, Some(server))
|
||||||
|
}
|
||||||
|
OperatingMode::OfflineFake => {
|
||||||
|
return Err(CliError("offline mode reached the live control plane".into()).into());
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
println!("grid agent session supervisor started; press Ctrl-C to stop");
|
println!("grid agent session supervisor started; press Ctrl-C to stop");
|
||||||
let mut signal_error = None;
|
let mut signal_error = None;
|
||||||
|
let mut control_failure = None;
|
||||||
loop {
|
loop {
|
||||||
tokio::select! {
|
tokio::select! {
|
||||||
signal = tokio::signal::ctrl_c() => {
|
signal = tokio::signal::ctrl_c() => {
|
||||||
@@ -243,6 +293,11 @@ async fn run_live(
|
|||||||
event = handle.next_observation() => {
|
event = handle.next_observation() => {
|
||||||
let Some(event) = event else { break; };
|
let Some(event) = event else { break; };
|
||||||
if let SessionObservation::Transition { status, reason, retry_in } = event {
|
if let SessionObservation::Transition { status, reason, retry_in } = event {
|
||||||
|
control_target.update_session(status);
|
||||||
|
control_plane.publish(ControlEventKind::StateChanged {
|
||||||
|
component: "session".into(),
|
||||||
|
state: status.state.as_str().into(),
|
||||||
|
});
|
||||||
println!(
|
println!(
|
||||||
"grid session state={} generation={} transport_connected={} agent_ready={} reason={reason:?} retry_in={retry_in:?}",
|
"grid session state={} generation={} transport_connected={} agent_ready={} reason={reason:?} retry_in={retry_in:?}",
|
||||||
status.state.as_str(),
|
status.state.as_str(),
|
||||||
@@ -262,19 +317,66 @@ async fn run_live(
|
|||||||
}
|
}
|
||||||
event = live.behavior.next_observation() => {
|
event = live.behavior.next_observation() => {
|
||||||
let Some(event) = event else { break; };
|
let Some(event) = event else { break; };
|
||||||
|
if let BehaviorObservation::Transition { to, .. } = &event {
|
||||||
|
control_target.update_behavior(*to);
|
||||||
|
control_plane.publish(ControlEventKind::StateChanged {
|
||||||
|
component: "behavior".into(),
|
||||||
|
state: format!("{to:?}").to_ascii_lowercase(),
|
||||||
|
});
|
||||||
|
}
|
||||||
println!("grid behavior event={event:?}");
|
println!("grid behavior event={event:?}");
|
||||||
}
|
}
|
||||||
|
command = control_commands.recv() => {
|
||||||
|
let Some(command) = command else { break; };
|
||||||
|
match command {
|
||||||
|
RuntimeControlCommand::Pause => {
|
||||||
|
if let Err(error) = handle.control(SessionControl::Pause).await {
|
||||||
|
control_failure = Some(error.to_string());
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
RuntimeControlCommand::Resume => {
|
||||||
|
if let Err(error) = handle.control(SessionControl::Resume).await {
|
||||||
|
control_failure = Some(error.to_string());
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
RuntimeControlCommand::ForceReconnect => {
|
||||||
|
if let Err(error) = handle.control(SessionControl::ForceReconnect).await {
|
||||||
|
control_failure = Some(error.to_string());
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
RuntimeControlCommand::OperatorMessage { message } => {
|
||||||
|
control_plane.publish(ControlEventKind::StateChanged {
|
||||||
|
component: "operator_message".into(),
|
||||||
|
state: format!("accepted_{}_bytes", message.len()),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
RuntimeControlCommand::GracefulShutdown => break,
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
control_target.mark_stopping();
|
||||||
let session_result = handle.shutdown().await;
|
let session_result = handle.shutdown().await;
|
||||||
let interaction_result = live.interaction.shutdown().await;
|
let interaction_result = live.interaction.shutdown().await;
|
||||||
let behavior_result = live.behavior.shutdown().await;
|
let behavior_result = live.behavior.shutdown().await;
|
||||||
|
let control_result = if let Some(server) = control_server {
|
||||||
|
server.shutdown().await
|
||||||
|
} else {
|
||||||
|
Ok(())
|
||||||
|
};
|
||||||
session_result?;
|
session_result?;
|
||||||
interaction_result?;
|
interaction_result?;
|
||||||
behavior_result?;
|
behavior_result?;
|
||||||
|
control_result?;
|
||||||
if let Some(error) = signal_error {
|
if let Some(error) = signal_error {
|
||||||
return Err(error.into());
|
return Err(error.into());
|
||||||
}
|
}
|
||||||
|
if let Some(error) = control_failure {
|
||||||
|
return Err(CliError(format!("control command failed: {error}")).into());
|
||||||
|
}
|
||||||
println!("grid agent stopped cleanly");
|
println!("grid agent stopped cleanly");
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
@@ -286,9 +388,13 @@ struct LiveInteractions {
|
|||||||
perception_observations:
|
perception_observations:
|
||||||
tokio::sync::mpsc::Receiver<metacrate_grid_agent::PerceptionObservation>,
|
tokio::sync::mpsc::Receiver<metacrate_grid_agent::PerceptionObservation>,
|
||||||
behavior: metacrate_grid_agent::BehaviorHandle,
|
behavior: metacrate_grid_agent::BehaviorHandle,
|
||||||
|
conversations: Arc<metacrate_grid_agent::ConversationStore>,
|
||||||
|
policy: Arc<metacrate_grid_agent::PolicyGateway>,
|
||||||
|
audit: Arc<metacrate_grid_agent::MemoryPolicyAudit>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(feature = "live-grid")]
|
#[cfg(feature = "live-grid")]
|
||||||
|
#[allow(clippy::too_many_lines)]
|
||||||
fn start_live_interactions(
|
fn start_live_interactions(
|
||||||
config: &metacrate_grid_agent::AgentConfig,
|
config: &metacrate_grid_agent::AgentConfig,
|
||||||
owner: &metacrate_grid_agent::LibremetaverseClientOwner,
|
owner: &metacrate_grid_agent::LibremetaverseClientOwner,
|
||||||
@@ -347,7 +453,7 @@ fn start_live_interactions(
|
|||||||
config.authorized_avatar_uuids.clone(),
|
config.authorized_avatar_uuids.clone(),
|
||||||
tools,
|
tools,
|
||||||
PolicyLimits::default(),
|
PolicyLimits::default(),
|
||||||
audit,
|
audit.clone(),
|
||||||
)?);
|
)?);
|
||||||
let loop_limits = ToolLoopLimits {
|
let loop_limits = ToolLoopLimits {
|
||||||
max_tool_calls_per_turn: config.limits.max_tool_calls,
|
max_tool_calls_per_turn: config.limits.max_tool_calls,
|
||||||
@@ -366,7 +472,7 @@ fn start_live_interactions(
|
|||||||
Arc::new(AuthorizedBackendRouter::new(routes)?);
|
Arc::new(AuthorizedBackendRouter::new(routes)?);
|
||||||
let responder = Arc::new(PolicyLlmResponder::new(
|
let responder = Arc::new(PolicyLlmResponder::new(
|
||||||
client,
|
client,
|
||||||
gateway,
|
gateway.clone(),
|
||||||
routed_backend,
|
routed_backend,
|
||||||
loop_limits,
|
loop_limits,
|
||||||
now,
|
now,
|
||||||
@@ -377,7 +483,7 @@ fn start_live_interactions(
|
|||||||
config.interaction.clone(),
|
config.interaction.clone(),
|
||||||
libremetaverse_types::UUID::zero(),
|
libremetaverse_types::UUID::zero(),
|
||||||
config.authorized_avatar_uuids.clone(),
|
config.authorized_avatar_uuids.clone(),
|
||||||
conversation,
|
conversation.clone(),
|
||||||
responder,
|
responder,
|
||||||
sink,
|
sink,
|
||||||
config.limits.grid_event_queue,
|
config.limits.grid_event_queue,
|
||||||
@@ -391,5 +497,8 @@ fn start_live_interactions(
|
|||||||
perception: perception_ingress,
|
perception: perception_ingress,
|
||||||
perception_observations,
|
perception_observations,
|
||||||
behavior,
|
behavior,
|
||||||
|
conversations: conversation,
|
||||||
|
policy: gateway,
|
||||||
|
audit,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ use std::collections::{BTreeMap, BTreeSet, VecDeque};
|
|||||||
use std::error::Error;
|
use std::error::Error;
|
||||||
use std::fmt;
|
use std::fmt;
|
||||||
use std::sync::{Arc, Mutex};
|
use std::sync::{Arc, Mutex};
|
||||||
|
use std::time::{SystemTime, UNIX_EPOCH};
|
||||||
|
|
||||||
const MAX_POLICY_TOOLS: usize = 64;
|
const MAX_POLICY_TOOLS: usize = 64;
|
||||||
const MAX_AUTHORIZED_AVATARS: usize = 1_024;
|
const MAX_AUTHORIZED_AVATARS: usize = 1_024;
|
||||||
@@ -238,6 +239,16 @@ pub struct ResourceCost {
|
|||||||
pub build_prims: u64,
|
pub build_prims: u64,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Secret-free operator projection of one pending approval.
|
||||||
|
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||||
|
pub struct PendingApprovalMetadata {
|
||||||
|
pub id: ApprovalId,
|
||||||
|
pub principal: String,
|
||||||
|
pub tool: String,
|
||||||
|
pub expires_at: u64,
|
||||||
|
pub cost: ResourceCost,
|
||||||
|
}
|
||||||
|
|
||||||
impl ResourceCost {
|
impl ResourceCost {
|
||||||
#[must_use]
|
#[must_use]
|
||||||
pub const fn one_call() -> Self {
|
pub const fn one_call() -> Self {
|
||||||
@@ -514,6 +525,7 @@ pub enum PolicyFinalOutcome {
|
|||||||
|
|
||||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||||
pub struct PolicyAuditRecord {
|
pub struct PolicyAuditRecord {
|
||||||
|
pub recorded_unix_millis: u64,
|
||||||
pub authorization_id: Option<u64>,
|
pub authorization_id: Option<u64>,
|
||||||
pub approval_id: Option<ApprovalId>,
|
pub approval_id: Option<ApprovalId>,
|
||||||
pub origin: OriginClass,
|
pub origin: OriginClass,
|
||||||
@@ -581,6 +593,18 @@ impl PolicyAuditSink for MemoryPolicyAudit {
|
|||||||
#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)]
|
#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)]
|
||||||
pub struct ApprovalId(u64);
|
pub struct ApprovalId(u64);
|
||||||
|
|
||||||
|
impl ApprovalId {
|
||||||
|
#[must_use]
|
||||||
|
pub const fn from_raw(value: u64) -> Option<Self> {
|
||||||
|
if value == 0 { None } else { Some(Self(value)) }
|
||||||
|
}
|
||||||
|
|
||||||
|
#[must_use]
|
||||||
|
pub const fn get(self) -> u64 {
|
||||||
|
self.0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)]
|
#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)]
|
||||||
pub struct SchedulerGrantId(u64);
|
pub struct SchedulerGrantId(u64);
|
||||||
|
|
||||||
@@ -588,6 +612,7 @@ pub struct SchedulerGrantId(u64);
|
|||||||
enum ApprovalStatus {
|
enum ApprovalStatus {
|
||||||
Pending,
|
Pending,
|
||||||
Granted,
|
Granted,
|
||||||
|
Denied,
|
||||||
Consumed,
|
Consumed,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -820,6 +845,31 @@ impl PolicyGateway {
|
|||||||
PolicySnapshot(lock(&self.state).clone())
|
PolicySnapshot(lock(&self.state).clone())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Returns bounded, content-free approval metadata for operator UIs.
|
||||||
|
#[must_use]
|
||||||
|
pub fn pending_approvals(&self, now: u64) -> Vec<PendingApprovalMetadata> {
|
||||||
|
lock(&self.state)
|
||||||
|
.approvals
|
||||||
|
.values()
|
||||||
|
.filter(|approval| {
|
||||||
|
approval.status == ApprovalStatus::Pending && approval.expires_at > now
|
||||||
|
})
|
||||||
|
.map(|approval| PendingApprovalMetadata {
|
||||||
|
id: approval.id,
|
||||||
|
principal: approval.principal.audit_label(),
|
||||||
|
tool: approval.tool.clone(),
|
||||||
|
expires_at: approval.expires_at,
|
||||||
|
cost: approval.cost,
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Current global budget use for a redacted control-plane projection.
|
||||||
|
#[must_use]
|
||||||
|
pub fn global_budget_usage(&self) -> ResourceCost {
|
||||||
|
lock(&self.state).global_used
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
pub(crate) fn pending_approval_count(&self) -> usize {
|
pub(crate) fn pending_approval_count(&self) -> usize {
|
||||||
lock(&self.state)
|
lock(&self.state)
|
||||||
@@ -1064,7 +1114,9 @@ impl PolicyGateway {
|
|||||||
match approval.status {
|
match approval.status {
|
||||||
ApprovalStatus::Pending => Some(PolicyReasonCode::ApprovalNotGranted),
|
ApprovalStatus::Pending => Some(PolicyReasonCode::ApprovalNotGranted),
|
||||||
ApprovalStatus::Granted => None,
|
ApprovalStatus::Granted => None,
|
||||||
ApprovalStatus::Consumed => Some(PolicyReasonCode::ApprovalReplayed),
|
ApprovalStatus::Denied | ApprovalStatus::Consumed => {
|
||||||
|
Some(PolicyReasonCode::ApprovalReplayed)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
if let Some(reason) = reason {
|
if let Some(reason) = reason {
|
||||||
@@ -1121,8 +1173,12 @@ impl PolicyGateway {
|
|||||||
}
|
}
|
||||||
if state.approvals.len() == MAX_APPROVALS {
|
if state.approvals.len() == MAX_APPROVALS {
|
||||||
let removable = state.approvals.iter().find_map(|(id, approval)| {
|
let removable = state.approvals.iter().find_map(|(id, approval)| {
|
||||||
(approval.expires_at <= now || approval.status == ApprovalStatus::Consumed)
|
(approval.expires_at <= now
|
||||||
.then_some(*id)
|
|| matches!(
|
||||||
|
approval.status,
|
||||||
|
ApprovalStatus::Denied | ApprovalStatus::Consumed
|
||||||
|
))
|
||||||
|
.then_some(*id)
|
||||||
});
|
});
|
||||||
let Some(removable) = removable else {
|
let Some(removable) = removable else {
|
||||||
drop(state);
|
drop(state);
|
||||||
@@ -1323,6 +1379,40 @@ impl PolicyGateway {
|
|||||||
Ok(PolicyReasonCode::Allowed)
|
Ok(PolicyReasonCode::Allowed)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn deny_approval(
|
||||||
|
&self,
|
||||||
|
id: ApprovalId,
|
||||||
|
_operator: &AuthenticatedPrincipal,
|
||||||
|
now: u64,
|
||||||
|
) -> Result<PolicyReasonCode, PolicyError> {
|
||||||
|
let mut state = lock(&self.state);
|
||||||
|
let Some(approval) = state.approvals.get_mut(&id) else {
|
||||||
|
return Ok(PolicyReasonCode::ApprovalUnknown);
|
||||||
|
};
|
||||||
|
if approval.expires_at <= now {
|
||||||
|
return Ok(PolicyReasonCode::ApprovalExpired);
|
||||||
|
}
|
||||||
|
if approval.status != ApprovalStatus::Pending {
|
||||||
|
return Ok(PolicyReasonCode::ApprovalReplayed);
|
||||||
|
}
|
||||||
|
let audit = audit_record(
|
||||||
|
&approval.context,
|
||||||
|
approval.origin,
|
||||||
|
&approval.principal,
|
||||||
|
BoundedText::new("policy.tool", approval.tool.clone())?,
|
||||||
|
None,
|
||||||
|
Some(id),
|
||||||
|
PolicyDisposition::Denied,
|
||||||
|
PolicyReasonCode::ApprovalNotGranted,
|
||||||
|
approval.cost,
|
||||||
|
approval.arguments_hash,
|
||||||
|
PolicyFinalOutcome::Denied,
|
||||||
|
)?;
|
||||||
|
self.emit(audit)?;
|
||||||
|
approval.status = ApprovalStatus::Denied;
|
||||||
|
Ok(PolicyReasonCode::ApprovalNotGranted)
|
||||||
|
}
|
||||||
|
|
||||||
/// Consumes the originating authorization so it cannot be replayed to
|
/// Consumes the originating authorization so it cannot be replayed to
|
||||||
/// create more than one scheduler grant.
|
/// create more than one scheduler grant.
|
||||||
#[allow(clippy::needless_pass_by_value)]
|
#[allow(clippy::needless_pass_by_value)]
|
||||||
@@ -1621,6 +1711,11 @@ fn audit_record(
|
|||||||
final_outcome: PolicyFinalOutcome,
|
final_outcome: PolicyFinalOutcome,
|
||||||
) -> Result<PolicyAuditRecord, PolicyError> {
|
) -> Result<PolicyAuditRecord, PolicyError> {
|
||||||
Ok(PolicyAuditRecord {
|
Ok(PolicyAuditRecord {
|
||||||
|
recorded_unix_millis: SystemTime::now()
|
||||||
|
.duration_since(UNIX_EPOCH)
|
||||||
|
.map_or(0, |duration| {
|
||||||
|
u64::try_from(duration.as_millis()).unwrap_or(u64::MAX)
|
||||||
|
}),
|
||||||
authorization_id,
|
authorization_id,
|
||||||
approval_id,
|
approval_id,
|
||||||
origin,
|
origin,
|
||||||
|
|||||||
@@ -411,6 +411,7 @@ fn approval_tool() -> PolicyTool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
#[allow(clippy::too_many_lines)] // One approval lifecycle includes deny and expiry paths.
|
||||||
fn approvals_bind_principal_canonical_arguments_expiry_and_one_execution() {
|
fn approvals_bind_principal_canonical_arguments_expiry_and_one_execution() {
|
||||||
let (gateway, audit) = gateway_with(vec![approval_tool()], PolicyLimits::default());
|
let (gateway, audit) = gateway_with(vec![approval_tool()], PolicyLimits::default());
|
||||||
let requester = context(OriginClass::AuthorizedIm);
|
let requester = context(OriginClass::AuthorizedIm);
|
||||||
@@ -507,6 +508,24 @@ fn approvals_bind_principal_canonical_arguments_expiry_and_one_execution() {
|
|||||||
.expect("expired result"),
|
.expect("expired result"),
|
||||||
PolicyReasonCode::ApprovalExpired
|
PolicyReasonCode::ApprovalExpired
|
||||||
);
|
);
|
||||||
|
let denied = gateway
|
||||||
|
.evaluate(&requester, &proposed, &arguments, None, 600)
|
||||||
|
.expect("denial pending")
|
||||||
|
.approval_id
|
||||||
|
.expect("approval ID");
|
||||||
|
assert_eq!(gateway.pending_approvals(600).len(), 1);
|
||||||
|
assert_eq!(
|
||||||
|
gateway.deny_approval(denied, &operator, 601).expect("deny"),
|
||||||
|
PolicyReasonCode::ApprovalNotGranted
|
||||||
|
);
|
||||||
|
assert!(gateway.pending_approvals(601).is_empty());
|
||||||
|
assert_eq!(
|
||||||
|
gateway
|
||||||
|
.evaluate(&requester, &proposed, &arguments, Some(denied), 601)
|
||||||
|
.expect("denied approval cannot execute")
|
||||||
|
.reason,
|
||||||
|
PolicyReasonCode::ApprovalReplayed
|
||||||
|
);
|
||||||
assert!(
|
assert!(
|
||||||
audit
|
audit
|
||||||
.snapshot()
|
.snapshot()
|
||||||
|
|||||||
@@ -2,14 +2,16 @@ use std::collections::BTreeSet;
|
|||||||
use std::fs;
|
use std::fs;
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
|
|
||||||
const ALLOWED_DEPENDENCIES: [&str; 8] = [
|
const ALLOWED_DEPENDENCIES: [&str; 10] = [
|
||||||
"libremetaverse",
|
"libremetaverse",
|
||||||
"libremetaverse-types",
|
"libremetaverse-types",
|
||||||
"reqwest",
|
"reqwest",
|
||||||
|
"rustls",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
"sha2",
|
"sha2",
|
||||||
"tokio",
|
"tokio",
|
||||||
|
"tokio-rustls",
|
||||||
"url",
|
"url",
|
||||||
];
|
];
|
||||||
|
|
||||||
@@ -46,7 +48,7 @@ fn runtime_source_has_no_subprocess_or_native_abi_escape_hatch() {
|
|||||||
let mut files = Vec::with_capacity(20);
|
let mut files = Vec::with_capacity(20);
|
||||||
collect_rust_files(&source, &mut files);
|
collect_rust_files(&source, &mut files);
|
||||||
assert!(
|
assert!(
|
||||||
files.len() <= 20,
|
files.len() <= 24,
|
||||||
"source-file count needs a reviewed bound update"
|
"source-file count needs a reviewed bound update"
|
||||||
);
|
);
|
||||||
for path in files {
|
for path in files {
|
||||||
|
|||||||
91
docs/grid-agent-control-plane.md
Normal file
91
docs/grid-agent-control-plane.md
Normal file
@@ -0,0 +1,91 @@
|
|||||||
|
# Grid-agent control plane v1
|
||||||
|
|
||||||
|
The grid agent exposes one transport-neutral request/response and event API.
|
||||||
|
Integrated clients receive an `InProcessControlClient`; split clients use the
|
||||||
|
same `ControlRequestEnvelope`, `ControlResponseEnvelope`, and `ControlEvent`
|
||||||
|
types over TCP. No terminal, Unix socket, named-pipe, permission-bit, or
|
||||||
|
platform-specific type appears in the core API.
|
||||||
|
|
||||||
|
## Transport and negotiation
|
||||||
|
|
||||||
|
TCP frames are a four-byte unsigned big-endian length followed by one UTF-8
|
||||||
|
JSON value. Empty, partial, malformed, idle, and oversized frames close the
|
||||||
|
connection within configured deadlines. The first frame must be:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{"frame":"hello","body":{"version":1,"token":"separate-operator-capability"}}
|
||||||
|
```
|
||||||
|
|
||||||
|
The server answers with a `hello` containing version `1`, the authenticated
|
||||||
|
`operator` or `observer` role, or a typed error. Every later client frame is a
|
||||||
|
`request`; server frames are `response` or `event`. Request IDs contain at most
|
||||||
|
96 ASCII identifier bytes and cannot be replayed on a connection. A caller can
|
||||||
|
cancel an active request by its ID. Responses and events can arrive in either
|
||||||
|
order, so clients correlate responses by request ID.
|
||||||
|
|
||||||
|
Plain TCP binds only to IPv4 or IPv6 loopback. Remote control is off by default
|
||||||
|
and a non-loopback address is rejected unless both DER certificate and private
|
||||||
|
key paths are explicitly configured. Remote mode uses Rustls before the hello
|
||||||
|
exchange and emits an operator warning at startup. The TLS client API requires
|
||||||
|
the embedding client to supply its certificate roots and server-name policy.
|
||||||
|
Every split connection must still authenticate with a dedicated control token;
|
||||||
|
observer and operator tokens must differ and configuration rejects reuse of an
|
||||||
|
LLM API key or grid password. Tokens are absent from protocol errors, audit,
|
||||||
|
serialization traits, and diagnostics.
|
||||||
|
|
||||||
|
Remote service configuration uses the JSON fields
|
||||||
|
`control.remote_tls_certificate_der` and
|
||||||
|
`control.remote_tls_private_key_der` together with a non-loopback
|
||||||
|
`control.listen` value. Both files are bounded, non-symlink DER files. Omitting
|
||||||
|
either field, or selecting a non-loopback address without both, fails
|
||||||
|
configuration before a listener is created.
|
||||||
|
|
||||||
|
## Requests
|
||||||
|
|
||||||
|
The JSON request envelope is stable and versioned. For example:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{"version":1,"request_id":"health-1","request":{"method":"health"}}
|
||||||
|
```
|
||||||
|
|
||||||
|
Observers can call `health`, `runtime`, `list_sessions`,
|
||||||
|
`list_scheduled_jobs`, `list_pending_approvals`, `list_audit_events`, and
|
||||||
|
`subscribe_events`. Operators can additionally call `cancel_request`,
|
||||||
|
`pause_autonomy`, `resume_autonomy`, `cancel_action`, `decide_approval`,
|
||||||
|
`force_reconnect`, `expire_conversation`, `set_roaming_job`,
|
||||||
|
`inject_operator_message`, and `graceful_shutdown`. Cancellation is a mutation
|
||||||
|
and is operator-only. List requests use an opaque numeric cursor and a page
|
||||||
|
size of 1 through 100.
|
||||||
|
|
||||||
|
Runtime projections contain lifecycle/readiness, session generation, safe
|
||||||
|
region and pose fields when known, behavior mode, control-queue utilization,
|
||||||
|
and aggregate budget use. Conversation responses contain metadata only. Audit
|
||||||
|
and approval responses exclude arguments, prompt contents, credentials,
|
||||||
|
authorization headers, capability URLs, model reasoning, and filesystem data.
|
||||||
|
`cancel_action` binds to the exact bounded action ID carried by behavior audit
|
||||||
|
observations; it can cancel a queued or executing embodied action without
|
||||||
|
preempting unrelated work. The built-in roaming job ID is `default-roaming`.
|
||||||
|
|
||||||
|
Errors are typed as `authentication_failed`, `version_mismatch`,
|
||||||
|
`permission_denied`, `invalid_request`, `replay`, `not_found`, `conflict`,
|
||||||
|
`cancelled`, `timed_out`, `busy`, `backpressure`, `frame_too_large`,
|
||||||
|
`idle_timeout`, `transport_closed`, or `internal`, with a bounded safe message
|
||||||
|
and a retryable flag.
|
||||||
|
|
||||||
|
## Events, bounds, and reconnect
|
||||||
|
|
||||||
|
Events carry monotonically increasing sequence numbers. A subscription can
|
||||||
|
resume after its last observed sequence. If retained history no longer covers
|
||||||
|
that point, its first item is an explicit `gap` with `first_available` and
|
||||||
|
`last_missed`, followed by retained events. Mutation events record the
|
||||||
|
authenticated role/connection principal, operation name, and completed or
|
||||||
|
rejected outcome, never request contents.
|
||||||
|
|
||||||
|
Connections, unauthenticated handshakes, in-flight request tasks,
|
||||||
|
subscriptions, command/event queues, replay history, event history, event
|
||||||
|
bytes, event rate, frames, pages, idle time, request time, and writes all have
|
||||||
|
validated hard bounds. A full event or writer queue disconnects the slow
|
||||||
|
consumer instead of blocking the headless service. With zero clients, event
|
||||||
|
publication retains only the configured history and creates no background
|
||||||
|
work. Server shutdown cancels and joins all listener, connection, request, and
|
||||||
|
writer tasks within its deadline.
|
||||||
Reference in New Issue
Block a user