feat(grid-agent): add portable control plane (#126)
This commit is contained in:
2
Cargo.lock
generated
2
Cargo.lock
generated
@@ -2201,10 +2201,12 @@ dependencies = [
|
||||
"libremetaverse",
|
||||
"libremetaverse-types",
|
||||
"reqwest",
|
||||
"rustls",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha2 0.11.0",
|
||||
"tokio",
|
||||
"tokio-rustls",
|
||||
"url",
|
||||
]
|
||||
|
||||
|
||||
@@ -12,10 +12,12 @@ publish = false
|
||||
libremetaverse = { version = "0.0.1", path = "../libremetaverse", default-features = false, optional = true }
|
||||
libremetaverse-types = { version = "0.0.1", path = "../libremetaverse-types" }
|
||||
reqwest = { version = "0.13.4", default-features = false, features = ["rustls"] }
|
||||
rustls = { version = "0.23.43", default-features = false, features = ["aws_lc_rs", "std", "tls12"] }
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
sha2 = "0.11"
|
||||
tokio = { version = "1.53.1", features = ["macros", "rt", "sync", "time"] }
|
||||
tokio-rustls = { version = "0.26.4", default-features = false, features = ["aws_lc_rs", "tls12"] }
|
||||
url = "2.5.8"
|
||||
|
||||
[dev-dependencies]
|
||||
|
||||
@@ -22,7 +22,10 @@ optional JSON file, its referenced secret files, then environment values (an
|
||||
environment-referenced secret file is below a direct environment secret).
|
||||
Supported secret environment variables are
|
||||
`METACRATE_AGENT_LLM_API_KEY[_FILE]` and
|
||||
`METACRATE_AGENT_GRID_PASSWORD[_FILE]`. Secret files must be bounded regular,
|
||||
`METACRATE_AGENT_GRID_PASSWORD[_FILE]`. Split control uses the separate
|
||||
`METACRATE_AGENT_CONTROL_OPERATOR_TOKEN[_FILE]` and optional
|
||||
`METACRATE_AGENT_CONTROL_OBSERVER_TOKEN[_FILE]`, plus
|
||||
`METACRATE_AGENT_CONTROL_LISTEN`. Secret files must be bounded regular,
|
||||
non-symlink UTF-8 files containing one line. Operators must restrict their OS
|
||||
ACLs to the service identity; the core uses only portable `std::fs` checks and
|
||||
does not assume Unix permission bits.
|
||||
@@ -51,3 +54,6 @@ shutdown deadline are specified in
|
||||
Per-avatar/channel expiry, compaction, redaction, optional atomic persistence,
|
||||
and metadata-only operator controls are specified in
|
||||
[`../../docs/grid-agent-conversation.md`](../../docs/grid-agent-conversation.md).
|
||||
The versioned integrated/TCP protocol, roles, framing, bounds, TLS remote-mode
|
||||
requirements, events, and management methods are specified in
|
||||
[`../../docs/grid-agent-control-plane.md`](../../docs/grid-agent-control-plane.md).
|
||||
|
||||
@@ -19,8 +19,8 @@ use serde_json::{Map, Value, json};
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::error::Error;
|
||||
use std::fmt;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::sync::{Arc, Mutex, MutexGuard};
|
||||
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
|
||||
use tokio::sync::{mpsc, oneshot, watch};
|
||||
use tokio::task::JoinHandle;
|
||||
@@ -36,6 +36,7 @@ pub const CURRENT_POSE_TOOL: &str = "behavior_current_pose";
|
||||
|
||||
const WALK_POLL: Duration = Duration::from_millis(250);
|
||||
const TURN_INTERVAL: Duration = Duration::from_millis(40);
|
||||
const MAX_ACTION_ID_BYTES: usize = 96;
|
||||
const MAX_TURN_STEP_DEGREES: f64 = 30.0;
|
||||
const ARRIVAL_METERS: f64 = 0.5;
|
||||
const STUCK_PROGRESS_METERS: f64 = 0.05;
|
||||
@@ -196,6 +197,7 @@ pub enum BehaviorObservation {
|
||||
trigger: BehaviorTrigger,
|
||||
},
|
||||
Action {
|
||||
action_id: String,
|
||||
generation: Option<u64>,
|
||||
region_id: Option<UUID>,
|
||||
action: String,
|
||||
@@ -243,15 +245,23 @@ impl BehaviorAction {
|
||||
}
|
||||
|
||||
struct ActionRequest {
|
||||
action_id: String,
|
||||
action: BehaviorAction,
|
||||
trigger: BehaviorTrigger,
|
||||
policy: BehaviorPolicyResult,
|
||||
reply: oneshot::Sender<Result<Value, BehaviorError>>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct ActionRegistry {
|
||||
active: BTreeSet<String>,
|
||||
cancelled: BTreeSet<String>,
|
||||
}
|
||||
|
||||
enum Command {
|
||||
Action(ActionRequest),
|
||||
Attention {
|
||||
action_id: String,
|
||||
delivery_id: String,
|
||||
avatar_id: UUID,
|
||||
reply: oneshot::Sender<Result<(), BehaviorError>>,
|
||||
@@ -266,6 +276,10 @@ pub struct BehaviorIngress {
|
||||
ready: watch::Sender<Option<ReadyState>>,
|
||||
paused: watch::Sender<bool>,
|
||||
emergency: watch::Sender<bool>,
|
||||
action_cancel: watch::Sender<u64>,
|
||||
actions: Arc<Mutex<ActionRegistry>>,
|
||||
next_action: Arc<AtomicU64>,
|
||||
action_capacity: usize,
|
||||
}
|
||||
|
||||
impl fmt::Debug for BehaviorIngress {
|
||||
@@ -313,6 +327,28 @@ impl BehaviorIngress {
|
||||
let _ = self.emergency.send(false);
|
||||
}
|
||||
|
||||
/// Cancels one exact queued or executing behavior action ID.
|
||||
pub fn cancel_action(&self, action_id: &str) -> Result<bool, BehaviorError> {
|
||||
if !valid_action_id(action_id) {
|
||||
return Err(BehaviorError::InvalidArguments);
|
||||
}
|
||||
let cancelled = {
|
||||
let mut actions = lock_actions(&self.actions);
|
||||
if actions.active.contains(action_id) {
|
||||
actions.cancelled.insert(action_id.to_owned());
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
};
|
||||
if cancelled {
|
||||
self.action_cancel.send_modify(|generation| {
|
||||
*generation = generation.saturating_add(1);
|
||||
});
|
||||
}
|
||||
Ok(cancelled)
|
||||
}
|
||||
|
||||
/// Marks a separately authorized scheduler task as roaming. The controller
|
||||
/// itself never invents a roaming route or exposes follow/wander behavior.
|
||||
pub fn set_roaming(&self, roaming: bool) -> Result<(), BehaviorError> {
|
||||
@@ -323,20 +359,30 @@ impl BehaviorIngress {
|
||||
|
||||
async fn request(
|
||||
&self,
|
||||
action_id: String,
|
||||
action: BehaviorAction,
|
||||
trigger: BehaviorTrigger,
|
||||
policy: BehaviorPolicyResult,
|
||||
) -> Result<Value, BehaviorError> {
|
||||
if !self.register_action(&action_id) {
|
||||
return Err(BehaviorError::InvalidArguments);
|
||||
}
|
||||
let (reply, receive) = oneshot::channel();
|
||||
self.commands
|
||||
if self
|
||||
.commands
|
||||
.send(Command::Action(ActionRequest {
|
||||
action_id: action_id.clone(),
|
||||
action,
|
||||
trigger,
|
||||
policy,
|
||||
reply,
|
||||
}))
|
||||
.await
|
||||
.map_err(|_| BehaviorError::QueueClosed)?;
|
||||
.is_err()
|
||||
{
|
||||
self.unregister_action(&action_id);
|
||||
return Err(BehaviorError::QueueClosed);
|
||||
}
|
||||
receive.await.map_err(|_| BehaviorError::QueueClosed)?
|
||||
}
|
||||
|
||||
@@ -345,17 +391,44 @@ impl BehaviorIngress {
|
||||
delivery_id: String,
|
||||
avatar_id: UUID,
|
||||
) -> Result<(), BehaviorError> {
|
||||
let action_id = format!(
|
||||
"attention-{}",
|
||||
self.next_action.fetch_add(1, Ordering::Relaxed)
|
||||
);
|
||||
if !self.register_action(&action_id) {
|
||||
return Err(BehaviorError::InvalidArguments);
|
||||
}
|
||||
let (reply, receive) = oneshot::channel();
|
||||
self.commands
|
||||
if self
|
||||
.commands
|
||||
.send(Command::Attention {
|
||||
action_id: action_id.clone(),
|
||||
delivery_id,
|
||||
avatar_id,
|
||||
reply,
|
||||
})
|
||||
.await
|
||||
.map_err(|_| BehaviorError::QueueClosed)?;
|
||||
.is_err()
|
||||
{
|
||||
self.unregister_action(&action_id);
|
||||
return Err(BehaviorError::QueueClosed);
|
||||
}
|
||||
receive.await.map_err(|_| BehaviorError::QueueClosed)?
|
||||
}
|
||||
|
||||
fn register_action(&self, action_id: &str) -> bool {
|
||||
if !valid_action_id(action_id) {
|
||||
return false;
|
||||
}
|
||||
let mut actions = lock_actions(&self.actions);
|
||||
actions.active.len() < self.action_capacity && actions.active.insert(action_id.to_owned())
|
||||
}
|
||||
|
||||
fn unregister_action(&self, action_id: &str) {
|
||||
let mut actions = lock_actions(&self.actions);
|
||||
actions.active.remove(action_id);
|
||||
actions.cancelled.remove(action_id);
|
||||
}
|
||||
}
|
||||
|
||||
impl ResponsePacer for BehaviorIngress {
|
||||
@@ -485,12 +558,18 @@ impl BehaviorController {
|
||||
let (ready, ready_rx) = watch::channel(None);
|
||||
let (paused, paused_rx) = watch::channel(false);
|
||||
let (emergency, emergency_rx) = watch::channel(false);
|
||||
let (action_cancel, action_cancel_rx) = watch::channel(0_u64);
|
||||
let actions = Arc::new(Mutex::new(ActionRegistry::default()));
|
||||
let (observations, observation_rx) = mpsc::channel(self.observation_capacity);
|
||||
let ingress = BehaviorIngress {
|
||||
commands,
|
||||
ready,
|
||||
paused,
|
||||
emergency,
|
||||
action_cancel,
|
||||
actions: actions.clone(),
|
||||
next_action: Arc::new(AtomicU64::new(1)),
|
||||
action_capacity: self.queue_capacity,
|
||||
};
|
||||
let shutdown_timeout = self.shutdown_timeout;
|
||||
let task = tokio::spawn(run_actor(
|
||||
@@ -499,6 +578,8 @@ impl BehaviorController {
|
||||
ready_rx,
|
||||
paused_rx,
|
||||
emergency_rx,
|
||||
action_cancel_rx,
|
||||
actions,
|
||||
observations,
|
||||
));
|
||||
BehaviorHandle {
|
||||
@@ -547,13 +628,15 @@ impl BehaviorRandom for SystemBehaviorRandom {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_lines)] // Lifecycle, preemption, idle, and commands share one ordered owner.
|
||||
#[allow(clippy::too_many_arguments, clippy::too_many_lines)] // One ordered lifecycle owner.
|
||||
async fn run_actor(
|
||||
controller: BehaviorController,
|
||||
mut commands: mpsc::Receiver<Command>,
|
||||
mut ready: watch::Receiver<Option<ReadyState>>,
|
||||
mut paused: watch::Receiver<bool>,
|
||||
mut emergency: watch::Receiver<bool>,
|
||||
action_cancel: watch::Receiver<u64>,
|
||||
actions: Arc<Mutex<ActionRegistry>>,
|
||||
observations: mpsc::Sender<BehaviorObservation>,
|
||||
) {
|
||||
let mut mode = BehaviorMode::Offline;
|
||||
@@ -616,8 +699,16 @@ async fn run_actor(
|
||||
() = &mut idle, if controller.settings.idle_look_enabled => {
|
||||
if mode == BehaviorMode::Available {
|
||||
let (reply, _) = oneshot::channel();
|
||||
let request = ActionRequest { action: BehaviorAction::LookAround, trigger: BehaviorTrigger::IdleTimer, policy: BehaviorPolicyResult::InternalIdle, reply };
|
||||
execute_request(&controller, request, &mut mode, &mut last_action, &ready, &paused, &emergency, &observations).await;
|
||||
let action_id = format!("idle-{}", unix_millis_now());
|
||||
let registered = {
|
||||
let mut registry = lock_actions(&actions);
|
||||
registry.active.len() < controller.queue_capacity
|
||||
&& registry.active.insert(action_id.clone())
|
||||
};
|
||||
if registered {
|
||||
let request = ActionRequest { action_id, action: BehaviorAction::LookAround, trigger: BehaviorTrigger::IdleTimer, policy: BehaviorPolicyResult::InternalIdle, reply };
|
||||
execute_request(&controller, request, &mut mode, &mut last_action, &ready, &paused, &emergency, &action_cancel, &actions, &observations).await;
|
||||
}
|
||||
}
|
||||
idle.as_mut().reset(tokio::time::Instant::now() + controller.settings.idle_interval);
|
||||
}
|
||||
@@ -642,9 +733,10 @@ async fn run_actor(
|
||||
};
|
||||
transition(&observations, &mut mode, next, BehaviorTrigger::IdleTimer).await;
|
||||
}
|
||||
Command::Action(request) => execute_request(&controller, request, &mut mode, &mut last_action, &ready, &paused, &emergency, &observations).await,
|
||||
Command::Attention { delivery_id, avatar_id, reply } => {
|
||||
Command::Action(request) => execute_request(&controller, request, &mut mode, &mut last_action, &ready, &paused, &emergency, &action_cancel, &actions, &observations).await,
|
||||
Command::Attention { action_id, delivery_id, avatar_id, reply } => {
|
||||
let request = ActionRequest {
|
||||
action_id,
|
||||
action: BehaviorAction::FaceAvatar(avatar_id),
|
||||
trigger: BehaviorTrigger::PublicResponse { delivery_id, avatar_id },
|
||||
policy: BehaviorPolicyResult::BuiltInAttention,
|
||||
@@ -652,7 +744,7 @@ async fn run_actor(
|
||||
};
|
||||
let delay = random_duration(&controller, controller.settings.response_delay_min, controller.settings.response_delay_max);
|
||||
tokio::time::sleep(delay).await;
|
||||
execute_request(&controller, request, &mut mode, &mut last_action, &ready, &paused, &emergency, &observations).await;
|
||||
execute_request(&controller, request, &mut mode, &mut last_action, &ready, &paused, &emergency, &action_cancel, &actions, &observations).await;
|
||||
if mode == BehaviorMode::Engaged {
|
||||
tokio::time::sleep(controller.settings.attention_dwell).await;
|
||||
if !*paused.borrow() && ready.borrow().is_some() { transition(&observations, &mut mode, BehaviorMode::Available, BehaviorTrigger::IdleTimer).await; }
|
||||
@@ -702,12 +794,17 @@ async fn execute_request(
|
||||
ready: &watch::Receiver<Option<ReadyState>>,
|
||||
paused: &watch::Receiver<bool>,
|
||||
emergency: &watch::Receiver<bool>,
|
||||
action_cancel: &watch::Receiver<u64>,
|
||||
actions: &Arc<Mutex<ActionRegistry>>,
|
||||
observations: &mpsc::Sender<BehaviorObservation>,
|
||||
) {
|
||||
let started = Instant::now();
|
||||
let action_id = request.action_id.clone();
|
||||
let action_name = request.action.name().to_owned();
|
||||
let state = *ready.borrow();
|
||||
let mut result = if *emergency.borrow() {
|
||||
let mut result = if take_action_cancellation(actions, &action_id) {
|
||||
Err(BehaviorError::Cancelled)
|
||||
} else if *emergency.borrow() {
|
||||
Err(BehaviorError::EmergencyStopped)
|
||||
} else if *paused.borrow() {
|
||||
Err(BehaviorError::Paused)
|
||||
@@ -737,11 +834,15 @@ async fn execute_request(
|
||||
let mut action_ready = ready.clone();
|
||||
let mut action_paused = paused.clone();
|
||||
let mut action_emergency = emergency.clone();
|
||||
let action_cancel = action_cancel.clone();
|
||||
let action_registry = actions.clone();
|
||||
let cancelled_id = action_id.clone();
|
||||
result = tokio::select! {
|
||||
value = tokio::time::timeout(controller.settings.action_timeout, perform_action(controller, state, &request.action)) => value.unwrap_or(Err(BehaviorError::TimedOut)),
|
||||
_ = action_ready.changed() => Err(BehaviorError::Cancelled),
|
||||
_ = action_paused.changed() => Err(BehaviorError::Paused),
|
||||
_ = action_emergency.changed() => Err(BehaviorError::EmergencyStopped),
|
||||
() = wait_for_action_cancellation(action_cancel, action_registry, cancelled_id) => Err(BehaviorError::Cancelled),
|
||||
};
|
||||
if result.is_err() && matches!(request.action, BehaviorAction::WalkShort { .. }) {
|
||||
let _ = controller
|
||||
@@ -811,6 +912,7 @@ async fn execute_request(
|
||||
};
|
||||
let _ = observations
|
||||
.send(BehaviorObservation::Action {
|
||||
action_id: action_id.clone(),
|
||||
generation: state.map(|value| value.generation),
|
||||
region_id: state.map(|value| value.region_id),
|
||||
action: action_name,
|
||||
@@ -821,6 +923,22 @@ async fn execute_request(
|
||||
})
|
||||
.await;
|
||||
let _ = request.reply.send(result);
|
||||
unregister_action(actions, &action_id);
|
||||
}
|
||||
|
||||
async fn wait_for_action_cancellation(
|
||||
mut signal: watch::Receiver<u64>,
|
||||
actions: Arc<Mutex<ActionRegistry>>,
|
||||
action_id: String,
|
||||
) {
|
||||
loop {
|
||||
if take_action_cancellation(&actions, &action_id) {
|
||||
return;
|
||||
}
|
||||
if signal.changed().await.is_err() {
|
||||
std::future::pending::<()>().await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn perform_action(
|
||||
@@ -1086,6 +1204,38 @@ fn random_duration(
|
||||
minimum + Duration::from_millis(u64::try_from(offset).unwrap_or(u64::MAX))
|
||||
}
|
||||
|
||||
fn valid_action_id(value: &str) -> bool {
|
||||
!value.is_empty()
|
||||
&& value.len() <= MAX_ACTION_ID_BYTES
|
||||
&& value.chars().all(|character| {
|
||||
character.is_ascii_alphanumeric() || matches!(character, '-' | '_' | '.' | ':')
|
||||
})
|
||||
}
|
||||
|
||||
fn lock_actions(value: &Mutex<ActionRegistry>) -> MutexGuard<'_, ActionRegistry> {
|
||||
value
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
}
|
||||
|
||||
fn take_action_cancellation(actions: &Mutex<ActionRegistry>, action_id: &str) -> bool {
|
||||
lock_actions(actions).cancelled.remove(action_id)
|
||||
}
|
||||
|
||||
fn unregister_action(actions: &Mutex<ActionRegistry>, action_id: &str) {
|
||||
let mut actions = lock_actions(actions);
|
||||
actions.active.remove(action_id);
|
||||
actions.cancelled.remove(action_id);
|
||||
}
|
||||
|
||||
fn unix_millis_now() -> u64 {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.map_or(0, |duration| {
|
||||
u64::try_from(duration.as_millis()).unwrap_or(u64::MAX)
|
||||
})
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct BehaviorBackend {
|
||||
ingress: BehaviorIngress,
|
||||
@@ -1136,7 +1286,7 @@ impl AuthorizedToolBackend for BehaviorBackend {
|
||||
let result = match parsed {
|
||||
Ok(behavior_action) => tokio::select! {
|
||||
() = cancellation.cancelled() => Err(BehaviorError::Cancelled),
|
||||
value = self.ingress.request(behavior_action, BehaviorTrigger::AuthorizedTool { authorization_id, tool: action.call().name.as_str().to_owned() }, BehaviorPolicyResult::Authorized(authorization_id)) => value,
|
||||
value = self.ingress.request(call_id.as_str().to_owned(), behavior_action, BehaviorTrigger::AuthorizedTool { authorization_id, tool: action.call().name.as_str().to_owned() }, BehaviorPolicyResult::Authorized(authorization_id)) => value,
|
||||
},
|
||||
Err(error) => Err(error),
|
||||
};
|
||||
|
||||
@@ -453,6 +453,48 @@ async fn pause_preempts_stuck_walk_and_blocks_all_motion() {
|
||||
handle.shutdown().await.expect("shutdown");
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn exact_action_id_cancels_only_the_matching_pending_action() {
|
||||
let region = uuid(10);
|
||||
let sink = Arc::new(FakeSink::new(1, region));
|
||||
sink.set_stuck(true);
|
||||
let mut config = settings();
|
||||
config.settle_delay = Duration::ZERO;
|
||||
let handle = BehaviorController::new(config.clone(), sink, 16, 32, Duration::from_secs(1))
|
||||
.expect("controller")
|
||||
.start();
|
||||
let ingress = handle.ingress();
|
||||
ingress.connected(1, region).expect("ready");
|
||||
tokio::task::yield_now().await;
|
||||
let backend = BehaviorBackend::new(ingress.clone());
|
||||
let task = tokio::spawn({
|
||||
let backend = backend.clone();
|
||||
let config = config.clone();
|
||||
async move {
|
||||
run_tool(
|
||||
&backend,
|
||||
&config,
|
||||
WALK_SHORT_TOOL,
|
||||
json!({"heading_degrees":90.0,"distance_meters":2.0}),
|
||||
)
|
||||
.await
|
||||
}
|
||||
});
|
||||
tokio::time::advance(Duration::from_millis(300)).await;
|
||||
assert!(!ingress.cancel_action("unrelated-action").expect("cancel"));
|
||||
assert!(ingress.cancel_action("behavior-call").expect("cancel"));
|
||||
tokio::task::yield_now().await;
|
||||
assert!(matches!(
|
||||
task.await.expect("task"),
|
||||
ToolCallOutcome::Rejected { .. }
|
||||
));
|
||||
assert!(matches!(
|
||||
run_tool(&backend, &config, SIT_TOOL, json!({})).await,
|
||||
ToolCallOutcome::Completed { .. }
|
||||
));
|
||||
handle.shutdown().await.expect("shutdown");
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn offline_and_emergency_stop_emit_zero_motion() {
|
||||
let region = uuid(10);
|
||||
|
||||
@@ -7,12 +7,15 @@ use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::error::Error;
|
||||
use std::fmt;
|
||||
use std::fs;
|
||||
use std::net::{IpAddr, Ipv4Addr, SocketAddr};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use url::Url;
|
||||
|
||||
const MAX_CONFIG_BYTES: u64 = 64 * 1024;
|
||||
const MAX_SECRET_BYTES: u64 = 16 * 1024;
|
||||
const MAX_TLS_DER_BYTES: u64 = 1024 * 1024;
|
||||
const MAX_AUTHORIZED_AVATARS: usize = 1_024;
|
||||
const MAX_QUEUE_CAPACITY: usize = 8_192;
|
||||
const MAX_BACKGROUND_TASKS: usize = 2;
|
||||
@@ -27,6 +30,11 @@ const ENV_GRID_AVATAR_NAME: &str = "METACRATE_AGENT_GRID_AVATAR_NAME";
|
||||
const ENV_GRID_PASSWORD: &str = "METACRATE_AGENT_GRID_PASSWORD";
|
||||
const ENV_GRID_PASSWORD_FILE: &str = "METACRATE_AGENT_GRID_PASSWORD_FILE";
|
||||
const ENV_AUTHORIZED_AVATARS: &str = "METACRATE_AGENT_AUTHORIZED_AVATAR_UUIDS";
|
||||
const ENV_CONTROL_LISTEN: &str = "METACRATE_AGENT_CONTROL_LISTEN";
|
||||
const ENV_CONTROL_OPERATOR_TOKEN: &str = "METACRATE_AGENT_CONTROL_OPERATOR_TOKEN";
|
||||
const ENV_CONTROL_OPERATOR_TOKEN_FILE: &str = "METACRATE_AGENT_CONTROL_OPERATOR_TOKEN_FILE";
|
||||
const ENV_CONTROL_OBSERVER_TOKEN: &str = "METACRATE_AGENT_CONTROL_OBSERVER_TOKEN";
|
||||
const ENV_CONTROL_OBSERVER_TOKEN_FILE: &str = "METACRATE_AGENT_CONTROL_OBSERVER_TOKEN_FILE";
|
||||
|
||||
/// Wrapper that never reveals its contents through `Debug` or `Display` and
|
||||
/// deliberately does not implement serialization.
|
||||
@@ -247,6 +255,66 @@ pub struct ConversationSettings {
|
||||
pub limits: crate::conversation::ConversationLimits,
|
||||
}
|
||||
|
||||
/// Portable control-plane settings. Plain TCP is loopback-only; a remote
|
||||
/// listener is admitted only with an explicit Rustls certificate and key.
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub struct ControlSettings {
|
||||
pub listen: SocketAddr,
|
||||
pub operator_token: Option<SecretString>,
|
||||
pub observer_token: Option<SecretString>,
|
||||
pub limits: crate::control_plane::ControlLimits,
|
||||
pub remote_tls: Option<RemoteTlsSettings>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub struct RemoteTlsSettings {
|
||||
pub certificate_der: PathBuf,
|
||||
pub private_key_der: PathBuf,
|
||||
}
|
||||
|
||||
impl RemoteTlsSettings {
|
||||
/// Loads bounded, regular DER files into a portable Rustls server config.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns a configuration error when either file is unavailable, unsafe,
|
||||
/// oversized, malformed, or does not form a valid certificate/key pair.
|
||||
pub fn server_config(&self) -> Result<Arc<rustls::ServerConfig>, ConfigError> {
|
||||
let certificate = read_bounded_regular_file(
|
||||
"control.remote_tls_certificate_der",
|
||||
&self.certificate_der,
|
||||
MAX_TLS_DER_BYTES,
|
||||
)?;
|
||||
let private_key = read_bounded_regular_file(
|
||||
"control.remote_tls_private_key_der",
|
||||
&self.private_key_der,
|
||||
MAX_TLS_DER_BYTES,
|
||||
)?;
|
||||
let private_key = rustls::pki_types::PrivateKeyDer::try_from(private_key)
|
||||
.map_err(|_| ConfigError::InvalidControl)?;
|
||||
let config = rustls::ServerConfig::builder()
|
||||
.with_no_client_auth()
|
||||
.with_single_cert(
|
||||
vec![rustls::pki_types::CertificateDer::from(certificate)],
|
||||
private_key,
|
||||
)
|
||||
.map_err(|_| ConfigError::InvalidControl)?;
|
||||
Ok(Arc::new(config))
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for ControlSettings {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
listen: SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 7943),
|
||||
operator_token: None,
|
||||
observer_token: None,
|
||||
limits: crate::control_plane::ControlLimits::default(),
|
||||
remote_tls: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Fully resolved configuration. It cannot be constructed without validation.
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub struct AgentConfig {
|
||||
@@ -261,6 +329,7 @@ pub struct AgentConfig {
|
||||
pub reconnect: crate::session::ReconnectPolicy,
|
||||
pub conversation: ConversationSettings,
|
||||
pub interaction: crate::interaction::InteractionSettings,
|
||||
pub control: ControlSettings,
|
||||
}
|
||||
|
||||
impl AgentConfig {
|
||||
@@ -300,6 +369,45 @@ impl AgentConfig {
|
||||
if !self.behavior.is_valid() {
|
||||
return Err(ConfigError::InvalidBehavior);
|
||||
}
|
||||
if (!self.control.listen.ip().is_loopback() && self.control.remote_tls.is_none())
|
||||
|| !self.control.limits.is_valid()
|
||||
{
|
||||
return Err(ConfigError::InvalidControl);
|
||||
}
|
||||
if self.mode == OperatingMode::SplitService && self.control.operator_token.is_none() {
|
||||
return Err(ConfigError::Missing {
|
||||
field: "control.operator_token",
|
||||
required_for: "split service mode",
|
||||
});
|
||||
}
|
||||
if let Some(operator) = &self.control.operator_token
|
||||
&& (self
|
||||
.control
|
||||
.observer_token
|
||||
.as_ref()
|
||||
.is_some_and(|observer| observer == operator)
|
||||
|| operator == &self.llm.api_key
|
||||
|| self
|
||||
.grid
|
||||
.as_ref()
|
||||
.is_some_and(|grid| operator == &grid.password))
|
||||
{
|
||||
return Err(ConfigError::InvalidControl);
|
||||
}
|
||||
if self
|
||||
.control
|
||||
.observer_token
|
||||
.as_ref()
|
||||
.is_some_and(|observer| {
|
||||
observer == &self.llm.api_key
|
||||
|| self
|
||||
.grid
|
||||
.as_ref()
|
||||
.is_some_and(|grid| observer == &grid.password)
|
||||
})
|
||||
{
|
||||
return Err(ConfigError::InvalidControl);
|
||||
}
|
||||
if self.mode != OperatingMode::OfflineFake && self.grid.is_none() {
|
||||
return Err(ConfigError::Missing {
|
||||
field: "grid",
|
||||
@@ -476,6 +584,7 @@ pub enum ConfigError {
|
||||
InvalidConversationMemory,
|
||||
InvalidInteraction,
|
||||
InvalidBehavior,
|
||||
InvalidControl,
|
||||
}
|
||||
|
||||
impl fmt::Display for ConfigError {
|
||||
@@ -531,6 +640,9 @@ impl fmt::Display for ConfigError {
|
||||
}
|
||||
Self::InvalidInteraction => formatter.write_str("invalid interaction bounds"),
|
||||
Self::InvalidBehavior => formatter.write_str("invalid embodied behavior bounds"),
|
||||
Self::InvalidControl => formatter.write_str(
|
||||
"invalid control plane: use distinct non-credential tokens and loopback TCP or an explicit TLS remote listener",
|
||||
),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -553,6 +665,7 @@ struct FileConfig {
|
||||
reconnect: RawReconnect,
|
||||
conversation: RawConversation,
|
||||
interaction: RawInteraction,
|
||||
control: RawControl,
|
||||
}
|
||||
|
||||
#[derive(Clone, Default, Deserialize)]
|
||||
@@ -575,6 +688,18 @@ struct RawGrid {
|
||||
struct RawSecretFiles {
|
||||
llm_api_key: Option<PathBuf>,
|
||||
grid_password: Option<PathBuf>,
|
||||
control_operator_token: Option<PathBuf>,
|
||||
control_observer_token: Option<PathBuf>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Default, Deserialize)]
|
||||
#[serde(default, deny_unknown_fields)]
|
||||
struct RawControl {
|
||||
listen: Option<String>,
|
||||
operator_token: Option<String>,
|
||||
observer_token: Option<String>,
|
||||
remote_tls_certificate_der: Option<PathBuf>,
|
||||
remote_tls_private_key_der: Option<PathBuf>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Default, Deserialize)]
|
||||
@@ -932,6 +1057,57 @@ fn resolve<E: Environment>(
|
||||
.max_debounce_fragments
|
||||
.unwrap_or(interaction_defaults.max_debounce_fragments),
|
||||
};
|
||||
let control_defaults = ControlSettings::default();
|
||||
let listen_text = environment
|
||||
.get(ENV_CONTROL_LISTEN)
|
||||
.or(raw.control.listen)
|
||||
.unwrap_or_else(|| control_defaults.listen.to_string());
|
||||
let listen = listen_text
|
||||
.parse::<SocketAddr>()
|
||||
.map_err(|_| ConfigError::InvalidControl)?;
|
||||
let operator_token = optional_secret_from_layers(
|
||||
environment.get(ENV_CONTROL_OPERATOR_TOKEN),
|
||||
environment
|
||||
.get(ENV_CONTROL_OPERATOR_TOKEN_FILE)
|
||||
.map(PathBuf::from),
|
||||
raw.control.operator_token,
|
||||
raw.secret_files.control_operator_token,
|
||||
base,
|
||||
"control.operator_token",
|
||||
)?;
|
||||
let observer_token = optional_secret_from_layers(
|
||||
environment.get(ENV_CONTROL_OBSERVER_TOKEN),
|
||||
environment
|
||||
.get(ENV_CONTROL_OBSERVER_TOKEN_FILE)
|
||||
.map(PathBuf::from),
|
||||
raw.control.observer_token,
|
||||
raw.secret_files.control_observer_token,
|
||||
base,
|
||||
"control.observer_token",
|
||||
)?;
|
||||
let remote_tls = match (
|
||||
raw.control.remote_tls_certificate_der,
|
||||
raw.control.remote_tls_private_key_der,
|
||||
) {
|
||||
(None, None) => None,
|
||||
(Some(certificate_der), Some(private_key_der)) => Some(RemoteTlsSettings {
|
||||
certificate_der: resolve_path(base, &certificate_der),
|
||||
private_key_der: resolve_path(base, &private_key_der),
|
||||
}),
|
||||
_ => return Err(ConfigError::InvalidControl),
|
||||
};
|
||||
let control = ControlSettings {
|
||||
listen,
|
||||
operator_token,
|
||||
observer_token,
|
||||
limits: crate::control_plane::ControlLimits {
|
||||
command_queue: limits.control_queue,
|
||||
request_timeout: timeouts.request.min(Duration::from_mins(2)),
|
||||
write_timeout: timeouts.shutdown.min(Duration::from_secs(30)),
|
||||
..crate::control_plane::ControlLimits::default()
|
||||
},
|
||||
remote_tls,
|
||||
};
|
||||
|
||||
let config = AgentConfig {
|
||||
mode,
|
||||
@@ -996,6 +1172,7 @@ fn resolve<E: Environment>(
|
||||
reconnect,
|
||||
conversation,
|
||||
interaction,
|
||||
control,
|
||||
};
|
||||
config.validate()?;
|
||||
Ok(config)
|
||||
@@ -1064,6 +1241,28 @@ fn secret_from_layers(
|
||||
})
|
||||
}
|
||||
|
||||
fn optional_secret_from_layers(
|
||||
environment_value: Option<String>,
|
||||
environment_file: Option<PathBuf>,
|
||||
file_value: Option<String>,
|
||||
file_path: Option<PathBuf>,
|
||||
base: &Path,
|
||||
field: &'static str,
|
||||
) -> Result<Option<SecretString>, ConfigError> {
|
||||
if let Some(value) = environment_value {
|
||||
return SecretString::new(field, value).map(Some);
|
||||
}
|
||||
if let Some(path) = environment_file {
|
||||
return read_secret(field, &resolve_path(base, &path)).map(Some);
|
||||
}
|
||||
if let Some(path) = file_path {
|
||||
return read_secret(field, &resolve_path(base, &path)).map(Some);
|
||||
}
|
||||
file_value
|
||||
.map(|value| SecretString::new(field, value))
|
||||
.transpose()
|
||||
}
|
||||
|
||||
fn resolve_path(base: &Path, path: &Path) -> PathBuf {
|
||||
if path.is_absolute() {
|
||||
path.to_owned()
|
||||
@@ -1437,6 +1636,50 @@ mod tests {
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn split_control_requires_distinct_tokens_and_tls_for_remote_bind() {
|
||||
let split = MapEnvironment::from_pairs([
|
||||
(ENV_SPLIT, "true"),
|
||||
(ENV_LLM_ENDPOINT, "https://llm.invalid/chat"),
|
||||
(ENV_LLM_API_KEY, "llm-key"),
|
||||
(ENV_GRID_LOGIN_URL, "https://grid.invalid/login"),
|
||||
(ENV_GRID_AVATAR_NAME, "Control Agent"),
|
||||
(ENV_GRID_PASSWORD, "grid-password"),
|
||||
(ENV_CONTROL_OPERATOR_TOKEN, "operator-capability"),
|
||||
(ENV_CONTROL_OBSERVER_TOKEN, "observer-capability"),
|
||||
]);
|
||||
let config = ConfigLoader::new()
|
||||
.with_environment(split.clone())
|
||||
.load()
|
||||
.expect("bounded loopback split control");
|
||||
assert_eq!(config.mode, OperatingMode::SplitService);
|
||||
let diagnostic = format!("{config:?}");
|
||||
for secret in [
|
||||
"operator-capability",
|
||||
"observer-capability",
|
||||
"llm-key",
|
||||
"grid-password",
|
||||
] {
|
||||
assert!(!diagnostic.contains(secret));
|
||||
}
|
||||
|
||||
let mut reused = split.clone();
|
||||
reused.insert(ENV_CONTROL_OPERATOR_TOKEN, "llm-key");
|
||||
assert_eq!(
|
||||
ConfigLoader::new().with_environment(reused).load(),
|
||||
Err(ConfigError::InvalidControl)
|
||||
);
|
||||
|
||||
let mut remote_plaintext = split;
|
||||
remote_plaintext.insert(ENV_CONTROL_LISTEN, "0.0.0.0:7943");
|
||||
assert_eq!(
|
||||
ConfigLoader::new()
|
||||
.with_environment(remote_plaintext)
|
||||
.load(),
|
||||
Err(ConfigError::InvalidControl)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn secret_file_is_bounded_regular_and_trailing_newline_is_removed() {
|
||||
let secret = temporary_file("api-key", "secret-from-file\n");
|
||||
|
||||
1780
crates/metacrate-grid-agent/src/control_plane.rs
Normal file
1780
crates/metacrate-grid-agent/src/control_plane.rs
Normal file
File diff suppressed because it is too large
Load Diff
746
crates/metacrate-grid-agent/src/control_plane_tests.rs
Normal file
746
crates/metacrate-grid-agent/src/control_plane_tests.rs
Normal file
@@ -0,0 +1,746 @@
|
||||
use crate::config::SecretString;
|
||||
use crate::control_plane::*;
|
||||
use libremetaverse_types::compat::CancellationToken;
|
||||
use serde_json::json;
|
||||
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::Duration;
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
use tokio::net::TcpStream;
|
||||
|
||||
#[derive(Default)]
|
||||
struct FakeTarget {
|
||||
mutations: Mutex<Vec<String>>,
|
||||
shutdown: AtomicBool,
|
||||
active: AtomicUsize,
|
||||
maximum_active: AtomicUsize,
|
||||
}
|
||||
|
||||
impl FakeTarget {
|
||||
fn mutation_count(&self) -> usize {
|
||||
self.mutations.lock().expect("mutations").len()
|
||||
}
|
||||
}
|
||||
|
||||
impl ControlTarget for FakeTarget {
|
||||
#[allow(clippy::too_many_lines)] // Exhaustive fake covers every response family.
|
||||
fn execute(
|
||||
&self,
|
||||
context: ControlContext,
|
||||
request: ControlRequest,
|
||||
cancellation: CancellationToken,
|
||||
) -> ControlFuture<'_> {
|
||||
Box::pin(async move {
|
||||
let active = self.active.fetch_add(1, Ordering::AcqRel) + 1;
|
||||
self.maximum_active.fetch_max(active, Ordering::AcqRel);
|
||||
let result = match request {
|
||||
ControlRequest::Health => Ok(ControlPayload::Health(HealthView {
|
||||
service_state: "running".into(),
|
||||
ready: true,
|
||||
uptime_seconds: 42,
|
||||
protocol_version: CONTROL_PROTOCOL_VERSION,
|
||||
})),
|
||||
ControlRequest::Runtime => Ok(ControlPayload::Runtime(RuntimeView {
|
||||
grid_state: "online".into(),
|
||||
generation: 7,
|
||||
transport_connected: true,
|
||||
agent_ready: true,
|
||||
region_id: Some("00000000-0000-4000-8000-000000000001".into()),
|
||||
region_name: Some("Safe Region".into()),
|
||||
position: Some([128.0, 128.0, 24.0]),
|
||||
behavior_mode: "available".into(),
|
||||
control_queue_used: 1,
|
||||
control_queue_capacity: 64,
|
||||
budget_tool_calls_used: 3,
|
||||
budget_movement_millimeters_used: 2_000,
|
||||
})),
|
||||
ControlRequest::ListSessions { page } => Ok(ControlPayload::Sessions(page_values(
|
||||
&page,
|
||||
(0..7)
|
||||
.map(|index| SessionMetadataView {
|
||||
session_id: format!("session-{index}"),
|
||||
avatar_id: format!("00000000-0000-4000-8000-{index:012x}"),
|
||||
channel: "direct_im".into(),
|
||||
created_unix_millis: 10,
|
||||
last_active_unix_millis: 20,
|
||||
turns: 2,
|
||||
bytes: 32,
|
||||
})
|
||||
.collect(),
|
||||
))),
|
||||
ControlRequest::ListScheduledJobs { page } => {
|
||||
Ok(ControlPayload::ScheduledJobs(page_values(
|
||||
&page,
|
||||
vec![ScheduledJobView {
|
||||
job_id: "roam-1".into(),
|
||||
kind: "landmark_roam".into(),
|
||||
enabled: true,
|
||||
next_run_unix_millis: Some(50),
|
||||
}],
|
||||
)))
|
||||
}
|
||||
ControlRequest::ListPendingApprovals { page } => {
|
||||
Ok(ControlPayload::PendingApprovals(page_values(
|
||||
&page,
|
||||
vec![PendingApprovalView {
|
||||
approval_id: 1,
|
||||
tool: "build".into(),
|
||||
principal: "operator:local".into(),
|
||||
expires_unix_seconds: 99,
|
||||
movement_millimeters: 0,
|
||||
inventory_operations: 0,
|
||||
build_prims: 1,
|
||||
}],
|
||||
)))
|
||||
}
|
||||
ControlRequest::ListAuditEvents { page } => {
|
||||
Ok(ControlPayload::AuditEvents(page_values(
|
||||
&page,
|
||||
vec![AuditEventView {
|
||||
sequence: 1,
|
||||
unix_millis: 1,
|
||||
principal: "operator:local".into(),
|
||||
operation: "pause_autonomy".into(),
|
||||
outcome: "completed".into(),
|
||||
authorization_id: None,
|
||||
}],
|
||||
)))
|
||||
}
|
||||
ControlRequest::InjectOperatorMessage { message } if message == "wait" => {
|
||||
cancellation.cancelled().await;
|
||||
Err(ControlError {
|
||||
code: ControlErrorCode::Cancelled,
|
||||
message: "operation cancelled",
|
||||
retryable: false,
|
||||
})
|
||||
}
|
||||
ControlRequest::GracefulShutdown => {
|
||||
self.shutdown.store(true, Ordering::Release);
|
||||
self.mutations
|
||||
.lock()
|
||||
.expect("mutations")
|
||||
.push("graceful_shutdown".into());
|
||||
Ok(ControlPayload::Completed)
|
||||
}
|
||||
other => {
|
||||
assert_eq!(context.role, ControlRole::Operator);
|
||||
self.mutations
|
||||
.lock()
|
||||
.expect("mutations")
|
||||
.push(format!("{other:?}"));
|
||||
Ok(ControlPayload::Completed)
|
||||
}
|
||||
};
|
||||
self.active.fetch_sub(1, Ordering::AcqRel);
|
||||
result
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn page_values<T>(page: &PageRequest, values: Vec<T>) -> Page<T> {
|
||||
let start = usize::try_from(page.cursor.unwrap_or(0))
|
||||
.expect("cursor")
|
||||
.min(values.len());
|
||||
let end = start
|
||||
.saturating_add(usize::from(page.limit))
|
||||
.min(values.len());
|
||||
let next_cursor = (end < values.len()).then(|| u64::try_from(end).expect("cursor"));
|
||||
Page {
|
||||
items: values.into_iter().skip(start).take(end - start).collect(),
|
||||
next_cursor,
|
||||
}
|
||||
}
|
||||
|
||||
fn token(value: &str) -> SecretString {
|
||||
SecretString::new("control.test_token", value).expect("token")
|
||||
}
|
||||
|
||||
fn limits() -> ControlLimits {
|
||||
ControlLimits {
|
||||
request_timeout: Duration::from_secs(5),
|
||||
idle_timeout: Duration::from_secs(5),
|
||||
write_timeout: Duration::from_secs(1),
|
||||
..ControlLimits::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn plane(target: Arc<FakeTarget>, configured: ControlLimits) -> Arc<ControlPlane> {
|
||||
ControlPlane::new(
|
||||
target,
|
||||
token("operator-secret"),
|
||||
Some(token("observer-secret")),
|
||||
configured,
|
||||
)
|
||||
.expect("plane")
|
||||
}
|
||||
|
||||
fn request(id: &str, request: ControlRequest) -> ControlRequestEnvelope {
|
||||
ControlRequestEnvelope {
|
||||
version: CONTROL_PROTOCOL_VERSION,
|
||||
request_id: id.into(),
|
||||
request,
|
||||
}
|
||||
}
|
||||
|
||||
enum Client {
|
||||
InProcess(InProcessControlClient),
|
||||
Tcp(TcpControlClient),
|
||||
}
|
||||
|
||||
impl Client {
|
||||
async fn request(&self, envelope: ControlRequestEnvelope) -> ControlResponseEnvelope {
|
||||
match self {
|
||||
Self::InProcess(client) => client.request(envelope).await,
|
||||
Self::Tcp(client) => client.request(envelope).await.expect("TCP response"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
enum Transport {
|
||||
InProcess,
|
||||
Tcp,
|
||||
}
|
||||
|
||||
struct Fixture {
|
||||
client: Client,
|
||||
plane: Arc<ControlPlane>,
|
||||
target: Arc<FakeTarget>,
|
||||
server: Option<TcpControlServer>,
|
||||
}
|
||||
|
||||
async fn fixture(transport: Transport, role: ControlRole, configured: ControlLimits) -> Fixture {
|
||||
let target = Arc::new(FakeTarget::default());
|
||||
let plane = plane(target.clone(), configured);
|
||||
let value = if role == ControlRole::Operator {
|
||||
"operator-secret"
|
||||
} else {
|
||||
"observer-secret"
|
||||
};
|
||||
match transport {
|
||||
Transport::InProcess => Fixture {
|
||||
client: Client::InProcess(plane.connect(value).expect("connect")),
|
||||
plane,
|
||||
target,
|
||||
server: None,
|
||||
},
|
||||
Transport::Tcp => {
|
||||
let server = TcpControlServer::bind(
|
||||
plane.clone(),
|
||||
TcpControlConfig {
|
||||
listen: "127.0.0.1:0".parse().expect("address"),
|
||||
limits: configured,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("bind");
|
||||
let client = TcpControlClient::connect(server.local_addr(), value, configured)
|
||||
.await
|
||||
.expect("connect");
|
||||
Fixture {
|
||||
client: Client::Tcp(client),
|
||||
plane,
|
||||
target,
|
||||
server: Some(server),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn close(fixture: Fixture) {
|
||||
drop(fixture.client);
|
||||
if let Some(server) = fixture.server {
|
||||
server.shutdown().await.expect("shutdown");
|
||||
}
|
||||
}
|
||||
|
||||
async fn conformance(transport: Transport) {
|
||||
let fixture = fixture(transport, ControlRole::Operator, limits()).await;
|
||||
let health = fixture
|
||||
.client
|
||||
.request(request("health-1", ControlRequest::Health))
|
||||
.await;
|
||||
assert_eq!(health.version, CONTROL_PROTOCOL_VERSION);
|
||||
assert!(matches!(
|
||||
health.result,
|
||||
Ok(ControlPayload::Health(HealthView { ready: true, .. }))
|
||||
));
|
||||
let page = fixture
|
||||
.client
|
||||
.request(request(
|
||||
"sessions-1",
|
||||
ControlRequest::ListSessions {
|
||||
page: PageRequest {
|
||||
cursor: Some(2),
|
||||
limit: 3,
|
||||
},
|
||||
},
|
||||
))
|
||||
.await;
|
||||
let Ok(ControlPayload::Sessions(page)) = page.result else {
|
||||
panic!("session page")
|
||||
};
|
||||
assert_eq!(page.items.len(), 3);
|
||||
assert_eq!(page.items[0].session_id, "session-2");
|
||||
assert_eq!(page.next_cursor, Some(5));
|
||||
assert!(
|
||||
fixture
|
||||
.client
|
||||
.request(request("pause-1", ControlRequest::PauseAutonomy))
|
||||
.await
|
||||
.result
|
||||
.is_ok()
|
||||
);
|
||||
let replay = fixture
|
||||
.client
|
||||
.request(request("pause-1", ControlRequest::ResumeAutonomy))
|
||||
.await;
|
||||
assert_eq!(
|
||||
replay.result.expect_err("replay").code,
|
||||
ControlErrorCode::Replay
|
||||
);
|
||||
assert_eq!(fixture.target.mutation_count(), 1);
|
||||
let shutdown = fixture
|
||||
.client
|
||||
.request(request("shutdown-1", ControlRequest::GracefulShutdown))
|
||||
.await;
|
||||
assert!(shutdown.result.is_ok());
|
||||
assert!(fixture.target.shutdown.load(Ordering::Acquire));
|
||||
close(fixture).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn same_conformance_suite_runs_in_process() {
|
||||
conformance(Transport::InProcess).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn same_conformance_suite_runs_over_loopback_tcp() {
|
||||
conformance(Transport::Tcp).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn observer_is_read_only_and_tokens_are_redacted() {
|
||||
let fixture = fixture(Transport::InProcess, ControlRole::Observer, limits()).await;
|
||||
assert!(matches!(
|
||||
fixture
|
||||
.client
|
||||
.request(request("read", ControlRequest::Runtime))
|
||||
.await
|
||||
.result,
|
||||
Ok(ControlPayload::Runtime(_))
|
||||
));
|
||||
let denied = fixture
|
||||
.client
|
||||
.request(request("mutate", ControlRequest::PauseAutonomy))
|
||||
.await;
|
||||
assert_eq!(
|
||||
denied.result.expect_err("denied").code,
|
||||
ControlErrorCode::PermissionDenied
|
||||
);
|
||||
assert_eq!(fixture.target.mutation_count(), 0);
|
||||
assert!(!format!("{:?}", fixture.plane).contains("operator-secret"));
|
||||
let Err(error) = fixture.plane.connect("wrong") else {
|
||||
panic!("bad token was accepted")
|
||||
};
|
||||
assert_eq!(error.code, ControlErrorCode::AuthenticationFailed);
|
||||
close(fixture).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn version_bounds_and_malformed_requests_are_typed() {
|
||||
let fixture = fixture(Transport::InProcess, ControlRole::Operator, limits()).await;
|
||||
let mut wrong = request("version", ControlRequest::Health);
|
||||
wrong.version = 99;
|
||||
assert_eq!(
|
||||
fixture
|
||||
.client
|
||||
.request(wrong)
|
||||
.await
|
||||
.result
|
||||
.expect_err("version")
|
||||
.code,
|
||||
ControlErrorCode::VersionMismatch
|
||||
);
|
||||
let invalid = fixture
|
||||
.client
|
||||
.request(request(
|
||||
"page",
|
||||
ControlRequest::ListSessions {
|
||||
page: PageRequest {
|
||||
cursor: None,
|
||||
limit: 101,
|
||||
},
|
||||
},
|
||||
))
|
||||
.await;
|
||||
assert_eq!(
|
||||
invalid.result.expect_err("page").code,
|
||||
ControlErrorCode::InvalidRequest
|
||||
);
|
||||
let secret_payload = fixture
|
||||
.client
|
||||
.request(request(
|
||||
"audit",
|
||||
ControlRequest::ListAuditEvents {
|
||||
page: PageRequest::default(),
|
||||
},
|
||||
))
|
||||
.await;
|
||||
let rendered = serde_json::to_string(&secret_payload).expect("JSON");
|
||||
for secret in ["operator-secret", "observer-secret", "Bearer ", "CAPS/"] {
|
||||
assert!(!rendered.contains(secret));
|
||||
}
|
||||
close(fixture).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancellation_is_per_request_and_concurrent_operators_stay_responsive() {
|
||||
for transport in [Transport::InProcess, Transport::Tcp] {
|
||||
let fixture = fixture(transport, ControlRole::Operator, limits()).await;
|
||||
let slow = match &fixture.client {
|
||||
Client::InProcess(value) => Client::InProcess(value.clone()),
|
||||
Client::Tcp(value) => Client::Tcp(value.clone()),
|
||||
};
|
||||
let task = tokio::spawn(async move {
|
||||
slow.request(request(
|
||||
"slow",
|
||||
ControlRequest::InjectOperatorMessage {
|
||||
message: "wait".into(),
|
||||
},
|
||||
))
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(1), async {
|
||||
while fixture.target.active.load(Ordering::Acquire) == 0 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("slow request became active");
|
||||
let concurrent_operator = match &fixture.server {
|
||||
Some(server) => Client::Tcp(
|
||||
TcpControlClient::connect(server.local_addr(), "operator-secret", limits())
|
||||
.await
|
||||
.expect("second TCP operator"),
|
||||
),
|
||||
None => Client::InProcess(
|
||||
fixture
|
||||
.plane
|
||||
.connect("operator-secret")
|
||||
.expect("second in-process operator"),
|
||||
),
|
||||
};
|
||||
let health = concurrent_operator
|
||||
.request(request("health-during", ControlRequest::Health))
|
||||
.await;
|
||||
assert!(health.result.is_ok());
|
||||
let cancelled = fixture
|
||||
.client
|
||||
.request(request(
|
||||
"cancel",
|
||||
ControlRequest::CancelRequest {
|
||||
target_request_id: "slow".into(),
|
||||
},
|
||||
))
|
||||
.await;
|
||||
assert!(matches!(
|
||||
cancelled.result,
|
||||
Ok(ControlPayload::Cancelled { .. })
|
||||
));
|
||||
let slow = task.await.expect("slow task");
|
||||
assert_eq!(
|
||||
slow.result.expect_err("cancelled").code,
|
||||
ControlErrorCode::Cancelled
|
||||
);
|
||||
assert!(fixture.target.maximum_active.load(Ordering::Acquire) >= 2);
|
||||
drop(concurrent_operator);
|
||||
close(fixture).await;
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn event_stream_resubscribes_with_explicit_gap_and_slow_consumers_are_cut_off() {
|
||||
let mut configured = limits();
|
||||
configured.event_queue = 2;
|
||||
configured.event_history = 3;
|
||||
let fixture = fixture(Transport::InProcess, ControlRole::Operator, configured).await;
|
||||
let mut in_subscription = None;
|
||||
if let Client::InProcess(client) = &fixture.client {
|
||||
let (_, subscription) = client
|
||||
.subscribe(request(
|
||||
"subscribe",
|
||||
ControlRequest::SubscribeEvents {
|
||||
after_sequence: Some(0),
|
||||
},
|
||||
))
|
||||
.await
|
||||
.expect("subscribe");
|
||||
in_subscription = Some(subscription);
|
||||
}
|
||||
for index in 0..10 {
|
||||
fixture.plane.publish(ControlEventKind::StateChanged {
|
||||
component: "grid".into(),
|
||||
state: format!("state-{index}"),
|
||||
});
|
||||
}
|
||||
assert_eq!(fixture.plane.retained_event_count(), 3);
|
||||
let first = in_subscription
|
||||
.as_mut()
|
||||
.expect("subscription")
|
||||
.recv()
|
||||
.await
|
||||
.expect("buffered");
|
||||
assert_eq!(first.sequence, 1);
|
||||
drop(in_subscription);
|
||||
let client = fixture.plane.connect("operator-secret").expect("reconnect");
|
||||
let (_, mut resumed) = client
|
||||
.subscribe(request(
|
||||
"resubscribe",
|
||||
ControlRequest::SubscribeEvents {
|
||||
after_sequence: Some(1),
|
||||
},
|
||||
))
|
||||
.await
|
||||
.expect("resubscribe");
|
||||
let gap = resumed.recv().await.expect("gap");
|
||||
assert!(matches!(
|
||||
gap.event,
|
||||
ControlEventKind::Gap {
|
||||
first_available: 9,
|
||||
last_missed: 8
|
||||
}
|
||||
));
|
||||
close(fixture).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn zero_client_event_flood_remains_bounded_and_headless() {
|
||||
let target = Arc::new(FakeTarget::default());
|
||||
let mut configured = limits();
|
||||
configured.event_history = 4;
|
||||
let plane = plane(target, configured);
|
||||
assert_eq!(plane.active_connections(), 0);
|
||||
for index in 0..10_000 {
|
||||
plane.publish(ControlEventKind::StateChanged {
|
||||
component: "load".into(),
|
||||
state: index.to_string(),
|
||||
});
|
||||
}
|
||||
assert_eq!(plane.retained_event_count(), 4);
|
||||
assert_eq!(plane.active_connections(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn subscriptions_have_a_hard_global_bound_and_release_on_drop() {
|
||||
let mut configured = limits();
|
||||
configured.max_subscriptions = 2;
|
||||
let target = Arc::new(FakeTarget::default());
|
||||
let plane = plane(target, configured);
|
||||
let client = plane.connect("operator-secret").expect("client");
|
||||
let (_, first) = client
|
||||
.subscribe(request(
|
||||
"subscribe-1",
|
||||
ControlRequest::SubscribeEvents {
|
||||
after_sequence: None,
|
||||
},
|
||||
))
|
||||
.await
|
||||
.expect("first subscription");
|
||||
let (_, second) = client
|
||||
.subscribe(request(
|
||||
"subscribe-2",
|
||||
ControlRequest::SubscribeEvents {
|
||||
after_sequence: None,
|
||||
},
|
||||
))
|
||||
.await
|
||||
.expect("second subscription");
|
||||
let error = client
|
||||
.subscribe(request(
|
||||
"subscribe-3",
|
||||
ControlRequest::SubscribeEvents {
|
||||
after_sequence: None,
|
||||
},
|
||||
))
|
||||
.await
|
||||
.err()
|
||||
.expect("subscription bound");
|
||||
assert_eq!(error.code, ControlErrorCode::Busy);
|
||||
drop(first);
|
||||
client
|
||||
.subscribe(request(
|
||||
"subscribe-4",
|
||||
ControlRequest::SubscribeEvents {
|
||||
after_sequence: None,
|
||||
},
|
||||
))
|
||||
.await
|
||||
.expect("released subscription");
|
||||
drop(second);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[allow(clippy::too_many_lines)] // One raw-transport matrix shares one listener.
|
||||
async fn tcp_rejects_bad_tokens_oversized_and_partial_frames() {
|
||||
let configured = limits();
|
||||
let target = Arc::new(FakeTarget::default());
|
||||
let plane = plane(target, configured);
|
||||
let server = TcpControlServer::bind(
|
||||
plane.clone(),
|
||||
TcpControlConfig {
|
||||
listen: "127.0.0.1:0".parse().expect("address"),
|
||||
limits: configured,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("server");
|
||||
let Err(error) = TcpControlClient::connect(server.local_addr(), "wrong", configured).await
|
||||
else {
|
||||
panic!("bad token was accepted")
|
||||
};
|
||||
assert_eq!(error.code, ControlErrorCode::AuthenticationFailed);
|
||||
|
||||
let mut wrong_version = TcpStream::connect(server.local_addr())
|
||||
.await
|
||||
.expect("version connect");
|
||||
let hello = serde_json::to_vec(&json!({
|
||||
"frame":"hello",
|
||||
"body":{"version":99,"token":"operator-secret"}
|
||||
}))
|
||||
.expect("hello JSON");
|
||||
wrong_version
|
||||
.write_all(
|
||||
&u32::try_from(hello.len())
|
||||
.expect("frame length")
|
||||
.to_be_bytes(),
|
||||
)
|
||||
.await
|
||||
.expect("version header");
|
||||
wrong_version.write_all(&hello).await.expect("version body");
|
||||
let mut response_header = [0_u8; 4];
|
||||
wrong_version
|
||||
.read_exact(&mut response_header)
|
||||
.await
|
||||
.expect("version response header");
|
||||
let mut response =
|
||||
vec![0_u8; usize::try_from(u32::from_be_bytes(response_header)).expect("response length")];
|
||||
wrong_version
|
||||
.read_exact(&mut response)
|
||||
.await
|
||||
.expect("version response");
|
||||
let response: serde_json::Value = serde_json::from_slice(&response).expect("version JSON");
|
||||
assert_eq!(response["body"]["error"]["code"], json!("version_mismatch"));
|
||||
|
||||
let subscriber = TcpControlClient::connect(server.local_addr(), "operator-secret", configured)
|
||||
.await
|
||||
.expect("subscriber");
|
||||
let subscription_response = subscriber
|
||||
.request(request(
|
||||
"events",
|
||||
ControlRequest::SubscribeEvents {
|
||||
after_sequence: None,
|
||||
},
|
||||
))
|
||||
.await
|
||||
.expect("subscription response");
|
||||
assert!(matches!(
|
||||
subscription_response.result,
|
||||
Ok(ControlPayload::Subscribed { .. })
|
||||
));
|
||||
plane.publish(ControlEventKind::StateChanged {
|
||||
component: "session".into(),
|
||||
state: "online".into(),
|
||||
});
|
||||
let event = tokio::time::timeout(Duration::from_secs(1), subscriber.next_event())
|
||||
.await
|
||||
.expect("event deadline")
|
||||
.expect("event");
|
||||
assert!(matches!(event.event, ControlEventKind::StateChanged { .. }));
|
||||
let last_sequence = event.sequence;
|
||||
drop(subscriber);
|
||||
plane.publish(ControlEventKind::StateChanged {
|
||||
component: "session".into(),
|
||||
state: "reconnecting".into(),
|
||||
});
|
||||
let reconnected = TcpControlClient::connect(server.local_addr(), "operator-secret", configured)
|
||||
.await
|
||||
.expect("reconnected subscriber");
|
||||
reconnected
|
||||
.request(request(
|
||||
"events-resumed",
|
||||
ControlRequest::SubscribeEvents {
|
||||
after_sequence: Some(last_sequence),
|
||||
},
|
||||
))
|
||||
.await
|
||||
.expect("resubscription response");
|
||||
let replayed = tokio::time::timeout(Duration::from_secs(1), reconnected.next_event())
|
||||
.await
|
||||
.expect("replay deadline")
|
||||
.expect("replayed event");
|
||||
assert_eq!(replayed.sequence, last_sequence.saturating_add(1));
|
||||
let mut oversized = TcpStream::connect(server.local_addr())
|
||||
.await
|
||||
.expect("raw connect");
|
||||
oversized
|
||||
.write_all(&(u32::try_from(configured.max_frame_bytes).expect("bound") + 1).to_be_bytes())
|
||||
.await
|
||||
.expect("header");
|
||||
let mut byte = [0_u8; 1];
|
||||
assert_eq!(
|
||||
tokio::time::timeout(Duration::from_secs(1), oversized.read(&mut byte))
|
||||
.await
|
||||
.expect("closed")
|
||||
.expect("read"),
|
||||
0
|
||||
);
|
||||
let mut partial = TcpStream::connect(server.local_addr())
|
||||
.await
|
||||
.expect("raw connect");
|
||||
partial
|
||||
.write_all(&100_u32.to_be_bytes())
|
||||
.await
|
||||
.expect("header");
|
||||
partial.write_all(b"{}").await.expect("partial");
|
||||
partial.shutdown().await.expect("close");
|
||||
drop(reconnected);
|
||||
server.shutdown().await.expect("shutdown");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn plaintext_server_rejects_non_loopback_before_binding() {
|
||||
let configured = limits();
|
||||
let target = Arc::new(FakeTarget::default());
|
||||
let plane = plane(target, configured);
|
||||
let Err(error) = TcpControlServer::bind(
|
||||
plane,
|
||||
TcpControlConfig {
|
||||
listen: "0.0.0.0:7943".parse().expect("address"),
|
||||
limits: configured,
|
||||
},
|
||||
)
|
||||
.await
|
||||
else {
|
||||
panic!("plaintext remote listener was accepted")
|
||||
};
|
||||
assert_eq!(error.code, ControlErrorCode::InvalidRequest);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn plain_tcp_is_loopback_only_and_protocol_json_is_stable() {
|
||||
let configured = limits();
|
||||
assert!(
|
||||
!TcpControlConfig {
|
||||
listen: "0.0.0.0:7943".parse().expect("address"),
|
||||
limits: configured
|
||||
}
|
||||
.listen
|
||||
.ip()
|
||||
.is_loopback()
|
||||
);
|
||||
let encoded = serde_json::to_value(request("health", ControlRequest::Health)).expect("JSON");
|
||||
assert_eq!(
|
||||
encoded,
|
||||
json!({"version":1,"request_id":"health","request":{"method":"health"}})
|
||||
);
|
||||
}
|
||||
522
crates/metacrate-grid-agent/src/control_runtime.rs
Normal file
522
crates/metacrate-grid-agent/src/control_runtime.rs
Normal file
@@ -0,0 +1,522 @@
|
||||
//! Production management target backed by the agent's bounded runtime stores.
|
||||
|
||||
#![allow(clippy::missing_errors_doc)]
|
||||
|
||||
use crate::behavior::{BehaviorIngress, BehaviorMode};
|
||||
use crate::control_plane::{
|
||||
AuditEventView, CONTROL_PROTOCOL_VERSION, ControlContext, ControlError, ControlErrorCode,
|
||||
ControlFuture, ControlPayload, ControlRequest, ControlTarget, ConversationChannelView,
|
||||
HealthView, Page, PageRequest, PendingApprovalView, RuntimeView, ScheduledJobView,
|
||||
SessionMetadataView,
|
||||
};
|
||||
use crate::conversation::{ConversationChannel, ConversationKey, ConversationStore};
|
||||
use crate::policy::{
|
||||
ApprovalId, AuthenticatedPrincipal, MemoryPolicyAudit, PolicyFinalOutcome, PolicyGateway,
|
||||
PolicyReasonCode,
|
||||
};
|
||||
use crate::session::{SessionState, SessionStatus};
|
||||
use libremetaverse_types::compat::CancellationToken;
|
||||
use std::collections::BTreeMap;
|
||||
use std::fmt;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::sync::{Arc, Mutex, MutexGuard};
|
||||
use std::time::{Instant, SystemTime, UNIX_EPOCH};
|
||||
use tokio::sync::mpsc;
|
||||
|
||||
const ROAMING_JOB_ID: &str = "default-roaming";
|
||||
|
||||
/// Commands whose ownership remains with the service lifecycle loop.
|
||||
pub enum RuntimeControlCommand {
|
||||
Pause,
|
||||
Resume,
|
||||
ForceReconnect,
|
||||
OperatorMessage { message: String },
|
||||
GracefulShutdown,
|
||||
}
|
||||
|
||||
impl fmt::Debug for RuntimeControlCommand {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
match self {
|
||||
Self::OperatorMessage { message } => formatter
|
||||
.debug_struct("OperatorMessage")
|
||||
.field("message_bytes", &message.len())
|
||||
.finish(),
|
||||
Self::Pause => formatter.write_str("Pause"),
|
||||
Self::Resume => formatter.write_str("Resume"),
|
||||
Self::ForceReconnect => formatter.write_str("ForceReconnect"),
|
||||
Self::GracefulShutdown => formatter.write_str("GracefulShutdown"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
struct RuntimeState {
|
||||
session: SessionStatus,
|
||||
behavior: BehaviorMode,
|
||||
service_state: &'static str,
|
||||
region_id: Option<String>,
|
||||
region_name: Option<String>,
|
||||
position: Option<[f64; 3]>,
|
||||
}
|
||||
|
||||
impl Default for RuntimeState {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
session: SessionStatus::default(),
|
||||
behavior: BehaviorMode::Offline,
|
||||
service_state: "starting",
|
||||
region_id: None,
|
||||
region_name: None,
|
||||
position: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Real control target shared by integrated and split transports.
|
||||
pub struct AgentControlTarget {
|
||||
started: Instant,
|
||||
state: Mutex<RuntimeState>,
|
||||
conversations: Arc<ConversationStore>,
|
||||
policy: Arc<PolicyGateway>,
|
||||
audit: Arc<MemoryPolicyAudit>,
|
||||
behavior: BehaviorIngress,
|
||||
commands: mpsc::Sender<RuntimeControlCommand>,
|
||||
command_capacity: usize,
|
||||
jobs: Mutex<BTreeMap<String, bool>>,
|
||||
next_operation: AtomicU64,
|
||||
}
|
||||
|
||||
impl fmt::Debug for AgentControlTarget {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter
|
||||
.debug_struct("AgentControlTarget")
|
||||
.field("state", &lock(&self.state))
|
||||
.field("command_capacity", &self.command_capacity)
|
||||
.finish_non_exhaustive()
|
||||
}
|
||||
}
|
||||
|
||||
impl AgentControlTarget {
|
||||
pub fn new(
|
||||
conversations: Arc<ConversationStore>,
|
||||
policy: Arc<PolicyGateway>,
|
||||
audit: Arc<MemoryPolicyAudit>,
|
||||
behavior: BehaviorIngress,
|
||||
command_capacity: usize,
|
||||
) -> Result<(Arc<Self>, mpsc::Receiver<RuntimeControlCommand>), ControlError> {
|
||||
if command_capacity == 0 || command_capacity > 8_192 {
|
||||
return Err(control_error(
|
||||
ControlErrorCode::InvalidRequest,
|
||||
"invalid runtime control queue capacity",
|
||||
false,
|
||||
));
|
||||
}
|
||||
let (commands, receiver) = mpsc::channel(command_capacity);
|
||||
let mut jobs = BTreeMap::new();
|
||||
jobs.insert(ROAMING_JOB_ID.to_owned(), false);
|
||||
Ok((
|
||||
Arc::new(Self {
|
||||
started: Instant::now(),
|
||||
state: Mutex::new(RuntimeState::default()),
|
||||
conversations,
|
||||
policy,
|
||||
audit,
|
||||
behavior,
|
||||
commands,
|
||||
command_capacity,
|
||||
jobs: Mutex::new(jobs),
|
||||
next_operation: AtomicU64::new(1),
|
||||
}),
|
||||
receiver,
|
||||
))
|
||||
}
|
||||
|
||||
pub fn update_session(&self, session: SessionStatus) {
|
||||
let mut state = lock(&self.state);
|
||||
state.session = session;
|
||||
state.service_state = if session.state == SessionState::ShuttingDown {
|
||||
"stopping"
|
||||
} else {
|
||||
"running"
|
||||
};
|
||||
}
|
||||
|
||||
pub fn update_behavior(&self, behavior: BehaviorMode) {
|
||||
lock(&self.state).behavior = behavior;
|
||||
}
|
||||
|
||||
pub fn update_region(
|
||||
&self,
|
||||
region_id: Option<String>,
|
||||
region_name: Option<String>,
|
||||
position: Option<[f64; 3]>,
|
||||
) {
|
||||
let mut state = lock(&self.state);
|
||||
state.region_id = region_id.filter(|value| value.len() <= 64);
|
||||
state.region_name = region_name.filter(|value| value.len() <= 256);
|
||||
state.position =
|
||||
position.filter(|value| value.iter().all(|component| component.is_finite()));
|
||||
}
|
||||
|
||||
pub fn mark_stopping(&self) {
|
||||
lock(&self.state).service_state = "stopping";
|
||||
}
|
||||
|
||||
fn enqueue(
|
||||
&self,
|
||||
operation: &'static str,
|
||||
command: RuntimeControlCommand,
|
||||
) -> Result<ControlPayload, ControlError> {
|
||||
self.commands
|
||||
.try_send(command)
|
||||
.map_err(|error| match error {
|
||||
mpsc::error::TrySendError::Full(_) => control_error(
|
||||
ControlErrorCode::Backpressure,
|
||||
"runtime control queue is full",
|
||||
true,
|
||||
),
|
||||
mpsc::error::TrySendError::Closed(_) => control_error(
|
||||
ControlErrorCode::TransportClosed,
|
||||
"runtime control queue is closed",
|
||||
true,
|
||||
),
|
||||
})?;
|
||||
Ok(ControlPayload::Accepted {
|
||||
operation_id: format!(
|
||||
"{operation}-{}",
|
||||
self.next_operation.fetch_add(1, Ordering::Relaxed)
|
||||
),
|
||||
})
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_lines)] // Exhaustive protocol-to-runtime mapping stays auditable.
|
||||
fn execute_now(
|
||||
&self,
|
||||
context: &ControlContext,
|
||||
request: ControlRequest,
|
||||
) -> Result<ControlPayload, ControlError> {
|
||||
match request {
|
||||
ControlRequest::Health => {
|
||||
let state = lock(&self.state);
|
||||
Ok(ControlPayload::Health(HealthView {
|
||||
service_state: state.service_state.to_owned(),
|
||||
ready: state.session.agent_ready,
|
||||
uptime_seconds: self.started.elapsed().as_secs(),
|
||||
protocol_version: CONTROL_PROTOCOL_VERSION,
|
||||
}))
|
||||
}
|
||||
ControlRequest::Runtime => {
|
||||
let state = lock(&self.state).clone();
|
||||
let usage = self.policy.global_budget_usage();
|
||||
Ok(ControlPayload::Runtime(RuntimeView {
|
||||
grid_state: state.session.state.as_str().to_owned(),
|
||||
generation: state.session.generation,
|
||||
transport_connected: state.session.transport_connected,
|
||||
agent_ready: state.session.agent_ready,
|
||||
region_id: state.region_id,
|
||||
region_name: state.region_name,
|
||||
position: state.position,
|
||||
behavior_mode: behavior_name(state.behavior).to_owned(),
|
||||
control_queue_used: self
|
||||
.command_capacity
|
||||
.saturating_sub(self.commands.capacity()),
|
||||
control_queue_capacity: self.command_capacity,
|
||||
budget_tool_calls_used: usage.tool_calls,
|
||||
budget_movement_millimeters_used: usage.movement_millimeters,
|
||||
}))
|
||||
}
|
||||
ControlRequest::ListSessions { page } => {
|
||||
let values = self
|
||||
.conversations
|
||||
.list_metadata()
|
||||
.into_iter()
|
||||
.map(|metadata| SessionMetadataView {
|
||||
session_id: metadata.session_id.as_str().to_owned(),
|
||||
avatar_id: metadata.avatar_id.to_string(),
|
||||
channel: match metadata.channel {
|
||||
ConversationChannel::PublicChat => "public_chat",
|
||||
ConversationChannel::DirectIm => "direct_im",
|
||||
}
|
||||
.to_owned(),
|
||||
created_unix_millis: metadata.created_unix_millis,
|
||||
last_active_unix_millis: metadata.last_active_unix_millis,
|
||||
turns: metadata.turns,
|
||||
bytes: metadata.bytes,
|
||||
})
|
||||
.collect();
|
||||
Ok(ControlPayload::Sessions(page_values(&page, values)))
|
||||
}
|
||||
ControlRequest::ListScheduledJobs { page } => {
|
||||
let values = lock(&self.jobs)
|
||||
.iter()
|
||||
.map(|(job_id, enabled)| ScheduledJobView {
|
||||
job_id: job_id.clone(),
|
||||
kind: "bounded_roaming".to_owned(),
|
||||
enabled: *enabled,
|
||||
next_run_unix_millis: None,
|
||||
})
|
||||
.collect();
|
||||
Ok(ControlPayload::ScheduledJobs(page_values(&page, values)))
|
||||
}
|
||||
ControlRequest::ListPendingApprovals { page } => {
|
||||
let values = self
|
||||
.policy
|
||||
.pending_approvals(unix_seconds())
|
||||
.into_iter()
|
||||
.map(|approval| PendingApprovalView {
|
||||
approval_id: approval.id.get(),
|
||||
tool: approval.tool,
|
||||
principal: approval.principal,
|
||||
expires_unix_seconds: approval.expires_at,
|
||||
movement_millimeters: approval.cost.movement_millimeters,
|
||||
inventory_operations: approval.cost.inventory_operations,
|
||||
build_prims: approval.cost.build_prims,
|
||||
})
|
||||
.collect();
|
||||
Ok(ControlPayload::PendingApprovals(page_values(&page, values)))
|
||||
}
|
||||
ControlRequest::ListAuditEvents { page } => {
|
||||
let values = self
|
||||
.audit
|
||||
.snapshot()
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
.map(|(index, record)| AuditEventView {
|
||||
sequence: u64::try_from(index).unwrap_or(u64::MAX).saturating_add(1),
|
||||
unix_millis: record.recorded_unix_millis,
|
||||
principal: record.principal.as_str().to_owned(),
|
||||
operation: record.tool.as_str().to_owned(),
|
||||
outcome: policy_outcome_name(record.final_outcome).to_owned(),
|
||||
authorization_id: record.authorization_id,
|
||||
})
|
||||
.collect();
|
||||
Ok(ControlPayload::AuditEvents(page_values(&page, values)))
|
||||
}
|
||||
ControlRequest::PauseAutonomy => {
|
||||
let response = self.enqueue("pause", RuntimeControlCommand::Pause)?;
|
||||
self.behavior.pause();
|
||||
Ok(response)
|
||||
}
|
||||
ControlRequest::ResumeAutonomy => {
|
||||
let response = self.enqueue("resume", RuntimeControlCommand::Resume)?;
|
||||
self.behavior.resume();
|
||||
Ok(response)
|
||||
}
|
||||
ControlRequest::CancelAction { action_id } => {
|
||||
let cancelled = self.behavior.cancel_action(&action_id).map_err(|_| {
|
||||
control_error(ControlErrorCode::InvalidRequest, "invalid action ID", false)
|
||||
})?;
|
||||
if !cancelled {
|
||||
return Err(control_error(
|
||||
ControlErrorCode::NotFound,
|
||||
"active behavior action not found",
|
||||
false,
|
||||
));
|
||||
}
|
||||
Ok(ControlPayload::Completed)
|
||||
}
|
||||
ControlRequest::DecideApproval {
|
||||
approval_id,
|
||||
approve,
|
||||
} => {
|
||||
let id = ApprovalId::from_raw(approval_id).ok_or_else(|| {
|
||||
control_error(
|
||||
ControlErrorCode::InvalidRequest,
|
||||
"invalid approval ID",
|
||||
false,
|
||||
)
|
||||
})?;
|
||||
let principal =
|
||||
AuthenticatedPrincipal::from_authenticated_control(context.principal.clone())
|
||||
.map_err(|_| {
|
||||
control_error(
|
||||
ControlErrorCode::PermissionDenied,
|
||||
"invalid operator principal",
|
||||
false,
|
||||
)
|
||||
})?;
|
||||
let reason = if approve {
|
||||
self.policy.grant_approval(id, &principal, unix_seconds())
|
||||
} else {
|
||||
self.policy.deny_approval(id, &principal, unix_seconds())
|
||||
}
|
||||
.map_err(|_| {
|
||||
control_error(
|
||||
ControlErrorCode::Internal,
|
||||
"approval decision failed",
|
||||
false,
|
||||
)
|
||||
})?;
|
||||
match reason {
|
||||
PolicyReasonCode::Allowed | PolicyReasonCode::ApprovalNotGranted => {
|
||||
Ok(ControlPayload::Completed)
|
||||
}
|
||||
PolicyReasonCode::ApprovalUnknown => Err(control_error(
|
||||
ControlErrorCode::NotFound,
|
||||
"approval not found",
|
||||
false,
|
||||
)),
|
||||
PolicyReasonCode::ApprovalExpired | PolicyReasonCode::ApprovalReplayed => {
|
||||
Err(control_error(
|
||||
ControlErrorCode::Conflict,
|
||||
"approval is no longer pending",
|
||||
false,
|
||||
))
|
||||
}
|
||||
_ => Err(control_error(
|
||||
ControlErrorCode::Conflict,
|
||||
"approval decision rejected",
|
||||
false,
|
||||
)),
|
||||
}
|
||||
}
|
||||
ControlRequest::ForceReconnect => {
|
||||
self.enqueue("reconnect", RuntimeControlCommand::ForceReconnect)
|
||||
}
|
||||
ControlRequest::ExpireConversation { avatar_id, channel } => {
|
||||
let avatar_id = libremetaverse_types::UUID::parse(avatar_id).map_err(|_| {
|
||||
control_error(ControlErrorCode::InvalidRequest, "invalid avatar ID", false)
|
||||
})?;
|
||||
let channel = match channel {
|
||||
ConversationChannelView::PublicChat => ConversationChannel::PublicChat,
|
||||
ConversationChannelView::DirectIm => ConversationChannel::DirectIm,
|
||||
};
|
||||
let key = ConversationKey::new(avatar_id, channel).map_err(|_| {
|
||||
control_error(
|
||||
ControlErrorCode::InvalidRequest,
|
||||
"invalid conversation key",
|
||||
false,
|
||||
)
|
||||
})?;
|
||||
if self.conversations.expire(key) {
|
||||
Ok(ControlPayload::Completed)
|
||||
} else {
|
||||
Err(control_error(
|
||||
ControlErrorCode::NotFound,
|
||||
"conversation not found",
|
||||
false,
|
||||
))
|
||||
}
|
||||
}
|
||||
ControlRequest::SetRoamingJob { job_id, enabled } => {
|
||||
let mut jobs = lock(&self.jobs);
|
||||
let Some(current) = jobs.get_mut(&job_id) else {
|
||||
return Err(control_error(
|
||||
ControlErrorCode::NotFound,
|
||||
"roaming job not found",
|
||||
false,
|
||||
));
|
||||
};
|
||||
self.behavior.set_roaming(enabled).map_err(|_| {
|
||||
control_error(
|
||||
ControlErrorCode::Backpressure,
|
||||
"behavior queue unavailable",
|
||||
true,
|
||||
)
|
||||
})?;
|
||||
*current = enabled;
|
||||
Ok(ControlPayload::Completed)
|
||||
}
|
||||
ControlRequest::InjectOperatorMessage { message } => self.enqueue(
|
||||
"operator-message",
|
||||
RuntimeControlCommand::OperatorMessage { message },
|
||||
),
|
||||
ControlRequest::GracefulShutdown => {
|
||||
let response = self.enqueue("shutdown", RuntimeControlCommand::GracefulShutdown)?;
|
||||
self.mark_stopping();
|
||||
Ok(response)
|
||||
}
|
||||
ControlRequest::SubscribeEvents { .. } | ControlRequest::CancelRequest { .. } => {
|
||||
Err(control_error(
|
||||
ControlErrorCode::Internal,
|
||||
"transport request reached runtime",
|
||||
false,
|
||||
))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ControlTarget for AgentControlTarget {
|
||||
fn execute(
|
||||
&self,
|
||||
context: ControlContext,
|
||||
request: ControlRequest,
|
||||
cancellation: CancellationToken,
|
||||
) -> ControlFuture<'_> {
|
||||
Box::pin(async move {
|
||||
if cancellation.is_cancellation_requested() {
|
||||
return Err(control_error(
|
||||
ControlErrorCode::Cancelled,
|
||||
"control request cancelled",
|
||||
false,
|
||||
));
|
||||
}
|
||||
self.execute_now(&context, request)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn page_values<T>(request: &PageRequest, values: Vec<T>) -> Page<T> {
|
||||
let start = usize::try_from(request.cursor.unwrap_or(0))
|
||||
.unwrap_or(usize::MAX)
|
||||
.min(values.len());
|
||||
let end = start
|
||||
.saturating_add(usize::from(request.limit))
|
||||
.min(values.len());
|
||||
let next_cursor = (end < values.len()).then(|| u64::try_from(end).unwrap_or(u64::MAX));
|
||||
Page {
|
||||
items: values.into_iter().skip(start).take(end - start).collect(),
|
||||
next_cursor,
|
||||
}
|
||||
}
|
||||
|
||||
const fn behavior_name(mode: BehaviorMode) -> &'static str {
|
||||
match mode {
|
||||
BehaviorMode::Offline => "offline",
|
||||
BehaviorMode::Settling => "settling",
|
||||
BehaviorMode::Available => "available",
|
||||
BehaviorMode::Engaged => "engaged",
|
||||
BehaviorMode::Executing => "executing",
|
||||
BehaviorMode::Roaming => "roaming",
|
||||
BehaviorMode::Paused => "paused",
|
||||
BehaviorMode::Recovering => "recovering",
|
||||
}
|
||||
}
|
||||
|
||||
const fn policy_outcome_name(outcome: PolicyFinalOutcome) -> &'static str {
|
||||
match outcome {
|
||||
PolicyFinalOutcome::Denied => "denied",
|
||||
PolicyFinalOutcome::ApprovalRequired => "approval_required",
|
||||
PolicyFinalOutcome::ApprovalGranted => "approval_granted",
|
||||
PolicyFinalOutcome::Authorized => "authorized",
|
||||
PolicyFinalOutcome::Completed => "completed",
|
||||
PolicyFinalOutcome::Rejected => "rejected",
|
||||
PolicyFinalOutcome::Failed => "failed",
|
||||
PolicyFinalOutcome::AmbiguousMutation => "ambiguous_mutation",
|
||||
}
|
||||
}
|
||||
|
||||
fn unix_seconds() -> u64 {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.map_or(0, |duration| duration.as_secs())
|
||||
}
|
||||
|
||||
const fn control_error(
|
||||
code: ControlErrorCode,
|
||||
message: &'static str,
|
||||
retryable: bool,
|
||||
) -> ControlError {
|
||||
ControlError {
|
||||
code,
|
||||
message,
|
||||
retryable,
|
||||
}
|
||||
}
|
||||
|
||||
fn lock<T>(value: &Mutex<T>) -> MutexGuard<'_, T> {
|
||||
value
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
}
|
||||
167
crates/metacrate-grid-agent/src/control_runtime_tests.rs
Normal file
167
crates/metacrate-grid-agent/src/control_runtime_tests.rs
Normal file
@@ -0,0 +1,167 @@
|
||||
use crate::behavior::{BehaviorController, BehaviorError, EmbodimentFuture, EmbodimentSink};
|
||||
use crate::control_plane::{
|
||||
CONTROL_PROTOCOL_VERSION, ControlLimits, ControlPayload, ControlPlane, ControlRequest,
|
||||
ControlRequestEnvelope,
|
||||
};
|
||||
use crate::control_runtime::{AgentControlTarget, RuntimeControlCommand};
|
||||
use crate::conversation::{ConversationLimits, ConversationStore};
|
||||
use crate::perception::WorldPosition;
|
||||
use crate::policy::{MemoryPolicyAudit, PolicyAuditSink, PolicyGateway, PolicyLimits};
|
||||
use crate::{AgentConfig, EmbodiedPose, SecretString};
|
||||
use libremetaverse_types::UUID;
|
||||
use libremetaverse_types::compat::CancellationToken;
|
||||
use std::collections::BTreeSet;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
struct Sink;
|
||||
|
||||
impl EmbodimentSink for Sink {
|
||||
fn current_pose(
|
||||
&self,
|
||||
generation: u64,
|
||||
_cancellation: CancellationToken,
|
||||
) -> EmbodimentFuture<'_, EmbodiedPose> {
|
||||
Box::pin(async move {
|
||||
Ok(EmbodiedPose {
|
||||
generation,
|
||||
region_id: UUID::zero(),
|
||||
position: WorldPosition {
|
||||
x: 128.0,
|
||||
y: 128.0,
|
||||
z: 24.0,
|
||||
},
|
||||
heading_degrees: 0.0,
|
||||
sitting: false,
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
fn resolve_avatar(
|
||||
&self,
|
||||
_generation: u64,
|
||||
_avatar_id: UUID,
|
||||
_cancellation: CancellationToken,
|
||||
) -> EmbodimentFuture<'_, WorldPosition> {
|
||||
Box::pin(async { Err(BehaviorError::TargetUnavailable) })
|
||||
}
|
||||
|
||||
fn face_point(
|
||||
&self,
|
||||
_generation: u64,
|
||||
_point: WorldPosition,
|
||||
_cancellation: CancellationToken,
|
||||
) -> EmbodimentFuture<'_, ()> {
|
||||
Box::pin(async { Ok(()) })
|
||||
}
|
||||
|
||||
fn begin_walk(
|
||||
&self,
|
||||
_generation: u64,
|
||||
_point: WorldPosition,
|
||||
_cancellation: CancellationToken,
|
||||
) -> EmbodimentFuture<'_, ()> {
|
||||
Box::pin(async { Ok(()) })
|
||||
}
|
||||
|
||||
fn validate_walk_target(
|
||||
&self,
|
||||
_generation: u64,
|
||||
_point: WorldPosition,
|
||||
_cancellation: CancellationToken,
|
||||
) -> EmbodimentFuture<'_, ()> {
|
||||
Box::pin(async { Ok(()) })
|
||||
}
|
||||
|
||||
fn stop(&self, _generation: u64, _cancellation: CancellationToken) -> EmbodimentFuture<'_, ()> {
|
||||
Box::pin(async { Ok(()) })
|
||||
}
|
||||
|
||||
fn sit(&self, _generation: u64, _cancellation: CancellationToken) -> EmbodimentFuture<'_, ()> {
|
||||
Box::pin(async { Ok(()) })
|
||||
}
|
||||
|
||||
fn stand(
|
||||
&self,
|
||||
_generation: u64,
|
||||
_cancellation: CancellationToken,
|
||||
) -> EmbodimentFuture<'_, ()> {
|
||||
Box::pin(async { Ok(()) })
|
||||
}
|
||||
}
|
||||
|
||||
fn envelope(id: &str, request: ControlRequest) -> ControlRequestEnvelope {
|
||||
ControlRequestEnvelope {
|
||||
version: CONTROL_PROTOCOL_VERSION,
|
||||
request_id: id.to_owned(),
|
||||
request,
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn production_target_projects_state_and_routes_real_mutations() {
|
||||
let settings = AgentConfig::offline("https://llm.invalid/chat", "llm-key")
|
||||
.expect("offline config")
|
||||
.behavior;
|
||||
let behavior = BehaviorController::new(settings, Arc::new(Sink), 8, 8, Duration::from_secs(1))
|
||||
.expect("behavior")
|
||||
.start();
|
||||
let audit = Arc::new(MemoryPolicyAudit::new(32).expect("audit"));
|
||||
let audit_sink: Arc<dyn PolicyAuditSink> = audit.clone();
|
||||
let policy = Arc::new(
|
||||
PolicyGateway::new(
|
||||
BTreeSet::new(),
|
||||
Vec::new(),
|
||||
PolicyLimits::default(),
|
||||
audit_sink,
|
||||
)
|
||||
.expect("policy"),
|
||||
);
|
||||
let conversations = Arc::new(
|
||||
ConversationStore::open(ConversationLimits::default(), None).expect("conversations"),
|
||||
);
|
||||
let (target, mut commands) =
|
||||
AgentControlTarget::new(conversations, policy, audit, behavior.ingress(), 8)
|
||||
.expect("target");
|
||||
target.update_region(
|
||||
Some("00000000-0000-4000-8000-000000000001".into()),
|
||||
Some("Test Region".into()),
|
||||
Some([128.0, 128.0, 24.0]),
|
||||
);
|
||||
let plane = ControlPlane::new(
|
||||
target,
|
||||
SecretString::new("test.operator", "operator-token").expect("token"),
|
||||
Some(SecretString::new("test.observer", "observer-token").expect("token")),
|
||||
ControlLimits::default(),
|
||||
)
|
||||
.expect("plane");
|
||||
let operator = plane.connect("operator-token").expect("operator");
|
||||
let runtime = operator
|
||||
.request(envelope("runtime", ControlRequest::Runtime))
|
||||
.await;
|
||||
let Ok(ControlPayload::Runtime(runtime)) = runtime.result else {
|
||||
panic!("runtime projection")
|
||||
};
|
||||
assert_eq!(runtime.region_name.as_deref(), Some("Test Region"));
|
||||
assert_eq!(runtime.control_queue_capacity, 8);
|
||||
|
||||
assert!(
|
||||
operator
|
||||
.request(envelope("pause", ControlRequest::PauseAutonomy))
|
||||
.await
|
||||
.result
|
||||
.is_ok()
|
||||
);
|
||||
assert!(matches!(
|
||||
commands.recv().await,
|
||||
Some(RuntimeControlCommand::Pause)
|
||||
));
|
||||
|
||||
let observer = plane.connect("observer-token").expect("observer");
|
||||
let denied = observer
|
||||
.request(envelope("shutdown", ControlRequest::GracefulShutdown))
|
||||
.await;
|
||||
assert!(denied.result.is_err());
|
||||
assert!(commands.try_recv().is_err());
|
||||
behavior.shutdown().await.expect("shutdown behavior");
|
||||
}
|
||||
@@ -7,6 +7,8 @@
|
||||
pub mod backend;
|
||||
pub mod behavior;
|
||||
pub mod config;
|
||||
pub mod control_plane;
|
||||
pub mod control_runtime;
|
||||
pub mod conversation;
|
||||
pub mod interaction;
|
||||
pub mod llm;
|
||||
@@ -20,6 +22,10 @@ pub mod types;
|
||||
#[cfg(test)]
|
||||
mod behavior_tests;
|
||||
#[cfg(test)]
|
||||
mod control_plane_tests;
|
||||
#[cfg(test)]
|
||||
mod control_runtime_tests;
|
||||
#[cfg(test)]
|
||||
mod conversation_tests;
|
||||
#[cfg(test)]
|
||||
mod interaction_tests;
|
||||
@@ -47,10 +53,19 @@ pub use behavior::{
|
||||
STOP_TOOL, WALK_SHORT_TOOL, behavior_policy_tools,
|
||||
};
|
||||
pub use config::{
|
||||
AgentConfig, BehaviorSettings, ConfigError, ConfigLoader, ConversationSettings, EndpointUrl,
|
||||
Environment, GridConnection, Limits, LlmConnection, MapEnvironment, OperatingMode,
|
||||
SecretString, StdEnvironment, Timeouts,
|
||||
AgentConfig, BehaviorSettings, ConfigError, ConfigLoader, ControlSettings,
|
||||
ConversationSettings, EndpointUrl, Environment, GridConnection, Limits, LlmConnection,
|
||||
MapEnvironment, OperatingMode, RemoteTlsSettings, SecretString, StdEnvironment, Timeouts,
|
||||
};
|
||||
pub use control_plane::{
|
||||
AuditEventView, CONTROL_PROTOCOL_VERSION, ControlContext, ControlError, ControlErrorBody,
|
||||
ControlErrorCode, ControlEvent, ControlEventKind, ControlFuture, ControlLimits, ControlPayload,
|
||||
ControlPlane, ControlRequest, ControlRequestEnvelope, ControlResponseEnvelope, ControlRole,
|
||||
ControlSubscription, ControlTarget, ConversationChannelView, HealthView,
|
||||
InProcessControlClient, Page, PageRequest, PendingApprovalView, RuntimeView, ScheduledJobView,
|
||||
SessionMetadataView, TcpControlClient, TcpControlConfig, TcpControlServer,
|
||||
};
|
||||
pub use control_runtime::{AgentControlTarget, RuntimeControlCommand};
|
||||
pub use conversation::{
|
||||
ConversationChannel, ConversationClock, ConversationContext, ConversationError,
|
||||
ConversationKey, ConversationLimits, ConversationMetadata, ConversationPersistence,
|
||||
@@ -81,10 +96,11 @@ pub use perception::{
|
||||
pub use policy::{
|
||||
ActionOrigin, AllowedOrigins, ApprovalId, ApprovalRule, AuthenticatedPrincipal,
|
||||
AuthorizedAction, BudgetLimits, Capability, FixedCost, Idempotency, MemoryPolicyAudit,
|
||||
OriginClass, PolicyAuditError, PolicyAuditRecord, PolicyAuditSink, PolicyDisposition,
|
||||
PolicyError, PolicyEvaluation, PolicyFinalOutcome, PolicyGateway, PolicyLimits,
|
||||
PolicyReasonCode, PolicyRequestContext, PolicySnapshot, PolicyTool, PolicyToolExecutor,
|
||||
ResourceCost, ResourceEstimator, Risk, SchedulerGrantId, UntrustedData, UntrustedSource,
|
||||
OriginClass, PendingApprovalMetadata, PolicyAuditError, PolicyAuditRecord, PolicyAuditSink,
|
||||
PolicyDisposition, PolicyError, PolicyEvaluation, PolicyFinalOutcome, PolicyGateway,
|
||||
PolicyLimits, PolicyReasonCode, PolicyRequestContext, PolicySnapshot, PolicyTool,
|
||||
PolicyToolExecutor, ResourceCost, ResourceEstimator, Risk, SchedulerGrantId, UntrustedData,
|
||||
UntrustedSource,
|
||||
};
|
||||
pub use service::{AgentService, ServiceError, ServiceHandle, ServiceState};
|
||||
pub use session::{
|
||||
|
||||
@@ -147,8 +147,10 @@ async fn run_live(
|
||||
run_once: bool,
|
||||
) -> Result<(), Box<dyn Error>> {
|
||||
use metacrate_grid_agent::{
|
||||
GridSessionBackend, LibremetaverseClientOwner, SessionObservation, SessionState,
|
||||
SessionSupervisor,
|
||||
AgentControlTarget, BehaviorObservation, ControlEventKind, ControlPlane, ControlTarget,
|
||||
GridSessionBackend, LibremetaverseClientOwner, OperatingMode, RuntimeControlCommand,
|
||||
SessionControl, SessionObservation, SessionState, SessionSupervisor, TcpControlConfig,
|
||||
TcpControlServer,
|
||||
};
|
||||
|
||||
let connection = config.grid.clone().ok_or_else(|| {
|
||||
@@ -230,8 +232,56 @@ async fn run_live(
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let (control_target, mut control_commands) = AgentControlTarget::new(
|
||||
live.conversations.clone(),
|
||||
live.policy.clone(),
|
||||
live.audit.clone(),
|
||||
live.behavior.ingress(),
|
||||
config.limits.control_queue,
|
||||
)?;
|
||||
control_target.update_session(handle.status());
|
||||
let erased_target: Arc<dyn ControlTarget> = control_target.clone();
|
||||
let (control_plane, _integrated_client, control_server) = match config.mode {
|
||||
OperatingMode::Integrated => {
|
||||
let (plane, client) = ControlPlane::integrated(erased_target, config.control.limits)?;
|
||||
(plane, Some(client), None)
|
||||
}
|
||||
OperatingMode::SplitService => {
|
||||
let operator = config.control.operator_token.clone().ok_or_else(|| {
|
||||
CliError("validated split configuration omitted its operator token".into())
|
||||
})?;
|
||||
let plane = ControlPlane::new(
|
||||
erased_target,
|
||||
operator,
|
||||
config.control.observer_token.clone(),
|
||||
config.control.limits,
|
||||
)?;
|
||||
let transport = TcpControlConfig {
|
||||
listen: config.control.listen,
|
||||
limits: config.control.limits,
|
||||
};
|
||||
let server = if let Some(tls) = &config.control.remote_tls {
|
||||
eprintln!(
|
||||
"WARNING: remote control is enabled with TLS; protect operator tokens and certificate keys"
|
||||
);
|
||||
TcpControlServer::bind_tls(plane.clone(), transport, tls.server_config()?).await?
|
||||
} else {
|
||||
TcpControlServer::bind(plane.clone(), transport).await?
|
||||
};
|
||||
println!(
|
||||
"grid agent control plane listening on {}",
|
||||
server.local_addr()
|
||||
);
|
||||
(plane, None, Some(server))
|
||||
}
|
||||
OperatingMode::OfflineFake => {
|
||||
return Err(CliError("offline mode reached the live control plane".into()).into());
|
||||
}
|
||||
};
|
||||
|
||||
println!("grid agent session supervisor started; press Ctrl-C to stop");
|
||||
let mut signal_error = None;
|
||||
let mut control_failure = None;
|
||||
loop {
|
||||
tokio::select! {
|
||||
signal = tokio::signal::ctrl_c() => {
|
||||
@@ -243,6 +293,11 @@ async fn run_live(
|
||||
event = handle.next_observation() => {
|
||||
let Some(event) = event else { break; };
|
||||
if let SessionObservation::Transition { status, reason, retry_in } = event {
|
||||
control_target.update_session(status);
|
||||
control_plane.publish(ControlEventKind::StateChanged {
|
||||
component: "session".into(),
|
||||
state: status.state.as_str().into(),
|
||||
});
|
||||
println!(
|
||||
"grid session state={} generation={} transport_connected={} agent_ready={} reason={reason:?} retry_in={retry_in:?}",
|
||||
status.state.as_str(),
|
||||
@@ -262,19 +317,66 @@ async fn run_live(
|
||||
}
|
||||
event = live.behavior.next_observation() => {
|
||||
let Some(event) = event else { break; };
|
||||
if let BehaviorObservation::Transition { to, .. } = &event {
|
||||
control_target.update_behavior(*to);
|
||||
control_plane.publish(ControlEventKind::StateChanged {
|
||||
component: "behavior".into(),
|
||||
state: format!("{to:?}").to_ascii_lowercase(),
|
||||
});
|
||||
}
|
||||
println!("grid behavior event={event:?}");
|
||||
}
|
||||
command = control_commands.recv() => {
|
||||
let Some(command) = command else { break; };
|
||||
match command {
|
||||
RuntimeControlCommand::Pause => {
|
||||
if let Err(error) = handle.control(SessionControl::Pause).await {
|
||||
control_failure = Some(error.to_string());
|
||||
break;
|
||||
}
|
||||
}
|
||||
RuntimeControlCommand::Resume => {
|
||||
if let Err(error) = handle.control(SessionControl::Resume).await {
|
||||
control_failure = Some(error.to_string());
|
||||
break;
|
||||
}
|
||||
}
|
||||
RuntimeControlCommand::ForceReconnect => {
|
||||
if let Err(error) = handle.control(SessionControl::ForceReconnect).await {
|
||||
control_failure = Some(error.to_string());
|
||||
break;
|
||||
}
|
||||
}
|
||||
RuntimeControlCommand::OperatorMessage { message } => {
|
||||
control_plane.publish(ControlEventKind::StateChanged {
|
||||
component: "operator_message".into(),
|
||||
state: format!("accepted_{}_bytes", message.len()),
|
||||
});
|
||||
}
|
||||
RuntimeControlCommand::GracefulShutdown => break,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
control_target.mark_stopping();
|
||||
let session_result = handle.shutdown().await;
|
||||
let interaction_result = live.interaction.shutdown().await;
|
||||
let behavior_result = live.behavior.shutdown().await;
|
||||
let control_result = if let Some(server) = control_server {
|
||||
server.shutdown().await
|
||||
} else {
|
||||
Ok(())
|
||||
};
|
||||
session_result?;
|
||||
interaction_result?;
|
||||
behavior_result?;
|
||||
control_result?;
|
||||
if let Some(error) = signal_error {
|
||||
return Err(error.into());
|
||||
}
|
||||
if let Some(error) = control_failure {
|
||||
return Err(CliError(format!("control command failed: {error}")).into());
|
||||
}
|
||||
println!("grid agent stopped cleanly");
|
||||
Ok(())
|
||||
}
|
||||
@@ -286,9 +388,13 @@ struct LiveInteractions {
|
||||
perception_observations:
|
||||
tokio::sync::mpsc::Receiver<metacrate_grid_agent::PerceptionObservation>,
|
||||
behavior: metacrate_grid_agent::BehaviorHandle,
|
||||
conversations: Arc<metacrate_grid_agent::ConversationStore>,
|
||||
policy: Arc<metacrate_grid_agent::PolicyGateway>,
|
||||
audit: Arc<metacrate_grid_agent::MemoryPolicyAudit>,
|
||||
}
|
||||
|
||||
#[cfg(feature = "live-grid")]
|
||||
#[allow(clippy::too_many_lines)]
|
||||
fn start_live_interactions(
|
||||
config: &metacrate_grid_agent::AgentConfig,
|
||||
owner: &metacrate_grid_agent::LibremetaverseClientOwner,
|
||||
@@ -347,7 +453,7 @@ fn start_live_interactions(
|
||||
config.authorized_avatar_uuids.clone(),
|
||||
tools,
|
||||
PolicyLimits::default(),
|
||||
audit,
|
||||
audit.clone(),
|
||||
)?);
|
||||
let loop_limits = ToolLoopLimits {
|
||||
max_tool_calls_per_turn: config.limits.max_tool_calls,
|
||||
@@ -366,7 +472,7 @@ fn start_live_interactions(
|
||||
Arc::new(AuthorizedBackendRouter::new(routes)?);
|
||||
let responder = Arc::new(PolicyLlmResponder::new(
|
||||
client,
|
||||
gateway,
|
||||
gateway.clone(),
|
||||
routed_backend,
|
||||
loop_limits,
|
||||
now,
|
||||
@@ -377,7 +483,7 @@ fn start_live_interactions(
|
||||
config.interaction.clone(),
|
||||
libremetaverse_types::UUID::zero(),
|
||||
config.authorized_avatar_uuids.clone(),
|
||||
conversation,
|
||||
conversation.clone(),
|
||||
responder,
|
||||
sink,
|
||||
config.limits.grid_event_queue,
|
||||
@@ -391,5 +497,8 @@ fn start_live_interactions(
|
||||
perception: perception_ingress,
|
||||
perception_observations,
|
||||
behavior,
|
||||
conversations: conversation,
|
||||
policy: gateway,
|
||||
audit,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -16,6 +16,7 @@ use std::collections::{BTreeMap, BTreeSet, VecDeque};
|
||||
use std::error::Error;
|
||||
use std::fmt;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
const MAX_POLICY_TOOLS: usize = 64;
|
||||
const MAX_AUTHORIZED_AVATARS: usize = 1_024;
|
||||
@@ -238,6 +239,16 @@ pub struct ResourceCost {
|
||||
pub build_prims: u64,
|
||||
}
|
||||
|
||||
/// Secret-free operator projection of one pending approval.
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub struct PendingApprovalMetadata {
|
||||
pub id: ApprovalId,
|
||||
pub principal: String,
|
||||
pub tool: String,
|
||||
pub expires_at: u64,
|
||||
pub cost: ResourceCost,
|
||||
}
|
||||
|
||||
impl ResourceCost {
|
||||
#[must_use]
|
||||
pub const fn one_call() -> Self {
|
||||
@@ -514,6 +525,7 @@ pub enum PolicyFinalOutcome {
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub struct PolicyAuditRecord {
|
||||
pub recorded_unix_millis: u64,
|
||||
pub authorization_id: Option<u64>,
|
||||
pub approval_id: Option<ApprovalId>,
|
||||
pub origin: OriginClass,
|
||||
@@ -581,6 +593,18 @@ impl PolicyAuditSink for MemoryPolicyAudit {
|
||||
#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)]
|
||||
pub struct ApprovalId(u64);
|
||||
|
||||
impl ApprovalId {
|
||||
#[must_use]
|
||||
pub const fn from_raw(value: u64) -> Option<Self> {
|
||||
if value == 0 { None } else { Some(Self(value)) }
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn get(self) -> u64 {
|
||||
self.0
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)]
|
||||
pub struct SchedulerGrantId(u64);
|
||||
|
||||
@@ -588,6 +612,7 @@ pub struct SchedulerGrantId(u64);
|
||||
enum ApprovalStatus {
|
||||
Pending,
|
||||
Granted,
|
||||
Denied,
|
||||
Consumed,
|
||||
}
|
||||
|
||||
@@ -820,6 +845,31 @@ impl PolicyGateway {
|
||||
PolicySnapshot(lock(&self.state).clone())
|
||||
}
|
||||
|
||||
/// Returns bounded, content-free approval metadata for operator UIs.
|
||||
#[must_use]
|
||||
pub fn pending_approvals(&self, now: u64) -> Vec<PendingApprovalMetadata> {
|
||||
lock(&self.state)
|
||||
.approvals
|
||||
.values()
|
||||
.filter(|approval| {
|
||||
approval.status == ApprovalStatus::Pending && approval.expires_at > now
|
||||
})
|
||||
.map(|approval| PendingApprovalMetadata {
|
||||
id: approval.id,
|
||||
principal: approval.principal.audit_label(),
|
||||
tool: approval.tool.clone(),
|
||||
expires_at: approval.expires_at,
|
||||
cost: approval.cost,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Current global budget use for a redacted control-plane projection.
|
||||
#[must_use]
|
||||
pub fn global_budget_usage(&self) -> ResourceCost {
|
||||
lock(&self.state).global_used
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn pending_approval_count(&self) -> usize {
|
||||
lock(&self.state)
|
||||
@@ -1064,7 +1114,9 @@ impl PolicyGateway {
|
||||
match approval.status {
|
||||
ApprovalStatus::Pending => Some(PolicyReasonCode::ApprovalNotGranted),
|
||||
ApprovalStatus::Granted => None,
|
||||
ApprovalStatus::Consumed => Some(PolicyReasonCode::ApprovalReplayed),
|
||||
ApprovalStatus::Denied | ApprovalStatus::Consumed => {
|
||||
Some(PolicyReasonCode::ApprovalReplayed)
|
||||
}
|
||||
}
|
||||
};
|
||||
if let Some(reason) = reason {
|
||||
@@ -1121,8 +1173,12 @@ impl PolicyGateway {
|
||||
}
|
||||
if state.approvals.len() == MAX_APPROVALS {
|
||||
let removable = state.approvals.iter().find_map(|(id, approval)| {
|
||||
(approval.expires_at <= now || approval.status == ApprovalStatus::Consumed)
|
||||
.then_some(*id)
|
||||
(approval.expires_at <= now
|
||||
|| matches!(
|
||||
approval.status,
|
||||
ApprovalStatus::Denied | ApprovalStatus::Consumed
|
||||
))
|
||||
.then_some(*id)
|
||||
});
|
||||
let Some(removable) = removable else {
|
||||
drop(state);
|
||||
@@ -1323,6 +1379,40 @@ impl PolicyGateway {
|
||||
Ok(PolicyReasonCode::Allowed)
|
||||
}
|
||||
|
||||
pub fn deny_approval(
|
||||
&self,
|
||||
id: ApprovalId,
|
||||
_operator: &AuthenticatedPrincipal,
|
||||
now: u64,
|
||||
) -> Result<PolicyReasonCode, PolicyError> {
|
||||
let mut state = lock(&self.state);
|
||||
let Some(approval) = state.approvals.get_mut(&id) else {
|
||||
return Ok(PolicyReasonCode::ApprovalUnknown);
|
||||
};
|
||||
if approval.expires_at <= now {
|
||||
return Ok(PolicyReasonCode::ApprovalExpired);
|
||||
}
|
||||
if approval.status != ApprovalStatus::Pending {
|
||||
return Ok(PolicyReasonCode::ApprovalReplayed);
|
||||
}
|
||||
let audit = audit_record(
|
||||
&approval.context,
|
||||
approval.origin,
|
||||
&approval.principal,
|
||||
BoundedText::new("policy.tool", approval.tool.clone())?,
|
||||
None,
|
||||
Some(id),
|
||||
PolicyDisposition::Denied,
|
||||
PolicyReasonCode::ApprovalNotGranted,
|
||||
approval.cost,
|
||||
approval.arguments_hash,
|
||||
PolicyFinalOutcome::Denied,
|
||||
)?;
|
||||
self.emit(audit)?;
|
||||
approval.status = ApprovalStatus::Denied;
|
||||
Ok(PolicyReasonCode::ApprovalNotGranted)
|
||||
}
|
||||
|
||||
/// Consumes the originating authorization so it cannot be replayed to
|
||||
/// create more than one scheduler grant.
|
||||
#[allow(clippy::needless_pass_by_value)]
|
||||
@@ -1621,6 +1711,11 @@ fn audit_record(
|
||||
final_outcome: PolicyFinalOutcome,
|
||||
) -> Result<PolicyAuditRecord, PolicyError> {
|
||||
Ok(PolicyAuditRecord {
|
||||
recorded_unix_millis: SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.map_or(0, |duration| {
|
||||
u64::try_from(duration.as_millis()).unwrap_or(u64::MAX)
|
||||
}),
|
||||
authorization_id,
|
||||
approval_id,
|
||||
origin,
|
||||
|
||||
@@ -411,6 +411,7 @@ fn approval_tool() -> PolicyTool {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[allow(clippy::too_many_lines)] // One approval lifecycle includes deny and expiry paths.
|
||||
fn approvals_bind_principal_canonical_arguments_expiry_and_one_execution() {
|
||||
let (gateway, audit) = gateway_with(vec![approval_tool()], PolicyLimits::default());
|
||||
let requester = context(OriginClass::AuthorizedIm);
|
||||
@@ -507,6 +508,24 @@ fn approvals_bind_principal_canonical_arguments_expiry_and_one_execution() {
|
||||
.expect("expired result"),
|
||||
PolicyReasonCode::ApprovalExpired
|
||||
);
|
||||
let denied = gateway
|
||||
.evaluate(&requester, &proposed, &arguments, None, 600)
|
||||
.expect("denial pending")
|
||||
.approval_id
|
||||
.expect("approval ID");
|
||||
assert_eq!(gateway.pending_approvals(600).len(), 1);
|
||||
assert_eq!(
|
||||
gateway.deny_approval(denied, &operator, 601).expect("deny"),
|
||||
PolicyReasonCode::ApprovalNotGranted
|
||||
);
|
||||
assert!(gateway.pending_approvals(601).is_empty());
|
||||
assert_eq!(
|
||||
gateway
|
||||
.evaluate(&requester, &proposed, &arguments, Some(denied), 601)
|
||||
.expect("denied approval cannot execute")
|
||||
.reason,
|
||||
PolicyReasonCode::ApprovalReplayed
|
||||
);
|
||||
assert!(
|
||||
audit
|
||||
.snapshot()
|
||||
|
||||
@@ -2,14 +2,16 @@ use std::collections::BTreeSet;
|
||||
use std::fs;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
const ALLOWED_DEPENDENCIES: [&str; 8] = [
|
||||
const ALLOWED_DEPENDENCIES: [&str; 10] = [
|
||||
"libremetaverse",
|
||||
"libremetaverse-types",
|
||||
"reqwest",
|
||||
"rustls",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha2",
|
||||
"tokio",
|
||||
"tokio-rustls",
|
||||
"url",
|
||||
];
|
||||
|
||||
@@ -46,7 +48,7 @@ fn runtime_source_has_no_subprocess_or_native_abi_escape_hatch() {
|
||||
let mut files = Vec::with_capacity(20);
|
||||
collect_rust_files(&source, &mut files);
|
||||
assert!(
|
||||
files.len() <= 20,
|
||||
files.len() <= 24,
|
||||
"source-file count needs a reviewed bound update"
|
||||
);
|
||||
for path in files {
|
||||
|
||||
91
docs/grid-agent-control-plane.md
Normal file
91
docs/grid-agent-control-plane.md
Normal file
@@ -0,0 +1,91 @@
|
||||
# Grid-agent control plane v1
|
||||
|
||||
The grid agent exposes one transport-neutral request/response and event API.
|
||||
Integrated clients receive an `InProcessControlClient`; split clients use the
|
||||
same `ControlRequestEnvelope`, `ControlResponseEnvelope`, and `ControlEvent`
|
||||
types over TCP. No terminal, Unix socket, named-pipe, permission-bit, or
|
||||
platform-specific type appears in the core API.
|
||||
|
||||
## Transport and negotiation
|
||||
|
||||
TCP frames are a four-byte unsigned big-endian length followed by one UTF-8
|
||||
JSON value. Empty, partial, malformed, idle, and oversized frames close the
|
||||
connection within configured deadlines. The first frame must be:
|
||||
|
||||
```json
|
||||
{"frame":"hello","body":{"version":1,"token":"separate-operator-capability"}}
|
||||
```
|
||||
|
||||
The server answers with a `hello` containing version `1`, the authenticated
|
||||
`operator` or `observer` role, or a typed error. Every later client frame is a
|
||||
`request`; server frames are `response` or `event`. Request IDs contain at most
|
||||
96 ASCII identifier bytes and cannot be replayed on a connection. A caller can
|
||||
cancel an active request by its ID. Responses and events can arrive in either
|
||||
order, so clients correlate responses by request ID.
|
||||
|
||||
Plain TCP binds only to IPv4 or IPv6 loopback. Remote control is off by default
|
||||
and a non-loopback address is rejected unless both DER certificate and private
|
||||
key paths are explicitly configured. Remote mode uses Rustls before the hello
|
||||
exchange and emits an operator warning at startup. The TLS client API requires
|
||||
the embedding client to supply its certificate roots and server-name policy.
|
||||
Every split connection must still authenticate with a dedicated control token;
|
||||
observer and operator tokens must differ and configuration rejects reuse of an
|
||||
LLM API key or grid password. Tokens are absent from protocol errors, audit,
|
||||
serialization traits, and diagnostics.
|
||||
|
||||
Remote service configuration uses the JSON fields
|
||||
`control.remote_tls_certificate_der` and
|
||||
`control.remote_tls_private_key_der` together with a non-loopback
|
||||
`control.listen` value. Both files are bounded, non-symlink DER files. Omitting
|
||||
either field, or selecting a non-loopback address without both, fails
|
||||
configuration before a listener is created.
|
||||
|
||||
## Requests
|
||||
|
||||
The JSON request envelope is stable and versioned. For example:
|
||||
|
||||
```json
|
||||
{"version":1,"request_id":"health-1","request":{"method":"health"}}
|
||||
```
|
||||
|
||||
Observers can call `health`, `runtime`, `list_sessions`,
|
||||
`list_scheduled_jobs`, `list_pending_approvals`, `list_audit_events`, and
|
||||
`subscribe_events`. Operators can additionally call `cancel_request`,
|
||||
`pause_autonomy`, `resume_autonomy`, `cancel_action`, `decide_approval`,
|
||||
`force_reconnect`, `expire_conversation`, `set_roaming_job`,
|
||||
`inject_operator_message`, and `graceful_shutdown`. Cancellation is a mutation
|
||||
and is operator-only. List requests use an opaque numeric cursor and a page
|
||||
size of 1 through 100.
|
||||
|
||||
Runtime projections contain lifecycle/readiness, session generation, safe
|
||||
region and pose fields when known, behavior mode, control-queue utilization,
|
||||
and aggregate budget use. Conversation responses contain metadata only. Audit
|
||||
and approval responses exclude arguments, prompt contents, credentials,
|
||||
authorization headers, capability URLs, model reasoning, and filesystem data.
|
||||
`cancel_action` binds to the exact bounded action ID carried by behavior audit
|
||||
observations; it can cancel a queued or executing embodied action without
|
||||
preempting unrelated work. The built-in roaming job ID is `default-roaming`.
|
||||
|
||||
Errors are typed as `authentication_failed`, `version_mismatch`,
|
||||
`permission_denied`, `invalid_request`, `replay`, `not_found`, `conflict`,
|
||||
`cancelled`, `timed_out`, `busy`, `backpressure`, `frame_too_large`,
|
||||
`idle_timeout`, `transport_closed`, or `internal`, with a bounded safe message
|
||||
and a retryable flag.
|
||||
|
||||
## Events, bounds, and reconnect
|
||||
|
||||
Events carry monotonically increasing sequence numbers. A subscription can
|
||||
resume after its last observed sequence. If retained history no longer covers
|
||||
that point, its first item is an explicit `gap` with `first_available` and
|
||||
`last_missed`, followed by retained events. Mutation events record the
|
||||
authenticated role/connection principal, operation name, and completed or
|
||||
rejected outcome, never request contents.
|
||||
|
||||
Connections, unauthenticated handshakes, in-flight request tasks,
|
||||
subscriptions, command/event queues, replay history, event history, event
|
||||
bytes, event rate, frames, pages, idle time, request time, and writes all have
|
||||
validated hard bounds. A full event or writer queue disconnects the slow
|
||||
consumer instead of blocking the headless service. With zero clients, event
|
||||
publication retains only the configured history and creates no background
|
||||
work. Server shutdown cancels and joins all listener, connection, request, and
|
||||
writer tasks within its deadline.
|
||||
Reference in New Issue
Block a user