feat(grid-agent): add portable control plane (#126)
Some checks failed
CI / rust-skia (Rust only) (push) Successful in 2m44s
CI / required (push) Failing after 2m43s

This commit is contained in:
2026-08-18 04:29:12 +00:00
parent 058ed10005
commit 962d17257d
16 changed files with 4023 additions and 31 deletions

2
Cargo.lock generated
View File

@@ -2201,10 +2201,12 @@ dependencies = [
"libremetaverse",
"libremetaverse-types",
"reqwest",
"rustls",
"serde",
"serde_json",
"sha2 0.11.0",
"tokio",
"tokio-rustls",
"url",
]

View File

@@ -12,10 +12,12 @@ publish = false
libremetaverse = { version = "0.0.1", path = "../libremetaverse", default-features = false, optional = true }
libremetaverse-types = { version = "0.0.1", path = "../libremetaverse-types" }
reqwest = { version = "0.13.4", default-features = false, features = ["rustls"] }
rustls = { version = "0.23.43", default-features = false, features = ["aws_lc_rs", "std", "tls12"] }
serde = { version = "1", features = ["derive"] }
serde_json = "1"
sha2 = "0.11"
tokio = { version = "1.53.1", features = ["macros", "rt", "sync", "time"] }
tokio-rustls = { version = "0.26.4", default-features = false, features = ["aws_lc_rs", "tls12"] }
url = "2.5.8"
[dev-dependencies]

View File

@@ -22,7 +22,10 @@ optional JSON file, its referenced secret files, then environment values (an
environment-referenced secret file is below a direct environment secret).
Supported secret environment variables are
`METACRATE_AGENT_LLM_API_KEY[_FILE]` and
`METACRATE_AGENT_GRID_PASSWORD[_FILE]`. Secret files must be bounded regular,
`METACRATE_AGENT_GRID_PASSWORD[_FILE]`. Split control uses the separate
`METACRATE_AGENT_CONTROL_OPERATOR_TOKEN[_FILE]` and optional
`METACRATE_AGENT_CONTROL_OBSERVER_TOKEN[_FILE]`, plus
`METACRATE_AGENT_CONTROL_LISTEN`. Secret files must be bounded regular,
non-symlink UTF-8 files containing one line. Operators must restrict their OS
ACLs to the service identity; the core uses only portable `std::fs` checks and
does not assume Unix permission bits.
@@ -51,3 +54,6 @@ shutdown deadline are specified in
Per-avatar/channel expiry, compaction, redaction, optional atomic persistence,
and metadata-only operator controls are specified in
[`../../docs/grid-agent-conversation.md`](../../docs/grid-agent-conversation.md).
The versioned integrated/TCP protocol, roles, framing, bounds, TLS remote-mode
requirements, events, and management methods are specified in
[`../../docs/grid-agent-control-plane.md`](../../docs/grid-agent-control-plane.md).

View File

@@ -19,8 +19,8 @@ use serde_json::{Map, Value, json};
use std::collections::{BTreeMap, BTreeSet};
use std::error::Error;
use std::fmt;
use std::sync::Arc;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, Mutex, MutexGuard};
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
use tokio::sync::{mpsc, oneshot, watch};
use tokio::task::JoinHandle;
@@ -36,6 +36,7 @@ pub const CURRENT_POSE_TOOL: &str = "behavior_current_pose";
const WALK_POLL: Duration = Duration::from_millis(250);
const TURN_INTERVAL: Duration = Duration::from_millis(40);
const MAX_ACTION_ID_BYTES: usize = 96;
const MAX_TURN_STEP_DEGREES: f64 = 30.0;
const ARRIVAL_METERS: f64 = 0.5;
const STUCK_PROGRESS_METERS: f64 = 0.05;
@@ -196,6 +197,7 @@ pub enum BehaviorObservation {
trigger: BehaviorTrigger,
},
Action {
action_id: String,
generation: Option<u64>,
region_id: Option<UUID>,
action: String,
@@ -243,15 +245,23 @@ impl BehaviorAction {
}
struct ActionRequest {
action_id: String,
action: BehaviorAction,
trigger: BehaviorTrigger,
policy: BehaviorPolicyResult,
reply: oneshot::Sender<Result<Value, BehaviorError>>,
}
#[derive(Default)]
struct ActionRegistry {
active: BTreeSet<String>,
cancelled: BTreeSet<String>,
}
enum Command {
Action(ActionRequest),
Attention {
action_id: String,
delivery_id: String,
avatar_id: UUID,
reply: oneshot::Sender<Result<(), BehaviorError>>,
@@ -266,6 +276,10 @@ pub struct BehaviorIngress {
ready: watch::Sender<Option<ReadyState>>,
paused: watch::Sender<bool>,
emergency: watch::Sender<bool>,
action_cancel: watch::Sender<u64>,
actions: Arc<Mutex<ActionRegistry>>,
next_action: Arc<AtomicU64>,
action_capacity: usize,
}
impl fmt::Debug for BehaviorIngress {
@@ -313,6 +327,28 @@ impl BehaviorIngress {
let _ = self.emergency.send(false);
}
/// Cancels one exact queued or executing behavior action ID.
pub fn cancel_action(&self, action_id: &str) -> Result<bool, BehaviorError> {
if !valid_action_id(action_id) {
return Err(BehaviorError::InvalidArguments);
}
let cancelled = {
let mut actions = lock_actions(&self.actions);
if actions.active.contains(action_id) {
actions.cancelled.insert(action_id.to_owned());
true
} else {
false
}
};
if cancelled {
self.action_cancel.send_modify(|generation| {
*generation = generation.saturating_add(1);
});
}
Ok(cancelled)
}
/// Marks a separately authorized scheduler task as roaming. The controller
/// itself never invents a roaming route or exposes follow/wander behavior.
pub fn set_roaming(&self, roaming: bool) -> Result<(), BehaviorError> {
@@ -323,20 +359,30 @@ impl BehaviorIngress {
async fn request(
&self,
action_id: String,
action: BehaviorAction,
trigger: BehaviorTrigger,
policy: BehaviorPolicyResult,
) -> Result<Value, BehaviorError> {
if !self.register_action(&action_id) {
return Err(BehaviorError::InvalidArguments);
}
let (reply, receive) = oneshot::channel();
self.commands
if self
.commands
.send(Command::Action(ActionRequest {
action_id: action_id.clone(),
action,
trigger,
policy,
reply,
}))
.await
.map_err(|_| BehaviorError::QueueClosed)?;
.is_err()
{
self.unregister_action(&action_id);
return Err(BehaviorError::QueueClosed);
}
receive.await.map_err(|_| BehaviorError::QueueClosed)?
}
@@ -345,17 +391,44 @@ impl BehaviorIngress {
delivery_id: String,
avatar_id: UUID,
) -> Result<(), BehaviorError> {
let action_id = format!(
"attention-{}",
self.next_action.fetch_add(1, Ordering::Relaxed)
);
if !self.register_action(&action_id) {
return Err(BehaviorError::InvalidArguments);
}
let (reply, receive) = oneshot::channel();
self.commands
if self
.commands
.send(Command::Attention {
action_id: action_id.clone(),
delivery_id,
avatar_id,
reply,
})
.await
.map_err(|_| BehaviorError::QueueClosed)?;
.is_err()
{
self.unregister_action(&action_id);
return Err(BehaviorError::QueueClosed);
}
receive.await.map_err(|_| BehaviorError::QueueClosed)?
}
fn register_action(&self, action_id: &str) -> bool {
if !valid_action_id(action_id) {
return false;
}
let mut actions = lock_actions(&self.actions);
actions.active.len() < self.action_capacity && actions.active.insert(action_id.to_owned())
}
fn unregister_action(&self, action_id: &str) {
let mut actions = lock_actions(&self.actions);
actions.active.remove(action_id);
actions.cancelled.remove(action_id);
}
}
impl ResponsePacer for BehaviorIngress {
@@ -485,12 +558,18 @@ impl BehaviorController {
let (ready, ready_rx) = watch::channel(None);
let (paused, paused_rx) = watch::channel(false);
let (emergency, emergency_rx) = watch::channel(false);
let (action_cancel, action_cancel_rx) = watch::channel(0_u64);
let actions = Arc::new(Mutex::new(ActionRegistry::default()));
let (observations, observation_rx) = mpsc::channel(self.observation_capacity);
let ingress = BehaviorIngress {
commands,
ready,
paused,
emergency,
action_cancel,
actions: actions.clone(),
next_action: Arc::new(AtomicU64::new(1)),
action_capacity: self.queue_capacity,
};
let shutdown_timeout = self.shutdown_timeout;
let task = tokio::spawn(run_actor(
@@ -499,6 +578,8 @@ impl BehaviorController {
ready_rx,
paused_rx,
emergency_rx,
action_cancel_rx,
actions,
observations,
));
BehaviorHandle {
@@ -547,13 +628,15 @@ impl BehaviorRandom for SystemBehaviorRandom {
}
}
#[allow(clippy::too_many_lines)] // Lifecycle, preemption, idle, and commands share one ordered owner.
#[allow(clippy::too_many_arguments, clippy::too_many_lines)] // One ordered lifecycle owner.
async fn run_actor(
controller: BehaviorController,
mut commands: mpsc::Receiver<Command>,
mut ready: watch::Receiver<Option<ReadyState>>,
mut paused: watch::Receiver<bool>,
mut emergency: watch::Receiver<bool>,
action_cancel: watch::Receiver<u64>,
actions: Arc<Mutex<ActionRegistry>>,
observations: mpsc::Sender<BehaviorObservation>,
) {
let mut mode = BehaviorMode::Offline;
@@ -616,8 +699,16 @@ async fn run_actor(
() = &mut idle, if controller.settings.idle_look_enabled => {
if mode == BehaviorMode::Available {
let (reply, _) = oneshot::channel();
let request = ActionRequest { action: BehaviorAction::LookAround, trigger: BehaviorTrigger::IdleTimer, policy: BehaviorPolicyResult::InternalIdle, reply };
execute_request(&controller, request, &mut mode, &mut last_action, &ready, &paused, &emergency, &observations).await;
let action_id = format!("idle-{}", unix_millis_now());
let registered = {
let mut registry = lock_actions(&actions);
registry.active.len() < controller.queue_capacity
&& registry.active.insert(action_id.clone())
};
if registered {
let request = ActionRequest { action_id, action: BehaviorAction::LookAround, trigger: BehaviorTrigger::IdleTimer, policy: BehaviorPolicyResult::InternalIdle, reply };
execute_request(&controller, request, &mut mode, &mut last_action, &ready, &paused, &emergency, &action_cancel, &actions, &observations).await;
}
}
idle.as_mut().reset(tokio::time::Instant::now() + controller.settings.idle_interval);
}
@@ -642,9 +733,10 @@ async fn run_actor(
};
transition(&observations, &mut mode, next, BehaviorTrigger::IdleTimer).await;
}
Command::Action(request) => execute_request(&controller, request, &mut mode, &mut last_action, &ready, &paused, &emergency, &observations).await,
Command::Attention { delivery_id, avatar_id, reply } => {
Command::Action(request) => execute_request(&controller, request, &mut mode, &mut last_action, &ready, &paused, &emergency, &action_cancel, &actions, &observations).await,
Command::Attention { action_id, delivery_id, avatar_id, reply } => {
let request = ActionRequest {
action_id,
action: BehaviorAction::FaceAvatar(avatar_id),
trigger: BehaviorTrigger::PublicResponse { delivery_id, avatar_id },
policy: BehaviorPolicyResult::BuiltInAttention,
@@ -652,7 +744,7 @@ async fn run_actor(
};
let delay = random_duration(&controller, controller.settings.response_delay_min, controller.settings.response_delay_max);
tokio::time::sleep(delay).await;
execute_request(&controller, request, &mut mode, &mut last_action, &ready, &paused, &emergency, &observations).await;
execute_request(&controller, request, &mut mode, &mut last_action, &ready, &paused, &emergency, &action_cancel, &actions, &observations).await;
if mode == BehaviorMode::Engaged {
tokio::time::sleep(controller.settings.attention_dwell).await;
if !*paused.borrow() && ready.borrow().is_some() { transition(&observations, &mut mode, BehaviorMode::Available, BehaviorTrigger::IdleTimer).await; }
@@ -702,12 +794,17 @@ async fn execute_request(
ready: &watch::Receiver<Option<ReadyState>>,
paused: &watch::Receiver<bool>,
emergency: &watch::Receiver<bool>,
action_cancel: &watch::Receiver<u64>,
actions: &Arc<Mutex<ActionRegistry>>,
observations: &mpsc::Sender<BehaviorObservation>,
) {
let started = Instant::now();
let action_id = request.action_id.clone();
let action_name = request.action.name().to_owned();
let state = *ready.borrow();
let mut result = if *emergency.borrow() {
let mut result = if take_action_cancellation(actions, &action_id) {
Err(BehaviorError::Cancelled)
} else if *emergency.borrow() {
Err(BehaviorError::EmergencyStopped)
} else if *paused.borrow() {
Err(BehaviorError::Paused)
@@ -737,11 +834,15 @@ async fn execute_request(
let mut action_ready = ready.clone();
let mut action_paused = paused.clone();
let mut action_emergency = emergency.clone();
let action_cancel = action_cancel.clone();
let action_registry = actions.clone();
let cancelled_id = action_id.clone();
result = tokio::select! {
value = tokio::time::timeout(controller.settings.action_timeout, perform_action(controller, state, &request.action)) => value.unwrap_or(Err(BehaviorError::TimedOut)),
_ = action_ready.changed() => Err(BehaviorError::Cancelled),
_ = action_paused.changed() => Err(BehaviorError::Paused),
_ = action_emergency.changed() => Err(BehaviorError::EmergencyStopped),
() = wait_for_action_cancellation(action_cancel, action_registry, cancelled_id) => Err(BehaviorError::Cancelled),
};
if result.is_err() && matches!(request.action, BehaviorAction::WalkShort { .. }) {
let _ = controller
@@ -811,6 +912,7 @@ async fn execute_request(
};
let _ = observations
.send(BehaviorObservation::Action {
action_id: action_id.clone(),
generation: state.map(|value| value.generation),
region_id: state.map(|value| value.region_id),
action: action_name,
@@ -821,6 +923,22 @@ async fn execute_request(
})
.await;
let _ = request.reply.send(result);
unregister_action(actions, &action_id);
}
async fn wait_for_action_cancellation(
mut signal: watch::Receiver<u64>,
actions: Arc<Mutex<ActionRegistry>>,
action_id: String,
) {
loop {
if take_action_cancellation(&actions, &action_id) {
return;
}
if signal.changed().await.is_err() {
std::future::pending::<()>().await;
}
}
}
async fn perform_action(
@@ -1086,6 +1204,38 @@ fn random_duration(
minimum + Duration::from_millis(u64::try_from(offset).unwrap_or(u64::MAX))
}
fn valid_action_id(value: &str) -> bool {
!value.is_empty()
&& value.len() <= MAX_ACTION_ID_BYTES
&& value.chars().all(|character| {
character.is_ascii_alphanumeric() || matches!(character, '-' | '_' | '.' | ':')
})
}
fn lock_actions(value: &Mutex<ActionRegistry>) -> MutexGuard<'_, ActionRegistry> {
value
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
}
fn take_action_cancellation(actions: &Mutex<ActionRegistry>, action_id: &str) -> bool {
lock_actions(actions).cancelled.remove(action_id)
}
fn unregister_action(actions: &Mutex<ActionRegistry>, action_id: &str) {
let mut actions = lock_actions(actions);
actions.active.remove(action_id);
actions.cancelled.remove(action_id);
}
fn unix_millis_now() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.map_or(0, |duration| {
u64::try_from(duration.as_millis()).unwrap_or(u64::MAX)
})
}
#[derive(Clone)]
pub struct BehaviorBackend {
ingress: BehaviorIngress,
@@ -1136,7 +1286,7 @@ impl AuthorizedToolBackend for BehaviorBackend {
let result = match parsed {
Ok(behavior_action) => tokio::select! {
() = cancellation.cancelled() => Err(BehaviorError::Cancelled),
value = self.ingress.request(behavior_action, BehaviorTrigger::AuthorizedTool { authorization_id, tool: action.call().name.as_str().to_owned() }, BehaviorPolicyResult::Authorized(authorization_id)) => value,
value = self.ingress.request(call_id.as_str().to_owned(), behavior_action, BehaviorTrigger::AuthorizedTool { authorization_id, tool: action.call().name.as_str().to_owned() }, BehaviorPolicyResult::Authorized(authorization_id)) => value,
},
Err(error) => Err(error),
};

View File

@@ -453,6 +453,48 @@ async fn pause_preempts_stuck_walk_and_blocks_all_motion() {
handle.shutdown().await.expect("shutdown");
}
#[tokio::test(start_paused = true)]
async fn exact_action_id_cancels_only_the_matching_pending_action() {
let region = uuid(10);
let sink = Arc::new(FakeSink::new(1, region));
sink.set_stuck(true);
let mut config = settings();
config.settle_delay = Duration::ZERO;
let handle = BehaviorController::new(config.clone(), sink, 16, 32, Duration::from_secs(1))
.expect("controller")
.start();
let ingress = handle.ingress();
ingress.connected(1, region).expect("ready");
tokio::task::yield_now().await;
let backend = BehaviorBackend::new(ingress.clone());
let task = tokio::spawn({
let backend = backend.clone();
let config = config.clone();
async move {
run_tool(
&backend,
&config,
WALK_SHORT_TOOL,
json!({"heading_degrees":90.0,"distance_meters":2.0}),
)
.await
}
});
tokio::time::advance(Duration::from_millis(300)).await;
assert!(!ingress.cancel_action("unrelated-action").expect("cancel"));
assert!(ingress.cancel_action("behavior-call").expect("cancel"));
tokio::task::yield_now().await;
assert!(matches!(
task.await.expect("task"),
ToolCallOutcome::Rejected { .. }
));
assert!(matches!(
run_tool(&backend, &config, SIT_TOOL, json!({})).await,
ToolCallOutcome::Completed { .. }
));
handle.shutdown().await.expect("shutdown");
}
#[tokio::test(start_paused = true)]
async fn offline_and_emergency_stop_emit_zero_motion() {
let region = uuid(10);

View File

@@ -7,12 +7,15 @@ use std::collections::{BTreeMap, BTreeSet};
use std::error::Error;
use std::fmt;
use std::fs;
use std::net::{IpAddr, Ipv4Addr, SocketAddr};
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::time::Duration;
use url::Url;
const MAX_CONFIG_BYTES: u64 = 64 * 1024;
const MAX_SECRET_BYTES: u64 = 16 * 1024;
const MAX_TLS_DER_BYTES: u64 = 1024 * 1024;
const MAX_AUTHORIZED_AVATARS: usize = 1_024;
const MAX_QUEUE_CAPACITY: usize = 8_192;
const MAX_BACKGROUND_TASKS: usize = 2;
@@ -27,6 +30,11 @@ const ENV_GRID_AVATAR_NAME: &str = "METACRATE_AGENT_GRID_AVATAR_NAME";
const ENV_GRID_PASSWORD: &str = "METACRATE_AGENT_GRID_PASSWORD";
const ENV_GRID_PASSWORD_FILE: &str = "METACRATE_AGENT_GRID_PASSWORD_FILE";
const ENV_AUTHORIZED_AVATARS: &str = "METACRATE_AGENT_AUTHORIZED_AVATAR_UUIDS";
const ENV_CONTROL_LISTEN: &str = "METACRATE_AGENT_CONTROL_LISTEN";
const ENV_CONTROL_OPERATOR_TOKEN: &str = "METACRATE_AGENT_CONTROL_OPERATOR_TOKEN";
const ENV_CONTROL_OPERATOR_TOKEN_FILE: &str = "METACRATE_AGENT_CONTROL_OPERATOR_TOKEN_FILE";
const ENV_CONTROL_OBSERVER_TOKEN: &str = "METACRATE_AGENT_CONTROL_OBSERVER_TOKEN";
const ENV_CONTROL_OBSERVER_TOKEN_FILE: &str = "METACRATE_AGENT_CONTROL_OBSERVER_TOKEN_FILE";
/// Wrapper that never reveals its contents through `Debug` or `Display` and
/// deliberately does not implement serialization.
@@ -247,6 +255,66 @@ pub struct ConversationSettings {
pub limits: crate::conversation::ConversationLimits,
}
/// Portable control-plane settings. Plain TCP is loopback-only; a remote
/// listener is admitted only with an explicit Rustls certificate and key.
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct ControlSettings {
pub listen: SocketAddr,
pub operator_token: Option<SecretString>,
pub observer_token: Option<SecretString>,
pub limits: crate::control_plane::ControlLimits,
pub remote_tls: Option<RemoteTlsSettings>,
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct RemoteTlsSettings {
pub certificate_der: PathBuf,
pub private_key_der: PathBuf,
}
impl RemoteTlsSettings {
/// Loads bounded, regular DER files into a portable Rustls server config.
///
/// # Errors
///
/// Returns a configuration error when either file is unavailable, unsafe,
/// oversized, malformed, or does not form a valid certificate/key pair.
pub fn server_config(&self) -> Result<Arc<rustls::ServerConfig>, ConfigError> {
let certificate = read_bounded_regular_file(
"control.remote_tls_certificate_der",
&self.certificate_der,
MAX_TLS_DER_BYTES,
)?;
let private_key = read_bounded_regular_file(
"control.remote_tls_private_key_der",
&self.private_key_der,
MAX_TLS_DER_BYTES,
)?;
let private_key = rustls::pki_types::PrivateKeyDer::try_from(private_key)
.map_err(|_| ConfigError::InvalidControl)?;
let config = rustls::ServerConfig::builder()
.with_no_client_auth()
.with_single_cert(
vec![rustls::pki_types::CertificateDer::from(certificate)],
private_key,
)
.map_err(|_| ConfigError::InvalidControl)?;
Ok(Arc::new(config))
}
}
impl Default for ControlSettings {
fn default() -> Self {
Self {
listen: SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 7943),
operator_token: None,
observer_token: None,
limits: crate::control_plane::ControlLimits::default(),
remote_tls: None,
}
}
}
/// Fully resolved configuration. It cannot be constructed without validation.
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct AgentConfig {
@@ -261,6 +329,7 @@ pub struct AgentConfig {
pub reconnect: crate::session::ReconnectPolicy,
pub conversation: ConversationSettings,
pub interaction: crate::interaction::InteractionSettings,
pub control: ControlSettings,
}
impl AgentConfig {
@@ -300,6 +369,45 @@ impl AgentConfig {
if !self.behavior.is_valid() {
return Err(ConfigError::InvalidBehavior);
}
if (!self.control.listen.ip().is_loopback() && self.control.remote_tls.is_none())
|| !self.control.limits.is_valid()
{
return Err(ConfigError::InvalidControl);
}
if self.mode == OperatingMode::SplitService && self.control.operator_token.is_none() {
return Err(ConfigError::Missing {
field: "control.operator_token",
required_for: "split service mode",
});
}
if let Some(operator) = &self.control.operator_token
&& (self
.control
.observer_token
.as_ref()
.is_some_and(|observer| observer == operator)
|| operator == &self.llm.api_key
|| self
.grid
.as_ref()
.is_some_and(|grid| operator == &grid.password))
{
return Err(ConfigError::InvalidControl);
}
if self
.control
.observer_token
.as_ref()
.is_some_and(|observer| {
observer == &self.llm.api_key
|| self
.grid
.as_ref()
.is_some_and(|grid| observer == &grid.password)
})
{
return Err(ConfigError::InvalidControl);
}
if self.mode != OperatingMode::OfflineFake && self.grid.is_none() {
return Err(ConfigError::Missing {
field: "grid",
@@ -476,6 +584,7 @@ pub enum ConfigError {
InvalidConversationMemory,
InvalidInteraction,
InvalidBehavior,
InvalidControl,
}
impl fmt::Display for ConfigError {
@@ -531,6 +640,9 @@ impl fmt::Display for ConfigError {
}
Self::InvalidInteraction => formatter.write_str("invalid interaction bounds"),
Self::InvalidBehavior => formatter.write_str("invalid embodied behavior bounds"),
Self::InvalidControl => formatter.write_str(
"invalid control plane: use distinct non-credential tokens and loopback TCP or an explicit TLS remote listener",
),
}
}
}
@@ -553,6 +665,7 @@ struct FileConfig {
reconnect: RawReconnect,
conversation: RawConversation,
interaction: RawInteraction,
control: RawControl,
}
#[derive(Clone, Default, Deserialize)]
@@ -575,6 +688,18 @@ struct RawGrid {
struct RawSecretFiles {
llm_api_key: Option<PathBuf>,
grid_password: Option<PathBuf>,
control_operator_token: Option<PathBuf>,
control_observer_token: Option<PathBuf>,
}
#[derive(Clone, Default, Deserialize)]
#[serde(default, deny_unknown_fields)]
struct RawControl {
listen: Option<String>,
operator_token: Option<String>,
observer_token: Option<String>,
remote_tls_certificate_der: Option<PathBuf>,
remote_tls_private_key_der: Option<PathBuf>,
}
#[derive(Clone, Default, Deserialize)]
@@ -932,6 +1057,57 @@ fn resolve<E: Environment>(
.max_debounce_fragments
.unwrap_or(interaction_defaults.max_debounce_fragments),
};
let control_defaults = ControlSettings::default();
let listen_text = environment
.get(ENV_CONTROL_LISTEN)
.or(raw.control.listen)
.unwrap_or_else(|| control_defaults.listen.to_string());
let listen = listen_text
.parse::<SocketAddr>()
.map_err(|_| ConfigError::InvalidControl)?;
let operator_token = optional_secret_from_layers(
environment.get(ENV_CONTROL_OPERATOR_TOKEN),
environment
.get(ENV_CONTROL_OPERATOR_TOKEN_FILE)
.map(PathBuf::from),
raw.control.operator_token,
raw.secret_files.control_operator_token,
base,
"control.operator_token",
)?;
let observer_token = optional_secret_from_layers(
environment.get(ENV_CONTROL_OBSERVER_TOKEN),
environment
.get(ENV_CONTROL_OBSERVER_TOKEN_FILE)
.map(PathBuf::from),
raw.control.observer_token,
raw.secret_files.control_observer_token,
base,
"control.observer_token",
)?;
let remote_tls = match (
raw.control.remote_tls_certificate_der,
raw.control.remote_tls_private_key_der,
) {
(None, None) => None,
(Some(certificate_der), Some(private_key_der)) => Some(RemoteTlsSettings {
certificate_der: resolve_path(base, &certificate_der),
private_key_der: resolve_path(base, &private_key_der),
}),
_ => return Err(ConfigError::InvalidControl),
};
let control = ControlSettings {
listen,
operator_token,
observer_token,
limits: crate::control_plane::ControlLimits {
command_queue: limits.control_queue,
request_timeout: timeouts.request.min(Duration::from_mins(2)),
write_timeout: timeouts.shutdown.min(Duration::from_secs(30)),
..crate::control_plane::ControlLimits::default()
},
remote_tls,
};
let config = AgentConfig {
mode,
@@ -996,6 +1172,7 @@ fn resolve<E: Environment>(
reconnect,
conversation,
interaction,
control,
};
config.validate()?;
Ok(config)
@@ -1064,6 +1241,28 @@ fn secret_from_layers(
})
}
fn optional_secret_from_layers(
environment_value: Option<String>,
environment_file: Option<PathBuf>,
file_value: Option<String>,
file_path: Option<PathBuf>,
base: &Path,
field: &'static str,
) -> Result<Option<SecretString>, ConfigError> {
if let Some(value) = environment_value {
return SecretString::new(field, value).map(Some);
}
if let Some(path) = environment_file {
return read_secret(field, &resolve_path(base, &path)).map(Some);
}
if let Some(path) = file_path {
return read_secret(field, &resolve_path(base, &path)).map(Some);
}
file_value
.map(|value| SecretString::new(field, value))
.transpose()
}
fn resolve_path(base: &Path, path: &Path) -> PathBuf {
if path.is_absolute() {
path.to_owned()
@@ -1437,6 +1636,50 @@ mod tests {
));
}
#[test]
fn split_control_requires_distinct_tokens_and_tls_for_remote_bind() {
let split = MapEnvironment::from_pairs([
(ENV_SPLIT, "true"),
(ENV_LLM_ENDPOINT, "https://llm.invalid/chat"),
(ENV_LLM_API_KEY, "llm-key"),
(ENV_GRID_LOGIN_URL, "https://grid.invalid/login"),
(ENV_GRID_AVATAR_NAME, "Control Agent"),
(ENV_GRID_PASSWORD, "grid-password"),
(ENV_CONTROL_OPERATOR_TOKEN, "operator-capability"),
(ENV_CONTROL_OBSERVER_TOKEN, "observer-capability"),
]);
let config = ConfigLoader::new()
.with_environment(split.clone())
.load()
.expect("bounded loopback split control");
assert_eq!(config.mode, OperatingMode::SplitService);
let diagnostic = format!("{config:?}");
for secret in [
"operator-capability",
"observer-capability",
"llm-key",
"grid-password",
] {
assert!(!diagnostic.contains(secret));
}
let mut reused = split.clone();
reused.insert(ENV_CONTROL_OPERATOR_TOKEN, "llm-key");
assert_eq!(
ConfigLoader::new().with_environment(reused).load(),
Err(ConfigError::InvalidControl)
);
let mut remote_plaintext = split;
remote_plaintext.insert(ENV_CONTROL_LISTEN, "0.0.0.0:7943");
assert_eq!(
ConfigLoader::new()
.with_environment(remote_plaintext)
.load(),
Err(ConfigError::InvalidControl)
);
}
#[test]
fn secret_file_is_bounded_regular_and_trailing_newline_is_removed() {
let secret = temporary_file("api-key", "secret-from-file\n");

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,746 @@
use crate::config::SecretString;
use crate::control_plane::*;
use libremetaverse_types::compat::CancellationToken;
use serde_json::json;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::sync::{Arc, Mutex};
use std::time::Duration;
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::net::TcpStream;
#[derive(Default)]
struct FakeTarget {
mutations: Mutex<Vec<String>>,
shutdown: AtomicBool,
active: AtomicUsize,
maximum_active: AtomicUsize,
}
impl FakeTarget {
fn mutation_count(&self) -> usize {
self.mutations.lock().expect("mutations").len()
}
}
impl ControlTarget for FakeTarget {
#[allow(clippy::too_many_lines)] // Exhaustive fake covers every response family.
fn execute(
&self,
context: ControlContext,
request: ControlRequest,
cancellation: CancellationToken,
) -> ControlFuture<'_> {
Box::pin(async move {
let active = self.active.fetch_add(1, Ordering::AcqRel) + 1;
self.maximum_active.fetch_max(active, Ordering::AcqRel);
let result = match request {
ControlRequest::Health => Ok(ControlPayload::Health(HealthView {
service_state: "running".into(),
ready: true,
uptime_seconds: 42,
protocol_version: CONTROL_PROTOCOL_VERSION,
})),
ControlRequest::Runtime => Ok(ControlPayload::Runtime(RuntimeView {
grid_state: "online".into(),
generation: 7,
transport_connected: true,
agent_ready: true,
region_id: Some("00000000-0000-4000-8000-000000000001".into()),
region_name: Some("Safe Region".into()),
position: Some([128.0, 128.0, 24.0]),
behavior_mode: "available".into(),
control_queue_used: 1,
control_queue_capacity: 64,
budget_tool_calls_used: 3,
budget_movement_millimeters_used: 2_000,
})),
ControlRequest::ListSessions { page } => Ok(ControlPayload::Sessions(page_values(
&page,
(0..7)
.map(|index| SessionMetadataView {
session_id: format!("session-{index}"),
avatar_id: format!("00000000-0000-4000-8000-{index:012x}"),
channel: "direct_im".into(),
created_unix_millis: 10,
last_active_unix_millis: 20,
turns: 2,
bytes: 32,
})
.collect(),
))),
ControlRequest::ListScheduledJobs { page } => {
Ok(ControlPayload::ScheduledJobs(page_values(
&page,
vec![ScheduledJobView {
job_id: "roam-1".into(),
kind: "landmark_roam".into(),
enabled: true,
next_run_unix_millis: Some(50),
}],
)))
}
ControlRequest::ListPendingApprovals { page } => {
Ok(ControlPayload::PendingApprovals(page_values(
&page,
vec![PendingApprovalView {
approval_id: 1,
tool: "build".into(),
principal: "operator:local".into(),
expires_unix_seconds: 99,
movement_millimeters: 0,
inventory_operations: 0,
build_prims: 1,
}],
)))
}
ControlRequest::ListAuditEvents { page } => {
Ok(ControlPayload::AuditEvents(page_values(
&page,
vec![AuditEventView {
sequence: 1,
unix_millis: 1,
principal: "operator:local".into(),
operation: "pause_autonomy".into(),
outcome: "completed".into(),
authorization_id: None,
}],
)))
}
ControlRequest::InjectOperatorMessage { message } if message == "wait" => {
cancellation.cancelled().await;
Err(ControlError {
code: ControlErrorCode::Cancelled,
message: "operation cancelled",
retryable: false,
})
}
ControlRequest::GracefulShutdown => {
self.shutdown.store(true, Ordering::Release);
self.mutations
.lock()
.expect("mutations")
.push("graceful_shutdown".into());
Ok(ControlPayload::Completed)
}
other => {
assert_eq!(context.role, ControlRole::Operator);
self.mutations
.lock()
.expect("mutations")
.push(format!("{other:?}"));
Ok(ControlPayload::Completed)
}
};
self.active.fetch_sub(1, Ordering::AcqRel);
result
})
}
}
fn page_values<T>(page: &PageRequest, values: Vec<T>) -> Page<T> {
let start = usize::try_from(page.cursor.unwrap_or(0))
.expect("cursor")
.min(values.len());
let end = start
.saturating_add(usize::from(page.limit))
.min(values.len());
let next_cursor = (end < values.len()).then(|| u64::try_from(end).expect("cursor"));
Page {
items: values.into_iter().skip(start).take(end - start).collect(),
next_cursor,
}
}
fn token(value: &str) -> SecretString {
SecretString::new("control.test_token", value).expect("token")
}
fn limits() -> ControlLimits {
ControlLimits {
request_timeout: Duration::from_secs(5),
idle_timeout: Duration::from_secs(5),
write_timeout: Duration::from_secs(1),
..ControlLimits::default()
}
}
fn plane(target: Arc<FakeTarget>, configured: ControlLimits) -> Arc<ControlPlane> {
ControlPlane::new(
target,
token("operator-secret"),
Some(token("observer-secret")),
configured,
)
.expect("plane")
}
fn request(id: &str, request: ControlRequest) -> ControlRequestEnvelope {
ControlRequestEnvelope {
version: CONTROL_PROTOCOL_VERSION,
request_id: id.into(),
request,
}
}
enum Client {
InProcess(InProcessControlClient),
Tcp(TcpControlClient),
}
impl Client {
async fn request(&self, envelope: ControlRequestEnvelope) -> ControlResponseEnvelope {
match self {
Self::InProcess(client) => client.request(envelope).await,
Self::Tcp(client) => client.request(envelope).await.expect("TCP response"),
}
}
}
enum Transport {
InProcess,
Tcp,
}
struct Fixture {
client: Client,
plane: Arc<ControlPlane>,
target: Arc<FakeTarget>,
server: Option<TcpControlServer>,
}
async fn fixture(transport: Transport, role: ControlRole, configured: ControlLimits) -> Fixture {
let target = Arc::new(FakeTarget::default());
let plane = plane(target.clone(), configured);
let value = if role == ControlRole::Operator {
"operator-secret"
} else {
"observer-secret"
};
match transport {
Transport::InProcess => Fixture {
client: Client::InProcess(plane.connect(value).expect("connect")),
plane,
target,
server: None,
},
Transport::Tcp => {
let server = TcpControlServer::bind(
plane.clone(),
TcpControlConfig {
listen: "127.0.0.1:0".parse().expect("address"),
limits: configured,
},
)
.await
.expect("bind");
let client = TcpControlClient::connect(server.local_addr(), value, configured)
.await
.expect("connect");
Fixture {
client: Client::Tcp(client),
plane,
target,
server: Some(server),
}
}
}
}
async fn close(fixture: Fixture) {
drop(fixture.client);
if let Some(server) = fixture.server {
server.shutdown().await.expect("shutdown");
}
}
async fn conformance(transport: Transport) {
let fixture = fixture(transport, ControlRole::Operator, limits()).await;
let health = fixture
.client
.request(request("health-1", ControlRequest::Health))
.await;
assert_eq!(health.version, CONTROL_PROTOCOL_VERSION);
assert!(matches!(
health.result,
Ok(ControlPayload::Health(HealthView { ready: true, .. }))
));
let page = fixture
.client
.request(request(
"sessions-1",
ControlRequest::ListSessions {
page: PageRequest {
cursor: Some(2),
limit: 3,
},
},
))
.await;
let Ok(ControlPayload::Sessions(page)) = page.result else {
panic!("session page")
};
assert_eq!(page.items.len(), 3);
assert_eq!(page.items[0].session_id, "session-2");
assert_eq!(page.next_cursor, Some(5));
assert!(
fixture
.client
.request(request("pause-1", ControlRequest::PauseAutonomy))
.await
.result
.is_ok()
);
let replay = fixture
.client
.request(request("pause-1", ControlRequest::ResumeAutonomy))
.await;
assert_eq!(
replay.result.expect_err("replay").code,
ControlErrorCode::Replay
);
assert_eq!(fixture.target.mutation_count(), 1);
let shutdown = fixture
.client
.request(request("shutdown-1", ControlRequest::GracefulShutdown))
.await;
assert!(shutdown.result.is_ok());
assert!(fixture.target.shutdown.load(Ordering::Acquire));
close(fixture).await;
}
#[tokio::test]
async fn same_conformance_suite_runs_in_process() {
conformance(Transport::InProcess).await;
}
#[tokio::test]
async fn same_conformance_suite_runs_over_loopback_tcp() {
conformance(Transport::Tcp).await;
}
#[tokio::test]
async fn observer_is_read_only_and_tokens_are_redacted() {
let fixture = fixture(Transport::InProcess, ControlRole::Observer, limits()).await;
assert!(matches!(
fixture
.client
.request(request("read", ControlRequest::Runtime))
.await
.result,
Ok(ControlPayload::Runtime(_))
));
let denied = fixture
.client
.request(request("mutate", ControlRequest::PauseAutonomy))
.await;
assert_eq!(
denied.result.expect_err("denied").code,
ControlErrorCode::PermissionDenied
);
assert_eq!(fixture.target.mutation_count(), 0);
assert!(!format!("{:?}", fixture.plane).contains("operator-secret"));
let Err(error) = fixture.plane.connect("wrong") else {
panic!("bad token was accepted")
};
assert_eq!(error.code, ControlErrorCode::AuthenticationFailed);
close(fixture).await;
}
#[tokio::test]
async fn version_bounds_and_malformed_requests_are_typed() {
let fixture = fixture(Transport::InProcess, ControlRole::Operator, limits()).await;
let mut wrong = request("version", ControlRequest::Health);
wrong.version = 99;
assert_eq!(
fixture
.client
.request(wrong)
.await
.result
.expect_err("version")
.code,
ControlErrorCode::VersionMismatch
);
let invalid = fixture
.client
.request(request(
"page",
ControlRequest::ListSessions {
page: PageRequest {
cursor: None,
limit: 101,
},
},
))
.await;
assert_eq!(
invalid.result.expect_err("page").code,
ControlErrorCode::InvalidRequest
);
let secret_payload = fixture
.client
.request(request(
"audit",
ControlRequest::ListAuditEvents {
page: PageRequest::default(),
},
))
.await;
let rendered = serde_json::to_string(&secret_payload).expect("JSON");
for secret in ["operator-secret", "observer-secret", "Bearer ", "CAPS/"] {
assert!(!rendered.contains(secret));
}
close(fixture).await;
}
#[tokio::test]
async fn cancellation_is_per_request_and_concurrent_operators_stay_responsive() {
for transport in [Transport::InProcess, Transport::Tcp] {
let fixture = fixture(transport, ControlRole::Operator, limits()).await;
let slow = match &fixture.client {
Client::InProcess(value) => Client::InProcess(value.clone()),
Client::Tcp(value) => Client::Tcp(value.clone()),
};
let task = tokio::spawn(async move {
slow.request(request(
"slow",
ControlRequest::InjectOperatorMessage {
message: "wait".into(),
},
))
.await
});
tokio::time::timeout(Duration::from_secs(1), async {
while fixture.target.active.load(Ordering::Acquire) == 0 {
tokio::task::yield_now().await;
}
})
.await
.expect("slow request became active");
let concurrent_operator = match &fixture.server {
Some(server) => Client::Tcp(
TcpControlClient::connect(server.local_addr(), "operator-secret", limits())
.await
.expect("second TCP operator"),
),
None => Client::InProcess(
fixture
.plane
.connect("operator-secret")
.expect("second in-process operator"),
),
};
let health = concurrent_operator
.request(request("health-during", ControlRequest::Health))
.await;
assert!(health.result.is_ok());
let cancelled = fixture
.client
.request(request(
"cancel",
ControlRequest::CancelRequest {
target_request_id: "slow".into(),
},
))
.await;
assert!(matches!(
cancelled.result,
Ok(ControlPayload::Cancelled { .. })
));
let slow = task.await.expect("slow task");
assert_eq!(
slow.result.expect_err("cancelled").code,
ControlErrorCode::Cancelled
);
assert!(fixture.target.maximum_active.load(Ordering::Acquire) >= 2);
drop(concurrent_operator);
close(fixture).await;
}
}
#[tokio::test]
async fn event_stream_resubscribes_with_explicit_gap_and_slow_consumers_are_cut_off() {
let mut configured = limits();
configured.event_queue = 2;
configured.event_history = 3;
let fixture = fixture(Transport::InProcess, ControlRole::Operator, configured).await;
let mut in_subscription = None;
if let Client::InProcess(client) = &fixture.client {
let (_, subscription) = client
.subscribe(request(
"subscribe",
ControlRequest::SubscribeEvents {
after_sequence: Some(0),
},
))
.await
.expect("subscribe");
in_subscription = Some(subscription);
}
for index in 0..10 {
fixture.plane.publish(ControlEventKind::StateChanged {
component: "grid".into(),
state: format!("state-{index}"),
});
}
assert_eq!(fixture.plane.retained_event_count(), 3);
let first = in_subscription
.as_mut()
.expect("subscription")
.recv()
.await
.expect("buffered");
assert_eq!(first.sequence, 1);
drop(in_subscription);
let client = fixture.plane.connect("operator-secret").expect("reconnect");
let (_, mut resumed) = client
.subscribe(request(
"resubscribe",
ControlRequest::SubscribeEvents {
after_sequence: Some(1),
},
))
.await
.expect("resubscribe");
let gap = resumed.recv().await.expect("gap");
assert!(matches!(
gap.event,
ControlEventKind::Gap {
first_available: 9,
last_missed: 8
}
));
close(fixture).await;
}
#[tokio::test]
async fn zero_client_event_flood_remains_bounded_and_headless() {
let target = Arc::new(FakeTarget::default());
let mut configured = limits();
configured.event_history = 4;
let plane = plane(target, configured);
assert_eq!(plane.active_connections(), 0);
for index in 0..10_000 {
plane.publish(ControlEventKind::StateChanged {
component: "load".into(),
state: index.to_string(),
});
}
assert_eq!(plane.retained_event_count(), 4);
assert_eq!(plane.active_connections(), 0);
}
#[tokio::test]
async fn subscriptions_have_a_hard_global_bound_and_release_on_drop() {
let mut configured = limits();
configured.max_subscriptions = 2;
let target = Arc::new(FakeTarget::default());
let plane = plane(target, configured);
let client = plane.connect("operator-secret").expect("client");
let (_, first) = client
.subscribe(request(
"subscribe-1",
ControlRequest::SubscribeEvents {
after_sequence: None,
},
))
.await
.expect("first subscription");
let (_, second) = client
.subscribe(request(
"subscribe-2",
ControlRequest::SubscribeEvents {
after_sequence: None,
},
))
.await
.expect("second subscription");
let error = client
.subscribe(request(
"subscribe-3",
ControlRequest::SubscribeEvents {
after_sequence: None,
},
))
.await
.err()
.expect("subscription bound");
assert_eq!(error.code, ControlErrorCode::Busy);
drop(first);
client
.subscribe(request(
"subscribe-4",
ControlRequest::SubscribeEvents {
after_sequence: None,
},
))
.await
.expect("released subscription");
drop(second);
}
#[tokio::test]
#[allow(clippy::too_many_lines)] // One raw-transport matrix shares one listener.
async fn tcp_rejects_bad_tokens_oversized_and_partial_frames() {
let configured = limits();
let target = Arc::new(FakeTarget::default());
let plane = plane(target, configured);
let server = TcpControlServer::bind(
plane.clone(),
TcpControlConfig {
listen: "127.0.0.1:0".parse().expect("address"),
limits: configured,
},
)
.await
.expect("server");
let Err(error) = TcpControlClient::connect(server.local_addr(), "wrong", configured).await
else {
panic!("bad token was accepted")
};
assert_eq!(error.code, ControlErrorCode::AuthenticationFailed);
let mut wrong_version = TcpStream::connect(server.local_addr())
.await
.expect("version connect");
let hello = serde_json::to_vec(&json!({
"frame":"hello",
"body":{"version":99,"token":"operator-secret"}
}))
.expect("hello JSON");
wrong_version
.write_all(
&u32::try_from(hello.len())
.expect("frame length")
.to_be_bytes(),
)
.await
.expect("version header");
wrong_version.write_all(&hello).await.expect("version body");
let mut response_header = [0_u8; 4];
wrong_version
.read_exact(&mut response_header)
.await
.expect("version response header");
let mut response =
vec![0_u8; usize::try_from(u32::from_be_bytes(response_header)).expect("response length")];
wrong_version
.read_exact(&mut response)
.await
.expect("version response");
let response: serde_json::Value = serde_json::from_slice(&response).expect("version JSON");
assert_eq!(response["body"]["error"]["code"], json!("version_mismatch"));
let subscriber = TcpControlClient::connect(server.local_addr(), "operator-secret", configured)
.await
.expect("subscriber");
let subscription_response = subscriber
.request(request(
"events",
ControlRequest::SubscribeEvents {
after_sequence: None,
},
))
.await
.expect("subscription response");
assert!(matches!(
subscription_response.result,
Ok(ControlPayload::Subscribed { .. })
));
plane.publish(ControlEventKind::StateChanged {
component: "session".into(),
state: "online".into(),
});
let event = tokio::time::timeout(Duration::from_secs(1), subscriber.next_event())
.await
.expect("event deadline")
.expect("event");
assert!(matches!(event.event, ControlEventKind::StateChanged { .. }));
let last_sequence = event.sequence;
drop(subscriber);
plane.publish(ControlEventKind::StateChanged {
component: "session".into(),
state: "reconnecting".into(),
});
let reconnected = TcpControlClient::connect(server.local_addr(), "operator-secret", configured)
.await
.expect("reconnected subscriber");
reconnected
.request(request(
"events-resumed",
ControlRequest::SubscribeEvents {
after_sequence: Some(last_sequence),
},
))
.await
.expect("resubscription response");
let replayed = tokio::time::timeout(Duration::from_secs(1), reconnected.next_event())
.await
.expect("replay deadline")
.expect("replayed event");
assert_eq!(replayed.sequence, last_sequence.saturating_add(1));
let mut oversized = TcpStream::connect(server.local_addr())
.await
.expect("raw connect");
oversized
.write_all(&(u32::try_from(configured.max_frame_bytes).expect("bound") + 1).to_be_bytes())
.await
.expect("header");
let mut byte = [0_u8; 1];
assert_eq!(
tokio::time::timeout(Duration::from_secs(1), oversized.read(&mut byte))
.await
.expect("closed")
.expect("read"),
0
);
let mut partial = TcpStream::connect(server.local_addr())
.await
.expect("raw connect");
partial
.write_all(&100_u32.to_be_bytes())
.await
.expect("header");
partial.write_all(b"{}").await.expect("partial");
partial.shutdown().await.expect("close");
drop(reconnected);
server.shutdown().await.expect("shutdown");
}
#[tokio::test]
async fn plaintext_server_rejects_non_loopback_before_binding() {
let configured = limits();
let target = Arc::new(FakeTarget::default());
let plane = plane(target, configured);
let Err(error) = TcpControlServer::bind(
plane,
TcpControlConfig {
listen: "0.0.0.0:7943".parse().expect("address"),
limits: configured,
},
)
.await
else {
panic!("plaintext remote listener was accepted")
};
assert_eq!(error.code, ControlErrorCode::InvalidRequest);
}
#[test]
fn plain_tcp_is_loopback_only_and_protocol_json_is_stable() {
let configured = limits();
assert!(
!TcpControlConfig {
listen: "0.0.0.0:7943".parse().expect("address"),
limits: configured
}
.listen
.ip()
.is_loopback()
);
let encoded = serde_json::to_value(request("health", ControlRequest::Health)).expect("JSON");
assert_eq!(
encoded,
json!({"version":1,"request_id":"health","request":{"method":"health"}})
);
}

View File

@@ -0,0 +1,522 @@
//! Production management target backed by the agent's bounded runtime stores.
#![allow(clippy::missing_errors_doc)]
use crate::behavior::{BehaviorIngress, BehaviorMode};
use crate::control_plane::{
AuditEventView, CONTROL_PROTOCOL_VERSION, ControlContext, ControlError, ControlErrorCode,
ControlFuture, ControlPayload, ControlRequest, ControlTarget, ConversationChannelView,
HealthView, Page, PageRequest, PendingApprovalView, RuntimeView, ScheduledJobView,
SessionMetadataView,
};
use crate::conversation::{ConversationChannel, ConversationKey, ConversationStore};
use crate::policy::{
ApprovalId, AuthenticatedPrincipal, MemoryPolicyAudit, PolicyFinalOutcome, PolicyGateway,
PolicyReasonCode,
};
use crate::session::{SessionState, SessionStatus};
use libremetaverse_types::compat::CancellationToken;
use std::collections::BTreeMap;
use std::fmt;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, Mutex, MutexGuard};
use std::time::{Instant, SystemTime, UNIX_EPOCH};
use tokio::sync::mpsc;
const ROAMING_JOB_ID: &str = "default-roaming";
/// Commands whose ownership remains with the service lifecycle loop.
pub enum RuntimeControlCommand {
Pause,
Resume,
ForceReconnect,
OperatorMessage { message: String },
GracefulShutdown,
}
impl fmt::Debug for RuntimeControlCommand {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::OperatorMessage { message } => formatter
.debug_struct("OperatorMessage")
.field("message_bytes", &message.len())
.finish(),
Self::Pause => formatter.write_str("Pause"),
Self::Resume => formatter.write_str("Resume"),
Self::ForceReconnect => formatter.write_str("ForceReconnect"),
Self::GracefulShutdown => formatter.write_str("GracefulShutdown"),
}
}
}
#[derive(Clone, Debug)]
struct RuntimeState {
session: SessionStatus,
behavior: BehaviorMode,
service_state: &'static str,
region_id: Option<String>,
region_name: Option<String>,
position: Option<[f64; 3]>,
}
impl Default for RuntimeState {
fn default() -> Self {
Self {
session: SessionStatus::default(),
behavior: BehaviorMode::Offline,
service_state: "starting",
region_id: None,
region_name: None,
position: None,
}
}
}
/// Real control target shared by integrated and split transports.
pub struct AgentControlTarget {
started: Instant,
state: Mutex<RuntimeState>,
conversations: Arc<ConversationStore>,
policy: Arc<PolicyGateway>,
audit: Arc<MemoryPolicyAudit>,
behavior: BehaviorIngress,
commands: mpsc::Sender<RuntimeControlCommand>,
command_capacity: usize,
jobs: Mutex<BTreeMap<String, bool>>,
next_operation: AtomicU64,
}
impl fmt::Debug for AgentControlTarget {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("AgentControlTarget")
.field("state", &lock(&self.state))
.field("command_capacity", &self.command_capacity)
.finish_non_exhaustive()
}
}
impl AgentControlTarget {
pub fn new(
conversations: Arc<ConversationStore>,
policy: Arc<PolicyGateway>,
audit: Arc<MemoryPolicyAudit>,
behavior: BehaviorIngress,
command_capacity: usize,
) -> Result<(Arc<Self>, mpsc::Receiver<RuntimeControlCommand>), ControlError> {
if command_capacity == 0 || command_capacity > 8_192 {
return Err(control_error(
ControlErrorCode::InvalidRequest,
"invalid runtime control queue capacity",
false,
));
}
let (commands, receiver) = mpsc::channel(command_capacity);
let mut jobs = BTreeMap::new();
jobs.insert(ROAMING_JOB_ID.to_owned(), false);
Ok((
Arc::new(Self {
started: Instant::now(),
state: Mutex::new(RuntimeState::default()),
conversations,
policy,
audit,
behavior,
commands,
command_capacity,
jobs: Mutex::new(jobs),
next_operation: AtomicU64::new(1),
}),
receiver,
))
}
pub fn update_session(&self, session: SessionStatus) {
let mut state = lock(&self.state);
state.session = session;
state.service_state = if session.state == SessionState::ShuttingDown {
"stopping"
} else {
"running"
};
}
pub fn update_behavior(&self, behavior: BehaviorMode) {
lock(&self.state).behavior = behavior;
}
pub fn update_region(
&self,
region_id: Option<String>,
region_name: Option<String>,
position: Option<[f64; 3]>,
) {
let mut state = lock(&self.state);
state.region_id = region_id.filter(|value| value.len() <= 64);
state.region_name = region_name.filter(|value| value.len() <= 256);
state.position =
position.filter(|value| value.iter().all(|component| component.is_finite()));
}
pub fn mark_stopping(&self) {
lock(&self.state).service_state = "stopping";
}
fn enqueue(
&self,
operation: &'static str,
command: RuntimeControlCommand,
) -> Result<ControlPayload, ControlError> {
self.commands
.try_send(command)
.map_err(|error| match error {
mpsc::error::TrySendError::Full(_) => control_error(
ControlErrorCode::Backpressure,
"runtime control queue is full",
true,
),
mpsc::error::TrySendError::Closed(_) => control_error(
ControlErrorCode::TransportClosed,
"runtime control queue is closed",
true,
),
})?;
Ok(ControlPayload::Accepted {
operation_id: format!(
"{operation}-{}",
self.next_operation.fetch_add(1, Ordering::Relaxed)
),
})
}
#[allow(clippy::too_many_lines)] // Exhaustive protocol-to-runtime mapping stays auditable.
fn execute_now(
&self,
context: &ControlContext,
request: ControlRequest,
) -> Result<ControlPayload, ControlError> {
match request {
ControlRequest::Health => {
let state = lock(&self.state);
Ok(ControlPayload::Health(HealthView {
service_state: state.service_state.to_owned(),
ready: state.session.agent_ready,
uptime_seconds: self.started.elapsed().as_secs(),
protocol_version: CONTROL_PROTOCOL_VERSION,
}))
}
ControlRequest::Runtime => {
let state = lock(&self.state).clone();
let usage = self.policy.global_budget_usage();
Ok(ControlPayload::Runtime(RuntimeView {
grid_state: state.session.state.as_str().to_owned(),
generation: state.session.generation,
transport_connected: state.session.transport_connected,
agent_ready: state.session.agent_ready,
region_id: state.region_id,
region_name: state.region_name,
position: state.position,
behavior_mode: behavior_name(state.behavior).to_owned(),
control_queue_used: self
.command_capacity
.saturating_sub(self.commands.capacity()),
control_queue_capacity: self.command_capacity,
budget_tool_calls_used: usage.tool_calls,
budget_movement_millimeters_used: usage.movement_millimeters,
}))
}
ControlRequest::ListSessions { page } => {
let values = self
.conversations
.list_metadata()
.into_iter()
.map(|metadata| SessionMetadataView {
session_id: metadata.session_id.as_str().to_owned(),
avatar_id: metadata.avatar_id.to_string(),
channel: match metadata.channel {
ConversationChannel::PublicChat => "public_chat",
ConversationChannel::DirectIm => "direct_im",
}
.to_owned(),
created_unix_millis: metadata.created_unix_millis,
last_active_unix_millis: metadata.last_active_unix_millis,
turns: metadata.turns,
bytes: metadata.bytes,
})
.collect();
Ok(ControlPayload::Sessions(page_values(&page, values)))
}
ControlRequest::ListScheduledJobs { page } => {
let values = lock(&self.jobs)
.iter()
.map(|(job_id, enabled)| ScheduledJobView {
job_id: job_id.clone(),
kind: "bounded_roaming".to_owned(),
enabled: *enabled,
next_run_unix_millis: None,
})
.collect();
Ok(ControlPayload::ScheduledJobs(page_values(&page, values)))
}
ControlRequest::ListPendingApprovals { page } => {
let values = self
.policy
.pending_approvals(unix_seconds())
.into_iter()
.map(|approval| PendingApprovalView {
approval_id: approval.id.get(),
tool: approval.tool,
principal: approval.principal,
expires_unix_seconds: approval.expires_at,
movement_millimeters: approval.cost.movement_millimeters,
inventory_operations: approval.cost.inventory_operations,
build_prims: approval.cost.build_prims,
})
.collect();
Ok(ControlPayload::PendingApprovals(page_values(&page, values)))
}
ControlRequest::ListAuditEvents { page } => {
let values = self
.audit
.snapshot()
.into_iter()
.enumerate()
.map(|(index, record)| AuditEventView {
sequence: u64::try_from(index).unwrap_or(u64::MAX).saturating_add(1),
unix_millis: record.recorded_unix_millis,
principal: record.principal.as_str().to_owned(),
operation: record.tool.as_str().to_owned(),
outcome: policy_outcome_name(record.final_outcome).to_owned(),
authorization_id: record.authorization_id,
})
.collect();
Ok(ControlPayload::AuditEvents(page_values(&page, values)))
}
ControlRequest::PauseAutonomy => {
let response = self.enqueue("pause", RuntimeControlCommand::Pause)?;
self.behavior.pause();
Ok(response)
}
ControlRequest::ResumeAutonomy => {
let response = self.enqueue("resume", RuntimeControlCommand::Resume)?;
self.behavior.resume();
Ok(response)
}
ControlRequest::CancelAction { action_id } => {
let cancelled = self.behavior.cancel_action(&action_id).map_err(|_| {
control_error(ControlErrorCode::InvalidRequest, "invalid action ID", false)
})?;
if !cancelled {
return Err(control_error(
ControlErrorCode::NotFound,
"active behavior action not found",
false,
));
}
Ok(ControlPayload::Completed)
}
ControlRequest::DecideApproval {
approval_id,
approve,
} => {
let id = ApprovalId::from_raw(approval_id).ok_or_else(|| {
control_error(
ControlErrorCode::InvalidRequest,
"invalid approval ID",
false,
)
})?;
let principal =
AuthenticatedPrincipal::from_authenticated_control(context.principal.clone())
.map_err(|_| {
control_error(
ControlErrorCode::PermissionDenied,
"invalid operator principal",
false,
)
})?;
let reason = if approve {
self.policy.grant_approval(id, &principal, unix_seconds())
} else {
self.policy.deny_approval(id, &principal, unix_seconds())
}
.map_err(|_| {
control_error(
ControlErrorCode::Internal,
"approval decision failed",
false,
)
})?;
match reason {
PolicyReasonCode::Allowed | PolicyReasonCode::ApprovalNotGranted => {
Ok(ControlPayload::Completed)
}
PolicyReasonCode::ApprovalUnknown => Err(control_error(
ControlErrorCode::NotFound,
"approval not found",
false,
)),
PolicyReasonCode::ApprovalExpired | PolicyReasonCode::ApprovalReplayed => {
Err(control_error(
ControlErrorCode::Conflict,
"approval is no longer pending",
false,
))
}
_ => Err(control_error(
ControlErrorCode::Conflict,
"approval decision rejected",
false,
)),
}
}
ControlRequest::ForceReconnect => {
self.enqueue("reconnect", RuntimeControlCommand::ForceReconnect)
}
ControlRequest::ExpireConversation { avatar_id, channel } => {
let avatar_id = libremetaverse_types::UUID::parse(avatar_id).map_err(|_| {
control_error(ControlErrorCode::InvalidRequest, "invalid avatar ID", false)
})?;
let channel = match channel {
ConversationChannelView::PublicChat => ConversationChannel::PublicChat,
ConversationChannelView::DirectIm => ConversationChannel::DirectIm,
};
let key = ConversationKey::new(avatar_id, channel).map_err(|_| {
control_error(
ControlErrorCode::InvalidRequest,
"invalid conversation key",
false,
)
})?;
if self.conversations.expire(key) {
Ok(ControlPayload::Completed)
} else {
Err(control_error(
ControlErrorCode::NotFound,
"conversation not found",
false,
))
}
}
ControlRequest::SetRoamingJob { job_id, enabled } => {
let mut jobs = lock(&self.jobs);
let Some(current) = jobs.get_mut(&job_id) else {
return Err(control_error(
ControlErrorCode::NotFound,
"roaming job not found",
false,
));
};
self.behavior.set_roaming(enabled).map_err(|_| {
control_error(
ControlErrorCode::Backpressure,
"behavior queue unavailable",
true,
)
})?;
*current = enabled;
Ok(ControlPayload::Completed)
}
ControlRequest::InjectOperatorMessage { message } => self.enqueue(
"operator-message",
RuntimeControlCommand::OperatorMessage { message },
),
ControlRequest::GracefulShutdown => {
let response = self.enqueue("shutdown", RuntimeControlCommand::GracefulShutdown)?;
self.mark_stopping();
Ok(response)
}
ControlRequest::SubscribeEvents { .. } | ControlRequest::CancelRequest { .. } => {
Err(control_error(
ControlErrorCode::Internal,
"transport request reached runtime",
false,
))
}
}
}
}
impl ControlTarget for AgentControlTarget {
fn execute(
&self,
context: ControlContext,
request: ControlRequest,
cancellation: CancellationToken,
) -> ControlFuture<'_> {
Box::pin(async move {
if cancellation.is_cancellation_requested() {
return Err(control_error(
ControlErrorCode::Cancelled,
"control request cancelled",
false,
));
}
self.execute_now(&context, request)
})
}
}
fn page_values<T>(request: &PageRequest, values: Vec<T>) -> Page<T> {
let start = usize::try_from(request.cursor.unwrap_or(0))
.unwrap_or(usize::MAX)
.min(values.len());
let end = start
.saturating_add(usize::from(request.limit))
.min(values.len());
let next_cursor = (end < values.len()).then(|| u64::try_from(end).unwrap_or(u64::MAX));
Page {
items: values.into_iter().skip(start).take(end - start).collect(),
next_cursor,
}
}
const fn behavior_name(mode: BehaviorMode) -> &'static str {
match mode {
BehaviorMode::Offline => "offline",
BehaviorMode::Settling => "settling",
BehaviorMode::Available => "available",
BehaviorMode::Engaged => "engaged",
BehaviorMode::Executing => "executing",
BehaviorMode::Roaming => "roaming",
BehaviorMode::Paused => "paused",
BehaviorMode::Recovering => "recovering",
}
}
const fn policy_outcome_name(outcome: PolicyFinalOutcome) -> &'static str {
match outcome {
PolicyFinalOutcome::Denied => "denied",
PolicyFinalOutcome::ApprovalRequired => "approval_required",
PolicyFinalOutcome::ApprovalGranted => "approval_granted",
PolicyFinalOutcome::Authorized => "authorized",
PolicyFinalOutcome::Completed => "completed",
PolicyFinalOutcome::Rejected => "rejected",
PolicyFinalOutcome::Failed => "failed",
PolicyFinalOutcome::AmbiguousMutation => "ambiguous_mutation",
}
}
fn unix_seconds() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.map_or(0, |duration| duration.as_secs())
}
const fn control_error(
code: ControlErrorCode,
message: &'static str,
retryable: bool,
) -> ControlError {
ControlError {
code,
message,
retryable,
}
}
fn lock<T>(value: &Mutex<T>) -> MutexGuard<'_, T> {
value
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
}

View File

@@ -0,0 +1,167 @@
use crate::behavior::{BehaviorController, BehaviorError, EmbodimentFuture, EmbodimentSink};
use crate::control_plane::{
CONTROL_PROTOCOL_VERSION, ControlLimits, ControlPayload, ControlPlane, ControlRequest,
ControlRequestEnvelope,
};
use crate::control_runtime::{AgentControlTarget, RuntimeControlCommand};
use crate::conversation::{ConversationLimits, ConversationStore};
use crate::perception::WorldPosition;
use crate::policy::{MemoryPolicyAudit, PolicyAuditSink, PolicyGateway, PolicyLimits};
use crate::{AgentConfig, EmbodiedPose, SecretString};
use libremetaverse_types::UUID;
use libremetaverse_types::compat::CancellationToken;
use std::collections::BTreeSet;
use std::sync::Arc;
use std::time::Duration;
struct Sink;
impl EmbodimentSink for Sink {
fn current_pose(
&self,
generation: u64,
_cancellation: CancellationToken,
) -> EmbodimentFuture<'_, EmbodiedPose> {
Box::pin(async move {
Ok(EmbodiedPose {
generation,
region_id: UUID::zero(),
position: WorldPosition {
x: 128.0,
y: 128.0,
z: 24.0,
},
heading_degrees: 0.0,
sitting: false,
})
})
}
fn resolve_avatar(
&self,
_generation: u64,
_avatar_id: UUID,
_cancellation: CancellationToken,
) -> EmbodimentFuture<'_, WorldPosition> {
Box::pin(async { Err(BehaviorError::TargetUnavailable) })
}
fn face_point(
&self,
_generation: u64,
_point: WorldPosition,
_cancellation: CancellationToken,
) -> EmbodimentFuture<'_, ()> {
Box::pin(async { Ok(()) })
}
fn begin_walk(
&self,
_generation: u64,
_point: WorldPosition,
_cancellation: CancellationToken,
) -> EmbodimentFuture<'_, ()> {
Box::pin(async { Ok(()) })
}
fn validate_walk_target(
&self,
_generation: u64,
_point: WorldPosition,
_cancellation: CancellationToken,
) -> EmbodimentFuture<'_, ()> {
Box::pin(async { Ok(()) })
}
fn stop(&self, _generation: u64, _cancellation: CancellationToken) -> EmbodimentFuture<'_, ()> {
Box::pin(async { Ok(()) })
}
fn sit(&self, _generation: u64, _cancellation: CancellationToken) -> EmbodimentFuture<'_, ()> {
Box::pin(async { Ok(()) })
}
fn stand(
&self,
_generation: u64,
_cancellation: CancellationToken,
) -> EmbodimentFuture<'_, ()> {
Box::pin(async { Ok(()) })
}
}
fn envelope(id: &str, request: ControlRequest) -> ControlRequestEnvelope {
ControlRequestEnvelope {
version: CONTROL_PROTOCOL_VERSION,
request_id: id.to_owned(),
request,
}
}
#[tokio::test]
async fn production_target_projects_state_and_routes_real_mutations() {
let settings = AgentConfig::offline("https://llm.invalid/chat", "llm-key")
.expect("offline config")
.behavior;
let behavior = BehaviorController::new(settings, Arc::new(Sink), 8, 8, Duration::from_secs(1))
.expect("behavior")
.start();
let audit = Arc::new(MemoryPolicyAudit::new(32).expect("audit"));
let audit_sink: Arc<dyn PolicyAuditSink> = audit.clone();
let policy = Arc::new(
PolicyGateway::new(
BTreeSet::new(),
Vec::new(),
PolicyLimits::default(),
audit_sink,
)
.expect("policy"),
);
let conversations = Arc::new(
ConversationStore::open(ConversationLimits::default(), None).expect("conversations"),
);
let (target, mut commands) =
AgentControlTarget::new(conversations, policy, audit, behavior.ingress(), 8)
.expect("target");
target.update_region(
Some("00000000-0000-4000-8000-000000000001".into()),
Some("Test Region".into()),
Some([128.0, 128.0, 24.0]),
);
let plane = ControlPlane::new(
target,
SecretString::new("test.operator", "operator-token").expect("token"),
Some(SecretString::new("test.observer", "observer-token").expect("token")),
ControlLimits::default(),
)
.expect("plane");
let operator = plane.connect("operator-token").expect("operator");
let runtime = operator
.request(envelope("runtime", ControlRequest::Runtime))
.await;
let Ok(ControlPayload::Runtime(runtime)) = runtime.result else {
panic!("runtime projection")
};
assert_eq!(runtime.region_name.as_deref(), Some("Test Region"));
assert_eq!(runtime.control_queue_capacity, 8);
assert!(
operator
.request(envelope("pause", ControlRequest::PauseAutonomy))
.await
.result
.is_ok()
);
assert!(matches!(
commands.recv().await,
Some(RuntimeControlCommand::Pause)
));
let observer = plane.connect("observer-token").expect("observer");
let denied = observer
.request(envelope("shutdown", ControlRequest::GracefulShutdown))
.await;
assert!(denied.result.is_err());
assert!(commands.try_recv().is_err());
behavior.shutdown().await.expect("shutdown behavior");
}

View File

@@ -7,6 +7,8 @@
pub mod backend;
pub mod behavior;
pub mod config;
pub mod control_plane;
pub mod control_runtime;
pub mod conversation;
pub mod interaction;
pub mod llm;
@@ -20,6 +22,10 @@ pub mod types;
#[cfg(test)]
mod behavior_tests;
#[cfg(test)]
mod control_plane_tests;
#[cfg(test)]
mod control_runtime_tests;
#[cfg(test)]
mod conversation_tests;
#[cfg(test)]
mod interaction_tests;
@@ -47,10 +53,19 @@ pub use behavior::{
STOP_TOOL, WALK_SHORT_TOOL, behavior_policy_tools,
};
pub use config::{
AgentConfig, BehaviorSettings, ConfigError, ConfigLoader, ConversationSettings, EndpointUrl,
Environment, GridConnection, Limits, LlmConnection, MapEnvironment, OperatingMode,
SecretString, StdEnvironment, Timeouts,
AgentConfig, BehaviorSettings, ConfigError, ConfigLoader, ControlSettings,
ConversationSettings, EndpointUrl, Environment, GridConnection, Limits, LlmConnection,
MapEnvironment, OperatingMode, RemoteTlsSettings, SecretString, StdEnvironment, Timeouts,
};
pub use control_plane::{
AuditEventView, CONTROL_PROTOCOL_VERSION, ControlContext, ControlError, ControlErrorBody,
ControlErrorCode, ControlEvent, ControlEventKind, ControlFuture, ControlLimits, ControlPayload,
ControlPlane, ControlRequest, ControlRequestEnvelope, ControlResponseEnvelope, ControlRole,
ControlSubscription, ControlTarget, ConversationChannelView, HealthView,
InProcessControlClient, Page, PageRequest, PendingApprovalView, RuntimeView, ScheduledJobView,
SessionMetadataView, TcpControlClient, TcpControlConfig, TcpControlServer,
};
pub use control_runtime::{AgentControlTarget, RuntimeControlCommand};
pub use conversation::{
ConversationChannel, ConversationClock, ConversationContext, ConversationError,
ConversationKey, ConversationLimits, ConversationMetadata, ConversationPersistence,
@@ -81,10 +96,11 @@ pub use perception::{
pub use policy::{
ActionOrigin, AllowedOrigins, ApprovalId, ApprovalRule, AuthenticatedPrincipal,
AuthorizedAction, BudgetLimits, Capability, FixedCost, Idempotency, MemoryPolicyAudit,
OriginClass, PolicyAuditError, PolicyAuditRecord, PolicyAuditSink, PolicyDisposition,
PolicyError, PolicyEvaluation, PolicyFinalOutcome, PolicyGateway, PolicyLimits,
PolicyReasonCode, PolicyRequestContext, PolicySnapshot, PolicyTool, PolicyToolExecutor,
ResourceCost, ResourceEstimator, Risk, SchedulerGrantId, UntrustedData, UntrustedSource,
OriginClass, PendingApprovalMetadata, PolicyAuditError, PolicyAuditRecord, PolicyAuditSink,
PolicyDisposition, PolicyError, PolicyEvaluation, PolicyFinalOutcome, PolicyGateway,
PolicyLimits, PolicyReasonCode, PolicyRequestContext, PolicySnapshot, PolicyTool,
PolicyToolExecutor, ResourceCost, ResourceEstimator, Risk, SchedulerGrantId, UntrustedData,
UntrustedSource,
};
pub use service::{AgentService, ServiceError, ServiceHandle, ServiceState};
pub use session::{

View File

@@ -147,8 +147,10 @@ async fn run_live(
run_once: bool,
) -> Result<(), Box<dyn Error>> {
use metacrate_grid_agent::{
GridSessionBackend, LibremetaverseClientOwner, SessionObservation, SessionState,
SessionSupervisor,
AgentControlTarget, BehaviorObservation, ControlEventKind, ControlPlane, ControlTarget,
GridSessionBackend, LibremetaverseClientOwner, OperatingMode, RuntimeControlCommand,
SessionControl, SessionObservation, SessionState, SessionSupervisor, TcpControlConfig,
TcpControlServer,
};
let connection = config.grid.clone().ok_or_else(|| {
@@ -230,8 +232,56 @@ async fn run_live(
return Ok(());
}
let (control_target, mut control_commands) = AgentControlTarget::new(
live.conversations.clone(),
live.policy.clone(),
live.audit.clone(),
live.behavior.ingress(),
config.limits.control_queue,
)?;
control_target.update_session(handle.status());
let erased_target: Arc<dyn ControlTarget> = control_target.clone();
let (control_plane, _integrated_client, control_server) = match config.mode {
OperatingMode::Integrated => {
let (plane, client) = ControlPlane::integrated(erased_target, config.control.limits)?;
(plane, Some(client), None)
}
OperatingMode::SplitService => {
let operator = config.control.operator_token.clone().ok_or_else(|| {
CliError("validated split configuration omitted its operator token".into())
})?;
let plane = ControlPlane::new(
erased_target,
operator,
config.control.observer_token.clone(),
config.control.limits,
)?;
let transport = TcpControlConfig {
listen: config.control.listen,
limits: config.control.limits,
};
let server = if let Some(tls) = &config.control.remote_tls {
eprintln!(
"WARNING: remote control is enabled with TLS; protect operator tokens and certificate keys"
);
TcpControlServer::bind_tls(plane.clone(), transport, tls.server_config()?).await?
} else {
TcpControlServer::bind(plane.clone(), transport).await?
};
println!(
"grid agent control plane listening on {}",
server.local_addr()
);
(plane, None, Some(server))
}
OperatingMode::OfflineFake => {
return Err(CliError("offline mode reached the live control plane".into()).into());
}
};
println!("grid agent session supervisor started; press Ctrl-C to stop");
let mut signal_error = None;
let mut control_failure = None;
loop {
tokio::select! {
signal = tokio::signal::ctrl_c() => {
@@ -243,6 +293,11 @@ async fn run_live(
event = handle.next_observation() => {
let Some(event) = event else { break; };
if let SessionObservation::Transition { status, reason, retry_in } = event {
control_target.update_session(status);
control_plane.publish(ControlEventKind::StateChanged {
component: "session".into(),
state: status.state.as_str().into(),
});
println!(
"grid session state={} generation={} transport_connected={} agent_ready={} reason={reason:?} retry_in={retry_in:?}",
status.state.as_str(),
@@ -262,19 +317,66 @@ async fn run_live(
}
event = live.behavior.next_observation() => {
let Some(event) = event else { break; };
if let BehaviorObservation::Transition { to, .. } = &event {
control_target.update_behavior(*to);
control_plane.publish(ControlEventKind::StateChanged {
component: "behavior".into(),
state: format!("{to:?}").to_ascii_lowercase(),
});
}
println!("grid behavior event={event:?}");
}
command = control_commands.recv() => {
let Some(command) = command else { break; };
match command {
RuntimeControlCommand::Pause => {
if let Err(error) = handle.control(SessionControl::Pause).await {
control_failure = Some(error.to_string());
break;
}
}
RuntimeControlCommand::Resume => {
if let Err(error) = handle.control(SessionControl::Resume).await {
control_failure = Some(error.to_string());
break;
}
}
RuntimeControlCommand::ForceReconnect => {
if let Err(error) = handle.control(SessionControl::ForceReconnect).await {
control_failure = Some(error.to_string());
break;
}
}
RuntimeControlCommand::OperatorMessage { message } => {
control_plane.publish(ControlEventKind::StateChanged {
component: "operator_message".into(),
state: format!("accepted_{}_bytes", message.len()),
});
}
RuntimeControlCommand::GracefulShutdown => break,
}
}
}
}
control_target.mark_stopping();
let session_result = handle.shutdown().await;
let interaction_result = live.interaction.shutdown().await;
let behavior_result = live.behavior.shutdown().await;
let control_result = if let Some(server) = control_server {
server.shutdown().await
} else {
Ok(())
};
session_result?;
interaction_result?;
behavior_result?;
control_result?;
if let Some(error) = signal_error {
return Err(error.into());
}
if let Some(error) = control_failure {
return Err(CliError(format!("control command failed: {error}")).into());
}
println!("grid agent stopped cleanly");
Ok(())
}
@@ -286,9 +388,13 @@ struct LiveInteractions {
perception_observations:
tokio::sync::mpsc::Receiver<metacrate_grid_agent::PerceptionObservation>,
behavior: metacrate_grid_agent::BehaviorHandle,
conversations: Arc<metacrate_grid_agent::ConversationStore>,
policy: Arc<metacrate_grid_agent::PolicyGateway>,
audit: Arc<metacrate_grid_agent::MemoryPolicyAudit>,
}
#[cfg(feature = "live-grid")]
#[allow(clippy::too_many_lines)]
fn start_live_interactions(
config: &metacrate_grid_agent::AgentConfig,
owner: &metacrate_grid_agent::LibremetaverseClientOwner,
@@ -347,7 +453,7 @@ fn start_live_interactions(
config.authorized_avatar_uuids.clone(),
tools,
PolicyLimits::default(),
audit,
audit.clone(),
)?);
let loop_limits = ToolLoopLimits {
max_tool_calls_per_turn: config.limits.max_tool_calls,
@@ -366,7 +472,7 @@ fn start_live_interactions(
Arc::new(AuthorizedBackendRouter::new(routes)?);
let responder = Arc::new(PolicyLlmResponder::new(
client,
gateway,
gateway.clone(),
routed_backend,
loop_limits,
now,
@@ -377,7 +483,7 @@ fn start_live_interactions(
config.interaction.clone(),
libremetaverse_types::UUID::zero(),
config.authorized_avatar_uuids.clone(),
conversation,
conversation.clone(),
responder,
sink,
config.limits.grid_event_queue,
@@ -391,5 +497,8 @@ fn start_live_interactions(
perception: perception_ingress,
perception_observations,
behavior,
conversations: conversation,
policy: gateway,
audit,
})
}

View File

@@ -16,6 +16,7 @@ use std::collections::{BTreeMap, BTreeSet, VecDeque};
use std::error::Error;
use std::fmt;
use std::sync::{Arc, Mutex};
use std::time::{SystemTime, UNIX_EPOCH};
const MAX_POLICY_TOOLS: usize = 64;
const MAX_AUTHORIZED_AVATARS: usize = 1_024;
@@ -238,6 +239,16 @@ pub struct ResourceCost {
pub build_prims: u64,
}
/// Secret-free operator projection of one pending approval.
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct PendingApprovalMetadata {
pub id: ApprovalId,
pub principal: String,
pub tool: String,
pub expires_at: u64,
pub cost: ResourceCost,
}
impl ResourceCost {
#[must_use]
pub const fn one_call() -> Self {
@@ -514,6 +525,7 @@ pub enum PolicyFinalOutcome {
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct PolicyAuditRecord {
pub recorded_unix_millis: u64,
pub authorization_id: Option<u64>,
pub approval_id: Option<ApprovalId>,
pub origin: OriginClass,
@@ -581,6 +593,18 @@ impl PolicyAuditSink for MemoryPolicyAudit {
#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)]
pub struct ApprovalId(u64);
impl ApprovalId {
#[must_use]
pub const fn from_raw(value: u64) -> Option<Self> {
if value == 0 { None } else { Some(Self(value)) }
}
#[must_use]
pub const fn get(self) -> u64 {
self.0
}
}
#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)]
pub struct SchedulerGrantId(u64);
@@ -588,6 +612,7 @@ pub struct SchedulerGrantId(u64);
enum ApprovalStatus {
Pending,
Granted,
Denied,
Consumed,
}
@@ -820,6 +845,31 @@ impl PolicyGateway {
PolicySnapshot(lock(&self.state).clone())
}
/// Returns bounded, content-free approval metadata for operator UIs.
#[must_use]
pub fn pending_approvals(&self, now: u64) -> Vec<PendingApprovalMetadata> {
lock(&self.state)
.approvals
.values()
.filter(|approval| {
approval.status == ApprovalStatus::Pending && approval.expires_at > now
})
.map(|approval| PendingApprovalMetadata {
id: approval.id,
principal: approval.principal.audit_label(),
tool: approval.tool.clone(),
expires_at: approval.expires_at,
cost: approval.cost,
})
.collect()
}
/// Current global budget use for a redacted control-plane projection.
#[must_use]
pub fn global_budget_usage(&self) -> ResourceCost {
lock(&self.state).global_used
}
#[cfg(test)]
pub(crate) fn pending_approval_count(&self) -> usize {
lock(&self.state)
@@ -1064,7 +1114,9 @@ impl PolicyGateway {
match approval.status {
ApprovalStatus::Pending => Some(PolicyReasonCode::ApprovalNotGranted),
ApprovalStatus::Granted => None,
ApprovalStatus::Consumed => Some(PolicyReasonCode::ApprovalReplayed),
ApprovalStatus::Denied | ApprovalStatus::Consumed => {
Some(PolicyReasonCode::ApprovalReplayed)
}
}
};
if let Some(reason) = reason {
@@ -1121,7 +1173,11 @@ impl PolicyGateway {
}
if state.approvals.len() == MAX_APPROVALS {
let removable = state.approvals.iter().find_map(|(id, approval)| {
(approval.expires_at <= now || approval.status == ApprovalStatus::Consumed)
(approval.expires_at <= now
|| matches!(
approval.status,
ApprovalStatus::Denied | ApprovalStatus::Consumed
))
.then_some(*id)
});
let Some(removable) = removable else {
@@ -1323,6 +1379,40 @@ impl PolicyGateway {
Ok(PolicyReasonCode::Allowed)
}
pub fn deny_approval(
&self,
id: ApprovalId,
_operator: &AuthenticatedPrincipal,
now: u64,
) -> Result<PolicyReasonCode, PolicyError> {
let mut state = lock(&self.state);
let Some(approval) = state.approvals.get_mut(&id) else {
return Ok(PolicyReasonCode::ApprovalUnknown);
};
if approval.expires_at <= now {
return Ok(PolicyReasonCode::ApprovalExpired);
}
if approval.status != ApprovalStatus::Pending {
return Ok(PolicyReasonCode::ApprovalReplayed);
}
let audit = audit_record(
&approval.context,
approval.origin,
&approval.principal,
BoundedText::new("policy.tool", approval.tool.clone())?,
None,
Some(id),
PolicyDisposition::Denied,
PolicyReasonCode::ApprovalNotGranted,
approval.cost,
approval.arguments_hash,
PolicyFinalOutcome::Denied,
)?;
self.emit(audit)?;
approval.status = ApprovalStatus::Denied;
Ok(PolicyReasonCode::ApprovalNotGranted)
}
/// Consumes the originating authorization so it cannot be replayed to
/// create more than one scheduler grant.
#[allow(clippy::needless_pass_by_value)]
@@ -1621,6 +1711,11 @@ fn audit_record(
final_outcome: PolicyFinalOutcome,
) -> Result<PolicyAuditRecord, PolicyError> {
Ok(PolicyAuditRecord {
recorded_unix_millis: SystemTime::now()
.duration_since(UNIX_EPOCH)
.map_or(0, |duration| {
u64::try_from(duration.as_millis()).unwrap_or(u64::MAX)
}),
authorization_id,
approval_id,
origin,

View File

@@ -411,6 +411,7 @@ fn approval_tool() -> PolicyTool {
}
#[test]
#[allow(clippy::too_many_lines)] // One approval lifecycle includes deny and expiry paths.
fn approvals_bind_principal_canonical_arguments_expiry_and_one_execution() {
let (gateway, audit) = gateway_with(vec![approval_tool()], PolicyLimits::default());
let requester = context(OriginClass::AuthorizedIm);
@@ -507,6 +508,24 @@ fn approvals_bind_principal_canonical_arguments_expiry_and_one_execution() {
.expect("expired result"),
PolicyReasonCode::ApprovalExpired
);
let denied = gateway
.evaluate(&requester, &proposed, &arguments, None, 600)
.expect("denial pending")
.approval_id
.expect("approval ID");
assert_eq!(gateway.pending_approvals(600).len(), 1);
assert_eq!(
gateway.deny_approval(denied, &operator, 601).expect("deny"),
PolicyReasonCode::ApprovalNotGranted
);
assert!(gateway.pending_approvals(601).is_empty());
assert_eq!(
gateway
.evaluate(&requester, &proposed, &arguments, Some(denied), 601)
.expect("denied approval cannot execute")
.reason,
PolicyReasonCode::ApprovalReplayed
);
assert!(
audit
.snapshot()

View File

@@ -2,14 +2,16 @@ use std::collections::BTreeSet;
use std::fs;
use std::path::{Path, PathBuf};
const ALLOWED_DEPENDENCIES: [&str; 8] = [
const ALLOWED_DEPENDENCIES: [&str; 10] = [
"libremetaverse",
"libremetaverse-types",
"reqwest",
"rustls",
"serde",
"serde_json",
"sha2",
"tokio",
"tokio-rustls",
"url",
];
@@ -46,7 +48,7 @@ fn runtime_source_has_no_subprocess_or_native_abi_escape_hatch() {
let mut files = Vec::with_capacity(20);
collect_rust_files(&source, &mut files);
assert!(
files.len() <= 20,
files.len() <= 24,
"source-file count needs a reviewed bound update"
);
for path in files {

View File

@@ -0,0 +1,91 @@
# Grid-agent control plane v1
The grid agent exposes one transport-neutral request/response and event API.
Integrated clients receive an `InProcessControlClient`; split clients use the
same `ControlRequestEnvelope`, `ControlResponseEnvelope`, and `ControlEvent`
types over TCP. No terminal, Unix socket, named-pipe, permission-bit, or
platform-specific type appears in the core API.
## Transport and negotiation
TCP frames are a four-byte unsigned big-endian length followed by one UTF-8
JSON value. Empty, partial, malformed, idle, and oversized frames close the
connection within configured deadlines. The first frame must be:
```json
{"frame":"hello","body":{"version":1,"token":"separate-operator-capability"}}
```
The server answers with a `hello` containing version `1`, the authenticated
`operator` or `observer` role, or a typed error. Every later client frame is a
`request`; server frames are `response` or `event`. Request IDs contain at most
96 ASCII identifier bytes and cannot be replayed on a connection. A caller can
cancel an active request by its ID. Responses and events can arrive in either
order, so clients correlate responses by request ID.
Plain TCP binds only to IPv4 or IPv6 loopback. Remote control is off by default
and a non-loopback address is rejected unless both DER certificate and private
key paths are explicitly configured. Remote mode uses Rustls before the hello
exchange and emits an operator warning at startup. The TLS client API requires
the embedding client to supply its certificate roots and server-name policy.
Every split connection must still authenticate with a dedicated control token;
observer and operator tokens must differ and configuration rejects reuse of an
LLM API key or grid password. Tokens are absent from protocol errors, audit,
serialization traits, and diagnostics.
Remote service configuration uses the JSON fields
`control.remote_tls_certificate_der` and
`control.remote_tls_private_key_der` together with a non-loopback
`control.listen` value. Both files are bounded, non-symlink DER files. Omitting
either field, or selecting a non-loopback address without both, fails
configuration before a listener is created.
## Requests
The JSON request envelope is stable and versioned. For example:
```json
{"version":1,"request_id":"health-1","request":{"method":"health"}}
```
Observers can call `health`, `runtime`, `list_sessions`,
`list_scheduled_jobs`, `list_pending_approvals`, `list_audit_events`, and
`subscribe_events`. Operators can additionally call `cancel_request`,
`pause_autonomy`, `resume_autonomy`, `cancel_action`, `decide_approval`,
`force_reconnect`, `expire_conversation`, `set_roaming_job`,
`inject_operator_message`, and `graceful_shutdown`. Cancellation is a mutation
and is operator-only. List requests use an opaque numeric cursor and a page
size of 1 through 100.
Runtime projections contain lifecycle/readiness, session generation, safe
region and pose fields when known, behavior mode, control-queue utilization,
and aggregate budget use. Conversation responses contain metadata only. Audit
and approval responses exclude arguments, prompt contents, credentials,
authorization headers, capability URLs, model reasoning, and filesystem data.
`cancel_action` binds to the exact bounded action ID carried by behavior audit
observations; it can cancel a queued or executing embodied action without
preempting unrelated work. The built-in roaming job ID is `default-roaming`.
Errors are typed as `authentication_failed`, `version_mismatch`,
`permission_denied`, `invalid_request`, `replay`, `not_found`, `conflict`,
`cancelled`, `timed_out`, `busy`, `backpressure`, `frame_too_large`,
`idle_timeout`, `transport_closed`, or `internal`, with a bounded safe message
and a retryable flag.
## Events, bounds, and reconnect
Events carry monotonically increasing sequence numbers. A subscription can
resume after its last observed sequence. If retained history no longer covers
that point, its first item is an explicit `gap` with `first_available` and
`last_missed`, followed by retained events. Mutation events record the
authenticated role/connection principal, operation name, and completed or
rejected outcome, never request contents.
Connections, unauthenticated handshakes, in-flight request tasks,
subscriptions, command/event queues, replay history, event history, event
bytes, event rate, frames, pages, idle time, request time, and writes all have
validated hard bounds. A full event or writer queue disconnects the slow
consumer instead of blocking the headless service. With zero clients, event
publication retains only the configured history and creates no background
work. Server shutdown cancels and joins all listener, connection, request, and
writer tasks within its deadline.