feat(grid-agent): add embodied behavior controller (#125)
Some checks failed
CI / rust-skia (Rust only) (push) Successful in 2m42s
CI / required (push) Failing after 2m40s

This commit is contained in:
2026-08-18 00:26:24 +00:00
parent 3a0ead7eb5
commit 058ed10005
11 changed files with 2607 additions and 22 deletions

View File

@@ -24,7 +24,19 @@
}, },
"storage_path": "data/grid-agent", "storage_path": "data/grid-agent",
"behavior": { "behavior": {
"heartbeat_seconds": 30 "heartbeat_seconds": 30,
"settle_milliseconds": 2000,
"response_delay_min_milliseconds": 350,
"response_delay_max_milliseconds": 1200,
"attention_dwell_milliseconds": 4000,
"idle_interval_seconds": 120,
"action_timeout_seconds": 15,
"max_walk_duration_seconds": 10,
"stuck_timeout_seconds": 3,
"min_action_interval_milliseconds": 750,
"max_walk_distance_meters": 8,
"max_attention_distance_meters": 96,
"idle_look_enabled": true
}, },
"reconnect": { "reconnect": {
"initial_delay_milliseconds": 1000, "initial_delay_milliseconds": 1000,

View File

@@ -107,6 +107,14 @@ pub struct LibremetaverseWorldSnapshotSource {
agent: Arc<libremetaverse::AgentManager>, agent: Arc<libremetaverse::AgentManager>,
} }
#[cfg(feature = "live-grid")]
#[derive(Clone)]
pub struct LibremetaverseEmbodimentSink {
client: libremetaverse::GridClient,
agent: Arc<libremetaverse::AgentManager>,
generation: Arc<std::sync::RwLock<Option<u64>>>,
}
#[cfg(feature = "live-grid")] #[cfg(feature = "live-grid")]
impl fmt::Debug for LibremetaverseClientOwner { impl fmt::Debug for LibremetaverseClientOwner {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
@@ -130,6 +138,7 @@ pub struct LibremetaverseSessionBackend {
password: crate::config::SecretString, password: crate::config::SecretString,
interaction: Option<crate::interaction::InteractionIngress>, interaction: Option<crate::interaction::InteractionIngress>,
perception: Option<crate::perception::PerceptionIngress>, perception: Option<crate::perception::PerceptionIngress>,
behavior: Option<crate::behavior::BehaviorIngress>,
delivery_generation: Arc<std::sync::RwLock<Option<u64>>>, delivery_generation: Arc<std::sync::RwLock<Option<u64>>>,
} }
@@ -187,7 +196,7 @@ impl LibremetaverseClientOwner {
&self, &self,
connection: crate::config::GridConnection, connection: crate::config::GridConnection,
) -> Result<LibremetaverseSessionBackend, BackendError> { ) -> Result<LibremetaverseSessionBackend, BackendError> {
self.session_backend_inner(connection, None, None) self.session_backend_inner(connection, None, None, None)
} }
/// Creates a supervised live session whose generation owns exactly one /// Creates a supervised live session whose generation owns exactly one
@@ -201,7 +210,7 @@ impl LibremetaverseClientOwner {
connection: crate::config::GridConnection, connection: crate::config::GridConnection,
ingress: crate::interaction::InteractionIngress, ingress: crate::interaction::InteractionIngress,
) -> Result<LibremetaverseSessionBackend, BackendError> { ) -> Result<LibremetaverseSessionBackend, BackendError> {
self.session_backend_inner(connection, Some(ingress), None) self.session_backend_inner(connection, Some(ingress), None, None)
} }
/// Creates a session that generation-fences interaction and perception. /// Creates a session that generation-fences interaction and perception.
@@ -215,7 +224,27 @@ impl LibremetaverseClientOwner {
interaction: crate::interaction::InteractionIngress, interaction: crate::interaction::InteractionIngress,
perception: crate::perception::PerceptionIngress, perception: crate::perception::PerceptionIngress,
) -> Result<LibremetaverseSessionBackend, BackendError> { ) -> Result<LibremetaverseSessionBackend, BackendError> {
self.session_backend_inner(connection, Some(interaction), Some(perception)) self.session_backend_inner(connection, Some(interaction), Some(perception), None)
}
/// Creates a session that generation-fences interaction, perception, and embodiment.
///
/// # Errors
///
/// Rejects a malformed avatar identity before any subscription or login.
pub fn session_backend_with_agent_services(
&self,
connection: crate::config::GridConnection,
interaction: crate::interaction::InteractionIngress,
perception: crate::perception::PerceptionIngress,
behavior: crate::behavior::BehaviorIngress,
) -> Result<LibremetaverseSessionBackend, BackendError> {
self.session_backend_inner(
connection,
Some(interaction),
Some(perception),
Some(behavior),
)
} }
fn session_backend_inner( fn session_backend_inner(
@@ -223,6 +252,7 @@ impl LibremetaverseClientOwner {
connection: crate::config::GridConnection, connection: crate::config::GridConnection,
interaction: Option<crate::interaction::InteractionIngress>, interaction: Option<crate::interaction::InteractionIngress>,
perception: Option<crate::perception::PerceptionIngress>, perception: Option<crate::perception::PerceptionIngress>,
behavior: Option<crate::behavior::BehaviorIngress>,
) -> Result<LibremetaverseSessionBackend, BackendError> { ) -> Result<LibremetaverseSessionBackend, BackendError> {
let avatar = connection.avatar_name.trim(); let avatar = connection.avatar_name.trim();
let (first_name, last_name) = avatar let (first_name, last_name) = avatar
@@ -242,6 +272,7 @@ impl LibremetaverseClientOwner {
password: connection.password, password: connection.password,
interaction, interaction,
perception, perception,
behavior,
delivery_generation: Arc::clone(&self.delivery_generation), delivery_generation: Arc::clone(&self.delivery_generation),
}) })
} }
@@ -261,6 +292,15 @@ impl LibremetaverseClientOwner {
agent: Arc::clone(&self.agent), agent: Arc::clone(&self.agent),
} }
} }
#[must_use]
pub fn embodiment_sink(&self) -> LibremetaverseEmbodimentSink {
LibremetaverseEmbodimentSink {
client: self.client.clone(),
agent: Arc::clone(&self.agent),
generation: Arc::clone(&self.delivery_generation),
}
}
} }
#[cfg(feature = "live-grid")] #[cfg(feature = "live-grid")]
@@ -437,6 +477,222 @@ impl crate::perception::WorldSnapshotSource for LibremetaverseWorldSnapshotSourc
} }
} }
#[cfg(feature = "live-grid")]
impl LibremetaverseEmbodimentSink {
fn ensure_generation(&self, generation: u64) -> Result<(), crate::behavior::BehaviorError> {
let current = *self
.generation
.read()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if current == Some(generation) && self.client.network().native_connected() {
Ok(())
} else {
Err(crate::behavior::BehaviorError::NotReady)
}
}
fn native_pose(
&self,
generation: u64,
) -> Result<crate::behavior::EmbodiedPose, crate::behavior::BehaviorError> {
self.ensure_generation(generation)?;
let simulator = self
.client
.network()
.native_current_sim()
.ok_or(crate::behavior::BehaviorError::NotReady)?;
let position = self.agent.sim_position();
let rotation = self.agent.sim_rotation();
let sin_yaw = 2.0 * f64::from(rotation.w * rotation.z + rotation.x * rotation.y);
let cos_yaw = 1.0 - 2.0 * f64::from(rotation.y * rotation.y + rotation.z * rotation.z);
Ok(crate::behavior::EmbodiedPose {
generation,
region_id: simulator.region_id,
position: native_position(position),
heading_degrees: sin_yaw.atan2(cos_yaw).to_degrees().rem_euclid(360.0),
sitting: self.agent.sitting_on() != 0,
})
}
}
#[cfg(feature = "live-grid")]
#[allow(clippy::cast_possible_truncation)]
impl crate::behavior::EmbodimentSink for LibremetaverseEmbodimentSink {
fn current_pose(
&self,
generation: u64,
cancellation: CancellationToken,
) -> crate::behavior::EmbodimentFuture<'_, crate::behavior::EmbodiedPose> {
Box::pin(async move {
if cancellation.is_cancellation_requested() {
return Err(crate::behavior::BehaviorError::Cancelled);
}
self.native_pose(generation)
})
}
fn resolve_avatar(
&self,
generation: u64,
avatar_id: UUID,
cancellation: CancellationToken,
) -> crate::behavior::EmbodimentFuture<'_, crate::perception::WorldPosition> {
Box::pin(async move {
if cancellation.is_cancellation_requested() {
return Err(crate::behavior::BehaviorError::Cancelled);
}
self.ensure_generation(generation)?;
let simulator = self
.client
.network()
.native_current_sim()
.ok_or(crate::behavior::BehaviorError::NotReady)?;
simulator
.objects_avatars
.read()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.values()
.find(|avatar| avatar.id == avatar_id)
.map(|avatar| native_position(avatar.position))
.ok_or(crate::behavior::BehaviorError::TargetUnavailable)
})
}
fn face_point(
&self,
generation: u64,
point: crate::perception::WorldPosition,
cancellation: CancellationToken,
) -> crate::behavior::EmbodimentFuture<'_, ()> {
Box::pin(async move {
if cancellation.is_cancellation_requested() {
return Err(crate::behavior::BehaviorError::Cancelled);
}
self.ensure_generation(generation)?;
self.agent
.movement
.turn_toward(
libremetaverse_types::Vector3 {
x: point.x as f32,
y: point.y as f32,
z: point.z as f32,
},
Some(true),
)
.map_err(|_| crate::behavior::BehaviorError::NativeOperation)?;
Ok(())
})
}
fn begin_walk(
&self,
generation: u64,
point: crate::perception::WorldPosition,
cancellation: CancellationToken,
) -> crate::behavior::EmbodimentFuture<'_, ()> {
Box::pin(async move {
if cancellation.is_cancellation_requested() {
return Err(crate::behavior::BehaviorError::Cancelled);
}
self.ensure_generation(generation)?;
self.agent
.auto_pilot_local(
point.x.round() as i32,
point.y.round() as i32,
point.z as f32,
)
.map_err(|_| crate::behavior::BehaviorError::NativeOperation)
})
}
fn validate_walk_target(
&self,
generation: u64,
point: crate::perception::WorldPosition,
cancellation: CancellationToken,
) -> crate::behavior::EmbodimentFuture<'_, ()> {
Box::pin(async move {
if cancellation.is_cancellation_requested() {
return Err(crate::behavior::BehaviorError::Cancelled);
}
self.ensure_generation(generation)?;
let simulator = self
.client
.network()
.native_current_sim()
.ok_or(crate::behavior::BehaviorError::NotReady)?;
let parcels = self.client.parcels();
let current = parcels
.get_parcel_local_id(simulator.clone(), self.agent.sim_position())
.map_err(|_| crate::behavior::BehaviorError::NativeOperation)?;
let target = parcels
.get_parcel_local_id(
simulator,
libremetaverse_types::Vector3 {
x: point.x as f32,
y: point.y as f32,
z: point.z as f32,
},
)
.map_err(|_| crate::behavior::BehaviorError::NativeOperation)?;
if current == 0 || current != target {
return Err(crate::behavior::BehaviorError::RegionBoundary);
}
Ok(())
})
}
fn stop(
&self,
generation: u64,
cancellation: CancellationToken,
) -> crate::behavior::EmbodimentFuture<'_, ()> {
Box::pin(async move {
if cancellation.is_cancellation_requested() {
return Err(crate::behavior::BehaviorError::Cancelled);
}
self.ensure_generation(generation)?;
self.agent
.auto_pilot_cancel()
.map(|_| ())
.map_err(|_| crate::behavior::BehaviorError::NativeOperation)
})
}
fn sit(
&self,
generation: u64,
cancellation: CancellationToken,
) -> crate::behavior::EmbodimentFuture<'_, ()> {
Box::pin(async move {
if cancellation.is_cancellation_requested() {
return Err(crate::behavior::BehaviorError::Cancelled);
}
self.ensure_generation(generation)?;
self.agent
.sit()
.map_err(|_| crate::behavior::BehaviorError::NativeOperation)
})
}
fn stand(
&self,
generation: u64,
cancellation: CancellationToken,
) -> crate::behavior::EmbodimentFuture<'_, ()> {
Box::pin(async move {
if cancellation.is_cancellation_requested() {
return Err(crate::behavior::BehaviorError::Cancelled);
}
self.ensure_generation(generation)?;
self.agent
.stand()
.map(|_| ())
.map_err(|_| crate::behavior::BehaviorError::NativeOperation)
})
}
}
#[cfg(feature = "live-grid")] #[cfg(feature = "live-grid")]
fn native_position(value: libremetaverse_types::Vector3) -> crate::perception::WorldPosition { fn native_position(value: libremetaverse_types::Vector3) -> crate::perception::WorldPosition {
crate::perception::WorldPosition { crate::perception::WorldPosition {
@@ -580,6 +836,7 @@ struct LibremetaverseSession {
subscriptions: Vec<libremetaverse_types::compat::Subscription>, subscriptions: Vec<libremetaverse_types::compat::Subscription>,
interaction: Option<crate::interaction::InteractionIngress>, interaction: Option<crate::interaction::InteractionIngress>,
perception: Option<crate::perception::PerceptionIngress>, perception: Option<crate::perception::PerceptionIngress>,
behavior: Option<crate::behavior::BehaviorIngress>,
delivery_generation: Arc<std::sync::RwLock<Option<u64>>>, delivery_generation: Arc<std::sync::RwLock<Option<u64>>>,
} }
@@ -616,16 +873,20 @@ impl crate::session::GridSession for LibremetaverseSession {
cancellation: CancellationToken, cancellation: CancellationToken,
) -> crate::session::SessionFuture<'static, Result<(), crate::session::SessionFailure>> { ) -> crate::session::SessionFuture<'static, Result<(), crate::session::SessionFailure>> {
Box::pin(async move { Box::pin(async move {
*self
.delivery_generation
.write()
.unwrap_or_else(std::sync::PoisonError::into_inner) = None;
if let Some(interaction) = &self.interaction { if let Some(interaction) = &self.interaction {
let _ = interaction.try_disconnected(); let _ = interaction.try_disconnected();
} }
if let Some(perception) = &self.perception { if let Some(perception) = &self.perception {
perception.disconnected(); perception.disconnected();
} }
if let Some(behavior) = &self.behavior {
behavior.disconnected();
tokio::task::yield_now().await;
}
*self
.delivery_generation
.write()
.unwrap_or_else(std::sync::PoisonError::into_inner) = None;
self.subscriptions.clear(); self.subscriptions.clear();
self.network self.network
.native_logout_async(Some(cancellation)) .native_logout_async(Some(cancellation))
@@ -694,6 +955,7 @@ impl crate::session::GridSessionBackend for LibremetaverseSessionBackend {
let disconnected_sender = sender.clone(); let disconnected_sender = sender.clone();
let disconnected_interaction = self.interaction.clone(); let disconnected_interaction = self.interaction.clone();
let disconnected_perception = self.perception.clone(); let disconnected_perception = self.perception.clone();
let disconnected_behavior = self.behavior.clone();
let disconnected_generation = Arc::clone(&self.delivery_generation); let disconnected_generation = Arc::clone(&self.delivery_generation);
let disconnected = self let disconnected = self
.network .network
@@ -707,6 +969,9 @@ impl crate::session::GridSessionBackend for LibremetaverseSessionBackend {
if let Some(perception) = &disconnected_perception { if let Some(perception) = &disconnected_perception {
perception.disconnected(); perception.disconnected();
} }
if let Some(behavior) = &disconnected_behavior {
behavior.disconnected();
}
let kind = match event.reason() { let kind = match event.reason() {
libremetaverse::NetworkManagerDisconnectType::NetworkTimeout libremetaverse::NetworkManagerDisconnectType::NetworkTimeout
| libremetaverse::NetworkManagerDisconnectType::ClientInitiated => { | libremetaverse::NetworkManagerDisconnectType::ClientInitiated => {
@@ -727,6 +992,7 @@ impl crate::session::GridSessionBackend for LibremetaverseSessionBackend {
let ready_sender = sender.clone(); let ready_sender = sender.clone();
let ready_interaction = self.interaction.clone(); let ready_interaction = self.interaction.clone();
let ready_perception = self.perception.clone(); let ready_perception = self.perception.clone();
let ready_behavior = self.behavior.clone();
let ready_network = self.network.clone(); let ready_network = self.network.clone();
let ready_agent = Arc::clone(&self.agent); let ready_agent = Arc::clone(&self.agent);
let ready_generation = Arc::clone(&self.delivery_generation); let ready_generation = Arc::clone(&self.delivery_generation);
@@ -747,18 +1013,27 @@ impl crate::session::GridSessionBackend for LibremetaverseSessionBackend {
{ {
let _ = perception.connected(generation, simulator.region_id); let _ = perception.connected(generation, simulator.region_id);
} }
if let (Some(behavior), Some(simulator)) =
(&ready_behavior, ready_network.native_current_sim())
{
let _ = behavior.connected(generation, simulator.region_id);
}
let _ = ready_sender.try_send(crate::session::SessionSignal::Ready); let _ = ready_sender.try_send(crate::session::SessionSignal::Ready);
} }
})); }));
let mut subscriptions = vec![disconnected, ready]; let mut subscriptions = vec![disconnected, ready];
if let Some(perception) = &self.perception { if let Some(perception) = &self.perception {
let crossing_perception = perception.clone(); let crossing_perception = perception.clone();
let crossing_behavior = self.behavior.clone();
let crossing_network = self.network.clone(); let crossing_network = self.network.clone();
subscriptions.push(self.network.native_subscribe_sim_changed(Arc::new( subscriptions.push(self.network.native_subscribe_sim_changed(Arc::new(
move |_| { move |_| {
if let Some(simulator) = crossing_network.native_current_sim() { if let Some(simulator) = crossing_network.native_current_sim() {
let _ = let _ =
crossing_perception.region_changed(generation, simulator.region_id); crossing_perception.region_changed(generation, simulator.region_id);
if let Some(behavior) = &crossing_behavior {
let _ = behavior.region_changed(generation, simulator.region_id);
}
} }
}, },
))); )));
@@ -806,6 +1081,11 @@ impl crate::session::GridSessionBackend for LibremetaverseSessionBackend {
{ {
let _ = perception.connected(generation, simulator.region_id); let _ = perception.connected(generation, simulator.region_id);
} }
if let (Some(behavior), Some(simulator)) =
(&self.behavior, self.network.native_current_sim())
{
let _ = behavior.connected(generation, simulator.region_id);
}
let _ = sender.try_send(crate::session::SessionSignal::Ready); let _ = sender.try_send(crate::session::SessionSignal::Ready);
} }
let session: Box<dyn crate::session::GridSession> = Box::new(LibremetaverseSession { let session: Box<dyn crate::session::GridSession> = Box::new(LibremetaverseSession {
@@ -815,6 +1095,7 @@ impl crate::session::GridSessionBackend for LibremetaverseSessionBackend {
subscriptions, subscriptions,
interaction: self.interaction.clone(), interaction: self.interaction.clone(),
perception: self.perception.clone(), perception: self.perception.clone(),
behavior: self.behavior.clone(),
delivery_generation: Arc::clone(&self.delivery_generation), delivery_generation: Arc::clone(&self.delivery_generation),
}); });
Ok(session) Ok(session)

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,562 @@
use crate::backend::AuthorizedToolBackend;
use crate::behavior::*;
use crate::config::BehaviorSettings;
use crate::perception::WorldPosition;
use crate::policy::{
ActionOrigin, MemoryPolicyAudit, PolicyAuditSink, PolicyGateway, PolicyLimits,
PolicyRequestContext,
};
use crate::types::{ProposedToolCall, ToolCallOutcome};
use libremetaverse_types::UUID;
use libremetaverse_types::compat::CancellationToken;
use serde_json::{Value, json};
use std::collections::{BTreeMap, BTreeSet};
use std::sync::{Arc, Mutex};
use std::time::Duration;
fn uuid(number: u64) -> UUID {
UUID::new_with_string(format!("00000000-0000-4000-8000-{number:012x}")).expect("UUID")
}
fn settings() -> BehaviorSettings {
BehaviorSettings {
heartbeat: Duration::from_secs(30),
settle_delay: Duration::from_millis(10),
response_delay_min: Duration::from_millis(20),
response_delay_max: Duration::from_millis(20),
attention_dwell: Duration::from_millis(30),
idle_interval: Duration::from_secs(15),
action_timeout: Duration::from_secs(5),
max_walk_duration: Duration::from_secs(4),
stuck_timeout: Duration::from_secs(1),
min_action_interval: Duration::ZERO,
max_walk_distance_meters: 8,
max_attention_distance_meters: 96,
idle_look_enabled: true,
}
}
#[derive(Clone)]
struct FakeSink {
state: Arc<Mutex<FakeState>>,
}
struct FakeState {
pose: EmbodiedPose,
avatars: BTreeMap<UUID, WorldPosition>,
calls: Vec<String>,
walk_target: Option<WorldPosition>,
stuck: bool,
walk_allowed: bool,
}
impl FakeSink {
fn new(generation: u64, region_id: UUID) -> Self {
Self {
state: Arc::new(Mutex::new(FakeState {
pose: EmbodiedPose {
generation,
region_id,
position: WorldPosition {
x: 128.0,
y: 128.0,
z: 24.0,
},
heading_degrees: 0.0,
sitting: false,
},
avatars: BTreeMap::from([(
uuid(2),
WorldPosition {
x: 132.0,
y: 128.0,
z: 24.0,
},
)]),
calls: Vec::new(),
walk_target: None,
stuck: false,
walk_allowed: true,
})),
}
}
fn calls(&self) -> Vec<String> {
self.state.lock().expect("state").calls.clone()
}
fn set_region(&self, region_id: UUID) {
self.state.lock().expect("state").pose.region_id = region_id;
}
fn set_stuck(&self, stuck: bool) {
self.state.lock().expect("state").stuck = stuck;
}
fn set_walk_allowed(&self, allowed: bool) {
self.state.lock().expect("state").walk_allowed = allowed;
}
}
impl EmbodimentSink for FakeSink {
fn current_pose(
&self,
generation: u64,
_: CancellationToken,
) -> EmbodimentFuture<'_, EmbodiedPose> {
Box::pin(async move {
let mut state = self.state.lock().expect("state");
if state.pose.generation != generation {
return Err(BehaviorError::NotReady);
}
if let Some(target) = state.walk_target
&& !state.stuck
{
state.pose.position = target;
}
Ok(state.pose.clone())
})
}
fn resolve_avatar(
&self,
generation: u64,
id: UUID,
_: CancellationToken,
) -> EmbodimentFuture<'_, WorldPosition> {
Box::pin(async move {
let state = self.state.lock().expect("state");
if state.pose.generation != generation {
return Err(BehaviorError::NotReady);
}
state
.avatars
.get(&id)
.copied()
.ok_or(BehaviorError::TargetUnavailable)
})
}
fn face_point(
&self,
_: u64,
_: WorldPosition,
_: CancellationToken,
) -> EmbodimentFuture<'_, ()> {
Box::pin(async move {
self.state.lock().expect("state").calls.push("face".into());
Ok(())
})
}
fn begin_walk(
&self,
_: u64,
point: WorldPosition,
_: CancellationToken,
) -> EmbodimentFuture<'_, ()> {
Box::pin(async move {
let mut state = self.state.lock().expect("state");
state.calls.push("walk".into());
state.walk_target = Some(point);
Ok(())
})
}
fn validate_walk_target(
&self,
_: u64,
_: WorldPosition,
_: CancellationToken,
) -> EmbodimentFuture<'_, ()> {
Box::pin(async move {
self.state
.lock()
.expect("state")
.walk_allowed
.then_some(())
.ok_or(BehaviorError::RegionBoundary)
})
}
fn stop(&self, _: u64, _: CancellationToken) -> EmbodimentFuture<'_, ()> {
Box::pin(async move {
let mut state = self.state.lock().expect("state");
state.calls.push("stop".into());
state.walk_target = None;
Ok(())
})
}
fn sit(&self, _: u64, _: CancellationToken) -> EmbodimentFuture<'_, ()> {
Box::pin(async move {
let mut state = self.state.lock().expect("state");
state.calls.push("sit".into());
state.pose.sitting = true;
Ok(())
})
}
fn stand(&self, _: u64, _: CancellationToken) -> EmbodimentFuture<'_, ()> {
Box::pin(async move {
let mut state = self.state.lock().expect("state");
state.calls.push("stand".into());
state.pose.sitting = false;
Ok(())
})
}
}
struct FixedRandom(u64);
impl BehaviorRandom for FixedRandom {
fn next_u64(&self) -> u64 {
self.0
}
}
fn authorize(
settings: &BehaviorSettings,
name: &str,
arguments: &Value,
) -> crate::policy::AuthorizedAction {
let avatar = uuid(900);
let audit: Arc<dyn PolicyAuditSink> = Arc::new(MemoryPolicyAudit::new(64).expect("audit"));
let gateway = PolicyGateway::new(
BTreeSet::from([avatar]),
behavior_policy_tools(settings).expect("tools"),
PolicyLimits::default(),
audit,
)
.expect("gateway");
let context = PolicyRequestContext::new(
ActionOrigin::instant_message(avatar),
"behavior-session",
"behavior-correlation",
)
.expect("context");
let call = ProposedToolCall::new("behavior-call", name, arguments.to_string()).expect("call");
gateway
.evaluate(&context, &call, arguments, None, 100)
.expect("evaluation")
.into_authorization()
.expect("authorization")
}
async fn run_tool(
backend: &BehaviorBackend,
settings: &BehaviorSettings,
name: &str,
args: Value,
) -> ToolCallOutcome {
backend
.apply(
authorize(settings, name, &args),
CancellationToken::default(),
)
.await
.expect("backend")
}
#[test]
fn policy_exposes_only_bounded_high_level_actions() {
let tools = behavior_policy_tools(&settings()).expect("tools");
assert_eq!(tools.len(), 8);
let names = tools
.iter()
.map(|tool| tool.definition.name.as_str())
.collect::<Vec<_>>();
assert_eq!(
names,
[
FACE_AVATAR_TOOL,
FACE_POINT_TOOL,
LOOK_AROUND_TOOL,
WALK_SHORT_TOOL,
STOP_TOOL,
SIT_TOOL,
STAND_TOOL,
CURRENT_POSE_TOOL
]
);
assert!(names.iter().all(|name| !name.contains("teleport")
&& !name.contains("control")
&& !name.contains("follow")));
}
#[tokio::test(start_paused = true)]
async fn public_attention_waits_turns_and_returns_to_available() {
let region = uuid(10);
let sink = Arc::new(FakeSink::new(1, region));
let handle = BehaviorController::with_random(
settings(),
sink.clone(),
16,
32,
Duration::from_secs(1),
Arc::new(FixedRandom(0)),
)
.expect("controller")
.start();
let ingress = handle.ingress();
ingress.connected(1, region).expect("ready");
tokio::time::advance(Duration::from_millis(11)).await;
tokio::task::yield_now().await;
let pacing = tokio::spawn(async move { ingress.attention("delivery-1".into(), uuid(2)).await });
tokio::time::advance(Duration::from_millis(21)).await;
tokio::task::yield_now().await;
assert!(pacing.await.expect("task").is_ok());
assert_eq!(sink.calls(), vec!["face"]);
tokio::time::advance(Duration::from_millis(31)).await;
handle.shutdown().await.expect("shutdown");
}
#[tokio::test(start_paused = true)]
async fn quarter_turn_uses_bounded_intermediate_camera_updates() {
let region = uuid(10);
let sink = Arc::new(FakeSink::new(1, region));
let mut config = settings();
config.settle_delay = Duration::ZERO;
let handle =
BehaviorController::new(config.clone(), sink.clone(), 16, 32, Duration::from_secs(1))
.expect("controller")
.start();
let ingress = handle.ingress();
ingress.connected(1, region).expect("ready");
tokio::task::yield_now().await;
let backend = BehaviorBackend::new(ingress);
let turn = tokio::spawn({
let config = config.clone();
async move {
run_tool(
&backend,
&config,
FACE_POINT_TOOL,
json!({"x":128.0,"y":132.0,"z":24.0}),
)
.await
}
});
tokio::task::yield_now().await;
tokio::time::advance(Duration::from_millis(100)).await;
tokio::task::yield_now().await;
assert!(matches!(
turn.await.expect("turn"),
ToolCallOutcome::Completed { .. }
));
assert_eq!(sink.calls(), vec!["face", "face", "face"]);
handle.shutdown().await.expect("shutdown");
}
#[tokio::test(start_paused = true)]
async fn bounded_walk_stops_and_region_change_rejects_stale_pose() {
let region = uuid(10);
let sink = Arc::new(FakeSink::new(1, region));
let mut config = settings();
config.settle_delay = Duration::ZERO;
let handle =
BehaviorController::new(config.clone(), sink.clone(), 16, 32, Duration::from_secs(1))
.expect("controller")
.start();
let ingress = handle.ingress();
ingress.connected(1, region).expect("ready");
tokio::task::yield_now().await;
let backend = BehaviorBackend::new(ingress.clone());
let task = tokio::spawn({
let backend = backend.clone();
let config = config.clone();
async move {
run_tool(
&backend,
&config,
WALK_SHORT_TOOL,
json!({"heading_degrees":0.0,"distance_meters":2.0}),
)
.await
}
});
tokio::time::advance(Duration::from_millis(300)).await;
tokio::task::yield_now().await;
assert!(matches!(
task.await.expect("task"),
ToolCallOutcome::Completed { .. }
));
assert_eq!(sink.calls(), vec!["face", "walk", "stop"]);
let new_region = uuid(11);
sink.set_region(new_region);
ingress.region_changed(1, new_region).expect("cross");
tokio::task::yield_now().await;
let pose = run_tool(&backend, &config, CURRENT_POSE_TOOL, json!({})).await;
assert!(matches!(pose, ToolCallOutcome::Completed { .. }));
handle.shutdown().await.expect("shutdown");
}
#[tokio::test(start_paused = true)]
async fn parcel_boundary_rejects_walk_before_any_movement_update() {
let region = uuid(10);
let sink = Arc::new(FakeSink::new(1, region));
sink.set_walk_allowed(false);
let mut config = settings();
config.settle_delay = Duration::ZERO;
let handle =
BehaviorController::new(config.clone(), sink.clone(), 16, 32, Duration::from_secs(1))
.expect("controller")
.start();
let ingress = handle.ingress();
ingress.connected(1, region).expect("ready");
tokio::task::yield_now().await;
let backend = BehaviorBackend::new(ingress);
assert!(matches!(
run_tool(
&backend,
&config,
WALK_SHORT_TOOL,
json!({"heading_degrees":0.0,"distance_meters":2.0}),
)
.await,
ToolCallOutcome::Rejected { .. }
));
assert_eq!(sink.calls(), vec!["stop"]);
handle.shutdown().await.expect("shutdown");
}
#[tokio::test(start_paused = true)]
async fn pause_preempts_stuck_walk_and_blocks_all_motion() {
let region = uuid(10);
let sink = Arc::new(FakeSink::new(1, region));
sink.set_stuck(true);
let mut config = settings();
config.settle_delay = Duration::ZERO;
let handle =
BehaviorController::new(config.clone(), sink.clone(), 16, 32, Duration::from_secs(1))
.expect("controller")
.start();
let ingress = handle.ingress();
ingress.connected(1, region).expect("ready");
tokio::task::yield_now().await;
let backend = BehaviorBackend::new(ingress.clone());
let task = tokio::spawn({
let backend = backend.clone();
let config = config.clone();
async move {
run_tool(
&backend,
&config,
WALK_SHORT_TOOL,
json!({"heading_degrees":90.0,"distance_meters":2.0}),
)
.await
}
});
tokio::time::advance(Duration::from_millis(300)).await;
ingress.pause();
tokio::task::yield_now().await;
assert!(matches!(
task.await.expect("task"),
ToolCallOutcome::Rejected { .. }
));
let before = sink.calls().len();
assert!(matches!(
run_tool(&backend, &config, SIT_TOOL, json!({})).await,
ToolCallOutcome::Rejected { .. }
));
assert_eq!(sink.calls().len(), before);
handle.shutdown().await.expect("shutdown");
}
#[tokio::test(start_paused = true)]
async fn offline_and_emergency_stop_emit_zero_motion() {
let region = uuid(10);
let sink = Arc::new(FakeSink::new(1, region));
let mut config = settings();
config.settle_delay = Duration::ZERO;
let handle =
BehaviorController::new(config.clone(), sink.clone(), 16, 32, Duration::from_secs(1))
.expect("controller")
.start();
let ingress = handle.ingress();
let backend = BehaviorBackend::new(ingress.clone());
assert!(matches!(
run_tool(&backend, &config, SIT_TOOL, json!({})).await,
ToolCallOutcome::Rejected { .. }
));
ingress.connected(1, region).expect("ready");
tokio::task::yield_now().await;
ingress.emergency_stop();
tokio::task::yield_now().await;
let after_stop = sink.calls().len();
assert!(matches!(
run_tool(&backend, &config, STAND_TOOL, json!({})).await,
ToolCallOutcome::Rejected { .. }
));
assert_eq!(sink.calls().len(), after_stop);
handle.shutdown().await.expect("shutdown");
}
#[tokio::test(start_paused = true)]
async fn seeded_idle_is_one_low_frequency_look_without_movement_or_chat() {
let region = uuid(10);
let sink = Arc::new(FakeSink::new(1, region));
let mut config = settings();
config.settle_delay = Duration::ZERO;
let handle = BehaviorController::with_random(
config,
sink.clone(),
16,
32,
Duration::from_secs(1),
Arc::new(FixedRandom(30)),
)
.expect("controller")
.start();
let ingress = handle.ingress();
ingress.connected(1, region).expect("ready");
tokio::task::yield_now().await;
tokio::time::advance(Duration::from_secs(14)).await;
tokio::task::yield_now().await;
assert!(sink.calls().is_empty());
tokio::time::advance(Duration::from_secs(2)).await;
tokio::task::yield_now().await;
assert_eq!(sink.calls(), vec!["face"]);
handle.shutdown().await.expect("shutdown");
}
#[tokio::test(start_paused = true)]
async fn region_change_preempts_walk_and_disappearance_cancels_attention() {
let region = uuid(10);
let sink = Arc::new(FakeSink::new(1, region));
sink.set_stuck(true);
let mut config = settings();
config.settle_delay = Duration::ZERO;
config.response_delay_min = Duration::ZERO;
config.response_delay_max = Duration::ZERO;
let handle =
BehaviorController::new(config.clone(), sink.clone(), 16, 32, Duration::from_secs(1))
.expect("controller")
.start();
let ingress = handle.ingress();
ingress.connected(1, region).expect("ready");
tokio::task::yield_now().await;
let backend = BehaviorBackend::new(ingress.clone());
let task = tokio::spawn({
let backend = backend.clone();
let config = config.clone();
async move {
run_tool(
&backend,
&config,
WALK_SHORT_TOOL,
json!({"heading_degrees":180.0,"distance_meters":2.0}),
)
.await
}
});
tokio::time::advance(Duration::from_millis(300)).await;
let next = uuid(11);
sink.set_region(next);
ingress.region_changed(1, next).expect("cross");
tokio::task::yield_now().await;
assert!(matches!(
task.await.expect("walk"),
ToolCallOutcome::Rejected { .. }
));
let missing = tokio::spawn({
let ingress = ingress.clone();
async move { ingress.attention("missing".into(), uuid(999)).await }
});
tokio::task::yield_now().await;
assert_eq!(
missing.await.expect("attention"),
Err(BehaviorError::TargetUnavailable)
);
handle.shutdown().await.expect("shutdown");
}

View File

@@ -200,6 +200,43 @@ impl Default for Limits {
#[derive(Clone, Debug, Eq, PartialEq)] #[derive(Clone, Debug, Eq, PartialEq)]
pub struct BehaviorSettings { pub struct BehaviorSettings {
pub heartbeat: Duration, pub heartbeat: Duration,
pub settle_delay: Duration,
pub response_delay_min: Duration,
pub response_delay_max: Duration,
pub attention_dwell: Duration,
pub idle_interval: Duration,
pub action_timeout: Duration,
pub max_walk_duration: Duration,
pub stuck_timeout: Duration,
pub min_action_interval: Duration,
pub max_walk_distance_meters: u32,
pub max_attention_distance_meters: u32,
pub idle_look_enabled: bool,
}
impl BehaviorSettings {
pub(crate) fn is_valid(&self) -> bool {
if self.heartbeat.is_zero()
|| self.settle_delay > Duration::from_secs(30)
|| self.response_delay_min > self.response_delay_max
|| self.response_delay_max > Duration::from_secs(30)
|| self.attention_dwell > Duration::from_mins(1)
|| self.idle_interval < Duration::from_secs(15)
|| self.idle_interval > Duration::from_hours(1)
|| self.action_timeout.is_zero()
|| self.action_timeout > Duration::from_mins(1)
|| self.max_walk_duration.is_zero()
|| self.max_walk_duration > self.action_timeout
|| self.stuck_timeout.is_zero()
|| self.stuck_timeout > self.max_walk_duration
|| self.min_action_interval > Duration::from_secs(30)
|| !(1..=20).contains(&self.max_walk_distance_meters)
|| !(1..=256).contains(&self.max_attention_distance_meters)
{
return false;
}
true
}
} }
/// Conversation-memory settings. Persistence is opt-in and uses /// Conversation-memory settings. Persistence is opt-in and uses
@@ -260,6 +297,9 @@ impl AgentConfig {
self.interaction self.interaction
.validate() .validate()
.map_err(|_| ConfigError::InvalidInteraction)?; .map_err(|_| ConfigError::InvalidInteraction)?;
if !self.behavior.is_valid() {
return Err(ConfigError::InvalidBehavior);
}
if self.mode != OperatingMode::OfflineFake && self.grid.is_none() { if self.mode != OperatingMode::OfflineFake && self.grid.is_none() {
return Err(ConfigError::Missing { return Err(ConfigError::Missing {
field: "grid", field: "grid",
@@ -435,6 +475,7 @@ pub enum ConfigError {
InvalidReconnect, InvalidReconnect,
InvalidConversationMemory, InvalidConversationMemory,
InvalidInteraction, InvalidInteraction,
InvalidBehavior,
} }
impl fmt::Display for ConfigError { impl fmt::Display for ConfigError {
@@ -489,6 +530,7 @@ impl fmt::Display for ConfigError {
formatter.write_str("invalid conversation-memory bounds") formatter.write_str("invalid conversation-memory bounds")
} }
Self::InvalidInteraction => formatter.write_str("invalid interaction bounds"), Self::InvalidInteraction => formatter.write_str("invalid interaction bounds"),
Self::InvalidBehavior => formatter.write_str("invalid embodied behavior bounds"),
} }
} }
} }
@@ -564,6 +606,18 @@ struct RawLimits {
#[serde(default, deny_unknown_fields)] #[serde(default, deny_unknown_fields)]
struct RawBehavior { struct RawBehavior {
heartbeat_seconds: Option<u64>, heartbeat_seconds: Option<u64>,
settle_milliseconds: Option<u64>,
response_delay_min_milliseconds: Option<u64>,
response_delay_max_milliseconds: Option<u64>,
attention_dwell_milliseconds: Option<u64>,
idle_interval_seconds: Option<u64>,
action_timeout_seconds: Option<u64>,
max_walk_duration_seconds: Option<u64>,
stuck_timeout_seconds: Option<u64>,
min_action_interval_milliseconds: Option<u64>,
max_walk_distance_meters: Option<u32>,
max_attention_distance_meters: Option<u32>,
idle_look_enabled: Option<bool>,
} }
#[derive(Clone, Default, Deserialize)] #[derive(Clone, Default, Deserialize)]
@@ -896,6 +950,48 @@ fn resolve<E: Environment>(
1, 1,
300, 300,
)?, )?,
settle_delay: Duration::from_millis(raw.behavior.settle_milliseconds.unwrap_or(2_000)),
response_delay_min: Duration::from_millis(
raw.behavior.response_delay_min_milliseconds.unwrap_or(350),
),
response_delay_max: Duration::from_millis(
raw.behavior
.response_delay_max_milliseconds
.unwrap_or(1_200),
),
attention_dwell: Duration::from_millis(
raw.behavior.attention_dwell_milliseconds.unwrap_or(4_000),
),
idle_interval: checked_duration(
"behavior.idle_interval_seconds",
raw.behavior.idle_interval_seconds.unwrap_or(120),
15,
3_600,
)?,
action_timeout: checked_duration(
"behavior.action_timeout_seconds",
raw.behavior.action_timeout_seconds.unwrap_or(15),
1,
60,
)?,
max_walk_duration: checked_duration(
"behavior.max_walk_duration_seconds",
raw.behavior.max_walk_duration_seconds.unwrap_or(10),
1,
60,
)?,
stuck_timeout: checked_duration(
"behavior.stuck_timeout_seconds",
raw.behavior.stuck_timeout_seconds.unwrap_or(3),
1,
60,
)?,
min_action_interval: Duration::from_millis(
raw.behavior.min_action_interval_milliseconds.unwrap_or(750),
),
max_walk_distance_meters: raw.behavior.max_walk_distance_meters.unwrap_or(8),
max_attention_distance_meters: raw.behavior.max_attention_distance_meters.unwrap_or(96),
idle_look_enabled: raw.behavior.idle_look_enabled.unwrap_or(true),
}, },
reconnect, reconnect,
conversation, conversation,

View File

@@ -270,6 +270,26 @@ pub trait InteractionResponder: Send + Sync + 'static {
) -> ResponderFuture<'_>; ) -> ResponderFuture<'_>;
} }
pub type PacerFuture<'a> =
Pin<Box<dyn Future<Output = Result<(), InteractionPacingError>> + Send + 'a>>;
/// Optional embodied response hook. It may turn toward a nearby speaker and
/// wait a bounded, human-scale delay before public delivery.
pub trait ResponsePacer: Send + Sync + 'static {
fn prepare_public_response(
&self,
delivery_id: String,
avatar_id: UUID,
cancellation: CancellationToken,
) -> PacerFuture<'_>;
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum InteractionPacingError {
Cancelled,
Unavailable,
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)] #[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum InteractionModelError { pub enum InteractionModelError {
Cancelled, Cancelled,
@@ -450,6 +470,7 @@ pub struct InteractionCoordinator {
conversation: Arc<ConversationStore>, conversation: Arc<ConversationStore>,
responder: Arc<dyn InteractionResponder>, responder: Arc<dyn InteractionResponder>,
sink: Arc<dyn InteractionSink>, sink: Arc<dyn InteractionSink>,
pacer: Option<Arc<dyn ResponsePacer>>,
input_capacity: usize, input_capacity: usize,
observation_capacity: usize, observation_capacity: usize,
shutdown_timeout: Duration, shutdown_timeout: Duration,
@@ -499,12 +520,19 @@ impl InteractionCoordinator {
conversation, conversation,
responder, responder,
sink, sink,
pacer: None,
input_capacity, input_capacity,
observation_capacity, observation_capacity,
shutdown_timeout, shutdown_timeout,
}) })
} }
#[must_use]
pub fn with_response_pacer(mut self, pacer: Arc<dyn ResponsePacer>) -> Self {
self.pacer = Some(pacer);
self
}
#[must_use] #[must_use]
pub fn start(self) -> InteractionHandle { pub fn start(self) -> InteractionHandle {
let (commands, command_receiver) = mpsc::channel(self.input_capacity); let (commands, command_receiver) = mpsc::channel(self.input_capacity);
@@ -947,6 +975,7 @@ fn schedule_ready(
Arc::clone(&coordinator.conversation), Arc::clone(&coordinator.conversation),
Arc::clone(&coordinator.responder), Arc::clone(&coordinator.responder),
Arc::clone(&coordinator.sink), Arc::clone(&coordinator.sink),
coordinator.pacer.clone(),
Arc::clone(limiter), Arc::clone(limiter),
observations.clone(), observations.clone(),
completions.clone(), completions.clone(),
@@ -966,6 +995,7 @@ async fn process_batch(
conversation: Arc<ConversationStore>, conversation: Arc<ConversationStore>,
responder: Arc<dyn InteractionResponder>, responder: Arc<dyn InteractionResponder>,
sink: Arc<dyn InteractionSink>, sink: Arc<dyn InteractionSink>,
pacer: Option<Arc<dyn ResponsePacer>>,
limiter: Arc<OutboundRateLimiter>, limiter: Arc<OutboundRateLimiter>,
observations: mpsc::Sender<InteractionObservation>, observations: mpsc::Sender<InteractionObservation>,
completions: mpsc::Sender<WorkCompletion>, completions: mpsc::Sender<WorkCompletion>,
@@ -1001,6 +1031,7 @@ async fn process_batch(
&sink, &sink,
&limiter, &limiter,
&cancellation, &cancellation,
pacer.as_ref(),
"I could not retain that message safely.", "I could not retain that message safely.",
) )
.await; .await;
@@ -1031,6 +1062,7 @@ async fn process_batch(
&sink, &sink,
&limiter, &limiter,
&cancellation, &cancellation,
pacer.as_ref(),
"For safety, commands must be requested through an authorized direct message.", "For safety, commands must be requested through an authorized direct message.",
) )
.await; .await;
@@ -1101,6 +1133,15 @@ async fn process_batch(
return; return;
}; };
let parts = split_utf8(&safe, settings.grid_chunk_bytes); let parts = split_utf8(&safe, settings.grid_chunk_bytes);
if public_engaged && let Some(pacer) = &pacer {
let _ = pacer
.prepare_public_response(
delivery_id.as_str().to_owned(),
trigger.sender_id,
cancellation.clone(),
)
.await;
}
let delivered = deliver_parts( let delivered = deliver_parts(
generation, generation,
trigger, trigger,
@@ -1173,6 +1214,7 @@ async fn process_batch(
&sink, &sink,
&limiter, &limiter,
&cancellation, &cancellation,
pacer.as_ref(),
"I could not answer that just now.", "I could not answer that just now.",
) )
.await; .await;
@@ -1209,6 +1251,7 @@ async fn fixed_failure(
sink: &Arc<dyn InteractionSink>, sink: &Arc<dyn InteractionSink>,
limiter: &Arc<OutboundRateLimiter>, limiter: &Arc<OutboundRateLimiter>,
cancellation: &CancellationToken, cancellation: &CancellationToken,
pacer: Option<&Arc<dyn ResponsePacer>>,
text: &str, text: &str,
) -> usize { ) -> usize {
let useful = trigger.channel == InteractionChannel::DirectIm let useful = trigger.channel == InteractionChannel::DirectIm
@@ -1218,6 +1261,18 @@ async fn fixed_failure(
let Some(session_id) = session_id.filter(|_| useful) else { let Some(session_id) = session_id.filter(|_| useful) else {
return 0; return 0;
}; };
if trigger.channel == InteractionChannel::PublicChat
&& trigger.nearby
&& let Some(pacer) = pacer
{
let _ = pacer
.prepare_public_response(
trigger.delivery_id.as_str().to_owned(),
trigger.sender_id,
cancellation.clone(),
)
.await;
}
let parts = split_utf8(text, settings.grid_chunk_bytes); let parts = split_utf8(text, settings.grid_chunk_bytes);
deliver_parts( deliver_parts(
generation, generation,

View File

@@ -165,6 +165,28 @@ impl InteractionSink for FakeSink {
} }
} }
struct FakePacer {
called: AtomicBool,
delay: Duration,
}
impl ResponsePacer for FakePacer {
fn prepare_public_response(
&self,
_delivery_id: String,
_avatar_id: UUID,
cancellation: libremetaverse_types::compat::CancellationToken,
) -> PacerFuture<'_> {
self.called.store(true, Ordering::Release);
Box::pin(async move {
tokio::select! {
() = cancellation.cancelled() => Err(InteractionPacingError::Cancelled),
() = tokio::time::sleep(self.delay) => Ok(()),
}
})
}
}
fn coordinator( fn coordinator(
interaction_settings: InteractionSettings, interaction_settings: InteractionSettings,
authorized: BTreeSet<UUID>, authorized: BTreeSet<UUID>,
@@ -235,6 +257,38 @@ fn request_text(request: &ResponseRequest) -> String {
.join("\n") .join("\n")
} }
#[tokio::test(start_paused = true)]
async fn public_response_pacer_runs_before_visible_delivery() {
let responder = Arc::new(FakeResponder::with_response("Hello there"));
let sink = Arc::new(FakeSink::default());
let pacer = Arc::new(FakePacer {
called: AtomicBool::new(false),
delay: Duration::from_millis(500),
});
let mut handle = coordinator(settings(), BTreeSet::new(), responder, sink.clone())
.with_response_pacer(pacer.clone())
.start();
handle.connected(1).expect("connect");
handle
.submit(inbound(
"paced",
1,
InteractionChannel::PublicChat,
"hello metacrate",
))
.await
.expect("submit");
settle_debounce().await;
assert!(pacer.called.load(Ordering::Acquire));
assert!(sink.messages().is_empty());
tokio::time::advance(Duration::from_millis(501)).await;
for _ in 0..4 {
tokio::task::yield_now().await;
}
assert_eq!(sink.messages().len(), 1);
handle.shutdown().await.expect("shutdown");
}
#[tokio::test(start_paused = true)] #[tokio::test(start_paused = true)]
#[allow(clippy::too_many_lines)] #[allow(clippy::too_many_lines)]
async fn mentions_aliases_greetings_ambient_and_public_routes_are_correct() { async fn mentions_aliases_greetings_ambient_and_public_routes_are_correct() {

View File

@@ -5,6 +5,7 @@
//! subprocess, provider-SDK, or platform-specific dependency. //! subprocess, provider-SDK, or platform-specific dependency.
pub mod backend; pub mod backend;
pub mod behavior;
pub mod config; pub mod config;
pub mod conversation; pub mod conversation;
pub mod interaction; pub mod interaction;
@@ -16,6 +17,8 @@ pub mod session;
pub mod tool_loop; pub mod tool_loop;
pub mod types; pub mod types;
#[cfg(test)]
mod behavior_tests;
#[cfg(test)] #[cfg(test)]
mod conversation_tests; mod conversation_tests;
#[cfg(test)] #[cfg(test)]
@@ -33,8 +36,15 @@ pub use backend::{
}; };
#[cfg(feature = "live-grid")] #[cfg(feature = "live-grid")]
pub use backend::{ pub use backend::{
LibremetaverseClientOwner, LibremetaverseInteractionSink, LibremetaverseSessionBackend, LibremetaverseClientOwner, LibremetaverseEmbodimentSink, LibremetaverseInteractionSink,
LibremetaverseWorldSnapshotSource, LibremetaverseSessionBackend, LibremetaverseWorldSnapshotSource,
};
pub use behavior::{
AuthorizedBackendRouter, BehaviorBackend, BehaviorController, BehaviorError, BehaviorHandle,
BehaviorIngress, BehaviorMode, BehaviorObservation, BehaviorOutcome, BehaviorPolicyResult,
BehaviorRandom, BehaviorTrigger, CURRENT_POSE_TOOL, EmbodiedPose, EmbodimentFuture,
EmbodimentSink, FACE_AVATAR_TOOL, FACE_POINT_TOOL, LOOK_AROUND_TOOL, SIT_TOOL, STAND_TOOL,
STOP_TOOL, WALK_SHORT_TOOL, behavior_policy_tools,
}; };
pub use config::{ pub use config::{
AgentConfig, BehaviorSettings, ConfigError, ConfigLoader, ConversationSettings, EndpointUrl, AgentConfig, BehaviorSettings, ConfigError, ConfigLoader, ConversationSettings, EndpointUrl,
@@ -51,9 +61,10 @@ pub use interaction::{
DeliveryFuture, DeliveryOutcome, ImDialogKind, InboundInteraction, InboundSource, DeliveryFuture, DeliveryOutcome, ImDialogKind, InboundInteraction, InboundSource,
InteractionChannel, InteractionCoordinator, InteractionDeliveryError, InteractionError, InteractionChannel, InteractionCoordinator, InteractionDeliveryError, InteractionError,
InteractionHandle, InteractionIngress, InteractionIntent, InteractionModelError, InteractionHandle, InteractionIngress, InteractionIntent, InteractionModelError,
InteractionObservation, InteractionOrigin, InteractionResponder, InteractionSettings, InteractionObservation, InteractionOrigin, InteractionPacingError, InteractionResponder,
InteractionSink, OutboundInteraction, PolicyLlmResponder, ResponderFuture, ResponseRequest, InteractionSettings, InteractionSink, OutboundInteraction, PacerFuture, PolicyLlmResponder,
SuppressionReason, VisibleResponse, split_utf8, ResponderFuture, ResponsePacer, ResponseRequest, SuppressionReason, VisibleResponse,
split_utf8,
}; };
pub use llm::{ pub use llm::{
Completion, CompletionMessage, ContentPart, ImageDetail, LlmClient, LlmError, Completion, CompletionMessage, ContentPart, ImageDetail, LlmClient, LlmError,

View File

@@ -156,14 +156,16 @@ async fn run_live(
})?; })?;
let owner = LibremetaverseClientOwner::new()?; let owner = LibremetaverseClientOwner::new()?;
let mut live = start_live_interactions(&config, &owner)?; let mut live = start_live_interactions(&config, &owner)?;
let backend = match owner.session_backend_with_services( let backend = match owner.session_backend_with_agent_services(
connection, connection,
live.interaction.ingress(), live.interaction.ingress(),
live.perception.clone(), live.perception.clone(),
live.behavior.ingress(),
) { ) {
Ok(backend) => backend, Ok(backend) => backend,
Err(error) => { Err(error) => {
live.interaction.shutdown().await?; live.interaction.shutdown().await?;
live.behavior.shutdown().await?;
return Err(error.into()); return Err(error.into());
} }
}; };
@@ -212,14 +214,18 @@ async fn run_live(
if let Err(error) = readiness { if let Err(error) = readiness {
let session_result = handle.shutdown().await; let session_result = handle.shutdown().await;
let interaction_result = live.interaction.shutdown().await; let interaction_result = live.interaction.shutdown().await;
let behavior_result = live.behavior.shutdown().await;
session_result?; session_result?;
interaction_result?; interaction_result?;
behavior_result?;
return Err(error.into()); return Err(error.into());
} }
let session_result = handle.shutdown().await; let session_result = handle.shutdown().await;
let interaction_result = live.interaction.shutdown().await; let interaction_result = live.interaction.shutdown().await;
let behavior_result = live.behavior.shutdown().await;
session_result?; session_result?;
interaction_result?; interaction_result?;
behavior_result?;
println!("grid agent completed one supervised login/logout cycle"); println!("grid agent completed one supervised login/logout cycle");
return Ok(()); return Ok(());
} }
@@ -254,12 +260,18 @@ async fn run_live(
let Some(event) = event else { break; }; let Some(event) = event else { break; };
println!("grid perception event={event:?}"); println!("grid perception event={event:?}");
} }
event = live.behavior.next_observation() => {
let Some(event) = event else { break; };
println!("grid behavior event={event:?}");
}
} }
} }
let session_result = handle.shutdown().await; let session_result = handle.shutdown().await;
let interaction_result = live.interaction.shutdown().await; let interaction_result = live.interaction.shutdown().await;
let behavior_result = live.behavior.shutdown().await;
session_result?; session_result?;
interaction_result?; interaction_result?;
behavior_result?;
if let Some(error) = signal_error { if let Some(error) = signal_error {
return Err(error.into()); return Err(error.into());
} }
@@ -273,6 +285,7 @@ struct LiveInteractions {
perception: metacrate_grid_agent::PerceptionIngress, perception: metacrate_grid_agent::PerceptionIngress,
perception_observations: perception_observations:
tokio::sync::mpsc::Receiver<metacrate_grid_agent::PerceptionObservation>, tokio::sync::mpsc::Receiver<metacrate_grid_agent::PerceptionObservation>,
behavior: metacrate_grid_agent::BehaviorHandle,
} }
#[cfg(feature = "live-grid")] #[cfg(feature = "live-grid")]
@@ -281,9 +294,10 @@ fn start_live_interactions(
owner: &metacrate_grid_agent::LibremetaverseClientOwner, owner: &metacrate_grid_agent::LibremetaverseClientOwner,
) -> Result<LiveInteractions, Box<dyn Error>> { ) -> Result<LiveInteractions, Box<dyn Error>> {
use metacrate_grid_agent::{ use metacrate_grid_agent::{
AuthorizedToolBackend, ConversationStore, InteractionCoordinator, LlmClient, AuthorizedBackendRouter, AuthorizedToolBackend, BehaviorBackend, BehaviorController,
LlmTransportLimits, MemoryPolicyAudit, PerceptionBackend, PolicyGateway, PolicyLimits, ConversationStore, InteractionCoordinator, LlmClient, LlmTransportLimits,
PolicyLlmResponder, ToolLoopLimits, perception_policy_tools, MemoryPolicyAudit, PerceptionBackend, PolicyGateway, PolicyLimits, PolicyLlmResponder,
ToolLoopLimits, behavior_policy_tools, perception_policy_tools,
}; };
let transport_limits = LlmTransportLimits { let transport_limits = LlmTransportLimits {
@@ -304,11 +318,34 @@ fn start_live_interactions(
let perception_observations = perception let perception_observations = perception
.take_observations() .take_observations()
.ok_or_else(|| CliError("perception observation receiver already claimed".into()))?; .ok_or_else(|| CliError("perception observation receiver already claimed".into()))?;
let behavior = BehaviorController::new(
config.behavior.clone(),
Arc::new(owner.embodiment_sink()),
config.limits.control_queue,
config.limits.observable_queue,
config.timeouts.shutdown,
)?
.start();
let behavior_ingress = behavior.ingress();
let client = Arc::new(LlmClient::new(config.llm.clone(), transport_limits)?); let client = Arc::new(LlmClient::new(config.llm.clone(), transport_limits)?);
let audit = Arc::new(MemoryPolicyAudit::new(config.limits.observable_queue)?); let audit = Arc::new(MemoryPolicyAudit::new(config.limits.observable_queue)?);
let mut tools = perception_policy_tools()?;
tools.extend(behavior_policy_tools(&config.behavior)?);
let routes = tools
.iter()
.map(|tool| tool.definition.name.as_str().to_owned())
.map(|name| {
let backend: Arc<dyn AuthorizedToolBackend> = if name.starts_with("behavior_") {
Arc::new(BehaviorBackend::new(behavior_ingress.clone()))
} else {
perception.clone()
};
(name, backend)
})
.collect::<Vec<_>>();
let gateway = Arc::new(PolicyGateway::new( let gateway = Arc::new(PolicyGateway::new(
config.authorized_avatar_uuids.clone(), config.authorized_avatar_uuids.clone(),
perception_policy_tools()?, tools,
PolicyLimits::default(), PolicyLimits::default(),
audit, audit,
)?); )?);
@@ -325,15 +362,17 @@ fn start_live_interactions(
.duration_since(UNIX_EPOCH) .duration_since(UNIX_EPOCH)
.map_or(0, |duration| duration.as_secs()) .map_or(0, |duration| duration.as_secs())
}); });
let perception_backend: Arc<dyn AuthorizedToolBackend> = perception; let routed_backend: Arc<dyn AuthorizedToolBackend> =
Arc::new(AuthorizedBackendRouter::new(routes)?);
let responder = Arc::new(PolicyLlmResponder::new( let responder = Arc::new(PolicyLlmResponder::new(
client, client,
gateway, gateway,
perception_backend, routed_backend,
loop_limits, loop_limits,
now, now,
)?); )?);
let sink = Arc::new(owner.interaction_sink()); let sink = Arc::new(owner.interaction_sink());
let pacer: Arc<dyn metacrate_grid_agent::ResponsePacer> = Arc::new(behavior_ingress);
let interaction = InteractionCoordinator::new( let interaction = InteractionCoordinator::new(
config.interaction.clone(), config.interaction.clone(),
libremetaverse_types::UUID::zero(), libremetaverse_types::UUID::zero(),
@@ -345,10 +384,12 @@ fn start_live_interactions(
config.limits.observable_queue, config.limits.observable_queue,
config.timeouts.shutdown, config.timeouts.shutdown,
)? )?
.with_response_pacer(pacer)
.start(); .start();
Ok(LiveInteractions { Ok(LiveInteractions {
interaction, interaction,
perception: perception_ingress, perception: perception_ingress,
perception_observations, perception_observations,
behavior,
}) })
} }

View File

@@ -43,10 +43,10 @@ fn package_has_only_reviewed_rust_dependencies_and_no_build_script() {
#[test] #[test]
fn runtime_source_has_no_subprocess_or_native_abi_escape_hatch() { fn runtime_source_has_no_subprocess_or_native_abi_escape_hatch() {
let source = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("src"); let source = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("src");
let mut files = Vec::with_capacity(18); let mut files = Vec::with_capacity(20);
collect_rust_files(&source, &mut files); collect_rust_files(&source, &mut files);
assert!( assert!(
files.len() <= 18, files.len() <= 20,
"source-file count needs a reviewed bound update" "source-file count needs a reviewed bound update"
); );
for path in files { for path in files {
@@ -116,6 +116,37 @@ fn live_session_adapter_reuses_native_lifecycle_and_messaging_managers() {
assert!(!backend.contains("reqwest")); assert!(!backend.contains("reqwest"));
} }
#[test]
fn embodied_adapter_uses_only_reviewed_high_level_native_movement() {
let root = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
let backend = fs::read_to_string(root.join("src/backend.rs")).expect("backend source");
for required in [
".movement\n .turn_toward(",
".auto_pilot_local(",
".auto_pilot_cancel()",
".get_parcel_local_id(",
".sit()",
".stand()",
] {
assert!(
backend.contains(required),
"missing native embodiment mapping {required}"
);
}
let behavior = fs::read_to_string(root.join("src/behavior.rs")).expect("behavior source");
for forbidden in [
"Teleport(",
"TouchObject(",
"set_agent_controls(",
"FollowAvatar(",
] {
assert!(
!behavior.contains(forbidden),
"model-facing behavior contains forbidden primitive {forbidden}"
);
}
}
fn collect_rust_files(directory: &Path, output: &mut Vec<PathBuf>) { fn collect_rust_files(directory: &Path, output: &mut Vec<PathBuf>) {
for entry in fs::read_dir(directory).expect("read source directory") { for entry in fs::read_dir(directory).expect("read source directory") {
let path = entry.expect("source entry").path(); let path = entry.expect("source entry").path();

View File

@@ -0,0 +1,19 @@
# Grid-agent embodied behavior
The embodied controller is the only route from an LLM-authorized action to avatar movement. Its public surface is deliberately high level: face a visible avatar or nearby point, make one small look shift, walk a short distance on a heading, stop, sit, stand, and read the current pose. It exposes no control flags, raw packets, follow/wander primitive, unrestricted autopilot, flight, teleport, touch, or arbitrary animation.
## State and preemption
The observable modes are `offline`, `settling`, `available`, `engaged`, `executing`, `roaming`, `paused`, and `recovering`. Session readiness enters a bounded settling period. Public-response pacing enters `engaged`; authorized actions enter `executing`; timeout, stuck, and native failures pass through `recovering`. Roaming is only a marker for a separately policy-authorized scheduler task—the controller does not invent routes. Operator pause and the global emergency stop preempt movement and send a stop request. Disconnects and region changes cancel in-flight work and invalidate every old target.
Every transition records its trigger. Every action record contains generation and region provenance, action name, trigger, policy authorization class or ID, duration, and outcome without including message text or tool arguments.
## Safety envelope
- Only authenticated IMs, the authenticated local operator, and authorized scheduler grants may invoke behavior tools. Public speaker attention is a fixed built-in response behavior, not a public tool authorization.
- Face targets must be present in the current avatar cache or be a finite nearby point in the current 256 m region.
- A short walk is limited by configured distance and duration. Before moving, the native adapter verifies that the target's cached 4 m parcel-map cell has the same nonzero parcel ID as the current position. The controller then faces the target, starts one local movement request, polls pose, and always cancels movement on arrival, timeout, stuck detection, pause, emergency stop, disconnect, or region change.
- Actions have a global timeout and minimum interval. Idle behavior is one low-frequency look shift; it never moves continuously and never emits chat.
- Response delay, attention dwell, idle interval, timeouts, distance bounds, and rate limits are validated configuration. Random pacing is injectable so paused-time tests are deterministic.
The live adapter reuses the single native client/manager graph. Turns use `AgentMovement::turn_toward`; bounded walking uses a local autopilot target that is always paired with `auto_pilot_cancel`; sit and stand use the existing agent manager. Native state reads and actions are generation-fenced before use.