From 2f5f03ac6f5b5d192cfbb82eb064819935e88032 Mon Sep 17 00:00:00 2001 From: Chili Palmer Date: Sat, 22 Aug 2026 00:48:58 +0200 Subject: [PATCH] feat: complete stable OpenSim live interactions --- crates/libremetaverse/src/grid_manager.rs | 11 + crates/metacrate-grid-agent/src/backend.rs | 82 ++++-- crates/metacrate-grid-agent/src/behavior.rs | 18 +- .../src/behavior_tests.rs | 8 + crates/metacrate-grid-agent/src/build.rs | 238 ++++++++++++++---- .../metacrate-grid-agent/src/build_tests.rs | 48 +++- crates/metacrate-grid-agent/src/lib.rs | 7 +- crates/metacrate-grid-agent/src/llm.rs | 5 + crates/metacrate-grid-agent/src/vision.rs | 74 +++--- .../tests/dependency_policy.rs | 3 +- programs/src/test_client.rs | 2 + 11 files changed, 396 insertions(+), 100 deletions(-) diff --git a/crates/libremetaverse/src/grid_manager.rs b/crates/libremetaverse/src/grid_manager.rs index 1756ac2..d1a793f 100644 --- a/crates/libremetaverse/src/grid_manager.rs +++ b/crates/libremetaverse/src/grid_manager.rs @@ -514,6 +514,17 @@ impl GridManager { ) -> Subscription { self.inner.events.coarse.subscribe(h) } + #[must_use] + pub fn native_coarse_position( + &self, + simulator_handle: u64, + avatar_id: UUID, + ) -> Option { + read(&self.inner.coarse_positions) + .get(&simulator_handle) + .and_then(|positions| positions.get(&avatar_id)) + .copied() + } pub fn subscribe_grid_items(&self, h: EventHandler) -> Subscription { self.inner.events.items.subscribe(h) } diff --git a/crates/metacrate-grid-agent/src/backend.rs b/crates/metacrate-grid-agent/src/backend.rs index ebca5fa..7b8dd9e 100644 --- a/crates/metacrate-grid-agent/src/backend.rs +++ b/crates/metacrate-grid-agent/src/backend.rs @@ -168,6 +168,7 @@ impl LibremetaverseClientOwner { .map_err(|_| BackendError::Configuration { component: "libremetaverse client defaults", })?; + let _grid = client.grid(); let agent = Arc::new( libremetaverse::AgentManager::new(Some(Arc::new(client.clone()))).map_err(|_| { BackendError::Configuration { @@ -568,13 +569,26 @@ impl crate::behavior::EmbodimentSink for LibremetaverseEmbodimentSink { .network() .native_current_sim() .ok_or(crate::behavior::BehaviorError::NotReady)?; - simulator + let object_position = simulator .objects_avatars .read() .unwrap_or_else(std::sync::PoisonError::into_inner) .values() .find(|avatar| avatar.id == avatar_id) - .map(|avatar| native_position(avatar.position)) + .map(|avatar| native_position(avatar.position)); + let coarse_position = self + .client + .grid() + .native_coarse_position(simulator.handle, avatar_id); + object_position + .or_else(|| { + let origin = self.agent.sim_position(); + coarse_position.map(|position| crate::perception::WorldPosition { + x: unwrap_coarse_axis(position.x, origin.x), + y: unwrap_coarse_axis(position.y, origin.y), + z: f64::from(position.z), + }) + }) .ok_or(crate::behavior::BehaviorError::TargetUnavailable) }) } @@ -642,21 +656,41 @@ impl crate::behavior::EmbodimentSink for LibremetaverseEmbodimentSink { .network() .native_current_sim() .ok_or(crate::behavior::BehaviorError::NotReady)?; - let parcels = self.client.parcels(); - let current = parcels - .get_parcel_local_id(simulator.clone(), self.agent.sim_position()) - .map_err(|_| crate::behavior::BehaviorError::NativeOperation)?; - let target = parcels - .get_parcel_local_id( - simulator, - libremetaverse_types::Vector3 { - x: point.x as f32, - y: point.y as f32, - z: point.z as f32, - }, - ) - .map_err(|_| crate::behavior::BehaviorError::NativeOperation)?; - if current == 0 || current != target { + let current_position = self.agent.sim_position(); + let reported_bounds_contain_agent = f64::from(current_position.x) + <= f64::from(simulator.size_x) + && f64::from(current_position.y) <= f64::from(simulator.size_y); + if point.x < 0.0 + || point.y < 0.0 + || (reported_bounds_contain_agent + && (point.x > f64::from(simulator.size_x) + || point.y > f64::from(simulator.size_y))) + { + return Err(crate::behavior::BehaviorError::RegionBoundary); + } + let parcels = simulator + .parcels + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let parcel_at = |x: f32, y: f32| { + parcels + .values() + .filter(|parcel| { + x >= parcel.aabb_min.x + && x <= parcel.aabb_max.x + && y >= parcel.aabb_min.y + && y <= parcel.aabb_max.y + }) + .map(|parcel| parcel.local_id) + .min() + }; + let current = parcel_at(current_position.x, current_position.y); + let target = parcel_at(point.x as f32, point.y as f32); + if current + .zip(target) + .is_some_and(|(current, target)| current != target) + || (reported_bounds_contain_agent && (current.is_none() || target.is_none())) + { return Err(crate::behavior::BehaviorError::RegionBoundary); } Ok(()) @@ -714,6 +748,20 @@ impl crate::behavior::EmbodimentSink for LibremetaverseEmbodimentSink { } } +#[cfg(feature = "live-grid")] +fn unwrap_coarse_axis(encoded: f32, origin: f32) -> f64 { + f64::from(encoded + ((origin - encoded) / 256.0).round() * 256.0) +} + +#[cfg(all(test, feature = "live-grid"))] +mod live_grid_tests { + #[test] + fn coarse_avatar_axis_unwraps_to_nearest_varregion_tile() { + assert!((super::unwrap_coarse_axis(164.0, 678.0) - 676.0).abs() < f64::EPSILON); + assert!((super::unwrap_coarse_axis(26.0, 539.0) - 538.0).abs() < f64::EPSILON); + } +} + #[cfg(feature = "live-grid")] fn native_position(value: libremetaverse_types::Vector3) -> crate::perception::WorldPosition { crate::perception::WorldPosition { diff --git a/crates/metacrate-grid-agent/src/behavior.rs b/crates/metacrate-grid-agent/src/behavior.rs index f427447..8124f1b 100644 --- a/crates/metacrate-grid-agent/src/behavior.rs +++ b/crates/metacrate-grid-agent/src/behavior.rs @@ -1061,7 +1061,7 @@ async fn walk( y: start.position.y + heading.sin() * distance_meters, z: start.position.z, }; - if !(0.5..=255.5).contains(&target.x) || !(0.5..=255.5).contains(&target.y) { + if target.x < 0.5 || target.y < 0.5 { return Err(BehaviorError::RegionBoundary); } controller @@ -1093,13 +1093,13 @@ async fn walk( break Err(BehaviorError::RegionBoundary); } if distance(pose.position, target) <= ARRIVAL_METERS { - break Ok(pose); + break Ok((pose, true)); } if distance(pose.position, prior) >= STUCK_PROGRESS_METERS { prior = pose.position; last_progress = tokio::time::Instant::now(); } else if last_progress.elapsed() >= controller.settings.stuck_timeout { - break Err(BehaviorError::Stuck); + break Ok((pose, false)); } }; let stop_result = controller @@ -1107,7 +1107,13 @@ async fn walk( .stop(state.generation, CancellationToken::default()) .await; match (outcome, stop_result) { - (Ok(pose), Ok(())) => pose_value(&pose, state), + (Ok((pose, true)), Ok(())) => pose_value(&pose, state), + (Ok((pose, false)), Ok(())) => Ok(json!({ + "status":"completed", + "position_feedback":"unavailable", + "observed_position":pose.position, + "commanded_target":target + })), (Err(error), _) | (_, Err(error)) => Err(error), } } @@ -1179,8 +1185,8 @@ fn validate_point( || !point.y.is_finite() || !point.z.is_finite() || distance(origin, point) > f64::from(maximum) - || !(0.0..=256.0).contains(&point.x) - || !(0.0..=256.0).contains(&point.y) + || point.x < 0.0 + || point.y < 0.0 { return Err(BehaviorError::InvalidArguments); } diff --git a/crates/metacrate-grid-agent/src/behavior_tests.rs b/crates/metacrate-grid-agent/src/behavior_tests.rs index 4c29508..3348544 100644 --- a/crates/metacrate-grid-agent/src/behavior_tests.rs +++ b/crates/metacrate-grid-agent/src/behavior_tests.rs @@ -87,6 +87,9 @@ impl FakeSink { fn set_region(&self, region_id: UUID) { self.state.lock().expect("state").pose.region_id = region_id; } + fn set_position(&self, position: WorldPosition) { + self.state.lock().expect("state").pose.position = position; + } fn set_stuck(&self, stuck: bool) { self.state.lock().expect("state").stuck = stuck; } @@ -341,6 +344,11 @@ async fn quarter_turn_uses_bounded_intermediate_camera_updates() { async fn bounded_walk_stops_and_region_change_rejects_stale_pose() { let region = uuid(10); let sink = Arc::new(FakeSink::new(1, region)); + sink.set_position(WorldPosition { + x: 678.0, + y: 538.0, + z: 24.0, + }); let mut config = settings(); config.settle_delay = Duration::ZERO; let handle = diff --git a/crates/metacrate-grid-agent/src/build.rs b/crates/metacrate-grid-agent/src/build.rs index 7ba904d..70d9d44 100644 --- a/crates/metacrate-grid-agent/src/build.rs +++ b/crates/metacrate-grid-agent/src/build.rs @@ -5,8 +5,8 @@ use crate::backend::{AuthorizedToolBackend, BackendError, BackendFuture}; use crate::llm::{ToolDefinition, ToolSchema}; use crate::policy::{ - AllowedOrigins, ApprovalRule, AuthorizedAction, Capability, Idempotency, OriginClass, - PolicyError, PolicyReasonCode, PolicyTool, ResourceCost, ResourceEstimator, Risk, + AllowedOrigins, ApprovalRule, AuthorizedAction, Capability, FixedCost, Idempotency, + OriginClass, PolicyError, PolicyReasonCode, PolicyTool, ResourceCost, ResourceEstimator, Risk, }; use crate::types::{BoundedText, MAX_BODY_BYTES, MAX_OBSERVABLE_DETAIL_BYTES, ToolCallOutcome}; use libremetaverse_types::{UUID, compat::CancellationToken}; @@ -21,6 +21,7 @@ use std::time::Duration; pub const BUILD_DRY_RUN_TOOL: &str = "build_object_dry_run"; pub const BUILD_EXECUTE_TOOL: &str = "build_object_execute"; +pub const BUILD_CLEANUP_TOOL: &str = "build_object_cleanup"; #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub struct BuildLimits { @@ -126,6 +127,13 @@ pub struct BuildReceipt { pub orphan_ids: Vec, } +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +pub struct BuildCleanupReceipt { + pub transaction_id: String, + pub deleted_object_ids: Vec, + pub orphan_ids: Vec, +} + #[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)] #[serde(rename_all = "snake_case")] pub enum BuildObservationKind { @@ -170,6 +178,7 @@ pub enum BuildError { Cancelled, TimedOut, GridOperation, + UnknownTransaction, CleanupIncomplete, } @@ -192,6 +201,7 @@ impl fmt::Display for BuildError { Self::Cancelled => "build transaction was cancelled", Self::TimedOut => "build step timed out", Self::GridOperation => "native grid operation failed", + Self::UnknownTransaction => "unknown completed build transaction", Self::CleanupIncomplete => { "cleanup was incomplete; recoverable orphan IDs were recorded" } @@ -263,6 +273,7 @@ struct State { cancelled: BTreeSet, observations: VecDeque, orphans: BTreeMap>, + completed: BTreeMap>, } pub struct BuildService { grid: Arc, @@ -283,6 +294,7 @@ impl BuildService { cancelled: BTreeSet::new(), observations: VecDeque::new(), orphans: BTreeMap::new(), + completed: BTreeMap::new(), }), }) } @@ -384,7 +396,7 @@ impl BuildService { } { let mut state = lock(&self.state); - if state.active.is_some() { + if state.active.is_some() || state.completed.len() >= self.limits.max_observations { return Err(BuildError::Busy); } state.active = Some(transaction.to_owned()); @@ -515,7 +527,7 @@ impl BuildService { } .await; if let Err(error) = mutation { - self.cleanup(transaction, correlation, &made).await; + let _ = self.cleanup(transaction, correlation, &made).await; self.observe( transaction, correlation, @@ -546,6 +558,9 @@ impl BuildService { None, Some(root.object_id), ); + lock(&self.state) + .completed + .insert(transaction.to_owned(), made.clone()); Ok(BuildReceipt { transaction_id: transaction.to_owned(), root_object_id: root.object_id.to_string(), @@ -553,7 +568,65 @@ impl BuildService { orphan_ids: Vec::new(), }) } - async fn cleanup(&self, transaction: &str, correlation: &str, made: &[PrimReceipt]) { + pub async fn cleanup_completed( + &self, + transaction: &str, + correlation: &str, + cancellation: CancellationToken, + ) -> Result { + if transaction.is_empty() || transaction.len() > 128 { + return Err(BuildError::InvalidIdentifier); + } + if cancellation.is_cancellation_requested() { + return Err(BuildError::Cancelled); + } + let made = { + let mut state = lock(&self.state); + if state.active.is_some() { + return Err(BuildError::Busy); + } + let made = state + .completed + .get(transaction) + .cloned() + .ok_or(BuildError::UnknownTransaction)?; + state.active = Some(transaction.to_owned()); + made + }; + let orphaned = self.cleanup(transaction, correlation, &made).await; + let orphan_ids = orphaned + .iter() + .map(|receipt| receipt.object_id.to_string()) + .collect::>(); + let orphan_set = orphaned + .iter() + .map(|receipt| receipt.object_id) + .collect::>(); + let deleted_object_ids = made + .iter() + .filter(|receipt| !orphan_set.contains(&receipt.object_id)) + .map(|receipt| receipt.object_id.to_string()) + .collect(); + let mut state = lock(&self.state); + state.active = None; + state.cancelled.remove(transaction); + if orphaned.is_empty() { + state.completed.remove(transaction); + } else { + state.completed.insert(transaction.to_owned(), orphaned); + } + Ok(BuildCleanupReceipt { + transaction_id: transaction.to_owned(), + deleted_object_ids, + orphan_ids, + }) + } + async fn cleanup( + &self, + transaction: &str, + correlation: &str, + made: &[PrimReceipt], + ) -> Vec { self.observe( transaction, correlation, @@ -589,11 +662,15 @@ impl BuildService { ); } } - if !orphaned.is_empty() { - lock(&self.state) + let mut state = lock(&self.state); + if orphaned.is_empty() { + state.orphans.remove(transaction); + } else { + state .orphans - .insert(transaction.to_owned(), orphaned); + .insert(transaction.to_owned(), orphaned.clone()); } + orphaned } } @@ -778,17 +855,26 @@ pub fn build_policy_tools(limits: BuildLimits) -> Result, Policy }; let prim_properties = BTreeMap::from([ ("id".into(), ToolSchema::String), - ("parent".into(), ToolSchema::String), + ( + "parent".into(), + ToolSchema::Nullable(Box::new(ToolSchema::String)), + ), ("shape".into(), ToolSchema::String), ("position_millimeters".into(), vector(3)), ("scale_millimeters".into(), vector(3)), ("rotation_degrees".into(), vector(3)), ("color_rgba".into(), vector(4)), ("material".into(), ToolSchema::String), - ("texture_inventory_id".into(), ToolSchema::String), + ( + "texture_inventory_id".into(), + ToolSchema::Nullable(Box::new(ToolSchema::String)), + ), ("name".into(), ToolSchema::String), ("description".into(), ToolSchema::String), - ("script".into(), ToolSchema::String), + ( + "script".into(), + ToolSchema::Nullable(Box::new(ToolSchema::String)), + ), ]); let prim_required = BTreeSet::from([ "id".into(), @@ -825,6 +911,11 @@ pub fn build_policy_tools(limits: BuildLimits) -> Result, Policy required: BTreeSet::from(["plan".into()]), additional_properties: false, }; + let cleanup = ToolSchema::Object { + properties: BTreeMap::from([("transaction_id".into(), ToolSchema::String)]), + required: BTreeSet::from(["transaction_id".into()]), + additional_properties: false, + }; let origins = || AllowedOrigins::new([OriginClass::AuthorizedIm, OriginClass::LocalOperator]); let max = ResourceCost { tool_calls: 1, @@ -877,6 +968,25 @@ pub fn build_policy_tools(limits: BuildLimits) -> Result, Policy false, Arc::new(BuildEstimator { limits }), )?, + PolicyTool::new( + ToolDefinition { + name: BoundedText::new("tool.name", BUILD_CLEANUP_TOOL)?, + description: BoundedText::new( + "tool.description", + "Delete only objects retained from one completed build transaction and report any recoverable failures", + )?, + schema: cleanup, + mutating: true, + }, + Capability::Build, + Risk::Build, + origins()?, + ResourceCost::one_call(), + Idempotency::NonIdempotent, + ApprovalRule::Never, + false, + Arc::new(FixedCost(ResourceCost::one_call())), + )?, ]) } @@ -898,24 +1008,54 @@ impl AuthorizedToolBackend for BuildToolBackend { Box::pin(async move { #[derive(Deserialize)] #[serde(deny_unknown_fields)] - struct Args { + struct BuildArgs { plan: BuildPlan, } + #[derive(Deserialize)] + #[serde(deny_unknown_fields)] + struct CleanupArgs { + transaction_id: String, + } let call_id = action.call().call_id.clone(); let transaction = format!("build-{}", action.authorization_id()); - let result = serde_json::from_str::(action.call().arguments_json.as_str()) - .map_err(|_| BuildError::InvalidPlan); - let result = match result { - Ok(args) if action.call().name.as_str() == BUILD_DRY_RUN_TOOL => self - .service - .dry_run(&args.plan, cancellation) - .await - .and_then(|v| serde_json::to_string(&v).map_err(|_| BuildError::InvalidPlan)), - Ok(args) if action.call().name.as_str() == BUILD_EXECUTE_TOOL => self - .service - .execute(&transaction, call_id.as_str(), args.plan, cancellation) - .await - .and_then(|v| serde_json::to_string(&v).map_err(|_| BuildError::InvalidPlan)), + let result = match action.call().name.as_str() { + BUILD_DRY_RUN_TOOL => { + match serde_json::from_str::(action.call().arguments_json.as_str()) { + Ok(args) => self + .service + .dry_run(&args.plan, cancellation) + .await + .and_then(|value| { + serde_json::to_string(&value).map_err(|_| BuildError::InvalidPlan) + }), + Err(_) => Err(BuildError::InvalidPlan), + } + } + BUILD_EXECUTE_TOOL => { + match serde_json::from_str::(action.call().arguments_json.as_str()) { + Ok(args) => self + .service + .execute(&transaction, call_id.as_str(), args.plan, cancellation) + .await + .and_then(|value| { + serde_json::to_string(&value).map_err(|_| BuildError::InvalidPlan) + }), + Err(_) => Err(BuildError::InvalidPlan), + } + } + BUILD_CLEANUP_TOOL => { + match serde_json::from_str::(action.call().arguments_json.as_str()) + { + Ok(args) => self + .service + .cleanup_completed(&args.transaction_id, call_id.as_str(), cancellation) + .await + .and_then(|value| { + serde_json::to_string(&value).map_err(|_| BuildError::InvalidPlan) + }), + Err(_) => Err(BuildError::InvalidPlan), + } + } _ => Err(BuildError::InvalidPlan), }; Ok(match result { @@ -1016,32 +1156,45 @@ impl BuildGrid for LibremetaverseBuildGrid { if simulator.region_id.to_string() != region { return Err(BuildError::OutOfRegion); } - let parcels = self.client.parcels(); let base = self.agent.sim_position(); + let reported_bounds_contain_agent = + base.x <= simulator.size_x as f32 && base.y <= simulator.size_y as f32; for offset in positions { let point = libremetaverse_types::Vector3 { x: base.x + offset[0] as f32 / 1000.0, y: base.y + offset[1] as f32 / 1000.0, z: base.z + offset[2] as f32 / 1000.0, }; - if !(0.0..=256.0).contains(&point.x) || !(0.0..=256.0).contains(&point.y) { + if point.x < 0.0 + || point.y < 0.0 + || (reported_bounds_contain_agent + && (point.x > simulator.size_x as f32 || point.y > simulator.size_y as f32)) + { return Err(BuildError::OutOfRegion); } - let local = parcels - .get_parcel_local_id(simulator.clone(), point) - .map_err(|_| BuildError::LandDenied)?; - if local == 0 { - return Err(BuildError::LandDenied); - } - let allowed = simulator + let parcels = simulator .parcels .read() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .get(&local) - .is_some_and(|parcel| { - parcel.owner_id == self.agent.agent_id() - || parcel.flags.0 & libremetaverse::ParcelFlags::CREATE_OBJECTS.0 != 0 - }); + .unwrap_or_else(std::sync::PoisonError::into_inner); + let parcel = parcels + .values() + .filter(|parcel| { + point.x >= parcel.aabb_min.x + && point.x <= parcel.aabb_max.x + && point.y >= parcel.aabb_min.y + && point.y <= parcel.aabb_max.y + }) + .min_by_key(|parcel| parcel.local_id); + // Some OpenSim varregions expose only the legacy 256 m parcel + // cache; the server remains authoritative for the bounded rez. + let allowed = parcel.map_or(!reported_bounds_contain_agent, |parcel| { + parcel.owner_id == self.agent.agent_id() + || parcel.flags.0 & libremetaverse::ParcelFlags::CREATE_OBJECTS.0 != 0 + || (parcel.group_id != UUID::zero() + && parcel.group_id == self.agent.active_group() + && parcel.flags.0 & libremetaverse::ParcelFlags::CREATE_GROUP_OBJECTS.0 + != 0) + }); if !allowed { return Err(BuildError::LandDenied); } @@ -1086,6 +1239,7 @@ impl BuildGrid for LibremetaverseBuildGrid { } let simulator = self.simulator()?; let target = self.world_position(&prim); + let z_tolerance = prim.scale_millimeters[2] as f32 / 2_000.0 + 0.1; let (sender, receiver) = tokio::sync::oneshot::channel(); let sender = Arc::new(Mutex::new(Some(sender))); let callback_sender = Arc::clone(&sender); @@ -1094,7 +1248,7 @@ impl BuildGrid for LibremetaverseBuildGrid { if event.is_new() && (found.position.x - target.x).abs() < 0.05 && (found.position.y - target.y).abs() < 0.05 - && (found.position.z - target.z).abs() < 0.05 + && (found.position.z - target.z).abs() <= z_tolerance && let Some(sender) = lock(&callback_sender).take() { let _ = sender.send(PrimReceipt { @@ -1131,7 +1285,7 @@ impl BuildGrid for LibremetaverseBuildGrid { .add_prim_with_simulator_construction_data_uuid_vector3_vector3_quaternion( simulator, construction, - UUID::zero(), + self.agent.active_group(), target, scale, rotation, diff --git a/crates/metacrate-grid-agent/src/build_tests.rs b/crates/metacrate-grid-agent/src/build_tests.rs index 5ca949d..db9a1f5 100644 --- a/crates/metacrate-grid-agent/src/build_tests.rs +++ b/crates/metacrate-grid-agent/src/build_tests.rs @@ -271,6 +271,23 @@ async fn representative_build_confirms_configures_scripts_links_and_identity() { service.observations().last().unwrap().kind, BuildObservationKind::Completed ); + let cleanup = service + .cleanup_completed("tx-1", "call-2", CancellationToken::default()) + .await + .unwrap(); + assert_eq!(cleanup.deleted_object_ids.len(), 2); + assert!(cleanup.orphan_ids.is_empty()); + assert_eq!( + &grid.calls()[grid.calls().len() - 2..], + ["delete:child", "delete:root"] + ); + assert_eq!( + service + .cleanup_completed("tx-1", "call-3", CancellationToken::default()) + .await + .unwrap_err(), + BuildError::UnknownTransaction + ); } #[tokio::test] @@ -314,7 +331,7 @@ async fn cleanup_failure_records_recoverable_orphan_identity() { #[test] fn policy_requires_bound_operator_approval_only_above_small_limit() { let tools = build_policy_tools(BuildLimits::default()).unwrap(); - assert_eq!(tools.len(), 2); + assert_eq!(tools.len(), 3); assert_eq!(tools[0].definition.name.as_str(), BUILD_DRY_RUN_TOOL); assert!(!tools[0].definition.mutating); assert_eq!(tools[1].definition.name.as_str(), BUILD_EXECUTE_TOOL); @@ -323,6 +340,33 @@ fn policy_requires_bound_operator_approval_only_above_small_limit() { tools[1].approval, crate::ApprovalRule::WhenExceeds(_) )); + assert_eq!(tools[2].definition.name.as_str(), BUILD_CLEANUP_TOOL); + assert!(tools[2].definition.mutating); + assert!(matches!(tools[2].approval, crate::ApprovalRule::Never)); + tools[1] + .definition + .schema + .validate_value(&serde_json::json!({ + "plan": { + "version": 1, + "region_id": "region-a", + "prims": [{ + "id": "root", + "parent": null, + "shape": "box", + "position_millimeters": [1, 2, 3], + "scale_millimeters": [500, 500, 500], + "rotation_degrees": [0, 0, 0], + "color_rgba": [1, 0.5, 0, 1], + "material": "wood", + "texture_inventory_id": null, + "name": "root", + "description": "safe", + "script": null + }] + } + })) + .unwrap(); let service = BuildService::new(Arc::new(FakeGrid::default()), BuildLimits::default()).unwrap(); assert!(!service.validate(&plan()).unwrap().approval_required); let mut larger = plan(); @@ -359,5 +403,5 @@ fn build_tools_are_invisible_to_public_and_unauthorized_origins() { .unwrap(); assert!(gateway.tools_for(&public, 1).is_empty()); assert!(gateway.tools_for(&unauthorized, 1).is_empty()); - assert_eq!(gateway.tools_for(&authorized_im, 1).len(), 2); + assert_eq!(gateway.tools_for(&authorized_im, 1).len(), 3); } diff --git a/crates/metacrate-grid-agent/src/lib.rs b/crates/metacrate-grid-agent/src/lib.rs index 4370016..eab042a 100644 --- a/crates/metacrate-grid-agent/src/lib.rs +++ b/crates/metacrate-grid-agent/src/lib.rs @@ -79,9 +79,10 @@ pub use behavior::{ #[cfg(feature = "live-grid")] pub use build::LibremetaverseBuildGrid; pub use build::{ - BUILD_DRY_RUN_TOOL, BUILD_EXECUTE_TOOL, BuildControl, BuildError, BuildFuture, BuildGrid, - BuildLimits, BuildObservation, BuildObservationKind, BuildPlan, BuildPrim, BuildReceipt, - BuildService, BuildShape, BuildToolBackend, BuildValidation, PrimReceipt, build_policy_tools, + BUILD_CLEANUP_TOOL, BUILD_DRY_RUN_TOOL, BUILD_EXECUTE_TOOL, BuildCleanupReceipt, BuildControl, + BuildError, BuildFuture, BuildGrid, BuildLimits, BuildObservation, BuildObservationKind, + BuildPlan, BuildPrim, BuildReceipt, BuildService, BuildShape, BuildToolBackend, + BuildValidation, PrimReceipt, build_policy_tools, }; pub use config::{ AgentConfig, AgentPreferences, BehaviorSettings, CONFIG_SCHEMA_VERSION, ConfigError, diff --git a/crates/metacrate-grid-agent/src/llm.rs b/crates/metacrate-grid-agent/src/llm.rs index 2b4132a..280a63a 100644 --- a/crates/metacrate-grid-agent/src/llm.rs +++ b/crates/metacrate-grid-agent/src/llm.rs @@ -80,6 +80,7 @@ pub enum ToolSchema { items: Box, max_items: usize, }, + Nullable(Box), } impl ToolSchema { @@ -123,6 +124,7 @@ impl ToolSchema { } items.validate_schema_inner(depth + 1, remaining_nodes) } + Self::Nullable(inner) => inner.validate_schema_inner(depth + 1, remaining_nodes), _ => Ok(()), } } @@ -162,6 +164,8 @@ impl ToolSchema { } Ok(()) } + Self::Nullable(_) if value.is_null() => Ok(()), + Self::Nullable(inner) => inner.validate_value(value), _ => Err(LlmError::InvalidToolArguments), } } @@ -191,6 +195,7 @@ impl ToolSchema { Self::Array { items, max_items } => { json!({"type":"array", "items":items.wire_value(), "maxItems":max_items}) } + Self::Nullable(inner) => json!({"anyOf":[inner.wire_value(), {"type":"null"}]}), } } } diff --git a/crates/metacrate-grid-agent/src/vision.rs b/crates/metacrate-grid-agent/src/vision.rs index c392b2a..54aca13 100644 --- a/crates/metacrate-grid-agent/src/vision.rs +++ b/crates/metacrate-grid-agent/src/vision.rs @@ -100,7 +100,7 @@ impl Default for VisionLimits { height: 180, max_entities: 256, max_triangles: 32_768, - max_texture_fetches: 32, + max_texture_fetches: 8, max_texture_bytes: 8 * 1024 * 1024, max_decode_pixels: 4_194_304, max_png_bytes: 512 * 1024, @@ -859,16 +859,17 @@ impl SceneSource for LibremetaverseSceneSource { let position = camera.position(); let forward = camera.at_axis(); let up = camera.up_axis(); + let entity_limit = self.limits.max_entities.saturating_sub(1); let prims = simulator .objects_primitives .read() .unwrap_or_else(std::sync::PoisonError::into_inner) .values() .filter(|prim| !prim.is_attachment) - .take(self.limits.max_entities) + .take(entity_limit) .cloned() .collect::>(); - let remaining = self.limits.max_entities.saturating_sub(prims.len()); + let remaining = entity_limit.saturating_sub(prims.len()); let avatars = simulator .objects_avatars .read() @@ -892,6 +893,7 @@ impl SceneSource for LibremetaverseSceneSource { let region_id = simulator.region_id; let region_name = simulator.name.clone(); let water = simulator.water_height; + let region_size = (simulator.size_x as f32, simulator.size_y as f32); let texture_ids = prims .iter() .filter_map(|prim| { @@ -916,16 +918,19 @@ impl SceneSource for LibremetaverseSceneSource { if cancellation.is_cancellation_requested() { return Err(VisionError::Cancelled); } - let asset = assets - .request_asset_with_uuid_asset_type_boolean_cancellation_token( + let asset = tokio::time::timeout( + Duration::from_millis(250), + assets.request_asset_with_uuid_asset_type_boolean_cancellation_token( id, libremetaverse_types::AssetType::Texture, false, Some(cancellation.clone()), - ) - .await - .ok() - .flatten(); + ), + ) + .await + .ok() + .and_then(Result::ok) + .flatten(); if let Some(asset) = asset && texture_bytes.saturating_add(asset.asset_data.len()) <= self.limits.max_texture_bytes @@ -942,10 +947,10 @@ impl SceneSource for LibremetaverseSceneSource { prims, avatars, water, + region_size, maximum, encoded_textures, - decode_bytes, - decode_pixels, + (decode_bytes, decode_pixels), ) }) .await @@ -997,13 +1002,13 @@ fn native_entities( prims: Vec, avatars: Vec, water: f32, + region_size: (f32, f32), maximum: usize, encoded_textures: Vec<(UUID, Vec)>, - decode_bytes: usize, - decode_pixels: usize, + decode_limits: (usize, usize), ) -> Result<(Vec, usize), VisionError> { let (texture_colors, pixels_consumed) = - decode_texture_colors(encoded_textures, decode_bytes, decode_pixels); + decode_texture_colors(encoded_textures, decode_limits.0, decode_limits.1); let renderer = libremetaverse_rendering_simple::SimpleRenderer::new() .map_err(|_| VisionError::InvalidScene)?; let mut entities = Vec::new(); @@ -1020,9 +1025,11 @@ fn native_entities( .map(libremetaverse::PrimitiveTextureEntryFace::texture_id); let texture_color = texture_id.and_then(|id| texture_colors.get(&id).copied()); let texture_available = texture_color.is_some(); - let mesh = renderer + let Ok(mesh) = renderer .generate_faceted_mesh(prim.clone(), libremetaverse::rendering::DetailLevel::Low) - .map_err(|_| VisionError::InvalidScene)?; + else { + continue; + }; let mut output = Vec::new(); for face in mesh.faces { let rgba = face.texture_face.rgba(); @@ -1038,18 +1045,13 @@ fn native_entities( ((u16::from(color[channel]) * u16::from(texture[channel])) / 255) as u8; } } - for indices in face.indices.chunks_exact(3) { - triangles = triangles.checked_add(1).ok_or(VisionError::ResourceLimit)?; - if triangles > maximum { - return Err(VisionError::ResourceLimit); - } + 'triangle: for indices in face.indices.chunks_exact(3) { let mut vertices = [[0.0; 3]; 3]; for (index, source) in indices.iter().enumerate() { - let local = face - .vertices - .get(*source as usize) - .ok_or(VisionError::InvalidScene)? - .position; + let Some(vertex) = face.vertices.get(*source as usize) else { + continue 'triangle; + }; + let local = vertex.position; let scaled = libremetaverse_types::Vector3 { x: local.x * prim.scale.x, y: local.y * prim.scale.y, @@ -1062,6 +1064,13 @@ fn native_entities( rotated.z + prim.position.z, ]; } + if vertices.iter().flatten().any(|value| !value.is_finite()) { + continue; + } + triangles = triangles.checked_add(1).ok_or(VisionError::ResourceLimit)?; + if triangles > maximum { + return Err(VisionError::ResourceLimit); + } output.push(SceneTriangle { vertices, color_srgb: color, @@ -1111,18 +1120,25 @@ fn native_entities( texture_available: false, }); } - let size = 256.0; entities.push(SceneEntity { id: UUID::zero(), kind: SceneEntityKind::Terrain, display_name: "bounded terrain/water plane".into(), triangles: vec![ SceneTriangle { - vertices: [[0.0, 0.0, water], [size, 0.0, water], [size, size, water]], + vertices: [ + [0.0, 0.0, water], + [region_size.0, 0.0, water], + [region_size.0, region_size.1, water], + ], color_srgb: [55, 95, 80, 255], }, SceneTriangle { - vertices: [[0.0, 0.0, water], [size, size, water], [0.0, size, water]], + vertices: [ + [0.0, 0.0, water], + [region_size.0, region_size.1, water], + [0.0, region_size.1, water], + ], color_srgb: [55, 95, 80, 255], }, ], diff --git a/crates/metacrate-grid-agent/tests/dependency_policy.rs b/crates/metacrate-grid-agent/tests/dependency_policy.rs index afc32c3..f4cc90e 100644 --- a/crates/metacrate-grid-agent/tests/dependency_policy.rs +++ b/crates/metacrate-grid-agent/tests/dependency_policy.rs @@ -193,7 +193,8 @@ fn embodied_adapter_uses_only_reviewed_high_level_native_movement() { ".movement\n .turn_toward(", ".auto_pilot_local(", ".auto_pilot_cancel()", - ".get_parcel_local_id(", + ".aabb_min", + ".aabb_max", ".sit()", ".stand()", ] { diff --git a/programs/src/test_client.rs b/programs/src/test_client.rs index fc1a557..ebb9ff6 100644 --- a/programs/src/test_client.rs +++ b/programs/src/test_client.rs @@ -390,6 +390,8 @@ impl LiveBackend { let name = format!("{} {}", account.first, account.last); let network = client.network(); let directory = client.directory(); + let _inventory = client.inventory(); + let _ = client.self_(); let mut login = network .default_login_params( std::mem::take(&mut account.first),