diff --git a/Cargo.lock b/Cargo.lock index 23b5a4f..5e7296c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2270,10 +2270,14 @@ dependencies = [ name = "metacrate-grid-agent" version = "0.0.1" dependencies = [ + "base64", "crossterm", "libremetaverse", + "libremetaverse-imaging", + "libremetaverse-rendering-simple", "libremetaverse-types", "metacrate-lsl-tools", + "png", "reqwest", "rustls", "serde", diff --git a/crates/metacrate-grid-agent/Cargo.toml b/crates/metacrate-grid-agent/Cargo.toml index 1252ed5..b44eb21 100644 --- a/crates/metacrate-grid-agent/Cargo.toml +++ b/crates/metacrate-grid-agent/Cargo.toml @@ -9,10 +9,14 @@ description = "Bounded pure-Rust OpenSim grid-agent service foundation" publish = false [dependencies] +base64 = "0.22" crossterm = "0.29" libremetaverse = { version = "0.0.1", path = "../libremetaverse", default-features = false, optional = true } +libremetaverse-imaging = { version = "0.0.1", path = "../libremetaverse-imaging", default-features = false, features = ["rust-j2k"], optional = true } +libremetaverse-rendering-simple = { version = "0.0.1", path = "../libremetaverse-rendering-simple", optional = true } libremetaverse-types = { version = "0.0.1", path = "../libremetaverse-types" } metacrate-lsl-tools = { version = "0.0.1", path = "../metacrate-lsl-tools" } +png = "0.17" reqwest = { version = "0.13.4", default-features = false, features = ["rustls"] } rustls = { version = "0.23.43", default-features = false, features = ["aws_lc_rs", "std", "tls12"] } serde = { version = "1", features = ["derive"] } @@ -31,7 +35,7 @@ tokio = { version = "1.53.1", features = ["io-util", "net", "rt-multi-thread", " [features] default = [] -live-grid = ["dep:libremetaverse"] +live-grid = ["dep:libremetaverse", "dep:libremetaverse-imaging", "dep:libremetaverse-rendering-simple"] [lints] workspace = true diff --git a/crates/metacrate-grid-agent/src/control_plane.rs b/crates/metacrate-grid-agent/src/control_plane.rs index d537c49..7bd6349 100644 --- a/crates/metacrate-grid-agent/src/control_plane.rs +++ b/crates/metacrate-grid-agent/src/control_plane.rs @@ -202,6 +202,9 @@ pub struct RuntimeView { pub active_build_transaction: Option, pub build_progress: Option, pub build_orphan_ids: Vec, + pub active_visual_capture: Option, + pub visual_progress: Option, + pub visual_image_sha256: Option, } #[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] diff --git a/crates/metacrate-grid-agent/src/control_plane_tests.rs b/crates/metacrate-grid-agent/src/control_plane_tests.rs index 072d699..7b66c6f 100644 --- a/crates/metacrate-grid-agent/src/control_plane_tests.rs +++ b/crates/metacrate-grid-agent/src/control_plane_tests.rs @@ -56,6 +56,9 @@ impl ControlTarget for FakeTarget { active_build_transaction: None, build_progress: None, build_orphan_ids: Vec::new(), + active_visual_capture: None, + visual_progress: None, + visual_image_sha256: None, })), ControlRequest::ListSessions { page } => Ok(ControlPayload::Sessions(page_values( &page, diff --git a/crates/metacrate-grid-agent/src/control_runtime.rs b/crates/metacrate-grid-agent/src/control_runtime.rs index 67dd850..6125610 100644 --- a/crates/metacrate-grid-agent/src/control_runtime.rs +++ b/crates/metacrate-grid-agent/src/control_runtime.rs @@ -85,6 +85,7 @@ pub struct AgentControlTarget { audit: Arc, observability: Mutex>>, build: Mutex>>, + vision: Mutex>>, behavior: BehaviorIngress, commands: mpsc::Sender, command_capacity: usize, @@ -129,6 +130,7 @@ impl AgentControlTarget { audit, observability: Mutex::new(None), build: Mutex::new(None), + vision: Mutex::new(None), behavior, commands, command_capacity, @@ -149,6 +151,11 @@ impl AgentControlTarget { *lock(&self.build) = Some(build); } + /// Attaches synthetic viewport cancellation and metadata to operator control. + pub fn attach_vision_control(&self, vision: Arc) { + *lock(&self.vision) = Some(vision); + } + pub fn update_session(&self, session: SessionStatus) { let mut state = lock(&self.state); state.session = session; @@ -245,6 +252,7 @@ impl AgentControlTarget { let state = lock(&self.state).clone(); let usage = self.policy.global_budget_usage(); let build = lock(&self.build); + let vision = lock(&self.vision); Ok(ControlPayload::Runtime(RuntimeView { grid_state: state.session.state.as_str().to_owned(), generation: state.session.generation, @@ -265,6 +273,13 @@ impl AgentControlTarget { build_orphan_ids: build .as_ref() .map_or_else(Vec::new, |build| build.build_orphans()), + active_visual_capture: vision + .as_ref() + .and_then(|vision| vision.active_capture()), + visual_progress: vision.as_ref().and_then(|vision| vision.capture_progress()), + visual_image_sha256: vision + .as_ref() + .and_then(|vision| vision.last_image_sha256()), })) } ControlRequest::ListSessions { page } => { @@ -345,6 +360,12 @@ impl AgentControlTarget { ControlRequest::PauseAutonomy => { let response = self.enqueue("pause", RuntimeControlCommand::Pause)?; self.behavior.pause(); + let vision = lock(&self.vision).clone(); + if let Some((vision, active)) = + vision.and_then(|vision| vision.active_capture().map(|active| (vision, active))) + { + let _ = vision.cancel_capture(&active); + } Ok(response) } ControlRequest::ResumeAutonomy => { @@ -356,10 +377,13 @@ impl AgentControlTarget { let build_cancelled = lock(&self.build) .as_ref() .is_some_and(|build| build.cancel_build(&action_id)); + let vision_cancelled = lock(&self.vision) + .as_ref() + .is_some_and(|vision| vision.cancel_capture(&action_id)); let cancelled = self.behavior.cancel_action(&action_id).map_err(|_| { control_error(ControlErrorCode::InvalidRequest, "invalid action ID", false) })?; - if !cancelled && !build_cancelled { + if !cancelled && !build_cancelled && !vision_cancelled { return Err(control_error( ControlErrorCode::NotFound, "active behavior action not found", diff --git a/crates/metacrate-grid-agent/src/control_runtime_tests.rs b/crates/metacrate-grid-agent/src/control_runtime_tests.rs index d036958..d0b3cf7 100644 --- a/crates/metacrate-grid-agent/src/control_runtime_tests.rs +++ b/crates/metacrate-grid-agent/src/control_runtime_tests.rs @@ -7,15 +7,39 @@ use crate::control_runtime::{AgentControlTarget, RuntimeControlCommand}; use crate::conversation::{ConversationLimits, ConversationStore}; use crate::perception::WorldPosition; use crate::policy::{MemoryPolicyAudit, PolicyAuditSink, PolicyGateway, PolicyLimits}; -use crate::{AgentConfig, EmbodiedPose, Observability, ObservabilityLimits, SecretString}; +use crate::{ + AgentConfig, EmbodiedPose, Observability, ObservabilityLimits, SecretString, VisionControl, +}; use libremetaverse_types::UUID; use libremetaverse_types::compat::CancellationToken; use std::collections::BTreeSet; -use std::sync::Arc; +use std::sync::{Arc, Mutex}; use std::time::Duration; struct Sink; +struct FakeVision(Mutex>); +impl crate::VisionControl for FakeVision { + fn cancel_capture(&self, correlation_id: &str) -> bool { + let mut active = self.0.lock().unwrap(); + if active.as_deref() == Some(correlation_id) { + *active = None; + true + } else { + false + } + } + fn active_capture(&self) -> Option { + self.0.lock().unwrap().clone() + } + fn capture_progress(&self) -> Option { + self.active_capture().map(|_| "rendering".into()) + } + fn last_image_sha256(&self) -> Option { + Some("abc123".into()) + } +} + impl EmbodimentSink for Sink { fn current_pose( &self, @@ -99,6 +123,7 @@ fn envelope(id: &str, request: ControlRequest) -> ControlRequestEnvelope { } #[tokio::test] +#[allow(clippy::too_many_lines)] async fn production_target_projects_state_and_routes_real_mutations() { let settings = AgentConfig::offline("https://llm.invalid/chat", "llm-key") .expect("offline config") @@ -125,6 +150,8 @@ async fn production_target_projects_state_and_routes_real_mutations() { .expect("target"); let observability = Observability::memory(ObservabilityLimits::default()).expect("events"); target.attach_observability(observability); + let vision = Arc::new(FakeVision(Mutex::new(Some("visual-1".into())))); + target.attach_vision_control(vision.clone()); target.update_region( Some("00000000-0000-4000-8000-000000000001".into()), Some("Test Region".into()), @@ -146,6 +173,9 @@ async fn production_target_projects_state_and_routes_real_mutations() { }; assert_eq!(runtime.region_name.as_deref(), Some("Test Region")); assert_eq!(runtime.control_queue_capacity, 8); + assert_eq!(runtime.active_visual_capture.as_deref(), Some("visual-1")); + assert_eq!(runtime.visual_progress.as_deref(), Some("rendering")); + assert_eq!(runtime.visual_image_sha256.as_deref(), Some("abc123")); let metrics = operator .request(envelope("metrics", ControlRequest::Metrics)) .await; @@ -165,6 +195,7 @@ async fn production_target_projects_state_and_routes_real_mutations() { commands.recv().await, Some(RuntimeControlCommand::Pause) )); + assert!(vision.active_capture().is_none(), "pause cancels capture"); let observer = plane.connect("observer-token").expect("observer"); let denied = observer diff --git a/crates/metacrate-grid-agent/src/interaction.rs b/crates/metacrate-grid-agent/src/interaction.rs index 1beea77..a091d63 100644 --- a/crates/metacrate-grid-agent/src/interaction.rs +++ b/crates/metacrate-grid-agent/src/interaction.rs @@ -296,6 +296,7 @@ pub enum InteractionModelError { Timeout, Failed, PolicyRejected, + MultimodalUnsupported, } impl fmt::Display for InteractionModelError { @@ -305,6 +306,7 @@ impl fmt::Display for InteractionModelError { Self::Timeout => "interaction model request timed out", Self::Failed => "interaction model request failed", Self::PolicyRejected => "interaction request was rejected by policy", + Self::MultimodalUnsupported => "configured endpoint does not support visual context", }) } } @@ -1152,7 +1154,9 @@ async fn process_batch( Ok(_) => InferenceOutcome::Completed, Err(InteractionModelError::Cancelled) => InferenceOutcome::Cancelled, Err(InteractionModelError::Timeout) => InferenceOutcome::TimedOut, - Err(InteractionModelError::Failed) => InferenceOutcome::Failed, + Err(InteractionModelError::Failed | InteractionModelError::MultimodalUnsupported) => { + InferenceOutcome::Failed + } Err(InteractionModelError::PolicyRejected) => InferenceOutcome::PolicyRejected, }; observe( @@ -1249,6 +1253,7 @@ async fn process_batch( InteractionModelError::Cancelled | InteractionModelError::Failed => { DeliveryOutcome::Failed } + InteractionModelError::MultimodalUnsupported => DeliveryOutcome::Failed, }; let delivered = if error == InteractionModelError::Cancelled { delivery_observation( @@ -1822,6 +1827,9 @@ impl InteractionResponder for PolicyLlmResponder { crate::tool_loop::ToolLoopError::WallClockTimeout => { InteractionModelError::Timeout } + crate::tool_loop::ToolLoopError::Transport( + crate::llm::LlmError::MultimodalUnsupported, + ) => InteractionModelError::MultimodalUnsupported, _ => InteractionModelError::Failed, })?; visible_text(&outcome.final_message) diff --git a/crates/metacrate-grid-agent/src/lib.rs b/crates/metacrate-grid-agent/src/lib.rs index 8649652..190bbcf 100644 --- a/crates/metacrate-grid-agent/src/lib.rs +++ b/crates/metacrate-grid-agent/src/lib.rs @@ -23,6 +23,7 @@ pub mod session; pub mod tool_loop; pub mod tui; pub mod types; +pub mod vision; #[cfg(test)] mod behavior_tests; @@ -50,6 +51,8 @@ mod script_delivery_tests; mod session_tests; #[cfg(test)] mod tui_tests; +#[cfg(test)] +mod vision_tests; pub use backend::{ AuthorizedToolBackend, BackendError, BackendFuture, GridBackend, OfflineGridBackend, @@ -172,3 +175,11 @@ pub use types::{ GridEvent, GridEventKind, LlmRequest, LlmResult, MessageRole, ObservableEvent, ProposedToolCall, ToolCallOutcome, }; +#[cfg(feature = "live-grid")] +pub use vision::LibremetaverseSceneSource; +pub use vision::{ + CameraPose, SceneEntity, SceneEntityKind, SceneSnapshot, SceneSource, SceneTriangle, + SnapshotCompleteness, VisionAugmentedResponder, VisionCapture, VisionControl, VisionError, + VisionFuture, VisionLimits, VisionObservation, VisionObservationKind, VisionService, + visual_question, +}; diff --git a/crates/metacrate-grid-agent/src/llm.rs b/crates/metacrate-grid-agent/src/llm.rs index 748dec5..9942f34 100644 --- a/crates/metacrate-grid-agent/src/llm.rs +++ b/crates/metacrate-grid-agent/src/llm.rs @@ -301,6 +301,7 @@ pub enum LlmError { HttpStatus(u16), MalformedJson, UnsupportedResponse, + MultimodalUnsupported, DuplicateToolCallId, } @@ -320,6 +321,9 @@ impl fmt::Display for LlmError { Self::HttpStatus(status) => write!(formatter, "LLM endpoint returned HTTP {status}"), Self::MalformedJson => formatter.write_str("LLM endpoint returned malformed JSON"), Self::UnsupportedResponse => formatter.write_str("unsupported LLM response shape"), + Self::MultimodalUnsupported => { + formatter.write_str("LLM endpoint does not support image input") + } Self::DuplicateToolCallId => { formatter.write_str("LLM response repeated a tool-call ID") } @@ -390,6 +394,13 @@ impl LlmClient { for tool in tools { tool.validate()?; } + let has_image = messages.iter().any(|message| { + message + .content + .as_slice() + .iter() + .any(|part| matches!(part, ContentPart::Image { .. })) + }); let body = request_body(messages, tools)?; if body.len() > self.limits.max_prompt_bytes { return Err(LlmError::PromptTooLarge); @@ -411,9 +422,14 @@ impl LlmClient { drop(permit); result }; - tokio::time::timeout(self.limits.total_timeout, operation) + let result = tokio::time::timeout(self.limits.total_timeout, operation) .await - .map_err(|_| LlmError::Timeout)? + .map_err(|_| LlmError::Timeout)?; + if has_image && matches!(result, Err(LlmError::HttpStatus(400 | 415 | 422))) { + Err(LlmError::MultimodalUnsupported) + } else { + result + } } async fn complete_with_retries( diff --git a/crates/metacrate-grid-agent/src/main.rs b/crates/metacrate-grid-agent/src/main.rs index 10d69ca..bdd9a12 100644 --- a/crates/metacrate-grid-agent/src/main.rs +++ b/crates/metacrate-grid-agent/src/main.rs @@ -224,6 +224,7 @@ async fn run_live( config.limits.observable_queue, )? .start(); + live.vision.set_generation(handle.status().generation); if run_once { let readiness = tokio::time::timeout(config.timeouts.startup, async { @@ -232,19 +233,21 @@ async fn run_live( Some(event @ SessionObservation::Transition { status, .. }) if status.agent_ready => { + live.vision.set_generation(status.generation); record_session_observation(&live.observability, &event); return Ok::<(), CliError>(()); } Some( event @ SessionObservation::Transition { status: - metacrate_grid_agent::SessionStatus { + status @ metacrate_grid_agent::SessionStatus { state: SessionState::AuthenticationBlocked, .. }, .. }, ) => { + live.vision.set_generation(status.generation); record_session_observation(&live.observability, &event); return Err(CliError( "grid authentication/configuration requires operator action".into(), @@ -292,6 +295,7 @@ async fn run_live( )?; control_target.attach_observability(live.observability.clone()); control_target.attach_build_control(live.build_control.clone()); + control_target.attach_vision_control(live.vision.clone()); control_target.update_session(handle.status()); let erased_target: Arc = control_target.clone(); let (control_plane, integrated_client, control_server) = match config.mode { @@ -360,6 +364,7 @@ async fn run_live( let Some(event) = event else { break; }; record_session_observation(&live.observability, &event); if let SessionObservation::Transition { status, reason, retry_in } = event { + live.vision.set_generation(status.generation); control_target.update_session(status); live.landmark_roaming .update_pause(|pause| pause.degraded = !status.agent_ready); @@ -492,6 +497,8 @@ struct LiveInteractions { _landmark_intake: metacrate_grid_agent::LibremetaverseLandmarkIntake, landmark_roaming: metacrate_grid_agent::LandmarkRoamingHandle, build_control: Arc, + vision: + Arc>, } #[cfg(feature = "live-grid")] @@ -504,12 +511,13 @@ fn start_live_interactions( AuthorizedBackendRouter, AuthorizedToolBackend, BehaviorBackend, BehaviorController, BuildLimits, BuildService, BuildToolBackend, ConversationStore, InteractionCoordinator, LandmarkLimits, LandmarkService, LandmarkToolBackend, LibremetaverseBuildGrid, - LibremetaverseLandmarkGrid, LibremetaverseLandmarkIntake, LibremetaverseScriptInventory, - LlmClient, LlmTransportLimits, MemoryPolicyAudit, Observability, ObservabilityLimits, - PerceptionBackend, PolicyAuditSink, PolicyGateway, PolicyLimits, PolicyLlmResponder, - ScriptDeliveryBackend, ScriptDeliverySettings, SystemRoamingRandom, ToolLoopLimits, - UnifiedPolicyAudit, behavior_policy_tools, build_policy_tools, landmark_policy_tools, - perception_policy_tools, script_delivery_policy_tool, + LibremetaverseLandmarkGrid, LibremetaverseLandmarkIntake, LibremetaverseSceneSource, + LibremetaverseScriptInventory, LlmClient, LlmTransportLimits, MemoryPolicyAudit, + Observability, ObservabilityLimits, PerceptionBackend, PolicyAuditSink, PolicyGateway, + PolicyLimits, PolicyLlmResponder, ScriptDeliveryBackend, ScriptDeliverySettings, + SystemRoamingRandom, ToolLoopLimits, UnifiedPolicyAudit, VisionAugmentedResponder, + VisionLimits, VisionService, behavior_policy_tools, build_policy_tools, + landmark_policy_tools, perception_policy_tools, script_delivery_policy_tool, }; let transport_limits = LlmTransportLimits { @@ -635,6 +643,12 @@ fn start_live_interactions( loop_limits, now, )?); + let vision_limits = VisionLimits::default(); + let vision = Arc::new(VisionService::new( + Arc::new(LibremetaverseSceneSource::new(owner, vision_limits)), + vision_limits, + )?); + let responder = Arc::new(VisionAugmentedResponder::new(vision.clone(), responder)); let sink = Arc::new(owner.interaction_sink()); let pacer: Arc = Arc::new(behavior_ingress); let interaction = InteractionCoordinator::new( @@ -662,6 +676,7 @@ fn start_live_interactions( _landmark_intake: landmark_intake, landmark_roaming, build_control, + vision, }) } diff --git a/crates/metacrate-grid-agent/src/tui.rs b/crates/metacrate-grid-agent/src/tui.rs index 036395f..93f7823 100644 --- a/crates/metacrate-grid-agent/src/tui.rs +++ b/crates/metacrate-grid-agent/src/tui.rs @@ -717,6 +717,14 @@ fn render_overview(s: &OperatorSnapshot, out: &mut Vec) { v.build_orphan_ids.len() )); } + if v.active_visual_capture.is_some() || v.visual_image_sha256.is_some() { + out.push(format!( + "vision={} progress={} last-image-sha256={}", + v.active_visual_capture.as_deref().unwrap_or("idle"), + v.visual_progress.as_deref().unwrap_or("none"), + v.visual_image_sha256.as_deref().unwrap_or("none") + )); + } } } fn render_queues(s: &OperatorSnapshot, out: &mut Vec) { diff --git a/crates/metacrate-grid-agent/src/tui_tests.rs b/crates/metacrate-grid-agent/src/tui_tests.rs index 3654179..b926d52 100644 --- a/crates/metacrate-grid-agent/src/tui_tests.rs +++ b/crates/metacrate-grid-agent/src/tui_tests.rs @@ -46,6 +46,9 @@ impl TuiTransport for FakeTransport { active_build_transaction: Some("build-7".into()), build_progress: Some("configured".into()), build_orphan_ids: Vec::new(), + active_visual_capture: None, + visual_progress: None, + visual_image_sha256: None, }), ControlRequest::ListSessions { .. } => ControlPayload::Sessions(crate::Page { items: Vec::new(), diff --git a/crates/metacrate-grid-agent/src/vision.rs b/crates/metacrate-grid-agent/src/vision.rs new file mode 100644 index 0000000..c392b2a --- /dev/null +++ b/crates/metacrate-grid-agent/src/vision.rs @@ -0,0 +1,1206 @@ +//! Bounded deterministic synthetic viewport rendering and capture ownership. + +#![allow(clippy::missing_errors_doc)] +#![allow( + clippy::cast_possible_truncation, + clippy::cast_possible_wrap, + clippy::cast_precision_loss, + clippy::cast_sign_loss +)] + +use base64::Engine as _; +use libremetaverse_types::{ + UUID, + compat::{CancellationToken, CancellationTokenSource}, +}; +use serde::Serialize; +use sha2::{Digest, Sha256}; +use std::fmt::{self, Write as _}; +use std::future::Future; +use std::pin::Pin; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; +use tokio::sync::Semaphore; + +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct CameraPose { + pub position: [f32; 3], + pub forward: [f32; 3], + pub up: [f32; 3], + pub vertical_fov_degrees: f32, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum SceneEntityKind { + Object, + Resident, + Terrain, + Environment, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct SceneTriangle { + pub vertices: [[f32; 3]; 3], + pub color_srgb: [u8; 4], +} + +#[derive(Clone, Debug, PartialEq)] +pub struct SceneEntity { + pub id: UUID, + pub kind: SceneEntityKind, + pub display_name: String, + pub triangles: Vec, + pub texture_available: bool, +} + +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Serialize)] +pub struct SnapshotCompleteness { + pub objects_truncated: bool, + pub avatars_truncated: bool, + pub textures_missing: u32, + pub terrain_available: bool, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct SceneSnapshot { + pub generation: u64, + pub observed_unix_millis: u64, + pub region_id: UUID, + pub region_name: String, + pub camera: CameraPose, + pub entities: Vec, + pub completeness: SnapshotCompleteness, + /// Work already performed by the scene source; validated again before rendering. + pub texture_fetches: usize, + pub texture_bytes: usize, + pub decoded_texture_pixels: usize, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct VisionLimits { + pub width: u32, + pub height: u32, + pub max_entities: usize, + pub max_triangles: usize, + pub max_texture_fetches: usize, + pub max_texture_bytes: usize, + pub max_decode_pixels: usize, + pub max_png_bytes: usize, + pub max_concurrent_captures: usize, + pub capture_timeout: Duration, + pub minimum_interval: Duration, +} + +impl Default for VisionLimits { + fn default() -> Self { + Self { + width: 320, + height: 180, + max_entities: 256, + max_triangles: 32_768, + max_texture_fetches: 32, + max_texture_bytes: 8 * 1024 * 1024, + max_decode_pixels: 4_194_304, + max_png_bytes: 512 * 1024, + max_concurrent_captures: 1, + capture_timeout: Duration::from_secs(10), + minimum_interval: Duration::from_secs(5), + } + } +} +impl VisionLimits { + fn valid(self) -> bool { + (64..=640).contains(&self.width) + && (64..=480).contains(&self.height) + && self.width as usize * self.height as usize <= 307_200 + && (1..=512).contains(&self.max_entities) + && (1..=65_536).contains(&self.max_triangles) + && self.max_texture_fetches <= 64 + && self.max_texture_bytes <= 16 * 1024 * 1024 + && self.max_decode_pixels <= 8_388_608 + && (4_096..=1024 * 1024).contains(&self.max_png_bytes) + && (1..=4).contains(&self.max_concurrent_captures) + && !self.capture_timeout.is_zero() + && self.capture_timeout <= Duration::from_secs(30) + && self.minimum_interval <= Duration::from_mins(10) + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum VisionError { + UnsafeLimits, + NotExplicit, + Unauthorized, + RateLimited, + Busy, + Cancelled, + Superseded, + TimedOut, + StaleGeneration, + InvalidScene, + ResourceLimit, + Encode, + EndpointUnsupported, +} +impl fmt::Display for VisionError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { + Self::UnsafeLimits => "unsafe viewport limits", + Self::NotExplicit => "message does not explicitly request visual context", + Self::Unauthorized => "visual capture origin is not authorized", + Self::RateLimited => "visual capture rate limit reached", + Self::Busy => "visual capture capacity is exhausted", + Self::Cancelled => "visual capture was cancelled", + Self::Superseded => "visual capture was superseded", + Self::TimedOut => "visual capture timed out", + Self::StaleGeneration => "visual capture belongs to a stale simulator generation", + Self::InvalidScene => "scene data is invalid", + Self::ResourceLimit => "visual capture exceeded a resource limit", + Self::Encode => "viewport image encoding failed", + Self::EndpointUnsupported => "configured endpoint rejected multimodal input", + }) + } +} +impl std::error::Error for VisionError {} + +pub type VisionFuture<'a, T> = Pin> + Send + 'a>>; +pub trait SceneSource: Send + Sync + 'static { + fn capture_scene( + &self, + generation: u64, + cancellation: CancellationToken, + ) -> VisionFuture<'_, SceneSnapshot>; +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum VisionObservationKind { + Started, + SceneCaptured, + Rendered, + Completed, + Cancelled, + Failed, +} +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +pub struct VisionObservation { + pub correlation_id: String, + pub generation: u64, + pub kind: VisionObservationKind, + pub image_sha256: Option, + pub detail: String, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct VisionCapture { + pub png: Vec, + pub data_url: String, + pub image_sha256: String, + pub summary: String, + pub width: u32, + pub height: u32, + pub generation: u64, + pub observed_unix_millis: u64, + pub completeness: SnapshotCompleteness, +} + +struct VisionState { + next_allowed: Instant, + active: Option<(String, CancellationTokenSource)>, + observations: Vec, +} +pub struct VisionService { + source: Arc, + limits: VisionLimits, + semaphore: Semaphore, + request_sequence: AtomicU64, + current_generation: AtomicU64, + state: Mutex, +} + +impl VisionService { + pub fn new(source: Arc, limits: VisionLimits) -> Result { + if !limits.valid() { + return Err(VisionError::UnsafeLimits); + } + Ok(Self { + source, + limits, + semaphore: Semaphore::new(limits.max_concurrent_captures), + request_sequence: AtomicU64::new(0), + current_generation: AtomicU64::new(0), + state: Mutex::new(VisionState { + next_allowed: Instant::now(), + active: None, + observations: Vec::new(), + }), + }) + } + pub fn set_generation(&self, generation: u64) { + if self.current_generation.swap(generation, Ordering::AcqRel) != generation { + self.cancel_active(); + } + } + #[must_use] + pub fn generation(&self) -> u64 { + self.current_generation.load(Ordering::Acquire) + } + pub fn cancel_active(&self) -> bool { + lock(&self.state).active.take().is_some_and(|(_, source)| { + source.cancel(); + true + }) + } + #[must_use] + pub fn observations(&self) -> Vec { + lock(&self.state).observations.clone() + } + fn observe( + &self, + correlation: &str, + generation: u64, + kind: VisionObservationKind, + hash: Option, + detail: &str, + ) { + let mut state = lock(&self.state); + if state.observations.len() == 256 { + state.observations.remove(0); + } + state.observations.push(VisionObservation { + correlation_id: correlation.to_owned(), + generation, + kind, + image_sha256: hash, + detail: detail.to_owned(), + }); + } + pub async fn capture( + &self, + correlation: &str, + generation: u64, + cancellation: CancellationToken, + ) -> Result { + if generation == 0 || generation != self.current_generation.load(Ordering::Acquire) { + return Err(VisionError::StaleGeneration); + } + let owned = CancellationTokenSource::new(); + { + let mut state = lock(&self.state); + if Instant::now() < state.next_allowed { + return Err(VisionError::RateLimited); + } + if let Some((_, previous)) = state.active.take() { + previous.cancel(); + } + state.next_allowed = Instant::now() + self.limits.minimum_interval; + } + let _permit = tokio::select! { + () = cancellation.cancelled() => return Err(VisionError::Cancelled), + permit = self.semaphore.acquire() => permit.map_err(|_| VisionError::Cancelled)?, + }; + lock(&self.state).active = Some((correlation.to_owned(), owned.clone())); + let request = self.request_sequence.fetch_add(1, Ordering::AcqRel) + 1; + self.observe( + correlation, + generation, + VisionObservationKind::Started, + None, + "synthetic viewport capture started", + ); + let token = owned.token(); + let result = tokio::select! {()=cancellation.cancelled()=>Err(VisionError::Cancelled),()=token.cancelled()=>Err(VisionError::Superseded),result=tokio::time::timeout(self.limits.capture_timeout,self.source.capture_scene(generation,token.clone()))=>result.map_err(|_|VisionError::TimedOut)?}; + let result = if result == Err(VisionError::Cancelled) + && token.is_cancellation_requested() + && !cancellation.is_cancellation_requested() + { + Err(VisionError::Superseded) + } else { + result + }; + let result = match result { + Ok(scene) => { + self.observe( + correlation, + generation, + VisionObservationKind::SceneCaptured, + None, + "bounded scene captured", + ); + if request != self.request_sequence.load(Ordering::Acquire) { + Err(VisionError::Superseded) + } else if generation != self.current_generation.load(Ordering::Acquire) { + Err(VisionError::StaleGeneration) + } else { + render_scene(&scene, self.limits).inspect(|capture| { + self.observe( + correlation, + generation, + VisionObservationKind::Rendered, + Some(capture.image_sha256.clone()), + "deterministic viewport rendered", + ); + }) + } + } + Err(error) => Err(error), + }; + lock(&self.state).active = None; + match &result { + Ok(capture) => self.observe( + correlation, + generation, + VisionObservationKind::Completed, + Some(capture.image_sha256.clone()), + "capture completed", + ), + Err(VisionError::Cancelled | VisionError::Superseded) => self.observe( + correlation, + generation, + VisionObservationKind::Cancelled, + None, + "capture cancelled", + ), + Err(_) => self.observe( + correlation, + generation, + VisionObservationKind::Failed, + None, + "capture failed", + ), + } + result + } +} + +/// Type-erased runtime control surface. It deliberately exposes metadata and +/// hashes, never captured image bytes. +pub trait VisionControl: Send + Sync { + fn cancel_capture(&self, correlation_id: &str) -> bool; + fn active_capture(&self) -> Option; + fn capture_progress(&self) -> Option; + fn last_image_sha256(&self) -> Option; +} + +impl VisionControl for VisionService { + fn cancel_capture(&self, correlation_id: &str) -> bool { + let mut state = lock(&self.state); + if state + .active + .as_ref() + .is_some_and(|(active, _)| active == correlation_id) + { + if let Some((_, source)) = state.active.take() { + source.cancel(); + } + true + } else { + false + } + } + + fn active_capture(&self) -> Option { + lock(&self.state) + .active + .as_ref() + .map(|(correlation, _)| correlation.clone()) + } + + fn capture_progress(&self) -> Option { + let state = lock(&self.state); + let active = state.active.as_ref()?.0.as_str(); + state + .observations + .iter() + .rev() + .find(|event| event.correlation_id == active) + .map(|event| event.detail.clone()) + } + + fn last_image_sha256(&self) -> Option { + lock(&self.state) + .observations + .iter() + .rev() + .find_map(|event| event.image_sha256.clone()) + } +} + +/// Provider-neutral responder decorator that adds a synthetic viewport only +/// for an explicit visual question. An endpoint capability rejection produces +/// a textual scene-summary response without retrying the large image request. +pub struct VisionAugmentedResponder { + vision: Arc>, + inner: Arc, +} + +impl VisionAugmentedResponder { + #[must_use] + pub fn new(vision: Arc>, inner: Arc) -> Self { + Self { vision, inner } + } +} + +impl + crate::interaction::InteractionResponder for VisionAugmentedResponder +{ + fn respond( + &self, + mut request: crate::interaction::ResponseRequest, + cancellation: CancellationToken, + ) -> crate::interaction::ResponderFuture<'_> { + Box::pin(async move { + let explicit = request + .messages + .iter() + .rev() + .find_map(|message| { + message + .content + .as_slice() + .iter() + .find_map(|part| match part { + crate::llm::ContentPart::Text(text) => { + Some(visual_question(text.as_str())) + } + crate::llm::ContentPart::Image { .. } => None, + }) + }) + .unwrap_or(false); + if !explicit { + return self.inner.respond(request, cancellation).await; + } + if request.origin != crate::interaction::InteractionOrigin::AuthorizedIm { + return Err(crate::interaction::InteractionModelError::Failed); + } + let capture = self + .vision + .capture( + request.delivery_id.as_str(), + self.vision.generation(), + cancellation.clone(), + ) + .await + .map_err(|error| match error { + VisionError::Cancelled | VisionError::Superseded => { + crate::interaction::InteractionModelError::Cancelled + } + VisionError::TimedOut => crate::interaction::InteractionModelError::Timeout, + _ => crate::interaction::InteractionModelError::Failed, + })?; + let target = request + .messages + .iter_mut() + .rev() + .find(|message| message.role == crate::types::MessageRole::Avatar) + .ok_or(crate::interaction::InteractionModelError::Failed)?; + target + .content + .try_push( + "vision.summary", + crate::llm::ContentPart::Text( + crate::types::BoundedText::new("vision.summary", capture.summary.clone()) + .map_err(|_| crate::interaction::InteractionModelError::Failed)?, + ), + ) + .map_err(|_| crate::interaction::InteractionModelError::Failed)?; + target + .content + .try_push( + "vision.image", + crate::llm::ContentPart::Image { + url: crate::types::BoundedText::new("vision.image", capture.data_url) + .map_err(|_| crate::interaction::InteractionModelError::Failed)?, + detail: crate::llm::ImageDetail::Low, + }, + ) + .map_err(|_| crate::interaction::InteractionModelError::Failed)?; + match self.inner.respond(request, cancellation).await { + Err(crate::interaction::InteractionModelError::MultimodalUnsupported) => { + crate::interaction::VisibleResponse::new(format!( + "The configured endpoint does not support visual input. Scene summary: {}", + capture.summary + )) + .map_err(|_| crate::interaction::InteractionModelError::Failed) + } + result => result, + } + }) + } +} + +#[must_use] +pub fn visual_question(text: &str) -> bool { + let lower = text.to_lowercase(); + [ + "what do you see", + "what can you see", + "look at", + "in front of you", + "visual", + "viewport", + "picture", + "image", + "screenshot", + ] + .iter() + .any(|needle| lower.contains(needle)) +} + +fn render_scene(scene: &SceneSnapshot, limits: VisionLimits) -> Result { + validate_scene(scene, limits)?; + let pixel_count = limits.width as usize * limits.height as usize; + let mut rgba = vec![0u8; pixel_count * 4]; + let sky = environment_color(scene); + for pixel in rgba.chunks_exact_mut(4) { + pixel.copy_from_slice(&sky); + } + let mut depth = vec![f32::INFINITY; pixel_count]; + let basis = camera_basis(scene.camera)?; + let mut triangles = scene + .entities + .iter() + .flat_map(|entity| { + entity + .triangles + .iter() + .map(move |triangle| (entity, triangle)) + }) + .collect::>(); + triangles.sort_by_key(|(entity, _)| entity.id.to_string()); + for (_, triangle) in triangles { + raster_triangle( + &mut rgba, + &mut depth, + limits.width, + limits.height, + scene.camera, + basis, + triangle, + ); + } + let png = encode_png(limits.width, limits.height, &rgba, limits.max_png_bytes)?; + let hash = Sha256::digest(&png) + .iter() + .fold(String::with_capacity(64), |mut output, byte| { + write!(output, "{byte:02x}").expect("writing to String cannot fail"); + output + }); + let summary = scene_summary(scene); + let data_url = format!( + "data:image/png;base64,{}", + base64::engine::general_purpose::STANDARD.encode(&png) + ); + Ok(VisionCapture { + png, + data_url, + image_sha256: hash, + summary, + width: limits.width, + height: limits.height, + generation: scene.generation, + observed_unix_millis: scene.observed_unix_millis, + completeness: scene.completeness, + }) +} +fn validate_scene(scene: &SceneSnapshot, limits: VisionLimits) -> Result<(), VisionError> { + if scene.generation == 0 + || scene.region_id == UUID::zero() + || scene.region_name.len() > 256 + || scene.entities.len() > limits.max_entities + { + return Err(VisionError::InvalidScene); + } + if scene.texture_fetches > limits.max_texture_fetches + || scene.texture_bytes > limits.max_texture_bytes + || scene.decoded_texture_pixels > limits.max_decode_pixels + { + return Err(VisionError::ResourceLimit); + } + let mut count = 0usize; + for entity in &scene.entities { + count = count + .checked_add(entity.triangles.len()) + .ok_or(VisionError::ResourceLimit)?; + if entity.display_name.len() > 256 + || entity + .triangles + .iter() + .flat_map(|t| t.vertices) + .flatten() + .any(|v| !v.is_finite()) + { + return Err(VisionError::InvalidScene); + } + } + if count > limits.max_triangles { + return Err(VisionError::ResourceLimit); + } + Ok(()) +} +type CameraBasis = ([f32; 3], [f32; 3], [f32; 3]); +fn camera_basis(camera: CameraPose) -> Result { + if !(10.0..=140.0).contains(&camera.vertical_fov_degrees) + || camera + .position + .iter() + .chain(camera.forward.iter()) + .chain(camera.up.iter()) + .any(|v| !v.is_finite()) + { + return Err(VisionError::InvalidScene); + } + let f = normalize(camera.forward)?; + let r = normalize(cross(f, camera.up))?; + let u = cross(r, f); + Ok((r, u, f)) +} +fn normalize(v: [f32; 3]) -> Result<[f32; 3], VisionError> { + let length = dot(v, v).sqrt(); + if length < 0.0001 { + return Err(VisionError::InvalidScene); + } + Ok([v[0] / length, v[1] / length, v[2] / length]) +} +fn dot(a: [f32; 3], b: [f32; 3]) -> f32 { + a[0] * b[0] + a[1] * b[1] + a[2] * b[2] +} +fn cross(a: [f32; 3], b: [f32; 3]) -> [f32; 3] { + [ + a[1] * b[2] - a[2] * b[1], + a[2] * b[0] - a[0] * b[2], + a[0] * b[1] - a[1] * b[0], + ] +} +fn raster_triangle( + rgba: &mut [u8], + depth: &mut [f32], + width: u32, + height: u32, + camera: CameraPose, + basis: ([f32; 3], [f32; 3], [f32; 3]), + triangle: &SceneTriangle, +) { + let mut p = [[0f32; 3]; 3]; + let aspect = width as f32 / height as f32; + let focal = 1.0 / (camera.vertical_fov_degrees.to_radians() / 2.0).tan(); + for (index, world) in triangle.vertices.iter().enumerate() { + let delta = [ + world[0] - camera.position[0], + world[1] - camera.position[1], + world[2] - camera.position[2], + ]; + let z = dot(delta, basis.2); + if z <= 0.05 { + return; + } + p[index] = [ + (dot(delta, basis.0) * focal / aspect / z + 1.0) * 0.5 * width as f32, + (1.0 - dot(delta, basis.1) * focal / z) * 0.5 * height as f32, + z, + ]; + } + let area = edge(p[0], p[1], p[2][0], p[2][1]); + if area.abs() < 0.0001 { + return; + } + let min_x = p + .iter() + .map(|v| v[0].floor() as i32) + .min() + .unwrap_or(0) + .clamp(0, width as i32 - 1); + let max_x = p + .iter() + .map(|v| v[0].ceil() as i32) + .max() + .unwrap_or(0) + .clamp(0, width as i32 - 1); + let min_y = p + .iter() + .map(|v| v[1].floor() as i32) + .min() + .unwrap_or(0) + .clamp(0, height as i32 - 1); + let max_y = p + .iter() + .map(|v| v[1].ceil() as i32) + .max() + .unwrap_or(0) + .clamp(0, height as i32 - 1); + for y in min_y..=max_y { + for x in min_x..=max_x { + let px = x as f32 + 0.5; + let py = y as f32 + 0.5; + let w0 = edge(p[1], p[2], px, py) / area; + let w1 = edge(p[2], p[0], px, py) / area; + let w2 = 1.0 - w0 - w1; + if w0 >= 0.0 && w1 >= 0.0 && w2 >= 0.0 { + let z = w0 * p[0][2] + w1 * p[1][2] + w2 * p[2][2]; + let index = y as usize * width as usize + x as usize; + if z < depth[index] { + depth[index] = z; + rgba[index * 4..index * 4 + 4].copy_from_slice(&triangle.color_srgb); + } + } + } + } +} +fn edge(a: [f32; 3], b: [f32; 3], x: f32, y: f32) -> f32 { + (x - a[0]) * (b[1] - a[1]) - (y - a[1]) * (b[0] - a[0]) +} +fn environment_color(scene: &SceneSnapshot) -> [u8; 4] { + let phase = (scene.observed_unix_millis / 1000 % 86_400) as f32 / 86_400.0; + let light = (phase * std::f32::consts::TAU) + .sin() + .mul_add(24.0, 72.0) + .clamp(32.0, 112.0) as u8; + [ + light, + light.saturating_add(24), + light.saturating_add(48), + 255, + ] +} +fn encode_png( + width: u32, + height: u32, + rgba: &[u8], + maximum: usize, +) -> Result, VisionError> { + let mut output = Vec::new(); + { + let mut encoder = png::Encoder::new(&mut output, width, height); + encoder.set_color(png::ColorType::Rgba); + encoder.set_depth(png::BitDepth::Eight); + encoder.set_source_srgb(png::SrgbRenderingIntent::Perceptual); + encoder.set_compression(png::Compression::Best); + encoder.set_filter(png::FilterType::Paeth); + let mut writer = encoder.write_header().map_err(|_| VisionError::Encode)?; + writer + .write_image_data(rgba) + .map_err(|_| VisionError::Encode)?; + } + if output.len() > maximum { + return Err(VisionError::ResourceLimit); + } + Ok(output) +} +fn scene_summary(scene: &SceneSnapshot) -> String { + let objects = scene + .entities + .iter() + .filter(|e| e.kind == SceneEntityKind::Object) + .count(); + let residents = scene + .entities + .iter() + .filter(|e| e.kind == SceneEntityKind::Resident) + .count(); + format!( + "Synthetic viewport of region {:?}: {objects} tracked objects and {residents} privacy-marked residents. generation={} observed_unix_millis={}. completeness: objects_truncated={}, avatars_truncated={}, textures_missing={}, terrain_available={}. This is reconstructed from available scene data, not a viewer framebuffer.", + sanitize(&scene.region_name), + scene.generation, + scene.observed_unix_millis, + scene.completeness.objects_truncated, + scene.completeness.avatars_truncated, + scene.completeness.textures_missing, + scene.completeness.terrain_available + ) +} +fn sanitize(value: &str) -> String { + value.chars().filter(|c| !c.is_control()).take(96).collect() +} +fn lock(value: &Mutex) -> std::sync::MutexGuard<'_, T> { + value + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) +} + +#[cfg(feature = "live-grid")] +pub struct LibremetaverseSceneSource { + client: libremetaverse::GridClient, + agent: Arc, + limits: VisionLimits, +} + +#[cfg(feature = "live-grid")] +impl LibremetaverseSceneSource { + #[must_use] + pub fn new(owner: &crate::backend::LibremetaverseClientOwner, limits: VisionLimits) -> Self { + Self { + client: owner.client().clone(), + agent: owner.agent(), + limits, + } + } +} + +#[cfg(feature = "live-grid")] +impl SceneSource for LibremetaverseSceneSource { + #[allow(clippy::too_many_lines)] + fn capture_scene( + &self, + generation: u64, + cancellation: CancellationToken, + ) -> VisionFuture<'_, SceneSnapshot> { + Box::pin(async move { + if cancellation.is_cancellation_requested() { + return Err(VisionError::Cancelled); + } + let simulator = self + .client + .network() + .current_sim() + .ok_or(VisionError::StaleGeneration)?; + let camera = &self.agent.movement.camera; + let position = camera.position(); + let forward = camera.at_axis(); + let up = camera.up_axis(); + let prims = simulator + .objects_primitives + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .values() + .filter(|prim| !prim.is_attachment) + .take(self.limits.max_entities) + .cloned() + .collect::>(); + let remaining = self.limits.max_entities.saturating_sub(prims.len()); + let avatars = simulator + .objects_avatars + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .values() + .take(remaining) + .cloned() + .collect::>(); + let objects_truncated = simulator + .objects_primitives + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .len() + > prims.len(); + let avatars_truncated = simulator + .objects_avatars + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .len() + > avatars.len(); + let region_id = simulator.region_id; + let region_name = simulator.name.clone(); + let water = simulator.water_height; + let texture_ids = prims + .iter() + .filter_map(|prim| { + prim.textures + .as_ref()? + .default_texture + .as_ref() + .map(libremetaverse::PrimitiveTextureEntryFace::texture_id) + }) + .filter(|id| *id != UUID::zero()) + .fold(Vec::new(), |mut ids, id| { + if !ids.contains(&id) && ids.len() < self.limits.max_texture_fetches { + ids.push(id); + } + ids + }); + let assets = self.client.assets(); + let texture_fetches = texture_ids.len(); + let mut encoded_textures = Vec::new(); + let mut texture_bytes = 0usize; + for id in texture_ids { + if cancellation.is_cancellation_requested() { + return Err(VisionError::Cancelled); + } + let asset = assets + .request_asset_with_uuid_asset_type_boolean_cancellation_token( + id, + libremetaverse_types::AssetType::Texture, + false, + Some(cancellation.clone()), + ) + .await + .ok() + .flatten(); + if let Some(asset) = asset + && texture_bytes.saturating_add(asset.asset_data.len()) + <= self.limits.max_texture_bytes + { + texture_bytes += asset.asset_data.len(); + encoded_textures.push((id, asset.asset_data)); + } + } + let maximum = self.limits.max_triangles; + let decode_pixels = self.limits.max_decode_pixels; + let decode_bytes = self.limits.max_texture_bytes; + let (entities, decoded_texture_pixels) = tokio::task::spawn_blocking(move || { + native_entities( + prims, + avatars, + water, + maximum, + encoded_textures, + decode_bytes, + decode_pixels, + ) + }) + .await + .map_err(|_| VisionError::InvalidScene)??; + if cancellation.is_cancellation_requested() { + return Err(VisionError::Cancelled); + } + let textures_missing = entities + .iter() + .filter(|entity| { + !entity.texture_available && entity.kind == SceneEntityKind::Object + }) + .count() + .try_into() + .unwrap_or(u32::MAX); + Ok(SceneSnapshot { + generation, + observed_unix_millis: std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |duration| { + duration.as_millis().try_into().unwrap_or(u64::MAX) + }), + region_id, + region_name, + camera: CameraPose { + position: [position.x, position.y, position.z], + forward: [forward.x, forward.y, forward.z], + up: [up.x, up.y, up.z], + vertical_fov_degrees: 60.0, + }, + entities, + completeness: SnapshotCompleteness { + objects_truncated, + avatars_truncated, + textures_missing, + terrain_available: true, + }, + texture_fetches, + texture_bytes, + decoded_texture_pixels, + }) + }) + } +} + +#[cfg(feature = "live-grid")] +#[allow(clippy::too_many_lines)] +fn native_entities( + prims: Vec, + avatars: Vec, + water: f32, + maximum: usize, + encoded_textures: Vec<(UUID, Vec)>, + decode_bytes: usize, + decode_pixels: usize, +) -> Result<(Vec, usize), VisionError> { + let (texture_colors, pixels_consumed) = + decode_texture_colors(encoded_textures, decode_bytes, decode_pixels); + let renderer = libremetaverse_rendering_simple::SimpleRenderer::new() + .map_err(|_| VisionError::InvalidScene)?; + let mut entities = Vec::new(); + let mut triangles = 0usize; + for prim in prims { + let name = prim.properties.as_ref().map_or_else( + || "tracked object".to_owned(), + |properties| properties.name.clone(), + ); + let texture_id = prim + .textures + .as_ref() + .and_then(|textures| textures.default_texture.as_ref()) + .map(libremetaverse::PrimitiveTextureEntryFace::texture_id); + let texture_color = texture_id.and_then(|id| texture_colors.get(&id).copied()); + let texture_available = texture_color.is_some(); + let mesh = renderer + .generate_faceted_mesh(prim.clone(), libremetaverse::rendering::DetailLevel::Low) + .map_err(|_| VisionError::InvalidScene)?; + let mut output = Vec::new(); + for face in mesh.faces { + let rgba = face.texture_face.rgba(); + let mut color = [ + unit_byte(rgba.r), + unit_byte(rgba.g), + unit_byte(rgba.b), + unit_byte(rgba.a), + ]; + if let Some(texture) = texture_color { + for channel in 0..3 { + color[channel] = + ((u16::from(color[channel]) * u16::from(texture[channel])) / 255) as u8; + } + } + for indices in face.indices.chunks_exact(3) { + triangles = triangles.checked_add(1).ok_or(VisionError::ResourceLimit)?; + if triangles > maximum { + return Err(VisionError::ResourceLimit); + } + let mut vertices = [[0.0; 3]; 3]; + for (index, source) in indices.iter().enumerate() { + let local = face + .vertices + .get(*source as usize) + .ok_or(VisionError::InvalidScene)? + .position; + let scaled = libremetaverse_types::Vector3 { + x: local.x * prim.scale.x, + y: local.y * prim.scale.y, + z: local.z * prim.scale.z, + }; + let rotated = rotate_vector(scaled, prim.rotation); + vertices[index] = [ + rotated.x + prim.position.x, + rotated.y + prim.position.y, + rotated.z + prim.position.z, + ]; + } + output.push(SceneTriangle { + vertices, + color_srgb: color, + }); + } + } + entities.push(SceneEntity { + id: prim.id, + kind: SceneEntityKind::Object, + display_name: name, + triangles: output, + texture_available, + }); + } + for avatar in avatars { + let p = avatar.position; + let half = 0.3; + let bottom = p.z; + let top = p.z + 1.8; + let front = p.y; + let triangles = vec![ + SceneTriangle { + vertices: [ + [p.x - half, front, bottom], + [p.x + half, front, bottom], + [p.x + half, front, top], + ], + color_srgb: [80, 140, 220, 255], + }, + SceneTriangle { + vertices: [ + [p.x - half, front, bottom], + [p.x + half, front, top], + [p.x - half, front, top], + ], + color_srgb: [80, 140, 220, 255], + }, + ]; + entities.push(SceneEntity { + id: avatar.id, + kind: SceneEntityKind::Resident, + display_name: format!( + "resident-{}", + crate::observability::pseudonymous_identifier(&avatar.id.to_string()) + ), + triangles, + texture_available: false, + }); + } + let size = 256.0; + entities.push(SceneEntity { + id: UUID::zero(), + kind: SceneEntityKind::Terrain, + display_name: "bounded terrain/water plane".into(), + triangles: vec![ + SceneTriangle { + vertices: [[0.0, 0.0, water], [size, 0.0, water], [size, size, water]], + color_srgb: [55, 95, 80, 255], + }, + SceneTriangle { + vertices: [[0.0, 0.0, water], [size, size, water], [0.0, size, water]], + color_srgb: [55, 95, 80, 255], + }, + ], + texture_available: false, + }); + Ok((entities, pixels_consumed)) +} + +#[cfg(feature = "live-grid")] +fn decode_texture_colors( + textures: Vec<(UUID, Vec)>, + max_bytes: usize, + max_pixels: usize, +) -> (std::collections::BTreeMap, usize) { + let mut output = std::collections::BTreeMap::new(); + let mut remaining_pixels = max_pixels; + for (id, bytes) in textures { + if remaining_pixels == 0 { + break; + } + let options = libremetaverse_imaging::J2kDecodeOptions::default() + .with_limits(max_bytes.min(bytes.len().max(1)), remaining_pixels); + let Ok(image) = libremetaverse_imaging::RustJ2kCodec::decode_bytes(&bytes, options) else { + continue; + }; + let Ok(width) = usize::try_from(image.width) else { + continue; + }; + let Ok(height) = usize::try_from(image.height) else { + continue; + }; + let Some(pixels) = width.checked_mul(height) else { + continue; + }; + if pixels == 0 || pixels > remaining_pixels || image.red.len() != pixels { + continue; + } + let average = |plane: &[u8], fallback: u8| -> u8 { + if plane.len() != pixels { + return fallback; + } + let sum = plane.iter().map(|value| u64::from(*value)).sum::(); + u8::try_from(sum / pixels as u64).unwrap_or(fallback) + }; + output.insert( + id, + [ + average(&image.red, 255), + average(&image.green, 255), + average(&image.blue, 255), + average(&image.alpha, 255), + ], + ); + remaining_pixels -= pixels; + } + (output, max_pixels - remaining_pixels) +} +#[cfg(feature = "live-grid")] +fn unit_byte(value: f32) -> u8 { + (value.clamp(0.0, 1.0) * 255.0).round() as u8 +} + +#[cfg(feature = "live-grid")] +fn rotate_vector( + value: libremetaverse_types::Vector3, + rotation: libremetaverse_types::Quaternion, +) -> libremetaverse_types::Vector3 { + // q * v * conjugate(q), expanded to avoid allocating an intermediate matrix. + let q = [rotation.x, rotation.y, rotation.z]; + let v = [value.x, value.y, value.z]; + let t = [ + 2.0 * (q[1] * v[2] - q[2] * v[1]), + 2.0 * (q[2] * v[0] - q[0] * v[2]), + 2.0 * (q[0] * v[1] - q[1] * v[0]), + ]; + libremetaverse_types::Vector3 { + x: v[0] + rotation.w * t[0] + q[1] * t[2] - q[2] * t[1], + y: v[1] + rotation.w * t[1] + q[2] * t[0] - q[0] * t[2], + z: v[2] + rotation.w * t[2] + q[0] * t[1] - q[1] * t[0], + } +} diff --git a/crates/metacrate-grid-agent/src/vision_tests.rs b/crates/metacrate-grid-agent/src/vision_tests.rs new file mode 100644 index 0000000..68e2a58 --- /dev/null +++ b/crates/metacrate-grid-agent/src/vision_tests.rs @@ -0,0 +1,376 @@ +use crate::vision::*; +use crate::{ + BoundedText, CompletionMessage, InteractionChannel, InteractionIntent, InteractionModelError, + InteractionOrigin, InteractionResponder, MessageRole, ResponderFuture, ResponseRequest, + VisibleResponse, +}; +use libremetaverse_types::{ + UUID, + compat::{CancellationToken, CancellationTokenSource}, +}; +use std::sync::{Arc, Mutex}; + +struct FakeScene { + scene: SceneSnapshot, + pending: Mutex, +} +impl SceneSource for FakeScene { + fn capture_scene( + &self, + _: u64, + cancellation: CancellationToken, + ) -> VisionFuture<'_, SceneSnapshot> { + Box::pin(async move { + if *self.pending.lock().unwrap() { + tokio::select! {()=cancellation.cancelled()=>Err(VisionError::Cancelled),()=std::future::pending()=>unreachable!()} + } else { + Ok(self.scene.clone()) + } + }) + } +} +fn uuid(value: u32) -> UUID { + UUID::new_with_string(format!("00000000-0000-4000-8000-{value:012}")).unwrap() +} +fn triangle(z: f32, color: [u8; 4]) -> SceneTriangle { + SceneTriangle { + vertices: [[-1.0, -1.0, z], [1.0, -1.0, z], [0.0, 1.0, z]], + color_srgb: color, + } +} +fn scene() -> SceneSnapshot { + SceneSnapshot { + generation: 7, + observed_unix_millis: 1_700_000_000_000, + region_id: uuid(1), + region_name: "Fixture Region".into(), + camera: CameraPose { + position: [0.0, 0.0, 0.0], + forward: [0.0, 0.0, 1.0], + up: [0.0, 1.0, 0.0], + vertical_fov_degrees: 60.0, + }, + entities: vec![ + SceneEntity { + id: uuid(3), + kind: SceneEntityKind::Resident, + display_name: "Private Resident".into(), + triangles: vec![triangle(4.0, [255, 0, 0, 255])], + texture_available: false, + }, + SceneEntity { + id: uuid(2), + kind: SceneEntityKind::Object, + display_name: "Near".into(), + triangles: vec![triangle(2.0, [0, 255, 0, 255])], + texture_available: true, + }, + ], + completeness: SnapshotCompleteness { + textures_missing: 1, + terrain_available: false, + ..SnapshotCompleteness::default() + }, + texture_fetches: 1, + texture_bytes: 128, + decoded_texture_pixels: 16, + } +} + +#[tokio::test] +async fn golden_scene_is_deterministic_depth_ordered_and_privacy_marked() { + let source = Arc::new(FakeScene { + scene: scene(), + pending: Mutex::new(false), + }); + let service = VisionService::new( + source, + VisionLimits { + width: 64, + height: 64, + minimum_interval: std::time::Duration::ZERO, + ..VisionLimits::default() + }, + ) + .unwrap(); + service.set_generation(7); + let first = service + .capture("one", 7, CancellationToken::default()) + .await + .unwrap(); + let second = service + .capture("two", 7, CancellationToken::default()) + .await + .unwrap(); + assert_eq!(first.png, second.png); + assert_eq!(first.image_sha256, second.image_sha256); + assert_eq!( + first.image_sha256, + "e943b14142da05fcc1471ee0b639ffc1c11c74bfde83432ad3a26f994e4e665a" + ); + assert!(first.summary.contains("privacy-marked residents")); + assert!(!first.summary.contains("Private Resident")); + assert!(first.summary.contains("textures_missing=1")); + assert!(first.data_url.starts_with("data:image/png;base64,")); + let decoder = png::Decoder::new(first.png.as_slice()); + let mut reader = decoder.read_info().unwrap(); + assert_eq!( + reader.info().srgb, + Some(png::SrgbRenderingIntent::Perceptual) + ); + let mut pixels = vec![0; reader.output_buffer_size()]; + let info = reader.next_frame(&mut pixels).unwrap(); + let center = ((info.height as usize / 2) * info.width as usize + info.width as usize / 2) * 4; + assert_eq!(&pixels[center..center + 4], &[0, 255, 0, 255]); +} + +#[tokio::test] +async fn invalid_camera_huge_scene_stale_generation_and_size_limit_fail_closed() { + let mut bad = scene(); + bad.camera.forward = [0.0, 0.0, 0.0]; + let source = Arc::new(FakeScene { + scene: bad, + pending: Mutex::new(false), + }); + let service = VisionService::new( + source, + VisionLimits { + width: 64, + height: 64, + minimum_interval: std::time::Duration::ZERO, + ..VisionLimits::default() + }, + ) + .unwrap(); + service.set_generation(7); + assert_eq!( + service + .capture("bad", 7, CancellationToken::default()) + .await + .unwrap_err(), + VisionError::InvalidScene + ); + assert_eq!( + service + .capture("stale", 6, CancellationToken::default()) + .await + .unwrap_err(), + VisionError::StaleGeneration + ); + let source = Arc::new(FakeScene { + scene: scene(), + pending: Mutex::new(false), + }); + let service = VisionService::new( + source, + VisionLimits { + width: 64, + height: 64, + max_png_bytes: 4096, + minimum_interval: std::time::Duration::ZERO, + ..VisionLimits::default() + }, + ) + .unwrap(); + service.set_generation(7); + let result = service + .capture("size", 7, CancellationToken::default()) + .await; + assert!(matches!(result, Ok(_) | Err(VisionError::ResourceLimit))); + + let mut excessive_texture_work = scene(); + excessive_texture_work.texture_fetches = VisionLimits::default().max_texture_fetches + 1; + let source = Arc::new(FakeScene { + scene: excessive_texture_work, + pending: Mutex::new(false), + }); + let service = VisionService::new( + source, + VisionLimits { + width: 64, + height: 64, + minimum_interval: std::time::Duration::ZERO, + ..VisionLimits::default() + }, + ) + .unwrap(); + service.set_generation(7); + assert_eq!( + service + .capture("texture-budget", 7, CancellationToken::default()) + .await + .unwrap_err(), + VisionError::ResourceLimit + ); +} + +#[tokio::test] +async fn cancellation_and_region_change_interrupt_nonblocking_capture() { + let source = Arc::new(FakeScene { + scene: scene(), + pending: Mutex::new(true), + }); + let service = Arc::new( + VisionService::new( + source, + VisionLimits { + width: 64, + height: 64, + minimum_interval: std::time::Duration::ZERO, + ..VisionLimits::default() + }, + ) + .unwrap(), + ); + service.set_generation(7); + let cancellation = CancellationTokenSource::new(); + cancellation.cancel(); + assert_eq!( + service + .capture("cancel", 7, cancellation.token()) + .await + .unwrap_err(), + VisionError::Cancelled + ); + let service2 = service.clone(); + let task = tokio::spawn(async move { + service2 + .capture("region", 7, CancellationToken::default()) + .await + }); + tokio::task::yield_now().await; + service.set_generation(8); + assert!(matches!( + task.await.unwrap(), + Err(VisionError::Superseded | VisionError::Cancelled) + )); + + service.set_generation(9); + let first_service = service.clone(); + let first = tokio::spawn(async move { + first_service + .capture("first", 9, CancellationToken::default()) + .await + }); + tokio::task::yield_now().await; + let second_cancel = CancellationTokenSource::new(); + let second_service = service.clone(); + let second_token = second_cancel.token(); + let second = + tokio::spawn(async move { second_service.capture("second", 9, second_token).await }); + assert_eq!( + tokio::time::timeout(std::time::Duration::from_secs(1), first) + .await + .unwrap() + .unwrap() + .unwrap_err(), + VisionError::Superseded + ); + second_cancel.cancel(); + assert_eq!(second.await.unwrap().unwrap_err(), VisionError::Cancelled); +} + +#[test] +fn explicit_visual_intent_is_narrow() { + assert!(visual_question("What do you see in front of you?")); + assert!(visual_question("Please look at this viewport")); + assert!(!visual_question("hello there")); + assert!(!visual_question("build a cube")); +} + +struct FakeResponder { + parts: Mutex, + unsupported: bool, +} +impl InteractionResponder for FakeResponder { + fn respond(&self, request: ResponseRequest, _: CancellationToken) -> ResponderFuture<'_> { + Box::pin(async move { + *self.parts.lock().unwrap() = request.messages.last().unwrap().content.len(); + if self.unsupported { + Err(InteractionModelError::MultimodalUnsupported) + } else { + VisibleResponse::new("seen").map_err(|_| InteractionModelError::Failed) + } + }) + } +} +fn request(text: &str) -> ResponseRequest { + ResponseRequest { + delivery_id: BoundedText::new("id", "delivery").unwrap(), + sender_id: uuid(9), + session_id: BoundedText::new("session", "session").unwrap(), + channel: InteractionChannel::DirectIm, + origin: InteractionOrigin::AuthorizedIm, + intent: InteractionIntent::Informational, + messages: vec![CompletionMessage::text(MessageRole::Avatar, text).unwrap()], + } +} + +#[tokio::test] +async fn responder_adds_generic_image_payload_and_returns_summary_on_capability_rejection() { + let source = Arc::new(FakeScene { + scene: scene(), + pending: Mutex::new(false), + }); + let vision = Arc::new( + VisionService::new( + source, + VisionLimits { + width: 64, + height: 64, + minimum_interval: std::time::Duration::ZERO, + ..VisionLimits::default() + }, + ) + .unwrap(), + ); + vision.set_generation(7); + let endpoint = Arc::new(FakeResponder { + parts: Mutex::new(0), + unsupported: true, + }); + let responder = VisionAugmentedResponder::new(vision, endpoint.clone()); + let response = responder + .respond(request("What do you see?"), CancellationToken::default()) + .await + .unwrap(); + assert!(response.as_str().contains("does not support visual input")); + assert!(response.as_str().contains("Synthetic viewport")); + assert_eq!(*endpoint.parts.lock().unwrap(), 3); +} + +#[tokio::test] +async fn responder_never_captures_for_ordinary_text() { + let source = Arc::new(FakeScene { + scene: scene(), + pending: Mutex::new(false), + }); + let vision = Arc::new( + VisionService::new( + source, + VisionLimits { + width: 64, + height: 64, + minimum_interval: std::time::Duration::ZERO, + ..VisionLimits::default() + }, + ) + .unwrap(), + ); + vision.set_generation(7); + let endpoint = Arc::new(FakeResponder { + parts: Mutex::new(0), + unsupported: false, + }); + let responder = VisionAugmentedResponder::new(vision.clone(), endpoint.clone()); + assert_eq!( + responder + .respond(request("hello"), CancellationToken::default()) + .await + .unwrap() + .as_str(), + "seen" + ); + assert_eq!(*endpoint.parts.lock().unwrap(), 1); + assert!(vision.observations().is_empty()); +} diff --git a/crates/metacrate-grid-agent/tests/dependency_policy.rs b/crates/metacrate-grid-agent/tests/dependency_policy.rs index 2eae49c..5556896 100644 --- a/crates/metacrate-grid-agent/tests/dependency_policy.rs +++ b/crates/metacrate-grid-agent/tests/dependency_policy.rs @@ -2,10 +2,14 @@ use std::collections::BTreeSet; use std::fs; use std::path::{Path, PathBuf}; -const ALLOWED_DEPENDENCIES: [&str; 13] = [ +const ALLOWED_DEPENDENCIES: [&str; 17] = [ + "base64", "crossterm", "libremetaverse", + "libremetaverse-imaging", + "libremetaverse-rendering-simple", "metacrate-lsl-tools", + "png", "libremetaverse-types", "reqwest", "rustls", @@ -48,10 +52,10 @@ fn package_has_only_reviewed_rust_dependencies_and_no_build_script() { #[test] fn runtime_source_has_no_subprocess_or_native_abi_escape_hatch() { let source = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("src"); - let mut files = Vec::with_capacity(34); + let mut files = Vec::with_capacity(36); collect_rust_files(&source, &mut files); assert!( - files.len() <= 34, + files.len() <= 36, "source-file count needs a reviewed bound update" ); for path in files { @@ -82,6 +86,27 @@ fn runtime_source_has_no_subprocess_or_native_abi_escape_hatch() { } } +#[test] +fn vision_path_selects_only_the_pure_rust_renderer_and_codec() { + let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + let manifest = fs::read_to_string(root.join("Cargo.toml")).expect("agent manifest"); + assert!(manifest.contains("default-features = false, features = [\"rust-j2k\"]")); + for forbidden in ["imaging-skia", "openjpeg", "skia-safe"] { + assert!(!manifest.contains(forbidden), "vision enables {forbidden}"); + } + let renderer = fs::read_to_string(root.join("../libremetaverse-rendering-simple/Cargo.toml")) + .expect("simple renderer manifest"); + assert!(!renderer.contains("imaging-skia")); + assert!(!renderer.contains("openjpeg")); + let vision = fs::read_to_string(root.join("src/vision.rs")).expect("vision source"); + for forbidden in ["screenshot crate", "gpu adapter", "std::process"] { + assert!( + !vision.to_ascii_lowercase().contains(forbidden), + "vision must remain synthetic and pure Rust: {forbidden}" + ); + } +} + #[test] fn compatibility_crates_never_depend_on_or_reexport_metacrate_crates() { let workspace = PathBuf::from(env!("CARGO_MANIFEST_DIR")) diff --git a/crates/metacrate-grid-agent/tests/llm_transport.rs b/crates/metacrate-grid-agent/tests/llm_transport.rs index 2692063..45d7195 100644 --- a/crates/metacrate-grid-agent/tests/llm_transport.rs +++ b/crates/metacrate-grid-agent/tests/llm_transport.rs @@ -503,6 +503,45 @@ async fn retry_classification_retry_after_timeout_and_cancellation_are_bounded() assert_eq!(request.await.expect_err("cancelled"), LlmError::Cancelled); } +#[tokio::test] +async fn multimodal_capability_rejection_is_typed_and_never_retried() { + let mut server = fake_server(vec![ResponsePlan::status(415)]).await; + let mut retry_limits = limits(); + retry_limits.max_retries = 2; + let image = CompletionMessage { + role: MessageRole::Avatar, + content: metacrate_grid_agent::BoundedVec::try_from_vec( + "message.content", + vec![ContentPart::Image { + url: metacrate_grid_agent::BoundedText::new( + "image.url", + "data:image/png;base64,iVBORw0KGgo=", + ) + .unwrap(), + detail: ImageDetail::Low, + }], + ) + .unwrap(), + tool_call_id: None, + proposed_calls: metacrate_grid_agent::BoundedVec::new(), + }; + assert_eq!( + client(&server.url, retry_limits) + .complete(&[image], &[], &CancellationToken::default()) + .await + .expect_err("multimodal rejection"), + LlmError::MultimodalUnsupported + ); + server.requests.recv().await.expect("one image request"); + assert!( + !matches!( + tokio::time::timeout(Duration::from_millis(100), server.requests.recv()).await, + Ok(Some(_)) + ), + "capability rejection must not resend the large image" + ); +} + #[tokio::test] async fn concurrent_slow_sessions_respect_the_shared_semaphore() { let plans = (0..6)