use crate::vision::*; use crate::{ BoundedText, Capability, CompletionMessage, InteractionChannel, InteractionIntent, InteractionModelError, InteractionOrigin, InteractionResponder, MessageRole, ResponderFuture, ResponseRequest, VisibleResponse, }; use libremetaverse_types::{ UUID, compat::{CancellationToken, CancellationTokenSource}, }; use std::sync::{Arc, Mutex}; struct FakeScene { scene: SceneSnapshot, pending: Mutex, } impl SceneSource for FakeScene { fn capture_scene( &self, _: u64, cancellation: CancellationToken, ) -> VisionFuture<'_, SceneSnapshot> { Box::pin(async move { if *self.pending.lock().unwrap() { tokio::select! {()=cancellation.cancelled()=>Err(VisionError::Cancelled),()=std::future::pending()=>unreachable!()} } else { Ok(self.scene.clone()) } }) } } fn uuid(value: u32) -> UUID { UUID::new_with_string(format!("00000000-0000-4000-8000-{value:012}")).unwrap() } fn triangle(z: f32, color: [u8; 4]) -> SceneTriangle { SceneTriangle { vertices: [[-1.0, -1.0, z], [1.0, -1.0, z], [0.0, 1.0, z]], color_srgb: color, } } fn scene() -> SceneSnapshot { SceneSnapshot { generation: 7, observed_unix_millis: 1_700_000_000_000, region_id: uuid(1), region_name: "Fixture Region".into(), camera: CameraPose { position: [0.0, 0.0, 0.0], forward: [0.0, 0.0, 1.0], up: [0.0, 1.0, 0.0], vertical_fov_degrees: 60.0, }, entities: vec![ SceneEntity { id: uuid(3), kind: SceneEntityKind::Resident, display_name: "Private Resident".into(), triangles: vec![triangle(4.0, [255, 0, 0, 255])], texture_available: false, }, SceneEntity { id: uuid(2), kind: SceneEntityKind::Object, display_name: "Near".into(), triangles: vec![triangle(2.0, [0, 255, 0, 255])], texture_available: true, }, ], completeness: SnapshotCompleteness { textures_missing: 1, terrain_available: false, ..SnapshotCompleteness::default() }, texture_fetches: 1, texture_bytes: 128, decoded_texture_pixels: 16, } } #[tokio::test] async fn golden_scene_is_deterministic_depth_ordered_and_privacy_marked() { let source = Arc::new(FakeScene { scene: scene(), pending: Mutex::new(false), }); let service = VisionService::new( source, VisionLimits { width: 64, height: 64, minimum_interval: std::time::Duration::ZERO, ..VisionLimits::default() }, ) .unwrap(); service.set_generation(7); let first = service .capture("one", 7, CancellationToken::default()) .await .unwrap(); let second = service .capture("two", 7, CancellationToken::default()) .await .unwrap(); assert_eq!(first.jpeg, second.jpeg); assert_eq!(first.image_sha256, second.image_sha256); assert_eq!( first.image_sha256, "b53d23f57ba0d6fb79856afde111ad6dc25559e810dcc028a5f107090914821d" ); assert!(first.summary.contains("privacy-marked residents")); assert!(!first.summary.contains("Private Resident")); assert!(first.summary.contains("textures_missing=1")); assert!(first.data_url.starts_with("data:image/jpeg;base64,")); let mut decoder = jpeg_decoder::Decoder::new(first.jpeg.as_slice()); let pixels = decoder.decode().unwrap(); let info = decoder.info().unwrap(); assert_eq!((info.width, info.height), (64, 64)); let center = ((info.height as usize / 2) * info.width as usize + info.width as usize / 2) * 3; assert!(pixels[center + 1] > 180 && pixels[center] < 80 && pixels[center + 2] < 80); } #[cfg(feature = "live-grid")] #[tokio::test(flavor = "multi_thread")] async fn offscreen_wgpu_renders_depth_ordered_scene_when_an_adapter_is_available() { let Ok(renderer) = WgpuRenderer::new().await else { return; }; let limits = VisionLimits { width: 64, height: 64, minimum_interval: std::time::Duration::ZERO, ..VisionLimits::default() }; let rgba = tokio::task::spawn_blocking(move || renderer.render(&scene(), limits)) .await .expect("render worker") .expect("offscreen render"); assert_eq!(rgba.len(), 64 * 64 * 4); let center = (32 * 64 + 32) * 4; assert!(rgba[center + 1] > 180 && rgba[center] < 80 && rgba[center + 2] < 80); } #[tokio::test] async fn invalid_camera_huge_scene_stale_generation_and_size_limit_fail_closed() { let mut bad = scene(); bad.camera.forward = [0.0, 0.0, 0.0]; let source = Arc::new(FakeScene { scene: bad, pending: Mutex::new(false), }); let service = VisionService::new( source, VisionLimits { width: 64, height: 64, minimum_interval: std::time::Duration::ZERO, ..VisionLimits::default() }, ) .unwrap(); service.set_generation(7); assert_eq!( service .capture("bad", 7, CancellationToken::default()) .await .unwrap_err(), VisionError::InvalidScene ); assert_eq!( service .capture("stale", 6, CancellationToken::default()) .await .unwrap_err(), VisionError::StaleGeneration ); let source = Arc::new(FakeScene { scene: scene(), pending: Mutex::new(false), }); let service = VisionService::new( source, VisionLimits { width: 64, height: 64, max_jpeg_bytes: 4096, minimum_interval: std::time::Duration::ZERO, ..VisionLimits::default() }, ) .unwrap(); service.set_generation(7); let result = service .capture("size", 7, CancellationToken::default()) .await; assert!(matches!(result, Ok(_) | Err(VisionError::ResourceLimit))); let mut excessive_texture_work = scene(); excessive_texture_work.texture_fetches = VisionLimits::default().max_texture_fetches + 1; let source = Arc::new(FakeScene { scene: excessive_texture_work, pending: Mutex::new(false), }); let service = VisionService::new( source, VisionLimits { width: 64, height: 64, minimum_interval: std::time::Duration::ZERO, ..VisionLimits::default() }, ) .unwrap(); service.set_generation(7); assert_eq!( service .capture("texture-budget", 7, CancellationToken::default()) .await .unwrap_err(), VisionError::ResourceLimit ); } #[tokio::test] async fn cancellation_and_region_change_interrupt_nonblocking_capture() { let source = Arc::new(FakeScene { scene: scene(), pending: Mutex::new(true), }); let service = Arc::new( VisionService::new( source, VisionLimits { width: 64, height: 64, minimum_interval: std::time::Duration::ZERO, ..VisionLimits::default() }, ) .unwrap(), ); service.set_generation(7); let cancellation = CancellationTokenSource::new(); cancellation.cancel(); assert_eq!( service .capture("cancel", 7, cancellation.token()) .await .unwrap_err(), VisionError::Cancelled ); let service2 = service.clone(); let task = tokio::spawn(async move { service2 .capture("region", 7, CancellationToken::default()) .await }); tokio::task::yield_now().await; service.set_generation(8); assert!(matches!( task.await.unwrap(), Err(VisionError::Superseded | VisionError::Cancelled) )); service.set_generation(9); let first_service = service.clone(); let first = tokio::spawn(async move { first_service .capture("first", 9, CancellationToken::default()) .await }); tokio::task::yield_now().await; let second_cancel = CancellationTokenSource::new(); let second_service = service.clone(); let second_token = second_cancel.token(); let second = tokio::spawn(async move { second_service.capture("second", 9, second_token).await }); assert_eq!( tokio::time::timeout(std::time::Duration::from_secs(1), first) .await .unwrap() .unwrap() .unwrap_err(), VisionError::Superseded ); second_cancel.cancel(); assert_eq!(second.await.unwrap().unwrap_err(), VisionError::Cancelled); } #[test] fn explicit_visual_intent_is_narrow() { assert!(visual_question("What do you see in front of you?")); assert!(visual_question("Please look at this viewport")); assert!(!visual_question("hello there")); assert!(!visual_question("build a cube")); } struct FakeResponder { parts: Mutex, unsupported: bool, } impl InteractionResponder for FakeResponder { fn respond(&self, request: ResponseRequest, _: CancellationToken) -> ResponderFuture<'_> { Box::pin(async move { *self.parts.lock().unwrap() = request.messages.last().unwrap().content.len(); if self.unsupported { Err(InteractionModelError::MultimodalUnsupported) } else { VisibleResponse::new("seen").map_err(|_| InteractionModelError::Failed) } }) } } fn request(text: &str) -> ResponseRequest { ResponseRequest { delivery_id: BoundedText::new("id", "delivery").unwrap(), sender_id: uuid(9), session_id: BoundedText::new("session", "session").unwrap(), channel: InteractionChannel::DirectIm, origin: InteractionOrigin::AuthorizedIm, intent: InteractionIntent::Informational, capabilities: std::collections::BTreeSet::from([Capability::Informational]), messages: vec![CompletionMessage::text(MessageRole::Avatar, text).unwrap()], } } #[tokio::test] async fn responder_adds_generic_image_payload_and_returns_summary_on_capability_rejection() { let source = Arc::new(FakeScene { scene: scene(), pending: Mutex::new(false), }); let vision = Arc::new( VisionService::new( source, VisionLimits { width: 64, height: 64, minimum_interval: std::time::Duration::ZERO, ..VisionLimits::default() }, ) .unwrap(), ); vision.set_generation(7); let endpoint = Arc::new(FakeResponder { parts: Mutex::new(0), unsupported: true, }); let responder = VisionAugmentedResponder::new(vision, endpoint.clone()); let response = responder .respond(request("What do you see?"), CancellationToken::default()) .await .unwrap(); assert!(response.as_str().contains("does not support visual input")); assert!(response.as_str().contains("Reconstructed viewport")); assert_eq!(*endpoint.parts.lock().unwrap(), 3); } #[tokio::test] async fn responder_never_captures_for_ordinary_text() { let source = Arc::new(FakeScene { scene: scene(), pending: Mutex::new(false), }); let vision = Arc::new( VisionService::new( source, VisionLimits { width: 64, height: 64, minimum_interval: std::time::Duration::ZERO, ..VisionLimits::default() }, ) .unwrap(), ); vision.set_generation(7); let endpoint = Arc::new(FakeResponder { parts: Mutex::new(0), unsupported: false, }); let responder = VisionAugmentedResponder::new(vision.clone(), endpoint.clone()); assert_eq!( responder .respond(request("hello"), CancellationToken::default()) .await .unwrap() .as_str(), "seen" ); assert_eq!(*endpoint.parts.lock().unwrap(), 1); assert!(vision.observations().is_empty()); }