Implement pure Rust visual snapshots (#132)

crates/metacrate-grid-agent/src/vision_tests.rs

@@ -0,0 +1,376 @@
+use crate::vision::*;
+use crate::{
+    BoundedText, CompletionMessage, InteractionChannel, InteractionIntent, InteractionModelError,
+    InteractionOrigin, InteractionResponder, MessageRole, ResponderFuture, ResponseRequest,
+    VisibleResponse,
+};
+use libremetaverse_types::{
+    UUID,
+    compat::{CancellationToken, CancellationTokenSource},
+};
+use std::sync::{Arc, Mutex};
+
+struct FakeScene {
+    scene: SceneSnapshot,
+    pending: Mutex<bool>,
+}
+impl SceneSource for FakeScene {
+    fn capture_scene(
+        &self,
+        _: u64,
+        cancellation: CancellationToken,
+    ) -> VisionFuture<'_, SceneSnapshot> {
+        Box::pin(async move {
+            if *self.pending.lock().unwrap() {
+                tokio::select! {()=cancellation.cancelled()=>Err(VisionError::Cancelled),()=std::future::pending()=>unreachable!()}
+            } else {
+                Ok(self.scene.clone())
+            }
+        })
+    }
+}
+fn uuid(value: u32) -> UUID {
+    UUID::new_with_string(format!("00000000-0000-4000-8000-{value:012}")).unwrap()
+}
+fn triangle(z: f32, color: [u8; 4]) -> SceneTriangle {
+    SceneTriangle {
+        vertices: [[-1.0, -1.0, z], [1.0, -1.0, z], [0.0, 1.0, z]],
+        color_srgb: color,
+    }
+}
+fn scene() -> SceneSnapshot {
+    SceneSnapshot {
+        generation: 7,
+        observed_unix_millis: 1_700_000_000_000,
+        region_id: uuid(1),
+        region_name: "Fixture Region".into(),
+        camera: CameraPose {
+            position: [0.0, 0.0, 0.0],
+            forward: [0.0, 0.0, 1.0],
+            up: [0.0, 1.0, 0.0],
+            vertical_fov_degrees: 60.0,
+        },
+        entities: vec![
+            SceneEntity {
+                id: uuid(3),
+                kind: SceneEntityKind::Resident,
+                display_name: "Private Resident".into(),
+                triangles: vec![triangle(4.0, [255, 0, 0, 255])],
+                texture_available: false,
+            },
+            SceneEntity {
+                id: uuid(2),
+                kind: SceneEntityKind::Object,
+                display_name: "Near".into(),
+                triangles: vec![triangle(2.0, [0, 255, 0, 255])],
+                texture_available: true,
+            },
+        ],
+        completeness: SnapshotCompleteness {
+            textures_missing: 1,
+            terrain_available: false,
+            ..SnapshotCompleteness::default()
+        },
+        texture_fetches: 1,
+        texture_bytes: 128,
+        decoded_texture_pixels: 16,
+    }
+}
+
+#[tokio::test]
+async fn golden_scene_is_deterministic_depth_ordered_and_privacy_marked() {
+    let source = Arc::new(FakeScene {
+        scene: scene(),
+        pending: Mutex::new(false),
+    });
+    let service = VisionService::new(
+        source,
+        VisionLimits {
+            width: 64,
+            height: 64,
+            minimum_interval: std::time::Duration::ZERO,
+            ..VisionLimits::default()
+        },
+    )
+    .unwrap();
+    service.set_generation(7);
+    let first = service
+        .capture("one", 7, CancellationToken::default())
+        .await
+        .unwrap();
+    let second = service
+        .capture("two", 7, CancellationToken::default())
+        .await
+        .unwrap();
+    assert_eq!(first.png, second.png);
+    assert_eq!(first.image_sha256, second.image_sha256);
+    assert_eq!(
+        first.image_sha256,
+        "e943b14142da05fcc1471ee0b639ffc1c11c74bfde83432ad3a26f994e4e665a"
+    );
+    assert!(first.summary.contains("privacy-marked residents"));
+    assert!(!first.summary.contains("Private Resident"));
+    assert!(first.summary.contains("textures_missing=1"));
+    assert!(first.data_url.starts_with("data:image/png;base64,"));
+    let decoder = png::Decoder::new(first.png.as_slice());
+    let mut reader = decoder.read_info().unwrap();
+    assert_eq!(
+        reader.info().srgb,
+        Some(png::SrgbRenderingIntent::Perceptual)
+    );
+    let mut pixels = vec![0; reader.output_buffer_size()];
+    let info = reader.next_frame(&mut pixels).unwrap();
+    let center = ((info.height as usize / 2) * info.width as usize + info.width as usize / 2) * 4;
+    assert_eq!(&pixels[center..center + 4], &[0, 255, 0, 255]);
+}
+
+#[tokio::test]
+async fn invalid_camera_huge_scene_stale_generation_and_size_limit_fail_closed() {
+    let mut bad = scene();
+    bad.camera.forward = [0.0, 0.0, 0.0];
+    let source = Arc::new(FakeScene {
+        scene: bad,
+        pending: Mutex::new(false),
+    });
+    let service = VisionService::new(
+        source,
+        VisionLimits {
+            width: 64,
+            height: 64,
+            minimum_interval: std::time::Duration::ZERO,
+            ..VisionLimits::default()
+        },
+    )
+    .unwrap();
+    service.set_generation(7);
+    assert_eq!(
+        service
+            .capture("bad", 7, CancellationToken::default())
+            .await
+            .unwrap_err(),
+        VisionError::InvalidScene
+    );
+    assert_eq!(
+        service
+            .capture("stale", 6, CancellationToken::default())
+            .await
+            .unwrap_err(),
+        VisionError::StaleGeneration
+    );
+    let source = Arc::new(FakeScene {
+        scene: scene(),
+        pending: Mutex::new(false),
+    });
+    let service = VisionService::new(
+        source,
+        VisionLimits {
+            width: 64,
+            height: 64,
+            max_png_bytes: 4096,
+            minimum_interval: std::time::Duration::ZERO,
+            ..VisionLimits::default()
+        },
+    )
+    .unwrap();
+    service.set_generation(7);
+    let result = service
+        .capture("size", 7, CancellationToken::default())
+        .await;
+    assert!(matches!(result, Ok(_) | Err(VisionError::ResourceLimit)));
+
+    let mut excessive_texture_work = scene();
+    excessive_texture_work.texture_fetches = VisionLimits::default().max_texture_fetches + 1;
+    let source = Arc::new(FakeScene {
+        scene: excessive_texture_work,
+        pending: Mutex::new(false),
+    });
+    let service = VisionService::new(
+        source,
+        VisionLimits {
+            width: 64,
+            height: 64,
+            minimum_interval: std::time::Duration::ZERO,
+            ..VisionLimits::default()
+        },
+    )
+    .unwrap();
+    service.set_generation(7);
+    assert_eq!(
+        service
+            .capture("texture-budget", 7, CancellationToken::default())
+            .await
+            .unwrap_err(),
+        VisionError::ResourceLimit
+    );
+}
+
+#[tokio::test]
+async fn cancellation_and_region_change_interrupt_nonblocking_capture() {
+    let source = Arc::new(FakeScene {
+        scene: scene(),
+        pending: Mutex::new(true),
+    });
+    let service = Arc::new(
+        VisionService::new(
+            source,
+            VisionLimits {
+                width: 64,
+                height: 64,
+                minimum_interval: std::time::Duration::ZERO,
+                ..VisionLimits::default()
+            },
+        )
+        .unwrap(),
+    );
+    service.set_generation(7);
+    let cancellation = CancellationTokenSource::new();
+    cancellation.cancel();
+    assert_eq!(
+        service
+            .capture("cancel", 7, cancellation.token())
+            .await
+            .unwrap_err(),
+        VisionError::Cancelled
+    );
+    let service2 = service.clone();
+    let task = tokio::spawn(async move {
+        service2
+            .capture("region", 7, CancellationToken::default())
+            .await
+    });
+    tokio::task::yield_now().await;
+    service.set_generation(8);
+    assert!(matches!(
+        task.await.unwrap(),
+        Err(VisionError::Superseded | VisionError::Cancelled)
+    ));
+
+    service.set_generation(9);
+    let first_service = service.clone();
+    let first = tokio::spawn(async move {
+        first_service
+            .capture("first", 9, CancellationToken::default())
+            .await
+    });
+    tokio::task::yield_now().await;
+    let second_cancel = CancellationTokenSource::new();
+    let second_service = service.clone();
+    let second_token = second_cancel.token();
+    let second =
+        tokio::spawn(async move { second_service.capture("second", 9, second_token).await });
+    assert_eq!(
+        tokio::time::timeout(std::time::Duration::from_secs(1), first)
+            .await
+            .unwrap()
+            .unwrap()
+            .unwrap_err(),
+        VisionError::Superseded
+    );
+    second_cancel.cancel();
+    assert_eq!(second.await.unwrap().unwrap_err(), VisionError::Cancelled);
+}
+
+#[test]
+fn explicit_visual_intent_is_narrow() {
+    assert!(visual_question("What do you see in front of you?"));
+    assert!(visual_question("Please look at this viewport"));
+    assert!(!visual_question("hello there"));
+    assert!(!visual_question("build a cube"));
+}
+
+struct FakeResponder {
+    parts: Mutex<usize>,
+    unsupported: bool,
+}
+impl InteractionResponder for FakeResponder {
+    fn respond(&self, request: ResponseRequest, _: CancellationToken) -> ResponderFuture<'_> {
+        Box::pin(async move {
+            *self.parts.lock().unwrap() = request.messages.last().unwrap().content.len();
+            if self.unsupported {
+                Err(InteractionModelError::MultimodalUnsupported)
+            } else {
+                VisibleResponse::new("seen").map_err(|_| InteractionModelError::Failed)
+            }
+        })
+    }
+}
+fn request(text: &str) -> ResponseRequest {
+    ResponseRequest {
+        delivery_id: BoundedText::new("id", "delivery").unwrap(),
+        sender_id: uuid(9),
+        session_id: BoundedText::new("session", "session").unwrap(),
+        channel: InteractionChannel::DirectIm,
+        origin: InteractionOrigin::AuthorizedIm,
+        intent: InteractionIntent::Informational,
+        messages: vec![CompletionMessage::text(MessageRole::Avatar, text).unwrap()],
+    }
+}
+
+#[tokio::test]
+async fn responder_adds_generic_image_payload_and_returns_summary_on_capability_rejection() {
+    let source = Arc::new(FakeScene {
+        scene: scene(),
+        pending: Mutex::new(false),
+    });
+    let vision = Arc::new(
+        VisionService::new(
+            source,
+            VisionLimits {
+                width: 64,
+                height: 64,
+                minimum_interval: std::time::Duration::ZERO,
+                ..VisionLimits::default()
+            },
+        )
+        .unwrap(),
+    );
+    vision.set_generation(7);
+    let endpoint = Arc::new(FakeResponder {
+        parts: Mutex::new(0),
+        unsupported: true,
+    });
+    let responder = VisionAugmentedResponder::new(vision, endpoint.clone());
+    let response = responder
+        .respond(request("What do you see?"), CancellationToken::default())
+        .await
+        .unwrap();
+    assert!(response.as_str().contains("does not support visual input"));
+    assert!(response.as_str().contains("Synthetic viewport"));
+    assert_eq!(*endpoint.parts.lock().unwrap(), 3);
+}
+
+#[tokio::test]
+async fn responder_never_captures_for_ordinary_text() {
+    let source = Arc::new(FakeScene {
+        scene: scene(),
+        pending: Mutex::new(false),
+    });
+    let vision = Arc::new(
+        VisionService::new(
+            source,
+            VisionLimits {
+                width: 64,
+                height: 64,
+                minimum_interval: std::time::Duration::ZERO,
+                ..VisionLimits::default()
+            },
+        )
+        .unwrap(),
+    );
+    vision.set_generation(7);
+    let endpoint = Arc::new(FakeResponder {
+        parts: Mutex::new(0),
+        unsupported: false,
+    });
+    let responder = VisionAugmentedResponder::new(vision.clone(), endpoint.clone());
+    assert_eq!(
+        responder
+            .respond(request("hello"), CancellationToken::default())
+            .await
+            .unwrap()
+            .as_str(),
+        "seen"
+    );
+    assert_eq!(*endpoint.parts.lock().unwrap(), 1);
+    assert!(vision.observations().is_empty());
+}