395 lines
12 KiB
Rust
395 lines
12 KiB
Rust
use crate::vision::*;
|
|
use crate::{
|
|
BoundedText, Capability, CompletionMessage, InteractionChannel, InteractionIntent,
|
|
InteractionModelError, InteractionOrigin, InteractionResponder, MessageRole, ResponderFuture,
|
|
ResponseRequest, VisibleResponse,
|
|
};
|
|
use libremetaverse_types::{
|
|
UUID,
|
|
compat::{CancellationToken, CancellationTokenSource},
|
|
};
|
|
use std::sync::{Arc, Mutex};
|
|
|
|
struct FakeScene {
|
|
scene: SceneSnapshot,
|
|
pending: Mutex<bool>,
|
|
}
|
|
impl SceneSource for FakeScene {
|
|
fn capture_scene(
|
|
&self,
|
|
_: u64,
|
|
cancellation: CancellationToken,
|
|
) -> VisionFuture<'_, SceneSnapshot> {
|
|
Box::pin(async move {
|
|
if *self.pending.lock().unwrap() {
|
|
tokio::select! {()=cancellation.cancelled()=>Err(VisionError::Cancelled),()=std::future::pending()=>unreachable!()}
|
|
} else {
|
|
Ok(self.scene.clone())
|
|
}
|
|
})
|
|
}
|
|
}
|
|
fn uuid(value: u32) -> UUID {
|
|
UUID::new_with_string(format!("00000000-0000-4000-8000-{value:012}")).unwrap()
|
|
}
|
|
fn triangle(z: f32, color: [u8; 4]) -> SceneTriangle {
|
|
SceneTriangle {
|
|
vertices: [[-1.0, -1.0, z], [1.0, -1.0, z], [0.0, 1.0, z]],
|
|
color_srgb: color,
|
|
}
|
|
}
|
|
fn scene() -> SceneSnapshot {
|
|
SceneSnapshot {
|
|
generation: 7,
|
|
observed_unix_millis: 1_700_000_000_000,
|
|
region_id: uuid(1),
|
|
region_name: "Fixture Region".into(),
|
|
camera: CameraPose {
|
|
position: [0.0, 0.0, 0.0],
|
|
forward: [0.0, 0.0, 1.0],
|
|
up: [0.0, 1.0, 0.0],
|
|
vertical_fov_degrees: 60.0,
|
|
},
|
|
entities: vec![
|
|
SceneEntity {
|
|
id: uuid(3),
|
|
kind: SceneEntityKind::Resident,
|
|
display_name: "Private Resident".into(),
|
|
triangles: vec![triangle(4.0, [255, 0, 0, 255])],
|
|
texture_available: false,
|
|
},
|
|
SceneEntity {
|
|
id: uuid(2),
|
|
kind: SceneEntityKind::Object,
|
|
display_name: "Near".into(),
|
|
triangles: vec![triangle(2.0, [0, 255, 0, 255])],
|
|
texture_available: true,
|
|
},
|
|
],
|
|
completeness: SnapshotCompleteness {
|
|
textures_missing: 1,
|
|
terrain_available: false,
|
|
..SnapshotCompleteness::default()
|
|
},
|
|
texture_fetches: 1,
|
|
texture_bytes: 128,
|
|
decoded_texture_pixels: 16,
|
|
}
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn golden_scene_is_deterministic_depth_ordered_and_privacy_marked() {
|
|
let source = Arc::new(FakeScene {
|
|
scene: scene(),
|
|
pending: Mutex::new(false),
|
|
});
|
|
let service = VisionService::new(
|
|
source,
|
|
VisionLimits {
|
|
width: 64,
|
|
height: 64,
|
|
minimum_interval: std::time::Duration::ZERO,
|
|
..VisionLimits::default()
|
|
},
|
|
)
|
|
.unwrap();
|
|
service.set_generation(7);
|
|
let first = service
|
|
.capture("one", 7, CancellationToken::default())
|
|
.await
|
|
.unwrap();
|
|
let second = service
|
|
.capture("two", 7, CancellationToken::default())
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(first.jpeg, second.jpeg);
|
|
assert_eq!(first.image_sha256, second.image_sha256);
|
|
assert_eq!(
|
|
first.image_sha256,
|
|
"b53d23f57ba0d6fb79856afde111ad6dc25559e810dcc028a5f107090914821d"
|
|
);
|
|
assert!(first.summary.contains("privacy-marked residents"));
|
|
assert!(!first.summary.contains("Private Resident"));
|
|
assert!(first.summary.contains("textures_missing=1"));
|
|
assert!(first.data_url.starts_with("data:image/jpeg;base64,"));
|
|
let mut decoder = jpeg_decoder::Decoder::new(first.jpeg.as_slice());
|
|
let pixels = decoder.decode().unwrap();
|
|
let info = decoder.info().unwrap();
|
|
assert_eq!((info.width, info.height), (64, 64));
|
|
let center = ((info.height as usize / 2) * info.width as usize + info.width as usize / 2) * 3;
|
|
assert!(pixels[center + 1] > 180 && pixels[center] < 80 && pixels[center + 2] < 80);
|
|
}
|
|
|
|
#[cfg(feature = "live-grid")]
|
|
#[tokio::test(flavor = "multi_thread")]
|
|
async fn offscreen_wgpu_renders_depth_ordered_scene_when_an_adapter_is_available() {
|
|
let Ok(renderer) = WgpuRenderer::new().await else {
|
|
return;
|
|
};
|
|
let limits = VisionLimits {
|
|
width: 64,
|
|
height: 64,
|
|
minimum_interval: std::time::Duration::ZERO,
|
|
..VisionLimits::default()
|
|
};
|
|
let rgba = tokio::task::spawn_blocking(move || renderer.render(&scene(), limits))
|
|
.await
|
|
.expect("render worker")
|
|
.expect("offscreen render");
|
|
assert_eq!(rgba.len(), 64 * 64 * 4);
|
|
let center = (32 * 64 + 32) * 4;
|
|
assert!(rgba[center + 1] > 180 && rgba[center] < 80 && rgba[center + 2] < 80);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn invalid_camera_huge_scene_stale_generation_and_size_limit_fail_closed() {
|
|
let mut bad = scene();
|
|
bad.camera.forward = [0.0, 0.0, 0.0];
|
|
let source = Arc::new(FakeScene {
|
|
scene: bad,
|
|
pending: Mutex::new(false),
|
|
});
|
|
let service = VisionService::new(
|
|
source,
|
|
VisionLimits {
|
|
width: 64,
|
|
height: 64,
|
|
minimum_interval: std::time::Duration::ZERO,
|
|
..VisionLimits::default()
|
|
},
|
|
)
|
|
.unwrap();
|
|
service.set_generation(7);
|
|
assert_eq!(
|
|
service
|
|
.capture("bad", 7, CancellationToken::default())
|
|
.await
|
|
.unwrap_err(),
|
|
VisionError::InvalidScene
|
|
);
|
|
assert_eq!(
|
|
service
|
|
.capture("stale", 6, CancellationToken::default())
|
|
.await
|
|
.unwrap_err(),
|
|
VisionError::StaleGeneration
|
|
);
|
|
let source = Arc::new(FakeScene {
|
|
scene: scene(),
|
|
pending: Mutex::new(false),
|
|
});
|
|
let service = VisionService::new(
|
|
source,
|
|
VisionLimits {
|
|
width: 64,
|
|
height: 64,
|
|
max_jpeg_bytes: 4096,
|
|
minimum_interval: std::time::Duration::ZERO,
|
|
..VisionLimits::default()
|
|
},
|
|
)
|
|
.unwrap();
|
|
service.set_generation(7);
|
|
let result = service
|
|
.capture("size", 7, CancellationToken::default())
|
|
.await;
|
|
assert!(matches!(result, Ok(_) | Err(VisionError::ResourceLimit)));
|
|
|
|
let mut excessive_texture_work = scene();
|
|
excessive_texture_work.texture_fetches = VisionLimits::default().max_texture_fetches + 1;
|
|
let source = Arc::new(FakeScene {
|
|
scene: excessive_texture_work,
|
|
pending: Mutex::new(false),
|
|
});
|
|
let service = VisionService::new(
|
|
source,
|
|
VisionLimits {
|
|
width: 64,
|
|
height: 64,
|
|
minimum_interval: std::time::Duration::ZERO,
|
|
..VisionLimits::default()
|
|
},
|
|
)
|
|
.unwrap();
|
|
service.set_generation(7);
|
|
assert_eq!(
|
|
service
|
|
.capture("texture-budget", 7, CancellationToken::default())
|
|
.await
|
|
.unwrap_err(),
|
|
VisionError::ResourceLimit
|
|
);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn cancellation_and_region_change_interrupt_nonblocking_capture() {
|
|
let source = Arc::new(FakeScene {
|
|
scene: scene(),
|
|
pending: Mutex::new(true),
|
|
});
|
|
let service = Arc::new(
|
|
VisionService::new(
|
|
source,
|
|
VisionLimits {
|
|
width: 64,
|
|
height: 64,
|
|
minimum_interval: std::time::Duration::ZERO,
|
|
..VisionLimits::default()
|
|
},
|
|
)
|
|
.unwrap(),
|
|
);
|
|
service.set_generation(7);
|
|
let cancellation = CancellationTokenSource::new();
|
|
cancellation.cancel();
|
|
assert_eq!(
|
|
service
|
|
.capture("cancel", 7, cancellation.token())
|
|
.await
|
|
.unwrap_err(),
|
|
VisionError::Cancelled
|
|
);
|
|
let service2 = service.clone();
|
|
let task = tokio::spawn(async move {
|
|
service2
|
|
.capture("region", 7, CancellationToken::default())
|
|
.await
|
|
});
|
|
tokio::task::yield_now().await;
|
|
service.set_generation(8);
|
|
assert!(matches!(
|
|
task.await.unwrap(),
|
|
Err(VisionError::Superseded | VisionError::Cancelled)
|
|
));
|
|
|
|
service.set_generation(9);
|
|
let first_service = service.clone();
|
|
let first = tokio::spawn(async move {
|
|
first_service
|
|
.capture("first", 9, CancellationToken::default())
|
|
.await
|
|
});
|
|
tokio::task::yield_now().await;
|
|
let second_cancel = CancellationTokenSource::new();
|
|
let second_service = service.clone();
|
|
let second_token = second_cancel.token();
|
|
let second =
|
|
tokio::spawn(async move { second_service.capture("second", 9, second_token).await });
|
|
assert_eq!(
|
|
tokio::time::timeout(std::time::Duration::from_secs(1), first)
|
|
.await
|
|
.unwrap()
|
|
.unwrap()
|
|
.unwrap_err(),
|
|
VisionError::Superseded
|
|
);
|
|
second_cancel.cancel();
|
|
assert_eq!(second.await.unwrap().unwrap_err(), VisionError::Cancelled);
|
|
}
|
|
|
|
#[test]
|
|
fn explicit_visual_intent_is_narrow() {
|
|
assert!(visual_question("What do you see in front of you?"));
|
|
assert!(visual_question("Please look at this viewport"));
|
|
assert!(!visual_question("hello there"));
|
|
assert!(!visual_question("build a cube"));
|
|
}
|
|
|
|
struct FakeResponder {
|
|
parts: Mutex<usize>,
|
|
unsupported: bool,
|
|
}
|
|
impl InteractionResponder for FakeResponder {
|
|
fn respond(&self, request: ResponseRequest, _: CancellationToken) -> ResponderFuture<'_> {
|
|
Box::pin(async move {
|
|
*self.parts.lock().unwrap() = request.messages.last().unwrap().content.len();
|
|
if self.unsupported {
|
|
Err(InteractionModelError::MultimodalUnsupported)
|
|
} else {
|
|
VisibleResponse::new("seen").map_err(|_| InteractionModelError::Failed)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
fn request(text: &str) -> ResponseRequest {
|
|
ResponseRequest {
|
|
delivery_id: BoundedText::new("id", "delivery").unwrap(),
|
|
sender_id: uuid(9),
|
|
session_id: BoundedText::new("session", "session").unwrap(),
|
|
channel: InteractionChannel::DirectIm,
|
|
origin: InteractionOrigin::AuthorizedIm,
|
|
intent: InteractionIntent::Informational,
|
|
capabilities: std::collections::BTreeSet::from([Capability::Informational]),
|
|
messages: vec![CompletionMessage::text(MessageRole::Avatar, text).unwrap()],
|
|
}
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn responder_adds_generic_image_payload_and_returns_summary_on_capability_rejection() {
|
|
let source = Arc::new(FakeScene {
|
|
scene: scene(),
|
|
pending: Mutex::new(false),
|
|
});
|
|
let vision = Arc::new(
|
|
VisionService::new(
|
|
source,
|
|
VisionLimits {
|
|
width: 64,
|
|
height: 64,
|
|
minimum_interval: std::time::Duration::ZERO,
|
|
..VisionLimits::default()
|
|
},
|
|
)
|
|
.unwrap(),
|
|
);
|
|
vision.set_generation(7);
|
|
let endpoint = Arc::new(FakeResponder {
|
|
parts: Mutex::new(0),
|
|
unsupported: true,
|
|
});
|
|
let responder = VisionAugmentedResponder::new(vision, endpoint.clone());
|
|
let response = responder
|
|
.respond(request("What do you see?"), CancellationToken::default())
|
|
.await
|
|
.unwrap();
|
|
assert!(response.as_str().contains("does not support visual input"));
|
|
assert!(response.as_str().contains("Reconstructed viewport"));
|
|
assert_eq!(*endpoint.parts.lock().unwrap(), 3);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn responder_never_captures_for_ordinary_text() {
|
|
let source = Arc::new(FakeScene {
|
|
scene: scene(),
|
|
pending: Mutex::new(false),
|
|
});
|
|
let vision = Arc::new(
|
|
VisionService::new(
|
|
source,
|
|
VisionLimits {
|
|
width: 64,
|
|
height: 64,
|
|
minimum_interval: std::time::Duration::ZERO,
|
|
..VisionLimits::default()
|
|
},
|
|
)
|
|
.unwrap(),
|
|
);
|
|
vision.set_generation(7);
|
|
let endpoint = Arc::new(FakeResponder {
|
|
parts: Mutex::new(0),
|
|
unsupported: false,
|
|
});
|
|
let responder = VisionAugmentedResponder::new(vision.clone(), endpoint.clone());
|
|
assert_eq!(
|
|
responder
|
|
.respond(request("hello"), CancellationToken::default())
|
|
.await
|
|
.unwrap()
|
|
.as_str(),
|
|
"seen"
|
|
);
|
|
assert_eq!(*endpoint.parts.lock().unwrap(), 1);
|
|
assert!(vision.observations().is_empty());
|
|
}
|