Complete Bevy OpenSim scene rendering

crates/metacrate-grid-agent/tests/dependency_policy.rs

@@ -104,7 +104,8 @@ fn vision_path_selects_only_reviewed_rust_renderers_and_codec() {
     assert!(manifest.contains("metacrate-rendering-wgpu ="));
     let renderer = fs::read_to_string(root.join("../metacrate-rendering-wgpu/Cargo.toml"))
         .expect("wgpu renderer manifest");
-    assert!(renderer.contains("wgpu = { version = \"30.0.1\""));
+    assert!(renderer.contains("bevy = { version = \"0.19.1\", default-features = false"));
+    assert!(renderer.contains("wgpu = [\"bevy-engine\"]"));
 }
 
 #[test]

crates/metacrate-grid-agent/tests/live_llm_vision.rs

@@ -0,0 +1,91 @@
+use base64::Engine as _;
+use mentra::{BuiltinProvider, ContentBlock, ModelInfo, Runtime};
+use metacrate_grid_agent::{AgentConfig, LlmClient};
+use std::{collections::BTreeMap, error::Error, path::PathBuf};
+
+#[tokio::test]
+#[ignore = "sends the live renderer JPEG to the configured vision model through Mentra SSE"]
+async fn luna_describes_live_renderer_evidence() -> Result<(), Box<dyn Error>> {
+    let environment = live_environment()?;
+    let mut connection = AgentConfig::offline(
+        required(&environment, "OPENAPI_URL")?,
+        required(&environment, "OPENAPI_KEY")?,
+    )?
+    .llm;
+    connection.model = Some(required(&environment, "OPENAPI_MODEL")?.to_owned());
+    let client = LlmClient::new(connection);
+    let runtime = Runtime::empty_builder()
+        .with_store(mentra::runtime::VolatileRuntimeStore::default())
+        .with_registered_provider(client.mentra_provider())
+        .build()?;
+    let root = std::env::temp_dir().join(format!("metacrate-live-vision-{}", std::process::id()));
+    let mut config = mentra::AgentConfig {
+        system: Some("You are in a virtual world.".to_owned()),
+        ..Default::default()
+    };
+    config.compaction.transcript_dir = root.join("transcripts");
+    config.task.tasks_dir = root.join("tasks");
+    config.team.team_dir = root.join("teams");
+    config.workspace.base_dir = root;
+    let mut agent = runtime.spawn_with_config(
+        "live-render-review",
+        ModelInfo::new(client.configured_model(), BuiltinProvider::OpenAI),
+        config,
+    )?;
+    let jpeg = std::fs::read(
+        std::env::var_os("METACRATE_LIVE_RENDER_OUTPUT").map_or_else(
+            || std::env::temp_dir().join("metacrate-live-render.jpg"),
+            PathBuf::from,
+        ),
+    )?;
+    let image = format!(
+        "data:image/jpeg;base64,{}",
+        base64::engine::general_purpose::STANDARD.encode(jpeg)
+    );
+    let response = agent
+        .send(vec![
+            ContentBlock::text(
+                "Describe this rendered virtual-world scene. State whether it contains recognizable textured terrain and mesh scenery, and identify any obvious rendering corruption.",
+            ),
+            ContentBlock::image_url(image),
+        ])
+        .await?;
+    for block in response.content {
+        if let ContentBlock::Text { text } = block {
+            println!("LIVE_LLM_VISION={text}");
+        }
+    }
+    Ok(())
+}
+
+fn live_environment() -> Result<BTreeMap<String, String>, Box<dyn Error>> {
+    let mut values = std::env::vars().collect::<BTreeMap<_, _>>();
+    let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../.env");
+    for line in std::fs::read_to_string(path)?.lines() {
+        let line = line.trim();
+        if line.is_empty() || line.starts_with('#') {
+            continue;
+        }
+        let Some((key, value)) = line.split_once('=') else {
+            continue;
+        };
+        values.entry(key.trim().to_owned()).or_insert_with(|| {
+            value
+                .trim()
+                .trim_matches(|character| character == '\'' || character == '"')
+                .to_owned()
+        });
+    }
+    Ok(values)
+}
+
+fn required<'a>(
+    environment: &'a BTreeMap<String, String>,
+    key: &'static str,
+) -> Result<&'a str, Box<dyn Error>> {
+    environment
+        .get(key)
+        .filter(|value| !value.is_empty())
+        .map(String::as_str)
+        .ok_or_else(|| format!("missing live test setting {key}").into())
+}

crates/metacrate-grid-agent/tests/live_renderer.rs

@@ -0,0 +1,443 @@
+#![cfg(feature = "live-grid")]
+
+use libremetaverse_types::{Vector3, compat::CancellationTokenSource};
+use metacrate_grid_agent::{
+    EndpointUrl, GridConnection, GridSession, GridSessionBackend, LibremetaverseClientOwner,
+    LibremetaverseSceneSource, SecretString, SessionSignal, VisionLimits, VisionService,
+};
+use std::{
+    collections::BTreeMap,
+    error::Error,
+    path::PathBuf,
+    sync::Arc,
+    time::{Duration, Instant},
+};
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+#[ignore = "logs two dedicated accounts into the configured live grid and writes a JPEG"]
+#[allow(clippy::too_many_lines)] // The one end-to-end live workflow stays readable in execution order.
+async fn live_bevy_capture_renders_the_target_accounts_scene() -> Result<(), Box<dyn Error>> {
+    let environment = live_environment()?;
+    let target_owner = LibremetaverseClientOwner::new()?;
+    let renderer_owner = LibremetaverseClientOwner::new()?;
+    let (mut target_session, target_cancel) = login(
+        &target_owner,
+        &environment,
+        "GRID_TEST_UNPRIVILEGED_USER",
+        "GRID_TEST_UNPRIVILEGED_PASSWORD",
+    )
+    .await?;
+    let (mut renderer_session, renderer_cancel) = match login(
+        &renderer_owner,
+        &environment,
+        "GRID_TEST_AUTHORIZED_USER",
+        "GRID_TEST_AUTHORIZED_PASSWORD",
+    )
+    .await
+    {
+        Ok(login) => login,
+        Err(error) => {
+            let _ = target_session.logout(target_cancel.token()).await;
+            return Err(error);
+        }
+    };
+
+    let result = async {
+        wait_ready("target", &mut target_session, &target_cancel).await?;
+        wait_ready("renderer", &mut renderer_session, &renderer_cancel).await?;
+        let renderer_start_simulator = renderer_owner
+            .client()
+            .network()
+            .current_sim()
+            .ok_or("renderer start simulator unavailable")?;
+        let renderer_start_handle = renderer_start_simulator.handle;
+        let renderer_start_position = conventional_start(&renderer_start_simulator);
+        let target_simulator = target_owner
+            .client()
+            .network()
+            .current_sim()
+            .ok_or("target simulator unavailable")?;
+        let target_position = target_owner.agent().sim_position();
+        let arrival = Vector3 {
+            x: target_position.x - 10.0,
+            y: target_position.y - 10.0,
+            z: target_position.z + 2.0,
+        };
+        if !renderer_owner
+            .agent()
+            .teleport_with_u_int64_vector3_vector3_cancellation_token(
+                target_simulator.handle,
+                arrival,
+                target_position,
+                Some(renderer_cancel.token()),
+            )
+            .await?
+        {
+            return Err("teleport did not complete".into());
+        }
+        tokio::time::sleep(Duration::from_secs(5)).await;
+        if !renderer_owner
+            .agent()
+            .teleport_with_u_int64_vector3_vector3_cancellation_token(
+                renderer_start_handle,
+                renderer_start_position,
+                Vector3 {
+                    x: renderer_start_position.x + 10.0,
+                    y: renderer_start_position.y,
+                    z: renderer_start_position.z,
+                },
+                Some(renderer_cancel.token()),
+            )
+            .await?
+        {
+            return Err("return teleport did not complete".into());
+        }
+        tokio::time::sleep(Duration::from_secs(5)).await;
+        let mut renderer_simulator = renderer_owner
+            .client()
+            .network()
+            .current_sim()
+            .ok_or("renderer simulator unavailable")?;
+        let mut settled_position = renderer_owner.agent().sim_position();
+        if let Some(ground) = terrain_ground(
+            &renderer_simulator,
+            renderer_start_position.x,
+            renderer_start_position.y,
+        ) {
+            let destination = Vector3 {
+                x: renderer_start_position.x,
+                y: renderer_start_position.y,
+                z: ground + 2.0,
+            };
+            if (settled_position.z - destination.z).abs() > 1.0 {
+                if !renderer_owner
+                    .agent()
+                    .teleport_with_u_int64_vector3_vector3_cancellation_token(
+                        renderer_start_handle,
+                        destination,
+                        Vector3 {
+                            x: destination.x + 10.0,
+                            ..destination
+                        },
+                        Some(renderer_cancel.token()),
+                    )
+                    .await?
+                {
+                    return Err("ground-settle teleport did not complete".into());
+                }
+                tokio::time::sleep(Duration::from_secs(5)).await;
+                renderer_simulator = renderer_owner
+                    .client()
+                    .network()
+                    .current_sim()
+                    .ok_or("settled renderer simulator unavailable")?;
+                settled_position = renderer_owner.agent().sim_position();
+            }
+        }
+        wait_scene_settled(&renderer_owner).await?;
+        let mut view_forward = Vector3::unit_x();
+        if let Some((destination, forward)) = scenic_location(&renderer_simulator) {
+            if !renderer_owner
+                .agent()
+                .teleport_with_u_int64_vector3_vector3_cancellation_token(
+                    renderer_start_handle,
+                    destination,
+                    Vector3 {
+                        x: destination.x + forward.x,
+                        y: destination.y + forward.y,
+                        z: destination.z + forward.z,
+                    },
+                    Some(renderer_cancel.token()),
+                )
+                .await?
+            {
+                return Err("scenic teleport did not complete".into());
+            }
+            wait_scene_settled(&renderer_owner).await?;
+            renderer_simulator = renderer_owner
+                .client()
+                .network()
+                .current_sim()
+                .ok_or("scenic renderer simulator unavailable")?;
+            settled_position = renderer_owner.agent().sim_position();
+            view_forward = forward;
+        }
+        {
+            let prims = renderer_simulator
+                .objects_primitives
+                .read()
+                .unwrap_or_else(std::sync::PoisonError::into_inner);
+            let largest_scale = prims.values().fold(0.0_f32, |largest, prim| {
+                largest.max(prim.scale.x.max(prim.scale.y).max(prim.scale.z))
+            });
+            println!(
+                "LIVE_SCENE_CACHE=prims:{} roots:{} children:{} attachments:{} sculpted:{} largest_scale:{largest_scale:.2}",
+                prims.len(),
+                prims.values().filter(|prim| prim.parent_id == 0).count(),
+                prims.values().filter(|prim| prim.parent_id != 0).count(),
+                prims.values().filter(|prim| prim.is_attachment).count(),
+                prims.values().filter(|prim| prim.sculpt.is_some()).count(),
+            );
+            println!(
+                "LIVE_SCENE_GEOMETRY=mesh:{} sculpt:{} legacy:{}",
+                prims
+                    .values()
+                    .filter(|prim| prim.sculpt.as_ref().is_some_and(|sculpt| sculpt.type_() == libremetaverse_types::SculptType::Mesh))
+                    .count(),
+                prims
+                    .values()
+                    .filter(|prim| prim.sculpt.as_ref().is_some_and(|sculpt| sculpt.type_() != libremetaverse_types::SculptType::Mesh))
+                    .count(),
+                prims.values().filter(|prim| prim.sculpt.is_none()).count(),
+            );
+        }
+        let (camera_position, camera_target) = avatar_camera(settled_position, view_forward)?;
+        renderer_owner
+            .agent()
+            .movement
+            .camera
+            .look_at_with_vector3_vector3(camera_position, camera_target)?;
+
+        let limits = VisionLimits {
+            minimum_interval: Duration::ZERO,
+            ..VisionLimits::default()
+        };
+        let vision = VisionService::new(
+            Arc::new(LibremetaverseSceneSource::new(&renderer_owner, limits)),
+            limits,
+        )?
+        .prefer_gpu();
+        vision.set_generation(1);
+        let mut capture = None;
+        let output = std::env::var_os("METACRATE_LIVE_RENDER_OUTPUT").map_or_else(
+            || std::env::temp_dir().join("metacrate-live-render.jpg"),
+            PathBuf::from,
+        );
+        let frames = std::env::var("METACRATE_LIVE_RENDER_FRAMES")
+            .ok()
+            .and_then(|value| value.parse::<u32>().ok())
+            .unwrap_or(8)
+            .clamp(1, 16);
+        for frame in 1..=frames {
+            let next = vision
+                .capture(
+                    &format!("live-renderer-evidence-{frame}"),
+                    1,
+                    renderer_cancel.token(),
+                )
+                .await?;
+            println!("LIVE_RENDER_FRAME_{frame}_SUMMARY={}", next.summary);
+            std::fs::write(&output, &next.jpeg)?;
+            capture = Some(next);
+        }
+        let capture = capture.ok_or("live renderer produced no frame")?;
+        println!("LIVE_RENDER_OUTPUT={}", output.display());
+        println!("LIVE_RENDER_SHA256={}", capture.image_sha256);
+        Ok::<(), Box<dyn Error>>(())
+    }
+    .await;
+
+    let _ = renderer_session.logout(renderer_cancel.token()).await;
+    let _ = target_session.logout(target_cancel.token()).await;
+    result
+}
+
+async fn login(
+    owner: &LibremetaverseClientOwner,
+    environment: &BTreeMap<String, String>,
+    user: &'static str,
+    password: &'static str,
+) -> Result<(Box<dyn GridSession>, CancellationTokenSource), Box<dyn Error>> {
+    let cancellation = CancellationTokenSource::new();
+    let backend = owner.session_backend(GridConnection {
+        login_url: EndpointUrl::parse("grid.login_url", &required(environment, "GRID_LOGIN_URL")?)?,
+        avatar_name: required(environment, user)?,
+        password: SecretString::new("grid.password", required(environment, password)?)?,
+    })?;
+    let session = backend.login(1, cancellation.token()).await?;
+    Ok((session, cancellation))
+}
+
+async fn wait_ready(
+    account: &str,
+    session: &mut Box<dyn GridSession>,
+    cancellation: &CancellationTokenSource,
+) -> Result<(), Box<dyn Error>> {
+    let signal = tokio::time::timeout(
+        Duration::from_secs(30),
+        session.next_signal(cancellation.token()),
+    )
+    .await
+    .map_err(|_| format!("timed out waiting for {account} session readiness"))?;
+    match signal {
+        SessionSignal::Ready => Ok(()),
+        signal => Err(format!("unexpected {account} session signal: {signal:?}").into()),
+    }
+}
+
+async fn wait_scene_settled(owner: &LibremetaverseClientOwner) -> Result<(), Box<dyn Error>> {
+    let deadline = Instant::now() + Duration::from_secs(30);
+    let minimum = Instant::now() + Duration::from_secs(10);
+    let mut previous = 0;
+    let mut stable = 0;
+    loop {
+        tokio::time::sleep(Duration::from_secs(1)).await;
+        let simulator = owner
+            .client()
+            .network()
+            .current_sim()
+            .ok_or("simulator unavailable while settling scene")?;
+        let count = simulator
+            .objects_primitives
+            .read()
+            .unwrap_or_else(std::sync::PoisonError::into_inner)
+            .len();
+        if count == previous && count != 0 {
+            stable += 1;
+        } else {
+            stable = 0;
+            previous = count;
+        }
+        if (Instant::now() >= minimum && stable >= 3) || Instant::now() >= deadline {
+            return Ok(());
+        }
+    }
+}
+
+fn live_environment() -> Result<BTreeMap<String, String>, Box<dyn Error>> {
+    let mut values = std::env::vars().collect::<BTreeMap<_, _>>();
+    let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../.env");
+    for line in std::fs::read_to_string(path)?.lines() {
+        let line = line.trim();
+        if line.is_empty() || line.starts_with('#') {
+            continue;
+        }
+        let Some((key, value)) = line.split_once('=') else {
+            continue;
+        };
+        values.entry(key.trim().to_owned()).or_insert_with(|| {
+            value
+                .trim()
+                .trim_matches(|character| character == '\'' || character == '"')
+                .to_owned()
+        });
+    }
+    Ok(values)
+}
+
+fn required(
+    environment: &BTreeMap<String, String>,
+    key: &'static str,
+) -> Result<String, Box<dyn Error>> {
+    environment
+        .get(key)
+        .filter(|value| !value.is_empty())
+        .cloned()
+        .ok_or_else(|| format!("missing live test setting {key}").into())
+}
+
+#[allow(clippy::cast_precision_loss)] // Simulator dimensions are small region coordinates.
+fn conventional_start(simulator: &libremetaverse::Simulator) -> Vector3 {
+    let x = 128.0_f32.min(simulator.size_x as f32 - 2.0);
+    let y = 128.0_f32.min(simulator.size_y as f32 - 2.0);
+    Vector3 { x, y, z: 50.0 }
+}
+
+#[allow(
+    clippy::cast_possible_truncation,
+    clippy::cast_possible_wrap,
+    clippy::cast_sign_loss
+)] // Coordinates are already clamped inside the non-negative simulator extent.
+fn terrain_ground(simulator: &libremetaverse::Simulator, x: f32, y: f32) -> Option<f32> {
+    let patches = simulator
+        .terrain
+        .read()
+        .unwrap_or_else(std::sync::PoisonError::into_inner);
+    let patch_x = (x as usize / 16) as i32;
+    let patch_y = (y as usize / 16) as i32;
+    let local_x = x as usize % 16;
+    let local_y = y as usize % 16;
+    patches
+        .iter()
+        .find(|patch| patch.x == patch_x && patch.y == patch_y)
+        .and_then(|patch| patch.data.get(local_y * 16 + local_x))
+        .copied()
+}
+
+#[allow(clippy::cast_precision_loss)] // Terrain patch indexes are small region coordinates.
+fn scenic_location(simulator: &libremetaverse::Simulator) -> Option<(Vector3, Vector3)> {
+    let roots = simulator
+        .objects_primitives
+        .read()
+        .unwrap_or_else(std::sync::PoisonError::into_inner)
+        .values()
+        .filter(|prim| prim.parent_id == 0)
+        .cloned()
+        .collect::<Vec<_>>();
+    let patches = simulator
+        .terrain
+        .read()
+        .unwrap_or_else(std::sync::PoisonError::into_inner);
+    patches
+        .iter()
+        .filter(|patch| patch.data.len() == 256)
+        .filter_map(|patch| {
+            let position = Vector3 {
+                x: patch.x as f32 * 16.0 + 8.0,
+                y: patch.y as f32 * 16.0 + 8.0,
+                z: patch.data[8 * 16 + 8] + 2.0,
+            };
+            if roots.iter().any(|prim| {
+                let extent = prim.scale.x.max(prim.scale.y).max(prim.scale.z);
+                let dx = prim.position.x - position.x;
+                let dy = prim.position.y - position.y;
+                let dz = prim.position.z - position.z;
+                (dx * dx + dy * dy + dz * dz).sqrt() < extent * 0.5 + 3.0
+            }) {
+                return None;
+            }
+            let mut score = 0.0_f32;
+            let mut forward = Vector3::zero();
+            for prim in &roots {
+                let dx = prim.position.x - position.x;
+                let dy = prim.position.y - position.y;
+                let horizontal = (dx * dx + dy * dy).sqrt();
+                let dz = prim.position.z - position.z;
+                let extent = prim.scale.x.max(prim.scale.y).max(prim.scale.z);
+                if !(10.0..=64.0).contains(&horizontal)
+                    || dz.abs() > 20.0
+                    || !(2.0..=32.0).contains(&extent)
+                {
+                    continue;
+                }
+                let weight = extent.min(10.0) / horizontal;
+                score += extent.min(10.0);
+                forward.x += dx * weight;
+                forward.y += dy * weight;
+                forward.z += dz.clamp(-horizontal * 0.35, horizontal * 0.35) * weight;
+            }
+            (score > 0.0).then_some((score, position, forward))
+        })
+        .max_by(|left, right| left.0.total_cmp(&right.0))
+        .map(|(_, position, forward)| (position, forward))
+}
+
+fn avatar_camera(avatar: Vector3, forward: Vector3) -> Result<(Vector3, Vector3), Box<dyn Error>> {
+    let length = (forward.x * forward.x + forward.y * forward.y + forward.z * forward.z).sqrt();
+    if !length.is_finite() || length < f32::EPSILON {
+        return Err("renderer start direction unavailable".into());
+    }
+    let position = Vector3 {
+        x: avatar.x,
+        y: avatar.y,
+        z: avatar.z + 2.0,
+    };
+    Ok((
+        position,
+        Vector3 {
+            x: position.x + forward.x / length * 30.0,
+            y: position.y + forward.y / length * 30.0,
+            z: position.z + forward.z / length * 30.0,
+        },
+    ))
+}