Complete Bevy OpenSim scene rendering

crates/metacrate-grid-agent/tests/live_llm_vision.rs

@@ -0,0 +1,91 @@
+use base64::Engine as _;
+use mentra::{BuiltinProvider, ContentBlock, ModelInfo, Runtime};
+use metacrate_grid_agent::{AgentConfig, LlmClient};
+use std::{collections::BTreeMap, error::Error, path::PathBuf};
+
+#[tokio::test]
+#[ignore = "sends the live renderer JPEG to the configured vision model through Mentra SSE"]
+async fn luna_describes_live_renderer_evidence() -> Result<(), Box<dyn Error>> {
+    let environment = live_environment()?;
+    let mut connection = AgentConfig::offline(
+        required(&environment, "OPENAPI_URL")?,
+        required(&environment, "OPENAPI_KEY")?,
+    )?
+    .llm;
+    connection.model = Some(required(&environment, "OPENAPI_MODEL")?.to_owned());
+    let client = LlmClient::new(connection);
+    let runtime = Runtime::empty_builder()
+        .with_store(mentra::runtime::VolatileRuntimeStore::default())
+        .with_registered_provider(client.mentra_provider())
+        .build()?;
+    let root = std::env::temp_dir().join(format!("metacrate-live-vision-{}", std::process::id()));
+    let mut config = mentra::AgentConfig {
+        system: Some("You are in a virtual world.".to_owned()),
+        ..Default::default()
+    };
+    config.compaction.transcript_dir = root.join("transcripts");
+    config.task.tasks_dir = root.join("tasks");
+    config.team.team_dir = root.join("teams");
+    config.workspace.base_dir = root;
+    let mut agent = runtime.spawn_with_config(
+        "live-render-review",
+        ModelInfo::new(client.configured_model(), BuiltinProvider::OpenAI),
+        config,
+    )?;
+    let jpeg = std::fs::read(
+        std::env::var_os("METACRATE_LIVE_RENDER_OUTPUT").map_or_else(
+            || std::env::temp_dir().join("metacrate-live-render.jpg"),
+            PathBuf::from,
+        ),
+    )?;
+    let image = format!(
+        "data:image/jpeg;base64,{}",
+        base64::engine::general_purpose::STANDARD.encode(jpeg)
+    );
+    let response = agent
+        .send(vec![
+            ContentBlock::text(
+                "Describe this rendered virtual-world scene. State whether it contains recognizable textured terrain and mesh scenery, and identify any obvious rendering corruption.",
+            ),
+            ContentBlock::image_url(image),
+        ])
+        .await?;
+    for block in response.content {
+        if let ContentBlock::Text { text } = block {
+            println!("LIVE_LLM_VISION={text}");
+        }
+    }
+    Ok(())
+}
+
+fn live_environment() -> Result<BTreeMap<String, String>, Box<dyn Error>> {
+    let mut values = std::env::vars().collect::<BTreeMap<_, _>>();
+    let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../.env");
+    for line in std::fs::read_to_string(path)?.lines() {
+        let line = line.trim();
+        if line.is_empty() || line.starts_with('#') {
+            continue;
+        }
+        let Some((key, value)) = line.split_once('=') else {
+            continue;
+        };
+        values.entry(key.trim().to_owned()).or_insert_with(|| {
+            value
+                .trim()
+                .trim_matches(|character| character == '\'' || character == '"')
+                .to_owned()
+        });
+    }
+    Ok(values)
+}
+
+fn required<'a>(
+    environment: &'a BTreeMap<String, String>,
+    key: &'static str,
+) -> Result<&'a str, Box<dyn Error>> {
+    environment
+        .get(key)
+        .filter(|value| !value.is_empty())
+        .map(String::as_str)
+        .ok_or_else(|| format!("missing live test setting {key}").into())
+}