use base64::Engine as _; use mentra::{BuiltinProvider, ContentBlock, ModelInfo, Runtime}; use metacrate_grid_agent::{AgentConfig, LlmClient}; use std::{collections::BTreeMap, error::Error, path::PathBuf}; #[tokio::test] #[ignore = "sends the live renderer JPEG to the configured vision model through Mentra SSE"] async fn luna_describes_live_renderer_evidence() -> Result<(), Box> { let environment = live_environment()?; let mut connection = AgentConfig::offline( required(&environment, "OPENAPI_URL")?, required(&environment, "OPENAPI_KEY")?, )? .llm; connection.model = Some(required(&environment, "OPENAPI_MODEL")?.to_owned()); let client = LlmClient::new(connection); let runtime = Runtime::empty_builder() .with_store(mentra::runtime::VolatileRuntimeStore::default()) .with_registered_provider(client.mentra_provider()) .build()?; let root = std::env::temp_dir().join(format!("metacrate-live-vision-{}", std::process::id())); let mut config = mentra::AgentConfig { system: Some("You are in a virtual world.".to_owned()), ..Default::default() }; config.compaction.transcript_dir = root.join("transcripts"); config.task.tasks_dir = root.join("tasks"); config.team.team_dir = root.join("teams"); config.workspace.base_dir = root; let mut agent = runtime.spawn_with_config( "live-render-review", ModelInfo::new(client.configured_model(), BuiltinProvider::OpenAI), config, )?; let jpeg = std::fs::read( std::env::var_os("METACRATE_LIVE_RENDER_OUTPUT").map_or_else( || std::env::temp_dir().join("metacrate-live-render.jpg"), PathBuf::from, ), )?; let image = format!( "data:image/jpeg;base64,{}", base64::engine::general_purpose::STANDARD.encode(jpeg) ); let response = agent .send(vec![ ContentBlock::text( "Describe this rendered virtual-world scene. State whether it contains recognizable detailed textured terrain and mesh scenery, and identify any obvious rendering corruption. End with `QUALITY: PASS` only if the scene is clearly recognizable and has no substantial rendering corruption; otherwise end with `QUALITY: FAIL`.", ), ContentBlock::image_url(image), ]) .await?; let descriptions = response .content .into_iter() .filter_map(|block| match block { ContentBlock::Text { text } if !text.trim().is_empty() => Some(text), _ => None, }) .collect::>(); if descriptions.is_empty() { return Err("Luna returned no snapshot description".into()); } let passed = descriptions .iter() .any(|description| description.contains("QUALITY: PASS")); for description in descriptions { println!("LIVE_LLM_VISION={description}"); } if passed { Ok(()) } else { Err("Luna rejected the rendered scene quality".into()) } } fn live_environment() -> Result, Box> { let mut values = std::env::vars().collect::>(); let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../.env"); for line in std::fs::read_to_string(path)?.lines() { let line = line.trim(); if line.is_empty() || line.starts_with('#') { continue; } let Some((key, value)) = line.split_once('=') else { continue; }; values.entry(key.trim().to_owned()).or_insert_with(|| { value .trim() .trim_matches(|character| character == '\'' || character == '"') .to_owned() }); } Ok(values) } fn required<'a>( environment: &'a BTreeMap, key: &'static str, ) -> Result<&'a str, Box> { environment .get(key) .filter(|value| !value.is_empty()) .map(String::as_str) .ok_or_else(|| format!("missing live test setting {key}").into()) }