101 lines
3.7 KiB
Rust
101 lines
3.7 KiB
Rust
use base64::Engine as _;
|
|
use mentra::{BuiltinProvider, ContentBlock, ModelInfo, Runtime};
|
|
use metacrate_grid_agent::{AgentConfig, LlmClient};
|
|
use std::{collections::BTreeMap, error::Error, path::PathBuf};
|
|
|
|
#[tokio::test]
|
|
#[ignore = "sends the live renderer JPEG to the configured vision model through Mentra SSE"]
|
|
async fn luna_describes_live_renderer_evidence() -> Result<(), Box<dyn Error>> {
|
|
let environment = live_environment()?;
|
|
let mut connection = AgentConfig::offline(
|
|
required(&environment, "OPENAPI_URL")?,
|
|
required(&environment, "OPENAPI_KEY")?,
|
|
)?
|
|
.llm;
|
|
connection.model = Some(required(&environment, "OPENAPI_MODEL")?.to_owned());
|
|
let client = LlmClient::new(connection);
|
|
let runtime = Runtime::empty_builder()
|
|
.with_store(mentra::runtime::VolatileRuntimeStore::default())
|
|
.with_registered_provider(client.mentra_provider())
|
|
.build()?;
|
|
let root = std::env::temp_dir().join(format!("metacrate-live-vision-{}", std::process::id()));
|
|
let mut config = mentra::AgentConfig {
|
|
system: Some("You are in a virtual world.".to_owned()),
|
|
..Default::default()
|
|
};
|
|
config.compaction.transcript_dir = root.join("transcripts");
|
|
config.task.tasks_dir = root.join("tasks");
|
|
config.team.team_dir = root.join("teams");
|
|
config.workspace.base_dir = root;
|
|
let mut agent = runtime.spawn_with_config(
|
|
"live-render-review",
|
|
ModelInfo::new(client.configured_model(), BuiltinProvider::OpenAI),
|
|
config,
|
|
)?;
|
|
let jpeg = std::fs::read(
|
|
std::env::var_os("METACRATE_LIVE_RENDER_OUTPUT").map_or_else(
|
|
|| std::env::temp_dir().join("metacrate-live-render.jpg"),
|
|
PathBuf::from,
|
|
),
|
|
)?;
|
|
let image = format!(
|
|
"data:image/jpeg;base64,{}",
|
|
base64::engine::general_purpose::STANDARD.encode(jpeg)
|
|
);
|
|
let response = agent
|
|
.send(vec![
|
|
ContentBlock::text(
|
|
"Describe this rendered virtual-world scene. State whether it contains recognizable textured terrain and mesh scenery, and identify any obvious rendering corruption.",
|
|
),
|
|
ContentBlock::image_url(image),
|
|
])
|
|
.await?;
|
|
let descriptions = response
|
|
.content
|
|
.into_iter()
|
|
.filter_map(|block| match block {
|
|
ContentBlock::Text { text } if !text.trim().is_empty() => Some(text),
|
|
_ => None,
|
|
})
|
|
.collect::<Vec<_>>();
|
|
if descriptions.is_empty() {
|
|
return Err("Luna returned no snapshot description".into());
|
|
}
|
|
for description in descriptions {
|
|
println!("LIVE_LLM_VISION={description}");
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
fn live_environment() -> Result<BTreeMap<String, String>, Box<dyn Error>> {
|
|
let mut values = std::env::vars().collect::<BTreeMap<_, _>>();
|
|
let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../.env");
|
|
for line in std::fs::read_to_string(path)?.lines() {
|
|
let line = line.trim();
|
|
if line.is_empty() || line.starts_with('#') {
|
|
continue;
|
|
}
|
|
let Some((key, value)) = line.split_once('=') else {
|
|
continue;
|
|
};
|
|
values.entry(key.trim().to_owned()).or_insert_with(|| {
|
|
value
|
|
.trim()
|
|
.trim_matches(|character| character == '\'' || character == '"')
|
|
.to_owned()
|
|
});
|
|
}
|
|
Ok(values)
|
|
}
|
|
|
|
fn required<'a>(
|
|
environment: &'a BTreeMap<String, String>,
|
|
key: &'static str,
|
|
) -> Result<&'a str, Box<dyn Error>> {
|
|
environment
|
|
.get(key)
|
|
.filter(|value| !value.is_empty())
|
|
.map(String::as_str)
|
|
.ok_or_else(|| format!("missing live test setting {key}").into())
|
|
}
|