Fix bounded avatar appearance recovery after login
Some checks failed
CI / rust-skia (Rust only) (push) Has been cancelled
CI / required (push) Has been cancelled

This commit is contained in:
2026-08-23 18:06:58 +02:00
parent 42f0c62b6b
commit 5a533dfa00
9 changed files with 1114 additions and 41 deletions

View File

@@ -732,6 +732,7 @@ struct LiveInteractions {
policy: Arc<metacrate_grid_agent::PolicyGateway>,
audit: Arc<metacrate_grid_agent::MemoryPolicyAudit>,
observability: Arc<metacrate_grid_agent::Observability>,
_appearance_recovery: libremetaverse_types::compat::Subscription,
_landmark_intake: metacrate_grid_agent::LibremetaverseLandmarkIntake,
landmark_roaming: metacrate_grid_agent::LandmarkRoamingHandle,
build_control: Arc<dyn metacrate_grid_agent::BuildControl>,
@@ -808,6 +809,11 @@ async fn start_live_interactions(
journal_queue: config.limits.observable_queue,
..ObservabilityLimits::default()
})?;
let appearance_observability = Arc::clone(&observability);
let appearance_recovery = libremetaverse::IGridClient::appearance(owner.client())
.subscribe_recovery(Arc::new(move |event| {
record_appearance_recovery(&appearance_observability, event);
}));
let audit_sink: Arc<dyn PolicyAuditSink> = Arc::new(UnifiedPolicyAudit::new(
audit.clone(),
observability.clone(),
@@ -930,6 +936,7 @@ async fn start_live_interactions(
policy: gateway,
audit,
observability,
_appearance_recovery: appearance_recovery,
_landmark_intake: landmark_intake,
landmark_roaming,
build_control,
@@ -941,6 +948,67 @@ async fn start_live_interactions(
})
}
#[cfg(feature = "live-grid")]
fn record_appearance_recovery(
observer: &metacrate_grid_agent::Observability,
event: libremetaverse::AppearanceRecoveryEvent,
) {
use libremetaverse::AppearanceRecoveryPhase as Phase;
use metacrate_grid_agent::{EventDraft, EventFamily, EventOrigin, EventSeverity};
let phase = match event.phase {
Phase::LoginReady => "login_ready",
Phase::OutfitReady => "outfit_ready",
Phase::BakeComplete => "bake_complete",
Phase::AppearanceSent => "appearance_sent",
Phase::AppearanceAcknowledged => "appearance_acknowledged",
Phase::AttachmentsRepaired => "attachments_repaired",
Phase::Retrying => "retrying",
Phase::Complete => "complete",
Phase::Failed => "failed",
Phase::Cancelled => "cancelled",
};
let severity = match event.phase {
Phase::Failed => EventSeverity::Error,
Phase::Retrying | Phase::Cancelled => EventSeverity::Warning,
_ => EventSeverity::Info,
};
let stage = match event.stage {
libremetaverse::AppearanceRecoveryStage::Starting => "starting",
libremetaverse::AppearanceRecoveryStage::Outfit => "outfit",
libremetaverse::AppearanceRecoveryStage::Cache => "cache",
libremetaverse::AppearanceRecoveryStage::Assets => "assets",
libremetaverse::AppearanceRecoveryStage::Bake => "bake",
libremetaverse::AppearanceRecoveryStage::Publish => "publish",
libremetaverse::AppearanceRecoveryStage::Acknowledgement => "acknowledgement",
libremetaverse::AppearanceRecoveryStage::Attachments => "attachments",
};
let draft = EventDraft::new(
EventFamily::DiagnosticEnvelope,
severity,
"appearance",
EventOrigin::Grid,
)
.and_then(|draft| draft.code_field("phase", phase))
.and_then(|draft| draft.code_field("stage", stage))
.map(|draft| {
draft
.retry_count(event.retry)
.duration_millis(event.elapsed_millis)
})
.and_then(|draft| draft.field("generation", event.generation.into()))
.and_then(|draft| draft.field("cof_version", event.cof_version.into()))
.and_then(|draft| draft.field("wearables", event.wearable_count.into()))
.and_then(|draft| draft.field("attachments", event.attachment_count.into()))
.and_then(|draft| draft.field("missing_attachments", event.missing_attachment_count.into()))
.and_then(|draft| draft.field("cached_bakes", event.cached_bake_count.into()))
.and_then(|draft| draft.field("server_baking", event.server_baking.into()))
.and_then(|draft| draft.field("appearance_serial", event.appearance_serial.into()))
.and_then(|draft| draft.field("acknowledged", event.acknowledged.into()));
if let Ok(draft) = draft {
let _ = observer.record(draft);
}
}
#[cfg(feature = "live-grid")]
fn record_session_observation(
observer: &metacrate_grid_agent::Observability,

View File

@@ -1332,13 +1332,14 @@ impl SceneSource for LibremetaverseSceneSource {
})
.map(|avatar| avatar.local_id)
.collect::<std::collections::BTreeSet<_>>();
let (mut world, mut attachments) = {
let (mut world, mut attachments, avatars_with_attachments) = {
let primitive_cache = simulator
.objects_primitives
.read()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let mut world = Vec::new();
let mut attachments = Vec::new();
let mut avatars_with_attachments = std::collections::BTreeSet::new();
for prim in primitive_cache.values().cloned() {
let Some((prim, attachment_avatar)) =
world_primitive(prim, &primitive_cache, &avatar_parents)
@@ -1347,6 +1348,7 @@ impl SceneSource for LibremetaverseSceneSource {
};
if let Some(avatar) = attachment_avatar {
if attachment_avatar_ids.contains(&avatar) {
avatars_with_attachments.insert(avatar);
attachments.push(prim);
}
} else if scene_within_distance(
@@ -1357,7 +1359,7 @@ impl SceneSource for LibremetaverseSceneSource {
world.push(prim);
}
}
(world, attachments)
(world, attachments, avatars_with_attachments)
};
world.sort_by(|left, right| {
let left_visible = scene_center_visible(
@@ -1385,17 +1387,18 @@ impl SceneSource for LibremetaverseSceneSource {
scene_distance_squared(left.position, camera_position)
.total_cmp(&scene_distance_squared(right.position, camera_position))
});
avatars.retain(|avatar| !avatars_with_attachments.contains(&avatar.local_id));
let avatar_limit = avatars.len().min(entity_limit);
let object_limit = entity_limit.saturating_sub(avatar_limit);
let world_limit = world.len().min(object_limit);
let attachment_limit = attachments.len().min(object_limit - world_limit);
let attachment_limit = attachments.len().min(object_limit);
let world_limit = world.len().min(object_limit - attachment_limit);
let objects_truncated =
world.len() > world_limit || attachments.len() > attachment_limit;
let avatars_truncated = avatars.len() > avatar_limit;
world.truncate(world_limit);
attachments.truncate(attachment_limit);
world.extend(attachments);
let prims = world;
attachments.extend(world);
let prims = attachments;
avatars.truncate(avatar_limit);
let region_id = simulator.region_id;
let region_name = simulator.name.clone();
@@ -2033,6 +2036,19 @@ fn native_entities(
))
});
let Some(mesh) = mesh else { continue };
let skinning = (prim.is_attachment)
.then(|| mesh.skin_data.as_ref())
.flatten()
.and_then(|skin| {
let skeleton = libremetaverse::rendering::LindenSkeleton::get_default().ok()?;
let matrices = libremetaverse::AnimeshSkinning::compute_skinning_matrices(
Some(std::collections::HashMap::new()),
Some(skeleton),
Some(skin.clone()),
)
.ok()?;
Some((matrices, matrix4(&skin.bind_shape_matrix)))
});
let object_triangles = mesh
.faces
.iter()
@@ -2069,7 +2085,16 @@ fn native_entities(
let gpu_vertices = face
.vertices
.iter()
.map(|vertex| native_gpu_vertex(vertex, &prim))
.enumerate()
.map(|(index, vertex)| {
let (position, normal) = skinning
.as_ref()
.and_then(|(matrices, bind_shape)| {
skinned_vertex(face, index, matrices, *bind_shape)
})
.unwrap_or((vertex.position, vertex.normal));
native_gpu_vertex(position, normal, vertex.tex_coord, &prim)
})
.collect::<Vec<_>>();
let gpu_indices = face
.indices
@@ -2327,21 +2352,101 @@ pub(crate) fn scene_detail_level(
}
}
#[cfg(feature = "live-grid")]
fn matrix4(values: &[f32]) -> libremetaverse_types::Matrix4 {
let mut matrix = [0.0; 16];
if values.len() == matrix.len() {
matrix.copy_from_slice(values);
} else {
matrix = [
1.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 1.0,
];
}
libremetaverse_types::Matrix4 {
m11: matrix[0],
m12: matrix[1],
m13: matrix[2],
m14: matrix[3],
m21: matrix[4],
m22: matrix[5],
m23: matrix[6],
m24: matrix[7],
m31: matrix[8],
m32: matrix[9],
m33: matrix[10],
m34: matrix[11],
m41: matrix[12],
m42: matrix[13],
m43: matrix[14],
m44: matrix[15],
}
}
#[cfg(feature = "live-grid")]
fn skinned_vertex(
face: &libremetaverse::rendering::Face,
index: usize,
matrices: &[libremetaverse_types::Matrix4],
bind_shape: libremetaverse_types::Matrix4,
) -> Option<(libremetaverse_types::Vector3, libremetaverse_types::Vector3)> {
let vertex = face.vertices.get(index)?;
let weights = face.weights.as_ref()?.get(index)?;
let position = libremetaverse_types::Vector3::transform(vertex.position, bind_shape).ok()?;
let influences = [
(weights.joint0, weights.weight0),
(weights.joint1, weights.weight1),
(weights.joint2, weights.weight2),
(weights.joint3, weights.weight3),
];
let mut output_position = libremetaverse_types::Vector3::zero();
let mut output_normal = libremetaverse_types::Vector3::zero();
for (joint, weight) in influences {
if weight <= 0.0 || !weight.is_finite() {
continue;
}
let matrix = matrices.get(usize::try_from(joint).ok()?)?;
output_position = libremetaverse_types::Vector3::add_with_vector3_vector3(
output_position,
libremetaverse_types::Vector3::multiply_with_vector3_single(
libremetaverse_types::Vector3::transform(position, *matrix).ok()?,
weight,
)
.ok()?,
)
.ok()?;
output_normal = libremetaverse_types::Vector3::add_with_vector3_vector3(
output_normal,
libremetaverse_types::Vector3::multiply_with_vector3_single(
libremetaverse_types::Vector3::transform_normal(vertex.normal, *matrix).ok()?,
weight,
)
.ok()?,
)
.ok()?;
}
Some((
output_position,
libremetaverse_types::Vector3::normalize(output_normal).ok()?,
))
}
#[cfg(feature = "live-grid")]
fn native_gpu_vertex(
vertex: &libremetaverse::rendering::Vertex,
vertex_position: libremetaverse_types::Vector3,
vertex_normal: libremetaverse_types::Vector3,
tex_coord: libremetaverse_types::Vector2,
prim: &libremetaverse::Primitive,
) -> metacrate_rendering_wgpu::Vertex {
let scaled = libremetaverse_types::Vector3 {
x: vertex.position.x * prim.scale.x,
y: vertex.position.y * prim.scale.y,
z: vertex.position.z * prim.scale.z,
x: vertex_position.x * prim.scale.x,
y: vertex_position.y * prim.scale.y,
z: vertex_position.z * prim.scale.z,
};
let position = rotate_vector(scaled, prim.rotation);
let inverse_scaled_normal = libremetaverse_types::Vector3 {
x: vertex.normal.x / prim.scale.x.abs().max(f32::EPSILON),
y: vertex.normal.y / prim.scale.y.abs().max(f32::EPSILON),
z: vertex.normal.z / prim.scale.z.abs().max(f32::EPSILON),
x: vertex_normal.x / prim.scale.x.abs().max(f32::EPSILON),
y: vertex_normal.y / prim.scale.y.abs().max(f32::EPSILON),
z: vertex_normal.z / prim.scale.z.abs().max(f32::EPSILON),
};
let normal = rotate_vector(inverse_scaled_normal, prim.rotation);
let normal_length = (normal.x * normal.x + normal.y * normal.y + normal.z * normal.z)
@@ -2358,7 +2463,7 @@ fn native_gpu_vertex(
normal.y / normal_length,
normal.z / normal_length,
],
tex_coord: [vertex.tex_coord.x, vertex.tex_coord.y],
tex_coord: [tex_coord.x, tex_coord.y],
color_srgb: [255; 4],
}
}

View File

@@ -74,6 +74,75 @@ async fn luna_describes_live_renderer_evidence() -> Result<(), Box<dyn Error>> {
}
}
#[tokio::test]
#[ignore = "sends three observer-rendered views of Myrddin to the configured vision model"]
async fn luna_confirms_myrddin_body_and_attachments() -> Result<(), Box<dyn Error>> {
let environment = live_environment()?;
let mut connection = AgentConfig::offline(
required(&environment, "OPENAPI_URL")?,
required(&environment, "OPENAPI_KEY")?,
)?
.llm;
connection.model = Some(required(&environment, "OPENAPI_MODEL")?.to_owned());
let client = LlmClient::new(connection);
let runtime = Runtime::empty_builder()
.with_store(mentra::runtime::VolatileRuntimeStore::default())
.with_registered_provider(client.mentra_provider())
.build()?;
let root = std::env::temp_dir().join(format!(
"metacrate-live-appearance-review-{}",
std::process::id()
));
let mut config = mentra::AgentConfig {
system: Some(
"Judge only visible evidence in virtual-world renders; do not infer success from the prompt."
.to_owned(),
),
..Default::default()
};
config.compaction.transcript_dir = root.join("transcripts");
config.task.tasks_dir = root.join("tasks");
config.team.team_dir = root.join("teams");
config.workspace.base_dir = root;
let mut agent = runtime.spawn_with_config(
"live-appearance-review",
ModelInfo::new(client.configured_model(), BuiltinProvider::OpenAI),
config,
)?;
let mut content = vec![ContentBlock::text(
"These are front, side, and back views rendered by a second logged-in agent. Determine whether Myrddin is visibly initialized as a recognizable male humanoid rather than a cloud, rectangle, placeholder, or missing avatar. Confirm whether a normal mesh body and coat-like worn attachment are visibly present and correctly follow the body, with no obvious detached, duplicated, or misplaced attachment geometry. Explain the visible evidence briefly. End with `APPEARANCE: PASS` only if every condition is visibly satisfied; otherwise end with `APPEARANCE: FAIL`.",
)];
for label in ["front", "side", "back"] {
let path = std::env::temp_dir().join(format!("metacrate-myrddin-appearance-{label}.jpg"));
let jpeg = std::fs::read(path)?;
content.push(ContentBlock::text(format!("{label} view:")));
content.push(ContentBlock::image_url(format!(
"data:image/jpeg;base64,{}",
base64::engine::general_purpose::STANDARD.encode(jpeg)
)));
}
let response = agent.send(content).await?;
let descriptions = response
.content
.into_iter()
.filter_map(|block| match block {
ContentBlock::Text { text } if !text.trim().is_empty() => Some(text),
_ => None,
})
.collect::<Vec<_>>();
for description in &descriptions {
println!("LIVE_APPEARANCE_LLM={description}");
}
if descriptions
.iter()
.any(|description| description.contains("APPEARANCE: PASS"))
{
Ok(())
} else {
Err("vision model rejected Myrddin appearance evidence".into())
}
}
fn live_environment() -> Result<BTreeMap<String, String>, Box<dyn Error>> {
let mut values = std::env::vars().collect::<BTreeMap<_, _>>();
let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../.env");

View File

@@ -349,6 +349,268 @@ async fn live_bevy_capture_renders_in_front_of_avatar() -> Result<(), Box<dyn Er
result
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[ignore = "logs Myrddin and a second account into the live grid and renders appearance evidence"]
#[allow(clippy::too_many_lines)] // One ordered two-account evidence workflow is easier to audit.
async fn live_observer_renders_myrddin_appearance() -> Result<(), Box<dyn Error>> {
let environment = live_environment()?;
let target = LibremetaverseClientOwner::new()?;
let (progress_tx, mut progress_rx) = tokio::sync::mpsc::unbounded_channel();
let _appearance = target
.client()
.appearance()
.subscribe_recovery(Arc::new(move |event| {
let _ = progress_tx.send(event);
}));
let (mut target_session, target_cancel) =
login(&target, &environment, "GRID_USER", "GRID_PASSWORD").await?;
wait_ready("Myrddin", &mut target_session, &target_cancel).await?;
let recovery = tokio::time::timeout(Duration::from_mins(2), async {
loop {
let event = progress_rx
.recv()
.await
.ok_or("appearance recovery stream closed")?;
println!("LIVE_APPEARANCE_RECOVERY={event:?}");
if matches!(
event.phase,
libremetaverse::AppearanceRecoveryPhase::AppearanceSent
| libremetaverse::AppearanceRecoveryPhase::Complete
) {
return Ok::<_, Box<dyn Error>>(event);
}
}
})
.await
.map_err(|_| "timed out waiting for Myrddin appearance recovery")??;
let observer = LibremetaverseClientOwner::new()?;
let (mut observer_session, observer_cancel) = login(
&observer,
&environment,
"GRID_TEST_AUTHORIZED_USER",
"GRID_TEST_AUTHORIZED_PASSWORD",
)
.await?;
let result = async {
wait_ready("observer", &mut observer_session, &observer_cancel).await?;
wait_scene_settled(&observer).await?;
let target_name = required(&environment, "GRID_USER")?;
let target_avatar = tokio::time::timeout(Duration::from_secs(30), async {
loop {
let simulator = observer
.client()
.network()
.current_sim()
.ok_or("observer simulator unavailable")?;
if let Some(avatar) = simulator
.objects_avatars
.read()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.values()
.find(|avatar| avatar.name() == target_name)
.cloned()
{
return Ok::<_, Box<dyn Error>>(avatar);
}
tokio::time::sleep(Duration::from_millis(250)).await;
}
})
.await
.map_err(|_| "observer did not see Myrddin")??;
let attachment_primitives = observer
.client()
.network()
.current_sim()
.ok_or("observer simulator unavailable")?
.objects_primitives
.read()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.values()
.filter(|primitive| primitive.parent_id == target_avatar.local_id)
.cloned()
.collect::<Vec<_>>();
let attachment_ids = attachment_primitives
.iter()
.map(|primitive| primitive.id)
.collect::<std::collections::BTreeSet<_>>();
println!(
"LIVE_APPEARANCE_TARGET=position:{:?},local_id:{},observer_attachments:{}",
target_avatar.position,
target_avatar.local_id,
attachment_ids.len()
);
if attachment_ids.len() < 2 {
return Err("observer did not receive Myrddin's mesh body and coat attachments".into());
}
let assets = observer.client().assets();
let mut mesh_loads = tokio::task::JoinSet::new();
for mesh_id in attachment_primitives
.iter()
.filter_map(|primitive| primitive.sculpt.as_ref())
.filter(|sculpt| sculpt.type_() == libremetaverse_types::SculptType::Mesh)
.map(|sculpt| sculpt.sculpt_texture)
{
let assets = assets.clone();
let token = observer_cancel.token();
mesh_loads.spawn(async move { assets.request_mesh(mesh_id, Some(token)).await });
}
let mut loaded_meshes = 0;
while let Some(result) = tokio::time::timeout(Duration::from_secs(30), mesh_loads.join_next())
.await
.map_err(|_| "timed out loading Myrddin attachment mesh")?
{
loaded_meshes += usize::from(matches!(result, Ok(Ok(Some(_)))));
}
println!("LIVE_APPEARANCE_MESHES_LOADED={loaded_meshes}");
if loaded_meshes < 2 {
return Err("observer could not load Myrddin's mesh body and coat assets".into());
}
let mut decoded_meshes = 0;
for primitive in &attachment_primitives {
let Some(sculpt) = primitive.sculpt.as_ref() else {
continue;
};
let Some(bytes) = assets
.cache
.get_cached_asset_bytes_with_uuid(sculpt.sculpt_texture)?
else {
continue;
};
let asset = libremetaverse::assets::AssetMesh::new_with_uuid_bytes(
sculpt.sculpt_texture,
bytes,
)?;
let mut mesh = None;
decoded_meshes += usize::from(
libremetaverse::rendering::FacetedMesh::try_decode_from_asset(
primitive.clone(),
asset,
libremetaverse::rendering::DetailLevel::Highest,
&mut mesh,
),
);
}
println!("LIVE_APPEARANCE_MESHES_DECODED={decoded_meshes}");
if decoded_meshes < 2 {
return Err("renderer could not decode Myrddin's mesh body and coat assets".into());
}
let limits = VisionLimits {
width: 640,
height: 640,
max_distance_meters: 8,
max_entities: 128,
max_texture_fetches: 128,
minimum_interval: Duration::ZERO,
..VisionLimits::default()
};
let source = Arc::new(LibremetaverseSceneSource::new(&observer, limits));
let vision = VisionService::new(source.clone(), limits)?.prefer_gpu();
if !vision.initialize_renderer().await {
return Err("appearance evidence renderer initialization failed".into());
}
let facing = Vector3::mul_with_vector3_quaternion(
Vector3::unit_x(),
target_avatar.rotation,
);
let length = (facing.x * facing.x + facing.y * facing.y).sqrt();
if !length.is_finite() || length < f32::EPSILON {
return Err("Myrddin facing direction unavailable".into());
}
let forward = Vector3 {
x: facing.x / length,
y: facing.y / length,
z: 0.0,
};
let side = Vector3 {
x: -forward.y,
y: forward.x,
z: 0.0,
};
let back = Vector3 {
x: -forward.x,
y: -forward.y,
z: 0.0,
};
for (view, (label, direction)) in [("front", forward), ("side", side), ("back", back)]
.into_iter()
.enumerate()
{
let generation = u64::try_from(view)? + 1;
let eye = Vector3 {
x: target_avatar.position.x,
y: target_avatar.position.y,
z: target_avatar.position.z + 0.9,
};
let camera = Vector3 {
x: eye.x + direction.x * 4.0,
y: eye.y + direction.y * 4.0,
z: eye.z,
};
observer
.agent()
.movement
.camera
.look_at_with_vector3_vector3(camera, eye)?;
observer
.agent()
.movement
.camera
.set_vertical_fov_angle(45_f32.to_radians())?;
tokio::time::sleep(Duration::from_secs(1)).await;
vision.set_generation(generation);
if view == 0 {
let scene = source
.capture_scene(generation, observer_cancel.token())
.await?;
let rendered_attachments = scene
.entities
.iter()
.filter(|entity| {
attachment_ids.contains(&entity.id) && !entity.renderables.is_empty()
})
.count();
println!(
"LIVE_APPEARANCE_SCENE=entities:{},rendered_attachments:{rendered_attachments},textures_missing:{},texture_fetches:{}",
scene.entities.len(),
scene.completeness.textures_missing,
scene.texture_fetches,
);
if rendered_attachments < 2 {
return Err("renderer did not decode Myrddin's mesh body and coat".into());
}
}
let capture = vision
.capture(
&format!("myrddin-appearance-{label}"),
generation,
observer_cancel.token(),
)
.await?;
let output = std::env::temp_dir().join(format!(
"metacrate-myrddin-appearance-{label}.jpg"
));
std::fs::write(&output, &capture.jpeg)?;
println!("LIVE_APPEARANCE_OUTPUT_{label}={}", output.display());
println!("LIVE_APPEARANCE_SHA256_{label}={}", capture.image_sha256);
}
source.stop_prefetch().await;
println!(
"LIVE_APPEARANCE_STATE=cof_version:{},wearables:{},attachments:{},missing_attachments:{},elapsed_ms:{}",
recovery.cof_version,
recovery.wearable_count,
recovery.attachment_count,
recovery.missing_attachment_count,
recovery.elapsed_millis,
);
Ok::<(), Box<dyn Error>>(())
}
.await;
let _ = observer_session.logout(observer_cancel.token()).await;
let _ = target_session.logout(target_cancel.token()).await;
result
}
#[allow(clippy::too_many_lines)] // One ordered live timing record keeps phase boundaries explicit.
async fn profile_warm_capture(
source: &LibremetaverseSceneSource,