feat(grid-agent): add structured observability and replay (#127)

crates/metacrate-grid-agent/src/control_plane.rs

@@ -4,6 +4,7 @@
 
 use crate::backend::BackendFuture;
 use crate::config::SecretString;
+use crate::observability::{MetricsSnapshot, StructuredEvent};
 use libremetaverse_types::compat::{CancellationToken, CancellationTokenSource};
 use serde::{Deserialize, Serialize};
 use std::collections::{BTreeMap, BTreeSet, VecDeque};
@@ -244,11 +245,13 @@ pub struct AuditEventView {
 #[serde(tag = "kind", content = "data", rename_all = "snake_case")]
 pub enum ControlPayload {
     Health(HealthView),
+    Metrics(MetricsSnapshot),
     Runtime(RuntimeView),
     Sessions(Page<SessionMetadataView>),
     ScheduledJobs(Page<ScheduledJobView>),
     PendingApprovals(Page<PendingApprovalView>),
     AuditEvents(Page<AuditEventView>),
+    ObservabilityEvents(Page<StructuredEvent>),
     Accepted { operation_id: String },
     Completed,
     Subscribed { current_sequence: u64 },
@@ -266,6 +269,7 @@ pub enum ConversationChannelView {
 #[serde(tag = "method", content = "parameters", rename_all = "snake_case")]
 pub enum ControlRequest {
     Health,
+    Metrics,
     Runtime,
     ListSessions {
         page: PageRequest,
@@ -279,6 +283,9 @@ pub enum ControlRequest {
     ListAuditEvents {
         page: PageRequest,
     },
+    ListObservabilityEvents {
+        page: PageRequest,
+    },
     SubscribeEvents {
         after_sequence: Option<u64>,
     },
@@ -326,11 +333,13 @@ impl ControlRequest {
         !matches!(
             self,
             Self::Health
+                | Self::Metrics
                 | Self::Runtime
                 | Self::ListSessions { .. }
                 | Self::ListScheduledJobs { .. }
                 | Self::ListPendingApprovals { .. }
                 | Self::ListAuditEvents { .. }
+                | Self::ListObservabilityEvents { .. }
                 | Self::SubscribeEvents { .. }
         )
     }
@@ -340,7 +349,8 @@ impl ControlRequest {
             Self::ListSessions { page }
             | Self::ListScheduledJobs { page }
             | Self::ListPendingApprovals { page }
-            | Self::ListAuditEvents { page } => page.valid(),
+            | Self::ListAuditEvents { page }
+            | Self::ListObservabilityEvents { page } => page.valid(),
             Self::CancelRequest { target_request_id } => valid_identifier(target_request_id),
             Self::CancelAction { action_id } => valid_identifier(action_id),
             Self::DecideApproval { approval_id, .. } => *approval_id != 0,
@@ -952,7 +962,7 @@ enum ClientFrame {
 #[serde(tag = "frame", content = "body", rename_all = "snake_case")]
 enum ServerFrame {
     Hello(ServerHello),
-    Response(ControlResponseEnvelope),
+    Response(Box<ControlResponseEnvelope>),
     Event(ControlEvent),
 }
 
@@ -1245,7 +1255,10 @@ where
                     true,
                 ),
             );
-            if outbound.try_send(ServerFrame::Response(response)).is_err() {
+            if outbound
+                .try_send(ServerFrame::Response(Box::new(response)))
+                .is_err()
+            {
                 break;
             }
             continue;
@@ -1256,7 +1269,7 @@ where
         tasks.spawn(async move {
             let (response, subscription) = request_core.request(envelope).await;
             if request_outbound
-                .try_send(ServerFrame::Response(response))
+                .try_send(ServerFrame::Response(Box::new(response)))
                 .is_err()
             {
                 request_cancel.cancel();
@@ -1534,7 +1547,7 @@ async fn tcp_client_reader(
         match frame {
             Ok(ServerFrame::Response(response)) => {
                 if let Some(sender) = lock(&shared.pending).remove(&response.request_id) {
-                    let _ = sender.send(response);
+                    let _ = sender.send(*response);
                 }
             }
             Ok(ServerFrame::Event(event)) => {
@@ -1744,11 +1757,13 @@ fn role_name(role: ControlRole) -> &'static str {
 fn request_name(request: &ControlRequest) -> &'static str {
     match request {
         ControlRequest::Health => "health",
+        ControlRequest::Metrics => "metrics",
         ControlRequest::Runtime => "runtime",
         ControlRequest::ListSessions { .. } => "list_sessions",
         ControlRequest::ListScheduledJobs { .. } => "list_scheduled_jobs",
         ControlRequest::ListPendingApprovals { .. } => "list_pending_approvals",
         ControlRequest::ListAuditEvents { .. } => "list_audit_events",
+        ControlRequest::ListObservabilityEvents { .. } => "list_observability_events",
         ControlRequest::SubscribeEvents { .. } => "subscribe_events",
         ControlRequest::CancelRequest { .. } => "cancel_request",
         ControlRequest::PauseAutonomy => "pause_autonomy",

crates/metacrate-grid-agent/src/control_runtime.rs

@@ -10,6 +10,10 @@ use crate::control_plane::{
     SessionMetadataView,
 };
 use crate::conversation::{ConversationChannel, ConversationKey, ConversationStore};
+use crate::observability::{
+    CorrelationIds, EventDraft, EventFamily, EventOrigin, EventSeverity, MetricsSnapshot,
+    Observability, pseudonymous_identifier,
+};
 use crate::policy::{
     ApprovalId, AuthenticatedPrincipal, MemoryPolicyAudit, PolicyFinalOutcome, PolicyGateway,
     PolicyReasonCode,
@@ -79,6 +83,7 @@ pub struct AgentControlTarget {
     conversations: Arc<ConversationStore>,
     policy: Arc<PolicyGateway>,
     audit: Arc<MemoryPolicyAudit>,
+    observability: Mutex<Option<Arc<Observability>>>,
     behavior: BehaviorIngress,
     commands: mpsc::Sender<RuntimeControlCommand>,
     command_capacity: usize,
@@ -121,6 +126,7 @@ impl AgentControlTarget {
                 conversations,
                 policy,
                 audit,
+                observability: Mutex::new(None),
                 behavior,
                 commands,
                 command_capacity,
@@ -131,6 +137,11 @@ impl AgentControlTarget {
         ))
     }
 
+    /// Attaches the unified recorder used by the live service and control API.
+    pub fn attach_observability(&self, observability: Arc<Observability>) {
+        *lock(&self.observability) = Some(observability);
+    }
+
     pub fn update_session(&self, session: SessionStatus) {
         let mut state = lock(&self.state);
         state.session = session;
@@ -205,6 +216,24 @@ impl AgentControlTarget {
                     protocol_version: CONTROL_PROTOCOL_VERSION,
                 }))
             }
+            ControlRequest::Metrics => {
+                let mut metrics = lock(&self.observability)
+                    .as_ref()
+                    .map_or_else(MetricsSnapshot::default, |observer| {
+                        observer.metrics().snapshot()
+                    });
+                let state = lock(&self.state);
+                metrics.ready = state.session.agent_ready;
+                metrics.active_sessions =
+                    u64::try_from(self.conversations.list_metadata().len()).unwrap_or(u64::MAX);
+                metrics.queue_depth = u64::try_from(
+                    self.command_capacity
+                        .saturating_sub(self.commands.capacity()),
+                )
+                .unwrap_or(u64::MAX);
+                metrics.queue_capacity = u64::try_from(self.command_capacity).unwrap_or(u64::MAX);
+                Ok(ControlPayload::Metrics(metrics))
+            }
             ControlRequest::Runtime => {
                 let state = lock(&self.state).clone();
                 let usage = self.policy.global_budget_usage();
@@ -284,7 +313,7 @@ impl AgentControlTarget {
                     .map(|(index, record)| AuditEventView {
                         sequence: u64::try_from(index).unwrap_or(u64::MAX).saturating_add(1),
                         unix_millis: record.recorded_unix_millis,
-                        principal: record.principal.as_str().to_owned(),
+                        principal: pseudonymous_identifier(record.principal.as_str()),
                         operation: record.tool.as_str().to_owned(),
                         outcome: policy_outcome_name(record.final_outcome).to_owned(),
                         authorization_id: record.authorization_id,
@@ -292,6 +321,14 @@ impl AgentControlTarget {
                     .collect();
                 Ok(ControlPayload::AuditEvents(page_values(&page, values)))
             }
+            ControlRequest::ListObservabilityEvents { page } => {
+                let values = lock(&self.observability)
+                    .as_ref()
+                    .map_or_else(Vec::new, |observer| observer.snapshot());
+                Ok(ControlPayload::ObservabilityEvents(page_values(
+                    &page, values,
+                )))
+            }
             ControlRequest::PauseAutonomy => {
                 let response = self.enqueue("pause", RuntimeControlCommand::Pause)?;
                 self.behavior.pause();
@@ -452,11 +489,185 @@ impl ControlTarget for AgentControlTarget {
                     false,
                 ));
             }
-            self.execute_now(&context, request)
+            let operation = runtime_request_name(&request);
+            let correlation = request_correlation(&request);
+            let domain_event = runtime_domain_event(&request).ok().flatten();
+            let result = self.execute_now(&context, request);
+            if let Some(observability) = lock(&self.observability).as_ref() {
+                let (severity, result_code) = match &result {
+                    Ok(_) => (EventSeverity::Info, "accepted"),
+                    Err(error) => (
+                        if error.retryable {
+                            EventSeverity::Warning
+                        } else {
+                            EventSeverity::Error
+                        },
+                        control_error_name(error.code),
+                    ),
+                };
+                let event = EventDraft::new(
+                    EventFamily::ControlCommand,
+                    severity,
+                    "control",
+                    EventOrigin::Control,
+                )
+                .and_then(|event| event.correlation(correlation))
+                .and_then(|event| event.result_code(result_code))
+                .and_then(|event| event.code_field("operation", operation))
+                .and_then(|event| {
+                    event.code_field(
+                        "role",
+                        match context.role {
+                            crate::control_plane::ControlRole::Observer => "observer",
+                            crate::control_plane::ControlRole::Operator => "operator",
+                        },
+                    )
+                })
+                .and_then(|event| event.redacted("operator_payload"));
+                if let Ok(event) = event {
+                    let _ = observability.record(event);
+                }
+                if result.is_ok()
+                    && let Some(event) = domain_event
+                {
+                    let _ = observability.record(event);
+                }
+            }
+            result
         })
     }
 }
 
+fn runtime_domain_event(
+    request: &ControlRequest,
+) -> Result<Option<EventDraft>, crate::observability::ObservabilityError> {
+    let event = match request {
+        ControlRequest::SetRoamingJob { job_id, enabled } => Some(
+            EventDraft::new(
+                EventFamily::ScheduledJob,
+                EventSeverity::Info,
+                "scheduler",
+                EventOrigin::Control,
+            )?
+            .correlation(CorrelationIds {
+                action_id: Some(job_id.clone()),
+                ..CorrelationIds::default()
+            })?
+            .result_code("accepted")?
+            .code_field("state", if *enabled { "enabled" } else { "disabled" })?,
+        ),
+        ControlRequest::ExpireConversation { avatar_id, channel } => Some(
+            EventDraft::new(
+                EventFamily::SessionExpired,
+                EventSeverity::Info,
+                "conversation",
+                EventOrigin::Control,
+            )?
+            .correlation(CorrelationIds {
+                avatar_id: Some(avatar_id.clone()),
+                ..CorrelationIds::default()
+            })?
+            .result_code("completed")?
+            .reason_code("operator_expired")?
+            .code_field(
+                "channel",
+                match channel {
+                    ConversationChannelView::PublicChat => "public_chat",
+                    ConversationChannelView::DirectIm => "direct_im",
+                },
+            )?,
+        ),
+        ControlRequest::DecideApproval {
+            approval_id,
+            approve,
+        } => Some(
+            EventDraft::new(
+                EventFamily::ApprovalDecision,
+                EventSeverity::Info,
+                "policy",
+                EventOrigin::Control,
+            )?
+            .correlation(CorrelationIds {
+                action_id: Some(format!("approval-{approval_id}")),
+                ..CorrelationIds::default()
+            })?
+            .result_code("completed")?
+            .code_field("decision", if *approve { "approved" } else { "denied" })?,
+        ),
+        ControlRequest::GracefulShutdown => Some(
+            EventDraft::new(
+                EventFamily::Shutdown,
+                EventSeverity::Info,
+                "service",
+                EventOrigin::Control,
+            )?
+            .result_code("requested")?,
+        ),
+        _ => None,
+    };
+    Ok(event)
+}
+
+fn request_correlation(request: &ControlRequest) -> CorrelationIds {
+    let action_id = match request {
+        ControlRequest::CancelAction { action_id } => Some(action_id.clone()),
+        ControlRequest::DecideApproval { approval_id, .. } => {
+            Some(format!("approval-{approval_id}"))
+        }
+        _ => None,
+    };
+    CorrelationIds {
+        action_id,
+        ..CorrelationIds::default()
+    }
+}
+
+const fn runtime_request_name(request: &ControlRequest) -> &'static str {
+    match request {
+        ControlRequest::Health => "health",
+        ControlRequest::Metrics => "metrics",
+        ControlRequest::Runtime => "runtime",
+        ControlRequest::ListSessions { .. } => "list_sessions",
+        ControlRequest::ListScheduledJobs { .. } => "list_scheduled_jobs",
+        ControlRequest::ListPendingApprovals { .. } => "list_pending_approvals",
+        ControlRequest::ListAuditEvents { .. } => "list_audit_events",
+        ControlRequest::ListObservabilityEvents { .. } => "list_observability_events",
+        ControlRequest::SubscribeEvents { .. } => "subscribe_events",
+        ControlRequest::CancelRequest { .. } => "cancel_request",
+        ControlRequest::PauseAutonomy => "pause_autonomy",
+        ControlRequest::ResumeAutonomy => "resume_autonomy",
+        ControlRequest::CancelAction { .. } => "cancel_action",
+        ControlRequest::DecideApproval { approve: true, .. } => "approve_proposal",
+        ControlRequest::DecideApproval { approve: false, .. } => "deny_proposal",
+        ControlRequest::ForceReconnect => "force_reconnect",
+        ControlRequest::ExpireConversation { .. } => "expire_conversation",
+        ControlRequest::SetRoamingJob { enabled: true, .. } => "enable_roaming_job",
+        ControlRequest::SetRoamingJob { enabled: false, .. } => "disable_roaming_job",
+        ControlRequest::InjectOperatorMessage { .. } => "inject_operator_message",
+        ControlRequest::GracefulShutdown => "graceful_shutdown",
+    }
+}
+
+const fn control_error_name(code: ControlErrorCode) -> &'static str {
+    match code {
+        ControlErrorCode::AuthenticationFailed => "authentication_failed",
+        ControlErrorCode::VersionMismatch => "version_mismatch",
+        ControlErrorCode::PermissionDenied => "permission_denied",
+        ControlErrorCode::InvalidRequest => "invalid_request",
+        ControlErrorCode::Replay => "replay",
+        ControlErrorCode::NotFound => "not_found",
+        ControlErrorCode::Conflict => "conflict",
+        ControlErrorCode::Cancelled => "cancelled",
+        ControlErrorCode::TimedOut => "timed_out",
+        ControlErrorCode::Busy => "busy",
+        ControlErrorCode::Backpressure => "backpressure",
+        ControlErrorCode::FrameTooLarge => "frame_too_large",
+        ControlErrorCode::IdleTimeout => "idle_timeout",
+        ControlErrorCode::TransportClosed => "transport_closed",
+        ControlErrorCode::Internal => "internal",
+    }
+}
+
 fn page_values<T>(request: &PageRequest, values: Vec<T>) -> Page<T> {
     let start = usize::try_from(request.cursor.unwrap_or(0))
         .unwrap_or(usize::MAX)

crates/metacrate-grid-agent/src/control_runtime_tests.rs

@@ -1,13 +1,13 @@
 use crate::behavior::{BehaviorController, BehaviorError, EmbodimentFuture, EmbodimentSink};
 use crate::control_plane::{
     CONTROL_PROTOCOL_VERSION, ControlLimits, ControlPayload, ControlPlane, ControlRequest,
-    ControlRequestEnvelope,
+    ControlRequestEnvelope, PageRequest,
 };
 use crate::control_runtime::{AgentControlTarget, RuntimeControlCommand};
 use crate::conversation::{ConversationLimits, ConversationStore};
 use crate::perception::WorldPosition;
 use crate::policy::{MemoryPolicyAudit, PolicyAuditSink, PolicyGateway, PolicyLimits};
-use crate::{AgentConfig, EmbodiedPose, SecretString};
+use crate::{AgentConfig, EmbodiedPose, Observability, ObservabilityLimits, SecretString};
 use libremetaverse_types::UUID;
 use libremetaverse_types::compat::CancellationToken;
 use std::collections::BTreeSet;
@@ -123,13 +123,15 @@ async fn production_target_projects_state_and_routes_real_mutations() {
     let (target, mut commands) =
         AgentControlTarget::new(conversations, policy, audit, behavior.ingress(), 8)
             .expect("target");
+    let observability = Observability::memory(ObservabilityLimits::default()).expect("events");
+    target.attach_observability(observability);
     target.update_region(
         Some("00000000-0000-4000-8000-000000000001".into()),
         Some("Test Region".into()),
         Some([128.0, 128.0, 24.0]),
     );
     let plane = ControlPlane::new(
-        target,
+        target.clone(),
         SecretString::new("test.operator", "operator-token").expect("token"),
         Some(SecretString::new("test.observer", "observer-token").expect("token")),
         ControlLimits::default(),
@@ -144,6 +146,13 @@ async fn production_target_projects_state_and_routes_real_mutations() {
     };
     assert_eq!(runtime.region_name.as_deref(), Some("Test Region"));
     assert_eq!(runtime.control_queue_capacity, 8);
+    let metrics = operator
+        .request(envelope("metrics", ControlRequest::Metrics))
+        .await;
+    let Ok(ControlPayload::Metrics(metrics)) = metrics.result else {
+        panic!("metrics projection")
+    };
+    assert_eq!(metrics.queue_capacity, 8);
 
     assert!(
         operator
@@ -163,5 +172,30 @@ async fn production_target_projects_state_and_routes_real_mutations() {
         .await;
     assert!(denied.result.is_err());
     assert!(commands.try_recv().is_err());
+    let events = observer
+        .request(envelope(
+            "events",
+            ControlRequest::ListObservabilityEvents {
+                page: PageRequest {
+                    cursor: None,
+                    limit: 100,
+                },
+            },
+        ))
+        .await;
+    let Ok(ControlPayload::ObservabilityEvents(events)) = events.result else {
+        panic!("observability events")
+    };
+    assert!(
+        events
+            .items
+            .iter()
+            .any(|event| event.family == "control_command")
+    );
+    assert!(events.items.iter().all(|event| {
+        !serde_json::to_string(event)
+            .expect("event JSON")
+            .contains("operator-token")
+    }));
     behavior.shutdown().await.expect("shutdown behavior");
 }

crates/metacrate-grid-agent/src/conversation.rs

@@ -232,6 +232,7 @@ impl MemoryRecord {
 /// Stable, content-free reasons suitable for audit and observability.
 #[derive(Clone, Copy, Debug, Eq, PartialEq)]
 pub enum MemoryReason {
+    SessionCreated,
     PublicExpired,
     DirectImExpired,
     TurnCompacted,
@@ -542,6 +543,10 @@ impl ConversationStore {
                 tool_results: 0,
             };
             state.sessions.insert(key, session);
+            if let Some(session) = state.sessions.get(&key) {
+                let event = event_for(MemoryReason::SessionCreated, session);
+                push_event(&mut state.events, event);
+            }
         }
         let effective_wall = state.sessions.get(&key).map_or(wall_now, |session| {
             wall_now.max(session.last_active_unix_millis)

crates/metacrate-grid-agent/src/interaction.rs

@@ -374,8 +374,20 @@ pub enum DeliveryOutcome {
     PolicyDenied,
 }
 
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub enum InferenceOutcome {
+    Completed,
+    Cancelled,
+    TimedOut,
+    Failed,
+    PolicyRejected,
+}
+
 #[derive(Clone, Debug, Eq, PartialEq)]
 pub enum InteractionObservation {
+    SessionMemory {
+        event: crate::conversation::MemoryEvent,
+    },
     Suppressed {
         delivery_id: BoundedText<MAX_IDENTIFIER_BYTES>,
         reason: SuppressionReason,
@@ -390,6 +402,17 @@ pub enum InteractionObservation {
         delivery_id: BoundedText<MAX_IDENTIFIER_BYTES>,
         avatar_id: UUID,
     },
+    InferenceStarted {
+        delivery_id: BoundedText<MAX_IDENTIFIER_BYTES>,
+        session_id: BoundedText<MAX_IDENTIFIER_BYTES>,
+        prompt_messages: usize,
+    },
+    InferenceFinished {
+        delivery_id: BoundedText<MAX_IDENTIFIER_BYTES>,
+        session_id: BoundedText<MAX_IDENTIFIER_BYTES>,
+        outcome: InferenceOutcome,
+        duration_millis: u64,
+    },
     Delivery {
         delivery_id: BoundedText<MAX_IDENTIFIER_BYTES>,
         session_id: BoundedText<MAX_IDENTIFIER_BYTES>,
@@ -1044,6 +1067,12 @@ async fn process_batch(
         (InteractionChannel::DirectIm, false) => InteractionOrigin::UnprivilegedIm,
     };
     let intent = classify_intent(trigger.channel, authorized, &body);
+    for event in update.events.iter().cloned() {
+        observe(
+            &observations,
+            InteractionObservation::SessionMemory { event },
+        );
+    }
     observe(
         &observations,
         InteractionObservation::IntentRouted {
@@ -1104,12 +1133,38 @@ async fn process_batch(
         intent,
         messages,
     };
+    observe(
+        &observations,
+        InteractionObservation::InferenceStarted {
+            delivery_id: delivery_id.clone(),
+            session_id: update.session_id.clone(),
+            prompt_messages: request.messages.len(),
+        },
+    );
+    let inference_started = Instant::now();
     let model = tokio::select! {
         () = cancellation.cancelled() => Err(InteractionModelError::Cancelled),
         result = tokio::time::timeout(settings.model_timeout, responder.respond(request, cancellation.clone())) => {
             result.unwrap_or(Err(InteractionModelError::Timeout))
         }
     };
+    let inference_outcome = match &model {
+        Ok(_) => InferenceOutcome::Completed,
+        Err(InteractionModelError::Cancelled) => InferenceOutcome::Cancelled,
+        Err(InteractionModelError::Timeout) => InferenceOutcome::TimedOut,
+        Err(InteractionModelError::Failed) => InferenceOutcome::Failed,
+        Err(InteractionModelError::PolicyRejected) => InferenceOutcome::PolicyRejected,
+    };
+    observe(
+        &observations,
+        InteractionObservation::InferenceFinished {
+            delivery_id: delivery_id.clone(),
+            session_id: update.session_id.clone(),
+            outcome: inference_outcome,
+            duration_millis: u64::try_from(inference_started.elapsed().as_millis())
+                .unwrap_or(u64::MAX),
+        },
+    );
     match model {
         Ok(response) => {
             let Some(safe) = safe_visible_response(response.as_str(), settings.max_response_bytes)

crates/metacrate-grid-agent/src/interaction_tests.rs

@@ -237,6 +237,9 @@ async fn observation_for(
             | InteractionObservation::Delivery {
                 delivery_id: id, ..
             } => id.as_str() == delivery_id,
+            InteractionObservation::SessionMemory { .. }
+            | InteractionObservation::InferenceStarted { .. }
+            | InteractionObservation::InferenceFinished { .. } => false,
         };
         if matches {
             return observation;

crates/metacrate-grid-agent/src/lib.rs

@@ -12,6 +12,7 @@ pub mod control_runtime;
 pub mod conversation;
 pub mod interaction;
 pub mod llm;
+pub mod observability;
 pub mod perception;
 pub mod policy;
 pub mod service;
@@ -30,6 +31,8 @@ mod conversation_tests;
 #[cfg(test)]
 mod interaction_tests;
 #[cfg(test)]
+mod observability_tests;
+#[cfg(test)]
 mod perception_tests;
 #[cfg(test)]
 mod policy_tests;
@@ -74,17 +77,25 @@ pub use conversation::{
 };
 pub use interaction::{
     DeliveryFuture, DeliveryOutcome, ImDialogKind, InboundInteraction, InboundSource,
-    InteractionChannel, InteractionCoordinator, InteractionDeliveryError, InteractionError,
-    InteractionHandle, InteractionIngress, InteractionIntent, InteractionModelError,
-    InteractionObservation, InteractionOrigin, InteractionPacingError, InteractionResponder,
-    InteractionSettings, InteractionSink, OutboundInteraction, PacerFuture, PolicyLlmResponder,
-    ResponderFuture, ResponsePacer, ResponseRequest, SuppressionReason, VisibleResponse,
-    split_utf8,
+    InferenceOutcome, InteractionChannel, InteractionCoordinator, InteractionDeliveryError,
+    InteractionError, InteractionHandle, InteractionIngress, InteractionIntent,
+    InteractionModelError, InteractionObservation, InteractionOrigin, InteractionPacingError,
+    InteractionResponder, InteractionSettings, InteractionSink, OutboundInteraction, PacerFuture,
+    PolicyLlmResponder, ResponderFuture, ResponsePacer, ResponseRequest, SuppressionReason,
+    VisibleResponse, split_utf8,
 };
 pub use llm::{
     Completion, CompletionMessage, ContentPart, ImageDetail, LlmClient, LlmError,
     LlmTransportLimits, ToolDefinition, ToolSchema, Usage,
 };
+pub use observability::{
+    AgentMetrics, CorrelationIds, DiagnosticDirection, EventDraft, EventFamily, EventOrigin,
+    EventSeverity, EventSubscription, JournalConfig, LatencyMetrics, MetricsSnapshot,
+    OBSERVABILITY_SCHEMA_VERSION, Observability, ObservabilityError, ObservabilityLimits,
+    ObservabilityRuntime, OutcomeCode, OutcomeMetrics, ReplayAction, ReplayState,
+    SensitiveDiagnosticPart, StructuredEvent, UnifiedPolicyAudit, event_json,
+    pseudonymous_identifier, replay_journal,
+};
 pub use perception::{
     AGENT_STATE_TOOL, AgentSnapshot, EnvironmentSnapshot, INVENTORY_SEARCH_TOOL, InventoryMetadata,
     LOCATION_TOOL, LandmarkMetadata, NEARBY_AVATARS_TOOL, ObservedAvatar, ObservedObject,
@@ -105,9 +116,9 @@ pub use policy::{
 pub use service::{AgentService, ServiceError, ServiceHandle, ServiceState};
 pub use session::{
     GridSession, GridSessionBackend, ReconnectPolicy, SessionControl, SessionFailure,
-    SessionFailureKind, SessionFuture, SessionObservation, SessionSignal, SessionState,
-    SessionStatus, SessionSupervisor, SessionSupervisorError, SessionSupervisorHandle, SessionWork,
-    WorkDisposition, WorkKind,
+    SessionFailureKind, SessionFuture, SessionObservation, SessionReason, SessionSignal,
+    SessionState, SessionStatus, SessionSupervisor, SessionSupervisorError,
+    SessionSupervisorHandle, SessionWork, WorkDisposition, WorkKind,
 };
 pub use tool_loop::{
     HistorySummarizer, SessionGeneration, ToolExecution, ToolExecutor, ToolFuture, ToolLoop,

crates/metacrate-grid-agent/src/main.rs

@@ -7,7 +7,7 @@ use std::path::PathBuf;
 #[cfg(feature = "live-grid")]
 use std::sync::Arc;
 #[cfg(feature = "live-grid")]
-use std::time::{SystemTime, UNIX_EPOCH};
+use std::time::{Duration, SystemTime, UNIX_EPOCH};
 
 const MAX_ARGUMENTS: usize = 8;
 
@@ -184,22 +184,28 @@ async fn run_live(
         let readiness = tokio::time::timeout(config.timeouts.startup, async {
             loop {
                 match handle.next_observation().await {
-                    Some(SessionObservation::Transition { status, .. }) if status.agent_ready => {
+                    Some(event @ SessionObservation::Transition { status, .. })
+                        if status.agent_ready =>
+                    {
+                        record_session_observation(&live.observability, &event);
                         return Ok::<(), CliError>(());
                     }
-                    Some(SessionObservation::Transition {
-                        status:
-                            metacrate_grid_agent::SessionStatus {
-                                state: SessionState::AuthenticationBlocked,
-                                ..
-                            },
-                        ..
-                    }) => {
+                    Some(
+                        event @ SessionObservation::Transition {
+                            status:
+                                metacrate_grid_agent::SessionStatus {
+                                    state: SessionState::AuthenticationBlocked,
+                                    ..
+                                },
+                            ..
+                        },
+                    ) => {
+                        record_session_observation(&live.observability, &event);
                         return Err(CliError(
                             "grid authentication/configuration requires operator action".into(),
                         ));
                     }
-                    Some(_) => {}
+                    Some(event) => record_session_observation(&live.observability, &event),
                     None => {
                         return Err(CliError(
                             "session supervisor stopped before readiness".into(),
@@ -239,6 +245,7 @@ async fn run_live(
         live.behavior.ingress(),
         config.limits.control_queue,
     )?;
+    control_target.attach_observability(live.observability.clone());
     control_target.update_session(handle.status());
     let erased_target: Arc<dyn ControlTarget> = control_target.clone();
     let (control_plane, _integrated_client, control_server) = match config.mode {
@@ -292,6 +299,7 @@ async fn run_live(
             }
             event = handle.next_observation() => {
                 let Some(event) = event else { break; };
+                record_session_observation(&live.observability, &event);
                 if let SessionObservation::Transition { status, reason, retry_in } = event {
                     control_target.update_session(status);
                     control_plane.publish(ControlEventKind::StateChanged {
@@ -309,14 +317,17 @@ async fn run_live(
             }
             event = live.interaction.next_observation() => {
                 let Some(event) = event else { break; };
+                record_interaction_observation(&live.observability, &event);
                 println!("grid interaction event={event:?}");
             }
             event = live.perception_observations.recv() => {
                 let Some(event) = event else { break; };
+                record_perception_observation(&live.observability, &event);
                 println!("grid perception event={event:?}");
             }
             event = live.behavior.next_observation() => {
                 let Some(event) = event else { break; };
+                record_behavior_observation(&live.observability, &event);
                 if let BehaviorObservation::Transition { to, .. } = &event {
                     control_target.update_behavior(*to);
                     control_plane.publish(ControlEventKind::StateChanged {
@@ -358,6 +369,14 @@ async fn run_live(
             }
         }
     }
+    let shutdown_event = metacrate_grid_agent::EventDraft::new(
+        metacrate_grid_agent::EventFamily::Shutdown,
+        metacrate_grid_agent::EventSeverity::Info,
+        "service",
+        metacrate_grid_agent::EventOrigin::Service,
+    )?
+    .result_code("requested")?;
+    let _ = live.observability.record(shutdown_event);
     control_target.mark_stopping();
     let session_result = handle.shutdown().await;
     let interaction_result = live.interaction.shutdown().await;
@@ -391,6 +410,7 @@ struct LiveInteractions {
     conversations: Arc<metacrate_grid_agent::ConversationStore>,
     policy: Arc<metacrate_grid_agent::PolicyGateway>,
     audit: Arc<metacrate_grid_agent::MemoryPolicyAudit>,
+    observability: Arc<metacrate_grid_agent::Observability>,
 }
 
 #[cfg(feature = "live-grid")]
@@ -402,8 +422,9 @@ fn start_live_interactions(
     use metacrate_grid_agent::{
         AuthorizedBackendRouter, AuthorizedToolBackend, BehaviorBackend, BehaviorController,
         ConversationStore, InteractionCoordinator, LlmClient, LlmTransportLimits,
-        MemoryPolicyAudit, PerceptionBackend, PolicyGateway, PolicyLimits, PolicyLlmResponder,
-        ToolLoopLimits, behavior_policy_tools, perception_policy_tools,
+        MemoryPolicyAudit, Observability, ObservabilityLimits, PerceptionBackend, PolicyAuditSink,
+        PolicyGateway, PolicyLimits, PolicyLlmResponder, ToolLoopLimits, UnifiedPolicyAudit,
+        behavior_policy_tools, perception_policy_tools,
     };
 
     let transport_limits = LlmTransportLimits {
@@ -435,6 +456,16 @@ fn start_live_interactions(
     let behavior_ingress = behavior.ingress();
     let client = Arc::new(LlmClient::new(config.llm.clone(), transport_limits)?);
     let audit = Arc::new(MemoryPolicyAudit::new(config.limits.observable_queue)?);
+    let observability = Observability::memory(ObservabilityLimits {
+        ring_events: config.limits.observable_queue,
+        subscriber_queue: config.limits.observable_queue.min(512),
+        journal_queue: config.limits.observable_queue,
+        ..ObservabilityLimits::default()
+    })?;
+    let audit_sink: Arc<dyn PolicyAuditSink> = Arc::new(UnifiedPolicyAudit::new(
+        audit.clone(),
+        observability.clone(),
+    ));
     let mut tools = perception_policy_tools()?;
     tools.extend(behavior_policy_tools(&config.behavior)?);
     let routes = tools
@@ -453,7 +484,7 @@ fn start_live_interactions(
         config.authorized_avatar_uuids.clone(),
         tools,
         PolicyLimits::default(),
-        audit.clone(),
+        audit_sink,
     )?);
     let loop_limits = ToolLoopLimits {
         max_tool_calls_per_turn: config.limits.max_tool_calls,
@@ -500,5 +531,497 @@ fn start_live_interactions(
         conversations: conversation,
         policy: gateway,
         audit,
+        observability,
     })
 }
+
+#[cfg(feature = "live-grid")]
+fn record_session_observation(
+    observer: &metacrate_grid_agent::Observability,
+    observation: &metacrate_grid_agent::SessionObservation,
+) {
+    use metacrate_grid_agent::{
+        EventDraft, EventFamily, EventOrigin, EventSeverity, SessionObservation,
+    };
+    let SessionObservation::Transition {
+        status,
+        reason,
+        retry_in,
+    } = observation
+    else {
+        return;
+    };
+    observer.metrics().set_ready(status.agent_ready);
+    if matches!(
+        reason,
+        metacrate_grid_agent::SessionReason::TransientTransport
+            | metacrate_grid_agent::SessionReason::Maintenance
+            | metacrate_grid_agent::SessionReason::Kicked
+            | metacrate_grid_agent::SessionReason::SimulatorDisconnected
+            | metacrate_grid_agent::SessionReason::ServerFailure
+            | metacrate_grid_agent::SessionReason::OperatorForceReconnect
+    ) {
+        observer.metrics().record_reconnect();
+    }
+    let event = EventDraft::new(
+        EventFamily::LifecycleTransition,
+        if status.agent_ready {
+            EventSeverity::Info
+        } else {
+            EventSeverity::Warning
+        },
+        "session",
+        EventOrigin::Grid,
+    )
+    .and_then(|event| event.reason_code(session_reason_code(*reason)))
+    .and_then(|event| {
+        event
+            .retry_count(status.consecutive_failures)
+            .code_field("to", status.state.as_str())
+    })
+    .and_then(|event| event.field("generation", serde_json::Value::from(status.generation)))
+    .and_then(|event| event.field("ready", serde_json::Value::from(status.agent_ready)))
+    .and_then(|event| {
+        event.field(
+            "transport_connected",
+            serde_json::Value::from(status.transport_connected),
+        )
+    })
+    .and_then(|event| {
+        event.field(
+            "retry_millis",
+            serde_json::Value::from(retry_in.map_or(0, |duration| {
+                u64::try_from(duration.as_millis()).unwrap_or(u64::MAX)
+            })),
+        )
+    });
+    if let Ok(event) = event {
+        let _ = observer.record(event);
+    }
+}
+
+#[cfg(feature = "live-grid")]
+#[allow(clippy::too_many_lines)]
+fn record_interaction_observation(
+    observer: &metacrate_grid_agent::Observability,
+    observation: &metacrate_grid_agent::InteractionObservation,
+) {
+    use metacrate_grid_agent::{
+        CorrelationIds, EventDraft, EventFamily, EventOrigin, EventSeverity, InferenceOutcome,
+        InteractionObservation, MemoryReason, OutcomeCode,
+    };
+    let event = match observation {
+        InteractionObservation::SessionMemory { event } => {
+            let (family, state) = match event.reason {
+                MemoryReason::SessionCreated => (EventFamily::SessionCreated, "created"),
+                MemoryReason::PublicExpired
+                | MemoryReason::DirectImExpired
+                | MemoryReason::SessionLimitEviction
+                | MemoryReason::TotalByteEviction
+                | MemoryReason::OperatorDeleted
+                | MemoryReason::OperatorExpired => (EventFamily::SessionExpired, "expired"),
+                MemoryReason::TurnCompacted
+                | MemoryReason::ByteCompacted
+                | MemoryReason::ToolResultCompacted => {
+                    (EventFamily::LifecycleTransition, "compacted")
+                }
+                MemoryReason::CorruptSnapshotQuarantined
+                | MemoryReason::UnsupportedSnapshotQuarantined
+                | MemoryReason::SnapshotTooLargeQuarantined
+                | MemoryReason::PermissionsNotVerified => {
+                    (EventFamily::LifecycleTransition, "quarantined")
+                }
+            };
+            EventDraft::new(
+                family,
+                EventSeverity::Info,
+                "conversation",
+                EventOrigin::Service,
+            )
+            .and_then(|draft| {
+                draft.correlation(CorrelationIds {
+                    avatar_id: event.avatar_id.map(|id| id.to_string()),
+                    session_id: event.session_id.as_ref().map(|id| id.as_str().to_owned()),
+                    ..CorrelationIds::default()
+                })
+            })
+            .and_then(|draft| draft.reason_code(memory_reason_code(event.reason)))
+            .and_then(|draft| draft.code_field("state", state))
+        }
+        InteractionObservation::Suppressed {
+            delivery_id,
+            reason,
+        } => EventDraft::new(
+            EventFamily::InboundMessage,
+            EventSeverity::Info,
+            "interaction",
+            EventOrigin::Grid,
+        )
+        .and_then(|event| {
+            event.correlation(CorrelationIds {
+                request_id: Some(delivery_id.as_str().to_owned()),
+                ..CorrelationIds::default()
+            })
+        })
+        .and_then(|event| event.result_code("suppressed"))
+        .and_then(|event| event.reason_code(suppression_reason_code(*reason)))
+        .and_then(|event| event.redacted("message_content")),
+        InteractionObservation::IntentRouted {
+            delivery_id,
+            session_id,
+            origin,
+            intent,
+        } => EventDraft::new(
+            EventFamily::InboundMessage,
+            EventSeverity::Info,
+            "interaction",
+            EventOrigin::Grid,
+        )
+        .and_then(|event| {
+            event.correlation(CorrelationIds {
+                session_id: Some(session_id.as_str().to_owned()),
+                request_id: Some(delivery_id.as_str().to_owned()),
+                ..CorrelationIds::default()
+            })
+        })
+        .and_then(|event| event.result_code("routed"))
+        .and_then(|event| event.code_field("origin", interaction_origin_code(*origin)))
+        .and_then(|event| event.code_field("intent", interaction_intent_code(*intent)))
+        .and_then(|event| event.redacted("message_content")),
+        InteractionObservation::AttentionRequested {
+            delivery_id,
+            avatar_id,
+        } => EventDraft::new(
+            EventFamily::ModelActionSummary,
+            EventSeverity::Info,
+            "behavior",
+            EventOrigin::Service,
+        )
+        .and_then(|event| {
+            event.correlation(CorrelationIds {
+                avatar_id: Some(avatar_id.to_string()),
+                request_id: Some(delivery_id.as_str().to_owned()),
+                ..CorrelationIds::default()
+            })
+        })
+        .and_then(|event| event.code_field("summary", "face_speaker")),
+        InteractionObservation::InferenceStarted {
+            delivery_id,
+            session_id,
+            prompt_messages,
+        } => EventDraft::new(
+            EventFamily::InferenceRequest,
+            EventSeverity::Info,
+            "llm",
+            EventOrigin::Service,
+        )
+        .and_then(|event| {
+            event.correlation(CorrelationIds {
+                session_id: Some(session_id.as_str().to_owned()),
+                request_id: Some(delivery_id.as_str().to_owned()),
+                ..CorrelationIds::default()
+            })
+        })
+        .and_then(|event| event.field("prompt_messages", serde_json::Value::from(*prompt_messages)))
+        .and_then(|event| event.redacted("prompt_content")),
+        InteractionObservation::InferenceFinished {
+            delivery_id,
+            session_id,
+            outcome,
+            duration_millis,
+        } => {
+            observer
+                .metrics()
+                .record_inference_latency(Duration::from_millis(*duration_millis));
+            let (severity, result, metric) = match outcome {
+                InferenceOutcome::Completed => {
+                    (EventSeverity::Info, "completed", OutcomeCode::Completed)
+                }
+                InferenceOutcome::Cancelled => {
+                    (EventSeverity::Warning, "cancelled", OutcomeCode::Cancelled)
+                }
+                InferenceOutcome::TimedOut => {
+                    (EventSeverity::Warning, "timed_out", OutcomeCode::Failed)
+                }
+                InferenceOutcome::Failed => (EventSeverity::Error, "failed", OutcomeCode::Failed),
+                InferenceOutcome::PolicyRejected => (
+                    EventSeverity::Warning,
+                    "policy_rejected",
+                    OutcomeCode::Denied,
+                ),
+            };
+            observer.metrics().record_inference_outcome(metric);
+            EventDraft::new(
+                EventFamily::InferenceResult,
+                severity,
+                "llm",
+                EventOrigin::Model,
+            )
+            .and_then(|event| {
+                event.correlation(CorrelationIds {
+                    session_id: Some(session_id.as_str().to_owned()),
+                    request_id: Some(delivery_id.as_str().to_owned()),
+                    ..CorrelationIds::default()
+                })
+            })
+            .map(|event| event.duration_millis(*duration_millis))
+            .and_then(|event| event.result_code(result))
+            .and_then(|event| event.redacted("response_content"))
+        }
+        InteractionObservation::Delivery {
+            delivery_id,
+            session_id,
+            channel,
+            outcome,
+            delivered_parts,
+        } => EventDraft::new(
+            EventFamily::OutboundMessage,
+            EventSeverity::Info,
+            "interaction",
+            EventOrigin::Service,
+        )
+        .and_then(|event| {
+            event.correlation(CorrelationIds {
+                session_id: Some(session_id.as_str().to_owned()),
+                request_id: Some(delivery_id.as_str().to_owned()),
+                ..CorrelationIds::default()
+            })
+        })
+        .and_then(|event| event.result_code(delivery_outcome_code(*outcome)))
+        .and_then(|event| event.code_field("channel", interaction_channel_code(*channel)))
+        .and_then(|event| event.field("delivered_parts", serde_json::Value::from(*delivered_parts)))
+        .and_then(|event| event.redacted("message_content")),
+    };
+    if let Ok(event) = event {
+        let _ = observer.record(event);
+    }
+}
+
+#[cfg(feature = "live-grid")]
+fn record_perception_observation(
+    observer: &metacrate_grid_agent::Observability,
+    observation: &metacrate_grid_agent::PerceptionObservation,
+) {
+    use metacrate_grid_agent::{
+        CorrelationIds, EventDraft, EventFamily, EventOrigin, EventSeverity, PerceptionOutcome,
+    };
+    observer
+        .metrics()
+        .record_tool_latency(Duration::from_millis(observation.duration_millis));
+    let result = match observation.outcome {
+        PerceptionOutcome::Completed => "completed",
+        PerceptionOutcome::Rejected => "rejected",
+    };
+    let event = EventDraft::new(
+        EventFamily::ToolResult,
+        EventSeverity::Info,
+        "perception",
+        EventOrigin::Service,
+    )
+    .and_then(|event| {
+        event.correlation(CorrelationIds {
+            action_id: Some(observation.call_id.as_str().to_owned()),
+            ..CorrelationIds::default()
+        })
+    })
+    .map(|event| event.duration_millis(observation.duration_millis))
+    .and_then(|event| event.result_code(result))
+    .and_then(|event| event.identifier_field("tool", observation.tool.as_str()))
+    .and_then(|event| {
+        event.field(
+            "result_bytes",
+            serde_json::Value::from(observation.result_bytes),
+        )
+    })
+    .and_then(|event| event.field("cache_hit", serde_json::Value::from(observation.cache_hit)))
+    .and_then(|event| event.redacted("tool_result"));
+    if let Ok(event) = event {
+        let _ = observer.record(event);
+    }
+}
+
+#[cfg(feature = "live-grid")]
+fn record_behavior_observation(
+    observer: &metacrate_grid_agent::Observability,
+    observation: &metacrate_grid_agent::BehaviorObservation,
+) {
+    use metacrate_grid_agent::{
+        BehaviorObservation, CorrelationIds, EventDraft, EventFamily, EventOrigin, EventSeverity,
+    };
+    let event = match observation {
+        BehaviorObservation::Transition { from, to, .. } => EventDraft::new(
+            EventFamily::BehaviorTransition,
+            EventSeverity::Info,
+            "behavior",
+            EventOrigin::Service,
+        )
+        .and_then(|event| event.code_field("from", behavior_mode_code(*from)))
+        .and_then(|event| event.code_field("to", behavior_mode_code(*to))),
+        BehaviorObservation::Action {
+            action_id,
+            generation,
+            action,
+            duration_millis,
+            outcome,
+            ..
+        } => EventDraft::new(
+            EventFamily::ToolResult,
+            EventSeverity::Info,
+            "behavior",
+            EventOrigin::Service,
+        )
+        .and_then(|event| {
+            event.correlation(CorrelationIds {
+                action_id: Some(action_id.clone()),
+                ..CorrelationIds::default()
+            })
+        })
+        .map(|event| event.duration_millis(*duration_millis))
+        .and_then(|event| event.result_code(behavior_outcome_code(*outcome)))
+        .and_then(|event| event.identifier_field("tool", action))
+        .and_then(|event| {
+            event.field(
+                "generation",
+                serde_json::Value::from(generation.unwrap_or(0)),
+            )
+        }),
+    };
+    if let Ok(event) = event {
+        let _ = observer.record(event);
+    }
+}
+
+#[cfg(feature = "live-grid")]
+const fn session_reason_code(reason: metacrate_grid_agent::SessionReason) -> &'static str {
+    use metacrate_grid_agent::SessionReason;
+    match reason {
+        SessionReason::Startup => "startup",
+        SessionReason::LoginSucceeded => "login_succeeded",
+        SessionReason::AgentReady => "agent_ready",
+        SessionReason::ReadinessLost => "readiness_lost",
+        SessionReason::TransientTransport => "transient_transport",
+        SessionReason::Maintenance => "maintenance",
+        SessionReason::Kicked => "kicked",
+        SessionReason::SimulatorDisconnected => "simulator_disconnected",
+        SessionReason::ServerFailure => "server_failure",
+        SessionReason::InvalidCredentials => "invalid_credentials",
+        SessionReason::InvalidConfiguration => "invalid_configuration",
+        SessionReason::StableSessionReset => "stable_session_reset",
+        SessionReason::OperatorPause => "operator_pause",
+        SessionReason::OperatorResume => "operator_resume",
+        SessionReason::OperatorForceReconnect => "operator_force_reconnect",
+        SessionReason::OperatorLogout => "operator_logout",
+        SessionReason::ShutdownRequested => "shutdown_requested",
+        SessionReason::ShutdownComplete => "shutdown_complete",
+    }
+}
+
+#[cfg(feature = "live-grid")]
+const fn memory_reason_code(reason: metacrate_grid_agent::MemoryReason) -> &'static str {
+    use metacrate_grid_agent::MemoryReason;
+    match reason {
+        MemoryReason::SessionCreated => "session_created",
+        MemoryReason::PublicExpired => "public_expired",
+        MemoryReason::DirectImExpired => "direct_im_expired",
+        MemoryReason::TurnCompacted => "turn_compacted",
+        MemoryReason::ByteCompacted => "byte_compacted",
+        MemoryReason::ToolResultCompacted => "tool_result_compacted",
+        MemoryReason::SessionLimitEviction => "session_limit_eviction",
+        MemoryReason::TotalByteEviction => "total_byte_eviction",
+        MemoryReason::OperatorDeleted => "operator_deleted",
+        MemoryReason::OperatorExpired => "operator_expired",
+        MemoryReason::CorruptSnapshotQuarantined => "corrupt_snapshot_quarantined",
+        MemoryReason::UnsupportedSnapshotQuarantined => "unsupported_snapshot_quarantined",
+        MemoryReason::SnapshotTooLargeQuarantined => "snapshot_too_large_quarantined",
+        MemoryReason::PermissionsNotVerified => "permissions_not_verified",
+    }
+}
+
+#[cfg(feature = "live-grid")]
+const fn suppression_reason_code(reason: metacrate_grid_agent::SuppressionReason) -> &'static str {
+    use metacrate_grid_agent::SuppressionReason;
+    match reason {
+        SuppressionReason::SelfEcho => "self_echo",
+        SuppressionReason::Duplicate => "duplicate",
+        SuppressionReason::Muted => "muted",
+        SuppressionReason::UnsupportedSource => "unsupported_source",
+        SuppressionReason::UnsupportedDialog => "unsupported_dialog",
+        SuppressionReason::AmbientPublicChat => "ambient_public_chat",
+        SuppressionReason::SenderQueueFull => "sender_queue_full",
+        SuppressionReason::SenderLimit => "sender_limit",
+        SuppressionReason::Disconnected => "disconnected",
+        SuppressionReason::UnsafeResponse => "unsafe_response",
+        SuppressionReason::Cancelled => "cancelled",
+    }
+}
+
+#[cfg(feature = "live-grid")]
+const fn interaction_origin_code(origin: metacrate_grid_agent::InteractionOrigin) -> &'static str {
+    use metacrate_grid_agent::InteractionOrigin;
+    match origin {
+        InteractionOrigin::Public => "public",
+        InteractionOrigin::UnprivilegedIm => "unprivileged_im",
+        InteractionOrigin::AuthorizedIm => "authorized_im",
+    }
+}
+
+#[cfg(feature = "live-grid")]
+const fn interaction_intent_code(intent: metacrate_grid_agent::InteractionIntent) -> &'static str {
+    use metacrate_grid_agent::InteractionIntent;
+    match intent {
+        InteractionIntent::Informational => "informational",
+        InteractionIntent::PublicCommandDenied => "public_command_denied",
+        InteractionIntent::PolicyGatedCommand => "policy_gated_command",
+        InteractionIntent::PolicyGatedLslRequest => "policy_gated_lsl_request",
+    }
+}
+
+#[cfg(feature = "live-grid")]
+const fn delivery_outcome_code(outcome: metacrate_grid_agent::DeliveryOutcome) -> &'static str {
+    use metacrate_grid_agent::DeliveryOutcome;
+    match outcome {
+        DeliveryOutcome::Succeeded => "succeeded",
+        DeliveryOutcome::Failed => "failed",
+        DeliveryOutcome::TimedOut => "timed_out",
+        DeliveryOutcome::PolicyDenied => "policy_denied",
+    }
+}
+
+#[cfg(feature = "live-grid")]
+const fn interaction_channel_code(
+    channel: metacrate_grid_agent::InteractionChannel,
+) -> &'static str {
+    use metacrate_grid_agent::InteractionChannel;
+    match channel {
+        InteractionChannel::PublicChat => "public_chat",
+        InteractionChannel::DirectIm => "direct_im",
+    }
+}
+
+#[cfg(feature = "live-grid")]
+const fn behavior_mode_code(mode: metacrate_grid_agent::BehaviorMode) -> &'static str {
+    use metacrate_grid_agent::BehaviorMode;
+    match mode {
+        BehaviorMode::Offline => "offline",
+        BehaviorMode::Settling => "settling",
+        BehaviorMode::Available => "available",
+        BehaviorMode::Engaged => "engaged",
+        BehaviorMode::Executing => "executing",
+        BehaviorMode::Roaming => "roaming",
+        BehaviorMode::Paused => "paused",
+        BehaviorMode::Recovering => "recovering",
+    }
+}
+
+#[cfg(feature = "live-grid")]
+const fn behavior_outcome_code(outcome: metacrate_grid_agent::BehaviorOutcome) -> &'static str {
+    use metacrate_grid_agent::BehaviorOutcome;
+    match outcome {
+        BehaviorOutcome::Completed => "completed",
+        BehaviorOutcome::Cancelled => "cancelled",
+        BehaviorOutcome::TimedOut => "timed_out",
+        BehaviorOutcome::Stuck => "stuck",
+        BehaviorOutcome::Rejected => "rejected",
+        BehaviorOutcome::Preempted => "preempted",
+    }
+}

crates/metacrate-grid-agent/src/observability_tests.rs

@@ -0,0 +1,423 @@
+use crate::observability::{
+    CorrelationIds, DiagnosticDirection, EventDraft, EventFamily, EventOrigin, EventSeverity,
+    JournalConfig, OBSERVABILITY_SCHEMA_VERSION, Observability, ObservabilityLimits, OutcomeCode,
+    SensitiveDiagnosticPart, StructuredEvent, UnifiedPolicyAudit, replay_journal,
+};
+use crate::policy::{
+    MemoryPolicyAudit, OriginClass, PolicyAuditRecord, PolicyAuditSink, PolicyDisposition,
+    PolicyFinalOutcome, PolicyReasonCode, ResourceCost,
+};
+use crate::types::{BoundedText, MAX_IDENTIFIER_BYTES};
+use serde_json::{Value, json};
+use std::fs::{self, OpenOptions};
+use std::io::Write;
+use std::path::PathBuf;
+use std::sync::atomic::{AtomicU64, Ordering};
+use std::time::Duration;
+
+static NEXT_TEMP: AtomicU64 = AtomicU64::new(1);
+const SECRET_CANARY: &str = "OBSERVABILITY-SECRET-CANARY-9f7d2";
+
+fn limits() -> ObservabilityLimits {
+    ObservabilityLimits {
+        ring_events: 8,
+        subscriber_queue: 2,
+        max_subscribers: 2,
+        journal_queue: 8,
+        max_event_bytes: 4_096,
+        max_diagnostic_entries: 4,
+    }
+}
+
+fn temp_dir(label: &str) -> PathBuf {
+    let path = std::env::temp_dir().join(format!(
+        "metacrate-observability-{label}-{}-{}",
+        std::process::id(),
+        NEXT_TEMP.fetch_add(1, Ordering::Relaxed)
+    ));
+    fs::create_dir_all(&path).expect("create test directory");
+    path
+}
+
+fn draft(family: EventFamily, request: &str) -> EventDraft {
+    EventDraft::new(family, EventSeverity::Info, "test", EventOrigin::Service)
+        .expect("draft")
+        .correlation(CorrelationIds {
+            request_id: Some(request.to_owned()),
+            ..CorrelationIds::default()
+        })
+        .expect("correlation")
+        .result_code("completed")
+        .expect("result")
+        .redacted("content_omitted")
+        .expect("redaction")
+        .field("count", Value::from(1))
+        .expect("field")
+}
+
+#[test]
+fn golden_schema_covers_every_event_family_and_preserves_extensions() {
+    let observer = Observability::memory(ObservabilityLimits {
+        ring_events: 32,
+        ..limits()
+    })
+    .expect("observer");
+    let families = [
+        EventFamily::LifecycleTransition,
+        EventFamily::InboundMessage,
+        EventFamily::OutboundMessage,
+        EventFamily::SessionCreated,
+        EventFamily::SessionExpired,
+        EventFamily::InferenceRequest,
+        EventFamily::InferenceResult,
+        EventFamily::ModelActionSummary,
+        EventFamily::ToolProposed,
+        EventFamily::PolicyDecision,
+        EventFamily::ApprovalDecision,
+        EventFamily::ToolStarted,
+        EventFamily::ToolProgress,
+        EventFamily::ToolResult,
+        EventFamily::BehaviorTransition,
+        EventFamily::ScheduledJob,
+        EventFamily::ControlCommand,
+        EventFamily::Shutdown,
+        EventFamily::DiagnosticEnvelope,
+    ];
+    for (index, family) in families.into_iter().enumerate() {
+        observer
+            .record(draft(family, &format!("request-{index}")))
+            .expect("record family");
+    }
+    let events = observer.snapshot();
+    assert_eq!(events.len(), families.len());
+    let event = events.last().expect("last event");
+    let value = serde_json::to_value(event).expect("JSON");
+    assert_eq!(value["schema_version"], OBSERVABILITY_SCHEMA_VERSION);
+    assert_eq!(value["family"], "diagnostic_envelope");
+    assert_eq!(value["redaction_flags"], json!(["content_omitted"]));
+
+    let mut value = value;
+    value["future_top_level"] = json!({"compatible": true});
+    value["fields"]["future_scalar"] = json!(42);
+    let decoded: StructuredEvent = serde_json::from_value(value).expect("future event");
+    assert_eq!(decoded.extensions["future_top_level"]["compatible"], true);
+    assert_eq!(decoded.fields["future_scalar"], 42);
+    let encoded = serde_json::to_value(decoded).expect("round trip");
+    assert_eq!(encoded["future_top_level"]["compatible"], true);
+}
+
+#[test]
+fn secret_canaries_are_unrepresentable_and_debug_is_redacted() {
+    let observer = Observability::memory(limits()).expect("observer");
+    for sensitive in [
+        SECRET_CANARY,
+        "Bearer OBSERVABILITY-SECRET-CANARY-9f7d2",
+        "https://grid.example/CAPS/OBSERVABILITY-SECRET-CANARY-9f7d2",
+        "password=OBSERVABILITY-SECRET-CANARY-9f7d2",
+        "api_key=OBSERVABILITY-SECRET-CANARY-9f7d2",
+    ] {
+        assert!(
+            EventDraft::new(
+                EventFamily::InboundMessage,
+                EventSeverity::Info,
+                "test",
+                EventOrigin::Grid
+            )
+            .expect("draft")
+            .field("unsafe", Value::String(sensitive.to_owned()))
+            .is_err()
+        );
+    }
+    let part = SensitiveDiagnosticPart {
+        role: SECRET_CANARY,
+        contents: SECRET_CANARY,
+    };
+    assert!(!format!("{part:?}").contains(SECRET_CANARY));
+    observer
+        .capture_diagnostic(DiagnosticDirection::Prompt, "request-safe", &[part])
+        .expect("diagnostic metadata");
+    let rendered = serde_json::to_string(&observer.snapshot()).expect("render events");
+    assert!(!rendered.contains(SECRET_CANARY));
+    assert!(!format!("{observer:?}").contains(SECRET_CANARY));
+    assert!(rendered.contains("content_digest"));
+    assert!(rendered.contains("content_omitted"));
+    let mut external = serde_json::to_value(&observer.snapshot()[0]).expect("external event");
+    external["component"] = Value::String(SECRET_CANARY.to_owned());
+    external["family"] = Value::String(SECRET_CANARY.to_owned());
+    external["result_code"] = Value::String(SECRET_CANARY.to_owned());
+    external["reason_code"] = Value::String(SECRET_CANARY.to_owned());
+    external["redaction_flags"] = serde_json::json!([SECRET_CANARY]);
+    external["future_secret"] = Value::String(SECRET_CANARY.to_owned());
+    let external: StructuredEvent = serde_json::from_value(external).expect("external event");
+    assert!(!format!("{external:?}").contains(SECRET_CANARY));
+}
+
+#[tokio::test]
+async fn ring_overflow_reports_gap_and_slow_subscribers_are_disconnected() {
+    let observer = Observability::memory(ObservabilityLimits {
+        ring_events: 3,
+        subscriber_queue: 1,
+        ..limits()
+    })
+    .expect("observer");
+    for index in 0..5 {
+        observer
+            .record(draft(
+                EventFamily::ToolProgress,
+                &format!("request-{index}"),
+            ))
+            .expect("record");
+    }
+    assert_eq!(observer.snapshot().len(), 3);
+    assert_eq!(observer.metrics().snapshot().dropped_ring_events, 2);
+    let mut replay = observer.subscribe(Some(0)).expect("subscription");
+    assert_eq!(replay.take_gap(), Some((1, 3)));
+    assert_eq!(replay.recv().await.expect("replay").event_id, 3);
+
+    let mut slow = observer.subscribe(None).expect("slow subscription");
+    for index in 5..9 {
+        observer
+            .record(draft(
+                EventFamily::ToolProgress,
+                &format!("request-{index}"),
+            ))
+            .expect("record");
+    }
+    while slow.recv().await.is_some() {}
+    assert!(observer.metrics().snapshot().dropped_subscriber_events >= 1);
+}
+
+#[tokio::test]
+async fn journal_rotates_recovers_truncated_tail_and_replays_without_execution() {
+    let directory = temp_dir("journal");
+    let config = JournalConfig {
+        directory: directory.clone(),
+        max_segment_bytes: 700,
+        max_segments: 3,
+        max_total_bytes: 2_100,
+        sync_each_record: false,
+    };
+    let (observer, runtime) = Observability::journaled(
+        ObservabilityLimits {
+            max_event_bytes: 512,
+            ..limits()
+        },
+        config,
+    )
+    .expect("journal observer");
+    for index in 0..12 {
+        let family = if index == 0 {
+            EventFamily::LifecycleTransition
+        } else if index == 1 {
+            EventFamily::BehaviorTransition
+        } else {
+            EventFamily::ToolResult
+        };
+        let mut event = draft(family, &format!("request-{index}"));
+        if family == EventFamily::LifecycleTransition || family == EventFamily::BehaviorTransition {
+            event = event.code_field("to", "available").expect("state");
+        }
+        if family == EventFamily::ToolResult {
+            event = event
+                .correlation(CorrelationIds {
+                    action_id: Some(format!("action-{index}")),
+                    ..CorrelationIds::default()
+                })
+                .expect("action");
+        }
+        observer.record(event).expect("journal event");
+    }
+    runtime.shutdown(&observer).await.expect("journal shutdown");
+    let last_before_restart = observer.snapshot().last().expect("last event").event_id;
+    let (restarted, restarted_runtime) = Observability::journaled(
+        ObservabilityLimits {
+            max_event_bytes: 512,
+            ..limits()
+        },
+        JournalConfig {
+            directory: directory.clone(),
+            max_segment_bytes: 700,
+            max_segments: 3,
+            max_total_bytes: 2_100,
+            sync_each_record: false,
+        },
+    )
+    .expect("restart journal");
+    let restarted_id = restarted
+        .record(draft(EventFamily::Shutdown, "restart"))
+        .expect("restart event");
+    assert!(restarted_id > last_before_restart);
+    restarted_runtime
+        .shutdown(&restarted)
+        .await
+        .expect("restart shutdown");
+    let active = directory.join("events-current.jsonl.tmp");
+    OpenOptions::new()
+        .append(true)
+        .open(&active)
+        .expect("active")
+        .write_all(b"{\"schema_version\":1")
+        .expect("torn record");
+    let replay = replay_journal(&directory, 4_096, 64).expect("recover replay");
+    assert!(!replay.timeline.is_empty());
+    assert!(replay.tool_outcomes > 0);
+    assert!(replay.unavailable_private_content);
+    assert!(
+        replay
+            .actions
+            .values()
+            .all(|action| action.final_result.is_some())
+    );
+    // Three retained segments, one active file, and one durable ID reservation.
+    assert!(fs::read_dir(&directory).expect("directory").count() <= 5);
+    let journal_bytes = fs::read_dir(&directory)
+        .expect("directory")
+        .filter_map(Result::ok)
+        .filter(|entry| {
+            let name = entry.file_name();
+            let name = name.to_string_lossy();
+            name.ends_with(".jsonl") || name == "events-current.jsonl.tmp"
+        })
+        .map(|entry| entry.metadata().expect("metadata").len())
+        .sum::<u64>();
+    assert!(journal_bytes <= 2_100);
+    fs::remove_dir_all(directory).expect("remove test directory");
+}
+
+#[test]
+fn metrics_have_fixed_fields_and_saturating_latency_summaries() {
+    let observer = Observability::memory(limits()).expect("observer");
+    let metrics = observer.metrics();
+    metrics.set_health(true, 3);
+    metrics.set_load(4, 5, 6, 7);
+    metrics.set_rate_limit(8, 9);
+    metrics.record_inference_latency(Duration::from_millis(11));
+    metrics.record_inference_latency(Duration::from_millis(13));
+    metrics.record_tool_latency(Duration::from_millis(17));
+    metrics.record_inference_outcome(OutcomeCode::Failed);
+    metrics.record_tool_outcome(OutcomeCode::Completed);
+    metrics.record_policy_outcome(OutcomeCode::Denied);
+    let snapshot = metrics.snapshot();
+    assert!(snapshot.ready);
+    assert_eq!(snapshot.reconnects, 3);
+    assert_eq!(snapshot.active_sessions, 4);
+    assert_eq!(snapshot.queue_capacity, 7);
+    assert_eq!(snapshot.inference_latency.samples, 2);
+    assert_eq!(snapshot.inference_latency.total_millis, 24);
+    assert_eq!(snapshot.inference_latency.maximum_millis, 13);
+    assert_eq!(snapshot.tool_latency.maximum_millis, 17);
+    assert_eq!(snapshot.inference_outcomes.failed, 1);
+    assert_eq!(snapshot.tool_outcomes.completed, 1);
+    assert_eq!(snapshot.policy_outcomes.denied, 1);
+    assert_eq!(
+        serde_json::to_value(snapshot)
+            .expect("metrics")
+            .as_object()
+            .expect("object")
+            .len(),
+        18
+    );
+}
+
+#[test]
+fn unified_policy_audit_correlates_outcomes_without_exposing_inputs() {
+    let probe_observer = Observability::memory(limits()).expect("probe observer");
+    let probe = EventDraft::new(
+        EventFamily::ToolResult,
+        EventSeverity::Info,
+        "policy",
+        EventOrigin::Policy,
+    )
+    .and_then(|event| {
+        event.correlation(CorrelationIds {
+            session_id: Some(SECRET_CANARY.to_owned()),
+            request_id: Some(SECRET_CANARY.to_owned()),
+            action_id: Some("authorization-7".to_owned()),
+            ..CorrelationIds::default()
+        })
+    })
+    .and_then(|event| event.result_code("completed"))
+    .and_then(|event| event.reason_code("allowed"))
+    .and_then(|event| event.identifier_field("tool", SECRET_CANARY))
+    .and_then(|event| event.code_field("disposition", "allowed"))
+    .and_then(|event| event.field("tool_calls", Value::from(1)))
+    .and_then(|event| event.field("movement_millimeters", Value::from(0)))
+    .and_then(|event| event.redacted("tool_arguments"))
+    .expect("policy event draft");
+    probe_observer.record(probe).expect("policy event record");
+
+    let observer = Observability::memory(limits()).expect("observer");
+    let memory = std::sync::Arc::new(MemoryPolicyAudit::new(8).expect("policy audit"));
+    let audit = UnifiedPolicyAudit::new(memory.clone(), observer.clone());
+    audit
+        .emit(PolicyAuditRecord {
+            recorded_unix_millis: 1,
+            authorization_id: Some(7),
+            approval_id: None,
+            origin: OriginClass::AuthorizedIm,
+            origin_avatar_id: None,
+            principal: BoundedText::<MAX_IDENTIFIER_BYTES>::new("principal", SECRET_CANARY)
+                .expect("principal"),
+            session_id: BoundedText::<MAX_IDENTIFIER_BYTES>::new("session", SECRET_CANARY)
+                .expect("session"),
+            correlation_id: BoundedText::<MAX_IDENTIFIER_BYTES>::new("correlation", SECRET_CANARY)
+                .expect("correlation"),
+            tool: BoundedText::<MAX_IDENTIFIER_BYTES>::new("tool", SECRET_CANARY).expect("tool"),
+            disposition: PolicyDisposition::Allowed,
+            reason: PolicyReasonCode::Allowed,
+            applied_budget: ResourceCost::one_call(),
+            arguments_hash: BoundedText::<64>::new("hash", SECRET_CANARY).expect("hash"),
+            final_outcome: PolicyFinalOutcome::Completed,
+        })
+        .expect("emit audit");
+    assert_eq!(memory.snapshot().len(), 1);
+    let rendered = serde_json::to_string(&observer.snapshot()).expect("events");
+    assert!(!rendered.contains(SECRET_CANARY));
+    let families = observer
+        .snapshot()
+        .iter()
+        .map(|event| event.family.clone())
+        .collect::<Vec<_>>();
+    assert!(
+        families.iter().any(|family| family == "tool_result"),
+        "{families:?}"
+    );
+    assert_eq!(observer.metrics().snapshot().policy_outcomes.completed, 1);
+}
+
+#[tokio::test]
+async fn nonblocking_flood_remains_bounded_when_journal_queue_is_saturated() {
+    let directory = temp_dir("flood");
+    let flood_limits = ObservabilityLimits {
+        ring_events: 16,
+        subscriber_queue: 1,
+        max_subscribers: 1,
+        journal_queue: 1,
+        max_event_bytes: 4_096,
+        max_diagnostic_entries: 0,
+    };
+    let (observer, runtime) = Observability::journaled(
+        flood_limits,
+        JournalConfig {
+            directory: directory.clone(),
+            max_segment_bytes: 4_096,
+            max_segments: 2,
+            max_total_bytes: 8_192,
+            sync_each_record: true,
+        },
+    )
+    .expect("journal observer");
+    let _slow = observer.subscribe(None).expect("subscriber");
+    for index in 0..10_000 {
+        observer
+            .record(draft(EventFamily::ToolProgress, &format!("flood-{index}")))
+            .expect("bounded record");
+    }
+    assert_eq!(observer.snapshot().len(), flood_limits.ring_events);
+    let snapshot = observer.metrics().snapshot();
+    assert_eq!(snapshot.recorded_events, 10_000);
+    assert!(snapshot.dropped_journal_events > 0);
+    assert!(snapshot.dropped_subscriber_events > 0);
+    runtime.shutdown(&observer).await.expect("shutdown");
+    fs::remove_dir_all(directory).expect("remove test directory");
+}