feat(grid-agent): add structured observability and replay (#127)

crates/metacrate-grid-agent/src/control_plane.rs

@@ -4,6 +4,7 @@
 
 use crate::backend::BackendFuture;
 use crate::config::SecretString;
+use crate::observability::{MetricsSnapshot, StructuredEvent};
 use libremetaverse_types::compat::{CancellationToken, CancellationTokenSource};
 use serde::{Deserialize, Serialize};
 use std::collections::{BTreeMap, BTreeSet, VecDeque};
@@ -244,11 +245,13 @@ pub struct AuditEventView {
 #[serde(tag = "kind", content = "data", rename_all = "snake_case")]
 pub enum ControlPayload {
     Health(HealthView),
+    Metrics(MetricsSnapshot),
     Runtime(RuntimeView),
     Sessions(Page<SessionMetadataView>),
     ScheduledJobs(Page<ScheduledJobView>),
     PendingApprovals(Page<PendingApprovalView>),
     AuditEvents(Page<AuditEventView>),
+    ObservabilityEvents(Page<StructuredEvent>),
     Accepted { operation_id: String },
     Completed,
     Subscribed { current_sequence: u64 },
@@ -266,6 +269,7 @@ pub enum ConversationChannelView {
 #[serde(tag = "method", content = "parameters", rename_all = "snake_case")]
 pub enum ControlRequest {
     Health,
+    Metrics,
     Runtime,
     ListSessions {
         page: PageRequest,
@@ -279,6 +283,9 @@ pub enum ControlRequest {
     ListAuditEvents {
         page: PageRequest,
     },
+    ListObservabilityEvents {
+        page: PageRequest,
+    },
     SubscribeEvents {
         after_sequence: Option<u64>,
     },
@@ -326,11 +333,13 @@ impl ControlRequest {
         !matches!(
             self,
             Self::Health
+                | Self::Metrics
                 | Self::Runtime
                 | Self::ListSessions { .. }
                 | Self::ListScheduledJobs { .. }
                 | Self::ListPendingApprovals { .. }
                 | Self::ListAuditEvents { .. }
+                | Self::ListObservabilityEvents { .. }
                 | Self::SubscribeEvents { .. }
         )
     }
@@ -340,7 +349,8 @@ impl ControlRequest {
             Self::ListSessions { page }
             | Self::ListScheduledJobs { page }
             | Self::ListPendingApprovals { page }
-            | Self::ListAuditEvents { page } => page.valid(),
+            | Self::ListAuditEvents { page }
+            | Self::ListObservabilityEvents { page } => page.valid(),
             Self::CancelRequest { target_request_id } => valid_identifier(target_request_id),
             Self::CancelAction { action_id } => valid_identifier(action_id),
             Self::DecideApproval { approval_id, .. } => *approval_id != 0,
@@ -952,7 +962,7 @@ enum ClientFrame {
 #[serde(tag = "frame", content = "body", rename_all = "snake_case")]
 enum ServerFrame {
     Hello(ServerHello),
-    Response(ControlResponseEnvelope),
+    Response(Box<ControlResponseEnvelope>),
     Event(ControlEvent),
 }
 
@@ -1245,7 +1255,10 @@ where
                     true,
                 ),
             );
-            if outbound.try_send(ServerFrame::Response(response)).is_err() {
+            if outbound
+                .try_send(ServerFrame::Response(Box::new(response)))
+                .is_err()
+            {
                 break;
             }
             continue;
@@ -1256,7 +1269,7 @@ where
         tasks.spawn(async move {
             let (response, subscription) = request_core.request(envelope).await;
             if request_outbound
-                .try_send(ServerFrame::Response(response))
+                .try_send(ServerFrame::Response(Box::new(response)))
                 .is_err()
             {
                 request_cancel.cancel();
@@ -1534,7 +1547,7 @@ async fn tcp_client_reader(
         match frame {
             Ok(ServerFrame::Response(response)) => {
                 if let Some(sender) = lock(&shared.pending).remove(&response.request_id) {
-                    let _ = sender.send(response);
+                    let _ = sender.send(*response);
                 }
             }
             Ok(ServerFrame::Event(event)) => {
@@ -1744,11 +1757,13 @@ fn role_name(role: ControlRole) -> &'static str {
 fn request_name(request: &ControlRequest) -> &'static str {
     match request {
         ControlRequest::Health => "health",
+        ControlRequest::Metrics => "metrics",
         ControlRequest::Runtime => "runtime",
         ControlRequest::ListSessions { .. } => "list_sessions",
         ControlRequest::ListScheduledJobs { .. } => "list_scheduled_jobs",
         ControlRequest::ListPendingApprovals { .. } => "list_pending_approvals",
         ControlRequest::ListAuditEvents { .. } => "list_audit_events",
+        ControlRequest::ListObservabilityEvents { .. } => "list_observability_events",
         ControlRequest::SubscribeEvents { .. } => "subscribe_events",
         ControlRequest::CancelRequest { .. } => "cancel_request",
         ControlRequest::PauseAutonomy => "pause_autonomy",