Add agent context compaction

src/runtime.rs

@@ -1,4 +1,4 @@
-use crate::engine::{ChatTurn, GenerationOutput, Generator};
+use crate::engine::{ChatTurn, CompactionOutput, GenerationOutput, Generator};
 use crate::metrics::{Metrics, WorkSource};
 use crate::settings::{EngineSettings, TurnSettings};
 use std::path::PathBuf;
@@ -56,6 +56,7 @@ pub(crate) enum GenerationEvent {
         tokens_per_second: Option<f32>,
     },
     Finished(Result<GenerationOutput, String>),
+    Compacted(Result<CompactionOutput, String>),
 }
 
 struct Command {
@@ -63,6 +64,7 @@ struct Command {
     turn: TurnSettings,
     messages: Vec<ChatTurn>,
     checkpoint: CheckpointTarget,
+    compact_reason: Option<String>,
     idle_timeout: Duration,
     cancel: Arc<AtomicBool>,
     events: Sender<GenerationEvent>,
@@ -98,6 +100,7 @@ impl GenerationService {
                 turn,
                 messages,
                 checkpoint,
+                compact_reason: None,
                 idle_timeout,
                 cancel: Arc::clone(&cancel),
                 events,
@@ -112,6 +115,35 @@ impl GenerationService {
             cancel,
         })
     }
+
+    pub(crate) fn compact(
+        &self,
+        engine: EngineSettings,
+        turn: TurnSettings,
+        messages: Vec<ChatTurn>,
+        reason: &str,
+        idle_timeout: Duration,
+    ) -> Result<ActiveGeneration, String> {
+        let cancel = Arc::new(AtomicBool::new(false));
+        let (events, receiver) = mpsc::channel();
+        self.metrics.request_queued(WorkSource::LocalChat);
+        self.commands
+            .send(Command {
+                engine,
+                turn,
+                messages,
+                checkpoint: CheckpointTarget::OneShot(PathBuf::new()),
+                compact_reason: Some(reason.to_owned()),
+                idle_timeout,
+                cancel: Arc::clone(&cancel),
+                events,
+            })
+            .map_err(|_| "The model runtime stopped unexpectedly.".to_owned())?;
+        Ok(ActiveGeneration {
+            events: receiver,
+            cancel,
+        })
+    }
 }
 
 fn run(commands: Receiver<Command>, metrics: Arc<Metrics>) {
@@ -228,6 +260,24 @@ fn run_command(
                 tokens_per_second,
             });
         };
+        if let Some(reason) = &command.compact_reason {
+            let result = generator.compact(
+                &command.messages,
+                &command.turn,
+                reason,
+                &command.cancel,
+                &mut progress,
+            );
+            match &result {
+                Ok(_) => {
+                    metrics.request_finished(source, request_started.elapsed(), 0, 0, 0, None, 0)
+                }
+                Err(_) => metrics.request_failed(request_started.elapsed()),
+            }
+            let _ = command.events.send(GenerationEvent::Compacted(result));
+            *last_used = Instant::now();
+            return;
+        }
         let result = match command.checkpoint {
             CheckpointTarget::Local(path) => generator.generate(
                 &path,