feat(grid-agent): add generic LLM tool loop (#119)
Some checks failed
CI / rust-skia (Rust only) (push) Successful in 2m50s
CI / required (push) Failing after 2m46s

This commit is contained in:
2026-08-17 20:45:56 +00:00
parent 1e1e95a58a
commit a46bc42a8f
11 changed files with 2365 additions and 6 deletions

View File

@@ -11,13 +11,14 @@ publish = false
[dependencies]
libremetaverse = { version = "0.0.1", path = "../libremetaverse", default-features = false, optional = true }
libremetaverse-types = { version = "0.0.1", path = "../libremetaverse-types" }
reqwest = { version = "0.13.4", default-features = false, features = ["rustls"] }
serde = { version = "1", features = ["derive"] }
serde_json = "1"
tokio = { version = "1.53.1", features = ["macros", "rt", "sync", "time"] }
url = "2.5.8"
[target.'cfg(any(unix, windows))'.dependencies]
tokio = { version = "1.53.1", features = ["rt-multi-thread", "signal"] }
tokio = { version = "1.53.1", features = ["io-util", "net", "rt-multi-thread", "signal"] }
[features]
default = []

View File

@@ -2,10 +2,11 @@
This package is the bounded, provider-neutral foundation for the MetaCrate
OpenSim grid agent. It contains a reusable library and the
`metacrate-grid-agent` service binary. The first implementation is deliberately
`metacrate-grid-agent` service binary. The default service graph is deliberately
offline: it publishes a deterministic ready event, accepts control commands,
and shuts down both owned tasks without contacting a grid or LLM. The
`live-grid` feature exposes the side-effect-free owner for the existing
and shuts down both owned tasks without contacting a grid or LLM. The library
also provides the bounded exact-endpoint LLM transport and tool loop for live
adapters. The `live-grid` feature exposes the side-effect-free owner for the existing
`libremetaverse::GridClient`; later live adapters must extend that manager graph
instead of adding a protocol client.
@@ -37,4 +38,6 @@ cargo run --locked -p metacrate-grid-agent -- \
```
See [`../../docs/grid-agent-architecture.md`](../../docs/grid-agent-architecture.md)
for queue/task ownership, shutdown, and trust boundaries.
for queue/task ownership, shutdown, and trust boundaries. See
[`../../docs/grid-agent-llm.md`](../../docs/grid-agent-llm.md) for the LLM wire
compatibility envelope, retry rules, and tool-loop safety contract.

View File

@@ -6,7 +6,9 @@
pub mod backend;
pub mod config;
pub mod llm;
pub mod service;
pub mod tool_loop;
pub mod types;
#[cfg(feature = "live-grid")]
@@ -17,7 +19,15 @@ pub use config::{
GridConnection, Limits, LlmConnection, MapEnvironment, OperatingMode, SecretString,
StdEnvironment, Timeouts,
};
pub use llm::{
Completion, CompletionMessage, ContentPart, ImageDetail, LlmClient, LlmError,
LlmTransportLimits, ToolDefinition, ToolSchema, Usage,
};
pub use service::{AgentService, ServiceError, ServiceHandle, ServiceState};
pub use tool_loop::{
HistorySummarizer, SessionGeneration, ToolExecution, ToolExecutor, ToolFuture, ToolLoop,
ToolLoopError, ToolLoopLimits, ToolLoopOutcome,
};
pub use types::{
BoundaryError, BoundedText, BoundedVec, ControlCommand, Conversation, ConversationMessage,
GridEvent, GridEventKind, LlmRequest, LlmResult, MessageRole, ObservableEvent, PolicyDecision,

View File

@@ -0,0 +1,764 @@
//! Provider-neutral exact-endpoint LLM transport and bounded wire envelope.
#![allow(clippy::missing_errors_doc)] // Public methods share the exhaustive typed LlmError boundary.
use crate::config::LlmConnection;
use crate::types::{
BoundedText, BoundedVec, MAX_BODY_BYTES, MAX_CONVERSATION_MESSAGES, MAX_IDENTIFIER_BYTES,
MAX_MESSAGE_BYTES, MAX_TOOL_CALLS, MessageRole, ProposedToolCall,
};
use libremetaverse_types::compat::CancellationToken;
use serde::Deserialize;
use serde_json::{Map, Value, json};
use std::collections::{BTreeMap, BTreeSet};
use std::error::Error;
use std::fmt;
use std::sync::Arc;
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::{Duration, Instant};
use tokio::sync::Semaphore;
const MAX_TOOLS: usize = 64;
const MAX_SCHEMA_PROPERTIES: usize = 128;
const MAX_SCHEMA_DEPTH: usize = 16;
const MAX_SCHEMA_NODES: usize = 1_024;
#[derive(Clone, Debug, Eq, PartialEq)]
pub enum ContentPart {
Text(BoundedText<MAX_MESSAGE_BYTES>),
Image {
url: BoundedText<MAX_BODY_BYTES>,
detail: ImageDetail,
},
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum ImageDetail {
Auto,
Low,
High,
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct CompletionMessage {
pub role: MessageRole,
pub content: BoundedVec<ContentPart, 16>,
pub tool_call_id: Option<BoundedText<MAX_IDENTIFIER_BYTES>>,
pub proposed_calls: BoundedVec<ProposedToolCall, MAX_TOOL_CALLS>,
}
impl CompletionMessage {
pub fn text(role: MessageRole, text: impl Into<String>) -> Result<Self, LlmError> {
let mut content = BoundedVec::new();
content
.try_push(
"completion_message.content",
ContentPart::Text(BoundedText::new("completion_message.text", text)?),
)
.map_err(LlmError::Boundary)?;
Ok(Self {
role,
content,
tool_call_id: None,
proposed_calls: BoundedVec::new(),
})
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub enum ToolSchema {
Object {
properties: BTreeMap<String, ToolSchema>,
required: BTreeSet<String>,
additional_properties: bool,
},
String,
Integer,
Number,
Boolean,
Array {
items: Box<ToolSchema>,
max_items: usize,
},
}
impl ToolSchema {
pub fn validate_schema(&self) -> Result<(), LlmError> {
let mut remaining_nodes = MAX_SCHEMA_NODES;
self.validate_schema_inner(0, &mut remaining_nodes)
}
fn validate_schema_inner(
&self,
depth: usize,
remaining_nodes: &mut usize,
) -> Result<(), LlmError> {
if depth > MAX_SCHEMA_DEPTH || *remaining_nodes == 0 {
return Err(LlmError::InvalidToolSchema);
}
*remaining_nodes -= 1;
match self {
Self::Object {
properties,
required,
..
} => {
if properties.len() > MAX_SCHEMA_PROPERTIES
|| required.len() > MAX_SCHEMA_PROPERTIES
|| required.iter().any(|name| !properties.contains_key(name))
|| properties
.keys()
.any(|name| name.is_empty() || name.len() > MAX_IDENTIFIER_BYTES)
{
return Err(LlmError::InvalidToolSchema);
}
for schema in properties.values() {
schema.validate_schema_inner(depth + 1, remaining_nodes)?;
}
Ok(())
}
Self::Array { items, max_items } => {
if *max_items == 0 || *max_items > 256 {
return Err(LlmError::InvalidToolSchema);
}
items.validate_schema_inner(depth + 1, remaining_nodes)
}
_ => Ok(()),
}
}
pub fn validate_value(&self, value: &Value) -> Result<(), LlmError> {
match self {
Self::Object {
properties,
required,
additional_properties,
} => {
let object = value.as_object().ok_or(LlmError::InvalidToolArguments)?;
if required.iter().any(|name| !object.contains_key(name))
|| (!additional_properties
&& object.keys().any(|name| !properties.contains_key(name)))
{
return Err(LlmError::InvalidToolArguments);
}
for (name, value) in object {
if let Some(schema) = properties.get(name) {
schema.validate_value(value)?;
}
}
Ok(())
}
Self::String if value.is_string() => Ok(()),
Self::Integer if value.as_i64().is_some() || value.as_u64().is_some() => Ok(()),
Self::Number if value.is_number() => Ok(()),
Self::Boolean if value.is_boolean() => Ok(()),
Self::Array { items, max_items } => {
let values = value.as_array().ok_or(LlmError::InvalidToolArguments)?;
if values.len() > *max_items {
return Err(LlmError::InvalidToolArguments);
}
for value in values {
items.validate_value(value)?;
}
Ok(())
}
_ => Err(LlmError::InvalidToolArguments),
}
}
fn wire_value(&self) -> Value {
match self {
Self::Object {
properties,
required,
additional_properties,
} => {
let properties = properties
.iter()
.map(|(name, schema)| (name.clone(), schema.wire_value()))
.collect::<Map<_, _>>();
json!({
"type": "object",
"properties": properties,
"required": required,
"additionalProperties": additional_properties
})
}
Self::String => json!({"type":"string"}),
Self::Integer => json!({"type":"integer"}),
Self::Number => json!({"type":"number"}),
Self::Boolean => json!({"type":"boolean"}),
Self::Array { items, max_items } => {
json!({"type":"array", "items":items.wire_value(), "maxItems":max_items})
}
}
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct ToolDefinition {
pub name: BoundedText<MAX_IDENTIFIER_BYTES>,
pub description: BoundedText<MAX_MESSAGE_BYTES>,
pub schema: ToolSchema,
pub mutating: bool,
}
impl ToolDefinition {
pub fn validate(&self) -> Result<(), LlmError> {
if !self
.name
.chars()
.all(|character| character.is_ascii_alphanumeric() || matches!(character, '_' | '-'))
{
return Err(LlmError::InvalidToolSchema);
}
self.schema.validate_schema()
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct Usage {
pub prompt_tokens: Option<u64>,
pub completion_tokens: Option<u64>,
pub total_tokens: Option<u64>,
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct Completion {
pub request_id: u64,
pub correlation_id: BoundedText<MAX_IDENTIFIER_BYTES>,
pub message: CompletionMessage,
pub usage: Option<Usage>,
pub latency: Duration,
pub attempts: usize,
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct LlmTransportLimits {
pub connect_timeout: Duration,
pub request_timeout: Duration,
pub read_idle_timeout: Duration,
pub pool_idle_timeout: Duration,
pub total_timeout: Duration,
pub max_prompt_bytes: usize,
pub max_response_bytes: usize,
pub max_concurrent_requests: usize,
pub max_retries: usize,
pub max_retry_delay: Duration,
}
impl Default for LlmTransportLimits {
fn default() -> Self {
Self {
connect_timeout: Duration::from_secs(10),
request_timeout: Duration::from_mins(1),
read_idle_timeout: Duration::from_secs(15),
pool_idle_timeout: Duration::from_secs(30),
total_timeout: Duration::from_secs(90),
max_prompt_bytes: 1024 * 1024,
max_response_bytes: 1024 * 1024,
max_concurrent_requests: 8,
max_retries: 2,
max_retry_delay: Duration::from_secs(5),
}
}
}
impl LlmTransportLimits {
pub fn validate(&self) -> Result<(), LlmError> {
if self.connect_timeout.is_zero()
|| self.request_timeout.is_zero()
|| self.read_idle_timeout.is_zero()
|| self.pool_idle_timeout.is_zero()
|| self.total_timeout.is_zero()
|| self.max_prompt_bytes == 0
|| self.max_prompt_bytes > MAX_BODY_BYTES
|| self.max_response_bytes == 0
|| self.max_response_bytes > MAX_BODY_BYTES
|| self.max_concurrent_requests == 0
|| self.max_concurrent_requests > 256
|| self.max_retries > 8
|| self.max_retry_delay > Duration::from_mins(1)
{
return Err(LlmError::UnsafeLimits);
}
Ok(())
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub enum LlmError {
Boundary(crate::types::BoundaryError),
UnsafeLimits,
InvalidToolSchema,
InvalidToolArguments,
PromptTooLarge,
ResponseTooLarge,
Cancelled,
Timeout,
Transport,
RedirectRefused,
HttpStatus(u16),
MalformedJson,
UnsupportedResponse,
DuplicateToolCallId,
}
impl fmt::Display for LlmError {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Boundary(error) => write!(formatter, "LLM boundary rejected input: {error}"),
Self::UnsafeLimits => formatter.write_str("unsafe LLM transport limits"),
Self::InvalidToolSchema => formatter.write_str("invalid registered tool schema"),
Self::InvalidToolArguments => formatter.write_str("tool arguments do not match schema"),
Self::PromptTooLarge => formatter.write_str("LLM prompt exceeds its byte bound"),
Self::ResponseTooLarge => formatter.write_str("LLM response exceeds its byte bound"),
Self::Cancelled => formatter.write_str("LLM request cancelled"),
Self::Timeout => formatter.write_str("LLM request timed out"),
Self::Transport => formatter.write_str("LLM transport failed"),
Self::RedirectRefused => formatter.write_str("LLM endpoint redirect refused"),
Self::HttpStatus(status) => write!(formatter, "LLM endpoint returned HTTP {status}"),
Self::MalformedJson => formatter.write_str("LLM endpoint returned malformed JSON"),
Self::UnsupportedResponse => formatter.write_str("unsupported LLM response shape"),
Self::DuplicateToolCallId => {
formatter.write_str("LLM response repeated a tool-call ID")
}
}
}
}
impl Error for LlmError {}
impl From<crate::types::BoundaryError> for LlmError {
fn from(value: crate::types::BoundaryError) -> Self {
Self::Boundary(value)
}
}
pub struct LlmClient {
connection: LlmConnection,
client: reqwest::Client,
limits: LlmTransportLimits,
slots: Arc<Semaphore>,
next_request_id: AtomicU64,
}
impl fmt::Debug for LlmClient {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("LlmClient")
.field("endpoint", &self.connection.endpoint_url)
.field("limits", &self.limits)
.field("available_slots", &self.slots.available_permits())
.finish_non_exhaustive()
}
}
impl LlmClient {
pub fn new(connection: LlmConnection, limits: LlmTransportLimits) -> Result<Self, LlmError> {
limits.validate()?;
let client = reqwest::Client::builder()
.redirect(reqwest::redirect::Policy::none())
.connect_timeout(limits.connect_timeout)
.timeout(limits.request_timeout)
.read_timeout(limits.read_idle_timeout)
.pool_idle_timeout(limits.pool_idle_timeout)
.pool_max_idle_per_host(limits.max_concurrent_requests)
.build()
.map_err(|_| LlmError::Transport)?;
Ok(Self {
connection,
client,
slots: Arc::new(Semaphore::new(limits.max_concurrent_requests)),
limits,
next_request_id: AtomicU64::new(1),
})
}
pub async fn complete(
&self,
messages: &[CompletionMessage],
tools: &[ToolDefinition],
cancellation: &CancellationToken,
) -> Result<Completion, LlmError> {
if messages.is_empty()
|| messages.len() > MAX_CONVERSATION_MESSAGES
|| tools.len() > MAX_TOOLS
{
return Err(LlmError::PromptTooLarge);
}
for tool in tools {
tool.validate()?;
}
let body = request_body(messages, tools)?;
if body.len() > self.limits.max_prompt_bytes {
return Err(LlmError::PromptTooLarge);
}
let request_id = self.next_request_id.fetch_add(1, Ordering::Relaxed);
let correlation_id = BoundedText::new(
"llm.correlation_id",
format!("agent-request-{request_id:016x}"),
)?;
let started = Instant::now();
let operation = async {
let permit = tokio::select! {
() = cancellation.cancelled() => return Err(LlmError::Cancelled),
permit = Arc::clone(&self.slots).acquire_owned() => permit.map_err(|_| LlmError::Cancelled)?,
};
let result = self
.complete_with_retries(request_id, correlation_id, body, cancellation, started)
.await;
drop(permit);
result
};
tokio::time::timeout(self.limits.total_timeout, operation)
.await
.map_err(|_| LlmError::Timeout)?
}
async fn complete_with_retries(
&self,
request_id: u64,
correlation_id: BoundedText<MAX_IDENTIFIER_BYTES>,
body: Vec<u8>,
cancellation: &CancellationToken,
started: Instant,
) -> Result<Completion, LlmError> {
for attempt in 0..=self.limits.max_retries {
match self
.complete_once(
request_id,
correlation_id.clone(),
&body,
cancellation,
started,
attempt + 1,
)
.await
{
Ok(completion) => return Ok(completion),
Err(AttemptError { error, retry_after })
if attempt < self.limits.max_retries && transient(&error) =>
{
let delay = retry_after
.unwrap_or_else(|| deterministic_backoff(request_id, attempt))
.min(self.limits.max_retry_delay);
tokio::select! {
() = cancellation.cancelled() => return Err(LlmError::Cancelled),
() = tokio::time::sleep(delay) => {}
}
}
Err(AttemptError { error, .. }) => return Err(error),
}
}
Err(LlmError::Transport)
}
async fn complete_once(
&self,
request_id: u64,
correlation_id: BoundedText<MAX_IDENTIFIER_BYTES>,
body: &[u8],
cancellation: &CancellationToken,
started: Instant,
attempts: usize,
) -> Result<Completion, AttemptError> {
let response = tokio::select! {
() = cancellation.cancelled() => return Err(AttemptError::new(LlmError::Cancelled)),
response = self.client
.post(self.connection.endpoint_url.expose_url())
.bearer_auth(self.connection.api_key.expose_secret())
.header("content-type", "application/json")
.header("accept", "application/json")
.header("x-correlation-id", correlation_id.as_str())
.body(body.to_vec())
.send() => response.map_err(|error| AttemptError::new(classify_reqwest(&error)))?,
};
let status = response.status();
if status.is_redirection() {
return Err(AttemptError::new(LlmError::RedirectRefused));
}
if !status.is_success() {
let retry_after = response
.headers()
.get("retry-after")
.and_then(|value| value.to_str().ok())
.and_then(|value| value.parse::<u64>().ok())
.map(Duration::from_secs);
return Err(AttemptError {
error: LlmError::HttpStatus(status.as_u16()),
retry_after,
});
}
let bytes = read_bounded(response, self.limits.max_response_bytes, cancellation).await?;
parse_completion(
request_id,
correlation_id,
&bytes,
started.elapsed(),
attempts,
)
.map_err(AttemptError::new)
}
}
struct AttemptError {
error: LlmError,
retry_after: Option<Duration>,
}
impl AttemptError {
const fn new(error: LlmError) -> Self {
Self {
error,
retry_after: None,
}
}
}
async fn read_bounded(
mut response: reqwest::Response,
maximum: usize,
cancellation: &CancellationToken,
) -> Result<Vec<u8>, AttemptError> {
if response
.content_length()
.is_some_and(|length| usize::try_from(length).map_or(true, |length| length > maximum))
{
return Err(AttemptError::new(LlmError::ResponseTooLarge));
}
let mut body = Vec::with_capacity(
response
.content_length()
.and_then(|value| usize::try_from(value).ok())
.unwrap_or(0)
.min(maximum),
);
loop {
let chunk = tokio::select! {
() = cancellation.cancelled() => return Err(AttemptError::new(LlmError::Cancelled)),
chunk = response.chunk() => chunk.map_err(|error| AttemptError::new(classify_reqwest(&error)))?,
};
let Some(chunk) = chunk else { break };
if body.len().saturating_add(chunk.len()) > maximum {
return Err(AttemptError::new(LlmError::ResponseTooLarge));
}
body.extend_from_slice(&chunk);
}
Ok(body)
}
fn request_body(
messages: &[CompletionMessage],
tools: &[ToolDefinition],
) -> Result<Vec<u8>, LlmError> {
let messages = messages.iter().map(message_wire_value).collect::<Vec<_>>();
let tools = tools
.iter()
.map(|tool| {
json!({
"type":"function",
"function": {
"name": tool.name.as_str(),
"description": tool.description.as_str(),
"parameters": tool.schema.wire_value()
}
})
})
.collect::<Vec<_>>();
serde_json::to_vec(&json!({"messages":messages, "tools":tools}))
.map_err(|_| LlmError::MalformedJson)
}
fn message_wire_value(message: &CompletionMessage) -> Value {
let role = match message.role {
MessageRole::System => "system",
MessageRole::Avatar => "user",
MessageRole::Agent => "assistant",
MessageRole::Tool => "tool",
};
let content = message.content.as_slice().iter().map(|part| match part {
ContentPart::Text(text) => json!({"type":"text", "text":text.as_str()}),
ContentPart::Image { url, detail } => json!({
"type":"image_url",
"image_url":{"url":url.as_str(), "detail":match detail { ImageDetail::Auto=>"auto", ImageDetail::Low=>"low", ImageDetail::High=>"high"}}
}),
}).collect::<Vec<_>>();
let mut value = json!({"role":role,"content":content});
if let Some(call_id) = &message.tool_call_id {
value["tool_call_id"] = Value::String(call_id.as_str().to_owned());
}
if !message.proposed_calls.is_empty() {
value["tool_calls"] = Value::Array(message.proposed_calls.as_slice().iter().map(|call| json!({
"id":call.call_id.as_str(), "type":"function", "function":{"name":call.name.as_str(),"arguments":call.arguments_json.as_str()}
})).collect());
}
value
}
#[derive(Deserialize)]
struct WireResponse {
choices: Vec<WireChoice>,
usage: Option<WireUsage>,
}
#[derive(Deserialize)]
struct WireChoice {
message: WireMessage,
}
#[derive(Deserialize)]
struct WireMessage {
content: Option<String>,
#[serde(default)]
tool_calls: Vec<WireToolCall>,
}
#[derive(Deserialize)]
struct WireToolCall {
id: String,
#[serde(rename = "type")]
kind: Option<String>,
function: WireFunction,
}
#[derive(Deserialize)]
struct WireFunction {
name: String,
arguments: String,
}
#[derive(Deserialize)]
struct WireUsage {
#[serde(rename = "prompt_tokens")]
prompt: Option<u64>,
#[serde(rename = "completion_tokens")]
completion: Option<u64>,
#[serde(rename = "total_tokens")]
total: Option<u64>,
}
fn parse_completion(
request_id: u64,
correlation_id: BoundedText<MAX_IDENTIFIER_BYTES>,
body: &[u8],
latency: Duration,
attempts: usize,
) -> Result<Completion, LlmError> {
let wire: WireResponse = serde_json::from_slice(body).map_err(|_| LlmError::MalformedJson)?;
let choice = wire
.choices
.into_iter()
.next()
.ok_or(LlmError::UnsupportedResponse)?;
let mut proposed_calls = BoundedVec::new();
let mut ids = BTreeSet::new();
for call in choice.message.tool_calls {
if call.kind.as_deref().is_some_and(|kind| kind != "function") {
return Err(LlmError::UnsupportedResponse);
}
if !ids.insert(call.id.clone()) {
return Err(LlmError::DuplicateToolCallId);
}
proposed_calls
.try_push(
"completion.tool_calls",
ProposedToolCall::from_model_output(
call.id,
call.function.name,
call.function.arguments,
)?,
)
.map_err(LlmError::Boundary)?;
}
let mut content = BoundedVec::new();
if let Some(text) = choice.message.content {
content
.try_push(
"completion.content",
ContentPart::Text(BoundedText::new_allow_empty("completion.text", text)?),
)
.map_err(LlmError::Boundary)?;
}
if content.is_empty() && proposed_calls.is_empty() {
return Err(LlmError::UnsupportedResponse);
}
Ok(Completion {
request_id,
correlation_id,
message: CompletionMessage {
role: MessageRole::Agent,
content,
tool_call_id: None,
proposed_calls,
},
usage: wire.usage.map(|usage| Usage {
prompt_tokens: usage.prompt,
completion_tokens: usage.completion,
total_tokens: usage.total,
}),
latency,
attempts,
})
}
fn classify_reqwest(error: &reqwest::Error) -> LlmError {
if error.is_timeout() {
LlmError::Timeout
} else {
LlmError::Transport
}
}
fn transient(error: &LlmError) -> bool {
matches!(error, LlmError::Transport | LlmError::Timeout)
|| matches!(
error,
LlmError::HttpStatus(408 | 425 | 429 | 500 | 502 | 503 | 504)
)
}
fn deterministic_backoff(request_id: u64, attempt: usize) -> Duration {
let exponent = u32::try_from(attempt).unwrap_or(8).min(8);
let base = 100_u64.saturating_mul(1_u64 << exponent);
let jitter = request_id
.wrapping_mul(6_364_136_223_846_793_005)
.rotate_left(exponent)
% 97;
Duration::from_millis(base.saturating_add(jitter))
}
#[cfg(test)]
mod tests {
use super::*;
use crate::config::AgentConfig;
#[tokio::test]
async fn total_timeout_includes_waiting_for_a_concurrency_slot() {
let limits = LlmTransportLimits {
total_timeout: Duration::from_millis(20),
max_concurrent_requests: 1,
..LlmTransportLimits::default()
};
let connection = AgentConfig::offline("http://127.0.0.1:9/exact", "test-key")
.expect("test config")
.llm;
let client = LlmClient::new(connection, limits).expect("test client");
let permit = Arc::clone(&client.slots)
.acquire_owned()
.await
.expect("semaphore open");
assert_eq!(
client
.complete(
&[CompletionMessage::text(MessageRole::Avatar, "hello").expect("message")],
&[],
&CancellationToken::default(),
)
.await
.expect_err("slot wait must time out"),
LlmError::Timeout
);
drop(permit);
}
}

View File

@@ -0,0 +1,534 @@
//! Bounded multi-turn tool orchestration above the provider-neutral transport.
use crate::llm::{CompletionMessage, ContentPart, LlmClient, LlmError, ToolDefinition, Usage};
use crate::types::{
BoundedText, BoundedVec, MAX_BODY_BYTES, MAX_CONVERSATION_MESSAGES, MAX_MESSAGE_BYTES,
MAX_TOOL_CALLS, MessageRole, ProposedToolCall,
};
use libremetaverse_types::compat::{CancellationToken, CancellationTokenSource};
use serde_json::Value;
use std::error::Error;
use std::fmt;
use std::future::Future;
use std::pin::Pin;
use std::sync::Arc;
use std::sync::Mutex;
use std::time::Duration;
const MAX_REGISTERED_TOOLS: usize = 64;
const MAX_LOOP_TURNS: usize = 32;
const MAX_SESSION_TOOL_CALLS: usize = 256;
pub type ToolFuture<'a> = Pin<Box<dyn Future<Output = ToolExecution> + Send + 'a>>;
/// Downstream execution boundary. Issue #120 can implement policy evaluation
/// here; this loop guarantees its input already passed name/schema checks.
pub trait ToolExecutor: Send + Sync {
fn execute<'a>(
&'a self,
definition: &'a ToolDefinition,
call: &'a ProposedToolCall,
arguments: &'a Value,
cancellation: &'a CancellationToken,
) -> ToolFuture<'a>;
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub enum ToolExecution {
Completed(BoundedText<MAX_BODY_BYTES>),
Rejected(BoundedText<MAX_MESSAGE_BYTES>),
Failed(BoundedText<MAX_MESSAGE_BYTES>),
/// A mutating operation may have reached the world, so it must never be
/// repeated or converted into another model request automatically.
AmbiguousMutation,
}
/// Optional deterministic history summarizer. Failure is explicitly safe: the
/// loop inserts a bounded truncation marker instead.
pub trait HistorySummarizer: Send + Sync {
/// Returns a deterministic bounded summary, or `None` to request the safe
/// truncation marker fallback.
fn summarize(&self, omitted: &[CompletionMessage]) -> Option<CompletionMessage>;
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct ToolLoopLimits {
pub max_turns: usize,
pub max_tool_calls_per_turn: usize,
pub max_tool_calls_per_session: usize,
pub max_history_messages: usize,
pub max_history_bytes: usize,
pub wall_clock_timeout: Duration,
}
impl Default for ToolLoopLimits {
fn default() -> Self {
Self {
max_turns: 8,
max_tool_calls_per_turn: 8,
max_tool_calls_per_session: 32,
max_history_messages: 64,
max_history_bytes: 512 * 1024,
wall_clock_timeout: Duration::from_mins(2),
}
}
}
impl ToolLoopLimits {
/// # Errors
///
/// Rejects zero or above-hard-ceiling loop limits.
pub fn validate(&self) -> Result<(), ToolLoopError> {
if self.max_turns == 0
|| self.max_turns > MAX_LOOP_TURNS
|| self.max_tool_calls_per_turn == 0
|| self.max_tool_calls_per_turn > MAX_TOOL_CALLS
|| self.max_tool_calls_per_session == 0
|| self.max_tool_calls_per_session > MAX_SESSION_TOOL_CALLS
|| self.max_history_messages < 2
|| self.max_history_messages > MAX_CONVERSATION_MESSAGES
|| self.max_history_bytes < 1024
|| self.max_history_bytes > MAX_BODY_BYTES
|| self.wall_clock_timeout.is_zero()
|| self.wall_clock_timeout > Duration::from_mins(10)
{
return Err(ToolLoopError::UnsafeLimits);
}
Ok(())
}
}
/// Monotonic session epoch. Disconnect, expiry, or operator replacement calls
/// `supersede`; late inference and tool results then fail closed.
#[derive(Debug)]
struct SessionState {
generation: u64,
cancellation: CancellationTokenSource,
}
/// Session generation and its owned cancellation source are updated under one
/// lock so a caller can never obtain an uncancelled token for a stale epoch.
#[derive(Debug)]
pub struct SessionGeneration(Mutex<SessionState>);
impl Default for SessionGeneration {
fn default() -> Self {
Self(Mutex::new(SessionState {
generation: 0,
cancellation: CancellationTokenSource::new(),
}))
}
}
impl SessionGeneration {
#[must_use]
pub fn current(&self) -> u64 {
self.state().generation
}
pub fn supersede(&self) -> u64 {
let mut state = self.state();
state.cancellation.cancel();
state.generation = state.generation.saturating_add(1);
state.cancellation = CancellationTokenSource::new();
state.generation
}
#[must_use]
pub fn is_current(&self, generation: u64) -> bool {
self.current() == generation
}
fn cancellation_for(&self, generation: u64) -> Option<CancellationToken> {
let state = self.state();
(state.generation == generation).then(|| state.cancellation.token())
}
fn state(&self) -> std::sync::MutexGuard<'_, SessionState> {
self.0
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct ToolLoopOutcome {
pub final_message: CompletionMessage,
pub turns: usize,
pub tool_calls: usize,
pub usage: BoundedVec<Usage, MAX_LOOP_TURNS>,
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub enum ToolLoopError {
Transport(LlmError),
UnsafeLimits,
DuplicateToolName,
DuplicateToolCallId,
TooManyTools,
EmptyHistory,
HistoryLimit,
EndlessToolLoop,
ToolCallLimit,
AmbiguousMutation,
Cancelled,
WallClockTimeout,
Superseded,
Boundary(crate::types::BoundaryError),
}
impl fmt::Display for ToolLoopError {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Transport(error) => write!(formatter, "LLM tool loop transport failed: {error}"),
Self::UnsafeLimits => formatter.write_str("unsafe tool-loop limits"),
Self::DuplicateToolName => formatter.write_str("duplicate registered tool name"),
Self::DuplicateToolCallId => {
formatter.write_str("model repeated a tool-call ID in the session")
}
Self::TooManyTools => formatter.write_str("too many registered tools"),
Self::EmptyHistory => formatter.write_str("tool loop requires initial history"),
Self::HistoryLimit => formatter.write_str("tool-loop history exceeds its hard bound"),
Self::EndlessToolLoop => formatter.write_str("model exceeded the bounded tool turns"),
Self::ToolCallLimit => {
formatter.write_str("model exceeded the session tool-call bound")
}
Self::AmbiguousMutation => formatter.write_str("mutating tool outcome is ambiguous"),
Self::Cancelled => formatter.write_str("tool loop cancelled"),
Self::WallClockTimeout => formatter.write_str("tool loop wall-clock bound elapsed"),
Self::Superseded => formatter.write_str("tool loop session was superseded"),
Self::Boundary(error) => write!(formatter, "tool loop boundary failed: {error}"),
}
}
}
impl Error for ToolLoopError {}
impl From<LlmError> for ToolLoopError {
fn from(value: LlmError) -> Self {
if value == LlmError::Cancelled {
Self::Cancelled
} else {
Self::Transport(value)
}
}
}
impl From<crate::types::BoundaryError> for ToolLoopError {
fn from(value: crate::types::BoundaryError) -> Self {
Self::Boundary(value)
}
}
pub struct ToolLoop {
client: Arc<LlmClient>,
tools: BoundedVec<ToolDefinition, MAX_REGISTERED_TOOLS>,
limits: ToolLoopLimits,
summarizer: Option<Arc<dyn HistorySummarizer>>,
}
impl fmt::Debug for ToolLoop {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("ToolLoop")
.field("client", &self.client)
.field("tool_count", &self.tools.len())
.field("limits", &self.limits)
.field("has_summarizer", &self.summarizer.is_some())
.finish()
}
}
impl ToolLoop {
/// # Errors
///
/// Rejects invalid limits, schemas, counts, or duplicate tool names.
pub fn new(
client: Arc<LlmClient>,
tools: Vec<ToolDefinition>,
limits: ToolLoopLimits,
) -> Result<Self, ToolLoopError> {
limits.validate()?;
if tools.len() > MAX_REGISTERED_TOOLS {
return Err(ToolLoopError::TooManyTools);
}
let mut names = std::collections::BTreeSet::new();
for tool in &tools {
tool.validate()?;
if !names.insert(tool.name.as_str()) {
return Err(ToolLoopError::DuplicateToolName);
}
}
Ok(Self {
client,
tools: BoundedVec::try_from_vec("tool_loop.tools", tools)?,
limits,
summarizer: None,
})
}
#[must_use]
pub fn with_summarizer(mut self, summarizer: Arc<dyn HistorySummarizer>) -> Self {
self.summarizer = Some(summarizer);
self
}
/// # Errors
///
/// Returns typed transport, bound, cancellation, supersession, endless-loop,
/// or ambiguous-mutation failures.
pub async fn run(
&self,
mut history: Vec<CompletionMessage>,
generation_owner: &SessionGeneration,
generation: u64,
cancellation: &CancellationToken,
executor: &dyn ToolExecutor,
) -> Result<ToolLoopOutcome, ToolLoopError> {
if history.is_empty() {
return Err(ToolLoopError::EmptyHistory);
}
if history.len() > MAX_CONVERSATION_MESSAGES {
return Err(ToolLoopError::HistoryLimit);
}
let session_cancellation = generation_owner
.cancellation_for(generation)
.ok_or(ToolLoopError::Superseded)?;
let linked_cancellation =
CancellationTokenSource::new_linked(&[cancellation.clone(), session_cancellation]);
let linked_token = linked_cancellation.token();
let future = self.run_inner(
&mut history,
generation_owner,
generation,
&linked_token,
executor,
);
let result = tokio::time::timeout(self.limits.wall_clock_timeout, future)
.await
.map_err(|_| ToolLoopError::WallClockTimeout)?;
drop(linked_cancellation);
result
}
async fn run_inner(
&self,
history: &mut Vec<CompletionMessage>,
generation_owner: &SessionGeneration,
generation: u64,
cancellation: &CancellationToken,
executor: &dyn ToolExecutor,
) -> Result<ToolLoopOutcome, ToolLoopError> {
let mut total_calls = 0;
let mut usage = BoundedVec::new();
let mut seen_call_ids = std::collections::BTreeSet::new();
for turn in 1..=self.limits.max_turns {
ensure_live(generation_owner, generation, cancellation)?;
compact_history(history, &self.limits, self.summarizer.as_deref())?;
let completion_result = self
.client
.complete(history, self.tools.as_slice(), cancellation)
.await;
ensure_live(generation_owner, generation, cancellation)?;
let completion = completion_result?;
if let Some(item) = completion.usage.clone() {
usage.try_push("tool_loop.usage", item)?;
}
let calls = completion.message.proposed_calls.clone().into_inner();
if calls.len() > self.limits.max_tool_calls_per_turn {
return Err(ToolLoopError::ToolCallLimit);
}
if calls
.iter()
.any(|call| !seen_call_ids.insert(call.call_id.as_str().to_owned()))
{
return Err(ToolLoopError::DuplicateToolCallId);
}
history.push(completion.message.clone());
if calls.is_empty() {
return Ok(ToolLoopOutcome {
final_message: completion.message,
turns: turn,
tool_calls: total_calls,
usage,
});
}
total_calls = total_calls.saturating_add(calls.len());
if total_calls > self.limits.max_tool_calls_per_session {
return Err(ToolLoopError::ToolCallLimit);
}
for call in calls {
ensure_live(generation_owner, generation, cancellation)?;
let observation = self
.evaluate_call(&call, generation_owner, generation, cancellation, executor)
.await?;
history.push(observation);
}
}
Err(ToolLoopError::EndlessToolLoop)
}
async fn evaluate_call(
&self,
call: &ProposedToolCall,
generation_owner: &SessionGeneration,
generation: u64,
cancellation: &CancellationToken,
executor: &dyn ToolExecutor,
) -> Result<CompletionMessage, ToolLoopError> {
let Some(definition) = self
.tools
.as_slice()
.iter()
.find(|definition| definition.name.as_str() == call.name.as_str())
else {
return tool_observation(call, "unknown tool; no execution occurred");
};
let Ok(arguments) = serde_json::from_str::<Value>(call.arguments_json.as_str()) else {
return tool_observation(call, "arguments are malformed JSON; no execution occurred");
};
if definition.schema.validate_value(&arguments).is_err() {
return tool_observation(call, "arguments rejected by registered schema");
}
ensure_live(generation_owner, generation, cancellation)?;
let execution = tokio::select! {
() = cancellation.cancelled() => {
ensure_live(generation_owner, generation, cancellation)?;
return Err(ToolLoopError::Cancelled);
}
execution = executor.execute(definition, call, &arguments, cancellation) => execution,
};
ensure_live(generation_owner, generation, cancellation)?;
match execution {
ToolExecution::Completed(result) => tool_observation(call, result.as_str()),
ToolExecution::Rejected(reason) => {
tool_observation(call, &format!("tool rejected: {}", reason.as_str()))
}
ToolExecution::Failed(reason) => {
tool_observation(call, &format!("tool failed safely: {}", reason.as_str()))
}
ToolExecution::AmbiguousMutation => Err(ToolLoopError::AmbiguousMutation),
}
}
}
fn ensure_live(
owner: &SessionGeneration,
generation: u64,
cancellation: &CancellationToken,
) -> Result<(), ToolLoopError> {
if !owner.is_current(generation) {
Err(ToolLoopError::Superseded)
} else if cancellation.is_cancellation_requested() {
Err(ToolLoopError::Cancelled)
} else {
Ok(())
}
}
fn tool_observation(
call: &ProposedToolCall,
body: &str,
) -> Result<CompletionMessage, ToolLoopError> {
const TRUNCATION_MARKER: &str = "\n[tool observation truncated]";
let body = if body.len() <= MAX_MESSAGE_BYTES {
body.to_owned()
} else {
let mut end = MAX_MESSAGE_BYTES - TRUNCATION_MARKER.len();
while !body.is_char_boundary(end) {
end -= 1;
}
format!("{}{TRUNCATION_MARKER}", &body[..end])
};
let mut message = CompletionMessage::text(MessageRole::Tool, body)?;
message.tool_call_id = Some(call.call_id.clone());
Ok(message)
}
fn compact_history(
history: &mut Vec<CompletionMessage>,
limits: &ToolLoopLimits,
summarizer: Option<&dyn HistorySummarizer>,
) -> Result<(), ToolLoopError> {
let mut omitted = Vec::new();
while history.len() > 1
&& (history.len() >= limits.max_history_messages
|| history_bytes(history) > limits.max_history_bytes)
{
omitted.push(history.remove(0));
}
if omitted.is_empty() {
return (history_bytes(history) <= limits.max_history_bytes)
.then_some(())
.ok_or(ToolLoopError::HistoryLimit);
}
if history_bytes(history) > limits.max_history_bytes {
return Err(ToolLoopError::HistoryLimit);
}
let fallback = CompletionMessage::text(
MessageRole::System,
format!(
"[history safely truncated: {} earlier messages omitted]",
omitted.len()
),
)?;
let summary = summarizer
.and_then(|summarizer| summarizer.summarize(&omitted))
.filter(valid_summary)
.filter(|summary| {
history_bytes(history).saturating_add(message_bytes(summary))
<= limits.max_history_bytes
})
.or_else(|| {
(history_bytes(history).saturating_add(message_bytes(&fallback))
<= limits.max_history_bytes)
.then_some(fallback)
});
if let Some(summary) = summary {
history.insert(0, summary);
}
while history.len() > limits.max_history_messages {
history.remove(1);
}
Ok(())
}
fn history_bytes(history: &[CompletionMessage]) -> usize {
history
.iter()
.map(message_bytes)
.fold(0, usize::saturating_add)
}
fn message_bytes(message: &CompletionMessage) -> usize {
let content = message
.content
.as_slice()
.iter()
.map(|part| match part {
ContentPart::Text(text) => text.len(),
ContentPart::Image { url, .. } => url.len(),
})
.fold(0, usize::saturating_add);
let call_id = message
.tool_call_id
.as_ref()
.map_or(0, |call_id| call_id.len());
message
.proposed_calls
.as_slice()
.iter()
.map(|call| {
call.call_id
.len()
.saturating_add(call.name.len())
.saturating_add(call.arguments_json.len())
})
.fold(content.saturating_add(call_id), usize::saturating_add)
}
fn valid_summary(message: &CompletionMessage) -> bool {
message.role == MessageRole::System
&& message.tool_call_id.is_none()
&& message.proposed_calls.is_empty()
}

View File

@@ -330,6 +330,21 @@ impl ProposedToolCall {
arguments_json,
})
}
/// Admits bounded model output before schema/JSON validation in the tool
/// loop. This is crate-private so application-created calls still use
/// [`Self::new`].
pub(crate) fn from_model_output(
call_id: impl Into<String>,
name: impl Into<String>,
arguments_json: impl Into<String>,
) -> Result<Self, BoundaryError> {
Ok(Self {
call_id: BoundedText::new("tool_call.call_id", call_id)?,
name: BoundedText::new("tool_call.name", name)?,
arguments_json: BoundedText::new("tool_call.arguments_json", arguments_json)?,
})
}
}
#[derive(Clone, Debug, Eq, PartialEq)]

View File

@@ -2,9 +2,10 @@ use std::collections::BTreeSet;
use std::fs;
use std::path::{Path, PathBuf};
const ALLOWED_DEPENDENCIES: [&str; 6] = [
const ALLOWED_DEPENDENCIES: [&str; 7] = [
"libremetaverse",
"libremetaverse-types",
"reqwest",
"serde",
"serde_json",
"tokio",

View File

@@ -0,0 +1,950 @@
use libremetaverse_types::compat::{CancellationToken, CancellationTokenSource};
use metacrate_grid_agent::{
AgentConfig, BoundedText, BoundedVec, CompletionMessage, ContentPart, HistorySummarizer,
ImageDetail, LlmClient, LlmError, LlmTransportLimits, MessageRole, SessionGeneration,
ToolDefinition, ToolExecution, ToolExecutor, ToolFuture, ToolLoop, ToolLoopError,
ToolLoopLimits, ToolSchema,
};
use serde_json::{Value, json};
use std::collections::{BTreeMap, BTreeSet};
use std::fmt::Write as _;
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::time::Duration;
use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _};
use tokio::net::TcpListener;
use tokio::sync::{Notify, mpsc};
const MAX_REQUEST_BYTES: usize = 2 * 1024 * 1024;
#[derive(Clone)]
struct ResponsePlan {
status: u16,
headers: Vec<(String, String)>,
chunks: Vec<Vec<u8>>,
delay: Duration,
}
impl ResponsePlan {
#[allow(clippy::needless_pass_by_value)] // Keeps nested JSON fixtures readable at call sites.
fn json(value: Value) -> Self {
Self {
status: 200,
headers: vec![("content-type".into(), "application/json".into())],
chunks: vec![serde_json::to_vec(&value).expect("fixture JSON")],
delay: Duration::ZERO,
}
}
fn status(status: u16) -> Self {
Self {
status,
headers: Vec::new(),
chunks: vec![b"<html>bounded error</html>".to_vec()],
delay: Duration::ZERO,
}
}
fn delayed(mut self, delay: Duration) -> Self {
self.delay = delay;
self
}
}
#[derive(Debug)]
struct CapturedRequest {
head: String,
body: Value,
}
struct FakeServer {
url: String,
requests: mpsc::Receiver<CapturedRequest>,
task: tokio::task::JoinHandle<()>,
max_active: Arc<AtomicUsize>,
}
impl Drop for FakeServer {
fn drop(&mut self) {
self.task.abort();
}
}
async fn fake_server(plans: Vec<ResponsePlan>) -> FakeServer {
let listener = TcpListener::bind((std::net::Ipv4Addr::LOCALHOST, 0))
.await
.expect("bind fake LLM");
let address = listener.local_addr().expect("fake address");
let (request_sender, requests) = mpsc::channel(plans.len().max(1));
let active = Arc::new(AtomicUsize::new(0));
let max_active = Arc::new(AtomicUsize::new(0));
let task_active = Arc::clone(&active);
let task_max = Arc::clone(&max_active);
let task = tokio::spawn(async move {
let mut handlers = Vec::with_capacity(plans.len());
for plan in plans {
let Ok((stream, _)) = listener.accept().await else {
break;
};
let sender = request_sender.clone();
let active = Arc::clone(&task_active);
let max_active = Arc::clone(&task_max);
handlers.push(tokio::spawn(async move {
handle_connection(stream, plan, sender, active, max_active).await;
}));
}
for handler in handlers {
let _ = handler.await;
}
});
FakeServer {
url: format!("http://{address}/exact/chat?route=operator"),
requests,
task,
max_active,
}
}
async fn handle_connection(
mut stream: tokio::net::TcpStream,
plan: ResponsePlan,
sender: mpsc::Sender<CapturedRequest>,
active: Arc<AtomicUsize>,
max_active: Arc<AtomicUsize>,
) {
let now = active.fetch_add(1, Ordering::AcqRel) + 1;
max_active.fetch_max(now, Ordering::AcqRel);
let request = read_request(&mut stream).await.expect("bounded request");
sender.send(request).await.expect("capture owner");
tokio::time::sleep(plan.delay).await;
let body_len = plan.chunks.iter().map(Vec::len).sum::<usize>();
let reason = match plan.status {
200 => "OK",
302 => "Found",
400 => "Bad Request",
429 => "Too Many Requests",
500 => "Internal Server Error",
503 => "Service Unavailable",
_ => "Response",
};
let mut head = format!(
"HTTP/1.1 {} {}\r\nContent-Length: {}\r\nConnection: close\r\n",
plan.status, reason, body_len
);
for (name, value) in plan.headers {
write!(head, "{name}: {value}\r\n").expect("write response header");
}
head.push_str("\r\n");
stream
.write_all(head.as_bytes())
.await
.expect("response head");
for chunk in plan.chunks {
stream.write_all(&chunk).await.expect("response chunk");
tokio::task::yield_now().await;
}
let _ = stream.shutdown().await;
active.fetch_sub(1, Ordering::AcqRel);
}
async fn read_request(stream: &mut tokio::net::TcpStream) -> Result<CapturedRequest, ()> {
let mut bytes = Vec::with_capacity(4096);
let header_end = loop {
if bytes.len() >= MAX_REQUEST_BYTES {
return Err(());
}
let mut chunk = [0_u8; 1024];
let read = stream.read(&mut chunk).await.map_err(|_| ())?;
if read == 0 {
return Err(());
}
bytes.extend_from_slice(&chunk[..read]);
if let Some(offset) = bytes.windows(4).position(|window| window == b"\r\n\r\n") {
break offset + 4;
}
};
let head = String::from_utf8(bytes[..header_end].to_vec()).map_err(|_| ())?;
let content_length = head
.lines()
.find_map(|line| {
line.split_once(':').and_then(|(name, value)| {
name.eq_ignore_ascii_case("content-length")
.then(|| value.trim().parse::<usize>().ok())
.flatten()
})
})
.ok_or(())?;
if content_length > MAX_REQUEST_BYTES {
return Err(());
}
while bytes.len() - header_end < content_length {
let mut chunk = [0_u8; 1024];
let read = stream.read(&mut chunk).await.map_err(|_| ())?;
if read == 0 {
return Err(());
}
bytes.extend_from_slice(&chunk[..read]);
}
let body =
serde_json::from_slice(&bytes[header_end..header_end + content_length]).map_err(|_| ())?;
Ok(CapturedRequest { head, body })
}
fn response_text(text: &str) -> Value {
json!({
"id":"ignored-provider-field",
"choices":[{"message":{"role":"assistant","content":text},"unknown":"kept-compatible"}],
"usage":{"prompt_tokens":3,"completion_tokens":2,"total_tokens":5},
"unknown_root":{"harmless":true}
})
}
#[allow(clippy::needless_pass_by_value)] // Keeps nested JSON fixtures readable at call sites.
fn response_calls(calls: Value) -> Value {
json!({"choices":[{"message":{"role":"assistant","content":null,"tool_calls":calls}}]})
}
fn call(id: &str, name: &str, arguments: &str) -> Value {
json!({"id":id,"type":"function","function":{"name":name,"arguments":arguments}})
}
fn limits() -> LlmTransportLimits {
LlmTransportLimits {
connect_timeout: Duration::from_secs(1),
request_timeout: Duration::from_secs(2),
read_idle_timeout: Duration::from_secs(1),
pool_idle_timeout: Duration::from_secs(1),
total_timeout: Duration::from_secs(4),
max_prompt_bytes: 64 * 1024,
max_response_bytes: 64 * 1024,
max_concurrent_requests: 2,
max_retries: 0,
max_retry_delay: Duration::from_millis(20),
}
}
fn client(url: &str, limits: LlmTransportLimits) -> Arc<LlmClient> {
let connection = AgentConfig::offline(url, "super-secret-api-key")
.expect("valid fake endpoint")
.llm;
Arc::new(LlmClient::new(connection, limits).expect("valid client"))
}
fn message() -> CompletionMessage {
CompletionMessage::text(MessageRole::Avatar, "hello").expect("bounded message")
}
fn system_message() -> CompletionMessage {
CompletionMessage::text(MessageRole::System, "use registered tools only")
.expect("bounded system message")
}
fn image_message() -> CompletionMessage {
CompletionMessage {
role: MessageRole::Avatar,
content: BoundedVec::try_from_vec(
"image.content",
vec![ContentPart::Image {
url: BoundedText::new("image.url", "https://assets.invalid/object.png")
.expect("image URL"),
detail: ImageDetail::Low,
}],
)
.expect("image content"),
tool_call_id: None,
proposed_calls: BoundedVec::new(),
}
}
fn tool(name: &str, mutating: bool) -> ToolDefinition {
let mut properties = BTreeMap::new();
properties.insert("target".into(), ToolSchema::String);
ToolDefinition {
name: BoundedText::new("tool.name", name).expect("tool name"),
description: BoundedText::new("tool.description", "test tool").expect("description"),
schema: ToolSchema::Object {
properties,
required: BTreeSet::from(["target".into()]),
additional_properties: false,
},
mutating,
}
}
#[tokio::test]
async fn exact_url_auth_schema_fragmentation_and_metadata_are_verified() {
let body = serde_json::to_vec(&response_text("done")).expect("response");
let midpoint = body.len() / 2;
let mut server = fake_server(vec![ResponsePlan {
status: 200,
headers: vec![("content-type".into(), "application/json".into())],
chunks: vec![body[..midpoint].to_vec(), body[midpoint..].to_vec()],
delay: Duration::ZERO,
}])
.await;
let client = client(&server.url, limits());
let source = CancellationTokenSource::new();
let messages = [system_message(), message(), image_message()];
let completion = client
.complete(&messages, &[tool("look", false)], &source.token())
.await
.expect("completion");
assert_eq!(completion.request_id, 1);
assert_eq!(completion.attempts, 1);
assert_eq!(completion.usage.expect("usage").total_tokens, Some(5));
let request = server.requests.recv().await.expect("captured request");
assert!(
request
.head
.starts_with("POST /exact/chat?route=operator HTTP/1.1")
);
assert!(
request
.head
.to_ascii_lowercase()
.contains("authorization: bearer super-secret-api-key")
);
assert!(
request
.head
.to_ascii_lowercase()
.contains("x-correlation-id: agent-request-0000000000000001")
);
assert!(request.body.get("model").is_none());
assert!(request.body.get("provider").is_none());
assert_eq!(request.body["tools"][0]["type"], "function");
assert_eq!(request.body["messages"][0]["role"], "system");
assert_eq!(
request.body["messages"][2]["content"][0]["type"],
"image_url"
);
assert_eq!(
request.body["messages"][2]["content"][0]["image_url"]["detail"],
"low"
);
let diagnostic = format!("{client:?}");
assert!(!diagnostic.contains("super-secret-api-key"));
assert!(!diagnostic.contains("route=operator"));
}
#[tokio::test]
async fn prompt_bytes_and_schema_complexity_fail_before_network_access() {
let mut tiny = limits();
tiny.max_prompt_bytes = 32;
let client = client("http://127.0.0.1:9/exact", tiny);
assert_eq!(
client
.complete(&[message()], &[], &CancellationToken::default())
.await
.expect_err("prompt cap"),
LlmError::PromptTooLarge
);
let mut schema = ToolSchema::String;
for _ in 0..18 {
schema = ToolSchema::Array {
items: Box::new(schema),
max_items: 1,
};
}
let definition = ToolDefinition {
name: BoundedText::new("tool.name", "deep").expect("name"),
description: BoundedText::new("tool.description", "too deep").expect("description"),
schema,
mutating: false,
};
assert_eq!(definition.validate(), Err(LlmError::InvalidToolSchema));
}
#[tokio::test]
async fn malformed_html_oversize_and_duplicate_calls_are_typed() {
let cases = [
(
ResponsePlan::json(json!({"not_choices":[]})),
LlmError::MalformedJson,
),
(
ResponsePlan {
status: 200,
headers: vec![("content-type".into(), "text/html".into())],
chunks: vec![b"<html>not JSON</html>".to_vec()],
delay: Duration::ZERO,
},
LlmError::MalformedJson,
),
(ResponsePlan::status(500), LlmError::HttpStatus(500)),
(
ResponsePlan::json(response_calls(json!([{
"id":"unsupported-1",
"type":"not-a-function",
"function":{"name":"look","arguments":"{}"}
}]))),
LlmError::UnsupportedResponse,
),
(
ResponsePlan::json(response_calls(json!([
call("same", "look", "{}"),
call("same", "look", "{}")
]))),
LlmError::DuplicateToolCallId,
),
];
for (plan, expected) in cases {
let server = fake_server(vec![plan]).await;
let error = client(&server.url, limits())
.complete(&[message()], &[], &CancellationToken::default())
.await
.expect_err("typed failure");
assert_eq!(error, expected);
}
let mut small = limits();
small.max_response_bytes = 32;
let server = fake_server(vec![ResponsePlan::json(response_text(
"this response is deliberately larger than thirty-two bytes",
))])
.await;
assert_eq!(
client(&server.url, small)
.complete(&[message()], &[], &CancellationToken::default())
.await
.expect_err("oversize rejected"),
LlmError::ResponseTooLarge
);
}
#[tokio::test]
async fn redirects_never_forward_bearer_credentials() {
let mut target = fake_server(vec![ResponsePlan::json(response_text("leaked"))]).await;
let redirect = ResponsePlan {
status: 302,
headers: vec![("location".into(), target.url.clone())],
chunks: Vec::new(),
delay: Duration::ZERO,
};
let mut source = fake_server(vec![redirect]).await;
let error = client(&source.url, limits())
.complete(&[message()], &[], &CancellationToken::default())
.await
.expect_err("redirect refused");
assert_eq!(error, LlmError::RedirectRefused);
source.requests.recv().await.expect("source request");
assert!(
tokio::time::timeout(Duration::from_millis(100), target.requests.recv())
.await
.is_err(),
"redirect target must receive no request"
);
}
#[tokio::test]
async fn retry_classification_retry_after_timeout_and_cancellation_are_bounded() {
let retry = ResponsePlan {
status: 503,
headers: vec![("retry-after".into(), "0".into())],
chunks: Vec::new(),
delay: Duration::ZERO,
};
let mut server = fake_server(vec![retry, ResponsePlan::json(response_text("recovered"))]).await;
let mut retry_limits = limits();
retry_limits.max_retries = 1;
let completion = client(&server.url, retry_limits)
.complete(&[message()], &[], &CancellationToken::default())
.await
.expect("transient retry");
assert_eq!(completion.attempts, 2);
server.requests.recv().await.expect("first attempt");
server.requests.recv().await.expect("second attempt");
let server = fake_server(vec![
ResponsePlan::status(400),
ResponsePlan::json(response_text("must not retry")),
])
.await;
let mut retry_limits = limits();
retry_limits.max_retries = 1;
assert_eq!(
client(&server.url, retry_limits)
.complete(&[message()], &[], &CancellationToken::default())
.await
.expect_err("400 not retried"),
LlmError::HttpStatus(400)
);
let mut slow_limits = limits();
slow_limits.request_timeout = Duration::from_millis(50);
slow_limits.total_timeout = Duration::from_millis(100);
let server = fake_server(vec![
ResponsePlan::json(response_text("late")).delayed(Duration::from_secs(1)),
])
.await;
assert_eq!(
client(&server.url, slow_limits)
.complete(&[message()], &[], &CancellationToken::default())
.await
.expect_err("timeout"),
LlmError::Timeout
);
let server = fake_server(vec![
ResponsePlan::json(response_text("late")).delayed(Duration::from_secs(1)),
])
.await;
let client = client(&server.url, limits());
let source = CancellationTokenSource::new();
let token = source.token();
let messages = [message()];
let request = client.complete(&messages, &[], &token);
tokio::pin!(request);
tokio::select! {
result = &mut request => panic!("request unexpectedly completed: {result:?}"),
() = tokio::time::sleep(Duration::from_millis(20)) => source.cancel(),
}
assert_eq!(request.await.expect_err("cancelled"), LlmError::Cancelled);
}
#[tokio::test]
async fn concurrent_slow_sessions_respect_the_shared_semaphore() {
let plans = (0..6)
.map(|_| ResponsePlan::json(response_text("done")).delayed(Duration::from_millis(80)))
.collect();
let server = fake_server(plans).await;
let client = client(&server.url, limits());
let mut tasks = Vec::with_capacity(6);
for _ in 0..6 {
let client = Arc::clone(&client);
tasks.push(tokio::spawn(async move {
client
.complete(&[message()], &[], &CancellationToken::default())
.await
}));
}
for task in tasks {
task.await
.expect("session task")
.expect("session completion");
}
assert_eq!(server.max_active.load(Ordering::Acquire), 2);
}
struct RecordingExecutor {
calls: AtomicUsize,
result: ToolExecution,
}
impl ToolExecutor for RecordingExecutor {
fn execute<'a>(
&'a self,
_definition: &'a ToolDefinition,
_call: &'a metacrate_grid_agent::ProposedToolCall,
_arguments: &'a Value,
_cancellation: &'a CancellationToken,
) -> ToolFuture<'a> {
self.calls.fetch_add(1, Ordering::AcqRel);
let result = self.result.clone();
Box::pin(async move { result })
}
}
fn loop_limits() -> ToolLoopLimits {
ToolLoopLimits {
max_turns: 3,
max_tool_calls_per_turn: 4,
max_tool_calls_per_session: 4,
max_history_messages: 8,
max_history_bytes: 4096,
wall_clock_timeout: Duration::from_secs(2),
}
}
#[tokio::test]
async fn valid_tool_round_trip_and_model_authored_summary_complete() {
let mut server = fake_server(vec![
ResponsePlan::json(response_calls(json!([call(
"call-1",
"look",
r#"{"target":"tree"}"#,
)]))),
ResponsePlan::json(response_text("I looked at the tree.")),
])
.await;
let executor = RecordingExecutor {
calls: AtomicUsize::new(0),
result: ToolExecution::Completed(
BoundedText::new("result", "tree is nearby").expect("result"),
),
};
let loop_ = ToolLoop::new(
client(&server.url, limits()),
vec![tool("look", false)],
loop_limits(),
)
.expect("loop");
let generation = SessionGeneration::default();
let outcome = loop_
.run(
vec![message()],
&generation,
generation.current(),
&CancellationToken::default(),
&executor,
)
.await
.expect("tool loop");
assert_eq!(outcome.turns, 2);
assert_eq!(outcome.tool_calls, 1);
assert_eq!(executor.calls.load(Ordering::Acquire), 1);
server.requests.recv().await.expect("tool request");
let round_trip = server.requests.recv().await.expect("observation request");
let serialized = round_trip.body.to_string();
assert!(serialized.contains("tree is nearby"));
assert!(serialized.contains("call-1"));
}
#[tokio::test]
async fn unknown_and_malformed_calls_become_observations_without_execution() {
let mut server = fake_server(vec![
ResponsePlan::json(response_calls(json!([
call("unknown-1", "missing", "{}"),
call("bad-1", "look", "not-json"),
call("schema-1", "look", r#"{"wrong":true}"#)
]))),
ResponsePlan::json(response_text("handled safely")),
])
.await;
let executor = RecordingExecutor {
calls: AtomicUsize::new(0),
result: ToolExecution::Completed(BoundedText::new("result", "unused").expect("result")),
};
let loop_ = ToolLoop::new(
client(&server.url, limits()),
vec![tool("look", false)],
loop_limits(),
)
.expect("loop");
let generation = SessionGeneration::default();
loop_
.run(
vec![message()],
&generation,
0,
&CancellationToken::default(),
&executor,
)
.await
.expect("safe observations");
assert_eq!(executor.calls.load(Ordering::Acquire), 0);
server.requests.recv().await.expect("first request");
let observations = server
.requests
.recv()
.await
.expect("second request")
.body
.to_string();
assert!(observations.contains("unknown tool"));
assert!(observations.contains("malformed JSON"));
assert!(observations.contains("registered schema"));
}
#[tokio::test]
async fn endless_loops_and_ambiguous_mutations_fail_without_reexecution() {
let server = fake_server(
(1..=3)
.map(|number| {
ResponsePlan::json(response_calls(json!([call(
&format!("call-{number}"),
"look",
r#"{"target":"tree"}"#,
)])))
})
.collect(),
)
.await;
let executor = RecordingExecutor {
calls: AtomicUsize::new(0),
result: ToolExecution::Completed(BoundedText::new("result", "again").expect("result")),
};
let loop_ = ToolLoop::new(
client(&server.url, limits()),
vec![tool("look", false)],
loop_limits(),
)
.expect("loop");
assert_eq!(
loop_
.run(
vec![message()],
&SessionGeneration::default(),
0,
&CancellationToken::default(),
&executor,
)
.await
.expect_err("endless loop"),
ToolLoopError::EndlessToolLoop
);
let server = fake_server(vec![ResponsePlan::json(response_calls(json!([call(
"mutate-1",
"rez",
r#"{"target":"cube"}"#,
)])))])
.await;
let executor = RecordingExecutor {
calls: AtomicUsize::new(0),
result: ToolExecution::AmbiguousMutation,
};
let loop_ = ToolLoop::new(
client(&server.url, limits()),
vec![tool("rez", true)],
loop_limits(),
)
.expect("loop");
assert_eq!(
loop_
.run(
vec![message()],
&SessionGeneration::default(),
0,
&CancellationToken::default(),
&executor,
)
.await
.expect_err("ambiguous mutation"),
ToolLoopError::AmbiguousMutation
);
assert_eq!(executor.calls.load(Ordering::Acquire), 1);
}
#[tokio::test]
async fn repeated_call_id_across_turns_is_rejected_before_reexecution() {
let duplicate = ResponsePlan::json(response_calls(json!([call(
"same-call",
"look",
r#"{"target":"tree"}"#,
)])));
let server = fake_server(vec![duplicate.clone(), duplicate]).await;
let executor = RecordingExecutor {
calls: AtomicUsize::new(0),
result: ToolExecution::Completed(BoundedText::new("result", "first").expect("result")),
};
let loop_ = ToolLoop::new(
client(&server.url, limits()),
vec![tool("look", false)],
loop_limits(),
)
.expect("loop");
assert_eq!(
loop_
.run(
vec![message()],
&SessionGeneration::default(),
0,
&CancellationToken::default(),
&executor,
)
.await
.expect_err("duplicate call ID"),
ToolLoopError::DuplicateToolCallId
);
assert_eq!(executor.calls.load(Ordering::Acquire), 1);
}
#[tokio::test]
async fn session_call_and_wall_clock_budgets_are_enforced() {
let server = fake_server(vec![ResponsePlan::json(response_calls(json!([
call("one", "look", r#"{"target":"tree"}"#),
call("two", "look", r#"{"target":"rock"}"#)
])))])
.await;
let executor = RecordingExecutor {
calls: AtomicUsize::new(0),
result: ToolExecution::Completed(BoundedText::new("result", "unused").expect("result")),
};
let mut one_call = loop_limits();
one_call.max_tool_calls_per_turn = 1;
one_call.max_tool_calls_per_session = 1;
let loop_ = ToolLoop::new(
client(&server.url, limits()),
vec![tool("look", false)],
one_call,
)
.expect("loop");
assert_eq!(
loop_
.run(
vec![message()],
&SessionGeneration::default(),
0,
&CancellationToken::default(),
&executor,
)
.await
.expect_err("call budget"),
ToolLoopError::ToolCallLimit
);
assert_eq!(executor.calls.load(Ordering::Acquire), 0);
let server = fake_server(vec![
ResponsePlan::json(response_text("too late")).delayed(Duration::from_secs(1)),
])
.await;
let mut short_loop = loop_limits();
short_loop.wall_clock_timeout = Duration::from_millis(30);
let loop_ = ToolLoop::new(client(&server.url, limits()), vec![], short_loop).expect("loop");
assert_eq!(
loop_
.run(
vec![message()],
&SessionGeneration::default(),
0,
&CancellationToken::default(),
&executor,
)
.await
.expect_err("wall-clock budget"),
ToolLoopError::WallClockTimeout
);
}
#[tokio::test]
async fn superseded_session_discards_late_model_result_before_execution() {
let mut server = fake_server(vec![
ResponsePlan::json(response_calls(json!([call(
"late-1",
"look",
r#"{"target":"tree"}"#,
)])))
.delayed(Duration::from_millis(100)),
])
.await;
let executor = Arc::new(RecordingExecutor {
calls: AtomicUsize::new(0),
result: ToolExecution::Completed(BoundedText::new("result", "late").expect("result")),
});
let loop_ = ToolLoop::new(
client(&server.url, limits()),
vec![tool("look", false)],
loop_limits(),
)
.expect("loop");
let generation = Arc::new(SessionGeneration::default());
let token = CancellationToken::default();
let run = loop_.run(vec![message()], &generation, 0, &token, executor.as_ref());
tokio::pin!(run);
tokio::select! {
request = server.requests.recv() => {
request.expect("in-flight request");
}
result = &mut run => panic!("session completed before supersession: {result:?}"),
}
generation.supersede();
assert_eq!(
run.await.expect_err("superseded"),
ToolLoopError::Superseded
);
assert_eq!(executor.calls.load(Ordering::Acquire), 0);
}
struct BlockingExecutor {
started: Notify,
completed: AtomicUsize,
}
impl ToolExecutor for BlockingExecutor {
fn execute<'a>(
&'a self,
_definition: &'a ToolDefinition,
_call: &'a metacrate_grid_agent::ProposedToolCall,
_arguments: &'a Value,
_cancellation: &'a CancellationToken,
) -> ToolFuture<'a> {
Box::pin(async move {
self.started.notify_one();
std::future::pending::<()>().await;
self.completed.fetch_add(1, Ordering::AcqRel);
ToolExecution::AmbiguousMutation
})
}
}
#[tokio::test]
async fn superseding_session_cancels_an_in_flight_tool_executor() {
let server = fake_server(vec![ResponsePlan::json(response_calls(json!([call(
"in-flight-1",
"look",
r#"{"target":"tree"}"#,
)])))])
.await;
let executor = Arc::new(BlockingExecutor {
started: Notify::new(),
completed: AtomicUsize::new(0),
});
let loop_ = ToolLoop::new(
client(&server.url, limits()),
vec![tool("look", false)],
loop_limits(),
)
.expect("loop");
let generation = SessionGeneration::default();
let token = CancellationToken::default();
let run = loop_.run(vec![message()], &generation, 0, &token, executor.as_ref());
tokio::pin!(run);
tokio::select! {
() = executor.started.notified() => {}
result = &mut run => panic!("tool loop completed before supersession: {result:?}"),
}
generation.supersede();
assert_eq!(
run.await.expect_err("superseded executor"),
ToolLoopError::Superseded
);
assert_eq!(executor.completed.load(Ordering::Acquire), 0);
}
struct FailingSummarizer;
impl HistorySummarizer for FailingSummarizer {
fn summarize(&self, _omitted: &[CompletionMessage]) -> Option<CompletionMessage> {
None
}
}
#[tokio::test]
async fn failed_history_summary_degrades_to_bounded_safe_truncation() {
let mut server = fake_server(vec![ResponsePlan::json(response_text("done"))]).await;
let mut compact = loop_limits();
compact.max_history_messages = 2;
let loop_ = ToolLoop::new(client(&server.url, limits()), vec![], compact)
.expect("loop")
.with_summarizer(Arc::new(FailingSummarizer));
let history = vec![
CompletionMessage::text(MessageRole::System, "old system").expect("message"),
CompletionMessage::text(MessageRole::Avatar, "old question").expect("message"),
CompletionMessage::text(MessageRole::Agent, "old answer").expect("message"),
message(),
];
loop_
.run(
history,
&SessionGeneration::default(),
0,
&CancellationToken::default(),
&RecordingExecutor {
calls: AtomicUsize::new(0),
result: ToolExecution::AmbiguousMutation,
},
)
.await
.expect("compacted completion");
let request = server
.requests
.recv()
.await
.expect("request")
.body
.to_string();
assert!(request.contains("history safely truncated"));
assert!(!request.contains("old question"));
}