use std::collections::{BTreeMap, HashMap}; use std::io::Read; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex, OnceLock, mpsc}; use std::time::Duration; use diesel::prelude::*; use reqwest::blocking::Client; use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; use uuid::Uuid; use crate::db::DbConnection as Connection; use crate::db::queries::chat as queries; use crate::db::schema::ai_models; use crate::engine::ai::{self, AiEndpointConfig, TokenUsage}; use crate::engine::{EngineError, EngineResult, chat_tools}; use crate::model::{ChatConversation, ChatMessage, ChatRole, NewChatConversation, NewChatMessage}; use crate::util::now_unix_ms; const DEFAULT_CONTEXT_TOKENS: usize = 32_768; const DEFAULT_OUTPUT_TOKENS: u64 = 16_384; const MAX_TOOL_ROUNDS: usize = 10; #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct ChatToolCall { pub id: String, pub name: String, pub arguments: Value, } #[derive(Debug, Clone, PartialEq, Eq)] pub struct ChatModelInfo { pub provider: String, pub id: String, pub name: String, pub family: Option, pub context_window: u64, pub max_output_tokens: u64, pub supports_tools: bool, } #[derive(Debug, Clone, PartialEq, Eq)] pub enum ChatEvent { Started { conversation_id: String, }, Content { conversation_id: String, content: String, }, ToolStarted { conversation_id: String, name: String, }, ToolFinished { conversation_id: String, name: String, }, Finished { conversation_id: String, usage: TokenUsage, }, Failed { conversation_id: String, message: String, }, Cancelled { conversation_id: String, }, Navigate { destination: String, entity_id: Option, }, } #[derive(Clone)] pub struct ChatSendOptions { pub endpoint: Option, pub model: Option, pub context_tokens: Option, pub max_output_tokens: u64, pub max_tool_rounds: usize, pub enable_tools: bool, pub event_handler: Option>, } impl Default for ChatSendOptions { fn default() -> Self { Self { endpoint: None, model: None, context_tokens: None, max_output_tokens: DEFAULT_OUTPUT_TOKENS, max_tool_rounds: MAX_TOOL_ROUNDS, enable_tools: true, event_handler: None, } } } #[derive(Debug, Clone, PartialEq, Eq, Default)] pub struct ChatTurnResult { pub content: String, pub usage: TokenUsage, pub cancelled: bool, } #[derive(Debug, Clone, PartialEq, Default)] pub struct AssembledResponse { pub content: String, pub tool_calls: Vec, pub usage: TokenUsage, pub session_id: Option, } #[derive(Default)] struct PartialToolCall { id: String, name: String, arguments: String, } #[derive(Default)] pub struct SseAssembler { buffer: Vec, content: String, tools: BTreeMap, usage: TokenUsage, session_id: Option, done: bool, } impl SseAssembler { pub fn feed(&mut self, bytes: &[u8]) -> EngineResult<()> { self.buffer.extend_from_slice(bytes); while let Some((end, separator_len)) = event_boundary(&self.buffer) { let event = self.buffer.drain(..end).collect::>(); self.buffer.drain(..separator_len); self.process_event(&event)?; } Ok(()) } pub fn snapshot(&self) -> &str { &self.content } pub fn finish(mut self) -> EngineResult { if !self.buffer.is_empty() { let trailing = std::mem::take(&mut self.buffer); self.process_event(&trailing)?; } let tool_calls = self .tools .into_values() .map(|tool| { let arguments = if tool.arguments.trim().is_empty() { json!({}) } else { serde_json::from_str(&tool.arguments).map_err(|error| { EngineError::Parse(format!( "invalid arguments for tool {}: {error}", tool.name )) })? }; Ok(ChatToolCall { id: tool.id, name: tool.name, arguments, }) }) .collect::>>()?; Ok(AssembledResponse { content: self.content, tool_calls, usage: self.usage, session_id: self.session_id, }) } fn process_event(&mut self, event: &[u8]) -> EngineResult<()> { let text = std::str::from_utf8(event) .map_err(|error| EngineError::Parse(format!("invalid SSE encoding: {error}")))?; let data = text .lines() .filter_map(|line| line.strip_prefix("data:")) .map(str::trim_start) .collect::>() .join("\n"); if data.is_empty() || data == "[DONE]" { self.done |= data == "[DONE]"; return Ok(()); } let body: Value = serde_json::from_str(&data) .map_err(|error| EngineError::Parse(format!("malformed SSE event: {error}")))?; self.apply_json(&body) } fn apply_json(&mut self, body: &Value) -> EngineResult<()> { if let Some(error) = body.get("error") { return Err(EngineError::Parse(format!("provider error: {error}"))); } if let Some(session_id) = body .get("session_id") .or_else(|| body.get("sessionId")) .and_then(Value::as_str) { self.session_id = Some(session_id.to_string()); } merge_usage(&mut self.usage, body); for choice in body .get("choices") .and_then(Value::as_array) .into_iter() .flatten() { let delta = choice .get("delta") .or_else(|| choice.get("message")) .unwrap_or(&Value::Null); if let Some(content) = delta.get("content").and_then(Value::as_str) { self.content.push_str(content); } for tool in delta .get("tool_calls") .and_then(Value::as_array) .into_iter() .flatten() { let index = tool.get("index").and_then(Value::as_u64).unwrap_or(0); let partial = self.tools.entry(index).or_default(); if let Some(id) = tool.get("id").and_then(Value::as_str) { partial.id.push_str(id); } let function = tool.get("function").unwrap_or(&Value::Null); if let Some(name) = function.get("name").and_then(Value::as_str) { partial.name.push_str(name); } if let Some(arguments) = function.get("arguments").and_then(Value::as_str) { partial.arguments.push_str(arguments); } } } Ok(()) } } pub fn create_conversation( conn: &Connection, model: Option<&str>, ) -> EngineResult { let model = model.filter(|value| !value.trim().is_empty()); let title = model .map(|model| format!("Chat with {model}")) .unwrap_or_else(|| "New Chat".to_string()); create_conversation_titled(conn, model, &title) } pub fn create_conversation_titled( conn: &Connection, model: Option<&str>, title: &str, ) -> EngineResult { let model = model.filter(|value| !value.trim().is_empty()); let title = title.trim(); if title.is_empty() { return Err(EngineError::Validation( "conversation title is required".to_string(), )); } let id = Uuid::new_v4().to_string(); let now = now_unix_ms(); Ok(queries::insert_conversation( conn, &NewChatConversation { id: &id, title, model, copilot_session_id: None, created_at: now, updated_at: now, }, )?) } pub fn rename_conversation( conn: &Connection, id: &str, title: &str, ) -> EngineResult { let title = title.trim(); if title.is_empty() { return Err(EngineError::Validation( "conversation title is required".to_string(), )); } Ok(queries::rename_conversation( conn, id, title, now_unix_ms(), )?) } pub fn set_conversation_model(conn: &Connection, id: &str, model: &str) -> EngineResult<()> { let model = model.trim(); if model.is_empty() { return Err(EngineError::Validation( "chat model is required".to_string(), )); } if queries::set_conversation_model(conn, id, model, now_unix_ms())? == 0 { return Err(EngineError::NotFound(format!("conversation {id}"))); } Ok(()) } pub fn list_conversations(conn: &Connection) -> EngineResult> { Ok(queries::list_conversations(conn)?) } pub fn list_models(conn: &Connection) -> EngineResult> { let rows = conn.with(|connection| { ai_models::table .select(( ai_models::provider, ai_models::model_id, ai_models::name, ai_models::family, ai_models::context_window, ai_models::max_output_tokens, ai_models::tool_call, )) .order((ai_models::provider.asc(), ai_models::name.asc())) .load::<(String, String, String, Option, i32, i32, i32)>(connection) })?; Ok(rows .into_iter() .map( |(provider, id, name, family, context_window, max_output_tokens, tool_call)| { ChatModelInfo { provider, id, name, family, context_window: context_window.max(0) as u64, max_output_tokens: max_output_tokens.max(0) as u64, supports_tools: tool_call != 0, } }, ) .collect()) } pub fn get_conversation(conn: &Connection, id: &str) -> EngineResult { queries::get_conversation(conn, id).map_err(|error| match error { diesel::result::Error::NotFound => EngineError::NotFound(format!("conversation {id}")), error => error.into(), }) } pub fn delete_conversation(conn: &Connection, id: &str) -> EngineResult<()> { cancel_chat(id); if queries::delete_conversation(conn, id)? == 0 { return Err(EngineError::NotFound(format!("conversation {id}"))); } Ok(()) } pub fn list_messages(conn: &Connection, id: &str) -> EngineResult> { Ok(queries::list_messages(conn, id)?) } pub fn insert_message( conn: &Connection, conversation_id: &str, role: ChatRole, content: Option<&str>, tool_call_id: Option<&str>, tool_calls: Option<&str>, usage: TokenUsage, ) -> EngineResult { let now = now_unix_ms(); Ok(queries::insert_message( conn, &NewChatMessage { conversation_id, role, content, tool_call_id, tool_calls, created_at: now, cache_read_tokens: token_i32(usage.cache_read_tokens), cache_write_tokens: token_i32(usage.cache_write_tokens), token_usage_input: token_i32(usage.input_tokens), token_usage_output: token_i32(usage.output_tokens), }, now, )?) } pub fn subscribe_events() -> mpsc::Receiver { let (sender, receiver) = mpsc::channel(); listeners() .lock() .expect("chat listeners lock") .push(sender); receiver } pub fn cancel_chat(conversation_id: &str) -> bool { let state = in_flight() .lock() .expect("chat cancellation lock") .get(conversation_id) .cloned(); if let Some(state) = state { state.store(true, Ordering::SeqCst); true } else { false } } #[allow(clippy::too_many_arguments)] pub fn send_chat_message( conn: &Connection, data_dir: &std::path::Path, project_id: &str, offline_mode: bool, conversation_id: &str, content: &str, options: ChatSendOptions, ) -> EngineResult { let content = content.trim(); if content.is_empty() { return Err(EngineError::Validation( "chat message is required".to_string(), )); } let conversation = get_conversation(conn, conversation_id)?; let endpoint = match options.endpoint.clone() { Some(endpoint) => validate_runtime_endpoint(endpoint)?, None => ai::active_endpoint(conn, offline_mode)?, }; let model = options .model .clone() .or(conversation.model.clone()) .filter(|value| !value.trim().is_empty()) .unwrap_or_else(|| endpoint.model.clone()); if model.trim().is_empty() { return Err(EngineError::Validation( "AI unavailable - configure a chat model in Settings".to_string(), )); } let cancelled = Arc::new(AtomicBool::new(false)); { let mut active = in_flight().lock().expect("chat cancellation lock"); if active.contains_key(conversation_id) { return Err(EngineError::Conflict( "a response is already streaming for this conversation".to_string(), )); } active.insert(conversation_id.to_string(), Arc::clone(&cancelled)); } let _guard = InFlightGuard(conversation_id.to_string()); emit( &options, ChatEvent::Started { conversation_id: conversation_id.to_string(), }, ); set_conversation_model(conn, conversation_id, &model)?; insert_message( conn, conversation_id, ChatRole::User, Some(content), None, None, TokenUsage::default(), )?; let result = run_turns( conn, data_dir, project_id, conversation_id, &endpoint, &model, &options, &cancelled, ); match &result { Ok(turn) if turn.cancelled => emit( &options, ChatEvent::Cancelled { conversation_id: conversation_id.to_string(), }, ), Ok(turn) => emit( &options, ChatEvent::Finished { conversation_id: conversation_id.to_string(), usage: turn.usage, }, ), Err(error) => emit( &options, ChatEvent::Failed { conversation_id: conversation_id.to_string(), message: error.to_string(), }, ), } result } #[allow(clippy::too_many_arguments)] fn run_turns( conn: &Connection, data_dir: &std::path::Path, project_id: &str, conversation_id: &str, endpoint: &AiEndpointConfig, model: &str, options: &ChatSendOptions, cancelled: &AtomicBool, ) -> EngineResult { let mut total_usage = TokenUsage::default(); let max_rounds = options.max_tool_rounds.min(MAX_TOOL_ROUNDS); let supports_tools = options.enable_tools && chat_tools::model_supports_tools(conn, model)?; let catalog_model = list_models(conn)? .into_iter() .find(|candidate| candidate.id == model); let context_window = options.context_tokens.unwrap_or_else(|| { catalog_model .as_ref() .map(|model| model.context_window as usize) .filter(|window| *window > 0) .unwrap_or(DEFAULT_CONTEXT_TOKENS) }); let max_output_tokens = catalog_model .as_ref() .map(|model| model.max_output_tokens) .filter(|limit| *limit > 0) .map_or(options.max_output_tokens, |limit| { limit.min(options.max_output_tokens) }); let tool_specs = supports_tools.then(chat_tools::tool_specs); let tool_budget = tool_specs .as_ref() .map(|tools| approximate_tokens(&Value::Array(tools.clone()))) .unwrap_or(0); let context_budget = context_window .saturating_sub(max_output_tokens as usize) .saturating_sub(tool_budget) .max(1_024.min(context_window)); for round in 0..=max_rounds { if cancelled.load(Ordering::SeqCst) { return Ok(ChatTurnResult { usage: total_usage, cancelled: true, ..Default::default() }); } let messages = build_context(conn, project_id, conversation_id, context_budget)?; let mut payload = json!({ "model": model, "messages": messages, "stream": true, "stream_options": {"include_usage": true}, "max_tokens": max_output_tokens, }); if let Some(tools) = tool_specs.as_ref() { payload["tools"] = Value::Array(tools.clone()); payload["tool_choice"] = json!("auto"); } let response = request_completion(endpoint, &payload, cancelled, |content| { emit( options, ChatEvent::Content { conversation_id: conversation_id.to_string(), content: content.to_string(), }, ); })?; add_usage(&mut total_usage, response.usage); if let Some(session_id) = response.session_id.as_deref() { queries::set_session_id(conn, conversation_id, Some(session_id), now_unix_ms())?; } if cancelled.load(Ordering::SeqCst) { if !response.content.is_empty() { insert_message( conn, conversation_id, ChatRole::Assistant, Some(&response.content), None, None, response.usage, )?; } return Ok(ChatTurnResult { content: response.content, usage: total_usage, cancelled: true, }); } if !response.tool_calls.is_empty() && round == max_rounds { if !response.content.is_empty() { insert_message( conn, conversation_id, ChatRole::Assistant, Some(&response.content), None, None, response.usage, )?; } return Err(EngineError::Validation(format!( "chat exceeded the {max_rounds}-round tool limit" ))); } let serialized_calls = (!response.tool_calls.is_empty()) .then(|| serialize_tool_calls(&response.tool_calls)) .transpose()?; insert_message( conn, conversation_id, ChatRole::Assistant, (!response.content.is_empty()).then_some(response.content.as_str()), None, serialized_calls.as_deref(), response.usage, )?; if response.tool_calls.is_empty() { return Ok(ChatTurnResult { content: response.content, usage: total_usage, cancelled: false, }); } for (index, call) in response.tool_calls.iter().enumerate() { if cancelled.load(Ordering::SeqCst) { persist_cancelled_tool_results( conn, conversation_id, &response.tool_calls[index..], )?; return Ok(ChatTurnResult { usage: total_usage, cancelled: true, ..Default::default() }); } emit( options, ChatEvent::ToolStarted { conversation_id: conversation_id.to_string(), name: call.name.clone(), }, ); if cancelled.load(Ordering::SeqCst) { persist_cancelled_tool_results( conn, conversation_id, &response.tool_calls[index..], )?; return Ok(ChatTurnResult { usage: total_usage, cancelled: true, ..Default::default() }); } let result = chat_tools::execute(conn, data_dir, project_id, &call.name, &call.arguments); let result = match result { Ok(result) => result, Err(error) => json!({"success": false, "error": error.to_string()}), }; if call.name == "navigate" && let Some(navigation) = result.get("navigation") && let Some(destination) = navigation.get("destination").and_then(Value::as_str) { emit( options, ChatEvent::Navigate { destination: destination.to_string(), entity_id: navigation .get("entity_id") .and_then(Value::as_str) .map(str::to_string), }, ); } let result_text = serde_json::to_string(&result)?; insert_message( conn, conversation_id, ChatRole::Tool, Some(&result_text), Some(&call.id), None, TokenUsage::default(), )?; emit( options, ChatEvent::ToolFinished { conversation_id: conversation_id.to_string(), name: call.name.clone(), }, ); } } unreachable!("bounded loop returns on its final iteration") } fn persist_cancelled_tool_results( conn: &Connection, conversation_id: &str, calls: &[ChatToolCall], ) -> EngineResult<()> { let result = json!({"success": false, "cancelled": true}).to_string(); for call in calls { insert_message( conn, conversation_id, ChatRole::Tool, Some(&result), Some(&call.id), None, TokenUsage::default(), )?; } Ok(()) } pub fn build_context( conn: &Connection, project_id: &str, conversation_id: &str, token_budget: usize, ) -> EngineResult> { let system = chat_tools::system_prompt(conn, project_id)?; let messages = list_messages(conn, conversation_id)?; let mut groups: Vec> = Vec::new(); let mut current = Vec::new(); for message in messages { let value = message_json(&message)?; if message.role == ChatRole::System { continue; } if message.role == ChatRole::User && !current.is_empty() { groups.push(std::mem::take(&mut current)); } current.push(value); } if !current.is_empty() { groups.push(current); } let system_value = json!({"role": "system", "content": system}); let mut used = approximate_tokens(&system_value); let mut selected = Vec::new(); for group in groups.into_iter().rev() { let cost = group.iter().map(approximate_tokens).sum::(); if used + cost > token_budget && !selected.is_empty() { continue; } used += cost; selected.push(group); if used >= token_budget { break; } } selected.reverse(); let mut result = vec![system_value]; result.extend(selected.into_iter().flatten()); Ok(result) } fn request_completion( endpoint: &AiEndpointConfig, payload: &Value, cancelled: &AtomicBool, mut on_content: impl FnMut(&str), ) -> EngineResult { let client = Client::builder() .connect_timeout(Duration::from_secs(5)) .timeout(Duration::from_secs(300)) .build()?; let mut request = client .post(chat_completions_url(&endpoint.url)) .json(payload); if let Some(api_key) = endpoint.api_key.as_deref() { request = request.bearer_auth(api_key); } let mut response = request.send()?.error_for_status()?; let is_stream = response .headers() .get(reqwest::header::CONTENT_TYPE) .and_then(|value| value.to_str().ok()) .is_some_and(|value| value.contains("text/event-stream")); if !is_stream { let body: Value = response.json()?; let mut assembler = SseAssembler::default(); assembler.apply_json(&body)?; return assembler.finish(); } let mut assembler = SseAssembler::default(); let mut chunk = [0_u8; 4096]; loop { if cancelled.load(Ordering::SeqCst) { break; } let count = response.read(&mut chunk)?; if count == 0 { break; } let old_len = assembler.snapshot().len(); assembler.feed(&chunk[..count])?; if assembler.snapshot().len() != old_len { on_content(assembler.snapshot()); } } assembler.finish() } fn message_json(message: &ChatMessage) -> EngineResult { let mut value = json!({"role": message.role.as_str()}); if let Some(content) = message.content.as_deref() { value["content"] = json!(content); } if let Some(tool_call_id) = message.tool_call_id.as_deref() { value["tool_call_id"] = json!(tool_call_id); } if let Some(tool_calls) = message.tool_calls.as_deref() { value["tool_calls"] = serde_json::from_str(tool_calls)?; } Ok(value) } fn serialize_tool_calls(calls: &[ChatToolCall]) -> EngineResult { let calls = calls .iter() .map(|call| { json!({ "id": call.id, "type": "function", "function": { "name": call.name, "arguments": call.arguments.to_string(), } }) }) .collect::>(); Ok(serde_json::to_string(&calls)?) } fn approximate_tokens(value: &Value) -> usize { value.to_string().chars().count().div_ceil(4).max(1) } fn merge_usage(usage: &mut TokenUsage, body: &Value) { let source = body.get("usage").unwrap_or(&Value::Null); usage.input_tokens = source .get("prompt_tokens") .and_then(Value::as_u64) .or(usage.input_tokens); usage.output_tokens = source .get("completion_tokens") .and_then(Value::as_u64) .or(usage.output_tokens); usage.cache_read_tokens = source .get("prompt_tokens_details") .and_then(|value| value.get("cached_tokens")) .and_then(Value::as_u64) .or_else(|| source.get("cache_read_tokens").and_then(Value::as_u64)) .or(usage.cache_read_tokens); usage.cache_write_tokens = source .get("completion_tokens_details") .and_then(|value| value.get("cached_tokens")) .and_then(Value::as_u64) .or_else(|| source.get("cache_write_tokens").and_then(Value::as_u64)) .or(usage.cache_write_tokens); } fn add_usage(total: &mut TokenUsage, current: TokenUsage) { total.input_tokens = add_optional(total.input_tokens, current.input_tokens); total.output_tokens = add_optional(total.output_tokens, current.output_tokens); total.cache_read_tokens = add_optional(total.cache_read_tokens, current.cache_read_tokens); total.cache_write_tokens = add_optional(total.cache_write_tokens, current.cache_write_tokens); } fn add_optional(left: Option, right: Option) -> Option { match (left, right) { (None, None) => None, (left, right) => Some(left.unwrap_or(0).saturating_add(right.unwrap_or(0))), } } fn token_i32(value: Option) -> Option { value.map(|value| i32::try_from(value).unwrap_or(i32::MAX)) } fn validate_runtime_endpoint(endpoint: AiEndpointConfig) -> EngineResult { if endpoint.url.trim().is_empty() || endpoint.model.trim().is_empty() { return Err(EngineError::Validation( "AI unavailable - configure endpoint and model in Settings".to_string(), )); } Ok(endpoint) } fn chat_completions_url(base_url: &str) -> String { let base = base_url.trim_end_matches('/'); if base.ends_with("/chat/completions") { base.to_string() } else if base.ends_with("/v1") { format!("{base}/chat/completions") } else { format!("{base}/v1/chat/completions") } } fn event_boundary(buffer: &[u8]) -> Option<(usize, usize)> { buffer .windows(4) .position(|window| window == b"\r\n\r\n") .map(|position| (position, 4)) .or_else(|| { buffer .windows(2) .position(|window| window == b"\n\n") .map(|position| (position, 2)) }) } fn in_flight() -> &'static Mutex>> { static IN_FLIGHT: OnceLock>>> = OnceLock::new(); IN_FLIGHT.get_or_init(|| Mutex::new(HashMap::new())) } fn listeners() -> &'static Mutex>> { static LISTENERS: OnceLock>>> = OnceLock::new(); LISTENERS.get_or_init(|| Mutex::new(Vec::new())) } fn emit(options: &ChatSendOptions, event: ChatEvent) { if let Some(handler) = options.event_handler.as_deref() { handler(event.clone()); } listeners() .lock() .expect("chat listeners lock") .retain(|sender| sender.send(event.clone()).is_ok()); } struct InFlightGuard(String); impl Drop for InFlightGuard { fn drop(&mut self) { in_flight() .lock() .expect("chat cancellation lock") .remove(&self.0); } }