diff --git a/Cargo.lock b/Cargo.lock index e77ff85..021b488 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2340,6 +2340,12 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "98de4bbd547a563b716d8dfa9aad1cb19bfab00f4fa09a6a4ed21dbcf44ce9c4" +[[package]] +name = "float_next_after" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bf7cc16383c4b8d58b9905a8509f02926ce3058053c056376248d958c9df1e8" + [[package]] name = "fluent-bundle" version = "0.16.0" @@ -3363,6 +3369,7 @@ dependencies = [ "image 0.24.9", "kamadak-exif", "log", + "lyon_path", "once_cell", "raw-window-handle", "rustc-hash 2.1.2", @@ -3427,6 +3434,7 @@ dependencies = [ "iced_glyphon", "iced_graphics", "log", + "lyon", "once_cell", "resvg", "rustc-hash 2.1.2", @@ -4352,6 +4360,58 @@ dependencies = [ "which", ] +[[package]] +name = "lyon" +version = "1.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd0578bdecb7d6d88987b8b2b1e3a4e2f81df9d0ece1078623324a567904e7b7" +dependencies = [ + "lyon_algorithms", + "lyon_tessellation", +] + +[[package]] +name = "lyon_algorithms" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8575c0d003ae459399623c4def180c63b77f343b1a7fee64f249b349e7699a31" +dependencies = [ + "lyon_path", + "num-traits", +] + +[[package]] +name = "lyon_geom" +version = "1.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4336502e29e32af93cf2dad2214ed6003c17ceb5bd499df77b1de663b9042b92" +dependencies = [ + "arrayvec", + "euclid", + "num-traits", +] + +[[package]] +name = "lyon_path" +version = "1.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c463f9c428b7fc5ec885dcd39ce4aa61e29111d0e33483f6f98c74e89d8621e" +dependencies = [ + "lyon_geom", + "num-traits", +] + +[[package]] +name = "lyon_tessellation" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e43b7e44161571868f5c931d12583592c223c5583eef86b08aa02b7048a3552" +dependencies = [ + "float_next_after", + "lyon_path", + "num-traits", +] + [[package]] name = "malloc_buf" version = "0.0.6" @@ -4758,6 +4818,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", + "libm", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 74fb059..b43ff8d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -54,7 +54,7 @@ mlua = { version = "0.12", features = ["lua54", "vendored", "serialize"] } url = "2.5" # UI framework -iced = { version = "0.13", features = ["wgpu", "advanced", "image", "svg", "tokio", "markdown"] } +iced = { version = "0.13", features = ["wgpu", "advanced", "canvas", "image", "svg", "tokio", "markdown"] } # Editor widget cosmic-text = "0.12" diff --git a/README.md b/README.md index 7d1ef32..548a916 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ The project is under active development. Core blogging workflows are broadly ava - Markdown/Liquid rendering with native macros, multilingual routes, feeds, sitemap, Pagefind, and incremental site generation through cancellable section task groups. - Local preview in the app or system browser. - Optional one-shot AI translation, description, analysis, taxonomy, and language-detection operations using online or local OpenAI-compatible endpoints with airplane-mode gating. -- Persistent conversational AI with safe Markdown, streamed and cancellable responses, model/session/token tracking, bounded project-aware blog tools, and localized conversation management in the Chat workspace. +- Persistent conversational AI with safe Markdown, streamed and cancellable responses, model/session/token tracking, bounded project-aware blog tools, and localized conversation management in the Chat workspace. Allowlisted render tools add persistent native cards, charts, forms, lists, metrics, mind maps, tables, and tabs without executing assistant-provided HTML or JavaScript. - SSH-agent-based SCP or rsync publishing. - Integrated Git workflow with repository initialization, Git LFS image tracking, status and diffs, branch/file history, commits, remotes, cancellable fetch/pull/push, and post-pull filesystem reconciliation; network actions respect airplane mode. - Site, media, and translation validation plus `ruds://new-post` Blogmark capture and Lua transforms; bDS2 keeps its separate `bds2://` bookmarklet protocol. diff --git a/RUST_PLAN_EXTENSION.md b/RUST_PLAN_EXTENSION.md index b067a7b..b35b807 100644 --- a/RUST_PLAN_EXTENSION.md +++ b/RUST_PLAN_EXTENSION.md @@ -39,10 +39,9 @@ Done: - OpenAI-compatible SSE streaming with split-frame content/tool assembly, provider-error handling, independent cancellation, bounded tool rounds, and context truncation that preserves system messages and tool pairs. - Project-aware tools over statistics, FTS search, posts, media, templates, scripts, tags/categories, metadata mutation through shared engines, and allowlisted workspace navigation. - Localized Chat sidebar/editor with conversation and model controls, safe GFM text rendering with blocked external images, streaming/tool state, multiline send/stop controls, and status-bar token totals. +- Fixed native A2UI cards, charts, forms, lists, metrics, mind maps, tables, and tabs with safe nested fallbacks, persistent stable-ID interaction state, debounced form values, and strictly allowlisted application actions. - Online/airplane endpoint routing uses the shared secure endpoint and model infrastructure; unavailable modes direct the user to the existing localized AI settings. -Persistent A2UI surfaces remain separately tracked below. - ### Embeddings, semantic search, and duplicates — Open Existing source contains schema placeholders only. @@ -118,11 +117,13 @@ Open: - Sidebar/editor navigation, editing, publishing, and live domain-event updates. - Remote operation through the headless server. -### A2UI surfaces — Open +### A2UI surfaces — Complete -Open: +Done: -- A2UI component renderer, surface state/data flow, and AI-assistant integration. +- Eight structured render tools and fixed native rendering for ten surface types, including all required chart and form variants. +- Per-conversation form, tab, and dismissal state tied to stable message/tool-index IDs and restored on reopen. +- Safe text/JSON fallbacks, inert assistant markup, persistent expanded surfaces during later streaming, and visibly rejected non-allowlisted actions. ## Suggested Order @@ -131,6 +132,6 @@ Open: 3. Embeddings and duplicate detection. 4. Documentation and menu UX. 5. Headless server and TUI. -6. A2UI after conversational AI exists. +6. A2UI after conversational AI exists. (Complete) The order may change when an extension directly unlocks a concrete user workflow; it must not create a parallel data model or bypass core engines. diff --git a/crates/bds-core/migrations/20260719163000_add_chat_surface_state/down.sql b/crates/bds-core/migrations/20260719163000_add_chat_surface_state/down.sql new file mode 100644 index 0000000..b83fcd9 --- /dev/null +++ b/crates/bds-core/migrations/20260719163000_add_chat_surface_state/down.sql @@ -0,0 +1,2 @@ +-- This file should undo anything in `up.sql` +ALTER TABLE chat_conversations DROP COLUMN surface_state; diff --git a/crates/bds-core/migrations/20260719163000_add_chat_surface_state/up.sql b/crates/bds-core/migrations/20260719163000_add_chat_surface_state/up.sql new file mode 100644 index 0000000..f5070ef --- /dev/null +++ b/crates/bds-core/migrations/20260719163000_add_chat_surface_state/up.sql @@ -0,0 +1,2 @@ +-- Your SQL goes here +ALTER TABLE chat_conversations ADD COLUMN surface_state TEXT; diff --git a/crates/bds-core/src/db/migrations.rs b/crates/bds-core/src/db/migrations.rs index 343bfde..3e14b9c 100644 --- a/crates/bds-core/src/db/migrations.rs +++ b/crates/bds-core/src/db/migrations.rs @@ -36,7 +36,7 @@ mod tests { let applied = db .conn() .with_migrations(|conn| conn.applied_migrations().unwrap().len()); - assert_eq!(applied, 5); + assert_eq!(applied, 6); } #[test] @@ -204,4 +204,40 @@ mod tests { .unwrap(); assert_eq!(usage, (Some(8), Some(5), None, None)); } + + #[test] + fn existing_conversations_are_preserved_when_surface_state_is_added() { + let db = Database::open_in_memory().unwrap(); + db.conn().with_migrations(|conn| { + for _ in 0..5 { + conn.run_next_migration(MIGRATIONS).unwrap(); + } + }); + db.conn() + .with(|conn| { + diesel::insert_into(chat_conversations::table) + .values(( + chat_conversations::id.eq("existing-surface-chat"), + chat_conversations::title.eq("Keep me"), + chat_conversations::created_at.eq(1_i64), + chat_conversations::updated_at.eq(1_i64), + )) + .execute(conn) + }) + .unwrap(); + + run_migrations(db.conn()).unwrap(); + + let (title, state) = db + .conn() + .with(|conn| { + chat_conversations::table + .filter(chat_conversations::id.eq("existing-surface-chat")) + .select((chat_conversations::title, chat_conversations::surface_state)) + .first::<(String, Option)>(conn) + }) + .unwrap(); + assert_eq!(title, "Keep me"); + assert_eq!(state, None); + } } diff --git a/crates/bds-core/src/db/queries/chat.rs b/crates/bds-core/src/db/queries/chat.rs index d09cc6d..d295f80 100644 --- a/crates/bds-core/src/db/queries/chat.rs +++ b/crates/bds-core/src/db/queries/chat.rs @@ -92,6 +92,22 @@ pub fn set_session_id( }) } +pub fn set_surface_state( + conn: &DbConnection, + id: &str, + state: &str, + updated_at: i64, +) -> QueryResult { + conn.with(|connection| { + diesel::update(chat_conversations::table.find(id)) + .set(( + chat_conversations::surface_state.eq(state), + chat_conversations::updated_at.eq(updated_at), + )) + .execute(connection) + }) +} + pub fn delete_conversation(conn: &DbConnection, id: &str) -> QueryResult { conn.with(|connection| { connection.transaction(|connection| { diff --git a/crates/bds-core/src/db/schema.rs b/crates/bds-core/src/db/schema.rs index 1f8b963..2124a5b 100644 --- a/crates/bds-core/src/db/schema.rs +++ b/crates/bds-core/src/db/schema.rs @@ -64,6 +64,7 @@ diesel::table! { title -> Text, model -> Nullable, copilot_session_id -> Nullable, + surface_state -> Nullable, created_at -> BigInt, updated_at -> BigInt, } diff --git a/crates/bds-core/src/engine/chat.rs b/crates/bds-core/src/engine/chat.rs index d278b0a..cef1c0d 100644 --- a/crates/bds-core/src/engine/chat.rs +++ b/crates/bds-core/src/engine/chat.rs @@ -52,6 +52,8 @@ pub enum ChatEvent { ToolStarted { conversation_id: String, name: String, + surface_id: String, + arguments: Value, }, ToolFinished { conversation_id: String, @@ -279,6 +281,7 @@ pub fn create_conversation_titled( title, model, copilot_session_id: None, + surface_state: None, created_at: now, updated_at: now, }, @@ -373,6 +376,38 @@ pub fn list_messages(conn: &Connection, id: &str) -> EngineResult EngineResult { + let conversation = match queries::get_conversation(conn, conversation_id) { + Ok(conversation) => conversation, + Err(diesel::result::Error::NotFound) => return Ok(Default::default()), + Err(error) => return Err(error.into()), + }; + conversation.surface_state.as_deref().map_or_else( + || Ok(Default::default()), + |state| { + serde_json::from_str(state) + .map_err(|error| EngineError::Parse(format!("invalid chat surface state: {error}"))) + }, + ) +} + +pub fn put_surface_state( + conn: &Connection, + conversation_id: &str, + state: &crate::engine::chat_surfaces::ChatSurfaceState, +) -> EngineResult<()> { + let state = serde_json::to_string(state)?; + if queries::set_surface_state(conn, conversation_id, &state, now_unix_ms())? == 0 { + return Err(EngineError::NotFound(format!( + "chat conversation {conversation_id}" + ))); + } + Ok(()) +} + pub fn insert_message( conn: &Connection, conversation_id: &str, @@ -633,7 +668,7 @@ fn run_turns( let serialized_calls = (!response.tool_calls.is_empty()) .then(|| serialize_tool_calls(&response.tool_calls)) .transpose()?; - insert_message( + let assistant_message = insert_message( conn, conversation_id, ChatRole::Assistant, @@ -667,6 +702,8 @@ fn run_turns( ChatEvent::ToolStarted { conversation_id: conversation_id.to_string(), name: call.name.clone(), + surface_id: format!("{}-surface-{index}", assistant_message.id), + arguments: call.arguments.clone(), }, ); if cancelled.load(Ordering::SeqCst) { diff --git a/crates/bds-core/src/engine/chat_surfaces.rs b/crates/bds-core/src/engine/chat_surfaces.rs new file mode 100644 index 0000000..f4feff5 --- /dev/null +++ b/crates/bds-core/src/engine/chat_surfaces.rs @@ -0,0 +1,583 @@ +use std::collections::{BTreeMap, BTreeSet}; + +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; + +use crate::model::ChatMessage; + +pub const RENDER_TOOL_NAMES: [&str; 8] = [ + "render_card", + "render_chart", + "render_form", + "render_list", + "render_metric", + "render_mindmap", + "render_table", + "render_tabs", +]; + +#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize)] +pub struct ChatSurfaceState { + #[serde(default)] + pub surface_data: BTreeMap>, + #[serde(default)] + pub surface_tabs: BTreeMap, + #[serde(default)] + pub dismissed_surfaces: BTreeSet, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SurfaceKind { + Card, + Chart, + Form, + List, + Metric, + Mindmap, + Table, + Tabs, + Text, + Json, +} + +impl SurfaceKind { + pub const fn as_str(self) -> &'static str { + match self { + Self::Card => "card", + Self::Chart => "chart", + Self::Form => "form", + Self::List => "list", + Self::Metric => "metric", + Self::Mindmap => "mindmap", + Self::Table => "table", + Self::Tabs => "tabs", + Self::Text => "text", + Self::Json => "json", + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ChartType { + Bar, + StackedBar, + Line, + Area, + Pie, + Donut, + Heatmap, +} + +impl ChartType { + pub const fn as_str(self) -> &'static str { + match self { + Self::Bar => "bar", + Self::StackedBar => "stacked-bar", + Self::Line => "line", + Self::Area => "area", + Self::Pie => "pie", + Self::Donut => "donut", + Self::Heatmap => "heatmap", + } + } + + fn parse(value: Option<&str>) -> Self { + match value { + Some("stacked-bar") => Self::StackedBar, + Some("line") => Self::Line, + Some("area") => Self::Area, + Some("pie") => Self::Pie, + Some("donut") => Self::Donut, + Some("heatmap") => Self::Heatmap, + _ => Self::Bar, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FormInputType { + Text, + Textarea, + Select, + Checkbox, + Date, + Number, +} + +impl FormInputType { + fn parse(value: Option<&str>) -> Self { + match value { + Some("textarea") => Self::Textarea, + Some("select") => Self::Select, + Some("checkbox") => Self::Checkbox, + Some("date") => Self::Date, + Some("number") => Self::Number, + _ => Self::Text, + } + } +} + +#[derive(Debug, Clone, PartialEq)] +pub struct InlineSurface { + pub id: String, + pub kind: SurfaceKind, + pub title: Option, + pub subtitle: Option, + pub body: Option, + pub actions: Vec, + pub columns: Vec, + pub rows: Vec>, + pub chart_type: Option, + pub series: Vec, + pub max_value: Option, + pub label: Option, + pub value: Option, + pub items: Vec, + pub nodes: Vec, + pub fields: Vec, + pub submit_label: Option, + pub submit_action: Option, + pub tabs: Vec, + pub selected_index: Option, + pub raw: Option, +} + +impl InlineSurface { + fn empty(id: String, kind: SurfaceKind) -> Self { + Self { + id, + kind, + title: None, + subtitle: None, + body: None, + actions: vec![], + columns: vec![], + rows: vec![], + chart_type: None, + series: vec![], + max_value: None, + label: None, + value: None, + items: vec![], + nodes: vec![], + fields: vec![], + submit_label: None, + submit_action: None, + tabs: vec![], + selected_index: None, + raw: None, + } + } +} + +#[derive(Debug, Clone, PartialEq)] +pub struct SurfaceAction { + pub label: String, + pub action: String, + pub payload: Value, +} +#[derive(Debug, Clone, PartialEq)] +pub struct ChartSeries { + pub label: String, + pub value: f64, + pub segments: Vec, +} +#[derive(Debug, Clone, PartialEq)] +pub struct ChartSegment { + pub label: String, + pub value: f64, +} +#[derive(Debug, Clone, PartialEq)] +pub struct MindmapNode { + pub id: Option, + pub label: String, + pub children: Vec, +} +#[derive(Debug, Clone, PartialEq)] +pub struct FormField { + pub key: String, + pub label: String, + pub input_type: FormInputType, + pub placeholder: Option, + pub value: Value, + pub options: Vec, + pub required: bool, +} +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FieldOption { + pub label: String, + pub value: String, +} +impl std::fmt::Display for FieldOption { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.label) + } +} +#[derive(Debug, Clone, PartialEq)] +pub struct TabPanel { + pub label: String, + pub content: Vec, +} +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SurfaceNavigation { + pub destination: String, + pub entity_id: Option, +} + +pub fn build_message_surfaces( + message: &ChatMessage, + state: &ChatSurfaceState, +) -> Vec { + let Some(raw) = message.tool_calls.as_deref() else { + return vec![]; + }; + let Ok(calls) = serde_json::from_str::>(raw) else { + return vec![]; + }; + calls + .iter() + .enumerate() + .filter_map(|(index, call)| { + let function = call.get("function").unwrap_or(call); + let name = function.get("name")?.as_str()?; + let raw = function + .get("arguments") + .cloned() + .unwrap_or_else(|| json!({})); + let arguments = raw + .as_str() + .and_then(|s| serde_json::from_str(s).ok()) + .unwrap_or(raw); + build_render_surface( + name, + &arguments, + format!("{}-surface-{index}", message.id), + state, + ) + }) + .filter(|surface| !state.dismissed_surfaces.contains(&surface.id)) + .collect() +} + +pub fn build_render_surface( + name: &str, + arguments: &Value, + id: String, + state: &ChatSurfaceState, +) -> Option { + match name { + "render_card" => Some(card(arguments, id)), + "render_chart" => Some(chart(arguments, id)), + "render_form" => Some(form(arguments, id, state)), + "render_list" => Some(list(arguments, id)), + "render_metric" => Some(metric(arguments, id)), + "render_mindmap" => Some(mindmap(arguments, id)), + "render_table" => Some(table(arguments, id)), + "render_tabs" => Some(tabs(arguments, id, state)), + unknown if unknown.starts_with("render_") => { + let mut surface = InlineSurface::empty(id, SurfaceKind::Json); + surface.raw = Some(arguments.clone()); + Some(surface) + } + _ => None, + } +} + +pub fn merge_form_data(payload: Value, surface_id: &str, state: &ChatSurfaceState) -> Value { + let mut result = payload.as_object().cloned().unwrap_or_default(); + if let Some(values) = state.surface_data.get(surface_id) { + result.insert( + "formData".into(), + Value::Object(values.clone().into_iter().collect()), + ); + } + Value::Object(result) +} + +pub fn resolve_surface_action(action: &str, payload: &Value) -> Result { + let map = payload + .as_object() + .ok_or_else(|| "surface action payload must be an object".to_string())?; + let normalized = action.replace('_', "").to_ascii_lowercase(); + let open = |destination: &str, keys: &[&str]| { + let id = keys + .iter() + .find_map(|key| map.get(*key).and_then(Value::as_str)) + .filter(|id| !id.trim().is_empty()) + .ok_or_else(|| format!("surface action {action} requires an identifier"))?; + Ok(SurfaceNavigation { + destination: destination.into(), + entity_id: Some(id.into()), + }) + }; + match normalized.as_str() { + "openpost" => open("posts", &["postId", "post_id", "id"]), + "openmedia" => open("media", &["mediaId", "media_id", "id"]), + "openchat" => open( + "chat", + &[ + "conversationId", + "conversation_id", + "chatId", + "chat_id", + "id", + ], + ), + "opensettings" => Ok(navigation("settings")), + "switchview" | "setview" | "setactiveview" => { + let view = map + .get("view") + .or_else(|| map.get("destination")) + .and_then(Value::as_str) + .filter(|view| { + [ + "posts", + "pages", + "media", + "templates", + "scripts", + "tags", + "chat", + "import", + "git", + "settings", + ] + .contains(view) + }) + .ok_or_else(|| format!("surface action {action} requires a valid view"))?; + Ok(navigation(view)) + } + "togglesidebar" => Ok(navigation("toggle_sidebar")), + "togglepanel" | "openpanel" => Ok(navigation("toggle_panel")), + "toggleassistantsidebar" => Ok(navigation("toggle_assistant_sidebar")), + _ => Err(format!("unsupported surface action: {action}")), + } +} + +fn navigation(destination: &str) -> SurfaceNavigation { + SurfaceNavigation { + destination: destination.into(), + entity_id: None, + } +} + +fn card(a: &Value, id: String) -> InlineSurface { + let mut s = InlineSurface::empty(id, SurfaceKind::Card); + s.title = string(a, "title"); + s.subtitle = string(a, "subtitle"); + s.body = string(a, "body"); + s.actions = array(a, "actions") + .iter() + .map(|v| SurfaceAction { + label: string(v, "label").unwrap_or_default(), + action: string(v, "action").unwrap_or_default(), + payload: v.get("payload").cloned().unwrap_or_else(|| json!({})), + }) + .collect(); + s +} + +fn chart(a: &Value, id: String) -> InlineSurface { + let mut s = InlineSurface::empty(id, SurfaceKind::Chart); + s.title = string(a, "title"); + s.chart_type = Some(ChartType::parse( + a.get("chartType") + .or_else(|| a.get("chart_type")) + .and_then(Value::as_str), + )); + s.series = array(a, "series") + .iter() + .map(|v| ChartSeries { + label: string(v, "label").unwrap_or_default(), + value: numeric(v.get("value")), + segments: array(v, "segments") + .iter() + .map(|seg| ChartSegment { + label: string(seg, "label").unwrap_or_default(), + value: numeric(seg.get("value")), + }) + .collect(), + }) + .collect(); + s.max_value = Some(s.series.iter().map(|v| v.value).fold(0.0_f64, f64::max)); + s +} + +fn form(a: &Value, id: String, state: &ChatSurfaceState) -> InlineSurface { + let mut s = InlineSurface::empty(id.clone(), SurfaceKind::Form); + s.title = string(a, "title"); + s.fields = array(a, "fields") + .iter() + .map(|v| { + let key = string(v, "key").unwrap_or_else(|| "field".into()); + let default = v + .get("defaultValue") + .or_else(|| v.get("default_value")) + .cloned() + .unwrap_or(Value::Null); + FormField { + label: string(v, "label").unwrap_or_else(|| key.clone()), + input_type: FormInputType::parse( + v.get("inputType") + .or_else(|| v.get("input_type")) + .and_then(Value::as_str), + ), + placeholder: string(v, "placeholder"), + value: state + .surface_data + .get(&id) + .and_then(|m| m.get(&key)) + .cloned() + .unwrap_or(default), + options: array(v, "options") + .iter() + .map(|o| FieldOption { + label: string(o, "label").unwrap_or_default(), + value: string(o, "value").unwrap_or_default(), + }) + .collect(), + required: v.get("required").and_then(Value::as_bool).unwrap_or(false), + key, + } + }) + .collect(); + s.submit_label = a + .get("submitLabel") + .or_else(|| a.get("submit_label")) + .and_then(Value::as_str) + .map(str::to_owned); + s.submit_action = a + .get("submitAction") + .or_else(|| a.get("submit_action")) + .and_then(Value::as_str) + .map(str::to_owned); + s +} + +fn list(a: &Value, id: String) -> InlineSurface { + let mut s = InlineSurface::empty(id, SurfaceKind::List); + s.title = string(a, "title"); + s.items = strings(a.get("items")); + s +} +fn metric(a: &Value, id: String) -> InlineSurface { + let mut s = InlineSurface::empty(id, SurfaceKind::Metric); + s.label = string(a, "label"); + s.value = scalar(a.get("value")); + s +} +fn mindmap(a: &Value, id: String) -> InlineSurface { + let mut s = InlineSurface::empty(id, SurfaceKind::Mindmap); + s.title = string(a, "title"); + s.nodes = array(a, "nodes") + .iter() + .map(|v| MindmapNode { + id: string(v, "id"), + label: string(v, "label").unwrap_or_default(), + children: strings(v.get("children")), + }) + .collect(); + s +} +fn table(a: &Value, id: String) -> InlineSurface { + let mut s = InlineSurface::empty(id, SurfaceKind::Table); + s.title = string(a, "title"); + s.columns = strings(a.get("columns")); + s.rows = array(a, "rows").iter().map(|v| strings(Some(v))).collect(); + s +} +fn tabs(a: &Value, id: String, state: &ChatSurfaceState) -> InlineSurface { + let mut s = InlineSurface::empty(id.clone(), SurfaceKind::Tabs); + s.title = string(a, "title"); + s.tabs = array(a, "tabs") + .iter() + .enumerate() + .map(|(ti, tab)| TabPanel { + label: string(tab, "label").unwrap_or_default(), + content: array(tab, "content") + .iter() + .enumerate() + .map(|(ci, v)| nested(v, format!("{id}-tab-{ti}-{ci}"), state)) + .collect(), + }) + .collect(); + s.selected_index = Some( + state + .surface_tabs + .get(&id) + .copied() + .unwrap_or(0) + .min(s.tabs.len().saturating_sub(1)), + ); + s +} +fn nested(v: &Value, id: String, state: &ChatSurfaceState) -> InlineSurface { + let Some(object) = v.as_object() else { + let mut s = InlineSurface::empty(id, SurfaceKind::Text); + s.body = Some(scalar(Some(v)).unwrap_or_default()); + return s; + }; + match object.get("type").and_then(Value::as_str).unwrap_or("text") { + "card" => card(v, id), + "chart" => chart(v, id), + "form" => form(v, id, state), + "list" => list(v, id), + "metric" => metric(v, id), + "mindmap" => mindmap(v, id), + "table" => table(v, id), + "tabs" => tabs(v, id, state), + "text" => { + let mut s = InlineSurface::empty(id, SurfaceKind::Text); + s.body = string(v, "body").or_else(|| string(v, "text")); + s + } + _ => { + let mut s = InlineSurface::empty(id, SurfaceKind::Json); + s.raw = Some(Value::Object(object.clone())); + s + } + } +} + +fn array<'a>(v: &'a Value, key: &str) -> &'a [Value] { + v.get(key) + .and_then(Value::as_array) + .map(Vec::as_slice) + .unwrap_or_default() +} +fn string(v: &Value, key: &str) -> Option { + scalar(v.get(key)) +} +fn strings(v: Option<&Value>) -> Vec { + v.and_then(Value::as_array) + .into_iter() + .flatten() + .filter_map(|v| scalar(Some(v))) + .collect() +} +fn scalar(v: Option<&Value>) -> Option { + match v? { + Value::Null => None, + Value::String(v) => Some(v.clone()), + Value::Bool(v) => Some(v.to_string()), + Value::Number(v) => Some(v.to_string()), + v => serde_json::to_string(v).ok(), + } +} +fn numeric(v: Option<&Value>) -> f64 { + v.and_then(|v| { + v.as_f64() + .or_else(|| v.as_str().and_then(|s| s.parse().ok())) + }) + .filter(|v| v.is_finite()) + .unwrap_or(0.0) +} + +pub fn render_tool_result(name: &str, arguments: &Value) -> Option { + let kind = name.strip_prefix("render_")?; + RENDER_TOOL_NAMES.contains(&name).then(|| { + let mut values = arguments.as_object().cloned().unwrap_or_default(); + values.insert("type".into(), Value::String(kind.into())); + Value::Object(values) + }) +} diff --git a/crates/bds-core/src/engine/chat_tools.rs b/crates/bds-core/src/engine/chat_tools.rs index c5695b2..3c6cb32 100644 --- a/crates/bds-core/src/engine/chat_tools.rs +++ b/crates/bds-core/src/engine/chat_tools.rs @@ -9,7 +9,7 @@ use serde_json::{Value, json}; use crate::db::DbConnection as Connection; use crate::db::queries::{media, post, post_link, post_media, script, template}; use crate::db::schema::ai_models; -use crate::engine::{EngineError, EngineResult}; +use crate::engine::{EngineError, EngineResult, chat_surfaces}; use crate::util::frontmatter::read_post_file; pub fn model_supports_tools(conn: &Connection, model: &str) -> EngineResult { @@ -48,7 +48,7 @@ pub fn system_prompt(conn: &Connection, project_id: &str) -> EngineResult Vec { "value": {"type": "string"} }), ), + spec( + "render_card", + "Render an information card with optional allow-listed actions.", + json!({ + "title": {"type": "string"}, "subtitle": {"type": "string"}, "body": {"type": "string"}, + "actions": {"type": "array", "items": {"type": "object", "properties": { + "label": {"type": "string"}, "action": {"type": "string"}, "payload": {"type": "object"} + }, "required": ["label", "action"]}} + }), + ), + spec( + "render_chart", + "Render a native chart; heatmap and stacked-bar series use labelled segments.", + json!({ + "chartType": {"type": "string", "enum": ["bar", "stacked-bar", "line", "area", "pie", "donut", "heatmap"]}, + "chart_type": {"type": "string", "enum": ["bar", "stacked-bar", "line", "area", "pie", "donut", "heatmap"]}, + "title": {"type": "string"}, + "series": {"type": "array", "items": {"type": "object", "properties": { + "label": {"type": "string"}, "value": {"type": "number"}, + "segments": {"type": "array", "items": {"type": "object", "properties": { + "label": {"type": "string"}, "value": {"type": "number"} + }, "required": ["label", "value"]}} + }, "required": ["label"]}} + }), + ), + spec( + "render_form", + "Render a native form that submits its current values with an allow-listed action.", + json!({ + "title": {"type": "string"}, + "fields": {"type": "array", "items": {"type": "object", "properties": { + "key": {"type": "string"}, "label": {"type": "string"}, + "inputType": {"type": "string", "enum": ["text", "textarea", "select", "checkbox", "date", "number"]}, + "input_type": {"type": "string", "enum": ["text", "textarea", "select", "checkbox", "date", "number"]}, + "placeholder": {"type": "string"}, "defaultValue": {}, "default_value": {}, "required": {"type": "boolean"}, + "options": {"type": "array", "items": {"type": "object", "properties": { + "label": {"type": "string"}, "value": {"type": "string"} + }}} + }, "required": ["key", "label", "inputType"]}}, + "submitLabel": {"type": "string"}, "submit_label": {"type": "string"}, + "submitAction": {"type": "string"}, "submit_action": {"type": "string"} + }), + ), + spec( + "render_list", + "Render a native list.", + json!({"title": {"type": "string"}, "items": {"type": "array", "items": {"type": "string"}}}), + ), + spec( + "render_metric", + "Render a prominent metric.", + json!({"label": {"type": "string"}, "value": {"type": "string"}}), + ), + spec( + "render_mindmap", + "Render a native hierarchical mind map.", + json!({"title": {"type": "string"}, "nodes": {"type": "array", "items": {"type": "object", "properties": { + "id": {"type": "string"}, "label": {"type": "string"}, "children": {"type": "array", "items": {"type": "string"}} + }, "required": ["label"]}}}), + ), + spec( + "render_table", + "Render a native data table.", + json!({ + "title": {"type": "string"}, "columns": {"type": "array", "items": {"type": "string"}}, + "rows": {"type": "array", "items": {"type": "array", "items": {"type": "string"}}} + }), + ), + spec( + "render_tabs", + "Render switchable tabs containing nested native surfaces or text.", + json!({"title": {"type": "string"}, "tabs": {"type": "array", "items": {"type": "object", "properties": { + "label": {"type": "string"}, "content": {"type": "array", "items": {"type": "object"}} + }, "required": ["label", "content"]}}}), + ), ] } @@ -234,6 +309,10 @@ pub fn execute( Ok(json!({"success": true, "script": item})) } "navigate" => navigate(arguments), + name if chat_surfaces::RENDER_TOOL_NAMES.contains(&name) => { + Ok(chat_surfaces::render_tool_result(name, arguments) + .expect("render tool allow-list and result builder must stay in sync")) + } _ => Ok(json!({"success": false, "error": "unknown_tool", "name": name})), } } @@ -858,4 +937,42 @@ mod tests { ); assert!(matches!(invalid, Err(EngineError::Validation(_)))); } + + #[test] + fn structured_render_tools_are_advertised_and_return_inert_native_data() { + let specs = tool_specs(); + let names = specs + .iter() + .filter_map(|spec| spec.pointer("/function/name").and_then(Value::as_str)) + .collect::>(); + for name in chat_surfaces::RENDER_TOOL_NAMES { + assert!(names.contains(name), "missing tool schema for {name}"); + } + let chart = specs + .iter() + .find(|spec| { + spec.pointer("/function/name").and_then(Value::as_str) == Some("render_chart") + }) + .unwrap(); + assert!( + chart + .pointer("/function/parameters/properties/chartType") + .is_some() + ); + assert!( + chart + .pointer("/function/parameters/properties/chart_type") + .is_some() + ); + + let db = setup(); + let dir = tempfile::tempdir().unwrap(); + let raw = json!({"title": "", "body": "data"}); + let result = execute(db.conn(), dir.path(), "p1", "render_card", &raw).unwrap(); + assert_eq!(result["type"], "card"); + assert_eq!(result["title"], raw["title"]); + assert_eq!(result["body"], raw["body"]); + assert!(result.get("html").is_none()); + assert!(result.get("javascript").is_none()); + } } diff --git a/crates/bds-core/src/engine/mod.rs b/crates/bds-core/src/engine/mod.rs index 19e3f07..8586369 100644 --- a/crates/bds-core/src/engine/mod.rs +++ b/crates/bds-core/src/engine/mod.rs @@ -3,6 +3,7 @@ pub mod auto_translation; pub mod blogmark; pub mod calendar; pub mod chat; +pub mod chat_surfaces; mod chat_tools; pub mod cli_launcher; pub mod cli_sync; diff --git a/crates/bds-core/src/model/chat.rs b/crates/bds-core/src/model/chat.rs index 1f04724..383945a 100644 --- a/crates/bds-core/src/model/chat.rs +++ b/crates/bds-core/src/model/chat.rs @@ -57,6 +57,7 @@ pub struct ChatConversation { pub title: String, pub model: Option, pub copilot_session_id: Option, + pub surface_state: Option, pub created_at: i64, pub updated_at: i64, } @@ -68,6 +69,7 @@ pub struct NewChatConversation<'a> { pub title: &'a str, pub model: Option<&'a str>, pub copilot_session_id: Option<&'a str>, + pub surface_state: Option<&'a str>, pub created_at: i64, pub updated_at: i64, } diff --git a/crates/bds-core/tests/chat.rs b/crates/bds-core/tests/chat.rs index 2e09210..eef0b9b 100644 --- a/crates/bds-core/tests/chat.rs +++ b/crates/bds-core/tests/chat.rs @@ -397,6 +397,55 @@ fn tool_round_limit_stops_without_leaving_an_unpaired_tool_call() { assert!(messages[1].tool_calls.as_deref().unwrap().contains("stats")); } +#[test] +fn render_tool_events_include_stable_surface_identity_and_inert_arguments() { + let (_root, db, project_id, data_dir) = setup(); + let conversation = chat::create_conversation(db.conn(), Some("tool-model")).unwrap(); + let tool_frame = "data: {\"choices\":[{\"delta\":{\"tool_calls\":[{\"index\":0,\"id\":\"card\",\"function\":{\"name\":\"render_card\",\"arguments\":\"{\\\"title\\\":\\\"Live\\\",\\\"body\\\":\\\"\\\"}\"}}]}}]}\n\ndata: [DONE]\n\n"; + let (url, server) = serve(vec![ + MockResponse::sse(vec![tool_frame]), + MockResponse::sse(vec![ + "data: {\"choices\":[{\"delta\":{\"content\":\"Done\"}}]}\n\n", + "data: [DONE]\n\n", + ]), + ]); + let events = Arc::new(std::sync::Mutex::new(Vec::new())); + let captured = events.clone(); + chat::send_chat_message( + db.conn(), + &data_dir, + &project_id, + false, + &conversation.id, + "Show it", + options(url, move |event| captured.lock().unwrap().push(event)), + ) + .unwrap(); + server.join().unwrap(); + + let messages = chat::list_messages(db.conn(), &conversation.id).unwrap(); + let assistant_call = messages + .iter() + .find(|message| message.tool_calls.is_some()) + .unwrap(); + let events = events.lock().unwrap(); + let event = events + .iter() + .find_map(|event| match event { + chat::ChatEvent::ToolStarted { + name, + surface_id, + arguments, + .. + } if name == "render_card" => Some((surface_id, arguments)), + _ => None, + }) + .unwrap(); + assert_eq!(event.0, &format!("{}-surface-0", assistant_call.id)); + assert_eq!(event.1["title"], "Live"); + assert_eq!(event.1["body"], ""); +} + #[test] fn cancellation_before_mutating_tool_execution_records_pairs_without_mutation() { let (_root, db, project_id, data_dir) = setup(); diff --git a/crates/bds-core/tests/chat_surfaces.rs b/crates/bds-core/tests/chat_surfaces.rs new file mode 100644 index 0000000..4a4b046 --- /dev/null +++ b/crates/bds-core/tests/chat_surfaces.rs @@ -0,0 +1,241 @@ +use std::collections::{BTreeMap, BTreeSet}; + +use bds_core::db::Database; +use bds_core::engine::chat; +use bds_core::engine::chat_surfaces::{ + ChartType, ChatSurfaceState, FormInputType, SurfaceKind, build_render_surface, merge_form_data, + resolve_surface_action, +}; +use serde_json::{Value, json}; + +#[test] +fn every_render_tool_builds_the_expected_fixed_surface_type() { + let state = ChatSurfaceState::default(); + for (index, (tool, expected)) in [ + ("render_card", SurfaceKind::Card), + ("render_chart", SurfaceKind::Chart), + ("render_form", SurfaceKind::Form), + ("render_list", SurfaceKind::List), + ("render_metric", SurfaceKind::Metric), + ("render_mindmap", SurfaceKind::Mindmap), + ("render_table", SurfaceKind::Table), + ("render_tabs", SurfaceKind::Tabs), + ] + .into_iter() + .enumerate() + { + let surface = + build_render_surface(tool, &json!({}), format!("message-surface-{index}"), &state) + .unwrap(); + assert_eq!(surface.kind, expected); + } + assert!(build_render_surface("read_post", &json!({}), "x".into(), &state).is_none()); +} + +#[test] +fn charts_accept_all_types_aliases_and_coerce_invalid_numbers_to_zero() { + let state = ChatSurfaceState::default(); + for chart_type in [ + ChartType::Bar, + ChartType::StackedBar, + ChartType::Line, + ChartType::Area, + ChartType::Pie, + ChartType::Donut, + ChartType::Heatmap, + ] { + let surface = build_render_surface( + "render_chart", + &json!({ + "chartType": chart_type.as_str(), + "series": [{ + "label": "A", + "value": "invalid", + "segments": [{"label": "one", "value": {}}, {"label": "two", "value": "2.5"}] + }] + }), + "chart".into(), + &state, + ) + .unwrap(); + assert_eq!(surface.chart_type, Some(chart_type)); + assert_eq!(surface.series[0].value, 0.0); + assert_eq!(surface.series[0].segments[0].value, 0.0); + assert_eq!(surface.series[0].segments[1].value, 2.5); + } + + let defaulted = + build_render_surface("render_chart", &json!({}), "default".into(), &state).unwrap(); + assert_eq!(defaulted.chart_type, Some(ChartType::Bar)); + let legacy = build_render_surface( + "render_chart", + &json!({"chart_type": "pie"}), + "legacy".into(), + &state, + ) + .unwrap(); + assert_eq!(legacy.chart_type, Some(ChartType::Pie)); +} + +#[test] +fn forms_support_every_input_type_and_restore_current_values() { + let mut state = ChatSurfaceState::default(); + state.surface_data.insert( + "form".into(), + BTreeMap::from([ + ("title".into(), json!("Restored")), + ("enabled".into(), json!(true)), + ]), + ); + let surface = build_render_surface( + "render_form", + &json!({ + "fields": [ + {"key": "title", "label": "Title", "inputType": "text", "defaultValue": "Default"}, + {"key": "body", "label": "Body", "inputType": "textarea"}, + {"key": "kind", "label": "Kind", "inputType": "select", "options": [{"label": "Post", "value": "post"}]}, + {"key": "enabled", "label": "Enabled", "inputType": "checkbox"}, + {"key": "date", "label": "Date", "inputType": "date"}, + {"key": "count", "label": "Count", "inputType": "number"} + ], + "submitAction": "openPost" + }), + "form".into(), + &state, + ) + .unwrap(); + assert_eq!( + surface + .fields + .iter() + .map(|field| field.input_type) + .collect::>(), + vec![ + FormInputType::Text, + FormInputType::Textarea, + FormInputType::Select, + FormInputType::Checkbox, + FormInputType::Date, + FormInputType::Number, + ] + ); + assert_eq!(surface.fields[0].value, json!("Restored")); + assert_eq!(surface.fields[3].value, json!(true)); + + let payload = merge_form_data(json!({"postId": "post-1"}), "form", &state); + assert_eq!(payload["postId"], "post-1"); + assert_eq!(payload["formData"]["title"], "Restored"); + assert_eq!(payload["formData"]["enabled"], true); +} + +#[test] +fn tabs_restore_selection_and_nested_unknown_content_is_safe_data() { + let state = ChatSurfaceState { + surface_tabs: BTreeMap::from([("tabs".into(), 1)]), + ..Default::default() + }; + let surface = build_render_surface( + "render_tabs", + &json!({"tabs": [ + {"label": "Known", "content": [{"type": "text", "body": ""}]}, + {"label": "Fallback", "content": [{"type": "future", "html": "not markup"}, 42]} + ]}), + "tabs".into(), + &state, + ) + .unwrap(); + assert_eq!(surface.selected_index, Some(1)); + assert_eq!(surface.tabs[0].content[0].kind, SurfaceKind::Text); + assert_eq!( + surface.tabs[0].content[0].body.as_deref(), + Some("") + ); + assert_eq!(surface.tabs[1].content[0].kind, SurfaceKind::Json); + assert_eq!(surface.tabs[1].content[1].kind, SurfaceKind::Text); +} + +#[test] +fn unknown_render_tool_is_inspectable_json_but_non_render_tools_are_not_surfaces() { + let state = ChatSurfaceState::default(); + let raw = json!({"html": "", "answer": 42}); + let surface = build_render_surface("render_future", &raw, "future-0".into(), &state).unwrap(); + assert_eq!(surface.kind, SurfaceKind::Json); + assert_eq!(surface.raw.as_ref(), Some(&raw)); + assert!(build_render_surface("run_script", &raw, "bad-0".into(), &state).is_none()); +} + +#[test] +fn surface_state_persists_and_reopens_against_stable_ids() { + let db = Database::open_in_memory().unwrap(); + db.migrate().unwrap(); + let conversation = chat::create_conversation(db.conn(), Some("model")).unwrap(); + let expected = ChatSurfaceState { + surface_data: BTreeMap::from([( + "17-surface-0".into(), + BTreeMap::from([("query".into(), json!("hello"))]), + )]), + surface_tabs: BTreeMap::from([("17-surface-1".into(), 2)]), + dismissed_surfaces: BTreeSet::from(["17-surface-2".into()]), + }; + chat::put_surface_state(db.conn(), &conversation.id, &expected).unwrap(); + assert_eq!( + chat::get_surface_state(db.conn(), &conversation.id).unwrap(), + expected + ); + assert_eq!( + chat::get_surface_state(db.conn(), "missing").unwrap(), + ChatSurfaceState::default() + ); +} + +#[test] +fn assistant_action_allow_list_accepts_aliases_and_refuses_bad_payloads() { + for (action, payload, destination, entity_id) in [ + ( + "openPost", + json!({"postId": "post-1"}), + "posts", + Some("post-1"), + ), + ( + "open_media", + json!({"media_id": "media-1"}), + "media", + Some("media-1"), + ), + ("openSettings", json!({}), "settings", None), + ( + "open_chat", + json!({"conversationId": "chat-1"}), + "chat", + Some("chat-1"), + ), + ( + "switchView", + json!({"view": "templates"}), + "templates", + None, + ), + ("set_view", json!({"view": "scripts"}), "scripts", None), + ("toggleSidebar", json!({}), "toggle_sidebar", None), + ("toggle_panel", json!({}), "toggle_panel", None), + ( + "toggleAssistantSidebar", + json!({}), + "toggle_assistant_sidebar", + None, + ), + ] { + let navigation = resolve_surface_action(action, &payload).unwrap(); + assert_eq!(navigation.destination, destination); + assert_eq!(navigation.entity_id.as_deref(), entity_id); + } + for (action, payload) in [ + ("runJavaScript", json!({"code": "alert(1)"})), + ("openPost", json!({"postId": ""})), + ("switchView", json!({"view": "root-shell"})), + ("openMedia", Value::String("media-1".into())), + ] { + assert!(resolve_surface_action(action, &payload).is_err()); + } +} diff --git a/crates/bds-ui/src/app.rs b/crates/bds-ui/src/app.rs index 6110ce5..2b138b3 100644 --- a/crates/bds-ui/src/app.rs +++ b/crates/bds-ui/src/app.rs @@ -334,6 +334,26 @@ pub enum Message { ChatSend, ChatCancel, ChatLinkClicked(String), + ChatSurfaceFieldChanged { + surface_id: String, + field: String, + value: serde_json::Value, + }, + ChatSurfaceTextareaAction { + surface_id: String, + field: String, + action: iced::widget::text_editor::Action, + }, + ChatSurfaceTabSelected { + surface_id: String, + index: usize, + }, + ChatSurfaceDismissed(String), + ChatSurfaceAction { + surface_id: String, + action: String, + payload: serde_json::Value, + }, ChatFinished { conversation_id: String, result: Result, @@ -1317,6 +1337,103 @@ impl BdsApp { } Task::none() } + Message::ChatSurfaceFieldChanged { + surface_id, + field, + value, + } => { + if let Some(state) = self.active_chat_state_mut() { + state + .surface_state + .surface_data + .entry(surface_id) + .or_default() + .insert(field, value); + state.surface_state_dirty_since = Some(std::time::Instant::now()); + state.rebuild_surfaces(); + } + Task::none() + } + Message::ChatSurfaceTextareaAction { + surface_id, + field, + action, + } => { + if let Some(state) = self.active_chat_state_mut() + && let Some(content) = state + .surface_textareas + .get_mut(&crate::views::chat_view::textarea_key(&surface_id, &field)) + { + content.perform(action); + let value = content.text(); + state + .surface_state + .surface_data + .entry(surface_id) + .or_default() + .insert(field, value.into()); + state.surface_state_dirty_since = Some(std::time::Instant::now()); + } + Task::none() + } + Message::ChatSurfaceTabSelected { surface_id, index } => { + let Some(conversation_id) = self.active_chat_id().map(str::to_string) else { + return Task::none(); + }; + if let Some(state) = self.chat_editors.get_mut(&conversation_id) { + state.surface_state.surface_tabs.insert(surface_id, index); + state.rebuild_surfaces(); + } + self.persist_chat_surface_state(&conversation_id); + Task::none() + } + Message::ChatSurfaceDismissed(surface_id) => { + let Some(conversation_id) = self.active_chat_id().map(str::to_string) else { + return Task::none(); + }; + if let Some(state) = self.chat_editors.get_mut(&conversation_id) { + state.surface_state.dismissed_surfaces.insert(surface_id); + state.rebuild_surfaces(); + } + self.persist_chat_surface_state(&conversation_id); + Task::none() + } + Message::ChatSurfaceAction { + surface_id, + action, + payload, + } => { + let result = self + .active_chat_id() + .and_then(|id| self.chat_editors.get(id)) + .map(|state| { + let payload = engine::chat_surfaces::merge_form_data( + payload, + &surface_id, + &state.surface_state, + ); + engine::chat_surfaces::resolve_surface_action(&action, &payload) + }); + match result { + Some(Ok(navigation)) => self.dispatch_chat_navigation( + &navigation.destination, + navigation.entity_id.as_deref(), + ), + Some(Err(reason)) => { + let message = tw( + self.ui_locale, + "chat.surface.actionRefused", + &[("reason", &reason)], + ); + if let Some(state) = self.active_chat_state_mut() { + state.error = Some(message.clone()); + } + self.notify(ToastLevel::Error, &message); + } + None => {} + } + Task::none() + } Message::ChatFinished { conversation_id, result, @@ -2011,6 +2128,7 @@ impl BdsApp { self.task_manager.evict_expired(); self.refresh_task_snapshots(); self.process_chat_events(); + self.persist_due_chat_surface_state(); if !self.search_index_rebuild_running { self.auto_save_due_post_editors(); } @@ -2654,6 +2772,48 @@ impl BdsApp { .unwrap_or_default(); } + fn persist_due_chat_surface_state(&mut self) { + let due = self + .chat_editors + .iter() + .filter_map(|(id, state)| { + state.surface_state_dirty_since.and_then(|since| { + (since.elapsed() >= std::time::Duration::from_millis(500)).then(|| id.clone()) + }) + }) + .collect::>(); + for id in due { + self.persist_chat_surface_state(&id); + } + } + + fn persist_chat_surface_state(&mut self, conversation_id: &str) { + let Some(surface_state) = self + .chat_editors + .get(conversation_id) + .map(|state| state.surface_state.clone()) + else { + return; + }; + let result = self + .db + .as_ref() + .ok_or_else(|| "database unavailable".to_string()) + .and_then(|db| { + engine::chat::put_surface_state(db.conn(), conversation_id, &surface_state) + .map_err(|error| error.to_string()) + }); + match result { + Ok(()) => { + if let Some(state) = self.chat_editors.get_mut(conversation_id) { + state.conversation.surface_state = serde_json::to_string(&surface_state).ok(); + state.surface_state_dirty_since = None; + } + } + Err(error) => self.notify(ToastLevel::Error, &error), + } + } + fn chat_model_options(&self) -> Vec { let mut models = self .db @@ -2826,9 +2986,12 @@ impl BdsApp { engine::chat::ChatEvent::ToolStarted { conversation_id, name, + surface_id, + arguments, } => { if let Some(state) = self.chat_editors.get_mut(&conversation_id) { - state.active_tool = Some(name); + state.active_tool = Some(name.clone()); + state.add_streaming_surface(&name, &arguments, surface_id); } } engine::chat::ChatEvent::ToolFinished { @@ -7800,6 +7963,7 @@ mod tests { use crate::state::ToastLevel; use crate::state::sidebar_filter::{MediaFilter, PostFilter}; use crate::state::tabs::{Tab, TabType}; + use crate::views::chat_view::ChatEditorState; use crate::views::media_editor::{MediaEditorMsg, MediaEditorState}; use crate::views::modal; use crate::views::post_editor::{PostEditorMsg, PostEditorState}; @@ -7811,9 +7975,9 @@ mod tests { use bds_core::db::queries::project::insert_project; use bds_core::engine::generation::GenerationReport; use bds_core::engine::task::{TaskStatus, TaskStatus::*}; - use bds_core::engine::{ai, media, post, script, template, wordpress_import}; + use bds_core::engine::{ai, chat, media, post, script, template, wordpress_import}; use bds_core::i18n::UiLocale; - use bds_core::model::{Project, ScriptKind, TemplateKind}; + use bds_core::model::{ChatRole, Project, ScriptKind, TemplateKind}; use chrono::{Datelike, TimeZone}; use std::io::{Read, Write}; use std::net::TcpListener; @@ -9725,4 +9889,116 @@ mod tests { .is_empty() ); } + + #[test] + fn chat_surfaces_stream_persist_reopen_and_refuse_unknown_actions() { + let (db, project, tmp) = setup(); + let conversation = + chat::create_conversation_titled(db.conn(), Some("test-model"), "Surfaces").unwrap(); + let tool_calls = serde_json::json!([ + { + "id": "form-call", + "type": "function", + "function": { + "name": "render_form", + "arguments": serde_json::json!({ + "title": "Preferences", + "fields": [{"key": "topic", "label": "Topic", "inputType": "text"}], + "submitAction": "switchView" + }).to_string() + } + }, + { + "id": "tabs-call", + "type": "function", + "function": { + "name": "render_tabs", + "arguments": serde_json::json!({ + "tabs": [ + {"label": "One", "content": [{"type": "text", "text": "First"}]}, + {"label": "Two", "content": [{"type": "text", "text": "Second"}]} + ] + }).to_string() + } + } + ]) + .to_string(); + let message = chat::insert_message( + db.conn(), + &conversation.id, + ChatRole::Assistant, + Some("Structured answer"), + None, + Some(&tool_calls), + bds_core::engine::ai::TokenUsage::default(), + ) + .unwrap(); + let mut app = make_app(db, project, &tmp); + let tab = Tab { + id: conversation.id.clone(), + tab_type: TabType::Chat, + title: conversation.title.clone(), + is_transient: false, + is_dirty: false, + }; + let _ = app.update(Message::OpenTab(tab)); + let form_id = format!("{}-surface-0", message.id); + let tabs_id = format!("{}-surface-1", message.id); + { + let state = app.chat_editors.get_mut(&conversation.id).unwrap(); + assert_eq!(state.message_surfaces[&message.id].len(), 2); + state.streaming = true; + state.add_streaming_surface( + "render_card", + &serde_json::json!({"title": "Later"}), + "later-message-surface-0".into(), + ); + assert_eq!(state.message_surfaces[&message.id].len(), 2); + assert_eq!(state.streaming_surfaces.len(), 1); + } + + let _ = app.update(Message::ChatSurfaceFieldChanged { + surface_id: form_id.clone(), + field: "topic".into(), + value: "Rust".into(), + }); + app.chat_editors + .get_mut(&conversation.id) + .unwrap() + .surface_state_dirty_since = + Some(std::time::Instant::now() - std::time::Duration::from_millis(501)); + let _ = app.update(Message::TaskTick); + let _ = app.update(Message::ChatSurfaceTabSelected { + surface_id: tabs_id.clone(), + index: 1, + }); + let _ = app.update(Message::ChatSurfaceDismissed(form_id.clone())); + + let reopened_conversation = + chat::get_conversation(app.db.as_ref().unwrap().conn(), &conversation.id).unwrap(); + let reopened = ChatEditorState::new( + reopened_conversation, + chat::list_messages(app.db.as_ref().unwrap().conn(), &conversation.id).unwrap(), + vec![], + ); + assert_eq!( + reopened.surface_state.surface_data[&form_id]["topic"], + "Rust" + ); + assert_eq!(reopened.surface_state.surface_tabs[&tabs_id], 1); + assert!(reopened.surface_state.dismissed_surfaces.contains(&form_id)); + assert_eq!(reopened.message_surfaces[&message.id].len(), 1); + + let _ = app.update(Message::ChatSurfaceAction { + surface_id: form_id, + action: "runJavaScript".into(), + payload: serde_json::json!({"script": "alert(1)"}), + }); + assert!( + app.chat_editors[&conversation.id] + .error + .as_deref() + .is_some_and(|error| error.contains("runJavaScript")) + ); + } } diff --git a/crates/bds-ui/src/views/chat_surfaces.rs b/crates/bds-ui/src/views/chat_surfaces.rs new file mode 100644 index 0000000..7439f1f --- /dev/null +++ b/crates/bds-ui/src/views/chat_surfaces.rs @@ -0,0 +1,926 @@ +use std::collections::{HashMap, HashSet}; +use std::f32::consts::{FRAC_PI_2, TAU}; + +use bds_core::engine::chat_surfaces::{ + ChartSeries, ChartType, ChatSurfaceState, FormInputType, InlineSurface, MindmapNode, + SurfaceKind, +}; +use bds_core::i18n::UiLocale; +use iced::widget::canvas::{self, Path, Stroke, path}; +use iced::widget::{ + Space, button, checkbox, column, container, row, scrollable, text, text_editor, +}; +use iced::{ + Alignment, Color, Element, Length, Point, Radians, Rectangle, Renderer, Size, Theme, mouse, +}; + +use crate::app::Message; +use crate::components::inputs; +use crate::i18n::t; +use crate::views::chat_view::textarea_key; + +const CHART_COLORS: [Color; 7] = [ + rgb8(0x4E, 0xA1, 0xE0), + rgb8(0x7B, 0xC9, 0x6F), + rgb8(0xF0, 0xB4, 0x4D), + rgb8(0xD9, 0x78, 0xAE), + rgb8(0x9B, 0x8A, 0xE6), + rgb8(0x54, 0xC6, 0xC0), + rgb8(0xE3, 0x73, 0x63), +]; + +const fn rgb8(red: u8, green: u8, blue: u8) -> Color { + Color { + r: red as f32 / 255.0, + g: green as f32 / 255.0, + b: blue as f32 / 255.0, + a: 1.0, + } +} + +pub fn view<'a>( + surface: &'a InlineSurface, + state: &'a ChatSurfaceState, + textareas: &'a HashMap, + locale: UiLocale, +) -> Element<'a, Message> { + surface_view(surface, state, textareas, locale, true) +} + +fn surface_view<'a>( + surface: &'a InlineSurface, + state: &'a ChatSurfaceState, + textareas: &'a HashMap, + locale: UiLocale, + dismissible: bool, +) -> Element<'a, Message> { + let has_title = surface.title.is_some(); + let title = surface.title.clone().unwrap_or_else(|| { + t( + locale, + &format!("chat.surface.type.{}", surface.kind.as_str()), + ) + }); + let mut header = row![text(title).size(14), Space::with_width(Length::Fill),] + .spacing(8) + .align_y(Alignment::Center); + if has_title { + header = header.push( + text(t( + locale, + &format!("chat.surface.type.{}", surface.kind.as_str()), + )) + .size(10) + .color(inputs::SECTION_COLOR), + ); + } + if dismissible { + header = header.push( + button(text(t(locale, "chat.surface.dismiss"))) + .on_press(Message::ChatSurfaceDismissed(surface.id.clone())) + .padding([4, 8]) + .style(inputs::secondary_button), + ); + } + + let content = surface_content(surface, state, textareas, locale); + inputs::card(column![header, content].spacing(10)).into() +} + +fn surface_content<'a>( + surface: &'a InlineSurface, + state: &'a ChatSurfaceState, + textareas: &'a HashMap, + locale: UiLocale, +) -> Element<'a, Message> { + match surface.kind { + SurfaceKind::Card => { + let mut children = Vec::new(); + if let Some(subtitle) = &surface.subtitle { + children.push( + text(subtitle.clone()) + .size(12) + .color(inputs::SECTION_COLOR) + .into(), + ); + } + if let Some(body) = &surface.body { + children.push(text(body.clone()).size(13).into()); + } + if !surface.actions.is_empty() { + let actions = surface + .actions + .iter() + .fold(row![].spacing(8), |actions, item| { + actions.push( + button(text(item.label.clone())) + .on_press(Message::ChatSurfaceAction { + surface_id: surface.id.clone(), + action: item.action.clone(), + payload: item.payload.clone(), + }) + .padding([7, 10]) + .style(inputs::secondary_button), + ) + }); + children.push(actions.into()); + } + empty_or_column(children, locale) + } + SurfaceKind::Chart => chart(surface), + SurfaceKind::Form => form(surface, textareas, locale), + SurfaceKind::List => { + let children = surface + .items + .iter() + .map(|item| text(format!("• {item}")).size(13).into()) + .collect(); + empty_or_column(children, locale) + } + SurfaceKind::Metric => column![ + text(surface.label.clone().unwrap_or_default()) + .size(12) + .color(inputs::SECTION_COLOR), + text(surface.value.clone().unwrap_or_default()).size(28), + ] + .spacing(4) + .into(), + SurfaceKind::Mindmap => mindmap(surface, locale), + SurfaceKind::Table => table(surface, locale), + SurfaceKind::Tabs => tabs(surface, state, textareas, locale), + SurfaceKind::Text => text(surface.body.clone().unwrap_or_default()) + .size(13) + .into(), + SurfaceKind::Json => text( + surface + .raw + .as_ref() + .and_then(|value| serde_json::to_string_pretty(value).ok()) + .unwrap_or_default(), + ) + .size(12) + .shaping(iced::widget::text::Shaping::Advanced) + .into(), + } +} + +fn empty_or_column<'a>( + mut children: Vec>, + locale: UiLocale, +) -> Element<'a, Message> { + if children.is_empty() { + children.push( + text(t(locale, "chat.surface.empty")) + .size(12) + .color(inputs::SECTION_COLOR) + .into(), + ); + } + iced::widget::Column::with_children(children) + .spacing(7) + .into() +} + +fn form<'a>( + surface: &'a InlineSurface, + textareas: &'a HashMap, + locale: UiLocale, +) -> Element<'a, Message> { + let mut children: Vec> = Vec::new(); + for field in &surface.fields { + let label = if field.required { + format!("{} *", field.label) + } else { + field.label.clone() + }; + let surface_id = surface.id.clone(); + let key = field.key.clone(); + let control: Element<'a, Message> = match field.input_type { + FormInputType::Checkbox => checkbox(label, field.value.as_bool().unwrap_or(false)) + .on_toggle(move |value| Message::ChatSurfaceFieldChanged { + surface_id: surface_id.clone(), + field: key.clone(), + value: value.into(), + }) + .size(16) + .text_size(13) + .into(), + FormInputType::Select => { + let selected = field.options.iter().find(|option| { + field + .value + .as_str() + .is_some_and(|value| value == option.value) + }); + inputs::labeled_select(&label, &field.options, selected, move |option| { + Message::ChatSurfaceFieldChanged { + surface_id: surface_id.clone(), + field: key.clone(), + value: option.value.into(), + } + }) + } + FormInputType::Textarea => { + let content_key = textarea_key(&surface.id, &field.key); + if let Some(content) = textareas.get(&content_key) { + column![ + text(label).size(12).color(inputs::LABEL_COLOR), + text_editor(content) + .placeholder(field.placeholder.clone().unwrap_or_default()) + .on_action(move |action| Message::ChatSurfaceTextareaAction { + surface_id: surface_id.clone(), + field: key.clone(), + action, + }) + .height(Length::Fixed(96.0)) + .style(inputs::text_editor_style), + ] + .spacing(6) + .into() + } else { + text(t(locale, "chat.surface.empty")).into() + } + } + FormInputType::Text | FormInputType::Date | FormInputType::Number => { + let value = match &field.value { + serde_json::Value::String(value) => value.clone(), + serde_json::Value::Number(value) => value.to_string(), + _ => String::new(), + }; + let placeholder = field.placeholder.clone().unwrap_or_else(|| { + t( + locale, + match field.input_type { + FormInputType::Date => "chat.surface.form.datePlaceholder", + FormInputType::Number => "chat.surface.form.numberPlaceholder", + _ => "chat.surface.form.textPlaceholder", + }, + ) + }); + inputs::labeled_input(&label, &placeholder, &value, move |value| { + let value = if field.input_type == FormInputType::Number { + value + .parse::() + .map(serde_json::Value::Number) + .unwrap_or_else(|_| value.into()) + } else { + value.into() + }; + Message::ChatSurfaceFieldChanged { + surface_id: surface_id.clone(), + field: key.clone(), + value, + } + }) + } + }; + children.push(control); + } + if let Some(action) = &surface.submit_action { + children.push( + button(text( + surface + .submit_label + .clone() + .unwrap_or_else(|| t(locale, "chat.surface.form.submit")), + )) + .on_press(Message::ChatSurfaceAction { + surface_id: surface.id.clone(), + action: action.clone(), + payload: serde_json::json!({}), + }) + .padding([8, 12]) + .style(inputs::primary_button) + .into(), + ); + } + empty_or_column(children, locale) +} + +fn table<'a>(surface: &'a InlineSurface, locale: UiLocale) -> Element<'a, Message> { + if surface.columns.is_empty() && surface.rows.is_empty() { + return text(t(locale, "chat.surface.empty")) + .size(12) + .color(inputs::SECTION_COLOR) + .into(); + } + let header = iced::widget::Row::with_children( + surface + .columns + .iter() + .map(|value| { + container(text(value.clone()).size(12)) + .width(Length::Fixed(140.0)) + .into() + }) + .collect::>(), + ) + .spacing(8); + let mut body: Vec> = vec![header.into()]; + for values in &surface.rows { + body.push( + iced::widget::Row::with_children( + values + .iter() + .map(|value| { + container(text(value.clone()).size(12)) + .width(Length::Fixed(140.0)) + .into() + }) + .collect::>(), + ) + .spacing(8) + .into(), + ); + } + scrollable(iced::widget::Column::with_children(body).spacing(6)) + .direction(scrollable::Direction::Horizontal( + scrollable::Scrollbar::default(), + )) + .into() +} + +fn mindmap<'a>(surface: &'a InlineSurface, locale: UiLocale) -> Element<'a, Message> { + let rows = mindmap_rows(&surface.nodes); + if rows.is_empty() { + return text(t(locale, "chat.surface.empty")) + .size(12) + .color(inputs::SECTION_COLOR) + .into(); + } + iced::widget::Column::with_children( + rows.into_iter() + .map(|(depth, label)| { + row![ + Space::with_width(Length::Fixed(depth as f32 * 18.0)), + text(if depth == 0 { "◆" } else { "└" }) + .size(11) + .color(inputs::SECTION_COLOR), + text(label).size(13), + ] + .spacing(6) + .into() + }) + .collect::>(), + ) + .spacing(6) + .into() +} + +fn mindmap_rows(nodes: &[MindmapNode]) -> Vec<(usize, String)> { + let by_id = nodes + .iter() + .filter_map(|node| node.id.as_deref().map(|id| (id, node))) + .collect::>(); + let children = nodes + .iter() + .flat_map(|node| node.children.iter().map(String::as_str)) + .collect::>(); + let roots = nodes + .iter() + .filter(|node| node.id.as_deref().is_none_or(|id| !children.contains(id))) + .collect::>(); + let mut rows = Vec::new(); + let mut visited = HashSet::new(); + fn walk( + node: &MindmapNode, + depth: usize, + by_id: &HashMap<&str, &MindmapNode>, + visited: &mut HashSet, + rows: &mut Vec<(usize, String)>, + ) { + if let Some(id) = &node.id + && !visited.insert(id.clone()) + { + return; + } + rows.push((depth, node.label.clone())); + for child in &node.children { + if let Some(node) = by_id.get(child.as_str()) { + walk(node, depth + 1, by_id, visited, rows); + } + } + } + for root in roots { + walk(root, 0, &by_id, &mut visited, &mut rows); + } + for node in nodes { + let missing = node.id.as_ref().is_some_and(|id| !visited.contains(id)); + if missing { + walk(node, 0, &by_id, &mut visited, &mut rows); + } + } + rows +} + +fn tabs<'a>( + surface: &'a InlineSurface, + state: &'a ChatSurfaceState, + textareas: &'a HashMap, + locale: UiLocale, +) -> Element<'a, Message> { + if surface.tabs.is_empty() { + return text(t(locale, "chat.surface.empty")).into(); + } + let selected = state + .surface_tabs + .get(&surface.id) + .copied() + .or(surface.selected_index) + .unwrap_or(0) + .min(surface.tabs.len() - 1); + let controls = + surface + .tabs + .iter() + .enumerate() + .fold(row![].spacing(6), |controls, (index, tab)| { + controls.push( + button(text(tab.label.clone())) + .on_press(Message::ChatSurfaceTabSelected { + surface_id: surface.id.clone(), + index, + }) + .padding([6, 10]) + .style(if index == selected { + inputs::primary_button + } else { + inputs::secondary_button + }), + ) + }); + let content = iced::widget::Column::with_children( + surface.tabs[selected] + .content + .iter() + .map(|child| surface_view(child, state, textareas, locale, false)) + .collect::>(), + ) + .spacing(8); + column![controls, content].spacing(8).into() +} + +fn chart<'a>(surface: &'a InlineSurface) -> Element<'a, Message> { + let chart_type = surface.chart_type.unwrap_or(ChartType::Bar); + let canvas = canvas::Canvas::new(NativeChart { + chart_type, + series: surface.series.clone(), + }) + .width(Length::Fill) + .height(Length::Fixed(180.0)); + let mut legend_items = Vec::new(); + if chart_type == ChartType::Heatmap { + let (columns, rows) = heatmap_layout(&surface.series); + for (row_label, values) in rows { + for (column, value) in columns.iter().zip(values) { + legend_items.push(format!( + "{row_label} / {column} — {}", + display_number(value) + )); + } + } + } else if chart_type == ChartType::StackedBar { + for item in &surface.series { + if item.segments.is_empty() { + legend_items.push(format!("{} — {}", item.label, display_number(item.value))); + } else { + for segment in &item.segments { + legend_items.push(format!( + "{} / {} — {}", + item.label, + segment.label, + display_number(segment.value) + )); + } + } + } + } else { + legend_items.extend(surface.series.iter().map(|item| { + format!( + "{} — {}", + item.label, + display_number(chart_series_total(chart_type, item)) + ) + })); + } + let legend = legend_items.into_iter().enumerate().fold( + iced::widget::Column::new().spacing(4), + |legend, (index, item)| { + legend.push( + text(item) + .size(11) + .color(CHART_COLORS[index % CHART_COLORS.len()]), + ) + }, + ); + column![canvas, legend].spacing(8).into() +} + +fn display_number(value: f64) -> String { + if value.fract() == 0.0 { + format!("{value:.0}") + } else { + format!("{value:.2}") + } +} + +fn chart_series_total(chart_type: ChartType, series: &ChartSeries) -> f64 { + if chart_type == ChartType::StackedBar && !series.segments.is_empty() { + series.segments.iter().map(|segment| segment.value).sum() + } else { + series.value + } +} + +#[derive(Debug, Clone)] +struct NativeChart { + chart_type: ChartType, + series: Vec, +} + +impl canvas::Program for NativeChart { + type State = (); + + fn draw( + &self, + _state: &Self::State, + renderer: &Renderer, + _theme: &Theme, + bounds: Rectangle, + _cursor: mouse::Cursor, + ) -> Vec { + let mut frame = canvas::Frame::new(renderer, bounds.size()); + match self.chart_type { + ChartType::Bar => draw_bars(&mut frame, &self.series, false), + ChartType::StackedBar => draw_bars(&mut frame, &self.series, true), + ChartType::Line => draw_line(&mut frame, &self.series, false), + ChartType::Area => draw_line(&mut frame, &self.series, true), + ChartType::Pie => draw_pie(&mut frame, &self.series, false), + ChartType::Donut => draw_pie(&mut frame, &self.series, true), + ChartType::Heatmap => draw_heatmap(&mut frame, &self.series), + } + vec![frame.into_geometry()] + } +} + +fn positive_max(series: &[ChartSeries], stacked: bool) -> f32 { + series + .iter() + .map(|item| { + if stacked && !item.segments.is_empty() { + item.segments.iter().map(|part| part.value.max(0.0)).sum() + } else { + item.value.max(0.0) + } + }) + .fold(0.0_f64, f64::max) + .max(1.0) as f32 +} + +fn draw_bars(frame: &mut canvas::Frame, series: &[ChartSeries], stacked: bool) { + if series.is_empty() { + return; + } + let size = frame.size(); + let gap = 8.0; + let width = ((size.width - gap * (series.len() as f32 + 1.0)) / series.len() as f32).max(3.0); + let max = positive_max(series, stacked); + for (index, item) in series.iter().enumerate() { + let x = gap + index as f32 * (width + gap); + if stacked && !item.segments.is_empty() { + let mut y = size.height - 8.0; + for (part_index, part) in item.segments.iter().enumerate() { + let height = (part.value.max(0.0) as f32 / max) * (size.height - 16.0); + y -= height; + frame.fill( + &Path::rectangle(Point::new(x, y), Size::new(width, height)), + CHART_COLORS[part_index % CHART_COLORS.len()], + ); + } + } else { + let height = (item.value.max(0.0) as f32 / max) * (size.height - 16.0); + frame.fill( + &Path::rectangle( + Point::new(x, size.height - height - 8.0), + Size::new(width, height), + ), + CHART_COLORS[index % CHART_COLORS.len()], + ); + } + } +} + +fn chart_points(size: Size, series: &[ChartSeries]) -> Vec { + let max = positive_max(series, false); + let denominator = series.len().saturating_sub(1).max(1) as f32; + series + .iter() + .enumerate() + .map(|(index, item)| { + Point::new( + 8.0 + index as f32 / denominator * (size.width - 16.0), + size.height - 8.0 - item.value.max(0.0) as f32 / max * (size.height - 16.0), + ) + }) + .collect() +} + +fn draw_line(frame: &mut canvas::Frame, series: &[ChartSeries], area: bool) { + let points = chart_points(frame.size(), series); + let Some(first) = points.first().copied() else { + return; + }; + let line = Path::new(|builder| { + builder.move_to(first); + for point in points.iter().skip(1) { + builder.line_to(*point); + } + }); + if area { + let size = frame.size(); + let fill = Path::new(|builder| { + builder.move_to(Point::new(first.x, size.height - 8.0)); + builder.line_to(first); + for point in points.iter().skip(1) { + builder.line_to(*point); + } + builder.line_to(Point::new( + points.last().map_or(first.x, |point| point.x), + size.height - 8.0, + )); + builder.close(); + }); + frame.fill(&fill, CHART_COLORS[0].scale_alpha(0.28)); + } + frame.stroke( + &line, + Stroke::default() + .with_color(CHART_COLORS[0]) + .with_width(2.0) + .with_line_cap(canvas::LineCap::Round), + ); + for point in points { + frame.fill(&Path::circle(point, 3.0), CHART_COLORS[0]); + } +} + +fn draw_pie(frame: &mut canvas::Frame, series: &[ChartSeries], donut: bool) { + let values = series + .iter() + .map(|item| item.value.max(0.0) as f32) + .collect::>(); + let total = values.iter().sum::(); + if total <= 0.0 { + return; + } + let center = frame.center(); + let radius = (frame.width().min(frame.height()) / 2.0 - 8.0).max(1.0); + let mut start = -FRAC_PI_2; + for (index, value) in values.into_iter().enumerate() { + let end = start + value / total * TAU; + let wedge = Path::new(|builder| { + builder.move_to(center); + builder.line_to(Point::new( + center.x + radius * start.cos(), + center.y + radius * start.sin(), + )); + builder.arc(path::Arc { + center, + radius, + start_angle: Radians(start), + end_angle: Radians(end), + }); + builder.close(); + }); + frame.fill(&wedge, CHART_COLORS[index % CHART_COLORS.len()]); + start = end; + } + if donut { + frame.fill( + &Path::circle(center, radius * 0.54), + Color::from_rgb8(0x25, 0x25, 0x26), + ); + } +} + +fn draw_heatmap(frame: &mut canvas::Frame, series: &[ChartSeries]) { + let (columns, rows) = heatmap_layout(series); + if columns.is_empty() || rows.is_empty() { + return; + } + let cell_width = frame.width() / columns.len() as f32; + let cell_height = frame.height() / rows.len() as f32; + let max = rows + .iter() + .flat_map(|(_, values)| values) + .copied() + .fold(0.0_f64, f64::max) + .max(1.0); + for (row_index, (_, values)) in rows.iter().enumerate() { + for (column_index, value) in values.iter().enumerate() { + let alpha = if *value <= 0.0 { + 0.08 + } else { + 0.18 + 0.82 * *value as f32 / max as f32 + }; + frame.fill( + &Path::rectangle( + Point::new( + column_index as f32 * cell_width + 2.0, + row_index as f32 * cell_height + 2.0, + ), + Size::new(cell_width - 4.0, cell_height - 4.0), + ), + CHART_COLORS[0].scale_alpha(alpha), + ); + } + } +} + +fn heatmap_layout(series: &[ChartSeries]) -> (Vec, Vec<(String, Vec)>) { + let mut columns = Vec::new(); + for item in series { + for segment in &item.segments { + if !columns.contains(&segment.label) { + columns.push(segment.label.clone()); + } + } + } + let rows = series + .iter() + .filter(|item| !item.segments.is_empty()) + .map(|item| { + let values = columns + .iter() + .map(|column| { + item.segments + .iter() + .find(|segment| segment.label == *column) + .map_or(0.0, |segment| segment.value) + }) + .collect(); + (item.label.clone(), values) + }) + .collect(); + (columns, rows) +} + +#[cfg(test)] +mod tests { + use super::*; + use bds_core::engine::chat_surfaces::build_render_surface; + + #[test] + fn chart_math_handles_every_fixed_chart_type() { + let series = vec![ChartSeries { + label: "A".into(), + value: 3.0, + segments: vec![], + }]; + for chart_type in [ + ChartType::Bar, + ChartType::StackedBar, + ChartType::Line, + ChartType::Area, + ChartType::Pie, + ChartType::Donut, + ChartType::Heatmap, + ] { + assert_eq!(chart_series_total(chart_type, &series[0]), 3.0); + } + assert_eq!(chart_points(Size::new(100.0, 100.0), &series).len(), 1); + } + + #[test] + fn heatmap_uses_labelled_segments_and_zero_fills_missing_cells() { + let series = vec![ + ChartSeries { + label: "First".into(), + value: 0.0, + segments: vec![ + bds_core::engine::chat_surfaces::ChartSegment { + label: "A".into(), + value: 2.0, + }, + bds_core::engine::chat_surfaces::ChartSegment { + label: "B".into(), + value: 4.0, + }, + ], + }, + ChartSeries { + label: "Second".into(), + value: 0.0, + segments: vec![bds_core::engine::chat_surfaces::ChartSegment { + label: "B".into(), + value: 3.0, + }], + }, + ]; + let (columns, rows) = heatmap_layout(&series); + assert_eq!(columns, ["A", "B"]); + assert_eq!(rows[0], ("First".into(), vec![2.0, 4.0])); + assert_eq!(rows[1], ("Second".into(), vec![0.0, 3.0])); + } + + #[test] + fn mindmap_is_cycle_safe_and_keeps_orphans() { + let nodes = vec![ + MindmapNode { + id: Some("a".into()), + label: "A".into(), + children: vec!["b".into()], + }, + MindmapNode { + id: Some("b".into()), + label: "B".into(), + children: vec!["a".into()], + }, + MindmapNode { + id: None, + label: "Loose".into(), + children: vec![], + }, + ]; + let rows = mindmap_rows(&nodes); + assert_eq!(rows.len(), 3); + assert!(rows.iter().any(|(_, label)| label == "Loose")); + } + + #[test] + fn every_surface_chart_and_form_control_builds_a_native_widget() { + let state = ChatSurfaceState::default(); + let form = serde_json::json!({ + "fields": [ + {"key": "text", "label": "Text", "inputType": "text"}, + {"key": "notes", "label": "Notes", "inputType": "textarea"}, + {"key": "choice", "label": "Choice", "inputType": "select", "options": [{"label": "A", "value": "a"}]}, + {"key": "enabled", "label": "Enabled", "inputType": "checkbox"}, + {"key": "date", "label": "Date", "inputType": "date"}, + {"key": "count", "label": "Count", "inputType": "number"} + ], + "submitAction": "switchView" + }); + let cases = [ + ( + "render_card", + serde_json::json!({"body": ""}), + ), + ("render_form", form), + ("render_list", serde_json::json!({"items": ["One"]})), + ( + "render_metric", + serde_json::json!({"label": "Count", "value": 2}), + ), + ( + "render_mindmap", + serde_json::json!({"nodes": [{"id": "a", "label": "A"}]}), + ), + ( + "render_table", + serde_json::json!({"columns": ["A"], "rows": [["B"]]}), + ), + ( + "render_tabs", + serde_json::json!({"tabs": [{"label": "Mixed", "content": ["plain", {"type": "future", "html": "data"}]}]}), + ), + ]; + for (index, (name, arguments)) in cases.into_iter().enumerate() { + let surface = + build_render_surface(name, &arguments, format!("surface-{index}"), &state).unwrap(); + let mut textareas = HashMap::new(); + if name == "render_form" { + textareas.insert( + textarea_key(&surface.id, "notes"), + text_editor::Content::new(), + ); + } + let _: Element<'_, Message> = view(&surface, &state, &textareas, UiLocale::En); + } + for (index, chart_type) in [ + "bar", + "stacked-bar", + "line", + "area", + "pie", + "donut", + "heatmap", + ] + .into_iter() + .enumerate() + { + let surface = build_render_surface( + "render_chart", + &serde_json::json!({ + "chartType": chart_type, + "series": [{"label": "A", "value": 2, "segments": [{"label": "X", "value": 2}]}] + }), + format!("chart-{index}"), + &state, + ) + .unwrap(); + let textareas = HashMap::new(); + let _: Element<'_, Message> = view(&surface, &state, &textareas, UiLocale::En); + } + } +} diff --git a/crates/bds-ui/src/views/chat_view.rs b/crates/bds-ui/src/views/chat_view.rs index f2d390b..0fbd25a 100644 --- a/crates/bds-ui/src/views/chat_view.rs +++ b/crates/bds-ui/src/views/chat_view.rs @@ -1,3 +1,9 @@ +use std::collections::HashMap; +use std::time::Instant; + +use bds_core::engine::chat_surfaces::{ + ChatSurfaceState, InlineSurface, build_message_surfaces, build_render_surface, +}; use bds_core::i18n::UiLocale; use bds_core::model::{ChatConversation, ChatMessage, ChatRole}; use iced::widget::text::Shaping; @@ -22,6 +28,11 @@ pub struct ChatEditorState { streaming_markdown: Vec, pub active_tool: Option, pub error: Option, + pub surface_state: ChatSurfaceState, + pub message_surfaces: HashMap>, + pub streaming_surfaces: Vec, + pub surface_textareas: HashMap, + pub surface_state_dirty_since: Option, } #[derive(Debug, Clone, PartialEq, Eq)] @@ -62,6 +73,13 @@ impl ChatEditorState { ) }) .collect(); + let surface_state = conversation + .surface_state + .as_deref() + .and_then(|value| serde_json::from_str(value).ok()) + .unwrap_or_default(); + let message_surfaces = build_surfaces(&messages, &surface_state); + let surface_textareas = build_textareas(&message_surfaces); Self { rename_input: conversation.title.clone(), conversation, @@ -74,6 +92,11 @@ impl ChatEditorState { streaming_markdown: Vec::new(), active_tool: None, error: None, + surface_state, + message_surfaces, + streaming_surfaces: Vec::new(), + surface_textareas, + surface_state_dirty_since: None, } } @@ -89,6 +112,7 @@ impl ChatEditorState { }) .collect(); self.messages = messages; + self.rebuild_surfaces(); } pub fn set_streaming_content(&mut self, content: String) { @@ -99,6 +123,24 @@ impl ChatEditorState { pub fn clear_streaming(&mut self) { self.streaming_content.clear(); self.streaming_markdown.clear(); + self.streaming_surfaces.clear(); + } + + pub fn add_streaming_surface(&mut self, name: &str, arguments: &serde_json::Value, id: String) { + if let Some(surface) = build_render_surface(name, arguments, id, &self.surface_state) + && !self.surface_state.dismissed_surfaces.contains(&surface.id) + { + add_textareas(&mut self.surface_textareas, &surface); + self.streaming_surfaces.push(surface); + } + } + + pub fn rebuild_surfaces(&mut self) { + self.message_surfaces = build_surfaces(&self.messages, &self.surface_state); + self.surface_textareas = build_textareas(&self.message_surfaces); + for surface in &self.streaming_surfaces { + add_textareas(&mut self.surface_textareas, surface); + } } pub fn token_totals(&self) -> (u64, u64, u64, u64) { @@ -114,6 +156,50 @@ impl ChatEditorState { } } +fn build_surfaces( + messages: &[ChatMessage], + state: &ChatSurfaceState, +) -> HashMap> { + messages + .iter() + .filter_map(|message| { + let surfaces = build_message_surfaces(message, state); + (!surfaces.is_empty()).then_some((message.id, surfaces)) + }) + .collect() +} + +fn build_textareas( + surfaces: &HashMap>, +) -> HashMap { + let mut result = HashMap::new(); + for surface in surfaces.values().flatten() { + add_textareas(&mut result, surface); + } + result +} + +fn add_textareas(result: &mut HashMap, surface: &InlineSurface) { + for field in &surface.fields { + if field.input_type == bds_core::engine::chat_surfaces::FormInputType::Textarea { + result + .entry(textarea_key(&surface.id, &field.key)) + .or_insert_with(|| { + text_editor::Content::with_text(field.value.as_str().unwrap_or_default()) + }); + } + } + for tab in &surface.tabs { + for child in &tab.content { + add_textareas(result, child); + } + } +} + +pub fn textarea_key(surface_id: &str, field: &str) -> String { + format!("{surface_id}\0{field}") +} + pub fn view<'a>( state: &'a ChatEditorState, locale: UiLocale, @@ -197,6 +283,19 @@ pub fn view<'a>( state.rendered_messages.get(&message.id), locale, )); + if let Some(surfaces) = state.message_surfaces.get(&message.id) { + for surface in surfaces { + message_elements.push(crate::views::chat_surfaces::view( + surface, + &state.surface_state, + &state.surface_textareas, + locale, + )); + } + } + if let Some(markers) = tool_markers_view(message, locale) { + message_elements.push(markers); + } } } if state.streaming && !state.streaming_content.is_empty() { @@ -205,6 +304,14 @@ pub fn view<'a>( t(locale, "chat.streaming"), )); } + for surface in &state.streaming_surfaces { + message_elements.push(crate::views::chat_surfaces::view( + surface, + &state.surface_state, + &state.surface_textareas, + locale, + )); + } if let Some(tool) = state.active_tool.as_deref() { message_elements.push( inputs::card( @@ -341,10 +448,18 @@ fn message_view<'a>( .shaping(Shaping::Advanced) .into() }; - let mut children: Vec> = vec![ + let children: Vec> = vec![ text(label).size(11).color(inputs::SECTION_COLOR).into(), body, ]; + inputs::card(iced::widget::Column::with_children(children).spacing(6)).into() +} + +fn tool_markers_view<'a>( + message: &'a ChatMessage, + locale: UiLocale, +) -> Option> { + let mut markers = Vec::new(); if let Some(tool_calls) = message.tool_calls.as_deref() && let Ok(calls) = serde_json::from_str::>(tool_calls) { @@ -371,10 +486,11 @@ fn message_view<'a>( args_preview ) }; - children.push(text(marker).size(11).color(inputs::SECTION_COLOR).into()); + markers.push(text(marker).size(11).color(inputs::SECTION_COLOR).into()); } } - inputs::card(iced::widget::Column::with_children(children).spacing(6)).into() + (!markers.is_empty()) + .then(|| inputs::card(iced::widget::Column::with_children(markers).spacing(4)).into()) } fn assistant_card<'a>(items: &'a [markdown::Item], label: String) -> Element<'a, Message> { diff --git a/crates/bds-ui/src/views/mod.rs b/crates/bds-ui/src/views/mod.rs index efd40d3..069b15c 100644 --- a/crates/bds-ui/src/views/mod.rs +++ b/crates/bds-ui/src/views/mod.rs @@ -1,4 +1,5 @@ pub mod activity_bar; +pub mod chat_surfaces; pub mod chat_view; pub mod dashboard; pub mod git; diff --git a/locales/ui/de.ftl b/locales/ui/de.ftl index e080ca7..5df4a2f 100644 --- a/locales/ui/de.ftl +++ b/locales/ui/de.ftl @@ -78,6 +78,23 @@ chat-role-tool = Werkzeug chat-deleted = Unterhaltung gelöscht chat-link-refused = Dieser Linktyp ist nicht erlaubt chat-navigation-invalid = Das angeforderte Blog-Element konnte nicht geöffnet werden +chat-surface-dismiss = Ausblenden +chat-surface-empty = Kein Inhalt +chat-surface-type-card = Karte +chat-surface-type-chart = Diagramm +chat-surface-type-form = Formular +chat-surface-type-list = Liste +chat-surface-type-metric = Kennzahl +chat-surface-type-mindmap = Mindmap +chat-surface-type-table = Tabelle +chat-surface-type-tabs = Registerkarten +chat-surface-type-text = Text +chat-surface-type-json = Daten +chat-surface-form-datePlaceholder = JJJJ-MM-TT +chat-surface-form-numberPlaceholder = Zahl eingeben +chat-surface-form-textPlaceholder = Wert eingeben +chat-surface-form-submit = Absenden +chat-surface-actionRefused = Assistentenaktion abgelehnt: { $reason } statusBar-chatTokens = KI ↑{ $input } ↓{ $output } Cache { $cacheRead }/{ $cacheWrite } common-settings = Einstellungen common-tasks = Aufgaben diff --git a/locales/ui/en.ftl b/locales/ui/en.ftl index 0c992ca..745e7a6 100644 --- a/locales/ui/en.ftl +++ b/locales/ui/en.ftl @@ -78,6 +78,23 @@ chat-role-tool = Tool chat-deleted = Conversation deleted chat-link-refused = This link type is not allowed chat-navigation-invalid = The requested blog item could not be opened +chat-surface-dismiss = Dismiss +chat-surface-empty = No content +chat-surface-type-card = Card +chat-surface-type-chart = Chart +chat-surface-type-form = Form +chat-surface-type-list = List +chat-surface-type-metric = Metric +chat-surface-type-mindmap = Mind map +chat-surface-type-table = Table +chat-surface-type-tabs = Tabs +chat-surface-type-text = Text +chat-surface-type-json = Data +chat-surface-form-datePlaceholder = YYYY-MM-DD +chat-surface-form-numberPlaceholder = Enter a number +chat-surface-form-textPlaceholder = Enter a value +chat-surface-form-submit = Submit +chat-surface-actionRefused = Assistant action refused: { $reason } statusBar-chatTokens = AI ↑{ $input } ↓{ $output } cache { $cacheRead }/{ $cacheWrite } common-settings = Settings common-tasks = Tasks diff --git a/locales/ui/es.ftl b/locales/ui/es.ftl index 0907750..df415d5 100644 --- a/locales/ui/es.ftl +++ b/locales/ui/es.ftl @@ -78,6 +78,23 @@ chat-role-tool = Herramienta chat-deleted = Conversación eliminada chat-link-refused = Este tipo de enlace no está permitido chat-navigation-invalid = No se pudo abrir el elemento del blog solicitado +chat-surface-dismiss = Descartar +chat-surface-empty = Sin contenido +chat-surface-type-card = Tarjeta +chat-surface-type-chart = Gráfico +chat-surface-type-form = Formulario +chat-surface-type-list = Lista +chat-surface-type-metric = Métrica +chat-surface-type-mindmap = Mapa mental +chat-surface-type-table = Tabla +chat-surface-type-tabs = Pestañas +chat-surface-type-text = Texto +chat-surface-type-json = Datos +chat-surface-form-datePlaceholder = AAAA-MM-DD +chat-surface-form-numberPlaceholder = Introduce un número +chat-surface-form-textPlaceholder = Introduce un valor +chat-surface-form-submit = Enviar +chat-surface-actionRefused = Acción del asistente rechazada: { $reason } statusBar-chatTokens = IA ↑{ $input } ↓{ $output } caché { $cacheRead }/{ $cacheWrite } common-settings = Configuración common-tasks = Tareas diff --git a/locales/ui/fr.ftl b/locales/ui/fr.ftl index 8354559..99bfb82 100644 --- a/locales/ui/fr.ftl +++ b/locales/ui/fr.ftl @@ -78,6 +78,23 @@ chat-role-tool = Outil chat-deleted = Conversation supprimée chat-link-refused = Ce type de lien n’est pas autorisé chat-navigation-invalid = L’élément du blog demandé n’a pas pu être ouvert +chat-surface-dismiss = Masquer +chat-surface-empty = Aucun contenu +chat-surface-type-card = Carte +chat-surface-type-chart = Graphique +chat-surface-type-form = Formulaire +chat-surface-type-list = Liste +chat-surface-type-metric = Indicateur +chat-surface-type-mindmap = Carte mentale +chat-surface-type-table = Tableau +chat-surface-type-tabs = Onglets +chat-surface-type-text = Texte +chat-surface-type-json = Données +chat-surface-form-datePlaceholder = AAAA-MM-JJ +chat-surface-form-numberPlaceholder = Saisir un nombre +chat-surface-form-textPlaceholder = Saisir une valeur +chat-surface-form-submit = Envoyer +chat-surface-actionRefused = Action de l’assistant refusée : { $reason } statusBar-chatTokens = IA ↑{ $input } ↓{ $output } cache { $cacheRead }/{ $cacheWrite } common-settings = Paramètres common-tasks = Tâches diff --git a/locales/ui/it.ftl b/locales/ui/it.ftl index ae3e8e4..137abb3 100644 --- a/locales/ui/it.ftl +++ b/locales/ui/it.ftl @@ -78,6 +78,23 @@ chat-role-tool = Strumento chat-deleted = Conversazione eliminata chat-link-refused = Questo tipo di collegamento non è consentito chat-navigation-invalid = Impossibile aprire l’elemento del blog richiesto +chat-surface-dismiss = Nascondi +chat-surface-empty = Nessun contenuto +chat-surface-type-card = Scheda +chat-surface-type-chart = Grafico +chat-surface-type-form = Modulo +chat-surface-type-list = Elenco +chat-surface-type-metric = Metrica +chat-surface-type-mindmap = Mappa mentale +chat-surface-type-table = Tabella +chat-surface-type-tabs = Schede +chat-surface-type-text = Testo +chat-surface-type-json = Dati +chat-surface-form-datePlaceholder = AAAA-MM-GG +chat-surface-form-numberPlaceholder = Inserisci un numero +chat-surface-form-textPlaceholder = Inserisci un valore +chat-surface-form-submit = Invia +chat-surface-actionRefused = Azione dell’assistente rifiutata: { $reason } statusBar-chatTokens = IA ↑{ $input } ↓{ $output } cache { $cacheRead }/{ $cacheWrite } common-settings = Impostazioni common-tasks = Attività