Implement persistent native A2UI chat surfaces

This commit is contained in:
2026-07-19 16:57:43 +02:00
parent ac611e3f7f
commit fd4fd744e0
25 changed files with 2571 additions and 18 deletions

61
Cargo.lock generated
View File

@@ -2340,6 +2340,12 @@ version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "98de4bbd547a563b716d8dfa9aad1cb19bfab00f4fa09a6a4ed21dbcf44ce9c4"
[[package]]
name = "float_next_after"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8bf7cc16383c4b8d58b9905a8509f02926ce3058053c056376248d958c9df1e8"
[[package]]
name = "fluent-bundle"
version = "0.16.0"
@@ -3363,6 +3369,7 @@ dependencies = [
"image 0.24.9",
"kamadak-exif",
"log",
"lyon_path",
"once_cell",
"raw-window-handle",
"rustc-hash 2.1.2",
@@ -3427,6 +3434,7 @@ dependencies = [
"iced_glyphon",
"iced_graphics",
"log",
"lyon",
"once_cell",
"resvg",
"rustc-hash 2.1.2",
@@ -4352,6 +4360,58 @@ dependencies = [
"which",
]
[[package]]
name = "lyon"
version = "1.0.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bd0578bdecb7d6d88987b8b2b1e3a4e2f81df9d0ece1078623324a567904e7b7"
dependencies = [
"lyon_algorithms",
"lyon_tessellation",
]
[[package]]
name = "lyon_algorithms"
version = "1.0.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8575c0d003ae459399623c4def180c63b77f343b1a7fee64f249b349e7699a31"
dependencies = [
"lyon_path",
"num-traits",
]
[[package]]
name = "lyon_geom"
version = "1.0.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4336502e29e32af93cf2dad2214ed6003c17ceb5bd499df77b1de663b9042b92"
dependencies = [
"arrayvec",
"euclid",
"num-traits",
]
[[package]]
name = "lyon_path"
version = "1.0.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c463f9c428b7fc5ec885dcd39ce4aa61e29111d0e33483f6f98c74e89d8621e"
dependencies = [
"lyon_geom",
"num-traits",
]
[[package]]
name = "lyon_tessellation"
version = "1.0.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e43b7e44161571868f5c931d12583592c223c5583eef86b08aa02b7048a3552"
dependencies = [
"float_next_after",
"lyon_path",
"num-traits",
]
[[package]]
name = "malloc_buf"
version = "0.0.6"
@@ -4758,6 +4818,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"
dependencies = [
"autocfg",
"libm",
]
[[package]]

View File

@@ -54,7 +54,7 @@ mlua = { version = "0.12", features = ["lua54", "vendored", "serialize"] }
url = "2.5"
# UI framework
iced = { version = "0.13", features = ["wgpu", "advanced", "image", "svg", "tokio", "markdown"] }
iced = { version = "0.13", features = ["wgpu", "advanced", "canvas", "image", "svg", "tokio", "markdown"] }
# Editor widget
cosmic-text = "0.12"

View File

@@ -18,7 +18,7 @@ The project is under active development. Core blogging workflows are broadly ava
- Markdown/Liquid rendering with native macros, multilingual routes, feeds, sitemap, Pagefind, and incremental site generation through cancellable section task groups.
- Local preview in the app or system browser.
- Optional one-shot AI translation, description, analysis, taxonomy, and language-detection operations using online or local OpenAI-compatible endpoints with airplane-mode gating.
- Persistent conversational AI with safe Markdown, streamed and cancellable responses, model/session/token tracking, bounded project-aware blog tools, and localized conversation management in the Chat workspace.
- Persistent conversational AI with safe Markdown, streamed and cancellable responses, model/session/token tracking, bounded project-aware blog tools, and localized conversation management in the Chat workspace. Allowlisted render tools add persistent native cards, charts, forms, lists, metrics, mind maps, tables, and tabs without executing assistant-provided HTML or JavaScript.
- SSH-agent-based SCP or rsync publishing.
- Integrated Git workflow with repository initialization, Git LFS image tracking, status and diffs, branch/file history, commits, remotes, cancellable fetch/pull/push, and post-pull filesystem reconciliation; network actions respect airplane mode.
- Site, media, and translation validation plus `ruds://new-post` Blogmark capture and Lua transforms; bDS2 keeps its separate `bds2://` bookmarklet protocol.

View File

@@ -39,10 +39,9 @@ Done:
- OpenAI-compatible SSE streaming with split-frame content/tool assembly, provider-error handling, independent cancellation, bounded tool rounds, and context truncation that preserves system messages and tool pairs.
- Project-aware tools over statistics, FTS search, posts, media, templates, scripts, tags/categories, metadata mutation through shared engines, and allowlisted workspace navigation.
- Localized Chat sidebar/editor with conversation and model controls, safe GFM text rendering with blocked external images, streaming/tool state, multiline send/stop controls, and status-bar token totals.
- Fixed native A2UI cards, charts, forms, lists, metrics, mind maps, tables, and tabs with safe nested fallbacks, persistent stable-ID interaction state, debounced form values, and strictly allowlisted application actions.
- Online/airplane endpoint routing uses the shared secure endpoint and model infrastructure; unavailable modes direct the user to the existing localized AI settings.
Persistent A2UI surfaces remain separately tracked below.
### Embeddings, semantic search, and duplicates — Open
Existing source contains schema placeholders only.
@@ -118,11 +117,13 @@ Open:
- Sidebar/editor navigation, editing, publishing, and live domain-event updates.
- Remote operation through the headless server.
### A2UI surfaces — Open
### A2UI surfaces — Complete
Open:
Done:
- A2UI component renderer, surface state/data flow, and AI-assistant integration.
- Eight structured render tools and fixed native rendering for ten surface types, including all required chart and form variants.
- Per-conversation form, tab, and dismissal state tied to stable message/tool-index IDs and restored on reopen.
- Safe text/JSON fallbacks, inert assistant markup, persistent expanded surfaces during later streaming, and visibly rejected non-allowlisted actions.
## Suggested Order
@@ -131,6 +132,6 @@ Open:
3. Embeddings and duplicate detection.
4. Documentation and menu UX.
5. Headless server and TUI.
6. A2UI after conversational AI exists.
6. A2UI after conversational AI exists. (Complete)
The order may change when an extension directly unlocks a concrete user workflow; it must not create a parallel data model or bypass core engines.

View File

@@ -0,0 +1,2 @@
-- This file should undo anything in `up.sql`
ALTER TABLE chat_conversations DROP COLUMN surface_state;

View File

@@ -0,0 +1,2 @@
-- Your SQL goes here
ALTER TABLE chat_conversations ADD COLUMN surface_state TEXT;

View File

@@ -36,7 +36,7 @@ mod tests {
let applied = db
.conn()
.with_migrations(|conn| conn.applied_migrations().unwrap().len());
assert_eq!(applied, 5);
assert_eq!(applied, 6);
}
#[test]
@@ -204,4 +204,40 @@ mod tests {
.unwrap();
assert_eq!(usage, (Some(8), Some(5), None, None));
}
#[test]
fn existing_conversations_are_preserved_when_surface_state_is_added() {
let db = Database::open_in_memory().unwrap();
db.conn().with_migrations(|conn| {
for _ in 0..5 {
conn.run_next_migration(MIGRATIONS).unwrap();
}
});
db.conn()
.with(|conn| {
diesel::insert_into(chat_conversations::table)
.values((
chat_conversations::id.eq("existing-surface-chat"),
chat_conversations::title.eq("Keep me"),
chat_conversations::created_at.eq(1_i64),
chat_conversations::updated_at.eq(1_i64),
))
.execute(conn)
})
.unwrap();
run_migrations(db.conn()).unwrap();
let (title, state) = db
.conn()
.with(|conn| {
chat_conversations::table
.filter(chat_conversations::id.eq("existing-surface-chat"))
.select((chat_conversations::title, chat_conversations::surface_state))
.first::<(String, Option<String>)>(conn)
})
.unwrap();
assert_eq!(title, "Keep me");
assert_eq!(state, None);
}
}

View File

@@ -92,6 +92,22 @@ pub fn set_session_id(
})
}
pub fn set_surface_state(
conn: &DbConnection,
id: &str,
state: &str,
updated_at: i64,
) -> QueryResult<usize> {
conn.with(|connection| {
diesel::update(chat_conversations::table.find(id))
.set((
chat_conversations::surface_state.eq(state),
chat_conversations::updated_at.eq(updated_at),
))
.execute(connection)
})
}
pub fn delete_conversation(conn: &DbConnection, id: &str) -> QueryResult<usize> {
conn.with(|connection| {
connection.transaction(|connection| {

View File

@@ -64,6 +64,7 @@ diesel::table! {
title -> Text,
model -> Nullable<Text>,
copilot_session_id -> Nullable<Text>,
surface_state -> Nullable<Text>,
created_at -> BigInt,
updated_at -> BigInt,
}

View File

@@ -52,6 +52,8 @@ pub enum ChatEvent {
ToolStarted {
conversation_id: String,
name: String,
surface_id: String,
arguments: Value,
},
ToolFinished {
conversation_id: String,
@@ -279,6 +281,7 @@ pub fn create_conversation_titled(
title,
model,
copilot_session_id: None,
surface_state: None,
created_at: now,
updated_at: now,
},
@@ -373,6 +376,38 @@ pub fn list_messages(conn: &Connection, id: &str) -> EngineResult<Vec<ChatMessag
Ok(queries::list_messages(conn, id)?)
}
pub fn get_surface_state(
conn: &Connection,
conversation_id: &str,
) -> EngineResult<crate::engine::chat_surfaces::ChatSurfaceState> {
let conversation = match queries::get_conversation(conn, conversation_id) {
Ok(conversation) => conversation,
Err(diesel::result::Error::NotFound) => return Ok(Default::default()),
Err(error) => return Err(error.into()),
};
conversation.surface_state.as_deref().map_or_else(
|| Ok(Default::default()),
|state| {
serde_json::from_str(state)
.map_err(|error| EngineError::Parse(format!("invalid chat surface state: {error}")))
},
)
}
pub fn put_surface_state(
conn: &Connection,
conversation_id: &str,
state: &crate::engine::chat_surfaces::ChatSurfaceState,
) -> EngineResult<()> {
let state = serde_json::to_string(state)?;
if queries::set_surface_state(conn, conversation_id, &state, now_unix_ms())? == 0 {
return Err(EngineError::NotFound(format!(
"chat conversation {conversation_id}"
)));
}
Ok(())
}
pub fn insert_message(
conn: &Connection,
conversation_id: &str,
@@ -633,7 +668,7 @@ fn run_turns(
let serialized_calls = (!response.tool_calls.is_empty())
.then(|| serialize_tool_calls(&response.tool_calls))
.transpose()?;
insert_message(
let assistant_message = insert_message(
conn,
conversation_id,
ChatRole::Assistant,
@@ -667,6 +702,8 @@ fn run_turns(
ChatEvent::ToolStarted {
conversation_id: conversation_id.to_string(),
name: call.name.clone(),
surface_id: format!("{}-surface-{index}", assistant_message.id),
arguments: call.arguments.clone(),
},
);
if cancelled.load(Ordering::SeqCst) {

View File

@@ -0,0 +1,583 @@
use std::collections::{BTreeMap, BTreeSet};
use serde::{Deserialize, Serialize};
use serde_json::{Value, json};
use crate::model::ChatMessage;
pub const RENDER_TOOL_NAMES: [&str; 8] = [
"render_card",
"render_chart",
"render_form",
"render_list",
"render_metric",
"render_mindmap",
"render_table",
"render_tabs",
];
#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize)]
pub struct ChatSurfaceState {
#[serde(default)]
pub surface_data: BTreeMap<String, BTreeMap<String, Value>>,
#[serde(default)]
pub surface_tabs: BTreeMap<String, usize>,
#[serde(default)]
pub dismissed_surfaces: BTreeSet<String>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum SurfaceKind {
Card,
Chart,
Form,
List,
Metric,
Mindmap,
Table,
Tabs,
Text,
Json,
}
impl SurfaceKind {
pub const fn as_str(self) -> &'static str {
match self {
Self::Card => "card",
Self::Chart => "chart",
Self::Form => "form",
Self::List => "list",
Self::Metric => "metric",
Self::Mindmap => "mindmap",
Self::Table => "table",
Self::Tabs => "tabs",
Self::Text => "text",
Self::Json => "json",
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ChartType {
Bar,
StackedBar,
Line,
Area,
Pie,
Donut,
Heatmap,
}
impl ChartType {
pub const fn as_str(self) -> &'static str {
match self {
Self::Bar => "bar",
Self::StackedBar => "stacked-bar",
Self::Line => "line",
Self::Area => "area",
Self::Pie => "pie",
Self::Donut => "donut",
Self::Heatmap => "heatmap",
}
}
fn parse(value: Option<&str>) -> Self {
match value {
Some("stacked-bar") => Self::StackedBar,
Some("line") => Self::Line,
Some("area") => Self::Area,
Some("pie") => Self::Pie,
Some("donut") => Self::Donut,
Some("heatmap") => Self::Heatmap,
_ => Self::Bar,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum FormInputType {
Text,
Textarea,
Select,
Checkbox,
Date,
Number,
}
impl FormInputType {
fn parse(value: Option<&str>) -> Self {
match value {
Some("textarea") => Self::Textarea,
Some("select") => Self::Select,
Some("checkbox") => Self::Checkbox,
Some("date") => Self::Date,
Some("number") => Self::Number,
_ => Self::Text,
}
}
}
#[derive(Debug, Clone, PartialEq)]
pub struct InlineSurface {
pub id: String,
pub kind: SurfaceKind,
pub title: Option<String>,
pub subtitle: Option<String>,
pub body: Option<String>,
pub actions: Vec<SurfaceAction>,
pub columns: Vec<String>,
pub rows: Vec<Vec<String>>,
pub chart_type: Option<ChartType>,
pub series: Vec<ChartSeries>,
pub max_value: Option<f64>,
pub label: Option<String>,
pub value: Option<String>,
pub items: Vec<String>,
pub nodes: Vec<MindmapNode>,
pub fields: Vec<FormField>,
pub submit_label: Option<String>,
pub submit_action: Option<String>,
pub tabs: Vec<TabPanel>,
pub selected_index: Option<usize>,
pub raw: Option<Value>,
}
impl InlineSurface {
fn empty(id: String, kind: SurfaceKind) -> Self {
Self {
id,
kind,
title: None,
subtitle: None,
body: None,
actions: vec![],
columns: vec![],
rows: vec![],
chart_type: None,
series: vec![],
max_value: None,
label: None,
value: None,
items: vec![],
nodes: vec![],
fields: vec![],
submit_label: None,
submit_action: None,
tabs: vec![],
selected_index: None,
raw: None,
}
}
}
#[derive(Debug, Clone, PartialEq)]
pub struct SurfaceAction {
pub label: String,
pub action: String,
pub payload: Value,
}
#[derive(Debug, Clone, PartialEq)]
pub struct ChartSeries {
pub label: String,
pub value: f64,
pub segments: Vec<ChartSegment>,
}
#[derive(Debug, Clone, PartialEq)]
pub struct ChartSegment {
pub label: String,
pub value: f64,
}
#[derive(Debug, Clone, PartialEq)]
pub struct MindmapNode {
pub id: Option<String>,
pub label: String,
pub children: Vec<String>,
}
#[derive(Debug, Clone, PartialEq)]
pub struct FormField {
pub key: String,
pub label: String,
pub input_type: FormInputType,
pub placeholder: Option<String>,
pub value: Value,
pub options: Vec<FieldOption>,
pub required: bool,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct FieldOption {
pub label: String,
pub value: String,
}
impl std::fmt::Display for FieldOption {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(&self.label)
}
}
#[derive(Debug, Clone, PartialEq)]
pub struct TabPanel {
pub label: String,
pub content: Vec<InlineSurface>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct SurfaceNavigation {
pub destination: String,
pub entity_id: Option<String>,
}
pub fn build_message_surfaces(
message: &ChatMessage,
state: &ChatSurfaceState,
) -> Vec<InlineSurface> {
let Some(raw) = message.tool_calls.as_deref() else {
return vec![];
};
let Ok(calls) = serde_json::from_str::<Vec<Value>>(raw) else {
return vec![];
};
calls
.iter()
.enumerate()
.filter_map(|(index, call)| {
let function = call.get("function").unwrap_or(call);
let name = function.get("name")?.as_str()?;
let raw = function
.get("arguments")
.cloned()
.unwrap_or_else(|| json!({}));
let arguments = raw
.as_str()
.and_then(|s| serde_json::from_str(s).ok())
.unwrap_or(raw);
build_render_surface(
name,
&arguments,
format!("{}-surface-{index}", message.id),
state,
)
})
.filter(|surface| !state.dismissed_surfaces.contains(&surface.id))
.collect()
}
pub fn build_render_surface(
name: &str,
arguments: &Value,
id: String,
state: &ChatSurfaceState,
) -> Option<InlineSurface> {
match name {
"render_card" => Some(card(arguments, id)),
"render_chart" => Some(chart(arguments, id)),
"render_form" => Some(form(arguments, id, state)),
"render_list" => Some(list(arguments, id)),
"render_metric" => Some(metric(arguments, id)),
"render_mindmap" => Some(mindmap(arguments, id)),
"render_table" => Some(table(arguments, id)),
"render_tabs" => Some(tabs(arguments, id, state)),
unknown if unknown.starts_with("render_") => {
let mut surface = InlineSurface::empty(id, SurfaceKind::Json);
surface.raw = Some(arguments.clone());
Some(surface)
}
_ => None,
}
}
pub fn merge_form_data(payload: Value, surface_id: &str, state: &ChatSurfaceState) -> Value {
let mut result = payload.as_object().cloned().unwrap_or_default();
if let Some(values) = state.surface_data.get(surface_id) {
result.insert(
"formData".into(),
Value::Object(values.clone().into_iter().collect()),
);
}
Value::Object(result)
}
pub fn resolve_surface_action(action: &str, payload: &Value) -> Result<SurfaceNavigation, String> {
let map = payload
.as_object()
.ok_or_else(|| "surface action payload must be an object".to_string())?;
let normalized = action.replace('_', "").to_ascii_lowercase();
let open = |destination: &str, keys: &[&str]| {
let id = keys
.iter()
.find_map(|key| map.get(*key).and_then(Value::as_str))
.filter(|id| !id.trim().is_empty())
.ok_or_else(|| format!("surface action {action} requires an identifier"))?;
Ok(SurfaceNavigation {
destination: destination.into(),
entity_id: Some(id.into()),
})
};
match normalized.as_str() {
"openpost" => open("posts", &["postId", "post_id", "id"]),
"openmedia" => open("media", &["mediaId", "media_id", "id"]),
"openchat" => open(
"chat",
&[
"conversationId",
"conversation_id",
"chatId",
"chat_id",
"id",
],
),
"opensettings" => Ok(navigation("settings")),
"switchview" | "setview" | "setactiveview" => {
let view = map
.get("view")
.or_else(|| map.get("destination"))
.and_then(Value::as_str)
.filter(|view| {
[
"posts",
"pages",
"media",
"templates",
"scripts",
"tags",
"chat",
"import",
"git",
"settings",
]
.contains(view)
})
.ok_or_else(|| format!("surface action {action} requires a valid view"))?;
Ok(navigation(view))
}
"togglesidebar" => Ok(navigation("toggle_sidebar")),
"togglepanel" | "openpanel" => Ok(navigation("toggle_panel")),
"toggleassistantsidebar" => Ok(navigation("toggle_assistant_sidebar")),
_ => Err(format!("unsupported surface action: {action}")),
}
}
fn navigation(destination: &str) -> SurfaceNavigation {
SurfaceNavigation {
destination: destination.into(),
entity_id: None,
}
}
fn card(a: &Value, id: String) -> InlineSurface {
let mut s = InlineSurface::empty(id, SurfaceKind::Card);
s.title = string(a, "title");
s.subtitle = string(a, "subtitle");
s.body = string(a, "body");
s.actions = array(a, "actions")
.iter()
.map(|v| SurfaceAction {
label: string(v, "label").unwrap_or_default(),
action: string(v, "action").unwrap_or_default(),
payload: v.get("payload").cloned().unwrap_or_else(|| json!({})),
})
.collect();
s
}
fn chart(a: &Value, id: String) -> InlineSurface {
let mut s = InlineSurface::empty(id, SurfaceKind::Chart);
s.title = string(a, "title");
s.chart_type = Some(ChartType::parse(
a.get("chartType")
.or_else(|| a.get("chart_type"))
.and_then(Value::as_str),
));
s.series = array(a, "series")
.iter()
.map(|v| ChartSeries {
label: string(v, "label").unwrap_or_default(),
value: numeric(v.get("value")),
segments: array(v, "segments")
.iter()
.map(|seg| ChartSegment {
label: string(seg, "label").unwrap_or_default(),
value: numeric(seg.get("value")),
})
.collect(),
})
.collect();
s.max_value = Some(s.series.iter().map(|v| v.value).fold(0.0_f64, f64::max));
s
}
fn form(a: &Value, id: String, state: &ChatSurfaceState) -> InlineSurface {
let mut s = InlineSurface::empty(id.clone(), SurfaceKind::Form);
s.title = string(a, "title");
s.fields = array(a, "fields")
.iter()
.map(|v| {
let key = string(v, "key").unwrap_or_else(|| "field".into());
let default = v
.get("defaultValue")
.or_else(|| v.get("default_value"))
.cloned()
.unwrap_or(Value::Null);
FormField {
label: string(v, "label").unwrap_or_else(|| key.clone()),
input_type: FormInputType::parse(
v.get("inputType")
.or_else(|| v.get("input_type"))
.and_then(Value::as_str),
),
placeholder: string(v, "placeholder"),
value: state
.surface_data
.get(&id)
.and_then(|m| m.get(&key))
.cloned()
.unwrap_or(default),
options: array(v, "options")
.iter()
.map(|o| FieldOption {
label: string(o, "label").unwrap_or_default(),
value: string(o, "value").unwrap_or_default(),
})
.collect(),
required: v.get("required").and_then(Value::as_bool).unwrap_or(false),
key,
}
})
.collect();
s.submit_label = a
.get("submitLabel")
.or_else(|| a.get("submit_label"))
.and_then(Value::as_str)
.map(str::to_owned);
s.submit_action = a
.get("submitAction")
.or_else(|| a.get("submit_action"))
.and_then(Value::as_str)
.map(str::to_owned);
s
}
fn list(a: &Value, id: String) -> InlineSurface {
let mut s = InlineSurface::empty(id, SurfaceKind::List);
s.title = string(a, "title");
s.items = strings(a.get("items"));
s
}
fn metric(a: &Value, id: String) -> InlineSurface {
let mut s = InlineSurface::empty(id, SurfaceKind::Metric);
s.label = string(a, "label");
s.value = scalar(a.get("value"));
s
}
fn mindmap(a: &Value, id: String) -> InlineSurface {
let mut s = InlineSurface::empty(id, SurfaceKind::Mindmap);
s.title = string(a, "title");
s.nodes = array(a, "nodes")
.iter()
.map(|v| MindmapNode {
id: string(v, "id"),
label: string(v, "label").unwrap_or_default(),
children: strings(v.get("children")),
})
.collect();
s
}
fn table(a: &Value, id: String) -> InlineSurface {
let mut s = InlineSurface::empty(id, SurfaceKind::Table);
s.title = string(a, "title");
s.columns = strings(a.get("columns"));
s.rows = array(a, "rows").iter().map(|v| strings(Some(v))).collect();
s
}
fn tabs(a: &Value, id: String, state: &ChatSurfaceState) -> InlineSurface {
let mut s = InlineSurface::empty(id.clone(), SurfaceKind::Tabs);
s.title = string(a, "title");
s.tabs = array(a, "tabs")
.iter()
.enumerate()
.map(|(ti, tab)| TabPanel {
label: string(tab, "label").unwrap_or_default(),
content: array(tab, "content")
.iter()
.enumerate()
.map(|(ci, v)| nested(v, format!("{id}-tab-{ti}-{ci}"), state))
.collect(),
})
.collect();
s.selected_index = Some(
state
.surface_tabs
.get(&id)
.copied()
.unwrap_or(0)
.min(s.tabs.len().saturating_sub(1)),
);
s
}
fn nested(v: &Value, id: String, state: &ChatSurfaceState) -> InlineSurface {
let Some(object) = v.as_object() else {
let mut s = InlineSurface::empty(id, SurfaceKind::Text);
s.body = Some(scalar(Some(v)).unwrap_or_default());
return s;
};
match object.get("type").and_then(Value::as_str).unwrap_or("text") {
"card" => card(v, id),
"chart" => chart(v, id),
"form" => form(v, id, state),
"list" => list(v, id),
"metric" => metric(v, id),
"mindmap" => mindmap(v, id),
"table" => table(v, id),
"tabs" => tabs(v, id, state),
"text" => {
let mut s = InlineSurface::empty(id, SurfaceKind::Text);
s.body = string(v, "body").or_else(|| string(v, "text"));
s
}
_ => {
let mut s = InlineSurface::empty(id, SurfaceKind::Json);
s.raw = Some(Value::Object(object.clone()));
s
}
}
}
fn array<'a>(v: &'a Value, key: &str) -> &'a [Value] {
v.get(key)
.and_then(Value::as_array)
.map(Vec::as_slice)
.unwrap_or_default()
}
fn string(v: &Value, key: &str) -> Option<String> {
scalar(v.get(key))
}
fn strings(v: Option<&Value>) -> Vec<String> {
v.and_then(Value::as_array)
.into_iter()
.flatten()
.filter_map(|v| scalar(Some(v)))
.collect()
}
fn scalar(v: Option<&Value>) -> Option<String> {
match v? {
Value::Null => None,
Value::String(v) => Some(v.clone()),
Value::Bool(v) => Some(v.to_string()),
Value::Number(v) => Some(v.to_string()),
v => serde_json::to_string(v).ok(),
}
}
fn numeric(v: Option<&Value>) -> f64 {
v.and_then(|v| {
v.as_f64()
.or_else(|| v.as_str().and_then(|s| s.parse().ok()))
})
.filter(|v| v.is_finite())
.unwrap_or(0.0)
}
pub fn render_tool_result(name: &str, arguments: &Value) -> Option<Value> {
let kind = name.strip_prefix("render_")?;
RENDER_TOOL_NAMES.contains(&name).then(|| {
let mut values = arguments.as_object().cloned().unwrap_or_default();
values.insert("type".into(), Value::String(kind.into()));
Value::Object(values)
})
}

View File

@@ -9,7 +9,7 @@ use serde_json::{Value, json};
use crate::db::DbConnection as Connection;
use crate::db::queries::{media, post, post_link, post_media, script, template};
use crate::db::schema::ai_models;
use crate::engine::{EngineError, EngineResult};
use crate::engine::{EngineError, EngineResult, chat_surfaces};
use crate::util::frontmatter::read_post_file;
pub fn model_supports_tools(conn: &Connection, model: &str) -> EngineResult<bool> {
@@ -48,7 +48,7 @@ pub fn system_prompt(conn: &Connection, project_id: &str) -> EngineResult<String
.len();
let configured = crate::engine::settings::get(conn, "ai.system_prompt")?.unwrap_or_default();
let contract = format!(
"You are the conversational assistant for this blog project. Use tools when facts from the project are needed. Never invent project content or identifiers. There are {} posts, {media_count} media items, {tags} tags, and {categories} categories. Keep answers concise and use GitHub-flavored Markdown when useful.",
"You are the conversational assistant for this blog project. Use tools when facts from the project are needed. Never invent project content or identifiers. There are {} posts, {media_count} media items, {tags} tags, and {categories} categories. Keep answers concise and use GitHub-flavored Markdown when useful. Use render tools for structured data, comparisons, and forms; their payloads are native UI data, never HTML or JavaScript.",
posts.len()
);
Ok(if configured.trim().is_empty() {
@@ -168,6 +168,81 @@ pub fn tool_specs() -> Vec<Value> {
"value": {"type": "string"}
}),
),
spec(
"render_card",
"Render an information card with optional allow-listed actions.",
json!({
"title": {"type": "string"}, "subtitle": {"type": "string"}, "body": {"type": "string"},
"actions": {"type": "array", "items": {"type": "object", "properties": {
"label": {"type": "string"}, "action": {"type": "string"}, "payload": {"type": "object"}
}, "required": ["label", "action"]}}
}),
),
spec(
"render_chart",
"Render a native chart; heatmap and stacked-bar series use labelled segments.",
json!({
"chartType": {"type": "string", "enum": ["bar", "stacked-bar", "line", "area", "pie", "donut", "heatmap"]},
"chart_type": {"type": "string", "enum": ["bar", "stacked-bar", "line", "area", "pie", "donut", "heatmap"]},
"title": {"type": "string"},
"series": {"type": "array", "items": {"type": "object", "properties": {
"label": {"type": "string"}, "value": {"type": "number"},
"segments": {"type": "array", "items": {"type": "object", "properties": {
"label": {"type": "string"}, "value": {"type": "number"}
}, "required": ["label", "value"]}}
}, "required": ["label"]}}
}),
),
spec(
"render_form",
"Render a native form that submits its current values with an allow-listed action.",
json!({
"title": {"type": "string"},
"fields": {"type": "array", "items": {"type": "object", "properties": {
"key": {"type": "string"}, "label": {"type": "string"},
"inputType": {"type": "string", "enum": ["text", "textarea", "select", "checkbox", "date", "number"]},
"input_type": {"type": "string", "enum": ["text", "textarea", "select", "checkbox", "date", "number"]},
"placeholder": {"type": "string"}, "defaultValue": {}, "default_value": {}, "required": {"type": "boolean"},
"options": {"type": "array", "items": {"type": "object", "properties": {
"label": {"type": "string"}, "value": {"type": "string"}
}}}
}, "required": ["key", "label", "inputType"]}},
"submitLabel": {"type": "string"}, "submit_label": {"type": "string"},
"submitAction": {"type": "string"}, "submit_action": {"type": "string"}
}),
),
spec(
"render_list",
"Render a native list.",
json!({"title": {"type": "string"}, "items": {"type": "array", "items": {"type": "string"}}}),
),
spec(
"render_metric",
"Render a prominent metric.",
json!({"label": {"type": "string"}, "value": {"type": "string"}}),
),
spec(
"render_mindmap",
"Render a native hierarchical mind map.",
json!({"title": {"type": "string"}, "nodes": {"type": "array", "items": {"type": "object", "properties": {
"id": {"type": "string"}, "label": {"type": "string"}, "children": {"type": "array", "items": {"type": "string"}}
}, "required": ["label"]}}}),
),
spec(
"render_table",
"Render a native data table.",
json!({
"title": {"type": "string"}, "columns": {"type": "array", "items": {"type": "string"}},
"rows": {"type": "array", "items": {"type": "array", "items": {"type": "string"}}}
}),
),
spec(
"render_tabs",
"Render switchable tabs containing nested native surfaces or text.",
json!({"title": {"type": "string"}, "tabs": {"type": "array", "items": {"type": "object", "properties": {
"label": {"type": "string"}, "content": {"type": "array", "items": {"type": "object"}}
}, "required": ["label", "content"]}}}),
),
]
}
@@ -234,6 +309,10 @@ pub fn execute(
Ok(json!({"success": true, "script": item}))
}
"navigate" => navigate(arguments),
name if chat_surfaces::RENDER_TOOL_NAMES.contains(&name) => {
Ok(chat_surfaces::render_tool_result(name, arguments)
.expect("render tool allow-list and result builder must stay in sync"))
}
_ => Ok(json!({"success": false, "error": "unknown_tool", "name": name})),
}
}
@@ -858,4 +937,42 @@ mod tests {
);
assert!(matches!(invalid, Err(EngineError::Validation(_))));
}
#[test]
fn structured_render_tools_are_advertised_and_return_inert_native_data() {
let specs = tool_specs();
let names = specs
.iter()
.filter_map(|spec| spec.pointer("/function/name").and_then(Value::as_str))
.collect::<BTreeSet<_>>();
for name in chat_surfaces::RENDER_TOOL_NAMES {
assert!(names.contains(name), "missing tool schema for {name}");
}
let chart = specs
.iter()
.find(|spec| {
spec.pointer("/function/name").and_then(Value::as_str) == Some("render_chart")
})
.unwrap();
assert!(
chart
.pointer("/function/parameters/properties/chartType")
.is_some()
);
assert!(
chart
.pointer("/function/parameters/properties/chart_type")
.is_some()
);
let db = setup();
let dir = tempfile::tempdir().unwrap();
let raw = json!({"title": "<script>alert(1)</script>", "body": "<b>data</b>"});
let result = execute(db.conn(), dir.path(), "p1", "render_card", &raw).unwrap();
assert_eq!(result["type"], "card");
assert_eq!(result["title"], raw["title"]);
assert_eq!(result["body"], raw["body"]);
assert!(result.get("html").is_none());
assert!(result.get("javascript").is_none());
}
}

View File

@@ -3,6 +3,7 @@ pub mod auto_translation;
pub mod blogmark;
pub mod calendar;
pub mod chat;
pub mod chat_surfaces;
mod chat_tools;
pub mod cli_launcher;
pub mod cli_sync;

View File

@@ -57,6 +57,7 @@ pub struct ChatConversation {
pub title: String,
pub model: Option<String>,
pub copilot_session_id: Option<String>,
pub surface_state: Option<String>,
pub created_at: i64,
pub updated_at: i64,
}
@@ -68,6 +69,7 @@ pub struct NewChatConversation<'a> {
pub title: &'a str,
pub model: Option<&'a str>,
pub copilot_session_id: Option<&'a str>,
pub surface_state: Option<&'a str>,
pub created_at: i64,
pub updated_at: i64,
}

View File

@@ -397,6 +397,55 @@ fn tool_round_limit_stops_without_leaving_an_unpaired_tool_call() {
assert!(messages[1].tool_calls.as_deref().unwrap().contains("stats"));
}
#[test]
fn render_tool_events_include_stable_surface_identity_and_inert_arguments() {
let (_root, db, project_id, data_dir) = setup();
let conversation = chat::create_conversation(db.conn(), Some("tool-model")).unwrap();
let tool_frame = "data: {\"choices\":[{\"delta\":{\"tool_calls\":[{\"index\":0,\"id\":\"card\",\"function\":{\"name\":\"render_card\",\"arguments\":\"{\\\"title\\\":\\\"Live\\\",\\\"body\\\":\\\"<script>data only</script>\\\"}\"}}]}}]}\n\ndata: [DONE]\n\n";
let (url, server) = serve(vec![
MockResponse::sse(vec![tool_frame]),
MockResponse::sse(vec![
"data: {\"choices\":[{\"delta\":{\"content\":\"Done\"}}]}\n\n",
"data: [DONE]\n\n",
]),
]);
let events = Arc::new(std::sync::Mutex::new(Vec::new()));
let captured = events.clone();
chat::send_chat_message(
db.conn(),
&data_dir,
&project_id,
false,
&conversation.id,
"Show it",
options(url, move |event| captured.lock().unwrap().push(event)),
)
.unwrap();
server.join().unwrap();
let messages = chat::list_messages(db.conn(), &conversation.id).unwrap();
let assistant_call = messages
.iter()
.find(|message| message.tool_calls.is_some())
.unwrap();
let events = events.lock().unwrap();
let event = events
.iter()
.find_map(|event| match event {
chat::ChatEvent::ToolStarted {
name,
surface_id,
arguments,
..
} if name == "render_card" => Some((surface_id, arguments)),
_ => None,
})
.unwrap();
assert_eq!(event.0, &format!("{}-surface-0", assistant_call.id));
assert_eq!(event.1["title"], "Live");
assert_eq!(event.1["body"], "<script>data only</script>");
}
#[test]
fn cancellation_before_mutating_tool_execution_records_pairs_without_mutation() {
let (_root, db, project_id, data_dir) = setup();

View File

@@ -0,0 +1,241 @@
use std::collections::{BTreeMap, BTreeSet};
use bds_core::db::Database;
use bds_core::engine::chat;
use bds_core::engine::chat_surfaces::{
ChartType, ChatSurfaceState, FormInputType, SurfaceKind, build_render_surface, merge_form_data,
resolve_surface_action,
};
use serde_json::{Value, json};
#[test]
fn every_render_tool_builds_the_expected_fixed_surface_type() {
let state = ChatSurfaceState::default();
for (index, (tool, expected)) in [
("render_card", SurfaceKind::Card),
("render_chart", SurfaceKind::Chart),
("render_form", SurfaceKind::Form),
("render_list", SurfaceKind::List),
("render_metric", SurfaceKind::Metric),
("render_mindmap", SurfaceKind::Mindmap),
("render_table", SurfaceKind::Table),
("render_tabs", SurfaceKind::Tabs),
]
.into_iter()
.enumerate()
{
let surface =
build_render_surface(tool, &json!({}), format!("message-surface-{index}"), &state)
.unwrap();
assert_eq!(surface.kind, expected);
}
assert!(build_render_surface("read_post", &json!({}), "x".into(), &state).is_none());
}
#[test]
fn charts_accept_all_types_aliases_and_coerce_invalid_numbers_to_zero() {
let state = ChatSurfaceState::default();
for chart_type in [
ChartType::Bar,
ChartType::StackedBar,
ChartType::Line,
ChartType::Area,
ChartType::Pie,
ChartType::Donut,
ChartType::Heatmap,
] {
let surface = build_render_surface(
"render_chart",
&json!({
"chartType": chart_type.as_str(),
"series": [{
"label": "A",
"value": "invalid",
"segments": [{"label": "one", "value": {}}, {"label": "two", "value": "2.5"}]
}]
}),
"chart".into(),
&state,
)
.unwrap();
assert_eq!(surface.chart_type, Some(chart_type));
assert_eq!(surface.series[0].value, 0.0);
assert_eq!(surface.series[0].segments[0].value, 0.0);
assert_eq!(surface.series[0].segments[1].value, 2.5);
}
let defaulted =
build_render_surface("render_chart", &json!({}), "default".into(), &state).unwrap();
assert_eq!(defaulted.chart_type, Some(ChartType::Bar));
let legacy = build_render_surface(
"render_chart",
&json!({"chart_type": "pie"}),
"legacy".into(),
&state,
)
.unwrap();
assert_eq!(legacy.chart_type, Some(ChartType::Pie));
}
#[test]
fn forms_support_every_input_type_and_restore_current_values() {
let mut state = ChatSurfaceState::default();
state.surface_data.insert(
"form".into(),
BTreeMap::from([
("title".into(), json!("Restored")),
("enabled".into(), json!(true)),
]),
);
let surface = build_render_surface(
"render_form",
&json!({
"fields": [
{"key": "title", "label": "Title", "inputType": "text", "defaultValue": "Default"},
{"key": "body", "label": "Body", "inputType": "textarea"},
{"key": "kind", "label": "Kind", "inputType": "select", "options": [{"label": "Post", "value": "post"}]},
{"key": "enabled", "label": "Enabled", "inputType": "checkbox"},
{"key": "date", "label": "Date", "inputType": "date"},
{"key": "count", "label": "Count", "inputType": "number"}
],
"submitAction": "openPost"
}),
"form".into(),
&state,
)
.unwrap();
assert_eq!(
surface
.fields
.iter()
.map(|field| field.input_type)
.collect::<Vec<_>>(),
vec![
FormInputType::Text,
FormInputType::Textarea,
FormInputType::Select,
FormInputType::Checkbox,
FormInputType::Date,
FormInputType::Number,
]
);
assert_eq!(surface.fields[0].value, json!("Restored"));
assert_eq!(surface.fields[3].value, json!(true));
let payload = merge_form_data(json!({"postId": "post-1"}), "form", &state);
assert_eq!(payload["postId"], "post-1");
assert_eq!(payload["formData"]["title"], "Restored");
assert_eq!(payload["formData"]["enabled"], true);
}
#[test]
fn tabs_restore_selection_and_nested_unknown_content_is_safe_data() {
let state = ChatSurfaceState {
surface_tabs: BTreeMap::from([("tabs".into(), 1)]),
..Default::default()
};
let surface = build_render_surface(
"render_tabs",
&json!({"tabs": [
{"label": "Known", "content": [{"type": "text", "body": "<script>bad()</script>"}]},
{"label": "Fallback", "content": [{"type": "future", "html": "<b>not markup</b>"}, 42]}
]}),
"tabs".into(),
&state,
)
.unwrap();
assert_eq!(surface.selected_index, Some(1));
assert_eq!(surface.tabs[0].content[0].kind, SurfaceKind::Text);
assert_eq!(
surface.tabs[0].content[0].body.as_deref(),
Some("<script>bad()</script>")
);
assert_eq!(surface.tabs[1].content[0].kind, SurfaceKind::Json);
assert_eq!(surface.tabs[1].content[1].kind, SurfaceKind::Text);
}
#[test]
fn unknown_render_tool_is_inspectable_json_but_non_render_tools_are_not_surfaces() {
let state = ChatSurfaceState::default();
let raw = json!({"html": "<script>alert(1)</script>", "answer": 42});
let surface = build_render_surface("render_future", &raw, "future-0".into(), &state).unwrap();
assert_eq!(surface.kind, SurfaceKind::Json);
assert_eq!(surface.raw.as_ref(), Some(&raw));
assert!(build_render_surface("run_script", &raw, "bad-0".into(), &state).is_none());
}
#[test]
fn surface_state_persists_and_reopens_against_stable_ids() {
let db = Database::open_in_memory().unwrap();
db.migrate().unwrap();
let conversation = chat::create_conversation(db.conn(), Some("model")).unwrap();
let expected = ChatSurfaceState {
surface_data: BTreeMap::from([(
"17-surface-0".into(),
BTreeMap::from([("query".into(), json!("hello"))]),
)]),
surface_tabs: BTreeMap::from([("17-surface-1".into(), 2)]),
dismissed_surfaces: BTreeSet::from(["17-surface-2".into()]),
};
chat::put_surface_state(db.conn(), &conversation.id, &expected).unwrap();
assert_eq!(
chat::get_surface_state(db.conn(), &conversation.id).unwrap(),
expected
);
assert_eq!(
chat::get_surface_state(db.conn(), "missing").unwrap(),
ChatSurfaceState::default()
);
}
#[test]
fn assistant_action_allow_list_accepts_aliases_and_refuses_bad_payloads() {
for (action, payload, destination, entity_id) in [
(
"openPost",
json!({"postId": "post-1"}),
"posts",
Some("post-1"),
),
(
"open_media",
json!({"media_id": "media-1"}),
"media",
Some("media-1"),
),
("openSettings", json!({}), "settings", None),
(
"open_chat",
json!({"conversationId": "chat-1"}),
"chat",
Some("chat-1"),
),
(
"switchView",
json!({"view": "templates"}),
"templates",
None,
),
("set_view", json!({"view": "scripts"}), "scripts", None),
("toggleSidebar", json!({}), "toggle_sidebar", None),
("toggle_panel", json!({}), "toggle_panel", None),
(
"toggleAssistantSidebar",
json!({}),
"toggle_assistant_sidebar",
None,
),
] {
let navigation = resolve_surface_action(action, &payload).unwrap();
assert_eq!(navigation.destination, destination);
assert_eq!(navigation.entity_id.as_deref(), entity_id);
}
for (action, payload) in [
("runJavaScript", json!({"code": "alert(1)"})),
("openPost", json!({"postId": ""})),
("switchView", json!({"view": "root-shell"})),
("openMedia", Value::String("media-1".into())),
] {
assert!(resolve_surface_action(action, &payload).is_err());
}
}

View File

@@ -334,6 +334,26 @@ pub enum Message {
ChatSend,
ChatCancel,
ChatLinkClicked(String),
ChatSurfaceFieldChanged {
surface_id: String,
field: String,
value: serde_json::Value,
},
ChatSurfaceTextareaAction {
surface_id: String,
field: String,
action: iced::widget::text_editor::Action,
},
ChatSurfaceTabSelected {
surface_id: String,
index: usize,
},
ChatSurfaceDismissed(String),
ChatSurfaceAction {
surface_id: String,
action: String,
payload: serde_json::Value,
},
ChatFinished {
conversation_id: String,
result: Result<engine::chat::ChatTurnResult, String>,
@@ -1317,6 +1337,103 @@ impl BdsApp {
}
Task::none()
}
Message::ChatSurfaceFieldChanged {
surface_id,
field,
value,
} => {
if let Some(state) = self.active_chat_state_mut() {
state
.surface_state
.surface_data
.entry(surface_id)
.or_default()
.insert(field, value);
state.surface_state_dirty_since = Some(std::time::Instant::now());
state.rebuild_surfaces();
}
Task::none()
}
Message::ChatSurfaceTextareaAction {
surface_id,
field,
action,
} => {
if let Some(state) = self.active_chat_state_mut()
&& let Some(content) = state
.surface_textareas
.get_mut(&crate::views::chat_view::textarea_key(&surface_id, &field))
{
content.perform(action);
let value = content.text();
state
.surface_state
.surface_data
.entry(surface_id)
.or_default()
.insert(field, value.into());
state.surface_state_dirty_since = Some(std::time::Instant::now());
}
Task::none()
}
Message::ChatSurfaceTabSelected { surface_id, index } => {
let Some(conversation_id) = self.active_chat_id().map(str::to_string) else {
return Task::none();
};
if let Some(state) = self.chat_editors.get_mut(&conversation_id) {
state.surface_state.surface_tabs.insert(surface_id, index);
state.rebuild_surfaces();
}
self.persist_chat_surface_state(&conversation_id);
Task::none()
}
Message::ChatSurfaceDismissed(surface_id) => {
let Some(conversation_id) = self.active_chat_id().map(str::to_string) else {
return Task::none();
};
if let Some(state) = self.chat_editors.get_mut(&conversation_id) {
state.surface_state.dismissed_surfaces.insert(surface_id);
state.rebuild_surfaces();
}
self.persist_chat_surface_state(&conversation_id);
Task::none()
}
Message::ChatSurfaceAction {
surface_id,
action,
payload,
} => {
let result = self
.active_chat_id()
.and_then(|id| self.chat_editors.get(id))
.map(|state| {
let payload = engine::chat_surfaces::merge_form_data(
payload,
&surface_id,
&state.surface_state,
);
engine::chat_surfaces::resolve_surface_action(&action, &payload)
});
match result {
Some(Ok(navigation)) => self.dispatch_chat_navigation(
&navigation.destination,
navigation.entity_id.as_deref(),
),
Some(Err(reason)) => {
let message = tw(
self.ui_locale,
"chat.surface.actionRefused",
&[("reason", &reason)],
);
if let Some(state) = self.active_chat_state_mut() {
state.error = Some(message.clone());
}
self.notify(ToastLevel::Error, &message);
}
None => {}
}
Task::none()
}
Message::ChatFinished {
conversation_id,
result,
@@ -2011,6 +2128,7 @@ impl BdsApp {
self.task_manager.evict_expired();
self.refresh_task_snapshots();
self.process_chat_events();
self.persist_due_chat_surface_state();
if !self.search_index_rebuild_running {
self.auto_save_due_post_editors();
}
@@ -2654,6 +2772,48 @@ impl BdsApp {
.unwrap_or_default();
}
fn persist_due_chat_surface_state(&mut self) {
let due = self
.chat_editors
.iter()
.filter_map(|(id, state)| {
state.surface_state_dirty_since.and_then(|since| {
(since.elapsed() >= std::time::Duration::from_millis(500)).then(|| id.clone())
})
})
.collect::<Vec<_>>();
for id in due {
self.persist_chat_surface_state(&id);
}
}
fn persist_chat_surface_state(&mut self, conversation_id: &str) {
let Some(surface_state) = self
.chat_editors
.get(conversation_id)
.map(|state| state.surface_state.clone())
else {
return;
};
let result = self
.db
.as_ref()
.ok_or_else(|| "database unavailable".to_string())
.and_then(|db| {
engine::chat::put_surface_state(db.conn(), conversation_id, &surface_state)
.map_err(|error| error.to_string())
});
match result {
Ok(()) => {
if let Some(state) = self.chat_editors.get_mut(conversation_id) {
state.conversation.surface_state = serde_json::to_string(&surface_state).ok();
state.surface_state_dirty_since = None;
}
}
Err(error) => self.notify(ToastLevel::Error, &error),
}
}
fn chat_model_options(&self) -> Vec<ChatModelChoice> {
let mut models = self
.db
@@ -2826,9 +2986,12 @@ impl BdsApp {
engine::chat::ChatEvent::ToolStarted {
conversation_id,
name,
surface_id,
arguments,
} => {
if let Some(state) = self.chat_editors.get_mut(&conversation_id) {
state.active_tool = Some(name);
state.active_tool = Some(name.clone());
state.add_streaming_surface(&name, &arguments, surface_id);
}
}
engine::chat::ChatEvent::ToolFinished {
@@ -7800,6 +7963,7 @@ mod tests {
use crate::state::ToastLevel;
use crate::state::sidebar_filter::{MediaFilter, PostFilter};
use crate::state::tabs::{Tab, TabType};
use crate::views::chat_view::ChatEditorState;
use crate::views::media_editor::{MediaEditorMsg, MediaEditorState};
use crate::views::modal;
use crate::views::post_editor::{PostEditorMsg, PostEditorState};
@@ -7811,9 +7975,9 @@ mod tests {
use bds_core::db::queries::project::insert_project;
use bds_core::engine::generation::GenerationReport;
use bds_core::engine::task::{TaskStatus, TaskStatus::*};
use bds_core::engine::{ai, media, post, script, template, wordpress_import};
use bds_core::engine::{ai, chat, media, post, script, template, wordpress_import};
use bds_core::i18n::UiLocale;
use bds_core::model::{Project, ScriptKind, TemplateKind};
use bds_core::model::{ChatRole, Project, ScriptKind, TemplateKind};
use chrono::{Datelike, TimeZone};
use std::io::{Read, Write};
use std::net::TcpListener;
@@ -9725,4 +9889,116 @@ mod tests {
.is_empty()
);
}
#[test]
fn chat_surfaces_stream_persist_reopen_and_refuse_unknown_actions() {
let (db, project, tmp) = setup();
let conversation =
chat::create_conversation_titled(db.conn(), Some("test-model"), "Surfaces").unwrap();
let tool_calls = serde_json::json!([
{
"id": "form-call",
"type": "function",
"function": {
"name": "render_form",
"arguments": serde_json::json!({
"title": "Preferences",
"fields": [{"key": "topic", "label": "Topic", "inputType": "text"}],
"submitAction": "switchView"
}).to_string()
}
},
{
"id": "tabs-call",
"type": "function",
"function": {
"name": "render_tabs",
"arguments": serde_json::json!({
"tabs": [
{"label": "One", "content": [{"type": "text", "text": "First"}]},
{"label": "Two", "content": [{"type": "text", "text": "Second"}]}
]
}).to_string()
}
}
])
.to_string();
let message = chat::insert_message(
db.conn(),
&conversation.id,
ChatRole::Assistant,
Some("Structured answer"),
None,
Some(&tool_calls),
bds_core::engine::ai::TokenUsage::default(),
)
.unwrap();
let mut app = make_app(db, project, &tmp);
let tab = Tab {
id: conversation.id.clone(),
tab_type: TabType::Chat,
title: conversation.title.clone(),
is_transient: false,
is_dirty: false,
};
let _ = app.update(Message::OpenTab(tab));
let form_id = format!("{}-surface-0", message.id);
let tabs_id = format!("{}-surface-1", message.id);
{
let state = app.chat_editors.get_mut(&conversation.id).unwrap();
assert_eq!(state.message_surfaces[&message.id].len(), 2);
state.streaming = true;
state.add_streaming_surface(
"render_card",
&serde_json::json!({"title": "Later"}),
"later-message-surface-0".into(),
);
assert_eq!(state.message_surfaces[&message.id].len(), 2);
assert_eq!(state.streaming_surfaces.len(), 1);
}
let _ = app.update(Message::ChatSurfaceFieldChanged {
surface_id: form_id.clone(),
field: "topic".into(),
value: "Rust".into(),
});
app.chat_editors
.get_mut(&conversation.id)
.unwrap()
.surface_state_dirty_since =
Some(std::time::Instant::now() - std::time::Duration::from_millis(501));
let _ = app.update(Message::TaskTick);
let _ = app.update(Message::ChatSurfaceTabSelected {
surface_id: tabs_id.clone(),
index: 1,
});
let _ = app.update(Message::ChatSurfaceDismissed(form_id.clone()));
let reopened_conversation =
chat::get_conversation(app.db.as_ref().unwrap().conn(), &conversation.id).unwrap();
let reopened = ChatEditorState::new(
reopened_conversation,
chat::list_messages(app.db.as_ref().unwrap().conn(), &conversation.id).unwrap(),
vec![],
);
assert_eq!(
reopened.surface_state.surface_data[&form_id]["topic"],
"Rust"
);
assert_eq!(reopened.surface_state.surface_tabs[&tabs_id], 1);
assert!(reopened.surface_state.dismissed_surfaces.contains(&form_id));
assert_eq!(reopened.message_surfaces[&message.id].len(), 1);
let _ = app.update(Message::ChatSurfaceAction {
surface_id: form_id,
action: "runJavaScript".into(),
payload: serde_json::json!({"script": "alert(1)"}),
});
assert!(
app.chat_editors[&conversation.id]
.error
.as_deref()
.is_some_and(|error| error.contains("runJavaScript"))
);
}
}

View File

@@ -0,0 +1,926 @@
use std::collections::{HashMap, HashSet};
use std::f32::consts::{FRAC_PI_2, TAU};
use bds_core::engine::chat_surfaces::{
ChartSeries, ChartType, ChatSurfaceState, FormInputType, InlineSurface, MindmapNode,
SurfaceKind,
};
use bds_core::i18n::UiLocale;
use iced::widget::canvas::{self, Path, Stroke, path};
use iced::widget::{
Space, button, checkbox, column, container, row, scrollable, text, text_editor,
};
use iced::{
Alignment, Color, Element, Length, Point, Radians, Rectangle, Renderer, Size, Theme, mouse,
};
use crate::app::Message;
use crate::components::inputs;
use crate::i18n::t;
use crate::views::chat_view::textarea_key;
const CHART_COLORS: [Color; 7] = [
rgb8(0x4E, 0xA1, 0xE0),
rgb8(0x7B, 0xC9, 0x6F),
rgb8(0xF0, 0xB4, 0x4D),
rgb8(0xD9, 0x78, 0xAE),
rgb8(0x9B, 0x8A, 0xE6),
rgb8(0x54, 0xC6, 0xC0),
rgb8(0xE3, 0x73, 0x63),
];
const fn rgb8(red: u8, green: u8, blue: u8) -> Color {
Color {
r: red as f32 / 255.0,
g: green as f32 / 255.0,
b: blue as f32 / 255.0,
a: 1.0,
}
}
pub fn view<'a>(
surface: &'a InlineSurface,
state: &'a ChatSurfaceState,
textareas: &'a HashMap<String, text_editor::Content>,
locale: UiLocale,
) -> Element<'a, Message> {
surface_view(surface, state, textareas, locale, true)
}
fn surface_view<'a>(
surface: &'a InlineSurface,
state: &'a ChatSurfaceState,
textareas: &'a HashMap<String, text_editor::Content>,
locale: UiLocale,
dismissible: bool,
) -> Element<'a, Message> {
let has_title = surface.title.is_some();
let title = surface.title.clone().unwrap_or_else(|| {
t(
locale,
&format!("chat.surface.type.{}", surface.kind.as_str()),
)
});
let mut header = row![text(title).size(14), Space::with_width(Length::Fill),]
.spacing(8)
.align_y(Alignment::Center);
if has_title {
header = header.push(
text(t(
locale,
&format!("chat.surface.type.{}", surface.kind.as_str()),
))
.size(10)
.color(inputs::SECTION_COLOR),
);
}
if dismissible {
header = header.push(
button(text(t(locale, "chat.surface.dismiss")))
.on_press(Message::ChatSurfaceDismissed(surface.id.clone()))
.padding([4, 8])
.style(inputs::secondary_button),
);
}
let content = surface_content(surface, state, textareas, locale);
inputs::card(column![header, content].spacing(10)).into()
}
fn surface_content<'a>(
surface: &'a InlineSurface,
state: &'a ChatSurfaceState,
textareas: &'a HashMap<String, text_editor::Content>,
locale: UiLocale,
) -> Element<'a, Message> {
match surface.kind {
SurfaceKind::Card => {
let mut children = Vec::new();
if let Some(subtitle) = &surface.subtitle {
children.push(
text(subtitle.clone())
.size(12)
.color(inputs::SECTION_COLOR)
.into(),
);
}
if let Some(body) = &surface.body {
children.push(text(body.clone()).size(13).into());
}
if !surface.actions.is_empty() {
let actions = surface
.actions
.iter()
.fold(row![].spacing(8), |actions, item| {
actions.push(
button(text(item.label.clone()))
.on_press(Message::ChatSurfaceAction {
surface_id: surface.id.clone(),
action: item.action.clone(),
payload: item.payload.clone(),
})
.padding([7, 10])
.style(inputs::secondary_button),
)
});
children.push(actions.into());
}
empty_or_column(children, locale)
}
SurfaceKind::Chart => chart(surface),
SurfaceKind::Form => form(surface, textareas, locale),
SurfaceKind::List => {
let children = surface
.items
.iter()
.map(|item| text(format!("{item}")).size(13).into())
.collect();
empty_or_column(children, locale)
}
SurfaceKind::Metric => column![
text(surface.label.clone().unwrap_or_default())
.size(12)
.color(inputs::SECTION_COLOR),
text(surface.value.clone().unwrap_or_default()).size(28),
]
.spacing(4)
.into(),
SurfaceKind::Mindmap => mindmap(surface, locale),
SurfaceKind::Table => table(surface, locale),
SurfaceKind::Tabs => tabs(surface, state, textareas, locale),
SurfaceKind::Text => text(surface.body.clone().unwrap_or_default())
.size(13)
.into(),
SurfaceKind::Json => text(
surface
.raw
.as_ref()
.and_then(|value| serde_json::to_string_pretty(value).ok())
.unwrap_or_default(),
)
.size(12)
.shaping(iced::widget::text::Shaping::Advanced)
.into(),
}
}
fn empty_or_column<'a>(
mut children: Vec<Element<'a, Message>>,
locale: UiLocale,
) -> Element<'a, Message> {
if children.is_empty() {
children.push(
text(t(locale, "chat.surface.empty"))
.size(12)
.color(inputs::SECTION_COLOR)
.into(),
);
}
iced::widget::Column::with_children(children)
.spacing(7)
.into()
}
fn form<'a>(
surface: &'a InlineSurface,
textareas: &'a HashMap<String, text_editor::Content>,
locale: UiLocale,
) -> Element<'a, Message> {
let mut children: Vec<Element<'a, Message>> = Vec::new();
for field in &surface.fields {
let label = if field.required {
format!("{} *", field.label)
} else {
field.label.clone()
};
let surface_id = surface.id.clone();
let key = field.key.clone();
let control: Element<'a, Message> = match field.input_type {
FormInputType::Checkbox => checkbox(label, field.value.as_bool().unwrap_or(false))
.on_toggle(move |value| Message::ChatSurfaceFieldChanged {
surface_id: surface_id.clone(),
field: key.clone(),
value: value.into(),
})
.size(16)
.text_size(13)
.into(),
FormInputType::Select => {
let selected = field.options.iter().find(|option| {
field
.value
.as_str()
.is_some_and(|value| value == option.value)
});
inputs::labeled_select(&label, &field.options, selected, move |option| {
Message::ChatSurfaceFieldChanged {
surface_id: surface_id.clone(),
field: key.clone(),
value: option.value.into(),
}
})
}
FormInputType::Textarea => {
let content_key = textarea_key(&surface.id, &field.key);
if let Some(content) = textareas.get(&content_key) {
column![
text(label).size(12).color(inputs::LABEL_COLOR),
text_editor(content)
.placeholder(field.placeholder.clone().unwrap_or_default())
.on_action(move |action| Message::ChatSurfaceTextareaAction {
surface_id: surface_id.clone(),
field: key.clone(),
action,
})
.height(Length::Fixed(96.0))
.style(inputs::text_editor_style),
]
.spacing(6)
.into()
} else {
text(t(locale, "chat.surface.empty")).into()
}
}
FormInputType::Text | FormInputType::Date | FormInputType::Number => {
let value = match &field.value {
serde_json::Value::String(value) => value.clone(),
serde_json::Value::Number(value) => value.to_string(),
_ => String::new(),
};
let placeholder = field.placeholder.clone().unwrap_or_else(|| {
t(
locale,
match field.input_type {
FormInputType::Date => "chat.surface.form.datePlaceholder",
FormInputType::Number => "chat.surface.form.numberPlaceholder",
_ => "chat.surface.form.textPlaceholder",
},
)
});
inputs::labeled_input(&label, &placeholder, &value, move |value| {
let value = if field.input_type == FormInputType::Number {
value
.parse::<serde_json::Number>()
.map(serde_json::Value::Number)
.unwrap_or_else(|_| value.into())
} else {
value.into()
};
Message::ChatSurfaceFieldChanged {
surface_id: surface_id.clone(),
field: key.clone(),
value,
}
})
}
};
children.push(control);
}
if let Some(action) = &surface.submit_action {
children.push(
button(text(
surface
.submit_label
.clone()
.unwrap_or_else(|| t(locale, "chat.surface.form.submit")),
))
.on_press(Message::ChatSurfaceAction {
surface_id: surface.id.clone(),
action: action.clone(),
payload: serde_json::json!({}),
})
.padding([8, 12])
.style(inputs::primary_button)
.into(),
);
}
empty_or_column(children, locale)
}
fn table<'a>(surface: &'a InlineSurface, locale: UiLocale) -> Element<'a, Message> {
if surface.columns.is_empty() && surface.rows.is_empty() {
return text(t(locale, "chat.surface.empty"))
.size(12)
.color(inputs::SECTION_COLOR)
.into();
}
let header = iced::widget::Row::with_children(
surface
.columns
.iter()
.map(|value| {
container(text(value.clone()).size(12))
.width(Length::Fixed(140.0))
.into()
})
.collect::<Vec<_>>(),
)
.spacing(8);
let mut body: Vec<Element<'a, Message>> = vec![header.into()];
for values in &surface.rows {
body.push(
iced::widget::Row::with_children(
values
.iter()
.map(|value| {
container(text(value.clone()).size(12))
.width(Length::Fixed(140.0))
.into()
})
.collect::<Vec<_>>(),
)
.spacing(8)
.into(),
);
}
scrollable(iced::widget::Column::with_children(body).spacing(6))
.direction(scrollable::Direction::Horizontal(
scrollable::Scrollbar::default(),
))
.into()
}
fn mindmap<'a>(surface: &'a InlineSurface, locale: UiLocale) -> Element<'a, Message> {
let rows = mindmap_rows(&surface.nodes);
if rows.is_empty() {
return text(t(locale, "chat.surface.empty"))
.size(12)
.color(inputs::SECTION_COLOR)
.into();
}
iced::widget::Column::with_children(
rows.into_iter()
.map(|(depth, label)| {
row![
Space::with_width(Length::Fixed(depth as f32 * 18.0)),
text(if depth == 0 { "" } else { "" })
.size(11)
.color(inputs::SECTION_COLOR),
text(label).size(13),
]
.spacing(6)
.into()
})
.collect::<Vec<_>>(),
)
.spacing(6)
.into()
}
fn mindmap_rows(nodes: &[MindmapNode]) -> Vec<(usize, String)> {
let by_id = nodes
.iter()
.filter_map(|node| node.id.as_deref().map(|id| (id, node)))
.collect::<HashMap<_, _>>();
let children = nodes
.iter()
.flat_map(|node| node.children.iter().map(String::as_str))
.collect::<HashSet<_>>();
let roots = nodes
.iter()
.filter(|node| node.id.as_deref().is_none_or(|id| !children.contains(id)))
.collect::<Vec<_>>();
let mut rows = Vec::new();
let mut visited = HashSet::new();
fn walk(
node: &MindmapNode,
depth: usize,
by_id: &HashMap<&str, &MindmapNode>,
visited: &mut HashSet<String>,
rows: &mut Vec<(usize, String)>,
) {
if let Some(id) = &node.id
&& !visited.insert(id.clone())
{
return;
}
rows.push((depth, node.label.clone()));
for child in &node.children {
if let Some(node) = by_id.get(child.as_str()) {
walk(node, depth + 1, by_id, visited, rows);
}
}
}
for root in roots {
walk(root, 0, &by_id, &mut visited, &mut rows);
}
for node in nodes {
let missing = node.id.as_ref().is_some_and(|id| !visited.contains(id));
if missing {
walk(node, 0, &by_id, &mut visited, &mut rows);
}
}
rows
}
fn tabs<'a>(
surface: &'a InlineSurface,
state: &'a ChatSurfaceState,
textareas: &'a HashMap<String, text_editor::Content>,
locale: UiLocale,
) -> Element<'a, Message> {
if surface.tabs.is_empty() {
return text(t(locale, "chat.surface.empty")).into();
}
let selected = state
.surface_tabs
.get(&surface.id)
.copied()
.or(surface.selected_index)
.unwrap_or(0)
.min(surface.tabs.len() - 1);
let controls =
surface
.tabs
.iter()
.enumerate()
.fold(row![].spacing(6), |controls, (index, tab)| {
controls.push(
button(text(tab.label.clone()))
.on_press(Message::ChatSurfaceTabSelected {
surface_id: surface.id.clone(),
index,
})
.padding([6, 10])
.style(if index == selected {
inputs::primary_button
} else {
inputs::secondary_button
}),
)
});
let content = iced::widget::Column::with_children(
surface.tabs[selected]
.content
.iter()
.map(|child| surface_view(child, state, textareas, locale, false))
.collect::<Vec<_>>(),
)
.spacing(8);
column![controls, content].spacing(8).into()
}
fn chart<'a>(surface: &'a InlineSurface) -> Element<'a, Message> {
let chart_type = surface.chart_type.unwrap_or(ChartType::Bar);
let canvas = canvas::Canvas::new(NativeChart {
chart_type,
series: surface.series.clone(),
})
.width(Length::Fill)
.height(Length::Fixed(180.0));
let mut legend_items = Vec::new();
if chart_type == ChartType::Heatmap {
let (columns, rows) = heatmap_layout(&surface.series);
for (row_label, values) in rows {
for (column, value) in columns.iter().zip(values) {
legend_items.push(format!(
"{row_label} / {column}{}",
display_number(value)
));
}
}
} else if chart_type == ChartType::StackedBar {
for item in &surface.series {
if item.segments.is_empty() {
legend_items.push(format!("{}{}", item.label, display_number(item.value)));
} else {
for segment in &item.segments {
legend_items.push(format!(
"{} / {}{}",
item.label,
segment.label,
display_number(segment.value)
));
}
}
}
} else {
legend_items.extend(surface.series.iter().map(|item| {
format!(
"{}{}",
item.label,
display_number(chart_series_total(chart_type, item))
)
}));
}
let legend = legend_items.into_iter().enumerate().fold(
iced::widget::Column::new().spacing(4),
|legend, (index, item)| {
legend.push(
text(item)
.size(11)
.color(CHART_COLORS[index % CHART_COLORS.len()]),
)
},
);
column![canvas, legend].spacing(8).into()
}
fn display_number(value: f64) -> String {
if value.fract() == 0.0 {
format!("{value:.0}")
} else {
format!("{value:.2}")
}
}
fn chart_series_total(chart_type: ChartType, series: &ChartSeries) -> f64 {
if chart_type == ChartType::StackedBar && !series.segments.is_empty() {
series.segments.iter().map(|segment| segment.value).sum()
} else {
series.value
}
}
#[derive(Debug, Clone)]
struct NativeChart {
chart_type: ChartType,
series: Vec<ChartSeries>,
}
impl<Message> canvas::Program<Message> for NativeChart {
type State = ();
fn draw(
&self,
_state: &Self::State,
renderer: &Renderer,
_theme: &Theme,
bounds: Rectangle,
_cursor: mouse::Cursor,
) -> Vec<canvas::Geometry> {
let mut frame = canvas::Frame::new(renderer, bounds.size());
match self.chart_type {
ChartType::Bar => draw_bars(&mut frame, &self.series, false),
ChartType::StackedBar => draw_bars(&mut frame, &self.series, true),
ChartType::Line => draw_line(&mut frame, &self.series, false),
ChartType::Area => draw_line(&mut frame, &self.series, true),
ChartType::Pie => draw_pie(&mut frame, &self.series, false),
ChartType::Donut => draw_pie(&mut frame, &self.series, true),
ChartType::Heatmap => draw_heatmap(&mut frame, &self.series),
}
vec![frame.into_geometry()]
}
}
fn positive_max(series: &[ChartSeries], stacked: bool) -> f32 {
series
.iter()
.map(|item| {
if stacked && !item.segments.is_empty() {
item.segments.iter().map(|part| part.value.max(0.0)).sum()
} else {
item.value.max(0.0)
}
})
.fold(0.0_f64, f64::max)
.max(1.0) as f32
}
fn draw_bars(frame: &mut canvas::Frame, series: &[ChartSeries], stacked: bool) {
if series.is_empty() {
return;
}
let size = frame.size();
let gap = 8.0;
let width = ((size.width - gap * (series.len() as f32 + 1.0)) / series.len() as f32).max(3.0);
let max = positive_max(series, stacked);
for (index, item) in series.iter().enumerate() {
let x = gap + index as f32 * (width + gap);
if stacked && !item.segments.is_empty() {
let mut y = size.height - 8.0;
for (part_index, part) in item.segments.iter().enumerate() {
let height = (part.value.max(0.0) as f32 / max) * (size.height - 16.0);
y -= height;
frame.fill(
&Path::rectangle(Point::new(x, y), Size::new(width, height)),
CHART_COLORS[part_index % CHART_COLORS.len()],
);
}
} else {
let height = (item.value.max(0.0) as f32 / max) * (size.height - 16.0);
frame.fill(
&Path::rectangle(
Point::new(x, size.height - height - 8.0),
Size::new(width, height),
),
CHART_COLORS[index % CHART_COLORS.len()],
);
}
}
}
fn chart_points(size: Size, series: &[ChartSeries]) -> Vec<Point> {
let max = positive_max(series, false);
let denominator = series.len().saturating_sub(1).max(1) as f32;
series
.iter()
.enumerate()
.map(|(index, item)| {
Point::new(
8.0 + index as f32 / denominator * (size.width - 16.0),
size.height - 8.0 - item.value.max(0.0) as f32 / max * (size.height - 16.0),
)
})
.collect()
}
fn draw_line(frame: &mut canvas::Frame, series: &[ChartSeries], area: bool) {
let points = chart_points(frame.size(), series);
let Some(first) = points.first().copied() else {
return;
};
let line = Path::new(|builder| {
builder.move_to(first);
for point in points.iter().skip(1) {
builder.line_to(*point);
}
});
if area {
let size = frame.size();
let fill = Path::new(|builder| {
builder.move_to(Point::new(first.x, size.height - 8.0));
builder.line_to(first);
for point in points.iter().skip(1) {
builder.line_to(*point);
}
builder.line_to(Point::new(
points.last().map_or(first.x, |point| point.x),
size.height - 8.0,
));
builder.close();
});
frame.fill(&fill, CHART_COLORS[0].scale_alpha(0.28));
}
frame.stroke(
&line,
Stroke::default()
.with_color(CHART_COLORS[0])
.with_width(2.0)
.with_line_cap(canvas::LineCap::Round),
);
for point in points {
frame.fill(&Path::circle(point, 3.0), CHART_COLORS[0]);
}
}
fn draw_pie(frame: &mut canvas::Frame, series: &[ChartSeries], donut: bool) {
let values = series
.iter()
.map(|item| item.value.max(0.0) as f32)
.collect::<Vec<_>>();
let total = values.iter().sum::<f32>();
if total <= 0.0 {
return;
}
let center = frame.center();
let radius = (frame.width().min(frame.height()) / 2.0 - 8.0).max(1.0);
let mut start = -FRAC_PI_2;
for (index, value) in values.into_iter().enumerate() {
let end = start + value / total * TAU;
let wedge = Path::new(|builder| {
builder.move_to(center);
builder.line_to(Point::new(
center.x + radius * start.cos(),
center.y + radius * start.sin(),
));
builder.arc(path::Arc {
center,
radius,
start_angle: Radians(start),
end_angle: Radians(end),
});
builder.close();
});
frame.fill(&wedge, CHART_COLORS[index % CHART_COLORS.len()]);
start = end;
}
if donut {
frame.fill(
&Path::circle(center, radius * 0.54),
Color::from_rgb8(0x25, 0x25, 0x26),
);
}
}
fn draw_heatmap(frame: &mut canvas::Frame, series: &[ChartSeries]) {
let (columns, rows) = heatmap_layout(series);
if columns.is_empty() || rows.is_empty() {
return;
}
let cell_width = frame.width() / columns.len() as f32;
let cell_height = frame.height() / rows.len() as f32;
let max = rows
.iter()
.flat_map(|(_, values)| values)
.copied()
.fold(0.0_f64, f64::max)
.max(1.0);
for (row_index, (_, values)) in rows.iter().enumerate() {
for (column_index, value) in values.iter().enumerate() {
let alpha = if *value <= 0.0 {
0.08
} else {
0.18 + 0.82 * *value as f32 / max as f32
};
frame.fill(
&Path::rectangle(
Point::new(
column_index as f32 * cell_width + 2.0,
row_index as f32 * cell_height + 2.0,
),
Size::new(cell_width - 4.0, cell_height - 4.0),
),
CHART_COLORS[0].scale_alpha(alpha),
);
}
}
}
fn heatmap_layout(series: &[ChartSeries]) -> (Vec<String>, Vec<(String, Vec<f64>)>) {
let mut columns = Vec::new();
for item in series {
for segment in &item.segments {
if !columns.contains(&segment.label) {
columns.push(segment.label.clone());
}
}
}
let rows = series
.iter()
.filter(|item| !item.segments.is_empty())
.map(|item| {
let values = columns
.iter()
.map(|column| {
item.segments
.iter()
.find(|segment| segment.label == *column)
.map_or(0.0, |segment| segment.value)
})
.collect();
(item.label.clone(), values)
})
.collect();
(columns, rows)
}
#[cfg(test)]
mod tests {
use super::*;
use bds_core::engine::chat_surfaces::build_render_surface;
#[test]
fn chart_math_handles_every_fixed_chart_type() {
let series = vec![ChartSeries {
label: "A".into(),
value: 3.0,
segments: vec![],
}];
for chart_type in [
ChartType::Bar,
ChartType::StackedBar,
ChartType::Line,
ChartType::Area,
ChartType::Pie,
ChartType::Donut,
ChartType::Heatmap,
] {
assert_eq!(chart_series_total(chart_type, &series[0]), 3.0);
}
assert_eq!(chart_points(Size::new(100.0, 100.0), &series).len(), 1);
}
#[test]
fn heatmap_uses_labelled_segments_and_zero_fills_missing_cells() {
let series = vec![
ChartSeries {
label: "First".into(),
value: 0.0,
segments: vec![
bds_core::engine::chat_surfaces::ChartSegment {
label: "A".into(),
value: 2.0,
},
bds_core::engine::chat_surfaces::ChartSegment {
label: "B".into(),
value: 4.0,
},
],
},
ChartSeries {
label: "Second".into(),
value: 0.0,
segments: vec![bds_core::engine::chat_surfaces::ChartSegment {
label: "B".into(),
value: 3.0,
}],
},
];
let (columns, rows) = heatmap_layout(&series);
assert_eq!(columns, ["A", "B"]);
assert_eq!(rows[0], ("First".into(), vec![2.0, 4.0]));
assert_eq!(rows[1], ("Second".into(), vec![0.0, 3.0]));
}
#[test]
fn mindmap_is_cycle_safe_and_keeps_orphans() {
let nodes = vec![
MindmapNode {
id: Some("a".into()),
label: "A".into(),
children: vec!["b".into()],
},
MindmapNode {
id: Some("b".into()),
label: "B".into(),
children: vec!["a".into()],
},
MindmapNode {
id: None,
label: "Loose".into(),
children: vec![],
},
];
let rows = mindmap_rows(&nodes);
assert_eq!(rows.len(), 3);
assert!(rows.iter().any(|(_, label)| label == "Loose"));
}
#[test]
fn every_surface_chart_and_form_control_builds_a_native_widget() {
let state = ChatSurfaceState::default();
let form = serde_json::json!({
"fields": [
{"key": "text", "label": "Text", "inputType": "text"},
{"key": "notes", "label": "Notes", "inputType": "textarea"},
{"key": "choice", "label": "Choice", "inputType": "select", "options": [{"label": "A", "value": "a"}]},
{"key": "enabled", "label": "Enabled", "inputType": "checkbox"},
{"key": "date", "label": "Date", "inputType": "date"},
{"key": "count", "label": "Count", "inputType": "number"}
],
"submitAction": "switchView"
});
let cases = [
(
"render_card",
serde_json::json!({"body": "<script>plain text</script>"}),
),
("render_form", form),
("render_list", serde_json::json!({"items": ["One"]})),
(
"render_metric",
serde_json::json!({"label": "Count", "value": 2}),
),
(
"render_mindmap",
serde_json::json!({"nodes": [{"id": "a", "label": "A"}]}),
),
(
"render_table",
serde_json::json!({"columns": ["A"], "rows": [["B"]]}),
),
(
"render_tabs",
serde_json::json!({"tabs": [{"label": "Mixed", "content": ["plain", {"type": "future", "html": "<b>data</b>"}]}]}),
),
];
for (index, (name, arguments)) in cases.into_iter().enumerate() {
let surface =
build_render_surface(name, &arguments, format!("surface-{index}"), &state).unwrap();
let mut textareas = HashMap::new();
if name == "render_form" {
textareas.insert(
textarea_key(&surface.id, "notes"),
text_editor::Content::new(),
);
}
let _: Element<'_, Message> = view(&surface, &state, &textareas, UiLocale::En);
}
for (index, chart_type) in [
"bar",
"stacked-bar",
"line",
"area",
"pie",
"donut",
"heatmap",
]
.into_iter()
.enumerate()
{
let surface = build_render_surface(
"render_chart",
&serde_json::json!({
"chartType": chart_type,
"series": [{"label": "A", "value": 2, "segments": [{"label": "X", "value": 2}]}]
}),
format!("chart-{index}"),
&state,
)
.unwrap();
let textareas = HashMap::new();
let _: Element<'_, Message> = view(&surface, &state, &textareas, UiLocale::En);
}
}
}

View File

@@ -1,3 +1,9 @@
use std::collections::HashMap;
use std::time::Instant;
use bds_core::engine::chat_surfaces::{
ChatSurfaceState, InlineSurface, build_message_surfaces, build_render_surface,
};
use bds_core::i18n::UiLocale;
use bds_core::model::{ChatConversation, ChatMessage, ChatRole};
use iced::widget::text::Shaping;
@@ -22,6 +28,11 @@ pub struct ChatEditorState {
streaming_markdown: Vec<markdown::Item>,
pub active_tool: Option<String>,
pub error: Option<String>,
pub surface_state: ChatSurfaceState,
pub message_surfaces: HashMap<i32, Vec<InlineSurface>>,
pub streaming_surfaces: Vec<InlineSurface>,
pub surface_textareas: HashMap<String, text_editor::Content>,
pub surface_state_dirty_since: Option<Instant>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
@@ -62,6 +73,13 @@ impl ChatEditorState {
)
})
.collect();
let surface_state = conversation
.surface_state
.as_deref()
.and_then(|value| serde_json::from_str(value).ok())
.unwrap_or_default();
let message_surfaces = build_surfaces(&messages, &surface_state);
let surface_textareas = build_textareas(&message_surfaces);
Self {
rename_input: conversation.title.clone(),
conversation,
@@ -74,6 +92,11 @@ impl ChatEditorState {
streaming_markdown: Vec::new(),
active_tool: None,
error: None,
surface_state,
message_surfaces,
streaming_surfaces: Vec::new(),
surface_textareas,
surface_state_dirty_since: None,
}
}
@@ -89,6 +112,7 @@ impl ChatEditorState {
})
.collect();
self.messages = messages;
self.rebuild_surfaces();
}
pub fn set_streaming_content(&mut self, content: String) {
@@ -99,6 +123,24 @@ impl ChatEditorState {
pub fn clear_streaming(&mut self) {
self.streaming_content.clear();
self.streaming_markdown.clear();
self.streaming_surfaces.clear();
}
pub fn add_streaming_surface(&mut self, name: &str, arguments: &serde_json::Value, id: String) {
if let Some(surface) = build_render_surface(name, arguments, id, &self.surface_state)
&& !self.surface_state.dismissed_surfaces.contains(&surface.id)
{
add_textareas(&mut self.surface_textareas, &surface);
self.streaming_surfaces.push(surface);
}
}
pub fn rebuild_surfaces(&mut self) {
self.message_surfaces = build_surfaces(&self.messages, &self.surface_state);
self.surface_textareas = build_textareas(&self.message_surfaces);
for surface in &self.streaming_surfaces {
add_textareas(&mut self.surface_textareas, surface);
}
}
pub fn token_totals(&self) -> (u64, u64, u64, u64) {
@@ -114,6 +156,50 @@ impl ChatEditorState {
}
}
fn build_surfaces(
messages: &[ChatMessage],
state: &ChatSurfaceState,
) -> HashMap<i32, Vec<InlineSurface>> {
messages
.iter()
.filter_map(|message| {
let surfaces = build_message_surfaces(message, state);
(!surfaces.is_empty()).then_some((message.id, surfaces))
})
.collect()
}
fn build_textareas(
surfaces: &HashMap<i32, Vec<InlineSurface>>,
) -> HashMap<String, text_editor::Content> {
let mut result = HashMap::new();
for surface in surfaces.values().flatten() {
add_textareas(&mut result, surface);
}
result
}
fn add_textareas(result: &mut HashMap<String, text_editor::Content>, surface: &InlineSurface) {
for field in &surface.fields {
if field.input_type == bds_core::engine::chat_surfaces::FormInputType::Textarea {
result
.entry(textarea_key(&surface.id, &field.key))
.or_insert_with(|| {
text_editor::Content::with_text(field.value.as_str().unwrap_or_default())
});
}
}
for tab in &surface.tabs {
for child in &tab.content {
add_textareas(result, child);
}
}
}
pub fn textarea_key(surface_id: &str, field: &str) -> String {
format!("{surface_id}\0{field}")
}
pub fn view<'a>(
state: &'a ChatEditorState,
locale: UiLocale,
@@ -197,6 +283,19 @@ pub fn view<'a>(
state.rendered_messages.get(&message.id),
locale,
));
if let Some(surfaces) = state.message_surfaces.get(&message.id) {
for surface in surfaces {
message_elements.push(crate::views::chat_surfaces::view(
surface,
&state.surface_state,
&state.surface_textareas,
locale,
));
}
}
if let Some(markers) = tool_markers_view(message, locale) {
message_elements.push(markers);
}
}
}
if state.streaming && !state.streaming_content.is_empty() {
@@ -205,6 +304,14 @@ pub fn view<'a>(
t(locale, "chat.streaming"),
));
}
for surface in &state.streaming_surfaces {
message_elements.push(crate::views::chat_surfaces::view(
surface,
&state.surface_state,
&state.surface_textareas,
locale,
));
}
if let Some(tool) = state.active_tool.as_deref() {
message_elements.push(
inputs::card(
@@ -341,10 +448,18 @@ fn message_view<'a>(
.shaping(Shaping::Advanced)
.into()
};
let mut children: Vec<Element<'a, Message>> = vec![
let children: Vec<Element<'a, Message>> = vec![
text(label).size(11).color(inputs::SECTION_COLOR).into(),
body,
];
inputs::card(iced::widget::Column::with_children(children).spacing(6)).into()
}
fn tool_markers_view<'a>(
message: &'a ChatMessage,
locale: UiLocale,
) -> Option<Element<'a, Message>> {
let mut markers = Vec::new();
if let Some(tool_calls) = message.tool_calls.as_deref()
&& let Ok(calls) = serde_json::from_str::<Vec<serde_json::Value>>(tool_calls)
{
@@ -371,10 +486,11 @@ fn message_view<'a>(
args_preview
)
};
children.push(text(marker).size(11).color(inputs::SECTION_COLOR).into());
markers.push(text(marker).size(11).color(inputs::SECTION_COLOR).into());
}
}
inputs::card(iced::widget::Column::with_children(children).spacing(6)).into()
(!markers.is_empty())
.then(|| inputs::card(iced::widget::Column::with_children(markers).spacing(4)).into())
}
fn assistant_card<'a>(items: &'a [markdown::Item], label: String) -> Element<'a, Message> {

View File

@@ -1,4 +1,5 @@
pub mod activity_bar;
pub mod chat_surfaces;
pub mod chat_view;
pub mod dashboard;
pub mod git;

View File

@@ -78,6 +78,23 @@ chat-role-tool = Werkzeug
chat-deleted = Unterhaltung gelöscht
chat-link-refused = Dieser Linktyp ist nicht erlaubt
chat-navigation-invalid = Das angeforderte Blog-Element konnte nicht geöffnet werden
chat-surface-dismiss = Ausblenden
chat-surface-empty = Kein Inhalt
chat-surface-type-card = Karte
chat-surface-type-chart = Diagramm
chat-surface-type-form = Formular
chat-surface-type-list = Liste
chat-surface-type-metric = Kennzahl
chat-surface-type-mindmap = Mindmap
chat-surface-type-table = Tabelle
chat-surface-type-tabs = Registerkarten
chat-surface-type-text = Text
chat-surface-type-json = Daten
chat-surface-form-datePlaceholder = JJJJ-MM-TT
chat-surface-form-numberPlaceholder = Zahl eingeben
chat-surface-form-textPlaceholder = Wert eingeben
chat-surface-form-submit = Absenden
chat-surface-actionRefused = Assistentenaktion abgelehnt: { $reason }
statusBar-chatTokens = KI ↑{ $input } ↓{ $output } Cache { $cacheRead }/{ $cacheWrite }
common-settings = Einstellungen
common-tasks = Aufgaben

View File

@@ -78,6 +78,23 @@ chat-role-tool = Tool
chat-deleted = Conversation deleted
chat-link-refused = This link type is not allowed
chat-navigation-invalid = The requested blog item could not be opened
chat-surface-dismiss = Dismiss
chat-surface-empty = No content
chat-surface-type-card = Card
chat-surface-type-chart = Chart
chat-surface-type-form = Form
chat-surface-type-list = List
chat-surface-type-metric = Metric
chat-surface-type-mindmap = Mind map
chat-surface-type-table = Table
chat-surface-type-tabs = Tabs
chat-surface-type-text = Text
chat-surface-type-json = Data
chat-surface-form-datePlaceholder = YYYY-MM-DD
chat-surface-form-numberPlaceholder = Enter a number
chat-surface-form-textPlaceholder = Enter a value
chat-surface-form-submit = Submit
chat-surface-actionRefused = Assistant action refused: { $reason }
statusBar-chatTokens = AI ↑{ $input } ↓{ $output } cache { $cacheRead }/{ $cacheWrite }
common-settings = Settings
common-tasks = Tasks

View File

@@ -78,6 +78,23 @@ chat-role-tool = Herramienta
chat-deleted = Conversación eliminada
chat-link-refused = Este tipo de enlace no está permitido
chat-navigation-invalid = No se pudo abrir el elemento del blog solicitado
chat-surface-dismiss = Descartar
chat-surface-empty = Sin contenido
chat-surface-type-card = Tarjeta
chat-surface-type-chart = Gráfico
chat-surface-type-form = Formulario
chat-surface-type-list = Lista
chat-surface-type-metric = Métrica
chat-surface-type-mindmap = Mapa mental
chat-surface-type-table = Tabla
chat-surface-type-tabs = Pestañas
chat-surface-type-text = Texto
chat-surface-type-json = Datos
chat-surface-form-datePlaceholder = AAAA-MM-DD
chat-surface-form-numberPlaceholder = Introduce un número
chat-surface-form-textPlaceholder = Introduce un valor
chat-surface-form-submit = Enviar
chat-surface-actionRefused = Acción del asistente rechazada: { $reason }
statusBar-chatTokens = IA ↑{ $input } ↓{ $output } caché { $cacheRead }/{ $cacheWrite }
common-settings = Configuración
common-tasks = Tareas

View File

@@ -78,6 +78,23 @@ chat-role-tool = Outil
chat-deleted = Conversation supprimée
chat-link-refused = Ce type de lien nest pas autorisé
chat-navigation-invalid = Lélément du blog demandé na pas pu être ouvert
chat-surface-dismiss = Masquer
chat-surface-empty = Aucun contenu
chat-surface-type-card = Carte
chat-surface-type-chart = Graphique
chat-surface-type-form = Formulaire
chat-surface-type-list = Liste
chat-surface-type-metric = Indicateur
chat-surface-type-mindmap = Carte mentale
chat-surface-type-table = Tableau
chat-surface-type-tabs = Onglets
chat-surface-type-text = Texte
chat-surface-type-json = Données
chat-surface-form-datePlaceholder = AAAA-MM-JJ
chat-surface-form-numberPlaceholder = Saisir un nombre
chat-surface-form-textPlaceholder = Saisir une valeur
chat-surface-form-submit = Envoyer
chat-surface-actionRefused = Action de lassistant refusée : { $reason }
statusBar-chatTokens = IA ↑{ $input } ↓{ $output } cache { $cacheRead }/{ $cacheWrite }
common-settings = Paramètres
common-tasks = Tâches

View File

@@ -78,6 +78,23 @@ chat-role-tool = Strumento
chat-deleted = Conversazione eliminata
chat-link-refused = Questo tipo di collegamento non è consentito
chat-navigation-invalid = Impossibile aprire lelemento del blog richiesto
chat-surface-dismiss = Nascondi
chat-surface-empty = Nessun contenuto
chat-surface-type-card = Scheda
chat-surface-type-chart = Grafico
chat-surface-type-form = Modulo
chat-surface-type-list = Elenco
chat-surface-type-metric = Metrica
chat-surface-type-mindmap = Mappa mentale
chat-surface-type-table = Tabella
chat-surface-type-tabs = Schede
chat-surface-type-text = Testo
chat-surface-type-json = Dati
chat-surface-form-datePlaceholder = AAAA-MM-GG
chat-surface-form-numberPlaceholder = Inserisci un numero
chat-surface-form-textPlaceholder = Inserisci un valore
chat-surface-form-submit = Invia
chat-surface-actionRefused = Azione dellassistente rifiutata: { $reason }
statusBar-chatTokens = IA ↑{ $input } ↓{ $output } cache { $cacheRead }/{ $cacheWrite }
common-settings = Impostazioni
common-tasks = Attività