From c9c2d8efd57fbd2f2663e2e7dde89a75a911af30 Mon Sep 17 00:00:00 2001 From: Georg Bauer Date: Mon, 27 Jul 2026 11:19:57 +0200 Subject: [PATCH] feat: a2ui interface to enablee the LLM to give structured information --- Cargo.lock | 117 +- Cargo.toml | 5 +- PLAN.md | 26 +- README.md | 38 +- assets/a2ui/catalog.json | 144 ++ build.rs | 8 + .../20260727120000_add_a2ui_messages/down.sql | 1 + .../20260727120000_add_a2ui_messages/up.sql | 10 + native/media/ds4_media.m | 66 + src/a2ui.rs | 2217 +++++++++++++++++ src/a2ui_validation.rs | 258 ++ src/app.rs | 145 +- src/app/generation.rs | 188 +- src/app/preferences.rs | 3 + src/app/projects.rs | 76 + src/app/view.rs | 11 +- src/app/view/a2ui.rs | 1909 ++++++++++++++ src/app/view/chat.rs | 3 + src/app/view/preferences.rs | 6 + src/config.rs | 5 +- src/database.rs | 75 +- src/main.rs | 11 + src/native_media.rs | 39 + src/schema.rs | 13 +- 24 files changed, 5322 insertions(+), 52 deletions(-) create mode 100644 assets/a2ui/catalog.json create mode 100644 migrations/20260727120000_add_a2ui_messages/down.sql create mode 100644 migrations/20260727120000_add_a2ui_messages/up.sql create mode 100644 native/media/ds4_media.m create mode 100644 src/a2ui.rs create mode 100644 src/a2ui_validation.rs create mode 100644 src/app/view/a2ui.rs create mode 100644 src/native_media.rs diff --git a/Cargo.lock b/Cargo.lock index 8b553e4..079b4eb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -993,14 +993,17 @@ dependencies = [ "diesel", "diesel_migrations", "iced", + "image", "memmap2", "muda", - "png", + "png 0.17.16", + "regex", "rfd", "serde", "serde_json", "serde_norway", "sha2", + "time", "ureq", "url", ] @@ -1495,6 +1498,16 @@ dependencies = [ "weezl", ] +[[package]] +name = "gif" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee8cfcc411d9adbbaba82fb72661cc1bcca13e8bba98b364e62b2dba8f960159" +dependencies = [ + "color_quant", + "weezl", +] + [[package]] name = "gio" version = "0.18.4" @@ -1854,6 +1867,7 @@ dependencies = [ "iced_runtime", "iced_widget", "iced_winit", + "image", "thiserror 2.0.19", ] @@ -1913,6 +1927,8 @@ dependencies = [ "half", "iced_core", "iced_futures", + "image", + "kamadak-exif", "log", "raw-window-handle", "rustc-hash 2.1.3", @@ -2149,6 +2165,24 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "image" +version = "0.25.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85ab80394333c02fe689eaf900ab500fbd0c2213da414687ebf995a65d5a6104" +dependencies = [ + "bytemuck", + "byteorder-lite", + "color_quant", + "gif 0.14.2", + "image-webp", + "moxcms", + "num-traits", + "png 0.18.1", + "zune-core 0.5.1", + "zune-jpeg 0.5.15", +] + [[package]] name = "image-webp" version = "0.2.4" @@ -2260,6 +2294,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "kamadak-exif" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1130d80c7374efad55a117d715a3af9368f0fa7a2c54573afc15a188cd984837" +dependencies = [ + "mutate_once", +] + [[package]] name = "keyboard-types" version = "0.7.0" @@ -2517,6 +2560,16 @@ dependencies = [ "simd-adler32", ] +[[package]] +name = "moxcms" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb85c154ba489f01b25c0d36ae69a87e4a1c73a72631fc6c0eb6dde34a73e44b" +dependencies = [ + "num-traits", + "pxfm", +] + [[package]] name = "muda" version = "0.15.3" @@ -2532,7 +2585,7 @@ dependencies = [ "objc2-app-kit 0.2.2", "objc2-foundation 0.2.2", "once_cell", - "png", + "png 0.17.16", "thiserror 1.0.69", "windows-sys 0.59.0", ] @@ -2562,6 +2615,12 @@ dependencies = [ "zbus", ] +[[package]] +name = "mutate_once" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13d2233c9842d08cfe13f9eac96e207ca6a2ea10b80259ebe8ad0268be27d2af" + [[package]] name = "naga" version = "27.0.3" @@ -3212,6 +3271,19 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "png" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60769b8b31b2a9f263dae2776c37b1b28ae246943cf719eb6946a1db05128a61" +dependencies = [ + "bitflags 2.13.1", + "crc32fast", + "fdeflate", + "flate2", + "miniz_oxide", +] + [[package]] name = "polling" version = "3.11.0" @@ -3364,6 +3436,12 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "007d8adb5ddab6f8e3f491ac63566a7d5002cc7ed73901f72057943fa71ae1ae" +[[package]] +name = "pxfm" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d55d956fa96f5ec02be2e13af0e20391a5aa83d6a074e3ad368959d0fab299ea" + [[package]] name = "quick-error" version = "2.0.1" @@ -3505,6 +3583,18 @@ dependencies = [ "bitflags 2.13.1", ] +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + [[package]] name = "regex-automata" version = "0.4.16" @@ -3534,7 +3624,7 @@ version = "0.45.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8928798c0a55e03c9ca6c4c6846f76377427d2c1e1f7e6de3c06ae57942df43" dependencies = [ - "gif", + "gif 0.13.3", "image-webp", "log", "pico-args", @@ -3542,7 +3632,7 @@ dependencies = [ "svgtypes", "tiny-skia", "usvg", - "zune-jpeg", + "zune-jpeg 0.4.21", ] [[package]] @@ -4336,7 +4426,7 @@ dependencies = [ "bytemuck", "cfg-if", "log", - "png", + "png 0.17.16", "tiny-skia-path", ] @@ -5846,13 +5936,28 @@ version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f423a2c17029964870cfaabb1f13dfab7d092a62a29a89264f4d36990ca414a" +[[package]] +name = "zune-core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb8a0807f7c01457d0379ba880ba6322660448ddebc890ce29bb64da71fb40f9" + [[package]] name = "zune-jpeg" version = "0.4.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29ce2c8a9384ad323cf564b67da86e21d3cfdff87908bc1223ed5c99bc792713" dependencies = [ - "zune-core", + "zune-core 0.4.12", +] + +[[package]] +name = "zune-jpeg" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27bc9d5b815bc103f142aa054f561d9187d191692ec7c2d1e2b4737f8dbd7296" +dependencies = [ + "zune-core 0.5.1", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index eb6b70d..bf366fd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,14 +14,17 @@ cc = "1.3.0" [dependencies] diesel = { version = "2.3.11", features = ["sqlite", "returning_clauses_for_sqlite_3_35", "64-column-tables"] } diesel_migrations = "2.3.2" -iced = { version = "0.14.0", features = ["advanced", "highlighter", "markdown", "svg", "tokio"] } +iced = { version = "0.14.0", features = ["advanced", "highlighter", "image-without-codecs", "markdown", "svg", "tokio"] } +image = { version = "0.25.10", default-features = false, features = ["gif", "jpeg", "png", "webp"] } memmap2 = "0.9.11" png = "0.17.16" +regex = "1.13.1" rfd = "0.15.4" serde = { version = "1.0.228", features = ["derive"] } serde_json = { version = "1.0.149", features = ["preserve_order", "raw_value"] } serde_norway = "0.9.42" sha2 = "0.11.0" +time = { version = "0.3.54", features = ["formatting", "parsing"] } ureq = { version = "3.3.0", default-features = false, features = ["rustls"] } url = "2.5.8" diff --git a/PLAN.md b/PLAN.md index c014de7..1852569 100644 --- a/PLAN.md +++ b/PLAN.md @@ -4,29 +4,7 @@ Only unfinished implementation work belongs here. DwarfStar remains the behavioral oracle for model execution, token processing, context accounting, KV-cache behavior, the HTTP API, and the built-in agent loop. -## 1. Add A2UI local-chat surfaces - -- Use the public [A2UI protocol](https://a2ui.org/) as the normative reference. - Implement its versioned, streaming JSON messages and keep `../bDS2` only as - a reference for the research-oriented components and presentation we want. -- Define a DS4Server catalog and native Iced renderer for the A2UI Basic Catalog - plus useful research surfaces such as charts, tables, metrics, timelines, - maps, mind maps, tabs, cards, lists, and forms. -- Support `createSurface`, incremental `updateComponents` and `updateDataModel` - messages, and `deleteSurface`. The model must be able to add, replace, and - update components and data over multiple responses without recreating the - whole surface. -- Persist A2UI messages with the chat and restore the resulting surfaces after - relaunch. Validate messages against the selected catalog, report validation - failures back to the model, and retain a readable fallback in the transcript. -- Implement catalog-declared local functions and agent events. UI-local edits - update the surface immediately; agent events return the action and current - data to the conversation so the model can respond with further A2UI updates. - The existing approval flow applies only if the resulting agent turn invokes - an ordinary side-effecting tool such as shell or file mutation. -- A2UI can be enabled/disabled in preferences to reduce the system prompt - -## 2. Finish native project and chat controls +## 1. Finish native project and chat controls - Add functional File, View, and Help menus; update native menu enabled state from the focused control; complete undo/redo behavior; and make transcript @@ -42,7 +20,7 @@ KV-cache behavior, the HTTP API, and the built-in agent loop. - the hellp menu must open a window with an actual documentation in it, that still needs to be constructed from the information and implementation -## 3. Add Dev Brain support +## 2. Add Dev Brain support - Let the user select one Obsidian vault and give the local agent bounded Markdown memory_search, memory_read, memory_create, and memory_append diff --git a/README.md b/README.md index 67b7c6b..b716832 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,38 @@ Projects, sessions, and transcripts use SQLite. Settings use a compact YAML file that contains only values changed from their defaults and can also be edited by hand. +## A2UI widgets + +DS4Server renders A2UI v1.0 surfaces with these widgets: + +- **Content:** Markdown text, images (including avatars), icons, video and audio + players, and dividers. +- **Layout:** rows, columns, lists, cards, modals, and tabs. +- **Controls:** buttons, text fields, checkboxes, sliders, date/time inputs, and + choice pickers. +- **Research:** bar, line, area, stacked-bar, pie, and donut charts; tables, + metrics, timelines, maps, mind maps, and forms. + +Video posters render in the surface; video and audio playback uses native +macOS AVKit player windows with standard transport, volume, and fullscreen +controls. + +### Live A2UI validation + +With DS4Server running and its local endpoint enabled, run the opt-in model +validation suite with: + +```sh +cargo run --release -- validate-a2ui +``` + +It sends natural-language chart, form, choice, and full-composition prompts +through the current A2UI tool description, then replays every returned protocol +message through the real catalog and component-graph validators. Use `--case +pie-natural`, `--case composed-basics`, `--attempts 3`, `--model MODEL_ID`, or +`--endpoint URL` to narrow or repeat runs. This slow, model-dependent suite is +intentionally separate from `cargo test`. + ## Build and run DS4Server targets macOS 13 or newer and requires Rust 1.97 plus @@ -53,9 +85,9 @@ open target/release/DS4Server.app ## Application data ```text -~/Library/Application Support/DS4Server.rfc1437.de/data.sqlite3 -~/Library/Application Support/DS4Server.rfc1437.de/config.yaml -~/Library/Application Support/DS4Server.rfc1437.de/kv-cache/ +~/Library/Application Support/de.rfc1437.ds4server/data.sqlite3 +~/Library/Application Support/de.rfc1437.ds4server/config.yaml +~/Library/Application Support/de.rfc1437.ds4server/kv-cache/ ``` Deleting a project or session removes DS4Server metadata and its associated diff --git a/assets/a2ui/catalog.json b/assets/a2ui/catalog.json new file mode 100644 index 0000000..513fee7 --- /dev/null +++ b/assets/a2ui/catalog.json @@ -0,0 +1,144 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://ds4server.local/a2ui/v1_0/catalog.json", + "title": "DS4Server A2UI Catalog", + "catalogId": "https://ds4server.local/a2ui/v1_0/catalog.json", + "instructions": "Use the Basic Catalog for interaction and layout. Use research components for compact analytical results.", + "components": { + "Text": { "$ref": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json#/components/Text" }, + "Image": { "$ref": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json#/components/Image" }, + "Icon": { "$ref": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json#/components/Icon" }, + "Video": { "$ref": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json#/components/Video" }, + "AudioPlayer": { "$ref": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json#/components/AudioPlayer" }, + "Divider": { "$ref": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json#/components/Divider" }, + "Row": { "$ref": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json#/components/Row" }, + "Column": { "$ref": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json#/components/Column" }, + "List": { "$ref": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json#/components/List" }, + "Card": { "$ref": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json#/components/Card" }, + "Modal": { "$ref": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json#/components/Modal" }, + "Tabs": { "$ref": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json#/components/Tabs" }, + "Button": { "$ref": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json#/components/Button" }, + "TextField": { "$ref": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json#/components/TextField" }, + "CheckBox": { "$ref": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json#/components/CheckBox" }, + "Slider": { "$ref": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json#/components/Slider" }, + "DateTimeInput": { "$ref": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json#/components/DateTimeInput" }, + "ChoicePicker": { "$ref": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json#/components/ChoicePicker" }, + "Chart": { "$ref": "#/$defs/chart" }, + "Table": { "$ref": "#/$defs/table" }, + "Metric": { "$ref": "#/$defs/metric" }, + "Timeline": { "$ref": "#/$defs/timeline" }, + "Map": { "$ref": "#/$defs/map" }, + "MindMap": { "$ref": "#/$defs/mindMap" }, + "Form": { "$ref": "#/$defs/form" } + }, + "functions": { + "required": { "$ref": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json#/functions/required" }, + "regex": { "$ref": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json#/functions/regex" }, + "length": { "$ref": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json#/functions/length" }, + "numeric": { "$ref": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json#/functions/numeric" }, + "email": { "$ref": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json#/functions/email" }, + "formatString": { "$ref": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json#/functions/formatString" }, + "formatNumber": { "$ref": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json#/functions/formatNumber" }, + "formatCurrency": { "$ref": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json#/functions/formatCurrency" }, + "formatDate": { "$ref": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json#/functions/formatDate" }, + "pluralize": { "$ref": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json#/functions/pluralize" }, + "openUrl": { "$ref": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json#/functions/openUrl" }, + "and": { "$ref": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json#/functions/and" }, + "or": { "$ref": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json#/functions/or" }, + "not": { "$ref": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json#/functions/not" } + }, + "$defs": { + "surfaceProperties": { + "type": "object", + "properties": { + "iconUrl": { "type": "string", "format": "uri" }, + "agentDisplayName": { "type": "string" } + }, + "additionalProperties": true + }, + "researchBase": { + "type": "object", + "properties": { + "id": { "type": "string" }, + "component": { "type": "string" }, + "title": { "type": "string" }, + "accessibility": { "type": "object" }, + "weight": { "type": "number" }, + "checks": { "type": "array" } + }, + "required": ["id", "component"] + }, + "chart": { + "allOf": [ + { "$ref": "#/$defs/researchBase" }, + { "properties": { "component": { "const": "Chart" }, "chartType": { "enum": ["bar", "line", "area", "stackedBar", "pie", "donut"] }, "series": {} }, "required": ["chartType", "series"] } + ], + "unevaluatedProperties": false + }, + "table": { + "allOf": [ + { "$ref": "#/$defs/researchBase" }, + { "properties": { "component": { "const": "Table" }, "columns": {}, "rows": {} }, "required": ["columns", "rows"] } + ], + "unevaluatedProperties": false + }, + "metric": { + "allOf": [ + { "$ref": "#/$defs/researchBase" }, + { "properties": { "component": { "const": "Metric" }, "label": {}, "value": {}, "detail": {}, "trend": {} }, "required": ["label", "value"] } + ], + "unevaluatedProperties": false + }, + "timeline": { + "allOf": [ + { "$ref": "#/$defs/researchBase" }, + { "properties": { "component": { "const": "Timeline" }, "events": {} }, "required": ["events"] } + ], + "unevaluatedProperties": false + }, + "map": { + "allOf": [ + { "$ref": "#/$defs/researchBase" }, + { "properties": { "component": { "const": "Map" }, "locations": {} }, "required": ["locations"] } + ], + "unevaluatedProperties": false + }, + "mindMap": { + "allOf": [ + { "$ref": "#/$defs/researchBase" }, + { "properties": { "component": { "const": "MindMap" }, "nodes": {} }, "required": ["nodes"] } + ], + "unevaluatedProperties": false + }, + "form": { + "allOf": [ + { "$ref": "#/$defs/researchBase" }, + { "properties": { "component": { "const": "Form" }, "children": { "$ref": "https://a2ui.org/specification/v1_0/common_types.json#/$defs/ChildList" }, "submitLabel": { "type": "string" }, "action": { "$ref": "https://a2ui.org/specification/v1_0/common_types.json#/$defs/Action" } }, "required": ["children"] } + ], + "unevaluatedProperties": false + }, + "anyComponent": { + "oneOf": [ + { "$ref": "#/components/Text" }, { "$ref": "#/components/Image" }, { "$ref": "#/components/Icon" }, + { "$ref": "#/components/Video" }, { "$ref": "#/components/AudioPlayer" }, { "$ref": "#/components/Divider" }, + { "$ref": "#/components/Row" }, { "$ref": "#/components/Column" }, { "$ref": "#/components/List" }, + { "$ref": "#/components/Card" }, { "$ref": "#/components/Modal" }, { "$ref": "#/components/Tabs" }, + { "$ref": "#/components/Button" }, { "$ref": "#/components/TextField" }, { "$ref": "#/components/CheckBox" }, + { "$ref": "#/components/Slider" }, { "$ref": "#/components/DateTimeInput" }, { "$ref": "#/components/ChoicePicker" }, + { "$ref": "#/components/Chart" }, { "$ref": "#/components/Table" }, { "$ref": "#/components/Metric" }, + { "$ref": "#/components/Timeline" }, { "$ref": "#/components/Map" }, { "$ref": "#/components/MindMap" }, + { "$ref": "#/components/Form" } + ], + "discriminator": { "propertyName": "component" } + }, + "anyFunction": { + "oneOf": [ + { "$ref": "#/functions/required" }, { "$ref": "#/functions/regex" }, { "$ref": "#/functions/length" }, + { "$ref": "#/functions/numeric" }, { "$ref": "#/functions/email" }, { "$ref": "#/functions/formatString" }, + { "$ref": "#/functions/formatNumber" }, { "$ref": "#/functions/formatCurrency" }, { "$ref": "#/functions/formatDate" }, + { "$ref": "#/functions/pluralize" }, { "$ref": "#/functions/openUrl" }, { "$ref": "#/functions/and" }, + { "$ref": "#/functions/or" }, { "$ref": "#/functions/not" } + ] + } + } +} diff --git a/build.rs b/build.rs index 90717cd..242ead2 100644 --- a/build.rs +++ b/build.rs @@ -13,6 +13,11 @@ fn main() { .flag("-fobjc-arc") .opt_level(3) .compile("ds4_metal"); + println!("cargo:rerun-if-changed=native/media"); + cc::Build::new() + .file("native/media/ds4_media.m") + .flag("-fobjc-arc") + .compile("ds4_media"); println!("cargo:rerun-if-changed=native/web"); cc::Build::new() .include("native/web") @@ -20,5 +25,8 @@ fn main() { .opt_level(2) .compile("ds4_web"); println!("cargo:rustc-link-lib=framework=Foundation"); + println!("cargo:rustc-link-lib=framework=AppKit"); + println!("cargo:rustc-link-lib=framework=AVFoundation"); + println!("cargo:rustc-link-lib=framework=AVKit"); println!("cargo:rustc-link-lib=framework=Metal"); } diff --git a/migrations/20260727120000_add_a2ui_messages/down.sql b/migrations/20260727120000_add_a2ui_messages/down.sql new file mode 100644 index 0000000..d1c2990 --- /dev/null +++ b/migrations/20260727120000_add_a2ui_messages/down.sql @@ -0,0 +1 @@ +DROP TABLE a2ui_messages; diff --git a/migrations/20260727120000_add_a2ui_messages/up.sql b/migrations/20260727120000_add_a2ui_messages/up.sql new file mode 100644 index 0000000..00cefd6 --- /dev/null +++ b/migrations/20260727120000_add_a2ui_messages/up.sql @@ -0,0 +1,10 @@ +CREATE TABLE a2ui_messages ( + id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + session_id INTEGER NOT NULL, + message_id INTEGER NOT NULL, + json TEXT NOT NULL, + FOREIGN KEY (session_id) REFERENCES sessions(id), + FOREIGN KEY (message_id) REFERENCES messages(id) +); + +CREATE INDEX a2ui_messages_session_id ON a2ui_messages(session_id, id); diff --git a/native/media/ds4_media.m b/native/media/ds4_media.m new file mode 100644 index 0000000..104aa44 --- /dev/null +++ b/native/media/ds4_media.m @@ -0,0 +1,66 @@ +#import +#import +#include + +@interface DS4MediaWindowController : NSWindowController +@end + +static NSMutableSet *DS4MediaWindows(void) { + static NSMutableSet *windows; + static dispatch_once_t once; + dispatch_once(&once, ^{ + windows = [NSMutableSet set]; + }); + return windows; +} + +@implementation DS4MediaWindowController + +- (void)windowWillClose:(NSNotification *)notification { + AVPlayerView *view = (AVPlayerView *)self.window.contentView; + [view.player pause]; + [DS4MediaWindows() removeObject:self]; +} + +@end + +bool ds4_media_open(const char *url_bytes, const char *title_bytes, bool video) { + if (url_bytes == NULL || title_bytes == NULL) { + return false; + } + NSString *url_string = [NSString stringWithUTF8String:url_bytes]; + NSString *title = [NSString stringWithUTF8String:title_bytes]; + NSURL *url = [NSURL URLWithString:url_string]; + if (url == nil || title == nil) { + return false; + } + + dispatch_async(dispatch_get_main_queue(), ^{ + NSRect frame = NSMakeRect(0, 0, video ? 800 : 560, video ? 500 : 180); + NSWindow *window = [[NSWindow alloc] + initWithContentRect:frame + styleMask:NSWindowStyleMaskTitled | + NSWindowStyleMaskClosable | + NSWindowStyleMaskMiniaturizable | + NSWindowStyleMaskResizable + backing:NSBackingStoreBuffered + defer:NO]; + window.title = title.length > 0 ? title : url.lastPathComponent; + window.minSize = video ? NSMakeSize(480, 300) : NSMakeSize(420, 150); + + AVPlayerView *view = [[AVPlayerView alloc] initWithFrame:frame]; + view.autoresizingMask = NSViewWidthSizable | NSViewHeightSizable; + view.player = [AVPlayer playerWithURL:url]; + window.contentView = view; + + DS4MediaWindowController *controller = + [[DS4MediaWindowController alloc] initWithWindow:window]; + window.delegate = controller; + [DS4MediaWindows() addObject:controller]; + [controller showWindow:nil]; + [window center]; + [window makeKeyAndOrderFront:nil]; + [view.player play]; + }); + return true; +} diff --git a/src/a2ui.rs b/src/a2ui.rs new file mode 100644 index 0000000..6add606 --- /dev/null +++ b/src/a2ui.rs @@ -0,0 +1,2217 @@ +use regex::Regex; +use serde_json::{Map, Value, json}; +use std::cell::Cell; +use std::collections::{BTreeMap, BTreeSet}; +use std::sync::OnceLock; + +pub(crate) const VERSION: &str = "v1.0"; +pub(crate) const CATALOG_ID: &str = "https://ds4server.local/a2ui/v1_0/catalog.json"; +pub(crate) const BASIC_CATALOG_ID: &str = + "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json"; +pub(crate) const CATALOG_JSON: &str = include_str!("../assets/a2ui/catalog.json"); + +thread_local! { + static TEMPLATE_INDEX: Cell> = const { Cell::new(None) }; +} +pub(crate) const BASIC_COMPONENTS: &[&str] = &[ + "Text", + "Image", + "Icon", + "Video", + "AudioPlayer", + "Divider", + "Row", + "Column", + "List", + "Card", + "Modal", + "Tabs", + "Button", + "TextField", + "CheckBox", + "Slider", + "DateTimeInput", + "ChoicePicker", +]; +pub(crate) const BASIC_NON_MEDIA_COMPONENTS: &[&str] = &[ + "Text", + "Image", + "Icon", + "Divider", + "Row", + "Column", + "List", + "Card", + "Modal", + "Tabs", + "Button", + "TextField", + "CheckBox", + "Slider", + "DateTimeInput", + "ChoicePicker", +]; +const FUNCTIONS: &[&str] = &[ + "required", + "regex", + "length", + "numeric", + "email", + "formatString", + "formatNumber", + "formatCurrency", + "formatDate", + "pluralize", + "openUrl", + "and", + "or", + "not", + "@index", +]; + +pub(crate) const SYSTEM_PROMPT: &str = r#"A2UI local-chat rendering is enabled. Use the newest A2UI v1.0 protocol. You may present interactive native UI by emitting newline-delimited messages inside a fenced `a2ui` block. Keep ordinary prose outside the block. Use catalogId `https://ds4server.local/a2ui/v1_0/catalog.json`. + +Every line must be one JSON object with `version":"v1.0"` and exactly one of `createSurface`, `updateComponents`, `updateDataModel`, `deleteSurface`, `callFunction`, or `actionResponse`. Create a surface before updating it. Components are a flat adjacency list and the root component has id `root`. Compose complete UIs by combining basic components through container child ids; include every referenced child, tab child, and list template component. Reuse a surfaceId to update it incrementally; never recreate an existing surface. `createSurface` may include initial `components`, `dataModel`, and `surfaceProperties`. For `actionResponse`, put `actionId` beside `version` and put only `value` or `error` inside `actionResponse`. + +Catalog components: +- Basic: Text(text Markdown,variant), Image(url,description,fit,variant including avatar), Icon(name), Video(url,posterUrl), AudioPlayer(url,description), Divider(axis), Row/Column(children,justify start|center|end|spaceBetween|spaceAround|spaceEvenly|stretch,align start|center|end|stretch), List(children,direction,align), Card(child), Modal(trigger,content), Tabs(tabs[{title,child}]), Button(child,variant,action:{event:{name,context,wantResponse}}), TextField(label,value:{path},variant shortText|longText|number|obscured,placeholder), CheckBox(label,value:{path}), Slider(value:{path},min,max), DateTimeInput(label,value:{path},enableDate,enableTime,min,max), ChoicePicker(label,options[{label,value}],value:{path},variant multipleSelection|mutuallyExclusive,displayStyle checkbox|chips,filterable). Put numeric weight on direct Row/Column children to distribute available space. +- Research: Chart(title,chartType bar|line|area|stackedBar|pie|donut,series[{label,value,segments}]), Table(title,columns,rows), Metric(label,value,detail,trend), Timeline(title,events[{time,title,description,status}]), Map(title,locations[{label,latitude,longitude,detail}]), MindMap(title,nodes[{id,label,children}]), Form(title,children,submitLabel,action). Use pie or donut for proportional breakdowns; donut displays the total in its center. +- Shared fields: id, accessibility, weight, checks. Checks use {"condition":{"call":"required","args":{...}},"message":"..."}. Bind dynamic values with {"path":"/json/pointer"}. The renderer supports every function in the v1.0 Basic Catalog, including validation, formatting, logic, pluralize, openUrl, and @index. Input edits are local and synchronous. Agent events receive their resolved context and current data model. Server-initiated `callFunction` messages are supported. + +Example: +```a2ui +{"version":"v1.0","createSurface":{"surfaceId":"answer","catalogId":"https://ds4server.local/a2ui/v1_0/catalog.json","sendDataModel":true,"surfaceProperties":{},"dataModel":{"name":""},"components":[{"id":"root","component":"Card","child":"body"},{"id":"body","component":"Column","children":["title","name","submit"]},{"id":"title","component":"Text","text":"Research result","variant":"body"},{"id":"name","component":"TextField","label":"Name","value":{"path":"/name"}},{"id":"submit-label","component":"Text","text":"Continue"},{"id":"submit","component":"Button","child":"submit-label","checks":[{"condition":{"call":"required","args":{"value":{"path":"/name"}}},"message":"Name is required"}],"action":{"event":{"name":"continue","wantResponse":true,"responsePath":"/result","context":{"name":{"path":"/name"}}}}}]}} +``` +If the renderer reports a `VALIDATION_FAILED` A2UI client error, correct the named message or component with another valid A2UI message."#; + +#[derive(Clone, Debug)] +pub(crate) struct Surface { + pub(crate) id: String, + pub(crate) catalog_id: String, + pub(crate) surface_properties: Value, + pub(crate) send_data_model: bool, + pub(crate) components: BTreeMap, + pub(crate) data: Value, + pub(crate) owner_message_id: i32, +} + +#[derive(Clone, Default)] +pub(crate) struct Store { + surfaces: BTreeMap, + pending_actions: BTreeMap)>, + next_action_id: u64, +} + +#[derive(Debug)] +pub(crate) struct ExtractedLine { + pub(crate) raw: String, + pub(crate) value: Result, +} + +#[derive(Debug)] +pub(crate) struct Applied { + pub(crate) raws: Vec, + pub(crate) reply: Option, + pub(crate) open_url: Option, +} + +pub(crate) fn validate_surface_composition(surface: &Surface) -> Result, String> { + fn visit( + surface: &Surface, + id: &str, + ancestors: &mut BTreeSet, + visited: &mut BTreeSet, + kinds: &mut BTreeSet, + ) -> Result<(), String> { + if visited.contains(id) { + return Ok(()); + } + if !ancestors.insert(id.to_owned()) { + return Err(format!("cyclic component reference at `{id}`")); + } + let component = surface + .components + .get(id) + .and_then(Value::as_object) + .ok_or_else(|| format!("referenced component `{id}` is missing"))?; + let kind = required_string(component, "component")?; + kinds.insert(kind.to_owned()); + + let mut children = Vec::new(); + match kind { + "Row" | "Column" | "List" | "Form" => { + if let Some(ids) = component.get("children").and_then(Value::as_array) { + children.extend(ids.iter().filter_map(Value::as_str)); + } else if let Some(id) = component + .get("children") + .and_then(Value::as_object) + .and_then(|template| template.get("componentId")) + .and_then(Value::as_str) + { + children.push(id); + } + } + "Card" | "Button" => { + children.extend(component.get("child").and_then(Value::as_str)); + } + "Modal" => { + children.extend(component.get("trigger").and_then(Value::as_str)); + children.extend(component.get("content").and_then(Value::as_str)); + } + "Tabs" => { + children.extend( + component + .get("tabs") + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter_map(|tab| tab.get("child").and_then(Value::as_str)), + ); + } + _ => {} + } + for child in children { + visit(surface, child, ancestors, visited, kinds)?; + } + ancestors.remove(id); + visited.insert(id.to_owned()); + Ok(()) + } + + let mut ancestors = BTreeSet::new(); + let mut visited = BTreeSet::new(); + let mut kinds = BTreeSet::new(); + visit(surface, "root", &mut ancestors, &mut visited, &mut kinds)?; + Ok(kinds) +} + +impl Store { + pub(crate) fn clear(&mut self) { + self.surfaces.clear(); + self.pending_actions.clear(); + self.next_action_id = 0; + } + + pub(crate) fn surfaces_for_message(&self, message_id: i32) -> impl Iterator { + self.surfaces + .values() + .filter(move |surface| surface.owner_message_id == message_id) + } + + pub(crate) fn surfaces(&self) -> impl Iterator { + self.surfaces.values() + } + + pub(crate) fn surface(&self, id: &str) -> Option<&Surface> { + self.surfaces.get(id) + } + + pub(crate) fn image_urls(&self) -> impl Iterator + '_ { + self.surfaces.values().flat_map(|surface| { + surface.components.values().filter_map(|component| { + let component = component.as_object()?; + let field = match component.get("component").and_then(Value::as_str) { + Some("Image") => "url", + Some("Video") => "posterUrl", + _ => return None, + }; + component.get(field).map(|value| { + display_value(&bound_value_at(Some(value), &surface.data, &surface.data)) + }) + }) + }) + } + + pub(crate) fn apply_raw( + &mut self, + raw: &str, + owner_message_id: i32, + ) -> Result { + let value: Value = serde_json::from_str(raw).map_err(|error| error.to_string())?; + self.apply(value, raw.to_owned(), owner_message_id) + } + + pub(crate) fn apply( + &mut self, + value: Value, + raw: String, + owner_message_id: i32, + ) -> Result { + let mut next = self.clone(); + let applied = next.apply_inner(value, raw, owner_message_id)?; + *self = next; + Ok(applied) + } + + fn apply_inner( + &mut self, + value: Value, + raw: String, + owner_message_id: i32, + ) -> Result { + let envelope = value + .as_object() + .ok_or_else(|| "A2UI message must be a JSON object".to_owned())?; + let version = envelope + .get("version") + .and_then(Value::as_str) + .ok_or_else(|| "A2UI message requires a string version".to_owned())?; + if version != VERSION { + return Err(format!("unsupported A2UI version `{version}`")); + } + let kinds = [ + "createSurface", + "updateComponents", + "updateDataModel", + "deleteSurface", + "callFunction", + "actionResponse", + ]; + let present = kinds + .iter() + .filter(|key| envelope.contains_key(**key)) + .copied() + .collect::>(); + if present.len() != 1 { + return Err("A2UI envelope must contain version and exactly one message type".into()); + } + let kind = present[0]; + let allowed_envelope: &[&str] = match kind { + "callFunction" => &["version", "callFunction", "functionCallId", "wantResponse"], + "actionResponse" => &["version", "actionResponse", "actionId"], + _ => &["version", kind], + }; + reject_unknown(envelope, allowed_envelope, "A2UI envelope")?; + let payload = object(envelope.get(kind), kind)?; + let mut raws = vec![raw]; + let mut reply = None; + let mut open_url = None; + match kind { + "createSurface" => { + reject_unknown( + payload, + &[ + "surfaceId", + "catalogId", + "surfaceProperties", + "sendDataModel", + "components", + "dataModel", + ], + "createSurface", + )?; + let id = required_string(payload, "surfaceId")?; + if self.surfaces.contains_key(id) { + return Err(format!("surface `{id}` already exists")); + } + let catalog_id = required_string(payload, "catalogId")?; + if !matches!(catalog_id, CATALOG_ID | BASIC_CATALOG_ID) { + return Err(format!("unsupported catalog `{catalog_id}`")); + } + let surface_properties = payload + .get("surfaceProperties") + .cloned() + .unwrap_or_else(|| json!({})); + if !surface_properties.is_object() { + return Err("createSurface surface properties must be an object".into()); + } + let properties = surface_properties.as_object().unwrap(); + if properties + .get("agentDisplayName") + .is_some_and(|value| !value.is_string()) + { + return Err("surfaceProperties.agentDisplayName must be a string".into()); + } + if let Some(icon) = properties.get("iconUrl") { + let icon = icon.as_str().ok_or_else(|| { + "surfaceProperties.iconUrl must be a URI string".to_owned() + })?; + url::Url::parse(icon) + .map_err(|error| format!("invalid surface iconUrl: {error}"))?; + } + let send_data_model = optional_bool(payload, "sendDataModel")?.unwrap_or(false); + self.surfaces.insert( + id.to_owned(), + Surface { + id: id.to_owned(), + catalog_id: catalog_id.to_owned(), + surface_properties, + send_data_model, + components: BTreeMap::new(), + data: json!({}), + owner_message_id, + }, + ); + let surface = self.surfaces.get_mut(id).unwrap(); + if let Some(data) = payload.get("dataModel") { + if !data.is_object() { + return Err("createSurface.dataModel must be an object".into()); + } + surface.data = data.clone(); + } + if let Some(components) = payload.get("components") { + let components = components + .as_array() + .filter(|components| !components.is_empty()) + .ok_or_else(|| { + "createSurface.components must be a non-empty array".to_owned() + })?; + for component in components { + let component = component + .as_object() + .ok_or_else(|| "every component must be an object".to_owned())?; + validate_component(component, &surface.catalog_id)?; + surface.components.insert( + required_string(component, "id")?.to_owned(), + Value::Object(component.clone()), + ); + } + } + } + "updateComponents" => { + reject_unknown(payload, &["surfaceId", "components"], "updateComponents")?; + let id = required_string(payload, "surfaceId")?; + let surface = self + .surfaces + .get_mut(id) + .ok_or_else(|| format!("surface `{id}` has not been created"))?; + let components = payload + .get("components") + .and_then(Value::as_array) + .ok_or_else(|| "updateComponents.components must be an array".to_owned())?; + if components.is_empty() { + return Err("updateComponents.components cannot be empty".into()); + } + let mut ids = BTreeSet::new(); + for component in components { + let component = component + .as_object() + .ok_or_else(|| "every component must be an object".to_owned())?; + validate_component(component, &surface.catalog_id)?; + let component_id = required_string(component, "id")?; + if !ids.insert(component_id) { + return Err(format!( + "component `{component_id}` occurs twice in one update" + )); + } + } + for component in components { + let component_id = component["id"].as_str().unwrap().to_owned(); + surface.components.insert(component_id, component.clone()); + } + } + "updateDataModel" => { + reject_unknown(payload, &["surfaceId", "path", "value"], "updateDataModel")?; + let id = required_string(payload, "surfaceId")?; + let surface = self + .surfaces + .get_mut(id) + .ok_or_else(|| format!("surface `{id}` has not been created"))?; + let path = payload.get("path").and_then(Value::as_str).unwrap_or("/"); + if path != "/" && !path.starts_with('/') { + return Err("updateDataModel.path must be a JSON Pointer".into()); + } + set_pointer(&mut surface.data, path, payload.get("value").cloned())?; + } + "deleteSurface" => { + reject_unknown(payload, &["surfaceId"], "deleteSurface")?; + let id = required_string(payload, "surfaceId")?; + if self.surfaces.remove(id).is_none() { + return Err(format!("surface `{id}` has not been created")); + } + } + "callFunction" => { + let call_id = required_string(envelope, "functionCallId")?; + reject_unknown(payload, &["call", "args"], "callFunction")?; + let call = required_string(payload, "call")?; + validate_function(&Value::Object(payload.clone()))?; + let value = if call == "openUrl" { + let url = payload + .get("args") + .and_then(|args| args.get("url")) + .and_then(Value::as_str) + .ok_or_else(|| "openUrl requires string `args.url`".to_owned())?; + let parsed = url::Url::parse(url).map_err(|error| error.to_string())?; + if !matches!(parsed.scheme(), "http" | "https") { + return Err("openUrl only supports HTTP(S) URLs".into()); + } + open_url = Some(parsed.to_string()); + Value::Null + } else { + evaluate( + call, + payload.get("args").unwrap_or(&json!({})), + &json!({}), + &json!({}), + )? + }; + if optional_bool(envelope, "wantResponse")?.unwrap_or(false) { + reply = Some(json!({ + "version": VERSION, + "functionResponse": { + "functionCallId": call_id, + "call": call, + "value": value + } + })); + } + } + "actionResponse" => { + let action_id = required_string(envelope, "actionId")?; + let response = payload + .get("value") + .or_else(|| payload.get("error")) + .cloned() + .ok_or_else(|| "actionResponse requires value or error".to_owned())?; + reject_unknown(payload, &["value", "error"], "actionResponse")?; + if payload.contains_key("value") == payload.contains_key("error") { + return Err("actionResponse requires exactly one of value or error".into()); + } + if let Some(error) = payload.get("error") { + let error = object(Some(error), "actionResponse.error")?; + reject_unknown(error, &["code", "message"], "actionResponse.error")?; + required_string(error, "code")?; + required_string(error, "message")?; + } + if let Some((surface_id, response_path)) = self.pending_actions.remove(action_id) + && payload.contains_key("value") + && let Some(path) = response_path + { + let surface = self.surfaces.get_mut(&surface_id).ok_or_else(|| { + format!("surface `{surface_id}` for action `{action_id}` no longer exists") + })?; + set_pointer(&mut surface.data, &path, Some(response.clone()))?; + raws.push( + serde_json::to_string(&json!({ + "version": VERSION, + "updateDataModel": { + "surfaceId": surface_id, + "path": path, + "value": response + } + })) + .map_err(|error| error.to_string())?, + ); + } + } + _ => unreachable!(), + } + Ok(Applied { + raws, + reply, + open_url, + }) + } + + pub(crate) fn local_update( + &mut self, + surface_id: &str, + path: &str, + value: Value, + ) -> Result { + let surface = self + .surfaces + .get_mut(surface_id) + .ok_or_else(|| format!("surface `{surface_id}` does not exist"))?; + set_pointer(&mut surface.data, path, Some(value.clone()))?; + serde_json::to_string(&json!({ + "version": VERSION, + "updateDataModel": {"surfaceId": surface_id, "path": path, "value": value} + })) + .map_err(|error| error.to_string()) + } + + pub(crate) fn action( + &mut self, + surface_id: &str, + component_id: &str, + context_path: Option<&str>, + ) -> Result { + let surface = self + .surfaces + .get(surface_id) + .ok_or_else(|| format!("surface `{surface_id}` does not exist"))?; + let component = surface + .components + .get(component_id) + .and_then(Value::as_object) + .ok_or_else(|| format!("component `{component_id}` does not exist"))?; + if let Some(error) = first_failed_check(component, &surface.data) { + return Err(error); + } + let event = component + .get("action") + .and_then(Value::as_object) + .and_then(|action| action.get("event")) + .and_then(Value::as_object) + .ok_or_else(|| format!("component `{component_id}` has no agent event"))?; + let name = required_string(event, "name")?; + let context_data = context_path + .and_then(|path| surface.data.pointer(path)) + .unwrap_or(&surface.data); + let context = resolve_at( + event.get("context").unwrap_or(&Value::Object(Map::new())), + &surface.data, + context_data, + )?; + self.next_action_id += 1; + let action_id = format!("ds4-action-{}", self.next_action_id); + let want_response = event + .get("wantResponse") + .and_then(Value::as_bool) + .unwrap_or(false); + let response_path = event + .get("responsePath") + .map(|value| { + value + .as_str() + .filter(|path| path.starts_with('/')) + .map(str::to_owned) + .ok_or_else(|| "action responsePath must be a JSON Pointer".to_owned()) + }) + .transpose()?; + let timestamp = time::OffsetDateTime::now_utc() + .format(&time::format_description::well_known::Rfc3339) + .map_err(|error| error.to_string())?; + let mut event_payload = json!({ + "name": name, + "surfaceId": surface_id, + "sourceComponentId": component_id, + "timestamp": timestamp, + "context": context, + "wantResponse": want_response + }); + if want_response { + event_payload["actionId"] = Value::String(action_id.clone()); + self.pending_actions + .insert(action_id, (surface_id.to_owned(), response_path)); + } + let action = json!({ + "version": VERSION, + "action": event_payload + }); + Ok(action) + } + + pub(crate) fn client_metadata(&self) -> Value { + let surfaces = self + .surfaces + .values() + .filter(|surface| surface.send_data_model) + .map(|surface| (surface.id.clone(), surface.data.clone())) + .collect::>(); + let mut metadata = json!({ + "a2uiClientCapabilities": { + (VERSION): { + "supportedCatalogIds": [CATALOG_ID, BASIC_CATALOG_ID], + "inlineCatalogs": [] + } + } + }); + if !surfaces.is_empty() { + metadata["a2uiClientDataModel"] = json!({ + "version": VERSION, + "surfaces": surfaces, + }); + } + metadata + } +} + +pub(crate) fn extract_lines(content: &str) -> Vec { + let mut lines = Vec::new(); + let mut in_block = false; + for part in content.split_inclusive('\n') { + let complete = part.ends_with('\n'); + let line = part.trim(); + if !in_block && line.eq_ignore_ascii_case("```a2ui") { + in_block = true; + } else if in_block && line.starts_with("```") { + in_block = false; + } else if in_block && complete && !line.is_empty() { + lines.push(ExtractedLine { + raw: line.to_owned(), + value: serde_json::from_str(line).map_err(|error| error.to_string()), + }); + } + } + lines +} + +pub(crate) fn message_surface_id(value: &Value) -> Option<&str> { + value + .as_object()? + .values() + .filter_map(Value::as_object) + .find_map(|payload| payload.get("surfaceId").and_then(Value::as_str)) +} + +pub(crate) fn transcript_fallback(content: &str) -> String { + let mut output = String::new(); + let mut in_block = false; + let mut fallback = Vec::new(); + for part in content.split_inclusive('\n') { + let line = part.trim(); + if !in_block && line.eq_ignore_ascii_case("```a2ui") { + in_block = true; + continue; + } + if in_block && line.starts_with("```") { + in_block = false; + if !fallback.is_empty() { + output.push_str("\n> "); + output.push_str(&fallback.join(" \n> ")); + output.push('\n'); + fallback.clear(); + } + continue; + } + if in_block { + if line.is_empty() { + continue; + } + fallback.push(match serde_json::from_str::(line) { + Ok(value) => message_summary(&value), + Err(error) => format!("A2UI message could not be read: {error}"), + }); + } else { + output.push_str(part); + } + } + if !fallback.is_empty() { + output.push_str("\n> "); + output.push_str(&fallback.join(" \n> ")); + } + output +} + +pub(crate) fn bound_value_at(value: Option<&Value>, data: &Value, context: &Value) -> Value { + value + .map(|value| resolve_at(value, data, context).unwrap_or(Value::Null)) + .unwrap_or(Value::Null) +} + +pub(crate) fn binding_path(value: Option<&Value>) -> Option<&str> { + value?.as_object()?.get("path")?.as_str() +} + +pub(crate) fn with_template_index(index: usize, render: impl FnOnce() -> T) -> T { + TEMPLATE_INDEX.with(|current| { + let previous = current.replace(Some(index)); + let rendered = render(); + current.set(previous); + rendered + }) +} + +pub(crate) fn first_failed_check(component: &Map, data: &Value) -> Option { + component + .get("checks") + .and_then(Value::as_array) + .and_then(|checks| { + checks.iter().find_map(|check| { + let check = check.as_object()?; + let fallback = Value::Object(check.clone()); + let condition = check.get("condition").unwrap_or(&fallback); + match resolve(condition, data) { + Ok(Value::Bool(true)) => None, + Ok(_) => Some( + check + .get("message") + .and_then(Value::as_str) + .unwrap_or("This value is invalid") + .to_owned(), + ), + Err(error) => Some(error), + } + }) + }) +} + +fn validate_component(component: &Map, catalog_id: &str) -> Result<(), String> { + let id = required_string(component, "id")?; + let kind = required_string(component, "component")?; + let allowed = if catalog_id == CATALOG_ID { + ds4_catalog() + .pointer("/components") + .and_then(Value::as_object) + .is_some_and(|components| components.contains_key(kind)) + } else { + BASIC_COMPONENTS.contains(&kind) + }; + if !allowed { + return Err(format!( + "component `{id}` uses `{kind}`, which is not in catalog `{catalog_id}`" + )); + } + let required: &[&str] = match kind { + "Text" => &["text"], + "Image" => &["url"], + "Icon" => &["name"], + "Video" | "AudioPlayer" => &["url"], + "Row" | "Column" | "List" => &["children"], + "Card" => &["child"], + "Modal" => &["trigger", "content"], + "Tabs" => &["tabs"], + "Button" => &["child", "action"], + "TextField" => &["label"], + "CheckBox" => &["label", "value"], + "Slider" => &["value", "max"], + "DateTimeInput" => &["value"], + "ChoicePicker" => &["options", "value"], + "Chart" => &["chartType", "series"], + "Table" => &["columns", "rows"], + "Metric" => &["label", "value"], + "Timeline" => &["events"], + "Map" => &["locations"], + "MindMap" => &["nodes"], + "Form" => &["children"], + _ => &[], + }; + if let Some(field) = required + .iter() + .find(|field| !component.contains_key(**field)) + { + return Err(format!("component `{id}` ({kind}) requires `{field}`")); + } + let common = ["id", "component", "accessibility", "weight", "checks"]; + let specific: &[&str] = match kind { + "Text" => &["text", "variant"], + "Image" => &["url", "description", "fit", "variant"], + "Icon" => &["name"], + "Video" => &["url", "posterUrl"], + "AudioPlayer" => &["url", "description"], + "Divider" => &["axis"], + "Row" | "Column" => &["children", "justify", "align"], + "List" => &["children", "direction", "align"], + "Card" => &["child"], + "Modal" => &["trigger", "content"], + "Tabs" => &["tabs"], + "Button" => &["child", "variant", "action"], + "TextField" => &["label", "value", "placeholder", "variant"], + "CheckBox" => &["label", "value"], + "Slider" => &["label", "min", "max", "value", "steps"], + "DateTimeInput" => &["label", "value", "enableDate", "enableTime", "min", "max"], + "ChoicePicker" => &[ + "label", + "variant", + "options", + "value", + "displayStyle", + "filterable", + ], + "Chart" => &["title", "chartType", "series"], + "Table" => &["title", "columns", "rows"], + "Metric" => &["label", "value", "detail", "trend"], + "Timeline" => &["title", "events"], + "Map" => &["title", "locations"], + "MindMap" => &["title", "nodes"], + "Form" => &["title", "children", "submitLabel", "action"], + _ => &[], + }; + if let Some(field) = component + .keys() + .find(|field| !common.contains(&field.as_str()) && !specific.contains(&field.as_str())) + { + return Err(format!( + "component `{id}` ({kind}) contains unknown property `{field}`" + )); + } + if let Some(accessibility) = component.get("accessibility") { + let accessibility = object(Some(accessibility), "accessibility")?; + for field in ["label", "description"] { + validate_optional_dynamic(accessibility, field, Value::is_string, id)?; + } + } + if component + .get("weight") + .is_some_and(|value| !value.is_number()) + { + return Err(format!("component `{id}` weight must be a number")); + } + if component + .get("title") + .is_some_and(|value| !value.is_string()) + { + return Err(format!("component `{id}` title must be a string")); + } + if catalog_id != CATALOG_ID + && component.contains_key("checks") + && !matches!( + kind, + "Button" | "TextField" | "CheckBox" | "ChoicePicker" | "Slider" | "DateTimeInput" + ) + { + return Err(format!("component `{id}` ({kind}) does not support checks")); + } + match kind { + "Row" | "Column" | "List" | "Form" => { + validate_children(component.get("children").unwrap(), id)? + } + "Card" | "Button" => expect_string(component, "child", id)?, + "Modal" => { + expect_string(component, "trigger", id)?; + expect_string(component, "content", id)?; + } + "Tabs" => validate_tabs(component.get("tabs").unwrap(), id)?, + "ChoicePicker" => validate_options(component.get("options").unwrap(), id)?, + "Slider" => { + expect_number(component, "max", id)?; + if let Some(min) = component.get("min") + && !min.is_number() + { + return Err(format!("component `{id}` min must be a number")); + } + if let Some(steps) = component.get("steps") + && !steps.as_u64().is_some_and(|steps| steps > 0) + { + return Err(format!("component `{id}` steps must be a positive integer")); + } + } + _ => {} + } + match kind { + "Text" => validate_dynamic_field(component, "text", Value::is_string, id)?, + "Image" | "Video" | "AudioPlayer" => { + validate_dynamic_field(component, "url", Value::is_string, id)?; + validate_optional_dynamic(component, "description", Value::is_string, id)?; + validate_optional_dynamic(component, "posterUrl", Value::is_string, id)?; + } + "Icon" => validate_dynamic_field(component, "name", Value::is_string, id)?, + "TextField" | "DateTimeInput" => { + validate_optional_dynamic(component, "label", Value::is_string, id)?; + validate_optional_dynamic(component, "value", Value::is_string, id)?; + } + "CheckBox" => { + validate_dynamic_field(component, "label", Value::is_string, id)?; + validate_dynamic_field(component, "value", Value::is_boolean, id)?; + } + "Slider" => validate_dynamic_field(component, "value", Value::is_number, id)?, + "ChoicePicker" => { + validate_optional_dynamic(component, "label", Value::is_string, id)?; + validate_dynamic_field( + component, + "value", + |value| { + value + .as_array() + .is_some_and(|values| values.iter().all(Value::is_string)) + }, + id, + )?; + } + "Chart" | "Table" | "Timeline" | "Map" | "MindMap" => { + for field in required { + if *field != "chartType" { + validate_dynamic_field(component, field, Value::is_array, id)?; + } + } + } + _ => {} + } + match kind { + "Text" => validate_enum(component, "variant", &["caption", "body"], id)?, + "Image" => { + validate_enum( + component, + "fit", + &["contain", "cover", "fill", "none", "scaleDown"], + id, + )?; + validate_enum( + component, + "variant", + &[ + "icon", + "avatar", + "smallFeature", + "mediumFeature", + "largeFeature", + "header", + ], + id, + )?; + } + "Icon" if component.get("name").is_some_and(Value::is_string) => validate_enum( + component, + "name", + &[ + "accountCircle", + "add", + "arrowBack", + "arrowForward", + "attachFile", + "calendarToday", + "call", + "camera", + "check", + "close", + "delete", + "download", + "edit", + "event", + "error", + "fastForward", + "favorite", + "favoriteOff", + "folder", + "help", + "home", + "info", + "locationOn", + "lock", + "lockOpen", + "mail", + "menu", + "moreVert", + "moreHoriz", + "notificationsOff", + "notifications", + "pause", + "payment", + "person", + "phone", + "photo", + "play", + "print", + "refresh", + "rewind", + "search", + "send", + "settings", + "share", + "shoppingCart", + "skipNext", + "skipPrevious", + "star", + "starHalf", + "starOff", + "stop", + "upload", + "visibility", + "visibilityOff", + "volumeDown", + "volumeMute", + "volumeOff", + "volumeUp", + "warning", + ], + id, + )?, + "Divider" => validate_enum(component, "axis", &["horizontal", "vertical"], id)?, + "Row" | "Column" => { + validate_enum( + component, + "justify", + &[ + "start", + "center", + "end", + "spaceBetween", + "spaceAround", + "spaceEvenly", + "stretch", + ], + id, + )?; + validate_enum( + component, + "align", + &["start", "center", "end", "stretch"], + id, + )?; + } + "List" => { + validate_enum(component, "direction", &["vertical", "horizontal"], id)?; + validate_enum( + component, + "align", + &["start", "center", "end", "stretch"], + id, + )?; + } + "Button" => validate_enum( + component, + "variant", + &["default", "primary", "borderless"], + id, + )?, + "TextField" => validate_enum( + component, + "variant", + &["longText", "number", "shortText", "obscured"], + id, + )?, + "ChoicePicker" => { + validate_enum( + component, + "variant", + &["multipleSelection", "mutuallyExclusive"], + id, + )?; + validate_enum(component, "displayStyle", &["checkbox", "chips"], id)?; + optional_bool(component, "filterable")?; + } + "DateTimeInput" => { + optional_bool(component, "enableDate")?; + optional_bool(component, "enableTime")?; + validate_optional_dynamic(component, "min", Value::is_string, id)?; + validate_optional_dynamic(component, "max", Value::is_string, id)?; + } + "Chart" => validate_enum( + component, + "chartType", + &["bar", "line", "area", "stackedBar", "pie", "donut"], + id, + )?, + _ => {} + } + if let Some(action) = component.get("action") { + validate_action(action, id)?; + } + if let Some(checks) = component.get("checks") { + let checks = checks + .as_array() + .ok_or_else(|| format!("component `{id}` checks must be an array"))?; + for check in checks { + let check = object(Some(check), "check")?; + reject_unknown(check, &["condition", "message"], "check")?; + let condition = check + .get("condition") + .ok_or_else(|| format!("component `{id}` check requires condition"))?; + required_string(check, "message")?; + validate_function(condition)?; + } + } + validate_dynamic_values(component)?; + Ok(()) +} + +fn validate_dynamic_field( + object: &Map, + field: &str, + literal: impl Fn(&Value) -> bool, + id: &str, +) -> Result<(), String> { + let value = object + .get(field) + .ok_or_else(|| format!("component `{id}` requires `{field}`"))?; + if literal(value) { + return Ok(()); + } + let dynamic = value + .as_object() + .ok_or_else(|| format!("component `{id}` {field} has the wrong literal or dynamic type"))?; + if let Some(path) = dynamic.get("path") { + if dynamic.len() == 1 && path.is_string() { + return Ok(()); + } + return Err(format!( + "component `{id}` {field} has an invalid data binding" + )); + } + validate_function(value).map_err(|error| format!("component `{id}` {field}: {error}")) +} + +fn validate_optional_dynamic( + object: &Map, + field: &str, + literal: impl Fn(&Value) -> bool, + id: &str, +) -> Result<(), String> { + if object.contains_key(field) { + validate_dynamic_field(object, field, literal, id) + } else { + Ok(()) + } +} + +fn ds4_catalog() -> &'static Value { + static CATALOG: OnceLock = OnceLock::new(); + CATALOG.get_or_init(|| { + serde_json::from_str(CATALOG_JSON).expect("embedded DS4Server A2UI catalog must be valid") + }) +} + +fn expect_string(object: &Map, field: &str, id: &str) -> Result<(), String> { + if !object.get(field).is_some_and(Value::is_string) { + return Err(format!("component `{id}` {field} must be a string")); + } + Ok(()) +} + +fn expect_number(object: &Map, field: &str, id: &str) -> Result<(), String> { + if !object.get(field).is_some_and(Value::is_number) { + return Err(format!("component `{id}` {field} must be a number")); + } + Ok(()) +} + +fn validate_enum( + object: &Map, + field: &str, + allowed: &[&str], + id: &str, +) -> Result<(), String> { + if let Some(value) = object.get(field) { + let value = value + .as_str() + .ok_or_else(|| format!("component `{id}` {field} must be a string"))?; + if !allowed.contains(&value) { + return Err(format!("component `{id}` has invalid {field} `{value}`")); + } + } + Ok(()) +} + +fn validate_children(value: &Value, id: &str) -> Result<(), String> { + if value + .as_array() + .is_some_and(|children| children.iter().all(Value::is_string)) + { + return Ok(()); + } + let template = object(Some(value), "children")?; + reject_unknown(template, &["componentId", "path"], "children template")?; + required_string(template, "componentId")?; + required_string(template, "path")?; + if !required_string(template, "path")?.starts_with('/') { + return Err(format!( + "component `{id}` child template path must be a JSON Pointer" + )); + } + Ok(()) +} + +fn validate_tabs(value: &Value, id: &str) -> Result<(), String> { + let tabs = value + .as_array() + .filter(|tabs| !tabs.is_empty()) + .ok_or_else(|| format!("component `{id}` tabs must be a non-empty array"))?; + for tab in tabs { + let tab = object(Some(tab), "tab")?; + reject_unknown(tab, &["title", "child"], "tab")?; + validate_dynamic_field(tab, "title", Value::is_string, id)?; + required_string(tab, "child")?; + } + Ok(()) +} + +fn validate_options(value: &Value, id: &str) -> Result<(), String> { + let options = value + .as_array() + .ok_or_else(|| format!("component `{id}` options must be an array"))?; + for option in options { + let option = object(Some(option), "choice option")?; + reject_unknown(option, &["label", "value"], "choice option")?; + validate_dynamic_field(option, "label", Value::is_string, id)?; + required_string(option, "value")?; + } + Ok(()) +} + +fn validate_action(value: &Value, id: &str) -> Result<(), String> { + let action = object(Some(value), "action")?; + if let Some(event) = action.get("event") { + reject_unknown(action, &["event"], "action")?; + let event = object(Some(event), "action.event")?; + reject_unknown( + event, + &["name", "context", "wantResponse", "responsePath"], + "action.event", + )?; + required_string(event, "name")?; + if let Some(context) = event.get("context") + && !context.is_object() + { + return Err(format!("component `{id}` action context must be an object")); + } + optional_bool(event, "wantResponse")?; + if let Some(path) = event.get("responsePath") + && !path.as_str().is_some_and(|path| path.starts_with('/')) + { + return Err(format!( + "component `{id}` responsePath must be a JSON Pointer" + )); + } + return Ok(()); + } + reject_unknown(action, &["functionCall"], "action")?; + validate_function( + action + .get("functionCall") + .ok_or_else(|| format!("component `{id}` action requires event or functionCall"))?, + ) +} + +fn validate_dynamic_values(value: &Map) -> Result<(), String> { + for value in value.values() { + if value.get("call").is_some() { + validate_function(value)?; + } + match value { + Value::Object(object) => validate_dynamic_values(object)?, + Value::Array(values) => { + for value in values { + if let Value::Object(object) = value { + validate_dynamic_values(object)?; + } + } + } + _ => {} + } + } + Ok(()) +} + +fn validate_function(value: &Value) -> Result<(), String> { + let function = object(Some(value), "function call")?; + reject_unknown(function, &["call", "args"], "function call")?; + let call = value + .get("call") + .and_then(Value::as_str) + .ok_or_else(|| "function call requires `call`".to_owned())?; + if !FUNCTIONS.contains(&call) { + return Err(format!("function `{call}` is not declared by the catalog")); + } + if value.get("args").is_some_and(|args| !args.is_object()) { + return Err(format!("function `{call}` requires object `args`")); + } + let args = value + .get("args") + .and_then(Value::as_object) + .cloned() + .unwrap_or_default(); + let allowed: &[&str] = match call { + "required" | "email" | "formatString" | "not" => &["value"], + "regex" => &["value", "pattern"], + "length" | "numeric" => &["value", "min", "max"], + "formatNumber" => &["value", "decimals", "grouping"], + "formatCurrency" => &["value", "currency", "decimals", "grouping"], + "formatDate" => &["value", "format"], + "pluralize" => &["value", "zero", "one", "two", "few", "many", "other"], + "openUrl" => &["url"], + "and" | "or" => &["values"], + "@index" => &["offset"], + _ => &[], + }; + reject_unknown(&args, allowed, &format!("function `{call}` args"))?; + let required: &[&str] = match call { + "required" | "regex" | "length" | "numeric" | "email" | "formatString" | "formatNumber" + | "not" | "pluralize" => &["value"], + "formatCurrency" => &["value", "currency"], + "formatDate" => &["value", "format"], + "openUrl" => &["url"], + "and" | "or" => &["values"], + "@index" => &[], + _ => &[], + }; + if let Some(field) = required.iter().find(|field| !args.contains_key(**field)) { + return Err(format!("function `{call}` requires argument `{field}`")); + } + if call == "regex" && !args.get("pattern").is_some_and(Value::is_string) { + return Err("function `regex` requires string argument `pattern`".into()); + } + if matches!(call, "length" | "numeric") + && !args.contains_key("min") + && !args.contains_key("max") + { + return Err(format!("function `{call}` requires `min` or `max`")); + } + if call == "pluralize" && !args.contains_key("other") { + return Err("function `pluralize` requires argument `other`".into()); + } + if matches!(call, "and" | "or") + && !args + .get("values") + .and_then(Value::as_array) + .is_some_and(|values| values.len() >= 2) + { + return Err(format!("function `{call}` requires at least two values")); + } + match call { + "regex" | "length" | "email" | "formatString" => { + validate_dynamic_field(&args, "value", Value::is_string, call)?; + } + "numeric" | "formatNumber" | "formatCurrency" | "pluralize" => { + validate_dynamic_field(&args, "value", Value::is_number, call)?; + } + "not" => validate_dynamic_field(&args, "value", Value::is_boolean, call)?, + "formatDate" => validate_dynamic_field(&args, "format", Value::is_string, call)?, + "openUrl" => { + let url = required_string(&args, "url")?; + url::Url::parse(url).map_err(|error| format!("invalid openUrl URL: {error}"))?; + } + "and" | "or" => { + for item in args["values"].as_array().unwrap() { + if !item.is_boolean() { + let item = item.as_object().ok_or_else(|| { + format!("function `{call}` values must be dynamic booleans") + })?; + if !item.contains_key("path") && !item.contains_key("call") { + return Err(format!("function `{call}` values must be dynamic booleans")); + } + } + } + } + _ => {} + } + for field in ["min", "max", "decimals", "offset"] { + if args.contains_key(field) { + validate_dynamic_field(&args, field, Value::is_number, call)?; + } + } + if args.contains_key("grouping") { + validate_dynamic_field(&args, "grouping", Value::is_boolean, call)?; + } + for field in ["currency", "zero", "one", "two", "few", "many", "other"] { + if args.contains_key(field) { + validate_dynamic_field(&args, field, Value::is_string, call)?; + } + } + Ok(()) +} + +fn resolve(value: &Value, data: &Value) -> Result { + resolve_at(value, data, data) +} + +fn resolve_at(value: &Value, data: &Value, context: &Value) -> Result { + if let Some(path) = value + .as_object() + .and_then(|value| value.get("path")) + .and_then(Value::as_str) + { + let source = if path.starts_with('/') { data } else { context }; + let pointer = if path.starts_with('/') { + normalize_pointer(path).to_owned() + } else { + format!("/{path}") + }; + return Ok(source.pointer(&pointer).cloned().unwrap_or(Value::Null)); + } + if let Some(call) = value + .as_object() + .and_then(|value| value.get("call")) + .and_then(Value::as_str) + { + return evaluate( + call, + value.get("args").unwrap_or(&Value::Null), + data, + context, + ); + } + match value { + Value::Array(values) => values + .iter() + .map(|value| resolve_at(value, data, context)) + .collect::, _>>() + .map(Value::Array), + Value::Object(values) => values + .iter() + .map(|(key, value)| Ok((key.clone(), resolve_at(value, data, context)?))) + .collect::, String>>() + .map(Value::Object), + _ => Ok(value.clone()), + } +} + +fn evaluate(call: &str, args: &Value, data: &Value, context: &Value) -> Result { + if !FUNCTIONS.contains(&call) { + return Err(format!("function `{call}` is not declared by the catalog")); + } + let args = args + .as_object() + .ok_or_else(|| format!("function `{call}` requires object args"))?; + let value = || resolve_at(args.get("value").unwrap_or(&Value::Null), data, context); + match call { + "required" => Ok(Value::Bool(match value()? { + Value::Null => false, + Value::String(value) => !value.is_empty(), + Value::Array(value) => !value.is_empty(), + Value::Object(value) => !value.is_empty(), + _ => true, + })), + "regex" => { + let pattern = args + .get("pattern") + .and_then(Value::as_str) + .ok_or_else(|| "regex requires a pattern".to_owned())?; + let regex = Regex::new(pattern).map_err(|error| format!("invalid regex: {error}"))?; + Ok(Value::Bool(regex.is_match(&display_value(&value()?)))) + } + "length" => { + let length = display_value(&value()?).chars().count() as u64; + let min = args.get("min").and_then(Value::as_u64).unwrap_or(0); + let max = args.get("max").and_then(Value::as_u64).unwrap_or(u64::MAX); + Ok(Value::Bool((min..=max).contains(&length))) + } + "numeric" => { + let resolved = value()?; + let number = resolved + .as_f64() + .or_else(|| resolved.as_str().and_then(|value| value.parse().ok())); + let min = args + .get("min") + .and_then(Value::as_f64) + .unwrap_or(f64::NEG_INFINITY); + let max = args + .get("max") + .and_then(Value::as_f64) + .unwrap_or(f64::INFINITY); + Ok(Value::Bool( + number.is_some_and(|number| (min..=max).contains(&number)), + )) + } + "email" => { + let email = display_value(&value()?); + let valid = Regex::new(r"^[^\s@]+@[^\s@]+\.[^\s@]+$") + .unwrap() + .is_match(&email); + Ok(Value::Bool(valid)) + } + "formatString" => Ok(Value::String(interpolate( + &display_value(&value()?), + data, + context, + )?)), + "formatNumber" => { + let number = value()?.as_f64().unwrap_or(0.0); + let digits = resolve_at(args.get("decimals").unwrap_or(&json!(2)), data, context)? + .as_f64() + .map(|value| value.max(0.0) as u64) + .unwrap_or(2) + .min(12) as usize; + let formatted = format!("{number:.digits$}"); + Ok(Value::String( + if resolve_at( + args.get("grouping").unwrap_or(&Value::Bool(true)), + data, + context, + )? + .as_bool() + .unwrap_or(true) + { + group_number(&formatted) + } else { + formatted + }, + )) + } + "formatCurrency" => { + let number = value()?.as_f64().unwrap_or(0.0); + let currency = display_value(&resolve_at( + args.get("currency").unwrap_or(&Value::Null), + data, + context, + )?); + let digits = resolve_at(args.get("decimals").unwrap_or(&json!(2)), data, context)? + .as_f64() + .map(|value| value.max(0.0) as u64) + .unwrap_or(2) + .min(12) as usize; + let formatted = format!("{number:.digits$}"); + let number = if resolve_at( + args.get("grouping").unwrap_or(&Value::Bool(true)), + data, + context, + )? + .as_bool() + .unwrap_or(true) + { + group_number(&formatted) + } else { + formatted + }; + Ok(Value::String(format!("{currency} {number}"))) + } + "formatDate" => { + let input = display_value(&value()?); + let date = + time::OffsetDateTime::parse(&input, &time::format_description::well_known::Rfc3339) + .map_err(|error| format!("formatDate requires an RFC 3339 value: {error}"))?; + let pattern = display_value(&resolve_at( + args.get("format").unwrap_or(&Value::Null), + data, + context, + )?); + Ok(Value::String(format_date(date, &pattern))) + } + "pluralize" => { + let number = value()?.as_f64().unwrap_or(0.0); + let key = if number == 0.0 && args.contains_key("zero") { + "zero" + } else if number == 1.0 && args.contains_key("one") { + "one" + } else if number == 2.0 && args.contains_key("two") { + "two" + } else { + "other" + }; + resolve_at(args.get(key).unwrap_or(&Value::Null), data, context) + } + "and" | "or" => { + let values = args + .get("values") + .and_then(Value::as_array) + .ok_or_else(|| format!("{call} requires array `values`"))?; + let values = values + .iter() + .map(|value| { + resolve_at(value, data, context)? + .as_bool() + .ok_or_else(|| format!("{call} requires boolean values")) + }) + .collect::, _>>()?; + Ok(Value::Bool(if call == "and" { + values.into_iter().all(|value| value) + } else { + values.into_iter().any(|value| value) + })) + } + "not" => { + Ok(Value::Bool(!value()?.as_bool().ok_or_else(|| { + "not requires a boolean value".to_owned() + })?)) + } + "openUrl" => Ok(Value::Null), + "@index" => { + let index = TEMPLATE_INDEX + .with(Cell::get) + .ok_or_else(|| "@index is only available in a list template".to_owned())?; + let offset = args + .get("offset") + .map(|value| resolve_at(value, data, context)) + .transpose()? + .and_then(|value| value.as_i64()) + .unwrap_or(0); + Ok(json!(index as i64 + offset)) + } + _ => unreachable!(), + } +} + +fn interpolate(template: &str, data: &Value, context: &Value) -> Result { + let mut output = String::new(); + let mut offset = 0; + while let Some(relative) = template[offset..].find("${") { + let start = offset + relative; + if start > offset && template.as_bytes()[start - 1] == b'\\' { + output.push_str(&template[offset..start - 1]); + output.push_str("${"); + offset = start + 2; + continue; + } + output.push_str(&template[offset..start]); + let end = expression_end(template, start + 2) + .ok_or_else(|| "formatString contains an unclosed expression".to_owned())?; + output.push_str(&display_value(&evaluate_expression( + &template[start + 2..end], + data, + context, + )?)); + offset = end + 1; + } + output.push_str(&template[offset..]); + Ok(output) +} + +fn expression_end(text: &str, mut offset: usize) -> Option { + let mut depth = 1; + let mut quote = None; + while offset < text.len() { + let character = text[offset..].chars().next()?; + if let Some(current) = quote { + if character == current && text.as_bytes().get(offset.wrapping_sub(1)) != Some(&b'\\') { + quote = None; + } + } else if matches!(character, '\'' | '"') { + quote = Some(character); + } else if text[offset..].starts_with("${") { + depth += 1; + offset += 2; + continue; + } else if character == '}' { + depth -= 1; + if depth == 0 { + return Some(offset); + } + } + offset += character.len_utf8(); + } + None +} + +fn evaluate_expression(expression: &str, data: &Value, context: &Value) -> Result { + let expression = expression.trim(); + if let Some(open) = expression.find('(') + && expression.ends_with(')') + { + let call = expression[..open].trim(); + let mut args = Map::new(); + for argument in split_expression_args(&expression[open + 1..expression.len() - 1]) { + let colon = top_level_separator(argument, ':') + .ok_or_else(|| format!("formatString argument `{argument}` must be named"))?; + let name = argument[..colon].trim(); + if name.is_empty() { + return Err("formatString contains an empty argument name".into()); + } + args.insert( + name.to_owned(), + expression_value(argument[colon + 1..].trim(), data, context)?, + ); + } + return evaluate(call, &Value::Object(args), data, context); + } + let (source, pointer) = if expression.starts_with('/') { + (data, normalize_pointer(expression).to_owned()) + } else { + (context, format!("/{expression}")) + }; + Ok(source.pointer(&pointer).cloned().unwrap_or(Value::Null)) +} + +fn split_expression_args(input: &str) -> Vec<&str> { + let mut arguments = Vec::new(); + let mut start = 0; + while let Some(relative) = top_level_separator(&input[start..], ',') { + arguments.push(input[start..start + relative].trim()); + start += relative + 1; + } + if !input[start..].trim().is_empty() { + arguments.push(input[start..].trim()); + } + arguments +} + +fn top_level_separator(input: &str, separator: char) -> Option { + let mut round = 0; + let mut braces = 0; + let mut quote = None; + for (index, character) in input.char_indices() { + if let Some(current) = quote { + if character == current && input.as_bytes().get(index.wrapping_sub(1)) != Some(&b'\\') { + quote = None; + } + continue; + } + match character { + '\'' | '"' => quote = Some(character), + '(' => round += 1, + ')' => round -= 1, + '{' => braces += 1, + '}' => braces -= 1, + _ if character == separator && round == 0 && braces == 0 => return Some(index), + _ => {} + } + } + None +} + +fn expression_value(expression: &str, data: &Value, context: &Value) -> Result { + if expression.starts_with("${") && expression.ends_with('}') { + return evaluate_expression(&expression[2..expression.len() - 1], data, context); + } + if expression.starts_with('\'') && expression.ends_with('\'') && expression.len() >= 2 { + return Ok(Value::String( + expression[1..expression.len() - 1].to_owned(), + )); + } + if let Ok(value) = serde_json::from_str(expression) { + return Ok(value); + } + if expression.starts_with('/') { + return Ok(data + .pointer(normalize_pointer(expression)) + .cloned() + .unwrap_or(Value::Null)); + } + Ok(Value::String(expression.to_owned())) +} + +fn group_number(number: &str) -> String { + let (whole, fraction) = number.split_once('.').unwrap_or((number, "")); + let (sign, digits) = whole + .strip_prefix('-') + .map_or(("", whole), |digits| ("-", digits)); + let mut grouped = String::with_capacity(number.len() + number.len() / 3); + grouped.push_str(sign); + for (index, digit) in digits.chars().enumerate() { + if index > 0 && (digits.len() - index).is_multiple_of(3) { + grouped.push(','); + } + grouped.push(digit); + } + if !fraction.is_empty() { + grouped.push('.'); + grouped.push_str(fraction); + } + grouped +} + +fn format_date(date: time::OffsetDateTime, pattern: &str) -> String { + const MONTHS: [&str; 12] = [ + "January", + "February", + "March", + "April", + "May", + "June", + "July", + "August", + "September", + "October", + "November", + "December", + ]; + const DAYS: [&str; 7] = [ + "Monday", + "Tuesday", + "Wednesday", + "Thursday", + "Friday", + "Saturday", + "Sunday", + ]; + let month = MONTHS[date.month() as usize - 1]; + let day = DAYS[date.weekday().number_days_from_monday() as usize]; + let hour_12 = match date.hour() % 12 { + 0 => 12, + hour => hour, + }; + let replacements = [ + ("EEEE", day.to_owned()), + ("MMMM", month.to_owned()), + ("yyyy", format!("{:04}", date.year())), + ("MMM", month[..3].to_owned()), + ("yy", format!("{:02}", date.year().rem_euclid(100))), + ("MM", format!("{:02}", date.month() as u8)), + ("dd", format!("{:02}", date.day())), + ("HH", format!("{:02}", date.hour())), + ("hh", format!("{hour_12:02}")), + ("mm", format!("{:02}", date.minute())), + ("ss", format!("{:02}", date.second())), + ("E", day[..3].to_owned()), + ("M", (date.month() as u8).to_string()), + ("d", date.day().to_string()), + ("H", date.hour().to_string()), + ("h", hour_12.to_string()), + ("a", if date.hour() < 12 { "AM" } else { "PM" }.to_owned()), + ]; + let mut output = String::new(); + let mut remaining = pattern; + while !remaining.is_empty() { + if let Some((token, replacement)) = replacements + .iter() + .find(|(token, _)| remaining.starts_with(token)) + { + output.push_str(replacement); + remaining = &remaining[token.len()..]; + } else { + let character = remaining.chars().next().unwrap(); + output.push(character); + remaining = &remaining[character.len_utf8()..]; + } + } + output +} + +fn set_pointer(root: &mut Value, path: &str, value: Option) -> Result<(), String> { + if path.is_empty() || path == "/" { + *root = value.unwrap_or(Value::Null); + return Ok(()); + } + let segments = path[1..] + .split('/') + .map(|segment| segment.replace("~1", "/").replace("~0", "~")) + .collect::>(); + let (last, parents) = segments.split_last().unwrap(); + let mut current = root; + for segment in parents { + match current { + Value::Object(object) => { + current = object + .entry(segment.clone()) + .or_insert_with(|| Value::Object(Map::new())); + } + Value::Array(array) => { + let index = segment + .parse::() + .map_err(|_| format!("`{segment}` is not an array index"))?; + while array.len() <= index { + array.push(Value::Object(Map::new())); + } + current = &mut array[index]; + } + _ => return Err(format!("cannot traverse through `{segment}`")), + } + } + match current { + Value::Object(object) => match value { + Some(value) => { + object.insert(last.clone(), value); + } + None => { + object.remove(last); + } + }, + Value::Array(array) => { + let index = last + .parse::() + .map_err(|_| format!("`{last}` is not an array index"))?; + if let Some(value) = value { + while array.len() <= index { + array.push(Value::Null); + } + array[index] = value; + } else if index < array.len() { + array.remove(index); + } + } + _ => return Err(format!("cannot update `{path}`")), + } + Ok(()) +} + +fn message_summary(value: &Value) -> String { + for (key, verb) in [ + ("createSurface", "Created"), + ("updateComponents", "Updated components on"), + ("updateDataModel", "Updated data on"), + ("deleteSurface", "Deleted"), + ("actionResponse", "Responded to action on"), + ] { + if let Some(id) = value + .get(key) + .and_then(|value| value.get("surfaceId")) + .and_then(Value::as_str) + { + return format!("{verb} interactive surface `{id}`."); + } + } + "A2UI interactive update.".into() +} + +pub(crate) fn display_value(value: &Value) -> String { + match value { + Value::Null => String::new(), + Value::String(value) => value.clone(), + Value::Bool(value) => value.to_string(), + Value::Number(value) => value.to_string(), + value => serde_json::to_string(value).unwrap_or_default(), + } +} + +fn normalize_pointer(path: &str) -> &str { + if path == "/" { "" } else { path } +} + +fn object<'a>(value: Option<&'a Value>, name: &str) -> Result<&'a Map, String> { + value + .and_then(Value::as_object) + .ok_or_else(|| format!("{name} must be an object")) +} + +fn reject_unknown(object: &Map, allowed: &[&str], name: &str) -> Result<(), String> { + if let Some(key) = object.keys().find(|key| !allowed.contains(&key.as_str())) { + return Err(format!("{name} contains unknown property `{key}`")); + } + Ok(()) +} + +fn required_string<'a>(object: &'a Map, key: &str) -> Result<&'a str, String> { + object + .get(key) + .and_then(Value::as_str) + .filter(|value| !value.is_empty()) + .ok_or_else(|| format!("`{key}` must be a non-empty string")) +} + +fn optional_bool(object: &Map, key: &str) -> Result, String> { + object + .get(key) + .map(|value| { + value + .as_bool() + .ok_or_else(|| format!("`{key}` must be a boolean")) + }) + .transpose() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn apply(store: &mut Store, value: Value) -> Result { + store.apply(value.clone(), value.to_string(), 7) + } + + #[test] + fn lifecycle_updates_incrementally_and_json_pointer_upserts() { + let mut store = Store::default(); + apply( + &mut store, + json!({"version":VERSION,"createSurface":{"surfaceId":"s","catalogId":CATALOG_ID}}), + ) + .unwrap(); + apply(&mut store, json!({"version":VERSION,"updateComponents":{"surfaceId":"s","components":[{"id":"root","component":"Text","text":{"path":"/user/name"}}]}})).unwrap(); + apply(&mut store, json!({"version":VERSION,"updateDataModel":{"surfaceId":"s","path":"/user/name","value":"Ada"}})).unwrap(); + assert_eq!( + store.surface("s").unwrap().data.pointer("/user/name"), + Some(&json!("Ada")) + ); + apply(&mut store, json!({"version":VERSION,"updateComponents":{"surfaceId":"s","components":[{"id":"root","component":"Metric","label":"Name","value":{"path":"/user/name"}}]}})).unwrap(); + assert_eq!( + store.surface("s").unwrap().components["root"]["component"], + "Metric" + ); + apply(&mut store, json!({"version":VERSION,"updateDataModel":{"surfaceId":"s","path":"/items","value":["first","second"]}})).unwrap(); + apply( + &mut store, + json!({"version":VERSION,"updateDataModel":{"surfaceId":"s","path":"/items/0"}}), + ) + .unwrap(); + assert_eq!(store.surface("s").unwrap().data["items"], json!(["second"])); + apply( + &mut store, + json!({"version":VERSION,"deleteSurface":{"surfaceId":"s"}}), + ) + .unwrap(); + assert!(store.surface("s").is_none()); + } + + #[test] + fn catalog_validation_and_actions_use_current_local_data() { + let mut store = Store::default(); + apply(&mut store, json!({"version":VERSION,"createSurface":{"surfaceId":"s","catalogId":CATALOG_ID,"sendDataModel":true}})).unwrap(); + apply(&mut store, json!({"version":VERSION,"updateComponents":{"surfaceId":"s","components":[{"id":"label","component":"Text","text":"Send"},{"id":"root","component":"Button","child":"label","checks":[{"condition":{"call":"required","args":{"value":{"path":"/name"}}},"message":"Name required"}],"action":{"event":{"name":"submit","context":{"name":{"path":"/name"}}}}}]}})).unwrap(); + assert_eq!( + store.action("s", "root", None).unwrap_err(), + "Name required" + ); + let raw = store.local_update("s", "/name", json!("Ada")).unwrap(); + assert!(raw.contains("updateDataModel")); + let action = store.action("s", "root", None).unwrap(); + assert_eq!(action.pointer("/action/context/name"), Some(&json!("Ada"))); + let metadata = store.client_metadata(); + assert_eq!( + metadata.pointer("/a2uiClientDataModel/surfaces/s/name"), + Some(&json!("Ada")) + ); + } + + #[test] + fn streaming_parser_only_accepts_complete_a2ui_jsonl_lines() { + let content = "Before\n```a2ui\n{\"version\":\"v1.0\",\"deleteSurface\":{\"surfaceId\":\"s\"}}\n{\"version\":"; + let lines = extract_lines(content); + assert_eq!(lines.len(), 1); + assert!(lines[0].value.is_ok()); + assert!(transcript_fallback(content).contains("Deleted interactive surface `s`")); + } + + #[test] + fn v1_action_and_function_round_trips_follow_current_schema() { + let mut store = Store::default(); + apply( + &mut store, + json!({ + "version": VERSION, + "createSurface": { + "surfaceId": "s", + "catalogId": CATALOG_ID, + "dataModel": {"result": null}, + "components": [ + {"id":"label","component":"Text","text":"Run"}, + {"id":"root","component":"Button","child":"label","action":{"event":{ + "name":"run","wantResponse":true,"responsePath":"/result" + }}} + ] + } + }), + ) + .unwrap(); + let action = store.action("s", "root", None).unwrap(); + assert!(action.pointer("/action/timestamp").is_some()); + let action_id = action["action"]["actionId"].as_str().unwrap(); + let response = apply( + &mut store, + json!({ + "version": VERSION, + "actionId": action_id, + "actionResponse": {"value": "done"} + }), + ) + .unwrap(); + assert_eq!(response.raws.len(), 2); + assert_eq!(store.surface("s").unwrap().data["result"], "done"); + + let applied = apply( + &mut store, + json!({ + "version": VERSION, + "functionCallId": "f1", + "wantResponse": true, + "callFunction": {"call":"formatNumber","args":{"value":1234.5,"decimals":1}} + }), + ) + .unwrap(); + assert_eq!( + applied.reply.unwrap().pointer("/functionResponse/value"), + Some(&json!("1,234.5")) + ); + } + + #[test] + fn embedded_catalog_and_validator_cover_the_same_components() { + let catalog = ds4_catalog(); + assert_eq!(catalog["catalogId"], CATALOG_ID); + let components = catalog["components"].as_object().unwrap(); + for component in BASIC_COMPONENTS { + assert!(components.contains_key(*component)); + } + for component in [ + "Chart", "Table", "Metric", "Timeline", "Map", "MindMap", "Form", + ] { + assert!(components.contains_key(component)); + } + let functions = catalog["functions"].as_object().unwrap(); + for function in FUNCTIONS.iter().filter(|function| **function != "@index") { + assert!(functions.contains_key(*function)); + } + } + + #[test] + fn all_basics_compose_into_one_surface() { + let mut store = Store::default(); + apply( + &mut store, + json!({ + "version": VERSION, + "createSurface": { + "surfaceId": "composed", + "catalogId": CATALOG_ID, + "sendDataModel": true, + "dataModel": { + "name": "Ada", + "active": true, + "priority": 3, + "due": "2026-08-01", + "language": ["rust"], + "items": [{"label": "Renderer"}, {"label": "Validator"}] + }, + "components": [ + {"id":"root","component":"Card","child":"layout"}, + {"id":"layout","component":"Column","children":["title","identity","divider","controls","tabs","items","media","modal"]}, + {"id":"title","component":"Text","text":"**Project dashboard**"}, + {"id":"identity","component":"Row","children":["icon","avatar"],"align":"center"}, + {"id":"icon","component":"Icon","name":"accountCircle"}, + {"id":"avatar","component":"Image","url":"https://example.com/avatar.png","description":"Project owner","variant":"avatar"}, + {"id":"divider","component":"Divider","axis":"horizontal"}, + {"id":"controls","component":"Row","children":["name","active","priority","due","language"]}, + {"id":"name","component":"TextField","label":"Name","value":{"path":"/name"},"weight":2}, + {"id":"active","component":"CheckBox","label":"Active","value":{"path":"/active"}}, + {"id":"priority","component":"Slider","label":"Priority","value":{"path":"/priority"},"min":1,"max":5}, + {"id":"due","component":"DateTimeInput","label":"Due","value":{"path":"/due"},"enableDate":true}, + {"id":"language","component":"ChoicePicker","label":"Language","options":[{"label":"Rust","value":"rust"},{"label":"Python","value":"python"}],"value":{"path":"/language"},"variant":"mutuallyExclusive","displayStyle":"chips","filterable":true}, + {"id":"tabs","component":"Tabs","tabs":[{"title":"Overview","child":"chart"},{"title":"Details","child":"details"}]}, + {"id":"chart","component":"Chart","chartType":"donut","series":[{"label":"Done","value":2},{"label":"Open","value":1}]}, + {"id":"details","component":"Card","child":"details-text"}, + {"id":"details-text","component":"Text","text":"All systems operational."}, + {"id":"items","component":"List","children":{"path":"/items","componentId":"item"}}, + {"id":"item","component":"Text","text":{"path":"label"}}, + {"id":"media","component":"Row","children":["video","audio"]}, + {"id":"video","component":"Video","url":"https://example.com/demo.mp4","posterUrl":"https://example.com/poster.jpg"}, + {"id":"audio","component":"AudioPlayer","url":"https://example.com/demo.mp3","description":"Project update"}, + {"id":"modal","component":"Modal","trigger":"open","content":"dialog"}, + {"id":"open","component":"Button","child":"open-label","variant":"primary","action":{"event":{"name":"openDetails","context":{"name":{"path":"/name"}}}}}, + {"id":"open-label","component":"Text","text":"Open details"}, + {"id":"dialog","component":"Card","child":"dialog-body"}, + {"id":"dialog-body","component":"Text","text":"Composed modal content"} + ] + } + }), + ) + .unwrap(); + + let surface = store.surface("composed").unwrap(); + let kinds = validate_surface_composition(surface).unwrap(); + for component in BASIC_COMPONENTS { + assert!(kinds.contains(*component), "missing {component}"); + } + assert!(kinds.contains("Chart")); + assert!( + store + .image_urls() + .any(|url| url == "https://example.com/poster.jpg") + ); + + let mut broken = surface.clone(); + broken.components.insert( + "root".into(), + json!({"id":"root","component":"Card","child":"missing"}), + ); + assert_eq!( + validate_surface_composition(&broken).unwrap_err(), + "referenced component `missing` is missing" + ); + broken.components.insert( + "root".into(), + json!({"id":"root","component":"Card","child":"root"}), + ); + assert_eq!( + validate_surface_composition(&broken).unwrap_err(), + "cyclic component reference at `root`" + ); + } + + #[test] + fn catalog_validation_rejects_invalid_variants_and_function_arguments() { + let mut store = Store::default(); + apply( + &mut store, + json!({"version":VERSION,"createSurface":{"surfaceId":"s","catalogId":CATALOG_ID}}), + ) + .unwrap(); + assert!(apply(&mut store, json!({"version":VERSION,"updateComponents":{"surfaceId":"s","components":[{"id":"root","component":"Text","text":"x","variant":"h1"}]}})).is_err()); + assert!(apply(&mut store, json!({"version":VERSION,"updateComponents":{"surfaceId":"s","components":[{"id":"root","component":"Text","text":{"call":"formatDate","args":{"value":"2026-01-01T00:00:00Z"}}}]}})).is_err()); + } + + #[test] + fn catalog_accepts_pie_and_donut_charts() { + let mut store = Store::default(); + apply( + &mut store, + json!({"version":VERSION,"createSurface":{"surfaceId":"s","catalogId":CATALOG_ID}}), + ) + .unwrap(); + for chart_type in ["pie", "donut"] { + apply( + &mut store, + json!({"version":VERSION,"updateComponents":{"surfaceId":"s","components":[{"id":"root","component":"Chart","chartType":chart_type,"series":[{"label":"rs","value":41}]}]}}), + ) + .unwrap(); + } + } + + #[test] + fn basic_formatters_follow_catalog_examples() { + let date = time::OffsetDateTime::parse( + "2026-03-16T14:30:00Z", + &time::format_description::well_known::Rfc3339, + ) + .unwrap(); + assert_eq!(format_date(date, "EEEE, d MMMM"), "Monday, 16 March"); + let data = json!({"name":"Ada","currentDate":"2026-03-16T14:30:00Z"}); + assert_eq!( + evaluate( + "formatString", + &json!({"value":"Hello ${/name}: ${formatDate(value:${/currentDate}, format:'MMM dd, yyyy')} \\${literal}"}), + &data, + &data, + ) + .unwrap(), + json!("Hello Ada: Mar 16, 2026 ${literal}") + ); + } +} diff --git a/src/a2ui_validation.rs b/src/a2ui_validation.rs new file mode 100644 index 0000000..6e34344 --- /dev/null +++ b/src/a2ui_validation.rs @@ -0,0 +1,258 @@ +use crate::a2ui::{BASIC_NON_MEDIA_COMPONENTS, Store, extract_lines, validate_surface_composition}; +use crate::config::Config; +use crate::model::ModelChoice; +use serde_json::{Value, json}; +use std::time::Duration; + +const CASES: &[Case] = &[ + Case { + name: "pie-natural", + prompt: "Do not call tools. The file extension counts are rs 41, md 7, and toml 2. Give me a pie chart with the numbers.", + components: &["Chart"], + chart_type: Some("pie"), + }, + Case { + name: "pie-single-slice", + prompt: "Do not call tools. There are 41 rs files and no other file extensions. Show that distribution as a one-slice pie chart.", + components: &["Chart"], + chart_type: Some("pie"), + }, + Case { + name: "donut-natural", + prompt: "Do not call tools. Show a donut chart for 62 completed, 23 active, and 15 blocked tasks. Include the total.", + components: &["Chart"], + chart_type: Some("donut"), + }, + Case { + name: "form-controls", + prompt: "Do not call tools. Build an A2UI Form for a name, multiline notes, due date, and multiple checkbox priorities, with a submit button.", + components: &["Form"], + chart_type: None, + }, + Case { + name: "filterable-choices", + prompt: "Do not call tools. Build an A2UI surface with a filterable mutually-exclusive ChoicePicker for Rust, Python, and TypeScript.", + components: &["ChoicePicker"], + chart_type: None, + }, + Case { + name: "composed-basics", + prompt: "Do not call tools. Build one complete A2UI project dashboard. Compose a Card and Column containing Markdown text, an avatar Image, an Icon, a Divider, a weighted Row, Tabs, a dynamic List, a Modal, a Button, TextField, CheckBox, Slider, DateTimeInput, and a filterable mutually-exclusive ChoicePicker. Bind the controls to the data model and give the button an event.", + components: BASIC_NON_MEDIA_COMPONENTS, + chart_type: None, + }, + Case { + name: "media-players", + prompt: "Do not call tools. Build one A2UI Card containing a Video with a poster URL and an AudioPlayer with a description. Use valid HTTPS media URLs and compose both into the root tree.", + components: &["Video", "AudioPlayer"], + chart_type: None, + }, +]; + +struct Case { + name: &'static str, + prompt: &'static str, + components: &'static [&'static str], + chart_type: Option<&'static str>, +} + +struct Options { + endpoint: Option, + model: Option, + case: Option, + attempts: u32, +} + +pub(crate) fn run(args: impl Iterator) -> Result<(), String> { + let options = parse_options(args)?; + let config = Config::load(&crate::app::config_path())?; + let endpoint = options + .endpoint + .unwrap_or_else(|| format!("http://127.0.0.1:{}", config.endpoint.port)); + let model_id = options + .model + .unwrap_or_else(|| config.model.id().to_owned()); + let model = + ModelChoice::from_id(&model_id).ok_or_else(|| format!("unknown model `{model_id}`"))?; + let cases = CASES + .iter() + .filter(|case| options.case.as_deref().is_none_or(|name| name == case.name)) + .collect::>(); + if cases.is_empty() { + return Err(format!( + "unknown case; choose one of: {}", + CASES + .iter() + .map(|case| case.name) + .collect::>() + .join(", ") + )); + } + let agent: ureq::Agent = ureq::Agent::config_builder() + .timeout_connect(Some(Duration::from_secs(5))) + .timeout_recv_response(Some(Duration::from_secs(30 * 60))) + .timeout_recv_body(Some(Duration::from_secs(30))) + .build() + .into(); + let mut system = crate::agent::system_prompt(model, &config.generation.system_prompt); + system.push_str("\n\n"); + system.push_str(crate::a2ui::SYSTEM_PROMPT); + let metadata = Store::default().client_metadata(); + let mut passed = 0; + let total = cases.len() as u32 * options.attempts; + println!("A2UI live validation: {model_id} at {endpoint}"); + for case in cases { + for attempt in 1..=options.attempts { + let user = format!("{}\n\nA2UI client metadata:\n{}", case.prompt, metadata); + let payload = json!({ + "model": model_id, + "messages": [ + {"role": "system", "content": system}, + {"role": "user", "content": user} + ], + "reasoning_effort": "none", + "temperature": 0, + "max_tokens": 4096 + }); + let content = request(&agent, &endpoint, &payload)?; + match validate(case, &content) { + Ok(summary) => { + passed += 1; + println!("PASS {}#{attempt}: {summary}", case.name); + } + Err(error) => { + println!("FAIL {}#{attempt}: {error}", case.name); + println!( + "--- response ---\n{}\n--- end response ---", + bounded(&content) + ); + } + } + } + } + println!("A2UI validation summary: {passed}/{total} passed"); + if passed == total { + Ok(()) + } else { + Err(format!("{} live validation case(s) failed", total - passed)) + } +} + +fn parse_options(mut args: impl Iterator) -> Result { + let mut options = Options { + endpoint: None, + model: None, + case: None, + attempts: 1, + }; + while let Some(argument) = args.next() { + let value = match argument.as_str() { + "--endpoint" | "--model" | "--case" | "--attempts" => args + .next() + .ok_or_else(|| format!("{argument} requires a value"))?, + "--help" | "-h" => { + println!( + "Usage: ds4-server validate-a2ui [--endpoint URL] [--model ID] [--case NAME] [--attempts N]" + ); + std::process::exit(0); + } + _ => return Err(format!("unknown argument `{argument}`")), + }; + match argument.as_str() { + "--endpoint" => options.endpoint = Some(value.trim_end_matches('/').to_owned()), + "--model" => options.model = Some(value), + "--case" => options.case = Some(value), + "--attempts" => { + options.attempts = value + .parse::() + .ok() + .filter(|attempts| *attempts > 0) + .ok_or_else(|| "--attempts must be a positive integer".to_owned())?; + } + _ => unreachable!(), + } + } + Ok(options) +} + +fn request(agent: &ureq::Agent, endpoint: &str, payload: &Value) -> Result { + let mut response = agent + .post(&format!("{endpoint}/v1/chat/completions")) + .header("Content-Type", "application/json") + .send(payload.to_string().as_bytes()) + .map_err(|error| format!("could not call the local endpoint: {error}"))?; + let body = response + .body_mut() + .read_to_string() + .map_err(|error| format!("could not read the local endpoint response: {error}"))?; + let value: Value = serde_json::from_str(&body) + .map_err(|error| format!("local endpoint returned invalid JSON: {error}"))?; + value + .pointer("/choices/0/message/content") + .and_then(Value::as_str) + .map(str::to_owned) + .ok_or_else(|| format!("local endpoint response has no assistant content: {body}")) +} + +fn validate(case: &Case, content: &str) -> Result { + let lines = extract_lines(content); + if lines.is_empty() { + return Err("the model emitted no complete fenced A2UI JSONL messages".into()); + } + let mut store = Store::default(); + let mut errors = Vec::new(); + for (index, line) in lines.iter().enumerate() { + if let Err(error) = &line.value { + errors.push(format!("line {} is invalid JSON: {error}", index + 1)); + } else if let Err(error) = store.apply_raw(&line.raw, 1) { + errors.push(format!( + "line {} failed catalog validation: {error}", + index + 1 + )); + } + } + if !errors.is_empty() { + return Err(errors.join("; ")); + } + let surfaces = store.surfaces().collect::>(); + let mut reachable_components = std::collections::BTreeSet::new(); + for surface in &surfaces { + reachable_components.extend( + validate_surface_composition(surface) + .map_err(|error| format!("surface `{}` is not composable: {error}", surface.id))?, + ); + } + let components = surfaces + .iter() + .flat_map(|surface| surface.components.values()) + .filter_map(Value::as_object) + .collect::>(); + for expected in case.components { + if !reachable_components.contains(*expected) { + return Err(format!( + "valid surface did not compose expected {expected} component into its root tree" + )); + } + } + if let Some(expected) = case.chart_type + && !components.iter().any(|component| { + component.get("component").and_then(Value::as_str) == Some("Chart") + && component.get("chartType").and_then(Value::as_str) == Some(expected) + }) + { + return Err(format!("valid surface omitted expected {expected} chart")); + } + Ok(format!( + "{} protocol message(s), {} surface(s), {} component(s)", + lines.len(), + surfaces.len(), + components.len() + )) +} + +fn bounded(content: &str) -> &str { + content + .char_indices() + .nth(8_000) + .map_or(content, |(end, _)| &content[..end]) +} diff --git a/src/app.rs b/src/app.rs index 9b34917..d029de9 100644 --- a/src/app.rs +++ b/src/app.rs @@ -25,7 +25,7 @@ use crate::settings::{ ReasoningMode, RuntimePreferences, SpeculativePreferences, SsdPreferences, SteeringPreferences, StreamingCacheBudget, }; -use iced::widget::{markdown, scrollable}; +use iced::widget::{markdown, scrollable, text_editor}; use iced::{Size, Subscription, Task, keyboard, mouse, window}; use rfd::AsyncFileDialog; use std::collections::{HashMap, HashSet, VecDeque}; @@ -78,6 +78,15 @@ pub(crate) struct App { pub(super) composer: String, pub(super) queued_inputs: VecDeque, pub(super) conversation: Vec, + pub(super) a2ui: crate::a2ui::Store, + pub(super) a2ui_tabs: HashMap<(String, String), usize>, + pub(super) a2ui_modals: HashSet<(String, String)>, + pub(super) a2ui_editors: HashMap<(String, String, String), text_editor::Content>, + pub(super) a2ui_markdown: HashMap<(String, String, String), markdown::Content>, + pub(super) a2ui_choice_filters: HashMap<(String, String, String), String>, + pub(super) a2ui_images: HashMap, + pub(super) a2ui_image_requests: HashSet, + pub(super) a2ui_image_loading: bool, pub(super) generating: bool, pub(super) context_used: u32, pub(super) context_limit: u32, @@ -163,6 +172,7 @@ pub(crate) enum Message { PreferenceLegacyMtpChanged(bool), PreferenceDsparkChanged(bool), PreferenceTimeoutChanged(String), + PreferenceA2uiChanged(bool), PreferenceEndpointPortChanged(String), PreferenceEndpointEnabledChanged(bool), PreferenceEndpointCorsChanged(bool), @@ -199,6 +209,14 @@ pub(crate) enum Message { PreferenceKvMinTokensChanged(String), PreferenceKvColdMaxChanged(String), PreferenceKvContinuedIntervalChanged(String), + A2uiDataChanged(String, String, serde_json::Value), + A2uiEditorAction(String, String, String, text_editor::Action), + A2uiChoiceFilterChanged(String, String, String, String), + A2uiAction(String, String, Option), + A2uiSelectTab(String, String, usize), + A2uiToggleModal(String, String), + A2uiImageLoaded(String, Result, String>), + A2uiPlayMedia(String, String, bool), ResetPreferences, SavePreferences, DownloadArtifact(ManagedArtifactId), @@ -309,6 +327,15 @@ impl App { composer: String::new(), queued_inputs: VecDeque::new(), conversation: Vec::new(), + a2ui: crate::a2ui::Store::default(), + a2ui_tabs: HashMap::new(), + a2ui_modals: HashSet::new(), + a2ui_editors: HashMap::new(), + a2ui_markdown: HashMap::new(), + a2ui_choice_filters: HashMap::new(), + a2ui_images: HashMap::new(), + a2ui_image_requests: HashSet::new(), + a2ui_image_loading: false, generating: false, context_used: 0, context_limit, @@ -415,6 +442,15 @@ impl App { composer: String::new(), queued_inputs: VecDeque::new(), conversation: Vec::new(), + a2ui: crate::a2ui::Store::default(), + a2ui_tabs: HashMap::new(), + a2ui_modals: HashSet::new(), + a2ui_editors: HashMap::new(), + a2ui_markdown: HashMap::new(), + a2ui_choice_filters: HashMap::new(), + a2ui_images: HashMap::new(), + a2ui_image_requests: HashSet::new(), + a2ui_image_loading: false, generating: false, context_used: 0, context_limit, @@ -593,6 +629,10 @@ impl App { self.preference_draft.idle_timeout_minutes = value; self.preference_error = None; } + Message::PreferenceA2uiChanged(enabled) => { + self.preference_draft.a2ui_enabled = enabled; + self.preference_error = None; + } Message::PreferenceEndpointPortChanged(value) => { self.preference_draft.endpoint_port = value; self.preference_error = None; @@ -802,6 +842,71 @@ impl App { } Message::DownloadProgressTick => self.update_download_progress(), Message::ComposerChanged(value) => self.composer = value, + Message::A2uiDataChanged(surface_id, path, value) => { + return self.change_a2ui_data(surface_id, path, value); + } + Message::A2uiEditorAction(surface_id, component_id, path, action) => { + let key = (surface_id.clone(), component_id, path.clone()); + let Some(editor) = self.a2ui_editors.get_mut(&key) else { + return Task::none(); + }; + editor.perform(action); + let value = serde_json::Value::String(editor.text()); + return self.change_a2ui_data(surface_id, path, value); + } + Message::A2uiChoiceFilterChanged(surface_id, component_id, context_path, value) => { + self.a2ui_choice_filters + .insert((surface_id, component_id, context_path), value); + } + Message::A2uiAction(surface_id, component_id, context_path) => { + match self + .a2ui + .action(&surface_id, &component_id, context_path.as_deref()) + { + Ok(action) => { + self.composer = format!( + "A2UI client event:\n{}\nA2UI client metadata:\n{}", + serde_json::to_string(&action).unwrap_or_default(), + self.a2ui.client_metadata() + ); + self.start_generation(); + return scroll_chat_to_end(); + } + Err(error) => self.error = Some(error), + } + } + Message::A2uiSelectTab(surface_id, component_id, index) => { + self.a2ui_tabs.insert((surface_id, component_id), index); + } + Message::A2uiToggleModal(surface_id, component_id) => { + let key = (surface_id, component_id); + if !self.a2ui_modals.remove(&key) { + self.a2ui_modals.insert(key); + } + } + Message::A2uiImageLoaded(url, result) => { + self.a2ui_image_loading = false; + match result { + Ok(bytes) => { + self.a2ui_images + .insert(url, iced::widget::image::Handle::from_bytes(bytes)); + } + Err(error) => { + self.error = Some(format!("Could not load an A2UI image: {error}")) + } + } + return self.load_next_a2ui_image(); + } + Message::A2uiPlayMedia(url, title, video) => { + #[cfg(target_os = "macos")] + if let Err(error) = crate::native_media::open(&url, &title, video) { + self.error = Some(format!("Could not play media: {error}")); + } + #[cfg(not(target_os = "macos"))] + if let Err(error) = std::process::Command::new("open").arg(url).spawn() { + self.error = Some(format!("Could not open media: {error}")); + } + } Message::ToggleReasoning(index) => { if let Some(message) = self.conversation.get_mut(index) && message.reasoning.is_some() @@ -869,9 +974,12 @@ impl App { Message::GenerationTick => { #[cfg(target_os = "macos")] self.poll_titling(); - if self.poll_generation() { - return scroll_chat_to_end(); + let changed = self.poll_generation(); + let images = self.load_next_a2ui_image(); + if changed { + return Task::batch([scroll_chat_to_end(), images]); } + return images; } Message::ChooseProjectFolder => { self.choosing_folder = true; @@ -948,6 +1056,7 @@ impl App { self.selected_project = None; self.selected_session = None; self.conversation.clear(); + self.clear_a2ui(); self.composer.clear(); self.queued_inputs.clear(); self.system_prompt_seen_at = 0; @@ -1102,9 +1211,26 @@ impl App { let Some(database) = &mut self.database else { return Task::none(); }; - match database.load_messages(session_id) { - Ok(messages) => { + let loaded = database.load_messages(session_id).and_then(|messages| { + database + .load_a2ui_messages(session_id) + .map(|a2ui| (messages, a2ui)) + }); + match loaded { + Ok((messages, a2ui)) => { self.conversation = messages.into_iter().map(ChatMessage::from).collect(); + self.clear_a2ui(); + for message in a2ui { + if let Err(error) = + self.a2ui.apply_raw(&message.json, message.message_id) + { + self.error = Some(format!( + "Could not restore A2UI message {}: {error}", + message.id + )); + } + } + self.sync_a2ui_renderer_state(); self.composer.clear(); self.remember_project(project_id); self.selected_session = Some(session_id); @@ -1119,7 +1245,7 @@ impl App { }; self.tokens_per_second = tokens_per_second; self.error = None; - return scroll_chat_to_end(); + return Task::batch([scroll_chat_to_end(), self.load_next_a2ui_image()]); } Err(error) => { self.error = Some(format!("Could not load the chat session: {error}")); @@ -1150,6 +1276,7 @@ impl App { if self.selected_session == Some(session_id) { self.selected_session = None; self.conversation.clear(); + self.clear_a2ui(); self.composer.clear(); self.queued_inputs.clear(); self.system_prompt_seen_at = 0; @@ -1424,7 +1551,7 @@ fn models_path() -> PathBuf { } /// The settings file, beside the project database. -fn config_path() -> PathBuf { +pub(crate) fn config_path() -> PathBuf { application_support_path().join("config.yaml") } @@ -1736,6 +1863,10 @@ mod tests { reasoning_open: true, content: String::new(), markdown: markdown::Content::new(), + a2ui_lines_processed: 0, + a2ui_errors: Vec::new(), + a2ui_replies: Vec::new(), + a2ui_open_urls: Vec::new(), }; message.append(true, "working it out"); message.append(false, "**final answer**"); diff --git a/src/app/generation.rs b/src/app/generation.rs index 306cd61..00849df 100644 --- a/src/app/generation.rs +++ b/src/app/generation.rs @@ -60,6 +60,10 @@ pub(crate) struct ChatMessage { pub(super) reasoning_open: bool, pub(super) content: String, pub(super) markdown: markdown::Content, + pub(super) a2ui_lines_processed: usize, + pub(super) a2ui_errors: Vec, + pub(super) a2ui_replies: Vec, + pub(super) a2ui_open_urls: Vec, } impl ChatMessage { @@ -75,10 +79,11 @@ impl ChatMessage { pub(super) fn refresh_markdown(&mut self) { if !self.user && !self.tool { let visible = crate::agent::visible_content(&self.content); + let visible = crate::a2ui::transcript_fallback(visible); let content = if self.reasoning.is_some() { visible.trim_start() } else { - visible + &visible }; self.markdown = markdown::Content::parse(content); } @@ -99,12 +104,59 @@ impl From for ChatMessage { reasoning_open: false, content: message.content, markdown: iced::widget::markdown::Content::new(), + a2ui_lines_processed: 0, + a2ui_errors: Vec::new(), + a2ui_replies: Vec::new(), + a2ui_open_urls: Vec::new(), }; message.refresh_markdown(); message } } +fn sync_a2ui_message( + store: &mut crate::a2ui::Store, + database: &mut Option, + session_id: Option, + message: &mut ChatMessage, +) { + let lines = crate::a2ui::extract_lines(&message.content); + for (index, line) in lines.iter().enumerate().skip(message.a2ui_lines_processed) { + let applied = match &line.value { + Ok(value) => store.apply(value.clone(), line.raw.clone(), message.id), + Err(error) => Err(error.clone()), + }; + match applied { + Ok(applied) => { + if let Some(reply) = applied.reply { + message.a2ui_replies.push(reply); + } + if let Some(url) = applied.open_url { + message.a2ui_open_urls.push(url); + } + if let (Some(session_id), Some(database)) = (session_id, database.as_mut()) { + for raw in applied.raws { + if let Err(error) = + database.insert_a2ui_message(session_id, message.id, &raw) + { + message.a2ui_errors.push(format!( + "line {} could not be persisted: {error}", + index + 1 + )); + break; + } + } + } + } + Err(error) => message + .a2ui_errors + .push(format!("line {}: {error}", index + 1)), + } + } + message.a2ui_lines_processed = lines.len(); + message.refresh_markdown(); +} + #[cfg(target_os = "macos")] fn chat_turn(message: &ChatMessage) -> ChatTurn { ChatTurn { @@ -168,6 +220,15 @@ fn has_chat_after_last_compaction(messages: &[ChatMessage]) -> bool { } impl App { + fn chat_system_prompt(&self, model: ModelChoice, prompt: &str) -> String { + let mut prompt = crate::agent::system_prompt(model, prompt); + if self.config.a2ui_enabled { + prompt.push_str("\n\n"); + prompt.push_str(crate::a2ui::SYSTEM_PROMPT); + } + prompt + } + pub(super) fn can_compact_session(&self, session_id: i32) -> bool { !self.generating && self.selected_session == Some(session_id) @@ -238,13 +299,22 @@ impl App { } }; effective.turn.system_prompt = - crate::agent::system_prompt(model, &effective.turn.system_prompt); + self.chat_system_prompt(model, &effective.turn.system_prompt); effective.turn.system_prompt = crate::compaction::summary_system_prompt( &effective.turn.system_prompt, self.compaction_summary(), ); let assistant_reasoning = effective.turn.reasoning_mode != ReasoningMode::Direct; #[cfg(target_os = "macos")] + let model_prompt = if self.config.a2ui_enabled { + format!( + "{prompt}\n\nA2UI client metadata:\n{}", + self.a2ui.client_metadata() + ) + } else { + prompt.clone() + }; + #[cfg(target_os = "macos")] let opening_turn = self.selected_session.is_none(); #[cfg(target_os = "macos")] let mut injected_system = Vec::new(); @@ -282,7 +352,7 @@ impl App { skip_previous_eos: false, reasoning: None, reasoning_complete: true, - content: prompt.clone(), + content: model_prompt, }); #[cfg(target_os = "macos")] @@ -374,6 +444,9 @@ impl App { fn system_prompt_reminders(&self, model: ModelChoice) -> Vec { let mut reminders = vec![crate::agent::system_prompt_reminder(model)]; + if self.config.a2ui_enabled { + reminders.push(crate::a2ui::SYSTEM_PROMPT.to_owned()); + } if !self.config.generation.system_prompt.trim().is_empty() { reminders.push(self.config.generation.system_prompt.clone()); } @@ -475,6 +548,10 @@ impl App { #[cfg(target_os = "macos")] let mut start_queued = false; #[cfg(target_os = "macos")] + let mut a2ui_feedback = None; + #[cfg(target_os = "macos")] + let mut a2ui_changed = false; + #[cfg(target_os = "macos")] loop { match active.events.try_recv() { Ok(GenerationEvent::Loading) => {} @@ -500,6 +577,18 @@ impl App { } transcript_changed = true; } + if !reasoning + && self.config.a2ui_enabled + && let Some(message) = self.conversation.last_mut() + { + sync_a2ui_message( + &mut self.a2ui, + &mut self.database, + self.selected_session, + message, + ); + a2ui_changed = true; + } } Ok(GenerationEvent::Context { used, @@ -530,6 +619,69 @@ impl App { !self.queued_inputs.is_empty() || self.manual_compaction_queued; } Ok(_) => { + let (validation_errors, replies, open_urls, error_surface_id) = self + .conversation + .last_mut() + .map(|message| { + let surface_id = crate::a2ui::extract_lines(&message.content) + .into_iter() + .rev() + .filter_map(|line| line.value.ok()) + .find_map(|value| { + crate::a2ui::message_surface_id(&value) + .map(str::to_owned) + }) + .unwrap_or_else(|| "unknown".to_owned()); + ( + std::mem::take(&mut message.a2ui_errors), + std::mem::take(&mut message.a2ui_replies), + std::mem::take(&mut message.a2ui_open_urls), + surface_id, + ) + }) + .unwrap_or_default(); + for url in open_urls { + if let Err(error) = + std::process::Command::new("open").arg(url).spawn() + { + self.error = + Some(format!("Could not open the A2UI link: {error}")); + } + } + if !validation_errors.is_empty() { + self.generating = false; + self.activity = Some("Correcting A2UI…".into()); + self.tool_cards.clear(); + a2ui_feedback = Some( + serde_json::json!({ + "version": crate::a2ui::VERSION, + "error": { + "code": "VALIDATION_FAILED", + "surfaceId": error_surface_id, + "path": "/", + "message": validation_errors.join("; ") + } + }) + .to_string(), + ); + self.active_generation = None; + break; + } + if !replies.is_empty() { + self.generating = false; + self.activity = Some("Continuing A2UI function call…".into()); + self.tool_cards.clear(); + a2ui_feedback = Some(format!( + "A2UI client response:\n{}", + replies + .iter() + .map(ToString::to_string) + .collect::>() + .join("\n") + )); + self.active_generation = None; + break; + } let model = self.config.model; let content = self .conversation @@ -586,6 +738,10 @@ impl App { } } #[cfg(target_os = "macos")] + if a2ui_changed { + self.sync_a2ui_renderer_state(); + } + #[cfg(target_os = "macos")] if transcript_changed && let Some(message) = self.conversation.last_mut() { message.refresh_markdown(); } @@ -633,6 +789,14 @@ impl App { self.start_next_queued(); } #[cfg(target_os = "macos")] + if let Some(feedback) = a2ui_feedback + && let Err(error) = self.continue_after_tool_result(&feedback) + { + self.generating = false; + self.activity = Some("Failed".into()); + self.error = Some(error); + } + #[cfg(target_os = "macos")] return transcript_changed; #[cfg(not(target_os = "macos"))] false @@ -698,7 +862,7 @@ impl App { &models_path(), )?; effective.turn.system_prompt = - crate::agent::system_prompt(model, &effective.turn.system_prompt); + self.chat_system_prompt(model, &effective.turn.system_prompt); effective.turn.system_prompt = crate::compaction::summary_system_prompt( &effective.turn.system_prompt, self.compaction_summary(), @@ -792,7 +956,7 @@ impl App { &models_path(), )?; effective.turn.system_prompt = - crate::agent::system_prompt(model, &effective.turn.system_prompt); + self.chat_system_prompt(model, &effective.turn.system_prompt); effective.turn.system_prompt = crate::compaction::summary_system_prompt( &effective.turn.system_prompt, self.compaction_summary(), @@ -941,7 +1105,7 @@ impl App { &models_path(), )?; effective.turn.system_prompt = - crate::agent::system_prompt(model, &effective.turn.system_prompt); + self.chat_system_prompt(model, &effective.turn.system_prompt); let rebuild_system_prompt = effective.turn.system_prompt.clone(); effective.turn.system_prompt = crate::compaction::summary_system_prompt( &effective.turn.system_prompt, @@ -1364,6 +1528,10 @@ mod tests { content: "### Core / Setup\n\n| File | Lines |\n|---|---:|\n| `src/app.rs` | **1,750** |\n| `src/engine.rs` | 2,400 |\n\n### Summary\n\nDone." .to_owned(), markdown: iced::widget::markdown::Content::new(), + a2ui_lines_processed: 0, + a2ui_errors: Vec::new(), + a2ui_replies: Vec::new(), + a2ui_open_urls: Vec::new(), }; message.refresh_markdown(); @@ -1394,6 +1562,10 @@ mod tests { reasoning_open: false, content: format!("message {id}"), markdown: iced::widget::markdown::Content::new(), + a2ui_lines_processed: 0, + a2ui_errors: Vec::new(), + a2ui_replies: Vec::new(), + a2ui_open_urls: Vec::new(), }; let history = vec![ message(1, false, None), @@ -1426,6 +1598,10 @@ mod tests { reasoning_open: false, content: format!("message {id}"), markdown: iced::widget::markdown::Content::new(), + a2ui_lines_processed: 0, + a2ui_errors: Vec::new(), + a2ui_replies: Vec::new(), + a2ui_open_urls: Vec::new(), }; let mut history = vec![ message(1, true, false, false, false), diff --git a/src/app/preferences.rs b/src/app/preferences.rs index 052e182..d3b99cf 100644 --- a/src/app/preferences.rs +++ b/src/app/preferences.rs @@ -6,6 +6,7 @@ pub(super) struct PreferenceDraft { pub(super) legacy_mtp_enabled: bool, pub(super) dspark_enabled: bool, pub(super) idle_timeout_minutes: String, + pub(super) a2ui_enabled: bool, pub(super) endpoint_port: String, pub(super) endpoint_enabled: bool, pub(super) endpoint_cors: bool, @@ -55,6 +56,7 @@ impl PreferenceDraft { legacy_mtp_enabled: speculative.legacy_mtp_enabled, dspark_enabled: speculative.dspark_enabled, idle_timeout_minutes: config.idle_timeout_minutes.to_string(), + a2ui_enabled: config.a2ui_enabled, endpoint_port: config.endpoint.port.to_string(), endpoint_enabled: config.endpoint.enabled, endpoint_cors: config.endpoint.cors, @@ -350,6 +352,7 @@ impl App { let config = Config { model: self.preference_draft.model, idle_timeout_minutes, + a2ui_enabled: self.preference_draft.a2ui_enabled, endpoint: EndpointConfig { port: i32::from(endpoint_port), enabled: self.preference_draft.endpoint_enabled, diff --git a/src/app/projects.rs b/src/app/projects.rs index a2922fc..13dbe0f 100644 --- a/src/app/projects.rs +++ b/src/app/projects.rs @@ -1,6 +1,79 @@ use super::*; impl App { + pub(super) fn clear_a2ui(&mut self) { + self.a2ui.clear(); + self.a2ui_tabs.clear(); + self.a2ui_modals.clear(); + self.a2ui_editors.clear(); + self.a2ui_markdown.clear(); + self.a2ui_choice_filters.clear(); + self.a2ui_images.clear(); + self.a2ui_image_requests.clear(); + self.a2ui_image_loading = false; + } + + pub(super) fn load_next_a2ui_image(&mut self) -> Task { + if self.a2ui_image_loading { + return Task::none(); + } + let Some(url) = self + .a2ui + .image_urls() + .find(|url| !self.a2ui_image_requests.contains(url)) + else { + return Task::none(); + }; + self.a2ui_image_requests.insert(url.clone()); + self.a2ui_image_loading = true; + let request_url = url.clone(); + Task::perform( + async move { + let mut response = ureq::get(&request_url) + .call() + .map_err(|error| error.to_string())?; + if !response + .headers() + .get("content-type") + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| value.starts_with("image/")) + { + return Err("the URL did not return an image".to_owned()); + } + response + .body_mut() + .read_to_vec() + .map_err(|error| error.to_string()) + }, + move |result| Message::A2uiImageLoaded(url, result), + ) + } + + pub(super) fn change_a2ui_data( + &mut self, + surface_id: String, + path: String, + value: serde_json::Value, + ) -> Task { + let owner = self + .a2ui + .surface(&surface_id) + .map(|surface| surface.owner_message_id); + match self.a2ui.local_update(&surface_id, &path, value) { + Ok(raw) => { + if let (Some(session_id), Some(message_id), Some(database)) = + (self.selected_session, owner, &mut self.database) + && let Err(error) = database.insert_a2ui_message(session_id, message_id, &raw) + { + self.error = Some(format!("Could not save the A2UI edit: {error}")); + } + } + Err(error) => self.error = Some(error), + } + self.sync_a2ui_renderer_state(); + self.load_next_a2ui_image() + } + pub(super) fn prepare_project(&mut self, path: PathBuf) { let Ok(path) = fs::canonicalize(path) else { self.error = Some("The selected folder is no longer available.".into()); @@ -49,6 +122,7 @@ impl App { Ok(project) => { self.remember_project(project.id); self.selected_session = None; + self.clear_a2ui(); self.system_prompt_seen_at = 0; self.pending_project_path = None; self.project_name_input.clear(); @@ -78,6 +152,7 @@ impl App { self.remember_project(project_id); self.selected_session = None; self.conversation.clear(); + self.clear_a2ui(); self.composer.clear(); self.queued_inputs.clear(); self.system_prompt_seen_at = 0; @@ -90,6 +165,7 @@ impl App { pub(super) fn discard_session(&mut self, project_id: i32) { if self.drafts.remove(&project_id).is_some() && self.draft_selected(project_id) { self.conversation.clear(); + self.clear_a2ui(); self.composer.clear(); self.queued_inputs.clear(); self.system_prompt_seen_at = 0; diff --git a/src/app/view.rs b/src/app/view.rs index 9ebd59e..8006e2a 100644 --- a/src/app/view.rs +++ b/src/app/view.rs @@ -1,3 +1,4 @@ +mod a2ui; mod chat; mod model_manager; mod preferences; @@ -16,8 +17,9 @@ use crate::model::{ use crate::settings::{GIB, REASONING_MODES}; use iced::theme::{Palette, palette}; use iced::widget::{ - Button, Space, Svg, Tooltip, button, checkbox, column, container, markdown, mouse_area, opaque, - pick_list, progress_bar, row, rule, scrollable, stack, svg, text, text_input, tooltip, + Button, Space, Svg, Tooltip, button, checkbox, column, container, image, markdown, mouse_area, + opaque, pick_list, progress_bar, row, rule, scrollable, slider, stack, svg, text, text_input, + tooltip, }; use iced::{Alignment, Background, Border, Color, Element, Length, Padding, Theme, window}; use std::collections::VecDeque; @@ -68,6 +70,7 @@ impl App { || self.pending_project_path.is_some() || self.session_rename.is_some() || self.menu_session().is_some() + || !self.a2ui_modals.is_empty() || { #[cfg(target_os = "macos")] { @@ -132,6 +135,8 @@ impl App { layers.push(self.rename_dialog(title)); } else if let Some(session) = self.menu_session() { layers.push(self.session_menu_panel(session)); + } else if let Some(panel) = self.a2ui_modal_panel() { + layers.push(panel); } #[cfg(not(target_os = "macos"))] if self.preferences_open { @@ -142,6 +147,8 @@ impl App { layers.push(self.rename_dialog(title)); } else if let Some(session) = self.menu_session() { layers.push(self.session_menu_panel(session)); + } else if let Some(panel) = self.a2ui_modal_panel() { + layers.push(panel); } stack(layers) diff --git a/src/app/view/a2ui.rs b/src/app/view/a2ui.rs new file mode 100644 index 0000000..d92b0bf --- /dev/null +++ b/src/app/view/a2ui.rs @@ -0,0 +1,1909 @@ +use super::*; +use crate::a2ui::{Surface, binding_path, bound_value_at, display_value}; +use iced::widget::{Column, Row, column, radio, text_editor}; +use serde_json::Value; +use std::collections::{BTreeSet, HashMap, HashSet}; +use time::{Month, OffsetDateTime}; + +impl App { + pub(crate) fn sync_a2ui_renderer_state(&mut self) { + let mut markdown = HashMap::new(); + let mut editors = HashMap::new(); + let mut filters = HashSet::new(); + for surface in self.a2ui.surfaces() { + collect_renderer_state( + surface, + "root", + &surface.data, + None, + BTreeSet::new(), + &mut markdown, + &mut editors, + &mut filters, + ); + } + self.a2ui_markdown = markdown + .into_iter() + .map(|(key, value)| (key, markdown::Content::parse(&value))) + .collect(); + self.a2ui_editors.retain(|key, _| editors.contains_key(key)); + for (key, value) in editors { + match self.a2ui_editors.entry(key) { + std::collections::hash_map::Entry::Occupied(mut entry) + if entry.get().text() != value => + { + entry.insert(iced::widget::text_editor::Content::with_text(&value)); + } + std::collections::hash_map::Entry::Vacant(entry) => { + entry.insert(iced::widget::text_editor::Content::with_text(&value)); + } + _ => {} + } + } + self.a2ui_choice_filters + .retain(|key, _| filters.contains(key)); + } + + pub(super) fn a2ui_surfaces(&self, message_id: i32) -> Element<'_, Message> { + let mut surfaces = column![].spacing(10); + for surface in self.a2ui.surfaces_for_message(message_id) { + surfaces = surfaces.push(self.a2ui_surface(surface)); + } + surfaces.into() + } + + fn a2ui_surface<'a>(&'a self, surface: &'a Surface) -> Element<'a, Message> { + let agent = surface + .surface_properties + .get("agentDisplayName") + .and_then(Value::as_str) + .unwrap_or("A2UI"); + let content = if surface.components.contains_key("root") { + self.a2ui_component(surface, "root", &surface.data, None, BTreeSet::new()) + } else { + text("Building interactive surface…") + .size(13) + .color(muted_text()) + .into() + }; + container( + column![ + row![ + text(agent).size(10).color(muted_text()), + Space::new().width(Length::Fill), + text(&surface.id).size(10).color(muted_text()), + ], + content, + ] + .spacing(10), + ) + .padding(14) + .width(Length::Fill) + .style(preference_group_style) + .into() + } + + fn a2ui_component<'a>( + &'a self, + surface: &'a Surface, + id: &str, + context: &'a Value, + context_path: Option, + mut ancestors: BTreeSet, + ) -> Element<'a, Message> { + if !ancestors.insert(id.to_owned()) { + return text(format!("Cyclic component reference: {id}")) + .style(iced::widget::text::danger) + .into(); + } + let Some(component) = surface.components.get(id).and_then(Value::as_object) else { + return text(format!("Waiting for component `{id}`…")) + .size(12) + .color(muted_text()) + .into(); + }; + let kind = component + .get("component") + .and_then(Value::as_str) + .unwrap_or("Unknown"); + match kind { + "Text" => { + let value = display_value(&bound_value_at( + component.get("text"), + &surface.data, + context, + )); + let size = match component.get("variant").and_then(Value::as_str) { + Some("caption") => 11, + _ => 14, + }; + let key = ( + surface.id.clone(), + id.to_owned(), + context_path.clone().unwrap_or_default(), + ); + self.a2ui_markdown.get(&key).map_or_else( + || text(value).size(size).into(), + |content| { + markdown::view( + content.items(), + markdown::Settings::with_text_size( + size, + markdown::Style::from_palette(app_theme().palette()), + ), + ) + .map(Message::OpenLink) + }, + ) + } + "Image" => { + let url = display_value(&bound_value_at( + component.get("url"), + &surface.data, + context, + )); + if let Some(handle) = self.a2ui_images.get(&url) { + let description = display_value(&bound_value_at( + component.get("description"), + &surface.data, + context, + )); + let variant = component.get("variant").and_then(Value::as_str); + let size = match variant { + Some("icon") => 24, + Some("avatar") => 40, + Some("smallFeature") => 100, + Some("largeFeature") => 300, + Some("header") => 200, + _ => 200, + }; + let mut rendered = image(handle.clone()) + .width( + if matches!(variant, Some("icon" | "avatar" | "smallFeature")) { + Length::Fixed(size as f32) + } else { + Length::Fill + }, + ) + .height(size) + .content_fit(match component.get("fit").and_then(Value::as_str) { + Some("cover") => iced::ContentFit::Cover, + Some("fill") => iced::ContentFit::Fill, + Some("none") => iced::ContentFit::None, + Some("scaleDown") => iced::ContentFit::ScaleDown, + _ => iced::ContentFit::Contain, + }); + if variant == Some("avatar") { + rendered = rendered.border_radius(20); + } + let mut content = Column::new().push(rendered); + if !description.is_empty() { + content = content.push(text(description).size(11).color(muted_text())); + } + content.spacing(5).into() + } else { + image_link(component, &surface.data, context) + } + } + "Video" | "AudioPlayer" => self.media_player(component, kind, &surface.data, context), + "Icon" => text(icon_glyph(&display_value(&bound_value_at( + component.get("name"), + &surface.data, + context, + )))) + .size(19) + .into(), + "Divider" => { + if component.get("axis").and_then(Value::as_str) == Some("vertical") { + rule::vertical(1).into() + } else { + rule::horizontal(1).into() + } + } + "Row" | "Column" => self.a2ui_children( + surface, + component.get("children"), + context, + context_path, + ancestors, + Some((component, kind == "Row")), + ), + "List" => { + let horizontal = + component.get("direction").and_then(Value::as_str) == Some("horizontal"); + let content = self.a2ui_children( + surface, + component.get("children"), + context, + context_path, + ancestors, + Some((component, horizontal)), + ); + let list = scrollable(content).height(280); + if horizontal { + list.direction(iced::widget::scrollable::Direction::Horizontal( + Default::default(), + )) + .into() + } else { + list.into() + } + } + "Card" => { + let child = component.get("child").and_then(Value::as_str).unwrap_or(""); + container(self.a2ui_component(surface, child, context, context_path, ancestors)) + .padding(14) + .width(Length::Fill) + .style(overview_style) + .into() + } + "Tabs" => { + let tabs = component + .get("tabs") + .and_then(Value::as_array) + .map(Vec::as_slice) + .unwrap_or(&[]); + let selected = self + .a2ui_tabs + .get(&(surface.id.clone(), id.to_owned())) + .copied() + .unwrap_or(0) + .min(tabs.len().saturating_sub(1)); + let mut labels = Row::new().spacing(6); + for (index, tab) in tabs.iter().enumerate() { + let title = + display_value(&bound_value_at(tab.get("title"), &surface.data, context)); + let mut tab_button = action_button(text(title).size(12)); + if index != selected { + tab_button = tab_button.on_press(Message::A2uiSelectTab( + surface.id.clone(), + id.to_owned(), + index, + )); + } + labels = labels.push(tab_button); + } + let child = tabs + .get(selected) + .and_then(|tab| tab.get("child")) + .and_then(Value::as_str) + .unwrap_or(""); + column![ + labels, + self.a2ui_component(surface, child, context, context_path, ancestors,) + ] + .spacing(10) + .into() + } + "Modal" => { + let trigger = component + .get("trigger") + .and_then(Value::as_str) + .unwrap_or(""); + let content = component + .get("content") + .and_then(Value::as_str) + .unwrap_or(""); + let key = (surface.id.clone(), id.to_owned()); + let trigger = mouse_area(self.a2ui_component( + surface, + trigger, + context, + context_path.clone(), + ancestors.clone(), + )) + .on_press(Message::A2uiToggleModal(key.0.clone(), key.1.clone())); + let _ = (content, context_path, ancestors); + trigger.into() + } + "Button" => { + let child = component.get("child").and_then(Value::as_str).unwrap_or(""); + let content = + self.a2ui_component(surface, child, context, context_path.clone(), ancestors); + let button = + if component.get("variant").and_then(Value::as_str) == Some("borderless") { + button(content).style(button::text) + } else { + action_button(content) + }; + if crate::a2ui::first_failed_check(component, &surface.data).is_some() { + button.into() + } else if let Some(url) = component + .get("action") + .and_then(|action| action.get("functionCall")) + .and_then(|call| call.get("args")) + .and_then(|args| args.get("url")) + .map(|url| display_value(&bound_value_at(Some(url), &surface.data, context))) + .filter(|url| { + url::Url::parse(url) + .is_ok_and(|url| matches!(url.scheme(), "http" | "https")) + }) + { + button.on_press(Message::OpenLink(url)).into() + } else { + button + .on_press(Message::A2uiAction( + surface.id.clone(), + id.to_owned(), + context_path, + )) + .into() + } + } + "TextField" => { + let binding = component.get("value"); + let value = display_value(&bound_value_at(binding, &surface.data, context)); + let label = display_value(&bound_value_at( + component.get("label"), + &surface.data, + context, + )); + let placeholder = component + .get("placeholder") + .map(|value| { + display_value(&bound_value_at(Some(value), &surface.data, context)) + }) + .filter(|value| !value.is_empty()) + .unwrap_or_else(|| label.clone()); + let path = local_path(binding_path(binding), context_path.as_deref()); + let variant = component.get("variant").and_then(Value::as_str); + let input: Element<'_, Message> = if variant == Some("longText") { + let key = path + .as_ref() + .map(|path| (surface.id.clone(), id.to_owned(), path.clone())); + if let Some((key, content)) = key + .as_ref() + .and_then(|key| self.a2ui_editors.get(key).map(|content| (key, content))) + { + let mut editor = text_editor(content) + .placeholder(placeholder.clone()) + .height(120) + .padding(9); + if path.is_some() { + let surface_id = surface.id.clone(); + let component_id = id.to_owned(); + let path = key.2.clone(); + editor = editor.on_action(move |action| { + Message::A2uiEditorAction( + surface_id.clone(), + component_id.clone(), + path.clone(), + action, + ) + }); + } + editor.into() + } else { + text_input(&placeholder, &value).padding(9).into() + } + } else { + let mut input = text_input(&placeholder, &value) + .secure(variant == Some("obscured")) + .padding(9); + if let Some(path) = path { + let surface_id = surface.id.clone(); + input = input.on_input(move |value| { + if variant == Some("number") && !valid_number_edit(&value) { + Message::Noop + } else { + Message::A2uiDataChanged( + surface_id.clone(), + path.clone(), + Value::String(value), + ) + } + }); + } + input.into() + }; + let mut field = column![text(label).size(12), input].spacing(5); + if let Some(error) = crate::a2ui::first_failed_check(component, &surface.data) { + field = field.push(text(error).size(11).style(iced::widget::text::danger)); + } + field.into() + } + "DateTimeInput" => date_time_input(component, surface, context, context_path), + "CheckBox" => { + let binding = component.get("value"); + let checked = bound_value_at(binding, &surface.data, context) + .as_bool() + .unwrap_or(false); + let label = display_value(&bound_value_at( + component.get("label"), + &surface.data, + context, + )); + let checkbox = checkbox(checked).label(label); + if let Some(path) = local_path(binding_path(binding), context_path.as_deref()) { + let surface_id = surface.id.clone(); + checkbox + .on_toggle(move |value| { + Message::A2uiDataChanged( + surface_id.clone(), + path.clone(), + Value::Bool(value), + ) + }) + .into() + } else { + checkbox.into() + } + } + "Slider" => { + let binding = component.get("value"); + let current = bound_value_at(binding, &surface.data, context) + .as_f64() + .unwrap_or(0.0) as f32; + let min = component.get("min").and_then(Value::as_f64).unwrap_or(0.0) as f32; + let max = component + .get("max") + .and_then(Value::as_f64) + .unwrap_or(100.0) as f32; + if let Some(path) = local_path(binding_path(binding), context_path.as_deref()) { + let surface_id = surface.id.clone(); + let mut control = slider(min..=max, current.clamp(min, max), move |value| { + Message::A2uiDataChanged( + surface_id.clone(), + path.clone(), + serde_json::Number::from_f64(value as f64) + .map(Value::Number) + .unwrap_or(Value::Null), + ) + }); + if let Some(steps) = component.get("steps").and_then(Value::as_u64) + && steps > 0 + { + control = control.step((max - min) / steps as f32); + } + column![ + text(display_value(&bound_value_at( + component.get("label"), + &surface.data, + context, + ))) + .size(12), + row![control, text(format!("{current}"))] + .spacing(8) + .align_y(Alignment::Center) + ] + .spacing(5) + .into() + } else { + progress_bar(min..=max, current.clamp(min, max)).into() + } + } + "ChoicePicker" => { + let binding = component.get("value"); + let selected = bound_value_at(binding, &surface.data, context); + let multiple = + component.get("variant").and_then(Value::as_str) == Some("multipleSelection"); + let path = local_path(binding_path(binding), context_path.as_deref()); + let filter_key = ( + surface.id.clone(), + id.to_owned(), + context_path.clone().unwrap_or_default(), + ); + let filter = self + .a2ui_choice_filters + .get(&filter_key) + .map_or("", String::as_str); + let normalized_filter = filter.to_lowercase(); + let options = component + .get("options") + .and_then(Value::as_array) + .map(Vec::as_slice) + .unwrap_or(&[]); + let selected_index = options.iter().position(|option| { + option.get("value").is_some_and(|value| { + selected + .as_array() + .is_some_and(|values| values.contains(value)) + }) + }); + let display_style = component + .get("displayStyle") + .and_then(Value::as_str) + .unwrap_or("checkbox"); + let mut choices: Column<'_, Message> = Column::new().spacing(6); + let mut chips: Row<'_, Message> = Row::new().spacing(6); + for (option_index, option) in options.iter().enumerate() { + let label = + display_value(&bound_value_at(option.get("label"), &surface.data, context)); + if !normalized_filter.is_empty() + && !label.to_lowercase().contains(&normalized_filter) + { + continue; + } + let option_value = option.get("value").cloned().unwrap_or(Value::Null); + let is_selected = selected + .as_array() + .is_some_and(|values| values.contains(&option_value)); + let next_value = + choice_selection(&selected, &option_value, multiple, !is_selected); + if display_style == "chips" { + let mut choice = action_button(text(if is_selected { + format!("✓ {label}") + } else { + label + })); + if let Some(path) = path.clone() { + choice = choice.on_press(Message::A2uiDataChanged( + surface.id.clone(), + path, + next_value, + )); + } + chips = chips.push(choice); + } else if multiple { + let mut choice = checkbox(is_selected).label(label); + if let Some(path) = path.clone() { + let surface_id = surface.id.clone(); + let option_value = option_value.clone(); + let selected = selected.clone(); + choice = choice.on_toggle(move |checked| { + Message::A2uiDataChanged( + surface_id.clone(), + path.clone(), + choice_selection(&selected, &option_value, true, checked), + ) + }); + } + choices = choices.push(choice); + } else { + let surface_id = surface.id.clone(); + let message_path = path.clone(); + choices = + choices.push(radio(label, option_index, selected_index, move |_| { + message_path.clone().map_or(Message::Noop, |path| { + Message::A2uiDataChanged( + surface_id.clone(), + path, + next_value.clone(), + ) + }) + })); + } + } + let mut picker = Column::new().spacing(5).push( + text(display_value(&bound_value_at( + component.get("label"), + &surface.data, + context, + ))) + .size(12), + ); + if component.get("filterable") == Some(&Value::Bool(true)) { + let key = filter_key.clone(); + picker = + picker.push(text_input("Filter choices…", filter).padding(7).on_input( + move |value| { + Message::A2uiChoiceFilterChanged( + key.0.clone(), + key.1.clone(), + key.2.clone(), + value, + ) + }, + )); + } + let choices: Element<'_, Message> = if display_style == "chips" { + chips.wrap().into() + } else { + choices.into() + }; + picker.push(choices).into() + } + "Chart" => research_chart(component, &surface.data, context), + "Table" => research_table(component, &surface.data, context), + "Metric" => research_metric(component, &surface.data, context), + "Timeline" => research_timeline(component, &surface.data, context), + "Map" => research_map(component, &surface.data, context), + "MindMap" => research_mind_map(component, &surface.data, context), + "Form" => { + let children = self.a2ui_children( + surface, + component.get("children"), + context, + context_path.clone(), + ancestors, + None, + ); + let label = component + .get("submitLabel") + .and_then(Value::as_str) + .unwrap_or("Submit"); + let button = action_button(label); + let button = if component.get("action").is_some() { + button.on_press(Message::A2uiAction( + surface.id.clone(), + id.to_owned(), + context_path, + )) + } else { + button + }; + column![ + component + .get("title") + .and_then(Value::as_str) + .map(|title| text(title.to_owned()).size(18)) + .unwrap_or_else(|| text("")), + children, + button, + ] + .spacing(10) + .into() + } + _ => text(format!("Unsupported component `{kind}`")) + .style(iced::widget::text::danger) + .into(), + } + } + + fn a2ui_children<'a>( + &'a self, + surface: &'a Surface, + children: Option<&'a Value>, + context: &'a Value, + context_path: Option, + ancestors: BTreeSet, + layout: Option<(&serde_json::Map, bool)>, + ) -> Element<'a, Message> { + let horizontal = layout.is_some_and(|(_, horizontal)| horizontal); + let layout = layout.map(|(component, _)| component); + let mut elements = Vec::new(); + if let Some(children) = children.and_then(Value::as_array) { + for child in children.iter().filter_map(Value::as_str) { + elements.push(( + self.a2ui_component( + surface, + child, + context, + context_path.clone(), + ancestors.clone(), + ), + component_weight(surface, child), + )); + } + } else if let Some(template) = children.and_then(Value::as_object) { + let component_id = template + .get("componentId") + .and_then(Value::as_str) + .unwrap_or(""); + let path = template.get("path").and_then(Value::as_str).unwrap_or("/"); + if let Some(items) = surface.data.pointer(path).and_then(Value::as_array) { + for (index, item) in items.iter().enumerate() { + elements.push(( + crate::a2ui::with_template_index(index, || { + self.a2ui_component( + surface, + component_id, + item, + Some(format!("{}/{index}", path.trim_end_matches('/'))), + ancestors.clone(), + ) + }), + component_weight(surface, component_id), + )); + } + } + } + let align_name = layout + .and_then(|component| component.get("align")) + .and_then(Value::as_str); + let align = match align_name { + Some("center") => Alignment::Center, + Some("end") => Alignment::End, + _ => Alignment::Start, + }; + let justify = layout + .and_then(|component| component.get("justify")) + .and_then(Value::as_str); + let expands_main_axis = elements.iter().any(|(_, weight)| weight.is_some()) + || matches!( + justify, + Some("center" | "end" | "spaceBetween" | "spaceAround" | "spaceEvenly" | "stretch") + ); + if horizontal { + let mut row = Row::new().spacing(10).align_y(align); + if matches!(justify, Some("spaceAround" | "spaceEvenly")) { + row = row.push(Space::new().width(Length::FillPortion(1))); + } + let count = elements.len(); + for (index, (element, weight)) in elements.into_iter().enumerate() { + if index > 0 + && matches!( + justify, + Some("spaceBetween" | "spaceAround" | "spaceEvenly") + ) + { + row = row.push(Space::new().width(Length::FillPortion( + if justify == Some("spaceAround") { 2 } else { 1 }, + ))); + } + let width = weight + .or((justify == Some("stretch")).then_some(1)) + .map_or(Length::Shrink, Length::FillPortion); + let mut child = container(element).width(width); + if align_name == Some("stretch") { + child = child.height(Length::Fill); + } + row = row.push(child); + } + if count > 0 && matches!(justify, Some("spaceAround" | "spaceEvenly")) { + row = row.push(Space::new().width(Length::FillPortion(1))); + } + if align_name == Some("stretch") { + row = row.height(Length::Fill); + } + let row = container(row).width(Length::Fill); + match justify { + Some("center") => row.align_x(Alignment::Center).into(), + Some("end") => row.align_x(Alignment::End).into(), + _ => row.into(), + } + } else { + let mut column = Column::new().spacing(10).align_x(align); + if matches!(justify, Some("spaceAround" | "spaceEvenly")) { + column = column.push(Space::new().height(Length::FillPortion(1))); + } + let count = elements.len(); + for (index, (element, weight)) in elements.into_iter().enumerate() { + if index > 0 + && matches!( + justify, + Some("spaceBetween" | "spaceAround" | "spaceEvenly") + ) + { + column = column.push(Space::new().height(Length::FillPortion( + if justify == Some("spaceAround") { 2 } else { 1 }, + ))); + } + let height = weight + .or((justify == Some("stretch")).then_some(1)) + .map_or(Length::Shrink, Length::FillPortion); + let mut child = container(element).height(height); + if align_name == Some("stretch") { + child = child.width(Length::Fill); + } + column = column.push(child); + } + if count > 0 && matches!(justify, Some("spaceAround" | "spaceEvenly")) { + column = column.push(Space::new().height(Length::FillPortion(1))); + } + if align_name == Some("stretch") { + column = column.width(Length::Fill); + } + let mut column = container(column); + if expands_main_axis { + column = column.height(Length::Fill); + } + match justify { + Some("center") => column.align_y(Alignment::Center).into(), + Some("end") => column.align_y(Alignment::End).into(), + _ => column.into(), + } + } + } + + fn media_player<'a>( + &'a self, + component: &'a serde_json::Map, + kind: &str, + data: &Value, + context: &Value, + ) -> Element<'a, Message> { + let url = display_value(&bound_value_at(component.get("url"), data, context)); + let description = + display_value(&bound_value_at(component.get("description"), data, context)); + let title = if description.is_empty() { + url::Url::parse(&url) + .ok() + .and_then(|url| { + url.path_segments() + .and_then(Iterator::last) + .filter(|name| !name.is_empty()) + .map(str::to_owned) + }) + .unwrap_or_else(|| kind.to_owned()) + } else { + description.clone() + }; + let video = kind == "Video"; + let mut content = Column::new().spacing(10); + if video { + let poster = display_value(&bound_value_at(component.get("posterUrl"), data, context)); + if let Some(handle) = self.a2ui_images.get(&poster) { + content = content.push( + image(handle.clone()) + .width(Length::Fill) + .height(240) + .content_fit(iced::ContentFit::Cover), + ); + } else if !poster.is_empty() { + content = content.push( + container(text("Loading video poster…").color(muted_text())) + .center(Length::Fill) + .height(180), + ); + } + } + let label = if video { + "▶ Play video" + } else { + "▶ Play audio" + }; + let mut play = action_button(text(label).size(12)); + if url::Url::parse(&url).is_ok_and(|url| matches!(url.scheme(), "http" | "https")) { + play = play.on_press(Message::A2uiPlayMedia(url.clone(), title.clone(), video)); + } + content = content.push( + row![ + column![text(title).size(13), text(url).size(10).color(muted_text())] + .spacing(3) + .width(Length::Fill), + play, + ] + .spacing(10) + .align_y(Alignment::Center), + ); + container(content) + .padding(12) + .width(Length::Fill) + .style(overview_style) + .into() + } + + pub(super) fn a2ui_modal_panel(&self) -> Option> { + let (surface_id, component_id) = self.a2ui_modals.iter().next()?; + let surface = self.a2ui.surface(surface_id)?; + let component = surface.components.get(component_id)?.as_object()?; + let content_id = component.get("content")?.as_str()?; + let content = + self.a2ui_component(surface, content_id, &surface.data, None, BTreeSet::new()); + let dialog = container( + column![ + row![ + text("Dialog").size(12), + Space::new().width(Length::Fill), + action_button("Close").on_press(Message::A2uiToggleModal( + surface_id.clone(), + component_id.clone(), + )), + ], + content, + ] + .spacing(12), + ) + .padding(22) + .width(560) + .style(overview_style); + Some(opaque( + container(dialog) + .center_x(Length::Fill) + .center_y(Length::Fill) + .style(|_| { + container::Style::default().background(Color::from_rgba8(0, 0, 0, 0.68)) + }), + )) + } +} + +#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] +struct DateTimeParts { + year: i32, + month: u8, + day: u8, + hour: u8, + minute: u8, + second: u8, +} + +fn component_weight(surface: &Surface, id: &str) -> Option { + let weight = surface.components.get(id)?.get("weight")?.as_f64()?; + (weight > 0.0).then(|| (weight * 1_000.0).round().clamp(1.0, u16::MAX as f64) as u16) +} + +#[derive(Clone, Copy)] +enum DatePart { + Year, + Month, + Day, + Hour, + Minute, +} + +#[derive(Clone)] +struct DateControl { + surface_id: String, + path: String, + value: DateTimeParts, + enable_date: bool, + enable_time: bool, + min: Option, + max: Option, +} + +impl DateControl { + fn message(&self, part: DatePart, value: String) -> Message { + let mut parts = self.value; + match part { + DatePart::Year => { + let Ok(value) = value.parse() else { + return Message::Noop; + }; + parts.year = value; + } + DatePart::Month => parts.month = value.parse().unwrap_or(parts.month), + DatePart::Day => parts.day = value.parse().unwrap_or(parts.day), + DatePart::Hour => parts.hour = value.parse().unwrap_or(parts.hour), + DatePart::Minute => parts.minute = value.parse().unwrap_or(parts.minute), + } + parts.day = parts.day.min(days_in_month(parts.year, parts.month)); + if let Some(min) = self.min { + parts = parts.max(min); + } + if let Some(max) = self.max { + parts = parts.min(max); + } + Message::A2uiDataChanged( + self.surface_id.clone(), + self.path.clone(), + Value::String(format_date_time(parts, self.enable_date, self.enable_time)), + ) + } +} + +fn date_time_input<'a>( + component: &serde_json::Map, + surface: &Surface, + context: &Value, + context_path: Option, +) -> Element<'a, Message> { + let label = display_value(&bound_value_at( + component.get("label"), + &surface.data, + context, + )); + let binding = component.get("value"); + let enable_date = component + .get("enableDate") + .and_then(Value::as_bool) + .unwrap_or(false); + let enable_time = component + .get("enableTime") + .and_then(Value::as_bool) + .unwrap_or(false); + let now = OffsetDateTime::now_utc(); + let fallback = DateTimeParts { + year: now.year(), + month: now.month() as u8, + day: now.day(), + hour: now.hour(), + minute: now.minute(), + second: now.second(), + }; + let raw = display_value(&bound_value_at(binding, &surface.data, context)); + let mut value = parse_date_time(&raw, enable_date, enable_time, fallback).unwrap_or(fallback); + let min = component + .get("min") + .map(|value| display_value(&bound_value_at(Some(value), &surface.data, context))) + .and_then(|value| parse_date_time(&value, enable_date, enable_time, fallback)); + let max = component + .get("max") + .map(|value| display_value(&bound_value_at(Some(value), &surface.data, context))) + .and_then(|value| parse_date_time(&value, enable_date, enable_time, fallback)); + if let Some(min) = min { + value = value.max(min); + } + if let Some(max) = max { + value = value.min(max); + } + let Some(path) = local_path(binding_path(binding), context_path.as_deref()) else { + return column![text(label).size(12), text(raw)].spacing(5).into(); + }; + if !enable_date && !enable_time { + return column![text(label).size(12), text(raw)].spacing(5).into(); + } + let control = DateControl { + surface_id: surface.id.clone(), + path, + value, + enable_date, + enable_time, + min, + max, + }; + let mut inputs = Row::new().spacing(6).align_y(Alignment::Center); + if enable_date { + let first_year = min + .map_or(value.year.saturating_sub(100), |parts| parts.year) + .clamp(0, 9999); + let last_year = max + .map_or(value.year.saturating_add(100), |parts| parts.year) + .clamp(0, 9999); + let years = (first_year.min(last_year)..=first_year.max(last_year)) + .map(|year| format!("{year:04}")) + .collect::>(); + let update = control.clone(); + inputs = inputs + .push(pick_list( + years, + Some(format!("{:04}", value.year)), + move |value| update.message(DatePart::Year, value), + )) + .push(text("–").color(muted_text())); + let update = control.clone(); + inputs = inputs + .push(pick_list( + (1..=12) + .map(|value| format!("{value:02}")) + .collect::>(), + Some(format!("{:02}", value.month)), + move |value| update.message(DatePart::Month, value), + )) + .push(text("–").color(muted_text())); + let update = control.clone(); + inputs = inputs.push(pick_list( + (1..=days_in_month(value.year, value.month)) + .map(|value| format!("{value:02}")) + .collect::>(), + Some(format!("{:02}", value.day)), + move |value| update.message(DatePart::Day, value), + )); + } + if enable_date && enable_time { + inputs = inputs.push(text("at").size(12).color(muted_text())); + } + if enable_time { + let update = control.clone(); + inputs = inputs + .push(pick_list( + (0..24) + .map(|value| format!("{value:02}")) + .collect::>(), + Some(format!("{:02}", value.hour)), + move |value| update.message(DatePart::Hour, value), + )) + .push(text(":").color(muted_text())); + inputs = inputs.push(pick_list( + (0..60) + .map(|value| format!("{value:02}")) + .collect::>(), + Some(format!("{:02}", value.minute)), + move |value| control.message(DatePart::Minute, value), + )); + } + let mut field = column![text(label).size(12), inputs].spacing(5); + if let Some(error) = crate::a2ui::first_failed_check(component, &surface.data) { + field = field.push(text(error).size(11).style(iced::widget::text::danger)); + } + field.into() +} + +fn parse_date_time( + value: &str, + enable_date: bool, + enable_time: bool, + mut parts: DateTimeParts, +) -> Option { + let (date, time) = match (enable_date, enable_time) { + (true, true) => value.split_once('T')?, + (true, false) => (value, ""), + (false, true) => ("", value), + (false, false) => return Some(parts), + }; + if enable_date { + let mut values = date.split('-'); + parts.year = values.next()?.parse().ok()?; + parts.month = values.next()?.parse().ok()?; + parts.day = values.next()?.parse().ok()?; + if values.next().is_some() + || !(0..=9999).contains(&parts.year) + || !(1..=12).contains(&parts.month) + || parts.day == 0 + || parts.day > days_in_month(parts.year, parts.month) + { + return None; + } + } + if enable_time { + let time = time + .trim_end_matches('Z') + .split(['+', '-']) + .next() + .unwrap_or(time) + .split('.') + .next() + .unwrap_or(time); + let mut values = time.split(':'); + parts.hour = values.next()?.parse().ok()?; + parts.minute = values.next()?.parse().ok()?; + parts.second = values.next().unwrap_or("0").parse().ok()?; + if values.next().is_some() || parts.hour > 23 || parts.minute > 59 || parts.second > 59 { + return None; + } + } + Some(parts) +} + +fn format_date_time(parts: DateTimeParts, enable_date: bool, enable_time: bool) -> String { + match (enable_date, enable_time) { + (true, true) => format!( + "{:04}-{:02}-{:02}T{:02}:{:02}:{:02}", + parts.year, parts.month, parts.day, parts.hour, parts.minute, parts.second + ), + (true, false) => format!("{:04}-{:02}-{:02}", parts.year, parts.month, parts.day), + (false, true) => format!("{:02}:{:02}:{:02}", parts.hour, parts.minute, parts.second), + (false, false) => String::new(), + } +} + +fn days_in_month(year: i32, month: u8) -> u8 { + Month::try_from(month) + .map(|month| month.length(year)) + .unwrap_or(31) +} + +fn valid_number_edit(value: &str) -> bool { + matches!(value, "" | "+" | "-" | "." | "+." | "-.") || value.parse::().is_ok() +} + +fn choice_selection(selected: &Value, option: &Value, multiple: bool, checked: bool) -> Value { + let mut values = selected.as_array().cloned().unwrap_or_default(); + if multiple { + values.retain(|value| value != option); + if checked { + values.push(option.clone()); + } + } else { + values = vec![option.clone()]; + } + Value::Array(values) +} + +#[allow(clippy::too_many_arguments)] +fn collect_renderer_state( + surface: &Surface, + id: &str, + context: &Value, + context_path: Option, + mut ancestors: BTreeSet, + markdown: &mut HashMap<(String, String, String), String>, + editors: &mut HashMap<(String, String, String), String>, + filters: &mut HashSet<(String, String, String)>, +) { + if !ancestors.insert(id.to_owned()) { + return; + } + let Some(component) = surface.components.get(id).and_then(Value::as_object) else { + return; + }; + let context_key = context_path.clone().unwrap_or_default(); + match component.get("component").and_then(Value::as_str) { + Some("Text") => { + markdown.insert( + (surface.id.clone(), id.to_owned(), context_key), + display_value(&bound_value_at( + component.get("text"), + &surface.data, + context, + )), + ); + } + Some("TextField") + if component.get("variant").and_then(Value::as_str) == Some("longText") => + { + if let Some(path) = local_path( + binding_path(component.get("value")), + context_path.as_deref(), + ) { + editors.insert( + (surface.id.clone(), id.to_owned(), path), + display_value(&bound_value_at( + component.get("value"), + &surface.data, + context, + )), + ); + } + } + Some("ChoicePicker") if component.get("filterable") == Some(&Value::Bool(true)) => { + filters.insert((surface.id.clone(), id.to_owned(), context_key)); + } + _ => {} + } + match component.get("component").and_then(Value::as_str) { + Some("Row" | "Column" | "List" | "Form") => collect_children_state( + surface, + component.get("children"), + context, + context_path, + ancestors, + markdown, + editors, + filters, + ), + Some("Card" | "Button") => { + if let Some(child) = component.get("child").and_then(Value::as_str) { + collect_renderer_state( + surface, + child, + context, + context_path, + ancestors, + markdown, + editors, + filters, + ); + } + } + Some("Tabs") => { + for child in component + .get("tabs") + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter_map(|tab| tab.get("child").and_then(Value::as_str)) + { + collect_renderer_state( + surface, + child, + context, + context_path.clone(), + ancestors.clone(), + markdown, + editors, + filters, + ); + } + } + Some("Modal") => { + for child in ["trigger", "content"] + .into_iter() + .filter_map(|field| component.get(field).and_then(Value::as_str)) + { + collect_renderer_state( + surface, + child, + context, + context_path.clone(), + ancestors.clone(), + markdown, + editors, + filters, + ); + } + } + _ => {} + } +} + +#[allow(clippy::too_many_arguments)] +fn collect_children_state( + surface: &Surface, + children: Option<&Value>, + context: &Value, + context_path: Option, + ancestors: BTreeSet, + markdown: &mut HashMap<(String, String, String), String>, + editors: &mut HashMap<(String, String, String), String>, + filters: &mut HashSet<(String, String, String)>, +) { + if let Some(children) = children.and_then(Value::as_array) { + for child in children.iter().filter_map(Value::as_str) { + collect_renderer_state( + surface, + child, + context, + context_path.clone(), + ancestors.clone(), + markdown, + editors, + filters, + ); + } + } else if let Some(template) = children.and_then(Value::as_object) { + let component_id = template + .get("componentId") + .and_then(Value::as_str) + .unwrap_or(""); + let path = template.get("path").and_then(Value::as_str).unwrap_or("/"); + for (index, item) in surface + .data + .pointer(path) + .and_then(Value::as_array) + .into_iter() + .flatten() + .enumerate() + { + collect_renderer_state( + surface, + component_id, + item, + Some(format!("{}/{index}", path.trim_end_matches('/'))), + ancestors.clone(), + markdown, + editors, + filters, + ); + } + } +} + +fn title<'a>(component: &serde_json::Map) -> Option> { + component + .get("title") + .and_then(Value::as_str) + .map(|title| text(title.to_owned()).size(17)) +} + +fn image_link<'a>( + component: &serde_json::Map, + data: &Value, + context: &Value, +) -> Element<'a, Message> { + let url = display_value(&bound_value_at(component.get("url"), data, context)); + let button = action_button(text("Open image").size(12)); + if url::Url::parse(&url).is_ok_and(|url| matches!(url.scheme(), "http" | "https")) { + button.on_press(Message::OpenLink(url)).into() + } else { + column![button, text(url).size(11).color(muted_text())] + .spacing(5) + .into() + } +} + +fn research_chart<'a>( + component: &serde_json::Map, + data: &Value, + context: &Value, +) -> Element<'a, Message> { + let series = bound_value_at(component.get("series"), data, context); + let series = series.as_array().map(Vec::as_slice).unwrap_or(&[]); + let chart_type = component + .get("chartType") + .and_then(Value::as_str) + .unwrap_or("bar"); + if matches!(chart_type, "pie" | "donut") { + return research_pie_chart(component, series, chart_type == "donut"); + } + let max = series + .iter() + .filter_map(|point| point.get("value").and_then(Value::as_f64)) + .fold(0.0_f64, f64::max) + .max(1.0) as f32; + let mut rows = Column::new().spacing(7); + if let Some(title) = title(component) { + rows = rows.push(title); + } + rows = rows.push(text(chart_type.to_uppercase()).size(10).color(muted_text())); + for point in series { + let label = point.get("label").map(display_value).unwrap_or_default(); + let value = point.get("value").and_then(Value::as_f64).unwrap_or(0.0) as f32; + rows = rows.push( + row![ + text(label).size(11).width(110), + progress_bar(0.0..=max, value.clamp(0.0, max)), + text(format!("{value}")).size(11).width(55), + ] + .spacing(8) + .align_y(Alignment::Center), + ); + if let Some(segments) = point.get("segments").and_then(Value::as_array) { + rows = rows.push( + text( + segments + .iter() + .map(|segment| { + format!( + "{} {}", + segment.get("label").map(display_value).unwrap_or_default(), + segment.get("value").map(display_value).unwrap_or_default() + ) + }) + .collect::>() + .join(" · "), + ) + .size(10) + .color(muted_text()), + ); + } + } + rows.into() +} + +fn research_pie_chart<'a>( + component: &serde_json::Map, + series: &[Value], + donut: bool, +) -> Element<'a, Message> { + let slices = series + .iter() + .filter_map(|point| { + let value = point.get("value")?.as_f64()?; + (value > 0.0).then(|| { + ( + point.get("label").map(display_value).unwrap_or_default(), + value, + point.get("value").map(display_value).unwrap_or_default(), + ) + }) + }) + .collect::>(); + let values = slices + .iter() + .map(|(_, value, _)| *value) + .collect::>(); + let mut chart = Column::new().spacing(8); + if let Some(title) = title(component) { + chart = chart.push(title); + } + chart = chart.push( + text(if donut { "DONUT" } else { "PIE" }) + .size(10) + .color(muted_text()), + ); + if slices.is_empty() { + return chart + .push(text("No positive values").size(11).color(muted_text())) + .into(); + } + chart = chart.push( + container( + svg(svg::Handle::from_memory( + pie_chart_svg(&values, donut).into_bytes(), + )) + .width(180) + .height(180), + ) + .width(Length::Fill) + .align_x(Alignment::Center), + ); + let total = values.iter().sum::(); + let mut legend: Row<'_, Message> = Row::new().spacing(12); + for (index, (label, _, value)) in slices.into_iter().enumerate() { + let color = chart_color(index); + legend = legend.push( + row![ + container(Space::new().width(10).height(10)) + .style(move |_| container::Style::default().background(color)), + text(label).size(11), + text(format!("{value} · {:.0}%", values[index] / total * 100.0)) + .size(10) + .color(muted_text()), + ] + .spacing(5) + .align_y(Alignment::Center), + ); + } + chart.push(legend.wrap()).into() +} + +fn pie_chart_svg(values: &[f64], donut: bool) -> String { + let total = values + .iter() + .copied() + .filter(|value| *value > 0.0) + .sum::(); + if total <= 0.0 { + return String::new(); + } + let mut current = 0.0; + let mut slices = String::new(); + for (index, value) in values.iter().copied().enumerate() { + if value <= 0.0 { + continue; + } + let fraction = value / total; + let color = hex(chart_color(index)); + if donut { + let circumference = 2.0 * std::f64::consts::PI * 46.0; + slices.push_str(&format!( + r#""#, + fraction * circumference, + (1.0 - fraction) * circumference, + -current * circumference, + )); + } else if fraction >= 0.999_999 { + slices.push_str(&format!( + r#""# + )); + } else { + let start = current * std::f64::consts::TAU - std::f64::consts::FRAC_PI_2; + let end = (current + fraction) * std::f64::consts::TAU - std::f64::consts::FRAC_PI_2; + let (x1, y1) = (70.0 + 56.0 * start.cos(), 70.0 + 56.0 * start.sin()); + let (x2, y2) = (70.0 + 56.0 * end.cos(), 70.0 + 56.0 * end.sin()); + let large = u8::from(fraction > 0.5); + slices.push_str(&format!( + r##""## + )); + } + current += fraction; + } + let center = donut.then(|| { + format!( + r#"{}"#, + hex(app_theme().palette().text), + format_chart_value(total), + ) + }); + format!( + r#"{slices}{}"#, + center.unwrap_or_default() + ) +} + +fn chart_color(index: usize) -> Color { + [ + Color::from_rgb8(117, 190, 255), + Color::from_rgb8(137, 209, 133), + Color::from_rgb8(209, 134, 22), + Color::from_rgb8(241, 76, 76), + Color::from_rgb8(177, 128, 215), + Color::from_rgb8(226, 196, 64), + ][index % 6] +} + +fn format_chart_value(value: f64) -> String { + if value.fract() == 0.0 { + format!("{value:.0}") + } else { + format!("{value:.2}") + } +} + +fn research_table<'a>( + component: &serde_json::Map, + data: &Value, + context: &Value, +) -> Element<'a, Message> { + let columns = bound_value_at(component.get("columns"), data, context); + let rows = bound_value_at(component.get("rows"), data, context); + let mut table = Column::new().spacing(0); + if let Some(title) = title(component) { + table = table.push(container(title).padding(8)); + } + if let Some(columns) = columns.as_array() { + table = table + .push(table_row(columns, true)) + .push(rule::horizontal(1)); + } + if let Some(rows) = rows.as_array() { + for (index, row) in rows.iter().filter_map(Value::as_array).enumerate() { + if index > 0 { + table = table.push(rule::horizontal(1)); + } + table = table.push(table_row(row, false)); + } + } + container(table) + .width(Length::Fill) + .style(overview_style) + .into() +} + +fn table_row<'a>(values: &[Value], header: bool) -> Element<'a, Message> { + values + .iter() + .fold(Row::new().spacing(8), |row, value| { + row.push( + text(display_value(value)) + .size(if header { 12 } else { 11 }) + .width(Length::Fill), + ) + }) + .padding(8) + .into() +} + +fn research_metric<'a>( + component: &serde_json::Map, + data: &Value, + context: &Value, +) -> Element<'a, Message> { + column![ + text(display_value(&bound_value_at( + component.get("label"), + data, + context + ))) + .size(11) + .color(muted_text()), + text(display_value(&bound_value_at( + component.get("value"), + data, + context + ))) + .size(28), + text(display_value(&bound_value_at( + component.get("detail"), + data, + context + ))) + .size(11) + .color(muted_text()), + ] + .spacing(4) + .into() +} + +fn research_timeline<'a>( + component: &serde_json::Map, + data: &Value, + context: &Value, +) -> Element<'a, Message> { + let events = bound_value_at(component.get("events"), data, context); + let mut timeline = Column::new().spacing(8); + if let Some(title) = title(component) { + timeline = timeline.push(title); + } + for event in events.as_array().map(Vec::as_slice).unwrap_or(&[]) { + timeline = timeline.push( + row![ + text("●").size(12), + column![ + text(event.get("title").map(display_value).unwrap_or_default()).size(13), + text(format!( + "{}{}", + event.get("time").map(display_value).unwrap_or_default(), + event + .get("description") + .map(|value| format!(" · {}", display_value(value))) + .unwrap_or_default() + )) + .size(11) + .color(muted_text()), + ] + .spacing(2), + ] + .spacing(8), + ); + } + timeline.into() +} + +fn research_map<'a>( + component: &serde_json::Map, + data: &Value, + context: &Value, +) -> Element<'a, Message> { + let locations = bound_value_at(component.get("locations"), data, context); + let mut map = Column::new().spacing(7); + if let Some(title) = title(component) { + map = map.push(title); + } + for location in locations.as_array().map(Vec::as_slice).unwrap_or(&[]) { + let label = location.get("label").map(display_value).unwrap_or_default(); + let latitude = location + .get("latitude") + .map(display_value) + .unwrap_or_default(); + let longitude = location + .get("longitude") + .map(display_value) + .unwrap_or_default(); + map = map.push( + row![ + text("⌖").size(18), + column![ + text(label).size(13), + text(format!("{latitude}, {longitude}")) + .size(11) + .color(muted_text()), + ] + ] + .spacing(8), + ); + } + container(map) + .padding(10) + .width(Length::Fill) + .style(overview_style) + .into() +} + +fn research_mind_map<'a>( + component: &serde_json::Map, + data: &Value, + context: &Value, +) -> Element<'a, Message> { + let nodes = bound_value_at(component.get("nodes"), data, context); + let nodes = nodes.as_array().map(Vec::as_slice).unwrap_or(&[]); + let mut map = Column::new().spacing(6); + if let Some(title) = title(component) { + map = map.push(title); + } + if let Some(root) = nodes.first() { + map = map.push(mind_node(nodes, root, 0, BTreeSet::new())); + } + map.into() +} + +fn mind_node<'a>( + nodes: &[Value], + node: &Value, + depth: usize, + mut seen: BTreeSet, +) -> Element<'a, Message> { + let id = node.get("id").map(display_value).unwrap_or_default(); + if !seen.insert(id) { + return text("Cycle").size(11).color(muted_text()).into(); + } + let mut branch = Column::new().spacing(4).push( + row![ + Space::new().width((depth * 18) as f32), + text(if depth == 0 { "◆" } else { "↳" }).size(12), + text(node.get("label").map(display_value).unwrap_or_default()).size(13), + ] + .spacing(6), + ); + for child in node + .get("children") + .and_then(Value::as_array) + .into_iter() + .flatten() + { + if let Some(child) = nodes.iter().find(|node| node.get("id") == Some(child)) { + branch = branch.push(mind_node(nodes, child, depth + 1, seen.clone())); + } + } + branch.into() +} + +fn local_path(path: Option<&str>, context_path: Option<&str>) -> Option { + let path = path?; + if path.starts_with('/') { + Some(path.to_owned()) + } else { + Some(format!( + "{}/{}", + context_path.unwrap_or("").trim_end_matches('/'), + path + )) + } +} + +fn icon_glyph(name: &str) -> &'static str { + match name { + "accountCircle" | "person" => "●", + "add" => "+", + "arrowBack" => "←", + "arrowForward" => "→", + "attachFile" => "⌕", + "calendarToday" | "event" => "▣", + "call" | "phone" => "☎", + "camera" => "◉", + "check" => "✓", + "close" => "×", + "delete" => "⌫", + "download" => "⇩", + "edit" => "✎", + "error" => "⊗", + "fastForward" => "≫", + "favorite" => "♥", + "favoriteOff" => "♡", + "folder" => "▰", + "help" => "?", + "home" => "⌂", + "warning" => "⚠", + "info" => "ⓘ", + "locationOn" => "⌖", + "lock" => "▣", + "lockOpen" => "□", + "search" => "⌕", + "mail" => "✉", + "menu" => "☰", + "moreVert" => "⋮", + "moreHoriz" => "…", + "notifications" => "◈", + "notificationsOff" => "◇", + "pause" => "Ⅱ", + "payment" => "¤", + "photo" => "▧", + "play" => "▶", + "print" => "▤", + "refresh" => "↻", + "rewind" => "≪", + "send" => "➤", + "settings" => "⚙", + "share" => "↗", + "shoppingCart" => "⌑", + "skipNext" => "▸|", + "skipPrevious" => "|◂", + "star" => "★", + "starHalf" => "☆", + "starOff" => "☆", + "stop" => "■", + "upload" => "⇧", + "visibility" => "◉", + "visibilityOff" => "○", + "volumeDown" => "◖", + "volumeMute" => "◁", + "volumeOff" => "×", + "volumeUp" => "◀", + _ => "•", + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn parts() -> DateTimeParts { + DateTimeParts { + year: 2024, + month: 1, + day: 31, + hour: 12, + minute: 30, + second: 45, + } + } + + #[test] + fn renderer_input_helpers_cover_dates_numbers_choices_and_weights() { + let fallback = parts(); + let leap = parse_date_time("2024-02-29T23:59:58Z", true, true, fallback).unwrap(); + assert_eq!(format_date_time(leap, true, true), "2024-02-29T23:59:58"); + assert!(parse_date_time("2023-02-29", true, false, fallback).is_none()); + assert_eq!( + format_date_time( + parse_date_time("08:09+02:00", false, true, fallback).unwrap(), + false, + true + ), + "08:09:00" + ); + assert!(valid_number_edit("-.")); + assert!(valid_number_edit("1.25e-3")); + assert!(!valid_number_edit("one")); + + assert_eq!( + choice_selection(&json!(["a"]), &json!("b"), true, true), + json!(["a", "b"]) + ); + assert_eq!( + choice_selection(&json!(["a", "b"]), &json!("a"), true, false), + json!(["b"]) + ); + assert_eq!( + choice_selection(&json!(["a"]), &json!("b"), false, true), + json!(["b"]) + ); + + let surface = Surface { + id: "surface".into(), + catalog_id: crate::a2ui::CATALOG_ID.into(), + surface_properties: json!({}), + send_data_model: true, + components: [("child".into(), json!({"weight": 1.5}))].into(), + data: json!({}), + owner_message_id: 1, + }; + assert_eq!(component_weight(&surface, "child"), Some(1_500)); + + let control = DateControl { + surface_id: "surface".into(), + path: "/when".into(), + value: parts(), + enable_date: true, + enable_time: false, + min: None, + max: None, + }; + let Message::A2uiDataChanged(_, _, Value::String(value)) = + control.message(DatePart::Month, "2".into()) + else { + panic!("date control did not update its binding"); + }; + assert_eq!(value, "2024-02-29"); + + let pie = pie_chart_svg(&[60.0, 25.0, 15.0], false); + assert_eq!(pie.matches("41")); + } +} diff --git a/src/app/view/chat.rs b/src/app/view/chat.rs index 34ed772..ea63428 100644 --- a/src/app/view/chat.rs +++ b/src/app/view/chat.rs @@ -179,6 +179,9 @@ impl App { if !cards.is_empty() { body = body.push(tool_cards(cards)); } + if self.a2ui.surfaces_for_message(message.id).next().is_some() { + body = body.push(self.a2ui_surfaces(message.id)); + } } let user = message.user; messages = messages.push( diff --git a/src/app/view/preferences.rs b/src/app/view/preferences.rs index fc69217..9a233ef 100644 --- a/src/app/view/preferences.rs +++ b/src/app/view/preferences.rs @@ -119,6 +119,12 @@ impl App { .spacing(10) .align_y(Alignment::Center), text("Enter a whole number from 1 to 1440.").size(12), + hint( + checkbox(self.preference_draft.a2ui_enabled) + .label("Enable interactive A2UI chat surfaces") + .on_toggle(Message::PreferenceA2uiChanged), + "Lets the local model build validated native charts, tables, forms and other interactive chat surfaces. Turning it off removes the A2UI catalog from the system prompt.", + ), ] .spacing(10), ); diff --git a/src/config.rs b/src/config.rs index 27d23f4..3888f21 100644 --- a/src/config.rs +++ b/src/config.rs @@ -14,6 +14,7 @@ use crate::settings::{GenerationPreferences, RuntimePreferences}; pub struct Config { pub model: ModelChoice, pub idle_timeout_minutes: i32, + pub a2ui_enabled: bool, pub endpoint: EndpointConfig, pub generation: GenerationPreferences, pub runtime: RuntimePreferences, @@ -25,6 +26,7 @@ impl Default for Config { Self { model: ModelChoice::default(), idle_timeout_minutes: 10, + a2ui_enabled: true, endpoint: EndpointConfig::default(), generation: GenerationPreferences::default(), runtime: RuntimePreferences::default(), @@ -153,6 +155,7 @@ mod tests { let path = directory.join("config.yaml"); let config = Config { model: ModelChoice::Glm52, + a2ui_enabled: false, generation: GenerationPreferences { context_tokens: 65_536, reasoning_mode: ReasoningMode::Direct, @@ -173,7 +176,7 @@ mod tests { let text = fs::read_to_string(&path).unwrap(); assert_eq!( text, - "model: glm-5.2\n\ + "model: glm-5.2\na2ui_enabled: false\n\ generation:\n context_tokens: 65536\n reasoning_mode: none\n\ runtime:\n ssd:\n enabled: true\n cache: 64GB\n" ); diff --git a/src/database.rs b/src/database.rs index 806aace..7c4427b 100644 --- a/src/database.rs +++ b/src/database.rs @@ -4,7 +4,7 @@ use std::collections::HashMap; use std::fs; use std::path::Path; -use crate::schema::{messages, projects, sessions}; +use crate::schema::{a2ui_messages, messages, projects, sessions}; pub const MIGRATIONS: EmbeddedMigrations = embed_migrations!("migrations"); @@ -137,6 +137,24 @@ struct NewMessage<'a> { compaction_tail_start: Option, } +#[derive(Clone, Debug, Identifiable, Queryable, Selectable)] +#[diesel(table_name = a2ui_messages)] +#[diesel(check_for_backend(diesel::sqlite::Sqlite))] +pub struct StoredA2uiMessage { + pub id: i32, + pub session_id: i32, + pub message_id: i32, + pub json: String, +} + +#[derive(Insertable)] +#[diesel(table_name = a2ui_messages)] +struct NewA2uiMessage<'a> { + session_id: i32, + message_id: i32, + json: &'a str, +} + #[derive(Debug)] pub struct ProjectWithSessions { pub project: Project, @@ -219,6 +237,16 @@ impl Database { pub fn delete_project(&mut self, project_id: i32) -> Result<(), String> { self.connection .transaction(|connection| { + diesel::delete( + a2ui_messages::table.filter( + a2ui_messages::session_id.eq_any( + sessions::table + .filter(sessions::project_id.eq(project_id)) + .select(sessions::id), + ), + ), + ) + .execute(connection)?; diesel::delete( messages::table.filter( messages::session_id.eq_any( @@ -272,6 +300,10 @@ impl Database { pub fn delete_session(&mut self, session_id: i32) -> Result<(), String> { self.connection .transaction(|connection| { + diesel::delete( + a2ui_messages::table.filter(a2ui_messages::session_id.eq(session_id)), + ) + .execute(connection)?; diesel::delete(messages::table.filter(messages::session_id.eq(session_id))) .execute(connection)?; diesel::delete(sessions::table.find(session_id)).execute(connection)?; @@ -289,6 +321,35 @@ impl Database { .map_err(|error| error.to_string()) } + pub fn load_a2ui_messages( + &mut self, + session_id: i32, + ) -> Result, String> { + a2ui_messages::table + .filter(a2ui_messages::session_id.eq(session_id)) + .order(a2ui_messages::id.asc()) + .select(StoredA2uiMessage::as_select()) + .load(&mut self.connection) + .map_err(|error| error.to_string()) + } + + pub fn insert_a2ui_message( + &mut self, + session_id: i32, + message_id: i32, + json: &str, + ) -> Result { + diesel::insert_into(a2ui_messages::table) + .values(NewA2uiMessage { + session_id, + message_id, + json, + }) + .returning(StoredA2uiMessage::as_returning()) + .get_result(&mut self.connection) + .map_err(|error| error.to_string()) + } + pub fn update_session_context( &mut self, session_id: i32, @@ -628,6 +689,13 @@ mod tests { database .update_message(assistant.id, Some("Reasoning"), true, "Answer") .unwrap(); + database + .insert_a2ui_message( + session.id, + assistant.id, + r#"{"version":"v1.0","createSurface":{"surfaceId":"saved","catalogId":"https://ds4server.local/a2ui/v1_0/catalog.json"}}"#, + ) + .unwrap(); database .continue_tool_turn( session.id, @@ -662,6 +730,10 @@ mod tests { assert_eq!(messages[5].content, "Tool reminder"); assert!(!messages[6].user); assert!(!messages[6].tool); + let a2ui = reopened.load_a2ui_messages(session.id).unwrap(); + assert_eq!(a2ui.len(), 1); + assert_eq!(a2ui[0].message_id, messages[2].id); + assert!(a2ui[0].json.contains("createSurface")); let first = reopened .record_compaction( session.id, @@ -744,6 +816,7 @@ mod tests { assert_eq!(history[15].content, "After third compaction"); reopened.delete_session(session.id).unwrap(); assert!(reopened.load_messages(session.id).unwrap().is_empty()); + assert!(reopened.load_a2ui_messages(session.id).unwrap().is_empty()); drop(reopened); fs::remove_file(path).unwrap(); } diff --git a/src/main.rs b/src/main.rs index 97211e4..099aecc 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,3 +1,5 @@ +mod a2ui; +mod a2ui_validation; mod agent; mod app; mod compaction; @@ -9,6 +11,8 @@ mod model; #[cfg(target_os = "macos")] mod native_edit; #[cfg(target_os = "macos")] +mod native_media; +#[cfg(target_os = "macos")] mod native_menu; #[cfg(target_os = "macos")] mod runtime; @@ -21,6 +25,13 @@ use app::{App, Message, app_icon, app_theme}; use iced::{Size, window}; fn main() -> iced::Result { + if std::env::args().nth(1).as_deref() == Some("validate-a2ui") { + if let Err(error) = a2ui_validation::run(std::env::args().skip(2)) { + eprintln!("A2UI validation failed: {error}"); + std::process::exit(1); + } + return Ok(()); + } #[cfg(target_os = "macos")] if let Err(error) = engine::configure_metal_sources() { eprintln!("DS4Server: {error}"); diff --git a/src/native_media.rs b/src/native_media.rs new file mode 100644 index 0000000..8c93bfb --- /dev/null +++ b/src/native_media.rs @@ -0,0 +1,39 @@ +use std::ffi::{CString, c_char}; + +unsafe extern "C" { + fn ds4_media_open(url: *const c_char, title: *const c_char, video: bool) -> bool; +} + +pub(crate) fn open(url: &str, title: &str, video: bool) -> Result<(), String> { + let url = playable_url(url)?; + let url = CString::new(url.as_str()).map_err(|error| error.to_string())?; + let title = CString::new(title).map_err(|error| error.to_string())?; + // SAFETY: Both C strings live for the duration of the call; Objective-C copies them. + if unsafe { ds4_media_open(url.as_ptr(), title.as_ptr(), video) } { + Ok(()) + } else { + Err("AVKit could not create the media player".into()) + } +} + +fn playable_url(value: &str) -> Result { + let url = url::Url::parse(value).map_err(|error| format!("invalid media URL: {error}"))?; + if matches!(url.scheme(), "http" | "https") { + Ok(url) + } else { + Err("media URLs must use HTTP(S)".into()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn media_player_only_accepts_remote_http_urls() { + assert!(playable_url("https://example.com/video.mp4").is_ok()); + assert!(playable_url("http://example.com/audio.mp3").is_ok()); + assert!(playable_url("file:///tmp/private.mov").is_err()); + assert!(playable_url("javascript:alert(1)").is_err()); + } +} diff --git a/src/schema.rs b/src/schema.rs index 8908f9e..bfaf338 100644 --- a/src/schema.rs +++ b/src/schema.rs @@ -1,3 +1,12 @@ +diesel::table! { + a2ui_messages (id) { + id -> Integer, + session_id -> Integer, + message_id -> Integer, + json -> Text, + } +} + diesel::table! { messages (id) { id -> Integer, @@ -36,5 +45,7 @@ diesel::table! { } diesel::joinable!(sessions -> projects (project_id)); +diesel::joinable!(a2ui_messages -> messages (message_id)); +diesel::joinable!(a2ui_messages -> sessions (session_id)); diesel::joinable!(messages -> sessions (session_id)); -diesel::allow_tables_to_appear_in_same_query!(messages, projects, sessions); +diesel::allow_tables_to_appear_in_same_query!(a2ui_messages, messages, projects, sessions);