feat: a2ui interface to enablee the LLM to give structured information
This commit is contained in:
Georg Bauer
117
Cargo.lock
generated
117
Cargo.lock
generated
@@ -993,14 +993,17 @@ dependencies = [
|
|||||||
"diesel",
|
"diesel",
|
||||||
"diesel_migrations",
|
"diesel_migrations",
|
||||||
"iced",
|
"iced",
|
||||||
|
"image",
|
||||||
"memmap2",
|
"memmap2",
|
||||||
"muda",
|
"muda",
|
||||||
"png",
|
"png 0.17.16",
|
||||||
|
"regex",
|
||||||
"rfd",
|
"rfd",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
"serde_norway",
|
"serde_norway",
|
||||||
"sha2",
|
"sha2",
|
||||||
|
"time",
|
||||||
"ureq",
|
"ureq",
|
||||||
"url",
|
"url",
|
||||||
]
|
]
|
||||||
@@ -1495,6 +1498,16 @@ dependencies = [
|
|||||||
"weezl",
|
"weezl",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "gif"
|
||||||
|
version = "0.14.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "ee8cfcc411d9adbbaba82fb72661cc1bcca13e8bba98b364e62b2dba8f960159"
|
||||||
|
dependencies = [
|
||||||
|
"color_quant",
|
||||||
|
"weezl",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "gio"
|
name = "gio"
|
||||||
version = "0.18.4"
|
version = "0.18.4"
|
||||||
@@ -1854,6 +1867,7 @@ dependencies = [
|
|||||||
"iced_runtime",
|
"iced_runtime",
|
||||||
"iced_widget",
|
"iced_widget",
|
||||||
"iced_winit",
|
"iced_winit",
|
||||||
|
"image",
|
||||||
"thiserror 2.0.19",
|
"thiserror 2.0.19",
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -1913,6 +1927,8 @@ dependencies = [
|
|||||||
"half",
|
"half",
|
||||||
"iced_core",
|
"iced_core",
|
||||||
"iced_futures",
|
"iced_futures",
|
||||||
|
"image",
|
||||||
|
"kamadak-exif",
|
||||||
"log",
|
"log",
|
||||||
"raw-window-handle",
|
"raw-window-handle",
|
||||||
"rustc-hash 2.1.3",
|
"rustc-hash 2.1.3",
|
||||||
@@ -2149,6 +2165,24 @@ dependencies = [
|
|||||||
"icu_properties",
|
"icu_properties",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "image"
|
||||||
|
version = "0.25.10"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "85ab80394333c02fe689eaf900ab500fbd0c2213da414687ebf995a65d5a6104"
|
||||||
|
dependencies = [
|
||||||
|
"bytemuck",
|
||||||
|
"byteorder-lite",
|
||||||
|
"color_quant",
|
||||||
|
"gif 0.14.2",
|
||||||
|
"image-webp",
|
||||||
|
"moxcms",
|
||||||
|
"num-traits",
|
||||||
|
"png 0.18.1",
|
||||||
|
"zune-core 0.5.1",
|
||||||
|
"zune-jpeg 0.5.15",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "image-webp"
|
name = "image-webp"
|
||||||
version = "0.2.4"
|
version = "0.2.4"
|
||||||
@@ -2260,6 +2294,15 @@ dependencies = [
|
|||||||
"wasm-bindgen",
|
"wasm-bindgen",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "kamadak-exif"
|
||||||
|
version = "0.6.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "1130d80c7374efad55a117d715a3af9368f0fa7a2c54573afc15a188cd984837"
|
||||||
|
dependencies = [
|
||||||
|
"mutate_once",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "keyboard-types"
|
name = "keyboard-types"
|
||||||
version = "0.7.0"
|
version = "0.7.0"
|
||||||
@@ -2517,6 +2560,16 @@ dependencies = [
|
|||||||
"simd-adler32",
|
"simd-adler32",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "moxcms"
|
||||||
|
version = "0.8.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "bb85c154ba489f01b25c0d36ae69a87e4a1c73a72631fc6c0eb6dde34a73e44b"
|
||||||
|
dependencies = [
|
||||||
|
"num-traits",
|
||||||
|
"pxfm",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "muda"
|
name = "muda"
|
||||||
version = "0.15.3"
|
version = "0.15.3"
|
||||||
@@ -2532,7 +2585,7 @@ dependencies = [
|
|||||||
"objc2-app-kit 0.2.2",
|
"objc2-app-kit 0.2.2",
|
||||||
"objc2-foundation 0.2.2",
|
"objc2-foundation 0.2.2",
|
||||||
"once_cell",
|
"once_cell",
|
||||||
"png",
|
"png 0.17.16",
|
||||||
"thiserror 1.0.69",
|
"thiserror 1.0.69",
|
||||||
"windows-sys 0.59.0",
|
"windows-sys 0.59.0",
|
||||||
]
|
]
|
||||||
@@ -2562,6 +2615,12 @@ dependencies = [
|
|||||||
"zbus",
|
"zbus",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "mutate_once"
|
||||||
|
version = "0.1.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "13d2233c9842d08cfe13f9eac96e207ca6a2ea10b80259ebe8ad0268be27d2af"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "naga"
|
name = "naga"
|
||||||
version = "27.0.3"
|
version = "27.0.3"
|
||||||
@@ -3212,6 +3271,19 @@ dependencies = [
|
|||||||
"miniz_oxide",
|
"miniz_oxide",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "png"
|
||||||
|
version = "0.18.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "60769b8b31b2a9f263dae2776c37b1b28ae246943cf719eb6946a1db05128a61"
|
||||||
|
dependencies = [
|
||||||
|
"bitflags 2.13.1",
|
||||||
|
"crc32fast",
|
||||||
|
"fdeflate",
|
||||||
|
"flate2",
|
||||||
|
"miniz_oxide",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "polling"
|
name = "polling"
|
||||||
version = "3.11.0"
|
version = "3.11.0"
|
||||||
@@ -3364,6 +3436,12 @@ version = "0.11.0"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "007d8adb5ddab6f8e3f491ac63566a7d5002cc7ed73901f72057943fa71ae1ae"
|
checksum = "007d8adb5ddab6f8e3f491ac63566a7d5002cc7ed73901f72057943fa71ae1ae"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "pxfm"
|
||||||
|
version = "0.1.30"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "d55d956fa96f5ec02be2e13af0e20391a5aa83d6a074e3ad368959d0fab299ea"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "quick-error"
|
name = "quick-error"
|
||||||
version = "2.0.1"
|
version = "2.0.1"
|
||||||
@@ -3505,6 +3583,18 @@ dependencies = [
|
|||||||
"bitflags 2.13.1",
|
"bitflags 2.13.1",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "regex"
|
||||||
|
version = "1.13.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d"
|
||||||
|
dependencies = [
|
||||||
|
"aho-corasick",
|
||||||
|
"memchr",
|
||||||
|
"regex-automata",
|
||||||
|
"regex-syntax",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "regex-automata"
|
name = "regex-automata"
|
||||||
version = "0.4.16"
|
version = "0.4.16"
|
||||||
@@ -3534,7 +3624,7 @@ version = "0.45.1"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "a8928798c0a55e03c9ca6c4c6846f76377427d2c1e1f7e6de3c06ae57942df43"
|
checksum = "a8928798c0a55e03c9ca6c4c6846f76377427d2c1e1f7e6de3c06ae57942df43"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"gif",
|
"gif 0.13.3",
|
||||||
"image-webp",
|
"image-webp",
|
||||||
"log",
|
"log",
|
||||||
"pico-args",
|
"pico-args",
|
||||||
@@ -3542,7 +3632,7 @@ dependencies = [
|
|||||||
"svgtypes",
|
"svgtypes",
|
||||||
"tiny-skia",
|
"tiny-skia",
|
||||||
"usvg",
|
"usvg",
|
||||||
"zune-jpeg",
|
"zune-jpeg 0.4.21",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -4336,7 +4426,7 @@ dependencies = [
|
|||||||
"bytemuck",
|
"bytemuck",
|
||||||
"cfg-if",
|
"cfg-if",
|
||||||
"log",
|
"log",
|
||||||
"png",
|
"png 0.17.16",
|
||||||
"tiny-skia-path",
|
"tiny-skia-path",
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -5846,13 +5936,28 @@ version = "0.4.12"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "3f423a2c17029964870cfaabb1f13dfab7d092a62a29a89264f4d36990ca414a"
|
checksum = "3f423a2c17029964870cfaabb1f13dfab7d092a62a29a89264f4d36990ca414a"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "zune-core"
|
||||||
|
version = "0.5.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "cb8a0807f7c01457d0379ba880ba6322660448ddebc890ce29bb64da71fb40f9"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "zune-jpeg"
|
name = "zune-jpeg"
|
||||||
version = "0.4.21"
|
version = "0.4.21"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "29ce2c8a9384ad323cf564b67da86e21d3cfdff87908bc1223ed5c99bc792713"
|
checksum = "29ce2c8a9384ad323cf564b67da86e21d3cfdff87908bc1223ed5c99bc792713"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"zune-core",
|
"zune-core 0.4.12",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "zune-jpeg"
|
||||||
|
version = "0.5.15"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "27bc9d5b815bc103f142aa054f561d9187d191692ec7c2d1e2b4737f8dbd7296"
|
||||||
|
dependencies = [
|
||||||
|
"zune-core 0.5.1",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
|
|||||||
@@ -14,14 +14,17 @@ cc = "1.3.0"
|
|||||||
[dependencies]
|
[dependencies]
|
||||||
diesel = { version = "2.3.11", features = ["sqlite", "returning_clauses_for_sqlite_3_35", "64-column-tables"] }
|
diesel = { version = "2.3.11", features = ["sqlite", "returning_clauses_for_sqlite_3_35", "64-column-tables"] }
|
||||||
diesel_migrations = "2.3.2"
|
diesel_migrations = "2.3.2"
|
||||||
iced = { version = "0.14.0", features = ["advanced", "highlighter", "markdown", "svg", "tokio"] }
|
iced = { version = "0.14.0", features = ["advanced", "highlighter", "image-without-codecs", "markdown", "svg", "tokio"] }
|
||||||
|
image = { version = "0.25.10", default-features = false, features = ["gif", "jpeg", "png", "webp"] }
|
||||||
memmap2 = "0.9.11"
|
memmap2 = "0.9.11"
|
||||||
png = "0.17.16"
|
png = "0.17.16"
|
||||||
|
regex = "1.13.1"
|
||||||
rfd = "0.15.4"
|
rfd = "0.15.4"
|
||||||
serde = { version = "1.0.228", features = ["derive"] }
|
serde = { version = "1.0.228", features = ["derive"] }
|
||||||
serde_json = { version = "1.0.149", features = ["preserve_order", "raw_value"] }
|
serde_json = { version = "1.0.149", features = ["preserve_order", "raw_value"] }
|
||||||
serde_norway = "0.9.42"
|
serde_norway = "0.9.42"
|
||||||
sha2 = "0.11.0"
|
sha2 = "0.11.0"
|
||||||
|
time = { version = "0.3.54", features = ["formatting", "parsing"] }
|
||||||
ureq = { version = "3.3.0", default-features = false, features = ["rustls"] }
|
ureq = { version = "3.3.0", default-features = false, features = ["rustls"] }
|
||||||
url = "2.5.8"
|
url = "2.5.8"
|
||||||
|
|
||||||
|
|||||||
26
PLAN.md
26
PLAN.md
@@ -4,29 +4,7 @@ Only unfinished implementation work belongs here. DwarfStar remains the
|
|||||||
behavioral oracle for model execution, token processing, context accounting,
|
behavioral oracle for model execution, token processing, context accounting,
|
||||||
KV-cache behavior, the HTTP API, and the built-in agent loop.
|
KV-cache behavior, the HTTP API, and the built-in agent loop.
|
||||||
|
|
||||||
## 1. Add A2UI local-chat surfaces
|
## 1. Finish native project and chat controls
|
||||||
|
|
||||||
- Use the public [A2UI protocol](https://a2ui.org/) as the normative reference.
|
|
||||||
Implement its versioned, streaming JSON messages and keep `../bDS2` only as
|
|
||||||
a reference for the research-oriented components and presentation we want.
|
|
||||||
- Define a DS4Server catalog and native Iced renderer for the A2UI Basic Catalog
|
|
||||||
plus useful research surfaces such as charts, tables, metrics, timelines,
|
|
||||||
maps, mind maps, tabs, cards, lists, and forms.
|
|
||||||
- Support `createSurface`, incremental `updateComponents` and `updateDataModel`
|
|
||||||
messages, and `deleteSurface`. The model must be able to add, replace, and
|
|
||||||
update components and data over multiple responses without recreating the
|
|
||||||
whole surface.
|
|
||||||
- Persist A2UI messages with the chat and restore the resulting surfaces after
|
|
||||||
relaunch. Validate messages against the selected catalog, report validation
|
|
||||||
failures back to the model, and retain a readable fallback in the transcript.
|
|
||||||
- Implement catalog-declared local functions and agent events. UI-local edits
|
|
||||||
update the surface immediately; agent events return the action and current
|
|
||||||
data to the conversation so the model can respond with further A2UI updates.
|
|
||||||
The existing approval flow applies only if the resulting agent turn invokes
|
|
||||||
an ordinary side-effecting tool such as shell or file mutation.
|
|
||||||
- A2UI can be enabled/disabled in preferences to reduce the system prompt
|
|
||||||
|
|
||||||
## 2. Finish native project and chat controls
|
|
||||||
|
|
||||||
- Add functional File, View, and Help menus; update native menu enabled state
|
- Add functional File, View, and Help menus; update native menu enabled state
|
||||||
from the focused control; complete undo/redo behavior; and make transcript
|
from the focused control; complete undo/redo behavior; and make transcript
|
||||||
@@ -42,7 +20,7 @@ KV-cache behavior, the HTTP API, and the built-in agent loop.
|
|||||||
- the hellp menu must open a window with an actual documentation in it,
|
- the hellp menu must open a window with an actual documentation in it,
|
||||||
that still needs to be constructed from the information and implementation
|
that still needs to be constructed from the information and implementation
|
||||||
|
|
||||||
## 3. Add Dev Brain support
|
## 2. Add Dev Brain support
|
||||||
|
|
||||||
- Let the user select one Obsidian vault and give the local agent bounded
|
- Let the user select one Obsidian vault and give the local agent bounded
|
||||||
Markdown memory_search, memory_read, memory_create, and memory_append
|
Markdown memory_search, memory_read, memory_create, and memory_append
|
||||||
|
|||||||
38
README.md
38
README.md
@@ -39,6 +39,38 @@ Projects, sessions, and transcripts use SQLite. Settings use a compact YAML file
|
|||||||
that contains only values changed from their defaults and can also be edited by
|
that contains only values changed from their defaults and can also be edited by
|
||||||
hand.
|
hand.
|
||||||
|
|
||||||
|
## A2UI widgets
|
||||||
|
|
||||||
|
DS4Server renders A2UI v1.0 surfaces with these widgets:
|
||||||
|
|
||||||
|
- **Content:** Markdown text, images (including avatars), icons, video and audio
|
||||||
|
players, and dividers.
|
||||||
|
- **Layout:** rows, columns, lists, cards, modals, and tabs.
|
||||||
|
- **Controls:** buttons, text fields, checkboxes, sliders, date/time inputs, and
|
||||||
|
choice pickers.
|
||||||
|
- **Research:** bar, line, area, stacked-bar, pie, and donut charts; tables,
|
||||||
|
metrics, timelines, maps, mind maps, and forms.
|
||||||
|
|
||||||
|
Video posters render in the surface; video and audio playback uses native
|
||||||
|
macOS AVKit player windows with standard transport, volume, and fullscreen
|
||||||
|
controls.
|
||||||
|
|
||||||
|
### Live A2UI validation
|
||||||
|
|
||||||
|
With DS4Server running and its local endpoint enabled, run the opt-in model
|
||||||
|
validation suite with:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
cargo run --release -- validate-a2ui
|
||||||
|
```
|
||||||
|
|
||||||
|
It sends natural-language chart, form, choice, and full-composition prompts
|
||||||
|
through the current A2UI tool description, then replays every returned protocol
|
||||||
|
message through the real catalog and component-graph validators. Use `--case
|
||||||
|
pie-natural`, `--case composed-basics`, `--attempts 3`, `--model MODEL_ID`, or
|
||||||
|
`--endpoint URL` to narrow or repeat runs. This slow, model-dependent suite is
|
||||||
|
intentionally separate from `cargo test`.
|
||||||
|
|
||||||
## Build and run
|
## Build and run
|
||||||
|
|
||||||
DS4Server targets macOS 13 or newer and requires Rust 1.97 plus
|
DS4Server targets macOS 13 or newer and requires Rust 1.97 plus
|
||||||
@@ -53,9 +85,9 @@ open target/release/DS4Server.app
|
|||||||
## Application data
|
## Application data
|
||||||
|
|
||||||
```text
|
```text
|
||||||
~/Library/Application Support/DS4Server.rfc1437.de/data.sqlite3
|
~/Library/Application Support/de.rfc1437.ds4server/data.sqlite3
|
||||||
~/Library/Application Support/DS4Server.rfc1437.de/config.yaml
|
~/Library/Application Support/de.rfc1437.ds4server/config.yaml
|
||||||
~/Library/Application Support/DS4Server.rfc1437.de/kv-cache/
|
~/Library/Application Support/de.rfc1437.ds4server/kv-cache/
|
||||||
```
|
```
|
||||||
|
|
||||||
Deleting a project or session removes DS4Server metadata and its associated
|
Deleting a project or session removes DS4Server metadata and its associated
|
||||||
|
|||||||
144
assets/a2ui/catalog.json
Normal file
144
assets/a2ui/catalog.json
Normal file
@@ -0,0 +1,144 @@
|
|||||||
|
{
|
||||||
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||||
|
"$id": "https://ds4server.local/a2ui/v1_0/catalog.json",
|
||||||
|
"title": "DS4Server A2UI Catalog",
|
||||||
|
"catalogId": "https://ds4server.local/a2ui/v1_0/catalog.json",
|
||||||
|
"instructions": "Use the Basic Catalog for interaction and layout. Use research components for compact analytical results.",
|
||||||
|
"components": {
|
||||||
|
"Text": { "$ref": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json#/components/Text" },
|
||||||
|
"Image": { "$ref": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json#/components/Image" },
|
||||||
|
"Icon": { "$ref": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json#/components/Icon" },
|
||||||
|
"Video": { "$ref": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json#/components/Video" },
|
||||||
|
"AudioPlayer": { "$ref": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json#/components/AudioPlayer" },
|
||||||
|
"Divider": { "$ref": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json#/components/Divider" },
|
||||||
|
"Row": { "$ref": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json#/components/Row" },
|
||||||
|
"Column": { "$ref": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json#/components/Column" },
|
||||||
|
"List": { "$ref": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json#/components/List" },
|
||||||
|
"Card": { "$ref": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json#/components/Card" },
|
||||||
|
"Modal": { "$ref": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json#/components/Modal" },
|
||||||
|
"Tabs": { "$ref": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json#/components/Tabs" },
|
||||||
|
"Button": { "$ref": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json#/components/Button" },
|
||||||
|
"TextField": { "$ref": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json#/components/TextField" },
|
||||||
|
"CheckBox": { "$ref": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json#/components/CheckBox" },
|
||||||
|
"Slider": { "$ref": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json#/components/Slider" },
|
||||||
|
"DateTimeInput": { "$ref": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json#/components/DateTimeInput" },
|
||||||
|
"ChoicePicker": { "$ref": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json#/components/ChoicePicker" },
|
||||||
|
"Chart": { "$ref": "#/$defs/chart" },
|
||||||
|
"Table": { "$ref": "#/$defs/table" },
|
||||||
|
"Metric": { "$ref": "#/$defs/metric" },
|
||||||
|
"Timeline": { "$ref": "#/$defs/timeline" },
|
||||||
|
"Map": { "$ref": "#/$defs/map" },
|
||||||
|
"MindMap": { "$ref": "#/$defs/mindMap" },
|
||||||
|
"Form": { "$ref": "#/$defs/form" }
|
||||||
|
},
|
||||||
|
"functions": {
|
||||||
|
"required": { "$ref": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json#/functions/required" },
|
||||||
|
"regex": { "$ref": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json#/functions/regex" },
|
||||||
|
"length": { "$ref": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json#/functions/length" },
|
||||||
|
"numeric": { "$ref": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json#/functions/numeric" },
|
||||||
|
"email": { "$ref": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json#/functions/email" },
|
||||||
|
"formatString": { "$ref": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json#/functions/formatString" },
|
||||||
|
"formatNumber": { "$ref": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json#/functions/formatNumber" },
|
||||||
|
"formatCurrency": { "$ref": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json#/functions/formatCurrency" },
|
||||||
|
"formatDate": { "$ref": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json#/functions/formatDate" },
|
||||||
|
"pluralize": { "$ref": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json#/functions/pluralize" },
|
||||||
|
"openUrl": { "$ref": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json#/functions/openUrl" },
|
||||||
|
"and": { "$ref": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json#/functions/and" },
|
||||||
|
"or": { "$ref": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json#/functions/or" },
|
||||||
|
"not": { "$ref": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json#/functions/not" }
|
||||||
|
},
|
||||||
|
"$defs": {
|
||||||
|
"surfaceProperties": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"iconUrl": { "type": "string", "format": "uri" },
|
||||||
|
"agentDisplayName": { "type": "string" }
|
||||||
|
},
|
||||||
|
"additionalProperties": true
|
||||||
|
},
|
||||||
|
"researchBase": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"id": { "type": "string" },
|
||||||
|
"component": { "type": "string" },
|
||||||
|
"title": { "type": "string" },
|
||||||
|
"accessibility": { "type": "object" },
|
||||||
|
"weight": { "type": "number" },
|
||||||
|
"checks": { "type": "array" }
|
||||||
|
},
|
||||||
|
"required": ["id", "component"]
|
||||||
|
},
|
||||||
|
"chart": {
|
||||||
|
"allOf": [
|
||||||
|
{ "$ref": "#/$defs/researchBase" },
|
||||||
|
{ "properties": { "component": { "const": "Chart" }, "chartType": { "enum": ["bar", "line", "area", "stackedBar", "pie", "donut"] }, "series": {} }, "required": ["chartType", "series"] }
|
||||||
|
],
|
||||||
|
"unevaluatedProperties": false
|
||||||
|
},
|
||||||
|
"table": {
|
||||||
|
"allOf": [
|
||||||
|
{ "$ref": "#/$defs/researchBase" },
|
||||||
|
{ "properties": { "component": { "const": "Table" }, "columns": {}, "rows": {} }, "required": ["columns", "rows"] }
|
||||||
|
],
|
||||||
|
"unevaluatedProperties": false
|
||||||
|
},
|
||||||
|
"metric": {
|
||||||
|
"allOf": [
|
||||||
|
{ "$ref": "#/$defs/researchBase" },
|
||||||
|
{ "properties": { "component": { "const": "Metric" }, "label": {}, "value": {}, "detail": {}, "trend": {} }, "required": ["label", "value"] }
|
||||||
|
],
|
||||||
|
"unevaluatedProperties": false
|
||||||
|
},
|
||||||
|
"timeline": {
|
||||||
|
"allOf": [
|
||||||
|
{ "$ref": "#/$defs/researchBase" },
|
||||||
|
{ "properties": { "component": { "const": "Timeline" }, "events": {} }, "required": ["events"] }
|
||||||
|
],
|
||||||
|
"unevaluatedProperties": false
|
||||||
|
},
|
||||||
|
"map": {
|
||||||
|
"allOf": [
|
||||||
|
{ "$ref": "#/$defs/researchBase" },
|
||||||
|
{ "properties": { "component": { "const": "Map" }, "locations": {} }, "required": ["locations"] }
|
||||||
|
],
|
||||||
|
"unevaluatedProperties": false
|
||||||
|
},
|
||||||
|
"mindMap": {
|
||||||
|
"allOf": [
|
||||||
|
{ "$ref": "#/$defs/researchBase" },
|
||||||
|
{ "properties": { "component": { "const": "MindMap" }, "nodes": {} }, "required": ["nodes"] }
|
||||||
|
],
|
||||||
|
"unevaluatedProperties": false
|
||||||
|
},
|
||||||
|
"form": {
|
||||||
|
"allOf": [
|
||||||
|
{ "$ref": "#/$defs/researchBase" },
|
||||||
|
{ "properties": { "component": { "const": "Form" }, "children": { "$ref": "https://a2ui.org/specification/v1_0/common_types.json#/$defs/ChildList" }, "submitLabel": { "type": "string" }, "action": { "$ref": "https://a2ui.org/specification/v1_0/common_types.json#/$defs/Action" } }, "required": ["children"] }
|
||||||
|
],
|
||||||
|
"unevaluatedProperties": false
|
||||||
|
},
|
||||||
|
"anyComponent": {
|
||||||
|
"oneOf": [
|
||||||
|
{ "$ref": "#/components/Text" }, { "$ref": "#/components/Image" }, { "$ref": "#/components/Icon" },
|
||||||
|
{ "$ref": "#/components/Video" }, { "$ref": "#/components/AudioPlayer" }, { "$ref": "#/components/Divider" },
|
||||||
|
{ "$ref": "#/components/Row" }, { "$ref": "#/components/Column" }, { "$ref": "#/components/List" },
|
||||||
|
{ "$ref": "#/components/Card" }, { "$ref": "#/components/Modal" }, { "$ref": "#/components/Tabs" },
|
||||||
|
{ "$ref": "#/components/Button" }, { "$ref": "#/components/TextField" }, { "$ref": "#/components/CheckBox" },
|
||||||
|
{ "$ref": "#/components/Slider" }, { "$ref": "#/components/DateTimeInput" }, { "$ref": "#/components/ChoicePicker" },
|
||||||
|
{ "$ref": "#/components/Chart" }, { "$ref": "#/components/Table" }, { "$ref": "#/components/Metric" },
|
||||||
|
{ "$ref": "#/components/Timeline" }, { "$ref": "#/components/Map" }, { "$ref": "#/components/MindMap" },
|
||||||
|
{ "$ref": "#/components/Form" }
|
||||||
|
],
|
||||||
|
"discriminator": { "propertyName": "component" }
|
||||||
|
},
|
||||||
|
"anyFunction": {
|
||||||
|
"oneOf": [
|
||||||
|
{ "$ref": "#/functions/required" }, { "$ref": "#/functions/regex" }, { "$ref": "#/functions/length" },
|
||||||
|
{ "$ref": "#/functions/numeric" }, { "$ref": "#/functions/email" }, { "$ref": "#/functions/formatString" },
|
||||||
|
{ "$ref": "#/functions/formatNumber" }, { "$ref": "#/functions/formatCurrency" }, { "$ref": "#/functions/formatDate" },
|
||||||
|
{ "$ref": "#/functions/pluralize" }, { "$ref": "#/functions/openUrl" }, { "$ref": "#/functions/and" },
|
||||||
|
{ "$ref": "#/functions/or" }, { "$ref": "#/functions/not" }
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
8
build.rs
8
build.rs
@@ -13,6 +13,11 @@ fn main() {
|
|||||||
.flag("-fobjc-arc")
|
.flag("-fobjc-arc")
|
||||||
.opt_level(3)
|
.opt_level(3)
|
||||||
.compile("ds4_metal");
|
.compile("ds4_metal");
|
||||||
|
println!("cargo:rerun-if-changed=native/media");
|
||||||
|
cc::Build::new()
|
||||||
|
.file("native/media/ds4_media.m")
|
||||||
|
.flag("-fobjc-arc")
|
||||||
|
.compile("ds4_media");
|
||||||
println!("cargo:rerun-if-changed=native/web");
|
println!("cargo:rerun-if-changed=native/web");
|
||||||
cc::Build::new()
|
cc::Build::new()
|
||||||
.include("native/web")
|
.include("native/web")
|
||||||
@@ -20,5 +25,8 @@ fn main() {
|
|||||||
.opt_level(2)
|
.opt_level(2)
|
||||||
.compile("ds4_web");
|
.compile("ds4_web");
|
||||||
println!("cargo:rustc-link-lib=framework=Foundation");
|
println!("cargo:rustc-link-lib=framework=Foundation");
|
||||||
|
println!("cargo:rustc-link-lib=framework=AppKit");
|
||||||
|
println!("cargo:rustc-link-lib=framework=AVFoundation");
|
||||||
|
println!("cargo:rustc-link-lib=framework=AVKit");
|
||||||
println!("cargo:rustc-link-lib=framework=Metal");
|
println!("cargo:rustc-link-lib=framework=Metal");
|
||||||
}
|
}
|
||||||
|
|||||||
1
migrations/20260727120000_add_a2ui_messages/down.sql
Normal file
1
migrations/20260727120000_add_a2ui_messages/down.sql
Normal file
@@ -0,0 +1 @@
|
|||||||
|
DROP TABLE a2ui_messages;
|
||||||
10
migrations/20260727120000_add_a2ui_messages/up.sql
Normal file
10
migrations/20260727120000_add_a2ui_messages/up.sql
Normal file
@@ -0,0 +1,10 @@
|
|||||||
|
CREATE TABLE a2ui_messages (
|
||||||
|
id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
|
||||||
|
session_id INTEGER NOT NULL,
|
||||||
|
message_id INTEGER NOT NULL,
|
||||||
|
json TEXT NOT NULL,
|
||||||
|
FOREIGN KEY (session_id) REFERENCES sessions(id),
|
||||||
|
FOREIGN KEY (message_id) REFERENCES messages(id)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX a2ui_messages_session_id ON a2ui_messages(session_id, id);
|
||||||
66
native/media/ds4_media.m
Normal file
66
native/media/ds4_media.m
Normal file
@@ -0,0 +1,66 @@
|
|||||||
|
#import <AppKit/AppKit.h>
|
||||||
|
#import <AVKit/AVKit.h>
|
||||||
|
#include <stdbool.h>
|
||||||
|
|
||||||
|
@interface DS4MediaWindowController : NSWindowController <NSWindowDelegate>
|
||||||
|
@end
|
||||||
|
|
||||||
|
static NSMutableSet<DS4MediaWindowController *> *DS4MediaWindows(void) {
|
||||||
|
static NSMutableSet<DS4MediaWindowController *> *windows;
|
||||||
|
static dispatch_once_t once;
|
||||||
|
dispatch_once(&once, ^{
|
||||||
|
windows = [NSMutableSet set];
|
||||||
|
});
|
||||||
|
return windows;
|
||||||
|
}
|
||||||
|
|
||||||
|
@implementation DS4MediaWindowController
|
||||||
|
|
||||||
|
- (void)windowWillClose:(NSNotification *)notification {
|
||||||
|
AVPlayerView *view = (AVPlayerView *)self.window.contentView;
|
||||||
|
[view.player pause];
|
||||||
|
[DS4MediaWindows() removeObject:self];
|
||||||
|
}
|
||||||
|
|
||||||
|
@end
|
||||||
|
|
||||||
|
bool ds4_media_open(const char *url_bytes, const char *title_bytes, bool video) {
|
||||||
|
if (url_bytes == NULL || title_bytes == NULL) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
NSString *url_string = [NSString stringWithUTF8String:url_bytes];
|
||||||
|
NSString *title = [NSString stringWithUTF8String:title_bytes];
|
||||||
|
NSURL *url = [NSURL URLWithString:url_string];
|
||||||
|
if (url == nil || title == nil) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
dispatch_async(dispatch_get_main_queue(), ^{
|
||||||
|
NSRect frame = NSMakeRect(0, 0, video ? 800 : 560, video ? 500 : 180);
|
||||||
|
NSWindow *window = [[NSWindow alloc]
|
||||||
|
initWithContentRect:frame
|
||||||
|
styleMask:NSWindowStyleMaskTitled |
|
||||||
|
NSWindowStyleMaskClosable |
|
||||||
|
NSWindowStyleMaskMiniaturizable |
|
||||||
|
NSWindowStyleMaskResizable
|
||||||
|
backing:NSBackingStoreBuffered
|
||||||
|
defer:NO];
|
||||||
|
window.title = title.length > 0 ? title : url.lastPathComponent;
|
||||||
|
window.minSize = video ? NSMakeSize(480, 300) : NSMakeSize(420, 150);
|
||||||
|
|
||||||
|
AVPlayerView *view = [[AVPlayerView alloc] initWithFrame:frame];
|
||||||
|
view.autoresizingMask = NSViewWidthSizable | NSViewHeightSizable;
|
||||||
|
view.player = [AVPlayer playerWithURL:url];
|
||||||
|
window.contentView = view;
|
||||||
|
|
||||||
|
DS4MediaWindowController *controller =
|
||||||
|
[[DS4MediaWindowController alloc] initWithWindow:window];
|
||||||
|
window.delegate = controller;
|
||||||
|
[DS4MediaWindows() addObject:controller];
|
||||||
|
[controller showWindow:nil];
|
||||||
|
[window center];
|
||||||
|
[window makeKeyAndOrderFront:nil];
|
||||||
|
[view.player play];
|
||||||
|
});
|
||||||
|
return true;
|
||||||
|
}
|
||||||
2217
src/a2ui.rs
Normal file
2217
src/a2ui.rs
Normal file
File diff suppressed because it is too large
Load Diff
258
src/a2ui_validation.rs
Normal file
258
src/a2ui_validation.rs
Normal file
@@ -0,0 +1,258 @@
|
|||||||
|
use crate::a2ui::{BASIC_NON_MEDIA_COMPONENTS, Store, extract_lines, validate_surface_composition};
|
||||||
|
use crate::config::Config;
|
||||||
|
use crate::model::ModelChoice;
|
||||||
|
use serde_json::{Value, json};
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
|
const CASES: &[Case] = &[
|
||||||
|
Case {
|
||||||
|
name: "pie-natural",
|
||||||
|
prompt: "Do not call tools. The file extension counts are rs 41, md 7, and toml 2. Give me a pie chart with the numbers.",
|
||||||
|
components: &["Chart"],
|
||||||
|
chart_type: Some("pie"),
|
||||||
|
},
|
||||||
|
Case {
|
||||||
|
name: "pie-single-slice",
|
||||||
|
prompt: "Do not call tools. There are 41 rs files and no other file extensions. Show that distribution as a one-slice pie chart.",
|
||||||
|
components: &["Chart"],
|
||||||
|
chart_type: Some("pie"),
|
||||||
|
},
|
||||||
|
Case {
|
||||||
|
name: "donut-natural",
|
||||||
|
prompt: "Do not call tools. Show a donut chart for 62 completed, 23 active, and 15 blocked tasks. Include the total.",
|
||||||
|
components: &["Chart"],
|
||||||
|
chart_type: Some("donut"),
|
||||||
|
},
|
||||||
|
Case {
|
||||||
|
name: "form-controls",
|
||||||
|
prompt: "Do not call tools. Build an A2UI Form for a name, multiline notes, due date, and multiple checkbox priorities, with a submit button.",
|
||||||
|
components: &["Form"],
|
||||||
|
chart_type: None,
|
||||||
|
},
|
||||||
|
Case {
|
||||||
|
name: "filterable-choices",
|
||||||
|
prompt: "Do not call tools. Build an A2UI surface with a filterable mutually-exclusive ChoicePicker for Rust, Python, and TypeScript.",
|
||||||
|
components: &["ChoicePicker"],
|
||||||
|
chart_type: None,
|
||||||
|
},
|
||||||
|
Case {
|
||||||
|
name: "composed-basics",
|
||||||
|
prompt: "Do not call tools. Build one complete A2UI project dashboard. Compose a Card and Column containing Markdown text, an avatar Image, an Icon, a Divider, a weighted Row, Tabs, a dynamic List, a Modal, a Button, TextField, CheckBox, Slider, DateTimeInput, and a filterable mutually-exclusive ChoicePicker. Bind the controls to the data model and give the button an event.",
|
||||||
|
components: BASIC_NON_MEDIA_COMPONENTS,
|
||||||
|
chart_type: None,
|
||||||
|
},
|
||||||
|
Case {
|
||||||
|
name: "media-players",
|
||||||
|
prompt: "Do not call tools. Build one A2UI Card containing a Video with a poster URL and an AudioPlayer with a description. Use valid HTTPS media URLs and compose both into the root tree.",
|
||||||
|
components: &["Video", "AudioPlayer"],
|
||||||
|
chart_type: None,
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
struct Case {
|
||||||
|
name: &'static str,
|
||||||
|
prompt: &'static str,
|
||||||
|
components: &'static [&'static str],
|
||||||
|
chart_type: Option<&'static str>,
|
||||||
|
}
|
||||||
|
|
||||||
|
struct Options {
|
||||||
|
endpoint: Option<String>,
|
||||||
|
model: Option<String>,
|
||||||
|
case: Option<String>,
|
||||||
|
attempts: u32,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn run(args: impl Iterator<Item = String>) -> Result<(), String> {
|
||||||
|
let options = parse_options(args)?;
|
||||||
|
let config = Config::load(&crate::app::config_path())?;
|
||||||
|
let endpoint = options
|
||||||
|
.endpoint
|
||||||
|
.unwrap_or_else(|| format!("http://127.0.0.1:{}", config.endpoint.port));
|
||||||
|
let model_id = options
|
||||||
|
.model
|
||||||
|
.unwrap_or_else(|| config.model.id().to_owned());
|
||||||
|
let model =
|
||||||
|
ModelChoice::from_id(&model_id).ok_or_else(|| format!("unknown model `{model_id}`"))?;
|
||||||
|
let cases = CASES
|
||||||
|
.iter()
|
||||||
|
.filter(|case| options.case.as_deref().is_none_or(|name| name == case.name))
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
if cases.is_empty() {
|
||||||
|
return Err(format!(
|
||||||
|
"unknown case; choose one of: {}",
|
||||||
|
CASES
|
||||||
|
.iter()
|
||||||
|
.map(|case| case.name)
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join(", ")
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let agent: ureq::Agent = ureq::Agent::config_builder()
|
||||||
|
.timeout_connect(Some(Duration::from_secs(5)))
|
||||||
|
.timeout_recv_response(Some(Duration::from_secs(30 * 60)))
|
||||||
|
.timeout_recv_body(Some(Duration::from_secs(30)))
|
||||||
|
.build()
|
||||||
|
.into();
|
||||||
|
let mut system = crate::agent::system_prompt(model, &config.generation.system_prompt);
|
||||||
|
system.push_str("\n\n");
|
||||||
|
system.push_str(crate::a2ui::SYSTEM_PROMPT);
|
||||||
|
let metadata = Store::default().client_metadata();
|
||||||
|
let mut passed = 0;
|
||||||
|
let total = cases.len() as u32 * options.attempts;
|
||||||
|
println!("A2UI live validation: {model_id} at {endpoint}");
|
||||||
|
for case in cases {
|
||||||
|
for attempt in 1..=options.attempts {
|
||||||
|
let user = format!("{}\n\nA2UI client metadata:\n{}", case.prompt, metadata);
|
||||||
|
let payload = json!({
|
||||||
|
"model": model_id,
|
||||||
|
"messages": [
|
||||||
|
{"role": "system", "content": system},
|
||||||
|
{"role": "user", "content": user}
|
||||||
|
],
|
||||||
|
"reasoning_effort": "none",
|
||||||
|
"temperature": 0,
|
||||||
|
"max_tokens": 4096
|
||||||
|
});
|
||||||
|
let content = request(&agent, &endpoint, &payload)?;
|
||||||
|
match validate(case, &content) {
|
||||||
|
Ok(summary) => {
|
||||||
|
passed += 1;
|
||||||
|
println!("PASS {}#{attempt}: {summary}", case.name);
|
||||||
|
}
|
||||||
|
Err(error) => {
|
||||||
|
println!("FAIL {}#{attempt}: {error}", case.name);
|
||||||
|
println!(
|
||||||
|
"--- response ---\n{}\n--- end response ---",
|
||||||
|
bounded(&content)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
println!("A2UI validation summary: {passed}/{total} passed");
|
||||||
|
if passed == total {
|
||||||
|
Ok(())
|
||||||
|
} else {
|
||||||
|
Err(format!("{} live validation case(s) failed", total - passed))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_options(mut args: impl Iterator<Item = String>) -> Result<Options, String> {
|
||||||
|
let mut options = Options {
|
||||||
|
endpoint: None,
|
||||||
|
model: None,
|
||||||
|
case: None,
|
||||||
|
attempts: 1,
|
||||||
|
};
|
||||||
|
while let Some(argument) = args.next() {
|
||||||
|
let value = match argument.as_str() {
|
||||||
|
"--endpoint" | "--model" | "--case" | "--attempts" => args
|
||||||
|
.next()
|
||||||
|
.ok_or_else(|| format!("{argument} requires a value"))?,
|
||||||
|
"--help" | "-h" => {
|
||||||
|
println!(
|
||||||
|
"Usage: ds4-server validate-a2ui [--endpoint URL] [--model ID] [--case NAME] [--attempts N]"
|
||||||
|
);
|
||||||
|
std::process::exit(0);
|
||||||
|
}
|
||||||
|
_ => return Err(format!("unknown argument `{argument}`")),
|
||||||
|
};
|
||||||
|
match argument.as_str() {
|
||||||
|
"--endpoint" => options.endpoint = Some(value.trim_end_matches('/').to_owned()),
|
||||||
|
"--model" => options.model = Some(value),
|
||||||
|
"--case" => options.case = Some(value),
|
||||||
|
"--attempts" => {
|
||||||
|
options.attempts = value
|
||||||
|
.parse::<u32>()
|
||||||
|
.ok()
|
||||||
|
.filter(|attempts| *attempts > 0)
|
||||||
|
.ok_or_else(|| "--attempts must be a positive integer".to_owned())?;
|
||||||
|
}
|
||||||
|
_ => unreachable!(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(options)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn request(agent: &ureq::Agent, endpoint: &str, payload: &Value) -> Result<String, String> {
|
||||||
|
let mut response = agent
|
||||||
|
.post(&format!("{endpoint}/v1/chat/completions"))
|
||||||
|
.header("Content-Type", "application/json")
|
||||||
|
.send(payload.to_string().as_bytes())
|
||||||
|
.map_err(|error| format!("could not call the local endpoint: {error}"))?;
|
||||||
|
let body = response
|
||||||
|
.body_mut()
|
||||||
|
.read_to_string()
|
||||||
|
.map_err(|error| format!("could not read the local endpoint response: {error}"))?;
|
||||||
|
let value: Value = serde_json::from_str(&body)
|
||||||
|
.map_err(|error| format!("local endpoint returned invalid JSON: {error}"))?;
|
||||||
|
value
|
||||||
|
.pointer("/choices/0/message/content")
|
||||||
|
.and_then(Value::as_str)
|
||||||
|
.map(str::to_owned)
|
||||||
|
.ok_or_else(|| format!("local endpoint response has no assistant content: {body}"))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn validate(case: &Case, content: &str) -> Result<String, String> {
|
||||||
|
let lines = extract_lines(content);
|
||||||
|
if lines.is_empty() {
|
||||||
|
return Err("the model emitted no complete fenced A2UI JSONL messages".into());
|
||||||
|
}
|
||||||
|
let mut store = Store::default();
|
||||||
|
let mut errors = Vec::new();
|
||||||
|
for (index, line) in lines.iter().enumerate() {
|
||||||
|
if let Err(error) = &line.value {
|
||||||
|
errors.push(format!("line {} is invalid JSON: {error}", index + 1));
|
||||||
|
} else if let Err(error) = store.apply_raw(&line.raw, 1) {
|
||||||
|
errors.push(format!(
|
||||||
|
"line {} failed catalog validation: {error}",
|
||||||
|
index + 1
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !errors.is_empty() {
|
||||||
|
return Err(errors.join("; "));
|
||||||
|
}
|
||||||
|
let surfaces = store.surfaces().collect::<Vec<_>>();
|
||||||
|
let mut reachable_components = std::collections::BTreeSet::new();
|
||||||
|
for surface in &surfaces {
|
||||||
|
reachable_components.extend(
|
||||||
|
validate_surface_composition(surface)
|
||||||
|
.map_err(|error| format!("surface `{}` is not composable: {error}", surface.id))?,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let components = surfaces
|
||||||
|
.iter()
|
||||||
|
.flat_map(|surface| surface.components.values())
|
||||||
|
.filter_map(Value::as_object)
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
for expected in case.components {
|
||||||
|
if !reachable_components.contains(*expected) {
|
||||||
|
return Err(format!(
|
||||||
|
"valid surface did not compose expected {expected} component into its root tree"
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if let Some(expected) = case.chart_type
|
||||||
|
&& !components.iter().any(|component| {
|
||||||
|
component.get("component").and_then(Value::as_str) == Some("Chart")
|
||||||
|
&& component.get("chartType").and_then(Value::as_str) == Some(expected)
|
||||||
|
})
|
||||||
|
{
|
||||||
|
return Err(format!("valid surface omitted expected {expected} chart"));
|
||||||
|
}
|
||||||
|
Ok(format!(
|
||||||
|
"{} protocol message(s), {} surface(s), {} component(s)",
|
||||||
|
lines.len(),
|
||||||
|
surfaces.len(),
|
||||||
|
components.len()
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn bounded(content: &str) -> &str {
|
||||||
|
content
|
||||||
|
.char_indices()
|
||||||
|
.nth(8_000)
|
||||||
|
.map_or(content, |(end, _)| &content[..end])
|
||||||
|
}
|
||||||
145
src/app.rs
145
src/app.rs
@@ -25,7 +25,7 @@ use crate::settings::{
|
|||||||
ReasoningMode, RuntimePreferences, SpeculativePreferences, SsdPreferences, SteeringPreferences,
|
ReasoningMode, RuntimePreferences, SpeculativePreferences, SsdPreferences, SteeringPreferences,
|
||||||
StreamingCacheBudget,
|
StreamingCacheBudget,
|
||||||
};
|
};
|
||||||
use iced::widget::{markdown, scrollable};
|
use iced::widget::{markdown, scrollable, text_editor};
|
||||||
use iced::{Size, Subscription, Task, keyboard, mouse, window};
|
use iced::{Size, Subscription, Task, keyboard, mouse, window};
|
||||||
use rfd::AsyncFileDialog;
|
use rfd::AsyncFileDialog;
|
||||||
use std::collections::{HashMap, HashSet, VecDeque};
|
use std::collections::{HashMap, HashSet, VecDeque};
|
||||||
@@ -78,6 +78,15 @@ pub(crate) struct App {
|
|||||||
pub(super) composer: String,
|
pub(super) composer: String,
|
||||||
pub(super) queued_inputs: VecDeque<String>,
|
pub(super) queued_inputs: VecDeque<String>,
|
||||||
pub(super) conversation: Vec<ChatMessage>,
|
pub(super) conversation: Vec<ChatMessage>,
|
||||||
|
pub(super) a2ui: crate::a2ui::Store,
|
||||||
|
pub(super) a2ui_tabs: HashMap<(String, String), usize>,
|
||||||
|
pub(super) a2ui_modals: HashSet<(String, String)>,
|
||||||
|
pub(super) a2ui_editors: HashMap<(String, String, String), text_editor::Content>,
|
||||||
|
pub(super) a2ui_markdown: HashMap<(String, String, String), markdown::Content>,
|
||||||
|
pub(super) a2ui_choice_filters: HashMap<(String, String, String), String>,
|
||||||
|
pub(super) a2ui_images: HashMap<String, iced::widget::image::Handle>,
|
||||||
|
pub(super) a2ui_image_requests: HashSet<String>,
|
||||||
|
pub(super) a2ui_image_loading: bool,
|
||||||
pub(super) generating: bool,
|
pub(super) generating: bool,
|
||||||
pub(super) context_used: u32,
|
pub(super) context_used: u32,
|
||||||
pub(super) context_limit: u32,
|
pub(super) context_limit: u32,
|
||||||
@@ -163,6 +172,7 @@ pub(crate) enum Message {
|
|||||||
PreferenceLegacyMtpChanged(bool),
|
PreferenceLegacyMtpChanged(bool),
|
||||||
PreferenceDsparkChanged(bool),
|
PreferenceDsparkChanged(bool),
|
||||||
PreferenceTimeoutChanged(String),
|
PreferenceTimeoutChanged(String),
|
||||||
|
PreferenceA2uiChanged(bool),
|
||||||
PreferenceEndpointPortChanged(String),
|
PreferenceEndpointPortChanged(String),
|
||||||
PreferenceEndpointEnabledChanged(bool),
|
PreferenceEndpointEnabledChanged(bool),
|
||||||
PreferenceEndpointCorsChanged(bool),
|
PreferenceEndpointCorsChanged(bool),
|
||||||
@@ -199,6 +209,14 @@ pub(crate) enum Message {
|
|||||||
PreferenceKvMinTokensChanged(String),
|
PreferenceKvMinTokensChanged(String),
|
||||||
PreferenceKvColdMaxChanged(String),
|
PreferenceKvColdMaxChanged(String),
|
||||||
PreferenceKvContinuedIntervalChanged(String),
|
PreferenceKvContinuedIntervalChanged(String),
|
||||||
|
A2uiDataChanged(String, String, serde_json::Value),
|
||||||
|
A2uiEditorAction(String, String, String, text_editor::Action),
|
||||||
|
A2uiChoiceFilterChanged(String, String, String, String),
|
||||||
|
A2uiAction(String, String, Option<String>),
|
||||||
|
A2uiSelectTab(String, String, usize),
|
||||||
|
A2uiToggleModal(String, String),
|
||||||
|
A2uiImageLoaded(String, Result<Vec<u8>, String>),
|
||||||
|
A2uiPlayMedia(String, String, bool),
|
||||||
ResetPreferences,
|
ResetPreferences,
|
||||||
SavePreferences,
|
SavePreferences,
|
||||||
DownloadArtifact(ManagedArtifactId),
|
DownloadArtifact(ManagedArtifactId),
|
||||||
@@ -309,6 +327,15 @@ impl App {
|
|||||||
composer: String::new(),
|
composer: String::new(),
|
||||||
queued_inputs: VecDeque::new(),
|
queued_inputs: VecDeque::new(),
|
||||||
conversation: Vec::new(),
|
conversation: Vec::new(),
|
||||||
|
a2ui: crate::a2ui::Store::default(),
|
||||||
|
a2ui_tabs: HashMap::new(),
|
||||||
|
a2ui_modals: HashSet::new(),
|
||||||
|
a2ui_editors: HashMap::new(),
|
||||||
|
a2ui_markdown: HashMap::new(),
|
||||||
|
a2ui_choice_filters: HashMap::new(),
|
||||||
|
a2ui_images: HashMap::new(),
|
||||||
|
a2ui_image_requests: HashSet::new(),
|
||||||
|
a2ui_image_loading: false,
|
||||||
generating: false,
|
generating: false,
|
||||||
context_used: 0,
|
context_used: 0,
|
||||||
context_limit,
|
context_limit,
|
||||||
@@ -415,6 +442,15 @@ impl App {
|
|||||||
composer: String::new(),
|
composer: String::new(),
|
||||||
queued_inputs: VecDeque::new(),
|
queued_inputs: VecDeque::new(),
|
||||||
conversation: Vec::new(),
|
conversation: Vec::new(),
|
||||||
|
a2ui: crate::a2ui::Store::default(),
|
||||||
|
a2ui_tabs: HashMap::new(),
|
||||||
|
a2ui_modals: HashSet::new(),
|
||||||
|
a2ui_editors: HashMap::new(),
|
||||||
|
a2ui_markdown: HashMap::new(),
|
||||||
|
a2ui_choice_filters: HashMap::new(),
|
||||||
|
a2ui_images: HashMap::new(),
|
||||||
|
a2ui_image_requests: HashSet::new(),
|
||||||
|
a2ui_image_loading: false,
|
||||||
generating: false,
|
generating: false,
|
||||||
context_used: 0,
|
context_used: 0,
|
||||||
context_limit,
|
context_limit,
|
||||||
@@ -593,6 +629,10 @@ impl App {
|
|||||||
self.preference_draft.idle_timeout_minutes = value;
|
self.preference_draft.idle_timeout_minutes = value;
|
||||||
self.preference_error = None;
|
self.preference_error = None;
|
||||||
}
|
}
|
||||||
|
Message::PreferenceA2uiChanged(enabled) => {
|
||||||
|
self.preference_draft.a2ui_enabled = enabled;
|
||||||
|
self.preference_error = None;
|
||||||
|
}
|
||||||
Message::PreferenceEndpointPortChanged(value) => {
|
Message::PreferenceEndpointPortChanged(value) => {
|
||||||
self.preference_draft.endpoint_port = value;
|
self.preference_draft.endpoint_port = value;
|
||||||
self.preference_error = None;
|
self.preference_error = None;
|
||||||
@@ -802,6 +842,71 @@ impl App {
|
|||||||
}
|
}
|
||||||
Message::DownloadProgressTick => self.update_download_progress(),
|
Message::DownloadProgressTick => self.update_download_progress(),
|
||||||
Message::ComposerChanged(value) => self.composer = value,
|
Message::ComposerChanged(value) => self.composer = value,
|
||||||
|
Message::A2uiDataChanged(surface_id, path, value) => {
|
||||||
|
return self.change_a2ui_data(surface_id, path, value);
|
||||||
|
}
|
||||||
|
Message::A2uiEditorAction(surface_id, component_id, path, action) => {
|
||||||
|
let key = (surface_id.clone(), component_id, path.clone());
|
||||||
|
let Some(editor) = self.a2ui_editors.get_mut(&key) else {
|
||||||
|
return Task::none();
|
||||||
|
};
|
||||||
|
editor.perform(action);
|
||||||
|
let value = serde_json::Value::String(editor.text());
|
||||||
|
return self.change_a2ui_data(surface_id, path, value);
|
||||||
|
}
|
||||||
|
Message::A2uiChoiceFilterChanged(surface_id, component_id, context_path, value) => {
|
||||||
|
self.a2ui_choice_filters
|
||||||
|
.insert((surface_id, component_id, context_path), value);
|
||||||
|
}
|
||||||
|
Message::A2uiAction(surface_id, component_id, context_path) => {
|
||||||
|
match self
|
||||||
|
.a2ui
|
||||||
|
.action(&surface_id, &component_id, context_path.as_deref())
|
||||||
|
{
|
||||||
|
Ok(action) => {
|
||||||
|
self.composer = format!(
|
||||||
|
"A2UI client event:\n{}\nA2UI client metadata:\n{}",
|
||||||
|
serde_json::to_string(&action).unwrap_or_default(),
|
||||||
|
self.a2ui.client_metadata()
|
||||||
|
);
|
||||||
|
self.start_generation();
|
||||||
|
return scroll_chat_to_end();
|
||||||
|
}
|
||||||
|
Err(error) => self.error = Some(error),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Message::A2uiSelectTab(surface_id, component_id, index) => {
|
||||||
|
self.a2ui_tabs.insert((surface_id, component_id), index);
|
||||||
|
}
|
||||||
|
Message::A2uiToggleModal(surface_id, component_id) => {
|
||||||
|
let key = (surface_id, component_id);
|
||||||
|
if !self.a2ui_modals.remove(&key) {
|
||||||
|
self.a2ui_modals.insert(key);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Message::A2uiImageLoaded(url, result) => {
|
||||||
|
self.a2ui_image_loading = false;
|
||||||
|
match result {
|
||||||
|
Ok(bytes) => {
|
||||||
|
self.a2ui_images
|
||||||
|
.insert(url, iced::widget::image::Handle::from_bytes(bytes));
|
||||||
|
}
|
||||||
|
Err(error) => {
|
||||||
|
self.error = Some(format!("Could not load an A2UI image: {error}"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return self.load_next_a2ui_image();
|
||||||
|
}
|
||||||
|
Message::A2uiPlayMedia(url, title, video) => {
|
||||||
|
#[cfg(target_os = "macos")]
|
||||||
|
if let Err(error) = crate::native_media::open(&url, &title, video) {
|
||||||
|
self.error = Some(format!("Could not play media: {error}"));
|
||||||
|
}
|
||||||
|
#[cfg(not(target_os = "macos"))]
|
||||||
|
if let Err(error) = std::process::Command::new("open").arg(url).spawn() {
|
||||||
|
self.error = Some(format!("Could not open media: {error}"));
|
||||||
|
}
|
||||||
|
}
|
||||||
Message::ToggleReasoning(index) => {
|
Message::ToggleReasoning(index) => {
|
||||||
if let Some(message) = self.conversation.get_mut(index)
|
if let Some(message) = self.conversation.get_mut(index)
|
||||||
&& message.reasoning.is_some()
|
&& message.reasoning.is_some()
|
||||||
@@ -869,9 +974,12 @@ impl App {
|
|||||||
Message::GenerationTick => {
|
Message::GenerationTick => {
|
||||||
#[cfg(target_os = "macos")]
|
#[cfg(target_os = "macos")]
|
||||||
self.poll_titling();
|
self.poll_titling();
|
||||||
if self.poll_generation() {
|
let changed = self.poll_generation();
|
||||||
return scroll_chat_to_end();
|
let images = self.load_next_a2ui_image();
|
||||||
|
if changed {
|
||||||
|
return Task::batch([scroll_chat_to_end(), images]);
|
||||||
}
|
}
|
||||||
|
return images;
|
||||||
}
|
}
|
||||||
Message::ChooseProjectFolder => {
|
Message::ChooseProjectFolder => {
|
||||||
self.choosing_folder = true;
|
self.choosing_folder = true;
|
||||||
@@ -948,6 +1056,7 @@ impl App {
|
|||||||
self.selected_project = None;
|
self.selected_project = None;
|
||||||
self.selected_session = None;
|
self.selected_session = None;
|
||||||
self.conversation.clear();
|
self.conversation.clear();
|
||||||
|
self.clear_a2ui();
|
||||||
self.composer.clear();
|
self.composer.clear();
|
||||||
self.queued_inputs.clear();
|
self.queued_inputs.clear();
|
||||||
self.system_prompt_seen_at = 0;
|
self.system_prompt_seen_at = 0;
|
||||||
@@ -1102,9 +1211,26 @@ impl App {
|
|||||||
let Some(database) = &mut self.database else {
|
let Some(database) = &mut self.database else {
|
||||||
return Task::none();
|
return Task::none();
|
||||||
};
|
};
|
||||||
match database.load_messages(session_id) {
|
let loaded = database.load_messages(session_id).and_then(|messages| {
|
||||||
Ok(messages) => {
|
database
|
||||||
|
.load_a2ui_messages(session_id)
|
||||||
|
.map(|a2ui| (messages, a2ui))
|
||||||
|
});
|
||||||
|
match loaded {
|
||||||
|
Ok((messages, a2ui)) => {
|
||||||
self.conversation = messages.into_iter().map(ChatMessage::from).collect();
|
self.conversation = messages.into_iter().map(ChatMessage::from).collect();
|
||||||
|
self.clear_a2ui();
|
||||||
|
for message in a2ui {
|
||||||
|
if let Err(error) =
|
||||||
|
self.a2ui.apply_raw(&message.json, message.message_id)
|
||||||
|
{
|
||||||
|
self.error = Some(format!(
|
||||||
|
"Could not restore A2UI message {}: {error}",
|
||||||
|
message.id
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
self.sync_a2ui_renderer_state();
|
||||||
self.composer.clear();
|
self.composer.clear();
|
||||||
self.remember_project(project_id);
|
self.remember_project(project_id);
|
||||||
self.selected_session = Some(session_id);
|
self.selected_session = Some(session_id);
|
||||||
@@ -1119,7 +1245,7 @@ impl App {
|
|||||||
};
|
};
|
||||||
self.tokens_per_second = tokens_per_second;
|
self.tokens_per_second = tokens_per_second;
|
||||||
self.error = None;
|
self.error = None;
|
||||||
return scroll_chat_to_end();
|
return Task::batch([scroll_chat_to_end(), self.load_next_a2ui_image()]);
|
||||||
}
|
}
|
||||||
Err(error) => {
|
Err(error) => {
|
||||||
self.error = Some(format!("Could not load the chat session: {error}"));
|
self.error = Some(format!("Could not load the chat session: {error}"));
|
||||||
@@ -1150,6 +1276,7 @@ impl App {
|
|||||||
if self.selected_session == Some(session_id) {
|
if self.selected_session == Some(session_id) {
|
||||||
self.selected_session = None;
|
self.selected_session = None;
|
||||||
self.conversation.clear();
|
self.conversation.clear();
|
||||||
|
self.clear_a2ui();
|
||||||
self.composer.clear();
|
self.composer.clear();
|
||||||
self.queued_inputs.clear();
|
self.queued_inputs.clear();
|
||||||
self.system_prompt_seen_at = 0;
|
self.system_prompt_seen_at = 0;
|
||||||
@@ -1424,7 +1551,7 @@ fn models_path() -> PathBuf {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// The settings file, beside the project database.
|
/// The settings file, beside the project database.
|
||||||
fn config_path() -> PathBuf {
|
pub(crate) fn config_path() -> PathBuf {
|
||||||
application_support_path().join("config.yaml")
|
application_support_path().join("config.yaml")
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1736,6 +1863,10 @@ mod tests {
|
|||||||
reasoning_open: true,
|
reasoning_open: true,
|
||||||
content: String::new(),
|
content: String::new(),
|
||||||
markdown: markdown::Content::new(),
|
markdown: markdown::Content::new(),
|
||||||
|
a2ui_lines_processed: 0,
|
||||||
|
a2ui_errors: Vec::new(),
|
||||||
|
a2ui_replies: Vec::new(),
|
||||||
|
a2ui_open_urls: Vec::new(),
|
||||||
};
|
};
|
||||||
message.append(true, "working it out");
|
message.append(true, "working it out");
|
||||||
message.append(false, "**final answer**");
|
message.append(false, "**final answer**");
|
||||||
|
|||||||
@@ -60,6 +60,10 @@ pub(crate) struct ChatMessage {
|
|||||||
pub(super) reasoning_open: bool,
|
pub(super) reasoning_open: bool,
|
||||||
pub(super) content: String,
|
pub(super) content: String,
|
||||||
pub(super) markdown: markdown::Content,
|
pub(super) markdown: markdown::Content,
|
||||||
|
pub(super) a2ui_lines_processed: usize,
|
||||||
|
pub(super) a2ui_errors: Vec<String>,
|
||||||
|
pub(super) a2ui_replies: Vec<serde_json::Value>,
|
||||||
|
pub(super) a2ui_open_urls: Vec<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl ChatMessage {
|
impl ChatMessage {
|
||||||
@@ -75,10 +79,11 @@ impl ChatMessage {
|
|||||||
pub(super) fn refresh_markdown(&mut self) {
|
pub(super) fn refresh_markdown(&mut self) {
|
||||||
if !self.user && !self.tool {
|
if !self.user && !self.tool {
|
||||||
let visible = crate::agent::visible_content(&self.content);
|
let visible = crate::agent::visible_content(&self.content);
|
||||||
|
let visible = crate::a2ui::transcript_fallback(visible);
|
||||||
let content = if self.reasoning.is_some() {
|
let content = if self.reasoning.is_some() {
|
||||||
visible.trim_start()
|
visible.trim_start()
|
||||||
} else {
|
} else {
|
||||||
visible
|
&visible
|
||||||
};
|
};
|
||||||
self.markdown = markdown::Content::parse(content);
|
self.markdown = markdown::Content::parse(content);
|
||||||
}
|
}
|
||||||
@@ -99,12 +104,59 @@ impl From<StoredMessage> for ChatMessage {
|
|||||||
reasoning_open: false,
|
reasoning_open: false,
|
||||||
content: message.content,
|
content: message.content,
|
||||||
markdown: iced::widget::markdown::Content::new(),
|
markdown: iced::widget::markdown::Content::new(),
|
||||||
|
a2ui_lines_processed: 0,
|
||||||
|
a2ui_errors: Vec::new(),
|
||||||
|
a2ui_replies: Vec::new(),
|
||||||
|
a2ui_open_urls: Vec::new(),
|
||||||
};
|
};
|
||||||
message.refresh_markdown();
|
message.refresh_markdown();
|
||||||
message
|
message
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn sync_a2ui_message(
|
||||||
|
store: &mut crate::a2ui::Store,
|
||||||
|
database: &mut Option<Database>,
|
||||||
|
session_id: Option<i32>,
|
||||||
|
message: &mut ChatMessage,
|
||||||
|
) {
|
||||||
|
let lines = crate::a2ui::extract_lines(&message.content);
|
||||||
|
for (index, line) in lines.iter().enumerate().skip(message.a2ui_lines_processed) {
|
||||||
|
let applied = match &line.value {
|
||||||
|
Ok(value) => store.apply(value.clone(), line.raw.clone(), message.id),
|
||||||
|
Err(error) => Err(error.clone()),
|
||||||
|
};
|
||||||
|
match applied {
|
||||||
|
Ok(applied) => {
|
||||||
|
if let Some(reply) = applied.reply {
|
||||||
|
message.a2ui_replies.push(reply);
|
||||||
|
}
|
||||||
|
if let Some(url) = applied.open_url {
|
||||||
|
message.a2ui_open_urls.push(url);
|
||||||
|
}
|
||||||
|
if let (Some(session_id), Some(database)) = (session_id, database.as_mut()) {
|
||||||
|
for raw in applied.raws {
|
||||||
|
if let Err(error) =
|
||||||
|
database.insert_a2ui_message(session_id, message.id, &raw)
|
||||||
|
{
|
||||||
|
message.a2ui_errors.push(format!(
|
||||||
|
"line {} could not be persisted: {error}",
|
||||||
|
index + 1
|
||||||
|
));
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(error) => message
|
||||||
|
.a2ui_errors
|
||||||
|
.push(format!("line {}: {error}", index + 1)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
message.a2ui_lines_processed = lines.len();
|
||||||
|
message.refresh_markdown();
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(target_os = "macos")]
|
#[cfg(target_os = "macos")]
|
||||||
fn chat_turn(message: &ChatMessage) -> ChatTurn {
|
fn chat_turn(message: &ChatMessage) -> ChatTurn {
|
||||||
ChatTurn {
|
ChatTurn {
|
||||||
@@ -168,6 +220,15 @@ fn has_chat_after_last_compaction(messages: &[ChatMessage]) -> bool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl App {
|
impl App {
|
||||||
|
fn chat_system_prompt(&self, model: ModelChoice, prompt: &str) -> String {
|
||||||
|
let mut prompt = crate::agent::system_prompt(model, prompt);
|
||||||
|
if self.config.a2ui_enabled {
|
||||||
|
prompt.push_str("\n\n");
|
||||||
|
prompt.push_str(crate::a2ui::SYSTEM_PROMPT);
|
||||||
|
}
|
||||||
|
prompt
|
||||||
|
}
|
||||||
|
|
||||||
pub(super) fn can_compact_session(&self, session_id: i32) -> bool {
|
pub(super) fn can_compact_session(&self, session_id: i32) -> bool {
|
||||||
!self.generating
|
!self.generating
|
||||||
&& self.selected_session == Some(session_id)
|
&& self.selected_session == Some(session_id)
|
||||||
@@ -238,13 +299,22 @@ impl App {
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
effective.turn.system_prompt =
|
effective.turn.system_prompt =
|
||||||
crate::agent::system_prompt(model, &effective.turn.system_prompt);
|
self.chat_system_prompt(model, &effective.turn.system_prompt);
|
||||||
effective.turn.system_prompt = crate::compaction::summary_system_prompt(
|
effective.turn.system_prompt = crate::compaction::summary_system_prompt(
|
||||||
&effective.turn.system_prompt,
|
&effective.turn.system_prompt,
|
||||||
self.compaction_summary(),
|
self.compaction_summary(),
|
||||||
);
|
);
|
||||||
let assistant_reasoning = effective.turn.reasoning_mode != ReasoningMode::Direct;
|
let assistant_reasoning = effective.turn.reasoning_mode != ReasoningMode::Direct;
|
||||||
#[cfg(target_os = "macos")]
|
#[cfg(target_os = "macos")]
|
||||||
|
let model_prompt = if self.config.a2ui_enabled {
|
||||||
|
format!(
|
||||||
|
"{prompt}\n\nA2UI client metadata:\n{}",
|
||||||
|
self.a2ui.client_metadata()
|
||||||
|
)
|
||||||
|
} else {
|
||||||
|
prompt.clone()
|
||||||
|
};
|
||||||
|
#[cfg(target_os = "macos")]
|
||||||
let opening_turn = self.selected_session.is_none();
|
let opening_turn = self.selected_session.is_none();
|
||||||
#[cfg(target_os = "macos")]
|
#[cfg(target_os = "macos")]
|
||||||
let mut injected_system = Vec::new();
|
let mut injected_system = Vec::new();
|
||||||
@@ -282,7 +352,7 @@ impl App {
|
|||||||
skip_previous_eos: false,
|
skip_previous_eos: false,
|
||||||
reasoning: None,
|
reasoning: None,
|
||||||
reasoning_complete: true,
|
reasoning_complete: true,
|
||||||
content: prompt.clone(),
|
content: model_prompt,
|
||||||
});
|
});
|
||||||
|
|
||||||
#[cfg(target_os = "macos")]
|
#[cfg(target_os = "macos")]
|
||||||
@@ -374,6 +444,9 @@ impl App {
|
|||||||
|
|
||||||
fn system_prompt_reminders(&self, model: ModelChoice) -> Vec<String> {
|
fn system_prompt_reminders(&self, model: ModelChoice) -> Vec<String> {
|
||||||
let mut reminders = vec![crate::agent::system_prompt_reminder(model)];
|
let mut reminders = vec![crate::agent::system_prompt_reminder(model)];
|
||||||
|
if self.config.a2ui_enabled {
|
||||||
|
reminders.push(crate::a2ui::SYSTEM_PROMPT.to_owned());
|
||||||
|
}
|
||||||
if !self.config.generation.system_prompt.trim().is_empty() {
|
if !self.config.generation.system_prompt.trim().is_empty() {
|
||||||
reminders.push(self.config.generation.system_prompt.clone());
|
reminders.push(self.config.generation.system_prompt.clone());
|
||||||
}
|
}
|
||||||
@@ -475,6 +548,10 @@ impl App {
|
|||||||
#[cfg(target_os = "macos")]
|
#[cfg(target_os = "macos")]
|
||||||
let mut start_queued = false;
|
let mut start_queued = false;
|
||||||
#[cfg(target_os = "macos")]
|
#[cfg(target_os = "macos")]
|
||||||
|
let mut a2ui_feedback = None;
|
||||||
|
#[cfg(target_os = "macos")]
|
||||||
|
let mut a2ui_changed = false;
|
||||||
|
#[cfg(target_os = "macos")]
|
||||||
loop {
|
loop {
|
||||||
match active.events.try_recv() {
|
match active.events.try_recv() {
|
||||||
Ok(GenerationEvent::Loading) => {}
|
Ok(GenerationEvent::Loading) => {}
|
||||||
@@ -500,6 +577,18 @@ impl App {
|
|||||||
}
|
}
|
||||||
transcript_changed = true;
|
transcript_changed = true;
|
||||||
}
|
}
|
||||||
|
if !reasoning
|
||||||
|
&& self.config.a2ui_enabled
|
||||||
|
&& let Some(message) = self.conversation.last_mut()
|
||||||
|
{
|
||||||
|
sync_a2ui_message(
|
||||||
|
&mut self.a2ui,
|
||||||
|
&mut self.database,
|
||||||
|
self.selected_session,
|
||||||
|
message,
|
||||||
|
);
|
||||||
|
a2ui_changed = true;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
Ok(GenerationEvent::Context {
|
Ok(GenerationEvent::Context {
|
||||||
used,
|
used,
|
||||||
@@ -530,6 +619,69 @@ impl App {
|
|||||||
!self.queued_inputs.is_empty() || self.manual_compaction_queued;
|
!self.queued_inputs.is_empty() || self.manual_compaction_queued;
|
||||||
}
|
}
|
||||||
Ok(_) => {
|
Ok(_) => {
|
||||||
|
let (validation_errors, replies, open_urls, error_surface_id) = self
|
||||||
|
.conversation
|
||||||
|
.last_mut()
|
||||||
|
.map(|message| {
|
||||||
|
let surface_id = crate::a2ui::extract_lines(&message.content)
|
||||||
|
.into_iter()
|
||||||
|
.rev()
|
||||||
|
.filter_map(|line| line.value.ok())
|
||||||
|
.find_map(|value| {
|
||||||
|
crate::a2ui::message_surface_id(&value)
|
||||||
|
.map(str::to_owned)
|
||||||
|
})
|
||||||
|
.unwrap_or_else(|| "unknown".to_owned());
|
||||||
|
(
|
||||||
|
std::mem::take(&mut message.a2ui_errors),
|
||||||
|
std::mem::take(&mut message.a2ui_replies),
|
||||||
|
std::mem::take(&mut message.a2ui_open_urls),
|
||||||
|
surface_id,
|
||||||
|
)
|
||||||
|
})
|
||||||
|
.unwrap_or_default();
|
||||||
|
for url in open_urls {
|
||||||
|
if let Err(error) =
|
||||||
|
std::process::Command::new("open").arg(url).spawn()
|
||||||
|
{
|
||||||
|
self.error =
|
||||||
|
Some(format!("Could not open the A2UI link: {error}"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !validation_errors.is_empty() {
|
||||||
|
self.generating = false;
|
||||||
|
self.activity = Some("Correcting A2UI…".into());
|
||||||
|
self.tool_cards.clear();
|
||||||
|
a2ui_feedback = Some(
|
||||||
|
serde_json::json!({
|
||||||
|
"version": crate::a2ui::VERSION,
|
||||||
|
"error": {
|
||||||
|
"code": "VALIDATION_FAILED",
|
||||||
|
"surfaceId": error_surface_id,
|
||||||
|
"path": "/",
|
||||||
|
"message": validation_errors.join("; ")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.to_string(),
|
||||||
|
);
|
||||||
|
self.active_generation = None;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
if !replies.is_empty() {
|
||||||
|
self.generating = false;
|
||||||
|
self.activity = Some("Continuing A2UI function call…".into());
|
||||||
|
self.tool_cards.clear();
|
||||||
|
a2ui_feedback = Some(format!(
|
||||||
|
"A2UI client response:\n{}",
|
||||||
|
replies
|
||||||
|
.iter()
|
||||||
|
.map(ToString::to_string)
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join("\n")
|
||||||
|
));
|
||||||
|
self.active_generation = None;
|
||||||
|
break;
|
||||||
|
}
|
||||||
let model = self.config.model;
|
let model = self.config.model;
|
||||||
let content = self
|
let content = self
|
||||||
.conversation
|
.conversation
|
||||||
@@ -586,6 +738,10 @@ impl App {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
#[cfg(target_os = "macos")]
|
#[cfg(target_os = "macos")]
|
||||||
|
if a2ui_changed {
|
||||||
|
self.sync_a2ui_renderer_state();
|
||||||
|
}
|
||||||
|
#[cfg(target_os = "macos")]
|
||||||
if transcript_changed && let Some(message) = self.conversation.last_mut() {
|
if transcript_changed && let Some(message) = self.conversation.last_mut() {
|
||||||
message.refresh_markdown();
|
message.refresh_markdown();
|
||||||
}
|
}
|
||||||
@@ -633,6 +789,14 @@ impl App {
|
|||||||
self.start_next_queued();
|
self.start_next_queued();
|
||||||
}
|
}
|
||||||
#[cfg(target_os = "macos")]
|
#[cfg(target_os = "macos")]
|
||||||
|
if let Some(feedback) = a2ui_feedback
|
||||||
|
&& let Err(error) = self.continue_after_tool_result(&feedback)
|
||||||
|
{
|
||||||
|
self.generating = false;
|
||||||
|
self.activity = Some("Failed".into());
|
||||||
|
self.error = Some(error);
|
||||||
|
}
|
||||||
|
#[cfg(target_os = "macos")]
|
||||||
return transcript_changed;
|
return transcript_changed;
|
||||||
#[cfg(not(target_os = "macos"))]
|
#[cfg(not(target_os = "macos"))]
|
||||||
false
|
false
|
||||||
@@ -698,7 +862,7 @@ impl App {
|
|||||||
&models_path(),
|
&models_path(),
|
||||||
)?;
|
)?;
|
||||||
effective.turn.system_prompt =
|
effective.turn.system_prompt =
|
||||||
crate::agent::system_prompt(model, &effective.turn.system_prompt);
|
self.chat_system_prompt(model, &effective.turn.system_prompt);
|
||||||
effective.turn.system_prompt = crate::compaction::summary_system_prompt(
|
effective.turn.system_prompt = crate::compaction::summary_system_prompt(
|
||||||
&effective.turn.system_prompt,
|
&effective.turn.system_prompt,
|
||||||
self.compaction_summary(),
|
self.compaction_summary(),
|
||||||
@@ -792,7 +956,7 @@ impl App {
|
|||||||
&models_path(),
|
&models_path(),
|
||||||
)?;
|
)?;
|
||||||
effective.turn.system_prompt =
|
effective.turn.system_prompt =
|
||||||
crate::agent::system_prompt(model, &effective.turn.system_prompt);
|
self.chat_system_prompt(model, &effective.turn.system_prompt);
|
||||||
effective.turn.system_prompt = crate::compaction::summary_system_prompt(
|
effective.turn.system_prompt = crate::compaction::summary_system_prompt(
|
||||||
&effective.turn.system_prompt,
|
&effective.turn.system_prompt,
|
||||||
self.compaction_summary(),
|
self.compaction_summary(),
|
||||||
@@ -941,7 +1105,7 @@ impl App {
|
|||||||
&models_path(),
|
&models_path(),
|
||||||
)?;
|
)?;
|
||||||
effective.turn.system_prompt =
|
effective.turn.system_prompt =
|
||||||
crate::agent::system_prompt(model, &effective.turn.system_prompt);
|
self.chat_system_prompt(model, &effective.turn.system_prompt);
|
||||||
let rebuild_system_prompt = effective.turn.system_prompt.clone();
|
let rebuild_system_prompt = effective.turn.system_prompt.clone();
|
||||||
effective.turn.system_prompt = crate::compaction::summary_system_prompt(
|
effective.turn.system_prompt = crate::compaction::summary_system_prompt(
|
||||||
&effective.turn.system_prompt,
|
&effective.turn.system_prompt,
|
||||||
@@ -1364,6 +1528,10 @@ mod tests {
|
|||||||
content: "### Core / Setup\n\n| File | Lines |\n|---|---:|\n| `src/app.rs` | **1,750** |\n| `src/engine.rs` | 2,400 |\n\n### Summary\n\nDone."
|
content: "### Core / Setup\n\n| File | Lines |\n|---|---:|\n| `src/app.rs` | **1,750** |\n| `src/engine.rs` | 2,400 |\n\n### Summary\n\nDone."
|
||||||
.to_owned(),
|
.to_owned(),
|
||||||
markdown: iced::widget::markdown::Content::new(),
|
markdown: iced::widget::markdown::Content::new(),
|
||||||
|
a2ui_lines_processed: 0,
|
||||||
|
a2ui_errors: Vec::new(),
|
||||||
|
a2ui_replies: Vec::new(),
|
||||||
|
a2ui_open_urls: Vec::new(),
|
||||||
};
|
};
|
||||||
|
|
||||||
message.refresh_markdown();
|
message.refresh_markdown();
|
||||||
@@ -1394,6 +1562,10 @@ mod tests {
|
|||||||
reasoning_open: false,
|
reasoning_open: false,
|
||||||
content: format!("message {id}"),
|
content: format!("message {id}"),
|
||||||
markdown: iced::widget::markdown::Content::new(),
|
markdown: iced::widget::markdown::Content::new(),
|
||||||
|
a2ui_lines_processed: 0,
|
||||||
|
a2ui_errors: Vec::new(),
|
||||||
|
a2ui_replies: Vec::new(),
|
||||||
|
a2ui_open_urls: Vec::new(),
|
||||||
};
|
};
|
||||||
let history = vec![
|
let history = vec![
|
||||||
message(1, false, None),
|
message(1, false, None),
|
||||||
@@ -1426,6 +1598,10 @@ mod tests {
|
|||||||
reasoning_open: false,
|
reasoning_open: false,
|
||||||
content: format!("message {id}"),
|
content: format!("message {id}"),
|
||||||
markdown: iced::widget::markdown::Content::new(),
|
markdown: iced::widget::markdown::Content::new(),
|
||||||
|
a2ui_lines_processed: 0,
|
||||||
|
a2ui_errors: Vec::new(),
|
||||||
|
a2ui_replies: Vec::new(),
|
||||||
|
a2ui_open_urls: Vec::new(),
|
||||||
};
|
};
|
||||||
let mut history = vec![
|
let mut history = vec![
|
||||||
message(1, true, false, false, false),
|
message(1, true, false, false, false),
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ pub(super) struct PreferenceDraft {
|
|||||||
pub(super) legacy_mtp_enabled: bool,
|
pub(super) legacy_mtp_enabled: bool,
|
||||||
pub(super) dspark_enabled: bool,
|
pub(super) dspark_enabled: bool,
|
||||||
pub(super) idle_timeout_minutes: String,
|
pub(super) idle_timeout_minutes: String,
|
||||||
|
pub(super) a2ui_enabled: bool,
|
||||||
pub(super) endpoint_port: String,
|
pub(super) endpoint_port: String,
|
||||||
pub(super) endpoint_enabled: bool,
|
pub(super) endpoint_enabled: bool,
|
||||||
pub(super) endpoint_cors: bool,
|
pub(super) endpoint_cors: bool,
|
||||||
@@ -55,6 +56,7 @@ impl PreferenceDraft {
|
|||||||
legacy_mtp_enabled: speculative.legacy_mtp_enabled,
|
legacy_mtp_enabled: speculative.legacy_mtp_enabled,
|
||||||
dspark_enabled: speculative.dspark_enabled,
|
dspark_enabled: speculative.dspark_enabled,
|
||||||
idle_timeout_minutes: config.idle_timeout_minutes.to_string(),
|
idle_timeout_minutes: config.idle_timeout_minutes.to_string(),
|
||||||
|
a2ui_enabled: config.a2ui_enabled,
|
||||||
endpoint_port: config.endpoint.port.to_string(),
|
endpoint_port: config.endpoint.port.to_string(),
|
||||||
endpoint_enabled: config.endpoint.enabled,
|
endpoint_enabled: config.endpoint.enabled,
|
||||||
endpoint_cors: config.endpoint.cors,
|
endpoint_cors: config.endpoint.cors,
|
||||||
@@ -350,6 +352,7 @@ impl App {
|
|||||||
let config = Config {
|
let config = Config {
|
||||||
model: self.preference_draft.model,
|
model: self.preference_draft.model,
|
||||||
idle_timeout_minutes,
|
idle_timeout_minutes,
|
||||||
|
a2ui_enabled: self.preference_draft.a2ui_enabled,
|
||||||
endpoint: EndpointConfig {
|
endpoint: EndpointConfig {
|
||||||
port: i32::from(endpoint_port),
|
port: i32::from(endpoint_port),
|
||||||
enabled: self.preference_draft.endpoint_enabled,
|
enabled: self.preference_draft.endpoint_enabled,
|
||||||
|
|||||||
@@ -1,6 +1,79 @@
|
|||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
impl App {
|
impl App {
|
||||||
|
pub(super) fn clear_a2ui(&mut self) {
|
||||||
|
self.a2ui.clear();
|
||||||
|
self.a2ui_tabs.clear();
|
||||||
|
self.a2ui_modals.clear();
|
||||||
|
self.a2ui_editors.clear();
|
||||||
|
self.a2ui_markdown.clear();
|
||||||
|
self.a2ui_choice_filters.clear();
|
||||||
|
self.a2ui_images.clear();
|
||||||
|
self.a2ui_image_requests.clear();
|
||||||
|
self.a2ui_image_loading = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn load_next_a2ui_image(&mut self) -> Task<Message> {
|
||||||
|
if self.a2ui_image_loading {
|
||||||
|
return Task::none();
|
||||||
|
}
|
||||||
|
let Some(url) = self
|
||||||
|
.a2ui
|
||||||
|
.image_urls()
|
||||||
|
.find(|url| !self.a2ui_image_requests.contains(url))
|
||||||
|
else {
|
||||||
|
return Task::none();
|
||||||
|
};
|
||||||
|
self.a2ui_image_requests.insert(url.clone());
|
||||||
|
self.a2ui_image_loading = true;
|
||||||
|
let request_url = url.clone();
|
||||||
|
Task::perform(
|
||||||
|
async move {
|
||||||
|
let mut response = ureq::get(&request_url)
|
||||||
|
.call()
|
||||||
|
.map_err(|error| error.to_string())?;
|
||||||
|
if !response
|
||||||
|
.headers()
|
||||||
|
.get("content-type")
|
||||||
|
.and_then(|value| value.to_str().ok())
|
||||||
|
.is_some_and(|value| value.starts_with("image/"))
|
||||||
|
{
|
||||||
|
return Err("the URL did not return an image".to_owned());
|
||||||
|
}
|
||||||
|
response
|
||||||
|
.body_mut()
|
||||||
|
.read_to_vec()
|
||||||
|
.map_err(|error| error.to_string())
|
||||||
|
},
|
||||||
|
move |result| Message::A2uiImageLoaded(url, result),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn change_a2ui_data(
|
||||||
|
&mut self,
|
||||||
|
surface_id: String,
|
||||||
|
path: String,
|
||||||
|
value: serde_json::Value,
|
||||||
|
) -> Task<Message> {
|
||||||
|
let owner = self
|
||||||
|
.a2ui
|
||||||
|
.surface(&surface_id)
|
||||||
|
.map(|surface| surface.owner_message_id);
|
||||||
|
match self.a2ui.local_update(&surface_id, &path, value) {
|
||||||
|
Ok(raw) => {
|
||||||
|
if let (Some(session_id), Some(message_id), Some(database)) =
|
||||||
|
(self.selected_session, owner, &mut self.database)
|
||||||
|
&& let Err(error) = database.insert_a2ui_message(session_id, message_id, &raw)
|
||||||
|
{
|
||||||
|
self.error = Some(format!("Could not save the A2UI edit: {error}"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(error) => self.error = Some(error),
|
||||||
|
}
|
||||||
|
self.sync_a2ui_renderer_state();
|
||||||
|
self.load_next_a2ui_image()
|
||||||
|
}
|
||||||
|
|
||||||
pub(super) fn prepare_project(&mut self, path: PathBuf) {
|
pub(super) fn prepare_project(&mut self, path: PathBuf) {
|
||||||
let Ok(path) = fs::canonicalize(path) else {
|
let Ok(path) = fs::canonicalize(path) else {
|
||||||
self.error = Some("The selected folder is no longer available.".into());
|
self.error = Some("The selected folder is no longer available.".into());
|
||||||
@@ -49,6 +122,7 @@ impl App {
|
|||||||
Ok(project) => {
|
Ok(project) => {
|
||||||
self.remember_project(project.id);
|
self.remember_project(project.id);
|
||||||
self.selected_session = None;
|
self.selected_session = None;
|
||||||
|
self.clear_a2ui();
|
||||||
self.system_prompt_seen_at = 0;
|
self.system_prompt_seen_at = 0;
|
||||||
self.pending_project_path = None;
|
self.pending_project_path = None;
|
||||||
self.project_name_input.clear();
|
self.project_name_input.clear();
|
||||||
@@ -78,6 +152,7 @@ impl App {
|
|||||||
self.remember_project(project_id);
|
self.remember_project(project_id);
|
||||||
self.selected_session = None;
|
self.selected_session = None;
|
||||||
self.conversation.clear();
|
self.conversation.clear();
|
||||||
|
self.clear_a2ui();
|
||||||
self.composer.clear();
|
self.composer.clear();
|
||||||
self.queued_inputs.clear();
|
self.queued_inputs.clear();
|
||||||
self.system_prompt_seen_at = 0;
|
self.system_prompt_seen_at = 0;
|
||||||
@@ -90,6 +165,7 @@ impl App {
|
|||||||
pub(super) fn discard_session(&mut self, project_id: i32) {
|
pub(super) fn discard_session(&mut self, project_id: i32) {
|
||||||
if self.drafts.remove(&project_id).is_some() && self.draft_selected(project_id) {
|
if self.drafts.remove(&project_id).is_some() && self.draft_selected(project_id) {
|
||||||
self.conversation.clear();
|
self.conversation.clear();
|
||||||
|
self.clear_a2ui();
|
||||||
self.composer.clear();
|
self.composer.clear();
|
||||||
self.queued_inputs.clear();
|
self.queued_inputs.clear();
|
||||||
self.system_prompt_seen_at = 0;
|
self.system_prompt_seen_at = 0;
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
|
mod a2ui;
|
||||||
mod chat;
|
mod chat;
|
||||||
mod model_manager;
|
mod model_manager;
|
||||||
mod preferences;
|
mod preferences;
|
||||||
@@ -16,8 +17,9 @@ use crate::model::{
|
|||||||
use crate::settings::{GIB, REASONING_MODES};
|
use crate::settings::{GIB, REASONING_MODES};
|
||||||
use iced::theme::{Palette, palette};
|
use iced::theme::{Palette, palette};
|
||||||
use iced::widget::{
|
use iced::widget::{
|
||||||
Button, Space, Svg, Tooltip, button, checkbox, column, container, markdown, mouse_area, opaque,
|
Button, Space, Svg, Tooltip, button, checkbox, column, container, image, markdown, mouse_area,
|
||||||
pick_list, progress_bar, row, rule, scrollable, stack, svg, text, text_input, tooltip,
|
opaque, pick_list, progress_bar, row, rule, scrollable, slider, stack, svg, text, text_input,
|
||||||
|
tooltip,
|
||||||
};
|
};
|
||||||
use iced::{Alignment, Background, Border, Color, Element, Length, Padding, Theme, window};
|
use iced::{Alignment, Background, Border, Color, Element, Length, Padding, Theme, window};
|
||||||
use std::collections::VecDeque;
|
use std::collections::VecDeque;
|
||||||
@@ -68,6 +70,7 @@ impl App {
|
|||||||
|| self.pending_project_path.is_some()
|
|| self.pending_project_path.is_some()
|
||||||
|| self.session_rename.is_some()
|
|| self.session_rename.is_some()
|
||||||
|| self.menu_session().is_some()
|
|| self.menu_session().is_some()
|
||||||
|
|| !self.a2ui_modals.is_empty()
|
||||||
|| {
|
|| {
|
||||||
#[cfg(target_os = "macos")]
|
#[cfg(target_os = "macos")]
|
||||||
{
|
{
|
||||||
@@ -132,6 +135,8 @@ impl App {
|
|||||||
layers.push(self.rename_dialog(title));
|
layers.push(self.rename_dialog(title));
|
||||||
} else if let Some(session) = self.menu_session() {
|
} else if let Some(session) = self.menu_session() {
|
||||||
layers.push(self.session_menu_panel(session));
|
layers.push(self.session_menu_panel(session));
|
||||||
|
} else if let Some(panel) = self.a2ui_modal_panel() {
|
||||||
|
layers.push(panel);
|
||||||
}
|
}
|
||||||
#[cfg(not(target_os = "macos"))]
|
#[cfg(not(target_os = "macos"))]
|
||||||
if self.preferences_open {
|
if self.preferences_open {
|
||||||
@@ -142,6 +147,8 @@ impl App {
|
|||||||
layers.push(self.rename_dialog(title));
|
layers.push(self.rename_dialog(title));
|
||||||
} else if let Some(session) = self.menu_session() {
|
} else if let Some(session) = self.menu_session() {
|
||||||
layers.push(self.session_menu_panel(session));
|
layers.push(self.session_menu_panel(session));
|
||||||
|
} else if let Some(panel) = self.a2ui_modal_panel() {
|
||||||
|
layers.push(panel);
|
||||||
}
|
}
|
||||||
|
|
||||||
stack(layers)
|
stack(layers)
|
||||||
|
|||||||
1909
src/app/view/a2ui.rs
Normal file
1909
src/app/view/a2ui.rs
Normal file
File diff suppressed because it is too large
Load Diff
@@ -179,6 +179,9 @@ impl App {
|
|||||||
if !cards.is_empty() {
|
if !cards.is_empty() {
|
||||||
body = body.push(tool_cards(cards));
|
body = body.push(tool_cards(cards));
|
||||||
}
|
}
|
||||||
|
if self.a2ui.surfaces_for_message(message.id).next().is_some() {
|
||||||
|
body = body.push(self.a2ui_surfaces(message.id));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
let user = message.user;
|
let user = message.user;
|
||||||
messages = messages.push(
|
messages = messages.push(
|
||||||
|
|||||||
@@ -119,6 +119,12 @@ impl App {
|
|||||||
.spacing(10)
|
.spacing(10)
|
||||||
.align_y(Alignment::Center),
|
.align_y(Alignment::Center),
|
||||||
text("Enter a whole number from 1 to 1440.").size(12),
|
text("Enter a whole number from 1 to 1440.").size(12),
|
||||||
|
hint(
|
||||||
|
checkbox(self.preference_draft.a2ui_enabled)
|
||||||
|
.label("Enable interactive A2UI chat surfaces")
|
||||||
|
.on_toggle(Message::PreferenceA2uiChanged),
|
||||||
|
"Lets the local model build validated native charts, tables, forms and other interactive chat surfaces. Turning it off removes the A2UI catalog from the system prompt.",
|
||||||
|
),
|
||||||
]
|
]
|
||||||
.spacing(10),
|
.spacing(10),
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ use crate::settings::{GenerationPreferences, RuntimePreferences};
|
|||||||
pub struct Config {
|
pub struct Config {
|
||||||
pub model: ModelChoice,
|
pub model: ModelChoice,
|
||||||
pub idle_timeout_minutes: i32,
|
pub idle_timeout_minutes: i32,
|
||||||
|
pub a2ui_enabled: bool,
|
||||||
pub endpoint: EndpointConfig,
|
pub endpoint: EndpointConfig,
|
||||||
pub generation: GenerationPreferences,
|
pub generation: GenerationPreferences,
|
||||||
pub runtime: RuntimePreferences,
|
pub runtime: RuntimePreferences,
|
||||||
@@ -25,6 +26,7 @@ impl Default for Config {
|
|||||||
Self {
|
Self {
|
||||||
model: ModelChoice::default(),
|
model: ModelChoice::default(),
|
||||||
idle_timeout_minutes: 10,
|
idle_timeout_minutes: 10,
|
||||||
|
a2ui_enabled: true,
|
||||||
endpoint: EndpointConfig::default(),
|
endpoint: EndpointConfig::default(),
|
||||||
generation: GenerationPreferences::default(),
|
generation: GenerationPreferences::default(),
|
||||||
runtime: RuntimePreferences::default(),
|
runtime: RuntimePreferences::default(),
|
||||||
@@ -153,6 +155,7 @@ mod tests {
|
|||||||
let path = directory.join("config.yaml");
|
let path = directory.join("config.yaml");
|
||||||
let config = Config {
|
let config = Config {
|
||||||
model: ModelChoice::Glm52,
|
model: ModelChoice::Glm52,
|
||||||
|
a2ui_enabled: false,
|
||||||
generation: GenerationPreferences {
|
generation: GenerationPreferences {
|
||||||
context_tokens: 65_536,
|
context_tokens: 65_536,
|
||||||
reasoning_mode: ReasoningMode::Direct,
|
reasoning_mode: ReasoningMode::Direct,
|
||||||
@@ -173,7 +176,7 @@ mod tests {
|
|||||||
let text = fs::read_to_string(&path).unwrap();
|
let text = fs::read_to_string(&path).unwrap();
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
text,
|
text,
|
||||||
"model: glm-5.2\n\
|
"model: glm-5.2\na2ui_enabled: false\n\
|
||||||
generation:\n context_tokens: 65536\n reasoning_mode: none\n\
|
generation:\n context_tokens: 65536\n reasoning_mode: none\n\
|
||||||
runtime:\n ssd:\n enabled: true\n cache: 64GB\n"
|
runtime:\n ssd:\n enabled: true\n cache: 64GB\n"
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ use std::collections::HashMap;
|
|||||||
use std::fs;
|
use std::fs;
|
||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
|
|
||||||
use crate::schema::{messages, projects, sessions};
|
use crate::schema::{a2ui_messages, messages, projects, sessions};
|
||||||
|
|
||||||
pub const MIGRATIONS: EmbeddedMigrations = embed_migrations!("migrations");
|
pub const MIGRATIONS: EmbeddedMigrations = embed_migrations!("migrations");
|
||||||
|
|
||||||
@@ -137,6 +137,24 @@ struct NewMessage<'a> {
|
|||||||
compaction_tail_start: Option<i32>,
|
compaction_tail_start: Option<i32>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, Identifiable, Queryable, Selectable)]
|
||||||
|
#[diesel(table_name = a2ui_messages)]
|
||||||
|
#[diesel(check_for_backend(diesel::sqlite::Sqlite))]
|
||||||
|
pub struct StoredA2uiMessage {
|
||||||
|
pub id: i32,
|
||||||
|
pub session_id: i32,
|
||||||
|
pub message_id: i32,
|
||||||
|
pub json: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Insertable)]
|
||||||
|
#[diesel(table_name = a2ui_messages)]
|
||||||
|
struct NewA2uiMessage<'a> {
|
||||||
|
session_id: i32,
|
||||||
|
message_id: i32,
|
||||||
|
json: &'a str,
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
pub struct ProjectWithSessions {
|
pub struct ProjectWithSessions {
|
||||||
pub project: Project,
|
pub project: Project,
|
||||||
@@ -219,6 +237,16 @@ impl Database {
|
|||||||
pub fn delete_project(&mut self, project_id: i32) -> Result<(), String> {
|
pub fn delete_project(&mut self, project_id: i32) -> Result<(), String> {
|
||||||
self.connection
|
self.connection
|
||||||
.transaction(|connection| {
|
.transaction(|connection| {
|
||||||
|
diesel::delete(
|
||||||
|
a2ui_messages::table.filter(
|
||||||
|
a2ui_messages::session_id.eq_any(
|
||||||
|
sessions::table
|
||||||
|
.filter(sessions::project_id.eq(project_id))
|
||||||
|
.select(sessions::id),
|
||||||
|
),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.execute(connection)?;
|
||||||
diesel::delete(
|
diesel::delete(
|
||||||
messages::table.filter(
|
messages::table.filter(
|
||||||
messages::session_id.eq_any(
|
messages::session_id.eq_any(
|
||||||
@@ -272,6 +300,10 @@ impl Database {
|
|||||||
pub fn delete_session(&mut self, session_id: i32) -> Result<(), String> {
|
pub fn delete_session(&mut self, session_id: i32) -> Result<(), String> {
|
||||||
self.connection
|
self.connection
|
||||||
.transaction(|connection| {
|
.transaction(|connection| {
|
||||||
|
diesel::delete(
|
||||||
|
a2ui_messages::table.filter(a2ui_messages::session_id.eq(session_id)),
|
||||||
|
)
|
||||||
|
.execute(connection)?;
|
||||||
diesel::delete(messages::table.filter(messages::session_id.eq(session_id)))
|
diesel::delete(messages::table.filter(messages::session_id.eq(session_id)))
|
||||||
.execute(connection)?;
|
.execute(connection)?;
|
||||||
diesel::delete(sessions::table.find(session_id)).execute(connection)?;
|
diesel::delete(sessions::table.find(session_id)).execute(connection)?;
|
||||||
@@ -289,6 +321,35 @@ impl Database {
|
|||||||
.map_err(|error| error.to_string())
|
.map_err(|error| error.to_string())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn load_a2ui_messages(
|
||||||
|
&mut self,
|
||||||
|
session_id: i32,
|
||||||
|
) -> Result<Vec<StoredA2uiMessage>, String> {
|
||||||
|
a2ui_messages::table
|
||||||
|
.filter(a2ui_messages::session_id.eq(session_id))
|
||||||
|
.order(a2ui_messages::id.asc())
|
||||||
|
.select(StoredA2uiMessage::as_select())
|
||||||
|
.load(&mut self.connection)
|
||||||
|
.map_err(|error| error.to_string())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn insert_a2ui_message(
|
||||||
|
&mut self,
|
||||||
|
session_id: i32,
|
||||||
|
message_id: i32,
|
||||||
|
json: &str,
|
||||||
|
) -> Result<StoredA2uiMessage, String> {
|
||||||
|
diesel::insert_into(a2ui_messages::table)
|
||||||
|
.values(NewA2uiMessage {
|
||||||
|
session_id,
|
||||||
|
message_id,
|
||||||
|
json,
|
||||||
|
})
|
||||||
|
.returning(StoredA2uiMessage::as_returning())
|
||||||
|
.get_result(&mut self.connection)
|
||||||
|
.map_err(|error| error.to_string())
|
||||||
|
}
|
||||||
|
|
||||||
pub fn update_session_context(
|
pub fn update_session_context(
|
||||||
&mut self,
|
&mut self,
|
||||||
session_id: i32,
|
session_id: i32,
|
||||||
@@ -628,6 +689,13 @@ mod tests {
|
|||||||
database
|
database
|
||||||
.update_message(assistant.id, Some("Reasoning"), true, "Answer")
|
.update_message(assistant.id, Some("Reasoning"), true, "Answer")
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
database
|
||||||
|
.insert_a2ui_message(
|
||||||
|
session.id,
|
||||||
|
assistant.id,
|
||||||
|
r#"{"version":"v1.0","createSurface":{"surfaceId":"saved","catalogId":"https://ds4server.local/a2ui/v1_0/catalog.json"}}"#,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
database
|
database
|
||||||
.continue_tool_turn(
|
.continue_tool_turn(
|
||||||
session.id,
|
session.id,
|
||||||
@@ -662,6 +730,10 @@ mod tests {
|
|||||||
assert_eq!(messages[5].content, "Tool reminder");
|
assert_eq!(messages[5].content, "Tool reminder");
|
||||||
assert!(!messages[6].user);
|
assert!(!messages[6].user);
|
||||||
assert!(!messages[6].tool);
|
assert!(!messages[6].tool);
|
||||||
|
let a2ui = reopened.load_a2ui_messages(session.id).unwrap();
|
||||||
|
assert_eq!(a2ui.len(), 1);
|
||||||
|
assert_eq!(a2ui[0].message_id, messages[2].id);
|
||||||
|
assert!(a2ui[0].json.contains("createSurface"));
|
||||||
let first = reopened
|
let first = reopened
|
||||||
.record_compaction(
|
.record_compaction(
|
||||||
session.id,
|
session.id,
|
||||||
@@ -744,6 +816,7 @@ mod tests {
|
|||||||
assert_eq!(history[15].content, "After third compaction");
|
assert_eq!(history[15].content, "After third compaction");
|
||||||
reopened.delete_session(session.id).unwrap();
|
reopened.delete_session(session.id).unwrap();
|
||||||
assert!(reopened.load_messages(session.id).unwrap().is_empty());
|
assert!(reopened.load_messages(session.id).unwrap().is_empty());
|
||||||
|
assert!(reopened.load_a2ui_messages(session.id).unwrap().is_empty());
|
||||||
drop(reopened);
|
drop(reopened);
|
||||||
fs::remove_file(path).unwrap();
|
fs::remove_file(path).unwrap();
|
||||||
}
|
}
|
||||||
|
|||||||
11
src/main.rs
11
src/main.rs
@@ -1,3 +1,5 @@
|
|||||||
|
mod a2ui;
|
||||||
|
mod a2ui_validation;
|
||||||
mod agent;
|
mod agent;
|
||||||
mod app;
|
mod app;
|
||||||
mod compaction;
|
mod compaction;
|
||||||
@@ -9,6 +11,8 @@ mod model;
|
|||||||
#[cfg(target_os = "macos")]
|
#[cfg(target_os = "macos")]
|
||||||
mod native_edit;
|
mod native_edit;
|
||||||
#[cfg(target_os = "macos")]
|
#[cfg(target_os = "macos")]
|
||||||
|
mod native_media;
|
||||||
|
#[cfg(target_os = "macos")]
|
||||||
mod native_menu;
|
mod native_menu;
|
||||||
#[cfg(target_os = "macos")]
|
#[cfg(target_os = "macos")]
|
||||||
mod runtime;
|
mod runtime;
|
||||||
@@ -21,6 +25,13 @@ use app::{App, Message, app_icon, app_theme};
|
|||||||
use iced::{Size, window};
|
use iced::{Size, window};
|
||||||
|
|
||||||
fn main() -> iced::Result {
|
fn main() -> iced::Result {
|
||||||
|
if std::env::args().nth(1).as_deref() == Some("validate-a2ui") {
|
||||||
|
if let Err(error) = a2ui_validation::run(std::env::args().skip(2)) {
|
||||||
|
eprintln!("A2UI validation failed: {error}");
|
||||||
|
std::process::exit(1);
|
||||||
|
}
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
#[cfg(target_os = "macos")]
|
#[cfg(target_os = "macos")]
|
||||||
if let Err(error) = engine::configure_metal_sources() {
|
if let Err(error) = engine::configure_metal_sources() {
|
||||||
eprintln!("DS4Server: {error}");
|
eprintln!("DS4Server: {error}");
|
||||||
|
|||||||
39
src/native_media.rs
Normal file
39
src/native_media.rs
Normal file
@@ -0,0 +1,39 @@
|
|||||||
|
use std::ffi::{CString, c_char};
|
||||||
|
|
||||||
|
unsafe extern "C" {
|
||||||
|
fn ds4_media_open(url: *const c_char, title: *const c_char, video: bool) -> bool;
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn open(url: &str, title: &str, video: bool) -> Result<(), String> {
|
||||||
|
let url = playable_url(url)?;
|
||||||
|
let url = CString::new(url.as_str()).map_err(|error| error.to_string())?;
|
||||||
|
let title = CString::new(title).map_err(|error| error.to_string())?;
|
||||||
|
// SAFETY: Both C strings live for the duration of the call; Objective-C copies them.
|
||||||
|
if unsafe { ds4_media_open(url.as_ptr(), title.as_ptr(), video) } {
|
||||||
|
Ok(())
|
||||||
|
} else {
|
||||||
|
Err("AVKit could not create the media player".into())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn playable_url(value: &str) -> Result<url::Url, String> {
|
||||||
|
let url = url::Url::parse(value).map_err(|error| format!("invalid media URL: {error}"))?;
|
||||||
|
if matches!(url.scheme(), "http" | "https") {
|
||||||
|
Ok(url)
|
||||||
|
} else {
|
||||||
|
Err("media URLs must use HTTP(S)".into())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn media_player_only_accepts_remote_http_urls() {
|
||||||
|
assert!(playable_url("https://example.com/video.mp4").is_ok());
|
||||||
|
assert!(playable_url("http://example.com/audio.mp3").is_ok());
|
||||||
|
assert!(playable_url("file:///tmp/private.mov").is_err());
|
||||||
|
assert!(playable_url("javascript:alert(1)").is_err());
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,3 +1,12 @@
|
|||||||
|
diesel::table! {
|
||||||
|
a2ui_messages (id) {
|
||||||
|
id -> Integer,
|
||||||
|
session_id -> Integer,
|
||||||
|
message_id -> Integer,
|
||||||
|
json -> Text,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
diesel::table! {
|
diesel::table! {
|
||||||
messages (id) {
|
messages (id) {
|
||||||
id -> Integer,
|
id -> Integer,
|
||||||
@@ -36,5 +45,7 @@ diesel::table! {
|
|||||||
}
|
}
|
||||||
|
|
||||||
diesel::joinable!(sessions -> projects (project_id));
|
diesel::joinable!(sessions -> projects (project_id));
|
||||||
|
diesel::joinable!(a2ui_messages -> messages (message_id));
|
||||||
|
diesel::joinable!(a2ui_messages -> sessions (session_id));
|
||||||
diesel::joinable!(messages -> sessions (session_id));
|
diesel::joinable!(messages -> sessions (session_id));
|
||||||
diesel::allow_tables_to_appear_in_same_query!(messages, projects, sessions);
|
diesel::allow_tables_to_appear_in_same_query!(a2ui_messages, messages, projects, sessions);
|
||||||
|
|||||||
Reference in New Issue
Block a user