hugo/DS4Server
1
0
Fork 0
Code Issues 7 Pull Requests Actions 1 Packages Projects Releases Wiki Activity

feat: a2ui interface to enablee the LLM to give structured information

Browse Source
This commit is contained in:
Georg Bauer
2026-07-27 11:19:57 +02:00
parent 27b10ee0e1
commit c9c2d8efd5
24 changed files with 5322 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2217
src/a2ui.rs Normal file
View File

File diff suppressed because it is too large Load Diff

258
src/a2ui_validation.rs Normal file
View File

@@ -0,0 +1,258 @@
use crate::a2ui::{BASIC_NON_MEDIA_COMPONENTS, Store, extract_lines, validate_surface_composition};
use crate::config::Config;
use crate::model::ModelChoice;
use serde_json::{Value, json};
use std::time::Duration;
const CASES: &[Case] = &[
Case {
name: "pie-natural",
prompt: "Do not call tools. The file extension counts are rs 41, md 7, and toml 2. Give me a pie chart with the numbers.",
components: &["Chart"],
chart_type: Some("pie"),
},
Case {
name: "pie-single-slice",
prompt: "Do not call tools. There are 41 rs files and no other file extensions. Show that distribution as a one-slice pie chart.",
components: &["Chart"],
chart_type: Some("pie"),
},
Case {
name: "donut-natural",
prompt: "Do not call tools. Show a donut chart for 62 completed, 23 active, and 15 blocked tasks. Include the total.",
components: &["Chart"],
chart_type: Some("donut"),
},
Case {
name: "form-controls",
prompt: "Do not call tools. Build an A2UI Form for a name, multiline notes, due date, and multiple checkbox priorities, with a submit button.",
components: &["Form"],
chart_type: None,
},
Case {
name: "filterable-choices",
prompt: "Do not call tools. Build an A2UI surface with a filterable mutually-exclusive ChoicePicker for Rust, Python, and TypeScript.",
components: &["ChoicePicker"],
chart_type: None,
},
Case {
name: "composed-basics",
prompt: "Do not call tools. Build one complete A2UI project dashboard. Compose a Card and Column containing Markdown text, an avatar Image, an Icon, a Divider, a weighted Row, Tabs, a dynamic List, a Modal, a Button, TextField, CheckBox, Slider, DateTimeInput, and a filterable mutually-exclusive ChoicePicker. Bind the controls to the data model and give the button an event.",
components: BASIC_NON_MEDIA_COMPONENTS,
chart_type: None,
},
Case {
name: "media-players",
prompt: "Do not call tools. Build one A2UI Card containing a Video with a poster URL and an AudioPlayer with a description. Use valid HTTPS media URLs and compose both into the root tree.",
components: &["Video", "AudioPlayer"],
chart_type: None,
},
];
struct Case {
name: &'static str,
prompt: &'static str,
components: &'static [&'static str],
chart_type: Option<&'static str>,
}
struct Options {
endpoint: Option<String>,
model: Option<String>,
case: Option<String>,
attempts: u32,
}
pub(crate) fn run(args: impl Iterator<Item = String>) -> Result<(), String> {
let options = parse_options(args)?;
let config = Config::load(&crate::app::config_path())?;
let endpoint = options
.endpoint
.unwrap_or_else(|| format!("http://127.0.0.1:{}", config.endpoint.port));
let model_id = options
.model
.unwrap_or_else(|| config.model.id().to_owned());
let model =
ModelChoice::from_id(&model_id).ok_or_else(|| format!("unknown model `{model_id}`"))?;
let cases = CASES
.iter()
.filter(|case| options.case.as_deref().is_none_or(|name| name == case.name))
.collect::<Vec<_>>();
if cases.is_empty() {
return Err(format!(
"unknown case; choose one of: {}",
CASES
.iter()
.map(|case| case.name)
.collect::<Vec<_>>()
.join(", ")
));
}
let agent: ureq::Agent = ureq::Agent::config_builder()
.timeout_connect(Some(Duration::from_secs(5)))
.timeout_recv_response(Some(Duration::from_secs(30 * 60)))
.timeout_recv_body(Some(Duration::from_secs(30)))
.build()
.into();
let mut system = crate::agent::system_prompt(model, &config.generation.system_prompt);
system.push_str("\n\n");
system.push_str(crate::a2ui::SYSTEM_PROMPT);
let metadata = Store::default().client_metadata();
let mut passed = 0;
let total = cases.len() as u32 * options.attempts;
println!("A2UI live validation: {model_id} at {endpoint}");
for case in cases {
for attempt in 1..=options.attempts {
let user = format!("{}\n\nA2UI client metadata:\n{}", case.prompt, metadata);
let payload = json!({
"model": model_id,
"messages": [
{"role": "system", "content": system},
{"role": "user", "content": user}
],
"reasoning_effort": "none",
"temperature": 0,
"max_tokens": 4096
});
let content = request(&agent, &endpoint, &payload)?;
match validate(case, &content) {
Ok(summary) => {
passed += 1;
println!("PASS {}#{attempt}: {summary}", case.name);
}
Err(error) => {
println!("FAIL {}#{attempt}: {error}", case.name);
println!(
"--- response ---\n{}\n--- end response ---",
bounded(&content)
);
}
}
}
}
println!("A2UI validation summary: {passed}/{total} passed");
if passed == total {
Ok(())
} else {
Err(format!("{} live validation case(s) failed", total - passed))
}
}
fn parse_options(mut args: impl Iterator<Item = String>) -> Result<Options, String> {
let mut options = Options {
endpoint: None,
model: None,
case: None,
attempts: 1,
};
while let Some(argument) = args.next() {
let value = match argument.as_str() {
"--endpoint" | "--model" | "--case" | "--attempts" => args
.next()
.ok_or_else(|| format!("{argument} requires a value"))?,
"--help" | "-h" => {
println!(
"Usage: ds4-server validate-a2ui [--endpoint URL] [--model ID] [--case NAME] [--attempts N]"
);
std::process::exit(0);
}
_ => return Err(format!("unknown argument `{argument}`")),
};
match argument.as_str() {
"--endpoint" => options.endpoint = Some(value.trim_end_matches('/').to_owned()),
"--model" => options.model = Some(value),
"--case" => options.case = Some(value),
"--attempts" => {
options.attempts = value
.parse::<u32>()
.ok()
.filter(|attempts| *attempts > 0)
.ok_or_else(|| "--attempts must be a positive integer".to_owned())?;
}
_ => unreachable!(),
}
}
Ok(options)
}
fn request(agent: &ureq::Agent, endpoint: &str, payload: &Value) -> Result<String, String> {
let mut response = agent
.post(&format!("{endpoint}/v1/chat/completions"))
.header("Content-Type", "application/json")
.send(payload.to_string().as_bytes())
.map_err(|error| format!("could not call the local endpoint: {error}"))?;
let body = response
.body_mut()
.read_to_string()
.map_err(|error| format!("could not read the local endpoint response: {error}"))?;
let value: Value = serde_json::from_str(&body)
.map_err(|error| format!("local endpoint returned invalid JSON: {error}"))?;
value
.pointer("/choices/0/message/content")
.and_then(Value::as_str)
.map(str::to_owned)
.ok_or_else(|| format!("local endpoint response has no assistant content: {body}"))
}
fn validate(case: &Case, content: &str) -> Result<String, String> {
let lines = extract_lines(content);
if lines.is_empty() {
return Err("the model emitted no complete fenced A2UI JSONL messages".into());
}
let mut store = Store::default();
let mut errors = Vec::new();
for (index, line) in lines.iter().enumerate() {
if let Err(error) = &line.value {
errors.push(format!("line {} is invalid JSON: {error}", index + 1));
} else if let Err(error) = store.apply_raw(&line.raw, 1) {
errors.push(format!(
"line {} failed catalog validation: {error}",
index + 1
));
}
}
if !errors.is_empty() {
return Err(errors.join("; "));
}
let surfaces = store.surfaces().collect::<Vec<_>>();
let mut reachable_components = std::collections::BTreeSet::new();
for surface in &surfaces {
reachable_components.extend(
validate_surface_composition(surface)
.map_err(|error| format!("surface `{}` is not composable: {error}", surface.id))?,
);
}
let components = surfaces
.iter()
.flat_map(|surface| surface.components.values())
.filter_map(Value::as_object)
.collect::<Vec<_>>();
for expected in case.components {
if !reachable_components.contains(*expected) {
return Err(format!(
"valid surface did not compose expected {expected} component into its root tree"
));
}
}
if let Some(expected) = case.chart_type
&& !components.iter().any(|component| {
component.get("component").and_then(Value::as_str) == Some("Chart")
&& component.get("chartType").and_then(Value::as_str) == Some(expected)
})
{
return Err(format!("valid surface omitted expected {expected} chart"));
}
Ok(format!(
"{} protocol message(s), {} surface(s), {} component(s)",
lines.len(),
surfaces.len(),
components.len()
))
}
fn bounded(content: &str) -> &str {
content
.char_indices()
.nth(8_000)
.map_or(content, |(end, _)| &content[..end])
}

145
src/app.rs
View File

@@ -25,7 +25,7 @@ use crate::settings::{
ReasoningMode, RuntimePreferences, SpeculativePreferences, SsdPreferences, SteeringPreferences,
StreamingCacheBudget,
};
use iced::widget::{markdown, scrollable};
use iced::widget::{markdown, scrollable, text_editor};
use iced::{Size, Subscription, Task, keyboard, mouse, window};
use rfd::AsyncFileDialog;
use std::collections::{HashMap, HashSet, VecDeque};
@@ -78,6 +78,15 @@ pub(crate) struct App {
pub(super) composer: String,
pub(super) queued_inputs: VecDeque<String>,
pub(super) conversation: Vec<ChatMessage>,
pub(super) a2ui: crate::a2ui::Store,
pub(super) a2ui_tabs: HashMap<(String, String), usize>,
pub(super) a2ui_modals: HashSet<(String, String)>,
pub(super) a2ui_editors: HashMap<(String, String, String), text_editor::Content>,
pub(super) a2ui_markdown: HashMap<(String, String, String), markdown::Content>,
pub(super) a2ui_choice_filters: HashMap<(String, String, String), String>,
pub(super) a2ui_images: HashMap<String, iced::widget::image::Handle>,
pub(super) a2ui_image_requests: HashSet<String>,
pub(super) a2ui_image_loading: bool,
pub(super) generating: bool,
pub(super) context_used: u32,
pub(super) context_limit: u32,
@@ -163,6 +172,7 @@ pub(crate) enum Message {
PreferenceLegacyMtpChanged(bool),
PreferenceDsparkChanged(bool),
PreferenceTimeoutChanged(String),
PreferenceA2uiChanged(bool),
PreferenceEndpointPortChanged(String),
PreferenceEndpointEnabledChanged(bool),
PreferenceEndpointCorsChanged(bool),
@@ -199,6 +209,14 @@ pub(crate) enum Message {
PreferenceKvMinTokensChanged(String),
PreferenceKvColdMaxChanged(String),
PreferenceKvContinuedIntervalChanged(String),
A2uiDataChanged(String, String, serde_json::Value),
A2uiEditorAction(String, String, String, text_editor::Action),
A2uiChoiceFilterChanged(String, String, String, String),
A2uiAction(String, String, Option<String>),
A2uiSelectTab(String, String, usize),
A2uiToggleModal(String, String),
A2uiImageLoaded(String, Result<Vec<u8>, String>),
A2uiPlayMedia(String, String, bool),
ResetPreferences,
SavePreferences,
DownloadArtifact(ManagedArtifactId),
@@ -309,6 +327,15 @@ impl App {
composer: String::new(),
queued_inputs: VecDeque::new(),
conversation: Vec::new(),
a2ui: crate::a2ui::Store::default(),
a2ui_tabs: HashMap::new(),
a2ui_modals: HashSet::new(),
a2ui_editors: HashMap::new(),
a2ui_markdown: HashMap::new(),
a2ui_choice_filters: HashMap::new(),
a2ui_images: HashMap::new(),
a2ui_image_requests: HashSet::new(),
a2ui_image_loading: false,
generating: false,
context_used: 0,
context_limit,
@@ -415,6 +442,15 @@ impl App {
composer: String::new(),
queued_inputs: VecDeque::new(),
conversation: Vec::new(),
a2ui: crate::a2ui::Store::default(),
a2ui_tabs: HashMap::new(),
a2ui_modals: HashSet::new(),
a2ui_editors: HashMap::new(),
a2ui_markdown: HashMap::new(),
a2ui_choice_filters: HashMap::new(),
a2ui_images: HashMap::new(),
a2ui_image_requests: HashSet::new(),
a2ui_image_loading: false,
generating: false,
context_used: 0,
context_limit,
@@ -593,6 +629,10 @@ impl App {
self.preference_draft.idle_timeout_minutes = value;
self.preference_error = None;
}
Message::PreferenceA2uiChanged(enabled) => {
self.preference_draft.a2ui_enabled = enabled;
self.preference_error = None;
}
Message::PreferenceEndpointPortChanged(value) => {
self.preference_draft.endpoint_port = value;
self.preference_error = None;
@@ -802,6 +842,71 @@ impl App {
}
Message::DownloadProgressTick => self.update_download_progress(),
Message::ComposerChanged(value) => self.composer = value,
Message::A2uiDataChanged(surface_id, path, value) => {
return self.change_a2ui_data(surface_id, path, value);
}
Message::A2uiEditorAction(surface_id, component_id, path, action) => {
let key = (surface_id.clone(), component_id, path.clone());
let Some(editor) = self.a2ui_editors.get_mut(&key) else {
return Task::none();
};
editor.perform(action);
let value = serde_json::Value::String(editor.text());
return self.change_a2ui_data(surface_id, path, value);
}
Message::A2uiChoiceFilterChanged(surface_id, component_id, context_path, value) => {
self.a2ui_choice_filters
.insert((surface_id, component_id, context_path), value);
}
Message::A2uiAction(surface_id, component_id, context_path) => {
match self
.a2ui
.action(&surface_id, &component_id, context_path.as_deref())
{
Ok(action) => {
self.composer = format!(
"A2UI client event:\n{}\nA2UI client metadata:\n{}",
serde_json::to_string(&action).unwrap_or_default(),
self.a2ui.client_metadata()
);
self.start_generation();
return scroll_chat_to_end();
}
Err(error) => self.error = Some(error),
}
}
Message::A2uiSelectTab(surface_id, component_id, index) => {
self.a2ui_tabs.insert((surface_id, component_id), index);
}
Message::A2uiToggleModal(surface_id, component_id) => {
let key = (surface_id, component_id);
if !self.a2ui_modals.remove(&key) {
self.a2ui_modals.insert(key);
}
}
Message::A2uiImageLoaded(url, result) => {
self.a2ui_image_loading = false;
match result {
Ok(bytes) => {
self.a2ui_images
.insert(url, iced::widget::image::Handle::from_bytes(bytes));
}
Err(error) => {
self.error = Some(format!("Could not load an A2UI image: {error}"))
}
}
return self.load_next_a2ui_image();
}
Message::A2uiPlayMedia(url, title, video) => {
#[cfg(target_os = "macos")]
if let Err(error) = crate::native_media::open(&url, &title, video) {
self.error = Some(format!("Could not play media: {error}"));
}
#[cfg(not(target_os = "macos"))]
if let Err(error) = std::process::Command::new("open").arg(url).spawn() {
self.error = Some(format!("Could not open media: {error}"));
}
}
Message::ToggleReasoning(index) => {
if let Some(message) = self.conversation.get_mut(index)
&& message.reasoning.is_some()
@@ -869,9 +974,12 @@ impl App {
Message::GenerationTick => {
#[cfg(target_os = "macos")]
self.poll_titling();
if self.poll_generation() {
return scroll_chat_to_end();
let changed = self.poll_generation();
let images = self.load_next_a2ui_image();
if changed {
return Task::batch([scroll_chat_to_end(), images]);
}
return images;
}
Message::ChooseProjectFolder => {
self.choosing_folder = true;
@@ -948,6 +1056,7 @@ impl App {
self.selected_project = None;
self.selected_session = None;
self.conversation.clear();
self.clear_a2ui();
self.composer.clear();
self.queued_inputs.clear();
self.system_prompt_seen_at = 0;
@@ -1102,9 +1211,26 @@ impl App {
let Some(database) = &mut self.database else {
return Task::none();
};
match database.load_messages(session_id) {
Ok(messages) => {
let loaded = database.load_messages(session_id).and_then(|messages| {
database
.load_a2ui_messages(session_id)
.map(|a2ui| (messages, a2ui))
});
match loaded {
Ok((messages, a2ui)) => {
self.conversation = messages.into_iter().map(ChatMessage::from).collect();
self.clear_a2ui();
for message in a2ui {
if let Err(error) =
self.a2ui.apply_raw(&message.json, message.message_id)
{
self.error = Some(format!(
"Could not restore A2UI message {}: {error}",
message.id
));
}
}
self.sync_a2ui_renderer_state();
self.composer.clear();
self.remember_project(project_id);
self.selected_session = Some(session_id);
@@ -1119,7 +1245,7 @@ impl App {
};
self.tokens_per_second = tokens_per_second;
self.error = None;
return scroll_chat_to_end();
return Task::batch([scroll_chat_to_end(), self.load_next_a2ui_image()]);
}
Err(error) => {
self.error = Some(format!("Could not load the chat session: {error}"));
@@ -1150,6 +1276,7 @@ impl App {
if self.selected_session == Some(session_id) {
self.selected_session = None;
self.conversation.clear();
self.clear_a2ui();
self.composer.clear();
self.queued_inputs.clear();
self.system_prompt_seen_at = 0;
@@ -1424,7 +1551,7 @@ fn models_path() -> PathBuf {
}
/// The settings file, beside the project database.
fn config_path() -> PathBuf {
pub(crate) fn config_path() -> PathBuf {
application_support_path().join("config.yaml")
}
@@ -1736,6 +1863,10 @@ mod tests {
reasoning_open: true,
content: String::new(),
markdown: markdown::Content::new(),
a2ui_lines_processed: 0,
a2ui_errors: Vec::new(),
a2ui_replies: Vec::new(),
a2ui_open_urls: Vec::new(),
};
message.append(true, "working it out");
message.append(false, "**final answer**");

188
src/app/generation.rs
View File

@@ -60,6 +60,10 @@ pub(crate) struct ChatMessage {
pub(super) reasoning_open: bool,
pub(super) content: String,
pub(super) markdown: markdown::Content,
pub(super) a2ui_lines_processed: usize,
pub(super) a2ui_errors: Vec<String>,
pub(super) a2ui_replies: Vec<serde_json::Value>,
pub(super) a2ui_open_urls: Vec<String>,
}
impl ChatMessage {
@@ -75,10 +79,11 @@ impl ChatMessage {
pub(super) fn refresh_markdown(&mut self) {
if !self.user && !self.tool {
let visible = crate::agent::visible_content(&self.content);
let visible = crate::a2ui::transcript_fallback(visible);
let content = if self.reasoning.is_some() {
visible.trim_start()
} else {
visible
&visible
};
self.markdown = markdown::Content::parse(content);
}
@@ -99,12 +104,59 @@ impl From<StoredMessage> for ChatMessage {
reasoning_open: false,
content: message.content,
markdown: iced::widget::markdown::Content::new(),
a2ui_lines_processed: 0,
a2ui_errors: Vec::new(),
a2ui_replies: Vec::new(),
a2ui_open_urls: Vec::new(),
};
message.refresh_markdown();
message
}
}
fn sync_a2ui_message(
store: &mut crate::a2ui::Store,
database: &mut Option<Database>,
session_id: Option<i32>,
message: &mut ChatMessage,
) {
let lines = crate::a2ui::extract_lines(&message.content);
for (index, line) in lines.iter().enumerate().skip(message.a2ui_lines_processed) {
let applied = match &line.value {
Ok(value) => store.apply(value.clone(), line.raw.clone(), message.id),
Err(error) => Err(error.clone()),
};
match applied {
Ok(applied) => {
if let Some(reply) = applied.reply {
message.a2ui_replies.push(reply);
}
if let Some(url) = applied.open_url {
message.a2ui_open_urls.push(url);
}
if let (Some(session_id), Some(database)) = (session_id, database.as_mut()) {
for raw in applied.raws {
if let Err(error) =
database.insert_a2ui_message(session_id, message.id, &raw)
{
message.a2ui_errors.push(format!(
"line {} could not be persisted: {error}",
index + 1
));
break;
}
}
}
}
Err(error) => message
.a2ui_errors
.push(format!("line {}: {error}", index + 1)),
}
}
message.a2ui_lines_processed = lines.len();
message.refresh_markdown();
}
#[cfg(target_os = "macos")]
fn chat_turn(message: &ChatMessage) -> ChatTurn {
ChatTurn {
@@ -168,6 +220,15 @@ fn has_chat_after_last_compaction(messages: &[ChatMessage]) -> bool {
}
impl App {
fn chat_system_prompt(&self, model: ModelChoice, prompt: &str) -> String {
let mut prompt = crate::agent::system_prompt(model, prompt);
if self.config.a2ui_enabled {
prompt.push_str("\n\n");
prompt.push_str(crate::a2ui::SYSTEM_PROMPT);
}
prompt
}
pub(super) fn can_compact_session(&self, session_id: i32) -> bool {
!self.generating
&& self.selected_session == Some(session_id)
@@ -238,13 +299,22 @@ impl App {
}
};
effective.turn.system_prompt =
crate::agent::system_prompt(model, &effective.turn.system_prompt);
self.chat_system_prompt(model, &effective.turn.system_prompt);
effective.turn.system_prompt = crate::compaction::summary_system_prompt(
&effective.turn.system_prompt,
self.compaction_summary(),
);
let assistant_reasoning = effective.turn.reasoning_mode != ReasoningMode::Direct;
#[cfg(target_os = "macos")]
let model_prompt = if self.config.a2ui_enabled {
format!(
"{prompt}\n\nA2UI client metadata:\n{}",
self.a2ui.client_metadata()
)
} else {
prompt.clone()
};
#[cfg(target_os = "macos")]
let opening_turn = self.selected_session.is_none();
#[cfg(target_os = "macos")]
let mut injected_system = Vec::new();
@@ -282,7 +352,7 @@ impl App {
skip_previous_eos: false,
reasoning: None,
reasoning_complete: true,
content: prompt.clone(),
content: model_prompt,
});
#[cfg(target_os = "macos")]
@@ -374,6 +444,9 @@ impl App {
fn system_prompt_reminders(&self, model: ModelChoice) -> Vec<String> {
let mut reminders = vec![crate::agent::system_prompt_reminder(model)];
if self.config.a2ui_enabled {
reminders.push(crate::a2ui::SYSTEM_PROMPT.to_owned());
}
if !self.config.generation.system_prompt.trim().is_empty() {
reminders.push(self.config.generation.system_prompt.clone());
}
@@ -475,6 +548,10 @@ impl App {
#[cfg(target_os = "macos")]
let mut start_queued = false;
#[cfg(target_os = "macos")]
let mut a2ui_feedback = None;
#[cfg(target_os = "macos")]
let mut a2ui_changed = false;
#[cfg(target_os = "macos")]
loop {
match active.events.try_recv() {
Ok(GenerationEvent::Loading) => {}
@@ -500,6 +577,18 @@ impl App {
}
transcript_changed = true;
}
if !reasoning
&& self.config.a2ui_enabled
&& let Some(message) = self.conversation.last_mut()
{
sync_a2ui_message(
&mut self.a2ui,
&mut self.database,
self.selected_session,
message,
);
a2ui_changed = true;
}
}
Ok(GenerationEvent::Context {
used,
@@ -530,6 +619,69 @@ impl App {
!self.queued_inputs.is_empty() || self.manual_compaction_queued;
}
Ok(_) => {
let (validation_errors, replies, open_urls, error_surface_id) = self
.conversation
.last_mut()
.map(|message| {
let surface_id = crate::a2ui::extract_lines(&message.content)
.into_iter()
.rev()
.filter_map(|line| line.value.ok())
.find_map(|value| {
crate::a2ui::message_surface_id(&value)
.map(str::to_owned)
})
.unwrap_or_else(|| "unknown".to_owned());
(
std::mem::take(&mut message.a2ui_errors),
std::mem::take(&mut message.a2ui_replies),
std::mem::take(&mut message.a2ui_open_urls),
surface_id,
)
})
.unwrap_or_default();
for url in open_urls {
if let Err(error) =
std::process::Command::new("open").arg(url).spawn()
{
self.error =
Some(format!("Could not open the A2UI link: {error}"));
}
}
if !validation_errors.is_empty() {
self.generating = false;
self.activity = Some("Correcting A2UI…".into());
self.tool_cards.clear();
a2ui_feedback = Some(
serde_json::json!({
"version": crate::a2ui::VERSION,
"error": {
"code": "VALIDATION_FAILED",
"surfaceId": error_surface_id,
"path": "/",
"message": validation_errors.join("; ")
}
})
.to_string(),
);
self.active_generation = None;
break;
}
if !replies.is_empty() {
self.generating = false;
self.activity = Some("Continuing A2UI function call…".into());
self.tool_cards.clear();
a2ui_feedback = Some(format!(
"A2UI client response:\n{}",
replies
.iter()
.map(ToString::to_string)
.collect::<Vec<_>>()
.join("\n")
));
self.active_generation = None;
break;
}
let model = self.config.model;
let content = self
.conversation
@@ -586,6 +738,10 @@ impl App {
}
}
#[cfg(target_os = "macos")]
if a2ui_changed {
self.sync_a2ui_renderer_state();
}
#[cfg(target_os = "macos")]
if transcript_changed && let Some(message) = self.conversation.last_mut() {
message.refresh_markdown();
}
@@ -633,6 +789,14 @@ impl App {
self.start_next_queued();
}
#[cfg(target_os = "macos")]
if let Some(feedback) = a2ui_feedback
&& let Err(error) = self.continue_after_tool_result(&feedback)
{
self.generating = false;
self.activity = Some("Failed".into());
self.error = Some(error);
}
#[cfg(target_os = "macos")]
return transcript_changed;
#[cfg(not(target_os = "macos"))]
false
@@ -698,7 +862,7 @@ impl App {
&models_path(),
)?;
effective.turn.system_prompt =
crate::agent::system_prompt(model, &effective.turn.system_prompt);
self.chat_system_prompt(model, &effective.turn.system_prompt);
effective.turn.system_prompt = crate::compaction::summary_system_prompt(
&effective.turn.system_prompt,
self.compaction_summary(),
@@ -792,7 +956,7 @@ impl App {
&models_path(),
)?;
effective.turn.system_prompt =
crate::agent::system_prompt(model, &effective.turn.system_prompt);
self.chat_system_prompt(model, &effective.turn.system_prompt);
effective.turn.system_prompt = crate::compaction::summary_system_prompt(
&effective.turn.system_prompt,
self.compaction_summary(),
@@ -941,7 +1105,7 @@ impl App {
&models_path(),
)?;
effective.turn.system_prompt =
crate::agent::system_prompt(model, &effective.turn.system_prompt);
self.chat_system_prompt(model, &effective.turn.system_prompt);
let rebuild_system_prompt = effective.turn.system_prompt.clone();
effective.turn.system_prompt = crate::compaction::summary_system_prompt(
&effective.turn.system_prompt,
@@ -1364,6 +1528,10 @@ mod tests {
content: "### Core / Setup\n\n| File | Lines |\n|---|---:|\n| `src/app.rs` | **1,750** |\n| `src/engine.rs` | 2,400 |\n\n### Summary\n\nDone."
.to_owned(),
markdown: iced::widget::markdown::Content::new(),
a2ui_lines_processed: 0,
a2ui_errors: Vec::new(),
a2ui_replies: Vec::new(),
a2ui_open_urls: Vec::new(),
};
message.refresh_markdown();
@@ -1394,6 +1562,10 @@ mod tests {
reasoning_open: false,
content: format!("message {id}"),
markdown: iced::widget::markdown::Content::new(),
a2ui_lines_processed: 0,
a2ui_errors: Vec::new(),
a2ui_replies: Vec::new(),
a2ui_open_urls: Vec::new(),
};
let history = vec![
message(1, false, None),
@@ -1426,6 +1598,10 @@ mod tests {
reasoning_open: false,
content: format!("message {id}"),
markdown: iced::widget::markdown::Content::new(),
a2ui_lines_processed: 0,
a2ui_errors: Vec::new(),
a2ui_replies: Vec::new(),
a2ui_open_urls: Vec::new(),
};
let mut history = vec![
message(1, true, false, false, false),

3
src/app/preferences.rs
View File

@@ -6,6 +6,7 @@ pub(super) struct PreferenceDraft {
pub(super) legacy_mtp_enabled: bool,
pub(super) dspark_enabled: bool,
pub(super) idle_timeout_minutes: String,
pub(super) a2ui_enabled: bool,
pub(super) endpoint_port: String,
pub(super) endpoint_enabled: bool,
pub(super) endpoint_cors: bool,
@@ -55,6 +56,7 @@ impl PreferenceDraft {
legacy_mtp_enabled: speculative.legacy_mtp_enabled,
dspark_enabled: speculative.dspark_enabled,
idle_timeout_minutes: config.idle_timeout_minutes.to_string(),
a2ui_enabled: config.a2ui_enabled,
endpoint_port: config.endpoint.port.to_string(),
endpoint_enabled: config.endpoint.enabled,
endpoint_cors: config.endpoint.cors,
@@ -350,6 +352,7 @@ impl App {
let config = Config {
model: self.preference_draft.model,
idle_timeout_minutes,
a2ui_enabled: self.preference_draft.a2ui_enabled,
endpoint: EndpointConfig {
port: i32::from(endpoint_port),
enabled: self.preference_draft.endpoint_enabled,

76
src/app/projects.rs
View File

@@ -1,6 +1,79 @@
use super::*;
impl App {
pub(super) fn clear_a2ui(&mut self) {
self.a2ui.clear();
self.a2ui_tabs.clear();
self.a2ui_modals.clear();
self.a2ui_editors.clear();
self.a2ui_markdown.clear();
self.a2ui_choice_filters.clear();
self.a2ui_images.clear();
self.a2ui_image_requests.clear();
self.a2ui_image_loading = false;
}
pub(super) fn load_next_a2ui_image(&mut self) -> Task<Message> {
if self.a2ui_image_loading {
return Task::none();
}
let Some(url) = self
.a2ui
.image_urls()
.find(|url| !self.a2ui_image_requests.contains(url))
else {
return Task::none();
};
self.a2ui_image_requests.insert(url.clone());
self.a2ui_image_loading = true;
let request_url = url.clone();
Task::perform(
async move {
let mut response = ureq::get(&request_url)
.call()
.map_err(|error| error.to_string())?;
if !response
.headers()
.get("content-type")
.and_then(|value| value.to_str().ok())
.is_some_and(|value| value.starts_with("image/"))
{
return Err("the URL did not return an image".to_owned());
}
response
.body_mut()
.read_to_vec()
.map_err(|error| error.to_string())
},
move |result| Message::A2uiImageLoaded(url, result),
)
}
pub(super) fn change_a2ui_data(
&mut self,
surface_id: String,
path: String,
value: serde_json::Value,
) -> Task<Message> {
let owner = self
.a2ui
.surface(&surface_id)
.map(|surface| surface.owner_message_id);
match self.a2ui.local_update(&surface_id, &path, value) {
Ok(raw) => {
if let (Some(session_id), Some(message_id), Some(database)) =
(self.selected_session, owner, &mut self.database)
&& let Err(error) = database.insert_a2ui_message(session_id, message_id, &raw)
{
self.error = Some(format!("Could not save the A2UI edit: {error}"));
}
}
Err(error) => self.error = Some(error),
}
self.sync_a2ui_renderer_state();
self.load_next_a2ui_image()
}
pub(super) fn prepare_project(&mut self, path: PathBuf) {
let Ok(path) = fs::canonicalize(path) else {
self.error = Some("The selected folder is no longer available.".into());
@@ -49,6 +122,7 @@ impl App {
Ok(project) => {
self.remember_project(project.id);
self.selected_session = None;
self.clear_a2ui();
self.system_prompt_seen_at = 0;
self.pending_project_path = None;
self.project_name_input.clear();
@@ -78,6 +152,7 @@ impl App {
self.remember_project(project_id);
self.selected_session = None;
self.conversation.clear();
self.clear_a2ui();
self.composer.clear();
self.queued_inputs.clear();
self.system_prompt_seen_at = 0;
@@ -90,6 +165,7 @@ impl App {
pub(super) fn discard_session(&mut self, project_id: i32) {
if self.drafts.remove(&project_id).is_some() && self.draft_selected(project_id) {
self.conversation.clear();
self.clear_a2ui();
self.composer.clear();
self.queued_inputs.clear();
self.system_prompt_seen_at = 0;

11
src/app/view.rs
View File

@@ -1,3 +1,4 @@
mod a2ui;
mod chat;
mod model_manager;
mod preferences;
@@ -16,8 +17,9 @@ use crate::model::{
use crate::settings::{GIB, REASONING_MODES};
use iced::theme::{Palette, palette};
use iced::widget::{
Button, Space, Svg, Tooltip, button, checkbox, column, container, markdown, mouse_area, opaque,
pick_list, progress_bar, row, rule, scrollable, stack, svg, text, text_input, tooltip,
Button, Space, Svg, Tooltip, button, checkbox, column, container, image, markdown, mouse_area,
opaque, pick_list, progress_bar, row, rule, scrollable, slider, stack, svg, text, text_input,
tooltip,
};
use iced::{Alignment, Background, Border, Color, Element, Length, Padding, Theme, window};
use std::collections::VecDeque;
@@ -68,6 +70,7 @@ impl App {
|| self.pending_project_path.is_some()
|| self.session_rename.is_some()
|| self.menu_session().is_some()
|| !self.a2ui_modals.is_empty()
|| {
#[cfg(target_os = "macos")]
{
@@ -132,6 +135,8 @@ impl App {
layers.push(self.rename_dialog(title));
} else if let Some(session) = self.menu_session() {
layers.push(self.session_menu_panel(session));
} else if let Some(panel) = self.a2ui_modal_panel() {
layers.push(panel);
}
#[cfg(not(target_os = "macos"))]
if self.preferences_open {
@@ -142,6 +147,8 @@ impl App {
layers.push(self.rename_dialog(title));
} else if let Some(session) = self.menu_session() {
layers.push(self.session_menu_panel(session));
} else if let Some(panel) = self.a2ui_modal_panel() {
layers.push(panel);
}
stack(layers)

1909
src/app/view/a2ui.rs Normal file
View File

File diff suppressed because it is too large Load Diff

3
src/app/view/chat.rs
View File

@@ -179,6 +179,9 @@ impl App {
if !cards.is_empty() {
body = body.push(tool_cards(cards));
}
if self.a2ui.surfaces_for_message(message.id).next().is_some() {
body = body.push(self.a2ui_surfaces(message.id));
}
}
let user = message.user;
messages = messages.push(

6
src/app/view/preferences.rs
View File

@@ -119,6 +119,12 @@ impl App {
.spacing(10)
.align_y(Alignment::Center),
text("Enter a whole number from 1 to 1440.").size(12),
hint(
checkbox(self.preference_draft.a2ui_enabled)
.label("Enable interactive A2UI chat surfaces")
.on_toggle(Message::PreferenceA2uiChanged),
"Lets the local model build validated native charts, tables, forms and other interactive chat surfaces. Turning it off removes the A2UI catalog from the system prompt.",
),
]
.spacing(10),
);

5
src/config.rs
View File

@@ -14,6 +14,7 @@ use crate::settings::{GenerationPreferences, RuntimePreferences};
pub struct Config {
pub model: ModelChoice,
pub idle_timeout_minutes: i32,
pub a2ui_enabled: bool,
pub endpoint: EndpointConfig,
pub generation: GenerationPreferences,
pub runtime: RuntimePreferences,
@@ -25,6 +26,7 @@ impl Default for Config {
Self {
model: ModelChoice::default(),
idle_timeout_minutes: 10,
a2ui_enabled: true,
endpoint: EndpointConfig::default(),
generation: GenerationPreferences::default(),
runtime: RuntimePreferences::default(),
@@ -153,6 +155,7 @@ mod tests {
let path = directory.join("config.yaml");
let config = Config {
model: ModelChoice::Glm52,
a2ui_enabled: false,
generation: GenerationPreferences {
context_tokens: 65_536,
reasoning_mode: ReasoningMode::Direct,
@@ -173,7 +176,7 @@ mod tests {
let text = fs::read_to_string(&path).unwrap();
assert_eq!(
text,
"model: glm-5.2\n\
"model: glm-5.2\na2ui_enabled: false\n\
generation:\n context_tokens: 65536\n reasoning_mode: none\n\
runtime:\n ssd:\n enabled: true\n cache: 64GB\n"
);

75
src/database.rs
View File

@@ -4,7 +4,7 @@ use std::collections::HashMap;
use std::fs;
use std::path::Path;
use crate::schema::{messages, projects, sessions};
use crate::schema::{a2ui_messages, messages, projects, sessions};
pub const MIGRATIONS: EmbeddedMigrations = embed_migrations!("migrations");
@@ -137,6 +137,24 @@ struct NewMessage<'a> {
compaction_tail_start: Option<i32>,
}
#[derive(Clone, Debug, Identifiable, Queryable, Selectable)]
#[diesel(table_name = a2ui_messages)]
#[diesel(check_for_backend(diesel::sqlite::Sqlite))]
pub struct StoredA2uiMessage {
pub id: i32,
pub session_id: i32,
pub message_id: i32,
pub json: String,
}
#[derive(Insertable)]
#[diesel(table_name = a2ui_messages)]
struct NewA2uiMessage<'a> {
session_id: i32,
message_id: i32,
json: &'a str,
}
#[derive(Debug)]
pub struct ProjectWithSessions {
pub project: Project,
@@ -219,6 +237,16 @@ impl Database {
pub fn delete_project(&mut self, project_id: i32) -> Result<(), String> {
self.connection
.transaction(|connection| {
diesel::delete(
a2ui_messages::table.filter(
a2ui_messages::session_id.eq_any(
sessions::table
.filter(sessions::project_id.eq(project_id))
.select(sessions::id),
),
),
)
.execute(connection)?;
diesel::delete(
messages::table.filter(
messages::session_id.eq_any(
@@ -272,6 +300,10 @@ impl Database {
pub fn delete_session(&mut self, session_id: i32) -> Result<(), String> {
self.connection
.transaction(|connection| {
diesel::delete(
a2ui_messages::table.filter(a2ui_messages::session_id.eq(session_id)),
)
.execute(connection)?;
diesel::delete(messages::table.filter(messages::session_id.eq(session_id)))
.execute(connection)?;
diesel::delete(sessions::table.find(session_id)).execute(connection)?;
@@ -289,6 +321,35 @@ impl Database {
.map_err(|error| error.to_string())
}
pub fn load_a2ui_messages(
&mut self,
session_id: i32,
) -> Result<Vec<StoredA2uiMessage>, String> {
a2ui_messages::table
.filter(a2ui_messages::session_id.eq(session_id))
.order(a2ui_messages::id.asc())
.select(StoredA2uiMessage::as_select())
.load(&mut self.connection)
.map_err(|error| error.to_string())
}
pub fn insert_a2ui_message(
&mut self,
session_id: i32,
message_id: i32,
json: &str,
) -> Result<StoredA2uiMessage, String> {
diesel::insert_into(a2ui_messages::table)
.values(NewA2uiMessage {
session_id,
message_id,
json,
})
.returning(StoredA2uiMessage::as_returning())
.get_result(&mut self.connection)
.map_err(|error| error.to_string())
}
pub fn update_session_context(
&mut self,
session_id: i32,
@@ -628,6 +689,13 @@ mod tests {
database
.update_message(assistant.id, Some("Reasoning"), true, "Answer")
.unwrap();
database
.insert_a2ui_message(
session.id,
assistant.id,
r#"{"version":"v1.0","createSurface":{"surfaceId":"saved","catalogId":"https://ds4server.local/a2ui/v1_0/catalog.json"}}"#,
)
.unwrap();
database
.continue_tool_turn(
session.id,
@@ -662,6 +730,10 @@ mod tests {
assert_eq!(messages[5].content, "Tool reminder");
assert!(!messages[6].user);
assert!(!messages[6].tool);
let a2ui = reopened.load_a2ui_messages(session.id).unwrap();
assert_eq!(a2ui.len(), 1);
assert_eq!(a2ui[0].message_id, messages[2].id);
assert!(a2ui[0].json.contains("createSurface"));
let first = reopened
.record_compaction(
session.id,
@@ -744,6 +816,7 @@ mod tests {
assert_eq!(history[15].content, "After third compaction");
reopened.delete_session(session.id).unwrap();
assert!(reopened.load_messages(session.id).unwrap().is_empty());
assert!(reopened.load_a2ui_messages(session.id).unwrap().is_empty());
drop(reopened);
fs::remove_file(path).unwrap();
}

11
src/main.rs
View File

@@ -1,3 +1,5 @@
mod a2ui;
mod a2ui_validation;
mod agent;
mod app;
mod compaction;
@@ -9,6 +11,8 @@ mod model;
#[cfg(target_os = "macos")]
mod native_edit;
#[cfg(target_os = "macos")]
mod native_media;
#[cfg(target_os = "macos")]
mod native_menu;
#[cfg(target_os = "macos")]
mod runtime;
@@ -21,6 +25,13 @@ use app::{App, Message, app_icon, app_theme};
use iced::{Size, window};
fn main() -> iced::Result {
if std::env::args().nth(1).as_deref() == Some("validate-a2ui") {
if let Err(error) = a2ui_validation::run(std::env::args().skip(2)) {
eprintln!("A2UI validation failed: {error}");
std::process::exit(1);
}
return Ok(());
}
#[cfg(target_os = "macos")]
if let Err(error) = engine::configure_metal_sources() {
eprintln!("DS4Server: {error}");

39
src/native_media.rs Normal file
View File

@@ -0,0 +1,39 @@
use std::ffi::{CString, c_char};
unsafe extern "C" {
fn ds4_media_open(url: *const c_char, title: *const c_char, video: bool) -> bool;
}
pub(crate) fn open(url: &str, title: &str, video: bool) -> Result<(), String> {
let url = playable_url(url)?;
let url = CString::new(url.as_str()).map_err(|error| error.to_string())?;
let title = CString::new(title).map_err(|error| error.to_string())?;
// SAFETY: Both C strings live for the duration of the call; Objective-C copies them.
if unsafe { ds4_media_open(url.as_ptr(), title.as_ptr(), video) } {
Ok(())
} else {
Err("AVKit could not create the media player".into())
}
}
fn playable_url(value: &str) -> Result<url::Url, String> {
let url = url::Url::parse(value).map_err(|error| format!("invalid media URL: {error}"))?;
if matches!(url.scheme(), "http" | "https") {
Ok(url)
} else {
Err("media URLs must use HTTP(S)".into())
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn media_player_only_accepts_remote_http_urls() {
assert!(playable_url("https://example.com/video.mp4").is_ok());
assert!(playable_url("http://example.com/audio.mp3").is_ok());
assert!(playable_url("file:///tmp/private.mov").is_err());
assert!(playable_url("javascript:alert(1)").is_err());
}
}

13
src/schema.rs
View File

@@ -1,3 +1,12 @@
diesel::table! {
a2ui_messages (id) {
id -> Integer,
session_id -> Integer,
message_id -> Integer,
json -> Text,
}
}
diesel::table! {
messages (id) {
id -> Integer,
@@ -36,5 +45,7 @@ diesel::table! {
}
diesel::joinable!(sessions -> projects (project_id));
diesel::joinable!(a2ui_messages -> messages (message_id));
diesel::joinable!(a2ui_messages -> sessions (session_id));
diesel::joinable!(messages -> sessions (session_id));
diesel::allow_tables_to_appear_in_same_query!(messages, projects, sessions);
diesel::allow_tables_to_appear_in_same_query!(a2ui_messages, messages, projects, sessions);