hugo/DS4Server
1
0
Fork 0
Code Issues 7 Pull Requests Actions 1 Packages Projects Releases Wiki Activity

feat: a2ui interface to enablee the LLM to give structured information

Browse Source
This commit is contained in:
Georg Bauer
2026-07-27 11:19:57 +02:00
parent 27b10ee0e1
commit c9c2d8efd5
24 changed files with 5322 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files

188
src/app/generation.rs
View File

@@ -60,6 +60,10 @@ pub(crate) struct ChatMessage {
pub(super) reasoning_open: bool,
pub(super) content: String,
pub(super) markdown: markdown::Content,
pub(super) a2ui_lines_processed: usize,
pub(super) a2ui_errors: Vec<String>,
pub(super) a2ui_replies: Vec<serde_json::Value>,
pub(super) a2ui_open_urls: Vec<String>,
}
impl ChatMessage {
@@ -75,10 +79,11 @@ impl ChatMessage {
pub(super) fn refresh_markdown(&mut self) {
if !self.user && !self.tool {
let visible = crate::agent::visible_content(&self.content);
let visible = crate::a2ui::transcript_fallback(visible);
let content = if self.reasoning.is_some() {
visible.trim_start()
} else {
visible
&visible
};
self.markdown = markdown::Content::parse(content);
}
@@ -99,12 +104,59 @@ impl From<StoredMessage> for ChatMessage {
reasoning_open: false,
content: message.content,
markdown: iced::widget::markdown::Content::new(),
a2ui_lines_processed: 0,
a2ui_errors: Vec::new(),
a2ui_replies: Vec::new(),
a2ui_open_urls: Vec::new(),
};
message.refresh_markdown();
message
}
}
fn sync_a2ui_message(
store: &mut crate::a2ui::Store,
database: &mut Option<Database>,
session_id: Option<i32>,
message: &mut ChatMessage,
) {
let lines = crate::a2ui::extract_lines(&message.content);
for (index, line) in lines.iter().enumerate().skip(message.a2ui_lines_processed) {
let applied = match &line.value {
Ok(value) => store.apply(value.clone(), line.raw.clone(), message.id),
Err(error) => Err(error.clone()),
};
match applied {
Ok(applied) => {
if let Some(reply) = applied.reply {
message.a2ui_replies.push(reply);
}
if let Some(url) = applied.open_url {
message.a2ui_open_urls.push(url);
}
if let (Some(session_id), Some(database)) = (session_id, database.as_mut()) {
for raw in applied.raws {
if let Err(error) =
database.insert_a2ui_message(session_id, message.id, &raw)
{
message.a2ui_errors.push(format!(
"line {} could not be persisted: {error}",
index + 1
));
break;
}
}
}
}
Err(error) => message
.a2ui_errors
.push(format!("line {}: {error}", index + 1)),
}
}
message.a2ui_lines_processed = lines.len();
message.refresh_markdown();
}
#[cfg(target_os = "macos")]
fn chat_turn(message: &ChatMessage) -> ChatTurn {
ChatTurn {
@@ -168,6 +220,15 @@ fn has_chat_after_last_compaction(messages: &[ChatMessage]) -> bool {
}
impl App {
fn chat_system_prompt(&self, model: ModelChoice, prompt: &str) -> String {
let mut prompt = crate::agent::system_prompt(model, prompt);
if self.config.a2ui_enabled {
prompt.push_str("\n\n");
prompt.push_str(crate::a2ui::SYSTEM_PROMPT);
}
prompt
}
pub(super) fn can_compact_session(&self, session_id: i32) -> bool {
!self.generating
&& self.selected_session == Some(session_id)
@@ -238,13 +299,22 @@ impl App {
}
};
effective.turn.system_prompt =
crate::agent::system_prompt(model, &effective.turn.system_prompt);
self.chat_system_prompt(model, &effective.turn.system_prompt);
effective.turn.system_prompt = crate::compaction::summary_system_prompt(
&effective.turn.system_prompt,
self.compaction_summary(),
);
let assistant_reasoning = effective.turn.reasoning_mode != ReasoningMode::Direct;
#[cfg(target_os = "macos")]
let model_prompt = if self.config.a2ui_enabled {
format!(
"{prompt}\n\nA2UI client metadata:\n{}",
self.a2ui.client_metadata()
)
} else {
prompt.clone()
};
#[cfg(target_os = "macos")]
let opening_turn = self.selected_session.is_none();
#[cfg(target_os = "macos")]
let mut injected_system = Vec::new();
@@ -282,7 +352,7 @@ impl App {
skip_previous_eos: false,
reasoning: None,
reasoning_complete: true,
content: prompt.clone(),
content: model_prompt,
});
#[cfg(target_os = "macos")]
@@ -374,6 +444,9 @@ impl App {
fn system_prompt_reminders(&self, model: ModelChoice) -> Vec<String> {
let mut reminders = vec![crate::agent::system_prompt_reminder(model)];
if self.config.a2ui_enabled {
reminders.push(crate::a2ui::SYSTEM_PROMPT.to_owned());
}
if !self.config.generation.system_prompt.trim().is_empty() {
reminders.push(self.config.generation.system_prompt.clone());
}
@@ -475,6 +548,10 @@ impl App {
#[cfg(target_os = "macos")]
let mut start_queued = false;
#[cfg(target_os = "macos")]
let mut a2ui_feedback = None;
#[cfg(target_os = "macos")]
let mut a2ui_changed = false;
#[cfg(target_os = "macos")]
loop {
match active.events.try_recv() {
Ok(GenerationEvent::Loading) => {}
@@ -500,6 +577,18 @@ impl App {
}
transcript_changed = true;
}
if !reasoning
&& self.config.a2ui_enabled
&& let Some(message) = self.conversation.last_mut()
{
sync_a2ui_message(
&mut self.a2ui,
&mut self.database,
self.selected_session,
message,
);
a2ui_changed = true;
}
}
Ok(GenerationEvent::Context {
used,
@@ -530,6 +619,69 @@ impl App {
!self.queued_inputs.is_empty() || self.manual_compaction_queued;
}
Ok(_) => {
let (validation_errors, replies, open_urls, error_surface_id) = self
.conversation
.last_mut()
.map(|message| {
let surface_id = crate::a2ui::extract_lines(&message.content)
.into_iter()
.rev()
.filter_map(|line| line.value.ok())
.find_map(|value| {
crate::a2ui::message_surface_id(&value)
.map(str::to_owned)
})
.unwrap_or_else(|| "unknown".to_owned());
(
std::mem::take(&mut message.a2ui_errors),
std::mem::take(&mut message.a2ui_replies),
std::mem::take(&mut message.a2ui_open_urls),
surface_id,
)
})
.unwrap_or_default();
for url in open_urls {
if let Err(error) =
std::process::Command::new("open").arg(url).spawn()
{
self.error =
Some(format!("Could not open the A2UI link: {error}"));
}
}
if !validation_errors.is_empty() {
self.generating = false;
self.activity = Some("Correcting A2UI…".into());
self.tool_cards.clear();
a2ui_feedback = Some(
serde_json::json!({
"version": crate::a2ui::VERSION,
"error": {
"code": "VALIDATION_FAILED",
"surfaceId": error_surface_id,
"path": "/",
"message": validation_errors.join("; ")
}
})
.to_string(),
);
self.active_generation = None;
break;
}
if !replies.is_empty() {
self.generating = false;
self.activity = Some("Continuing A2UI function call…".into());
self.tool_cards.clear();
a2ui_feedback = Some(format!(
"A2UI client response:\n{}",
replies
.iter()
.map(ToString::to_string)
.collect::<Vec<_>>()
.join("\n")
));
self.active_generation = None;
break;
}
let model = self.config.model;
let content = self
.conversation
@@ -586,6 +738,10 @@ impl App {
}
}
#[cfg(target_os = "macos")]
if a2ui_changed {
self.sync_a2ui_renderer_state();
}
#[cfg(target_os = "macos")]
if transcript_changed && let Some(message) = self.conversation.last_mut() {
message.refresh_markdown();
}
@@ -633,6 +789,14 @@ impl App {
self.start_next_queued();
}
#[cfg(target_os = "macos")]
if let Some(feedback) = a2ui_feedback
&& let Err(error) = self.continue_after_tool_result(&feedback)
{
self.generating = false;
self.activity = Some("Failed".into());
self.error = Some(error);
}
#[cfg(target_os = "macos")]
return transcript_changed;
#[cfg(not(target_os = "macos"))]
false
@@ -698,7 +862,7 @@ impl App {
&models_path(),
)?;
effective.turn.system_prompt =
crate::agent::system_prompt(model, &effective.turn.system_prompt);
self.chat_system_prompt(model, &effective.turn.system_prompt);
effective.turn.system_prompt = crate::compaction::summary_system_prompt(
&effective.turn.system_prompt,
self.compaction_summary(),
@@ -792,7 +956,7 @@ impl App {
&models_path(),
)?;
effective.turn.system_prompt =
crate::agent::system_prompt(model, &effective.turn.system_prompt);
self.chat_system_prompt(model, &effective.turn.system_prompt);
effective.turn.system_prompt = crate::compaction::summary_system_prompt(
&effective.turn.system_prompt,
self.compaction_summary(),
@@ -941,7 +1105,7 @@ impl App {
&models_path(),
)?;
effective.turn.system_prompt =
crate::agent::system_prompt(model, &effective.turn.system_prompt);
self.chat_system_prompt(model, &effective.turn.system_prompt);
let rebuild_system_prompt = effective.turn.system_prompt.clone();
effective.turn.system_prompt = crate::compaction::summary_system_prompt(
&effective.turn.system_prompt,
@@ -1364,6 +1528,10 @@ mod tests {
content: "### Core / Setup\n\n| File | Lines |\n|---|---:|\n| `src/app.rs` | **1,750** |\n| `src/engine.rs` | 2,400 |\n\n### Summary\n\nDone."
.to_owned(),
markdown: iced::widget::markdown::Content::new(),
a2ui_lines_processed: 0,
a2ui_errors: Vec::new(),
a2ui_replies: Vec::new(),
a2ui_open_urls: Vec::new(),
};
message.refresh_markdown();
@@ -1394,6 +1562,10 @@ mod tests {
reasoning_open: false,
content: format!("message {id}"),
markdown: iced::widget::markdown::Content::new(),
a2ui_lines_processed: 0,
a2ui_errors: Vec::new(),
a2ui_replies: Vec::new(),
a2ui_open_urls: Vec::new(),
};
let history = vec![
message(1, false, None),
@@ -1426,6 +1598,10 @@ mod tests {
reasoning_open: false,
content: format!("message {id}"),
markdown: iced::widget::markdown::Content::new(),
a2ui_lines_processed: 0,
a2ui_errors: Vec::new(),
a2ui_replies: Vec::new(),
a2ui_open_urls: Vec::new(),
};
let mut history = vec![
message(1, true, false, false, false),

3
src/app/preferences.rs
View File

@@ -6,6 +6,7 @@ pub(super) struct PreferenceDraft {
pub(super) legacy_mtp_enabled: bool,
pub(super) dspark_enabled: bool,
pub(super) idle_timeout_minutes: String,
pub(super) a2ui_enabled: bool,
pub(super) endpoint_port: String,
pub(super) endpoint_enabled: bool,
pub(super) endpoint_cors: bool,
@@ -55,6 +56,7 @@ impl PreferenceDraft {
legacy_mtp_enabled: speculative.legacy_mtp_enabled,
dspark_enabled: speculative.dspark_enabled,
idle_timeout_minutes: config.idle_timeout_minutes.to_string(),
a2ui_enabled: config.a2ui_enabled,
endpoint_port: config.endpoint.port.to_string(),
endpoint_enabled: config.endpoint.enabled,
endpoint_cors: config.endpoint.cors,
@@ -350,6 +352,7 @@ impl App {
let config = Config {
model: self.preference_draft.model,
idle_timeout_minutes,
a2ui_enabled: self.preference_draft.a2ui_enabled,
endpoint: EndpointConfig {
port: i32::from(endpoint_port),
enabled: self.preference_draft.endpoint_enabled,

76
src/app/projects.rs
View File

@@ -1,6 +1,79 @@
use super::*;
impl App {
pub(super) fn clear_a2ui(&mut self) {
self.a2ui.clear();
self.a2ui_tabs.clear();
self.a2ui_modals.clear();
self.a2ui_editors.clear();
self.a2ui_markdown.clear();
self.a2ui_choice_filters.clear();
self.a2ui_images.clear();
self.a2ui_image_requests.clear();
self.a2ui_image_loading = false;
}
pub(super) fn load_next_a2ui_image(&mut self) -> Task<Message> {
if self.a2ui_image_loading {
return Task::none();
}
let Some(url) = self
.a2ui
.image_urls()
.find(|url| !self.a2ui_image_requests.contains(url))
else {
return Task::none();
};
self.a2ui_image_requests.insert(url.clone());
self.a2ui_image_loading = true;
let request_url = url.clone();
Task::perform(
async move {
let mut response = ureq::get(&request_url)
.call()
.map_err(|error| error.to_string())?;
if !response
.headers()
.get("content-type")
.and_then(|value| value.to_str().ok())
.is_some_and(|value| value.starts_with("image/"))
{
return Err("the URL did not return an image".to_owned());
}
response
.body_mut()
.read_to_vec()
.map_err(|error| error.to_string())
},
move |result| Message::A2uiImageLoaded(url, result),
)
}
pub(super) fn change_a2ui_data(
&mut self,
surface_id: String,
path: String,
value: serde_json::Value,
) -> Task<Message> {
let owner = self
.a2ui
.surface(&surface_id)
.map(|surface| surface.owner_message_id);
match self.a2ui.local_update(&surface_id, &path, value) {
Ok(raw) => {
if let (Some(session_id), Some(message_id), Some(database)) =
(self.selected_session, owner, &mut self.database)
&& let Err(error) = database.insert_a2ui_message(session_id, message_id, &raw)
{
self.error = Some(format!("Could not save the A2UI edit: {error}"));
}
}
Err(error) => self.error = Some(error),
}
self.sync_a2ui_renderer_state();
self.load_next_a2ui_image()
}
pub(super) fn prepare_project(&mut self, path: PathBuf) {
let Ok(path) = fs::canonicalize(path) else {
self.error = Some("The selected folder is no longer available.".into());
@@ -49,6 +122,7 @@ impl App {
Ok(project) => {
self.remember_project(project.id);
self.selected_session = None;
self.clear_a2ui();
self.system_prompt_seen_at = 0;
self.pending_project_path = None;
self.project_name_input.clear();
@@ -78,6 +152,7 @@ impl App {
self.remember_project(project_id);
self.selected_session = None;
self.conversation.clear();
self.clear_a2ui();
self.composer.clear();
self.queued_inputs.clear();
self.system_prompt_seen_at = 0;
@@ -90,6 +165,7 @@ impl App {
pub(super) fn discard_session(&mut self, project_id: i32) {
if self.drafts.remove(&project_id).is_some() && self.draft_selected(project_id) {
self.conversation.clear();
self.clear_a2ui();
self.composer.clear();
self.queued_inputs.clear();
self.system_prompt_seen_at = 0;

11
src/app/view.rs
View File

@@ -1,3 +1,4 @@
mod a2ui;
mod chat;
mod model_manager;
mod preferences;
@@ -16,8 +17,9 @@ use crate::model::{
use crate::settings::{GIB, REASONING_MODES};
use iced::theme::{Palette, palette};
use iced::widget::{
Button, Space, Svg, Tooltip, button, checkbox, column, container, markdown, mouse_area, opaque,
pick_list, progress_bar, row, rule, scrollable, stack, svg, text, text_input, tooltip,
Button, Space, Svg, Tooltip, button, checkbox, column, container, image, markdown, mouse_area,
opaque, pick_list, progress_bar, row, rule, scrollable, slider, stack, svg, text, text_input,
tooltip,
};
use iced::{Alignment, Background, Border, Color, Element, Length, Padding, Theme, window};
use std::collections::VecDeque;
@@ -68,6 +70,7 @@ impl App {
|| self.pending_project_path.is_some()
|| self.session_rename.is_some()
|| self.menu_session().is_some()
|| !self.a2ui_modals.is_empty()
|| {
#[cfg(target_os = "macos")]
{
@@ -132,6 +135,8 @@ impl App {
layers.push(self.rename_dialog(title));
} else if let Some(session) = self.menu_session() {
layers.push(self.session_menu_panel(session));
} else if let Some(panel) = self.a2ui_modal_panel() {
layers.push(panel);
}
#[cfg(not(target_os = "macos"))]
if self.preferences_open {
@@ -142,6 +147,8 @@ impl App {
layers.push(self.rename_dialog(title));
} else if let Some(session) = self.menu_session() {
layers.push(self.session_menu_panel(session));
} else if let Some(panel) = self.a2ui_modal_panel() {
layers.push(panel);
}
stack(layers)

1909
src/app/view/a2ui.rs Normal file
View File

File diff suppressed because it is too large Load Diff

3
src/app/view/chat.rs
View File

@@ -179,6 +179,9 @@ impl App {
if !cards.is_empty() {
body = body.push(tool_cards(cards));
}
if self.a2ui.surfaces_for_message(message.id).next().is_some() {
body = body.push(self.a2ui_surfaces(message.id));
}
}
let user = message.user;
messages = messages.push(

6
src/app/view/preferences.rs
View File

@@ -119,6 +119,12 @@ impl App {
.spacing(10)
.align_y(Alignment::Center),
text("Enter a whole number from 1 to 1440.").size(12),
hint(
checkbox(self.preference_draft.a2ui_enabled)
.label("Enable interactive A2UI chat surfaces")
.on_toggle(Message::PreferenceA2uiChanged),
"Lets the local model build validated native charts, tables, forms and other interactive chat surfaces. Turning it off removes the A2UI catalog from the system prompt.",
),
]
.spacing(10),
);