hugo/DS4Server
1
0
Fork 0
Code Issues 7 Pull Requests Actions 1 Packages Projects Releases Wiki Activity

feat: a2ui interface to enablee the LLM to give structured information

Browse Source
This commit is contained in:
Georg Bauer
2026-07-27 11:19:57 +02:00
parent 27b10ee0e1
commit c9c2d8efd5
24 changed files with 5322 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files

145
src/app.rs
View File

@@ -25,7 +25,7 @@ use crate::settings::{
ReasoningMode, RuntimePreferences, SpeculativePreferences, SsdPreferences, SteeringPreferences,
StreamingCacheBudget,
};
use iced::widget::{markdown, scrollable};
use iced::widget::{markdown, scrollable, text_editor};
use iced::{Size, Subscription, Task, keyboard, mouse, window};
use rfd::AsyncFileDialog;
use std::collections::{HashMap, HashSet, VecDeque};
@@ -78,6 +78,15 @@ pub(crate) struct App {
pub(super) composer: String,
pub(super) queued_inputs: VecDeque<String>,
pub(super) conversation: Vec<ChatMessage>,
pub(super) a2ui: crate::a2ui::Store,
pub(super) a2ui_tabs: HashMap<(String, String), usize>,
pub(super) a2ui_modals: HashSet<(String, String)>,
pub(super) a2ui_editors: HashMap<(String, String, String), text_editor::Content>,
pub(super) a2ui_markdown: HashMap<(String, String, String), markdown::Content>,
pub(super) a2ui_choice_filters: HashMap<(String, String, String), String>,
pub(super) a2ui_images: HashMap<String, iced::widget::image::Handle>,
pub(super) a2ui_image_requests: HashSet<String>,
pub(super) a2ui_image_loading: bool,
pub(super) generating: bool,
pub(super) context_used: u32,
pub(super) context_limit: u32,
@@ -163,6 +172,7 @@ pub(crate) enum Message {
PreferenceLegacyMtpChanged(bool),
PreferenceDsparkChanged(bool),
PreferenceTimeoutChanged(String),
PreferenceA2uiChanged(bool),
PreferenceEndpointPortChanged(String),
PreferenceEndpointEnabledChanged(bool),
PreferenceEndpointCorsChanged(bool),
@@ -199,6 +209,14 @@ pub(crate) enum Message {
PreferenceKvMinTokensChanged(String),
PreferenceKvColdMaxChanged(String),
PreferenceKvContinuedIntervalChanged(String),
A2uiDataChanged(String, String, serde_json::Value),
A2uiEditorAction(String, String, String, text_editor::Action),
A2uiChoiceFilterChanged(String, String, String, String),
A2uiAction(String, String, Option<String>),
A2uiSelectTab(String, String, usize),
A2uiToggleModal(String, String),
A2uiImageLoaded(String, Result<Vec<u8>, String>),
A2uiPlayMedia(String, String, bool),
ResetPreferences,
SavePreferences,
DownloadArtifact(ManagedArtifactId),
@@ -309,6 +327,15 @@ impl App {
composer: String::new(),
queued_inputs: VecDeque::new(),
conversation: Vec::new(),
a2ui: crate::a2ui::Store::default(),
a2ui_tabs: HashMap::new(),
a2ui_modals: HashSet::new(),
a2ui_editors: HashMap::new(),
a2ui_markdown: HashMap::new(),
a2ui_choice_filters: HashMap::new(),
a2ui_images: HashMap::new(),
a2ui_image_requests: HashSet::new(),
a2ui_image_loading: false,
generating: false,
context_used: 0,
context_limit,
@@ -415,6 +442,15 @@ impl App {
composer: String::new(),
queued_inputs: VecDeque::new(),
conversation: Vec::new(),
a2ui: crate::a2ui::Store::default(),
a2ui_tabs: HashMap::new(),
a2ui_modals: HashSet::new(),
a2ui_editors: HashMap::new(),
a2ui_markdown: HashMap::new(),
a2ui_choice_filters: HashMap::new(),
a2ui_images: HashMap::new(),
a2ui_image_requests: HashSet::new(),
a2ui_image_loading: false,
generating: false,
context_used: 0,
context_limit,
@@ -593,6 +629,10 @@ impl App {
self.preference_draft.idle_timeout_minutes = value;
self.preference_error = None;
}
Message::PreferenceA2uiChanged(enabled) => {
self.preference_draft.a2ui_enabled = enabled;
self.preference_error = None;
}
Message::PreferenceEndpointPortChanged(value) => {
self.preference_draft.endpoint_port = value;
self.preference_error = None;
@@ -802,6 +842,71 @@ impl App {
}
Message::DownloadProgressTick => self.update_download_progress(),
Message::ComposerChanged(value) => self.composer = value,
Message::A2uiDataChanged(surface_id, path, value) => {
return self.change_a2ui_data(surface_id, path, value);
}
Message::A2uiEditorAction(surface_id, component_id, path, action) => {
let key = (surface_id.clone(), component_id, path.clone());
let Some(editor) = self.a2ui_editors.get_mut(&key) else {
return Task::none();
};
editor.perform(action);
let value = serde_json::Value::String(editor.text());
return self.change_a2ui_data(surface_id, path, value);
}
Message::A2uiChoiceFilterChanged(surface_id, component_id, context_path, value) => {
self.a2ui_choice_filters
.insert((surface_id, component_id, context_path), value);
}
Message::A2uiAction(surface_id, component_id, context_path) => {
match self
.a2ui
.action(&surface_id, &component_id, context_path.as_deref())
{
Ok(action) => {
self.composer = format!(
"A2UI client event:\n{}\nA2UI client metadata:\n{}",
serde_json::to_string(&action).unwrap_or_default(),
self.a2ui.client_metadata()
);
self.start_generation();
return scroll_chat_to_end();
}
Err(error) => self.error = Some(error),
}
}
Message::A2uiSelectTab(surface_id, component_id, index) => {
self.a2ui_tabs.insert((surface_id, component_id), index);
}
Message::A2uiToggleModal(surface_id, component_id) => {
let key = (surface_id, component_id);
if !self.a2ui_modals.remove(&key) {
self.a2ui_modals.insert(key);
}
}
Message::A2uiImageLoaded(url, result) => {
self.a2ui_image_loading = false;
match result {
Ok(bytes) => {
self.a2ui_images
.insert(url, iced::widget::image::Handle::from_bytes(bytes));
}
Err(error) => {
self.error = Some(format!("Could not load an A2UI image: {error}"))
}
}
return self.load_next_a2ui_image();
}
Message::A2uiPlayMedia(url, title, video) => {
#[cfg(target_os = "macos")]
if let Err(error) = crate::native_media::open(&url, &title, video) {
self.error = Some(format!("Could not play media: {error}"));
}
#[cfg(not(target_os = "macos"))]
if let Err(error) = std::process::Command::new("open").arg(url).spawn() {
self.error = Some(format!("Could not open media: {error}"));
}
}
Message::ToggleReasoning(index) => {
if let Some(message) = self.conversation.get_mut(index)
&& message.reasoning.is_some()
@@ -869,9 +974,12 @@ impl App {
Message::GenerationTick => {
#[cfg(target_os = "macos")]
self.poll_titling();
if self.poll_generation() {
return scroll_chat_to_end();
let changed = self.poll_generation();
let images = self.load_next_a2ui_image();
if changed {
return Task::batch([scroll_chat_to_end(), images]);
}
return images;
}
Message::ChooseProjectFolder => {
self.choosing_folder = true;
@@ -948,6 +1056,7 @@ impl App {
self.selected_project = None;
self.selected_session = None;
self.conversation.clear();
self.clear_a2ui();
self.composer.clear();
self.queued_inputs.clear();
self.system_prompt_seen_at = 0;
@@ -1102,9 +1211,26 @@ impl App {
let Some(database) = &mut self.database else {
return Task::none();
};
match database.load_messages(session_id) {
Ok(messages) => {
let loaded = database.load_messages(session_id).and_then(|messages| {
database
.load_a2ui_messages(session_id)
.map(|a2ui| (messages, a2ui))
});
match loaded {
Ok((messages, a2ui)) => {
self.conversation = messages.into_iter().map(ChatMessage::from).collect();
self.clear_a2ui();
for message in a2ui {
if let Err(error) =
self.a2ui.apply_raw(&message.json, message.message_id)
{
self.error = Some(format!(
"Could not restore A2UI message {}: {error}",
message.id
));
}
}
self.sync_a2ui_renderer_state();
self.composer.clear();
self.remember_project(project_id);
self.selected_session = Some(session_id);
@@ -1119,7 +1245,7 @@ impl App {
};
self.tokens_per_second = tokens_per_second;
self.error = None;
return scroll_chat_to_end();
return Task::batch([scroll_chat_to_end(), self.load_next_a2ui_image()]);
}
Err(error) => {
self.error = Some(format!("Could not load the chat session: {error}"));
@@ -1150,6 +1276,7 @@ impl App {
if self.selected_session == Some(session_id) {
self.selected_session = None;
self.conversation.clear();
self.clear_a2ui();
self.composer.clear();
self.queued_inputs.clear();
self.system_prompt_seen_at = 0;
@@ -1424,7 +1551,7 @@ fn models_path() -> PathBuf {
}
/// The settings file, beside the project database.
fn config_path() -> PathBuf {
pub(crate) fn config_path() -> PathBuf {
application_support_path().join("config.yaml")
}
@@ -1736,6 +1863,10 @@ mod tests {
reasoning_open: true,
content: String::new(),
markdown: markdown::Content::new(),
a2ui_lines_processed: 0,
a2ui_errors: Vec::new(),
a2ui_replies: Vec::new(),
a2ui_open_urls: Vec::new(),
};
message.append(true, "working it out");
message.append(false, "**final answer**");