fix: hopefully fixed the background task progress situation

This commit is contained in:
2026-07-23 22:48:53 +02:00
parent ea90167a5a
commit 87318c8b22
16 changed files with 225 additions and 42 deletions

View File

@@ -22,7 +22,7 @@ The project is under active development. Core blogging workflows are broadly ava
- Local MCP automation over stdio or a localhost-only stateless HTTP endpoint, with project resources, read/search/count tools, uniquely identified inert write proposals, clean duplicate-pending rejection, explicit desktop approval, and opt-in Claude Code/Copilot configuration.
- A fully localized Ratatui terminal workspace, available locally through `bds-cli tui`/`BDS_MODE=tui` and remotely through authenticated SSH shell sessions, with shared post/template/script editing and publishing, project/search/command overlays, settings, tags, Git, reports, task progress, live multi-client locale updates, and airplane-mode AI gating.
- `bds-cli server` hosting the shared application engines over a loopback-by-default, public-key-only SSH service, with restrictive private key material, live authorization updates, terminal-session transport, CLI-change synchronization, ordered domain/task events, and native desktop remote-project selection.
- bDS2-compatible Markdown/Liquid rendering with built-in macros rendered from bundled Liquid templates in isolated scopes (customizable with `macros/*` partial-template slugs), category-controlled list title visibility, descriptive category archive titles, canonical multilingual and flat page routes for every configured blog language, recursive menus, calendar archives, feeds, a root hreflang sitemap, Pagefind, change-aware and forced full renders from the native Blog menu/CLI/TUI, and fast route/mtime-based incremental validation whose targeted repair refreshes affected aggregate pages through cancellable section task groups.
- bDS2-compatible Markdown/Liquid rendering with built-in macros rendered from bundled Liquid templates in isolated scopes (customizable with `macros/*` partial-template slugs), category-controlled list title visibility, descriptive category archive titles, canonical multilingual and flat page routes for every configured blog language, recursive menus, calendar archives, feeds, a root hreflang sitemap, Pagefind, change-aware and forced full renders from the native Blog menu/CLI/TUI, and fast route/mtime-based incremental validation whose targeted repair refreshes affected aggregate pages through cancellable section task groups with bDS2-style per-URL progress.
- Navigable generated-route preview in the app or system browser, with draft database overlays and published filesystem content.
- Optional one-shot AI translation, description, analysis, taxonomy, and language-detection operations run in background tasks with editor-level waiting indicators, using provider-portable JSON-only requests through independent online and local OpenAI-compatible profiles. Each profile has secure credentials, persistently discovered chat/title/image model selections, explicit tool/vision overrides, chat testing, and restart-persistent status-bar airplane-mode routing.
- Persistent conversational AI with safe Markdown, streamed and cancellable responses, model/session/token tracking, bounded project-aware blog tools, and localized conversation management in the Chat workspace. Allowlisted render tools add persistent native cards, charts, forms, lists, metrics, mind maps, tables, and tabs without executing assistant-provided HTML or JavaScript.

View File

@@ -161,6 +161,7 @@ fn generate_starter_site_with_progress_mode(
posts,
section,
force,
&|_| {},
&mut on_page,
|| false,
)?);
@@ -189,6 +190,7 @@ pub fn render_site_section_with_progress(
posts: &[PublishedPostSource],
section: GenerationSection,
mut on_page: impl FnMut(usize, usize, &str),
on_page_rendered: impl Fn(&str) + Sync,
mut is_cancelled: impl FnMut() -> bool,
) -> EngineResult<GenerationReport> {
render_site_section_with_progress_mode(
@@ -199,6 +201,7 @@ pub fn render_site_section_with_progress(
posts,
section,
false,
&on_page_rendered,
&mut on_page,
&mut is_cancelled,
)
@@ -216,6 +219,7 @@ pub fn render_site_section_forced_with_progress(
posts: &[PublishedPostSource],
section: GenerationSection,
mut on_page: impl FnMut(usize, usize, &str),
on_page_rendered: impl Fn(&str) + Sync,
mut is_cancelled: impl FnMut() -> bool,
) -> EngineResult<GenerationReport> {
render_site_section_with_progress_mode(
@@ -226,6 +230,7 @@ pub fn render_site_section_forced_with_progress(
posts,
section,
true,
&on_page_rendered,
&mut on_page,
&mut is_cancelled,
)
@@ -243,6 +248,7 @@ fn render_site_section_with_progress_mode(
posts: &[PublishedPostSource],
section: GenerationSection,
force: bool,
on_page_rendered: &(dyn Fn(&str) + Sync),
mut on_page: impl FnMut(usize, usize, &str),
mut is_cancelled: impl FnMut() -> bool,
) -> EngineResult<GenerationReport> {
@@ -261,6 +267,7 @@ fn render_site_section_with_progress_mode(
metadata,
&input_posts,
section,
on_page_rendered,
)
.map_err(|error| EngineError::Parse(error.to_string()))?;
let mut report = GenerationReport::default();
@@ -361,6 +368,7 @@ pub fn apply_validation_sections(
validation,
*section,
|_current, _total, _url| {},
|_| {},
|| false,
)?);
}
@@ -384,6 +392,7 @@ pub fn apply_validation_section_with_progress(
validation: &SiteValidationReport,
section: GenerationSection,
mut on_page: impl FnMut(usize, usize, &str),
on_page_rendered: impl Fn(&str) + Sync,
mut is_cancelled: impl FnMut() -> bool,
) -> EngineResult<GenerationReport> {
if is_cancelled() {
@@ -414,6 +423,7 @@ pub fn apply_validation_section_with_progress(
metadata,
&input_posts,
section,
&on_page_rendered,
)
} else {
build_targeted_site_section_render_artifacts(
@@ -424,6 +434,7 @@ pub fn apply_validation_section_with_progress(
&input_posts,
section,
&requested,
&on_page_rendered,
)
}
.map_err(|error| EngineError::Parse(error.to_string()))?;

View File

@@ -23,7 +23,7 @@ pub use routes::{
pub use site::{
PagefindDocument, PreviewRenderResult, SitePage, SiteRenderArtifacts, build_preview_response,
build_site_render_artifacts, build_site_route_manifest, build_site_section_render_artifacts,
build_targeted_site_section_render_artifacts,
build_targeted_site_section_render_artifacts, estimate_site_render_pages,
};
pub use template_lookup::{
RenderCategorySettings, RenderTemplateLookup, TemplateLookupError, resolve_post_template,

View File

@@ -119,6 +119,7 @@ pub fn build_site_render_artifacts(
false,
None,
None,
&|_| {},
)
}
@@ -188,6 +189,7 @@ pub fn build_site_section_render_artifacts(
metadata: &ProjectMetadata,
published_posts: &[(Post, String)],
section: GenerationSection,
on_page_rendered: &(dyn Fn(&str) + Sync),
) -> Result<SiteRenderArtifacts, Box<dyn Error + Send + Sync>> {
build_site_render_artifacts_with_mode(
conn,
@@ -198,6 +200,7 @@ pub fn build_site_section_render_artifacts(
false,
Some(section),
None,
on_page_rendered,
)
}
@@ -209,6 +212,7 @@ pub fn build_targeted_site_section_render_artifacts(
published_posts: &[(Post, String)],
section: GenerationSection,
requested_paths: &HashSet<String>,
on_page_rendered: &(dyn Fn(&str) + Sync),
) -> Result<SiteRenderArtifacts, Box<dyn Error + Send + Sync>> {
build_site_render_artifacts_with_mode(
conn,
@@ -219,9 +223,28 @@ pub fn build_targeted_site_section_render_artifacts(
false,
Some(section),
Some(requested_paths),
on_page_rendered,
)
}
pub fn estimate_site_render_pages(
data_dir: &Path,
metadata: &ProjectMetadata,
published_posts: &[Post],
) -> Result<HashMap<GenerationSection, usize>, Box<dyn Error + Send + Sync>> {
let mut estimates = GenerationSection::ALL
.into_iter()
.map(|section| (section, 0))
.collect::<HashMap<_, _>>();
for page in build_site_route_manifest(data_dir, metadata, published_posts)? {
if let Some(section) = classify_generated_path(&page.relative_path, metadata) {
*estimates.get_mut(&section).unwrap() += 1;
}
}
*estimates.get_mut(&GenerationSection::Core).unwrap() += render_languages(metadata).len();
Ok(estimates)
}
#[expect(
clippy::too_many_arguments,
reason = "the shared renderer accepts optional section and path filters"
@@ -235,6 +258,7 @@ fn build_site_render_artifacts_with_mode(
is_preview: bool,
section: Option<GenerationSection>,
requested_paths: Option<&HashSet<String>>,
on_page_rendered: &(dyn Fn(&str) + Sync),
) -> Result<SiteRenderArtifacts, Box<dyn Error + Send + Sync>> {
let bundle = load_template_bundle(conn, data_dir, project_id)?;
let main_language = main_language(metadata).to_string();
@@ -316,11 +340,14 @@ fn build_site_render_artifacts_with_mode(
&bundle,
is_preview,
)
.map(|html| SitePage {
language: language.clone(),
relative_path: route.relative_path.clone(),
url_path: route.url_path.clone(),
html,
.map(|html| {
on_page_rendered(&route.url_path);
SitePage {
language: language.clone(),
relative_path: route.relative_path.clone(),
url_path: route.url_path.clone(),
html,
}
})
})
.collect::<Result<Vec<_>, _>>()?;
@@ -343,17 +370,14 @@ fn build_site_render_artifacts_with_mode(
};
if requested_paths.is_none_or(|requested| requested.contains(&relative_path)) {
let url_path = format!("/{}", relative_path.trim_end_matches(".html"));
let html =
render_not_found_route(&bundle, metadata, &language, &url_path, &menu_items)?;
on_page_rendered(&url_path);
artifacts.pages.push(SitePage {
language: language.clone(),
relative_path,
url_path: url_path.clone(),
html: render_not_found_route(
&bundle,
metadata,
&language,
&url_path,
&menu_items,
)?,
html,
});
}
}
@@ -410,6 +434,7 @@ fn build_site_render_artifacts_with_mode(
&bundle,
is_preview,
)?;
on_page_rendered(&url_path);
artifacts.pagefind_documents.push(PagefindDocument {
language: language.clone(),
relative_path: relative_path.clone(),
@@ -448,6 +473,7 @@ pub fn build_preview_response(
true,
None,
Some(&requested_paths),
&|_| {},
)?;
if let Some(page) = artifacts
.pages

View File

@@ -15,6 +15,7 @@ use bds_core::model::{
CategorySettings, Post, PostStatus, PostTranslation, Project, ProjectMetadata, Template,
TemplateKind, TemplateStatus,
};
use bds_core::render::estimate_site_render_pages;
use tempfile::TempDir;
fn make_project() -> Project {
@@ -435,6 +436,7 @@ fn section_generation_reports_its_urls_and_defers_pagefind() {
&posts,
GenerationSection::Single,
|current, total, url| urls.push((current, total, url.to_string())),
|_| {},
|| false,
)
.unwrap();
@@ -469,6 +471,7 @@ fn section_generation_reports_its_urls_and_defers_pagefind() {
&changed_posts,
GenerationSection::Single,
|_current, _total, _url| {},
|_| {},
|| false,
)
.unwrap();
@@ -477,6 +480,43 @@ fn section_generation_reports_its_urls_and_defers_pagefind() {
assert!(!dir.path().join(old_fragment).exists());
}
#[test]
fn section_generation_reports_while_html_is_rendered_before_files_are_written() {
let (db, dir) = setup();
let metadata = make_metadata();
let posts = vec![PublishedPostSource {
post: make_post("hello", 1_710_000_000_000),
body_markdown: "Hello **world**".into(),
}];
let output = dir.path().join("2024/03/09/hello/index.html");
let rendered_before_write = std::sync::atomic::AtomicBool::new(false);
let rendered = std::sync::atomic::AtomicUsize::new(0);
let estimated = estimate_site_render_pages(dir.path(), &metadata, &[posts[0].post.clone()])
.unwrap()[&GenerationSection::Single];
render_site_section_with_progress(
db.conn(),
dir.path(),
"p1",
&metadata,
&posts,
GenerationSection::Single,
|_current, _total, _url| {},
|_| {
rendered.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
rendered_before_write.store(!output.exists(), std::sync::atomic::Ordering::Relaxed);
},
|| false,
)
.unwrap();
assert!(rendered_before_write.load(std::sync::atomic::Ordering::Relaxed));
assert_eq!(
rendered.load(std::sync::atomic::Ordering::Relaxed),
estimated
);
}
#[test]
fn validation_apply_rewrites_only_the_reported_urls() {
let (db, dir) = setup();
@@ -508,6 +548,7 @@ fn validation_apply_rewrites_only_the_reported_urls() {
&validation,
GenerationSection::Single,
|current, total, url| urls.push((current, total, url.to_string())),
|_| {},
|| false,
)
.unwrap();
@@ -599,6 +640,7 @@ fn page_aliases_belong_to_core_while_dated_posts_belong_to_single() {
&posts,
GenerationSection::Single,
|_current, _total, _url| {},
|_| {},
|| false,
)
.unwrap();
@@ -613,6 +655,7 @@ fn page_aliases_belong_to_core_while_dated_posts_belong_to_single() {
&posts,
GenerationSection::Core,
|_current, _total, _url| {},
|_| {},
|| false,
)
.unwrap();

View File

@@ -11217,6 +11217,13 @@ mod tests {
.collect::<Vec<_>>(),
vec![Running, Running, Running, Pending, Pending]
);
assert!(snapshots[..3].iter().all(|task| {
task.progress.is_some()
&& task
.message
.as_deref()
.is_some_and(|message| message.contains("0/"))
}));
}
#[test]

View File

@@ -105,6 +105,26 @@ impl BdsApp {
let project_id = project.id.clone();
let db_path = self.db_path.clone();
let data_dir = data_dir.clone();
let published_posts = match self.db.as_ref().and_then(|db| {
bds_core::db::queries::post::list_posts_by_project(db.conn(), &project_id).ok()
}) {
Some(posts) => posts
.into_iter()
.filter(engine::generation::has_published_snapshot)
.collect::<Vec<_>>(),
None => return Task::none(),
};
let page_estimates = match bds_core::render::estimate_site_render_pages(
&data_dir,
&metadata,
&published_posts,
) {
Ok(estimates) => estimates,
Err(error) => {
self.notify(ToastLevel::Error, &error.to_string());
return Task::none();
}
};
let group_id = format!("site-generation:{}", Uuid::new_v4());
let group_name = t(
self.ui_locale,
@@ -120,9 +140,19 @@ impl BdsApp {
for section in sections {
let label = t(self.ui_locale, generation_section_label_key(section));
self.add_output(&label);
let page_work = page_estimates[&section];
let task_id = self
.task_manager
.submit_grouped(&label, &group_id, &group_name);
self.task_manager.report_progress(
task_id,
Some(if page_work == 0 { 1.0 } else { 0.0 }),
Some(tw(
self.ui_locale,
"engine.renderingPages",
&[("current", "0"), ("total", &page_work.to_string())],
)),
);
render_task_ids.push(task_id);
let task_manager = Arc::clone(&self.task_manager);
let task_db_path = db_path.clone();
@@ -144,6 +174,7 @@ impl BdsApp {
task_validation,
force,
locale,
page_work,
)
})
.await
@@ -452,10 +483,20 @@ fn run_site_generation_section(
validation: Option<engine::validate_site::SiteValidationReport>,
force: bool,
locale: UiLocale,
expected_pages: usize,
) -> Result<engine::generation::GenerationReport, String> {
if !task_manager.wait_until_runnable(task_id) {
return Err("cancelled".to_string());
}
task_manager.report_progress(
task_id,
Some(if expected_pages == 0 { 1.0 } else { 0.0 }),
Some(tw(
locale,
"engine.renderingPages",
&[("current", "0"), ("total", &expected_pages.to_string())],
)),
);
let db = Database::open(&db_path).map_err(|error| error.to_string())?;
let metadata = engine::meta::read_project_json(&data_dir).map_err(|error| error.to_string())?;
let posts = bds_core::db::queries::post::list_posts_by_project(db.conn(), &project_id)
@@ -476,29 +517,27 @@ fn run_site_generation_section(
}
let output_dir = data_dir.join("html");
std::fs::create_dir_all(&output_dir).map_err(|error| error.to_string())?;
let progress_manager = Arc::clone(&task_manager);
let render_manager = Arc::clone(&task_manager);
let cancel_manager = Arc::clone(&task_manager);
let progress_key = if validation.is_some() {
"engine.rewrotePage"
} else {
"engine.generatedPage"
};
let on_page = move |current: usize, total: usize, url: &str| {
let progress = if total == 0 {
1.0
} else {
current as f32 / total as f32
};
progress_manager.report_progress(
let rendered = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let rendered_count = Arc::clone(&rendered);
let on_page = |_current: usize, _total: usize, _url: &str| {};
let on_rendered = move |url: &str| {
let current = rendered_count.fetch_add(1, std::sync::atomic::Ordering::Relaxed) + 1;
render_manager.report_progress(
task_id,
Some(progress),
Some(if expected_pages == 0 {
1.0
} else {
current.min(expected_pages) as f32 / expected_pages as f32
}),
Some(tw(
locale,
progress_key,
"engine.renderingPage",
&[
("url", url),
("current", &current.to_string()),
("total", &total.to_string()),
("total", &expected_pages.to_string()),
],
)),
);
@@ -514,6 +553,7 @@ fn run_site_generation_section(
&validation,
section,
on_page,
on_rendered,
is_cancelled,
),
None if force => engine::generation::render_site_section_forced_with_progress(
@@ -524,6 +564,7 @@ fn run_site_generation_section(
&sources,
section,
on_page,
on_rendered,
is_cancelled,
),
None => engine::generation::render_site_section_with_progress(
@@ -534,6 +575,7 @@ fn run_site_generation_section(
&sources,
section,
on_page,
on_rendered,
is_cancelled,
),
}

View File

@@ -42,6 +42,22 @@ fn task_row(snapshot: &TaskSnapshot, locale: UiLocale) -> Element<'static, Messa
content.into()
}
fn group_progress(members: &[&TaskSnapshot]) -> Option<f32> {
(!members.is_empty()).then(|| {
members
.iter()
.map(|member| {
if member.is_cancellable {
member.progress.unwrap_or(0.0)
} else {
1.0
}
})
.sum::<f32>()
/ members.len() as f32
})
}
/// Panel background style.
fn panel_style(_theme: &Theme) -> container::Style {
container::Style {
@@ -96,6 +112,33 @@ fn close_btn_style(_theme: &Theme, status: button::Status) -> button::Style {
}
}
#[cfg(test)]
mod progress_tests {
use super::*;
#[test]
fn group_progress_includes_pending_tasks_as_zero_like_bds2() {
let complete = TaskSnapshot {
id: 1,
label: String::new(),
group_id: None,
group_name: None,
status: String::new(),
progress: Some(1.0),
message: None,
is_cancellable: false,
};
let pending = TaskSnapshot {
id: 2,
progress: None,
is_cancellable: true,
..complete.clone()
};
assert_eq!(group_progress(&[&complete, &pending]), Some(0.5));
}
}
#[expect(
clippy::too_many_arguments,
reason = "arguments are independent panel state slices"
@@ -219,17 +262,8 @@ pub fn view(
.copied()
.filter(|member| member.group_id.as_ref() == Some(group_id))
.collect::<Vec<_>>();
let progress_values = members
.iter()
.filter_map(|member| member.progress)
.collect::<Vec<_>>();
let progress = (!progress_values.is_empty()).then(|| {
format!(
" ({:.0}%)",
progress_values.iter().sum::<f32>() / progress_values.len() as f32
* 100.0
)
});
let progress = group_progress(&members)
.map(|progress| format!(" ({:.0}%)", progress * 100.0));
let collapsed = collapsed_task_groups.contains(group_id);
let group_name = snapshot.group_name.as_deref().unwrap_or(group_id);
items.push(

View File

@@ -674,6 +674,8 @@ blogmark-noProject = Öffnen Sie vor dem Import eines Blogmarks ein Projekt.
blogmark-importing = Blogmark wird importiert
blogmark-imported = Blogmark importiert
blogmark-failed = Blogmark-Import fehlgeschlagen: { $error }
engine-renderingPage = { $url } wird gerendert ({ $current }/{ $total })
engine-renderingPages = Seiten werden gerendert ({ $current }/{ $total })
engine-generatedPage = Erstellt { $url } ({ $current }/{ $total })
engine-rewrotePage = Neu geschrieben { $url } ({ $current }/{ $total })
engine-renderSiteGroup = Website rendern

View File

@@ -674,6 +674,8 @@ blogmark-noProject = Open a project before importing a blogmark.
blogmark-importing = Importing blogmark
blogmark-imported = Blogmark imported
blogmark-failed = Blogmark import failed: { $error }
engine-renderingPage = Rendering { $url } ({ $current }/{ $total })
engine-renderingPages = Rendering pages ({ $current }/{ $total })
engine-generatedPage = Generated { $url } ({ $current }/{ $total })
engine-rewrotePage = Rewrote { $url } ({ $current }/{ $total })
engine-renderSiteGroup = Render Site

View File

@@ -674,6 +674,8 @@ blogmark-noProject = Abre un proyecto antes de importar un blogmark.
blogmark-importing = Importando blogmark
blogmark-imported = Blogmark importado
blogmark-failed = Error al importar el blogmark: { $error }
engine-renderingPage = Renderizando { $url } ({ $current }/{ $total })
engine-renderingPages = Renderizando páginas ({ $current }/{ $total })
engine-generatedPage = Generado { $url } ({ $current }/{ $total })
engine-rewrotePage = Reescrito { $url } ({ $current }/{ $total })
engine-renderSiteGroup = Renderizar sitio

View File

@@ -674,6 +674,8 @@ blogmark-noProject = Ouvrez un projet avant dimporter un blogmark.
blogmark-importing = Importation du blogmark
blogmark-imported = Blogmark importé
blogmark-failed = Échec de limport du blogmark : { $error }
engine-renderingPage = Rendu de { $url } ({ $current }/{ $total })
engine-renderingPages = Rendu des pages ({ $current }/{ $total })
engine-generatedPage = Généré { $url } ({ $current }/{ $total })
engine-rewrotePage = Réécrit { $url } ({ $current }/{ $total })
engine-renderSiteGroup = Générer le site

View File

@@ -674,6 +674,8 @@ blogmark-noProject = Apri un progetto prima di importare un blogmark.
blogmark-importing = Importazione del blogmark
blogmark-imported = Blogmark importato
blogmark-failed = Importazione del blogmark non riuscita: { $error }
engine-renderingPage = Rendering di { $url } ({ $current }/{ $total })
engine-renderingPages = Rendering delle pagine ({ $current }/{ $total })
engine-generatedPage = Generato { $url } ({ $current }/{ $total })
engine-rewrotePage = Riscritto { $url } ({ $current }/{ $total })
engine-renderSiteGroup = Genera sito

View File

@@ -44,6 +44,10 @@ surface GenerationRuntimeSurface {
-- "Generated" for a full render and "Rewrote" for a validation apply —
-- and advances the bar by the number of URLs written.
--
-- Before section work begins, generation counts its planned URLs. The
-- task reports 0/total immediately and advances once per rendered URL,
-- matching bDS2's route-based generation progress.
--
-- Full generation and validation apply share the same structuring: a
-- task group containing one task per section (Render Site Core, Render
-- Single Posts, Render Category Archives, Render Tag Archives, Render

View File

@@ -263,7 +263,8 @@ invariant PanelTabFallback {
}
-- Tasks tab: last 10 tasks, newest first, with progress/cancel.
-- Tasks with shared group_id are collapsible groups showing aggregate progress.
-- Tasks with shared group_id are collapsible groups showing bDS2-style
-- average child progress; pending children contribute zero.
-- Output tab: log entries with copy-all button.
-- The panel opens on Output automatically when a blogmark transform fails
-- or a transform script writes output.

View File

@@ -116,6 +116,11 @@ invariant ProgressThrottled {
-- At most one progress event per 250ms per task
}
invariant GroupProgressAggregatesChildren {
-- Matches bDS2: group progress averages every child task contribution.
-- Pending tasks contribute 0 and completed tasks contribute 1.
}
invariant FinishedTaskRetention {
-- The status snapshot surfaces only the most recent finished tasks:
-- completed/failed/cancelled tasks beyond config.recent_finished_limit