From 36cbaa0cc5ef18c33cb97a02a5a783fd1ffb626a Mon Sep 17 00:00:00 2001 From: Chili Palmer Date: Mon, 3 Aug 2026 09:24:03 +0200 Subject: [PATCH] Add live CLI progress for long-running jobs --- CLI.md | 2 + README.md | 2 +- crates/bds-cli/src/lib.rs | 299 ++++++++++++++++++++--- crates/bds-cli/src/main.rs | 32 ++- crates/bds-cli/tests/process.rs | 13 + crates/bds-core/src/engine/generation.rs | 76 +++++- crates/bds-core/src/engine/rebuild.rs | 40 ++- specs/cli.allium | 8 + 8 files changed, 421 insertions(+), 51 deletions(-) diff --git a/CLI.md b/CLI.md index 1f85b37..7718a4f 100644 --- a/CLI.md +++ b/CLI.md @@ -30,6 +30,8 @@ Install the launcher from the desktop app under **Settings → Data**, or run `b The JSON envelope always has the shape `{"ok": bool, "command": string, "message": string, "data": object, "progress": [string], "notices": [string]}`, which makes the CLI safe to drive from scripts and LLM agents. Errors exit with code 1 and a formatted message on stderr; unknown commands and invalid options do the same. +Long-running rebuild and render commands show live progress while comparing metadata, validating generated output, and rendering pages. An interactive terminal updates one Hugging Face-style progress bar in place; redirected output receives one stable line per update. `--json` never emits interim output and includes the ordered progress history in the final envelope. + With `--airplane`, `upload`, `push`, and `pull` are refused, and AI-assisted steps (language detection, translation, image enrichment) use the configured airplane endpoint. When no local endpoint is configured, the CLI prints a notice and falls back to offline heuristics where they exist — the command-line equivalent of the desktop's airplane-mode toast. ## Command reference diff --git a/README.md b/README.md index cfcd30a..3084f22 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ The project is under active development. Core blogging workflows are broadly ava - A localized Tags workspace manages tags and category settings; its category table includes the main language and every configured translation, whose titles are used by matching category archives and menu entries. - A localized OPML menu editor manages pages, submenus, and category archives with protected Home ordering, keyboard-accessible tree controls, drag-and-drop, and bDS2-compatible persistence. - Project-scoped typed domain events synchronize desktop views and cached runtime settings with shared-engine and CLI mutations even when Preferences is closed; persisted CLI notifications are consumed once, and the selected UI language is shared through settings. -- Headless `bds-cli` automation for rebuild/repair/render, publishing and Git sync, post/media/gallery creation, effective shared settings with secret-presence redaction, projects, utility Lua tasks, JSON I/O, airplane-mode AI routing, and guarded launcher installation from Settings → Data or `bds-cli install`. +- Headless `bds-cli` automation for rebuild/repair/render, with live terminal progress through metadata comparison, site validation, and page rendering; publishing and Git sync; post/media/gallery creation; effective shared settings with secret-presence redaction; projects; utility Lua tasks; JSON I/O; airplane-mode AI routing; and guarded launcher installation from Settings → Data or `bds-cli install`. - Local MCP automation over stdio or a localhost-only stateless HTTP endpoint, with project resources, read/search/count tools, uniquely identified inert write proposals, clean duplicate-pending rejection, explicit desktop approval, and opt-in Claude Code/Copilot configuration. - A fully localized Ratatui terminal workspace, available locally through `bds-cli tui`/`BDS_MODE=tui` and remotely through authenticated SSH shell sessions, with shared post/template/script editing and publishing, project/search/command overlays, settings, tags, Git, reports, task progress, live multi-client locale updates, and airplane-mode AI gating. - `bds-cli server` hosting the shared application engines over a loopback-by-default, public-key-only SSH service, with restrictive private key material, live authorization updates, terminal-session transport, CLI-change synchronization, ordered domain/task events, and native desktop remote-project selection. diff --git a/crates/bds-cli/src/lib.rs b/crates/bds-cli/src/lib.rs index 68052b0..b4f727f 100644 --- a/crates/bds-cli/src/lib.rs +++ b/crates/bds-cli/src/lib.rs @@ -6,7 +6,7 @@ use std::sync::{Arc, Mutex}; use anyhow::{Context, Result, anyhow, bail}; use bds_core::db::{Database, DbConnection}; use bds_core::engine::{self, cli_sync, domain_events}; -use bds_core::i18n::UiLocale; +use bds_core::i18n::{UiLocale, detect_os_locale, normalize_language, translate, translate_with}; use bds_core::model::{DomainEntity, NotificationAction, Project, ScriptKind}; use bds_core::scripting::{CoreHost, ExecutionControl, ExecutionKind, execute_many_with_host}; use clap::{Args, Parser, Subcommand, ValueEnum}; @@ -195,6 +195,112 @@ impl RunContext { } } +#[derive(Debug, Clone, PartialEq)] +pub struct CliProgress { + pub value: f32, + pub message: String, +} + +pub type ProgressSink = Arc; + +#[derive(Clone)] +struct ProgressReporter { + history: Arc>>, + sink: Option, + locale: UiLocale, +} + +impl ProgressReporter { + fn new(sink: Option) -> Self { + Self { + history: Arc::new(Mutex::new(Vec::new())), + sink, + locale: detect_os_locale(), + } + } + + fn with_locale(mut self, locale: UiLocale) -> Self { + self.locale = locale; + self + } + + fn report(&self, value: f32, message: String) { + let value = if value.is_finite() { + value.clamp(0.0, 1.0) + } else { + 0.0 + }; + let line = format!("[{:>3}%] {message}", (value * 100.0).round() as i32); + let mut history = self + .history + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + if history.last() == Some(&line) { + return; + } + history.push(line); + drop(history); + if let Some(sink) = &self.sink { + sink(CliProgress { value, message }); + } + } + + fn history(&self) -> Vec { + self.history + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .clone() + } +} + +pub struct HumanProgress { + terminal: bool, + active: bool, +} + +impl HumanProgress { + pub fn new(terminal: bool) -> Self { + Self { + terminal, + active: false, + } + } + + pub fn write( + &mut self, + progress: &CliProgress, + mut writer: impl std::io::Write, + ) -> std::io::Result<()> { + let percent = (progress.value * 100.0).round() as usize; + if self.terminal { + const WIDTH: usize = 24; + let filled = percent * WIDTH / 100; + write!( + writer, + "\r\x1b[2K{}: {:>3}%|{}{}|", + progress.message, + percent, + "█".repeat(filled), + " ".repeat(WIDTH - filled), + )?; + writer.flush()?; + } else { + writeln!(writer, "[{:>3}%] {}", percent, progress.message)?; + } + self.active = true; + Ok(()) + } + + pub fn finish(&mut self, mut writer: impl std::io::Write) -> std::io::Result<()> { + if self.terminal && self.active { + writeln!(writer)?; + writer.flush()?; + } + self.active = false; + Ok(()) + } +} + #[derive(Debug, Serialize)] pub struct CommandOutput { pub command: &'static str, @@ -266,14 +372,29 @@ where } pub fn run(cli: Cli, context: RunContext) -> Result { + run_with_progress(cli, context, None) +} + +pub fn run_with_progress( + cli: Cli, + context: RunContext, + progress_sink: Option, +) -> Result { let command_name = command_name(&cli.command); - let mut output = execute(cli.command, &context, cli.airplane)?; + let progress = ProgressReporter::new(progress_sink); + let mut output = execute(cli.command, &context, cli.airplane, &progress)?; output.command = command_name; output.json = cli.json; + output.progress = progress.history(); Ok(output) } -fn execute(command: Command, context: &RunContext, airplane: bool) -> Result { +fn execute( + command: Command, + context: &RunContext, + airplane: bool, + progress: &ProgressReporter, +) -> Result { if matches!(command, Command::Install) { return install_launcher(context); } @@ -282,13 +403,18 @@ fn execute(command: Command, context: &RunContext, airplane: bool) -> Result rebuild(&db, incremental), + Command::Rebuild { incremental } => rebuild(&db, incremental, &progress), Command::Repair { part } => repair(&db, part), - Command::Render { incremental, force } => render(&db, incremental, force), + Command::Render { incremental, force } => render(&db, incremental, force, &progress), Command::Upload => upload(&db, airplane), Command::Push => git_push(&db, airplane), - Command::Pull => git_pull(&db, airplane), + Command::Pull => git_pull(&db, airplane, &progress), Command::Post(args) => create_post(&db, args, &context.stdin, airplane), Command::Media { file, language } => import_media(&db, &file, language, airplane), Command::Gallery(args) => create_gallery(&db, args, &context.stdin, airplane), @@ -333,11 +459,21 @@ fn active_project(db: &Database) -> Result<(Project, PathBuf)> { Ok((project, data_dir)) } -fn rebuild(db: &Database, incremental: bool) -> Result { +fn rebuild(db: &Database, incremental: bool, progress: &ProgressReporter) -> Result { let (project, data_dir) = active_project(db)?; if incremental { + let message = translate(progress.locale, "metadataDiff.running"); + progress.report(0.0, message.clone()); let report = cli_sync::run_cli_mutation(db.conn(), || { - let report = engine::rebuild::rebuild_incremental(db.conn(), &data_dir, &project.id)?; + let report = engine::rebuild::rebuild_incremental_with_progress( + db.conn(), + &data_dir, + &project.id, + |value| { + progress.report(value, message.clone()); + true + }, + )?; if report.differences_applied > 0 || report.orphans_imported > 0 { emit_bulk(&project.id); } @@ -353,29 +489,21 @@ fn rebuild(db: &Database, incremental: bool) -> Result { )); } - let progress = Arc::new(Mutex::new(Vec::new())); - let progress_sink = Arc::clone(&progress); let report = cli_sync::run_cli_mutation(db.conn(), || { + let progress = progress.clone(); let report = engine::rebuild::rebuild_from_filesystem_with_progress( db.conn(), &data_dir, &project.id, Some(Arc::new(move |value, event| { - let message = event.localized(UiLocale::En); - let line = format!("[{:>3}%] {message}", (value * 100.0).round() as i32); - let mut progress = progress_sink - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()); - if progress.last() != Some(&line) { - progress.push(line); - } + progress.report(value, event.localized(progress.locale)); true })), )?; emit_bulk(&project.id); Ok(report) })?; - let mut result = output( + Ok(output( "Rebuild complete", json!({ "posts_created": report.posts_created, @@ -391,12 +519,7 @@ fn rebuild(db: &Database, incremental: bool) -> Result { "thumbnails_generated": report.thumbnails_generated, "thumbnail_media_failed": report.thumbnail_media_failed, }), - ); - result.progress = progress - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()) - .clone(); - Ok(result) + )) } fn repair(db: &Database, part: RepairPart) -> Result { @@ -497,16 +620,34 @@ fn repair(db: &Database, part: RepairPart) -> Result { } } -fn render(db: &Database, incremental: bool, force: bool) -> Result { +fn render( + db: &Database, + incremental: bool, + force: bool, + progress: &ProgressReporter, +) -> Result { let (project, data_dir) = active_project(db)?; let metadata = engine::meta::read_project_json(&data_dir)?; let posts = published_sources(db.conn(), &data_dir, &project.id)?; let output_dir = data_dir.join("html"); std::fs::create_dir_all(&output_dir)?; if incremental { - let validation = engine::validate_site::validate_site(db.conn(), &data_dir, &project.id)?; + let validating = translate(progress.locale, "siteValidation.running"); + progress.report(0.0, validating.clone()); + let validation = engine::validate_site::validate_site_with_progress( + db.conn(), + &data_dir, + &project.id, + |current, total| { + progress.report( + 0.25 * current as f32 / total.max(1) as f32, + validating.clone(), + ); + true + }, + )?; let sections = engine::generation::sections_from_validation_report(&validation, &metadata); - let report = engine::generation::apply_validation_sections( + let report = engine::generation::apply_validation_sections_with_progress( db.conn(), &output_dir, &project.id, @@ -514,38 +655,69 @@ fn render(db: &Database, incremental: bool, force: bool) -> Result Result { Ok(output("Pushed", json!({"output": result.output}))) } -fn git_pull(db: &Database, airplane: bool) -> Result { +fn git_pull(db: &Database, airplane: bool, progress: &ProgressReporter) -> Result { if airplane { bail!("git pull is unavailable in airplane mode"); } let (_project, data_dir) = active_project(db)?; let result = engine::git::GitEngine::new(data_dir).pull(|| false, |_| {})?; - let mut rebuilt = rebuild(db, true)?; + let mut rebuilt = rebuild(db, true, progress)?; rebuilt.message = "Pulled and reconciled the cache database".into(); rebuilt.data["git_output"] = Value::String(result.output); Ok(rebuilt) @@ -1248,6 +1420,19 @@ mod tests { run(cli, self.context(stdin)) } + fn run_with_progress(&self, args: &[&str]) -> Result<(CommandOutput, Vec)> { + let cli = Cli::try_parse_from(std::iter::once("bds-cli").chain(args.iter().copied()))?; + let updates = Arc::new(Mutex::new(Vec::new())); + let sink = Arc::clone(&updates); + let output = super::run_with_progress( + cli, + self.context(""), + Some(Arc::new(move |update| sink.lock().unwrap().push(update))), + )?; + let updates = updates.lock().unwrap().clone(); + Ok((output, updates)) + } + fn image(&self, name: &str) -> PathBuf { let path = self._root.path().join(name); std::fs::write( @@ -1291,6 +1476,29 @@ mod tests { assert!(Cli::try_parse_from(["bds-cli", "repair", "unknown"]).is_err()); } + #[test] + fn human_progress_uses_a_live_bar_only_for_terminals() { + let update = CliProgress { + value: 0.5, + message: "Rendering".into(), + }; + let mut terminal = HumanProgress::new(true); + let mut terminal_output = Vec::new(); + terminal.write(&update, &mut terminal_output).unwrap(); + terminal.finish(&mut terminal_output).unwrap(); + let terminal_output = String::from_utf8(terminal_output).unwrap(); + assert!(terminal_output.contains("Rendering: 50%|████████████")); + assert!(terminal_output.starts_with("\r\x1b[2K")); + + let mut redirected = HumanProgress::new(false); + let mut redirected_output = Vec::new(); + redirected.write(&update, &mut redirected_output).unwrap(); + assert_eq!( + String::from_utf8(redirected_output).unwrap(), + "[ 50%] Rendering\n" + ); + } + #[test] fn config_and_project_families_dispatch_success_and_failure() { let fixture = Fixture::new(false); @@ -1578,6 +1786,33 @@ mod tests { assert!(no_project.run(&["render"], "").is_err()); } + #[test] + fn long_running_commands_stream_progress_while_preserving_result_history() { + let fixture = Fixture::new(true); + for args in [ + &["rebuild"][..], + &["rebuild", "--incremental"][..], + &["render"][..], + &["render", "--incremental"][..], + ] { + let (output, updates) = fixture.run_with_progress(args).unwrap(); + assert!( + !updates.is_empty(), + "{} did not stream progress", + args.join(" ") + ); + assert_eq!(updates.last().unwrap().value, 1.0); + assert!( + updates + .windows(2) + .all(|pair| pair[0].value <= pair[1].value), + "{} progress moved backwards: {updates:?}", + args.join(" ") + ); + assert_eq!(output.progress.len(), updates.len()); + } + } + #[test] fn lua_family_runs_only_enabled_utility_scripts() { let fixture = Fixture::new(true); diff --git a/crates/bds-cli/src/main.rs b/crates/bds-cli/src/main.rs index 65b2600..7c5b55e 100644 --- a/crates/bds-cli/src/main.rs +++ b/crates/bds-cli/src/main.rs @@ -1,5 +1,6 @@ -use std::io::Read as _; +use std::io::{IsTerminal as _, Read as _}; use std::process::ExitCode; +use std::sync::{Arc, Mutex}; use clap::Parser; @@ -74,8 +75,33 @@ fn main() -> ExitCode { return ExitCode::from(1); } - match bds_cli::run(cli, context) { - Ok(output) => { + let progress = (!json).then(|| { + Arc::new(Mutex::new(bds_cli::HumanProgress::new( + std::io::stdout().is_terminal(), + ))) + }); + let sink = progress.as_ref().map(|progress| { + let progress = Arc::clone(progress); + Arc::new(move |update: bds_cli::CliProgress| { + let _ = progress + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .write(&update, std::io::stdout().lock()); + }) as bds_cli::ProgressSink + }); + let result = bds_cli::run_with_progress(cli, context, sink); + if let Some(progress) = &progress { + let _ = progress + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .finish(std::io::stdout().lock()); + } + + match result { + Ok(mut output) => { + if !json { + output.progress.clear(); + } println!("{output}"); ExitCode::SUCCESS } diff --git a/crates/bds-cli/tests/process.rs b/crates/bds-cli/tests/process.rs index 6a0195e..e498b09 100644 --- a/crates/bds-cli/tests/process.rs +++ b/crates/bds-cli/tests/process.rs @@ -45,6 +45,19 @@ fn process_exit_codes_help_and_shared_state_roundtrip() { .success() ); + let rebuilt = cli(&home, &["rebuild"]); + assert!(rebuilt.status.success()); + assert!(String::from_utf8_lossy(&rebuilt.stdout).contains("%]")); + + let rebuilt_json = cli(&home, &["--json", "rebuild", "--incremental"]); + assert!(rebuilt_json.status.success()); + let rebuilt_json: serde_json::Value = serde_json::from_slice(&rebuilt_json.stdout).unwrap(); + assert!( + rebuilt_json["progress"] + .as_array() + .is_some_and(|items| !items.is_empty()) + ); + let created = Command::new(env!("CARGO_BIN_EXE_bds-cli")) .env("HOME", &home) .args(["--json", "post", "--stdin", "--no-translate"]) diff --git a/crates/bds-core/src/engine/generation.rs b/crates/bds-core/src/engine/generation.rs index 4c035b1..0d4eb62 100644 --- a/crates/bds-core/src/engine/generation.rs +++ b/crates/bds-core/src/engine/generation.rs @@ -182,18 +182,31 @@ pub fn generate_starter_site_forced( posts: &[PublishedPostSource], language: &str, ) -> EngineResult { - generate_starter_site_with_progress_mode( + generate_starter_site_forced_with_progress( conn, output_dir, project_id, metadata, posts, language, - true, |_current, _total, _path| {}, ) } +pub fn generate_starter_site_forced_with_progress( + conn: &Connection, + output_dir: &Path, + project_id: &str, + metadata: &ProjectMetadata, + posts: &[PublishedPostSource], + language: &str, + on_page: impl FnMut(usize, usize, &str), +) -> EngineResult { + generate_starter_site_with_progress_mode( + conn, output_dir, project_id, metadata, posts, language, true, on_page, + ) +} + pub fn generate_starter_site_with_progress( conn: &Connection, output_dir: &Path, @@ -224,8 +237,17 @@ fn generate_starter_site_with_progress_mode( ) -> EngineResult { let data_dir = project_data_dir(output_dir); let prepared = prepare_site_generation(conn, &data_dir, project_id, metadata, posts)?; + let total_pages = GenerationSection::ALL + .into_iter() + .map(|section| prepared_section_page_count(&prepared, None, section)) + .sum(); + let mut current_page = 0; let mut report = GenerationReport::default(); for section in GenerationSection::ALL { + let mut on_section_page = |_current, _total, path: &str| { + current_page += 1; + on_page(current_page, total_pages, path); + }; report.append(render_prepared_site_section_with_progress( conn, output_dir, @@ -234,7 +256,7 @@ fn generate_starter_site_with_progress_mode( section, force, &|_| {}, - &mut on_page, + &mut on_section_page, || false, )?); } @@ -446,25 +468,61 @@ pub fn apply_validation_sections( posts: &[PublishedPostSource], validation: &SiteValidationReport, sections: &[GenerationSection], +) -> EngineResult { + apply_validation_sections_with_progress( + conn, + output_dir, + project_id, + metadata, + posts, + validation, + sections, + |_current, _total, _url| {}, + ) +} + +#[expect( + clippy::too_many_arguments, + reason = "validation application adds progress to the existing generation context" +)] +pub fn apply_validation_sections_with_progress( + conn: &Connection, + output_dir: &Path, + project_id: &str, + metadata: &ProjectMetadata, + posts: &[PublishedPostSource], + validation: &SiteValidationReport, + sections: &[GenerationSection], + mut on_page: impl FnMut(usize, usize, &str), ) -> EngineResult { if sections.is_empty() { return Ok(GenerationReport::default()); } + let data_dir = project_data_dir(output_dir); + let prepared = prepare_site_generation(conn, &data_dir, project_id, metadata, posts)?; + let total_pages = sections + .iter() + .map(|section| prepared_section_page_count(&prepared, Some(validation), *section)) + .sum(); + let mut current_page = 0; let mut report = GenerationReport::default(); for section in sections { - report.append(apply_validation_section_with_progress( + let mut on_section_page = |_current, _total, url: &str| { + current_page += 1; + on_page(current_page, total_pages, url); + }; + report.append(apply_validation_prepared_section_with_progress( conn, output_dir, project_id, - metadata, - posts, + &prepared, validation, *section, - |_current, _total, _url| {}, - |_| {}, - || false, + &mut on_section_page, + &|_| {}, + &mut || false, )?); } report.append(build_site_search_index( diff --git a/crates/bds-core/src/engine/rebuild.rs b/crates/bds-core/src/engine/rebuild.rs index 3d8eb17..3a130f5 100644 --- a/crates/bds-core/src/engine/rebuild.rs +++ b/crates/bds-core/src/engine/rebuild.rs @@ -45,12 +45,33 @@ pub fn rebuild_incremental( data_dir: &Path, project_id: &str, ) -> EngineResult { - let report = super::metadata_diff::compute_metadata_diff(conn, data_dir, project_id)?; + rebuild_incremental_with_progress(conn, data_dir, project_id, |_| true) +} + +pub fn rebuild_incremental_with_progress( + conn: &Connection, + data_dir: &Path, + project_id: &str, + mut on_progress: impl FnMut(f32) -> bool, +) -> EngineResult { + let report = super::metadata_diff::compute_metadata_diff_with_progress( + conn, + data_dir, + project_id, + |current, total| on_progress(0.5 * current as f32 / total.max(1) as f32), + )?; if !report.errors.is_empty() { return Err(crate::engine::EngineError::Validation( report.errors.join("; "), )); } + let orphans = report + .orphans + .iter() + .filter(|orphan| orphan.reason == "file_without_db_entry") + .collect::>(); + let total = report.diffs.len() + orphans.len(); + let mut current = 0; for item in &report.diffs { super::metadata_diff::repair_metadata_diff_item( conn, @@ -59,20 +80,27 @@ pub fn rebuild_incremental( super::metadata_diff::RepairDirection::FileToDatabase, item, )?; + current += 1; + if !on_progress(0.5 + 0.5 * current as f32 / total.max(1) as f32) { + return Err(crate::engine::EngineError::Cancelled); + } } let mut result = IncrementalRebuildReport { differences_applied: report.diffs.len(), ..Default::default() }; - for orphan in report - .orphans - .iter() - .filter(|orphan| orphan.reason == "file_without_db_entry") - { + for orphan in orphans { match super::metadata_diff::import_orphan_file(conn, data_dir, project_id, orphan) { Ok(()) => result.orphans_imported += 1, Err(_) => result.orphans_failed += 1, } + current += 1; + if !on_progress(0.5 + 0.5 * current as f32 / total.max(1) as f32) { + return Err(crate::engine::EngineError::Cancelled); + } + } + if total == 0 && !on_progress(1.0) { + return Err(crate::engine::EngineError::Cancelled); } Ok(result) } diff --git a/specs/cli.allium b/specs/cli.allium index c677476..0280a9e 100644 --- a/specs/cli.allium +++ b/specs/cli.allium @@ -57,6 +57,14 @@ rule MachineOutput { ensures: CliInvocation.exit_code.updated() } +invariant LongRunningProgress { + -- Human-readable rebuild and render invocations stream localized progress + -- while metadata comparison, site validation, and page rendering run. + -- Interactive terminals update one bounded progress bar; redirected output + -- emits stable progress lines. JSON mode emits no interim text and includes + -- the same ordered progress history in its final envelope. +} + rule AirplaneGate { when: CliCommandExecuted(command) requires: CliInvocation.airplane