Add live CLI progress for long-running jobs

This commit is contained in:
2026-08-03 09:24:03 +02:00
parent c96fd9c041
commit 36cbaa0cc5
8 changed files with 421 additions and 51 deletions

View File

@@ -6,7 +6,7 @@ use std::sync::{Arc, Mutex};
use anyhow::{Context, Result, anyhow, bail};
use bds_core::db::{Database, DbConnection};
use bds_core::engine::{self, cli_sync, domain_events};
use bds_core::i18n::UiLocale;
use bds_core::i18n::{UiLocale, detect_os_locale, normalize_language, translate, translate_with};
use bds_core::model::{DomainEntity, NotificationAction, Project, ScriptKind};
use bds_core::scripting::{CoreHost, ExecutionControl, ExecutionKind, execute_many_with_host};
use clap::{Args, Parser, Subcommand, ValueEnum};
@@ -195,6 +195,112 @@ impl RunContext {
}
}
#[derive(Debug, Clone, PartialEq)]
pub struct CliProgress {
pub value: f32,
pub message: String,
}
pub type ProgressSink = Arc<dyn Fn(CliProgress) + Send + Sync>;
#[derive(Clone)]
struct ProgressReporter {
history: Arc<Mutex<Vec<String>>>,
sink: Option<ProgressSink>,
locale: UiLocale,
}
impl ProgressReporter {
fn new(sink: Option<ProgressSink>) -> Self {
Self {
history: Arc::new(Mutex::new(Vec::new())),
sink,
locale: detect_os_locale(),
}
}
fn with_locale(mut self, locale: UiLocale) -> Self {
self.locale = locale;
self
}
fn report(&self, value: f32, message: String) {
let value = if value.is_finite() {
value.clamp(0.0, 1.0)
} else {
0.0
};
let line = format!("[{:>3}%] {message}", (value * 100.0).round() as i32);
let mut history = self
.history
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner());
if history.last() == Some(&line) {
return;
}
history.push(line);
drop(history);
if let Some(sink) = &self.sink {
sink(CliProgress { value, message });
}
}
fn history(&self) -> Vec<String> {
self.history
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.clone()
}
}
pub struct HumanProgress {
terminal: bool,
active: bool,
}
impl HumanProgress {
pub fn new(terminal: bool) -> Self {
Self {
terminal,
active: false,
}
}
pub fn write(
&mut self,
progress: &CliProgress,
mut writer: impl std::io::Write,
) -> std::io::Result<()> {
let percent = (progress.value * 100.0).round() as usize;
if self.terminal {
const WIDTH: usize = 24;
let filled = percent * WIDTH / 100;
write!(
writer,
"\r\x1b[2K{}: {:>3}%|{}{}|",
progress.message,
percent,
"".repeat(filled),
" ".repeat(WIDTH - filled),
)?;
writer.flush()?;
} else {
writeln!(writer, "[{:>3}%] {}", percent, progress.message)?;
}
self.active = true;
Ok(())
}
pub fn finish(&mut self, mut writer: impl std::io::Write) -> std::io::Result<()> {
if self.terminal && self.active {
writeln!(writer)?;
writer.flush()?;
}
self.active = false;
Ok(())
}
}
#[derive(Debug, Serialize)]
pub struct CommandOutput {
pub command: &'static str,
@@ -266,14 +372,29 @@ where
}
pub fn run(cli: Cli, context: RunContext) -> Result<CommandOutput> {
run_with_progress(cli, context, None)
}
pub fn run_with_progress(
cli: Cli,
context: RunContext,
progress_sink: Option<ProgressSink>,
) -> Result<CommandOutput> {
let command_name = command_name(&cli.command);
let mut output = execute(cli.command, &context, cli.airplane)?;
let progress = ProgressReporter::new(progress_sink);
let mut output = execute(cli.command, &context, cli.airplane, &progress)?;
output.command = command_name;
output.json = cli.json;
output.progress = progress.history();
Ok(output)
}
fn execute(command: Command, context: &RunContext, airplane: bool) -> Result<CommandOutput> {
fn execute(
command: Command,
context: &RunContext,
airplane: bool,
progress: &ProgressReporter,
) -> Result<CommandOutput> {
if matches!(command, Command::Install) {
return install_launcher(context);
}
@@ -282,13 +403,18 @@ fn execute(command: Command, context: &RunContext, airplane: bool) -> Result<Com
}
let db = open_database(&context.database_path)?;
let locale = engine::settings::ui_language(db.conn())
.ok()
.flatten()
.map_or_else(detect_os_locale, |language| normalize_language(&language));
let progress = progress.clone().with_locale(locale);
match command {
Command::Rebuild { incremental } => rebuild(&db, incremental),
Command::Rebuild { incremental } => rebuild(&db, incremental, &progress),
Command::Repair { part } => repair(&db, part),
Command::Render { incremental, force } => render(&db, incremental, force),
Command::Render { incremental, force } => render(&db, incremental, force, &progress),
Command::Upload => upload(&db, airplane),
Command::Push => git_push(&db, airplane),
Command::Pull => git_pull(&db, airplane),
Command::Pull => git_pull(&db, airplane, &progress),
Command::Post(args) => create_post(&db, args, &context.stdin, airplane),
Command::Media { file, language } => import_media(&db, &file, language, airplane),
Command::Gallery(args) => create_gallery(&db, args, &context.stdin, airplane),
@@ -333,11 +459,21 @@ fn active_project(db: &Database) -> Result<(Project, PathBuf)> {
Ok((project, data_dir))
}
fn rebuild(db: &Database, incremental: bool) -> Result<CommandOutput> {
fn rebuild(db: &Database, incremental: bool, progress: &ProgressReporter) -> Result<CommandOutput> {
let (project, data_dir) = active_project(db)?;
if incremental {
let message = translate(progress.locale, "metadataDiff.running");
progress.report(0.0, message.clone());
let report = cli_sync::run_cli_mutation(db.conn(), || {
let report = engine::rebuild::rebuild_incremental(db.conn(), &data_dir, &project.id)?;
let report = engine::rebuild::rebuild_incremental_with_progress(
db.conn(),
&data_dir,
&project.id,
|value| {
progress.report(value, message.clone());
true
},
)?;
if report.differences_applied > 0 || report.orphans_imported > 0 {
emit_bulk(&project.id);
}
@@ -353,29 +489,21 @@ fn rebuild(db: &Database, incremental: bool) -> Result<CommandOutput> {
));
}
let progress = Arc::new(Mutex::new(Vec::new()));
let progress_sink = Arc::clone(&progress);
let report = cli_sync::run_cli_mutation(db.conn(), || {
let progress = progress.clone();
let report = engine::rebuild::rebuild_from_filesystem_with_progress(
db.conn(),
&data_dir,
&project.id,
Some(Arc::new(move |value, event| {
let message = event.localized(UiLocale::En);
let line = format!("[{:>3}%] {message}", (value * 100.0).round() as i32);
let mut progress = progress_sink
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner());
if progress.last() != Some(&line) {
progress.push(line);
}
progress.report(value, event.localized(progress.locale));
true
})),
)?;
emit_bulk(&project.id);
Ok(report)
})?;
let mut result = output(
Ok(output(
"Rebuild complete",
json!({
"posts_created": report.posts_created,
@@ -391,12 +519,7 @@ fn rebuild(db: &Database, incremental: bool) -> Result<CommandOutput> {
"thumbnails_generated": report.thumbnails_generated,
"thumbnail_media_failed": report.thumbnail_media_failed,
}),
);
result.progress = progress
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.clone();
Ok(result)
))
}
fn repair(db: &Database, part: RepairPart) -> Result<CommandOutput> {
@@ -497,16 +620,34 @@ fn repair(db: &Database, part: RepairPart) -> Result<CommandOutput> {
}
}
fn render(db: &Database, incremental: bool, force: bool) -> Result<CommandOutput> {
fn render(
db: &Database,
incremental: bool,
force: bool,
progress: &ProgressReporter,
) -> Result<CommandOutput> {
let (project, data_dir) = active_project(db)?;
let metadata = engine::meta::read_project_json(&data_dir)?;
let posts = published_sources(db.conn(), &data_dir, &project.id)?;
let output_dir = data_dir.join("html");
std::fs::create_dir_all(&output_dir)?;
if incremental {
let validation = engine::validate_site::validate_site(db.conn(), &data_dir, &project.id)?;
let validating = translate(progress.locale, "siteValidation.running");
progress.report(0.0, validating.clone());
let validation = engine::validate_site::validate_site_with_progress(
db.conn(),
&data_dir,
&project.id,
|current, total| {
progress.report(
0.25 * current as f32 / total.max(1) as f32,
validating.clone(),
);
true
},
)?;
let sections = engine::generation::sections_from_validation_report(&validation, &metadata);
let report = engine::generation::apply_validation_sections(
let report = engine::generation::apply_validation_sections_with_progress(
db.conn(),
&output_dir,
&project.id,
@@ -514,38 +655,69 @@ fn render(db: &Database, incremental: bool, force: bool) -> Result<CommandOutput
&posts,
&validation,
&sections,
|current, total, url| report_render_progress(progress, current, total, url, 0.25, 1.0),
)?;
progress.report(1.0, translate(progress.locale, "engine.generationComplete"));
return Ok(output(
"Validation differences applied",
json!({"written": report.written_paths.len(), "skipped": report.skipped_paths.len(), "deleted": report.deleted_paths.len()}),
));
}
let language = metadata.main_language.as_deref().unwrap_or("en");
progress.report(
0.0,
translate(progress.locale, "engine.generateSiteStarted"),
);
let report = if force {
engine::generation::generate_starter_site_forced(
engine::generation::generate_starter_site_forced_with_progress(
db.conn(),
&output_dir,
&project.id,
&metadata,
&posts,
language,
|current, total, url| report_render_progress(progress, current, total, url, 0.0, 1.0),
)?
} else {
engine::generation::generate_starter_site(
engine::generation::generate_starter_site_with_progress(
db.conn(),
&output_dir,
&project.id,
&metadata,
&posts,
language,
|current, total, url| report_render_progress(progress, current, total, url, 0.0, 1.0),
)?
};
progress.report(1.0, translate(progress.locale, "engine.generationComplete"));
Ok(output(
"Site rendered",
json!({"written": report.written_paths.len(), "skipped": report.skipped_paths.len(), "deleted": report.deleted_paths.len(), "force": force}),
))
}
fn report_render_progress(
progress: &ProgressReporter,
current: usize,
total: usize,
url: &str,
start: f32,
end: f32,
) {
progress.report(
start + (end - start) * current as f32 / total.max(1) as f32,
translate_with(
progress.locale,
"engine.renderingPage",
&[
("url", url),
("current", &current.to_string()),
("total", &total.to_string()),
],
),
);
}
fn published_sources(
conn: &DbConnection,
data_dir: &Path,
@@ -593,13 +765,13 @@ fn git_push(db: &Database, airplane: bool) -> Result<CommandOutput> {
Ok(output("Pushed", json!({"output": result.output})))
}
fn git_pull(db: &Database, airplane: bool) -> Result<CommandOutput> {
fn git_pull(db: &Database, airplane: bool, progress: &ProgressReporter) -> Result<CommandOutput> {
if airplane {
bail!("git pull is unavailable in airplane mode");
}
let (_project, data_dir) = active_project(db)?;
let result = engine::git::GitEngine::new(data_dir).pull(|| false, |_| {})?;
let mut rebuilt = rebuild(db, true)?;
let mut rebuilt = rebuild(db, true, progress)?;
rebuilt.message = "Pulled and reconciled the cache database".into();
rebuilt.data["git_output"] = Value::String(result.output);
Ok(rebuilt)
@@ -1248,6 +1420,19 @@ mod tests {
run(cli, self.context(stdin))
}
fn run_with_progress(&self, args: &[&str]) -> Result<(CommandOutput, Vec<CliProgress>)> {
let cli = Cli::try_parse_from(std::iter::once("bds-cli").chain(args.iter().copied()))?;
let updates = Arc::new(Mutex::new(Vec::new()));
let sink = Arc::clone(&updates);
let output = super::run_with_progress(
cli,
self.context(""),
Some(Arc::new(move |update| sink.lock().unwrap().push(update))),
)?;
let updates = updates.lock().unwrap().clone();
Ok((output, updates))
}
fn image(&self, name: &str) -> PathBuf {
let path = self._root.path().join(name);
std::fs::write(
@@ -1291,6 +1476,29 @@ mod tests {
assert!(Cli::try_parse_from(["bds-cli", "repair", "unknown"]).is_err());
}
#[test]
fn human_progress_uses_a_live_bar_only_for_terminals() {
let update = CliProgress {
value: 0.5,
message: "Rendering".into(),
};
let mut terminal = HumanProgress::new(true);
let mut terminal_output = Vec::new();
terminal.write(&update, &mut terminal_output).unwrap();
terminal.finish(&mut terminal_output).unwrap();
let terminal_output = String::from_utf8(terminal_output).unwrap();
assert!(terminal_output.contains("Rendering: 50%|████████████"));
assert!(terminal_output.starts_with("\r\x1b[2K"));
let mut redirected = HumanProgress::new(false);
let mut redirected_output = Vec::new();
redirected.write(&update, &mut redirected_output).unwrap();
assert_eq!(
String::from_utf8(redirected_output).unwrap(),
"[ 50%] Rendering\n"
);
}
#[test]
fn config_and_project_families_dispatch_success_and_failure() {
let fixture = Fixture::new(false);
@@ -1578,6 +1786,33 @@ mod tests {
assert!(no_project.run(&["render"], "").is_err());
}
#[test]
fn long_running_commands_stream_progress_while_preserving_result_history() {
let fixture = Fixture::new(true);
for args in [
&["rebuild"][..],
&["rebuild", "--incremental"][..],
&["render"][..],
&["render", "--incremental"][..],
] {
let (output, updates) = fixture.run_with_progress(args).unwrap();
assert!(
!updates.is_empty(),
"{} did not stream progress",
args.join(" ")
);
assert_eq!(updates.last().unwrap().value, 1.0);
assert!(
updates
.windows(2)
.all(|pair| pair[0].value <= pair[1].value),
"{} progress moved backwards: {updates:?}",
args.join(" ")
);
assert_eq!(output.progress.len(), updates.len());
}
}
#[test]
fn lua_family_runs_only_enabled_utility_scripts() {
let fixture = Fixture::new(true);

View File

@@ -1,5 +1,6 @@
use std::io::Read as _;
use std::io::{IsTerminal as _, Read as _};
use std::process::ExitCode;
use std::sync::{Arc, Mutex};
use clap::Parser;
@@ -74,8 +75,33 @@ fn main() -> ExitCode {
return ExitCode::from(1);
}
match bds_cli::run(cli, context) {
Ok(output) => {
let progress = (!json).then(|| {
Arc::new(Mutex::new(bds_cli::HumanProgress::new(
std::io::stdout().is_terminal(),
)))
});
let sink = progress.as_ref().map(|progress| {
let progress = Arc::clone(progress);
Arc::new(move |update: bds_cli::CliProgress| {
let _ = progress
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.write(&update, std::io::stdout().lock());
}) as bds_cli::ProgressSink
});
let result = bds_cli::run_with_progress(cli, context, sink);
if let Some(progress) = &progress {
let _ = progress
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.finish(std::io::stdout().lock());
}
match result {
Ok(mut output) => {
if !json {
output.progress.clear();
}
println!("{output}");
ExitCode::SUCCESS
}

View File

@@ -45,6 +45,19 @@ fn process_exit_codes_help_and_shared_state_roundtrip() {
.success()
);
let rebuilt = cli(&home, &["rebuild"]);
assert!(rebuilt.status.success());
assert!(String::from_utf8_lossy(&rebuilt.stdout).contains("%]"));
let rebuilt_json = cli(&home, &["--json", "rebuild", "--incremental"]);
assert!(rebuilt_json.status.success());
let rebuilt_json: serde_json::Value = serde_json::from_slice(&rebuilt_json.stdout).unwrap();
assert!(
rebuilt_json["progress"]
.as_array()
.is_some_and(|items| !items.is_empty())
);
let created = Command::new(env!("CARGO_BIN_EXE_bds-cli"))
.env("HOME", &home)
.args(["--json", "post", "--stdin", "--no-translate"])

View File

@@ -182,18 +182,31 @@ pub fn generate_starter_site_forced(
posts: &[PublishedPostSource],
language: &str,
) -> EngineResult<GenerationReport> {
generate_starter_site_with_progress_mode(
generate_starter_site_forced_with_progress(
conn,
output_dir,
project_id,
metadata,
posts,
language,
true,
|_current, _total, _path| {},
)
}
pub fn generate_starter_site_forced_with_progress(
conn: &Connection,
output_dir: &Path,
project_id: &str,
metadata: &ProjectMetadata,
posts: &[PublishedPostSource],
language: &str,
on_page: impl FnMut(usize, usize, &str),
) -> EngineResult<GenerationReport> {
generate_starter_site_with_progress_mode(
conn, output_dir, project_id, metadata, posts, language, true, on_page,
)
}
pub fn generate_starter_site_with_progress(
conn: &Connection,
output_dir: &Path,
@@ -224,8 +237,17 @@ fn generate_starter_site_with_progress_mode(
) -> EngineResult<GenerationReport> {
let data_dir = project_data_dir(output_dir);
let prepared = prepare_site_generation(conn, &data_dir, project_id, metadata, posts)?;
let total_pages = GenerationSection::ALL
.into_iter()
.map(|section| prepared_section_page_count(&prepared, None, section))
.sum();
let mut current_page = 0;
let mut report = GenerationReport::default();
for section in GenerationSection::ALL {
let mut on_section_page = |_current, _total, path: &str| {
current_page += 1;
on_page(current_page, total_pages, path);
};
report.append(render_prepared_site_section_with_progress(
conn,
output_dir,
@@ -234,7 +256,7 @@ fn generate_starter_site_with_progress_mode(
section,
force,
&|_| {},
&mut on_page,
&mut on_section_page,
|| false,
)?);
}
@@ -446,25 +468,61 @@ pub fn apply_validation_sections(
posts: &[PublishedPostSource],
validation: &SiteValidationReport,
sections: &[GenerationSection],
) -> EngineResult<GenerationReport> {
apply_validation_sections_with_progress(
conn,
output_dir,
project_id,
metadata,
posts,
validation,
sections,
|_current, _total, _url| {},
)
}
#[expect(
clippy::too_many_arguments,
reason = "validation application adds progress to the existing generation context"
)]
pub fn apply_validation_sections_with_progress(
conn: &Connection,
output_dir: &Path,
project_id: &str,
metadata: &ProjectMetadata,
posts: &[PublishedPostSource],
validation: &SiteValidationReport,
sections: &[GenerationSection],
mut on_page: impl FnMut(usize, usize, &str),
) -> EngineResult<GenerationReport> {
if sections.is_empty() {
return Ok(GenerationReport::default());
}
let data_dir = project_data_dir(output_dir);
let prepared = prepare_site_generation(conn, &data_dir, project_id, metadata, posts)?;
let total_pages = sections
.iter()
.map(|section| prepared_section_page_count(&prepared, Some(validation), *section))
.sum();
let mut current_page = 0;
let mut report = GenerationReport::default();
for section in sections {
report.append(apply_validation_section_with_progress(
let mut on_section_page = |_current, _total, url: &str| {
current_page += 1;
on_page(current_page, total_pages, url);
};
report.append(apply_validation_prepared_section_with_progress(
conn,
output_dir,
project_id,
metadata,
posts,
&prepared,
validation,
*section,
|_current, _total, _url| {},
|_| {},
|| false,
&mut on_section_page,
&|_| {},
&mut || false,
)?);
}
report.append(build_site_search_index(

View File

@@ -45,12 +45,33 @@ pub fn rebuild_incremental(
data_dir: &Path,
project_id: &str,
) -> EngineResult<IncrementalRebuildReport> {
let report = super::metadata_diff::compute_metadata_diff(conn, data_dir, project_id)?;
rebuild_incremental_with_progress(conn, data_dir, project_id, |_| true)
}
pub fn rebuild_incremental_with_progress(
conn: &Connection,
data_dir: &Path,
project_id: &str,
mut on_progress: impl FnMut(f32) -> bool,
) -> EngineResult<IncrementalRebuildReport> {
let report = super::metadata_diff::compute_metadata_diff_with_progress(
conn,
data_dir,
project_id,
|current, total| on_progress(0.5 * current as f32 / total.max(1) as f32),
)?;
if !report.errors.is_empty() {
return Err(crate::engine::EngineError::Validation(
report.errors.join("; "),
));
}
let orphans = report
.orphans
.iter()
.filter(|orphan| orphan.reason == "file_without_db_entry")
.collect::<Vec<_>>();
let total = report.diffs.len() + orphans.len();
let mut current = 0;
for item in &report.diffs {
super::metadata_diff::repair_metadata_diff_item(
conn,
@@ -59,20 +80,27 @@ pub fn rebuild_incremental(
super::metadata_diff::RepairDirection::FileToDatabase,
item,
)?;
current += 1;
if !on_progress(0.5 + 0.5 * current as f32 / total.max(1) as f32) {
return Err(crate::engine::EngineError::Cancelled);
}
}
let mut result = IncrementalRebuildReport {
differences_applied: report.diffs.len(),
..Default::default()
};
for orphan in report
.orphans
.iter()
.filter(|orphan| orphan.reason == "file_without_db_entry")
{
for orphan in orphans {
match super::metadata_diff::import_orphan_file(conn, data_dir, project_id, orphan) {
Ok(()) => result.orphans_imported += 1,
Err(_) => result.orphans_failed += 1,
}
current += 1;
if !on_progress(0.5 + 0.5 * current as f32 / total.max(1) as f32) {
return Err(crate::engine::EngineError::Cancelled);
}
}
if total == 0 && !on_progress(1.0) {
return Err(crate::engine::EngineError::Cancelled);
}
Ok(result)
}