From 6314eb577e89e7195db2ab8648e47561848c5f8a Mon Sep 17 00:00:00 2001 From: Chili Palmer Date: Sat, 25 Apr 2026 13:59:19 +0200 Subject: [PATCH] fix: better progress reporting on tasks --- lib/bds/desktop/shell_commands.ex | 12 +- lib/bds/maintenance.ex | 10 +- lib/bds/media.ex | 61 ++++++- lib/bds/posts.ex | 46 ++++- lib/bds/scripts.ex | 45 ++++- lib/bds/templates.ex | 45 ++++- test/bds/desktop/shell_commands_test.exs | 72 ++++++++ test/bds/maintenance_test.exs | 205 +++++++++++++++++++++++ 8 files changed, 466 insertions(+), 30 deletions(-) diff --git a/lib/bds/desktop/shell_commands.ex b/lib/bds/desktop/shell_commands.ex index 593827a..db846b6 100644 --- a/lib/bds/desktop/shell_commands.ex +++ b/lib/bds/desktop/shell_commands.ex @@ -122,32 +122,28 @@ defmodule BDS.Desktop.ShellCommands do {:ok, posts_task} = Tasks.submit_task("Rebuild Posts From Files", fn report -> - report.(0.0, "Scanning post files") - {:ok, posts} = Maintenance.rebuild_from_filesystem(project.id, "post") + {:ok, posts} = Maintenance.rebuild_from_filesystem(project.id, "post", on_progress: report) report.(1.0, "Post rebuild complete") %{project_id: project.id, counts: %{posts: length(posts)}} end, attrs) {:ok, _media_task} = Tasks.submit_task("Rebuild Media From Files", fn report -> - report.(0.0, "Scanning media files") - {:ok, media} = Maintenance.rebuild_from_filesystem(project.id, "media") + {:ok, media} = Maintenance.rebuild_from_filesystem(project.id, "media", on_progress: report) report.(1.0, "Media rebuild complete") %{project_id: project.id, counts: %{media: length(media)}} end, attrs) {:ok, _scripts_task} = Tasks.submit_task("Rebuild Scripts From Files", fn report -> - report.(0.0, "Scanning script files") - {:ok, scripts} = Maintenance.rebuild_from_filesystem(project.id, "script") + {:ok, scripts} = Maintenance.rebuild_from_filesystem(project.id, "script", on_progress: report) report.(1.0, "Script rebuild complete") %{project_id: project.id, counts: %{scripts: length(scripts)}} end, attrs) {:ok, _templates_task} = Tasks.submit_task("Rebuild Templates From Files", fn report -> - report.(0.0, "Scanning template files") - {:ok, templates} = Maintenance.rebuild_from_filesystem(project.id, "template") + {:ok, templates} = Maintenance.rebuild_from_filesystem(project.id, "template", on_progress: report) report.(1.0, "Template rebuild complete") %{project_id: project.id, counts: %{templates: length(templates)}} end, attrs) diff --git a/lib/bds/maintenance.ex b/lib/bds/maintenance.ex index e7b7bf2..9aed98e 100644 --- a/lib/bds/maintenance.ex +++ b/lib/bds/maintenance.ex @@ -15,12 +15,12 @@ defmodule BDS.Maintenance do alias BDS.Sidecar alias BDS.Templates.Template - def rebuild_from_filesystem(project_id, entity_type) do + def rebuild_from_filesystem(project_id, entity_type, opts \\ []) do case normalize_entity_type(entity_type) do - :post -> BDS.Posts.rebuild_posts_from_files(project_id) - :media -> BDS.Media.rebuild_media_from_files(project_id) - :script -> BDS.Scripts.rebuild_scripts_from_files(project_id) - :template -> BDS.Templates.rebuild_templates_from_files(project_id) + :post -> BDS.Posts.rebuild_posts_from_files(project_id, opts) + :media -> BDS.Media.rebuild_media_from_files(project_id, opts) + :script -> BDS.Scripts.rebuild_scripts_from_files(project_id, opts) + :template -> BDS.Templates.rebuild_templates_from_files(project_id, opts) :embedding -> Embeddings.rebuild_project(project_id) :unsupported -> {:error, :unsupported_entity_type} end diff --git a/lib/bds/media.ex b/lib/bds/media.ex index 8304a39..4fa6c92 100644 --- a/lib/bds/media.ex +++ b/lib/bds/media.ex @@ -311,8 +311,9 @@ defmodule BDS.Media do end) end - def rebuild_media_from_files(project_id) do + def rebuild_media_from_files(project_id, opts \\ []) do project = Projects.get_project!(project_id) + on_progress = progress_callback(opts) canonical_sidecars = project @@ -322,19 +323,36 @@ defmodule BDS.Media do |> Enum.filter(&canonical_sidecar?/1) |> Enum.filter(&binary_exists_for_sidecar?/1) - media_items = Enum.map(canonical_sidecars, &upsert_media_from_sidecar(project, &1)) + translation_sidecars = + project + |> Projects.project_data_dir() + |> Path.join("media") + |> list_matching_files("*.meta") + |> Enum.filter(&translation_sidecar?/1) + + total_files = length(canonical_sidecars) + length(translation_sidecars) + :ok = report_rebuild_started(on_progress, total_files, "media files") + + media_items = + canonical_sidecars + |> Enum.with_index(1) + |> Enum.map(fn {sidecar_path, index} -> + media = upsert_media_from_sidecar(project, sidecar_path) + :ok = report_rebuild_progress(on_progress, index, total_files, "media files") + media + end) canonical_media_by_binary_path = Map.new(media_items, fn media -> {Path.join(Projects.project_data_dir(project), media.file_path), media} end) - project - |> Projects.project_data_dir() - |> Path.join("media") - |> list_matching_files("*.meta") - |> Enum.filter(&translation_sidecar?/1) - |> Enum.each(&upsert_translation_from_sidecar(project, canonical_media_by_binary_path, &1)) + translation_sidecars + |> Enum.with_index(length(canonical_sidecars) + 1) + |> Enum.each(fn {sidecar_path, index} -> + upsert_translation_from_sidecar(project, canonical_media_by_binary_path, sidecar_path) + :ok = report_rebuild_progress(on_progress, index, total_files, "media files") + end) {:ok, media_items} end @@ -629,4 +647,31 @@ defmodule BDS.Media do true -> nil end end + + defp progress_callback(opts) do + case Keyword.get(opts, :on_progress) do + callback when is_function(callback, 2) -> callback + _other -> nil + end + end + + defp report_rebuild_started(nil, _total, _label), do: :ok + + defp report_rebuild_started(callback, 0, label) do + callback.(1.0, "No #{label} found") + :ok + end + + defp report_rebuild_started(callback, total, label) do + callback.(0.05, "Rebuilding #{label} (0/#{total})") + :ok + end + + defp report_rebuild_progress(nil, _current, _total, _label), do: :ok + defp report_rebuild_progress(_callback, _current, 0, _label), do: :ok + + defp report_rebuild_progress(callback, current, total, label) do + callback.(0.05 + 0.95 * (current / total), "Rebuilding #{label} (#{current}/#{total})") + :ok + end end diff --git a/lib/bds/posts.ex b/lib/bds/posts.ex index 73b55cc..09c39c1 100644 --- a/lib/bds/posts.ex +++ b/lib/bds/posts.ex @@ -136,8 +136,9 @@ defmodule BDS.Posts do end end - def rebuild_posts_from_files(project_id) do + def rebuild_posts_from_files(project_id, opts \\ []) do project = Projects.get_project!(project_id) + on_progress = progress_callback(opts) rebuild_files = project @@ -146,14 +147,26 @@ defmodule BDS.Posts do |> list_matching_files("*.md") |> Enum.map(&parse_rebuild_file(project, &1)) + total_files = length(rebuild_files) + :ok = report_rebuild_started(on_progress, total_files, "post files") + {translation_files, post_files} = Enum.split_with(rebuild_files, &translation_rebuild_file?/1) posts = post_files - |> Enum.map(&upsert_post_from_rebuild_file(project_id, &1)) + |> Enum.with_index(1) + |> Enum.map(fn {file, index} -> + post = upsert_post_from_rebuild_file(project_id, file) + :ok = report_rebuild_progress(on_progress, index, total_files, "post files") + post + end) translation_files - |> Enum.map(&upsert_post_translation_from_rebuild_file(project_id, &1)) + |> Enum.with_index(length(post_files) + 1) + |> Enum.each(fn {file, index} -> + upsert_post_translation_from_rebuild_file(project_id, file) + :ok = report_rebuild_progress(on_progress, index, total_files, "post files") + end) {:ok, posts} end @@ -941,4 +954,31 @@ defmodule BDS.Posts do true -> nil end end + + defp progress_callback(opts) do + case Keyword.get(opts, :on_progress) do + callback when is_function(callback, 2) -> callback + _other -> nil + end + end + + defp report_rebuild_started(nil, _total, _label), do: :ok + + defp report_rebuild_started(callback, 0, label) do + callback.(1.0, "No #{label} found") + :ok + end + + defp report_rebuild_started(callback, total, label) do + callback.(0.05, "Rebuilding #{label} (0/#{total})") + :ok + end + + defp report_rebuild_progress(nil, _current, _total, _label), do: :ok + defp report_rebuild_progress(_callback, _current, 0, _label), do: :ok + + defp report_rebuild_progress(callback, current, total, label) do + callback.(0.05 + 0.95 * (current / total), "Rebuilding #{label} (#{current}/#{total})") + :ok + end end diff --git a/lib/bds/scripts.ex b/lib/bds/scripts.ex index b1559ff..d9bee5f 100644 --- a/lib/bds/scripts.ex +++ b/lib/bds/scripts.ex @@ -115,15 +115,27 @@ defmodule BDS.Scripts do end end - def rebuild_scripts_from_files(project_id) do + def rebuild_scripts_from_files(project_id, opts \\ []) do project = Projects.get_project!(project_id) - scripts = + script_paths = project |> Projects.project_data_dir() |> Path.join("scripts") |> list_matching_files("*.lua") - |> Enum.map(&upsert_script_from_file(project_id, project, &1)) + + total_files = length(script_paths) + on_progress = progress_callback(opts) + :ok = report_rebuild_started(on_progress, total_files, "script files") + + scripts = + script_paths + |> Enum.with_index(1) + |> Enum.map(fn {path, index} -> + script = upsert_script_from_file(project_id, project, path) + :ok = report_rebuild_progress(on_progress, index, total_files, "script files") + script + end) {:ok, scripts} end @@ -259,4 +271,31 @@ defmodule BDS.Scripts do true -> nil end end + + defp progress_callback(opts) do + case Keyword.get(opts, :on_progress) do + callback when is_function(callback, 2) -> callback + _other -> nil + end + end + + defp report_rebuild_started(nil, _total, _label), do: :ok + + defp report_rebuild_started(callback, 0, label) do + callback.(1.0, "No #{label} found") + :ok + end + + defp report_rebuild_started(callback, total, label) do + callback.(0.05, "Rebuilding #{label} (0/#{total})") + :ok + end + + defp report_rebuild_progress(nil, _current, _total, _label), do: :ok + defp report_rebuild_progress(_callback, _current, 0, _label), do: :ok + + defp report_rebuild_progress(callback, current, total, label) do + callback.(0.05 + 0.95 * (current / total), "Rebuilding #{label} (#{current}/#{total})") + :ok + end end diff --git a/lib/bds/templates.ex b/lib/bds/templates.ex index 48db5b2..055a19b 100644 --- a/lib/bds/templates.ex +++ b/lib/bds/templates.ex @@ -133,15 +133,27 @@ defmodule BDS.Templates do end end - def rebuild_templates_from_files(project_id) do + def rebuild_templates_from_files(project_id, opts \\ []) do project = Projects.get_project!(project_id) - templates = + template_paths = project |> Projects.project_data_dir() |> Path.join("templates") |> list_matching_files("*.liquid") - |> Enum.map(&upsert_template_from_file(project_id, project, &1)) + + total_files = length(template_paths) + on_progress = progress_callback(opts) + :ok = report_rebuild_started(on_progress, total_files, "template files") + + templates = + template_paths + |> Enum.with_index(1) + |> Enum.map(fn {path, index} -> + template = upsert_template_from_file(project_id, project, path) + :ok = report_rebuild_progress(on_progress, index, total_files, "template files") + template + end) {:ok, templates} end @@ -410,4 +422,31 @@ defmodule BDS.Templates do true -> nil end end + + defp progress_callback(opts) do + case Keyword.get(opts, :on_progress) do + callback when is_function(callback, 2) -> callback + _other -> nil + end + end + + defp report_rebuild_started(nil, _total, _label), do: :ok + + defp report_rebuild_started(callback, 0, label) do + callback.(1.0, "No #{label} found") + :ok + end + + defp report_rebuild_started(callback, total, label) do + callback.(0.05, "Rebuilding #{label} (0/#{total})") + :ok + end + + defp report_rebuild_progress(nil, _current, _total, _label), do: :ok + defp report_rebuild_progress(_callback, _current, 0, _label), do: :ok + + defp report_rebuild_progress(callback, current, total, label) do + callback.(0.05 + 0.95 * (current / total), "Rebuilding #{label} (#{current}/#{total})") + :ok + end end diff --git a/test/bds/desktop/shell_commands_test.exs b/test/bds/desktop/shell_commands_test.exs index 6b5e42c..4c66202 100644 --- a/test/bds/desktop/shell_commands_test.exs +++ b/test/bds/desktop/shell_commands_test.exs @@ -134,6 +134,63 @@ defmodule BDS.Desktop.ShellCommandsTest do assert Enum.all?(tasks, &(&1.status == :completed)) end + test "rebuild_database exposes live in-task progress for rebuild work", %{temp_dir: temp_dir} do + original = Application.get_env(:bds, :tasks, []) + + Application.put_env( + :bds, + :tasks, + original + |> Keyword.put(:max_concurrent, 1) + |> Keyword.put(:progress_throttle_ms, 0) + ) + + on_exit(fn -> Application.put_env(:bds, :tasks, original) end) + + posts_dir = Path.join([temp_dir, "posts", "2026", "04"]) + File.mkdir_p!(posts_dir) + + Enum.each(1..80, fn index -> + slug = "progress-post-#{index}" + + File.write!( + Path.join(posts_dir, "#{slug}.md"), + [ + "---", + "id: #{slug}", + "title: Progress Post #{index}", + "slug: #{slug}", + "status: published", + "createdAt: 1711843200", + "updatedAt: 1711929600", + "publishedAt: 1712016000", + "tags:", + "categories:", + "---", + "Body #{index}", + "" + ] + |> Enum.join("\n") + ) + end) + + assert {:ok, result} = ShellCommands.execute("rebuild_database") + assert result.kind == "task_queued" + + progressed = + wait_for_named_task( + "Rebuild Posts From Files", + &(&1.status == :running and is_number(&1.progress) and &1.progress > 0.0 and &1.progress < 1.0), + 5_000 + ) + + assert progressed.group_name == "Maintenance" + assert progressed.message =~ "Rebuilding post files" + + assert wait_for_task(progressed.id, &(&1.status == :completed and &1.progress == 1.0), 5_000).status == + :completed + end + test "reindex_text queues a tracked background task for the active project", %{project: project} do assert {:ok, result} = ShellCommands.execute("reindex_text") @@ -192,4 +249,19 @@ defmodule BDS.Desktop.ShellCommandsTest do wait_for_tasks_by_name(names, matcher, timeout - 50) end end + + defp wait_for_named_task(_name, _matcher, timeout) when timeout <= 0 do + flunk("named task did not reach expected state") + end + + defp wait_for_named_task(name, matcher, timeout) do + task = Enum.find(BDS.Tasks.list_tasks(), &(&1.name == name)) + + if task && matcher.(task) do + task + else + Process.sleep(20) + wait_for_named_task(name, matcher, timeout - 20) + end + end end diff --git a/test/bds/maintenance_test.exs b/test/bds/maintenance_test.exs index 2176323..2166337 100644 --- a/test/bds/maintenance_test.exs +++ b/test/bds/maintenance_test.exs @@ -50,6 +50,7 @@ defmodule BDS.MaintenanceTest do media_dir = Path.join([temp_dir, "media", "2026", "04"]) File.mkdir_p!(media_dir) + File.write!(Path.join(media_dir, "asset.txt"), "hello media") File.write!( @@ -136,6 +137,197 @@ defmodule BDS.MaintenanceTest do BDS.Maintenance.rebuild_from_filesystem(project.id, "unknown") end + test "rebuild_from_filesystem reports incremental progress for file-backed rebuilders", %{ + project: project, + temp_dir: temp_dir + } do + parent = self() + on_progress = fn value, message -> send(parent, {:rebuild_progress, value, message}) end + + posts_dir = Path.join([temp_dir, "posts", "2026", "04"]) + File.mkdir_p!(posts_dir) + + File.write!( + Path.join(posts_dir, "first-post.md"), + [ + "---", + "id: first-post", + "title: First Post", + "slug: first-post", + "status: published", + "createdAt: 1711843200", + "updatedAt: 1711929600", + "publishedAt: 1712016000", + "tags:", + "categories:", + "---", + "Body one", + "" + ] + |> Enum.join("\n") + ) + + File.write!( + Path.join(posts_dir, "second-post.md"), + [ + "---", + "id: second-post", + "title: Second Post", + "slug: second-post", + "status: published", + "createdAt: 1711843200", + "updatedAt: 1711929600", + "publishedAt: 1712016000", + "tags:", + "categories:", + "---", + "Body two", + "" + ] + |> Enum.join("\n") + ) + + media_dir = Path.join([temp_dir, "media", "2026", "04"]) + File.mkdir_p!(media_dir) + + File.write!(Path.join(media_dir, "first.txt"), "first media") + File.write!(Path.join(media_dir, "second.txt"), "second media") + + File.write!( + Path.join(media_dir, "first.txt.meta"), + [ + "id: first-media", + "originalName: first.txt", + "mimeType: text/plain", + "size: 11", + "createdAt: 1711843200", + "updatedAt: 1711929600", + "tags:", + "" + ] + |> Enum.join("\n") + ) + + File.write!( + Path.join(media_dir, "second.txt.meta"), + [ + "id: second-media", + "originalName: second.txt", + "mimeType: text/plain", + "size: 12", + "createdAt: 1711843200", + "updatedAt: 1711929600", + "tags:", + "" + ] + |> Enum.join("\n") + ) + + template_dir = Path.join(temp_dir, "templates") + File.mkdir_p!(template_dir) + + File.write!( + Path.join(template_dir, "first-template.liquid"), + [ + "---", + "id: first-template", + "slug: first-template", + "title: First Template", + "kind: list", + "enabled: true", + "version: 1", + "createdAt: 101", + "updatedAt: 202", + "---", + "
First
", + "" + ] + |> Enum.join("\n") + ) + + File.write!( + Path.join(template_dir, "second-template.liquid"), + [ + "---", + "id: second-template", + "slug: second-template", + "title: Second Template", + "kind: post", + "enabled: true", + "version: 1", + "createdAt: 303", + "updatedAt: 404", + "---", + "
Second
", + "" + ] + |> Enum.join("\n") + ) + + script_dir = Path.join(temp_dir, "scripts") + File.mkdir_p!(script_dir) + + File.write!( + Path.join(script_dir, "first-script.lua"), + [ + "---", + "id: first-script", + "slug: first-script", + "title: First Script", + "kind: utility", + "entrypoint: main", + "enabled: true", + "version: 1", + "createdAt: 505", + "updatedAt: 606", + "---", + "function main() return true end", + "" + ] + |> Enum.join("\n") + ) + + File.write!( + Path.join(script_dir, "second-script.lua"), + [ + "---", + "id: second-script", + "slug: second-script", + "title: Second Script", + "kind: transform", + "entrypoint: main", + "enabled: true", + "version: 1", + "createdAt: 707", + "updatedAt: 808", + "---", + "function main() return true end", + "" + ] + |> Enum.join("\n") + ) + + assert {:ok, _posts} = + BDS.Maintenance.rebuild_from_filesystem(project.id, "post", on_progress: on_progress) + + assert_incremental_progress(collect_progress_events()) + + assert {:ok, _media} = + BDS.Maintenance.rebuild_from_filesystem(project.id, "media", on_progress: on_progress) + + assert_incremental_progress(collect_progress_events()) + + assert {:ok, _scripts} = + BDS.Maintenance.rebuild_from_filesystem(project.id, "script", on_progress: on_progress) + + assert_incremental_progress(collect_progress_events()) + + assert {:ok, _templates} = + BDS.Maintenance.rebuild_from_filesystem(project.id, "template", on_progress: on_progress) + + assert_incremental_progress(collect_progress_events()) + end + test "maintenance rebuilds and diffs embedding state explicitly", %{project: project} do assert {:ok, _metadata} = BDS.Metadata.update_project_metadata(project.id, %{semantic_similarity_enabled: true}) @@ -507,4 +699,17 @@ defmodule BDS.MaintenanceTest do assert "scripts/orphan.lua" in orphan_paths assert "templates/orphan-view.liquid" in orphan_paths end + + defp collect_progress_events(acc \\ []) do + receive do + {:rebuild_progress, value, message} -> collect_progress_events([{value, message} | acc]) + after + 0 -> Enum.reverse(acc) + end + end + + defp assert_incremental_progress(events) do + assert Enum.any?(events, fn {value, _message} -> value > 0.0 and value < 1.0 end) + assert Enum.any?(events, fn {value, _message} -> value == 1.0 end) + end end