diff --git a/lib/bds/desktop/shell_commands.ex b/lib/bds/desktop/shell_commands.ex
index 593827a..db846b6 100644
--- a/lib/bds/desktop/shell_commands.ex
+++ b/lib/bds/desktop/shell_commands.ex
@@ -122,32 +122,28 @@ defmodule BDS.Desktop.ShellCommands do
{:ok, posts_task} =
Tasks.submit_task("Rebuild Posts From Files", fn report ->
- report.(0.0, "Scanning post files")
- {:ok, posts} = Maintenance.rebuild_from_filesystem(project.id, "post")
+ {:ok, posts} = Maintenance.rebuild_from_filesystem(project.id, "post", on_progress: report)
report.(1.0, "Post rebuild complete")
%{project_id: project.id, counts: %{posts: length(posts)}}
end, attrs)
{:ok, _media_task} =
Tasks.submit_task("Rebuild Media From Files", fn report ->
- report.(0.0, "Scanning media files")
- {:ok, media} = Maintenance.rebuild_from_filesystem(project.id, "media")
+ {:ok, media} = Maintenance.rebuild_from_filesystem(project.id, "media", on_progress: report)
report.(1.0, "Media rebuild complete")
%{project_id: project.id, counts: %{media: length(media)}}
end, attrs)
{:ok, _scripts_task} =
Tasks.submit_task("Rebuild Scripts From Files", fn report ->
- report.(0.0, "Scanning script files")
- {:ok, scripts} = Maintenance.rebuild_from_filesystem(project.id, "script")
+ {:ok, scripts} = Maintenance.rebuild_from_filesystem(project.id, "script", on_progress: report)
report.(1.0, "Script rebuild complete")
%{project_id: project.id, counts: %{scripts: length(scripts)}}
end, attrs)
{:ok, _templates_task} =
Tasks.submit_task("Rebuild Templates From Files", fn report ->
- report.(0.0, "Scanning template files")
- {:ok, templates} = Maintenance.rebuild_from_filesystem(project.id, "template")
+ {:ok, templates} = Maintenance.rebuild_from_filesystem(project.id, "template", on_progress: report)
report.(1.0, "Template rebuild complete")
%{project_id: project.id, counts: %{templates: length(templates)}}
end, attrs)
diff --git a/lib/bds/maintenance.ex b/lib/bds/maintenance.ex
index e7b7bf2..9aed98e 100644
--- a/lib/bds/maintenance.ex
+++ b/lib/bds/maintenance.ex
@@ -15,12 +15,12 @@ defmodule BDS.Maintenance do
alias BDS.Sidecar
alias BDS.Templates.Template
- def rebuild_from_filesystem(project_id, entity_type) do
+ def rebuild_from_filesystem(project_id, entity_type, opts \\ []) do
case normalize_entity_type(entity_type) do
- :post -> BDS.Posts.rebuild_posts_from_files(project_id)
- :media -> BDS.Media.rebuild_media_from_files(project_id)
- :script -> BDS.Scripts.rebuild_scripts_from_files(project_id)
- :template -> BDS.Templates.rebuild_templates_from_files(project_id)
+ :post -> BDS.Posts.rebuild_posts_from_files(project_id, opts)
+ :media -> BDS.Media.rebuild_media_from_files(project_id, opts)
+ :script -> BDS.Scripts.rebuild_scripts_from_files(project_id, opts)
+ :template -> BDS.Templates.rebuild_templates_from_files(project_id, opts)
:embedding -> Embeddings.rebuild_project(project_id)
:unsupported -> {:error, :unsupported_entity_type}
end
diff --git a/lib/bds/media.ex b/lib/bds/media.ex
index 8304a39..4fa6c92 100644
--- a/lib/bds/media.ex
+++ b/lib/bds/media.ex
@@ -311,8 +311,9 @@ defmodule BDS.Media do
end)
end
- def rebuild_media_from_files(project_id) do
+ def rebuild_media_from_files(project_id, opts \\ []) do
project = Projects.get_project!(project_id)
+ on_progress = progress_callback(opts)
canonical_sidecars =
project
@@ -322,19 +323,36 @@ defmodule BDS.Media do
|> Enum.filter(&canonical_sidecar?/1)
|> Enum.filter(&binary_exists_for_sidecar?/1)
- media_items = Enum.map(canonical_sidecars, &upsert_media_from_sidecar(project, &1))
+ translation_sidecars =
+ project
+ |> Projects.project_data_dir()
+ |> Path.join("media")
+ |> list_matching_files("*.meta")
+ |> Enum.filter(&translation_sidecar?/1)
+
+ total_files = length(canonical_sidecars) + length(translation_sidecars)
+ :ok = report_rebuild_started(on_progress, total_files, "media files")
+
+ media_items =
+ canonical_sidecars
+ |> Enum.with_index(1)
+ |> Enum.map(fn {sidecar_path, index} ->
+ media = upsert_media_from_sidecar(project, sidecar_path)
+ :ok = report_rebuild_progress(on_progress, index, total_files, "media files")
+ media
+ end)
canonical_media_by_binary_path =
Map.new(media_items, fn media ->
{Path.join(Projects.project_data_dir(project), media.file_path), media}
end)
- project
- |> Projects.project_data_dir()
- |> Path.join("media")
- |> list_matching_files("*.meta")
- |> Enum.filter(&translation_sidecar?/1)
- |> Enum.each(&upsert_translation_from_sidecar(project, canonical_media_by_binary_path, &1))
+ translation_sidecars
+ |> Enum.with_index(length(canonical_sidecars) + 1)
+ |> Enum.each(fn {sidecar_path, index} ->
+ upsert_translation_from_sidecar(project, canonical_media_by_binary_path, sidecar_path)
+ :ok = report_rebuild_progress(on_progress, index, total_files, "media files")
+ end)
{:ok, media_items}
end
@@ -629,4 +647,31 @@ defmodule BDS.Media do
true -> nil
end
end
+
+ defp progress_callback(opts) do
+ case Keyword.get(opts, :on_progress) do
+ callback when is_function(callback, 2) -> callback
+ _other -> nil
+ end
+ end
+
+ defp report_rebuild_started(nil, _total, _label), do: :ok
+
+ defp report_rebuild_started(callback, 0, label) do
+ callback.(1.0, "No #{label} found")
+ :ok
+ end
+
+ defp report_rebuild_started(callback, total, label) do
+ callback.(0.05, "Rebuilding #{label} (0/#{total})")
+ :ok
+ end
+
+ defp report_rebuild_progress(nil, _current, _total, _label), do: :ok
+ defp report_rebuild_progress(_callback, _current, 0, _label), do: :ok
+
+ defp report_rebuild_progress(callback, current, total, label) do
+ callback.(0.05 + 0.95 * (current / total), "Rebuilding #{label} (#{current}/#{total})")
+ :ok
+ end
end
diff --git a/lib/bds/posts.ex b/lib/bds/posts.ex
index 73b55cc..09c39c1 100644
--- a/lib/bds/posts.ex
+++ b/lib/bds/posts.ex
@@ -136,8 +136,9 @@ defmodule BDS.Posts do
end
end
- def rebuild_posts_from_files(project_id) do
+ def rebuild_posts_from_files(project_id, opts \\ []) do
project = Projects.get_project!(project_id)
+ on_progress = progress_callback(opts)
rebuild_files =
project
@@ -146,14 +147,26 @@ defmodule BDS.Posts do
|> list_matching_files("*.md")
|> Enum.map(&parse_rebuild_file(project, &1))
+ total_files = length(rebuild_files)
+ :ok = report_rebuild_started(on_progress, total_files, "post files")
+
{translation_files, post_files} = Enum.split_with(rebuild_files, &translation_rebuild_file?/1)
posts =
post_files
- |> Enum.map(&upsert_post_from_rebuild_file(project_id, &1))
+ |> Enum.with_index(1)
+ |> Enum.map(fn {file, index} ->
+ post = upsert_post_from_rebuild_file(project_id, file)
+ :ok = report_rebuild_progress(on_progress, index, total_files, "post files")
+ post
+ end)
translation_files
- |> Enum.map(&upsert_post_translation_from_rebuild_file(project_id, &1))
+ |> Enum.with_index(length(post_files) + 1)
+ |> Enum.each(fn {file, index} ->
+ upsert_post_translation_from_rebuild_file(project_id, file)
+ :ok = report_rebuild_progress(on_progress, index, total_files, "post files")
+ end)
{:ok, posts}
end
@@ -941,4 +954,31 @@ defmodule BDS.Posts do
true -> nil
end
end
+
+ defp progress_callback(opts) do
+ case Keyword.get(opts, :on_progress) do
+ callback when is_function(callback, 2) -> callback
+ _other -> nil
+ end
+ end
+
+ defp report_rebuild_started(nil, _total, _label), do: :ok
+
+ defp report_rebuild_started(callback, 0, label) do
+ callback.(1.0, "No #{label} found")
+ :ok
+ end
+
+ defp report_rebuild_started(callback, total, label) do
+ callback.(0.05, "Rebuilding #{label} (0/#{total})")
+ :ok
+ end
+
+ defp report_rebuild_progress(nil, _current, _total, _label), do: :ok
+ defp report_rebuild_progress(_callback, _current, 0, _label), do: :ok
+
+ defp report_rebuild_progress(callback, current, total, label) do
+ callback.(0.05 + 0.95 * (current / total), "Rebuilding #{label} (#{current}/#{total})")
+ :ok
+ end
end
diff --git a/lib/bds/scripts.ex b/lib/bds/scripts.ex
index b1559ff..d9bee5f 100644
--- a/lib/bds/scripts.ex
+++ b/lib/bds/scripts.ex
@@ -115,15 +115,27 @@ defmodule BDS.Scripts do
end
end
- def rebuild_scripts_from_files(project_id) do
+ def rebuild_scripts_from_files(project_id, opts \\ []) do
project = Projects.get_project!(project_id)
- scripts =
+ script_paths =
project
|> Projects.project_data_dir()
|> Path.join("scripts")
|> list_matching_files("*.lua")
- |> Enum.map(&upsert_script_from_file(project_id, project, &1))
+
+ total_files = length(script_paths)
+ on_progress = progress_callback(opts)
+ :ok = report_rebuild_started(on_progress, total_files, "script files")
+
+ scripts =
+ script_paths
+ |> Enum.with_index(1)
+ |> Enum.map(fn {path, index} ->
+ script = upsert_script_from_file(project_id, project, path)
+ :ok = report_rebuild_progress(on_progress, index, total_files, "script files")
+ script
+ end)
{:ok, scripts}
end
@@ -259,4 +271,31 @@ defmodule BDS.Scripts do
true -> nil
end
end
+
+ defp progress_callback(opts) do
+ case Keyword.get(opts, :on_progress) do
+ callback when is_function(callback, 2) -> callback
+ _other -> nil
+ end
+ end
+
+ defp report_rebuild_started(nil, _total, _label), do: :ok
+
+ defp report_rebuild_started(callback, 0, label) do
+ callback.(1.0, "No #{label} found")
+ :ok
+ end
+
+ defp report_rebuild_started(callback, total, label) do
+ callback.(0.05, "Rebuilding #{label} (0/#{total})")
+ :ok
+ end
+
+ defp report_rebuild_progress(nil, _current, _total, _label), do: :ok
+ defp report_rebuild_progress(_callback, _current, 0, _label), do: :ok
+
+ defp report_rebuild_progress(callback, current, total, label) do
+ callback.(0.05 + 0.95 * (current / total), "Rebuilding #{label} (#{current}/#{total})")
+ :ok
+ end
end
diff --git a/lib/bds/templates.ex b/lib/bds/templates.ex
index 48db5b2..055a19b 100644
--- a/lib/bds/templates.ex
+++ b/lib/bds/templates.ex
@@ -133,15 +133,27 @@ defmodule BDS.Templates do
end
end
- def rebuild_templates_from_files(project_id) do
+ def rebuild_templates_from_files(project_id, opts \\ []) do
project = Projects.get_project!(project_id)
- templates =
+ template_paths =
project
|> Projects.project_data_dir()
|> Path.join("templates")
|> list_matching_files("*.liquid")
- |> Enum.map(&upsert_template_from_file(project_id, project, &1))
+
+ total_files = length(template_paths)
+ on_progress = progress_callback(opts)
+ :ok = report_rebuild_started(on_progress, total_files, "template files")
+
+ templates =
+ template_paths
+ |> Enum.with_index(1)
+ |> Enum.map(fn {path, index} ->
+ template = upsert_template_from_file(project_id, project, path)
+ :ok = report_rebuild_progress(on_progress, index, total_files, "template files")
+ template
+ end)
{:ok, templates}
end
@@ -410,4 +422,31 @@ defmodule BDS.Templates do
true -> nil
end
end
+
+ defp progress_callback(opts) do
+ case Keyword.get(opts, :on_progress) do
+ callback when is_function(callback, 2) -> callback
+ _other -> nil
+ end
+ end
+
+ defp report_rebuild_started(nil, _total, _label), do: :ok
+
+ defp report_rebuild_started(callback, 0, label) do
+ callback.(1.0, "No #{label} found")
+ :ok
+ end
+
+ defp report_rebuild_started(callback, total, label) do
+ callback.(0.05, "Rebuilding #{label} (0/#{total})")
+ :ok
+ end
+
+ defp report_rebuild_progress(nil, _current, _total, _label), do: :ok
+ defp report_rebuild_progress(_callback, _current, 0, _label), do: :ok
+
+ defp report_rebuild_progress(callback, current, total, label) do
+ callback.(0.05 + 0.95 * (current / total), "Rebuilding #{label} (#{current}/#{total})")
+ :ok
+ end
end
diff --git a/test/bds/desktop/shell_commands_test.exs b/test/bds/desktop/shell_commands_test.exs
index 6b5e42c..4c66202 100644
--- a/test/bds/desktop/shell_commands_test.exs
+++ b/test/bds/desktop/shell_commands_test.exs
@@ -134,6 +134,63 @@ defmodule BDS.Desktop.ShellCommandsTest do
assert Enum.all?(tasks, &(&1.status == :completed))
end
+ test "rebuild_database exposes live in-task progress for rebuild work", %{temp_dir: temp_dir} do
+ original = Application.get_env(:bds, :tasks, [])
+
+ Application.put_env(
+ :bds,
+ :tasks,
+ original
+ |> Keyword.put(:max_concurrent, 1)
+ |> Keyword.put(:progress_throttle_ms, 0)
+ )
+
+ on_exit(fn -> Application.put_env(:bds, :tasks, original) end)
+
+ posts_dir = Path.join([temp_dir, "posts", "2026", "04"])
+ File.mkdir_p!(posts_dir)
+
+ Enum.each(1..80, fn index ->
+ slug = "progress-post-#{index}"
+
+ File.write!(
+ Path.join(posts_dir, "#{slug}.md"),
+ [
+ "---",
+ "id: #{slug}",
+ "title: Progress Post #{index}",
+ "slug: #{slug}",
+ "status: published",
+ "createdAt: 1711843200",
+ "updatedAt: 1711929600",
+ "publishedAt: 1712016000",
+ "tags:",
+ "categories:",
+ "---",
+ "Body #{index}",
+ ""
+ ]
+ |> Enum.join("\n")
+ )
+ end)
+
+ assert {:ok, result} = ShellCommands.execute("rebuild_database")
+ assert result.kind == "task_queued"
+
+ progressed =
+ wait_for_named_task(
+ "Rebuild Posts From Files",
+ &(&1.status == :running and is_number(&1.progress) and &1.progress > 0.0 and &1.progress < 1.0),
+ 5_000
+ )
+
+ assert progressed.group_name == "Maintenance"
+ assert progressed.message =~ "Rebuilding post files"
+
+ assert wait_for_task(progressed.id, &(&1.status == :completed and &1.progress == 1.0), 5_000).status ==
+ :completed
+ end
+
test "reindex_text queues a tracked background task for the active project", %{project: project} do
assert {:ok, result} = ShellCommands.execute("reindex_text")
@@ -192,4 +249,19 @@ defmodule BDS.Desktop.ShellCommandsTest do
wait_for_tasks_by_name(names, matcher, timeout - 50)
end
end
+
+ defp wait_for_named_task(_name, _matcher, timeout) when timeout <= 0 do
+ flunk("named task did not reach expected state")
+ end
+
+ defp wait_for_named_task(name, matcher, timeout) do
+ task = Enum.find(BDS.Tasks.list_tasks(), &(&1.name == name))
+
+ if task && matcher.(task) do
+ task
+ else
+ Process.sleep(20)
+ wait_for_named_task(name, matcher, timeout - 20)
+ end
+ end
end
diff --git a/test/bds/maintenance_test.exs b/test/bds/maintenance_test.exs
index 2176323..2166337 100644
--- a/test/bds/maintenance_test.exs
+++ b/test/bds/maintenance_test.exs
@@ -50,6 +50,7 @@ defmodule BDS.MaintenanceTest do
media_dir = Path.join([temp_dir, "media", "2026", "04"])
File.mkdir_p!(media_dir)
+
File.write!(Path.join(media_dir, "asset.txt"), "hello media")
File.write!(
@@ -136,6 +137,197 @@ defmodule BDS.MaintenanceTest do
BDS.Maintenance.rebuild_from_filesystem(project.id, "unknown")
end
+ test "rebuild_from_filesystem reports incremental progress for file-backed rebuilders", %{
+ project: project,
+ temp_dir: temp_dir
+ } do
+ parent = self()
+ on_progress = fn value, message -> send(parent, {:rebuild_progress, value, message}) end
+
+ posts_dir = Path.join([temp_dir, "posts", "2026", "04"])
+ File.mkdir_p!(posts_dir)
+
+ File.write!(
+ Path.join(posts_dir, "first-post.md"),
+ [
+ "---",
+ "id: first-post",
+ "title: First Post",
+ "slug: first-post",
+ "status: published",
+ "createdAt: 1711843200",
+ "updatedAt: 1711929600",
+ "publishedAt: 1712016000",
+ "tags:",
+ "categories:",
+ "---",
+ "Body one",
+ ""
+ ]
+ |> Enum.join("\n")
+ )
+
+ File.write!(
+ Path.join(posts_dir, "second-post.md"),
+ [
+ "---",
+ "id: second-post",
+ "title: Second Post",
+ "slug: second-post",
+ "status: published",
+ "createdAt: 1711843200",
+ "updatedAt: 1711929600",
+ "publishedAt: 1712016000",
+ "tags:",
+ "categories:",
+ "---",
+ "Body two",
+ ""
+ ]
+ |> Enum.join("\n")
+ )
+
+ media_dir = Path.join([temp_dir, "media", "2026", "04"])
+ File.mkdir_p!(media_dir)
+
+ File.write!(Path.join(media_dir, "first.txt"), "first media")
+ File.write!(Path.join(media_dir, "second.txt"), "second media")
+
+ File.write!(
+ Path.join(media_dir, "first.txt.meta"),
+ [
+ "id: first-media",
+ "originalName: first.txt",
+ "mimeType: text/plain",
+ "size: 11",
+ "createdAt: 1711843200",
+ "updatedAt: 1711929600",
+ "tags:",
+ ""
+ ]
+ |> Enum.join("\n")
+ )
+
+ File.write!(
+ Path.join(media_dir, "second.txt.meta"),
+ [
+ "id: second-media",
+ "originalName: second.txt",
+ "mimeType: text/plain",
+ "size: 12",
+ "createdAt: 1711843200",
+ "updatedAt: 1711929600",
+ "tags:",
+ ""
+ ]
+ |> Enum.join("\n")
+ )
+
+ template_dir = Path.join(temp_dir, "templates")
+ File.mkdir_p!(template_dir)
+
+ File.write!(
+ Path.join(template_dir, "first-template.liquid"),
+ [
+ "---",
+ "id: first-template",
+ "slug: first-template",
+ "title: First Template",
+ "kind: list",
+ "enabled: true",
+ "version: 1",
+ "createdAt: 101",
+ "updatedAt: 202",
+ "---",
+ "",
+ ""
+ ]
+ |> Enum.join("\n")
+ )
+
+ File.write!(
+ Path.join(template_dir, "second-template.liquid"),
+ [
+ "---",
+ "id: second-template",
+ "slug: second-template",
+ "title: Second Template",
+ "kind: post",
+ "enabled: true",
+ "version: 1",
+ "createdAt: 303",
+ "updatedAt: 404",
+ "---",
+ "",
+ ""
+ ]
+ |> Enum.join("\n")
+ )
+
+ script_dir = Path.join(temp_dir, "scripts")
+ File.mkdir_p!(script_dir)
+
+ File.write!(
+ Path.join(script_dir, "first-script.lua"),
+ [
+ "---",
+ "id: first-script",
+ "slug: first-script",
+ "title: First Script",
+ "kind: utility",
+ "entrypoint: main",
+ "enabled: true",
+ "version: 1",
+ "createdAt: 505",
+ "updatedAt: 606",
+ "---",
+ "function main() return true end",
+ ""
+ ]
+ |> Enum.join("\n")
+ )
+
+ File.write!(
+ Path.join(script_dir, "second-script.lua"),
+ [
+ "---",
+ "id: second-script",
+ "slug: second-script",
+ "title: Second Script",
+ "kind: transform",
+ "entrypoint: main",
+ "enabled: true",
+ "version: 1",
+ "createdAt: 707",
+ "updatedAt: 808",
+ "---",
+ "function main() return true end",
+ ""
+ ]
+ |> Enum.join("\n")
+ )
+
+ assert {:ok, _posts} =
+ BDS.Maintenance.rebuild_from_filesystem(project.id, "post", on_progress: on_progress)
+
+ assert_incremental_progress(collect_progress_events())
+
+ assert {:ok, _media} =
+ BDS.Maintenance.rebuild_from_filesystem(project.id, "media", on_progress: on_progress)
+
+ assert_incremental_progress(collect_progress_events())
+
+ assert {:ok, _scripts} =
+ BDS.Maintenance.rebuild_from_filesystem(project.id, "script", on_progress: on_progress)
+
+ assert_incremental_progress(collect_progress_events())
+
+ assert {:ok, _templates} =
+ BDS.Maintenance.rebuild_from_filesystem(project.id, "template", on_progress: on_progress)
+
+ assert_incremental_progress(collect_progress_events())
+ end
+
test "maintenance rebuilds and diffs embedding state explicitly", %{project: project} do
assert {:ok, _metadata} =
BDS.Metadata.update_project_metadata(project.id, %{semantic_similarity_enabled: true})
@@ -507,4 +699,17 @@ defmodule BDS.MaintenanceTest do
assert "scripts/orphan.lua" in orphan_paths
assert "templates/orphan-view.liquid" in orphan_paths
end
+
+ defp collect_progress_events(acc \\ []) do
+ receive do
+ {:rebuild_progress, value, message} -> collect_progress_events([{value, message} | acc])
+ after
+ 0 -> Enum.reverse(acc)
+ end
+ end
+
+ defp assert_incremental_progress(events) do
+ assert Enum.any?(events, fn {value, _message} -> value > 0.0 and value < 1.0 end)
+ assert Enum.any?(events, fn {value, _message} -> value == 1.0 end)
+ end
end