fix: better progress reporting on tasks

lib/bds/desktop/shell_commands.ex

@@ -122,32 +122,28 @@ defmodule BDS.Desktop.ShellCommands do
 
     {:ok, posts_task} =
       Tasks.submit_task("Rebuild Posts From Files", fn report ->
-        report.(0.0, "Scanning post files")
+        {:ok, posts} = Maintenance.rebuild_from_filesystem(project.id, "post", on_progress: report)
-        {:ok, posts} = Maintenance.rebuild_from_filesystem(project.id, "post")
         report.(1.0, "Post rebuild complete")
         %{project_id: project.id, counts: %{posts: length(posts)}}
       end, attrs)
 
     {:ok, _media_task} =
       Tasks.submit_task("Rebuild Media From Files", fn report ->
-        report.(0.0, "Scanning media files")
+        {:ok, media} = Maintenance.rebuild_from_filesystem(project.id, "media", on_progress: report)
-        {:ok, media} = Maintenance.rebuild_from_filesystem(project.id, "media")
         report.(1.0, "Media rebuild complete")
         %{project_id: project.id, counts: %{media: length(media)}}
       end, attrs)
 
     {:ok, _scripts_task} =
       Tasks.submit_task("Rebuild Scripts From Files", fn report ->
-        report.(0.0, "Scanning script files")
+        {:ok, scripts} = Maintenance.rebuild_from_filesystem(project.id, "script", on_progress: report)
-        {:ok, scripts} = Maintenance.rebuild_from_filesystem(project.id, "script")
         report.(1.0, "Script rebuild complete")
         %{project_id: project.id, counts: %{scripts: length(scripts)}}
       end, attrs)
 
     {:ok, _templates_task} =
       Tasks.submit_task("Rebuild Templates From Files", fn report ->
-        report.(0.0, "Scanning template files")
+        {:ok, templates} = Maintenance.rebuild_from_filesystem(project.id, "template", on_progress: report)
-        {:ok, templates} = Maintenance.rebuild_from_filesystem(project.id, "template")
         report.(1.0, "Template rebuild complete")
         %{project_id: project.id, counts: %{templates: length(templates)}}
       end, attrs)

lib/bds/maintenance.ex

@@ -15,12 +15,12 @@ defmodule BDS.Maintenance do
   alias BDS.Sidecar
   alias BDS.Templates.Template
 
-  def rebuild_from_filesystem(project_id, entity_type) do
+  def rebuild_from_filesystem(project_id, entity_type, opts \\ []) do
     case normalize_entity_type(entity_type) do
-      :post -> BDS.Posts.rebuild_posts_from_files(project_id)
+      :post -> BDS.Posts.rebuild_posts_from_files(project_id, opts)
-      :media -> BDS.Media.rebuild_media_from_files(project_id)
+      :media -> BDS.Media.rebuild_media_from_files(project_id, opts)
-      :script -> BDS.Scripts.rebuild_scripts_from_files(project_id)
+      :script -> BDS.Scripts.rebuild_scripts_from_files(project_id, opts)
-      :template -> BDS.Templates.rebuild_templates_from_files(project_id)
+      :template -> BDS.Templates.rebuild_templates_from_files(project_id, opts)
       :embedding -> Embeddings.rebuild_project(project_id)
       :unsupported -> {:error, :unsupported_entity_type}
     end

lib/bds/media.ex

@@ -311,8 +311,9 @@ defmodule BDS.Media do
     end)
   end
 
-  def rebuild_media_from_files(project_id) do
+  def rebuild_media_from_files(project_id, opts \\ []) do
     project = Projects.get_project!(project_id)
+    on_progress = progress_callback(opts)
 
     canonical_sidecars =
       project
@@ -322,19 +323,36 @@ defmodule BDS.Media do
       |> Enum.filter(&canonical_sidecar?/1)
       |> Enum.filter(&binary_exists_for_sidecar?/1)
 
-    media_items = Enum.map(canonical_sidecars, &upsert_media_from_sidecar(project, &1))
+    translation_sidecars =
+      project
+      |> Projects.project_data_dir()
+      |> Path.join("media")
+      |> list_matching_files("*.meta")
+      |> Enum.filter(&translation_sidecar?/1)
+
+    total_files = length(canonical_sidecars) + length(translation_sidecars)
+    :ok = report_rebuild_started(on_progress, total_files, "media files")
+
+    media_items =
+      canonical_sidecars
+      |> Enum.with_index(1)
+      |> Enum.map(fn {sidecar_path, index} ->
+        media = upsert_media_from_sidecar(project, sidecar_path)
+        :ok = report_rebuild_progress(on_progress, index, total_files, "media files")
+        media
+      end)
 
     canonical_media_by_binary_path =
       Map.new(media_items, fn media ->
         {Path.join(Projects.project_data_dir(project), media.file_path), media}
       end)
 
-    project
+    translation_sidecars
-    |> Projects.project_data_dir()
+    |> Enum.with_index(length(canonical_sidecars) + 1)
-    |> Path.join("media")
+    |> Enum.each(fn {sidecar_path, index} ->
-    |> list_matching_files("*.meta")
+      upsert_translation_from_sidecar(project, canonical_media_by_binary_path, sidecar_path)
-    |> Enum.filter(&translation_sidecar?/1)
+      :ok = report_rebuild_progress(on_progress, index, total_files, "media files")
-    |> Enum.each(&upsert_translation_from_sidecar(project, canonical_media_by_binary_path, &1))
+    end)
 
     {:ok, media_items}
   end
@@ -629,4 +647,31 @@ defmodule BDS.Media do
       true -> nil
     end
   end
+
+  defp progress_callback(opts) do
+    case Keyword.get(opts, :on_progress) do
+      callback when is_function(callback, 2) -> callback
+      _other -> nil
+    end
+  end
+
+  defp report_rebuild_started(nil, _total, _label), do: :ok
+
+  defp report_rebuild_started(callback, 0, label) do
+    callback.(1.0, "No #{label} found")
+    :ok
+  end
+
+  defp report_rebuild_started(callback, total, label) do
+    callback.(0.05, "Rebuilding #{label} (0/#{total})")
+    :ok
+  end
+
+  defp report_rebuild_progress(nil, _current, _total, _label), do: :ok
+  defp report_rebuild_progress(_callback, _current, 0, _label), do: :ok
+
+  defp report_rebuild_progress(callback, current, total, label) do
+    callback.(0.05 + 0.95 * (current / total), "Rebuilding #{label} (#{current}/#{total})")
+    :ok
+  end
 end

lib/bds/posts.ex

@@ -136,8 +136,9 @@ defmodule BDS.Posts do
     end
   end
 
-  def rebuild_posts_from_files(project_id) do
+  def rebuild_posts_from_files(project_id, opts \\ []) do
     project = Projects.get_project!(project_id)
+    on_progress = progress_callback(opts)
 
     rebuild_files =
       project
@@ -146,14 +147,26 @@ defmodule BDS.Posts do
       |> list_matching_files("*.md")
       |> Enum.map(&parse_rebuild_file(project, &1))
 
+    total_files = length(rebuild_files)
+    :ok = report_rebuild_started(on_progress, total_files, "post files")
+
     {translation_files, post_files} = Enum.split_with(rebuild_files, &translation_rebuild_file?/1)
 
     posts =
       post_files
-      |> Enum.map(&upsert_post_from_rebuild_file(project_id, &1))
+      |> Enum.with_index(1)
+      |> Enum.map(fn {file, index} ->
+        post = upsert_post_from_rebuild_file(project_id, file)
+        :ok = report_rebuild_progress(on_progress, index, total_files, "post files")
+        post
+      end)
 
     translation_files
-    |> Enum.map(&upsert_post_translation_from_rebuild_file(project_id, &1))
+    |> Enum.with_index(length(post_files) + 1)
+    |> Enum.each(fn {file, index} ->
+      upsert_post_translation_from_rebuild_file(project_id, file)
+      :ok = report_rebuild_progress(on_progress, index, total_files, "post files")
+    end)
 
     {:ok, posts}
   end
@@ -941,4 +954,31 @@ defmodule BDS.Posts do
       true -> nil
     end
   end
+
+  defp progress_callback(opts) do
+    case Keyword.get(opts, :on_progress) do
+      callback when is_function(callback, 2) -> callback
+      _other -> nil
+    end
+  end
+
+  defp report_rebuild_started(nil, _total, _label), do: :ok
+
+  defp report_rebuild_started(callback, 0, label) do
+    callback.(1.0, "No #{label} found")
+    :ok
+  end
+
+  defp report_rebuild_started(callback, total, label) do
+    callback.(0.05, "Rebuilding #{label} (0/#{total})")
+    :ok
+  end
+
+  defp report_rebuild_progress(nil, _current, _total, _label), do: :ok
+  defp report_rebuild_progress(_callback, _current, 0, _label), do: :ok
+
+  defp report_rebuild_progress(callback, current, total, label) do
+    callback.(0.05 + 0.95 * (current / total), "Rebuilding #{label} (#{current}/#{total})")
+    :ok
+  end
 end

lib/bds/scripts.ex

@@ -115,15 +115,27 @@ defmodule BDS.Scripts do
     end
   end
 
-  def rebuild_scripts_from_files(project_id) do
+  def rebuild_scripts_from_files(project_id, opts \\ []) do
     project = Projects.get_project!(project_id)
 
-    scripts =
+    script_paths =
       project
       |> Projects.project_data_dir()
       |> Path.join("scripts")
       |> list_matching_files("*.lua")
-      |> Enum.map(&upsert_script_from_file(project_id, project, &1))
+
+    total_files = length(script_paths)
+    on_progress = progress_callback(opts)
+    :ok = report_rebuild_started(on_progress, total_files, "script files")
+
+    scripts =
+      script_paths
+      |> Enum.with_index(1)
+      |> Enum.map(fn {path, index} ->
+        script = upsert_script_from_file(project_id, project, path)
+        :ok = report_rebuild_progress(on_progress, index, total_files, "script files")
+        script
+      end)
 
     {:ok, scripts}
   end
@@ -259,4 +271,31 @@ defmodule BDS.Scripts do
       true -> nil
     end
   end
+
+  defp progress_callback(opts) do
+    case Keyword.get(opts, :on_progress) do
+      callback when is_function(callback, 2) -> callback
+      _other -> nil
+    end
+  end
+
+  defp report_rebuild_started(nil, _total, _label), do: :ok
+
+  defp report_rebuild_started(callback, 0, label) do
+    callback.(1.0, "No #{label} found")
+    :ok
+  end
+
+  defp report_rebuild_started(callback, total, label) do
+    callback.(0.05, "Rebuilding #{label} (0/#{total})")
+    :ok
+  end
+
+  defp report_rebuild_progress(nil, _current, _total, _label), do: :ok
+  defp report_rebuild_progress(_callback, _current, 0, _label), do: :ok
+
+  defp report_rebuild_progress(callback, current, total, label) do
+    callback.(0.05 + 0.95 * (current / total), "Rebuilding #{label} (#{current}/#{total})")
+    :ok
+  end
 end

lib/bds/templates.ex

@@ -133,15 +133,27 @@ defmodule BDS.Templates do
     end
   end
 
-  def rebuild_templates_from_files(project_id) do
+  def rebuild_templates_from_files(project_id, opts \\ []) do
     project = Projects.get_project!(project_id)
 
-    templates =
+    template_paths =
       project
       |> Projects.project_data_dir()
       |> Path.join("templates")
       |> list_matching_files("*.liquid")
-      |> Enum.map(&upsert_template_from_file(project_id, project, &1))
+
+    total_files = length(template_paths)
+    on_progress = progress_callback(opts)
+    :ok = report_rebuild_started(on_progress, total_files, "template files")
+
+    templates =
+      template_paths
+      |> Enum.with_index(1)
+      |> Enum.map(fn {path, index} ->
+        template = upsert_template_from_file(project_id, project, path)
+        :ok = report_rebuild_progress(on_progress, index, total_files, "template files")
+        template
+      end)
 
     {:ok, templates}
   end
@@ -410,4 +422,31 @@ defmodule BDS.Templates do
       true -> nil
     end
   end
+
+  defp progress_callback(opts) do
+    case Keyword.get(opts, :on_progress) do
+      callback when is_function(callback, 2) -> callback
+      _other -> nil
+    end
+  end
+
+  defp report_rebuild_started(nil, _total, _label), do: :ok
+
+  defp report_rebuild_started(callback, 0, label) do
+    callback.(1.0, "No #{label} found")
+    :ok
+  end
+
+  defp report_rebuild_started(callback, total, label) do
+    callback.(0.05, "Rebuilding #{label} (0/#{total})")
+    :ok
+  end
+
+  defp report_rebuild_progress(nil, _current, _total, _label), do: :ok
+  defp report_rebuild_progress(_callback, _current, 0, _label), do: :ok
+
+  defp report_rebuild_progress(callback, current, total, label) do
+    callback.(0.05 + 0.95 * (current / total), "Rebuilding #{label} (#{current}/#{total})")
+    :ok
+  end
 end

test/bds/desktop/shell_commands_test.exs

@@ -134,6 +134,63 @@ defmodule BDS.Desktop.ShellCommandsTest do
     assert Enum.all?(tasks, &(&1.status == :completed))
   end
 
+  test "rebuild_database exposes live in-task progress for rebuild work", %{temp_dir: temp_dir} do
+    original = Application.get_env(:bds, :tasks, [])
+
+    Application.put_env(
+      :bds,
+      :tasks,
+      original
+      |> Keyword.put(:max_concurrent, 1)
+      |> Keyword.put(:progress_throttle_ms, 0)
+    )
+
+    on_exit(fn -> Application.put_env(:bds, :tasks, original) end)
+
+    posts_dir = Path.join([temp_dir, "posts", "2026", "04"])
+    File.mkdir_p!(posts_dir)
+
+    Enum.each(1..80, fn index ->
+      slug = "progress-post-#{index}"
+
+      File.write!(
+        Path.join(posts_dir, "#{slug}.md"),
+        [
+          "---",
+          "id: #{slug}",
+          "title: Progress Post #{index}",
+          "slug: #{slug}",
+          "status: published",
+          "createdAt: 1711843200",
+          "updatedAt: 1711929600",
+          "publishedAt: 1712016000",
+          "tags:",
+          "categories:",
+          "---",
+          "Body #{index}",
+          ""
+        ]
+        |> Enum.join("\n")
+      )
+    end)
+
+    assert {:ok, result} = ShellCommands.execute("rebuild_database")
+    assert result.kind == "task_queued"
+
+    progressed =
+      wait_for_named_task(
+        "Rebuild Posts From Files",
+        &(&1.status == :running and is_number(&1.progress) and &1.progress > 0.0 and &1.progress < 1.0),
+        5_000
+      )
+
+    assert progressed.group_name == "Maintenance"
+    assert progressed.message =~ "Rebuilding post files"
+
+    assert wait_for_task(progressed.id, &(&1.status == :completed and &1.progress == 1.0), 5_000).status ==
+             :completed
+  end
+
   test "reindex_text queues a tracked background task for the active project", %{project: project} do
     assert {:ok, result} = ShellCommands.execute("reindex_text")
 
@@ -192,4 +249,19 @@ defmodule BDS.Desktop.ShellCommandsTest do
       wait_for_tasks_by_name(names, matcher, timeout - 50)
     end
   end
+
+  defp wait_for_named_task(_name, _matcher, timeout) when timeout <= 0 do
+    flunk("named task did not reach expected state")
+  end
+
+  defp wait_for_named_task(name, matcher, timeout) do
+    task = Enum.find(BDS.Tasks.list_tasks(), &(&1.name == name))
+
+    if task && matcher.(task) do
+      task
+    else
+      Process.sleep(20)
+      wait_for_named_task(name, matcher, timeout - 20)
+    end
+  end
 end

test/bds/maintenance_test.exs

@@ -50,6 +50,7 @@ defmodule BDS.MaintenanceTest do
 
     media_dir = Path.join([temp_dir, "media", "2026", "04"])
     File.mkdir_p!(media_dir)
+
     File.write!(Path.join(media_dir, "asset.txt"), "hello media")
 
     File.write!(
@@ -136,6 +137,197 @@ defmodule BDS.MaintenanceTest do
              BDS.Maintenance.rebuild_from_filesystem(project.id, "unknown")
   end
 
+  test "rebuild_from_filesystem reports incremental progress for file-backed rebuilders", %{
+    project: project,
+    temp_dir: temp_dir
+  } do
+    parent = self()
+    on_progress = fn value, message -> send(parent, {:rebuild_progress, value, message}) end
+
+    posts_dir = Path.join([temp_dir, "posts", "2026", "04"])
+    File.mkdir_p!(posts_dir)
+
+    File.write!(
+      Path.join(posts_dir, "first-post.md"),
+      [
+        "---",
+        "id: first-post",
+        "title: First Post",
+        "slug: first-post",
+        "status: published",
+        "createdAt: 1711843200",
+        "updatedAt: 1711929600",
+        "publishedAt: 1712016000",
+        "tags:",
+        "categories:",
+        "---",
+        "Body one",
+        ""
+      ]
+      |> Enum.join("\n")
+    )
+
+    File.write!(
+      Path.join(posts_dir, "second-post.md"),
+      [
+        "---",
+        "id: second-post",
+        "title: Second Post",
+        "slug: second-post",
+        "status: published",
+        "createdAt: 1711843200",
+        "updatedAt: 1711929600",
+        "publishedAt: 1712016000",
+        "tags:",
+        "categories:",
+        "---",
+        "Body two",
+        ""
+      ]
+      |> Enum.join("\n")
+    )
+
+    media_dir = Path.join([temp_dir, "media", "2026", "04"])
+    File.mkdir_p!(media_dir)
+
+    File.write!(Path.join(media_dir, "first.txt"), "first media")
+    File.write!(Path.join(media_dir, "second.txt"), "second media")
+
+    File.write!(
+      Path.join(media_dir, "first.txt.meta"),
+      [
+        "id: first-media",
+        "originalName: first.txt",
+        "mimeType: text/plain",
+        "size: 11",
+        "createdAt: 1711843200",
+        "updatedAt: 1711929600",
+        "tags:",
+        ""
+      ]
+      |> Enum.join("\n")
+    )
+
+    File.write!(
+      Path.join(media_dir, "second.txt.meta"),
+      [
+        "id: second-media",
+        "originalName: second.txt",
+        "mimeType: text/plain",
+        "size: 12",
+        "createdAt: 1711843200",
+        "updatedAt: 1711929600",
+        "tags:",
+        ""
+      ]
+      |> Enum.join("\n")
+    )
+
+    template_dir = Path.join(temp_dir, "templates")
+    File.mkdir_p!(template_dir)
+
+    File.write!(
+      Path.join(template_dir, "first-template.liquid"),
+      [
+        "---",
+        "id: first-template",
+        "slug: first-template",
+        "title: First Template",
+        "kind: list",
+        "enabled: true",
+        "version: 1",
+        "createdAt: 101",
+        "updatedAt: 202",
+        "---",
+        "<section>First</section>",
+        ""
+      ]
+      |> Enum.join("\n")
+    )
+
+    File.write!(
+      Path.join(template_dir, "second-template.liquid"),
+      [
+        "---",
+        "id: second-template",
+        "slug: second-template",
+        "title: Second Template",
+        "kind: post",
+        "enabled: true",
+        "version: 1",
+        "createdAt: 303",
+        "updatedAt: 404",
+        "---",
+        "<section>Second</section>",
+        ""
+      ]
+      |> Enum.join("\n")
+    )
+
+    script_dir = Path.join(temp_dir, "scripts")
+    File.mkdir_p!(script_dir)
+
+    File.write!(
+      Path.join(script_dir, "first-script.lua"),
+      [
+        "---",
+        "id: first-script",
+        "slug: first-script",
+        "title: First Script",
+        "kind: utility",
+        "entrypoint: main",
+        "enabled: true",
+        "version: 1",
+        "createdAt: 505",
+        "updatedAt: 606",
+        "---",
+        "function main() return true end",
+        ""
+      ]
+      |> Enum.join("\n")
+    )
+
+    File.write!(
+      Path.join(script_dir, "second-script.lua"),
+      [
+        "---",
+        "id: second-script",
+        "slug: second-script",
+        "title: Second Script",
+        "kind: transform",
+        "entrypoint: main",
+        "enabled: true",
+        "version: 1",
+        "createdAt: 707",
+        "updatedAt: 808",
+        "---",
+        "function main() return true end",
+        ""
+      ]
+      |> Enum.join("\n")
+    )
+
+    assert {:ok, _posts} =
+             BDS.Maintenance.rebuild_from_filesystem(project.id, "post", on_progress: on_progress)
+
+    assert_incremental_progress(collect_progress_events())
+
+    assert {:ok, _media} =
+             BDS.Maintenance.rebuild_from_filesystem(project.id, "media", on_progress: on_progress)
+
+    assert_incremental_progress(collect_progress_events())
+
+    assert {:ok, _scripts} =
+             BDS.Maintenance.rebuild_from_filesystem(project.id, "script", on_progress: on_progress)
+
+    assert_incremental_progress(collect_progress_events())
+
+    assert {:ok, _templates} =
+             BDS.Maintenance.rebuild_from_filesystem(project.id, "template", on_progress: on_progress)
+
+    assert_incremental_progress(collect_progress_events())
+  end
+
   test "maintenance rebuilds and diffs embedding state explicitly", %{project: project} do
     assert {:ok, _metadata} =
              BDS.Metadata.update_project_metadata(project.id, %{semantic_similarity_enabled: true})
@@ -507,4 +699,17 @@ defmodule BDS.MaintenanceTest do
     assert "scripts/orphan.lua" in orphan_paths
     assert "templates/orphan-view.liquid" in orphan_paths
   end
+
+  defp collect_progress_events(acc \\ []) do
+    receive do
+      {:rebuild_progress, value, message} -> collect_progress_events([{value, message} | acc])
+    after
+      0 -> Enum.reverse(acc)
+    end
+  end
+
+  defp assert_incremental_progress(events) do
+    assert Enum.any?(events, fn {value, _message} -> value > 0.0 and value < 1.0 end)
+    assert Enum.any?(events, fn {value, _message} -> value == 1.0 end)
+  end
 end