diff --git a/config/config.exs b/config/config.exs index adf8ce5..3a7be60 100644 --- a/config/config.exs +++ b/config/config.exs @@ -6,6 +6,8 @@ config :bds, config :bds, BDS.Repo, database: Path.expand("../priv/data/bds_dev.db", __DIR__), pool_size: 5, + busy_timeout: 15_000, + log: false, stacktrace: true, show_sensitive_data_on_connection_error: true diff --git a/lib/bds/desktop/shell_commands.ex b/lib/bds/desktop/shell_commands.ex index db846b6..60c8883 100644 --- a/lib/bds/desktop/shell_commands.ex +++ b/lib/bds/desktop/shell_commands.ex @@ -13,6 +13,7 @@ defmodule BDS.Desktop.ShellCommands do alias BDS.Tasks @site_sections [:core, :single, :category, :tag, :date] + @rebuild_phase_timeout 600_000 def execute(action, params \\ %{}) @@ -119,44 +120,16 @@ defmodule BDS.Desktop.ShellCommands do defp dispatch("rebuild_database", project, _params) do group_id = task_group_id("rebuild_database") attrs = %{group_id: group_id, group_name: "Maintenance"} + [first_step | remaining_steps] = rebuild_database_steps(project) {:ok, posts_task} = - Tasks.submit_task("Rebuild Posts From Files", fn report -> - {:ok, posts} = Maintenance.rebuild_from_filesystem(project.id, "post", on_progress: report) - report.(1.0, "Post rebuild complete") - %{project_id: project.id, counts: %{posts: length(posts)}} - end, attrs) - - {:ok, _media_task} = - Tasks.submit_task("Rebuild Media From Files", fn report -> - {:ok, media} = Maintenance.rebuild_from_filesystem(project.id, "media", on_progress: report) - report.(1.0, "Media rebuild complete") - %{project_id: project.id, counts: %{media: length(media)}} - end, attrs) - - {:ok, _scripts_task} = - Tasks.submit_task("Rebuild Scripts From Files", fn report -> - {:ok, scripts} = Maintenance.rebuild_from_filesystem(project.id, "script", on_progress: report) - report.(1.0, "Script rebuild complete") - %{project_id: project.id, counts: %{scripts: length(scripts)}} - end, attrs) - - {:ok, _templates_task} = - Tasks.submit_task("Rebuild Templates From Files", fn report -> - {:ok, templates} = Maintenance.rebuild_from_filesystem(project.id, "template", on_progress: report) - report.(1.0, "Template rebuild complete") - %{project_id: project.id, counts: %{templates: length(templates)}} - end, attrs) + Tasks.submit_task(first_step.name, first_step.work, attrs) Task.start(fn -> - wait_for_group_phase(group_id, [ - "Rebuild Posts From Files", - "Rebuild Media From Files", - "Rebuild Scripts From Files", - "Rebuild Templates From Files" - ]) - - submit_rebuild_followups(project, attrs) + case wait_for_group_phase(group_id, [first_step.name], @rebuild_phase_timeout) do + :ok -> run_rebuild_sequence(group_id, attrs, remaining_steps) + _other -> :ok + end end) {:ok, @@ -258,35 +231,80 @@ defmodule BDS.Desktop.ShellCommands do }} end - defp submit_rebuild_followups(project, attrs) do - {:ok, _links_task} = - Tasks.submit_task("Rebuild Post Links", fn report -> - report.(0.0, "Rebuilding link graph") - :ok = Posts.rebuild_post_links(project.id) - report.(1.0, "Post links rebuilt") - %{project_id: project.id} - end, attrs) - - {:ok, _thumbs_task} = - Tasks.submit_task("Regenerate Missing Thumbnails", fn report -> - report.(0.0, "Checking missing thumbnails") - result = BDS.Media.regenerate_missing_thumbnails(project.id) - report.(1.0, "Missing thumbnails regenerated") - Map.put(result, :project_id, project.id) - end, attrs) - - {:ok, _embeddings_task} = - Tasks.submit_task("Rebuild Embedding Index", fn report -> - report.(0.0, "Rebuilding semantic index") - {:ok, rebuilt_post_ids} = Embeddings.rebuild_project(project.id) - report.(1.0, "Embedding index rebuilt") - %{project_id: project.id, rebuilt_post_ids: rebuilt_post_ids, rebuilt_count: length(rebuilt_post_ids)} - end, attrs) - - :ok + defp rebuild_database_steps(project) do + [ + %{ + name: "Rebuild Posts From Files", + work: fn report -> + {:ok, posts} = Maintenance.rebuild_from_filesystem(project.id, "post", on_progress: report) + report.(1.0, "Post rebuild complete") + %{project_id: project.id, counts: %{posts: length(posts)}} + end + }, + %{ + name: "Rebuild Media From Files", + work: fn report -> + {:ok, media} = Maintenance.rebuild_from_filesystem(project.id, "media", on_progress: report) + report.(1.0, "Media rebuild complete") + %{project_id: project.id, counts: %{media: length(media)}} + end + }, + %{ + name: "Rebuild Scripts From Files", + work: fn report -> + {:ok, scripts} = Maintenance.rebuild_from_filesystem(project.id, "script", on_progress: report) + report.(1.0, "Script rebuild complete") + %{project_id: project.id, counts: %{scripts: length(scripts)}} + end + }, + %{ + name: "Rebuild Templates From Files", + work: fn report -> + {:ok, templates} = Maintenance.rebuild_from_filesystem(project.id, "template", on_progress: report) + report.(1.0, "Template rebuild complete") + %{project_id: project.id, counts: %{templates: length(templates)}} + end + }, + %{ + name: "Rebuild Post Links", + work: fn report -> + report.(0.0, "Rebuilding link graph") + :ok = Posts.rebuild_post_links(project.id) + report.(1.0, "Post links rebuilt") + %{project_id: project.id} + end + }, + %{ + name: "Regenerate Missing Thumbnails", + work: fn report -> + report.(0.0, "Checking missing thumbnails") + result = BDS.Media.regenerate_missing_thumbnails(project.id) + report.(1.0, "Missing thumbnails regenerated") + Map.put(result, :project_id, project.id) + end + }, + %{ + name: "Rebuild Embedding Index", + work: fn report -> + report.(0.0, "Rebuilding semantic index") + {:ok, rebuilt_post_ids} = Embeddings.rebuild_project(project.id) + report.(1.0, "Embedding index rebuilt") + %{project_id: project.id, rebuilt_post_ids: rebuilt_post_ids, rebuilt_count: length(rebuilt_post_ids)} + end + } + ] end - defp wait_for_group_phase(group_id, names, timeout \\ 30_000) + defp run_rebuild_sequence(_group_id, _attrs, []), do: :ok + + defp run_rebuild_sequence(group_id, attrs, [step | remaining_steps]) do + {:ok, _task} = Tasks.submit_task(step.name, step.work, attrs) + + case wait_for_group_phase(group_id, [step.name], @rebuild_phase_timeout) do + :ok -> run_rebuild_sequence(group_id, attrs, remaining_steps) + _other -> :ok + end + end defp wait_for_group_phase(_group_id, _names, timeout) when timeout <= 0, do: :timeout diff --git a/test/bds/desktop/shell_commands_test.exs b/test/bds/desktop/shell_commands_test.exs index 4c66202..487c292 100644 --- a/test/bds/desktop/shell_commands_test.exs +++ b/test/bds/desktop/shell_commands_test.exs @@ -134,6 +134,144 @@ defmodule BDS.Desktop.ShellCommandsTest do assert Enum.all?(tasks, &(&1.status == :completed)) end + test "rebuild_database does not run multiple rebuild writers at once", %{temp_dir: temp_dir} do + original = Application.get_env(:bds, :tasks, []) + + Application.put_env( + :bds, + :tasks, + original + |> Keyword.put(:max_concurrent, 4) + |> Keyword.put(:progress_throttle_ms, 0) + ) + + on_exit(fn -> Application.put_env(:bds, :tasks, original) end) + + posts_dir = Path.join([temp_dir, "posts", "2026", "04"]) + media_dir = Path.join([temp_dir, "media", "2026", "04"]) + scripts_dir = Path.join(temp_dir, "scripts") + templates_dir = Path.join(temp_dir, "templates") + + File.mkdir_p!(posts_dir) + File.mkdir_p!(media_dir) + File.mkdir_p!(scripts_dir) + File.mkdir_p!(templates_dir) + + Enum.each(1..80, fn index -> + slug = "serial-post-#{index}" + + File.write!( + Path.join(posts_dir, "#{slug}.md"), + [ + "---", + "id: #{slug}", + "title: Serial Post #{index}", + "slug: #{slug}", + "status: published", + "createdAt: 1711843200", + "updatedAt: 1711929600", + "publishedAt: 1712016000", + "tags:", + "categories:", + "---", + "Body #{index}", + "" + ] + |> Enum.join("\n") + ) + + File.write!(Path.join(media_dir, "asset-#{index}.txt"), "asset #{index}") + + File.write!( + Path.join(media_dir, "asset-#{index}.txt.meta"), + [ + "id: serial-media-#{index}", + "originalName: asset-#{index}.txt", + "mimeType: text/plain", + "size: 7", + "createdAt: 1711843200", + "updatedAt: 1711929600", + "tags:", + "" + ] + |> Enum.join("\n") + ) + + File.write!( + Path.join(scripts_dir, "serial-script-#{index}.lua"), + [ + "---", + "id: serial-script-#{index}", + "slug: serial-script-#{index}", + "title: Serial Script #{index}", + "kind: utility", + "entrypoint: main", + "enabled: true", + "version: 1", + "createdAt: 301", + "updatedAt: 404", + "---", + "function main() return true end", + "" + ] + |> Enum.join("\n") + ) + + File.write!( + Path.join(templates_dir, "serial-template-#{index}.liquid"), + [ + "---", + "id: serial-template-#{index}", + "slug: serial-template-#{index}", + "title: Serial Template #{index}", + "kind: list", + "enabled: true", + "version: 1", + "createdAt: 101", + "updatedAt: 202", + "---", + "
Template #{index}
", + "" + ] + |> Enum.join("\n") + ) + end) + + assert {:ok, _result} = ShellCommands.execute("rebuild_database") + + _posts_task = + wait_for_named_task( + "Rebuild Posts From Files", + &(&1.status == :running and is_number(&1.progress) and &1.progress > 0.0 and &1.progress < 1.0), + 10_000 + ) + + phase_one_tasks = + BDS.Tasks.list_tasks() + |> Enum.filter(&(&1.name in [ + "Rebuild Posts From Files", + "Rebuild Media From Files", + "Rebuild Scripts From Files", + "Rebuild Templates From Files" + ])) + + assert Enum.count(phase_one_tasks, &(&1.status == :running)) == 1 + + assert Enum.find(phase_one_tasks, &(&1.status == :running)).name == "Rebuild Posts From Files" + + tasks = wait_for_tasks_by_name([ + "Rebuild Posts From Files", + "Rebuild Media From Files", + "Rebuild Scripts From Files", + "Rebuild Templates From Files", + "Rebuild Post Links", + "Regenerate Missing Thumbnails", + "Rebuild Embedding Index" + ], &(&1.status == :completed), 20_000) + + assert Enum.all?(tasks, &(&1.status == :completed)) + end + test "rebuild_database exposes live in-task progress for rebuild work", %{temp_dir: temp_dir} do original = Application.get_env(:bds, :tasks, []) diff --git a/test/bds/repo/bootstrap_test.exs b/test/bds/repo/bootstrap_test.exs index 8487661..1c6d56c 100644 --- a/test/bds/repo/bootstrap_test.exs +++ b/test/bds/repo/bootstrap_test.exs @@ -55,6 +55,30 @@ defmodule BDS.Repo.BootstrapTest do assert %Project{id: "default", name: "My Blog", is_active: true} = BDS.Projects.get_active_project() end + test "dev repo config disables query logging by default" do + config_path = Path.expand("../../../config/config.exs", __DIR__) + config = Config.Reader.read!(config_path, env: :dev) + + repo_config = + config + |> Keyword.fetch!(:bds) + |> Keyword.fetch!(BDS.Repo) + + assert repo_config[:log] == false + end + + test "dev repo config sets a rebuild-safe sqlite busy timeout" do + config_path = Path.expand("../../../config/config.exs", __DIR__) + config = Config.Reader.read!(config_path, env: :dev) + + repo_config = + config + |> Keyword.fetch!(:bds) + |> Keyword.fetch!(BDS.Repo) + + assert repo_config[:busy_timeout] == 15_000 + end + defmodule RepoConfigBackup do def put_env do Process.put({__MODULE__, :temp_repo_config}, Application.get_env(:bds, TempRepo))