From 07730dc93ec22b2ed6cd4dea78deca80716bf171 Mon Sep 17 00:00:00 2001 From: Chili Palmer Date: Mon, 27 Apr 2026 10:38:36 +0200 Subject: [PATCH] fix: more work on metadata diff --- lib/bds/desktop/shell_commands.ex | 38 +++++++--- lib/bds/document_fields.ex | 43 ++++++++++++ lib/bds/maintenance.ex | 84 +++++++++++++++------- lib/bds/media.ex | 57 +++++++-------- lib/bds/posts.ex | 37 +++++----- lib/bds/scripts.ex | 13 ++-- lib/bds/templates.ex | 13 ++-- test/bds/desktop/shell_commands_test.exs | 39 +++++++++++ test/bds/desktop/shell_live_test.exs | 66 ++++++++++++++++++ test/bds/maintenance_test.exs | 88 ++++++++++++++++++++++++ 10 files changed, 384 insertions(+), 94 deletions(-) create mode 100644 lib/bds/document_fields.ex diff --git a/lib/bds/desktop/shell_commands.ex b/lib/bds/desktop/shell_commands.ex index bbedbfe..cc6d76b 100644 --- a/lib/bds/desktop/shell_commands.ex +++ b/lib/bds/desktop/shell_commands.ex @@ -215,8 +215,7 @@ defmodule BDS.Desktop.ShellCommands do defp dispatch("metadata_diff", project, _params) do queue_task(project, "metadata_diff", "Metadata Diff", "Maintenance", fn report -> - report.(0.2, "Comparing database and filesystem metadata") - {:ok, metadata_diff} = Maintenance.metadata_diff(project.id) + {:ok, metadata_diff} = Maintenance.metadata_diff(project.id, on_progress: report) report.(1.0, "Metadata diff complete") metadata_diff_result(project.id, metadata_diff) end) @@ -230,10 +229,16 @@ defmodule BDS.Desktop.ShellCommands do {:error, %{action: "repair_metadata_diff", message: "No metadata diff items selected"}} else queue_task(project, "repair_metadata_diff", "Repair Metadata Diff", "Maintenance", fn report -> - report.(0.2, "Repairing metadata differences") - {:ok, _repair} = Maintenance.repair_metadata_diff(project.id, direction, items) - report.(0.9, "Refreshing metadata diff") - {:ok, metadata_diff} = Maintenance.metadata_diff(project.id) + {:ok, _repair} = + Maintenance.repair_metadata_diff(project.id, direction, items, + on_progress: scaled_progress_reporter(report, 0.0, 0.8) + ) + + {:ok, metadata_diff} = + Maintenance.metadata_diff(project.id, + on_progress: scaled_progress_reporter(report, 0.8, 0.98) + ) + report.(1.0, "Metadata diff repair complete") metadata_diff_result(project.id, metadata_diff) end) @@ -247,10 +252,16 @@ defmodule BDS.Desktop.ShellCommands do {:error, %{action: "import_metadata_diff_orphans", message: "No orphan files selected"}} else queue_task(project, "import_metadata_diff_orphans", "Import Metadata Diff Orphans", "Maintenance", fn report -> - report.(0.2, "Importing orphan files") - {:ok, _import} = Maintenance.import_metadata_diff_orphans(project.id, orphans) - report.(0.9, "Refreshing metadata diff") - {:ok, metadata_diff} = Maintenance.metadata_diff(project.id) + {:ok, _import} = + Maintenance.import_metadata_diff_orphans(project.id, orphans, + on_progress: scaled_progress_reporter(report, 0.0, 0.8) + ) + + {:ok, metadata_diff} = + Maintenance.metadata_diff(project.id, + on_progress: scaled_progress_reporter(report, 0.8, 0.98) + ) + report.(1.0, "Metadata diff import complete") metadata_diff_result(project.id, metadata_diff) end) @@ -316,6 +327,13 @@ defmodule BDS.Desktop.ShellCommands do }} end + defp scaled_progress_reporter(report, start_value, end_value) when is_function(report, 2) do + fn value, message -> + scaled_value = start_value + (end_value - start_value) * value + report.(scaled_value, message) + end + end + defp rebuild_database_steps(project) do [ %{ diff --git a/lib/bds/document_fields.ex b/lib/bds/document_fields.ex new file mode 100644 index 0000000..606a1b4 --- /dev/null +++ b/lib/bds/document_fields.ex @@ -0,0 +1,43 @@ +defmodule BDS.DocumentFields do + @moduledoc false + + def get(fields, key, default \\ nil) when is_map(fields) and is_binary(key) do + case fetch(fields, key) do + {:ok, value} -> value + :error -> default + end + end + + def fetch!(fields, key) when is_map(fields) and is_binary(key) do + case fetch(fields, key) do + {:ok, value} -> value + :error -> raise KeyError, key: key, term: fields + end + end + + def has_key?(fields, key) when is_map(fields) and is_binary(key) do + match?({:ok, _value}, fetch(fields, key)) + end + + def fetch(fields, key) when is_map(fields) and is_binary(key) do + key + |> aliases_for() + |> Enum.find_value(:error, fn alias_key -> + if Map.has_key?(fields, alias_key) do + {:ok, Map.get(fields, alias_key)} + end + end) + end + + defp aliases_for(key) do + [key, Macro.underscore(key), lower_camelize(key)] + |> Enum.uniq() + end + + defp lower_camelize(value) do + case Macro.camelize(value) do + <> -> String.downcase(<>) <> rest + "" -> "" + end + end +end diff --git a/lib/bds/maintenance.ex b/lib/bds/maintenance.ex index 8e8bdb5..d2b28b9 100644 --- a/lib/bds/maintenance.ex +++ b/lib/bds/maintenance.ex @@ -4,6 +4,7 @@ defmodule BDS.Maintenance do import Ecto.Query alias BDS.Frontmatter + alias BDS.DocumentFields alias BDS.Metadata alias BDS.Media.Media alias BDS.Media.Translation, as: MediaTranslation @@ -78,20 +79,36 @@ defmodule BDS.Maintenance do end end - def metadata_diff(project_id) when is_binary(project_id) do + def metadata_diff(project_id, opts \\ []) + + def metadata_diff(project_id, opts) when is_binary(project_id) and is_list(opts) do project = Projects.get_project!(project_id) + on_progress = progress_callback(opts) + + phases = [ + {"Comparing project metadata", fn -> project_metadata_diff_reports(project_id) end}, + {"Comparing post metadata", fn -> post_diff_reports(project_id, project) end}, + {"Comparing post translations", fn -> post_translation_diff_reports(project_id, project) end}, + {"Comparing media metadata", fn -> media_diff_reports(project_id, project) end}, + {"Comparing media translations", fn -> media_translation_diff_reports(project_id, project) end}, + {"Comparing script metadata", fn -> script_diff_reports(project_id, project) end}, + {"Comparing template metadata", fn -> template_diff_reports(project_id, project) end}, + {"Comparing embeddings", fn -> Embeddings.diff_reports(project_id) end} + ] + + total_phases = length(phases) + 1 diff_reports = - project_metadata_diff_reports(project_id) ++ - post_diff_reports(project_id, project) ++ - post_translation_diff_reports(project_id, project) ++ - media_diff_reports(project_id, project) ++ - media_translation_diff_reports(project_id, project) ++ - script_diff_reports(project_id, project) ++ - template_diff_reports(project_id, project) ++ - Embeddings.diff_reports(project_id) + phases + |> Enum.with_index(1) + |> Enum.flat_map(fn {{label, fun}, index} -> + :ok = report_metadata_diff_phase(on_progress, index, total_phases, label) + fun.() + end) + :ok = report_metadata_diff_phase(on_progress, total_phases, total_phases, "Scanning orphan files") orphan_reports = orphan_reports(project_id, project) + :ok = report_metadata_diff_complete(on_progress) {:ok, %{diff_reports: diff_reports, orphan_reports: orphan_reports}} end @@ -189,11 +206,11 @@ defmodule BDS.Maintenance do diff_field("excerpt", post.excerpt, Map.get(fields, "excerpt")), diff_field("author", post.author, Map.get(fields, "author")), diff_field("language", post.language, Map.get(fields, "language")), - diff_field("status", post.status, Map.get(fields, "status")), - diff_field("template_slug", post.template_slug, Map.get(fields, "templateSlug")), - diff_field("created_at", post.created_at, Map.get(fields, "createdAt")), - diff_field("updated_at", post.updated_at, Map.get(fields, "updatedAt")), - diff_field("published_at", post.published_at, Map.get(fields, "publishedAt")), + diff_field("status", post.status, DocumentFields.get(fields, "status")), + diff_field("template_slug", post.template_slug, DocumentFields.get(fields, "templateSlug")), + diff_field("created_at", post.created_at, DocumentFields.get(fields, "createdAt")), + diff_field("updated_at", post.updated_at, DocumentFields.get(fields, "updatedAt")), + diff_field("published_at", post.published_at, DocumentFields.get(fields, "publishedAt")), diff_field("tags", post.tags, Map.get(fields, "tags", [])), diff_field("categories", post.categories, Map.get(fields, "categories", [])) ] @@ -228,8 +245,8 @@ defmodule BDS.Maintenance do diff_field("caption", media.caption, Map.get(fields, "caption")), diff_field("author", media.author, Map.get(fields, "author")), diff_field("language", media.language, Map.get(fields, "language")), - diff_field("created_at", media.created_at, Map.get(fields, "createdAt")), - diff_field("updated_at", media.updated_at, Map.get(fields, "updatedAt")), + diff_field("created_at", media.created_at, DocumentFields.get(fields, "createdAt")), + diff_field("updated_at", media.updated_at, DocumentFields.get(fields, "updatedAt")), diff_field("tags", media.tags, Map.get(fields, "tags", [])) ] |> Enum.reject(&is_nil/1) @@ -261,18 +278,18 @@ defmodule BDS.Maintenance do diff_field("title", translation.title, Map.get(fields, "title")), diff_field("excerpt", translation.excerpt, Map.get(fields, "excerpt")), diff_field("language", translation.language, Map.get(fields, "language")), - diff_field("status", translation.status, Map.get(fields, "status")), + diff_field("status", translation.status, DocumentFields.get(fields, "status")), diff_field( "translation_for", translation.translation_for, - Map.get(fields, "translationFor") + DocumentFields.get(fields, "translationFor") ), - diff_field("created_at", translation.created_at, Map.get(fields, "createdAt")), - diff_field("updated_at", translation.updated_at, Map.get(fields, "updatedAt")), + diff_field("created_at", translation.created_at, DocumentFields.get(fields, "createdAt")), + diff_field("updated_at", translation.updated_at, DocumentFields.get(fields, "updatedAt")), diff_field( "published_at", translation.published_at, - Map.get(fields, "publishedAt") + DocumentFields.get(fields, "publishedAt") ) ] |> Enum.reject(&is_nil/1) @@ -311,7 +328,7 @@ defmodule BDS.Maintenance do diff_field( "translation_for", translation.translation_for, - Map.get(fields, "translationFor") + DocumentFields.get(fields, "translationFor") ) ] |> Enum.reject(&is_nil/1) @@ -349,8 +366,8 @@ defmodule BDS.Maintenance do diff_field("title", script.title, Map.get(fields, "title")), diff_field("entrypoint", script.entrypoint, Map.get(fields, "entrypoint")), diff_field("enabled", script.enabled, Map.get(fields, "enabled")), - diff_field("created_at", script.created_at, Map.get(fields, "createdAt")), - diff_field("updated_at", script.updated_at, Map.get(fields, "updatedAt")) + diff_field("created_at", script.created_at, DocumentFields.get(fields, "createdAt")), + diff_field("updated_at", script.updated_at, DocumentFields.get(fields, "updatedAt")) ] |> Enum.reject(&is_nil/1) @@ -380,8 +397,8 @@ defmodule BDS.Maintenance do [ diff_field("title", template.title, Map.get(fields, "title")), diff_field("enabled", template.enabled, Map.get(fields, "enabled")), - diff_field("created_at", template.created_at, Map.get(fields, "createdAt")), - diff_field("updated_at", template.updated_at, Map.get(fields, "updatedAt")) + diff_field("created_at", template.created_at, DocumentFields.get(fields, "createdAt")), + diff_field("updated_at", template.updated_at, DocumentFields.get(fields, "updatedAt")) ] |> Enum.reject(&is_nil/1) @@ -669,6 +686,21 @@ defmodule BDS.Maintenance do end end + defp report_metadata_diff_phase(nil, _current, _total, _label), do: :ok + + defp report_metadata_diff_phase(callback, current, total, label) do + value = if total <= 1, do: 0.0, else: (current - 1) / total + callback.(value, "#{label} (#{current}/#{total})") + :ok + end + + defp report_metadata_diff_complete(nil), do: :ok + + defp report_metadata_diff_complete(callback) do + callback.(1.0, "Metadata diff complete") + :ok + end + defp report_started(nil, _total, _label), do: :ok defp report_started(callback, 0, label) do diff --git a/lib/bds/media.ex b/lib/bds/media.ex index bdda560..706c973 100644 --- a/lib/bds/media.ex +++ b/lib/bds/media.ex @@ -3,6 +3,7 @@ defmodule BDS.Media do import Ecto.Query + alias BDS.DocumentFields alias BDS.Media.Media alias BDS.Media.Translation alias BDS.Persistence @@ -156,10 +157,10 @@ defmodule BDS.Media do if File.exists?(sidecar_path) do sidecar = parse_translation_sidecar(sidecar_path) - case upsert_media_translation(media.id, Map.fetch!(sidecar.fields, "language"), %{ - title: Map.get(sidecar.fields, "title"), - alt: Map.get(sidecar.fields, "alt"), - caption: Map.get(sidecar.fields, "caption") + case upsert_media_translation(media.id, DocumentFields.fetch!(sidecar.fields, "language"), %{ + title: DocumentFields.get(sidecar.fields, "title"), + alt: DocumentFields.get(sidecar.fields, "alt"), + caption: DocumentFields.get(sidecar.fields, "caption") }) do {:ok, updated_translation} -> {:ok, updated_translation} error -> error @@ -188,20 +189,20 @@ defmodule BDS.Media do if File.exists?(sidecar_path) do sidecar = parse_translation_sidecar(sidecar_path) - case Repo.get(Media, Map.get(sidecar.fields, "translationFor")) do + case Repo.get(Media, DocumentFields.get(sidecar.fields, "translationFor")) do nil -> {:error, :not_found} media -> case Repo.get_by(Translation, translation_for: media.id, - language: Map.fetch!(sidecar.fields, "language") + language: DocumentFields.fetch!(sidecar.fields, "language") ) do nil -> - upsert_media_translation(media.id, Map.fetch!(sidecar.fields, "language"), %{ - title: Map.get(sidecar.fields, "title"), - alt: Map.get(sidecar.fields, "alt"), - caption: Map.get(sidecar.fields, "caption") + upsert_media_translation(media.id, DocumentFields.fetch!(sidecar.fields, "language"), %{ + title: DocumentFields.get(sidecar.fields, "title"), + alt: DocumentFields.get(sidecar.fields, "alt"), + caption: DocumentFields.get(sidecar.fields, "caption") }) _translation -> @@ -562,25 +563,25 @@ defmodule BDS.Media do now = Persistence.now_ms() attrs = %{ - id: Map.get(sidecar.fields, "id") || Ecto.UUID.generate(), + id: DocumentFields.get(sidecar.fields, "id") || Ecto.UUID.generate(), project_id: project.id, filename: sidecar.filename, - original_name: Map.get(sidecar.fields, "originalName") || sidecar.filename, - mime_type: Map.get(sidecar.fields, "mimeType") || detect_mime(sidecar.filename), - size: Map.get(sidecar.fields, "size", 0), - width: blank_to_nil(Map.get(sidecar.fields, "width")), - height: blank_to_nil(Map.get(sidecar.fields, "height")), - title: Map.get(sidecar.fields, "title"), - alt: Map.get(sidecar.fields, "alt"), - caption: Map.get(sidecar.fields, "caption"), - author: Map.get(sidecar.fields, "author"), - language: Map.get(sidecar.fields, "language"), + original_name: DocumentFields.get(sidecar.fields, "originalName") || sidecar.filename, + mime_type: DocumentFields.get(sidecar.fields, "mimeType") || detect_mime(sidecar.filename), + size: DocumentFields.get(sidecar.fields, "size", 0), + width: blank_to_nil(DocumentFields.get(sidecar.fields, "width")), + height: blank_to_nil(DocumentFields.get(sidecar.fields, "height")), + title: DocumentFields.get(sidecar.fields, "title"), + alt: DocumentFields.get(sidecar.fields, "alt"), + caption: DocumentFields.get(sidecar.fields, "caption"), + author: DocumentFields.get(sidecar.fields, "author"), + language: DocumentFields.get(sidecar.fields, "language"), file_path: sidecar.relative_file_path, sidecar_path: sidecar.relative_sidecar_path, checksum: nil, - tags: Map.get(sidecar.fields, "tags", []), - created_at: Map.get(sidecar.fields, "createdAt", now), - updated_at: Map.get(sidecar.fields, "updatedAt", now) + tags: DocumentFields.get(sidecar.fields, "tags", []), + created_at: DocumentFields.get(sidecar.fields, "createdAt", now), + updated_at: DocumentFields.get(sidecar.fields, "updatedAt", now) } media = @@ -653,7 +654,7 @@ defmodule BDS.Media do media -> now = Persistence.now_ms() - language = Map.fetch!(sidecar.fields, "language") + language = DocumentFields.fetch!(sidecar.fields, "language") translation = Repo.get_by(Translation, translation_for: media.id, language: language) || @@ -665,9 +666,9 @@ defmodule BDS.Media do project_id: project.id, translation_for: media.id, language: language, - title: Map.get(sidecar.fields, "title"), - alt: Map.get(sidecar.fields, "alt"), - caption: Map.get(sidecar.fields, "caption"), + title: DocumentFields.get(sidecar.fields, "title"), + alt: DocumentFields.get(sidecar.fields, "alt"), + caption: DocumentFields.get(sidecar.fields, "caption"), created_at: translation.created_at || now, updated_at: now }) diff --git a/lib/bds/posts.ex b/lib/bds/posts.ex index bd7f5fa..ae5370f 100644 --- a/lib/bds/posts.ex +++ b/lib/bds/posts.ex @@ -3,6 +3,7 @@ defmodule BDS.Posts do import Ecto.Query + alias BDS.DocumentFields alias BDS.Frontmatter alias BDS.Embeddings alias BDS.AI @@ -846,24 +847,24 @@ defmodule BDS.Posts do now = Persistence.now_ms() attrs = %{ - id: Map.get(fields, "id") || Ecto.UUID.generate(), + id: DocumentFields.get(fields, "id") || Ecto.UUID.generate(), project_id: project_id, - title: Map.get(fields, "title") || "", - slug: Map.fetch!(fields, "slug"), + title: DocumentFields.get(fields, "title") || "", + slug: DocumentFields.fetch!(fields, "slug"), excerpt: Map.get(fields, "excerpt"), content: nil, - status: parse_post_status(Map.get(fields, "status", "published")), + status: parse_post_status(DocumentFields.get(fields, "status", "published")), author: Map.get(fields, "author"), - created_at: Map.get(fields, "createdAt", now), - updated_at: Map.get(fields, "updatedAt", now), - published_at: Map.get(fields, "publishedAt"), + created_at: DocumentFields.get(fields, "createdAt", now), + updated_at: DocumentFields.get(fields, "updatedAt", now), + published_at: DocumentFields.get(fields, "publishedAt"), file_path: rebuild_file.relative_path, checksum: nil, tags: Map.get(fields, "tags", []), categories: Map.get(fields, "categories", []), - template_slug: Map.get(fields, "templateSlug"), + template_slug: DocumentFields.get(fields, "templateSlug"), language: Map.get(fields, "language"), - do_not_translate: Map.get(fields, "doNotTranslate", false), + do_not_translate: DocumentFields.get(fields, "doNotTranslate", false), published_title: nil, published_content: nil, published_tags: nil, @@ -894,26 +895,26 @@ defmodule BDS.Posts do defp upsert_post_translation_from_rebuild_file(project_id, rebuild_file, opts) do fields = rebuild_file.fields - source_post_id = Map.fetch!(fields, "translationFor") + source_post_id = DocumentFields.fetch!(fields, "translationFor") source_post = Repo.get_by!(Post, project_id: project_id, id: source_post_id) now = Persistence.now_ms() - language = normalize_language(Map.fetch!(fields, "language")) + language = normalize_language(DocumentFields.fetch!(fields, "language")) translation = Repo.get_by(Translation, translation_for: source_post_id, language: language) || %Translation{} attrs = %{ - id: Map.get(fields, "id") || Ecto.UUID.generate(), + id: DocumentFields.get(fields, "id") || Ecto.UUID.generate(), project_id: project_id, translation_for: source_post_id, language: language, - title: Map.get(fields, "title") || "", + title: DocumentFields.get(fields, "title") || "", excerpt: Map.get(fields, "excerpt"), content: nil, - status: parse_translation_status(Map.get(fields, "status", "published")), - created_at: Map.get(fields, "createdAt", source_post.created_at || now), - updated_at: Map.get(fields, "updatedAt", source_post.updated_at || source_post.created_at || now), - published_at: Map.get(fields, "publishedAt", source_post.published_at), + status: parse_translation_status(DocumentFields.get(fields, "status", "published")), + created_at: DocumentFields.get(fields, "createdAt", source_post.created_at || now), + updated_at: DocumentFields.get(fields, "updatedAt", source_post.updated_at || source_post.created_at || now), + published_at: DocumentFields.get(fields, "publishedAt", source_post.published_at), file_path: rebuild_file.relative_path, checksum: nil } @@ -946,7 +947,7 @@ defmodule BDS.Posts do end defp translation_rebuild_file?(%{fields: fields}) do - Map.has_key?(fields, "translationFor") and not Map.has_key?(fields, "slug") + DocumentFields.has_key?(fields, "translationFor") and not DocumentFields.has_key?(fields, "slug") end defp list_matching_files(dir, pattern) do diff --git a/lib/bds/scripts.ex b/lib/bds/scripts.ex index bf3642f..db48ee4 100644 --- a/lib/bds/scripts.ex +++ b/lib/bds/scripts.ex @@ -3,6 +3,7 @@ defmodule BDS.Scripts do import Ecto.Query + alias BDS.DocumentFields alias BDS.Frontmatter alias BDS.Persistence alias BDS.Projects @@ -283,19 +284,19 @@ defmodule BDS.Scripts do now = Persistence.now_ms() attrs = %{ - id: Map.get(fields, "id") || Ecto.UUID.generate(), + id: DocumentFields.get(fields, "id") || Ecto.UUID.generate(), project_id: project_id, - slug: Map.fetch!(fields, "slug"), - title: Map.get(fields, "title") || "", - kind: parse_script_kind(Map.fetch!(fields, "kind")), + slug: DocumentFields.fetch!(fields, "slug"), + title: DocumentFields.get(fields, "title") || "", + kind: parse_script_kind(DocumentFields.fetch!(fields, "kind")), entrypoint: Map.get(fields, "entrypoint") || "main", enabled: Map.get(fields, "enabled", true), version: Map.get(fields, "version", 1), file_path: relative_path, status: :published, content: nil, - created_at: Map.get(fields, "createdAt", now), - updated_at: Map.get(fields, "updatedAt", now) + created_at: DocumentFields.get(fields, "createdAt", now), + updated_at: DocumentFields.get(fields, "updatedAt", now) } script = Repo.get_by(Script, project_id: project_id, slug: attrs.slug) || %Script{} diff --git a/lib/bds/templates.ex b/lib/bds/templates.ex index d0396d3..ebbb7b8 100644 --- a/lib/bds/templates.ex +++ b/lib/bds/templates.ex @@ -3,6 +3,7 @@ defmodule BDS.Templates do import Ecto.Query + alias BDS.DocumentFields alias BDS.Frontmatter alias BDS.Persistence alias BDS.Posts @@ -408,18 +409,18 @@ defmodule BDS.Templates do now = Persistence.now_ms() attrs = %{ - id: Map.get(fields, "id") || Ecto.UUID.generate(), + id: DocumentFields.get(fields, "id") || Ecto.UUID.generate(), project_id: project_id, - slug: Map.fetch!(fields, "slug"), - title: Map.get(fields, "title") || "", - kind: parse_template_kind(Map.fetch!(fields, "kind")), + slug: DocumentFields.fetch!(fields, "slug"), + title: DocumentFields.get(fields, "title") || "", + kind: parse_template_kind(DocumentFields.fetch!(fields, "kind")), enabled: Map.get(fields, "enabled", true), version: Map.get(fields, "version", 1), file_path: relative_path, status: :published, content: nil, - created_at: Map.get(fields, "createdAt", now), - updated_at: Map.get(fields, "updatedAt", now) + created_at: DocumentFields.get(fields, "createdAt", now), + updated_at: DocumentFields.get(fields, "updatedAt", now) } template = Repo.get_by(Template, project_id: project_id, slug: attrs.slug) || %Template{} diff --git a/test/bds/desktop/shell_commands_test.exs b/test/bds/desktop/shell_commands_test.exs index f8ec43e..a091c5c 100644 --- a/test/bds/desktop/shell_commands_test.exs +++ b/test/bds/desktop/shell_commands_test.exs @@ -99,6 +99,45 @@ defmodule BDS.Desktop.ShellCommandsTest do assert is_map(completed.result.payload.summary) end + test "repair_metadata_diff exposes live in-task progress from the repair worker", %{project: project} do + original = Application.get_env(:bds, :tasks, []) + + Application.put_env( + :bds, + :tasks, + original + |> Keyword.put(:max_concurrent, 1) + |> Keyword.put(:progress_throttle_ms, 0) + ) + + on_exit(fn -> Application.put_env(:bds, :tasks, original) end) + + items = + Enum.map(1..40, fn _index -> + %{"entity_type" => "project", "entity_id" => project.id} + end) + + assert {:ok, result} = + ShellCommands.execute("repair_metadata_diff", %{ + "direction" => "file_to_db", + "items" => items + }) + + progressed = + wait_for_task( + result.task_id, + &(&1.status == :running and is_number(&1.progress) and &1.progress > 0.2 and &1.progress < 1.0), + 5_000 + ) + + assert progressed.group_name == "Maintenance" + assert String.contains?(progressed.message, "Repairing") + assert String.contains?(progressed.message, "/") + + assert wait_for_task(result.task_id, &(&1.status == :completed and &1.progress == 1.0), 5_000).status == + :completed + end + test "find_duplicates queues a tracked embeddings task and returns the report as an editor payload" do assert {:ok, result} = ShellCommands.execute("find_duplicates") diff --git a/test/bds/desktop/shell_live_test.exs b/test/bds/desktop/shell_live_test.exs index dfb2f20..066c09b 100644 --- a/test/bds/desktop/shell_live_test.exs +++ b/test/bds/desktop/shell_live_test.exs @@ -1034,6 +1034,72 @@ defmodule BDS.Desktop.ShellLiveTest do refute Enum.any?(diff.diff_reports, &(&1.entity_id == published_post.id)) end + test "metadata diff refresh reruns the diff and replaces the current result", %{ + project: project, + temp_dir: temp_dir + } do + :ok = BDS.Tasks.clear_finished() + + assert {:ok, post} = + Posts.create_post(%{ + project_id: project.id, + title: "Database Post", + content: "Body", + excerpt: "Summary", + language: "en" + }) + + assert {:ok, published_post} = Posts.publish_post(post.id) + post_path = Path.join(temp_dir, published_post.file_path) + + {:ok, view, _html} = live_isolated(build_conn(), BDS.Desktop.ShellLive) + + assert {:ok, queued} = BDS.Desktop.ShellCommands.execute("metadata_diff") + completed_task!(queued.task_id) + send(view.pid, :refresh_task_status) + + html = render(view) + refute html =~ "Filesystem Post" + + File.write!( + post_path, + [ + "---", + "id: #{published_post.id}", + "title: Filesystem Post", + "slug: #{published_post.slug}", + "excerpt: Summary", + "status: published", + "language: en", + "createdAt: #{published_post.created_at}", + "updatedAt: #{published_post.updated_at + 1}", + "publishedAt: #{published_post.published_at}", + "tags:", + "categories:", + "---", + "Body", + "" + ] + |> Enum.join("\n") + ) + + existing_ids = MapSet.new(Enum.map(BDS.Tasks.list_tasks(), & &1.id)) + + html = + view + |> element("button[phx-click='rerun_misc_editor']") + |> render_click() + + assert html =~ "Metadata Diff" + + refresh_task = new_task!(existing_ids, "Metadata Diff") + completed_task!(refresh_task.id) + send(view.pid, :refresh_task_status) + + html = render(view) + assert html =~ "Filesystem Post" + end + test "metadata diff orphan import action queues an import task and removes the orphan", %{ project: project, temp_dir: temp_dir diff --git a/test/bds/maintenance_test.exs b/test/bds/maintenance_test.exs index b69fbfa..c6b2c06 100644 --- a/test/bds/maintenance_test.exs +++ b/test/bds/maintenance_test.exs @@ -328,6 +328,49 @@ defmodule BDS.MaintenanceTest do assert_incremental_progress(collect_progress_events()) end + test "metadata_diff reports incremental progress across comparison phases", %{ + project: project, + temp_dir: temp_dir + } do + parent = self() + on_progress = fn value, message -> send(parent, {:rebuild_progress, value, message}) end + + posts_dir = Path.join([temp_dir, "posts", "2026", "04"]) + File.mkdir_p!(posts_dir) + + Enum.each(1..40, fn index -> + slug = "diff-progress-post-#{index}" + + File.write!( + Path.join(posts_dir, "#{slug}.md"), + [ + "---", + "id: #{slug}", + "title: Diff Progress Post #{index}", + "slug: #{slug}", + "status: published", + "createdAt: 1711843200", + "updatedAt: 1711929600", + "publishedAt: 1712016000", + "tags:", + "categories:", + "---", + "Body #{index}", + "" + ] + |> Enum.join("\n") + ) + end) + + assert {:ok, _posts} = BDS.Maintenance.rebuild_from_filesystem(project.id, "post") + + assert {:ok, _diff} = BDS.Maintenance.metadata_diff(project.id, on_progress: on_progress) + + events = collect_progress_events() + assert_incremental_progress(events) + assert Enum.any?(events, fn {_value, message} -> String.contains?(message, "Comparing") end) + end + test "maintenance rebuilds and diffs embedding state explicitly", %{project: project} do assert {:ok, _metadata} = BDS.Metadata.update_project_metadata(project.id, %{semantic_similarity_enabled: true}) @@ -749,6 +792,51 @@ defmodule BDS.MaintenanceTest do end) end + test "metadata_diff accepts legacy snake_case post frontmatter keys for status and timestamps", %{ + project: project, + temp_dir: temp_dir + } do + assert {:ok, post} = + BDS.Posts.create_post(%{ + project_id: project.id, + title: "Legacy Keys Post", + content: "Body", + language: "en" + }) + + assert {:ok, published_post} = BDS.Posts.publish_post(post.id) + + post_path = Path.join(temp_dir, published_post.file_path) + + File.write!( + post_path, + [ + "---", + "id: #{published_post.id}", + "title: #{published_post.title}", + "slug: #{published_post.slug}", + "status: published", + "language: en", + "created_at: #{published_post.created_at}", + "updated_at: #{published_post.updated_at}", + "published_at: #{published_post.published_at}", + "tags:", + "categories:", + "---", + "Body", + "" + ] + |> Enum.join("\n") + ) + + assert {:ok, %{diff_reports: diff_reports}} = BDS.Maintenance.metadata_diff(project.id) + + refute Enum.any?(diff_reports, fn report -> + report.entity_type == "post" and report.entity_id == published_post.id and + Enum.any?(report.differences, &(&1.name in ["status", "created_at", "updated_at", "published_at"])) + end) + end + test "metadata_diff includes project-level metadata drift", %{project: project, temp_dir: temp_dir} do assert {:ok, _metadata} = BDS.Metadata.update_project_metadata(project.id, %{