hugo/bDS2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: more work on metadata diff

Browse Source
This commit is contained in:
Chili Palmer
2026-04-27 10:38:36 +02:00
parent e7ccf02d40
commit 07730dc93e
10 changed files with 384 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
lib/bds/desktop/shell_commands.ex
View File

@@ -215,8 +215,7 @@ defmodule BDS.Desktop.ShellCommands do
defp dispatch("metadata_diff", project, _params) do
queue_task(project, "metadata_diff", "Metadata Diff", "Maintenance", fn report ->
report.(0.2, "Comparing database and filesystem metadata")
{:ok, metadata_diff} = Maintenance.metadata_diff(project.id)
{:ok, metadata_diff} = Maintenance.metadata_diff(project.id, on_progress: report)
report.(1.0, "Metadata diff complete")
metadata_diff_result(project.id, metadata_diff)
end)
@@ -230,10 +229,16 @@ defmodule BDS.Desktop.ShellCommands do
{:error, %{action: "repair_metadata_diff", message: "No metadata diff items selected"}}
else
queue_task(project, "repair_metadata_diff", "Repair Metadata Diff", "Maintenance", fn report ->
report.(0.2, "Repairing metadata differences")
{:ok, _repair} = Maintenance.repair_metadata_diff(project.id, direction, items)
report.(0.9, "Refreshing metadata diff")
{:ok, metadata_diff} = Maintenance.metadata_diff(project.id)
{:ok, _repair} =
Maintenance.repair_metadata_diff(project.id, direction, items,
on_progress: scaled_progress_reporter(report, 0.0, 0.8)
)
{:ok, metadata_diff} =
Maintenance.metadata_diff(project.id,
on_progress: scaled_progress_reporter(report, 0.8, 0.98)
)
report.(1.0, "Metadata diff repair complete")
metadata_diff_result(project.id, metadata_diff)
end)
@@ -247,10 +252,16 @@ defmodule BDS.Desktop.ShellCommands do
{:error, %{action: "import_metadata_diff_orphans", message: "No orphan files selected"}}
else
queue_task(project, "import_metadata_diff_orphans", "Import Metadata Diff Orphans", "Maintenance", fn report ->
report.(0.2, "Importing orphan files")
{:ok, _import} = Maintenance.import_metadata_diff_orphans(project.id, orphans)
report.(0.9, "Refreshing metadata diff")
{:ok, metadata_diff} = Maintenance.metadata_diff(project.id)
{:ok, _import} =
Maintenance.import_metadata_diff_orphans(project.id, orphans,
on_progress: scaled_progress_reporter(report, 0.0, 0.8)
)
{:ok, metadata_diff} =
Maintenance.metadata_diff(project.id,
on_progress: scaled_progress_reporter(report, 0.8, 0.98)
)
report.(1.0, "Metadata diff import complete")
metadata_diff_result(project.id, metadata_diff)
end)
@@ -316,6 +327,13 @@ defmodule BDS.Desktop.ShellCommands do
}}
end
defp scaled_progress_reporter(report, start_value, end_value) when is_function(report, 2) do
fn value, message ->
scaled_value = start_value + (end_value - start_value) * value
report.(scaled_value, message)
end
end
defp rebuild_database_steps(project) do
[
%{

43
lib/bds/document_fields.ex Normal file
View File

@@ -0,0 +1,43 @@
defmodule BDS.DocumentFields do
@moduledoc false
def get(fields, key, default \\ nil) when is_map(fields) and is_binary(key) do
case fetch(fields, key) do
{:ok, value} -> value
:error -> default
end
end
def fetch!(fields, key) when is_map(fields) and is_binary(key) do
case fetch(fields, key) do
{:ok, value} -> value
:error -> raise KeyError, key: key, term: fields
end
end
def has_key?(fields, key) when is_map(fields) and is_binary(key) do
match?({:ok, _value}, fetch(fields, key))
end
def fetch(fields, key) when is_map(fields) and is_binary(key) do
key
|> aliases_for()
|> Enum.find_value(:error, fn alias_key ->
if Map.has_key?(fields, alias_key) do
{:ok, Map.get(fields, alias_key)}
end
end)
end
defp aliases_for(key) do
[key, Macro.underscore(key), lower_camelize(key)]
|> Enum.uniq()
end
defp lower_camelize(value) do
case Macro.camelize(value) do
<<first::utf8, rest::binary>> -> String.downcase(<<first::utf8>>) <> rest
"" -> ""
end
end
end

84
lib/bds/maintenance.ex
View File

@@ -4,6 +4,7 @@ defmodule BDS.Maintenance do
import Ecto.Query
alias BDS.Frontmatter
alias BDS.DocumentFields
alias BDS.Metadata
alias BDS.Media.Media
alias BDS.Media.Translation, as: MediaTranslation
@@ -78,20 +79,36 @@ defmodule BDS.Maintenance do
end
end
def metadata_diff(project_id) when is_binary(project_id) do
def metadata_diff(project_id, opts \\ [])
def metadata_diff(project_id, opts) when is_binary(project_id) and is_list(opts) do
project = Projects.get_project!(project_id)
on_progress = progress_callback(opts)
phases = [
{"Comparing project metadata", fn -> project_metadata_diff_reports(project_id) end},
{"Comparing post metadata", fn -> post_diff_reports(project_id, project) end},
{"Comparing post translations", fn -> post_translation_diff_reports(project_id, project) end},
{"Comparing media metadata", fn -> media_diff_reports(project_id, project) end},
{"Comparing media translations", fn -> media_translation_diff_reports(project_id, project) end},
{"Comparing script metadata", fn -> script_diff_reports(project_id, project) end},
{"Comparing template metadata", fn -> template_diff_reports(project_id, project) end},
{"Comparing embeddings", fn -> Embeddings.diff_reports(project_id) end}
]
total_phases = length(phases) + 1
diff_reports =
project_metadata_diff_reports(project_id) ++
post_diff_reports(project_id, project) ++
post_translation_diff_reports(project_id, project) ++
media_diff_reports(project_id, project) ++
media_translation_diff_reports(project_id, project) ++
script_diff_reports(project_id, project) ++
template_diff_reports(project_id, project) ++
Embeddings.diff_reports(project_id)
phases
|> Enum.with_index(1)
|> Enum.flat_map(fn {{label, fun}, index} ->
:ok = report_metadata_diff_phase(on_progress, index, total_phases, label)
fun.()
end)
:ok = report_metadata_diff_phase(on_progress, total_phases, total_phases, "Scanning orphan files")
orphan_reports = orphan_reports(project_id, project)
:ok = report_metadata_diff_complete(on_progress)
{:ok, %{diff_reports: diff_reports, orphan_reports: orphan_reports}}
end
@@ -189,11 +206,11 @@ defmodule BDS.Maintenance do
diff_field("excerpt", post.excerpt, Map.get(fields, "excerpt")),
diff_field("author", post.author, Map.get(fields, "author")),
diff_field("language", post.language, Map.get(fields, "language")),
diff_field("status", post.status, Map.get(fields, "status")),
diff_field("template_slug", post.template_slug, Map.get(fields, "templateSlug")),
diff_field("created_at", post.created_at, Map.get(fields, "createdAt")),
diff_field("updated_at", post.updated_at, Map.get(fields, "updatedAt")),
diff_field("published_at", post.published_at, Map.get(fields, "publishedAt")),
diff_field("status", post.status, DocumentFields.get(fields, "status")),
diff_field("template_slug", post.template_slug, DocumentFields.get(fields, "templateSlug")),
diff_field("created_at", post.created_at, DocumentFields.get(fields, "createdAt")),
diff_field("updated_at", post.updated_at, DocumentFields.get(fields, "updatedAt")),
diff_field("published_at", post.published_at, DocumentFields.get(fields, "publishedAt")),
diff_field("tags", post.tags, Map.get(fields, "tags", [])),
diff_field("categories", post.categories, Map.get(fields, "categories", []))
]
@@ -228,8 +245,8 @@ defmodule BDS.Maintenance do
diff_field("caption", media.caption, Map.get(fields, "caption")),
diff_field("author", media.author, Map.get(fields, "author")),
diff_field("language", media.language, Map.get(fields, "language")),
diff_field("created_at", media.created_at, Map.get(fields, "createdAt")),
diff_field("updated_at", media.updated_at, Map.get(fields, "updatedAt")),
diff_field("created_at", media.created_at, DocumentFields.get(fields, "createdAt")),
diff_field("updated_at", media.updated_at, DocumentFields.get(fields, "updatedAt")),
diff_field("tags", media.tags, Map.get(fields, "tags", []))
]
|> Enum.reject(&is_nil/1)
@@ -261,18 +278,18 @@ defmodule BDS.Maintenance do
diff_field("title", translation.title, Map.get(fields, "title")),
diff_field("excerpt", translation.excerpt, Map.get(fields, "excerpt")),
diff_field("language", translation.language, Map.get(fields, "language")),
diff_field("status", translation.status, Map.get(fields, "status")),
diff_field("status", translation.status, DocumentFields.get(fields, "status")),
diff_field(
"translation_for",
translation.translation_for,
Map.get(fields, "translationFor")
DocumentFields.get(fields, "translationFor")
),
diff_field("created_at", translation.created_at, Map.get(fields, "createdAt")),
diff_field("updated_at", translation.updated_at, Map.get(fields, "updatedAt")),
diff_field("created_at", translation.created_at, DocumentFields.get(fields, "createdAt")),
diff_field("updated_at", translation.updated_at, DocumentFields.get(fields, "updatedAt")),
diff_field(
"published_at",
translation.published_at,
Map.get(fields, "publishedAt")
DocumentFields.get(fields, "publishedAt")
)
]
|> Enum.reject(&is_nil/1)
@@ -311,7 +328,7 @@ defmodule BDS.Maintenance do
diff_field(
"translation_for",
translation.translation_for,
Map.get(fields, "translationFor")
DocumentFields.get(fields, "translationFor")
)
]
|> Enum.reject(&is_nil/1)
@@ -349,8 +366,8 @@ defmodule BDS.Maintenance do
diff_field("title", script.title, Map.get(fields, "title")),
diff_field("entrypoint", script.entrypoint, Map.get(fields, "entrypoint")),
diff_field("enabled", script.enabled, Map.get(fields, "enabled")),
diff_field("created_at", script.created_at, Map.get(fields, "createdAt")),
diff_field("updated_at", script.updated_at, Map.get(fields, "updatedAt"))
diff_field("created_at", script.created_at, DocumentFields.get(fields, "createdAt")),
diff_field("updated_at", script.updated_at, DocumentFields.get(fields, "updatedAt"))
]
|> Enum.reject(&is_nil/1)
@@ -380,8 +397,8 @@ defmodule BDS.Maintenance do
[
diff_field("title", template.title, Map.get(fields, "title")),
diff_field("enabled", template.enabled, Map.get(fields, "enabled")),
diff_field("created_at", template.created_at, Map.get(fields, "createdAt")),
diff_field("updated_at", template.updated_at, Map.get(fields, "updatedAt"))
diff_field("created_at", template.created_at, DocumentFields.get(fields, "createdAt")),
diff_field("updated_at", template.updated_at, DocumentFields.get(fields, "updatedAt"))
]
|> Enum.reject(&is_nil/1)
@@ -669,6 +686,21 @@ defmodule BDS.Maintenance do
end
end
defp report_metadata_diff_phase(nil, _current, _total, _label), do: :ok
defp report_metadata_diff_phase(callback, current, total, label) do
value = if total <= 1, do: 0.0, else: (current - 1) / total
callback.(value, "#{label} (#{current}/#{total})")
:ok
end
defp report_metadata_diff_complete(nil), do: :ok
defp report_metadata_diff_complete(callback) do
callback.(1.0, "Metadata diff complete")
:ok
end
defp report_started(nil, _total, _label), do: :ok
defp report_started(callback, 0, label) do

57
lib/bds/media.ex
View File

@@ -3,6 +3,7 @@ defmodule BDS.Media do
import Ecto.Query
alias BDS.DocumentFields
alias BDS.Media.Media
alias BDS.Media.Translation
alias BDS.Persistence
@@ -156,10 +157,10 @@ defmodule BDS.Media do
if File.exists?(sidecar_path) do
sidecar = parse_translation_sidecar(sidecar_path)
case upsert_media_translation(media.id, Map.fetch!(sidecar.fields, "language"), %{
title: Map.get(sidecar.fields, "title"),
alt: Map.get(sidecar.fields, "alt"),
caption: Map.get(sidecar.fields, "caption")
case upsert_media_translation(media.id, DocumentFields.fetch!(sidecar.fields, "language"), %{
title: DocumentFields.get(sidecar.fields, "title"),
alt: DocumentFields.get(sidecar.fields, "alt"),
caption: DocumentFields.get(sidecar.fields, "caption")
}) do
{:ok, updated_translation} -> {:ok, updated_translation}
error -> error
@@ -188,20 +189,20 @@ defmodule BDS.Media do
if File.exists?(sidecar_path) do
sidecar = parse_translation_sidecar(sidecar_path)
case Repo.get(Media, Map.get(sidecar.fields, "translationFor")) do
case Repo.get(Media, DocumentFields.get(sidecar.fields, "translationFor")) do
nil ->
{:error, :not_found}
media ->
case Repo.get_by(Translation,
translation_for: media.id,
language: Map.fetch!(sidecar.fields, "language")
language: DocumentFields.fetch!(sidecar.fields, "language")
) do
nil ->
upsert_media_translation(media.id, Map.fetch!(sidecar.fields, "language"), %{
title: Map.get(sidecar.fields, "title"),
alt: Map.get(sidecar.fields, "alt"),
caption: Map.get(sidecar.fields, "caption")
upsert_media_translation(media.id, DocumentFields.fetch!(sidecar.fields, "language"), %{
title: DocumentFields.get(sidecar.fields, "title"),
alt: DocumentFields.get(sidecar.fields, "alt"),
caption: DocumentFields.get(sidecar.fields, "caption")
})
_translation ->
@@ -562,25 +563,25 @@ defmodule BDS.Media do
now = Persistence.now_ms()
attrs = %{
id: Map.get(sidecar.fields, "id") || Ecto.UUID.generate(),
id: DocumentFields.get(sidecar.fields, "id") || Ecto.UUID.generate(),
project_id: project.id,
filename: sidecar.filename,
original_name: Map.get(sidecar.fields, "originalName") || sidecar.filename,
mime_type: Map.get(sidecar.fields, "mimeType") || detect_mime(sidecar.filename),
size: Map.get(sidecar.fields, "size", 0),
width: blank_to_nil(Map.get(sidecar.fields, "width")),
height: blank_to_nil(Map.get(sidecar.fields, "height")),
title: Map.get(sidecar.fields, "title"),
alt: Map.get(sidecar.fields, "alt"),
caption: Map.get(sidecar.fields, "caption"),
author: Map.get(sidecar.fields, "author"),
language: Map.get(sidecar.fields, "language"),
original_name: DocumentFields.get(sidecar.fields, "originalName") || sidecar.filename,
mime_type: DocumentFields.get(sidecar.fields, "mimeType") || detect_mime(sidecar.filename),
size: DocumentFields.get(sidecar.fields, "size", 0),
width: blank_to_nil(DocumentFields.get(sidecar.fields, "width")),
height: blank_to_nil(DocumentFields.get(sidecar.fields, "height")),
title: DocumentFields.get(sidecar.fields, "title"),
alt: DocumentFields.get(sidecar.fields, "alt"),
caption: DocumentFields.get(sidecar.fields, "caption"),
author: DocumentFields.get(sidecar.fields, "author"),
language: DocumentFields.get(sidecar.fields, "language"),
file_path: sidecar.relative_file_path,
sidecar_path: sidecar.relative_sidecar_path,
checksum: nil,
tags: Map.get(sidecar.fields, "tags", []),
created_at: Map.get(sidecar.fields, "createdAt", now),
updated_at: Map.get(sidecar.fields, "updatedAt", now)
tags: DocumentFields.get(sidecar.fields, "tags", []),
created_at: DocumentFields.get(sidecar.fields, "createdAt", now),
updated_at: DocumentFields.get(sidecar.fields, "updatedAt", now)
}
media =
@@ -653,7 +654,7 @@ defmodule BDS.Media do
media ->
now = Persistence.now_ms()
language = Map.fetch!(sidecar.fields, "language")
language = DocumentFields.fetch!(sidecar.fields, "language")
translation =
Repo.get_by(Translation, translation_for: media.id, language: language) ||
@@ -665,9 +666,9 @@ defmodule BDS.Media do
project_id: project.id,
translation_for: media.id,
language: language,
title: Map.get(sidecar.fields, "title"),
alt: Map.get(sidecar.fields, "alt"),
caption: Map.get(sidecar.fields, "caption"),
title: DocumentFields.get(sidecar.fields, "title"),
alt: DocumentFields.get(sidecar.fields, "alt"),
caption: DocumentFields.get(sidecar.fields, "caption"),
created_at: translation.created_at || now,
updated_at: now
})

37
lib/bds/posts.ex
View File

@@ -3,6 +3,7 @@ defmodule BDS.Posts do
import Ecto.Query
alias BDS.DocumentFields
alias BDS.Frontmatter
alias BDS.Embeddings
alias BDS.AI
@@ -846,24 +847,24 @@ defmodule BDS.Posts do
now = Persistence.now_ms()
attrs = %{
id: Map.get(fields, "id") || Ecto.UUID.generate(),
id: DocumentFields.get(fields, "id") || Ecto.UUID.generate(),
project_id: project_id,
title: Map.get(fields, "title") || "",
slug: Map.fetch!(fields, "slug"),
title: DocumentFields.get(fields, "title") || "",
slug: DocumentFields.fetch!(fields, "slug"),
excerpt: Map.get(fields, "excerpt"),
content: nil,
status: parse_post_status(Map.get(fields, "status", "published")),
status: parse_post_status(DocumentFields.get(fields, "status", "published")),
author: Map.get(fields, "author"),
created_at: Map.get(fields, "createdAt", now),
updated_at: Map.get(fields, "updatedAt", now),
published_at: Map.get(fields, "publishedAt"),
created_at: DocumentFields.get(fields, "createdAt", now),
updated_at: DocumentFields.get(fields, "updatedAt", now),
published_at: DocumentFields.get(fields, "publishedAt"),
file_path: rebuild_file.relative_path,
checksum: nil,
tags: Map.get(fields, "tags", []),
categories: Map.get(fields, "categories", []),
template_slug: Map.get(fields, "templateSlug"),
template_slug: DocumentFields.get(fields, "templateSlug"),
language: Map.get(fields, "language"),
do_not_translate: Map.get(fields, "doNotTranslate", false),
do_not_translate: DocumentFields.get(fields, "doNotTranslate", false),
published_title: nil,
published_content: nil,
published_tags: nil,
@@ -894,26 +895,26 @@ defmodule BDS.Posts do
defp upsert_post_translation_from_rebuild_file(project_id, rebuild_file, opts) do
fields = rebuild_file.fields
source_post_id = Map.fetch!(fields, "translationFor")
source_post_id = DocumentFields.fetch!(fields, "translationFor")
source_post = Repo.get_by!(Post, project_id: project_id, id: source_post_id)
now = Persistence.now_ms()
language = normalize_language(Map.fetch!(fields, "language"))
language = normalize_language(DocumentFields.fetch!(fields, "language"))
translation =
Repo.get_by(Translation, translation_for: source_post_id, language: language) || %Translation{}
attrs = %{
id: Map.get(fields, "id") || Ecto.UUID.generate(),
id: DocumentFields.get(fields, "id") || Ecto.UUID.generate(),
project_id: project_id,
translation_for: source_post_id,
language: language,
title: Map.get(fields, "title") || "",
title: DocumentFields.get(fields, "title") || "",
excerpt: Map.get(fields, "excerpt"),
content: nil,
status: parse_translation_status(Map.get(fields, "status", "published")),
created_at: Map.get(fields, "createdAt", source_post.created_at || now),
updated_at: Map.get(fields, "updatedAt", source_post.updated_at || source_post.created_at || now),
published_at: Map.get(fields, "publishedAt", source_post.published_at),
status: parse_translation_status(DocumentFields.get(fields, "status", "published")),
created_at: DocumentFields.get(fields, "createdAt", source_post.created_at || now),
updated_at: DocumentFields.get(fields, "updatedAt", source_post.updated_at || source_post.created_at || now),
published_at: DocumentFields.get(fields, "publishedAt", source_post.published_at),
file_path: rebuild_file.relative_path,
checksum: nil
}
@@ -946,7 +947,7 @@ defmodule BDS.Posts do
end
defp translation_rebuild_file?(%{fields: fields}) do
Map.has_key?(fields, "translationFor") and not Map.has_key?(fields, "slug")
DocumentFields.has_key?(fields, "translationFor") and not DocumentFields.has_key?(fields, "slug")
end
defp list_matching_files(dir, pattern) do

13
lib/bds/scripts.ex
View File

@@ -3,6 +3,7 @@ defmodule BDS.Scripts do
import Ecto.Query
alias BDS.DocumentFields
alias BDS.Frontmatter
alias BDS.Persistence
alias BDS.Projects
@@ -283,19 +284,19 @@ defmodule BDS.Scripts do
now = Persistence.now_ms()
attrs = %{
id: Map.get(fields, "id") || Ecto.UUID.generate(),
id: DocumentFields.get(fields, "id") || Ecto.UUID.generate(),
project_id: project_id,
slug: Map.fetch!(fields, "slug"),
title: Map.get(fields, "title") || "",
kind: parse_script_kind(Map.fetch!(fields, "kind")),
slug: DocumentFields.fetch!(fields, "slug"),
title: DocumentFields.get(fields, "title") || "",
kind: parse_script_kind(DocumentFields.fetch!(fields, "kind")),
entrypoint: Map.get(fields, "entrypoint") || "main",
enabled: Map.get(fields, "enabled", true),
version: Map.get(fields, "version", 1),
file_path: relative_path,
status: :published,
content: nil,
created_at: Map.get(fields, "createdAt", now),
updated_at: Map.get(fields, "updatedAt", now)
created_at: DocumentFields.get(fields, "createdAt", now),
updated_at: DocumentFields.get(fields, "updatedAt", now)
}
script = Repo.get_by(Script, project_id: project_id, slug: attrs.slug) || %Script{}

13
lib/bds/templates.ex
View File

@@ -3,6 +3,7 @@ defmodule BDS.Templates do
import Ecto.Query
alias BDS.DocumentFields
alias BDS.Frontmatter
alias BDS.Persistence
alias BDS.Posts
@@ -408,18 +409,18 @@ defmodule BDS.Templates do
now = Persistence.now_ms()
attrs = %{
id: Map.get(fields, "id") || Ecto.UUID.generate(),
id: DocumentFields.get(fields, "id") || Ecto.UUID.generate(),
project_id: project_id,
slug: Map.fetch!(fields, "slug"),
title: Map.get(fields, "title") || "",
kind: parse_template_kind(Map.fetch!(fields, "kind")),
slug: DocumentFields.fetch!(fields, "slug"),
title: DocumentFields.get(fields, "title") || "",
kind: parse_template_kind(DocumentFields.fetch!(fields, "kind")),
enabled: Map.get(fields, "enabled", true),
version: Map.get(fields, "version", 1),
file_path: relative_path,
status: :published,
content: nil,
created_at: Map.get(fields, "createdAt", now),
updated_at: Map.get(fields, "updatedAt", now)
created_at: DocumentFields.get(fields, "createdAt", now),
updated_at: DocumentFields.get(fields, "updatedAt", now)
}
template = Repo.get_by(Template, project_id: project_id, slug: attrs.slug) || %Template{}

39
test/bds/desktop/shell_commands_test.exs
View File

@@ -99,6 +99,45 @@ defmodule BDS.Desktop.ShellCommandsTest do
assert is_map(completed.result.payload.summary)
end
test "repair_metadata_diff exposes live in-task progress from the repair worker", %{project: project} do
original = Application.get_env(:bds, :tasks, [])
Application.put_env(
:bds,
:tasks,
original
|> Keyword.put(:max_concurrent, 1)
|> Keyword.put(:progress_throttle_ms, 0)
)
on_exit(fn -> Application.put_env(:bds, :tasks, original) end)
items =
Enum.map(1..40, fn _index ->
%{"entity_type" => "project", "entity_id" => project.id}
end)
assert {:ok, result} =
ShellCommands.execute("repair_metadata_diff", %{
"direction" => "file_to_db",
"items" => items
})
progressed =
wait_for_task(
result.task_id,
&(&1.status == :running and is_number(&1.progress) and &1.progress > 0.2 and &1.progress < 1.0),
5_000
)
assert progressed.group_name == "Maintenance"
assert String.contains?(progressed.message, "Repairing")
assert String.contains?(progressed.message, "/")
assert wait_for_task(result.task_id, &(&1.status == :completed and &1.progress == 1.0), 5_000).status ==
:completed
end
test "find_duplicates queues a tracked embeddings task and returns the report as an editor payload" do
assert {:ok, result} = ShellCommands.execute("find_duplicates")

66
test/bds/desktop/shell_live_test.exs
View File

@@ -1034,6 +1034,72 @@ defmodule BDS.Desktop.ShellLiveTest do
refute Enum.any?(diff.diff_reports, &(&1.entity_id == published_post.id))
end
test "metadata diff refresh reruns the diff and replaces the current result", %{
project: project,
temp_dir: temp_dir
} do
:ok = BDS.Tasks.clear_finished()
assert {:ok, post} =
Posts.create_post(%{
project_id: project.id,
title: "Database Post",
content: "Body",
excerpt: "Summary",
language: "en"
})
assert {:ok, published_post} = Posts.publish_post(post.id)
post_path = Path.join(temp_dir, published_post.file_path)
{:ok, view, _html} = live_isolated(build_conn(), BDS.Desktop.ShellLive)
assert {:ok, queued} = BDS.Desktop.ShellCommands.execute("metadata_diff")
completed_task!(queued.task_id)
send(view.pid, :refresh_task_status)
html = render(view)
refute html =~ "Filesystem Post"
File.write!(
post_path,
[
"---",
"id: #{published_post.id}",
"title: Filesystem Post",
"slug: #{published_post.slug}",
"excerpt: Summary",
"status: published",
"language: en",
"createdAt: #{published_post.created_at}",
"updatedAt: #{published_post.updated_at + 1}",
"publishedAt: #{published_post.published_at}",
"tags:",
"categories:",
"---",
"Body",
""
]
|> Enum.join("\n")
)
existing_ids = MapSet.new(Enum.map(BDS.Tasks.list_tasks(), & &1.id))
html =
view
|> element("button[phx-click='rerun_misc_editor']")
|> render_click()
assert html =~ "Metadata Diff"
refresh_task = new_task!(existing_ids, "Metadata Diff")
completed_task!(refresh_task.id)
send(view.pid, :refresh_task_status)
html = render(view)
assert html =~ "Filesystem Post"
end
test "metadata diff orphan import action queues an import task and removes the orphan", %{
project: project,
temp_dir: temp_dir

88
test/bds/maintenance_test.exs
View File

@@ -328,6 +328,49 @@ defmodule BDS.MaintenanceTest do
assert_incremental_progress(collect_progress_events())
end
test "metadata_diff reports incremental progress across comparison phases", %{
project: project,
temp_dir: temp_dir
} do
parent = self()
on_progress = fn value, message -> send(parent, {:rebuild_progress, value, message}) end
posts_dir = Path.join([temp_dir, "posts", "2026", "04"])
File.mkdir_p!(posts_dir)
Enum.each(1..40, fn index ->
slug = "diff-progress-post-#{index}"
File.write!(
Path.join(posts_dir, "#{slug}.md"),
[
"---",
"id: #{slug}",
"title: Diff Progress Post #{index}",
"slug: #{slug}",
"status: published",
"createdAt: 1711843200",
"updatedAt: 1711929600",
"publishedAt: 1712016000",
"tags:",
"categories:",
"---",
"Body #{index}",
""
]
|> Enum.join("\n")
)
end)
assert {:ok, _posts} = BDS.Maintenance.rebuild_from_filesystem(project.id, "post")
assert {:ok, _diff} = BDS.Maintenance.metadata_diff(project.id, on_progress: on_progress)
events = collect_progress_events()
assert_incremental_progress(events)
assert Enum.any?(events, fn {_value, message} -> String.contains?(message, "Comparing") end)
end
test "maintenance rebuilds and diffs embedding state explicitly", %{project: project} do
assert {:ok, _metadata} =
BDS.Metadata.update_project_metadata(project.id, %{semantic_similarity_enabled: true})
@@ -749,6 +792,51 @@ defmodule BDS.MaintenanceTest do
end)
end
test "metadata_diff accepts legacy snake_case post frontmatter keys for status and timestamps", %{
project: project,
temp_dir: temp_dir
} do
assert {:ok, post} =
BDS.Posts.create_post(%{
project_id: project.id,
title: "Legacy Keys Post",
content: "Body",
language: "en"
})
assert {:ok, published_post} = BDS.Posts.publish_post(post.id)
post_path = Path.join(temp_dir, published_post.file_path)
File.write!(
post_path,
[
"---",
"id: #{published_post.id}",
"title: #{published_post.title}",
"slug: #{published_post.slug}",
"status: published",
"language: en",
"created_at: #{published_post.created_at}",
"updated_at: #{published_post.updated_at}",
"published_at: #{published_post.published_at}",
"tags:",
"categories:",
"---",
"Body",
""
]
|> Enum.join("\n")
)
assert {:ok, %{diff_reports: diff_reports}} = BDS.Maintenance.metadata_diff(project.id)
refute Enum.any?(diff_reports, fn report ->
report.entity_type == "post" and report.entity_id == published_post.id and
Enum.any?(report.differences, &(&1.name in ["status", "created_at", "updated_at", "published_at"]))
end)
end
test "metadata_diff includes project-level metadata drift", %{project: project, temp_dir: temp_dir} do
assert {:ok, _metadata} =
BDS.Metadata.update_project_metadata(project.id, %{