hugo/bDS2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: metadata diff hopefully implemented now

Browse Source
This commit is contained in:
Chili Palmer
2026-04-27 08:18:02 +02:00
parent adb49ceb6e
commit e7ccf02d40
19 changed files with 2006 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

54
lib/bds/desktop/shell_commands.ex
View File

@@ -222,6 +222,41 @@ defmodule BDS.Desktop.ShellCommands do
end)
end
defp dispatch("repair_metadata_diff", project, params) do
items = normalize_metadata_diff_items(Map.get(params, "items", Map.get(params, :items, [])))
direction = Map.get(params, "direction", Map.get(params, :direction))
if items == [] do
{:error, %{action: "repair_metadata_diff", message: "No metadata diff items selected"}}
else
queue_task(project, "repair_metadata_diff", "Repair Metadata Diff", "Maintenance", fn report ->
report.(0.2, "Repairing metadata differences")
{:ok, _repair} = Maintenance.repair_metadata_diff(project.id, direction, items)
report.(0.9, "Refreshing metadata diff")
{:ok, metadata_diff} = Maintenance.metadata_diff(project.id)
report.(1.0, "Metadata diff repair complete")
metadata_diff_result(project.id, metadata_diff)
end)
end
end
defp dispatch("import_metadata_diff_orphans", project, params) do
orphans = normalize_metadata_diff_orphans(Map.get(params, "orphans", Map.get(params, :orphans, [])))
if orphans == [] do
{:error, %{action: "import_metadata_diff_orphans", message: "No orphan files selected"}}
else
queue_task(project, "import_metadata_diff_orphans", "Import Metadata Diff Orphans", "Maintenance", fn report ->
report.(0.2, "Importing orphan files")
{:ok, _import} = Maintenance.import_metadata_diff_orphans(project.id, orphans)
report.(0.9, "Refreshing metadata diff")
{:ok, metadata_diff} = Maintenance.metadata_diff(project.id)
report.(1.0, "Metadata diff import complete")
metadata_diff_result(project.id, metadata_diff)
end)
end
end
defp dispatch("validate_translations", project, _params) do
queue_task(project, "validate_translations", "Validate Translations", "Validation", fn report ->
report.(0.2, "Checking published translations")
@@ -519,6 +554,25 @@ defmodule BDS.Desktop.ShellCommands do
Map.new(map, fn {key, value} -> {to_string(key), stringify_value(value)} end)
end
defp normalize_metadata_diff_items(items) when is_list(items) do
Enum.map(items, fn item ->
%{
entity_type: Map.get(item, :entity_type) || Map.get(item, "entity_type"),
entity_id: Map.get(item, :entity_id) || Map.get(item, "entity_id")
}
end)
end
defp normalize_metadata_diff_items(_items), do: []
defp normalize_metadata_diff_orphans(orphans) when is_list(orphans) do
Enum.map(orphans, fn orphan ->
%{file_path: Map.get(orphan, :file_path) || Map.get(orphan, "file_path")}
end)
end
defp normalize_metadata_diff_orphans(_orphans), do: []
defp stringify_value(value) when is_map(value), do: stringify_map(value)
defp stringify_value(value) when is_list(value), do: Enum.map(value, &stringify_value/1)
defp stringify_value(value) when is_atom(value), do: Atom.to_string(value)

46
lib/bds/desktop/shell_live.ex
View File

@@ -98,6 +98,8 @@ defmodule BDS.Desktop.ShellLive do
|> assign(:template_editor_drafts, %{})
|> assign(:chat_editor_inputs, %{})
|> assign(:misc_editor_selected_pairs, %{})
|> assign(:metadata_diff_active_tabs, %{})
|> assign(:metadata_diff_field_filters, %{})
|> assign(:shell_overlay, nil)
|> assign(:output_entries, [])
|> reload_shell(workbench)}
@@ -711,6 +713,46 @@ defmodule BDS.Desktop.ShellLive do
{:noreply, MiscEditor.dismiss_selected(socket, &reload_shell/2, &append_output_entry/5)}
end
def handle_event("repair_metadata_diff", %{"field" => field, "direction" => direction}, socket) do
case MiscEditor.metadata_diff_repair_request(socket, field, direction) do
{:ok, params} -> {:noreply, apply_shell_command(socket, "repair_metadata_diff", params)}
{:error, message} -> {:noreply, append_output_entry(socket, translate_for_socket(socket, "Metadata Diff"), message, nil, "error")}
end
end
def handle_event("import_metadata_diff_orphans", _params, socket) do
case MiscEditor.metadata_diff_orphan_import_request(socket) do
{:ok, params} -> {:noreply, apply_shell_command(socket, "import_metadata_diff_orphans", params)}
{:error, message} -> {:noreply, append_output_entry(socket, translate_for_socket(socket, "Metadata Diff"), message, nil, "error")}
end
end
def handle_event("select_metadata_diff_tab", %{"tab" => tab}, socket) do
tab_id = socket.assigns.current_tab.id
socket =
socket
|> assign(:metadata_diff_active_tabs, Map.put(socket.assigns.metadata_diff_active_tabs, tab_id, tab))
|> assign(:metadata_diff_field_filters, Map.delete(socket.assigns.metadata_diff_field_filters, tab_id))
|> assign_misc_editor()
{:noreply, socket}
end
def handle_event("toggle_metadata_diff_field", %{"field" => field}, socket) do
tab_id = socket.assigns.current_tab.id
current = Map.get(socket.assigns.metadata_diff_field_filters, tab_id)
next_filters =
if current == field do
Map.delete(socket.assigns.metadata_diff_field_filters, tab_id)
else
Map.put(socket.assigns.metadata_diff_field_filters, tab_id, field)
end
{:noreply, socket |> assign(:metadata_diff_field_filters, next_filters) |> assign_misc_editor()}
end
def handle_event("open_duplicate_post", %{"id" => id, "title" => title}, socket) do
{:noreply, open_sidebar_item(socket, %{"route" => "post", "id" => id, "title" => title, "subtitle" => "draft"}, :preview)}
end
@@ -1586,8 +1628,8 @@ defmodule BDS.Desktop.ShellLive do
ArgumentError -> nil
end
defp apply_shell_command(socket, action) do
case ShellCommands.execute(action) do
defp apply_shell_command(socket, action, params \\ %{}) do
case ShellCommands.execute(action, params) do
{:ok, result} -> apply_shell_command_result(socket, result)
{:error, %{message: message}} -> append_output_entry(socket, command_title(action), message, nil, "error")
{:error, reason} -> append_output_entry(socket, command_title(action), inspect(reason), nil, "error")

231
lib/bds/desktop/shell_live/misc_editor.ex
View File

@@ -101,13 +101,64 @@ defmodule BDS.Desktop.ShellLive.MiscEditor do
end
end
def metadata_diff_repair_request(socket, field, direction) do
meta = meta(socket.assigns)
payload = Map.get(meta, :payload, %{})
items = Enum.map(Map.get(payload, :diff_reports, []), &normalize_metadata_diff_item/1)
tabs = metadata_diff_tabs(items, [])
active_tab = metadata_diff_active_tab(socket.assigns, tabs)
repair_items =
items
|> Enum.filter(&(&1.tab_id == active_tab))
|> Enum.filter(fn item -> Enum.any?(item.differences, &(diff_name(&1) == field)) end)
|> Enum.map(&%{"entity_type" => &1.entity_type, "entity_id" => &1.entity_id})
cond do
not metadata_diff_repairable_tab?(active_tab) ->
{:error, translated("No repair action available")}
repair_items == [] ->
{:error, translated("No metadata diff items selected")}
true ->
{:ok,
%{
"direction" => direction,
"field" => field,
"tab" => active_tab,
"items" => repair_items
}}
end
end
def metadata_diff_orphan_import_request(socket) do
meta = meta(socket.assigns)
payload = Map.get(meta, :payload, %{})
items = Enum.map(Map.get(payload, :diff_reports, []), &normalize_metadata_diff_item/1)
orphan_files = Enum.map(Map.get(payload, :orphan_reports, []), &normalize_metadata_diff_orphan/1)
tabs = metadata_diff_tabs(items, orphan_files)
active_tab = metadata_diff_active_tab(socket.assigns, tabs)
selected_orphans =
orphan_files
|> Enum.filter(&(&1.tab_id == active_tab))
|> Enum.map(&%{"file_path" => &1.file_path})
if selected_orphans == [] do
{:error, translated("No orphan files selected")}
else
{:ok, %{"tab" => active_tab, "orphans" => selected_orphans}}
end
end
def build(%{current_tab: %{type: type}} = assigns) when type in @misc_routes do
meta = meta(assigns)
payload = Map.get(meta, :payload, %{})
case type do
:site_validation -> build_site_validation(meta, payload)
:metadata_diff -> build_metadata_diff(meta, payload)
:metadata_diff -> build_metadata_diff(assigns, meta, payload)
:translation_validation -> build_translation_validation(meta, payload)
:find_duplicates -> build_duplicates(assigns, meta, payload)
:git_diff -> build_git_diff(assigns, meta)
@@ -151,17 +202,28 @@ defmodule BDS.Desktop.ShellLive.MiscEditor do
}
end
defp build_metadata_diff(meta, payload) do
items = Map.get(payload, :diff_reports, [])
defp build_metadata_diff(assigns, meta, payload) do
items = Enum.map(Map.get(payload, :diff_reports, []), &normalize_metadata_diff_item/1)
orphan_files = Enum.map(Map.get(payload, :orphan_reports, []), &normalize_metadata_diff_orphan/1)
tabs = metadata_diff_tabs(items, orphan_files)
active_tab = metadata_diff_active_tab(assigns, tabs)
active_field = metadata_diff_active_field(assigns)
current_tab = Enum.find(tabs, &(&1.id == active_tab)) || List.first(tabs) || empty_metadata_diff_tab()
filtered_items = metadata_diff_filtered_items(current_tab.items, active_field)
%{
kind: :metadata_diff,
title: Map.get(meta, :title, translated("Metadata Diff")),
subtitle: Map.get(meta, :subtitle, ""),
summary: Map.get(payload, :summary, %{}),
field_summaries: field_summaries(items),
items: items,
orphan_files: Map.get(payload, :orphan_reports, [])
tabs: Enum.map(tabs, &Map.take(&1, [:id, :label, :badge_count, :diff_count, :orphan_count])),
active_tab: current_tab.id,
active_field: active_field,
repair_enabled: metadata_diff_repairable_tab?(current_tab.id),
field_summaries: field_summaries(current_tab.items),
items: filtered_items,
orphan_files: if(is_nil(active_field), do: current_tab.orphan_files, else: []),
empty_message: translated("No items")
}
end
@@ -247,8 +309,163 @@ defmodule BDS.Desktop.ShellLive.MiscEditor do
defp field_summaries(items) do
items
|> Enum.flat_map(fn item -> Map.get(item, :differences) || Map.get(item, "differences") || [] end)
|> Enum.group_by(fn diff -> Map.get(diff, :field) || Map.get(diff, "field") end)
|> Enum.group_by(&diff_name/1)
|> Enum.map(fn {field, diffs} -> %{field_name: field, diff_count: length(diffs)} end)
|> Enum.sort_by(&{&1.diff_count * -1, &1.field_name})
end
defp metadata_diff_tabs(items, orphan_files) do
tab_ids =
(Enum.map(items, & &1.tab_id) ++ Enum.map(orphan_files, & &1.tab_id))
|> Enum.uniq()
|> Enum.sort_by(&metadata_diff_tab_sort_key/1)
if tab_ids == [] do
[empty_metadata_diff_tab()]
else
Enum.map(tab_ids, fn tab_id ->
tab_items = Enum.filter(items, &(&1.tab_id == tab_id))
tab_orphans = Enum.filter(orphan_files, &(&1.tab_id == tab_id))
%{
id: tab_id,
label: metadata_diff_tab_label(tab_id),
items: tab_items,
orphan_files: tab_orphans,
diff_count: length(tab_items),
orphan_count: length(tab_orphans),
badge_count: length(tab_items) + length(tab_orphans)
}
end)
end
end
defp empty_metadata_diff_tab do
%{id: "posts", label: translated("Posts"), items: [], orphan_files: [], diff_count: 0, orphan_count: 0, badge_count: 0}
end
defp metadata_diff_active_tab(assigns, tabs) do
tab_id = Map.get(assigns.metadata_diff_active_tabs || %{}, assigns.current_tab.id)
if Enum.any?(tabs, &(&1.id == tab_id)) do
tab_id
else
tabs |> List.first() |> Map.get(:id)
end
end
defp metadata_diff_active_field(assigns) do
Map.get(assigns.metadata_diff_field_filters || %{}, assigns.current_tab.id)
end
defp metadata_diff_filtered_items(items, nil), do: items
defp metadata_diff_filtered_items(items, field) do
Enum.filter(items, fn item -> Enum.any?(item.differences, &(diff_name(&1) == field)) end)
end
defp normalize_metadata_diff_item(item) do
entity_type = Map.get(item, :entity_type) || Map.get(item, "entity_type") || "post"
entity_id = Map.get(item, :entity_id) || Map.get(item, "entity_id") || ""
differences =
item
|> Map.get(:differences, Map.get(item, "differences", []))
|> Enum.map(&normalize_metadata_diff_difference/1)
%{
tab_id: metadata_diff_tab_id(entity_type),
entity_type: entity_type,
entity_id: entity_id,
label: metadata_diff_item_label(item, entity_id),
display_entity_type: metadata_diff_item_type_label(entity_type),
differences: differences
}
end
defp normalize_metadata_diff_difference(diff) do
%{
field: diff_name(diff),
db_value: format_metadata_diff_value(Map.get(diff, :db_value) || Map.get(diff, "db_value")),
file_value: format_metadata_diff_value(Map.get(diff, :file_value) || Map.get(diff, "file_value"))
}
end
defp normalize_metadata_diff_orphan(orphan) do
path = Map.get(orphan, :file_path) || Map.get(orphan, "file_path") || Map.get(orphan, :path) || Map.get(orphan, "path") || ""
entity_type = Map.get(orphan, :entity_type) || Map.get(orphan, "entity_type") || metadata_diff_orphan_entity_type(path)
%{
tab_id: metadata_diff_tab_id(entity_type),
entity_type: entity_type,
file_path: path,
slug: Path.basename(path) |> String.trim(),
id: Map.get(orphan, :id) || Map.get(orphan, "id")
}
end
defp metadata_diff_item_label(item, entity_id) do
Map.get(item, :label) || Map.get(item, "label") || Map.get(item, :title) || Map.get(item, "title") || Map.get(item, :slug) || Map.get(item, "slug") || entity_id
end
defp metadata_diff_item_type_label("post"), do: translated("Post")
defp metadata_diff_item_type_label("post_translation"), do: translated("Translations")
defp metadata_diff_item_type_label("media"), do: translated("Media")
defp metadata_diff_item_type_label("media_translation"), do: translated("Translations")
defp metadata_diff_item_type_label("script"), do: translated("Script")
defp metadata_diff_item_type_label("template"), do: translated("Template")
defp metadata_diff_item_type_label("project"), do: translated("Project")
defp metadata_diff_item_type_label("publishing"), do: translated("Publishing")
defp metadata_diff_item_type_label("categories"), do: translated("Categories")
defp metadata_diff_item_type_label("category_meta"), do: translated("Categories")
defp metadata_diff_item_type_label("embedding"), do: translated("Embeddings")
defp metadata_diff_item_type_label(entity_type), do: entity_type |> String.replace("_", " ") |> String.capitalize()
defp metadata_diff_tab_id("post"), do: "posts"
defp metadata_diff_tab_id("post_translation"), do: "posts"
defp metadata_diff_tab_id("media"), do: "media"
defp metadata_diff_tab_id("media_translation"), do: "media"
defp metadata_diff_tab_id("script"), do: "scripts"
defp metadata_diff_tab_id("template"), do: "templates"
defp metadata_diff_tab_id("project"), do: "project"
defp metadata_diff_tab_id("publishing"), do: "project"
defp metadata_diff_tab_id("categories"), do: "project"
defp metadata_diff_tab_id("category_meta"), do: "project"
defp metadata_diff_tab_id("embedding"), do: "embeddings"
defp metadata_diff_tab_id(_entity_type), do: "project"
defp metadata_diff_tab_label("posts"), do: translated("Posts")
defp metadata_diff_tab_label("media"), do: translated("Media")
defp metadata_diff_tab_label("scripts"), do: translated("Scripts")
defp metadata_diff_tab_label("templates"), do: translated("Templates")
defp metadata_diff_tab_label("project"), do: translated("Project")
defp metadata_diff_tab_label("embeddings"), do: translated("Embeddings")
defp metadata_diff_tab_label(tab_id), do: tab_id |> String.replace("_", " ") |> String.capitalize()
defp metadata_diff_tab_sort_key("posts"), do: 0
defp metadata_diff_tab_sort_key("media"), do: 1
defp metadata_diff_tab_sort_key("scripts"), do: 2
defp metadata_diff_tab_sort_key("templates"), do: 3
defp metadata_diff_tab_sort_key("project"), do: 4
defp metadata_diff_tab_sort_key("embeddings"), do: 5
defp metadata_diff_tab_sort_key(other), do: {6, other}
defp metadata_diff_orphan_entity_type(path) do
cond do
String.starts_with?(path, "posts/") -> "post"
String.starts_with?(path, "media/") -> "media"
String.starts_with?(path, "scripts/") -> "script"
String.starts_with?(path, "templates/") -> "template"
true -> "project"
end
end
defp metadata_diff_repairable_tab?(tab_id), do: tab_id in ["posts", "media", "scripts", "templates", "project"]
defp format_metadata_diff_value(nil), do: "-"
defp format_metadata_diff_value(""), do: "-"
defp format_metadata_diff_value(value), do: to_string(value)
defp diff_name(diff) do
Map.get(diff, :field) || Map.get(diff, "field") || Map.get(diff, :name) || Map.get(diff, "name") || "value"
end
end

151
lib/bds/desktop/shell_live/misc_editor_html/misc_editor.html.heex
View File

@@ -31,10 +31,153 @@
</div>
<% :metadata_diff -> %>
<div class="misc-columns">
<section class="misc-card"><h3><%= translated("Field Summary") %></h3><div class="misc-summary-grid"><%= for field <- @misc_editor.field_summaries do %><span class="misc-summary-pill"><%= field.field_name %> <strong><%= field.diff_count %></strong></span><% end %></div></section>
<section class="misc-card"><h3><%= translated("Diff Items") %></h3><div class="misc-list"><%= for item <- @misc_editor.items do %><article class="misc-list-item"><header><strong><%= Map.get(item, :entity_type) || Map.get(item, "entity_type") %></strong> <span><%= Map.get(item, :entity_id) || Map.get(item, "entity_id") %></span></header><ul><%= for diff <- Map.get(item, :differences) || Map.get(item, "differences") || [] do %><li><strong><%= Map.get(diff, :field) || Map.get(diff, "field") %></strong><span><%= inspect(Map.get(diff, :db_value) || Map.get(diff, "db_value")) %></span><span><%= inspect(Map.get(diff, :file_value) || Map.get(diff, "file_value")) %></span></li><% end %></ul></article><% end %></div></section>
<section class="misc-card"><h3><%= translated("Orphan Files") %></h3><ul><%= for orphan <- @misc_editor.orphan_files do %><li><%= inspect(orphan) %></li><% end %></ul></section>
<div class="metadata-diff-tool">
<div class="metadata-diff-tabs" role="tablist">
<%= for tab <- @misc_editor.tabs do %>
<button
class={["metadata-diff-tab", if(@misc_editor.active_tab == tab.id, do: "active")]}
data-testid="metadata-diff-tab"
data-entity-tab={tab.id}
type="button"
phx-click="select_metadata_diff_tab"
phx-value-tab={tab.id}
>
<span><%= tab.label %></span>
<%= if tab.badge_count > 0 do %>
<span class="tab-badge"><%= tab.badge_count %></span>
<% end %>
</button>
<% end %>
</div>
<%= if @misc_editor.field_summaries != [] do %>
<div class="metadata-diff-field-pills">
<%= for field <- @misc_editor.field_summaries do %>
<div class={["metadata-diff-field-pill", if(@misc_editor.active_field == field.field_name, do: "active")]}>
<button
class="metadata-diff-field-pill-toggle"
data-testid="metadata-diff-field-pill"
data-field={field.field_name}
type="button"
phx-click="toggle_metadata_diff_field"
phx-value-field={field.field_name}
>
<span class="field-pill-label"><%= field.field_name %></span>
<span class="field-pill-count"><%= field.diff_count %></span>
</button>
<%= if @misc_editor.repair_enabled do %>
<div class="metadata-diff-field-pill-actions">
<button
class="secondary metadata-diff-action-button"
data-testid="metadata-diff-repair-button"
data-direction="db_to_file"
data-field={field.field_name}
type="button"
phx-click="repair_metadata_diff"
phx-value-direction="db_to_file"
phx-value-field={field.field_name}
>
<%= translated("DB to File") %>
</button>
<button
class="secondary metadata-diff-action-button"
data-testid="metadata-diff-repair-button"
data-direction="file_to_db"
data-field={field.field_name}
type="button"
phx-click="repair_metadata_diff"
phx-value-direction="file_to_db"
phx-value-field={field.field_name}
>
<%= translated("File to DB") %>
</button>
</div>
<% end %>
</div>
<% end %>
</div>
<% end %>
<div class="metadata-diff-results">
<%= if @misc_editor.items == [] do %>
<section class="misc-card metadata-diff-empty">
<p><%= @misc_editor.empty_message %></p>
</section>
<% else %>
<div class="diff-item-list">
<%= for item <- @misc_editor.items do %>
<article class="diff-item-card" data-entity-type={item.entity_type}>
<header class="diff-item-header">
<div>
<strong><%= item.label %></strong>
<div class="diff-item-meta"><%= item.display_entity_type %> · <%= item.entity_id %></div>
</div>
</header>
<div class="diff-item-fields">
<%= for diff <- item.differences do %>
<div class="diff-field-row" data-field={diff.field}>
<div class="diff-field-name"><%= diff.field %></div>
<div class="diff-field-values">
<div class="diff-field-value db-value">
<span class="diff-source-label">DB</span>
<span><%= diff.db_value %></span>
</div>
<div class="diff-field-value file-value">
<span class="diff-source-label"><%= translated("File") %></span>
<span><%= diff.file_value %></span>
</div>
</div>
</div>
<% end %>
</div>
</article>
<% end %>
</div>
<% end %>
<%= if @misc_editor.active_field == nil and @misc_editor.orphan_files != [] do %>
<section class="orphan-files-section" data-testid="metadata-diff-orphans">
<div class="orphan-files-header">
<h3><%= translated("Orphan Files") %></h3>
<div class="orphan-files-actions">
<span class="misc-summary-pill"><%= length(@misc_editor.orphan_files) %></span>
<button
class="secondary metadata-diff-action-button"
data-testid="metadata-diff-import-button"
type="button"
phx-click="import_metadata_diff_orphans"
>
<%= translated("Import") %>
</button>
</div>
</div>
<div class="diff-item-list">
<%= for orphan <- @misc_editor.orphan_files do %>
<article class="diff-item-card orphan-file">
<header class="diff-item-header">
<div>
<strong><%= orphan.slug %></strong>
<div class="diff-item-meta"><%= translated("Orphan Files") %></div>
</div>
</header>
<div class="diff-item-fields">
<div class="diff-field-row">
<div class="diff-field-name"><%= translated("Path") %></div>
<div class="diff-field-values">
<div class="diff-field-value file-value orphan-path"><span><%= orphan.file_path %></span></div>
</div>
</div>
</div>
</article>
<% end %>
</div>
</section>
<% end %>
</div>
</div>
<% :translation_validation -> %>

5
lib/bds/frontmatter.ex
View File

@@ -34,7 +34,10 @@ defmodule BDS.Frontmatter do
defp serialize_field({_key, nil}), do: []
defp serialize_field({_key, ""}), do: []
defp serialize_field({_key, false}), do: []
defp serialize_field({key, false}) do
["#{key}: false"]
end
defp serialize_field({key, true}) do
["#{key}: true"]

141
lib/bds/maintenance.ex
View File

@@ -16,6 +16,57 @@ defmodule BDS.Maintenance do
alias BDS.Sidecar
alias BDS.Templates.Template
def repair_metadata_diff(project_id, direction, items, opts \\ [])
def repair_metadata_diff(project_id, direction, items, opts)
when is_binary(project_id) and is_list(items) do
on_progress = progress_callback(opts)
total = length(items)
:ok = report_started(on_progress, total, "Repairing metadata differences")
result =
items
|> Enum.with_index(1)
|> Enum.reduce(%{repaired: 0, failed: 0}, fn {item, index}, acc ->
next_acc =
case repair_metadata_diff_item(project_id, direction, item) do
:ok -> %{acc | repaired: acc.repaired + 1}
{:ok, _value} -> %{acc | repaired: acc.repaired + 1}
_error -> %{acc | failed: acc.failed + 1}
end
:ok = report_progress(on_progress, index, total, "Repairing metadata differences")
next_acc
end)
{:ok, result}
end
def import_metadata_diff_orphans(project_id, orphans, opts \\ [])
def import_metadata_diff_orphans(project_id, orphans, opts)
when is_binary(project_id) and is_list(orphans) do
on_progress = progress_callback(opts)
total = length(orphans)
:ok = report_started(on_progress, total, "Importing orphan files")
result =
orphans
|> Enum.with_index(1)
|> Enum.reduce(%{imported: 0, failed: 0}, fn {orphan, index}, acc ->
next_acc =
case import_metadata_diff_orphan(project_id, orphan) do
{:ok, _value} -> %{acc | imported: acc.imported + 1}
_error -> %{acc | failed: acc.failed + 1}
end
:ok = report_progress(on_progress, index, total, "Importing orphan files")
next_acc
end)
{:ok, result}
end
def rebuild_from_filesystem(project_id, entity_type, opts \\ []) do
case normalize_entity_type(entity_type) do
:post -> BDS.Posts.rebuild_posts_from_files(project_id, opts)
@@ -547,4 +598,94 @@ defmodule BDS.Maintenance do
file_path -> "#{file_path}.#{translation.language}.meta"
end
end
defp repair_metadata_diff_item(project_id, direction, item) do
entity_type = Map.get(item, :entity_type) || Map.get(item, "entity_type")
entity_id = Map.get(item, :entity_id) || Map.get(item, "entity_id")
case {normalize_repair_direction(direction), entity_type} do
{:file_to_db, entity_type} when entity_type in ["project", "categories", "category_meta", "publishing"] ->
Metadata.sync_project_metadata_from_filesystem(project_id)
{:db_to_file, entity_type} when entity_type in ["project", "categories", "category_meta", "publishing"] ->
Metadata.flush_project_metadata_to_filesystem(project_id)
{:file_to_db, "post"} -> BDS.Posts.sync_post_from_file(entity_id)
{:db_to_file, "post"} -> BDS.Posts.rewrite_published_post(entity_id)
{:file_to_db, "post_translation"} -> BDS.Posts.sync_post_translation_from_file(entity_id)
{:db_to_file, "post_translation"} -> BDS.Posts.rewrite_published_post_translation(entity_id)
{:file_to_db, "media"} -> BDS.Media.sync_media_from_sidecar(entity_id)
{:db_to_file, "media"} -> BDS.Media.sync_media_sidecar(entity_id)
{:file_to_db, "media_translation"} -> BDS.Media.sync_media_translation_from_sidecar(entity_id)
{:db_to_file, "media_translation"} -> BDS.Media.sync_media_translation_sidecar(entity_id)
{:file_to_db, "script"} -> BDS.Scripts.sync_script_from_file(entity_id)
{:db_to_file, "script"} -> BDS.Scripts.sync_published_script_file(entity_id)
{:file_to_db, "template"} -> BDS.Templates.sync_template_from_file(entity_id)
{:db_to_file, "template"} -> BDS.Templates.sync_published_template_file(entity_id)
_other -> {:error, :unsupported}
end
end
defp import_metadata_diff_orphan(project_id, orphan) do
file_path = Map.get(orphan, :file_path) || Map.get(orphan, "file_path")
cond do
is_nil(file_path) ->
{:error, :not_found}
translation_post_file?(file_path) ->
BDS.Posts.import_orphan_post_translation_file(project_id, file_path)
String.ends_with?(file_path, ".md") ->
BDS.Posts.import_orphan_post_file(project_id, file_path)
translation_media_sidecar?(file_path) ->
BDS.Media.import_orphan_media_translation_sidecar(project_id, file_path)
canonical_media_sidecar?(file_path) and String.ends_with?(file_path, ".meta") ->
BDS.Media.import_orphan_media_sidecar(project_id, file_path)
String.ends_with?(file_path, ".lua") ->
BDS.Scripts.import_orphan_script_file(project_id, file_path)
String.ends_with?(file_path, ".liquid") ->
BDS.Templates.import_orphan_template_file(project_id, file_path)
true ->
{:error, :unsupported}
end
end
defp normalize_repair_direction(:file_to_db), do: :file_to_db
defp normalize_repair_direction(:db_to_file), do: :db_to_file
defp normalize_repair_direction("file_to_db"), do: :file_to_db
defp normalize_repair_direction("db_to_file"), do: :db_to_file
defp normalize_repair_direction(_direction), do: :unsupported
defp progress_callback(opts) do
case Keyword.get(opts, :on_progress) do
callback when is_function(callback, 2) -> callback
_other -> nil
end
end
defp report_started(nil, _total, _label), do: :ok
defp report_started(callback, 0, label) do
callback.(1.0, label)
:ok
end
defp report_started(callback, total, label) do
callback.(0.05, "#{label} (0/#{total})")
:ok
end
defp report_progress(nil, _current, _total, _label), do: :ok
defp report_progress(_callback, _current, 0, _label), do: :ok
defp report_progress(callback, current, total, label) do
callback.(0.05 + 0.95 * (current / total), "#{label} (#{current}/#{total})")
:ok
end
end

100
lib/bds/media.ex
View File

@@ -113,6 +113,106 @@ defmodule BDS.Media do
end
end
def sync_media_from_sidecar(media_id) do
case Repo.get(Media, media_id) do
nil ->
{:error, :not_found}
%Media{} = media ->
project = Projects.get_project!(media.project_id)
sidecar_path = Path.join(Projects.project_data_dir(project), media.sidecar_path)
if File.exists?(sidecar_path) do
{:ok, upsert_media_from_sidecar(project, parse_canonical_sidecar(project, sidecar_path), sync_search: true)}
else
{:error, :not_found}
end
end
end
def sync_media_translation_sidecar(translation_id) do
case Repo.get(Translation, translation_id) do
nil ->
{:error, :not_found}
%Translation{} = translation ->
media = Repo.get!(Media, translation.translation_for)
project = Projects.get_project!(media.project_id)
:ok = write_translation_sidecar(project, media, translation)
{:ok, translation}
end
end
def sync_media_translation_from_sidecar(translation_id) do
case Repo.get(Translation, translation_id) do
nil ->
{:error, :not_found}
%Translation{} = translation ->
media = Repo.get!(Media, translation.translation_for)
project = Projects.get_project!(media.project_id)
sidecar_path = Path.join(Projects.project_data_dir(project), translation_sidecar_path(media, translation.language))
if File.exists?(sidecar_path) do
sidecar = parse_translation_sidecar(sidecar_path)
case upsert_media_translation(media.id, Map.fetch!(sidecar.fields, "language"), %{
title: Map.get(sidecar.fields, "title"),
alt: Map.get(sidecar.fields, "alt"),
caption: Map.get(sidecar.fields, "caption")
}) do
{:ok, updated_translation} -> {:ok, updated_translation}
error -> error
end
else
{:error, :not_found}
end
end
end
def import_orphan_media_sidecar(project_id, relative_path) do
project = Projects.get_project!(project_id)
sidecar_path = Path.join(Projects.project_data_dir(project), relative_path)
if File.exists?(sidecar_path) do
{:ok, upsert_media_from_sidecar(project, parse_canonical_sidecar(project, sidecar_path), sync_search: true)}
else
{:error, :not_found}
end
end
def import_orphan_media_translation_sidecar(project_id, relative_path) do
project = Projects.get_project!(project_id)
sidecar_path = Path.join(Projects.project_data_dir(project), relative_path)
if File.exists?(sidecar_path) do
sidecar = parse_translation_sidecar(sidecar_path)
case Repo.get(Media, Map.get(sidecar.fields, "translationFor")) do
nil ->
{:error, :not_found}
media ->
case Repo.get_by(Translation,
translation_for: media.id,
language: Map.fetch!(sidecar.fields, "language")
) do
nil ->
upsert_media_translation(media.id, Map.fetch!(sidecar.fields, "language"), %{
title: Map.get(sidecar.fields, "title"),
alt: Map.get(sidecar.fields, "alt"),
caption: Map.get(sidecar.fields, "caption")
})
_translation ->
{:error, :conflict}
end
end
else
{:error, :not_found}
end
end
def delete_media(media_id) do
case Repo.get(Media, media_id) do
nil ->

9
lib/bds/metadata.ex
View File

@@ -167,6 +167,15 @@ defmodule BDS.Metadata do
|> unwrap_transaction()
end
def flush_project_metadata_to_filesystem(project_id) do
project = Projects.get_project!(project_id)
state = load_state(project)
write_project_metadata_files(project, state, state)
{:ok, state}
end
defp update_state(project_id, updater) do
project = Projects.get_project!(project_id)
state = load_state(project)

133
lib/bds/posts.ex
View File

@@ -239,6 +239,119 @@ defmodule BDS.Posts do
def editor_body(_record), do: ""
def sync_post_from_file(post_id) do
case Repo.get(Post, post_id) do
nil ->
{:error, :not_found}
%Post{file_path: file_path} when file_path in [nil, ""] ->
{:error, :not_found}
%Post{} = post ->
project = Projects.get_project!(post.project_id)
full_path = Path.join(Projects.project_data_dir(project), post.file_path)
if File.exists?(full_path) do
repaired_post = upsert_post_from_file(post.project_id, project, full_path)
:ok = PostLinks.sync_post_links(repaired_post)
{:ok, repaired_post}
else
{:error, :not_found}
end
end
end
def sync_post_translation_from_file(translation_id) do
case Repo.get(Translation, translation_id) do
nil ->
{:error, :not_found}
%Translation{file_path: file_path} when file_path in [nil, ""] ->
{:error, :not_found}
%Translation{} = translation ->
project = Projects.get_project!(translation.project_id)
full_path = Path.join(Projects.project_data_dir(project), translation.file_path)
if File.exists?(full_path) do
rebuild_file = parse_rebuild_file(project, full_path)
{:ok, upsert_post_translation_from_rebuild_file(translation.project_id, rebuild_file, sync_search: true)}
else
{:error, :not_found}
end
end
end
def rewrite_published_post_translation(translation_id) do
case Repo.get(Translation, translation_id) do
nil ->
{:error, :not_found}
%Translation{file_path: file_path, status: status} = translation
when file_path not in [nil, ""] and status == :published ->
post = Repo.get!(Post, translation.translation_for)
:ok = publish_translation(post, translation)
{:ok, Repo.get!(Translation, translation_id)}
%Translation{} ->
{:error, :not_found}
end
end
def import_orphan_post_file(project_id, relative_path) do
project = Projects.get_project!(project_id)
full_path = Path.join(Projects.project_data_dir(project), relative_path)
if File.exists?(full_path) do
rebuild_file = parse_rebuild_file(project, full_path)
if translation_rebuild_file?(rebuild_file) do
{:error, :unsupported_file}
else
fields =
rebuild_file.fields
|> Map.put("id", unique_post_id(Map.get(rebuild_file.fields, "id")))
|> Map.put("slug", unique_slug_for_import(project_id, Map.fetch!(rebuild_file.fields, "slug")))
{:ok, upsert_post_from_rebuild_file(project_id, %{rebuild_file | fields: fields})}
end
else
{:error, :not_found}
end
end
def import_orphan_post_translation_file(project_id, relative_path) do
project = Projects.get_project!(project_id)
full_path = Path.join(Projects.project_data_dir(project), relative_path)
if File.exists?(full_path) do
rebuild_file = parse_rebuild_file(project, full_path)
if translation_rebuild_file?(rebuild_file) do
source_post_id = Map.fetch!(rebuild_file.fields, "translationFor")
language = normalize_language(Map.fetch!(rebuild_file.fields, "language"))
case Repo.get(Post, source_post_id) do
nil ->
{:error, :not_found}
%Post{} = post ->
if normalize_language(post.language) == language or
Repo.get_by(Translation, translation_for: source_post_id, language: language) do
{:error, :conflict}
else
fields = Map.put(rebuild_file.fields, "id", Ecto.UUID.generate())
{:ok, upsert_post_translation_from_rebuild_file(project_id, %{rebuild_file | fields: fields}, sync_search: true)}
end
end
else
{:error, :unsupported_file}
end
else
{:error, :not_found}
end
end
def delete_post(post_id) do
case Repo.get(Post, post_id) do
nil ->
@@ -632,6 +745,26 @@ defmodule BDS.Posts do
defp maybe_put(map, _key, nil), do: map
defp maybe_put(map, key, value), do: Map.put(map, key, value)
defp unique_slug_for_import(project_id, slug) do
normalized = default_slug_source(slug) |> Slug.slugify()
if slug_available?(project_id, normalized) do
normalized
else
find_unique_slug(project_id, normalized, 2)
end
end
defp unique_post_id(nil), do: Ecto.UUID.generate()
defp unique_post_id(id) do
if Repo.get(Post, id) || Repo.get(Translation, id) do
Ecto.UUID.generate()
else
id
end
end
defp normalize_title(nil), do: ""
defp normalize_title(title), do: title

63
lib/bds/scripts.ex
View File

@@ -140,6 +140,54 @@ defmodule BDS.Scripts do
{:ok, scripts}
end
def sync_script_from_file(script_id) do
case Repo.get(Script, script_id) do
nil ->
{:error, :not_found}
%Script{file_path: file_path} when file_path in [nil, ""] ->
{:error, :not_found}
%Script{} = script ->
project = Projects.get_project!(script.project_id)
full_path = Path.join(Projects.project_data_dir(project), script.file_path)
if File.exists?(full_path) do
{:ok, upsert_script_from_file(script.project_id, project, full_path)}
else
{:error, :not_found}
end
end
end
def sync_published_script_file(script_id) do
case Repo.get(Script, script_id) do
nil ->
{:error, :not_found}
%Script{file_path: file_path, status: status} = script
when file_path not in [nil, ""] and status == :published ->
full_path = full_file_path(script.project_id, script.file_path)
body = published_script_body(script)
:ok = Persistence.atomic_write(full_path, serialize_script_file(script, body))
{:ok, script}
%Script{} ->
{:error, :not_found}
end
end
def import_orphan_script_file(project_id, relative_path) do
project = Projects.get_project!(project_id)
full_path = Path.join(Projects.project_data_dir(project), relative_path)
if File.exists?(full_path) do
{:ok, upsert_script_from_file(project_id, project, full_path)}
else
{:error, :not_found}
end
end
defp default_entrypoint(:macro), do: "render"
defp default_entrypoint(_kind), do: "main"
@@ -213,6 +261,21 @@ defmodule BDS.Scripts do
end
end
defp published_script_body(%Script{content: content}) when is_binary(content), do: content
defp published_script_body(script) do
case File.read(full_file_path(script.project_id, script.file_path)) do
{:ok, contents} ->
case Frontmatter.parse_document(contents) do
{:ok, %{body: body}} -> body
{:error, _reason} -> ""
end
{:error, _reason} ->
""
end
end
defp upsert_script_from_file(project_id, project, path) do
contents = File.read!(path)
{:ok, %{fields: fields}} = Frontmatter.parse_document(contents)

47
lib/bds/templates.ex
View File

@@ -186,6 +186,53 @@ defmodule BDS.Templates do
end
end
def sync_template_from_file(template_id) do
case Repo.get(Template, template_id) do
nil ->
{:error, :not_found}
%Template{file_path: file_path} when file_path in [nil, ""] ->
{:error, :not_found}
%Template{} = template ->
project = Projects.get_project!(template.project_id)
full_path = Path.join(Projects.project_data_dir(project), template.file_path)
if File.exists?(full_path) do
{:ok, upsert_template_from_file(template.project_id, project, full_path)}
else
{:error, :not_found}
end
end
end
def sync_published_template_file(template_id) do
case Repo.get(Template, template_id) do
nil ->
{:error, :not_found}
%Template{file_path: file_path, status: status} = template
when file_path not in [nil, ""] and status == :published ->
full_path = full_file_path(template.project_id, template.file_path)
:ok = Persistence.atomic_write(full_path, serialize_template_file(template, published_template_body(template)))
{:ok, template}
%Template{} ->
{:error, :not_found}
end
end
def import_orphan_template_file(project_id, relative_path) do
project = Projects.get_project!(project_id)
full_path = Path.join(Projects.project_data_dir(project), relative_path)
if File.exists?(full_path) do
{:ok, upsert_template_from_file(project_id, project, full_path)}
else
{:error, :not_found}
end
end
defp unique_slug(project_id, base_slug, fallback, exclude_id \\ nil) do
normalized = if base_slug == "", do: fallback, else: base_slug