feat: step 5 claimed done

lib/bds/ai.ex

@@ -317,6 +317,48 @@ defmodule BDS.AI do
     |> Enum.map(&format_conversation/1)
   end
 
+  def available_chat_models(current_model \\ nil) do
+    endpoint_models =
+      [:online, :airplane]
+      |> Enum.flat_map(fn kind ->
+        case get_endpoint(kind) do
+          {:ok, %{model: model}} when is_binary(model) and model != "" -> [model]
+          _other -> []
+        end
+      end)
+
+    preference_models =
+      [:chat, :airplane_chat]
+      |> Enum.flat_map(fn key ->
+        case get_model_preference(key) do
+          {:ok, model} when is_binary(model) and model != "" -> [model]
+          _other -> []
+        end
+      end)
+
+    [current_model | endpoint_models ++ preference_models]
+    |> Enum.filter(&(is_binary(&1) and String.trim(&1) != ""))
+    |> Enum.uniq()
+    |> Enum.map(&%{id: &1, name: &1})
+  end
+
+  def set_conversation_model(conversation_id, model_id)
+      when is_binary(conversation_id) and is_binary(model_id) do
+    case Repo.get(ChatConversation, conversation_id) do
+      nil ->
+        {:error, :not_found}
+
+      %ChatConversation{} = conversation ->
+        conversation
+        |> ChatConversation.changeset(%{model: model_id, updated_at: Persistence.now_ms()})
+        |> Repo.update()
+        |> case do
+          {:ok, updated_conversation} -> {:ok, format_conversation(updated_conversation)}
+          error -> error
+        end
+    end
+  end
+
   def list_chat_messages(conversation_id) when is_binary(conversation_id) do
     Repo.all(
       from message in ChatMessage,

lib/bds/desktop/shell_commands.ex

@@ -514,14 +514,10 @@ defmodule BDS.Desktop.ShellCommands do
 
   defp normalize_translation_validation(report) do
     %{
-      summary: %{
-        missing_count: length(report.missing),
-        orphan_count: length(report.orphan_files),
-        do_not_translate_count: length(report.do_not_translate_posts)
-      },
-      missing: Enum.map(report.missing, &stringify_map/1),
-      orphan_files: report.orphan_files,
-      do_not_translate_posts: report.do_not_translate_posts
+      checked_database_row_count: report.checked_database_row_count,
+      checked_filesystem_file_count: report.checked_filesystem_file_count,
+      invalid_database_rows: Enum.map(report.invalid_database_rows, &stringify_map/1),
+      invalid_filesystem_files: Enum.map(report.invalid_filesystem_files, &stringify_map/1)
     }
   end
 
@@ -532,11 +528,10 @@ defmodule BDS.Desktop.ShellCommands do
       project_id: project_id,
       route: "translation_validation",
       title: "Translation Validation",
-      subtitle: "Published posts checked against required blog languages",
+      subtitle: "Database rows and translation files checked for invalid state",
       editorMeta: [
-        %{label: "Missing", value: Integer.to_string(length(report.missing))},
-        %{label: "Orphan Files", value: Integer.to_string(length(report.orphan_files))},
-        %{label: "Skipped", value: Integer.to_string(length(report.do_not_translate_posts))}
+        %{label: "Invalid DB", value: Integer.to_string(length(report.invalid_database_rows))},
+        %{label: "Invalid Files", value: Integer.to_string(length(report.invalid_filesystem_files))}
       ],
       payload: normalize_translation_validation(report)
     }

lib/bds/desktop/shell_live.ex

@@ -97,7 +97,9 @@ defmodule BDS.Desktop.ShellLive do
         |> assign(:script_editor_drafts, %{})
         |> assign(:template_editor_drafts, %{})
         |> assign(:chat_editor_inputs, %{})
+        |> assign(:chat_model_selectors_open, %{})
         |> assign(:misc_editor_selected_pairs, %{})
+        |> assign(:misc_editor_git_selected_files, %{})
         |> assign(:metadata_diff_active_tabs, %{})
         |> assign(:metadata_diff_field_filters, %{})
         |> assign(:shell_overlay, nil)
@@ -683,6 +685,14 @@ defmodule BDS.Desktop.ShellLive do
     {:noreply, ChatEditor.update_input(socket, message, &reload_shell/2)}
   end
 
+  def handle_event("toggle_chat_model_selector", _params, socket) do
+    {:noreply, ChatEditor.toggle_model_selector(socket, &reload_shell/2)}
+  end
+
+  def handle_event("select_chat_model", %{"model" => model_id}, socket) do
+    {:noreply, ChatEditor.set_model(socket, model_id, &reload_shell/2, &append_output_entry/5)}
+  end
+
   def handle_event("send_chat_editor_message", _params, socket) do
     {:noreply, ChatEditor.send_message(socket, &reload_shell/2, &append_output_entry/5)}
   end
@@ -701,6 +711,17 @@ defmodule BDS.Desktop.ShellLive do
     end
   end
 
+  def handle_event("fix_translation_validation", _params, socket) do
+    case MiscEditor.fix_translation_validation(socket, &append_output_entry/5) do
+      {:rerun, next_socket} -> {:noreply, apply_shell_command(next_socket, "validate_translations")}
+      {:socket, next_socket} -> {:noreply, next_socket}
+    end
+  end
+
+  def handle_event("select_git_diff_file", %{"path" => path}, socket) do
+    {:noreply, socket |> MiscEditor.select_git_diff_file(path) |> assign_misc_editor()}
+  end
+
   def handle_event("toggle_duplicate_pair", %{"pair-id" => pair_id}, socket) do
     {:noreply, MiscEditor.toggle_duplicate(socket, pair_id, &reload_shell/2)}
   end

lib/bds/desktop/shell_live/chat_editor.ex

@@ -13,6 +13,31 @@ defmodule BDS.Desktop.ShellLive.ChatEditor do
     assign(socket, :chat_editor, build(socket.assigns))
   end
 
+  def toggle_model_selector(socket, reload) do
+    %{id: conversation_id} = socket.assigns.current_tab
+    current = Map.get(socket.assigns.chat_model_selectors_open, conversation_id, false)
+
+    socket
+    |> assign(:chat_model_selectors_open, Map.put(socket.assigns.chat_model_selectors_open, conversation_id, not current))
+    |> reload.(socket.assigns.workbench)
+  end
+
+  def set_model(socket, model_id, reload, append_output) do
+    %{id: conversation_id} = socket.assigns.current_tab
+
+    case AI.set_conversation_model(conversation_id, model_id) do
+      {:ok, _conversation} ->
+        socket
+        |> assign(:chat_model_selectors_open, Map.put(socket.assigns.chat_model_selectors_open, conversation_id, false))
+        |> reload.(socket.assigns.workbench)
+
+      {:error, reason} ->
+        socket
+        |> append_output.(translated("Chat"), inspect(reason), nil, "error")
+        |> reload.(socket.assigns.workbench)
+    end
+  end
+
   def update_input(socket, value, reload) do
     %{id: conversation_id} = socket.assigns.current_tab
 
@@ -53,10 +78,12 @@ defmodule BDS.Desktop.ShellLive.ChatEditor do
       %ChatConversation{} = conversation ->
         %{
           id: conversation.id,
-          title: conversation.title || translated("New Chat"),
+          title: conversation.title || translated("chat.newChat"),
           model: conversation.model,
+          available_models: AI.available_chat_models(conversation.model),
+          model_selector_open?: Map.get(assigns.chat_model_selectors_open, conversation.id, false),
           input: Map.get(assigns.chat_editor_inputs, conversation.id, ""),
-          messages: AI.list_chat_messages(conversation.id),
+          messages: build_entries(AI.list_chat_messages(conversation.id)),
           offline?: Map.get(assigns, :offline_mode, true)
         }
     end
@@ -64,5 +91,89 @@ defmodule BDS.Desktop.ShellLive.ChatEditor do
 
   def build(_assigns), do: nil
 
+  def message_role_label(:user), do: translated("chat.role.you")
+  def message_role_label(_role), do: translated("chat.role.assistant")
+
+  def tool_call_name(tool_call) when is_map(tool_call) do
+    Map.get(tool_call, "name") || Map.get(tool_call, :name) || "tool"
+  end
+
+  def tool_surface_type(surface), do: Map.get(surface, :type, "json")
+
+  defp build_entries(messages) do
+    {entries, current_entry} =
+      Enum.reduce(messages, {[], nil}, fn message, {entries, current_entry} ->
+        case message.role do
+          :tool ->
+            if current_entry && current_entry.role == :assistant do
+              {entries, append_tool_surface(current_entry, message)}
+            else
+              {entries, current_entry}
+            end
+
+          :system ->
+            {entries, current_entry}
+
+          _other ->
+            entries = finalize_entry(entries, current_entry)
+            {entries, start_entry(message)}
+        end
+      end)
+
+    entries
+    |> finalize_entry(current_entry)
+    |> Enum.reverse()
+  end
+
+  defp finalize_entry(entries, nil), do: entries
+  defp finalize_entry(entries, entry), do: [entry | entries]
+
+  defp start_entry(message) do
+    %{
+      id: message.id,
+      role: message.role,
+      content: message.content || "",
+      tool_markers: normalize_tool_calls(message.tool_calls),
+      tool_surfaces: []
+    }
+  end
+
+  defp append_tool_surface(entry, message) do
+    case normalize_tool_surface(message.content) do
+      nil -> entry
+      surface -> update_in(entry.tool_surfaces, &(&1 ++ [surface]))
+    end
+  end
+
+  defp normalize_tool_calls(tool_calls) when is_list(tool_calls) do
+    Enum.map(tool_calls, fn tool_call ->
+      %{
+        name: tool_call_name(tool_call),
+        arguments: Map.get(tool_call, "arguments") || Map.get(tool_call, :arguments) || Map.get(tool_call, "args") || Map.get(tool_call, :args) || %{}
+      }
+    end)
+  end
+
+  defp normalize_tool_calls(_tool_calls), do: []
+
+  defp normalize_tool_surface(content) when is_binary(content) do
+    case Jason.decode(content) do
+      {:ok, %{"type" => type} = decoded} ->
+        %{
+          type: type,
+          title: decoded["title"],
+          columns: List.wrap(decoded["columns"]),
+          rows: Enum.map(List.wrap(decoded["rows"]), &List.wrap/1),
+          fields: List.wrap(decoded["fields"]),
+          data: decoded
+        }
+
+      _other ->
+        nil
+    end
+  end
+
+  defp normalize_tool_surface(_content), do: nil
+
   def translated(text, bindings \\ %{}), do: ShellData.translate(text, bindings, Process.get(:bds_ui_locale))
 end

lib/bds/desktop/shell_live/chat_editor_html/chat_editor.html.heex

@@ -1,37 +1,113 @@
 <div class="chat-panel" data-testid="chat-editor">
   <div class="chat-panel-header">
     <div class="chat-panel-title"><%= @chat_editor.title %></div>
+
+    <div class="chat-panel-header-actions">
+      <button
+        class="chat-model-selector-button"
+        type="button"
+        phx-click="toggle_chat_model_selector"
+        data-testid="chat-model-selector-button"
+      >
+        <span><%= @chat_editor.model || translated("chat.newChat") %></span>
+        <span class="chat-model-selector-caret">▾</span>
+      </button>
+
+      <%= if @chat_editor.model_selector_open? and @chat_editor.available_models != [] do %>
+        <div class="chat-model-selector-menu">
+          <%= for model <- @chat_editor.available_models do %>
+            <button
+              class={[
+                "chat-model-selector-option",
+                if(model.id == @chat_editor.model, do: "active")
+              ]}
+              type="button"
+              phx-click="select_chat_model"
+              phx-value-model={model.id}
+            >
+              <%= model.name %>
+            </button>
+          <% end %>
+        </div>
+      <% end %>
+    </div>
   </div>
 
   <div class="chat-messages">
     <%= if Enum.empty?(@chat_editor.messages) do %>
       <div class="chat-welcome">
         <div class="chat-welcome-icon">🤖</div>
-        <h2><%= translated("New Chat") %></h2>
-        <p><%= translated("Ask the assistant about the active project.") %></p>
+        <h2><%= translated("chat.welcomeTitle") %></h2>
+        <p><%= translated("chat.welcomeDescription") %></p>
         <ul>
-          <li><%= translated("Search posts and media") %></li>
-          <li><%= translated("Inspect metadata") %></li>
-          <li><%= translated("Open related tabs") %></li>
-          <li><%= translated("Review generated output") %></li>
-          <li><%= translated("Navigate settings") %></li>
+          <li><%= translated("chat.welcomeTipSearch") %></li>
+          <li><%= translated("chat.welcomeTipChart") %></li>
+          <li><%= translated("chat.welcomeTipTable") %></li>
+          <li><%= translated("chat.welcomeTipMetadata") %></li>
+          <li><%= translated("chat.welcomeTipTabs") %></li>
         </ul>
       </div>
     <% else %>
       <%= for message <- @chat_editor.messages do %>
         <div class={["chat-message", to_string(message.role || "assistant")]}>
           <div class="chat-message-content">
-            <div class="chat-message-header"><span class="chat-message-role"><%= String.capitalize(to_string(message.role || "assistant")) %></span></div>
+            <div class="chat-message-header"><span class="chat-message-role"><%= message_role_label(message.role) %></span></div>
+
+            <%= if message.tool_markers != [] do %>
+              <div class="chat-tool-markers">
+                <%= for tool_call <- message.tool_markers do %>
+                  <div class="chat-tool-marker" data-testid="chat-tool-marker">
+                    <span class="chat-tool-marker-name"><%= tool_call_name(tool_call) %></span>
+                  </div>
+                <% end %>
+              </div>
+            <% end %>
+
             <div class="chat-message-text"><%= message.content || "" %></div>
           </div>
         </div>
+
+        <%= for surface <- message.tool_surfaces do %>
+          <article class="chat-tool-surface" data-testid="chat-tool-surface">
+            <%= if surface.title do %>
+              <h3><%= surface.title %></h3>
+            <% end %>
+
+            <%= case tool_surface_type(surface) do %>
+              <% "table" -> %>
+                <div class="chat-tool-surface-table-wrap">
+                  <table class="chat-tool-surface-table">
+                    <thead>
+                      <tr>
+                        <%= for column <- surface.columns do %>
+                          <th><%= column %></th>
+                        <% end %>
+                      </tr>
+                    </thead>
+                    <tbody>
+                      <%= for row <- surface.rows do %>
+                        <tr>
+                          <%= for value <- row do %>
+                            <td><%= value %></td>
+                          <% end %>
+                        </tr>
+                      <% end %>
+                    </tbody>
+                  </table>
+                </div>
+
+              <% _other -> %>
+                <pre class="chat-tool-surface-json"><%= Jason.encode!(surface.data, pretty: true) %></pre>
+            <% end %>
+          </article>
+        <% end %>
       <% end %>
     <% end %>
   </div>
 
   <div class="chat-input-container">
     <form class="chat-input-wrapper" phx-change="change_chat_editor_input">
-      <textarea class="chat-input chat-surface-input" name="message" rows="1" placeholder={translated("Send a message")}><%= @chat_editor.input %></textarea>
+      <textarea class="chat-input chat-surface-input" name="message" rows="1" placeholder={translated("chat.inputPlaceholder")}><%= @chat_editor.input %></textarea>
       <button class="chat-send-button" type="button" phx-click="send_chat_editor_message" disabled={String.trim(@chat_editor.input || "") == "" or @chat_editor.offline?}>↑</button>
     </form>
   </div>

lib/bds/desktop/shell_live/misc_editor.ex

@@ -3,8 +3,11 @@ defmodule BDS.Desktop.ShellLive.MiscEditor do
 
   use Phoenix.Component
 
-  alias BDS.{Embeddings, Generation, Git}
+  import Ecto.Query
+
+  alias BDS.{Embeddings, Generation, Git, Posts, Repo}
   alias BDS.Desktop.ShellData
+  alias BDS.Settings.Setting
 
   embed_templates "misc_editor_html/*"
 
@@ -114,6 +117,37 @@ defmodule BDS.Desktop.ShellLive.MiscEditor do
     end
   end
 
+  def fix_translation_validation(socket, append_output) do
+    report =
+      socket.assigns
+      |> meta()
+      |> Map.get(:payload, %{})
+      |> normalize_translation_validation_report()
+
+    {:ok, result} = Posts.fix_invalid_translations(report)
+
+    {:rerun,
+     socket
+     |> append_output.(
+       translated("Translation Validation"),
+       translated("translationValidation.toast.fixSuccess", %{
+         dbRows: result.deleted_database_rows,
+         files: result.deleted_files,
+         flushed: result.flushed_translations
+       })
+     )}
+  rescue
+    error -> {:socket, append_output.(socket, translated("Translation Validation"), inspect(error), nil, "error")}
+  end
+
+  def select_git_diff_file(socket, file_path) do
+    assign(
+      socket,
+      :misc_editor_git_selected_files,
+      Map.put(socket.assigns.misc_editor_git_selected_files, socket.assigns.current_tab.id, file_path)
+    )
+  end
+
   def metadata_diff_repair_request(socket, field, direction) do
     meta = meta(socket.assigns)
     payload = Map.get(meta, :payload, %{})
@@ -245,14 +279,23 @@ defmodule BDS.Desktop.ShellLive.MiscEditor do
   end
 
   defp build_translation_validation(meta, payload) do
+    report = normalize_translation_validation_report(payload)
+
     %{
       kind: :translation_validation,
       title: Map.get(meta, :title, translated("Translation Validation")),
       subtitle: Map.get(meta, :subtitle, ""),
-      summary: Map.get(payload, :summary, %{}),
-      missing: Map.get(payload, :missing, []),
-      orphan_files: Map.get(payload, :orphan_files, []),
-      do_not_translate_posts: Map.get(payload, :do_not_translate_posts, [])
+      summary: %{},
+      summary_text:
+        translated("translationValidation.summary", %{
+          dbRows: report.checked_database_row_count,
+          files: report.checked_filesystem_file_count,
+          invalidDb: length(report.invalid_database_rows),
+          invalidFiles: length(report.invalid_filesystem_files)
+        }),
+      invalid_database_rows: report.invalid_database_rows,
+      invalid_filesystem_files: report.invalid_filesystem_files,
+      can_fix?: report.invalid_database_rows != [] or report.invalid_filesystem_files != []
     }
   end
 
@@ -270,29 +313,93 @@ defmodule BDS.Desktop.ShellLive.MiscEditor do
   end
 
   defp build_git_diff(assigns, meta) do
-    diff_text =
-      case Git.diff(assigns.projects.active_project_id) do
-        {:ok, %{staged_diff: staged, unstaged_diff: unstaged}} ->
-          [
-            "# Staged Changes\n\n",
-            if(String.trim(staged) == "", do: translated("No staged changes"), else: staged),
-            "\n\n# Working Tree\n\n",
-            if(String.trim(unstaged) == "", do: translated("No unstaged changes"), else: unstaged)
-          ]
-          |> IO.iodata_to_binary()
+    project_id = assigns.projects.active_project_id
+    selected_files = Map.get(assigns, :misc_editor_git_selected_files, %{})
 
-        {:error, reason} -> inspect(reason)
+    {files, diff, error_message} =
+      case Git.status(project_id) do
+        {:ok, %{files: files}} ->
+          file_paths = files |> Enum.map(&Map.get(&1, :path)) |> Enum.reject(&is_nil/1) |> Enum.uniq() |> Enum.sort()
+          selected_file_path = select_git_diff_path(assigns.current_tab.id, file_paths, selected_files)
+
+          diff =
+            case selected_file_path do
+              nil ->
+                empty_git_diff(project_id)
+
+              file_path ->
+                case Git.get_diff_content(project_id, file_path) do
+                  {:ok, diff} -> diff
+                  {:error, reason} -> Map.merge(empty_git_diff(project_id), %{file_path: file_path, error: inspect(reason)})
+                end
+            end
+
+          {file_paths, diff, nil}
+
+        {:error, reason} ->
+          {[], empty_git_diff(project_id), inspect(reason)}
       end
 
+    preferences = git_diff_preferences()
+
     %{
       kind: :git_diff,
       title: Map.get(meta, :title, translated("Git Diff")),
       subtitle: Map.get(meta, :subtitle, ""),
-      diff_text: diff_text,
-      summary: %{}
+      summary: %{},
+      files: files,
+      selected_file_path: diff.file_path,
+      active_diff: Map.put(diff, :language, git_diff_language(diff.file_path)),
+      preferences: preferences,
+      empty_message: error_message || translated("No unstaged changes")
     }
   end
 
+  def translation_issue_label(issue) do
+    case issue_value(issue, :issue) do
+      "same-language-as-canonical" -> translated("translationValidation.issue.sameLanguage")
+      "do-not-translate-has-translations" -> translated("translationValidation.issue.doNotTranslate")
+      "content-in-database" -> translated("translationValidation.issue.contentInDatabase")
+      _other -> translated("translationValidation.issue.missingSource")
+    end
+  end
+
+  def translation_issue_languages(issue) do
+    canonical_language = issue_value(issue, :canonical_language)
+    translation_language = issue_value(issue, :translation_language)
+
+    if canonical_language in [nil, ""] do
+      translation_language
+    else
+      translated("translationValidation.languagesWithCanonical", %{
+        canonical: canonical_language,
+        translation: translation_language
+      })
+    end
+  end
+
+  def translation_issue_value(issue, key), do: issue_value(issue, key)
+
+  def git_diff_language(nil), do: "plaintext"
+
+  def git_diff_language(file_path) do
+    case file_path |> Path.extname() |> String.downcase() do
+      ".md" -> "markdown"
+      ".markdown" -> "markdown"
+      ".mdx" -> "markdown"
+      ".ts" -> "typescript"
+      ".tsx" -> "typescript"
+      ".js" -> "javascript"
+      ".jsx" -> "javascript"
+      ".json" -> "json"
+      ".css" -> "css"
+      ".html" -> "html"
+      ".yml" -> "yaml"
+      ".yaml" -> "yaml"
+      _other -> "plaintext"
+    end
+  end
+
   defp meta(assigns) do
     Map.get(assigns.tab_meta, {assigns.current_tab.type, assigns.current_tab.id}, %{})
   end
@@ -496,4 +603,60 @@ defmodule BDS.Desktop.ShellLive.MiscEditor do
   defp diff_name(diff) do
     Map.get(diff, :field) || Map.get(diff, "field") || Map.get(diff, :name) || Map.get(diff, "name") || "value"
   end
+
+  defp normalize_translation_validation_report(payload) when is_map(payload) do
+    %{
+      checked_database_row_count: issue_value(payload, :checked_database_row_count) || 0,
+      checked_filesystem_file_count: issue_value(payload, :checked_filesystem_file_count) || 0,
+      invalid_database_rows:
+        payload
+        |> issue_value(:invalid_database_rows)
+        |> List.wrap(),
+      invalid_filesystem_files:
+        payload
+        |> issue_value(:invalid_filesystem_files)
+        |> List.wrap()
+    }
+  end
+
+  defp normalize_translation_validation_report(_payload) do
+    %{
+      checked_database_row_count: 0,
+      checked_filesystem_file_count: 0,
+      invalid_database_rows: [],
+      invalid_filesystem_files: []
+    }
+  end
+
+  defp issue_value(issue, key) when is_map(issue) do
+    Map.get(issue, key) || Map.get(issue, Atom.to_string(key))
+  end
+
+  defp issue_value(_issue, _key), do: nil
+
+  defp select_git_diff_path(tab_id, files, selected_files) do
+    selected = Map.get(selected_files, tab_id)
+
+    if selected in files do
+      selected
+    else
+      List.first(files)
+    end
+  end
+
+  defp empty_git_diff(file_path) do
+    %{file_path: file_path, original: "", modified: "", error: nil}
+  end
+
+  defp git_diff_preferences do
+    %{
+      view_style: get_global_setting("ui.git_diff_view_style") || "inline",
+      word_wrap: get_global_setting("ui.git_diff_word_wrap") == "true",
+      hide_unchanged_regions: get_global_setting("ui.git_diff_hide_unchanged_regions") == "true"
+    }
+  end
+
+  defp get_global_setting(key) do
+    Repo.one(from setting in Setting, where: setting.key == ^key, select: setting.value)
+  end
 end

lib/bds/desktop/shell_live/misc_editor_html/misc_editor.html.heex

@@ -181,10 +181,79 @@
         </div>
 
       <% :translation_validation -> %>
-        <div class="misc-columns">
-          <section class="misc-card"><h3><%= translated("Missing") %></h3><ul><%= for issue <- @misc_editor.missing do %><li><%= inspect(issue) %></li><% end %></ul></section>
-          <section class="misc-card"><h3><%= translated("Orphan Files") %></h3><ul><%= for file <- @misc_editor.orphan_files do %><li><%= file %></li><% end %></ul></section>
-          <section class="misc-card"><h3><%= translated("Do Not Translate") %></h3><ul><%= for post <- @misc_editor.do_not_translate_posts do %><li><%= inspect(post) %></li><% end %></ul></section>
+        <div class="translation-validation-view">
+          <section class="translation-validation-summary">
+            <p><%= @misc_editor.summary_text %></p>
+          </section>
+
+          <section class="translation-validation-section">
+            <h3><%= translated("translationValidation.databaseTitle") %></h3>
+
+            <%= if @misc_editor.invalid_database_rows == [] do %>
+              <p class="translation-validation-empty"><%= translated("translationValidation.noneDatabase") %></p>
+            <% else %>
+              <div class="translation-validation-list">
+                <%= for issue <- @misc_editor.invalid_database_rows do %>
+                  <article class="translation-validation-card translation-validation-card-db" data-testid="translation-validation-card">
+                    <p class="translation-validation-card-title"><%= translation_issue_label(issue) %></p>
+                    <dl class="translation-validation-card-meta">
+                      <dt><%= translated("translationValidation.field.translationFor") %></dt>
+                      <dd><%= translation_issue_value(issue, :translation_for) %></dd>
+                      <%= if translation_issue_value(issue, :translation_id) do %>
+                        <dt><%= translated("translationValidation.field.translationId") %></dt>
+                        <dd><%= translation_issue_value(issue, :translation_id) %></dd>
+                      <% end %>
+                      <%= if translation_issue_value(issue, :title) do %>
+                        <dt><%= translated("translationValidation.field.title") %></dt>
+                        <dd><%= translation_issue_value(issue, :title) %></dd>
+                      <% end %>
+                      <dt><%= translated("translationValidation.field.languages") %></dt>
+                      <dd><%= translation_issue_languages(issue) %></dd>
+                      <%= if translation_issue_value(issue, :file_path) do %>
+                        <dt><%= translated("translationValidation.field.filePath") %></dt>
+                        <dd><%= translation_issue_value(issue, :file_path) %></dd>
+                      <% end %>
+                    </dl>
+                  </article>
+                <% end %>
+              </div>
+            <% end %>
+          </section>
+
+          <section class="translation-validation-section">
+            <h3><%= translated("translationValidation.filesystemTitle") %></h3>
+
+            <%= if @misc_editor.invalid_filesystem_files == [] do %>
+              <p class="translation-validation-empty"><%= translated("translationValidation.noneFilesystem") %></p>
+            <% else %>
+              <div class="translation-validation-list">
+                <%= for issue <- @misc_editor.invalid_filesystem_files do %>
+                  <article class="translation-validation-card translation-validation-card-file" data-testid="translation-validation-card">
+                    <p class="translation-validation-card-title"><%= translation_issue_label(issue) %></p>
+                    <dl class="translation-validation-card-meta">
+                      <dt><%= translated("translationValidation.field.translationFor") %></dt>
+                      <dd><%= translation_issue_value(issue, :translation_for) %></dd>
+                      <%= if translation_issue_value(issue, :title) do %>
+                        <dt><%= translated("translationValidation.field.title") %></dt>
+                        <dd><%= translation_issue_value(issue, :title) %></dd>
+                      <% end %>
+                      <dt><%= translated("translationValidation.field.languages") %></dt>
+                      <dd><%= translation_issue_languages(issue) %></dd>
+                      <%= if translation_issue_value(issue, :file_path) do %>
+                        <dt><%= translated("translationValidation.field.filePath") %></dt>
+                        <dd><%= translation_issue_value(issue, :file_path) %></dd>
+                      <% end %>
+                    </dl>
+                  </article>
+                <% end %>
+              </div>
+            <% end %>
+          </section>
+
+          <div class="translation-validation-actions">
+            <button class="secondary" type="button" phx-click="rerun_misc_editor" data-testid="translation-validation-revalidate"><%= translated("translationValidation.revalidate") %></button>
+            <button class="primary" type="button" phx-click="fix_translation_validation" data-testid="translation-validation-fix" disabled={not @misc_editor.can_fix?}><%= translated("translationValidation.fix") %></button>
+          </div>
         </div>
 
       <% :find_duplicates -> %>
@@ -202,7 +271,35 @@
         </div>
 
       <% :git_diff -> %>
-        <div class="misc-card misc-code-card"><pre><code><%= @misc_editor.diff_text %></code></pre></div>
+        <div class="git-diff-view">
+          <%= if @misc_editor.files == [] do %>
+            <p class="git-diff-empty"><%= @misc_editor.empty_message %></p>
+          <% else %>
+            <form class="git-diff-toolbar" phx-change="select_git_diff_file">
+              <label for="git-diff-file-select"><%= translated("gitDiff.changedFiles") %></label>
+              <select id="git-diff-file-select" data-testid="git-diff-file-select" name="path">
+                <%= for file_path <- @misc_editor.files do %>
+                  <option value={file_path} selected={file_path == @misc_editor.selected_file_path}><%= file_path %></option>
+                <% end %>
+              </select>
+            </form>
+
+            <div
+              id={"git-diff-editor-#{String.replace(@misc_editor.active_diff.file_path || "working-tree", ~r/[^a-zA-Z0-9_-]+/, "-")}"}
+              class="git-diff-editor"
+              phx-hook="MonacoDiffEditor"
+              data-monaco-diff-language={@misc_editor.active_diff.language}
+              data-monaco-diff-file-path={@misc_editor.active_diff.file_path}
+              data-monaco-diff-view-style={@misc_editor.preferences.view_style}
+              data-monaco-diff-word-wrap={if(@misc_editor.preferences.word_wrap, do: "on", else: "off")}
+              data-monaco-diff-hide-unchanged={if(@misc_editor.preferences.hide_unchanged_regions, do: "true", else: "false")}
+            >
+              <textarea class="monaco-diff-original" hidden><%= @misc_editor.active_diff.original %></textarea>
+              <textarea class="monaco-diff-modified" hidden><%= @misc_editor.active_diff.modified %></textarea>
+              <div class="monaco-diff-editor-instance"></div>
+            </div>
+          <% end %>
+        </div>
     <% end %>
   </div>
 </div>

lib/bds/git.ex

@@ -87,6 +87,27 @@ defmodule BDS.Git do
     end
   end
 
+  def get_diff_content(project_id, file_path, opts \\ [])
+      when is_binary(project_id) and is_binary(file_path) and is_list(opts) do
+    with {:ok, project_dir} <- project_dir(project_id) do
+      runner = Keyword.get(opts, :runner, &system_runner/3)
+
+      original =
+        case runner.("git", ["show", "HEAD:#{file_path}"], command_opts(project_dir)) do
+          {output, 0} -> output
+          {_output, _status} -> ""
+        end
+
+      modified =
+        case File.read(Path.join(project_dir, file_path)) do
+          {:ok, contents} -> contents
+          {:error, _reason} -> ""
+        end
+
+      {:ok, %{file_path: file_path, original: original, modified: modified}}
+    end
+  end
+
   def history(project_id, branch, opts \\ [])
       when is_binary(project_id) and is_binary(branch) and is_list(opts) do
     with {:ok, project_dir} <- project_dir(project_id),

lib/bds/posts.ex

@@ -580,72 +580,119 @@ defmodule BDS.Posts do
   end
 
   def validate_translations(project_id, opts \\ []) do
+    project = Projects.get_project!(project_id)
     {:ok, metadata} = Metadata.get_project_metadata(project_id)
     on_progress = progress_callback(opts)
 
-    posts =
+    source_posts =
       Repo.all(
         from post in Post,
-          where: post.project_id == ^project_id and post.status == :published,
+          where: post.project_id == ^project_id,
           order_by: [asc: post.created_at, asc: post.slug]
       )
 
-        total_posts = length(posts)
-        :ok = report_rebuild_started(on_progress, total_posts, "published posts")
+    source_post_map = Map.new(source_posts, &{&1.id, &1})
 
-    translation_languages =
+    translation_rows =
       Repo.all(
         from translation in Translation,
-          join: post in Post,
-          on: post.id == translation.translation_for,
-          where: post.project_id == ^project_id,
-          select: {translation.translation_for, translation.language}
+          where: translation.project_id == ^project_id,
+          order_by: [asc: translation.translation_for, asc: translation.language, asc: translation.id]
       )
-      |> Enum.group_by(fn {post_id, _language} -> post_id end, fn {_post_id, language} ->
-        language
-      end)
 
-    required_languages =
-      metadata.blog_languages
-      |> Enum.map(&normalize_language/1)
-      |> Enum.reject(&(&1 == normalize_language(metadata.main_language)))
-      |> Enum.uniq()
-      |> Enum.sort()
+    project_data_dir = Projects.project_data_dir(project)
 
-    missing =
-      posts
+    markdown_files =
+      project_data_dir
+      |> Path.join("posts")
+      |> list_markdown_files_recursive()
+
+    total_items = length(translation_rows) + length(markdown_files)
+    :ok = report_rebuild_started(on_progress, total_items, "translations")
+
+    invalid_database_rows =
+      translation_rows
       |> Enum.with_index(1)
-      |> Enum.flat_map(fn {post, index} ->
-        available = Map.get(translation_languages, post.id, [])
+      |> Enum.flat_map(fn {translation, index} ->
+        :ok = report_rebuild_progress(on_progress, index, total_items, "translations")
 
-        :ok = report_rebuild_progress(on_progress, index, total_posts, "published posts")
+        case invalid_database_translation_issue(translation, source_post_map, metadata) do
+          nil -> []
+          issue -> [issue]
+        end
+      end)
+      |> Enum.sort_by(&translation_validation_issue_sort_key/1)
 
-        cond do
-          post.do_not_translate ->
-            []
+    {checked_filesystem_file_count, invalid_filesystem_files} =
+      markdown_files
+      |> Enum.with_index(length(translation_rows) + 1)
+      |> Enum.reduce({0, []}, fn {file_path, index}, {count, issues} ->
+        :ok = report_rebuild_progress(on_progress, index, total_items, "translations")
 
-          true ->
-            required_languages
-            |> Enum.reject(&(&1 in available))
-            |> Enum.map(&%{post_id: post.id, language: &1})
+        case invalid_filesystem_translation_issue(file_path, source_post_map, metadata) do
+          {:ok, nil} ->
+            {count + 1, issues}
+
+          {:ok, issue} ->
+            {count + 1, [issue | issues]}
+
+          :skip ->
+            {count, issues}
         end
       end)
 
-    do_not_translate_posts =
-      posts
-      |> Enum.filter(& &1.do_not_translate)
-      |> Enum.map(& &1.id)
-
-    orphan_files = orphan_translation_files(project_id)
+     missing = legacy_missing_translation_entries(source_posts, translation_rows, metadata)
+     orphan_files = legacy_orphan_translation_files(invalid_filesystem_files, project_data_dir)
+     do_not_translate_posts = legacy_do_not_translate_posts(source_posts)
 
     {:ok,
      %{
+       checked_database_row_count: length(translation_rows),
+       checked_filesystem_file_count: checked_filesystem_file_count,
+       invalid_database_rows: invalid_database_rows,
+       invalid_filesystem_files: Enum.reverse(invalid_filesystem_files) |> Enum.sort_by(&translation_validation_issue_sort_key/1),
        missing: missing,
        orphan_files: orphan_files,
        do_not_translate_posts: do_not_translate_posts
      }}
   end
 
+  def fix_invalid_translations(report) when is_map(report) do
+    normalized_report = normalize_translation_validation_report(report)
+
+    {deleted_database_rows, flushed_translations, synced_post_ids} =
+      Enum.reduce(normalized_report.invalid_database_rows, {0, 0, MapSet.new()}, fn issue, {deleted, flushed, synced_ids} ->
+        case fix_invalid_database_translation(issue) do
+          {:deleted, post_id} ->
+            {deleted + 1, flushed, maybe_put_synced_post(synced_ids, post_id)}
+
+          {:flushed, post_id} ->
+            {deleted, flushed + 1, maybe_put_synced_post(synced_ids, post_id)}
+
+          :noop ->
+            {deleted, flushed, synced_ids}
+        end
+      end)
+
+    deleted_files =
+      Enum.reduce(normalized_report.invalid_filesystem_files, 0, fn issue, count ->
+        if delete_translation_validation_file(issue.file_path) do
+          count + 1
+        else
+          count
+        end
+      end)
+
+    Enum.each(synced_post_ids, &Search.sync_post/1)
+
+    {:ok,
+     %{
+       deleted_database_rows: deleted_database_rows,
+       deleted_files: deleted_files,
+       flushed_translations: flushed_translations
+     }}
+  end
+
   def rewrite_published_post(post_id) do
     post = Repo.get!(Post, post_id)
 
@@ -997,6 +1044,12 @@ defmodule BDS.Posts do
   end
 
   defp normalize_translation_updates(post, %Translation{} = translation, language, attrs, now) do
+    requested_status =
+      case attr(attrs, :status) do
+        nil -> nil
+        status -> parse_translation_status(status)
+      end
+
     updates =
       %{}
       |> maybe_put(:title, attr(attrs, :title))
@@ -1006,6 +1059,8 @@ defmodule BDS.Posts do
     reopened? =
       translation.status == :published and translation_content_change?(translation, updates)
 
+    status = if(reopened?, do: :draft, else: requested_status || translation.status || :draft)
+
     %{
       id: translation.id || Ecto.UUID.generate(),
       project_id: post.project_id,
@@ -1014,10 +1069,10 @@ defmodule BDS.Posts do
       title: Map.get(updates, :title, translation.title),
       excerpt: Map.get(updates, :excerpt, translation.excerpt),
       content: Map.get(updates, :content, translation.content),
-      status: if(reopened?, do: :draft, else: translation.status || :draft),
+      status: status,
       created_at: translation.created_at || now,
       updated_at: now,
-      published_at: translation.published_at,
+      published_at: translation.published_at || if(status == :published, do: now, else: nil),
       file_path: translation.file_path || "",
       checksum: translation.checksum
     }
@@ -1303,32 +1358,290 @@ defmodule BDS.Posts do
   defp present?(value) when is_binary(value), do: String.trim(value) != ""
   defp present?(value), do: not is_nil(value)
 
-  defp orphan_translation_files(project_id) do
-    project = Projects.get_project!(project_id)
-
-    translation_paths =
-      MapSet.new(
-        Repo.all(
-          from translation in Translation,
-            where: translation.project_id == ^project_id,
-            select: translation.file_path
-        )
-      )
-
-    project
-    |> Projects.project_data_dir()
-    |> Path.join("posts")
-    |> list_matching_files("*.md")
-    |> Enum.map(&Path.relative_to(&1, Projects.project_data_dir(project)))
-    |> Enum.filter(&translation_file?/1)
-    |> Enum.reject(&MapSet.member?(translation_paths, &1))
+  defp list_markdown_files_recursive(dir) do
+    ["*.md", "*.markdown", "*.mdx"]
+    |> Enum.flat_map(&list_matching_files(dir, &1))
+    |> Enum.uniq()
     |> Enum.sort()
   end
 
-  defp translation_file?(relative_path) do
-    Regex.match?(~r/\.[a-z]{2}\.md$/i, relative_path)
+  defp invalid_database_translation_issue(%Translation{} = translation, source_post_map, metadata) do
+    source_post = Map.get(source_post_map, translation.translation_for)
+    normalized_language = normalize_language(translation.language)
+
+    cond do
+      is_nil(source_post) ->
+        translation_validation_issue(%{
+          issue: "missing-source-post",
+          translation_id: translation.id,
+          translation_for: translation.translation_for,
+          translation_language: normalized_language,
+          title: translation.title,
+          file_path: blank_to_nil(translation.file_path)
+        })
+
+      canonical_translation_language?(source_post, normalized_language, metadata) ->
+        translation_validation_issue(%{
+          issue: "same-language-as-canonical",
+          translation_id: translation.id,
+          translation_for: translation.translation_for,
+          canonical_language: canonical_translation_language(source_post, metadata),
+          translation_language: normalized_language,
+          title: translation.title,
+          file_path: blank_to_nil(translation.file_path)
+        })
+
+      source_post.do_not_translate ->
+        translation_validation_issue(%{
+          issue: "do-not-translate-has-translations",
+          translation_id: translation.id,
+          translation_for: translation.translation_for,
+          translation_language: normalized_language,
+          title: translation.title,
+          file_path: blank_to_nil(translation.file_path)
+        })
+
+      translation.status == :published and present?(translation.content) ->
+        translation_validation_issue(%{
+          issue: "content-in-database",
+          translation_id: translation.id,
+          translation_for: translation.translation_for,
+          translation_language: normalized_language,
+          title: translation.title,
+          file_path: blank_to_nil(translation.file_path)
+        })
+
+      true ->
+        nil
+    end
   end
 
+  defp invalid_filesystem_translation_issue(file_path, source_post_map, metadata) do
+    with {:ok, contents} <- File.read(file_path),
+         {:ok, %{fields: fields}} <- Frontmatter.parse_document(contents),
+         true <- translation_rebuild_file?(%{fields: fields}) do
+      translation_for = DocumentFields.get(fields, "translationFor")
+      source_post = Map.get(source_post_map, translation_for)
+      normalized_language = normalize_language(DocumentFields.get(fields, "language"))
+      title = DocumentFields.get(fields, "title")
+
+      issue =
+        cond do
+          is_nil(source_post) ->
+            translation_validation_issue(%{
+              issue: "missing-source-post",
+              translation_for: translation_for,
+              translation_language: normalized_language,
+              title: title,
+              file_path: file_path
+            })
+
+          canonical_translation_language?(source_post, normalized_language, metadata) ->
+            translation_validation_issue(%{
+              issue: "same-language-as-canonical",
+              translation_for: translation_for,
+              canonical_language: canonical_translation_language(source_post, metadata),
+              translation_language: normalized_language,
+              title: title,
+              file_path: file_path
+            })
+
+          source_post.do_not_translate ->
+            translation_validation_issue(%{
+              issue: "do-not-translate-has-translations",
+              translation_for: translation_for,
+              translation_language: normalized_language,
+              title: title,
+              file_path: file_path
+            })
+
+          true ->
+            nil
+        end
+
+      {:ok, issue}
+    else
+      false -> :skip
+      _other -> :skip
+    end
+  end
+
+  defp normalize_translation_validation_report(report) do
+    %{
+      checked_database_row_count: map_value(report, :checked_database_row_count, 0),
+      checked_filesystem_file_count: map_value(report, :checked_filesystem_file_count, 0),
+      invalid_database_rows:
+        report
+        |> map_value(:invalid_database_rows, [])
+        |> Enum.map(&normalize_translation_validation_issue/1),
+      invalid_filesystem_files:
+        report
+        |> map_value(:invalid_filesystem_files, [])
+        |> Enum.map(&normalize_translation_validation_issue/1)
+    }
+  end
+
+  defp legacy_missing_translation_entries(source_posts, translation_rows, metadata) do
+    configured_languages =
+      ([Map.get(metadata, :main_language)] ++ Map.get(metadata, :blog_languages, []))
+      |> Enum.map(&normalize_language/1)
+      |> Enum.reject(&(&1 in [nil, ""]))
+      |> Enum.uniq()
+
+    existing_languages_by_post =
+      Enum.reduce(translation_rows, %{}, fn translation, acc ->
+        Map.update(
+          acc,
+          translation.translation_for,
+          MapSet.new([normalize_language(translation.language)]),
+          &MapSet.put(&1, normalize_language(translation.language))
+        )
+      end)
+
+    source_posts
+    |> Enum.filter(&(&1.status == :published and not &1.do_not_translate))
+    |> Enum.flat_map(fn post ->
+      canonical_language = canonical_translation_language(post, metadata)
+      existing_languages = Map.get(existing_languages_by_post, post.id, MapSet.new())
+
+      configured_languages
+      |> Enum.reject(&(&1 == canonical_language or MapSet.member?(existing_languages, &1)))
+      |> Enum.map(&%{post_id: post.id, language: &1})
+    end)
+    |> Enum.sort_by(&{&1.post_id, &1.language})
+  end
+
+  defp legacy_orphan_translation_files(invalid_filesystem_files, project_data_dir) do
+    invalid_filesystem_files
+    |> Enum.filter(&(Map.get(&1, :issue) == "missing-source-post"))
+    |> Enum.map(fn issue ->
+      issue
+      |> Map.get(:file_path)
+      |> relative_project_data_path(project_data_dir)
+    end)
+    |> Enum.reject(&is_nil/1)
+    |> Enum.sort()
+  end
+
+  defp legacy_do_not_translate_posts(source_posts) do
+    source_posts
+    |> Enum.filter(&(&1.status == :published and &1.do_not_translate))
+    |> Enum.map(& &1.id)
+    |> Enum.sort()
+  end
+
+  defp normalize_translation_validation_issue(issue) when is_map(issue) do
+    %{
+      issue: map_value(issue, :issue),
+      translation_id: blank_to_nil(map_value(issue, :translation_id)),
+      translation_for: map_value(issue, :translation_for),
+      canonical_language: blank_to_nil(map_value(issue, :canonical_language)),
+      translation_language: map_value(issue, :translation_language),
+      title: blank_to_nil(map_value(issue, :title)),
+      file_path: blank_to_nil(map_value(issue, :file_path))
+    }
+  end
+
+  defp fix_invalid_database_translation(%{issue: "content-in-database", translation_id: translation_id})
+       when is_binary(translation_id) do
+    case Repo.get(Translation, translation_id) do
+      %Translation{} = translation ->
+        case Repo.get(Post, translation.translation_for) do
+          %Post{} = post ->
+            :ok = publish_translation(post, translation)
+            {:flushed, translation.translation_for}
+
+          nil ->
+            :noop
+        end
+
+      nil ->
+        :noop
+    end
+  end
+
+  defp fix_invalid_database_translation(%{translation_id: translation_id, translation_for: translation_for})
+       when is_binary(translation_id) do
+    case Repo.get(Translation, translation_id) do
+      %Translation{} = translation ->
+        Repo.delete!(translation)
+        {:deleted, translation_for}
+
+      nil ->
+        :noop
+    end
+  end
+
+  defp fix_invalid_database_translation(_issue), do: :noop
+
+  defp delete_translation_validation_file(file_path) when file_path in [nil, ""], do: false
+
+  defp delete_translation_validation_file(file_path) do
+    case File.rm(file_path) do
+      :ok -> true
+      {:error, :enoent} -> false
+      {:error, _reason} -> false
+    end
+  end
+
+  defp translation_validation_issue(attrs) do
+    %{
+      issue: Map.get(attrs, :issue),
+      translation_id: Map.get(attrs, :translation_id),
+      translation_for: Map.get(attrs, :translation_for),
+      canonical_language: Map.get(attrs, :canonical_language),
+      translation_language: Map.get(attrs, :translation_language),
+      title: Map.get(attrs, :title),
+      file_path: Map.get(attrs, :file_path)
+    }
+  end
+
+  defp translation_validation_issue_sort_key(issue) do
+    [Map.get(issue, :translation_for), Map.get(issue, :translation_id), Map.get(issue, :file_path)]
+    |> Enum.map(&to_string(&1 || ""))
+    |> Enum.join(":")
+  end
+
+  defp canonical_translation_language(source_post, metadata) do
+    language = normalize_language(source_post.language)
+
+    if language == "" do
+      normalize_language(Map.get(metadata, :main_language))
+    else
+      language
+    end
+  end
+
+  defp canonical_translation_language?(source_post, language, metadata) do
+    canonical_language = canonical_translation_language(source_post, metadata)
+    canonical_language != "" and canonical_language == normalize_language(language)
+  end
+
+  defp map_value(map, key, default \\ nil) when is_map(map) do
+    Map.get(map, key, Map.get(map, Atom.to_string(key), default))
+  end
+
+  defp blank_to_nil(value) when is_binary(value) do
+    case String.trim(value) do
+      "" -> nil
+      trimmed -> trimmed
+    end
+  end
+
+  defp blank_to_nil(value), do: value
+
+  defp relative_project_data_path(nil, _project_data_dir), do: nil
+
+  defp relative_project_data_path(file_path, project_data_dir) do
+    case Path.relative_to(file_path, project_data_dir) do
+      relative_path when relative_path == file_path -> file_path
+      relative_path -> relative_path
+    end
+  end
+
+  defp maybe_put_synced_post(set, post_id) when is_binary(post_id) and post_id != "", do: MapSet.put(set, post_id)
+  defp maybe_put_synced_post(set, _post_id), do: set
+
   defp normalize_language(nil), do: ""
 
   defp normalize_language(language) do