diff --git a/PLAN.md b/PLAN.md index f04097b..7d55358 100644 --- a/PLAN.md +++ b/PLAN.md @@ -83,8 +83,8 @@ Only these two audit tracks matter for the current pass. The follow-on missing-f 11. Restore desktop-side CLI mutation watching parity. Completed 2026-04-29. A supervised CLI-sync watcher now polls the persisted notification store on the old-app timing budget, broadcasts `entity:changed` events through PubSub, and the LiveView shell refreshes sidebar/editor state while closing stale post/media tabs on external deletes. -12. Restore import execution and editor parity. - Extend the existing stored import definitions into the old WXR analysis/execution pipeline and add the dedicated editor surface so import behavior, workflow, and look and feel match the old app. +12. Restore import execution and editor parity. Completed 2026-04-29. + The stored import-definition flow now runs through the old analysis/execution pipeline again with progress callbacks, dedicated import-editor detail sections, inline taxonomy mapping pills plus AI-backed mapping, and focused import proof plus clean compile, dialyzer, and full-suite validation. ## Batch 3 Audit Matrix diff --git a/lib/bds/ai.ex b/lib/bds/ai.ex index 91050d8..1f7998d 100644 --- a/lib/bds/ai.ex +++ b/lib/bds/ai.ex @@ -212,6 +212,40 @@ defmodule BDS.AI do end end + def analyze_import_taxonomy(import_terms, existing_terms, opts \\ []) + when is_map(import_terms) and is_map(existing_terms) and is_list(opts) do + payload = %{ + import_categories: normalize_string_list(Map.get(import_terms, :categories) || Map.get(import_terms, "categories")), + import_tags: normalize_string_list(Map.get(import_terms, :tags) || Map.get(import_terms, "tags")), + existing_categories: normalize_string_list(Map.get(existing_terms, :categories) || Map.get(existing_terms, "categories")), + existing_tags: normalize_string_list(Map.get(existing_terms, :tags) || Map.get(existing_terms, "tags")) + } + + run_one_shot( + :import_taxonomy_mapping, + payload, + opts, + fn json, usage -> + {:ok, + %{ + category_mappings: + filter_taxonomy_mapping_response( + json["categoryMappings"] || json["category_mappings"], + payload.import_categories, + payload.existing_categories + ), + tag_mappings: + filter_taxonomy_mapping_response( + json["tagMappings"] || json["tag_mappings"], + payload.import_tags, + payload.existing_tags + ), + usage: usage + }} + end + ) + end + def analyze_post(post_input, opts \\ []) when is_list(opts) do with {:ok, post} <- normalize_post_input(post_input) do run_one_shot( @@ -559,7 +593,7 @@ defmodule BDS.AI do defp run_one_shot(operation, payload, opts, formatter) do runtime = Keyword.get(opts, :runtime, OpenAICompatibleRuntime) - with {:ok, endpoint, model, mode} <- resolve_runtime_target(operation, secret_backend: Keyword.get(opts, :secret_backend, SecretBackend)), + with {:ok, endpoint, model, mode} <- resolve_runtime_target(operation, opts), :ok <- validate_runtime_target(operation, model, mode), request <- build_one_shot_request(operation, payload, model), {:ok, response} <- runtime.generate(endpoint_with_model(endpoint, model), request, opts), @@ -903,20 +937,20 @@ defmodule BDS.AI do {:ok, get_model_preference_value(:chat) || endpoint.model} end - defp resolve_model_for_operation(:analyze_image, :airplane, endpoint, _extra) do - {:ok, get_model_preference_value(:airplane_image_analysis) || endpoint.model} + defp resolve_model_for_operation(:analyze_image, :airplane, endpoint, extra) do + {:ok, Keyword.get(extra, :model) || get_model_preference_value(:airplane_image_analysis) || endpoint.model} end - defp resolve_model_for_operation(:analyze_image, :online, endpoint, _extra) do - {:ok, get_model_preference_value(:image_analysis) || endpoint.model} + defp resolve_model_for_operation(:analyze_image, :online, endpoint, extra) do + {:ok, Keyword.get(extra, :model) || get_model_preference_value(:image_analysis) || endpoint.model} end - defp resolve_model_for_operation(_operation, :airplane, endpoint, _extra) do - {:ok, get_model_preference_value(:airplane_title) || endpoint.model} + defp resolve_model_for_operation(_operation, :airplane, endpoint, extra) do + {:ok, Keyword.get(extra, :model) || get_model_preference_value(:airplane_title) || endpoint.model} end - defp resolve_model_for_operation(_operation, :online, endpoint, _extra) do - {:ok, get_model_preference_value(:title) || endpoint.model} + defp resolve_model_for_operation(_operation, :online, endpoint, extra) do + {:ok, Keyword.get(extra, :model) || get_model_preference_value(:title) || endpoint.model} end defp validate_runtime_target(:analyze_image, model, _mode) do @@ -990,6 +1024,49 @@ defmodule BDS.AI do |> MapSet.size() end + defp normalize_string_list(values) do + values + |> List.wrap() + |> Enum.map(&to_string/1) + |> Enum.map(&String.trim/1) + |> Enum.reject(&(&1 == "")) + |> Enum.uniq() + end + + defp filter_taxonomy_mapping_response(mappings, import_terms, existing_terms) when is_map(mappings) do + import_lookup = canonical_term_lookup(import_terms) + existing_lookup = canonical_term_lookup(existing_terms) + + Enum.reduce(mappings, %{}, fn {source, target}, acc -> + with {:ok, canonical_source} <- resolve_canonical_term(source, import_lookup), + {:ok, canonical_target} <- resolve_canonical_term(target, existing_lookup) do + Map.put(acc, canonical_source, canonical_target) + else + _other -> acc + end + end) + end + + defp filter_taxonomy_mapping_response(_mappings, _import_terms, _existing_terms), do: %{} + + defp canonical_term_lookup(terms) do + Map.new(terms, fn term -> {normalize_term(term), term} end) + end + + defp resolve_canonical_term(term, lookup) do + case Map.get(lookup, normalize_term(term)) do + nil -> :error + canonical -> {:ok, canonical} + end + end + + defp normalize_term(term) do + term + |> to_string() + |> String.trim() + |> String.downcase() + end + defp one_shot_system_prompt(:detect_language) do "Return JSON with exactly one key: language_code." end @@ -998,6 +1075,10 @@ defmodule BDS.AI do "Return JSON with keys tags and categories, each an array of short strings." end + defp one_shot_system_prompt(:import_taxonomy_mapping) do + "You are helping import WordPress taxonomy into an existing blog. Return JSON with exactly two keys: categoryMappings and tagMappings. Each value must be an object mapping imported term names to existing project term names. Only map when the imported term should reuse an existing term to avoid duplicates. Do not invent target terms. Leave unmapped items out of the objects." + end + defp one_shot_system_prompt(:analyze_post) do "Return JSON with keys title, excerpt, and slug." end @@ -1022,6 +1103,27 @@ defmodule BDS.AI do "Suggest categories and tags for the following post.\nTitle: #{post.title}\nExcerpt: #{post.excerpt}\nContent: #{truncate_text(post.content, 2000)}" end + defp one_shot_user_content(:import_taxonomy_mapping, payload) do + [ + "Analyze these imported taxonomy terms and suggest which ones should map to existing project terms.", + "", + "Imported categories:", + Enum.join(payload.import_categories, ", "), + "", + "Imported tags:", + Enum.join(payload.import_tags, ", "), + "", + "Existing project categories:", + Enum.join(payload.existing_categories, ", "), + "", + "Existing project tags:", + Enum.join(payload.existing_tags, ", "), + "", + "Return JSON only." + ] + |> Enum.join("\n") + end + defp one_shot_user_content(:analyze_post, post) do "Suggest an improved title, excerpt, and slug.\nTitle: #{post.title}\nExcerpt: #{post.excerpt}\nContent: #{truncate_text(post.content, 2000)}" end diff --git a/lib/bds/desktop/shell_live.ex b/lib/bds/desktop/shell_live.ex index 07e6b8f..5e81dda 100644 --- a/lib/bds/desktop/shell_live.ex +++ b/lib/bds/desktop/shell_live.ex @@ -105,8 +105,12 @@ defmodule BDS.Desktop.ShellLive do |> assign(:chat_editor_surface_data, %{}) |> assign(:chat_editor_surface_tabs, %{}) |> assign(:chat_editor_action_errors, %{}) + |> assign(:import_editor_analysis_states, %{}) + |> assign(:import_editor_analysis_task_refs, %{}) |> assign(:import_editor_execution_states, %{}) + |> assign(:import_editor_execution_task_refs, %{}) |> assign(:import_editor_sections, %{}) + |> assign(:import_editor_taxonomy_edits, %{}) |> assign(:import_editor_model_selectors_open, %{}) |> assign(:import_editor_selected_models, %{}) |> assign(:misc_editor_selected_pairs, %{}) @@ -791,8 +795,20 @@ defmodule BDS.Desktop.ShellLive do {:noreply, ImportEditor.change_conflict_resolution(socket, params, &reload_shell/2)} end - def handle_event("change_import_taxonomy_mapping", params, socket) do - {:noreply, ImportEditor.change_taxonomy_mapping(socket, params, &reload_shell/2)} + def handle_event("start_import_taxonomy_edit", params, socket) do + {:noreply, ImportEditor.start_taxonomy_edit(socket, params, &reload_shell/2)} + end + + def handle_event("save_import_taxonomy_edit", params, socket) do + {:noreply, ImportEditor.save_taxonomy_edit(socket, params, &reload_shell/2)} + end + + def handle_event("cancel_import_taxonomy_edit", _params, socket) do + {:noreply, ImportEditor.cancel_taxonomy_edit(socket, &reload_shell/2)} + end + + def handle_event("clear_import_taxonomy_mapping", params, socket) do + {:noreply, ImportEditor.clear_taxonomy_mapping(socket, params, &reload_shell/2)} end def handle_event("toggle_import_section", %{"section" => section}, socket) do @@ -1184,19 +1200,46 @@ defmodule BDS.Desktop.ShellLive do @impl true def handle_info({ref, result}, socket) when is_reference(ref) do Process.demonitor(ref, [:flush]) - {:noreply, ChatEditor.finish_request(socket, ref, result, &reload_shell/2, &append_output_entry/5)} + + cond do + Map.has_key?(socket.assigns.import_editor_analysis_task_refs, ref) -> + {:noreply, ImportEditor.finish_analysis(socket, ref, result, &reload_shell/2, &append_output_entry/5)} + + Map.has_key?(socket.assigns.import_editor_execution_task_refs, ref) -> + {:noreply, ImportEditor.finish_execution(socket, ref, result, &reload_shell/2, &append_output_entry/5)} + + true -> + {:noreply, ChatEditor.finish_request(socket, ref, result, &reload_shell/2, &append_output_entry/5)} + end end def handle_info({:DOWN, ref, :process, _pid, reason}, socket) when is_reference(ref) do next_socket = - case reason do - :normal -> socket - _other -> ChatEditor.finish_request(socket, ref, {:error, :cancelled}, &reload_shell/2, &append_output_entry/5) + cond do + Map.has_key?(socket.assigns.import_editor_analysis_task_refs, ref) -> + ImportEditor.handle_task_down(socket, :analysis, ref, reason, &reload_shell/2, &append_output_entry/5) + + Map.has_key?(socket.assigns.import_editor_execution_task_refs, ref) -> + ImportEditor.handle_task_down(socket, :execution, ref, reason, &reload_shell/2, &append_output_entry/5) + + true -> + case reason do + :normal -> socket + _other -> ChatEditor.finish_request(socket, ref, {:error, :cancelled}, &reload_shell/2, &append_output_entry/5) + end end {:noreply, next_socket} end + def handle_info({:import_analysis_progress, definition_id, step, detail}, socket) do + {:noreply, ImportEditor.note_analysis_progress(socket, definition_id, step, detail, &reload_shell/2)} + end + + def handle_info({:import_execution_progress, definition_id, phase, current, total, detail}, socket) do + {:noreply, ImportEditor.note_execution_progress(socket, definition_id, phase, current, total, detail, &reload_shell/2)} + end + def handle_info({:chat_tool_call, conversation_id, tool_call}, socket) do {:noreply, ChatEditor.note_tool_call(socket, conversation_id, tool_call, &reload_shell/2)} end diff --git a/lib/bds/desktop/shell_live/import_editor.ex b/lib/bds/desktop/shell_live/import_editor.ex index c96920c..0c32144 100644 --- a/lib/bds/desktop/shell_live/import_editor.ex +++ b/lib/bds/desktop/shell_live/import_editor.ex @@ -15,7 +15,8 @@ defmodule BDS.Desktop.ShellLive.ImportEditor do definition -> report = ImportDefinitions.decode_analysis_result(definition) - existing_terms = socket.assigns.projects.active_project_id |> Tags.list_tags() |> Enum.map(& &1.name) + taxonomy_terms = existing_taxonomy_terms(socket.assigns.projects.active_project_id) + analysis_state = Map.get(socket.assigns.import_editor_analysis_states, definition.id, default_analysis_state()) execution_state = Map.get(socket.assigns.import_editor_execution_states, definition.id, default_execution_state()) sections = Map.get(socket.assigns.import_editor_sections, definition.id, default_sections()) selected_model = selected_model(socket.assigns, definition.id) @@ -27,7 +28,9 @@ defmodule BDS.Desktop.ShellLive.ImportEditor do uploads_folder_path: definition.uploads_folder_path, wxr_file_path: definition.wxr_file_path, report: report, - existing_terms: existing_terms, + taxonomy_terms: taxonomy_terms, + taxonomy_edit: Map.get(socket.assigns.import_editor_taxonomy_edits, definition.id), + analysis_state: analysis_state, execution_state: execution_state, importable_counts: importable_counts(report), sections: sections, @@ -36,7 +39,7 @@ defmodule BDS.Desktop.ShellLive.ImportEditor do model_selector_open?: Map.get(socket.assigns.import_editor_model_selectors_open, definition.id, false), available_models: available_models, offline?: Map.get(socket.assigns, :offline_mode, true), - is_loading: false + is_loading: analysis_state.loading } socket @@ -91,24 +94,39 @@ defmodule BDS.Desktop.ShellLive.ImportEditor do {:ok, wxr_file_path} -> project_id = socket.assigns.projects.active_project_id - case ImportAnalysis.analyze_wxr(project_id, wxr_file_path, definition.uploads_folder_path) do - {:ok, report} -> - {:ok, _definition} = - ImportDefinitions.update_definition(definition_id, %{ - wxr_file_path: wxr_file_path, - last_analysis_result: report - }) + {:ok, _definition} = + ImportDefinitions.update_definition(definition_id, %{ + wxr_file_path: wxr_file_path, + last_analysis_result: nil + }) - socket - |> assign(:import_editor_execution_states, Map.delete(socket.assigns.import_editor_execution_states, definition_id)) - |> append_output.(translated("activity.import"), translated("importAnalysis.analyzingWxr"), Path.basename(wxr_file_path), "info") - |> reload.(socket.assigns.workbench) + live_view_pid = self() - {:error, %{message: message}} -> - socket - |> append_output.(translated("activity.import"), message, nil, "error") - |> reload.(socket.assigns.workbench) - end + task = + Task.Supervisor.async_nolink(BDS.Tasks.TaskSupervisor, fn -> + ImportAnalysis.analyze_wxr(project_id, wxr_file_path, definition.uploads_folder_path, + on_progress: fn step, detail -> + send(live_view_pid, {:import_analysis_progress, definition_id, step, detail}) + end + ) + end) + + :ok = allow_repo_sandbox(task.pid) + + socket + |> assign( + :import_editor_analysis_states, + Map.put(socket.assigns.import_editor_analysis_states, definition_id, %{ + loading: true, + step: translated("importAnalysis.analyzingWxr"), + detail: Path.basename(wxr_file_path), + file_path: wxr_file_path, + ref: task.ref + }) + ) + |> assign(:import_editor_analysis_task_refs, Map.put(socket.assigns.import_editor_analysis_task_refs, task.ref, definition_id)) + |> assign(:import_editor_execution_states, Map.delete(socket.assigns.import_editor_execution_states, definition_id)) + |> reload.(socket.assigns.workbench) :cancel -> reload.(socket, socket.assigns.workbench) @@ -123,30 +141,50 @@ defmodule BDS.Desktop.ShellLive.ImportEditor do end end - def execute_import(socket, reload, append_output) do + def execute_import(socket, reload, _append_output) do with %{id: definition_id} <- socket.assigns.current_tab, %{} = definition <- ImportDefinitions.get_definition(definition_id), %{} = report <- ImportDefinitions.decode_analysis_result(definition) do project_id = socket.assigns.projects.active_project_id default_author = default_author(project_id) + counts = importable_counts(report) - case ImportExecution.execute_import(project_id, report, - uploads_folder_path: definition.uploads_folder_path, - default_author: default_author - ) do - {:ok, result} -> - counts = importable_counts(report) + if counts.total == 0 do + reload.(socket, socket.assigns.workbench) + else + live_view_pid = self() - socket - |> assign(:import_editor_execution_states, Map.put(socket.assigns.import_editor_execution_states, definition_id, %{completed: true, error: nil, count: counts.total, result: result})) - |> append_output.(translated("activity.import"), translated("importAnalysis.importComplete", %{count: counts.total}), nil, "info") - |> reload.(socket.assigns.workbench) + task = + Task.Supervisor.async_nolink(BDS.Tasks.TaskSupervisor, fn -> + ImportExecution.execute_import(project_id, report, + uploads_folder_path: definition.uploads_folder_path, + default_author: default_author, + on_progress: fn phase, current, total, detail -> + send(live_view_pid, {:import_execution_progress, definition_id, phase, current, total, detail}) + end + ) + end) - {:error, %{message: message}} -> - socket - |> assign(:import_editor_execution_states, Map.put(socket.assigns.import_editor_execution_states, definition_id, %{completed: false, error: message, count: 0, result: nil})) - |> append_output.(translated("activity.import"), message, nil, "error") - |> reload.(socket.assigns.workbench) + :ok = allow_repo_sandbox(task.pid) + + socket + |> assign( + :import_editor_execution_states, + Map.put(socket.assigns.import_editor_execution_states, definition_id, %{ + is_executing: true, + completed: false, + error: nil, + count: counts.total, + result: nil, + phase: translated("importAnalysis.executionStarting"), + current: 0, + total: counts.total, + detail: nil, + ref: task.ref + }) + ) + |> assign(:import_editor_execution_task_refs, Map.put(socket.assigns.import_editor_execution_task_refs, task.ref, definition_id)) + |> reload.(socket.assigns.workbench) end else _other -> reload.(socket, socket.assigns.workbench) @@ -165,21 +203,55 @@ defmodule BDS.Desktop.ShellLive.ImportEditor do end end - def change_taxonomy_mapping(socket, %{"type" => type, "name" => name, "mapped_to" => mapped_to}, reload) do - with %{id: definition_id} <- socket.assigns.current_tab, - %{} = definition <- ImportDefinitions.get_definition(definition_id), - %{} = report <- ImportDefinitions.decode_analysis_result(definition), - updated_report <- update_taxonomy_mapping(report, type, name, mapped_to), - {:ok, _definition} <- ImportDefinitions.update_definition(definition_id, %{last_analysis_result: updated_report}) do - reload.(socket, socket.assigns.workbench) + def start_taxonomy_edit(socket, %{"type" => type, "name" => name, "mapped_to" => mapped_to}, reload) do + with %{id: definition_id} <- socket.assigns.current_tab do + socket + |> assign( + :import_editor_taxonomy_edits, + Map.put(socket.assigns.import_editor_taxonomy_edits, definition_id, %{ + type: type, + name: name, + value: mapped_to |> to_string() |> blank_to_nil() + }) + ) + |> reload.(socket.assigns.workbench) else _other -> reload.(socket, socket.assigns.workbench) end end + def cancel_taxonomy_edit(socket, reload) do + with %{id: definition_id} <- socket.assigns.current_tab do + socket + |> assign(:import_editor_taxonomy_edits, Map.delete(socket.assigns.import_editor_taxonomy_edits, definition_id)) + |> reload.(socket.assigns.workbench) + else + _other -> reload.(socket, socket.assigns.workbench) + end + end + + def save_taxonomy_edit(socket, %{"type" => type, "name" => name, "mapped_to" => mapped_to}, reload) do + with %{id: definition_id} <- socket.assigns.current_tab, + %{} = definition <- ImportDefinitions.get_definition(definition_id), + %{} = report <- ImportDefinitions.decode_analysis_result(definition), + normalized_value <- normalize_taxonomy_mapping_value(socket.assigns.projects.active_project_id, type, mapped_to), + updated_report <- update_taxonomy_mapping(report, type, name, normalized_value), + {:ok, _definition} <- ImportDefinitions.update_definition(definition_id, %{last_analysis_result: updated_report}) do + socket + |> assign(:import_editor_taxonomy_edits, Map.delete(socket.assigns.import_editor_taxonomy_edits, definition_id)) + |> reload.(socket.assigns.workbench) + else + _other -> reload.(socket, socket.assigns.workbench) + end + end + + def clear_taxonomy_mapping(socket, %{"type" => type, "name" => name}, reload) do + save_taxonomy_edit(socket, %{"type" => type, "name" => name, "mapped_to" => ""}, reload) + end + def toggle_section(socket, section, reload) do with %{id: definition_id} <- socket.assigns.current_tab, - section_key when section_key in ["conflicts", "taxonomy", "macros"] <- section do + section_key when section_key in ["post_conflicts", "page_conflicts", "posts", "other", "pages", "media", "taxonomy", "macros"] <- section do next_sections = socket.assigns.import_editor_sections |> Map.get(definition_id, default_sections()) @@ -227,28 +299,236 @@ defmodule BDS.Desktop.ShellLive.ImportEditor do |> reload.(socket.assigns.workbench) true -> - updated_report = auto_map_taxonomies(report, socket.assigns.projects.active_project_id |> Tags.list_tags() |> Enum.map(& &1.name)) - {:ok, _definition} = ImportDefinitions.update_definition(definition_id, %{last_analysis_result: updated_report}) - mapped_count = auto_mapped_count(report, updated_report) + taxonomy_terms = existing_taxonomy_terms(socket.assigns.projects.active_project_id) - socket - |> append_output.(translated("activity.import"), translated("importAnalysis.mappedCount", %{count: mapped_count}), Map.get(socket.assigns.import_editor_selected_models, definition_id), "info") - |> reload.(socket.assigns.workbench) + import_terms = %{ + categories: Enum.map(Map.get(report.items, :categories, []), & &1.name), + tags: Enum.map(Map.get(report.items, :tags, []), & &1.name) + } + + opts = maybe_put_option([], :model, Map.get(socket.assigns.import_editor_selected_models, definition_id)) + + case AI.analyze_import_taxonomy(import_terms, taxonomy_terms, opts) do + {:ok, analysis} -> + updated_report = apply_taxonomy_mappings(report, analysis) + {:ok, _definition} = ImportDefinitions.update_definition(definition_id, %{last_analysis_result: updated_report}) + mapped_count = auto_mapped_count(report, updated_report) + + socket + |> append_output.(translated("activity.import"), translated("importAnalysis.mappedCount", %{count: mapped_count}), Map.get(socket.assigns.import_editor_selected_models, definition_id), "info") + |> reload.(socket.assigns.workbench) + + {:error, reason} -> + socket + |> append_output.(translated("activity.import"), inspect(reason), Map.get(socket.assigns.import_editor_selected_models, definition_id), "error") + |> reload.(socket.assigns.workbench) + end end else _other -> reload.(socket, socket.assigns.workbench) end end + def note_analysis_progress(socket, definition_id, step, detail, reload) do + socket + |> assign( + :import_editor_analysis_states, + Map.update(socket.assigns.import_editor_analysis_states, definition_id, default_analysis_state(), fn state -> + state + |> Map.put(:loading, true) + |> Map.put(:step, step) + |> Map.put(:detail, detail) + end) + ) + |> reload.(socket.assigns.workbench) + end + + def note_execution_progress(socket, definition_id, phase, current, total, detail, reload) do + socket + |> assign( + :import_editor_execution_states, + Map.update(socket.assigns.import_editor_execution_states, definition_id, default_execution_state(), fn state -> + state + |> Map.put(:is_executing, true) + |> Map.put(:phase, phase) + |> Map.put(:current, current) + |> Map.put(:total, total) + |> Map.put(:detail, detail) + end) + ) + |> reload.(socket.assigns.workbench) + end + + def finish_analysis(socket, ref, result, reload, append_output) do + case Map.get(socket.assigns.import_editor_analysis_task_refs, ref) do + nil -> + socket + + definition_id -> + analysis_state = Map.get(socket.assigns.import_editor_analysis_states, definition_id, default_analysis_state()) + + socket = + socket + |> assign(:import_editor_analysis_task_refs, Map.delete(socket.assigns.import_editor_analysis_task_refs, ref)) + |> assign(:import_editor_analysis_states, Map.delete(socket.assigns.import_editor_analysis_states, definition_id)) + + case result do + {:ok, report} -> + attrs = + %{ + wxr_file_path: analysis_state.file_path, + last_analysis_result: report + } + |> maybe_put(:name, suggested_definition_name(report)) + + case ImportDefinitions.update_definition(definition_id, attrs) do + {:ok, _definition} -> reload.(socket, socket.assigns.workbench) + + {:error, reason} -> + socket + |> append_output.(translated("activity.import"), inspect(reason), nil, "error") + |> reload.(socket.assigns.workbench) + end + + {:error, %{message: message}} -> + socket + |> append_output.(translated("activity.import"), message, nil, "error") + |> reload.(socket.assigns.workbench) + + {:error, reason} -> + socket + |> append_output.(translated("activity.import"), inspect(reason), nil, "error") + |> reload.(socket.assigns.workbench) + end + end + end + + def finish_execution(socket, ref, result, reload, append_output) do + case Map.get(socket.assigns.import_editor_execution_task_refs, ref) do + nil -> + socket + + definition_id -> + previous_state = Map.get(socket.assigns.import_editor_execution_states, definition_id, default_execution_state()) + + socket = + socket + |> assign(:import_editor_execution_task_refs, Map.delete(socket.assigns.import_editor_execution_task_refs, ref)) + + case result do + {:ok, execution_result} -> + socket + |> assign( + :import_editor_execution_states, + Map.put(socket.assigns.import_editor_execution_states, definition_id, %{ + previous_state + | is_executing: false, + completed: true, + error: nil, + current: previous_state.total, + detail: nil, + result: execution_result, + ref: nil + }) + ) + |> append_output.(translated("activity.import"), translated("importAnalysis.importComplete", %{count: previous_state.count}), nil, "info") + |> reload.(socket.assigns.workbench) + + {:error, %{message: message}} -> + socket + |> assign( + :import_editor_execution_states, + Map.put(socket.assigns.import_editor_execution_states, definition_id, %{ + previous_state + | is_executing: false, + completed: false, + error: message, + ref: nil + }) + ) + |> append_output.(translated("activity.import"), message, nil, "error") + |> reload.(socket.assigns.workbench) + + {:error, reason} -> + message = inspect(reason) + + socket + |> assign( + :import_editor_execution_states, + Map.put(socket.assigns.import_editor_execution_states, definition_id, %{ + previous_state + | is_executing: false, + completed: false, + error: message, + ref: nil + }) + ) + |> append_output.(translated("activity.import"), message, nil, "error") + |> reload.(socket.assigns.workbench) + end + end + end + + def handle_task_down(socket, kind, ref, reason, reload, append_output) when reason not in [:normal, :shutdown] do + message = inspect(reason) + + case kind do + :analysis -> + case Map.get(socket.assigns.import_editor_analysis_task_refs, ref) do + nil -> socket + + definition_id -> + socket + |> assign(:import_editor_analysis_task_refs, Map.delete(socket.assigns.import_editor_analysis_task_refs, ref)) + |> assign(:import_editor_analysis_states, Map.delete(socket.assigns.import_editor_analysis_states, definition_id)) + |> append_output.(translated("activity.import"), message, nil, "error") + |> reload.(socket.assigns.workbench) + end + + :execution -> + case Map.get(socket.assigns.import_editor_execution_task_refs, ref) do + nil -> socket + + definition_id -> + previous_state = Map.get(socket.assigns.import_editor_execution_states, definition_id, default_execution_state()) + + socket + |> assign(:import_editor_execution_task_refs, Map.delete(socket.assigns.import_editor_execution_task_refs, ref)) + |> assign( + :import_editor_execution_states, + Map.put(socket.assigns.import_editor_execution_states, definition_id, %{ + previous_state + | is_executing: false, + completed: false, + error: message, + ref: nil + }) + ) + |> append_output.(translated("activity.import"), message, nil, "error") + |> reload.(socket.assigns.workbench) + end + end + end + + def handle_task_down(socket, _kind, _ref, _reason, _reload, _append_output), do: socket + attr :import_editor, :map, required: true def import_editor(assigns) do assigns = assigns |> assign(:report, Map.get(assigns.import_editor, :report)) + |> assign(:analysis_state, Map.get(assigns.import_editor, :analysis_state, default_analysis_state())) |> assign(:execution_state, Map.get(assigns.import_editor, :execution_state)) |> assign(:counts, Map.get(assigns.import_editor, :importable_counts, %{total: 0, tags: 0, posts: 0, media: 0, pages: 0})) |> assign(:sections, Map.get(assigns.import_editor, :sections, default_sections())) + |> assign(:detail_posts, detail_items(Map.get(assigns.import_editor, :report), :posts)) + |> assign(:detail_pages, detail_items(Map.get(assigns.import_editor, :report), :pages)) + |> assign(:detail_media, detail_items(Map.get(assigns.import_editor, :report), :media)) + |> assign(:post_conflicts, Enum.filter(detail_items(Map.get(assigns.import_editor, :report), :posts), &(&1.status == "conflict"))) + |> assign(:page_conflicts, Enum.filter(detail_items(Map.get(assigns.import_editor, :report), :pages), &(&1.status == "conflict"))) + |> assign(:post_items, Enum.filter(detail_items(Map.get(assigns.import_editor, :report), :posts), &(Map.get(&1, :post_type, "post") == "post"))) + |> assign(:other_items, Enum.reject(detail_items(Map.get(assigns.import_editor, :report), :posts), &(Map.get(&1, :post_type, "post") == "post"))) ~H"""
@@ -281,7 +561,19 @@ defmodule BDS.Desktop.ShellLive.ImportEditor do
- <%= if @report do %> + <%= if @import_editor.is_loading do %> +
+
+
+
<%= @analysis_state.step || translated("importAnalysis.analyzingWxr") %>
+ <%= if present?(@analysis_state.detail) do %> +
<%= @analysis_state.detail %>
+ <% end %> +
+
+ <% end %> + + <%= if not is_nil(@report) and not @import_editor.is_loading do %>
<%= translated("importAnalysis.site") %> @@ -326,23 +618,43 @@ defmodule BDS.Desktop.ShellLive.ImportEditor do
<% end %> -
-
- <%= translated("importAnalysis.readyToImport") %> - <%= if @counts.tags > 0 do %><%= @counts.tags %> <%= translated("importAnalysis.tagsCategories") %><% end %> - <%= if @counts.posts > 0 do %><%= @counts.posts %> <%= translated("importAnalysis.posts") %><% end %> - <%= if @counts.media > 0 do %><%= @counts.media %> <%= translated("importAnalysis.media") %><% end %> - <%= if @counts.pages > 0 do %><%= @counts.pages %> <%= translated("importAnalysis.pages") %><% end %> + <%= if @execution_state.is_executing do %> +
+
+

<%= translated("importAnalysis.importing") %>

+
+
+
+
+
+ <%= @execution_state.phase || translated("importAnalysis.executionStarting") %> + <%= if present?(@execution_state.detail) do %> + <%= @execution_state.detail %> + <% end %> + <%= @execution_state.current || 0 %> / <%= @execution_state.total || @counts.total %> +
+ <% end %> - -
+ <%= if not @execution_state.is_executing and not @execution_state.completed do %> +
+
+ <%= translated("importAnalysis.readyToImport") %> + <%= if @counts.tags > 0 do %><%= @counts.tags %> <%= translated("importAnalysis.tagsCategories") %><% end %> + <%= if @counts.posts > 0 do %><%= @counts.posts %> <%= translated("importAnalysis.posts") %><% end %> + <%= if @counts.media > 0 do %><%= @counts.media %> <%= translated("importAnalysis.media") %><% end %> + <%= if @counts.pages > 0 do %><%= @counts.pages %> <%= translated("importAnalysis.pages") %><% end %> +
+ + +
+ <% end %> <%= if @execution_state.completed do %>
@@ -356,46 +668,28 @@ defmodule BDS.Desktop.ShellLive.ImportEditor do
<% end %> - <%= if Enum.any?(Map.get(@report, :conflicts, [])) do %> -
- + <%= if Enum.any?(@post_conflicts) do %> + <.conflict_section title={translated("importAnalysis.postSlugConflicts")} items={@post_conflicts} expanded={@sections.post_conflicts} section="post_conflicts" /> + <% end %> - <%= if @sections.conflicts do %> - - - - - - - - - - - <%= for conflict <- @report.conflicts do %> - - - - - - - <% end %> - -
<%= translated("importAnalysis.slug") %><%= translated("importAnalysis.newEntryWxr") %><%= translated("importAnalysis.existingEntry") %><%= translated("importAnalysis.resolution") %>
<%= conflict.item_name %><%= conflict.source_title %><%= conflict.existing_title || translated("importAnalysis.none") %> -
- - - -
-
- <% end %> -
+ <%= if Enum.any?(@page_conflicts) do %> + <.conflict_section title={translated("importAnalysis.pageSlugConflicts")} items={@page_conflicts} expanded={@sections.page_conflicts} section="page_conflicts" /> + <% end %> + + <%= if Enum.any?(@post_items) do %> + <.post_detail_section title={translated("importAnalysis.postsWithCount", %{count: length(@post_items)})} items={@post_items} expanded={@sections.posts} section="posts" /> + <% end %> + + <%= if Enum.any?(@other_items) do %> + <.post_detail_section title={translated("importAnalysis.otherWithCount", %{count: length(@other_items)})} items={@other_items} expanded={@sections.other} section="other" show_type={true} /> + <% end %> + + <%= if Enum.any?(@detail_pages) do %> + <.post_detail_section title={translated("importAnalysis.pagesWithCount", %{count: length(@detail_pages)})} items={@detail_pages} expanded={@sections.pages} section="pages" /> + <% end %> + + <%= if Enum.any?(@detail_media) do %> + <.media_detail_section title={translated("importAnalysis.mediaWithCount", %{count: length(@detail_media)})} items={@detail_media} expanded={@sections.media} section="media" /> <% end %> <%= if Enum.any?(Map.get(@report.items, :categories, [])) or Enum.any?(Map.get(@report.items, :tags, [])) do %> @@ -428,8 +722,20 @@ defmodule BDS.Desktop.ShellLive.ImportEditor do
- <.taxonomy_group title={translated("importAnalysis.categories")} items={Map.get(@report.items, :categories, [])} existing_terms={@import_editor.existing_terms} type="categories" /> - <.taxonomy_group title={translated("importAnalysis.tags")} items={Map.get(@report.items, :tags, [])} existing_terms={@import_editor.existing_terms} type="tags" /> + <.taxonomy_group + title={translated("importAnalysis.categories")} + items={Map.get(@report.items, :categories, [])} + suggestions={Map.get(@import_editor.taxonomy_terms, :categories, [])} + edit={@import_editor.taxonomy_edit} + type="categories" + /> + <.taxonomy_group + title={translated("importAnalysis.tags")} + items={Map.get(@report.items, :tags, [])} + suggestions={Map.get(@import_editor.taxonomy_terms, :tags, [])} + edit={@import_editor.taxonomy_edit} + type="tags" + />
<% end %> @@ -458,17 +764,160 @@ defmodule BDS.Desktop.ShellLive.ImportEditor do <% end %> <% else %> + <%= if @import_editor.is_loading do %> + <% else %>

<%= translated("importAnalysis.emptyState") %>

+ <% end %> <% end %>
""" end + attr :title, :string, required: true + attr :items, :list, required: true + attr :expanded, :boolean, required: true + attr :section, :string, required: true + + def conflict_section(assigns) do + ~H""" +
+ + + <%= if @expanded do %> + + + + + + + + + + + <%= for item <- @items do %> + + + + + + + <% end %> + +
<%= translated("importAnalysis.slug") %><%= translated("importAnalysis.newEntryWxr") %><%= translated("importAnalysis.existingEntry") %><%= translated("importAnalysis.resolution") %>
<%= Map.get(item, :slug) %><%= Map.get(item, :title) %><%= Map.get(item, :existing_title) || translated("importAnalysis.none") %> +
+ + + +
+
+ <% end %> +
+ """ + end + + attr :title, :string, required: true + attr :items, :list, required: true + attr :expanded, :boolean, required: true + attr :section, :string, required: true + attr :show_type, :boolean, default: false + + def post_detail_section(assigns) do + ~H""" +
+ + + <%= if @expanded do %> + + + + + <%= if @show_type do %> + + <% end %> + + + + + + + + + <%= for item <- @items do %> + + + <%= if @show_type do %> + + <% end %> + + + + + + + <% end %> + +
<%= translated("importAnalysis.status") %><%= translated("importAnalysis.type") %><%= translated("importAnalysis.title") %><%= translated("importAnalysis.slug") %><%= translated("importAnalysis.categories") %><%= translated("importAnalysis.wpStatus") %><%= translated("importAnalysis.existingMatch") %>
<%= item.status %><%= Map.get(item, :post_type, Map.get(item, :item_type)) %><%= Map.get(item, :title) %><%= Map.get(item, :slug) %><%= joined_or_none(Map.get(item, :categories)) %><%= Map.get(item, :wp_status) || translated("importAnalysis.none") %><%= Map.get(item, :existing_title) || translated("importAnalysis.none") %>
+ <% end %> +
+ """ + end + + attr :title, :string, required: true + attr :items, :list, required: true + attr :expanded, :boolean, required: true + attr :section, :string, required: true + + def media_detail_section(assigns) do + ~H""" +
+ + + <%= if @expanded do %> + + + + + + + + + + + + <%= for item <- @items do %> + + + + + + + + <% end %> + +
<%= translated("importAnalysis.status") %><%= translated("importAnalysis.filename") %><%= translated("importAnalysis.type") %><%= translated("importAnalysis.path") %><%= translated("importAnalysis.existingMatch") %>
<%= item.status %><%= Map.get(item, :filename) %><%= Map.get(item, :mime_type) || translated("importAnalysis.none") %><%= Map.get(item, :relative_path) %><%= Map.get(item, :existing_title) || translated("importAnalysis.none") %>
+ <% end %> +
+ """ + end + attr :label, :string, required: true attr :stats, :map, required: true @@ -525,26 +974,71 @@ defmodule BDS.Desktop.ShellLive.ImportEditor do attr :title, :string, required: true attr :items, :list, required: true - attr :existing_terms, :list, required: true + attr :suggestions, :list, required: true + attr :edit, :map, default: nil attr :type, :string, required: true def taxonomy_group(assigns) do ~H"""

<%= @title %>

+ + <%= for term <- @suggestions do %> + + <% end %> +
<%= for item <- @items do %> -
- - - <%= item.name %> - + + <%= item.name %> + + + + <%= if present?(item.mapped_to) do %> + <% end %> - -
+ + <% else %> +
+ <%= if item.exists_in_project do %> + <%= item.name %> + <% else %> + + <% end %> + + <%= if present?(item.mapped_to) do %> + + + + <% end %> +
+ <% end %> <% end %>
@@ -633,6 +1127,27 @@ defmodule BDS.Desktop.ShellLive.ImportEditor do end) end + defp detail_items(nil, _bucket), do: [] + + defp detail_items(report, bucket) do + get_in(report, [:details, bucket]) || get_in(report, [:items, bucket]) || [] + end + + defp execution_progress_width(state) do + current = Map.get(state, :current, 0) + total = Map.get(state, :total, 0) + + cond do + total <= 0 -> 0 + true -> min(current / total * 100, 100) + end + end + + defp joined_or_none(values) when is_list(values) and values != [], do: Enum.join(values, ", ") + defp joined_or_none(_values), do: translated("importAnalysis.none") + + defp status_badge_class(status), do: ["status-badge", status] + defp distribution_width(value, rows, key) do max_value = rows |> Enum.map(&Map.get(&1, key, 0)) |> Enum.max(fn -> 1 end) max(8, value / max(max_value, 1) * 100) @@ -649,18 +1164,54 @@ defmodule BDS.Desktop.ShellLive.ImportEditor do end end + defp taxonomy_item_editing?(%{type: type, name: name}, type, name), do: true + defp taxonomy_item_editing?(_edit, _type, _name), do: false + + defp taxonomy_mapping_tooltip(item) do + action = + if present?(item.mapped_to), + do: translated("importAnalysis.mappingActionEdit"), + else: translated("importAnalysis.mappingActionAdd") + + translated("importAnalysis.mappingTooltip", %{action: action}) + end + defp translated(text, bindings \\ %{}), do: ShellData.translate(text, bindings, Process.get(:bds_ui_locale)) defp present?(value), do: value not in [nil, ""] defp blank?(value), do: value in [nil, ""] defp blank_to_nil(""), do: nil defp blank_to_nil(value), do: value - defp default_execution_state do - %{completed: false, error: nil, count: 0, result: nil} + defp default_analysis_state do + %{loading: false, step: nil, detail: nil, file_path: nil, ref: nil} end defp default_sections do - %{conflicts: true, taxonomy: true, macros: true} + %{ + post_conflicts: true, + page_conflicts: true, + posts: false, + other: false, + pages: false, + media: false, + taxonomy: true, + macros: true + } + end + + defp default_execution_state do + %{ + is_executing: false, + completed: false, + error: nil, + count: 0, + result: nil, + phase: nil, + current: 0, + total: 0, + detail: nil, + ref: nil + } end defp selected_model(assigns, definition_id) do @@ -686,39 +1237,6 @@ defmodule BDS.Desktop.ShellLive.ImportEditor do end end - defp auto_map_taxonomies(report, existing_terms) do - report - |> update_in([:items, :categories], &auto_map_taxonomy_items(&1, existing_terms)) - |> update_in([:items, :tags], &auto_map_taxonomy_items(&1, existing_terms)) - |> then(fn updated_report -> - updated_report - |> Map.put(:category_stats, rebuild_taxonomy_stats(get_in(updated_report, [:items, :categories]) || [])) - |> Map.put(:tag_stats, rebuild_taxonomy_stats(get_in(updated_report, [:items, :tags]) || [])) - end) - end - - defp auto_map_taxonomy_items(items, existing_terms) do - Enum.map(items || [], fn item -> - cond do - item.exists_in_project or present?(item.mapped_to) -> item - suggestion = best_taxonomy_match(item.name, existing_terms) -> %{item | mapped_to: suggestion} - true -> item - end - end) - end - - defp best_taxonomy_match(term, existing_terms) do - normalized_term = normalize_term(term) - - existing_terms - |> Enum.map(fn candidate -> {candidate, String.jaro_distance(normalized_term, normalize_term(candidate))} end) - |> Enum.max_by(fn {_candidate, score} -> score end, fn -> {nil, 0.0} end) - |> case do - {candidate, score} when is_binary(candidate) and score >= 0.94 -> candidate - _other -> nil - end - end - defp auto_mapped_count(previous_report, next_report) do previous_count = (Map.get(previous_report.items, :categories, []) ++ Map.get(previous_report.items, :tags, [])) @@ -731,15 +1249,74 @@ defmodule BDS.Desktop.ShellLive.ImportEditor do max(next_count - previous_count, 0) end - defp normalize_term(term) do - term - |> to_string() - |> String.downcase() - |> String.replace(~r/[^a-z0-9]+/u, "") + defp apply_taxonomy_mappings(report, analysis) do + report + |> update_in([:items, :categories], &apply_taxonomy_mapping_bucket(&1, Map.get(analysis, :category_mappings, %{}))) + |> update_in([:items, :tags], &apply_taxonomy_mapping_bucket(&1, Map.get(analysis, :tag_mappings, %{}))) + |> then(fn updated_report -> + updated_report + |> Map.put(:category_stats, rebuild_taxonomy_stats(get_in(updated_report, [:items, :categories]) || [])) + |> Map.put(:tag_stats, rebuild_taxonomy_stats(get_in(updated_report, [:items, :tags]) || [])) + end) end + defp apply_taxonomy_mapping_bucket(items, mappings) do + Enum.map(items || [], fn item -> + case Map.fetch(mappings, item.name) do + {:ok, mapped_to} -> %{item | mapped_to: mapped_to} + :error -> item + end + end) + end + + defp existing_taxonomy_terms(project_id) do + {:ok, metadata} = Metadata.get_project_metadata(project_id) + + %{ + categories: Enum.uniq(Map.get(metadata, :categories, []) || []), + tags: project_id |> Tags.list_tags() |> Enum.map(& &1.name) |> Enum.uniq() + } + end + + defp normalize_taxonomy_mapping_value(project_id, type, mapped_to) do + normalized_value = mapped_to |> to_string() |> String.trim() |> blank_to_nil() + + case normalized_value do + nil -> nil + value -> + project_id + |> existing_taxonomy_terms() + |> Map.get(String.to_existing_atom(type), []) + |> Enum.find(fn term -> String.downcase(term) == String.downcase(value) end) + end + end + + defp maybe_put_option(opts, _key, nil), do: opts + defp maybe_put_option(opts, key, value), do: Keyword.put(opts, key, value) + defp default_author(project_id) do {:ok, metadata} = Metadata.get_project_metadata(project_id) Map.get(metadata, :default_author) end + + defp suggested_definition_name(report) do + get_in(report, [:site_info, :url]) || get_in(report, [:site_info, :title]) + end + + defp maybe_put(map, _key, nil), do: map + defp maybe_put(map, key, value), do: Map.put(map, key, value) + + defp allow_repo_sandbox(pid) when is_pid(pid) do + if Code.ensure_loaded?(Ecto.Adapters.SQL.Sandbox) do + try do + Ecto.Adapters.SQL.Sandbox.allow(BDS.Repo, self(), pid) + rescue + _error -> :ok + end + else + :ok + end + + :ok + end end diff --git a/lib/bds/import_analysis.ex b/lib/bds/import_analysis.ex index 89da898..194840c 100644 --- a/lib/bds/import_analysis.ex +++ b/lib/bds/import_analysis.ex @@ -12,17 +12,30 @@ defmodule BDS.ImportAnalysis do @shortcode_regex ~r/(? :ok end) wxr_data = WxrParser.parse_file(wxr_file_path) - {:ok, build_report(project_id, wxr_data, wxr_file_path, uploads_folder_path)} + {:ok, build_report(project_id, wxr_data, wxr_file_path, uploads_folder_path, on_progress)} rescue error -> {:error, %{message: Exception.message(error)}} end - defp build_report(project_id, wxr_data, wxr_file_path, uploads_folder_path) do + defp build_report(project_id, wxr_data, wxr_file_path, uploads_folder_path, on_progress) do + notify_progress(on_progress, "Loading existing posts...") existing_posts = Repo.all(from post in Post, where: post.project_id == ^project_id) + + notify_progress(on_progress, "Loading existing media...", "#{length(existing_posts)} posts in project") existing_media = Repo.all(from media in Media, where: media.project_id == ^project_id) + + notify_progress(on_progress, "Loading existing tags...", "#{length(existing_media)} media in project") existing_tag_names = Repo.all(from tag in Tag, where: tag.project_id == ^project_id, select: tag.name) existing_tag_set = existing_tag_names |> Enum.map(&String.downcase/1) |> MapSet.new() @@ -40,15 +53,22 @@ defmodule BDS.ImportAnalysis do |> Enum.reject(&is_nil(&1.checksum)) |> Map.new(&{&1.checksum, &1}) + notify_progress(on_progress, "Analyzing posts...", "#{length(wxr_data.posts)} posts to analyze") analyzed_posts = Enum.map(wxr_data.posts, &analyze_post_item(&1, posts_by_slug, posts_by_checksum, "post")) + + notify_progress(on_progress, "Analyzing pages...", "#{length(wxr_data.pages)} pages to analyze") analyzed_pages = Enum.map(wxr_data.pages, &analyze_post_item(&1, posts_by_slug, posts_by_checksum, "page")) + notify_progress(on_progress, "Analyzing media files...", "#{length(wxr_data.media)} media files to analyze") analyzed_media = Enum.map(wxr_data.media, &analyze_media_item(&1, uploads_folder_path, media_by_name, media_by_checksum)) + notify_progress(on_progress, "Processing categories and tags...") category_items = Enum.map(wxr_data.categories, &analyze_taxonomy_item(&1, existing_tag_set)) tag_items = Enum.map(wxr_data.tags, &analyze_taxonomy_item(&1, existing_tag_set)) + notify_progress(on_progress, "Discovering macros...") + %{ source_file: wxr_file_path, site_info: %{ @@ -312,6 +332,16 @@ defmodule BDS.ImportAnalysis do defp count_status(items, status), do: Enum.count(items, &(&1.status == status)) + defp notify_progress(callback, step, detail \\ nil) when is_function(callback, 2) do + try do + callback.(step, detail) + rescue + _error -> :ok + end + + :ok + end + defp sha256(value) do :sha256 |> :crypto.hash(value) diff --git a/lib/bds/import_execution.ex b/lib/bds/import_execution.ex index 21dd01a..dc0a0f7 100644 --- a/lib/bds/import_execution.ex +++ b/lib/bds/import_execution.ex @@ -11,6 +11,12 @@ defmodule BDS.ImportExecution do def execute_import(project_id, report, opts \\ []) when is_binary(project_id) and is_map(report) do normalized_report = normalize_report(report) default_author = Keyword.get(opts, :default_author) || project_default_author(project_id) + uploads_folder_path = Keyword.get(opts, :uploads_folder_path) + on_progress = Keyword.get(opts, :on_progress, fn _phase, _current, _total, _detail -> :ok end) + taxonomies = taxonomy_items(normalized_report) + post_items = import_items(normalized_report, :posts) + page_items = import_items(normalized_report, :pages) + media_items = import_items(normalized_report, :media) result = %{ success: true, @@ -21,49 +27,73 @@ defmodule BDS.ImportExecution do errors: [] } - result = execute_taxonomies(normalized_report, project_id, result) - result = execute_posts(normalized_report, project_id, default_author, result) - result = execute_pages(normalized_report, project_id, default_author, result) + notify_progress(on_progress, "tags", 0, length(taxonomies), "Creating tags...") + result = execute_taxonomies(taxonomies, project_id, result, on_progress) - {:ok, execute_media(normalized_report, project_id, default_author, result)} + notify_progress(on_progress, "posts", 0, length(post_items), "Importing posts...") + result = execute_posts(post_items, project_id, default_author, result, on_progress) + + notify_progress(on_progress, "pages", 0, length(page_items), "Importing pages...") + result = execute_pages(page_items, project_id, default_author, result, on_progress) + + notify_progress(on_progress, "media", 0, length(media_items), "Importing media...") + result = execute_media(media_items, project_id, default_author, result, on_progress, uploads_folder_path) + + notify_progress(on_progress, "complete", 1, 1, "Import complete") + {:ok, result} rescue error -> {:error, %{message: Exception.message(error)}} end - defp execute_taxonomies(report, project_id, result) do - taxonomies = List.wrap(get_in(report, [:items, :categories])) ++ List.wrap(get_in(report, [:items, :tags])) - + defp execute_taxonomies(taxonomies, project_id, result, on_progress) do Enum.reduce(taxonomies, result, fn item, acc -> + current = acc.tags.created + acc.tags.skipped + 1 + if item.exists_in_project || item.mapped_to do + notify_progress(on_progress, "tags", current, length(taxonomies), "Skipping tag: #{item.name}") put_in(acc, [:tags, :skipped], acc.tags.skipped + 1) else case Tags.create_tag(%{project_id: project_id, name: item.name}) do - {:ok, _tag} -> put_in(acc, [:tags, :created], acc.tags.created + 1) - {:error, _reason} -> put_in(acc, [:tags, :skipped], acc.tags.skipped + 1) + {:ok, _tag} -> + notify_progress(on_progress, "tags", current, length(taxonomies), "Created tag: #{item.name}") + put_in(acc, [:tags, :created], acc.tags.created + 1) + + {:error, _reason} -> + notify_progress(on_progress, "tags", current, length(taxonomies), "Skipping tag: #{item.name}") + put_in(acc, [:tags, :skipped], acc.tags.skipped + 1) end end end) end - defp execute_posts(report, project_id, default_author, result) do - items = import_items(report, :posts) + defp execute_posts(items, project_id, default_author, result, on_progress) do + total = length(items) - Enum.reduce(items, result, fn item, acc -> + Enum.with_index(items, 1) + |> Enum.reduce(result, fn {item, index}, acc -> + notify_progress(on_progress, "posts", index, total, "Processing: #{item.title}") execute_post_item(project_id, item, acc, :posts, default_author) end) end - defp execute_pages(report, project_id, default_author, result) do - items = import_items(report, :pages) + defp execute_pages(items, project_id, default_author, result, on_progress) do + total = length(items) - Enum.reduce(items, result, fn item, acc -> + Enum.with_index(items, 1) + |> Enum.reduce(result, fn {item, index}, acc -> + notify_progress(on_progress, "pages", index, total, "Processing: #{item.title}") execute_post_item(project_id, ensure_page_category(item), acc, :pages, default_author) end) end - defp execute_media(report, project_id, default_author, result) do - import_items(report, :media) - |> Enum.reduce(result, fn item, acc -> + defp execute_media(items, project_id, default_author, result, on_progress, uploads_folder_path) do + total = length(items) + + items + |> Enum.with_index(1) + |> Enum.reduce(result, fn {item, index}, acc -> + notify_progress(on_progress, "media", index, total, "Processing: #{item.filename}") + cond do item.status in ["update", "duplicate", "missing"] -> put_in(acc, [:media, :skipped], acc.media.skipped + 1) @@ -72,7 +102,7 @@ defmodule BDS.ImportExecution do put_in(acc, [:media, :skipped], acc.media.skipped + 1) true -> - case import_media_item(project_id, item, default_author) do + case import_media_item(project_id, item, default_author, uploads_folder_path) do {:ok, _media} -> put_in(acc, [:media, :imported], acc.media.imported + 1) {:error, reason} -> acc @@ -141,8 +171,8 @@ defmodule BDS.ImportExecution do end end - defp import_media_item(project_id, item, default_author) do - source_path = item.source_file || Path.join("", item.relative_path) + defp import_media_item(project_id, item, default_author, uploads_folder_path) do + source_path = item.source_file || uploads_source_path(item.relative_path, uploads_folder_path) checksum = if(source_path != nil and File.exists?(source_path), do: md5(File.read!(source_path)), else: nil) if source_path && File.exists?(source_path) do @@ -293,6 +323,29 @@ defmodule BDS.ImportExecution do defp parse_timestamp(_value), do: nil + defp taxonomy_items(report) do + List.wrap(get_in(report, [:items, :categories])) ++ List.wrap(get_in(report, [:items, :tags])) + end + + defp uploads_source_path(relative_path, uploads_folder_path) + + defp uploads_source_path(relative_path, uploads_folder_path) + when is_binary(relative_path) and is_binary(uploads_folder_path) and uploads_folder_path != "" do + Path.join(uploads_folder_path, relative_path) + end + + defp uploads_source_path(_relative_path, _uploads_folder_path), do: nil + + defp notify_progress(callback, phase, current, total, detail) when is_function(callback, 4) do + try do + callback.(phase, current, total, detail) + rescue + _error -> :ok + end + + :ok + end + defp md5(binary) do :md5 |> :crypto.hash(binary) diff --git a/priv/ui/app.css b/priv/ui/app.css index 4835086..f83ce52 100644 --- a/priv/ui/app.css +++ b/priv/ui/app.css @@ -7256,6 +7256,48 @@ button svg * { cursor: not-allowed; } +.import-loading { + display: flex; + align-items: center; + gap: 12px; + padding: 16px; + border: 1px solid var(--vscode-panel-border); + border-radius: 10px; + background: color-mix(in srgb, var(--vscode-editor-background) 84%, var(--vscode-input-background)); +} + +.import-spinner { + width: 18px; + height: 18px; + border: 2px solid var(--vscode-descriptionForeground); + border-top-color: var(--vscode-button-background); + border-radius: 50%; + animation: import-spinner-rotate 0.8s linear infinite; + flex-shrink: 0; +} + +.import-progress { + display: flex; + flex-direction: column; + gap: 2px; +} + +.import-progress-step { + font-size: 13px; + color: var(--vscode-foreground); +} + +.import-progress-detail { + font-size: 11px; + color: var(--vscode-descriptionForeground); +} + +@keyframes import-spinner-rotate { + to { + transform: rotate(360deg); + } +} + .import-site-info { display: grid; grid-template-columns: repeat(4, minmax(0, 1fr)); @@ -7458,6 +7500,56 @@ button svg * { font-weight: 600; } +.import-execution-progress { + display: grid; + gap: 10px; + padding: 16px; + border: 1px solid var(--vscode-panel-border); + border-radius: 10px; + background: color-mix(in srgb, var(--vscode-editor-background) 84%, var(--vscode-input-background)); +} + +.import-execution-header { + display: flex; + align-items: center; + justify-content: space-between; + gap: 12px; +} + +.import-execution-header h3 { + margin: 0; + font-size: 14px; +} + +.import-progress-bar { + height: 10px; + border-radius: 999px; + overflow: hidden; + background: var(--vscode-input-background); +} + +.import-progress-fill { + height: 100%; + background: linear-gradient(90deg, rgba(117, 190, 255, 0.85), rgba(117, 190, 255, 0.45)); +} + +.import-progress-info { + display: flex; + align-items: center; + gap: 12px; + flex-wrap: wrap; + font-size: 12px; +} + +.import-phase { + font-weight: 600; +} + +.import-detail, +.import-counter { + color: var(--vscode-descriptionForeground); +} + .import-detail-table { width: 100%; border-collapse: collapse; @@ -7480,8 +7572,55 @@ button svg * { letter-spacing: 0.04em; } +.import-detail-table .status-badge { + display: inline-flex; + align-items: center; + border-radius: 999px; + padding: 4px 9px; + font-size: 10px; + font-weight: 700; + text-transform: uppercase; + letter-spacing: 0.04em; +} + +.import-detail-table .status-badge.new { + background: rgba(117, 190, 255, 0.16); + color: #75beff; +} + +.import-detail-table .status-badge.update { + background: rgba(115, 201, 145, 0.16); + color: #73c991; +} + +.import-detail-table .status-badge.conflict { + background: rgba(255, 166, 87, 0.16); + color: #ffb169; +} + +.import-detail-table .status-badge.duplicate, +.import-detail-table .status-badge.missing { + background: rgba(204, 167, 0, 0.16); + color: #cca700; +} + +.categories-cell, +.existing-match, +.mime-type-cell, +.post-type-cell { + font-size: 11px; + color: var(--vscode-descriptionForeground); +} + +.mime-type-cell, +.post-type-cell, +.existing-match, +.slug-cell { + font-family: var(--vscode-editor-font-family, ui-monospace, monospace); +} + .resolution-select, -.import-taxonomy-form select { +.taxonomy-mapping-input { min-width: 150px; background: var(--vscode-dropdown-background, var(--vscode-input-background)); color: var(--vscode-dropdown-foreground, var(--vscode-foreground)); @@ -7556,12 +7695,58 @@ button svg * { gap: 12px; } -.import-taxonomy-form { +.import-taxonomy-entry, +.import-taxonomy-edit-form { display: inline-flex; align-items: center; gap: 8px; } +.import-taxonomy-entry, +.import-taxonomy-edit-form { + flex-wrap: wrap; +} + +.import-taxonomy-pill { + border: none; + cursor: default; +} + +button.import-taxonomy-pill { + cursor: pointer; +} + +.mapped-target { + background: rgba(115, 201, 145, 0.1); +} + +.taxonomy-mapping-arrow { + color: var(--vscode-descriptionForeground); + font-size: 12px; +} + +.taxonomy-mapping-input { + min-width: 170px; + border-radius: 6px; +} + +.taxonomy-edit-btn, +.taxonomy-clear-btn { + min-width: 28px; + min-height: 28px; + padding: 0 8px !important; + display: inline-flex; + align-items: center; + justify-content: center; +} + +.taxonomy-edit-btn.ghost, +.taxonomy-clear-btn { + background: transparent !important; + border: 1px solid var(--vscode-panel-border) !important; + color: var(--vscode-descriptionForeground) !important; +} + .macros-list { display: grid; gap: 10px; @@ -7645,7 +7830,7 @@ button svg * { .import-analysis button, .resolution-select, - .import-taxonomy-form select { + .taxonomy-mapping-input { width: 100%; } @@ -7654,7 +7839,8 @@ button svg * { align-items: stretch; } - .import-taxonomy-form { + .import-taxonomy-entry, + .import-taxonomy-edit-form { width: 100%; flex-direction: column; align-items: stretch; diff --git a/test/bds/ai_test.exs b/test/bds/ai_test.exs index 115c33f..071dd18 100644 --- a/test/bds/ai_test.exs +++ b/test/bds/ai_test.exs @@ -119,6 +119,16 @@ defmodule BDS.AITest do usage: usage(13, 9, 0, 0) }} + :import_taxonomy_mapping -> + {:ok, + %{ + json: %{ + "categoryMappings" => %{"General" => "article", "Unknown" => "missing"}, + "tagMappings" => %{"News" => "updates", "Ghost" => "missing"} + }, + usage: usage(19, 7, 0, 0) + }} + :chat -> if Enum.any?(request.messages, &(&1["role"] == "tool")) do {:ok, @@ -309,6 +319,36 @@ defmodule BDS.AITest do assert request.model == "gpt-4.1-mini" end + test "analyze_import_taxonomy uses the selected model override and returns only valid existing-term mappings" do + assert {:ok, _endpoint} = + BDS.AI.put_endpoint(:online, %{ + url: "https://api.example.test/v1", + api_key: "online-secret", + model: "gpt-4o-mini" + }, secret_backend: FakeSecretBackend) + + assert :ok = BDS.AI.set_airplane_mode(false) + assert :ok = BDS.AI.put_model_preference(:title, "gpt-4.1-mini") + + assert {:ok, result} = + BDS.AI.analyze_import_taxonomy( + %{categories: ["General"], tags: ["News"]}, + %{categories: ["article", "page"], tags: ["updates"]}, + runtime: FakeRuntime, + test_pid: self(), + secret_backend: FakeSecretBackend, + model: "gpt-4o" + ) + + assert result.category_mappings == %{"General" => "article"} + assert result.tag_mappings == %{"News" => "updates"} + + assert_received {:runtime_request, endpoint, request} + assert endpoint.kind == :online + assert request.operation == :import_taxonomy_mapping + assert request.model == "gpt-4o" + end + test "analyze_image requires a vision-capable airplane model before sending image input" do assert {:ok, _endpoint} = BDS.AI.put_endpoint(:airplane, %{ diff --git a/test/bds/desktop/import_shell_live_test.exs b/test/bds/desktop/import_shell_live_test.exs index 6b9e0c7..f9a2126 100644 --- a/test/bds/desktop/import_shell_live_test.exs +++ b/test/bds/desktop/import_shell_live_test.exs @@ -53,8 +53,29 @@ defmodule BDS.Desktop.ImportShellLiveTest do assert html =~ "Ready to import:" assert html =~ "Import 5 Items" assert html =~ "Post Slug Conflicts" - assert html =~ "Analyze with..." - refute html =~ "Desktop workbench content routed through the Elixir shell." + assert html =~ "Analyze with..." + assert html =~ "Posts (2)" + assert html =~ "Pages (1)" + assert html =~ "Media (1)" + assert html =~ "Click to add mapping" + refute html =~ ~s(name="mapped_to") + refute html =~ "Desktop workbench content routed through the Elixir shell." + + posts_html = + view + |> element("button[phx-value-section='posts']") + |> render_click() + + assert posts_html =~ "Existing Match" + assert posts_html =~ "WP Status" + + media_html = + view + |> element("button[phx-value-section='media']") + |> render_click() + + assert media_html =~ "Filename" + assert media_html =~ "Path" end defp cached_report(wxr_path, uploads_dir) do @@ -102,6 +123,70 @@ defmodule BDS.Desktop.ImportShellLiveTest do ], categories: [%{name: "General", exists_in_project: false, mapped_to: nil}], tags: [%{name: "News", exists_in_project: false, mapped_to: nil}] + }, + details: %{ + posts: [ + %{ + item_type: "post", + title: "Hello World", + slug: "hello-world", + status: "new", + wp_status: "publish", + author: "Importer", + categories: ["General"], + tags: ["News"], + published_at: "2024-05-01 12:00:00", + excerpt: "Legacy hello", + content_markdown: "Hello world", + content_preview: "Hello world" + }, + %{ + item_type: "post", + title: "Conflict Me", + slug: "conflict-me", + status: "conflict", + resolution: "skip", + wp_status: "publish", + author: "Importer", + categories: ["General"], + tags: ["News"], + published_at: "2024-05-02 12:00:00", + excerpt: "Legacy conflict", + existing_title: "Existing Conflict", + content_markdown: "Incoming conflict body", + content_preview: "Incoming conflict body" + } + ], + pages: [ + %{ + item_type: "page", + title: "About", + slug: "about", + status: "new", + wp_status: "publish", + author: "Importer", + categories: ["General"], + tags: [], + published_at: "2024-05-03 12:00:00", + excerpt: "About page", + content_markdown: "About page", + content_preview: "About page" + } + ], + media: [ + %{ + item_type: "media", + title: "Import Asset", + filename: "import-asset.txt", + relative_path: "2024/05/import-asset.txt", + source_file: Path.join(uploads_dir, "2024/05/import-asset.txt"), + status: "new", + mime_type: "text/plain", + description: "Legacy text attachment", + parent_wp_id: 101, + created_at: "2024-05-03 12:00:00" + } + ] } } end diff --git a/test/bds/import_analysis_test.exs b/test/bds/import_analysis_test.exs index f12a071..ed6d0f5 100644 --- a/test/bds/import_analysis_test.exs +++ b/test/bds/import_analysis_test.exs @@ -149,6 +149,28 @@ defmodule BDS.ImportAnalysisTest do assert Enum.any?(report.items.media, &(&1.filename == "missing-asset.txt" and &1.status == "missing")) end + test "analyze_wxr reports legacy progress steps while building the import report", %{project: project, temp_dir: temp_dir} do + uploads_dir = Path.join(temp_dir, "uploads") + File.mkdir_p!(Path.join(uploads_dir, "2024/05")) + File.write!(Path.join(uploads_dir, "2024/05/import-asset.txt"), "legacy attachment") + + wxr_path = Path.join(temp_dir, "legacy.xml") + File.write!(wxr_path, basic_wxr_xml()) + + assert {:ok, _report} = + ImportAnalysis.analyze_wxr(project.id, wxr_path, uploads_dir, + on_progress: fn step, detail -> + send(self(), {:analysis_progress, step, detail}) + end + ) + + assert_received {:analysis_progress, "Loading existing posts...", nil} + assert_received {:analysis_progress, "Analyzing posts...", "1 posts to analyze"} + assert_received {:analysis_progress, "Analyzing pages...", "1 pages to analyze"} + assert_received {:analysis_progress, "Analyzing media files...", "1 media files to analyze"} + assert_received {:analysis_progress, "Discovering macros...", nil} + end + defp sha256(value) do :sha256 |> :crypto.hash(value) diff --git a/test/bds/import_execution_test.exs b/test/bds/import_execution_test.exs index 3748ce2..43765a5 100644 --- a/test/bds/import_execution_test.exs +++ b/test/bds/import_execution_test.exs @@ -97,6 +97,32 @@ defmodule BDS.ImportExecutionTest do assert Enum.any?(slugs, &(&1 != "conflict-me")) end + test "execute_import reports phase progress while importing", %{project: project, temp_dir: temp_dir} do + uploads_dir = Path.join(temp_dir, "uploads") + File.mkdir_p!(Path.join(uploads_dir, "2024/05")) + File.write!(Path.join(uploads_dir, "2024/05/import-asset.txt"), "legacy attachment") + + wxr_path = Path.join(temp_dir, "legacy.xml") + File.write!(wxr_path, basic_wxr_xml()) + + assert {:ok, report} = ImportAnalysis.analyze_wxr(project.id, wxr_path, uploads_dir) + + assert {:ok, _result} = + ImportExecution.execute_import(project.id, report, + uploads_folder_path: uploads_dir, + default_author: "Imported Author", + on_progress: fn phase, current, total, detail -> + send(self(), {:execution_progress, phase, current, total, detail}) + end + ) + + assert_received {:execution_progress, "tags", 0, 2, "Creating tags..."} + assert_received {:execution_progress, "posts", 0, 1, "Importing posts..."} + assert_received {:execution_progress, "media", 0, 1, "Importing media..."} + assert_received {:execution_progress, "pages", 0, 1, "Importing pages..."} + assert_received {:execution_progress, "complete", 1, 1, "Import complete"} + end + defp sha256(value) do :sha256 |> :crypto.hash(value)