hugo/bDS2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: step 12 is done again. huh?

Browse Source
This commit is contained in:
Chili Palmer
2026-04-29 20:33:59 +02:00
parent f178b5b207
commit a6033cb86a
11 changed files with 1369 additions and 205 deletions
Download Patch File Download Diff File Expand all files Collapse all files

120
lib/bds/ai.ex
View File

@@ -212,6 +212,40 @@ defmodule BDS.AI do
end
end
def analyze_import_taxonomy(import_terms, existing_terms, opts \\ [])
when is_map(import_terms) and is_map(existing_terms) and is_list(opts) do
payload = %{
import_categories: normalize_string_list(Map.get(import_terms, :categories) || Map.get(import_terms, "categories")),
import_tags: normalize_string_list(Map.get(import_terms, :tags) || Map.get(import_terms, "tags")),
existing_categories: normalize_string_list(Map.get(existing_terms, :categories) || Map.get(existing_terms, "categories")),
existing_tags: normalize_string_list(Map.get(existing_terms, :tags) || Map.get(existing_terms, "tags"))
}
run_one_shot(
:import_taxonomy_mapping,
payload,
opts,
fn json, usage ->
{:ok,
%{
category_mappings:
filter_taxonomy_mapping_response(
json["categoryMappings"] || json["category_mappings"],
payload.import_categories,
payload.existing_categories
),
tag_mappings:
filter_taxonomy_mapping_response(
json["tagMappings"] || json["tag_mappings"],
payload.import_tags,
payload.existing_tags
),
usage: usage
}}
end
)
end
def analyze_post(post_input, opts \\ []) when is_list(opts) do
with {:ok, post} <- normalize_post_input(post_input) do
run_one_shot(
@@ -559,7 +593,7 @@ defmodule BDS.AI do
defp run_one_shot(operation, payload, opts, formatter) do
runtime = Keyword.get(opts, :runtime, OpenAICompatibleRuntime)
with {:ok, endpoint, model, mode} <- resolve_runtime_target(operation, secret_backend: Keyword.get(opts, :secret_backend, SecretBackend)),
with {:ok, endpoint, model, mode} <- resolve_runtime_target(operation, opts),
:ok <- validate_runtime_target(operation, model, mode),
request <- build_one_shot_request(operation, payload, model),
{:ok, response} <- runtime.generate(endpoint_with_model(endpoint, model), request, opts),
@@ -903,20 +937,20 @@ defmodule BDS.AI do
{:ok, get_model_preference_value(:chat) || endpoint.model}
end
defp resolve_model_for_operation(:analyze_image, :airplane, endpoint, _extra) do
{:ok, get_model_preference_value(:airplane_image_analysis) || endpoint.model}
defp resolve_model_for_operation(:analyze_image, :airplane, endpoint, extra) do
{:ok, Keyword.get(extra, :model) || get_model_preference_value(:airplane_image_analysis) || endpoint.model}
end
defp resolve_model_for_operation(:analyze_image, :online, endpoint, _extra) do
{:ok, get_model_preference_value(:image_analysis) || endpoint.model}
defp resolve_model_for_operation(:analyze_image, :online, endpoint, extra) do
{:ok, Keyword.get(extra, :model) || get_model_preference_value(:image_analysis) || endpoint.model}
end
defp resolve_model_for_operation(_operation, :airplane, endpoint, _extra) do
{:ok, get_model_preference_value(:airplane_title) || endpoint.model}
defp resolve_model_for_operation(_operation, :airplane, endpoint, extra) do
{:ok, Keyword.get(extra, :model) || get_model_preference_value(:airplane_title) || endpoint.model}
end
defp resolve_model_for_operation(_operation, :online, endpoint, _extra) do
{:ok, get_model_preference_value(:title) || endpoint.model}
defp resolve_model_for_operation(_operation, :online, endpoint, extra) do
{:ok, Keyword.get(extra, :model) || get_model_preference_value(:title) || endpoint.model}
end
defp validate_runtime_target(:analyze_image, model, _mode) do
@@ -990,6 +1024,49 @@ defmodule BDS.AI do
|> MapSet.size()
end
defp normalize_string_list(values) do
values
|> List.wrap()
|> Enum.map(&to_string/1)
|> Enum.map(&String.trim/1)
|> Enum.reject(&(&1 == ""))
|> Enum.uniq()
end
defp filter_taxonomy_mapping_response(mappings, import_terms, existing_terms) when is_map(mappings) do
import_lookup = canonical_term_lookup(import_terms)
existing_lookup = canonical_term_lookup(existing_terms)
Enum.reduce(mappings, %{}, fn {source, target}, acc ->
with {:ok, canonical_source} <- resolve_canonical_term(source, import_lookup),
{:ok, canonical_target} <- resolve_canonical_term(target, existing_lookup) do
Map.put(acc, canonical_source, canonical_target)
else
_other -> acc
end
end)
end
defp filter_taxonomy_mapping_response(_mappings, _import_terms, _existing_terms), do: %{}
defp canonical_term_lookup(terms) do
Map.new(terms, fn term -> {normalize_term(term), term} end)
end
defp resolve_canonical_term(term, lookup) do
case Map.get(lookup, normalize_term(term)) do
nil -> :error
canonical -> {:ok, canonical}
end
end
defp normalize_term(term) do
term
|> to_string()
|> String.trim()
|> String.downcase()
end
defp one_shot_system_prompt(:detect_language) do
"Return JSON with exactly one key: language_code."
end
@@ -998,6 +1075,10 @@ defmodule BDS.AI do
"Return JSON with keys tags and categories, each an array of short strings."
end
defp one_shot_system_prompt(:import_taxonomy_mapping) do
"You are helping import WordPress taxonomy into an existing blog. Return JSON with exactly two keys: categoryMappings and tagMappings. Each value must be an object mapping imported term names to existing project term names. Only map when the imported term should reuse an existing term to avoid duplicates. Do not invent target terms. Leave unmapped items out of the objects."
end
defp one_shot_system_prompt(:analyze_post) do
"Return JSON with keys title, excerpt, and slug."
end
@@ -1022,6 +1103,27 @@ defmodule BDS.AI do
"Suggest categories and tags for the following post.\nTitle: #{post.title}\nExcerpt: #{post.excerpt}\nContent: #{truncate_text(post.content, 2000)}"
end
defp one_shot_user_content(:import_taxonomy_mapping, payload) do
[
"Analyze these imported taxonomy terms and suggest which ones should map to existing project terms.",
"",
"Imported categories:",
Enum.join(payload.import_categories, ", "),
"",
"Imported tags:",
Enum.join(payload.import_tags, ", "),
"",
"Existing project categories:",
Enum.join(payload.existing_categories, ", "),
"",
"Existing project tags:",
Enum.join(payload.existing_tags, ", "),
"",
"Return JSON only."
]
|> Enum.join("\n")
end
defp one_shot_user_content(:analyze_post, post) do
"Suggest an improved title, excerpt, and slug.\nTitle: #{post.title}\nExcerpt: #{post.excerpt}\nContent: #{truncate_text(post.content, 2000)}"
end

55
lib/bds/desktop/shell_live.ex
View File

@@ -105,8 +105,12 @@ defmodule BDS.Desktop.ShellLive do
|> assign(:chat_editor_surface_data, %{})
|> assign(:chat_editor_surface_tabs, %{})
|> assign(:chat_editor_action_errors, %{})
|> assign(:import_editor_analysis_states, %{})
|> assign(:import_editor_analysis_task_refs, %{})
|> assign(:import_editor_execution_states, %{})
|> assign(:import_editor_execution_task_refs, %{})
|> assign(:import_editor_sections, %{})
|> assign(:import_editor_taxonomy_edits, %{})
|> assign(:import_editor_model_selectors_open, %{})
|> assign(:import_editor_selected_models, %{})
|> assign(:misc_editor_selected_pairs, %{})
@@ -791,8 +795,20 @@ defmodule BDS.Desktop.ShellLive do
{:noreply, ImportEditor.change_conflict_resolution(socket, params, &reload_shell/2)}
end
def handle_event("change_import_taxonomy_mapping", params, socket) do
{:noreply, ImportEditor.change_taxonomy_mapping(socket, params, &reload_shell/2)}
def handle_event("start_import_taxonomy_edit", params, socket) do
{:noreply, ImportEditor.start_taxonomy_edit(socket, params, &reload_shell/2)}
end
def handle_event("save_import_taxonomy_edit", params, socket) do
{:noreply, ImportEditor.save_taxonomy_edit(socket, params, &reload_shell/2)}
end
def handle_event("cancel_import_taxonomy_edit", _params, socket) do
{:noreply, ImportEditor.cancel_taxonomy_edit(socket, &reload_shell/2)}
end
def handle_event("clear_import_taxonomy_mapping", params, socket) do
{:noreply, ImportEditor.clear_taxonomy_mapping(socket, params, &reload_shell/2)}
end
def handle_event("toggle_import_section", %{"section" => section}, socket) do
@@ -1184,19 +1200,46 @@ defmodule BDS.Desktop.ShellLive do
@impl true
def handle_info({ref, result}, socket) when is_reference(ref) do
Process.demonitor(ref, [:flush])
{:noreply, ChatEditor.finish_request(socket, ref, result, &reload_shell/2, &append_output_entry/5)}
cond do
Map.has_key?(socket.assigns.import_editor_analysis_task_refs, ref) ->
{:noreply, ImportEditor.finish_analysis(socket, ref, result, &reload_shell/2, &append_output_entry/5)}
Map.has_key?(socket.assigns.import_editor_execution_task_refs, ref) ->
{:noreply, ImportEditor.finish_execution(socket, ref, result, &reload_shell/2, &append_output_entry/5)}
true ->
{:noreply, ChatEditor.finish_request(socket, ref, result, &reload_shell/2, &append_output_entry/5)}
end
end
def handle_info({:DOWN, ref, :process, _pid, reason}, socket) when is_reference(ref) do
next_socket =
case reason do
:normal -> socket
_other -> ChatEditor.finish_request(socket, ref, {:error, :cancelled}, &reload_shell/2, &append_output_entry/5)
cond do
Map.has_key?(socket.assigns.import_editor_analysis_task_refs, ref) ->
ImportEditor.handle_task_down(socket, :analysis, ref, reason, &reload_shell/2, &append_output_entry/5)
Map.has_key?(socket.assigns.import_editor_execution_task_refs, ref) ->
ImportEditor.handle_task_down(socket, :execution, ref, reason, &reload_shell/2, &append_output_entry/5)
true ->
case reason do
:normal -> socket
_other -> ChatEditor.finish_request(socket, ref, {:error, :cancelled}, &reload_shell/2, &append_output_entry/5)
end
end
{:noreply, next_socket}
end
def handle_info({:import_analysis_progress, definition_id, step, detail}, socket) do
{:noreply, ImportEditor.note_analysis_progress(socket, definition_id, step, detail, &reload_shell/2)}
end
def handle_info({:import_execution_progress, definition_id, phase, current, total, detail}, socket) do
{:noreply, ImportEditor.note_execution_progress(socket, definition_id, phase, current, total, detail, &reload_shell/2)}
end
def handle_info({:chat_tool_call, conversation_id, tool_call}, socket) do
{:noreply, ChatEditor.note_tool_call(socket, conversation_id, tool_call, &reload_shell/2)}
end

893
lib/bds/desktop/shell_live/import_editor.ex
View File

File diff suppressed because it is too large Load Diff

36
lib/bds/import_analysis.ex
View File

@@ -12,17 +12,30 @@ defmodule BDS.ImportAnalysis do
@shortcode_regex ~r/(?<!\[)\[(\w+)([^\]]*?)(?:\s*\/)?\](?!\])/u
@param_regex ~r/(\w+)=(?:"([^"]*)"|'([^']*)'|([^\s\]"']+))/u
def analyze_wxr(project_id, wxr_file_path, uploads_folder_path \\ nil)
def analyze_wxr(project_id, wxr_file_path), do: analyze_wxr(project_id, wxr_file_path, nil, [])
def analyze_wxr(project_id, wxr_file_path, uploads_folder_path)
when is_binary(project_id) and is_binary(wxr_file_path) do
analyze_wxr(project_id, wxr_file_path, uploads_folder_path, [])
end
def analyze_wxr(project_id, wxr_file_path, uploads_folder_path, opts)
when is_binary(project_id) and is_binary(wxr_file_path) and is_list(opts) do
on_progress = Keyword.get(opts, :on_progress, fn _step, _detail -> :ok end)
wxr_data = WxrParser.parse_file(wxr_file_path)
{:ok, build_report(project_id, wxr_data, wxr_file_path, uploads_folder_path)}
{:ok, build_report(project_id, wxr_data, wxr_file_path, uploads_folder_path, on_progress)}
rescue
error -> {:error, %{message: Exception.message(error)}}
end
defp build_report(project_id, wxr_data, wxr_file_path, uploads_folder_path) do
defp build_report(project_id, wxr_data, wxr_file_path, uploads_folder_path, on_progress) do
notify_progress(on_progress, "Loading existing posts...")
existing_posts = Repo.all(from post in Post, where: post.project_id == ^project_id)
notify_progress(on_progress, "Loading existing media...", "#{length(existing_posts)} posts in project")
existing_media = Repo.all(from media in Media, where: media.project_id == ^project_id)
notify_progress(on_progress, "Loading existing tags...", "#{length(existing_media)} media in project")
existing_tag_names = Repo.all(from tag in Tag, where: tag.project_id == ^project_id, select: tag.name)
existing_tag_set = existing_tag_names |> Enum.map(&String.downcase/1) |> MapSet.new()
@@ -40,15 +53,22 @@ defmodule BDS.ImportAnalysis do
|> Enum.reject(&is_nil(&1.checksum))
|> Map.new(&{&1.checksum, &1})
notify_progress(on_progress, "Analyzing posts...", "#{length(wxr_data.posts)} posts to analyze")
analyzed_posts = Enum.map(wxr_data.posts, &analyze_post_item(&1, posts_by_slug, posts_by_checksum, "post"))
notify_progress(on_progress, "Analyzing pages...", "#{length(wxr_data.pages)} pages to analyze")
analyzed_pages = Enum.map(wxr_data.pages, &analyze_post_item(&1, posts_by_slug, posts_by_checksum, "page"))
notify_progress(on_progress, "Analyzing media files...", "#{length(wxr_data.media)} media files to analyze")
analyzed_media =
Enum.map(wxr_data.media, &analyze_media_item(&1, uploads_folder_path, media_by_name, media_by_checksum))
notify_progress(on_progress, "Processing categories and tags...")
category_items = Enum.map(wxr_data.categories, &analyze_taxonomy_item(&1, existing_tag_set))
tag_items = Enum.map(wxr_data.tags, &analyze_taxonomy_item(&1, existing_tag_set))
notify_progress(on_progress, "Discovering macros...")
%{
source_file: wxr_file_path,
site_info: %{
@@ -312,6 +332,16 @@ defmodule BDS.ImportAnalysis do
defp count_status(items, status), do: Enum.count(items, &(&1.status == status))
defp notify_progress(callback, step, detail \\ nil) when is_function(callback, 2) do
try do
callback.(step, detail)
rescue
_error -> :ok
end
:ok
end
defp sha256(value) do
:sha256
|> :crypto.hash(value)

95
lib/bds/import_execution.ex
View File

@@ -11,6 +11,12 @@ defmodule BDS.ImportExecution do
def execute_import(project_id, report, opts \\ []) when is_binary(project_id) and is_map(report) do
normalized_report = normalize_report(report)
default_author = Keyword.get(opts, :default_author) || project_default_author(project_id)
uploads_folder_path = Keyword.get(opts, :uploads_folder_path)
on_progress = Keyword.get(opts, :on_progress, fn _phase, _current, _total, _detail -> :ok end)
taxonomies = taxonomy_items(normalized_report)
post_items = import_items(normalized_report, :posts)
page_items = import_items(normalized_report, :pages)
media_items = import_items(normalized_report, :media)
result = %{
success: true,
@@ -21,49 +27,73 @@ defmodule BDS.ImportExecution do
errors: []
}
result = execute_taxonomies(normalized_report, project_id, result)
result = execute_posts(normalized_report, project_id, default_author, result)
result = execute_pages(normalized_report, project_id, default_author, result)
notify_progress(on_progress, "tags", 0, length(taxonomies), "Creating tags...")
result = execute_taxonomies(taxonomies, project_id, result, on_progress)
{:ok, execute_media(normalized_report, project_id, default_author, result)}
notify_progress(on_progress, "posts", 0, length(post_items), "Importing posts...")
result = execute_posts(post_items, project_id, default_author, result, on_progress)
notify_progress(on_progress, "pages", 0, length(page_items), "Importing pages...")
result = execute_pages(page_items, project_id, default_author, result, on_progress)
notify_progress(on_progress, "media", 0, length(media_items), "Importing media...")
result = execute_media(media_items, project_id, default_author, result, on_progress, uploads_folder_path)
notify_progress(on_progress, "complete", 1, 1, "Import complete")
{:ok, result}
rescue
error -> {:error, %{message: Exception.message(error)}}
end
defp execute_taxonomies(report, project_id, result) do
taxonomies = List.wrap(get_in(report, [:items, :categories])) ++ List.wrap(get_in(report, [:items, :tags]))
defp execute_taxonomies(taxonomies, project_id, result, on_progress) do
Enum.reduce(taxonomies, result, fn item, acc ->
current = acc.tags.created + acc.tags.skipped + 1
if item.exists_in_project || item.mapped_to do
notify_progress(on_progress, "tags", current, length(taxonomies), "Skipping tag: #{item.name}")
put_in(acc, [:tags, :skipped], acc.tags.skipped + 1)
else
case Tags.create_tag(%{project_id: project_id, name: item.name}) do
{:ok, _tag} -> put_in(acc, [:tags, :created], acc.tags.created + 1)
{:error, _reason} -> put_in(acc, [:tags, :skipped], acc.tags.skipped + 1)
{:ok, _tag} ->
notify_progress(on_progress, "tags", current, length(taxonomies), "Created tag: #{item.name}")
put_in(acc, [:tags, :created], acc.tags.created + 1)
{:error, _reason} ->
notify_progress(on_progress, "tags", current, length(taxonomies), "Skipping tag: #{item.name}")
put_in(acc, [:tags, :skipped], acc.tags.skipped + 1)
end
end
end)
end
defp execute_posts(report, project_id, default_author, result) do
items = import_items(report, :posts)
defp execute_posts(items, project_id, default_author, result, on_progress) do
total = length(items)
Enum.reduce(items, result, fn item, acc ->
Enum.with_index(items, 1)
|> Enum.reduce(result, fn {item, index}, acc ->
notify_progress(on_progress, "posts", index, total, "Processing: #{item.title}")
execute_post_item(project_id, item, acc, :posts, default_author)
end)
end
defp execute_pages(report, project_id, default_author, result) do
items = import_items(report, :pages)
defp execute_pages(items, project_id, default_author, result, on_progress) do
total = length(items)
Enum.reduce(items, result, fn item, acc ->
Enum.with_index(items, 1)
|> Enum.reduce(result, fn {item, index}, acc ->
notify_progress(on_progress, "pages", index, total, "Processing: #{item.title}")
execute_post_item(project_id, ensure_page_category(item), acc, :pages, default_author)
end)
end
defp execute_media(report, project_id, default_author, result) do
import_items(report, :media)
|> Enum.reduce(result, fn item, acc ->
defp execute_media(items, project_id, default_author, result, on_progress, uploads_folder_path) do
total = length(items)
items
|> Enum.with_index(1)
|> Enum.reduce(result, fn {item, index}, acc ->
notify_progress(on_progress, "media", index, total, "Processing: #{item.filename}")
cond do
item.status in ["update", "duplicate", "missing"] ->
put_in(acc, [:media, :skipped], acc.media.skipped + 1)
@@ -72,7 +102,7 @@ defmodule BDS.ImportExecution do
put_in(acc, [:media, :skipped], acc.media.skipped + 1)
true ->
case import_media_item(project_id, item, default_author) do
case import_media_item(project_id, item, default_author, uploads_folder_path) do
{:ok, _media} -> put_in(acc, [:media, :imported], acc.media.imported + 1)
{:error, reason} ->
acc
@@ -141,8 +171,8 @@ defmodule BDS.ImportExecution do
end
end
defp import_media_item(project_id, item, default_author) do
source_path = item.source_file || Path.join("", item.relative_path)
defp import_media_item(project_id, item, default_author, uploads_folder_path) do
source_path = item.source_file || uploads_source_path(item.relative_path, uploads_folder_path)
checksum = if(source_path != nil and File.exists?(source_path), do: md5(File.read!(source_path)), else: nil)
if source_path && File.exists?(source_path) do
@@ -293,6 +323,29 @@ defmodule BDS.ImportExecution do
defp parse_timestamp(_value), do: nil
defp taxonomy_items(report) do
List.wrap(get_in(report, [:items, :categories])) ++ List.wrap(get_in(report, [:items, :tags]))
end
defp uploads_source_path(relative_path, uploads_folder_path)
defp uploads_source_path(relative_path, uploads_folder_path)
when is_binary(relative_path) and is_binary(uploads_folder_path) and uploads_folder_path != "" do
Path.join(uploads_folder_path, relative_path)
end
defp uploads_source_path(_relative_path, _uploads_folder_path), do: nil
defp notify_progress(callback, phase, current, total, detail) when is_function(callback, 4) do
try do
callback.(phase, current, total, detail)
rescue
_error -> :ok
end
:ok
end
defp md5(binary) do
:md5
|> :crypto.hash(binary)