fix: fixed media quick actions usage for images

2026-05-03 14:24:59 +02:00
parent 5bc2b4a338
commit 556f33711f
11 changed files with 560 additions and 39 deletions
--- a/lib/bds/ai/one_shot.ex
+++ b/lib/bds/ai/one_shot.ex
@@ -1,6 +1,8 @@
 defmodule BDS.AI.OneShot do
  @moduledoc false

+  require Logger
+
  alias BDS.AI.Chat
  alias BDS.AI.OpenAICompatibleRuntime
  alias BDS.AI.Runtime
@@ -8,6 +10,7 @@ defmodule BDS.AI.OneShot do
  alias BDS.MapUtils
  alias BDS.Posts
  alias BDS.Posts.Post
+  alias BDS.Projects
  alias BDS.Repo

  @default_max_output_tokens 16_384
@@ -163,6 +166,22 @@ defmodule BDS.AI.OneShot do
    end
  end

+  defp run_one_shot(:analyze_image = operation, payload, opts, formatter) do
+    runtime = Keyword.get(opts, :runtime, OpenAICompatibleRuntime)
+
+    with {:ok, endpoint, model, mode} <- Runtime.resolve_target(operation, opts),
+         :ok <- Runtime.validate_target(operation, model, mode),
+         {:ok, payload} <- resolve_image_data_url(payload),
+         request <- build_one_shot_request(operation, payload, model),
+         {:ok, response} <-
+           runtime.generate(Runtime.endpoint_with_model(endpoint, model), request, opts),
+         {:ok, json} <- extract_json_response(response),
+         usage <- Chat.normalize_usage(response.usage),
+         {:ok, result} <- formatter.(json, usage) do
+      {:ok, result}
+    end
+  end
+
  defp run_one_shot(operation, payload, opts, formatter) do
    runtime = Keyword.get(opts, :runtime, OpenAICompatibleRuntime)

@@ -273,12 +292,25 @@ defmodule BDS.AI.OneShot do

  defp extract_json_response(%{content: content}) when is_binary(content) do
    case Jason.decode(content) do
-      {:ok, json} when is_map(json) -> {:ok, json}
-      _other -> {:error, %{kind: :invalid_json_response}}
+      {:ok, json} when is_map(json) ->
+        {:ok, json}
+
+      _other ->
+        Logger.error(
+          "AI extract_json_response failed to parse content as JSON. Content: #{String.slice(content, 0, 1000)}"
+        )
+
+        {:error, %{kind: :invalid_json_response, content: content}}
    end
  end

-  defp extract_json_response(_response), do: {:error, %{kind: :invalid_json_response}}
+  defp extract_json_response(response) do
+    Logger.error(
+      "AI extract_json_response received response with no JSON and no content: #{inspect(Map.take(response, [:content, :json, :tool_calls]))}"
+    )
+
+    {:error, %{kind: :invalid_json_response}}
+  end

  defp normalize_post_input(%Post{} = post) do
    {:ok, %{title: post.title || "", excerpt: post.excerpt || "", content: Posts.editor_body(post)}}
@@ -307,7 +339,9 @@ defmodule BDS.AI.OneShot do
       title: media.title || "",
       alt: media.alt || "",
       caption: media.caption || "",
-       image_url: Map.get(media, :image_url) || media_path_to_file_url(media.file_path)
+       image_url: Map.get(media, :image_url),
+       file_path: media.file_path,
+       project_id: media.project_id
     }}
  end

@@ -325,15 +359,69 @@ defmodule BDS.AI.OneShot do
       title: MapUtils.attr(attrs, :title) || "",
       alt: MapUtils.attr(attrs, :alt) || "",
       caption: MapUtils.attr(attrs, :caption) || "",
-       image_url: MapUtils.attr(attrs, :image_url)
+       image_url: MapUtils.attr(attrs, :image_url),
+       file_path: MapUtils.attr(attrs, :file_path),
+       project_id: MapUtils.attr(attrs, :project_id)
     }}
  end

  defp ensure_image_media(%{mime_type: "image/" <> _rest}), do: :ok
  defp ensure_image_media(_media), do: {:error, %{kind: :invalid_media_type}}

-  defp media_path_to_file_url(nil), do: nil
-  defp media_path_to_file_url(path), do: "file://" <> path
+  defp resolve_image_data_url(%{image_url: "data:" <> _} = media) do
+    Logger.debug("AI analyze_image using existing data URL")
+    {:ok, media}
+  end
+
+  defp resolve_image_data_url(%{image_url: "http" <> _} = media) do
+    Logger.debug("AI analyze_image using HTTP URL: #{media.image_url}")
+    {:ok, media}
+  end
+
+  defp resolve_image_data_url(%{image_url: "file://" <> path, mime_type: mime_type} = media) do
+    with {:ok, binary} <- File.read(path) do
+      data_url = "data:#{mime_type};base64," <> Base.encode64(binary)
+      Logger.debug("AI analyze_image converted file://#{path} to data URL (#{byte_size(data_url)} chars)")
+      {:ok, %{media | image_url: data_url}}
+    else
+      {:error, reason} ->
+        Logger.error("AI analyze_image failed to read file://#{path}: #{inspect(reason)}")
+        {:error, :file_not_found}
+    end
+  end
+
+  defp resolve_image_data_url(%{file_path: file_path, project_id: project_id, mime_type: mime_type} = media)
+       when is_binary(file_path) and is_binary(project_id) do
+    case Projects.get_project(project_id) do
+      nil ->
+        Logger.error("AI analyze_image project not found: #{project_id}")
+        {:error, :file_not_found}
+
+      project ->
+        absolute_path = Path.join(Projects.project_data_dir(project), file_path)
+
+        case File.read(absolute_path) do
+          {:ok, binary} ->
+            data_url = "data:#{mime_type};base64," <> Base.encode64(binary)
+            Logger.debug("AI analyze_image converted #{absolute_path} to data URL (#{byte_size(data_url)} chars)")
+            {:ok, %{media | image_url: data_url}}
+
+          {:error, reason} ->
+            Logger.error("AI analyze_image failed to read #{absolute_path}: #{inspect(reason)}")
+            {:error, :file_not_found}
+        end
+    end
+  end
+
+  defp resolve_image_data_url(%{image_url: url} = media) when is_binary(url) and url != "" do
+    Logger.debug("AI analyze_image using URL: #{url}")
+    {:ok, media}
+  end
+
+  defp resolve_image_data_url(_media) do
+    Logger.error("AI analyze_image missing image source (no file_path, project_id, or image_url)")
+    {:error, :missing_image_source}
+  end

  defp normalize_string_list(values) do
    values
--- a/lib/bds/ai/openai_compatible_runtime.ex
+++ b/lib/bds/ai/openai_compatible_runtime.ex
@@ -41,16 +41,43 @@ defmodule BDS.AI.OpenAICompatibleRuntime do
      |> maybe_disable_thinking(request.model)
      |> maybe_put_tools(Map.get(request, :tools, []))

+    payload_json = Jason.encode!(payload)
+
    Logger.debug(
-      "AI OpenAI-compatible request operation=#{inspect(Map.get(request, :operation))} model=#{inspect(request.model)} url=#{url} tools=#{payload |> Map.get("tools", []) |> length()}"
+      "AI OpenAI-compatible request operation=#{inspect(Map.get(request, :operation))} model=#{inspect(request.model)} url=#{url} tools=#{payload |> Map.get("tools", []) |> length()} payload_size=#{byte_size(payload_json)}"
    )

-    with {:ok, response} <- HttpClient.post(url, headers, Jason.encode!(payload)),
-         200 <- response.status do
-      normalize_response(response.body)
-    else
-      status when is_integer(status) -> {:error, %{kind: :http_error, status: status}}
-      {:error, reason} -> {:error, %{kind: :http_error, reason: reason}}
+    case HttpClient.post(url, headers, payload_json) do
+      {:ok, %{status: 200, body: body}} ->
+        result = normalize_response(body)
+
+        case result do
+          {:ok, %{json: nil, content: content}} when is_binary(content) ->
+            Logger.warning(
+              "AI OpenAI-compatible response parsed but content is not valid JSON. Content: #{String.slice(content, 0, 500)}"
+            )
+
+          {:ok, _} ->
+            :ok
+
+          {:error, reason} ->
+            Logger.error(
+              "AI OpenAI-compatible response normalization failed: #{inspect(reason)} body=#{String.slice(body, 0, 1000)}"
+            )
+        end
+
+        result
+
+      {:ok, %{status: status, body: body}} ->
+        Logger.error(
+          "AI OpenAI-compatible HTTP error status=#{status} body=#{String.slice(body, 0, 2000)}"
+        )
+
+        {:error, %{kind: :http_error, status: status, body: body}}
+
+      {:error, reason} ->
+        Logger.error("AI OpenAI-compatible HTTP request failed: #{inspect(reason)}")
+        {:error, %{kind: :http_error, reason: reason}}
    end
  end

--- a/lib/bds/ai/runtime.ex
+++ b/lib/bds/ai/runtime.ex
@@ -32,10 +32,17 @@ defmodule BDS.AI.Runtime do

  @spec validate_target(atom(), String.t(), :airplane | :online) :: :ok | {:error, term()}
  def validate_target(:analyze_image, model, _mode) do
-    if Catalog.model_capabilities(model).supports_attachment do
-      :ok
-    else
-      {:error, %{kind: :model_capability_missing, capability: :supports_attachment, model: model}}
+    capabilities = Catalog.model_capabilities(model)
+
+    cond do
+      capabilities.supports_attachment ->
+        :ok
+
+      capabilities.supports_attachment == false ->
+        {:error, %{kind: :model_capability_missing, capability: :supports_attachment, model: model}}
+
+      true ->
+        :ok
    end
  end

--- a/lib/bds/desktop/overlay.ex
+++ b/lib/bds/desktop/overlay.ex
@@ -248,6 +248,12 @@ defmodule BDS.Desktop.Overlay do

  def set_ai_suggestions(overlay, _suggestions), do: overlay

+  def set_ai_suggestions_error(%{kind: :ai_suggestions} = overlay, error_message) do
+    Map.put(overlay, :error, error_message)
+  end
+
+  def set_ai_suggestions_error(overlay, _error_message), do: overlay
+
  defp normalize_ai_fields(fields) do
    Enum.map(fields, fn field ->
      %{
--- a/lib/bds/desktop/shell_live.ex
+++ b/lib/bds/desktop/shell_live.ex
@@ -3,6 +3,8 @@ defmodule BDS.Desktop.ShellLive do

  use Phoenix.LiveView

+  require Logger
+
  import Phoenix.HTML

  alias BDS.{AI, BoundedAtoms, ImportDefinitions, Media, Posts, Scripts}
@@ -1174,20 +1176,22 @@ defmodule BDS.Desktop.ShellLive do
  end

  def handle_info({:ai_suggestions_error, type, id, reason}, socket) do
+    Logger.error("AI suggestions error type=#{type} id=#{id} reason=#{inspect(reason)}")
+
    socket =
      case socket.assigns[:shell_overlay] do
-        %{kind: :ai_suggestions} ->
+        %{kind: :ai_suggestions} = overlay ->
          current_tab = socket.assigns.current_tab

          if current_tab && current_tab.type == type && current_tab.id == id do
-            socket
-            |> assign(:shell_overlay, nil)
-            |> append_output_entry(
-              translated("AI Suggestions"),
-              inspect(reason),
-              nil,
-              "error"
-            )
+            message =
+              if is_map(reason) and Map.has_key?(reason, :kind) do
+                "#{reason.kind}: #{inspect(Map.drop(reason, [:kind]))}"
+              else
+                inspect(reason)
+              end
+
+            assign(socket, :shell_overlay, Overlay.set_ai_suggestions_error(overlay, message))
          else
            socket
          end
--- a/lib/bds/desktop/shell_live/overlay_html/shell_overlay.html.heex
+++ b/lib/bds/desktop/shell_live/overlay_html/shell_overlay.html.heex
@@ -9,6 +9,12 @@
            <button class="ai-suggestions-modal-close" type="button" phx-click="close_overlay">×</button>
          </div>
          <div class="ai-suggestions-modal-body">
+            <%= if Map.get(@shell_overlay, :error) do %>
+              <div class="ai-suggestions-error">
+                <strong><%= translated("Error") %></strong>
+                <span><%= @shell_overlay.error %></span>
+              </div>
+            <% end %>
            <div class="ai-suggestions-list">
              <%= for field <- @shell_overlay.fields do %>
                <div class="ai-suggestion-item">
--- a/lib/bds/desktop/shell_live/settings_editor/ai_settings.ex
+++ b/lib/bds/desktop/shell_live/settings_editor/ai_settings.ex
@@ -27,6 +27,10 @@ defmodule BDS.Desktop.ShellLive.SettingsEditor.AISettings do
        ),
      "online_title_model" => get_model_preference(:title),
      "online_image_analysis_model" => get_model_preference(:image_analysis),
+      "online_chat_images" =>
+        model_supports_images?(
+          get_model_preference(:image_analysis)
+        ),
      "offline_url" => Map.get(airplane_endpoint || %{}, :url, ""),
      "offline_api_key" => Map.get(airplane_endpoint || %{}, :api_key, ""),
      "offline_mode" => Map.get(assigns, :offline_mode, AI.airplane_mode?(true)),
@@ -43,6 +47,10 @@ defmodule BDS.Desktop.ShellLive.SettingsEditor.AISettings do
        ),
      "offline_title_model" => get_model_preference(:airplane_title),
      "offline_image_analysis_model" => get_model_preference(:airplane_image_analysis),
+      "offline_chat_images" =>
+        model_supports_images?(
+          get_model_preference(:airplane_image_analysis)
+        ),
      "system_prompt" => EditorSettings.get_global_setting("ai.system_prompt") || ""
    }
  end
@@ -112,9 +120,14 @@ defmodule BDS.Desktop.ShellLive.SettingsEditor.AISettings do
             attrs.online_chat_tools,
             attrs.online_chat_disable_reasoning
           ),
-         :ok <- maybe_put_model_preference(:title, attrs.online_title_model),
-         :ok <- maybe_put_model_preference(:image_analysis, attrs.online_image_analysis_model),
-         :ok <- maybe_put_model_preference(:airplane_chat, attrs.offline_chat_model),
+          :ok <- maybe_put_model_preference(:title, attrs.online_title_model),
+          :ok <- maybe_put_model_preference(:image_analysis, attrs.online_image_analysis_model),
+          :ok <-
+            maybe_put_image_model_capabilities(
+              attrs.online_image_analysis_model,
+              attrs.online_chat_images
+            ),
+          :ok <- maybe_put_model_preference(:airplane_chat, attrs.offline_chat_model),
         :ok <-
           maybe_put_chat_model_capabilities(
             attrs.offline_chat_model,
@@ -122,12 +135,17 @@ defmodule BDS.Desktop.ShellLive.SettingsEditor.AISettings do
             attrs.offline_chat_disable_reasoning
           ),
         :ok <- maybe_put_model_preference(:airplane_title, attrs.offline_title_model),
-         :ok <-
-           maybe_put_model_preference(
-             :airplane_image_analysis,
-             attrs.offline_image_analysis_model
-           ),
-         :ok <- EditorSettings.put_global_setting("ai.system_prompt", attrs.system_prompt) do
+          :ok <-
+            maybe_put_model_preference(
+              :airplane_image_analysis,
+              attrs.offline_image_analysis_model
+            ),
+          :ok <-
+            maybe_put_image_model_capabilities(
+              attrs.offline_image_analysis_model,
+              attrs.offline_chat_images
+            ),
+          :ok <- EditorSettings.put_global_setting("ai.system_prompt", attrs.system_prompt) do
      socket
      |> assign(:settings_editor_ai_draft, %{})
      |> assign(:offline_mode, attrs.offline_mode)
@@ -166,6 +184,7 @@ defmodule BDS.Desktop.ShellLive.SettingsEditor.AISettings do
      online_chat_disable_reasoning: truthy?(Map.get(draft, "online_chat_disable_reasoning")),
      online_title_model: blank_to_nil(Map.get(draft, "online_title_model")),
      online_image_analysis_model: blank_to_nil(Map.get(draft, "online_image_analysis_model")),
+      online_chat_images: truthy?(Map.get(draft, "online_chat_images")),
      offline_url: blank_to_nil(Map.get(draft, "offline_url")),
      offline_api_key: blank_to_nil(Map.get(draft, "offline_api_key")),
      offline_mode: truthy?(Map.get(draft, "offline_mode")),
@@ -174,6 +193,7 @@ defmodule BDS.Desktop.ShellLive.SettingsEditor.AISettings do
      offline_chat_disable_reasoning: truthy?(Map.get(draft, "offline_chat_disable_reasoning")),
      offline_title_model: blank_to_nil(Map.get(draft, "offline_title_model")),
      offline_image_analysis_model: blank_to_nil(Map.get(draft, "offline_image_analysis_model")),
+      offline_chat_images: truthy?(Map.get(draft, "offline_chat_images")),
      system_prompt: Map.get(draft, "system_prompt", "")
    }
  end
@@ -188,6 +208,7 @@ defmodule BDS.Desktop.ShellLive.SettingsEditor.AISettings do
        truthy?(Map.get(params, "online_chat_disable_reasoning")),
      "online_title_model" => Map.get(params, "online_title_model", ""),
      "online_image_analysis_model" => Map.get(params, "online_image_analysis_model", ""),
+      "online_chat_images" => truthy?(Map.get(params, "online_chat_images")),
      "offline_url" => Map.get(params, "offline_url", ""),
      "offline_api_key" => Map.get(params, "offline_api_key", ""),
      "offline_mode" => truthy?(Map.get(params, "offline_mode")),
@@ -197,6 +218,7 @@ defmodule BDS.Desktop.ShellLive.SettingsEditor.AISettings do
        truthy?(Map.get(params, "offline_chat_disable_reasoning")),
      "offline_title_model" => Map.get(params, "offline_title_model", ""),
      "offline_image_analysis_model" => Map.get(params, "offline_image_analysis_model", ""),
+      "offline_chat_images" => truthy?(Map.get(params, "offline_chat_images")),
      "system_prompt" => Map.get(params, "system_prompt", "")
    }
  end
@@ -225,12 +247,31 @@ defmodule BDS.Desktop.ShellLive.SettingsEditor.AISettings do
    })
  end

+  defp maybe_put_image_model_capabilities(nil, _supports_images), do: :ok
+  defp maybe_put_image_model_capabilities("", _supports_images), do: :ok
+
+  defp maybe_put_image_model_capabilities(model, supports_images) do
+    existing = BDS.AI.Catalog.model_capabilities(model)
+
+    AI.put_model_capabilities(model, %{
+      supports_attachment: supports_images,
+      supports_tool_calls: existing.supports_tool_calls,
+      disables_reasoning: existing.disables_reasoning
+    })
+  end
+
  defp model_supports_tool_calls?(nil), do: false
  defp model_supports_tool_calls?(""), do: false

  defp model_supports_tool_calls?(model),
    do: BDS.AI.Catalog.model_capabilities(model).supports_tool_calls

+  defp model_supports_images?(nil), do: false
+  defp model_supports_images?(""), do: false
+
+  defp model_supports_images?(model),
+    do: BDS.AI.Catalog.model_capabilities(model).supports_attachment
+
  defp model_disables_reasoning?(nil), do: false
  defp model_disables_reasoning?(""), do: false

--- a/lib/bds/desktop/shell_live/settings_editor_html/settings_editor.html.heex
+++ b/lib/bds/desktop/shell_live/settings_editor_html/settings_editor.html.heex
@@ -245,6 +245,10 @@
              <div class="setting-info"><label class="setting-label"><%= translated("Online Image Analysis Model") %></label></div>
              <div class="setting-control"><input type="text" list="settings-ai-online-models" name="settings_ai[online_image_analysis_model]" value={@settings_editor.ai["online_image_analysis_model"]} /></div>
            </div>
+            <div class="setting-row">
+              <div class="setting-info"><label class="setting-label"><%= translated("Online Image Support") %></label></div>
+              <div class="setting-control"><label><input type="checkbox" name="settings_ai[online_chat_images]" checked={@settings_editor.ai["online_chat_images"]} /> <%= translated("Enable image analysis for the online image analysis model") %></label></div>
+            </div>
            <div class="setting-row">
              <div class="setting-info"><label class="setting-label"><%= translated("Offline Endpoint URL") %></label></div>
              <div class="setting-control">
@@ -282,6 +286,10 @@
              <div class="setting-info"><label class="setting-label"><%= translated("Offline Image Analysis Model") %></label></div>
              <div class="setting-control"><input type="text" list="settings-ai-offline-models" name="settings_ai[offline_image_analysis_model]" value={@settings_editor.ai["offline_image_analysis_model"]} /></div>
            </div>
+            <div class="setting-row">
+              <div class="setting-info"><label class="setting-label"><%= translated("Offline Image Support") %></label></div>
+              <div class="setting-control"><label><input type="checkbox" name="settings_ai[offline_chat_images]" checked={@settings_editor.ai["offline_chat_images"]} /> <%= translated("Enable image analysis for the offline image analysis model") %></label></div>
+            </div>
            <div class="setting-row">
              <div class="setting-info"><label class="setting-label"><%= translated("System Prompt") %></label></div>
              <div class="setting-control"><textarea name="settings_ai[system_prompt]" rows="12"><%= @settings_editor.ai["system_prompt"] %></textarea></div>