fix: fixed media quick actions usage for images

This commit is contained in:
2026-05-03 14:24:59 +02:00
parent 5bc2b4a338
commit 556f33711f
11 changed files with 560 additions and 39 deletions

View File

@@ -1,6 +1,8 @@
defmodule BDS.AI.OneShot do defmodule BDS.AI.OneShot do
@moduledoc false @moduledoc false
require Logger
alias BDS.AI.Chat alias BDS.AI.Chat
alias BDS.AI.OpenAICompatibleRuntime alias BDS.AI.OpenAICompatibleRuntime
alias BDS.AI.Runtime alias BDS.AI.Runtime
@@ -8,6 +10,7 @@ defmodule BDS.AI.OneShot do
alias BDS.MapUtils alias BDS.MapUtils
alias BDS.Posts alias BDS.Posts
alias BDS.Posts.Post alias BDS.Posts.Post
alias BDS.Projects
alias BDS.Repo alias BDS.Repo
@default_max_output_tokens 16_384 @default_max_output_tokens 16_384
@@ -163,6 +166,22 @@ defmodule BDS.AI.OneShot do
end end
end end
defp run_one_shot(:analyze_image = operation, payload, opts, formatter) do
runtime = Keyword.get(opts, :runtime, OpenAICompatibleRuntime)
with {:ok, endpoint, model, mode} <- Runtime.resolve_target(operation, opts),
:ok <- Runtime.validate_target(operation, model, mode),
{:ok, payload} <- resolve_image_data_url(payload),
request <- build_one_shot_request(operation, payload, model),
{:ok, response} <-
runtime.generate(Runtime.endpoint_with_model(endpoint, model), request, opts),
{:ok, json} <- extract_json_response(response),
usage <- Chat.normalize_usage(response.usage),
{:ok, result} <- formatter.(json, usage) do
{:ok, result}
end
end
defp run_one_shot(operation, payload, opts, formatter) do defp run_one_shot(operation, payload, opts, formatter) do
runtime = Keyword.get(opts, :runtime, OpenAICompatibleRuntime) runtime = Keyword.get(opts, :runtime, OpenAICompatibleRuntime)
@@ -273,12 +292,25 @@ defmodule BDS.AI.OneShot do
defp extract_json_response(%{content: content}) when is_binary(content) do defp extract_json_response(%{content: content}) when is_binary(content) do
case Jason.decode(content) do case Jason.decode(content) do
{:ok, json} when is_map(json) -> {:ok, json} {:ok, json} when is_map(json) ->
_other -> {:error, %{kind: :invalid_json_response}} {:ok, json}
_other ->
Logger.error(
"AI extract_json_response failed to parse content as JSON. Content: #{String.slice(content, 0, 1000)}"
)
{:error, %{kind: :invalid_json_response, content: content}}
end end
end end
defp extract_json_response(_response), do: {:error, %{kind: :invalid_json_response}} defp extract_json_response(response) do
Logger.error(
"AI extract_json_response received response with no JSON and no content: #{inspect(Map.take(response, [:content, :json, :tool_calls]))}"
)
{:error, %{kind: :invalid_json_response}}
end
defp normalize_post_input(%Post{} = post) do defp normalize_post_input(%Post{} = post) do
{:ok, %{title: post.title || "", excerpt: post.excerpt || "", content: Posts.editor_body(post)}} {:ok, %{title: post.title || "", excerpt: post.excerpt || "", content: Posts.editor_body(post)}}
@@ -307,7 +339,9 @@ defmodule BDS.AI.OneShot do
title: media.title || "", title: media.title || "",
alt: media.alt || "", alt: media.alt || "",
caption: media.caption || "", caption: media.caption || "",
image_url: Map.get(media, :image_url) || media_path_to_file_url(media.file_path) image_url: Map.get(media, :image_url),
file_path: media.file_path,
project_id: media.project_id
}} }}
end end
@@ -325,15 +359,69 @@ defmodule BDS.AI.OneShot do
title: MapUtils.attr(attrs, :title) || "", title: MapUtils.attr(attrs, :title) || "",
alt: MapUtils.attr(attrs, :alt) || "", alt: MapUtils.attr(attrs, :alt) || "",
caption: MapUtils.attr(attrs, :caption) || "", caption: MapUtils.attr(attrs, :caption) || "",
image_url: MapUtils.attr(attrs, :image_url) image_url: MapUtils.attr(attrs, :image_url),
file_path: MapUtils.attr(attrs, :file_path),
project_id: MapUtils.attr(attrs, :project_id)
}} }}
end end
defp ensure_image_media(%{mime_type: "image/" <> _rest}), do: :ok defp ensure_image_media(%{mime_type: "image/" <> _rest}), do: :ok
defp ensure_image_media(_media), do: {:error, %{kind: :invalid_media_type}} defp ensure_image_media(_media), do: {:error, %{kind: :invalid_media_type}}
defp media_path_to_file_url(nil), do: nil defp resolve_image_data_url(%{image_url: "data:" <> _} = media) do
defp media_path_to_file_url(path), do: "file://" <> path Logger.debug("AI analyze_image using existing data URL")
{:ok, media}
end
defp resolve_image_data_url(%{image_url: "http" <> _} = media) do
Logger.debug("AI analyze_image using HTTP URL: #{media.image_url}")
{:ok, media}
end
defp resolve_image_data_url(%{image_url: "file://" <> path, mime_type: mime_type} = media) do
with {:ok, binary} <- File.read(path) do
data_url = "data:#{mime_type};base64," <> Base.encode64(binary)
Logger.debug("AI analyze_image converted file://#{path} to data URL (#{byte_size(data_url)} chars)")
{:ok, %{media | image_url: data_url}}
else
{:error, reason} ->
Logger.error("AI analyze_image failed to read file://#{path}: #{inspect(reason)}")
{:error, :file_not_found}
end
end
defp resolve_image_data_url(%{file_path: file_path, project_id: project_id, mime_type: mime_type} = media)
when is_binary(file_path) and is_binary(project_id) do
case Projects.get_project(project_id) do
nil ->
Logger.error("AI analyze_image project not found: #{project_id}")
{:error, :file_not_found}
project ->
absolute_path = Path.join(Projects.project_data_dir(project), file_path)
case File.read(absolute_path) do
{:ok, binary} ->
data_url = "data:#{mime_type};base64," <> Base.encode64(binary)
Logger.debug("AI analyze_image converted #{absolute_path} to data URL (#{byte_size(data_url)} chars)")
{:ok, %{media | image_url: data_url}}
{:error, reason} ->
Logger.error("AI analyze_image failed to read #{absolute_path}: #{inspect(reason)}")
{:error, :file_not_found}
end
end
end
defp resolve_image_data_url(%{image_url: url} = media) when is_binary(url) and url != "" do
Logger.debug("AI analyze_image using URL: #{url}")
{:ok, media}
end
defp resolve_image_data_url(_media) do
Logger.error("AI analyze_image missing image source (no file_path, project_id, or image_url)")
{:error, :missing_image_source}
end
defp normalize_string_list(values) do defp normalize_string_list(values) do
values values

View File

@@ -41,16 +41,43 @@ defmodule BDS.AI.OpenAICompatibleRuntime do
|> maybe_disable_thinking(request.model) |> maybe_disable_thinking(request.model)
|> maybe_put_tools(Map.get(request, :tools, [])) |> maybe_put_tools(Map.get(request, :tools, []))
payload_json = Jason.encode!(payload)
Logger.debug( Logger.debug(
"AI OpenAI-compatible request operation=#{inspect(Map.get(request, :operation))} model=#{inspect(request.model)} url=#{url} tools=#{payload |> Map.get("tools", []) |> length()}" "AI OpenAI-compatible request operation=#{inspect(Map.get(request, :operation))} model=#{inspect(request.model)} url=#{url} tools=#{payload |> Map.get("tools", []) |> length()} payload_size=#{byte_size(payload_json)}"
) )
with {:ok, response} <- HttpClient.post(url, headers, Jason.encode!(payload)), case HttpClient.post(url, headers, payload_json) do
200 <- response.status do {:ok, %{status: 200, body: body}} ->
normalize_response(response.body) result = normalize_response(body)
else
status when is_integer(status) -> {:error, %{kind: :http_error, status: status}} case result do
{:error, reason} -> {:error, %{kind: :http_error, reason: reason}} {:ok, %{json: nil, content: content}} when is_binary(content) ->
Logger.warning(
"AI OpenAI-compatible response parsed but content is not valid JSON. Content: #{String.slice(content, 0, 500)}"
)
{:ok, _} ->
:ok
{:error, reason} ->
Logger.error(
"AI OpenAI-compatible response normalization failed: #{inspect(reason)} body=#{String.slice(body, 0, 1000)}"
)
end
result
{:ok, %{status: status, body: body}} ->
Logger.error(
"AI OpenAI-compatible HTTP error status=#{status} body=#{String.slice(body, 0, 2000)}"
)
{:error, %{kind: :http_error, status: status, body: body}}
{:error, reason} ->
Logger.error("AI OpenAI-compatible HTTP request failed: #{inspect(reason)}")
{:error, %{kind: :http_error, reason: reason}}
end end
end end

View File

@@ -32,10 +32,17 @@ defmodule BDS.AI.Runtime do
@spec validate_target(atom(), String.t(), :airplane | :online) :: :ok | {:error, term()} @spec validate_target(atom(), String.t(), :airplane | :online) :: :ok | {:error, term()}
def validate_target(:analyze_image, model, _mode) do def validate_target(:analyze_image, model, _mode) do
if Catalog.model_capabilities(model).supports_attachment do capabilities = Catalog.model_capabilities(model)
:ok
else cond do
{:error, %{kind: :model_capability_missing, capability: :supports_attachment, model: model}} capabilities.supports_attachment ->
:ok
capabilities.supports_attachment == false ->
{:error, %{kind: :model_capability_missing, capability: :supports_attachment, model: model}}
true ->
:ok
end end
end end

View File

@@ -248,6 +248,12 @@ defmodule BDS.Desktop.Overlay do
def set_ai_suggestions(overlay, _suggestions), do: overlay def set_ai_suggestions(overlay, _suggestions), do: overlay
def set_ai_suggestions_error(%{kind: :ai_suggestions} = overlay, error_message) do
Map.put(overlay, :error, error_message)
end
def set_ai_suggestions_error(overlay, _error_message), do: overlay
defp normalize_ai_fields(fields) do defp normalize_ai_fields(fields) do
Enum.map(fields, fn field -> Enum.map(fields, fn field ->
%{ %{

View File

@@ -3,6 +3,8 @@ defmodule BDS.Desktop.ShellLive do
use Phoenix.LiveView use Phoenix.LiveView
require Logger
import Phoenix.HTML import Phoenix.HTML
alias BDS.{AI, BoundedAtoms, ImportDefinitions, Media, Posts, Scripts} alias BDS.{AI, BoundedAtoms, ImportDefinitions, Media, Posts, Scripts}
@@ -1174,20 +1176,22 @@ defmodule BDS.Desktop.ShellLive do
end end
def handle_info({:ai_suggestions_error, type, id, reason}, socket) do def handle_info({:ai_suggestions_error, type, id, reason}, socket) do
Logger.error("AI suggestions error type=#{type} id=#{id} reason=#{inspect(reason)}")
socket = socket =
case socket.assigns[:shell_overlay] do case socket.assigns[:shell_overlay] do
%{kind: :ai_suggestions} -> %{kind: :ai_suggestions} = overlay ->
current_tab = socket.assigns.current_tab current_tab = socket.assigns.current_tab
if current_tab && current_tab.type == type && current_tab.id == id do if current_tab && current_tab.type == type && current_tab.id == id do
socket message =
|> assign(:shell_overlay, nil) if is_map(reason) and Map.has_key?(reason, :kind) do
|> append_output_entry( "#{reason.kind}: #{inspect(Map.drop(reason, [:kind]))}"
translated("AI Suggestions"), else
inspect(reason), inspect(reason)
nil, end
"error"
) assign(socket, :shell_overlay, Overlay.set_ai_suggestions_error(overlay, message))
else else
socket socket
end end

View File

@@ -9,6 +9,12 @@
<button class="ai-suggestions-modal-close" type="button" phx-click="close_overlay">×</button> <button class="ai-suggestions-modal-close" type="button" phx-click="close_overlay">×</button>
</div> </div>
<div class="ai-suggestions-modal-body"> <div class="ai-suggestions-modal-body">
<%= if Map.get(@shell_overlay, :error) do %>
<div class="ai-suggestions-error">
<strong><%= translated("Error") %></strong>
<span><%= @shell_overlay.error %></span>
</div>
<% end %>
<div class="ai-suggestions-list"> <div class="ai-suggestions-list">
<%= for field <- @shell_overlay.fields do %> <%= for field <- @shell_overlay.fields do %>
<div class="ai-suggestion-item"> <div class="ai-suggestion-item">

View File

@@ -27,6 +27,10 @@ defmodule BDS.Desktop.ShellLive.SettingsEditor.AISettings do
), ),
"online_title_model" => get_model_preference(:title), "online_title_model" => get_model_preference(:title),
"online_image_analysis_model" => get_model_preference(:image_analysis), "online_image_analysis_model" => get_model_preference(:image_analysis),
"online_chat_images" =>
model_supports_images?(
get_model_preference(:image_analysis)
),
"offline_url" => Map.get(airplane_endpoint || %{}, :url, ""), "offline_url" => Map.get(airplane_endpoint || %{}, :url, ""),
"offline_api_key" => Map.get(airplane_endpoint || %{}, :api_key, ""), "offline_api_key" => Map.get(airplane_endpoint || %{}, :api_key, ""),
"offline_mode" => Map.get(assigns, :offline_mode, AI.airplane_mode?(true)), "offline_mode" => Map.get(assigns, :offline_mode, AI.airplane_mode?(true)),
@@ -43,6 +47,10 @@ defmodule BDS.Desktop.ShellLive.SettingsEditor.AISettings do
), ),
"offline_title_model" => get_model_preference(:airplane_title), "offline_title_model" => get_model_preference(:airplane_title),
"offline_image_analysis_model" => get_model_preference(:airplane_image_analysis), "offline_image_analysis_model" => get_model_preference(:airplane_image_analysis),
"offline_chat_images" =>
model_supports_images?(
get_model_preference(:airplane_image_analysis)
),
"system_prompt" => EditorSettings.get_global_setting("ai.system_prompt") || "" "system_prompt" => EditorSettings.get_global_setting("ai.system_prompt") || ""
} }
end end
@@ -112,9 +120,14 @@ defmodule BDS.Desktop.ShellLive.SettingsEditor.AISettings do
attrs.online_chat_tools, attrs.online_chat_tools,
attrs.online_chat_disable_reasoning attrs.online_chat_disable_reasoning
), ),
:ok <- maybe_put_model_preference(:title, attrs.online_title_model), :ok <- maybe_put_model_preference(:title, attrs.online_title_model),
:ok <- maybe_put_model_preference(:image_analysis, attrs.online_image_analysis_model), :ok <- maybe_put_model_preference(:image_analysis, attrs.online_image_analysis_model),
:ok <- maybe_put_model_preference(:airplane_chat, attrs.offline_chat_model), :ok <-
maybe_put_image_model_capabilities(
attrs.online_image_analysis_model,
attrs.online_chat_images
),
:ok <- maybe_put_model_preference(:airplane_chat, attrs.offline_chat_model),
:ok <- :ok <-
maybe_put_chat_model_capabilities( maybe_put_chat_model_capabilities(
attrs.offline_chat_model, attrs.offline_chat_model,
@@ -122,12 +135,17 @@ defmodule BDS.Desktop.ShellLive.SettingsEditor.AISettings do
attrs.offline_chat_disable_reasoning attrs.offline_chat_disable_reasoning
), ),
:ok <- maybe_put_model_preference(:airplane_title, attrs.offline_title_model), :ok <- maybe_put_model_preference(:airplane_title, attrs.offline_title_model),
:ok <- :ok <-
maybe_put_model_preference( maybe_put_model_preference(
:airplane_image_analysis, :airplane_image_analysis,
attrs.offline_image_analysis_model attrs.offline_image_analysis_model
), ),
:ok <- EditorSettings.put_global_setting("ai.system_prompt", attrs.system_prompt) do :ok <-
maybe_put_image_model_capabilities(
attrs.offline_image_analysis_model,
attrs.offline_chat_images
),
:ok <- EditorSettings.put_global_setting("ai.system_prompt", attrs.system_prompt) do
socket socket
|> assign(:settings_editor_ai_draft, %{}) |> assign(:settings_editor_ai_draft, %{})
|> assign(:offline_mode, attrs.offline_mode) |> assign(:offline_mode, attrs.offline_mode)
@@ -166,6 +184,7 @@ defmodule BDS.Desktop.ShellLive.SettingsEditor.AISettings do
online_chat_disable_reasoning: truthy?(Map.get(draft, "online_chat_disable_reasoning")), online_chat_disable_reasoning: truthy?(Map.get(draft, "online_chat_disable_reasoning")),
online_title_model: blank_to_nil(Map.get(draft, "online_title_model")), online_title_model: blank_to_nil(Map.get(draft, "online_title_model")),
online_image_analysis_model: blank_to_nil(Map.get(draft, "online_image_analysis_model")), online_image_analysis_model: blank_to_nil(Map.get(draft, "online_image_analysis_model")),
online_chat_images: truthy?(Map.get(draft, "online_chat_images")),
offline_url: blank_to_nil(Map.get(draft, "offline_url")), offline_url: blank_to_nil(Map.get(draft, "offline_url")),
offline_api_key: blank_to_nil(Map.get(draft, "offline_api_key")), offline_api_key: blank_to_nil(Map.get(draft, "offline_api_key")),
offline_mode: truthy?(Map.get(draft, "offline_mode")), offline_mode: truthy?(Map.get(draft, "offline_mode")),
@@ -174,6 +193,7 @@ defmodule BDS.Desktop.ShellLive.SettingsEditor.AISettings do
offline_chat_disable_reasoning: truthy?(Map.get(draft, "offline_chat_disable_reasoning")), offline_chat_disable_reasoning: truthy?(Map.get(draft, "offline_chat_disable_reasoning")),
offline_title_model: blank_to_nil(Map.get(draft, "offline_title_model")), offline_title_model: blank_to_nil(Map.get(draft, "offline_title_model")),
offline_image_analysis_model: blank_to_nil(Map.get(draft, "offline_image_analysis_model")), offline_image_analysis_model: blank_to_nil(Map.get(draft, "offline_image_analysis_model")),
offline_chat_images: truthy?(Map.get(draft, "offline_chat_images")),
system_prompt: Map.get(draft, "system_prompt", "") system_prompt: Map.get(draft, "system_prompt", "")
} }
end end
@@ -188,6 +208,7 @@ defmodule BDS.Desktop.ShellLive.SettingsEditor.AISettings do
truthy?(Map.get(params, "online_chat_disable_reasoning")), truthy?(Map.get(params, "online_chat_disable_reasoning")),
"online_title_model" => Map.get(params, "online_title_model", ""), "online_title_model" => Map.get(params, "online_title_model", ""),
"online_image_analysis_model" => Map.get(params, "online_image_analysis_model", ""), "online_image_analysis_model" => Map.get(params, "online_image_analysis_model", ""),
"online_chat_images" => truthy?(Map.get(params, "online_chat_images")),
"offline_url" => Map.get(params, "offline_url", ""), "offline_url" => Map.get(params, "offline_url", ""),
"offline_api_key" => Map.get(params, "offline_api_key", ""), "offline_api_key" => Map.get(params, "offline_api_key", ""),
"offline_mode" => truthy?(Map.get(params, "offline_mode")), "offline_mode" => truthy?(Map.get(params, "offline_mode")),
@@ -197,6 +218,7 @@ defmodule BDS.Desktop.ShellLive.SettingsEditor.AISettings do
truthy?(Map.get(params, "offline_chat_disable_reasoning")), truthy?(Map.get(params, "offline_chat_disable_reasoning")),
"offline_title_model" => Map.get(params, "offline_title_model", ""), "offline_title_model" => Map.get(params, "offline_title_model", ""),
"offline_image_analysis_model" => Map.get(params, "offline_image_analysis_model", ""), "offline_image_analysis_model" => Map.get(params, "offline_image_analysis_model", ""),
"offline_chat_images" => truthy?(Map.get(params, "offline_chat_images")),
"system_prompt" => Map.get(params, "system_prompt", "") "system_prompt" => Map.get(params, "system_prompt", "")
} }
end end
@@ -225,12 +247,31 @@ defmodule BDS.Desktop.ShellLive.SettingsEditor.AISettings do
}) })
end end
defp maybe_put_image_model_capabilities(nil, _supports_images), do: :ok
defp maybe_put_image_model_capabilities("", _supports_images), do: :ok
defp maybe_put_image_model_capabilities(model, supports_images) do
existing = BDS.AI.Catalog.model_capabilities(model)
AI.put_model_capabilities(model, %{
supports_attachment: supports_images,
supports_tool_calls: existing.supports_tool_calls,
disables_reasoning: existing.disables_reasoning
})
end
defp model_supports_tool_calls?(nil), do: false defp model_supports_tool_calls?(nil), do: false
defp model_supports_tool_calls?(""), do: false defp model_supports_tool_calls?(""), do: false
defp model_supports_tool_calls?(model), defp model_supports_tool_calls?(model),
do: BDS.AI.Catalog.model_capabilities(model).supports_tool_calls do: BDS.AI.Catalog.model_capabilities(model).supports_tool_calls
defp model_supports_images?(nil), do: false
defp model_supports_images?(""), do: false
defp model_supports_images?(model),
do: BDS.AI.Catalog.model_capabilities(model).supports_attachment
defp model_disables_reasoning?(nil), do: false defp model_disables_reasoning?(nil), do: false
defp model_disables_reasoning?(""), do: false defp model_disables_reasoning?(""), do: false

View File

@@ -245,6 +245,10 @@
<div class="setting-info"><label class="setting-label"><%= translated("Online Image Analysis Model") %></label></div> <div class="setting-info"><label class="setting-label"><%= translated("Online Image Analysis Model") %></label></div>
<div class="setting-control"><input type="text" list="settings-ai-online-models" name="settings_ai[online_image_analysis_model]" value={@settings_editor.ai["online_image_analysis_model"]} /></div> <div class="setting-control"><input type="text" list="settings-ai-online-models" name="settings_ai[online_image_analysis_model]" value={@settings_editor.ai["online_image_analysis_model"]} /></div>
</div> </div>
<div class="setting-row">
<div class="setting-info"><label class="setting-label"><%= translated("Online Image Support") %></label></div>
<div class="setting-control"><label><input type="checkbox" name="settings_ai[online_chat_images]" checked={@settings_editor.ai["online_chat_images"]} /> <%= translated("Enable image analysis for the online image analysis model") %></label></div>
</div>
<div class="setting-row"> <div class="setting-row">
<div class="setting-info"><label class="setting-label"><%= translated("Offline Endpoint URL") %></label></div> <div class="setting-info"><label class="setting-label"><%= translated("Offline Endpoint URL") %></label></div>
<div class="setting-control"> <div class="setting-control">
@@ -282,6 +286,10 @@
<div class="setting-info"><label class="setting-label"><%= translated("Offline Image Analysis Model") %></label></div> <div class="setting-info"><label class="setting-label"><%= translated("Offline Image Analysis Model") %></label></div>
<div class="setting-control"><input type="text" list="settings-ai-offline-models" name="settings_ai[offline_image_analysis_model]" value={@settings_editor.ai["offline_image_analysis_model"]} /></div> <div class="setting-control"><input type="text" list="settings-ai-offline-models" name="settings_ai[offline_image_analysis_model]" value={@settings_editor.ai["offline_image_analysis_model"]} /></div>
</div> </div>
<div class="setting-row">
<div class="setting-info"><label class="setting-label"><%= translated("Offline Image Support") %></label></div>
<div class="setting-control"><label><input type="checkbox" name="settings_ai[offline_chat_images]" checked={@settings_editor.ai["offline_chat_images"]} /> <%= translated("Enable image analysis for the offline image analysis model") %></label></div>
</div>
<div class="setting-row"> <div class="setting-row">
<div class="setting-info"><label class="setting-label"><%= translated("System Prompt") %></label></div> <div class="setting-info"><label class="setting-label"><%= translated("System Prompt") %></label></div>
<div class="setting-control"><textarea name="settings_ai[system_prompt]" rows="12"><%= @settings_editor.ai["system_prompt"] %></textarea></div> <div class="setting-control"><textarea name="settings_ai[system_prompt]" rows="12"><%= @settings_editor.ai["system_prompt"] %></textarea></div>

View File

@@ -2505,6 +2505,18 @@ button svg * {
font-style: italic; font-style: italic;
} }
.ai-suggestions-error {
display: flex;
flex-direction: column;
gap: 4px;
padding: 12px 16px;
margin-bottom: 16px;
border-radius: 6px;
background: rgba(220, 50, 50, 0.12);
border: 1px solid rgba(220, 50, 50, 0.35);
color: #ff6b6b;
}
.ai-suggestions-modal-footer, .ai-suggestions-modal-footer,
.confirm-delete-modal-footer, .confirm-delete-modal-footer,
.confirm-dialog-actions { .confirm-dialog-actions {

View File

@@ -130,6 +130,47 @@ defmodule BDS.AITest do
end end
end end
defmodule ErrorCompletionServer do
use Plug.Router
plug(:match)
plug(:dispatch)
post "/v1/chat/completions" do
{:ok, body, conn} = Plug.Conn.read_body(conn)
send(Application.fetch_env!(:bds, :test_pid), {:error_payload, Jason.decode!(body)})
response = %{
"error" => %{
"message" => "Invalid image data",
"type" => "invalid_request_error"
}
}
conn
|> Plug.Conn.put_resp_content_type("application/json")
|> Plug.Conn.send_resp(400, Jason.encode!(response))
end
end
defmodule NonJsonContentServer do
use Plug.Router
plug(:match)
plug(:dispatch)
post "/v1/chat/completions" do
response = %{
"choices" => [%{"message" => %{"content" => "This is not valid JSON"}}],
"usage" => %{"prompt_tokens" => 4, "completion_tokens" => 2}
}
conn
|> Plug.Conn.put_resp_content_type("application/json")
|> Plug.Conn.send_resp(200, Jason.encode!(response))
end
end
defmodule FakeRuntime do defmodule FakeRuntime do
def generate(endpoint, request, opts) do def generate(endpoint, request, opts) do
test_pid = Keyword.fetch!(opts, :test_pid) test_pid = Keyword.fetch!(opts, :test_pid)
@@ -410,6 +451,99 @@ defmodule BDS.AITest do
assert payload["chat_template_kwargs"] == %{"enable_thinking" => false} assert payload["chat_template_kwargs"] == %{"enable_thinking" => false}
end end
test "openai-compatible generation includes response body in HTTP error details" do
Application.put_env(:bds, :test_pid, self())
server =
start_supervised!({Bandit, plug: ErrorCompletionServer, port: 0, startup_log: false})
{:ok, {_address, port}} = ThousandIsland.listener_info(server)
previous_level = Logger.level()
Logger.configure(level: :error)
log =
capture_log(fn ->
assert {:error, %{kind: :http_error, status: 400, body: body}} =
BDS.AI.OpenAICompatibleRuntime.generate(
%{url: "http://127.0.0.1:#{port}/v1", api_key: nil},
%{
model: "gpt-test",
messages: [%{"role" => "user", "content" => "Hello"}],
max_output_tokens: 128,
tools: []
},
[]
)
assert body =~ "Invalid image data"
end)
Logger.configure(level: previous_level)
assert log =~ "AI OpenAI-compatible HTTP error status=400"
assert log =~ "Invalid image data"
assert_received {:error_payload, payload}
assert payload["model"] == "gpt-test"
end
test "analyze_image logs non-JSON content when the model returns invalid JSON" do
assert {:ok, _endpoint} =
BDS.AI.put_endpoint(
:airplane,
%{
url: "http://localhost:11434/v1",
api_key: nil,
model: "llama-default"
},
secret_backend: FakeSecretBackend
)
assert :ok = BDS.AI.set_airplane_mode(true)
assert :ok = BDS.AI.put_model_preference(:airplane_image_analysis, "llava")
assert :ok =
BDS.AI.put_model_capabilities("llava", %{
supports_attachment: true,
supports_tool_calls: false,
disables_reasoning: false
})
defmodule NonJsonContentRuntime do
def generate(_endpoint, _request, _opts) do
{:ok,
%{
content: "This is not valid JSON",
json: nil,
tool_calls: [],
usage: %{input_tokens: 4, output_tokens: 2}
}}
end
end
previous_level = Logger.level()
Logger.configure(level: :error)
log =
capture_log(fn ->
assert {:error, %{kind: :invalid_json_response, content: "This is not valid JSON"}} =
BDS.AI.analyze_image(
%{
mime_type: "image/png",
image_url: "data:image/png;base64,abc123"
},
runtime: NonJsonContentRuntime,
test_pid: self(),
secret_backend: FakeSecretBackend
)
end)
Logger.configure(level: previous_level)
assert log =~ "AI extract_json_response failed to parse content as JSON"
assert log =~ "This is not valid JSON"
end
test "airplane mode routes title tasks to airplane endpoint and offline title model" do test "airplane mode routes title tasks to airplane endpoint and offline title model" do
assert {:ok, _endpoint} = assert {:ok, _endpoint} =
BDS.AI.put_endpoint( BDS.AI.put_endpoint(
@@ -582,7 +716,7 @@ defmodule BDS.AITest do
title: "Source", title: "Source",
alt: nil, alt: nil,
caption: nil, caption: nil,
image_url: "file:///tmp/test.png" image_url: "https://example.com/test.png"
}, },
runtime: FakeRuntime, runtime: FakeRuntime,
test_pid: self(), test_pid: self(),
@@ -603,7 +737,7 @@ defmodule BDS.AITest do
title: "Source", title: "Source",
alt: nil, alt: nil,
caption: nil, caption: nil,
image_url: "file:///tmp/test.png" image_url: "https://example.com/test.png"
}, },
runtime: FakeRuntime, runtime: FakeRuntime,
test_pid: self(), test_pid: self(),
@@ -619,6 +753,120 @@ defmodule BDS.AITest do
assert BDS.AI.Catalog.model_capabilities("llama3.2").disables_reasoning assert BDS.AI.Catalog.model_capabilities("llama3.2").disables_reasoning
end end
test "analyze_image converts file:// URLs to base64 data URLs before sending" do
assert {:ok, _endpoint} =
BDS.AI.put_endpoint(
:airplane,
%{
url: "http://localhost:11434/v1",
api_key: nil,
model: "llama-default"
},
secret_backend: FakeSecretBackend
)
assert :ok = BDS.AI.set_airplane_mode(true)
assert :ok = BDS.AI.put_model_preference(:airplane_image_analysis, "llama3.2")
assert :ok =
BDS.AI.put_model_capabilities("llama3.2", %{
supports_attachment: true,
supports_tool_calls: false,
disables_reasoning: false
})
tmp_path =
Path.join(System.tmp_dir!(), "bds-test-image-#{System.unique_integer([:positive])}.png")
File.write!(tmp_path, <<137, 80, 78, 71, 13, 10, 26, 10>>)
on_exit(fn -> File.rm(tmp_path) end)
assert {:ok, analysis} =
BDS.AI.analyze_image(
%{
mime_type: "image/png",
title: "Source",
alt: nil,
caption: nil,
image_url: "file://#{tmp_path}"
},
runtime: FakeRuntime,
test_pid: self(),
secret_backend: FakeSecretBackend
)
assert analysis.title == "Sunset"
assert_received {:runtime_request, _endpoint, request}
user_message = Enum.at(request.messages, 1)
image_content = Enum.at(user_message["content"], 1)
assert image_content["type"] == "image_url"
assert image_content["image_url"]["url"] =~ ~r/^data:image\/png;base64,/
end
test "analyze_image reads local media files and sends them as base64 data URLs" do
{:ok, project} = create_project_fixture("Image Analysis")
image_dir = Path.join(project.data_path, "media")
File.mkdir_p!(image_dir)
image_path = Path.join(image_dir, "test.png")
File.write!(image_path, <<137, 80, 78, 71, 13, 10, 26, 10>>)
media =
Repo.insert!(
Media.changeset(%Media{}, %{
id: Ecto.UUID.generate(),
project_id: project.id,
filename: "test.png",
original_name: "test.png",
mime_type: "image/png",
size: 8,
title: "Test",
file_path: "media/test.png",
sidecar_path: "media/test.png.meta",
created_at: Persistence.now_ms(),
updated_at: Persistence.now_ms()
})
)
assert {:ok, _endpoint} =
BDS.AI.put_endpoint(
:airplane,
%{
url: "http://localhost:11434/v1",
api_key: nil,
model: "llama-default"
},
secret_backend: FakeSecretBackend
)
assert :ok = BDS.AI.set_airplane_mode(true)
assert :ok = BDS.AI.put_model_preference(:airplane_image_analysis, "llama3.2")
assert :ok =
BDS.AI.put_model_capabilities("llama3.2", %{
supports_attachment: true,
supports_tool_calls: false,
disables_reasoning: false
})
assert {:ok, analysis} =
BDS.AI.analyze_image(
media.id,
runtime: FakeRuntime,
test_pid: self(),
secret_backend: FakeSecretBackend
)
assert analysis.title == "Sunset"
assert_received {:runtime_request, _endpoint, request}
user_message = Enum.at(request.messages, 1)
image_content = Enum.at(user_message["content"], 1)
assert image_content["type"] == "image_url"
assert image_content["image_url"]["url"] =~ ~r/^data:image\/png;base64,/
end
test "chat persists user, tool, and assistant messages with usage and blog stats prompt augmentation" do test "chat persists user, tool, and assistant messages with usage and blog stats prompt augmentation" do
{:ok, project} = create_project_fixture("AI Chat") {:ok, project} = create_project_fixture("AI Chat")
_fixtures = seed_project_content(project.id) _fixtures = seed_project_content(project.id)

View File

@@ -2460,6 +2460,78 @@ defmodule BDS.Desktop.ShellLiveTest do
assert html =~ "Automatic AI actions stay gated by airplane mode" assert html =~ "Automatic AI actions stay gated by airplane mode"
end end
test "ai suggestions overlay fetches async results for media when online", %{project: project} do
Application.put_env(:bds, :test_pid, self())
server =
start_supervised!({Bandit, plug: AiSuggestionsServer, port: 0, startup_log: false})
{:ok, {_address, port}} = ThousandIsland.listener_info(server)
assert :ok = AI.set_airplane_mode(false)
assert {:ok, _endpoint} =
AI.put_endpoint(:online, %{
url: "http://127.0.0.1:#{port}/v1",
api_key: "test-secret",
model: "gpt-test"
})
assert :ok = AI.put_model_preference(:image_analysis, "gpt-test")
assert :ok = AI.put_model_capabilities("gpt-test", %{supports_attachment: true})
temp_dir =
Path.join(System.tmp_dir!(), "bds-shell-live-#{System.unique_integer([:positive])}")
File.mkdir_p!(temp_dir)
media_source_path = Path.join(temp_dir, "online-media.jpg")
File.write!(media_source_path, "fake image body")
{:ok, media} =
Media.import_media(%{
project_id: project.id,
source_path: media_source_path,
title: "Online Media"
})
{:ok, view, _html} = live_isolated(build_conn(), BDS.Desktop.ShellLive)
html =
render_click(view, "pin_sidebar_item", %{
"route" => "media",
"id" => media.id,
"title" => media.title,
"subtitle" => "draft"
})
assert html =~ ~s(data-testid="media-editor")
html =
view
|> element("[data-testid='media-editor'] .quick-actions-btn")
|> render_click()
assert html =~ "quick-actions-menu"
html =
view
|> element("[phx-click='open_overlay'][phx-value-kind='ai_suggestions']")
|> render_click()
assert html =~ "ai-suggestions-modal"
assert html =~ "Loading"
assert_receive {:ai_suggestions_request, _request}, 2_000
Process.sleep(200)
html = render(view)
assert html =~ "AI Image Title"
assert html =~ "AI Alt Text"
assert html =~ "AI Caption"
refute html =~ "Loading"
end
test "ai suggestions async error closes overlay and shows toast", %{project: project} do test "ai suggestions async error closes overlay and shows toast", %{project: project} do
Application.put_env(:bds, :test_pid, self()) Application.put_env(:bds, :test_pid, self())
@@ -2515,7 +2587,9 @@ defmodule BDS.Desktop.ShellLiveTest do
send(view.pid, {:ai_suggestions_error, :post, post.id, :test_error}) send(view.pid, {:ai_suggestions_error, :post, post.id, :test_error})
html = render(view) html = render(view)
refute html =~ "ai-suggestions-modal" assert html =~ "ai-suggestions-modal"
assert html =~ "ai-suggestions-error"
assert html =~ "test_error"
end end
test "script and template editors surface lifecycle state, load published file content, and allow publishing drafts", test "script and template editors surface lifecycle state, load published file content, and allow publishing drafts",