hugo/bDS2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: fixed media quick actions usage for images

Browse Source
This commit is contained in:
Chili Palmer
2026-05-03 14:24:59 +02:00
parent 5bc2b4a338
commit 556f33711f
11 changed files with 560 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

102
lib/bds/ai/one_shot.ex
View File

@@ -1,6 +1,8 @@
defmodule BDS.AI.OneShot do
@moduledoc false
require Logger
alias BDS.AI.Chat
alias BDS.AI.OpenAICompatibleRuntime
alias BDS.AI.Runtime
@@ -8,6 +10,7 @@ defmodule BDS.AI.OneShot do
alias BDS.MapUtils
alias BDS.Posts
alias BDS.Posts.Post
alias BDS.Projects
alias BDS.Repo
@default_max_output_tokens 16_384
@@ -163,6 +166,22 @@ defmodule BDS.AI.OneShot do
end
end
defp run_one_shot(:analyze_image = operation, payload, opts, formatter) do
runtime = Keyword.get(opts, :runtime, OpenAICompatibleRuntime)
with {:ok, endpoint, model, mode} <- Runtime.resolve_target(operation, opts),
:ok <- Runtime.validate_target(operation, model, mode),
{:ok, payload} <- resolve_image_data_url(payload),
request <- build_one_shot_request(operation, payload, model),
{:ok, response} <-
runtime.generate(Runtime.endpoint_with_model(endpoint, model), request, opts),
{:ok, json} <- extract_json_response(response),
usage <- Chat.normalize_usage(response.usage),
{:ok, result} <- formatter.(json, usage) do
{:ok, result}
end
end
defp run_one_shot(operation, payload, opts, formatter) do
runtime = Keyword.get(opts, :runtime, OpenAICompatibleRuntime)
@@ -273,12 +292,25 @@ defmodule BDS.AI.OneShot do
defp extract_json_response(%{content: content}) when is_binary(content) do
case Jason.decode(content) do
{:ok, json} when is_map(json) -> {:ok, json}
_other -> {:error, %{kind: :invalid_json_response}}
{:ok, json} when is_map(json) ->
{:ok, json}
_other ->
Logger.error(
"AI extract_json_response failed to parse content as JSON. Content: #{String.slice(content, 0, 1000)}"
)
{:error, %{kind: :invalid_json_response, content: content}}
end
end
defp extract_json_response(_response), do: {:error, %{kind: :invalid_json_response}}
defp extract_json_response(response) do
Logger.error(
"AI extract_json_response received response with no JSON and no content: #{inspect(Map.take(response, [:content, :json, :tool_calls]))}"
)
{:error, %{kind: :invalid_json_response}}
end
defp normalize_post_input(%Post{} = post) do
{:ok, %{title: post.title || "", excerpt: post.excerpt || "", content: Posts.editor_body(post)}}
@@ -307,7 +339,9 @@ defmodule BDS.AI.OneShot do
title: media.title || "",
alt: media.alt || "",
caption: media.caption || "",
image_url: Map.get(media, :image_url) || media_path_to_file_url(media.file_path)
image_url: Map.get(media, :image_url),
file_path: media.file_path,
project_id: media.project_id
}}
end
@@ -325,15 +359,69 @@ defmodule BDS.AI.OneShot do
title: MapUtils.attr(attrs, :title) || "",
alt: MapUtils.attr(attrs, :alt) || "",
caption: MapUtils.attr(attrs, :caption) || "",
image_url: MapUtils.attr(attrs, :image_url)
image_url: MapUtils.attr(attrs, :image_url),
file_path: MapUtils.attr(attrs, :file_path),
project_id: MapUtils.attr(attrs, :project_id)
}}
end
defp ensure_image_media(%{mime_type: "image/" <> _rest}), do: :ok
defp ensure_image_media(_media), do: {:error, %{kind: :invalid_media_type}}
defp media_path_to_file_url(nil), do: nil
defp media_path_to_file_url(path), do: "file://" <> path
defp resolve_image_data_url(%{image_url: "data:" <> _} = media) do
Logger.debug("AI analyze_image using existing data URL")
{:ok, media}
end
defp resolve_image_data_url(%{image_url: "http" <> _} = media) do
Logger.debug("AI analyze_image using HTTP URL: #{media.image_url}")
{:ok, media}
end
defp resolve_image_data_url(%{image_url: "file://" <> path, mime_type: mime_type} = media) do
with {:ok, binary} <- File.read(path) do
data_url = "data:#{mime_type};base64," <> Base.encode64(binary)
Logger.debug("AI analyze_image converted file://#{path} to data URL (#{byte_size(data_url)} chars)")
{:ok, %{media | image_url: data_url}}
else
{:error, reason} ->
Logger.error("AI analyze_image failed to read file://#{path}: #{inspect(reason)}")
{:error, :file_not_found}
end
end
defp resolve_image_data_url(%{file_path: file_path, project_id: project_id, mime_type: mime_type} = media)
when is_binary(file_path) and is_binary(project_id) do
case Projects.get_project(project_id) do
nil ->
Logger.error("AI analyze_image project not found: #{project_id}")
{:error, :file_not_found}
project ->
absolute_path = Path.join(Projects.project_data_dir(project), file_path)
case File.read(absolute_path) do
{:ok, binary} ->
data_url = "data:#{mime_type};base64," <> Base.encode64(binary)
Logger.debug("AI analyze_image converted #{absolute_path} to data URL (#{byte_size(data_url)} chars)")
{:ok, %{media | image_url: data_url}}
{:error, reason} ->
Logger.error("AI analyze_image failed to read #{absolute_path}: #{inspect(reason)}")
{:error, :file_not_found}
end
end
end
defp resolve_image_data_url(%{image_url: url} = media) when is_binary(url) and url != "" do
Logger.debug("AI analyze_image using URL: #{url}")
{:ok, media}
end
defp resolve_image_data_url(_media) do
Logger.error("AI analyze_image missing image source (no file_path, project_id, or image_url)")
{:error, :missing_image_source}
end
defp normalize_string_list(values) do
values

41
lib/bds/ai/openai_compatible_runtime.ex
View File

@@ -41,16 +41,43 @@ defmodule BDS.AI.OpenAICompatibleRuntime do
|> maybe_disable_thinking(request.model)
|> maybe_put_tools(Map.get(request, :tools, []))
payload_json = Jason.encode!(payload)
Logger.debug(
"AI OpenAI-compatible request operation=#{inspect(Map.get(request, :operation))} model=#{inspect(request.model)} url=#{url} tools=#{payload |> Map.get("tools", []) |> length()}"
"AI OpenAI-compatible request operation=#{inspect(Map.get(request, :operation))} model=#{inspect(request.model)} url=#{url} tools=#{payload |> Map.get("tools", []) |> length()} payload_size=#{byte_size(payload_json)}"
)
with {:ok, response} <- HttpClient.post(url, headers, Jason.encode!(payload)),
200 <- response.status do
normalize_response(response.body)
else
status when is_integer(status) -> {:error, %{kind: :http_error, status: status}}
{:error, reason} -> {:error, %{kind: :http_error, reason: reason}}
case HttpClient.post(url, headers, payload_json) do
{:ok, %{status: 200, body: body}} ->
result = normalize_response(body)
case result do
{:ok, %{json: nil, content: content}} when is_binary(content) ->
Logger.warning(
"AI OpenAI-compatible response parsed but content is not valid JSON. Content: #{String.slice(content, 0, 500)}"
)
{:ok, _} ->
:ok
{:error, reason} ->
Logger.error(
"AI OpenAI-compatible response normalization failed: #{inspect(reason)} body=#{String.slice(body, 0, 1000)}"
)
end
result
{:ok, %{status: status, body: body}} ->
Logger.error(
"AI OpenAI-compatible HTTP error status=#{status} body=#{String.slice(body, 0, 2000)}"
)
{:error, %{kind: :http_error, status: status, body: body}}
{:error, reason} ->
Logger.error("AI OpenAI-compatible HTTP request failed: #{inspect(reason)}")
{:error, %{kind: :http_error, reason: reason}}
end
end

15
lib/bds/ai/runtime.ex
View File

@@ -32,10 +32,17 @@ defmodule BDS.AI.Runtime do
@spec validate_target(atom(), String.t(), :airplane | :online) :: :ok | {:error, term()}
def validate_target(:analyze_image, model, _mode) do
if Catalog.model_capabilities(model).supports_attachment do
:ok
else
{:error, %{kind: :model_capability_missing, capability: :supports_attachment, model: model}}
capabilities = Catalog.model_capabilities(model)
cond do
capabilities.supports_attachment ->
:ok
capabilities.supports_attachment == false ->
{:error, %{kind: :model_capability_missing, capability: :supports_attachment, model: model}}
true ->
:ok
end
end

6
lib/bds/desktop/overlay.ex
View File

@@ -248,6 +248,12 @@ defmodule BDS.Desktop.Overlay do
def set_ai_suggestions(overlay, _suggestions), do: overlay
def set_ai_suggestions_error(%{kind: :ai_suggestions} = overlay, error_message) do
Map.put(overlay, :error, error_message)
end
def set_ai_suggestions_error(overlay, _error_message), do: overlay
defp normalize_ai_fields(fields) do
Enum.map(fields, fn field ->
%{

22
lib/bds/desktop/shell_live.ex
View File

@@ -3,6 +3,8 @@ defmodule BDS.Desktop.ShellLive do
use Phoenix.LiveView
require Logger
import Phoenix.HTML
alias BDS.{AI, BoundedAtoms, ImportDefinitions, Media, Posts, Scripts}
@@ -1174,20 +1176,22 @@ defmodule BDS.Desktop.ShellLive do
end
def handle_info({:ai_suggestions_error, type, id, reason}, socket) do
Logger.error("AI suggestions error type=#{type} id=#{id} reason=#{inspect(reason)}")
socket =
case socket.assigns[:shell_overlay] do
%{kind: :ai_suggestions} ->
%{kind: :ai_suggestions} = overlay ->
current_tab = socket.assigns.current_tab
if current_tab && current_tab.type == type && current_tab.id == id do
socket
|> assign(:shell_overlay, nil)
|> append_output_entry(
translated("AI Suggestions"),
inspect(reason),
nil,
"error"
)
message =
if is_map(reason) and Map.has_key?(reason, :kind) do
"#{reason.kind}: #{inspect(Map.drop(reason, [:kind]))}"
else
inspect(reason)
end
assign(socket, :shell_overlay, Overlay.set_ai_suggestions_error(overlay, message))
else
socket
end

6
lib/bds/desktop/shell_live/overlay_html/shell_overlay.html.heex
View File

@@ -9,6 +9,12 @@
<button class="ai-suggestions-modal-close" type="button" phx-click="close_overlay">×</button>
</div>
<div class="ai-suggestions-modal-body">
<%= if Map.get(@shell_overlay, :error) do %>
<div class="ai-suggestions-error">
<strong><%= translated("Error") %></strong>
<span><%= @shell_overlay.error %></span>
</div>
<% end %>
<div class="ai-suggestions-list">
<%= for field <- @shell_overlay.fields do %>
<div class="ai-suggestion-item">

59
lib/bds/desktop/shell_live/settings_editor/ai_settings.ex
View File

@@ -27,6 +27,10 @@ defmodule BDS.Desktop.ShellLive.SettingsEditor.AISettings do
),
"online_title_model" => get_model_preference(:title),
"online_image_analysis_model" => get_model_preference(:image_analysis),
"online_chat_images" =>
model_supports_images?(
get_model_preference(:image_analysis)
),
"offline_url" => Map.get(airplane_endpoint || %{}, :url, ""),
"offline_api_key" => Map.get(airplane_endpoint || %{}, :api_key, ""),
"offline_mode" => Map.get(assigns, :offline_mode, AI.airplane_mode?(true)),
@@ -43,6 +47,10 @@ defmodule BDS.Desktop.ShellLive.SettingsEditor.AISettings do
),
"offline_title_model" => get_model_preference(:airplane_title),
"offline_image_analysis_model" => get_model_preference(:airplane_image_analysis),
"offline_chat_images" =>
model_supports_images?(
get_model_preference(:airplane_image_analysis)
),
"system_prompt" => EditorSettings.get_global_setting("ai.system_prompt") || ""
}
end
@@ -112,9 +120,14 @@ defmodule BDS.Desktop.ShellLive.SettingsEditor.AISettings do
attrs.online_chat_tools,
attrs.online_chat_disable_reasoning
),
:ok <- maybe_put_model_preference(:title, attrs.online_title_model),
:ok <- maybe_put_model_preference(:image_analysis, attrs.online_image_analysis_model),
:ok <- maybe_put_model_preference(:airplane_chat, attrs.offline_chat_model),
:ok <- maybe_put_model_preference(:title, attrs.online_title_model),
:ok <- maybe_put_model_preference(:image_analysis, attrs.online_image_analysis_model),
:ok <-
maybe_put_image_model_capabilities(
attrs.online_image_analysis_model,
attrs.online_chat_images
),
:ok <- maybe_put_model_preference(:airplane_chat, attrs.offline_chat_model),
:ok <-
maybe_put_chat_model_capabilities(
attrs.offline_chat_model,
@@ -122,12 +135,17 @@ defmodule BDS.Desktop.ShellLive.SettingsEditor.AISettings do
attrs.offline_chat_disable_reasoning
),
:ok <- maybe_put_model_preference(:airplane_title, attrs.offline_title_model),
:ok <-
maybe_put_model_preference(
:airplane_image_analysis,
attrs.offline_image_analysis_model
),
:ok <- EditorSettings.put_global_setting("ai.system_prompt", attrs.system_prompt) do
:ok <-
maybe_put_model_preference(
:airplane_image_analysis,
attrs.offline_image_analysis_model
),
:ok <-
maybe_put_image_model_capabilities(
attrs.offline_image_analysis_model,
attrs.offline_chat_images
),
:ok <- EditorSettings.put_global_setting("ai.system_prompt", attrs.system_prompt) do
socket
|> assign(:settings_editor_ai_draft, %{})
|> assign(:offline_mode, attrs.offline_mode)
@@ -166,6 +184,7 @@ defmodule BDS.Desktop.ShellLive.SettingsEditor.AISettings do
online_chat_disable_reasoning: truthy?(Map.get(draft, "online_chat_disable_reasoning")),
online_title_model: blank_to_nil(Map.get(draft, "online_title_model")),
online_image_analysis_model: blank_to_nil(Map.get(draft, "online_image_analysis_model")),
online_chat_images: truthy?(Map.get(draft, "online_chat_images")),
offline_url: blank_to_nil(Map.get(draft, "offline_url")),
offline_api_key: blank_to_nil(Map.get(draft, "offline_api_key")),
offline_mode: truthy?(Map.get(draft, "offline_mode")),
@@ -174,6 +193,7 @@ defmodule BDS.Desktop.ShellLive.SettingsEditor.AISettings do
offline_chat_disable_reasoning: truthy?(Map.get(draft, "offline_chat_disable_reasoning")),
offline_title_model: blank_to_nil(Map.get(draft, "offline_title_model")),
offline_image_analysis_model: blank_to_nil(Map.get(draft, "offline_image_analysis_model")),
offline_chat_images: truthy?(Map.get(draft, "offline_chat_images")),
system_prompt: Map.get(draft, "system_prompt", "")
}
end
@@ -188,6 +208,7 @@ defmodule BDS.Desktop.ShellLive.SettingsEditor.AISettings do
truthy?(Map.get(params, "online_chat_disable_reasoning")),
"online_title_model" => Map.get(params, "online_title_model", ""),
"online_image_analysis_model" => Map.get(params, "online_image_analysis_model", ""),
"online_chat_images" => truthy?(Map.get(params, "online_chat_images")),
"offline_url" => Map.get(params, "offline_url", ""),
"offline_api_key" => Map.get(params, "offline_api_key", ""),
"offline_mode" => truthy?(Map.get(params, "offline_mode")),
@@ -197,6 +218,7 @@ defmodule BDS.Desktop.ShellLive.SettingsEditor.AISettings do
truthy?(Map.get(params, "offline_chat_disable_reasoning")),
"offline_title_model" => Map.get(params, "offline_title_model", ""),
"offline_image_analysis_model" => Map.get(params, "offline_image_analysis_model", ""),
"offline_chat_images" => truthy?(Map.get(params, "offline_chat_images")),
"system_prompt" => Map.get(params, "system_prompt", "")
}
end
@@ -225,12 +247,31 @@ defmodule BDS.Desktop.ShellLive.SettingsEditor.AISettings do
})
end
defp maybe_put_image_model_capabilities(nil, _supports_images), do: :ok
defp maybe_put_image_model_capabilities("", _supports_images), do: :ok
defp maybe_put_image_model_capabilities(model, supports_images) do
existing = BDS.AI.Catalog.model_capabilities(model)
AI.put_model_capabilities(model, %{
supports_attachment: supports_images,
supports_tool_calls: existing.supports_tool_calls,
disables_reasoning: existing.disables_reasoning
})
end
defp model_supports_tool_calls?(nil), do: false
defp model_supports_tool_calls?(""), do: false
defp model_supports_tool_calls?(model),
do: BDS.AI.Catalog.model_capabilities(model).supports_tool_calls
defp model_supports_images?(nil), do: false
defp model_supports_images?(""), do: false
defp model_supports_images?(model),
do: BDS.AI.Catalog.model_capabilities(model).supports_attachment
defp model_disables_reasoning?(nil), do: false
defp model_disables_reasoning?(""), do: false

8
lib/bds/desktop/shell_live/settings_editor_html/settings_editor.html.heex
View File

@@ -245,6 +245,10 @@
<div class="setting-info"><label class="setting-label"><%= translated("Online Image Analysis Model") %></label></div>
<div class="setting-control"><input type="text" list="settings-ai-online-models" name="settings_ai[online_image_analysis_model]" value={@settings_editor.ai["online_image_analysis_model"]} /></div>
</div>
<div class="setting-row">
<div class="setting-info"><label class="setting-label"><%= translated("Online Image Support") %></label></div>
<div class="setting-control"><label><input type="checkbox" name="settings_ai[online_chat_images]" checked={@settings_editor.ai["online_chat_images"]} /> <%= translated("Enable image analysis for the online image analysis model") %></label></div>
</div>
<div class="setting-row">
<div class="setting-info"><label class="setting-label"><%= translated("Offline Endpoint URL") %></label></div>
<div class="setting-control">
@@ -282,6 +286,10 @@
<div class="setting-info"><label class="setting-label"><%= translated("Offline Image Analysis Model") %></label></div>
<div class="setting-control"><input type="text" list="settings-ai-offline-models" name="settings_ai[offline_image_analysis_model]" value={@settings_editor.ai["offline_image_analysis_model"]} /></div>
</div>
<div class="setting-row">
<div class="setting-info"><label class="setting-label"><%= translated("Offline Image Support") %></label></div>
<div class="setting-control"><label><input type="checkbox" name="settings_ai[offline_chat_images]" checked={@settings_editor.ai["offline_chat_images"]} /> <%= translated("Enable image analysis for the offline image analysis model") %></label></div>
</div>
<div class="setting-row">
<div class="setting-info"><label class="setting-label"><%= translated("System Prompt") %></label></div>
<div class="setting-control"><textarea name="settings_ai[system_prompt]" rows="12"><%= @settings_editor.ai["system_prompt"] %></textarea></div>

12
priv/ui/app.css
View File

@@ -2505,6 +2505,18 @@ button svg * {
font-style: italic;
}
.ai-suggestions-error {
display: flex;
flex-direction: column;
gap: 4px;
padding: 12px 16px;
margin-bottom: 16px;
border-radius: 6px;
background: rgba(220, 50, 50, 0.12);
border: 1px solid rgba(220, 50, 50, 0.35);
color: #ff6b6b;
}
.ai-suggestions-modal-footer,
.confirm-delete-modal-footer,
.confirm-dialog-actions {

252
test/bds/ai_test.exs
View File

@@ -130,6 +130,47 @@ defmodule BDS.AITest do
end
end
defmodule ErrorCompletionServer do
use Plug.Router
plug(:match)
plug(:dispatch)
post "/v1/chat/completions" do
{:ok, body, conn} = Plug.Conn.read_body(conn)
send(Application.fetch_env!(:bds, :test_pid), {:error_payload, Jason.decode!(body)})
response = %{
"error" => %{
"message" => "Invalid image data",
"type" => "invalid_request_error"
}
}
conn
|> Plug.Conn.put_resp_content_type("application/json")
|> Plug.Conn.send_resp(400, Jason.encode!(response))
end
end
defmodule NonJsonContentServer do
use Plug.Router
plug(:match)
plug(:dispatch)
post "/v1/chat/completions" do
response = %{
"choices" => [%{"message" => %{"content" => "This is not valid JSON"}}],
"usage" => %{"prompt_tokens" => 4, "completion_tokens" => 2}
}
conn
|> Plug.Conn.put_resp_content_type("application/json")
|> Plug.Conn.send_resp(200, Jason.encode!(response))
end
end
defmodule FakeRuntime do
def generate(endpoint, request, opts) do
test_pid = Keyword.fetch!(opts, :test_pid)
@@ -410,6 +451,99 @@ defmodule BDS.AITest do
assert payload["chat_template_kwargs"] == %{"enable_thinking" => false}
end
test "openai-compatible generation includes response body in HTTP error details" do
Application.put_env(:bds, :test_pid, self())
server =
start_supervised!({Bandit, plug: ErrorCompletionServer, port: 0, startup_log: false})
{:ok, {_address, port}} = ThousandIsland.listener_info(server)
previous_level = Logger.level()
Logger.configure(level: :error)
log =
capture_log(fn ->
assert {:error, %{kind: :http_error, status: 400, body: body}} =
BDS.AI.OpenAICompatibleRuntime.generate(
%{url: "http://127.0.0.1:#{port}/v1", api_key: nil},
%{
model: "gpt-test",
messages: [%{"role" => "user", "content" => "Hello"}],
max_output_tokens: 128,
tools: []
},
[]
)
assert body =~ "Invalid image data"
end)
Logger.configure(level: previous_level)
assert log =~ "AI OpenAI-compatible HTTP error status=400"
assert log =~ "Invalid image data"
assert_received {:error_payload, payload}
assert payload["model"] == "gpt-test"
end
test "analyze_image logs non-JSON content when the model returns invalid JSON" do
assert {:ok, _endpoint} =
BDS.AI.put_endpoint(
:airplane,
%{
url: "http://localhost:11434/v1",
api_key: nil,
model: "llama-default"
},
secret_backend: FakeSecretBackend
)
assert :ok = BDS.AI.set_airplane_mode(true)
assert :ok = BDS.AI.put_model_preference(:airplane_image_analysis, "llava")
assert :ok =
BDS.AI.put_model_capabilities("llava", %{
supports_attachment: true,
supports_tool_calls: false,
disables_reasoning: false
})
defmodule NonJsonContentRuntime do
def generate(_endpoint, _request, _opts) do
{:ok,
%{
content: "This is not valid JSON",
json: nil,
tool_calls: [],
usage: %{input_tokens: 4, output_tokens: 2}
}}
end
end
previous_level = Logger.level()
Logger.configure(level: :error)
log =
capture_log(fn ->
assert {:error, %{kind: :invalid_json_response, content: "This is not valid JSON"}} =
BDS.AI.analyze_image(
%{
mime_type: "image/png",
image_url: "data:image/png;base64,abc123"
},
runtime: NonJsonContentRuntime,
test_pid: self(),
secret_backend: FakeSecretBackend
)
end)
Logger.configure(level: previous_level)
assert log =~ "AI extract_json_response failed to parse content as JSON"
assert log =~ "This is not valid JSON"
end
test "airplane mode routes title tasks to airplane endpoint and offline title model" do
assert {:ok, _endpoint} =
BDS.AI.put_endpoint(
@@ -582,7 +716,7 @@ defmodule BDS.AITest do
title: "Source",
alt: nil,
caption: nil,
image_url: "file:///tmp/test.png"
image_url: "https://example.com/test.png"
},
runtime: FakeRuntime,
test_pid: self(),
@@ -603,7 +737,7 @@ defmodule BDS.AITest do
title: "Source",
alt: nil,
caption: nil,
image_url: "file:///tmp/test.png"
image_url: "https://example.com/test.png"
},
runtime: FakeRuntime,
test_pid: self(),
@@ -619,6 +753,120 @@ defmodule BDS.AITest do
assert BDS.AI.Catalog.model_capabilities("llama3.2").disables_reasoning
end
test "analyze_image converts file:// URLs to base64 data URLs before sending" do
assert {:ok, _endpoint} =
BDS.AI.put_endpoint(
:airplane,
%{
url: "http://localhost:11434/v1",
api_key: nil,
model: "llama-default"
},
secret_backend: FakeSecretBackend
)
assert :ok = BDS.AI.set_airplane_mode(true)
assert :ok = BDS.AI.put_model_preference(:airplane_image_analysis, "llama3.2")
assert :ok =
BDS.AI.put_model_capabilities("llama3.2", %{
supports_attachment: true,
supports_tool_calls: false,
disables_reasoning: false
})
tmp_path =
Path.join(System.tmp_dir!(), "bds-test-image-#{System.unique_integer([:positive])}.png")
File.write!(tmp_path, <<137, 80, 78, 71, 13, 10, 26, 10>>)
on_exit(fn -> File.rm(tmp_path) end)
assert {:ok, analysis} =
BDS.AI.analyze_image(
%{
mime_type: "image/png",
title: "Source",
alt: nil,
caption: nil,
image_url: "file://#{tmp_path}"
},
runtime: FakeRuntime,
test_pid: self(),
secret_backend: FakeSecretBackend
)
assert analysis.title == "Sunset"
assert_received {:runtime_request, _endpoint, request}
user_message = Enum.at(request.messages, 1)
image_content = Enum.at(user_message["content"], 1)
assert image_content["type"] == "image_url"
assert image_content["image_url"]["url"] =~ ~r/^data:image\/png;base64,/
end
test "analyze_image reads local media files and sends them as base64 data URLs" do
{:ok, project} = create_project_fixture("Image Analysis")
image_dir = Path.join(project.data_path, "media")
File.mkdir_p!(image_dir)
image_path = Path.join(image_dir, "test.png")
File.write!(image_path, <<137, 80, 78, 71, 13, 10, 26, 10>>)
media =
Repo.insert!(
Media.changeset(%Media{}, %{
id: Ecto.UUID.generate(),
project_id: project.id,
filename: "test.png",
original_name: "test.png",
mime_type: "image/png",
size: 8,
title: "Test",
file_path: "media/test.png",
sidecar_path: "media/test.png.meta",
created_at: Persistence.now_ms(),
updated_at: Persistence.now_ms()
})
)
assert {:ok, _endpoint} =
BDS.AI.put_endpoint(
:airplane,
%{
url: "http://localhost:11434/v1",
api_key: nil,
model: "llama-default"
},
secret_backend: FakeSecretBackend
)
assert :ok = BDS.AI.set_airplane_mode(true)
assert :ok = BDS.AI.put_model_preference(:airplane_image_analysis, "llama3.2")
assert :ok =
BDS.AI.put_model_capabilities("llama3.2", %{
supports_attachment: true,
supports_tool_calls: false,
disables_reasoning: false
})
assert {:ok, analysis} =
BDS.AI.analyze_image(
media.id,
runtime: FakeRuntime,
test_pid: self(),
secret_backend: FakeSecretBackend
)
assert analysis.title == "Sunset"
assert_received {:runtime_request, _endpoint, request}
user_message = Enum.at(request.messages, 1)
image_content = Enum.at(user_message["content"], 1)
assert image_content["type"] == "image_url"
assert image_content["image_url"]["url"] =~ ~r/^data:image\/png;base64,/
end
test "chat persists user, tool, and assistant messages with usage and blog stats prompt augmentation" do
{:ok, project} = create_project_fixture("AI Chat")
_fixtures = seed_project_content(project.id)

76
test/bds/desktop/shell_live_test.exs
View File

@@ -2460,6 +2460,78 @@ defmodule BDS.Desktop.ShellLiveTest do
assert html =~ "Automatic AI actions stay gated by airplane mode"
end
test "ai suggestions overlay fetches async results for media when online", %{project: project} do
Application.put_env(:bds, :test_pid, self())
server =
start_supervised!({Bandit, plug: AiSuggestionsServer, port: 0, startup_log: false})
{:ok, {_address, port}} = ThousandIsland.listener_info(server)
assert :ok = AI.set_airplane_mode(false)
assert {:ok, _endpoint} =
AI.put_endpoint(:online, %{
url: "http://127.0.0.1:#{port}/v1",
api_key: "test-secret",
model: "gpt-test"
})
assert :ok = AI.put_model_preference(:image_analysis, "gpt-test")
assert :ok = AI.put_model_capabilities("gpt-test", %{supports_attachment: true})
temp_dir =
Path.join(System.tmp_dir!(), "bds-shell-live-#{System.unique_integer([:positive])}")
File.mkdir_p!(temp_dir)
media_source_path = Path.join(temp_dir, "online-media.jpg")
File.write!(media_source_path, "fake image body")
{:ok, media} =
Media.import_media(%{
project_id: project.id,
source_path: media_source_path,
title: "Online Media"
})
{:ok, view, _html} = live_isolated(build_conn(), BDS.Desktop.ShellLive)
html =
render_click(view, "pin_sidebar_item", %{
"route" => "media",
"id" => media.id,
"title" => media.title,
"subtitle" => "draft"
})
assert html =~ ~s(data-testid="media-editor")
html =
view
|> element("[data-testid='media-editor'] .quick-actions-btn")
|> render_click()
assert html =~ "quick-actions-menu"
html =
view
|> element("[phx-click='open_overlay'][phx-value-kind='ai_suggestions']")
|> render_click()
assert html =~ "ai-suggestions-modal"
assert html =~ "Loading"
assert_receive {:ai_suggestions_request, _request}, 2_000
Process.sleep(200)
html = render(view)
assert html =~ "AI Image Title"
assert html =~ "AI Alt Text"
assert html =~ "AI Caption"
refute html =~ "Loading"
end
test "ai suggestions async error closes overlay and shows toast", %{project: project} do
Application.put_env(:bds, :test_pid, self())
@@ -2515,7 +2587,9 @@ defmodule BDS.Desktop.ShellLiveTest do
send(view.pid, {:ai_suggestions_error, :post, post.id, :test_error})
html = render(view)
refute html =~ "ai-suggestions-modal"
assert html =~ "ai-suggestions-modal"
assert html =~ "ai-suggestions-error"
assert html =~ "test_error"
end
test "script and template editors surface lifecycle state, load published file content, and allow publishing drafts",