fix: fixed media quick actions usage for images
This commit is contained in:
@@ -130,6 +130,47 @@ defmodule BDS.AITest do
|
||||
end
|
||||
end
|
||||
|
||||
defmodule ErrorCompletionServer do
|
||||
use Plug.Router
|
||||
|
||||
plug(:match)
|
||||
plug(:dispatch)
|
||||
|
||||
post "/v1/chat/completions" do
|
||||
{:ok, body, conn} = Plug.Conn.read_body(conn)
|
||||
send(Application.fetch_env!(:bds, :test_pid), {:error_payload, Jason.decode!(body)})
|
||||
|
||||
response = %{
|
||||
"error" => %{
|
||||
"message" => "Invalid image data",
|
||||
"type" => "invalid_request_error"
|
||||
}
|
||||
}
|
||||
|
||||
conn
|
||||
|> Plug.Conn.put_resp_content_type("application/json")
|
||||
|> Plug.Conn.send_resp(400, Jason.encode!(response))
|
||||
end
|
||||
end
|
||||
|
||||
defmodule NonJsonContentServer do
|
||||
use Plug.Router
|
||||
|
||||
plug(:match)
|
||||
plug(:dispatch)
|
||||
|
||||
post "/v1/chat/completions" do
|
||||
response = %{
|
||||
"choices" => [%{"message" => %{"content" => "This is not valid JSON"}}],
|
||||
"usage" => %{"prompt_tokens" => 4, "completion_tokens" => 2}
|
||||
}
|
||||
|
||||
conn
|
||||
|> Plug.Conn.put_resp_content_type("application/json")
|
||||
|> Plug.Conn.send_resp(200, Jason.encode!(response))
|
||||
end
|
||||
end
|
||||
|
||||
defmodule FakeRuntime do
|
||||
def generate(endpoint, request, opts) do
|
||||
test_pid = Keyword.fetch!(opts, :test_pid)
|
||||
@@ -410,6 +451,99 @@ defmodule BDS.AITest do
|
||||
assert payload["chat_template_kwargs"] == %{"enable_thinking" => false}
|
||||
end
|
||||
|
||||
test "openai-compatible generation includes response body in HTTP error details" do
|
||||
Application.put_env(:bds, :test_pid, self())
|
||||
|
||||
server =
|
||||
start_supervised!({Bandit, plug: ErrorCompletionServer, port: 0, startup_log: false})
|
||||
|
||||
{:ok, {_address, port}} = ThousandIsland.listener_info(server)
|
||||
|
||||
previous_level = Logger.level()
|
||||
Logger.configure(level: :error)
|
||||
|
||||
log =
|
||||
capture_log(fn ->
|
||||
assert {:error, %{kind: :http_error, status: 400, body: body}} =
|
||||
BDS.AI.OpenAICompatibleRuntime.generate(
|
||||
%{url: "http://127.0.0.1:#{port}/v1", api_key: nil},
|
||||
%{
|
||||
model: "gpt-test",
|
||||
messages: [%{"role" => "user", "content" => "Hello"}],
|
||||
max_output_tokens: 128,
|
||||
tools: []
|
||||
},
|
||||
[]
|
||||
)
|
||||
|
||||
assert body =~ "Invalid image data"
|
||||
end)
|
||||
|
||||
Logger.configure(level: previous_level)
|
||||
|
||||
assert log =~ "AI OpenAI-compatible HTTP error status=400"
|
||||
assert log =~ "Invalid image data"
|
||||
assert_received {:error_payload, payload}
|
||||
assert payload["model"] == "gpt-test"
|
||||
end
|
||||
|
||||
test "analyze_image logs non-JSON content when the model returns invalid JSON" do
|
||||
assert {:ok, _endpoint} =
|
||||
BDS.AI.put_endpoint(
|
||||
:airplane,
|
||||
%{
|
||||
url: "http://localhost:11434/v1",
|
||||
api_key: nil,
|
||||
model: "llama-default"
|
||||
},
|
||||
secret_backend: FakeSecretBackend
|
||||
)
|
||||
|
||||
assert :ok = BDS.AI.set_airplane_mode(true)
|
||||
assert :ok = BDS.AI.put_model_preference(:airplane_image_analysis, "llava")
|
||||
|
||||
assert :ok =
|
||||
BDS.AI.put_model_capabilities("llava", %{
|
||||
supports_attachment: true,
|
||||
supports_tool_calls: false,
|
||||
disables_reasoning: false
|
||||
})
|
||||
|
||||
defmodule NonJsonContentRuntime do
|
||||
def generate(_endpoint, _request, _opts) do
|
||||
{:ok,
|
||||
%{
|
||||
content: "This is not valid JSON",
|
||||
json: nil,
|
||||
tool_calls: [],
|
||||
usage: %{input_tokens: 4, output_tokens: 2}
|
||||
}}
|
||||
end
|
||||
end
|
||||
|
||||
previous_level = Logger.level()
|
||||
Logger.configure(level: :error)
|
||||
|
||||
log =
|
||||
capture_log(fn ->
|
||||
assert {:error, %{kind: :invalid_json_response, content: "This is not valid JSON"}} =
|
||||
BDS.AI.analyze_image(
|
||||
%{
|
||||
mime_type: "image/png",
|
||||
image_url: "data:image/png;base64,abc123"
|
||||
},
|
||||
runtime: NonJsonContentRuntime,
|
||||
test_pid: self(),
|
||||
secret_backend: FakeSecretBackend
|
||||
)
|
||||
end)
|
||||
|
||||
Logger.configure(level: previous_level)
|
||||
|
||||
assert log =~ "AI extract_json_response failed to parse content as JSON"
|
||||
assert log =~ "This is not valid JSON"
|
||||
end
|
||||
|
||||
test "airplane mode routes title tasks to airplane endpoint and offline title model" do
|
||||
assert {:ok, _endpoint} =
|
||||
BDS.AI.put_endpoint(
|
||||
@@ -582,7 +716,7 @@ defmodule BDS.AITest do
|
||||
title: "Source",
|
||||
alt: nil,
|
||||
caption: nil,
|
||||
image_url: "file:///tmp/test.png"
|
||||
image_url: "https://example.com/test.png"
|
||||
},
|
||||
runtime: FakeRuntime,
|
||||
test_pid: self(),
|
||||
@@ -603,7 +737,7 @@ defmodule BDS.AITest do
|
||||
title: "Source",
|
||||
alt: nil,
|
||||
caption: nil,
|
||||
image_url: "file:///tmp/test.png"
|
||||
image_url: "https://example.com/test.png"
|
||||
},
|
||||
runtime: FakeRuntime,
|
||||
test_pid: self(),
|
||||
@@ -619,6 +753,120 @@ defmodule BDS.AITest do
|
||||
assert BDS.AI.Catalog.model_capabilities("llama3.2").disables_reasoning
|
||||
end
|
||||
|
||||
test "analyze_image converts file:// URLs to base64 data URLs before sending" do
|
||||
assert {:ok, _endpoint} =
|
||||
BDS.AI.put_endpoint(
|
||||
:airplane,
|
||||
%{
|
||||
url: "http://localhost:11434/v1",
|
||||
api_key: nil,
|
||||
model: "llama-default"
|
||||
},
|
||||
secret_backend: FakeSecretBackend
|
||||
)
|
||||
|
||||
assert :ok = BDS.AI.set_airplane_mode(true)
|
||||
assert :ok = BDS.AI.put_model_preference(:airplane_image_analysis, "llama3.2")
|
||||
|
||||
assert :ok =
|
||||
BDS.AI.put_model_capabilities("llama3.2", %{
|
||||
supports_attachment: true,
|
||||
supports_tool_calls: false,
|
||||
disables_reasoning: false
|
||||
})
|
||||
|
||||
tmp_path =
|
||||
Path.join(System.tmp_dir!(), "bds-test-image-#{System.unique_integer([:positive])}.png")
|
||||
|
||||
File.write!(tmp_path, <<137, 80, 78, 71, 13, 10, 26, 10>>)
|
||||
on_exit(fn -> File.rm(tmp_path) end)
|
||||
|
||||
assert {:ok, analysis} =
|
||||
BDS.AI.analyze_image(
|
||||
%{
|
||||
mime_type: "image/png",
|
||||
title: "Source",
|
||||
alt: nil,
|
||||
caption: nil,
|
||||
image_url: "file://#{tmp_path}"
|
||||
},
|
||||
runtime: FakeRuntime,
|
||||
test_pid: self(),
|
||||
secret_backend: FakeSecretBackend
|
||||
)
|
||||
|
||||
assert analysis.title == "Sunset"
|
||||
|
||||
assert_received {:runtime_request, _endpoint, request}
|
||||
user_message = Enum.at(request.messages, 1)
|
||||
image_content = Enum.at(user_message["content"], 1)
|
||||
assert image_content["type"] == "image_url"
|
||||
assert image_content["image_url"]["url"] =~ ~r/^data:image\/png;base64,/
|
||||
end
|
||||
|
||||
test "analyze_image reads local media files and sends them as base64 data URLs" do
|
||||
{:ok, project} = create_project_fixture("Image Analysis")
|
||||
|
||||
image_dir = Path.join(project.data_path, "media")
|
||||
File.mkdir_p!(image_dir)
|
||||
image_path = Path.join(image_dir, "test.png")
|
||||
File.write!(image_path, <<137, 80, 78, 71, 13, 10, 26, 10>>)
|
||||
|
||||
media =
|
||||
Repo.insert!(
|
||||
Media.changeset(%Media{}, %{
|
||||
id: Ecto.UUID.generate(),
|
||||
project_id: project.id,
|
||||
filename: "test.png",
|
||||
original_name: "test.png",
|
||||
mime_type: "image/png",
|
||||
size: 8,
|
||||
title: "Test",
|
||||
file_path: "media/test.png",
|
||||
sidecar_path: "media/test.png.meta",
|
||||
created_at: Persistence.now_ms(),
|
||||
updated_at: Persistence.now_ms()
|
||||
})
|
||||
)
|
||||
|
||||
assert {:ok, _endpoint} =
|
||||
BDS.AI.put_endpoint(
|
||||
:airplane,
|
||||
%{
|
||||
url: "http://localhost:11434/v1",
|
||||
api_key: nil,
|
||||
model: "llama-default"
|
||||
},
|
||||
secret_backend: FakeSecretBackend
|
||||
)
|
||||
|
||||
assert :ok = BDS.AI.set_airplane_mode(true)
|
||||
assert :ok = BDS.AI.put_model_preference(:airplane_image_analysis, "llama3.2")
|
||||
|
||||
assert :ok =
|
||||
BDS.AI.put_model_capabilities("llama3.2", %{
|
||||
supports_attachment: true,
|
||||
supports_tool_calls: false,
|
||||
disables_reasoning: false
|
||||
})
|
||||
|
||||
assert {:ok, analysis} =
|
||||
BDS.AI.analyze_image(
|
||||
media.id,
|
||||
runtime: FakeRuntime,
|
||||
test_pid: self(),
|
||||
secret_backend: FakeSecretBackend
|
||||
)
|
||||
|
||||
assert analysis.title == "Sunset"
|
||||
|
||||
assert_received {:runtime_request, _endpoint, request}
|
||||
user_message = Enum.at(request.messages, 1)
|
||||
image_content = Enum.at(user_message["content"], 1)
|
||||
assert image_content["type"] == "image_url"
|
||||
assert image_content["image_url"]["url"] =~ ~r/^data:image\/png;base64,/
|
||||
end
|
||||
|
||||
test "chat persists user, tool, and assistant messages with usage and blog stats prompt augmentation" do
|
||||
{:ok, project} = create_project_fixture("AI Chat")
|
||||
_fixtures = seed_project_content(project.id)
|
||||
|
||||
@@ -2460,6 +2460,78 @@ defmodule BDS.Desktop.ShellLiveTest do
|
||||
assert html =~ "Automatic AI actions stay gated by airplane mode"
|
||||
end
|
||||
|
||||
test "ai suggestions overlay fetches async results for media when online", %{project: project} do
|
||||
Application.put_env(:bds, :test_pid, self())
|
||||
|
||||
server =
|
||||
start_supervised!({Bandit, plug: AiSuggestionsServer, port: 0, startup_log: false})
|
||||
|
||||
{:ok, {_address, port}} = ThousandIsland.listener_info(server)
|
||||
|
||||
assert :ok = AI.set_airplane_mode(false)
|
||||
|
||||
assert {:ok, _endpoint} =
|
||||
AI.put_endpoint(:online, %{
|
||||
url: "http://127.0.0.1:#{port}/v1",
|
||||
api_key: "test-secret",
|
||||
model: "gpt-test"
|
||||
})
|
||||
|
||||
assert :ok = AI.put_model_preference(:image_analysis, "gpt-test")
|
||||
assert :ok = AI.put_model_capabilities("gpt-test", %{supports_attachment: true})
|
||||
|
||||
temp_dir =
|
||||
Path.join(System.tmp_dir!(), "bds-shell-live-#{System.unique_integer([:positive])}")
|
||||
|
||||
File.mkdir_p!(temp_dir)
|
||||
media_source_path = Path.join(temp_dir, "online-media.jpg")
|
||||
File.write!(media_source_path, "fake image body")
|
||||
|
||||
{:ok, media} =
|
||||
Media.import_media(%{
|
||||
project_id: project.id,
|
||||
source_path: media_source_path,
|
||||
title: "Online Media"
|
||||
})
|
||||
|
||||
{:ok, view, _html} = live_isolated(build_conn(), BDS.Desktop.ShellLive)
|
||||
|
||||
html =
|
||||
render_click(view, "pin_sidebar_item", %{
|
||||
"route" => "media",
|
||||
"id" => media.id,
|
||||
"title" => media.title,
|
||||
"subtitle" => "draft"
|
||||
})
|
||||
|
||||
assert html =~ ~s(data-testid="media-editor")
|
||||
|
||||
html =
|
||||
view
|
||||
|> element("[data-testid='media-editor'] .quick-actions-btn")
|
||||
|> render_click()
|
||||
|
||||
assert html =~ "quick-actions-menu"
|
||||
|
||||
html =
|
||||
view
|
||||
|> element("[phx-click='open_overlay'][phx-value-kind='ai_suggestions']")
|
||||
|> render_click()
|
||||
|
||||
assert html =~ "ai-suggestions-modal"
|
||||
assert html =~ "Loading"
|
||||
|
||||
assert_receive {:ai_suggestions_request, _request}, 2_000
|
||||
|
||||
Process.sleep(200)
|
||||
html = render(view)
|
||||
|
||||
assert html =~ "AI Image Title"
|
||||
assert html =~ "AI Alt Text"
|
||||
assert html =~ "AI Caption"
|
||||
refute html =~ "Loading"
|
||||
end
|
||||
|
||||
test "ai suggestions async error closes overlay and shows toast", %{project: project} do
|
||||
Application.put_env(:bds, :test_pid, self())
|
||||
|
||||
@@ -2515,7 +2587,9 @@ defmodule BDS.Desktop.ShellLiveTest do
|
||||
send(view.pid, {:ai_suggestions_error, :post, post.id, :test_error})
|
||||
html = render(view)
|
||||
|
||||
refute html =~ "ai-suggestions-modal"
|
||||
assert html =~ "ai-suggestions-modal"
|
||||
assert html =~ "ai-suggestions-error"
|
||||
assert html =~ "test_error"
|
||||
end
|
||||
|
||||
test "script and template editors surface lifecycle state, load published file content, and allow publishing drafts",
|
||||
|
||||
Reference in New Issue
Block a user