fix: fix airplane mode for AI usage and qwen 3.6 one-shot parsing

This commit is contained in:
2026-06-11 22:28:44 +02:00
parent d8b24c9b72
commit 8546080a3d
12 changed files with 269 additions and 24 deletions

View File

@@ -111,6 +111,19 @@ defmodule BDS.AI do
end end
end end
@doc """
True when the airplane (local) endpoint has both a URL and a model
configured, so gated AI features can run against the local model.
"""
@spec airplane_endpoint_configured?() :: boolean()
def airplane_endpoint_configured? do
present_setting?(get_setting("ai.airplane.url")) and
present_setting?(get_setting("ai.airplane.model"))
end
defp present_setting?(value) when is_binary(value), do: String.trim(value) != ""
defp present_setting?(_value), do: false
@spec put_model_preference(atom(), String.t()) :: @spec put_model_preference(atom(), String.t()) ::
:ok | {:error, :unknown_model_preference | term()} :ok | {:error, :unknown_model_preference | term()}
def put_model_preference(key, model) when is_atom(key) and is_binary(model) do def put_model_preference(key, model) when is_atom(key) and is_binary(model) do

View File

@@ -4,6 +4,7 @@ defmodule BDS.AI.OneShot do
require Logger require Logger
alias BDS.AI.Chat alias BDS.AI.Chat
alias BDS.AI.JsonContent
alias BDS.AI.OpenAICompatibleRuntime alias BDS.AI.OpenAICompatibleRuntime
alias BDS.AI.Runtime alias BDS.AI.Runtime
alias BDS.Media.Media alias BDS.Media.Media
@@ -213,7 +214,9 @@ defmodule BDS.AI.OneShot do
messages: [ messages: [
%{ %{
"role" => "system", "role" => "system",
"content" => one_shot_system_prompt(operation, language, source_language) "content" =>
one_shot_system_prompt(operation, language, source_language) <>
" Output raw JSON only, without markdown code fences."
}, },
%{ %{
"role" => "user", "role" => "user",
@@ -351,11 +354,11 @@ defmodule BDS.AI.OneShot do
defp extract_json_response(%{json: json}) when is_map(json), do: {:ok, json} defp extract_json_response(%{json: json}) when is_map(json), do: {:ok, json}
defp extract_json_response(%{content: content}) when is_binary(content) do defp extract_json_response(%{content: content}) when is_binary(content) do
case Jason.decode(content) do case JsonContent.decode(content) do
{:ok, json} when is_map(json) -> json when is_map(json) ->
{:ok, json} {:ok, json}
_other -> nil ->
Logger.error( Logger.error(
"AI extract_json_response failed to parse content as JSON. Content: #{String.slice(content, 0, 1000)}" "AI extract_json_response failed to parse content as JSON. Content: #{String.slice(content, 0, 1000)}"
) )

View File

@@ -182,14 +182,7 @@ defmodule BDS.AI.OpenAICompatibleRuntime do
end end
end end
defp decode_json_content(nil), do: nil defp decode_json_content(content), do: BDS.AI.JsonContent.decode(content)
defp decode_json_content(content) when is_binary(content) do
case Jason.decode(content) do
{:ok, decoded} when is_map(decoded) -> decoded
_other -> nil
end
end
defp completions_url(url) do defp completions_url(url) do
cond do cond do

View File

@@ -414,7 +414,7 @@ defmodule BDS.Desktop.ShellLive do
do: OverlayManager.handle_event("overlay_lightbox_next", params, socket, overlay_callbacks()) do: OverlayManager.handle_event("overlay_lightbox_next", params, socket, overlay_callbacks())
def handle_event("add_gallery_images", %{"post-id" => post_id}, socket) do def handle_event("add_gallery_images", %{"post-id" => post_id}, socket) do
if socket.assigns.offline_mode do if socket.assigns.offline_mode and not AI.airplane_endpoint_configured?() do
{:noreply, {:noreply,
append_output_entry( append_output_entry(
socket, socket,

View File

@@ -230,7 +230,7 @@ defmodule BDS.Desktop.ShellLive.ChatEditor do
not is_nil(socket.assigns.request) -> not is_nil(socket.assigns.request) ->
build_data(socket) build_data(socket)
socket.assigns.offline_mode -> socket.assigns.offline_mode and not AI.airplane_endpoint_configured?() ->
Notify.output( Notify.output(
dgettext("ui", "Chat"), dgettext("ui", "Chat"),
dgettext("ui", "Automatic AI actions stay gated by airplane mode."), dgettext("ui", "Automatic AI actions stay gated by airplane mode."),
@@ -239,7 +239,7 @@ defmodule BDS.Desktop.ShellLive.ChatEditor do
build_data(socket) build_data(socket)
ModelSelection.needs_api_key?(false) -> ModelSelection.needs_api_key?(socket.assigns.offline_mode) ->
build_data(socket) build_data(socket)
true -> true ->

View File

@@ -434,7 +434,7 @@ defmodule BDS.Desktop.ShellLive.ImportEditor do
socket = socket =
with %{} = definition <- ImportDefinitions.get_definition(definition_id), with %{} = definition <- ImportDefinitions.get_definition(definition_id),
%{} = report <- ImportDefinitions.decode_analysis_result(definition) do %{} = report <- ImportDefinitions.decode_analysis_result(definition) do
if socket.assigns.offline_mode? do if socket.assigns.offline_mode? and not AI.airplane_endpoint_configured?() do
notify_output( notify_output(
dgettext("ui", "Import"), dgettext("ui", "Import"),
BDS.Gettext.lgettext( BDS.Gettext.lgettext(

View File

@@ -82,7 +82,7 @@ defmodule BDS.Desktop.ShellLive.ImportEditor.TaxonomyEditing do
%{} = definition <- ImportDefinitions.get_definition(definition_id), %{} = definition <- ImportDefinitions.get_definition(definition_id),
%{} = report <- ImportDefinitions.decode_analysis_result(definition) do %{} = report <- ImportDefinitions.decode_analysis_result(definition) do
cond do cond do
socket.assigns.offline_mode -> socket.assigns.offline_mode and not AI.airplane_endpoint_configured?() ->
socket socket
|> append_output.( |> append_output.(
dgettext("ui", "Import"), dgettext("ui", "Import"),

View File

@@ -153,7 +153,7 @@ defmodule BDS.Desktop.ShellLive.MediaEditor do
end end
def handle_event("detect_media_editor_language", _params, socket) do def handle_event("detect_media_editor_language", _params, socket) do
if socket.assigns.offline_mode do if socket.assigns.offline_mode and not AI.airplane_endpoint_configured?() do
notify_output( notify_output(
socket, socket,
dgettext("ui", "Detect Language"), dgettext("ui", "Detect Language"),
@@ -346,7 +346,7 @@ defmodule BDS.Desktop.ShellLive.MediaEditor do
def handle_event("refresh_media_translation", %{"language" => language}, socket) do def handle_event("refresh_media_translation", %{"language" => language}, socket) do
media = socket.assigns.media media = socket.assigns.media
if socket.assigns.offline_mode do if socket.assigns.offline_mode and not AI.airplane_endpoint_configured?() do
notify_output( notify_output(
socket, socket,
dgettext("ui", "Translate"), dgettext("ui", "Translate"),
@@ -539,7 +539,7 @@ defmodule BDS.Desktop.ShellLive.MediaEditor do
end end
defp do_translate(socket, language) do defp do_translate(socket, language) do
if socket.assigns.offline_mode do if socket.assigns.offline_mode and not AI.airplane_endpoint_configured?() do
notify_output( notify_output(
socket, socket,
dgettext("ui", "Translate"), dgettext("ui", "Translate"),

View File

@@ -66,7 +66,7 @@ defmodule BDS.Desktop.ShellLive.OverlayManager do
socket = socket =
if kind == "ai_suggestions" and not is_nil(overlay) do if kind == "ai_suggestions" and not is_nil(overlay) do
if socket.assigns.offline_mode do if socket.assigns.offline_mode and not AI.airplane_endpoint_configured?() do
callbacks.append_output.( callbacks.append_output.(
socket, socket,
dgettext("ui", "AI Suggestions"), dgettext("ui", "AI Suggestions"),

View File

@@ -707,7 +707,7 @@ defmodule BDS.Desktop.ShellLive.PostEditor do
end end
defp do_detect_language(socket) do defp do_detect_language(socket) do
if Map.get(socket.assigns, :offline_mode, true) do if Map.get(socket.assigns, :offline_mode, true) and not AI.airplane_endpoint_configured?() do
notify_output( notify_output(
socket, socket,
dgettext("ui", "Detect Language"), dgettext("ui", "Detect Language"),
@@ -756,7 +756,7 @@ defmodule BDS.Desktop.ShellLive.PostEditor do
end end
defp do_translate(socket, language) do defp do_translate(socket, language) do
if Map.get(socket.assigns, :offline_mode, true) do if Map.get(socket.assigns, :offline_mode, true) and not AI.airplane_endpoint_configured?() do
notify_output( notify_output(
socket, socket,
dgettext("ui", "Translate"), dgettext("ui", "Translate"),

View File

@@ -3,7 +3,6 @@ defmodule BDS.AITest do
import ExUnit.CaptureLog import ExUnit.CaptureLog
import Ecto.Query import Ecto.Query
require Logger
alias BDS.Media.Media alias BDS.Media.Media
alias BDS.Persistence alias BDS.Persistence
@@ -366,6 +365,31 @@ defmodule BDS.AITest do
assert Repo.get(Setting, "__encrypted_ai.online.api_key") == nil assert Repo.get(Setting, "__encrypted_ai.online.api_key") == nil
end end
test "airplane_endpoint_configured? reflects the airplane endpoint url and model" do
refute BDS.AI.airplane_endpoint_configured?()
assert {:ok, _endpoint} =
BDS.AI.put_endpoint(
:airplane,
%{url: "http://localhost:11434/v1", api_key: nil, model: ""},
secret_backend: FakeSecretBackend
)
refute BDS.AI.airplane_endpoint_configured?()
assert {:ok, _endpoint} =
BDS.AI.put_endpoint(
:airplane,
%{url: "http://localhost:11434/v1", api_key: nil, model: "llama3.3"},
secret_backend: FakeSecretBackend
)
assert BDS.AI.airplane_endpoint_configured?()
assert :ok = BDS.AI.delete_endpoint(:airplane)
refute BDS.AI.airplane_endpoint_configured?()
end
test "refresh_model_catalog stores providers, models, modalities, and etag metadata" do test "refresh_model_catalog stores providers, models, modalities, and etag metadata" do
assert {:ok, result} = assert {:ok, result} =
BDS.AI.refresh_model_catalog(http_client: FakeHttpClient) BDS.AI.refresh_model_catalog(http_client: FakeHttpClient)
@@ -611,6 +635,95 @@ defmodule BDS.AITest do
assert log =~ "This is not valid JSON" assert log =~ "This is not valid JSON"
end end
test "analyze_image accepts JSON wrapped in markdown fences from local models" do
assert {:ok, _endpoint} =
BDS.AI.put_endpoint(
:airplane,
%{
url: "http://localhost:11434/v1",
api_key: nil,
model: "llama-default"
},
secret_backend: FakeSecretBackend
)
assert :ok = BDS.AI.set_airplane_mode(true)
assert :ok = BDS.AI.put_model_preference(:airplane_image_analysis, "qwen-vision")
assert :ok =
BDS.AI.put_model_capabilities("qwen-vision", %{
supports_attachment: true,
supports_tool_calls: false,
disables_reasoning: false
})
defmodule FencedJsonContentRuntime do
def generate(_endpoint, _request, _opts) do
content = """
```json
{
"title": "Ahornblätter im Herbstlicht",
"alt": "Nahaufnahme von Ahornblättern",
"caption": "Einige Ahornblätter verfärben sich."
}
```
"""
{:ok,
%{
content: content,
json: nil,
tool_calls: [],
usage: %{input_tokens: 4, output_tokens: 2}
}}
end
end
assert {:ok, result} =
BDS.AI.analyze_image(
%{
mime_type: "image/png",
image_url: "data:image/png;base64,abc123"
},
runtime: FencedJsonContentRuntime,
test_pid: self(),
secret_backend: FakeSecretBackend
)
assert result.title == "Ahornblätter im Herbstlicht"
assert result.alt == "Nahaufnahme von Ahornblättern"
assert result.caption == "Einige Ahornblätter verfärben sich."
end
test "one-shot system prompts demand raw JSON without markdown fences" do
assert {:ok, _endpoint} =
BDS.AI.put_endpoint(
:airplane,
%{
url: "http://localhost:11434/v1",
api_key: nil,
model: "llama-default"
},
secret_backend: FakeSecretBackend
)
assert :ok = BDS.AI.set_airplane_mode(true)
assert {:ok, _result} =
BDS.AI.analyze_post(
%{title: "Title", excerpt: "Excerpt", content: "Content"},
runtime: FakeRuntime,
test_pid: self(),
secret_backend: FakeSecretBackend
)
assert_received {:runtime_request, _endpoint, request}
system_content = get_in(request.messages, [Access.at(0), "content"])
assert system_content =~ "raw JSON only"
assert system_content =~ "without markdown code fences"
end
test "airplane mode routes title tasks to airplane endpoint and offline title model" do test "airplane mode routes title tasks to airplane endpoint and offline title model" do
assert {:ok, _endpoint} = assert {:ok, _endpoint} =
BDS.AI.put_endpoint( BDS.AI.put_endpoint(

View File

@@ -3230,6 +3230,129 @@ defmodule BDS.Desktop.ShellLiveTest do
assert html =~ "Automatic AI actions stay gated by airplane mode" assert html =~ "Automatic AI actions stay gated by airplane mode"
end end
test "ai suggestions overlay uses the local model in airplane mode for media", %{
project: project
} do
Application.put_env(:bds, :test_pid, self())
server =
start_supervised!({Bandit, plug: AiSuggestionsServer, port: 0, startup_log: false})
{:ok, {_address, port}} = ThousandIsland.listener_info(server)
assert :ok = AI.set_airplane_mode(true)
assert {:ok, _endpoint} =
AI.put_endpoint(:airplane, %{
url: "http://127.0.0.1:#{port}/v1",
api_key: nil,
model: "llava-local"
})
assert :ok = AI.put_model_preference(:airplane_image_analysis, "llava-local")
assert :ok = AI.put_model_capabilities("llava-local", %{supports_attachment: true})
temp_dir =
Path.join(System.tmp_dir!(), "bds-shell-live-#{System.unique_integer([:positive])}")
File.mkdir_p!(temp_dir)
media_source_path = Path.join(temp_dir, "airplane-media.jpg")
File.write!(media_source_path, "fake image body")
{:ok, media} =
Media.import_media(%{
project_id: project.id,
source_path: media_source_path,
title: "Airplane Media"
})
{:ok, view, _html} = live_isolated(build_conn(), BDS.Desktop.ShellLive)
html =
render_click(view, "pin_sidebar_item", %{
"route" => "media",
"id" => media.id,
"title" => media.title,
"subtitle" => "draft"
})
assert html =~ ~s(data-testid="media-editor")
html =
view
|> element("[data-testid='media-editor'] .quick-actions-btn")
|> render_click()
assert html =~ "quick-actions-menu"
html =
view
|> element("[phx-click='open_overlay'][phx-value-kind='ai_suggestions']")
|> render_click()
assert html =~ "ai-suggestions-modal"
assert_receive {:ai_suggestions_request, request}, 2_000
assert request["model"] == "llava-local"
Process.sleep(200)
html = render(view)
assert html =~ "AI Image Title"
assert html =~ "AI Alt Text"
assert html =~ "AI Caption"
end
test "chat editor sends messages to the local model in airplane mode" do
Application.put_env(:bds, :test_pid, self())
server =
start_supervised!({Bandit, plug: TitleChatServer, port: 0, startup_log: false})
{:ok, {_address, port}} = ThousandIsland.listener_info(server)
assert :ok = AI.set_airplane_mode(true)
assert {:ok, _endpoint} =
AI.put_endpoint(:airplane, %{
url: "http://127.0.0.1:#{port}/v1",
api_key: nil,
model: "llama-local"
})
assert {:ok, conversation} = AI.start_chat(%{title: "New Chat"})
{:ok, view, _html} = live_isolated(build_conn(), BDS.Desktop.ShellLive)
html =
render_click(view, "pin_sidebar_item", %{
"route" => "chat",
"id" => conversation.id,
"title" => conversation.title,
"subtitle" => "chat"
})
assert html =~ ~s(data-testid="chat-send-button")
_html =
view
|> element(".chat-input-wrapper")
|> render_change(%{"message" => "Beschreibe das neueste Bild"})
_html =
view
|> element("[data-testid='chat-send-button']")
|> render_click()
assert_receive {:title_chat_request, request}, 2_000
assert request["model"] == "llama-local"
Process.sleep(350)
html = render(view)
assert html =~ "Ich habe die Posts pro Monat ermittelt."
end
test "ai suggestions overlay fetches async results for media when online", %{project: project} do test "ai suggestions overlay fetches async results for media when online", %{project: project} do
Application.put_env(:bds, :test_pid, self()) Application.put_env(:bds, :test_pid, self())