From 8546080a3d15e24f7e8e1882ff282354e12f5eed Mon Sep 17 00:00:00 2001 From: Chili Palmer Date: Thu, 11 Jun 2026 22:28:44 +0200 Subject: [PATCH] fix: fix airplane mode for AI usage and qwen 3.6 one-shot parsing --- lib/bds/ai.ex | 13 ++ lib/bds/ai/one_shot.ex | 11 +- lib/bds/ai/openai_compatible_runtime.ex | 9 +- lib/bds/desktop/shell_live.ex | 2 +- lib/bds/desktop/shell_live/chat_editor.ex | 4 +- lib/bds/desktop/shell_live/import_editor.ex | 2 +- .../import_editor/taxonomy_editing.ex | 2 +- lib/bds/desktop/shell_live/media_editor.ex | 6 +- lib/bds/desktop/shell_live/overlay_manager.ex | 2 +- lib/bds/desktop/shell_live/post_editor.ex | 4 +- test/bds/ai_test.exs | 115 +++++++++++++++- test/bds/desktop/shell_live_test.exs | 123 ++++++++++++++++++ 12 files changed, 269 insertions(+), 24 deletions(-) diff --git a/lib/bds/ai.ex b/lib/bds/ai.ex index 7dc5520..aa30ac5 100644 --- a/lib/bds/ai.ex +++ b/lib/bds/ai.ex @@ -111,6 +111,19 @@ defmodule BDS.AI do end end + @doc """ + True when the airplane (local) endpoint has both a URL and a model + configured, so gated AI features can run against the local model. + """ + @spec airplane_endpoint_configured?() :: boolean() + def airplane_endpoint_configured? do + present_setting?(get_setting("ai.airplane.url")) and + present_setting?(get_setting("ai.airplane.model")) + end + + defp present_setting?(value) when is_binary(value), do: String.trim(value) != "" + defp present_setting?(_value), do: false + @spec put_model_preference(atom(), String.t()) :: :ok | {:error, :unknown_model_preference | term()} def put_model_preference(key, model) when is_atom(key) and is_binary(model) do diff --git a/lib/bds/ai/one_shot.ex b/lib/bds/ai/one_shot.ex index 7cfb2eb..a2ebbe2 100644 --- a/lib/bds/ai/one_shot.ex +++ b/lib/bds/ai/one_shot.ex @@ -4,6 +4,7 @@ defmodule BDS.AI.OneShot do require Logger alias BDS.AI.Chat + alias BDS.AI.JsonContent alias BDS.AI.OpenAICompatibleRuntime alias BDS.AI.Runtime alias BDS.Media.Media @@ -213,7 +214,9 @@ defmodule BDS.AI.OneShot do messages: [ %{ "role" => "system", - "content" => one_shot_system_prompt(operation, language, source_language) + "content" => + one_shot_system_prompt(operation, language, source_language) <> + " Output raw JSON only, without markdown code fences." }, %{ "role" => "user", @@ -351,11 +354,11 @@ defmodule BDS.AI.OneShot do defp extract_json_response(%{json: json}) when is_map(json), do: {:ok, json} defp extract_json_response(%{content: content}) when is_binary(content) do - case Jason.decode(content) do - {:ok, json} when is_map(json) -> + case JsonContent.decode(content) do + json when is_map(json) -> {:ok, json} - _other -> + nil -> Logger.error( "AI extract_json_response failed to parse content as JSON. Content: #{String.slice(content, 0, 1000)}" ) diff --git a/lib/bds/ai/openai_compatible_runtime.ex b/lib/bds/ai/openai_compatible_runtime.ex index 6f8150a..cec342a 100644 --- a/lib/bds/ai/openai_compatible_runtime.ex +++ b/lib/bds/ai/openai_compatible_runtime.ex @@ -182,14 +182,7 @@ defmodule BDS.AI.OpenAICompatibleRuntime do end end - defp decode_json_content(nil), do: nil - - defp decode_json_content(content) when is_binary(content) do - case Jason.decode(content) do - {:ok, decoded} when is_map(decoded) -> decoded - _other -> nil - end - end + defp decode_json_content(content), do: BDS.AI.JsonContent.decode(content) defp completions_url(url) do cond do diff --git a/lib/bds/desktop/shell_live.ex b/lib/bds/desktop/shell_live.ex index 23f52a2..0e2b090 100644 --- a/lib/bds/desktop/shell_live.ex +++ b/lib/bds/desktop/shell_live.ex @@ -414,7 +414,7 @@ defmodule BDS.Desktop.ShellLive do do: OverlayManager.handle_event("overlay_lightbox_next", params, socket, overlay_callbacks()) def handle_event("add_gallery_images", %{"post-id" => post_id}, socket) do - if socket.assigns.offline_mode do + if socket.assigns.offline_mode and not AI.airplane_endpoint_configured?() do {:noreply, append_output_entry( socket, diff --git a/lib/bds/desktop/shell_live/chat_editor.ex b/lib/bds/desktop/shell_live/chat_editor.ex index 578cada..d649bb2 100644 --- a/lib/bds/desktop/shell_live/chat_editor.ex +++ b/lib/bds/desktop/shell_live/chat_editor.ex @@ -230,7 +230,7 @@ defmodule BDS.Desktop.ShellLive.ChatEditor do not is_nil(socket.assigns.request) -> build_data(socket) - socket.assigns.offline_mode -> + socket.assigns.offline_mode and not AI.airplane_endpoint_configured?() -> Notify.output( dgettext("ui", "Chat"), dgettext("ui", "Automatic AI actions stay gated by airplane mode."), @@ -239,7 +239,7 @@ defmodule BDS.Desktop.ShellLive.ChatEditor do build_data(socket) - ModelSelection.needs_api_key?(false) -> + ModelSelection.needs_api_key?(socket.assigns.offline_mode) -> build_data(socket) true -> diff --git a/lib/bds/desktop/shell_live/import_editor.ex b/lib/bds/desktop/shell_live/import_editor.ex index dcc61e8..567f02d 100644 --- a/lib/bds/desktop/shell_live/import_editor.ex +++ b/lib/bds/desktop/shell_live/import_editor.ex @@ -434,7 +434,7 @@ defmodule BDS.Desktop.ShellLive.ImportEditor do socket = with %{} = definition <- ImportDefinitions.get_definition(definition_id), %{} = report <- ImportDefinitions.decode_analysis_result(definition) do - if socket.assigns.offline_mode? do + if socket.assigns.offline_mode? and not AI.airplane_endpoint_configured?() do notify_output( dgettext("ui", "Import"), BDS.Gettext.lgettext( diff --git a/lib/bds/desktop/shell_live/import_editor/taxonomy_editing.ex b/lib/bds/desktop/shell_live/import_editor/taxonomy_editing.ex index bdf7884..a57c6fc 100644 --- a/lib/bds/desktop/shell_live/import_editor/taxonomy_editing.ex +++ b/lib/bds/desktop/shell_live/import_editor/taxonomy_editing.ex @@ -82,7 +82,7 @@ defmodule BDS.Desktop.ShellLive.ImportEditor.TaxonomyEditing do %{} = definition <- ImportDefinitions.get_definition(definition_id), %{} = report <- ImportDefinitions.decode_analysis_result(definition) do cond do - socket.assigns.offline_mode -> + socket.assigns.offline_mode and not AI.airplane_endpoint_configured?() -> socket |> append_output.( dgettext("ui", "Import"), diff --git a/lib/bds/desktop/shell_live/media_editor.ex b/lib/bds/desktop/shell_live/media_editor.ex index 4bd21de..2155fa9 100644 --- a/lib/bds/desktop/shell_live/media_editor.ex +++ b/lib/bds/desktop/shell_live/media_editor.ex @@ -153,7 +153,7 @@ defmodule BDS.Desktop.ShellLive.MediaEditor do end def handle_event("detect_media_editor_language", _params, socket) do - if socket.assigns.offline_mode do + if socket.assigns.offline_mode and not AI.airplane_endpoint_configured?() do notify_output( socket, dgettext("ui", "Detect Language"), @@ -346,7 +346,7 @@ defmodule BDS.Desktop.ShellLive.MediaEditor do def handle_event("refresh_media_translation", %{"language" => language}, socket) do media = socket.assigns.media - if socket.assigns.offline_mode do + if socket.assigns.offline_mode and not AI.airplane_endpoint_configured?() do notify_output( socket, dgettext("ui", "Translate"), @@ -539,7 +539,7 @@ defmodule BDS.Desktop.ShellLive.MediaEditor do end defp do_translate(socket, language) do - if socket.assigns.offline_mode do + if socket.assigns.offline_mode and not AI.airplane_endpoint_configured?() do notify_output( socket, dgettext("ui", "Translate"), diff --git a/lib/bds/desktop/shell_live/overlay_manager.ex b/lib/bds/desktop/shell_live/overlay_manager.ex index 5b10e2e..28d07af 100644 --- a/lib/bds/desktop/shell_live/overlay_manager.ex +++ b/lib/bds/desktop/shell_live/overlay_manager.ex @@ -66,7 +66,7 @@ defmodule BDS.Desktop.ShellLive.OverlayManager do socket = if kind == "ai_suggestions" and not is_nil(overlay) do - if socket.assigns.offline_mode do + if socket.assigns.offline_mode and not AI.airplane_endpoint_configured?() do callbacks.append_output.( socket, dgettext("ui", "AI Suggestions"), diff --git a/lib/bds/desktop/shell_live/post_editor.ex b/lib/bds/desktop/shell_live/post_editor.ex index 25249da..fb0d5f8 100644 --- a/lib/bds/desktop/shell_live/post_editor.ex +++ b/lib/bds/desktop/shell_live/post_editor.ex @@ -707,7 +707,7 @@ defmodule BDS.Desktop.ShellLive.PostEditor do end defp do_detect_language(socket) do - if Map.get(socket.assigns, :offline_mode, true) do + if Map.get(socket.assigns, :offline_mode, true) and not AI.airplane_endpoint_configured?() do notify_output( socket, dgettext("ui", "Detect Language"), @@ -756,7 +756,7 @@ defmodule BDS.Desktop.ShellLive.PostEditor do end defp do_translate(socket, language) do - if Map.get(socket.assigns, :offline_mode, true) do + if Map.get(socket.assigns, :offline_mode, true) and not AI.airplane_endpoint_configured?() do notify_output( socket, dgettext("ui", "Translate"), diff --git a/test/bds/ai_test.exs b/test/bds/ai_test.exs index 45f82fb..c7d7135 100644 --- a/test/bds/ai_test.exs +++ b/test/bds/ai_test.exs @@ -3,7 +3,6 @@ defmodule BDS.AITest do import ExUnit.CaptureLog import Ecto.Query - require Logger alias BDS.Media.Media alias BDS.Persistence @@ -366,6 +365,31 @@ defmodule BDS.AITest do assert Repo.get(Setting, "__encrypted_ai.online.api_key") == nil end + test "airplane_endpoint_configured? reflects the airplane endpoint url and model" do + refute BDS.AI.airplane_endpoint_configured?() + + assert {:ok, _endpoint} = + BDS.AI.put_endpoint( + :airplane, + %{url: "http://localhost:11434/v1", api_key: nil, model: ""}, + secret_backend: FakeSecretBackend + ) + + refute BDS.AI.airplane_endpoint_configured?() + + assert {:ok, _endpoint} = + BDS.AI.put_endpoint( + :airplane, + %{url: "http://localhost:11434/v1", api_key: nil, model: "llama3.3"}, + secret_backend: FakeSecretBackend + ) + + assert BDS.AI.airplane_endpoint_configured?() + + assert :ok = BDS.AI.delete_endpoint(:airplane) + refute BDS.AI.airplane_endpoint_configured?() + end + test "refresh_model_catalog stores providers, models, modalities, and etag metadata" do assert {:ok, result} = BDS.AI.refresh_model_catalog(http_client: FakeHttpClient) @@ -611,6 +635,95 @@ defmodule BDS.AITest do assert log =~ "This is not valid JSON" end + test "analyze_image accepts JSON wrapped in markdown fences from local models" do + assert {:ok, _endpoint} = + BDS.AI.put_endpoint( + :airplane, + %{ + url: "http://localhost:11434/v1", + api_key: nil, + model: "llama-default" + }, + secret_backend: FakeSecretBackend + ) + + assert :ok = BDS.AI.set_airplane_mode(true) + assert :ok = BDS.AI.put_model_preference(:airplane_image_analysis, "qwen-vision") + + assert :ok = + BDS.AI.put_model_capabilities("qwen-vision", %{ + supports_attachment: true, + supports_tool_calls: false, + disables_reasoning: false + }) + + defmodule FencedJsonContentRuntime do + def generate(_endpoint, _request, _opts) do + content = """ + ```json + { + "title": "Ahornblätter im Herbstlicht", + "alt": "Nahaufnahme von Ahornblättern", + "caption": "Einige Ahornblätter verfärben sich." + } + ``` + """ + + {:ok, + %{ + content: content, + json: nil, + tool_calls: [], + usage: %{input_tokens: 4, output_tokens: 2} + }} + end + end + + assert {:ok, result} = + BDS.AI.analyze_image( + %{ + mime_type: "image/png", + image_url: "data:image/png;base64,abc123" + }, + runtime: FencedJsonContentRuntime, + test_pid: self(), + secret_backend: FakeSecretBackend + ) + + assert result.title == "Ahornblätter im Herbstlicht" + assert result.alt == "Nahaufnahme von Ahornblättern" + assert result.caption == "Einige Ahornblätter verfärben sich." + end + + test "one-shot system prompts demand raw JSON without markdown fences" do + assert {:ok, _endpoint} = + BDS.AI.put_endpoint( + :airplane, + %{ + url: "http://localhost:11434/v1", + api_key: nil, + model: "llama-default" + }, + secret_backend: FakeSecretBackend + ) + + assert :ok = BDS.AI.set_airplane_mode(true) + + assert {:ok, _result} = + BDS.AI.analyze_post( + %{title: "Title", excerpt: "Excerpt", content: "Content"}, + runtime: FakeRuntime, + test_pid: self(), + secret_backend: FakeSecretBackend + ) + + assert_received {:runtime_request, _endpoint, request} + + system_content = get_in(request.messages, [Access.at(0), "content"]) + assert system_content =~ "raw JSON only" + assert system_content =~ "without markdown code fences" + end + test "airplane mode routes title tasks to airplane endpoint and offline title model" do assert {:ok, _endpoint} = BDS.AI.put_endpoint( diff --git a/test/bds/desktop/shell_live_test.exs b/test/bds/desktop/shell_live_test.exs index a674110..421d01f 100644 --- a/test/bds/desktop/shell_live_test.exs +++ b/test/bds/desktop/shell_live_test.exs @@ -3230,6 +3230,129 @@ defmodule BDS.Desktop.ShellLiveTest do assert html =~ "Automatic AI actions stay gated by airplane mode" end + test "ai suggestions overlay uses the local model in airplane mode for media", %{ + project: project + } do + Application.put_env(:bds, :test_pid, self()) + + server = + start_supervised!({Bandit, plug: AiSuggestionsServer, port: 0, startup_log: false}) + + {:ok, {_address, port}} = ThousandIsland.listener_info(server) + + assert :ok = AI.set_airplane_mode(true) + + assert {:ok, _endpoint} = + AI.put_endpoint(:airplane, %{ + url: "http://127.0.0.1:#{port}/v1", + api_key: nil, + model: "llava-local" + }) + + assert :ok = AI.put_model_preference(:airplane_image_analysis, "llava-local") + assert :ok = AI.put_model_capabilities("llava-local", %{supports_attachment: true}) + + temp_dir = + Path.join(System.tmp_dir!(), "bds-shell-live-#{System.unique_integer([:positive])}") + + File.mkdir_p!(temp_dir) + media_source_path = Path.join(temp_dir, "airplane-media.jpg") + File.write!(media_source_path, "fake image body") + + {:ok, media} = + Media.import_media(%{ + project_id: project.id, + source_path: media_source_path, + title: "Airplane Media" + }) + + {:ok, view, _html} = live_isolated(build_conn(), BDS.Desktop.ShellLive) + + html = + render_click(view, "pin_sidebar_item", %{ + "route" => "media", + "id" => media.id, + "title" => media.title, + "subtitle" => "draft" + }) + + assert html =~ ~s(data-testid="media-editor") + + html = + view + |> element("[data-testid='media-editor'] .quick-actions-btn") + |> render_click() + + assert html =~ "quick-actions-menu" + + html = + view + |> element("[phx-click='open_overlay'][phx-value-kind='ai_suggestions']") + |> render_click() + + assert html =~ "ai-suggestions-modal" + + assert_receive {:ai_suggestions_request, request}, 2_000 + assert request["model"] == "llava-local" + + Process.sleep(200) + html = render(view) + + assert html =~ "AI Image Title" + assert html =~ "AI Alt Text" + assert html =~ "AI Caption" + end + + test "chat editor sends messages to the local model in airplane mode" do + Application.put_env(:bds, :test_pid, self()) + + server = + start_supervised!({Bandit, plug: TitleChatServer, port: 0, startup_log: false}) + + {:ok, {_address, port}} = ThousandIsland.listener_info(server) + + assert :ok = AI.set_airplane_mode(true) + + assert {:ok, _endpoint} = + AI.put_endpoint(:airplane, %{ + url: "http://127.0.0.1:#{port}/v1", + api_key: nil, + model: "llama-local" + }) + + assert {:ok, conversation} = AI.start_chat(%{title: "New Chat"}) + + {:ok, view, _html} = live_isolated(build_conn(), BDS.Desktop.ShellLive) + + html = + render_click(view, "pin_sidebar_item", %{ + "route" => "chat", + "id" => conversation.id, + "title" => conversation.title, + "subtitle" => "chat" + }) + + assert html =~ ~s(data-testid="chat-send-button") + + _html = + view + |> element(".chat-input-wrapper") + |> render_change(%{"message" => "Beschreibe das neueste Bild"}) + + _html = + view + |> element("[data-testid='chat-send-button']") + |> render_click() + + assert_receive {:title_chat_request, request}, 2_000 + assert request["model"] == "llama-local" + + Process.sleep(350) + html = render(view) + + assert html =~ "Ich habe die Posts pro Monat ermittelt." + end + test "ai suggestions overlay fetches async results for media when online", %{project: project} do Application.put_env(:bds, :test_pid, self())