From 78609377be77745a244c20af6d6ce331476ee52b Mon Sep 17 00:00:00 2001 From: Chili Palmer Date: Fri, 24 Apr 2026 13:56:42 +0200 Subject: [PATCH] feat: start on AI integration --- .vscode/settings.json | 3 +- lib/bds/ai.ex | 1314 +++++++++++++++++ lib/bds/ai/catalog_meta.ex | 18 + lib/bds/ai/catalog_provider.ex | 23 + lib/bds/ai/chat_conversation.ex | 22 + lib/bds/ai/chat_message.ex | 41 + lib/bds/ai/http_client.ex | 51 + lib/bds/ai/in_flight.ex | 29 + lib/bds/ai/model.ex | 64 + lib/bds/ai/model_modality.ex | 21 + lib/bds/ai/openai_compatible_runtime.ex | 108 ++ lib/bds/ai/secret_backend.ex | 33 + ...0260424110431_add_ai_chat_usage_fields.exs | 12 + specs/ai.allium | 216 ++- test/bds/ai_test.exs | 459 ++++++ test/bds/repo/schema_migration_test.exs | 4 + 16 files changed, 2410 insertions(+), 8 deletions(-) create mode 100644 lib/bds/ai.ex create mode 100644 lib/bds/ai/catalog_meta.ex create mode 100644 lib/bds/ai/catalog_provider.ex create mode 100644 lib/bds/ai/chat_conversation.ex create mode 100644 lib/bds/ai/chat_message.ex create mode 100644 lib/bds/ai/http_client.ex create mode 100644 lib/bds/ai/in_flight.ex create mode 100644 lib/bds/ai/model.ex create mode 100644 lib/bds/ai/model_modality.ex create mode 100644 lib/bds/ai/openai_compatible_runtime.ex create mode 100644 lib/bds/ai/secret_backend.ex create mode 100644 priv/repo/migrations/20260424110431_add_ai_chat_usage_fields.exs create mode 100644 test/bds/ai_test.exs diff --git a/.vscode/settings.json b/.vscode/settings.json index a6888d0..4bd8082 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,6 +1,7 @@ { "chat.tools.terminal.autoApprove": { "mix": true, - "allium": true + "allium": true, + "command": true } } \ No newline at end of file diff --git a/lib/bds/ai.ex b/lib/bds/ai.ex new file mode 100644 index 0000000..539f6c3 --- /dev/null +++ b/lib/bds/ai.ex @@ -0,0 +1,1314 @@ +defmodule BDS.AI do + @moduledoc false + + import Ecto.Query + + alias BDS.AI.CatalogMeta + alias BDS.AI.CatalogProvider + alias BDS.AI.ChatConversation + alias BDS.AI.ChatMessage + alias BDS.AI.InFlight + alias BDS.AI.Model + alias BDS.AI.ModelModality + alias BDS.AI.OpenAICompatibleRuntime + alias BDS.AI.SecretBackend + alias BDS.Media.Media + alias BDS.Persistence + alias BDS.Posts.Post + alias BDS.Projects.Project + alias BDS.Repo + alias BDS.Settings.Setting + + @catalog_url "https://models.dev/api.json" + @default_system_prompt "You are the bDS AI backend. Be precise, prefer structured JSON when asked, and avoid inventing blog facts." + @default_max_output_tokens 16_384 + @chat_max_tool_rounds 10 + @default_context_window 128_000 + @model_preference_keys %{ + default: "ai.model.default", + chat: "ai.model.chat", + title: "ai.model.title", + image_analysis: "ai.model.image_analysis", + airplane_chat: "ai.airplane.model.chat", + airplane_title: "ai.airplane.model.title", + airplane_image_analysis: "ai.airplane.model.image_analysis" + } + + def put_endpoint(kind, attrs, opts \\ []) when is_atom(kind) and is_map(attrs) and is_list(opts) do + backend = Keyword.get(opts, :secret_backend, SecretBackend) + kind_key = Atom.to_string(kind) + + url = Map.get(attrs, :url) || Map.get(attrs, "url") + model = Map.get(attrs, :model) || Map.get(attrs, "model") + api_key = Map.get(attrs, :api_key) || Map.get(attrs, "api_key") + + with :ok <- put_setting("ai.#{kind_key}.url", url), + :ok <- put_setting("ai.#{kind_key}.model", model), + :ok <- put_secret("ai.#{kind_key}.api_key", api_key, backend) do + {:ok, %{kind: kind, url: url, api_key: api_key, model: model}} + end + end + + def get_endpoint(kind, opts \\ []) when is_atom(kind) and is_list(opts) do + backend = Keyword.get(opts, :secret_backend, SecretBackend) + kind_key = Atom.to_string(kind) + url = get_setting("ai.#{kind_key}.url") + model = get_setting("ai.#{kind_key}.model") + encrypted_api_key = get_setting(encrypted_key("ai.#{kind_key}.api_key")) + + cond do + is_nil(url) and is_nil(model) and is_nil(encrypted_api_key) -> + {:ok, nil} + + true -> + with {:ok, api_key} <- get_secret(encrypted_api_key, backend) do + {:ok, %{kind: kind, url: url, api_key: api_key, model: model}} + end + end + end + + def delete_endpoint(kind) when is_atom(kind) do + kind_key = Atom.to_string(kind) + delete_setting("ai.#{kind_key}.url") + delete_setting("ai.#{kind_key}.model") + delete_setting(encrypted_key("ai.#{kind_key}.api_key")) + :ok + end + + def refresh_model_catalog(opts \\ []) when is_list(opts) do + http_client = Keyword.get(opts, :http_client, BDS.AI.HttpClient) + + headers = + %{"accept" => "application/json"} + |> maybe_put_header("if-none-match", get_catalog_meta_value("etag")) + + with {:ok, response} <- http_get(http_client, @catalog_url, headers) do + case response.status do + 304 -> + :ok = put_catalog_meta("last_fetched_at", DateTime.utc_now() |> DateTime.to_iso8601()) + {:ok, %{success: true, models_updated: 0, not_modified: true}} + + 200 -> + payload = Jason.decode!(response.body) + models_updated = persist_catalog(payload) + + if etag = response.headers["etag"] do + :ok = put_catalog_meta("etag", etag) + end + + :ok = put_catalog_meta("last_fetched_at", DateTime.utc_now() |> DateTime.to_iso8601()) + + {:ok, %{success: true, models_updated: models_updated, not_modified: false}} + + status -> + {:error, %{kind: :http_error, status: status}} + end + end + end + + def list_catalog_providers do + Repo.all(from provider in CatalogProvider, order_by: [asc: provider.id]) + |> Enum.map(fn provider -> + %{ + id: provider.id, + name: provider.name, + env_keys: decode_json_list(provider.env_keys), + package_ref: provider.package_ref, + api_url: provider.api_url, + doc_url: provider.doc_url, + updated_at: provider.updated_at + } + end) + end + + def get_catalog_model(model_id, provider_id \\ nil) when is_binary(model_id) do + query = + from model in Model, + where: model.model_id == ^model_id, + order_by: [asc: model.provider] + + query = + case provider_id do + nil -> query + provider -> from model in query, where: model.provider == ^provider + end + + case Repo.one(query) do + nil -> {:error, :not_found} + model -> {:ok, format_model(model)} + end + end + + def catalog_meta(key) when is_binary(key) do + {:ok, get_catalog_meta_value(key)} + end + + def set_airplane_mode(enabled) when is_boolean(enabled) do + put_setting("ai.airplane_mode_enabled", Atom.to_string(enabled)) + end + + def airplane_mode? do + get_setting("ai.airplane_mode_enabled") == "true" + end + + def put_model_preference(key, model) when is_atom(key) and is_binary(model) do + case Map.fetch(@model_preference_keys, key) do + {:ok, setting_key} -> put_setting(setting_key, model) + :error -> {:error, :unknown_model_preference} + end + end + + def get_model_preference(key) when is_atom(key) do + case Map.fetch(@model_preference_keys, key) do + {:ok, setting_key} -> {:ok, get_setting(setting_key)} + :error -> {:error, :unknown_model_preference} + end + end + + def put_model_capabilities(model_id, attrs) when is_binary(model_id) and is_map(attrs) do + capabilities = %{ + supports_attachment: truthy?(Map.get(attrs, :supports_attachment) || Map.get(attrs, "supports_attachment")), + supports_tool_calls: truthy?(Map.get(attrs, :supports_tool_calls) || Map.get(attrs, "supports_tool_calls")) + } + + put_setting("ai.model_capabilities.#{model_id}", Jason.encode!(capabilities)) + end + + def detect_language(text, opts \\ []) when is_binary(text) and is_list(opts) do + run_one_shot( + :detect_language, + %{text: text}, + opts, + fn json, usage -> + {:ok, %{language_code: json["language_code"], usage: usage}} + end + ) + end + + def analyze_taxonomy(post_input, opts \\ []) when is_list(opts) do + with {:ok, post} <- normalize_post_input(post_input) do + run_one_shot( + :analyze_taxonomy, + post, + opts, + fn json, usage -> + {:ok, + %{ + tags: json["tags"] || [], + categories: json["categories"] || [], + usage: usage + }} + end + ) + end + end + + def analyze_post(post_input, opts \\ []) when is_list(opts) do + with {:ok, post} <- normalize_post_input(post_input) do + run_one_shot( + :analyze_post, + post, + opts, + fn json, usage -> + {:ok, + %{ + title: json["title"], + excerpt: json["excerpt"], + slug: json["slug"], + usage: usage + }} + end + ) + end + end + + def translate_post(post_input, target_language, opts \\ []) + when is_binary(target_language) and is_list(opts) do + with {:ok, post} <- normalize_post_input(post_input) do + run_one_shot( + :translate_post, + Map.put(post, :target_language, target_language), + opts, + fn json, usage -> + {:ok, + %{ + title: json["title"], + excerpt: json["excerpt"], + content: json["content"], + usage: usage + }} + end + ) + end + end + + def analyze_image(media_input, opts \\ []) when is_list(opts) do + with {:ok, media} <- normalize_media_input(media_input), + :ok <- ensure_image_media(media) do + run_one_shot( + :analyze_image, + media, + opts, + fn json, usage -> + {:ok, + %{ + title: json["title"], + alt: json["alt"], + caption: json["caption"], + usage: usage + }} + end + ) + end + end + + def translate_media(media_input, target_language, opts \\ []) + when is_binary(target_language) and is_list(opts) do + with {:ok, media} <- normalize_media_input(media_input) do + run_one_shot( + :translate_media, + Map.put(media, :target_language, target_language), + opts, + fn json, usage -> + {:ok, + %{ + title: json["title"], + alt: json["alt"], + caption: json["caption"], + usage: usage + }} + end + ) + end + end + + def start_chat(attrs \\ %{}) when is_map(attrs) do + now = Persistence.now_ms() + model = Map.get(attrs, :model) || Map.get(attrs, "model") + title = Map.get(attrs, :title) || Map.get(attrs, "title") || generated_chat_title(model) + + %ChatConversation{} + |> ChatConversation.changeset(%{ + id: Ecto.UUID.generate(), + title: title, + model: model, + copilot_session_id: Map.get(attrs, :copilot_session_id) || Map.get(attrs, "copilot_session_id"), + created_at: now, + updated_at: now + }) + |> Repo.insert() + |> case do + {:ok, conversation} -> {:ok, format_conversation(conversation)} + error -> error + end + end + + def list_chat_conversations do + Repo.all(from conversation in ChatConversation, order_by: [desc: conversation.updated_at]) + |> Enum.map(&format_conversation/1) + end + + def list_chat_messages(conversation_id) when is_binary(conversation_id) do + Repo.all( + from message in ChatMessage, + where: message.conversation_id == ^conversation_id, + order_by: [asc: message.created_at, asc: message.id] + ) + |> Enum.map(&format_chat_message/1) + end + + def send_chat_message(conversation_id, content, opts \\ []) + when is_binary(conversation_id) and is_binary(content) and is_list(opts) do + with %ChatConversation{} = conversation <- Repo.get(ChatConversation, conversation_id), + {:ok, user_message} <- persist_chat_message(%{ + conversation_id: conversation.id, + role: :user, + content: content, + created_at: Persistence.now_ms() + }) do + task = + Task.Supervisor.async_nolink(BDS.Tasks.TaskSupervisor, fn -> + receive do + :sandbox_ready -> :ok + end + + do_send_chat_message(conversation, user_message, opts) + end) + + InFlight.register(conversation.id, task.pid) + :ok = allow_repo_sandbox(task.pid) + send(task.pid, :sandbox_ready) + + try do + await_chat_task(task) + after + InFlight.unregister(conversation.id) + end + else + nil -> {:error, :not_found} + error -> error + end + end + + def cancel_chat(conversation_id) when is_binary(conversation_id) do + case InFlight.lookup(conversation_id) do + nil -> :ok + pid -> + _ = Task.Supervisor.terminate_child(BDS.Tasks.TaskSupervisor, pid) + :ok + end + end + + defp format_model(model) do + modalities = + Repo.all( + from modality in ModelModality, + where: modality.provider == ^model.provider and modality.model_id == ^model.model_id + ) + + %{ + provider: model.provider, + model_id: model.model_id, + name: model.name, + family: model.family, + supports_attachment: model.supports_attachment, + supports_reasoning: model.supports_reasoning, + supports_tool_calls: model.supports_tool_calls, + supports_structured_output: model.supports_structured_output, + supports_temperature: model.supports_temperature, + knowledge: model.knowledge, + release_date: model.release_date, + last_updated_date: model.last_updated_date, + open_weights: model.open_weights, + input_price: model.input_price, + output_price: model.output_price, + cache_read_price: model.cache_read_price, + cache_write_price: model.cache_write_price, + context_window: model.context_window, + max_input_tokens: model.max_input_tokens, + max_output_tokens: model.max_output_tokens, + interleaved: model.interleaved, + status: model.status, + updated_at: model.updated_at, + input_modalities: + modalities + |> Enum.filter(&(&1.direction == :input)) + |> Enum.map(&Atom.to_string(&1.modality)), + output_modalities: + modalities + |> Enum.filter(&(&1.direction == :output)) + |> Enum.map(&Atom.to_string(&1.modality)) + } + end + + defp format_conversation(conversation) do + %{ + id: conversation.id, + title: conversation.title, + model: conversation.model, + copilot_session_id: conversation.copilot_session_id, + created_at: conversation.created_at, + updated_at: conversation.updated_at + } + end + + defp format_chat_message(message) do + %{ + id: message.id, + conversation_id: message.conversation_id, + role: message.role, + content: message.content, + tool_call_id: message.tool_call_id, + tool_calls: decode_nullable_json(message.tool_calls), + token_usage_input: message.token_usage_input, + token_usage_output: message.token_usage_output, + cache_read_tokens: message.cache_read_tokens, + cache_write_tokens: message.cache_write_tokens, + created_at: message.created_at + } + end + + defp run_one_shot(operation, payload, opts, formatter) do + runtime = Keyword.get(opts, :runtime, OpenAICompatibleRuntime) + + with {:ok, endpoint, model, mode} <- resolve_runtime_target(operation, secret_backend: Keyword.get(opts, :secret_backend, SecretBackend)), + :ok <- validate_runtime_target(operation, model, mode), + request <- build_one_shot_request(operation, payload, model), + {:ok, response} <- runtime.generate(endpoint_with_model(endpoint, model), request, opts), + {:ok, json} <- extract_json_response(response), + usage <- normalize_usage(response.usage), + {:ok, result} <- formatter.(json, usage) do + {:ok, result} + end + end + + defp do_send_chat_message(conversation, _user_message, opts) do + runtime = Keyword.get(opts, :runtime, OpenAICompatibleRuntime) + project_id = Keyword.get(opts, :project_id, active_project_id()) + + with {:ok, endpoint, model, mode} <- + resolve_runtime_target( + :chat, + conversation: conversation, + secret_backend: Keyword.get(opts, :secret_backend, SecretBackend) + ), + :ok <- validate_runtime_target(:chat, model, mode), + messages <- load_chat_messages(conversation.id), + tools <- available_chat_tools(project_id, model), + {:ok, reply} <- chat_round(conversation, messages, endpoint, model, project_id, tools, runtime, opts, @chat_max_tool_rounds) do + {:ok, reply} + end + end + + defp chat_round(_conversation, _messages, _endpoint, _model, _project_id, _tools, _runtime, _opts, 0) do + {:error, %{kind: :tool_loop_exhausted}} + end + + defp chat_round(conversation, messages, endpoint, model, project_id, tools, runtime, opts, rounds_left) do + request = build_chat_request(conversation, messages, model, project_id, tools) + + with {:ok, response} <- runtime.generate(endpoint_with_model(endpoint, model), request, opts), + {:ok, assistant_message} <- persist_assistant_response(conversation.id, response), + :ok <- touch_conversation(conversation.id) do + tool_calls = decode_tool_calls(Map.get(response, :tool_calls)) + + cond do + tool_calls != [] and tools != [] -> + with {:ok, tool_messages} <- execute_tool_calls(conversation.id, tool_calls, project_id), + updated_messages <- load_chat_messages(conversation.id), + {:ok, reply} <- + chat_round( + Repo.get!(ChatConversation, conversation.id), + updated_messages, + endpoint, + model, + project_id, + tools, + runtime, + opts, + rounds_left - 1 + ) do + {:ok, Map.put(reply, :tool_messages, tool_messages)} + end + + true -> + {:ok, + %{ + conversation: format_conversation(Repo.get!(ChatConversation, conversation.id)), + assistant_message: format_chat_message(assistant_message), + tool_messages: [] + }} + end + end + end + + defp persist_assistant_response(conversation_id, response) do + usage = normalize_usage(response.usage) + + content = + case Map.get(response, :content) do + nil -> encode_nullable(Map.get(response, :json)) + value -> value + end + + persist_chat_message(%{ + conversation_id: conversation_id, + role: :assistant, + content: content, + tool_calls: encode_nullable(Map.get(response, :tool_calls)), + token_usage_input: usage.input_tokens, + token_usage_output: usage.output_tokens, + cache_read_tokens: usage.cache_read_tokens, + cache_write_tokens: usage.cache_write_tokens, + created_at: Persistence.now_ms() + }) + end + + defp execute_tool_calls(conversation_id, tool_calls, project_id) do + tool_messages = + Enum.map(tool_calls, fn tool_call -> + result = execute_tool(tool_call.name, tool_call.arguments || %{}, project_id) + + {:ok, message} = + persist_chat_message(%{ + conversation_id: conversation_id, + role: :tool, + content: Jason.encode!(result), + tool_call_id: tool_call.id, + created_at: Persistence.now_ms() + }) + + format_chat_message(message) + end) + + {:ok, tool_messages} + end + + defp execute_tool("blog_stats", _arguments, project_id) do + project_id = project_id || active_project_id() + + %{ + post_count: Repo.aggregate(from(post in Post, where: post.project_id == ^project_id), :count, :id), + media_count: Repo.aggregate(from(media in Media, where: media.project_id == ^project_id), :count, :id), + tag_count: count_distinct_string_list(Post, :tags, project_id), + category_count: count_distinct_string_list(Post, :categories, project_id) + } + end + + defp execute_tool("list_posts", arguments, project_id) do + limit = normalize_limit(arguments["limit"]) + + Repo.all( + from post in Post, + where: post.project_id == ^project_id, + order_by: [desc: post.updated_at], + limit: ^limit, + select: %{id: post.id, title: post.title, slug: post.slug, status: post.status} + ) + end + + defp execute_tool("list_media", arguments, project_id) do + limit = normalize_limit(arguments["limit"]) + + Repo.all( + from media in Media, + where: media.project_id == ^project_id, + order_by: [desc: media.updated_at], + limit: ^limit, + select: %{id: media.id, title: media.title, mime_type: media.mime_type, filename: media.filename} + ) + end + + defp execute_tool("render_table", arguments, _project_id) do + %{ + type: "table", + title: arguments["title"], + columns: arguments["columns"] || [], + rows: arguments["rows"] || [] + } + end + + defp execute_tool("render_chart", arguments, _project_id) do + %{ + type: "chart", + title: arguments["title"], + chart_type: arguments["chart_type"] || "bar", + series: arguments["series"] || [] + } + end + + defp execute_tool("render_form", arguments, _project_id) do + %{ + type: "form", + title: arguments["title"], + fields: arguments["fields"] || [] + } + end + + defp execute_tool(name, _arguments, _project_id) do + %{error: "unknown_tool", name: name} + end + + defp build_one_shot_request(operation, payload, model) do + %{ + operation: operation, + model: model, + max_output_tokens: @default_max_output_tokens, + messages: [ + %{"role" => "system", "content" => one_shot_system_prompt(operation)}, + %{"role" => "user", "content" => one_shot_user_content(operation, payload)} + ] + } + end + + defp build_chat_request(conversation, messages, model, project_id, tools) do + system_message = %{"role" => "system", "content" => chat_system_prompt(project_id)} + + %{ + operation: :chat, + conversation_id: conversation.id, + model: model, + max_output_tokens: @default_max_output_tokens, + tools: Enum.map(tools, & &1.spec), + messages: + [system_message | Enum.map(messages, &message_for_runtime/1)] + |> truncate_chat_messages(model, tools) + } + end + + defp message_for_runtime(%ChatMessage{} = message) do + base = %{"role" => Atom.to_string(message.role)} + + base = if is_binary(message.content), do: Map.put(base, "content", message.content), else: base + base = if is_binary(message.tool_call_id), do: Map.put(base, "tool_call_id", message.tool_call_id), else: base + + case decode_nullable_json(message.tool_calls) do + nil -> base + tool_calls -> Map.put(base, "tool_calls", tool_calls) + end + end + + defp truncate_chat_messages(messages, model, tools) do + context_window = model_context_window(model) + reserve = min(@default_max_output_tokens, max(div(context_window, 4), 512)) + tool_budget = length(tools) * 120 + max_budget = max(context_window - reserve - tool_budget, 512) + + [system | remainder] = messages + + {kept, _tokens} = + Enum.reduce(Enum.reverse(remainder), {[], approximate_message_tokens(system)}, fn message, {acc, used} -> + message_tokens = approximate_message_tokens(message) + + if used + message_tokens <= max_budget do + {[message | acc], used + message_tokens} + else + {acc, used} + end + end) + + [system | kept] + end + + defp available_chat_tools(project_id, model) do + if model_capabilities(model).supports_tool_calls do + project_tools = + if is_binary(project_id) do + [ + %{name: "blog_stats", spec: tool_spec("blog_stats", "Return aggregate blog statistics", %{"type" => "object", "properties" => %{}})}, + %{name: "list_posts", spec: tool_spec("list_posts", "List recent posts in the active project", limit_schema())}, + %{name: "list_media", spec: tool_spec("list_media", "List recent media items in the active project", limit_schema())} + ] + else + [] + end + + project_tools ++ + [ + %{name: "render_table", spec: tool_spec("render_table", "Return a structured table payload", render_table_schema())}, + %{name: "render_chart", spec: tool_spec("render_chart", "Return a structured chart payload", render_chart_schema())}, + %{name: "render_form", spec: tool_spec("render_form", "Return a structured form payload", render_form_schema())} + ] + else + [] + end + end + + defp resolve_runtime_target(operation, extra) do + mode = if airplane_mode?(), do: :airplane, else: :online + secret_backend = Keyword.get(extra, :secret_backend, SecretBackend) + + with {:ok, endpoint} <- fetch_endpoint_for_mode(mode, secret_backend), + {:ok, model} <- resolve_model_for_operation(operation, mode, endpoint, extra) do + {:ok, endpoint, model, mode} + end + end + + defp resolve_model_for_operation(:chat, :airplane, endpoint, _extra) do + {:ok, get_model_preference_value(:airplane_chat) || endpoint.model} + end + + defp resolve_model_for_operation(:chat, :online, endpoint, conversation: conversation) do + {:ok, conversation.model || get_model_preference_value(:chat) || get_model_preference_value(:default) || endpoint.model} + end + + defp resolve_model_for_operation(:chat, :online, endpoint, _extra) do + {:ok, get_model_preference_value(:chat) || get_model_preference_value(:default) || endpoint.model} + end + + defp resolve_model_for_operation(:analyze_image, :airplane, endpoint, _extra) do + {:ok, get_model_preference_value(:airplane_image_analysis) || endpoint.model} + end + + defp resolve_model_for_operation(:analyze_image, :online, endpoint, _extra) do + {:ok, get_model_preference_value(:image_analysis) || get_model_preference_value(:default) || endpoint.model} + end + + defp resolve_model_for_operation(_operation, :airplane, endpoint, _extra) do + {:ok, get_model_preference_value(:airplane_title) || endpoint.model} + end + + defp resolve_model_for_operation(_operation, :online, endpoint, _extra) do + {:ok, get_model_preference_value(:title) || get_model_preference_value(:default) || endpoint.model} + end + + defp validate_runtime_target(:analyze_image, model, _mode) do + if model_capabilities(model).supports_attachment do + :ok + else + {:error, %{kind: :model_capability_missing, capability: :supports_attachment, model: model}} + end + end + + defp validate_runtime_target(_operation, _model, _mode), do: :ok + + defp fetch_endpoint_for_mode(mode, secret_backend) do + with {:ok, endpoint} <- get_endpoint(mode, secret_backend: secret_backend) do + case endpoint do + %{url: url, model: model} = loaded when is_binary(url) and url != "" and is_binary(model) and model != "" -> + if mode == :online and blank?(loaded.api_key) do + {:error, %{kind: :endpoint_not_configured, endpoint: mode}} + else + {:ok, loaded} + end + + _other -> + {:error, %{kind: :endpoint_not_configured, endpoint: mode}} + end + end + end + + defp endpoint_with_model(endpoint, model), do: Map.put(endpoint, :model, model) + + defp get_model_preference_value(key) do + case get_model_preference(key) do + {:ok, value} -> value + _other -> nil + end + end + + defp chat_system_prompt(project_id) do + base = get_setting("ai.system_prompt") || @default_system_prompt + + case project_stats_summary(project_id) do + nil -> base + summary -> base <> "\n\nCurrent blog statistics:\n" <> summary + end + end + + defp project_stats_summary(nil), do: nil + + defp project_stats_summary(project_id) do + post_count = Repo.aggregate(from(post in Post, where: post.project_id == ^project_id), :count, :id) + media_count = Repo.aggregate(from(media in Media, where: media.project_id == ^project_id), :count, :id) + tag_count = count_distinct_string_list(Post, :tags, project_id) + category_count = count_distinct_string_list(Post, :categories, project_id) + + Enum.join( + [ + "Posts: #{post_count}", + "Media: #{media_count}", + "Tags: #{tag_count}", + "Categories: #{category_count}" + ], + "\n" + ) + end + + defp count_distinct_string_list(schema, field, project_id) do + Repo.all(from record in schema, where: field(record, :project_id) == ^project_id, select: field(record, ^field)) + |> List.flatten() + |> Enum.reject(&blank?/1) + |> MapSet.new() + |> MapSet.size() + end + + defp one_shot_system_prompt(:detect_language) do + "Return JSON with exactly one key: language_code." + end + + defp one_shot_system_prompt(:analyze_taxonomy) do + "Return JSON with keys tags and categories, each an array of short strings." + end + + defp one_shot_system_prompt(:analyze_post) do + "Return JSON with keys title, excerpt, and slug." + end + + defp one_shot_system_prompt(:translate_post) do + "Return JSON with keys title, excerpt, and content. Preserve Markdown structure." + end + + defp one_shot_system_prompt(:analyze_image) do + "Return JSON with keys title, alt, and caption for the provided image." + end + + defp one_shot_system_prompt(:translate_media) do + "Return JSON with keys title, alt, and caption translated to the requested language." + end + + defp one_shot_user_content(:detect_language, %{text: text}) do + "Detect the language of this text: #{text}" + end + + defp one_shot_user_content(:analyze_taxonomy, post) do + "Suggest categories and tags for the following post.\nTitle: #{post.title}\nExcerpt: #{post.excerpt}\nContent: #{truncate_text(post.content, 2000)}" + end + + defp one_shot_user_content(:analyze_post, post) do + "Suggest an improved title, excerpt, and slug.\nTitle: #{post.title}\nExcerpt: #{post.excerpt}\nContent: #{truncate_text(post.content, 2000)}" + end + + defp one_shot_user_content(:translate_post, post) do + "Translate this post to #{post.target_language}.\nTitle: #{post.title}\nExcerpt: #{post.excerpt}\nContent: #{post.content}" + end + + defp one_shot_user_content(:analyze_image, media) do + [ + %{"type" => "text", "text" => "Analyze this image and return title, alt text, and caption."}, + %{"type" => "image_url", "image_url" => %{"url" => media.image_url}} + ] + end + + defp one_shot_user_content(:translate_media, media) do + "Translate this media metadata to #{media.target_language}.\nTitle: #{media.title}\nAlt: #{media.alt}\nCaption: #{media.caption}" + end + + defp extract_json_response(%{json: json}) when is_map(json), do: {:ok, json} + + defp extract_json_response(%{content: content}) when is_binary(content) do + case Jason.decode(content) do + {:ok, json} when is_map(json) -> {:ok, json} + _other -> {:error, %{kind: :invalid_json_response}} + end + end + + defp extract_json_response(_response), do: {:error, %{kind: :invalid_json_response}} + + defp normalize_post_input(%Post{} = post) do + {:ok, %{title: post.title || "", excerpt: post.excerpt || "", content: post.content || ""}} + end + + defp normalize_post_input(post_id) when is_binary(post_id) do + case Repo.get(Post, post_id) do + nil -> {:error, :not_found} + post -> normalize_post_input(post) + end + end + + defp normalize_post_input(attrs) when is_map(attrs) do + {:ok, + %{ + title: Map.get(attrs, :title) || Map.get(attrs, "title") || "", + excerpt: Map.get(attrs, :excerpt) || Map.get(attrs, "excerpt") || "", + content: Map.get(attrs, :content) || Map.get(attrs, "content") || "" + }} + end + + defp normalize_media_input(%Media{} = media) do + {:ok, + %{ + mime_type: media.mime_type, + title: media.title || "", + alt: media.alt || "", + caption: media.caption || "", + image_url: Map.get(media, :image_url) || media_path_to_file_url(media.file_path) + }} + end + + defp normalize_media_input(media_id) when is_binary(media_id) do + case Repo.get(Media, media_id) do + nil -> {:error, :not_found} + media -> normalize_media_input(media) + end + end + + defp normalize_media_input(attrs) when is_map(attrs) do + {:ok, + %{ + mime_type: Map.get(attrs, :mime_type) || Map.get(attrs, "mime_type"), + title: Map.get(attrs, :title) || Map.get(attrs, "title") || "", + alt: Map.get(attrs, :alt) || Map.get(attrs, "alt") || "", + caption: Map.get(attrs, :caption) || Map.get(attrs, "caption") || "", + image_url: Map.get(attrs, :image_url) || Map.get(attrs, "image_url") + }} + end + + defp ensure_image_media(%{mime_type: "image/" <> _rest}), do: :ok + defp ensure_image_media(_media), do: {:error, %{kind: :invalid_media_type}} + + defp media_path_to_file_url(nil), do: nil + defp media_path_to_file_url(path), do: "file://" <> path + + defp model_capabilities(model_id) do + overrides = decode_model_capabilities_override(model_id) + + from_catalog = + case get_catalog_model(model_id) do + {:ok, model} -> + %{ + supports_attachment: model.supports_attachment or ("image" in model.input_modalities), + supports_tool_calls: model.supports_tool_calls + } + + _other -> + inferred_model_capabilities(model_id) + end + + Map.merge(from_catalog, overrides) + end + + defp inferred_model_capabilities(model_id) do + normalized = String.downcase(model_id) + + %{ + supports_attachment: + String.contains?(normalized, "4o") or String.contains?(normalized, "vision") or + String.contains?(normalized, "llava"), + supports_tool_calls: + String.contains?(normalized, "gpt") or String.contains?(normalized, "claude") or + String.contains?(normalized, "tool") + } + end + + defp decode_model_capabilities_override(model_id) do + case get_setting("ai.model_capabilities.#{model_id}") do + nil -> %{} + value -> Jason.decode!(value) |> atomize_map_keys() + end + end + + defp atomize_map_keys(map) do + Enum.into(map, %{}, fn {key, value} -> {String.to_atom(key), value} end) + end + + defp generated_chat_title(nil), do: "New Chat" + defp generated_chat_title(model), do: "Chat with #{model}" + + defp load_chat_messages(conversation_id) do + Repo.all( + from message in ChatMessage, + where: message.conversation_id == ^conversation_id, + order_by: [asc: message.created_at, asc: message.id] + ) + end + + defp persist_chat_message(attrs) do + %ChatMessage{} + |> ChatMessage.changeset(attrs) + |> Repo.insert() + end + + defp touch_conversation(conversation_id) do + now = Persistence.now_ms() + + Repo.update_all( + from(conversation in ChatConversation, where: conversation.id == ^conversation_id), + set: [updated_at: now] + ) + + :ok + end + + defp await_chat_task(task) do + receive do + {ref, result} when ref == task.ref -> + Process.demonitor(task.ref, [:flush]) + result + + {:DOWN, ref, :process, _pid, reason} when ref == task.ref -> + case reason do + :normal -> + receive do + {^ref, result} -> result + after + 10 -> {:error, :cancelled} + end + + :shutdown -> {:error, :cancelled} + {:shutdown, _detail} -> {:error, :cancelled} + _other -> {:error, :cancelled} + end + end + end + + defp decode_tool_calls(nil), do: [] + + defp decode_tool_calls(tool_calls) when is_list(tool_calls) do + Enum.map(tool_calls, fn tool_call -> + %{ + id: tool_call[:id] || tool_call["id"], + name: tool_call[:name] || tool_call["name"], + arguments: tool_call[:arguments] || tool_call["arguments"] || %{} + } + end) + end + + defp approximate_message_tokens(message) when is_map(message) do + message + |> Map.values() + |> Enum.map(&approximate_value_tokens/1) + |> Enum.sum() + |> Kernel.+(4) + end + + defp approximate_value_tokens(value) when is_binary(value), do: div(String.length(value), 4) + 1 + defp approximate_value_tokens(value) when is_list(value), do: Enum.map(value, &approximate_value_tokens/1) |> Enum.sum() + defp approximate_value_tokens(value) when is_map(value), do: Jason.encode!(value) |> approximate_value_tokens() + defp approximate_value_tokens(_value), do: 1 + + defp model_context_window(model_id) do + case get_catalog_model(model_id) do + {:ok, model} when is_integer(model.context_window) and model.context_window > 0 -> model.context_window + _other -> @default_context_window + end + end + + defp normalize_usage(usage) when is_map(usage) do + %{ + input_tokens: usage[:input_tokens] || usage["input_tokens"], + output_tokens: usage[:output_tokens] || usage["output_tokens"], + cache_read_tokens: usage[:cache_read_tokens] || usage["cache_read_tokens"], + cache_write_tokens: usage[:cache_write_tokens] || usage["cache_write_tokens"] + } + end + + defp normalize_usage(_usage) do + %{ + input_tokens: nil, + output_tokens: nil, + cache_read_tokens: nil, + cache_write_tokens: nil + } + end + + defp tool_spec(name, description, parameters) do + %{ + "type" => "function", + "function" => %{ + "name" => name, + "description" => description, + "parameters" => parameters + } + } + end + + defp limit_schema do + %{ + "type" => "object", + "properties" => %{ + "limit" => %{"type" => "integer", "minimum" => 1, "maximum" => 50} + } + } + end + + defp render_table_schema do + %{ + "type" => "object", + "properties" => %{ + "title" => %{"type" => "string"}, + "columns" => %{"type" => "array"}, + "rows" => %{"type" => "array"} + } + } + end + + defp render_chart_schema do + %{ + "type" => "object", + "properties" => %{ + "title" => %{"type" => "string"}, + "chart_type" => %{"type" => "string"}, + "series" => %{"type" => "array"} + } + } + end + + defp render_form_schema do + %{ + "type" => "object", + "properties" => %{ + "title" => %{"type" => "string"}, + "fields" => %{"type" => "array"} + } + } + end + + defp normalize_limit(value) when is_integer(value) and value > 0 and value <= 50, do: value + defp normalize_limit(_value), do: 10 + + defp truncate_text(nil, _max_length), do: "" + + defp truncate_text(text, max_length) when is_binary(text) do + if String.length(text) <= max_length do + text + else + String.slice(text, 0, max_length) + end + end + + defp active_project_id do + Repo.one(from project in Project, where: project.is_active == true, select: project.id) + end + + defp allow_repo_sandbox(pid) when is_pid(pid) do + if Code.ensure_loaded?(Ecto.Adapters.SQL.Sandbox) do + try do + Ecto.Adapters.SQL.Sandbox.allow(BDS.Repo, self(), pid) + rescue + _error -> :ok + end + else + :ok + end + + :ok + end + + defp persist_catalog(payload) do + now = Persistence.now_ms() + + Repo.transaction(fn -> + Repo.delete_all(ModelModality) + Repo.delete_all(Model) + Repo.delete_all(CatalogProvider) + + Enum.reduce(payload, 0, fn {provider_id, provider_data}, count -> + provider_attrs = %{ + id: provider_id, + name: Map.get(provider_data, "name", provider_id), + env_keys: Jason.encode!(Map.get(provider_data, "env", [])), + package_ref: Map.get(provider_data, "npm"), + api_url: Map.get(provider_data, "api"), + doc_url: Map.get(provider_data, "doc"), + updated_at: now + } + + %CatalogProvider{} + |> CatalogProvider.changeset(provider_attrs) + |> Repo.insert!() + + models = Map.get(provider_data, "models", %{}) + + Enum.reduce(models, count, fn {model_id, model_data}, inner_count -> + model_attrs = %{ + provider: provider_id, + model_id: model_id, + name: Map.get(model_data, "name", model_id), + family: Map.get(model_data, "family"), + supports_attachment: Map.get(model_data, "attachment", false), + supports_reasoning: Map.get(model_data, "reasoning", false), + supports_tool_calls: Map.get(model_data, "tool_call", false), + supports_structured_output: Map.get(model_data, "structured_output", false), + supports_temperature: Map.get(model_data, "temperature", false), + knowledge: Map.get(model_data, "knowledge"), + release_date: Map.get(model_data, "release_date"), + last_updated_date: Map.get(model_data, "last_updated"), + open_weights: Map.get(model_data, "open_weights", false), + input_price: get_in(model_data, ["cost", "input"]), + output_price: get_in(model_data, ["cost", "output"]), + cache_read_price: get_in(model_data, ["cost", "cache_read"]), + cache_write_price: get_in(model_data, ["cost", "cache_write"]), + context_window: get_in(model_data, ["limit", "context"]) || 0, + max_input_tokens: get_in(model_data, ["limit", "input"]) || 0, + max_output_tokens: get_in(model_data, ["limit", "output"]) || 0, + interleaved: encode_nullable(Map.get(model_data, "interleaved")), + status: Map.get(model_data, "status"), + updated_at: now + } + + %Model{} + |> Model.changeset(model_attrs) + |> Repo.insert!() + + insert_modalities(provider_id, model_id, Map.get(model_data, "input_modalities", []), :input) + insert_modalities(provider_id, model_id, Map.get(model_data, "output_modalities", []), :output) + + inner_count + 1 + end) + end) + end) + |> case do + {:ok, count} -> count + {:error, reason} -> raise reason + end + end + + defp insert_modalities(provider_id, model_id, modalities, direction) do + Enum.each(modalities, fn modality -> + %ModelModality{} + |> ModelModality.changeset(%{ + provider: provider_id, + model_id: model_id, + direction: direction, + modality: parse_modality(modality) + }) + |> Repo.insert!() + end) + end + + defp parse_modality("text"), do: :text + defp parse_modality("image"), do: :image + defp parse_modality("audio"), do: :audio + defp parse_modality("file"), do: :file + defp parse_modality("tool"), do: :tool + defp parse_modality(other) when is_binary(other), do: String.to_atom(other) + + defp encode_nullable(nil), do: nil + defp encode_nullable(value), do: Jason.encode!(value) + + defp decode_nullable_json(nil), do: nil + defp decode_nullable_json(value) when is_binary(value), do: Jason.decode!(value) + + defp http_get(client, url, headers) when is_atom(client), do: client.get(url, headers) + defp http_get(client, url, headers) when is_function(client, 2), do: client.(url, headers) + + defp put_secret(_key, nil, _backend), do: :ok + + defp put_secret(key, value, backend) do + with {:ok, encrypted_value} <- backend.encrypt(value) do + put_setting(encrypted_key(key), encrypted_value) + end + end + + defp get_secret(nil, _backend), do: {:ok, nil} + + defp get_secret(value, backend) do + backend.decrypt(value) + end + + defp encrypted_key(key), do: "__encrypted_#{key}" + + defp get_catalog_meta_value(key) do + case Repo.get(CatalogMeta, key) do + nil -> get_setting("ai.catalog.#{key}") + meta -> meta.value + end + end + + defp put_catalog_meta(key, value) do + %CatalogMeta{} + |> CatalogMeta.changeset(%{key: key, value: value}) + |> Repo.insert( + on_conflict: [set: [value: value]], + conflict_target: [:key] + ) + + :ok + end + + defp get_setting(key) do + case Repo.get(Setting, key) do + nil -> nil + setting -> setting.value + end + end + + defp put_setting(key, value) when is_binary(key) and is_binary(value) do + now = Persistence.now_ms() + + (%Setting{} + |> Setting.changeset(%{key: key, value: value, updated_at: now})) + |> Repo.insert( + on_conflict: [set: [value: value, updated_at: now]], + conflict_target: [:key] + ) + + :ok + end + + defp delete_setting(key) do + Repo.delete_all(from setting in Setting, where: setting.key == ^key) + :ok + end + + defp blank?(value), do: value in [nil, ""] + + defp truthy?(value), do: value in [true, "true", 1, "1"] + + defp maybe_put_header(headers, _key, nil), do: headers + defp maybe_put_header(headers, key, value), do: Map.put(headers, key, value) + + defp decode_json_list(nil), do: [] + defp decode_json_list(value), do: Jason.decode!(value) +end diff --git a/lib/bds/ai/catalog_meta.ex b/lib/bds/ai/catalog_meta.ex new file mode 100644 index 0000000..3da526d --- /dev/null +++ b/lib/bds/ai/catalog_meta.ex @@ -0,0 +1,18 @@ +defmodule BDS.AI.CatalogMeta do + @moduledoc false + + use Ecto.Schema + import Ecto.Changeset + + @primary_key {:key, :string, autogenerate: false} + + schema "ai_catalog_meta" do + field :value, :string + end + + def changeset(meta, attrs) do + meta + |> cast(attrs, [:key, :value], empty_values: [nil]) + |> validate_required([:key, :value]) + end +end diff --git a/lib/bds/ai/catalog_provider.ex b/lib/bds/ai/catalog_provider.ex new file mode 100644 index 0000000..12f44e9 --- /dev/null +++ b/lib/bds/ai/catalog_provider.ex @@ -0,0 +1,23 @@ +defmodule BDS.AI.CatalogProvider do + @moduledoc false + + use Ecto.Schema + import Ecto.Changeset + + @primary_key {:id, :string, autogenerate: false} + + schema "ai_providers" do + field :name, :string + field :env_keys, :string, source: :env + field :package_ref, :string + field :api_url, :string, source: :api + field :doc_url, :string, source: :doc + field :updated_at, :integer + end + + def changeset(provider, attrs) do + provider + |> cast(attrs, [:id, :name, :env_keys, :package_ref, :api_url, :doc_url, :updated_at], empty_values: [nil]) + |> validate_required([:id, :name, :updated_at]) + end +end diff --git a/lib/bds/ai/chat_conversation.ex b/lib/bds/ai/chat_conversation.ex new file mode 100644 index 0000000..8751079 --- /dev/null +++ b/lib/bds/ai/chat_conversation.ex @@ -0,0 +1,22 @@ +defmodule BDS.AI.ChatConversation do + @moduledoc false + + use Ecto.Schema + import Ecto.Changeset + + @primary_key {:id, :string, autogenerate: false} + + schema "chat_conversations" do + field :title, :string + field :model, :string + field :copilot_session_id, :string + field :created_at, :integer + field :updated_at, :integer + end + + def changeset(conversation, attrs) do + conversation + |> cast(attrs, [:id, :title, :model, :copilot_session_id, :created_at, :updated_at], empty_values: [nil]) + |> validate_required([:id, :title, :created_at, :updated_at]) + end +end diff --git a/lib/bds/ai/chat_message.ex b/lib/bds/ai/chat_message.ex new file mode 100644 index 0000000..b472368 --- /dev/null +++ b/lib/bds/ai/chat_message.ex @@ -0,0 +1,41 @@ +defmodule BDS.AI.ChatMessage do + @moduledoc false + + use Ecto.Schema + import Ecto.Changeset + + @foreign_key_type :string + @roles [:system, :user, :assistant, :tool] + + schema "chat_messages" do + belongs_to :conversation, BDS.AI.ChatConversation, references: :id, type: :string + + field :role, Ecto.Enum, values: @roles + field :content, :string + field :tool_call_id, :string + field :tool_calls, :string + field :token_usage_input, :integer + field :token_usage_output, :integer + field :cache_read_tokens, :integer + field :cache_write_tokens, :integer + field :created_at, :integer + end + + def changeset(message, attrs) do + message + |> cast(attrs, [ + :conversation_id, + :role, + :content, + :tool_call_id, + :tool_calls, + :token_usage_input, + :token_usage_output, + :cache_read_tokens, + :cache_write_tokens, + :created_at + ], empty_values: [nil]) + |> validate_required([:conversation_id, :role, :created_at]) + |> assoc_constraint(:conversation) + end +end diff --git a/lib/bds/ai/http_client.ex b/lib/bds/ai/http_client.ex new file mode 100644 index 0000000..1e73f4d --- /dev/null +++ b/lib/bds/ai/http_client.ex @@ -0,0 +1,51 @@ +defmodule BDS.AI.HttpClient do + @moduledoc false + + def get(url, headers) when is_binary(url) and is_map(headers) do + request = {String.to_charlist(url), Enum.map(headers, fn {key, value} -> {String.to_charlist(key), String.to_charlist(value)} end)} + + :inets.start() + :ssl.start() + + case :httpc.request(:get, request, [], body_format: :binary) do + {:ok, {{_version, status, _reason}, response_headers, body}} -> + {:ok, + %{ + status: status, + headers: normalize_headers(response_headers), + body: body + }} + + {:error, reason} -> + {:error, reason} + end + end + + def post(url, headers, body) + when is_binary(url) and is_map(headers) and is_binary(body) do + request = + {String.to_charlist(url), Enum.map(headers, fn {key, value} -> {String.to_charlist(key), String.to_charlist(value)} end), ~c"application/json", body} + + :inets.start() + :ssl.start() + + case :httpc.request(:post, request, [], body_format: :binary) do + {:ok, {{_version, status, _reason}, response_headers, response_body}} -> + {:ok, + %{ + status: status, + headers: normalize_headers(response_headers), + body: response_body + }} + + {:error, reason} -> + {:error, reason} + end + end + + defp normalize_headers(headers) do + Enum.into(headers, %{}, fn {key, value} -> + {key |> to_string() |> String.downcase(), to_string(value)} + end) + end +end diff --git a/lib/bds/ai/in_flight.ex b/lib/bds/ai/in_flight.ex new file mode 100644 index 0000000..13e1e98 --- /dev/null +++ b/lib/bds/ai/in_flight.ex @@ -0,0 +1,29 @@ +defmodule BDS.AI.InFlight do + @moduledoc false + + @table :bds_ai_in_flight + + def register(conversation_id, pid) when is_binary(conversation_id) and is_pid(pid) do + :ets.insert(table(), {conversation_id, pid}) + :ok + end + + def unregister(conversation_id) when is_binary(conversation_id) do + :ets.delete(table(), conversation_id) + :ok + end + + def lookup(conversation_id) when is_binary(conversation_id) do + case :ets.lookup(table(), conversation_id) do + [{^conversation_id, pid}] -> pid + _other -> nil + end + end + + defp table do + case :ets.whereis(@table) do + :undefined -> :ets.new(@table, [:named_table, :public, :set, read_concurrency: true]) + table -> table + end + end +end diff --git a/lib/bds/ai/model.ex b/lib/bds/ai/model.ex new file mode 100644 index 0000000..f2132cd --- /dev/null +++ b/lib/bds/ai/model.ex @@ -0,0 +1,64 @@ +defmodule BDS.AI.Model do + @moduledoc false + + use Ecto.Schema + import Ecto.Changeset + + @primary_key false + + schema "ai_models" do + field :provider, :string, primary_key: true + field :model_id, :string, primary_key: true + field :name, :string + field :family, :string + field :supports_attachment, :boolean, source: :attachment, default: false + field :supports_reasoning, :boolean, source: :reasoning, default: false + field :supports_tool_calls, :boolean, source: :tool_call, default: false + field :supports_structured_output, :boolean, source: :structured_output, default: false + field :supports_temperature, :boolean, source: :temperature, default: false + field :knowledge, :string + field :release_date, :string + field :last_updated_date, :string + field :open_weights, :boolean, default: false + field :input_price, :integer + field :output_price, :integer + field :cache_read_price, :integer + field :cache_write_price, :integer + field :context_window, :integer + field :max_input_tokens, :integer + field :max_output_tokens, :integer + field :interleaved, :string + field :status, :string + field :updated_at, :integer + end + + def changeset(model, attrs) do + model + |> cast(attrs, [ + :provider, + :model_id, + :name, + :family, + :supports_attachment, + :supports_reasoning, + :supports_tool_calls, + :supports_structured_output, + :supports_temperature, + :knowledge, + :release_date, + :last_updated_date, + :open_weights, + :input_price, + :output_price, + :cache_read_price, + :cache_write_price, + :context_window, + :max_input_tokens, + :max_output_tokens, + :interleaved, + :status, + :updated_at + ], empty_values: [nil]) + |> validate_required([:provider, :model_id, :name, :context_window, :max_input_tokens, :max_output_tokens, :updated_at]) + end +end diff --git a/lib/bds/ai/model_modality.ex b/lib/bds/ai/model_modality.ex new file mode 100644 index 0000000..e3300e4 --- /dev/null +++ b/lib/bds/ai/model_modality.ex @@ -0,0 +1,21 @@ +defmodule BDS.AI.ModelModality do + @moduledoc false + + use Ecto.Schema + import Ecto.Changeset + + @primary_key false + + schema "ai_model_modalities" do + field :provider, :string, primary_key: true + field :model_id, :string, primary_key: true + field :direction, Ecto.Enum, values: [:input, :output], primary_key: true + field :modality, Ecto.Enum, values: [:text, :image, :audio, :file, :tool], primary_key: true + end + + def changeset(modality, attrs) do + modality + |> cast(attrs, [:provider, :model_id, :direction, :modality], empty_values: [nil]) + |> validate_required([:provider, :model_id, :direction, :modality]) + end +end diff --git a/lib/bds/ai/openai_compatible_runtime.ex b/lib/bds/ai/openai_compatible_runtime.ex new file mode 100644 index 0000000..9071193 --- /dev/null +++ b/lib/bds/ai/openai_compatible_runtime.ex @@ -0,0 +1,108 @@ +defmodule BDS.AI.OpenAICompatibleRuntime do + @moduledoc false + + alias BDS.AI.HttpClient + + def generate(endpoint, request, _opts) when is_map(endpoint) and is_map(request) do + url = completions_url(endpoint.url) + + headers = + %{ + "content-type" => "application/json", + "accept" => "application/json" + } + |> maybe_put_auth(endpoint.api_key) + + payload = %{ + "model" => request.model, + "messages" => request.messages, + "max_tokens" => request.max_output_tokens + } + |> maybe_put_tools(request.tools) + + with {:ok, response} <- HttpClient.post(url, headers, Jason.encode!(payload)), + 200 <- response.status do + normalize_response(response.body) + else + status when is_integer(status) -> {:error, %{kind: :http_error, status: status}} + {:error, reason} -> {:error, %{kind: :http_error, reason: reason}} + end + end + + defp normalize_response(body) do + payload = Jason.decode!(body) + message = get_in(payload, ["choices", Access.at(0), "message"]) || %{} + content = normalize_content(message["content"]) + tool_calls = normalize_tool_calls(message["tool_calls"] || []) + usage = normalize_usage(payload["usage"] || %{}) + + json = + case content do + nil -> nil + value when is_binary(value) -> + case Jason.decode(value) do + {:ok, decoded} when is_map(decoded) -> decoded + _other -> nil + end + end + + {:ok, %{content: content, json: json, tool_calls: tool_calls, usage: usage}} + end + + defp completions_url(url) do + cond do + String.ends_with?(url, "/chat/completions") -> url + String.ends_with?(url, "/") -> url <> "chat/completions" + true -> url <> "/chat/completions" + end + end + + defp maybe_put_auth(headers, nil), do: headers + defp maybe_put_auth(headers, ""), do: headers + defp maybe_put_auth(headers, api_key), do: Map.put(headers, "authorization", "Bearer #{api_key}") + + defp maybe_put_tools(payload, []), do: payload + defp maybe_put_tools(payload, nil), do: payload + + defp maybe_put_tools(payload, tools) do + Map.put(payload, "tools", tools) + |> Map.put("tool_choice", "auto") + end + + defp normalize_tool_calls(tool_calls) do + Enum.map(tool_calls, fn tool_call -> + %{ + id: tool_call["id"], + name: get_in(tool_call, ["function", "name"]), + arguments: decode_arguments(get_in(tool_call, ["function", "arguments"])) + } + end) + end + + defp decode_arguments(nil), do: %{} + + defp decode_arguments(arguments) when is_binary(arguments) do + case Jason.decode(arguments) do + {:ok, decoded} when is_map(decoded) -> decoded + _other -> %{} + end + end + + defp normalize_content(nil), do: nil + defp normalize_content(content) when is_binary(content), do: content + + defp normalize_content(content) when is_list(content) do + content + |> Enum.map(fn item -> item["text"] || "" end) + |> Enum.join() + end + + defp normalize_usage(usage) do + %{ + input_tokens: usage["prompt_tokens"], + output_tokens: usage["completion_tokens"], + cache_read_tokens: get_in(usage, ["prompt_tokens_details", "cached_tokens"]), + cache_write_tokens: get_in(usage, ["completion_tokens_details", "cached_tokens"]) + } + end +end diff --git a/lib/bds/ai/secret_backend.ex b/lib/bds/ai/secret_backend.ex new file mode 100644 index 0000000..9e92f32 --- /dev/null +++ b/lib/bds/ai/secret_backend.ex @@ -0,0 +1,33 @@ +defmodule BDS.AI.SecretBackend do + @moduledoc false + + @aad "bds-ai-secret" + + def encrypt(value) when is_binary(value) do + key = secret_key() + iv = :crypto.strong_rand_bytes(12) + + {ciphertext, tag} = + :crypto.crypto_one_time_aead(:aes_256_gcm, key, iv, value, @aad, true) + + {:ok, Base.encode64(iv <> tag <> ciphertext)} + end + + def decrypt(encoded) when is_binary(encoded) do + with {:ok, binary} <- Base.decode64(encoded), + <> <- binary, + plaintext when is_binary(plaintext) <- + :crypto.crypto_one_time_aead(:aes_256_gcm, secret_key(), iv, ciphertext, @aad, tag, false) do + {:ok, plaintext} + else + _other -> {:error, :invalid_ciphertext} + end + end + + defp secret_key do + case Application.get_env(:bds, :ai_secret_key) do + key when is_binary(key) and byte_size(key) >= 32 -> binary_part(key, 0, 32) + _other -> :crypto.hash(:sha256, Atom.to_string(node()) <> ":bds:ai") + end + end +end diff --git a/priv/repo/migrations/20260424110431_add_ai_chat_usage_fields.exs b/priv/repo/migrations/20260424110431_add_ai_chat_usage_fields.exs new file mode 100644 index 0000000..9fc6636 --- /dev/null +++ b/priv/repo/migrations/20260424110431_add_ai_chat_usage_fields.exs @@ -0,0 +1,12 @@ +defmodule BDS.Repo.Migrations.AddAiChatUsageFields do + use Ecto.Migration + + def change do + alter table(:chat_messages) do + add :token_usage_input, :integer + add :token_usage_output, :integer + add :cache_read_tokens, :integer + add :cache_write_tokens, :integer + end + end +end diff --git a/specs/ai.allium b/specs/ai.allium index 9869947..9173e7d 100644 --- a/specs/ai.allium +++ b/specs/ai.allium @@ -18,6 +18,61 @@ entity AiEndpoint { -- airplane: local model (Ollama, LM Studio, etc.) } +entity AiCatalogProvider { + id: String + name: String + env_keys: Set + package_ref: String? + api_url: String? + doc_url: String? + updated_at: Timestamp +} + +entity AiModel { + provider: AiCatalogProvider + model_id: String + name: String + family: String? + supports_attachment: Boolean + supports_reasoning: Boolean + supports_tool_calls: Boolean + supports_structured_output: Boolean + supports_temperature: Boolean + knowledge: String? + release_date: String? + last_updated_date: String? + open_weights: Boolean + input_price: Integer? + output_price: Integer? + cache_read_price: Integer? + cache_write_price: Integer? + context_window: Integer + max_input_tokens: Integer + max_output_tokens: Integer + interleaved: String? + status: String? + updated_at: Timestamp + + -- Relationships + modalities: AiModelModality with provider = this.provider and model_id = this.model_id + + -- Derived + input_modalities: modalities where direction = input -> modality + output_modalities: modalities where direction = output -> modality +} + +entity AiModelModality { + provider: AiCatalogProvider + model_id: String + direction: input | output + modality: text | image | audio | file | tool +} + +entity AiCatalogMeta { + key: String + value: String +} + surface AiEndpointSurface { context endpoint: AiEndpoint @@ -28,6 +83,37 @@ surface AiEndpointSurface { endpoint.model } + surface AiModelSurface { + context catalog_model: AiModel + + exposes: + catalog_model.provider + catalog_model.model_id + catalog_model.name + catalog_model.family when catalog_model.family != null + catalog_model.supports_attachment + catalog_model.supports_reasoning + catalog_model.supports_tool_calls + catalog_model.supports_structured_output + catalog_model.supports_temperature + catalog_model.knowledge when catalog_model.knowledge != null + catalog_model.release_date when catalog_model.release_date != null + catalog_model.last_updated_date when catalog_model.last_updated_date != null + catalog_model.open_weights + catalog_model.input_price when catalog_model.input_price != null + catalog_model.output_price when catalog_model.output_price != null + catalog_model.cache_read_price when catalog_model.cache_read_price != null + catalog_model.cache_write_price when catalog_model.cache_write_price != null + catalog_model.context_window + catalog_model.max_input_tokens + catalog_model.max_output_tokens + catalog_model.interleaved when catalog_model.interleaved != null + catalog_model.status when catalog_model.status != null + catalog_model.input_modalities + catalog_model.output_modalities + catalog_model.updated_at + } + entity SecureKeyStore { -- Encrypts API keys using the host operating system's secure storage. -- Stored in application settings in encrypted form. @@ -62,8 +148,12 @@ entity ChatMessage { conversation: ChatConversation role: system | user | assistant | tool content: String + tool_call_id: String? + tool_calls: String? token_usage_input: Integer? token_usage_output: Integer? + cache_read_tokens: Integer? + cache_write_tokens: Integer? created_at: Timestamp } @@ -74,11 +164,24 @@ surface ChatMessageSurface { message.conversation message.role message.content + message.tool_call_id when message.tool_call_id != null + message.tool_calls when message.tool_calls != null message.token_usage_input when message.token_usage_input != null message.token_usage_output when message.token_usage_output != null + message.cache_read_tokens when message.cache_read_tokens != null + message.cache_write_tokens when message.cache_write_tokens != null message.created_at } +surface AiConfigurationSurface { + facing _: AiOperator + + provides: + SetAiEndpointRequested(kind, url, api_key, model) + RemoveAiEndpointRequested(kind) + RefreshModelCatalogRequested(source) +} + surface OneShotAiSurface { facing _: AiOperator @@ -97,7 +200,31 @@ surface AiChatSurface { provides: StartChatRequested(model) SendChatMessageRequested(conversation, content) - RefreshModelCatalogRequested(endpoint) + CancelChatRequested(conversation) +} + +config { + model_catalog_ttl: Duration = 5.minutes + chat_max_tool_rounds: Integer = 10 + default_max_output_tokens: Integer = 16384 +} + +rule SetAiEndpoint { + when: SetAiEndpointRequested(kind, url, api_key, model) + ensures: + let endpoint = AiEndpoint.created( + kind: kind, + url: url, + api_key: api_key, + model: model + ) + endpoint.kind = kind +} + +rule RemoveAiEndpoint { + when: RemoveAiEndpointRequested(kind) + for endpoint in AiEndpoints where endpoint.kind = kind: + ensures: not exists endpoint } -- One-shot AI tasks (core scope, no streaming) @@ -173,17 +300,24 @@ rule SendChatMessage { ensures: conversation.updated_at = now ensures: AiStreamingResponse(conversation) -- Streaming response with bounded tool-call loop. - -- Blog data tools for post/media querying during chat. - -- Token usage tracking (input, output, cache read/write). + -- Blog data tools for post/media querying and mutation during chat. + -- Render tools may emit structured chart/table/form payloads. + -- Token usage tracking includes input, output, cache read, cache write. +} + +rule CancelChat { + when: CancelChatRequested(conversation) + ensures: AiStreamingResponseCancelled(conversation) } -- Model catalog rule RefreshModelCatalog { - when: RefreshModelCatalogRequested(endpoint) - -- Queries the endpoint's model list API - -- 5-minute cache TTL - ensures: ModelCatalogUpdated(endpoint) + when: RefreshModelCatalogRequested(source) + -- Refreshes advisory provider/model metadata used for capability checks, + -- default token budgeting, and model selection UX. + -- Uses conditional GET with ETag where supported. + ensures: ModelCatalogUpdated() } invariant AirplaneModeGating { @@ -196,6 +330,14 @@ invariant AirplaneModeGating { -- show toast "AI unavailable — configure {online|airplane} endpoint in Settings" } +invariant AirplaneModeModelSwap { + -- In airplane mode, cloud models are never contacted. + -- Chat uses the configured offline chat model when needed. + -- Image analysis uses the configured offline vision-capable model when needed. + -- If no suitable offline model is configured, the operation fails with + -- actionable guidance instead of silently falling back to the online endpoint. +} + invariant TwoEndpointModel { -- Two configurable OpenAI-compatible endpoints: -- online: for cloud providers (requires API key) @@ -204,6 +346,66 @@ invariant TwoEndpointModel { -- Endpoint selection is configurable rather than tied to hard-coded providers. } +invariant AdvisoryModelCatalog { + -- Model metadata is stored separately from runtime endpoint configuration. + -- It supplies capability hints such as context window, tool-call support, + -- structured output support, vision/input modalities, and pricing metadata. + -- The catalog remains usable offline after the last successful refresh. +} + +invariant ConditionalCatalogRefresh { + -- Model catalog refresh uses conditional HTTP requests when possible. + -- The latest ETag and fetch timestamp are persisted in AiCatalogMeta. + -- A 304 response updates freshness metadata without rewriting model rows. + exists meta in AiCatalogMeta where meta.key = "etag" or meta.key = "last_fetched_at" +} + +invariant ProviderDetection { + -- Runtime provider selection may be inferred from model identifiers, + -- local-endpoint registration, or explicit endpoint configuration. + -- The system does not rely on a single hard-coded provider list for routing. +} + +invariant VisionCapabilityGate { + -- AnalyzeImage only runs against models that accept image input. + -- Local/offline models must advertise or be configured with image capability + -- before the runtime sends multimodal requests to them. +} + +invariant ChatContextTruncation { + -- Chat requests are trimmed to fit within the selected model's context window. + -- Oldest user/assistant pairs are dropped first. + -- The system prompt, tool schema budget, and output-token reserve are preserved. +} + +invariant BoundedToolLoop { + -- Chat tool execution is bounded by config.chat_max_tool_rounds. + -- Tool-capable models may call blog-domain tools and render tools. + -- Non-tool-capable models skip tool exposure entirely. +} + +invariant TokenUsageAccounting { + -- Chat turn accounting tracks input, output, cache-read, and cache-write tokens. + -- Usage is reported per turn and accumulated per conversation. + -- Cache token accounting is surfaced when the underlying provider reports it. +} + +invariant ChatCancellation { + -- Each in-flight chat turn can be aborted independently. + -- Cancellation stops streaming and tool execution for that request only. +} + +invariant StructuredRenderTools { + -- Chat may emit structured render payloads for charts, tables, and forms. + -- These payloads are data contracts, not arbitrary HTML strings. +} + +invariant BlogStatsPromptAugmentation { + -- The base system prompt may be augmented with current blog statistics + -- such as post counts, media counts, tag/category totals, and date ranges + -- so long as the augmentation reflects current project state. +} + invariant AiSpecPartitioning { -- This file covers two distinct but related AI contracts: -- 1. Core one-shot operations (taxonomy, vision, translation, language detection) diff --git a/test/bds/ai_test.exs b/test/bds/ai_test.exs new file mode 100644 index 0000000..89029ca --- /dev/null +++ b/test/bds/ai_test.exs @@ -0,0 +1,459 @@ +defmodule BDS.AITest do + use ExUnit.Case, async: false + + import Ecto.Query + + alias BDS.Media.Media + alias BDS.Persistence + alias BDS.Posts.Post + alias BDS.Projects + alias BDS.Repo + alias BDS.Settings.Setting + + defmodule FakeSecretBackend do + def encrypt(value), do: {:ok, "enc:" <> value} + + def decrypt("enc:" <> value), do: {:ok, value} + def decrypt(_other), do: {:error, :invalid_ciphertext} + end + + defmodule FakeHttpClient do + def get("https://models.dev/api.json", headers) do + send(self(), {:http_headers, headers}) + + if Map.has_key?(headers, "if-none-match") do + {:ok, %{status: 304, headers: %{"etag" => headers["if-none-match"]}, body: ""}} + else + {:ok, + %{ + status: 200, + headers: %{"etag" => "W/\"catalog-v1\""}, + body: Jason.encode!(sample_catalog()) + }} + end + end + + def sample_catalog do + %{ + "openai" => %{ + "name" => "OpenAI", + "env" => ["OPENAI_API_KEY"], + "api" => "https://api.openai.com/v1", + "doc" => "https://platform.openai.com/docs", + "models" => %{ + "gpt-4o-mini" => %{ + "name" => "GPT-4o mini", + "family" => "gpt-4o", + "attachment" => false, + "reasoning" => false, + "tool_call" => true, + "structured_output" => true, + "temperature" => true, + "open_weights" => false, + "cost" => %{"input" => 15, "output" => 60, "cache_read" => 5, "cache_write" => 10}, + "limit" => %{"context" => 128_000, "input" => 128_000, "output" => 16_384}, + "input_modalities" => ["text"], + "output_modalities" => ["text"] + }, + "gpt-4o" => %{ + "name" => "GPT-4o", + "family" => "gpt-4o", + "attachment" => true, + "reasoning" => false, + "tool_call" => true, + "structured_output" => true, + "temperature" => true, + "open_weights" => false, + "cost" => %{"input" => 250, "output" => 1000}, + "limit" => %{"context" => 128_000, "input" => 128_000, "output" => 16_384}, + "input_modalities" => ["text", "image"], + "output_modalities" => ["text"] + } + } + } + } + end + end + + defmodule FakeRuntime do + def generate(endpoint, request, opts) do + test_pid = Keyword.fetch!(opts, :test_pid) + send(test_pid, {:runtime_request, endpoint, request}) + + case request.operation do + :detect_language -> + {:ok, %{json: %{"language_code" => "fr"}, usage: usage(11, 3, 0, 0)}} + + :translate_post -> + {:ok, + %{ + json: %{ + "title" => "Hallo Welt", + "excerpt" => "Kurze Zusammenfassung", + "content" => "# Hallo Welt\n\nUbersetzter Inhalt" + }, + usage: usage(22, 14, 0, 0) + }} + + :analyze_image -> + {:ok, + %{ + json: %{ + "title" => "Sunset", + "alt" => "Orange sunset over calm water", + "caption" => "Evening light over the bay" + }, + usage: usage(13, 9, 0, 0) + }} + + :chat -> + if Enum.any?(request.messages, &(&1["role"] == "tool")) do + {:ok, + %{ + content: "You currently have 1 post and 1 media item.", + usage: usage(64, 21, 3, 1) + }} + else + {:ok, + %{ + tool_calls: [ + %{ + id: "call-blog-stats", + name: "blog_stats", + arguments: %{} + } + ], + usage: usage(31, 8, 0, 0) + }} + end + end + end + + defp usage(input_tokens, output_tokens, cache_read_tokens, cache_write_tokens) do + %{ + input_tokens: input_tokens, + output_tokens: output_tokens, + cache_read_tokens: cache_read_tokens, + cache_write_tokens: cache_write_tokens + } + end + end + + defmodule BlockingRuntime do + def generate(endpoint, request, opts) do + test_pid = Keyword.fetch!(opts, :test_pid) + send(test_pid, {:blocking_runtime_started, endpoint, request}) + Process.sleep(5_000) + {:ok, %{content: "too late", usage: %{input_tokens: 1, output_tokens: 1}}} + end + end + + setup do + :ok = Ecto.Adapters.SQL.Sandbox.checkout(BDS.Repo) + :ok + end + + test "put_endpoint, get_endpoint, and delete_endpoint manage encrypted endpoint settings" do + assert {:ok, endpoint} = + BDS.AI.put_endpoint(:online, %{ + url: "https://api.example.test/v1", + api_key: "top-secret", + model: "gpt-4o-mini" + }, secret_backend: FakeSecretBackend) + + assert endpoint.kind == :online + assert endpoint.url == "https://api.example.test/v1" + assert endpoint.api_key == "top-secret" + assert endpoint.model == "gpt-4o-mini" + + assert %Setting{value: "https://api.example.test/v1"} = Repo.get(Setting, "ai.online.url") + assert %Setting{value: "gpt-4o-mini"} = Repo.get(Setting, "ai.online.model") + assert %Setting{value: encrypted_value} = Repo.get(Setting, "__encrypted_ai.online.api_key") + refute encrypted_value == "top-secret" + + assert {:ok, fetched} = BDS.AI.get_endpoint(:online, secret_backend: FakeSecretBackend) + assert fetched.api_key == "top-secret" + + assert :ok = BDS.AI.delete_endpoint(:online) + assert {:ok, nil} = BDS.AI.get_endpoint(:online, secret_backend: FakeSecretBackend) + assert Repo.get(Setting, "ai.online.url") == nil + assert Repo.get(Setting, "__encrypted_ai.online.api_key") == nil + end + + test "refresh_model_catalog stores providers, models, modalities, and etag metadata" do + assert {:ok, result} = + BDS.AI.refresh_model_catalog(http_client: FakeHttpClient) + + assert result.success == true + assert result.models_updated == 2 + assert_received {:http_headers, %{"accept" => "application/json"}} + + providers = BDS.AI.list_catalog_providers() + assert [%{id: "openai", name: "OpenAI"}] = providers + + assert {:ok, model} = BDS.AI.get_catalog_model("gpt-4o") + assert model.provider == "openai" + assert model.supports_tool_calls == true + assert model.context_window == 128_000 + assert "image" in model.input_modalities + + modalities = + Repo.all( + from modality in "ai_model_modalities", + where: modality.provider == ^"openai" and modality.model_id == ^"gpt-4o", + select: {modality.direction, modality.modality} + ) + + assert {"input", "image"} in modalities + + assert {:ok, "W/\"catalog-v1\""} = BDS.AI.catalog_meta("etag") + assert {:ok, last_fetched_at} = BDS.AI.catalog_meta("last_fetched_at") + assert is_binary(last_fetched_at) + end + + test "refresh_model_catalog uses conditional fetch metadata on subsequent refreshes" do + assert {:ok, _first} = BDS.AI.refresh_model_catalog(http_client: FakeHttpClient) + + http_client = fn url, headers -> + send(self(), {:conditional_headers, headers}) + FakeHttpClient.get(url, Map.put(headers, "if-none-match", "W/\"catalog-v1\"")) + end + + assert {:ok, result} = BDS.AI.refresh_model_catalog(http_client: http_client) + assert result.not_modified == true + assert_received {:conditional_headers, %{"accept" => "application/json", "if-none-match" => "W/\"catalog-v1\""}} + end + + test "airplane mode routes title tasks to airplane endpoint and offline title model" do + assert {:ok, _endpoint} = + BDS.AI.put_endpoint(:online, %{ + url: "https://api.example.test/v1", + api_key: "online-secret", + model: "gpt-4o-mini" + }, secret_backend: FakeSecretBackend) + + assert {:ok, _endpoint} = + BDS.AI.put_endpoint(:airplane, %{ + url: "http://localhost:11434/v1", + api_key: nil, + model: "llama-default" + }, secret_backend: FakeSecretBackend) + + assert :ok = BDS.AI.set_airplane_mode(true) + assert :ok = BDS.AI.put_model_preference(:airplane_title, "llama3.1") + + assert {:ok, result} = + BDS.AI.detect_language("Bonjour tout le monde", + runtime: FakeRuntime, + test_pid: self(), + secret_backend: FakeSecretBackend + ) + + assert result.language_code == "fr" + + assert_received {:runtime_request, endpoint, request} + assert endpoint.kind == :airplane + assert endpoint.url == "http://localhost:11434/v1" + assert request.operation == :detect_language + assert request.model == "llama3.1" + end + + test "translate_post uses the online title model when airplane mode is disabled" do + assert {:ok, _endpoint} = + BDS.AI.put_endpoint(:online, %{ + url: "https://api.example.test/v1", + api_key: "online-secret", + model: "gpt-4o-mini" + }, secret_backend: FakeSecretBackend) + + assert :ok = BDS.AI.set_airplane_mode(false) + assert :ok = BDS.AI.put_model_preference(:title, "gpt-4.1-mini") + + assert {:ok, translation} = + BDS.AI.translate_post( + %{title: "Hello World", excerpt: "Short summary", content: "# Hello\n\nSource body"}, + "de", + runtime: FakeRuntime, + test_pid: self(), + secret_backend: FakeSecretBackend + ) + + assert translation.title == "Hallo Welt" + assert translation.excerpt == "Kurze Zusammenfassung" + + assert_received {:runtime_request, endpoint, request} + assert endpoint.kind == :online + assert request.operation == :translate_post + assert request.model == "gpt-4.1-mini" + end + + test "analyze_image requires a vision-capable airplane model before sending image input" do + assert {:ok, _endpoint} = + BDS.AI.put_endpoint(:airplane, %{ + url: "http://localhost:11434/v1", + api_key: nil, + model: "llama-default" + }, secret_backend: FakeSecretBackend) + + assert :ok = BDS.AI.set_airplane_mode(true) + assert :ok = BDS.AI.put_model_preference(:airplane_image_analysis, "llama3.2") + + assert {:error, %{kind: :model_capability_missing}} = + BDS.AI.analyze_image(%{ + mime_type: "image/png", + title: "Source", + alt: nil, + caption: nil, + image_url: "file:///tmp/test.png" + }, runtime: FakeRuntime, test_pid: self(), secret_backend: FakeSecretBackend) + + assert :ok = + BDS.AI.put_model_capabilities("llama3.2", %{ + supports_attachment: true, + supports_tool_calls: false + }) + + assert {:ok, analysis} = + BDS.AI.analyze_image(%{ + mime_type: "image/png", + title: "Source", + alt: nil, + caption: nil, + image_url: "file:///tmp/test.png" + }, runtime: FakeRuntime, test_pid: self(), secret_backend: FakeSecretBackend) + + assert analysis.alt == "Orange sunset over calm water" + + assert_received {:runtime_request, endpoint, request} + assert endpoint.kind == :airplane + assert request.operation == :analyze_image + assert request.model == "llama3.2" + end + + test "chat persists user, tool, and assistant messages with usage and blog stats prompt augmentation" do + {:ok, project} = create_project_fixture("AI Chat") + :ok = seed_project_content(project.id) + + assert {:ok, _endpoint} = + BDS.AI.put_endpoint(:online, %{ + url: "https://api.example.test/v1", + api_key: "online-secret", + model: "gpt-4o-mini" + }, secret_backend: FakeSecretBackend) + + assert :ok = BDS.AI.set_airplane_mode(false) + assert {:ok, conversation} = BDS.AI.start_chat(%{model: "gpt-4o-mini"}) + + assert {:ok, reply} = + BDS.AI.send_chat_message(conversation.id, "How many items are in the blog?", + runtime: FakeRuntime, + test_pid: self(), + project_id: project.id, + secret_backend: FakeSecretBackend + ) + + assert reply.assistant_message.content == "You currently have 1 post and 1 media item." + + messages = BDS.AI.list_chat_messages(conversation.id) + assert Enum.map(messages, & &1.role) == [:user, :assistant, :tool, :assistant] + + assistant_tool_call = Enum.at(messages, 1) + tool_message = Enum.at(messages, 2) + assistant_message = Enum.at(messages, 3) + + assert [%{"id" => "call-blog-stats", "name" => "blog_stats"}] = assistant_tool_call.tool_calls + assert tool_message.tool_call_id == "call-blog-stats" + assert tool_message.content =~ "post_count" + assert assistant_message.token_usage_input == 64 + assert assistant_message.token_usage_output == 21 + assert assistant_message.cache_read_tokens == 3 + assert assistant_message.cache_write_tokens == 1 + + assert_received {:runtime_request, _endpoint, first_request} + assert_received {:runtime_request, _endpoint, second_request} + + assert Enum.any?(first_request.messages, fn message -> + message["role"] == "system" and String.contains?(message["content"], "Posts: 1") and + String.contains?(message["content"], "Media: 1") + end) + + assert Enum.any?(second_request.messages, fn message -> message["role"] == "tool" end) + end + + test "cancel_chat aborts an in-flight chat turn" do + assert {:ok, _endpoint} = + BDS.AI.put_endpoint(:online, %{ + url: "https://api.example.test/v1", + api_key: "online-secret", + model: "gpt-4o-mini" + }, secret_backend: FakeSecretBackend) + + assert {:ok, conversation} = BDS.AI.start_chat(%{model: "gpt-4o-mini"}) + + parent = self() + + task = + Task.async(fn -> + BDS.AI.send_chat_message(conversation.id, "Please wait", + runtime: BlockingRuntime, + test_pid: parent, + secret_backend: FakeSecretBackend + ) + end) + + assert_receive {:blocking_runtime_started, _endpoint, request}, 500 + assert request.operation == :chat + + assert :ok = BDS.AI.cancel_chat(conversation.id) + assert {:error, :cancelled} = Task.await(task) + + messages = BDS.AI.list_chat_messages(conversation.id) + assert Enum.map(messages, & &1.role) == [:user] + end + + defp create_project_fixture(name) do + temp_dir = Path.join(System.tmp_dir!(), "bds-ai-#{System.unique_integer([:positive])}") + on_exit(fn -> File.rm_rf(temp_dir) end) + + with {:ok, project} <- Projects.create_project(%{name: name, data_path: temp_dir}), + {:ok, _active} <- Projects.set_active_project(project.id) do + {:ok, project} + end + end + + defp seed_project_content(project_id) do + now = Persistence.now_ms() + + Repo.insert!( + Post.changeset(%Post{}, %{ + id: Ecto.UUID.generate(), + project_id: project_id, + title: "AI Post", + slug: "ai-post", + excerpt: "Summary", + content: "Body", + status: :draft, + created_at: now, + updated_at: now, + do_not_translate: false + }) + ) + + Repo.insert!( + Media.changeset(%Media{}, %{ + id: Ecto.UUID.generate(), + project_id: project_id, + filename: "image.png", + original_name: "image.png", + mime_type: "image/png", + size: 128, + file_path: "media/image.png", + sidecar_path: "media/image.png.meta", + created_at: now, + updated_at: now + }) + ) + + :ok + end +end diff --git a/test/bds/repo/schema_migration_test.exs b/test/bds/repo/schema_migration_test.exs index 1407506..e36ac0c 100644 --- a/test/bds/repo/schema_migration_test.exs +++ b/test/bds/repo/schema_migration_test.exs @@ -148,6 +148,10 @@ defmodule BDS.Repo.SchemaMigrationTest do "content", "tool_call_id", "tool_calls", + "token_usage_input", + "token_usage_output", + "cache_read_tokens", + "cache_write_tokens", "created_at" ], "ai_providers" => ["id", "name", "env", "package_ref", "api", "doc", "updated_at"],