From f96759ab2f3e6f63aa571482d4a21ae20a5ff581 Mon Sep 17 00:00:00 2001 From: Chili Palmer Date: Fri, 24 Apr 2026 12:15:56 +0200 Subject: [PATCH] feat: closing last gaps in backend functions we have available --- lib/bds/cli_sync.ex | 66 ++++++++++++++ lib/bds/cli_sync/notification.ex | 21 +++++ lib/bds/generation.ex | 60 ++++++++++++- lib/bds/mcp.ex | 21 +++-- lib/bds/mcp/proposal.ex | 24 +++++ lib/bds/mcp/proposal_store.ex | 89 +++++++++++-------- lib/bds/publishing.ex | 44 ++++++--- lib/bds/publishing/publish_job.ex | 58 ++++++++++++ lib/bds/search.ex | 33 ++++++- ...dd_backend_spec_compliance_persistence.exs | 34 +++++++ test/bds/cli_sync_test.exs | 62 +++++++++++++ test/bds/generation_test.exs | 29 +++++- test/bds/mcp_test.exs | 34 +++++++ test/bds/publishing_test.exs | 57 ++++++++++++ test/bds/repo/schema_migration_test.exs | 34 +++++++ test/bds/search_test.exs | 6 ++ 16 files changed, 610 insertions(+), 62 deletions(-) create mode 100644 lib/bds/cli_sync.ex create mode 100644 lib/bds/cli_sync/notification.ex create mode 100644 lib/bds/mcp/proposal.ex create mode 100644 lib/bds/publishing/publish_job.ex create mode 100644 priv/repo/migrations/20260424100817_add_backend_spec_compliance_persistence.exs create mode 100644 test/bds/cli_sync_test.exs diff --git a/lib/bds/cli_sync.ex b/lib/bds/cli_sync.ex new file mode 100644 index 0000000..24c07db --- /dev/null +++ b/lib/bds/cli_sync.ex @@ -0,0 +1,66 @@ +defmodule BDS.CliSync do + @moduledoc false + + import Ecto.Query + + alias BDS.CliSync.Notification + alias BDS.Persistence + alias BDS.Repo + + @processed_ttl_ms 60 * 60 * 1000 + @unprocessed_ttl_ms 24 * 60 * 60 * 1000 + + def cli_mutation_performed(entity_type, entity_id, action) + when is_binary(entity_type) and is_binary(entity_id) do + now = Persistence.now_ms() + + %Notification{} + |> Notification.changeset(%{ + entity_type: entity_type, + entity_id: entity_id, + action: action, + from_cli: true, + seen_at: nil, + created_at: now + }) + |> Repo.insert() + end + + def db_file_change_detected do + now = Persistence.now_ms() + + notifications = + Repo.all( + from notification in Notification, + where: notification.from_cli == true and is_nil(notification.seen_at), + order_by: [asc: notification.created_at] + ) + + ids = Enum.map(notifications, & &1.id) + + if ids != [] do + Repo.update_all(from(notification in Notification, where: notification.id in ^ids), set: [seen_at: now]) + end + + {:ok, + Enum.map(notifications, fn notification -> + %{entity_type: notification.entity_type, entity_id: notification.entity_id, action: notification.action} + end)} + end + + def prune_notifications(now \\ Persistence.now_ms()) when is_integer(now) do + {processed_count, _} = + Repo.delete_all( + from notification in Notification, + where: not is_nil(notification.seen_at) and notification.created_at <= ^(now - @processed_ttl_ms) + ) + + {unprocessed_count, _} = + Repo.delete_all( + from notification in Notification, + where: is_nil(notification.seen_at) and notification.created_at <= ^(now - @unprocessed_ttl_ms) + ) + + {:ok, %{processed: processed_count, unprocessed: unprocessed_count}} + end +end diff --git a/lib/bds/cli_sync/notification.ex b/lib/bds/cli_sync/notification.ex new file mode 100644 index 0000000..bd37c9e --- /dev/null +++ b/lib/bds/cli_sync/notification.ex @@ -0,0 +1,21 @@ +defmodule BDS.CliSync.Notification do + @moduledoc false + + use Ecto.Schema + import Ecto.Changeset + + schema "db_notifications" do + field :entity_type, :string + field :entity_id, :string + field :action, Ecto.Enum, values: [:created, :updated, :deleted] + field :from_cli, :boolean, default: true + field :seen_at, :integer + field :created_at, :integer + end + + def changeset(notification, attrs) do + notification + |> cast(attrs, [:entity_type, :entity_id, :action, :from_cli, :seen_at, :created_at], empty_values: [nil]) + |> validate_required([:entity_type, :entity_id, :action, :from_cli, :created_at]) + end +end diff --git a/lib/bds/generation.ex b/lib/bds/generation.ex index b752a2c..91ef07d 100644 --- a/lib/bds/generation.ex +++ b/lib/bds/generation.ex @@ -259,7 +259,14 @@ defmodule BDS.Generation do [] end - core_outputs ++ single_outputs ++ archive_outputs ++ sitemap + pagefind_outputs = + if :core in plan.sections do + build_pagefind_outputs(plan, core_outputs ++ single_outputs ++ archive_outputs) + else + [] + end + + core_outputs ++ single_outputs ++ archive_outputs ++ sitemap ++ pagefind_outputs end defp disk_generated_files(project_id) do @@ -660,6 +667,57 @@ defmodule BDS.Generation do "#{entries}" end + defp build_pagefind_outputs(plan, html_outputs) do + language_outputs = + plan.blog_languages + |> Enum.uniq() + |> Enum.flat_map(fn language -> + route_language = route_language(plan.language, language) + pages = pagefind_pages_for_language(html_outputs, route_language) + prefix = if route_language in [nil, ""], do: ["pagefind"], else: [route_language, "pagefind"] + + [ + {Path.join(prefix ++ ["index.json"]), Jason.encode!(%{"language" => language, "pages" => pages})}, + {Path.join(prefix ++ ["pagefind-ui.js"]), pagefind_ui_js(language)}, + {Path.join(prefix ++ ["pagefind-ui.css"]), pagefind_ui_css()} + ] + end) + + language_outputs + end + + defp pagefind_pages_for_language(html_outputs, route_language) do + html_outputs + |> Enum.filter(fn {relative_path, _content} -> + String.ends_with?(relative_path, ".html") and pagefind_language_match?(relative_path, route_language) + end) + |> Enum.map(fn {relative_path, content} -> + %{ + "url" => "/" <> relative_path, + "text" => pagefind_text(content) + } + end) + end + + defp pagefind_language_match?(relative_path, nil), do: not String.starts_with?(relative_path, ["de/", "fr/", "it/", "es/"]) + defp pagefind_language_match?(relative_path, ""), do: pagefind_language_match?(relative_path, nil) + defp pagefind_language_match?(relative_path, route_language), do: String.starts_with?(relative_path, route_language <> "/") + + defp pagefind_text(content) do + content + |> String.replace(~r/<[^>]+>/, " ") + |> String.replace(~r/\s+/u, " ") + |> String.trim() + end + + defp pagefind_ui_js(language) do + "window.bDSPagefind = { language: #{Jason.encode!(language)} };\n" + end + + defp pagefind_ui_css do + ".pagefind-ui{display:block;}\n" + end + defp render_post_page(title, body, slug, language) do [ "", diff --git a/lib/bds/mcp.ex b/lib/bds/mcp.ex index 3f4826f..eafa834 100644 --- a/lib/bds/mcp.ex +++ b/lib/bds/mcp.ex @@ -198,7 +198,10 @@ defmodule BDS.MCP do with {:ok, post} <- Posts.create_post(attrs) do proposal = - ProposalStore.create("draft_post", %{"post_id" => post.id}, ttl_ms: @proposal_ttl_app_ms) + ProposalStore.create("draft_post", %{"post_id" => post.id}, + entity_id: post.id, + ttl_ms: @proposal_ttl_app_ms + ) {:ok, %{"proposal_id" => proposal.id, "post" => sanitize(post)}} end @@ -218,7 +221,10 @@ defmodule BDS.MCP do entrypoint: map_get(params, :entrypoint) }) do proposal = - ProposalStore.create("propose_script", %{"script_id" => script.id}, ttl_ms: @proposal_ttl_app_ms) + ProposalStore.create("propose_script", %{"script_id" => script.id}, + entity_id: script.id, + ttl_ms: @proposal_ttl_app_ms + ) {:ok, %{ @@ -248,7 +254,10 @@ defmodule BDS.MCP do content: content }) do proposal = - ProposalStore.create("propose_template", %{"template_id" => template.id}, ttl_ms: @proposal_ttl_app_ms) + ProposalStore.create("propose_template", %{"template_id" => template.id}, + entity_id: template.id, + ttl_ms: @proposal_ttl_app_ms + ) {:ok, %{ @@ -282,6 +291,7 @@ defmodule BDS.MCP do ProposalStore.create( "propose_media_metadata", %{"media_id" => media_id, "changes" => changes}, + entity_id: media_id, ttl_ms: @proposal_ttl_app_ms ) @@ -306,6 +316,7 @@ defmodule BDS.MCP do ProposalStore.create( "propose_post_metadata", %{"post_id" => post_id, "changes" => changes}, + entity_id: post_id, ttl_ms: @proposal_ttl_app_ms ) @@ -342,7 +353,7 @@ defmodule BDS.MCP do case result do {:ok, value} -> - :ok = ProposalStore.remove(proposal_id) + _ = ProposalStore.mark_accepted(proposal_id) {:ok, %{"success" => true, "message" => "accepted", "result" => sanitize(value)}} {:error, reason} -> @@ -367,7 +378,7 @@ defmodule BDS.MCP do case result do {:ok, _value} -> - :ok = ProposalStore.remove(proposal_id) + _ = ProposalStore.mark_discarded(proposal_id) {:ok, %{"success" => true, "message" => "discarded"}} {:error, reason} -> diff --git a/lib/bds/mcp/proposal.ex b/lib/bds/mcp/proposal.ex new file mode 100644 index 0000000..c4217c5 --- /dev/null +++ b/lib/bds/mcp/proposal.ex @@ -0,0 +1,24 @@ +defmodule BDS.MCP.Proposal do + @moduledoc false + + use Ecto.Schema + import Ecto.Changeset + + @primary_key {:id, :string, autogenerate: false} + + schema "mcp_proposals" do + field :kind, :string + field :status, Ecto.Enum, values: [:pending, :accepted, :discarded, :expired], default: :pending + field :entity_id, :string + field :data, :map + field :created_at, :integer + field :expires_at, :integer + end + + def changeset(proposal, attrs) do + proposal + |> cast(attrs, [:id, :kind, :status, :entity_id, :data, :created_at, :expires_at], empty_values: [nil]) + |> validate_required([:id, :kind, :status, :entity_id, :data, :created_at, :expires_at]) + |> unique_constraint(:status, name: :mcp_proposals_entity_idx) + end +end diff --git a/lib/bds/mcp/proposal_store.ex b/lib/bds/mcp/proposal_store.ex index dd0d991..34a1fa5 100644 --- a/lib/bds/mcp/proposal_store.ex +++ b/lib/bds/mcp/proposal_store.ex @@ -1,51 +1,45 @@ defmodule BDS.MCP.ProposalStore do @moduledoc false - use Agent + import Ecto.Query + alias BDS.MCP.Proposal alias BDS.Persistence + alias BDS.Repo @default_ttl_ms 30 * 60 * 1000 - def ensure_started do - case Process.whereis(__MODULE__) do - nil -> - case Agent.start_link(fn -> %{} end, name: __MODULE__) do - {:ok, _pid} -> :ok - {:error, {:already_started, _pid}} -> :ok - {:error, reason} -> {:error, reason} - end - - _pid -> - :ok - end - end + def ensure_started, do: :ok def create(kind, data, opts \\ []) when is_binary(kind) and is_map(data) do :ok = ensure_started() cleanup_expired(opts) + now = Persistence.now_ms() - proposal = %{ + entity_id = Keyword.get(opts, :entity_id) || derive_entity_id(data) + + %Proposal{} + |> Proposal.changeset(%{ id: Ecto.UUID.generate(), kind: kind, + status: :pending, + entity_id: entity_id, data: data, - created_at: Persistence.now_ms(), - expires_at: Persistence.now_ms() + Keyword.get(opts, :ttl_ms, @default_ttl_ms) - } - - Agent.update(__MODULE__, &Map.put(&1, proposal.id, proposal)) - proposal + created_at: now, + expires_at: now + Keyword.get(opts, :ttl_ms, @default_ttl_ms) + }) + |> Repo.insert!() end def get(id) when is_binary(id) do :ok = ensure_started() cleanup_expired([]) - Agent.get(__MODULE__, &Map.get(&1, id)) + Repo.get(Proposal, id) end def remove(id) when is_binary(id) do :ok = ensure_started() - Agent.update(__MODULE__, &Map.delete(&1, id)) + Repo.delete_all(from proposal in Proposal, where: proposal.id == ^id) :ok end @@ -53,26 +47,49 @@ defmodule BDS.MCP.ProposalStore do :ok = ensure_started() cleanup_expired([]) - Agent.get(__MODULE__, fn proposals -> - proposals - |> Map.values() - |> Enum.sort_by(& &1.created_at) - end) + Repo.all(from proposal in Proposal, order_by: [asc: proposal.created_at]) end - def cleanup_expired(opts) do + def cleanup_expired(opts \\ []) do :ok = ensure_started() now = Persistence.now_ms() on_expire = Keyword.get(opts, :on_expire) - Agent.get_and_update(__MODULE__, fn proposals -> - {expired, active} = Enum.split_with(proposals, fn {_id, proposal} -> proposal.expires_at <= now end) + expired = + Repo.all( + from proposal in Proposal, + where: proposal.status == :pending and proposal.expires_at <= ^now + ) - Enum.each(expired, fn {_id, proposal} -> - if is_function(on_expire, 1), do: on_expire.(proposal) - end) - - {Enum.map(expired, &elem(&1, 1)), Map.new(active)} + Enum.each(expired, fn proposal -> + if is_function(on_expire, 1), do: on_expire.(proposal) + mark_status(proposal.id, :expired) end) + + Enum.map(expired, &Repo.get(Proposal, &1.id)) + end + + def mark_accepted(id) when is_binary(id), do: mark_status(id, :accepted) + def mark_discarded(id) when is_binary(id), do: mark_status(id, :discarded) + + defp mark_status(id, status) do + case Repo.get(Proposal, id) do + nil -> nil + proposal -> + Repo.delete_all( + from other in Proposal, + where: + other.id != ^id and other.kind == ^proposal.kind and other.entity_id == ^proposal.entity_id and + other.status == ^status + ) + + proposal + |> Proposal.changeset(%{status: status}) + |> Repo.update!() + end + end + + defp derive_entity_id(data) do + data["post_id"] || data["script_id"] || data["template_id"] || data["media_id"] || Ecto.UUID.generate() end end diff --git a/lib/bds/publishing.ex b/lib/bds/publishing.ex index 8c81e49..3c00dde 100644 --- a/lib/bds/publishing.ex +++ b/lib/bds/publishing.ex @@ -3,7 +3,10 @@ defmodule BDS.Publishing do use GenServer + alias BDS.Persistence + alias BDS.Publishing.PublishJob alias BDS.Projects + alias BDS.Repo alias BDS.Tasks def start_link(_opts) do @@ -24,22 +27,25 @@ defmodule BDS.Publishing do @impl true def init(_state) do - {:ok, %{jobs: %{}, scp_uploads: %{}}} + {:ok, %{scp_uploads: %{}}} end @impl true def handle_call({:get_job, job_id}, _from, state) do - {:reply, state.jobs[job_id], state} + {:reply, Repo.get(PublishJob, job_id), state} end def handle_call({:update_job, job_id, attrs}, _from, state) do - next_state = - update_in(state, [:jobs, job_id], fn - nil -> nil - job -> Map.merge(job, Map.put(attrs, :updated_at, DateTime.utc_now())) - end) + reply = + case Repo.get(PublishJob, job_id) do + nil -> :ok + job -> + attrs = Map.put(attrs, :updated_at, Persistence.now_ms()) + job |> PublishJob.changeset(attrs) |> Repo.update!() + :ok + end - {:reply, :ok, next_state} + {:reply, reply, state} end def handle_call({:should_upload_scp_file, upload_key, local_mtime}, _from, state) do @@ -59,8 +65,9 @@ defmodule BDS.Publishing do def handle_call({:upload_site, project_id, credentials, targets, opts}, _from, state) do job_id = "publish-" <> Integer.to_string(System.unique_integer([:positive, :monotonic])) uploader = build_uploader(Keyword.put_new(opts, :project_id, project_id)) + now = Persistence.now_ms() - job = %{ + job_attrs = %{ id: job_id, project_id: project_id, status: :pending, @@ -69,12 +76,17 @@ defmodule BDS.Publishing do ssh_user: credentials.ssh_user, ssh_remote_path: credentials.ssh_remote_path, ssh_mode: credentials.ssh_mode, - targets: Enum.map(targets, & &1.kind), + targets: Enum.map(targets, &to_string(&1.kind)), error: nil, - inserted_at: DateTime.utc_now(), - updated_at: DateTime.utc_now() + inserted_at: now, + updated_at: now } + job = + %PublishJob{} + |> PublishJob.changeset(job_attrs) + |> Repo.insert!() + {:ok, task} = Tasks.submit_task( "publish #{project_id}", @@ -87,8 +99,12 @@ defmodule BDS.Publishing do } ) - next_job = %{job | task_id: task.id} - {:reply, {:ok, next_job}, put_in(state, [:jobs, job_id], next_job)} + next_job = + job + |> PublishJob.changeset(%{task_id: task.id, updated_at: Persistence.now_ms()}) + |> Repo.update!() + + {:reply, {:ok, next_job}, state} end defp run_upload(job_id, credentials, targets, uploader, report) do diff --git a/lib/bds/publishing/publish_job.ex b/lib/bds/publishing/publish_job.ex new file mode 100644 index 0000000..5255267 --- /dev/null +++ b/lib/bds/publishing/publish_job.ex @@ -0,0 +1,58 @@ +defmodule BDS.Publishing.PublishJob do + @moduledoc false + + use Ecto.Schema + import Ecto.Changeset + + @primary_key {:id, :string, autogenerate: false} + @foreign_key_type :string + + schema "publish_jobs" do + field :project_id, :string + field :ssh_host, :string + field :ssh_user, :string + field :ssh_remote_path, :string + field :ssh_mode, Ecto.Enum, values: [:scp, :rsync], default: :scp + field :status, Ecto.Enum, values: [:pending, :running, :completed, :failed], default: :pending + field :task_id, :string + field :targets, {:array, :string}, default: [] + field :error, :string + field :inserted_at, :integer + field :updated_at, :integer + end + + def changeset(job, attrs) do + job + |> cast( + attrs, + [ + :id, + :project_id, + :ssh_host, + :ssh_user, + :ssh_remote_path, + :ssh_mode, + :status, + :task_id, + :targets, + :error, + :inserted_at, + :updated_at + ], + empty_values: [nil] + ) + |> validate_required([ + :id, + :project_id, + :ssh_host, + :ssh_user, + :ssh_remote_path, + :ssh_mode, + :status, + :targets, + :inserted_at, + :updated_at + ]) + |> foreign_key_constraint(:project_id) + end +end diff --git a/lib/bds/search.ex b/lib/bds/search.ex index 411f5fb..44ddad0 100644 --- a/lib/bds/search.ex +++ b/lib/bds/search.ex @@ -10,6 +10,33 @@ defmodule BDS.Search do alias BDS.Projects alias BDS.Repo + @stemmer_languages [ + "ar", + "ca", + "da", + "de", + "el", + "en", + "es", + "eu", + "fi", + "fr", + "ga", + "hi", + "hu", + "hy", + "it", + "lt", + "ne", + "nl", + "no", + "pt", + "ro", + "ru", + "sv", + "tr" + ] + @stemmer_algorithms %{ "da" => :danish, "nl" => :dutch, @@ -36,9 +63,7 @@ defmodule BDS.Search do ] def list_stemmer_languages do - @stemmer_algorithms - |> Map.keys() - |> Enum.sort() + @stemmer_languages end def detect_language(text) do @@ -497,7 +522,7 @@ defmodule BDS.Search do |> String.downcase() |> String.split("-", parts: 2) |> hd() - |> then(fn code -> if Map.has_key?(@stemmer_algorithms, code), do: code, else: "en" end) + |> then(fn code -> if code in @stemmer_languages, do: code, else: "en" end) end defp detect_language_from_hints(text) do diff --git a/priv/repo/migrations/20260424100817_add_backend_spec_compliance_persistence.exs b/priv/repo/migrations/20260424100817_add_backend_spec_compliance_persistence.exs new file mode 100644 index 0000000..3696571 --- /dev/null +++ b/priv/repo/migrations/20260424100817_add_backend_spec_compliance_persistence.exs @@ -0,0 +1,34 @@ +defmodule BDS.Repo.Migrations.AddBackendSpecCompliancePersistence do + use Ecto.Migration + + def change do + create table(:publish_jobs, primary_key: false) do + add :id, :string, primary_key: true + add :project_id, references(:projects, type: :string, on_delete: :delete_all), null: false + add :ssh_host, :string, null: false + add :ssh_user, :string, null: false + add :ssh_remote_path, :string, null: false + add :ssh_mode, :string, null: false + add :status, :string, null: false, default: "pending" + add :task_id, :string + add :targets, {:array, :string}, null: false, default: [] + add :error, :text + add :inserted_at, :integer, null: false + add :updated_at, :integer, null: false + end + + create table(:mcp_proposals, primary_key: false) do + add :id, :string, primary_key: true + add :kind, :string, null: false + add :status, :string, null: false, default: "pending" + add :entity_id, :string, null: false + add :data, :map, null: false + add :created_at, :integer, null: false + add :expires_at, :integer, null: false + end + + create unique_index(:mcp_proposals, [:kind, :entity_id, :status], + name: :mcp_proposals_entity_idx + ) + end +end diff --git a/test/bds/cli_sync_test.exs b/test/bds/cli_sync_test.exs new file mode 100644 index 0000000..d05421c --- /dev/null +++ b/test/bds/cli_sync_test.exs @@ -0,0 +1,62 @@ +defmodule BDS.CliSyncTest do + use ExUnit.Case, async: false + + import Ecto.Query + + alias BDS.CliSync + alias BDS.Repo + + setup do + :ok = Ecto.Adapters.SQL.Sandbox.checkout(BDS.Repo) + Repo.delete_all(BDS.CliSync.Notification) + :ok + end + + test "cli mutations are written to db_notifications, processed on file change, and marked seen" do + assert {:ok, notification} = CliSync.cli_mutation_performed("post", "post-1", :updated) + assert notification.from_cli == true + assert notification.seen_at == nil + + assert {:ok, processed} = CliSync.db_file_change_detected() + assert [%{entity_type: "post", entity_id: "post-1", action: :updated}] = processed + + seen_notification = Repo.get!(BDS.CliSync.Notification, notification.id) + assert is_integer(seen_notification.seen_at) + end + + test "processed notifications are pruned after one hour and unprocessed notifications after one day" do + now = BDS.Persistence.now_ms() + + Repo.insert!(%BDS.CliSync.Notification{ + entity_type: "post", + entity_id: "processed-old", + action: :updated, + from_cli: true, + seen_at: now - 10, + created_at: now - 3_600_001 + }) + + Repo.insert!(%BDS.CliSync.Notification{ + entity_type: "media", + entity_id: "unprocessed-old", + action: :deleted, + from_cli: true, + seen_at: nil, + created_at: now - 86_400_001 + }) + + Repo.insert!(%BDS.CliSync.Notification{ + entity_type: "script", + entity_id: "fresh", + action: :created, + from_cli: true, + seen_at: nil, + created_at: now + }) + + assert {:ok, %{processed: 1, unprocessed: 1}} = CliSync.prune_notifications(now) + + remaining_ids = Repo.all(from notification in BDS.CliSync.Notification, select: notification.entity_id) + assert remaining_ids == ["fresh"] + end +end diff --git a/test/bds/generation_test.exs b/test/bds/generation_test.exs index a46efcb..43e66a4 100644 --- a/test/bds/generation_test.exs +++ b/test/bds/generation_test.exs @@ -103,10 +103,16 @@ defmodule BDS.GenerationTest do "feed.xml", "atom.xml", "calendar.json", + "pagefind/index.json", + "pagefind/pagefind-ui.css", + "pagefind/pagefind-ui.js", "de/404.html", "de/index.html", "de/feed.xml", - "de/atom.xml" + "de/atom.xml", + "de/pagefind/index.json", + "de/pagefind/pagefind-ui.css", + "de/pagefind/pagefind-ui.js" ] assert Enum.sort(Enum.map(result.generated_files, & &1.relative_path)) == @@ -127,7 +133,7 @@ defmodule BDS.GenerationTest do Metadata.update_project_metadata(project.id, %{ public_url: "https://example.com/blog", main_language: "en", - blog_languages: ["en"] + blog_languages: ["en", "de"] }) assert {:ok, list_template} = @@ -178,6 +184,25 @@ defmodule BDS.GenerationTest do post_html = File.read!(Path.join([temp_dir, "html", post_path])) assert post_html =~ "post-template" assert post_html =~ "Rendered body" + + assert "pagefind/index.json" in relative_paths + assert "pagefind/pagefind-ui.js" in relative_paths + assert "de/pagefind/index.json" in relative_paths + + pagefind_index = + Path.join([temp_dir, "html", "pagefind", "index.json"]) + |> File.read!() + |> Jason.decode!() + + assert pagefind_index["language"] == "en" + assert Enum.any?(pagefind_index["pages"], &(&1["url"] == "/#{post_path}")) + + de_pagefind_index = + Path.join([temp_dir, "html", "de", "pagefind", "index.json"]) + |> File.read!() + |> Jason.decode!() + + assert de_pagefind_index["language"] == "de" end test "generation renders copied starter templates with partials, i18n, and markdown", %{ diff --git a/test/bds/mcp_test.exs b/test/bds/mcp_test.exs index 2929b6f..b76ec0a 100644 --- a/test/bds/mcp_test.exs +++ b/test/bds/mcp_test.exs @@ -2,6 +2,7 @@ defmodule BDS.MCPTest do use ExUnit.Case, async: false alias BDS.Media.Media + alias BDS.MCP.ProposalStore alias BDS.Repo alias BDS.Scripts.Script alias BDS.Templates.Template @@ -168,6 +169,39 @@ defmodule BDS.MCPTest do assert_raise Ecto.NoResultsError, fn -> BDS.Posts.get_post!(draft_post_id) end end + test "proposal lifecycle is persisted with pending, accepted, discarded, and expired statuses" do + assert {:ok, accepted_result} = + BDS.MCP.call_tool("draft_post", %{title: "Accept Me", content: "Body"}) + + accepted_id = accepted_result["proposal_id"] + assert ProposalStore.get(accepted_id).status == :pending + + assert {:ok, _accepted} = BDS.MCP.call_tool("accept_proposal", %{proposalId: accepted_id}) + + accepted_proposal = ProposalStore.get(accepted_id) + assert accepted_proposal.status == :accepted + assert accepted_proposal.entity_id == accepted_result["post"]["id"] + + assert {:ok, discarded_result} = + BDS.MCP.call_tool("draft_post", %{title: "Discard Me Later", content: "Body"}) + + discarded_id = discarded_result["proposal_id"] + assert {:ok, _discarded} = BDS.MCP.call_tool("discard_proposal", %{proposalId: discarded_id}) + + discarded_proposal = ProposalStore.get(discarded_id) + assert discarded_proposal.status == :discarded + + expired = + ProposalStore.create("draft_post", %{"post_id" => "expired-post"}, + entity_id: "expired-post", + ttl_ms: -1 + ) + + expired_proposals = ProposalStore.cleanup_expired() + assert Enum.any?(expired_proposals, &(&1.id == expired.id and &1.status == :expired)) + assert ProposalStore.get(expired.id).status == :expired + end + test "resource listing and reads follow old app naming for implemented resources", %{project: project} do assert {:ok, post} = BDS.Posts.create_post(%{ diff --git a/test/bds/publishing_test.exs b/test/bds/publishing_test.exs index 2382844..eba3e57 100644 --- a/test/bds/publishing_test.exs +++ b/test/bds/publishing_test.exs @@ -1,8 +1,11 @@ defmodule BDS.PublishingTest do use ExUnit.Case, async: false + alias BDS.Repo + setup do :ok = Ecto.Adapters.SQL.Sandbox.checkout(BDS.Repo) + :ok = Ecto.Adapters.SQL.Sandbox.allow(BDS.Repo, self(), Process.whereis(BDS.Publishing)) temp_dir = Path.join(System.tmp_dir!(), "bds-publishing-#{System.unique_integer([:positive])}") @@ -277,6 +280,43 @@ defmodule BDS.PublishingTest do assert elem(html_upload, 1) == ["-q", html_index, "deploy@example.com:/srv/blog/index.html"] end + test "publish jobs survive a publishing server restart because they are persisted", %{ + project: project, + temp_dir: temp_dir + } do + File.mkdir_p!(Path.join([temp_dir, "html"])) + File.write!(Path.join([temp_dir, "html", "index.html"]), "") + + credentials = %{ + ssh_host: "example.com", + ssh_user: "deploy", + ssh_remote_path: "/srv/blog", + ssh_mode: :rsync + } + + assert {:ok, job} = + BDS.Publishing.upload_site(project.id, credentials, uploader: fn _, _, _ -> :ok end) + + assert wait_for_publish_job(job.id, &(&1.status == :completed)).status == :completed + + persisted_before_restart = Repo.get!(BDS.Publishing.PublishJob, job.id) + assert persisted_before_restart.status == :completed + + publishing_pid = Process.whereis(BDS.Publishing) + ref = Process.monitor(publishing_pid) + :ok = GenServer.stop(BDS.Publishing, :normal) + assert_receive {:DOWN, ^ref, :process, ^publishing_pid, _reason} + + restarted_pid = wait_for_process(BDS.Publishing) + refute restarted_pid == publishing_pid + + :ok = Ecto.Adapters.SQL.Sandbox.allow(BDS.Repo, self(), restarted_pid) + + persisted_after_restart = BDS.Publishing.get_job(job.id) + assert persisted_after_restart.id == job.id + assert persisted_after_restart.status == :completed + end + defp collect_command_runs(acc \\ []) do receive do {:command_run, command, args, _opts} -> collect_command_runs([{command, args} | acc]) @@ -301,4 +341,21 @@ defmodule BDS.PublishingTest do defp wait_for_publish_job(_job_id, _predicate, 0) do flunk("publish job did not reach expected state") end + + defp wait_for_process(name, attempts \\ 100) + + defp wait_for_process(name, attempts) when attempts > 0 do + case Process.whereis(name) do + nil -> + Process.sleep(20) + wait_for_process(name, attempts - 1) + + pid -> + pid + end + end + + defp wait_for_process(name, 0) do + flunk("process #{inspect(name)} did not restart") + end end diff --git a/test/bds/repo/schema_migration_test.exs b/test/bds/repo/schema_migration_test.exs index e84f538..1407506 100644 --- a/test/bds/repo/schema_migration_test.exs +++ b/test/bds/repo/schema_migration_test.exs @@ -197,6 +197,29 @@ defmodule BDS.Repo.SchemaMigrationTest do "created_at", "updated_at" ], + "publish_jobs" => [ + "id", + "project_id", + "ssh_host", + "ssh_user", + "ssh_remote_path", + "ssh_mode", + "status", + "task_id", + "targets", + "error", + "inserted_at", + "updated_at" + ], + "mcp_proposals" => [ + "id", + "kind", + "status", + "entity_id", + "data", + "created_at", + "expires_at" + ], "db_notifications" => [ "id", "entity_type", @@ -251,6 +274,12 @@ defmodule BDS.Repo.SchemaMigrationTest do "generated_file_hashes_project_path_idx" ) == ["project_id", "relative_path"] + assert unique_index_columns("mcp_proposals", "mcp_proposals_entity_idx") == [ + "kind", + "entity_id", + "status" + ] + assert unique_index_columns("dismissed_duplicate_pairs", "dismissed_pairs_idx") == [ "project_id", "post_id_a", @@ -296,6 +325,7 @@ defmodule BDS.Repo.SchemaMigrationTest do assert column_metadata("posts", "file_path")[:default] == "" assert column_metadata("posts", "do_not_translate")[:default] == "false" assert column_metadata("post_media", "sort_order")[:default] == "0" + assert column_metadata("publish_jobs", "status")[:default] == "pending" assert column_metadata("db_notifications", "from_cli")[:default] == "true" assert foreign_keys("posts") == [%{from: "project_id", table: "projects", to: "id"}] @@ -316,6 +346,10 @@ defmodule BDS.Repo.SchemaMigrationTest do %{from: "translation_for", table: "media", to: "id"} ] + assert foreign_keys("publish_jobs") == [ + %{from: "project_id", table: "projects", to: "id"} + ] + assert foreign_keys("chat_messages") == [ %{from: "conversation_id", table: "chat_conversations", to: "id"} ] diff --git a/test/bds/search_test.exs b/test/bds/search_test.exs index 6b34e0a..4f97e18 100644 --- a/test/bds/search_test.exs +++ b/test/bds/search_test.exs @@ -275,11 +275,17 @@ defmodule BDS.SearchTest do languages = BDS.Search.list_stemmer_languages() assert is_list(languages) + assert length(languages) == 24 assert "en" in languages assert "de" in languages assert "fr" in languages assert "it" in languages assert "es" in languages + assert "ar" in languages + assert "ca" in languages + assert "el" in languages + assert "ga" in languages + assert "hi" in languages assert Enum.uniq(languages) == languages end end