hugo/bDS2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: closing last gaps in backend functions we have available

Browse Source
This commit is contained in:
Chili Palmer
2026-04-24 12:15:56 +02:00
parent 13ace08a41
commit f96759ab2f
16 changed files with 610 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files

66
lib/bds/cli_sync.ex Normal file
View File

@@ -0,0 +1,66 @@
defmodule BDS.CliSync do
@moduledoc false
import Ecto.Query
alias BDS.CliSync.Notification
alias BDS.Persistence
alias BDS.Repo
@processed_ttl_ms 60 * 60 * 1000
@unprocessed_ttl_ms 24 * 60 * 60 * 1000
def cli_mutation_performed(entity_type, entity_id, action)
when is_binary(entity_type) and is_binary(entity_id) do
now = Persistence.now_ms()
%Notification{}
|> Notification.changeset(%{
entity_type: entity_type,
entity_id: entity_id,
action: action,
from_cli: true,
seen_at: nil,
created_at: now
})
|> Repo.insert()
end
def db_file_change_detected do
now = Persistence.now_ms()
notifications =
Repo.all(
from notification in Notification,
where: notification.from_cli == true and is_nil(notification.seen_at),
order_by: [asc: notification.created_at]
)
ids = Enum.map(notifications, & &1.id)
if ids != [] do
Repo.update_all(from(notification in Notification, where: notification.id in ^ids), set: [seen_at: now])
end
{:ok,
Enum.map(notifications, fn notification ->
%{entity_type: notification.entity_type, entity_id: notification.entity_id, action: notification.action}
end)}
end
def prune_notifications(now \\ Persistence.now_ms()) when is_integer(now) do
{processed_count, _} =
Repo.delete_all(
from notification in Notification,
where: not is_nil(notification.seen_at) and notification.created_at <= ^(now - @processed_ttl_ms)
)
{unprocessed_count, _} =
Repo.delete_all(
from notification in Notification,
where: is_nil(notification.seen_at) and notification.created_at <= ^(now - @unprocessed_ttl_ms)
)
{:ok, %{processed: processed_count, unprocessed: unprocessed_count}}
end
end

21
lib/bds/cli_sync/notification.ex Normal file
View File

@@ -0,0 +1,21 @@
defmodule BDS.CliSync.Notification do
@moduledoc false
use Ecto.Schema
import Ecto.Changeset
schema "db_notifications" do
field :entity_type, :string
field :entity_id, :string
field :action, Ecto.Enum, values: [:created, :updated, :deleted]
field :from_cli, :boolean, default: true
field :seen_at, :integer
field :created_at, :integer
end
def changeset(notification, attrs) do
notification
|> cast(attrs, [:entity_type, :entity_id, :action, :from_cli, :seen_at, :created_at], empty_values: [nil])
|> validate_required([:entity_type, :entity_id, :action, :from_cli, :created_at])
end
end

60
lib/bds/generation.ex
View File

@@ -259,7 +259,14 @@ defmodule BDS.Generation do
[]
end
core_outputs ++ single_outputs ++ archive_outputs ++ sitemap
pagefind_outputs =
if :core in plan.sections do
build_pagefind_outputs(plan, core_outputs ++ single_outputs ++ archive_outputs)
else
[]
end
core_outputs ++ single_outputs ++ archive_outputs ++ sitemap ++ pagefind_outputs
end
defp disk_generated_files(project_id) do
@@ -660,6 +667,57 @@ defmodule BDS.Generation do
"<urlset>#{entries}</urlset>"
end
defp build_pagefind_outputs(plan, html_outputs) do
language_outputs =
plan.blog_languages
|> Enum.uniq()
|> Enum.flat_map(fn language ->
route_language = route_language(plan.language, language)
pages = pagefind_pages_for_language(html_outputs, route_language)
prefix = if route_language in [nil, ""], do: ["pagefind"], else: [route_language, "pagefind"]
[
{Path.join(prefix ++ ["index.json"]), Jason.encode!(%{"language" => language, "pages" => pages})},
{Path.join(prefix ++ ["pagefind-ui.js"]), pagefind_ui_js(language)},
{Path.join(prefix ++ ["pagefind-ui.css"]), pagefind_ui_css()}
]
end)
language_outputs
end
defp pagefind_pages_for_language(html_outputs, route_language) do
html_outputs
|> Enum.filter(fn {relative_path, _content} ->
String.ends_with?(relative_path, ".html") and pagefind_language_match?(relative_path, route_language)
end)
|> Enum.map(fn {relative_path, content} ->
%{
"url" => "/" <> relative_path,
"text" => pagefind_text(content)
}
end)
end
defp pagefind_language_match?(relative_path, nil), do: not String.starts_with?(relative_path, ["de/", "fr/", "it/", "es/"])
defp pagefind_language_match?(relative_path, ""), do: pagefind_language_match?(relative_path, nil)
defp pagefind_language_match?(relative_path, route_language), do: String.starts_with?(relative_path, route_language <> "/")
defp pagefind_text(content) do
content
|> String.replace(~r/<[^>]+>/, " ")
|> String.replace(~r/\s+/u, " ")
|> String.trim()
end
defp pagefind_ui_js(language) do
"window.bDSPagefind = { language: #{Jason.encode!(language)} };\n"
end
defp pagefind_ui_css do
".pagefind-ui{display:block;}\n"
end
defp render_post_page(title, body, slug, language) do
[
"<html>",

21
lib/bds/mcp.ex
View File

@@ -198,7 +198,10 @@ defmodule BDS.MCP do
with {:ok, post} <- Posts.create_post(attrs) do
proposal =
ProposalStore.create("draft_post", %{"post_id" => post.id}, ttl_ms: @proposal_ttl_app_ms)
ProposalStore.create("draft_post", %{"post_id" => post.id},
entity_id: post.id,
ttl_ms: @proposal_ttl_app_ms
)
{:ok, %{"proposal_id" => proposal.id, "post" => sanitize(post)}}
end
@@ -218,7 +221,10 @@ defmodule BDS.MCP do
entrypoint: map_get(params, :entrypoint)
}) do
proposal =
ProposalStore.create("propose_script", %{"script_id" => script.id}, ttl_ms: @proposal_ttl_app_ms)
ProposalStore.create("propose_script", %{"script_id" => script.id},
entity_id: script.id,
ttl_ms: @proposal_ttl_app_ms
)
{:ok,
%{
@@ -248,7 +254,10 @@ defmodule BDS.MCP do
content: content
}) do
proposal =
ProposalStore.create("propose_template", %{"template_id" => template.id}, ttl_ms: @proposal_ttl_app_ms)
ProposalStore.create("propose_template", %{"template_id" => template.id},
entity_id: template.id,
ttl_ms: @proposal_ttl_app_ms
)
{:ok,
%{
@@ -282,6 +291,7 @@ defmodule BDS.MCP do
ProposalStore.create(
"propose_media_metadata",
%{"media_id" => media_id, "changes" => changes},
entity_id: media_id,
ttl_ms: @proposal_ttl_app_ms
)
@@ -306,6 +316,7 @@ defmodule BDS.MCP do
ProposalStore.create(
"propose_post_metadata",
%{"post_id" => post_id, "changes" => changes},
entity_id: post_id,
ttl_ms: @proposal_ttl_app_ms
)
@@ -342,7 +353,7 @@ defmodule BDS.MCP do
case result do
{:ok, value} ->
:ok = ProposalStore.remove(proposal_id)
_ = ProposalStore.mark_accepted(proposal_id)
{:ok, %{"success" => true, "message" => "accepted", "result" => sanitize(value)}}
{:error, reason} ->
@@ -367,7 +378,7 @@ defmodule BDS.MCP do
case result do
{:ok, _value} ->
:ok = ProposalStore.remove(proposal_id)
_ = ProposalStore.mark_discarded(proposal_id)
{:ok, %{"success" => true, "message" => "discarded"}}
{:error, reason} ->

24
lib/bds/mcp/proposal.ex Normal file
View File

@@ -0,0 +1,24 @@
defmodule BDS.MCP.Proposal do
@moduledoc false
use Ecto.Schema
import Ecto.Changeset
@primary_key {:id, :string, autogenerate: false}
schema "mcp_proposals" do
field :kind, :string
field :status, Ecto.Enum, values: [:pending, :accepted, :discarded, :expired], default: :pending
field :entity_id, :string
field :data, :map
field :created_at, :integer
field :expires_at, :integer
end
def changeset(proposal, attrs) do
proposal
|> cast(attrs, [:id, :kind, :status, :entity_id, :data, :created_at, :expires_at], empty_values: [nil])
|> validate_required([:id, :kind, :status, :entity_id, :data, :created_at, :expires_at])
|> unique_constraint(:status, name: :mcp_proposals_entity_idx)
end
end

89
lib/bds/mcp/proposal_store.ex
View File

@@ -1,51 +1,45 @@
defmodule BDS.MCP.ProposalStore do
@moduledoc false
use Agent
import Ecto.Query
alias BDS.MCP.Proposal
alias BDS.Persistence
alias BDS.Repo
@default_ttl_ms 30 * 60 * 1000
def ensure_started do
case Process.whereis(__MODULE__) do
nil ->
case Agent.start_link(fn -> %{} end, name: __MODULE__) do
{:ok, _pid} -> :ok
{:error, {:already_started, _pid}} -> :ok
{:error, reason} -> {:error, reason}
end
_pid ->
:ok
end
end
def ensure_started, do: :ok
def create(kind, data, opts \\ []) when is_binary(kind) and is_map(data) do
:ok = ensure_started()
cleanup_expired(opts)
now = Persistence.now_ms()
proposal = %{
entity_id = Keyword.get(opts, :entity_id) || derive_entity_id(data)
%Proposal{}
|> Proposal.changeset(%{
id: Ecto.UUID.generate(),
kind: kind,
status: :pending,
entity_id: entity_id,
data: data,
created_at: Persistence.now_ms(),
expires_at: Persistence.now_ms() + Keyword.get(opts, :ttl_ms, @default_ttl_ms)
}
Agent.update(__MODULE__, &Map.put(&1, proposal.id, proposal))
proposal
created_at: now,
expires_at: now + Keyword.get(opts, :ttl_ms, @default_ttl_ms)
})
|> Repo.insert!()
end
def get(id) when is_binary(id) do
:ok = ensure_started()
cleanup_expired([])
Agent.get(__MODULE__, &Map.get(&1, id))
Repo.get(Proposal, id)
end
def remove(id) when is_binary(id) do
:ok = ensure_started()
Agent.update(__MODULE__, &Map.delete(&1, id))
Repo.delete_all(from proposal in Proposal, where: proposal.id == ^id)
:ok
end
@@ -53,26 +47,49 @@ defmodule BDS.MCP.ProposalStore do
:ok = ensure_started()
cleanup_expired([])
Agent.get(__MODULE__, fn proposals ->
proposals
|> Map.values()
|> Enum.sort_by(& &1.created_at)
end)
Repo.all(from proposal in Proposal, order_by: [asc: proposal.created_at])
end
def cleanup_expired(opts) do
def cleanup_expired(opts \\ []) do
:ok = ensure_started()
now = Persistence.now_ms()
on_expire = Keyword.get(opts, :on_expire)
Agent.get_and_update(__MODULE__, fn proposals ->
{expired, active} = Enum.split_with(proposals, fn {_id, proposal} -> proposal.expires_at <= now end)
expired =
Repo.all(
from proposal in Proposal,
where: proposal.status == :pending and proposal.expires_at <= ^now
)
Enum.each(expired, fn {_id, proposal} ->
if is_function(on_expire, 1), do: on_expire.(proposal)
end)
{Enum.map(expired, &elem(&1, 1)), Map.new(active)}
Enum.each(expired, fn proposal ->
if is_function(on_expire, 1), do: on_expire.(proposal)
mark_status(proposal.id, :expired)
end)
Enum.map(expired, &Repo.get(Proposal, &1.id))
end
def mark_accepted(id) when is_binary(id), do: mark_status(id, :accepted)
def mark_discarded(id) when is_binary(id), do: mark_status(id, :discarded)
defp mark_status(id, status) do
case Repo.get(Proposal, id) do
nil -> nil
proposal ->
Repo.delete_all(
from other in Proposal,
where:
other.id != ^id and other.kind == ^proposal.kind and other.entity_id == ^proposal.entity_id and
other.status == ^status
)
proposal
|> Proposal.changeset(%{status: status})
|> Repo.update!()
end
end
defp derive_entity_id(data) do
data["post_id"] || data["script_id"] || data["template_id"] || data["media_id"] || Ecto.UUID.generate()
end
end

44
lib/bds/publishing.ex
View File

@@ -3,7 +3,10 @@ defmodule BDS.Publishing do
use GenServer
alias BDS.Persistence
alias BDS.Publishing.PublishJob
alias BDS.Projects
alias BDS.Repo
alias BDS.Tasks
def start_link(_opts) do
@@ -24,22 +27,25 @@ defmodule BDS.Publishing do
@impl true
def init(_state) do
{:ok, %{jobs: %{}, scp_uploads: %{}}}
{:ok, %{scp_uploads: %{}}}
end
@impl true
def handle_call({:get_job, job_id}, _from, state) do
{:reply, state.jobs[job_id], state}
{:reply, Repo.get(PublishJob, job_id), state}
end
def handle_call({:update_job, job_id, attrs}, _from, state) do
next_state =
update_in(state, [:jobs, job_id], fn
nil -> nil
job -> Map.merge(job, Map.put(attrs, :updated_at, DateTime.utc_now()))
end)
reply =
case Repo.get(PublishJob, job_id) do
nil -> :ok
job ->
attrs = Map.put(attrs, :updated_at, Persistence.now_ms())
job |> PublishJob.changeset(attrs) |> Repo.update!()
:ok
end
{:reply, :ok, next_state}
{:reply, reply, state}
end
def handle_call({:should_upload_scp_file, upload_key, local_mtime}, _from, state) do
@@ -59,8 +65,9 @@ defmodule BDS.Publishing do
def handle_call({:upload_site, project_id, credentials, targets, opts}, _from, state) do
job_id = "publish-" <> Integer.to_string(System.unique_integer([:positive, :monotonic]))
uploader = build_uploader(Keyword.put_new(opts, :project_id, project_id))
now = Persistence.now_ms()
job = %{
job_attrs = %{
id: job_id,
project_id: project_id,
status: :pending,
@@ -69,12 +76,17 @@ defmodule BDS.Publishing do
ssh_user: credentials.ssh_user,
ssh_remote_path: credentials.ssh_remote_path,
ssh_mode: credentials.ssh_mode,
targets: Enum.map(targets, & &1.kind),
targets: Enum.map(targets, &to_string(&1.kind)),
error: nil,
inserted_at: DateTime.utc_now(),
updated_at: DateTime.utc_now()
inserted_at: now,
updated_at: now
}
job =
%PublishJob{}
|> PublishJob.changeset(job_attrs)
|> Repo.insert!()
{:ok, task} =
Tasks.submit_task(
"publish #{project_id}",
@@ -87,8 +99,12 @@ defmodule BDS.Publishing do
}
)
next_job = %{job | task_id: task.id}
{:reply, {:ok, next_job}, put_in(state, [:jobs, job_id], next_job)}
next_job =
job
|> PublishJob.changeset(%{task_id: task.id, updated_at: Persistence.now_ms()})
|> Repo.update!()
{:reply, {:ok, next_job}, state}
end
defp run_upload(job_id, credentials, targets, uploader, report) do

58
lib/bds/publishing/publish_job.ex Normal file
View File

@@ -0,0 +1,58 @@
defmodule BDS.Publishing.PublishJob do
@moduledoc false
use Ecto.Schema
import Ecto.Changeset
@primary_key {:id, :string, autogenerate: false}
@foreign_key_type :string
schema "publish_jobs" do
field :project_id, :string
field :ssh_host, :string
field :ssh_user, :string
field :ssh_remote_path, :string
field :ssh_mode, Ecto.Enum, values: [:scp, :rsync], default: :scp
field :status, Ecto.Enum, values: [:pending, :running, :completed, :failed], default: :pending
field :task_id, :string
field :targets, {:array, :string}, default: []
field :error, :string
field :inserted_at, :integer
field :updated_at, :integer
end
def changeset(job, attrs) do
job
|> cast(
attrs,
[
:id,
:project_id,
:ssh_host,
:ssh_user,
:ssh_remote_path,
:ssh_mode,
:status,
:task_id,
:targets,
:error,
:inserted_at,
:updated_at
],
empty_values: [nil]
)
|> validate_required([
:id,
:project_id,
:ssh_host,
:ssh_user,
:ssh_remote_path,
:ssh_mode,
:status,
:targets,
:inserted_at,
:updated_at
])
|> foreign_key_constraint(:project_id)
end
end

33
lib/bds/search.ex
View File

@@ -10,6 +10,33 @@ defmodule BDS.Search do
alias BDS.Projects
alias BDS.Repo
@stemmer_languages [
"ar",
"ca",
"da",
"de",
"el",
"en",
"es",
"eu",
"fi",
"fr",
"ga",
"hi",
"hu",
"hy",
"it",
"lt",
"ne",
"nl",
"no",
"pt",
"ro",
"ru",
"sv",
"tr"
]
@stemmer_algorithms %{
"da" => :danish,
"nl" => :dutch,
@@ -36,9 +63,7 @@ defmodule BDS.Search do
]
def list_stemmer_languages do
@stemmer_algorithms
|> Map.keys()
|> Enum.sort()
@stemmer_languages
end
def detect_language(text) do
@@ -497,7 +522,7 @@ defmodule BDS.Search do
|> String.downcase()
|> String.split("-", parts: 2)
|> hd()
|> then(fn code -> if Map.has_key?(@stemmer_algorithms, code), do: code, else: "en" end)
|> then(fn code -> if code in @stemmer_languages, do: code, else: "en" end)
end
defp detect_language_from_hints(text) do