feat: more stuff around persistence of data

This commit is contained in:
2026-04-23 15:54:55 +02:00
parent a8bc945be9
commit 82f2ed57dd
11 changed files with 1106 additions and 2 deletions

23
lib/bds/maintenance.ex Normal file
View File

@@ -0,0 +1,23 @@
defmodule BDS.Maintenance do
@moduledoc false
def rebuild_from_filesystem(project_id, entity_type) do
case normalize_entity_type(entity_type) do
:post -> BDS.Posts.rebuild_posts_from_files(project_id)
:media -> BDS.Media.rebuild_media_from_files(project_id)
:script -> BDS.Scripts.rebuild_scripts_from_files(project_id)
:template -> BDS.Templates.rebuild_templates_from_files(project_id)
:unsupported -> {:error, :unsupported_entity_type}
end
end
defp normalize_entity_type(:post), do: :post
defp normalize_entity_type(:media), do: :media
defp normalize_entity_type(:script), do: :script
defp normalize_entity_type(:template), do: :template
defp normalize_entity_type("post"), do: :post
defp normalize_entity_type("media"), do: :media
defp normalize_entity_type("script"), do: :script
defp normalize_entity_type("template"), do: :template
defp normalize_entity_type(_entity_type), do: :unsupported
end

247
lib/bds/media.ex Normal file
View File

@@ -0,0 +1,247 @@
defmodule BDS.Media do
@moduledoc false
alias BDS.Media.Media
alias BDS.Projects
alias BDS.Repo
alias BDS.Sidecar
def import_media(attrs) do
project = Projects.get_project!(attr(attrs, :project_id))
source_path = attr(attrs, :source_path)
original_name = Path.basename(source_path)
now = System.system_time(:second)
file_name = Ecto.UUID.generate() <> Path.extname(original_name)
file_path = media_file_path(file_name, now)
sidecar_path = file_path <> ".meta"
destination = Path.join(Projects.project_data_dir(project), file_path)
stat = File.stat!(source_path)
Repo.transaction(fn ->
media =
%Media{}
|> Media.changeset(%{
id: Ecto.UUID.generate(),
project_id: project.id,
filename: file_name,
original_name: original_name,
mime_type: detect_mime(original_name),
size: stat.size,
width: attr(attrs, :width),
height: attr(attrs, :height),
title: attr(attrs, :title),
alt: attr(attrs, :alt),
caption: attr(attrs, :caption),
author: attr(attrs, :author),
language: attr(attrs, :language),
file_path: file_path,
sidecar_path: sidecar_path,
checksum: attr(attrs, :checksum),
tags: attr(attrs, :tags) || [],
created_at: now,
updated_at: now
})
|> Repo.insert!()
:ok = File.mkdir_p(Path.dirname(destination))
:ok = File.cp(source_path, destination)
:ok = write_sidecar(project, media)
media
end)
|> case do
{:ok, media} -> {:ok, media}
{:error, reason} -> {:error, reason}
end
end
def update_media(media_id, attrs) do
case Repo.get(Media, media_id) do
nil ->
{:error, :not_found}
media ->
updates = %{}
|> maybe_put(:title, attr(attrs, :title))
|> maybe_put(:alt, attr(attrs, :alt))
|> maybe_put(:caption, attr(attrs, :caption))
|> maybe_put(:author, attr(attrs, :author))
|> maybe_put(:language, attr(attrs, :language))
|> maybe_put(:tags, attr(attrs, :tags))
|> maybe_put(:width, attr(attrs, :width))
|> maybe_put(:height, attr(attrs, :height))
|> Map.put(:updated_at, System.system_time(:second))
project = Projects.get_project!(media.project_id)
Repo.transaction(fn ->
updated_media =
media
|> Media.changeset(updates)
|> Repo.update!()
:ok = write_sidecar(project, updated_media)
updated_media
end)
|> case do
{:ok, updated_media} -> {:ok, updated_media}
{:error, reason} -> {:error, reason}
end
end
end
def delete_media(media_id) do
case Repo.get(Media, media_id) do
nil ->
{:error, :not_found}
media ->
delete_file_if_present(media.project_id, media.file_path)
delete_file_if_present(media.project_id, media.sidecar_path)
Repo.delete!(media)
{:ok, :deleted}
end
end
def rebuild_media_from_files(project_id) do
project = Projects.get_project!(project_id)
media_items =
project
|> Projects.project_data_dir()
|> Path.join("media")
|> list_matching_files("*.meta")
|> Enum.filter(&binary_exists_for_sidecar?/1)
|> Enum.map(&upsert_media_from_sidecar(project, &1))
{:ok, media_items}
end
defp upsert_media_from_sidecar(project, sidecar_path) do
{:ok, fields} = sidecar_path |> File.read!() |> Sidecar.parse_document()
relative_sidecar_path = Path.relative_to(sidecar_path, Projects.project_data_dir(project))
relative_file_path = String.trim_trailing(relative_sidecar_path, ".meta")
filename = Path.basename(relative_file_path)
now = System.system_time(:second)
attrs = %{
id: Map.get(fields, "id") || Ecto.UUID.generate(),
project_id: project.id,
filename: filename,
original_name: Map.get(fields, "original_name") || filename,
mime_type: Map.get(fields, "mime_type") || detect_mime(filename),
size: Map.get(fields, "size", 0),
width: blank_to_nil(Map.get(fields, "width")),
height: blank_to_nil(Map.get(fields, "height")),
title: Map.get(fields, "title"),
alt: Map.get(fields, "alt"),
caption: Map.get(fields, "caption"),
author: Map.get(fields, "author"),
language: Map.get(fields, "language"),
file_path: relative_file_path,
sidecar_path: relative_sidecar_path,
checksum: nil,
tags: Map.get(fields, "tags", []),
created_at: Map.get(fields, "created_at", now),
updated_at: Map.get(fields, "updated_at", now)
}
media = Repo.get(Media, attrs.id) || Repo.get_by(Media, project_id: project.id, file_path: relative_file_path) || %Media{}
media
|> Media.changeset(attrs)
|> Repo.insert_or_update!()
end
defp write_sidecar(project, media) do
path = Path.join(Projects.project_data_dir(project), media.sidecar_path)
:ok = File.mkdir_p(Path.dirname(path))
atomic_write(
path,
Sidecar.serialize_document([
{:id, media.id},
{:original_name, media.original_name},
{:mime_type, media.mime_type},
{:size, media.size},
{:width, media.width},
{:height, media.height},
{:title, media.title},
{:alt, media.alt},
{:caption, media.caption},
{:author, media.author},
{:language, media.language},
{:created_at, media.created_at},
{:updated_at, media.updated_at},
{:tags, media.tags || []}
])
)
end
defp media_file_path(file_name, timestamp) do
datetime = DateTime.from_unix!(timestamp)
year = Integer.to_string(datetime.year)
month = datetime.month |> Integer.to_string() |> String.pad_leading(2, "0")
Path.join(["media", year, month, file_name])
end
defp detect_mime(file_name) do
case String.downcase(Path.extname(file_name)) do
".txt" -> "text/plain"
".md" -> "text/markdown"
".jpg" -> "image/jpeg"
".jpeg" -> "image/jpeg"
".png" -> "image/png"
".gif" -> "image/gif"
".webp" -> "image/webp"
_ -> "application/octet-stream"
end
end
defp binary_exists_for_sidecar?(sidecar_path) do
sidecar_path
|> String.trim_trailing(".meta")
|> File.exists?()
end
defp list_matching_files(dir, pattern) do
if File.dir?(dir) do
Path.join([dir, "**", pattern])
|> Path.wildcard()
|> Enum.sort()
else
[]
end
end
defp delete_file_if_present(project_id, relative_path) do
project = Projects.get_project!(project_id)
full_path = Path.join(Projects.project_data_dir(project), relative_path)
case File.rm(full_path) do
:ok -> :ok
{:error, :enoent} -> :ok
{:error, reason} -> {:error, reason}
end
end
defp atomic_write(path, contents) do
temp_path = path <> ".tmp"
:ok = File.write(temp_path, contents)
File.rename(temp_path, path)
end
defp blank_to_nil(nil), do: nil
defp blank_to_nil(""), do: nil
defp blank_to_nil(value), do: value
defp maybe_put(map, _key, nil), do: map
defp maybe_put(map, key, value), do: Map.put(map, key, value)
defp attr(attrs, key) do
cond do
Map.has_key?(attrs, key) -> Map.get(attrs, key)
Map.has_key?(attrs, Atom.to_string(key)) -> Map.get(attrs, Atom.to_string(key))
true -> nil
end
end
end

64
lib/bds/media/media.ex Normal file
View File

@@ -0,0 +1,64 @@
defmodule BDS.Media.Media do
@moduledoc false
use Ecto.Schema
import Ecto.Changeset
alias BDS.Types.StringList
@primary_key {:id, :string, autogenerate: false}
@foreign_key_type :string
schema "media" do
belongs_to :project, BDS.Projects.Project, type: :string
field :filename, :string
field :original_name, :string
field :mime_type, :string
field :size, :integer
field :width, :integer
field :height, :integer
field :title, :string
field :alt, :string
field :caption, :string
field :author, :string
field :language, :string
field :file_path, :string
field :sidecar_path, :string
field :checksum, :string
field :tags, StringList, default: []
field :created_at, :integer
field :updated_at, :integer
end
def changeset(media, attrs) do
media
|> cast(
attrs,
[
:id,
:project_id,
:filename,
:original_name,
:mime_type,
:size,
:width,
:height,
:title,
:alt,
:caption,
:author,
:language,
:file_path,
:sidecar_path,
:checksum,
:tags,
:created_at,
:updated_at
],
empty_values: [nil]
)
|> validate_required([:id, :project_id, :filename, :original_name, :mime_type, :size, :file_path, :sidecar_path, :created_at, :updated_at])
|> assoc_constraint(:project)
end
end

289
lib/bds/metadata.ex Normal file
View File

@@ -0,0 +1,289 @@
defmodule BDS.Metadata do
@moduledoc false
alias BDS.Projects
alias BDS.Projects.Project
alias BDS.Repo
alias BDS.Settings.Setting
@default_max_posts_per_page 50
@default_categories ["article", "aside", "page", "picture"]
def get_project_metadata(project_id) do
project = Projects.get_project!(project_id)
{:ok, load_state(project)}
end
def update_project_metadata(project_id, attrs) do
project = Projects.get_project!(project_id)
state = load_state(project)
now = System.system_time(:second)
project_metadata =
state
|> Map.take([:name, :description, :public_url, :main_language, :default_author, :max_posts_per_page, :blogmark_category, :pico_theme, :semantic_similarity_enabled, :blog_languages])
|> Map.merge(normalize_project_metadata_attrs(attrs, project))
Repo.transaction(fn ->
updated_project =
project
|> Project.changeset(%{name: project_metadata.name, description: project_metadata.description, updated_at: now})
|> Repo.update!()
persist_setting(project_id, "project", stringify_project_metadata(project_metadata), now)
write_project_metadata_files(updated_project, state, project_metadata)
load_state(updated_project)
end)
|> unwrap_transaction()
end
def add_category(project_id, name) do
update_state(project_id, fn project, state, now ->
categories =
state.categories
|> Kernel.++([String.trim(name)])
|> Enum.reject(&(&1 == ""))
|> Enum.uniq()
|> Enum.sort()
persist_setting(project.id, "categories", %{"categories" => categories}, now)
write_categories_json(project, categories)
%{state | categories: categories}
end)
end
def remove_category(project_id, name) do
update_state(project_id, fn project, state, now ->
categories = Enum.reject(state.categories, &(&1 == name))
category_settings = Map.delete(state.category_settings, name)
persist_setting(project.id, "categories", %{"categories" => categories}, now)
persist_setting(project.id, "category_meta", %{"categories" => category_settings}, now)
write_categories_json(project, categories)
write_category_meta_json(project, category_settings)
%{state | categories: categories, category_settings: category_settings}
end)
end
def update_category_settings(project_id, category, settings) do
update_state(project_id, fn project, state, now ->
normalized = normalize_category_settings(settings)
category_settings = Map.put(state.category_settings, category, normalized)
persist_setting(project.id, "category_meta", %{"categories" => category_settings}, now)
write_category_meta_json(project, category_settings)
%{state | category_settings: category_settings}
end)
end
def set_publishing_preferences(project_id, prefs) do
update_state(project_id, fn project, state, now ->
publishing_preferences = normalize_publishing_preferences(prefs)
persist_setting(project.id, "publishing", publishing_preferences, now)
write_publishing_json(project, publishing_preferences)
%{state | publishing_preferences: publishing_preferences}
end)
end
def sync_project_metadata_from_filesystem(project_id) do
project = Projects.get_project!(project_id)
now = System.system_time(:second)
project_metadata_from_files = read_json(project, "project.json") || stringify_project_metadata(default_project_metadata(project))
categories_from_files = read_json(project, "categories.json") || %{"categories" => @default_categories}
category_meta_from_files = read_json(project, "category-meta.json") || %{"categories" => %{}}
publishing_from_files = read_json(project, "publishing.json") || %{"ssh_mode" => "scp"}
Repo.transaction(fn ->
updated_project =
project
|> Project.changeset(%{
name: Map.get(project_metadata_from_files, "name", project.name),
description: Map.get(project_metadata_from_files, "description"),
updated_at: now
})
|> Repo.update!()
persist_setting(project_id, "project", project_metadata_from_files, now)
persist_setting(project_id, "categories", categories_from_files, now)
persist_setting(project_id, "category_meta", category_meta_from_files, now)
persist_setting(project_id, "publishing", publishing_from_files, now)
load_state(updated_project)
end)
|> unwrap_transaction()
end
defp update_state(project_id, updater) do
project = Projects.get_project!(project_id)
state = load_state(project)
now = System.system_time(:second)
Repo.transaction(fn ->
updater.(project, state, now)
end)
|> unwrap_transaction()
end
defp load_state(project) do
project_metadata = load_setting(project.id, "project") || stringify_project_metadata(default_project_metadata(project))
categories = (load_setting(project.id, "categories") || %{"categories" => @default_categories})["categories"]
category_settings =
(load_setting(project.id, "category_meta") || %{"categories" => %{}})["categories"]
publishing_preferences = load_setting(project.id, "publishing") || %{"ssh_mode" => "scp"}
%{
name: Map.get(project_metadata, "name", project.name),
description: Map.get(project_metadata, "description"),
public_url: Map.get(project_metadata, "public_url"),
main_language: Map.get(project_metadata, "main_language"),
default_author: Map.get(project_metadata, "default_author"),
max_posts_per_page: Map.get(project_metadata, "max_posts_per_page", @default_max_posts_per_page),
blogmark_category: Map.get(project_metadata, "blogmark_category"),
pico_theme: Map.get(project_metadata, "pico_theme"),
semantic_similarity_enabled: Map.get(project_metadata, "semantic_similarity_enabled", false),
blog_languages: Map.get(project_metadata, "blog_languages", []),
categories: categories,
category_settings: category_settings,
publishing_preferences: publishing_preferences
}
end
defp default_project_metadata(project) do
%{
name: project.name,
description: project.description,
public_url: nil,
main_language: nil,
default_author: nil,
max_posts_per_page: @default_max_posts_per_page,
blogmark_category: nil,
pico_theme: nil,
semantic_similarity_enabled: false,
blog_languages: []
}
end
defp normalize_project_metadata_attrs(attrs, project) do
%{
name: attr(attrs, :name) || project.name,
description: attr(attrs, :description),
public_url: attr(attrs, :public_url),
main_language: attr(attrs, :main_language),
default_author: attr(attrs, :default_author),
max_posts_per_page: attr(attrs, :max_posts_per_page) || @default_max_posts_per_page,
blogmark_category: attr(attrs, :blogmark_category),
pico_theme: attr(attrs, :pico_theme),
semantic_similarity_enabled: attr(attrs, :semantic_similarity_enabled) || false,
blog_languages: normalize_string_list(attr(attrs, :blog_languages) || [])
}
end
defp normalize_category_settings(settings) do
%{
"render_in_lists" => Map.get(settings, :render_in_lists, Map.get(settings, "render_in_lists", true)),
"show_title" => Map.get(settings, :show_title, Map.get(settings, "show_title", true)),
"post_template_slug" => Map.get(settings, :post_template_slug, Map.get(settings, "post_template_slug")),
"list_template_slug" => Map.get(settings, :list_template_slug, Map.get(settings, "list_template_slug"))
}
end
defp normalize_publishing_preferences(prefs) do
%{
"ssh_host" => attr(prefs, :ssh_host),
"ssh_user" => attr(prefs, :ssh_user),
"ssh_remote_path" => attr(prefs, :ssh_remote_path),
"ssh_mode" => attr(prefs, :ssh_mode) || "scp"
}
end
defp stringify_project_metadata(project_metadata) do
%{
"name" => project_metadata.name,
"description" => project_metadata.description,
"public_url" => project_metadata.public_url,
"main_language" => project_metadata.main_language,
"default_author" => project_metadata.default_author,
"max_posts_per_page" => project_metadata.max_posts_per_page,
"blogmark_category" => project_metadata.blogmark_category,
"pico_theme" => project_metadata.pico_theme,
"semantic_similarity_enabled" => project_metadata.semantic_similarity_enabled,
"blog_languages" => project_metadata.blog_languages
}
end
defp write_project_metadata_files(project, state, project_metadata) do
write_project_json(project, stringify_project_metadata(project_metadata))
write_categories_json(project, state.categories)
write_category_meta_json(project, state.category_settings)
write_publishing_json(project, state.publishing_preferences)
end
defp write_project_json(project, project_json), do: write_json(project, "project.json", project_json)
defp write_categories_json(project, categories) do
write_json(project, "categories.json", %{"categories" => Enum.sort(categories)})
end
defp write_category_meta_json(project, category_settings) do
write_json(project, "category-meta.json", %{"categories" => category_settings})
end
defp write_publishing_json(project, publishing_preferences) do
write_json(project, "publishing.json", publishing_preferences)
end
defp write_json(project, file_name, payload) do
meta_dir = Path.join(Projects.project_data_dir(project), "meta")
:ok = File.mkdir_p(meta_dir)
path = Path.join(meta_dir, file_name)
temp_path = path <> ".tmp"
:ok = File.write(temp_path, Jason.encode!(payload))
File.rename(temp_path, path)
end
defp read_json(project, file_name) do
path = Path.join([Projects.project_data_dir(project), "meta", file_name])
case File.read(path) do
{:ok, contents} -> Jason.decode!(contents)
{:error, :enoent} -> nil
end
end
defp load_setting(project_id, suffix) do
case Repo.get(Setting, setting_key(project_id, suffix)) do
nil -> nil
setting -> Jason.decode!(setting.value)
end
end
defp persist_setting(project_id, suffix, payload, now) do
key = setting_key(project_id, suffix)
setting = Repo.get(Setting, key) || %Setting{}
setting
|> Setting.changeset(%{key: key, value: Jason.encode!(payload), updated_at: now})
|> Repo.insert_or_update!()
end
defp setting_key(project_id, suffix), do: "project:#{project_id}:#{suffix}"
defp normalize_string_list(values) do
values
|> Enum.reject(&(&1 in [nil, ""]))
|> Enum.uniq()
end
defp unwrap_transaction({:ok, result}), do: {:ok, result}
defp unwrap_transaction({:error, reason}), do: {:error, reason}
defp attr(attrs, key) do
cond do
Map.has_key?(attrs, key) -> Map.get(attrs, key)
Map.has_key?(attrs, Atom.to_string(key)) -> Map.get(attrs, Atom.to_string(key))
true -> nil
end
end
end

View File

@@ -218,7 +218,9 @@ defmodule BDS.Scripts do
end
defp parse_script_kind(kind) when is_atom(kind), do: kind
defp parse_script_kind(kind), do: String.to_existing_atom(kind)
defp parse_script_kind("macro"), do: :macro
defp parse_script_kind("utility"), do: :utility
defp parse_script_kind("transform"), do: :transform
defp list_matching_files(dir, pattern) do
if File.dir?(dir) do

View File

@@ -0,0 +1,19 @@
defmodule BDS.Settings.Setting do
@moduledoc false
use Ecto.Schema
import Ecto.Changeset
@primary_key {:key, :string, autogenerate: false}
schema "settings" do
field :value, :string
field :updated_at, :integer
end
def changeset(setting, attrs) do
setting
|> cast(attrs, [:key, :value, :updated_at], empty_values: [nil])
|> validate_required([:key, :value, :updated_at])
end
end

77
lib/bds/sidecar.ex Normal file
View File

@@ -0,0 +1,77 @@
defmodule BDS.Sidecar do
@moduledoc false
@list_item_prefix " - "
def serialize_document(fields) when is_list(fields) do
fields
|> Enum.flat_map(&serialize_field/1)
|> Enum.join("\n")
|> Kernel.<>("\n")
end
def parse_document(contents) when is_binary(contents) do
{:ok,
contents
|> String.split("\n", trim: true)
|> parse_lines(%{})}
end
defp serialize_field({_key, nil}), do: []
defp serialize_field({_key, ""}), do: []
defp serialize_field({key, values}) when is_list(values) do
["#{key}:" | Enum.map(values, &" - #{&1}")]
end
defp serialize_field({key, value}) when is_boolean(value) do
["#{key}: #{if(value, do: "true", else: "false")}"]
end
defp serialize_field({key, value}) do
["#{key}: #{value}"]
end
defp parse_lines([], acc), do: acc
defp parse_lines([line | rest], acc) do
cond do
String.starts_with?(line, @list_item_prefix) ->
parse_lines(rest, acc)
String.ends_with?(line, ":") ->
key = String.trim_trailing(line, ":")
{items, remaining} = take_list_items(rest, [])
parse_lines(remaining, Map.put(acc, key, Enum.reverse(items)))
String.contains?(line, ": ") ->
[key, raw_value] = String.split(line, ": ", parts: 2)
parse_lines(rest, Map.put(acc, key, parse_scalar(raw_value)))
true ->
parse_lines(rest, acc)
end
end
defp take_list_items([line | rest], items) do
if String.starts_with?(line, @list_item_prefix) do
value = line |> String.replace_prefix(@list_item_prefix, "") |> parse_scalar()
take_list_items(rest, [value | items])
else
{items, [line | rest]}
end
end
defp take_list_items([], items), do: {items, []}
defp parse_scalar("true"), do: true
defp parse_scalar("false"), do: false
defp parse_scalar(value) do
if Regex.match?(~r/^-?\d+$/, value) do
String.to_integer(value)
else
value
end
end
end

View File

@@ -324,7 +324,10 @@ defmodule BDS.Templates do
end
defp parse_template_kind(kind) when is_atom(kind), do: kind
defp parse_template_kind(kind), do: String.to_existing_atom(kind)
defp parse_template_kind("post"), do: :post
defp parse_template_kind("list"), do: :list
defp parse_template_kind("not_found"), do: :not_found
defp parse_template_kind("partial"), do: :partial
defp list_matching_files(dir, pattern) do
if File.dir?(dir) do