feat: metadata, frontmatter, write atomicity should now be in

This commit is contained in:
2026-04-24 10:16:21 +02:00
parent 1d15de60de
commit a3f2c4a5f7
25 changed files with 479 additions and 136 deletions

View File

@@ -26,6 +26,7 @@ This document provides context and best practices for GitHub Copilot when workin
- HEREDOCs don't work most of the time. Don't use them. Use editor tools to create proper scripots - HEREDOCs don't work most of the time. Don't use them. Use editor tools to create proper scripots
- we have an allium spec in the specs/ folder. you must weed the specs against built code to make sure you follow the spec. - we have an allium spec in the specs/ folder. you must weed the specs against built code to make sure you follow the spec.
- when changing the spec, validate the spec with the available command line tool. - when changing the spec, validate the spec with the available command line tool.
- test with command line tools at least once to capture compile errors in tests, do not use the integrated testing of vscode, as that blocks on compile errors
--- ---

View File

@@ -3,6 +3,7 @@ defmodule BDS.Embeddings do
import Ecto.Query import Ecto.Query
alias BDS.Persistence
alias BDS.Embeddings.DismissedDuplicatePair alias BDS.Embeddings.DismissedDuplicatePair
alias BDS.Embeddings.Index alias BDS.Embeddings.Index
alias BDS.Embeddings.Key alias BDS.Embeddings.Key
@@ -317,7 +318,7 @@ defmodule BDS.Embeddings do
project_id: post_a.project_id, project_id: post_a.project_id,
post_id_a: sorted_a, post_id_a: sorted_a,
post_id_b: sorted_b, post_id_b: sorted_b,
dismissed_at: System.system_time(:second) dismissed_at: Persistence.now_ms()
}) })
|> Repo.insert_or_update!() |> Repo.insert_or_update!()

View File

@@ -3,6 +3,7 @@ defmodule BDS.Embeddings.Index do
import Ecto.Query import Ecto.Query
alias BDS.Persistence
alias BDS.Embeddings.Key alias BDS.Embeddings.Key
alias BDS.Projects alias BDS.Projects
alias BDS.Repo alias BDS.Repo
@@ -43,7 +44,7 @@ defmodule BDS.Embeddings.Index do
"project_id" => project_id, "project_id" => project_id,
"model_id" => model_id, "model_id" => model_id,
"dimensions" => dimensions, "dimensions" => dimensions,
"updated_at" => System.system_time(:second), "updated_at" => Persistence.now_ms(),
"entries" => entries "entries" => entries
} }
@@ -123,10 +124,7 @@ defmodule BDS.Embeddings.Index do
end end
defp write_snapshot(snapshot_path, payload) do defp write_snapshot(snapshot_path, payload) do
:ok = File.mkdir_p(Path.dirname(snapshot_path)) :ok = Persistence.atomic_write(snapshot_path, Jason.encode!(payload))
temp_path = snapshot_path <> ".tmp"
:ok = File.write(temp_path, Jason.encode!(payload))
:ok = File.rename(temp_path, snapshot_path)
legacy_path = legacy_path(snapshot_path) legacy_path = legacy_path(snapshot_path)
if File.exists?(legacy_path) do if File.exists?(legacy_path) do

View File

@@ -1,6 +1,8 @@
defmodule BDS.Frontmatter do defmodule BDS.Frontmatter do
@moduledoc false @moduledoc false
alias BDS.Persistence
@list_item_prefix " - " @list_item_prefix " - "
def serialize_document(fields, body) when is_list(fields) do def serialize_document(fields, body) when is_list(fields) do
@@ -38,11 +40,26 @@ defmodule BDS.Frontmatter do
end end
defp serialize_field({key, values}) when is_list(values) do defp serialize_field({key, values}) when is_list(values) do
["#{key}:" | Enum.map(values, &" - #{&1}")] ["#{key}:" | Enum.map(values, &" - #{serialize_scalar(nil, &1)}")]
end
defp serialize_field({key, value}) when is_atom(value) do
["#{key}: #{Atom.to_string(value)}"]
end
defp serialize_field({key, value}) when is_integer(value) do
rendered =
if timestamp_key?(key) do
Persistence.timestamp_to_iso8601(value)
else
Integer.to_string(value)
end
["#{key}: #{rendered}"]
end end
defp serialize_field({key, value}) do defp serialize_field({key, value}) do
["#{key}: #{value}"] ["#{key}: #{serialize_scalar(key, value)}"]
end end
defp parse_frontmatter(frontmatter) do defp parse_frontmatter(frontmatter) do
@@ -65,7 +82,7 @@ defmodule BDS.Frontmatter do
String.contains?(line, ": ") -> String.contains?(line, ": ") ->
[key, raw_value] = String.split(line, ": ", parts: 2) [key, raw_value] = String.split(line, ": ", parts: 2)
parse_lines(rest, Map.put(acc, key, parse_scalar(raw_value))) parse_lines(rest, Map.put(acc, key, parse_scalar(key, raw_value)))
true -> true ->
parse_lines(rest, acc) parse_lines(rest, acc)
@@ -74,7 +91,7 @@ defmodule BDS.Frontmatter do
defp take_list_items([line | rest], items) do defp take_list_items([line | rest], items) do
if String.starts_with?(line, @list_item_prefix) do if String.starts_with?(line, @list_item_prefix) do
value = line |> String.replace_prefix(@list_item_prefix, "") |> parse_scalar() value = line |> String.replace_prefix(@list_item_prefix, "") |> then(&parse_scalar(nil, &1))
take_list_items(rest, [value | items]) take_list_items(rest, [value | items])
else else
{items, [line | rest]} {items, [line | rest]}
@@ -83,14 +100,80 @@ defmodule BDS.Frontmatter do
defp take_list_items([], items), do: {items, []} defp take_list_items([], items), do: {items, []}
defp parse_scalar("true"), do: true defp parse_scalar(key, value) when is_binary(key) and is_binary(value) do
defp parse_scalar("false"), do: false trimmed = String.trim(value)
defp parse_scalar(value) do cond do
timestamp_key?(key) ->
Persistence.parse_timestamp(trimmed) || parse_generic_scalar(trimmed)
true ->
parse_generic_scalar(trimmed)
end
end
defp parse_scalar(nil, value) when is_binary(value) do
value
|> String.trim()
|> parse_generic_scalar()
end
defp parse_generic_scalar("true"), do: true
defp parse_generic_scalar("false"), do: false
defp parse_generic_scalar(value) do
if Regex.match?(~r/^-?\d+$/, value) do if Regex.match?(~r/^-?\d+$/, value) do
String.to_integer(value) String.to_integer(value)
else else
parse_string(value)
end
end
defp parse_string("\"" <> rest) do
rest
|> String.trim_trailing("\"")
|> String.replace("\\n", "\n")
|> String.replace("\\\"", "\"")
|> String.replace("\\\\", "\\")
end
defp parse_string(value), do: value
defp serialize_scalar(_key, value) when is_boolean(value) do
if(value, do: "true", else: "false")
end
defp serialize_scalar(_key, value) when is_atom(value) do
Atom.to_string(value)
end
defp serialize_scalar(key, value) when is_integer(value) do
if is_binary(key) and timestamp_key?(key) do
Persistence.timestamp_to_iso8601(value)
else
Integer.to_string(value)
end
end
defp serialize_scalar(_key, value) do
value value
|> to_string()
|> maybe_quote_string()
end
defp maybe_quote_string(value) do
if Regex.match?(~r/^[\p{L}\p{N} ._\/-]+$/u, value) do
value
else
escaped =
value
|> String.replace("\\", "\\\\")
|> String.replace("\"", "\\\"")
|> String.replace("\n", "\\n")
"\"#{escaped}\""
end end
end end
defp timestamp_key?(key), do: String.ends_with?(to_string(key), "_at")
end end

View File

@@ -5,6 +5,7 @@ defmodule BDS.Generation do
alias BDS.Generation.GeneratedFileHash alias BDS.Generation.GeneratedFileHash
alias BDS.Metadata alias BDS.Metadata
alias BDS.Persistence
alias BDS.Posts.Post alias BDS.Posts.Post
alias BDS.Posts.Translation alias BDS.Posts.Translation
alias BDS.Projects alias BDS.Projects
@@ -52,7 +53,7 @@ defmodule BDS.Generation do
def post_output_path(%Post{} = post), do: post_output_path(post, nil) def post_output_path(%Post{} = post), do: post_output_path(post, nil)
def post_output_path(%Post{} = post, language) do def post_output_path(%Post{} = post, language) do
datetime = DateTime.from_unix!(post.created_at) datetime = Persistence.from_unix_ms!(post.created_at)
year = Integer.to_string(datetime.year) year = Integer.to_string(datetime.year)
month = datetime.month |> Integer.to_string() |> String.pad_leading(2, "0") month = datetime.month |> Integer.to_string() |> String.pad_leading(2, "0")
day = datetime.day |> Integer.to_string() |> String.pad_leading(2, "0") day = datetime.day |> Integer.to_string() |> String.pad_leading(2, "0")
@@ -70,7 +71,7 @@ defmodule BDS.Generation do
when is_binary(project_id) and is_binary(relative_path) and is_binary(content) do when is_binary(project_id) and is_binary(relative_path) and is_binary(content) do
project = Projects.get_project!(project_id) project = Projects.get_project!(project_id)
content_hash = sha256(content) content_hash = sha256(content)
now = System.system_time(:second) now = Persistence.now_ms()
case Repo.get_by(GeneratedFileHash, project_id: project_id, relative_path: relative_path) do case Repo.get_by(GeneratedFileHash, project_id: project_id, relative_path: relative_path) do
%GeneratedFileHash{content_hash: ^content_hash} -> %GeneratedFileHash{content_hash: ^content_hash} ->
@@ -78,8 +79,7 @@ defmodule BDS.Generation do
_existing -> _existing ->
full_path = output_path(project, relative_path) full_path = output_path(project, relative_path)
:ok = File.mkdir_p(Path.dirname(full_path)) :ok = Persistence.atomic_write(full_path, content)
:ok = File.write(full_path, content)
attrs = %{ attrs = %{
project_id: project_id, project_id: project_id,
@@ -495,7 +495,7 @@ defmodule BDS.Generation do
defp render_calendar(published_posts) do defp render_calendar(published_posts) do
published_posts published_posts
|> Enum.map(fn post -> |> Enum.map(fn post ->
datetime = DateTime.from_unix!(post.created_at) datetime = Persistence.from_unix_ms!(post.created_at)
%{date: Date.to_iso8601(DateTime.to_date(datetime)), slug: post.slug, title: post.title} %{date: Date.to_iso8601(DateTime.to_date(datetime)), slug: post.slug, title: post.title}
end) end)
|> Jason.encode!() |> Jason.encode!()
@@ -637,13 +637,13 @@ defmodule BDS.Generation do
defp year_key(created_at) do defp year_key(created_at) do
created_at created_at
|> DateTime.from_unix!() |> Persistence.from_unix_ms!()
|> Map.fetch!(:year) |> Map.fetch!(:year)
|> Integer.to_string() |> Integer.to_string()
end end
defp month_key(created_at) do defp month_key(created_at) do
datetime = DateTime.from_unix!(created_at) datetime = Persistence.from_unix_ms!(created_at)
{Integer.to_string(datetime.year), {Integer.to_string(datetime.year),
Integer.to_string(datetime.month) |> String.pad_leading(2, "0")} Integer.to_string(datetime.month) |> String.pad_leading(2, "0")}

View File

@@ -5,6 +5,7 @@ defmodule BDS.Media do
alias BDS.Media.Media alias BDS.Media.Media
alias BDS.Media.Translation alias BDS.Media.Translation
alias BDS.Persistence
alias BDS.Projects alias BDS.Projects
alias BDS.Repo alias BDS.Repo
alias BDS.Search alias BDS.Search
@@ -16,7 +17,7 @@ defmodule BDS.Media do
original_name = Path.basename(source_path) original_name = Path.basename(source_path)
mime_type = detect_mime(original_name) mime_type = detect_mime(original_name)
{width, height} = image_dimensions(source_path, mime_type) {width, height} = image_dimensions(source_path, mime_type)
now = System.system_time(:second) now = Persistence.now_ms()
file_name = Ecto.UUID.generate() <> Path.extname(original_name) file_name = Ecto.UUID.generate() <> Path.extname(original_name)
file_path = media_file_path(file_name, now) file_path = media_file_path(file_name, now)
sidecar_path = file_path <> ".meta" sidecar_path = file_path <> ".meta"
@@ -78,7 +79,7 @@ defmodule BDS.Media do
|> maybe_put(:tags, attr(attrs, :tags)) |> maybe_put(:tags, attr(attrs, :tags))
|> maybe_put(:width, attr(attrs, :width)) |> maybe_put(:width, attr(attrs, :width))
|> maybe_put(:height, attr(attrs, :height)) |> maybe_put(:height, attr(attrs, :height))
|> Map.put(:updated_at, System.system_time(:second)) |> Map.put(:updated_at, Persistence.now_ms())
project = Projects.get_project!(media.project_id) project = Projects.get_project!(media.project_id)
@@ -136,7 +137,7 @@ defmodule BDS.Media do
media -> media ->
project = Projects.get_project!(media.project_id) project = Projects.get_project!(media.project_id)
now = System.system_time(:second) now = Persistence.now_ms()
translation = translation =
Repo.get_by(Translation, translation_for: media.id, language: language) || Repo.get_by(Translation, translation_for: media.id, language: language) ||
@@ -227,7 +228,7 @@ defmodule BDS.Media do
relative_sidecar_path = Path.relative_to(sidecar_path, Projects.project_data_dir(project)) relative_sidecar_path = Path.relative_to(sidecar_path, Projects.project_data_dir(project))
relative_file_path = String.trim_trailing(relative_sidecar_path, ".meta") relative_file_path = String.trim_trailing(relative_sidecar_path, ".meta")
filename = Path.basename(relative_file_path) filename = Path.basename(relative_file_path)
now = System.system_time(:second) now = Persistence.now_ms()
attrs = %{ attrs = %{
id: Map.get(fields, "id") || Ecto.UUID.generate(), id: Map.get(fields, "id") || Ecto.UUID.generate(),
@@ -317,7 +318,7 @@ defmodule BDS.Media do
media -> media ->
{:ok, fields} = sidecar_path |> File.read!() |> Sidecar.parse_document() {:ok, fields} = sidecar_path |> File.read!() |> Sidecar.parse_document()
now = System.system_time(:second) now = Persistence.now_ms()
language = Map.fetch!(fields, "language") language = Map.fetch!(fields, "language")
translation = translation =
@@ -408,7 +409,7 @@ defmodule BDS.Media do
end end
defp media_file_path(file_name, timestamp) do defp media_file_path(file_name, timestamp) do
datetime = DateTime.from_unix!(timestamp) datetime = Persistence.from_unix_ms!(timestamp)
year = Integer.to_string(datetime.year) year = Integer.to_string(datetime.year)
month = datetime.month |> Integer.to_string() |> String.pad_leading(2, "0") month = datetime.month |> Integer.to_string() |> String.pad_leading(2, "0")
Path.join(["media", year, month, file_name]) Path.join(["media", year, month, file_name])
@@ -487,9 +488,7 @@ defmodule BDS.Media do
end end
defp atomic_write(path, contents) do defp atomic_write(path, contents) do
temp_path = path <> ".tmp" Persistence.atomic_write(path, contents)
:ok = File.write(temp_path, contents)
File.rename(temp_path, path)
end end
defp blank_to_nil(nil), do: nil defp blank_to_nil(nil), do: nil

View File

@@ -3,6 +3,7 @@ defmodule BDS.Menu do
require Record require Record
alias BDS.Persistence
alias BDS.Projects alias BDS.Projects
Record.defrecord(:xmlElement, Record.extract(:xmlElement, from_lib: "xmerl/include/xmerl.hrl")) Record.defrecord(:xmlElement, Record.extract(:xmlElement, from_lib: "xmerl/include/xmerl.hrl"))
@@ -44,14 +45,8 @@ defmodule BDS.Menu do
end end
defp write_menu_file(project, menu) do defp write_menu_file(project, menu) do
meta_dir = Path.dirname(menu_path(project))
:ok = File.mkdir_p(meta_dir)
path = menu_path(project) path = menu_path(project)
temp_path = path <> ".tmp" :ok = Persistence.atomic_write(path, serialize_opml(project, menu.items))
:ok = File.write(temp_path, serialize_opml(menu.items))
File.rename(temp_path, path)
end end
defp menu_path(project) do defp menu_path(project) do
@@ -84,7 +79,9 @@ defmodule BDS.Menu do
end end
end end
defp serialize_opml(items) do defp serialize_opml(project, items) do
timestamp = project.updated_at || project.created_at || Persistence.now_ms()
rendered_items = rendered_items =
items items
|> Enum.map(&render_item(&1, 2)) |> Enum.map(&render_item(&1, 2))
@@ -93,6 +90,11 @@ defmodule BDS.Menu do
[ [
~s(<?xml version="1.0" encoding="UTF-8"?>), ~s(<?xml version="1.0" encoding="UTF-8"?>),
~s(<opml version="2.0">), ~s(<opml version="2.0">),
~s( <head>),
~s( <title>#{xml_escape(project.name)}</title>),
~s( <dateCreated>#{Persistence.timestamp_to_iso8601(timestamp)}</dateCreated>),
~s( <dateModified>#{Persistence.timestamp_to_iso8601(timestamp)}</dateModified>),
~s( </head>),
~s( <body>), ~s( <body>),
rendered_items, rendered_items,
~s( </body>), ~s( </body>),

View File

@@ -2,6 +2,8 @@ defmodule BDS.Metadata do
@moduledoc false @moduledoc false
alias BDS.Embeddings alias BDS.Embeddings
alias BDS.I18n
alias BDS.Persistence
alias BDS.Projects alias BDS.Projects
alias BDS.Projects.Project alias BDS.Projects.Project
alias BDS.Repo alias BDS.Repo
@@ -9,6 +11,30 @@ defmodule BDS.Metadata do
@default_max_posts_per_page 50 @default_max_posts_per_page 50
@default_categories ["article", "aside", "page", "picture"] @default_categories ["article", "aside", "page", "picture"]
@min_posts_per_page 1
@max_posts_per_page 500
@supported_pico_themes MapSet.new([
"default",
"amber",
"blue",
"cyan",
"fuchsia",
"green",
"grey",
"indigo",
"jade",
"lime",
"orange",
"pink",
"pumpkin",
"purple",
"red",
"sand",
"slate",
"violet",
"yellow",
"zinc"
])
def get_project_metadata(project_id) do def get_project_metadata(project_id) do
project = Projects.get_project!(project_id) project = Projects.get_project!(project_id)
@@ -18,7 +44,7 @@ defmodule BDS.Metadata do
def update_project_metadata(project_id, attrs) do def update_project_metadata(project_id, attrs) do
project = Projects.get_project!(project_id) project = Projects.get_project!(project_id)
state = load_state(project) state = load_state(project)
now = System.system_time(:second) now = Persistence.now_ms()
project_metadata = project_metadata =
state state
@@ -104,7 +130,7 @@ defmodule BDS.Metadata do
def sync_project_metadata_from_filesystem(project_id) do def sync_project_metadata_from_filesystem(project_id) do
project = Projects.get_project!(project_id) project = Projects.get_project!(project_id)
now = System.system_time(:second) now = Persistence.now_ms()
project_metadata_from_files = project_metadata_from_files =
read_json(project, "project.json") || read_json(project, "project.json") ||
@@ -130,6 +156,10 @@ defmodule BDS.Metadata do
persist_setting(project_id, "categories", categories_from_files, now) persist_setting(project_id, "categories", categories_from_files, now)
persist_setting(project_id, "category_meta", category_meta_from_files, now) persist_setting(project_id, "category_meta", category_meta_from_files, now)
persist_setting(project_id, "publishing", publishing_from_files, now) persist_setting(project_id, "publishing", publishing_from_files, now)
write_project_json(updated_project, project_metadata_from_files)
write_categories_json(updated_project, categories_from_files["categories"] || @default_categories)
write_category_meta_json(updated_project, category_meta_from_files["categories"] || %{})
write_publishing_json(updated_project, publishing_from_files)
load_state(updated_project) load_state(updated_project)
end) end)
|> unwrap_transaction() |> unwrap_transaction()
@@ -138,7 +168,7 @@ defmodule BDS.Metadata do
defp update_state(project_id, updater) do defp update_state(project_id, updater) do
project = Projects.get_project!(project_id) project = Projects.get_project!(project_id)
state = load_state(project) state = load_state(project)
now = System.system_time(:second) now = Persistence.now_ms()
Repo.transaction(fn -> Repo.transaction(fn ->
updater.(project, state, now) updater.(project, state, now)
@@ -200,13 +230,13 @@ defmodule BDS.Metadata do
name: attr(attrs, :name) || project.name, name: attr(attrs, :name) || project.name,
description: attr(attrs, :description), description: attr(attrs, :description),
public_url: attr(attrs, :public_url), public_url: attr(attrs, :public_url),
main_language: attr(attrs, :main_language), main_language: normalize_optional_language(attr(attrs, :main_language)),
default_author: attr(attrs, :default_author), default_author: attr(attrs, :default_author),
max_posts_per_page: attr(attrs, :max_posts_per_page) || @default_max_posts_per_page, max_posts_per_page: normalize_posts_per_page(attr(attrs, :max_posts_per_page)),
blogmark_category: attr(attrs, :blogmark_category), blogmark_category: attr(attrs, :blogmark_category),
pico_theme: attr(attrs, :pico_theme), pico_theme: normalize_pico_theme(attr(attrs, :pico_theme)),
semantic_similarity_enabled: attr(attrs, :semantic_similarity_enabled) || false, semantic_similarity_enabled: attr(attrs, :semantic_similarity_enabled) || false,
blog_languages: normalize_string_list(attr(attrs, :blog_languages) || []) blog_languages: normalize_language_list(attr(attrs, :blog_languages) || [])
} }
end end
@@ -227,7 +257,7 @@ defmodule BDS.Metadata do
"ssh_host" => attr(prefs, :ssh_host), "ssh_host" => attr(prefs, :ssh_host),
"ssh_user" => attr(prefs, :ssh_user), "ssh_user" => attr(prefs, :ssh_user),
"ssh_remote_path" => attr(prefs, :ssh_remote_path), "ssh_remote_path" => attr(prefs, :ssh_remote_path),
"ssh_mode" => attr(prefs, :ssh_mode) || "scp" "ssh_mode" => normalize_ssh_mode(attr(prefs, :ssh_mode))
} }
end end
@@ -270,11 +300,8 @@ defmodule BDS.Metadata do
defp write_json(project, file_name, payload) do defp write_json(project, file_name, payload) do
meta_dir = Path.join(Projects.project_data_dir(project), "meta") meta_dir = Path.join(Projects.project_data_dir(project), "meta")
:ok = File.mkdir_p(meta_dir)
path = Path.join(meta_dir, file_name) path = Path.join(meta_dir, file_name)
temp_path = path <> ".tmp" :ok = Persistence.atomic_write(path, Jason.encode!(payload))
:ok = File.write(temp_path, Jason.encode!(payload))
File.rename(temp_path, path)
end end
defp read_json(project, file_name) do defp read_json(project, file_name) do
@@ -304,12 +331,56 @@ defmodule BDS.Metadata do
defp setting_key(project_id, suffix), do: "project:#{project_id}:#{suffix}" defp setting_key(project_id, suffix), do: "project:#{project_id}:#{suffix}"
defp normalize_string_list(values) do defp normalize_posts_per_page(nil), do: @default_max_posts_per_page
defp normalize_posts_per_page(value) when is_integer(value) do
value
|> max(@min_posts_per_page)
|> min(@max_posts_per_page)
end
defp normalize_posts_per_page(value) when is_binary(value) do
case Integer.parse(String.trim(value)) do
{integer, ""} -> normalize_posts_per_page(integer)
_ -> @default_max_posts_per_page
end
end
defp normalize_posts_per_page(_value), do: @default_max_posts_per_page
defp normalize_optional_language(nil), do: nil
defp normalize_optional_language(""), do: nil
defp normalize_optional_language(value) do
normalized = value |> to_string() |> String.trim() |> String.downcase()
supported_language_codes = Enum.map(I18n.supported_languages(), & &1.code)
if normalized in supported_language_codes do
normalized
else
nil
end
end
defp normalize_language_list(values) do
values values
|> Enum.reject(&(&1 in [nil, ""])) |> Enum.map(&normalize_optional_language/1)
|> Enum.reject(&is_nil/1)
|> Enum.uniq() |> Enum.uniq()
end end
defp normalize_pico_theme(nil), do: nil
defp normalize_pico_theme(""), do: nil
defp normalize_pico_theme(value) do
normalized = value |> to_string() |> String.trim()
if MapSet.member?(@supported_pico_themes, normalized), do: normalized, else: nil
end
defp normalize_ssh_mode(:rsync), do: "rsync"
defp normalize_ssh_mode("rsync"), do: "rsync"
defp normalize_ssh_mode(_mode), do: "scp"
defp unwrap_transaction({:ok, result}), do: {:ok, result} defp unwrap_transaction({:ok, result}), do: {:ok, result}
defp unwrap_transaction({:error, reason}), do: {:error, reason} defp unwrap_transaction({:error, reason}), do: {:error, reason}

85
lib/bds/persistence.ex Normal file
View File

@@ -0,0 +1,85 @@
defmodule BDS.Persistence do
@moduledoc false
def now_ms, do: System.system_time(:millisecond)
def normalize_unix_timestamp(nil), do: nil
def normalize_unix_timestamp(value) when is_integer(value) do
if abs(value) < 100_000_000_000 do
value * 1000
else
value
end
end
def normalize_unix_timestamp(value) when is_binary(value) do
value
|> String.trim()
|> case do
"" -> nil
trimmed ->
case Integer.parse(trimmed) do
{integer, ""} -> normalize_unix_timestamp(integer)
_ -> nil
end
end
end
def normalize_unix_timestamp(_value), do: nil
def from_unix_ms!(value) when is_integer(value) do
value
|> normalize_unix_timestamp()
|> DateTime.from_unix!(:millisecond)
end
def timestamp_to_iso8601(nil), do: nil
def timestamp_to_iso8601(value) when is_integer(value) do
value
|> from_unix_ms!()
|> DateTime.to_iso8601()
end
def parse_timestamp(nil), do: nil
def parse_timestamp(value) when is_integer(value), do: normalize_unix_timestamp(value)
def parse_timestamp(value) when is_binary(value) do
trimmed = String.trim(value)
cond do
trimmed == "" ->
nil
Regex.match?(~r/^-?\d+$/, trimmed) ->
normalize_unix_timestamp(trimmed)
true ->
case DateTime.from_iso8601(trimmed) do
{:ok, datetime, _offset} -> DateTime.to_unix(datetime, :millisecond)
_ -> nil
end
end
end
def parse_timestamp(_value), do: nil
def atomic_write(path, contents) when is_binary(path) and is_binary(contents) do
:ok = File.mkdir_p(Path.dirname(path))
temp_path = path <> ".tmp"
with :ok <- File.write(temp_path, contents),
:ok <- File.rename(temp_path, path) do
:ok
else
{:error, _reason} = error ->
_ = File.rm(temp_path)
error
error ->
_ = File.rm(temp_path)
error
end
end
end

View File

@@ -3,6 +3,7 @@ defmodule BDS.PostLinks do
import Ecto.Query import Ecto.Query
alias BDS.Persistence
alias BDS.Posts.Link alias BDS.Posts.Link
alias BDS.Posts.Post alias BDS.Posts.Post
alias BDS.Projects alias BDS.Projects
@@ -22,7 +23,7 @@ defmodule BDS.PostLinks do
Repo.transaction(fn -> Repo.transaction(fn ->
Repo.delete_all(from link in Link, where: link.source_post_id == ^post.id) Repo.delete_all(from link in Link, where: link.source_post_id == ^post.id)
now = System.system_time(:second) now = Persistence.now_ms()
Enum.each(links, fn %{target_post_id: target_post_id, link_text: link_text} -> Enum.each(links, fn %{target_post_id: target_post_id, link_text: link_text} ->
%Link{} %Link{}

View File

@@ -6,6 +6,7 @@ defmodule BDS.Posts do
alias BDS.Frontmatter alias BDS.Frontmatter
alias BDS.Embeddings alias BDS.Embeddings
alias BDS.Metadata alias BDS.Metadata
alias BDS.Persistence
alias BDS.PostLinks alias BDS.PostLinks
alias BDS.Posts.Post alias BDS.Posts.Post
alias BDS.Posts.Translation alias BDS.Posts.Translation
@@ -15,7 +16,7 @@ defmodule BDS.Posts do
alias BDS.Slug alias BDS.Slug
def create_post(attrs) do def create_post(attrs) do
now = System.system_time(:second) now = Persistence.now_ms()
project_id = attr(attrs, :project_id) project_id = attr(attrs, :project_id)
title = normalize_title(attr(attrs, :title)) title = normalize_title(attr(attrs, :title))
base_slug = title |> default_slug_source() |> Slug.slugify() base_slug = title |> default_slug_source() |> Slug.slugify()
@@ -65,7 +66,7 @@ defmodule BDS.Posts do
post -> post ->
with :ok <- validate_slug_change(post, attrs) do with :ok <- validate_slug_change(post, attrs) do
now = System.system_time(:second) now = Persistence.now_ms()
updates = updates =
attrs attrs
@@ -99,16 +100,14 @@ defmodule BDS.Posts do
%Post{} = post -> %Post{} = post ->
project = Projects.get_project!(post.project_id) project = Projects.get_project!(post.project_id)
published_at = post.published_at || System.system_time(:second) published_at = post.published_at || Persistence.now_ms()
relative_path = build_post_relative_path(post.slug, post.created_at) relative_path = build_post_relative_path(post.slug, post.created_at)
full_path = Path.join(Projects.project_data_dir(project), relative_path) full_path = Path.join(Projects.project_data_dir(project), relative_path)
updated_at = System.system_time(:second) updated_at = Persistence.now_ms()
body = publishable_post_body(post, full_path, project) body = publishable_post_body(post, full_path, project)
:ok = File.mkdir_p(Path.dirname(full_path))
:ok = :ok =
File.write( Persistence.atomic_write(
full_path, full_path,
serialize_post_file(%{post | updated_at: updated_at, content: body}, published_at) serialize_post_file(%{post | updated_at: updated_at, content: body}, published_at)
) )
@@ -171,7 +170,7 @@ defmodule BDS.Posts do
%Post{status: status} = post when status in [:draft, :published] -> %Post{status: status} = post when status in [:draft, :published] ->
post post
|> Post.changeset(%{status: :archived, updated_at: System.system_time(:second)}) |> Post.changeset(%{status: :archived, updated_at: Persistence.now_ms()})
|> Repo.update() |> Repo.update()
|> case do |> case do
{:ok, updated_post} -> {:ok, updated_post} ->
@@ -218,7 +217,7 @@ defmodule BDS.Posts do
)} )}
%Post{} = post -> %Post{} = post ->
now = System.system_time(:second) now = Persistence.now_ms()
normalized_language = normalize_language(language) normalized_language = normalize_language(language)
translation = translation =
@@ -322,14 +321,12 @@ defmodule BDS.Posts do
project = Projects.get_project!(post.project_id) project = Projects.get_project!(post.project_id)
full_path = Path.join(Projects.project_data_dir(project), post.file_path) full_path = Path.join(Projects.project_data_dir(project), post.file_path)
body = published_post_body(post, full_path) body = published_post_body(post, full_path)
:ok = File.mkdir_p(Path.dirname(full_path))
:ok = :ok =
File.write( Persistence.atomic_write(
full_path, full_path,
serialize_post_file( serialize_post_file(
%{post | content: body}, %{post | content: body},
post.published_at || System.system_time(:second) post.published_at || Persistence.now_ms()
) )
) )
end end
@@ -451,7 +448,7 @@ defmodule BDS.Posts do
defp default_slug_source(title), do: title defp default_slug_source(title), do: title
defp build_post_relative_path(slug, created_at) do defp build_post_relative_path(slug, created_at) do
datetime = DateTime.from_unix!(created_at) datetime = Persistence.from_unix_ms!(created_at)
year = Integer.to_string(datetime.year) year = Integer.to_string(datetime.year)
month = datetime.month |> Integer.to_string() |> String.pad_leading(2, "0") month = datetime.month |> Integer.to_string() |> String.pad_leading(2, "0")
Path.join(["posts", year, month, "#{slug}.md"]) Path.join(["posts", year, month, "#{slug}.md"])
@@ -513,7 +510,7 @@ defmodule BDS.Posts do
contents = File.read!(path) contents = File.read!(path)
{:ok, %{fields: fields}} = Frontmatter.parse_document(contents) {:ok, %{fields: fields}} = Frontmatter.parse_document(contents)
relative_path = Path.relative_to(path, Projects.project_data_dir(project)) relative_path = Path.relative_to(path, Projects.project_data_dir(project))
now = System.system_time(:second) now = Persistence.now_ms()
attrs = %{ attrs = %{
id: Map.get(fields, "id") || Ecto.UUID.generate(), id: Map.get(fields, "id") || Ecto.UUID.generate(),
@@ -626,16 +623,14 @@ defmodule BDS.Posts do
defp publish_translation(%Post{} = post, %Translation{} = translation) do defp publish_translation(%Post{} = post, %Translation{} = translation) do
project = Projects.get_project!(post.project_id) project = Projects.get_project!(post.project_id)
published_at = translation.published_at || System.system_time(:second) published_at = translation.published_at || Persistence.now_ms()
relative_path = build_translation_relative_path(post, translation.language) relative_path = build_translation_relative_path(post, translation.language)
full_path = Path.join(Projects.project_data_dir(project), relative_path) full_path = Path.join(Projects.project_data_dir(project), relative_path)
updated_at = System.system_time(:second) updated_at = Persistence.now_ms()
body = publishable_translation_body(translation, full_path) body = publishable_translation_body(translation, full_path)
:ok = File.mkdir_p(Path.dirname(full_path))
:ok = :ok =
File.write( Persistence.atomic_write(
full_path, full_path,
serialize_translation_file( serialize_translation_file(
%{translation | updated_at: updated_at, content: body}, %{translation | updated_at: updated_at, content: body},
@@ -657,7 +652,7 @@ defmodule BDS.Posts do
end end
defp build_translation_relative_path(post, language) do defp build_translation_relative_path(post, language) do
datetime = DateTime.from_unix!(post.created_at) datetime = Persistence.from_unix_ms!(post.created_at)
year = Integer.to_string(datetime.year) year = Integer.to_string(datetime.year)
month = datetime.month |> Integer.to_string() |> String.pad_leading(2, "0") month = datetime.month |> Integer.to_string() |> String.pad_leading(2, "0")
Path.join(["posts", year, month, "#{post.slug}.#{language}.md"]) Path.join(["posts", year, month, "#{post.slug}.#{language}.md"])

View File

@@ -4,6 +4,7 @@ defmodule BDS.Projects do
import Ecto.Query import Ecto.Query
alias Ecto.Multi alias Ecto.Multi
alias BDS.Persistence
alias BDS.Projects.Project alias BDS.Projects.Project
alias BDS.Repo alias BDS.Repo
alias BDS.StarterTemplates alias BDS.StarterTemplates
@@ -26,7 +27,7 @@ defmodule BDS.Projects do
{:ok, project} {:ok, project}
nil -> nil ->
now = System.system_time(:second) now = Persistence.now_ms()
is_active = not Repo.exists?(from project in Project, where: project.is_active == true) is_active = not Repo.exists?(from project in Project, where: project.is_active == true)
Repo.transaction(fn -> Repo.transaction(fn ->
@@ -60,7 +61,7 @@ defmodule BDS.Projects do
end end
def create_project(attrs) do def create_project(attrs) do
now = System.system_time(:second) now = Persistence.now_ms()
name = attr(attrs, :name) || "" name = attr(attrs, :name) || ""
slug = unique_slug(attr(attrs, :slug) || Slug.slugify(name)) slug = unique_slug(attr(attrs, :slug) || Slug.slugify(name))
@@ -95,7 +96,7 @@ defmodule BDS.Projects do
{:error, :not_found} {:error, :not_found}
project -> project ->
now = System.system_time(:second) now = Persistence.now_ms()
Multi.new() Multi.new()
|> Multi.update_all(:clear_previous, from(p in Project, where: p.is_active == true), |> Multi.update_all(:clear_previous, from(p in Project, where: p.is_active == true),

View File

@@ -4,6 +4,7 @@ defmodule BDS.Rendering do
import Ecto.Query import Ecto.Query
alias BDS.Frontmatter alias BDS.Frontmatter
alias BDS.Persistence
alias BDS.Media.Media, as: MediaAsset alias BDS.Media.Media, as: MediaAsset
alias BDS.Menu alias BDS.Menu
alias BDS.Metadata alias BDS.Metadata
@@ -69,7 +70,7 @@ defmodule BDS.Rendering do
template.project_id == ^project_id and template.kind == ^kind and template.project_id == ^project_id and template.kind == ^kind and
template.status == :published and template.status == :published and
template.enabled == true, template.enabled == true,
order_by: [asc: template.created_at, asc: template.slug], order_by: [desc: template.created_at, desc: template.slug],
limit: 1 limit: 1
) )
end end
@@ -455,7 +456,7 @@ defmodule BDS.Rendering do
defp canonical_media_path_by_source_path(project_id) do defp canonical_media_path_by_source_path(project_id) do
Repo.all(from media in MediaAsset, where: media.project_id == ^project_id) Repo.all(from media in MediaAsset, where: media.project_id == ^project_id)
|> Enum.reduce(%{}, fn media, acc -> |> Enum.reduce(%{}, fn media, acc ->
datetime = DateTime.from_unix!(media.created_at) datetime = Persistence.from_unix_ms!(media.created_at)
source_key = source_key =
Path.join([ Path.join([
@@ -476,7 +477,7 @@ defmodule BDS.Rendering do
end end
defp post_path(post, nil) do defp post_path(post, nil) do
datetime = DateTime.from_unix!(post.created_at) datetime = Persistence.from_unix_ms!(post.created_at)
Path.join([ Path.join([
Integer.to_string(datetime.year), Integer.to_string(datetime.year),
@@ -630,7 +631,7 @@ defmodule BDS.Rendering do
grouped_blocks = grouped_blocks =
posts posts
|> Enum.filter(&is_integer(Map.get(&1, :created_at))) |> Enum.filter(&is_integer(Map.get(&1, :created_at)))
|> Enum.group_by(&DateTime.from_unix!(Map.get(&1, :created_at)) |> DateTime.to_date() |> Date.to_iso8601()) |> Enum.group_by(&(Map.get(&1, :created_at) |> Persistence.from_unix_ms!() |> DateTime.to_date() |> Date.to_iso8601()))
|> Enum.sort_by(fn {label, _posts} -> label end) |> Enum.sort_by(fn {label, _posts} -> label end)
grouped_blocks grouped_blocks
@@ -676,12 +677,12 @@ defmodule BDS.Rendering do
defp href_for_language(prefix), do: prefix <> "/" defp href_for_language(prefix), do: prefix <> "/"
defp calendar_initial_year(%{created_at: created_at}) when is_integer(created_at), defp calendar_initial_year(%{created_at: created_at}) when is_integer(created_at),
do: DateTime.from_unix!(created_at).year do: Persistence.from_unix_ms!(created_at).year
defp calendar_initial_year(_post), do: nil defp calendar_initial_year(_post), do: nil
defp calendar_initial_month(%{created_at: created_at}) when is_integer(created_at), defp calendar_initial_month(%{created_at: created_at}) when is_integer(created_at),
do: DateTime.from_unix!(created_at).month do: Persistence.from_unix_ms!(created_at).month
defp calendar_initial_month(_post), do: nil defp calendar_initial_month(_post), do: nil

View File

@@ -4,13 +4,14 @@ defmodule BDS.Scripts do
import Ecto.Query import Ecto.Query
alias BDS.Frontmatter alias BDS.Frontmatter
alias BDS.Persistence
alias BDS.Projects alias BDS.Projects
alias BDS.Repo alias BDS.Repo
alias BDS.Scripts.Script alias BDS.Scripts.Script
alias BDS.Slug alias BDS.Slug
def create_script(attrs) do def create_script(attrs) do
now = System.system_time(:second) now = Persistence.now_ms()
project_id = attr(attrs, :project_id) project_id = attr(attrs, :project_id)
title = attr(attrs, :title) || "" title = attr(attrs, :title) || ""
kind = attr(attrs, :kind) kind = attr(attrs, :kind)
@@ -42,13 +43,11 @@ defmodule BDS.Scripts do
script -> script ->
file_path = script_file_path(script.slug) file_path = script_file_path(script.slug)
full_path = full_file_path(script.project_id, file_path) full_path = full_file_path(script.project_id, file_path)
updated_at = System.system_time(:second) updated_at = Persistence.now_ms()
content = script.content || "" content = script.content || ""
:ok = File.mkdir_p(Path.dirname(full_path))
:ok = :ok =
File.write( Persistence.atomic_write(
full_path, full_path,
serialize_script_file( serialize_script_file(
%{script | status: :published, file_path: file_path, updated_at: updated_at}, %{script | status: :published, file_path: file_path, updated_at: updated_at},
@@ -81,7 +80,7 @@ defmodule BDS.Scripts do
end end
content_changed? = has_attr?(attrs, :content) and attr(attrs, :content) != script.content content_changed? = has_attr?(attrs, :content) and attr(attrs, :content) != script.content
now = System.system_time(:second) now = Persistence.now_ms()
updates = updates =
%{} %{}
@@ -205,7 +204,7 @@ defmodule BDS.Scripts do
contents = File.read!(path) contents = File.read!(path)
{:ok, %{fields: fields}} = Frontmatter.parse_document(contents) {:ok, %{fields: fields}} = Frontmatter.parse_document(contents)
relative_path = Path.relative_to(path, Projects.project_data_dir(project)) relative_path = Path.relative_to(path, Projects.project_data_dir(project))
now = System.system_time(:second) now = Persistence.now_ms()
attrs = %{ attrs = %{
id: Map.get(fields, "id") || Ecto.UUID.generate(), id: Map.get(fields, "id") || Ecto.UUID.generate(),

View File

@@ -5,6 +5,7 @@ defmodule BDS.Search do
alias BDS.Media.Media alias BDS.Media.Media
alias BDS.Media.Translation, as: MediaTranslation alias BDS.Media.Translation, as: MediaTranslation
alias BDS.Persistence
alias BDS.Posts.Post alias BDS.Posts.Post
alias BDS.Projects alias BDS.Projects
alias BDS.Repo alias BDS.Repo
@@ -269,10 +270,10 @@ defmodule BDS.Search do
defp matches_exact?(value, expected), do: value == expected defp matches_exact?(value, expected), do: value == expected
defp matches_year?(_post, nil), do: true defp matches_year?(_post, nil), do: true
defp matches_year?(post, year), do: DateTime.from_unix!(post.created_at).year == year defp matches_year?(post, year), do: Persistence.from_unix_ms!(post.created_at).year == year
defp matches_month?(_post, nil), do: true defp matches_month?(_post, nil), do: true
defp matches_month?(post, month), do: DateTime.from_unix!(post.created_at).month == month defp matches_month?(post, month), do: Persistence.from_unix_ms!(post.created_at).month == month
defp matches_from?(_post, nil), do: true defp matches_from?(_post, nil), do: true
defp matches_from?(post, from_unix), do: post.created_at >= from_unix defp matches_from?(post, from_unix), do: post.created_at >= from_unix
@@ -545,14 +546,14 @@ defmodule BDS.Search do
defp normalize_non_negative_integer(value, default), do: normalize_integer(value) || default defp normalize_non_negative_integer(value, default), do: normalize_integer(value) || default
defp normalize_timestamp(nil, _position), do: nil defp normalize_timestamp(nil, _position), do: nil
defp normalize_timestamp(value, _position) when is_integer(value), do: value defp normalize_timestamp(value, _position) when is_integer(value), do: Persistence.normalize_unix_timestamp(value)
defp normalize_timestamp(value, position) when is_binary(value) do defp normalize_timestamp(value, position) when is_binary(value) do
case Date.from_iso8601(value) do case Date.from_iso8601(value) do
{:ok, date} -> {:ok, date} ->
time = if position == :start, do: ~T[00:00:00], else: ~T[23:59:59] time = if position == :start, do: ~T[00:00:00], else: ~T[23:59:59]
{:ok, datetime} = DateTime.new(date, time, "Etc/UTC") {:ok, datetime} = DateTime.new(date, time, "Etc/UTC")
DateTime.to_unix(datetime) DateTime.to_unix(datetime, :millisecond)
{:error, _reason} -> {:error, _reason} ->
nil nil

View File

@@ -1,6 +1,8 @@
defmodule BDS.Sidecar do defmodule BDS.Sidecar do
@moduledoc false @moduledoc false
alias BDS.Persistence
@list_item_prefix " - " @list_item_prefix " - "
def serialize_document(fields) when is_list(fields) do def serialize_document(fields) when is_list(fields) do
@@ -21,15 +23,26 @@ defmodule BDS.Sidecar do
defp serialize_field({_key, ""}), do: [] defp serialize_field({_key, ""}), do: []
defp serialize_field({key, values}) when is_list(values) do defp serialize_field({key, values}) when is_list(values) do
["#{key}:" | Enum.map(values, &" - #{&1}")] ["#{key}:" | Enum.map(values, &" - #{serialize_scalar(nil, &1)}")]
end end
defp serialize_field({key, value}) when is_boolean(value) do defp serialize_field({key, value}) when is_boolean(value) do
["#{key}: #{if(value, do: "true", else: "false")}"] ["#{key}: #{if(value, do: "true", else: "false")}"]
end end
defp serialize_field({key, value}) when is_integer(value) do
rendered =
if timestamp_key?(key) do
Persistence.timestamp_to_iso8601(value)
else
Integer.to_string(value)
end
["#{key}: #{rendered}"]
end
defp serialize_field({key, value}) do defp serialize_field({key, value}) do
["#{key}: #{value}"] ["#{key}: #{serialize_scalar(key, value)}"]
end end
defp parse_lines([], acc), do: acc defp parse_lines([], acc), do: acc
@@ -46,7 +59,7 @@ defmodule BDS.Sidecar do
String.contains?(line, ": ") -> String.contains?(line, ": ") ->
[key, raw_value] = String.split(line, ": ", parts: 2) [key, raw_value] = String.split(line, ": ", parts: 2)
parse_lines(rest, Map.put(acc, key, parse_scalar(raw_value))) parse_lines(rest, Map.put(acc, key, parse_scalar(key, raw_value)))
true -> true ->
parse_lines(rest, acc) parse_lines(rest, acc)
@@ -55,7 +68,7 @@ defmodule BDS.Sidecar do
defp take_list_items([line | rest], items) do defp take_list_items([line | rest], items) do
if String.starts_with?(line, @list_item_prefix) do if String.starts_with?(line, @list_item_prefix) do
value = line |> String.replace_prefix(@list_item_prefix, "") |> parse_scalar() value = line |> String.replace_prefix(@list_item_prefix, "") |> then(&parse_scalar(nil, &1))
take_list_items(rest, [value | items]) take_list_items(rest, [value | items])
else else
{items, [line | rest]} {items, [line | rest]}
@@ -64,14 +77,73 @@ defmodule BDS.Sidecar do
defp take_list_items([], items), do: {items, []} defp take_list_items([], items), do: {items, []}
defp parse_scalar("true"), do: true defp parse_scalar(key, value) when is_binary(key) and is_binary(value) do
defp parse_scalar("false"), do: false trimmed = String.trim(value)
defp parse_scalar(value) do cond do
timestamp_key?(key) -> Persistence.parse_timestamp(trimmed) || parse_generic_scalar(trimmed)
true -> parse_generic_scalar(trimmed)
end
end
defp parse_scalar(nil, value) when is_binary(value) do
value
|> String.trim()
|> parse_generic_scalar()
end
defp parse_generic_scalar("true"), do: true
defp parse_generic_scalar("false"), do: false
defp parse_generic_scalar(value) do
if Regex.match?(~r/^-?\d+$/, value) do if Regex.match?(~r/^-?\d+$/, value) do
String.to_integer(value) String.to_integer(value)
else else
parse_string(value)
end
end
defp parse_string("\"" <> rest) do
rest
|> String.trim_trailing("\"")
|> String.replace("\\n", "\n")
|> String.replace("\\\"", "\"")
|> String.replace("\\\\", "\\")
end
defp parse_string(value), do: value
defp serialize_scalar(_key, value) when is_boolean(value) do
if(value, do: "true", else: "false")
end
defp serialize_scalar(key, value) when is_integer(value) do
if is_binary(key) and timestamp_key?(key) do
Persistence.timestamp_to_iso8601(value)
else
Integer.to_string(value)
end
end
defp serialize_scalar(_key, value) do
value value
|> to_string()
|> maybe_quote_string()
end
defp maybe_quote_string(value) do
if Regex.match?(~r/^[\p{L}\p{N} ._\/-]+$/u, value) do
value
else
escaped =
value
|> String.replace("\\", "\\\\")
|> String.replace("\"", "\\\"")
|> String.replace("\n", "\\n")
"\"#{escaped}\""
end end
end end
defp timestamp_key?(key), do: String.ends_with?(to_string(key), "_at")
end end

View File

@@ -3,6 +3,7 @@ defmodule BDS.Tags do
import Ecto.Query import Ecto.Query
alias BDS.Persistence
alias BDS.Posts alias BDS.Posts
alias BDS.Posts.Post alias BDS.Posts.Post
alias BDS.Projects alias BDS.Projects
@@ -14,7 +15,7 @@ defmodule BDS.Tags do
name = attr(attrs, :name) |> to_string() |> String.trim() name = attr(attrs, :name) |> to_string() |> String.trim()
with :ok <- validate_unique_name(project_id, name) do with :ok <- validate_unique_name(project_id, name) do
now = System.system_time(:second) now = Persistence.now_ms()
%Tag{} %Tag{}
|> Tag.changeset(%{ |> Tag.changeset(%{
@@ -70,7 +71,7 @@ defmodule BDS.Tags do
|> elem(1) |> elem(1)
|> Enum.reverse() |> Enum.reverse()
now = System.system_time(:second) now = Persistence.now_ms()
Enum.each(missing_names, fn name -> Enum.each(missing_names, fn name ->
%Tag{} %Tag{}
@@ -102,7 +103,7 @@ defmodule BDS.Tags do
updates = %{ updates = %{
color: attr(attrs, :color), color: attr(attrs, :color),
post_template_slug: attr(attrs, :post_template_slug), post_template_slug: attr(attrs, :post_template_slug),
updated_at: System.system_time(:second) updated_at: Persistence.now_ms()
} }
tag tag
@@ -164,7 +165,7 @@ defmodule BDS.Tags do
updated_tag = updated_tag =
tag tag
|> Tag.changeset(%{name: normalized_name, updated_at: System.system_time(:second)}) |> Tag.changeset(%{name: normalized_name, updated_at: Persistence.now_ms()})
|> Repo.update!() |> Repo.update!()
write_tags_json(tag.project_id) write_tags_json(tag.project_id)
@@ -226,7 +227,7 @@ defmodule BDS.Tags do
end) end)
} }
File.write!(path, Jason.encode!(payload)) :ok = Persistence.atomic_write(path, Jason.encode!(payload))
end end
defp validate_unique_name(project_id, name) do defp validate_unique_name(project_id, name) do
@@ -311,7 +312,7 @@ defmodule BDS.Tags do
defp update_post_tags(post, updated_tags) do defp update_post_tags(post, updated_tags) do
post post
|> Post.changeset(%{tags: updated_tags, updated_at: System.system_time(:second)}) |> Post.changeset(%{tags: updated_tags, updated_at: Persistence.now_ms()})
|> Repo.update!() |> Repo.update!()
Posts.rewrite_published_post(post.id) Posts.rewrite_published_post(post.id)

View File

@@ -4,6 +4,7 @@ defmodule BDS.Templates do
import Ecto.Query import Ecto.Query
alias BDS.Frontmatter alias BDS.Frontmatter
alias BDS.Persistence
alias BDS.Posts alias BDS.Posts
alias BDS.Projects alias BDS.Projects
alias BDS.Repo alias BDS.Repo
@@ -12,7 +13,7 @@ defmodule BDS.Templates do
alias BDS.Templates.Template alias BDS.Templates.Template
def create_template(attrs) do def create_template(attrs) do
now = System.system_time(:second) now = Persistence.now_ms()
project_id = attr(attrs, :project_id) project_id = attr(attrs, :project_id)
title = attr(attrs, :title) || "" title = attr(attrs, :title) || ""
@@ -42,13 +43,11 @@ defmodule BDS.Templates do
template -> template ->
file_path = template_file_path(template.slug) file_path = template_file_path(template.slug)
full_path = full_file_path(template.project_id, file_path) full_path = full_file_path(template.project_id, file_path)
updated_at = System.system_time(:second) updated_at = Persistence.now_ms()
content = template.content || "" content = template.content || ""
:ok = File.mkdir_p(Path.dirname(full_path))
:ok = :ok =
File.write( Persistence.atomic_write(
full_path, full_path,
serialize_template_file( serialize_template_file(
%{template | status: :published, file_path: file_path, updated_at: updated_at}, %{template | status: :published, file_path: file_path, updated_at: updated_at},
@@ -89,7 +88,7 @@ defmodule BDS.Templates do
has_attr?(attrs, :content) and attr(attrs, :content) != template.content has_attr?(attrs, :content) and attr(attrs, :content) != template.content
slug_changed? = next_slug != template.slug slug_changed? = next_slug != template.slug
now = System.system_time(:second) now = Persistence.now_ms()
next_status = next_status =
if(template.status == :published and content_changed?, if(template.status == :published and content_changed?,
@@ -254,7 +253,7 @@ defmodule BDS.Templates do
end end
defp clear_template_references(template) do defp clear_template_references(template) do
now = System.system_time(:second) now = Persistence.now_ms()
affected_posts = affected_posts =
Repo.all( Repo.all(
@@ -316,8 +315,7 @@ defmodule BDS.Templates do
defp rewrite_template_file(original_template, updated_template) do defp rewrite_template_file(original_template, updated_template) do
body = published_template_body(original_template) body = published_template_body(original_template)
new_full_path = full_file_path(updated_template.project_id, updated_template.file_path) new_full_path = full_file_path(updated_template.project_id, updated_template.file_path)
:ok = File.mkdir_p(Path.dirname(new_full_path)) :ok = Persistence.atomic_write(new_full_path, serialize_template_file(updated_template, body))
:ok = File.write(new_full_path, serialize_template_file(updated_template, body))
if original_template.file_path != updated_template.file_path do if original_template.file_path != updated_template.file_path do
_ = delete_file_if_present(original_template.project_id, original_template.file_path) _ = delete_file_if_present(original_template.project_id, original_template.file_path)
@@ -345,7 +343,7 @@ defmodule BDS.Templates do
contents = File.read!(path) contents = File.read!(path)
{:ok, %{fields: fields}} = Frontmatter.parse_document(contents) {:ok, %{fields: fields}} = Frontmatter.parse_document(contents)
relative_path = Path.relative_to(path, Projects.project_data_dir(project)) relative_path = Path.relative_to(path, Projects.project_data_dir(project))
now = System.system_time(:second) now = Persistence.now_ms()
attrs = %{ attrs = %{
id: Map.get(fields, "id") || Ecto.UUID.generate(), id: Map.get(fields, "id") || Ecto.UUID.generate(),

View File

@@ -51,6 +51,9 @@ defmodule BDS.MediaTest do
assert sidecar =~ "author: Writer\n" assert sidecar =~ "author: Writer\n"
assert sidecar =~ "language: en\n" assert sidecar =~ "language: en\n"
assert sidecar =~ "tags:\n - alpha\n" assert sidecar =~ "tags:\n - alpha\n"
assert sidecar =~ ~r/created_at: \d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z\n/
assert sidecar =~ ~r/updated_at: \d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z\n/
refute File.exists?(Path.join(temp_dir, media.sidecar_path <> ".tmp"))
end end
test "update_media rewrites the sidecar metadata", %{project: project, temp_dir: temp_dir} do test "update_media rewrites the sidecar metadata", %{project: project, temp_dir: temp_dir} do
@@ -137,8 +140,8 @@ defmodule BDS.MediaTest do
"caption: Recovered caption", "caption: Recovered caption",
"author: Writer", "author: Writer",
"language: en", "language: en",
"created_at: 1711843200", "created_at: 2024-03-30T21:20:00.000Z",
"updated_at: 1711929600", "updated_at: 2024-03-31T21:20:00.000Z",
"tags:", "tags:",
" - alpha", " - alpha",
"" ""
@@ -175,6 +178,8 @@ defmodule BDS.MediaTest do
assert media.author == "Writer" assert media.author == "Writer"
assert media.language == "en" assert media.language == "en"
assert media.tags == ["alpha"] assert media.tags == ["alpha"]
assert media.created_at == 1_711_833_600_000
assert media.updated_at == 1_711_920_000_000
assert media.file_path == "media/2026/04/asset.jpg" assert media.file_path == "media/2026/04/asset.jpg"
assert media.sidecar_path == "media/2026/04/asset.jpg.meta" assert media.sidecar_path == "media/2026/04/asset.jpg.meta"

View File

@@ -138,4 +138,26 @@ defmodule BDS.MetadataTest do
assert BDS.Repo.get_by(BDS.Embeddings.Key, project_id: project.id, post_id: post.id) != nil assert BDS.Repo.get_by(BDS.Embeddings.Key, project_id: project.id, post_id: post.id) != nil
assert File.exists?(BDS.Embeddings.index_path(project.id)) assert File.exists?(BDS.Embeddings.index_path(project.id))
end end
test "sync_project_metadata_from_filesystem materializes the canonical metadata files when missing",
%{project: project, temp_dir: temp_dir} do
meta_dir = Path.join(temp_dir, "meta")
refute File.exists?(Path.join(meta_dir, "project.json"))
refute File.exists?(Path.join(meta_dir, "categories.json"))
refute File.exists?(Path.join(meta_dir, "category-meta.json"))
refute File.exists?(Path.join(meta_dir, "publishing.json"))
assert {:ok, metadata} = BDS.Metadata.sync_project_metadata_from_filesystem(project.id)
assert metadata.name == project.name
assert File.exists?(Path.join(meta_dir, "project.json"))
assert File.exists?(Path.join(meta_dir, "categories.json"))
assert File.exists?(Path.join(meta_dir, "category-meta.json"))
assert File.exists?(Path.join(meta_dir, "publishing.json"))
refute File.exists?(Path.join(meta_dir, "project.json.tmp"))
refute File.exists?(Path.join(meta_dir, "categories.json.tmp"))
refute File.exists?(Path.join(meta_dir, "category-meta.json.tmp"))
refute File.exists?(Path.join(meta_dir, "publishing.json.tmp"))
end
end end

View File

@@ -74,7 +74,7 @@ defmodule BDS.PostLinksTest do
end end
defp canonical_post_href(post) do defp canonical_post_href(post) do
datetime = DateTime.from_unix!(post.created_at) datetime = DateTime.from_unix!(post.created_at, :millisecond)
Path.join([ Path.join([
"", "",

View File

@@ -151,7 +151,12 @@ defmodule BDS.PostsTest do
assert file_contents =~ "template_slug: article\n" assert file_contents =~ "template_slug: article\n"
assert file_contents =~ "tags:\n - alpha\n" assert file_contents =~ "tags:\n - alpha\n"
assert file_contents =~ "categories:\n - notes\n" assert file_contents =~ "categories:\n - notes\n"
assert file_contents =~ ~r/created_at: \d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z\n/
assert file_contents =~ ~r/updated_at: \d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z\n/
assert file_contents =~ ~r/published_at: \d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z\n/
assert file_contents =~ "\n---\nHello from markdown\n" assert file_contents =~ "\n---\nHello from markdown\n"
refute File.exists?(full_path <> ".tmp")
end end
test "delete_post removes the database row and published markdown file when present" do test "delete_post removes the database row and published markdown file when present" do
@@ -271,9 +276,9 @@ defmodule BDS.PostsTest do
"language: en", "language: en",
"do_not_translate: true", "do_not_translate: true",
"template_slug: article", "template_slug: article",
"created_at: 1711843200", "created_at: 2024-03-30T21:20:00.000Z",
"updated_at: 1711929600", "updated_at: 2024-03-31T21:20:00.000Z",
"published_at: 1712016000", "published_at: 2024-04-01T21:20:00.000Z",
"tags:", "tags:",
" - alpha", " - alpha",
"categories:", "categories:",
@@ -299,9 +304,9 @@ defmodule BDS.PostsTest do
assert post.language == "en" assert post.language == "en"
assert post.do_not_translate == true assert post.do_not_translate == true
assert post.template_slug == "article" assert post.template_slug == "article"
assert post.created_at == 1_711_843_200 assert post.created_at == 1_711_833_600_000
assert post.updated_at == 1_711_929_600 assert post.updated_at == 1_711_920_000_000
assert post.published_at == 1_712_016_000 assert post.published_at == 1_712_006_400_000
assert post.tags == ["alpha"] assert post.tags == ["alpha"]
assert post.categories == ["notes"] assert post.categories == ["notes"]
assert post.file_path == "posts/2026/04/recovered-post.md" assert post.file_path == "posts/2026/04/recovered-post.md"

View File

@@ -312,7 +312,7 @@ defmodule BDS.RenderingTest do
end end
defp canonical_post_href(post) do defp canonical_post_href(post) do
datetime = DateTime.from_unix!(post.created_at) datetime = DateTime.from_unix!(post.created_at, :millisecond)
"/#{datetime.year}/#{String.pad_leading(Integer.to_string(datetime.month), 2, "0")}/#{String.pad_leading(Integer.to_string(datetime.day), 2, "0")}/#{post.slug}/" "/#{datetime.year}/#{String.pad_leading(Integer.to_string(datetime.month), 2, "0")}/#{String.pad_leading(Integer.to_string(datetime.day), 2, "0")}/#{post.slug}/"
end end

View File

@@ -71,9 +71,10 @@ defmodule BDS.ScriptsTest do
assert contents =~ "entrypoint: main\n" assert contents =~ "entrypoint: main\n"
assert contents =~ "enabled: true\n" assert contents =~ "enabled: true\n"
assert contents =~ "version: 1\n" assert contents =~ "version: 1\n"
assert contents =~ "created_at: #{published.created_at}\n" assert contents =~ ~r/created_at: \d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z\n/
assert contents =~ "updated_at: #{published.updated_at}\n" assert contents =~ ~r/updated_at: \d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z\n/
assert contents =~ "\n---\nfunction main() return 'ok' end\n" assert contents =~ "\n---\nfunction main() return 'ok' end\n"
refute File.exists?(full_path <> ".tmp")
end end
test "update_script bumps version and reopens a published script when content changes", %{ test "update_script bumps version and reopens a published script when content changes", %{
@@ -144,8 +145,8 @@ defmodule BDS.ScriptsTest do
"entrypoint: main", "entrypoint: main",
"enabled: true", "enabled: true",
"version: 4", "version: 4",
"created_at: 301", "created_at: 1970-01-01T00:00:00.301Z",
"updated_at: 404", "updated_at: 1970-01-01T00:00:00.404Z",
"---", "---",
"function main() return 'restored' end", "function main() return 'restored' end",
"" ""

View File

@@ -71,9 +71,10 @@ defmodule BDS.TemplatesTest do
assert contents =~ "kind: list\n" assert contents =~ "kind: list\n"
assert contents =~ "enabled: true\n" assert contents =~ "enabled: true\n"
assert contents =~ "version: 1\n" assert contents =~ "version: 1\n"
assert contents =~ "created_at: #{published.created_at}\n" assert contents =~ ~r/created_at: \d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z\n/
assert contents =~ "updated_at: #{published.updated_at}\n" assert contents =~ ~r/updated_at: \d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z\n/
assert contents =~ "\n---\n<section>{{ page_title }}</section>\n" assert contents =~ "\n---\n<section>{{ page_title }}</section>\n"
refute File.exists?(full_path <> ".tmp")
end end
test "update_template bumps version and reopens a published template when content changes", %{ test "update_template bumps version and reopens a published template when content changes", %{
@@ -235,8 +236,8 @@ defmodule BDS.TemplatesTest do
"kind: list", "kind: list",
"enabled: true", "enabled: true",
"version: 3", "version: 3",
"created_at: 101", "created_at: 1970-01-01T00:00:00.101Z",
"updated_at: 202", "updated_at: 1970-01-01T00:00:00.202Z",
"---", "---",
"<section>Recovered</section>", "<section>Recovered</section>",
"" ""