hugo/bDS2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: metadata, frontmatter, write atomicity should now be in

Browse Source
This commit is contained in:
Chili Palmer
2026-04-24 10:16:21 +02:00
parent 1d15de60de
commit a3f2c4a5f7
25 changed files with 479 additions and 136 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
lib/bds/embeddings.ex
View File

@@ -3,6 +3,7 @@ defmodule BDS.Embeddings do
import Ecto.Query
alias BDS.Persistence
alias BDS.Embeddings.DismissedDuplicatePair
alias BDS.Embeddings.Index
alias BDS.Embeddings.Key
@@ -317,7 +318,7 @@ defmodule BDS.Embeddings do
project_id: post_a.project_id,
post_id_a: sorted_a,
post_id_b: sorted_b,
dismissed_at: System.system_time(:second)
dismissed_at: Persistence.now_ms()
})
|> Repo.insert_or_update!()

8
lib/bds/embeddings/index.ex
View File

@@ -3,6 +3,7 @@ defmodule BDS.Embeddings.Index do
import Ecto.Query
alias BDS.Persistence
alias BDS.Embeddings.Key
alias BDS.Projects
alias BDS.Repo
@@ -43,7 +44,7 @@ defmodule BDS.Embeddings.Index do
"project_id" => project_id,
"model_id" => model_id,
"dimensions" => dimensions,
"updated_at" => System.system_time(:second),
"updated_at" => Persistence.now_ms(),
"entries" => entries
}
@@ -123,10 +124,7 @@ defmodule BDS.Embeddings.Index do
end
defp write_snapshot(snapshot_path, payload) do
:ok = File.mkdir_p(Path.dirname(snapshot_path))
temp_path = snapshot_path <> ".tmp"
:ok = File.write(temp_path, Jason.encode!(payload))
:ok = File.rename(temp_path, snapshot_path)
:ok = Persistence.atomic_write(snapshot_path, Jason.encode!(payload))
legacy_path = legacy_path(snapshot_path)
if File.exists?(legacy_path) do

99
lib/bds/frontmatter.ex
View File

@@ -1,6 +1,8 @@
defmodule BDS.Frontmatter do
@moduledoc false
alias BDS.Persistence
@list_item_prefix " - "
def serialize_document(fields, body) when is_list(fields) do
@@ -38,11 +40,26 @@ defmodule BDS.Frontmatter do
end
defp serialize_field({key, values}) when is_list(values) do
["#{key}:" | Enum.map(values, &" - #{&1}")]
["#{key}:" | Enum.map(values, &" - #{serialize_scalar(nil, &1)}")]
end
defp serialize_field({key, value}) when is_atom(value) do
["#{key}: #{Atom.to_string(value)}"]
end
defp serialize_field({key, value}) when is_integer(value) do
rendered =
if timestamp_key?(key) do
Persistence.timestamp_to_iso8601(value)
else
Integer.to_string(value)
end
["#{key}: #{rendered}"]
end
defp serialize_field({key, value}) do
["#{key}: #{value}"]
["#{key}: #{serialize_scalar(key, value)}"]
end
defp parse_frontmatter(frontmatter) do
@@ -65,7 +82,7 @@ defmodule BDS.Frontmatter do
String.contains?(line, ": ") ->
[key, raw_value] = String.split(line, ": ", parts: 2)
parse_lines(rest, Map.put(acc, key, parse_scalar(raw_value)))
parse_lines(rest, Map.put(acc, key, parse_scalar(key, raw_value)))
true ->
parse_lines(rest, acc)
@@ -74,7 +91,7 @@ defmodule BDS.Frontmatter do
defp take_list_items([line | rest], items) do
if String.starts_with?(line, @list_item_prefix) do
value = line |> String.replace_prefix(@list_item_prefix, "") |> parse_scalar()
value = line |> String.replace_prefix(@list_item_prefix, "") |> then(&parse_scalar(nil, &1))
take_list_items(rest, [value | items])
else
{items, [line | rest]}
@@ -83,14 +100,80 @@ defmodule BDS.Frontmatter do
defp take_list_items([], items), do: {items, []}
defp parse_scalar("true"), do: true
defp parse_scalar("false"), do: false
defp parse_scalar(key, value) when is_binary(key) and is_binary(value) do
trimmed = String.trim(value)
defp parse_scalar(value) do
cond do
timestamp_key?(key) ->
Persistence.parse_timestamp(trimmed) || parse_generic_scalar(trimmed)
true ->
parse_generic_scalar(trimmed)
end
end
defp parse_scalar(nil, value) when is_binary(value) do
value
|> String.trim()
|> parse_generic_scalar()
end
defp parse_generic_scalar("true"), do: true
defp parse_generic_scalar("false"), do: false
defp parse_generic_scalar(value) do
if Regex.match?(~r/^-?\d+$/, value) do
String.to_integer(value)
else
value
parse_string(value)
end
end
defp parse_string("\"" <> rest) do
rest
|> String.trim_trailing("\"")
|> String.replace("\\n", "\n")
|> String.replace("\\\"", "\"")
|> String.replace("\\\\", "\\")
end
defp parse_string(value), do: value
defp serialize_scalar(_key, value) when is_boolean(value) do
if(value, do: "true", else: "false")
end
defp serialize_scalar(_key, value) when is_atom(value) do
Atom.to_string(value)
end
defp serialize_scalar(key, value) when is_integer(value) do
if is_binary(key) and timestamp_key?(key) do
Persistence.timestamp_to_iso8601(value)
else
Integer.to_string(value)
end
end
defp serialize_scalar(_key, value) do
value
|> to_string()
|> maybe_quote_string()
end
defp maybe_quote_string(value) do
if Regex.match?(~r/^[\p{L}\p{N} ._\/-]+$/u, value) do
value
else
escaped =
value
|> String.replace("\\", "\\\\")
|> String.replace("\"", "\\\"")
|> String.replace("\n", "\\n")
"\"#{escaped}\""
end
end
defp timestamp_key?(key), do: String.ends_with?(to_string(key), "_at")
end

14
lib/bds/generation.ex
View File

@@ -5,6 +5,7 @@ defmodule BDS.Generation do
alias BDS.Generation.GeneratedFileHash
alias BDS.Metadata
alias BDS.Persistence
alias BDS.Posts.Post
alias BDS.Posts.Translation
alias BDS.Projects
@@ -52,7 +53,7 @@ defmodule BDS.Generation do
def post_output_path(%Post{} = post), do: post_output_path(post, nil)
def post_output_path(%Post{} = post, language) do
datetime = DateTime.from_unix!(post.created_at)
datetime = Persistence.from_unix_ms!(post.created_at)
year = Integer.to_string(datetime.year)
month = datetime.month |> Integer.to_string() |> String.pad_leading(2, "0")
day = datetime.day |> Integer.to_string() |> String.pad_leading(2, "0")
@@ -70,7 +71,7 @@ defmodule BDS.Generation do
when is_binary(project_id) and is_binary(relative_path) and is_binary(content) do
project = Projects.get_project!(project_id)
content_hash = sha256(content)
now = System.system_time(:second)
now = Persistence.now_ms()
case Repo.get_by(GeneratedFileHash, project_id: project_id, relative_path: relative_path) do
%GeneratedFileHash{content_hash: ^content_hash} ->
@@ -78,8 +79,7 @@ defmodule BDS.Generation do
_existing ->
full_path = output_path(project, relative_path)
:ok = File.mkdir_p(Path.dirname(full_path))
:ok = File.write(full_path, content)
:ok = Persistence.atomic_write(full_path, content)
attrs = %{
project_id: project_id,
@@ -495,7 +495,7 @@ defmodule BDS.Generation do
defp render_calendar(published_posts) do
published_posts
|> Enum.map(fn post ->
datetime = DateTime.from_unix!(post.created_at)
datetime = Persistence.from_unix_ms!(post.created_at)
%{date: Date.to_iso8601(DateTime.to_date(datetime)), slug: post.slug, title: post.title}
end)
|> Jason.encode!()
@@ -637,13 +637,13 @@ defmodule BDS.Generation do
defp year_key(created_at) do
created_at
|> DateTime.from_unix!()
|> Persistence.from_unix_ms!()
|> Map.fetch!(:year)
|> Integer.to_string()
end
defp month_key(created_at) do
datetime = DateTime.from_unix!(created_at)
datetime = Persistence.from_unix_ms!(created_at)
{Integer.to_string(datetime.year),
Integer.to_string(datetime.month) |> String.pad_leading(2, "0")}

17
lib/bds/media.ex
View File

@@ -5,6 +5,7 @@ defmodule BDS.Media do
alias BDS.Media.Media
alias BDS.Media.Translation
alias BDS.Persistence
alias BDS.Projects
alias BDS.Repo
alias BDS.Search
@@ -16,7 +17,7 @@ defmodule BDS.Media do
original_name = Path.basename(source_path)
mime_type = detect_mime(original_name)
{width, height} = image_dimensions(source_path, mime_type)
now = System.system_time(:second)
now = Persistence.now_ms()
file_name = Ecto.UUID.generate() <> Path.extname(original_name)
file_path = media_file_path(file_name, now)
sidecar_path = file_path <> ".meta"
@@ -78,7 +79,7 @@ defmodule BDS.Media do
|> maybe_put(:tags, attr(attrs, :tags))
|> maybe_put(:width, attr(attrs, :width))
|> maybe_put(:height, attr(attrs, :height))
|> Map.put(:updated_at, System.system_time(:second))
|> Map.put(:updated_at, Persistence.now_ms())
project = Projects.get_project!(media.project_id)
@@ -136,7 +137,7 @@ defmodule BDS.Media do
media ->
project = Projects.get_project!(media.project_id)
now = System.system_time(:second)
now = Persistence.now_ms()
translation =
Repo.get_by(Translation, translation_for: media.id, language: language) ||
@@ -227,7 +228,7 @@ defmodule BDS.Media do
relative_sidecar_path = Path.relative_to(sidecar_path, Projects.project_data_dir(project))
relative_file_path = String.trim_trailing(relative_sidecar_path, ".meta")
filename = Path.basename(relative_file_path)
now = System.system_time(:second)
now = Persistence.now_ms()
attrs = %{
id: Map.get(fields, "id") || Ecto.UUID.generate(),
@@ -317,7 +318,7 @@ defmodule BDS.Media do
media ->
{:ok, fields} = sidecar_path |> File.read!() |> Sidecar.parse_document()
now = System.system_time(:second)
now = Persistence.now_ms()
language = Map.fetch!(fields, "language")
translation =
@@ -408,7 +409,7 @@ defmodule BDS.Media do
end
defp media_file_path(file_name, timestamp) do
datetime = DateTime.from_unix!(timestamp)
datetime = Persistence.from_unix_ms!(timestamp)
year = Integer.to_string(datetime.year)
month = datetime.month |> Integer.to_string() |> String.pad_leading(2, "0")
Path.join(["media", year, month, file_name])
@@ -487,9 +488,7 @@ defmodule BDS.Media do
end
defp atomic_write(path, contents) do
temp_path = path <> ".tmp"
:ok = File.write(temp_path, contents)
File.rename(temp_path, path)
Persistence.atomic_write(path, contents)
end
defp blank_to_nil(nil), do: nil

18
lib/bds/menu.ex
View File

@@ -3,6 +3,7 @@ defmodule BDS.Menu do
require Record
alias BDS.Persistence
alias BDS.Projects
Record.defrecord(:xmlElement, Record.extract(:xmlElement, from_lib: "xmerl/include/xmerl.hrl"))
@@ -44,14 +45,8 @@ defmodule BDS.Menu do
end
defp write_menu_file(project, menu) do
meta_dir = Path.dirname(menu_path(project))
:ok = File.mkdir_p(meta_dir)
path = menu_path(project)
temp_path = path <> ".tmp"
:ok = File.write(temp_path, serialize_opml(menu.items))
File.rename(temp_path, path)
:ok = Persistence.atomic_write(path, serialize_opml(project, menu.items))
end
defp menu_path(project) do
@@ -84,7 +79,9 @@ defmodule BDS.Menu do
end
end
defp serialize_opml(items) do
defp serialize_opml(project, items) do
timestamp = project.updated_at || project.created_at || Persistence.now_ms()
rendered_items =
items
|> Enum.map(&render_item(&1, 2))
@@ -93,6 +90,11 @@ defmodule BDS.Menu do
[
~s(<?xml version="1.0" encoding="UTF-8"?>),
~s(<opml version="2.0">),
~s( <head>),
~s( <title>#{xml_escape(project.name)}</title>),
~s( <dateCreated>#{Persistence.timestamp_to_iso8601(timestamp)}</dateCreated>),
~s( <dateModified>#{Persistence.timestamp_to_iso8601(timestamp)}</dateModified>),
~s( </head>),
~s( <body>),
rendered_items,
~s( </body>),

99
lib/bds/metadata.ex
View File

@@ -2,6 +2,8 @@ defmodule BDS.Metadata do
@moduledoc false
alias BDS.Embeddings
alias BDS.I18n
alias BDS.Persistence
alias BDS.Projects
alias BDS.Projects.Project
alias BDS.Repo
@@ -9,6 +11,30 @@ defmodule BDS.Metadata do
@default_max_posts_per_page 50
@default_categories ["article", "aside", "page", "picture"]
@min_posts_per_page 1
@max_posts_per_page 500
@supported_pico_themes MapSet.new([
"default",
"amber",
"blue",
"cyan",
"fuchsia",
"green",
"grey",
"indigo",
"jade",
"lime",
"orange",
"pink",
"pumpkin",
"purple",
"red",
"sand",
"slate",
"violet",
"yellow",
"zinc"
])
def get_project_metadata(project_id) do
project = Projects.get_project!(project_id)
@@ -18,7 +44,7 @@ defmodule BDS.Metadata do
def update_project_metadata(project_id, attrs) do
project = Projects.get_project!(project_id)
state = load_state(project)
now = System.system_time(:second)
now = Persistence.now_ms()
project_metadata =
state
@@ -104,7 +130,7 @@ defmodule BDS.Metadata do
def sync_project_metadata_from_filesystem(project_id) do
project = Projects.get_project!(project_id)
now = System.system_time(:second)
now = Persistence.now_ms()
project_metadata_from_files =
read_json(project, "project.json") ||
@@ -130,6 +156,10 @@ defmodule BDS.Metadata do
persist_setting(project_id, "categories", categories_from_files, now)
persist_setting(project_id, "category_meta", category_meta_from_files, now)
persist_setting(project_id, "publishing", publishing_from_files, now)
write_project_json(updated_project, project_metadata_from_files)
write_categories_json(updated_project, categories_from_files["categories"] || @default_categories)
write_category_meta_json(updated_project, category_meta_from_files["categories"] || %{})
write_publishing_json(updated_project, publishing_from_files)
load_state(updated_project)
end)
|> unwrap_transaction()
@@ -138,7 +168,7 @@ defmodule BDS.Metadata do
defp update_state(project_id, updater) do
project = Projects.get_project!(project_id)
state = load_state(project)
now = System.system_time(:second)
now = Persistence.now_ms()
Repo.transaction(fn ->
updater.(project, state, now)
@@ -200,13 +230,13 @@ defmodule BDS.Metadata do
name: attr(attrs, :name) || project.name,
description: attr(attrs, :description),
public_url: attr(attrs, :public_url),
main_language: attr(attrs, :main_language),
main_language: normalize_optional_language(attr(attrs, :main_language)),
default_author: attr(attrs, :default_author),
max_posts_per_page: attr(attrs, :max_posts_per_page) || @default_max_posts_per_page,
max_posts_per_page: normalize_posts_per_page(attr(attrs, :max_posts_per_page)),
blogmark_category: attr(attrs, :blogmark_category),
pico_theme: attr(attrs, :pico_theme),
pico_theme: normalize_pico_theme(attr(attrs, :pico_theme)),
semantic_similarity_enabled: attr(attrs, :semantic_similarity_enabled) || false,
blog_languages: normalize_string_list(attr(attrs, :blog_languages) || [])
blog_languages: normalize_language_list(attr(attrs, :blog_languages) || [])
}
end
@@ -227,7 +257,7 @@ defmodule BDS.Metadata do
"ssh_host" => attr(prefs, :ssh_host),
"ssh_user" => attr(prefs, :ssh_user),
"ssh_remote_path" => attr(prefs, :ssh_remote_path),
"ssh_mode" => attr(prefs, :ssh_mode) || "scp"
"ssh_mode" => normalize_ssh_mode(attr(prefs, :ssh_mode))
}
end
@@ -270,11 +300,8 @@ defmodule BDS.Metadata do
defp write_json(project, file_name, payload) do
meta_dir = Path.join(Projects.project_data_dir(project), "meta")
:ok = File.mkdir_p(meta_dir)
path = Path.join(meta_dir, file_name)
temp_path = path <> ".tmp"
:ok = File.write(temp_path, Jason.encode!(payload))
File.rename(temp_path, path)
:ok = Persistence.atomic_write(path, Jason.encode!(payload))
end
defp read_json(project, file_name) do
@@ -304,12 +331,56 @@ defmodule BDS.Metadata do
defp setting_key(project_id, suffix), do: "project:#{project_id}:#{suffix}"
defp normalize_string_list(values) do
defp normalize_posts_per_page(nil), do: @default_max_posts_per_page
defp normalize_posts_per_page(value) when is_integer(value) do
value
|> max(@min_posts_per_page)
|> min(@max_posts_per_page)
end
defp normalize_posts_per_page(value) when is_binary(value) do
case Integer.parse(String.trim(value)) do
{integer, ""} -> normalize_posts_per_page(integer)
_ -> @default_max_posts_per_page
end
end
defp normalize_posts_per_page(_value), do: @default_max_posts_per_page
defp normalize_optional_language(nil), do: nil
defp normalize_optional_language(""), do: nil
defp normalize_optional_language(value) do
normalized = value |> to_string() |> String.trim() |> String.downcase()
supported_language_codes = Enum.map(I18n.supported_languages(), & &1.code)
if normalized in supported_language_codes do
normalized
else
nil
end
end
defp normalize_language_list(values) do
values
|> Enum.reject(&(&1 in [nil, ""]))
|> Enum.map(&normalize_optional_language/1)
|> Enum.reject(&is_nil/1)
|> Enum.uniq()
end
defp normalize_pico_theme(nil), do: nil
defp normalize_pico_theme(""), do: nil
defp normalize_pico_theme(value) do
normalized = value |> to_string() |> String.trim()
if MapSet.member?(@supported_pico_themes, normalized), do: normalized, else: nil
end
defp normalize_ssh_mode(:rsync), do: "rsync"
defp normalize_ssh_mode("rsync"), do: "rsync"
defp normalize_ssh_mode(_mode), do: "scp"
defp unwrap_transaction({:ok, result}), do: {:ok, result}
defp unwrap_transaction({:error, reason}), do: {:error, reason}

85
lib/bds/persistence.ex Normal file
View File

@@ -0,0 +1,85 @@
defmodule BDS.Persistence do
@moduledoc false
def now_ms, do: System.system_time(:millisecond)
def normalize_unix_timestamp(nil), do: nil
def normalize_unix_timestamp(value) when is_integer(value) do
if abs(value) < 100_000_000_000 do
value * 1000
else
value
end
end
def normalize_unix_timestamp(value) when is_binary(value) do
value
|> String.trim()
|> case do
"" -> nil
trimmed ->
case Integer.parse(trimmed) do
{integer, ""} -> normalize_unix_timestamp(integer)
_ -> nil
end
end
end
def normalize_unix_timestamp(_value), do: nil
def from_unix_ms!(value) when is_integer(value) do
value
|> normalize_unix_timestamp()
|> DateTime.from_unix!(:millisecond)
end
def timestamp_to_iso8601(nil), do: nil
def timestamp_to_iso8601(value) when is_integer(value) do
value
|> from_unix_ms!()
|> DateTime.to_iso8601()
end
def parse_timestamp(nil), do: nil
def parse_timestamp(value) when is_integer(value), do: normalize_unix_timestamp(value)
def parse_timestamp(value) when is_binary(value) do
trimmed = String.trim(value)
cond do
trimmed == "" ->
nil
Regex.match?(~r/^-?\d+$/, trimmed) ->
normalize_unix_timestamp(trimmed)
true ->
case DateTime.from_iso8601(trimmed) do
{:ok, datetime, _offset} -> DateTime.to_unix(datetime, :millisecond)
_ -> nil
end
end
end
def parse_timestamp(_value), do: nil
def atomic_write(path, contents) when is_binary(path) and is_binary(contents) do
:ok = File.mkdir_p(Path.dirname(path))
temp_path = path <> ".tmp"
with :ok <- File.write(temp_path, contents),
:ok <- File.rename(temp_path, path) do
:ok
else
{:error, _reason} = error ->
_ = File.rm(temp_path)
error
error ->
_ = File.rm(temp_path)
error
end
end
end

3
lib/bds/post_links.ex
View File

@@ -3,6 +3,7 @@ defmodule BDS.PostLinks do
import Ecto.Query
alias BDS.Persistence
alias BDS.Posts.Link
alias BDS.Posts.Post
alias BDS.Projects
@@ -22,7 +23,7 @@ defmodule BDS.PostLinks do
Repo.transaction(fn ->
Repo.delete_all(from link in Link, where: link.source_post_id == ^post.id)
now = System.system_time(:second)
now = Persistence.now_ms()
Enum.each(links, fn %{target_post_id: target_post_id, link_text: link_text} ->
%Link{}

37
lib/bds/posts.ex
View File

@@ -6,6 +6,7 @@ defmodule BDS.Posts do
alias BDS.Frontmatter
alias BDS.Embeddings
alias BDS.Metadata
alias BDS.Persistence
alias BDS.PostLinks
alias BDS.Posts.Post
alias BDS.Posts.Translation
@@ -15,7 +16,7 @@ defmodule BDS.Posts do
alias BDS.Slug
def create_post(attrs) do
now = System.system_time(:second)
now = Persistence.now_ms()
project_id = attr(attrs, :project_id)
title = normalize_title(attr(attrs, :title))
base_slug = title |> default_slug_source() |> Slug.slugify()
@@ -65,7 +66,7 @@ defmodule BDS.Posts do
post ->
with :ok <- validate_slug_change(post, attrs) do
now = System.system_time(:second)
now = Persistence.now_ms()
updates =
attrs
@@ -99,16 +100,14 @@ defmodule BDS.Posts do
%Post{} = post ->
project = Projects.get_project!(post.project_id)
published_at = post.published_at || System.system_time(:second)
published_at = post.published_at || Persistence.now_ms()
relative_path = build_post_relative_path(post.slug, post.created_at)
full_path = Path.join(Projects.project_data_dir(project), relative_path)
updated_at = System.system_time(:second)
updated_at = Persistence.now_ms()
body = publishable_post_body(post, full_path, project)
:ok = File.mkdir_p(Path.dirname(full_path))
:ok =
File.write(
Persistence.atomic_write(
full_path,
serialize_post_file(%{post | updated_at: updated_at, content: body}, published_at)
)
@@ -171,7 +170,7 @@ defmodule BDS.Posts do
%Post{status: status} = post when status in [:draft, :published] ->
post
|> Post.changeset(%{status: :archived, updated_at: System.system_time(:second)})
|> Post.changeset(%{status: :archived, updated_at: Persistence.now_ms()})
|> Repo.update()
|> case do
{:ok, updated_post} ->
@@ -218,7 +217,7 @@ defmodule BDS.Posts do
)}
%Post{} = post ->
now = System.system_time(:second)
now = Persistence.now_ms()
normalized_language = normalize_language(language)
translation =
@@ -322,14 +321,12 @@ defmodule BDS.Posts do
project = Projects.get_project!(post.project_id)
full_path = Path.join(Projects.project_data_dir(project), post.file_path)
body = published_post_body(post, full_path)
:ok = File.mkdir_p(Path.dirname(full_path))
:ok =
File.write(
Persistence.atomic_write(
full_path,
serialize_post_file(
%{post | content: body},
post.published_at || System.system_time(:second)
post.published_at || Persistence.now_ms()
)
)
end
@@ -451,7 +448,7 @@ defmodule BDS.Posts do
defp default_slug_source(title), do: title
defp build_post_relative_path(slug, created_at) do
datetime = DateTime.from_unix!(created_at)
datetime = Persistence.from_unix_ms!(created_at)
year = Integer.to_string(datetime.year)
month = datetime.month |> Integer.to_string() |> String.pad_leading(2, "0")
Path.join(["posts", year, month, "#{slug}.md"])
@@ -513,7 +510,7 @@ defmodule BDS.Posts do
contents = File.read!(path)
{:ok, %{fields: fields}} = Frontmatter.parse_document(contents)
relative_path = Path.relative_to(path, Projects.project_data_dir(project))
now = System.system_time(:second)
now = Persistence.now_ms()
attrs = %{
id: Map.get(fields, "id") || Ecto.UUID.generate(),
@@ -626,16 +623,14 @@ defmodule BDS.Posts do
defp publish_translation(%Post{} = post, %Translation{} = translation) do
project = Projects.get_project!(post.project_id)
published_at = translation.published_at || System.system_time(:second)
published_at = translation.published_at || Persistence.now_ms()
relative_path = build_translation_relative_path(post, translation.language)
full_path = Path.join(Projects.project_data_dir(project), relative_path)
updated_at = System.system_time(:second)
updated_at = Persistence.now_ms()
body = publishable_translation_body(translation, full_path)
:ok = File.mkdir_p(Path.dirname(full_path))
:ok =
File.write(
Persistence.atomic_write(
full_path,
serialize_translation_file(
%{translation | updated_at: updated_at, content: body},
@@ -657,7 +652,7 @@ defmodule BDS.Posts do
end
defp build_translation_relative_path(post, language) do
datetime = DateTime.from_unix!(post.created_at)
datetime = Persistence.from_unix_ms!(post.created_at)
year = Integer.to_string(datetime.year)
month = datetime.month |> Integer.to_string() |> String.pad_leading(2, "0")
Path.join(["posts", year, month, "#{post.slug}.#{language}.md"])

7
lib/bds/projects.ex
View File

@@ -4,6 +4,7 @@ defmodule BDS.Projects do
import Ecto.Query
alias Ecto.Multi
alias BDS.Persistence
alias BDS.Projects.Project
alias BDS.Repo
alias BDS.StarterTemplates
@@ -26,7 +27,7 @@ defmodule BDS.Projects do
{:ok, project}
nil ->
now = System.system_time(:second)
now = Persistence.now_ms()
is_active = not Repo.exists?(from project in Project, where: project.is_active == true)
Repo.transaction(fn ->
@@ -60,7 +61,7 @@ defmodule BDS.Projects do
end
def create_project(attrs) do
now = System.system_time(:second)
now = Persistence.now_ms()
name = attr(attrs, :name) || ""
slug = unique_slug(attr(attrs, :slug) || Slug.slugify(name))
@@ -95,7 +96,7 @@ defmodule BDS.Projects do
{:error, :not_found}
project ->
now = System.system_time(:second)
now = Persistence.now_ms()
Multi.new()
|> Multi.update_all(:clear_previous, from(p in Project, where: p.is_active == true),

13
lib/bds/rendering.ex
View File

@@ -4,6 +4,7 @@ defmodule BDS.Rendering do
import Ecto.Query
alias BDS.Frontmatter
alias BDS.Persistence
alias BDS.Media.Media, as: MediaAsset
alias BDS.Menu
alias BDS.Metadata
@@ -69,7 +70,7 @@ defmodule BDS.Rendering do
template.project_id == ^project_id and template.kind == ^kind and
template.status == :published and
template.enabled == true,
order_by: [asc: template.created_at, asc: template.slug],
order_by: [desc: template.created_at, desc: template.slug],
limit: 1
)
end
@@ -455,7 +456,7 @@ defmodule BDS.Rendering do
defp canonical_media_path_by_source_path(project_id) do
Repo.all(from media in MediaAsset, where: media.project_id == ^project_id)
|> Enum.reduce(%{}, fn media, acc ->
datetime = DateTime.from_unix!(media.created_at)
datetime = Persistence.from_unix_ms!(media.created_at)
source_key =
Path.join([
@@ -476,7 +477,7 @@ defmodule BDS.Rendering do
end
defp post_path(post, nil) do
datetime = DateTime.from_unix!(post.created_at)
datetime = Persistence.from_unix_ms!(post.created_at)
Path.join([
Integer.to_string(datetime.year),
@@ -630,7 +631,7 @@ defmodule BDS.Rendering do
grouped_blocks =
posts
|> Enum.filter(&is_integer(Map.get(&1, :created_at)))
|> Enum.group_by(&DateTime.from_unix!(Map.get(&1, :created_at)) |> DateTime.to_date() |> Date.to_iso8601())
|> Enum.group_by(&(Map.get(&1, :created_at) |> Persistence.from_unix_ms!() |> DateTime.to_date() |> Date.to_iso8601()))
|> Enum.sort_by(fn {label, _posts} -> label end)
grouped_blocks
@@ -676,12 +677,12 @@ defmodule BDS.Rendering do
defp href_for_language(prefix), do: prefix <> "/"
defp calendar_initial_year(%{created_at: created_at}) when is_integer(created_at),
do: DateTime.from_unix!(created_at).year
do: Persistence.from_unix_ms!(created_at).year
defp calendar_initial_year(_post), do: nil
defp calendar_initial_month(%{created_at: created_at}) when is_integer(created_at),
do: DateTime.from_unix!(created_at).month
do: Persistence.from_unix_ms!(created_at).month
defp calendar_initial_month(_post), do: nil

13
lib/bds/scripts.ex
View File

@@ -4,13 +4,14 @@ defmodule BDS.Scripts do
import Ecto.Query
alias BDS.Frontmatter
alias BDS.Persistence
alias BDS.Projects
alias BDS.Repo
alias BDS.Scripts.Script
alias BDS.Slug
def create_script(attrs) do
now = System.system_time(:second)
now = Persistence.now_ms()
project_id = attr(attrs, :project_id)
title = attr(attrs, :title) || ""
kind = attr(attrs, :kind)
@@ -42,13 +43,11 @@ defmodule BDS.Scripts do
script ->
file_path = script_file_path(script.slug)
full_path = full_file_path(script.project_id, file_path)
updated_at = System.system_time(:second)
updated_at = Persistence.now_ms()
content = script.content || ""
:ok = File.mkdir_p(Path.dirname(full_path))
:ok =
File.write(
Persistence.atomic_write(
full_path,
serialize_script_file(
%{script | status: :published, file_path: file_path, updated_at: updated_at},
@@ -81,7 +80,7 @@ defmodule BDS.Scripts do
end
content_changed? = has_attr?(attrs, :content) and attr(attrs, :content) != script.content
now = System.system_time(:second)
now = Persistence.now_ms()
updates =
%{}
@@ -205,7 +204,7 @@ defmodule BDS.Scripts do
contents = File.read!(path)
{:ok, %{fields: fields}} = Frontmatter.parse_document(contents)
relative_path = Path.relative_to(path, Projects.project_data_dir(project))
now = System.system_time(:second)
now = Persistence.now_ms()
attrs = %{
id: Map.get(fields, "id") || Ecto.UUID.generate(),

9
lib/bds/search.ex
View File

@@ -5,6 +5,7 @@ defmodule BDS.Search do
alias BDS.Media.Media
alias BDS.Media.Translation, as: MediaTranslation
alias BDS.Persistence
alias BDS.Posts.Post
alias BDS.Projects
alias BDS.Repo
@@ -269,10 +270,10 @@ defmodule BDS.Search do
defp matches_exact?(value, expected), do: value == expected
defp matches_year?(_post, nil), do: true
defp matches_year?(post, year), do: DateTime.from_unix!(post.created_at).year == year
defp matches_year?(post, year), do: Persistence.from_unix_ms!(post.created_at).year == year
defp matches_month?(_post, nil), do: true
defp matches_month?(post, month), do: DateTime.from_unix!(post.created_at).month == month
defp matches_month?(post, month), do: Persistence.from_unix_ms!(post.created_at).month == month
defp matches_from?(_post, nil), do: true
defp matches_from?(post, from_unix), do: post.created_at >= from_unix
@@ -545,14 +546,14 @@ defmodule BDS.Search do
defp normalize_non_negative_integer(value, default), do: normalize_integer(value) || default
defp normalize_timestamp(nil, _position), do: nil
defp normalize_timestamp(value, _position) when is_integer(value), do: value
defp normalize_timestamp(value, _position) when is_integer(value), do: Persistence.normalize_unix_timestamp(value)
defp normalize_timestamp(value, position) when is_binary(value) do
case Date.from_iso8601(value) do
{:ok, date} ->
time = if position == :start, do: ~T[00:00:00], else: ~T[23:59:59]
{:ok, datetime} = DateTime.new(date, time, "Etc/UTC")
DateTime.to_unix(datetime)
DateTime.to_unix(datetime, :millisecond)
{:error, _reason} ->
nil

88
lib/bds/sidecar.ex
View File

@@ -1,6 +1,8 @@
defmodule BDS.Sidecar do
@moduledoc false
alias BDS.Persistence
@list_item_prefix " - "
def serialize_document(fields) when is_list(fields) do
@@ -21,15 +23,26 @@ defmodule BDS.Sidecar do
defp serialize_field({_key, ""}), do: []
defp serialize_field({key, values}) when is_list(values) do
["#{key}:" | Enum.map(values, &" - #{&1}")]
["#{key}:" | Enum.map(values, &" - #{serialize_scalar(nil, &1)}")]
end
defp serialize_field({key, value}) when is_boolean(value) do
["#{key}: #{if(value, do: "true", else: "false")}"]
end
defp serialize_field({key, value}) when is_integer(value) do
rendered =
if timestamp_key?(key) do
Persistence.timestamp_to_iso8601(value)
else
Integer.to_string(value)
end
["#{key}: #{rendered}"]
end
defp serialize_field({key, value}) do
["#{key}: #{value}"]
["#{key}: #{serialize_scalar(key, value)}"]
end
defp parse_lines([], acc), do: acc
@@ -46,7 +59,7 @@ defmodule BDS.Sidecar do
String.contains?(line, ": ") ->
[key, raw_value] = String.split(line, ": ", parts: 2)
parse_lines(rest, Map.put(acc, key, parse_scalar(raw_value)))
parse_lines(rest, Map.put(acc, key, parse_scalar(key, raw_value)))
true ->
parse_lines(rest, acc)
@@ -55,7 +68,7 @@ defmodule BDS.Sidecar do
defp take_list_items([line | rest], items) do
if String.starts_with?(line, @list_item_prefix) do
value = line |> String.replace_prefix(@list_item_prefix, "") |> parse_scalar()
value = line |> String.replace_prefix(@list_item_prefix, "") |> then(&parse_scalar(nil, &1))
take_list_items(rest, [value | items])
else
{items, [line | rest]}
@@ -64,14 +77,73 @@ defmodule BDS.Sidecar do
defp take_list_items([], items), do: {items, []}
defp parse_scalar("true"), do: true
defp parse_scalar("false"), do: false
defp parse_scalar(key, value) when is_binary(key) and is_binary(value) do
trimmed = String.trim(value)
defp parse_scalar(value) do
cond do
timestamp_key?(key) -> Persistence.parse_timestamp(trimmed) || parse_generic_scalar(trimmed)
true -> parse_generic_scalar(trimmed)
end
end
defp parse_scalar(nil, value) when is_binary(value) do
value
|> String.trim()
|> parse_generic_scalar()
end
defp parse_generic_scalar("true"), do: true
defp parse_generic_scalar("false"), do: false
defp parse_generic_scalar(value) do
if Regex.match?(~r/^-?\d+$/, value) do
String.to_integer(value)
else
value
parse_string(value)
end
end
defp parse_string("\"" <> rest) do
rest
|> String.trim_trailing("\"")
|> String.replace("\\n", "\n")
|> String.replace("\\\"", "\"")
|> String.replace("\\\\", "\\")
end
defp parse_string(value), do: value
defp serialize_scalar(_key, value) when is_boolean(value) do
if(value, do: "true", else: "false")
end
defp serialize_scalar(key, value) when is_integer(value) do
if is_binary(key) and timestamp_key?(key) do
Persistence.timestamp_to_iso8601(value)
else
Integer.to_string(value)
end
end
defp serialize_scalar(_key, value) do
value
|> to_string()
|> maybe_quote_string()
end
defp maybe_quote_string(value) do
if Regex.match?(~r/^[\p{L}\p{N} ._\/-]+$/u, value) do
value
else
escaped =
value
|> String.replace("\\", "\\\\")
|> String.replace("\"", "\\\"")
|> String.replace("\n", "\\n")
"\"#{escaped}\""
end
end
defp timestamp_key?(key), do: String.ends_with?(to_string(key), "_at")
end

13
lib/bds/tags.ex
View File

@@ -3,6 +3,7 @@ defmodule BDS.Tags do
import Ecto.Query
alias BDS.Persistence
alias BDS.Posts
alias BDS.Posts.Post
alias BDS.Projects
@@ -14,7 +15,7 @@ defmodule BDS.Tags do
name = attr(attrs, :name) |> to_string() |> String.trim()
with :ok <- validate_unique_name(project_id, name) do
now = System.system_time(:second)
now = Persistence.now_ms()
%Tag{}
|> Tag.changeset(%{
@@ -70,7 +71,7 @@ defmodule BDS.Tags do
|> elem(1)
|> Enum.reverse()
now = System.system_time(:second)
now = Persistence.now_ms()
Enum.each(missing_names, fn name ->
%Tag{}
@@ -102,7 +103,7 @@ defmodule BDS.Tags do
updates = %{
color: attr(attrs, :color),
post_template_slug: attr(attrs, :post_template_slug),
updated_at: System.system_time(:second)
updated_at: Persistence.now_ms()
}
tag
@@ -164,7 +165,7 @@ defmodule BDS.Tags do
updated_tag =
tag
|> Tag.changeset(%{name: normalized_name, updated_at: System.system_time(:second)})
|> Tag.changeset(%{name: normalized_name, updated_at: Persistence.now_ms()})
|> Repo.update!()
write_tags_json(tag.project_id)
@@ -226,7 +227,7 @@ defmodule BDS.Tags do
end)
}
File.write!(path, Jason.encode!(payload))
:ok = Persistence.atomic_write(path, Jason.encode!(payload))
end
defp validate_unique_name(project_id, name) do
@@ -311,7 +312,7 @@ defmodule BDS.Tags do
defp update_post_tags(post, updated_tags) do
post
|> Post.changeset(%{tags: updated_tags, updated_at: System.system_time(:second)})
|> Post.changeset(%{tags: updated_tags, updated_at: Persistence.now_ms()})
|> Repo.update!()
Posts.rewrite_published_post(post.id)

18
lib/bds/templates.ex
View File

@@ -4,6 +4,7 @@ defmodule BDS.Templates do
import Ecto.Query
alias BDS.Frontmatter
alias BDS.Persistence
alias BDS.Posts
alias BDS.Projects
alias BDS.Repo
@@ -12,7 +13,7 @@ defmodule BDS.Templates do
alias BDS.Templates.Template
def create_template(attrs) do
now = System.system_time(:second)
now = Persistence.now_ms()
project_id = attr(attrs, :project_id)
title = attr(attrs, :title) || ""
@@ -42,13 +43,11 @@ defmodule BDS.Templates do
template ->
file_path = template_file_path(template.slug)
full_path = full_file_path(template.project_id, file_path)
updated_at = System.system_time(:second)
updated_at = Persistence.now_ms()
content = template.content || ""
:ok = File.mkdir_p(Path.dirname(full_path))
:ok =
File.write(
Persistence.atomic_write(
full_path,
serialize_template_file(
%{template | status: :published, file_path: file_path, updated_at: updated_at},
@@ -89,7 +88,7 @@ defmodule BDS.Templates do
has_attr?(attrs, :content) and attr(attrs, :content) != template.content
slug_changed? = next_slug != template.slug
now = System.system_time(:second)
now = Persistence.now_ms()
next_status =
if(template.status == :published and content_changed?,
@@ -254,7 +253,7 @@ defmodule BDS.Templates do
end
defp clear_template_references(template) do
now = System.system_time(:second)
now = Persistence.now_ms()
affected_posts =
Repo.all(
@@ -316,8 +315,7 @@ defmodule BDS.Templates do
defp rewrite_template_file(original_template, updated_template) do
body = published_template_body(original_template)
new_full_path = full_file_path(updated_template.project_id, updated_template.file_path)
:ok = File.mkdir_p(Path.dirname(new_full_path))
:ok = File.write(new_full_path, serialize_template_file(updated_template, body))
:ok = Persistence.atomic_write(new_full_path, serialize_template_file(updated_template, body))
if original_template.file_path != updated_template.file_path do
_ = delete_file_if_present(original_template.project_id, original_template.file_path)
@@ -345,7 +343,7 @@ defmodule BDS.Templates do
contents = File.read!(path)
{:ok, %{fields: fields}} = Frontmatter.parse_document(contents)
relative_path = Path.relative_to(path, Projects.project_data_dir(project))
now = System.system_time(:second)
now = Persistence.now_ms()
attrs = %{
id: Map.get(fields, "id") || Ecto.UUID.generate(),