chore: posts.ex also refactored
Co-authored-by: Copilot <copilot@github.com>
This commit is contained in:
320
lib/bds/posts/rebuild_from_files.ex
Normal file
320
lib/bds/posts/rebuild_from_files.ex
Normal file
@@ -0,0 +1,320 @@
|
||||
defmodule BDS.Posts.RebuildFromFiles do
|
||||
@moduledoc false
|
||||
|
||||
alias BDS.DocumentFields
|
||||
alias BDS.Embeddings
|
||||
alias BDS.Frontmatter
|
||||
alias BDS.Persistence
|
||||
alias BDS.Posts.Post
|
||||
alias BDS.Posts.Slugs
|
||||
alias BDS.Posts.Translation
|
||||
alias BDS.Posts.TranslationValidation
|
||||
alias BDS.Projects
|
||||
alias BDS.Rebuild
|
||||
alias BDS.Repo
|
||||
alias BDS.Search
|
||||
|
||||
@spec rebuild_posts_from_files(String.t(), keyword()) :: {:ok, [Post.t()]}
|
||||
def rebuild_posts_from_files(project_id, opts \\ []) do
|
||||
project = Projects.get_project!(project_id)
|
||||
on_progress = progress_callback(opts)
|
||||
|
||||
rebuild_files =
|
||||
project
|
||||
|> Projects.project_data_dir()
|
||||
|> Path.join("posts")
|
||||
|> TranslationValidation.list_matching_files("*.md")
|
||||
|> Rebuild.parallel_map(&parse_rebuild_file(project, &1))
|
||||
|
||||
total_files = length(rebuild_files)
|
||||
:ok = report_rebuild_started(on_progress, total_files, "post files")
|
||||
|
||||
{translation_files, post_files} =
|
||||
Enum.split_with(rebuild_files, &TranslationValidation.translation_rebuild_file?/1)
|
||||
|
||||
posts =
|
||||
post_files
|
||||
|> Enum.with_index(1)
|
||||
|> Enum.map(fn {file, index} ->
|
||||
post =
|
||||
upsert_post_from_rebuild_file(project_id, file,
|
||||
sync_search: false,
|
||||
sync_embeddings: false
|
||||
)
|
||||
|
||||
:ok = report_rebuild_progress(on_progress, index, total_files, "post files")
|
||||
post
|
||||
end)
|
||||
|
||||
translation_files
|
||||
|> Enum.with_index(length(post_files) + 1)
|
||||
|> Enum.each(fn {file, index} ->
|
||||
upsert_post_translation_from_rebuild_file(project_id, file, sync_search: false)
|
||||
:ok = report_rebuild_progress(on_progress, index, total_files, "post files")
|
||||
end)
|
||||
|
||||
if Keyword.get(opts, :reindex_search, true) do
|
||||
:ok = report_rebuild_phase(on_progress, 0.97, "Refreshing post search index")
|
||||
|
||||
:ok =
|
||||
Search.reindex_posts(project_id,
|
||||
on_progress: scaled_progress_reporter(on_progress, 0.97, 0.99)
|
||||
)
|
||||
end
|
||||
|
||||
if Keyword.get(opts, :rebuild_embeddings, true) do
|
||||
:ok = report_rebuild_phase(on_progress, 0.99, "Refreshing post embeddings")
|
||||
|
||||
{:ok, _rebuilt_post_ids} =
|
||||
Embeddings.rebuild_project(project_id,
|
||||
on_progress: scaled_progress_reporter(on_progress, 0.99, 1.0)
|
||||
)
|
||||
end
|
||||
|
||||
{:ok, posts}
|
||||
end
|
||||
|
||||
@spec import_orphan_post_file(String.t(), String.t()) ::
|
||||
{:ok, Post.t()} | {:error, :not_found | :unsupported_file}
|
||||
def import_orphan_post_file(project_id, relative_path) do
|
||||
project = Projects.get_project!(project_id)
|
||||
full_path = Path.join(Projects.project_data_dir(project), relative_path)
|
||||
|
||||
if File.exists?(full_path) do
|
||||
rebuild_file = parse_rebuild_file(project, full_path)
|
||||
|
||||
if TranslationValidation.translation_rebuild_file?(rebuild_file) do
|
||||
{:error, :unsupported_file}
|
||||
else
|
||||
fields =
|
||||
rebuild_file.fields
|
||||
|> Map.put("id", unique_post_id(Map.get(rebuild_file.fields, "id")))
|
||||
|> Map.put(
|
||||
"slug",
|
||||
Slugs.unique_for_import(project_id, Map.fetch!(rebuild_file.fields, "slug"))
|
||||
)
|
||||
|
||||
{:ok, upsert_post_from_rebuild_file(project_id, %{rebuild_file | fields: fields})}
|
||||
end
|
||||
else
|
||||
{:error, :not_found}
|
||||
end
|
||||
end
|
||||
|
||||
@spec import_orphan_post_translation_file(String.t(), String.t()) ::
|
||||
{:ok, Translation.t()} | {:error, :not_found | :unsupported_file | :conflict}
|
||||
def import_orphan_post_translation_file(project_id, relative_path) do
|
||||
project = Projects.get_project!(project_id)
|
||||
full_path = Path.join(Projects.project_data_dir(project), relative_path)
|
||||
|
||||
if File.exists?(full_path) do
|
||||
rebuild_file = parse_rebuild_file(project, full_path)
|
||||
|
||||
if TranslationValidation.translation_rebuild_file?(rebuild_file) do
|
||||
source_post_id = Map.fetch!(rebuild_file.fields, "translationFor")
|
||||
language = TranslationValidation.normalize_language(Map.fetch!(rebuild_file.fields, "language"))
|
||||
|
||||
case Repo.get(Post, source_post_id) do
|
||||
nil ->
|
||||
{:error, :not_found}
|
||||
|
||||
%Post{} = post ->
|
||||
if TranslationValidation.normalize_language(post.language) == language or
|
||||
Repo.get_by(Translation, translation_for: source_post_id, language: language) do
|
||||
{:error, :conflict}
|
||||
else
|
||||
fields = Map.put(rebuild_file.fields, "id", Ecto.UUID.generate())
|
||||
|
||||
{:ok,
|
||||
upsert_post_translation_from_rebuild_file(
|
||||
project_id,
|
||||
%{rebuild_file | fields: fields},
|
||||
sync_search: true
|
||||
)}
|
||||
end
|
||||
end
|
||||
else
|
||||
{:error, :unsupported_file}
|
||||
end
|
||||
else
|
||||
{:error, :not_found}
|
||||
end
|
||||
end
|
||||
|
||||
@doc false
|
||||
def upsert_post_from_file(project_id, project, path) do
|
||||
rebuild_file = parse_rebuild_file(project, path)
|
||||
upsert_post_from_rebuild_file(project_id, rebuild_file)
|
||||
end
|
||||
|
||||
@doc false
|
||||
def upsert_post_from_rebuild_file(project_id, rebuild_file, opts \\ []) do
|
||||
fields = rebuild_file.fields
|
||||
now = Persistence.now_ms()
|
||||
|
||||
attrs = %{
|
||||
id: DocumentFields.get(fields, "id") || Ecto.UUID.generate(),
|
||||
project_id: project_id,
|
||||
title: DocumentFields.get(fields, "title") || "",
|
||||
slug: DocumentFields.fetch!(fields, "slug"),
|
||||
excerpt: Map.get(fields, "excerpt"),
|
||||
content: nil,
|
||||
status: parse_post_status(DocumentFields.get(fields, "status", "published")),
|
||||
author: Map.get(fields, "author"),
|
||||
created_at: DocumentFields.get(fields, "createdAt", now),
|
||||
updated_at: DocumentFields.get(fields, "updatedAt", now),
|
||||
published_at: DocumentFields.get(fields, "publishedAt"),
|
||||
file_path: rebuild_file.relative_path,
|
||||
checksum: nil,
|
||||
tags: Map.get(fields, "tags", []),
|
||||
categories: Map.get(fields, "categories", []),
|
||||
template_slug: DocumentFields.get(fields, "templateSlug"),
|
||||
language: Map.get(fields, "language"),
|
||||
do_not_translate: DocumentFields.get(fields, "doNotTranslate", false),
|
||||
published_title: nil,
|
||||
published_content: nil,
|
||||
published_tags: nil,
|
||||
published_categories: nil,
|
||||
published_excerpt: nil
|
||||
}
|
||||
|
||||
post =
|
||||
Repo.get(Post, attrs.id) ||
|
||||
Repo.get_by(Post, project_id: project_id, file_path: rebuild_file.relative_path) ||
|
||||
Repo.get_by(Post, project_id: project_id, slug: attrs.slug) || %Post{}
|
||||
|
||||
post =
|
||||
post
|
||||
|> Post.changeset(attrs)
|
||||
|> Repo.insert_or_update!()
|
||||
|
||||
if Keyword.get(opts, :sync_search, true) do
|
||||
:ok = Search.sync_post(post)
|
||||
end
|
||||
|
||||
if Keyword.get(opts, :sync_embeddings, true) do
|
||||
:ok = Embeddings.sync_post(post)
|
||||
end
|
||||
|
||||
post
|
||||
end
|
||||
|
||||
@doc false
|
||||
def upsert_post_translation_from_rebuild_file(project_id, rebuild_file, opts) do
|
||||
fields = rebuild_file.fields
|
||||
source_post_id = DocumentFields.fetch!(fields, "translationFor")
|
||||
source_post = Repo.get_by!(Post, project_id: project_id, id: source_post_id)
|
||||
now = Persistence.now_ms()
|
||||
language = TranslationValidation.normalize_language(DocumentFields.fetch!(fields, "language"))
|
||||
|
||||
translation =
|
||||
Repo.get_by(Translation, translation_for: source_post_id, language: language) ||
|
||||
%Translation{}
|
||||
|
||||
attrs = %{
|
||||
id: DocumentFields.get(fields, "id") || Ecto.UUID.generate(),
|
||||
project_id: project_id,
|
||||
translation_for: source_post_id,
|
||||
language: language,
|
||||
title: DocumentFields.get(fields, "title") || "",
|
||||
excerpt: Map.get(fields, "excerpt"),
|
||||
content: nil,
|
||||
status: parse_translation_status(DocumentFields.get(fields, "status", "published")),
|
||||
created_at: DocumentFields.get(fields, "createdAt", source_post.created_at || now),
|
||||
updated_at:
|
||||
DocumentFields.get(
|
||||
fields,
|
||||
"updatedAt",
|
||||
source_post.updated_at || source_post.created_at || now
|
||||
),
|
||||
published_at: DocumentFields.get(fields, "publishedAt", source_post.published_at),
|
||||
file_path: rebuild_file.relative_path,
|
||||
checksum: nil
|
||||
}
|
||||
|
||||
translation
|
||||
|> Translation.changeset(attrs)
|
||||
|> Repo.insert_or_update!()
|
||||
|> tap(fn _translation ->
|
||||
if Keyword.get(opts, :sync_search, true) do
|
||||
:ok = Search.sync_post(source_post_id)
|
||||
end
|
||||
end)
|
||||
end
|
||||
|
||||
@doc false
|
||||
def parse_rebuild_file(project, path) do
|
||||
contents = File.read!(path)
|
||||
{:ok, %{fields: fields}} = Frontmatter.parse_document(contents)
|
||||
|
||||
%{
|
||||
path: path,
|
||||
relative_path: Path.relative_to(path, Projects.project_data_dir(project)),
|
||||
fields: fields
|
||||
}
|
||||
end
|
||||
|
||||
@doc false
|
||||
def parse_post_status(status) when is_atom(status), do: status
|
||||
def parse_post_status(status), do: String.to_existing_atom(status)
|
||||
|
||||
@doc false
|
||||
def parse_translation_status(status) when is_atom(status), do: status
|
||||
def parse_translation_status(status), do: String.to_existing_atom(status)
|
||||
|
||||
@doc false
|
||||
def progress_callback(opts) do
|
||||
case Keyword.get(opts, :on_progress) do
|
||||
callback when is_function(callback, 2) -> callback
|
||||
_other -> nil
|
||||
end
|
||||
end
|
||||
|
||||
@doc false
|
||||
def report_rebuild_started(nil, _total, _label), do: :ok
|
||||
|
||||
def report_rebuild_started(callback, 0, label) do
|
||||
callback.(1.0, "No #{label} found")
|
||||
:ok
|
||||
end
|
||||
|
||||
def report_rebuild_started(callback, total, label) do
|
||||
callback.(0.05, "Rebuilding #{label} (0/#{total})")
|
||||
:ok
|
||||
end
|
||||
|
||||
@doc false
|
||||
def report_rebuild_progress(nil, _current, _total, _label), do: :ok
|
||||
def report_rebuild_progress(_callback, _current, 0, _label), do: :ok
|
||||
|
||||
def report_rebuild_progress(callback, current, total, label) do
|
||||
callback.(0.05 + 0.95 * (current / total), "Rebuilding #{label} (#{current}/#{total})")
|
||||
:ok
|
||||
end
|
||||
|
||||
defp scaled_progress_reporter(nil, _start_value, _end_value), do: nil
|
||||
|
||||
defp scaled_progress_reporter(report, start_value, end_value) when is_function(report, 2) do
|
||||
fn value, message ->
|
||||
scaled_value = start_value + (end_value - start_value) * value
|
||||
report.(scaled_value, message)
|
||||
end
|
||||
end
|
||||
|
||||
defp report_rebuild_phase(nil, _progress, _message), do: :ok
|
||||
|
||||
defp report_rebuild_phase(callback, progress, message) do
|
||||
callback.(progress, message)
|
||||
:ok
|
||||
end
|
||||
|
||||
defp unique_post_id(nil), do: Ecto.UUID.generate()
|
||||
|
||||
defp unique_post_id(id) do
|
||||
if Repo.get(Post, id) || Repo.get(Translation, id) do
|
||||
Ecto.UUID.generate()
|
||||
else
|
||||
id
|
||||
end
|
||||
end
|
||||
end
|
||||
Reference in New Issue
Block a user