Compare commits
2 Commits
9691d931b3
...
753f742b99
| Author | SHA1 | Date | |
|---|---|---|---|
| 753f742b99 | |||
| 96402bb4f3 |
14
CODESMELL.md
14
CODESMELL.md
@@ -2,7 +2,7 @@
|
||||
|
||||
Living document. Each section lists **status**, then **open work in priority order**, then a brief notes block. Completed work is listed compactly at the bottom (`## Changelog`).
|
||||
|
||||
Last refreshed: 2026-05-01.
|
||||
Last refreshed: 2026-05-02.
|
||||
|
||||
---
|
||||
|
||||
@@ -14,8 +14,6 @@ Last refreshed: 2026-05-01.
|
||||
|
||||
| # | Module | Current lines | Target | Strategy |
|
||||
|---|---|---|---|---|
|
||||
| 1 | `BDS.Maintenance` | 810 | ≤ 250 | Extract `DiffReports` (~240), `DiffComputation` (~160), `Repair` (~160), `FileScan` (~140), `Progress` (~60). Coordinator keeps the 4 public entrypoints. |
|
||||
| 2 | `BDS.Media` | 993 | ≤ 250 | Extract `Thumbnails` (~140), `Sidecars` (~150), `FileOps` (~180), `Rebuild` (~130), `Linking` (~80). Main keeps CRUD + translation API. |
|
||||
| 3 | `BDS.Desktop.ShellLive.ImportEditor` | 1436 | ≤ 600 | Extract `ConflictResolution` (~150), `TaxonomyEditing` (~120), `AnalysisState` (~150), `ProgressTracking` (~120). Components stay in main file. |
|
||||
| 4 | `BDS.Rendering` | 838 | ≤ 200 | Extract `TemplateSelection` (~120), `PostRendering` (~180), `ListArchive` (~150), `Metadata` (~140), `LinksAndLanguages` (~100). Main keeps the 3 public renders. |
|
||||
| 5 | `BDS.Desktop.ShellLive.MenuEditor` | 871 | ≤ 350 | Extract `TreeOps` (~280), `TreePredicates` (~60), `DraftManagement` (~140), `PageCategory` (~120), `State` (~80). |
|
||||
@@ -33,6 +31,8 @@ Last refreshed: 2026-05-01.
|
||||
- `BDS.Scripting.Capabilities` 1715 → 194 (89 %)
|
||||
- `BDS.Posts` 1781 → 569 (68 %)
|
||||
- `BDS.Desktop.ShellLive` 2607 → 1545 (41 %)
|
||||
- `BDS.Maintenance` 810 → 141 (83 %)
|
||||
- `BDS.Media` 993 → 324 (67 %)
|
||||
|
||||
---
|
||||
|
||||
@@ -166,9 +166,17 @@ Most tests share the SQLite repo and named GenServers (`BDS.Tasks`, `BDS.Search`
|
||||
|
||||
## Changelog
|
||||
|
||||
### 2026-05-02
|
||||
|
||||
### 2026-05-02
|
||||
|
||||
- **God modules**:
|
||||
- `BDS.Media` 993 → 324 (67 %). Submodules under `lib/bds/media/`: `FileOps` (150, attr/maybe_put/blank_to_nil/atomic_write/delete_file_if_present/list_matching_files/media_file_path/detect_mime/image_dimensions/image_mime?/progress callbacks), `Thumbnails` (165, thumbnail_paths/regenerate_thumbnails/regenerate_missing_thumbnails/ensure_thumbnails/delete_thumbnail_files + private render/write helpers), `Sidecars` (329, write_sidecar/write_translation_sidecar/parse_canonical_sidecar/parse_translation_sidecar/upsert_media_from_sidecar/upsert_translation_from_sidecar + sync/import-orphan public API + translation_sidecar_path/canonical_sidecar?/translation_sidecar?/binary_path_for_translation_sidecar/binary_exists_for_sidecar?), `Linking` (125, list_linked_posts/link_media_to_post/unlink_media_from_post/linked_post_ids), `Rebuilder` (82, rebuild_media_from_files/2). Public API preserved via `defdelegate`; coordinator keeps import_media/update_media/delete_media/upsert_media_translation/delete_media_translation/replace_media_file/list_media_translations and uses `import only:` for shared helpers.
|
||||
|
||||
### 2026-05-01
|
||||
|
||||
- **God modules**:
|
||||
- `BDS.Maintenance` 810 → 141 (83 %). Submodules under `lib/bds/maintenance/`: `Progress` (45), `FileScan` (158), `DiffComputation` (93), `DiffReports` (315), `Repair` (145). Coordinator keeps the 4 public entrypoints (`repair_metadata_diff/4`, `import_metadata_diff_orphans/3`, `rebuild_from_filesystem/3`, `metadata_diff/2`); submodules wired via `import only:`.
|
||||
- `BDS.Scripting.Capabilities` 1715 → 194 (89 %). Submodules: `Util` (301), `Posts` (270), `Media` (254), `Crud` (284), `Projects` (204), `AppShell` (134), `Bridges` (176). Public `for_project/2` preserved.
|
||||
- Fixed real race in `test/bds/desktop/shell_live_test.exs:1149` (metadata-diff editor open) — was diagnosed as flake but was a missing `completed_task!(task.id)` synchronization between the worker `:DOWN` and the next `:refresh_task_status` tick.
|
||||
|
||||
|
||||
@@ -1,22 +1,39 @@
|
||||
defmodule BDS.Maintenance do
|
||||
@moduledoc false
|
||||
|
||||
import Ecto.Query
|
||||
import BDS.Maintenance.Progress,
|
||||
only: [
|
||||
progress_callback: 1,
|
||||
report_metadata_diff_phase: 4,
|
||||
report_metadata_diff_complete: 1,
|
||||
report_started: 3,
|
||||
report_progress: 4
|
||||
]
|
||||
|
||||
import BDS.Maintenance.Repair,
|
||||
only: [
|
||||
normalize_entity_type: 1,
|
||||
normalize_repair_direction: 1,
|
||||
repair_metadata_diff_item: 3,
|
||||
repair_embedding_batch: 5,
|
||||
import_metadata_diff_orphan: 2
|
||||
]
|
||||
|
||||
import BDS.Maintenance.DiffReports,
|
||||
only: [
|
||||
project_metadata_diff_reports: 1,
|
||||
post_diff_reports: 2,
|
||||
post_translation_diff_reports: 2,
|
||||
media_diff_reports: 2,
|
||||
media_translation_diff_reports: 2,
|
||||
script_diff_reports: 2,
|
||||
template_diff_reports: 2
|
||||
]
|
||||
|
||||
import BDS.Maintenance.FileScan, only: [orphan_reports: 2]
|
||||
|
||||
alias BDS.Frontmatter
|
||||
alias BDS.DocumentFields
|
||||
alias BDS.Metadata
|
||||
alias BDS.Media.Media
|
||||
alias BDS.Media.Translation, as: MediaTranslation
|
||||
alias BDS.Embeddings
|
||||
alias BDS.Posts.Post
|
||||
alias BDS.Posts.Translation, as: PostTranslation
|
||||
alias BDS.Persistence
|
||||
alias BDS.Projects
|
||||
alias BDS.Repo
|
||||
alias BDS.Scripts.Script
|
||||
alias BDS.Sidecar
|
||||
alias BDS.Templates.Template
|
||||
|
||||
def repair_metadata_diff(project_id, direction, items, opts \\ [])
|
||||
|
||||
@@ -121,690 +138,4 @@ defmodule BDS.Maintenance do
|
||||
|
||||
{:ok, %{diff_reports: diff_reports, orphan_reports: orphan_reports}}
|
||||
end
|
||||
|
||||
defp project_metadata_diff_reports(project_id) do
|
||||
{:ok, db_state} = Metadata.get_project_metadata(project_id)
|
||||
{:ok, filesystem_state} = Metadata.read_project_metadata_from_filesystem(project_id)
|
||||
|
||||
[
|
||||
build_diff_report("project", project_id, [
|
||||
diff_field("name", db_state.name, filesystem_state.name),
|
||||
diff_field("description", db_state.description, filesystem_state.description),
|
||||
diff_field("public_url", db_state.public_url, filesystem_state.public_url),
|
||||
diff_field("main_language", db_state.main_language, filesystem_state.main_language),
|
||||
diff_field("default_author", db_state.default_author, filesystem_state.default_author),
|
||||
diff_field(
|
||||
"max_posts_per_page",
|
||||
db_state.max_posts_per_page,
|
||||
filesystem_state.max_posts_per_page
|
||||
),
|
||||
diff_field(
|
||||
"blogmark_category",
|
||||
db_state.blogmark_category,
|
||||
filesystem_state.blogmark_category
|
||||
),
|
||||
diff_field("pico_theme", db_state.pico_theme, filesystem_state.pico_theme),
|
||||
diff_field(
|
||||
"semantic_similarity_enabled",
|
||||
db_state.semantic_similarity_enabled,
|
||||
filesystem_state.semantic_similarity_enabled
|
||||
),
|
||||
diff_field("blog_languages", db_state.blog_languages, filesystem_state.blog_languages)
|
||||
]),
|
||||
build_diff_report("categories", project_id, [
|
||||
diff_field("categories", db_state.categories, filesystem_state.categories)
|
||||
]),
|
||||
build_diff_report("category_meta", project_id, [
|
||||
diff_field(
|
||||
"category_settings",
|
||||
db_state.category_settings,
|
||||
filesystem_state.category_settings
|
||||
)
|
||||
]),
|
||||
build_diff_report("publishing", project_id, [
|
||||
diff_field(
|
||||
"ssh_host",
|
||||
Map.get(db_state.publishing_preferences, "ssh_host"),
|
||||
Map.get(filesystem_state.publishing_preferences, "ssh_host")
|
||||
),
|
||||
diff_field(
|
||||
"ssh_user",
|
||||
Map.get(db_state.publishing_preferences, "ssh_user"),
|
||||
Map.get(filesystem_state.publishing_preferences, "ssh_user")
|
||||
),
|
||||
diff_field(
|
||||
"ssh_remote_path",
|
||||
Map.get(db_state.publishing_preferences, "ssh_remote_path"),
|
||||
Map.get(filesystem_state.publishing_preferences, "ssh_remote_path")
|
||||
),
|
||||
diff_field(
|
||||
"ssh_mode",
|
||||
Map.get(db_state.publishing_preferences, "ssh_mode"),
|
||||
Map.get(filesystem_state.publishing_preferences, "ssh_mode")
|
||||
)
|
||||
])
|
||||
]
|
||||
|> Enum.reject(&is_nil/1)
|
||||
end
|
||||
|
||||
defp normalize_entity_type(:post), do: :post
|
||||
defp normalize_entity_type(:media), do: :media
|
||||
defp normalize_entity_type(:script), do: :script
|
||||
defp normalize_entity_type(:template), do: :template
|
||||
defp normalize_entity_type(:embedding), do: :embedding
|
||||
defp normalize_entity_type("post"), do: :post
|
||||
defp normalize_entity_type("media"), do: :media
|
||||
defp normalize_entity_type("script"), do: :script
|
||||
defp normalize_entity_type("template"), do: :template
|
||||
defp normalize_entity_type("embedding"), do: :embedding
|
||||
defp normalize_entity_type("embeddings"), do: :embedding
|
||||
defp normalize_entity_type(_entity_type), do: :unsupported
|
||||
|
||||
defp post_diff_reports(project_id, project) do
|
||||
Repo.all(
|
||||
from post in Post,
|
||||
where:
|
||||
post.project_id == ^project_id and not is_nil(post.file_path) and post.file_path != ""
|
||||
)
|
||||
|> Enum.flat_map(fn post ->
|
||||
case read_frontmatter_document(project, post.file_path) do
|
||||
{:ok, %{fields: fields}} ->
|
||||
differences =
|
||||
[
|
||||
diff_field("title", post.title, Map.get(fields, "title")),
|
||||
diff_field("excerpt", post.excerpt, Map.get(fields, "excerpt")),
|
||||
diff_field("author", post.author, Map.get(fields, "author")),
|
||||
diff_field("language", post.language, Map.get(fields, "language")),
|
||||
diff_field("status", post.status, DocumentFields.get(fields, "status")),
|
||||
diff_field("template_slug", post.template_slug, DocumentFields.get(fields, "templateSlug")),
|
||||
diff_field("created_at", post.created_at, DocumentFields.get(fields, "createdAt")),
|
||||
diff_field("updated_at", post.updated_at, DocumentFields.get(fields, "updatedAt")),
|
||||
diff_field("published_at", post.published_at, DocumentFields.get(fields, "publishedAt")),
|
||||
diff_field("tags", post.tags, Map.get(fields, "tags", [])),
|
||||
diff_field("categories", post.categories, Map.get(fields, "categories", []))
|
||||
]
|
||||
|> Enum.reject(&is_nil/1)
|
||||
|
||||
if differences == [] do
|
||||
[]
|
||||
else
|
||||
[
|
||||
build_diff_report("post", post.id, differences,
|
||||
label: metadata_diff_entity_label(post.title, post.slug, post.id),
|
||||
meta_label: metadata_diff_timestamp_label(post.created_at)
|
||||
)
|
||||
]
|
||||
end
|
||||
|
||||
{:error, _reason} ->
|
||||
[]
|
||||
end
|
||||
end)
|
||||
end
|
||||
|
||||
defp media_diff_reports(project_id, project) do
|
||||
Repo.all(
|
||||
from media in Media,
|
||||
where:
|
||||
media.project_id == ^project_id and not is_nil(media.sidecar_path) and
|
||||
media.sidecar_path != ""
|
||||
)
|
||||
|> Enum.flat_map(fn media ->
|
||||
case read_sidecar_document(project, media.sidecar_path) do
|
||||
{:ok, fields} ->
|
||||
differences =
|
||||
[
|
||||
diff_field("title", media.title, Map.get(fields, "title")),
|
||||
diff_field("alt", media.alt, Map.get(fields, "alt")),
|
||||
diff_field("caption", media.caption, Map.get(fields, "caption")),
|
||||
diff_field("author", media.author, Map.get(fields, "author")),
|
||||
diff_field("language", media.language, Map.get(fields, "language")),
|
||||
diff_field("created_at", media.created_at, DocumentFields.get(fields, "createdAt")),
|
||||
diff_field("updated_at", media.updated_at, DocumentFields.get(fields, "updatedAt")),
|
||||
diff_field("tags", media.tags, Map.get(fields, "tags", []))
|
||||
]
|
||||
|> Enum.reject(&is_nil/1)
|
||||
|
||||
if differences == [] do
|
||||
[]
|
||||
else
|
||||
[%{entity_type: "media", entity_id: media.id, differences: differences}]
|
||||
end
|
||||
|
||||
{:error, _reason} ->
|
||||
[]
|
||||
end
|
||||
end)
|
||||
end
|
||||
|
||||
defp post_translation_diff_reports(project_id, project) do
|
||||
Repo.all(
|
||||
from translation in PostTranslation,
|
||||
where:
|
||||
translation.project_id == ^project_id and not is_nil(translation.file_path) and
|
||||
translation.file_path != ""
|
||||
)
|
||||
|> Enum.flat_map(fn translation ->
|
||||
case read_frontmatter_document(project, translation.file_path) do
|
||||
{:ok, %{fields: fields}} ->
|
||||
differences =
|
||||
[
|
||||
diff_field("title", translation.title, Map.get(fields, "title")),
|
||||
diff_field("excerpt", translation.excerpt, Map.get(fields, "excerpt")),
|
||||
diff_field("language", translation.language, Map.get(fields, "language")),
|
||||
diff_field(
|
||||
"translation_for",
|
||||
translation.translation_for,
|
||||
DocumentFields.get(fields, "translationFor")
|
||||
)
|
||||
]
|
||||
|> Enum.reject(&is_nil/1)
|
||||
|
||||
if differences == [] do
|
||||
[]
|
||||
else
|
||||
[
|
||||
build_diff_report("post_translation", translation.id, differences,
|
||||
label: metadata_diff_entity_label(translation.title, nil, translation.id),
|
||||
meta_label: translation.language
|
||||
)
|
||||
]
|
||||
end
|
||||
|
||||
{:error, _reason} ->
|
||||
[]
|
||||
end
|
||||
end)
|
||||
end
|
||||
|
||||
defp media_translation_diff_reports(project_id, project) do
|
||||
Repo.all(from translation in MediaTranslation, where: translation.project_id == ^project_id)
|
||||
|> Enum.flat_map(fn translation ->
|
||||
sidecar_path = media_translation_sidecar_path(project_id, translation)
|
||||
|
||||
case sidecar_path && read_sidecar_document(project, sidecar_path) do
|
||||
{:ok, fields} ->
|
||||
differences =
|
||||
[
|
||||
diff_field("title", translation.title, Map.get(fields, "title")),
|
||||
diff_field("alt", translation.alt, Map.get(fields, "alt")),
|
||||
diff_field("caption", translation.caption, Map.get(fields, "caption")),
|
||||
diff_field("language", translation.language, Map.get(fields, "language")),
|
||||
diff_field(
|
||||
"translation_for",
|
||||
translation.translation_for,
|
||||
DocumentFields.get(fields, "translationFor")
|
||||
)
|
||||
]
|
||||
|> Enum.reject(&is_nil/1)
|
||||
|
||||
if differences == [] do
|
||||
[]
|
||||
else
|
||||
[
|
||||
%{
|
||||
entity_type: "media_translation",
|
||||
entity_id: translation.id,
|
||||
differences: differences
|
||||
}
|
||||
]
|
||||
end
|
||||
|
||||
_ ->
|
||||
[]
|
||||
end
|
||||
end)
|
||||
end
|
||||
|
||||
defp script_diff_reports(project_id, project) do
|
||||
Repo.all(
|
||||
from script in Script,
|
||||
where:
|
||||
script.project_id == ^project_id and not is_nil(script.file_path) and
|
||||
script.file_path != ""
|
||||
)
|
||||
|> Enum.flat_map(fn script ->
|
||||
case read_frontmatter_document(project, script.file_path) do
|
||||
{:ok, %{fields: fields}} ->
|
||||
differences =
|
||||
[
|
||||
diff_field("title", script.title, Map.get(fields, "title")),
|
||||
diff_field("entrypoint", script.entrypoint, Map.get(fields, "entrypoint")),
|
||||
diff_field("enabled", script.enabled, Map.get(fields, "enabled")),
|
||||
diff_field("created_at", script.created_at, DocumentFields.get(fields, "createdAt")),
|
||||
diff_field("updated_at", script.updated_at, DocumentFields.get(fields, "updatedAt"))
|
||||
]
|
||||
|> Enum.reject(&is_nil/1)
|
||||
|
||||
if differences == [] do
|
||||
[]
|
||||
else
|
||||
[%{entity_type: "script", entity_id: script.id, differences: differences}]
|
||||
end
|
||||
|
||||
{:error, _reason} ->
|
||||
[]
|
||||
end
|
||||
end)
|
||||
end
|
||||
|
||||
defp template_diff_reports(project_id, project) do
|
||||
Repo.all(
|
||||
from template in Template,
|
||||
where:
|
||||
template.project_id == ^project_id and not is_nil(template.file_path) and
|
||||
template.file_path != ""
|
||||
)
|
||||
|> Enum.flat_map(fn template ->
|
||||
case read_frontmatter_document(project, template.file_path) do
|
||||
{:ok, %{fields: fields}} ->
|
||||
differences =
|
||||
[
|
||||
diff_field("title", template.title, Map.get(fields, "title")),
|
||||
diff_field("enabled", template.enabled, Map.get(fields, "enabled")),
|
||||
diff_field("created_at", template.created_at, DocumentFields.get(fields, "createdAt")),
|
||||
diff_field("updated_at", template.updated_at, DocumentFields.get(fields, "updatedAt"))
|
||||
]
|
||||
|> Enum.reject(&is_nil/1)
|
||||
|
||||
if differences == [] do
|
||||
[]
|
||||
else
|
||||
[%{entity_type: "template", entity_id: template.id, differences: differences}]
|
||||
end
|
||||
|
||||
{:error, _reason} ->
|
||||
[]
|
||||
end
|
||||
end)
|
||||
end
|
||||
|
||||
defp orphan_reports(project_id, project) do
|
||||
post_paths =
|
||||
MapSet.new(
|
||||
Repo.all(from post in Post, where: post.project_id == ^project_id, select: post.file_path)
|
||||
)
|
||||
|
||||
media_paths =
|
||||
MapSet.new(
|
||||
Repo.all(
|
||||
from media in Media, where: media.project_id == ^project_id, select: media.sidecar_path
|
||||
)
|
||||
)
|
||||
|
||||
post_translation_paths =
|
||||
MapSet.new(
|
||||
Repo.all(
|
||||
from translation in PostTranslation,
|
||||
where: translation.project_id == ^project_id,
|
||||
select: translation.file_path
|
||||
)
|
||||
)
|
||||
|
||||
media_translation_paths = MapSet.new(media_translation_sidecar_paths(project_id))
|
||||
|
||||
script_paths =
|
||||
MapSet.new(
|
||||
Repo.all(
|
||||
from script in Script, where: script.project_id == ^project_id, select: script.file_path
|
||||
)
|
||||
)
|
||||
|
||||
template_paths =
|
||||
MapSet.new(
|
||||
Repo.all(
|
||||
from template in Template,
|
||||
where: template.project_id == ^project_id,
|
||||
select: template.file_path
|
||||
)
|
||||
)
|
||||
|
||||
post_orphans =
|
||||
project
|
||||
|> list_project_files("posts/**/*.md")
|
||||
|> Enum.map(&Path.relative_to(&1, Projects.project_data_dir(project)))
|
||||
|> Enum.reject(&translation_post_file?/1)
|
||||
|> Enum.reject(&MapSet.member?(post_paths, &1))
|
||||
|
||||
post_translation_orphans =
|
||||
project
|
||||
|> list_project_files("posts/**/*.md")
|
||||
|> Enum.map(&Path.relative_to(&1, Projects.project_data_dir(project)))
|
||||
|> Enum.filter(&translation_post_file?/1)
|
||||
|> Enum.reject(&MapSet.member?(post_translation_paths, &1))
|
||||
|
||||
media_orphans =
|
||||
project
|
||||
|> list_project_files("media/**/*.meta")
|
||||
|> Enum.map(&Path.relative_to(&1, Projects.project_data_dir(project)))
|
||||
|> Enum.filter(&canonical_media_sidecar?/1)
|
||||
|> Enum.reject(&MapSet.member?(media_paths, &1))
|
||||
|
||||
media_translation_orphans =
|
||||
project
|
||||
|> list_project_files("media/**/*.meta")
|
||||
|> Enum.map(&Path.relative_to(&1, Projects.project_data_dir(project)))
|
||||
|> Enum.filter(&translation_media_sidecar?/1)
|
||||
|> Enum.reject(&MapSet.member?(media_translation_paths, &1))
|
||||
|
||||
script_orphans =
|
||||
project
|
||||
|> list_project_files("scripts/**/*.lua")
|
||||
|> Enum.map(&Path.relative_to(&1, Projects.project_data_dir(project)))
|
||||
|> Enum.reject(&MapSet.member?(script_paths, &1))
|
||||
|
||||
template_orphans =
|
||||
project
|
||||
|> list_project_files("templates/*.liquid")
|
||||
|> Enum.map(&Path.relative_to(&1, Projects.project_data_dir(project)))
|
||||
|> Enum.reject(&MapSet.member?(template_paths, &1))
|
||||
|
||||
(post_orphans ++
|
||||
post_translation_orphans ++
|
||||
media_orphans ++ media_translation_orphans ++ script_orphans ++ template_orphans)
|
||||
|> Enum.sort()
|
||||
|> Enum.map(&%{file_path: &1})
|
||||
end
|
||||
|
||||
defp build_diff_report(entity_type, entity_id, differences) do
|
||||
build_diff_report(entity_type, entity_id, differences, [])
|
||||
end
|
||||
|
||||
defp build_diff_report(entity_type, entity_id, differences, opts) do
|
||||
normalized = Enum.reject(differences, &is_nil/1)
|
||||
|
||||
if normalized == [] do
|
||||
nil
|
||||
else
|
||||
%{
|
||||
entity_type: entity_type,
|
||||
entity_id: entity_id,
|
||||
differences: normalized,
|
||||
label: Keyword.get(opts, :label),
|
||||
meta_label: Keyword.get(opts, :meta_label)
|
||||
}
|
||||
end
|
||||
end
|
||||
|
||||
defp metadata_diff_entity_label(title, slug, fallback_id) do
|
||||
blank_to_nil(title) || blank_to_nil(slug) || fallback_id
|
||||
end
|
||||
|
||||
defp metadata_diff_timestamp_label(nil), do: nil
|
||||
defp metadata_diff_timestamp_label(timestamp), do: Persistence.timestamp_to_iso8601(timestamp)
|
||||
|
||||
defp blank_to_nil(nil), do: nil
|
||||
|
||||
defp blank_to_nil(value) when is_binary(value) do
|
||||
case String.trim(value) do
|
||||
"" -> nil
|
||||
trimmed -> trimmed
|
||||
end
|
||||
end
|
||||
|
||||
defp blank_to_nil(value), do: value
|
||||
|
||||
defp diff_field(name, db_value, file_value) do
|
||||
if equal_diff_values?(db_value, file_value) do
|
||||
nil
|
||||
else
|
||||
%{name: name, db_value: stringify_value(db_value), file_value: stringify_value(file_value)}
|
||||
end
|
||||
end
|
||||
|
||||
defp equal_diff_values?(left, right) when is_list(left) and is_list(right) do
|
||||
normalize_list_diff_values(left) == normalize_list_diff_values(right)
|
||||
end
|
||||
|
||||
defp equal_diff_values?(left, right) when is_map(left) and is_map(right) do
|
||||
normalize_map_diff_values(left) == normalize_map_diff_values(right)
|
||||
end
|
||||
|
||||
defp equal_diff_values?(left, right), do: stringify_value(left) == stringify_value(right)
|
||||
|
||||
defp normalize_list_diff_values(values) do
|
||||
values
|
||||
|> Enum.map(&stringify_value/1)
|
||||
|> Enum.sort()
|
||||
end
|
||||
|
||||
defp stringify_value(nil), do: ""
|
||||
defp stringify_value(value) when is_atom(value), do: Atom.to_string(value)
|
||||
defp stringify_value(value) when is_boolean(value), do: to_string(value)
|
||||
defp stringify_value(value) when is_integer(value), do: Integer.to_string(value)
|
||||
defp stringify_value(value) when is_binary(value), do: value
|
||||
|
||||
defp stringify_value(value) when is_map(value),
|
||||
do: value |> normalize_map_diff_values() |> Jason.encode!()
|
||||
|
||||
defp stringify_value(value) when is_list(value),
|
||||
do: Enum.map_join(value, ",", &stringify_value/1)
|
||||
|
||||
defp stringify_value(value), do: to_string(value)
|
||||
|
||||
defp normalize_map_diff_values(values) when is_map(values) do
|
||||
values
|
||||
|> Enum.map(fn {key, value} -> {to_string(key), normalize_nested_diff_value(value)} end)
|
||||
|> Enum.sort_by(&elem(&1, 0))
|
||||
|> Map.new()
|
||||
end
|
||||
|
||||
defp normalize_nested_diff_value(value) when is_map(value), do: normalize_map_diff_values(value)
|
||||
defp normalize_nested_diff_value(value) when is_list(value), do: Enum.map(value, &normalize_nested_diff_value/1)
|
||||
defp normalize_nested_diff_value(value) when is_atom(value), do: Atom.to_string(value)
|
||||
defp normalize_nested_diff_value(value), do: value
|
||||
|
||||
defp read_frontmatter_document(project, relative_path) do
|
||||
full_path = Path.join(Projects.project_data_dir(project), relative_path)
|
||||
|
||||
case File.read(full_path) do
|
||||
{:ok, contents} -> Frontmatter.parse_document(contents)
|
||||
{:error, reason} -> {:error, reason}
|
||||
end
|
||||
end
|
||||
|
||||
defp read_sidecar_document(project, relative_path) do
|
||||
full_path = Path.join(Projects.project_data_dir(project), relative_path)
|
||||
|
||||
case File.read(full_path) do
|
||||
{:ok, contents} -> Sidecar.parse_document(contents)
|
||||
{:error, reason} -> {:error, reason}
|
||||
end
|
||||
end
|
||||
|
||||
defp list_project_files(project, glob) do
|
||||
project
|
||||
|> Projects.project_data_dir()
|
||||
|> Path.join(glob)
|
||||
|> Path.wildcard()
|
||||
|> Enum.sort()
|
||||
end
|
||||
|
||||
defp canonical_media_sidecar?(relative_path) do
|
||||
not Regex.match?(~r/\.[a-z]{2}\.meta$/i, relative_path)
|
||||
end
|
||||
|
||||
defp translation_post_file?(relative_path) do
|
||||
Regex.match?(~r/\.[a-z]{2}\.md$/i, relative_path)
|
||||
end
|
||||
|
||||
defp translation_media_sidecar?(relative_path) do
|
||||
Regex.match?(~r/\.[a-z]{2}\.meta$/i, relative_path)
|
||||
end
|
||||
|
||||
defp media_translation_sidecar_paths(project_id) do
|
||||
Repo.all(from translation in MediaTranslation, where: translation.project_id == ^project_id)
|
||||
|> Enum.map(&media_translation_sidecar_path(project_id, &1))
|
||||
|> Enum.reject(&is_nil/1)
|
||||
end
|
||||
|
||||
defp media_translation_sidecar_path(project_id, translation) do
|
||||
case Repo.one(
|
||||
from media in Media,
|
||||
where: media.project_id == ^project_id and media.id == ^translation.translation_for,
|
||||
select: media.file_path
|
||||
) do
|
||||
nil -> nil
|
||||
file_path -> "#{file_path}.#{translation.language}.meta"
|
||||
end
|
||||
end
|
||||
|
||||
defp repair_metadata_diff_item(project_id, direction, item) do
|
||||
entity_type = Map.get(item, :entity_type) || Map.get(item, "entity_type")
|
||||
entity_id = Map.get(item, :entity_id) || Map.get(item, "entity_id")
|
||||
|
||||
case {normalize_repair_direction(direction), entity_type} do
|
||||
{:file_to_db, entity_type} when entity_type in ["project", "categories", "category_meta", "publishing"] ->
|
||||
Metadata.sync_project_metadata_from_filesystem(project_id)
|
||||
|
||||
{:db_to_file, entity_type} when entity_type in ["project", "categories", "category_meta", "publishing"] ->
|
||||
Metadata.flush_project_metadata_to_filesystem(project_id)
|
||||
|
||||
{:file_to_db, "post"} -> BDS.Posts.sync_post_from_file(entity_id)
|
||||
{:db_to_file, "post"} -> BDS.Posts.rewrite_published_post(entity_id)
|
||||
{:file_to_db, "post_translation"} -> BDS.Posts.sync_post_translation_from_file(entity_id)
|
||||
{:db_to_file, "post_translation"} -> BDS.Posts.rewrite_published_post_translation(entity_id)
|
||||
{:file_to_db, "media"} -> BDS.Media.sync_media_from_sidecar(entity_id)
|
||||
{:db_to_file, "media"} -> BDS.Media.sync_media_sidecar(entity_id)
|
||||
{:file_to_db, "media_translation"} -> BDS.Media.sync_media_translation_from_sidecar(entity_id)
|
||||
{:db_to_file, "media_translation"} -> BDS.Media.sync_media_translation_sidecar(entity_id)
|
||||
{:file_to_db, "script"} -> BDS.Scripts.sync_script_from_file(entity_id)
|
||||
{:db_to_file, "script"} -> BDS.Scripts.sync_published_script_file(entity_id)
|
||||
{:file_to_db, "template"} -> BDS.Templates.sync_template_from_file(entity_id)
|
||||
{:db_to_file, "template"} -> BDS.Templates.sync_published_template_file(entity_id)
|
||||
{:file_to_db, "embedding"} -> BDS.Embeddings.sync_post(entity_id)
|
||||
{:db_to_file, "embedding"} -> BDS.Embeddings.refresh_snapshot(project_id)
|
||||
_other -> {:error, :unsupported}
|
||||
end
|
||||
end
|
||||
|
||||
defp repair_embedding_batch(project_id, direction, items, on_progress, total)
|
||||
when direction in [:file_to_db, :db_to_file] do
|
||||
if items != [] and Enum.all?(items, &(metadata_diff_item_entity_type(&1) == "embedding")) do
|
||||
result =
|
||||
case direction do
|
||||
:file_to_db ->
|
||||
post_ids = Enum.map(items, &metadata_diff_item_entity_id/1)
|
||||
|
||||
{:ok, repaired_post_ids} = Embeddings.repair_posts(project_id, post_ids)
|
||||
repaired_post_ids = MapSet.new(repaired_post_ids)
|
||||
|
||||
build_batch_repair_result(items, total, on_progress, fn item ->
|
||||
MapSet.member?(repaired_post_ids, metadata_diff_item_entity_id(item))
|
||||
end)
|
||||
|
||||
:db_to_file ->
|
||||
repaired? = Embeddings.refresh_snapshot(project_id) == :ok
|
||||
build_batch_repair_result(items, total, on_progress, fn _item -> repaired? end)
|
||||
end
|
||||
|
||||
{:ok, result}
|
||||
else
|
||||
:unsupported
|
||||
end
|
||||
end
|
||||
|
||||
defp repair_embedding_batch(_project_id, _direction, _items, _on_progress, _total), do: :unsupported
|
||||
|
||||
defp build_batch_repair_result(items, total, on_progress, repaired?) do
|
||||
items
|
||||
|> Enum.with_index(1)
|
||||
|> Enum.reduce(%{repaired: 0, failed: 0}, fn {item, index}, acc ->
|
||||
next_acc =
|
||||
if repaired?.(item) do
|
||||
%{acc | repaired: acc.repaired + 1}
|
||||
else
|
||||
%{acc | failed: acc.failed + 1}
|
||||
end
|
||||
|
||||
:ok = report_progress(on_progress, index, total, "Repairing metadata differences")
|
||||
next_acc
|
||||
end)
|
||||
end
|
||||
|
||||
defp metadata_diff_item_entity_type(item) do
|
||||
Map.get(item, :entity_type) || Map.get(item, "entity_type")
|
||||
end
|
||||
|
||||
defp metadata_diff_item_entity_id(item) do
|
||||
Map.get(item, :entity_id) || Map.get(item, "entity_id")
|
||||
end
|
||||
|
||||
defp import_metadata_diff_orphan(project_id, orphan) do
|
||||
file_path = Map.get(orphan, :file_path) || Map.get(orphan, "file_path")
|
||||
|
||||
cond do
|
||||
is_nil(file_path) ->
|
||||
{:error, :not_found}
|
||||
|
||||
translation_post_file?(file_path) ->
|
||||
BDS.Posts.import_orphan_post_translation_file(project_id, file_path)
|
||||
|
||||
String.ends_with?(file_path, ".md") ->
|
||||
BDS.Posts.import_orphan_post_file(project_id, file_path)
|
||||
|
||||
translation_media_sidecar?(file_path) ->
|
||||
BDS.Media.import_orphan_media_translation_sidecar(project_id, file_path)
|
||||
|
||||
canonical_media_sidecar?(file_path) and String.ends_with?(file_path, ".meta") ->
|
||||
BDS.Media.import_orphan_media_sidecar(project_id, file_path)
|
||||
|
||||
String.ends_with?(file_path, ".lua") ->
|
||||
BDS.Scripts.import_orphan_script_file(project_id, file_path)
|
||||
|
||||
String.ends_with?(file_path, ".liquid") ->
|
||||
BDS.Templates.import_orphan_template_file(project_id, file_path)
|
||||
|
||||
true ->
|
||||
{:error, :unsupported}
|
||||
end
|
||||
end
|
||||
|
||||
defp normalize_repair_direction(:file_to_db), do: :file_to_db
|
||||
defp normalize_repair_direction(:db_to_file), do: :db_to_file
|
||||
defp normalize_repair_direction("file_to_db"), do: :file_to_db
|
||||
defp normalize_repair_direction("db_to_file"), do: :db_to_file
|
||||
defp normalize_repair_direction(_direction), do: :unsupported
|
||||
|
||||
defp progress_callback(opts) do
|
||||
case Keyword.get(opts, :on_progress) do
|
||||
callback when is_function(callback, 2) -> callback
|
||||
_other -> nil
|
||||
end
|
||||
end
|
||||
|
||||
defp report_metadata_diff_phase(nil, _current, _total, _label), do: :ok
|
||||
|
||||
defp report_metadata_diff_phase(callback, current, total, label) do
|
||||
value = if total <= 1, do: 0.0, else: (current - 1) / total
|
||||
callback.(value, "#{label} (#{current}/#{total})")
|
||||
:ok
|
||||
end
|
||||
|
||||
defp report_metadata_diff_complete(nil), do: :ok
|
||||
|
||||
defp report_metadata_diff_complete(callback) do
|
||||
callback.(1.0, "Metadata diff complete")
|
||||
:ok
|
||||
end
|
||||
|
||||
defp report_started(nil, _total, _label), do: :ok
|
||||
|
||||
defp report_started(callback, 0, label) do
|
||||
callback.(1.0, label)
|
||||
:ok
|
||||
end
|
||||
|
||||
defp report_started(callback, total, label) do
|
||||
callback.(0.05, "#{label} (0/#{total})")
|
||||
:ok
|
||||
end
|
||||
|
||||
defp report_progress(nil, _current, _total, _label), do: :ok
|
||||
defp report_progress(_callback, _current, 0, _label), do: :ok
|
||||
|
||||
defp report_progress(callback, current, total, label) do
|
||||
callback.(0.05 + 0.95 * (current / total), "#{label} (#{current}/#{total})")
|
||||
:ok
|
||||
end
|
||||
end
|
||||
|
||||
93
lib/bds/maintenance/diff_computation.ex
Normal file
93
lib/bds/maintenance/diff_computation.ex
Normal file
@@ -0,0 +1,93 @@
|
||||
defmodule BDS.Maintenance.DiffComputation do
|
||||
@moduledoc false
|
||||
|
||||
alias BDS.Persistence
|
||||
|
||||
def build_diff_report(entity_type, entity_id, differences) do
|
||||
build_diff_report(entity_type, entity_id, differences, [])
|
||||
end
|
||||
|
||||
def build_diff_report(entity_type, entity_id, differences, opts) do
|
||||
normalized = Enum.reject(differences, &is_nil/1)
|
||||
|
||||
if normalized == [] do
|
||||
nil
|
||||
else
|
||||
%{
|
||||
entity_type: entity_type,
|
||||
entity_id: entity_id,
|
||||
differences: normalized,
|
||||
label: Keyword.get(opts, :label),
|
||||
meta_label: Keyword.get(opts, :meta_label)
|
||||
}
|
||||
end
|
||||
end
|
||||
|
||||
def metadata_diff_entity_label(title, slug, fallback_id) do
|
||||
blank_to_nil(title) || blank_to_nil(slug) || fallback_id
|
||||
end
|
||||
|
||||
def metadata_diff_timestamp_label(nil), do: nil
|
||||
def metadata_diff_timestamp_label(timestamp), do: Persistence.timestamp_to_iso8601(timestamp)
|
||||
|
||||
def blank_to_nil(nil), do: nil
|
||||
|
||||
def blank_to_nil(value) when is_binary(value) do
|
||||
case String.trim(value) do
|
||||
"" -> nil
|
||||
trimmed -> trimmed
|
||||
end
|
||||
end
|
||||
|
||||
def blank_to_nil(value), do: value
|
||||
|
||||
def diff_field(name, db_value, file_value) do
|
||||
if equal_diff_values?(db_value, file_value) do
|
||||
nil
|
||||
else
|
||||
%{name: name, db_value: stringify_value(db_value), file_value: stringify_value(file_value)}
|
||||
end
|
||||
end
|
||||
|
||||
def equal_diff_values?(left, right) when is_list(left) and is_list(right) do
|
||||
normalize_list_diff_values(left) == normalize_list_diff_values(right)
|
||||
end
|
||||
|
||||
def equal_diff_values?(left, right) when is_map(left) and is_map(right) do
|
||||
normalize_map_diff_values(left) == normalize_map_diff_values(right)
|
||||
end
|
||||
|
||||
def equal_diff_values?(left, right), do: stringify_value(left) == stringify_value(right)
|
||||
|
||||
def normalize_list_diff_values(values) do
|
||||
values
|
||||
|> Enum.map(&stringify_value/1)
|
||||
|> Enum.sort()
|
||||
end
|
||||
|
||||
def stringify_value(nil), do: ""
|
||||
def stringify_value(value) when is_atom(value), do: Atom.to_string(value)
|
||||
def stringify_value(value) when is_boolean(value), do: to_string(value)
|
||||
def stringify_value(value) when is_integer(value), do: Integer.to_string(value)
|
||||
def stringify_value(value) when is_binary(value), do: value
|
||||
|
||||
def stringify_value(value) when is_map(value),
|
||||
do: value |> normalize_map_diff_values() |> Jason.encode!()
|
||||
|
||||
def stringify_value(value) when is_list(value),
|
||||
do: Enum.map_join(value, ",", &stringify_value/1)
|
||||
|
||||
def stringify_value(value), do: to_string(value)
|
||||
|
||||
def normalize_map_diff_values(values) when is_map(values) do
|
||||
values
|
||||
|> Enum.map(fn {key, value} -> {to_string(key), normalize_nested_diff_value(value)} end)
|
||||
|> Enum.sort_by(&elem(&1, 0))
|
||||
|> Map.new()
|
||||
end
|
||||
|
||||
def normalize_nested_diff_value(value) when is_map(value), do: normalize_map_diff_values(value)
|
||||
def normalize_nested_diff_value(value) when is_list(value), do: Enum.map(value, &normalize_nested_diff_value/1)
|
||||
def normalize_nested_diff_value(value) when is_atom(value), do: Atom.to_string(value)
|
||||
def normalize_nested_diff_value(value), do: value
|
||||
end
|
||||
315
lib/bds/maintenance/diff_reports.ex
Normal file
315
lib/bds/maintenance/diff_reports.ex
Normal file
@@ -0,0 +1,315 @@
|
||||
defmodule BDS.Maintenance.DiffReports do
|
||||
@moduledoc false
|
||||
|
||||
import Ecto.Query
|
||||
|
||||
import BDS.Maintenance.DiffComputation,
|
||||
only: [
|
||||
build_diff_report: 3,
|
||||
build_diff_report: 4,
|
||||
diff_field: 3,
|
||||
metadata_diff_entity_label: 3,
|
||||
metadata_diff_timestamp_label: 1
|
||||
]
|
||||
|
||||
import BDS.Maintenance.FileScan,
|
||||
only: [
|
||||
read_frontmatter_document: 2,
|
||||
read_sidecar_document: 2,
|
||||
media_translation_sidecar_path: 2
|
||||
]
|
||||
|
||||
alias BDS.DocumentFields
|
||||
alias BDS.Media.Media
|
||||
alias BDS.Media.Translation, as: MediaTranslation
|
||||
alias BDS.Metadata
|
||||
alias BDS.Posts.Post
|
||||
alias BDS.Posts.Translation, as: PostTranslation
|
||||
alias BDS.Repo
|
||||
alias BDS.Scripts.Script
|
||||
alias BDS.Templates.Template
|
||||
|
||||
def project_metadata_diff_reports(project_id) do
|
||||
{:ok, db_state} = Metadata.get_project_metadata(project_id)
|
||||
{:ok, filesystem_state} = Metadata.read_project_metadata_from_filesystem(project_id)
|
||||
|
||||
[
|
||||
build_diff_report("project", project_id, [
|
||||
diff_field("name", db_state.name, filesystem_state.name),
|
||||
diff_field("description", db_state.description, filesystem_state.description),
|
||||
diff_field("public_url", db_state.public_url, filesystem_state.public_url),
|
||||
diff_field("main_language", db_state.main_language, filesystem_state.main_language),
|
||||
diff_field("default_author", db_state.default_author, filesystem_state.default_author),
|
||||
diff_field(
|
||||
"max_posts_per_page",
|
||||
db_state.max_posts_per_page,
|
||||
filesystem_state.max_posts_per_page
|
||||
),
|
||||
diff_field(
|
||||
"blogmark_category",
|
||||
db_state.blogmark_category,
|
||||
filesystem_state.blogmark_category
|
||||
),
|
||||
diff_field("pico_theme", db_state.pico_theme, filesystem_state.pico_theme),
|
||||
diff_field(
|
||||
"semantic_similarity_enabled",
|
||||
db_state.semantic_similarity_enabled,
|
||||
filesystem_state.semantic_similarity_enabled
|
||||
),
|
||||
diff_field("blog_languages", db_state.blog_languages, filesystem_state.blog_languages)
|
||||
]),
|
||||
build_diff_report("categories", project_id, [
|
||||
diff_field("categories", db_state.categories, filesystem_state.categories)
|
||||
]),
|
||||
build_diff_report("category_meta", project_id, [
|
||||
diff_field(
|
||||
"category_settings",
|
||||
db_state.category_settings,
|
||||
filesystem_state.category_settings
|
||||
)
|
||||
]),
|
||||
build_diff_report("publishing", project_id, [
|
||||
diff_field(
|
||||
"ssh_host",
|
||||
Map.get(db_state.publishing_preferences, "ssh_host"),
|
||||
Map.get(filesystem_state.publishing_preferences, "ssh_host")
|
||||
),
|
||||
diff_field(
|
||||
"ssh_user",
|
||||
Map.get(db_state.publishing_preferences, "ssh_user"),
|
||||
Map.get(filesystem_state.publishing_preferences, "ssh_user")
|
||||
),
|
||||
diff_field(
|
||||
"ssh_remote_path",
|
||||
Map.get(db_state.publishing_preferences, "ssh_remote_path"),
|
||||
Map.get(filesystem_state.publishing_preferences, "ssh_remote_path")
|
||||
),
|
||||
diff_field(
|
||||
"ssh_mode",
|
||||
Map.get(db_state.publishing_preferences, "ssh_mode"),
|
||||
Map.get(filesystem_state.publishing_preferences, "ssh_mode")
|
||||
)
|
||||
])
|
||||
]
|
||||
|> Enum.reject(&is_nil/1)
|
||||
end
|
||||
|
||||
def post_diff_reports(project_id, project) do
|
||||
Repo.all(
|
||||
from post in Post,
|
||||
where:
|
||||
post.project_id == ^project_id and not is_nil(post.file_path) and post.file_path != ""
|
||||
)
|
||||
|> Enum.flat_map(fn post ->
|
||||
case read_frontmatter_document(project, post.file_path) do
|
||||
{:ok, %{fields: fields}} ->
|
||||
differences =
|
||||
[
|
||||
diff_field("title", post.title, Map.get(fields, "title")),
|
||||
diff_field("excerpt", post.excerpt, Map.get(fields, "excerpt")),
|
||||
diff_field("author", post.author, Map.get(fields, "author")),
|
||||
diff_field("language", post.language, Map.get(fields, "language")),
|
||||
diff_field("status", post.status, DocumentFields.get(fields, "status")),
|
||||
diff_field("template_slug", post.template_slug, DocumentFields.get(fields, "templateSlug")),
|
||||
diff_field("created_at", post.created_at, DocumentFields.get(fields, "createdAt")),
|
||||
diff_field("updated_at", post.updated_at, DocumentFields.get(fields, "updatedAt")),
|
||||
diff_field("published_at", post.published_at, DocumentFields.get(fields, "publishedAt")),
|
||||
diff_field("tags", post.tags, Map.get(fields, "tags", [])),
|
||||
diff_field("categories", post.categories, Map.get(fields, "categories", []))
|
||||
]
|
||||
|> Enum.reject(&is_nil/1)
|
||||
|
||||
if differences == [] do
|
||||
[]
|
||||
else
|
||||
[
|
||||
build_diff_report("post", post.id, differences,
|
||||
label: metadata_diff_entity_label(post.title, post.slug, post.id),
|
||||
meta_label: metadata_diff_timestamp_label(post.created_at)
|
||||
)
|
||||
]
|
||||
end
|
||||
|
||||
{:error, _reason} ->
|
||||
[]
|
||||
end
|
||||
end)
|
||||
end
|
||||
|
||||
def media_diff_reports(project_id, project) do
|
||||
Repo.all(
|
||||
from media in Media,
|
||||
where:
|
||||
media.project_id == ^project_id and not is_nil(media.sidecar_path) and
|
||||
media.sidecar_path != ""
|
||||
)
|
||||
|> Enum.flat_map(fn media ->
|
||||
case read_sidecar_document(project, media.sidecar_path) do
|
||||
{:ok, fields} ->
|
||||
differences =
|
||||
[
|
||||
diff_field("title", media.title, Map.get(fields, "title")),
|
||||
diff_field("alt", media.alt, Map.get(fields, "alt")),
|
||||
diff_field("caption", media.caption, Map.get(fields, "caption")),
|
||||
diff_field("author", media.author, Map.get(fields, "author")),
|
||||
diff_field("language", media.language, Map.get(fields, "language")),
|
||||
diff_field("created_at", media.created_at, DocumentFields.get(fields, "createdAt")),
|
||||
diff_field("updated_at", media.updated_at, DocumentFields.get(fields, "updatedAt")),
|
||||
diff_field("tags", media.tags, Map.get(fields, "tags", []))
|
||||
]
|
||||
|> Enum.reject(&is_nil/1)
|
||||
|
||||
if differences == [] do
|
||||
[]
|
||||
else
|
||||
[%{entity_type: "media", entity_id: media.id, differences: differences}]
|
||||
end
|
||||
|
||||
{:error, _reason} ->
|
||||
[]
|
||||
end
|
||||
end)
|
||||
end
|
||||
|
||||
def post_translation_diff_reports(project_id, project) do
|
||||
Repo.all(
|
||||
from translation in PostTranslation,
|
||||
where:
|
||||
translation.project_id == ^project_id and not is_nil(translation.file_path) and
|
||||
translation.file_path != ""
|
||||
)
|
||||
|> Enum.flat_map(fn translation ->
|
||||
case read_frontmatter_document(project, translation.file_path) do
|
||||
{:ok, %{fields: fields}} ->
|
||||
differences =
|
||||
[
|
||||
diff_field("title", translation.title, Map.get(fields, "title")),
|
||||
diff_field("excerpt", translation.excerpt, Map.get(fields, "excerpt")),
|
||||
diff_field("language", translation.language, Map.get(fields, "language")),
|
||||
diff_field(
|
||||
"translation_for",
|
||||
translation.translation_for,
|
||||
DocumentFields.get(fields, "translationFor")
|
||||
)
|
||||
]
|
||||
|> Enum.reject(&is_nil/1)
|
||||
|
||||
if differences == [] do
|
||||
[]
|
||||
else
|
||||
[
|
||||
build_diff_report("post_translation", translation.id, differences,
|
||||
label: metadata_diff_entity_label(translation.title, nil, translation.id),
|
||||
meta_label: translation.language
|
||||
)
|
||||
]
|
||||
end
|
||||
|
||||
{:error, _reason} ->
|
||||
[]
|
||||
end
|
||||
end)
|
||||
end
|
||||
|
||||
def media_translation_diff_reports(project_id, project) do
|
||||
Repo.all(from translation in MediaTranslation, where: translation.project_id == ^project_id)
|
||||
|> Enum.flat_map(fn translation ->
|
||||
sidecar_path = media_translation_sidecar_path(project_id, translation)
|
||||
|
||||
case sidecar_path && read_sidecar_document(project, sidecar_path) do
|
||||
{:ok, fields} ->
|
||||
differences =
|
||||
[
|
||||
diff_field("title", translation.title, Map.get(fields, "title")),
|
||||
diff_field("alt", translation.alt, Map.get(fields, "alt")),
|
||||
diff_field("caption", translation.caption, Map.get(fields, "caption")),
|
||||
diff_field("language", translation.language, Map.get(fields, "language")),
|
||||
diff_field(
|
||||
"translation_for",
|
||||
translation.translation_for,
|
||||
DocumentFields.get(fields, "translationFor")
|
||||
)
|
||||
]
|
||||
|> Enum.reject(&is_nil/1)
|
||||
|
||||
if differences == [] do
|
||||
[]
|
||||
else
|
||||
[
|
||||
%{
|
||||
entity_type: "media_translation",
|
||||
entity_id: translation.id,
|
||||
differences: differences
|
||||
}
|
||||
]
|
||||
end
|
||||
|
||||
_ ->
|
||||
[]
|
||||
end
|
||||
end)
|
||||
end
|
||||
|
||||
def script_diff_reports(project_id, project) do
|
||||
Repo.all(
|
||||
from script in Script,
|
||||
where:
|
||||
script.project_id == ^project_id and not is_nil(script.file_path) and
|
||||
script.file_path != ""
|
||||
)
|
||||
|> Enum.flat_map(fn script ->
|
||||
case read_frontmatter_document(project, script.file_path) do
|
||||
{:ok, %{fields: fields}} ->
|
||||
differences =
|
||||
[
|
||||
diff_field("title", script.title, Map.get(fields, "title")),
|
||||
diff_field("entrypoint", script.entrypoint, Map.get(fields, "entrypoint")),
|
||||
diff_field("enabled", script.enabled, Map.get(fields, "enabled")),
|
||||
diff_field("created_at", script.created_at, DocumentFields.get(fields, "createdAt")),
|
||||
diff_field("updated_at", script.updated_at, DocumentFields.get(fields, "updatedAt"))
|
||||
]
|
||||
|> Enum.reject(&is_nil/1)
|
||||
|
||||
if differences == [] do
|
||||
[]
|
||||
else
|
||||
[%{entity_type: "script", entity_id: script.id, differences: differences}]
|
||||
end
|
||||
|
||||
{:error, _reason} ->
|
||||
[]
|
||||
end
|
||||
end)
|
||||
end
|
||||
|
||||
def template_diff_reports(project_id, project) do
|
||||
Repo.all(
|
||||
from template in Template,
|
||||
where:
|
||||
template.project_id == ^project_id and not is_nil(template.file_path) and
|
||||
template.file_path != ""
|
||||
)
|
||||
|> Enum.flat_map(fn template ->
|
||||
case read_frontmatter_document(project, template.file_path) do
|
||||
{:ok, %{fields: fields}} ->
|
||||
differences =
|
||||
[
|
||||
diff_field("title", template.title, Map.get(fields, "title")),
|
||||
diff_field("enabled", template.enabled, Map.get(fields, "enabled")),
|
||||
diff_field("created_at", template.created_at, DocumentFields.get(fields, "createdAt")),
|
||||
diff_field("updated_at", template.updated_at, DocumentFields.get(fields, "updatedAt"))
|
||||
]
|
||||
|> Enum.reject(&is_nil/1)
|
||||
|
||||
if differences == [] do
|
||||
[]
|
||||
else
|
||||
[%{entity_type: "template", entity_id: template.id, differences: differences}]
|
||||
end
|
||||
|
||||
{:error, _reason} ->
|
||||
[]
|
||||
end
|
||||
end)
|
||||
end
|
||||
end
|
||||
158
lib/bds/maintenance/file_scan.ex
Normal file
158
lib/bds/maintenance/file_scan.ex
Normal file
@@ -0,0 +1,158 @@
|
||||
defmodule BDS.Maintenance.FileScan do
|
||||
@moduledoc false
|
||||
|
||||
import Ecto.Query
|
||||
|
||||
alias BDS.Frontmatter
|
||||
alias BDS.Media.Media
|
||||
alias BDS.Media.Translation, as: MediaTranslation
|
||||
alias BDS.Posts.Post
|
||||
alias BDS.Posts.Translation, as: PostTranslation
|
||||
alias BDS.Projects
|
||||
alias BDS.Repo
|
||||
alias BDS.Scripts.Script
|
||||
alias BDS.Sidecar
|
||||
alias BDS.Templates.Template
|
||||
|
||||
def orphan_reports(project_id, project) do
|
||||
post_paths =
|
||||
MapSet.new(
|
||||
Repo.all(from post in Post, where: post.project_id == ^project_id, select: post.file_path)
|
||||
)
|
||||
|
||||
media_paths =
|
||||
MapSet.new(
|
||||
Repo.all(
|
||||
from media in Media, where: media.project_id == ^project_id, select: media.sidecar_path
|
||||
)
|
||||
)
|
||||
|
||||
post_translation_paths =
|
||||
MapSet.new(
|
||||
Repo.all(
|
||||
from translation in PostTranslation,
|
||||
where: translation.project_id == ^project_id,
|
||||
select: translation.file_path
|
||||
)
|
||||
)
|
||||
|
||||
media_translation_paths = MapSet.new(media_translation_sidecar_paths(project_id))
|
||||
|
||||
script_paths =
|
||||
MapSet.new(
|
||||
Repo.all(
|
||||
from script in Script, where: script.project_id == ^project_id, select: script.file_path
|
||||
)
|
||||
)
|
||||
|
||||
template_paths =
|
||||
MapSet.new(
|
||||
Repo.all(
|
||||
from template in Template,
|
||||
where: template.project_id == ^project_id,
|
||||
select: template.file_path
|
||||
)
|
||||
)
|
||||
|
||||
post_orphans =
|
||||
project
|
||||
|> list_project_files("posts/**/*.md")
|
||||
|> Enum.map(&Path.relative_to(&1, Projects.project_data_dir(project)))
|
||||
|> Enum.reject(&translation_post_file?/1)
|
||||
|> Enum.reject(&MapSet.member?(post_paths, &1))
|
||||
|
||||
post_translation_orphans =
|
||||
project
|
||||
|> list_project_files("posts/**/*.md")
|
||||
|> Enum.map(&Path.relative_to(&1, Projects.project_data_dir(project)))
|
||||
|> Enum.filter(&translation_post_file?/1)
|
||||
|> Enum.reject(&MapSet.member?(post_translation_paths, &1))
|
||||
|
||||
media_orphans =
|
||||
project
|
||||
|> list_project_files("media/**/*.meta")
|
||||
|> Enum.map(&Path.relative_to(&1, Projects.project_data_dir(project)))
|
||||
|> Enum.filter(&canonical_media_sidecar?/1)
|
||||
|> Enum.reject(&MapSet.member?(media_paths, &1))
|
||||
|
||||
media_translation_orphans =
|
||||
project
|
||||
|> list_project_files("media/**/*.meta")
|
||||
|> Enum.map(&Path.relative_to(&1, Projects.project_data_dir(project)))
|
||||
|> Enum.filter(&translation_media_sidecar?/1)
|
||||
|> Enum.reject(&MapSet.member?(media_translation_paths, &1))
|
||||
|
||||
script_orphans =
|
||||
project
|
||||
|> list_project_files("scripts/**/*.lua")
|
||||
|> Enum.map(&Path.relative_to(&1, Projects.project_data_dir(project)))
|
||||
|> Enum.reject(&MapSet.member?(script_paths, &1))
|
||||
|
||||
template_orphans =
|
||||
project
|
||||
|> list_project_files("templates/*.liquid")
|
||||
|> Enum.map(&Path.relative_to(&1, Projects.project_data_dir(project)))
|
||||
|> Enum.reject(&MapSet.member?(template_paths, &1))
|
||||
|
||||
(post_orphans ++
|
||||
post_translation_orphans ++
|
||||
media_orphans ++ media_translation_orphans ++ script_orphans ++ template_orphans)
|
||||
|> Enum.sort()
|
||||
|> Enum.map(&%{file_path: &1})
|
||||
end
|
||||
|
||||
def read_frontmatter_document(project, relative_path) do
|
||||
full_path = Path.join(Projects.project_data_dir(project), relative_path)
|
||||
|
||||
case File.read(full_path) do
|
||||
{:ok, contents} -> Frontmatter.parse_document(contents)
|
||||
{:error, reason} -> {:error, reason}
|
||||
end
|
||||
end
|
||||
|
||||
def read_sidecar_document(project, relative_path) do
|
||||
full_path = Path.join(Projects.project_data_dir(project), relative_path)
|
||||
|
||||
case File.read(full_path) do
|
||||
{:ok, contents} -> Sidecar.parse_document(contents)
|
||||
{:error, reason} -> {:error, reason}
|
||||
end
|
||||
end
|
||||
|
||||
def list_project_files(project, glob) do
|
||||
project
|
||||
|> Projects.project_data_dir()
|
||||
|> Path.join(glob)
|
||||
|> Path.wildcard()
|
||||
|> Enum.sort()
|
||||
end
|
||||
|
||||
def canonical_media_sidecar?(relative_path) do
|
||||
not Regex.match?(~r/\.[a-z]{2}\.meta$/i, relative_path)
|
||||
end
|
||||
|
||||
def translation_post_file?(relative_path) do
|
||||
Regex.match?(~r/\.[a-z]{2}\.md$/i, relative_path)
|
||||
end
|
||||
|
||||
def translation_media_sidecar?(relative_path) do
|
||||
Regex.match?(~r/\.[a-z]{2}\.meta$/i, relative_path)
|
||||
end
|
||||
|
||||
def media_translation_sidecar_paths(project_id) do
|
||||
Repo.all(from translation in MediaTranslation, where: translation.project_id == ^project_id)
|
||||
|> Enum.map(&media_translation_sidecar_path(project_id, &1))
|
||||
|> Enum.reject(&is_nil/1)
|
||||
end
|
||||
|
||||
def media_translation_sidecar_path(project_id, translation) do
|
||||
case Repo.one(
|
||||
from media in Media,
|
||||
where: media.project_id == ^project_id and media.id == ^translation.translation_for,
|
||||
select: media.file_path
|
||||
) do
|
||||
nil -> nil
|
||||
file_path -> "#{file_path}.#{translation.language}.meta"
|
||||
end
|
||||
end
|
||||
end
|
||||
45
lib/bds/maintenance/progress.ex
Normal file
45
lib/bds/maintenance/progress.ex
Normal file
@@ -0,0 +1,45 @@
|
||||
defmodule BDS.Maintenance.Progress do
|
||||
@moduledoc false
|
||||
|
||||
def progress_callback(opts) do
|
||||
case Keyword.get(opts, :on_progress) do
|
||||
callback when is_function(callback, 2) -> callback
|
||||
_other -> nil
|
||||
end
|
||||
end
|
||||
|
||||
def report_metadata_diff_phase(nil, _current, _total, _label), do: :ok
|
||||
|
||||
def report_metadata_diff_phase(callback, current, total, label) do
|
||||
value = if total <= 1, do: 0.0, else: (current - 1) / total
|
||||
callback.(value, "#{label} (#{current}/#{total})")
|
||||
:ok
|
||||
end
|
||||
|
||||
def report_metadata_diff_complete(nil), do: :ok
|
||||
|
||||
def report_metadata_diff_complete(callback) do
|
||||
callback.(1.0, "Metadata diff complete")
|
||||
:ok
|
||||
end
|
||||
|
||||
def report_started(nil, _total, _label), do: :ok
|
||||
|
||||
def report_started(callback, 0, label) do
|
||||
callback.(1.0, label)
|
||||
:ok
|
||||
end
|
||||
|
||||
def report_started(callback, total, label) do
|
||||
callback.(0.05, "#{label} (0/#{total})")
|
||||
:ok
|
||||
end
|
||||
|
||||
def report_progress(nil, _current, _total, _label), do: :ok
|
||||
def report_progress(_callback, _current, 0, _label), do: :ok
|
||||
|
||||
def report_progress(callback, current, total, label) do
|
||||
callback.(0.05 + 0.95 * (current / total), "#{label} (#{current}/#{total})")
|
||||
:ok
|
||||
end
|
||||
end
|
||||
145
lib/bds/maintenance/repair.ex
Normal file
145
lib/bds/maintenance/repair.ex
Normal file
@@ -0,0 +1,145 @@
|
||||
defmodule BDS.Maintenance.Repair do
|
||||
@moduledoc false
|
||||
|
||||
import BDS.Maintenance.FileScan,
|
||||
only: [
|
||||
canonical_media_sidecar?: 1,
|
||||
translation_post_file?: 1,
|
||||
translation_media_sidecar?: 1
|
||||
]
|
||||
|
||||
import BDS.Maintenance.Progress, only: [report_progress: 4]
|
||||
|
||||
alias BDS.Embeddings
|
||||
alias BDS.Metadata
|
||||
|
||||
def normalize_entity_type(:post), do: :post
|
||||
def normalize_entity_type(:media), do: :media
|
||||
def normalize_entity_type(:script), do: :script
|
||||
def normalize_entity_type(:template), do: :template
|
||||
def normalize_entity_type(:embedding), do: :embedding
|
||||
def normalize_entity_type("post"), do: :post
|
||||
def normalize_entity_type("media"), do: :media
|
||||
def normalize_entity_type("script"), do: :script
|
||||
def normalize_entity_type("template"), do: :template
|
||||
def normalize_entity_type("embedding"), do: :embedding
|
||||
def normalize_entity_type("embeddings"), do: :embedding
|
||||
def normalize_entity_type(_entity_type), do: :unsupported
|
||||
|
||||
def normalize_repair_direction(:file_to_db), do: :file_to_db
|
||||
def normalize_repair_direction(:db_to_file), do: :db_to_file
|
||||
def normalize_repair_direction("file_to_db"), do: :file_to_db
|
||||
def normalize_repair_direction("db_to_file"), do: :db_to_file
|
||||
def normalize_repair_direction(_direction), do: :unsupported
|
||||
|
||||
def repair_metadata_diff_item(project_id, direction, item) do
|
||||
entity_type = Map.get(item, :entity_type) || Map.get(item, "entity_type")
|
||||
entity_id = Map.get(item, :entity_id) || Map.get(item, "entity_id")
|
||||
|
||||
case {normalize_repair_direction(direction), entity_type} do
|
||||
{:file_to_db, entity_type} when entity_type in ["project", "categories", "category_meta", "publishing"] ->
|
||||
Metadata.sync_project_metadata_from_filesystem(project_id)
|
||||
|
||||
{:db_to_file, entity_type} when entity_type in ["project", "categories", "category_meta", "publishing"] ->
|
||||
Metadata.flush_project_metadata_to_filesystem(project_id)
|
||||
|
||||
{:file_to_db, "post"} -> BDS.Posts.sync_post_from_file(entity_id)
|
||||
{:db_to_file, "post"} -> BDS.Posts.rewrite_published_post(entity_id)
|
||||
{:file_to_db, "post_translation"} -> BDS.Posts.sync_post_translation_from_file(entity_id)
|
||||
{:db_to_file, "post_translation"} -> BDS.Posts.rewrite_published_post_translation(entity_id)
|
||||
{:file_to_db, "media"} -> BDS.Media.sync_media_from_sidecar(entity_id)
|
||||
{:db_to_file, "media"} -> BDS.Media.sync_media_sidecar(entity_id)
|
||||
{:file_to_db, "media_translation"} -> BDS.Media.sync_media_translation_from_sidecar(entity_id)
|
||||
{:db_to_file, "media_translation"} -> BDS.Media.sync_media_translation_sidecar(entity_id)
|
||||
{:file_to_db, "script"} -> BDS.Scripts.sync_script_from_file(entity_id)
|
||||
{:db_to_file, "script"} -> BDS.Scripts.sync_published_script_file(entity_id)
|
||||
{:file_to_db, "template"} -> BDS.Templates.sync_template_from_file(entity_id)
|
||||
{:db_to_file, "template"} -> BDS.Templates.sync_published_template_file(entity_id)
|
||||
{:file_to_db, "embedding"} -> BDS.Embeddings.sync_post(entity_id)
|
||||
{:db_to_file, "embedding"} -> BDS.Embeddings.refresh_snapshot(project_id)
|
||||
_other -> {:error, :unsupported}
|
||||
end
|
||||
end
|
||||
|
||||
def repair_embedding_batch(project_id, direction, items, on_progress, total)
|
||||
when direction in [:file_to_db, :db_to_file] do
|
||||
if items != [] and Enum.all?(items, &(metadata_diff_item_entity_type(&1) == "embedding")) do
|
||||
result =
|
||||
case direction do
|
||||
:file_to_db ->
|
||||
post_ids = Enum.map(items, &metadata_diff_item_entity_id/1)
|
||||
|
||||
{:ok, repaired_post_ids} = Embeddings.repair_posts(project_id, post_ids)
|
||||
repaired_post_ids = MapSet.new(repaired_post_ids)
|
||||
|
||||
build_batch_repair_result(items, total, on_progress, fn item ->
|
||||
MapSet.member?(repaired_post_ids, metadata_diff_item_entity_id(item))
|
||||
end)
|
||||
|
||||
:db_to_file ->
|
||||
repaired? = Embeddings.refresh_snapshot(project_id) == :ok
|
||||
build_batch_repair_result(items, total, on_progress, fn _item -> repaired? end)
|
||||
end
|
||||
|
||||
{:ok, result}
|
||||
else
|
||||
:unsupported
|
||||
end
|
||||
end
|
||||
|
||||
def repair_embedding_batch(_project_id, _direction, _items, _on_progress, _total), do: :unsupported
|
||||
|
||||
defp build_batch_repair_result(items, total, on_progress, repaired?) do
|
||||
items
|
||||
|> Enum.with_index(1)
|
||||
|> Enum.reduce(%{repaired: 0, failed: 0}, fn {item, index}, acc ->
|
||||
next_acc =
|
||||
if repaired?.(item) do
|
||||
%{acc | repaired: acc.repaired + 1}
|
||||
else
|
||||
%{acc | failed: acc.failed + 1}
|
||||
end
|
||||
|
||||
:ok = report_progress(on_progress, index, total, "Repairing metadata differences")
|
||||
next_acc
|
||||
end)
|
||||
end
|
||||
|
||||
defp metadata_diff_item_entity_type(item) do
|
||||
Map.get(item, :entity_type) || Map.get(item, "entity_type")
|
||||
end
|
||||
|
||||
defp metadata_diff_item_entity_id(item) do
|
||||
Map.get(item, :entity_id) || Map.get(item, "entity_id")
|
||||
end
|
||||
|
||||
def import_metadata_diff_orphan(project_id, orphan) do
|
||||
file_path = Map.get(orphan, :file_path) || Map.get(orphan, "file_path")
|
||||
|
||||
cond do
|
||||
is_nil(file_path) ->
|
||||
{:error, :not_found}
|
||||
|
||||
translation_post_file?(file_path) ->
|
||||
BDS.Posts.import_orphan_post_translation_file(project_id, file_path)
|
||||
|
||||
String.ends_with?(file_path, ".md") ->
|
||||
BDS.Posts.import_orphan_post_file(project_id, file_path)
|
||||
|
||||
translation_media_sidecar?(file_path) ->
|
||||
BDS.Media.import_orphan_media_translation_sidecar(project_id, file_path)
|
||||
|
||||
canonical_media_sidecar?(file_path) and String.ends_with?(file_path, ".meta") ->
|
||||
BDS.Media.import_orphan_media_sidecar(project_id, file_path)
|
||||
|
||||
String.ends_with?(file_path, ".lua") ->
|
||||
BDS.Scripts.import_orphan_script_file(project_id, file_path)
|
||||
|
||||
String.ends_with?(file_path, ".liquid") ->
|
||||
BDS.Templates.import_orphan_template_file(project_id, file_path)
|
||||
|
||||
true ->
|
||||
{:error, :unsupported}
|
||||
end
|
||||
end
|
||||
end
|
||||
769
lib/bds/media.ex
769
lib/bds/media.ex
@@ -1,18 +1,37 @@
|
||||
defmodule BDS.Media do
|
||||
@moduledoc false
|
||||
|
||||
import BDS.Media.FileOps,
|
||||
only: [
|
||||
attr: 2,
|
||||
delete_file_if_present: 2,
|
||||
detect_mime: 1,
|
||||
image_dimensions: 2,
|
||||
maybe_put: 3,
|
||||
media_file_path: 2
|
||||
]
|
||||
|
||||
import BDS.Media.Sidecars,
|
||||
only: [
|
||||
translation_sidecar_path: 2,
|
||||
write_sidecar: 2,
|
||||
write_translation_sidecar: 3
|
||||
]
|
||||
|
||||
import BDS.Media.Thumbnails,
|
||||
only: [
|
||||
delete_thumbnail_files: 2,
|
||||
ensure_thumbnails: 2
|
||||
]
|
||||
|
||||
import Ecto.Query
|
||||
|
||||
alias BDS.DocumentFields
|
||||
alias BDS.Media.Media
|
||||
alias BDS.Media.Translation
|
||||
alias BDS.Persistence
|
||||
alias BDS.Posts.PostMedia
|
||||
alias BDS.Projects
|
||||
alias BDS.Rebuild
|
||||
alias BDS.Repo
|
||||
alias BDS.Search
|
||||
alias BDS.Sidecar
|
||||
|
||||
@typedoc "An attribute map that may use atom or string keys."
|
||||
@type attrs :: %{optional(atom()) => term(), optional(String.t()) => term()}
|
||||
@@ -20,6 +39,28 @@ defmodule BDS.Media do
|
||||
@typedoc "Options accepted by long-running rebuild operations."
|
||||
@type rebuild_opts :: keyword()
|
||||
|
||||
# Public API delegations to submodules
|
||||
defdelegate thumbnail_paths(media), to: BDS.Media.Thumbnails
|
||||
defdelegate regenerate_thumbnails(media_id), to: BDS.Media.Thumbnails
|
||||
defdelegate regenerate_missing_thumbnails(project_id), to: BDS.Media.Thumbnails
|
||||
defdelegate regenerate_missing_thumbnails(project_id, opts), to: BDS.Media.Thumbnails
|
||||
|
||||
defdelegate sync_media_sidecar(media_id), to: BDS.Media.Sidecars
|
||||
defdelegate sync_media_from_sidecar(media_id), to: BDS.Media.Sidecars
|
||||
defdelegate sync_media_translation_sidecar(translation_id), to: BDS.Media.Sidecars
|
||||
defdelegate sync_media_translation_from_sidecar(translation_id), to: BDS.Media.Sidecars
|
||||
defdelegate import_orphan_media_sidecar(project_id, relative_path), to: BDS.Media.Sidecars
|
||||
|
||||
defdelegate import_orphan_media_translation_sidecar(project_id, relative_path),
|
||||
to: BDS.Media.Sidecars
|
||||
|
||||
defdelegate list_linked_posts(media_id), to: BDS.Media.Linking
|
||||
defdelegate link_media_to_post(media_id, post_id), to: BDS.Media.Linking
|
||||
defdelegate unlink_media_from_post(media_id, post_id), to: BDS.Media.Linking
|
||||
|
||||
defdelegate rebuild_media_from_files(project_id), to: BDS.Media.Rebuilder
|
||||
defdelegate rebuild_media_from_files(project_id, opts), to: BDS.Media.Rebuilder
|
||||
|
||||
@spec import_media(attrs()) :: {:ok, Media.t()} | {:error, Ecto.Changeset.t() | term()}
|
||||
def import_media(attrs) do
|
||||
project = Projects.get_project!(attr(attrs, :project_id))
|
||||
@@ -112,129 +153,6 @@ defmodule BDS.Media do
|
||||
end
|
||||
end
|
||||
|
||||
@spec sync_media_sidecar(String.t()) :: :ok | {:error, :not_found | term()}
|
||||
def sync_media_sidecar(media_id) do
|
||||
case Repo.get(Media, media_id) do
|
||||
nil ->
|
||||
{:error, :not_found}
|
||||
|
||||
media ->
|
||||
project = Projects.get_project!(media.project_id)
|
||||
:ok = write_sidecar(project, media)
|
||||
:ok
|
||||
end
|
||||
end
|
||||
|
||||
@spec sync_media_from_sidecar(String.t()) ::
|
||||
{:ok, Media.t()} | {:error, :not_found | term()}
|
||||
def sync_media_from_sidecar(media_id) do
|
||||
case Repo.get(Media, media_id) do
|
||||
nil ->
|
||||
{:error, :not_found}
|
||||
|
||||
%Media{} = media ->
|
||||
project = Projects.get_project!(media.project_id)
|
||||
sidecar_path = Path.join(Projects.project_data_dir(project), media.sidecar_path)
|
||||
|
||||
if File.exists?(sidecar_path) do
|
||||
{:ok, upsert_media_from_sidecar(project, parse_canonical_sidecar(project, sidecar_path), sync_search: true)}
|
||||
else
|
||||
{:error, :not_found}
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
@spec sync_media_translation_sidecar(String.t()) ::
|
||||
{:ok, Translation.t()} | {:error, :not_found | term()}
|
||||
def sync_media_translation_sidecar(translation_id) do
|
||||
case Repo.get(Translation, translation_id) do
|
||||
nil ->
|
||||
{:error, :not_found}
|
||||
|
||||
%Translation{} = translation ->
|
||||
media = Repo.get!(Media, translation.translation_for)
|
||||
project = Projects.get_project!(media.project_id)
|
||||
:ok = write_translation_sidecar(project, media, translation)
|
||||
{:ok, translation}
|
||||
end
|
||||
end
|
||||
|
||||
@spec sync_media_translation_from_sidecar(String.t()) ::
|
||||
{:ok, Translation.t()} | {:error, :not_found | term()}
|
||||
def sync_media_translation_from_sidecar(translation_id) do
|
||||
case Repo.get(Translation, translation_id) do
|
||||
nil ->
|
||||
{:error, :not_found}
|
||||
|
||||
%Translation{} = translation ->
|
||||
media = Repo.get!(Media, translation.translation_for)
|
||||
project = Projects.get_project!(media.project_id)
|
||||
sidecar_path = Path.join(Projects.project_data_dir(project), translation_sidecar_path(media, translation.language))
|
||||
|
||||
if File.exists?(sidecar_path) do
|
||||
sidecar = parse_translation_sidecar(sidecar_path)
|
||||
|
||||
case upsert_media_translation(media.id, DocumentFields.fetch!(sidecar.fields, "language"), %{
|
||||
title: DocumentFields.get(sidecar.fields, "title"),
|
||||
alt: DocumentFields.get(sidecar.fields, "alt"),
|
||||
caption: DocumentFields.get(sidecar.fields, "caption")
|
||||
}) do
|
||||
{:ok, updated_translation} -> {:ok, updated_translation}
|
||||
error -> error
|
||||
end
|
||||
else
|
||||
{:error, :not_found}
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
@spec import_orphan_media_sidecar(String.t(), String.t()) ::
|
||||
{:ok, Media.t()} | {:error, term()}
|
||||
def import_orphan_media_sidecar(project_id, relative_path) do
|
||||
project = Projects.get_project!(project_id)
|
||||
sidecar_path = Path.join(Projects.project_data_dir(project), relative_path)
|
||||
|
||||
if File.exists?(sidecar_path) do
|
||||
{:ok, upsert_media_from_sidecar(project, parse_canonical_sidecar(project, sidecar_path), sync_search: true)}
|
||||
else
|
||||
{:error, :not_found}
|
||||
end
|
||||
end
|
||||
|
||||
@spec import_orphan_media_translation_sidecar(String.t(), String.t()) ::
|
||||
{:ok, Translation.t()} | {:error, term()}
|
||||
def import_orphan_media_translation_sidecar(project_id, relative_path) do
|
||||
project = Projects.get_project!(project_id)
|
||||
sidecar_path = Path.join(Projects.project_data_dir(project), relative_path)
|
||||
|
||||
if File.exists?(sidecar_path) do
|
||||
sidecar = parse_translation_sidecar(sidecar_path)
|
||||
|
||||
case Repo.get(Media, DocumentFields.get(sidecar.fields, "translationFor")) do
|
||||
nil ->
|
||||
{:error, :not_found}
|
||||
|
||||
media ->
|
||||
case Repo.get_by(Translation,
|
||||
translation_for: media.id,
|
||||
language: DocumentFields.fetch!(sidecar.fields, "language")
|
||||
) do
|
||||
nil ->
|
||||
upsert_media_translation(media.id, DocumentFields.fetch!(sidecar.fields, "language"), %{
|
||||
title: DocumentFields.get(sidecar.fields, "title"),
|
||||
alt: DocumentFields.get(sidecar.fields, "alt"),
|
||||
caption: DocumentFields.get(sidecar.fields, "caption")
|
||||
})
|
||||
|
||||
_translation ->
|
||||
{:error, :conflict}
|
||||
end
|
||||
end
|
||||
else
|
||||
{:error, :not_found}
|
||||
end
|
||||
end
|
||||
|
||||
@spec delete_media(String.t()) :: {:ok, :deleted} | {:error, :not_found}
|
||||
def delete_media(media_id) do
|
||||
case Repo.get(Media, media_id) do
|
||||
@@ -328,7 +246,11 @@ defmodule BDS.Media do
|
||||
|
||||
case Repo.transaction(fn -> Repo.delete!(translation) end) do
|
||||
{:ok, _deleted} ->
|
||||
delete_file_if_present(media.project_id, translation_sidecar_path(media, normalized_language))
|
||||
delete_file_if_present(
|
||||
media.project_id,
|
||||
translation_sidecar_path(media, normalized_language)
|
||||
)
|
||||
|
||||
:ok = Search.sync_media(media)
|
||||
:ok = write_sidecar(project, media)
|
||||
{:ok, true}
|
||||
@@ -399,595 +321,4 @@ defmodule BDS.Media do
|
||||
order_by: [asc: translation.language]
|
||||
)
|
||||
end
|
||||
|
||||
@spec list_linked_posts(String.t()) :: [%{post_id: String.t(), title: String.t(), sort_order: integer()}]
|
||||
def list_linked_posts(media_id) when is_binary(media_id) do
|
||||
Repo.all(
|
||||
from post in BDS.Posts.Post,
|
||||
join: pm in PostMedia,
|
||||
on: pm.post_id == post.id,
|
||||
where: pm.media_id == ^media_id,
|
||||
order_by: [asc: pm.sort_order, asc: post.updated_at],
|
||||
select: %{
|
||||
post_id: post.id,
|
||||
title: fragment("COALESCE(?, ?, ?)", post.title, post.slug, post.id),
|
||||
sort_order: pm.sort_order
|
||||
}
|
||||
)
|
||||
end
|
||||
|
||||
@spec link_media_to_post(String.t(), String.t()) ::
|
||||
{:ok, :linked} | {:error, :not_found | term()}
|
||||
def link_media_to_post(media_id, post_id) when is_binary(media_id) and is_binary(post_id) do
|
||||
case {Repo.get(Media, media_id), Repo.get(BDS.Posts.Post, post_id)} do
|
||||
{nil, _post} ->
|
||||
{:error, :not_found}
|
||||
|
||||
{_media, nil} ->
|
||||
{:error, :not_found}
|
||||
|
||||
{%Media{} = media, %BDS.Posts.Post{} = post} ->
|
||||
project = Projects.get_project!(media.project_id)
|
||||
|
||||
case Repo.transaction(fn ->
|
||||
if Repo.exists?(from pm in PostMedia, where: pm.post_id == ^post.id and pm.media_id == ^media.id) do
|
||||
:already_linked
|
||||
else
|
||||
sort_order = next_sort_order(media.id)
|
||||
|
||||
%PostMedia{}
|
||||
|> PostMedia.changeset(%{
|
||||
id: Ecto.UUID.generate(),
|
||||
project_id: media.project_id,
|
||||
post_id: post.id,
|
||||
media_id: media.id,
|
||||
sort_order: sort_order,
|
||||
created_at: Persistence.now_ms()
|
||||
})
|
||||
|> Repo.insert!()
|
||||
|
||||
:linked
|
||||
end
|
||||
end) do
|
||||
{:ok, _result} ->
|
||||
:ok = write_sidecar(project, media)
|
||||
{:ok, :linked}
|
||||
|
||||
{:error, reason} ->
|
||||
{:error, reason}
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
@spec unlink_media_from_post(String.t(), String.t()) ::
|
||||
{:ok, :unlinked} | {:error, :not_found | term()}
|
||||
def unlink_media_from_post(media_id, post_id) when is_binary(media_id) and is_binary(post_id) do
|
||||
case Repo.get(Media, media_id) do
|
||||
nil ->
|
||||
{:error, :not_found}
|
||||
|
||||
%Media{} = media ->
|
||||
project = Projects.get_project!(media.project_id)
|
||||
|
||||
case Repo.transaction(fn ->
|
||||
{_count, _} =
|
||||
Repo.delete_all(
|
||||
from pm in PostMedia, where: pm.media_id == ^media.id and pm.post_id == ^post_id
|
||||
)
|
||||
|
||||
:ok
|
||||
end) do
|
||||
{:ok, :ok} ->
|
||||
:ok = write_sidecar(project, media)
|
||||
{:ok, :unlinked}
|
||||
|
||||
{:error, reason} ->
|
||||
{:error, reason}
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
@spec thumbnail_paths(Media.t()) :: %{required(atom()) => String.t()}
|
||||
def thumbnail_paths(%Media{id: id}) do
|
||||
prefix = String.slice(id, 0, 2)
|
||||
|
||||
%{
|
||||
small: Path.join(["thumbnails", prefix, "#{id}-small.webp"]),
|
||||
medium: Path.join(["thumbnails", prefix, "#{id}-medium.webp"]),
|
||||
large: Path.join(["thumbnails", prefix, "#{id}-large.webp"]),
|
||||
ai: Path.join(["thumbnails", prefix, "#{id}-ai.jpg"])
|
||||
}
|
||||
end
|
||||
|
||||
@spec regenerate_thumbnails(String.t()) :: {:ok, Media.t()} | {:error, :not_found | term()}
|
||||
def regenerate_thumbnails(media_id) do
|
||||
case Repo.get(Media, media_id) do
|
||||
nil ->
|
||||
{:error, :not_found}
|
||||
|
||||
media ->
|
||||
project = Projects.get_project!(media.project_id)
|
||||
:ok = ensure_thumbnails(project, media)
|
||||
{:ok, media}
|
||||
end
|
||||
end
|
||||
|
||||
@spec regenerate_missing_thumbnails(String.t(), rebuild_opts()) ::
|
||||
%{processed: non_neg_integer(), generated: non_neg_integer(), failed: non_neg_integer()}
|
||||
def regenerate_missing_thumbnails(project_id, opts \\ []) do
|
||||
project = Projects.get_project!(project_id)
|
||||
on_progress = progress_callback(opts)
|
||||
|
||||
media_items =
|
||||
Repo.all(
|
||||
from(media in Media,
|
||||
where: media.project_id == ^project_id,
|
||||
order_by: [asc: media.created_at]
|
||||
)
|
||||
)
|
||||
|> Enum.filter(fn media ->
|
||||
String.starts_with?(media.mime_type || "", "image/") and
|
||||
not String.contains?(media.mime_type || "", "svg")
|
||||
end)
|
||||
|
||||
total_media = length(media_items)
|
||||
:ok = report_rebuild_started(on_progress, total_media, "image assets")
|
||||
|
||||
media_items
|
||||
|> Enum.with_index(1)
|
||||
|> Enum.reduce(%{processed: 0, generated: 0, failed: 0}, fn {media, index}, acc ->
|
||||
missing_paths =
|
||||
media
|
||||
|> thumbnail_paths()
|
||||
|> Enum.map(fn {_size, relative_path} -> Path.join(Projects.project_data_dir(project), relative_path) end)
|
||||
|> Enum.reject(&File.exists?/1)
|
||||
|
||||
next_acc =
|
||||
if missing_paths == [] do
|
||||
%{acc | processed: acc.processed + 1}
|
||||
else
|
||||
try do
|
||||
:ok = ensure_thumbnails(project, media)
|
||||
|
||||
%{
|
||||
processed: acc.processed + 1,
|
||||
generated: acc.generated + length(missing_paths),
|
||||
failed: acc.failed
|
||||
}
|
||||
rescue
|
||||
_error ->
|
||||
%{acc | processed: acc.processed + 1, failed: acc.failed + 1}
|
||||
end
|
||||
end
|
||||
|
||||
:ok = report_rebuild_progress(on_progress, index, total_media, "image assets")
|
||||
next_acc
|
||||
end)
|
||||
end
|
||||
|
||||
@spec rebuild_media_from_files(String.t(), rebuild_opts()) :: {:ok, [Media.t()]}
|
||||
def rebuild_media_from_files(project_id, opts \\ []) do
|
||||
project = Projects.get_project!(project_id)
|
||||
on_progress = progress_callback(opts)
|
||||
|
||||
canonical_sidecars =
|
||||
project
|
||||
|> Projects.project_data_dir()
|
||||
|> Path.join("media")
|
||||
|> list_matching_files("*.meta")
|
||||
|> Enum.filter(&canonical_sidecar?/1)
|
||||
|> Enum.filter(&binary_exists_for_sidecar?/1)
|
||||
|> Rebuild.parallel_map(&parse_canonical_sidecar(project, &1))
|
||||
|
||||
translation_sidecars =
|
||||
project
|
||||
|> Projects.project_data_dir()
|
||||
|> Path.join("media")
|
||||
|> list_matching_files("*.meta")
|
||||
|> Enum.filter(&translation_sidecar?/1)
|
||||
|> Rebuild.parallel_map(&parse_translation_sidecar(&1))
|
||||
|
||||
total_files = length(canonical_sidecars) + length(translation_sidecars)
|
||||
:ok = report_rebuild_started(on_progress, total_files, "media files")
|
||||
|
||||
media_items =
|
||||
canonical_sidecars
|
||||
|> Enum.with_index(1)
|
||||
|> Enum.map(fn {sidecar, index} ->
|
||||
media = upsert_media_from_sidecar(project, sidecar, sync_search: false)
|
||||
:ok = report_rebuild_progress(on_progress, index, total_files, "media files")
|
||||
media
|
||||
end)
|
||||
|
||||
canonical_media_by_binary_path =
|
||||
Map.new(media_items, fn media ->
|
||||
{Path.join(Projects.project_data_dir(project), media.file_path), media}
|
||||
end)
|
||||
|
||||
translation_sidecars
|
||||
|> Enum.with_index(length(canonical_sidecars) + 1)
|
||||
|> Enum.each(fn {sidecar, index} ->
|
||||
upsert_translation_from_sidecar(project, canonical_media_by_binary_path, sidecar, sync_search: false)
|
||||
:ok = report_rebuild_progress(on_progress, index, total_files, "media files")
|
||||
end)
|
||||
|
||||
if Keyword.get(opts, :reindex_search, true) do
|
||||
:ok = report_rebuild_phase(on_progress, 0.99, "Refreshing media search index")
|
||||
:ok =
|
||||
Search.reindex_media(project.id,
|
||||
on_progress: scaled_progress_reporter(on_progress, 0.99, 1.0)
|
||||
)
|
||||
end
|
||||
|
||||
{:ok, media_items}
|
||||
end
|
||||
|
||||
defp upsert_media_from_sidecar(project, sidecar, opts) do
|
||||
now = Persistence.now_ms()
|
||||
|
||||
attrs = %{
|
||||
id: DocumentFields.get(sidecar.fields, "id") || Ecto.UUID.generate(),
|
||||
project_id: project.id,
|
||||
filename: sidecar.filename,
|
||||
original_name: DocumentFields.get(sidecar.fields, "originalName") || sidecar.filename,
|
||||
mime_type: DocumentFields.get(sidecar.fields, "mimeType") || detect_mime(sidecar.filename),
|
||||
size: DocumentFields.get(sidecar.fields, "size", 0),
|
||||
width: blank_to_nil(DocumentFields.get(sidecar.fields, "width")),
|
||||
height: blank_to_nil(DocumentFields.get(sidecar.fields, "height")),
|
||||
title: DocumentFields.get(sidecar.fields, "title"),
|
||||
alt: DocumentFields.get(sidecar.fields, "alt"),
|
||||
caption: DocumentFields.get(sidecar.fields, "caption"),
|
||||
author: DocumentFields.get(sidecar.fields, "author"),
|
||||
language: DocumentFields.get(sidecar.fields, "language"),
|
||||
file_path: sidecar.relative_file_path,
|
||||
sidecar_path: sidecar.relative_sidecar_path,
|
||||
checksum: nil,
|
||||
tags: DocumentFields.get(sidecar.fields, "tags", []),
|
||||
created_at: DocumentFields.get(sidecar.fields, "createdAt", now),
|
||||
updated_at: DocumentFields.get(sidecar.fields, "updatedAt", now)
|
||||
}
|
||||
|
||||
media =
|
||||
Repo.get(Media, attrs.id) ||
|
||||
Repo.get_by(Media, project_id: project.id, file_path: sidecar.relative_file_path) || %Media{}
|
||||
|
||||
media =
|
||||
media
|
||||
|> Media.changeset(attrs)
|
||||
|> Repo.insert_or_update!()
|
||||
|
||||
if Keyword.get(opts, :sync_search, true) do
|
||||
:ok = Search.sync_media(media)
|
||||
end
|
||||
|
||||
media
|
||||
end
|
||||
|
||||
defp write_sidecar(project, media) do
|
||||
path = Path.join(Projects.project_data_dir(project), media.sidecar_path)
|
||||
:ok = File.mkdir_p(Path.dirname(path))
|
||||
|
||||
atomic_write(
|
||||
path,
|
||||
Sidecar.serialize_document([
|
||||
{"id", media.id},
|
||||
{"originalName", media.original_name},
|
||||
{"mimeType", media.mime_type},
|
||||
{"size", media.size},
|
||||
{"width", media.width},
|
||||
{"height", media.height},
|
||||
{"title", media.title},
|
||||
{"alt", media.alt},
|
||||
{"caption", media.caption},
|
||||
{"author", media.author},
|
||||
{"language", media.language},
|
||||
{"createdAt", media.created_at},
|
||||
{"updatedAt", media.updated_at},
|
||||
{"linkedPostIds", linked_post_ids(media.id)},
|
||||
{"tags", media.tags || []}
|
||||
])
|
||||
)
|
||||
end
|
||||
|
||||
defp write_translation_sidecar(project, media, translation) do
|
||||
path =
|
||||
Path.join(
|
||||
Projects.project_data_dir(project),
|
||||
translation_sidecar_path(media, translation.language)
|
||||
)
|
||||
|
||||
:ok = File.mkdir_p(Path.dirname(path))
|
||||
|
||||
atomic_write(
|
||||
path,
|
||||
Sidecar.serialize_document([
|
||||
{"translationFor", media.id},
|
||||
{"language", translation.language},
|
||||
{"title", translation.title},
|
||||
{"alt", translation.alt},
|
||||
{"caption", translation.caption}
|
||||
])
|
||||
)
|
||||
end
|
||||
|
||||
defp upsert_translation_from_sidecar(project, canonical_media_by_binary_path, sidecar, opts) do
|
||||
case Map.get(canonical_media_by_binary_path, sidecar.binary_path) do
|
||||
nil ->
|
||||
:skip
|
||||
|
||||
media ->
|
||||
now = Persistence.now_ms()
|
||||
language = DocumentFields.fetch!(sidecar.fields, "language")
|
||||
|
||||
translation =
|
||||
Repo.get_by(Translation, translation_for: media.id, language: language) ||
|
||||
%Translation{id: Ecto.UUID.generate(), created_at: now}
|
||||
|
||||
translation
|
||||
|> Translation.changeset(%{
|
||||
id: translation.id,
|
||||
project_id: project.id,
|
||||
translation_for: media.id,
|
||||
language: language,
|
||||
title: DocumentFields.get(sidecar.fields, "title"),
|
||||
alt: DocumentFields.get(sidecar.fields, "alt"),
|
||||
caption: DocumentFields.get(sidecar.fields, "caption"),
|
||||
created_at: translation.created_at || now,
|
||||
updated_at: now
|
||||
})
|
||||
|> Repo.insert_or_update!()
|
||||
|
||||
if Keyword.get(opts, :sync_search, true) do
|
||||
:ok = Search.sync_media(media.id)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
defp parse_canonical_sidecar(project, sidecar_path) do
|
||||
{:ok, fields} = sidecar_path |> File.read!() |> Sidecar.parse_document()
|
||||
relative_sidecar_path = Path.relative_to(sidecar_path, Projects.project_data_dir(project))
|
||||
relative_file_path = String.trim_trailing(relative_sidecar_path, ".meta")
|
||||
|
||||
%{
|
||||
fields: fields,
|
||||
relative_sidecar_path: relative_sidecar_path,
|
||||
relative_file_path: relative_file_path,
|
||||
filename: Path.basename(relative_file_path)
|
||||
}
|
||||
end
|
||||
|
||||
defp parse_translation_sidecar(sidecar_path) do
|
||||
{:ok, fields} = sidecar_path |> File.read!() |> Sidecar.parse_document()
|
||||
|
||||
%{
|
||||
fields: fields,
|
||||
binary_path: binary_path_for_translation_sidecar(sidecar_path)
|
||||
}
|
||||
end
|
||||
|
||||
defp ensure_thumbnails(project, media) do
|
||||
if image_mime?(media.mime_type) do
|
||||
source_path = Path.join(Projects.project_data_dir(project), media.file_path)
|
||||
|
||||
case Image.open(source_path) do
|
||||
{:ok, image} ->
|
||||
image
|
||||
|> Image.autorotate!()
|
||||
|> write_all_thumbnails(project, media)
|
||||
|
||||
{:error, _reason} ->
|
||||
:ok
|
||||
end
|
||||
end
|
||||
|
||||
:ok
|
||||
end
|
||||
|
||||
defp write_all_thumbnails(image, project, media) do
|
||||
thumbnail_paths(media)
|
||||
|> Enum.each(fn {size, relative_path} ->
|
||||
destination = Path.join(Projects.project_data_dir(project), relative_path)
|
||||
:ok = File.mkdir_p(Path.dirname(destination))
|
||||
|
||||
image
|
||||
|> render_thumbnail(size)
|
||||
|> write_thumbnail(destination, size)
|
||||
end)
|
||||
|
||||
:ok
|
||||
end
|
||||
|
||||
defp render_thumbnail(image, :small), do: bounded_thumbnail(image, 150, 150)
|
||||
defp render_thumbnail(image, :medium), do: bounded_thumbnail(image, 400, 400)
|
||||
defp render_thumbnail(image, :large), do: bounded_thumbnail(image, 800, 800)
|
||||
|
||||
defp render_thumbnail(image, :ai) do
|
||||
image
|
||||
|> Image.thumbnail!("448x448", fit: :contain, resize: :both, autorotate: false)
|
||||
|> Image.embed!(448, 448, x: :center, y: :center, background_color: :black)
|
||||
end
|
||||
|
||||
defp bounded_thumbnail(image, width, height) do
|
||||
Image.thumbnail!(image, "#{width}x#{height}", fit: :contain, resize: :down, autorotate: false)
|
||||
end
|
||||
|
||||
defp write_thumbnail(image, destination, :ai) do
|
||||
flattened = Image.flatten!(image, background_color: :black)
|
||||
Image.write!(flattened, destination, quality: 85, strip_metadata: true)
|
||||
:ok
|
||||
end
|
||||
|
||||
defp write_thumbnail(image, destination, _size) do
|
||||
Image.write!(image, destination, quality: 80, strip_metadata: true)
|
||||
:ok
|
||||
end
|
||||
|
||||
defp delete_thumbnail_files(project_id, media) do
|
||||
Enum.each(Map.values(thumbnail_paths(media)), fn path ->
|
||||
delete_file_if_present(project_id, path)
|
||||
end)
|
||||
|
||||
:ok
|
||||
end
|
||||
|
||||
defp media_file_path(file_name, timestamp) do
|
||||
datetime = Persistence.from_unix_ms!(timestamp)
|
||||
year = Integer.to_string(datetime.year)
|
||||
month = datetime.month |> Integer.to_string() |> String.pad_leading(2, "0")
|
||||
Path.join(["media", year, month, file_name])
|
||||
end
|
||||
|
||||
defp detect_mime(file_name) do
|
||||
case String.downcase(Path.extname(file_name)) do
|
||||
".txt" -> "text/plain"
|
||||
".md" -> "text/markdown"
|
||||
".jpg" -> "image/jpeg"
|
||||
".jpeg" -> "image/jpeg"
|
||||
".png" -> "image/png"
|
||||
".gif" -> "image/gif"
|
||||
".webp" -> "image/webp"
|
||||
".tif" -> "image/tiff"
|
||||
".tiff" -> "image/tiff"
|
||||
".bmp" -> "image/bmp"
|
||||
".heic" -> "image/heic"
|
||||
".heif" -> "image/heif"
|
||||
_ -> "application/octet-stream"
|
||||
end
|
||||
end
|
||||
|
||||
defp image_dimensions(source_path, mime_type) do
|
||||
if image_mime?(mime_type) do
|
||||
case Image.open(source_path) do
|
||||
{:ok, image} -> {Image.width(image), Image.height(image)}
|
||||
{:error, _reason} -> {nil, nil}
|
||||
end
|
||||
else
|
||||
{nil, nil}
|
||||
end
|
||||
end
|
||||
|
||||
defp image_mime?(mime_type), do: String.starts_with?(mime_type || "", "image/")
|
||||
|
||||
defp binary_exists_for_sidecar?(sidecar_path) do
|
||||
sidecar_path
|
||||
|> String.trim_trailing(".meta")
|
||||
|> File.exists?()
|
||||
end
|
||||
|
||||
defp list_matching_files(dir, pattern) do
|
||||
if File.dir?(dir) do
|
||||
Path.join([dir, "**", pattern])
|
||||
|> Path.wildcard()
|
||||
|> Enum.sort()
|
||||
else
|
||||
[]
|
||||
end
|
||||
end
|
||||
|
||||
defp canonical_sidecar?(sidecar_path) do
|
||||
not translation_sidecar?(sidecar_path)
|
||||
end
|
||||
|
||||
defp translation_sidecar?(sidecar_path) do
|
||||
Regex.match?(~r/\.[a-z]{2}\.meta$/i, sidecar_path)
|
||||
end
|
||||
|
||||
defp binary_path_for_translation_sidecar(sidecar_path) do
|
||||
Regex.replace(~r/\.[a-z]{2}\.meta$/i, sidecar_path, "")
|
||||
end
|
||||
|
||||
defp translation_sidecar_path(media, language), do: "#{media.file_path}.#{language}.meta"
|
||||
|
||||
defp delete_file_if_present(project_id, relative_path) do
|
||||
project = Projects.get_project!(project_id)
|
||||
full_path = Path.join(Projects.project_data_dir(project), relative_path)
|
||||
|
||||
case File.rm(full_path) do
|
||||
:ok -> :ok
|
||||
{:error, :enoent} -> :ok
|
||||
{:error, reason} -> {:error, reason}
|
||||
end
|
||||
end
|
||||
|
||||
defp atomic_write(path, contents) do
|
||||
Persistence.atomic_write(path, contents)
|
||||
end
|
||||
|
||||
defp linked_post_ids(media_id) do
|
||||
Repo.all(
|
||||
from pm in PostMedia,
|
||||
where: pm.media_id == ^media_id,
|
||||
order_by: [asc: pm.sort_order, asc: pm.post_id],
|
||||
select: pm.post_id
|
||||
)
|
||||
end
|
||||
|
||||
defp next_sort_order(media_id) do
|
||||
case Repo.one(
|
||||
from pm in PostMedia,
|
||||
where: pm.media_id == ^media_id,
|
||||
select: max(pm.sort_order)
|
||||
) do
|
||||
value when is_integer(value) -> value + 1
|
||||
_other -> 0
|
||||
end
|
||||
end
|
||||
|
||||
defp blank_to_nil(nil), do: nil
|
||||
defp blank_to_nil(""), do: nil
|
||||
defp blank_to_nil(value), do: value
|
||||
|
||||
defp maybe_put(map, _key, nil), do: map
|
||||
defp maybe_put(map, key, value), do: Map.put(map, key, value)
|
||||
|
||||
defp attr(attrs, key) do
|
||||
cond do
|
||||
Map.has_key?(attrs, key) -> Map.get(attrs, key)
|
||||
Map.has_key?(attrs, Atom.to_string(key)) -> Map.get(attrs, Atom.to_string(key))
|
||||
true -> nil
|
||||
end
|
||||
end
|
||||
|
||||
defp progress_callback(opts) do
|
||||
case Keyword.get(opts, :on_progress) do
|
||||
callback when is_function(callback, 2) -> callback
|
||||
_other -> nil
|
||||
end
|
||||
end
|
||||
|
||||
defp scaled_progress_reporter(nil, _start_value, _end_value), do: nil
|
||||
|
||||
defp scaled_progress_reporter(report, start_value, end_value) when is_function(report, 2) do
|
||||
fn value, message ->
|
||||
scaled_value = start_value + (end_value - start_value) * value
|
||||
report.(scaled_value, message)
|
||||
end
|
||||
end
|
||||
|
||||
defp report_rebuild_started(nil, _total, _label), do: :ok
|
||||
|
||||
defp report_rebuild_started(callback, 0, label) do
|
||||
callback.(1.0, "No #{label} found")
|
||||
:ok
|
||||
end
|
||||
|
||||
defp report_rebuild_started(callback, total, label) do
|
||||
callback.(0.05, "Rebuilding #{label} (0/#{total})")
|
||||
:ok
|
||||
end
|
||||
|
||||
defp report_rebuild_progress(nil, _current, _total, _label), do: :ok
|
||||
defp report_rebuild_progress(_callback, _current, 0, _label), do: :ok
|
||||
|
||||
defp report_rebuild_progress(callback, current, total, label) do
|
||||
callback.(0.05 + 0.95 * (current / total), "Rebuilding #{label} (#{current}/#{total})")
|
||||
:ok
|
||||
end
|
||||
|
||||
defp report_rebuild_phase(nil, _progress, _message), do: :ok
|
||||
|
||||
defp report_rebuild_phase(callback, progress, message) do
|
||||
callback.(progress, message)
|
||||
:ok
|
||||
end
|
||||
end
|
||||
|
||||
150
lib/bds/media/file_ops.ex
Normal file
150
lib/bds/media/file_ops.ex
Normal file
@@ -0,0 +1,150 @@
|
||||
defmodule BDS.Media.FileOps do
|
||||
@moduledoc false
|
||||
|
||||
alias BDS.Persistence
|
||||
alias BDS.Projects
|
||||
|
||||
@typedoc "An attribute map that may use atom or string keys."
|
||||
@type attrs :: %{optional(atom()) => term(), optional(String.t()) => term()}
|
||||
|
||||
@spec attr(attrs(), atom()) :: term()
|
||||
def attr(attrs, key) do
|
||||
cond do
|
||||
Map.has_key?(attrs, key) -> Map.get(attrs, key)
|
||||
Map.has_key?(attrs, Atom.to_string(key)) -> Map.get(attrs, Atom.to_string(key))
|
||||
true -> nil
|
||||
end
|
||||
end
|
||||
|
||||
@spec maybe_put(map(), atom(), term()) :: map()
|
||||
def maybe_put(map, _key, nil), do: map
|
||||
def maybe_put(map, key, value), do: Map.put(map, key, value)
|
||||
|
||||
@spec blank_to_nil(term()) :: term()
|
||||
def blank_to_nil(nil), do: nil
|
||||
def blank_to_nil(""), do: nil
|
||||
def blank_to_nil(value), do: value
|
||||
|
||||
@spec atomic_write(Path.t(), iodata()) :: :ok | {:error, term()}
|
||||
def atomic_write(path, contents), do: Persistence.atomic_write(path, contents)
|
||||
|
||||
@spec delete_file_if_present(String.t(), Path.t()) :: :ok | {:error, term()}
|
||||
def delete_file_if_present(project_id, relative_path) do
|
||||
project = Projects.get_project!(project_id)
|
||||
full_path = Path.join(Projects.project_data_dir(project), relative_path)
|
||||
|
||||
case File.rm(full_path) do
|
||||
:ok -> :ok
|
||||
{:error, :enoent} -> :ok
|
||||
{:error, reason} -> {:error, reason}
|
||||
end
|
||||
end
|
||||
|
||||
@spec list_matching_files(Path.t(), String.t()) :: [Path.t()]
|
||||
def list_matching_files(dir, pattern) do
|
||||
if File.dir?(dir) do
|
||||
Path.join([dir, "**", pattern])
|
||||
|> Path.wildcard()
|
||||
|> Enum.sort()
|
||||
else
|
||||
[]
|
||||
end
|
||||
end
|
||||
|
||||
@spec media_file_path(String.t(), integer()) :: Path.t()
|
||||
def media_file_path(file_name, timestamp) do
|
||||
datetime = Persistence.from_unix_ms!(timestamp)
|
||||
year = Integer.to_string(datetime.year)
|
||||
month = datetime.month |> Integer.to_string() |> String.pad_leading(2, "0")
|
||||
Path.join(["media", year, month, file_name])
|
||||
end
|
||||
|
||||
@spec detect_mime(String.t()) :: String.t()
|
||||
def detect_mime(file_name) do
|
||||
case String.downcase(Path.extname(file_name)) do
|
||||
".txt" -> "text/plain"
|
||||
".md" -> "text/markdown"
|
||||
".jpg" -> "image/jpeg"
|
||||
".jpeg" -> "image/jpeg"
|
||||
".png" -> "image/png"
|
||||
".gif" -> "image/gif"
|
||||
".webp" -> "image/webp"
|
||||
".tif" -> "image/tiff"
|
||||
".tiff" -> "image/tiff"
|
||||
".bmp" -> "image/bmp"
|
||||
".heic" -> "image/heic"
|
||||
".heif" -> "image/heif"
|
||||
_ -> "application/octet-stream"
|
||||
end
|
||||
end
|
||||
|
||||
@spec image_dimensions(Path.t(), String.t() | nil) ::
|
||||
{non_neg_integer() | nil, non_neg_integer() | nil}
|
||||
def image_dimensions(source_path, mime_type) do
|
||||
if image_mime?(mime_type) do
|
||||
case Image.open(source_path) do
|
||||
{:ok, image} -> {Image.width(image), Image.height(image)}
|
||||
{:error, _reason} -> {nil, nil}
|
||||
end
|
||||
else
|
||||
{nil, nil}
|
||||
end
|
||||
end
|
||||
|
||||
@spec image_mime?(String.t() | nil) :: boolean()
|
||||
def image_mime?(mime_type), do: String.starts_with?(mime_type || "", "image/")
|
||||
|
||||
@spec progress_callback(keyword()) :: (float(), String.t() -> any()) | nil
|
||||
def progress_callback(opts) do
|
||||
case Keyword.get(opts, :on_progress) do
|
||||
callback when is_function(callback, 2) -> callback
|
||||
_other -> nil
|
||||
end
|
||||
end
|
||||
|
||||
@spec scaled_progress_reporter((float(), String.t() -> any()) | nil, float(), float()) ::
|
||||
(float(), String.t() -> any()) | nil
|
||||
def scaled_progress_reporter(nil, _start_value, _end_value), do: nil
|
||||
|
||||
def scaled_progress_reporter(report, start_value, end_value) when is_function(report, 2) do
|
||||
fn value, message ->
|
||||
scaled_value = start_value + (end_value - start_value) * value
|
||||
report.(scaled_value, message)
|
||||
end
|
||||
end
|
||||
|
||||
@spec report_rebuild_started((float(), String.t() -> any()) | nil, non_neg_integer(), String.t()) :: :ok
|
||||
def report_rebuild_started(nil, _total, _label), do: :ok
|
||||
|
||||
def report_rebuild_started(callback, 0, label) do
|
||||
callback.(1.0, "No #{label} found")
|
||||
:ok
|
||||
end
|
||||
|
||||
def report_rebuild_started(callback, total, label) do
|
||||
callback.(0.05, "Rebuilding #{label} (0/#{total})")
|
||||
:ok
|
||||
end
|
||||
|
||||
@spec report_rebuild_progress(
|
||||
(float(), String.t() -> any()) | nil,
|
||||
non_neg_integer(),
|
||||
non_neg_integer(),
|
||||
String.t()
|
||||
) :: :ok
|
||||
def report_rebuild_progress(nil, _current, _total, _label), do: :ok
|
||||
def report_rebuild_progress(_callback, _current, 0, _label), do: :ok
|
||||
|
||||
def report_rebuild_progress(callback, current, total, label) do
|
||||
callback.(0.05 + 0.95 * (current / total), "Rebuilding #{label} (#{current}/#{total})")
|
||||
:ok
|
||||
end
|
||||
|
||||
@spec report_rebuild_phase((float(), String.t() -> any()) | nil, float(), String.t()) :: :ok
|
||||
def report_rebuild_phase(nil, _progress, _message), do: :ok
|
||||
|
||||
def report_rebuild_phase(callback, progress, message) do
|
||||
callback.(progress, message)
|
||||
:ok
|
||||
end
|
||||
end
|
||||
125
lib/bds/media/linking.ex
Normal file
125
lib/bds/media/linking.ex
Normal file
@@ -0,0 +1,125 @@
|
||||
defmodule BDS.Media.Linking do
|
||||
@moduledoc false
|
||||
|
||||
import Ecto.Query
|
||||
|
||||
alias BDS.Media.Media
|
||||
alias BDS.Media.Sidecars
|
||||
alias BDS.Persistence
|
||||
alias BDS.Posts.PostMedia
|
||||
alias BDS.Projects
|
||||
alias BDS.Repo
|
||||
|
||||
@spec list_linked_posts(String.t()) ::
|
||||
[%{post_id: String.t(), title: String.t(), sort_order: integer()}]
|
||||
def list_linked_posts(media_id) when is_binary(media_id) do
|
||||
Repo.all(
|
||||
from post in BDS.Posts.Post,
|
||||
join: pm in PostMedia,
|
||||
on: pm.post_id == post.id,
|
||||
where: pm.media_id == ^media_id,
|
||||
order_by: [asc: pm.sort_order, asc: post.updated_at],
|
||||
select: %{
|
||||
post_id: post.id,
|
||||
title: fragment("COALESCE(?, ?, ?)", post.title, post.slug, post.id),
|
||||
sort_order: pm.sort_order
|
||||
}
|
||||
)
|
||||
end
|
||||
|
||||
@spec link_media_to_post(String.t(), String.t()) ::
|
||||
{:ok, :linked} | {:error, :not_found | term()}
|
||||
def link_media_to_post(media_id, post_id) when is_binary(media_id) and is_binary(post_id) do
|
||||
case {Repo.get(Media, media_id), Repo.get(BDS.Posts.Post, post_id)} do
|
||||
{nil, _post} ->
|
||||
{:error, :not_found}
|
||||
|
||||
{_media, nil} ->
|
||||
{:error, :not_found}
|
||||
|
||||
{%Media{} = media, %BDS.Posts.Post{} = post} ->
|
||||
project = Projects.get_project!(media.project_id)
|
||||
|
||||
case Repo.transaction(fn ->
|
||||
if Repo.exists?(
|
||||
from pm in PostMedia,
|
||||
where: pm.post_id == ^post.id and pm.media_id == ^media.id
|
||||
) do
|
||||
:already_linked
|
||||
else
|
||||
sort_order = next_sort_order(media.id)
|
||||
|
||||
%PostMedia{}
|
||||
|> PostMedia.changeset(%{
|
||||
id: Ecto.UUID.generate(),
|
||||
project_id: media.project_id,
|
||||
post_id: post.id,
|
||||
media_id: media.id,
|
||||
sort_order: sort_order,
|
||||
created_at: Persistence.now_ms()
|
||||
})
|
||||
|> Repo.insert!()
|
||||
|
||||
:linked
|
||||
end
|
||||
end) do
|
||||
{:ok, _result} ->
|
||||
:ok = Sidecars.write_sidecar(project, media)
|
||||
{:ok, :linked}
|
||||
|
||||
{:error, reason} ->
|
||||
{:error, reason}
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
@spec unlink_media_from_post(String.t(), String.t()) ::
|
||||
{:ok, :unlinked} | {:error, :not_found | term()}
|
||||
def unlink_media_from_post(media_id, post_id) when is_binary(media_id) and is_binary(post_id) do
|
||||
case Repo.get(Media, media_id) do
|
||||
nil ->
|
||||
{:error, :not_found}
|
||||
|
||||
%Media{} = media ->
|
||||
project = Projects.get_project!(media.project_id)
|
||||
|
||||
case Repo.transaction(fn ->
|
||||
{_count, _} =
|
||||
Repo.delete_all(
|
||||
from pm in PostMedia,
|
||||
where: pm.media_id == ^media.id and pm.post_id == ^post_id
|
||||
)
|
||||
|
||||
:ok
|
||||
end) do
|
||||
{:ok, :ok} ->
|
||||
:ok = Sidecars.write_sidecar(project, media)
|
||||
{:ok, :unlinked}
|
||||
|
||||
{:error, reason} ->
|
||||
{:error, reason}
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
@spec linked_post_ids(String.t()) :: [String.t()]
|
||||
def linked_post_ids(media_id) do
|
||||
Repo.all(
|
||||
from pm in PostMedia,
|
||||
where: pm.media_id == ^media_id,
|
||||
order_by: [asc: pm.sort_order, asc: pm.post_id],
|
||||
select: pm.post_id
|
||||
)
|
||||
end
|
||||
|
||||
defp next_sort_order(media_id) do
|
||||
case Repo.one(
|
||||
from pm in PostMedia,
|
||||
where: pm.media_id == ^media_id,
|
||||
select: max(pm.sort_order)
|
||||
) do
|
||||
value when is_integer(value) -> value + 1
|
||||
_other -> 0
|
||||
end
|
||||
end
|
||||
end
|
||||
82
lib/bds/media/rebuilder.ex
Normal file
82
lib/bds/media/rebuilder.ex
Normal file
@@ -0,0 +1,82 @@
|
||||
defmodule BDS.Media.Rebuilder do
|
||||
@moduledoc false
|
||||
|
||||
import BDS.Media.FileOps,
|
||||
only: [
|
||||
list_matching_files: 2,
|
||||
progress_callback: 1,
|
||||
report_rebuild_phase: 3,
|
||||
report_rebuild_progress: 4,
|
||||
report_rebuild_started: 3,
|
||||
scaled_progress_reporter: 3
|
||||
]
|
||||
|
||||
alias BDS.Media.Media
|
||||
alias BDS.Media.Sidecars
|
||||
alias BDS.Projects
|
||||
alias BDS.Rebuild
|
||||
alias BDS.Search
|
||||
|
||||
@type rebuild_opts :: keyword()
|
||||
|
||||
@spec rebuild_media_from_files(String.t(), rebuild_opts()) :: {:ok, [Media.t()]}
|
||||
def rebuild_media_from_files(project_id, opts \\ []) do
|
||||
project = Projects.get_project!(project_id)
|
||||
on_progress = progress_callback(opts)
|
||||
|
||||
canonical_sidecars =
|
||||
project
|
||||
|> Projects.project_data_dir()
|
||||
|> Path.join("media")
|
||||
|> list_matching_files("*.meta")
|
||||
|> Enum.filter(&Sidecars.canonical_sidecar?/1)
|
||||
|> Enum.filter(&Sidecars.binary_exists_for_sidecar?/1)
|
||||
|> Rebuild.parallel_map(&Sidecars.parse_canonical_sidecar(project, &1))
|
||||
|
||||
translation_sidecars =
|
||||
project
|
||||
|> Projects.project_data_dir()
|
||||
|> Path.join("media")
|
||||
|> list_matching_files("*.meta")
|
||||
|> Enum.filter(&Sidecars.translation_sidecar?/1)
|
||||
|> Rebuild.parallel_map(&Sidecars.parse_translation_sidecar(&1))
|
||||
|
||||
total_files = length(canonical_sidecars) + length(translation_sidecars)
|
||||
:ok = report_rebuild_started(on_progress, total_files, "media files")
|
||||
|
||||
media_items =
|
||||
canonical_sidecars
|
||||
|> Enum.with_index(1)
|
||||
|> Enum.map(fn {sidecar, index} ->
|
||||
media = Sidecars.upsert_media_from_sidecar(project, sidecar, sync_search: false)
|
||||
:ok = report_rebuild_progress(on_progress, index, total_files, "media files")
|
||||
media
|
||||
end)
|
||||
|
||||
canonical_media_by_binary_path =
|
||||
Map.new(media_items, fn media ->
|
||||
{Path.join(Projects.project_data_dir(project), media.file_path), media}
|
||||
end)
|
||||
|
||||
translation_sidecars
|
||||
|> Enum.with_index(length(canonical_sidecars) + 1)
|
||||
|> Enum.each(fn {sidecar, index} ->
|
||||
Sidecars.upsert_translation_from_sidecar(project, canonical_media_by_binary_path, sidecar,
|
||||
sync_search: false
|
||||
)
|
||||
|
||||
:ok = report_rebuild_progress(on_progress, index, total_files, "media files")
|
||||
end)
|
||||
|
||||
if Keyword.get(opts, :reindex_search, true) do
|
||||
:ok = report_rebuild_phase(on_progress, 0.99, "Refreshing media search index")
|
||||
|
||||
:ok =
|
||||
Search.reindex_media(project.id,
|
||||
on_progress: scaled_progress_reporter(on_progress, 0.99, 1.0)
|
||||
)
|
||||
end
|
||||
|
||||
{:ok, media_items}
|
||||
end
|
||||
end
|
||||
329
lib/bds/media/sidecars.ex
Normal file
329
lib/bds/media/sidecars.ex
Normal file
@@ -0,0 +1,329 @@
|
||||
defmodule BDS.Media.Sidecars do
|
||||
@moduledoc false
|
||||
|
||||
import BDS.Media.FileOps,
|
||||
only: [
|
||||
atomic_write: 2,
|
||||
blank_to_nil: 1,
|
||||
detect_mime: 1
|
||||
]
|
||||
|
||||
alias BDS.DocumentFields
|
||||
alias BDS.Media.Linking
|
||||
alias BDS.Media.Media
|
||||
alias BDS.Media.Translation
|
||||
alias BDS.Persistence
|
||||
alias BDS.Projects
|
||||
alias BDS.Repo
|
||||
alias BDS.Search
|
||||
alias BDS.Sidecar
|
||||
|
||||
@spec write_sidecar(BDS.Projects.Project.t(), Media.t()) :: :ok
|
||||
def write_sidecar(project, media) do
|
||||
path = Path.join(Projects.project_data_dir(project), media.sidecar_path)
|
||||
:ok = File.mkdir_p(Path.dirname(path))
|
||||
|
||||
atomic_write(
|
||||
path,
|
||||
Sidecar.serialize_document([
|
||||
{"id", media.id},
|
||||
{"originalName", media.original_name},
|
||||
{"mimeType", media.mime_type},
|
||||
{"size", media.size},
|
||||
{"width", media.width},
|
||||
{"height", media.height},
|
||||
{"title", media.title},
|
||||
{"alt", media.alt},
|
||||
{"caption", media.caption},
|
||||
{"author", media.author},
|
||||
{"language", media.language},
|
||||
{"createdAt", media.created_at},
|
||||
{"updatedAt", media.updated_at},
|
||||
{"linkedPostIds", Linking.linked_post_ids(media.id)},
|
||||
{"tags", media.tags || []}
|
||||
])
|
||||
)
|
||||
end
|
||||
|
||||
@spec write_translation_sidecar(BDS.Projects.Project.t(), Media.t(), Translation.t()) :: :ok
|
||||
def write_translation_sidecar(project, media, translation) do
|
||||
path =
|
||||
Path.join(
|
||||
Projects.project_data_dir(project),
|
||||
translation_sidecar_path(media, translation.language)
|
||||
)
|
||||
|
||||
:ok = File.mkdir_p(Path.dirname(path))
|
||||
|
||||
atomic_write(
|
||||
path,
|
||||
Sidecar.serialize_document([
|
||||
{"translationFor", media.id},
|
||||
{"language", translation.language},
|
||||
{"title", translation.title},
|
||||
{"alt", translation.alt},
|
||||
{"caption", translation.caption}
|
||||
])
|
||||
)
|
||||
end
|
||||
|
||||
@spec parse_canonical_sidecar(BDS.Projects.Project.t(), Path.t()) :: map()
|
||||
def parse_canonical_sidecar(project, sidecar_path) do
|
||||
{:ok, fields} = sidecar_path |> File.read!() |> Sidecar.parse_document()
|
||||
relative_sidecar_path = Path.relative_to(sidecar_path, Projects.project_data_dir(project))
|
||||
relative_file_path = String.trim_trailing(relative_sidecar_path, ".meta")
|
||||
|
||||
%{
|
||||
fields: fields,
|
||||
relative_sidecar_path: relative_sidecar_path,
|
||||
relative_file_path: relative_file_path,
|
||||
filename: Path.basename(relative_file_path)
|
||||
}
|
||||
end
|
||||
|
||||
@spec parse_translation_sidecar(Path.t()) :: map()
|
||||
def parse_translation_sidecar(sidecar_path) do
|
||||
{:ok, fields} = sidecar_path |> File.read!() |> Sidecar.parse_document()
|
||||
|
||||
%{
|
||||
fields: fields,
|
||||
binary_path: binary_path_for_translation_sidecar(sidecar_path)
|
||||
}
|
||||
end
|
||||
|
||||
@spec upsert_media_from_sidecar(BDS.Projects.Project.t(), map(), keyword()) :: Media.t()
|
||||
def upsert_media_from_sidecar(project, sidecar, opts) do
|
||||
now = Persistence.now_ms()
|
||||
|
||||
attrs = %{
|
||||
id: DocumentFields.get(sidecar.fields, "id") || Ecto.UUID.generate(),
|
||||
project_id: project.id,
|
||||
filename: sidecar.filename,
|
||||
original_name: DocumentFields.get(sidecar.fields, "originalName") || sidecar.filename,
|
||||
mime_type: DocumentFields.get(sidecar.fields, "mimeType") || detect_mime(sidecar.filename),
|
||||
size: DocumentFields.get(sidecar.fields, "size", 0),
|
||||
width: blank_to_nil(DocumentFields.get(sidecar.fields, "width")),
|
||||
height: blank_to_nil(DocumentFields.get(sidecar.fields, "height")),
|
||||
title: DocumentFields.get(sidecar.fields, "title"),
|
||||
alt: DocumentFields.get(sidecar.fields, "alt"),
|
||||
caption: DocumentFields.get(sidecar.fields, "caption"),
|
||||
author: DocumentFields.get(sidecar.fields, "author"),
|
||||
language: DocumentFields.get(sidecar.fields, "language"),
|
||||
file_path: sidecar.relative_file_path,
|
||||
sidecar_path: sidecar.relative_sidecar_path,
|
||||
checksum: nil,
|
||||
tags: DocumentFields.get(sidecar.fields, "tags", []),
|
||||
created_at: DocumentFields.get(sidecar.fields, "createdAt", now),
|
||||
updated_at: DocumentFields.get(sidecar.fields, "updatedAt", now)
|
||||
}
|
||||
|
||||
media =
|
||||
Repo.get(Media, attrs.id) ||
|
||||
Repo.get_by(Media, project_id: project.id, file_path: sidecar.relative_file_path) ||
|
||||
%Media{}
|
||||
|
||||
media =
|
||||
media
|
||||
|> Media.changeset(attrs)
|
||||
|> Repo.insert_or_update!()
|
||||
|
||||
if Keyword.get(opts, :sync_search, true) do
|
||||
:ok = Search.sync_media(media)
|
||||
end
|
||||
|
||||
media
|
||||
end
|
||||
|
||||
@spec upsert_translation_from_sidecar(BDS.Projects.Project.t(), %{required(Path.t()) => Media.t()}, map(), keyword()) ::
|
||||
Translation.t() | :skip | :ok
|
||||
def upsert_translation_from_sidecar(project, canonical_media_by_binary_path, sidecar, opts) do
|
||||
case Map.get(canonical_media_by_binary_path, sidecar.binary_path) do
|
||||
nil ->
|
||||
:skip
|
||||
|
||||
media ->
|
||||
now = Persistence.now_ms()
|
||||
language = DocumentFields.fetch!(sidecar.fields, "language")
|
||||
|
||||
translation =
|
||||
Repo.get_by(Translation, translation_for: media.id, language: language) ||
|
||||
%Translation{id: Ecto.UUID.generate(), created_at: now}
|
||||
|
||||
translation
|
||||
|> Translation.changeset(%{
|
||||
id: translation.id,
|
||||
project_id: project.id,
|
||||
translation_for: media.id,
|
||||
language: language,
|
||||
title: DocumentFields.get(sidecar.fields, "title"),
|
||||
alt: DocumentFields.get(sidecar.fields, "alt"),
|
||||
caption: DocumentFields.get(sidecar.fields, "caption"),
|
||||
created_at: translation.created_at || now,
|
||||
updated_at: now
|
||||
})
|
||||
|> Repo.insert_or_update!()
|
||||
|
||||
if Keyword.get(opts, :sync_search, true) do
|
||||
:ok = Search.sync_media(media.id)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
@spec sync_media_sidecar(String.t()) :: :ok | {:error, :not_found | term()}
|
||||
def sync_media_sidecar(media_id) do
|
||||
case Repo.get(Media, media_id) do
|
||||
nil ->
|
||||
{:error, :not_found}
|
||||
|
||||
media ->
|
||||
project = Projects.get_project!(media.project_id)
|
||||
:ok = write_sidecar(project, media)
|
||||
:ok
|
||||
end
|
||||
end
|
||||
|
||||
@spec sync_media_from_sidecar(String.t()) :: {:ok, Media.t()} | {:error, :not_found | term()}
|
||||
def sync_media_from_sidecar(media_id) do
|
||||
case Repo.get(Media, media_id) do
|
||||
nil ->
|
||||
{:error, :not_found}
|
||||
|
||||
%Media{} = media ->
|
||||
project = Projects.get_project!(media.project_id)
|
||||
sidecar_path = Path.join(Projects.project_data_dir(project), media.sidecar_path)
|
||||
|
||||
if File.exists?(sidecar_path) do
|
||||
{:ok, upsert_media_from_sidecar(project, parse_canonical_sidecar(project, sidecar_path), sync_search: true)}
|
||||
else
|
||||
{:error, :not_found}
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
@spec sync_media_translation_sidecar(String.t()) ::
|
||||
{:ok, Translation.t()} | {:error, :not_found | term()}
|
||||
def sync_media_translation_sidecar(translation_id) do
|
||||
case Repo.get(Translation, translation_id) do
|
||||
nil ->
|
||||
{:error, :not_found}
|
||||
|
||||
%Translation{} = translation ->
|
||||
media = Repo.get!(Media, translation.translation_for)
|
||||
project = Projects.get_project!(media.project_id)
|
||||
:ok = write_translation_sidecar(project, media, translation)
|
||||
{:ok, translation}
|
||||
end
|
||||
end
|
||||
|
||||
@spec sync_media_translation_from_sidecar(String.t()) ::
|
||||
{:ok, Translation.t()} | {:error, :not_found | term()}
|
||||
def sync_media_translation_from_sidecar(translation_id) do
|
||||
case Repo.get(Translation, translation_id) do
|
||||
nil ->
|
||||
{:error, :not_found}
|
||||
|
||||
%Translation{} = translation ->
|
||||
media = Repo.get!(Media, translation.translation_for)
|
||||
project = Projects.get_project!(media.project_id)
|
||||
|
||||
sidecar_path =
|
||||
Path.join(
|
||||
Projects.project_data_dir(project),
|
||||
translation_sidecar_path(media, translation.language)
|
||||
)
|
||||
|
||||
if File.exists?(sidecar_path) do
|
||||
sidecar = parse_translation_sidecar(sidecar_path)
|
||||
|
||||
case BDS.Media.upsert_media_translation(
|
||||
media.id,
|
||||
DocumentFields.fetch!(sidecar.fields, "language"),
|
||||
%{
|
||||
title: DocumentFields.get(sidecar.fields, "title"),
|
||||
alt: DocumentFields.get(sidecar.fields, "alt"),
|
||||
caption: DocumentFields.get(sidecar.fields, "caption")
|
||||
}
|
||||
) do
|
||||
{:ok, updated_translation} -> {:ok, updated_translation}
|
||||
error -> error
|
||||
end
|
||||
else
|
||||
{:error, :not_found}
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
@spec import_orphan_media_sidecar(String.t(), String.t()) ::
|
||||
{:ok, Media.t()} | {:error, term()}
|
||||
def import_orphan_media_sidecar(project_id, relative_path) do
|
||||
project = Projects.get_project!(project_id)
|
||||
sidecar_path = Path.join(Projects.project_data_dir(project), relative_path)
|
||||
|
||||
if File.exists?(sidecar_path) do
|
||||
{:ok, upsert_media_from_sidecar(project, parse_canonical_sidecar(project, sidecar_path), sync_search: true)}
|
||||
else
|
||||
{:error, :not_found}
|
||||
end
|
||||
end
|
||||
|
||||
@spec import_orphan_media_translation_sidecar(String.t(), String.t()) ::
|
||||
{:ok, Translation.t()} | {:error, term()}
|
||||
def import_orphan_media_translation_sidecar(project_id, relative_path) do
|
||||
project = Projects.get_project!(project_id)
|
||||
sidecar_path = Path.join(Projects.project_data_dir(project), relative_path)
|
||||
|
||||
if File.exists?(sidecar_path) do
|
||||
sidecar = parse_translation_sidecar(sidecar_path)
|
||||
|
||||
case Repo.get(Media, DocumentFields.get(sidecar.fields, "translationFor")) do
|
||||
nil ->
|
||||
{:error, :not_found}
|
||||
|
||||
media ->
|
||||
case Repo.get_by(Translation,
|
||||
translation_for: media.id,
|
||||
language: DocumentFields.fetch!(sidecar.fields, "language")
|
||||
) do
|
||||
nil ->
|
||||
BDS.Media.upsert_media_translation(
|
||||
media.id,
|
||||
DocumentFields.fetch!(sidecar.fields, "language"),
|
||||
%{
|
||||
title: DocumentFields.get(sidecar.fields, "title"),
|
||||
alt: DocumentFields.get(sidecar.fields, "alt"),
|
||||
caption: DocumentFields.get(sidecar.fields, "caption")
|
||||
}
|
||||
)
|
||||
|
||||
_translation ->
|
||||
{:error, :conflict}
|
||||
end
|
||||
end
|
||||
else
|
||||
{:error, :not_found}
|
||||
end
|
||||
end
|
||||
|
||||
@spec translation_sidecar_path(Media.t(), String.t()) :: String.t()
|
||||
def translation_sidecar_path(media, language), do: "#{media.file_path}.#{language}.meta"
|
||||
|
||||
@spec canonical_sidecar?(Path.t()) :: boolean()
|
||||
def canonical_sidecar?(sidecar_path), do: not translation_sidecar?(sidecar_path)
|
||||
|
||||
@spec translation_sidecar?(Path.t()) :: boolean()
|
||||
def translation_sidecar?(sidecar_path) do
|
||||
Regex.match?(~r/\.[a-z]{2}\.meta$/i, sidecar_path)
|
||||
end
|
||||
|
||||
@spec binary_path_for_translation_sidecar(Path.t()) :: Path.t()
|
||||
def binary_path_for_translation_sidecar(sidecar_path) do
|
||||
Regex.replace(~r/\.[a-z]{2}\.meta$/i, sidecar_path, "")
|
||||
end
|
||||
|
||||
@spec binary_exists_for_sidecar?(Path.t()) :: boolean()
|
||||
def binary_exists_for_sidecar?(sidecar_path) do
|
||||
sidecar_path
|
||||
|> String.trim_trailing(".meta")
|
||||
|> File.exists?()
|
||||
end
|
||||
end
|
||||
165
lib/bds/media/thumbnails.ex
Normal file
165
lib/bds/media/thumbnails.ex
Normal file
@@ -0,0 +1,165 @@
|
||||
defmodule BDS.Media.Thumbnails do
|
||||
@moduledoc false
|
||||
|
||||
import BDS.Media.FileOps,
|
||||
only: [
|
||||
delete_file_if_present: 2,
|
||||
image_mime?: 1,
|
||||
progress_callback: 1,
|
||||
report_rebuild_progress: 4,
|
||||
report_rebuild_started: 3
|
||||
]
|
||||
|
||||
import Ecto.Query
|
||||
|
||||
alias BDS.Media.Media
|
||||
alias BDS.Projects
|
||||
alias BDS.Repo
|
||||
|
||||
@type rebuild_opts :: keyword()
|
||||
|
||||
@spec thumbnail_paths(Media.t()) :: %{required(atom()) => String.t()}
|
||||
def thumbnail_paths(%Media{id: id}) do
|
||||
prefix = String.slice(id, 0, 2)
|
||||
|
||||
%{
|
||||
small: Path.join(["thumbnails", prefix, "#{id}-small.webp"]),
|
||||
medium: Path.join(["thumbnails", prefix, "#{id}-medium.webp"]),
|
||||
large: Path.join(["thumbnails", prefix, "#{id}-large.webp"]),
|
||||
ai: Path.join(["thumbnails", prefix, "#{id}-ai.jpg"])
|
||||
}
|
||||
end
|
||||
|
||||
@spec regenerate_thumbnails(String.t()) :: {:ok, Media.t()} | {:error, :not_found | term()}
|
||||
def regenerate_thumbnails(media_id) do
|
||||
case Repo.get(Media, media_id) do
|
||||
nil ->
|
||||
{:error, :not_found}
|
||||
|
||||
media ->
|
||||
project = Projects.get_project!(media.project_id)
|
||||
:ok = ensure_thumbnails(project, media)
|
||||
{:ok, media}
|
||||
end
|
||||
end
|
||||
|
||||
@spec regenerate_missing_thumbnails(String.t(), rebuild_opts()) ::
|
||||
%{processed: non_neg_integer(), generated: non_neg_integer(), failed: non_neg_integer()}
|
||||
def regenerate_missing_thumbnails(project_id, opts \\ []) do
|
||||
project = Projects.get_project!(project_id)
|
||||
on_progress = progress_callback(opts)
|
||||
|
||||
media_items =
|
||||
Repo.all(
|
||||
from(media in Media,
|
||||
where: media.project_id == ^project_id,
|
||||
order_by: [asc: media.created_at]
|
||||
)
|
||||
)
|
||||
|> Enum.filter(fn media ->
|
||||
String.starts_with?(media.mime_type || "", "image/") and
|
||||
not String.contains?(media.mime_type || "", "svg")
|
||||
end)
|
||||
|
||||
total_media = length(media_items)
|
||||
:ok = report_rebuild_started(on_progress, total_media, "image assets")
|
||||
|
||||
media_items
|
||||
|> Enum.with_index(1)
|
||||
|> Enum.reduce(%{processed: 0, generated: 0, failed: 0}, fn {media, index}, acc ->
|
||||
missing_paths =
|
||||
media
|
||||
|> thumbnail_paths()
|
||||
|> Enum.map(fn {_size, relative_path} -> Path.join(Projects.project_data_dir(project), relative_path) end)
|
||||
|> Enum.reject(&File.exists?/1)
|
||||
|
||||
next_acc =
|
||||
if missing_paths == [] do
|
||||
%{acc | processed: acc.processed + 1}
|
||||
else
|
||||
try do
|
||||
:ok = ensure_thumbnails(project, media)
|
||||
|
||||
%{
|
||||
processed: acc.processed + 1,
|
||||
generated: acc.generated + length(missing_paths),
|
||||
failed: acc.failed
|
||||
}
|
||||
rescue
|
||||
_error ->
|
||||
%{acc | processed: acc.processed + 1, failed: acc.failed + 1}
|
||||
end
|
||||
end
|
||||
|
||||
:ok = report_rebuild_progress(on_progress, index, total_media, "image assets")
|
||||
next_acc
|
||||
end)
|
||||
end
|
||||
|
||||
@spec ensure_thumbnails(BDS.Projects.Project.t(), Media.t()) :: :ok
|
||||
def ensure_thumbnails(project, media) do
|
||||
if image_mime?(media.mime_type) do
|
||||
source_path = Path.join(Projects.project_data_dir(project), media.file_path)
|
||||
|
||||
case Image.open(source_path) do
|
||||
{:ok, image} ->
|
||||
image
|
||||
|> Image.autorotate!()
|
||||
|> write_all_thumbnails(project, media)
|
||||
|
||||
{:error, _reason} ->
|
||||
:ok
|
||||
end
|
||||
end
|
||||
|
||||
:ok
|
||||
end
|
||||
|
||||
@spec delete_thumbnail_files(String.t(), Media.t()) :: :ok
|
||||
def delete_thumbnail_files(project_id, media) do
|
||||
Enum.each(Map.values(thumbnail_paths(media)), fn path ->
|
||||
delete_file_if_present(project_id, path)
|
||||
end)
|
||||
|
||||
:ok
|
||||
end
|
||||
|
||||
defp write_all_thumbnails(image, project, media) do
|
||||
thumbnail_paths(media)
|
||||
|> Enum.each(fn {size, relative_path} ->
|
||||
destination = Path.join(Projects.project_data_dir(project), relative_path)
|
||||
:ok = File.mkdir_p(Path.dirname(destination))
|
||||
|
||||
image
|
||||
|> render_thumbnail(size)
|
||||
|> write_thumbnail(destination, size)
|
||||
end)
|
||||
|
||||
:ok
|
||||
end
|
||||
|
||||
defp render_thumbnail(image, :small), do: bounded_thumbnail(image, 150, 150)
|
||||
defp render_thumbnail(image, :medium), do: bounded_thumbnail(image, 400, 400)
|
||||
defp render_thumbnail(image, :large), do: bounded_thumbnail(image, 800, 800)
|
||||
|
||||
defp render_thumbnail(image, :ai) do
|
||||
image
|
||||
|> Image.thumbnail!("448x448", fit: :contain, resize: :both, autorotate: false)
|
||||
|> Image.embed!(448, 448, x: :center, y: :center, background_color: :black)
|
||||
end
|
||||
|
||||
defp bounded_thumbnail(image, width, height) do
|
||||
Image.thumbnail!(image, "#{width}x#{height}", fit: :contain, resize: :down, autorotate: false)
|
||||
end
|
||||
|
||||
defp write_thumbnail(image, destination, :ai) do
|
||||
flattened = Image.flatten!(image, background_color: :black)
|
||||
Image.write!(flattened, destination, quality: 85, strip_metadata: true)
|
||||
:ok
|
||||
end
|
||||
|
||||
defp write_thumbnail(image, destination, _size) do
|
||||
Image.write!(image, destination, quality: 80, strip_metadata: true)
|
||||
:ok
|
||||
end
|
||||
end
|
||||
Reference in New Issue
Block a user