diff --git a/lib/bds/maintenance.ex b/lib/bds/maintenance.ex index 92b3bc0..34b8a97 100644 --- a/lib/bds/maintenance.ex +++ b/lib/bds/maintenance.ex @@ -4,6 +4,7 @@ defmodule BDS.Maintenance do import Ecto.Query alias BDS.Frontmatter + alias BDS.Metadata alias BDS.Media.Media alias BDS.Media.Translation, as: MediaTranslation alias BDS.Embeddings @@ -30,7 +31,8 @@ defmodule BDS.Maintenance do project = Projects.get_project!(project_id) diff_reports = - post_diff_reports(project_id, project) ++ + project_metadata_diff_reports(project_id) ++ + post_diff_reports(project_id, project) ++ post_translation_diff_reports(project_id, project) ++ media_diff_reports(project_id, project) ++ media_translation_diff_reports(project_id, project) ++ @@ -43,6 +45,71 @@ defmodule BDS.Maintenance do {:ok, %{diff_reports: diff_reports, orphan_reports: orphan_reports}} end + defp project_metadata_diff_reports(project_id) do + {:ok, db_state} = Metadata.get_project_metadata(project_id) + {:ok, filesystem_state} = Metadata.read_project_metadata_from_filesystem(project_id) + + [ + build_diff_report("project", project_id, [ + diff_field("name", db_state.name, filesystem_state.name), + diff_field("description", db_state.description, filesystem_state.description), + diff_field("public_url", db_state.public_url, filesystem_state.public_url), + diff_field("main_language", db_state.main_language, filesystem_state.main_language), + diff_field("default_author", db_state.default_author, filesystem_state.default_author), + diff_field( + "max_posts_per_page", + db_state.max_posts_per_page, + filesystem_state.max_posts_per_page + ), + diff_field( + "blogmark_category", + db_state.blogmark_category, + filesystem_state.blogmark_category + ), + diff_field("pico_theme", db_state.pico_theme, filesystem_state.pico_theme), + diff_field( + "semantic_similarity_enabled", + db_state.semantic_similarity_enabled, + filesystem_state.semantic_similarity_enabled + ), + diff_field("blog_languages", db_state.blog_languages, filesystem_state.blog_languages) + ]), + build_diff_report("categories", project_id, [ + diff_field("categories", db_state.categories, filesystem_state.categories) + ]), + build_diff_report("category_meta", project_id, [ + diff_field( + "category_settings", + db_state.category_settings, + filesystem_state.category_settings + ) + ]), + build_diff_report("publishing", project_id, [ + diff_field( + "ssh_host", + Map.get(db_state.publishing_preferences, "ssh_host"), + Map.get(filesystem_state.publishing_preferences, "ssh_host") + ), + diff_field( + "ssh_user", + Map.get(db_state.publishing_preferences, "ssh_user"), + Map.get(filesystem_state.publishing_preferences, "ssh_user") + ), + diff_field( + "ssh_remote_path", + Map.get(db_state.publishing_preferences, "ssh_remote_path"), + Map.get(filesystem_state.publishing_preferences, "ssh_remote_path") + ), + diff_field( + "ssh_mode", + Map.get(db_state.publishing_preferences, "ssh_mode"), + Map.get(filesystem_state.publishing_preferences, "ssh_mode") + ) + ]) + ] + |> Enum.reject(&is_nil/1) + end + defp normalize_entity_type(:post), do: :post defp normalize_entity_type(:media), do: :media defp normalize_entity_type(:script), do: :script @@ -366,6 +433,16 @@ defmodule BDS.Maintenance do |> Enum.map(&%{file_path: &1}) end + defp build_diff_report(entity_type, entity_id, differences) do + normalized = Enum.reject(differences, &is_nil/1) + + if normalized == [] do + nil + else + %{entity_type: entity_type, entity_id: entity_id, differences: normalized} + end + end + defp diff_field(name, db_value, file_value) do if equal_diff_values?(db_value, file_value) do nil @@ -378,6 +455,10 @@ defmodule BDS.Maintenance do normalize_list_diff_values(left) == normalize_list_diff_values(right) end + defp equal_diff_values?(left, right) when is_map(left) and is_map(right) do + normalize_map_diff_values(left) == normalize_map_diff_values(right) + end + defp equal_diff_values?(left, right), do: stringify_value(left) == stringify_value(right) defp normalize_list_diff_values(values) do @@ -392,11 +473,26 @@ defmodule BDS.Maintenance do defp stringify_value(value) when is_integer(value), do: Integer.to_string(value) defp stringify_value(value) when is_binary(value), do: value + defp stringify_value(value) when is_map(value), + do: value |> normalize_map_diff_values() |> Jason.encode!() + defp stringify_value(value) when is_list(value), do: Enum.map_join(value, ",", &stringify_value/1) defp stringify_value(value), do: to_string(value) + defp normalize_map_diff_values(values) when is_map(values) do + values + |> Enum.map(fn {key, value} -> {to_string(key), normalize_nested_diff_value(value)} end) + |> Enum.sort_by(&elem(&1, 0)) + |> Map.new() + end + + defp normalize_nested_diff_value(value) when is_map(value), do: normalize_map_diff_values(value) + defp normalize_nested_diff_value(value) when is_list(value), do: Enum.map(value, &normalize_nested_diff_value/1) + defp normalize_nested_diff_value(value) when is_atom(value), do: Atom.to_string(value) + defp normalize_nested_diff_value(value), do: value + defp read_frontmatter_document(project, relative_path) do full_path = Path.join(Projects.project_data_dir(project), relative_path) diff --git a/lib/bds/metadata.ex b/lib/bds/metadata.ex index e2128bb..027c804 100644 --- a/lib/bds/metadata.ex +++ b/lib/bds/metadata.ex @@ -41,6 +41,11 @@ defmodule BDS.Metadata do {:ok, load_state(project)} end + def read_project_metadata_from_filesystem(project_id) do + project = Projects.get_project!(project_id) + {:ok, load_state_from_filesystem(project)} + end + def update_project_metadata(project_id, attrs) do project = Projects.get_project!(project_id) state = load_state(project) @@ -131,35 +136,32 @@ defmodule BDS.Metadata do def sync_project_metadata_from_filesystem(project_id) do project = Projects.get_project!(project_id) now = Persistence.now_ms() - - project_metadata_from_files = - read_json(project, "project.json") || - stringify_project_metadata(default_project_metadata(project)) - - categories_from_files = - read_json(project, "categories.json") || %{"categories" => @default_categories} - - category_meta_from_files = read_json(project, "category-meta.json") || %{"categories" => %{}} - publishing_from_files = read_json(project, "publishing.json") || %{"ssh_mode" => "scp"} + filesystem_state = load_state_from_filesystem(project) Repo.transaction(fn -> updated_project = project |> Project.changeset(%{ - name: Map.get(project_metadata_from_files, "name", project.name), - description: Map.get(project_metadata_from_files, "description"), + name: filesystem_state.name, + description: filesystem_state.description, updated_at: now }) |> Repo.update!() - persist_setting(project_id, "project", project_metadata_from_files, now) - persist_setting(project_id, "categories", categories_from_files, now) - persist_setting(project_id, "category_meta", category_meta_from_files, now) - persist_setting(project_id, "publishing", publishing_from_files, now) - write_project_json(updated_project, project_metadata_from_files) - write_categories_json(updated_project, normalized_categories(categories_from_files)) - write_category_meta_json(updated_project, normalized_category_settings(category_meta_from_files)) - write_publishing_json(updated_project, publishing_from_files) + persist_setting(project_id, "project", stringify_project_metadata(filesystem_state), now) + persist_setting(project_id, "categories", %{"categories" => filesystem_state.categories}, now) + persist_setting( + project_id, + "category_meta", + %{"categories" => filesystem_state.category_settings}, + now + ) + + persist_setting(project_id, "publishing", filesystem_state.publishing_preferences, now) + write_project_json(updated_project, stringify_project_metadata(filesystem_state)) + write_categories_json(updated_project, filesystem_state.categories) + write_category_meta_json(updated_project, filesystem_state.category_settings) + write_publishing_json(updated_project, filesystem_state.publishing_preferences) load_state(updated_project) end) |> unwrap_transaction() @@ -210,6 +212,37 @@ defmodule BDS.Metadata do } end + defp load_state_from_filesystem(project) do + project_metadata = + read_json(project, "project.json") || + stringify_project_metadata(default_project_metadata(project)) + + categories = normalized_categories(read_json(project, "categories.json") || %{"categories" => @default_categories}) + + category_settings = + normalized_category_settings(read_json(project, "category-meta.json") || %{"categories" => %{}}) + + publishing_preferences = read_json(project, "publishing.json") || %{"ssh_mode" => "scp"} + + %{ + name: Map.get(project_metadata, "name", project.name), + description: Map.get(project_metadata, "description"), + public_url: Map.get(project_metadata, "public_url"), + main_language: Map.get(project_metadata, "main_language"), + default_author: Map.get(project_metadata, "default_author"), + max_posts_per_page: + Map.get(project_metadata, "max_posts_per_page", @default_max_posts_per_page), + blogmark_category: Map.get(project_metadata, "blogmark_category"), + pico_theme: Map.get(project_metadata, "pico_theme"), + semantic_similarity_enabled: + Map.get(project_metadata, "semantic_similarity_enabled", false), + blog_languages: Map.get(project_metadata, "blog_languages", []), + categories: categories, + category_settings: category_settings, + publishing_preferences: publishing_preferences + } + end + defp default_project_metadata(project) do %{ name: project.name, diff --git a/lib/bds/projects.ex b/lib/bds/projects.ex index ba185ce..6d1d832 100644 --- a/lib/bds/projects.ex +++ b/lib/bds/projects.ex @@ -3,6 +3,7 @@ defmodule BDS.Projects do import Ecto.Query + alias BDS.Metadata alias BDS.Persistence alias BDS.Projects.Project alias BDS.Repo @@ -96,7 +97,7 @@ defmodule BDS.Projects do project end) |> case do - {:ok, project} -> {:ok, project} + {:ok, project} -> sync_filesystem_metadata(project) {:error, reason} -> {:error, reason} end end @@ -168,6 +169,15 @@ defmodule BDS.Projects do } end + defp sync_filesystem_metadata(%Project{data_path: nil} = project), do: {:ok, project} + + defp sync_filesystem_metadata(%Project{} = project) do + case Metadata.sync_project_metadata_from_filesystem(project.id) do + {:ok, _metadata} -> {:ok, get_project!(project.id)} + {:error, reason} -> {:error, reason} + end + end + defp unique_slug(base_slug) do normalized = if base_slug in [nil, ""], do: "project", else: base_slug diff --git a/test/bds/maintenance_test.exs b/test/bds/maintenance_test.exs index f3f059d..4bbfff9 100644 --- a/test/bds/maintenance_test.exs +++ b/test/bds/maintenance_test.exs @@ -749,6 +749,70 @@ defmodule BDS.MaintenanceTest do end) end + test "metadata_diff includes project-level metadata drift", %{project: project, temp_dir: temp_dir} do + assert {:ok, _metadata} = + BDS.Metadata.update_project_metadata(project.id, %{ + name: "Database Blog", + description: "Database description", + public_url: "https://database.example", + main_language: "en", + default_author: "Database Author", + max_posts_per_page: 25, + blogmark_category: "article", + pico_theme: "blue", + semantic_similarity_enabled: false, + blog_languages: ["de"] + }) + + assert {:ok, _metadata} = + BDS.Metadata.set_publishing_preferences(project.id, %{ + ssh_host: "db.example", + ssh_user: "db-user", + ssh_remote_path: "/srv/db", + ssh_mode: "scp" + }) + + meta_dir = Path.join(temp_dir, "meta") + + File.write!( + Path.join(meta_dir, "project.json"), + Jason.encode!(%{ + "name" => "Filesystem Blog", + "description" => "Filesystem description", + "publicUrl" => "https://filesystem.example", + "mainLanguage" => "fr", + "defaultAuthor" => "Filesystem Author", + "maxPostsPerPage" => 12, + "blogmarkCategory" => "notes", + "picoTheme" => "slate", + "semanticSimilarityEnabled" => true, + "blogLanguages" => ["it"] + }) + ) + + File.write!( + Path.join(meta_dir, "publishing.json"), + Jason.encode!(%{ + "sshHost" => "files.example", + "sshUser" => "files-user", + "sshRemotePath" => "/srv/files", + "sshMode" => "rsync" + }) + ) + + assert {:ok, diff} = BDS.Maintenance.metadata_diff(project.id) + + assert Enum.any?(diff.diff_reports, fn report -> + report.entity_type == "project" and + Enum.any?(report.differences, &(&1.name == "main_language" and &1.db_value == "en" and &1.file_value == "fr")) + end) + + assert Enum.any?(diff.diff_reports, fn report -> + report.entity_type == "publishing" and + Enum.any?(report.differences, &(&1.name == "ssh_mode" and &1.db_value == "scp" and &1.file_value == "rsync")) + end) + end + defp collect_progress_events(acc \\ []) do receive do {:rebuild_progress, value, message} -> collect_progress_events([{value, message} | acc]) diff --git a/test/bds/metadata_test.exs b/test/bds/metadata_test.exs index 45a1519..a3f7813 100644 --- a/test/bds/metadata_test.exs +++ b/test/bds/metadata_test.exs @@ -221,6 +221,9 @@ defmodule BDS.MetadataTest do test "sync_project_metadata_from_filesystem materializes the canonical metadata files when missing", %{project: project, temp_dir: temp_dir} do meta_dir = Path.join(temp_dir, "meta") + + File.rm_rf!(meta_dir) + refute File.exists?(Path.join(meta_dir, "project.json")) refute File.exists?(Path.join(meta_dir, "categories.json")) refute File.exists?(Path.join(meta_dir, "category-meta.json")) diff --git a/test/bds/projects_test.exs b/test/bds/projects_test.exs index c9ce6ca..43fbf4a 100644 --- a/test/bds/projects_test.exs +++ b/test/bds/projects_test.exs @@ -3,6 +3,7 @@ defmodule BDS.ProjectsTest do import Ecto.Query + alias BDS.Metadata alias BDS.Projects.Project alias BDS.Repo alias BDS.Templates.Template @@ -157,4 +158,60 @@ defmodule BDS.ProjectsTest do assert BDS.Projects.get_project(external_project.id) == nil assert File.read!(marker_path) == "preserve me" end + + test "create_project loads project metadata from an existing filesystem-backed blog", %{temp_root: temp_root} do + external_dir = Path.join(temp_root, "imported-blog") + meta_dir = Path.join(external_dir, "meta") + File.mkdir_p!(meta_dir) + + File.write!( + Path.join(meta_dir, "project.json"), + Jason.encode!(%{ + "name" => "Imported Blog", + "description" => "Filesystem metadata", + "publicUrl" => "https://imported.example", + "mainLanguage" => "de", + "defaultAuthor" => "Importer", + "maxPostsPerPage" => 17, + "blogmarkCategory" => "notes", + "picoTheme" => "slate", + "semanticSimilarityEnabled" => true, + "blogLanguages" => ["en", "fr"] + }) + ) + + File.write!( + Path.join(meta_dir, "publishing.json"), + Jason.encode!(%{ + "sshHost" => "upload.example", + "sshUser" => "deploy", + "sshRemotePath" => "/srv/imported", + "sshMode" => "rsync" + }) + ) + + assert {:ok, project} = + BDS.Projects.create_project(%{name: "Placeholder", data_path: external_dir}) + + assert BDS.Projects.get_project!(project.id).name == "Imported Blog" + + assert {:ok, metadata} = Metadata.get_project_metadata(project.id) + assert metadata.name == "Imported Blog" + assert metadata.description == "Filesystem metadata" + assert metadata.public_url == "https://imported.example" + assert metadata.main_language == "de" + assert metadata.default_author == "Importer" + assert metadata.max_posts_per_page == 17 + assert metadata.blogmark_category == "notes" + assert metadata.pico_theme == "slate" + assert metadata.semantic_similarity_enabled == true + assert metadata.blog_languages == ["en", "fr"] + + assert metadata.publishing_preferences == %{ + "ssh_host" => "upload.example", + "ssh_user" => "deploy", + "ssh_remote_path" => "/srv/imported", + "ssh_mode" => "rsync" + } + end end