From 28141deb8bcd600b7d8445a2a990351c984c6246 Mon Sep 17 00:00:00 2001
From: Chili Palmer <hugoms@me.com>
Date: Thu, 23 Apr 2026 12:14:54 +0200
Subject: [PATCH] feat: first database migration

---
 ...0423120000_create_persistence_contract.exs | 347 ++++++++++++++++
 test/bds/repo/schema_migration_test.exs       | 381 ++++++++++++++++++
 2 files changed, 728 insertions(+)
 create mode 100644 priv/repo/migrations/20260423120000_create_persistence_contract.exs
 create mode 100644 test/bds/repo/schema_migration_test.exs

diff --git a/priv/repo/migrations/20260423120000_create_persistence_contract.exs b/priv/repo/migrations/20260423120000_create_persistence_contract.exs
new file mode 100644
index 0000000..f69e337
--- /dev/null
+++ b/priv/repo/migrations/20260423120000_create_persistence_contract.exs
@@ -0,0 +1,347 @@
+defmodule BDS.Repo.Migrations.CreatePersistenceContract do
+  use Ecto.Migration
+
+  def change do
+    create table(:projects, primary_key: false) do
+      add :id, :string, primary_key: true
+      add :name, :string, null: false
+      add :slug, :string, null: false
+      add :description, :text
+      add :data_path, :string
+      add :created_at, :integer, null: false
+      add :updated_at, :integer, null: false
+      add :is_active, :boolean, null: false, default: false
+    end
+
+    create unique_index(:projects, [:slug])
+
+    create table(:posts, primary_key: false) do
+      add :id, :string, primary_key: true
+      add :project_id, references(:projects, type: :string, on_delete: :delete_all), null: false
+      add :title, :string, null: false
+      add :slug, :string, null: false
+      add :excerpt, :text
+      add :content, :text
+      add :status, :string, null: false, default: "draft"
+      add :author, :string
+      add :created_at, :integer, null: false
+      add :updated_at, :integer, null: false
+      add :published_at, :integer
+      add :file_path, :string, null: false, default: ""
+      add :checksum, :string
+      add :tags, :text
+      add :categories, :text
+      add :template_slug, :string
+      add :language, :string
+      add :do_not_translate, :boolean, null: false, default: false
+      add :published_title, :text
+      add :published_content, :text
+      add :published_tags, :text
+      add :published_categories, :text
+      add :published_excerpt, :text
+    end
+
+    create unique_index(:posts, [:project_id, :slug], name: :posts_project_slug_idx)
+
+    create table(:post_translations, primary_key: false) do
+      add :id, :string, primary_key: true
+      add :project_id, references(:projects, type: :string, on_delete: :delete_all), null: false
+
+      add :translation_for, references(:posts, column: :id, type: :string, on_delete: :delete_all),
+        null: false
+
+      add :language, :string, null: false
+      add :title, :string, null: false
+      add :excerpt, :text
+      add :content, :text
+      add :status, :string, null: false, default: "draft"
+      add :created_at, :integer, null: false
+      add :updated_at, :integer, null: false
+      add :published_at, :integer
+      add :file_path, :string, null: false, default: ""
+      add :checksum, :string
+    end
+
+    create unique_index(:post_translations, [:translation_for, :language],
+             name: :post_translations_translation_language_idx
+           )
+
+    create table(:media, primary_key: false) do
+      add :id, :string, primary_key: true
+      add :project_id, references(:projects, type: :string, on_delete: :delete_all), null: false
+      add :filename, :string, null: false
+      add :original_name, :string, null: false
+      add :mime_type, :string, null: false
+      add :size, :integer, null: false
+      add :width, :integer
+      add :height, :integer
+      add :title, :string
+      add :alt, :text
+      add :caption, :text
+      add :author, :string
+      add :file_path, :string, null: false
+      add :sidecar_path, :string, null: false
+      add :created_at, :integer, null: false
+      add :updated_at, :integer, null: false
+      add :checksum, :string
+      add :tags, :text
+      add :language, :string
+    end
+
+    create table(:media_translations, primary_key: false) do
+      add :id, :string, primary_key: true
+      add :project_id, references(:projects, type: :string, on_delete: :delete_all), null: false
+
+      add :translation_for, references(:media, column: :id, type: :string, on_delete: :delete_all),
+        null: false
+
+      add :language, :string, null: false
+      add :title, :string
+      add :alt, :text
+      add :caption, :text
+      add :created_at, :integer, null: false
+      add :updated_at, :integer, null: false
+    end
+
+    create unique_index(:media_translations, [:translation_for, :language],
+             name: :media_translations_translation_language_idx
+           )
+
+    create table(:tags, primary_key: false) do
+      add :id, :string, primary_key: true
+      add :project_id, references(:projects, type: :string, on_delete: :delete_all), null: false
+      add :name, :string, null: false
+      add :color, :string
+      add :post_template_slug, :string
+      add :created_at, :integer, null: false
+      add :updated_at, :integer, null: false
+    end
+
+    create unique_index(:tags, [:project_id, :name], name: :tags_project_name_idx)
+
+    create table(:templates, primary_key: false) do
+      add :id, :string, primary_key: true
+      add :project_id, references(:projects, type: :string, on_delete: :delete_all), null: false
+      add :slug, :string, null: false
+      add :title, :string, null: false
+      add :kind, :string, null: false
+      add :enabled, :boolean, null: false, default: true
+      add :version, :integer, null: false, default: 1
+      add :file_path, :string, null: false
+      add :status, :string, null: false, default: "draft"
+      add :content, :text
+      add :created_at, :integer, null: false
+      add :updated_at, :integer, null: false
+    end
+
+    create unique_index(:templates, [:project_id, :slug], name: :templates_project_slug_idx)
+
+    create table(:scripts, primary_key: false) do
+      add :id, :string, primary_key: true
+      add :project_id, references(:projects, type: :string, on_delete: :delete_all), null: false
+      add :slug, :string, null: false
+      add :title, :string, null: false
+      add :kind, :string, null: false
+      add :entrypoint, :string, null: false
+      add :enabled, :boolean, null: false, default: true
+      add :version, :integer, null: false, default: 1
+      add :file_path, :string, null: false
+      add :status, :string, null: false, default: "draft"
+      add :content, :text
+      add :created_at, :integer, null: false
+      add :updated_at, :integer, null: false
+    end
+
+    create unique_index(:scripts, [:project_id, :slug], name: :scripts_project_slug_idx)
+
+    create table(:post_links, primary_key: false) do
+      add :id, :string, primary_key: true
+
+      add :source_post_id, references(:posts, column: :id, type: :string, on_delete: :delete_all),
+        null: false
+
+      add :target_post_id, references(:posts, column: :id, type: :string, on_delete: :delete_all),
+        null: false
+
+      add :link_text, :text
+      add :created_at, :integer, null: false
+    end
+
+    create table(:post_media, primary_key: false) do
+      add :id, :string, primary_key: true
+      add :project_id, references(:projects, type: :string, on_delete: :delete_all), null: false
+      add :post_id, references(:posts, column: :id, type: :string, on_delete: :delete_all), null: false
+      add :media_id, references(:media, column: :id, type: :string, on_delete: :delete_all), null: false
+      add :sort_order, :integer, null: false, default: 0
+      add :created_at, :integer, null: false
+    end
+
+    create unique_index(:post_media, [:post_id, :media_id], name: :post_media_post_media_idx)
+
+    create table(:settings, primary_key: false) do
+      add :key, :string, primary_key: true
+      add :value, :text, null: false
+      add :updated_at, :integer, null: false
+    end
+
+    create table(:generated_file_hashes, primary_key: false) do
+      add :project_id, references(:projects, type: :string, on_delete: :delete_all), null: false
+      add :relative_path, :string, null: false
+      add :content_hash, :string, null: false
+      add :updated_at, :integer, null: false
+    end
+
+    create unique_index(:generated_file_hashes, [:project_id, :relative_path],
+             name: :generated_file_hashes_project_path_idx
+           )
+
+    create table(:chat_conversations, primary_key: false) do
+      add :id, :string, primary_key: true
+      add :title, :string, null: false
+      add :model, :string
+      add :copilot_session_id, :string
+      add :created_at, :integer, null: false
+      add :updated_at, :integer, null: false
+    end
+
+    create table(:chat_messages) do
+      add :conversation_id,
+          references(:chat_conversations, column: :id, type: :string, on_delete: :delete_all),
+          null: false
+
+      add :role, :string, null: false
+      add :content, :text
+      add :tool_call_id, :string
+      add :tool_calls, :text
+      add :created_at, :integer, null: false
+    end
+
+    create table(:ai_providers, primary_key: false) do
+      add :id, :string, primary_key: true
+      add :name, :string, null: false
+      add :env, :string
+      add :package_ref, :string
+      add :api, :string
+      add :doc, :string
+      add :updated_at, :integer, null: false
+    end
+
+    create table(:ai_models, primary_key: false) do
+      add :provider, references(:ai_providers, type: :string, on_delete: :delete_all),
+        primary_key: true,
+        null: false
+
+      add :model_id, :string, primary_key: true
+      add :name, :string, null: false
+      add :family, :string
+      add :attachment, :boolean, null: false, default: false
+      add :reasoning, :boolean, null: false, default: false
+      add :tool_call, :boolean, null: false, default: false
+      add :structured_output, :boolean, null: false, default: false
+      add :temperature, :boolean, null: false, default: false
+      add :knowledge, :string
+      add :release_date, :string
+      add :last_updated_date, :string
+      add :open_weights, :boolean, null: false, default: false
+      add :input_price, :integer
+      add :output_price, :integer
+      add :cache_read_price, :integer
+      add :cache_write_price, :integer
+      add :context_window, :integer, null: false
+      add :max_input_tokens, :integer, null: false
+      add :max_output_tokens, :integer, null: false
+      add :interleaved, :string
+      add :status, :string
+      add :provider_package_ref, :string
+      add :updated_at, :integer, null: false
+    end
+
+    create table(:ai_model_modalities, primary_key: false) do
+      add :provider, references(:ai_providers, type: :string, on_delete: :delete_all),
+        primary_key: true,
+        null: false
+
+      add :model_id, :string, primary_key: true
+      add :direction, :string, primary_key: true
+      add :modality, :string, primary_key: true
+    end
+
+    create table(:ai_catalog_meta, primary_key: false) do
+      add :key, :string, primary_key: true
+      add :value, :string, null: false
+    end
+
+    create table(:embedding_keys, primary_key: false) do
+      add :label, :integer, primary_key: true
+      add :post_id, references(:posts, column: :id, type: :string, on_delete: :delete_all), null: false
+      add :project_id, references(:projects, type: :string, on_delete: :delete_all), null: false
+      add :content_hash, :string, null: false
+      add :vector, :text
+    end
+
+    create table(:dismissed_duplicate_pairs, primary_key: false) do
+      add :id, :string, primary_key: true
+      add :project_id, references(:projects, type: :string, on_delete: :delete_all), null: false
+
+      add :post_id_a, references(:posts, column: :id, type: :string, on_delete: :delete_all),
+        null: false
+
+      add :post_id_b, references(:posts, column: :id, type: :string, on_delete: :delete_all),
+        null: false
+
+      add :dismissed_at, :integer, null: false
+    end
+
+    create unique_index(:dismissed_duplicate_pairs, [:project_id, :post_id_a, :post_id_b],
+             name: :dismissed_pairs_idx
+           )
+
+    create table(:import_definitions, primary_key: false) do
+      add :id, :string, primary_key: true
+      add :project_id, references(:projects, type: :string, on_delete: :delete_all), null: false
+      add :name, :string, null: false
+      add :wxr_file_path, :string
+      add :uploads_folder_path, :string
+      add :last_analysis_result, :text
+      add :created_at, :integer, null: false
+      add :updated_at, :integer, null: false
+    end
+
+    create table(:db_notifications) do
+      add :entity_type, :string, null: false
+      add :entity_id, :string, null: false
+      add :action, :string, null: false
+      add :from_cli, :boolean, null: false, default: true
+      add :seen_at, :integer
+      add :created_at, :integer, null: false
+    end
+
+    execute(
+      """
+      CREATE VIRTUAL TABLE posts_fts USING fts5(
+        post_id UNINDEXED,
+        title,
+        excerpt,
+        content,
+        tags,
+        categories
+      )
+      """,
+      "DROP TABLE IF EXISTS posts_fts"
+    )
+
+    execute(
+      """
+      CREATE VIRTUAL TABLE media_fts USING fts5(
+        media_id UNINDEXED,
+        title,
+        alt,
+        caption,
+        original_name,
+        tags
+      )
+      """,
+      "DROP TABLE IF EXISTS media_fts"
+    )
+  end
+end
diff --git a/test/bds/repo/schema_migration_test.exs b/test/bds/repo/schema_migration_test.exs
new file mode 100644
index 0000000..a31431d
--- /dev/null
+++ b/test/bds/repo/schema_migration_test.exs
@@ -0,0 +1,381 @@
+defmodule BDS.Repo.SchemaMigrationTest do
+  use ExUnit.Case, async: false
+
+  alias Ecto.Adapters.SQL
+
+  setup do
+    :ok = Ecto.Adapters.SQL.Sandbox.checkout(BDS.Repo)
+  end
+
+  test "creates the persisted tables and columns declared by the schema spec" do
+    expected_columns = %{
+      "projects" => [
+        "id",
+        "name",
+        "slug",
+        "description",
+        "data_path",
+        "created_at",
+        "updated_at",
+        "is_active"
+      ],
+      "posts" => [
+        "project_id",
+        "id",
+        "title",
+        "slug",
+        "excerpt",
+        "content",
+        "status",
+        "author",
+        "created_at",
+        "updated_at",
+        "published_at",
+        "file_path",
+        "checksum",
+        "tags",
+        "categories",
+        "template_slug",
+        "language",
+        "do_not_translate",
+        "published_title",
+        "published_content",
+        "published_tags",
+        "published_categories",
+        "published_excerpt"
+      ],
+      "post_translations" => [
+        "id",
+        "project_id",
+        "translation_for",
+        "language",
+        "title",
+        "excerpt",
+        "content",
+        "status",
+        "created_at",
+        "updated_at",
+        "published_at",
+        "file_path",
+        "checksum"
+      ],
+      "media" => [
+        "project_id",
+        "id",
+        "filename",
+        "original_name",
+        "mime_type",
+        "size",
+        "width",
+        "height",
+        "title",
+        "alt",
+        "caption",
+        "author",
+        "file_path",
+        "sidecar_path",
+        "created_at",
+        "updated_at",
+        "checksum",
+        "tags",
+        "language"
+      ],
+      "media_translations" => [
+        "id",
+        "project_id",
+        "translation_for",
+        "language",
+        "title",
+        "alt",
+        "caption",
+        "created_at",
+        "updated_at"
+      ],
+      "tags" => [
+        "id",
+        "project_id",
+        "name",
+        "color",
+        "post_template_slug",
+        "created_at",
+        "updated_at"
+      ],
+      "templates" => [
+        "id",
+        "project_id",
+        "slug",
+        "title",
+        "kind",
+        "enabled",
+        "version",
+        "file_path",
+        "status",
+        "content",
+        "created_at",
+        "updated_at"
+      ],
+      "scripts" => [
+        "id",
+        "project_id",
+        "slug",
+        "title",
+        "kind",
+        "entrypoint",
+        "enabled",
+        "version",
+        "file_path",
+        "status",
+        "content",
+        "created_at",
+        "updated_at"
+      ],
+      "post_links" => ["id", "source_post_id", "target_post_id", "link_text", "created_at"],
+      "post_media" => ["id", "project_id", "post_id", "media_id", "sort_order", "created_at"],
+      "settings" => ["key", "value", "updated_at"],
+      "generated_file_hashes" => ["project_id", "relative_path", "content_hash", "updated_at"],
+      "chat_conversations" => [
+        "id",
+        "title",
+        "model",
+        "copilot_session_id",
+        "created_at",
+        "updated_at"
+      ],
+      "chat_messages" => [
+        "id",
+        "conversation_id",
+        "role",
+        "content",
+        "tool_call_id",
+        "tool_calls",
+        "created_at"
+      ],
+      "ai_providers" => ["id", "name", "env", "package_ref", "api", "doc", "updated_at"],
+      "ai_models" => [
+        "provider",
+        "model_id",
+        "name",
+        "family",
+        "attachment",
+        "reasoning",
+        "tool_call",
+        "structured_output",
+        "temperature",
+        "knowledge",
+        "release_date",
+        "last_updated_date",
+        "open_weights",
+        "input_price",
+        "output_price",
+        "cache_read_price",
+        "cache_write_price",
+        "context_window",
+        "max_input_tokens",
+        "max_output_tokens",
+        "interleaved",
+        "status",
+        "provider_package_ref",
+        "updated_at"
+      ],
+      "ai_model_modalities" => ["provider", "model_id", "direction", "modality"],
+      "ai_catalog_meta" => ["key", "value"],
+      "embedding_keys" => ["label", "post_id", "project_id", "content_hash", "vector"],
+      "dismissed_duplicate_pairs" => ["id", "project_id", "post_id_a", "post_id_b", "dismissed_at"],
+      "import_definitions" => [
+        "id",
+        "project_id",
+        "name",
+        "wxr_file_path",
+        "uploads_folder_path",
+        "last_analysis_result",
+        "created_at",
+        "updated_at"
+      ],
+      "db_notifications" => [
+        "id",
+        "entity_type",
+        "entity_id",
+        "action",
+        "from_cli",
+        "seen_at",
+        "created_at"
+      ]
+    }
+
+    actual_tables =
+      query_rows("SELECT name FROM sqlite_master WHERE type = 'table'")
+      |> Enum.map(&hd/1)
+      |> MapSet.new()
+
+    for {table, columns} <- expected_columns do
+      assert MapSet.member?(actual_tables, table), "expected table #{table} to exist"
+      assert MapSet.new(table_columns(table)) == MapSet.new(columns)
+    end
+  end
+
+  test "creates the unique indexes and primary keys declared by the schema spec" do
+    assert has_unique_index?("projects", ["slug"])
+    assert unique_index_columns("posts", "posts_project_slug_idx") == ["project_id", "slug"]
+
+    assert unique_index_columns(
+             "post_translations",
+             "post_translations_translation_language_idx"
+           ) == ["translation_for", "language"]
+
+    assert unique_index_columns(
+             "media_translations",
+             "media_translations_translation_language_idx"
+           ) == ["translation_for", "language"]
+
+    assert unique_index_columns("tags", "tags_project_name_idx") == ["project_id", "name"]
+    assert unique_index_columns("scripts", "scripts_project_slug_idx") == ["project_id", "slug"]
+    assert unique_index_columns("templates", "templates_project_slug_idx") == ["project_id", "slug"]
+    assert unique_index_columns("post_media", "post_media_post_media_idx") == ["post_id", "media_id"]
+
+    assert unique_index_columns(
+             "generated_file_hashes",
+             "generated_file_hashes_project_path_idx"
+           ) == ["project_id", "relative_path"]
+
+    assert unique_index_columns("dismissed_duplicate_pairs", "dismissed_pairs_idx") == [
+             "project_id",
+             "post_id_a",
+             "post_id_b"
+           ]
+
+    assert primary_key_columns("ai_models") == ["provider", "model_id"]
+
+    assert primary_key_columns("ai_model_modalities") == [
+             "provider",
+             "model_id",
+             "direction",
+             "modality"
+           ]
+  end
+
+  test "creates standalone FTS5 virtual tables for post and media search" do
+    posts_fts_sql = table_sql("posts_fts")
+    media_fts_sql = table_sql("media_fts")
+
+    assert posts_fts_sql =~ "CREATE VIRTUAL TABLE posts_fts USING fts5"
+    assert posts_fts_sql =~ "post_id UNINDEXED"
+    assert posts_fts_sql =~ "title"
+    assert posts_fts_sql =~ "excerpt"
+    assert posts_fts_sql =~ "content"
+    assert posts_fts_sql =~ "tags"
+    assert posts_fts_sql =~ "categories"
+    refute posts_fts_sql =~ "content="
+
+    assert media_fts_sql =~ "CREATE VIRTUAL TABLE media_fts USING fts5"
+    assert media_fts_sql =~ "media_id UNINDEXED"
+    assert media_fts_sql =~ "title"
+    assert media_fts_sql =~ "alt"
+    assert media_fts_sql =~ "caption"
+    assert media_fts_sql =~ "original_name"
+    assert media_fts_sql =~ "tags"
+    refute media_fts_sql =~ "content="
+  end
+
+  test "applies key defaults and foreign keys from the persistence contract" do
+    assert column_metadata("projects", "is_active")[:default] == "false"
+    assert column_metadata("posts", "status")[:default] == "draft"
+    assert column_metadata("posts", "file_path")[:default] == ""
+    assert column_metadata("posts", "do_not_translate")[:default] == "false"
+    assert column_metadata("post_media", "sort_order")[:default] == "0"
+    assert column_metadata("db_notifications", "from_cli")[:default] == "true"
+
+    assert foreign_keys("posts") == [%{from: "project_id", table: "projects", to: "id"}]
+
+    assert Enum.sort_by(foreign_keys("post_media"), & &1.from) == [
+             %{from: "media_id", table: "media", to: "id"},
+             %{from: "post_id", table: "posts", to: "id"},
+             %{from: "project_id", table: "projects", to: "id"}
+           ]
+
+    assert Enum.sort_by(foreign_keys("post_translations"), & &1.from) == [
+             %{from: "project_id", table: "projects", to: "id"},
+             %{from: "translation_for", table: "posts", to: "id"}
+           ]
+
+    assert Enum.sort_by(foreign_keys("media_translations"), & &1.from) == [
+             %{from: "project_id", table: "projects", to: "id"},
+             %{from: "translation_for", table: "media", to: "id"}
+           ]
+
+    assert foreign_keys("chat_messages") == [
+             %{from: "conversation_id", table: "chat_conversations", to: "id"}
+           ]
+
+    assert foreign_keys("ai_models") == [%{from: "provider", table: "ai_providers", to: "id"}]
+  end
+
+  defp table_columns(table) do
+    query_rows("PRAGMA table_info(#{table})")
+    |> Enum.map(fn [_cid, name, _type, _notnull, _default, _pk] -> name end)
+  end
+
+  defp column_metadata(table, column_name) do
+    query_rows("PRAGMA table_info(#{table})")
+    |> Enum.find_value(fn [_cid, name, type, notnull, default, pk] ->
+      if name == column_name do
+        %{
+          type: type,
+          notnull: notnull,
+          default: normalize_default(default),
+          pk: pk
+        }
+      end
+    end)
+  end
+
+  defp foreign_keys(table) do
+    query_rows("PRAGMA foreign_key_list(#{table})")
+    |> Enum.map(fn [_id, _seq, foreign_table, from, to, _on_update, _on_delete, _match] ->
+      %{from: from, table: foreign_table, to: to}
+    end)
+  end
+
+  defp primary_key_columns(table) do
+    query_rows("PRAGMA table_info(#{table})")
+    |> Enum.filter(fn [_cid, _name, _type, _notnull, _default, pk] -> pk > 0 end)
+    |> Enum.sort_by(fn [_cid, _name, _type, _notnull, _default, pk] -> pk end)
+    |> Enum.map(fn [_cid, name, _type, _notnull, _default, _pk] -> name end)
+  end
+
+  defp unique_index_columns(table, index_name) do
+    indexes = query_rows("PRAGMA index_list(#{table})")
+
+    assert Enum.any?(indexes, fn [_seq, name, unique | _rest] -> name == index_name and unique == 1 end),
+           "expected unique index #{index_name} on #{table}"
+
+    query_rows("PRAGMA index_info(#{index_name})")
+    |> Enum.sort_by(fn [seqno | _rest] -> seqno end)
+    |> Enum.map(fn [_seqno, _cid, name] -> name end)
+  end
+
+  defp has_unique_index?(table, columns) do
+    query_rows("PRAGMA index_list(#{table})")
+    |> Enum.filter(fn [_seq, _name, unique | _rest] -> unique == 1 end)
+    |> Enum.any?(fn [_seq, name, _unique | _rest] ->
+      unique_index_columns(table, name) == columns
+    end)
+  end
+
+  defp table_sql(table) do
+    [[sql]] = query_rows("SELECT sql FROM sqlite_master WHERE name = '#{table}'")
+    sql
+  end
+
+  defp query_rows(statement) do
+    %{rows: rows} = SQL.query!(BDS.Repo, statement, [])
+    rows
+  end
+
+  defp normalize_default(nil), do: nil
+
+  defp normalize_default(default) do
+    default
+    |> String.trim_leading("'")
+    |> String.trim_trailing("'")
+  end
+end