fix: fix CSM-005

CODESMELL.md

@@ -86,19 +86,35 @@
 
 ---
 
-### CSM-005 — Client-Side Filtering of Entire Tables
+### ~~CSM-005 — Client-Side Filtering of Entire Tables~~ ✅ FIXED
-- **Files:** `lib/bds/ui/sidebar.ex`, `lib/bds/tags.ex`, `lib/bds/ui/dashboard.ex`
+- **Fixed:** 2026-05-08
-- **What:**
+- **What was done:**
-  - `UI.Sidebar.list_posts/1` (Z. 464-482) loads every post for a project, then `apply_post_filters/1` (Z. 608-615) filters in Elixir. Translation counts are a separate query but still unbounded.
+  - **Sidebar** (`lib/bds/ui/sidebar.ex`):
-  - `Tags.posts_with_tag/2` (Z. 310-312) and `Tags.posts_with_any_tag/2` (Z. 314-316) load **all posts** to filter by tag name. `Tags.post_tag_names/1` (Z. 320-322) loads all posts to extract tag names.
+    - Removed `list_posts/1` and `list_media/1` that loaded all records into memory.
-  - `UI.Dashboard.snapshot/1` (Z. 14-35) loads ALL posts and ALL media for counts and stats. Post stats (Z. 77-87) and media stats (Z. 89-96) are computed in Elixir via `Enum.reduce`.
+    - Replaced `apply_post_filters/1` and `apply_media_filters/1` (Elixir-side filtering) with SQL `WHERE` clauses using Ecto dynamic queries and SQLite `json_each` fragments.
-  - `Posts.dashboard_stats/1` (posts.ex:374-394) loads all post statuses and counts in Elixir.
+    - Page/non-page split now uses `EXISTS (SELECT 1 FROM json_each(categories) WHERE lower(value) = 'page')` in SQL.
-- **Fix:**
+    - Search, year/month, tag, and category filters all push to SQL via `maybe_where_search`, `maybe_where_year`, `maybe_where_month`, `maybe_where_all_tags`, `maybe_where_all_categories`.
-  - Sidebar: Use `Repo.aggregate` for counts, `where` clauses for filters, `limit`/`offset` for pagination. Preload tag colors separately.
+    - Aggregate queries (`year_month_counts`, `available_tags`, `available_categories`) use `Ecto.Adapters.SQL.query!` with `json_each` cross-joins, `GROUP BY`, and `DISTINCT`.
-  - Tags: Use `where: fragment("? IN (?)", ^tag_name, post.tags)` or JSON functions. For `post_tag_names`, use `Repo.aggregate` + `distinct`.
+    - Pagination uses SQL `LIMIT` instead of `Enum.take`.
-  - Dashboard: Use `Repo.aggregate(:count)` with `group_by` for status counts, media counts, tag clouds, and category counts. No need to load full records.
+    - `tag_count/1` replaces `list_tags/1` + `length/1` with `Repo.one(select: count(tag.id))`.
-  - `dashboard_stats`: Replace with `from post in Post, where: post.project_id == ^project_id, group_by: post.status, select: {post.status, count(post.id)}`.
+    - Fixed `group_posts/1` O(n²) `acc.draft ++ [post]` pattern — now uses `Enum.group_by/2` (also fixes CSM-024).
-- **Test:** Create 10,000 posts; open the sidebar; assert the LiveView process memory stays bounded.
+  - **Tags** (`lib/bds/tags.ex`):
+    - `posts_with_tag/2` now uses `EXISTS (SELECT 1 FROM json_each(?) WHERE value = ?)` instead of loading all posts.
+    - `posts_with_any_tag/2` now uses `json_each` cross-join with a JSON parameter for the tag name list.
+    - `post_tag_names/1` now selects only the `tags` column instead of loading full post records.
+  - **Dashboard** (`lib/bds/ui/dashboard.ex`):
+    - `post_stats` uses `GROUP BY post.status, SELECT {status, count(id)}` — no longer loads all posts.
+    - `media_stats` uses `SELECT count(id), coalesce(sum(size), 0)` and a separate image count query with `LIKE 'image/%'`.
+    - `tag_cloud_items` and `category_counts` use raw SQL with `json_each` cross-joins and `GROUP BY`.
+    - `timeline_entries` uses SQL `strftime` + `GROUP BY` for year/month aggregation.
+    - `recent_posts` uses SQL `ORDER BY updated_at DESC LIMIT 5`.
+  - **Posts** (`lib/bds/posts.ex`):
+    - `dashboard_stats/1` uses `GROUP BY post.status, SELECT {status, count(id)}` instead of loading all statuses.
+  - **Capabilities** (`lib/bds/scripting/capabilities/`):
+    - `tag_post_ids/2` uses `json_each` fragment + `SELECT post.id` instead of loading all posts.
+    - `names_with_counts/2` uses raw SQL with `json_each` + `GROUP BY` instead of loading all posts.
+    - `posts_by_status/2` filters at SQL level instead of loading all posts and filtering in Elixir.
+  - Added 20 tests in `test/bds/csm005_sql_filtering_test.exs` covering dashboard stats, tag cloud, sidebar page/post separation, tag/search/year-month filters, available aggregates, and media filtering.
 
 ---
 

lib/bds/posts.ex

@@ -380,20 +380,17 @@ defmodule BDS.Posts do
     Repo.all(
       from(post in Post,
         where: post.project_id == ^project_id,
-        select: post.status
+        group_by: post.status,
+        select: {post.status, count(post.id)}
       )
     )
     |> Enum.reduce(
       %{total_posts: 0, draft_count: 0, published_count: 0, archived_count: 0},
-      fn status, acc ->
+      fn
-        acc
+        {:draft, n}, acc -> %{acc | total_posts: acc.total_posts + n, draft_count: n}
-        |> Map.update!(:total_posts, &(&1 + 1))
+        {:published, n}, acc -> %{acc | total_posts: acc.total_posts + n, published_count: n}
-        |> case do
+        {:archived, n}, acc -> %{acc | total_posts: acc.total_posts + n, archived_count: n}
-          counts when status == :draft -> Map.update!(counts, :draft_count, &(&1 + 1))
+        {_other, n}, acc -> %{acc | total_posts: acc.total_posts + n}
-          counts when status == :published -> Map.update!(counts, :published_count, &(&1 + 1))
-          counts when status == :archived -> Map.update!(counts, :archived_count, &(&1 + 1))
-          counts -> counts
-        end
       end
     )
   end

lib/bds/scripting/capabilities/crud.ex

@@ -212,12 +212,17 @@ defmodule BDS.Scripting.Capabilities.Crud do
       %Tag{name: tag_name} ->
         Repo.all(
           from(post in Post,
-            where: post.project_id == ^project_id,
+            where:
-            order_by: [asc: post.created_at]
+              post.project_id == ^project_id and
+                fragment(
+                  "EXISTS (SELECT 1 FROM json_each(?) WHERE value = ?)",
+                  post.tags,
+                  ^tag_name
+                ),
+            order_by: [asc: post.created_at],
+            select: post.id
           )
         )
-        |> Enum.filter(&(tag_name in (&1.tags || [])))
-        |> Enum.map(& &1.id)
 
       _other ->
         []

lib/bds/scripting/capabilities/posts.ex

@@ -96,9 +96,13 @@ defmodule BDS.Scripting.Capabilities.Posts do
     normalized_status = string_or_nil(status) || ""
 
     Repo.all(
-      from(post in Post, where: post.project_id == ^project_id, order_by: [asc: post.created_at])
+      from(post in Post,
+        where:
+          post.project_id == ^project_id and
+            fragment("CAST(? AS TEXT) = ?", post.status, ^normalized_status),
+        order_by: [asc: post.created_at]
+      )
     )
-    |> Enum.filter(&(to_string(&1.status) == normalized_status))
     |> Enum.map(&post_payload/1)
   end
 
@@ -278,15 +282,18 @@ defmodule BDS.Scripting.Capabilities.Posts do
   end
 
   def names_with_counts(project_id, field) when field in [:tags, :categories] do
-    Repo.all(
+    column = Atom.to_string(field)
-      from(post in Post,
+
-        where: post.project_id == ^project_id,
+    %{rows: rows} =
-        order_by: [asc: post.created_at]
+      Ecto.Adapters.SQL.query!(
+        Repo,
+        "SELECT trim(je.value) AS name, COUNT(*) AS cnt " <>
+          "FROM posts, json_each(posts.#{column}) je " <>
+          "WHERE posts.project_id = ?1 AND trim(je.value) != '' " <>
+          "GROUP BY name ORDER BY lower(name), cnt",
+        [project_id]
       )
-    )
+
-    |> Enum.flat_map(&(Map.get(&1, field) || []))
+    Enum.map(rows, fn [name, count] -> %{"name" => name, "count" => count} end)
-    |> Enum.reduce(%{}, fn name, acc -> Map.update(acc, name, 1, &(&1 + 1)) end)
-    |> Enum.map(fn {name, count} -> %{"name" => name, "count" => count} end)
-    |> Enum.sort_by(&{String.downcase(&1["name"]), &1["count"]})
   end
 end

lib/bds/tags.ex

@@ -316,19 +316,41 @@ defmodule BDS.Tags do
   end
 
   defp posts_with_tag(project_id, tag_name) do
-    Repo.all(from post in Post, where: post.project_id == ^project_id)
+    Repo.all(
-    |> Enum.filter(fn post -> tag_name in (post.tags || []) end)
+      from post in Post,
+        where:
+          post.project_id == ^project_id and
+            fragment(
+              "EXISTS (SELECT 1 FROM json_each(?) WHERE value = ?)",
+              post.tags,
+              ^tag_name
+            )
+    )
   end
 
   defp posts_with_any_tag(project_id, tag_names) do
-    Repo.all(from post in Post, where: post.project_id == ^project_id)
+    tags_json = Jason.encode!(tag_names)
-    |> Enum.filter(fn post -> Enum.any?(post.tags || [], &(&1 in tag_names)) end)
+
+    Repo.all(
+      from post in Post,
+        where:
+          post.project_id == ^project_id and
+            fragment(
+              "EXISTS (SELECT 1 FROM json_each(?) je, json_each(?) jt WHERE je.value = jt.value)",
+              post.tags,
+              ^tags_json
+            )
+    )
   end
 
   defp post_tag_names(project_id) do
-    Repo.all(from post in Post, where: post.project_id == ^project_id)
+    Repo.all(
-    |> Enum.flat_map(fn post ->
+      from post in Post,
-      post.tags
+        where: post.project_id == ^project_id,
+        select: post.tags
+    )
+    |> Enum.flat_map(fn tags ->
+      tags
       |> Kernel.||([])
       |> Enum.map(&String.trim/1)
       |> Enum.reject(&(&1 == ""))

lib/bds/ui/dashboard.ex

@@ -11,53 +11,15 @@ defmodule BDS.UI.Dashboard do
   def snapshot(nil), do: empty_snapshot()
 
   def snapshot(project_id) when is_binary(project_id) do
-    posts =
-      Repo.all(
-        from post in Post,
-          where: post.project_id == ^project_id,
-          select: %{
-            id: post.id,
-            title: post.title,
-            slug: post.slug,
-            status: post.status,
-            tags: post.tags,
-            categories: post.categories,
-            created_at: post.created_at,
-            updated_at: post.updated_at
-          }
-      )
-
-    media_items =
-      Repo.all(
-        from media in Media,
-          where: media.project_id == ^project_id,
-          select: %{mime_type: media.mime_type, size: media.size}
-      )
-
-    tag_colors =
-      Repo.all(
-        from tag in Tag,
-          where: tag.project_id == ^project_id,
-          select: %{name: tag.name, color: tag.color}
-      )
-      |> Enum.reduce(%{}, fn %{name: name, color: color}, acc ->
-        if blank?(color), do: acc, else: Map.put(acc, name, color)
-      end)
-
-    post_stats = post_stats(posts)
-    media_stats = media_stats(media_items)
-    tag_cloud_items = tag_cloud_items(posts, tag_colors)
-    category_counts = category_counts(posts)
-
     %{
       title: "dashboard.title",
       subtitle: "dashboard.subtitle",
-      post_stats: post_stats,
+      post_stats: post_stats(project_id),
-      media_stats: media_stats,
+      media_stats: media_stats(project_id),
-      timeline_entries: timeline_entries(posts),
+      timeline_entries: timeline_entries(project_id),
-      tag_cloud_items: tag_cloud_items,
+      tag_cloud_items: tag_cloud_items(project_id),
-      category_counts: category_counts,
+      category_counts: category_counts(project_id),
-      recent_posts: recent_posts(posts)
+      recent_posts: recent_posts(project_id)
     }
   end
 
@@ -74,61 +36,142 @@ defmodule BDS.UI.Dashboard do
     }
   end
 
-  defp post_stats(posts) do
+  defp post_stats(project_id) do
-    Enum.reduce(
+    Repo.all(
-      posts,
+      from post in Post,
+        where: post.project_id == ^project_id,
+        group_by: post.status,
+        select: {post.status, count(post.id)}
+    )
+    |> Enum.reduce(
       %{total_posts: 0, draft_count: 0, published_count: 0, archived_count: 0},
-      fn post, acc ->
+      fn
-        acc
+        {:draft, n}, acc -> %{acc | total_posts: acc.total_posts + n, draft_count: n}
-        |> Map.update!(:total_posts, &(&1 + 1))
+        {:published, n}, acc -> %{acc | total_posts: acc.total_posts + n, published_count: n}
-        |> increment_status(post.status)
+        {:archived, n}, acc -> %{acc | total_posts: acc.total_posts + n, archived_count: n}
+        {_other, n}, acc -> %{acc | total_posts: acc.total_posts + n}
       end
     )
   end
 
-  defp media_stats(media_items) do
+  defp media_stats(project_id) do
-    Enum.reduce(media_items, %{media_count: 0, image_count: 0, total_bytes: 0}, fn media, acc ->
+    {media_count, total_bytes} =
-      acc
+      Repo.one(
-      |> Map.update!(:media_count, &(&1 + 1))
+        from media in Media,
-      |> Map.update!(:total_bytes, &(&1 + (media.size || 0)))
+          where: media.project_id == ^project_id,
-      |> maybe_increment_image_count(media.mime_type)
+          select: {count(media.id), coalesce(sum(media.size), 0)}
-    end)
+      )
+
+    image_count =
+      Repo.one(
+        from media in Media,
+          where: media.project_id == ^project_id and like(media.mime_type, "image/%"),
+          select: count(media.id)
+      )
+
+    %{media_count: media_count, image_count: image_count, total_bytes: total_bytes}
   end
 
-  defp timeline_entries(posts) do
+  defp timeline_entries(project_id) do
-    posts
+    Repo.all(
-    |> Enum.reduce(%{}, fn post, acc ->
+      from post in Post,
-      datetime = DateTime.from_unix!(post.created_at, :millisecond)
+        where: post.project_id == ^project_id,
-      key = {datetime.year, datetime.month}
+        group_by: [
-      Map.update(acc, key, 1, &(&1 + 1))
+          fragment("CAST(strftime('%Y', datetime(? / 1000, 'unixepoch')) AS INTEGER)", post.created_at),
-    end)
+          fragment("CAST(strftime('%m', datetime(? / 1000, 'unixepoch')) AS INTEGER)", post.created_at)
-    |> Enum.map(fn {{year, month}, count} -> %{year: year, month: month, count: count} end)
+        ],
-    |> Enum.sort_by(&{&1.year, &1.month})
+        select: %{
+          year:
+            fragment(
+              "CAST(strftime('%Y', datetime(? / 1000, 'unixepoch')) AS INTEGER)",
+              post.created_at
+            ),
+          month:
+            fragment(
+              "CAST(strftime('%m', datetime(? / 1000, 'unixepoch')) AS INTEGER)",
+              post.created_at
+            ),
+          count: count(post.id)
+        },
+        order_by: [
+          asc:
+            fragment(
+              "CAST(strftime('%Y', datetime(? / 1000, 'unixepoch')) AS INTEGER)",
+              post.created_at
+            ),
+          asc:
+            fragment(
+              "CAST(strftime('%m', datetime(? / 1000, 'unixepoch')) AS INTEGER)",
+              post.created_at
+            )
+        ]
+    )
     |> Enum.take(-12)
   end
 
-  defp tag_cloud_items(posts, tag_colors) do
+  defp tag_cloud_items(project_id) do
-    posts
+    tag_colors =
-    |> Enum.flat_map(&normalize_terms(&1.tags))
+      Repo.all(
-    |> Enum.frequencies()
+        from tag in Tag,
-    |> Enum.map(fn {tag, count} -> %{tag: tag, count: count, color: Map.get(tag_colors, tag)} end)
+          where: tag.project_id == ^project_id,
-    |> Enum.sort_by(fn %{tag: tag, count: count} -> {-count, String.downcase(tag)} end)
+          where: not is_nil(tag.color) and tag.color != "",
-  end
+          select: {tag.name, tag.color}
+      )
+      |> Map.new()
 
-  defp category_counts(posts) do
+    %{rows: rows} =
-    posts
+      Ecto.Adapters.SQL.query!(
-    |> Enum.flat_map(&normalize_terms(&1.categories))
+        Repo,
-    |> Enum.frequencies()
+        """
-    |> Enum.map(fn {category, count} -> %{category: category, count: count} end)
+        SELECT trim(je.value) AS tag, COUNT(*) AS cnt
-    |> Enum.sort_by(fn %{category: category, count: count} ->
+        FROM posts, json_each(posts.tags) je
-      {-count, String.downcase(category)}
+        WHERE posts.project_id = ?1
+          AND trim(je.value) != ''
+        GROUP BY tag
+        ORDER BY cnt DESC, lower(tag) ASC
+        """,
+        [project_id]
+      )
+
+    Enum.map(rows, fn [tag, count] ->
+      %{tag: tag, count: count, color: Map.get(tag_colors, tag)}
     end)
   end
 
-  defp recent_posts(posts) do
+  defp category_counts(project_id) do
-    posts
+    %{rows: rows} =
-    |> Enum.sort_by(& &1.updated_at, :desc)
+      Ecto.Adapters.SQL.query!(
-    |> Enum.take(5)
+        Repo,
+        """
+        SELECT trim(je.value) AS category, COUNT(*) AS cnt
+        FROM posts, json_each(posts.categories) je
+        WHERE posts.project_id = ?1
+          AND trim(je.value) != ''
+        GROUP BY category
+        ORDER BY cnt DESC, lower(category) ASC
+        """,
+        [project_id]
+      )
+
+    Enum.map(rows, fn [category, count] ->
+      %{category: category, count: count}
+    end)
+  end
+
+  defp recent_posts(project_id) do
+    Repo.all(
+      from post in Post,
+        where: post.project_id == ^project_id,
+        order_by: [desc: post.updated_at],
+        limit: 5,
+        select: %{
+          id: post.id,
+          title: post.title,
+          slug: post.slug,
+          status: post.status,
+          updated_at: post.updated_at
+        }
+    )
     |> Enum.map(fn post ->
       %{
         id: post.id,
@@ -139,30 +182,9 @@ defmodule BDS.UI.Dashboard do
     end)
   end
 
-  defp normalize_terms(values) do
-    values
-    |> Kernel.||([])
-    |> Enum.map(&to_string/1)
-    |> Enum.map(&String.trim/1)
-    |> Enum.reject(&(&1 == ""))
-  end
-
   defp display_title(post) do
     if blank?(post.title), do: post.slug || "", else: post.title
   end
 
-  defp increment_status(counts, :draft), do: Map.update!(counts, :draft_count, &(&1 + 1))
-  defp increment_status(counts, :published), do: Map.update!(counts, :published_count, &(&1 + 1))
-  defp increment_status(counts, :archived), do: Map.update!(counts, :archived_count, &(&1 + 1))
-  defp increment_status(counts, _status), do: counts
-
-  defp maybe_increment_image_count(counts, mime_type) when is_binary(mime_type) do
-    if String.starts_with?(mime_type, "image/"),
-      do: Map.update!(counts, :image_count, &(&1 + 1)),
-      else: counts
-  end
-
-  defp maybe_increment_image_count(counts, _mime_type), do: counts
-
   defp blank?(value), do: value in [nil, ""]
 end

lib/bds/ui/sidebar.ex

@@ -14,7 +14,6 @@ defmodule BDS.UI.Sidebar do
   alias BDS.Tags.Tag
   alias BDS.Templates.Template
 
-  @page_category "page"
   @default_page_size 500
 
   def snapshot(nil), do: empty_snapshot()
@@ -74,7 +73,7 @@ defmodule BDS.UI.Sidebar do
         )
 
       "tags" ->
-        tags_nav_view(list_tags(project_id))
+        tags_nav_view(tag_count(project_id))
 
       "chat" ->
         entity_list_view(
@@ -122,7 +121,7 @@ defmodule BDS.UI.Sidebar do
           "templates",
           []
         ),
-      "tags" => tags_nav_view([]),
+      "tags" => tags_nav_view(0),
       "chat" =>
         entity_list_view(
           dgettext("ui", "Chat"),
@@ -142,9 +141,9 @@ defmodule BDS.UI.Sidebar do
     }
   end
 
-  defp empty_view("posts"), do: posts_view_data([], [], %{}, false, empty_filter_params(), %{})
+  defp empty_view("posts"), do: build_posts_view([], %{}, false, empty_filter_params(), %{}, [], [], [])
-  defp empty_view("pages"), do: posts_view_data([], [], %{}, true, empty_filter_params(), %{})
+  defp empty_view("pages"), do: build_posts_view([], %{}, true, empty_filter_params(), %{}, [], [], [])
-  defp empty_view("media"), do: media_view_data([], [], empty_filter_params(), %{})
+  defp empty_view("media"), do: build_media_view([], empty_filter_params(), %{}, [], [], 0)
 
   defp empty_view("scripts"),
     do:
@@ -164,7 +163,7 @@ defmodule BDS.UI.Sidebar do
         []
       )
 
-  defp empty_view("tags"), do: tags_nav_view([])
+  defp empty_view("tags"), do: tags_nav_view(0)
 
   defp empty_view("chat"),
     do:
@@ -196,29 +195,70 @@ defmodule BDS.UI.Sidebar do
       empty_message: dgettext("ui", "No items")
     }
 
+  # ---------------------------------------------------------------------------
+  # Posts view (SQL-level filtering)
+  # ---------------------------------------------------------------------------
+
   defp posts_view(project_id, params, pages?) do
-    posts = list_posts(project_id)
+    filters = normalize_filter_params(params)
     translation_counts = translation_counts(project_id)
     tag_colors = tag_color_map(project_id)
-    filters = normalize_filter_params(params)
-    base_posts = Enum.filter(posts, &(page_post?(&1) == pages?))
-    filtered_posts = apply_post_filters(base_posts, filters)
 
-    posts_view_data(base_posts, filtered_posts, translation_counts, pages?, filters, tag_colors)
+    year_months = base_year_month_counts(project_id, pages?)
+    avail_tags = base_available_tags(project_id, pages?)
+    avail_categories = base_available_categories(project_id, pages?)
+
+    filtered_query =
+      base_posts_query(project_id, pages?)
+      |> apply_post_query_filters(filters)
+
+    total_count = Repo.one(from p in filtered_query, select: count(p.id))
+
+    limited_posts =
+      Repo.all(
+        from p in filtered_query,
+          order_by: [desc: p.created_at],
+          limit: ^filters.display_limit,
+          select: %{
+            id: p.id,
+            title: p.title,
+            slug: p.slug,
+            excerpt: p.excerpt,
+            status: p.status,
+            tags: p.tags,
+            categories: p.categories,
+            updated_at: p.updated_at,
+            published_at: p.published_at,
+            language: p.language
+          }
+      )
+
+    build_posts_view(
+      limited_posts,
+      translation_counts,
+      pages?,
+      filters,
+      tag_colors,
+      year_months,
+      avail_tags,
+      avail_categories,
+      total_count
+    )
   end
 
-  defp posts_view_data(
+  defp build_posts_view(
-         base_posts,
+         limited_posts,
-         filtered_posts,
          translation_counts,
          pages?,
          filters,
-         tag_colors
+         tag_colors,
+         year_month_counts,
+         available_tags,
+         available_categories,
+         total_count \\ 0
        ) do
-    limited_posts = Enum.take(filtered_posts, filters.display_limit)
     grouped_posts = group_posts(limited_posts)
-    available_tags = available_tags(base_posts, & &1.tags)
+    loaded_count = length(limited_posts)
-    available_categories = available_categories(base_posts, pages?)
 
     %{
       title: if(pages?, do: dgettext("ui", "Pages"), else: dgettext("ui", "Posts")),
@@ -247,15 +287,15 @@ defmodule BDS.UI.Sidebar do
         results_label: dgettext("ui", "results"),
         results_for_label: dgettext("ui", "results for"),
         no_results_label: dgettext("ui", "No matching posts"),
-        year_month_counts: year_month_counts(base_posts, &post_filter_timestamp/1),
+        year_month_counts: year_month_counts,
         available_tags: available_tags,
         available_tag_colors: Map.take(tag_colors, available_tags),
         available_categories: available_categories,
         max_items: @default_page_size,
         display_limit: filters.display_limit,
-        loaded_count: length(limited_posts),
+        loaded_count: loaded_count,
-        total_count: length(filtered_posts),
+        total_count: total_count,
-        has_more: length(filtered_posts) > filters.display_limit,
+        has_more: total_count > filters.display_limit,
         has_active_filters: filter_active?(filters),
         selected: %{
           search: filters.search,
@@ -291,18 +331,237 @@ defmodule BDS.UI.Sidebar do
     }
   end
 
-  defp media_view(project_id, params) do
+  defp base_posts_query(project_id, true = _pages?) do
-    media_items = list_media(project_id)
+    from post in Post,
-    tag_colors = tag_color_map(project_id)
+      where: post.project_id == ^project_id,
-    filters = normalize_filter_params(params)
+      where:
-    filtered_media = apply_media_filters(media_items, filters)
+        fragment(
-
+          "EXISTS (SELECT 1 FROM json_each(?) WHERE lower(value) = 'page')",
-    media_view_data(media_items, filtered_media, filters, tag_colors)
+          post.categories
+        )
   end
 
-  defp media_view_data(base_media, filtered_media, filters, tag_colors) do
+  defp base_posts_query(project_id, false = _pages?) do
-    limited_media = Enum.take(filtered_media, filters.display_limit)
+    from post in Post,
-    available_tags = available_tags(base_media, & &1.tags)
+      where: post.project_id == ^project_id,
+      where:
+        fragment(
+          "NOT EXISTS (SELECT 1 FROM json_each(?) WHERE lower(value) = 'page')",
+          post.categories
+        )
+  end
+
+  defp apply_post_query_filters(query, filters) do
+    query
+    |> maybe_where_search(filters.search)
+    |> maybe_where_year(filters.year)
+    |> maybe_where_month(filters.month)
+    |> maybe_where_all_tags(filters.tags)
+    |> maybe_where_all_categories(filters.categories)
+  end
+
+  defp maybe_where_search(query, nil), do: query
+
+  defp maybe_where_search(query, search) do
+    search_term = "%" <> String.downcase(search) <> "%"
+
+    where(
+      query,
+      [p],
+      fragment(
+        """
+        lower(COALESCE(?,'') || ' ' || COALESCE(?,'') || ' ' || COALESCE(?,'')
+              || ' ' || COALESCE(?,'[]') || ' ' || COALESCE(?,'[]')) LIKE ?
+        """,
+        p.title,
+        p.slug,
+        p.excerpt,
+        p.tags,
+        p.categories,
+        ^search_term
+      )
+    )
+  end
+
+  defp maybe_where_year(query, nil), do: query
+
+  defp maybe_where_year(query, year) do
+    year_str = to_string(year)
+
+    where(
+      query,
+      [p],
+      fragment(
+        "strftime('%Y', datetime(COALESCE(?, ?) / 1000, 'unixepoch')) = ?",
+        p.published_at,
+        p.updated_at,
+        ^year_str
+      )
+    )
+  end
+
+  defp maybe_where_month(query, nil), do: query
+
+  defp maybe_where_month(query, month) do
+    month_str = String.pad_leading(to_string(month), 2, "0")
+
+    where(
+      query,
+      [p],
+      fragment(
+        "strftime('%m', datetime(COALESCE(?, ?) / 1000, 'unixepoch')) = ?",
+        p.published_at,
+        p.updated_at,
+        ^month_str
+      )
+    )
+  end
+
+  defp maybe_where_all_tags(query, []), do: query
+
+  defp maybe_where_all_tags(query, tags) do
+    Enum.reduce(tags, query, fn tag, q ->
+      where(
+        q,
+        [p],
+        fragment(
+          "EXISTS (SELECT 1 FROM json_each(?) WHERE lower(value) = lower(?))",
+          p.tags,
+          ^tag
+        )
+      )
+    end)
+  end
+
+  defp maybe_where_all_categories(query, []), do: query
+
+  defp maybe_where_all_categories(query, categories) do
+    Enum.reduce(categories, query, fn category, q ->
+      where(
+        q,
+        [p],
+        fragment(
+          "EXISTS (SELECT 1 FROM json_each(?) WHERE lower(value) = lower(?) AND lower(value) != 'page')",
+          p.categories,
+          ^category
+        )
+      )
+    end)
+  end
+
+  defp base_year_month_counts(project_id, pages?) do
+    is_page = if pages?, do: 1, else: 0
+
+    %{rows: rows} =
+      Ecto.Adapters.SQL.query!(
+        Repo,
+        """
+        SELECT CAST(strftime('%Y', datetime(COALESCE(published_at, updated_at) / 1000, 'unixepoch')) AS INTEGER),
+               CAST(strftime('%m', datetime(COALESCE(published_at, updated_at) / 1000, 'unixepoch')) AS INTEGER),
+               COUNT(*)
+        FROM posts
+        WHERE project_id = ?1
+          AND COALESCE(published_at, updated_at) IS NOT NULL
+          AND (
+            (?2 = 1 AND EXISTS (SELECT 1 FROM json_each(categories) WHERE lower(value) = 'page'))
+            OR
+            (?2 = 0 AND NOT EXISTS (SELECT 1 FROM json_each(categories) WHERE lower(value) = 'page'))
+          )
+        GROUP BY 1, 2
+        ORDER BY 1 DESC, 2 DESC
+        """,
+        [project_id, is_page]
+      )
+
+    Enum.map(rows, fn [year, month, count] -> %{year: year, month: month, count: count} end)
+  end
+
+  defp base_available_tags(project_id, pages?) do
+    is_page = if pages?, do: 1, else: 0
+
+    %{rows: rows} =
+      Ecto.Adapters.SQL.query!(
+        Repo,
+        """
+        SELECT DISTINCT trim(je.value)
+        FROM posts, json_each(posts.tags) je
+        WHERE posts.project_id = ?1
+          AND trim(je.value) != ''
+          AND (
+            (?2 = 1 AND EXISTS (SELECT 1 FROM json_each(posts.categories) WHERE lower(value) = 'page'))
+            OR
+            (?2 = 0 AND NOT EXISTS (SELECT 1 FROM json_each(posts.categories) WHERE lower(value) = 'page'))
+          )
+        ORDER BY lower(trim(je.value))
+        """,
+        [project_id, is_page]
+      )
+
+    Enum.map(rows, fn [tag] -> tag end)
+  end
+
+  defp base_available_categories(project_id, pages?) do
+    is_page = if pages?, do: 1, else: 0
+
+    %{rows: rows} =
+      Ecto.Adapters.SQL.query!(
+        Repo,
+        """
+        SELECT DISTINCT trim(je.value)
+        FROM posts, json_each(posts.categories) je
+        WHERE posts.project_id = ?1
+          AND trim(je.value) != ''
+          AND lower(trim(je.value)) != 'page'
+          AND (
+            (?2 = 1 AND EXISTS (SELECT 1 FROM json_each(posts.categories) jc WHERE lower(jc.value) = 'page'))
+            OR
+            (?2 = 0 AND NOT EXISTS (SELECT 1 FROM json_each(posts.categories) jc WHERE lower(jc.value) = 'page'))
+          )
+        ORDER BY lower(trim(je.value))
+        """,
+        [project_id, is_page]
+      )
+
+    Enum.map(rows, fn [category] -> category end)
+  end
+
+  # ---------------------------------------------------------------------------
+  # Media view (SQL-level filtering)
+  # ---------------------------------------------------------------------------
+
+  defp media_view(project_id, params) do
+    filters = normalize_filter_params(params)
+    tag_colors = tag_color_map(project_id)
+
+    year_months = media_year_month_counts(project_id)
+    avail_tags = media_available_tags(project_id)
+
+    filtered_query = media_filtered_query(project_id, filters)
+    total_count = Repo.one(from m in filtered_query, select: count(m.id))
+
+    limited_media =
+      Repo.all(
+        from m in filtered_query,
+          order_by: [desc: m.created_at],
+          limit: ^filters.display_limit,
+          select: %{
+            id: m.id,
+            title: m.title,
+            original_name: m.original_name,
+            mime_type: m.mime_type,
+            size: m.size,
+            tags: m.tags,
+            alt: m.alt,
+            caption: m.caption,
+            updated_at: m.updated_at
+          }
+      )
+
+    build_media_view(limited_media, filters, tag_colors, year_months, avail_tags, total_count)
+  end
+
+  defp build_media_view(limited_media, filters, tag_colors, year_month_counts, available_tags, total_count) do
+    loaded_count = length(limited_media)
 
     %{
       title: dgettext("ui", "Media"),
@@ -320,15 +579,15 @@ defmodule BDS.UI.Sidebar do
         results_label: dgettext("ui", "results"),
         results_for_label: dgettext("ui", "results for"),
         no_results_label: dgettext("ui", "No media files"),
-        year_month_counts: year_month_counts(base_media, &Map.get(&1, :updated_at)),
+        year_month_counts: year_month_counts,
         available_tags: available_tags,
         available_tag_colors: Map.take(tag_colors, available_tags),
         available_categories: [],
         max_items: @default_page_size,
         display_limit: filters.display_limit,
-        loaded_count: length(limited_media),
+        loaded_count: loaded_count,
-        total_count: length(filtered_media),
+        total_count: total_count,
-        has_more: length(filtered_media) > filters.display_limit,
+        has_more: total_count > filters.display_limit,
         has_active_filters: filter_active?(filters),
         selected: %{
           search: filters.search,
@@ -354,7 +613,127 @@ defmodule BDS.UI.Sidebar do
     }
   end
 
-  defp tags_nav_view(tags) do
+  defp media_filtered_query(project_id, filters) do
+    from(media in Media, where: media.project_id == ^project_id)
+    |> maybe_where_media_search(filters.search)
+    |> maybe_where_media_year(filters.year)
+    |> maybe_where_media_month(filters.month)
+    |> maybe_where_all_media_tags(filters.tags)
+  end
+
+  defp maybe_where_media_search(query, nil), do: query
+
+  defp maybe_where_media_search(query, search) do
+    search_term = "%" <> String.downcase(search) <> "%"
+
+    where(
+      query,
+      [m],
+      fragment(
+        """
+        lower(COALESCE(?,'') || ' ' || COALESCE(?,'') || ' ' || COALESCE(?,'')
+              || ' ' || COALESCE(?,'') || ' ' || COALESCE(?,'[]')) LIKE ?
+        """,
+        m.title,
+        m.original_name,
+        m.alt,
+        m.caption,
+        m.tags,
+        ^search_term
+      )
+    )
+  end
+
+  defp maybe_where_media_year(query, nil), do: query
+
+  defp maybe_where_media_year(query, year) do
+    year_str = to_string(year)
+
+    where(
+      query,
+      [m],
+      fragment(
+        "strftime('%Y', datetime(? / 1000, 'unixepoch')) = ?",
+        m.updated_at,
+        ^year_str
+      )
+    )
+  end
+
+  defp maybe_where_media_month(query, nil), do: query
+
+  defp maybe_where_media_month(query, month) do
+    month_str = String.pad_leading(to_string(month), 2, "0")
+
+    where(
+      query,
+      [m],
+      fragment(
+        "strftime('%m', datetime(? / 1000, 'unixepoch')) = ?",
+        m.updated_at,
+        ^month_str
+      )
+    )
+  end
+
+  defp maybe_where_all_media_tags(query, []), do: query
+
+  defp maybe_where_all_media_tags(query, tags) do
+    Enum.reduce(tags, query, fn tag, q ->
+      where(
+        q,
+        [m],
+        fragment(
+          "EXISTS (SELECT 1 FROM json_each(?) WHERE lower(value) = lower(?))",
+          m.tags,
+          ^tag
+        )
+      )
+    end)
+  end
+
+  defp media_year_month_counts(project_id) do
+    %{rows: rows} =
+      Ecto.Adapters.SQL.query!(
+        Repo,
+        """
+        SELECT CAST(strftime('%Y', datetime(updated_at / 1000, 'unixepoch')) AS INTEGER),
+               CAST(strftime('%m', datetime(updated_at / 1000, 'unixepoch')) AS INTEGER),
+               COUNT(*)
+        FROM media
+        WHERE project_id = ?1
+          AND updated_at IS NOT NULL
+        GROUP BY 1, 2
+        ORDER BY 1 DESC, 2 DESC
+        """,
+        [project_id]
+      )
+
+    Enum.map(rows, fn [year, month, count] -> %{year: year, month: month, count: count} end)
+  end
+
+  defp media_available_tags(project_id) do
+    %{rows: rows} =
+      Ecto.Adapters.SQL.query!(
+        Repo,
+        """
+        SELECT DISTINCT trim(je.value)
+        FROM media, json_each(media.tags) je
+        WHERE media.project_id = ?1
+          AND trim(je.value) != ''
+        ORDER BY lower(trim(je.value))
+        """,
+        [project_id]
+      )
+
+    Enum.map(rows, fn [tag] -> tag end)
+  end
+
+  # ---------------------------------------------------------------------------
+  # Navigation views
+  # ---------------------------------------------------------------------------
+
+  defp tags_nav_view(count) do
     %{
       title: dgettext("ui", "Tags"),
       subtitle: dgettext("ui", "Tag management"),
@@ -364,7 +743,7 @@ defmodule BDS.UI.Sidebar do
         %{id: "tags-manage", title: dgettext("ui", "Create / Edit"), icon: "✏️", route: "tags"},
         %{id: "tags-merge", title: dgettext("ui", "Merge Tags"), icon: "🔀", route: "tags"}
       ],
-      summary_badge: length(tags)
+      summary_badge: count
     }
   end
 
@@ -434,6 +813,10 @@ defmodule BDS.UI.Sidebar do
     }
   end
 
+  # ---------------------------------------------------------------------------
+  # Post section builder
+  # ---------------------------------------------------------------------------
+
   defp build_post_section(title, status, posts, translation_counts, published_meta?) do
     %{
       id: Atom.to_string(status),
@@ -461,25 +844,9 @@ defmodule BDS.UI.Sidebar do
     }
   end
 
-  defp list_posts(project_id) do
+  # ---------------------------------------------------------------------------
-    Repo.all(
+  # Data queries
-      from post in Post,
+  # ---------------------------------------------------------------------------
-        where: post.project_id == ^project_id,
-        order_by: [desc: post.created_at],
-        select: %{
-          id: post.id,
-          title: post.title,
-          slug: post.slug,
-          excerpt: post.excerpt,
-          status: post.status,
-          tags: post.tags,
-          categories: post.categories,
-          updated_at: post.updated_at,
-          published_at: post.published_at,
-          language: post.language
-        }
-    )
-  end
 
   defp translation_counts(project_id) do
     Repo.all(
@@ -491,25 +858,6 @@ defmodule BDS.UI.Sidebar do
     |> Map.new()
   end
 
-  defp list_media(project_id) do
-    Repo.all(
-      from media in Media,
-        where: media.project_id == ^project_id,
-        order_by: [desc: media.created_at],
-        select: %{
-          id: media.id,
-          title: media.title,
-          original_name: media.original_name,
-          mime_type: media.mime_type,
-          size: media.size,
-          tags: media.tags,
-          alt: media.alt,
-          caption: media.caption,
-          updated_at: media.updated_at
-        }
-    )
-  end
-
   defp list_scripts(project_id) do
     Repo.all(
       from script in Script,
@@ -532,13 +880,8 @@ defmodule BDS.UI.Sidebar do
     ImportDefinitions.list_definitions(project_id)
   end
 
-  defp list_tags(project_id) do
+  defp tag_count(project_id) do
-    Repo.all(
+    Repo.one(from tag in Tag, where: tag.project_id == ^project_id, select: count(tag.id))
-      from tag in Tag,
-        where: tag.project_id == ^project_id,
-        order_by: [asc: tag.name],
-        select: %{id: tag.id, title: tag.name, updated_at: tag.updated_at}
-    )
   end
 
   defp list_conversations do
@@ -554,19 +897,28 @@ defmodule BDS.UI.Sidebar do
   end
 
   defp group_posts(posts) do
-    Enum.reduce(posts, %{draft: [], published: [], archived: []}, fn post, acc ->
+    grouped = Enum.group_by(posts, & &1.status)
-      case post.status do
+
-        :draft -> %{acc | draft: acc.draft ++ [post]}
+    %{
-        :published -> %{acc | published: acc.published ++ [post]}
+      draft: Map.get(grouped, :draft, []),
-        :archived -> %{acc | archived: acc.archived ++ [post]}
+      published: Map.get(grouped, :published, []),
-        _other -> acc
+      archived: Map.get(grouped, :archived, [])
+    }
+  end
+
+  defp tag_color_map(project_id) do
+    Repo.all(from tag in Tag, where: tag.project_id == ^project_id, select: {tag.name, tag.color})
+    |> Enum.reduce(%{}, fn {name, color}, acc ->
+      case String.trim(to_string(color || "")) do
+        "" -> acc
+        trimmed -> Map.put(acc, to_string(name), trimmed)
       end
     end)
   end
 
-  defp page_post?(post) do
+  # ---------------------------------------------------------------------------
-    Enum.any?(post.categories || [], &(String.downcase(to_string(&1)) == @page_category))
+  # Helpers
-  end
+  # ---------------------------------------------------------------------------
 
   defp normalize_view_id(view_id) when is_atom(view_id), do: Atom.to_string(view_id)
   defp normalize_view_id(view_id) when is_binary(view_id), do: view_id
@@ -605,104 +957,6 @@ defmodule BDS.UI.Sidebar do
       filters.categories != []
   end
 
-  defp apply_post_filters(posts, filters) do
-    Enum.filter(posts, fn post ->
-      matches_search?(post_search_blob(post), filters.search) and
-        matches_year_month?(post_filter_timestamp(post), filters.year, filters.month) and
-        matches_overlap?(post.tags, filters.tags) and
-        matches_overlap?(filtered_categories(post.categories), filters.categories)
-    end)
-  end
-
-  defp apply_media_filters(media_items, filters) do
-    Enum.filter(media_items, fn media ->
-      matches_search?(media_search_blob(media), filters.search) and
-        matches_year_month?(media.updated_at, filters.year, filters.month) and
-        matches_overlap?(media.tags, filters.tags)
-    end)
-  end
-
-  defp matches_search?(_text, nil), do: true
-
-  defp matches_search?(text, search) do
-    String.contains?(String.downcase(text), String.downcase(search))
-  end
-
-  defp matches_year_month?(_timestamp, nil, _month), do: true
-  defp matches_year_month?(nil, _year, _month), do: false
-
-  defp matches_year_month?(timestamp, year, month) do
-    datetime = DateTime.from_unix!(timestamp, :millisecond)
-
-    datetime.year == year and
-      (is_nil(month) or datetime.month == month)
-  end
-
-  defp matches_overlap?(_values, []), do: true
-
-  defp matches_overlap?(values, filters) do
-    normalized_values = MapSet.new(Enum.map(values || [], &normalize_term/1))
-
-    Enum.all?(filters, fn filter ->
-      MapSet.member?(normalized_values, normalize_term(filter))
-    end)
-  end
-
-  defp year_month_counts(items, timestamp_fun) do
-    items
-    |> Enum.reduce(%{}, fn item, acc ->
-      case timestamp_fun.(item) do
-        timestamp when is_integer(timestamp) ->
-          datetime = DateTime.from_unix!(timestamp, :millisecond)
-          Map.update(acc, {datetime.year, datetime.month}, 1, &(&1 + 1))
-
-        _other ->
-          acc
-      end
-    end)
-    |> Enum.map(fn {{year, month}, count} -> %{year: year, month: month, count: count} end)
-    |> Enum.sort_by(fn entry -> {-entry.year, -entry.month} end)
-  end
-
-  defp available_tags(items, getter) do
-    items
-    |> Enum.flat_map(fn item -> getter.(item) || [] end)
-    |> Enum.map(&to_string/1)
-    |> Enum.reject(&(&1 == ""))
-    |> Enum.uniq_by(&String.downcase/1)
-    |> Enum.sort_by(&String.downcase/1)
-  end
-
-  defp available_categories(posts, pages?) do
-    posts
-    |> Enum.flat_map(&filtered_categories(&1.categories || []))
-    |> then(fn categories ->
-      if pages?,
-        do: Enum.reject(categories, &(normalize_term(&1) == @page_category)),
-        else: categories
-    end)
-    |> Enum.map(&to_string/1)
-    |> Enum.reject(&(&1 == ""))
-    |> Enum.uniq_by(&String.downcase/1)
-    |> Enum.sort_by(&String.downcase/1)
-  end
-
-  defp tag_color_map(project_id) do
-    Repo.all(from tag in Tag, where: tag.project_id == ^project_id, select: {tag.name, tag.color})
-    |> Enum.reduce(%{}, fn {name, color}, acc ->
-      case String.trim(to_string(color || "")) do
-        "" -> acc
-        trimmed -> Map.put(acc, to_string(name), trimmed)
-      end
-    end)
-  end
-
-  defp filtered_categories(categories) do
-    Enum.reject(categories || [], &(normalize_term(&1) == @page_category))
-  end
-
-  defp post_filter_timestamp(post), do: post.published_at || post.updated_at
-
   defp post_search_blob(post) do
     [
       post.title,
@@ -748,8 +1002,6 @@ defmodule BDS.UI.Sidebar do
 
   defp normalize_string_list(_values), do: []
 
-  defp normalize_term(value), do: value |> to_string() |> String.downcase()
-
   defp display_post_title(post) do
     cond do
       present?(post.title) -> post.title