diff --git a/lib/bds/generation.ex b/lib/bds/generation.ex index 760294c..ed39edc 100644 --- a/lib/bds/generation.ex +++ b/lib/bds/generation.ex @@ -21,7 +21,7 @@ defmodule BDS.Generation do import BDS.Generation.Progress import BDS.Generation.Outputs import BDS.Generation.Data - import BDS.Generation.Renderers, only: [build_list_posts: 3] + import BDS.Generation.Renderers, only: [build_list_posts: 3, plan_render_target: 1] import BDS.Generation.Validation alias BDS.Generation.GeneratedFileHash @@ -31,6 +31,7 @@ defmodule BDS.Generation do alias BDS.PreviewAssets alias BDS.Posts.Post alias BDS.Projects + alias BDS.Rendering.RenderContext alias BDS.Repo @core_sections [:core, :single, :category, :tag, :date] @@ -108,6 +109,7 @@ defmodule BDS.Generation do on_progress = callback(opts) with {:ok, plan} <- plan_generation(project_id, [section]) do + plan = with_render_context(plan) render_data = validation_render_data(plan) route_total = section_route_total(plan, render_data) @@ -141,13 +143,17 @@ defmodule BDS.Generation do pagefind_outputs = BDS.Generation.Pagefind.build_outputs(plan, html_outputs) total = max(length(pagefind_outputs), 1) + write_ctx = stream_context(project_id, nil, 0, verb: "Built", gerund: "Building") + pagefind_outputs |> Enum.with_index(1) |> Enum.each(fn {{relative_path, content}, index} -> - _ = write_generated_file(project_id, relative_path, content) + :ok = write_output_tracked(write_ctx, relative_path, content, false) :ok = report_validation_progress(on_progress, index / total, "Built #{relative_path}") end) + flush_pending_hashes(write_ctx) + {:ok, %{written_count: length(pagefind_outputs)}} end end @@ -162,7 +168,7 @@ defmodule BDS.Generation do on_output ) ++ build_page_outputs( - plan.project_id, + plan_render_target(plan), plan.language, data.published_posts, render_data.translations_by_post_language, @@ -176,7 +182,7 @@ defmodule BDS.Generation do data = render_data.data build_single_outputs( - plan.project_id, + plan_render_target(plan), plan.language, data.published_posts, render_data.translations_by_post_language, @@ -238,6 +244,13 @@ defmodule BDS.Generation do main + additional end + # Attach a load-once render context to the plan so every page render works + # from preloaded in-memory data instead of re-querying per page. Idempotent: + # a plan that already carries a context keeps it. + defp with_render_context(plan) do + Map.put_new_lazy(plan, :render_context, fn -> RenderContext.build(plan.project_id) end) + end + defp report_outputs(outputs, nil), do: outputs defp report_outputs(outputs, on_output) do @@ -618,6 +631,7 @@ defmodule BDS.Generation do end defp build_outputs(plan, on_output \\ nil) do + plan = with_render_context(plan) data = generation_data(plan) published_translations = flattened_generation_translations(data.translations_by_post) translations_by_post_language = translation_lookup_map(published_translations) @@ -674,7 +688,7 @@ defmodule BDS.Generation do page_outputs = if :core in plan.sections do build_page_outputs( - plan.project_id, + plan_render_target(plan), plan.language, data.published_posts, translations_by_post_language, @@ -688,7 +702,7 @@ defmodule BDS.Generation do single_outputs = if :single in plan.sections do build_single_outputs( - plan.project_id, + plan_render_target(plan), plan.language, data.published_posts, translations_by_post_language, @@ -856,6 +870,7 @@ defmodule BDS.Generation do defp validation_apply_plan(project_id, sections, report) do with {:ok, plan} <- plan_generation(project_id, sections) do + plan = with_render_context(plan) published_posts = list_published_posts(project_id) targeted_plan = @@ -913,15 +928,22 @@ defmodule BDS.Generation do ctx = stream_context(project_id, on_progress, route_total, opts) Enum.each(outputs, fn output -> emit_output(output, ctx) end) + flush_pending_hashes(ctx) :counters.get(ctx.counter, 1) end defp stream_render(project_id, on_progress, route_total, opts, build_fn) do ctx = stream_context(project_id, on_progress, route_total, opts) _ = build_fn.(fn output -> emit_output(output, ctx) end) + flush_pending_hashes(ctx) :counters.get(ctx.counter, 1) end + # The stream context mirrors the old app's generation worker hash store: the + # hashes of every previously generated file are loaded once up front, writes + # compare against that in-memory map, and the changed hashes are accumulated + # in a (cross-process safe) ETS table that `flush_pending_hashes/1` persists + # in a few batched upserts — instead of 3 database roundtrips per file. defp stream_context(project_id, on_progress, route_total, opts) do gerund = Keyword.fetch!(opts, :gerund) @@ -930,6 +952,9 @@ defmodule BDS.Generation do %{ project_id: project_id, + project: Projects.get_project!(project_id), + existing_hashes: existing_hash_map(project_id), + pending_hashes: :ets.new(:bds_generation_pending_hashes, [:set, :public]), on_progress: on_progress, route_total: route_total, verb: Keyword.fetch!(opts, :verb), @@ -938,13 +963,19 @@ defmodule BDS.Generation do } end + defp existing_hash_map(project_id) do + Repo.all( + from generated_file in GeneratedFileHash, + where: generated_file.project_id == ^project_id, + select: {generated_file.relative_path, generated_file.content_hash} + ) + |> Map.new() + end + defp emit_output({relative_path, content} = output, ctx) do route? = route_html_path?(relative_path) - _ = - write_generated_file(ctx.project_id, relative_path, content, - refresh_timestamp_on_unchanged: ctx.refresh_routes? and route? - ) + :ok = write_output_tracked(ctx, relative_path, content, ctx.refresh_routes? and route?) if route? do :counters.add(ctx.counter, 1, 1) @@ -962,6 +993,62 @@ defmodule BDS.Generation do output end + defp write_output_tracked(ctx, relative_path, content, refresh_timestamp?) do + content_hash = sha256(content) + full_path = output_path(ctx.project, relative_path) + + case Map.get(ctx.existing_hashes, relative_path) do + ^content_hash -> + cond do + not File.exists?(full_path) -> + :ok = Persistence.atomic_write(full_path, content) + true = :ets.insert(ctx.pending_hashes, {relative_path, content_hash}) + :ok + + refresh_timestamp? -> + true = :ets.insert(ctx.pending_hashes, {relative_path, content_hash}) + :ok + + true -> + :ok + end + + _other -> + :ok = Persistence.atomic_write(full_path, content) + true = :ets.insert(ctx.pending_hashes, {relative_path, content_hash}) + :ok + end + end + + # SQLite allows at most 999 bound parameters per statement; 4 columns per row + # keeps 200-row chunks comfortably below that. + @hash_flush_chunk_size 200 + + defp flush_pending_hashes(ctx) do + rows = :ets.tab2list(ctx.pending_hashes) + :ets.delete(ctx.pending_hashes) + now = Persistence.now_ms() + + rows + |> Enum.map(fn {relative_path, content_hash} -> + %{ + project_id: ctx.project_id, + relative_path: relative_path, + content_hash: content_hash, + updated_at: now + } + end) + |> Enum.chunk_every(@hash_flush_chunk_size) + |> Enum.each(fn chunk -> + Repo.insert_all(GeneratedFileHash, chunk, + on_conflict: {:replace, [:content_hash, :updated_at]}, + conflict_target: [:project_id, :relative_path] + ) + end) + + :ok + end + defp url_progress_started_message(gerund, 0), do: "No URLs to #{String.downcase(gerund)}" defp url_progress_started_message(gerund, 1), do: "#{gerund} 1 URL..." defp url_progress_started_message(gerund, total), do: "#{gerund} #{total} URLs..." @@ -1021,7 +1108,7 @@ defmodule BDS.Generation do main_root = if targeted_plan.request_root_routes do - posts = build_list_posts(plan.base_url, data.published_list_posts, nil) + posts = build_list_posts(plan, data.published_list_posts, nil) build_root_outputs(plan, plan.language, posts) else [] @@ -1034,7 +1121,7 @@ defmodule BDS.Generation do if language_plan.request_root_routes do source = Map.get(render_data.localized_list_posts_by_language, language, []) prefix = route_language(plan.language, language) - posts = build_list_posts(plan.base_url, source, prefix) + posts = build_list_posts(plan, source, prefix) build_root_outputs(plan, language, posts) else [] @@ -1056,7 +1143,7 @@ defmodule BDS.Generation do page_outputs = build_page_outputs( - plan.project_id, + plan_render_target(plan), plan.language, main_page_posts, render_data.translations_by_post_language, @@ -1080,7 +1167,7 @@ defmodule BDS.Generation do end) build_single_outputs( - plan.project_id, + plan_render_target(plan), plan.language, main_posts, render_data.translations_by_post_language, diff --git a/lib/bds/generation/outputs.ex b/lib/bds/generation/outputs.ex index 25d1e8a..37e13ca 100644 --- a/lib/bds/generation/outputs.ex +++ b/lib/bds/generation/outputs.ex @@ -5,9 +5,11 @@ defmodule BDS.Generation.Outputs do import BDS.Generation.Renderers import BDS.Generation.Sitemap, only: [render_feed: 3, render_atom: 3, render_calendar: 1] + alias BDS.Rendering.RenderContext alias BDS.Rendering.TemplateSelection @type output_callback :: ({String.t(), iodata()} -> any()) | nil + @type render_target :: RenderContext.t() | String.t() @spec additional_languages(map()) :: [String.t()] def additional_languages(plan) do @@ -209,7 +211,7 @@ defmodule BDS.Generation.Outputs do {String.t(), iodata()} ] def build_category_outputs(plan, posts_by_category, languages, on_output \\ nil) do - Enum.flat_map(posts_by_category, fn {category, posts} -> + concurrent_flat_map(posts_by_category, fn {category, posts} -> paginated_posts = Enum.chunk_every(posts, max(plan.max_posts_per_page, 1)) category_slug = archive_route_segment(category) total_pages = length(paginated_posts) @@ -267,7 +269,7 @@ defmodule BDS.Generation.Outputs do {String.t(), iodata()} ] def build_tag_outputs(plan, posts_by_tag, languages, on_output \\ nil) do - Enum.flat_map(posts_by_tag, fn {tag, posts} -> + concurrent_flat_map(posts_by_tag, fn {tag, posts} -> tag_slug = archive_route_segment(tag) build_paginated_archive_outputs( @@ -288,7 +290,7 @@ defmodule BDS.Generation.Outputs do ] def build_date_outputs(plan, post_index, languages, on_output \\ nil) do year_outputs = - Enum.flat_map(post_index.posts_by_year, fn {year, posts} -> + concurrent_flat_map(post_index.posts_by_year, fn {year, posts} -> build_paginated_archive_outputs( plan, languages, @@ -309,7 +311,7 @@ defmodule BDS.Generation.Outputs do end) month_outputs = - Enum.flat_map(post_index.posts_by_year_month, fn {year_month, posts} -> + concurrent_flat_map(post_index.posts_by_year_month, fn {year_month, posts} -> [year, month] = String.split(year_month, "/", parts: 2) build_paginated_archive_outputs( @@ -332,7 +334,7 @@ defmodule BDS.Generation.Outputs do end) day_outputs = - Enum.flat_map(post_index.posts_by_year_month_day, fn {year_month_day, posts} -> + concurrent_flat_map(post_index.posts_by_year_month_day, fn {year_month_day, posts} -> [year, month, day] = String.split(year_month_day, "/", parts: 3) build_paginated_archive_outputs( @@ -366,7 +368,7 @@ defmodule BDS.Generation.Outputs do def build_core_outputs(plan, published_posts, localized_posts_by_language, on_output \\ nil) do language = plan.language additional_languages = Enum.reject(plan.blog_languages, &(&1 == language)) - main_posts = build_list_posts(plan.base_url, published_posts, nil) + main_posts = build_list_posts(plan, published_posts, nil) main_static_outputs = [ @@ -384,7 +386,7 @@ defmodule BDS.Generation.Outputs do localized_source_posts = Map.get(localized_posts_by_language, localized_language, []) localized_posts = - build_list_posts(plan.base_url, localized_source_posts, localized_prefix) + build_list_posts(plan, localized_source_posts, localized_prefix) # `build_root_outputs` is called without `on_output` here because the # combined list (roots + static files) is reported exactly once by the @@ -404,11 +406,12 @@ defmodule BDS.Generation.Outputs do end) end - @spec build_page_outputs(String.t(), String.t(), [map()], map(), map(), output_callback()) :: [ - {String.t(), iodata()} - ] + @spec build_page_outputs(render_target(), String.t(), [map()], map(), map(), output_callback()) :: + [ + {String.t(), iodata()} + ] def build_page_outputs( - project_id, + render_target, main_language, published_posts, translations_by_post_language, @@ -418,17 +421,17 @@ defmodule BDS.Generation.Outputs do page_outputs = published_posts |> Enum.filter(&("page" in (&1.categories || []))) - |> Enum.map(fn post -> + |> concurrent_map(fn post -> canonical_variant = Map.get(translations_by_post_language, {post.id, main_language}, post) render_language = effective_render_language(canonical_variant.language, main_language) - body = load_body(project_id, canonical_variant.file_path, canonical_variant.content) + body = load_body(render_target, canonical_variant.file_path, canonical_variant.content) - effective_slug = effective_template_slug(project_id, post) + effective_slug = effective_template_slug(render_target, post) output = {page_output_path(post.slug, nil), render_post_output( - project_id, + render_target, effective_slug, %{ id: canonical_variant.id, @@ -457,16 +460,16 @@ defmodule BDS.Generation.Outputs do |> Enum.flat_map(fn {language, posts} -> posts |> Enum.filter(&("page" in (&1.categories || []))) - |> Enum.map(fn post -> + |> concurrent_map(fn post -> render_language = effective_render_language(post.language, language) - body = load_body(project_id, post.file_path, post.content) + body = load_body(render_target, post.file_path, post.content) - effective_slug = effective_template_slug(project_id, post) + effective_slug = effective_template_slug(render_target, post) output = {page_output_path(post.slug, language), render_post_output( - project_id, + render_target, effective_slug, %{ id: post.id, @@ -497,7 +500,7 @@ defmodule BDS.Generation.Outputs do posts |> paginate_posts(plan.max_posts_per_page) |> Enum.with_index(1) - |> Enum.map(fn {page_posts, page_number} -> + |> concurrent_map(fn {page_posts, page_number} -> route_language = route_language(plan.language, language) output = @@ -570,12 +573,12 @@ defmodule BDS.Generation.Outputs do end) end - @spec build_single_outputs(String.t(), String.t(), [map()], map(), map(), output_callback()) :: + @spec build_single_outputs(render_target(), String.t(), [map()], map(), map(), output_callback()) :: [ {String.t(), iodata()} ] def build_single_outputs( - project_id, + render_target, main_language, published_posts, translations_by_post_language, @@ -583,17 +586,17 @@ defmodule BDS.Generation.Outputs do on_output \\ nil ) do post_outputs = - Enum.map(published_posts, fn post -> + concurrent_map(published_posts, fn post -> canonical_variant = Map.get(translations_by_post_language, {post.id, main_language}, post) render_language = effective_render_language(canonical_variant.language, main_language) - body = load_body(project_id, canonical_variant.file_path, canonical_variant.content) + body = load_body(render_target, canonical_variant.file_path, canonical_variant.content) - effective_slug = effective_template_slug(project_id, post) + effective_slug = effective_template_slug(render_target, post) output = {post_output_path(post), render_post_output( - project_id, + render_target, effective_slug, %{ id: canonical_variant.id, @@ -620,16 +623,16 @@ defmodule BDS.Generation.Outputs do translation_outputs = localized_posts_by_language |> Enum.flat_map(fn {language, posts} -> - Enum.map(posts, fn post -> + concurrent_map(posts, fn post -> render_language = effective_render_language(post.language, language) - body = load_body(project_id, post.file_path, post.content) + body = load_body(render_target, post.file_path, post.content) - effective_slug = effective_template_slug(project_id, post) + effective_slug = effective_template_slug(render_target, post) output = {post_output_path(post, language), render_post_output( - project_id, + render_target, effective_slug, %{ id: post.id, @@ -650,14 +653,14 @@ defmodule BDS.Generation.Outputs do post_outputs ++ translation_outputs end - defp effective_template_slug(project_id, post) do + defp effective_template_slug(render_target, post) do slug = Map.get(post, :template_slug) if is_binary(slug) and slug != "" do slug else TemplateSelection.resolve_post_template_slug( - project_id, + render_target, Map.get(post, :tags) || [], Map.get(post, :categories) || [] ) @@ -682,4 +685,23 @@ defmodule BDS.Generation.Outputs do on_output.(output) output end + + # Render pages on all cores, like the old app's worker threads — safe because + # the render context is immutable data and the output callback only touches + # cross-process-safe state (file writes, ETS, counters). Order is preserved. + defp concurrent_map(enum, fun) do + enum + |> Task.async_stream(fun, + timeout: :infinity, + max_concurrency: System.schedulers_online(), + ordered: true + ) + |> Enum.map(fn {:ok, value} -> value end) + end + + defp concurrent_flat_map(enum, fun) do + enum + |> concurrent_map(fun) + |> Enum.concat() + end end diff --git a/lib/bds/generation/renderers.ex b/lib/bds/generation/renderers.ex index 1b2df68..5af14c7 100644 --- a/lib/bds/generation/renderers.ex +++ b/lib/bds/generation/renderers.ex @@ -4,6 +4,7 @@ defmodule BDS.Generation.Renderers do alias BDS.Generation.Paths alias BDS.Projects alias BDS.Rendering + alias BDS.Rendering.RenderContext @doc "Render the home page (HTML) using the project's template engine." @spec render_home(map(), String.t() | nil) :: String.t() @@ -80,7 +81,7 @@ defmodule BDS.Generation.Renderers do plan, language, label, - build_list_posts(plan.base_url, posts, Paths.route_language(plan.language, language)), + build_list_posts(plan, posts, Paths.route_language(plan.language, language)), archive_context, pagination, fallback @@ -88,9 +89,13 @@ defmodule BDS.Generation.Renderers do end @doc "Try the project's post template; on error, fall back to the inline `fallback` thunk." - @spec render_post_output(String.t(), String.t() | nil, map(), (-> String.t())) :: String.t() - def render_post_output(project_id, template_slug, assigns, fallback) do - render_with_fallback(fn -> Rendering.render_post_page(project_id, template_slug, assigns) end, fallback) + @spec render_post_output(RenderContext.t() | String.t(), String.t() | nil, map(), (-> String.t())) :: + String.t() + def render_post_output(render_target, template_slug, assigns, fallback) do + render_with_fallback( + fn -> Rendering.render_post_page(render_target, template_slug, assigns) end, + fallback + ) end @doc "Render a list/archive page through the project template, falling back to inline." @@ -105,7 +110,7 @@ defmodule BDS.Generation.Renderers do ) :: String.t() def render_list_output( - %{project_id: project_id, language: main_language}, + %{project_id: project_id, language: main_language} = plan, language, page_title, posts, @@ -116,7 +121,7 @@ defmodule BDS.Generation.Renderers do when is_binary(project_id) do render_with_fallback( fn -> - Rendering.render_list_page(project_id, %{ + Rendering.render_list_page(plan_render_target(plan), %{ language: language, language_prefix: Paths.language_prefix(language, main_language), page_title: page_title, @@ -131,11 +136,14 @@ defmodule BDS.Generation.Renderers do @doc "Render the project's 404 page via its template, falling back to a static page." @spec render_not_found_output(map(), String.t() | nil) :: String.t() - def render_not_found_output(%{project_id: project_id, language: main_language}, language) + def render_not_found_output( + %{project_id: project_id, language: main_language} = plan, + language + ) when is_binary(project_id) do render_with_fallback( fn -> - Rendering.render_not_found_page(project_id, %{ + Rendering.render_not_found_page(plan_render_target(plan), %{ language: language, language_prefix: Paths.language_prefix(language, main_language) }) @@ -144,6 +152,15 @@ defmodule BDS.Generation.Renderers do ) end + @doc "The render context stashed in the plan, or the bare project id when absent." + @spec plan_render_target(map()) :: RenderContext.t() | String.t() + def plan_render_target(plan) do + case Map.get(plan, :render_context) do + %RenderContext{} = ctx -> ctx + _other -> plan.project_id + end + end + @doc "Static fallback HTML for a 404 page." @spec render_not_found_page(String.t() | nil) :: String.t() def render_not_found_page(language) do @@ -156,41 +173,49 @@ defmodule BDS.Generation.Renderers do end @doc "Build the list-of-posts payload (with hrefs and bodies) for archive/list templates." - @spec build_list_posts(String.t() | nil, [map()], String.t() | nil) :: [map()] - def build_list_posts(base_url, posts, language_prefix) do + @spec build_list_posts(map(), [map()], String.t() | nil) :: [map()] + def build_list_posts(plan, posts, language_prefix) do + render_target = plan_render_target(plan) + Enum.map(posts, fn post -> %{ id: post.id, slug: post.slug, title: post.title, - href: Paths.url_for_output(base_url, Paths.post_output_path(post, language_prefix)), + href: Paths.url_for_output(plan.base_url, Paths.post_output_path(post, language_prefix)), excerpt: post.excerpt, - content: load_body(post.project_id, post.file_path, post.content) + content: load_body(render_target, post.file_path, post.content) } end) end @doc "Load the post body from disk (or pass-through inline content) for list rendering." - @spec load_body(String.t() | nil, String.t() | nil, String.t() | nil) :: String.t() - def load_body(_project_id, _file_path, inline_content) when is_binary(inline_content), + @spec load_body(RenderContext.t() | String.t() | nil, String.t() | nil, String.t() | nil) :: + String.t() + def load_body(_render_target, _file_path, inline_content) when is_binary(inline_content), do: inline_content - def load_body(project_id, file_path, _inline_content) do - case file_path do - nil -> - "" + def load_body(%RenderContext{} = ctx, file_path, _inline_content) + when is_binary(file_path) and file_path != "" do + RenderContext.memoize(ctx, {:post_body, file_path}, fn -> + read_body_file(Path.expand(file_path, ctx.project_data_dir)) + end) + end - "" -> - "" + def load_body(project_id, file_path, _inline_content) + when is_binary(project_id) and is_binary(file_path) and file_path != "" do + project_id + |> Projects.get_project!() + |> Projects.project_data_dir() + |> then(&read_body_file(Path.expand(file_path, &1))) + end - value -> - project_path = - Path.expand(value, Projects.project_data_dir(Projects.get_project!(project_id))) + def load_body(_render_target, _file_path, _inline_content), do: "" - case File.read(project_path) do - {:ok, contents} -> parse_frontmatter_body(contents) - {:error, _reason} -> "" - end + defp read_body_file(full_path) do + case File.read(full_path) do + {:ok, contents} -> parse_frontmatter_body(contents) + {:error, _reason} -> "" end end diff --git a/lib/bds/rendering.ex b/lib/bds/rendering.ex index 40464d1..acf74e0 100644 --- a/lib/bds/rendering.ex +++ b/lib/bds/rendering.ex @@ -3,43 +3,58 @@ defmodule BDS.Rendering do alias BDS.Rendering.ListArchive alias BDS.Rendering.PostRendering + alias BDS.Rendering.RenderContext alias BDS.Rendering.TemplateSelection def render_post_page(project_id, template_slug, assigns) when is_binary(project_id) and is_map(assigns) do + render_post_page(RenderContext.build(project_id), template_slug, assigns) + end + + def render_post_page(%RenderContext{} = ctx, template_slug, assigns) when is_map(assigns) do with {:ok, template_source} <- - TemplateSelection.load_template_source(project_id, :post, template_slug), + TemplateSelection.load_template_source(ctx, :post, template_slug), {:ok, rendered} <- TemplateSelection.render_template( - project_id, + ctx, template_source, - PostRendering.post_assigns(project_id, assigns) + PostRendering.post_assigns(ctx, assigns) ) do {:ok, rendered} end end def render_list_page(project_id, assigns) when is_binary(project_id) and is_map(assigns) do - with {:ok, template_source} <- TemplateSelection.load_template_source(project_id, :list, nil), + render_list_page(RenderContext.build(project_id), assigns) + end + + def render_list_page(%RenderContext{} = ctx, assigns) when is_map(assigns) do + with {:ok, template_source} <- TemplateSelection.load_template_source(ctx, :list, nil), {:ok, rendered} <- TemplateSelection.render_template( - project_id, + ctx, template_source, - ListArchive.list_assigns(project_id, assigns) + ListArchive.list_assigns(ctx, assigns) ) do {:ok, rendered} end end - def render_not_found_page(project_id, assigns \\ %{}) + def render_not_found_page(ctx_or_project_id, assigns \\ %{}) + + def render_not_found_page(project_id, assigns) when is_binary(project_id) and is_map(assigns) do + render_not_found_page(RenderContext.build(project_id), assigns) + end + + def render_not_found_page(%RenderContext{} = ctx, assigns) when is_map(assigns) do with {:ok, template_source} <- - TemplateSelection.load_template_source(project_id, :not_found, nil), + TemplateSelection.load_template_source(ctx, :not_found, nil), {:ok, rendered} <- TemplateSelection.render_template( - project_id, + ctx, template_source, - ListArchive.not_found_assigns(project_id, assigns) + ListArchive.not_found_assigns(ctx, assigns) ) do {:ok, rendered} end diff --git a/lib/bds/rendering/links_and_languages.ex b/lib/bds/rendering/links_and_languages.ex index daee62b..63d2df1 100644 --- a/lib/bds/rendering/links_and_languages.ex +++ b/lib/bds/rendering/links_and_languages.ex @@ -84,19 +84,21 @@ defmodule BDS.Rendering.LinksAndLanguages do end case Repo.get(Post, linked_post_id) do - nil -> - nil - - linked_post -> - %{ - href: post_path(linked_post, nil), - title: linked_post.title, - display_slug: linked_post.slug, - language: normalize_language(linked_post.language, main_language) - } + nil -> nil + linked_post -> link_context_for_post(linked_post, main_language) end end + @spec link_context_for_post(map(), String.t()) :: map() + def link_context_for_post(linked_post, main_language) do + %{ + href: post_path(linked_post, nil), + title: linked_post.title, + display_slug: linked_post.slug, + language: normalize_language(linked_post.language, main_language) + } + end + @spec language_prefix(String.t() | nil, String.t()) :: String.t() def language_prefix(language, main_language) when language == main_language, do: "" def language_prefix(nil, _main_language), do: "" diff --git a/lib/bds/rendering/list_archive.ex b/lib/bds/rendering/list_archive.ex index cc708c1..23f38aa 100644 --- a/lib/bds/rendering/list_archive.ex +++ b/lib/bds/rendering/list_archive.ex @@ -7,43 +7,25 @@ defmodule BDS.Rendering.ListArchive do alias BDS.Rendering.LinksAndLanguages alias BDS.Rendering.Metadata, as: RenderMetadata alias BDS.Rendering.PostRendering - alias BDS.Rendering.TemplateSelection + alias BDS.Rendering.RenderContext use Gettext, backend: BDS.Gettext - @spec list_assigns(String.t(), map()) :: map() - def list_assigns(project_id, assigns) do - metadata = RenderMetadata.project_metadata(project_id) - template_context = TemplateSelection.template_render_context(project_id) + @spec list_assigns(RenderContext.t() | String.t(), map()) :: map() + def list_assigns(project_id, assigns) when is_binary(project_id) do + list_assigns(RenderContext.build(project_id), assigns) + end + + def list_assigns(%RenderContext{} = ctx, assigns) do + metadata = ctx.metadata language = MapUtils.attr(assigns, :language, metadata.main_language || "en") main_language = metadata.main_language || language archive_context = MapUtils.attr(assigns, :archive_context, %{}) - canonical_post_paths = - LinksAndLanguages.canonical_post_path_by_slug(project_id, main_language) - - canonical_media_paths = LinksAndLanguages.canonical_media_path_by_source_path(project_id) - input_posts = MapUtils.attr(assigns, :posts, []) - post_ids = - input_posts - |> Enum.map(&MapUtils.attr(&1, :id)) - |> Enum.reject(&is_nil/1) - |> Enum.uniq() - - post_records = PostRendering.load_post_records_batch(post_ids) - - posts = - normalize_list_posts( - input_posts, - post_records, - canonical_post_paths, - canonical_media_paths, - language, - template_context - ) + posts = normalize_list_posts(ctx, input_posts, language) pagination = normalize_pagination(MapUtils.attr(assigns, :pagination), posts) @@ -73,7 +55,7 @@ defmodule BDS.Rendering.ListArchive do assign_override(assigns, :html_theme_attribute, "html_theme_attribute", nil), blog_languages: RenderMetadata.blog_languages(metadata, language), alternate_links: [], - menu_items: RenderMetadata.menu_items(project_id), + menu_items: ctx.menu_items, calendar_initial_year: calendar_initial_year_from_posts(posts), calendar_initial_month: calendar_initial_month_from_posts(posts), archive_context: normalized_archive_context, @@ -92,8 +74,8 @@ defmodule BDS.Rendering.ListArchive do total_pages: pagination.total_pages, total_items: pagination.total_items, items_per_page: pagination.items_per_page, - canonical_post_path_by_slug: canonical_post_paths, - canonical_media_path_by_source_path: canonical_media_paths, + canonical_post_path_by_slug: ctx.canonical_post_paths, + canonical_media_path_by_source_path: ctx.canonical_media_paths, post_data_json_by_id: Enum.into(posts, %{}, fn post -> {post.id, PostRendering.post_data_json_value(post)} end), day_blocks: day_blocks, @@ -103,9 +85,13 @@ defmodule BDS.Rendering.ListArchive do } end - @spec not_found_assigns(String.t(), map()) :: map() - def not_found_assigns(project_id, assigns) do - metadata = RenderMetadata.project_metadata(project_id) + @spec not_found_assigns(RenderContext.t() | String.t(), map()) :: map() + def not_found_assigns(project_id, assigns) when is_binary(project_id) do + not_found_assigns(RenderContext.build(project_id), assigns) + end + + def not_found_assigns(%RenderContext{} = ctx, assigns) do + metadata = ctx.metadata language = MapUtils.attr(assigns, :language, metadata.main_language || "en") @@ -129,7 +115,7 @@ defmodule BDS.Rendering.ListArchive do html_theme_attribute: assign_override(assigns, :html_theme_attribute, "html_theme_attribute", nil), blog_languages: RenderMetadata.blog_languages(metadata, language), - menu_items: RenderMetadata.menu_items(project_id), + menu_items: ctx.menu_items, alternate_links: [], not_found_message: assign_override( @@ -149,17 +135,10 @@ defmodule BDS.Rendering.ListArchive do } end - defp normalize_list_posts( - posts, - post_records, - canonical_post_paths, - canonical_media_paths, - language, - template_context - ) do + defp normalize_list_posts(ctx, posts, language) do Enum.map(posts, fn post -> post_id = MapUtils.attr(post, :id) - post_record = Map.get(post_records, post_id) + post_record = Map.get(ctx.record_by_id, post_id) raw_content = Map.get( @@ -172,14 +151,7 @@ defmodule BDS.Rendering.ListArchive do id: MapUtils.attr(post, :id), slug: MapUtils.attr(post, :slug), title: MapUtils.attr(post, :title), - content: - PostRendering.render_post_content( - raw_content, - canonical_post_paths, - canonical_media_paths, - language, - template_context - ), + content: PostRendering.cached_post_content(ctx, raw_content, language), raw_content: raw_content, excerpt: MapUtils.attr(post, :excerpt, record_value(post_record, :excerpt)), author: MapUtils.attr(post, :author, record_value(post_record, :author)), diff --git a/lib/bds/rendering/metadata.ex b/lib/bds/rendering/metadata.ex index 6374134..3625188 100644 --- a/lib/bds/rendering/metadata.ex +++ b/lib/bds/rendering/metadata.ex @@ -103,6 +103,13 @@ defmodule BDS.Rendering.Metadata do order_by: [asc: translation.language] ) + alternate_links_from(post, translations, main_language) + end + + @spec alternate_links_from(Post.t() | nil, [Translation.t()], String.t()) :: [map()] + def alternate_links_from(nil, _translations, _main_language), do: [] + + def alternate_links_from(%Post{} = post, translations, main_language) do [ %{ href: LinksAndLanguages.post_path(post, nil), diff --git a/lib/bds/rendering/post_rendering.ex b/lib/bds/rendering/post_rendering.ex index de37ce6..fde9fb2 100644 --- a/lib/bds/rendering/post_rendering.ex +++ b/lib/bds/rendering/post_rendering.ex @@ -7,49 +7,34 @@ defmodule BDS.Rendering.PostRendering do alias BDS.Rendering.Labels alias BDS.Rendering.LinksAndLanguages alias BDS.Rendering.Metadata, as: RenderMetadata - alias BDS.Rendering.TemplateSelection + alias BDS.Rendering.RenderContext alias BDS.MapUtils alias BDS.Posts.Post - alias BDS.Posts.PostMedia alias BDS.Posts.Translation - alias BDS.Media.Media, as: MediaRecord alias BDS.Repo - @spec post_assigns(String.t(), map()) :: map() - def post_assigns(project_id, assigns) do - metadata = RenderMetadata.project_metadata(project_id) - template_context = TemplateSelection.template_render_context(project_id) + @spec post_assigns(RenderContext.t() | String.t(), map()) :: map() + def post_assigns(project_id, assigns) when is_binary(project_id) do + post_assigns(RenderContext.build(project_id), assigns) + end + + def post_assigns(%RenderContext{} = ctx, assigns) do + metadata = ctx.metadata language = MapUtils.attr(assigns, :language) || metadata.main_language || "en" main_language = metadata.main_language || language - post_record = Map.get(assigns, :_post_record) || load_post_record(assigns) - canonical_post = canonical_post_record(post_record) + post_record = Map.get(assigns, :_post_record) || load_post_record(ctx, assigns) + canonical_post = canonical_post_record(ctx, post_record) post_id = canonical_post_id(post_record, assigns) post_categories = record_list(post_record, :categories) post_tags = record_list(post_record, :tags) - canonical_post_paths = - LinksAndLanguages.canonical_post_path_by_slug(project_id, main_language) - - canonical_media_paths = LinksAndLanguages.canonical_media_path_by_source_path(project_id) raw_content = MapUtils.attr(assigns, :content) + rendered_content = cached_post_content(ctx, raw_content, language, post_id) - rendered_content = - render_post_content( - raw_content, - canonical_post_paths, - canonical_media_paths, - language, - template_context, - post_id - ) - - incoming_links = - LinksAndLanguages.link_contexts(project_id, post_id, :incoming, main_language) - - outgoing_links = - LinksAndLanguages.link_contexts(project_id, post_id, :outgoing, main_language) + incoming_links = RenderContext.link_contexts(ctx, post_id, :incoming) + outgoing_links = RenderContext.link_contexts(ctx, post_id, :outgoing) post_assigns = assigns @@ -81,30 +66,59 @@ defmodule BDS.Rendering.PostRendering do html_theme_attribute: assign_override(assigns, :html_theme_attribute, "html_theme_attribute", nil), blog_languages: RenderMetadata.blog_languages(metadata, language), - alternate_links: RenderMetadata.alternate_links(canonical_post, project_id, main_language), - menu_items: RenderMetadata.menu_items(project_id), + alternate_links: + RenderMetadata.alternate_links_from( + canonical_post, + ctx_translations(ctx, canonical_post), + main_language + ), + menu_items: ctx.menu_items, calendar_initial_year: RenderMetadata.calendar_initial_year(post_record), calendar_initial_month: RenderMetadata.calendar_initial_month(post_record), post_categories: post_categories, post_tags: post_tags, - tag_color_by_name: RenderMetadata.tag_color_by_name(project_id), + tag_color_by_name: ctx.tag_color_by_name, backlinks: RenderMetadata.backlinks(incoming_links), - canonical_post_path_by_slug: canonical_post_paths, - canonical_media_path_by_source_path: canonical_media_paths, - post_data_json_by_id: post_data_json(post_assigns, post_record), - post: build_post_context(post_assigns, post_record, incoming_links, outgoing_links), + canonical_post_path_by_slug: ctx.canonical_post_paths, + canonical_media_path_by_source_path: ctx.canonical_media_paths, + post_data_json_by_id: + post_data_json(ctx, post_assigns, post_record, incoming_links, outgoing_links), + post: + build_post_context(ctx, post_assigns, post_record, incoming_links, outgoing_links), labels: Labels.for_language(language) } end - @spec load_post_record(map()) :: Post.t() | Translation.t() | nil - def load_post_record(assigns) do - case MapUtils.attr(assigns, :id) do - nil -> nil - post_id -> Repo.get(Post, post_id) || Repo.get(Translation, post_id) - end + @doc """ + Render post markdown through the context's canonical URL maps, memoizing by + content, language, and post id so the same content is only rendered once per + full-site render (posts appear on many list/archive pages). + """ + @spec cached_post_content(RenderContext.t(), term(), String.t(), term()) :: String.t() + def cached_post_content(%RenderContext{} = ctx, content, language, post_id \\ nil) do + raw = to_string(content || "") + + RenderContext.memoize( + ctx, + {:rendered_markdown, :erlang.md5(raw), language, post_id}, + fn -> + render_post_content( + raw, + ctx.canonical_post_paths, + ctx.canonical_media_paths, + language, + ctx.template_context, + post_id + ) + end + ) end + defp ctx_translations(_ctx, nil), do: [] + + defp ctx_translations(%RenderContext{} = ctx, %Post{id: post_id}), + do: Map.get(ctx.translations_by_post, post_id, []) + @spec load_post_records_batch([String.t()]) :: %{String.t() => Post.t() | Translation.t() | nil} def load_post_records_batch([]), do: %{} @@ -135,38 +149,35 @@ defmodule BDS.Rendering.PostRendering do Map.merge(posts, Map.merge(translations, nil_entries)) end - defp canonical_post_record(%Translation{translation_for: post_id}), do: Repo.get(Post, post_id) - defp canonical_post_record(%Post{} = post), do: post - defp canonical_post_record(_other), do: nil + defp load_post_record(%RenderContext{} = ctx, assigns) do + case MapUtils.attr(assigns, :id) do + nil -> nil + post_id -> Map.get(ctx.record_by_id, post_id) + end + end + + defp canonical_post_record(%RenderContext{} = ctx, %Translation{translation_for: post_id}) do + case Map.get(ctx.record_by_id, post_id) do + %Post{} = post -> post + _other -> nil + end + end + + defp canonical_post_record(_ctx, %Post{} = post), do: post + defp canonical_post_record(_ctx, _other), do: nil defp canonical_post_id(%Translation{translation_for: post_id}, _assigns), do: post_id defp canonical_post_id(%Post{id: post_id}, _assigns), do: post_id defp canonical_post_id(_post_record, assigns), do: MapUtils.attr(assigns, :id) - defp post_data_json(assigns, post_record) do + defp post_data_json(ctx, assigns, post_record, incoming_links, outgoing_links) do id = MapUtils.attr(assigns, :id) if is_binary(id) do - incoming_links = - LinksAndLanguages.link_contexts( - Map.get(post_record || %{}, :project_id), - canonical_post_id(post_record, assigns), - :incoming, - Map.get(post_record || %{}, :language) - ) - - outgoing_links = - LinksAndLanguages.link_contexts( - Map.get(post_record || %{}, :project_id), - canonical_post_id(post_record, assigns), - :outgoing, - Map.get(post_record || %{}, :language) - ) - %{ id => post_data_json_value( - build_post_context(assigns, post_record, incoming_links, outgoing_links) + build_post_context(ctx, assigns, post_record, incoming_links, outgoing_links) ) } else @@ -194,7 +205,7 @@ defmodule BDS.Rendering.PostRendering do end end - defp build_post_context(assigns, post_record, incoming_links, outgoing_links) do + defp build_post_context(ctx, assigns, post_record, incoming_links, outgoing_links) do %{ id: MapUtils.attr(assigns, :id), slug: MapUtils.attr(assigns, :slug), @@ -228,7 +239,7 @@ defmodule BDS.Rendering.PostRendering do MapUtils.attr(assigns, :template_slug) ), do_not_translate: record_value(post_record, :do_not_translate, false), - linked_media: linked_media_images(assigns), + linked_media: linked_media_images(ctx, assigns), outgoing_links: outgoing_links, incoming_links: incoming_links } @@ -260,19 +271,11 @@ defmodule BDS.Rendering.PostRendering do ) end - defp linked_media_images(assigns) do + defp linked_media_images(%RenderContext{} = ctx, assigns) do post_id = MapUtils.attr(assigns, :id) if is_binary(post_id) do - Repo.all( - from pm in PostMedia, - join: m in MediaRecord, - on: pm.media_id == m.id, - where: pm.post_id == ^post_id, - where: like(m.mime_type, "image/%"), - order_by: [asc: pm.sort_order, asc: pm.media_id], - select: m - ) + Map.get(ctx.linked_media_by_post, post_id, []) else [] end diff --git a/lib/bds/rendering/render_context.ex b/lib/bds/rendering/render_context.ex new file mode 100644 index 0000000..ad6f312 --- /dev/null +++ b/lib/bds/rendering/render_context.ex @@ -0,0 +1,201 @@ +defmodule BDS.Rendering.RenderContext do + @moduledoc """ + Load-once bundle of everything page rendering needs. + + Mirrors the old app's generation worker design: all project-wide data + (metadata, menu, canonical URL maps, link graphs, media, template roots) is + loaded up front in a handful of queries, and per-page rendering then works + from immutable in-memory data instead of re-querying the database and + re-reading config files for every page. An ETS-backed `memoize/3` cache holds + parsed template ASTs, post bodies, and rendered markdown for the lifetime of + the context. + + The ETS cache table is owned by the process that calls `build/1` and is + reclaimed automatically when that process exits, so a context must not + outlive its creator. + """ + + import Ecto.Query + + alias BDS.Media.Media, as: MediaRecord + alias BDS.Metadata, as: ProjectMetadata + alias BDS.Posts.Link + alias BDS.Posts.Post + alias BDS.Posts.PostMedia + alias BDS.Posts.Translation + alias BDS.Projects + alias BDS.Rendering.FileSystem + alias BDS.Rendering.Filters + alias BDS.Rendering.LinksAndLanguages + alias BDS.Rendering.Metadata, as: RenderMetadata + alias BDS.Repo + alias BDS.StarterTemplates + alias BDS.Tags.Tag + + defstruct [ + :project, + :project_id, + :project_data_dir, + :metadata, + :main_language, + :menu_items, + :tag_color_by_name, + :canonical_post_paths, + :canonical_media_paths, + :file_system, + :template_context, + :record_by_id, + :incoming_links_by_post, + :outgoing_links_by_post, + :translations_by_post, + :linked_media_by_post, + :tag_template_slug_by_name, + :cache + ] + + @type t :: %__MODULE__{} + + @spec build(String.t()) :: t() + def build(project_id) when is_binary(project_id) do + project = Projects.get_project!(project_id) + {:ok, metadata} = ProjectMetadata.get_project_metadata(project_id) + main_language = metadata.main_language || "en" + + posts = Repo.all(from post in Post, where: post.project_id == ^project_id) + post_by_id = Map.new(posts, &{&1.id, &1}) + + translations = + Repo.all(from translation in Translation, where: translation.project_id == ^project_id) + + {incoming_links_by_post, outgoing_links_by_post} = + build_link_context_maps(project_id, post_by_id, main_language) + + file_system = FileSystem.new(StarterTemplates.template_roots(project)) + + %__MODULE__{ + project: project, + project_id: project_id, + project_data_dir: Projects.project_data_dir(project), + metadata: metadata, + main_language: main_language, + menu_items: RenderMetadata.menu_items(project_id), + tag_color_by_name: RenderMetadata.tag_color_by_name(project_id), + canonical_post_paths: + LinksAndLanguages.canonical_post_path_by_slug(project_id, main_language), + canonical_media_paths: LinksAndLanguages.canonical_media_path_by_source_path(project_id), + file_system: file_system, + template_context: + Liquex.Context.new(%{}, + static_environment: %{}, + filter_module: Filters, + file_system: file_system + ), + record_by_id: Map.merge(Map.new(translations, &{&1.id, &1}), post_by_id), + incoming_links_by_post: incoming_links_by_post, + outgoing_links_by_post: outgoing_links_by_post, + translations_by_post: build_translations_by_post(translations), + linked_media_by_post: build_linked_media_by_post(project_id), + tag_template_slug_by_name: build_tag_template_slugs(project_id), + cache: :ets.new(:bds_render_context_cache, [:set, :public, {:read_concurrency, true}]) + } + end + + @doc """ + Return the cached value for `key`, computing and caching it with `fun` on the + first call. Safe to call from concurrently rendering processes; concurrent + first calls may compute twice but always return a consistent value. + """ + @spec memoize(t(), term(), (-> value)) :: value when value: term() + def memoize(%__MODULE__{cache: cache}, key, fun) when is_function(fun, 0) do + case :ets.lookup(cache, key) do + [{^key, value}] -> + value + + [] -> + value = fun.() + :ets.insert(cache, {key, value}) + value + end + end + + @spec link_contexts(t(), String.t() | nil, :incoming | :outgoing) :: [map()] + def link_contexts(_ctx, nil, _direction), do: [] + + def link_contexts(%__MODULE__{} = ctx, post_id, :incoming), + do: Map.get(ctx.incoming_links_by_post, post_id, []) + + def link_contexts(%__MODULE__{} = ctx, post_id, :outgoing), + do: Map.get(ctx.outgoing_links_by_post, post_id, []) + + ## --- internals ----------------------------------------------------------- + + defp build_link_context_maps(project_id, post_by_id, main_language) do + links = + Repo.all( + from link in Link, + join: source in Post, + on: link.source_post_id == source.id, + where: source.project_id == ^project_id, + order_by: [asc: link.created_at] + ) + + incoming = + links + |> Enum.group_by(& &1.target_post_id) + |> Map.new(fn {post_id, post_links} -> + {post_id, link_contexts_for(post_links, & &1.source_post_id, post_by_id, main_language)} + end) + + outgoing = + links + |> Enum.group_by(& &1.source_post_id) + |> Map.new(fn {post_id, post_links} -> + {post_id, link_contexts_for(post_links, & &1.target_post_id, post_by_id, main_language)} + end) + + {incoming, outgoing} + end + + defp link_contexts_for(links, linked_id_fun, post_by_id, main_language) do + links + |> Enum.map(fn link -> + case Map.get(post_by_id, linked_id_fun.(link)) do + nil -> nil + linked_post -> LinksAndLanguages.link_context_for_post(linked_post, main_language) + end + end) + |> Enum.reject(&is_nil/1) + end + + defp build_translations_by_post(translations) do + translations + |> Enum.filter(&(&1.status == :published)) + |> Enum.sort_by(& &1.language) + |> Enum.group_by(& &1.translation_for) + end + + defp build_linked_media_by_post(project_id) do + Repo.all( + from post_media in PostMedia, + join: media in MediaRecord, + on: post_media.media_id == media.id, + where: post_media.project_id == ^project_id, + where: like(media.mime_type, "image/%"), + order_by: [asc: post_media.post_id, asc: post_media.sort_order, asc: post_media.media_id], + select: {post_media.post_id, media} + ) + |> Enum.group_by(&elem(&1, 0), &elem(&1, 1)) + end + + defp build_tag_template_slugs(project_id) do + Repo.all( + from tag in Tag, + where: + tag.project_id == ^project_id and + not is_nil(tag.post_template_slug) and + tag.post_template_slug != "", + select: {tag.name, tag.post_template_slug} + ) + |> Map.new() + end +end diff --git a/lib/bds/rendering/template_selection.ex b/lib/bds/rendering/template_selection.ex index 559a9f8..056db04 100644 --- a/lib/bds/rendering/template_selection.ex +++ b/lib/bds/rendering/template_selection.ex @@ -8,13 +8,19 @@ defmodule BDS.Rendering.TemplateSelection do alias BDS.Projects alias BDS.Rendering.FileSystem alias BDS.Rendering.Filters + alias BDS.Rendering.RenderContext alias BDS.Repo alias BDS.StarterTemplates alias BDS.Tags.Tag alias BDS.Templates.Template - @spec resolve_post_template_slug(String.t(), [String.t()], [String.t()]) :: + @spec resolve_post_template_slug(RenderContext.t() | String.t(), [String.t()], [String.t()]) :: String.t() | nil + def resolve_post_template_slug(%RenderContext{} = ctx, tag_names, category_names) do + Enum.find_value(tag_names, &Map.get(ctx.tag_template_slug_by_name, &1)) || + resolve_from_category_settings(ctx.metadata.category_settings || %{}, category_names) + end + def resolve_post_template_slug(project_id, tag_names, category_names) do resolve_from_tags(project_id, tag_names) || resolve_from_categories(project_id, category_names) @@ -40,8 +46,12 @@ defmodule BDS.Rendering.TemplateSelection do defp resolve_from_categories(project_id, category_names) do {:ok, state} = Metadata.get_project_metadata(project_id) - settings = state.category_settings || %{} + resolve_from_category_settings(state.category_settings || %{}, category_names) + end + defp resolve_from_category_settings(_settings, []), do: nil + + defp resolve_from_category_settings(settings, category_names) do Enum.find_value(category_names, fn cat_name -> case Map.get(settings, cat_name) do %{"post_template_slug" => slug} when is_binary(slug) and slug != "" -> slug @@ -50,14 +60,22 @@ defmodule BDS.Rendering.TemplateSelection do end) end - @spec load_template_source(String.t(), atom(), String.t() | nil) :: + @spec load_template_source(RenderContext.t() | String.t(), atom(), String.t() | nil) :: {:ok, String.t()} | {:error, term()} - def load_template_source(project_id, kind, slug) do - project = Projects.get_project!(project_id) + def load_template_source(%RenderContext{} = ctx, kind, slug) do + RenderContext.memoize(ctx, {:template_source, kind, slug}, fn -> + do_load_template_source(ctx.project, ctx.project_id, kind, slug) + end) + end + def load_template_source(project_id, kind, slug) when is_binary(project_id) do + do_load_template_source(Projects.get_project!(project_id), project_id, kind, slug) + end + + defp do_load_template_source(project, project_id, kind, slug) do case select_template(project_id, kind, slug) do %Template{} = template -> - case published_template_body(template) do + case published_template_body(project, template) do {:ok, _source} = ok -> ok @@ -105,11 +123,10 @@ defmodule BDS.Rendering.TemplateSelection do limit: 1 end - defp published_template_body(%Template{content: content}) when is_binary(content), + defp published_template_body(_project, %Template{content: content}) when is_binary(content), do: {:ok, content} - defp published_template_body(%Template{} = template) do - project = Projects.get_project!(template.project_id) + defp published_template_body(project, %Template{} = template) do full_path = Path.join(Projects.project_data_dir(project), template.file_path) case File.read(full_path) do @@ -124,11 +141,16 @@ defmodule BDS.Rendering.TemplateSelection do end end - @spec render_template(String.t(), String.t(), map()) :: + @spec render_template(RenderContext.t() | String.t(), String.t(), map()) :: {:ok, String.t()} | {:error, String.t()} - def render_template(project_id, source, assigns) do - with {:ok, template_ast} <- Liquex.parse(source, BDS.Rendering.LiquidParser), - {:ok, _rendered} = ok <- safe_liquex_render(template_ast, project_id, assigns) do + def render_template(%RenderContext{} = ctx, source, assigns) do + parse_result = + RenderContext.memoize(ctx, {:template_ast, :erlang.md5(source)}, fn -> + Liquex.parse(source, BDS.Rendering.LiquidParser) + end) + + with {:ok, template_ast} <- parse_result, + {:ok, _rendered} = ok <- safe_liquex_render(template_ast, ctx.file_system, assigns) do ok else {:error, reason, line} when is_integer(line) -> {:error, "#{reason} at line #{line}"} @@ -136,14 +158,25 @@ defmodule BDS.Rendering.TemplateSelection do end end - defp safe_liquex_render(template_ast, project_id, assigns) do + def render_template(project_id, source, assigns) when is_binary(project_id) do project = Projects.get_project!(project_id) + file_system = FileSystem.new(StarterTemplates.template_roots(project)) + with {:ok, template_ast} <- Liquex.parse(source, BDS.Rendering.LiquidParser), + {:ok, _rendered} = ok <- safe_liquex_render(template_ast, file_system, assigns) do + ok + else + {:error, reason, line} when is_integer(line) -> {:error, "#{reason} at line #{line}"} + {:error, _message} = error -> error + end + end + + defp safe_liquex_render(template_ast, file_system, assigns) do context = Liquex.Context.new(assigns, static_environment: assigns, filter_module: Filters, - file_system: FileSystem.new(StarterTemplates.template_roots(project)) + file_system: file_system ) {result, _context} = Liquex.render!(template_ast, context) @@ -183,7 +216,10 @@ defmodule BDS.Rendering.TemplateSelection do defp bundled_template_slug(_kind, slug) when is_binary(slug) and slug != "", do: slug defp bundled_template_slug(kind, _slug), do: StarterTemplates.default_slug(kind) - @spec template_render_context(String.t()) :: Liquex.Context.t() + @spec template_render_context(RenderContext.t() | String.t()) :: Liquex.Context.t() + def template_render_context(%RenderContext{template_context: template_context}), + do: template_context + def template_render_context(project_id) do project = Projects.get_project!(project_id) diff --git a/test/bds/rendering/render_context_test.exs b/test/bds/rendering/render_context_test.exs new file mode 100644 index 0000000..a2c4942 --- /dev/null +++ b/test/bds/rendering/render_context_test.exs @@ -0,0 +1,292 @@ +defmodule BDS.Rendering.RenderContextTest do + use ExUnit.Case, async: false + + alias BDS.Generation + alias BDS.Rendering + alias BDS.Rendering.RenderContext + + setup do + :ok = Ecto.Adapters.SQL.Sandbox.checkout(BDS.Repo) + + temp_dir = + Path.join(System.tmp_dir!(), "bds-render-context-#{System.unique_integer([:positive])}") + + File.mkdir_p!(temp_dir) + on_exit(fn -> File.rm_rf(temp_dir) end) + + {:ok, project} = BDS.Projects.create_project(%{name: "RenderContext", data_path: temp_dir}) + + {:ok, _metadata} = + BDS.Metadata.update_project_metadata(project.id, %{ + main_language: "en", + blog_languages: ["en", "de"] + }) + + %{project: project, temp_dir: temp_dir} + end + + defp create_published_post(project, attrs) do + {:ok, post} = + BDS.Posts.create_post( + Map.merge( + %{project_id: project.id, language: "en", content: "Body"}, + attrs + ) + ) + + {:ok, published} = BDS.Posts.publish_post(post.id) + published + end + + test "build/1 preloads canonical paths, menu, tag colors, and link maps", %{project: project} do + target = create_published_post(project, %{title: "Ctx Target", tags: ["alpha"]}) + + _source = + create_published_post(project, %{ + title: "Ctx Source", + content: "See [Target](/#{Enum.join(post_date_parts(target), "/")}/#{target.slug}/)" + }) + + ctx = RenderContext.build(project.id) + + assert ctx.project.id == project.id + assert ctx.main_language == "en" + assert Map.has_key?(ctx.canonical_post_paths, target.slug) + assert is_map(ctx.canonical_media_paths) + assert is_map(ctx.tag_color_by_name) + assert is_list(ctx.menu_items) + + incoming = Map.get(ctx.incoming_links_by_post, target.id, []) + assert Enum.any?(incoming, fn link -> link.title == "Ctx Source" end) + end + + test "memoize/3 executes the computation only once per key", %{project: project} do + ctx = RenderContext.build(project.id) + {:ok, counter} = Agent.start_link(fn -> 0 end) + + compute = fn -> + Agent.update(counter, &(&1 + 1)) + :computed + end + + assert RenderContext.memoize(ctx, {:test, "key"}, compute) == :computed + assert RenderContext.memoize(ctx, {:test, "key"}, compute) == :computed + assert Agent.get(counter, & &1) == 1 + end + + test "render_post_page with a context matches the project_id render", %{project: project} do + {:ok, template} = + BDS.Templates.create_template(%{ + project_id: project.id, + title: "Ctx Parity", + kind: :post, + content: + "{{ page_title }}|{{ post.author }}|{{ post.tags.size }}|{% for lang in blog_languages %}[{{ lang.code }}]{% endfor %}|{{ content }}" + }) + + {:ok, published_template} = BDS.Templates.publish_template(template.id) + + post = + create_published_post(project, %{ + title: "Parity Post", + author: "Writer", + tags: ["alpha", "beta"], + content: "Hello **world**", + template_slug: published_template.slug + }) + + assigns = %{ + id: post.id, + title: post.title, + content: "Hello **world**", + slug: post.slug, + language: "en", + excerpt: post.excerpt, + template_slug: post.template_slug + } + + assert {:ok, from_project_id} = + Rendering.render_post_page(project.id, published_template.slug, assigns) + + ctx = RenderContext.build(project.id) + + assert {:ok, from_ctx} = + Rendering.render_post_page(ctx, published_template.slug, assigns) + + assert from_ctx == from_project_id + end + + test "render_list_page with a context matches the project_id render", %{project: project} do + post = create_published_post(project, %{title: "List Parity", content: "List body"}) + + assigns = %{ + language: "en", + page_title: "Home", + posts: [ + %{ + id: post.id, + slug: post.slug, + title: post.title, + href: "/#{post.slug}/", + excerpt: post.excerpt, + content: "List body" + } + ], + archive_context: %{kind: "core"}, + pagination: %{ + current_page: 1, + total_pages: 1, + total_items: 1, + items_per_page: 10, + has_prev_page: false, + prev_page_href: "", + has_next_page: false, + next_page_href: "" + } + } + + assert {:ok, from_project_id} = Rendering.render_list_page(project.id, assigns) + + ctx = RenderContext.build(project.id) + assert {:ok, from_ctx} = Rendering.render_list_page(ctx, assigns) + + assert from_ctx == from_project_id + end + + test "render_template caches the parsed template AST in the context", %{project: project} do + ctx = RenderContext.build(project.id) + source = "Hello {{ name }}" + + assert {:ok, "Hello A"} = + BDS.Rendering.TemplateSelection.render_template(ctx, source, %{"name" => "A"}) + + assert {:ok, "Hello B"} = + BDS.Rendering.TemplateSelection.render_template(ctx, source, %{"name" => "B"}) + + ast_entries = + :ets.tab2list(ctx.cache) + |> Enum.filter(fn {key, _value} -> match?({:template_ast, _hash}, key) end) + + assert length(ast_entries) == 1 + end + + test "load_body caches post bodies for the lifetime of the context", %{ + project: project, + temp_dir: temp_dir + } do + relative_path = "posts/cached-body.md" + full_path = Path.join(temp_dir, relative_path) + File.mkdir_p!(Path.dirname(full_path)) + File.write!(full_path, "---\ntitle: Cached\n---\nCached body content\n") + + ctx = RenderContext.build(project.id) + + assert BDS.Generation.Renderers.load_body(ctx, relative_path, nil) == + "Cached body content" + + File.rm!(full_path) + + assert BDS.Generation.Renderers.load_body(ctx, relative_path, nil) == + "Cached body content" + end + + test "generate_site batches hash bookkeeping and skips unchanged outputs", %{ + project: project, + temp_dir: temp_dir + } do + _post = create_published_post(project, %{title: "Site Post", content: "Site body"}) + + assert {:ok, %{generated_files: files}} = + Generation.generate_site(project.id, [:core, :single]) + + assert files != [] + + Enum.each(files, fn file -> + disk_path = Path.join([temp_dir, "html", file.relative_path]) + assert File.exists?(disk_path) + + disk_hash = + :crypto.hash(:sha256, File.read!(disk_path)) |> Base.encode16(case: :lower) + + assert disk_hash == file.content_hash + end) + + first_by_path = Map.new(files, &{&1.relative_path, &1}) + + Process.sleep(5) + + assert {:ok, %{generated_files: second_files}} = + Generation.generate_site(project.id, [:core, :single]) + + Enum.each(second_files, fn file -> + first = Map.fetch!(first_by_path, file.relative_path) + assert file.content_hash == first.content_hash + + assert file.updated_at == first.updated_at, + "expected unchanged output #{file.relative_path} to keep its updated_at" + end) + end + + test "full section render issues a bounded number of queries regardless of post count", %{ + project: project + } do + for index <- 1..12 do + create_published_post(project, %{ + title: "Query Bound #{index}", + content: "Body #{index}", + tags: ["tag-#{rem(index, 3)}"], + categories: ["article"] + }) + end + + query_count = + count_queries(fn -> + assert {:ok, %{rendered_url_count: rendered}} = + Generation.render_site_section(project.id, :single) + + assert rendered >= 12 + end) + + # Before the render-context rework this was ~15 queries *per page*; the + # bound only holds if all per-page lookups stay preloaded. + assert query_count <= 40, + "expected a bounded query count for the section render, got #{query_count}" + end + + defp count_queries(func) do + test_pid = self() + ref = make_ref() + handler_id = "render-context-query-counter-#{inspect(ref)}" + + :telemetry.attach( + handler_id, + [:bds, :repo, :query], + fn _event, _measurements, _metadata, _config -> + send(test_pid, {:query_executed, ref}) + end, + nil + ) + + func.() + :telemetry.detach(handler_id) + count_query_messages(ref, 0) + end + + defp count_query_messages(ref, acc) do + receive do + {:query_executed, ^ref} -> count_query_messages(ref, acc + 1) + after + 0 -> acc + end + end + + defp post_date_parts(post) do + datetime = BDS.Persistence.from_unix_ms!(post.created_at) + + [ + Integer.to_string(datetime.year), + String.pad_leading(Integer.to_string(datetime.month), 2, "0"), + String.pad_leading(Integer.to_string(datetime.day), 2, "0") + ] + end +end