hugo/bDS2
1
0
Fork 0
Code Issues 4 Pull Requests Actions Packages Projects Releases Wiki Activity

fix: better progress tracking for full site render

Browse Source
This commit is contained in:
Chili Palmer
2026-07-02 21:52:20 +02:00
parent 1487c0337e
commit 4b52ac6156
6 changed files with 544 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files

99
lib/bds/desktop/shell_commands.ex
View File

@@ -239,18 +239,33 @@ defmodule BDS.Desktop.ShellCommands do
end
defp dispatch("generate_sitemap", project, _params) do
queue_task(project, "generate_sitemap", "Generate Site", "Generation", fn report ->
{:ok, generation} =
Generation.generate_site(project.id, @site_sections, on_progress: report)
group_id = task_group_id("generate_sitemap")
attrs = %{group_id: group_id, group_name: "Render Site"}
report.(1.0, "Generated site output")
{:ok, first_task} =
Tasks.submit_task(
section_task_name(:core),
fn report ->
{:ok, _result} = Generation.render_site_section(project.id, :core, on_progress: report)
report.(1.0, section_complete_message(:core))
:ok
end,
attrs
)
%{
project_id: project.id,
sections: generation.sections,
generated_count: length(generation.generated_files)
}
end)
Task.start(fn -> run_render_site_sequence(project, group_id, attrs) end)
{:ok,
%{
kind: "task_queued",
action: "generate_sitemap",
title: "Render Site",
message: "Render Site tasks queued",
project_id: project.id,
task_id: first_task.id,
task_group_id: group_id,
panel_tab: "tasks"
}}
end
defp dispatch("validate_site", project, _params) do
@@ -475,6 +490,44 @@ defmodule BDS.Desktop.ShellCommands do
end
end
defp run_render_site_sequence(project, group_id, attrs) do
remaining_sections = [:single, :category, :tag, :date]
Enum.each(remaining_sections, fn section ->
{:ok, _task} =
Tasks.submit_task(
section_task_name(section),
fn report ->
{:ok, _result} =
Generation.render_site_section(project.id, section, on_progress: report)
report.(1.0, section_complete_message(section))
:ok
end,
attrs
)
end)
section_names = Enum.map([:core | remaining_sections], &section_task_name/1)
with :ok <- wait_for_group_phase(group_id, section_names, @rebuild_phase_timeout) do
{:ok, _task} =
Tasks.submit_task(
"Build Search Index",
fn report ->
{:ok, _result} = Generation.build_site_search_index(project.id, on_progress: report)
report.(1.0, "Build Search Index complete")
:ok
end,
attrs
)
wait_for_group_phase(group_id, ["Build Search Index"], @rebuild_phase_timeout)
end
:ok
end
defp run_apply_validation_sequence(project, validation_report, group_id, attrs, prepare_task_id) do
with :ok <-
wait_for_group_phase(group_id, ["Prepare Validation Apply"], @rebuild_phase_timeout),
@@ -499,10 +552,10 @@ defmodule BDS.Desktop.ShellCommands do
defp run_apply_validation_render_tasks(project, validation_report, group_id, attrs, preparation) do
sections = Map.get(preparation, :sections_to_render, [])
task_names = Enum.map(sections, &apply_validation_section_task_name/1)
task_names = Enum.map(sections, &section_task_name/1)
Enum.each(sections, fn section ->
task_name = apply_validation_section_task_name(section)
task_name = section_task_name(section)
{:ok, _task} =
Tasks.submit_task(
@@ -513,7 +566,7 @@ defmodule BDS.Desktop.ShellCommands do
on_progress: report
)
report.(1.0, apply_validation_section_complete_message(section))
report.(1.0, section_complete_message(section))
:ok
end,
attrs
@@ -574,20 +627,20 @@ defmodule BDS.Desktop.ShellCommands do
Map.get(preparation, :deleted_url_count, 0) > 0
end
defp apply_validation_section_task_name(:core), do: "Render Site Core"
defp apply_validation_section_task_name(:single), do: "Render Single Posts"
defp apply_validation_section_task_name(:category), do: "Render Category Archives"
defp apply_validation_section_task_name(:tag), do: "Render Tag Archives"
defp apply_validation_section_task_name(:date), do: "Render Date Archives"
defp section_task_name(:core), do: "Render Site Core"
defp section_task_name(:single), do: "Render Single Posts"
defp section_task_name(:category), do: "Render Category Archives"
defp section_task_name(:tag), do: "Render Tag Archives"
defp section_task_name(:date), do: "Render Date Archives"
defp apply_validation_section_complete_message(:core), do: "Render Site Core complete"
defp apply_validation_section_complete_message(:single), do: "Render Single Posts complete"
defp section_complete_message(:core), do: "Render Site Core complete"
defp section_complete_message(:single), do: "Render Single Posts complete"
defp apply_validation_section_complete_message(:category),
defp section_complete_message(:category),
do: "Render Category Archives complete"
defp apply_validation_section_complete_message(:tag), do: "Render Tag Archives complete"
defp apply_validation_section_complete_message(:date), do: "Render Date Archives complete"
defp section_complete_message(:tag), do: "Render Tag Archives complete"
defp section_complete_message(:date), do: "Render Date Archives complete"
defp apply_validation_refresh_task_name(:calendar), do: "Regenerate Calendar"

310
lib/bds/generation.ex
View File

@@ -79,22 +79,201 @@ defmodule BDS.Generation do
with {:ok, plan} <- plan_generation(project_id, sections) do
outputs = build_outputs(plan)
on_progress = callback(opts)
total_outputs = length(outputs)
:ok = report_generation_started(on_progress, total_outputs, "generated files")
outputs
|> Enum.with_index(1)
|> Enum.each(fn {{relative_path, content}, index} ->
{:ok, _write} = write_generated_file(project_id, relative_path, content)
:ok = report_generation_progress(on_progress, index, total_outputs, "generated files")
end)
_generated_url_count =
stream_write_outputs(project_id, outputs, on_progress,
verb: "Generated",
gerund: "Generating",
refresh_route_timestamps: false
)
{:ok, generated_files} = list_generated_files(project_id)
{:ok, %{sections: plan.sections, generated_files: generated_files}}
end
end
@doc """
Render one section of the full site, streaming one progress message per URL
(exactly like validation apply, only rendering ALL URLs of the section rather
than just the affected ones). The `:core` section also writes the complete
sitemap, feeds, calendar and preview assets. The Pagefind search index is
built separately by `build_site_search_index/2`.
"""
@spec render_site_section(String.t(), section(), generation_opts()) ::
{:ok, map()} | {:error, term()}
def render_site_section(project_id, section, opts \\ [])
def render_site_section(project_id, section, opts)
when is_binary(project_id) and section in @core_sections and is_list(opts) do
on_progress = callback(opts)
with {:ok, plan} <- plan_generation(project_id, [section]) do
render_data = validation_render_data(plan)
route_total = section_route_total(plan, render_data)
rendered_url_count =
stream_render(project_id, on_progress, route_total,
[verb: "Generated", gerund: "Generating", refresh_route_timestamps: false],
fn on_output -> full_section_outputs(plan, render_data, section, on_output) end
)
{:ok, %{section: section, rendered_url_count: rendered_url_count}}
end
end
@doc """
Build the Pagefind search index for the whole site from the HTML already on
disk (matching the old app's separate `Build Search Index` step that runs
after every section has been rendered).
"""
@spec build_site_search_index(String.t(), generation_opts()) ::
{:ok, map()} | {:error, term()}
def build_site_search_index(project_id, opts \\ [])
def build_site_search_index(project_id, opts)
when is_binary(project_id) and is_list(opts) do
on_progress = callback(opts)
:ok = report_validation_progress(on_progress, 0.0, "Building search index...")
with {:ok, plan} <- plan_generation(project_id, @core_sections) do
project = Projects.get_project!(project_id)
html_outputs = read_disk_html_outputs(output_path(project, ""))
pagefind_outputs = BDS.Generation.Pagefind.build_outputs(plan, html_outputs)
total = max(length(pagefind_outputs), 1)
pagefind_outputs
|> Enum.with_index(1)
|> Enum.each(fn {{relative_path, content}, index} ->
_ = write_generated_file(project_id, relative_path, content)
:ok = report_validation_progress(on_progress, index / total, "Built #{relative_path}")
end)
{:ok, %{written_count: length(pagefind_outputs)}}
end
end
defp full_section_outputs(plan, render_data, :core, on_output) do
data = render_data.data
build_core_outputs(
plan,
data.published_list_posts,
render_data.localized_list_posts_by_language,
on_output
) ++
build_page_outputs(
plan.project_id,
plan.language,
data.published_posts,
render_data.translations_by_post_language,
render_data.localized_posts_by_language,
on_output
) ++
report_outputs(core_sitemap_outputs(plan, data) ++ PreviewAssets.generated_outputs(), on_output)
end
defp full_section_outputs(plan, render_data, :single, on_output) do
data = render_data.data
build_single_outputs(
plan.project_id,
plan.language,
data.published_posts,
render_data.translations_by_post_language,
render_data.localized_posts_by_language,
on_output
)
end
defp full_section_outputs(plan, render_data, :category, on_output) do
full_localized_archive_outputs(plan, render_data, fn plan_arg, index, languages ->
build_category_outputs(plan_arg, index.posts_by_category, languages, on_output)
end)
end
defp full_section_outputs(plan, render_data, :tag, on_output) do
full_localized_archive_outputs(plan, render_data, fn plan_arg, index, languages ->
build_tag_outputs(plan_arg, index.posts_by_tag, languages, on_output)
end)
end
defp full_section_outputs(plan, render_data, :date, on_output) do
full_localized_archive_outputs(plan, render_data, fn plan_arg, index, languages ->
build_date_outputs(plan_arg, index, languages, on_output)
end)
end
# Total number of route (HTML) pages the section will render, computed from the
# post index without rendering any content, so streaming progress can report an
# accurate `n/total` from the very first page.
defp section_route_total(plan, render_data) do
data = render_data.data
main =
length(
build_validation_route_paths(
plan,
data.published_posts,
data.published_list_posts,
data.post_index,
nil
)
)
additional =
additional_languages(plan)
|> Enum.map(fn language ->
length(
build_validation_route_paths(
plan,
Map.get(render_data.localized_posts_by_language, language, []),
Map.get(render_data.localized_list_posts_by_language, language, []),
Map.get(render_data.localized_post_indexes, language, empty_generation_post_index()),
language
)
)
end)
|> Enum.sum()
main + additional
end
defp report_outputs(outputs, nil), do: outputs
defp report_outputs(outputs, on_output) do
Enum.each(outputs, on_output)
outputs
end
defp full_localized_archive_outputs(plan, render_data, builder) do
main_outputs = builder.(plan, render_data.data.post_index, [plan.language])
language_outputs =
Enum.flat_map(additional_languages(plan), fn language ->
index = Map.get(render_data.localized_post_indexes, language, empty_generation_post_index())
builder.(plan, index, [language])
end)
main_outputs ++ language_outputs
end
defp core_sitemap_outputs(plan, data) do
published_route_posts =
suppress_subtree_translation_variants(data.published_route_posts, additional_languages(plan))
{_sitemap_content, sitemap_to_write, _expected_paths, _timestamp_checks} =
build_validation_sitemap_artifacts(plan, data, published_route_posts, %{}, nil)
[{"sitemap.xml", sitemap_to_write}]
end
defp read_disk_html_outputs(html_root) do
Path.join(html_root, "**/*.html")
|> Path.wildcard()
|> Enum.map(fn full_path -> {Path.relative_to(full_path, html_root), File.read!(full_path)} end)
|> Enum.sort_by(&elem(&1, 0))
end
@spec validate_site(String.t(), [section()], generation_opts()) ::
{:ok, validation_report()} | {:error, term()}
def validate_site(project_id, sections \\ @core_sections, opts \\ [])
@@ -706,45 +885,86 @@ defmodule BDS.Generation do
end)
end
end
# Write the targeted outputs, streaming one progress message per rewritten URL
# (exactly like the old app's `Generated /url` reporting) so the task shows the
# actual URLs and advances by the number of URLs rewritten.
# Write the targeted validation outputs, streaming one progress message per
# rewritten URL so the task shows the actual URLs and advances by the number
# of URLs rewritten.
defp write_validation_outputs(project_id, outputs, on_progress) do
route_total = Enum.count(outputs, fn {relative_path, _content} -> route_html_path?(relative_path) end)
:ok = report_validation_progress(on_progress, 0.0, validation_rewrite_started_message(route_total))
{rendered, _total} =
Enum.reduce(outputs, {0, route_total}, fn {relative_path, content}, {rendered, total} ->
_ =
write_generated_file(project_id, relative_path, content,
refresh_timestamp_on_unchanged: route_html_path?(relative_path)
)
if route_html_path?(relative_path) do
rewritten = rendered + 1
progress = if total > 0, do: rewritten / total, else: 1.0
:ok =
report_validation_progress(
on_progress,
progress,
"Rewrote #{relative_path_to_url_path(relative_path)} (#{rewritten}/#{total})"
)
{rewritten, total}
else
{rendered, total}
end
end)
rendered
stream_write_outputs(project_id, outputs, on_progress,
verb: "Rewrote",
gerund: "Rewriting",
refresh_route_timestamps: true
)
end
defp validation_rewrite_started_message(0), do: "No URLs to rewrite"
defp validation_rewrite_started_message(1), do: "Rewriting 1 URL..."
defp validation_rewrite_started_message(total), do: "Rewriting #{total} URLs..."
# Shared per-URL streaming writer used by both full site generation and
# validation apply. Emits one `<verb> /url (n/total)` progress message per
# rendered route page (matching the old app's `Generated /url` reporting) and
# advances the task by the number of URLs written. Ancillary files
# (sitemap.xml, feeds, calendar, pagefind, assets) are written without a URL
# message. Returns the number of route pages written.
#
# `stream_write_outputs` drives a pre-built list of outputs (used by validation
# apply, whose targeted render is cheap); `stream_render` drives the render
# itself via the builders' `on_output` callback so a full-site render reports
# progress as each page is produced rather than only after every page is built.
defp stream_write_outputs(project_id, outputs, on_progress, opts) do
route_total =
Enum.count(outputs, fn {relative_path, _content} -> route_html_path?(relative_path) end)
ctx = stream_context(project_id, on_progress, route_total, opts)
Enum.each(outputs, fn output -> emit_output(output, ctx) end)
:counters.get(ctx.counter, 1)
end
defp stream_render(project_id, on_progress, route_total, opts, build_fn) do
ctx = stream_context(project_id, on_progress, route_total, opts)
_ = build_fn.(fn output -> emit_output(output, ctx) end)
:counters.get(ctx.counter, 1)
end
defp stream_context(project_id, on_progress, route_total, opts) do
gerund = Keyword.fetch!(opts, :gerund)
:ok =
report_validation_progress(on_progress, 0.0, url_progress_started_message(gerund, route_total))
%{
project_id: project_id,
on_progress: on_progress,
route_total: route_total,
verb: Keyword.fetch!(opts, :verb),
refresh_routes?: Keyword.get(opts, :refresh_route_timestamps, false),
counter: :counters.new(1, [])
}
end
defp emit_output({relative_path, content} = output, ctx) do
route? = route_html_path?(relative_path)
_ =
write_generated_file(ctx.project_id, relative_path, content,
refresh_timestamp_on_unchanged: ctx.refresh_routes? and route?
)
if route? do
:counters.add(ctx.counter, 1, 1)
count = :counters.get(ctx.counter, 1)
progress = if ctx.route_total > 0, do: min(count / ctx.route_total, 1.0), else: 1.0
:ok =
report_validation_progress(
ctx.on_progress,
progress,
"#{ctx.verb} #{relative_path_to_url_path(relative_path)} (#{count}/#{ctx.route_total})"
)
end
output
end
defp url_progress_started_message(gerund, 0), do: "No URLs to #{String.downcase(gerund)}"
defp url_progress_started_message(gerund, 1), do: "#{gerund} 1 URL..."
defp url_progress_started_message(gerund, total), do: "#{gerund} #{total} URLs..."
defp validation_render_data(plan) do
data = generation_data(plan)

7
lib/bds/generation/outputs.ex
View File

@@ -386,7 +386,12 @@ defmodule BDS.Generation.Outputs do
localized_posts =
build_list_posts(plan.base_url, localized_source_posts, localized_prefix)
(build_root_outputs(plan, localized_language, localized_posts, on_output) ++
# `build_root_outputs` is called without `on_output` here because the
# combined list (roots + static files) is reported exactly once by the
# `Enum.map(&report_output/2)` below. Passing `on_output` to both would
# report (and, for streaming callers, write and count) each localized
# root page twice.
(build_root_outputs(plan, localized_language, localized_posts) ++
[
{Path.join(localized_language, "404.html"),
render_not_found_output(plan, localized_language)},