Group site generation by section

This commit is contained in:
2026-07-19 10:27:12 +02:00
parent 055dd5efc5
commit 90a9002124
19 changed files with 1467 additions and 399 deletions

View File

@@ -11,7 +11,7 @@ The project is under active development. Core blogging workflows are broadly ava
- Media import, thumbnails, metadata translations, filters, validation, and post assignment. - Media import, thumbnails, metadata translations, filters, validation, and post assignment.
- Template and Lua script management using a custom Ropey/Syntect/Cosmic Text editor and the documented, bDS2-signature-compatible project-scoped [`bds` host API](docs/scripting/API_REFERENCE.md) across utilities, rendered macros, and Blogmark transforms. - Template and Lua script management using a custom Ropey/Syntect/Cosmic Text editor and the documented, bDS2-signature-compatible project-scoped [`bds` host API](docs/scripting/API_REFERENCE.md) across utilities, rendered macros, and Blogmark transforms.
- SQLite and filesystem persistence with frontmatter, sidecars, rebuild, metadata diff/repair, and FTS5 search. - SQLite and filesystem persistence with frontmatter, sidecars, rebuild, metadata diff/repair, and FTS5 search.
- Markdown/Liquid rendering with native macros, multilingual routes, feeds, sitemap, Pagefind, and incremental site generation. - Markdown/Liquid rendering with native macros, multilingual routes, feeds, sitemap, Pagefind, and incremental site generation through cancellable section task groups.
- Local preview in the app or system browser. - Local preview in the app or system browser.
- Optional one-shot AI translation, description, analysis, taxonomy, and language-detection operations using online or local OpenAI-compatible endpoints with airplane-mode gating. - Optional one-shot AI translation, description, analysis, taxonomy, and language-detection operations using online or local OpenAI-compatible endpoints with airplane-mode gating.
- SSH-agent-based SCP or rsync publishing. - SSH-agent-based SCP or rsync publishing.

View File

@@ -85,12 +85,11 @@ Available:
- Native gallery, YouTube, Vimeo, photo archive, and tag cloud macros. - Native gallery, YouTube, Vimeo, photo archive, and tag cloud macros.
- User-authored Lua macro invocation during rendering. - User-authored Lua macro invocation during rendering.
- Localhost-only Axum preview server, draft routes, embedded Wry preview, and external-browser preview. - Localhost-only Axum preview server, draft routes, embedded Wry preview, and external-browser preview.
- Complete site generation with pages, archives, feeds, sitemap, static assets, changed-file tracking, parallel page rendering, and Pagefind output. - Complete site generation with pages, archives, feeds, sitemap, static assets, changed-file tracking, parallel page rendering, and Pagefind output; full generation and validation apply run as grouped section tasks followed by search indexing.
- OPML/menu document loading and normalized Home-first menu output. - OPML/menu document loading and normalized Home-first menu output.
Open: Open:
- Represent generation as the specified group of section tasks followed by a final search-index task, rather than one coarse application task.
- Keep closing concrete output differences found against bDS2; approved normalization differences belong in this document when discovered. - Keep closing concrete output differences found against bDS2; approved normalization differences belong in this document when discovered.
### One-shot AI — Mostly done ### One-shot AI — Mostly done

View File

@@ -70,10 +70,6 @@ where
}) })
} }
#[expect(
clippy::too_many_arguments,
reason = "the shared gallery workflow needs its persisted post and project context"
)]
pub fn import_gallery_images( pub fn import_gallery_images(
db_path: &Path, db_path: &Path,
data_dir: &Path, data_dir: &Path,

View File

@@ -1,4 +1,4 @@
use std::collections::{HashMap, HashSet}; use std::collections::{BTreeMap, HashMap, HashSet};
use std::path::Path; use std::path::Path;
use crate::db::DbConnection as Connection; use crate::db::DbConnection as Connection;
@@ -14,7 +14,8 @@ use crate::engine::{EngineError, EngineResult};
use crate::model::{CategorySettings, Post, ProjectMetadata}; use crate::model::{CategorySettings, Post, ProjectMetadata};
use crate::render::{ use crate::render::{
GeneratedWriteOutcome, build_calendar_json, build_canonical_post_path, GeneratedWriteOutcome, build_calendar_json, build_canonical_post_path,
build_site_render_artifacts, render_markdown_to_html, write_generated_bytes, build_site_render_artifacts, build_site_section_render_artifacts,
build_targeted_site_section_render_artifacts, render_markdown_to_html, write_generated_bytes,
write_generated_file, write_generated_file,
}; };
@@ -65,6 +66,24 @@ pub enum GenerationSection {
Date, Date,
} }
impl GenerationSection {
pub const ALL: [Self; 5] = [
Self::Core,
Self::Single,
Self::Category,
Self::Tag,
Self::Date,
];
}
impl GenerationReport {
pub fn append(&mut self, mut other: Self) {
self.written_paths.append(&mut other.written_paths);
self.skipped_paths.append(&mut other.skipped_paths);
self.deleted_paths.append(&mut other.deleted_paths);
}
}
pub fn generate_starter_site( pub fn generate_starter_site(
conn: &Connection, conn: &Connection,
output_dir: &Path, output_dir: &Path,
@@ -94,6 +113,41 @@ pub fn generate_starter_site_with_progress(
mut on_page: impl FnMut(usize, usize, &str), mut on_page: impl FnMut(usize, usize, &str),
) -> EngineResult<GenerationReport> { ) -> EngineResult<GenerationReport> {
let mut report = GenerationReport::default(); let mut report = GenerationReport::default();
for section in GenerationSection::ALL {
report.append(render_site_section_with_progress(
conn,
output_dir,
project_id,
metadata,
posts,
section,
&mut on_page,
|| false,
)?);
}
report.append(build_site_search_index(
conn, output_dir, project_id, metadata,
)?);
Ok(report)
}
#[expect(
clippy::too_many_arguments,
reason = "section rendering uses the existing generation context and two callbacks"
)]
pub fn render_site_section_with_progress(
conn: &Connection,
output_dir: &Path,
project_id: &str,
metadata: &ProjectMetadata,
posts: &[PublishedPostSource],
section: GenerationSection,
mut on_page: impl FnMut(usize, usize, &str),
mut is_cancelled: impl FnMut() -> bool,
) -> EngineResult<GenerationReport> {
if is_cancelled() {
return Err(EngineError::Validation("cancelled".to_string()));
}
let data_dir = project_data_dir(output_dir); let data_dir = project_data_dir(output_dir);
let category_settings = load_category_settings(&data_dir); let category_settings = load_category_settings(&data_dir);
let list_posts = filter_posts_for_lists(posts, &category_settings); let list_posts = filter_posts_for_lists(posts, &category_settings);
@@ -101,12 +155,21 @@ pub fn generate_starter_site_with_progress(
.iter() .iter()
.map(|source| (source.post.clone(), source.body_markdown.clone())) .map(|source| (source.post.clone(), source.body_markdown.clone()))
.collect::<Vec<_>>(); .collect::<Vec<_>>();
let artifacts = let artifacts = build_site_section_render_artifacts(
build_site_render_artifacts(conn, &data_dir, project_id, metadata, &input_posts) conn,
&data_dir,
project_id,
metadata,
&input_posts,
section,
)
.map_err(|error| EngineError::Parse(error.to_string()))?; .map_err(|error| EngineError::Parse(error.to_string()))?;
let mut report = GenerationReport::default();
let total_pages = artifacts.pages.len(); let total_pages = artifacts.pages.len();
for (index, page) in artifacts.pages.iter().enumerate() { for (index, page) in artifacts.pages.iter().enumerate() {
if is_cancelled() {
return Err(EngineError::Validation("cancelled".to_string()));
}
write_out( write_out(
conn, conn,
output_dir, output_dir,
@@ -118,78 +181,20 @@ pub fn generate_starter_site_with_progress(
on_page(index + 1, total_pages, &page.url_path); on_page(index + 1, total_pages, &page.url_path);
} }
write_bundled_site_assets(conn, output_dir, project_id, &mut report)?; if section == GenerationSection::Core {
write_core_outputs(
write_out(
conn, conn,
output_dir, output_dir,
project_id, project_id,
"calendar.json",
&build_calendar_json(
&list_posts
.iter()
.map(|source| source.post.clone())
.collect::<Vec<_>>(),
)?,
&mut report,
)?;
for render_language in render_languages(metadata) {
let localized_posts =
localized_sources(conn, &data_dir, &list_posts, &render_language, metadata)?;
let prefix = if render_language
== metadata
.main_language
.clone()
.unwrap_or_else(|| "en".to_string())
{
String::new()
} else {
format!("{}/", render_language)
};
let rss = build_rss_xml(metadata, &localized_posts, &render_language);
if prefix.is_empty() {
write_out(conn, output_dir, project_id, "rss.xml", &rss, &mut report)?;
}
write_out(
conn,
output_dir,
project_id,
&format!("{prefix}feed.xml"),
&rss,
&mut report,
)?;
write_out(
conn,
output_dir,
project_id,
&format!("{prefix}atom.xml"),
&build_atom_xml(metadata, &localized_posts, &render_language),
&mut report,
)?;
write_out(
conn,
output_dir,
project_id,
&format!("{prefix}sitemap.xml"),
&build_sitemap_xml(
metadata, metadata,
&artifacts.pages, &data_dir,
&localized_posts, &list_posts,
&render_language, &artifacts.route_manifest,
), None,
&mut report, &mut report,
&mut is_cancelled,
)?; )?;
} }
write_pagefind_indexes(
conn,
output_dir,
project_id,
&artifacts.pagefind_documents,
&mut report,
)?;
Ok(report) Ok(report)
} }
@@ -236,8 +241,6 @@ pub fn apply_validation_sections(
let section_set = sections.iter().copied().collect::<HashSet<_>>(); let section_set = sections.iter().copied().collect::<HashSet<_>>();
let data_dir = project_data_dir(output_dir); let data_dir = project_data_dir(output_dir);
let category_settings = load_category_settings(&data_dir);
let list_posts = filter_posts_for_lists(posts, &category_settings);
let input_posts = posts let input_posts = posts
.iter() .iter()
.map(|source| (source.post.clone(), source.body_markdown.clone())) .map(|source| (source.post.clone(), source.body_markdown.clone()))
@@ -248,8 +251,91 @@ pub fn apply_validation_sections(
let mut report = GenerationReport::default(); let mut report = GenerationReport::default();
let expected_paths = expected_paths_for_sections(metadata, &artifacts.pages, &section_set); let expected_paths = expected_paths_for_sections(metadata, &artifacts.pages, &section_set);
for page in &artifacts.pages { for section in sections {
if path_matches_sections(&page.relative_path, &section_set) { report.append(render_site_section_with_progress(
conn,
output_dir,
project_id,
metadata,
posts,
*section,
|_current, _total, _url| {},
|| false,
)?);
}
remove_extra_section_paths(output_dir, &expected_paths, &section_set, &mut report)?;
report.append(build_site_search_index(
conn, output_dir, project_id, metadata,
)?);
Ok(report)
}
#[expect(
clippy::too_many_arguments,
reason = "targeted apply adds its validation report to the existing generation context"
)]
pub fn apply_validation_section_with_progress(
conn: &Connection,
output_dir: &Path,
project_id: &str,
metadata: &ProjectMetadata,
posts: &[PublishedPostSource],
validation: &SiteValidationReport,
section: GenerationSection,
mut on_page: impl FnMut(usize, usize, &str),
mut is_cancelled: impl FnMut() -> bool,
) -> EngineResult<GenerationReport> {
if is_cancelled() {
return Err(EngineError::Validation("cancelled".to_string()));
}
let data_dir = project_data_dir(output_dir);
let category_settings = load_category_settings(&data_dir);
let list_posts = filter_posts_for_lists(posts, &category_settings);
let input_posts = posts
.iter()
.map(|source| (source.post.clone(), source.body_markdown.clone()))
.collect::<Vec<_>>();
let requested = validation
.missing_pages
.iter()
.chain(validation.stale_pages.iter())
.cloned()
.collect::<HashSet<_>>();
let fallback = validation
.missing_pages
.iter()
.chain(validation.extra_pages.iter())
.chain(validation.stale_pages.iter())
.any(|path| classify_generated_path(path).is_none());
let artifacts = if fallback {
build_site_section_render_artifacts(
conn,
&data_dir,
project_id,
metadata,
&input_posts,
section,
)
} else {
build_targeted_site_section_render_artifacts(
conn,
&data_dir,
project_id,
metadata,
&input_posts,
section,
&requested,
)
}
.map_err(|error| EngineError::Parse(error.to_string()))?;
let mut report = GenerationReport::default();
let total_pages = artifacts.pages.len();
for (index, page) in artifacts.pages.iter().enumerate() {
if is_cancelled() {
return Err(EngineError::Validation("cancelled".to_string()));
}
write_out( write_out(
conn, conn,
output_dir, output_dir,
@@ -258,84 +344,95 @@ pub fn apply_validation_sections(
&page.html, &page.html,
&mut report, &mut report,
)?; )?;
} on_page(index + 1, total_pages, &page.url_path);
} }
if section_set.contains(&GenerationSection::Core) { if section == GenerationSection::Core {
write_bundled_site_assets(conn, output_dir, project_id, &mut report)?; write_core_outputs(
write_out(
conn, conn,
output_dir, output_dir,
project_id, project_id,
"calendar.json", metadata,
&build_calendar_json( &data_dir,
&list_posts,
&artifacts.route_manifest,
(!fallback).then_some(&requested),
&mut report,
&mut is_cancelled,
)?;
}
for path in &validation.extra_pages {
if is_cancelled() {
return Err(EngineError::Validation("cancelled".to_string()));
}
let owned_by_section = classify_generated_path(path)
.map_or(section == GenerationSection::Core, |owner| owner == section);
if owned_by_section && output_dir.join(path).is_file() {
std::fs::remove_file(output_dir.join(path)).map_err(EngineError::Io)?;
report.deleted_paths.push(path.clone());
}
}
Ok(report)
}
#[expect(
clippy::too_many_arguments,
reason = "generation context is existing domain data"
)]
fn write_core_outputs(
conn: &Connection,
output_dir: &Path,
project_id: &str,
metadata: &ProjectMetadata,
data_dir: &Path,
list_posts: &[PublishedPostSource],
route_manifest: &[crate::render::SitePage],
requested: Option<&HashSet<String>>,
report: &mut GenerationReport,
is_cancelled: &mut impl FnMut() -> bool,
) -> EngineResult<()> {
if requested.is_none() {
write_bundled_site_assets(conn, output_dir, project_id, report)?;
}
let mut outputs = vec![(
"calendar.json".to_string(),
build_calendar_json(
&list_posts &list_posts
.iter() .iter()
.map(|source| source.post.clone()) .map(|source| source.post.clone())
.collect::<Vec<_>>(), .collect::<Vec<_>>(),
)?, )?,
&mut report, )];
)?;
for render_language in render_languages(metadata) { for render_language in render_languages(metadata) {
let localized_posts = let localized_posts =
localized_sources(conn, &data_dir, &list_posts, &render_language, metadata)?; localized_sources(conn, data_dir, list_posts, &render_language, metadata)?;
let prefix = if render_language let prefix = if render_language == metadata.main_language.as_deref().unwrap_or("en") {
== metadata
.main_language
.clone()
.unwrap_or_else(|| "en".to_string())
{
String::new() String::new()
} else { } else {
format!("{}/", render_language) format!("{render_language}/")
}; };
let rss = build_rss_xml(metadata, &localized_posts, &render_language); let rss = build_rss_xml(metadata, &localized_posts, &render_language);
if prefix.is_empty() { outputs.push((format!("{prefix}rss.xml"), rss.clone()));
write_out(conn, output_dir, project_id, "rss.xml", &rss, &mut report)?; outputs.push((format!("{prefix}feed.xml"), rss));
outputs.push((
format!("{prefix}atom.xml"),
build_atom_xml(metadata, &localized_posts, &render_language),
));
outputs.push((
format!("{prefix}sitemap.xml"),
build_sitemap_xml(metadata, route_manifest, &localized_posts, &render_language),
));
} }
write_out( for (path, content) in outputs {
conn, if is_cancelled() {
output_dir, return Err(EngineError::Validation("cancelled".to_string()));
project_id, }
&format!("{prefix}feed.xml"), if requested.is_none_or(|requested| requested.contains(&path)) {
&rss, write_out(conn, output_dir, project_id, &path, &content, report)?;
&mut report,
)?;
write_out(
conn,
output_dir,
project_id,
&format!("{prefix}atom.xml"),
&build_atom_xml(metadata, &localized_posts, &render_language),
&mut report,
)?;
write_out(
conn,
output_dir,
project_id,
&format!("{prefix}sitemap.xml"),
&build_sitemap_xml(
metadata,
&artifacts.pages,
&localized_posts,
&render_language,
),
&mut report,
)?;
} }
} }
Ok(())
remove_extra_section_paths(output_dir, &expected_paths, &section_set, &mut report)?;
write_pagefind_indexes(
conn,
output_dir,
project_id,
&artifacts.pagefind_documents,
&mut report,
)?;
Ok(report)
} }
fn write_out( fn write_out(
@@ -357,35 +454,100 @@ fn write_out(
Ok(()) Ok(())
} }
fn write_pagefind_indexes( pub fn build_site_search_index(
conn: &Connection, conn: &Connection,
output_dir: &Path, output_dir: &Path,
project_id: &str, project_id: &str,
documents: &[crate::render::PagefindDocument], metadata: &ProjectMetadata,
report: &mut GenerationReport, ) -> EngineResult<GenerationReport> {
) -> EngineResult<()> { build_site_search_index_with_progress(
conn,
output_dir,
project_id,
metadata,
|_current, _total, _path| {},
|| false,
)
}
pub fn build_site_search_index_with_progress(
conn: &Connection,
output_dir: &Path,
project_id: &str,
metadata: &ProjectMetadata,
mut on_file: impl FnMut(usize, usize, &str),
mut is_cancelled: impl FnMut() -> bool,
) -> EngineResult<GenerationReport> {
let mut documents = Vec::new();
if output_dir.exists() {
for entry in WalkDir::new(output_dir).into_iter().filter_map(Result::ok) {
if !entry.file_type().is_file() {
continue;
}
let relative_path = entry
.path()
.strip_prefix(output_dir)
.unwrap_or(entry.path())
.to_string_lossy()
.replace('\\', "/");
if !relative_path.ends_with(".html")
|| relative_path.starts_with("pagefind/")
|| relative_path.contains("/pagefind/")
{
continue;
}
let language = render_languages(metadata)
.into_iter()
.find(|language| relative_path.starts_with(&format!("{language}/")))
.unwrap_or_else(|| {
metadata
.main_language
.clone()
.unwrap_or_else(|| "en".into())
});
documents.push(crate::render::PagefindDocument {
language,
url_path: String::new(),
html: std::fs::read_to_string(entry.path()).map_err(EngineError::Io)?,
relative_path,
});
}
}
documents.sort_by(|left, right| left.relative_path.cmp(&right.relative_path));
let runtime = tokio::runtime::Builder::new_current_thread() let runtime = tokio::runtime::Builder::new_current_thread()
.enable_all() .enable_all()
.build() .build()
.map_err(EngineError::Io)?; .map_err(EngineError::Io)?;
let grouped = documents.iter().fold( let grouped = documents.iter().fold(
HashMap::<String, Vec<&crate::render::PagefindDocument>>::new(), BTreeMap::<String, Vec<&crate::render::PagefindDocument>>::new(),
|mut acc, doc| { |mut acc, doc| {
acc.entry(doc.language.clone()).or_default().push(doc); acc.entry(doc.language.clone()).or_default().push(doc);
acc acc
}, },
); );
let mut outputs = Vec::new();
for (language, docs) in grouped { for (language, docs) in grouped {
if is_cancelled() {
return Err(EngineError::Validation("cancelled".to_string()));
}
let config = PagefindServiceConfig::builder() let config = PagefindServiceConfig::builder()
.keep_index_url(true) .keep_index_url(true)
.force_language(language.clone()) .force_language(language.clone())
.build(); .build();
let mut index = PagefindIndex::new(Some(config)) let mut index = PagefindIndex::new(Some(config))
.map_err(|error| EngineError::Parse(error.to_string()))?; .map_err(|error| EngineError::Parse(error.to_string()))?;
let output_prefix = if language == metadata.main_language.as_deref().unwrap_or("en") {
"pagefind".to_string()
} else {
format!("{language}/pagefind")
};
runtime.block_on(async { runtime.block_on(async {
for doc in docs { for doc in docs {
if is_cancelled() {
return Err(EngineError::Validation("cancelled".to_string()));
}
index index
.add_html_file(Some(doc.relative_path.clone()), None, doc.html.clone()) .add_html_file(Some(doc.relative_path.clone()), None, doc.html.clone())
.await .await
@@ -396,23 +558,71 @@ fn write_pagefind_indexes(
.await .await
.map_err(|error| EngineError::Parse(error.to_string()))?; .map_err(|error| EngineError::Parse(error.to_string()))?;
for file in files { for file in files {
let relative = file outputs.push((
.filename format!(
.to_string_lossy() "{output_prefix}/{}",
.trim_start_matches('/') file.filename.to_string_lossy().trim_start_matches('/')
.to_string(); ),
match write_generated_bytes(conn, output_dir, project_id, &relative, &file.contents) file.contents,
.map_err(|error| EngineError::Parse(error.to_string()))? ));
{
GeneratedWriteOutcome::Written => report.written_paths.push(relative),
GeneratedWriteOutcome::SkippedUnchanged => report.skipped_paths.push(relative),
}
} }
Ok::<(), EngineError>(()) Ok::<(), EngineError>(())
})?; })?;
} }
outputs.sort_by(|left, right| {
Ok(()) left.0
.ends_with("pagefind-entry.json")
.cmp(&right.0.ends_with("pagefind-entry.json"))
.then_with(|| left.0.cmp(&right.0))
});
let total = outputs.len();
let mut report = GenerationReport::default();
let expected = outputs
.iter()
.map(|(relative, _)| relative.clone())
.collect::<HashSet<_>>();
for (index, (relative, contents)) in outputs.into_iter().enumerate() {
if is_cancelled() {
return Err(EngineError::Validation("cancelled".to_string()));
}
match write_generated_bytes(conn, output_dir, project_id, &relative, &contents)
.map_err(|error| EngineError::Parse(error.to_string()))?
{
GeneratedWriteOutcome::Written => report.written_paths.push(relative.clone()),
GeneratedWriteOutcome::SkippedUnchanged => report.skipped_paths.push(relative.clone()),
}
on_file(index + 1, total, &relative);
}
for language in render_languages(metadata) {
let prefix = if language == metadata.main_language.as_deref().unwrap_or("en") {
"pagefind".to_string()
} else {
format!("{language}/pagefind")
};
let index_dir = output_dir.join(&prefix);
if !index_dir.exists() {
continue;
}
for entry in WalkDir::new(&index_dir).into_iter().filter_map(Result::ok) {
if is_cancelled() {
return Err(EngineError::Validation("cancelled".to_string()));
}
if !entry.file_type().is_file() {
continue;
}
let relative = entry
.path()
.strip_prefix(output_dir)
.unwrap_or(entry.path())
.to_string_lossy()
.replace('\\', "/");
if !expected.contains(&relative) {
std::fs::remove_file(entry.path()).map_err(EngineError::Io)?;
report.deleted_paths.push(relative);
}
}
}
Ok(report)
} }
fn project_data_dir(output_dir: &Path) -> std::path::PathBuf { fn project_data_dir(output_dir: &Path) -> std::path::PathBuf {
@@ -448,6 +658,7 @@ fn expected_paths_for_sections(
} else { } else {
format!("{language}/") format!("{language}/")
}; };
expected.insert(format!("{prefix}rss.xml"));
expected.insert(format!("{prefix}feed.xml")); expected.insert(format!("{prefix}feed.xml"));
expected.insert(format!("{prefix}atom.xml")); expected.insert(format!("{prefix}atom.xml"));
expected.insert(format!("{prefix}sitemap.xml")); expected.insert(format!("{prefix}sitemap.xml"));
@@ -508,7 +719,7 @@ fn path_matches_sections(path: &str, sections: &HashSet<GenerationSection>) -> b
.unwrap_or(false) .unwrap_or(false)
} }
fn classify_generated_path(path: &str) -> Option<GenerationSection> { pub(crate) fn classify_generated_path(path: &str) -> Option<GenerationSection> {
if path.ends_with(".xml") || path.ends_with(".json") { if path.ends_with(".xml") || path.ends_with(".json") {
return Some(GenerationSection::Core); return Some(GenerationSection::Core);
} }
@@ -522,13 +733,29 @@ fn classify_generated_path(path: &str) -> Option<GenerationSection> {
} }
match parts.as_slice() { match parts.as_slice() {
["index.html"] => Some(GenerationSection::Core), ["index.html"] | ["404.html"] | ["page", _, "index.html"] => Some(GenerationSection::Core),
["category", ..] => Some(GenerationSection::Category), ["category", ..] => Some(GenerationSection::Category),
["tag", ..] => Some(GenerationSection::Tag), ["tag", ..] => Some(GenerationSection::Tag),
[year, "index.html"] if is_year_segment(year) => Some(GenerationSection::Date), [year, "index.html"] if is_year_segment(year) => Some(GenerationSection::Date),
[year, "page", _, "index.html"] if is_year_segment(year) => Some(GenerationSection::Date),
[year, month, "index.html"] if is_year_segment(year) && is_month_segment(month) => { [year, month, "index.html"] if is_year_segment(year) && is_month_segment(month) => {
Some(GenerationSection::Date) Some(GenerationSection::Date)
} }
[year, month, "page", _, "index.html"]
if is_year_segment(year) && is_month_segment(month) =>
{
Some(GenerationSection::Date)
}
[year, month, day, "index.html"]
if is_year_segment(year) && is_month_segment(month) && is_day_segment(day) =>
{
Some(GenerationSection::Date)
}
[year, month, day, "page", _, "index.html"]
if is_year_segment(year) && is_month_segment(month) && is_day_segment(day) =>
{
Some(GenerationSection::Date)
}
[year, month, day, _slug, "index.html"] [year, month, day, _slug, "index.html"]
if is_year_segment(year) && is_month_segment(month) && is_day_segment(day) => if is_year_segment(year) && is_month_segment(month) && is_day_segment(day) =>
{ {
@@ -545,6 +772,8 @@ fn has_language_prefix(parts: &[&str]) -> bool {
&& *first != "category" && *first != "category"
&& *first != "tag" && *first != "tag"
&& (*second == "index.html" && (*second == "index.html"
|| *second == "404.html"
|| *second == "page"
|| is_year_segment(second) || is_year_segment(second)
|| *second == "category" || *second == "category"
|| *second == "tag") || *second == "tag")
@@ -570,13 +799,7 @@ fn matches_generated_extension(path: &str) -> bool {
} }
fn all_sections() -> Vec<GenerationSection> { fn all_sections() -> Vec<GenerationSection> {
vec![ GenerationSection::ALL.to_vec()
GenerationSection::Core,
GenerationSection::Single,
GenerationSection::Category,
GenerationSection::Tag,
GenerationSection::Date,
]
} }
fn section_sort_key(section: &GenerationSection) -> u8 { fn section_sort_key(section: &GenerationSection) -> u8 {

View File

@@ -169,6 +169,32 @@ impl TaskManager {
self.state_changed.notify_all(); self.state_changed.notify_all();
} }
/// Cancel every active task in a group and release their workers.
pub fn cancel_group(&self, group_id: &str) {
let mut tasks = self.tasks.lock().unwrap();
let now = Instant::now();
for entry in tasks.iter_mut().filter(|task| {
task.group_id.as_deref() == Some(group_id)
&& matches!(task.status, TaskStatus::Running | TaskStatus::Pending)
}) {
entry.cancel_flag.store(true, Ordering::Release);
entry.status = TaskStatus::Cancelled;
entry.finished_at = Some(now);
}
Self::promote_next(&mut tasks, self.max_concurrent);
self.state_changed.notify_all();
}
/// Return the group containing a task, if any.
pub fn group_id(&self, task_id: TaskId) -> Option<String> {
self.tasks
.lock()
.unwrap()
.iter()
.find(|task| task.id == task_id)
.and_then(|task| task.group_id.clone())
}
/// Check whether a task has been cancelled. /// Check whether a task has been cancelled.
pub fn is_cancelled(&self, task_id: TaskId) -> bool { pub fn is_cancelled(&self, task_id: TaskId) -> bool {
let tasks = self.tasks.lock().unwrap(); let tasks = self.tasks.lock().unwrap();
@@ -292,14 +318,19 @@ impl TaskManager {
/// Promote the next queued task to running if capacity allows. /// Promote the next queued task to running if capacity allows.
fn promote_next(tasks: &mut [TaskEntry], max_concurrent: usize) { fn promote_next(tasks: &mut [TaskEntry], max_concurrent: usize) {
let running = tasks while tasks
.iter() .iter()
.filter(|t| t.status == TaskStatus::Running) .filter(|task| task.status == TaskStatus::Running)
.count(); .count()
if running < max_concurrent < max_concurrent
&& let Some(t) = tasks.iter_mut().find(|t| t.status == TaskStatus::Pending)
{ {
t.status = TaskStatus::Running; let Some(task) = tasks
.iter_mut()
.find(|task| task.status == TaskStatus::Pending)
else {
break;
};
task.status = TaskStatus::Running;
} }
} }
} }
@@ -483,6 +514,22 @@ mod tests {
assert_eq!(mgr.status(id), Some(TaskStatus::Completed)); assert_eq!(mgr.status(id), Some(TaskStatus::Completed));
} }
#[test]
fn cancelling_group_settles_every_active_task() {
let mgr = TaskManager::new(2);
let first = mgr.submit_grouped("first", "generation-1", "Render Site");
let second = mgr.submit_grouped("second", "generation-1", "Render Site");
let third = mgr.submit_grouped("third", "generation-1", "Render Site");
let unrelated = mgr.submit("unrelated");
mgr.cancel_group("generation-1");
assert_eq!(mgr.status(first), Some(TaskStatus::Cancelled));
assert_eq!(mgr.status(second), Some(TaskStatus::Cancelled));
assert_eq!(mgr.status(third), Some(TaskStatus::Cancelled));
assert_ne!(mgr.status(unrelated), Some(TaskStatus::Cancelled));
}
#[test] #[test]
fn report_progress_updates_task() { fn report_progress_updates_task() {
let mgr = TaskManager::default(); let mgr = TaskManager::default();

View File

@@ -6,7 +6,7 @@ use walkdir::WalkDir;
use crate::db::queries; use crate::db::queries;
use crate::engine::{EngineError, EngineResult}; use crate::engine::{EngineError, EngineResult};
use crate::model::{Post, PostStatus}; use crate::model::Post;
use crate::render::build_site_render_artifacts; use crate::render::build_site_render_artifacts;
use crate::util::file_hash; use crate::util::file_hash;
@@ -35,7 +35,6 @@ pub fn validate_site(
.map(|page| page.relative_path.clone()) .map(|page| page.relative_path.clone())
.collect::<HashSet<_>>(); .collect::<HashSet<_>>();
expected.insert("calendar.json".to_string()); expected.insert("calendar.json".to_string());
expected.insert("rss.xml".to_string());
for language in render_languages(&metadata) { for language in render_languages(&metadata) {
let prefix = if language let prefix = if language
== metadata == metadata
@@ -47,6 +46,7 @@ pub fn validate_site(
} else { } else {
format!("{language}/") format!("{language}/")
}; };
expected.insert(format!("{prefix}rss.xml"));
expected.insert(format!("{prefix}feed.xml")); expected.insert(format!("{prefix}feed.xml"));
expected.insert(format!("{prefix}atom.xml")); expected.insert(format!("{prefix}atom.xml"));
expected.insert(format!("{prefix}sitemap.xml")); expected.insert(format!("{prefix}sitemap.xml"));
@@ -124,20 +124,12 @@ fn load_published_posts(
let mut published = Vec::new(); let mut published = Vec::new();
for post in posts for post in posts
.into_iter() .into_iter()
.filter(|post| post.status == PostStatus::Published) .filter(crate::engine::generation::has_published_snapshot)
{ {
let body = if let Some(content) = &post.content { if let Some(source) = crate::engine::generation::load_published_post_source(data_dir, post)?
content.clone() {
} else if let Some(content) = &post.published_content { published.push((source.post, source.body_markdown));
content.clone() }
} else {
let raw =
std::fs::read_to_string(data_dir.join(post.file_path.trim_start_matches('/')))?;
crate::util::frontmatter::read_post_file(&raw)
.map(|(_, body)| body)
.map_err(EngineError::Parse)?
};
published.push((post, body));
} }
Ok(published) Ok(published)
} }

View File

@@ -96,7 +96,9 @@ pub fn write_generated_bytes(
pub fn build_core_generation_paths(main_language: &str, blog_languages: &[String]) -> Vec<String> { pub fn build_core_generation_paths(main_language: &str, blog_languages: &[String]) -> Vec<String> {
let mut paths = vec![ let mut paths = vec![
"index.html".to_string(), "index.html".to_string(),
"404.html".to_string(),
"sitemap.xml".to_string(), "sitemap.xml".to_string(),
"rss.xml".to_string(),
"feed.xml".to_string(), "feed.xml".to_string(),
"atom.xml".to_string(), "atom.xml".to_string(),
"calendar.json".to_string(), "calendar.json".to_string(),
@@ -105,6 +107,9 @@ pub fn build_core_generation_paths(main_language: &str, blog_languages: &[String
for language in blog_languages { for language in blog_languages {
if language != main_language { if language != main_language {
paths.push(format!("{language}/index.html")); paths.push(format!("{language}/index.html"));
paths.push(format!("{language}/404.html"));
paths.push(format!("{language}/sitemap.xml"));
paths.push(format!("{language}/rss.xml"));
paths.push(format!("{language}/feed.xml")); paths.push(format!("{language}/feed.xml"));
paths.push(format!("{language}/atom.xml")); paths.push(format!("{language}/atom.xml"));
} }

View File

@@ -20,7 +20,8 @@ pub use routes::{
}; };
pub use site::{ pub use site::{
PagefindDocument, PreviewRenderResult, SitePage, SiteRenderArtifacts, build_preview_response, PagefindDocument, PreviewRenderResult, SitePage, SiteRenderArtifacts, build_preview_response,
build_site_render_artifacts, build_site_render_artifacts, build_site_section_render_artifacts,
build_targeted_site_section_render_artifacts,
}; };
pub use template_lookup::{ pub use template_lookup::{
RenderCategorySettings, RenderTemplateLookup, TemplateLookupError, resolve_post_template, RenderCategorySettings, RenderTemplateLookup, TemplateLookupError, resolve_post_template,

View File

@@ -1,4 +1,4 @@
use std::collections::{BTreeMap, HashMap}; use std::collections::{BTreeMap, HashMap, HashSet};
use std::error::Error; use std::error::Error;
use std::fs; use std::fs;
use std::path::Path; use std::path::Path;
@@ -10,6 +10,7 @@ use rayon::prelude::*;
use serde_json::{Value, json}; use serde_json::{Value, json};
use crate::db::queries; use crate::db::queries;
use crate::engine::generation::{GenerationSection, classify_generated_path};
use crate::engine::menu::{self, MenuItemKind}; use crate::engine::menu::{self, MenuItemKind};
use crate::model::{ use crate::model::{
CategorySettings, Media, Post, ProjectMetadata, ScriptKind, Tag, Template, TemplateKind, CategorySettings, Media, Post, ProjectMetadata, ScriptKind, Tag, Template, TemplateKind,
@@ -56,6 +57,7 @@ pub struct PagefindDocument {
pub struct SiteRenderArtifacts { pub struct SiteRenderArtifacts {
pub pages: Vec<SitePage>, pub pages: Vec<SitePage>,
pub pagefind_documents: Vec<PagefindDocument>, pub pagefind_documents: Vec<PagefindDocument>,
pub route_manifest: Vec<SitePage>,
} }
#[derive(Debug, Clone, PartialEq, Eq)] #[derive(Debug, Clone, PartialEq, Eq)]
@@ -110,9 +112,56 @@ pub fn build_site_render_artifacts(
metadata, metadata,
published_posts, published_posts,
false, false,
None,
None,
) )
} }
pub fn build_site_section_render_artifacts(
conn: &Connection,
data_dir: &Path,
project_id: &str,
metadata: &ProjectMetadata,
published_posts: &[(Post, String)],
section: GenerationSection,
) -> Result<SiteRenderArtifacts, Box<dyn Error + Send + Sync>> {
build_site_render_artifacts_with_mode(
conn,
data_dir,
project_id,
metadata,
published_posts,
false,
Some(section),
None,
)
}
pub fn build_targeted_site_section_render_artifacts(
conn: &Connection,
data_dir: &Path,
project_id: &str,
metadata: &ProjectMetadata,
published_posts: &[(Post, String)],
section: GenerationSection,
requested_paths: &HashSet<String>,
) -> Result<SiteRenderArtifacts, Box<dyn Error + Send + Sync>> {
build_site_render_artifacts_with_mode(
conn,
data_dir,
project_id,
metadata,
published_posts,
false,
Some(section),
Some(requested_paths),
)
}
#[expect(
clippy::too_many_arguments,
reason = "the shared renderer accepts optional section and path filters"
)]
fn build_site_render_artifacts_with_mode( fn build_site_render_artifacts_with_mode(
conn: &Connection, conn: &Connection,
data_dir: &Path, data_dir: &Path,
@@ -120,6 +169,8 @@ fn build_site_render_artifacts_with_mode(
metadata: &ProjectMetadata, metadata: &ProjectMetadata,
published_posts: &[(Post, String)], published_posts: &[(Post, String)],
is_preview: bool, is_preview: bool,
section: Option<GenerationSection>,
requested_paths: Option<&HashSet<String>>,
) -> Result<SiteRenderArtifacts, Box<dyn Error + Send + Sync>> { ) -> Result<SiteRenderArtifacts, Box<dyn Error + Send + Sync>> {
let bundle = load_template_bundle(conn, data_dir, project_id)?; let bundle = load_template_bundle(conn, data_dir, project_id)?;
let main_language = main_language(metadata).to_string(); let main_language = main_language(metadata).to_string();
@@ -144,10 +195,24 @@ fn build_site_render_artifacts_with_mode(
&tags, &tags,
&category_settings, &category_settings,
); );
artifacts
.route_manifest
.extend(routes.iter().map(|route| SitePage {
language: language.clone(),
relative_path: route.relative_path.clone(),
url_path: route.url_path.clone(),
html: String::new(),
}));
let post_data_json_by_id = build_post_data_json_by_id(&localized_posts); let post_data_json_by_id = build_post_data_json_by_id(&localized_posts);
let menu_items = build_menu_items(data_dir, &language, &main_language)?; let menu_items = build_menu_items(data_dir, &language, &main_language)?;
let rendered_list_pages = routes let rendered_list_pages = routes
.par_iter() .par_iter()
.filter(|route| {
section.is_none_or(|section| {
classify_generated_path(&route.relative_path) == Some(section)
}) && requested_paths
.is_none_or(|requested| requested.contains(&route.relative_path))
})
.map(|route| { .map(|route| {
render_list_route( render_list_route(
route, route,
@@ -180,9 +245,46 @@ fn build_site_render_artifacts_with_mode(
artifacts.pages.push(page); artifacts.pages.push(page);
} }
if section.is_none_or(|section| section == GenerationSection::Core) {
let relative_path = if language == main_language {
"404.html".to_string()
} else {
format!("{language}/404.html")
};
if requested_paths.is_none_or(|requested| requested.contains(&relative_path)) {
let url_path = format!("/{}", relative_path.trim_end_matches(".html"));
artifacts.pages.push(SitePage {
language: language.clone(),
relative_path,
url_path: url_path.clone(),
html: render_not_found_route(
&bundle,
metadata,
&language,
&url_path,
&menu_items,
)?,
});
}
}
let canonical_map = let canonical_map =
canonical_post_path_by_slug(&localized_posts, &language, &main_language); canonical_post_path_by_slug(&localized_posts, &language, &main_language);
for record in &localized_posts { for record in &localized_posts {
let canonical_path = build_canonical_post_path(&record.post, &language, &main_language);
let relative_path = format!("{}/index.html", canonical_path.trim_start_matches('/'));
artifacts.route_manifest.push(SitePage {
language: language.clone(),
relative_path: relative_path.clone(),
url_path: canonical_path.clone(),
html: String::new(),
});
if section.is_some_and(|section| section != GenerationSection::Single) {
continue;
}
if requested_paths.is_some_and(|requested| !requested.contains(&relative_path)) {
continue;
}
let html = render_post_route( let html = render_post_route(
conn, conn,
metadata, metadata,
@@ -199,8 +301,6 @@ fn build_site_render_artifacts_with_mode(
&bundle, &bundle,
is_preview, is_preview,
)?; )?;
let canonical_path = build_canonical_post_path(&record.post, &language, &main_language);
let relative_path = format!("{}/index.html", canonical_path.trim_start_matches('/'));
artifacts.pagefind_documents.push(PagefindDocument { artifacts.pagefind_documents.push(PagefindDocument {
language: language.clone(), language: language.clone(),
relative_path: relative_path.clone(), relative_path: relative_path.clone(),
@@ -234,6 +334,8 @@ pub fn build_preview_response(
metadata, metadata,
published_posts, published_posts,
true, true,
None,
None,
)?; )?;
let normalized = normalize_request_path(requested_path); let normalized = normalize_request_path(requested_path);
if let Some(page) = artifacts if let Some(page) = artifacts
@@ -464,6 +566,7 @@ fn build_language_routes(
let mut tag_posts: BTreeMap<String, Vec<RenderPostRecord>> = BTreeMap::new(); let mut tag_posts: BTreeMap<String, Vec<RenderPostRecord>> = BTreeMap::new();
let mut year_posts: BTreeMap<i32, Vec<RenderPostRecord>> = BTreeMap::new(); let mut year_posts: BTreeMap<i32, Vec<RenderPostRecord>> = BTreeMap::new();
let mut month_posts: BTreeMap<(i32, u32), Vec<RenderPostRecord>> = BTreeMap::new(); let mut month_posts: BTreeMap<(i32, u32), Vec<RenderPostRecord>> = BTreeMap::new();
let mut day_posts: BTreeMap<(i32, u32, u32), Vec<RenderPostRecord>> = BTreeMap::new();
for record in posts { for record in posts {
for category in &record.post.categories { for category in &record.post.categories {
@@ -490,6 +593,10 @@ fn build_language_routes(
.entry((timestamp.year(), timestamp.month())) .entry((timestamp.year(), timestamp.month()))
.or_default() .or_default()
.push(record.clone()); .push(record.clone());
day_posts
.entry((timestamp.year(), timestamp.month(), timestamp.day()))
.or_default()
.push(record.clone());
} }
} }
@@ -552,6 +659,20 @@ fn build_language_routes(
)); ));
} }
for ((year, month, day), records) in day_posts {
routes.extend(paginated_route_specs(
&records,
per_page,
format!(
"{}/{year}/{month:02}/{day:02}",
language_root_prefix(language, metadata)
),
format!("{} {year}-{month:02}-{day:02}", metadata.name),
Some(json!({"kind": "day", "year": year, "month": month, "day": day})),
None,
));
}
routes routes
} }

View File

@@ -4,8 +4,9 @@ use bds_core::db::Database;
use bds_core::db::queries::project::insert_project; use bds_core::db::queries::project::insert_project;
use bds_core::db::queries::template::insert_template; use bds_core::db::queries::template::insert_template;
use bds_core::engine::generation::{ use bds_core::engine::generation::{
PublishedPostSource, apply_validation_sections, generate_starter_site, GenerationSection, PublishedPostSource, apply_validation_section_with_progress,
load_published_post_source, sections_from_validation_report, apply_validation_sections, build_site_search_index, generate_starter_site,
load_published_post_source, render_site_section_with_progress, sections_from_validation_report,
}; };
use bds_core::engine::meta::write_category_meta_json; use bds_core::engine::meta::write_category_meta_json;
use bds_core::engine::validate_site::validate_site; use bds_core::engine::validate_site::validate_site;
@@ -99,6 +100,14 @@ fn setup() -> (Database, TempDir) {
(db, dir) (db, dir)
} }
fn write_published_snapshot(dir: &TempDir, post: &mut Post, body: &str) {
post.file_path = format!("posts/{}.md", post.slug);
let frontmatter = bds_core::util::frontmatter::PostFrontmatter::from_post(post).to_yaml();
let file = bds_core::util::frontmatter::format_frontmatter(&frontmatter, body);
std::fs::create_dir_all(dir.path().join("posts")).unwrap();
std::fs::write(dir.path().join(&post.file_path), file).unwrap();
}
#[test] #[test]
fn reopened_draft_generation_uses_last_published_file() { fn reopened_draft_generation_uses_last_published_file() {
let (_db, dir) = setup(); let (_db, dir) = setup();
@@ -117,17 +126,47 @@ fn reopened_draft_generation_uses_last_published_file() {
assert_eq!(source.body_markdown, "Published body"); assert_eq!(source.body_markdown, "Published body");
} }
#[test]
fn validation_keeps_reopened_draft_published_snapshots() {
let (db, dir) = setup();
let mut post = make_post("reopened", 1_710_000_000_000);
post.status = PostStatus::Draft;
post.content = Some("Unpublished draft body".into());
write_published_snapshot(&dir, &mut post, "Published body");
bds_core::db::queries::post::insert_post(db.conn(), &post).unwrap();
let source = load_published_post_source(dir.path(), post)
.unwrap()
.unwrap();
generate_starter_site(
db.conn(),
dir.path(),
"p1",
&make_metadata(),
&[source],
"en",
)
.unwrap();
let validation = validate_site(db.conn(), dir.path(), "p1").unwrap();
assert!(validation.missing_pages.is_empty());
assert!(validation.extra_pages.is_empty());
assert!(validation.stale_pages.is_empty());
}
#[test] #[test]
fn generation_engine_writes_core_and_single_post_artifacts() { fn generation_engine_writes_core_and_single_post_artifacts() {
let (db, dir) = setup(); let (db, dir) = setup();
let metadata = make_metadata(); let metadata = make_metadata();
let hello = make_post("hello", 1_710_000_000_000);
let next = make_post("next", 1_710_086_400_000);
let posts = vec![ let posts = vec![
PublishedPostSource { PublishedPostSource {
post: make_post("hello", 1_710_000_000_000), post: hello,
body_markdown: "Hello **world**".into(), body_markdown: "Hello **world**".into(),
}, },
PublishedPostSource { PublishedPostSource {
post: make_post("next", 1_710_086_400_000), post: next,
body_markdown: "Next post".into(), body_markdown: "Next post".into(),
}, },
]; ];
@@ -141,6 +180,12 @@ fn generation_engine_writes_core_and_single_post_artifacts() {
assert!(report.written_paths.contains(&"feed.xml".to_string())); assert!(report.written_paths.contains(&"feed.xml".to_string()));
assert!(report.written_paths.contains(&"atom.xml".to_string())); assert!(report.written_paths.contains(&"atom.xml".to_string()));
assert!(report.written_paths.contains(&"sitemap.xml".to_string())); assert!(report.written_paths.contains(&"sitemap.xml".to_string()));
assert!(report.written_paths.contains(&"404.html".to_string()));
assert!(
report
.written_paths
.contains(&"2024/03/09/index.html".to_string())
);
assert!( assert!(
report report
.written_paths .written_paths
@@ -180,6 +225,110 @@ fn generation_engine_writes_core_and_single_post_artifacts() {
assert!(sitemap.contains("https://example.com/category/article")); assert!(sitemap.contains("https://example.com/category/article"));
} }
#[test]
fn section_generation_reports_its_urls_and_defers_pagefind() {
let (db, dir) = setup();
let metadata = make_metadata();
let posts = vec![PublishedPostSource {
post: make_post("hello", 1_710_000_000_000),
body_markdown: "Hello **world**".into(),
}];
let mut urls = Vec::new();
let report = render_site_section_with_progress(
db.conn(),
dir.path(),
"p1",
&metadata,
&posts,
GenerationSection::Single,
|current, total, url| urls.push((current, total, url.to_string())),
|| false,
)
.unwrap();
assert_eq!(urls, vec![(1, 1, "/2024/03/09/hello".to_string())]);
assert_eq!(report.written_paths, vec!["2024/03/09/hello/index.html"]);
assert!(!dir.path().join("pagefind").exists());
let index_report = build_site_search_index(db.conn(), dir.path(), "p1", &metadata).unwrap();
assert!(!index_report.written_paths.is_empty());
assert!(
dir.path().join("pagefind/pagefind-ui.js").exists(),
"pagefind outputs: {:?}",
index_report.written_paths
);
let old_fragment = index_report
.written_paths
.iter()
.find(|path| path.contains("/fragment/"))
.cloned()
.unwrap();
let changed_posts = vec![PublishedPostSource {
post: make_post("hello", 1_710_000_000_000),
body_markdown: "Changed body".into(),
}];
render_site_section_with_progress(
db.conn(),
dir.path(),
"p1",
&metadata,
&changed_posts,
GenerationSection::Single,
|_current, _total, _url| {},
|| false,
)
.unwrap();
let rebuilt = build_site_search_index(db.conn(), dir.path(), "p1", &metadata).unwrap();
assert!(rebuilt.deleted_paths.contains(&old_fragment));
assert!(!dir.path().join(old_fragment).exists());
}
#[test]
fn validation_apply_rewrites_only_the_reported_urls() {
let (db, dir) = setup();
let metadata = make_metadata();
let posts = vec![
PublishedPostSource {
post: make_post("hello", 1_710_000_000_000),
body_markdown: "Hello **world**".into(),
},
PublishedPostSource {
post: make_post("next", 1_710_086_400_000),
body_markdown: "Next post".into(),
},
];
generate_starter_site(db.conn(), dir.path(), "p1", &metadata, &posts, "en").unwrap();
std::fs::write(dir.path().join("2024/03/09/hello/index.html"), "tampered").unwrap();
let validation = bds_core::engine::validate_site::SiteValidationReport {
stale_pages: vec!["2024/03/09/hello/index.html".into()],
..Default::default()
};
let mut urls = Vec::new();
let report = apply_validation_section_with_progress(
db.conn(),
dir.path(),
"p1",
&metadata,
&posts,
&validation,
GenerationSection::Single,
|current, total, url| urls.push((current, total, url.to_string())),
|| false,
)
.unwrap();
assert_eq!(urls, vec![(1, 1, "/2024/03/09/hello".to_string())]);
assert_eq!(report.written_paths, vec!["2024/03/09/hello/index.html"]);
assert!(
!report
.skipped_paths
.contains(&"2024/03/10/next/index.html".to_string())
);
}
#[test] #[test]
fn multilingual_generation_writes_language_aware_atom_and_sitemap_routes() { fn multilingual_generation_writes_language_aware_atom_and_sitemap_routes() {
let (db, dir) = setup(); let (db, dir) = setup();
@@ -195,7 +344,14 @@ fn multilingual_generation_writes_language_aware_atom_and_sitemap_routes() {
generate_starter_site(db.conn(), dir.path(), "p1", &metadata, &posts, "de").unwrap(); generate_starter_site(db.conn(), dir.path(), "p1", &metadata, &posts, "de").unwrap();
assert!(report.written_paths.contains(&"en/atom.xml".to_string())); assert!(report.written_paths.contains(&"en/atom.xml".to_string()));
assert!(report.written_paths.contains(&"en/rss.xml".to_string()));
assert!(report.written_paths.contains(&"en/sitemap.xml".to_string())); assert!(report.written_paths.contains(&"en/sitemap.xml".to_string()));
assert!(report.written_paths.contains(&"en/404.html".to_string()));
assert!(
report
.written_paths
.contains(&"en/pagefind/pagefind-ui.js".to_string())
);
let atom = std::fs::read_to_string(dir.path().join("en/atom.xml")).unwrap(); let atom = std::fs::read_to_string(dir.path().join("en/atom.xml")).unwrap();
assert!( assert!(
@@ -340,7 +496,8 @@ fn generation_engine_skips_unchanged_outputs_on_second_run() {
fn site_validation_detects_stale_and_missing_outputs() { fn site_validation_detects_stale_and_missing_outputs() {
let (db, dir) = setup(); let (db, dir) = setup();
let metadata = make_metadata(); let metadata = make_metadata();
let post = make_post("hello", 1_710_000_000_000); let mut post = make_post("hello", 1_710_000_000_000);
write_published_snapshot(&dir, &mut post, "Hello **world**");
bds_core::db::queries::post::insert_post(db.conn(), &post).unwrap(); bds_core::db::queries::post::insert_post(db.conn(), &post).unwrap();
let posts = vec![PublishedPostSource { let posts = vec![PublishedPostSource {
post, post,
@@ -362,7 +519,8 @@ fn site_validation_detects_stale_and_missing_outputs() {
fn apply_validation_repairs_core_section_outputs() { fn apply_validation_repairs_core_section_outputs() {
let (db, dir) = setup(); let (db, dir) = setup();
let metadata = make_metadata(); let metadata = make_metadata();
let post = make_post("hello", 1_710_000_000_000); let mut post = make_post("hello", 1_710_000_000_000);
write_published_snapshot(&dir, &mut post, "Hello **world**");
bds_core::db::queries::post::insert_post(db.conn(), &post).unwrap(); bds_core::db::queries::post::insert_post(db.conn(), &post).unwrap();
let posts = vec![PublishedPostSource { let posts = vec![PublishedPostSource {
post, post,
@@ -390,7 +548,8 @@ fn apply_validation_repairs_core_section_outputs() {
fn apply_validation_removes_extra_section_outputs() { fn apply_validation_removes_extra_section_outputs() {
let (db, dir) = setup(); let (db, dir) = setup();
let metadata = make_metadata(); let metadata = make_metadata();
let post = make_post("hello", 1_710_000_000_000); let mut post = make_post("hello", 1_710_000_000_000);
write_published_snapshot(&dir, &mut post, "Hello **world**");
bds_core::db::queries::post::insert_post(db.conn(), &post).unwrap(); bds_core::db::queries::post::insert_post(db.conn(), &post).unwrap();
let posts = vec![PublishedPostSource { let posts = vec![PublishedPostSource {
post, post,
@@ -426,7 +585,8 @@ fn apply_validation_removes_extra_section_outputs() {
fn site_validation_uses_html_output_directory_when_present() { fn site_validation_uses_html_output_directory_when_present() {
let (db, dir) = setup(); let (db, dir) = setup();
let metadata = make_metadata(); let metadata = make_metadata();
let post = make_post("hello", 1_710_000_000_000); let mut post = make_post("hello", 1_710_000_000_000);
write_published_snapshot(&dir, &mut post, "Hello **world**");
bds_core::db::queries::post::insert_post(db.conn(), &post).unwrap(); bds_core::db::queries::post::insert_post(db.conn(), &post).unwrap();
let posts = vec![PublishedPostSource { let posts = vec![PublishedPostSource {
post, post,

View File

@@ -114,11 +114,16 @@ fn generated_write_rewrites_stale_file_even_when_db_hash_matches() {
fn core_generation_paths_include_language_prefixed_variants() { fn core_generation_paths_include_language_prefixed_variants() {
let paths = build_core_generation_paths("en", &["en".into(), "de".into(), "fr".into()]); let paths = build_core_generation_paths("en", &["en".into(), "de".into(), "fr".into()]);
assert!(paths.contains(&"index.html".to_string())); assert!(paths.contains(&"index.html".to_string()));
assert!(paths.contains(&"404.html".to_string()));
assert!(paths.contains(&"sitemap.xml".to_string())); assert!(paths.contains(&"sitemap.xml".to_string()));
assert!(paths.contains(&"rss.xml".to_string()));
assert!(paths.contains(&"feed.xml".to_string())); assert!(paths.contains(&"feed.xml".to_string()));
assert!(paths.contains(&"atom.xml".to_string())); assert!(paths.contains(&"atom.xml".to_string()));
assert!(paths.contains(&"calendar.json".to_string())); assert!(paths.contains(&"calendar.json".to_string()));
assert!(paths.contains(&"de/index.html".to_string())); assert!(paths.contains(&"de/index.html".to_string()));
assert!(paths.contains(&"de/404.html".to_string()));
assert!(paths.contains(&"de/sitemap.xml".to_string()));
assert!(paths.contains(&"de/rss.xml".to_string()));
assert!(paths.contains(&"de/feed.xml".to_string())); assert!(paths.contains(&"de/feed.xml".to_string()));
assert!(paths.contains(&"de/atom.xml".to_string())); assert!(paths.contains(&"de/atom.xml".to_string()));
assert!(paths.contains(&"fr/index.html".to_string())); assert!(paths.contains(&"fr/index.html".to_string()));

View File

@@ -201,8 +201,17 @@ pub enum Message {
label: String, label: String,
result: Result<String, String>, result: Result<String, String>,
}, },
SiteGenerationSectionDone {
group_id: String,
task_id: TaskId,
result: Result<engine::generation::GenerationReport, String>,
},
SiteGenerationIndexDone {
group_id: String,
task_id: TaskId,
result: Result<engine::generation::GenerationReport, String>,
},
SiteValidationLoaded(Result<engine::validate_site::SiteValidationReport, String>), SiteValidationLoaded(Result<engine::validate_site::SiteValidationReport, String>),
SiteValidationApplied(Result<String, String>),
// Editor views // Editor views
PostEditor(PostEditorMsg), PostEditor(PostEditorMsg),
@@ -242,6 +251,24 @@ pub enum Message {
InitMenuBar, InitMenuBar,
} }
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum SiteGenerationKind {
Full,
Validation,
}
#[derive(Debug, Clone)]
struct SiteGenerationWorkflow {
kind: SiteGenerationKind,
db_path: PathBuf,
project_id: String,
data_dir: PathBuf,
group_name: String,
render_task_ids: Vec<TaskId>,
index_task_id: Option<TaskId>,
report: engine::generation::GenerationReport,
}
enum PersistedPostState { enum PersistedPostState {
Canonical(Box<Post>), Canonical(Box<Post>),
Translation(Box<bds_core::model::PostTranslation>), Translation(Box<bds_core::model::PostTranslation>),
@@ -754,6 +781,7 @@ pub struct BdsApp {
search_index_rebuild_required: bool, search_index_rebuild_required: bool,
search_index_rebuild_running: bool, search_index_rebuild_running: bool,
search_index_rebuild_task_id: Option<TaskId>, search_index_rebuild_task_id: Option<TaskId>,
site_generation_workflows: HashMap<String, SiteGenerationWorkflow>,
// Platform // Platform
_menu_bar: Option<muda::Menu>, _menu_bar: Option<muda::Menu>,
@@ -908,6 +936,7 @@ impl BdsApp {
search_index_rebuild_required, search_index_rebuild_required,
search_index_rebuild_running: false, search_index_rebuild_running: false,
search_index_rebuild_task_id: None, search_index_rebuild_task_id: None,
site_generation_workflows: HashMap::new(),
_menu_bar: Some(menu_bar), _menu_bar: Some(menu_bar),
menu_registry: registry, menu_registry: registry,
ui_locale: locale, ui_locale: locale,
@@ -974,6 +1003,7 @@ impl BdsApp {
search_index_rebuild_required: false, search_index_rebuild_required: false,
search_index_rebuild_running: false, search_index_rebuild_running: false,
search_index_rebuild_task_id: None, search_index_rebuild_task_id: None,
site_generation_workflows: HashMap::new(),
_menu_bar: None, _menu_bar: None,
menu_registry: MenuRegistry::empty(), menu_registry: MenuRegistry::empty(),
ui_locale: UiLocale::En, ui_locale: UiLocale::En,
@@ -1610,6 +1640,9 @@ impl BdsApp {
) )
} }
Message::CancelTask(task_id) => { Message::CancelTask(task_id) => {
if self.cancel_site_generation_task(task_id) {
return Task::none();
}
self.task_manager.cancel(task_id); self.task_manager.cancel(task_id);
self.refresh_task_snapshots(); self.refresh_task_snapshots();
Task::none() Task::none()
@@ -1831,8 +1864,9 @@ impl BdsApp {
| Message::RunSiteValidation | Message::RunSiteValidation
| Message::ApplySiteValidation | Message::ApplySiteValidation
| Message::EngineTaskDone { .. } | Message::EngineTaskDone { .. }
| Message::SiteValidationLoaded(_) | Message::SiteGenerationSectionDone { .. }
| Message::SiteValidationApplied(_)) => self.handle_engine_message(message), | Message::SiteGenerationIndexDone { .. }
| Message::SiteValidationLoaded(_)) => self.handle_engine_message(message),
// ── Toasts ── // ── Toasts ──
Message::ShowToast(level, msg) => { Message::ShowToast(level, msg) => {
@@ -2660,68 +2694,9 @@ impl BdsApp {
return Task::none(); return Task::none();
} }
let Some(project_id) = self
.active_project
.as_ref()
.map(|project| project.id.clone())
else {
self.site_validation_state.error_message =
Some(t(self.ui_locale, "engine.generateSiteNoProject"));
return Task::none();
};
let Some(data_dir) = self.data_dir.clone() else {
self.site_validation_state.error_message =
Some(t(self.ui_locale, "engine.previewDataDirUnavailable"));
return Task::none();
};
self.site_validation_state.is_applying = true; self.site_validation_state.is_applying = true;
self.site_validation_state.error_message = None; self.site_validation_state.error_message = None;
let db_path = self.db_path.clone(); self.queue_site_generation(Some(report))
let applied_label = t(self.ui_locale, "siteValidation.apply");
Task::perform(
async move {
let db = Database::open(&db_path).map_err(|error| error.to_string())?;
let metadata = engine::meta::read_project_json(&data_dir)
.map_err(|error| error.to_string())?;
let all_posts =
bds_core::db::queries::post::list_posts_by_project(db.conn(), &project_id)
.map_err(|error| error.to_string())?;
let mut sources = Vec::new();
for post in all_posts
.into_iter()
.filter(engine::generation::has_published_snapshot)
{
if let Some(source) =
engine::generation::load_published_post_source(&data_dir, post)
.map_err(|error| error.to_string())?
{
sources.push(source);
}
}
let output_dir = data_dir.join("html");
std::fs::create_dir_all(&output_dir).map_err(|error| error.to_string())?;
let apply_report = engine::generation::apply_validation_sections(
db.conn(),
&output_dir,
&project_id,
&metadata,
&sources,
&sections,
)
.map_err(|error| error.to_string())?;
Ok(format!(
"{}: written={}, skipped={}, deleted={}, output={}",
applied_label,
apply_report.written_paths.len(),
apply_report.skipped_paths.len(),
apply_report.deleted_paths.len(),
output_dir.display(),
))
},
Message::SiteValidationApplied,
)
} }
fn refresh_counts(&mut self) -> Task<Message> { fn refresh_counts(&mut self) -> Task<Message> {
@@ -6236,6 +6211,8 @@ mod tests {
use bds_core::db::Database; use bds_core::db::Database;
use bds_core::db::fts::ensure_fts_tables; use bds_core::db::fts::ensure_fts_tables;
use bds_core::db::queries::project::insert_project; use bds_core::db::queries::project::insert_project;
use bds_core::engine::generation::GenerationReport;
use bds_core::engine::task::{TaskStatus, TaskStatus::*};
use bds_core::engine::{ai, media, post, script, template}; use bds_core::engine::{ai, media, post, script, template};
use bds_core::model::{Project, ScriptKind, TemplateKind}; use bds_core::model::{Project, ScriptKind, TemplateKind};
use chrono::{Datelike, TimeZone}; use chrono::{Datelike, TimeZone};
@@ -6317,6 +6294,12 @@ mod tests {
BdsApp::new_for_tests(db, project, tmp.path().to_path_buf()) BdsApp::new_for_tests(db, project, tmp.path().to_path_buf())
} }
fn enable_generation(tmp: &TempDir) {
let mut metadata = bds_core::engine::meta::read_project_json(tmp.path()).unwrap();
metadata.public_url = Some("https://example.com".to_string());
bds_core::engine::meta::write_project_json(tmp.path(), &metadata).unwrap();
}
fn open_post_editor(app: &mut BdsApp, post: &bds_core::model::Post) { fn open_post_editor(app: &mut BdsApp, post: &bds_core::model::Post) {
let tab = crate::state::tabs::Tab { let tab = crate::state::tabs::Tab {
id: post.id.clone(), id: post.id.clone(),
@@ -6462,6 +6445,162 @@ mod tests {
)); ));
} }
#[test]
fn full_generation_queues_five_ordered_section_tasks_before_indexing() {
let (db, project, tmp) = setup();
enable_generation(&tmp);
let mut app = make_app(db, project, &tmp);
let _task = app.queue_site_generation(None);
let snapshots = app.task_manager.snapshots();
assert_eq!(
snapshots
.iter()
.map(|task| task.label.as_str())
.collect::<Vec<_>>(),
vec![
"Render Site Core",
"Render Single Posts",
"Render Category Archives",
"Render Tag Archives",
"Render Date Archives",
]
);
assert!(
snapshots
.iter()
.all(|task| task.group_name.as_deref() == Some("Render Site"))
);
assert_eq!(
snapshots
.iter()
.map(|task| task.status.clone())
.collect::<Vec<_>>(),
vec![Running, Running, Running, Pending, Pending]
);
}
#[test]
fn search_index_is_queued_only_after_every_render_task_succeeds() {
let (db, project, tmp) = setup();
enable_generation(&tmp);
let mut app = make_app(db, project, &tmp);
let _task = app.queue_site_generation(None);
let group_id = app.task_manager.snapshots()[0].group_id.clone().unwrap();
let render_ids = app.site_generation_workflows[&group_id]
.render_task_ids
.clone();
for (index, task_id) in render_ids.into_iter().enumerate() {
let _task = app.handle_engine_message(Message::SiteGenerationSectionDone {
group_id: group_id.clone(),
task_id,
result: Ok(GenerationReport::default()),
});
let has_index = app
.task_manager
.snapshots()
.iter()
.any(|task| task.label == "Build Search Index");
assert_eq!(has_index, index == 4);
}
}
#[test]
fn failed_render_cancels_its_group_and_never_queues_index() {
let (db, project, tmp) = setup();
enable_generation(&tmp);
let mut app = make_app(db, project, &tmp);
let _task = app.queue_site_generation(None);
let snapshots = app.task_manager.snapshots();
let group_id = snapshots[0].group_id.clone().unwrap();
let first_id = snapshots[0].id;
let _task = app.handle_engine_message(Message::SiteGenerationSectionDone {
group_id: group_id.clone(),
task_id: first_id,
result: Err("render failed".to_string()),
});
let snapshots = app.task_manager.snapshots();
assert_eq!(
app.task_manager.status(first_id),
Some(TaskStatus::Failed("render failed".to_string()))
);
assert!(
snapshots
.iter()
.filter(|task| task.group_id.as_deref() == Some(&group_id))
.all(|task| matches!(task.status, TaskStatus::Failed(_) | TaskStatus::Cancelled))
);
assert!(
!snapshots
.iter()
.any(|task| task.label == "Build Search Index")
);
assert!(!app.site_generation_workflows.contains_key(&group_id));
}
#[test]
fn cancelling_a_render_task_cancels_the_generation_group() {
let (db, project, tmp) = setup();
enable_generation(&tmp);
let mut app = make_app(db, project, &tmp);
let _task = app.queue_site_generation(None);
let snapshots = app.task_manager.snapshots();
let group_id = snapshots[0].group_id.clone().unwrap();
let _task = app.update(Message::CancelTask(snapshots[0].id));
let snapshots = app.task_manager.snapshots();
assert!(
snapshots
.iter()
.filter(|task| task.group_id.as_deref() == Some(&group_id))
.all(|task| task.status == TaskStatus::Cancelled)
);
assert!(
!snapshots
.iter()
.any(|task| task.label == "Build Search Index")
);
assert!(!app.site_generation_workflows.contains_key(&group_id));
}
#[test]
fn validation_apply_queues_only_affected_sections_then_index() {
let (db, project, tmp) = setup();
enable_generation(&tmp);
let mut app = make_app(db, project, &tmp);
let validation = bds_core::engine::validate_site::SiteValidationReport {
stale_pages: vec!["2024/03/09/hello/index.html".to_string()],
..Default::default()
};
let _task = app.queue_site_generation(Some(validation));
let snapshots = app.task_manager.snapshots();
assert_eq!(snapshots.len(), 1);
assert_eq!(snapshots[0].label, "Render Single Posts");
assert_eq!(
snapshots[0].group_name.as_deref(),
Some("Apply Site Validation")
);
let group_id = snapshots[0].group_id.clone().unwrap();
let _task = app.handle_engine_message(Message::SiteGenerationSectionDone {
group_id,
task_id: snapshots[0].id,
result: Ok(GenerationReport::default()),
});
assert!(
app.task_manager
.snapshots()
.iter()
.any(|task| task.label == "Build Search Index")
);
}
#[test] #[test]
fn post_editor_save_flow_persists_changes() { fn post_editor_save_flow_persists_changes() {
let (db, project, tmp) = setup(); let (db, project, tmp) = setup();

View File

@@ -139,109 +139,7 @@ impl BdsApp {
}, },
) )
} }
Message::GenerateSite => { Message::GenerateSite => self.queue_site_generation(None),
let locale = self.ui_locale;
self.spawn_engine_task(
"engine.generateSiteStarted",
move |db_path, project_id, data_dir, tm, tid| {
let db = Database::open(&db_path).map_err(|e| e.to_string())?;
let metadata = engine::meta::read_project_json(&data_dir)
.map_err(|e| e.to_string())?;
if metadata
.public_url
.as_deref()
.unwrap_or("")
.trim()
.is_empty()
{
return Err(
"public URL is required before generating the site".to_string()
);
}
let main_language = metadata
.main_language
.clone()
.unwrap_or_else(|| "en".to_string());
let all_posts = bds_core::db::queries::post::list_posts_by_project(
db.conn(),
&project_id,
)
.map_err(|e| e.to_string())?;
let published_posts = all_posts
.into_iter()
.filter(engine::generation::has_published_snapshot)
.collect::<Vec<_>>();
let total = published_posts.len().max(1) as f32;
let mut sources = Vec::new();
for (index, post) in published_posts.into_iter().enumerate() {
if tm.is_cancelled(tid) {
return Err("cancelled".to_string());
}
tm.report_progress(
tid,
Some(((index as f32) / total) * 0.7),
Some(tw(locale, "engine.renderingPost", &[("post", &post.slug)])),
);
if let Some(source) =
engine::generation::load_published_post_source(&data_dir, post)
.map_err(|error| error.to_string())?
{
sources.push(source);
}
}
let output_dir = data_dir.join("html");
std::fs::create_dir_all(&output_dir).map_err(|e| e.to_string())?;
tm.report_progress(
tid,
Some(0.85),
Some(t(locale, "engine.writingGeneratedFiles")),
);
let progress_start = 0.70_f32;
let progress_span = 0.28_f32;
let task_manager = Arc::clone(&tm);
let report = engine::generation::generate_starter_site_with_progress(
db.conn(),
&output_dir,
&project_id,
&metadata,
&sources,
&main_language,
move |current, total, path| {
let fraction = if total == 0 {
1.0
} else {
current as f32 / total as f32
};
task_manager.report_progress(
tid,
Some(progress_start + fraction * progress_span),
Some(tw(
locale,
"engine.generatedPage",
&[
("url", path),
("current", &current.to_string()),
("total", &total.to_string()),
],
)),
);
},
)
.map_err(|e| e.to_string())?;
tm.report_progress(
tid,
Some(1.0),
Some(t(locale, "engine.generationComplete")),
);
Ok(format!(
"written={}, skipped={}, output={}",
report.written_paths.len(),
report.skipped_paths.len(),
output_dir.display(),
))
},
)
}
Message::RunMetadataDiff => { Message::RunMetadataDiff => {
self.open_singleton_tab(TabType::MetadataDiff, "tabBar.metadataDiff"); self.open_singleton_tab(TabType::MetadataDiff, "tabBar.metadataDiff");
self.start_metadata_diff() self.start_metadata_diff()
@@ -349,6 +247,125 @@ impl BdsApp {
self.refresh_task_snapshots(); self.refresh_task_snapshots();
sidebar_task sidebar_task
} }
Message::SiteGenerationSectionDone {
group_id,
task_id,
result,
} => {
if self.task_manager.status(task_id) == Some(TaskStatus::Cancelled) {
self.refresh_task_snapshots();
return Task::none();
}
match result {
Ok(report) => {
self.task_manager.complete(task_id);
if let Some(workflow) = self.site_generation_workflows.get_mut(&group_id) {
workflow.report.append(report);
}
}
Err(error) => {
self.task_manager.fail(task_id, error.clone());
self.task_manager.cancel_group(&group_id);
if let Some(workflow) = self.site_generation_workflows.remove(&group_id)
&& workflow.kind == SiteGenerationKind::Validation
{
self.site_validation_state.is_applying = false;
self.site_validation_state.error_message = Some(error.clone());
}
let message = tw(
self.ui_locale,
"common.operationFailed",
&[
("operation", &t(self.ui_locale, "engine.renderSiteGroup")),
("error", &error),
],
);
self.notify(ToastLevel::Error, &message);
self.refresh_task_snapshots();
return Task::none();
}
}
let should_index =
self.site_generation_workflows
.get(&group_id)
.is_some_and(|workflow| {
workflow.index_task_id.is_none()
&& workflow.render_task_ids.iter().all(|task_id| {
self.task_manager.status(*task_id)
== Some(TaskStatus::Completed)
})
});
self.refresh_task_snapshots();
if should_index {
self.queue_site_search_index(&group_id)
} else {
Task::none()
}
}
Message::SiteGenerationIndexDone {
group_id,
task_id,
result,
} => {
if self.task_manager.status(task_id) == Some(TaskStatus::Cancelled) {
self.refresh_task_snapshots();
return Task::none();
}
match result {
Ok(report) => {
self.task_manager.complete(task_id);
let Some(mut workflow) = self.site_generation_workflows.remove(&group_id)
else {
self.refresh_task_snapshots();
return Task::none();
};
workflow.report.append(report);
let message = tw(
self.ui_locale,
"engine.generationSummary",
&[
("written", &workflow.report.written_paths.len().to_string()),
("skipped", &workflow.report.skipped_paths.len().to_string()),
("deleted", &workflow.report.deleted_paths.len().to_string()),
(
"output",
&workflow.data_dir.join("html").display().to_string(),
),
],
);
self.notify(ToastLevel::Success, &message);
self.refresh_task_snapshots();
if workflow.kind == SiteGenerationKind::Validation {
self.site_validation_state.is_applying = false;
self.site_validation_state.error_message = None;
self.start_site_validation()
} else {
Task::none()
}
}
Err(error) => {
self.task_manager.fail(task_id, error.clone());
if let Some(workflow) = self.site_generation_workflows.remove(&group_id)
&& workflow.kind == SiteGenerationKind::Validation
{
self.site_validation_state.is_applying = false;
self.site_validation_state.error_message = Some(error.clone());
}
let message = tw(
self.ui_locale,
"common.operationFailed",
&[
("operation", &t(self.ui_locale, "engine.buildSearchIndex")),
("error", &error),
],
);
self.notify(ToastLevel::Error, &message);
self.refresh_task_snapshots();
Task::none()
}
}
}
Message::SiteValidationLoaded(result) => { Message::SiteValidationLoaded(result) => {
self.site_validation_state.is_running = false; self.site_validation_state.is_running = false;
self.site_validation_state.has_run = true; self.site_validation_state.has_run = true;
@@ -391,21 +408,6 @@ impl BdsApp {
} }
Task::none() Task::none()
} }
Message::SiteValidationApplied(result) => {
self.site_validation_state.is_applying = false;
match result {
Ok(detail) => {
self.site_validation_state.error_message = None;
self.notify(ToastLevel::Success, &detail);
self.start_site_validation()
}
Err(error) => {
self.site_validation_state.error_message = Some(error.clone());
self.notify(ToastLevel::Error, &error);
Task::none()
}
}
}
_ => unreachable!("non-engine message routed to engine handler"), _ => unreachable!("non-engine message routed to engine handler"),
} }
} }

View File

@@ -32,6 +32,218 @@ impl BdsApp {
}) })
.collect(); .collect();
} }
pub(super) fn queue_site_generation(
&mut self,
validation: Option<engine::validate_site::SiteValidationReport>,
) -> Task<Message> {
let kind = if validation.is_some() {
SiteGenerationKind::Validation
} else {
SiteGenerationKind::Full
};
let (Some(project), Some(data_dir)) = (&self.active_project, &self.data_dir) else {
if kind == SiteGenerationKind::Validation {
self.site_validation_state.is_applying = false;
}
self.notify(
ToastLevel::Error,
&t(self.ui_locale, "engine.generateSiteNoProject"),
);
return Task::none();
};
let metadata = match engine::meta::read_project_json(data_dir) {
Ok(metadata) => metadata,
Err(error) => {
if kind == SiteGenerationKind::Validation {
self.site_validation_state.is_applying = false;
self.site_validation_state.error_message = Some(error.to_string());
}
self.notify(ToastLevel::Error, &error.to_string());
return Task::none();
}
};
if metadata
.public_url
.as_deref()
.unwrap_or("")
.trim()
.is_empty()
{
if kind == SiteGenerationKind::Validation {
self.site_validation_state.is_applying = false;
}
self.notify(
ToastLevel::Error,
&t(self.ui_locale, "engine.publicUrlRequired"),
);
return Task::none();
}
let sections = validation.as_ref().map_or_else(
|| engine::generation::GenerationSection::ALL.to_vec(),
engine::generation::sections_from_validation_report,
);
if sections.is_empty() {
return Task::none();
}
let project_id = project.id.clone();
let db_path = self.db_path.clone();
let data_dir = data_dir.clone();
let group_id = format!("site-generation:{}", Uuid::new_v4());
let group_name = t(
self.ui_locale,
if kind == SiteGenerationKind::Full {
"engine.renderSiteGroup"
} else {
"engine.applyValidationGroup"
},
);
let mut render_task_ids = Vec::new();
let mut tasks = Vec::new();
for section in sections {
let label = t(self.ui_locale, generation_section_label_key(section));
self.add_output(&label);
let task_id = self
.task_manager
.submit_grouped(&label, &group_id, &group_name);
render_task_ids.push(task_id);
let task_manager = Arc::clone(&self.task_manager);
let task_db_path = db_path.clone();
let task_project_id = project_id.clone();
let task_data_dir = data_dir.clone();
let task_group_id = group_id.clone();
let task_validation = validation.clone();
let locale = self.ui_locale;
tasks.push(Task::perform(
async move {
tokio::task::spawn_blocking(move || {
run_site_generation_section(
task_db_path,
task_project_id,
task_data_dir,
task_manager,
task_id,
section,
task_validation,
locale,
)
})
.await
.unwrap_or_else(|error| Err(format!("task panicked: {error}")))
},
move |result| Message::SiteGenerationSectionDone {
group_id: task_group_id.clone(),
task_id,
result,
},
));
}
self.site_generation_workflows.insert(
group_id,
SiteGenerationWorkflow {
kind,
db_path,
project_id,
data_dir,
group_name,
render_task_ids,
index_task_id: None,
report: engine::generation::GenerationReport::default(),
},
);
self.refresh_task_snapshots();
Task::batch(tasks)
}
pub(super) fn queue_site_search_index(&mut self, group_id: &str) -> Task<Message> {
let Some(workflow) = self.site_generation_workflows.get(group_id).cloned() else {
return Task::none();
};
let label = t(self.ui_locale, "engine.buildSearchIndex");
self.add_output(&label);
let task_id = self
.task_manager
.submit_grouped(&label, group_id, &workflow.group_name);
if let Some(workflow) = self.site_generation_workflows.get_mut(group_id) {
workflow.index_task_id = Some(task_id);
}
self.refresh_task_snapshots();
let locale = self.ui_locale;
let task_manager = Arc::clone(&self.task_manager);
let task_group_id = group_id.to_string();
Task::perform(
async move {
tokio::task::spawn_blocking(move || {
if !task_manager.wait_until_runnable(task_id) {
return Err("cancelled".to_string());
}
let db =
Database::open(&workflow.db_path).map_err(|error| error.to_string())?;
let metadata = engine::meta::read_project_json(&workflow.data_dir)
.map_err(|error| error.to_string())?;
let output_dir = workflow.data_dir.join("html");
let progress_manager = Arc::clone(&task_manager);
let cancel_manager = Arc::clone(&task_manager);
engine::generation::build_site_search_index_with_progress(
db.conn(),
&output_dir,
&workflow.project_id,
&metadata,
move |current, total, path| {
let progress = if total == 0 {
1.0
} else {
current as f32 / total as f32
};
progress_manager.report_progress(
task_id,
Some(progress),
Some(tw(
locale,
"engine.builtSearchFile",
&[
("path", path),
("current", &current.to_string()),
("total", &total.to_string()),
],
)),
);
},
move || cancel_manager.is_cancelled(task_id),
)
.map_err(|error| error.to_string())
})
.await
.unwrap_or_else(|error| Err(format!("task panicked: {error}")))
},
move |result| Message::SiteGenerationIndexDone {
group_id: task_group_id.clone(),
task_id,
result,
},
)
}
pub(super) fn cancel_site_generation_task(&mut self, task_id: TaskId) -> bool {
let Some(group_id) = self.task_manager.group_id(task_id) else {
return false;
};
let Some(workflow) = self.site_generation_workflows.remove(&group_id) else {
return false;
};
self.task_manager.cancel_group(&group_id);
if workflow.kind == SiteGenerationKind::Validation {
self.site_validation_state.is_applying = false;
}
self.add_output(&t(self.ui_locale, "engine.generationCancelled"));
self.refresh_task_snapshots();
true
}
/// Rebuild the shared search index while the modal blocks editor writes. /// Rebuild the shared search index while the modal blocks editor writes.
pub(super) fn start_search_index_rebuild(&mut self) -> Task<Message> { pub(super) fn start_search_index_rebuild(&mut self) -> Task<Message> {
if self.db.is_none() || self.search_index_rebuild_running { if self.db.is_none() || self.search_index_rebuild_running {
@@ -185,3 +397,104 @@ impl BdsApp {
) )
} }
} }
fn generation_section_label_key(section: engine::generation::GenerationSection) -> &'static str {
match section {
engine::generation::GenerationSection::Core => "engine.renderSiteCore",
engine::generation::GenerationSection::Single => "engine.renderSinglePosts",
engine::generation::GenerationSection::Category => "engine.renderCategoryArchives",
engine::generation::GenerationSection::Tag => "engine.renderTagArchives",
engine::generation::GenerationSection::Date => "engine.renderDateArchives",
}
}
#[expect(
clippy::too_many_arguments,
reason = "task input is captured generation context"
)]
fn run_site_generation_section(
db_path: PathBuf,
project_id: String,
data_dir: PathBuf,
task_manager: Arc<TaskManager>,
task_id: TaskId,
section: engine::generation::GenerationSection,
validation: Option<engine::validate_site::SiteValidationReport>,
locale: UiLocale,
) -> Result<engine::generation::GenerationReport, String> {
if !task_manager.wait_until_runnable(task_id) {
return Err("cancelled".to_string());
}
let db = Database::open(&db_path).map_err(|error| error.to_string())?;
let metadata = engine::meta::read_project_json(&data_dir).map_err(|error| error.to_string())?;
let posts = bds_core::db::queries::post::list_posts_by_project(db.conn(), &project_id)
.map_err(|error| error.to_string())?;
let mut sources = Vec::new();
for post in posts
.into_iter()
.filter(engine::generation::has_published_snapshot)
{
if task_manager.is_cancelled(task_id) {
return Err("cancelled".to_string());
}
if let Some(source) = engine::generation::load_published_post_source(&data_dir, post)
.map_err(|error| error.to_string())?
{
sources.push(source);
}
}
let output_dir = data_dir.join("html");
std::fs::create_dir_all(&output_dir).map_err(|error| error.to_string())?;
let progress_manager = Arc::clone(&task_manager);
let cancel_manager = Arc::clone(&task_manager);
let progress_key = if validation.is_some() {
"engine.rewrotePage"
} else {
"engine.generatedPage"
};
let on_page = move |current: usize, total: usize, url: &str| {
let progress = if total == 0 {
1.0
} else {
current as f32 / total as f32
};
progress_manager.report_progress(
task_id,
Some(progress),
Some(tw(
locale,
progress_key,
&[
("url", url),
("current", &current.to_string()),
("total", &total.to_string()),
],
)),
);
};
let is_cancelled = move || cancel_manager.is_cancelled(task_id);
match validation {
Some(validation) => engine::generation::apply_validation_section_with_progress(
db.conn(),
&output_dir,
&project_id,
&metadata,
&sources,
&validation,
section,
on_page,
is_cancelled,
),
None => engine::generation::render_site_section_with_progress(
db.conn(),
&output_dir,
&project_id,
&metadata,
&sources,
section,
on_page,
is_cancelled,
),
}
.map_err(|error| error.to_string())
}

View File

@@ -465,6 +465,19 @@ blogmark-importing = Blogmark wird importiert
blogmark-imported = Blogmark importiert blogmark-imported = Blogmark importiert
blogmark-failed = Blogmark-Import fehlgeschlagen: { $error } blogmark-failed = Blogmark-Import fehlgeschlagen: { $error }
engine-generatedPage = Erstellt { $url } ({ $current }/{ $total }) engine-generatedPage = Erstellt { $url } ({ $current }/{ $total })
engine-rewrotePage = Neu geschrieben { $url } ({ $current }/{ $total })
engine-renderSiteGroup = Website rendern
engine-applyValidationGroup = Website-Validierung anwenden
engine-renderSiteCore = Website-Kern rendern
engine-renderSinglePosts = Einzelbeiträge rendern
engine-renderCategoryArchives = Kategoriearchive rendern
engine-renderTagArchives = Schlagwortarchive rendern
engine-renderDateArchives = Datumsarchive rendern
engine-buildSearchIndex = Suchindex erstellen
engine-builtSearchFile = { $path } erstellt ({ $current }/{ $total })
engine-generationCancelled = Website-Erstellung abgebrochen
engine-generationSummary = Website-Erstellung abgeschlossen: geschrieben={ $written }, übersprungen={ $skipped }, gelöscht={ $deleted }, Ausgabe={ $output }
engine-publicUrlRequired = Vor der Website-Erstellung ist eine öffentliche URL erforderlich
settings-imageImportConcurrency = Gleichzeitige Bildimporte (18) settings-imageImportConcurrency = Gleichzeitige Bildimporte (18)
settings-imageImportConcurrencyInvalid = Die Anzahl gleichzeitiger Bildimporte muss zwischen 1 und 8 liegen settings-imageImportConcurrencyInvalid = Die Anzahl gleichzeitiger Bildimporte muss zwischen 1 und 8 liegen
find-titleFind = Suchen find-titleFind = Suchen

View File

@@ -465,6 +465,19 @@ blogmark-importing = Importing blogmark
blogmark-imported = Blogmark imported blogmark-imported = Blogmark imported
blogmark-failed = Blogmark import failed: { $error } blogmark-failed = Blogmark import failed: { $error }
engine-generatedPage = Generated { $url } ({ $current }/{ $total }) engine-generatedPage = Generated { $url } ({ $current }/{ $total })
engine-rewrotePage = Rewrote { $url } ({ $current }/{ $total })
engine-renderSiteGroup = Render Site
engine-applyValidationGroup = Apply Site Validation
engine-renderSiteCore = Render Site Core
engine-renderSinglePosts = Render Single Posts
engine-renderCategoryArchives = Render Category Archives
engine-renderTagArchives = Render Tag Archives
engine-renderDateArchives = Render Date Archives
engine-buildSearchIndex = Build Search Index
engine-builtSearchFile = Built { $path } ({ $current }/{ $total })
engine-generationCancelled = Site generation cancelled
engine-generationSummary = Site generation complete: written={ $written }, skipped={ $skipped }, deleted={ $deleted }, output={ $output }
engine-publicUrlRequired = Public URL is required before generating the site
settings-imageImportConcurrency = Image Import Concurrency (18) settings-imageImportConcurrency = Image Import Concurrency (18)
settings-imageImportConcurrencyInvalid = Image import concurrency must be a number from 1 to 8 settings-imageImportConcurrencyInvalid = Image import concurrency must be a number from 1 to 8
find-titleFind = Find find-titleFind = Find

View File

@@ -465,6 +465,19 @@ blogmark-importing = Importando blogmark
blogmark-imported = Blogmark importado blogmark-imported = Blogmark importado
blogmark-failed = Error al importar el blogmark: { $error } blogmark-failed = Error al importar el blogmark: { $error }
engine-generatedPage = Generado { $url } ({ $current }/{ $total }) engine-generatedPage = Generado { $url } ({ $current }/{ $total })
engine-rewrotePage = Reescrito { $url } ({ $current }/{ $total })
engine-renderSiteGroup = Renderizar sitio
engine-applyValidationGroup = Aplicar validación del sitio
engine-renderSiteCore = Renderizar núcleo del sitio
engine-renderSinglePosts = Renderizar entradas individuales
engine-renderCategoryArchives = Renderizar archivos de categorías
engine-renderTagArchives = Renderizar archivos de etiquetas
engine-renderDateArchives = Renderizar archivos por fecha
engine-buildSearchIndex = Crear índice de búsqueda
engine-builtSearchFile = Creado { $path } ({ $current }/{ $total })
engine-generationCancelled = Generación del sitio cancelada
engine-generationSummary = Generación completada: escritos={ $written }, omitidos={ $skipped }, eliminados={ $deleted }, salida={ $output }
engine-publicUrlRequired = Se requiere una URL pública antes de generar el sitio
settings-imageImportConcurrency = Importaciones de imágenes simultáneas (18) settings-imageImportConcurrency = Importaciones de imágenes simultáneas (18)
settings-imageImportConcurrencyInvalid = El número de importaciones simultáneas debe estar entre 1 y 8 settings-imageImportConcurrencyInvalid = El número de importaciones simultáneas debe estar entre 1 y 8
find-titleFind = Buscar find-titleFind = Buscar

View File

@@ -465,6 +465,19 @@ blogmark-importing = Importation du blogmark
blogmark-imported = Blogmark importé blogmark-imported = Blogmark importé
blogmark-failed = Échec de limport du blogmark : { $error } blogmark-failed = Échec de limport du blogmark : { $error }
engine-generatedPage = Généré { $url } ({ $current }/{ $total }) engine-generatedPage = Généré { $url } ({ $current }/{ $total })
engine-rewrotePage = Réécrit { $url } ({ $current }/{ $total })
engine-renderSiteGroup = Générer le site
engine-applyValidationGroup = Appliquer la validation du site
engine-renderSiteCore = Générer le cœur du site
engine-renderSinglePosts = Générer les articles individuels
engine-renderCategoryArchives = Générer les archives de catégories
engine-renderTagArchives = Générer les archives de mots-clés
engine-renderDateArchives = Générer les archives chronologiques
engine-buildSearchIndex = Construire lindex de recherche
engine-builtSearchFile = Construit { $path } ({ $current }/{ $total })
engine-generationCancelled = Génération du site annulée
engine-generationSummary = Génération terminée : écrits={ $written }, ignorés={ $skipped }, supprimés={ $deleted }, sortie={ $output }
engine-publicUrlRequired = Une URL publique est requise avant de générer le site
settings-imageImportConcurrency = Imports dimages simultanés (18) settings-imageImportConcurrency = Imports dimages simultanés (18)
settings-imageImportConcurrencyInvalid = Le nombre dimports simultanés doit être compris entre 1 et 8 settings-imageImportConcurrencyInvalid = Le nombre dimports simultanés doit être compris entre 1 et 8
find-titleFind = Rechercher find-titleFind = Rechercher

View File

@@ -465,6 +465,19 @@ blogmark-importing = Importazione del blogmark
blogmark-imported = Blogmark importato blogmark-imported = Blogmark importato
blogmark-failed = Importazione del blogmark non riuscita: { $error } blogmark-failed = Importazione del blogmark non riuscita: { $error }
engine-generatedPage = Generato { $url } ({ $current }/{ $total }) engine-generatedPage = Generato { $url } ({ $current }/{ $total })
engine-rewrotePage = Riscritto { $url } ({ $current }/{ $total })
engine-renderSiteGroup = Genera sito
engine-applyValidationGroup = Applica convalida del sito
engine-renderSiteCore = Genera nucleo del sito
engine-renderSinglePosts = Genera articoli singoli
engine-renderCategoryArchives = Genera archivi delle categorie
engine-renderTagArchives = Genera archivi dei tag
engine-renderDateArchives = Genera archivi per data
engine-buildSearchIndex = Crea indice di ricerca
engine-builtSearchFile = Creato { $path } ({ $current }/{ $total })
engine-generationCancelled = Generazione del sito annullata
engine-generationSummary = Generazione completata: scritti={ $written }, ignorati={ $skipped }, eliminati={ $deleted }, output={ $output }
engine-publicUrlRequired = È richiesto un URL pubblico prima di generare il sito
settings-imageImportConcurrency = Importazioni immagini simultanee (18) settings-imageImportConcurrency = Importazioni immagini simultanee (18)
settings-imageImportConcurrencyInvalid = Il numero di importazioni simultanee deve essere compreso tra 1 e 8 settings-imageImportConcurrencyInvalid = Il numero di importazioni simultanee deve essere compreso tra 1 e 8
find-titleFind = Trova find-titleFind = Trova