hugo/RuDS
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

Group site generation by section

Browse Source
This commit is contained in:
Chili Palmer
2026-07-19 10:27:12 +02:00
parent 055dd5efc5
commit 90a9002124
19 changed files with 1467 additions and 399 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
crates/bds-core/src/engine/gallery_import.rs
View File

@@ -70,10 +70,6 @@ where
})
}
#[expect(
clippy::too_many_arguments,
reason = "the shared gallery workflow needs its persisted post and project context"
)]
pub fn import_gallery_images(
db_path: &Path,
data_dir: &Path,

579
crates/bds-core/src/engine/generation.rs
View File

@@ -1,4 +1,4 @@
use std::collections::{HashMap, HashSet};
use std::collections::{BTreeMap, HashMap, HashSet};
use std::path::Path;
use crate::db::DbConnection as Connection;
@@ -14,7 +14,8 @@ use crate::engine::{EngineError, EngineResult};
use crate::model::{CategorySettings, Post, ProjectMetadata};
use crate::render::{
GeneratedWriteOutcome, build_calendar_json, build_canonical_post_path,
build_site_render_artifacts, render_markdown_to_html, write_generated_bytes,
build_site_render_artifacts, build_site_section_render_artifacts,
build_targeted_site_section_render_artifacts, render_markdown_to_html, write_generated_bytes,
write_generated_file,
};
@@ -65,6 +66,24 @@ pub enum GenerationSection {
Date,
}
impl GenerationSection {
pub const ALL: [Self; 5] = [
Self::Core,
Self::Single,
Self::Category,
Self::Tag,
Self::Date,
];
}
impl GenerationReport {
pub fn append(&mut self, mut other: Self) {
self.written_paths.append(&mut other.written_paths);
self.skipped_paths.append(&mut other.skipped_paths);
self.deleted_paths.append(&mut other.deleted_paths);
}
}
pub fn generate_starter_site(
conn: &Connection,
output_dir: &Path,
@@ -94,6 +113,41 @@ pub fn generate_starter_site_with_progress(
mut on_page: impl FnMut(usize, usize, &str),
) -> EngineResult<GenerationReport> {
let mut report = GenerationReport::default();
for section in GenerationSection::ALL {
report.append(render_site_section_with_progress(
conn,
output_dir,
project_id,
metadata,
posts,
section,
&mut on_page,
|| false,
)?);
}
report.append(build_site_search_index(
conn, output_dir, project_id, metadata,
)?);
Ok(report)
}
#[expect(
clippy::too_many_arguments,
reason = "section rendering uses the existing generation context and two callbacks"
)]
pub fn render_site_section_with_progress(
conn: &Connection,
output_dir: &Path,
project_id: &str,
metadata: &ProjectMetadata,
posts: &[PublishedPostSource],
section: GenerationSection,
mut on_page: impl FnMut(usize, usize, &str),
mut is_cancelled: impl FnMut() -> bool,
) -> EngineResult<GenerationReport> {
if is_cancelled() {
return Err(EngineError::Validation("cancelled".to_string()));
}
let data_dir = project_data_dir(output_dir);
let category_settings = load_category_settings(&data_dir);
let list_posts = filter_posts_for_lists(posts, &category_settings);
@@ -101,12 +155,21 @@ pub fn generate_starter_site_with_progress(
.iter()
.map(|source| (source.post.clone(), source.body_markdown.clone()))
.collect::<Vec<_>>();
let artifacts =
build_site_render_artifacts(conn, &data_dir, project_id, metadata, &input_posts)
.map_err(|error| EngineError::Parse(error.to_string()))?;
let artifacts = build_site_section_render_artifacts(
conn,
&data_dir,
project_id,
metadata,
&input_posts,
section,
)
.map_err(|error| EngineError::Parse(error.to_string()))?;
let mut report = GenerationReport::default();
let total_pages = artifacts.pages.len();
for (index, page) in artifacts.pages.iter().enumerate() {
if is_cancelled() {
return Err(EngineError::Validation("cancelled".to_string()));
}
write_out(
conn,
output_dir,
@@ -118,78 +181,20 @@ pub fn generate_starter_site_with_progress(
on_page(index + 1, total_pages, &page.url_path);
}
write_bundled_site_assets(conn, output_dir, project_id, &mut report)?;
write_out(
conn,
output_dir,
project_id,
"calendar.json",
&build_calendar_json(
&list_posts
.iter()
.map(|source| source.post.clone())
.collect::<Vec<_>>(),
)?,
&mut report,
)?;
for render_language in render_languages(metadata) {
let localized_posts =
localized_sources(conn, &data_dir, &list_posts, &render_language, metadata)?;
let prefix = if render_language
== metadata
.main_language
.clone()
.unwrap_or_else(|| "en".to_string())
{
String::new()
} else {
format!("{}/", render_language)
};
let rss = build_rss_xml(metadata, &localized_posts, &render_language);
if prefix.is_empty() {
write_out(conn, output_dir, project_id, "rss.xml", &rss, &mut report)?;
}
write_out(
if section == GenerationSection::Core {
write_core_outputs(
conn,
output_dir,
project_id,
&format!("{prefix}feed.xml"),
&rss,
&mut report,
)?;
write_out(
conn,
output_dir,
project_id,
&format!("{prefix}atom.xml"),
&build_atom_xml(metadata, &localized_posts, &render_language),
&mut report,
)?;
write_out(
conn,
output_dir,
project_id,
&format!("{prefix}sitemap.xml"),
&build_sitemap_xml(
metadata,
&artifacts.pages,
&localized_posts,
&render_language,
),
metadata,
&data_dir,
&list_posts,
&artifacts.route_manifest,
None,
&mut report,
&mut is_cancelled,
)?;
}
write_pagefind_indexes(
conn,
output_dir,
project_id,
&artifacts.pagefind_documents,
&mut report,
)?;
Ok(report)
}
@@ -236,8 +241,6 @@ pub fn apply_validation_sections(
let section_set = sections.iter().copied().collect::<HashSet<_>>();
let data_dir = project_data_dir(output_dir);
let category_settings = load_category_settings(&data_dir);
let list_posts = filter_posts_for_lists(posts, &category_settings);
let input_posts = posts
.iter()
.map(|source| (source.post.clone(), source.body_markdown.clone()))
@@ -248,96 +251,190 @@ pub fn apply_validation_sections(
let mut report = GenerationReport::default();
let expected_paths = expected_paths_for_sections(metadata, &artifacts.pages, &section_set);
for page in &artifacts.pages {
if path_matches_sections(&page.relative_path, &section_set) {
write_out(
conn,
output_dir,
project_id,
&page.relative_path,
&page.html,
&mut report,
)?;
}
for section in sections {
report.append(render_site_section_with_progress(
conn,
output_dir,
project_id,
metadata,
posts,
*section,
|_current, _total, _url| {},
|| false,
)?);
}
if section_set.contains(&GenerationSection::Core) {
write_bundled_site_assets(conn, output_dir, project_id, &mut report)?;
remove_extra_section_paths(output_dir, &expected_paths, &section_set, &mut report)?;
report.append(build_site_search_index(
conn, output_dir, project_id, metadata,
)?);
Ok(report)
}
#[expect(
clippy::too_many_arguments,
reason = "targeted apply adds its validation report to the existing generation context"
)]
pub fn apply_validation_section_with_progress(
conn: &Connection,
output_dir: &Path,
project_id: &str,
metadata: &ProjectMetadata,
posts: &[PublishedPostSource],
validation: &SiteValidationReport,
section: GenerationSection,
mut on_page: impl FnMut(usize, usize, &str),
mut is_cancelled: impl FnMut() -> bool,
) -> EngineResult<GenerationReport> {
if is_cancelled() {
return Err(EngineError::Validation("cancelled".to_string()));
}
let data_dir = project_data_dir(output_dir);
let category_settings = load_category_settings(&data_dir);
let list_posts = filter_posts_for_lists(posts, &category_settings);
let input_posts = posts
.iter()
.map(|source| (source.post.clone(), source.body_markdown.clone()))
.collect::<Vec<_>>();
let requested = validation
.missing_pages
.iter()
.chain(validation.stale_pages.iter())
.cloned()
.collect::<HashSet<_>>();
let fallback = validation
.missing_pages
.iter()
.chain(validation.extra_pages.iter())
.chain(validation.stale_pages.iter())
.any(|path| classify_generated_path(path).is_none());
let artifacts = if fallback {
build_site_section_render_artifacts(
conn,
&data_dir,
project_id,
metadata,
&input_posts,
section,
)
} else {
build_targeted_site_section_render_artifacts(
conn,
&data_dir,
project_id,
metadata,
&input_posts,
section,
&requested,
)
}
.map_err(|error| EngineError::Parse(error.to_string()))?;
let mut report = GenerationReport::default();
let total_pages = artifacts.pages.len();
for (index, page) in artifacts.pages.iter().enumerate() {
if is_cancelled() {
return Err(EngineError::Validation("cancelled".to_string()));
}
write_out(
conn,
output_dir,
project_id,
"calendar.json",
&build_calendar_json(
&list_posts
.iter()
.map(|source| source.post.clone())
.collect::<Vec<_>>(),
)?,
&page.relative_path,
&page.html,
&mut report,
)?;
for render_language in render_languages(metadata) {
let localized_posts =
localized_sources(conn, &data_dir, &list_posts, &render_language, metadata)?;
let prefix = if render_language
== metadata
.main_language
.clone()
.unwrap_or_else(|| "en".to_string())
{
String::new()
} else {
format!("{}/", render_language)
};
let rss = build_rss_xml(metadata, &localized_posts, &render_language);
if prefix.is_empty() {
write_out(conn, output_dir, project_id, "rss.xml", &rss, &mut report)?;
}
write_out(
conn,
output_dir,
project_id,
&format!("{prefix}feed.xml"),
&rss,
&mut report,
)?;
write_out(
conn,
output_dir,
project_id,
&format!("{prefix}atom.xml"),
&build_atom_xml(metadata, &localized_posts, &render_language),
&mut report,
)?;
write_out(
conn,
output_dir,
project_id,
&format!("{prefix}sitemap.xml"),
&build_sitemap_xml(
metadata,
&artifacts.pages,
&localized_posts,
&render_language,
),
&mut report,
)?;
}
on_page(index + 1, total_pages, &page.url_path);
}
remove_extra_section_paths(output_dir, &expected_paths, &section_set, &mut report)?;
write_pagefind_indexes(
conn,
output_dir,
project_id,
&artifacts.pagefind_documents,
&mut report,
)?;
if section == GenerationSection::Core {
write_core_outputs(
conn,
output_dir,
project_id,
metadata,
&data_dir,
&list_posts,
&artifacts.route_manifest,
(!fallback).then_some(&requested),
&mut report,
&mut is_cancelled,
)?;
}
for path in &validation.extra_pages {
if is_cancelled() {
return Err(EngineError::Validation("cancelled".to_string()));
}
let owned_by_section = classify_generated_path(path)
.map_or(section == GenerationSection::Core, |owner| owner == section);
if owned_by_section && output_dir.join(path).is_file() {
std::fs::remove_file(output_dir.join(path)).map_err(EngineError::Io)?;
report.deleted_paths.push(path.clone());
}
}
Ok(report)
}
#[expect(
clippy::too_many_arguments,
reason = "generation context is existing domain data"
)]
fn write_core_outputs(
conn: &Connection,
output_dir: &Path,
project_id: &str,
metadata: &ProjectMetadata,
data_dir: &Path,
list_posts: &[PublishedPostSource],
route_manifest: &[crate::render::SitePage],
requested: Option<&HashSet<String>>,
report: &mut GenerationReport,
is_cancelled: &mut impl FnMut() -> bool,
) -> EngineResult<()> {
if requested.is_none() {
write_bundled_site_assets(conn, output_dir, project_id, report)?;
}
let mut outputs = vec![(
"calendar.json".to_string(),
build_calendar_json(
&list_posts
.iter()
.map(|source| source.post.clone())
.collect::<Vec<_>>(),
)?,
)];
for render_language in render_languages(metadata) {
let localized_posts =
localized_sources(conn, data_dir, list_posts, &render_language, metadata)?;
let prefix = if render_language == metadata.main_language.as_deref().unwrap_or("en") {
String::new()
} else {
format!("{render_language}/")
};
let rss = build_rss_xml(metadata, &localized_posts, &render_language);
outputs.push((format!("{prefix}rss.xml"), rss.clone()));
outputs.push((format!("{prefix}feed.xml"), rss));
outputs.push((
format!("{prefix}atom.xml"),
build_atom_xml(metadata, &localized_posts, &render_language),
));
outputs.push((
format!("{prefix}sitemap.xml"),
build_sitemap_xml(metadata, route_manifest, &localized_posts, &render_language),
));
}
for (path, content) in outputs {
if is_cancelled() {
return Err(EngineError::Validation("cancelled".to_string()));
}
if requested.is_none_or(|requested| requested.contains(&path)) {
write_out(conn, output_dir, project_id, &path, &content, report)?;
}
}
Ok(())
}
fn write_out(
conn: &Connection,
output_dir: &Path,
@@ -357,35 +454,100 @@ fn write_out(
Ok(())
}
fn write_pagefind_indexes(
pub fn build_site_search_index(
conn: &Connection,
output_dir: &Path,
project_id: &str,
documents: &[crate::render::PagefindDocument],
report: &mut GenerationReport,
) -> EngineResult<()> {
metadata: &ProjectMetadata,
) -> EngineResult<GenerationReport> {
build_site_search_index_with_progress(
conn,
output_dir,
project_id,
metadata,
|_current, _total, _path| {},
|| false,
)
}
pub fn build_site_search_index_with_progress(
conn: &Connection,
output_dir: &Path,
project_id: &str,
metadata: &ProjectMetadata,
mut on_file: impl FnMut(usize, usize, &str),
mut is_cancelled: impl FnMut() -> bool,
) -> EngineResult<GenerationReport> {
let mut documents = Vec::new();
if output_dir.exists() {
for entry in WalkDir::new(output_dir).into_iter().filter_map(Result::ok) {
if !entry.file_type().is_file() {
continue;
}
let relative_path = entry
.path()
.strip_prefix(output_dir)
.unwrap_or(entry.path())
.to_string_lossy()
.replace('\\', "/");
if !relative_path.ends_with(".html")
|| relative_path.starts_with("pagefind/")
|| relative_path.contains("/pagefind/")
{
continue;
}
let language = render_languages(metadata)
.into_iter()
.find(|language| relative_path.starts_with(&format!("{language}/")))
.unwrap_or_else(|| {
metadata
.main_language
.clone()
.unwrap_or_else(|| "en".into())
});
documents.push(crate::render::PagefindDocument {
language,
url_path: String::new(),
html: std::fs::read_to_string(entry.path()).map_err(EngineError::Io)?,
relative_path,
});
}
}
documents.sort_by(|left, right| left.relative_path.cmp(&right.relative_path));
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.map_err(EngineError::Io)?;
let grouped = documents.iter().fold(
HashMap::<String, Vec<&crate::render::PagefindDocument>>::new(),
BTreeMap::<String, Vec<&crate::render::PagefindDocument>>::new(),
|mut acc, doc| {
acc.entry(doc.language.clone()).or_default().push(doc);
acc
},
);
let mut outputs = Vec::new();
for (language, docs) in grouped {
if is_cancelled() {
return Err(EngineError::Validation("cancelled".to_string()));
}
let config = PagefindServiceConfig::builder()
.keep_index_url(true)
.force_language(language.clone())
.build();
let mut index = PagefindIndex::new(Some(config))
.map_err(|error| EngineError::Parse(error.to_string()))?;
let output_prefix = if language == metadata.main_language.as_deref().unwrap_or("en") {
"pagefind".to_string()
} else {
format!("{language}/pagefind")
};
runtime.block_on(async {
for doc in docs {
if is_cancelled() {
return Err(EngineError::Validation("cancelled".to_string()));
}
index
.add_html_file(Some(doc.relative_path.clone()), None, doc.html.clone())
.await
@@ -396,23 +558,71 @@ fn write_pagefind_indexes(
.await
.map_err(|error| EngineError::Parse(error.to_string()))?;
for file in files {
let relative = file
.filename
.to_string_lossy()
.trim_start_matches('/')
.to_string();
match write_generated_bytes(conn, output_dir, project_id, &relative, &file.contents)
.map_err(|error| EngineError::Parse(error.to_string()))?
{
GeneratedWriteOutcome::Written => report.written_paths.push(relative),
GeneratedWriteOutcome::SkippedUnchanged => report.skipped_paths.push(relative),
}
outputs.push((
format!(
"{output_prefix}/{}",
file.filename.to_string_lossy().trim_start_matches('/')
),
file.contents,
));
}
Ok::<(), EngineError>(())
})?;
}
Ok(())
outputs.sort_by(|left, right| {
left.0
.ends_with("pagefind-entry.json")
.cmp(&right.0.ends_with("pagefind-entry.json"))
.then_with(|| left.0.cmp(&right.0))
});
let total = outputs.len();
let mut report = GenerationReport::default();
let expected = outputs
.iter()
.map(|(relative, _)| relative.clone())
.collect::<HashSet<_>>();
for (index, (relative, contents)) in outputs.into_iter().enumerate() {
if is_cancelled() {
return Err(EngineError::Validation("cancelled".to_string()));
}
match write_generated_bytes(conn, output_dir, project_id, &relative, &contents)
.map_err(|error| EngineError::Parse(error.to_string()))?
{
GeneratedWriteOutcome::Written => report.written_paths.push(relative.clone()),
GeneratedWriteOutcome::SkippedUnchanged => report.skipped_paths.push(relative.clone()),
}
on_file(index + 1, total, &relative);
}
for language in render_languages(metadata) {
let prefix = if language == metadata.main_language.as_deref().unwrap_or("en") {
"pagefind".to_string()
} else {
format!("{language}/pagefind")
};
let index_dir = output_dir.join(&prefix);
if !index_dir.exists() {
continue;
}
for entry in WalkDir::new(&index_dir).into_iter().filter_map(Result::ok) {
if is_cancelled() {
return Err(EngineError::Validation("cancelled".to_string()));
}
if !entry.file_type().is_file() {
continue;
}
let relative = entry
.path()
.strip_prefix(output_dir)
.unwrap_or(entry.path())
.to_string_lossy()
.replace('\\', "/");
if !expected.contains(&relative) {
std::fs::remove_file(entry.path()).map_err(EngineError::Io)?;
report.deleted_paths.push(relative);
}
}
}
Ok(report)
}
fn project_data_dir(output_dir: &Path) -> std::path::PathBuf {
@@ -448,6 +658,7 @@ fn expected_paths_for_sections(
} else {
format!("{language}/")
};
expected.insert(format!("{prefix}rss.xml"));
expected.insert(format!("{prefix}feed.xml"));
expected.insert(format!("{prefix}atom.xml"));
expected.insert(format!("{prefix}sitemap.xml"));
@@ -508,7 +719,7 @@ fn path_matches_sections(path: &str, sections: &HashSet<GenerationSection>) -> b
.unwrap_or(false)
}
fn classify_generated_path(path: &str) -> Option<GenerationSection> {
pub(crate) fn classify_generated_path(path: &str) -> Option<GenerationSection> {
if path.ends_with(".xml") || path.ends_with(".json") {
return Some(GenerationSection::Core);
}
@@ -522,13 +733,29 @@ fn classify_generated_path(path: &str) -> Option<GenerationSection> {
}
match parts.as_slice() {
["index.html"] => Some(GenerationSection::Core),
["index.html"] | ["404.html"] | ["page", _, "index.html"] => Some(GenerationSection::Core),
["category", ..] => Some(GenerationSection::Category),
["tag", ..] => Some(GenerationSection::Tag),
[year, "index.html"] if is_year_segment(year) => Some(GenerationSection::Date),
[year, "page", _, "index.html"] if is_year_segment(year) => Some(GenerationSection::Date),
[year, month, "index.html"] if is_year_segment(year) && is_month_segment(month) => {
Some(GenerationSection::Date)
}
[year, month, "page", _, "index.html"]
if is_year_segment(year) && is_month_segment(month) =>
{
Some(GenerationSection::Date)
}
[year, month, day, "index.html"]
if is_year_segment(year) && is_month_segment(month) && is_day_segment(day) =>
{
Some(GenerationSection::Date)
}
[year, month, day, "page", _, "index.html"]
if is_year_segment(year) && is_month_segment(month) && is_day_segment(day) =>
{
Some(GenerationSection::Date)
}
[year, month, day, _slug, "index.html"]
if is_year_segment(year) && is_month_segment(month) && is_day_segment(day) =>
{
@@ -545,6 +772,8 @@ fn has_language_prefix(parts: &[&str]) -> bool {
&& *first != "category"
&& *first != "tag"
&& (*second == "index.html"
|| *second == "404.html"
|| *second == "page"
|| is_year_segment(second)
|| *second == "category"
|| *second == "tag")
@@ -570,13 +799,7 @@ fn matches_generated_extension(path: &str) -> bool {
}
fn all_sections() -> Vec<GenerationSection> {
vec![
GenerationSection::Core,
GenerationSection::Single,
GenerationSection::Category,
GenerationSection::Tag,
GenerationSection::Date,
]
GenerationSection::ALL.to_vec()
}
fn section_sort_key(section: &GenerationSection) -> u8 {

59
crates/bds-core/src/engine/task.rs
View File

@@ -169,6 +169,32 @@ impl TaskManager {
self.state_changed.notify_all();
}
/// Cancel every active task in a group and release their workers.
pub fn cancel_group(&self, group_id: &str) {
let mut tasks = self.tasks.lock().unwrap();
let now = Instant::now();
for entry in tasks.iter_mut().filter(|task| {
task.group_id.as_deref() == Some(group_id)
&& matches!(task.status, TaskStatus::Running | TaskStatus::Pending)
}) {
entry.cancel_flag.store(true, Ordering::Release);
entry.status = TaskStatus::Cancelled;
entry.finished_at = Some(now);
}
Self::promote_next(&mut tasks, self.max_concurrent);
self.state_changed.notify_all();
}
/// Return the group containing a task, if any.
pub fn group_id(&self, task_id: TaskId) -> Option<String> {
self.tasks
.lock()
.unwrap()
.iter()
.find(|task| task.id == task_id)
.and_then(|task| task.group_id.clone())
}
/// Check whether a task has been cancelled.
pub fn is_cancelled(&self, task_id: TaskId) -> bool {
let tasks = self.tasks.lock().unwrap();
@@ -292,14 +318,19 @@ impl TaskManager {
/// Promote the next queued task to running if capacity allows.
fn promote_next(tasks: &mut [TaskEntry], max_concurrent: usize) {
let running = tasks
while tasks
.iter()
.filter(|t| t.status == TaskStatus::Running)
.count();
if running < max_concurrent
&& let Some(t) = tasks.iter_mut().find(|t| t.status == TaskStatus::Pending)
.filter(|task| task.status == TaskStatus::Running)
.count()
< max_concurrent
{
t.status = TaskStatus::Running;
let Some(task) = tasks
.iter_mut()
.find(|task| task.status == TaskStatus::Pending)
else {
break;
};
task.status = TaskStatus::Running;
}
}
}
@@ -483,6 +514,22 @@ mod tests {
assert_eq!(mgr.status(id), Some(TaskStatus::Completed));
}
#[test]
fn cancelling_group_settles_every_active_task() {
let mgr = TaskManager::new(2);
let first = mgr.submit_grouped("first", "generation-1", "Render Site");
let second = mgr.submit_grouped("second", "generation-1", "Render Site");
let third = mgr.submit_grouped("third", "generation-1", "Render Site");
let unrelated = mgr.submit("unrelated");
mgr.cancel_group("generation-1");
assert_eq!(mgr.status(first), Some(TaskStatus::Cancelled));
assert_eq!(mgr.status(second), Some(TaskStatus::Cancelled));
assert_eq!(mgr.status(third), Some(TaskStatus::Cancelled));
assert_ne!(mgr.status(unrelated), Some(TaskStatus::Cancelled));
}
#[test]
fn report_progress_updates_task() {
let mgr = TaskManager::default();

22
crates/bds-core/src/engine/validate_site.rs
View File

@@ -6,7 +6,7 @@ use walkdir::WalkDir;
use crate::db::queries;
use crate::engine::{EngineError, EngineResult};
use crate::model::{Post, PostStatus};
use crate::model::Post;
use crate::render::build_site_render_artifacts;
use crate::util::file_hash;
@@ -35,7 +35,6 @@ pub fn validate_site(
.map(|page| page.relative_path.clone())
.collect::<HashSet<_>>();
expected.insert("calendar.json".to_string());
expected.insert("rss.xml".to_string());
for language in render_languages(&metadata) {
let prefix = if language
== metadata
@@ -47,6 +46,7 @@ pub fn validate_site(
} else {
format!("{language}/")
};
expected.insert(format!("{prefix}rss.xml"));
expected.insert(format!("{prefix}feed.xml"));
expected.insert(format!("{prefix}atom.xml"));
expected.insert(format!("{prefix}sitemap.xml"));
@@ -124,20 +124,12 @@ fn load_published_posts(
let mut published = Vec::new();
for post in posts
.into_iter()
.filter(|post| post.status == PostStatus::Published)
.filter(crate::engine::generation::has_published_snapshot)
{
let body = if let Some(content) = &post.content {
content.clone()
} else if let Some(content) = &post.published_content {
content.clone()
} else {
let raw =
std::fs::read_to_string(data_dir.join(post.file_path.trim_start_matches('/')))?;
crate::util::frontmatter::read_post_file(&raw)
.map(|(_, body)| body)
.map_err(EngineError::Parse)?
};
published.push((post, body));
if let Some(source) = crate::engine::generation::load_published_post_source(data_dir, post)?
{
published.push((source.post, source.body_markdown));
}
}
Ok(published)
}

5
crates/bds-core/src/render/generation.rs
View File

@@ -96,7 +96,9 @@ pub fn write_generated_bytes(
pub fn build_core_generation_paths(main_language: &str, blog_languages: &[String]) -> Vec<String> {
let mut paths = vec![
"index.html".to_string(),
"404.html".to_string(),
"sitemap.xml".to_string(),
"rss.xml".to_string(),
"feed.xml".to_string(),
"atom.xml".to_string(),
"calendar.json".to_string(),
@@ -105,6 +107,9 @@ pub fn build_core_generation_paths(main_language: &str, blog_languages: &[String
for language in blog_languages {
if language != main_language {
paths.push(format!("{language}/index.html"));
paths.push(format!("{language}/404.html"));
paths.push(format!("{language}/sitemap.xml"));
paths.push(format!("{language}/rss.xml"));
paths.push(format!("{language}/feed.xml"));
paths.push(format!("{language}/atom.xml"));
}

3
crates/bds-core/src/render/mod.rs
View File

@@ -20,7 +20,8 @@ pub use routes::{
};
pub use site::{
PagefindDocument, PreviewRenderResult, SitePage, SiteRenderArtifacts, build_preview_response,
build_site_render_artifacts,
build_site_render_artifacts, build_site_section_render_artifacts,
build_targeted_site_section_render_artifacts,
};
pub use template_lookup::{
RenderCategorySettings, RenderTemplateLookup, TemplateLookupError, resolve_post_template,

127
crates/bds-core/src/render/site.rs
View File

@@ -1,4 +1,4 @@
use std::collections::{BTreeMap, HashMap};
use std::collections::{BTreeMap, HashMap, HashSet};
use std::error::Error;
use std::fs;
use std::path::Path;
@@ -10,6 +10,7 @@ use rayon::prelude::*;
use serde_json::{Value, json};
use crate::db::queries;
use crate::engine::generation::{GenerationSection, classify_generated_path};
use crate::engine::menu::{self, MenuItemKind};
use crate::model::{
CategorySettings, Media, Post, ProjectMetadata, ScriptKind, Tag, Template, TemplateKind,
@@ -56,6 +57,7 @@ pub struct PagefindDocument {
pub struct SiteRenderArtifacts {
pub pages: Vec<SitePage>,
pub pagefind_documents: Vec<PagefindDocument>,
pub route_manifest: Vec<SitePage>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
@@ -110,9 +112,56 @@ pub fn build_site_render_artifacts(
metadata,
published_posts,
false,
None,
None,
)
}
pub fn build_site_section_render_artifacts(
conn: &Connection,
data_dir: &Path,
project_id: &str,
metadata: &ProjectMetadata,
published_posts: &[(Post, String)],
section: GenerationSection,
) -> Result<SiteRenderArtifacts, Box<dyn Error + Send + Sync>> {
build_site_render_artifacts_with_mode(
conn,
data_dir,
project_id,
metadata,
published_posts,
false,
Some(section),
None,
)
}
pub fn build_targeted_site_section_render_artifacts(
conn: &Connection,
data_dir: &Path,
project_id: &str,
metadata: &ProjectMetadata,
published_posts: &[(Post, String)],
section: GenerationSection,
requested_paths: &HashSet<String>,
) -> Result<SiteRenderArtifacts, Box<dyn Error + Send + Sync>> {
build_site_render_artifacts_with_mode(
conn,
data_dir,
project_id,
metadata,
published_posts,
false,
Some(section),
Some(requested_paths),
)
}
#[expect(
clippy::too_many_arguments,
reason = "the shared renderer accepts optional section and path filters"
)]
fn build_site_render_artifacts_with_mode(
conn: &Connection,
data_dir: &Path,
@@ -120,6 +169,8 @@ fn build_site_render_artifacts_with_mode(
metadata: &ProjectMetadata,
published_posts: &[(Post, String)],
is_preview: bool,
section: Option<GenerationSection>,
requested_paths: Option<&HashSet<String>>,
) -> Result<SiteRenderArtifacts, Box<dyn Error + Send + Sync>> {
let bundle = load_template_bundle(conn, data_dir, project_id)?;
let main_language = main_language(metadata).to_string();
@@ -144,10 +195,24 @@ fn build_site_render_artifacts_with_mode(
&tags,
&category_settings,
);
artifacts
.route_manifest
.extend(routes.iter().map(|route| SitePage {
language: language.clone(),
relative_path: route.relative_path.clone(),
url_path: route.url_path.clone(),
html: String::new(),
}));
let post_data_json_by_id = build_post_data_json_by_id(&localized_posts);
let menu_items = build_menu_items(data_dir, &language, &main_language)?;
let rendered_list_pages = routes
.par_iter()
.filter(|route| {
section.is_none_or(|section| {
classify_generated_path(&route.relative_path) == Some(section)
}) && requested_paths
.is_none_or(|requested| requested.contains(&route.relative_path))
})
.map(|route| {
render_list_route(
route,
@@ -180,9 +245,46 @@ fn build_site_render_artifacts_with_mode(
artifacts.pages.push(page);
}
if section.is_none_or(|section| section == GenerationSection::Core) {
let relative_path = if language == main_language {
"404.html".to_string()
} else {
format!("{language}/404.html")
};
if requested_paths.is_none_or(|requested| requested.contains(&relative_path)) {
let url_path = format!("/{}", relative_path.trim_end_matches(".html"));
artifacts.pages.push(SitePage {
language: language.clone(),
relative_path,
url_path: url_path.clone(),
html: render_not_found_route(
&bundle,
metadata,
&language,
&url_path,
&menu_items,
)?,
});
}
}
let canonical_map =
canonical_post_path_by_slug(&localized_posts, &language, &main_language);
for record in &localized_posts {
let canonical_path = build_canonical_post_path(&record.post, &language, &main_language);
let relative_path = format!("{}/index.html", canonical_path.trim_start_matches('/'));
artifacts.route_manifest.push(SitePage {
language: language.clone(),
relative_path: relative_path.clone(),
url_path: canonical_path.clone(),
html: String::new(),
});
if section.is_some_and(|section| section != GenerationSection::Single) {
continue;
}
if requested_paths.is_some_and(|requested| !requested.contains(&relative_path)) {
continue;
}
let html = render_post_route(
conn,
metadata,
@@ -199,8 +301,6 @@ fn build_site_render_artifacts_with_mode(
&bundle,
is_preview,
)?;
let canonical_path = build_canonical_post_path(&record.post, &language, &main_language);
let relative_path = format!("{}/index.html", canonical_path.trim_start_matches('/'));
artifacts.pagefind_documents.push(PagefindDocument {
language: language.clone(),
relative_path: relative_path.clone(),
@@ -234,6 +334,8 @@ pub fn build_preview_response(
metadata,
published_posts,
true,
None,
None,
)?;
let normalized = normalize_request_path(requested_path);
if let Some(page) = artifacts
@@ -464,6 +566,7 @@ fn build_language_routes(
let mut tag_posts: BTreeMap<String, Vec<RenderPostRecord>> = BTreeMap::new();
let mut year_posts: BTreeMap<i32, Vec<RenderPostRecord>> = BTreeMap::new();
let mut month_posts: BTreeMap<(i32, u32), Vec<RenderPostRecord>> = BTreeMap::new();
let mut day_posts: BTreeMap<(i32, u32, u32), Vec<RenderPostRecord>> = BTreeMap::new();
for record in posts {
for category in &record.post.categories {
@@ -490,6 +593,10 @@ fn build_language_routes(
.entry((timestamp.year(), timestamp.month()))
.or_default()
.push(record.clone());
day_posts
.entry((timestamp.year(), timestamp.month(), timestamp.day()))
.or_default()
.push(record.clone());
}
}
@@ -552,6 +659,20 @@ fn build_language_routes(
));
}
for ((year, month, day), records) in day_posts {
routes.extend(paginated_route_specs(
&records,
per_page,
format!(
"{}/{year}/{month:02}/{day:02}",
language_root_prefix(language, metadata)
),
format!("{} {year}-{month:02}-{day:02}", metadata.name),
Some(json!({"kind": "day", "year": year, "month": month, "day": day})),
None,
));
}
routes
}

176
crates/bds-core/tests/m4_generation_engine.rs
View File

@@ -4,8 +4,9 @@ use bds_core::db::Database;
use bds_core::db::queries::project::insert_project;
use bds_core::db::queries::template::insert_template;
use bds_core::engine::generation::{
PublishedPostSource, apply_validation_sections, generate_starter_site,
load_published_post_source, sections_from_validation_report,
GenerationSection, PublishedPostSource, apply_validation_section_with_progress,
apply_validation_sections, build_site_search_index, generate_starter_site,
load_published_post_source, render_site_section_with_progress, sections_from_validation_report,
};
use bds_core::engine::meta::write_category_meta_json;
use bds_core::engine::validate_site::validate_site;
@@ -99,6 +100,14 @@ fn setup() -> (Database, TempDir) {
(db, dir)
}
fn write_published_snapshot(dir: &TempDir, post: &mut Post, body: &str) {
post.file_path = format!("posts/{}.md", post.slug);
let frontmatter = bds_core::util::frontmatter::PostFrontmatter::from_post(post).to_yaml();
let file = bds_core::util::frontmatter::format_frontmatter(&frontmatter, body);
std::fs::create_dir_all(dir.path().join("posts")).unwrap();
std::fs::write(dir.path().join(&post.file_path), file).unwrap();
}
#[test]
fn reopened_draft_generation_uses_last_published_file() {
let (_db, dir) = setup();
@@ -117,17 +126,47 @@ fn reopened_draft_generation_uses_last_published_file() {
assert_eq!(source.body_markdown, "Published body");
}
#[test]
fn validation_keeps_reopened_draft_published_snapshots() {
let (db, dir) = setup();
let mut post = make_post("reopened", 1_710_000_000_000);
post.status = PostStatus::Draft;
post.content = Some("Unpublished draft body".into());
write_published_snapshot(&dir, &mut post, "Published body");
bds_core::db::queries::post::insert_post(db.conn(), &post).unwrap();
let source = load_published_post_source(dir.path(), post)
.unwrap()
.unwrap();
generate_starter_site(
db.conn(),
dir.path(),
"p1",
&make_metadata(),
&[source],
"en",
)
.unwrap();
let validation = validate_site(db.conn(), dir.path(), "p1").unwrap();
assert!(validation.missing_pages.is_empty());
assert!(validation.extra_pages.is_empty());
assert!(validation.stale_pages.is_empty());
}
#[test]
fn generation_engine_writes_core_and_single_post_artifacts() {
let (db, dir) = setup();
let metadata = make_metadata();
let hello = make_post("hello", 1_710_000_000_000);
let next = make_post("next", 1_710_086_400_000);
let posts = vec![
PublishedPostSource {
post: make_post("hello", 1_710_000_000_000),
post: hello,
body_markdown: "Hello **world**".into(),
},
PublishedPostSource {
post: make_post("next", 1_710_086_400_000),
post: next,
body_markdown: "Next post".into(),
},
];
@@ -141,6 +180,12 @@ fn generation_engine_writes_core_and_single_post_artifacts() {
assert!(report.written_paths.contains(&"feed.xml".to_string()));
assert!(report.written_paths.contains(&"atom.xml".to_string()));
assert!(report.written_paths.contains(&"sitemap.xml".to_string()));
assert!(report.written_paths.contains(&"404.html".to_string()));
assert!(
report
.written_paths
.contains(&"2024/03/09/index.html".to_string())
);
assert!(
report
.written_paths
@@ -180,6 +225,110 @@ fn generation_engine_writes_core_and_single_post_artifacts() {
assert!(sitemap.contains("https://example.com/category/article"));
}
#[test]
fn section_generation_reports_its_urls_and_defers_pagefind() {
let (db, dir) = setup();
let metadata = make_metadata();
let posts = vec![PublishedPostSource {
post: make_post("hello", 1_710_000_000_000),
body_markdown: "Hello **world**".into(),
}];
let mut urls = Vec::new();
let report = render_site_section_with_progress(
db.conn(),
dir.path(),
"p1",
&metadata,
&posts,
GenerationSection::Single,
|current, total, url| urls.push((current, total, url.to_string())),
|| false,
)
.unwrap();
assert_eq!(urls, vec![(1, 1, "/2024/03/09/hello".to_string())]);
assert_eq!(report.written_paths, vec!["2024/03/09/hello/index.html"]);
assert!(!dir.path().join("pagefind").exists());
let index_report = build_site_search_index(db.conn(), dir.path(), "p1", &metadata).unwrap();
assert!(!index_report.written_paths.is_empty());
assert!(
dir.path().join("pagefind/pagefind-ui.js").exists(),
"pagefind outputs: {:?}",
index_report.written_paths
);
let old_fragment = index_report
.written_paths
.iter()
.find(|path| path.contains("/fragment/"))
.cloned()
.unwrap();
let changed_posts = vec![PublishedPostSource {
post: make_post("hello", 1_710_000_000_000),
body_markdown: "Changed body".into(),
}];
render_site_section_with_progress(
db.conn(),
dir.path(),
"p1",
&metadata,
&changed_posts,
GenerationSection::Single,
|_current, _total, _url| {},
|| false,
)
.unwrap();
let rebuilt = build_site_search_index(db.conn(), dir.path(), "p1", &metadata).unwrap();
assert!(rebuilt.deleted_paths.contains(&old_fragment));
assert!(!dir.path().join(old_fragment).exists());
}
#[test]
fn validation_apply_rewrites_only_the_reported_urls() {
let (db, dir) = setup();
let metadata = make_metadata();
let posts = vec![
PublishedPostSource {
post: make_post("hello", 1_710_000_000_000),
body_markdown: "Hello **world**".into(),
},
PublishedPostSource {
post: make_post("next", 1_710_086_400_000),
body_markdown: "Next post".into(),
},
];
generate_starter_site(db.conn(), dir.path(), "p1", &metadata, &posts, "en").unwrap();
std::fs::write(dir.path().join("2024/03/09/hello/index.html"), "tampered").unwrap();
let validation = bds_core::engine::validate_site::SiteValidationReport {
stale_pages: vec!["2024/03/09/hello/index.html".into()],
..Default::default()
};
let mut urls = Vec::new();
let report = apply_validation_section_with_progress(
db.conn(),
dir.path(),
"p1",
&metadata,
&posts,
&validation,
GenerationSection::Single,
|current, total, url| urls.push((current, total, url.to_string())),
|| false,
)
.unwrap();
assert_eq!(urls, vec![(1, 1, "/2024/03/09/hello".to_string())]);
assert_eq!(report.written_paths, vec!["2024/03/09/hello/index.html"]);
assert!(
!report
.skipped_paths
.contains(&"2024/03/10/next/index.html".to_string())
);
}
#[test]
fn multilingual_generation_writes_language_aware_atom_and_sitemap_routes() {
let (db, dir) = setup();
@@ -195,7 +344,14 @@ fn multilingual_generation_writes_language_aware_atom_and_sitemap_routes() {
generate_starter_site(db.conn(), dir.path(), "p1", &metadata, &posts, "de").unwrap();
assert!(report.written_paths.contains(&"en/atom.xml".to_string()));
assert!(report.written_paths.contains(&"en/rss.xml".to_string()));
assert!(report.written_paths.contains(&"en/sitemap.xml".to_string()));
assert!(report.written_paths.contains(&"en/404.html".to_string()));
assert!(
report
.written_paths
.contains(&"en/pagefind/pagefind-ui.js".to_string())
);
let atom = std::fs::read_to_string(dir.path().join("en/atom.xml")).unwrap();
assert!(
@@ -340,7 +496,8 @@ fn generation_engine_skips_unchanged_outputs_on_second_run() {
fn site_validation_detects_stale_and_missing_outputs() {
let (db, dir) = setup();
let metadata = make_metadata();
let post = make_post("hello", 1_710_000_000_000);
let mut post = make_post("hello", 1_710_000_000_000);
write_published_snapshot(&dir, &mut post, "Hello **world**");
bds_core::db::queries::post::insert_post(db.conn(), &post).unwrap();
let posts = vec![PublishedPostSource {
post,
@@ -362,7 +519,8 @@ fn site_validation_detects_stale_and_missing_outputs() {
fn apply_validation_repairs_core_section_outputs() {
let (db, dir) = setup();
let metadata = make_metadata();
let post = make_post("hello", 1_710_000_000_000);
let mut post = make_post("hello", 1_710_000_000_000);
write_published_snapshot(&dir, &mut post, "Hello **world**");
bds_core::db::queries::post::insert_post(db.conn(), &post).unwrap();
let posts = vec![PublishedPostSource {
post,
@@ -390,7 +548,8 @@ fn apply_validation_repairs_core_section_outputs() {
fn apply_validation_removes_extra_section_outputs() {
let (db, dir) = setup();
let metadata = make_metadata();
let post = make_post("hello", 1_710_000_000_000);
let mut post = make_post("hello", 1_710_000_000_000);
write_published_snapshot(&dir, &mut post, "Hello **world**");
bds_core::db::queries::post::insert_post(db.conn(), &post).unwrap();
let posts = vec![PublishedPostSource {
post,
@@ -426,7 +585,8 @@ fn apply_validation_removes_extra_section_outputs() {
fn site_validation_uses_html_output_directory_when_present() {
let (db, dir) = setup();
let metadata = make_metadata();
let post = make_post("hello", 1_710_000_000_000);
let mut post = make_post("hello", 1_710_000_000_000);
write_published_snapshot(&dir, &mut post, "Hello **world**");
bds_core::db::queries::post::insert_post(db.conn(), &post).unwrap();
let posts = vec![PublishedPostSource {
post,

5
crates/bds-core/tests/m4_generation_primitives.rs
View File

@@ -114,11 +114,16 @@ fn generated_write_rewrites_stale_file_even_when_db_hash_matches() {
fn core_generation_paths_include_language_prefixed_variants() {
let paths = build_core_generation_paths("en", &["en".into(), "de".into(), "fr".into()]);
assert!(paths.contains(&"index.html".to_string()));
assert!(paths.contains(&"404.html".to_string()));
assert!(paths.contains(&"sitemap.xml".to_string()));
assert!(paths.contains(&"rss.xml".to_string()));
assert!(paths.contains(&"feed.xml".to_string()));
assert!(paths.contains(&"atom.xml".to_string()));
assert!(paths.contains(&"calendar.json".to_string()));
assert!(paths.contains(&"de/index.html".to_string()));
assert!(paths.contains(&"de/404.html".to_string()));
assert!(paths.contains(&"de/sitemap.xml".to_string()));
assert!(paths.contains(&"de/rss.xml".to_string()));
assert!(paths.contains(&"de/feed.xml".to_string()));
assert!(paths.contains(&"de/atom.xml".to_string()));
assert!(paths.contains(&"fr/index.html".to_string()));

265
crates/bds-ui/src/app.rs
View File

@@ -201,8 +201,17 @@ pub enum Message {
label: String,
result: Result<String, String>,
},
SiteGenerationSectionDone {
group_id: String,
task_id: TaskId,
result: Result<engine::generation::GenerationReport, String>,
},
SiteGenerationIndexDone {
group_id: String,
task_id: TaskId,
result: Result<engine::generation::GenerationReport, String>,
},
SiteValidationLoaded(Result<engine::validate_site::SiteValidationReport, String>),
SiteValidationApplied(Result<String, String>),
// Editor views
PostEditor(PostEditorMsg),
@@ -242,6 +251,24 @@ pub enum Message {
InitMenuBar,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum SiteGenerationKind {
Full,
Validation,
}
#[derive(Debug, Clone)]
struct SiteGenerationWorkflow {
kind: SiteGenerationKind,
db_path: PathBuf,
project_id: String,
data_dir: PathBuf,
group_name: String,
render_task_ids: Vec<TaskId>,
index_task_id: Option<TaskId>,
report: engine::generation::GenerationReport,
}
enum PersistedPostState {
Canonical(Box<Post>),
Translation(Box<bds_core::model::PostTranslation>),
@@ -754,6 +781,7 @@ pub struct BdsApp {
search_index_rebuild_required: bool,
search_index_rebuild_running: bool,
search_index_rebuild_task_id: Option<TaskId>,
site_generation_workflows: HashMap<String, SiteGenerationWorkflow>,
// Platform
_menu_bar: Option<muda::Menu>,
@@ -908,6 +936,7 @@ impl BdsApp {
search_index_rebuild_required,
search_index_rebuild_running: false,
search_index_rebuild_task_id: None,
site_generation_workflows: HashMap::new(),
_menu_bar: Some(menu_bar),
menu_registry: registry,
ui_locale: locale,
@@ -974,6 +1003,7 @@ impl BdsApp {
search_index_rebuild_required: false,
search_index_rebuild_running: false,
search_index_rebuild_task_id: None,
site_generation_workflows: HashMap::new(),
_menu_bar: None,
menu_registry: MenuRegistry::empty(),
ui_locale: UiLocale::En,
@@ -1610,6 +1640,9 @@ impl BdsApp {
)
}
Message::CancelTask(task_id) => {
if self.cancel_site_generation_task(task_id) {
return Task::none();
}
self.task_manager.cancel(task_id);
self.refresh_task_snapshots();
Task::none()
@@ -1831,8 +1864,9 @@ impl BdsApp {
| Message::RunSiteValidation
| Message::ApplySiteValidation
| Message::EngineTaskDone { .. }
| Message::SiteValidationLoaded(_)
| Message::SiteValidationApplied(_)) => self.handle_engine_message(message),
| Message::SiteGenerationSectionDone { .. }
| Message::SiteGenerationIndexDone { .. }
| Message::SiteValidationLoaded(_)) => self.handle_engine_message(message),
// ── Toasts ──
Message::ShowToast(level, msg) => {
@@ -2660,68 +2694,9 @@ impl BdsApp {
return Task::none();
}
let Some(project_id) = self
.active_project
.as_ref()
.map(|project| project.id.clone())
else {
self.site_validation_state.error_message =
Some(t(self.ui_locale, "engine.generateSiteNoProject"));
return Task::none();
};
let Some(data_dir) = self.data_dir.clone() else {
self.site_validation_state.error_message =
Some(t(self.ui_locale, "engine.previewDataDirUnavailable"));
return Task::none();
};
self.site_validation_state.is_applying = true;
self.site_validation_state.error_message = None;
let db_path = self.db_path.clone();
let applied_label = t(self.ui_locale, "siteValidation.apply");
Task::perform(
async move {
let db = Database::open(&db_path).map_err(|error| error.to_string())?;
let metadata = engine::meta::read_project_json(&data_dir)
.map_err(|error| error.to_string())?;
let all_posts =
bds_core::db::queries::post::list_posts_by_project(db.conn(), &project_id)
.map_err(|error| error.to_string())?;
let mut sources = Vec::new();
for post in all_posts
.into_iter()
.filter(engine::generation::has_published_snapshot)
{
if let Some(source) =
engine::generation::load_published_post_source(&data_dir, post)
.map_err(|error| error.to_string())?
{
sources.push(source);
}
}
let output_dir = data_dir.join("html");
std::fs::create_dir_all(&output_dir).map_err(|error| error.to_string())?;
let apply_report = engine::generation::apply_validation_sections(
db.conn(),
&output_dir,
&project_id,
&metadata,
&sources,
&sections,
)
.map_err(|error| error.to_string())?;
Ok(format!(
"{}: written={}, skipped={}, deleted={}, output={}",
applied_label,
apply_report.written_paths.len(),
apply_report.skipped_paths.len(),
apply_report.deleted_paths.len(),
output_dir.display(),
))
},
Message::SiteValidationApplied,
)
self.queue_site_generation(Some(report))
}
fn refresh_counts(&mut self) -> Task<Message> {
@@ -6236,6 +6211,8 @@ mod tests {
use bds_core::db::Database;
use bds_core::db::fts::ensure_fts_tables;
use bds_core::db::queries::project::insert_project;
use bds_core::engine::generation::GenerationReport;
use bds_core::engine::task::{TaskStatus, TaskStatus::*};
use bds_core::engine::{ai, media, post, script, template};
use bds_core::model::{Project, ScriptKind, TemplateKind};
use chrono::{Datelike, TimeZone};
@@ -6317,6 +6294,12 @@ mod tests {
BdsApp::new_for_tests(db, project, tmp.path().to_path_buf())
}
fn enable_generation(tmp: &TempDir) {
let mut metadata = bds_core::engine::meta::read_project_json(tmp.path()).unwrap();
metadata.public_url = Some("https://example.com".to_string());
bds_core::engine::meta::write_project_json(tmp.path(), &metadata).unwrap();
}
fn open_post_editor(app: &mut BdsApp, post: &bds_core::model::Post) {
let tab = crate::state::tabs::Tab {
id: post.id.clone(),
@@ -6462,6 +6445,162 @@ mod tests {
));
}
#[test]
fn full_generation_queues_five_ordered_section_tasks_before_indexing() {
let (db, project, tmp) = setup();
enable_generation(&tmp);
let mut app = make_app(db, project, &tmp);
let _task = app.queue_site_generation(None);
let snapshots = app.task_manager.snapshots();
assert_eq!(
snapshots
.iter()
.map(|task| task.label.as_str())
.collect::<Vec<_>>(),
vec![
"Render Site Core",
"Render Single Posts",
"Render Category Archives",
"Render Tag Archives",
"Render Date Archives",
]
);
assert!(
snapshots
.iter()
.all(|task| task.group_name.as_deref() == Some("Render Site"))
);
assert_eq!(
snapshots
.iter()
.map(|task| task.status.clone())
.collect::<Vec<_>>(),
vec![Running, Running, Running, Pending, Pending]
);
}
#[test]
fn search_index_is_queued_only_after_every_render_task_succeeds() {
let (db, project, tmp) = setup();
enable_generation(&tmp);
let mut app = make_app(db, project, &tmp);
let _task = app.queue_site_generation(None);
let group_id = app.task_manager.snapshots()[0].group_id.clone().unwrap();
let render_ids = app.site_generation_workflows[&group_id]
.render_task_ids
.clone();
for (index, task_id) in render_ids.into_iter().enumerate() {
let _task = app.handle_engine_message(Message::SiteGenerationSectionDone {
group_id: group_id.clone(),
task_id,
result: Ok(GenerationReport::default()),
});
let has_index = app
.task_manager
.snapshots()
.iter()
.any(|task| task.label == "Build Search Index");
assert_eq!(has_index, index == 4);
}
}
#[test]
fn failed_render_cancels_its_group_and_never_queues_index() {
let (db, project, tmp) = setup();
enable_generation(&tmp);
let mut app = make_app(db, project, &tmp);
let _task = app.queue_site_generation(None);
let snapshots = app.task_manager.snapshots();
let group_id = snapshots[0].group_id.clone().unwrap();
let first_id = snapshots[0].id;
let _task = app.handle_engine_message(Message::SiteGenerationSectionDone {
group_id: group_id.clone(),
task_id: first_id,
result: Err("render failed".to_string()),
});
let snapshots = app.task_manager.snapshots();
assert_eq!(
app.task_manager.status(first_id),
Some(TaskStatus::Failed("render failed".to_string()))
);
assert!(
snapshots
.iter()
.filter(|task| task.group_id.as_deref() == Some(&group_id))
.all(|task| matches!(task.status, TaskStatus::Failed(_) | TaskStatus::Cancelled))
);
assert!(
!snapshots
.iter()
.any(|task| task.label == "Build Search Index")
);
assert!(!app.site_generation_workflows.contains_key(&group_id));
}
#[test]
fn cancelling_a_render_task_cancels_the_generation_group() {
let (db, project, tmp) = setup();
enable_generation(&tmp);
let mut app = make_app(db, project, &tmp);
let _task = app.queue_site_generation(None);
let snapshots = app.task_manager.snapshots();
let group_id = snapshots[0].group_id.clone().unwrap();
let _task = app.update(Message::CancelTask(snapshots[0].id));
let snapshots = app.task_manager.snapshots();
assert!(
snapshots
.iter()
.filter(|task| task.group_id.as_deref() == Some(&group_id))
.all(|task| task.status == TaskStatus::Cancelled)
);
assert!(
!snapshots
.iter()
.any(|task| task.label == "Build Search Index")
);
assert!(!app.site_generation_workflows.contains_key(&group_id));
}
#[test]
fn validation_apply_queues_only_affected_sections_then_index() {
let (db, project, tmp) = setup();
enable_generation(&tmp);
let mut app = make_app(db, project, &tmp);
let validation = bds_core::engine::validate_site::SiteValidationReport {
stale_pages: vec!["2024/03/09/hello/index.html".to_string()],
..Default::default()
};
let _task = app.queue_site_generation(Some(validation));
let snapshots = app.task_manager.snapshots();
assert_eq!(snapshots.len(), 1);
assert_eq!(snapshots[0].label, "Render Single Posts");
assert_eq!(
snapshots[0].group_name.as_deref(),
Some("Apply Site Validation")
);
let group_id = snapshots[0].group_id.clone().unwrap();
let _task = app.handle_engine_message(Message::SiteGenerationSectionDone {
group_id,
task_id: snapshots[0].id,
result: Ok(GenerationReport::default()),
});
assert!(
app.task_manager
.snapshots()
.iter()
.any(|task| task.label == "Build Search Index")
);
}
#[test]
fn post_editor_save_flow_persists_changes() {
let (db, project, tmp) = setup();

238
crates/bds-ui/src/app/engine_handlers.rs
View File

@@ -139,109 +139,7 @@ impl BdsApp {
},
)
}
Message::GenerateSite => {
let locale = self.ui_locale;
self.spawn_engine_task(
"engine.generateSiteStarted",
move |db_path, project_id, data_dir, tm, tid| {
let db = Database::open(&db_path).map_err(|e| e.to_string())?;
let metadata = engine::meta::read_project_json(&data_dir)
.map_err(|e| e.to_string())?;
if metadata
.public_url
.as_deref()
.unwrap_or("")
.trim()
.is_empty()
{
return Err(
"public URL is required before generating the site".to_string()
);
}
let main_language = metadata
.main_language
.clone()
.unwrap_or_else(|| "en".to_string());
let all_posts = bds_core::db::queries::post::list_posts_by_project(
db.conn(),
&project_id,
)
.map_err(|e| e.to_string())?;
let published_posts = all_posts
.into_iter()
.filter(engine::generation::has_published_snapshot)
.collect::<Vec<_>>();
let total = published_posts.len().max(1) as f32;
let mut sources = Vec::new();
for (index, post) in published_posts.into_iter().enumerate() {
if tm.is_cancelled(tid) {
return Err("cancelled".to_string());
}
tm.report_progress(
tid,
Some(((index as f32) / total) * 0.7),
Some(tw(locale, "engine.renderingPost", &[("post", &post.slug)])),
);
if let Some(source) =
engine::generation::load_published_post_source(&data_dir, post)
.map_err(|error| error.to_string())?
{
sources.push(source);
}
}
let output_dir = data_dir.join("html");
std::fs::create_dir_all(&output_dir).map_err(|e| e.to_string())?;
tm.report_progress(
tid,
Some(0.85),
Some(t(locale, "engine.writingGeneratedFiles")),
);
let progress_start = 0.70_f32;
let progress_span = 0.28_f32;
let task_manager = Arc::clone(&tm);
let report = engine::generation::generate_starter_site_with_progress(
db.conn(),
&output_dir,
&project_id,
&metadata,
&sources,
&main_language,
move |current, total, path| {
let fraction = if total == 0 {
1.0
} else {
current as f32 / total as f32
};
task_manager.report_progress(
tid,
Some(progress_start + fraction * progress_span),
Some(tw(
locale,
"engine.generatedPage",
&[
("url", path),
("current", &current.to_string()),
("total", &total.to_string()),
],
)),
);
},
)
.map_err(|e| e.to_string())?;
tm.report_progress(
tid,
Some(1.0),
Some(t(locale, "engine.generationComplete")),
);
Ok(format!(
"written={}, skipped={}, output={}",
report.written_paths.len(),
report.skipped_paths.len(),
output_dir.display(),
))
},
)
}
Message::GenerateSite => self.queue_site_generation(None),
Message::RunMetadataDiff => {
self.open_singleton_tab(TabType::MetadataDiff, "tabBar.metadataDiff");
self.start_metadata_diff()
@@ -349,6 +247,125 @@ impl BdsApp {
self.refresh_task_snapshots();
sidebar_task
}
Message::SiteGenerationSectionDone {
group_id,
task_id,
result,
} => {
if self.task_manager.status(task_id) == Some(TaskStatus::Cancelled) {
self.refresh_task_snapshots();
return Task::none();
}
match result {
Ok(report) => {
self.task_manager.complete(task_id);
if let Some(workflow) = self.site_generation_workflows.get_mut(&group_id) {
workflow.report.append(report);
}
}
Err(error) => {
self.task_manager.fail(task_id, error.clone());
self.task_manager.cancel_group(&group_id);
if let Some(workflow) = self.site_generation_workflows.remove(&group_id)
&& workflow.kind == SiteGenerationKind::Validation
{
self.site_validation_state.is_applying = false;
self.site_validation_state.error_message = Some(error.clone());
}
let message = tw(
self.ui_locale,
"common.operationFailed",
&[
("operation", &t(self.ui_locale, "engine.renderSiteGroup")),
("error", &error),
],
);
self.notify(ToastLevel::Error, &message);
self.refresh_task_snapshots();
return Task::none();
}
}
let should_index =
self.site_generation_workflows
.get(&group_id)
.is_some_and(|workflow| {
workflow.index_task_id.is_none()
&& workflow.render_task_ids.iter().all(|task_id| {
self.task_manager.status(*task_id)
== Some(TaskStatus::Completed)
})
});
self.refresh_task_snapshots();
if should_index {
self.queue_site_search_index(&group_id)
} else {
Task::none()
}
}
Message::SiteGenerationIndexDone {
group_id,
task_id,
result,
} => {
if self.task_manager.status(task_id) == Some(TaskStatus::Cancelled) {
self.refresh_task_snapshots();
return Task::none();
}
match result {
Ok(report) => {
self.task_manager.complete(task_id);
let Some(mut workflow) = self.site_generation_workflows.remove(&group_id)
else {
self.refresh_task_snapshots();
return Task::none();
};
workflow.report.append(report);
let message = tw(
self.ui_locale,
"engine.generationSummary",
&[
("written", &workflow.report.written_paths.len().to_string()),
("skipped", &workflow.report.skipped_paths.len().to_string()),
("deleted", &workflow.report.deleted_paths.len().to_string()),
(
"output",
&workflow.data_dir.join("html").display().to_string(),
),
],
);
self.notify(ToastLevel::Success, &message);
self.refresh_task_snapshots();
if workflow.kind == SiteGenerationKind::Validation {
self.site_validation_state.is_applying = false;
self.site_validation_state.error_message = None;
self.start_site_validation()
} else {
Task::none()
}
}
Err(error) => {
self.task_manager.fail(task_id, error.clone());
if let Some(workflow) = self.site_generation_workflows.remove(&group_id)
&& workflow.kind == SiteGenerationKind::Validation
{
self.site_validation_state.is_applying = false;
self.site_validation_state.error_message = Some(error.clone());
}
let message = tw(
self.ui_locale,
"common.operationFailed",
&[
("operation", &t(self.ui_locale, "engine.buildSearchIndex")),
("error", &error),
],
);
self.notify(ToastLevel::Error, &message);
self.refresh_task_snapshots();
Task::none()
}
}
}
Message::SiteValidationLoaded(result) => {
self.site_validation_state.is_running = false;
self.site_validation_state.has_run = true;
@@ -391,21 +408,6 @@ impl BdsApp {
}
Task::none()
}
Message::SiteValidationApplied(result) => {
self.site_validation_state.is_applying = false;
match result {
Ok(detail) => {
self.site_validation_state.error_message = None;
self.notify(ToastLevel::Success, &detail);
self.start_site_validation()
}
Err(error) => {
self.site_validation_state.error_message = Some(error.clone());
self.notify(ToastLevel::Error, &error);
Task::none()
}
}
}
_ => unreachable!("non-engine message routed to engine handler"),
}
}

313
crates/bds-ui/src/app/tasks.rs
View File

@@ -32,6 +32,218 @@ impl BdsApp {
})
.collect();
}
pub(super) fn queue_site_generation(
&mut self,
validation: Option<engine::validate_site::SiteValidationReport>,
) -> Task<Message> {
let kind = if validation.is_some() {
SiteGenerationKind::Validation
} else {
SiteGenerationKind::Full
};
let (Some(project), Some(data_dir)) = (&self.active_project, &self.data_dir) else {
if kind == SiteGenerationKind::Validation {
self.site_validation_state.is_applying = false;
}
self.notify(
ToastLevel::Error,
&t(self.ui_locale, "engine.generateSiteNoProject"),
);
return Task::none();
};
let metadata = match engine::meta::read_project_json(data_dir) {
Ok(metadata) => metadata,
Err(error) => {
if kind == SiteGenerationKind::Validation {
self.site_validation_state.is_applying = false;
self.site_validation_state.error_message = Some(error.to_string());
}
self.notify(ToastLevel::Error, &error.to_string());
return Task::none();
}
};
if metadata
.public_url
.as_deref()
.unwrap_or("")
.trim()
.is_empty()
{
if kind == SiteGenerationKind::Validation {
self.site_validation_state.is_applying = false;
}
self.notify(
ToastLevel::Error,
&t(self.ui_locale, "engine.publicUrlRequired"),
);
return Task::none();
}
let sections = validation.as_ref().map_or_else(
|| engine::generation::GenerationSection::ALL.to_vec(),
engine::generation::sections_from_validation_report,
);
if sections.is_empty() {
return Task::none();
}
let project_id = project.id.clone();
let db_path = self.db_path.clone();
let data_dir = data_dir.clone();
let group_id = format!("site-generation:{}", Uuid::new_v4());
let group_name = t(
self.ui_locale,
if kind == SiteGenerationKind::Full {
"engine.renderSiteGroup"
} else {
"engine.applyValidationGroup"
},
);
let mut render_task_ids = Vec::new();
let mut tasks = Vec::new();
for section in sections {
let label = t(self.ui_locale, generation_section_label_key(section));
self.add_output(&label);
let task_id = self
.task_manager
.submit_grouped(&label, &group_id, &group_name);
render_task_ids.push(task_id);
let task_manager = Arc::clone(&self.task_manager);
let task_db_path = db_path.clone();
let task_project_id = project_id.clone();
let task_data_dir = data_dir.clone();
let task_group_id = group_id.clone();
let task_validation = validation.clone();
let locale = self.ui_locale;
tasks.push(Task::perform(
async move {
tokio::task::spawn_blocking(move || {
run_site_generation_section(
task_db_path,
task_project_id,
task_data_dir,
task_manager,
task_id,
section,
task_validation,
locale,
)
})
.await
.unwrap_or_else(|error| Err(format!("task panicked: {error}")))
},
move |result| Message::SiteGenerationSectionDone {
group_id: task_group_id.clone(),
task_id,
result,
},
));
}
self.site_generation_workflows.insert(
group_id,
SiteGenerationWorkflow {
kind,
db_path,
project_id,
data_dir,
group_name,
render_task_ids,
index_task_id: None,
report: engine::generation::GenerationReport::default(),
},
);
self.refresh_task_snapshots();
Task::batch(tasks)
}
pub(super) fn queue_site_search_index(&mut self, group_id: &str) -> Task<Message> {
let Some(workflow) = self.site_generation_workflows.get(group_id).cloned() else {
return Task::none();
};
let label = t(self.ui_locale, "engine.buildSearchIndex");
self.add_output(&label);
let task_id = self
.task_manager
.submit_grouped(&label, group_id, &workflow.group_name);
if let Some(workflow) = self.site_generation_workflows.get_mut(group_id) {
workflow.index_task_id = Some(task_id);
}
self.refresh_task_snapshots();
let locale = self.ui_locale;
let task_manager = Arc::clone(&self.task_manager);
let task_group_id = group_id.to_string();
Task::perform(
async move {
tokio::task::spawn_blocking(move || {
if !task_manager.wait_until_runnable(task_id) {
return Err("cancelled".to_string());
}
let db =
Database::open(&workflow.db_path).map_err(|error| error.to_string())?;
let metadata = engine::meta::read_project_json(&workflow.data_dir)
.map_err(|error| error.to_string())?;
let output_dir = workflow.data_dir.join("html");
let progress_manager = Arc::clone(&task_manager);
let cancel_manager = Arc::clone(&task_manager);
engine::generation::build_site_search_index_with_progress(
db.conn(),
&output_dir,
&workflow.project_id,
&metadata,
move |current, total, path| {
let progress = if total == 0 {
1.0
} else {
current as f32 / total as f32
};
progress_manager.report_progress(
task_id,
Some(progress),
Some(tw(
locale,
"engine.builtSearchFile",
&[
("path", path),
("current", &current.to_string()),
("total", &total.to_string()),
],
)),
);
},
move || cancel_manager.is_cancelled(task_id),
)
.map_err(|error| error.to_string())
})
.await
.unwrap_or_else(|error| Err(format!("task panicked: {error}")))
},
move |result| Message::SiteGenerationIndexDone {
group_id: task_group_id.clone(),
task_id,
result,
},
)
}
pub(super) fn cancel_site_generation_task(&mut self, task_id: TaskId) -> bool {
let Some(group_id) = self.task_manager.group_id(task_id) else {
return false;
};
let Some(workflow) = self.site_generation_workflows.remove(&group_id) else {
return false;
};
self.task_manager.cancel_group(&group_id);
if workflow.kind == SiteGenerationKind::Validation {
self.site_validation_state.is_applying = false;
}
self.add_output(&t(self.ui_locale, "engine.generationCancelled"));
self.refresh_task_snapshots();
true
}
/// Rebuild the shared search index while the modal blocks editor writes.
pub(super) fn start_search_index_rebuild(&mut self) -> Task<Message> {
if self.db.is_none() || self.search_index_rebuild_running {
@@ -185,3 +397,104 @@ impl BdsApp {
)
}
}
fn generation_section_label_key(section: engine::generation::GenerationSection) -> &'static str {
match section {
engine::generation::GenerationSection::Core => "engine.renderSiteCore",
engine::generation::GenerationSection::Single => "engine.renderSinglePosts",
engine::generation::GenerationSection::Category => "engine.renderCategoryArchives",
engine::generation::GenerationSection::Tag => "engine.renderTagArchives",
engine::generation::GenerationSection::Date => "engine.renderDateArchives",
}
}
#[expect(
clippy::too_many_arguments,
reason = "task input is captured generation context"
)]
fn run_site_generation_section(
db_path: PathBuf,
project_id: String,
data_dir: PathBuf,
task_manager: Arc<TaskManager>,
task_id: TaskId,
section: engine::generation::GenerationSection,
validation: Option<engine::validate_site::SiteValidationReport>,
locale: UiLocale,
) -> Result<engine::generation::GenerationReport, String> {
if !task_manager.wait_until_runnable(task_id) {
return Err("cancelled".to_string());
}
let db = Database::open(&db_path).map_err(|error| error.to_string())?;
let metadata = engine::meta::read_project_json(&data_dir).map_err(|error| error.to_string())?;
let posts = bds_core::db::queries::post::list_posts_by_project(db.conn(), &project_id)
.map_err(|error| error.to_string())?;
let mut sources = Vec::new();
for post in posts
.into_iter()
.filter(engine::generation::has_published_snapshot)
{
if task_manager.is_cancelled(task_id) {
return Err("cancelled".to_string());
}
if let Some(source) = engine::generation::load_published_post_source(&data_dir, post)
.map_err(|error| error.to_string())?
{
sources.push(source);
}
}
let output_dir = data_dir.join("html");
std::fs::create_dir_all(&output_dir).map_err(|error| error.to_string())?;
let progress_manager = Arc::clone(&task_manager);
let cancel_manager = Arc::clone(&task_manager);
let progress_key = if validation.is_some() {
"engine.rewrotePage"
} else {
"engine.generatedPage"
};
let on_page = move |current: usize, total: usize, url: &str| {
let progress = if total == 0 {
1.0
} else {
current as f32 / total as f32
};
progress_manager.report_progress(
task_id,
Some(progress),
Some(tw(
locale,
progress_key,
&[
("url", url),
("current", &current.to_string()),
("total", &total.to_string()),
],
)),
);
};
let is_cancelled = move || cancel_manager.is_cancelled(task_id);
match validation {
Some(validation) => engine::generation::apply_validation_section_with_progress(
db.conn(),
&output_dir,
&project_id,
&metadata,
&sources,
&validation,
section,
on_page,
is_cancelled,
),
None => engine::generation::render_site_section_with_progress(
db.conn(),
&output_dir,
&project_id,
&metadata,
&sources,
section,
on_page,
is_cancelled,
),
}
.map_err(|error| error.to_string())
}