hugo/RuDS
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

Group site generation by section

Browse Source
This commit is contained in:
Chili Palmer
2026-07-19 10:27:12 +02:00
parent 055dd5efc5
commit 90a9002124
19 changed files with 1467 additions and 399 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
crates/bds-core/src/engine/gallery_import.rs
View File

@@ -70,10 +70,6 @@ where
})
}
#[expect(
clippy::too_many_arguments,
reason = "the shared gallery workflow needs its persisted post and project context"
)]
pub fn import_gallery_images(
db_path: &Path,
data_dir: &Path,

579
crates/bds-core/src/engine/generation.rs
View File

@@ -1,4 +1,4 @@
use std::collections::{HashMap, HashSet};
use std::collections::{BTreeMap, HashMap, HashSet};
use std::path::Path;
use crate::db::DbConnection as Connection;
@@ -14,7 +14,8 @@ use crate::engine::{EngineError, EngineResult};
use crate::model::{CategorySettings, Post, ProjectMetadata};
use crate::render::{
GeneratedWriteOutcome, build_calendar_json, build_canonical_post_path,
build_site_render_artifacts, render_markdown_to_html, write_generated_bytes,
build_site_render_artifacts, build_site_section_render_artifacts,
build_targeted_site_section_render_artifacts, render_markdown_to_html, write_generated_bytes,
write_generated_file,
};
@@ -65,6 +66,24 @@ pub enum GenerationSection {
Date,
}
impl GenerationSection {
pub const ALL: [Self; 5] = [
Self::Core,
Self::Single,
Self::Category,
Self::Tag,
Self::Date,
];
}
impl GenerationReport {
pub fn append(&mut self, mut other: Self) {
self.written_paths.append(&mut other.written_paths);
self.skipped_paths.append(&mut other.skipped_paths);
self.deleted_paths.append(&mut other.deleted_paths);
}
}
pub fn generate_starter_site(
conn: &Connection,
output_dir: &Path,
@@ -94,6 +113,41 @@ pub fn generate_starter_site_with_progress(
mut on_page: impl FnMut(usize, usize, &str),
) -> EngineResult<GenerationReport> {
let mut report = GenerationReport::default();
for section in GenerationSection::ALL {
report.append(render_site_section_with_progress(
conn,
output_dir,
project_id,
metadata,
posts,
section,
&mut on_page,
|| false,
)?);
}
report.append(build_site_search_index(
conn, output_dir, project_id, metadata,
)?);
Ok(report)
}
#[expect(
clippy::too_many_arguments,
reason = "section rendering uses the existing generation context and two callbacks"
)]
pub fn render_site_section_with_progress(
conn: &Connection,
output_dir: &Path,
project_id: &str,
metadata: &ProjectMetadata,
posts: &[PublishedPostSource],
section: GenerationSection,
mut on_page: impl FnMut(usize, usize, &str),
mut is_cancelled: impl FnMut() -> bool,
) -> EngineResult<GenerationReport> {
if is_cancelled() {
return Err(EngineError::Validation("cancelled".to_string()));
}
let data_dir = project_data_dir(output_dir);
let category_settings = load_category_settings(&data_dir);
let list_posts = filter_posts_for_lists(posts, &category_settings);
@@ -101,12 +155,21 @@ pub fn generate_starter_site_with_progress(
.iter()
.map(|source| (source.post.clone(), source.body_markdown.clone()))
.collect::<Vec<_>>();
let artifacts =
build_site_render_artifacts(conn, &data_dir, project_id, metadata, &input_posts)
.map_err(|error| EngineError::Parse(error.to_string()))?;
let artifacts = build_site_section_render_artifacts(
conn,
&data_dir,
project_id,
metadata,
&input_posts,
section,
)
.map_err(|error| EngineError::Parse(error.to_string()))?;
let mut report = GenerationReport::default();
let total_pages = artifacts.pages.len();
for (index, page) in artifacts.pages.iter().enumerate() {
if is_cancelled() {
return Err(EngineError::Validation("cancelled".to_string()));
}
write_out(
conn,
output_dir,
@@ -118,78 +181,20 @@ pub fn generate_starter_site_with_progress(
on_page(index + 1, total_pages, &page.url_path);
}
write_bundled_site_assets(conn, output_dir, project_id, &mut report)?;
write_out(
conn,
output_dir,
project_id,
"calendar.json",
&build_calendar_json(
&list_posts
.iter()
.map(|source| source.post.clone())
.collect::<Vec<_>>(),
)?,
&mut report,
)?;
for render_language in render_languages(metadata) {
let localized_posts =
localized_sources(conn, &data_dir, &list_posts, &render_language, metadata)?;
let prefix = if render_language
== metadata
.main_language
.clone()
.unwrap_or_else(|| "en".to_string())
{
String::new()
} else {
format!("{}/", render_language)
};
let rss = build_rss_xml(metadata, &localized_posts, &render_language);
if prefix.is_empty() {
write_out(conn, output_dir, project_id, "rss.xml", &rss, &mut report)?;
}
write_out(
if section == GenerationSection::Core {
write_core_outputs(
conn,
output_dir,
project_id,
&format!("{prefix}feed.xml"),
&rss,
&mut report,
)?;
write_out(
conn,
output_dir,
project_id,
&format!("{prefix}atom.xml"),
&build_atom_xml(metadata, &localized_posts, &render_language),
&mut report,
)?;
write_out(
conn,
output_dir,
project_id,
&format!("{prefix}sitemap.xml"),
&build_sitemap_xml(
metadata,
&artifacts.pages,
&localized_posts,
&render_language,
),
metadata,
&data_dir,
&list_posts,
&artifacts.route_manifest,
None,
&mut report,
&mut is_cancelled,
)?;
}
write_pagefind_indexes(
conn,
output_dir,
project_id,
&artifacts.pagefind_documents,
&mut report,
)?;
Ok(report)
}
@@ -236,8 +241,6 @@ pub fn apply_validation_sections(
let section_set = sections.iter().copied().collect::<HashSet<_>>();
let data_dir = project_data_dir(output_dir);
let category_settings = load_category_settings(&data_dir);
let list_posts = filter_posts_for_lists(posts, &category_settings);
let input_posts = posts
.iter()
.map(|source| (source.post.clone(), source.body_markdown.clone()))
@@ -248,96 +251,190 @@ pub fn apply_validation_sections(
let mut report = GenerationReport::default();
let expected_paths = expected_paths_for_sections(metadata, &artifacts.pages, &section_set);
for page in &artifacts.pages {
if path_matches_sections(&page.relative_path, &section_set) {
write_out(
conn,
output_dir,
project_id,
&page.relative_path,
&page.html,
&mut report,
)?;
}
for section in sections {
report.append(render_site_section_with_progress(
conn,
output_dir,
project_id,
metadata,
posts,
*section,
|_current, _total, _url| {},
|| false,
)?);
}
if section_set.contains(&GenerationSection::Core) {
write_bundled_site_assets(conn, output_dir, project_id, &mut report)?;
remove_extra_section_paths(output_dir, &expected_paths, &section_set, &mut report)?;
report.append(build_site_search_index(
conn, output_dir, project_id, metadata,
)?);
Ok(report)
}
#[expect(
clippy::too_many_arguments,
reason = "targeted apply adds its validation report to the existing generation context"
)]
pub fn apply_validation_section_with_progress(
conn: &Connection,
output_dir: &Path,
project_id: &str,
metadata: &ProjectMetadata,
posts: &[PublishedPostSource],
validation: &SiteValidationReport,
section: GenerationSection,
mut on_page: impl FnMut(usize, usize, &str),
mut is_cancelled: impl FnMut() -> bool,
) -> EngineResult<GenerationReport> {
if is_cancelled() {
return Err(EngineError::Validation("cancelled".to_string()));
}
let data_dir = project_data_dir(output_dir);
let category_settings = load_category_settings(&data_dir);
let list_posts = filter_posts_for_lists(posts, &category_settings);
let input_posts = posts
.iter()
.map(|source| (source.post.clone(), source.body_markdown.clone()))
.collect::<Vec<_>>();
let requested = validation
.missing_pages
.iter()
.chain(validation.stale_pages.iter())
.cloned()
.collect::<HashSet<_>>();
let fallback = validation
.missing_pages
.iter()
.chain(validation.extra_pages.iter())
.chain(validation.stale_pages.iter())
.any(|path| classify_generated_path(path).is_none());
let artifacts = if fallback {
build_site_section_render_artifacts(
conn,
&data_dir,
project_id,
metadata,
&input_posts,
section,
)
} else {
build_targeted_site_section_render_artifacts(
conn,
&data_dir,
project_id,
metadata,
&input_posts,
section,
&requested,
)
}
.map_err(|error| EngineError::Parse(error.to_string()))?;
let mut report = GenerationReport::default();
let total_pages = artifacts.pages.len();
for (index, page) in artifacts.pages.iter().enumerate() {
if is_cancelled() {
return Err(EngineError::Validation("cancelled".to_string()));
}
write_out(
conn,
output_dir,
project_id,
"calendar.json",
&build_calendar_json(
&list_posts
.iter()
.map(|source| source.post.clone())
.collect::<Vec<_>>(),
)?,
&page.relative_path,
&page.html,
&mut report,
)?;
for render_language in render_languages(metadata) {
let localized_posts =
localized_sources(conn, &data_dir, &list_posts, &render_language, metadata)?;
let prefix = if render_language
== metadata
.main_language
.clone()
.unwrap_or_else(|| "en".to_string())
{
String::new()
} else {
format!("{}/", render_language)
};
let rss = build_rss_xml(metadata, &localized_posts, &render_language);
if prefix.is_empty() {
write_out(conn, output_dir, project_id, "rss.xml", &rss, &mut report)?;
}
write_out(
conn,
output_dir,
project_id,
&format!("{prefix}feed.xml"),
&rss,
&mut report,
)?;
write_out(
conn,
output_dir,
project_id,
&format!("{prefix}atom.xml"),
&build_atom_xml(metadata, &localized_posts, &render_language),
&mut report,
)?;
write_out(
conn,
output_dir,
project_id,
&format!("{prefix}sitemap.xml"),
&build_sitemap_xml(
metadata,
&artifacts.pages,
&localized_posts,
&render_language,
),
&mut report,
)?;
}
on_page(index + 1, total_pages, &page.url_path);
}
remove_extra_section_paths(output_dir, &expected_paths, &section_set, &mut report)?;
write_pagefind_indexes(
conn,
output_dir,
project_id,
&artifacts.pagefind_documents,
&mut report,
)?;
if section == GenerationSection::Core {
write_core_outputs(
conn,
output_dir,
project_id,
metadata,
&data_dir,
&list_posts,
&artifacts.route_manifest,
(!fallback).then_some(&requested),
&mut report,
&mut is_cancelled,
)?;
}
for path in &validation.extra_pages {
if is_cancelled() {
return Err(EngineError::Validation("cancelled".to_string()));
}
let owned_by_section = classify_generated_path(path)
.map_or(section == GenerationSection::Core, |owner| owner == section);
if owned_by_section && output_dir.join(path).is_file() {
std::fs::remove_file(output_dir.join(path)).map_err(EngineError::Io)?;
report.deleted_paths.push(path.clone());
}
}
Ok(report)
}
#[expect(
clippy::too_many_arguments,
reason = "generation context is existing domain data"
)]
fn write_core_outputs(
conn: &Connection,
output_dir: &Path,
project_id: &str,
metadata: &ProjectMetadata,
data_dir: &Path,
list_posts: &[PublishedPostSource],
route_manifest: &[crate::render::SitePage],
requested: Option<&HashSet<String>>,
report: &mut GenerationReport,
is_cancelled: &mut impl FnMut() -> bool,
) -> EngineResult<()> {
if requested.is_none() {
write_bundled_site_assets(conn, output_dir, project_id, report)?;
}
let mut outputs = vec![(
"calendar.json".to_string(),
build_calendar_json(
&list_posts
.iter()
.map(|source| source.post.clone())
.collect::<Vec<_>>(),
)?,
)];
for render_language in render_languages(metadata) {
let localized_posts =
localized_sources(conn, data_dir, list_posts, &render_language, metadata)?;
let prefix = if render_language == metadata.main_language.as_deref().unwrap_or("en") {
String::new()
} else {
format!("{render_language}/")
};
let rss = build_rss_xml(metadata, &localized_posts, &render_language);
outputs.push((format!("{prefix}rss.xml"), rss.clone()));
outputs.push((format!("{prefix}feed.xml"), rss));
outputs.push((
format!("{prefix}atom.xml"),
build_atom_xml(metadata, &localized_posts, &render_language),
));
outputs.push((
format!("{prefix}sitemap.xml"),
build_sitemap_xml(metadata, route_manifest, &localized_posts, &render_language),
));
}
for (path, content) in outputs {
if is_cancelled() {
return Err(EngineError::Validation("cancelled".to_string()));
}
if requested.is_none_or(|requested| requested.contains(&path)) {
write_out(conn, output_dir, project_id, &path, &content, report)?;
}
}
Ok(())
}
fn write_out(
conn: &Connection,
output_dir: &Path,
@@ -357,35 +454,100 @@ fn write_out(
Ok(())
}
fn write_pagefind_indexes(
pub fn build_site_search_index(
conn: &Connection,
output_dir: &Path,
project_id: &str,
documents: &[crate::render::PagefindDocument],
report: &mut GenerationReport,
) -> EngineResult<()> {
metadata: &ProjectMetadata,
) -> EngineResult<GenerationReport> {
build_site_search_index_with_progress(
conn,
output_dir,
project_id,
metadata,
|_current, _total, _path| {},
|| false,
)
}
pub fn build_site_search_index_with_progress(
conn: &Connection,
output_dir: &Path,
project_id: &str,
metadata: &ProjectMetadata,
mut on_file: impl FnMut(usize, usize, &str),
mut is_cancelled: impl FnMut() -> bool,
) -> EngineResult<GenerationReport> {
let mut documents = Vec::new();
if output_dir.exists() {
for entry in WalkDir::new(output_dir).into_iter().filter_map(Result::ok) {
if !entry.file_type().is_file() {
continue;
}
let relative_path = entry
.path()
.strip_prefix(output_dir)
.unwrap_or(entry.path())
.to_string_lossy()
.replace('\\', "/");
if !relative_path.ends_with(".html")
|| relative_path.starts_with("pagefind/")
|| relative_path.contains("/pagefind/")
{
continue;
}
let language = render_languages(metadata)
.into_iter()
.find(|language| relative_path.starts_with(&format!("{language}/")))
.unwrap_or_else(|| {
metadata
.main_language
.clone()
.unwrap_or_else(|| "en".into())
});
documents.push(crate::render::PagefindDocument {
language,
url_path: String::new(),
html: std::fs::read_to_string(entry.path()).map_err(EngineError::Io)?,
relative_path,
});
}
}
documents.sort_by(|left, right| left.relative_path.cmp(&right.relative_path));
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.map_err(EngineError::Io)?;
let grouped = documents.iter().fold(
HashMap::<String, Vec<&crate::render::PagefindDocument>>::new(),
BTreeMap::<String, Vec<&crate::render::PagefindDocument>>::new(),
|mut acc, doc| {
acc.entry(doc.language.clone()).or_default().push(doc);
acc
},
);
let mut outputs = Vec::new();
for (language, docs) in grouped {
if is_cancelled() {
return Err(EngineError::Validation("cancelled".to_string()));
}
let config = PagefindServiceConfig::builder()
.keep_index_url(true)
.force_language(language.clone())
.build();
let mut index = PagefindIndex::new(Some(config))
.map_err(|error| EngineError::Parse(error.to_string()))?;
let output_prefix = if language == metadata.main_language.as_deref().unwrap_or("en") {
"pagefind".to_string()
} else {
format!("{language}/pagefind")
};
runtime.block_on(async {
for doc in docs {
if is_cancelled() {
return Err(EngineError::Validation("cancelled".to_string()));
}
index
.add_html_file(Some(doc.relative_path.clone()), None, doc.html.clone())
.await
@@ -396,23 +558,71 @@ fn write_pagefind_indexes(
.await
.map_err(|error| EngineError::Parse(error.to_string()))?;
for file in files {
let relative = file
.filename
.to_string_lossy()
.trim_start_matches('/')
.to_string();
match write_generated_bytes(conn, output_dir, project_id, &relative, &file.contents)
.map_err(|error| EngineError::Parse(error.to_string()))?
{
GeneratedWriteOutcome::Written => report.written_paths.push(relative),
GeneratedWriteOutcome::SkippedUnchanged => report.skipped_paths.push(relative),
}
outputs.push((
format!(
"{output_prefix}/{}",
file.filename.to_string_lossy().trim_start_matches('/')
),
file.contents,
));
}
Ok::<(), EngineError>(())
})?;
}
Ok(())
outputs.sort_by(|left, right| {
left.0
.ends_with("pagefind-entry.json")
.cmp(&right.0.ends_with("pagefind-entry.json"))
.then_with(|| left.0.cmp(&right.0))
});
let total = outputs.len();
let mut report = GenerationReport::default();
let expected = outputs
.iter()
.map(|(relative, _)| relative.clone())
.collect::<HashSet<_>>();
for (index, (relative, contents)) in outputs.into_iter().enumerate() {
if is_cancelled() {
return Err(EngineError::Validation("cancelled".to_string()));
}
match write_generated_bytes(conn, output_dir, project_id, &relative, &contents)
.map_err(|error| EngineError::Parse(error.to_string()))?
{
GeneratedWriteOutcome::Written => report.written_paths.push(relative.clone()),
GeneratedWriteOutcome::SkippedUnchanged => report.skipped_paths.push(relative.clone()),
}
on_file(index + 1, total, &relative);
}
for language in render_languages(metadata) {
let prefix = if language == metadata.main_language.as_deref().unwrap_or("en") {
"pagefind".to_string()
} else {
format!("{language}/pagefind")
};
let index_dir = output_dir.join(&prefix);
if !index_dir.exists() {
continue;
}
for entry in WalkDir::new(&index_dir).into_iter().filter_map(Result::ok) {
if is_cancelled() {
return Err(EngineError::Validation("cancelled".to_string()));
}
if !entry.file_type().is_file() {
continue;
}
let relative = entry
.path()
.strip_prefix(output_dir)
.unwrap_or(entry.path())
.to_string_lossy()
.replace('\\', "/");
if !expected.contains(&relative) {
std::fs::remove_file(entry.path()).map_err(EngineError::Io)?;
report.deleted_paths.push(relative);
}
}
}
Ok(report)
}
fn project_data_dir(output_dir: &Path) -> std::path::PathBuf {
@@ -448,6 +658,7 @@ fn expected_paths_for_sections(
} else {
format!("{language}/")
};
expected.insert(format!("{prefix}rss.xml"));
expected.insert(format!("{prefix}feed.xml"));
expected.insert(format!("{prefix}atom.xml"));
expected.insert(format!("{prefix}sitemap.xml"));
@@ -508,7 +719,7 @@ fn path_matches_sections(path: &str, sections: &HashSet<GenerationSection>) -> b
.unwrap_or(false)
}
fn classify_generated_path(path: &str) -> Option<GenerationSection> {
pub(crate) fn classify_generated_path(path: &str) -> Option<GenerationSection> {
if path.ends_with(".xml") || path.ends_with(".json") {
return Some(GenerationSection::Core);
}
@@ -522,13 +733,29 @@ fn classify_generated_path(path: &str) -> Option<GenerationSection> {
}
match parts.as_slice() {
["index.html"] => Some(GenerationSection::Core),
["index.html"] | ["404.html"] | ["page", _, "index.html"] => Some(GenerationSection::Core),
["category", ..] => Some(GenerationSection::Category),
["tag", ..] => Some(GenerationSection::Tag),
[year, "index.html"] if is_year_segment(year) => Some(GenerationSection::Date),
[year, "page", _, "index.html"] if is_year_segment(year) => Some(GenerationSection::Date),
[year, month, "index.html"] if is_year_segment(year) && is_month_segment(month) => {
Some(GenerationSection::Date)
}
[year, month, "page", _, "index.html"]
if is_year_segment(year) && is_month_segment(month) =>
{
Some(GenerationSection::Date)
}
[year, month, day, "index.html"]
if is_year_segment(year) && is_month_segment(month) && is_day_segment(day) =>
{
Some(GenerationSection::Date)
}
[year, month, day, "page", _, "index.html"]
if is_year_segment(year) && is_month_segment(month) && is_day_segment(day) =>
{
Some(GenerationSection::Date)
}
[year, month, day, _slug, "index.html"]
if is_year_segment(year) && is_month_segment(month) && is_day_segment(day) =>
{
@@ -545,6 +772,8 @@ fn has_language_prefix(parts: &[&str]) -> bool {
&& *first != "category"
&& *first != "tag"
&& (*second == "index.html"
|| *second == "404.html"
|| *second == "page"
|| is_year_segment(second)
|| *second == "category"
|| *second == "tag")
@@ -570,13 +799,7 @@ fn matches_generated_extension(path: &str) -> bool {
}
fn all_sections() -> Vec<GenerationSection> {
vec![
GenerationSection::Core,
GenerationSection::Single,
GenerationSection::Category,
GenerationSection::Tag,
GenerationSection::Date,
]
GenerationSection::ALL.to_vec()
}
fn section_sort_key(section: &GenerationSection) -> u8 {

59
crates/bds-core/src/engine/task.rs
View File

@@ -169,6 +169,32 @@ impl TaskManager {
self.state_changed.notify_all();
}
/// Cancel every active task in a group and release their workers.
pub fn cancel_group(&self, group_id: &str) {
let mut tasks = self.tasks.lock().unwrap();
let now = Instant::now();
for entry in tasks.iter_mut().filter(|task| {
task.group_id.as_deref() == Some(group_id)
&& matches!(task.status, TaskStatus::Running | TaskStatus::Pending)
}) {
entry.cancel_flag.store(true, Ordering::Release);
entry.status = TaskStatus::Cancelled;
entry.finished_at = Some(now);
}
Self::promote_next(&mut tasks, self.max_concurrent);
self.state_changed.notify_all();
}
/// Return the group containing a task, if any.
pub fn group_id(&self, task_id: TaskId) -> Option<String> {
self.tasks
.lock()
.unwrap()
.iter()
.find(|task| task.id == task_id)
.and_then(|task| task.group_id.clone())
}
/// Check whether a task has been cancelled.
pub fn is_cancelled(&self, task_id: TaskId) -> bool {
let tasks = self.tasks.lock().unwrap();
@@ -292,14 +318,19 @@ impl TaskManager {
/// Promote the next queued task to running if capacity allows.
fn promote_next(tasks: &mut [TaskEntry], max_concurrent: usize) {
let running = tasks
while tasks
.iter()
.filter(|t| t.status == TaskStatus::Running)
.count();
if running < max_concurrent
&& let Some(t) = tasks.iter_mut().find(|t| t.status == TaskStatus::Pending)
.filter(|task| task.status == TaskStatus::Running)
.count()
< max_concurrent
{
t.status = TaskStatus::Running;
let Some(task) = tasks
.iter_mut()
.find(|task| task.status == TaskStatus::Pending)
else {
break;
};
task.status = TaskStatus::Running;
}
}
}
@@ -483,6 +514,22 @@ mod tests {
assert_eq!(mgr.status(id), Some(TaskStatus::Completed));
}
#[test]
fn cancelling_group_settles_every_active_task() {
let mgr = TaskManager::new(2);
let first = mgr.submit_grouped("first", "generation-1", "Render Site");
let second = mgr.submit_grouped("second", "generation-1", "Render Site");
let third = mgr.submit_grouped("third", "generation-1", "Render Site");
let unrelated = mgr.submit("unrelated");
mgr.cancel_group("generation-1");
assert_eq!(mgr.status(first), Some(TaskStatus::Cancelled));
assert_eq!(mgr.status(second), Some(TaskStatus::Cancelled));
assert_eq!(mgr.status(third), Some(TaskStatus::Cancelled));
assert_ne!(mgr.status(unrelated), Some(TaskStatus::Cancelled));
}
#[test]
fn report_progress_updates_task() {
let mgr = TaskManager::default();

22
crates/bds-core/src/engine/validate_site.rs
View File

@@ -6,7 +6,7 @@ use walkdir::WalkDir;
use crate::db::queries;
use crate::engine::{EngineError, EngineResult};
use crate::model::{Post, PostStatus};
use crate::model::Post;
use crate::render::build_site_render_artifacts;
use crate::util::file_hash;
@@ -35,7 +35,6 @@ pub fn validate_site(
.map(|page| page.relative_path.clone())
.collect::<HashSet<_>>();
expected.insert("calendar.json".to_string());
expected.insert("rss.xml".to_string());
for language in render_languages(&metadata) {
let prefix = if language
== metadata
@@ -47,6 +46,7 @@ pub fn validate_site(
} else {
format!("{language}/")
};
expected.insert(format!("{prefix}rss.xml"));
expected.insert(format!("{prefix}feed.xml"));
expected.insert(format!("{prefix}atom.xml"));
expected.insert(format!("{prefix}sitemap.xml"));
@@ -124,20 +124,12 @@ fn load_published_posts(
let mut published = Vec::new();
for post in posts
.into_iter()
.filter(|post| post.status == PostStatus::Published)
.filter(crate::engine::generation::has_published_snapshot)
{
let body = if let Some(content) = &post.content {
content.clone()
} else if let Some(content) = &post.published_content {
content.clone()
} else {
let raw =
std::fs::read_to_string(data_dir.join(post.file_path.trim_start_matches('/')))?;
crate::util::frontmatter::read_post_file(&raw)
.map(|(_, body)| body)
.map_err(EngineError::Parse)?
};
published.push((post, body));
if let Some(source) = crate::engine::generation::load_published_post_source(data_dir, post)?
{
published.push((source.post, source.body_markdown));
}
}
Ok(published)
}

5
crates/bds-core/src/render/generation.rs
View File

@@ -96,7 +96,9 @@ pub fn write_generated_bytes(
pub fn build_core_generation_paths(main_language: &str, blog_languages: &[String]) -> Vec<String> {
let mut paths = vec![
"index.html".to_string(),
"404.html".to_string(),
"sitemap.xml".to_string(),
"rss.xml".to_string(),
"feed.xml".to_string(),
"atom.xml".to_string(),
"calendar.json".to_string(),
@@ -105,6 +107,9 @@ pub fn build_core_generation_paths(main_language: &str, blog_languages: &[String
for language in blog_languages {
if language != main_language {
paths.push(format!("{language}/index.html"));
paths.push(format!("{language}/404.html"));
paths.push(format!("{language}/sitemap.xml"));
paths.push(format!("{language}/rss.xml"));
paths.push(format!("{language}/feed.xml"));
paths.push(format!("{language}/atom.xml"));
}

3
crates/bds-core/src/render/mod.rs
View File

@@ -20,7 +20,8 @@ pub use routes::{
};
pub use site::{
PagefindDocument, PreviewRenderResult, SitePage, SiteRenderArtifacts, build_preview_response,
build_site_render_artifacts,
build_site_render_artifacts, build_site_section_render_artifacts,
build_targeted_site_section_render_artifacts,
};
pub use template_lookup::{
RenderCategorySettings, RenderTemplateLookup, TemplateLookupError, resolve_post_template,

127
crates/bds-core/src/render/site.rs
View File

@@ -1,4 +1,4 @@
use std::collections::{BTreeMap, HashMap};
use std::collections::{BTreeMap, HashMap, HashSet};
use std::error::Error;
use std::fs;
use std::path::Path;
@@ -10,6 +10,7 @@ use rayon::prelude::*;
use serde_json::{Value, json};
use crate::db::queries;
use crate::engine::generation::{GenerationSection, classify_generated_path};
use crate::engine::menu::{self, MenuItemKind};
use crate::model::{
CategorySettings, Media, Post, ProjectMetadata, ScriptKind, Tag, Template, TemplateKind,
@@ -56,6 +57,7 @@ pub struct PagefindDocument {
pub struct SiteRenderArtifacts {
pub pages: Vec<SitePage>,
pub pagefind_documents: Vec<PagefindDocument>,
pub route_manifest: Vec<SitePage>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
@@ -110,9 +112,56 @@ pub fn build_site_render_artifacts(
metadata,
published_posts,
false,
None,
None,
)
}
pub fn build_site_section_render_artifacts(
conn: &Connection,
data_dir: &Path,
project_id: &str,
metadata: &ProjectMetadata,
published_posts: &[(Post, String)],
section: GenerationSection,
) -> Result<SiteRenderArtifacts, Box<dyn Error + Send + Sync>> {
build_site_render_artifacts_with_mode(
conn,
data_dir,
project_id,
metadata,
published_posts,
false,
Some(section),
None,
)
}
pub fn build_targeted_site_section_render_artifacts(
conn: &Connection,
data_dir: &Path,
project_id: &str,
metadata: &ProjectMetadata,
published_posts: &[(Post, String)],
section: GenerationSection,
requested_paths: &HashSet<String>,
) -> Result<SiteRenderArtifacts, Box<dyn Error + Send + Sync>> {
build_site_render_artifacts_with_mode(
conn,
data_dir,
project_id,
metadata,
published_posts,
false,
Some(section),
Some(requested_paths),
)
}
#[expect(
clippy::too_many_arguments,
reason = "the shared renderer accepts optional section and path filters"
)]
fn build_site_render_artifacts_with_mode(
conn: &Connection,
data_dir: &Path,
@@ -120,6 +169,8 @@ fn build_site_render_artifacts_with_mode(
metadata: &ProjectMetadata,
published_posts: &[(Post, String)],
is_preview: bool,
section: Option<GenerationSection>,
requested_paths: Option<&HashSet<String>>,
) -> Result<SiteRenderArtifacts, Box<dyn Error + Send + Sync>> {
let bundle = load_template_bundle(conn, data_dir, project_id)?;
let main_language = main_language(metadata).to_string();
@@ -144,10 +195,24 @@ fn build_site_render_artifacts_with_mode(
&tags,
&category_settings,
);
artifacts
.route_manifest
.extend(routes.iter().map(|route| SitePage {
language: language.clone(),
relative_path: route.relative_path.clone(),
url_path: route.url_path.clone(),
html: String::new(),
}));
let post_data_json_by_id = build_post_data_json_by_id(&localized_posts);
let menu_items = build_menu_items(data_dir, &language, &main_language)?;
let rendered_list_pages = routes
.par_iter()
.filter(|route| {
section.is_none_or(|section| {
classify_generated_path(&route.relative_path) == Some(section)
}) && requested_paths
.is_none_or(|requested| requested.contains(&route.relative_path))
})
.map(|route| {
render_list_route(
route,
@@ -180,9 +245,46 @@ fn build_site_render_artifacts_with_mode(
artifacts.pages.push(page);
}
if section.is_none_or(|section| section == GenerationSection::Core) {
let relative_path = if language == main_language {
"404.html".to_string()
} else {
format!("{language}/404.html")
};
if requested_paths.is_none_or(|requested| requested.contains(&relative_path)) {
let url_path = format!("/{}", relative_path.trim_end_matches(".html"));
artifacts.pages.push(SitePage {
language: language.clone(),
relative_path,
url_path: url_path.clone(),
html: render_not_found_route(
&bundle,
metadata,
&language,
&url_path,
&menu_items,
)?,
});
}
}
let canonical_map =
canonical_post_path_by_slug(&localized_posts, &language, &main_language);
for record in &localized_posts {
let canonical_path = build_canonical_post_path(&record.post, &language, &main_language);
let relative_path = format!("{}/index.html", canonical_path.trim_start_matches('/'));
artifacts.route_manifest.push(SitePage {
language: language.clone(),
relative_path: relative_path.clone(),
url_path: canonical_path.clone(),
html: String::new(),
});
if section.is_some_and(|section| section != GenerationSection::Single) {
continue;
}
if requested_paths.is_some_and(|requested| !requested.contains(&relative_path)) {
continue;
}
let html = render_post_route(
conn,
metadata,
@@ -199,8 +301,6 @@ fn build_site_render_artifacts_with_mode(
&bundle,
is_preview,
)?;
let canonical_path = build_canonical_post_path(&record.post, &language, &main_language);
let relative_path = format!("{}/index.html", canonical_path.trim_start_matches('/'));
artifacts.pagefind_documents.push(PagefindDocument {
language: language.clone(),
relative_path: relative_path.clone(),
@@ -234,6 +334,8 @@ pub fn build_preview_response(
metadata,
published_posts,
true,
None,
None,
)?;
let normalized = normalize_request_path(requested_path);
if let Some(page) = artifacts
@@ -464,6 +566,7 @@ fn build_language_routes(
let mut tag_posts: BTreeMap<String, Vec<RenderPostRecord>> = BTreeMap::new();
let mut year_posts: BTreeMap<i32, Vec<RenderPostRecord>> = BTreeMap::new();
let mut month_posts: BTreeMap<(i32, u32), Vec<RenderPostRecord>> = BTreeMap::new();
let mut day_posts: BTreeMap<(i32, u32, u32), Vec<RenderPostRecord>> = BTreeMap::new();
for record in posts {
for category in &record.post.categories {
@@ -490,6 +593,10 @@ fn build_language_routes(
.entry((timestamp.year(), timestamp.month()))
.or_default()
.push(record.clone());
day_posts
.entry((timestamp.year(), timestamp.month(), timestamp.day()))
.or_default()
.push(record.clone());
}
}
@@ -552,6 +659,20 @@ fn build_language_routes(
));
}
for ((year, month, day), records) in day_posts {
routes.extend(paginated_route_specs(
&records,
per_page,
format!(
"{}/{year}/{month:02}/{day:02}",
language_root_prefix(language, metadata)
),
format!("{} {year}-{month:02}-{day:02}", metadata.name),
Some(json!({"kind": "day", "year": year, "month": month, "day": day})),
None,
));
}
routes
}

176
crates/bds-core/tests/m4_generation_engine.rs
View File

@@ -4,8 +4,9 @@ use bds_core::db::Database;
use bds_core::db::queries::project::insert_project;
use bds_core::db::queries::template::insert_template;
use bds_core::engine::generation::{
PublishedPostSource, apply_validation_sections, generate_starter_site,
load_published_post_source, sections_from_validation_report,
GenerationSection, PublishedPostSource, apply_validation_section_with_progress,
apply_validation_sections, build_site_search_index, generate_starter_site,
load_published_post_source, render_site_section_with_progress, sections_from_validation_report,
};
use bds_core::engine::meta::write_category_meta_json;
use bds_core::engine::validate_site::validate_site;
@@ -99,6 +100,14 @@ fn setup() -> (Database, TempDir) {
(db, dir)
}
fn write_published_snapshot(dir: &TempDir, post: &mut Post, body: &str) {
post.file_path = format!("posts/{}.md", post.slug);
let frontmatter = bds_core::util::frontmatter::PostFrontmatter::from_post(post).to_yaml();
let file = bds_core::util::frontmatter::format_frontmatter(&frontmatter, body);
std::fs::create_dir_all(dir.path().join("posts")).unwrap();
std::fs::write(dir.path().join(&post.file_path), file).unwrap();
}
#[test]
fn reopened_draft_generation_uses_last_published_file() {
let (_db, dir) = setup();
@@ -117,17 +126,47 @@ fn reopened_draft_generation_uses_last_published_file() {
assert_eq!(source.body_markdown, "Published body");
}
#[test]
fn validation_keeps_reopened_draft_published_snapshots() {
let (db, dir) = setup();
let mut post = make_post("reopened", 1_710_000_000_000);
post.status = PostStatus::Draft;
post.content = Some("Unpublished draft body".into());
write_published_snapshot(&dir, &mut post, "Published body");
bds_core::db::queries::post::insert_post(db.conn(), &post).unwrap();
let source = load_published_post_source(dir.path(), post)
.unwrap()
.unwrap();
generate_starter_site(
db.conn(),
dir.path(),
"p1",
&make_metadata(),
&[source],
"en",
)
.unwrap();
let validation = validate_site(db.conn(), dir.path(), "p1").unwrap();
assert!(validation.missing_pages.is_empty());
assert!(validation.extra_pages.is_empty());
assert!(validation.stale_pages.is_empty());
}
#[test]
fn generation_engine_writes_core_and_single_post_artifacts() {
let (db, dir) = setup();
let metadata = make_metadata();
let hello = make_post("hello", 1_710_000_000_000);
let next = make_post("next", 1_710_086_400_000);
let posts = vec![
PublishedPostSource {
post: make_post("hello", 1_710_000_000_000),
post: hello,
body_markdown: "Hello **world**".into(),
},
PublishedPostSource {
post: make_post("next", 1_710_086_400_000),
post: next,
body_markdown: "Next post".into(),
},
];
@@ -141,6 +180,12 @@ fn generation_engine_writes_core_and_single_post_artifacts() {
assert!(report.written_paths.contains(&"feed.xml".to_string()));
assert!(report.written_paths.contains(&"atom.xml".to_string()));
assert!(report.written_paths.contains(&"sitemap.xml".to_string()));
assert!(report.written_paths.contains(&"404.html".to_string()));
assert!(
report
.written_paths
.contains(&"2024/03/09/index.html".to_string())
);
assert!(
report
.written_paths
@@ -180,6 +225,110 @@ fn generation_engine_writes_core_and_single_post_artifacts() {
assert!(sitemap.contains("https://example.com/category/article"));
}
#[test]
fn section_generation_reports_its_urls_and_defers_pagefind() {
let (db, dir) = setup();
let metadata = make_metadata();
let posts = vec![PublishedPostSource {
post: make_post("hello", 1_710_000_000_000),
body_markdown: "Hello **world**".into(),
}];
let mut urls = Vec::new();
let report = render_site_section_with_progress(
db.conn(),
dir.path(),
"p1",
&metadata,
&posts,
GenerationSection::Single,
|current, total, url| urls.push((current, total, url.to_string())),
|| false,
)
.unwrap();
assert_eq!(urls, vec![(1, 1, "/2024/03/09/hello".to_string())]);
assert_eq!(report.written_paths, vec!["2024/03/09/hello/index.html"]);
assert!(!dir.path().join("pagefind").exists());
let index_report = build_site_search_index(db.conn(), dir.path(), "p1", &metadata).unwrap();
assert!(!index_report.written_paths.is_empty());
assert!(
dir.path().join("pagefind/pagefind-ui.js").exists(),
"pagefind outputs: {:?}",
index_report.written_paths
);
let old_fragment = index_report
.written_paths
.iter()
.find(|path| path.contains("/fragment/"))
.cloned()
.unwrap();
let changed_posts = vec![PublishedPostSource {
post: make_post("hello", 1_710_000_000_000),
body_markdown: "Changed body".into(),
}];
render_site_section_with_progress(
db.conn(),
dir.path(),
"p1",
&metadata,
&changed_posts,
GenerationSection::Single,
|_current, _total, _url| {},
|| false,
)
.unwrap();
let rebuilt = build_site_search_index(db.conn(), dir.path(), "p1", &metadata).unwrap();
assert!(rebuilt.deleted_paths.contains(&old_fragment));
assert!(!dir.path().join(old_fragment).exists());
}
#[test]
fn validation_apply_rewrites_only_the_reported_urls() {
let (db, dir) = setup();
let metadata = make_metadata();
let posts = vec![
PublishedPostSource {
post: make_post("hello", 1_710_000_000_000),
body_markdown: "Hello **world**".into(),
},
PublishedPostSource {
post: make_post("next", 1_710_086_400_000),
body_markdown: "Next post".into(),
},
];
generate_starter_site(db.conn(), dir.path(), "p1", &metadata, &posts, "en").unwrap();
std::fs::write(dir.path().join("2024/03/09/hello/index.html"), "tampered").unwrap();
let validation = bds_core::engine::validate_site::SiteValidationReport {
stale_pages: vec!["2024/03/09/hello/index.html".into()],
..Default::default()
};
let mut urls = Vec::new();
let report = apply_validation_section_with_progress(
db.conn(),
dir.path(),
"p1",
&metadata,
&posts,
&validation,
GenerationSection::Single,
|current, total, url| urls.push((current, total, url.to_string())),
|| false,
)
.unwrap();
assert_eq!(urls, vec![(1, 1, "/2024/03/09/hello".to_string())]);
assert_eq!(report.written_paths, vec!["2024/03/09/hello/index.html"]);
assert!(
!report
.skipped_paths
.contains(&"2024/03/10/next/index.html".to_string())
);
}
#[test]
fn multilingual_generation_writes_language_aware_atom_and_sitemap_routes() {
let (db, dir) = setup();
@@ -195,7 +344,14 @@ fn multilingual_generation_writes_language_aware_atom_and_sitemap_routes() {
generate_starter_site(db.conn(), dir.path(), "p1", &metadata, &posts, "de").unwrap();
assert!(report.written_paths.contains(&"en/atom.xml".to_string()));
assert!(report.written_paths.contains(&"en/rss.xml".to_string()));
assert!(report.written_paths.contains(&"en/sitemap.xml".to_string()));
assert!(report.written_paths.contains(&"en/404.html".to_string()));
assert!(
report
.written_paths
.contains(&"en/pagefind/pagefind-ui.js".to_string())
);
let atom = std::fs::read_to_string(dir.path().join("en/atom.xml")).unwrap();
assert!(
@@ -340,7 +496,8 @@ fn generation_engine_skips_unchanged_outputs_on_second_run() {
fn site_validation_detects_stale_and_missing_outputs() {
let (db, dir) = setup();
let metadata = make_metadata();
let post = make_post("hello", 1_710_000_000_000);
let mut post = make_post("hello", 1_710_000_000_000);
write_published_snapshot(&dir, &mut post, "Hello **world**");
bds_core::db::queries::post::insert_post(db.conn(), &post).unwrap();
let posts = vec![PublishedPostSource {
post,
@@ -362,7 +519,8 @@ fn site_validation_detects_stale_and_missing_outputs() {
fn apply_validation_repairs_core_section_outputs() {
let (db, dir) = setup();
let metadata = make_metadata();
let post = make_post("hello", 1_710_000_000_000);
let mut post = make_post("hello", 1_710_000_000_000);
write_published_snapshot(&dir, &mut post, "Hello **world**");
bds_core::db::queries::post::insert_post(db.conn(), &post).unwrap();
let posts = vec![PublishedPostSource {
post,
@@ -390,7 +548,8 @@ fn apply_validation_repairs_core_section_outputs() {
fn apply_validation_removes_extra_section_outputs() {
let (db, dir) = setup();
let metadata = make_metadata();
let post = make_post("hello", 1_710_000_000_000);
let mut post = make_post("hello", 1_710_000_000_000);
write_published_snapshot(&dir, &mut post, "Hello **world**");
bds_core::db::queries::post::insert_post(db.conn(), &post).unwrap();
let posts = vec![PublishedPostSource {
post,
@@ -426,7 +585,8 @@ fn apply_validation_removes_extra_section_outputs() {
fn site_validation_uses_html_output_directory_when_present() {
let (db, dir) = setup();
let metadata = make_metadata();
let post = make_post("hello", 1_710_000_000_000);
let mut post = make_post("hello", 1_710_000_000_000);
write_published_snapshot(&dir, &mut post, "Hello **world**");
bds_core::db::queries::post::insert_post(db.conn(), &post).unwrap();
let posts = vec![PublishedPostSource {
post,

5
crates/bds-core/tests/m4_generation_primitives.rs
View File

@@ -114,11 +114,16 @@ fn generated_write_rewrites_stale_file_even_when_db_hash_matches() {
fn core_generation_paths_include_language_prefixed_variants() {
let paths = build_core_generation_paths("en", &["en".into(), "de".into(), "fr".into()]);
assert!(paths.contains(&"index.html".to_string()));
assert!(paths.contains(&"404.html".to_string()));
assert!(paths.contains(&"sitemap.xml".to_string()));
assert!(paths.contains(&"rss.xml".to_string()));
assert!(paths.contains(&"feed.xml".to_string()));
assert!(paths.contains(&"atom.xml".to_string()));
assert!(paths.contains(&"calendar.json".to_string()));
assert!(paths.contains(&"de/index.html".to_string()));
assert!(paths.contains(&"de/404.html".to_string()));
assert!(paths.contains(&"de/sitemap.xml".to_string()));
assert!(paths.contains(&"de/rss.xml".to_string()));
assert!(paths.contains(&"de/feed.xml".to_string()));
assert!(paths.contains(&"de/atom.xml".to_string()));
assert!(paths.contains(&"fr/index.html".to_string()));