hugo/RuDS
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

Add forced full-site rendering

Browse Source
This commit is contained in:
Chili Palmer
2026-07-23 10:23:02 +02:00
parent d87e80f96b
commit 2844ed5a47
21 changed files with 611 additions and 122 deletions
Download Patch File Download Diff File Expand all files Collapse all files

210
crates/bds-core/src/engine/generation.rs
View File

@@ -15,7 +15,8 @@ use crate::model::{CategorySettings, Post, ProjectMetadata};
use crate::render::{
GeneratedWriteOutcome, PostLanguageVariant, build_calendar_json, build_canonical_post_path,
build_site_section_render_artifacts, build_targeted_site_section_render_artifacts,
select_post_language_variant, write_generated_bytes, write_generated_file,
select_post_language_variant, write_generated_bytes, write_generated_bytes_forced,
write_generated_file, write_generated_file_forced, write_generated_file_verified,
};
#[derive(Debug, Clone)]
@@ -102,18 +103,24 @@ pub fn generate_starter_site(
)
}
/// Forget stored generated-file hashes so the next render writes every
/// artifact while repopulating the cache with its current content hash.
pub fn clear_generation_cache(conn: &Connection, project_id: &str) -> EngineResult<usize> {
use crate::db::schema::generated_file_hashes;
use diesel::prelude::*;
Ok(conn.with(|connection| {
diesel::delete(
generated_file_hashes::table.filter(generated_file_hashes::project_id.eq(project_id)),
)
.execute(connection)
})?)
pub fn generate_starter_site_forced(
conn: &Connection,
output_dir: &Path,
project_id: &str,
metadata: &ProjectMetadata,
posts: &[PublishedPostSource],
language: &str,
) -> EngineResult<GenerationReport> {
generate_starter_site_with_progress_mode(
conn,
output_dir,
project_id,
metadata,
posts,
language,
true,
|_current, _total, _path| {},
)
}
pub fn generate_starter_site_with_progress(
@@ -123,23 +130,49 @@ pub fn generate_starter_site_with_progress(
metadata: &ProjectMetadata,
posts: &[PublishedPostSource],
_language: &str,
on_page: impl FnMut(usize, usize, &str),
) -> EngineResult<GenerationReport> {
generate_starter_site_with_progress_mode(
conn, output_dir, project_id, metadata, posts, _language, false, on_page,
)
}
#[expect(
clippy::too_many_arguments,
reason = "full generation adds write mode to the existing generation context"
)]
fn generate_starter_site_with_progress_mode(
conn: &Connection,
output_dir: &Path,
project_id: &str,
metadata: &ProjectMetadata,
posts: &[PublishedPostSource],
_language: &str,
force: bool,
mut on_page: impl FnMut(usize, usize, &str),
) -> EngineResult<GenerationReport> {
let mut report = GenerationReport::default();
for section in GenerationSection::ALL {
report.append(render_site_section_with_progress(
report.append(render_site_section_with_progress_mode(
conn,
output_dir,
project_id,
metadata,
posts,
section,
force,
&mut on_page,
|| false,
)?);
}
report.append(build_site_search_index(
conn, output_dir, project_id, metadata,
report.append(build_site_search_index_with_progress_mode(
conn,
output_dir,
project_id,
metadata,
force,
|_current, _total, _path| {},
|| false,
)?);
Ok(report)
}
@@ -157,6 +190,61 @@ pub fn render_site_section_with_progress(
section: GenerationSection,
mut on_page: impl FnMut(usize, usize, &str),
mut is_cancelled: impl FnMut() -> bool,
) -> EngineResult<GenerationReport> {
render_site_section_with_progress_mode(
conn,
output_dir,
project_id,
metadata,
posts,
section,
false,
&mut on_page,
&mut is_cancelled,
)
}
#[expect(
clippy::too_many_arguments,
reason = "forced section rendering adds write mode to the existing generation context"
)]
pub fn render_site_section_forced_with_progress(
conn: &Connection,
output_dir: &Path,
project_id: &str,
metadata: &ProjectMetadata,
posts: &[PublishedPostSource],
section: GenerationSection,
mut on_page: impl FnMut(usize, usize, &str),
mut is_cancelled: impl FnMut() -> bool,
) -> EngineResult<GenerationReport> {
render_site_section_with_progress_mode(
conn,
output_dir,
project_id,
metadata,
posts,
section,
true,
&mut on_page,
&mut is_cancelled,
)
}
#[expect(
clippy::too_many_arguments,
reason = "section rendering uses the existing generation context, write mode, and callbacks"
)]
fn render_site_section_with_progress_mode(
conn: &Connection,
output_dir: &Path,
project_id: &str,
metadata: &ProjectMetadata,
posts: &[PublishedPostSource],
section: GenerationSection,
force: bool,
mut on_page: impl FnMut(usize, usize, &str),
mut is_cancelled: impl FnMut() -> bool,
) -> EngineResult<GenerationReport> {
if is_cancelled() {
return Err(EngineError::Validation("cancelled".to_string()));
@@ -188,6 +276,8 @@ pub fn render_site_section_with_progress(
&page.relative_path,
&page.html,
&mut report,
force,
false,
)?;
on_page(index + 1, total_pages, &page.url_path);
}
@@ -204,6 +294,8 @@ pub fn render_site_section_with_progress(
None,
&mut report,
&mut is_cancelled,
force,
false,
)?;
}
Ok(report)
@@ -345,6 +437,8 @@ pub fn apply_validation_section_with_progress(
&page.relative_path,
&page.html,
&mut report,
false,
true,
)?;
on_page(index + 1, total_pages, &page.url_path);
}
@@ -361,6 +455,8 @@ pub fn apply_validation_section_with_progress(
(!fallback).then_some(&requested),
&mut report,
&mut is_cancelled,
false,
true,
)?;
}
@@ -415,9 +511,11 @@ fn write_core_outputs(
requested: Option<&HashSet<String>>,
report: &mut GenerationReport,
is_cancelled: &mut impl FnMut() -> bool,
force: bool,
verify_output: bool,
) -> EngineResult<()> {
if requested.is_none() {
write_bundled_site_assets(conn, output_dir, project_id, report)?;
write_bundled_site_assets(conn, output_dir, project_id, report, force)?;
}
let mut outputs = vec![(
"calendar.json".to_string(),
@@ -483,12 +581,25 @@ fn write_core_outputs(
return Err(EngineError::Validation("cancelled".to_string()));
}
if requested.is_none_or(|requested| requested.contains(&path)) {
write_out(conn, output_dir, project_id, &path, &content, report)?;
write_out(
conn,
output_dir,
project_id,
&path,
&content,
report,
force,
verify_output,
)?;
}
}
Ok(())
}
#[expect(
clippy::too_many_arguments,
reason = "generated output needs its render context, report, and write mode"
)]
fn write_out(
conn: &Connection,
output_dir: &Path,
@@ -496,10 +607,18 @@ fn write_out(
relative_path: &str,
content: &str,
report: &mut GenerationReport,
force: bool,
verify_output: bool,
) -> EngineResult<()> {
match write_generated_file(conn, output_dir, project_id, relative_path, content)
.map_err(|error| EngineError::Parse(error.to_string()))?
{
let outcome = if force {
write_generated_file_forced(conn, output_dir, project_id, relative_path, content)
} else if verify_output {
write_generated_file_verified(conn, output_dir, project_id, relative_path, content)
} else {
write_generated_file(conn, output_dir, project_id, relative_path, content)
}
.map_err(|error| EngineError::Parse(error.to_string()))?;
match outcome {
GeneratedWriteOutcome::Written => report.written_paths.push(relative_path.to_string()),
GeneratedWriteOutcome::SkippedUnchanged => {
report.skipped_paths.push(relative_path.to_string())
@@ -524,6 +643,25 @@ pub fn build_site_search_index(
)
}
pub fn build_site_search_index_forced_with_progress(
conn: &Connection,
output_dir: &Path,
project_id: &str,
metadata: &ProjectMetadata,
on_file: impl FnMut(usize, usize, &str),
is_cancelled: impl FnMut() -> bool,
) -> EngineResult<GenerationReport> {
build_site_search_index_with_progress_mode(
conn,
output_dir,
project_id,
metadata,
true,
on_file,
is_cancelled,
)
}
pub fn build_site_search_index_with_progress(
conn: &Connection,
output_dir: &Path,
@@ -531,6 +669,26 @@ pub fn build_site_search_index_with_progress(
metadata: &ProjectMetadata,
mut on_file: impl FnMut(usize, usize, &str),
mut is_cancelled: impl FnMut() -> bool,
) -> EngineResult<GenerationReport> {
build_site_search_index_with_progress_mode(
conn,
output_dir,
project_id,
metadata,
false,
&mut on_file,
&mut is_cancelled,
)
}
fn build_site_search_index_with_progress_mode(
conn: &Connection,
output_dir: &Path,
project_id: &str,
metadata: &ProjectMetadata,
force: bool,
mut on_file: impl FnMut(usize, usize, &str),
mut is_cancelled: impl FnMut() -> bool,
) -> EngineResult<GenerationReport> {
let mut documents = Vec::new();
if output_dir.exists() {
@@ -639,9 +797,13 @@ pub fn build_site_search_index_with_progress(
if is_cancelled() {
return Err(EngineError::Validation("cancelled".to_string()));
}
match write_generated_bytes(conn, output_dir, project_id, &relative, &contents)
.map_err(|error| EngineError::Parse(error.to_string()))?
{
let outcome = if force {
write_generated_bytes_forced(conn, output_dir, project_id, &relative, &contents)
} else {
write_generated_bytes(conn, output_dir, project_id, &relative, &contents)
}
.map_err(|error| EngineError::Parse(error.to_string()))?;
match outcome {
GeneratedWriteOutcome::Written => report.written_paths.push(relative.clone()),
GeneratedWriteOutcome::SkippedUnchanged => report.skipped_paths.push(relative.clone()),
}

29
crates/bds-core/src/engine/site_assets.rs
View File

@@ -257,17 +257,28 @@ pub(crate) fn write_bundled_site_assets(
output_dir: &Path,
project_id: &str,
report: &mut crate::engine::generation::GenerationReport,
force: bool,
) -> EngineResult<()> {
for asset in BUNDLED_SITE_ASSETS {
match write_generated_bytes(
conn,
output_dir,
project_id,
asset.relative_path,
asset.bytes,
)
.map_err(|error| EngineError::Parse(error.to_string()))?
{
let outcome = if force {
crate::render::write_generated_bytes_forced(
conn,
output_dir,
project_id,
asset.relative_path,
asset.bytes,
)
} else {
write_generated_bytes(
conn,
output_dir,
project_id,
asset.relative_path,
asset.bytes,
)
}
.map_err(|error| EngineError::Parse(error.to_string()))?;
match outcome {
GeneratedWriteOutcome::Written => {
report.written_paths.push(asset.relative_path.to_string())
}

86
crates/bds-core/src/render/generation.rs
View File

@@ -29,13 +29,70 @@ pub fn write_generated_file(
project_id: &str,
relative_path: &str,
content: &str,
) -> Result<GeneratedWriteOutcome, Box<dyn std::error::Error + Send + Sync>> {
write_generated_file_with_mode(
conn,
output_dir,
project_id,
relative_path,
content,
false,
false,
)
}
pub fn write_generated_file_verified(
conn: &Connection,
output_dir: &Path,
project_id: &str,
relative_path: &str,
content: &str,
) -> Result<GeneratedWriteOutcome, Box<dyn std::error::Error + Send + Sync>> {
write_generated_file_with_mode(
conn,
output_dir,
project_id,
relative_path,
content,
false,
true,
)
}
pub fn write_generated_file_forced(
conn: &Connection,
output_dir: &Path,
project_id: &str,
relative_path: &str,
content: &str,
) -> Result<GeneratedWriteOutcome, Box<dyn std::error::Error + Send + Sync>> {
write_generated_file_with_mode(
conn,
output_dir,
project_id,
relative_path,
content,
true,
false,
)
}
fn write_generated_file_with_mode(
conn: &Connection,
output_dir: &Path,
project_id: &str,
relative_path: &str,
content: &str,
force: bool,
verify_output: bool,
) -> Result<GeneratedWriteOutcome, Box<dyn std::error::Error + Send + Sync>> {
let hash = content_hash(content.as_bytes());
let target_path = output_dir.join(relative_path);
if let Ok(existing) = qhash::get_generated_file_hash(conn, project_id, relative_path)
if !force
&& let Ok(existing) = qhash::get_generated_file_hash(conn, project_id, relative_path)
&& existing.content_hash == hash
&& target_path.exists()
&& file_hash(&target_path)? == hash
&& (!verify_output || file_hash(&target_path)? == hash)
{
return Ok(GeneratedWriteOutcome::SkippedUnchanged);
}
@@ -64,13 +121,34 @@ pub fn write_generated_bytes(
project_id: &str,
relative_path: &str,
content: &[u8],
) -> Result<GeneratedWriteOutcome, Box<dyn std::error::Error + Send + Sync>> {
write_generated_bytes_with_force(conn, output_dir, project_id, relative_path, content, false)
}
pub fn write_generated_bytes_forced(
conn: &Connection,
output_dir: &Path,
project_id: &str,
relative_path: &str,
content: &[u8],
) -> Result<GeneratedWriteOutcome, Box<dyn std::error::Error + Send + Sync>> {
write_generated_bytes_with_force(conn, output_dir, project_id, relative_path, content, true)
}
fn write_generated_bytes_with_force(
conn: &Connection,
output_dir: &Path,
project_id: &str,
relative_path: &str,
content: &[u8],
force: bool,
) -> Result<GeneratedWriteOutcome, Box<dyn std::error::Error + Send + Sync>> {
let hash = content_hash(content);
let target_path = output_dir.join(relative_path);
if let Ok(existing) = qhash::get_generated_file_hash(conn, project_id, relative_path)
if !force
&& let Ok(existing) = qhash::get_generated_file_hash(conn, project_id, relative_path)
&& existing.content_hash == hash
&& target_path.exists()
&& file_hash(&target_path)? == hash
{
return Ok(GeneratedWriteOutcome::SkippedUnchanged);
}

3
crates/bds-core/src/render/mod.rs
View File

@@ -8,7 +8,8 @@ mod template_lookup;
pub use generation::{
CalendarArchiveData, GeneratedWriteOutcome, build_calendar_json, build_core_generation_paths,
write_generated_bytes, write_generated_file,
write_generated_bytes, write_generated_bytes_forced, write_generated_file,
write_generated_file_forced, write_generated_file_verified,
};
pub use markdown::render_markdown_to_html;
pub use page_renderer::{RenderError, render_liquid_template};

58
crates/bds-core/tests/m4_generation_engine.rs
View File

@@ -6,7 +6,8 @@ use bds_core::db::queries::template::insert_template;
use bds_core::engine::generation::{
GenerationSection, PublishedPostSource, apply_validation_section_with_progress,
apply_validation_sections, build_site_search_index, generate_starter_site,
load_published_post_source, render_site_section_with_progress, sections_from_validation_report,
generate_starter_site_forced, load_published_post_source, render_site_section_with_progress,
sections_from_validation_report,
};
use bds_core::engine::meta::write_category_meta_json;
use bds_core::engine::validate_site::validate_site;
@@ -742,6 +743,61 @@ fn generation_engine_skips_unchanged_outputs_on_second_run() {
);
}
#[test]
fn forced_generation_rewrites_unchanged_outputs_without_erasing_unrelated_cache_records() {
let (db, dir) = setup();
let metadata = make_metadata();
let posts = vec![PublishedPostSource {
post: make_post("hello", 1_710_000_000_000),
body_markdown: "Hello **world**".into(),
}];
generate_starter_site(db.conn(), dir.path(), "p1", &metadata, &posts, "en").unwrap();
let unchanged =
generate_starter_site(db.conn(), dir.path(), "p1", &metadata, &posts, "en").unwrap();
assert!(unchanged.skipped_paths.contains(&"index.html".to_string()));
std::fs::write(dir.path().join("index.html"), "tampered").unwrap();
let normal_after_drift =
generate_starter_site(db.conn(), dir.path(), "p1", &metadata, &posts, "en").unwrap();
assert!(
normal_after_drift
.skipped_paths
.contains(&"index.html".to_string())
);
assert_eq!(
std::fs::read_to_string(dir.path().join("index.html")).unwrap(),
"tampered"
);
bds_core::db::queries::generated_file_hash::upsert_generated_file_hash(
db.conn(),
&bds_core::model::GeneratedFileHash {
project_id: "p1".into(),
relative_path: "legacy-output.txt".into(),
content_hash: "legacy".into(),
updated_at: 42,
},
)
.unwrap();
let forced =
generate_starter_site_forced(db.conn(), dir.path(), "p1", &metadata, &posts, "en").unwrap();
assert!(forced.written_paths.contains(&"index.html".to_string()));
assert!(forced.skipped_paths.is_empty());
assert_ne!(
std::fs::read_to_string(dir.path().join("index.html")).unwrap(),
"tampered"
);
assert!(
bds_core::db::queries::generated_file_hash::get_generated_file_hash(
db.conn(),
"p1",
"legacy-output.txt",
)
.is_ok()
);
}
#[test]
fn site_validation_matches_bds2_index_route_comparison() {
let (db, dir) = setup();

21
crates/bds-core/tests/m4_generation_primitives.rs
View File

@@ -6,6 +6,7 @@ use bds_core::db::queries::project::insert_project;
use bds_core::model::{Post, PostStatus, Project};
use bds_core::render::{
GeneratedWriteOutcome, build_calendar_json, build_core_generation_paths, write_generated_file,
write_generated_file_forced,
};
use tempfile::TempDir;
@@ -95,7 +96,7 @@ fn generated_write_rewrites_missing_file_even_when_hash_matches() {
}
#[test]
fn generated_write_rewrites_stale_file_even_when_db_hash_matches() {
fn generated_write_trusts_the_cached_hash_like_bds2() {
let (db, dir) = setup();
let first = write_generated_file(db.conn(), dir.path(), "p1", "index.html", "hello").unwrap();
@@ -103,7 +104,23 @@ fn generated_write_rewrites_stale_file_even_when_db_hash_matches() {
let second = write_generated_file(db.conn(), dir.path(), "p1", "index.html", "hello").unwrap();
assert_eq!(first, GeneratedWriteOutcome::Written);
assert_eq!(second, GeneratedWriteOutcome::Written);
assert_eq!(second, GeneratedWriteOutcome::SkippedUnchanged);
assert_eq!(
fs::read_to_string(dir.path().join("index.html")).unwrap(),
"tampered"
);
}
#[test]
fn forced_generated_write_ignores_the_cached_hash() {
let (db, dir) = setup();
write_generated_file(db.conn(), dir.path(), "p1", "index.html", "hello").unwrap();
fs::write(dir.path().join("index.html"), "tampered").unwrap();
let forced =
write_generated_file_forced(db.conn(), dir.path(), "p1", "index.html", "hello").unwrap();
assert_eq!(forced, GeneratedWriteOutcome::Written);
assert_eq!(
fs::read_to_string(dir.path().join("index.html")).unwrap(),
"hello"