From 473c48937d7bfa67a92e017189575266eb286b29 Mon Sep 17 00:00:00 2001 From: Chili Palmer Date: Fri, 24 Jul 2026 06:22:45 +0200 Subject: [PATCH] fix: align rendering process with bds2 --- README.md | 2 +- .../src/db/queries/generated_file_hash.rs | 58 ++ crates/bds-core/src/db/queries/post_link.rs | 21 +- crates/bds-core/src/db/queries/post_media.rs | 31 + .../src/db/queries/post_translation.rs | 25 + crates/bds-core/src/engine/generation.rs | 421 ++++++------ crates/bds-core/src/engine/preview.rs | 12 +- crates/bds-core/src/engine/site_assets.rs | 47 +- crates/bds-core/src/model/generation.rs | 3 +- crates/bds-core/src/render/generation.rs | 114 +++- crates/bds-core/src/render/mod.rs | 9 +- crates/bds-core/src/render/page_renderer.rs | 129 +++- crates/bds-core/src/render/site.rs | 601 +++++++++++------- crates/bds-ui/src/app/tasks.rs | 76 +-- crates/bds-ui/src/views/panel.rs | 54 +- 15 files changed, 1022 insertions(+), 581 deletions(-) diff --git a/README.md b/README.md index 2a953fb..60e66d3 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ The project is under active development. Core blogging workflows are broadly ava - Local MCP automation over stdio or a localhost-only stateless HTTP endpoint, with project resources, read/search/count tools, uniquely identified inert write proposals, clean duplicate-pending rejection, explicit desktop approval, and opt-in Claude Code/Copilot configuration. - A fully localized Ratatui terminal workspace, available locally through `bds-cli tui`/`BDS_MODE=tui` and remotely through authenticated SSH shell sessions, with shared post/template/script editing and publishing, project/search/command overlays, settings, tags, Git, reports, task progress, live multi-client locale updates, and airplane-mode AI gating. - `bds-cli server` hosting the shared application engines over a loopback-by-default, public-key-only SSH service, with restrictive private key material, live authorization updates, terminal-session transport, CLI-change synchronization, ordered domain/task events, and native desktop remote-project selection. -- bDS2-compatible Markdown/Liquid rendering with built-in macros rendered from bundled Liquid templates in isolated scopes (customizable with `macros/*` partial-template slugs), category-controlled list title visibility, descriptive category archive titles, canonical multilingual and flat page routes for every configured blog language, recursive menus, calendar archives, feeds, a root hreflang sitemap, Pagefind, change-aware and forced full renders from the native Blog menu/CLI/TUI, and fast route/mtime-based incremental validation whose targeted repair refreshes affected aggregate pages through cancellable section task groups with bDS2-style per-URL progress. +- bDS2-compatible Markdown/Liquid rendering with built-in macros rendered from bundled Liquid templates in isolated scopes (customizable with `macros/*` partial-template slugs), category-controlled list title visibility, descriptive category archive titles, canonical multilingual and flat page routes for every configured blog language, recursive menus, calendar archives, feeds, a root hreflang sitemap, Pagefind, shared cached multicore full-site rendering, change-aware and forced full renders from the native Blog menu/CLI/TUI, and fast route/mtime-based incremental validation whose targeted repair refreshes affected aggregate pages through cancellable section task groups with bDS2-style per-URL progress. - Navigable generated-route preview in the app or system browser, with draft database overlays and published filesystem content. - Optional one-shot AI translation, description, analysis, taxonomy, and language-detection operations run in background tasks with editor-level waiting indicators, using provider-portable JSON-only requests through independent online and local OpenAI-compatible profiles. Each profile has secure credentials, persistently discovered chat/title/image model selections, explicit tool/vision overrides, chat testing, and restart-persistent status-bar airplane-mode routing. - Persistent conversational AI with safe Markdown, streamed and cancellable responses, model/session/token tracking, bounded project-aware blog tools, and localized conversation management in the Chat workspace. Allowlisted render tools add persistent native cards, charts, forms, lists, metrics, mind maps, tables, and tabs without executing assistant-provided HTML or JavaScript. diff --git a/crates/bds-core/src/db/queries/generated_file_hash.rs b/crates/bds-core/src/db/queries/generated_file_hash.rs index 0cac706..ed3a5fa 100644 --- a/crates/bds-core/src/db/queries/generated_file_hash.rs +++ b/crates/bds-core/src/db/queries/generated_file_hash.rs @@ -51,6 +51,32 @@ pub fn upsert_generated_file_hash( }) } +pub fn upsert_generated_file_hashes( + conn: &DbConnection, + hashes: &[GeneratedFileHash], +) -> QueryResult<()> { + conn.with(|c| { + hashes.chunks(200).try_for_each(|chunk| { + diesel::insert_into(generated_file_hashes::table) + .values(chunk) + .on_conflict(( + generated_file_hashes::project_id, + generated_file_hashes::relative_path, + )) + .do_update() + .set(( + generated_file_hashes::content_hash.eq(diesel::upsert::excluded( + generated_file_hashes::content_hash, + )), + generated_file_hashes::updated_at + .eq(diesel::upsert::excluded(generated_file_hashes::updated_at)), + )) + .execute(c) + .map(|_| ()) + }) + }) +} + pub fn touch_generated_file_hashes( conn: &DbConnection, project_id: &str, @@ -111,4 +137,36 @@ mod tests { assert_eq!(stored.content_hash, "def"); assert_eq!(stored.updated_at, 99); } + + #[test] + fn batch_upsert_inserts_and_updates_hashes() { + let db = setup(); + let mut hashes = vec![ + GeneratedFileHash { + project_id: "p1".into(), + relative_path: "index.html".into(), + content_hash: "one".into(), + updated_at: 1, + }, + GeneratedFileHash { + project_id: "p1".into(), + relative_path: "rss.xml".into(), + content_hash: "two".into(), + updated_at: 1, + }, + ]; + upsert_generated_file_hashes(db.conn(), &hashes).unwrap(); + hashes[0].content_hash = "changed".into(); + hashes[0].updated_at = 2; + upsert_generated_file_hashes(db.conn(), &hashes[..1]).unwrap(); + + let stored = list_generated_file_hashes(db.conn(), "p1").unwrap(); + assert_eq!(stored.len(), 2); + assert_eq!( + get_generated_file_hash(db.conn(), "p1", "index.html") + .unwrap() + .content_hash, + "changed" + ); + } } diff --git a/crates/bds-core/src/db/queries/post_link.rs b/crates/bds-core/src/db/queries/post_link.rs index 567bdad..0f04803 100644 --- a/crates/bds-core/src/db/queries/post_link.rs +++ b/crates/bds-core/src/db/queries/post_link.rs @@ -1,7 +1,7 @@ use diesel::prelude::*; use crate::db::DbConnection; -use crate::db::schema::post_links; +use crate::db::schema::{post_links, posts}; use crate::model::PostLink; pub fn insert_post_link(conn: &DbConnection, link: &PostLink) -> QueryResult<()> { @@ -55,6 +55,17 @@ pub fn list_links_by_target( }) } +pub fn list_links_by_project(conn: &DbConnection, project_id: &str) -> QueryResult> { + conn.with(|c| { + post_links::table + .inner_join(posts::table.on(posts::id.eq(post_links::source_post_id))) + .filter(posts::project_id.eq(project_id)) + .order(post_links::created_at) + .select(PostLink::as_select()) + .load(c) + }) +} + #[cfg(test)] mod tests { use super::*; @@ -100,6 +111,14 @@ mod tests { assert_eq!(links.len(), 2); } + #[test] + fn list_by_project() { + let db = setup(); + insert_post_link(db.conn(), &make_link("l1", "a", "b")).unwrap(); + insert_post_link(db.conn(), &make_link("l2", "b", "c")).unwrap(); + assert_eq!(list_links_by_project(db.conn(), "p1").unwrap().len(), 2); + } + #[test] fn delete_by_source() { let db = setup(); diff --git a/crates/bds-core/src/db/queries/post_media.rs b/crates/bds-core/src/db/queries/post_media.rs index a198d63..f9eb8a5 100644 --- a/crates/bds-core/src/db/queries/post_media.rs +++ b/crates/bds-core/src/db/queries/post_media.rs @@ -56,6 +56,23 @@ pub fn list_post_media_by_media( }) } +pub fn list_post_media_by_project( + conn: &DbConnection, + project_id: &str, +) -> QueryResult> { + conn.with(|c| { + post_media::table + .filter(post_media::project_id.eq(project_id)) + .order(( + post_media::post_id.asc(), + post_media::sort_order.asc(), + post_media::media_id.asc(), + )) + .select(PostMedia::as_select()) + .load(c) + }) +} + pub fn update_sort_order( conn: &DbConnection, post_id: &str, @@ -123,6 +140,20 @@ mod tests { assert_eq!(list[0].post_id, "post1"); } + #[test] + fn list_by_project_preserves_post_order() { + let db = setup(); + link_media(db.conn(), &make_pm("pm1", "m1", 1)).unwrap(); + link_media(db.conn(), &make_pm("pm2", "m2", 0)).unwrap(); + let list = list_post_media_by_project(db.conn(), "p1").unwrap(); + assert_eq!( + list.iter() + .map(|link| link.media_id.as_str()) + .collect::>(), + ["m2", "m1"] + ); + } + #[test] fn unlink_removes_association() { let db = setup(); diff --git a/crates/bds-core/src/db/queries/post_translation.rs b/crates/bds-core/src/db/queries/post_translation.rs index 4349938..bcc51dd 100644 --- a/crates/bds-core/src/db/queries/post_translation.rs +++ b/crates/bds-core/src/db/queries/post_translation.rs @@ -54,6 +54,22 @@ pub fn list_post_translations_by_post( }) } +pub fn list_post_translations_by_project( + conn: &DbConnection, + project_id: &str, +) -> QueryResult> { + conn.with(|c| { + post_translations::table + .filter(post_translations::project_id.eq(project_id)) + .order(( + post_translations::translation_for, + post_translations::language, + )) + .select(PostTranslation::as_select()) + .load(c) + }) +} + pub fn update_post_translation(conn: &DbConnection, t: &PostTranslation) -> QueryResult<()> { if !t.status.is_valid_for_translation() { return Err(diesel::result::Error::SerializationError( @@ -151,6 +167,15 @@ mod tests { assert_eq!(list[1].language, "fr"); } + #[test] + fn list_by_project() { + let db = setup(); + insert_post_translation(db.conn(), &make_translation("t1", "de")).unwrap(); + insert_post_translation(db.conn(), &make_translation("t2", "fr")).unwrap(); + let list = list_post_translations_by_project(db.conn(), "p1").unwrap(); + assert_eq!(list.len(), 2); + } + #[test] fn update_translation() { let db = setup(); diff --git a/crates/bds-core/src/engine/generation.rs b/crates/bds-core/src/engine/generation.rs index fd6473d..5b8a17f 100644 --- a/crates/bds-core/src/engine/generation.rs +++ b/crates/bds-core/src/engine/generation.rs @@ -1,5 +1,6 @@ use std::collections::{BTreeMap, HashMap, HashSet}; use std::path::Path; +use std::sync::Arc; use crate::db::DbConnection as Connection; use chrono::{DateTime, TimeZone, Utc}; @@ -8,15 +9,13 @@ use pagefind::options::PagefindServiceConfig; use walkdir::WalkDir; use crate::db::queries; -use crate::engine::site_assets::write_bundled_site_assets; +use crate::engine::site_assets::bundled_site_assets; use crate::engine::validate_site::SiteValidationReport; use crate::engine::{EngineError, EngineResult}; use crate::model::{CategorySettings, Post, ProjectMetadata}; use crate::render::{ - GeneratedWriteOutcome, PostLanguageVariant, build_calendar_json, build_canonical_post_path, - build_site_section_render_artifacts, build_targeted_site_section_render_artifacts, - select_post_language_variant, write_generated_bytes, write_generated_bytes_forced, - write_generated_file, write_generated_file_forced, write_generated_file_verified, + GeneratedFileWriter, GeneratedWriteOutcome, build_calendar_json, build_canonical_post_path, + build_site_render_artifacts_from_context, prepare_site_render_context, write_generated_file, }; #[derive(Debug, Clone)] @@ -57,6 +56,41 @@ pub struct GenerationReport { pub deleted_paths: Vec, } +pub struct PreparedSiteGeneration { + metadata: ProjectMetadata, + sources: Vec, + render: crate::render::SiteRenderContext, + generated_hashes: Arc>, +} + +pub fn prepare_site_generation( + conn: &Connection, + data_dir: &Path, + project_id: &str, + metadata: &ProjectMetadata, + sources: &[PublishedPostSource], +) -> EngineResult { + let input_posts = sources + .iter() + .map(|source| (source.post.clone(), source.body_markdown.clone())) + .collect::>(); + let render = + prepare_site_render_context(conn, data_dir, project_id, metadata, &input_posts, false) + .map_err(|error| EngineError::Parse(error.to_string()))?; + let generated_hashes = Arc::new( + queries::generated_file_hash::list_generated_file_hashes(conn, project_id)? + .into_iter() + .map(|hash| (hash.relative_path, hash.content_hash)) + .collect(), + ); + Ok(PreparedSiteGeneration { + metadata: metadata.clone(), + sources: sources.to_vec(), + render, + generated_hashes, + }) +} + #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum GenerationSection { Core, @@ -151,14 +185,15 @@ fn generate_starter_site_with_progress_mode( force: bool, mut on_page: impl FnMut(usize, usize, &str), ) -> EngineResult { + let data_dir = project_data_dir(output_dir); + let prepared = prepare_site_generation(conn, &data_dir, project_id, metadata, posts)?; let mut report = GenerationReport::default(); for section in GenerationSection::ALL { - report.append(render_site_section_with_progress_mode( + report.append(render_prepared_site_section_with_progress( conn, output_dir, project_id, - metadata, - posts, + &prepared, section, force, &|_| {}, @@ -256,55 +291,78 @@ fn render_site_section_with_progress_mode( return Err(EngineError::Validation("cancelled".to_string())); } let data_dir = project_data_dir(output_dir); - let input_posts = posts - .iter() - .map(|source| (source.post.clone(), source.body_markdown.clone())) - .collect::>(); - let artifacts = build_site_section_render_artifacts( + let prepared = prepare_site_generation(conn, &data_dir, project_id, metadata, posts)?; + render_prepared_site_section_with_progress( conn, - &data_dir, + output_dir, project_id, - metadata, - &input_posts, + &prepared, section, + force, + on_page_rendered, + &mut on_page, + &mut is_cancelled, + ) +} + +#[expect( + clippy::too_many_arguments, + reason = "prepared section rendering keeps write mode and callbacks" +)] +pub fn render_prepared_site_section_with_progress( + conn: &Connection, + output_dir: &Path, + project_id: &str, + prepared: &PreparedSiteGeneration, + section: GenerationSection, + force: bool, + on_page_rendered: &(dyn Fn(&str) + Sync), + mut on_page: impl FnMut(usize, usize, &str), + mut is_cancelled: impl FnMut() -> bool, +) -> EngineResult { + if is_cancelled() { + return Err(EngineError::Validation("cancelled".to_string())); + } + let artifacts = build_site_render_artifacts_from_context( + &prepared.render, + Some(section), + None, on_page_rendered, ) .map_err(|error| EngineError::Parse(error.to_string()))?; let mut report = GenerationReport::default(); + let mut writer = GeneratedFileWriter::with_existing( + conn, + output_dir, + project_id, + Arc::clone(&prepared.generated_hashes), + force, + false, + ) + .map_err(|error| EngineError::Parse(error.to_string()))?; let total_pages = artifacts.pages.len(); for (index, page) in artifacts.pages.iter().enumerate() { if is_cancelled() { return Err(EngineError::Validation("cancelled".to_string())); } - write_out( - conn, - output_dir, - project_id, - &page.relative_path, - &page.html, - &mut report, - force, - false, - )?; + write_out(&mut writer, &page.relative_path, &page.html, &mut report)?; on_page(index + 1, total_pages, &page.url_path); } if section == GenerationSection::Core { write_core_outputs( - conn, - output_dir, - project_id, - metadata, - &data_dir, - posts, + &mut writer, + prepared, + &project_data_dir(output_dir), &artifacts.route_manifest, None, &mut report, &mut is_cancelled, - force, - false, )?; } + writer + .finish() + .map_err(|error| EngineError::Parse(error.to_string()))?; Ok(report) } @@ -399,10 +457,36 @@ pub fn apply_validation_section_with_progress( return Err(EngineError::Validation("cancelled".to_string())); } let data_dir = project_data_dir(output_dir); - let input_posts = posts - .iter() - .map(|source| (source.post.clone(), source.body_markdown.clone())) - .collect::>(); + let prepared = prepare_site_generation(conn, &data_dir, project_id, metadata, posts)?; + apply_validation_prepared_section_with_progress( + conn, + output_dir, + project_id, + &prepared, + validation, + section, + &mut on_page, + &on_page_rendered, + &mut is_cancelled, + ) +} + +#[expect( + clippy::too_many_arguments, + reason = "prepared targeted apply keeps validation and callbacks" +)] +pub fn apply_validation_prepared_section_with_progress( + conn: &Connection, + output_dir: &Path, + project_id: &str, + prepared: &PreparedSiteGeneration, + validation: &SiteValidationReport, + section: GenerationSection, + mut on_page: impl FnMut(usize, usize, &str), + on_page_rendered: &(dyn Fn(&str) + Sync), + mut is_cancelled: impl FnMut() -> bool, +) -> EngineResult { + let metadata = &prepared.metadata; let requested = validation .missing_pages .iter() @@ -415,64 +499,46 @@ pub fn apply_validation_section_with_progress( .chain(validation.extra_pages.iter()) .chain(validation.stale_pages.iter()) .any(|path| classify_generated_path(path, metadata).is_none()); - let artifacts = if fallback { - build_site_section_render_artifacts( - conn, - &data_dir, - project_id, - metadata, - &input_posts, - section, - &on_page_rendered, - ) - } else { - build_targeted_site_section_render_artifacts( - conn, - &data_dir, - project_id, - metadata, - &input_posts, - section, - &requested, - &on_page_rendered, - ) - } + let artifacts = build_site_render_artifacts_from_context( + &prepared.render, + Some(section), + (!fallback).then_some(&requested), + on_page_rendered, + ) .map_err(|error| EngineError::Parse(error.to_string()))?; let mut report = GenerationReport::default(); + let mut writer = GeneratedFileWriter::with_existing( + conn, + output_dir, + project_id, + Arc::clone(&prepared.generated_hashes), + false, + true, + ) + .map_err(|error| EngineError::Parse(error.to_string()))?; let total_pages = artifacts.pages.len(); for (index, page) in artifacts.pages.iter().enumerate() { if is_cancelled() { return Err(EngineError::Validation("cancelled".to_string())); } - write_out( - conn, - output_dir, - project_id, - &page.relative_path, - &page.html, - &mut report, - false, - true, - )?; + write_out(&mut writer, &page.relative_path, &page.html, &mut report)?; on_page(index + 1, total_pages, &page.url_path); } if section == GenerationSection::Core { write_core_outputs( - conn, - output_dir, - project_id, - metadata, - &data_dir, - posts, + &mut writer, + prepared, + &project_data_dir(output_dir), &artifacts.route_manifest, (!fallback).then_some(&requested), &mut report, &mut is_cancelled, - false, - true, )?; } + writer + .finish() + .map_err(|error| EngineError::Parse(error.to_string()))?; for path in &validation.extra_pages { if is_cancelled() { @@ -510,26 +576,24 @@ fn refresh_route_timestamps( Ok(()) } -#[expect( - clippy::too_many_arguments, - reason = "generation context is existing domain data" -)] fn write_core_outputs( - conn: &Connection, - output_dir: &Path, - project_id: &str, - metadata: &ProjectMetadata, + writer: &mut GeneratedFileWriter<'_>, + prepared: &PreparedSiteGeneration, data_dir: &Path, - published_posts: &[PublishedPostSource], route_manifest: &[crate::render::SitePage], requested: Option<&HashSet>, report: &mut GenerationReport, is_cancelled: &mut impl FnMut() -> bool, - force: bool, - verify_output: bool, ) -> EngineResult<()> { + let metadata = &prepared.metadata; + let published_posts = &prepared.sources; if requested.is_none() { - write_bundled_site_assets(conn, output_dir, project_id, report, force)?; + for asset in bundled_site_assets() { + let outcome = writer + .write_bytes(asset.relative_path, asset.bytes) + .map_err(|error| EngineError::Parse(error.to_string()))?; + record_write_outcome(report, asset.relative_path, outcome); + } } let mut outputs = vec![( "calendar.json".to_string(), @@ -541,8 +605,6 @@ fn write_core_outputs( )?, )]; for render_language in render_languages(metadata) { - let localized_posts = - localized_sources(conn, data_dir, published_posts, &render_language, metadata)?; let is_main = render_language == metadata.main_language.as_deref().unwrap_or("en"); let prefix = if is_main { String::new() @@ -550,28 +612,35 @@ fn write_core_outputs( format!("{render_language}/") }; let mut feed_posts = if is_main { - published_posts.to_vec() - } else { - localized_posts + published_posts .iter() - .filter(|source| { - source - .post - .language + .map(|source| &source.post) + .collect::>() + } else { + prepared + .render + .localized_posts(&render_language) + .filter(|post| { + post.language .as_deref() .is_some_and(|language| language.eq_ignore_ascii_case(&render_language)) }) - .cloned() .collect::>() }; - sort_published_sources(&mut feed_posts); + feed_posts.sort_by(|left, right| { + right + .created_at + .cmp(&left.created_at) + .then_with(|| right.published_at.cmp(&left.published_at)) + .then_with(|| left.slug.cmp(&right.slug)) + }); outputs.push(( format!("{prefix}rss.xml"), - build_rss_xml(metadata, &feed_posts, &render_language), + build_rss_xml(metadata, feed_posts.iter().copied(), &render_language), )); outputs.push(( format!("{prefix}atom.xml"), - build_atom_xml(metadata, &feed_posts, &render_language), + build_atom_xml(metadata, feed_posts.iter().copied(), &render_language), )); if is_main { let category_settings = load_category_settings(data_dir); @@ -595,50 +664,36 @@ fn write_core_outputs( return Err(EngineError::Validation("cancelled".to_string())); } if requested.is_none_or(|requested| requested.contains(&path)) { - write_out( - conn, - output_dir, - project_id, - &path, - &content, - report, - force, - verify_output, - )?; + write_out(writer, &path, &content, report)?; } } Ok(()) } -#[expect( - clippy::too_many_arguments, - reason = "generated output needs its render context, report, and write mode" -)] fn write_out( - conn: &Connection, - output_dir: &Path, - project_id: &str, + writer: &mut GeneratedFileWriter<'_>, relative_path: &str, content: &str, report: &mut GenerationReport, - force: bool, - verify_output: bool, ) -> EngineResult<()> { - let outcome = if force { - write_generated_file_forced(conn, output_dir, project_id, relative_path, content) - } else if verify_output { - write_generated_file_verified(conn, output_dir, project_id, relative_path, content) - } else { - write_generated_file(conn, output_dir, project_id, relative_path, content) - } - .map_err(|error| EngineError::Parse(error.to_string()))?; + let outcome = writer + .write_str(relative_path, content) + .map_err(|error| EngineError::Parse(error.to_string()))?; + record_write_outcome(report, relative_path, outcome); + Ok(()) +} + +fn record_write_outcome( + report: &mut GenerationReport, + relative_path: &str, + outcome: GeneratedWriteOutcome, +) { match outcome { GeneratedWriteOutcome::Written => report.written_paths.push(relative_path.to_string()), GeneratedWriteOutcome::SkippedUnchanged => { report.skipped_paths.push(relative_path.to_string()) } } - Ok(()) } pub fn build_site_search_index( @@ -803,6 +858,8 @@ fn build_site_search_index_with_progress_mode( }); let total = outputs.len(); let mut report = GenerationReport::default(); + let mut writer = GeneratedFileWriter::new(conn, output_dir, project_id, force, false) + .map_err(|error| EngineError::Parse(error.to_string()))?; let expected = outputs .iter() .map(|(relative, _)| relative.clone()) @@ -811,18 +868,15 @@ fn build_site_search_index_with_progress_mode( if is_cancelled() { return Err(EngineError::Validation("cancelled".to_string())); } - let outcome = if force { - write_generated_bytes_forced(conn, output_dir, project_id, &relative, &contents) - } else { - write_generated_bytes(conn, output_dir, project_id, &relative, &contents) - } - .map_err(|error| EngineError::Parse(error.to_string()))?; - match outcome { - GeneratedWriteOutcome::Written => report.written_paths.push(relative.clone()), - GeneratedWriteOutcome::SkippedUnchanged => report.skipped_paths.push(relative.clone()), - } + let outcome = writer + .write_bytes(&relative, &contents) + .map_err(|error| EngineError::Parse(error.to_string()))?; + record_write_outcome(&mut report, &relative, outcome); on_file(index + 1, total, &relative); } + writer + .finish() + .map_err(|error| EngineError::Parse(error.to_string()))?; for language in render_languages(metadata) { let prefix = if language == metadata.main_language.as_deref().unwrap_or("en") { "pagefind".to_string() @@ -969,67 +1023,6 @@ fn render_languages(metadata: &ProjectMetadata) -> Vec { languages } -fn localized_sources( - conn: &Connection, - data_dir: &Path, - posts: &[PublishedPostSource], - language: &str, - metadata: &ProjectMetadata, -) -> EngineResult> { - let main_language = metadata.main_language.as_deref().unwrap_or("en"); - let mut localized = Vec::new(); - for source in posts { - let translation = queries::post_translation::get_post_translation_by_post_and_language( - conn, - &source.post.id, - language, - ) - .ok() - .filter(|translation| { - !translation.file_path.trim().is_empty() - && data_dir - .join(translation.file_path.trim_start_matches('/')) - .is_file() - }); - match select_post_language_variant( - &source.post, - language, - main_language, - translation.is_some(), - ) { - Some(PostLanguageVariant::Base) => localized.push(source.clone()), - Some(PostLanguageVariant::Translation) => { - let Some(translation) = translation else { - continue; - }; - let raw = std::fs::read_to_string( - data_dir.join(translation.file_path.trim_start_matches('/')), - ) - .map_err(EngineError::Io)?; - let (_, body) = crate::util::frontmatter::read_translation_file(&raw) - .map_err(EngineError::Parse)?; - let mut translated_post = source.post.clone(); - translated_post.id = translation.id.clone(); - translated_post.title = translation.title.clone(); - translated_post.excerpt = translation.excerpt.clone(); - translated_post.language = Some(translation.language.clone()); - translated_post.status = translation.status.clone(); - translated_post.file_path = translation.file_path.clone(); - translated_post.updated_at = translation.updated_at; - translated_post.published_at = - translation.published_at.or(source.post.published_at); - localized.push(PublishedPostSource { - post: translated_post, - body_markdown: body, - }); - } - None => {} - } - } - sort_published_sources(&mut localized); - Ok(localized) -} - fn sort_published_sources(posts: &mut [PublishedPostSource]) { posts.sort_by(|left, right| { right @@ -1093,9 +1086,9 @@ pub(crate) fn refresh_validation_sitemap( Ok(()) } -pub(crate) fn build_rss_xml( +pub(crate) fn build_rss_xml<'a>( metadata: &ProjectMetadata, - posts: &[PublishedPostSource], + posts: impl IntoIterator, language: &str, ) -> String { let base_url = metadata @@ -1109,20 +1102,20 @@ pub(crate) fn build_rss_xml( escape_xml(language) ); - for source in posts { - let url = post_absolute_url(base_url, metadata, source, language); + for post in posts { + let url = post_absolute_url(base_url, metadata, post, language); xml.push_str(&format!( "{}{url}", - escape_xml(&source.post.title) + escape_xml(&post.title) )); } xml.push_str(""); xml } -pub(crate) fn build_atom_xml( +pub(crate) fn build_atom_xml<'a>( metadata: &ProjectMetadata, - posts: &[PublishedPostSource], + posts: impl IntoIterator, language: &str, ) -> String { let base_url = metadata @@ -1136,11 +1129,11 @@ pub(crate) fn build_atom_xml( escape_xml(language) ); - for source in posts { - let url = post_absolute_url(base_url, metadata, source, language); + for post in posts { + let url = post_absolute_url(base_url, metadata, post, language); xml.push_str(&format!( "{}{url}", - escape_xml(&source.post.title) + escape_xml(&post.title) )); } xml.push_str(""); @@ -1150,13 +1143,13 @@ pub(crate) fn build_atom_xml( fn post_absolute_url( base_url: &str, metadata: &ProjectMetadata, - source: &PublishedPostSource, + post: &Post, language: &str, ) -> String { format!( "{base_url}{}/", build_canonical_post_path( - &source.post, + post, language, metadata.main_language.as_deref().unwrap_or("en") ) diff --git a/crates/bds-core/src/engine/preview.rs b/crates/bds-core/src/engine/preview.rs index d115949..f02f67b 100644 --- a/crates/bds-core/src/engine/preview.rs +++ b/crates/bds-core/src/engine/preview.rs @@ -243,11 +243,19 @@ fn render_preview_response( let (content_type, xml) = match kind { PreviewFeedKind::Rss => ( "application/rss+xml; charset=utf-8", - crate::engine::generation::build_rss_xml(&metadata, &localized_posts, &language), + crate::engine::generation::build_rss_xml( + &metadata, + localized_posts.iter().map(|source| &source.post), + &language, + ), ), PreviewFeedKind::Atom => ( "application/atom+xml; charset=utf-8", - crate::engine::generation::build_atom_xml(&metadata, &localized_posts, &language), + crate::engine::generation::build_atom_xml( + &metadata, + localized_posts.iter().map(|source| &source.post), + &language, + ), ), }; return Ok((StatusCode::OK, [(header::CONTENT_TYPE, content_type)], xml).into_response()); diff --git a/crates/bds-core/src/engine/site_assets.rs b/crates/bds-core/src/engine/site_assets.rs index 6214575..e5aee35 100644 --- a/crates/bds-core/src/engine/site_assets.rs +++ b/crates/bds-core/src/engine/site_assets.rs @@ -1,10 +1,7 @@ use std::fs; use std::path::Path; -use crate::db::DbConnection as Connection; - -use crate::engine::{EngineError, EngineResult}; -use crate::render::{GeneratedWriteOutcome, write_generated_bytes}; +use crate::engine::EngineResult; #[derive(Debug, Clone, Copy)] pub(crate) struct BundledSiteAsset { @@ -238,6 +235,10 @@ pub(crate) fn bundled_site_asset(relative_path: &str) -> Option<&'static [u8]> { .map(|asset| asset.bytes) } +pub(crate) fn bundled_site_assets() -> &'static [BundledSiteAsset] { + BUNDLED_SITE_ASSETS +} + pub(crate) fn copy_bundled_site_assets(project_dir: &Path) -> EngineResult<()> { for asset in BUNDLED_SITE_ASSETS { let target = project_dir.join(asset.relative_path); @@ -252,44 +253,6 @@ pub(crate) fn copy_bundled_site_assets(project_dir: &Path) -> EngineResult<()> { Ok(()) } -pub(crate) fn write_bundled_site_assets( - conn: &Connection, - output_dir: &Path, - project_id: &str, - report: &mut crate::engine::generation::GenerationReport, - force: bool, -) -> EngineResult<()> { - for asset in BUNDLED_SITE_ASSETS { - let outcome = if force { - crate::render::write_generated_bytes_forced( - conn, - output_dir, - project_id, - asset.relative_path, - asset.bytes, - ) - } else { - write_generated_bytes( - conn, - output_dir, - project_id, - asset.relative_path, - asset.bytes, - ) - } - .map_err(|error| EngineError::Parse(error.to_string()))?; - match outcome { - GeneratedWriteOutcome::Written => { - report.written_paths.push(asset.relative_path.to_string()) - } - GeneratedWriteOutcome::SkippedUnchanged => { - report.skipped_paths.push(asset.relative_path.to_string()) - } - } - } - Ok(()) -} - #[cfg(test)] mod tests { use super::bundled_site_asset; diff --git a/crates/bds-core/src/model/generation.rs b/crates/bds-core/src/model/generation.rs index 5dacfa6..2326ab1 100644 --- a/crates/bds-core/src/model/generation.rs +++ b/crates/bds-core/src/model/generation.rs @@ -14,7 +14,8 @@ use serde::{Deserialize, Serialize}; )] #[diesel( table_name = crate::db::schema::generated_file_hashes, - check_for_backend(diesel::sqlite::Sqlite) + check_for_backend(diesel::sqlite::Sqlite), + treat_none_as_default_value = false )] pub struct GeneratedFileHash { pub project_id: String, diff --git a/crates/bds-core/src/render/generation.rs b/crates/bds-core/src/render/generation.rs index 3712967..cf9820f 100644 --- a/crates/bds-core/src/render/generation.rs +++ b/crates/bds-core/src/render/generation.rs @@ -1,6 +1,7 @@ -use std::collections::BTreeMap; +use std::collections::{BTreeMap, HashMap}; use std::fs; use std::path::Path; +use std::sync::Arc; use crate::db::DbConnection as Connection; use chrono::{Datelike, Local, TimeZone}; @@ -16,6 +17,117 @@ pub enum GeneratedWriteOutcome { SkippedUnchanged, } +pub(crate) struct GeneratedFileWriter<'a> { + conn: &'a Connection, + output_dir: &'a Path, + project_id: &'a str, + existing: Arc>, + pending: HashMap, + force: bool, + verify_output: bool, +} + +impl<'a> GeneratedFileWriter<'a> { + pub(crate) fn new( + conn: &'a Connection, + output_dir: &'a Path, + project_id: &'a str, + force: bool, + verify_output: bool, + ) -> Result> { + let existing = Arc::new( + qhash::list_generated_file_hashes(conn, project_id)? + .into_iter() + .map(|hash| (hash.relative_path, hash.content_hash)) + .collect(), + ); + Self::with_existing(conn, output_dir, project_id, existing, force, verify_output) + } + + pub(crate) fn with_existing( + conn: &'a Connection, + output_dir: &'a Path, + project_id: &'a str, + existing: Arc>, + force: bool, + verify_output: bool, + ) -> Result> { + Ok(Self { + conn, + output_dir, + project_id, + existing, + pending: HashMap::new(), + force, + verify_output, + }) + } + + pub(crate) fn write_str( + &mut self, + relative_path: &str, + content: &str, + ) -> Result> { + self.write(relative_path, content.as_bytes(), |path| { + atomic_write_str(path, content) + }) + } + + pub(crate) fn write_bytes( + &mut self, + relative_path: &str, + content: &[u8], + ) -> Result> { + self.write(relative_path, content, |path| { + crate::util::atomic_write(path, content) + }) + } + + fn write( + &mut self, + relative_path: &str, + content: &[u8], + write: impl FnOnce(&Path) -> std::io::Result<()>, + ) -> Result> { + let hash = content_hash(content); + let target_path = self.output_dir.join(relative_path); + let existing_hash = self + .pending + .get(relative_path) + .map(|pending| &pending.content_hash) + .or_else(|| self.existing.get(relative_path)); + if !self.force + && existing_hash == Some(&hash) + && target_path.exists() + && (!self.verify_output || file_hash(&target_path)? == hash) + { + return Ok(GeneratedWriteOutcome::SkippedUnchanged); + } + if let Some(parent) = target_path.parent() { + fs::create_dir_all(parent)?; + } + write(&target_path)?; + self.pending.insert( + relative_path.to_string(), + GeneratedFileHash { + project_id: self.project_id.to_string(), + relative_path: relative_path.to_string(), + content_hash: hash, + updated_at: now_unix_ms(), + }, + ); + Ok(GeneratedWriteOutcome::Written) + } + + pub(crate) fn finish(self) -> Result<(), Box> { + qhash::upsert_generated_file_hashes( + self.conn, + &self.pending.into_values().collect::>(), + )?; + Ok(()) + } +} + #[derive(Debug, Clone, PartialEq, Eq, Serialize)] pub struct CalendarArchiveData { pub years: BTreeMap, diff --git a/crates/bds-core/src/render/mod.rs b/crates/bds-core/src/render/mod.rs index 2e3f585..8520a0e 100644 --- a/crates/bds-core/src/render/mod.rs +++ b/crates/bds-core/src/render/mod.rs @@ -6,14 +6,15 @@ mod routes; mod site; mod template_lookup; +pub(crate) use generation::GeneratedFileWriter; pub use generation::{ CalendarArchiveData, GeneratedWriteOutcome, build_calendar_json, build_core_generation_paths, write_generated_bytes, write_generated_bytes_forced, write_generated_file, write_generated_file_forced, write_generated_file_verified, }; pub use markdown::render_markdown_to_html; +pub(crate) use page_renderer::validate_liquid_template_syntax; pub use page_renderer::{RenderError, render_liquid_template}; -pub(crate) use page_renderer::{render_liquid_template_with_host, validate_liquid_template_syntax}; pub(crate) use routes::{PostLanguageVariant, blog_page_title, select_post_language_variant}; pub use routes::{ RenderedPage, build_canonical_post_path, render_starter_list_page, @@ -21,9 +22,11 @@ pub use routes::{ render_starter_single_post_page_with_media_map, }; pub use site::{ - PagefindDocument, PreviewRenderResult, SitePage, SiteRenderArtifacts, build_preview_response, - build_site_render_artifacts, build_site_route_manifest, build_site_section_render_artifacts, + PagefindDocument, PreviewRenderResult, SitePage, SiteRenderArtifacts, SiteRenderContext, + build_preview_response, build_site_render_artifacts, build_site_render_artifacts_from_context, + build_site_route_manifest, build_site_section_render_artifacts, build_targeted_site_section_render_artifacts, estimate_site_render_pages, + prepare_site_render_context, }; pub use template_lookup::{ RenderCategorySettings, RenderTemplateLookup, TemplateLookupError, resolve_post_template, diff --git a/crates/bds-core/src/render/page_renderer.rs b/crates/bds-core/src/render/page_renderer.rs index bcb9ece..f102ee1 100644 --- a/crates/bds-core/src/render/page_renderer.rs +++ b/crates/bds-core/src/render/page_renderer.rs @@ -1,6 +1,6 @@ use std::collections::HashMap; use std::fmt; -use std::sync::Arc; +use std::sync::{Arc, Mutex, OnceLock}; use liquid::ParserBuilder; use liquid::partials::{EagerCompiler, InMemorySource}; @@ -18,7 +18,7 @@ use crate::i18n::translate_render; use crate::render::macros::{MacroRenderContext, expand_builtin_macros}; use crate::render::render_markdown_to_html; use crate::scripting::{HostApi, UnavailableHost}; -use crate::util::slugify; +use crate::util::{content_hash, slugify}; #[derive(Debug, Error)] pub enum RenderError { @@ -26,6 +26,44 @@ pub enum RenderError { Liquid(#[from] liquid::Error), } +#[derive(Clone)] +pub(crate) struct CompiledLiquidTemplate(Arc); + +#[derive(Clone)] +pub(crate) struct LiquidRenderer { + parser: liquid::Parser, +} + +impl LiquidRenderer { + pub(crate) fn new( + partials: &HashMap, + host: Arc, + ) -> Result { + let mut compiled_partials: EagerCompiler = EagerCompiler::empty(); + for (name, content) in partials { + compiled_partials.add(format!("{name}.liquid"), content.clone()); + } + Ok(Self { + parser: liquid_parser_builder(host, Default::default()) + .partials(compiled_partials) + .build()?, + }) + } + + pub(crate) fn compile(&self, source: &str) -> Result { + Ok(CompiledLiquidTemplate(Arc::new(self.parser.parse(source)?))) + } + + pub(crate) fn render( + &self, + template: &CompiledLiquidTemplate, + context: &T, + ) -> Result { + let globals = liquid::to_object(context)?; + Ok(template.0.render(&globals)?) + } +} + #[derive(Debug, Clone, Default)] struct HtmlRewriteContext { canonical_post_path_by_slug: HashMap, @@ -51,21 +89,13 @@ pub(crate) fn render_liquid_template_with_host( context: &T, host: Arc, ) -> Result { - let mut compiled_partials: EagerCompiler = EagerCompiler::empty(); - for (name, content) in partials { - compiled_partials.add(format!("{name}.liquid"), content.clone()); - } - - let parser = liquid_parser_builder(host) - .partials(compiled_partials) - .build()?; - let template = parser.parse(template_source)?; - let globals = liquid::to_object(context)?; - Ok(template.render(&globals)?) + let renderer = LiquidRenderer::new(partials, host)?; + let template = renderer.compile(template_source)?; + renderer.render(&template, context) } pub(crate) fn validate_liquid_template_syntax(template_source: &str) -> Result<(), String> { - let parser = liquid_parser_builder(Arc::new(UnavailableHost)) + let parser = liquid_parser_builder(Arc::new(UnavailableHost), Default::default()) .build() .map_err(|error| error.to_string())?; parser @@ -74,10 +104,15 @@ pub(crate) fn validate_liquid_template_syntax(template_source: &str) -> Result<( .map_err(|error| error.to_string()) } -fn liquid_parser_builder(host: Arc) -> ParserBuilder { +type MarkdownCache = Arc>>>>; + +fn liquid_parser_builder(host: Arc, markdown_cache: MarkdownCache) -> ParserBuilder { ParserBuilder::with_stdlib() .filter(I18n) - .filter(Markdown { host }) + .filter(Markdown { + host, + cache: markdown_cache, + }) .filter(Slugify) } @@ -162,6 +197,7 @@ struct MarkdownArgs { )] struct Markdown { host: Arc, + cache: MarkdownCache, } impl ParseFilter for Markdown { @@ -169,6 +205,7 @@ impl ParseFilter for Markdown { Ok(Box::new(MarkdownFilter { args: MarkdownArgs::from_args(args)?, host: Arc::clone(&self.host), + cache: Arc::clone(&self.cache), })) } @@ -182,6 +219,7 @@ impl ParseFilter for Markdown { struct MarkdownFilter { args: MarkdownArgs, host: Arc, + cache: MarkdownCache, } impl fmt::Debug for MarkdownFilter { @@ -208,31 +246,54 @@ impl Filter for MarkdownFilter { .map(value_to_string_map) .unwrap_or_default(), }; + let post_id = args + .post_id + .as_ref() + .and_then(|value| value.as_scalar().map(|scalar| scalar.to_kstr().to_string())) + .or_else(|| { + runtime + .try_get(&[ScalarCow::new("post"), ScalarCow::new("id")]) + .and_then(|value| value.as_scalar().map(|scalar| scalar.to_kstr().to_string())) + }); + let language = args + .language + .as_ref() + .and_then(|value| value.as_scalar()) + .map(|scalar| scalar.to_kstr().to_string()) + .unwrap_or_default(); let macro_context = MacroRenderContext { roots: collect_macro_roots(runtime), - post_id: args - .post_id - .as_ref() - .and_then(|value| value.as_scalar().map(|scalar| scalar.to_kstr().to_string())) - .or_else(|| { - runtime - .try_get(&[ScalarCow::new("post"), ScalarCow::new("id")]) - .and_then(|value| { - value.as_scalar().map(|scalar| scalar.to_kstr().to_string()) - }) - }), + post_id: post_id.clone(), host: Arc::clone(&self.host), }; - - let expanded = expand_builtin_macros(markdown.as_str(), ¯o_context); - let rendered = render_markdown_to_html(&expanded); - Ok(Value::scalar(rewrite_rendered_html_urls( - &rendered, - &rewrite_context, - ))) + let key = MarkdownCacheKey { + content_hash: content_hash(markdown.as_bytes()), + language, + post_id, + }; + let entry = { + let mut cache = self.cache.lock().unwrap_or_else(|error| error.into_inner()); + Arc::clone(cache.entry(key).or_default()) + }; + Ok(Value::scalar( + entry + .get_or_init(|| { + let expanded = expand_builtin_macros(markdown.as_str(), ¯o_context); + let rendered = render_markdown_to_html(&expanded); + rewrite_rendered_html_urls(&rendered, &rewrite_context) + }) + .clone(), + )) } } +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +struct MarkdownCacheKey { + content_hash: String, + language: String, + post_id: Option, +} + fn collect_macro_roots(runtime: &dyn Runtime) -> JsonMap { let mut roots = JsonMap::new(); diff --git a/crates/bds-core/src/render/site.rs b/crates/bds-core/src/render/site.rs index 8e6f8a3..31a4dfe 100644 --- a/crates/bds-core/src/render/site.rs +++ b/crates/bds-core/src/render/site.rs @@ -9,17 +9,17 @@ use chrono::{Datelike, Local, TimeZone, Utc}; use rayon::prelude::*; use serde_json::{Value, json}; +use super::page_renderer::{CompiledLiquidTemplate, LiquidRenderer}; use crate::db::queries; use crate::engine::generation::{GenerationSection, classify_generated_path}; use crate::engine::menu::{self, MenuItemKind}; use crate::model::{ - CategorySettings, Media, Post, PostStatus, ProjectMetadata, ScriptKind, Tag, Template, - TemplateKind, TemplateStatus, + CategorySettings, Media, Post, PostLink, PostMedia, PostStatus, PostTranslation, + ProjectMetadata, ScriptKind, Tag, Template, TemplateKind, TemplateStatus, }; use crate::render::{ PostLanguageVariant, RenderCategorySettings, RenderTemplateLookup, blog_page_title, - build_canonical_post_path, render_liquid_template_with_host, resolve_post_template, - select_post_language_variant, + build_canonical_post_path, resolve_post_template, select_post_language_variant, }; use crate::scripting::{CoreHost, HostApi, UnavailableHost}; use crate::util::frontmatter::{read_script_file, read_template_file, read_translation_file}; @@ -59,7 +59,6 @@ pub struct PagefindDocument { #[derive(Debug, Clone, Default)] pub struct SiteRenderArtifacts { pub pages: Vec, - pub pagefind_documents: Vec, pub route_manifest: Vec, } @@ -72,14 +71,13 @@ pub struct PreviewRenderResult { #[derive(Clone)] struct TemplateBundle { post_templates: Vec