From d9085cf681573b94ea8483179dc143cc01aaaddb Mon Sep 17 00:00:00 2001 From: Chili Palmer Date: Fri, 24 Jul 2026 07:10:57 +0200 Subject: [PATCH] fix: still working on rendering speed --- crates/bds-core/src/engine/task.rs | 19 ++- crates/bds-core/src/render/page_renderer.rs | 37 ++++-- crates/bds-core/src/render/site.rs | 120 +++++++----------- crates/bds-core/tests/m4_generation_engine.rs | 44 +++++++ crates/bds-ui/src/app.rs | 19 ++- crates/bds-ui/src/app/tasks.rs | 57 ++++----- specs/task.allium | 3 +- 7 files changed, 176 insertions(+), 123 deletions(-) diff --git a/crates/bds-core/src/engine/task.rs b/crates/bds-core/src/engine/task.rs index 888be1c..1dc80c6 100644 --- a/crates/bds-core/src/engine/task.rs +++ b/crates/bds-core/src/engine/task.rs @@ -337,7 +337,11 @@ impl TaskManager { impl Default for TaskManager { fn default() -> Self { - Self::new(3) + Self::new( + std::thread::available_parallelism() + .map(usize::from) + .unwrap_or(1), + ) } } @@ -358,6 +362,19 @@ mod tests { assert_eq!(mgr.status(id), Some(TaskStatus::Running)); } + #[test] + fn default_uses_every_online_worker_like_bds2() { + let mgr = TaskManager::default(); + let expected = std::thread::available_parallelism() + .map(usize::from) + .unwrap_or(1); + for index in 0..expected { + mgr.submit(&format!("task {index}")); + } + + assert_eq!(mgr.running_count(), expected); + } + #[test] fn max_concurrent_enforced() { let mgr = TaskManager::new(3); diff --git a/crates/bds-core/src/render/page_renderer.rs b/crates/bds-core/src/render/page_renderer.rs index f102ee1..67b0817 100644 --- a/crates/bds-core/src/render/page_renderer.rs +++ b/crates/bds-core/src/render/page_renderer.rs @@ -4,7 +4,7 @@ use std::sync::{Arc, Mutex, OnceLock}; use liquid::ParserBuilder; use liquid::partials::{EagerCompiler, InMemorySource}; -use liquid_core::model::ScalarCow; +use liquid_core::model::{Object, ScalarCow, ValueCow}; use liquid_core::parser::FilterArguments; use liquid_core::{ Display_filter, Expression, Filter, FilterParameters, FilterReflection, FromFilterParameters, @@ -62,6 +62,21 @@ impl LiquidRenderer { let globals = liquid::to_object(context)?; Ok(template.0.render(&globals)?) } + + pub(crate) fn render_with_base( + &self, + template: &CompiledLiquidTemplate, + base: &Object, + context: &T, + ) -> Result { + let page = liquid::to_object(context)?; + let globals = base + .iter() + .chain(page.iter()) + .map(|(key, value)| (key.to_string(), ValueCow::from(value))) + .collect::>(); + Ok(template.0.render(&globals)?) + } } #[derive(Debug, Clone, Default)] @@ -261,15 +276,10 @@ impl Filter for MarkdownFilter { .and_then(|value| value.as_scalar()) .map(|scalar| scalar.to_kstr().to_string()) .unwrap_or_default(); - let macro_context = MacroRenderContext { - roots: collect_macro_roots(runtime), - post_id: post_id.clone(), - host: Arc::clone(&self.host), - }; let key = MarkdownCacheKey { content_hash: content_hash(markdown.as_bytes()), language, - post_id, + post_id: post_id.clone(), }; let entry = { let mut cache = self.cache.lock().unwrap_or_else(|error| error.into_inner()); @@ -278,8 +288,17 @@ impl Filter for MarkdownFilter { Ok(Value::scalar( entry .get_or_init(|| { - let expanded = expand_builtin_macros(markdown.as_str(), ¯o_context); - let rendered = render_markdown_to_html(&expanded); + let rendered = if markdown.contains("[[") { + let macro_context = MacroRenderContext { + roots: collect_macro_roots(runtime), + post_id, + host: Arc::clone(&self.host), + }; + let expanded = expand_builtin_macros(markdown.as_str(), ¯o_context); + render_markdown_to_html(&expanded) + } else { + render_markdown_to_html(markdown.as_str()) + }; rewrite_rendered_html_urls(&rendered, &rewrite_context) }) .clone(), diff --git a/crates/bds-core/src/render/site.rs b/crates/bds-core/src/render/site.rs index 31a4dfe..b89d519 100644 --- a/crates/bds-core/src/render/site.rs +++ b/crates/bds-core/src/render/site.rs @@ -84,7 +84,7 @@ struct TemplateBundle { struct RenderPostRecord { post: Post, source_post_id: String, - body_markdown: String, + body_markdown: Arc, } #[derive(Debug, Clone)] @@ -118,13 +118,9 @@ struct LinkContext { pub struct SiteRenderContext { metadata: ProjectMetadata, - is_preview: bool, main_language: String, tags: Vec, category_settings: HashMap, - canonical_media_map: HashMap, - project_media: Vec, - project_tags: Vec, render_categories: HashMap, bundle: TemplateBundle, languages: Vec, @@ -149,9 +145,9 @@ struct LanguageRenderContext { linked_media_by_post_id: HashMap>, post_data_json_by_id: HashMap, menu_items: Vec, - canonical_post_path_by_slug: HashMap, taxonomy: TaxonomyContext, links: LinkContext, + liquid_globals: liquid::Object, } pub fn build_site_render_artifacts( @@ -191,7 +187,7 @@ pub fn build_site_route_manifest( .map(|post| RenderPostRecord { post: post.clone(), source_post_id: post.id.clone(), - body_markdown: String::new(), + body_markdown: Arc::from(""), }) .collect::>(); let list_posts = filter_posts_for_lists(&posts, &category_settings); @@ -400,6 +396,21 @@ pub fn prepare_site_render_context( canonical_post_path_by_slug(&posts, &language, &main_language); let taxonomy = build_taxonomy_context(&posts, &tags); let links = build_link_context(&posts, &post_links, &language, &main_language); + let liquid_globals = liquid::to_object(&json!({ + "language": language, + "language_prefix": language_prefix(&language, &main_language), + "main_language": main_language, + "is_preview": is_preview, + "macro_scripts": bundle.macro_scripts, + "macro_templates": bundle.macro_templates, + "pico_stylesheet_href": pico_stylesheet_href(metadata), + "menu_items": menu_items, + "canonical_post_path_by_slug": canonical_post_path_by_slug, + "canonical_media_path_by_source_path": canonical_media_map, + "project": { "media": project_media }, + "Tags": project_tags, + "tag_color_by_name": taxonomy.tag_colors, + }))?; language_contexts.push(LanguageRenderContext { language, posts, @@ -407,21 +418,17 @@ pub fn prepare_site_render_context( linked_media_by_post_id, post_data_json_by_id, menu_items, - canonical_post_path_by_slug, taxonomy, links, + liquid_globals, }); } Ok(SiteRenderContext { metadata: metadata.clone(), - is_preview, main_language, tags, category_settings, - canonical_media_map, - project_media, - project_tags, render_categories, bundle, languages: language_contexts, @@ -467,15 +474,10 @@ pub fn build_site_render_artifacts_from_context( metadata, language, &context.category_settings, - &language_context.menu_items, - &language_context.canonical_post_path_by_slug, &language_context.taxonomy, &language_context.post_data_json_by_id, - &context.canonical_media_map, - &context.project_media, - &context.project_tags, &context.bundle, - context.is_preview, + &language_context.liquid_globals, ) .map(|html| { on_page_rendered(&route.url_path); @@ -556,21 +558,15 @@ pub fn build_site_render_artifacts_from_context( render_post_route( metadata, language, - main_language, record, &context.tags, &context.render_categories, &language_context.linked_media_by_post_id, &language_context.links, - &language_context.canonical_post_path_by_slug, - &language_context.menu_items, &language_context.taxonomy, &language_context.post_data_json_by_id, - &context.canonical_media_map, - &context.project_media, - &context.project_tags, &context.bundle, - context.is_preview, + &language_context.liquid_globals, ) .map(|html| { on_page_rendered(url_path); @@ -811,7 +807,7 @@ fn load_language_posts( Some(PostLanguageVariant::Base) => posts.push(RenderPostRecord { post: post.clone(), source_post_id: post.id.clone(), - body_markdown: body.clone(), + body_markdown: Arc::from(body.as_str()), }), Some(PostLanguageVariant::Translation) => { let Some(translation) = translation else { @@ -837,7 +833,7 @@ fn load_language_posts( posts.push(RenderPostRecord { post: translated_post, source_post_id: post.id.clone(), - body_markdown: translated_body, + body_markdown: Arc::from(translated_body), }); } None => {} @@ -1141,35 +1137,22 @@ fn render_list_route( metadata: &ProjectMetadata, language: &str, category_settings: &HashMap, - menu_items: &[Value], - canonical_post_path_by_slug: &HashMap, taxonomy: &TaxonomyContext, post_data_json_by_id: &HashMap, - canonical_media_path_by_source_path: &HashMap, - project_media: &[Value], - project_tags: &[Value], bundle: &TemplateBundle, - is_preview: bool, + liquid_globals: &liquid::Object, ) -> Result> { - let main_language = main_language(metadata); + let post_data_json_by_id = page_post_data(&route.posts, post_data_json_by_id); let list_template = route .list_template_slug .as_deref() .and_then(|slug| bundle.list_template_by_slug.get(slug)) .unwrap_or(&bundle.default_list_template); let context = json!({ - "language": language, - "language_prefix": language_prefix(language, main_language), - "main_language": main_language, - "is_preview": is_preview, - "macro_scripts": bundle.macro_scripts, - "macro_templates": bundle.macro_templates, "html_theme_attribute": serde_json::Value::Null, "page_title": route.page_title, - "pico_stylesheet_href": pico_stylesheet_href(metadata), "blog_languages": build_list_blog_languages(metadata, language, &route.url_path), "alternate_links": build_alternate_list_links(metadata, &route.url_path), - "menu_items": menu_items, "calendar_initial_year": route.posts.first().map(|post| calendar_initial_parts(&post.post).0).unwrap_or(1970), "calendar_initial_month": route.posts.first().map(|post| calendar_initial_parts(&post.post).1).unwrap_or(1), "archive_context": route.archive_context, @@ -1188,41 +1171,32 @@ fn render_list_route( "total_pages": route.total_pages, "total_items": route.total_items, "items_per_page": route.items_per_page, - "canonical_post_path_by_slug": canonical_post_path_by_slug, - "canonical_media_path_by_source_path": canonical_media_path_by_source_path, "post_data_json_by_id": post_data_json_by_id, - "project": { "media": project_media }, - "Tags": project_tags, "post_categories": taxonomy.categories, "post_tags": taxonomy.tags, - "tag_color_by_name": taxonomy.tag_colors, "backlinks": Vec::::new(), "not_found_message": serde_json::Value::Null, "not_found_back_label": serde_json::Value::Null, }); - Ok(bundle.renderer.render(list_template, &context)?) + Ok(bundle + .renderer + .render_with_base(list_template, liquid_globals, &context)?) } #[allow(clippy::too_many_arguments)] fn render_post_route( metadata: &ProjectMetadata, language: &str, - main_language: &str, record: &RenderPostRecord, tags: &[Tag], render_categories: &HashMap, linked_media_by_post_id: &HashMap>, links: &LinkContext, - canonical_post_path_by_slug: &HashMap, - menu_items: &[Value], taxonomy: &TaxonomyContext, post_data_json_by_id: &HashMap, - canonical_media_path_by_source_path: &HashMap, - project_media: &[Value], - project_tags: &[Value], bundle: &TemplateBundle, - is_preview: bool, + liquid_globals: &liquid::Object, ) -> Result> { let resolved = resolve_post_template(RenderTemplateLookup { post: &record.post, @@ -1257,32 +1231,20 @@ fn render_post_route( .get(&record.source_post_id) .cloned() .unwrap_or_default(); + let post_data_json_by_id = page_post_data(std::slice::from_ref(record), post_data_json_by_id); let context = json!({ - "language": language, - "language_prefix": language_prefix(language, main_language), - "main_language": main_language, - "is_preview": is_preview, - "macro_scripts": bundle.macro_scripts, - "macro_templates": bundle.macro_templates, "page_title": record.post.title, - "pico_stylesheet_href": pico_stylesheet_href(metadata), "html_theme_attribute": serde_json::Value::Null, "alternate_links": build_alternate_post_links(&record.post, metadata), "blog_languages": build_post_blog_languages(&record.post, metadata, language), - "menu_items": menu_items, "calendar_initial_year": calendar_initial_parts(&record.post).0, "calendar_initial_month": calendar_initial_parts(&record.post).1, - "post": post_context(&record.post, &record.body_markdown, linked_media, outgoing_link_context, incoming_link_context), + "post": post_context(&record.post, record.body_markdown.as_ref(), linked_media, outgoing_link_context, incoming_link_context), "post_categories": taxonomy_items_for_categories(&record.post.categories, taxonomy), "post_tags": taxonomy_items_for_tags(&record.post.tags, taxonomy, tags), - "tag_color_by_name": taxonomy.tag_colors, "backlinks": backlinks, - "canonical_post_path_by_slug": canonical_post_path_by_slug, - "canonical_media_path_by_source_path": canonical_media_path_by_source_path, "post_data_json_by_id": post_data_json_by_id, - "project": { "media": project_media }, - "Tags": project_tags, "day_blocks": Vec::::new(), "archive_context": serde_json::Value::Null, "show_archive_range_heading": false, @@ -1299,7 +1261,9 @@ fn render_post_route( "not_found_back_label": serde_json::Value::Null, }); - Ok(bundle.renderer.render(&template, &context)?) + Ok(bundle + .renderer + .render_with_base(&template, liquid_globals, &context)?) } fn render_not_found_route( @@ -1521,7 +1485,7 @@ fn resolve_list_content( if show_title && !excerpt.is_empty() { record.post.excerpt.clone().unwrap_or_default() } else { - record.body_markdown.clone() + record.body_markdown.to_string() } } @@ -1622,6 +1586,20 @@ fn build_post_data_json_by_id( .collect() } +fn page_post_data<'a>( + posts: &'a [RenderPostRecord], + post_data_json_by_id: &'a HashMap, +) -> HashMap<&'a str, &'a Value> { + posts + .iter() + .filter_map(|record| { + post_data_json_by_id + .get(&record.post.id) + .map(|value| (record.post.id.as_str(), value)) + }) + .collect() +} + fn build_published_tag_counts(posts: &[(Post, String)], tags: &[Tag]) -> Vec { let mut counts = HashMap::::new(); for (post, _) in posts { diff --git a/crates/bds-core/tests/m4_generation_engine.rs b/crates/bds-core/tests/m4_generation_engine.rs index 71955c8..984d03f 100644 --- a/crates/bds-core/tests/m4_generation_engine.rs +++ b/crates/bds-core/tests/m4_generation_engine.rs @@ -480,6 +480,50 @@ fn section_generation_reports_its_urls_and_defers_pagefind() { assert!(!dir.path().join(old_fragment).exists()); } +#[test] +fn list_pages_receive_only_their_page_post_data_like_bds2() { + let (db, dir) = setup(); + insert_template( + db.conn(), + &make_list_template("list", "POST_DATA={{ post_data_json_by_id | size }}"), + ) + .unwrap(); + let mut metadata = make_metadata(); + metadata.max_posts_per_page = 1; + let posts = vec![ + PublishedPostSource { + post: make_post("hello", 1_710_000_000_000), + body_markdown: "Hello".into(), + }, + PublishedPostSource { + post: make_post("next", 1_710_086_400_000), + body_markdown: "Next".into(), + }, + ]; + + render_site_section_with_progress( + db.conn(), + dir.path(), + "p1", + &metadata, + &posts, + GenerationSection::Core, + |_, _, _| {}, + |_| {}, + || false, + ) + .unwrap(); + + assert_eq!( + std::fs::read_to_string(dir.path().join("index.html")).unwrap(), + "POST_DATA=1" + ); + assert_eq!( + std::fs::read_to_string(dir.path().join("page/2/index.html")).unwrap(), + "POST_DATA=1" + ); +} + #[test] fn section_generation_reports_while_html_is_rendered_before_files_are_written() { let (db, dir) = setup(); diff --git a/crates/bds-ui/src/app.rs b/crates/bds-ui/src/app.rs index a329690..d23c686 100644 --- a/crates/bds-ui/src/app.rs +++ b/crates/bds-ui/src/app.rs @@ -11210,14 +11210,21 @@ mod tests { .iter() .all(|task| task.group_name.as_deref() == Some("Render Site")) ); - assert_eq!( - snapshots + let running = std::thread::available_parallelism() + .map(usize::from) + .unwrap_or(1) + .min(snapshots.len()); + assert!( + snapshots[..running] .iter() - .map(|task| task.status.clone()) - .collect::>(), - vec![Running, Running, Running, Pending, Pending] + .all(|task| task.status == Running) ); - assert!(snapshots[..3].iter().all(|task| { + assert!( + snapshots[running..] + .iter() + .all(|task| task.status == Pending) + ); + assert!(snapshots[..running].iter().all(|task| { task.progress.is_some() && task .message diff --git a/crates/bds-ui/src/app/tasks.rs b/crates/bds-ui/src/app/tasks.rs index 31666dc..bfdf784 100644 --- a/crates/bds-ui/src/app/tasks.rs +++ b/crates/bds-ui/src/app/tasks.rs @@ -149,7 +149,6 @@ impl BdsApp { ); let mut render_task_ids = Vec::new(); let mut tasks = Vec::new(); - let prepared_generation = Arc::new(std::sync::OnceLock::new()); for section in sections { let label = t(self.ui_locale, generation_section_label_key(section)); @@ -174,7 +173,6 @@ impl BdsApp { let task_data_dir = data_dir.clone(); let task_group_id = group_id.clone(); let task_validation = validation.clone(); - let task_prepared_generation = Arc::clone(&prepared_generation); let locale = self.ui_locale; tasks.push(Task::perform( async move { @@ -187,7 +185,6 @@ impl BdsApp { task_id, section, task_validation, - task_prepared_generation, force, locale, page_work, @@ -497,9 +494,6 @@ fn run_site_generation_section( task_id: TaskId, section: engine::generation::GenerationSection, validation: Option, - prepared_generation: Arc< - std::sync::OnceLock, String>>, - >, force: bool, locale: UiLocale, expected_pages: usize, @@ -517,33 +511,26 @@ fn run_site_generation_section( )), ); let db = Database::open(&db_path).map_err(|error| error.to_string())?; - let prepared = prepared_generation - .get_or_init(|| { - let metadata = - engine::meta::read_project_json(&data_dir).map_err(|error| error.to_string())?; - let posts = bds_core::db::queries::post::list_posts_by_project(db.conn(), &project_id) - .map_err(|error| error.to_string())?; - let sources = posts - .into_iter() - .filter(engine::generation::has_published_snapshot) - .map(|post| engine::generation::load_published_post_source(&data_dir, post)) - .collect::, _>>() - .map_err(|error| error.to_string())? - .into_iter() - .flatten() - .collect::>(); - engine::generation::prepare_site_generation( - db.conn(), - &data_dir, - &project_id, - &metadata, - &sources, - ) - .map(Arc::new) - .map_err(|error| error.to_string()) - }) - .as_ref() - .map_err(Clone::clone)?; + let metadata = engine::meta::read_project_json(&data_dir).map_err(|error| error.to_string())?; + let posts = bds_core::db::queries::post::list_posts_by_project(db.conn(), &project_id) + .map_err(|error| error.to_string())?; + let sources = posts + .into_iter() + .filter(engine::generation::has_published_snapshot) + .map(|post| engine::generation::load_published_post_source(&data_dir, post)) + .collect::, _>>() + .map_err(|error| error.to_string())? + .into_iter() + .flatten() + .collect::>(); + let prepared = engine::generation::prepare_site_generation( + db.conn(), + &data_dir, + &project_id, + &metadata, + &sources, + ) + .map_err(|error| error.to_string())?; let output_dir = data_dir.join("html"); std::fs::create_dir_all(&output_dir).map_err(|error| error.to_string())?; let render_manager = Arc::clone(&task_manager); @@ -578,7 +565,7 @@ fn run_site_generation_section( db.conn(), &output_dir, &project_id, - prepared, + &prepared, &validation, section, on_page, @@ -589,7 +576,7 @@ fn run_site_generation_section( db.conn(), &output_dir, &project_id, - prepared, + &prepared, section, force, &on_rendered, diff --git a/specs/task.allium b/specs/task.allium index f563224..2e795bb 100644 --- a/specs/task.allium +++ b/specs/task.allium @@ -54,7 +54,8 @@ surface TaskSurface { } config { - max_concurrent: Integer = 3 + -- Runtime default: all online CPU workers, matching bDS2. + max_concurrent: Integer progress_throttle: Duration = 250.milliseconds finished_task_ttl: Duration = 1.hour recent_finished_limit: Integer = 10