hugo/RuDS
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

fix: still working on rendering speed

Browse Source
This commit is contained in:
Chili Palmer
2026-07-24 07:10:57 +02:00
parent 473c48937d
commit d9085cf681
7 changed files with 176 additions and 123 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
crates/bds-core/src/engine/task.rs
View File

@@ -337,7 +337,11 @@ impl TaskManager {
impl Default for TaskManager {
fn default() -> Self {
Self::new(3)
Self::new(
std::thread::available_parallelism()
.map(usize::from)
.unwrap_or(1),
)
}
}
@@ -358,6 +362,19 @@ mod tests {
assert_eq!(mgr.status(id), Some(TaskStatus::Running));
}
#[test]
fn default_uses_every_online_worker_like_bds2() {
let mgr = TaskManager::default();
let expected = std::thread::available_parallelism()
.map(usize::from)
.unwrap_or(1);
for index in 0..expected {
mgr.submit(&format!("task {index}"));
}
assert_eq!(mgr.running_count(), expected);
}
#[test]
fn max_concurrent_enforced() {
let mgr = TaskManager::new(3);

37
crates/bds-core/src/render/page_renderer.rs
View File

@@ -4,7 +4,7 @@ use std::sync::{Arc, Mutex, OnceLock};
use liquid::ParserBuilder;
use liquid::partials::{EagerCompiler, InMemorySource};
use liquid_core::model::ScalarCow;
use liquid_core::model::{Object, ScalarCow, ValueCow};
use liquid_core::parser::FilterArguments;
use liquid_core::{
Display_filter, Expression, Filter, FilterParameters, FilterReflection, FromFilterParameters,
@@ -62,6 +62,21 @@ impl LiquidRenderer {
let globals = liquid::to_object(context)?;
Ok(template.0.render(&globals)?)
}
pub(crate) fn render_with_base<T: Serialize>(
&self,
template: &CompiledLiquidTemplate,
base: &Object,
context: &T,
) -> Result<String, RenderError> {
let page = liquid::to_object(context)?;
let globals = base
.iter()
.chain(page.iter())
.map(|(key, value)| (key.to_string(), ValueCow::from(value)))
.collect::<HashMap<_, _>>();
Ok(template.0.render(&globals)?)
}
}
#[derive(Debug, Clone, Default)]
@@ -261,15 +276,10 @@ impl Filter for MarkdownFilter {
.and_then(|value| value.as_scalar())
.map(|scalar| scalar.to_kstr().to_string())
.unwrap_or_default();
let macro_context = MacroRenderContext {
roots: collect_macro_roots(runtime),
post_id: post_id.clone(),
host: Arc::clone(&self.host),
};
let key = MarkdownCacheKey {
content_hash: content_hash(markdown.as_bytes()),
language,
post_id,
post_id: post_id.clone(),
};
let entry = {
let mut cache = self.cache.lock().unwrap_or_else(|error| error.into_inner());
@@ -278,8 +288,17 @@ impl Filter for MarkdownFilter {
Ok(Value::scalar(
entry
.get_or_init(|| {
let expanded = expand_builtin_macros(markdown.as_str(), &macro_context);
let rendered = render_markdown_to_html(&expanded);
let rendered = if markdown.contains("[[") {
let macro_context = MacroRenderContext {
roots: collect_macro_roots(runtime),
post_id,
host: Arc::clone(&self.host),
};
let expanded = expand_builtin_macros(markdown.as_str(), &macro_context);
render_markdown_to_html(&expanded)
} else {
render_markdown_to_html(markdown.as_str())
};
rewrite_rendered_html_urls(&rendered, &rewrite_context)
})
.clone(),

120
crates/bds-core/src/render/site.rs
View File

@@ -84,7 +84,7 @@ struct TemplateBundle {
struct RenderPostRecord {
post: Post,
source_post_id: String,
body_markdown: String,
body_markdown: Arc<str>,
}
#[derive(Debug, Clone)]
@@ -118,13 +118,9 @@ struct LinkContext {
pub struct SiteRenderContext {
metadata: ProjectMetadata,
is_preview: bool,
main_language: String,
tags: Vec<Tag>,
category_settings: HashMap<String, CategorySettings>,
canonical_media_map: HashMap<String, String>,
project_media: Vec<Value>,
project_tags: Vec<Value>,
render_categories: HashMap<String, RenderCategorySettings>,
bundle: TemplateBundle,
languages: Vec<LanguageRenderContext>,
@@ -149,9 +145,9 @@ struct LanguageRenderContext {
linked_media_by_post_id: HashMap<String, Vec<Value>>,
post_data_json_by_id: HashMap<String, Value>,
menu_items: Vec<Value>,
canonical_post_path_by_slug: HashMap<String, String>,
taxonomy: TaxonomyContext,
links: LinkContext,
liquid_globals: liquid::Object,
}
pub fn build_site_render_artifacts(
@@ -191,7 +187,7 @@ pub fn build_site_route_manifest(
.map(|post| RenderPostRecord {
post: post.clone(),
source_post_id: post.id.clone(),
body_markdown: String::new(),
body_markdown: Arc::from(""),
})
.collect::<Vec<_>>();
let list_posts = filter_posts_for_lists(&posts, &category_settings);
@@ -400,6 +396,21 @@ pub fn prepare_site_render_context(
canonical_post_path_by_slug(&posts, &language, &main_language);
let taxonomy = build_taxonomy_context(&posts, &tags);
let links = build_link_context(&posts, &post_links, &language, &main_language);
let liquid_globals = liquid::to_object(&json!({
"language": language,
"language_prefix": language_prefix(&language, &main_language),
"main_language": main_language,
"is_preview": is_preview,
"macro_scripts": bundle.macro_scripts,
"macro_templates": bundle.macro_templates,
"pico_stylesheet_href": pico_stylesheet_href(metadata),
"menu_items": menu_items,
"canonical_post_path_by_slug": canonical_post_path_by_slug,
"canonical_media_path_by_source_path": canonical_media_map,
"project": { "media": project_media },
"Tags": project_tags,
"tag_color_by_name": taxonomy.tag_colors,
}))?;
language_contexts.push(LanguageRenderContext {
language,
posts,
@@ -407,21 +418,17 @@ pub fn prepare_site_render_context(
linked_media_by_post_id,
post_data_json_by_id,
menu_items,
canonical_post_path_by_slug,
taxonomy,
links,
liquid_globals,
});
}
Ok(SiteRenderContext {
metadata: metadata.clone(),
is_preview,
main_language,
tags,
category_settings,
canonical_media_map,
project_media,
project_tags,
render_categories,
bundle,
languages: language_contexts,
@@ -467,15 +474,10 @@ pub fn build_site_render_artifacts_from_context(
metadata,
language,
&context.category_settings,
&language_context.menu_items,
&language_context.canonical_post_path_by_slug,
&language_context.taxonomy,
&language_context.post_data_json_by_id,
&context.canonical_media_map,
&context.project_media,
&context.project_tags,
&context.bundle,
context.is_preview,
&language_context.liquid_globals,
)
.map(|html| {
on_page_rendered(&route.url_path);
@@ -556,21 +558,15 @@ pub fn build_site_render_artifacts_from_context(
render_post_route(
metadata,
language,
main_language,
record,
&context.tags,
&context.render_categories,
&language_context.linked_media_by_post_id,
&language_context.links,
&language_context.canonical_post_path_by_slug,
&language_context.menu_items,
&language_context.taxonomy,
&language_context.post_data_json_by_id,
&context.canonical_media_map,
&context.project_media,
&context.project_tags,
&context.bundle,
context.is_preview,
&language_context.liquid_globals,
)
.map(|html| {
on_page_rendered(url_path);
@@ -811,7 +807,7 @@ fn load_language_posts(
Some(PostLanguageVariant::Base) => posts.push(RenderPostRecord {
post: post.clone(),
source_post_id: post.id.clone(),
body_markdown: body.clone(),
body_markdown: Arc::from(body.as_str()),
}),
Some(PostLanguageVariant::Translation) => {
let Some(translation) = translation else {
@@ -837,7 +833,7 @@ fn load_language_posts(
posts.push(RenderPostRecord {
post: translated_post,
source_post_id: post.id.clone(),
body_markdown: translated_body,
body_markdown: Arc::from(translated_body),
});
}
None => {}
@@ -1141,35 +1137,22 @@ fn render_list_route(
metadata: &ProjectMetadata,
language: &str,
category_settings: &HashMap<String, CategorySettings>,
menu_items: &[Value],
canonical_post_path_by_slug: &HashMap<String, String>,
taxonomy: &TaxonomyContext,
post_data_json_by_id: &HashMap<String, Value>,
canonical_media_path_by_source_path: &HashMap<String, String>,
project_media: &[Value],
project_tags: &[Value],
bundle: &TemplateBundle,
is_preview: bool,
liquid_globals: &liquid::Object,
) -> Result<String, Box<dyn Error + Send + Sync>> {
let main_language = main_language(metadata);
let post_data_json_by_id = page_post_data(&route.posts, post_data_json_by_id);
let list_template = route
.list_template_slug
.as_deref()
.and_then(|slug| bundle.list_template_by_slug.get(slug))
.unwrap_or(&bundle.default_list_template);
let context = json!({
"language": language,
"language_prefix": language_prefix(language, main_language),
"main_language": main_language,
"is_preview": is_preview,
"macro_scripts": bundle.macro_scripts,
"macro_templates": bundle.macro_templates,
"html_theme_attribute": serde_json::Value::Null,
"page_title": route.page_title,
"pico_stylesheet_href": pico_stylesheet_href(metadata),
"blog_languages": build_list_blog_languages(metadata, language, &route.url_path),
"alternate_links": build_alternate_list_links(metadata, &route.url_path),
"menu_items": menu_items,
"calendar_initial_year": route.posts.first().map(|post| calendar_initial_parts(&post.post).0).unwrap_or(1970),
"calendar_initial_month": route.posts.first().map(|post| calendar_initial_parts(&post.post).1).unwrap_or(1),
"archive_context": route.archive_context,
@@ -1188,41 +1171,32 @@ fn render_list_route(
"total_pages": route.total_pages,
"total_items": route.total_items,
"items_per_page": route.items_per_page,
"canonical_post_path_by_slug": canonical_post_path_by_slug,
"canonical_media_path_by_source_path": canonical_media_path_by_source_path,
"post_data_json_by_id": post_data_json_by_id,
"project": { "media": project_media },
"Tags": project_tags,
"post_categories": taxonomy.categories,
"post_tags": taxonomy.tags,
"tag_color_by_name": taxonomy.tag_colors,
"backlinks": Vec::<Value>::new(),
"not_found_message": serde_json::Value::Null,
"not_found_back_label": serde_json::Value::Null,
});
Ok(bundle.renderer.render(list_template, &context)?)
Ok(bundle
.renderer
.render_with_base(list_template, liquid_globals, &context)?)
}
#[allow(clippy::too_many_arguments)]
fn render_post_route(
metadata: &ProjectMetadata,
language: &str,
main_language: &str,
record: &RenderPostRecord,
tags: &[Tag],
render_categories: &HashMap<String, RenderCategorySettings>,
linked_media_by_post_id: &HashMap<String, Vec<Value>>,
links: &LinkContext,
canonical_post_path_by_slug: &HashMap<String, String>,
menu_items: &[Value],
taxonomy: &TaxonomyContext,
post_data_json_by_id: &HashMap<String, Value>,
canonical_media_path_by_source_path: &HashMap<String, String>,
project_media: &[Value],
project_tags: &[Value],
bundle: &TemplateBundle,
is_preview: bool,
liquid_globals: &liquid::Object,
) -> Result<String, Box<dyn Error + Send + Sync>> {
let resolved = resolve_post_template(RenderTemplateLookup {
post: &record.post,
@@ -1257,32 +1231,20 @@ fn render_post_route(
.get(&record.source_post_id)
.cloned()
.unwrap_or_default();
let post_data_json_by_id = page_post_data(std::slice::from_ref(record), post_data_json_by_id);
let context = json!({
"language": language,
"language_prefix": language_prefix(language, main_language),
"main_language": main_language,
"is_preview": is_preview,
"macro_scripts": bundle.macro_scripts,
"macro_templates": bundle.macro_templates,
"page_title": record.post.title,
"pico_stylesheet_href": pico_stylesheet_href(metadata),
"html_theme_attribute": serde_json::Value::Null,
"alternate_links": build_alternate_post_links(&record.post, metadata),
"blog_languages": build_post_blog_languages(&record.post, metadata, language),
"menu_items": menu_items,
"calendar_initial_year": calendar_initial_parts(&record.post).0,
"calendar_initial_month": calendar_initial_parts(&record.post).1,
"post": post_context(&record.post, &record.body_markdown, linked_media, outgoing_link_context, incoming_link_context),
"post": post_context(&record.post, record.body_markdown.as_ref(), linked_media, outgoing_link_context, incoming_link_context),
"post_categories": taxonomy_items_for_categories(&record.post.categories, taxonomy),
"post_tags": taxonomy_items_for_tags(&record.post.tags, taxonomy, tags),
"tag_color_by_name": taxonomy.tag_colors,
"backlinks": backlinks,
"canonical_post_path_by_slug": canonical_post_path_by_slug,
"canonical_media_path_by_source_path": canonical_media_path_by_source_path,
"post_data_json_by_id": post_data_json_by_id,
"project": { "media": project_media },
"Tags": project_tags,
"day_blocks": Vec::<Value>::new(),
"archive_context": serde_json::Value::Null,
"show_archive_range_heading": false,
@@ -1299,7 +1261,9 @@ fn render_post_route(
"not_found_back_label": serde_json::Value::Null,
});
Ok(bundle.renderer.render(&template, &context)?)
Ok(bundle
.renderer
.render_with_base(&template, liquid_globals, &context)?)
}
fn render_not_found_route(
@@ -1521,7 +1485,7 @@ fn resolve_list_content(
if show_title && !excerpt.is_empty() {
record.post.excerpt.clone().unwrap_or_default()
} else {
record.body_markdown.clone()
record.body_markdown.to_string()
}
}
@@ -1622,6 +1586,20 @@ fn build_post_data_json_by_id(
.collect()
}
fn page_post_data<'a>(
posts: &'a [RenderPostRecord],
post_data_json_by_id: &'a HashMap<String, Value>,
) -> HashMap<&'a str, &'a Value> {
posts
.iter()
.filter_map(|record| {
post_data_json_by_id
.get(&record.post.id)
.map(|value| (record.post.id.as_str(), value))
})
.collect()
}
fn build_published_tag_counts(posts: &[(Post, String)], tags: &[Tag]) -> Vec<Value> {
let mut counts = HashMap::<String, usize>::new();
for (post, _) in posts {

44
crates/bds-core/tests/m4_generation_engine.rs
View File

@@ -480,6 +480,50 @@ fn section_generation_reports_its_urls_and_defers_pagefind() {
assert!(!dir.path().join(old_fragment).exists());
}
#[test]
fn list_pages_receive_only_their_page_post_data_like_bds2() {
let (db, dir) = setup();
insert_template(
db.conn(),
&make_list_template("list", "POST_DATA={{ post_data_json_by_id | size }}"),
)
.unwrap();
let mut metadata = make_metadata();
metadata.max_posts_per_page = 1;
let posts = vec![
PublishedPostSource {
post: make_post("hello", 1_710_000_000_000),
body_markdown: "Hello".into(),
},
PublishedPostSource {
post: make_post("next", 1_710_086_400_000),
body_markdown: "Next".into(),
},
];
render_site_section_with_progress(
db.conn(),
dir.path(),
"p1",
&metadata,
&posts,
GenerationSection::Core,
|_, _, _| {},
|_| {},
|| false,
)
.unwrap();
assert_eq!(
std::fs::read_to_string(dir.path().join("index.html")).unwrap(),
"POST_DATA=1"
);
assert_eq!(
std::fs::read_to_string(dir.path().join("page/2/index.html")).unwrap(),
"POST_DATA=1"
);
}
#[test]
fn section_generation_reports_while_html_is_rendered_before_files_are_written() {
let (db, dir) = setup();

19
crates/bds-ui/src/app.rs
View File

@@ -11210,14 +11210,21 @@ mod tests {
.iter()
.all(|task| task.group_name.as_deref() == Some("Render Site"))
);
assert_eq!(
snapshots
let running = std::thread::available_parallelism()
.map(usize::from)
.unwrap_or(1)
.min(snapshots.len());
assert!(
snapshots[..running]
.iter()
.map(|task| task.status.clone())
.collect::<Vec<_>>(),
vec![Running, Running, Running, Pending, Pending]
.all(|task| task.status == Running)
);
assert!(snapshots[..3].iter().all(|task| {
assert!(
snapshots[running..]
.iter()
.all(|task| task.status == Pending)
);
assert!(snapshots[..running].iter().all(|task| {
task.progress.is_some()
&& task
.message

57
crates/bds-ui/src/app/tasks.rs
View File

@@ -149,7 +149,6 @@ impl BdsApp {
);
let mut render_task_ids = Vec::new();
let mut tasks = Vec::new();
let prepared_generation = Arc::new(std::sync::OnceLock::new());
for section in sections {
let label = t(self.ui_locale, generation_section_label_key(section));
@@ -174,7 +173,6 @@ impl BdsApp {
let task_data_dir = data_dir.clone();
let task_group_id = group_id.clone();
let task_validation = validation.clone();
let task_prepared_generation = Arc::clone(&prepared_generation);
let locale = self.ui_locale;
tasks.push(Task::perform(
async move {
@@ -187,7 +185,6 @@ impl BdsApp {
task_id,
section,
task_validation,
task_prepared_generation,
force,
locale,
page_work,
@@ -497,9 +494,6 @@ fn run_site_generation_section(
task_id: TaskId,
section: engine::generation::GenerationSection,
validation: Option<engine::validate_site::SiteValidationReport>,
prepared_generation: Arc<
std::sync::OnceLock<Result<Arc<engine::generation::PreparedSiteGeneration>, String>>,
>,
force: bool,
locale: UiLocale,
expected_pages: usize,
@@ -517,33 +511,26 @@ fn run_site_generation_section(
)),
);
let db = Database::open(&db_path).map_err(|error| error.to_string())?;
let prepared = prepared_generation
.get_or_init(|| {
let metadata =
engine::meta::read_project_json(&data_dir).map_err(|error| error.to_string())?;
let posts = bds_core::db::queries::post::list_posts_by_project(db.conn(), &project_id)
.map_err(|error| error.to_string())?;
let sources = posts
.into_iter()
.filter(engine::generation::has_published_snapshot)
.map(|post| engine::generation::load_published_post_source(&data_dir, post))
.collect::<Result<Vec<_>, _>>()
.map_err(|error| error.to_string())?
.into_iter()
.flatten()
.collect::<Vec<_>>();
engine::generation::prepare_site_generation(
db.conn(),
&data_dir,
&project_id,
&metadata,
&sources,
)
.map(Arc::new)
.map_err(|error| error.to_string())
})
.as_ref()
.map_err(Clone::clone)?;
let metadata = engine::meta::read_project_json(&data_dir).map_err(|error| error.to_string())?;
let posts = bds_core::db::queries::post::list_posts_by_project(db.conn(), &project_id)
.map_err(|error| error.to_string())?;
let sources = posts
.into_iter()
.filter(engine::generation::has_published_snapshot)
.map(|post| engine::generation::load_published_post_source(&data_dir, post))
.collect::<Result<Vec<_>, _>>()
.map_err(|error| error.to_string())?
.into_iter()
.flatten()
.collect::<Vec<_>>();
let prepared = engine::generation::prepare_site_generation(
db.conn(),
&data_dir,
&project_id,
&metadata,
&sources,
)
.map_err(|error| error.to_string())?;
let output_dir = data_dir.join("html");
std::fs::create_dir_all(&output_dir).map_err(|error| error.to_string())?;
let render_manager = Arc::clone(&task_manager);
@@ -578,7 +565,7 @@ fn run_site_generation_section(
db.conn(),
&output_dir,
&project_id,
prepared,
&prepared,
&validation,
section,
on_page,
@@ -589,7 +576,7 @@ fn run_site_generation_section(
db.conn(),
&output_dir,
&project_id,
prepared,
&prepared,
section,
force,
&on_rendered,

3
specs/task.allium
View File

@@ -54,7 +54,8 @@ surface TaskSurface {
}
config {
max_concurrent: Integer = 3
-- Runtime default: all online CPU workers, matching bDS2.
max_concurrent: Integer
progress_throttle: Duration = 250.milliseconds
finished_task_ttl: Duration = 1.hour
recent_finished_limit: Integer = 10