diff --git a/README.md b/README.md index 5657707..9fe5a65 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ The project is under active development. Core blogging workflows are broadly ava - Local MCP automation over stdio or a localhost-only stateless HTTP endpoint, with project resources, read/search/count tools, uniquely identified inert write proposals, clean duplicate-pending rejection, explicit desktop approval, and opt-in Claude Code/Copilot configuration. - A fully localized Ratatui terminal workspace, available locally through `bds-cli tui`/`BDS_MODE=tui` and remotely through authenticated SSH shell sessions, with shared post/template/script editing and publishing, project/search/command overlays, settings, tags, Git, reports, task progress, live multi-client locale updates, and airplane-mode AI gating. - `bds-cli server` hosting the shared application engines over a loopback-by-default, public-key-only SSH service, with restrictive private key material, live authorization updates, terminal-session transport, CLI-change synchronization, ordered domain/task events, and native desktop remote-project selection. -- bDS2-compatible Markdown/Liquid rendering with built-in macros rendered from bundled Liquid templates in isolated scopes (customizable with `macros/*` partial-template slugs), descriptive category archive titles, canonical multilingual and flat page routes for every configured blog language, recursive menus, calendar archives, feeds, a root hreflang sitemap, Pagefind, and incremental site generation and validation repair through cancellable section task groups. +- bDS2-compatible Markdown/Liquid rendering with built-in macros rendered from bundled Liquid templates in isolated scopes (customizable with `macros/*` partial-template slugs), descriptive category archive titles, canonical multilingual and flat page routes for every configured blog language, recursive menus, calendar archives, feeds, a root hreflang sitemap, Pagefind, and fast route/mtime-based incremental validation with targeted repair through cancellable section task groups. - Navigable generated-route preview in the app or system browser, with draft database overlays and published filesystem content. - Optional one-shot AI translation, description, analysis, taxonomy, and language-detection operations run in background tasks with editor-level waiting indicators, using provider-portable JSON-only requests through independent online and local OpenAI-compatible profiles. Each profile has secure credentials, persistently discovered chat/title/image model selections, explicit tool/vision overrides, chat testing, and restart-persistent status-bar airplane-mode routing. - Persistent conversational AI with safe Markdown, streamed and cancellable responses, model/session/token tracking, bounded project-aware blog tools, and localized conversation management in the Chat workspace. Allowlisted render tools add persistent native cards, charts, forms, lists, metrics, mind maps, tables, and tabs without executing assistant-provided HTML or JavaScript. diff --git a/crates/bds-cli/src/lib.rs b/crates/bds-cli/src/lib.rs index 245a94c..79c5443 100644 --- a/crates/bds-cli/src/lib.rs +++ b/crates/bds-cli/src/lib.rs @@ -513,6 +513,7 @@ fn render(db: &Database, incremental: bool, force: bool) -> Result QueryResult> { + conn.with(|c| { + generated_file_hashes::table + .filter(generated_file_hashes::project_id.eq(project_id)) + .select(GeneratedFileHash::as_select()) + .load(c) + }) +} + pub fn upsert_generated_file_hash( conn: &DbConnection, hash: &GeneratedFileHash, @@ -39,6 +51,26 @@ pub fn upsert_generated_file_hash( }) } +pub fn touch_generated_file_hashes( + conn: &DbConnection, + project_id: &str, + relative_paths: &[String], + updated_at: i64, +) -> QueryResult { + conn.with(|c| { + relative_paths.chunks(500).try_fold(0, |updated, paths| { + diesel::update( + generated_file_hashes::table + .filter(generated_file_hashes::project_id.eq(project_id)) + .filter(generated_file_hashes::relative_path.eq_any(paths)), + ) + .set(generated_file_hashes::updated_at.eq(updated_at)) + .execute(c) + .map(|count| updated + count) + }) + }) +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/bds-core/src/engine/generation.rs b/crates/bds-core/src/engine/generation.rs index 110b686..13bcb17 100644 --- a/crates/bds-core/src/engine/generation.rs +++ b/crates/bds-core/src/engine/generation.rs @@ -14,9 +14,8 @@ use crate::engine::{EngineError, EngineResult}; use crate::model::{CategorySettings, Post, ProjectMetadata}; use crate::render::{ GeneratedWriteOutcome, PostLanguageVariant, build_calendar_json, build_canonical_post_path, - build_site_render_artifacts, build_site_section_render_artifacts, - build_targeted_site_section_render_artifacts, select_post_language_variant, - write_generated_bytes, write_generated_file, + build_site_section_render_artifacts, build_targeted_site_section_render_artifacts, + select_post_language_variant, write_generated_bytes, write_generated_file, }; #[derive(Debug, Clone)] @@ -248,44 +247,28 @@ pub fn apply_validation_sections( project_id: &str, metadata: &ProjectMetadata, posts: &[PublishedPostSource], + validation: &SiteValidationReport, sections: &[GenerationSection], ) -> EngineResult { if sections.is_empty() { return Ok(GenerationReport::default()); } - let section_set = sections.iter().copied().collect::>(); - let data_dir = project_data_dir(output_dir); - let input_posts = posts - .iter() - .map(|source| (source.post.clone(), source.body_markdown.clone())) - .collect::>(); - let artifacts = - build_site_render_artifacts(conn, &data_dir, project_id, metadata, &input_posts) - .map_err(|error| EngineError::Parse(error.to_string()))?; let mut report = GenerationReport::default(); - let expected_paths = expected_paths_for_sections(metadata, &artifacts.pages, §ion_set); for section in sections { - report.append(render_site_section_with_progress( + report.append(apply_validation_section_with_progress( conn, output_dir, project_id, metadata, posts, + validation, *section, |_current, _total, _url| {}, || false, )?); } - - remove_extra_section_paths( - output_dir, - &expected_paths, - metadata, - §ion_set, - &mut report, - )?; report.append(build_site_search_index( conn, output_dir, project_id, metadata, )?); @@ -392,9 +375,31 @@ pub fn apply_validation_section_with_progress( report.deleted_paths.push(path.clone()); } } + refresh_route_timestamps(conn, project_id, &report)?; Ok(report) } +fn refresh_route_timestamps( + conn: &Connection, + project_id: &str, + report: &GenerationReport, +) -> EngineResult<()> { + let paths = report + .written_paths + .iter() + .chain(&report.skipped_paths) + .filter(|path| *path == "index.html" || path.ends_with("/index.html")) + .cloned() + .collect::>(); + queries::generated_file_hash::touch_generated_file_hashes( + conn, + project_id, + &paths, + crate::util::now_unix_ms(), + )?; + Ok(()) +} + #[expect( clippy::too_many_arguments, reason = "generation context is existing domain data" @@ -682,96 +687,6 @@ fn project_data_dir(output_dir: &Path) -> std::path::PathBuf { } } -fn expected_paths_for_sections( - metadata: &ProjectMetadata, - pages: &[crate::render::SitePage], - sections: &HashSet, -) -> HashSet { - let mut expected = pages - .iter() - .filter(|page| path_matches_sections(&page.relative_path, metadata, sections)) - .map(|page| page.relative_path.clone()) - .collect::>(); - - if sections.contains(&GenerationSection::Core) { - expected.insert("calendar.json".to_string()); - expected.insert("rss.xml".to_string()); - for language in render_languages(metadata) { - let prefix = if language - == metadata - .main_language - .clone() - .unwrap_or_else(|| "en".to_string()) - { - String::new() - } else { - format!("{language}/") - }; - expected.insert(format!("{prefix}rss.xml")); - expected.insert(format!("{prefix}atom.xml")); - } - expected.insert("sitemap.xml".to_string()); - } - - expected -} - -fn remove_extra_section_paths( - output_dir: &Path, - expected: &HashSet, - metadata: &ProjectMetadata, - sections: &HashSet, - report: &mut GenerationReport, -) -> EngineResult<()> { - if !output_dir.exists() { - return Ok(()); - } - - let mut deleted = Vec::new(); - for entry in WalkDir::new(output_dir).into_iter().filter_map(Result::ok) { - if !entry.file_type().is_file() { - continue; - } - let rel = entry - .path() - .strip_prefix(output_dir) - .unwrap_or(entry.path()) - .to_string_lossy() - .replace('\\', "/"); - if rel.starts_with("meta/") - || rel.starts_with("posts/") - || rel.starts_with("media/") - || rel.starts_with("assets/") - || rel.starts_with("pagefind") - || rel.contains("/pagefind/") - { - continue; - } - if !matches_generated_extension(&rel) - || !path_matches_sections(&rel, metadata, sections) - || expected.contains(&rel) - { - continue; - } - std::fs::remove_file(entry.path()).map_err(EngineError::Io)?; - deleted.push(rel); - } - - deleted.sort(); - report.deleted_paths.extend(deleted); - Ok(()) -} - -fn path_matches_sections( - path: &str, - metadata: &ProjectMetadata, - sections: &HashSet, -) -> bool { - classify_generated_path(path, metadata) - .map(|section| sections.contains(§ion)) - .unwrap_or(false) -} - pub(crate) fn classify_generated_path( path: &str, metadata: &ProjectMetadata, @@ -841,10 +756,6 @@ fn is_day_segment(value: &str) -> bool { is_month_segment(value) } -fn matches_generated_extension(path: &str) -> bool { - path.ends_with(".html") || path.ends_with(".xml") || path.ends_with(".json") -} - fn all_sections() -> Vec { GenerationSection::ALL.to_vec() } @@ -975,6 +886,37 @@ fn filter_posts_for_lists( .collect() } +pub(crate) fn refresh_validation_sitemap( + conn: &Connection, + output_dir: &Path, + project_id: &str, + data_dir: &Path, + metadata: &ProjectMetadata, + posts: &[Post], + route_manifest: &[crate::render::SitePage], +) -> EngineResult<()> { + let mut sources = posts + .iter() + .cloned() + .map(|post| PublishedPostSource { + post, + body_markdown: String::new(), + }) + .collect::>(); + sort_published_sources(&mut sources); + let list_posts = filter_posts_for_lists(&sources, &load_category_settings(data_dir)); + let content = build_sitemap_xml( + metadata, + route_manifest, + &sources, + &list_posts, + metadata.main_language.as_deref().unwrap_or("en"), + ); + write_generated_file(conn, output_dir, project_id, "sitemap.xml", &content) + .map_err(|error| EngineError::Parse(error.to_string()))?; + Ok(()) +} + pub(crate) fn build_rss_xml( metadata: &ProjectMetadata, posts: &[PublishedPostSource], diff --git a/crates/bds-core/src/engine/validate_site.rs b/crates/bds-core/src/engine/validate_site.rs index 3e785f7..986bfa4 100644 --- a/crates/bds-core/src/engine/validate_site.rs +++ b/crates/bds-core/src/engine/validate_site.rs @@ -1,14 +1,17 @@ -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use std::path::Path; +use std::time::{SystemTime, UNIX_EPOCH}; use crate::db::DbConnection as Connection; use walkdir::WalkDir; use crate::db::queries; -use crate::engine::{EngineError, EngineResult}; +use crate::engine::EngineResult; +use crate::engine::generation::has_published_snapshot; use crate::model::Post; -use crate::render::build_site_render_artifacts; -use crate::util::file_hash; +use crate::render::{build_canonical_post_path, build_site_route_manifest}; + +const MTIME_GRANULARITY_TOLERANCE_MS: i64 = 1_000; #[derive(Debug, Clone, Default)] pub struct SiteValidationReport { @@ -24,79 +27,55 @@ pub fn validate_site( ) -> EngineResult { let metadata = crate::engine::meta::read_project_json(data_dir)?; let output_dir = generated_output_dir(data_dir); - let published_posts = load_published_posts(data_dir, conn, project_id)?; - let artifacts = - build_site_render_artifacts(conn, data_dir, project_id, &metadata, &published_posts) - .map_err(|error| EngineError::Parse(error.to_string()))?; - - let mut expected = artifacts - .pages - .iter() - .map(|page| page.relative_path.clone()) + let published_posts = load_published_posts(conn, project_id)?; + let route_manifest = build_site_route_manifest(data_dir, &metadata, &published_posts) + .map_err(|error| crate::engine::EngineError::Parse(error.to_string()))?; + crate::engine::generation::refresh_validation_sitemap( + conn, + &output_dir, + project_id, + data_dir, + &metadata, + &published_posts, + &route_manifest, + )?; + let expected = route_manifest + .into_iter() + .map(|page| page.relative_path) .collect::>(); - expected.insert("calendar.json".to_string()); - for language in render_languages(&metadata) { - let prefix = if language - == metadata - .main_language - .clone() - .unwrap_or_else(|| "en".to_string()) - { - String::new() - } else { - format!("{language}/") - }; - expected.insert(format!("{prefix}rss.xml")); - expected.insert(format!("{prefix}atom.xml")); - } - expected.insert("sitemap.xml".to_string()); let mut actual = HashSet::new(); if output_dir.exists() { for entry in WalkDir::new(&output_dir).into_iter().filter_map(Result::ok) { - if !entry.file_type().is_file() { + if !entry.file_type().is_file() || entry.file_name() != "index.html" { continue; } - let rel = entry - .path() - .strip_prefix(&output_dir) - .unwrap_or(entry.path()) - .to_string_lossy() - .replace('\\', "/"); - if rel.starts_with("meta/") - || rel.starts_with("posts/") - || rel.starts_with("media/") - || rel.starts_with("assets/") - { - continue; - } - if rel.starts_with("pagefind") || rel.contains("/pagefind/") { - continue; - } - if rel.ends_with(".html") || rel.ends_with(".xml") || rel.ends_with(".json") { - actual.insert(rel); + if entry.metadata().is_ok_and(|metadata| metadata.len() > 0) { + actual.insert(relative_path(&output_dir, entry.path())); } } } let mut missing_pages = expected.difference(&actual).cloned().collect::>(); let mut extra_pages = actual.difference(&expected).cloned().collect::>(); - - let mut stale_pages = Vec::new(); - for rel in expected.intersection(&actual) { - if let Ok(stored) = - queries::generated_file_hash::get_generated_file_hash(conn, project_id, rel) - { - let actual_hash = file_hash(&output_dir.join(rel))?; - if actual_hash != stored.content_hash { - stale_pages.push(rel.clone()); - } - } - } + let generated_at = queries::generated_file_hash::list_generated_file_hashes(conn, project_id)? + .into_iter() + .map(|file| (file.relative_path, file.updated_at)) + .collect::>(); + let mut stale_pages = stale_post_paths( + data_dir, + &output_dir, + &metadata, + &published_posts, + &expected, + &actual, + &generated_at, + ); missing_pages.sort(); extra_pages.sort(); stale_pages.sort(); + stale_pages.dedup(); Ok(SiteValidationReport { missing_pages, @@ -105,6 +84,77 @@ pub fn validate_site( }) } +fn stale_post_paths( + data_dir: &Path, + output_dir: &Path, + metadata: &crate::model::ProjectMetadata, + published_posts: &[Post], + expected: &HashSet, + actual: &HashSet, + generated_at: &HashMap, +) -> Vec { + let main_language = metadata.main_language.as_deref().unwrap_or("en"); + let mut languages = vec![main_language.to_string()]; + for language in &metadata.blog_languages { + if !languages + .iter() + .any(|known| known.eq_ignore_ascii_case(language)) + { + languages.push(language.clone()); + } + } + let mut stale = Vec::new(); + + for post in published_posts { + let Some(source_modified) = modified_ms(&data_dir.join(&post.file_path)) else { + continue; + }; + for language in &languages { + if language != main_language && post.do_not_translate { + continue; + } + let relative_path = format!( + "{}/index.html", + build_canonical_post_path(post, language, main_language).trim_start_matches('/') + ); + if !expected.contains(&relative_path) || !actual.contains(&relative_path) { + continue; + } + let Some(output_modified) = modified_ms(&output_dir.join(&relative_path)) else { + continue; + }; + let effective_generated = output_modified.max( + generated_at + .get(&relative_path) + .copied() + .unwrap_or_default(), + ); + if source_modified > effective_generated + MTIME_GRANULARITY_TOLERANCE_MS { + stale.push(relative_path); + } + } + } + stale +} + +fn modified_ms(path: &Path) -> Option { + let modified = path.metadata().ok()?.modified().ok()?; + Some(system_time_ms(modified)) +} + +fn system_time_ms(time: SystemTime) -> i64 { + time.duration_since(UNIX_EPOCH) + .map(|duration| duration.as_millis() as i64) + .unwrap_or_default() +} + +fn relative_path(root: &Path, path: &Path) -> String { + path.strip_prefix(root) + .unwrap_or(path) + .to_string_lossy() + .replace('\\', "/") +} + fn generated_output_dir(data_dir: &Path) -> std::path::PathBuf { let html_dir = data_dir.join("html"); if html_dir.exists() { @@ -114,38 +164,9 @@ fn generated_output_dir(data_dir: &Path) -> std::path::PathBuf { } } -fn load_published_posts( - data_dir: &Path, - conn: &Connection, - project_id: &str, -) -> EngineResult> { - let posts = queries::post::list_posts_by_project(conn, project_id)?; - let mut published = Vec::new(); - for post in posts +fn load_published_posts(conn: &Connection, project_id: &str) -> EngineResult> { + Ok(queries::post::list_posts_by_project(conn, project_id)? .into_iter() - .filter(crate::engine::generation::has_published_snapshot) - { - if let Some(source) = crate::engine::generation::load_published_post_source(data_dir, post)? - { - published.push((source.post, source.body_markdown)); - } - } - Ok(published) -} - -fn render_languages(metadata: &crate::model::ProjectMetadata) -> Vec { - let main = metadata - .main_language - .clone() - .unwrap_or_else(|| "en".to_string()); - let mut languages = vec![main.clone()]; - for language in &metadata.blog_languages { - if !languages - .iter() - .any(|existing| existing.eq_ignore_ascii_case(language)) - { - languages.push(language.clone()); - } - } - languages + .filter(has_published_snapshot) + .collect()) } diff --git a/crates/bds-core/src/render/mod.rs b/crates/bds-core/src/render/mod.rs index d64f8ff..f7b5df1 100644 --- a/crates/bds-core/src/render/mod.rs +++ b/crates/bds-core/src/render/mod.rs @@ -21,7 +21,7 @@ pub use routes::{ }; pub use site::{ PagefindDocument, PreviewRenderResult, SitePage, SiteRenderArtifacts, build_preview_response, - build_site_render_artifacts, build_site_section_render_artifacts, + build_site_render_artifacts, build_site_route_manifest, build_site_section_render_artifacts, build_targeted_site_section_render_artifacts, }; pub use template_lookup::{ diff --git a/crates/bds-core/src/render/site.rs b/crates/bds-core/src/render/site.rs index d7efb91..ce4d929 100644 --- a/crates/bds-core/src/render/site.rs +++ b/crates/bds-core/src/render/site.rs @@ -122,6 +122,65 @@ pub fn build_site_render_artifacts( ) } +/// Build the sitemap route set without loading templates, media, or rendering HTML. +pub fn build_site_route_manifest( + data_dir: &Path, + metadata: &ProjectMetadata, + published_posts: &[Post], +) -> Result, Box> { + let main_language = main_language(metadata).to_string(); + let category_settings = queries_category_settings(data_dir)?; + let mut manifest = Vec::new(); + + for language in render_languages(metadata) { + let posts = published_posts + .iter() + .filter(|post| language == main_language || !post.do_not_translate) + .map(|post| RenderPostRecord { + post: post.clone(), + source_post_id: post.id.clone(), + body_markdown: String::new(), + }) + .collect::>(); + let list_posts = filter_posts_for_lists(&posts, &category_settings); + manifest.extend( + build_language_routes(&list_posts, metadata, &language, &[], &category_settings) + .into_iter() + .map(|route| SitePage { + language: language.clone(), + relative_path: route.relative_path, + url_path: route.url_path, + html: String::new(), + }), + ); + + for record in posts { + let canonical_path = build_canonical_post_path(&record.post, &language, &main_language); + let mut paths = vec![canonical_path]; + if record + .post + .categories + .iter() + .any(|category| category == "page") + { + paths.push(if language == main_language { + format!("/{}", record.post.slug) + } else { + format!("/{language}/{}", record.post.slug) + }); + } + manifest.extend(paths.into_iter().map(|url_path| SitePage { + language: language.clone(), + relative_path: format!("{}/index.html", url_path.trim_start_matches('/')), + url_path, + html: String::new(), + })); + } + } + + Ok(manifest) +} + pub fn build_site_section_render_artifacts( conn: &Connection, data_dir: &Path, diff --git a/crates/bds-core/tests/m4_generation_engine.rs b/crates/bds-core/tests/m4_generation_engine.rs index 805a876..546a60b 100644 --- a/crates/bds-core/tests/m4_generation_engine.rs +++ b/crates/bds-core/tests/m4_generation_engine.rs @@ -743,7 +743,7 @@ fn generation_engine_skips_unchanged_outputs_on_second_run() { } #[test] -fn site_validation_detects_stale_and_missing_outputs() { +fn site_validation_matches_bds2_index_route_comparison() { let (db, dir) = setup(); let metadata = make_metadata(); let mut post = make_post("hello", 1_710_000_000_000); @@ -755,16 +755,93 @@ fn site_validation_detects_stale_and_missing_outputs() { }]; generate_starter_site(db.conn(), dir.path(), "p1", &metadata, &posts, "en").unwrap(); - std::fs::write(dir.path().join("index.html"), "tampered").unwrap(); + std::fs::write(dir.path().join("index.html"), "tampered but non-empty").unwrap(); std::fs::remove_file(dir.path().join("atom.xml")).unwrap(); + std::fs::write(dir.path().join("ghost.xml"), "not a route").unwrap(); let report = validate_site(db.conn(), dir.path(), "p1").unwrap(); - assert!(report.stale_pages.contains(&"index.html".to_string())); - assert!(report.missing_pages.contains(&"atom.xml".to_string())); + assert!(report.stale_pages.is_empty()); + assert!(report.missing_pages.is_empty()); assert!(report.extra_pages.is_empty()); } +#[test] +fn site_validation_detects_missing_zero_byte_and_extra_index_routes() { + let (db, dir) = setup(); + let metadata = make_metadata(); + let mut post = make_post("hello", 1_710_000_000_000); + write_published_snapshot(&dir, &mut post, "Hello **world**"); + bds_core::db::queries::post::insert_post(db.conn(), &post).unwrap(); + let posts = vec![PublishedPostSource { + post, + body_markdown: "Hello **world**".into(), + }]; + + generate_starter_site(db.conn(), dir.path(), "p1", &metadata, &posts, "en").unwrap(); + std::fs::write(dir.path().join("index.html"), "").unwrap(); + let extra = dir.path().join("ghost/index.html"); + std::fs::create_dir_all(extra.parent().unwrap()).unwrap(); + std::fs::write(extra, "ghost").unwrap(); + + let report = validate_site(db.conn(), dir.path(), "p1").unwrap(); + + assert!(report.missing_pages.contains(&"index.html".to_string())); + assert!(report.extra_pages.contains(&"ghost/index.html".to_string())); +} + +#[test] +fn site_validation_detects_post_sources_newer_than_generated_routes() { + use std::fs::{File, FileTimes}; + use std::time::{Duration, SystemTime}; + + let (db, dir) = setup(); + let metadata = make_metadata(); + let mut post = make_post("hello", 1_710_000_000_000); + write_published_snapshot(&dir, &mut post, "Hello **world**"); + let source_path = dir.path().join(&post.file_path); + bds_core::db::queries::post::insert_post(db.conn(), &post).unwrap(); + let posts = vec![PublishedPostSource { + post, + body_markdown: "Hello **world**".into(), + }]; + + generate_starter_site(db.conn(), dir.path(), "p1", &metadata, &posts, "en").unwrap(); + File::options() + .write(true) + .open(source_path) + .unwrap() + .set_times(FileTimes::new().set_modified(SystemTime::now() + Duration::from_secs(5))) + .unwrap(); + + let report = validate_site(db.conn(), dir.path(), "p1").unwrap(); + + assert!( + report + .stale_pages + .contains(&"2024/03/09/hello/index.html".to_string()) + ); +} + +#[test] +fn site_validation_refreshes_sitemap_without_rendering_pages() { + let (db, dir) = setup(); + let mut post = make_post("hello", 1_710_000_000_000); + write_published_snapshot(&dir, &mut post, "Hello **world**"); + bds_core::db::queries::post::insert_post(db.conn(), &post).unwrap(); + + let report = validate_site(db.conn(), dir.path(), "p1").unwrap(); + let sitemap = std::fs::read_to_string(dir.path().join("sitemap.xml")).unwrap(); + + assert!( + report + .missing_pages + .contains(&"2024/03/09/hello/index.html".to_string()) + ); + assert!(sitemap.contains("https://example.com/2024/03/09/hello/")); + assert!(!dir.path().join("2024/03/09/hello/index.html").exists()); +} + #[test] fn apply_validation_repairs_core_section_outputs() { let (db, dir) = setup(); @@ -778,14 +855,20 @@ fn apply_validation_repairs_core_section_outputs() { }]; generate_starter_site(db.conn(), dir.path(), "p1", &metadata, &posts, "en").unwrap(); - std::fs::write(dir.path().join("index.html"), "tampered").unwrap(); - std::fs::remove_file(dir.path().join("atom.xml")).unwrap(); + std::fs::remove_file(dir.path().join("index.html")).unwrap(); let report = validate_site(db.conn(), dir.path(), "p1").unwrap(); let sections = sections_from_validation_report(&report, &metadata); - let apply_report = - apply_validation_sections(db.conn(), dir.path(), "p1", &metadata, &posts, §ions) - .unwrap(); + let apply_report = apply_validation_sections( + db.conn(), + dir.path(), + "p1", + &metadata, + &posts, + &report, + §ions, + ) + .unwrap(); let repaired = validate_site(db.conn(), dir.path(), "p1").unwrap(); assert!(!apply_report.written_paths.is_empty() || !apply_report.skipped_paths.is_empty()); @@ -794,6 +877,66 @@ fn apply_validation_repairs_core_section_outputs() { assert!(repaired.stale_pages.is_empty()); } +#[test] +fn validation_apply_clears_unchanged_touched_post_routes() { + use std::fs::{File, FileTimes}; + use std::time::{Duration, UNIX_EPOCH}; + + let (db, dir) = setup(); + let metadata = make_metadata(); + let mut post = make_post("hello", 1_710_000_000_000); + write_published_snapshot(&dir, &mut post, "Hello **world**"); + let source_path = dir.path().join(&post.file_path); + bds_core::db::queries::post::insert_post(db.conn(), &post).unwrap(); + let posts = vec![PublishedPostSource { + post, + body_markdown: "Hello **world**".into(), + }]; + + generate_starter_site(db.conn(), dir.path(), "p1", &metadata, &posts, "en").unwrap(); + let relative_path = "2024/03/09/hello/index.html"; + let output_path = dir.path().join(relative_path); + File::options() + .write(true) + .open(&output_path) + .unwrap() + .set_times(FileTimes::new().set_modified(UNIX_EPOCH + Duration::from_secs(10))) + .unwrap(); + File::options() + .write(true) + .open(source_path) + .unwrap() + .set_times(FileTimes::new().set_modified(UNIX_EPOCH + Duration::from_secs(20))) + .unwrap(); + let mut hash = bds_core::db::queries::generated_file_hash::get_generated_file_hash( + db.conn(), + "p1", + relative_path, + ) + .unwrap(); + hash.updated_at = 10_000; + bds_core::db::queries::generated_file_hash::upsert_generated_file_hash(db.conn(), &hash) + .unwrap(); + + let validation = validate_site(db.conn(), dir.path(), "p1").unwrap(); + assert_eq!(validation.stale_pages, vec![relative_path.to_string()]); + let sections = sections_from_validation_report(&validation, &metadata); + let applied = apply_validation_sections( + db.conn(), + dir.path(), + "p1", + &metadata, + &posts, + &validation, + §ions, + ) + .unwrap(); + let repaired = validate_site(db.conn(), dir.path(), "p1").unwrap(); + + assert!(applied.skipped_paths.contains(&relative_path.to_string())); + assert!(repaired.stale_pages.is_empty()); +} + #[test] fn apply_validation_removes_extra_section_outputs() { let (db, dir) = setup(); @@ -818,9 +961,16 @@ fn apply_validation_removes_extra_section_outputs() { .contains(&"tag/ghost/index.html".to_string()) ); let sections = sections_from_validation_report(&report, &metadata); - let apply_report = - apply_validation_sections(db.conn(), dir.path(), "p1", &metadata, &posts, §ions) - .unwrap(); + let apply_report = apply_validation_sections( + db.conn(), + dir.path(), + "p1", + &metadata, + &posts, + &report, + §ions, + ) + .unwrap(); let repaired = validate_site(db.conn(), dir.path(), "p1").unwrap(); assert!( @@ -884,15 +1034,22 @@ fn validation_apply_repairs_and_cleans_localized_page_outputs() { generate_starter_site(db.conn(), dir.path(), "p1", &metadata, &posts, "en").unwrap(); let localized_page = dir.path().join("de/about/index.html"); let localized_extra = dir.path().join("de/ghost/index.html"); - std::fs::write(&localized_page, "tampered").unwrap(); + std::fs::write(&localized_page, "").unwrap(); std::fs::create_dir_all(localized_extra.parent().unwrap()).unwrap(); std::fs::write(&localized_extra, "ghost").unwrap(); let validation = validate_site(db.conn(), dir.path(), "p1").unwrap(); let sections = sections_from_validation_report(&validation, &metadata); - let applied = - apply_validation_sections(db.conn(), dir.path(), "p1", &metadata, &posts, §ions) - .unwrap(); + let applied = apply_validation_sections( + db.conn(), + dir.path(), + "p1", + &metadata, + &posts, + &validation, + §ions, + ) + .unwrap(); let repaired = validate_site(db.conn(), dir.path(), "p1").unwrap(); assert_eq!(sections, vec![GenerationSection::Core]); @@ -933,9 +1090,16 @@ fn validation_apply_removes_extra_output_under_configured_language() { let validation = validate_site(db.conn(), dir.path(), "p1").unwrap(); let sections = sections_from_validation_report(&validation, &metadata); - let applied = - apply_validation_sections(db.conn(), dir.path(), "p1", &metadata, &posts, §ions) - .unwrap(); + let applied = apply_validation_sections( + db.conn(), + dir.path(), + "p1", + &metadata, + &posts, + &validation, + §ions, + ) + .unwrap(); assert_eq!(sections, vec![GenerationSection::Category]); assert!( diff --git a/crates/bds-server/src/tui.rs b/crates/bds-server/src/tui.rs index 57cd215..3321216 100644 --- a/crates/bds-server/src/tui.rs +++ b/crates/bds-server/src/tui.rs @@ -2973,6 +2973,7 @@ impl TuiApp { &project_id, &metadata, &posts, + &validation, §ions, )?; self.status = self.tr_with( diff --git a/specs/generation.allium b/specs/generation.allium index 3422842..9a978f3 100644 --- a/specs/generation.allium +++ b/specs/generation.allium @@ -262,14 +262,24 @@ rule RenderPage { rule ValidateSite { when: ValidateSiteRequested(project) - -- Compares sitemap URLs to HTML files on disk - -- Detects: missing pages, extra (stale) pages, sitemap/file mismatches + -- Matches bDS2's cheap route comparison without rendering page content: + -- sitemap.xml is refreshed from the current route set before comparison + -- expected: sitemap-eligible routes derived from published snapshots + -- actual: non-empty **/index.html files on disk + -- missing: expected routes with no non-empty index.html + -- extra: non-empty index.html routes absent from the expected set + -- stale: existing post routes whose source mtime is more than one second + -- newer than both the output mtime and tracked generation time + -- Standalone HTML, XML, JSON, assets, Pagefind files, and content hashes are + -- not validation comparison inputs. ensures: ValidationReport(missing_pages, extra_pages, stale_pages) } rule ApplyValidation { when: ApplyValidationRequested(project_id, sections) - -- Targeted re-rendering for affected sections only + -- Targeted re-rendering for affected report paths only. Applying an + -- unchanged post route refreshes its tracked generation time so the + -- automatic follow-up validation does not report it stale again. ensures: GenerateSiteRequested(plan_generation(project_id, sections)) }