feat: even more M4 closing

crates/bds-core/src/engine/generation.rs

@@ -1,13 +1,15 @@
+use std::collections::{HashMap, HashSet};
 use std::path::Path;
-use std::collections::HashMap;
 
 use chrono::{DateTime, TimeZone, Utc};
 use pagefind::api::PagefindIndex;
 use pagefind::options::PagefindServiceConfig;
 use rusqlite::Connection;
+use walkdir::WalkDir;
 
 use crate::db::queries;
 use crate::engine::site_assets::write_bundled_site_assets;
+use crate::engine::validate_site::SiteValidationReport;
 use crate::engine::{EngineError, EngineResult};
 use crate::model::{Post, ProjectMetadata};
 use crate::render::{
@@ -26,6 +28,16 @@ pub struct PublishedPostSource {
 pub struct GenerationReport {
     pub written_paths: Vec<String>,
     pub skipped_paths: Vec<String>,
+    pub deleted_paths: Vec<String>,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub enum GenerationSection {
+    Core,
+    Single,
+    Category,
+    Tag,
+    Date,
 }
 
 pub fn generate_starter_site(
@@ -87,6 +99,124 @@ pub fn generate_starter_site(
     Ok(report)
 }
 
+pub fn sections_from_validation_report(report: &SiteValidationReport) -> Vec<GenerationSection> {
+    let mut sections = HashSet::new();
+    let mut saw_unknown = false;
+
+    for path in report
+        .missing_pages
+        .iter()
+        .chain(report.extra_pages.iter())
+        .chain(report.stale_pages.iter())
+    {
+        match classify_generated_path(path) {
+            Some(section) => {
+                sections.insert(section);
+            }
+            None => {
+                saw_unknown = true;
+            }
+        }
+    }
+
+    if saw_unknown && !report_is_empty(report) {
+        return all_sections();
+    }
+
+    let mut ordered = sections.into_iter().collect::<Vec<_>>();
+    ordered.sort_by_key(section_sort_key);
+    ordered
+}
+
+pub fn apply_validation_sections(
+    conn: &Connection,
+    output_dir: &Path,
+    project_id: &str,
+    metadata: &ProjectMetadata,
+    posts: &[PublishedPostSource],
+    sections: &[GenerationSection],
+) -> EngineResult<GenerationReport> {
+    if sections.is_empty() {
+        return Ok(GenerationReport::default());
+    }
+
+    let section_set = sections.iter().copied().collect::<HashSet<_>>();
+    let data_dir = project_data_dir(output_dir);
+    let input_posts = posts
+        .iter()
+        .map(|source| (source.post.clone(), source.body_markdown.clone()))
+        .collect::<Vec<_>>();
+    let artifacts = build_site_render_artifacts(
+        conn,
+        &data_dir,
+        project_id,
+        metadata,
+        &input_posts,
+    )
+    .map_err(|error| EngineError::Parse(error.to_string()))?;
+    let mut report = GenerationReport::default();
+    let expected_paths = expected_paths_for_sections(metadata, &artifacts.pages, &section_set);
+
+    for page in &artifacts.pages {
+        if path_matches_sections(&page.relative_path, &section_set) {
+            write_out(conn, output_dir, project_id, &page.relative_path, &page.html, &mut report)?;
+        }
+    }
+
+    if section_set.contains(&GenerationSection::Core) {
+        write_bundled_site_assets(conn, output_dir, project_id, &mut report)?;
+        write_out(
+            conn,
+            output_dir,
+            project_id,
+            "calendar.json",
+            &build_calendar_json(&posts.iter().map(|source| source.post.clone()).collect::<Vec<_>>())?,
+            &mut report,
+        )?;
+
+        for render_language in render_languages(metadata) {
+            let localized_posts = localized_sources(
+                conn,
+                &data_dir,
+                posts,
+                &render_language,
+                metadata,
+            )?;
+            let prefix = if render_language == metadata.main_language.clone().unwrap_or_else(|| "en".to_string()) {
+                String::new()
+            } else {
+                format!("{}/", render_language)
+            };
+            let rss = build_rss_xml(metadata, &localized_posts, &render_language);
+            if prefix.is_empty() {
+                write_out(conn, output_dir, project_id, "rss.xml", &rss, &mut report)?;
+            }
+            write_out(conn, output_dir, project_id, &format!("{prefix}feed.xml"), &rss, &mut report)?;
+            write_out(
+                conn,
+                output_dir,
+                project_id,
+                &format!("{prefix}atom.xml"),
+                &build_atom_xml(metadata, &localized_posts, &render_language),
+                &mut report,
+            )?;
+            write_out(
+                conn,
+                output_dir,
+                project_id,
+                &format!("{prefix}sitemap.xml"),
+                &build_sitemap_xml(metadata, &artifacts.pages, &localized_posts, &render_language),
+                &mut report,
+            )?;
+        }
+    }
+
+    remove_extra_section_paths(output_dir, &expected_paths, &section_set, &mut report)?;
+    write_pagefind_indexes(conn, output_dir, project_id, &artifacts.pagefind_documents, &mut report)?;
+
+    Ok(report)
+}
+
 fn build_media_rewrite_map(
     conn: &Connection,
     project_id: &str,
@@ -174,6 +304,168 @@ fn write_pagefind_indexes(
     Ok(())
 }
 
+fn project_data_dir(output_dir: &Path) -> std::path::PathBuf {
+    if output_dir.join("meta").exists() {
+        output_dir.to_path_buf()
+    } else {
+        output_dir.parent().unwrap_or(output_dir).to_path_buf()
+    }
+}
+
+fn expected_paths_for_sections(
+    metadata: &ProjectMetadata,
+    pages: &[crate::render::SitePage],
+    sections: &HashSet<GenerationSection>,
+) -> HashSet<String> {
+    let mut expected = pages
+        .iter()
+        .filter(|page| path_matches_sections(&page.relative_path, sections))
+        .map(|page| page.relative_path.clone())
+        .collect::<HashSet<_>>();
+
+    if sections.contains(&GenerationSection::Core) {
+        expected.insert("calendar.json".to_string());
+        expected.insert("rss.xml".to_string());
+        for language in render_languages(metadata) {
+            let prefix = if language == metadata.main_language.clone().unwrap_or_else(|| "en".to_string()) {
+                String::new()
+            } else {
+                format!("{language}/")
+            };
+            expected.insert(format!("{prefix}feed.xml"));
+            expected.insert(format!("{prefix}atom.xml"));
+            expected.insert(format!("{prefix}sitemap.xml"));
+        }
+    }
+
+    expected
+}
+
+fn remove_extra_section_paths(
+    output_dir: &Path,
+    expected: &HashSet<String>,
+    sections: &HashSet<GenerationSection>,
+    report: &mut GenerationReport,
+) -> EngineResult<()> {
+    if !output_dir.exists() {
+        return Ok(());
+    }
+
+    let mut deleted = Vec::new();
+    for entry in WalkDir::new(output_dir).into_iter().filter_map(Result::ok) {
+        if !entry.file_type().is_file() {
+            continue;
+        }
+        let rel = entry
+            .path()
+            .strip_prefix(output_dir)
+            .unwrap_or(entry.path())
+            .to_string_lossy()
+            .replace('\\', "/");
+        if rel.starts_with("meta/")
+            || rel.starts_with("posts/")
+            || rel.starts_with("media/")
+            || rel.starts_with("assets/")
+            || rel.starts_with("pagefind")
+            || rel.contains("/pagefind/")
+        {
+            continue;
+        }
+        if !matches_generated_extension(&rel) || !path_matches_sections(&rel, sections) || expected.contains(&rel) {
+            continue;
+        }
+        std::fs::remove_file(entry.path()).map_err(EngineError::Io)?;
+        deleted.push(rel);
+    }
+
+    deleted.sort();
+    report.deleted_paths.extend(deleted);
+    Ok(())
+}
+
+fn path_matches_sections(path: &str, sections: &HashSet<GenerationSection>) -> bool {
+    classify_generated_path(path)
+        .map(|section| sections.contains(&section))
+        .unwrap_or(false)
+}
+
+fn classify_generated_path(path: &str) -> Option<GenerationSection> {
+    if path.ends_with(".xml") || path.ends_with(".json") {
+        return Some(GenerationSection::Core);
+    }
+
+    let mut parts = path.split('/').collect::<Vec<_>>();
+    if parts.is_empty() {
+        return None;
+    }
+    if has_language_prefix(&parts) {
+        parts.remove(0);
+    }
+
+    match parts.as_slice() {
+        ["index.html"] => Some(GenerationSection::Core),
+        ["category", ..] => Some(GenerationSection::Category),
+        ["tag", ..] => Some(GenerationSection::Tag),
+        [year, "index.html"] if is_year_segment(year) => Some(GenerationSection::Date),
+        [year, month, "index.html"] if is_year_segment(year) && is_month_segment(month) => Some(GenerationSection::Date),
+        [year, month, day, _slug, "index.html"]
+            if is_year_segment(year) && is_month_segment(month) && is_day_segment(day) => Some(GenerationSection::Single),
+        _ => None,
+    }
+}
+
+fn has_language_prefix(parts: &[&str]) -> bool {
+    match parts {
+        [first, second, ..] => {
+            !is_year_segment(first)
+                && *first != "category"
+                && *first != "tag"
+                && (*second == "index.html" || is_year_segment(second) || *second == "category" || *second == "tag")
+        }
+        _ => false,
+    }
+}
+
+fn is_year_segment(value: &str) -> bool {
+    value.len() == 4 && value.chars().all(|ch| ch.is_ascii_digit())
+}
+
+fn is_month_segment(value: &str) -> bool {
+    value.len() == 2 && value.chars().all(|ch| ch.is_ascii_digit())
+}
+
+fn is_day_segment(value: &str) -> bool {
+    is_month_segment(value)
+}
+
+fn matches_generated_extension(path: &str) -> bool {
+    path.ends_with(".html") || path.ends_with(".xml") || path.ends_with(".json")
+}
+
+fn all_sections() -> Vec<GenerationSection> {
+    vec![
+        GenerationSection::Core,
+        GenerationSection::Single,
+        GenerationSection::Category,
+        GenerationSection::Tag,
+        GenerationSection::Date,
+    ]
+}
+
+fn section_sort_key(section: &GenerationSection) -> u8 {
+    match section {
+        GenerationSection::Core => 0,
+        GenerationSection::Single => 1,
+        GenerationSection::Category => 2,
+        GenerationSection::Tag => 3,
+        GenerationSection::Date => 4,
+    }
+}
+
+fn report_is_empty(report: &SiteValidationReport) -> bool {
+    report.missing_pages.is_empty() && report.extra_pages.is_empty() && report.stale_pages.is_empty()
+}
+
 fn render_languages(metadata: &ProjectMetadata) -> Vec<String> {
     let main = metadata.main_language.clone().unwrap_or_else(|| "en".to_string());
     let mut languages = vec![main.clone()];

crates/bds-core/src/engine/validate_site.rs

@@ -23,7 +23,7 @@ pub fn validate_site(
     project_id: &str,
 ) -> EngineResult<SiteValidationReport> {
     let metadata = crate::engine::meta::read_project_json(data_dir)?;
-    let output_dir = data_dir.to_path_buf();
+    let output_dir = generated_output_dir(data_dir);
     let published_posts = load_published_posts(data_dir, conn, project_id)?;
     let artifacts = build_site_render_artifacts(conn, data_dir, project_id, &metadata, &published_posts)
         .map_err(|error| EngineError::Parse(error.to_string()))?;
@@ -98,6 +98,15 @@ pub fn validate_site(
     })
 }
 
+fn generated_output_dir(data_dir: &Path) -> std::path::PathBuf {
+    let html_dir = data_dir.join("html");
+    if html_dir.exists() {
+        html_dir
+    } else {
+        data_dir.to_path_buf()
+    }
+}
+
 fn load_published_posts(
     data_dir: &Path,
     conn: &Connection,

crates/bds-core/src/render/generation.rs

@@ -8,7 +8,7 @@ use serde::Serialize;
 
 use crate::db::queries::generated_file_hash as qhash;
 use crate::model::{GeneratedFileHash, Post};
-use crate::util::{atomic_write_str, content_hash, now_unix_ms};
+use crate::util::{atomic_write_str, content_hash, file_hash, now_unix_ms};
 
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub enum GeneratedWriteOutcome {
@@ -31,13 +31,16 @@ pub fn write_generated_file(
     content: &str,
 ) -> Result<GeneratedWriteOutcome, Box<dyn std::error::Error + Send + Sync>> {
     let hash = content_hash(content.as_bytes());
+    let target_path = output_dir.join(relative_path);
     if let Ok(existing) = qhash::get_generated_file_hash(conn, project_id, relative_path) {
-        if existing.content_hash == hash {
+        if existing.content_hash == hash
+            && target_path.exists()
+            && file_hash(&target_path)? == hash
+        {
             return Ok(GeneratedWriteOutcome::SkippedUnchanged);
         }
     }
 
-    let target_path = output_dir.join(relative_path);
     if let Some(parent) = target_path.parent() {
         fs::create_dir_all(parent)?;
     }
@@ -64,13 +67,16 @@ pub fn write_generated_bytes(
     content: &[u8],
 ) -> Result<GeneratedWriteOutcome, Box<dyn std::error::Error + Send + Sync>> {
     let hash = content_hash(content);
+    let target_path = output_dir.join(relative_path);
     if let Ok(existing) = qhash::get_generated_file_hash(conn, project_id, relative_path) {
-        if existing.content_hash == hash {
+        if existing.content_hash == hash
+            && target_path.exists()
+            && file_hash(&target_path)? == hash
+        {
             return Ok(GeneratedWriteOutcome::SkippedUnchanged);
         }
     }
 
-    let target_path = output_dir.join(relative_path);
     if let Some(parent) = target_path.parent() {
         fs::create_dir_all(parent)?;
     }

crates/bds-core/tests/m4_generation_engine.rs

@@ -1,6 +1,9 @@
 use bds_core::db::queries::project::insert_project;
 use bds_core::db::Database;
-use bds_core::engine::generation::{PublishedPostSource, generate_starter_site};
+use bds_core::engine::generation::{
+    PublishedPostSource, apply_validation_sections, generate_starter_site,
+    sections_from_validation_report,
+};
 use bds_core::engine::validate_site::validate_site;
 use bds_core::model::{Post, PostStatus, Project, ProjectMetadata};
 use tempfile::TempDir;
@@ -184,4 +187,78 @@ fn site_validation_detects_stale_and_missing_outputs() {
     assert!(report.stale_pages.contains(&"index.html".to_string()));
     assert!(report.missing_pages.contains(&"feed.xml".to_string()));
     assert!(report.extra_pages.is_empty());
+}
+
+#[test]
+fn apply_validation_repairs_core_section_outputs() {
+    let (db, dir) = setup();
+    let metadata = make_metadata();
+    let post = make_post("hello", 1_710_000_000_000);
+    bds_core::db::queries::post::insert_post(db.conn(), &post).unwrap();
+    let posts = vec![PublishedPostSource {
+        post,
+        body_markdown: "Hello **world**".into(),
+    }];
+
+    generate_starter_site(db.conn(), dir.path(), "p1", &metadata, &posts, "en").unwrap();
+    std::fs::write(dir.path().join("index.html"), "tampered").unwrap();
+    std::fs::remove_file(dir.path().join("feed.xml")).unwrap();
+
+    let report = validate_site(db.conn(), dir.path(), "p1").unwrap();
+    let sections = sections_from_validation_report(&report);
+    let apply_report = apply_validation_sections(db.conn(), dir.path(), "p1", &metadata, &posts, &sections).unwrap();
+    let repaired = validate_site(db.conn(), dir.path(), "p1").unwrap();
+
+    assert!(!apply_report.written_paths.is_empty() || !apply_report.skipped_paths.is_empty());
+    assert!(repaired.missing_pages.is_empty());
+    assert!(repaired.extra_pages.is_empty());
+    assert!(repaired.stale_pages.is_empty());
+}
+
+#[test]
+fn apply_validation_removes_extra_section_outputs() {
+    let (db, dir) = setup();
+    let metadata = make_metadata();
+    let post = make_post("hello", 1_710_000_000_000);
+    bds_core::db::queries::post::insert_post(db.conn(), &post).unwrap();
+    let posts = vec![PublishedPostSource {
+        post,
+        body_markdown: "Hello **world**".into(),
+    }];
+
+    generate_starter_site(db.conn(), dir.path(), "p1", &metadata, &posts, "en").unwrap();
+    let extra_dir = dir.path().join("tag/ghost");
+    std::fs::create_dir_all(&extra_dir).unwrap();
+    std::fs::write(extra_dir.join("index.html"), "ghost").unwrap();
+
+    let report = validate_site(db.conn(), dir.path(), "p1").unwrap();
+    assert!(report.extra_pages.contains(&"tag/ghost/index.html".to_string()));
+    let sections = sections_from_validation_report(&report);
+    let apply_report = apply_validation_sections(db.conn(), dir.path(), "p1", &metadata, &posts, &sections).unwrap();
+    let repaired = validate_site(db.conn(), dir.path(), "p1").unwrap();
+
+    assert!(apply_report.deleted_paths.contains(&"tag/ghost/index.html".to_string()));
+    assert!(repaired.extra_pages.is_empty());
+}
+
+#[test]
+fn site_validation_uses_html_output_directory_when_present() {
+    let (db, dir) = setup();
+    let metadata = make_metadata();
+    let post = make_post("hello", 1_710_000_000_000);
+    bds_core::db::queries::post::insert_post(db.conn(), &post).unwrap();
+    let posts = vec![PublishedPostSource {
+        post,
+        body_markdown: "Hello **world**".into(),
+    }];
+
+    let output_dir = dir.path().join("html");
+    std::fs::create_dir_all(&output_dir).unwrap();
+    generate_starter_site(db.conn(), &output_dir, "p1", &metadata, &posts, "en").unwrap();
+
+    let report = validate_site(db.conn(), dir.path(), "p1").unwrap();
+
+    assert!(report.missing_pages.is_empty());
+    assert!(report.extra_pages.is_empty());
+    assert!(report.stale_pages.is_empty());
 }

crates/bds-core/tests/m4_generation_primitives.rs

@@ -75,6 +75,32 @@ fn generated_write_skips_unchanged_content() {
     assert!(fs::read_to_string(dir.path().join("index.html")).unwrap().contains("changed"));
 }
 
+#[test]
+fn generated_write_rewrites_missing_file_even_when_hash_matches() {
+    let (db, dir) = setup();
+
+    let first = write_generated_file(db.conn(), dir.path(), "p1", "index.html", "hello").unwrap();
+    fs::remove_file(dir.path().join("index.html")).unwrap();
+    let second = write_generated_file(db.conn(), dir.path(), "p1", "index.html", "hello").unwrap();
+
+    assert_eq!(first, GeneratedWriteOutcome::Written);
+    assert_eq!(second, GeneratedWriteOutcome::Written);
+    assert_eq!(fs::read_to_string(dir.path().join("index.html")).unwrap(), "hello");
+}
+
+#[test]
+fn generated_write_rewrites_stale_file_even_when_db_hash_matches() {
+    let (db, dir) = setup();
+
+    let first = write_generated_file(db.conn(), dir.path(), "p1", "index.html", "hello").unwrap();
+    fs::write(dir.path().join("index.html"), "tampered").unwrap();
+    let second = write_generated_file(db.conn(), dir.path(), "p1", "index.html", "hello").unwrap();
+
+    assert_eq!(first, GeneratedWriteOutcome::Written);
+    assert_eq!(second, GeneratedWriteOutcome::Written);
+    assert_eq!(fs::read_to_string(dir.path().join("index.html")).unwrap(), "hello");
+}
+
 #[test]
 fn core_generation_paths_include_language_prefixed_variants() {
     let paths = build_core_generation_paths("en", &["en".into(), "de".into(), "fr".into()]);