hugo/RuDS
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

fix: better database rebuild

Browse Source
This commit is contained in:
Chili Palmer
2026-07-18 20:46:27 +02:00
parent 638202b96c
commit b438a99999
6 changed files with 694 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files

280
crates/bds-core/src/engine/rebuild.rs
View File

@@ -2,6 +2,7 @@ use std::path::Path;
use std::sync::Arc;
use crate::db::DbConnection as Connection;
use diesel::prelude::*;
use crate::db::fts;
use crate::engine::EngineResult;
@@ -33,8 +34,8 @@ pub type ProgressFn = Arc<dyn Fn(f32, &str) + Send + Sync>;
/// Orchestrate a full rebuild from filesystem into the database.
///
/// Ensures FTS tables exist, then rebuilds posts, media, templates, and scripts
/// from their respective filesystem directories. Returns an aggregated report.
/// Replaces the project-scoped database state with project metadata, content,
/// and relationships reconstructed from the filesystem.
pub fn rebuild_from_filesystem(
conn: &Connection,
data_dir: &Path,
@@ -49,6 +50,25 @@ pub fn rebuild_from_filesystem_with_progress(
data_dir: &Path,
project_id: &str,
on_progress: Option<ProgressFn>,
) -> EngineResult<FullRebuildReport> {
conn.begin_savepoint()?;
match rebuild_from_filesystem_inner(conn, data_dir, project_id, on_progress) {
Ok(report) => {
conn.release_savepoint()?;
Ok(report)
}
Err(error) => {
let _ = conn.rollback_savepoint();
Err(error)
}
}
}
fn rebuild_from_filesystem_inner(
conn: &Connection,
data_dir: &Path,
project_id: &str,
on_progress: Option<ProgressFn>,
) -> EngineResult<FullRebuildReport> {
let mut report = FullRebuildReport::default();
let progress = |pct: f32, msg: &str| {
@@ -59,9 +79,11 @@ pub fn rebuild_from_filesystem_with_progress(
// Phase weights: posts 0.0..0.35, media 0.35..0.70, templates 0.70..0.85, scripts 0.85..1.0
// 1. Ensure FTS tables exist
progress(0.0, "Ensuring FTS tables...");
// 1. Load portable project metadata and clear all reconstructible rows.
progress(0.0, "Loading project metadata...");
fts::ensure_fts_tables(conn)?;
crate::engine::meta::sync_project_from_file(conn, data_dir, project_id)?;
clear_project_rows(conn, project_id)?;
// 2. Rebuild posts (0.00 .. 0.35)
progress(0.01, "Scanning posts...");
@@ -135,15 +157,93 @@ pub fn rebuild_from_filesystem_with_progress(
report.scripts_updated = script_report.updated;
report.errors.extend(script_report.errors);
// 6. Import tags from tags.json and sync from posts (0.95 .. 1.0)
// 6. Restore relationships and tags (0.95 .. 1.0)
progress(0.95, "Importing tags...");
let _ = super::tag::import_tags_from_file(conn, data_dir, project_id);
let _ = super::tag::sync_tags_from_posts(conn, project_id);
super::tag::import_tags_from_file(conn, data_dir, project_id)?;
super::tag::sync_tags_from_posts(conn, project_id)?;
post::rebuild_all_links(conn, data_dir, project_id)?;
if !report.errors.is_empty() {
return Err(crate::engine::EngineError::Validation(format!(
"rebuild failed: {}",
report.errors.join("; ")
)));
}
progress(1.0, "Rebuild complete");
Ok(report)
}
fn clear_project_rows(conn: &Connection, project_id: &str) -> EngineResult<()> {
use crate::db::schema::{
dismissed_duplicate_pairs, embedding_keys, generated_file_hashes, import_definitions,
media, media_translations, post_links, post_media, post_translations, posts, scripts, tags,
templates,
};
let post_ids = crate::db::queries::post::list_posts_by_project(conn, project_id)?
.into_iter()
.map(|post| post.id)
.collect::<Vec<_>>();
let media_ids = crate::db::queries::media::list_media_by_project(conn, project_id)?
.into_iter()
.map(|media| media.id)
.collect::<Vec<_>>();
for id in &post_ids {
fts::remove_post_from_index(conn, id)?;
}
for id in &media_ids {
fts::remove_media_from_index(conn, id)?;
}
conn.with(|connection| {
diesel::delete(
post_links::table.filter(
post_links::source_post_id
.eq_any(&post_ids)
.or(post_links::target_post_id.eq_any(&post_ids)),
),
)
.execute(connection)?;
diesel::delete(post_media::table.filter(post_media::project_id.eq(project_id)))
.execute(connection)?;
diesel::delete(
post_translations::table.filter(post_translations::project_id.eq(project_id)),
)
.execute(connection)?;
diesel::delete(
media_translations::table.filter(media_translations::project_id.eq(project_id)),
)
.execute(connection)?;
diesel::delete(
generated_file_hashes::table.filter(generated_file_hashes::project_id.eq(project_id)),
)
.execute(connection)?;
diesel::delete(embedding_keys::table.filter(embedding_keys::project_id.eq(project_id)))
.execute(connection)?;
diesel::delete(
dismissed_duplicate_pairs::table
.filter(dismissed_duplicate_pairs::project_id.eq(project_id)),
)
.execute(connection)?;
diesel::delete(
import_definitions::table.filter(import_definitions::project_id.eq(project_id)),
)
.execute(connection)?;
diesel::delete(tags::table.filter(tags::project_id.eq(project_id))).execute(connection)?;
diesel::delete(scripts::table.filter(scripts::project_id.eq(project_id)))
.execute(connection)?;
diesel::delete(templates::table.filter(templates::project_id.eq(project_id)))
.execute(connection)?;
diesel::delete(media::table.filter(media::project_id.eq(project_id)))
.execute(connection)?;
diesel::delete(posts::table.filter(posts::project_id.eq(project_id)))
.execute(connection)?;
Ok(())
})?;
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
@@ -152,7 +252,12 @@ mod tests {
use crate::db::queries::post as qp;
use crate::db::queries::project::{insert_project, make_test_project};
use crate::db::queries::script as qs;
use crate::db::queries::tag as qtag;
use crate::db::queries::template as qtpl;
use crate::model::metadata::ProjectMetadata;
use crate::model::{
Script, ScriptKind, ScriptStatus, Tag, Template, TemplateKind, TemplateStatus,
};
use std::fs;
use tempfile::TempDir;
@@ -162,6 +267,23 @@ mod tests {
fts::ensure_fts_tables(db.conn()).unwrap();
insert_project(db.conn(), &make_test_project("p1", "blog")).unwrap();
let dir = TempDir::new().unwrap();
crate::engine::meta::write_project_json(
dir.path(),
&ProjectMetadata {
name: "Project p1".into(),
description: Some("A test project".into()),
public_url: None,
main_language: None,
default_author: None,
max_posts_per_page: 50,
image_import_concurrency: 4,
blogmark_category: None,
pico_theme: None,
semantic_similarity_enabled: false,
blog_languages: vec![],
},
)
.unwrap();
(db, dir)
}
@@ -253,6 +375,96 @@ tags: []
fn rebuild_all_entity_types() {
let (db, dir) = setup();
crate::engine::meta::write_project_json(
dir.path(),
&ProjectMetadata {
name: "Rebuilt Project".into(),
description: None,
public_url: None,
main_language: Some("en".into()),
default_author: None,
max_posts_per_page: 50,
image_import_concurrency: 4,
blogmark_category: None,
pico_theme: None,
semantic_similarity_enabled: false,
blog_languages: vec!["en".into()],
},
)
.unwrap();
qs::insert_script(
db.conn(),
&Script {
id: "stale-script".into(),
project_id: "p1".into(),
slug: "stale-script".into(),
title: "Stale Script".into(),
kind: ScriptKind::Utility,
entrypoint: "main".into(),
enabled: true,
version: 1,
file_path: "scripts/stale.lua".into(),
status: ScriptStatus::Published,
content: None,
created_at: 1,
updated_at: 1,
},
)
.unwrap();
qtpl::insert_template(
db.conn(),
&Template {
id: "stale-template".into(),
project_id: "p1".into(),
slug: "stale-template".into(),
title: "Stale Template".into(),
kind: TemplateKind::Post,
enabled: true,
version: 1,
file_path: "templates/stale.liquid".into(),
status: TemplateStatus::Published,
content: None,
created_at: 1,
updated_at: 1,
},
)
.unwrap();
qp::insert_post(
db.conn(),
&qp::make_test_post("stale-post", "p1", "stale-post"),
)
.unwrap();
qm::insert_media(db.conn(), &qm::make_test_media("stale-media", "p1")).unwrap();
qtag::insert_tag(
db.conn(),
&Tag {
id: "stale-tag".into(),
project_id: "p1".into(),
name: "stale-tag".into(),
color: None,
post_template_slug: None,
created_at: 1,
updated_at: 1,
},
)
.unwrap();
db.conn()
.with(|connection| {
use crate::db::schema::import_definitions;
diesel::insert_into(import_definitions::table)
.values((
import_definitions::id.eq("stale-import"),
import_definitions::project_id.eq("p1"),
import_definitions::name.eq("Stale Import"),
import_definitions::created_at.eq(1_i64),
import_definitions::updated_at.eq(1_i64),
))
.execute(connection)?;
Ok(())
})
.unwrap();
// Post
let posts_dir = dir.path().join("posts").join("2024").join("01");
fs::create_dir_all(&posts_dir).unwrap();
@@ -288,6 +500,7 @@ height: 600
createdAt: 2024-01-15T12:00:00.000Z
updatedAt: 2024-01-15T12:00:00.000Z
tags: []
linkedPostIds: [\"test-post-1\"]
---
";
fs::write(media_dir.join("test-media-1.jpg.meta"), sidecar_content).unwrap();
@@ -328,6 +541,7 @@ updatedAt: \"2024-01-15T12:00:00.000Z\"
function render() end
";
fs::write(scripts_dir.join("test-script.lua"), script_content).unwrap();
fs::write(scripts_dir.join("legacy.py"), "print('ignore me')").unwrap();
let report = rebuild_from_filesystem(db.conn(), dir.path(), "p1").unwrap();
@@ -349,6 +563,36 @@ function render() end
let script = qs::get_script_by_id(db.conn(), "test-script-1").unwrap();
assert_eq!(script.title, "Test Script");
assert!(qs::get_script_by_id(db.conn(), "stale-script").is_err());
assert!(qtpl::get_template_by_id(db.conn(), "stale-template").is_err());
assert!(qp::get_post_by_id(db.conn(), "stale-post").is_err());
assert!(qm::get_media_by_id(db.conn(), "stale-media").is_err());
assert!(qtag::get_tag_by_id(db.conn(), "stale-tag").is_err());
let import_count = db
.conn()
.with(|connection| {
use crate::db::schema::import_definitions::dsl::*;
import_definitions
.filter(project_id.eq("p1"))
.count()
.get_result::<i64>(connection)
})
.unwrap();
assert_eq!(import_count, 0);
assert_eq!(
qs::list_scripts_by_project(db.conn(), "p1").unwrap().len(),
1
);
let project = crate::db::queries::project::get_project_by_id(db.conn(), "p1").unwrap();
assert_eq!(project.name, "Rebuilt Project");
assert!(project.description.is_none());
let links =
crate::db::queries::post_media::list_post_media_by_media(db.conn(), "test-media-1")
.unwrap();
assert_eq!(links.len(), 1);
assert_eq!(links[0].post_id, "test-post-1");
}
#[test]
@@ -398,12 +642,24 @@ updatedAt: \"2024-01-15T12:00:00.000Z\"
assert_eq!(r1.templates_updated, 0);
assert!(r1.errors.is_empty(), "errors: {:?}", r1.errors);
// Second rebuild - should update, not create
// A full rebuild clears the project rows before importing them again.
let r2 = rebuild_from_filesystem(db.conn(), dir.path(), "p1").unwrap();
assert_eq!(r2.posts_created, 0);
assert_eq!(r2.posts_updated, 1);
assert_eq!(r2.templates_created, 0);
assert_eq!(r2.templates_updated, 1);
assert_eq!(r2.posts_created, 1);
assert_eq!(r2.posts_updated, 0);
assert_eq!(r2.templates_created, 1);
assert_eq!(r2.templates_updated, 0);
assert!(r2.errors.is_empty(), "errors: {:?}", r2.errors);
}
#[test]
fn failed_rebuild_rolls_back_truncation() {
let (db, dir) = setup();
qp::insert_post(db.conn(), &qp::make_test_post("keep-me", "p1", "keep-me")).unwrap();
let posts_dir = dir.path().join("posts");
fs::create_dir_all(&posts_dir).unwrap();
fs::write(posts_dir.join("broken.md"), "not frontmatter").unwrap();
assert!(rebuild_from_filesystem(db.conn(), dir.path(), "p1").is_err());
assert!(qp::get_post_by_id(db.conn(), "keep-me").is_ok());
}
}