hugo/RuDS
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

fix: better database rebuild

Browse Source
This commit is contained in:
Chili Palmer
2026-07-18 20:46:27 +02:00
parent 638202b96c
commit b438a99999
6 changed files with 694 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files

97
crates/bds-core/src/engine/media.rs
View File

@@ -8,9 +8,10 @@ use walkdir::WalkDir;
use crate::db::fts;
use crate::db::queries::media as qm;
use crate::db::queries::media_translation as qmt;
use crate::db::queries::post as qp;
use crate::db::queries::post_media as qpm;
use crate::engine::{EngineError, EngineResult};
use crate::model::{Media, MediaTranslation};
use crate::model::{Media, MediaTranslation, PostMedia};
use crate::util::sidecar::{
MediaSidecar, MediaTranslationSidecar, read_sidecar, read_translation_sidecar,
};
@@ -641,31 +642,6 @@ pub(crate) fn rebuild_canonical_media(
.unwrap_or(&sidecar_rel)
.to_string();
let abs_file = data_dir.join(&file_path);
// Get file size (if file exists)
let file_size = if abs_file.exists() {
fs::metadata(&abs_file)?.len() as i64
} else {
sc.size
};
// Get checksum (if file exists) — streamed to avoid loading large files into memory
let checksum = if abs_file.exists() {
Some(crate::util::file_hash(&abs_file)?)
} else {
None
};
// Get dimensions (if file exists and is an image)
let (width, height) = if abs_file.exists() {
image_dimensions(&abs_file)
.map(|(w, h)| (Some(w as i32), Some(h as i32)))
.unwrap_or((sc.width, sc.height))
} else {
(sc.width, sc.height)
};
// Derive filename from file_path
let filename = Path::new(&file_path)
.file_name()
@@ -673,7 +649,7 @@ pub(crate) fn rebuild_canonical_media(
.unwrap_or("")
.to_string();
let now = now_unix_ms();
let linked_post_ids = sc.linked_post_ids.clone();
let existing = qm::get_media_by_id(conn, &sc.id);
let created = match existing {
@@ -681,9 +657,9 @@ pub(crate) fn rebuild_canonical_media(
// Update existing media
media.original_name = sc.original_name;
media.mime_type = sc.mime_type;
media.size = file_size;
media.width = width;
media.height = height;
media.size = sc.size;
media.width = sc.width;
media.height = sc.height;
media.title = sc.title;
media.alt = sc.alt;
media.caption = sc.caption;
@@ -691,9 +667,9 @@ pub(crate) fn rebuild_canonical_media(
media.language = sc.language;
media.file_path = file_path;
media.sidecar_path = sidecar_rel;
media.checksum = checksum;
media.checksum = None;
media.tags = sc.tags;
media.updated_at = now;
media.updated_at = sc.updated_at;
qm::update_media(conn, &media)?;
false
}
@@ -704,9 +680,9 @@ pub(crate) fn rebuild_canonical_media(
filename,
original_name: sc.original_name,
mime_type: sc.mime_type,
size: file_size,
width,
height,
size: sc.size,
width: sc.width,
height: sc.height,
title: sc.title,
alt: sc.alt,
caption: sc.caption,
@@ -714,20 +690,35 @@ pub(crate) fn rebuild_canonical_media(
language: sc.language,
file_path,
sidecar_path: sidecar_rel,
checksum,
checksum: None,
tags: sc.tags,
created_at: sc.created_at,
updated_at: now,
updated_at: sc.updated_at,
};
qm::insert_media(conn, &media)?;
true
}
};
// Regenerate thumbnails if the binary file exists
if abs_file.exists() {
let thumbnails_dir = data_dir.join("thumbnails");
let _ = generate_all_thumbnails(&abs_file, &thumbnails_dir, &sc.id);
let existing_post_ids: std::collections::HashSet<_> =
qpm::list_post_media_by_media(conn, &sc.id)?
.into_iter()
.map(|link| link.post_id)
.collect();
for post_id in linked_post_ids {
if !existing_post_ids.contains(&post_id) && qp::get_post_by_id(conn, &post_id).is_ok() {
qpm::link_media(
conn,
&PostMedia {
id: Uuid::new_v4().to_string(),
project_id: project_id.to_string(),
post_id,
media_id: sc.id.clone(),
sort_order: 0,
created_at: now_unix_ms(),
},
)?;
}
}
Ok(created)
@@ -1227,6 +1218,30 @@ alt: \"Ein Bild\"
assert_eq!(report2.translations_updated, 1);
}
#[test]
fn rebuild_media_trusts_sidecars_without_generating_thumbnails() {
let (db, dir) = setup();
let media_subdir = dir.path().join("media/2024/01");
fs::create_dir_all(&media_subdir).unwrap();
DynamicImage::new_rgb8(100, 80)
.save(media_subdir.join("sidecar-only.png"))
.unwrap();
fs::write(
media_subdir.join("sidecar-only.png.meta"),
"---\nid: sidecar-only\noriginalName: \"photo.png\"\nmimeType: image/png\nsize: 123\nwidth: 640\nheight: 480\ncreatedAt: 2024-01-15T12:00:00.000Z\nupdatedAt: 2024-01-15T12:00:00.000Z\ntags: []\n---",
)
.unwrap();
rebuild_media_from_filesystem(db.conn(), dir.path(), "p1").unwrap();
let media = qm::get_media_by_id(db.conn(), "sidecar-only").unwrap();
assert_eq!(
(media.size, media.width, media.height),
(123, Some(640), Some(480))
);
assert!(!dir.path().join("thumbnails").exists());
}
#[test]
fn is_translation_sidecar_detection() {
assert!(is_translation_sidecar("photo.jpg.de.meta"));

20
crates/bds-core/src/engine/meta.rs
View File

@@ -2,6 +2,7 @@ use std::collections::HashMap;
use std::fs;
use std::path::Path;
use crate::db::DbConnection as Connection;
use crate::engine::EngineResult;
use crate::model::PublishingPreferences;
use crate::model::metadata::{CategorySettings, ProjectMetadata, TagEntry};
@@ -25,6 +26,25 @@ pub fn write_project_json(data_dir: &Path, meta: &ProjectMetadata) -> EngineResu
Ok(())
}
/// Copy the project fields persisted in both representations from project.json
/// into the machine-local project row.
pub fn sync_project_from_file(
conn: &Connection,
data_dir: &Path,
project_id: &str,
) -> EngineResult<()> {
let metadata = read_project_json(data_dir)?;
metadata
.validate()
.map_err(crate::engine::EngineError::Validation)?;
let mut project = crate::db::queries::project::get_project_by_id(conn, project_id)?;
project.name = metadata.name;
project.description = metadata.description;
project.updated_at = crate::util::now_unix_ms();
crate::db::queries::project::update_project(conn, &project)?;
Ok(())
}
// ── categories.json ─────────────────────────────────────────────────
/// Read meta/categories.json as a sorted array of strings.

65
crates/bds-core/src/engine/metadata_diff.rs
View File

@@ -255,13 +255,7 @@ fn sync_project_from_file(
data_dir: &Path,
project_id: &str,
) -> EngineResult<()> {
let metadata = crate::engine::meta::read_project_json(data_dir)?;
let mut project = qproject::get_project_by_id(conn, project_id)?;
project.name = metadata.name;
project.description = metadata.description;
project.updated_at = crate::util::now_unix_ms();
qproject::update_project(conn, &project)?;
Ok(())
crate::engine::meta::sync_project_from_file(conn, data_dir, project_id)
}
fn rewrite_project_from_database(
@@ -865,7 +859,7 @@ fn detect_orphan_files(
"posts" => ext == "md",
"media" => ext == "meta",
"templates" => ext == "liquid",
"scripts" => ext == "lua" || ext == "py",
"scripts" => ext == "lua",
_ => false,
};
@@ -967,8 +961,8 @@ mod tests {
use crate::db::queries::template::insert_template;
use crate::engine::post::{create_post, publish_post};
use crate::model::{
Media, Post, PostStatus, Script, ScriptKind, ScriptStatus, Template, TemplateKind,
TemplateStatus,
Media, Post, PostStatus, ProjectMetadata, Script, ScriptKind, ScriptStatus, Template,
TemplateKind, TemplateStatus,
};
use crate::util::frontmatter::{
ScriptFrontmatter, TemplateFrontmatter, write_script_file, write_template_file,
@@ -1067,6 +1061,57 @@ mod tests {
assert_eq!(title_diffs[0].file_value, "Tampered Title");
}
#[test]
fn detects_missing_project_description() {
let (db, dir) = setup();
crate::engine::meta::write_project_json(
dir.path(),
&ProjectMetadata {
name: "Project p1".into(),
description: None,
public_url: None,
main_language: None,
default_author: None,
max_posts_per_page: 50,
image_import_concurrency: 4,
blogmark_category: None,
pico_theme: None,
semantic_similarity_enabled: false,
blog_languages: vec![],
},
)
.unwrap();
let report = compute_metadata_diff(db.conn(), dir.path(), "p1").unwrap();
let project = report
.diffs
.iter()
.find(|item| item.entity_type == "project")
.unwrap();
assert!(project.fields.iter().any(|field| {
field.field_name == "description"
&& field.db_value == "A test project"
&& field.file_value.is_empty()
}));
}
#[test]
fn ignores_python_scripts_during_orphan_scan() {
let (db, dir) = setup();
let scripts = dir.path().join("scripts");
fs::create_dir_all(&scripts).unwrap();
fs::write(scripts.join("legacy.py"), "print('legacy')").unwrap();
let report = compute_metadata_diff(db.conn(), dir.path(), "p1").unwrap();
assert!(
report
.orphans
.iter()
.all(|orphan| !orphan.file_path.ends_with(".py"))
);
}
#[test]
fn repairs_one_post_diff_from_file_to_database() {
let (db, dir) = setup();

280
crates/bds-core/src/engine/rebuild.rs
View File

@@ -2,6 +2,7 @@ use std::path::Path;
use std::sync::Arc;
use crate::db::DbConnection as Connection;
use diesel::prelude::*;
use crate::db::fts;
use crate::engine::EngineResult;
@@ -33,8 +34,8 @@ pub type ProgressFn = Arc<dyn Fn(f32, &str) + Send + Sync>;
/// Orchestrate a full rebuild from filesystem into the database.
///
/// Ensures FTS tables exist, then rebuilds posts, media, templates, and scripts
/// from their respective filesystem directories. Returns an aggregated report.
/// Replaces the project-scoped database state with project metadata, content,
/// and relationships reconstructed from the filesystem.
pub fn rebuild_from_filesystem(
conn: &Connection,
data_dir: &Path,
@@ -49,6 +50,25 @@ pub fn rebuild_from_filesystem_with_progress(
data_dir: &Path,
project_id: &str,
on_progress: Option<ProgressFn>,
) -> EngineResult<FullRebuildReport> {
conn.begin_savepoint()?;
match rebuild_from_filesystem_inner(conn, data_dir, project_id, on_progress) {
Ok(report) => {
conn.release_savepoint()?;
Ok(report)
}
Err(error) => {
let _ = conn.rollback_savepoint();
Err(error)
}
}
}
fn rebuild_from_filesystem_inner(
conn: &Connection,
data_dir: &Path,
project_id: &str,
on_progress: Option<ProgressFn>,
) -> EngineResult<FullRebuildReport> {
let mut report = FullRebuildReport::default();
let progress = |pct: f32, msg: &str| {
@@ -59,9 +79,11 @@ pub fn rebuild_from_filesystem_with_progress(
// Phase weights: posts 0.0..0.35, media 0.35..0.70, templates 0.70..0.85, scripts 0.85..1.0
// 1. Ensure FTS tables exist
progress(0.0, "Ensuring FTS tables...");
// 1. Load portable project metadata and clear all reconstructible rows.
progress(0.0, "Loading project metadata...");
fts::ensure_fts_tables(conn)?;
crate::engine::meta::sync_project_from_file(conn, data_dir, project_id)?;
clear_project_rows(conn, project_id)?;
// 2. Rebuild posts (0.00 .. 0.35)
progress(0.01, "Scanning posts...");
@@ -135,15 +157,93 @@ pub fn rebuild_from_filesystem_with_progress(
report.scripts_updated = script_report.updated;
report.errors.extend(script_report.errors);
// 6. Import tags from tags.json and sync from posts (0.95 .. 1.0)
// 6. Restore relationships and tags (0.95 .. 1.0)
progress(0.95, "Importing tags...");
let _ = super::tag::import_tags_from_file(conn, data_dir, project_id);
let _ = super::tag::sync_tags_from_posts(conn, project_id);
super::tag::import_tags_from_file(conn, data_dir, project_id)?;
super::tag::sync_tags_from_posts(conn, project_id)?;
post::rebuild_all_links(conn, data_dir, project_id)?;
if !report.errors.is_empty() {
return Err(crate::engine::EngineError::Validation(format!(
"rebuild failed: {}",
report.errors.join("; ")
)));
}
progress(1.0, "Rebuild complete");
Ok(report)
}
fn clear_project_rows(conn: &Connection, project_id: &str) -> EngineResult<()> {
use crate::db::schema::{
dismissed_duplicate_pairs, embedding_keys, generated_file_hashes, import_definitions,
media, media_translations, post_links, post_media, post_translations, posts, scripts, tags,
templates,
};
let post_ids = crate::db::queries::post::list_posts_by_project(conn, project_id)?
.into_iter()
.map(|post| post.id)
.collect::<Vec<_>>();
let media_ids = crate::db::queries::media::list_media_by_project(conn, project_id)?
.into_iter()
.map(|media| media.id)
.collect::<Vec<_>>();
for id in &post_ids {
fts::remove_post_from_index(conn, id)?;
}
for id in &media_ids {
fts::remove_media_from_index(conn, id)?;
}
conn.with(|connection| {
diesel::delete(
post_links::table.filter(
post_links::source_post_id
.eq_any(&post_ids)
.or(post_links::target_post_id.eq_any(&post_ids)),
),
)
.execute(connection)?;
diesel::delete(post_media::table.filter(post_media::project_id.eq(project_id)))
.execute(connection)?;
diesel::delete(
post_translations::table.filter(post_translations::project_id.eq(project_id)),
)
.execute(connection)?;
diesel::delete(
media_translations::table.filter(media_translations::project_id.eq(project_id)),
)
.execute(connection)?;
diesel::delete(
generated_file_hashes::table.filter(generated_file_hashes::project_id.eq(project_id)),
)
.execute(connection)?;
diesel::delete(embedding_keys::table.filter(embedding_keys::project_id.eq(project_id)))
.execute(connection)?;
diesel::delete(
dismissed_duplicate_pairs::table
.filter(dismissed_duplicate_pairs::project_id.eq(project_id)),
)
.execute(connection)?;
diesel::delete(
import_definitions::table.filter(import_definitions::project_id.eq(project_id)),
)
.execute(connection)?;
diesel::delete(tags::table.filter(tags::project_id.eq(project_id))).execute(connection)?;
diesel::delete(scripts::table.filter(scripts::project_id.eq(project_id)))
.execute(connection)?;
diesel::delete(templates::table.filter(templates::project_id.eq(project_id)))
.execute(connection)?;
diesel::delete(media::table.filter(media::project_id.eq(project_id)))
.execute(connection)?;
diesel::delete(posts::table.filter(posts::project_id.eq(project_id)))
.execute(connection)?;
Ok(())
})?;
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
@@ -152,7 +252,12 @@ mod tests {
use crate::db::queries::post as qp;
use crate::db::queries::project::{insert_project, make_test_project};
use crate::db::queries::script as qs;
use crate::db::queries::tag as qtag;
use crate::db::queries::template as qtpl;
use crate::model::metadata::ProjectMetadata;
use crate::model::{
Script, ScriptKind, ScriptStatus, Tag, Template, TemplateKind, TemplateStatus,
};
use std::fs;
use tempfile::TempDir;
@@ -162,6 +267,23 @@ mod tests {
fts::ensure_fts_tables(db.conn()).unwrap();
insert_project(db.conn(), &make_test_project("p1", "blog")).unwrap();
let dir = TempDir::new().unwrap();
crate::engine::meta::write_project_json(
dir.path(),
&ProjectMetadata {
name: "Project p1".into(),
description: Some("A test project".into()),
public_url: None,
main_language: None,
default_author: None,
max_posts_per_page: 50,
image_import_concurrency: 4,
blogmark_category: None,
pico_theme: None,
semantic_similarity_enabled: false,
blog_languages: vec![],
},
)
.unwrap();
(db, dir)
}
@@ -253,6 +375,96 @@ tags: []
fn rebuild_all_entity_types() {
let (db, dir) = setup();
crate::engine::meta::write_project_json(
dir.path(),
&ProjectMetadata {
name: "Rebuilt Project".into(),
description: None,
public_url: None,
main_language: Some("en".into()),
default_author: None,
max_posts_per_page: 50,
image_import_concurrency: 4,
blogmark_category: None,
pico_theme: None,
semantic_similarity_enabled: false,
blog_languages: vec!["en".into()],
},
)
.unwrap();
qs::insert_script(
db.conn(),
&Script {
id: "stale-script".into(),
project_id: "p1".into(),
slug: "stale-script".into(),
title: "Stale Script".into(),
kind: ScriptKind::Utility,
entrypoint: "main".into(),
enabled: true,
version: 1,
file_path: "scripts/stale.lua".into(),
status: ScriptStatus::Published,
content: None,
created_at: 1,
updated_at: 1,
},
)
.unwrap();
qtpl::insert_template(
db.conn(),
&Template {
id: "stale-template".into(),
project_id: "p1".into(),
slug: "stale-template".into(),
title: "Stale Template".into(),
kind: TemplateKind::Post,
enabled: true,
version: 1,
file_path: "templates/stale.liquid".into(),
status: TemplateStatus::Published,
content: None,
created_at: 1,
updated_at: 1,
},
)
.unwrap();
qp::insert_post(
db.conn(),
&qp::make_test_post("stale-post", "p1", "stale-post"),
)
.unwrap();
qm::insert_media(db.conn(), &qm::make_test_media("stale-media", "p1")).unwrap();
qtag::insert_tag(
db.conn(),
&Tag {
id: "stale-tag".into(),
project_id: "p1".into(),
name: "stale-tag".into(),
color: None,
post_template_slug: None,
created_at: 1,
updated_at: 1,
},
)
.unwrap();
db.conn()
.with(|connection| {
use crate::db::schema::import_definitions;
diesel::insert_into(import_definitions::table)
.values((
import_definitions::id.eq("stale-import"),
import_definitions::project_id.eq("p1"),
import_definitions::name.eq("Stale Import"),
import_definitions::created_at.eq(1_i64),
import_definitions::updated_at.eq(1_i64),
))
.execute(connection)?;
Ok(())
})
.unwrap();
// Post
let posts_dir = dir.path().join("posts").join("2024").join("01");
fs::create_dir_all(&posts_dir).unwrap();
@@ -288,6 +500,7 @@ height: 600
createdAt: 2024-01-15T12:00:00.000Z
updatedAt: 2024-01-15T12:00:00.000Z
tags: []
linkedPostIds: [\"test-post-1\"]
---
";
fs::write(media_dir.join("test-media-1.jpg.meta"), sidecar_content).unwrap();
@@ -328,6 +541,7 @@ updatedAt: \"2024-01-15T12:00:00.000Z\"
function render() end
";
fs::write(scripts_dir.join("test-script.lua"), script_content).unwrap();
fs::write(scripts_dir.join("legacy.py"), "print('ignore me')").unwrap();
let report = rebuild_from_filesystem(db.conn(), dir.path(), "p1").unwrap();
@@ -349,6 +563,36 @@ function render() end
let script = qs::get_script_by_id(db.conn(), "test-script-1").unwrap();
assert_eq!(script.title, "Test Script");
assert!(qs::get_script_by_id(db.conn(), "stale-script").is_err());
assert!(qtpl::get_template_by_id(db.conn(), "stale-template").is_err());
assert!(qp::get_post_by_id(db.conn(), "stale-post").is_err());
assert!(qm::get_media_by_id(db.conn(), "stale-media").is_err());
assert!(qtag::get_tag_by_id(db.conn(), "stale-tag").is_err());
let import_count = db
.conn()
.with(|connection| {
use crate::db::schema::import_definitions::dsl::*;
import_definitions
.filter(project_id.eq("p1"))
.count()
.get_result::<i64>(connection)
})
.unwrap();
assert_eq!(import_count, 0);
assert_eq!(
qs::list_scripts_by_project(db.conn(), "p1").unwrap().len(),
1
);
let project = crate::db::queries::project::get_project_by_id(db.conn(), "p1").unwrap();
assert_eq!(project.name, "Rebuilt Project");
assert!(project.description.is_none());
let links =
crate::db::queries::post_media::list_post_media_by_media(db.conn(), "test-media-1")
.unwrap();
assert_eq!(links.len(), 1);
assert_eq!(links[0].post_id, "test-post-1");
}
#[test]
@@ -398,12 +642,24 @@ updatedAt: \"2024-01-15T12:00:00.000Z\"
assert_eq!(r1.templates_updated, 0);
assert!(r1.errors.is_empty(), "errors: {:?}", r1.errors);
// Second rebuild - should update, not create
// A full rebuild clears the project rows before importing them again.
let r2 = rebuild_from_filesystem(db.conn(), dir.path(), "p1").unwrap();
assert_eq!(r2.posts_created, 0);
assert_eq!(r2.posts_updated, 1);
assert_eq!(r2.templates_created, 0);
assert_eq!(r2.templates_updated, 1);
assert_eq!(r2.posts_created, 1);
assert_eq!(r2.posts_updated, 0);
assert_eq!(r2.templates_created, 1);
assert_eq!(r2.templates_updated, 0);
assert!(r2.errors.is_empty(), "errors: {:?}", r2.errors);
}
#[test]
fn failed_rebuild_rolls_back_truncation() {
let (db, dir) = setup();
qp::insert_post(db.conn(), &qp::make_test_post("keep-me", "p1", "keep-me")).unwrap();
let posts_dir = dir.path().join("posts");
fs::create_dir_all(&posts_dir).unwrap();
fs::write(posts_dir.join("broken.md"), "not frontmatter").unwrap();
assert!(rebuild_from_filesystem(db.conn(), dir.path(), "p1").is_err());
assert!(qp::get_post_by_id(db.conn(), "keep-me").is_ok());
}
}