hugo/RuDS
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

Align checksums with bDS2.

Browse Source
This commit is contained in:
Chili Palmer
2026-07-21 21:18:47 +02:00
parent 8e78a5fc2b
commit b3bf134980
16 changed files with 162 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
crates/bds-core/Cargo.toml
View File

@@ -14,6 +14,7 @@ serde_json = { workspace = true }
serde_yaml = { workspace = true }
chrono = { workspace = true }
sha2 = { workspace = true }
md5 = { workspace = true }
unicode-normalization = { workspace = true }
thiserror = { workspace = true }
walkdir = { workspace = true }

47
crates/bds-core/src/engine/media.rs
View File

@@ -20,7 +20,7 @@ use crate::util::thumbnail::{
mime_from_extension,
};
use crate::util::{
atomic_write_str, content_hash, media_dir_path, media_sidecar_path,
atomic_write_str, media_dir_path, media_file_hash, media_sidecar_path,
media_translation_sidecar_path, now_unix_ms,
};
@@ -193,6 +193,7 @@ pub fn import_media(
author,
language,
tags,
None,
now_unix_ms(),
)
}
@@ -216,6 +217,7 @@ pub(crate) fn import_media_at(
author: Option<&str>,
language: Option<&str>,
tags: Vec<String>,
checksum: Option<&str>,
created_at: i64,
) -> EngineResult<Media> {
// Validate file type per spec
@@ -254,10 +256,6 @@ pub(crate) fn import_media_at(
// Compute sidecar path
let sidecar_rel = media_sidecar_path(&rel_file_path);
// Compute checksum of the copied file
let file_bytes = fs::read(&abs_file_path)?;
let checksum = content_hash(&file_bytes);
// Generate thumbnails (silently ignore errors for non-image files)
let thumbnails_dir = data_dir.join("thumbnails");
let _ = generate_all_thumbnails(&abs_file_path, &thumbnails_dir, &id);
@@ -278,7 +276,7 @@ pub(crate) fn import_media_at(
language: language.map(|s| s.to_string()),
file_path: rel_file_path,
sidecar_path: sidecar_rel.clone(),
checksum: Some(checksum),
checksum: checksum.map(str::to_owned),
tags,
created_at,
updated_at: created_at,
@@ -373,7 +371,7 @@ pub fn replace_media_file(
)));
}
let replacement_checksum = crate::util::file_hash(new_source_path)?;
let replacement_checksum = media_file_hash(new_source_path)?;
if media.checksum.as_deref() == Some(replacement_checksum.as_str()) {
return Ok(None);
}
@@ -994,7 +992,7 @@ mod tests {
assert_eq!(from_db.mime_type, "image/png");
assert_eq!(from_db.width, Some(100));
assert_eq!(from_db.height, Some(80));
assert!(from_db.checksum.is_some());
assert_eq!(from_db.checksum, None);
assert!(from_db.size > 0);
// Verify binary file exists
@@ -1234,7 +1232,12 @@ mod tests {
)
.unwrap();
let stored = dir.path().join(&media.file_path);
let updated_at = media.updated_at;
let duplicate = dir.path().join("duplicate.png");
fs::copy(&stored, &duplicate).unwrap();
let established = replace_media_file(db.conn(), dir.path(), &media.id, &duplicate)
.unwrap()
.unwrap();
let updated_at = established.updated_at;
assert!(
replace_media_file(db.conn(), dir.path(), &media.id, &stored)
@@ -1249,6 +1252,32 @@ mod tests {
);
}
#[test]
fn import_media_at_preserves_caller_supplied_bds2_checksum() {
let (db, dir) = setup();
let source = create_test_png(dir.path());
let checksum = crate::util::media_file_hash(&source).unwrap();
let media = import_media_at(
db.conn(),
dir.path(),
"p1",
&source,
"photo.png",
None,
None,
None,
None,
None,
vec![],
Some(&checksum),
now_unix_ms(),
)
.unwrap();
assert_eq!(media.checksum.as_deref(), Some(checksum.as_str()));
}
#[test]
fn delete_media_removes_everything() {
let (db, dir) = setup();

66
crates/bds-core/src/engine/post.rs
View File

@@ -15,8 +15,7 @@ use crate::util::frontmatter::{
read_post_file, read_translation_file, write_post_file, write_translation_file,
};
use crate::util::{
atomic_write_str, content_hash, ensure_unique, now_unix_ms, post_file_path, slugify,
translation_file_path,
atomic_write_str, ensure_unique, now_unix_ms, post_file_path, slugify, translation_file_path,
};
/// Report returned by `rebuild_posts_from_filesystem`.
@@ -255,10 +254,6 @@ fn publish_post_in_savepoint(
let file_content = write_post_file(&post, &body);
atomic_write_str(&abs_path, &file_content)?;
// Compute checksum
let hash = content_hash(file_content.as_bytes());
post.checksum = Some(hash);
// Set published snapshot fields
let tags_json = serde_json::to_string(&post.tags).unwrap_or_else(|_| "[]".into());
let cats_json = serde_json::to_string(&post.categories).unwrap_or_else(|_| "[]".into());
@@ -387,7 +382,7 @@ pub fn discard_post_draft(conn: &Connection, data_dir: &Path, post_id: &str) ->
fm.do_not_translate,
fm.tags,
fm.categories,
Some(content_hash(raw.as_bytes())),
None,
)
} else {
(
@@ -887,8 +882,6 @@ fn publish_translation(
let file_content = write_translation_file(t, &body);
atomic_write_str(&abs_path, &file_content)?;
let hash = content_hash(file_content.as_bytes());
t.checksum = Some(hash);
t.content = None;
qt::update_post_translation(conn, t)?;
@@ -984,7 +977,6 @@ pub(crate) fn rebuild_canonical_post(
.to_string_lossy()
.to_string();
let hash = content_hash(content.as_bytes());
let status = match fm.status.as_str() {
"published" => PostStatus::Published,
"archived" => PostStatus::Archived,
@@ -1010,7 +1002,7 @@ pub(crate) fn rebuild_canonical_post(
post.do_not_translate = fm.do_not_translate;
post.template_slug = fm.template_slug;
post.file_path = rel_path;
post.checksum = Some(hash);
post.checksum = None;
post.tags = fm.tags;
post.categories = fm.categories;
post.created_at = fm.created_at;
@@ -1038,7 +1030,7 @@ pub(crate) fn rebuild_canonical_post(
do_not_translate: fm.do_not_translate,
template_slug: fm.template_slug,
file_path: rel_path,
checksum: Some(hash),
checksum: None,
tags: fm.tags,
categories: fm.categories,
published_title: None,
@@ -1072,8 +1064,6 @@ pub(crate) fn rebuild_translation(
.to_string_lossy()
.to_string();
let hash = content_hash(content.as_bytes());
// Check if parent post exists
let parent = qp::get_post_by_id(conn, &fm.translation_for);
if parent.is_err() {
@@ -1110,7 +1100,7 @@ pub(crate) fn rebuild_translation(
};
t.status = status;
t.file_path = rel_path;
t.checksum = Some(hash);
t.checksum = None;
t.created_at = created_at;
t.updated_at = updated_at;
t.published_at = published_at;
@@ -1132,7 +1122,7 @@ pub(crate) fn rebuild_translation(
},
status,
file_path: rel_path,
checksum: Some(hash),
checksum: None,
created_at,
updated_at,
published_at,
@@ -1643,6 +1633,48 @@ mod tests {
);
}
#[test]
fn publish_preserves_caller_supplied_checksums() {
let (db, dir) = setup();
let mut post = create_post(
db.conn(),
dir.path(),
"p1",
"Imported",
Some("body"),
vec![],
vec![],
None,
None,
None,
)
.unwrap();
post.checksum = Some("bds2-post-checksum".into());
qp::update_post(db.conn(), &post).unwrap();
let mut translation = upsert_translation(
db.conn(),
dir.path(),
&post.id,
"de",
"Importiert",
None,
Some("Inhalt"),
)
.unwrap();
translation.checksum = Some("bds2-translation-checksum".into());
qt::update_post_translation(db.conn(), &translation).unwrap();
let published = publish_post(db.conn(), dir.path(), &post.id).unwrap();
let published_translation =
qt::get_post_translation_by_id(db.conn(), &translation.id).unwrap();
assert_eq!(published.checksum.as_deref(), Some("bds2-post-checksum"));
assert_eq!(
published_translation.checksum.as_deref(),
Some("bds2-translation-checksum")
);
}
#[test]
fn delete_post_removes_everything() {
let (db, dir) = setup();
@@ -1873,12 +1905,14 @@ mod tests {
assert_eq!(post.slug, "rebuilt-post");
assert_eq!(post.tags, vec!["test"]);
assert_eq!(post.updated_at, 1_705_320_000_000);
assert_eq!(post.checksum, None);
// Verify translation in DB
let trans =
qt::get_post_translation_by_post_and_language(db.conn(), "rebuild-post-1", "de")
.unwrap();
assert_eq!(trans.title, "Wiederhergestellter Beitrag");
assert_eq!(trans.checksum, None);
// Run rebuild again - should update, not create
let report2 = rebuild_posts_from_filesystem(db.conn(), dir.path(), "p1").unwrap();

8
crates/bds-core/src/engine/wordpress_import.rs
View File

@@ -25,8 +25,8 @@ use crate::model::{
TaxonomyKind,
};
use crate::util::{
atomic_write_str, content_hash, file_hash, media_sidecar_path, media_translation_sidecar_path,
now_unix_ms, post_file_path, slugify,
atomic_write_str, content_hash, media_file_hash, media_sidecar_path,
media_translation_sidecar_path, now_unix_ms, post_file_path, slugify,
};
const TRANSACTION_BATCH_SIZE: usize = 500;
@@ -678,7 +678,7 @@ fn analyze_media(
let checksum = source_path
.as_ref()
.filter(|path| path.is_file())
.and_then(|path| file_hash(path).ok());
.and_then(|path| media_file_hash(path).ok());
let existing_by_name = media_by_name.get(&item.filename.to_lowercase()).copied();
let existing_by_checksum = checksum
.as_ref()
@@ -1369,7 +1369,6 @@ fn import_post_item(
imported.published_at = item.published_at.or(Some(imported.created_at));
imported.file_path = post_file_path(imported.created_at, &imported.slug);
let serialized = crate::util::frontmatter::write_post_file(&imported, &body);
imported.checksum = Some(content_hash(serialized.as_bytes()));
atomic_write_str(&data_dir.join(&imported.file_path), &serialized)?;
if !previous_file_path.is_empty() && previous_file_path != imported.file_path {
remove_file_if_present(&data_dir.join(previous_file_path))?;
@@ -1476,6 +1475,7 @@ fn execute_media_phase(
default_author,
None,
Vec::new(),
item.checksum.as_deref(),
item.created_at.unwrap_or_else(now_unix_ms),
)?
};

29
crates/bds-core/src/util/checksum.rs
View File

@@ -25,6 +25,26 @@ pub fn file_hash(path: &Path) -> std::io::Result<String> {
Ok(encode_hex(hasher.finalize()))
}
/// Compute the bDS2-compatible lower-hex MD5 change signal for media bytes.
pub fn media_content_hash(content: &[u8]) -> String {
format!("{:x}", md5::compute(content))
}
/// Compute the bDS2-compatible lower-hex MD5 change signal for a media file.
pub fn media_file_hash(path: &Path) -> std::io::Result<String> {
let mut file = std::fs::File::open(path)?;
let mut context = md5::Context::new();
let mut buf = [0u8; 8192];
loop {
let n = file.read(&mut buf)?;
if n == 0 {
break;
}
context.consume(&buf[..n]);
}
Ok(format!("{:x}", context.finalize()))
}
fn encode_hex(bytes: impl AsRef<[u8]>) -> String {
bytes
.as_ref()
@@ -55,4 +75,13 @@ mod tests {
"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
);
}
#[test]
fn media_hash_matches_bds2_fixture() {
// Base.encode16(:crypto.hash(:md5, "hello"), case: :lower) in bDS2.
assert_eq!(
media_content_hash(b"hello"),
"5d41402abc4b2a76b9719d911017c592"
);
}
}

2
crates/bds-core/src/util/mod.rs
View File

@@ -10,7 +10,7 @@ pub mod timestamp;
pub use app_paths::{application_data_dir, application_database_path, default_project_data_dir};
pub use atomic_write::{atomic_write, atomic_write_str};
pub use checksum::{content_hash, file_hash};
pub use checksum::{content_hash, file_hash, media_content_hash, media_file_hash};
pub use paths::*;
pub use slug::{ensure_unique, slugify};
pub use timestamp::{

18
crates/bds-core/tests/wordpress_import.rs
View File

@@ -163,7 +163,7 @@ fn analysis_distinguishes_new_updates_conflicts_duplicates_and_missing_media() {
.unwrap();
let old_different = dir.path().join("old-different.png");
DynamicImage::new_rgb8(2, 2).save(&old_different).unwrap();
bds_core::engine::media::import_media(
let same = bds_core::engine::media::import_media(
db.conn(),
dir.path(),
&project.id,
@@ -177,7 +177,14 @@ fn analysis_distinguishes_new_updates_conflicts_duplicates_and_missing_media() {
Vec::new(),
)
.unwrap();
bds_core::engine::media::import_media(
bds_core::engine::media::replace_media_file(
db.conn(),
dir.path(),
&same.id,
&uploads.join("same.png"),
)
.unwrap();
let different = bds_core::engine::media::import_media(
db.conn(),
dir.path(),
&project.id,
@@ -191,6 +198,13 @@ fn analysis_distinguishes_new_updates_conflicts_duplicates_and_missing_media() {
Vec::new(),
)
.unwrap();
bds_core::engine::media::replace_media_file(
db.conn(),
dir.path(),
&different.id,
&old_different,
)
.unwrap();
let same_body = "Hello **world**.\n\n[[gallery ids=\"1,2\"]]";
let update = post::create_post(

1
crates/bds-ui/Cargo.toml
View File

@@ -25,7 +25,6 @@ wry = "0.55.1"
[target.'cfg(target_os = "macos")'.dependencies]
objc2 = "0.6"
objc2-foundation = { version = "0.3", features = ["objc2-core-services"] }
objc2-app-kit = "0.3"
[dev-dependencies]
fluent-syntax = { workspace = true }