Align checksums with bDS2.

This commit is contained in:
2026-07-21 21:18:47 +02:00
parent 8e78a5fc2b
commit b3bf134980
16 changed files with 162 additions and 110 deletions

77
Cargo.lock generated
View File

@@ -938,6 +938,7 @@ dependencies = [
"libsqlite3-sys",
"liquid",
"liquid-core",
"md5",
"mlua",
"ort",
"pagefind",
@@ -1021,7 +1022,6 @@ dependencies = [
"iced",
"muda",
"objc2 0.6.4",
"objc2-app-kit 0.3.2",
"objc2-foundation 0.3.2",
"open",
"regex",
@@ -6129,8 +6129,8 @@ dependencies = [
"block2 0.5.1",
"libc",
"objc2 0.5.2",
"objc2-core-data 0.2.2",
"objc2-core-image 0.2.2",
"objc2-core-data",
"objc2-core-image",
"objc2-foundation 0.2.2",
"objc2-quartz-core 0.2.2",
]
@@ -6143,17 +6143,9 @@ checksum = "d49e936b501e5c5bf01fda3a9452ff86dc3ea98ad5f283e1455153142d97518c"
dependencies = [
"bitflags 2.13.1",
"block2 0.6.2",
"libc",
"objc2 0.6.4",
"objc2-cloud-kit 0.3.2",
"objc2-core-data 0.3.2",
"objc2-core-foundation",
"objc2-core-graphics",
"objc2-core-image 0.3.2",
"objc2-core-text",
"objc2-core-video",
"objc2-foundation 0.3.2",
"objc2-quartz-core 0.3.2",
]
[[package]]
@@ -6169,17 +6161,6 @@ dependencies = [
"objc2-foundation 0.2.2",
]
[[package]]
name = "objc2-cloud-kit"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "73ad74d880bb43877038da939b7427bba67e9dd42004a18b809ba7d87cee241c"
dependencies = [
"bitflags 2.13.1",
"objc2 0.6.4",
"objc2-foundation 0.3.2",
]
[[package]]
name = "objc2-contacts"
version = "0.2.2"
@@ -6203,17 +6184,6 @@ dependencies = [
"objc2-foundation 0.2.2",
]
[[package]]
name = "objc2-core-data"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b402a653efbb5e82ce4df10683b6b28027616a2715e90009947d50b8dd298fa"
dependencies = [
"bitflags 2.13.1",
"objc2 0.6.4",
"objc2-foundation 0.3.2",
]
[[package]]
name = "objc2-core-foundation"
version = "0.3.2"
@@ -6250,16 +6220,6 @@ dependencies = [
"objc2-metal",
]
[[package]]
name = "objc2-core-image"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e5d563b38d2b97209f8e861173de434bd0214cf020e3423a52624cd1d989f006"
dependencies = [
"objc2 0.6.4",
"objc2-foundation 0.3.2",
]
[[package]]
name = "objc2-core-location"
version = "0.2.2"
@@ -6284,31 +6244,6 @@ dependencies = [
"objc2-security",
]
[[package]]
name = "objc2-core-text"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0cde0dfb48d25d2b4862161a4d5fcc0e3c24367869ad306b0c9ec0073bfed92d"
dependencies = [
"bitflags 2.13.1",
"objc2 0.6.4",
"objc2-core-foundation",
"objc2-core-graphics",
]
[[package]]
name = "objc2-core-video"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d425caf1df73233f29fd8a5c3e5edbc30d2d4307870f802d18f00d83dc5141a6"
dependencies = [
"bitflags 2.13.1",
"objc2 0.6.4",
"objc2-core-foundation",
"objc2-core-graphics",
"objc2-io-surface",
]
[[package]]
name = "objc2-encode"
version = "4.1.0"
@@ -6441,9 +6376,9 @@ dependencies = [
"bitflags 2.13.1",
"block2 0.5.1",
"objc2 0.5.2",
"objc2-cloud-kit 0.2.2",
"objc2-core-data 0.2.2",
"objc2-core-image 0.2.2",
"objc2-cloud-kit",
"objc2-core-data",
"objc2-core-image",
"objc2-core-location",
"objc2-foundation 0.2.2",
"objc2-link-presentation",

View File

@@ -24,6 +24,7 @@ serde_json = "1"
serde_yaml = "0.9"
chrono = { version = "0.4", features = ["serde"] }
sha2 = "0.10"
md5 = "0.8.1"
unicode-normalization = "0.1"
thiserror = "2"
anyhow = "1"

View File

@@ -11,7 +11,7 @@ The project is under active development. Core blogging workflows are broadly ava
- Media import, thumbnails, metadata translations, filters, validation, post assignment, and sequential drag-and-drop insertion into post editors.
- WordPress WXR migration with saved analyses, HTML-to-Markdown and shortcode conversion, conflict/taxonomy review, recoverable 500-item execution batches, media-parent linking, progress reporting, and optional AI-assisted taxonomy mapping.
- Post, Liquid template, and Lua script editing with dedicated syntax highlighting and explicit syntax-check feedback, including publish-time enforcement of the bDS2 Liquid tag/filter/operator subset, using a custom Ropey/Syntect/Cosmic Text editor and the documented, bDS2-signature-compatible project-scoped [`bds` host API](docs/scripting/API_REFERENCE.md) across utilities, rendered macros, and Blogmark transforms, including airplane-gated Git sync.
- SQLite and filesystem persistence with frontmatter, sidecars, rebuild, metadata diff/repair, bDS2-compatible NFD slug generation, and FTS5 search.
- SQLite and filesystem persistence with frontmatter, sidecars, rebuild, metadata diff/repair, bDS2-compatible checksums and NFD slug generation, and FTS5 search.
- Optional on-device multilingual semantic search and similar-post tag suggestions backed by a persistent USearch index, plus always-available partial-name tag autocomplete and duplicate-post review in the desktop workspace.
- Read-only in-app browsers in the Help menu for the bundled global `DOCUMENTATION.md`, the generated Lua API reference with public types and runnable examples, the [CLI/server/TUI documentation](CLI.md), and the [MCP server documentation](MCP.md), with safe GFM rendering and confirmed external links.
- A localized OPML menu editor manages pages, submenus, and category archives with protected Home ordering, keyboard-accessible tree controls, drag-and-drop, and bDS2-compatible persistence.

View File

@@ -14,6 +14,7 @@ serde_json = { workspace = true }
serde_yaml = { workspace = true }
chrono = { workspace = true }
sha2 = { workspace = true }
md5 = { workspace = true }
unicode-normalization = { workspace = true }
thiserror = { workspace = true }
walkdir = { workspace = true }

View File

@@ -20,7 +20,7 @@ use crate::util::thumbnail::{
mime_from_extension,
};
use crate::util::{
atomic_write_str, content_hash, media_dir_path, media_sidecar_path,
atomic_write_str, media_dir_path, media_file_hash, media_sidecar_path,
media_translation_sidecar_path, now_unix_ms,
};
@@ -193,6 +193,7 @@ pub fn import_media(
author,
language,
tags,
None,
now_unix_ms(),
)
}
@@ -216,6 +217,7 @@ pub(crate) fn import_media_at(
author: Option<&str>,
language: Option<&str>,
tags: Vec<String>,
checksum: Option<&str>,
created_at: i64,
) -> EngineResult<Media> {
// Validate file type per spec
@@ -254,10 +256,6 @@ pub(crate) fn import_media_at(
// Compute sidecar path
let sidecar_rel = media_sidecar_path(&rel_file_path);
// Compute checksum of the copied file
let file_bytes = fs::read(&abs_file_path)?;
let checksum = content_hash(&file_bytes);
// Generate thumbnails (silently ignore errors for non-image files)
let thumbnails_dir = data_dir.join("thumbnails");
let _ = generate_all_thumbnails(&abs_file_path, &thumbnails_dir, &id);
@@ -278,7 +276,7 @@ pub(crate) fn import_media_at(
language: language.map(|s| s.to_string()),
file_path: rel_file_path,
sidecar_path: sidecar_rel.clone(),
checksum: Some(checksum),
checksum: checksum.map(str::to_owned),
tags,
created_at,
updated_at: created_at,
@@ -373,7 +371,7 @@ pub fn replace_media_file(
)));
}
let replacement_checksum = crate::util::file_hash(new_source_path)?;
let replacement_checksum = media_file_hash(new_source_path)?;
if media.checksum.as_deref() == Some(replacement_checksum.as_str()) {
return Ok(None);
}
@@ -994,7 +992,7 @@ mod tests {
assert_eq!(from_db.mime_type, "image/png");
assert_eq!(from_db.width, Some(100));
assert_eq!(from_db.height, Some(80));
assert!(from_db.checksum.is_some());
assert_eq!(from_db.checksum, None);
assert!(from_db.size > 0);
// Verify binary file exists
@@ -1234,7 +1232,12 @@ mod tests {
)
.unwrap();
let stored = dir.path().join(&media.file_path);
let updated_at = media.updated_at;
let duplicate = dir.path().join("duplicate.png");
fs::copy(&stored, &duplicate).unwrap();
let established = replace_media_file(db.conn(), dir.path(), &media.id, &duplicate)
.unwrap()
.unwrap();
let updated_at = established.updated_at;
assert!(
replace_media_file(db.conn(), dir.path(), &media.id, &stored)
@@ -1249,6 +1252,32 @@ mod tests {
);
}
#[test]
fn import_media_at_preserves_caller_supplied_bds2_checksum() {
let (db, dir) = setup();
let source = create_test_png(dir.path());
let checksum = crate::util::media_file_hash(&source).unwrap();
let media = import_media_at(
db.conn(),
dir.path(),
"p1",
&source,
"photo.png",
None,
None,
None,
None,
None,
vec![],
Some(&checksum),
now_unix_ms(),
)
.unwrap();
assert_eq!(media.checksum.as_deref(), Some(checksum.as_str()));
}
#[test]
fn delete_media_removes_everything() {
let (db, dir) = setup();

View File

@@ -15,8 +15,7 @@ use crate::util::frontmatter::{
read_post_file, read_translation_file, write_post_file, write_translation_file,
};
use crate::util::{
atomic_write_str, content_hash, ensure_unique, now_unix_ms, post_file_path, slugify,
translation_file_path,
atomic_write_str, ensure_unique, now_unix_ms, post_file_path, slugify, translation_file_path,
};
/// Report returned by `rebuild_posts_from_filesystem`.
@@ -255,10 +254,6 @@ fn publish_post_in_savepoint(
let file_content = write_post_file(&post, &body);
atomic_write_str(&abs_path, &file_content)?;
// Compute checksum
let hash = content_hash(file_content.as_bytes());
post.checksum = Some(hash);
// Set published snapshot fields
let tags_json = serde_json::to_string(&post.tags).unwrap_or_else(|_| "[]".into());
let cats_json = serde_json::to_string(&post.categories).unwrap_or_else(|_| "[]".into());
@@ -387,7 +382,7 @@ pub fn discard_post_draft(conn: &Connection, data_dir: &Path, post_id: &str) ->
fm.do_not_translate,
fm.tags,
fm.categories,
Some(content_hash(raw.as_bytes())),
None,
)
} else {
(
@@ -887,8 +882,6 @@ fn publish_translation(
let file_content = write_translation_file(t, &body);
atomic_write_str(&abs_path, &file_content)?;
let hash = content_hash(file_content.as_bytes());
t.checksum = Some(hash);
t.content = None;
qt::update_post_translation(conn, t)?;
@@ -984,7 +977,6 @@ pub(crate) fn rebuild_canonical_post(
.to_string_lossy()
.to_string();
let hash = content_hash(content.as_bytes());
let status = match fm.status.as_str() {
"published" => PostStatus::Published,
"archived" => PostStatus::Archived,
@@ -1010,7 +1002,7 @@ pub(crate) fn rebuild_canonical_post(
post.do_not_translate = fm.do_not_translate;
post.template_slug = fm.template_slug;
post.file_path = rel_path;
post.checksum = Some(hash);
post.checksum = None;
post.tags = fm.tags;
post.categories = fm.categories;
post.created_at = fm.created_at;
@@ -1038,7 +1030,7 @@ pub(crate) fn rebuild_canonical_post(
do_not_translate: fm.do_not_translate,
template_slug: fm.template_slug,
file_path: rel_path,
checksum: Some(hash),
checksum: None,
tags: fm.tags,
categories: fm.categories,
published_title: None,
@@ -1072,8 +1064,6 @@ pub(crate) fn rebuild_translation(
.to_string_lossy()
.to_string();
let hash = content_hash(content.as_bytes());
// Check if parent post exists
let parent = qp::get_post_by_id(conn, &fm.translation_for);
if parent.is_err() {
@@ -1110,7 +1100,7 @@ pub(crate) fn rebuild_translation(
};
t.status = status;
t.file_path = rel_path;
t.checksum = Some(hash);
t.checksum = None;
t.created_at = created_at;
t.updated_at = updated_at;
t.published_at = published_at;
@@ -1132,7 +1122,7 @@ pub(crate) fn rebuild_translation(
},
status,
file_path: rel_path,
checksum: Some(hash),
checksum: None,
created_at,
updated_at,
published_at,
@@ -1643,6 +1633,48 @@ mod tests {
);
}
#[test]
fn publish_preserves_caller_supplied_checksums() {
let (db, dir) = setup();
let mut post = create_post(
db.conn(),
dir.path(),
"p1",
"Imported",
Some("body"),
vec![],
vec![],
None,
None,
None,
)
.unwrap();
post.checksum = Some("bds2-post-checksum".into());
qp::update_post(db.conn(), &post).unwrap();
let mut translation = upsert_translation(
db.conn(),
dir.path(),
&post.id,
"de",
"Importiert",
None,
Some("Inhalt"),
)
.unwrap();
translation.checksum = Some("bds2-translation-checksum".into());
qt::update_post_translation(db.conn(), &translation).unwrap();
let published = publish_post(db.conn(), dir.path(), &post.id).unwrap();
let published_translation =
qt::get_post_translation_by_id(db.conn(), &translation.id).unwrap();
assert_eq!(published.checksum.as_deref(), Some("bds2-post-checksum"));
assert_eq!(
published_translation.checksum.as_deref(),
Some("bds2-translation-checksum")
);
}
#[test]
fn delete_post_removes_everything() {
let (db, dir) = setup();
@@ -1873,12 +1905,14 @@ mod tests {
assert_eq!(post.slug, "rebuilt-post");
assert_eq!(post.tags, vec!["test"]);
assert_eq!(post.updated_at, 1_705_320_000_000);
assert_eq!(post.checksum, None);
// Verify translation in DB
let trans =
qt::get_post_translation_by_post_and_language(db.conn(), "rebuild-post-1", "de")
.unwrap();
assert_eq!(trans.title, "Wiederhergestellter Beitrag");
assert_eq!(trans.checksum, None);
// Run rebuild again - should update, not create
let report2 = rebuild_posts_from_filesystem(db.conn(), dir.path(), "p1").unwrap();

View File

@@ -25,8 +25,8 @@ use crate::model::{
TaxonomyKind,
};
use crate::util::{
atomic_write_str, content_hash, file_hash, media_sidecar_path, media_translation_sidecar_path,
now_unix_ms, post_file_path, slugify,
atomic_write_str, content_hash, media_file_hash, media_sidecar_path,
media_translation_sidecar_path, now_unix_ms, post_file_path, slugify,
};
const TRANSACTION_BATCH_SIZE: usize = 500;
@@ -678,7 +678,7 @@ fn analyze_media(
let checksum = source_path
.as_ref()
.filter(|path| path.is_file())
.and_then(|path| file_hash(path).ok());
.and_then(|path| media_file_hash(path).ok());
let existing_by_name = media_by_name.get(&item.filename.to_lowercase()).copied();
let existing_by_checksum = checksum
.as_ref()
@@ -1369,7 +1369,6 @@ fn import_post_item(
imported.published_at = item.published_at.or(Some(imported.created_at));
imported.file_path = post_file_path(imported.created_at, &imported.slug);
let serialized = crate::util::frontmatter::write_post_file(&imported, &body);
imported.checksum = Some(content_hash(serialized.as_bytes()));
atomic_write_str(&data_dir.join(&imported.file_path), &serialized)?;
if !previous_file_path.is_empty() && previous_file_path != imported.file_path {
remove_file_if_present(&data_dir.join(previous_file_path))?;
@@ -1476,6 +1475,7 @@ fn execute_media_phase(
default_author,
None,
Vec::new(),
item.checksum.as_deref(),
item.created_at.unwrap_or_else(now_unix_ms),
)?
};

View File

@@ -25,6 +25,26 @@ pub fn file_hash(path: &Path) -> std::io::Result<String> {
Ok(encode_hex(hasher.finalize()))
}
/// Compute the bDS2-compatible lower-hex MD5 change signal for media bytes.
pub fn media_content_hash(content: &[u8]) -> String {
format!("{:x}", md5::compute(content))
}
/// Compute the bDS2-compatible lower-hex MD5 change signal for a media file.
pub fn media_file_hash(path: &Path) -> std::io::Result<String> {
let mut file = std::fs::File::open(path)?;
let mut context = md5::Context::new();
let mut buf = [0u8; 8192];
loop {
let n = file.read(&mut buf)?;
if n == 0 {
break;
}
context.consume(&buf[..n]);
}
Ok(format!("{:x}", context.finalize()))
}
fn encode_hex(bytes: impl AsRef<[u8]>) -> String {
bytes
.as_ref()
@@ -55,4 +75,13 @@ mod tests {
"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
);
}
#[test]
fn media_hash_matches_bds2_fixture() {
// Base.encode16(:crypto.hash(:md5, "hello"), case: :lower) in bDS2.
assert_eq!(
media_content_hash(b"hello"),
"5d41402abc4b2a76b9719d911017c592"
);
}
}

View File

@@ -10,7 +10,7 @@ pub mod timestamp;
pub use app_paths::{application_data_dir, application_database_path, default_project_data_dir};
pub use atomic_write::{atomic_write, atomic_write_str};
pub use checksum::{content_hash, file_hash};
pub use checksum::{content_hash, file_hash, media_content_hash, media_file_hash};
pub use paths::*;
pub use slug::{ensure_unique, slugify};
pub use timestamp::{

View File

@@ -163,7 +163,7 @@ fn analysis_distinguishes_new_updates_conflicts_duplicates_and_missing_media() {
.unwrap();
let old_different = dir.path().join("old-different.png");
DynamicImage::new_rgb8(2, 2).save(&old_different).unwrap();
bds_core::engine::media::import_media(
let same = bds_core::engine::media::import_media(
db.conn(),
dir.path(),
&project.id,
@@ -177,7 +177,14 @@ fn analysis_distinguishes_new_updates_conflicts_duplicates_and_missing_media() {
Vec::new(),
)
.unwrap();
bds_core::engine::media::import_media(
bds_core::engine::media::replace_media_file(
db.conn(),
dir.path(),
&same.id,
&uploads.join("same.png"),
)
.unwrap();
let different = bds_core::engine::media::import_media(
db.conn(),
dir.path(),
&project.id,
@@ -191,6 +198,13 @@ fn analysis_distinguishes_new_updates_conflicts_duplicates_and_missing_media() {
Vec::new(),
)
.unwrap();
bds_core::engine::media::replace_media_file(
db.conn(),
dir.path(),
&different.id,
&old_different,
)
.unwrap();
let same_body = "Hello **world**.\n\n[[gallery ids=\"1,2\"]]";
let update = post::create_post(

View File

@@ -25,7 +25,6 @@ wry = "0.55.1"
[target.'cfg(target_os = "macos")'.dependencies]
objc2 = "0.6"
objc2-foundation = { version = "0.3", features = ["objc2-core-services"] }
objc2-app-kit = "0.3"
[dev-dependencies]
fluent-syntax = { workspace = true }

View File

@@ -264,6 +264,8 @@ invariant WxrConversionParity {
}
invariant AnalysisClassification {
-- Post/page checksums are lower-hex SHA-256 of converted Markdown; media
-- checksums are lower-hex MD5 of source bytes, matching bDS2.
-- Posts/pages: same slug and checksum => update; same slug with different
-- content => conflict; checksum matching another post => content duplicate;
-- otherwise new. Media applies the same rules by original filename and

View File

@@ -254,7 +254,8 @@ invariant MediaTranslationFileLayout {
rule ImportMedia {
when: ImportMediaRequested(source_path, project, metadata)
-- metadata is optional import context: title, alt, caption, author,
-- language, and tags may be supplied by the caller.
-- language, tags, and a checksum may be supplied by the caller. WXR
-- supplies a lower-hex MD5 checksum; interactive imports leave it nil.
-- 1. Validate file type (must be supported image)
-- 2. Generate UUID v4 filename
-- 3. Copy to media/{YYYY}/{MM}/{uuid}.{ext}
@@ -276,7 +277,7 @@ rule ImportMedia {
tags: metadata.tags,
file_path: format("media/{yyyy}/{mm}/{uuid}.{ext}"),
sidecar_path: format("media/{yyyy}/{mm}/{uuid}.{ext}.meta"),
checksum: sha256(source_path)
checksum: metadata.checksum
)
ensures: ThumbnailsGenerated(media_id)
ensures: SearchIndexUpdated(media_id)

View File

@@ -98,6 +98,8 @@ entity Post {
template_slug: String?
file_path: String
checksum: String?
-- Caller-supplied SHA-256 content change signal. Publish preserves it;
-- filesystem rebuild/sync clears it to match bDS2.
tags: List<String>
categories: List<String>
created_at: Timestamp
@@ -210,6 +212,7 @@ rule PublishPost {
-- Writes frontmatter + markdown to posts/YYYY/MM/{slug}.md
ensures: post.content = null
-- Content cleared from DB; now lives in filesystem only
-- post.checksum is preserved unchanged.
ensures: SearchIndexUpdated(post)
ensures: PostLinksUpdated(post)
-- Parse inter-post links, update link graph
@@ -242,6 +245,7 @@ rule SyncPostFromFile {
-- record from it (the filesystem is treated as truth for that one post).
-- Re-syncs the post's link graph. Errors out if the file is missing.
ensures: PostFieldsUpdated(post, parse_post_file(post.file_path))
ensures: post.checksum = null
ensures: PostLinksUpdated(post)
}

View File

@@ -55,7 +55,7 @@ entity Post {
updated_at: Timestamp
published_at: Timestamp?
file_path: String -- Empty for never-published drafts
checksum: String? -- SHA-256 of content
checksum: String? -- Caller-supplied SHA-256 content change signal; nil after filesystem rebuild
tags: Set<String> -- JSON array stored as text
categories: Set<String> -- JSON array stored as text
template_slug: String? -- User template override
@@ -83,7 +83,7 @@ entity PostTranslation {
updated_at: Timestamp
published_at: Timestamp?
file_path: String
checksum: String?
checksum: String? -- Caller-supplied content change signal; nil after filesystem rebuild
}
entity Media {
@@ -103,7 +103,7 @@ entity Media {
sidecar_path: String -- Path to .meta sidecar file
created_at: Timestamp
updated_at: Timestamp
checksum: String?
checksum: String? -- Lower-hex MD5 of media bytes when known
tags: Set<String> -- JSON array stored as text
language: String? -- ISO 639-1 code
}

View File

@@ -51,6 +51,8 @@ entity PostTranslation {
status: PostTranslationStatus
file_path: String
checksum: String?
-- Caller-supplied content change signal. Publish preserves it;
-- filesystem rebuild/sync clears it to match bDS2.
created_at: Timestamp
updated_at: Timestamp
published_at: Timestamp?
@@ -143,6 +145,7 @@ rule PublishTranslation {
ensures: TranslationFileWritten(translation)
ensures: translation.content = null
-- Content moves to filesystem
-- translation.checksum is preserved unchanged.
}
rule ReopenPublishedTranslation {