diff --git a/Cargo.lock b/Cargo.lock index ccbef4d..2535e69 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -939,7 +939,6 @@ dependencies = [ "axum", "base64 0.22.1", "chrono", - "deunicode", "diesel", "diesel_migrations", "dirs 5.0.1", @@ -969,6 +968,7 @@ dependencies = [ "tempfile", "thiserror 2.0.19", "tokio", + "unicode-normalization", "url", "usearch", "uuid", @@ -2489,12 +2489,6 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "21d8ad60dd5b13a4ee6bd8fa2d5d88965c597c67bce32b5fc49c94f55cb50810" -[[package]] -name = "deunicode" -version = "1.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abd57806937c9cc163efc8ea3910e00a62e2aeb0b8119f1793a978088f8f6b04" - [[package]] name = "diesel" version = "2.3.11" diff --git a/Cargo.toml b/Cargo.toml index 7ccc384..e388107 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,7 +24,7 @@ serde_json = "1" serde_yaml = "0.9" chrono = { version = "0.4", features = ["serde"] } sha2 = "0.10" -deunicode = "1" +unicode-normalization = "0.1" thiserror = "2" anyhow = "1" clap = { version = "4", features = ["derive"] } diff --git a/README.md b/README.md index 1a83756..56e4f45 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ The project is under active development. Core blogging workflows are broadly ava - Media import, thumbnails, metadata translations, filters, validation, post assignment, and sequential drag-and-drop insertion into post editors. - WordPress WXR migration with saved analyses, HTML-to-Markdown and shortcode conversion, conflict/taxonomy review, recoverable 500-item execution batches, media-parent linking, progress reporting, and optional AI-assisted taxonomy mapping. - Post, Liquid template, and Lua script editing with dedicated syntax highlighting and explicit syntax-check feedback, using a custom Ropey/Syntect/Cosmic Text editor and the documented, bDS2-signature-compatible project-scoped [`bds` host API](docs/scripting/API_REFERENCE.md) across utilities, rendered macros, and Blogmark transforms, including airplane-gated Git sync. -- SQLite and filesystem persistence with frontmatter, sidecars, rebuild, metadata diff/repair, and FTS5 search. +- SQLite and filesystem persistence with frontmatter, sidecars, rebuild, metadata diff/repair, bDS2-compatible NFD slug generation, and FTS5 search. - Optional on-device multilingual semantic search and similar-post tag suggestions backed by a persistent USearch index, plus always-available partial-name tag autocomplete and duplicate-post review in the desktop workspace. - Read-only in-app browsers in the Help menu for the bundled global `DOCUMENTATION.md`, the generated Lua API reference with public types and runnable examples, the [CLI/server/TUI documentation](CLI.md), and the [MCP server documentation](MCP.md), with safe GFM rendering and confirmed external links. - A localized OPML menu editor manages pages, submenus, and category archives with protected Home ordering, keyboard-accessible tree controls, drag-and-drop, and bDS2-compatible persistence. diff --git a/crates/bds-core/Cargo.toml b/crates/bds-core/Cargo.toml index 2dc8879..09132ce 100644 --- a/crates/bds-core/Cargo.toml +++ b/crates/bds-core/Cargo.toml @@ -14,7 +14,7 @@ serde_json = { workspace = true } serde_yaml = { workspace = true } chrono = { workspace = true } sha2 = { workspace = true } -deunicode = { workspace = true } +unicode-normalization = { workspace = true } thiserror = { workspace = true } walkdir = { workspace = true } image = { workspace = true } diff --git a/crates/bds-core/src/engine/preview.rs b/crates/bds-core/src/engine/preview.rs index b949995..7e4355f 100644 --- a/crates/bds-core/src/engine/preview.rs +++ b/crates/bds-core/src/engine/preview.rs @@ -1605,7 +1605,7 @@ mod tests { oldest.post.id = "post-1".into(); oldest.post.slug = "alpha".into(); oldest.post.title = "Alpha".into(); - oldest.post.tags = vec!["Rust".into()]; + oldest.post.tags = vec!["Über Öl".into()]; oldest.post.published_at = Some(1_709_568_000_000); oldest.post.created_at = 1_709_568_000_000; oldest.body_markdown = "Alpha body".into(); @@ -1614,7 +1614,7 @@ mod tests { newest.post.id = "post-2".into(); newest.post.slug = "beta".into(); newest.post.title = "Beta".into(); - newest.post.tags = vec!["Rust".into()]; + newest.post.tags = vec!["Über Öl".into()]; newest.post.published_at = Some(1_710_086_400_000); newest.post.created_at = 1_710_086_400_000; newest.body_markdown = "Beta body".into(); @@ -1640,7 +1640,7 @@ mod tests { "project-1", &metadata, &posts, - "/tag/rust", + "/tag/uber-ol", ) .unwrap(); assert_eq!(tag_first.status_code, 200); @@ -1653,7 +1653,7 @@ mod tests { "project-1", &metadata, &posts, - "/tag/rust/page/2", + "/tag/uber-ol/page/2", ) .unwrap(); assert_eq!(tag_second.status_code, 200); diff --git a/crates/bds-core/src/render/page_renderer.rs b/crates/bds-core/src/render/page_renderer.rs index 71491e3..905fb79 100644 --- a/crates/bds-core/src/render/page_renderer.rs +++ b/crates/bds-core/src/render/page_renderer.rs @@ -592,7 +592,7 @@ mod tests { ) .unwrap(); - assert_eq!(rendered, "ueber-die-bruecke"); + assert_eq!(rendered, "uber-die-brucke"); } #[test] diff --git a/crates/bds-core/src/render/site.rs b/crates/bds-core/src/render/site.rs index 99a2307..92b5808 100644 --- a/crates/bds-core/src/render/site.rs +++ b/crates/bds-core/src/render/site.rs @@ -1826,8 +1826,8 @@ mod menu_tests { slug: None, children: vec![MenuItem { kind: MenuItemKind::CategoryArchive, - label: "Long Form".into(), - slug: Some("Long Form".into()), + label: "Über Öl".into(), + slug: Some("Über Öl".into()), children: Vec::new(), }], }, @@ -1838,12 +1838,12 @@ mod menu_tests { let rendered = build_menu_items(dir.path(), "en", "en").unwrap(); assert_eq!(rendered[0]["href"], "/"); assert_eq!(rendered[1]["href"], "/about/"); - assert_eq!(rendered[2]["children"][0]["href"], "/category/long-form/"); + assert_eq!(rendered[2]["children"][0]["href"], "/category/uber-ol/"); let translated = build_menu_items(dir.path(), "de", "en").unwrap(); assert_eq!(translated[1]["href"], "/de/about/"); assert_eq!( translated[2]["children"][0]["href"], - "/de/category/long-form/" + "/de/category/uber-ol/" ); } } diff --git a/crates/bds-core/src/util/slug.rs b/crates/bds-core/src/util/slug.rs index 606af97..2f490ad 100644 --- a/crates/bds-core/src/util/slug.rs +++ b/crates/bds-core/src/util/slug.rs @@ -1,34 +1,21 @@ -use deunicode::deunicode; +use unicode_normalization::UnicodeNormalization; -/// Pre-process German characters to match TypeScript `transliteration` npm output. -/// deunicode maps ä→a, ö→o, ü→u but TypeScript produces ä→ae, ö→oe, ü→ue. -/// We replace these before deunicode so the slug output is compatible. +/// Apply the one German replacement that canonical decomposition cannot provide. fn german_transliterate(input: &str) -> String { - let mut result = String::with_capacity(input.len() + 16); - for c in input.chars() { - match c { - 'ä' => result.push_str("ae"), - 'ö' => result.push_str("oe"), - 'ü' => result.push_str("ue"), - 'Ä' => result.push_str("Ae"), - 'Ö' => result.push_str("Oe"), - 'Ü' => result.push_str("Ue"), - _ => result.push(c), - } - } - result + input.replace('ß', "ss") } /// Generate a URL-safe slug from a title string. /// -/// Transliterates Unicode to ASCII, lowercases, replaces non-alphanumeric -/// chars with hyphens, and collapses/trims hyphens. -/// -/// German umlauts (ä/ö/ü/Ä/Ö/Ü) are pre-processed to ae/oe/ue/Ae/Oe/Ue -/// to match TypeScript `transliteration` npm output. ß→ss is handled by deunicode. +/// Matches bDS2 exactly: replace ß with `ss`, canonically decompose Unicode, +/// discard non-ASCII code points, lowercase, replace non-alphanumeric runs +/// with hyphens, and trim leading/trailing hyphens. pub fn slugify(input: &str) -> String { let preprocessed = german_transliterate(input); - let ascii = deunicode(&preprocessed); + let ascii = preprocessed + .nfd() + .filter(char::is_ascii) + .collect::(); let lowered = ascii.to_lowercase(); let mut slug = String::with_capacity(lowered.len()); let mut prev_hyphen = true; // avoid leading hyphen @@ -79,7 +66,7 @@ mod tests { #[test] fn unicode_slug() { - assert_eq!(slugify("Über die Brücke"), "ueber-die-bruecke"); + assert_eq!(slugify("Über die Brücke"), "uber-die-brucke"); } #[test] @@ -122,24 +109,34 @@ mod tests { assert_eq!(slug, "hello-4"); } - // German umlaut tests — spec: "only German and English letters are used. - // Verify deunicode handles ä/ö/ü/ß/ÄÖÜ correctly against transliteration npm." - // Pre-processing maps ä→ae, ö→oe, ü→ue to match TypeScript transliteration npm. - // ß→ss is handled correctly by deunicode without pre-processing. + // German corpus copied from the bDS2 golden-master tests. + #[test] + fn bds2_german_transliteration_corpus() { + assert_eq!(slugify("Straße"), "strasse"); + assert_eq!(slugify("Öl"), "ol"); + assert_eq!(slugify("Äpfel"), "apfel"); + assert_eq!(slugify("Über"), "uber"); + assert_eq!(slugify("ÄÖÜäöüß"), "aouaouss"); + } + + #[test] + fn bds2_nfd_discards_non_ascii_instead_of_transliterating_it() { + assert_eq!(slugify("Crème 東京 œ"), "creme"); + } #[test] fn german_umlaut_ae() { - assert_eq!(slugify("Ärger"), "aerger"); + assert_eq!(slugify("Ärger"), "arger"); } #[test] fn german_umlaut_oe() { - assert_eq!(slugify("Öffnung"), "oeffnung"); + assert_eq!(slugify("Öffnung"), "offnung"); } #[test] fn german_umlaut_ue() { - assert_eq!(slugify("Über"), "ueber"); + assert_eq!(slugify("Über"), "uber"); } #[test] @@ -149,12 +146,12 @@ mod tests { #[test] fn german_mixed_umlauts() { - assert_eq!(slugify("Größe über Maße"), "groesse-ueber-masse"); + assert_eq!(slugify("Größe über Maße"), "grosse-uber-masse"); } #[test] fn german_uppercase_umlauts() { - assert_eq!(slugify("ÄÖÜ Test"), "aeoeue-test"); + assert_eq!(slugify("ÄÖÜ Test"), "aou-test"); } // spec: CreatePost uses Slug.generate(title ?? "untitled") diff --git a/crates/bds-core/tests/spec_claims.rs b/crates/bds-core/tests/spec_claims.rs index cc86fa2..1147f71 100644 --- a/crates/bds-core/tests/spec_claims.rs +++ b/crates/bds-core/tests/spec_claims.rs @@ -284,7 +284,7 @@ fn slug_generation_matches_spec() { assert_eq!(slugify("a --- b"), "a-b"); assert_eq!(slugify("---hello---"), "hello"); assert_eq!(slugify("café"), "cafe"); - assert_eq!(slugify("über"), "ueber"); + assert_eq!(slugify("über"), "uber"); assert_eq!(slugify("Straße"), "strasse"); assert_eq!(ensure_unique("test", |value| value == "test"), "test-2"); } diff --git a/specs/bds.allium b/specs/bds.allium index c3e26a0..3d71234 100644 --- a/specs/bds.allium +++ b/specs/bds.allium @@ -74,9 +74,9 @@ use "./metadata_diff.allium" as metadata_diff -- DB/filesystem diff and rebuil -- Resolved questions: -- --- 1. Slug generation scope: only German and English letters are used. --- Verify transliteration preserves the established bDS behaviour for --- ä/ö/ü/ß/ÄÖÜ. +-- 1. Slug generation is identical to bDS2: replace ß with "ss", normalize to +-- Unicode NFD, discard non-ASCII code points, lowercase, hyphenate +-- non-alphanumeric runs, and trim hyphens. Thus ä/ö/ü become a/o/u. -- -- 2. Liquid subset: see template.allium for the exact subset. -- Only 5 tags, 4 standard filters, 2 custom filters, 5 operators. diff --git a/specs/post.allium b/specs/post.allium index f5c9168..530fa40 100644 --- a/specs/post.allium +++ b/specs/post.allium @@ -14,10 +14,11 @@ enum PostStatus { value Slug { value: String - -- Generated by: transliterate unicode to ASCII, lowercase, - -- replace [^a-z0-9]+ with hyphens, strip leading/trailing hyphens - -- Transliteration scope: only German (ä/ö/ü/ß/ÄÖÜ) and English letters used. - -- Verify transliteration matches the established bDS behaviour for this set. + -- Generated exactly as bDS2: replace ß with "ss", normalize to Unicode NFD, + -- discard every non-ASCII code point, lowercase, replace [^a-z0-9]+ with + -- hyphens, then strip leading/trailing hyphens. + -- Therefore ä/ö/ü/Ä/Ö/Ü become a/o/u/A/O/U through NFD decomposition; + -- examples: "Über" -> "uber", "ÄÖÜäöüß" -> "aouaouss". -- Uniqueness: tries base, then {slug}-2, {slug}-3, … (unbounded numeric suffix) }