Align slug transliteration with bDS2

Cargo.lock

@@ -939,7 +939,6 @@ dependencies = [
  "axum",
  "base64 0.22.1",
  "chrono",
- "deunicode",
  "diesel",
  "diesel_migrations",
  "dirs 5.0.1",
@@ -969,6 +968,7 @@ dependencies = [
  "tempfile",
  "thiserror 2.0.19",
  "tokio",
+ "unicode-normalization",
  "url",
  "usearch",
  "uuid",
@@ -2489,12 +2489,6 @@ version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "21d8ad60dd5b13a4ee6bd8fa2d5d88965c597c67bce32b5fc49c94f55cb50810"
 
-[[package]]
-name = "deunicode"
-version = "1.6.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "abd57806937c9cc163efc8ea3910e00a62e2aeb0b8119f1793a978088f8f6b04"
-
 [[package]]
 name = "diesel"
 version = "2.3.11"

Cargo.toml

@@ -24,7 +24,7 @@ serde_json = "1"
 serde_yaml = "0.9"
 chrono = { version = "0.4", features = ["serde"] }
 sha2 = "0.10"
-deunicode = "1"
+unicode-normalization = "0.1"
 thiserror = "2"
 anyhow = "1"
 clap = { version = "4", features = ["derive"] }

README.md

@@ -11,7 +11,7 @@ The project is under active development. Core blogging workflows are broadly ava
 - Media import, thumbnails, metadata translations, filters, validation, post assignment, and sequential drag-and-drop insertion into post editors.
 - WordPress WXR migration with saved analyses, HTML-to-Markdown and shortcode conversion, conflict/taxonomy review, recoverable 500-item execution batches, media-parent linking, progress reporting, and optional AI-assisted taxonomy mapping.
 - Post, Liquid template, and Lua script editing with dedicated syntax highlighting and explicit syntax-check feedback, using a custom Ropey/Syntect/Cosmic Text editor and the documented, bDS2-signature-compatible project-scoped [`bds` host API](docs/scripting/API_REFERENCE.md) across utilities, rendered macros, and Blogmark transforms, including airplane-gated Git sync.
-- SQLite and filesystem persistence with frontmatter, sidecars, rebuild, metadata diff/repair, and FTS5 search.
+- SQLite and filesystem persistence with frontmatter, sidecars, rebuild, metadata diff/repair, bDS2-compatible NFD slug generation, and FTS5 search.
 - Optional on-device multilingual semantic search and similar-post tag suggestions backed by a persistent USearch index, plus always-available partial-name tag autocomplete and duplicate-post review in the desktop workspace.
 - Read-only in-app browsers in the Help menu for the bundled global `DOCUMENTATION.md`, the generated Lua API reference with public types and runnable examples, the [CLI/server/TUI documentation](CLI.md), and the [MCP server documentation](MCP.md), with safe GFM rendering and confirmed external links.
 - A localized OPML menu editor manages pages, submenus, and category archives with protected Home ordering, keyboard-accessible tree controls, drag-and-drop, and bDS2-compatible persistence.

crates/bds-core/Cargo.toml

@@ -14,7 +14,7 @@ serde_json = { workspace = true }
 serde_yaml = { workspace = true }
 chrono = { workspace = true }
 sha2 = { workspace = true }
-deunicode = { workspace = true }
+unicode-normalization = { workspace = true }
 thiserror = { workspace = true }
 walkdir = { workspace = true }
 image = { workspace = true }

crates/bds-core/src/engine/preview.rs

@@ -1605,7 +1605,7 @@ mod tests {
         oldest.post.id = "post-1".into();
         oldest.post.slug = "alpha".into();
         oldest.post.title = "Alpha".into();
-        oldest.post.tags = vec!["Rust".into()];
+        oldest.post.tags = vec!["Über Öl".into()];
         oldest.post.published_at = Some(1_709_568_000_000);
         oldest.post.created_at = 1_709_568_000_000;
         oldest.body_markdown = "Alpha body".into();
@@ -1614,7 +1614,7 @@ mod tests {
         newest.post.id = "post-2".into();
         newest.post.slug = "beta".into();
         newest.post.title = "Beta".into();
-        newest.post.tags = vec!["Rust".into()];
+        newest.post.tags = vec!["Über Öl".into()];
         newest.post.published_at = Some(1_710_086_400_000);
         newest.post.created_at = 1_710_086_400_000;
         newest.body_markdown = "Beta body".into();
@@ -1640,7 +1640,7 @@ mod tests {
             "project-1",
             &metadata,
             &posts,
-            "/tag/rust",
+            "/tag/uber-ol",
         )
         .unwrap();
         assert_eq!(tag_first.status_code, 200);
@@ -1653,7 +1653,7 @@ mod tests {
             "project-1",
             &metadata,
             &posts,
-            "/tag/rust/page/2",
+            "/tag/uber-ol/page/2",
         )
         .unwrap();
         assert_eq!(tag_second.status_code, 200);

crates/bds-core/src/render/page_renderer.rs

@@ -592,7 +592,7 @@ mod tests {
         )
         .unwrap();
 
-        assert_eq!(rendered, "ueber-die-bruecke");
+        assert_eq!(rendered, "uber-die-brucke");
     }
 
     #[test]

crates/bds-core/src/render/site.rs

@@ -1826,8 +1826,8 @@ mod menu_tests {
                     slug: None,
                     children: vec![MenuItem {
                         kind: MenuItemKind::CategoryArchive,
-                        label: "Long Form".into(),
+                        label: "Über Öl".into(),
-                        slug: Some("Long Form".into()),
+                        slug: Some("Über Öl".into()),
                         children: Vec::new(),
                     }],
                 },
@@ -1838,12 +1838,12 @@ mod menu_tests {
         let rendered = build_menu_items(dir.path(), "en", "en").unwrap();
         assert_eq!(rendered[0]["href"], "/");
         assert_eq!(rendered[1]["href"], "/about/");
-        assert_eq!(rendered[2]["children"][0]["href"], "/category/long-form/");
+        assert_eq!(rendered[2]["children"][0]["href"], "/category/uber-ol/");
         let translated = build_menu_items(dir.path(), "de", "en").unwrap();
         assert_eq!(translated[1]["href"], "/de/about/");
         assert_eq!(
             translated[2]["children"][0]["href"],
-            "/de/category/long-form/"
+            "/de/category/uber-ol/"
         );
     }
 }

crates/bds-core/src/util/slug.rs

@@ -1,34 +1,21 @@
-use deunicode::deunicode;
+use unicode_normalization::UnicodeNormalization;
 
-/// Pre-process German characters to match TypeScript `transliteration` npm output.
+/// Apply the one German replacement that canonical decomposition cannot provide.
-/// deunicode maps ä→a, ö→o, ü→u but TypeScript produces ä→ae, ö→oe, ü→ue.
-/// We replace these before deunicode so the slug output is compatible.
 fn german_transliterate(input: &str) -> String {
-    let mut result = String::with_capacity(input.len() + 16);
+    input.replace('ß', "ss")
-    for c in input.chars() {
-        match c {
-            'ä' => result.push_str("ae"),
-            'ö' => result.push_str("oe"),
-            'ü' => result.push_str("ue"),
-            'Ä' => result.push_str("Ae"),
-            'Ö' => result.push_str("Oe"),
-            'Ü' => result.push_str("Ue"),
-            _ => result.push(c),
-        }
-    }
-    result
 }
 
 /// Generate a URL-safe slug from a title string.
 ///
-/// Transliterates Unicode to ASCII, lowercases, replaces non-alphanumeric
+/// Matches bDS2 exactly: replace ß with `ss`, canonically decompose Unicode,
-/// chars with hyphens, and collapses/trims hyphens.
+/// discard non-ASCII code points, lowercase, replace non-alphanumeric runs
-///
+/// with hyphens, and trim leading/trailing hyphens.
-/// German umlauts (ä/ö/ü/Ä/Ö/Ü) are pre-processed to ae/oe/ue/Ae/Oe/Ue
-/// to match TypeScript `transliteration` npm output. ß→ss is handled by deunicode.
 pub fn slugify(input: &str) -> String {
     let preprocessed = german_transliterate(input);
-    let ascii = deunicode(&preprocessed);
+    let ascii = preprocessed
+        .nfd()
+        .filter(char::is_ascii)
+        .collect::<String>();
     let lowered = ascii.to_lowercase();
     let mut slug = String::with_capacity(lowered.len());
     let mut prev_hyphen = true; // avoid leading hyphen
@@ -79,7 +66,7 @@ mod tests {
 
     #[test]
     fn unicode_slug() {
-        assert_eq!(slugify("Über die Brücke"), "ueber-die-bruecke");
+        assert_eq!(slugify("Über die Brücke"), "uber-die-brucke");
     }
 
     #[test]
@@ -122,24 +109,34 @@ mod tests {
         assert_eq!(slug, "hello-4");
     }
 
-    // German umlaut tests — spec: "only German and English letters are used.
+    // German corpus copied from the bDS2 golden-master tests.
-    // Verify deunicode handles ä/ö/ü/ß/ÄÖÜ correctly against transliteration npm."
+    #[test]
-    // Pre-processing maps ä→ae, ö→oe, ü→ue to match TypeScript transliteration npm.
+    fn bds2_german_transliteration_corpus() {
-    // ß→ss is handled correctly by deunicode without pre-processing.
+        assert_eq!(slugify("Straße"), "strasse");
+        assert_eq!(slugify("Öl"), "ol");
+        assert_eq!(slugify("Äpfel"), "apfel");
+        assert_eq!(slugify("Über"), "uber");
+        assert_eq!(slugify("ÄÖÜäöüß"), "aouaouss");
+    }
+
+    #[test]
+    fn bds2_nfd_discards_non_ascii_instead_of_transliterating_it() {
+        assert_eq!(slugify("Crème 東京 œ"), "creme");
+    }
 
     #[test]
     fn german_umlaut_ae() {
-        assert_eq!(slugify("Ärger"), "aerger");
+        assert_eq!(slugify("Ärger"), "arger");
     }
 
     #[test]
     fn german_umlaut_oe() {
-        assert_eq!(slugify("Öffnung"), "oeffnung");
+        assert_eq!(slugify("Öffnung"), "offnung");
     }
 
     #[test]
     fn german_umlaut_ue() {
-        assert_eq!(slugify("Über"), "ueber");
+        assert_eq!(slugify("Über"), "uber");
     }
 
     #[test]
@@ -149,12 +146,12 @@ mod tests {
 
     #[test]
     fn german_mixed_umlauts() {
-        assert_eq!(slugify("Größe über Maße"), "groesse-ueber-masse");
+        assert_eq!(slugify("Größe über Maße"), "grosse-uber-masse");
     }
 
     #[test]
     fn german_uppercase_umlauts() {
-        assert_eq!(slugify("ÄÖÜ Test"), "aeoeue-test");
+        assert_eq!(slugify("ÄÖÜ Test"), "aou-test");
     }
 
     // spec: CreatePost uses Slug.generate(title ?? "untitled")

crates/bds-core/tests/spec_claims.rs

@@ -284,7 +284,7 @@ fn slug_generation_matches_spec() {
     assert_eq!(slugify("a --- b"), "a-b");
     assert_eq!(slugify("---hello---"), "hello");
     assert_eq!(slugify("café"), "cafe");
-    assert_eq!(slugify("über"), "ueber");
+    assert_eq!(slugify("über"), "uber");
     assert_eq!(slugify("Straße"), "strasse");
     assert_eq!(ensure_unique("test", |value| value == "test"), "test-2");
 }

specs/bds.allium

@@ -74,9 +74,9 @@ use "./metadata_diff.allium" as metadata_diff   -- DB/filesystem diff and rebuil
 
 -- Resolved questions:
 --
--- 1. Slug generation scope: only German and English letters are used.
+-- 1. Slug generation is identical to bDS2: replace ß with "ss", normalize to
---    Verify transliteration preserves the established bDS behaviour for
+--    Unicode NFD, discard non-ASCII code points, lowercase, hyphenate
---    ä/ö/ü/ß/ÄÖÜ.
+--    non-alphanumeric runs, and trim hyphens. Thus ä/ö/ü become a/o/u.
 --
 -- 2. Liquid subset: see template.allium for the exact subset.
 --    Only 5 tags, 4 standard filters, 2 custom filters, 5 operators.

specs/post.allium

@@ -14,10 +14,11 @@ enum PostStatus {
 value Slug {
     value: String
 
-    -- Generated by: transliterate unicode to ASCII, lowercase,
+    -- Generated exactly as bDS2: replace ß with "ss", normalize to Unicode NFD,
-    -- replace [^a-z0-9]+ with hyphens, strip leading/trailing hyphens
+    -- discard every non-ASCII code point, lowercase, replace [^a-z0-9]+ with
-    -- Transliteration scope: only German (ä/ö/ü/ß/ÄÖÜ) and English letters used.
+    -- hyphens, then strip leading/trailing hyphens.
-    -- Verify transliteration matches the established bDS behaviour for this set.
+    -- Therefore ä/ö/ü/Ä/Ö/Ü become a/o/u/A/O/U through NFD decomposition;
+    -- examples: "Über" -> "uber", "ÄÖÜäöüß" -> "aouaouss".
     -- Uniqueness: tries base, then {slug}-2, {slug}-3, … (unbounded numeric suffix)
 }