hugo/RuDS
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

Align slug transliteration with bDS2

Browse Source
This commit is contained in:
Chili Palmer
2026-07-20 23:01:12 +02:00
parent b641f6492b
commit ac993d8743
11 changed files with 52 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
Cargo.lock generated
View File

@@ -939,7 +939,6 @@ dependencies = [
"axum",
"base64 0.22.1",
"chrono",
"deunicode",
"diesel",
"diesel_migrations",
"dirs 5.0.1",
@@ -969,6 +968,7 @@ dependencies = [
"tempfile",
"thiserror 2.0.19",
"tokio",
"unicode-normalization",
"url",
"usearch",
"uuid",
@@ -2489,12 +2489,6 @@ version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "21d8ad60dd5b13a4ee6bd8fa2d5d88965c597c67bce32b5fc49c94f55cb50810"
[[package]]
name = "deunicode"
version = "1.6.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "abd57806937c9cc163efc8ea3910e00a62e2aeb0b8119f1793a978088f8f6b04"
[[package]]
name = "diesel"
version = "2.3.11"

2
Cargo.toml
View File

@@ -24,7 +24,7 @@ serde_json = "1"
serde_yaml = "0.9"
chrono = { version = "0.4", features = ["serde"] }
sha2 = "0.10"
deunicode = "1"
unicode-normalization = "0.1"
thiserror = "2"
anyhow = "1"
clap = { version = "4", features = ["derive"] }

2
README.md
View File

@@ -11,7 +11,7 @@ The project is under active development. Core blogging workflows are broadly ava
- Media import, thumbnails, metadata translations, filters, validation, post assignment, and sequential drag-and-drop insertion into post editors.
- WordPress WXR migration with saved analyses, HTML-to-Markdown and shortcode conversion, conflict/taxonomy review, recoverable 500-item execution batches, media-parent linking, progress reporting, and optional AI-assisted taxonomy mapping.
- Post, Liquid template, and Lua script editing with dedicated syntax highlighting and explicit syntax-check feedback, using a custom Ropey/Syntect/Cosmic Text editor and the documented, bDS2-signature-compatible project-scoped [`bds` host API](docs/scripting/API_REFERENCE.md) across utilities, rendered macros, and Blogmark transforms, including airplane-gated Git sync.
- SQLite and filesystem persistence with frontmatter, sidecars, rebuild, metadata diff/repair, and FTS5 search.
- SQLite and filesystem persistence with frontmatter, sidecars, rebuild, metadata diff/repair, bDS2-compatible NFD slug generation, and FTS5 search.
- Optional on-device multilingual semantic search and similar-post tag suggestions backed by a persistent USearch index, plus always-available partial-name tag autocomplete and duplicate-post review in the desktop workspace.
- Read-only in-app browsers in the Help menu for the bundled global `DOCUMENTATION.md`, the generated Lua API reference with public types and runnable examples, the [CLI/server/TUI documentation](CLI.md), and the [MCP server documentation](MCP.md), with safe GFM rendering and confirmed external links.
- A localized OPML menu editor manages pages, submenus, and category archives with protected Home ordering, keyboard-accessible tree controls, drag-and-drop, and bDS2-compatible persistence.

2
crates/bds-core/Cargo.toml
View File

@@ -14,7 +14,7 @@ serde_json = { workspace = true }
serde_yaml = { workspace = true }
chrono = { workspace = true }
sha2 = { workspace = true }
deunicode = { workspace = true }
unicode-normalization = { workspace = true }
thiserror = { workspace = true }
walkdir = { workspace = true }
image = { workspace = true }

8
crates/bds-core/src/engine/preview.rs
View File

@@ -1605,7 +1605,7 @@ mod tests {
oldest.post.id = "post-1".into();
oldest.post.slug = "alpha".into();
oldest.post.title = "Alpha".into();
oldest.post.tags = vec!["Rust".into()];
oldest.post.tags = vec!["Über Öl".into()];
oldest.post.published_at = Some(1_709_568_000_000);
oldest.post.created_at = 1_709_568_000_000;
oldest.body_markdown = "Alpha body".into();
@@ -1614,7 +1614,7 @@ mod tests {
newest.post.id = "post-2".into();
newest.post.slug = "beta".into();
newest.post.title = "Beta".into();
newest.post.tags = vec!["Rust".into()];
newest.post.tags = vec!["Über Öl".into()];
newest.post.published_at = Some(1_710_086_400_000);
newest.post.created_at = 1_710_086_400_000;
newest.body_markdown = "Beta body".into();
@@ -1640,7 +1640,7 @@ mod tests {
"project-1",
&metadata,
&posts,
"/tag/rust",
"/tag/uber-ol",
)
.unwrap();
assert_eq!(tag_first.status_code, 200);
@@ -1653,7 +1653,7 @@ mod tests {
"project-1",
&metadata,
&posts,
"/tag/rust/page/2",
"/tag/uber-ol/page/2",
)
.unwrap();
assert_eq!(tag_second.status_code, 200);

2
crates/bds-core/src/render/page_renderer.rs
View File

@@ -592,7 +592,7 @@ mod tests {
)
.unwrap();
assert_eq!(rendered, "ueber-die-bruecke");
assert_eq!(rendered, "uber-die-brucke");
}
#[test]

8
crates/bds-core/src/render/site.rs
View File

@@ -1826,8 +1826,8 @@ mod menu_tests {
slug: None,
children: vec![MenuItem {
kind: MenuItemKind::CategoryArchive,
label: "Long Form".into(),
slug: Some("Long Form".into()),
label: "Über Öl".into(),
slug: Some("Über Öl".into()),
children: Vec::new(),
}],
},
@@ -1838,12 +1838,12 @@ mod menu_tests {
let rendered = build_menu_items(dir.path(), "en", "en").unwrap();
assert_eq!(rendered[0]["href"], "/");
assert_eq!(rendered[1]["href"], "/about/");
assert_eq!(rendered[2]["children"][0]["href"], "/category/long-form/");
assert_eq!(rendered[2]["children"][0]["href"], "/category/uber-ol/");
let translated = build_menu_items(dir.path(), "de", "en").unwrap();
assert_eq!(translated[1]["href"], "/de/about/");
assert_eq!(
translated[2]["children"][0]["href"],
"/de/category/long-form/"
"/de/category/uber-ol/"
);
}
}

63
crates/bds-core/src/util/slug.rs
View File

@@ -1,34 +1,21 @@
use deunicode::deunicode;
use unicode_normalization::UnicodeNormalization;
/// Pre-process German characters to match TypeScript `transliteration` npm output.
/// deunicode maps ä→a, ö→o, ü→u but TypeScript produces ä→ae, ö→oe, ü→ue.
/// We replace these before deunicode so the slug output is compatible.
/// Apply the one German replacement that canonical decomposition cannot provide.
fn german_transliterate(input: &str) -> String {
let mut result = String::with_capacity(input.len() + 16);
for c in input.chars() {
match c {
'ä' => result.push_str("ae"),
'ö' => result.push_str("oe"),
'ü' => result.push_str("ue"),
'Ä' => result.push_str("Ae"),
'Ö' => result.push_str("Oe"),
'Ü' => result.push_str("Ue"),
_ => result.push(c),
}
}
result
input.replace('ß', "ss")
}
/// Generate a URL-safe slug from a title string.
///
/// Transliterates Unicode to ASCII, lowercases, replaces non-alphanumeric
/// chars with hyphens, and collapses/trims hyphens.
///
/// German umlauts (ä/ö/ü/Ä/Ö/Ü) are pre-processed to ae/oe/ue/Ae/Oe/Ue
/// to match TypeScript `transliteration` npm output. ß→ss is handled by deunicode.
/// Matches bDS2 exactly: replace ß with `ss`, canonically decompose Unicode,
/// discard non-ASCII code points, lowercase, replace non-alphanumeric runs
/// with hyphens, and trim leading/trailing hyphens.
pub fn slugify(input: &str) -> String {
let preprocessed = german_transliterate(input);
let ascii = deunicode(&preprocessed);
let ascii = preprocessed
.nfd()
.filter(char::is_ascii)
.collect::<String>();
let lowered = ascii.to_lowercase();
let mut slug = String::with_capacity(lowered.len());
let mut prev_hyphen = true; // avoid leading hyphen
@@ -79,7 +66,7 @@ mod tests {
#[test]
fn unicode_slug() {
assert_eq!(slugify("Über die Brücke"), "ueber-die-bruecke");
assert_eq!(slugify("Über die Brücke"), "uber-die-brucke");
}
#[test]
@@ -122,24 +109,34 @@ mod tests {
assert_eq!(slug, "hello-4");
}
// German umlaut tests — spec: "only German and English letters are used.
// Verify deunicode handles ä/ö/ü/ß/ÄÖÜ correctly against transliteration npm."
// Pre-processing maps ä→ae, ö→oe, ü→ue to match TypeScript transliteration npm.
// ß→ss is handled correctly by deunicode without pre-processing.
// German corpus copied from the bDS2 golden-master tests.
#[test]
fn bds2_german_transliteration_corpus() {
assert_eq!(slugify("Straße"), "strasse");
assert_eq!(slugify("Öl"), "ol");
assert_eq!(slugify("Äpfel"), "apfel");
assert_eq!(slugify("Über"), "uber");
assert_eq!(slugify("ÄÖÜäöüß"), "aouaouss");
}
#[test]
fn bds2_nfd_discards_non_ascii_instead_of_transliterating_it() {
assert_eq!(slugify("Crème 東京 œ"), "creme");
}
#[test]
fn german_umlaut_ae() {
assert_eq!(slugify("Ärger"), "aerger");
assert_eq!(slugify("Ärger"), "arger");
}
#[test]
fn german_umlaut_oe() {
assert_eq!(slugify("Öffnung"), "oeffnung");
assert_eq!(slugify("Öffnung"), "offnung");
}
#[test]
fn german_umlaut_ue() {
assert_eq!(slugify("Über"), "ueber");
assert_eq!(slugify("Über"), "uber");
}
#[test]
@@ -149,12 +146,12 @@ mod tests {
#[test]
fn german_mixed_umlauts() {
assert_eq!(slugify("Größe über Maße"), "groesse-ueber-masse");
assert_eq!(slugify("Größe über Maße"), "grosse-uber-masse");
}
#[test]
fn german_uppercase_umlauts() {
assert_eq!(slugify("ÄÖÜ Test"), "aeoeue-test");
assert_eq!(slugify("ÄÖÜ Test"), "aou-test");
}
// spec: CreatePost uses Slug.generate(title ?? "untitled")

2
crates/bds-core/tests/spec_claims.rs
View File

@@ -284,7 +284,7 @@ fn slug_generation_matches_spec() {
assert_eq!(slugify("a --- b"), "a-b");
assert_eq!(slugify("---hello---"), "hello");
assert_eq!(slugify("café"), "cafe");
assert_eq!(slugify("über"), "ueber");
assert_eq!(slugify("über"), "uber");
assert_eq!(slugify("Straße"), "strasse");
assert_eq!(ensure_unique("test", |value| value == "test"), "test-2");
}

6
specs/bds.allium
View File

@@ -74,9 +74,9 @@ use "./metadata_diff.allium" as metadata_diff -- DB/filesystem diff and rebuil
-- Resolved questions:
--
-- 1. Slug generation scope: only German and English letters are used.
-- Verify transliteration preserves the established bDS behaviour for
-- ä/ö/ü/ß/ÄÖÜ.
-- 1. Slug generation is identical to bDS2: replace ß with "ss", normalize to
-- Unicode NFD, discard non-ASCII code points, lowercase, hyphenate
-- non-alphanumeric runs, and trim hyphens. Thus ä/ö/ü become a/o/u.
--
-- 2. Liquid subset: see template.allium for the exact subset.
-- Only 5 tags, 4 standard filters, 2 custom filters, 5 operators.

9
specs/post.allium
View File

@@ -14,10 +14,11 @@ enum PostStatus {
value Slug {
value: String
-- Generated by: transliterate unicode to ASCII, lowercase,
-- replace [^a-z0-9]+ with hyphens, strip leading/trailing hyphens
-- Transliteration scope: only German (ä/ö/ü/ß/ÄÖÜ) and English letters used.
-- Verify transliteration matches the established bDS behaviour for this set.
-- Generated exactly as bDS2: replace ß with "ss", normalize to Unicode NFD,
-- discard every non-ASCII code point, lowercase, replace [^a-z0-9]+ with
-- hyphens, then strip leading/trailing hyphens.
-- Therefore ä/ö/ü/Ä/Ö/Ü become a/o/u/A/O/U through NFD decomposition;
-- examples: "Über" -> "uber", "ÄÖÜäöüß" -> "aouaouss".
-- Uniqueness: tries base, then {slug}-2, {slug}-3, … (unbounded numeric suffix)
}