hugo/RuDS
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

Align slug transliteration with bDS2

Browse Source
This commit is contained in:
Chili Palmer
2026-07-20 23:01:12 +02:00
parent b641f6492b
commit ac993d8743
11 changed files with 52 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
crates/bds-core/Cargo.toml
View File

@@ -14,7 +14,7 @@ serde_json = { workspace = true }
serde_yaml = { workspace = true }
chrono = { workspace = true }
sha2 = { workspace = true }
deunicode = { workspace = true }
unicode-normalization = { workspace = true }
thiserror = { workspace = true }
walkdir = { workspace = true }
image = { workspace = true }

8
crates/bds-core/src/engine/preview.rs
View File

@@ -1605,7 +1605,7 @@ mod tests {
oldest.post.id = "post-1".into();
oldest.post.slug = "alpha".into();
oldest.post.title = "Alpha".into();
oldest.post.tags = vec!["Rust".into()];
oldest.post.tags = vec!["Über Öl".into()];
oldest.post.published_at = Some(1_709_568_000_000);
oldest.post.created_at = 1_709_568_000_000;
oldest.body_markdown = "Alpha body".into();
@@ -1614,7 +1614,7 @@ mod tests {
newest.post.id = "post-2".into();
newest.post.slug = "beta".into();
newest.post.title = "Beta".into();
newest.post.tags = vec!["Rust".into()];
newest.post.tags = vec!["Über Öl".into()];
newest.post.published_at = Some(1_710_086_400_000);
newest.post.created_at = 1_710_086_400_000;
newest.body_markdown = "Beta body".into();
@@ -1640,7 +1640,7 @@ mod tests {
"project-1",
&metadata,
&posts,
"/tag/rust",
"/tag/uber-ol",
)
.unwrap();
assert_eq!(tag_first.status_code, 200);
@@ -1653,7 +1653,7 @@ mod tests {
"project-1",
&metadata,
&posts,
"/tag/rust/page/2",
"/tag/uber-ol/page/2",
)
.unwrap();
assert_eq!(tag_second.status_code, 200);

2
crates/bds-core/src/render/page_renderer.rs
View File

@@ -592,7 +592,7 @@ mod tests {
)
.unwrap();
assert_eq!(rendered, "ueber-die-bruecke");
assert_eq!(rendered, "uber-die-brucke");
}
#[test]

8
crates/bds-core/src/render/site.rs
View File

@@ -1826,8 +1826,8 @@ mod menu_tests {
slug: None,
children: vec![MenuItem {
kind: MenuItemKind::CategoryArchive,
label: "Long Form".into(),
slug: Some("Long Form".into()),
label: "Über Öl".into(),
slug: Some("Über Öl".into()),
children: Vec::new(),
}],
},
@@ -1838,12 +1838,12 @@ mod menu_tests {
let rendered = build_menu_items(dir.path(), "en", "en").unwrap();
assert_eq!(rendered[0]["href"], "/");
assert_eq!(rendered[1]["href"], "/about/");
assert_eq!(rendered[2]["children"][0]["href"], "/category/long-form/");
assert_eq!(rendered[2]["children"][0]["href"], "/category/uber-ol/");
let translated = build_menu_items(dir.path(), "de", "en").unwrap();
assert_eq!(translated[1]["href"], "/de/about/");
assert_eq!(
translated[2]["children"][0]["href"],
"/de/category/long-form/"
"/de/category/uber-ol/"
);
}
}

63
crates/bds-core/src/util/slug.rs
View File

@@ -1,34 +1,21 @@
use deunicode::deunicode;
use unicode_normalization::UnicodeNormalization;
/// Pre-process German characters to match TypeScript `transliteration` npm output.
/// deunicode maps ä→a, ö→o, ü→u but TypeScript produces ä→ae, ö→oe, ü→ue.
/// We replace these before deunicode so the slug output is compatible.
/// Apply the one German replacement that canonical decomposition cannot provide.
fn german_transliterate(input: &str) -> String {
let mut result = String::with_capacity(input.len() + 16);
for c in input.chars() {
match c {
'ä' => result.push_str("ae"),
'ö' => result.push_str("oe"),
'ü' => result.push_str("ue"),
'Ä' => result.push_str("Ae"),
'Ö' => result.push_str("Oe"),
'Ü' => result.push_str("Ue"),
_ => result.push(c),
}
}
result
input.replace('ß', "ss")
}
/// Generate a URL-safe slug from a title string.
///
/// Transliterates Unicode to ASCII, lowercases, replaces non-alphanumeric
/// chars with hyphens, and collapses/trims hyphens.
///
/// German umlauts (ä/ö/ü/Ä/Ö/Ü) are pre-processed to ae/oe/ue/Ae/Oe/Ue
/// to match TypeScript `transliteration` npm output. ß→ss is handled by deunicode.
/// Matches bDS2 exactly: replace ß with `ss`, canonically decompose Unicode,
/// discard non-ASCII code points, lowercase, replace non-alphanumeric runs
/// with hyphens, and trim leading/trailing hyphens.
pub fn slugify(input: &str) -> String {
let preprocessed = german_transliterate(input);
let ascii = deunicode(&preprocessed);
let ascii = preprocessed
.nfd()
.filter(char::is_ascii)
.collect::<String>();
let lowered = ascii.to_lowercase();
let mut slug = String::with_capacity(lowered.len());
let mut prev_hyphen = true; // avoid leading hyphen
@@ -79,7 +66,7 @@ mod tests {
#[test]
fn unicode_slug() {
assert_eq!(slugify("Über die Brücke"), "ueber-die-bruecke");
assert_eq!(slugify("Über die Brücke"), "uber-die-brucke");
}
#[test]
@@ -122,24 +109,34 @@ mod tests {
assert_eq!(slug, "hello-4");
}
// German umlaut tests — spec: "only German and English letters are used.
// Verify deunicode handles ä/ö/ü/ß/ÄÖÜ correctly against transliteration npm."
// Pre-processing maps ä→ae, ö→oe, ü→ue to match TypeScript transliteration npm.
// ß→ss is handled correctly by deunicode without pre-processing.
// German corpus copied from the bDS2 golden-master tests.
#[test]
fn bds2_german_transliteration_corpus() {
assert_eq!(slugify("Straße"), "strasse");
assert_eq!(slugify("Öl"), "ol");
assert_eq!(slugify("Äpfel"), "apfel");
assert_eq!(slugify("Über"), "uber");
assert_eq!(slugify("ÄÖÜäöüß"), "aouaouss");
}
#[test]
fn bds2_nfd_discards_non_ascii_instead_of_transliterating_it() {
assert_eq!(slugify("Crème 東京 œ"), "creme");
}
#[test]
fn german_umlaut_ae() {
assert_eq!(slugify("Ärger"), "aerger");
assert_eq!(slugify("Ärger"), "arger");
}
#[test]
fn german_umlaut_oe() {
assert_eq!(slugify("Öffnung"), "oeffnung");
assert_eq!(slugify("Öffnung"), "offnung");
}
#[test]
fn german_umlaut_ue() {
assert_eq!(slugify("Über"), "ueber");
assert_eq!(slugify("Über"), "uber");
}
#[test]
@@ -149,12 +146,12 @@ mod tests {
#[test]
fn german_mixed_umlauts() {
assert_eq!(slugify("Größe über Maße"), "groesse-ueber-masse");
assert_eq!(slugify("Größe über Maße"), "grosse-uber-masse");
}
#[test]
fn german_uppercase_umlauts() {
assert_eq!(slugify("ÄÖÜ Test"), "aeoeue-test");
assert_eq!(slugify("ÄÖÜ Test"), "aou-test");
}
// spec: CreatePost uses Slug.generate(title ?? "untitled")

2
crates/bds-core/tests/spec_claims.rs
View File

@@ -284,7 +284,7 @@ fn slug_generation_matches_spec() {
assert_eq!(slugify("a --- b"), "a-b");
assert_eq!(slugify("---hello---"), "hello");
assert_eq!(slugify("café"), "cafe");
assert_eq!(slugify("über"), "ueber");
assert_eq!(slugify("über"), "uber");
assert_eq!(slugify("Straße"), "strasse");
assert_eq!(ensure_unique("test", |value| value == "test"), "test-2");
}