Apply Qwen NFC normalization before token splitting
This commit is contained in:
1 parent
5129c450ee
commit
6507a494bc
36 files changed
+1831
-39
No files matched your search
Generated
+10
@@ -834,6 +834,7 @@ dependencies = [
|
||||
"shlex",
|
||||
"time",
|
||||
"turbovault-parser",
|
||||
"unicode-normalization-alignments",
|
||||
"ureq",
|
||||
"url",
|
||||
"usvg",
|
||||
@@ -3948,6 +3949,15 @@ version = "0.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3b09c83c3c29d37506a3e260c08c03743a6bb66a9cd432c6934ab501a190571f"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-normalization-alignments"
|
||||
version = "0.1.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "43f613e4fa046e69818dd287fdc4bc78175ff20331479dab6e1b0f98d57062de"
|
||||
dependencies = [
|
||||
"smallvec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "unicode-properties"
|
||||
version = "0.1.4"
|
||||
|
||||
Reference in new issue
Block a user