Apply Qwen NFC normalization before token splitting
This commit is contained in:
1 parent
5129c450ee
commit
6507a494bc
36 files changed
+1831
-39
No files matched your search
@@ -35,6 +35,7 @@ sha2 = "0.11.0"
|
||||
shlex = "2.0.1"
|
||||
time = { version = "0.3.54", features = ["formatting", "parsing"] }
|
||||
turbovault-parser = "1.6.0"
|
||||
unicode-normalization-alignments = "0.1.12"
|
||||
ureq = { version = "3.3.0", default-features = false, features = ["rustls"] }
|
||||
url = "2.5.8"
|
||||
usvg = "0.45.1"
|
||||
|
||||
Reference in new issue
Block a user