Apply Qwen NFC normalization before token splitting
This commit is contained in:
1 parent
5129c450ee
commit
6507a494bc
36 files changed
+1831
-39
No files matched your search
@@ -0,0 +1,16 @@
|
||||
|
||||
running 3 tests
|
||||
test engine::tokenizer::issue111::issue111_ds4_families ... {"case":"ds4_control","family":"glm","cases":19,"plain_rendered_exact":true}
|
||||
{"case":"ds4_control","family":"deepseek","cases":19,"plain_rendered_exact":true}
|
||||
ok
|
||||
test engine::tokenizer::issue111::issue111_every_unicode_scalar_split ... {"case":"unicode_split","scalars":1112064,"pieces":2082513,"sha256":"7ef8755fb56062d9695b38bc132f325ea59c3c65f198e1b1ab20bff1afef632a","exact":true}
|
||||
ok
|
||||
test engine::tokenizer::issue111::issue111_qwen_tokens_and_requests ... {"case":"request","effort":"direct","context_tokens":59,"exact":true}
|
||||
{"case":"request","effort":"low","context_tokens":93,"exact":true}
|
||||
{"case":"request","effort":"medium","context_tokens":63,"exact":true}
|
||||
{"case":"request","effort":"xhigh","context_tokens":105,"exact":true}
|
||||
{"case":"tokenizer","number_count":1911,"cases":5752,"plain_rendered_roundtrip_exact":true}
|
||||
ok
|
||||
|
||||
test result: ok. 3 passed; 0 failed; 0 ignored; 0 measured; 552 filtered out; finished in 6.32s
|
||||
|
||||
Reference in new issue
Block a user