20 lines
3.1 KiB
Plaintext
20 lines
3.1 KiB
Plaintext
|
||
running 1 test
|
||
test engine::metal::qwen_mtplx::issue103_analysis::issue103_tokenizer_receipts ... {"case":"native_tokenizer","text":"भारत एक विशाल देश है जिसकी संस्कृति बहुत पुरानी है।","actual_ids":[149755,148504,76519,171549,165997,151224,197734,170046,184864,160357,172281,237627,158827,153570,199638,80752,151859,181117,170046,154845],"expected_ids":[149755,148504,76519,171549,165997,151224,197734,170046,184864,160357,172281,237627,158827,153570,199638,80752,151859,181117,170046,154845],"matches":true}
|
||
{"case":"native_tokenizer","text":"ประเทศไทยมีวัฒนธรรมที่เก่าแก่และสวยงามมาก","actual_ids":[156150,148450,186319,148285,178118,155545,149119,162566,149196],"expected_ids":[156150,148450,186319,148285,178118,155545,149119,162566,149196],"matches":true}
|
||
{"case":"native_tokenizer","text":"إِنَّ اللُّغَةَ الْعَرَبِيَّةَ جَمِيلَةٌ","actual_ids":[88792,164372,148784,159613,78958,191556,205629,153458,150765,151003,178402,27004,79333,152397,216808,233790],"expected_ids":[88792,164372,148784,159613,78958,191556,205629,153458,150765,151003,178402,27004,79333,152397,216808,233790],"matches":true}
|
||
{"case":"native_tokenizer","text":"The quick brown fox jumps over the lazy dog near the river bank.","actual_ids":[760,3841,13477,37550,33075,888,279,15217,5388,3043,279,14367,5883,13],"expected_ids":[760,3841,13477,37550,33075,888,279,15217,5388,3043,279,14367,5883,13],"matches":true}
|
||
{"case":"native_tokenizer","text":"def foo(x):\n return x + 1\n","actual_ids":[727,14785,2007,1590,198,262,460,830,478,220,16,198],"expected_ids":[727,14785,2007,1590,198,262,460,830,478,220,16,198],"matches":true}
|
||
{"case":"native_tokenizer","text":"中国的文化历史悠久。","actual_ids":[108733,96655,125188,1710],"expected_ids":[108733,96655,125188,1710],"matches":true}
|
||
{"case":"native_tokenizer","text":"a🙂b ©c ∑d —e …f €g ™h\n","actual_ids":[64,169171,65,7013,66,11654,239,67,1892,68,4444,69,12612,70,27717,95,71,198],"expected_ids":[64,169171,65,7013,66,11654,239,67,1892,68,4444,69,12612,70,27717,95,71,198],"matches":true}
|
||
{"case":"native_tokenizer","text":"x́y १२३ ١٢٣ ①Ⅷ²Z","actual_ids":[87,52033,88,220,12256,100,12256,101,12256,102,220,149,94,149,95,149,96,2765,239,254,68086,100,28495,373,235,57],"expected_ids":[87,52033,88,220,12256,100,12256,101,12256,102,220,149,94,149,95,149,96,220,46701,254,68086,100,28495,373,235,57],"matches":false}
|
||
{"case":"native_tokenizer","text":" ①","actual_ids":[2765,239,254],"expected_ids":[220,46701,254],"matches":false}
|
||
{"case":"native_tokenizer","text":" Ⅷ","actual_ids":[2765,88608],"expected_ids":[220,68086,100],"matches":false}
|
||
{"case":"native_tokenizer","text":" ²","actual_ids":[2071,110],"expected_ids":[220,28495],"matches":false}
|
||
{"case":"native_tokenizer","text":" 𝟘","actual_ids":[78449,253,246],"expected_ids":[220,54362,253,246],"matches":false}
|
||
{"case":"tokenizer_summary","mismatches":5}
|
||
ok
|
||
|
||
test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 516 filtered out; finished in 0.47s
|
||
|