{"case": "native_tokenizer", "text": "ประเทศไทยมีวัฒนธรรมที่เก่าแก่และสวยงามมาก", "actual_ids": [156150, 148450, 186319, 148285, 178118, 155545, 149119, 162566, 149196], "expected_ids": [156150, 148450, 186319, 148285, 178118, 155545, 149119, 162566, 149196], "matches": true}
{"case": "native_tokenizer", "text": "إِنَّ اللُّغَةَ الْعَرَبِيَّةَ جَمِيلَةٌ", "actual_ids": [88792, 164372, 148784, 159613, 78958, 191556, 205629, 153458, 150765, 151003, 178402, 27004, 79333, 152397, 216808, 233790], "expected_ids": [88792, 164372, 148784, 159613, 78958, 191556, 205629, 153458, 150765, 151003, 178402, 27004, 79333, 152397, 216808, 233790], "matches": true}
{"case": "native_tokenizer", "text": "The quick brown fox jumps over the lazy dog near the river bank.", "actual_ids": [760, 3841, 13477, 37550, 33075, 888, 279, 15217, 5388, 3043, 279, 14367, 5883, 13], "expected_ids": [760, 3841, 13477, 37550, 33075, 888, 279, 15217, 5388, 3043, 279, 14367, 5883, 13], "matches": true}
{"case": "native_tokenizer", "text": "def foo(x):\n    return x + 1\n", "actual_ids": [727, 14785, 2007, 1590, 198, 262, 460, 830, 478, 220, 16, 198], "expected_ids": [727, 14785, 2007, 1590, 198, 262, 460, 830, 478, 220, 16, 198], "matches": true}
{"case": "native_tokenizer", "text": "中国的文化历史悠久。", "actual_ids": [108733, 96655, 125188, 1710], "expected_ids": [108733, 96655, 125188, 1710], "matches": true}
{"case": "native_tokenizer", "text": "a🙂b ©c ∑d —e …f €g ™h\n", "actual_ids": [64, 169171, 65, 7013, 66, 11654, 239, 67, 1892, 68, 4444, 69, 12612, 70, 27717, 95, 71, 198], "expected_ids": [64, 169171, 65, 7013, 66, 11654, 239, 67, 1892, 68, 4444, 69, 12612, 70, 27717, 95, 71, 198], "matches": true}
{"case": "native_tokenizer", "text": "x́y १२३ ١٢٣ ①Ⅷ²‍Z", "actual_ids": [87, 52033, 88, 220, 12256, 100, 12256, 101, 12256, 102, 220, 149, 94, 149, 95, 149, 96, 2765, 239, 254, 68086, 100, 28495, 373, 235, 57], "expected_ids": [87, 52033, 88, 220, 12256, 100, 12256, 101, 12256, 102, 220, 149, 94, 149, 95, 149, 96, 220, 46701, 254, 68086, 100, 28495, 373, 235, 57], "matches": false}
{"case": "native_tokenizer", "text": " ①", "actual_ids": [2765, 239, 254], "expected_ids": [220, 46701, 254], "matches": false}
{"case": "native_tokenizer", "text": " Ⅷ", "actual_ids": [2765, 88608], "expected_ids": [220, 68086, 100], "matches": false}
{"case": "native_tokenizer", "text": " ²", "actual_ids": [2071, 110], "expected_ids": [220, 28495], "matches": false}
{"case": "native_tokenizer", "text": " 𝟘", "actual_ids": [78449, 253, 246], "expected_ids": [220, 54362, 253, 246], "matches": false}
{"case": "tokenizer_summary", "mismatches": 5}