Match Qwen Unicode number token boundaries

docs/issue111-tokenizer/qwen.json

@@ -0,0 +1 @@
+{"number_count": 1911, "cases": [{"text": " ①", "ids": [220, 46701, 254], "full_ids": [220, 46701, 254], "pieces": [" ", "①"]}, {"text": " Ⅷ", "ids": [220, 68086, 100], "full_ids": [220, 68086, 100], "pieces": [" ", "Ⅷ"]}, {"text": " ²", "ids": [220, 28495], "full_ids": [220, 28495], "pieces": [" ", "²"]}, {"text": " 𝟘", "ids": [220, 54362, 253, 246], "full_ids": [220, 54362, 253, 246], "pieces": [" ", "𝟘"]}, {"text": "Item ①, Ⅷ, ², 𝟘 and 123.", "ids": [1192, 220, 46701, 254, 11, 220, 68086, 100, 11, 220, 28495, 11, 220, 54362, 253, 246, 321, 220, 16, 17, 18, 13], "full_ids": [1192, 220, 46701, 254, 11, 220, 68086, 100, 11, 220, 28495, 11, 220, 54362, 253, 246, 321, 220, 16, 17, 18, 13], "pieces": ["Item", " ", "①", ",", " ", "Ⅷ", ",", " ", "²", ",", " ", "𝟘", " and", " ", "1", "2", "3", "."]}, {"text": "भारत एक विशाल देश है जिसकी संस्कृति बहुत पुरानी है।", "ids": [149755, 148504, 76519, 171549, 165997, 151224, 197734, 170046, 184864, 160357, 172281, 237627, 158827, 153570, 199638, 80752, 151859, 181117, 170046, 154845], "full_ids": [149755, 148504, 76519, 171549, 165997, 151224, 197734, 170046, 184864, 160357, 172281, 237627, 158827, 153570, 199638, 80752, 151859, 181117, 170046, 154845], "pieces": ["भारत", " एक", " विशाल", " देश", " है", " जिसकी", " संस्कृति", " बहुत", " पुरानी", " है", "।"]}, {"text": "ประเทศไทยมีวัฒนธรรมที่เก่าแก่และสวยงามมาก", "ids": [156150, 148450, 186319, 148285, 178118, 155545, 149119, 162566, 149196], "full_ids": [156150, 148450, 186319, 148285, 178118, 155545, 149119, 162566, 149196], "pieces": ["ประเทศไทยมีวัฒนธรรมที่เก่าแก่และสวยงามมาก"]}, {"text": "الْعَرَبِيَّةُ", "ids": [30350, 187581, 153458, 150765, 151003, 178402, 62422], "full_ids": [30350, 187581, 153458, 150765, 151003, 178402, 62422], "pieces": ["الْعَرَبِيَّةُ"]}, {"text": "The quick brown fox jumps over the lazy dog.", "ids": [760, 3841, 13477, 37550, 33075, 888, 279, 15217, 5388, 13], "full_ids": [760, 3841, 13477, 37550, 33075, 888, 279, 15217, 5388, 13], "pieces": ["The", " quick", " brown", " fox", " jumps", " over", " the", " lazy", " dog", "."]}, {"text": "中文，标点。", "ids": [99986, 3709, 141111, 1710], "full_ids": [99986, 3709, 141111, 1710], "pieces": ["中文", "，标点", "。"]}, {"text": "def f(x):\n    return x + 123\n", "ids": [727, 281, 2007, 1590, 198, 262, 460, 830, 478, 220, 16, 17, 18, 198], "full_ids": [727, 281, 2007, 1590, 198, 262, 460, 830, 478, 220, 16, 17, 18, 198], "pieces": ["def", " f", "(x", "):\n", "   ", " return", " x", " +", " ", "1", "2", "3", "\n"]}, {"text": "I'm WE'RE it's he'd I'll you've can't 'ſx", "ids": [40, 2688, 19076, 90902, 424, 579, 551, 4035, 353, 3172, 488, 2908, 628, 914, 359, 129, 123, 87], "full_ids": [40, 2688, 19076, 90902, 424, 579, 551, 4035, 353, 3172, 488, 2908, 628, 914, 359, 129, 123, 87], "pieces": ["I", "'m", " WE", "'RE", " it", "'s", " he", "'d", " I", "'ll", " you", "'ve", " can", "'t", " '", "ſx"]}, {"text": "́a á ́ १́b", "ids": [52033, 64, 264, 52033, 220, 52033, 220, 12256, 100, 52033, 65], "full_ids": [52033, 64, 18135, 220, 52033, 220, 12256, 100, 52033, 65], "pieces": ["́a", " á", " ́", " ", "१", "́b"]}, {"text": "a\r\n  b\t\t", "ids": [64, 317, 220, 292, 297], "full_ids": [64, 317, 220, 292, 297], "pieces": ["a", "\r\n", " ", " b", "\t\t"]}, {"text": "a   ", "ids": [64, 262], "full_ids": [64, 262], "pieces": ["a", "   "]}, {"text": "a\t  b", "ids": [64, 3551, 373, 225, 65], "full_ids": [64, 3551, 373, 225, 65], "pieces": ["a", "\t ", " b"]}, {"text": "a… b", "ids": [64, 126, 227, 3966, 65], "full_ids": [64, 126, 227, 3966, 65], "pieces": ["a", "…", " b"]}, {"text": "x \n \r\n   y", "ids": [87, 695, 2449, 256, 374], "full_ids": [87, 695, 2449, 256, 374], "pieces": ["x", " \n \r\n", "  ", " y"]}, {"text": "😀 ①🙂Ⅷ ²+𝟘", "ids": [169706, 220, 46701, 254, 169171, 68086, 100, 220, 28495, 10, 54362, 253, 246], "full_ids": [169706, 220, 46701, 254, 169171, 68086, 100, 220, 28495, 10, 54362, 253, 246], "pieces": ["😀", " ", "①", "🙂", "Ⅷ", " ", "²", "+", "𝟘"]}, {"text": " 0", "ids": [220, 15], "full_ids": [220, 15], "pieces": [" ", "0"]}, {"text": "a0b", "ids": [64, 15, 65], "full_ids": [64, 15, 65], "pieces": ["a", "0", "b"]}, {"text": " ́0́x", "ids": [220, 52033, 15, 52033, 87], "full_ids": [220, 52033, 15, 52033, 87], "pieces": [" ́", "0", "́x"]}, {"text": " 1", "ids": [220, 16], "full_ids": [220, 16], "pieces": [" ", "1"]}, {"text": "a1b", "ids": [64, 16, 65], "full_ids": [64, 16, 65], "pieces": ["a", "1", "b"]}, {"text": " ́1́x", "ids": [220, 52033, 16, 52033, 87], "full_ids": [220, 52033, 16, 52033,Line truncated