Match Qwen Unicode number token boundaries

docs/issue111-tokenizer/ds4.json

@@ -0,0 +1 @@
+{"revision": "ec7642cdd9ec81d01ad4b1fd8f8a3d1511533748", "binary_sha256": "53eae11b7d98f4f73435d28753870b6338a8740cd6084e028d8ed9fdd45d3111", "families": [{"family": "glm", "model": "/Users/gb/Library/Application Support/de.rfc1437.ds4server/models/glm-5.3-flash/GLM-5.3-Flash-Q2.gguf", "cases": [{"text": " ①", "ids": [111229]}, {"text": " Ⅷ", "ids": [2858, 91001]}, {"text": " ²", "ids": [2139, 110]}, {"text": " 𝟘", "ids": [80590, 253, 246]}, {"text": "Item ①, Ⅷ, ², 𝟘 and 123.", "ids": [1234, 111229, 11, 2858, 91001, 11, 2139, 110, 11, 80590, 253, 246, 323, 220, 108714, 13]}, {"text": "भारत एक विशाल देश है जिसकी संस्कृति बहुत पुरानी है।", "ids": [151387, 31280, 108, 78601, 14913, 237, 64234, 14913, 113, 42069, 114, 31280, 110, 14913, 99, 54418, 114, 83615, 151404, 14913, 250, 42069, 116, 64234, 43397, 67662, 71759, 77740, 30366, 243, 151400, 78601, 38643, 14913, 105, 93158, 72094, 78601, 82946, 72094, 43923, 31280, 101, 43397, 83615, 151404, 151430]}, {"text": "ประเทศไทยมีวัฒนธรรมที่เก่าแก่และสวยงามมาก", "ids": [54318, 22230, 71712, 27796, 35705, 153221, 49438, 35705, 34339, 26201, 28221, 37019, 23208, 153214, 20144, 153215, 22230, 22230, 26201, 35705, 46880, 27796, 25122, 64270, 70767, 25122, 18592, 70767, 31649, 71712, 35470, 37019, 34339, 30317, 20728, 26201, 26201, 20728, 25122]}, {"text": "الْعَرَبِيَّةُ", "ids": [31251, 58985, 23161, 27815, 11066, 27815, 21310, 52357, 14548, 132158, 25790, 64137]}, {"text": "The quick brown fox jumps over the lazy dog.", "ids": [785, 3974, 13867, 38627, 34041, 916, 279, 15666, 5562, 13]}, {"text": "中文，标点。", "ids": [102169, 3837, 98593, 98428, 1773]}, {"text": "def f(x):\n    return x + 123\n", "ids": [750, 282, 2075, 982, 262, 470, 856, 488, 220, 108714, 198]}, {"text": "I'm WE'RE it's he'd I'll you've can't 'ſx", "ids": [40, 2776, 19640, 93362, 432, 594, 566, 4172, 358, 3278, 498, 3003, 646, 944, 364, 153000, 87]}, {"text": "́a á ́ १́b", "ids": [53481, 64, 264, 53481, 220, 53481, 220, 151433, 53481, 65]}, {"text": "a\r\n  b\t\t", "ids": [64, 319, 220, 293, 298]}, {"text": "a   ", "ids": [64, 262]}, {"text": "a\t  b", "ids": [64, 3677, 103245, 65]}, {"text": "a… b", "ids": [64, 126, 227, 4102, 65]}, {"text": "x \n \r\n   y", "ids": [87, 715, 2529, 256, 379]}, {"text": "😀 ①🙂Ⅷ ²+𝟘", "ids": [152336, 111229, 152402, 69942, 100, 2139, 110, 10, 55873, 253, 246]}]}, {"family": "deepseek", "model": "/Users/gb/Library/Application Support/de.rfc1437.ds4server/models/deepseek-v4-flash-0731/DeepSeek-V4-Flash-IQ2XXS-w2Q2K-AProjQ8-SExpQ8-OutQ8-chat-v2-imatrix-0731.gguf", "cases": [{"text": " ①", "ids": [1327, 242, 257]}, {"text": " Ⅷ", "ids": [1327, 230, 103]}, {"text": " ²", "ids": [1203, 113]}, {"text": " 𝟘", "ids": [223, 70958, 249]}, {"text": "Item ①, Ⅷ, ², 𝟘 and 123.", "ids": [7519, 1327, 242, 257, 14, 1327, 230, 103, 14, 1203, 113, 14, 223, 70958, 249, 305, 223, 6895, 16]}, {"text": "भारत एक विशाल देश है जिसकी संस्कृति बहुत पुरानी है।", "ids": [34952, 17989, 8422, 66862, 48957, 117, 31639, 26227, 79882, 35674, 27039, 64689, 11018, 8714, 58501, 11931, 61515, 32774, 52058, 28568, 18359, 11634, 8422, 14410, 56275, 22336, 8714, 35674, 4361]}, {"text": "ประเทศไทยมีวัฒนธรรมที่เก่าแก่และสวยงามมาก", "ids": [113424, 9289, 125977, 5689, 13307, 15316, 64120, 8429, 2952, 22401, 1979, 6565, 24208]}, {"text": "الْعَرَبِيَّةُ", "ids": [1712, 9821, 1183, 52748, 61079, 47029, 106817, 7680]}, {"text": "The quick brown fox jumps over the lazy dog.", "ids": [671, 4787, 13769, 46012, 54994, 1060, 270, 41638, 6397, 16]}, {"text": "中文，标点。", "ids": [21134, 303, 1701, 985, 320]}, {"text": "def f(x):\n    return x + 123\n", "ids": [3465, 285, 4042, 3395, 361, 1354, 1527, 940, 223, 6895, 201]}, {"text": "I'm WE'RE it's he'd I'll you've can't 'ſx", "ids": [43, 4571, 33418, 9, 3827, 436, 734, 468, 7485, 342, 5922, 440, 5270, 588, 1664, 905, 75929, 90]}, {"text": "́a á ́ १́b", "ids": [17793, 67, 260, 17793, 223, 17793, 223, 26020, 17793, 68]}, {"text": "a\r\n  b\t\t", "ids": [67, 204, 201, 223, 291, 998]}, {"text": "a   ", "ids": [67, 361]}, {"text": "a\t  b", "ids": [67, 200, 663, 228, 68]}, {"text": "a… b", "ids": [67, 129, 230, 2162, 68]}, {"text": "x \n \r\n   y", "ids": [90, 539, 223, 204, 201, 262, 383]}, {"text": "😀 ①🙂Ⅷ ²+𝟘", "ids": [28927, 225, 1327, 242, 257, 80300, 227, 23603, 103, 1203, 113, 13, 70958, 249]}]}]}