Files
DS4Server/docs/issue111-tokenizer/ds4.json
T

1 line
4.7 KiB
JSON
Raw Blame History

This file contains invisible Unicode characters
This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
{"revision": "ec7642cdd9ec81d01ad4b1fd8f8a3d1511533748", "binary_sha256": "53eae11b7d98f4f73435d28753870b6338a8740cd6084e028d8ed9fdd45d3111", "families": [{"family": "glm", "model": "/Users/gb/Library/Application Support/de.rfc1437.ds4server/models/glm-5.3-flash/GLM-5.3-Flash-Q2.gguf", "cases": [{"text": " ①", "ids": [111229]}, {"text": " Ⅷ", "ids": [2858, 91001]}, {"text": " ²", "ids": [2139, 110]}, {"text": " 𝟘", "ids": [80590, 253, 246]}, {"text": "Item ①, Ⅷ, ², 𝟘 and 123.", "ids": [1234, 111229, 11, 2858, 91001, 11, 2139, 110, 11, 80590, 253, 246, 323, 220, 108714, 13]}, {"text": "भारत एक विशाल देश है जिसकी संस्कृति बहुत पुरानी है।", "ids": [151387, 31280, 108, 78601, 14913, 237, 64234, 14913, 113, 42069, 114, 31280, 110, 14913, 99, 54418, 114, 83615, 151404, 14913, 250, 42069, 116, 64234, 43397, 67662, 71759, 77740, 30366, 243, 151400, 78601, 38643, 14913, 105, 93158, 72094, 78601, 82946, 72094, 43923, 31280, 101, 43397, 83615, 151404, 151430]}, {"text": "ประเทศไทยมีวัฒนธรรมที่เก่าแก่และสวยงามมาก", "ids": [54318, 22230, 71712, 27796, 35705, 153221, 49438, 35705, 34339, 26201, 28221, 37019, 23208, 153214, 20144, 153215, 22230, 22230, 26201, 35705, 46880, 27796, 25122, 64270, 70767, 25122, 18592, 70767, 31649, 71712, 35470, 37019, 34339, 30317, 20728, 26201, 26201, 20728, 25122]}, {"text": "الْعَرَبِيَّةُ", "ids": [31251, 58985, 23161, 27815, 11066, 27815, 21310, 52357, 14548, 132158, 25790, 64137]}, {"text": "The quick brown fox jumps over the lazy dog.", "ids": [785, 3974, 13867, 38627, 34041, 916, 279, 15666, 5562, 13]}, {"text": "中文,标点。", "ids": [102169, 3837, 98593, 98428, 1773]}, {"text": "def f(x):\n return x + 123\n", "ids": [750, 282, 2075, 982, 262, 470, 856, 488, 220, 108714, 198]}, {"text": "I'm WE'RE it's he'd I'll you've can't 'ſx", "ids": [40, 2776, 19640, 93362, 432, 594, 566, 4172, 358, 3278, 498, 3003, 646, 944, 364, 153000, 87]}, {"text": "́a á ́ १́b", "ids": [53481, 64, 264, 53481, 220, 53481, 220, 151433, 53481, 65]}, {"text": "a\r\n b\t\t", "ids": [64, 319, 220, 293, 298]}, {"text": "a ", "ids": [64, 262]}, {"text": "a\t  b", "ids": [64, 3677, 103245, 65]}, {"text": "a… b", "ids": [64, 126, 227, 4102, 65]}, {"text": "x \n \r\n y", "ids": [87, 715, 2529, 256, 379]}, {"text": "😀 ①🙂Ⅷ ²+𝟘", "ids": [152336, 111229, 152402, 69942, 100, 2139, 110, 10, 55873, 253, 246]}]}, {"family": "deepseek", "model": "/Users/gb/Library/Application Support/de.rfc1437.ds4server/models/deepseek-v4-flash-0731/DeepSeek-V4-Flash-IQ2XXS-w2Q2K-AProjQ8-SExpQ8-OutQ8-chat-v2-imatrix-0731.gguf", "cases": [{"text": " ①", "ids": [1327, 242, 257]}, {"text": " Ⅷ", "ids": [1327, 230, 103]}, {"text": " ²", "ids": [1203, 113]}, {"text": " 𝟘", "ids": [223, 70958, 249]}, {"text": "Item ①, Ⅷ, ², 𝟘 and 123.", "ids": [7519, 1327, 242, 257, 14, 1327, 230, 103, 14, 1203, 113, 14, 223, 70958, 249, 305, 223, 6895, 16]}, {"text": "भारत एक विशाल देश है जिसकी संस्कृति बहुत पुरानी है।", "ids": [34952, 17989, 8422, 66862, 48957, 117, 31639, 26227, 79882, 35674, 27039, 64689, 11018, 8714, 58501, 11931, 61515, 32774, 52058, 28568, 18359, 11634, 8422, 14410, 56275, 22336, 8714, 35674, 4361]}, {"text": "ประเทศไทยมีวัฒนธรรมที่เก่าแก่และสวยงามมาก", "ids": [113424, 9289, 125977, 5689, 13307, 15316, 64120, 8429, 2952, 22401, 1979, 6565, 24208]}, {"text": "الْعَرَبِيَّةُ", "ids": [1712, 9821, 1183, 52748, 61079, 47029, 106817, 7680]}, {"text": "The quick brown fox jumps over the lazy dog.", "ids": [671, 4787, 13769, 46012, 54994, 1060, 270, 41638, 6397, 16]}, {"text": "中文,标点。", "ids": [21134, 303, 1701, 985, 320]}, {"text": "def f(x):\n return x + 123\n", "ids": [3465, 285, 4042, 3395, 361, 1354, 1527, 940, 223, 6895, 201]}, {"text": "I'm WE'RE it's he'd I'll you've can't 'ſx", "ids": [43, 4571, 33418, 9, 3827, 436, 734, 468, 7485, 342, 5922, 440, 5270, 588, 1664, 905, 75929, 90]}, {"text": "́a á ́ १́b", "ids": [17793, 67, 260, 17793, 223, 17793, 223, 26020, 17793, 68]}, {"text": "a\r\n b\t\t", "ids": [67, 204, 201, 223, 291, 998]}, {"text": "a ", "ids": [67, 361]}, {"text": "a\t  b", "ids": [67, 200, 663, 228, 68]}, {"text": "a… b", "ids": [67, 129, 230, 2162, 68]}, {"text": "x \n \r\n y", "ids": [90, 539, 223, 204, 201, 262, 383]}, {"text": "😀 ①🙂Ⅷ ²+𝟘", "ids": [28927, 225, 1327, 242, 257, 80300, 227, 23603, 103, 1203, 113, 13, 70958, 249]}]}]}