Apply Qwen NFC normalization before token splitting

docs/issue116-nfc/tokens.json

@@ -0,0 +1 @@
+[{"text": "", "normalized": "", "plain_ids": [], "ids": [], "decoded": ""}, {"text": "ASCII 123\n", "normalized": "ASCII 123\n", "plain_ids": [54556, 220, 16, 17, 18, 198], "ids": [54556, 220, 16, 17, 18, 198], "decoded": "ASCII 123\n"}, {"text": "́a á ́ १́b", "normalized": "́a á ́ १́b", "plain_ids": [52033, 64, 18135, 220, 52033, 220, 12256, 100, 52033, 65], "ids": [52033, 64, 18135, 220, 52033, 220, 12256, 100, 52033, 65], "decoded": "́a á ́ १́b"}, {"text": "é", "normalized": "é", "plain_ids": [933], "ids": [933], "decoded": "é"}, {"text": "é", "normalized": "é", "plain_ids": [933], "ids": [933], "decoded": "é"}, {"text": "ạ́", "normalized": "ạ́", "plain_ids": [19614, 52033], "ids": [19614, 52033], "decoded": "ạ́"}, {"text": "ạ́", "normalized": "ạ́", "plain_ids": [19614, 52033], "ids": [19614, 52033], "decoded": "ạ́"}, {"text": "̣́a", "normalized": "̣́a", "plain_ids": [136, 96, 52033, 64], "ids": [136, 96, 52033, 64], "decoded": "̣́a"}, {"text": "각", "normalized": "각", "plain_ids": [149252], "ids": [149252], "decoded": "각"}, {"text": "각", "normalized": "각", "plain_ids": [149252], "ids": [149252], "decoded": "각"}, {"text": "① Ⅷ ² 𝟘", "normalized": "① Ⅷ ² 𝟘", "plain_ids": [46701, 254, 220, 68086, 100, 220, 28495, 220, 54362, 253, 246], "ids": [46701, 254, 220, 68086, 100, 220, 28495, 220, 54362, 253, 246], "decoded": "① Ⅷ ² 𝟘"}, {"text": "Å Ω K", "normalized": "Å Ω K", "plain_ids": [169111, 208307, 710], "ids": [169111, 208307, 710], "decoded": "Å Ω K"}, {"text": "x́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́", "normalized": "x́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́", "plain_ids": [87, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033], "ids": [87, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033, 52033], "decoded": "x́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́́"}, {"text": "a͏̣́", "normalized": "a͏̣́", "plain_ids": [64, 137, 237, 136, 96, 52033], "ids": [64, 137, 237, 136, 96, 52033], "decoded": "a͏̣́"}, {"text": "̣𞓬a", "normalized": "̣𞓬a", "plain_ids": [172, 252, 241, 105, 136, 96, 64], "ids": [172, 252, 241, 105, 136, 96, 64], "decoded": "̣𞓬a"}, {"text": "À", "normalized": "À", "plain_ids": [69508], "ids": [69508], "decoded": "À"}, {"text": "Á", "normalized": "Á", "plain_ids": [42254], "ids": [42254], "decoded": "Á"}, {"text": "Â", "normalized": "Â", "plain_ids": [31709], "ids": [31709], "decoded": "Â"}, {"text": "Ã", "normalized": "Ã", "plain_ids": [18224], "ids": [18224], "decoded": "Ã"}, {"text": "Ä", "normalized": "Ä", "plain_ids": [83975], "ids": [83975], "decoded": "Ä"}, {"text": "Å", "normalized": "Å", "plain_ids": [169111], "ids": [169111], "decoded": "Å"}, {"text": "Ç", "normalized": "Ç", "plain_ids": [44382], "ids": [44382], "decoded": "Ç"}, {"text": "È", "normalized": "È", "plain_ids": [168975], "ids": [168975], "decoded": "È"}, {"text": "É", "normalized": "É", "plain_ids": [25933], "ids": [25933], "decoded": "É"}, {"text": "Ê", "normalized": "Ê", "plain_ids": [150003], "ids": [150003], "decoded": "Ê"}, {"text": "Ë", "normalized": "Ë", "plain_ids": [127, 233], "ids": [127, 233], "decoded": "Ë"}, {"text": "Ì", "normalized": "Ì", "plain_ids": [152116], "ids": [152116], "decoded": "Ì"}, {"text": "Í", "normalized": "Í", "plain_ids": [50467], "ids": [50467], "decoded": "Í"}, {"text": "Î", "normalized": "Î", "plain_ids": [69368], "ids": [69368], "decoded": "Î"}, {"text": "Ï", "normalized": "Ï", "plain_ids": [127, 237], "ids": [127, 237], "decoded": "Ï"}, {"text": "Ñ", "normalized": "Ñ", "plain_ids": [68933], "ids": [68933], "decoded": "Ñ"}, {"text": "Ò", "normalized": "Ò", "plain_ids": [154990], "ids": [154990], "decoded": "Ò"}, {"text": "Ó", "normalized": "Ó", "plain_ids": [37769], "ids": [37769], "decoded": "Ó"}, {"text": "Ô", "normalized": "Ô", "plain_ids": [150209], "ids": [1Line truncated