Apply Qwen NFC normalization before token splitting

tools/issue116-reference.py

@@ -0,0 +1,44 @@
+"""Independent NFC, full tokenizer and request receipts from pinned MTPLX."""
+import hashlib, json, os, subprocess, sys, unicodedata
+from pathlib import Path
+checkout, root, out = map(Path, sys.argv[1:])
+assert subprocess.check_output(['git','-C',str(checkout),'rev-parse','HEAD'],text=True).strip() == '9882703f3105363ddc37eca9f97aa09a1d387112'
+os.environ.update(HF_HUB_OFFLINE='1', TRANSFORMERS_OFFLINE='1')
+sys.path.insert(0, str(checkout))
+from mtplx.runtime import _load_tokenizer_resilient
+from mtplx.server import openai as server
+from tokenizers import Tokenizer
+import tokenizers
+out.mkdir(parents=True, exist_ok=True)
+tok = _load_tokenizer_resilient(root, json.loads((root/'config.json').read_text()))
+normalizer = tok.backend_tokenizer.normalizer
+assert json.loads((root/'tokenizer.json').read_text())['normalizer'] == {'type':'NFC'}
+backend = json.loads(tok.backend_tokenizer.to_str())
+backend['added_tokens'] = []
+plain = Tokenizer.from_str(json.dumps(backend))
+legacy = Tokenizer.from_str(tok.backend_tokenizer.to_str()); legacy.normalizer = None
+text = ''.join('a'+chr(i)+'\u0301\u0323|' for i in range(0x110000) if not 0xd800 <= i <= 0xdfff)
+normalized = normalizer.normalize_str(text)
+(out/'scalar-nfc.json').write_text(json.dumps(dict(scalars=1112064,bytes=len(normalized.encode()),sha256=hashlib.sha256(normalized.encode()).hexdigest(),tokenizers=tokenizers.__version__)))
+texts = ['', 'ASCII 123\n', '́a á ́ १́b', 'e\u0301', 'é', 'a\u0301\u0323', 'a\u0323\u0301', '\u0301\u0323a', '각', '각', '① Ⅷ ² 𝟘', 'Å Ω K', 'x'+ '\u0301'*64, 'a\u034f\u0323\u0301', '\U0001e4ec\u0323a']
+texts += [unicodedata.normalize('NFD', chr(i)) for i in range(0x110000) if not 0xd800 <= i <= 0xdfff and unicodedata.decomposition(chr(i)) and not unicodedata.decomposition(chr(i)).startswith('<')]
+texts += [unicodedata.normalize('NFD',chr(i)) for i in range(0xac00,0xd7a4)]
+texts += ['e\u0301'+t['content']+'\u0301a\u0323\u0301' for t in json.loads((root/'tokenizer.json').read_text())['added_tokens']]
+records=[]
+for text in texts:
+ ids=tok.encode(text,add_special_tokens=False)
+ records.append(dict(text=text, normalized=normalizer.normalize_str(text), plain_ids=plain.encode(text,add_special_tokens=False).ids, ids=ids, decoded=tok.decode(ids,skip_special_tokens=False,clean_up_tokenization_spaces=False)))
+(out/'tokens.json').write_text(json.dumps(records,ensure_ascii=False))
+requests=[]
+for effort in ['direct','low','medium','xhigh']:
+ for system in ['', 'Cafe\u0301 각.']:
+  messages=[dict(role='user',content=' ́a á ́ १́b 각\n '),dict(role='assistant',content='Cafe\u0301 ①.',reasoning_content='' if effort=='direct' else 'a\u0301\u0323 thought.'),dict(role='user',content='Continue e\u0301 and 가.\n   x')]
+  full=([dict(role='system',content=system)] if system else [])+messages
+  ids=server._encode_messages_uncached(tok,[server.ChatMessage(**m) for m in full],enable_thinking=effort!='direct',reasoning_effort=None if effort=='direct' else effort,preserve_reasoning_history=True,tools=None)
+  rendered=tok.apply_chat_template(full,tokenize=False,add_generation_prompt=True,enable_thinking=effort!='direct',reasoning_effort=None if effort=='direct' else effort,preserve_reasoning_history=True)
+  # The server uses this template; keep pre-normalization text to build old IDs.
+  assert tok.encode(rendered,add_special_tokens=False)==ids
+  history_rendered=tok.apply_chat_template(full[:-1],tokenize=False,add_generation_prompt=False,enable_thinking=effort!='direct',reasoning_effort=None if effort=='direct' else effort,preserve_reasoning_history=True)
+  requests.append(dict(legacy_history_ids=legacy.encode(history_rendered,add_special_tokens=False).ids,effort=effort,system=system,messages=messages,ids=ids,legacy_ids=legacy.encode(rendered,add_special_tokens=False).ids,decoded=tok.decode(ids,skip_special_tokens=False,clean_up_tokenization_spaces=False)))
+(out/'requests.json').write_text(json.dumps(requests,ensure_ascii=False))
+print(json.dumps(dict(cases=len(records),requests=len(requests),scalar_sha256=hashlib.sha256(normalized.encode()).hexdigest(),tokenizer_sha256=hashlib.sha256((root/'tokenizer.json').read_bytes()).hexdigest(),tokenizers=tokenizers.__version__)),flush=True)