"""Independent NFC, full tokenizer and request receipts from pinned MTPLX."""
import hashlib, json, os, subprocess, sys, unicodedata
from pathlib import Path
checkout, root, out = map(Path, sys.argv[1:])
assert subprocess.check_output(['git','-C',str(checkout),'rev-parse','HEAD'],text=True).strip() == '9882703f3105363ddc37eca9f97aa09a1d387112'
os.environ.update(HF_HUB_OFFLINE='1', TRANSFORMERS_OFFLINE='1')
sys.path.insert(0, str(checkout))
from mtplx.runtime import _load_tokenizer_resilient
from mtplx.server import openai as server
from tokenizers import Tokenizer
import tokenizers
out.mkdir(parents=True, exist_ok=True)
tok = _load_tokenizer_resilient(root, json.loads((root/'config.json').read_text()))
normalizer = tok.backend_tokenizer.normalizer
assert json.loads((root/'tokenizer.json').read_text())['normalizer'] == {'type':'NFC'}
backend = json.loads(tok.backend_tokenizer.to_str())
backend['added_tokens'] = []
plain = Tokenizer.from_str(json.dumps(backend))
legacy = Tokenizer.from_str(tok.backend_tokenizer.to_str()); legacy.normalizer = None
text = ''.join('a'+chr(i)+'\u0301\u0323|' for i in range(0x110000) if not 0xd800 <= i <= 0xdfff)
normalized = normalizer.normalize_str(text)
(out/'scalar-nfc.json').write_text(json.dumps(dict(scalars=1112064,bytes=len(normalized.encode()),sha256=hashlib.sha256(normalized.encode()).hexdigest(),tokenizers=tokenizers.__version__)))
texts = ['', 'ASCII 123\n', '́a á ́ १́b', 'e\u0301', 'é', 'a\u0301\u0323', 'a\u0323\u0301', '\u0301\u0323a', '각', '각', '① Ⅷ ² 𝟘', 'Å Ω K', 'x'+ '\u0301'*64, 'a\u034f\u0323\u0301', '\U0001e4ec\u0323a']
texts += [unicodedata.normalize('NFD', chr(i)) for i in range(0x110000) if not 0xd800 <= i <= 0xdfff and unicodedata.decomposition(chr(i)) and not unicodedata.decomposition(chr(i)).startswith('<')]
texts += [unicodedata.normalize('NFD',chr(i)) for i in range(0xac00,0xd7a4)]
texts += ['e\u0301'+t['content']+'\u0301a\u0323\u0301' for t in json.loads((root/'tokenizer.json').read_text())['added_tokens']]
records=[]
for text in texts:
 ids=tok.encode(text,add_special_tokens=False)
 records.append(dict(text=text, normalized=normalizer.normalize_str(text), plain_ids=plain.encode(text,add_special_tokens=False).ids, ids=ids, decoded=tok.decode(ids,skip_special_tokens=False,clean_up_tokenization_spaces=False)))
(out/'tokens.json').write_text(json.dumps(records,ensure_ascii=False))
requests=[]
for effort in ['direct','low','medium','xhigh']:
 for system in ['', 'Cafe\u0301 각.']:
  messages=[dict(role='user',content=' ́a á ́ १́b 각\n '),dict(role='assistant',content='Cafe\u0301 ①.',reasoning_content='' if effort=='direct' else 'a\u0301\u0323 thought.'),dict(role='user',content='Continue e\u0301 and 가.\n   x')]
  full=([dict(role='system',content=system)] if system else [])+messages
  ids=server._encode_messages_uncached(tok,[server.ChatMessage(**m) for m in full],enable_thinking=effort!='direct',reasoning_effort=None if effort=='direct' else effort,preserve_reasoning_history=True,tools=None)
  rendered=tok.apply_chat_template(full,tokenize=False,add_generation_prompt=True,enable_thinking=effort!='direct',reasoning_effort=None if effort=='direct' else effort,preserve_reasoning_history=True)
  # The server uses this template; keep pre-normalization text to build old IDs.
  assert tok.encode(rendered,add_special_tokens=False)==ids
  history_rendered=tok.apply_chat_template(full[:-1],tokenize=False,add_generation_prompt=False,enable_thinking=effort!='direct',reasoning_effort=None if effort=='direct' else effort,preserve_reasoning_history=True)
  requests.append(dict(legacy_history_ids=legacy.encode(history_rendered,add_special_tokens=False).ids,effort=effort,system=system,messages=messages,ids=ids,legacy_ids=legacy.encode(rendered,add_special_tokens=False).ids,decoded=tok.decode(ids,skip_special_tokens=False,clean_up_tokenization_spaces=False)))
(out/'requests.json').write_text(json.dumps(requests,ensure_ascii=False))
print(json.dumps(dict(cases=len(records),requests=len(requests),scalar_sha256=hashlib.sha256(normalized.encode()).hexdigest(),tokenizer_sha256=hashlib.sha256((root/'tokenizer.json').read_bytes()).hexdigest(),tokenizers=tokenizers.__version__)),flush=True)