45 lines
4.2 KiB
Python
45 lines
4.2 KiB
Python
"""Independent NFC, full tokenizer and request receipts from pinned MTPLX."""
|
||
import hashlib, json, os, subprocess, sys, unicodedata
|
||
from pathlib import Path
|
||
checkout, root, out = map(Path, sys.argv[1:])
|
||
assert subprocess.check_output(['git','-C',str(checkout),'rev-parse','HEAD'],text=True).strip() == '9882703f3105363ddc37eca9f97aa09a1d387112'
|
||
os.environ.update(HF_HUB_OFFLINE='1', TRANSFORMERS_OFFLINE='1')
|
||
sys.path.insert(0, str(checkout))
|
||
from mtplx.runtime import _load_tokenizer_resilient
|
||
from mtplx.server import openai as server
|
||
from tokenizers import Tokenizer
|
||
import tokenizers
|
||
out.mkdir(parents=True, exist_ok=True)
|
||
tok = _load_tokenizer_resilient(root, json.loads((root/'config.json').read_text()))
|
||
normalizer = tok.backend_tokenizer.normalizer
|
||
assert json.loads((root/'tokenizer.json').read_text())['normalizer'] == {'type':'NFC'}
|
||
backend = json.loads(tok.backend_tokenizer.to_str())
|
||
backend['added_tokens'] = []
|
||
plain = Tokenizer.from_str(json.dumps(backend))
|
||
legacy = Tokenizer.from_str(tok.backend_tokenizer.to_str()); legacy.normalizer = None
|
||
text = ''.join('a'+chr(i)+'\u0301\u0323|' for i in range(0x110000) if not 0xd800 <= i <= 0xdfff)
|
||
normalized = normalizer.normalize_str(text)
|
||
(out/'scalar-nfc.json').write_text(json.dumps(dict(scalars=1112064,bytes=len(normalized.encode()),sha256=hashlib.sha256(normalized.encode()).hexdigest(),tokenizers=tokenizers.__version__)))
|
||
texts = ['', 'ASCII 123\n', '́a á ́ १́b', 'e\u0301', 'é', 'a\u0301\u0323', 'a\u0323\u0301', '\u0301\u0323a', '각', '각', '① Ⅷ ² 𝟘', 'Å Ω K', 'x'+ '\u0301'*64, 'a\u034f\u0323\u0301', '\U0001e4ec\u0323a']
|
||
texts += [unicodedata.normalize('NFD', chr(i)) for i in range(0x110000) if not 0xd800 <= i <= 0xdfff and unicodedata.decomposition(chr(i)) and not unicodedata.decomposition(chr(i)).startswith('<')]
|
||
texts += [unicodedata.normalize('NFD',chr(i)) for i in range(0xac00,0xd7a4)]
|
||
texts += ['e\u0301'+t['content']+'\u0301a\u0323\u0301' for t in json.loads((root/'tokenizer.json').read_text())['added_tokens']]
|
||
records=[]
|
||
for text in texts:
|
||
ids=tok.encode(text,add_special_tokens=False)
|
||
records.append(dict(text=text, normalized=normalizer.normalize_str(text), plain_ids=plain.encode(text,add_special_tokens=False).ids, ids=ids, decoded=tok.decode(ids,skip_special_tokens=False,clean_up_tokenization_spaces=False)))
|
||
(out/'tokens.json').write_text(json.dumps(records,ensure_ascii=False))
|
||
requests=[]
|
||
for effort in ['direct','low','medium','xhigh']:
|
||
for system in ['', 'Cafe\u0301 각.']:
|
||
messages=[dict(role='user',content=' ́a á ́ १́b 각\n '),dict(role='assistant',content='Cafe\u0301 ①.',reasoning_content='' if effort=='direct' else 'a\u0301\u0323 thought.'),dict(role='user',content='Continue e\u0301 and 가.\n x')]
|
||
full=([dict(role='system',content=system)] if system else [])+messages
|
||
ids=server._encode_messages_uncached(tok,[server.ChatMessage(**m) for m in full],enable_thinking=effort!='direct',reasoning_effort=None if effort=='direct' else effort,preserve_reasoning_history=True,tools=None)
|
||
rendered=tok.apply_chat_template(full,tokenize=False,add_generation_prompt=True,enable_thinking=effort!='direct',reasoning_effort=None if effort=='direct' else effort,preserve_reasoning_history=True)
|
||
# The server uses this template; keep pre-normalization text to build old IDs.
|
||
assert tok.encode(rendered,add_special_tokens=False)==ids
|
||
history_rendered=tok.apply_chat_template(full[:-1],tokenize=False,add_generation_prompt=False,enable_thinking=effort!='direct',reasoning_effort=None if effort=='direct' else effort,preserve_reasoning_history=True)
|
||
requests.append(dict(legacy_history_ids=legacy.encode(history_rendered,add_special_tokens=False).ids,effort=effort,system=system,messages=messages,ids=ids,legacy_ids=legacy.encode(rendered,add_special_tokens=False).ids,decoded=tok.decode(ids,skip_special_tokens=False,clean_up_tokenization_spaces=False)))
|
||
(out/'requests.json').write_text(json.dumps(requests,ensure_ascii=False))
|
||
print(json.dumps(dict(cases=len(records),requests=len(requests),scalar_sha256=hashlib.sha256(normalized.encode()).hexdigest(),tokenizer_sha256=hashlib.sha256((root/'tokenizer.json').read_bytes()).hexdigest(),tokenizers=tokenizers.__version__)),flush=True)
|