Add Qwen vision loading and Metal inference

2026-09-11 18:32:41 +02:00
parent 48c2f751b4
commit e32954ac1b
30 changed files with 1687 additions and 100 deletions
@@ -0,0 +1,32 @@
+"""Describe a neutral image and run a no-image control through pinned MTPLX.
+Usage: test-supervisor ... --command python tools/qwen-vision-chat-reference.py MODEL_DIR IMAGE
+Uses the reference runtime only as an oracle; application inference is Rust/Metal.
+"""
+import os,sys,json,time,argparse
+from pathlib import Path
+root=Path(sys.argv[1]);image=Path(sys.argv[2])
+from mtplx.server.openai import _server_runtime_env_overrides
+os.environ.update(_server_runtime_env_overrides(argparse.Namespace(model=str(root),generation_mode='mtp'),None))
+os.environ['MTPLX_SUSTAINED_PREFILL']='1'
+os.environ['MTPLX_PREFILL_CHUNK_SIZE']='2048'
+import mlx.core as mx
+from mlx_lm.utils import load_model
+from mtplx.models import qwen4_exp as qwen
+from mtplx.runtime import _load_tokenizer_resilient,MTPLXRuntime
+from mtplx.mtp_patch import MTPContract
+from mtplx.sampling import SamplerConfig
+from mtplx.generation import generate_mtpk
+from mtplx.server.openai import ChatMessage,_encode_messages_uncached,_materialize_vision_splice
+print(json.dumps(dict(event='loading')),flush=True)
+tokenizer=_load_tokenizer_resilient(root,json.loads((root/'config.json').read_text()))
+model,_=load_model(root,lazy=False,strict=True,get_model_classes=lambda **_:(qwen.Model,qwen.ModelArgs))
+model.post_weight_load(root);assert model.attach_mtp(root)
+rt=MTPLXRuntime(model,tokenizer,root,True,MTPContract())
+for with_image in [True,False]:
+ text='Describe this image'+('<|vision_start|><|image_pad|><|vision_end|>' if with_image else '')
+ ids=_encode_messages_uncached(tokenizer,[ChatMessage(role='user',content=text)],enable_thinking=True,reasoning_effort='low',preserve_reasoning_history=True,tools=None)
+ splice=None
+ if with_image:ids,splice=_materialize_vision_splice(argparse.Namespace(args=argparse.Namespace(model=str(root))),[image.read_bytes()],ids)
+ print(json.dumps(dict(event='input',with_image=with_image,prompt='Describe this image',ids=ids)),flush=True)
+ out=generate_mtpk(rt,ids,max_tokens=1024,sampler=SamplerConfig(temperature=0.,top_p=.95,top_k=20),speculative_depth=3,seed=1,stop_token_ids=set(tokenizer.eos_token_ids),mtp_history_policy='committed',verify_strategy='batched',vision_splice=splice,capture_final_state=True,prefill_callback=lambda d:print(json.dumps(dict(event='prefill',**d)),flush=True),token_callback=lambda ids:print(json.dumps(dict(event='tokens',ids=ids)),flush=True))
+ print(json.dumps(dict(event='result',with_image=with_image,text=out.text,tokens=out.tokens,finish_reason=out.finish_reason)),flush=True)