Add Qwen vision loading and Metal inference

src/engine/metal/qwen_mtplx/model.rs

@@ -89,6 +89,7 @@ pub(super) struct TextModel<'a> {
     pub(super) mixer: hyper::GatedResidual<'a>,
     pub(super) head: Option<Linear<'a>>, // None means tied embedding weights.
     pub(super) mtp: Option<super::mtp::Mtp<'a>>,
+    pub(super) vision: Option<super::vision::Input>,
     pub(super) last_widened: Option<Array>,
     policy: CompilePolicy,
     decode_runs: Option<Vec<Run>>,
@@ -111,7 +112,7 @@ pub(super) struct Output {
     pub(super) hidden: Option<Array>,
 }
 
-fn embedding(layer: Linear<'_>, ids: &Array, stream: Stream) -> Result<Array, String> {
+pub(super) fn embedding(layer: Linear<'_>, ids: &Array, stream: Stream) -> Result<Array, String> {
     match layer {
         Linear::Dense(weight) => indexing::take_rows(weight, ids, stream),
         Linear::Quantized(layer) => linear::dequantize(
@@ -171,6 +172,7 @@ impl<'a> TextModel<'a> {
             head,
             parameter_order: Vec::new(),
             mtp: None,
+            vision: None,
             last_widened: None,
             policy,
             decode_runs: None,
@@ -213,6 +215,8 @@ impl<'a> TextModel<'a> {
         streams: &Streams,
         stream: Stream,
     ) -> Result<Output, String> {
+        let image = self.vision.clone();
+        let vision = vision.or_else(|| image.as_ref().map(|v| (Some(&v.table), v.delta)));
         let emb = embedding(self.embedding, ids, stream)?;
         let mtp = self
             .mtp
@@ -242,7 +246,18 @@ impl<'a> TextModel<'a> {
         streams: &Streams,
         stream: Stream,
     ) -> Result<Array, String> {
-        let emb = match input_embeddings {
+        let image = self.vision.clone();
+        let vision = vision.or_else(|| image.as_ref().map(|v| (Some(&v.table), v.delta)));
+        let spliced = match (&image, input_embeddings) {
+            (Some(v), None) => v.embeddings(
+                self.embedding,
+                ids,
+                super::vision::cache_position(cache.as_deref()) + 1,
+                stream,
+            )?,
+            _ => None,
+        };
+        let emb = match input_embeddings.or(spliced.as_ref()) {
             Some(emb) => emb.clone(),
             None => embedding(self.embedding, ids, stream)?,
         };
@@ -440,7 +455,18 @@ impl<'a> TextModel<'a> {
         streams: &Streams,
         stream: Stream,
     ) -> Result<Output, String> {
-        let h = match input_embeddings {
+        let image = self.vision.clone();
+        let vision = vision.or_else(|| image.as_ref().map(|v| (Some(&v.table), v.delta)));
+        let spliced = match (&image, input_embeddings) {
+            (Some(v), None) => v.embeddings(
+                self.embedding,
+                ids,
+                super::vision::cache_position(cache.as_deref()),
+                stream,
+            )?,
+            _ => None,
+        };
+        let h = match input_embeddings.or(spliced.as_ref()) {
             Some(h) => h.clone(),
             None => embedding(self.embedding, ids, stream)?,
         };