diff --git a/src/mobius/_registry.py b/src/mobius/_registry.py
index a3534c0b..dd0aa18b 100644
--- a/src/mobius/_registry.py
+++ b/src/mobius/_registry.py
@@ -476,14 +476,14 @@ def _detect_fallback_registration(hf_config) -> ModelRegistration | None:
     "llava_next": ModelRegistration(LLaVAModel, task="vision-language"),
     "llava_next_video": ModelRegistration(LLaVAModel, task="vision-language"),
     "llava_onevision": ModelRegistration(LLaVAModel, task="vision-language"),
-    "mistral3": ModelRegistration(LLaVAModel, task="vision-language"),
+    "mistral3": ModelRegistration(LLaVAModel, task="pixtral-vl"),
     "mllama": ModelRegistration(MllamaCausalLMModel, task="mllama-vision-language"),
     "molmo": ModelRegistration(LLaVAModel, task="vision-language"),
     "ovis2": ModelRegistration(LLaVAModel, task="vision-language"),
     "paligemma": ModelRegistration(LLaVAModel, task="vision-language"),
     "phi4_multimodal": ModelRegistration(Phi4MMMultiModalModel, task="phi4mm-multimodal"),
     "phi4mm": ModelRegistration(Phi4MMMultiModalModel, task="phi4mm-multimodal"),
-    "pixtral": ModelRegistration(LLaVAModel, task="vision-language"),
+    "pixtral": ModelRegistration(LLaVAModel, task="pixtral-vl"),
     "qwen2_5_vl": ModelRegistration(Qwen25VLCausalLMModel, task="qwen-vl"),
     "qwen2_5_vl_text": ModelRegistration(Qwen25VLTextModel),
     "qwen2_vl": ModelRegistration(Qwen25VLCausalLMModel, task="qwen-vl"),
diff --git a/src/mobius/_weight_loading.py b/src/mobius/_weight_loading.py
index 55778052..c3724e24 100644
--- a/src/mobius/_weight_loading.py
+++ b/src/mobius/_weight_loading.py
@@ -147,6 +147,58 @@ def _parallel_download(
     return [path_map[f] for f in filenames]
 
 
+def _dequantize_fp8_weights(state_dict: dict[str, torch.Tensor]) -> dict[str, torch.Tensor]:
+    """Dequantize FP8 weights and return a new dict with float tensors.
+
+    Some HuggingFace checkpoints (e.g. Ministral-3-3B) store linear layer
+    weights as float8_e4m3fn with a scalar ``weight_scale_inv`` tensor.
+    The real weight value is ``fp8_weight.to(bfloat16) * weight_scale_inv``.
+
+    Dequantization targets bfloat16 because that is the native training
+    dtype for FP8-quantized checkpoints — the FP8 values represent
+    bfloat16 values scaled into the FP8 range.
+
+    This function detects FP8 tensors, applies the scale, and removes the
+    auxiliary ``weight_scale_inv``, ``activation_scale``, and
+    ``input_scale`` tensors.
+
+    Returns:
+        A new dict with FP8 weights dequantized to bfloat16 and the
+        auxiliary scale tensors removed.  Always returns a new dict,
+        even when no FP8 weights are found.
+    """
+    fp8_dtypes = {torch.float8_e4m3fn, torch.float8_e5m2}
+    fp8_keys = [k for k, v in state_dict.items() if v.dtype in fp8_dtypes]
+    if not fp8_keys:
+        return dict(state_dict)
+
+    # Work on a copy to avoid mutating the caller's dict
+    result = dict(state_dict)
+
+    logger.info("Dequantizing %d FP8 weights", len(fp8_keys))
+    for key in fp8_keys:
+        # Derive scale key from weight key using suffix replacement to avoid
+        # replacing '.weight' substrings that appear in the middle of the key
+        # (e.g. 'model.weight_proj.weight' → 'model.weight_proj.weight_scale_inv').
+        if key.endswith(".weight"):
+            scale_key = key[: -len(".weight")] + ".weight_scale_inv"
+        else:
+            scale_key = key + "_scale_inv"
+
+        if scale_key in result:
+            # Cast scale to bfloat16 to guarantee the output dtype is bfloat16,
+            # even when weight_scale_inv is stored as FP32 in the checkpoint.
+            scale = result[scale_key].to(torch.bfloat16)
+            result[key] = result[key].to(torch.bfloat16) * scale
+        else:
+            logger.warning("FP8 weight '%s' has no scale_inv — casting without scaling", key)
+            result[key] = result[key].to(torch.bfloat16)
+
+    # Remove auxiliary FP8 tensors (not needed in the ONNX graph)
+    aux_suffixes = (".weight_scale_inv", ".activation_scale", ".input_scale")
+    return {k: v for k, v in result.items() if not any(k.endswith(s) for s in aux_suffixes)}
+
+
 def _download_weights(model_id: str) -> dict[str, torch.Tensor]:
     """Download weights from HuggingFace and return as a state dict.
 
@@ -171,4 +223,6 @@ def _download_weights(model_id: str) -> dict[str, torch.Tensor]:
     state_dict: dict[str, torch.Tensor] = {}
     for path in tqdm.tqdm(paths, desc="Loading weights"):
         state_dict.update(safetensors.torch.load_file(path))
+
+    state_dict = _dequantize_fp8_weights(state_dict)
     return state_dict
diff --git a/src/mobius/_weight_loading_test.py b/src/mobius/_weight_loading_test.py
index 8f74b15e..1d40f7cd 100644
--- a/src/mobius/_weight_loading_test.py
+++ b/src/mobius/_weight_loading_test.py
@@ -518,3 +518,108 @@ def build(self, module, config):
         module = CausalLMModel(config)
         pkg = build_from_module(module, config, task=StubTask())
         assert pkg["model"].graph.name == "stub"
+
+
+class TestDequantizeFP8Weights:
+    """Tests for _dequantize_fp8_weights."""
+
+    def test_no_fp8_returns_unchanged(self):
+        """Non-FP8 state dicts pass through unchanged."""
+        from mobius._weight_loading import _dequantize_fp8_weights
+
+        state_dict = {
+            "layer.weight": torch.randn(4, 4),
+            "layer.bias": torch.randn(4),
+        }
+        result = _dequantize_fp8_weights(state_dict)
+        assert set(result.keys()) == set(state_dict.keys())
+        assert torch.equal(result["layer.weight"], state_dict["layer.weight"])
+
+    def test_fp8_e4m3fn_dequantized(self):
+        """FP8 weights are multiplied by weight_scale_inv."""
+        from mobius._weight_loading import _dequantize_fp8_weights
+
+        fp8_weight = torch.tensor([1.0, 2.0, -1.0, 0.5], dtype=torch.float32).to(
+            torch.float8_e4m3fn
+        )
+        scale_inv = torch.tensor(0.5, dtype=torch.bfloat16)
+        state_dict = {
+            "proj.weight": fp8_weight,
+            "proj.weight_scale_inv": scale_inv,
+        }
+        result = _dequantize_fp8_weights(state_dict)
+        assert "proj.weight" in result
+        assert "proj.weight_scale_inv" not in result  # aux tensor removed
+        assert result["proj.weight"].dtype == torch.bfloat16
+        # Verify dequant: fp8→bf16 * scale_inv
+        expected = fp8_weight.to(torch.bfloat16) * scale_inv
+        assert torch.allclose(result["proj.weight"], expected)
+
+    def test_activation_scale_removed(self):
+        """Auxiliary activation_scale tensors are removed."""
+        from mobius._weight_loading import _dequantize_fp8_weights
+
+        state_dict = {
+            "proj.weight": torch.tensor([1.0], dtype=torch.float32).to(torch.float8_e4m3fn),
+            "proj.weight_scale_inv": torch.tensor(1.0, dtype=torch.bfloat16),
+            "proj.activation_scale": torch.tensor(1.0, dtype=torch.bfloat16),
+        }
+        result = _dequantize_fp8_weights(state_dict)
+        assert "proj.activation_scale" not in result
+
+    def test_suffix_replace_not_greedy(self):
+        """The scale key derivation uses suffix replacement, not global replace.
+
+        For a key like 'model.weight_proj.weight', the scale key should be
+        'model.weight_proj.weight_scale_inv' (not 'model.weight_scale_inv_proj.weight_scale_inv').
+        """
+        from mobius._weight_loading import _dequantize_fp8_weights
+
+        fp8_weight = torch.tensor([1.0], dtype=torch.float32).to(torch.float8_e4m3fn)
+        scale = torch.tensor(2.0, dtype=torch.bfloat16)
+        state_dict = {
+            "model.weight_proj.weight": fp8_weight,
+            "model.weight_proj.weight_scale_inv": scale,
+        }
+        result = _dequantize_fp8_weights(state_dict)
+        assert "model.weight_proj.weight" in result
+        assert result["model.weight_proj.weight"].dtype == torch.bfloat16
+
+    def test_missing_scale_casts_without_scaling(self):
+        """FP8 weight without scale_inv is cast to bfloat16 without scaling."""
+        from mobius._weight_loading import _dequantize_fp8_weights
+
+        fp8_weight = torch.tensor([1.0, 2.0], dtype=torch.float32).to(torch.float8_e4m3fn)
+        state_dict = {"orphan.weight": fp8_weight}
+        result = _dequantize_fp8_weights(state_dict)
+        assert result["orphan.weight"].dtype == torch.bfloat16
+
+    def test_fp32_scale_produces_bf16_output(self):
+        """FP32 weight_scale_inv should still produce bfloat16 output."""
+        from mobius._weight_loading import _dequantize_fp8_weights
+
+        fp8_weight = torch.tensor([1.0, 2.0], dtype=torch.float32).to(torch.float8_e4m3fn)
+        # Scale stored as FP32 (common for scalar scales in real checkpoints)
+        scale_inv = torch.tensor(0.5, dtype=torch.float32)
+        state_dict = {
+            "proj.weight": fp8_weight,
+            "proj.weight_scale_inv": scale_inv,
+        }
+        result = _dequantize_fp8_weights(state_dict)
+        assert result["proj.weight"].dtype == torch.bfloat16, (
+            f"Expected bfloat16, got {result['proj.weight'].dtype}"
+        )
+
+    def test_does_not_mutate_input(self):
+        """_dequantize_fp8_weights should not mutate the input dict."""
+        from mobius._weight_loading import _dequantize_fp8_weights
+
+        fp8_weight = torch.tensor([1.0], dtype=torch.float32).to(torch.float8_e4m3fn)
+        scale = torch.tensor(1.0, dtype=torch.bfloat16)
+        original = {
+            "proj.weight": fp8_weight,
+            "proj.weight_scale_inv": scale,
+        }
+        original_keys = set(original.keys())
+        _dequantize_fp8_weights(original)
+        assert set(original.keys()) == original_keys, "Input dict was mutated"
diff --git a/src/mobius/components/_attention.py b/src/mobius/components/_attention.py
index 73db9fe9..09596bda 100644
--- a/src/mobius/components/_attention.py
+++ b/src/mobius/components/_attention.py
@@ -304,6 +304,17 @@ def forward(
 
         # Apply rotary position embeddings (skip when not provided)
         if position_embeddings is not None:
+            # Apply llama_4_attn_scale if present (Ministral3/Mistral4).
+            # The scale is computed from position_ids by the RoPE module
+            # and passed as the 3rd element of position_embeddings.
+            # Applied BEFORE RoPE so the graph keeps the
+            # RotaryEmbedding → Attention pattern that the
+            # RotaryAttentionToGQA rewrite rule matches. Scaling
+            # commutes with rotation: scale(RoPE(q)) == RoPE(scale(q)).
+            if len(position_embeddings) > 2:
+                attn_scale = position_embeddings[2]
+                query_states = op.Mul(query_states, attn_scale)
+
             query_states = apply_rotary_pos_emb(
                 op,
                 x=query_states,
diff --git a/src/mobius/components/_pixtral_vision.py b/src/mobius/components/_pixtral_vision.py
index 165adea1..c15fec7d 100644
--- a/src/mobius/components/_pixtral_vision.py
+++ b/src/mobius/components/_pixtral_vision.py
@@ -24,6 +24,7 @@
 from onnxscript import nn
 from onnxscript._internal import builder
 
+from mobius._build_context import ep_capabilities
 from mobius._configs import ArchitectureConfig
 from mobius.components._common import Linear
 from mobius.components._conv import Conv2dNoBias
@@ -174,17 +175,31 @@ def forward(
             num_heads=self._num_heads,
         )
 
-        # Bidirectional attention (is_causal=0, no KV cache)
+        # Bidirectional attention (no causal mask, no KV cache).
+        # Use com.microsoft.MultiHeadAttention for all EPs that support
+        # custom-domain ops (it has fused kernels on CUDA/DML and runs
+        # correctly on CPU). Fall back to standard opset-23 Attention
+        # for onnx-standard EP which prohibits custom-domain ops.
         scale = float(1.0 / (self._head_dim**0.5))
-        attn_output = op.Attention(
-            q,
-            k,
-            v,
-            q_num_heads=self._num_heads,
-            kv_num_heads=self._num_heads,
-            scale=scale,
-            is_causal=0,
-        )
+        if ep_capabilities().name == "onnx-standard":
+            attn_output = op.Attention(
+                q,
+                k,
+                v,
+                q_num_heads=self._num_heads,
+                kv_num_heads=self._num_heads,
+                scale=scale,
+                is_causal=0,
+            )
+        else:
+            attn_output = op.MultiHeadAttention(
+                q,
+                k,
+                v,
+                num_heads=self._num_heads,
+                scale=scale,
+                _domain="com.microsoft",
+            )
         return self.o_proj(op, attn_output)
 
 
@@ -349,10 +364,15 @@ def forward(
         )
         x = op.Reshape(hidden_states, shape_6d)
 
-        # Transpose: [batch, H/ms, W/ms, ms, ms, D]
-        x = op.Transpose(x, perm=[0, 1, 3, 2, 4, 5])
+        # Transpose to match HuggingFace F.unfold ordering (dim-major).
+        # F.unfold groups elements as [D, ms_h, ms_w] per spatial position,
+        # meaning the hidden dim D is the outermost loop. To reproduce
+        # this with reshape + transpose:
+        #   from: [batch, H/ms, ms, W/ms, ms, D]
+        #     to: [batch, H/ms, W/ms, D, ms, ms]
+        x = op.Transpose(x, perm=[0, 1, 3, 5, 2, 4])
 
-        # Flatten: [batch, (H/ms)*(W/ms), ms*ms*D]
+        # Flatten: [batch, (H/ms)*(W/ms), D*ms*ms]
         merged_count = op.Mul(h_m_1d, w_m_1d)
         shape_3d = op.Concat(
             batch,
diff --git a/src/mobius/components/_pixtral_vision_test.py b/src/mobius/components/_pixtral_vision_test.py
index e1eae7e9..7e8d6374 100644
--- a/src/mobius/components/_pixtral_vision_test.py
+++ b/src/mobius/components/_pixtral_vision_test.py
@@ -6,6 +6,8 @@
 from __future__ import annotations
 
 import numpy as np
+import onnx_ir as ir
+import onnxruntime as ort
 
 from mobius._configs import ArchitectureConfig, VisionConfig
 from mobius.components._pixtral_vision import (
@@ -103,3 +105,98 @@ def test_patch_merger_builds():
     merger = Mistral3PatchMerger(hidden_size=32, spatial_merge_size=2)
     # input_dim = 32 * 2 * 2 = 128, output_dim = 32
     assert list(merger.merging_layer.weight.shape) == [32, 128]
+
+
+def test_patch_merger_matches_hf_unfold_ordering():
+    """PatchMerger element ordering matches HuggingFace F.unfold (dim-major).
+
+    HF uses ``F.unfold(image_grid, kernel_size=ms, stride=ms)`` which
+    groups elements as ``[D, ms_h, ms_w]`` per spatial position (dim is
+    the outermost loop). The ONNX implementation must reproduce this
+    ordering so the learned ``merging_layer`` projection is correct.
+    """
+    import tempfile
+
+    import torch
+    from onnxscript._internal.builder import GraphBuilder
+
+    hidden_size = 8
+    ms = 2
+    grid_h, grid_w = 4, 4
+    seq_len = grid_h * grid_w
+
+    rng = np.random.default_rng(42)
+    x = rng.standard_normal((1, seq_len, hidden_size)).astype(np.float32)
+
+    # HF reference: F.unfold ordering
+    x_torch = torch.from_numpy(x.squeeze(0))  # (seq_len, D)
+    image_grid = x_torch.view(grid_h, grid_w, hidden_size).permute(2, 0, 1).unsqueeze(0)
+    grid = torch.nn.functional.unfold(image_grid, kernel_size=ms, stride=ms)
+    hf_merged = grid.view(hidden_size * ms * ms, -1).t().numpy()  # (num_merged, D*ms*ms)
+
+    # Build ONNX model that performs only the reshape+transpose+flatten
+    # (no linear projection) so we can compare the raw merge ordering.
+    x_input = ir.Value(
+        name="x",
+        shape=ir.Shape([1, seq_len, hidden_size]),
+        type=ir.TensorType(ir.DataType.FLOAT),
+    )
+    gh_input = ir.Value(
+        name="grid_h",
+        shape=ir.Shape([]),
+        type=ir.TensorType(ir.DataType.INT64),
+    )
+    gw_input = ir.Value(
+        name="grid_w",
+        shape=ir.Shape([]),
+        type=ir.TensorType(ir.DataType.INT64),
+    )
+    graph = ir.Graph(
+        inputs=[x_input, gh_input, gw_input],
+        outputs=[],
+        nodes=[],
+        name="test_merge_ordering",
+        opset_imports={"": 23},
+    )
+    gb = GraphBuilder(graph)
+    op = gb.op
+
+    # Reproduce the PatchMerger reshape+transpose+flatten logic
+    batch = op.Shape(x_input, start=0, end=1)
+    d = op.Shape(x_input, start=2, end=3)
+    ms_scalar = op.Constant(value_int=ms)
+    h_m = op.Div(gh_input, ms_scalar)
+    w_m = op.Div(gw_input, ms_scalar)
+    ms_1d = op.Constant(value_ints=[ms])
+    h_m_1d = op.Reshape(h_m, op.Constant(value_ints=[1]))
+    w_m_1d = op.Reshape(w_m, op.Constant(value_ints=[1]))
+    shape_6d = op.Concat(batch, h_m_1d, ms_1d, w_m_1d, ms_1d, d, axis=0)
+    merged = op.Reshape(x_input, shape_6d)
+    merged = op.Transpose(merged, perm=[0, 1, 3, 5, 2, 4])
+    merged_count = op.Mul(h_m_1d, w_m_1d)
+    shape_3d = op.Concat(batch, merged_count, op.Constant(value_ints=[-1]), axis=0)
+    result = op.Reshape(merged, shape_3d)
+    result.name = "output"
+    graph.outputs.append(result)
+
+    model = ir.Model(graph, ir_version=11)
+
+    with tempfile.NamedTemporaryFile(suffix=".onnx", delete=True) as f:
+        ir.save(model, f.name)
+        sess = ort.InferenceSession(f.name, providers=["CPUExecutionProvider"])
+        onnx_out = sess.run(
+            None,
+            {
+                "x": x,
+                "grid_h": np.array(grid_h, dtype=np.int64),
+                "grid_w": np.array(grid_w, dtype=np.int64),
+            },
+        )[0]
+
+    np.testing.assert_allclose(
+        onnx_out.squeeze(0),
+        hf_merged,
+        atol=1e-5,
+        rtol=1e-5,
+        err_msg="PatchMerger ordering does not match HF F.unfold",
+    )
diff --git a/src/mobius/components/_rotary_embedding.py b/src/mobius/components/_rotary_embedding.py
index 76047fb7..f1729a62 100644
--- a/src/mobius/components/_rotary_embedding.py
+++ b/src/mobius/components/_rotary_embedding.py
@@ -67,8 +67,11 @@ def apply_rotary_pos_emb(
     Args:
         op: The OpBuilder.
         x: Input tensor of shape ``(batch_size, seq_length, num_heads * head_dim)``.
-        position_embeddings: Tuple of ``(cos, sin)`` embeddings, each
-            ``(batch_size, seq_length, rotary_dim)``.
+        position_embeddings: Tuple of ``(cos, sin)`` or ``(cos, sin, attn_scale)``
+            embeddings. The cos/sin tensors have shape
+            ``(batch_size, seq_length, rotary_dim)``. The optional attn_scale
+            (used by Ministral3/Mistral4) is not consumed here — it is applied
+            separately in the attention module after RoPE.
         num_heads: Number of attention heads.
         rotary_embedding_dim: Dimension for partial RoPE (0 = full embedding).
         interleaved: If True, use interleaved RoPE layout where real/imag
@@ -78,7 +81,7 @@ def apply_rotary_pos_emb(
     Returns:
         Tensor with RoPE applied, same shape as input.
     """
-    cos, sin = position_embeddings
+    cos, sin = position_embeddings[0], position_embeddings[1]
     return op.RotaryEmbedding(
         x,
         cos,
@@ -223,8 +226,13 @@ def forward(self, op: builder.OpBuilder, position_ids: ir.Value):
 class YarnRope(BaseRope):
     """YaRN (Yet another RoPE extensioN) rotary embeddings.
 
-    Used by DeepSeek-V2/V3. Blends interpolated and extrapolated
-    frequencies with a linear ramp, and applies mscale attention factor.
+    Used by DeepSeek-V2/V3 and Ministral3 (Pixtral). Blends interpolated
+    and extrapolated frequencies with a linear ramp, and applies mscale
+    attention factor.
+
+    For Ministral3/Mistral4 models with ``llama_4_scaling_beta`` in
+    rope_scaling, ``forward()`` returns a 3-tuple ``(cos, sin, attn_scale)``
+    where ``attn_scale`` is a position-dependent query scaling factor.
 
     Reference: https://huggingface.co/papers/2309.00071
     """
@@ -291,6 +299,32 @@ def find_correction_dim(num_rotations):
         )
         super().__init__(cos_cache, sin_cache)
 
+        # Store llama_4_scaling_beta for Ministral3 position-dependent query scaling.
+        # When set, forward() returns (cos, sin, attn_scale) instead of (cos, sin).
+        self._llama4_beta = rope_scaling.get("llama_4_scaling_beta")
+        self._llama4_original_max_pos = float(original_max_pos)
+
+    def forward(self, op: builder.OpBuilder, position_ids: ir.Value):
+        cos_sin = get_rotary_pos_emb(op, position_ids, self.cos_cache, self.sin_cache)
+        if self._llama4_beta is None:
+            return cos_sin
+
+        # Compute position-dependent attention scale for Ministral3/Mistral4:
+        #   scale = 1 + beta * log(1 + floor(position_ids / original_max_pos))
+        # For pos < original_max_pos: floor(pos / max) = 0 → scale = 1.0
+        # Computed in FP32 for precision, then cast to match model dtype.
+        pos_float = op.Cast(position_ids, to=ir.DataType.FLOAT)
+        floored = op.Floor(op.Div(pos_float, float(self._llama4_original_max_pos)))
+        log_term = op.Log(op.Add(floored, 1.0))
+        attn_scale = op.Add(op.Mul(log_term, float(self._llama4_beta)), 1.0)
+        # Cast to match model dtype (e.g. FP16) — cos_cache has the target dtype
+        cos_dtype = self.cos_cache.dtype
+        if cos_dtype is not None and cos_dtype != ir.DataType.FLOAT:
+            attn_scale = op.Cast(attn_scale, to=cos_dtype)
+        # Unsqueeze to [batch, seq_len, 1] for broadcasting with 3D query states
+        attn_scale = op.Unsqueeze(attn_scale, [-1])
+        return (cos_sin[0], cos_sin[1], attn_scale)
+
 
 class _MRopeBase(BaseRope):
     """Base class for multi-dimensional RoPE variants.
diff --git a/src/mobius/components/_rotary_embedding_test.py b/src/mobius/components/_rotary_embedding_test.py
index 71ee35d1..da87cce6 100644
--- a/src/mobius/components/_rotary_embedding_test.py
+++ b/src/mobius/components/_rotary_embedding_test.py
@@ -254,3 +254,70 @@ def test_get_rotary_pos_emb(self):
         cos, sin = get_rotary_pos_emb(op, pos_ids, cos_cache, sin_cache)
         assert cos is not None
         assert sin is not None
+
+
+class TestYarnRopeAttnScale:
+    """Tests for YarnRope with llama_4_attn_scale (Ministral3)."""
+
+    def test_yarn_without_attn_scale_returns_2tuple(self):
+        """Standard YaRN (no llama_4_scaling_beta) returns (cos, sin)."""
+        from mobius.components._rotary_embedding import YarnRope
+
+        config = make_config(
+            head_dim=128,
+            max_position_embeddings=16384,
+            rope_theta=1000000.0,
+            rope_scaling={
+                "rope_type": "yarn",
+                "factor": 16.0,
+                "beta_fast": 32.0,
+                "beta_slow": 1.0,
+                "mscale": 1.0,
+                "mscale_all_dim": 1.0,
+                "original_max_position_embeddings": 16384,
+            },
+        )
+        rope = YarnRope(config)
+        builder, op, _graph = create_test_builder()
+        pos_ids = create_test_input(builder, "pos_ids", [1, 4])
+        result = rope.forward(op, pos_ids)
+        assert len(result) == 2, "Without llama_4_scaling_beta, should return (cos, sin)"
+
+    def test_yarn_with_attn_scale_returns_3tuple(self):
+        """YaRN with llama_4_scaling_beta returns (cos, sin, attn_scale)."""
+        from mobius.components._rotary_embedding import YarnRope
+
+        config = make_config(
+            head_dim=128,
+            max_position_embeddings=262144,
+            rope_theta=1000000.0,
+            rope_scaling={
+                "rope_type": "yarn",
+                "factor": 16.0,
+                "beta_fast": 32.0,
+                "beta_slow": 1.0,
+                "mscale": 1.0,
+                "mscale_all_dim": 1.0,
+                "original_max_position_embeddings": 16384,
+                "llama_4_scaling_beta": 0.1,
+            },
+        )
+        rope = YarnRope(config)
+        builder, op, _graph = create_test_builder()
+        pos_ids = create_test_input(builder, "pos_ids", [1, 4])
+        result = rope.forward(op, pos_ids)
+        assert len(result) == 3, (
+            "With llama_4_scaling_beta, should return (cos, sin, attn_scale)"
+        )
+
+    def test_apply_rotary_pos_emb_ignores_3rd_element(self):
+        """apply_rotary_pos_emb should work with both 2-tuple and 3-tuple."""
+        builder, op, _graph = create_test_builder()
+        x = create_test_input(builder, "x", [1, 4, 64])
+        cos = create_test_input(builder, "cos", [1, 4, 8])
+        sin = create_test_input(builder, "sin", [1, 4, 8])
+        scale = create_test_input(builder, "scale", [1, 4, 1])
+
+        # 3-tuple should work — apply_rotary_pos_emb only uses [0] and [1]
+        result = apply_rotary_pos_emb(op, x, (cos, sin, scale), num_heads=4)
+        assert result is not None
diff --git a/src/mobius/models/llava.py b/src/mobius/models/llava.py
index 944d9006..484cf2f8 100644
--- a/src/mobius/models/llava.py
+++ b/src/mobius/models/llava.py
@@ -129,7 +129,20 @@ def forward(self, op: builder.OpBuilder, input_ids: ir.Value, image_features: ir
         indices = op.Sub(cumsum, op.Constant(value_int=1))
         indices = op.Clip(indices, op.Constant(value_int=0))
 
-        gathered = op.Gather(image_features, indices, axis=0)
+        # Pad image_features with one zero row so Gather is valid even when
+        # image_features is empty (text-only input: num_image_tokens == 0).
+        # The Where mask ensures the padding row is never used in the output.
+        pad_row = op.Expand(
+            op.CastLike(op.Constant(value_float=0.0), image_features),
+            op.Concat(
+                op.Constant(value_ints=[1]),
+                op.Shape(image_features, start=1, end=2),
+                axis=0,
+            ),
+        )
+        padded_features = op.Concat(image_features, pad_row, axis=0)
+
+        gathered = op.Gather(padded_features, indices, axis=0)
         return op.Where(image_mask_3d, gathered, text_embeds)
 
     def preprocess_weights(
diff --git a/src/mobius/tasks/__init__.py b/src/mobius/tasks/__init__.py
index 181dbca5..1a23964c 100644
--- a/src/mobius/tasks/__init__.py
+++ b/src/mobius/tasks/__init__.py
@@ -36,6 +36,7 @@
     "OPSET_VERSION",
     "ObjectDetectionTask",
     "Phi4MMMultiModalTask",
+    "PixtralVLTask",
     "Qwen3VLVisionLanguageTask",
     "QwenImageVAETask",
     "QwenVLTask",
@@ -87,6 +88,7 @@
 from mobius.tasks._vision_language_3model import (
     HybridQwenVLTask,
     MllamaVisionLanguageTask,
+    PixtralVLTask,
     QwenVLTask,
     VisionLanguageTask,
 )
@@ -110,6 +112,7 @@
     "vae": VAETask,
     "qwen-image-vae": QwenImageVAETask,
     "vision-language": VisionLanguageTask,
+    "pixtral-vl": PixtralVLTask,
     "mllama-vision-language": MllamaVisionLanguageTask,
     "qwen-vl": QwenVLTask,
     "hybrid-qwen-vl": HybridQwenVLTask,
diff --git a/src/mobius/tasks/_vision_language_3model.py b/src/mobius/tasks/_vision_language_3model.py
index dd5fcd98..9f6a9487 100644
--- a/src/mobius/tasks/_vision_language_3model.py
+++ b/src/mobius/tasks/_vision_language_3model.py
@@ -187,6 +187,57 @@ def build(
         return ModelPackage(models, config=config)
 
 
+class PixtralVLTask(VisionLanguageTask):
+    """Vision-language task with dynamic-resolution Pixtral vision encoder.
+
+    Pixtral's internal computations (patch embedding, 2D RoPE, spatial merge)
+    are fully dynamic — grid_h and grid_w are derived from ``op.Shape()`` at
+    runtime.  This subclass replaces the static ``image_size`` input shape
+    with symbolic ``height`` / ``width`` dimensions so the exported ONNX
+    model accepts variable-resolution images.
+
+    Constraints (enforced at runtime, not in the graph):
+      - H, W ≥ ``patch_size * spatial_merge_size`` (28 for Pixtral)
+      - H, W ≤ ``image_size`` (1540 for Pixtral) due to RoPE cache limits
+    """
+
+    def _build_vision(
+        self,
+        vision: nn.Module,
+        config: ArchitectureConfig,
+    ) -> ir.Model:
+        """Build Pixtral vision encoder with dynamic HxW input."""
+        batch = ir.SymbolicDim("batch")
+        height = ir.SymbolicDim("height")
+        width = ir.SymbolicDim("width")
+
+        pixel_values = ir.Value(
+            name="pixel_values",
+            shape=ir.Shape([batch, 3, height, width]),
+            type=ir.TensorType(config.dtype),
+        )
+
+        graph_inputs = [pixel_values]
+
+        graph, graph_builder = _make_graph(graph_inputs, name="vision")
+        op = graph_builder.op
+
+        image_features = vision(
+            op,
+            pixel_values=pixel_values,
+        )
+
+        # Squeeze batch dim: [batch, num_patches, hidden] → [num_patches, hidden]
+        # The runtime (ort-genai) expects rank-2 vision features because the
+        # vision encoder always processes one image at a time.
+        image_features = op.Squeeze(image_features, [0])
+
+        image_features.name = "image_features"
+        graph.outputs.append(image_features)
+
+        return _make_model(graph)
+
+
 class MllamaVisionLanguageTask(VisionLanguageTask):
     """Mllama VL task with cross-attention KV caching.
 
diff --git a/testdata/cases/vision-language/ministral-3-3b.yaml b/testdata/cases/vision-language/ministral-3-3b.yaml
new file mode 100644
index 00000000..81ca56ee
--- /dev/null
+++ b/testdata/cases/vision-language/ministral-3-3b.yaml
@@ -0,0 +1,19 @@
+model_id: "mistralai/Ministral-3-3B-Instruct-2512"
+revision: "main"
+task_type: "image-text-to-text"
+dtype: "float32"
+
+inputs:
+  prompts:
+    - "What is in this image?"
+  images:
+    - "pipeline-cat-chonk.jpeg"
+
+level: "L4+L5"
+
+generation:
+  max_new_tokens: 30
+  do_sample: false
+
+skip_reason: "Gated repo (mistralai/Ministral-3-3B-Instruct-2512 requires authentication)."
+notes: "Ministral 3-3B (Pixtral VLM). PixtralVisionTower + Mistral3MultiModalProjector + MistralDecoder. 3-model split (vision/embedding/decoder). Uses 2D RoPE for vision, YaRN 1D RoPE for text decoder."
diff --git a/tests/integration_test.py b/tests/integration_test.py
index fc6e1606..a8cb4dd3 100644
--- a/tests/integration_test.py
+++ b/tests/integration_test.py
@@ -1512,6 +1512,103 @@ def test_3model_vision_pipeline(self, model_id: str):
             atol=2e-1,
         )
 
+    def test_vision_features_parity(self, model_id: str):
+        """Vision model output features match HF PyTorch reference.
+
+        Catches regressions in:
+        - Attention/RotaryEmbedding op attribute types (the swapped INT/FLOAT bug)
+        - Weight dequantization correctness
+        - 2D RoPE positional encoding
+        - PatchMerger spatial reshaping
+        """
+        import math
+
+        pkg = _build_mistral3_3model(model_id)
+        hf_config = transformers.AutoConfig.from_pretrained(model_id)
+
+        torch_model, _, _ = load_torch_multimodal_model(model_id)
+        torch_model.eval()
+
+        # Use the standard test image
+        image = Image.open("testdata/pipeline-cat-chonk.jpeg").convert("RGB")
+        w, h = image.size
+
+        # HF Pixtral resize: scale longest side to max_image_size, ceil to patch_size
+        patch_size = hf_config.vision_config.image_size // (
+            hf_config.vision_config.image_size // hf_config.vision_config.patch_size
+        )
+        max_image_size = hf_config.vision_config.image_size
+        merge_size = getattr(hf_config.vision_config, "spatial_merge_size", 2)
+        effective_patch = patch_size * merge_size
+
+        scale = max_image_size / max(h, w)
+        new_h = math.ceil(h * scale / patch_size) * patch_size
+        new_w = math.ceil(w * scale / patch_size) * patch_size
+        if new_h % effective_patch != 0:
+            new_h = math.ceil(new_h / effective_patch) * effective_patch
+        if new_w % effective_patch != 0:
+            new_w = math.ceil(new_w / effective_patch) * effective_patch
+
+        resized = image.resize((new_w, new_h), Image.BICUBIC)
+        arr = np.array(resized, dtype=np.float32) / 255.0
+        mean = np.array([0.48145466, 0.4578275, 0.40821073])
+        std = np.array([0.26862954, 0.26130258, 0.27577711])
+        arr = (arr - mean) / std
+        pixel_values = np.transpose(arr, (2, 0, 1))[np.newaxis, ...]
+
+        # HF reference: vision_tower + multi_modal_projector
+        pv_torch = torch.from_numpy(pixel_values).to(torch_model.dtype)
+        with torch.no_grad():
+            raw = torch_model.model.vision_tower(pv_torch).last_hidden_state.squeeze(0)
+            hf_features = (
+                torch_model.model.multi_modal_projector(raw, torch.tensor([[new_h, new_w]]))
+                .float()
+                .numpy()
+            )
+
+        # ONNX vision model
+        vision_session = OnnxModelSession(pkg["vision"])
+        onnx_out = vision_session.run({"pixel_values": pixel_values.astype(np.float32)})
+        vision_session.close()
+        onnx_features = onnx_out["image_features"].astype(np.float32)
+
+        # Shape check
+        assert hf_features.shape == onnx_features.shape, (
+            f"Shape mismatch: HF={hf_features.shape}, ONNX={onnx_features.shape}"
+        )
+
+        # Cosine similarity (must be very high — catches attribute type bugs)
+        cosine_sim = np.dot(hf_features.flatten(), onnx_features.flatten()) / (
+            np.linalg.norm(hf_features) * np.linalg.norm(onnx_features)
+        )
+        assert cosine_sim > 0.99, (
+            f"Vision cosine similarity {cosine_sim:.6f} < 0.99 — "
+            f"ONNX vision model produces different features than HF. "
+            f"HF norm={np.linalg.norm(hf_features):.2f}, "
+            f"ONNX norm={np.linalg.norm(onnx_features):.2f}"
+        )
+
+        # Norm ratio (catches scale factor bugs like FP8 dequant issues)
+        norm_ratio = np.linalg.norm(onnx_features) / np.linalg.norm(hf_features)
+        assert 0.9 < norm_ratio < 1.1, (
+            f"Vision norm ratio {norm_ratio:.4f} outside [0.9, 1.1] — "
+            f"scale factor mismatch between ONNX and HF"
+        )
+
+        # Random independence check (catches attribute zero bugs)
+        rng = np.random.RandomState(42)
+        r1 = rng.randn(1, 3, new_h, new_w).astype(np.float32)
+        r2 = rng.randn(1, 3, new_h, new_w).astype(np.float32)
+        vision_session = OnnxModelSession(pkg["vision"])
+        o1 = vision_session.run({"pixel_values": r1})["image_features"].flatten()
+        o2 = vision_session.run({"pixel_values": r2})["image_features"].flatten()
+        vision_session.close()
+        random_cosine = np.dot(o1, o2) / (np.linalg.norm(o1) * np.linalg.norm(o2))
+        assert random_cosine < 0.9, (
+            f"Random input cosine similarity {random_cosine:.4f} > 0.9 — "
+            f"vision model is not differentiating inputs (possible broken attention)"
+        )
+
 
 # ---------------------------------------------------------------------------
 # Encoder-only models (BERT, DistilBERT, etc.)