From 2e94d9dd7c3c3eea2dd266bccee82682f6bd9667 Mon Sep 17 00:00:00 2001 From: Justin Chu Date: Fri, 7 Aug 2026 17:48:05 -0700 Subject: [PATCH 1/6] Add MiniCPM-V 4.6 support Implement the packed variable-resolution SigLIP2 vision tower, ragged crop batching, 16x/4x mergers, Qwen3.5 hybrid decoder integration, image/video token mixing, config extraction, standardized three-model export, and ORT GenAI metadata. Add synthetic and real-weight parity tests plus committed image prefill and deterministic generation goldens. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: Justin Chu --- README.md | 2 +- src/mobius/_configs/_base.py | 2 + src/mobius/_configs/_sub_configs.py | 5 + src/mobius/_configs/per_model/__init__.py | 1 + .../_configs/per_model/_minicpmv4_6_vision.py | 38 + src/mobius/_registry.py | 6 + .../integrations/ort_genai/auto_export.py | 31 +- src/mobius/models/__init__.py | 2 + src/mobius/models/minicpmv4_6.py | 732 ++++++++++++++++++ src/mobius/models/minicpmv4_6_test.py | 246 ++++++ src/mobius/tasks/__init__.py | 3 + src/mobius/tasks/_vision_language_3model.py | 62 ++ .../cases/vision-language/minicpm-v-4_6.yaml | 19 + .../golden/vision-language/minicpm-v-4_6.json | 384 +++++++++ .../minicpm-v-4_6_generation.json | 37 + tests/_test_configs.py | 38 + tests/integration_test.py | 69 ++ tests/ort_genai_test.py | 23 + 18 files changed, 1698 insertions(+), 2 deletions(-) create mode 100644 src/mobius/_configs/per_model/_minicpmv4_6_vision.py create mode 100644 src/mobius/models/minicpmv4_6.py create mode 100644 src/mobius/models/minicpmv4_6_test.py create mode 100644 testdata/cases/vision-language/minicpm-v-4_6.yaml create mode 100644 testdata/golden/vision-language/minicpm-v-4_6.json create mode 100644 testdata/golden/vision-language/minicpm-v-4_6_generation.json diff --git a/README.md b/README.md index f9f2320b6..da21660e2 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ multi-component export for pipelines. |---|---| | **Text Generation** | Llama 2/3/4, Mistral, Qwen 2/2.5/3/3.5/3.6, Phi-3/3.5, Gemma 1/2/3/4, Granite, GPT-2, OPT, OLMo, SmolLM3, and many more | | **Mixture of Experts** | PhiMoE, GPTOSS, Mixtral, OLMoE, DeepSeek-V2/V3, Qwen2-MoE, Qwen3-MoE, Qwen3-Next, GLM-4-MoE, Arctic, DBRX, Jamba | -| **Multimodal** | Gemma 3/4, Phi-4MM (vision + audio + LoRA), LLaVA, InternVL2, Qwen2.5-VL, Qwen3-VL, Qwen3.5/3.6-VL, Pixtral | +| **Multimodal** | Gemma 3/4, Phi-4MM (vision + audio + LoRA), LLaVA, InternVL2, MiniCPM-V 4.6, Qwen2.5-VL, Qwen3-VL, Qwen3.5/3.6-VL, Pixtral | | **Encoder-only** | BERT, RoBERTa, ALBERT, DeBERTa, DistilBERT, ELECTRA, XLNet | | **Encoder-Decoder** | BART, T5/mT5, Marian, M2M-100, Pegasus, BigBird-Pegasus | | **Speech-to-Text** | Whisper, FastConformer-RNNT, FunASR, Qwen3-ASR, SenseVoice | diff --git a/src/mobius/_configs/_base.py b/src/mobius/_configs/_base.py index 475b69b9a..d5fd36ece 100644 --- a/src/mobius/_configs/_base.py +++ b/src/mobius/_configs/_base.py @@ -494,6 +494,8 @@ class ArchitectureConfig(BaseModelConfig): # Vision shared fields (accessed as top-level config.X by tasks) mm_tokens_per_image: int | None = None image_token_id: int | None = None + video_token_id: int | None = None + downsample_mode: str = "16x" spatial_merge_size: int = 2 temporal_patch_size: int = 2 deepstack_visual_indexes: list[int] | None = None diff --git a/src/mobius/_configs/_sub_configs.py b/src/mobius/_configs/_sub_configs.py index 79dfddc5f..a06e4924a 100644 --- a/src/mobius/_configs/_sub_configs.py +++ b/src/mobius/_configs/_sub_configs.py @@ -82,6 +82,11 @@ class VisionConfig: # (HuggingFace convention, e.g. -2 for Phi-3.5-Vision). ``None`` means use # the final hidden state (all layers + post_layernorm). feature_layer: int | None = None + # MiniCPM-V packed-NaViT vision encoder and its two spatial mergers. + insert_layer_id: int | None = None + window_kernel_size: tuple[int, int] = (2, 2) + merge_kernel_size: tuple[int, int] = (2, 2) + merger_times: int = 1 @dataclasses.dataclass diff --git a/src/mobius/_configs/per_model/__init__.py b/src/mobius/_configs/per_model/__init__.py index 28e637637..a57e2ba55 100644 --- a/src/mobius/_configs/per_model/__init__.py +++ b/src/mobius/_configs/per_model/__init__.py @@ -29,6 +29,7 @@ _gemma4_unified_vision, _hunyuan_vl_mot_vision, _internvl_vision, + _minicpmv4_6_vision, _phi4mm_audio, _phi4mm_vision, _phi_vision, diff --git a/src/mobius/_configs/per_model/_minicpmv4_6_vision.py b/src/mobius/_configs/per_model/_minicpmv4_6_vision.py new file mode 100644 index 000000000..cd091a854 --- /dev/null +++ b/src/mobius/_configs/per_model/_minicpmv4_6_vision.py @@ -0,0 +1,38 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""MiniCPM-V-4.6 vision extractor hook.""" + +from __future__ import annotations + +from mobius._configs._extractors import register_vision_hook + + +@register_vision_hook("qwen3_5_text") +def _minicpmv4_6_vision(config, parent_config, model_type: str, fields: dict): + """Extract the packed SigLIP2 geometry from the composite MiniCPM config.""" + if getattr(parent_config, "model_type", None) != "minicpmv4_6": + return None + + vision = parent_config.vision_config + image_size = getattr(vision, "image_size", 980) + patch_size = getattr(vision, "patch_size", 14) + fields.update( + model_type="minicpmv4_6_vision", + hidden_size=vision.hidden_size, + intermediate_size=vision.intermediate_size, + num_hidden_layers=vision.num_hidden_layers, + num_attention_heads=vision.num_attention_heads, + image_size=image_size, + patch_size=patch_size, + norm_eps=vision.layer_norm_eps, + hidden_act=getattr(vision, "hidden_act", "gelu_pytorch_tanh"), + in_channels=getattr(vision, "num_channels", 3), + num_position_embeddings=(image_size // patch_size) ** 2, + image_token_id=parent_config.image_token_id, + insert_layer_id=getattr(parent_config, "insert_layer_id", 6), + window_kernel_size=tuple(getattr(vision, "window_kernel_size", (2, 2))), + merge_kernel_size=tuple(getattr(parent_config, "merge_kernel_size", (2, 2))), + merger_times=getattr(parent_config, "merger_times", 1), + ) + return None diff --git a/src/mobius/_registry.py b/src/mobius/_registry.py index 1c611e97c..4a90198e6 100644 --- a/src/mobius/_registry.py +++ b/src/mobius/_registry.py @@ -126,6 +126,7 @@ from mobius.models.llava import LLaVAModel from mobius.models.longcat_flash import LongcatFlashCausalLMModel from mobius.models.mamba import Mamba2CausalLMModel, MambaCausalLMModel +from mobius.models.minicpmv4_6 import MiniCPMV46ForConditionalGeneration from mobius.models.minimax import MiniMaxCausalLMModel from mobius.models.mllama import MllamaCausalLMModel from mobius.models.modernbert import ModernBertDecoderModel, ModernBertModel @@ -598,6 +599,10 @@ def _detect_fallback_registration(hf_config) -> ModelRegistration | None: "llava_next_video": ModelRegistration(LLaVAModel, task="vision-language"), "llava_onevision": ModelRegistration(LLaVAModel, task="vision-language"), "mistral3": ModelRegistration(LLaVAModel, task="pixtral-vl"), + "minicpmv4_6": ModelRegistration( + MiniCPMV46ForConditionalGeneration, + task="minicpm-vl", + ), "mllama": ModelRegistration(MllamaCausalLMModel, task="mllama-vision-language"), "molmo": ModelRegistration(LLaVAModel, task="vision-language"), "ovis2": ModelRegistration(LLaVAModel, task="vision-language"), @@ -979,6 +984,7 @@ def _create_default_registry() -> ModelRegistry: "llava_onevision": "llava-hf/llava-onevision-qwen2-0.5b-ov-hf", "molmo": "allenai/MolmoE-1B-0924", "mistral3": "mistralai/Ministral-3-3B-Instruct-2512", + "minicpmv4_6": "openbmb/MiniCPM-V-4.6", "aya_vision": "CohereForAI/aya-vision-8b", "chameleon": "facebook/chameleon-7b", "cohere2_vision": "CohereForAI/c4ai-command-r7b-12-2024", diff --git a/src/mobius/integrations/ort_genai/auto_export.py b/src/mobius/integrations/ort_genai/auto_export.py index 3b4786b5d..8c9dc4964 100644 --- a/src/mobius/integrations/ort_genai/auto_export.py +++ b/src/mobius/integrations/ort_genai/auto_export.py @@ -93,6 +93,10 @@ "qwen3_vl_text": "qwen3_vl", "qwen3_5": "qwen2_5_vl", "qwen3_5_vl": "qwen2_5_vl", + # MiniCPM uses standard 1D decoder position IDs (unlike Qwen-VL MRoPE). + # The phi3v multimodal runtime provides that contract; callers supply + # HF-preprocessed packed pixels through Generator.set_inputs(). + "minicpmv4_6": "phi3v", } _GEMMA4_MODEL_TYPES = frozenset( @@ -107,6 +111,7 @@ # must preprocess with the HuggingFace processor and feed tensors via # ``Generator.set_inputs`` (see examples/gemma4_unified_ort_genai.py). _GEMMA4_UNIFIED_MODEL_TYPES = frozenset({"gemma4_unified", "gemma4_unified_text"}) +_MINICPM_MODEL_TYPES = frozenset({"minicpmv4_6"}) # gemma-3 multimodal. build() unwraps the composite HF config to its text # sub-config, so at export time ``config.model_type`` is "gemma3_text" (not # "gemma3"). @@ -134,6 +139,9 @@ "merges.txt", # BPE "vocab.json", # BPE "chat_template.jinja", # Chat template for ORT GenAI + # Preserve HuggingFace processor metadata for VLMs whose preprocessing + # cannot be represented by an ort-extensions image_processor.json. + "preprocessor_config.json", ] @@ -463,6 +471,17 @@ def _write_vision_processor_config( model_type, ) return None + if model_type in _MINICPM_MODEL_TYPES: + # MiniCPM needs adaptive slicing and NaViT horizontal patch packing. + # ort-extensions has no equivalent transform, so preserving the HF + # processor output and injecting it through set_inputs is the only + # numerically faithful runtime path. + logger.info( + "Skipping image_processor.json for %s " + "(use MiniCPMV4_6Processor + Generator.set_inputs)", + model_type, + ) + return None vision_model_type = getattr(vision, "model_type", None) is_pixtral = vision_model_type == "pixtral" or model_type in _PIXTRAL_MODEL_TYPES @@ -848,16 +867,26 @@ def _write_genai_config( if image_token_id is not None: vision_input_mapping = _introspect_inputs(pkg, "vision_encoder") embedding_input_mapping = _introspect_inputs(pkg, "embedding") + if ( + model_type := getattr(config, "model_type", "") + ) in _MINICPM_MODEL_TYPES and vision_input_mapping is not None: + # ORT GenAI's VisionInputs schema only accepts its predefined + # semantic keys. ``target_sizes`` remains an ONNX graph input + # and is supplied as a named tensor through set_inputs(). + vision_input_mapping.pop("target_sizes", None) # spatial_merge_size and config_filename are config-level # properties that cannot be inferred from the graph. vision_kwargs: dict[str, Any] = {} - model_type = getattr(config, "model_type", "") if model_type in _GEMMA4_MODEL_TYPES: vision_cfg = getattr(config, "vision", None) vision_kwargs["spatial_merge_size"] = getattr( vision_cfg, "spatial_merge_size", 2 ) + elif model_type in _MINICPM_MODEL_TYPES: + # MiniCPM performs both 2x2 merges inside the ONNX vision + # graph and consumes HF-prepacked pixels, not Qwen grid_thw. + vision_kwargs["spatial_merge_size"] = None elif has_speech: vision_kwargs["spatial_merge_size"] = None elif ( diff --git a/src/mobius/models/__init__.py b/src/mobius/models/__init__.py index 9ca9e2ce3..7755d2068 100644 --- a/src/mobius/models/__init__.py +++ b/src/mobius/models/__init__.py @@ -80,6 +80,7 @@ "Mamba2CausalLMModel", "MambaCausalLMModel", "MiniMaxCausalLMModel", + "MiniCPMV46ForConditionalGeneration", "MimiModel", "MoshiDepformerModel", "MoshiTemporalModel", @@ -217,6 +218,7 @@ from mobius.models.longcat_flash import LongcatFlashCausalLMModel from mobius.models.mamba import Mamba2CausalLMModel, MambaCausalLMModel from mobius.models.mimi import MimiModel, mimi_default_config +from mobius.models.minicpmv4_6 import MiniCPMV46ForConditionalGeneration from mobius.models.minimax import MiniMaxCausalLMModel from mobius.models.moe import ( Ernie45MoECausalLMModel, diff --git a/src/mobius/models/minicpmv4_6.py b/src/mobius/models/minicpmv4_6.py new file mode 100644 index 000000000..19846e40d --- /dev/null +++ b/src/mobius/models/minicpmv4_6.py @@ -0,0 +1,732 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""OpenBMB MiniCPM-V-4.6 vision-language model. + +Replicates ``MiniCPMV4_6ForConditionalGeneration`` from Transformers 5.7: +packed variable-resolution SigLIP2 vision, an in-tower window-attention +merger, a second MLP spatial merger, and the Qwen3.5 hybrid text decoder. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import torch +from onnxscript import OpBuilder, nn + +from mobius._configs import ArchitectureConfig +from mobius.components import ( + Embedding, + LayerNorm, + Linear, + VisionAttention, + VisionEncoderLayer, + get_activation, +) +from mobius.models.qwen35 import Qwen35TextModel + +if TYPE_CHECKING: + import onnx_ir as ir + + +class MiniCPMV46Config(ArchitectureConfig): + """Mobius configuration preserving MiniCPM's composite model identity.""" + + @classmethod + def from_transformers(cls, config, parent_config=None) -> MiniCPMV46Config: + result = super().from_transformers(config, parent_config=parent_config) + composite = parent_config or config + result.model_type = "minicpmv4_6" + result.video_token_id = getattr(composite, "video_token_id", None) + result.downsample_mode = getattr(composite, "downsample_mode", "16x") + return result + + +def _grid_columns(op: OpBuilder, target_sizes: ir.Value) -> tuple[ir.Value, ir.Value]: + """Return the per-visual-unit patch-grid heights and widths.""" + target_sizes = op.Cast(target_sizes, to=7) + heights = op.Squeeze( + op.Slice(target_sizes, [0], [1], axes=[1]), + [1], + ) + widths = op.Squeeze( + op.Slice(target_sizes, [1], [2], axes=[1]), + [1], + ) + return heights, widths + + +def _packed_patch_coordinates( + op: OpBuilder, target_sizes: ir.Value +) -> tuple[ir.Value, ir.Value, ir.Value]: + """Map each packed patch to ``(visual_unit, row, column)``.""" + heights, widths = _grid_columns(op, target_sizes) + lengths = op.Mul(heights, widths) + ends = op.CumSum(lengths, 0) + starts = op.Sub(ends, lengths) + total = op.ReduceSum(lengths, keepdims=0) + packed_index = op.Range(0, total, 1) + + # The first cumulative end greater than each packed index identifies its + # source image crop or video frame. + before_end = op.Less( + op.Unsqueeze(packed_index, [1]), + op.Unsqueeze(ends, [0]), + ) + visual_index = op.ArgMax( + op.Cast(before_end, to=7), + axis=1, + keepdims=0, + ) + local_index = op.Sub(packed_index, op.Gather(starts, visual_index)) + local_width = op.Gather(widths, visual_index) + rows = op.Div(local_index, local_width) + columns = op.Mod(local_index, local_width) + return visual_index, rows, columns + + +def _max_grid_size(op: OpBuilder, target_sizes: ir.Value) -> tuple[ir.Value, ir.Value]: + heights, widths = _grid_columns(op, target_sizes) + return ( + op.ReduceMax(heights, keepdims=1), + op.ReduceMax(widths, keepdims=1), + ) + + +def _grid_mask( + op: OpBuilder, + target_sizes: ir.Value, + max_height: ir.Value, + max_width: ir.Value, +) -> ir.Value: + """Boolean ``[N, max_h, max_w]`` mask for a padded ragged patch grid.""" + heights, widths = _grid_columns(op, target_sizes) + row_ids = op.Range(0, op.Squeeze(max_height, [0]), 1) + column_ids = op.Range(0, op.Squeeze(max_width, [0]), 1) + valid_rows = op.Less( + op.Unsqueeze(row_ids, [0]), + op.Unsqueeze(heights, [1]), + ) + valid_columns = op.Less( + op.Unsqueeze(column_ids, [0]), + op.Unsqueeze(widths, [1]), + ) + return op.And( + op.Unsqueeze(valid_rows, [2]), + op.Unsqueeze(valid_columns, [1]), + ) + + +def _unpack_padded_grid( + op: OpBuilder, + hidden_states: ir.Value, + target_sizes: ir.Value, + hidden_size: int, +) -> ir.Value: + """Scatter packed patches into ``[N, max_h*max_w, D]`` padded batches.""" + visual_index, rows, columns = _packed_patch_coordinates(op, target_sizes) + coordinates = op.Concat( + op.Unsqueeze(visual_index, [1]), + op.Unsqueeze(rows, [1]), + op.Unsqueeze(columns, [1]), + axis=1, + ) + max_height, max_width = _max_grid_size(op, target_sizes) + grid_shape = op.Concat( + op.Shape(target_sizes, start=0, end=1), + max_height, + max_width, + op.Constant(value_ints=[hidden_size]), + axis=0, + ) + patches = op.Squeeze(hidden_states, [0]) + padded = op.Expand(op.CastLike(0.0, patches), grid_shape) + padded = op.ScatterND(padded, coordinates, patches) + return op.Reshape( + padded, + op.Concat( + op.Shape(target_sizes, start=0, end=1), + op.Mul(max_height, max_width), + op.Constant(value_ints=[hidden_size]), + axis=0, + ), + ) + + +def _vision_attention_bias( + op: OpBuilder, + hidden_states: ir.Value, + target_sizes: ir.Value, +) -> ir.Value: + """Build a full additive mask for padded per-image patch batches.""" + max_height, max_width = _max_grid_size(op, target_sizes) + valid = op.Reshape( + _grid_mask(op, target_sizes, max_height, max_width), + op.Concat( + op.Shape(target_sizes, start=0, end=1), + op.Mul(max_height, max_width), + axis=0, + ), + ) + key_bias = op.Where( + op.Unsqueeze(valid, [1, 2]), + op.CastLike(0.0, hidden_states), + op.CastLike(-10_000.0, hidden_states), + ) + sequence_length = op.Mul(max_height, max_width) + return op.Expand( + key_bias, + op.Concat( + op.Shape(target_sizes, start=0, end=1), + op.Constant(value_ints=[1]), + sequence_length, + sequence_length, + axis=0, + ), + ) + + +def _spatial_windows( + op: OpBuilder, + hidden_states: ir.Value, + target_sizes: ir.Value, + hidden_size: int, + kernel_size: tuple[int, int], +) -> tuple[ir.Value, ir.Value, ir.Value]: + """Arrange ``[B, H*W, D]`` as ``[B, H/k, W/k, k, k, D]`` windows.""" + batch = op.Shape(target_sizes, start=0, end=1) + height, width = _max_grid_size(op, target_sizes) + kernel_h, kernel_w = kernel_size + merged_h = op.Div(height, op.Constant(value_ints=[kernel_h])) + merged_w = op.Div(width, op.Constant(value_ints=[kernel_w])) + grid = op.Reshape( + hidden_states, + op.Concat( + batch, + merged_h, + op.Constant(value_ints=[kernel_h]), + merged_w, + op.Constant(value_ints=[kernel_w, hidden_size]), + axis=0, + ), + ) + # (B, H/k, k, W/k, k, D) -> (B, H/k, W/k, k, k, D) + grid = op.Transpose(grid, perm=[0, 1, 3, 2, 4, 5]) + return grid, merged_h, merged_w + + +class _MiniCPMVisionEmbeddings(nn.Module): + """NaViT patch embedding with nearest-neighbor learned 2D positions.""" + + def __init__( + self, + image_size: int, + patch_size: int, + hidden_size: int, + num_channels: int, + ): + super().__init__() + self.patch_size = patch_size + self.hidden_size = hidden_size + self.position_side = image_size // patch_size + self.patch_embedding = nn.Parameter( + [hidden_size, num_channels, patch_size, patch_size], + name="patch_embedding.weight", + ) + self.patch_embedding_bias = nn.Parameter([hidden_size], name="patch_embedding.bias") + self.position_embedding = nn.Parameter( + [self.position_side**2, hidden_size], + name="position_embedding.weight", + ) + + def forward( + self, + op: OpBuilder, + pixel_values: ir.Value, + target_sizes: ir.Value, + ): + # NaViT input is packed horizontally: (1, C, patch_size, total_patch_width). + patches = op.Conv( + pixel_values, + self.patch_embedding, + self.patch_embedding_bias, + kernel_shape=[self.patch_size, self.patch_size], + strides=[self.patch_size, self.patch_size], + ) + # (packed_batch, D, 1, patches) -> (1, total_patches, D) + patches = op.Transpose(patches, perm=[0, 2, 3, 1]) + patches = op.Reshape( + patches, + op.Constant(value_ints=[1, -1, self.hidden_size]), + ) + + visual_index, rows, columns = _packed_patch_coordinates(op, target_sizes) + heights, widths = _grid_columns(op, target_sizes) + patch_heights = op.Gather(heights, visual_index) + patch_widths = op.Gather(widths, visual_index) + side = op.Constant(value_int=self.position_side) + bucket_h = op.Div(op.Mul(rows, side), patch_heights) + bucket_w = op.Div(op.Mul(columns, side), patch_widths) + position_ids = op.Add(op.Mul(bucket_h, side), bucket_w) + position_embeddings = op.Gather(self.position_embedding, position_ids) + return op.Add(patches, op.Unsqueeze(position_embeddings, [0])) + + +class _MiniCPMViTWindowAttentionMerger(nn.Module): + """2x2 local attention followed by spatial MLP compression.""" + + def __init__(self, config: ArchitectureConfig): + super().__init__() + vision = config.vision + assert vision is not None + assert vision.hidden_size is not None + assert vision.intermediate_size is not None + self.hidden_size = vision.hidden_size + self.kernel_size = vision.window_kernel_size + window_tokens = self.kernel_size[0] * self.kernel_size[1] + window_hidden = self.hidden_size * window_tokens + window_intermediate = vision.intermediate_size * window_tokens + self.self_attn = VisionAttention(self.hidden_size, vision.num_attention_heads or 1) + self.layer_norm1 = LayerNorm(self.hidden_size, eps=vision.norm_eps) + self.pre_norm = LayerNorm(window_hidden, eps=vision.norm_eps) + self.linear_1 = Linear(window_hidden, window_intermediate, bias=True) + self.act = get_activation("gelu_pytorch_tanh") + self.linear_2 = Linear(window_intermediate, self.hidden_size, bias=True) + + def forward( + self, + op: OpBuilder, + hidden_states: ir.Value, + target_sizes: ir.Value, + ): + residual = hidden_states + normalized = self.layer_norm1(op, hidden_states) + grid, merged_h, merged_w = _spatial_windows( + op, normalized, target_sizes, self.hidden_size, self.kernel_size + ) + window_tokens = self.kernel_size[0] * self.kernel_size[1] + windows = op.Reshape( + grid, + op.Constant(value_ints=[-1, window_tokens, self.hidden_size]), + ) + windows = self.self_attn(op, windows) + grid = op.Reshape( + windows, + op.Concat( + op.Shape(target_sizes, start=0, end=1), + merged_h, + merged_w, + op.Constant( + value_ints=[ + self.kernel_size[0], + self.kernel_size[1], + self.hidden_size, + ] + ), + axis=0, + ), + ) + # Restore raster order before the attention residual. + grid = op.Transpose(grid, perm=[0, 1, 3, 2, 4, 5]) + attended = op.Reshape( + grid, + op.Concat( + op.Shape(target_sizes, start=0, end=1), + op.Mul( + op.Mul(merged_h, merged_w), + op.Constant(value_ints=[window_tokens]), + ), + op.Constant(value_ints=[self.hidden_size]), + axis=0, + ), + ) + hidden_states = op.Add(residual, attended) + + # Merge each 2x2 block: (B, H, W, D) -> (B*H/2*W/2, 4D). + merged_grid, _, _ = _spatial_windows( + op, hidden_states, target_sizes, self.hidden_size, self.kernel_size + ) + merge_residual = op.ReduceMean(merged_grid, [3, 4], keepdims=0) + merge_residual = op.Reshape( + merge_residual, + op.Constant(value_ints=[-1, self.hidden_size]), + ) + flat = op.Reshape( + merged_grid, + op.Constant( + value_ints=[ + -1, + window_tokens * self.hidden_size, + ] + ), + ) + flat = self.pre_norm(op, flat) + flat = self.linear_1(op, flat) + flat = self.act(op, flat) + flat = self.linear_2(op, flat) + flat = op.Add(flat, merge_residual) + return op.Reshape( + flat, + op.Concat( + op.Shape(target_sizes, start=0, end=1), + op.Mul(merged_h, merged_w), + op.Constant(value_ints=[self.hidden_size]), + axis=0, + ), + ) + + +class _MiniCPMVisionTower(nn.Module): + """Variable-resolution SigLIP2 tower with an inserted attention merger.""" + + def __init__(self, config: ArchitectureConfig): + super().__init__() + vision = config.vision + assert vision is not None + assert vision.image_size is not None + assert vision.patch_size is not None + assert vision.hidden_size is not None + assert vision.intermediate_size is not None + assert vision.num_hidden_layers is not None + assert vision.num_attention_heads is not None + self.hidden_size = vision.hidden_size + self.insert_layer_id = ( + vision.insert_layer_id if vision.insert_layer_id is not None else 6 + ) + self.use_vit_merger = config.downsample_mode != "4x" + self.embeddings = _MiniCPMVisionEmbeddings( + vision.image_size, + vision.patch_size, + vision.hidden_size, + vision.in_channels, + ) + self.encoder = _MiniCPMVisionEncoder(config) + self.post_layernorm = LayerNorm(vision.hidden_size, eps=vision.norm_eps) + + def forward( + self, + op: OpBuilder, + pixel_values: ir.Value, + target_sizes: ir.Value, + ): + hidden_states = self.embeddings(op, pixel_values, target_sizes) + hidden_states = _unpack_padded_grid( + op, + hidden_states, + target_sizes, + self.hidden_size, + ) + hidden_states = self.encoder( + op, + hidden_states, + target_sizes=target_sizes, + insert_layer_id=self.insert_layer_id if self.use_vit_merger else -1, + ) + return self.post_layernorm(op, hidden_states) + + +class _MiniCPMVisionEncoder(nn.Module): + """Container matching HuggingFace's ``vision_tower.encoder.layers`` names.""" + + def __init__(self, config: ArchitectureConfig): + super().__init__() + vision = config.vision + assert vision is not None + assert vision.hidden_size is not None + assert vision.intermediate_size is not None + assert vision.num_hidden_layers is not None + assert vision.num_attention_heads is not None + self.layers = nn.ModuleList( + [ + VisionEncoderLayer( + vision.hidden_size, + vision.intermediate_size, + vision.num_attention_heads, + vision.norm_eps, + ) + for _ in range(vision.num_hidden_layers) + ] + ) + # The upstream module registers vit_merger beside encoder. Keeping it + # here lets the encoder call every layer through its own scope so layer + # initializer names retain ``encoder.layers.N``; preprocessing adds + # this one extra ``encoder.`` segment for the merger weights. + self.vit_merger = _MiniCPMViTWindowAttentionMerger(config) + + def forward( + self, + op: OpBuilder, + hidden_states: ir.Value, + target_sizes: ir.Value, + insert_layer_id: int, + ): + current_sizes = op.Cast(target_sizes, to=7) + attention_bias = _vision_attention_bias( + op, + hidden_states, + current_sizes, + ) + for layer_idx, layer in enumerate(self.layers): + hidden_states = layer(op, hidden_states, attention_bias) + if layer_idx == insert_layer_id: + hidden_states = self.vit_merger(op, hidden_states, current_sizes) + current_sizes = op.Div(current_sizes, 2) + attention_bias = _vision_attention_bias( + op, + hidden_states, + current_sizes, + ) + return hidden_states + + +class _MiniCPMDownsampleMLP(nn.Module): + """One 2x2 spatial merge and projection, matching ``merger.mlp.N``.""" + + def __init__(self, hidden_size: int, output_size: int): + super().__init__() + merged_hidden_size = hidden_size * 4 + self.pre_norm = LayerNorm(merged_hidden_size, eps=1e-6) + self.linear_1 = Linear(merged_hidden_size, merged_hidden_size, bias=True) + self.act = get_activation("gelu") + self.linear_2 = Linear(merged_hidden_size, output_size, bias=True) + + def forward(self, op: OpBuilder, hidden_states: ir.Value): + hidden_states = self.pre_norm(op, hidden_states) + hidden_states = self.linear_1(op, hidden_states) + hidden_states = self.act(op, hidden_states) + return self.linear_2(op, hidden_states) + + +class _MiniCPMMerger(nn.Module): + """Final spatial merger from vision width into the text embedding width.""" + + def __init__(self, config: ArchitectureConfig): + super().__init__() + vision = config.vision + assert vision is not None and vision.hidden_size is not None + self.hidden_size = vision.hidden_size + self.kernel_size = vision.merge_kernel_size + self.merger_times = vision.merger_times + self.vision_downsample = 1 if config.downsample_mode == "4x" else 2 + self.output_sizes = [ + self.hidden_size if i < self.merger_times - 1 else config.hidden_size + for i in range(self.merger_times) + ] + mlps = [ + _MiniCPMDownsampleMLP( + self.hidden_size, + self.output_sizes[i], + ) + for i in range(self.merger_times) + ] + self.mlp = nn.ModuleList(mlps) + + def forward( + self, + op: OpBuilder, + hidden_states: ir.Value, + target_sizes: ir.Value, + ): + # Default 16x mode enters after the in-tower 2x2 merger. In 4x mode + # the tower is unmerged, so this projector performs the only 2x2 merge. + target_sizes = op.Div( + op.Cast(target_sizes, to=7), + self.vision_downsample, + ) + current_hidden = self.hidden_size + for index, mlp in enumerate(self.mlp): + grid, merged_h, merged_w = _spatial_windows( + op, + hidden_states, + target_sizes, + current_hidden, + self.kernel_size, + ) + merged_dim = self.kernel_size[0] * self.kernel_size[1] * current_hidden + hidden_states = op.Reshape( + grid, + op.Constant(value_ints=[-1, merged_dim]), + ) + hidden_states = mlp(op, hidden_states) + target_sizes = op.Concat( + op.Div( + op.Slice(target_sizes, [0], [1], axes=[1]), + self.kernel_size[0], + ), + op.Div( + op.Slice(target_sizes, [1], [2], axes=[1]), + self.kernel_size[1], + ), + axis=1, + ) + current_hidden = self.output_sizes[index] + hidden_states = op.Reshape( + hidden_states, + op.Concat( + op.Shape(target_sizes, start=0, end=1), + op.Mul(merged_h, merged_w), + op.Shape(hidden_states, start=1, end=2), + axis=0, + ), + ) + max_height, max_width = _max_grid_size(op, target_sizes) + valid = op.Reshape( + _grid_mask(op, target_sizes, max_height, max_width), + [-1], + ) + hidden_states = op.Reshape( + hidden_states, + op.Concat( + op.Constant(value_ints=[-1]), + op.Shape(hidden_states, start=2, end=3), + axis=0, + ), + ) + return op.Compress(hidden_states, valid, axis=0) + + +class _MiniCPMVisionEncoderModel(nn.Module): + """Packed SigLIP2 encoder plus both MiniCPM visual token mergers.""" + + def __init__(self, config: ArchitectureConfig): + super().__init__() + self.vision_tower = _MiniCPMVisionTower(config) + self.merger = _MiniCPMMerger(config) + + def forward( + self, + op: OpBuilder, + pixel_values: ir.Value, + target_sizes: ir.Value, + ): + hidden_states = self.vision_tower(op, pixel_values, target_sizes) + return self.merger(op, hidden_states, target_sizes) + + +class _MiniCPMDecoderModel(nn.Module): + """Qwen3.5 hybrid decoder taking pre-fused ``inputs_embeds``.""" + + def __init__(self, config: ArchitectureConfig): + super().__init__() + self.model = Qwen35TextModel(config) + self.lm_head = Linear(config.hidden_size, config.vocab_size, bias=False) + + def forward( + self, + op: OpBuilder, + inputs_embeds: ir.Value, + attention_mask: ir.Value, + position_ids: ir.Value, + past_key_values: list | None = None, + ): + hidden_states, present_key_values = self.model( + op, + input_ids=None, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + ) + return self.lm_head(op, hidden_states), present_key_values + + +class _MiniCPMEmbeddingModel(nn.Module): + """Fuse packed image or video features into the token embedding sequence.""" + + def __init__(self, config: ArchitectureConfig): + super().__init__() + self.embed_tokens = Embedding( + config.vocab_size, config.hidden_size, config.pad_token_id + ) + self.image_token_id = config.image_token_id or 248056 + self.video_token_id = config.video_token_id or 248057 + + def forward( + self, + op: OpBuilder, + input_ids: ir.Value, + image_features: ir.Value, + ): + text_embeds = self.embed_tokens(op, input_ids) + media_mask = op.Or( + op.Equal(input_ids, self.image_token_id), + op.Equal(input_ids, self.video_token_id), + ) + media_indices = op.Clip( + op.Sub( + op.CumSum(op.Cast(media_mask, to=7), 1), + 1, + ), + 0, + ) + # Keep text-only inference valid when the feature tensor has zero rows. + padding = op.Expand( + op.CastLike(0.0, image_features), + op.Concat( + op.Constant(value_ints=[1]), + op.Shape(image_features, start=1, end=2), + axis=0, + ), + ) + features = op.Gather( + op.Concat(image_features, padding, axis=0), + media_indices, + axis=0, + ) + return op.Where(op.Unsqueeze(media_mask, [-1]), features, text_embeds) + + +class MiniCPMV46ForConditionalGeneration(nn.Module): + """OpenBMB MiniCPM-V-4.6 image/video conditional generation model.""" + + default_task: str = "minicpm-vl" + category: str = "Multimodal" + config_class: type = MiniCPMV46Config + + def __init__(self, config: ArchitectureConfig): + super().__init__() + self.config = config + self.decoder = _MiniCPMDecoderModel(config) + self.vision_encoder = _MiniCPMVisionEncoderModel(config) + self.embedding = _MiniCPMEmbeddingModel(config) + + def forward(self, op: OpBuilder, **kwargs): + raise NotImplementedError( + "MiniCPMV46ForConditionalGeneration uses MiniCPMVLTask, which " + "builds decoder, vision_encoder, and embedding graphs separately." + ) + + def preprocess_weights( + self, state_dict: dict[str, torch.Tensor] + ) -> dict[str, torch.Tensor]: + """Route the HuggingFace checkpoint into the three ONNX sub-models.""" + renamed: dict[str, torch.Tensor] = {} + embed_weight: torch.Tensor | None = None + for key, value in state_dict.items(): + if key.startswith(("mtp_", "mtp.")): + continue + if key.startswith("model.vision_tower."): + name = "vision_encoder." + key[len("model.") :] + name = name.replace(".mlp.fc1.", ".mlp.up_proj.") + name = name.replace(".mlp.fc2.", ".mlp.down_proj.") + name = name.replace( + ".vision_tower.vit_merger.", + ".vision_tower.encoder.vit_merger.", + ) + renamed[name] = value + elif key.startswith("model.merger."): + renamed["vision_encoder." + key[len("model.") :]] = value + elif key == "model.language_model.embed_tokens.weight": + embed_weight = value + renamed["decoder.model.embed_tokens.weight"] = value + renamed["embedding.embed_tokens.weight"] = value + elif key.startswith("model.language_model."): + suffix = key[len("model.language_model.") :] + renamed[f"decoder.model.{suffix}"] = value + elif key == "lm_head.weight": + renamed["decoder.lm_head.weight"] = value + + if self.config.tie_word_embeddings and embed_weight is not None: + renamed["decoder.lm_head.weight"] = embed_weight + return renamed diff --git a/src/mobius/models/minicpmv4_6_test.py b/src/mobius/models/minicpmv4_6_test.py new file mode 100644 index 000000000..f9c2baabb --- /dev/null +++ b/src/mobius/models/minicpmv4_6_test.py @@ -0,0 +1,246 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +from __future__ import annotations + +import dataclasses + +import numpy as np +import onnx_ir as ir +import torch + +from mobius._configs import ArchitectureConfig, VisionConfig +from mobius._testing.ort_inference import OnnxModelSession +from mobius._weight_loading import apply_weights +from mobius.models.minicpmv4_6 import MiniCPMV46ForConditionalGeneration +from mobius.tasks import MiniCPMVLTask + + +def _tiny_config() -> ArchitectureConfig: + return ArchitectureConfig( + hidden_size=64, + intermediate_size=128, + num_attention_heads=4, + num_key_value_heads=2, + head_dim=16, + num_hidden_layers=4, + vocab_size=256, + max_position_embeddings=128, + hidden_act="silu", + rms_norm_eps=1e-6, + rope_type="default", + rope_theta=10_000.0, + partial_rotary_factor=0.25, + layer_types=[ + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + ], + linear_num_key_heads=2, + linear_key_head_dim=16, + linear_num_value_heads=2, + linear_value_head_dim=16, + linear_conv_kernel_dim=4, + image_token_id=250, + video_token_id=251, + vision=VisionConfig( + hidden_size=32, + intermediate_size=64, + num_hidden_layers=2, + num_attention_heads=2, + image_size=56, + patch_size=14, + norm_eps=1e-6, + in_channels=3, + num_position_embeddings=16, + insert_layer_id=0, + window_kernel_size=(2, 2), + merge_kernel_size=(2, 2), + merger_times=1, + ), + dtype=ir.DataType.FLOAT, + ) + + +def test_minicpmv4_6_synthetic_vision_parity(): + """L3: packed vision + both mergers match Transformers with random weights.""" + from transformers.models.minicpmv4_6.configuration_minicpmv4_6 import ( + MiniCPMV4_6Config, + MiniCPMV4_6VisionConfig, + ) + from transformers.models.minicpmv4_6.modeling_minicpmv4_6 import ( + MiniCPMV4_6Merger, + MiniCPMV4_6VisionModel, + ) + + torch.manual_seed(42) + vision_config = MiniCPMV4_6VisionConfig( + hidden_size=32, + intermediate_size=64, + num_hidden_layers=2, + num_attention_heads=2, + image_size=56, + patch_size=14, + ) + hf_config = MiniCPMV4_6Config( + vision_config=vision_config.to_dict(), + insert_layer_id=0, + text_config={ + "model_type": "qwen3_5_text", + "hidden_size": 64, + "intermediate_size": 128, + "num_hidden_layers": 4, + "num_attention_heads": 4, + "num_key_value_heads": 2, + "head_dim": 16, + "vocab_size": 256, + "layer_types": [ + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + ], + "linear_num_key_heads": 2, + "linear_key_head_dim": 16, + "linear_num_value_heads": 2, + "linear_value_head_dim": 16, + }, + ) + hf_vision = MiniCPMV4_6VisionModel(hf_config.vision_config).eval() + hf_merger = MiniCPMV4_6Merger(hf_config).eval() + + config = _tiny_config() + module = MiniCPMV46ForConditionalGeneration(config) + package = MiniCPMVLTask().build(module, config) + state_dict = { + **{ + f"model.vision_tower.{name}": value + for name, value in hf_vision.state_dict().items() + }, + **{f"model.merger.{name}": value for name, value in hf_merger.state_dict().items()}, + } + processed_weights = module.preprocess_weights(state_dict) + graph_parameters = { + name + for name in package["vision_encoder"].graph.initializers + if name.startswith("vision_encoder.") + } + assert set(processed_weights) == graph_parameters + apply_weights( + package["vision_encoder"], + processed_weights, + ) + + pixel_values = torch.randn(1, 3, 14, 224) + target_sizes = torch.tensor([[4, 4]], dtype=torch.int32) + with torch.no_grad(): + hidden_states = hf_vision( + pixel_values, + target_sizes=target_sizes, + ).last_hidden_state + expected = torch.cat( + hf_merger(hidden_states, target_sizes // 2), + dim=0, + ).numpy() + + session = OnnxModelSession(package["vision_encoder"]) + actual = session.run( + { + "pixel_values": pixel_values.numpy(), + "target_sizes": target_sizes.numpy(), + } + )["image_features"] + np.testing.assert_allclose(actual, expected, rtol=1e-4, atol=1e-4) + + # Default slice-mode preprocessing commonly emits an overview crop and + # detail crops with different grids. Compare the packed ONNX call against + # independent HF calls, which are the ragged reference semantics. + target_sizes = torch.tensor([[4, 4], [4, 8]], dtype=torch.int32) + pixel_values = torch.randn(1, 3, 14, (4 * 4 + 4 * 8) * 14) + expected_parts = [] + start = 0 + with torch.no_grad(): + for size in target_sizes: + num_patches = int(size.prod()) + end = start + num_patches * 14 + unit_pixels = pixel_values[:, :, :, start:end] + unit_size = size.unsqueeze(0) + hidden_states = hf_vision( + unit_pixels, + target_sizes=unit_size, + ).last_hidden_state + expected_parts.extend(hf_merger(hidden_states, unit_size // 2)) + start = end + expected = torch.cat(expected_parts, dim=0).numpy() + actual = session.run( + { + "pixel_values": pixel_values.numpy(), + "target_sizes": target_sizes.numpy(), + } + )["image_features"] + session.close() + np.testing.assert_allclose(actual, expected, rtol=1e-4, atol=1e-4) + + # The optional 4x mode skips only the in-tower merger, leaving the final + # 2x2 projector to produce four times as many visual tokens as 16x mode. + config_4x = dataclasses.replace(config, downsample_mode="4x") + module_4x = MiniCPMV46ForConditionalGeneration(config_4x) + package_4x = MiniCPMVLTask().build(module_4x, config_4x) + processed_4x = module_4x.preprocess_weights(state_dict) + graph_parameters_4x = set(package_4x["vision_encoder"].graph.initializers) + apply_weights( + package_4x["vision_encoder"], + {name: value for name, value in processed_4x.items() if name in graph_parameters_4x}, + ) + target_sizes = torch.tensor([[4, 4]], dtype=torch.int32) + pixel_values = torch.randn(1, 3, 14, 224) + with torch.no_grad(): + hidden_states = hf_vision( + pixel_values, + target_sizes=target_sizes, + use_vit_merger=False, + ).last_hidden_state + expected = torch.cat( + hf_merger(hidden_states, target_sizes), + dim=0, + ).numpy() + session = OnnxModelSession(package_4x["vision_encoder"]) + actual = session.run( + { + "pixel_values": pixel_values.numpy(), + "target_sizes": target_sizes.numpy(), + } + )["image_features"] + session.close() + assert actual.shape[0] == 4 + np.testing.assert_allclose(actual, expected, rtol=1e-4, atol=1e-4) + + +def test_minicpmv4_6_embedding_mixes_image_and_video_tokens(): + """Image and video placeholders share the packed visual-feature stream.""" + config = _tiny_config() + package = MiniCPMVLTask().build( + MiniCPMV46ForConditionalGeneration(config), + config, + ) + rng = np.random.default_rng(42) + for initializer in package["embedding"].graph.initializers.values(): + if initializer.const_value is None: + initializer.const_value = ir.tensor( + rng.standard_normal(initializer.shape).astype(np.float32) + ) + + input_ids = np.array([[1, 250, 2, 251]], dtype=np.int64) + features = rng.standard_normal((2, config.hidden_size)).astype(np.float32) + session = OnnxModelSession(package["embedding"]) + result = session.run( + { + "input_ids": input_ids, + "image_features": features, + } + )["inputs_embeds"] + session.close() + + np.testing.assert_array_equal(result[0, 1], features[0]) + np.testing.assert_array_equal(result[0, 3], features[1]) diff --git a/src/mobius/tasks/__init__.py b/src/mobius/tasks/__init__.py index d7f947ffc..6295fc776 100644 --- a/src/mobius/tasks/__init__.py +++ b/src/mobius/tasks/__init__.py @@ -46,6 +46,7 @@ "ImageClassificationTask", "ModelTask", "MllamaVisionLanguageTask", + "MiniCPMVLTask", "MaskedDiffusionTask", "MoshiDepformerTask", "MoshiTemporalTask", @@ -124,6 +125,7 @@ from mobius.tasks._vision_language_3model import ( Cosmos3EdgeVLTask, HybridQwenVLTask, + MiniCPMVLTask, MllamaVisionLanguageTask, PixtralVLTask, QwenVLTask, @@ -165,6 +167,7 @@ "mllama-vision-language": MllamaVisionLanguageTask, "qwen-vl": QwenVLTask, "hybrid-qwen-vl": HybridQwenVLTask, + "minicpm-vl": MiniCPMVLTask, "qwen3-vl-vision-language": Qwen3VLVisionLanguageTask, "gemma4": Gemma4Task, "gemma4-text-generation": Gemma4TextCausalLMTask, diff --git a/src/mobius/tasks/_vision_language_3model.py b/src/mobius/tasks/_vision_language_3model.py index 42d804ed9..90cf9c022 100644 --- a/src/mobius/tasks/_vision_language_3model.py +++ b/src/mobius/tasks/_vision_language_3model.py @@ -250,6 +250,68 @@ def build( return ModelPackage(models, config=config) +class MiniCPMVLTask(VisionLanguageTask): + """MiniCPM-V packed-NaViT vision with a Qwen3.5 hybrid decoder.""" + + def build( + self, + module: nn.Module, + config: ArchitectureConfig, + ) -> ModelPackage: + self._validate_components(module) + models: dict[str, ir.Model] = {} + models["decoder"] = build_decoder_from_embeds( + module.decoder, + config, + hybrid=True, + ) + models["vision_encoder"] = self._build_vision(module.vision_encoder, config) + models["embedding"] = build_embedding_from_features( + module.embedding, + config, + feature_name="image_features", + feature_dim=config.hidden_size, + ) + return ModelPackage(models, config=config) + + def _build_vision( + self, + vision: nn.Module, + config: ArchitectureConfig, + ) -> ir.Model: + """Build packed pixels + patch-grid sizes -> compressed visual tokens.""" + packed_batch = ir.SymbolicDim("packed_batch") + packed_width = ir.SymbolicDim("packed_width") + num_visual_units = ir.SymbolicDim("num_visual_units") + vision_config = config.vision + assert vision_config is not None + patch_size = vision_config.patch_size or 14 + + graph, builder = _make_graph(name="vision_encoder") + pixel_values = builder.input( + "pixel_values", + dtype=config.dtype, + shape=[ + packed_batch, + vision_config.in_channels, + patch_size, + packed_width, + ], + ) + target_sizes = builder.input( + "target_sizes", + dtype=ir.DataType.INT32, + shape=[num_visual_units, 2], + ) + image_features = vision( + builder.op, + pixel_values=pixel_values, + target_sizes=target_sizes, + ) + builder.add_output(image_features, "image_features") + return _make_model(graph) + + class PixtralVLTask(VisionLanguageTask): """Vision-language task with dynamic-resolution Pixtral vision encoder. diff --git a/testdata/cases/vision-language/minicpm-v-4_6.yaml b/testdata/cases/vision-language/minicpm-v-4_6.yaml new file mode 100644 index 000000000..2d8b31fc2 --- /dev/null +++ b/testdata/cases/vision-language/minicpm-v-4_6.yaml @@ -0,0 +1,19 @@ +model_id: "openbmb/MiniCPM-V-4.6" +model_type: "minicpmv4_6" +revision: "8169864629825dc1d755a5aa1cd8b5935dcbc83f" +task_type: "image-text-to-text" +dtype: "float32" + +inputs: + prompts: + - "Describe this image in detail." + images: + - "pipeline-cat-chonk.jpeg" + +level: "L4+L5" + +generation: + max_new_tokens: 30 + do_sample: false + +notes: "MiniCPM-V-4.6 with packed variable-resolution SigLIP2 and Qwen3.5 hybrid text." diff --git a/testdata/golden/vision-language/minicpm-v-4_6.json b/testdata/golden/vision-language/minicpm-v-4_6.json new file mode 100644 index 000000000..ff56f8097 --- /dev/null +++ b/testdata/golden/vision-language/minicpm-v-4_6.json @@ -0,0 +1,384 @@ +{ + "top1_id": 760, + "top2_id": 561, + "top10_ids": [ + 760, + 561, + 92157, + 10288, + 3067, + 16721, + 1719, + 85190, + 15877, + 50821 + ], + "top10_logits": [ + "0x1.3c8a280000000p+5", + "0x1.f0b58a0000000p+4", + "0x1.7dfebc0000000p+4", + "0x1.64a30c0000000p+4", + "0x1.64762c0000000p+4", + "0x1.6281aa0000000p+4", + "0x1.603c1e0000000p+4", + "0x1.5be7220000000p+4", + "0x1.57dc380000000p+4", + "0x1.4c0c540000000p+4" + ], + "logits_summary": [ + "0x1.3c8a280000000p+5", + "-0x1.472b7a0000000p+3", + "0x1.c8f268d10cb0ap-1", + "0x1.26c048f97daeep+1" + ], + "input_ids": [ + 248045, + 846, + 198, + 248090, + 15, + 248091, + 248078, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248079, + 248088, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248089, + 248088, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248089, + 198, + 248088, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248089, + 248088, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248089, + 198, + 72240, + 411, + 2099, + 303, + 7472, + 13, + 248046, + 198, + 248045, + 74455, + 198, + 248068, + 271, + 248069, + 271 + ] +} diff --git a/testdata/golden/vision-language/minicpm-v-4_6_generation.json b/testdata/golden/vision-language/minicpm-v-4_6_generation.json new file mode 100644 index 000000000..b1b37f04e --- /dev/null +++ b/testdata/golden/vision-language/minicpm-v-4_6_generation.json @@ -0,0 +1,37 @@ +{ + "model_id": "openbmb/MiniCPM-V-4.6", + "prompt": "Describe this image in detail.", + "generated_tokens": [ + 760, + 2099, + 4774, + 264, + 3349, + 11, + 65283, + 9572, + 10932, + 303, + 264, + 86684, + 4424, + 13, + 561, + 9572, + 682, + 264, + 21094, + 11, + 28704, + 88, + 10771, + 11, + 440, + 264, + 11702, + 11, + 71044, + 22162 + ], + "generated_text": "The image shows a large, fluffy animal standing in a snowy environment. The animal has a robust, bushy appearance, with a thick, textured coat" +} diff --git a/tests/_test_configs.py b/tests/_test_configs.py index 19ea16689..9d8971409 100644 --- a/tests/_test_configs.py +++ b/tests/_test_configs.py @@ -2089,6 +2089,22 @@ def _base_config(config_cls=None, **overrides) -> ArchitectureConfig: window_size=4, ) +_TINY_MINICPMV46_VISION = VisionConfig( + hidden_size=32, + intermediate_size=64, + num_hidden_layers=2, + num_attention_heads=2, + image_size=56, + patch_size=14, + norm_eps=1e-6, + in_channels=3, + num_position_embeddings=16, + insert_layer_id=0, + window_kernel_size=(2, 2), + merge_kernel_size=(2, 2), + merger_times=1, +) + _TINY_COSMOS3_EDGE_VISION = VisionConfig( hidden_size=32, @@ -2113,6 +2129,28 @@ def _base_config(config_cls=None, **overrides) -> ArchitectureConfig: VL_CONFIGS: list[tuple[str, dict, bool]] = [ # --- LLaVA family (vision-language, 3-model split) --- ("llava", {"vision": _TINY_VISION, "image_token_id": 32000}, True), + ( + "minicpmv4_6", + { + "vision": _TINY_MINICPMV46_VISION, + "image_token_id": 250, + "video_token_id": 251, + "num_hidden_layers": 4, + "layer_types": [ + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + ], + "partial_rotary_factor": 0.25, + "linear_num_key_heads": 2, + "linear_key_head_dim": 16, + "linear_num_value_heads": 2, + "linear_value_head_dim": 16, + "linear_conv_kernel_dim": 4, + }, + True, + ), ( "cosmos3_edge", { diff --git a/tests/integration_test.py b/tests/integration_test.py index 99218321c..747483596 100644 --- a/tests/integration_test.py +++ b/tests/integration_test.py @@ -68,6 +68,75 @@ def _model_accessible(model_id: str) -> bool: return True +@pytest.mark.integration +def test_minicpmv4_6_real_weight_vision_parity(): + """Real nonzero pixels match through SigLIP2 and both visual mergers.""" + import gc + + from transformers import AutoModelForImageTextToText, AutoProcessor + + model_id = "openbmb/MiniCPM-V-4.6" + processor = AutoProcessor.from_pretrained(model_id) + image = Image.open("testdata/pipeline-cat-chonk.jpeg").convert("RGB") + # A square source triggers the default overview + slice path with + # non-uniform patch grids (e.g. 32x32 overview and 40x24 slices). + image = image.resize((1024, 1024)) + prompt = processor.apply_chat_template( + [ + { + "role": "user", + "content": [ + {"type": "image", "image": "testdata/pipeline-cat-chonk.jpeg"}, + {"type": "text", "text": "Describe this image in detail."}, + ], + } + ], + tokenize=False, + add_generation_prompt=True, + ) + inputs = processor( + text=prompt, + images=[image], + return_tensors="pt", + ) + + hf_model = AutoModelForImageTextToText.from_pretrained( + model_id, + dtype=torch.float32, + ).eval() + expected_parts = [] + start = 0 + with torch.no_grad(): + for size in inputs["target_sizes"]: + num_patches = int(size.prod()) + end = start + num_patches * 14 + unit_pixels = inputs["pixel_values"][:, :, :, start:end] + expected_parts.extend( + hf_model.get_image_features( + unit_pixels, + size.unsqueeze(0), + ).pooler_output + ) + start = end + expected = torch.cat(expected_parts, dim=0).numpy() + del hf_model + gc.collect() + + package = build(model_id, dtype="float32", load_weights=True) + session = _make_session(package["vision_encoder"]) + actual = session.run( + { + "pixel_values": inputs["pixel_values"].numpy(), + "target_sizes": inputs["target_sizes"].numpy(), + } + )["image_features"] + session.close() + + assert float(np.linalg.norm(inputs["pixel_values"].numpy())) > 0.0 + assert np.unique(inputs["target_sizes"].numpy(), axis=0).shape[0] > 1 + np.testing.assert_allclose(actual, expected, rtol=1e-3, atol=1e-3) + + # --------------------------------------------------------------------------- # Model catalogue: small models for each supported architecture # diff --git a/tests/ort_genai_test.py b/tests/ort_genai_test.py index 162c98537..c48736a5d 100644 --- a/tests/ort_genai_test.py +++ b/tests/ort_genai_test.py @@ -390,3 +390,26 @@ def test_text_generation(self, model_id: str, tmp_path): "Model should generate at least one token" ) del generator + + +@pytest.mark.integration +@pytest.mark.integration_slow +def test_minicpmv4_6_package_loads(tmp_path): + """MiniCPM's custom packed vision graph loads through ORT GenAI.""" + from mobius import build + from mobius.integrations.ort_genai import export_package + + model_id = "openbmb/MiniCPM-V-4.6" + package = build(model_id, dtype="f32", load_weights=True) + output_dir = str(tmp_path / "minicpm-v-4.6") + manifest = export_package( + package, + output_dir, + hf_model_id=model_id, + progress_bar=False, + ) + + assert os.path.isfile(manifest["genai_config"]) + assert os.path.isfile(os.path.join(output_dir, "preprocessor_config.json")) + model = ort_genai.Model(output_dir) + assert ort_genai.Tokenizer(model) is not None From 0542aae1fa730a34a38e6baa78fb5e34567845da Mon Sep 17 00:00:00 2001 From: Justin Chu Date: Fri, 7 Aug 2026 18:47:19 -0700 Subject: [PATCH 2/6] Fix MiniCPM CUDA and BF16 execution Keep packed patch coordinate mapping on CUDA by replacing ArgMax with a segment-count reduction. Cast through float for ReduceMean and Compress so BF16 vision graphs load and run while restoring the model dtype at each boundary. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: Justin Chu --- src/mobius/models/minicpmv4_6.py | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/src/mobius/models/minicpmv4_6.py b/src/mobius/models/minicpmv4_6.py index 19846e40d..3972a91d7 100644 --- a/src/mobius/models/minicpmv4_6.py +++ b/src/mobius/models/minicpmv4_6.py @@ -70,13 +70,13 @@ def _packed_patch_coordinates( # The first cumulative end greater than each packed index identifies its # source image crop or video frame. - before_end = op.Less( + reached_end = op.GreaterOrEqual( op.Unsqueeze(packed_index, [1]), op.Unsqueeze(ends, [0]), ) - visual_index = op.ArgMax( - op.Cast(before_end, to=7), - axis=1, + visual_index = op.ReduceSum( + op.Cast(reached_end, to=7), + [1], keepdims=0, ) local_index = op.Sub(packed_index, op.Gather(starts, visual_index)) @@ -347,7 +347,15 @@ def forward( merged_grid, _, _ = _spatial_windows( op, hidden_states, target_sizes, self.hidden_size, self.kernel_size ) - merge_residual = op.ReduceMean(merged_grid, [3, 4], keepdims=0) + # ReduceMean does not have an ORT BF16 kernel on CPU or CUDA. + merge_residual = op.CastLike( + op.ReduceMean( + op.CastLike(merged_grid, 0.0), + [3, 4], + keepdims=0, + ), + merged_grid, + ) merge_residual = op.Reshape( merge_residual, op.Constant(value_ints=[-1, self.hidden_size]), @@ -583,7 +591,14 @@ def forward( axis=0, ), ) - return op.Compress(hidden_states, valid, axis=0) + # ONNX Compress excludes BF16 from its type constraint. Compaction is + # data movement only, so cast through FLOAT and restore the model dtype. + compacted = op.Compress( + op.CastLike(hidden_states, 0.0), + valid, + axis=0, + ) + return op.CastLike(compacted, hidden_states) class _MiniCPMVisionEncoderModel(nn.Module): From 6616732bfc17b491c341bd18351a6d54712b6174 Mon Sep 17 00:00:00 2001 From: Justin Chu Date: Sat, 8 Aug 2026 04:38:50 -0700 Subject: [PATCH 3/6] Optimize MiniCPM reduced-precision vision graph Emit float compatibility casts around ReduceMean and Compress only for BF16, where ORT requires them. Keep native FP16 kernels on the fast path and remove four unnecessary casts without changing outputs. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: Justin Chu --- src/mobius/models/minicpmv4_6.py | 47 +++++++++++++++++--------------- 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/src/mobius/models/minicpmv4_6.py b/src/mobius/models/minicpmv4_6.py index 3972a91d7..833f0817c 100644 --- a/src/mobius/models/minicpmv4_6.py +++ b/src/mobius/models/minicpmv4_6.py @@ -10,8 +10,7 @@ from __future__ import annotations -from typing import TYPE_CHECKING - +import onnx_ir as ir import torch from onnxscript import OpBuilder, nn @@ -26,9 +25,6 @@ ) from mobius.models.qwen35 import Qwen35TextModel -if TYPE_CHECKING: - import onnx_ir as ir - class MiniCPMV46Config(ArchitectureConfig): """Mobius configuration preserving MiniCPM's composite model identity.""" @@ -284,6 +280,7 @@ def __init__(self, config: ArchitectureConfig): assert vision.intermediate_size is not None self.hidden_size = vision.hidden_size self.kernel_size = vision.window_kernel_size + self._needs_bf16_cast = config.dtype == ir.DataType.BFLOAT16 window_tokens = self.kernel_size[0] * self.kernel_size[1] window_hidden = self.hidden_size * window_tokens window_intermediate = vision.intermediate_size * window_tokens @@ -347,15 +344,18 @@ def forward( merged_grid, _, _ = _spatial_windows( op, hidden_states, target_sizes, self.hidden_size, self.kernel_size ) - # ReduceMean does not have an ORT BF16 kernel on CPU or CUDA. - merge_residual = op.CastLike( - op.ReduceMean( - op.CastLike(merged_grid, 0.0), - [3, 4], - keepdims=0, - ), - merged_grid, - ) + if self._needs_bf16_cast: + # ReduceMean does not have an ORT BF16 kernel on CPU or CUDA. + merge_residual = op.CastLike( + op.ReduceMean( + op.CastLike(merged_grid, 0.0), + [3, 4], + keepdims=0, + ), + merged_grid, + ) + else: + merge_residual = op.ReduceMean(merged_grid, [3, 4], keepdims=0) merge_residual = op.Reshape( merge_residual, op.Constant(value_ints=[-1, self.hidden_size]), @@ -517,6 +517,7 @@ def __init__(self, config: ArchitectureConfig): self.kernel_size = vision.merge_kernel_size self.merger_times = vision.merger_times self.vision_downsample = 1 if config.downsample_mode == "4x" else 2 + self._needs_bf16_cast = config.dtype == ir.DataType.BFLOAT16 self.output_sizes = [ self.hidden_size if i < self.merger_times - 1 else config.hidden_size for i in range(self.merger_times) @@ -591,14 +592,16 @@ def forward( axis=0, ), ) - # ONNX Compress excludes BF16 from its type constraint. Compaction is - # data movement only, so cast through FLOAT and restore the model dtype. - compacted = op.Compress( - op.CastLike(hidden_states, 0.0), - valid, - axis=0, - ) - return op.CastLike(compacted, hidden_states) + if self._needs_bf16_cast: + # ONNX Compress excludes BF16 from its type constraint. Compaction + # is data movement only, so cast through FLOAT and restore dtype. + compacted = op.Compress( + op.CastLike(hidden_states, 0.0), + valid, + axis=0, + ) + return op.CastLike(compacted, hidden_states) + return op.Compress(hidden_states, valid, axis=0) class _MiniCPMVisionEncoderModel(nn.Module): From cd34f4b5f84a35b2239ee1dae36b48d192992d51 Mon Sep 17 00:00:00 2001 From: Justin Chu Date: Sat, 8 Aug 2026 04:47:05 -0700 Subject: [PATCH 4/6] Harden MiniCPM merger config extraction Fall back to the architecture's 2x2 merger kernels when Hugging Face explicitly supplies null values, and add a regression test for both window and final merger fields. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: Justin Chu --- src/mobius/_configs/_extractors_test.py | 32 +++++++++++++++++++ .../_configs/per_model/_minicpmv4_6_vision.py | 4 +-- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/src/mobius/_configs/_extractors_test.py b/src/mobius/_configs/_extractors_test.py index 3d3159466..f76691443 100644 --- a/src/mobius/_configs/_extractors_test.py +++ b/src/mobius/_configs/_extractors_test.py @@ -343,3 +343,35 @@ def test_hunyuan_vl_mot_image_token_id_survives_default_hook(loaded_vision_hooks out = _extractors.extract_vision_config(cfg, None, "hunyuan_vl_mot") assert out["vision"].image_token_id == 12 assert out.get("image_token_id") == 12 + + +def test_minicpm_vision_defaults_explicit_none_kernels(loaded_vision_hooks): + """Explicit None merger kernels fall back to MiniCPM's 2x2 defaults.""" + vision_config = _FakeHFConfig( + model_type="minicpmv4_6_vision", + hidden_size=32, + intermediate_size=64, + num_hidden_layers=2, + num_attention_heads=2, + image_size=56, + patch_size=14, + layer_norm_eps=1e-6, + num_channels=3, + window_kernel_size=None, + ) + parent_config = _FakeHFConfig( + model_type="minicpmv4_6", + vision_config=vision_config, + image_token_id=250, + merge_kernel_size=None, + ) + text_config = _FakeHFConfig(model_type="qwen3_5_text") + + out = _extractors.extract_vision_config( + text_config, + parent_config, + "qwen3_5_text", + ) + + assert out["vision"].window_kernel_size == (2, 2) + assert out["vision"].merge_kernel_size == (2, 2) diff --git a/src/mobius/_configs/per_model/_minicpmv4_6_vision.py b/src/mobius/_configs/per_model/_minicpmv4_6_vision.py index cd091a854..ad74b93d6 100644 --- a/src/mobius/_configs/per_model/_minicpmv4_6_vision.py +++ b/src/mobius/_configs/per_model/_minicpmv4_6_vision.py @@ -31,8 +31,8 @@ def _minicpmv4_6_vision(config, parent_config, model_type: str, fields: dict): num_position_embeddings=(image_size // patch_size) ** 2, image_token_id=parent_config.image_token_id, insert_layer_id=getattr(parent_config, "insert_layer_id", 6), - window_kernel_size=tuple(getattr(vision, "window_kernel_size", (2, 2))), - merge_kernel_size=tuple(getattr(parent_config, "merge_kernel_size", (2, 2))), + window_kernel_size=tuple(getattr(vision, "window_kernel_size", None) or (2, 2)), + merge_kernel_size=tuple(getattr(parent_config, "merge_kernel_size", None) or (2, 2)), merger_times=getattr(parent_config, "merger_times", 1), ) return None From f1467bcb261fd71c65eaed73680c760c9cb47acc Mon Sep 17 00:00:00 2001 From: justinchuby Date: Tue, 11 Aug 2026 15:33:00 +0000 Subject: [PATCH 5/6] Fix MiniCPM batch media feature indexing Flatten the media placeholder mask before cumulative indexing so packed image and video features are consumed globally in HF masked_scatter order across batch rows. Add two-row parity and empty decode execution coverage. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: justinchuby --- src/mobius/models/minicpmv4_6.py | 6 +++- src/mobius/models/minicpmv4_6_test.py | 45 +++++++++++++++++++++------ 2 files changed, 40 insertions(+), 11 deletions(-) diff --git a/src/mobius/models/minicpmv4_6.py b/src/mobius/models/minicpmv4_6.py index 833f0817c..fe6ba40f8 100644 --- a/src/mobius/models/minicpmv4_6.py +++ b/src/mobius/models/minicpmv4_6.py @@ -671,9 +671,12 @@ def forward( op.Equal(input_ids, self.image_token_id), op.Equal(input_ids, self.video_token_id), ) + # HF masked_scatter consumes the packed feature stream in batch-major + # order, so flatten before CumSum instead of restarting at each row. + flat_media_mask = op.Reshape(media_mask, [-1]) media_indices = op.Clip( op.Sub( - op.CumSum(op.Cast(media_mask, to=7), 1), + op.CumSum(op.Cast(flat_media_mask, to=ir.DataType.INT64), 0), 1, ), 0, @@ -692,6 +695,7 @@ def forward( media_indices, axis=0, ) + features = op.Reshape(features, op.Shape(text_embeds)) return op.Where(op.Unsqueeze(media_mask, [-1]), features, text_embeds) diff --git a/src/mobius/models/minicpmv4_6_test.py b/src/mobius/models/minicpmv4_6_test.py index f9c2baabb..f627ed862 100644 --- a/src/mobius/models/minicpmv4_6_test.py +++ b/src/mobius/models/minicpmv4_6_test.py @@ -218,21 +218,32 @@ def test_minicpmv4_6_synthetic_vision_parity(): def test_minicpmv4_6_embedding_mixes_image_and_video_tokens(): - """Image and video placeholders share the packed visual-feature stream.""" + """Packed features follow HF masked_scatter order across batch rows.""" config = _tiny_config() package = MiniCPMVLTask().build( MiniCPMV46ForConditionalGeneration(config), config, ) rng = np.random.default_rng(42) - for initializer in package["embedding"].graph.initializers.values(): + embedding_weight = None + for name, initializer in package["embedding"].graph.initializers.items(): if initializer.const_value is None: - initializer.const_value = ir.tensor( - rng.standard_normal(initializer.shape).astype(np.float32) - ) + value = rng.standard_normal(initializer.shape).astype(np.float32) + initializer.const_value = ir.tensor(value) + if name.endswith("embed_tokens.weight"): + embedding_weight = value + assert embedding_weight is not None - input_ids = np.array([[1, 250, 2, 251]], dtype=np.int64) - features = rng.standard_normal((2, config.hidden_size)).astype(np.float32) + input_ids = np.array( + [ + [1, 250, 2, 251], + [251, 3, 250, 4], + ], + dtype=np.int64, + ) + features = np.arange(4 * config.hidden_size, dtype=np.float32).reshape( + 4, config.hidden_size + ) session = OnnxModelSession(package["embedding"]) result = session.run( { @@ -240,7 +251,21 @@ def test_minicpmv4_6_embedding_mixes_image_and_video_tokens(): "image_features": features, } )["inputs_embeds"] - session.close() - np.testing.assert_array_equal(result[0, 1], features[0]) - np.testing.assert_array_equal(result[0, 3], features[1]) + media_mask = (input_ids == config.image_token_id) | ( + input_ids == config.video_token_id + ) + expected = embedding_weight[input_ids].copy() + expected[media_mask] = features + np.testing.assert_array_equal(result, expected) + + # Decode steps without new media pass an empty packed feature stream. + decode_ids = np.array([[5], [6]], dtype=np.int64) + decode_result = session.run( + { + "input_ids": decode_ids, + "image_features": np.empty((0, config.hidden_size), dtype=np.float32), + } + )["inputs_embeds"] + session.close() + np.testing.assert_array_equal(decode_result, embedding_weight[decode_ids]) From 1400b7bd09e612a4acb6440be724f59f396264c9 Mon Sep 17 00:00:00 2001 From: justinchuby Date: Tue, 11 Aug 2026 15:45:22 +0000 Subject: [PATCH 6/6] Format MiniCPM media embedding test Apply the repository formatter to the batch-global image and video feature routing coverage. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: justinchuby --- src/mobius/models/minicpmv4_6_test.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/mobius/models/minicpmv4_6_test.py b/src/mobius/models/minicpmv4_6_test.py index f627ed862..ca9009166 100644 --- a/src/mobius/models/minicpmv4_6_test.py +++ b/src/mobius/models/minicpmv4_6_test.py @@ -252,9 +252,7 @@ def test_minicpmv4_6_embedding_mixes_image_and_video_tokens(): } )["inputs_embeds"] - media_mask = (input_ids == config.image_token_id) | ( - input_ids == config.video_token_id - ) + media_mask = (input_ids == config.image_token_id) | (input_ids == config.video_token_id) expected = embedding_weight[input_ids].copy() expected[media_mask] = features np.testing.assert_array_equal(result, expected)