diff --git a/README.md b/README.md index f9f2320b6..da21660e2 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ multi-component export for pipelines. |---|---| | **Text Generation** | Llama 2/3/4, Mistral, Qwen 2/2.5/3/3.5/3.6, Phi-3/3.5, Gemma 1/2/3/4, Granite, GPT-2, OPT, OLMo, SmolLM3, and many more | | **Mixture of Experts** | PhiMoE, GPTOSS, Mixtral, OLMoE, DeepSeek-V2/V3, Qwen2-MoE, Qwen3-MoE, Qwen3-Next, GLM-4-MoE, Arctic, DBRX, Jamba | -| **Multimodal** | Gemma 3/4, Phi-4MM (vision + audio + LoRA), LLaVA, InternVL2, Qwen2.5-VL, Qwen3-VL, Qwen3.5/3.6-VL, Pixtral | +| **Multimodal** | Gemma 3/4, Phi-4MM (vision + audio + LoRA), LLaVA, InternVL2, MiniCPM-V 4.6, Qwen2.5-VL, Qwen3-VL, Qwen3.5/3.6-VL, Pixtral | | **Encoder-only** | BERT, RoBERTa, ALBERT, DeBERTa, DistilBERT, ELECTRA, XLNet | | **Encoder-Decoder** | BART, T5/mT5, Marian, M2M-100, Pegasus, BigBird-Pegasus | | **Speech-to-Text** | Whisper, FastConformer-RNNT, FunASR, Qwen3-ASR, SenseVoice | diff --git a/src/mobius/_configs/_base.py b/src/mobius/_configs/_base.py index 475b69b9a..d5fd36ece 100644 --- a/src/mobius/_configs/_base.py +++ b/src/mobius/_configs/_base.py @@ -494,6 +494,8 @@ class ArchitectureConfig(BaseModelConfig): # Vision shared fields (accessed as top-level config.X by tasks) mm_tokens_per_image: int | None = None image_token_id: int | None = None + video_token_id: int | None = None + downsample_mode: str = "16x" spatial_merge_size: int = 2 temporal_patch_size: int = 2 deepstack_visual_indexes: list[int] | None = None diff --git a/src/mobius/_configs/_extractors_test.py b/src/mobius/_configs/_extractors_test.py index 3d3159466..f76691443 100644 --- a/src/mobius/_configs/_extractors_test.py +++ b/src/mobius/_configs/_extractors_test.py @@ -343,3 +343,35 @@ def test_hunyuan_vl_mot_image_token_id_survives_default_hook(loaded_vision_hooks out = _extractors.extract_vision_config(cfg, None, "hunyuan_vl_mot") assert out["vision"].image_token_id == 12 assert out.get("image_token_id") == 12 + + +def test_minicpm_vision_defaults_explicit_none_kernels(loaded_vision_hooks): + """Explicit None merger kernels fall back to MiniCPM's 2x2 defaults.""" + vision_config = _FakeHFConfig( + model_type="minicpmv4_6_vision", + hidden_size=32, + intermediate_size=64, + num_hidden_layers=2, + num_attention_heads=2, + image_size=56, + patch_size=14, + layer_norm_eps=1e-6, + num_channels=3, + window_kernel_size=None, + ) + parent_config = _FakeHFConfig( + model_type="minicpmv4_6", + vision_config=vision_config, + image_token_id=250, + merge_kernel_size=None, + ) + text_config = _FakeHFConfig(model_type="qwen3_5_text") + + out = _extractors.extract_vision_config( + text_config, + parent_config, + "qwen3_5_text", + ) + + assert out["vision"].window_kernel_size == (2, 2) + assert out["vision"].merge_kernel_size == (2, 2) diff --git a/src/mobius/_configs/_sub_configs.py b/src/mobius/_configs/_sub_configs.py index 79dfddc5f..a06e4924a 100644 --- a/src/mobius/_configs/_sub_configs.py +++ b/src/mobius/_configs/_sub_configs.py @@ -82,6 +82,11 @@ class VisionConfig: # (HuggingFace convention, e.g. -2 for Phi-3.5-Vision). ``None`` means use # the final hidden state (all layers + post_layernorm). feature_layer: int | None = None + # MiniCPM-V packed-NaViT vision encoder and its two spatial mergers. + insert_layer_id: int | None = None + window_kernel_size: tuple[int, int] = (2, 2) + merge_kernel_size: tuple[int, int] = (2, 2) + merger_times: int = 1 @dataclasses.dataclass diff --git a/src/mobius/_configs/per_model/__init__.py b/src/mobius/_configs/per_model/__init__.py index 28e637637..a57e2ba55 100644 --- a/src/mobius/_configs/per_model/__init__.py +++ b/src/mobius/_configs/per_model/__init__.py @@ -29,6 +29,7 @@ _gemma4_unified_vision, _hunyuan_vl_mot_vision, _internvl_vision, + _minicpmv4_6_vision, _phi4mm_audio, _phi4mm_vision, _phi_vision, diff --git a/src/mobius/_configs/per_model/_minicpmv4_6_vision.py b/src/mobius/_configs/per_model/_minicpmv4_6_vision.py new file mode 100644 index 000000000..ad74b93d6 --- /dev/null +++ b/src/mobius/_configs/per_model/_minicpmv4_6_vision.py @@ -0,0 +1,38 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""MiniCPM-V-4.6 vision extractor hook.""" + +from __future__ import annotations + +from mobius._configs._extractors import register_vision_hook + + +@register_vision_hook("qwen3_5_text") +def _minicpmv4_6_vision(config, parent_config, model_type: str, fields: dict): + """Extract the packed SigLIP2 geometry from the composite MiniCPM config.""" + if getattr(parent_config, "model_type", None) != "minicpmv4_6": + return None + + vision = parent_config.vision_config + image_size = getattr(vision, "image_size", 980) + patch_size = getattr(vision, "patch_size", 14) + fields.update( + model_type="minicpmv4_6_vision", + hidden_size=vision.hidden_size, + intermediate_size=vision.intermediate_size, + num_hidden_layers=vision.num_hidden_layers, + num_attention_heads=vision.num_attention_heads, + image_size=image_size, + patch_size=patch_size, + norm_eps=vision.layer_norm_eps, + hidden_act=getattr(vision, "hidden_act", "gelu_pytorch_tanh"), + in_channels=getattr(vision, "num_channels", 3), + num_position_embeddings=(image_size // patch_size) ** 2, + image_token_id=parent_config.image_token_id, + insert_layer_id=getattr(parent_config, "insert_layer_id", 6), + window_kernel_size=tuple(getattr(vision, "window_kernel_size", None) or (2, 2)), + merge_kernel_size=tuple(getattr(parent_config, "merge_kernel_size", None) or (2, 2)), + merger_times=getattr(parent_config, "merger_times", 1), + ) + return None diff --git a/src/mobius/_registry.py b/src/mobius/_registry.py index 1c611e97c..4a90198e6 100644 --- a/src/mobius/_registry.py +++ b/src/mobius/_registry.py @@ -126,6 +126,7 @@ from mobius.models.llava import LLaVAModel from mobius.models.longcat_flash import LongcatFlashCausalLMModel from mobius.models.mamba import Mamba2CausalLMModel, MambaCausalLMModel +from mobius.models.minicpmv4_6 import MiniCPMV46ForConditionalGeneration from mobius.models.minimax import MiniMaxCausalLMModel from mobius.models.mllama import MllamaCausalLMModel from mobius.models.modernbert import ModernBertDecoderModel, ModernBertModel @@ -598,6 +599,10 @@ def _detect_fallback_registration(hf_config) -> ModelRegistration | None: "llava_next_video": ModelRegistration(LLaVAModel, task="vision-language"), "llava_onevision": ModelRegistration(LLaVAModel, task="vision-language"), "mistral3": ModelRegistration(LLaVAModel, task="pixtral-vl"), + "minicpmv4_6": ModelRegistration( + MiniCPMV46ForConditionalGeneration, + task="minicpm-vl", + ), "mllama": ModelRegistration(MllamaCausalLMModel, task="mllama-vision-language"), "molmo": ModelRegistration(LLaVAModel, task="vision-language"), "ovis2": ModelRegistration(LLaVAModel, task="vision-language"), @@ -979,6 +984,7 @@ def _create_default_registry() -> ModelRegistry: "llava_onevision": "llava-hf/llava-onevision-qwen2-0.5b-ov-hf", "molmo": "allenai/MolmoE-1B-0924", "mistral3": "mistralai/Ministral-3-3B-Instruct-2512", + "minicpmv4_6": "openbmb/MiniCPM-V-4.6", "aya_vision": "CohereForAI/aya-vision-8b", "chameleon": "facebook/chameleon-7b", "cohere2_vision": "CohereForAI/c4ai-command-r7b-12-2024", diff --git a/src/mobius/integrations/ort_genai/auto_export.py b/src/mobius/integrations/ort_genai/auto_export.py index 3b4786b5d..8c9dc4964 100644 --- a/src/mobius/integrations/ort_genai/auto_export.py +++ b/src/mobius/integrations/ort_genai/auto_export.py @@ -93,6 +93,10 @@ "qwen3_vl_text": "qwen3_vl", "qwen3_5": "qwen2_5_vl", "qwen3_5_vl": "qwen2_5_vl", + # MiniCPM uses standard 1D decoder position IDs (unlike Qwen-VL MRoPE). + # The phi3v multimodal runtime provides that contract; callers supply + # HF-preprocessed packed pixels through Generator.set_inputs(). + "minicpmv4_6": "phi3v", } _GEMMA4_MODEL_TYPES = frozenset( @@ -107,6 +111,7 @@ # must preprocess with the HuggingFace processor and feed tensors via # ``Generator.set_inputs`` (see examples/gemma4_unified_ort_genai.py). _GEMMA4_UNIFIED_MODEL_TYPES = frozenset({"gemma4_unified", "gemma4_unified_text"}) +_MINICPM_MODEL_TYPES = frozenset({"minicpmv4_6"}) # gemma-3 multimodal. build() unwraps the composite HF config to its text # sub-config, so at export time ``config.model_type`` is "gemma3_text" (not # "gemma3"). @@ -134,6 +139,9 @@ "merges.txt", # BPE "vocab.json", # BPE "chat_template.jinja", # Chat template for ORT GenAI + # Preserve HuggingFace processor metadata for VLMs whose preprocessing + # cannot be represented by an ort-extensions image_processor.json. + "preprocessor_config.json", ] @@ -463,6 +471,17 @@ def _write_vision_processor_config( model_type, ) return None + if model_type in _MINICPM_MODEL_TYPES: + # MiniCPM needs adaptive slicing and NaViT horizontal patch packing. + # ort-extensions has no equivalent transform, so preserving the HF + # processor output and injecting it through set_inputs is the only + # numerically faithful runtime path. + logger.info( + "Skipping image_processor.json for %s " + "(use MiniCPMV4_6Processor + Generator.set_inputs)", + model_type, + ) + return None vision_model_type = getattr(vision, "model_type", None) is_pixtral = vision_model_type == "pixtral" or model_type in _PIXTRAL_MODEL_TYPES @@ -848,16 +867,26 @@ def _write_genai_config( if image_token_id is not None: vision_input_mapping = _introspect_inputs(pkg, "vision_encoder") embedding_input_mapping = _introspect_inputs(pkg, "embedding") + if ( + model_type := getattr(config, "model_type", "") + ) in _MINICPM_MODEL_TYPES and vision_input_mapping is not None: + # ORT GenAI's VisionInputs schema only accepts its predefined + # semantic keys. ``target_sizes`` remains an ONNX graph input + # and is supplied as a named tensor through set_inputs(). + vision_input_mapping.pop("target_sizes", None) # spatial_merge_size and config_filename are config-level # properties that cannot be inferred from the graph. vision_kwargs: dict[str, Any] = {} - model_type = getattr(config, "model_type", "") if model_type in _GEMMA4_MODEL_TYPES: vision_cfg = getattr(config, "vision", None) vision_kwargs["spatial_merge_size"] = getattr( vision_cfg, "spatial_merge_size", 2 ) + elif model_type in _MINICPM_MODEL_TYPES: + # MiniCPM performs both 2x2 merges inside the ONNX vision + # graph and consumes HF-prepacked pixels, not Qwen grid_thw. + vision_kwargs["spatial_merge_size"] = None elif has_speech: vision_kwargs["spatial_merge_size"] = None elif ( diff --git a/src/mobius/models/__init__.py b/src/mobius/models/__init__.py index 9ca9e2ce3..7755d2068 100644 --- a/src/mobius/models/__init__.py +++ b/src/mobius/models/__init__.py @@ -80,6 +80,7 @@ "Mamba2CausalLMModel", "MambaCausalLMModel", "MiniMaxCausalLMModel", + "MiniCPMV46ForConditionalGeneration", "MimiModel", "MoshiDepformerModel", "MoshiTemporalModel", @@ -217,6 +218,7 @@ from mobius.models.longcat_flash import LongcatFlashCausalLMModel from mobius.models.mamba import Mamba2CausalLMModel, MambaCausalLMModel from mobius.models.mimi import MimiModel, mimi_default_config +from mobius.models.minicpmv4_6 import MiniCPMV46ForConditionalGeneration from mobius.models.minimax import MiniMaxCausalLMModel from mobius.models.moe import ( Ernie45MoECausalLMModel, diff --git a/src/mobius/models/minicpmv4_6.py b/src/mobius/models/minicpmv4_6.py new file mode 100644 index 000000000..fe6ba40f8 --- /dev/null +++ b/src/mobius/models/minicpmv4_6.py @@ -0,0 +1,754 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""OpenBMB MiniCPM-V-4.6 vision-language model. + +Replicates ``MiniCPMV4_6ForConditionalGeneration`` from Transformers 5.7: +packed variable-resolution SigLIP2 vision, an in-tower window-attention +merger, a second MLP spatial merger, and the Qwen3.5 hybrid text decoder. +""" + +from __future__ import annotations + +import onnx_ir as ir +import torch +from onnxscript import OpBuilder, nn + +from mobius._configs import ArchitectureConfig +from mobius.components import ( + Embedding, + LayerNorm, + Linear, + VisionAttention, + VisionEncoderLayer, + get_activation, +) +from mobius.models.qwen35 import Qwen35TextModel + + +class MiniCPMV46Config(ArchitectureConfig): + """Mobius configuration preserving MiniCPM's composite model identity.""" + + @classmethod + def from_transformers(cls, config, parent_config=None) -> MiniCPMV46Config: + result = super().from_transformers(config, parent_config=parent_config) + composite = parent_config or config + result.model_type = "minicpmv4_6" + result.video_token_id = getattr(composite, "video_token_id", None) + result.downsample_mode = getattr(composite, "downsample_mode", "16x") + return result + + +def _grid_columns(op: OpBuilder, target_sizes: ir.Value) -> tuple[ir.Value, ir.Value]: + """Return the per-visual-unit patch-grid heights and widths.""" + target_sizes = op.Cast(target_sizes, to=7) + heights = op.Squeeze( + op.Slice(target_sizes, [0], [1], axes=[1]), + [1], + ) + widths = op.Squeeze( + op.Slice(target_sizes, [1], [2], axes=[1]), + [1], + ) + return heights, widths + + +def _packed_patch_coordinates( + op: OpBuilder, target_sizes: ir.Value +) -> tuple[ir.Value, ir.Value, ir.Value]: + """Map each packed patch to ``(visual_unit, row, column)``.""" + heights, widths = _grid_columns(op, target_sizes) + lengths = op.Mul(heights, widths) + ends = op.CumSum(lengths, 0) + starts = op.Sub(ends, lengths) + total = op.ReduceSum(lengths, keepdims=0) + packed_index = op.Range(0, total, 1) + + # The first cumulative end greater than each packed index identifies its + # source image crop or video frame. + reached_end = op.GreaterOrEqual( + op.Unsqueeze(packed_index, [1]), + op.Unsqueeze(ends, [0]), + ) + visual_index = op.ReduceSum( + op.Cast(reached_end, to=7), + [1], + keepdims=0, + ) + local_index = op.Sub(packed_index, op.Gather(starts, visual_index)) + local_width = op.Gather(widths, visual_index) + rows = op.Div(local_index, local_width) + columns = op.Mod(local_index, local_width) + return visual_index, rows, columns + + +def _max_grid_size(op: OpBuilder, target_sizes: ir.Value) -> tuple[ir.Value, ir.Value]: + heights, widths = _grid_columns(op, target_sizes) + return ( + op.ReduceMax(heights, keepdims=1), + op.ReduceMax(widths, keepdims=1), + ) + + +def _grid_mask( + op: OpBuilder, + target_sizes: ir.Value, + max_height: ir.Value, + max_width: ir.Value, +) -> ir.Value: + """Boolean ``[N, max_h, max_w]`` mask for a padded ragged patch grid.""" + heights, widths = _grid_columns(op, target_sizes) + row_ids = op.Range(0, op.Squeeze(max_height, [0]), 1) + column_ids = op.Range(0, op.Squeeze(max_width, [0]), 1) + valid_rows = op.Less( + op.Unsqueeze(row_ids, [0]), + op.Unsqueeze(heights, [1]), + ) + valid_columns = op.Less( + op.Unsqueeze(column_ids, [0]), + op.Unsqueeze(widths, [1]), + ) + return op.And( + op.Unsqueeze(valid_rows, [2]), + op.Unsqueeze(valid_columns, [1]), + ) + + +def _unpack_padded_grid( + op: OpBuilder, + hidden_states: ir.Value, + target_sizes: ir.Value, + hidden_size: int, +) -> ir.Value: + """Scatter packed patches into ``[N, max_h*max_w, D]`` padded batches.""" + visual_index, rows, columns = _packed_patch_coordinates(op, target_sizes) + coordinates = op.Concat( + op.Unsqueeze(visual_index, [1]), + op.Unsqueeze(rows, [1]), + op.Unsqueeze(columns, [1]), + axis=1, + ) + max_height, max_width = _max_grid_size(op, target_sizes) + grid_shape = op.Concat( + op.Shape(target_sizes, start=0, end=1), + max_height, + max_width, + op.Constant(value_ints=[hidden_size]), + axis=0, + ) + patches = op.Squeeze(hidden_states, [0]) + padded = op.Expand(op.CastLike(0.0, patches), grid_shape) + padded = op.ScatterND(padded, coordinates, patches) + return op.Reshape( + padded, + op.Concat( + op.Shape(target_sizes, start=0, end=1), + op.Mul(max_height, max_width), + op.Constant(value_ints=[hidden_size]), + axis=0, + ), + ) + + +def _vision_attention_bias( + op: OpBuilder, + hidden_states: ir.Value, + target_sizes: ir.Value, +) -> ir.Value: + """Build a full additive mask for padded per-image patch batches.""" + max_height, max_width = _max_grid_size(op, target_sizes) + valid = op.Reshape( + _grid_mask(op, target_sizes, max_height, max_width), + op.Concat( + op.Shape(target_sizes, start=0, end=1), + op.Mul(max_height, max_width), + axis=0, + ), + ) + key_bias = op.Where( + op.Unsqueeze(valid, [1, 2]), + op.CastLike(0.0, hidden_states), + op.CastLike(-10_000.0, hidden_states), + ) + sequence_length = op.Mul(max_height, max_width) + return op.Expand( + key_bias, + op.Concat( + op.Shape(target_sizes, start=0, end=1), + op.Constant(value_ints=[1]), + sequence_length, + sequence_length, + axis=0, + ), + ) + + +def _spatial_windows( + op: OpBuilder, + hidden_states: ir.Value, + target_sizes: ir.Value, + hidden_size: int, + kernel_size: tuple[int, int], +) -> tuple[ir.Value, ir.Value, ir.Value]: + """Arrange ``[B, H*W, D]`` as ``[B, H/k, W/k, k, k, D]`` windows.""" + batch = op.Shape(target_sizes, start=0, end=1) + height, width = _max_grid_size(op, target_sizes) + kernel_h, kernel_w = kernel_size + merged_h = op.Div(height, op.Constant(value_ints=[kernel_h])) + merged_w = op.Div(width, op.Constant(value_ints=[kernel_w])) + grid = op.Reshape( + hidden_states, + op.Concat( + batch, + merged_h, + op.Constant(value_ints=[kernel_h]), + merged_w, + op.Constant(value_ints=[kernel_w, hidden_size]), + axis=0, + ), + ) + # (B, H/k, k, W/k, k, D) -> (B, H/k, W/k, k, k, D) + grid = op.Transpose(grid, perm=[0, 1, 3, 2, 4, 5]) + return grid, merged_h, merged_w + + +class _MiniCPMVisionEmbeddings(nn.Module): + """NaViT patch embedding with nearest-neighbor learned 2D positions.""" + + def __init__( + self, + image_size: int, + patch_size: int, + hidden_size: int, + num_channels: int, + ): + super().__init__() + self.patch_size = patch_size + self.hidden_size = hidden_size + self.position_side = image_size // patch_size + self.patch_embedding = nn.Parameter( + [hidden_size, num_channels, patch_size, patch_size], + name="patch_embedding.weight", + ) + self.patch_embedding_bias = nn.Parameter([hidden_size], name="patch_embedding.bias") + self.position_embedding = nn.Parameter( + [self.position_side**2, hidden_size], + name="position_embedding.weight", + ) + + def forward( + self, + op: OpBuilder, + pixel_values: ir.Value, + target_sizes: ir.Value, + ): + # NaViT input is packed horizontally: (1, C, patch_size, total_patch_width). + patches = op.Conv( + pixel_values, + self.patch_embedding, + self.patch_embedding_bias, + kernel_shape=[self.patch_size, self.patch_size], + strides=[self.patch_size, self.patch_size], + ) + # (packed_batch, D, 1, patches) -> (1, total_patches, D) + patches = op.Transpose(patches, perm=[0, 2, 3, 1]) + patches = op.Reshape( + patches, + op.Constant(value_ints=[1, -1, self.hidden_size]), + ) + + visual_index, rows, columns = _packed_patch_coordinates(op, target_sizes) + heights, widths = _grid_columns(op, target_sizes) + patch_heights = op.Gather(heights, visual_index) + patch_widths = op.Gather(widths, visual_index) + side = op.Constant(value_int=self.position_side) + bucket_h = op.Div(op.Mul(rows, side), patch_heights) + bucket_w = op.Div(op.Mul(columns, side), patch_widths) + position_ids = op.Add(op.Mul(bucket_h, side), bucket_w) + position_embeddings = op.Gather(self.position_embedding, position_ids) + return op.Add(patches, op.Unsqueeze(position_embeddings, [0])) + + +class _MiniCPMViTWindowAttentionMerger(nn.Module): + """2x2 local attention followed by spatial MLP compression.""" + + def __init__(self, config: ArchitectureConfig): + super().__init__() + vision = config.vision + assert vision is not None + assert vision.hidden_size is not None + assert vision.intermediate_size is not None + self.hidden_size = vision.hidden_size + self.kernel_size = vision.window_kernel_size + self._needs_bf16_cast = config.dtype == ir.DataType.BFLOAT16 + window_tokens = self.kernel_size[0] * self.kernel_size[1] + window_hidden = self.hidden_size * window_tokens + window_intermediate = vision.intermediate_size * window_tokens + self.self_attn = VisionAttention(self.hidden_size, vision.num_attention_heads or 1) + self.layer_norm1 = LayerNorm(self.hidden_size, eps=vision.norm_eps) + self.pre_norm = LayerNorm(window_hidden, eps=vision.norm_eps) + self.linear_1 = Linear(window_hidden, window_intermediate, bias=True) + self.act = get_activation("gelu_pytorch_tanh") + self.linear_2 = Linear(window_intermediate, self.hidden_size, bias=True) + + def forward( + self, + op: OpBuilder, + hidden_states: ir.Value, + target_sizes: ir.Value, + ): + residual = hidden_states + normalized = self.layer_norm1(op, hidden_states) + grid, merged_h, merged_w = _spatial_windows( + op, normalized, target_sizes, self.hidden_size, self.kernel_size + ) + window_tokens = self.kernel_size[0] * self.kernel_size[1] + windows = op.Reshape( + grid, + op.Constant(value_ints=[-1, window_tokens, self.hidden_size]), + ) + windows = self.self_attn(op, windows) + grid = op.Reshape( + windows, + op.Concat( + op.Shape(target_sizes, start=0, end=1), + merged_h, + merged_w, + op.Constant( + value_ints=[ + self.kernel_size[0], + self.kernel_size[1], + self.hidden_size, + ] + ), + axis=0, + ), + ) + # Restore raster order before the attention residual. + grid = op.Transpose(grid, perm=[0, 1, 3, 2, 4, 5]) + attended = op.Reshape( + grid, + op.Concat( + op.Shape(target_sizes, start=0, end=1), + op.Mul( + op.Mul(merged_h, merged_w), + op.Constant(value_ints=[window_tokens]), + ), + op.Constant(value_ints=[self.hidden_size]), + axis=0, + ), + ) + hidden_states = op.Add(residual, attended) + + # Merge each 2x2 block: (B, H, W, D) -> (B*H/2*W/2, 4D). + merged_grid, _, _ = _spatial_windows( + op, hidden_states, target_sizes, self.hidden_size, self.kernel_size + ) + if self._needs_bf16_cast: + # ReduceMean does not have an ORT BF16 kernel on CPU or CUDA. + merge_residual = op.CastLike( + op.ReduceMean( + op.CastLike(merged_grid, 0.0), + [3, 4], + keepdims=0, + ), + merged_grid, + ) + else: + merge_residual = op.ReduceMean(merged_grid, [3, 4], keepdims=0) + merge_residual = op.Reshape( + merge_residual, + op.Constant(value_ints=[-1, self.hidden_size]), + ) + flat = op.Reshape( + merged_grid, + op.Constant( + value_ints=[ + -1, + window_tokens * self.hidden_size, + ] + ), + ) + flat = self.pre_norm(op, flat) + flat = self.linear_1(op, flat) + flat = self.act(op, flat) + flat = self.linear_2(op, flat) + flat = op.Add(flat, merge_residual) + return op.Reshape( + flat, + op.Concat( + op.Shape(target_sizes, start=0, end=1), + op.Mul(merged_h, merged_w), + op.Constant(value_ints=[self.hidden_size]), + axis=0, + ), + ) + + +class _MiniCPMVisionTower(nn.Module): + """Variable-resolution SigLIP2 tower with an inserted attention merger.""" + + def __init__(self, config: ArchitectureConfig): + super().__init__() + vision = config.vision + assert vision is not None + assert vision.image_size is not None + assert vision.patch_size is not None + assert vision.hidden_size is not None + assert vision.intermediate_size is not None + assert vision.num_hidden_layers is not None + assert vision.num_attention_heads is not None + self.hidden_size = vision.hidden_size + self.insert_layer_id = ( + vision.insert_layer_id if vision.insert_layer_id is not None else 6 + ) + self.use_vit_merger = config.downsample_mode != "4x" + self.embeddings = _MiniCPMVisionEmbeddings( + vision.image_size, + vision.patch_size, + vision.hidden_size, + vision.in_channels, + ) + self.encoder = _MiniCPMVisionEncoder(config) + self.post_layernorm = LayerNorm(vision.hidden_size, eps=vision.norm_eps) + + def forward( + self, + op: OpBuilder, + pixel_values: ir.Value, + target_sizes: ir.Value, + ): + hidden_states = self.embeddings(op, pixel_values, target_sizes) + hidden_states = _unpack_padded_grid( + op, + hidden_states, + target_sizes, + self.hidden_size, + ) + hidden_states = self.encoder( + op, + hidden_states, + target_sizes=target_sizes, + insert_layer_id=self.insert_layer_id if self.use_vit_merger else -1, + ) + return self.post_layernorm(op, hidden_states) + + +class _MiniCPMVisionEncoder(nn.Module): + """Container matching HuggingFace's ``vision_tower.encoder.layers`` names.""" + + def __init__(self, config: ArchitectureConfig): + super().__init__() + vision = config.vision + assert vision is not None + assert vision.hidden_size is not None + assert vision.intermediate_size is not None + assert vision.num_hidden_layers is not None + assert vision.num_attention_heads is not None + self.layers = nn.ModuleList( + [ + VisionEncoderLayer( + vision.hidden_size, + vision.intermediate_size, + vision.num_attention_heads, + vision.norm_eps, + ) + for _ in range(vision.num_hidden_layers) + ] + ) + # The upstream module registers vit_merger beside encoder. Keeping it + # here lets the encoder call every layer through its own scope so layer + # initializer names retain ``encoder.layers.N``; preprocessing adds + # this one extra ``encoder.`` segment for the merger weights. + self.vit_merger = _MiniCPMViTWindowAttentionMerger(config) + + def forward( + self, + op: OpBuilder, + hidden_states: ir.Value, + target_sizes: ir.Value, + insert_layer_id: int, + ): + current_sizes = op.Cast(target_sizes, to=7) + attention_bias = _vision_attention_bias( + op, + hidden_states, + current_sizes, + ) + for layer_idx, layer in enumerate(self.layers): + hidden_states = layer(op, hidden_states, attention_bias) + if layer_idx == insert_layer_id: + hidden_states = self.vit_merger(op, hidden_states, current_sizes) + current_sizes = op.Div(current_sizes, 2) + attention_bias = _vision_attention_bias( + op, + hidden_states, + current_sizes, + ) + return hidden_states + + +class _MiniCPMDownsampleMLP(nn.Module): + """One 2x2 spatial merge and projection, matching ``merger.mlp.N``.""" + + def __init__(self, hidden_size: int, output_size: int): + super().__init__() + merged_hidden_size = hidden_size * 4 + self.pre_norm = LayerNorm(merged_hidden_size, eps=1e-6) + self.linear_1 = Linear(merged_hidden_size, merged_hidden_size, bias=True) + self.act = get_activation("gelu") + self.linear_2 = Linear(merged_hidden_size, output_size, bias=True) + + def forward(self, op: OpBuilder, hidden_states: ir.Value): + hidden_states = self.pre_norm(op, hidden_states) + hidden_states = self.linear_1(op, hidden_states) + hidden_states = self.act(op, hidden_states) + return self.linear_2(op, hidden_states) + + +class _MiniCPMMerger(nn.Module): + """Final spatial merger from vision width into the text embedding width.""" + + def __init__(self, config: ArchitectureConfig): + super().__init__() + vision = config.vision + assert vision is not None and vision.hidden_size is not None + self.hidden_size = vision.hidden_size + self.kernel_size = vision.merge_kernel_size + self.merger_times = vision.merger_times + self.vision_downsample = 1 if config.downsample_mode == "4x" else 2 + self._needs_bf16_cast = config.dtype == ir.DataType.BFLOAT16 + self.output_sizes = [ + self.hidden_size if i < self.merger_times - 1 else config.hidden_size + for i in range(self.merger_times) + ] + mlps = [ + _MiniCPMDownsampleMLP( + self.hidden_size, + self.output_sizes[i], + ) + for i in range(self.merger_times) + ] + self.mlp = nn.ModuleList(mlps) + + def forward( + self, + op: OpBuilder, + hidden_states: ir.Value, + target_sizes: ir.Value, + ): + # Default 16x mode enters after the in-tower 2x2 merger. In 4x mode + # the tower is unmerged, so this projector performs the only 2x2 merge. + target_sizes = op.Div( + op.Cast(target_sizes, to=7), + self.vision_downsample, + ) + current_hidden = self.hidden_size + for index, mlp in enumerate(self.mlp): + grid, merged_h, merged_w = _spatial_windows( + op, + hidden_states, + target_sizes, + current_hidden, + self.kernel_size, + ) + merged_dim = self.kernel_size[0] * self.kernel_size[1] * current_hidden + hidden_states = op.Reshape( + grid, + op.Constant(value_ints=[-1, merged_dim]), + ) + hidden_states = mlp(op, hidden_states) + target_sizes = op.Concat( + op.Div( + op.Slice(target_sizes, [0], [1], axes=[1]), + self.kernel_size[0], + ), + op.Div( + op.Slice(target_sizes, [1], [2], axes=[1]), + self.kernel_size[1], + ), + axis=1, + ) + current_hidden = self.output_sizes[index] + hidden_states = op.Reshape( + hidden_states, + op.Concat( + op.Shape(target_sizes, start=0, end=1), + op.Mul(merged_h, merged_w), + op.Shape(hidden_states, start=1, end=2), + axis=0, + ), + ) + max_height, max_width = _max_grid_size(op, target_sizes) + valid = op.Reshape( + _grid_mask(op, target_sizes, max_height, max_width), + [-1], + ) + hidden_states = op.Reshape( + hidden_states, + op.Concat( + op.Constant(value_ints=[-1]), + op.Shape(hidden_states, start=2, end=3), + axis=0, + ), + ) + if self._needs_bf16_cast: + # ONNX Compress excludes BF16 from its type constraint. Compaction + # is data movement only, so cast through FLOAT and restore dtype. + compacted = op.Compress( + op.CastLike(hidden_states, 0.0), + valid, + axis=0, + ) + return op.CastLike(compacted, hidden_states) + return op.Compress(hidden_states, valid, axis=0) + + +class _MiniCPMVisionEncoderModel(nn.Module): + """Packed SigLIP2 encoder plus both MiniCPM visual token mergers.""" + + def __init__(self, config: ArchitectureConfig): + super().__init__() + self.vision_tower = _MiniCPMVisionTower(config) + self.merger = _MiniCPMMerger(config) + + def forward( + self, + op: OpBuilder, + pixel_values: ir.Value, + target_sizes: ir.Value, + ): + hidden_states = self.vision_tower(op, pixel_values, target_sizes) + return self.merger(op, hidden_states, target_sizes) + + +class _MiniCPMDecoderModel(nn.Module): + """Qwen3.5 hybrid decoder taking pre-fused ``inputs_embeds``.""" + + def __init__(self, config: ArchitectureConfig): + super().__init__() + self.model = Qwen35TextModel(config) + self.lm_head = Linear(config.hidden_size, config.vocab_size, bias=False) + + def forward( + self, + op: OpBuilder, + inputs_embeds: ir.Value, + attention_mask: ir.Value, + position_ids: ir.Value, + past_key_values: list | None = None, + ): + hidden_states, present_key_values = self.model( + op, + input_ids=None, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + ) + return self.lm_head(op, hidden_states), present_key_values + + +class _MiniCPMEmbeddingModel(nn.Module): + """Fuse packed image or video features into the token embedding sequence.""" + + def __init__(self, config: ArchitectureConfig): + super().__init__() + self.embed_tokens = Embedding( + config.vocab_size, config.hidden_size, config.pad_token_id + ) + self.image_token_id = config.image_token_id or 248056 + self.video_token_id = config.video_token_id or 248057 + + def forward( + self, + op: OpBuilder, + input_ids: ir.Value, + image_features: ir.Value, + ): + text_embeds = self.embed_tokens(op, input_ids) + media_mask = op.Or( + op.Equal(input_ids, self.image_token_id), + op.Equal(input_ids, self.video_token_id), + ) + # HF masked_scatter consumes the packed feature stream in batch-major + # order, so flatten before CumSum instead of restarting at each row. + flat_media_mask = op.Reshape(media_mask, [-1]) + media_indices = op.Clip( + op.Sub( + op.CumSum(op.Cast(flat_media_mask, to=ir.DataType.INT64), 0), + 1, + ), + 0, + ) + # Keep text-only inference valid when the feature tensor has zero rows. + padding = op.Expand( + op.CastLike(0.0, image_features), + op.Concat( + op.Constant(value_ints=[1]), + op.Shape(image_features, start=1, end=2), + axis=0, + ), + ) + features = op.Gather( + op.Concat(image_features, padding, axis=0), + media_indices, + axis=0, + ) + features = op.Reshape(features, op.Shape(text_embeds)) + return op.Where(op.Unsqueeze(media_mask, [-1]), features, text_embeds) + + +class MiniCPMV46ForConditionalGeneration(nn.Module): + """OpenBMB MiniCPM-V-4.6 image/video conditional generation model.""" + + default_task: str = "minicpm-vl" + category: str = "Multimodal" + config_class: type = MiniCPMV46Config + + def __init__(self, config: ArchitectureConfig): + super().__init__() + self.config = config + self.decoder = _MiniCPMDecoderModel(config) + self.vision_encoder = _MiniCPMVisionEncoderModel(config) + self.embedding = _MiniCPMEmbeddingModel(config) + + def forward(self, op: OpBuilder, **kwargs): + raise NotImplementedError( + "MiniCPMV46ForConditionalGeneration uses MiniCPMVLTask, which " + "builds decoder, vision_encoder, and embedding graphs separately." + ) + + def preprocess_weights( + self, state_dict: dict[str, torch.Tensor] + ) -> dict[str, torch.Tensor]: + """Route the HuggingFace checkpoint into the three ONNX sub-models.""" + renamed: dict[str, torch.Tensor] = {} + embed_weight: torch.Tensor | None = None + for key, value in state_dict.items(): + if key.startswith(("mtp_", "mtp.")): + continue + if key.startswith("model.vision_tower."): + name = "vision_encoder." + key[len("model.") :] + name = name.replace(".mlp.fc1.", ".mlp.up_proj.") + name = name.replace(".mlp.fc2.", ".mlp.down_proj.") + name = name.replace( + ".vision_tower.vit_merger.", + ".vision_tower.encoder.vit_merger.", + ) + renamed[name] = value + elif key.startswith("model.merger."): + renamed["vision_encoder." + key[len("model.") :]] = value + elif key == "model.language_model.embed_tokens.weight": + embed_weight = value + renamed["decoder.model.embed_tokens.weight"] = value + renamed["embedding.embed_tokens.weight"] = value + elif key.startswith("model.language_model."): + suffix = key[len("model.language_model.") :] + renamed[f"decoder.model.{suffix}"] = value + elif key == "lm_head.weight": + renamed["decoder.lm_head.weight"] = value + + if self.config.tie_word_embeddings and embed_weight is not None: + renamed["decoder.lm_head.weight"] = embed_weight + return renamed diff --git a/src/mobius/models/minicpmv4_6_test.py b/src/mobius/models/minicpmv4_6_test.py new file mode 100644 index 000000000..ca9009166 --- /dev/null +++ b/src/mobius/models/minicpmv4_6_test.py @@ -0,0 +1,269 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +from __future__ import annotations + +import dataclasses + +import numpy as np +import onnx_ir as ir +import torch + +from mobius._configs import ArchitectureConfig, VisionConfig +from mobius._testing.ort_inference import OnnxModelSession +from mobius._weight_loading import apply_weights +from mobius.models.minicpmv4_6 import MiniCPMV46ForConditionalGeneration +from mobius.tasks import MiniCPMVLTask + + +def _tiny_config() -> ArchitectureConfig: + return ArchitectureConfig( + hidden_size=64, + intermediate_size=128, + num_attention_heads=4, + num_key_value_heads=2, + head_dim=16, + num_hidden_layers=4, + vocab_size=256, + max_position_embeddings=128, + hidden_act="silu", + rms_norm_eps=1e-6, + rope_type="default", + rope_theta=10_000.0, + partial_rotary_factor=0.25, + layer_types=[ + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + ], + linear_num_key_heads=2, + linear_key_head_dim=16, + linear_num_value_heads=2, + linear_value_head_dim=16, + linear_conv_kernel_dim=4, + image_token_id=250, + video_token_id=251, + vision=VisionConfig( + hidden_size=32, + intermediate_size=64, + num_hidden_layers=2, + num_attention_heads=2, + image_size=56, + patch_size=14, + norm_eps=1e-6, + in_channels=3, + num_position_embeddings=16, + insert_layer_id=0, + window_kernel_size=(2, 2), + merge_kernel_size=(2, 2), + merger_times=1, + ), + dtype=ir.DataType.FLOAT, + ) + + +def test_minicpmv4_6_synthetic_vision_parity(): + """L3: packed vision + both mergers match Transformers with random weights.""" + from transformers.models.minicpmv4_6.configuration_minicpmv4_6 import ( + MiniCPMV4_6Config, + MiniCPMV4_6VisionConfig, + ) + from transformers.models.minicpmv4_6.modeling_minicpmv4_6 import ( + MiniCPMV4_6Merger, + MiniCPMV4_6VisionModel, + ) + + torch.manual_seed(42) + vision_config = MiniCPMV4_6VisionConfig( + hidden_size=32, + intermediate_size=64, + num_hidden_layers=2, + num_attention_heads=2, + image_size=56, + patch_size=14, + ) + hf_config = MiniCPMV4_6Config( + vision_config=vision_config.to_dict(), + insert_layer_id=0, + text_config={ + "model_type": "qwen3_5_text", + "hidden_size": 64, + "intermediate_size": 128, + "num_hidden_layers": 4, + "num_attention_heads": 4, + "num_key_value_heads": 2, + "head_dim": 16, + "vocab_size": 256, + "layer_types": [ + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + ], + "linear_num_key_heads": 2, + "linear_key_head_dim": 16, + "linear_num_value_heads": 2, + "linear_value_head_dim": 16, + }, + ) + hf_vision = MiniCPMV4_6VisionModel(hf_config.vision_config).eval() + hf_merger = MiniCPMV4_6Merger(hf_config).eval() + + config = _tiny_config() + module = MiniCPMV46ForConditionalGeneration(config) + package = MiniCPMVLTask().build(module, config) + state_dict = { + **{ + f"model.vision_tower.{name}": value + for name, value in hf_vision.state_dict().items() + }, + **{f"model.merger.{name}": value for name, value in hf_merger.state_dict().items()}, + } + processed_weights = module.preprocess_weights(state_dict) + graph_parameters = { + name + for name in package["vision_encoder"].graph.initializers + if name.startswith("vision_encoder.") + } + assert set(processed_weights) == graph_parameters + apply_weights( + package["vision_encoder"], + processed_weights, + ) + + pixel_values = torch.randn(1, 3, 14, 224) + target_sizes = torch.tensor([[4, 4]], dtype=torch.int32) + with torch.no_grad(): + hidden_states = hf_vision( + pixel_values, + target_sizes=target_sizes, + ).last_hidden_state + expected = torch.cat( + hf_merger(hidden_states, target_sizes // 2), + dim=0, + ).numpy() + + session = OnnxModelSession(package["vision_encoder"]) + actual = session.run( + { + "pixel_values": pixel_values.numpy(), + "target_sizes": target_sizes.numpy(), + } + )["image_features"] + np.testing.assert_allclose(actual, expected, rtol=1e-4, atol=1e-4) + + # Default slice-mode preprocessing commonly emits an overview crop and + # detail crops with different grids. Compare the packed ONNX call against + # independent HF calls, which are the ragged reference semantics. + target_sizes = torch.tensor([[4, 4], [4, 8]], dtype=torch.int32) + pixel_values = torch.randn(1, 3, 14, (4 * 4 + 4 * 8) * 14) + expected_parts = [] + start = 0 + with torch.no_grad(): + for size in target_sizes: + num_patches = int(size.prod()) + end = start + num_patches * 14 + unit_pixels = pixel_values[:, :, :, start:end] + unit_size = size.unsqueeze(0) + hidden_states = hf_vision( + unit_pixels, + target_sizes=unit_size, + ).last_hidden_state + expected_parts.extend(hf_merger(hidden_states, unit_size // 2)) + start = end + expected = torch.cat(expected_parts, dim=0).numpy() + actual = session.run( + { + "pixel_values": pixel_values.numpy(), + "target_sizes": target_sizes.numpy(), + } + )["image_features"] + session.close() + np.testing.assert_allclose(actual, expected, rtol=1e-4, atol=1e-4) + + # The optional 4x mode skips only the in-tower merger, leaving the final + # 2x2 projector to produce four times as many visual tokens as 16x mode. + config_4x = dataclasses.replace(config, downsample_mode="4x") + module_4x = MiniCPMV46ForConditionalGeneration(config_4x) + package_4x = MiniCPMVLTask().build(module_4x, config_4x) + processed_4x = module_4x.preprocess_weights(state_dict) + graph_parameters_4x = set(package_4x["vision_encoder"].graph.initializers) + apply_weights( + package_4x["vision_encoder"], + {name: value for name, value in processed_4x.items() if name in graph_parameters_4x}, + ) + target_sizes = torch.tensor([[4, 4]], dtype=torch.int32) + pixel_values = torch.randn(1, 3, 14, 224) + with torch.no_grad(): + hidden_states = hf_vision( + pixel_values, + target_sizes=target_sizes, + use_vit_merger=False, + ).last_hidden_state + expected = torch.cat( + hf_merger(hidden_states, target_sizes), + dim=0, + ).numpy() + session = OnnxModelSession(package_4x["vision_encoder"]) + actual = session.run( + { + "pixel_values": pixel_values.numpy(), + "target_sizes": target_sizes.numpy(), + } + )["image_features"] + session.close() + assert actual.shape[0] == 4 + np.testing.assert_allclose(actual, expected, rtol=1e-4, atol=1e-4) + + +def test_minicpmv4_6_embedding_mixes_image_and_video_tokens(): + """Packed features follow HF masked_scatter order across batch rows.""" + config = _tiny_config() + package = MiniCPMVLTask().build( + MiniCPMV46ForConditionalGeneration(config), + config, + ) + rng = np.random.default_rng(42) + embedding_weight = None + for name, initializer in package["embedding"].graph.initializers.items(): + if initializer.const_value is None: + value = rng.standard_normal(initializer.shape).astype(np.float32) + initializer.const_value = ir.tensor(value) + if name.endswith("embed_tokens.weight"): + embedding_weight = value + assert embedding_weight is not None + + input_ids = np.array( + [ + [1, 250, 2, 251], + [251, 3, 250, 4], + ], + dtype=np.int64, + ) + features = np.arange(4 * config.hidden_size, dtype=np.float32).reshape( + 4, config.hidden_size + ) + session = OnnxModelSession(package["embedding"]) + result = session.run( + { + "input_ids": input_ids, + "image_features": features, + } + )["inputs_embeds"] + + media_mask = (input_ids == config.image_token_id) | (input_ids == config.video_token_id) + expected = embedding_weight[input_ids].copy() + expected[media_mask] = features + np.testing.assert_array_equal(result, expected) + + # Decode steps without new media pass an empty packed feature stream. + decode_ids = np.array([[5], [6]], dtype=np.int64) + decode_result = session.run( + { + "input_ids": decode_ids, + "image_features": np.empty((0, config.hidden_size), dtype=np.float32), + } + )["inputs_embeds"] + session.close() + np.testing.assert_array_equal(decode_result, embedding_weight[decode_ids]) diff --git a/src/mobius/tasks/__init__.py b/src/mobius/tasks/__init__.py index d7f947ffc..6295fc776 100644 --- a/src/mobius/tasks/__init__.py +++ b/src/mobius/tasks/__init__.py @@ -46,6 +46,7 @@ "ImageClassificationTask", "ModelTask", "MllamaVisionLanguageTask", + "MiniCPMVLTask", "MaskedDiffusionTask", "MoshiDepformerTask", "MoshiTemporalTask", @@ -124,6 +125,7 @@ from mobius.tasks._vision_language_3model import ( Cosmos3EdgeVLTask, HybridQwenVLTask, + MiniCPMVLTask, MllamaVisionLanguageTask, PixtralVLTask, QwenVLTask, @@ -165,6 +167,7 @@ "mllama-vision-language": MllamaVisionLanguageTask, "qwen-vl": QwenVLTask, "hybrid-qwen-vl": HybridQwenVLTask, + "minicpm-vl": MiniCPMVLTask, "qwen3-vl-vision-language": Qwen3VLVisionLanguageTask, "gemma4": Gemma4Task, "gemma4-text-generation": Gemma4TextCausalLMTask, diff --git a/src/mobius/tasks/_vision_language_3model.py b/src/mobius/tasks/_vision_language_3model.py index 42d804ed9..90cf9c022 100644 --- a/src/mobius/tasks/_vision_language_3model.py +++ b/src/mobius/tasks/_vision_language_3model.py @@ -250,6 +250,68 @@ def build( return ModelPackage(models, config=config) +class MiniCPMVLTask(VisionLanguageTask): + """MiniCPM-V packed-NaViT vision with a Qwen3.5 hybrid decoder.""" + + def build( + self, + module: nn.Module, + config: ArchitectureConfig, + ) -> ModelPackage: + self._validate_components(module) + models: dict[str, ir.Model] = {} + models["decoder"] = build_decoder_from_embeds( + module.decoder, + config, + hybrid=True, + ) + models["vision_encoder"] = self._build_vision(module.vision_encoder, config) + models["embedding"] = build_embedding_from_features( + module.embedding, + config, + feature_name="image_features", + feature_dim=config.hidden_size, + ) + return ModelPackage(models, config=config) + + def _build_vision( + self, + vision: nn.Module, + config: ArchitectureConfig, + ) -> ir.Model: + """Build packed pixels + patch-grid sizes -> compressed visual tokens.""" + packed_batch = ir.SymbolicDim("packed_batch") + packed_width = ir.SymbolicDim("packed_width") + num_visual_units = ir.SymbolicDim("num_visual_units") + vision_config = config.vision + assert vision_config is not None + patch_size = vision_config.patch_size or 14 + + graph, builder = _make_graph(name="vision_encoder") + pixel_values = builder.input( + "pixel_values", + dtype=config.dtype, + shape=[ + packed_batch, + vision_config.in_channels, + patch_size, + packed_width, + ], + ) + target_sizes = builder.input( + "target_sizes", + dtype=ir.DataType.INT32, + shape=[num_visual_units, 2], + ) + image_features = vision( + builder.op, + pixel_values=pixel_values, + target_sizes=target_sizes, + ) + builder.add_output(image_features, "image_features") + return _make_model(graph) + + class PixtralVLTask(VisionLanguageTask): """Vision-language task with dynamic-resolution Pixtral vision encoder. diff --git a/testdata/cases/vision-language/minicpm-v-4_6.yaml b/testdata/cases/vision-language/minicpm-v-4_6.yaml new file mode 100644 index 000000000..2d8b31fc2 --- /dev/null +++ b/testdata/cases/vision-language/minicpm-v-4_6.yaml @@ -0,0 +1,19 @@ +model_id: "openbmb/MiniCPM-V-4.6" +model_type: "minicpmv4_6" +revision: "8169864629825dc1d755a5aa1cd8b5935dcbc83f" +task_type: "image-text-to-text" +dtype: "float32" + +inputs: + prompts: + - "Describe this image in detail." + images: + - "pipeline-cat-chonk.jpeg" + +level: "L4+L5" + +generation: + max_new_tokens: 30 + do_sample: false + +notes: "MiniCPM-V-4.6 with packed variable-resolution SigLIP2 and Qwen3.5 hybrid text." diff --git a/testdata/golden/vision-language/minicpm-v-4_6.json b/testdata/golden/vision-language/minicpm-v-4_6.json new file mode 100644 index 000000000..ff56f8097 --- /dev/null +++ b/testdata/golden/vision-language/minicpm-v-4_6.json @@ -0,0 +1,384 @@ +{ + "top1_id": 760, + "top2_id": 561, + "top10_ids": [ + 760, + 561, + 92157, + 10288, + 3067, + 16721, + 1719, + 85190, + 15877, + 50821 + ], + "top10_logits": [ + "0x1.3c8a280000000p+5", + "0x1.f0b58a0000000p+4", + "0x1.7dfebc0000000p+4", + "0x1.64a30c0000000p+4", + "0x1.64762c0000000p+4", + "0x1.6281aa0000000p+4", + "0x1.603c1e0000000p+4", + "0x1.5be7220000000p+4", + "0x1.57dc380000000p+4", + "0x1.4c0c540000000p+4" + ], + "logits_summary": [ + "0x1.3c8a280000000p+5", + "-0x1.472b7a0000000p+3", + "0x1.c8f268d10cb0ap-1", + "0x1.26c048f97daeep+1" + ], + "input_ids": [ + 248045, + 846, + 198, + 248090, + 15, + 248091, + 248078, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248079, + 248088, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248089, + 248088, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248089, + 198, + 248088, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248089, + 248088, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248056, + 248089, + 198, + 72240, + 411, + 2099, + 303, + 7472, + 13, + 248046, + 198, + 248045, + 74455, + 198, + 248068, + 271, + 248069, + 271 + ] +} diff --git a/testdata/golden/vision-language/minicpm-v-4_6_generation.json b/testdata/golden/vision-language/minicpm-v-4_6_generation.json new file mode 100644 index 000000000..b1b37f04e --- /dev/null +++ b/testdata/golden/vision-language/minicpm-v-4_6_generation.json @@ -0,0 +1,37 @@ +{ + "model_id": "openbmb/MiniCPM-V-4.6", + "prompt": "Describe this image in detail.", + "generated_tokens": [ + 760, + 2099, + 4774, + 264, + 3349, + 11, + 65283, + 9572, + 10932, + 303, + 264, + 86684, + 4424, + 13, + 561, + 9572, + 682, + 264, + 21094, + 11, + 28704, + 88, + 10771, + 11, + 440, + 264, + 11702, + 11, + 71044, + 22162 + ], + "generated_text": "The image shows a large, fluffy animal standing in a snowy environment. The animal has a robust, bushy appearance, with a thick, textured coat" +} diff --git a/tests/_test_configs.py b/tests/_test_configs.py index 19ea16689..9d8971409 100644 --- a/tests/_test_configs.py +++ b/tests/_test_configs.py @@ -2089,6 +2089,22 @@ def _base_config(config_cls=None, **overrides) -> ArchitectureConfig: window_size=4, ) +_TINY_MINICPMV46_VISION = VisionConfig( + hidden_size=32, + intermediate_size=64, + num_hidden_layers=2, + num_attention_heads=2, + image_size=56, + patch_size=14, + norm_eps=1e-6, + in_channels=3, + num_position_embeddings=16, + insert_layer_id=0, + window_kernel_size=(2, 2), + merge_kernel_size=(2, 2), + merger_times=1, +) + _TINY_COSMOS3_EDGE_VISION = VisionConfig( hidden_size=32, @@ -2113,6 +2129,28 @@ def _base_config(config_cls=None, **overrides) -> ArchitectureConfig: VL_CONFIGS: list[tuple[str, dict, bool]] = [ # --- LLaVA family (vision-language, 3-model split) --- ("llava", {"vision": _TINY_VISION, "image_token_id": 32000}, True), + ( + "minicpmv4_6", + { + "vision": _TINY_MINICPMV46_VISION, + "image_token_id": 250, + "video_token_id": 251, + "num_hidden_layers": 4, + "layer_types": [ + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + ], + "partial_rotary_factor": 0.25, + "linear_num_key_heads": 2, + "linear_key_head_dim": 16, + "linear_num_value_heads": 2, + "linear_value_head_dim": 16, + "linear_conv_kernel_dim": 4, + }, + True, + ), ( "cosmos3_edge", { diff --git a/tests/integration_test.py b/tests/integration_test.py index 99218321c..747483596 100644 --- a/tests/integration_test.py +++ b/tests/integration_test.py @@ -68,6 +68,75 @@ def _model_accessible(model_id: str) -> bool: return True +@pytest.mark.integration +def test_minicpmv4_6_real_weight_vision_parity(): + """Real nonzero pixels match through SigLIP2 and both visual mergers.""" + import gc + + from transformers import AutoModelForImageTextToText, AutoProcessor + + model_id = "openbmb/MiniCPM-V-4.6" + processor = AutoProcessor.from_pretrained(model_id) + image = Image.open("testdata/pipeline-cat-chonk.jpeg").convert("RGB") + # A square source triggers the default overview + slice path with + # non-uniform patch grids (e.g. 32x32 overview and 40x24 slices). + image = image.resize((1024, 1024)) + prompt = processor.apply_chat_template( + [ + { + "role": "user", + "content": [ + {"type": "image", "image": "testdata/pipeline-cat-chonk.jpeg"}, + {"type": "text", "text": "Describe this image in detail."}, + ], + } + ], + tokenize=False, + add_generation_prompt=True, + ) + inputs = processor( + text=prompt, + images=[image], + return_tensors="pt", + ) + + hf_model = AutoModelForImageTextToText.from_pretrained( + model_id, + dtype=torch.float32, + ).eval() + expected_parts = [] + start = 0 + with torch.no_grad(): + for size in inputs["target_sizes"]: + num_patches = int(size.prod()) + end = start + num_patches * 14 + unit_pixels = inputs["pixel_values"][:, :, :, start:end] + expected_parts.extend( + hf_model.get_image_features( + unit_pixels, + size.unsqueeze(0), + ).pooler_output + ) + start = end + expected = torch.cat(expected_parts, dim=0).numpy() + del hf_model + gc.collect() + + package = build(model_id, dtype="float32", load_weights=True) + session = _make_session(package["vision_encoder"]) + actual = session.run( + { + "pixel_values": inputs["pixel_values"].numpy(), + "target_sizes": inputs["target_sizes"].numpy(), + } + )["image_features"] + session.close() + + assert float(np.linalg.norm(inputs["pixel_values"].numpy())) > 0.0 + assert np.unique(inputs["target_sizes"].numpy(), axis=0).shape[0] > 1 + np.testing.assert_allclose(actual, expected, rtol=1e-3, atol=1e-3) + + # --------------------------------------------------------------------------- # Model catalogue: small models for each supported architecture # diff --git a/tests/ort_genai_test.py b/tests/ort_genai_test.py index 162c98537..c48736a5d 100644 --- a/tests/ort_genai_test.py +++ b/tests/ort_genai_test.py @@ -390,3 +390,26 @@ def test_text_generation(self, model_id: str, tmp_path): "Model should generate at least one token" ) del generator + + +@pytest.mark.integration +@pytest.mark.integration_slow +def test_minicpmv4_6_package_loads(tmp_path): + """MiniCPM's custom packed vision graph loads through ORT GenAI.""" + from mobius import build + from mobius.integrations.ort_genai import export_package + + model_id = "openbmb/MiniCPM-V-4.6" + package = build(model_id, dtype="f32", load_weights=True) + output_dir = str(tmp_path / "minicpm-v-4.6") + manifest = export_package( + package, + output_dir, + hf_model_id=model_id, + progress_bar=False, + ) + + assert os.path.isfile(manifest["genai_config"]) + assert os.path.isfile(os.path.join(output_dir, "preprocessor_config.json")) + model = ort_genai.Model(output_dir) + assert ort_genai.Tokenizer(model) is not None