Add OpenBMB MiniCPM-V 4.6 support

README.md

@@ -29,7 +29,7 @@ multi-component export for pipelines.
 |---|---|
 | **Text Generation** | Llama 2/3/4, Mistral, Qwen 2/2.5/3/3.5/3.6, Phi-3/3.5, Gemma 1/2/3/4, Granite, GPT-2, OPT, OLMo, SmolLM3, and many more |
 | **Mixture of Experts** | PhiMoE, GPTOSS, Mixtral, OLMoE, DeepSeek-V2/V3, Qwen2-MoE, Qwen3-MoE, Qwen3-Next, GLM-4-MoE, Arctic, DBRX, Jamba |
-| **Multimodal** | Gemma 3/4, Phi-4MM (vision + audio + LoRA), LLaVA, InternVL2, Qwen2.5-VL, Qwen3-VL, Qwen3.5/3.6-VL, Pixtral |
+| **Multimodal** | Gemma 3/4, Phi-4MM (vision + audio + LoRA), LLaVA, InternVL2, MiniCPM-V 4.6, Qwen2.5-VL, Qwen3-VL, Qwen3.5/3.6-VL, Pixtral |
 | **Encoder-only** | BERT, RoBERTa, ALBERT, DeBERTa, DistilBERT, ELECTRA, XLNet |
 | **Encoder-Decoder** | BART, T5/mT5, Marian, M2M-100, Pegasus, BigBird-Pegasus |
 | **Speech-to-Text** | Whisper, FastConformer-RNNT, FunASR, Qwen3-ASR, SenseVoice |

src/mobius/_configs/_base.py

@@ -494,6 +494,8 @@ class ArchitectureConfig(BaseModelConfig):
     # Vision shared fields (accessed as top-level config.X by tasks)
     mm_tokens_per_image: int | None = None
     image_token_id: int | None = None
+    video_token_id: int | None = None
+    downsample_mode: str = "16x"
     spatial_merge_size: int = 2
     temporal_patch_size: int = 2
     deepstack_visual_indexes: list[int] | None = None

src/mobius/_configs/_extractors_test.py

@@ -343,3 +343,35 @@ def test_hunyuan_vl_mot_image_token_id_survives_default_hook(loaded_vision_hooks
     out = _extractors.extract_vision_config(cfg, None, "hunyuan_vl_mot")
     assert out["vision"].image_token_id == 12
     assert out.get("image_token_id") == 12
+
+
+def test_minicpm_vision_defaults_explicit_none_kernels(loaded_vision_hooks):
+    """Explicit None merger kernels fall back to MiniCPM's 2x2 defaults."""
+    vision_config = _FakeHFConfig(
+        model_type="minicpmv4_6_vision",
+        hidden_size=32,
+        intermediate_size=64,
+        num_hidden_layers=2,
+        num_attention_heads=2,
+        image_size=56,
+        patch_size=14,
+        layer_norm_eps=1e-6,
+        num_channels=3,
+        window_kernel_size=None,
+    )
+    parent_config = _FakeHFConfig(
+        model_type="minicpmv4_6",
+        vision_config=vision_config,
+        image_token_id=250,
+        merge_kernel_size=None,
+    )
+    text_config = _FakeHFConfig(model_type="qwen3_5_text")
+
+    out = _extractors.extract_vision_config(
+        text_config,
+        parent_config,
+        "qwen3_5_text",
+    )
+
+    assert out["vision"].window_kernel_size == (2, 2)
+    assert out["vision"].merge_kernel_size == (2, 2)

src/mobius/_configs/_sub_configs.py

@@ -82,6 +82,11 @@ class VisionConfig:
     # (HuggingFace convention, e.g. -2 for Phi-3.5-Vision). ``None`` means use
     # the final hidden state (all layers + post_layernorm).
     feature_layer: int | None = None
+    # MiniCPM-V packed-NaViT vision encoder and its two spatial mergers.
+    insert_layer_id: int | None = None
+    window_kernel_size: tuple[int, int] = (2, 2)
+    merge_kernel_size: tuple[int, int] = (2, 2)
+    merger_times: int = 1
 
 
 @dataclasses.dataclass

src/mobius/_configs/per_model/__init__.py

@@ -29,6 +29,7 @@
     _gemma4_unified_vision,
     _hunyuan_vl_mot_vision,
     _internvl_vision,
+    _minicpmv4_6_vision,
     _phi4mm_audio,
     _phi4mm_vision,
     _phi_vision,

src/mobius/_configs/per_model/_minicpmv4_6_vision.py

@@ -0,0 +1,38 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+"""MiniCPM-V-4.6 vision extractor hook."""
+
+from __future__ import annotations
+
+from mobius._configs._extractors import register_vision_hook
+
+
+@register_vision_hook("qwen3_5_text")
+def _minicpmv4_6_vision(config, parent_config, model_type: str, fields: dict):
+    """Extract the packed SigLIP2 geometry from the composite MiniCPM config."""
+    if getattr(parent_config, "model_type", None) != "minicpmv4_6":
+        return None
+
+    vision = parent_config.vision_config
+    image_size = getattr(vision, "image_size", 980)
+    patch_size = getattr(vision, "patch_size", 14)
+    fields.update(
+        model_type="minicpmv4_6_vision",
+        hidden_size=vision.hidden_size,
+        intermediate_size=vision.intermediate_size,
+        num_hidden_layers=vision.num_hidden_layers,
+        num_attention_heads=vision.num_attention_heads,
+        image_size=image_size,
+        patch_size=patch_size,
+        norm_eps=vision.layer_norm_eps,
+        hidden_act=getattr(vision, "hidden_act", "gelu_pytorch_tanh"),
+        in_channels=getattr(vision, "num_channels", 3),
+        num_position_embeddings=(image_size // patch_size) ** 2,
+        image_token_id=parent_config.image_token_id,
+        insert_layer_id=getattr(parent_config, "insert_layer_id", 6),
+        window_kernel_size=tuple(getattr(vision, "window_kernel_size", None) or (2, 2)),
+        merge_kernel_size=tuple(getattr(parent_config, "merge_kernel_size", None) or (2, 2)),
+        merger_times=getattr(parent_config, "merger_times", 1),
+    )
+    return None

src/mobius/_registry.py

@@ -126,6 +126,7 @@
 from mobius.models.llava import LLaVAModel
 from mobius.models.longcat_flash import LongcatFlashCausalLMModel
 from mobius.models.mamba import Mamba2CausalLMModel, MambaCausalLMModel
+from mobius.models.minicpmv4_6 import MiniCPMV46ForConditionalGeneration
 from mobius.models.minimax import MiniMaxCausalLMModel
 from mobius.models.mllama import MllamaCausalLMModel
 from mobius.models.modernbert import ModernBertDecoderModel, ModernBertModel
@@ -598,6 +599,10 @@ def _detect_fallback_registration(hf_config) -> ModelRegistration | None:
     "llava_next_video": ModelRegistration(LLaVAModel, task="vision-language"),
     "llava_onevision": ModelRegistration(LLaVAModel, task="vision-language"),
     "mistral3": ModelRegistration(LLaVAModel, task="pixtral-vl"),
+    "minicpmv4_6": ModelRegistration(
+        MiniCPMV46ForConditionalGeneration,
+        task="minicpm-vl",
+    ),
     "mllama": ModelRegistration(MllamaCausalLMModel, task="mllama-vision-language"),
     "molmo": ModelRegistration(LLaVAModel, task="vision-language"),
     "ovis2": ModelRegistration(LLaVAModel, task="vision-language"),
@@ -979,6 +984,7 @@ def _create_default_registry() -> ModelRegistry:
     "llava_onevision": "llava-hf/llava-onevision-qwen2-0.5b-ov-hf",
     "molmo": "allenai/MolmoE-1B-0924",
     "mistral3": "mistralai/Ministral-3-3B-Instruct-2512",
+    "minicpmv4_6": "openbmb/MiniCPM-V-4.6",
     "aya_vision": "CohereForAI/aya-vision-8b",
     "chameleon": "facebook/chameleon-7b",
     "cohere2_vision": "CohereForAI/c4ai-command-r7b-12-2024",

src/mobius/integrations/ort_genai/auto_export.py

@@ -93,6 +93,10 @@
     "qwen3_vl_text": "qwen3_vl",
     "qwen3_5": "qwen2_5_vl",
     "qwen3_5_vl": "qwen2_5_vl",
+    # MiniCPM uses standard 1D decoder position IDs (unlike Qwen-VL MRoPE).
+    # The phi3v multimodal runtime provides that contract; callers supply
+    # HF-preprocessed packed pixels through Generator.set_inputs().
+    "minicpmv4_6": "phi3v",
 }
 
 _GEMMA4_MODEL_TYPES = frozenset(
@@ -107,6 +111,7 @@
 # must preprocess with the HuggingFace processor and feed tensors via
 # ``Generator.set_inputs`` (see examples/gemma4_unified_ort_genai.py).
 _GEMMA4_UNIFIED_MODEL_TYPES = frozenset({"gemma4_unified", "gemma4_unified_text"})
+_MINICPM_MODEL_TYPES = frozenset({"minicpmv4_6"})
 # gemma-3 multimodal. build() unwraps the composite HF config to its text
 # sub-config, so at export time ``config.model_type`` is "gemma3_text" (not
 # "gemma3").
@@ -134,6 +139,9 @@
     "merges.txt",  # BPE
     "vocab.json",  # BPE
     "chat_template.jinja",  # Chat template for ORT GenAI
+    # Preserve HuggingFace processor metadata for VLMs whose preprocessing
+    # cannot be represented by an ort-extensions image_processor.json.
+    "preprocessor_config.json",
 ]
 
 
@@ -463,6 +471,17 @@ def _write_vision_processor_config(
             model_type,
         )
         return None
+    if model_type in _MINICPM_MODEL_TYPES:
+        # MiniCPM needs adaptive slicing and NaViT horizontal patch packing.
+        # ort-extensions has no equivalent transform, so preserving the HF
+        # processor output and injecting it through set_inputs is the only
+        # numerically faithful runtime path.
+        logger.info(
+            "Skipping image_processor.json for %s "
+            "(use MiniCPMV4_6Processor + Generator.set_inputs)",
+            model_type,
+        )
+        return None
 
     vision_model_type = getattr(vision, "model_type", None)
     is_pixtral = vision_model_type == "pixtral" or model_type in _PIXTRAL_MODEL_TYPES
@@ -848,16 +867,26 @@ def _write_genai_config(
         if image_token_id is not None:
             vision_input_mapping = _introspect_inputs(pkg, "vision_encoder")
             embedding_input_mapping = _introspect_inputs(pkg, "embedding")
+            if (
+                model_type := getattr(config, "model_type", "")
+            ) in _MINICPM_MODEL_TYPES and vision_input_mapping is not None:
+                # ORT GenAI's VisionInputs schema only accepts its predefined
+                # semantic keys. ``target_sizes`` remains an ONNX graph input
+                # and is supplied as a named tensor through set_inputs().
+                vision_input_mapping.pop("target_sizes", None)
 
             # spatial_merge_size and config_filename are config-level
             # properties that cannot be inferred from the graph.
             vision_kwargs: dict[str, Any] = {}
-            model_type = getattr(config, "model_type", "")
             if model_type in _GEMMA4_MODEL_TYPES:
                 vision_cfg = getattr(config, "vision", None)
                 vision_kwargs["spatial_merge_size"] = getattr(
                     vision_cfg, "spatial_merge_size", 2
                 )
+            elif model_type in _MINICPM_MODEL_TYPES:
+                # MiniCPM performs both 2x2 merges inside the ONNX vision
+                # graph and consumes HF-prepacked pixels, not Qwen grid_thw.
+                vision_kwargs["spatial_merge_size"] = None
             elif has_speech:
                 vision_kwargs["spatial_merge_size"] = None
             elif (

src/mobius/models/__init__.py

@@ -80,6 +80,7 @@
     "Mamba2CausalLMModel",
     "MambaCausalLMModel",
     "MiniMaxCausalLMModel",
+    "MiniCPMV46ForConditionalGeneration",
     "MimiModel",
     "MoshiDepformerModel",
     "MoshiTemporalModel",
@@ -217,6 +218,7 @@
 from mobius.models.longcat_flash import LongcatFlashCausalLMModel
 from mobius.models.mamba import Mamba2CausalLMModel, MambaCausalLMModel
 from mobius.models.mimi import MimiModel, mimi_default_config
+from mobius.models.minicpmv4_6 import MiniCPMV46ForConditionalGeneration
 from mobius.models.minimax import MiniMaxCausalLMModel
 from mobius.models.moe import (
     Ernie45MoECausalLMModel,