[Bugfix][Multimodal] Honor modality-scoped mm_processor_kwargs in every model that reads them

tests/models/multimodal/processing/test_glm4_1v.py

@@ -200,3 +200,92 @@ def test_video_loader_consistency(
         static_outputs["mm_kwargs"].get_data(),
         dynamic_outputs["mm_kwargs"].get_data(),
     )
+
+
+# Far above any stock GLM-4.1V pixel budget, so an override is unmistakable.
+_SCOPED_MAX_PIXELS = 469762048
+
+
+def _probe_max_pixels(
+    model_id: str, mm_processor_kwargs: dict | None
+) -> tuple[int, int]:
+    """Return the (video, image) pixel budgets vLLM computes."""
+    ctx = build_model_context(
+        model_id,
+        mm_processor_kwargs=mm_processor_kwargs,
+        limit_mm_per_prompt={"image": 1, "video": 1},
+    )
+    info = MULTIMODAL_REGISTRY.create_processor(ctx.model_config).info
+    return info._get_video_max_pixels(), info._get_image_max_pixels()
+
+
+@pytest.mark.skip_global_cleanup
+@pytest.mark.parametrize("model_id", ["zai-org/GLM-4.1V-9B-Thinking"])
+def test_videos_kwargs_max_pixels_does_not_leak_into_image_budget(model_id: str):
+    """A scoped ``videos_kwargs`` override must reach only the video budget.
+
+    The HF processor already honors the nested dict, so when vLLM's own
+    budget reads ignore it the two disagree about how many tokens a video
+    expands to.
+    """
+    stock_video, stock_image = _probe_max_pixels(model_id, None)
+    assert stock_video != _SCOPED_MAX_PIXELS
+    assert stock_image != _SCOPED_MAX_PIXELS
+
+    scoped_video, scoped_image = _probe_max_pixels(
+        model_id, {"videos_kwargs": {"max_pixels": _SCOPED_MAX_PIXELS}}
+    )
+    assert scoped_video == _SCOPED_MAX_PIXELS
+    assert scoped_image == stock_image
+
+    # A flat override keeps the previous shared-namespace behavior.
+    flat_video, flat_image = _probe_max_pixels(
+        model_id, {"max_pixels": _SCOPED_MAX_PIXELS}
+    )
+    assert flat_video == _SCOPED_MAX_PIXELS
+    assert flat_image == _SCOPED_MAX_PIXELS
+
+
+# Well below any stock GLM-4.1V image budget, so a leak into the shared
+# upper bound is unmistakable.
+_SMALL_MAX_PIXELS = 1_003_520
+
+
+def _probe_budgets(model_id: str, mm_processor_kwargs: dict | None) -> dict:
+    ctx = build_model_context(
+        model_id,
+        mm_processor_kwargs=mm_processor_kwargs,
+        limit_mm_per_prompt={"image": 1, "video": 1},
+    )
+    info = MULTIMODAL_REGISTRY.create_processor(ctx.model_config).info
+    return {
+        "image_max_pixels": info._get_image_max_pixels(),
+        "size_bound": tuple(info.get_image_size_with_most_features()),
+        "video_frames": info._get_max_video_frames(30_000),
+    }
+
+
+@pytest.mark.skip_global_cleanup
+@pytest.mark.parametrize("model_id", ["zai-org/GLM-4.1V-9B-Thinking"])
+def test_images_kwargs_max_pixels_does_not_leak_into_video_budget(model_id: str):
+    """An image-scoped override must not move the shared size upper bound.
+
+    ``get_image_size_with_most_features`` feeds the video frame budget and the
+    dummy data as well as the image budget. Scoping that bound to ``image``
+    would let an image-only override shrink the profiled video budget, which
+    no override of that modality should touch.
+    """
+    stock = _probe_budgets(model_id, None)
+    scoped = _probe_budgets(
+        model_id, {"images_kwargs": {"max_pixels": _SMALL_MAX_PIXELS}}
+    )
+
+    # The override reaches the per-item image read it is meant for.
+    assert stock["image_max_pixels"] != _SMALL_MAX_PIXELS
+    assert scoped["image_max_pixels"] == _SMALL_MAX_PIXELS
+
+    # It must not reach the bound that video sizing and dummy data share.
+    assert (scoped["size_bound"], scoped["video_frames"]) == (
+        stock["size_bound"],
+        stock["video_frames"],
+    )

tests/models/multimodal/processing/test_transformers_image.py

@@ -314,3 +314,69 @@ def test_nested_image_fields_split_per_image(processor_cls):
     for item in items:
         pixel_values = item["pixel_values"].data
         assert pixel_values.shape[1] == int(item["num_image_patches"].data)
+
+
+_MODEL_ID = "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
+# The stock processor size is 384x384, so this changes the feature count.
+_SCOPED_SIZE = {"height": 768, "width": 768}
+
+
+def _probe_num_image_tokens(mm_processor_kwargs, request_kwargs=None) -> list[int]:
+    """The per-image token counts vLLM predicts for a single image."""
+    from vllm.config import ModelConfig
+    from vllm.model_executor.models.transformers.multimodal import (
+        MultiModalDummyInputsBuilder,
+        MultiModalProcessingInfo,
+    )
+    from vllm.multimodal.processing import InputProcessingContext
+    from vllm.tokenizers.registry import cached_tokenizer_from_config
+
+    model_config = ModelConfig(
+        model=_MODEL_ID,
+        model_impl="transformers",
+        mm_processor_kwargs=mm_processor_kwargs,
+    )
+    info = MultiModalProcessingInfo(
+        InputProcessingContext(model_config, cached_tokenizer_from_config(model_config))
+    )
+    mm_processor = LegacyMultiModalProcessor(info, MultiModalDummyInputsBuilder(info))
+    image = ImageAsset("cherry_blossom").pil_image
+    mm_items = mm_processor.info.parse_mm_data({"image": image})
+    tokens = mm_processor._get_num_multimodal_tokens(mm_items, request_kwargs or {})
+    return list(tokens["num_image_tokens"])
+
+
+def test_scoped_images_kwargs_reach_the_token_count():
+    """A nested ``images_kwargs`` override must reach vLLM's own token count.
+
+    ``_get_num_multimodal_tokens`` is how vLLM predicts how many placeholder
+    tokens an image expands to. The HF processor honors a nested
+    ``images_kwargs`` in its ``__call__``, so a vLLM-side read that only looks
+    at the flat namespace makes the two disagree.
+    """
+    stock = _probe_num_image_tokens(None)
+    flat = _probe_num_image_tokens({"size": _SCOPED_SIZE})
+    # Precondition: this override really does move the count, so the
+    # assertion below cannot pass by coincidence.
+    assert flat != stock
+
+    scoped = _probe_num_image_tokens({"images_kwargs": {"size": _SCOPED_SIZE}})
+    assert scoped == flat
+
+
+def test_request_mm_processor_kwargs_reach_the_token_count():
+    """Per-request ``mm_processor_kwargs`` must reach vLLM's own token count too.
+
+    The request overrides build the HF processor that produces the features, so
+    a token count that only merges the model-config overrides predicts a
+    different number of placeholder tokens than the processor actually emits.
+    """
+    stock = _probe_num_image_tokens(None)
+    flat = _probe_num_image_tokens({"size": _SCOPED_SIZE})
+    assert flat != stock
+
+    for request_kwargs in (
+        {"size": _SCOPED_SIZE},
+        {"images_kwargs": {"size": _SCOPED_SIZE}},
+    ):
+        assert _probe_num_image_tokens(None, request_kwargs) == flat

tests/models/multimodal/processing/transformers_backend.py

@@ -27,10 +27,14 @@
 ]
 
 
-def create_processor(model_id: str, processor_cls):
+def create_processor(model_id: str, processor_cls, mm_processor_kwargs=None):
     """Build a processor directly, because the registry only ever builds the one the
     installed transformers version selects, leaving the other path untested."""
-    model_config = ModelConfig(model=model_id, model_impl="transformers")
+    model_config = ModelConfig(
+        model=model_id,
+        model_impl="transformers",
+        mm_processor_kwargs=mm_processor_kwargs,
+    )
     ctx = InputProcessingContext(
         model_config, cached_tokenizer_from_config(model_config)
     )

vllm/model_executor/models/cohere2_vision.py

@@ -184,7 +184,7 @@ def get_num_patches(
         return image_processor.get_number_of_image_patches(
             image_height,
             image_width,
-            self.ctx.get_merged_mm_kwargs(mm_kwargs),
+            self.ctx.get_merged_mm_kwargs(mm_kwargs, modality="image"),
         )
 
 

vllm/model_executor/models/cohere_compass.py

@@ -1089,6 +1089,7 @@ def get_num_image_tokens(
             num_frames=1,
             image_processor=image_processor,
             mm_kwargs=mm_kwargs,
+            modality="image",
         )
         return num_image_tokens
 
@@ -1116,6 +1117,9 @@ def get_image_size_with_most_features(
         if max_pixels is None:
             image_processor = self.get_image_processor()
 
+            # Unscoped on purpose: this bound also sizes the dummy data used
+            # for profiling, so a modality-scoped override must not move it.
+            # get_num_image_tokens re-resizes it with the image cap.
             mm_kwargs = self.ctx.get_merged_mm_kwargs({})
             size = image_processor.size
             if override_size := mm_kwargs.get("size"):
@@ -1184,14 +1188,15 @@ def _get_vision_info(
         do_resize: bool = True,
         image_processor: CohereCompassImageProcessor,
         mm_kwargs: Mapping[str, object],
+        modality: str | None = None,
     ) -> tuple[ImageSize, int]:
         hf_config = self.get_hf_config()
         vision_config = hf_config.vision_config
         patch_size = vision_config.patch_size
         merge_size = vision_config.spatial_merge_size
         temporal_patch_size = vision_config.temporal_patch_size
 
-        mm_kwargs = self.ctx.get_merged_mm_kwargs(mm_kwargs)
+        mm_kwargs = self.ctx.get_merged_mm_kwargs(mm_kwargs, modality=modality)
         size = image_processor.size
         if override_size := mm_kwargs.get("size"):
             size = size | override_size

vllm/model_executor/models/ernie45_vl.py

@@ -913,6 +913,7 @@ def _get_vision_info(
         do_resize: bool = True,
         image_processor: BaseImageProcessor,
         mm_kwargs: Mapping[str, object],
+        modality: str | None = None,
     ) -> tuple[ImageSize, int]:
         hf_config = self.get_hf_config()
         vision_config = hf_config.vision_config
@@ -930,7 +931,7 @@ def _get_vision_info(
             min_pixels_key = "shortest_edge"
             max_pixels_key = "longest_edge"
 
-        mm_kwargs = self.ctx.get_merged_mm_kwargs(mm_kwargs)
+        mm_kwargs = self.ctx.get_merged_mm_kwargs(mm_kwargs, modality=modality)
         size = image_processor.size
         if override_size := mm_kwargs.get("size"):
             size = size | override_size
@@ -973,6 +974,7 @@ def get_num_image_tokens(
             image_height=image_height,
             image_processor=image_processor,
             mm_kwargs=mm_kwargs,
+            modality="image",
         )
         return num_image_tokens
 
@@ -991,12 +993,17 @@ def get_num_video_tokens(
             num_frames=num_frames,
             image_processor=image_processor,
             mm_kwargs=mm_kwargs,
+            modality="video",
         )
         return num_video_tokens
 
     def get_image_size_with_most_features(self) -> ImageSize:
         image_processor = self.get_image_processor()
 
+        # Unscoped on purpose: this bound is shared by the image budget,
+        # the video budget and the dummy data, so a modality-scoped override
+        # must not move it. get_num_{image,video}_tokens re-resize it with
+        # the cap for their own modality.
         max_image_size, _ = self._get_vision_info(
             image_width=9999999,
             image_height=9999999,

vllm/model_executor/models/glm4_1v.py

@@ -1084,15 +1084,15 @@ def _get_vision_info(
 
         return preprocessed_size, num_vision_tokens
 
-    def _get_image_max_pixels(self) -> int:
+    def _get_image_max_pixels(self, modality: str | None = "image") -> int:
         """Read max_pixels from the HF image processor config.
 
         Despite the name, ``longest_edge`` is a pixel **area** (total pixel
         count), not an edge length.  The HF processor passes it directly to
         ``smart_resize`` as the ``max_pixels`` argument, which constrains
         ``t_bar * h_bar * w_bar <= max_pixels``.
         """
-        mm_kwargs = self.ctx.get_merged_mm_kwargs({})
+        mm_kwargs = self.ctx.get_merged_mm_kwargs({}, modality=modality)
         if (override_max_pixels := mm_kwargs.get("max_pixels")) is not None:
             return int(override_max_pixels)
 
@@ -1109,7 +1109,7 @@ def _get_image_max_pixels(self) -> int:
         return self._get_longest_edge(size, "GLM4V image processor size")
 
     def _get_video_max_pixels(self) -> int:
-        mm_kwargs = self.ctx.get_merged_mm_kwargs({})
+        mm_kwargs = self.ctx.get_merged_mm_kwargs({}, modality="video")
         if (override_max_pixels := mm_kwargs.get("max_pixels")) is not None:
             return int(override_max_pixels)
 
@@ -1130,11 +1130,15 @@ def get_image_size_with_most_features(self) -> ImageSize:
         # underestimating the spatial budget for a single image and
         # causing encoder cache overflow for large images
         # (see https://github.com/vllm-project/vllm/issues/34040).
+        # The pixel bound is deliberately unscoped: it is shared by the image
+        # budget, the video budget and the dummy data, so a modality-scoped
+        # override must not move it. get_num_image_tokens and
+        # _get_max_video_frames re-resize it with their own cap.
         max_image_size, _ = self._get_vision_info(
             image_width=9999999,
             image_height=9999999,
             num_frames=1,
-            max_image_pixels=self._get_image_max_pixels(),
+            max_image_pixels=self._get_image_max_pixels(modality=None),
         )
         return max_image_size
 

vllm/model_executor/models/idefics3.py

@@ -120,7 +120,7 @@ def _get_image_feature_grid_size(
         return image_processor.get_number_of_image_patches(
             image_height,
             image_width,
-            self.ctx.get_merged_mm_kwargs(mm_kwargs),
+            self.ctx.get_merged_mm_kwargs(mm_kwargs, modality="image"),
         )
 
     def get_num_patches(

vllm/model_executor/models/interns1.py

@@ -211,7 +211,7 @@ def get_num_image_tokens(
         num_image_patches = image_processor.get_number_of_image_patches(
             image_height,
             image_width,
-            self.ctx.get_merged_mm_kwargs(mm_kwargs),
+            self.ctx.get_merged_mm_kwargs(mm_kwargs, modality="image"),
         )
 
         return processor.image_seq_length * num_image_patches

vllm/model_executor/models/keye.py

@@ -965,14 +965,15 @@ def _get_vision_info(
         do_resize: bool = True,
         image_processor: BaseImageProcessor,
         mm_kwargs: Mapping[str, object],
+        modality: str | None = None,
     ) -> tuple[ImageSize, int]:
         hf_config = self.get_hf_config()
         vision_config = hf_config.vision_config
         patch_size = vision_config.patch_size
         merge_size = vision_config.spatial_merge_size
         temporal_patch_size = 1
 
-        mm_kwargs = self.ctx.get_merged_mm_kwargs(mm_kwargs)
+        mm_kwargs = self.ctx.get_merged_mm_kwargs(mm_kwargs, modality=modality)
         size = image_processor.size
         if override_size := mm_kwargs.get("size"):
             size = size | override_size
@@ -1017,6 +1018,7 @@ def get_num_image_tokens(
             image_height=image_height,
             image_processor=image_processor,
             mm_kwargs=mm_kwargs,
+            modality="image",
         )
         return num_image_tokens
 
@@ -1035,12 +1037,17 @@ def get_num_video_tokens(
             num_frames=num_frames,
             image_processor=image_processor,
             mm_kwargs=mm_kwargs,
+            modality="video",
         )
         return num_video_tokens
 
     def get_image_size_with_most_features(self) -> ImageSize:
         image_processor = self.get_image_processor()
 
+        # Unscoped on purpose: this bound is shared by the image budget,
+        # the video budget and the dummy data, so a modality-scoped override
+        # must not move it. get_num_{image,video}_tokens re-resize it with
+        # the cap for their own modality.
         max_image_size, _ = self._get_vision_info(
             image_width=self.get_max_image_size(),
             image_height=self.get_max_image_size(),

vllm/model_executor/models/lfm2_vl.py

@@ -216,7 +216,7 @@ def _get_image_feature_grid_size(
     ) -> tuple[int, int, int]:
         image_processor: Lfm2VlImageProcessorFast = processor.image_processor
 
-        mm_kwargs = self.ctx.get_merged_mm_kwargs(mm_kwargs)
+        mm_kwargs = self.ctx.get_merged_mm_kwargs(mm_kwargs, modality="image")
         downsample_factor = mm_kwargs.get(
             "downsample_factor", image_processor.downsample_factor
         )
@@ -339,7 +339,7 @@ def get_num_image_tokens(
     ) -> tuple[int, int]:
         image_processor: Lfm2VlImageProcessorFast = processor.image_processor
 
-        mm_kwargs = self.ctx.get_merged_mm_kwargs(mm_kwargs)
+        mm_kwargs = self.ctx.get_merged_mm_kwargs(mm_kwargs, modality="image")
         downsample_factor = mm_kwargs.get(
             "downsample_factor", image_processor.downsample_factor
         )