diff --git a/tests/models/multimodal/processing/test_glm4_1v.py b/tests/models/multimodal/processing/test_glm4_1v.py index adf7219db1b7..6df291de022d 100644 --- a/tests/models/multimodal/processing/test_glm4_1v.py +++ b/tests/models/multimodal/processing/test_glm4_1v.py @@ -199,3 +199,92 @@ def test_video_loader_consistency( static_outputs["mm_kwargs"].get_data(), dynamic_outputs["mm_kwargs"].get_data(), ) + + +# Far above any stock GLM-4.1V pixel budget, so an override is unmistakable. +_SCOPED_MAX_PIXELS = 469762048 + + +def _probe_max_pixels( + model_id: str, mm_processor_kwargs: dict | None +) -> tuple[int, int]: + """Return the (video, image) pixel budgets vLLM computes.""" + ctx = build_model_context( + model_id, + mm_processor_kwargs=mm_processor_kwargs, + limit_mm_per_prompt={"image": 1, "video": 1}, + ) + info = MULTIMODAL_REGISTRY.create_processor(ctx.model_config).info + return info._get_video_max_pixels(), info._get_image_max_pixels() + + +@pytest.mark.skip_global_cleanup +@pytest.mark.parametrize("model_id", ["zai-org/GLM-4.1V-9B-Thinking"]) +def test_videos_kwargs_max_pixels_does_not_leak_into_image_budget(model_id: str): + """A scoped ``videos_kwargs`` override must reach only the video budget. + + The HF processor already honors the nested dict, so when vLLM's own + budget reads ignore it the two disagree about how many tokens a video + expands to. + """ + stock_video, stock_image = _probe_max_pixels(model_id, None) + assert stock_video != _SCOPED_MAX_PIXELS + assert stock_image != _SCOPED_MAX_PIXELS + + scoped_video, scoped_image = _probe_max_pixels( + model_id, {"videos_kwargs": {"max_pixels": _SCOPED_MAX_PIXELS}} + ) + assert scoped_video == _SCOPED_MAX_PIXELS + assert scoped_image == stock_image + + # A flat override keeps the previous shared-namespace behavior. + flat_video, flat_image = _probe_max_pixels( + model_id, {"max_pixels": _SCOPED_MAX_PIXELS} + ) + assert flat_video == _SCOPED_MAX_PIXELS + assert flat_image == _SCOPED_MAX_PIXELS + + +# Well below any stock GLM-4.1V image budget, so a leak into the shared +# upper bound is unmistakable. +_SMALL_MAX_PIXELS = 1_003_520 + + +def _probe_budgets(model_id: str, mm_processor_kwargs: dict | None) -> dict: + ctx = build_model_context( + model_id, + mm_processor_kwargs=mm_processor_kwargs, + limit_mm_per_prompt={"image": 1, "video": 1}, + ) + info = MULTIMODAL_REGISTRY.create_processor(ctx.model_config).info + return { + "image_max_pixels": info._get_image_max_pixels(), + "size_bound": tuple(info.get_image_size_with_most_features()), + "video_frames": info._get_max_video_frames(30_000), + } + + +@pytest.mark.skip_global_cleanup +@pytest.mark.parametrize("model_id", ["zai-org/GLM-4.1V-9B-Thinking"]) +def test_images_kwargs_max_pixels_does_not_leak_into_video_budget(model_id: str): + """An image-scoped override must not move the shared size upper bound. + + ``get_image_size_with_most_features`` feeds the video frame budget and the + dummy data as well as the image budget. Scoping that bound to ``image`` + would let an image-only override shrink the profiled video budget, which + no override of that modality should touch. + """ + stock = _probe_budgets(model_id, None) + scoped = _probe_budgets( + model_id, {"images_kwargs": {"max_pixels": _SMALL_MAX_PIXELS}} + ) + + # The override reaches the per-item image read it is meant for. + assert stock["image_max_pixels"] != _SMALL_MAX_PIXELS + assert scoped["image_max_pixels"] == _SMALL_MAX_PIXELS + + # It must not reach the bound that video sizing and dummy data share. + assert (scoped["size_bound"], scoped["video_frames"]) == ( + stock["size_bound"], + stock["video_frames"], + ) diff --git a/tests/models/multimodal/processing/test_transformers_image.py b/tests/models/multimodal/processing/test_transformers_image.py index f5de7d66f4ff..756792ed50dd 100644 --- a/tests/models/multimodal/processing/test_transformers_image.py +++ b/tests/models/multimodal/processing/test_transformers_image.py @@ -314,3 +314,69 @@ def test_nested_image_fields_split_per_image(processor_cls): for item in items: pixel_values = item["pixel_values"].data assert pixel_values.shape[1] == int(item["num_image_patches"].data) + + +_MODEL_ID = "llava-hf/llava-onevision-qwen2-0.5b-ov-hf" +# The stock processor size is 384x384, so this changes the feature count. +_SCOPED_SIZE = {"height": 768, "width": 768} + + +def _probe_num_image_tokens(mm_processor_kwargs, request_kwargs=None) -> list[int]: + """The per-image token counts vLLM predicts for a single image.""" + from vllm.config import ModelConfig + from vllm.model_executor.models.transformers.multimodal import ( + MultiModalDummyInputsBuilder, + MultiModalProcessingInfo, + ) + from vllm.multimodal.processing import InputProcessingContext + from vllm.tokenizers.registry import cached_tokenizer_from_config + + model_config = ModelConfig( + model=_MODEL_ID, + model_impl="transformers", + mm_processor_kwargs=mm_processor_kwargs, + ) + info = MultiModalProcessingInfo( + InputProcessingContext(model_config, cached_tokenizer_from_config(model_config)) + ) + mm_processor = LegacyMultiModalProcessor(info, MultiModalDummyInputsBuilder(info)) + image = ImageAsset("cherry_blossom").pil_image + mm_items = mm_processor.info.parse_mm_data({"image": image}) + tokens = mm_processor._get_num_multimodal_tokens(mm_items, request_kwargs or {}) + return list(tokens["num_image_tokens"]) + + +def test_scoped_images_kwargs_reach_the_token_count(): + """A nested ``images_kwargs`` override must reach vLLM's own token count. + + ``_get_num_multimodal_tokens`` is how vLLM predicts how many placeholder + tokens an image expands to. The HF processor honors a nested + ``images_kwargs`` in its ``__call__``, so a vLLM-side read that only looks + at the flat namespace makes the two disagree. + """ + stock = _probe_num_image_tokens(None) + flat = _probe_num_image_tokens({"size": _SCOPED_SIZE}) + # Precondition: this override really does move the count, so the + # assertion below cannot pass by coincidence. + assert flat != stock + + scoped = _probe_num_image_tokens({"images_kwargs": {"size": _SCOPED_SIZE}}) + assert scoped == flat + + +def test_request_mm_processor_kwargs_reach_the_token_count(): + """Per-request ``mm_processor_kwargs`` must reach vLLM's own token count too. + + The request overrides build the HF processor that produces the features, so + a token count that only merges the model-config overrides predicts a + different number of placeholder tokens than the processor actually emits. + """ + stock = _probe_num_image_tokens(None) + flat = _probe_num_image_tokens({"size": _SCOPED_SIZE}) + assert flat != stock + + for request_kwargs in ( + {"size": _SCOPED_SIZE}, + {"images_kwargs": {"size": _SCOPED_SIZE}}, + ): + assert _probe_num_image_tokens(None, request_kwargs) == flat diff --git a/tests/models/multimodal/processing/transformers_backend.py b/tests/models/multimodal/processing/transformers_backend.py index dc49036e8352..8c6188c6f63a 100644 --- a/tests/models/multimodal/processing/transformers_backend.py +++ b/tests/models/multimodal/processing/transformers_backend.py @@ -27,10 +27,14 @@ ] -def create_processor(model_id: str, processor_cls): +def create_processor(model_id: str, processor_cls, mm_processor_kwargs=None): """Build a processor directly, because the registry only ever builds the one the installed transformers version selects, leaving the other path untested.""" - model_config = ModelConfig(model=model_id, model_impl="transformers") + model_config = ModelConfig( + model=model_id, + model_impl="transformers", + mm_processor_kwargs=mm_processor_kwargs, + ) ctx = InputProcessingContext( model_config, cached_tokenizer_from_config(model_config) ) diff --git a/vllm/model_executor/models/cohere2_vision.py b/vllm/model_executor/models/cohere2_vision.py index a4748932d67e..5f7d02c60ae1 100644 --- a/vllm/model_executor/models/cohere2_vision.py +++ b/vllm/model_executor/models/cohere2_vision.py @@ -183,7 +183,7 @@ def get_num_patches( return image_processor.get_number_of_image_patches( image_height, image_width, - self.ctx.get_merged_mm_kwargs(mm_kwargs), + self.ctx.get_merged_mm_kwargs(mm_kwargs, modality="image"), ) diff --git a/vllm/model_executor/models/cohere_compass.py b/vllm/model_executor/models/cohere_compass.py index 1d0dd8c35b05..283ca5041f3a 100644 --- a/vllm/model_executor/models/cohere_compass.py +++ b/vllm/model_executor/models/cohere_compass.py @@ -1088,6 +1088,7 @@ def get_num_image_tokens( num_frames=1, image_processor=image_processor, mm_kwargs=mm_kwargs, + modality="image", ) return num_image_tokens @@ -1115,6 +1116,9 @@ def get_image_size_with_most_features( if max_pixels is None: image_processor = self.get_image_processor() + # Unscoped on purpose: this bound also sizes the dummy data used + # for profiling, so a modality-scoped override must not move it. + # get_num_image_tokens re-resizes it with the image cap. mm_kwargs = self.ctx.get_merged_mm_kwargs({}) size = image_processor.size if override_size := mm_kwargs.get("size"): @@ -1183,6 +1187,7 @@ def _get_vision_info( do_resize: bool = True, image_processor: CohereCompassImageProcessor, mm_kwargs: Mapping[str, object], + modality: str | None = None, ) -> tuple[ImageSize, int]: hf_config = self.get_hf_config() vision_config = hf_config.vision_config @@ -1190,7 +1195,7 @@ def _get_vision_info( merge_size = vision_config.spatial_merge_size temporal_patch_size = vision_config.temporal_patch_size - mm_kwargs = self.ctx.get_merged_mm_kwargs(mm_kwargs) + mm_kwargs = self.ctx.get_merged_mm_kwargs(mm_kwargs, modality=modality) size = image_processor.size if override_size := mm_kwargs.get("size"): size = size | override_size diff --git a/vllm/model_executor/models/ernie45_vl.py b/vllm/model_executor/models/ernie45_vl.py index e1b847c2940f..5c95029a76f8 100644 --- a/vllm/model_executor/models/ernie45_vl.py +++ b/vllm/model_executor/models/ernie45_vl.py @@ -913,6 +913,7 @@ def _get_vision_info( do_resize: bool = True, image_processor: BaseImageProcessor, mm_kwargs: Mapping[str, object], + modality: str | None = None, ) -> tuple[ImageSize, int]: hf_config = self.get_hf_config() vision_config = hf_config.vision_config @@ -930,7 +931,7 @@ def _get_vision_info( min_pixels_key = "shortest_edge" max_pixels_key = "longest_edge" - mm_kwargs = self.ctx.get_merged_mm_kwargs(mm_kwargs) + mm_kwargs = self.ctx.get_merged_mm_kwargs(mm_kwargs, modality=modality) size = image_processor.size if override_size := mm_kwargs.get("size"): size = size | override_size @@ -973,6 +974,7 @@ def get_num_image_tokens( image_height=image_height, image_processor=image_processor, mm_kwargs=mm_kwargs, + modality="image", ) return num_image_tokens @@ -991,12 +993,17 @@ def get_num_video_tokens( num_frames=num_frames, image_processor=image_processor, mm_kwargs=mm_kwargs, + modality="video", ) return num_video_tokens def get_image_size_with_most_features(self) -> ImageSize: image_processor = self.get_image_processor() + # Unscoped on purpose: this bound is shared by the image budget, + # the video budget and the dummy data, so a modality-scoped override + # must not move it. get_num_{image,video}_tokens re-resize it with + # the cap for their own modality. max_image_size, _ = self._get_vision_info( image_width=9999999, image_height=9999999, diff --git a/vllm/model_executor/models/glm4_1v.py b/vllm/model_executor/models/glm4_1v.py index 8f3db8a3d110..690cc38f5e8f 100644 --- a/vllm/model_executor/models/glm4_1v.py +++ b/vllm/model_executor/models/glm4_1v.py @@ -1080,7 +1080,7 @@ def _get_vision_info( return preprocessed_size, num_vision_tokens - def _get_image_max_pixels(self) -> int: + def _get_image_max_pixels(self, modality: str | None = "image") -> int: """Read max_pixels from the HF image processor config. Despite the name, ``longest_edge`` is a pixel **area** (total pixel @@ -1088,7 +1088,7 @@ def _get_image_max_pixels(self) -> int: ``smart_resize`` as the ``max_pixels`` argument, which constrains ``t_bar * h_bar * w_bar <= max_pixels``. """ - mm_kwargs = self.ctx.get_merged_mm_kwargs({}) + mm_kwargs = self.ctx.get_merged_mm_kwargs({}, modality=modality) if (override_max_pixels := mm_kwargs.get("max_pixels")) is not None: return int(override_max_pixels) @@ -1105,7 +1105,7 @@ def _get_image_max_pixels(self) -> int: return self._get_longest_edge(size, "GLM4V image processor size") def _get_video_max_pixels(self) -> int: - mm_kwargs = self.ctx.get_merged_mm_kwargs({}) + mm_kwargs = self.ctx.get_merged_mm_kwargs({}, modality="video") if (override_max_pixels := mm_kwargs.get("max_pixels")) is not None: return int(override_max_pixels) @@ -1126,11 +1126,15 @@ def get_image_size_with_most_features(self) -> ImageSize: # underestimating the spatial budget for a single image and # causing encoder cache overflow for large images # (see https://github.com/vllm-project/vllm/issues/34040). + # The pixel bound is deliberately unscoped: it is shared by the image + # budget, the video budget and the dummy data, so a modality-scoped + # override must not move it. get_num_image_tokens and + # _get_max_video_frames re-resize it with their own cap. max_image_size, _ = self._get_vision_info( image_width=9999999, image_height=9999999, num_frames=1, - max_image_pixels=self._get_image_max_pixels(), + max_image_pixels=self._get_image_max_pixels(modality=None), ) return max_image_size diff --git a/vllm/model_executor/models/idefics3.py b/vllm/model_executor/models/idefics3.py index dd03e17d1ec0..dbd04a744bb3 100644 --- a/vllm/model_executor/models/idefics3.py +++ b/vllm/model_executor/models/idefics3.py @@ -119,7 +119,7 @@ def _get_image_feature_grid_size( return image_processor.get_number_of_image_patches( image_height, image_width, - self.ctx.get_merged_mm_kwargs(mm_kwargs), + self.ctx.get_merged_mm_kwargs(mm_kwargs, modality="image"), ) def get_num_patches( diff --git a/vllm/model_executor/models/interns1.py b/vllm/model_executor/models/interns1.py index 4a4264660d54..20a59f5c874c 100644 --- a/vllm/model_executor/models/interns1.py +++ b/vllm/model_executor/models/interns1.py @@ -207,7 +207,7 @@ def get_num_image_tokens( num_image_patches = image_processor.get_number_of_image_patches( image_height, image_width, - self.ctx.get_merged_mm_kwargs(mm_kwargs), + self.ctx.get_merged_mm_kwargs(mm_kwargs, modality="image"), ) return processor.image_seq_length * num_image_patches diff --git a/vllm/model_executor/models/keye.py b/vllm/model_executor/models/keye.py index 1322920900bb..5b44283bf5f4 100644 --- a/vllm/model_executor/models/keye.py +++ b/vllm/model_executor/models/keye.py @@ -961,6 +961,7 @@ def _get_vision_info( do_resize: bool = True, image_processor: BaseImageProcessor, mm_kwargs: Mapping[str, object], + modality: str | None = None, ) -> tuple[ImageSize, int]: hf_config = self.get_hf_config() vision_config = hf_config.vision_config @@ -968,7 +969,7 @@ def _get_vision_info( merge_size = vision_config.spatial_merge_size temporal_patch_size = 1 - mm_kwargs = self.ctx.get_merged_mm_kwargs(mm_kwargs) + mm_kwargs = self.ctx.get_merged_mm_kwargs(mm_kwargs, modality=modality) size = image_processor.size if override_size := mm_kwargs.get("size"): size = size | override_size @@ -1013,6 +1014,7 @@ def get_num_image_tokens( image_height=image_height, image_processor=image_processor, mm_kwargs=mm_kwargs, + modality="image", ) return num_image_tokens @@ -1031,12 +1033,17 @@ def get_num_video_tokens( num_frames=num_frames, image_processor=image_processor, mm_kwargs=mm_kwargs, + modality="video", ) return num_video_tokens def get_image_size_with_most_features(self) -> ImageSize: image_processor = self.get_image_processor() + # Unscoped on purpose: this bound is shared by the image budget, + # the video budget and the dummy data, so a modality-scoped override + # must not move it. get_num_{image,video}_tokens re-resize it with + # the cap for their own modality. max_image_size, _ = self._get_vision_info( image_width=self.get_max_image_size(), image_height=self.get_max_image_size(), diff --git a/vllm/model_executor/models/lfm2_vl.py b/vllm/model_executor/models/lfm2_vl.py index 1cd51b93a2ed..2be1d2fb224a 100644 --- a/vllm/model_executor/models/lfm2_vl.py +++ b/vllm/model_executor/models/lfm2_vl.py @@ -215,7 +215,7 @@ def _get_image_feature_grid_size( ) -> tuple[int, int, int]: image_processor: Lfm2VlImageProcessorFast = processor.image_processor - mm_kwargs = self.ctx.get_merged_mm_kwargs(mm_kwargs) + mm_kwargs = self.ctx.get_merged_mm_kwargs(mm_kwargs, modality="image") downsample_factor = mm_kwargs.get( "downsample_factor", image_processor.downsample_factor ) @@ -338,7 +338,7 @@ def get_num_image_tokens( ) -> tuple[int, int]: image_processor: Lfm2VlImageProcessorFast = processor.image_processor - mm_kwargs = self.ctx.get_merged_mm_kwargs(mm_kwargs) + mm_kwargs = self.ctx.get_merged_mm_kwargs(mm_kwargs, modality="image") downsample_factor = mm_kwargs.get( "downsample_factor", image_processor.downsample_factor ) diff --git a/vllm/model_executor/models/mimo_v2_omni.py b/vllm/model_executor/models/mimo_v2_omni.py index d74edd6d4ad0..df4b91100756 100644 --- a/vllm/model_executor/models/mimo_v2_omni.py +++ b/vllm/model_executor/models/mimo_v2_omni.py @@ -715,6 +715,7 @@ def _get_vision_info( do_resize: bool = True, image_processor, mm_kwargs: Mapping[str, object], + modality: str | None = None, ) -> tuple[ImageSize, int]: hf_config = self.get_hf_config() vision_config = hf_config.vision_config @@ -723,7 +724,7 @@ def _get_vision_info( temporal_patch_size = vision_config.temporal_patch_size tokens_per_second = vision_config.tokens_per_second - mm_kwargs = self.ctx.get_merged_mm_kwargs(mm_kwargs) + mm_kwargs = self.ctx.get_merged_mm_kwargs(mm_kwargs, modality=modality) size = image_processor.size if override_size := mm_kwargs.get("size"): size = size | override_size @@ -773,6 +774,7 @@ def get_num_image_tokens( num_frames=1, image_processor=image_processor, mm_kwargs=mm_kwargs, + modality="image", ) return num_image_tokens @@ -791,6 +793,7 @@ def get_num_video_tokens( num_frames=num_frames, image_processor=image_processor, mm_kwargs=mm_kwargs, + modality="video", ) return num_video_tokens @@ -804,6 +807,10 @@ def get_image_size_with_most_features( if max_pixels is None: image_processor = self.get_image_processor() + # Unscoped on purpose: this bound is shared by the image budget, + # the video budget and the dummy data, so a modality-scoped + # override must not move it. get_num_{image,video}_tokens + # re-resize it with the cap for their own modality. mm_kwargs = self.ctx.get_merged_mm_kwargs({}) size = image_processor.size if override_size := mm_kwargs.get("size"): diff --git a/vllm/model_executor/models/mistral3.py b/vllm/model_executor/models/mistral3.py index f9d7dafe70b3..60fcccaea03e 100644 --- a/vllm/model_executor/models/mistral3.py +++ b/vllm/model_executor/models/mistral3.py @@ -217,7 +217,9 @@ def get_vision_encoder_info( ) -> PixtralHFEncoderInfo: processor = self.get_hf_processor() size = processor.image_processor.size - merged_kwargs = self.ctx.get_merged_mm_kwargs(mm_processor_kwargs or {}) + merged_kwargs = self.ctx.get_merged_mm_kwargs( + mm_processor_kwargs or {}, modality="image" + ) if override_size := merged_kwargs.get("size"): size = size | override_size diff --git a/vllm/model_executor/models/paddleocr_vl.py b/vllm/model_executor/models/paddleocr_vl.py index 976d82032f20..3a2e2011d05f 100644 --- a/vllm/model_executor/models/paddleocr_vl.py +++ b/vllm/model_executor/models/paddleocr_vl.py @@ -160,7 +160,7 @@ def get_num_image_tokens( min_pixels_key = "shortest_edge" max_pixels_key = "longest_edge" - mm_kwargs = self.ctx.get_merged_mm_kwargs(mm_kwargs) + mm_kwargs = self.ctx.get_merged_mm_kwargs(mm_kwargs, modality="image") size = image_processor.size if override_size := mm_kwargs.get("size"): size = size | override_size diff --git a/vllm/model_executor/models/qwen2_5_omni_thinker.py b/vllm/model_executor/models/qwen2_5_omni_thinker.py index cecc8345e5ee..b55230b99a77 100644 --- a/vllm/model_executor/models/qwen2_5_omni_thinker.py +++ b/vllm/model_executor/models/qwen2_5_omni_thinker.py @@ -539,6 +539,10 @@ def _get_hf_mm_inputs( "Video doesn't have audio track with `audio_in_video=True`" ) + # No `modality` here: the synthesized `size` below is flat, and images + # and videos go through a single combined HF call, so overlaying a + # nested `videos_kwargs` would leak the video size onto the images. + # HF's own kwarg merging already scopes the nested dict correctly. merged = self.info.ctx.get_merged_mm_kwargs(normalized_kwargs) if hf_data.get("videos") and ( merged.keys() & {"size", "min_pixels", "max_pixels"} diff --git a/vllm/model_executor/models/transformers/multimodal.py b/vllm/model_executor/models/transformers/multimodal.py index 855b267c02f5..cc6d89997fa5 100644 --- a/vllm/model_executor/models/transformers/multimodal.py +++ b/vllm/model_executor/models/transformers/multimodal.py @@ -17,7 +17,7 @@ """Transformers modeling backend mixin for multi-modal models.""" from collections import defaultdict -from collections.abc import Mapping, Sequence +from collections.abc import Callable, Mapping, Sequence from contextlib import ExitStack, contextmanager from typing import TYPE_CHECKING, Any @@ -59,6 +59,7 @@ cached_encode, ) from vllm.sequence import IntermediateTensors +from vllm.utils.func_utils import get_allowed_kwarg_only_overrides from vllm.utils.gpu_sync_debug import gpu_sync_allowed from vllm.utils.torch_utils import async_tensor_h2d @@ -407,6 +408,26 @@ def _unpad_images(self, hf_inputs: "BatchFeature") -> None: ] +def _num_multimodal_tokens_kwargs( + count_tokens: Callable[..., object], + merged_mm_kwargs: Mapping[str, object], +) -> dict[str, Any]: + """The subset of `merged_mm_kwargs` that `count_tokens` accepts. + + `_get_num_multimodal_tokens` is a sizing helper rather than `__call__`, so it + does not take every processor override. Filtering mirrors what + `call_hf_processor` does before splatting the same kwargs into the processor, + and keeps an override meant for `__call__` from turning the count into a + `TypeError`. + """ + return get_allowed_kwarg_only_overrides( + count_tokens, + merged_mm_kwargs, + requires_kw_only=False, + allow_var_kwargs=True, + ) + + class LegacyMultiModalProcessor(_MultiModalProcessorBase): """Locates placeholders by searching the prompt the HF processor has already expanded for the tokens of each modality. @@ -502,9 +523,15 @@ def _get_num_multimodal_tokens( (size.height, size.width) for size in map(images.get_image_size, range(len(images))) ] - return processor._get_num_multimodal_tokens( + count_tokens = processor._get_num_multimodal_tokens + return count_tokens( image_sizes=image_sizes, - **self.info.ctx.get_merged_mm_kwargs({}), + **_num_multimodal_tokens_kwargs( + count_tokens, + self.info.ctx.get_merged_mm_kwargs( + hf_processor_mm_kwargs, modality="image" + ), + ), ) def _apply_vision( @@ -583,7 +610,11 @@ def apply( # transforms outputs to `MultiModalKwargs` which is not going to # work for Transformers. The vision path has logic tied to # `mm_tokens_per_modality` in _apply_vision() - hf_processor_mm_kwargs = { + # These options belong to the HF processor call only. They stay out + # of `hf_processor_mm_kwargs` because that dict also reaches the + # sizing helpers, which read every kwarg as an image processor + # override. + call_mm_kwargs = { # vLLM needs the untruncated sequence to keep placeholder # tokens aligned. Note that the text inputs are just dummy # text, not the original prompt. The original prompt is @@ -597,7 +628,7 @@ def apply( } processor_data, _, passthrough_data = self._get_hf_mm_inputs( - mm_items, hf_processor_mm_kwargs + mm_items, call_mm_kwargs ) # The real prompt is fed to the processor below and the placeholder @@ -613,9 +644,9 @@ def apply( try: processed_data = self.info.ctx.call_hf_processor( - self.info.get_hf_processor(**hf_processor_mm_kwargs), + self.info.get_hf_processor(**call_mm_kwargs), dict(text=prompt_text, **processor_data), - hf_processor_mm_kwargs, + call_mm_kwargs, ) except ValueError: if has_mm_data: @@ -627,7 +658,7 @@ def apply( dict(input_ids=[prompt_ids]), tensor_type="pt" ) self._unpad_images(processed_data) - self._unpad_audios(processed_data, processor_data, hf_processor_mm_kwargs) + self._unpad_audios(processed_data, processor_data, call_mm_kwargs) processed_data.update(passthrough_data) input_ids = processed_data.pop("input_ids") @@ -744,6 +775,7 @@ def _get_num_image_patches( hf_inputs: "BatchFeature", mm_data: Mapping[str, object], num_images: int, + hf_processor_mm_kwargs: Mapping[str, object], ) -> torch.Tensor: """How many rows of the image fields belong to each image. @@ -752,7 +784,9 @@ def _get_num_image_patches( """ if (grid := hf_inputs.get("image_grid_thw")) is not None: num_patches = grid.prod(-1) - elif (counts := self._get_num_patches_per_image(mm_data)) is not None: + elif ( + counts := self._get_num_patches_per_image(mm_data, hf_processor_mm_kwargs) + ) is not None: num_patches = torch.tensor(counts) else: num_patches = torch.ones(num_images, dtype=torch.long) @@ -773,16 +807,26 @@ def _get_num_image_patches( return num_patches def _get_num_patches_per_image( - self, mm_data: Mapping[str, object] + self, + mm_data: Mapping[str, object], + hf_processor_mm_kwargs: Mapping[str, object], ) -> list[int] | None: """Ask the HF processor how many rows of image data each image produces.""" images = mm_data.get("images") if not images: return None try: + processor = self.info.get_hf_processor(**hf_processor_mm_kwargs) + count_tokens = processor._get_num_multimodal_tokens sizes = [(image.height, image.width) for image in images] - mm_tokens = self.info.get_hf_processor()._get_num_multimodal_tokens( - image_sizes=sizes, **self.info.ctx.get_merged_mm_kwargs({}) + mm_tokens = count_tokens( + image_sizes=sizes, + **_num_multimodal_tokens_kwargs( + count_tokens, + self.info.ctx.get_merged_mm_kwargs( + hf_processor_mm_kwargs, modality="image" + ), + ), ) return list(mm_tokens["num_image_patches"]) except (AttributeError, KeyError, TypeError): @@ -796,6 +840,10 @@ def _apply_hf_processor_main( mm_items: MultiModalDataItems, hf_kwargs: Mapping[str, object], ) -> "BatchFeature": + # `_get_hf_mm_inputs` folds call-only options such as `truncation` into + # the kwargs it returns, which a sizing helper would read as an image + # processor override, so keep the request kwargs for those. + request_mm_kwargs = hf_kwargs hf_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs( mm_items, hf_kwargs ) @@ -868,7 +916,7 @@ def _apply_hf_processor_main( ) if modality == "image": hf_inputs["num_image_patches"] = self._get_num_image_patches( - hf_inputs, hf_data, len(seqs) + hf_inputs, hf_data, len(seqs), request_mm_kwargs ) elif modality == "audio": counts = [] diff --git a/vllm/models/glm5next/common/multimodal.py b/vllm/models/glm5next/common/multimodal.py index d2b21345d052..50aa10aa6566 100644 --- a/vllm/models/glm5next/common/multimodal.py +++ b/vllm/models/glm5next/common/multimodal.py @@ -644,14 +644,14 @@ def _processor_pixel_budget(self, proc) -> tuple[int, int]: proc.temporal_patch_size, ) - def _get_image_max_pixels(self) -> int: - mm_kwargs = self.ctx.get_merged_mm_kwargs({}) + def _get_image_max_pixels(self, modality: str | None = "image") -> int: + mm_kwargs = self.ctx.get_merged_mm_kwargs({}, modality=modality) if (override := mm_kwargs.get("max_pixels")) is not None: return int(override) return self._processor_pixel_budget(self.get_hf_processor().image_processor)[1] def _get_video_max_pixels(self) -> int: - mm_kwargs = self.ctx.get_merged_mm_kwargs({}) + mm_kwargs = self.ctx.get_merged_mm_kwargs({}, modality="video") if (override := mm_kwargs.get("max_pixels")) is not None: return int(override) return self._processor_pixel_budget(self.get_hf_processor().video_processor)[1]