From 93b86789fd98893f9717026759bd3aef9e61a441 Mon Sep 17 00:00:00 2001
From: Reagan Lee <“reaganjlee@gmail.com”>
Date: Sat, 17 Jan 2026 23:17:50 +0000
Subject: [PATCH 01/17] change
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Reagan Lee <“reaganjlee@gmail.com”>
Signed-off-by: Reagan Lee <reaganjlee@gmail.com>
Signed-off-by: Reagan Lee <“reaganjlee@gmail.com”>
---
 vllm/config/multimodal.py               |  4 ++
 vllm/entrypoints/chat_utils.py          |  7 ++-
 vllm/multimodal/budget.py               | 21 +++++++
 vllm/multimodal/processing/processor.py | 73 ++++++++++++++++++++++++
 vllm/multimodal/registry.py             |  5 ++
 vllm/v1/worker/gpu_model_runner.py      | 74 +++++++++++++++----------
 6 files changed, 153 insertions(+), 31 deletions(-)

diff --git a/vllm/config/multimodal.py b/vllm/config/multimodal.py
index 48eea6f4ef5e..de68b185488c 100644
--- a/vllm/config/multimodal.py
+++ b/vllm/config/multimodal.py
@@ -76,6 +76,10 @@ class MultiModalConfig:
     for the OpenAI-compatible server, this refers to chat messages with content
     `"type": "*_embeds"`.
 
+    When enabled, precomputed embeddings bypass the `--limit-mm-per-prompt` count
+    restriction. This allows unlimited embeddings even when limit is 0, enabling
+    memory savings by not loading encoder modules while still processing embeddings.
+
     WARNING: The vLLM engine may crash if incorrect shape of embeddings is passed.
     Only enable this flag for trusted users!"""
     media_io_kwargs: dict[str, dict[str, Any]] = Field(default_factory=dict)
diff --git a/vllm/entrypoints/chat_utils.py b/vllm/entrypoints/chat_utils.py
index 0077a897d53e..94e88c3582e5 100644
--- a/vllm/entrypoints/chat_utils.py
+++ b/vllm/entrypoints/chat_utils.py
@@ -528,7 +528,12 @@ def add(self, modality: ModalityStr, item: _T) -> str | None:
         else:
             num_items = len(self._items_by_modality[original_modality]) + 1
 
-        self.mm_processor.info.validate_num_items(input_modality, num_items)
+        mm_config = self.model_config.multimodal_config
+        if mm_config.enable_mm_embeds and mm_config.get_limit_per_prompt(input_modality) == 0:
+            # Additional check with info logging for this specific case in BaseMultiModalProcessor
+            pass
+        else:
+            self.mm_processor.validate_num_items(input_modality, num_items)
 
         # Track original modality for vision_chunk items
         if use_vision_chunk:
diff --git a/vllm/multimodal/budget.py b/vllm/multimodal/budget.py
index 1380ec1ba7c3..fa003fdd7f11 100644
--- a/vllm/multimodal/budget.py
+++ b/vllm/multimodal/budget.py
@@ -3,11 +3,14 @@
 from collections.abc import Mapping
 
 from vllm.config import ModelConfig, VllmConfig
+from vllm.logger import init_logger
 from vllm.multimodal.processing import BaseMultiModalProcessor
 from vllm.multimodal.registry import MultiModalRegistry
 from vllm.utils.torch_utils import set_default_torch_num_threads
 from vllm.v1.core.encoder_cache_manager import compute_mm_encoder_budget
 
+logger = init_logger(__name__)
+
 
 def get_mm_max_toks_per_item(
     model_config: ModelConfig,
@@ -87,6 +90,24 @@ def __init__(
             mm_max_toks_per_item,
         )
 
+        # When enable_mm_embeds=True and all limits are 0, we still need
+        # encoder cache space to store pre-computed embeddings. Use the
+        # scheduler's default settings as a minimum.
+        mm_config = model_config.get_multimodal_config()
+        if (mm_config is not None and mm_config.enable_mm_embeds
+                and encoder_compute_budget == 0
+                and encoder_cache_size == 0):
+            encoder_compute_budget = (
+                scheduler_config.max_num_encoder_input_tokens)
+            encoder_cache_size = scheduler_config.encoder_cache_size
+            logger.info(
+                "enable_mm_embeds is True with all modality limits=0. "
+                "Using default encoder cache settings for embeddings: "
+                "compute_budget=%d, cache_size=%d",
+                encoder_compute_budget,
+                encoder_cache_size,
+            )
+
         self.encoder_compute_budget = encoder_compute_budget
         self.encoder_cache_size = encoder_cache_size
 
diff --git a/vllm/multimodal/processing/processor.py b/vllm/multimodal/processing/processor.py
index fe697c5ceab4..ee5ee3f7bea5 100644
--- a/vllm/multimodal/processing/processor.py
+++ b/vllm/multimodal/processing/processor.py
@@ -1018,6 +1018,79 @@ def __call__(
     ) -> MultiModalInputs:
         return self.apply(prompt, mm_items, hf_processor_mm_kwargs, mm_uuids=mm_uuids)
 
+    def _get_data_parser(self) -> MultiModalDataParser:
+        """
+        Construct a parser to preprocess multi-modal data items
+        before passing them to
+        [`_get_hf_mm_data`][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_data].
+
+        You can support additional modalities by creating a subclass
+        of [`MultiModalDataParser`][vllm.multimodal.parse.MultiModalDataParser]
+        that has additional subparsers.
+        """
+        # Get expected hidden size for embedding validation if mm_embeds enabled
+        # This validates hidden dimensions to prevent vulnerabilities: embeddings
+        # with correct ndim but wrong shape could cause crashes at inference time
+        mm_config = self.info.ctx.model_config.get_multimodal_config()
+        expected_hidden_size = None
+        if mm_config.enable_mm_embeds:
+            expected_hidden_size = self.info.ctx.model_config.get_inputs_embeds_size()
+
+        return MultiModalDataParser(expected_hidden_size=expected_hidden_size)
+
+    def validate_num_items(
+        self,
+        modality: str,
+        num_items: int,
+    ) -> None:
+        supported_limit = self.supported_mm_limits.get(modality, 0)
+        allowed_limit = self.allowed_mm_limits.get(modality, 0)
+
+        if supported_limit is None:
+            supported_limit = allowed_limit
+
+        limit = min(supported_limit, allowed_limit)
+
+        if num_items > limit:
+            msg = f"At most {limit} {modality}(s) may be provided in one prompt."
+
+            if num_items <= supported_limit:
+                msg += " Set `--limit-mm-per-prompt` to increase this limit."
+
+            raise ValueError(msg)
+
+    def _to_mm_items(
+        self,
+        mm_data: MultiModalDataDict,
+    ) -> MultiModalDataItems:
+        """
+        Normalize
+        [`MultiModalDataDict`][vllm.multimodal.inputs.MultiModalDataDict]
+        to [`MultiModalDataItems`][vllm.multimodal.parse.MultiModalDataItems]
+        before passing them to
+        [`_get_hf_mm_data`][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_data].
+        """
+        mm_items = self.data_parser.parse_mm_data(mm_data)
+
+        mm_config = self.info.ctx.model_config.get_multimodal_config()
+
+        for modality, items in mm_items.items():
+            if isinstance(items, (EmbeddingItems, DictEmbeddingItems)):
+                if not mm_config.enable_mm_embeds:
+                    raise ValueError(
+                        f"You must set `--enable-mm-embeds` to input "
+                        f"`{modality}_embeds`"
+                    )
+                if mm_config.get_limit_per_prompt(modality) == 0:
+                    logger.info(
+                        f"Skipping count validation for modality '{modality}' "
+                        f"(embeddings with limit=0)"
+                    )
+                    continue
+            self.validate_num_items(modality, len(items))
+
+        return mm_items
+
     @abstractmethod
     def _get_mm_fields_config(
         self,
diff --git a/vllm/multimodal/registry.py b/vllm/multimodal/registry.py
index 82329d17f1dc..6c7e86a4fb73 100644
--- a/vllm/multimodal/registry.py
+++ b/vllm/multimodal/registry.py
@@ -138,6 +138,11 @@ def supports_multimodal_inputs(self, model_config: "ModelConfig") -> bool:
             mm_config.get_limit_per_prompt(modality) == 0
             for modality in info.supported_mm_limits
         ):
+            # If enable_mm_embeds is True, we still need MM infrastructure
+            # to process pre-computed embeddings even though encoder won't run
+            if mm_config.enable_mm_embeds:
+                return True
+
             logger.info_once(
                 "All limits of multimodal modalities supported by the model "
                 "are set to 0, running in text-only mode."
diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py
index ec36e159147d..d0a55d2ff5ef 100644
--- a/vllm/v1/worker/gpu_model_runner.py
+++ b/vllm/v1/worker/gpu_model_runner.py
@@ -5116,40 +5116,54 @@ def profile_run(self) -> None:
                 assert mm_budget is not None
 
                 if (encoder_budget := mm_budget.get_encoder_budget()) > 0:
-                    # NOTE: Currently model is profiled with a single non-text
-                    # modality with the max possible input tokens even when
-                    # it supports multiple.
-                    dummy_modality = mm_budget.get_modality_with_max_tokens()
-                    max_mm_items_per_batch = mm_budget.mm_max_items_per_batch[
-                        dummy_modality
-                    ]
+                    if not mm_budget.mm_max_toks_per_item:
+                        # All modality limits are 0 — embedding-only mode.
+                        # Budget is non-zero for embedding storage, but
+                        # there's no encoder to profile.
+                        logger.info(
+                            "Skipping encoder profiling for embedding-only "
+                            "mode (all modality limits=0 with "
+                            "enable_mm_embeds=True).",
+                        )
+                    else:
+                        # NOTE: Currently model is profiled with a single
+                        # non-text modality with the max possible input
+                        # tokens even when it supports multiple.
+                        dummy_modality = mm_budget.get_modality_with_max_tokens()
+                        max_mm_items_per_batch = mm_budget.mm_max_items_per_batch[
+                            dummy_modality
+                        ]
 
-                    logger.info(
-                        "Encoder cache will be initialized with a budget of "
-                        "%s tokens, and profiled with %s %s items of the "
-                        "maximum feature size.",
-                        encoder_budget,
-                        max_mm_items_per_batch,
-                        dummy_modality,
-                    )
+                        logger.info(
+                            "Encoder cache will be initialized with a "
+                            "budget of %s tokens, and profiled with "
+                            "%s %s items of the maximum feature size.",
+                            encoder_budget,
+                            max_mm_items_per_batch,
+                            dummy_modality,
+                        )
 
-                    # Create dummy batch of multimodal inputs.
-                    batched_dummy_mm_inputs = self._get_mm_dummy_batch(
-                        dummy_modality,
-                        max_mm_items_per_batch,
-                    )
+                        # Create dummy batch of multimodal inputs.
+                        batched_dummy_mm_inputs = self._get_mm_dummy_batch(
+                            dummy_modality,
+                            max_mm_items_per_batch,
+                        )
 
-                    # Run multimodal encoder.
-                    dummy_encoder_outputs = self.model.embed_multimodal(
-                        **batched_dummy_mm_inputs
-                    )
+                        # Run multimodal encoder.
+                        dummy_encoder_outputs = (
+                            self.model.embed_multimodal(
+                                **batched_dummy_mm_inputs
+                            )
+                        )
 
-                    sanity_check_mm_encoder_outputs(
-                        dummy_encoder_outputs,
-                        expected_num_items=max_mm_items_per_batch,
-                    )
-                    for i, output in enumerate(dummy_encoder_outputs):
-                        self.encoder_cache[f"tmp_{i}"] = output
+                        sanity_check_mm_encoder_outputs(
+                            dummy_encoder_outputs,
+                            expected_num_items=max_mm_items_per_batch,
+                        )
+                        for i, output in enumerate(
+                            dummy_encoder_outputs
+                        ):
+                            self.encoder_cache[f"tmp_{i}"] = output
 
         # Add `is_profile` here to pre-allocate communication buffers
         hidden_states, last_hidden_states = self._dummy_run(

From ec8a28eeb7c031c1830934a2ea7728fe1960a1f3 Mon Sep 17 00:00:00 2001
From: Reagan Lee <reaganjlee@gmail.com>
Date: Thu, 5 Feb 2026 04:34:09 +0000
Subject: [PATCH 02/17] change
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Reagan Lee <reaganjlee@gmail.com>
Signed-off-by: Reagan Lee <“reaganjlee@gmail.com”>
---
 vllm/entrypoints/chat_utils.py          |  4 +-
 vllm/multimodal/processing/context.py   | 11 ++--
 vllm/multimodal/processing/processor.py | 73 -------------------------
 3 files changed, 8 insertions(+), 80 deletions(-)

diff --git a/vllm/entrypoints/chat_utils.py b/vllm/entrypoints/chat_utils.py
index 94e88c3582e5..fa26d9be4e57 100644
--- a/vllm/entrypoints/chat_utils.py
+++ b/vllm/entrypoints/chat_utils.py
@@ -530,10 +530,10 @@ def add(self, modality: ModalityStr, item: _T) -> str | None:
 
         mm_config = self.model_config.multimodal_config
         if mm_config.enable_mm_embeds and mm_config.get_limit_per_prompt(input_modality) == 0:
-            # Additional check with info logging for this specific case in BaseMultiModalProcessor
+            # Skip validation: embeddings bypass limit when enable_mm_embeds=True
             pass
         else:
-            self.mm_processor.validate_num_items(input_modality, num_items)
+            self.mm_processor.info.validate_num_items(input_modality, num_items)
 
         # Track original modality for vision_chunk items
         if use_vision_chunk:
diff --git a/vllm/multimodal/processing/context.py b/vllm/multimodal/processing/context.py
index 9a98692b5340..77eda3757b63 100644
--- a/vllm/multimodal/processing/context.py
+++ b/vllm/multimodal/processing/context.py
@@ -682,15 +682,16 @@ def parse_mm_data(
 
         if validate:
             mm_config = self.ctx.model_config.get_multimodal_config()
-            if not mm_config.enable_mm_embeds:
-                for modality, items in mm_items.items():
-                    if isinstance(items, (EmbeddingItems, DictEmbeddingItems)):
+
+            for modality, items in mm_items.items():
+                if isinstance(items, (EmbeddingItems, DictEmbeddingItems)):
+                    if not mm_config.enable_mm_embeds:
                         raise ValueError(
                             f"You must set `--enable-mm-embeds` to input "
                             f"`{modality}_embeds`"
                         )
-
-            for modality, items in mm_items.items():
+                    if mm_config.get_limit_per_prompt(modality) == 0:
+                        continue
                 self.validate_num_items(modality, len(items))
 
         return mm_items
diff --git a/vllm/multimodal/processing/processor.py b/vllm/multimodal/processing/processor.py
index ee5ee3f7bea5..fe697c5ceab4 100644
--- a/vllm/multimodal/processing/processor.py
+++ b/vllm/multimodal/processing/processor.py
@@ -1018,79 +1018,6 @@ def __call__(
     ) -> MultiModalInputs:
         return self.apply(prompt, mm_items, hf_processor_mm_kwargs, mm_uuids=mm_uuids)
 
-    def _get_data_parser(self) -> MultiModalDataParser:
-        """
-        Construct a parser to preprocess multi-modal data items
-        before passing them to
-        [`_get_hf_mm_data`][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_data].
-
-        You can support additional modalities by creating a subclass
-        of [`MultiModalDataParser`][vllm.multimodal.parse.MultiModalDataParser]
-        that has additional subparsers.
-        """
-        # Get expected hidden size for embedding validation if mm_embeds enabled
-        # This validates hidden dimensions to prevent vulnerabilities: embeddings
-        # with correct ndim but wrong shape could cause crashes at inference time
-        mm_config = self.info.ctx.model_config.get_multimodal_config()
-        expected_hidden_size = None
-        if mm_config.enable_mm_embeds:
-            expected_hidden_size = self.info.ctx.model_config.get_inputs_embeds_size()
-
-        return MultiModalDataParser(expected_hidden_size=expected_hidden_size)
-
-    def validate_num_items(
-        self,
-        modality: str,
-        num_items: int,
-    ) -> None:
-        supported_limit = self.supported_mm_limits.get(modality, 0)
-        allowed_limit = self.allowed_mm_limits.get(modality, 0)
-
-        if supported_limit is None:
-            supported_limit = allowed_limit
-
-        limit = min(supported_limit, allowed_limit)
-
-        if num_items > limit:
-            msg = f"At most {limit} {modality}(s) may be provided in one prompt."
-
-            if num_items <= supported_limit:
-                msg += " Set `--limit-mm-per-prompt` to increase this limit."
-
-            raise ValueError(msg)
-
-    def _to_mm_items(
-        self,
-        mm_data: MultiModalDataDict,
-    ) -> MultiModalDataItems:
-        """
-        Normalize
-        [`MultiModalDataDict`][vllm.multimodal.inputs.MultiModalDataDict]
-        to [`MultiModalDataItems`][vllm.multimodal.parse.MultiModalDataItems]
-        before passing them to
-        [`_get_hf_mm_data`][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_data].
-        """
-        mm_items = self.data_parser.parse_mm_data(mm_data)
-
-        mm_config = self.info.ctx.model_config.get_multimodal_config()
-
-        for modality, items in mm_items.items():
-            if isinstance(items, (EmbeddingItems, DictEmbeddingItems)):
-                if not mm_config.enable_mm_embeds:
-                    raise ValueError(
-                        f"You must set `--enable-mm-embeds` to input "
-                        f"`{modality}_embeds`"
-                    )
-                if mm_config.get_limit_per_prompt(modality) == 0:
-                    logger.info(
-                        f"Skipping count validation for modality '{modality}' "
-                        f"(embeddings with limit=0)"
-                    )
-                    continue
-            self.validate_num_items(modality, len(items))
-
-        return mm_items
-
     @abstractmethod
     def _get_mm_fields_config(
         self,

From 79b122a970b96bc394a5226bd2b5083d6027d5c6 Mon Sep 17 00:00:00 2001
From: Reagan Lee <reaganjlee@gmail.com>
Date: Thu, 5 Feb 2026 04:53:00 +0000
Subject: [PATCH 03/17] change
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Reagan Lee <reaganjlee@gmail.com>
Signed-off-by: Reagan Lee <“reaganjlee@gmail.com”>
---
 vllm/multimodal/processing/context.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/vllm/multimodal/processing/context.py b/vllm/multimodal/processing/context.py
index 77eda3757b63..a451e34c315d 100644
--- a/vllm/multimodal/processing/context.py
+++ b/vllm/multimodal/processing/context.py
@@ -691,6 +691,9 @@ def parse_mm_data(
                             f"`{modality}_embeds`"
                         )
                     if mm_config.get_limit_per_prompt(modality) == 0:
+                        logger.info(
+                            "Skipping count validation for modality "
+                            "'%s' (embeddings with limit=0)", modality)
                         continue
                 self.validate_num_items(modality, len(items))
 

From bd5ec6d50b83652c21a570b81c1fb5219a6ebf76 Mon Sep 17 00:00:00 2001
From: Reagan Lee <reaganjlee@gmail.com>
Date: Thu, 5 Feb 2026 05:00:19 +0000
Subject: [PATCH 04/17] change
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Reagan Lee <reaganjlee@gmail.com>
Signed-off-by: Reagan Lee <“reaganjlee@gmail.com”>
---
 vllm/entrypoints/chat_utils.py        |  5 ++++-
 vllm/multimodal/budget.py             | 12 +++++++-----
 vllm/multimodal/processing/context.py |  4 +++-
 vllm/v1/worker/gpu_model_runner.py    | 10 +++-------
 4 files changed, 17 insertions(+), 14 deletions(-)

diff --git a/vllm/entrypoints/chat_utils.py b/vllm/entrypoints/chat_utils.py
index fa26d9be4e57..13aa40d9157a 100644
--- a/vllm/entrypoints/chat_utils.py
+++ b/vllm/entrypoints/chat_utils.py
@@ -529,7 +529,10 @@ def add(self, modality: ModalityStr, item: _T) -> str | None:
             num_items = len(self._items_by_modality[original_modality]) + 1
 
         mm_config = self.model_config.multimodal_config
-        if mm_config.enable_mm_embeds and mm_config.get_limit_per_prompt(input_modality) == 0:
+        if (
+            mm_config.enable_mm_embeds
+            and mm_config.get_limit_per_prompt(input_modality) == 0
+        ):
             # Skip validation: embeddings bypass limit when enable_mm_embeds=True
             pass
         else:
diff --git a/vllm/multimodal/budget.py b/vllm/multimodal/budget.py
index fa003fdd7f11..a4564d8cc39b 100644
--- a/vllm/multimodal/budget.py
+++ b/vllm/multimodal/budget.py
@@ -94,11 +94,13 @@ def __init__(
         # encoder cache space to store pre-computed embeddings. Use the
         # scheduler's default settings as a minimum.
         mm_config = model_config.get_multimodal_config()
-        if (mm_config is not None and mm_config.enable_mm_embeds
-                and encoder_compute_budget == 0
-                and encoder_cache_size == 0):
-            encoder_compute_budget = (
-                scheduler_config.max_num_encoder_input_tokens)
+        if (
+            mm_config is not None
+            and mm_config.enable_mm_embeds
+            and encoder_compute_budget == 0
+            and encoder_cache_size == 0
+        ):
+            encoder_compute_budget = scheduler_config.max_num_encoder_input_tokens
             encoder_cache_size = scheduler_config.encoder_cache_size
             logger.info(
                 "enable_mm_embeds is True with all modality limits=0. "
diff --git a/vllm/multimodal/processing/context.py b/vllm/multimodal/processing/context.py
index a451e34c315d..8b9a47bb35eb 100644
--- a/vllm/multimodal/processing/context.py
+++ b/vllm/multimodal/processing/context.py
@@ -693,7 +693,9 @@ def parse_mm_data(
                     if mm_config.get_limit_per_prompt(modality) == 0:
                         logger.info(
                             "Skipping count validation for modality "
-                            "'%s' (embeddings with limit=0)", modality)
+                            "'%s' (embeddings with limit=0)",
+                            modality,
+                        )
                         continue
                 self.validate_num_items(modality, len(items))
 
diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py
index d0a55d2ff5ef..b592f1eb901a 100644
--- a/vllm/v1/worker/gpu_model_runner.py
+++ b/vllm/v1/worker/gpu_model_runner.py
@@ -5150,19 +5150,15 @@ def profile_run(self) -> None:
                         )
 
                         # Run multimodal encoder.
-                        dummy_encoder_outputs = (
-                            self.model.embed_multimodal(
-                                **batched_dummy_mm_inputs
-                            )
+                        dummy_encoder_outputs = self.model.embed_multimodal(
+                            **batched_dummy_mm_inputs
                         )
 
                         sanity_check_mm_encoder_outputs(
                             dummy_encoder_outputs,
                             expected_num_items=max_mm_items_per_batch,
                         )
-                        for i, output in enumerate(
-                            dummy_encoder_outputs
-                        ):
+                        for i, output in enumerate(dummy_encoder_outputs):
                             self.encoder_cache[f"tmp_{i}"] = output
 
         # Add `is_profile` here to pre-allocate communication buffers

From e5eaa360eddae4001e2a7a04ed8dd88cad956a8d Mon Sep 17 00:00:00 2001
From: Reagan Lee <reaganjlee@gmail.com>
Date: Thu, 5 Feb 2026 05:14:22 +0000
Subject: [PATCH 05/17] change
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Reagan Lee <reaganjlee@gmail.com>
Signed-off-by: Reagan Lee <“reaganjlee@gmail.com”>
---
 vllm/entrypoints/chat_utils.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/vllm/entrypoints/chat_utils.py b/vllm/entrypoints/chat_utils.py
index 13aa40d9157a..c48d7bea983c 100644
--- a/vllm/entrypoints/chat_utils.py
+++ b/vllm/entrypoints/chat_utils.py
@@ -530,8 +530,10 @@ def add(self, modality: ModalityStr, item: _T) -> str | None:
 
         mm_config = self.model_config.multimodal_config
         if (
-            mm_config.enable_mm_embeds
+            mm_config is not None
+            and mm_config.enable_mm_embeds
             and mm_config.get_limit_per_prompt(input_modality) == 0
+            and original_modality.endswith("_embeds")
         ):
             # Skip validation: embeddings bypass limit when enable_mm_embeds=True
             pass

From 6a4a569a3a49757935398d087ca15906d9b95bd9 Mon Sep 17 00:00:00 2001
From: Reagan Lee <reaganjlee@gmail.com>
Date: Thu, 5 Feb 2026 06:24:49 +0000
Subject: [PATCH 06/17] logger.debug + doc
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Reagan Lee <reaganjlee@gmail.com>
Signed-off-by: Reagan Lee <“reaganjlee@gmail.com”>
---
 vllm/config/multimodal.py             | 7 ++++---
 vllm/multimodal/processing/context.py | 2 +-
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/vllm/config/multimodal.py b/vllm/config/multimodal.py
index de68b185488c..30305e4be96b 100644
--- a/vllm/config/multimodal.py
+++ b/vllm/config/multimodal.py
@@ -76,9 +76,10 @@ class MultiModalConfig:
     for the OpenAI-compatible server, this refers to chat messages with content
     `"type": "*_embeds"`.
 
-    When enabled, precomputed embeddings bypass the `--limit-mm-per-prompt` count
-    restriction. This allows unlimited embeddings even when limit is 0, enabling
-    memory savings by not loading encoder modules while still processing embeddings.
+    When enabled with `--limit-mm-per-prompt` set to 0 for a modality,
+    precomputed embeddings skip count validation for that modality, 
+    saving memory by not loading encoder modules while still enabling 
+    embeddings as an input. Limits greater than 0 still apply to embeddings.
 
     WARNING: The vLLM engine may crash if incorrect shape of embeddings is passed.
     Only enable this flag for trusted users!"""
diff --git a/vllm/multimodal/processing/context.py b/vllm/multimodal/processing/context.py
index 8b9a47bb35eb..0eea1c6e28fc 100644
--- a/vllm/multimodal/processing/context.py
+++ b/vllm/multimodal/processing/context.py
@@ -691,7 +691,7 @@ def parse_mm_data(
                             f"`{modality}_embeds`"
                         )
                     if mm_config.get_limit_per_prompt(modality) == 0:
-                        logger.info(
+                        logger.debug(
                             "Skipping count validation for modality "
                             "'%s' (embeddings with limit=0)",
                             modality,

From ad36e655c809e6ee016175b8c90964e6d8cb4207 Mon Sep 17 00:00:00 2001
From: Reagan Lee <“reaganjlee@gmail.com”>
Date: Fri, 6 Feb 2026 11:29:04 +0000
Subject: [PATCH 07/17] change
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Reagan Lee <“reaganjlee@gmail.com”>
---
 vllm/multimodal/budget.py          | 62 +++++++++++++++++-------------
 vllm/v1/worker/gpu_model_runner.py |  3 ++
 2 files changed, 39 insertions(+), 26 deletions(-)

diff --git a/vllm/multimodal/budget.py b/vllm/multimodal/budget.py
index a4564d8cc39b..e8d9c2287f26 100644
--- a/vllm/multimodal/budget.py
+++ b/vllm/multimodal/budget.py
@@ -62,12 +62,26 @@ def __init__(
             processor = mm_registry.create_processor(model_config, cache=cache)
 
             self.cache = cache
+            mm_config = model_config.get_multimodal_config()
+            enable_mm_embeds = (mm_config is not None
+                                and mm_config.enable_mm_embeds)
+
+            supported_mm_limits = processor.info.supported_mm_limits
             self.mm_limits = mm_limits = processor.info.allowed_mm_limits
 
-            active_modalities = {
-                modality for modality, limit in mm_limits.items() if limit > 0
+            # Modalities that pass through the MM encoder tower
+            tower_modalities = {
+                modality for modality in supported_mm_limits
+                if mm_limits.get(modality, 0) > 0
+            }
+            # Modalities that bypass the tower (pre-computed embeddings only)
+            embed_only_modalities = {
+                modality for modality in supported_mm_limits
+                if enable_mm_embeds and mm_limits.get(modality, 0) == 0
             }
 
+            active_modalities = tower_modalities | embed_only_modalities
+
             all_mm_max_toks_per_item = get_mm_max_toks_per_item(
                 model_config,
                 mm_registry,
@@ -75,54 +89,50 @@ def __init__(
                 mm_counts=dict.fromkeys(active_modalities, 1),
             )
 
+        if embed_only_modalities:
+            logger.info(
+                "enable_mm_embeds is True; modalities handled as "
+                "embedding-only: %s",
+                embed_only_modalities,
+            )
+
         # Some models (e.g., Qwen3Omni with use_audio_in_video=True) share
         # placeholders between modalities, so not all active modalities will
         # have their own entry in the returned dict. We filter to only include
         # modalities that have independent placeholder tokens.
-        mm_max_toks_per_item = {
+        active_mm_max_toks_per_item = {
             modality: all_mm_max_toks_per_item[modality]
             for modality in active_modalities
             if modality in all_mm_max_toks_per_item
         }
+        tower_mm_max_toks_per_item = {
+            modality: active_mm_max_toks_per_item[modality]
+            for modality in tower_modalities
+            if modality in active_mm_max_toks_per_item
+        }
 
+        # Encoder budget is computed from all active modalities (including
+        # embedding-only ones that need encoder cache space).
         encoder_compute_budget, encoder_cache_size = compute_mm_encoder_budget(
             scheduler_config,
-            mm_max_toks_per_item,
+            active_mm_max_toks_per_item,
         )
 
-        # When enable_mm_embeds=True and all limits are 0, we still need
-        # encoder cache space to store pre-computed embeddings. Use the
-        # scheduler's default settings as a minimum.
-        mm_config = model_config.get_multimodal_config()
-        if (
-            mm_config is not None
-            and mm_config.enable_mm_embeds
-            and encoder_compute_budget == 0
-            and encoder_cache_size == 0
-        ):
-            encoder_compute_budget = scheduler_config.max_num_encoder_input_tokens
-            encoder_cache_size = scheduler_config.encoder_cache_size
-            logger.info(
-                "enable_mm_embeds is True with all modality limits=0. "
-                "Using default encoder cache settings for embeddings: "
-                "compute_budget=%d, cache_size=%d",
-                encoder_compute_budget,
-                encoder_cache_size,
-            )
-
         self.encoder_compute_budget = encoder_compute_budget
         self.encoder_cache_size = encoder_cache_size
 
         mm_max_items_per_prompt = dict[str, int]()
         mm_max_items_per_batch = dict[str, int]()
 
-        for modality, max_toks_per_item in mm_max_toks_per_item.items():
+        # Per-prompt/per-batch limits are only relevant for tower modalities
+        # (embedding-only modalities don't go through the encoder tower).
+        for modality, max_toks_per_item in tower_mm_max_toks_per_item.items():
             (
                 mm_max_items_per_prompt[modality],
                 mm_max_items_per_batch[modality],
             ) = self._get_max_items(modality, max_toks_per_item)
 
-        self.mm_max_toks_per_item = mm_max_toks_per_item
+        self.mm_max_toks_per_item = tower_mm_max_toks_per_item
         self.mm_max_items_per_prompt: Mapping[str, int] = mm_max_items_per_prompt
         self.mm_max_items_per_batch: Mapping[str, int] = mm_max_items_per_batch
 
diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py
index b592f1eb901a..24a9470da45d 100644
--- a/vllm/v1/worker/gpu_model_runner.py
+++ b/vllm/v1/worker/gpu_model_runner.py
@@ -1259,6 +1259,9 @@ def _dummy_mm_kwargs(self, num_seqs: int) -> BatchedTensorInputs:
         mm_budget = self.mm_budget
         assert mm_budget is not None
 
+        if not mm_budget.mm_max_toks_per_item:
+            return {}  # No tower modalities (embed-only mode)
+
         dummy_modality = mm_budget.get_modality_with_max_tokens()
         return self._get_mm_dummy_batch(dummy_modality, num_seqs)
 

From f068bf015d1792508a2674f31a9dba436e5e30d7 Mon Sep 17 00:00:00 2001
From: Reagan Lee <96998476+reaganjlee@users.noreply.github.com>
Date: Fri, 6 Feb 2026 19:33:47 -0800
Subject: [PATCH 08/17] Update vllm/multimodal/processing/context.py
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Signed-off-by: Reagan Lee <96998476+reaganjlee@users.noreply.github.com>
Signed-off-by: Reagan Lee <“reaganjlee@gmail.com”>
---
 vllm/multimodal/processing/context.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/vllm/multimodal/processing/context.py b/vllm/multimodal/processing/context.py
index 0eea1c6e28fc..d5c14310c77d 100644
--- a/vllm/multimodal/processing/context.py
+++ b/vllm/multimodal/processing/context.py
@@ -681,7 +681,7 @@ def parse_mm_data(
         mm_items = self.data_parser.parse_mm_data(mm_data)
 
         if validate:
-            mm_config = self.ctx.model_config.get_multimodal_config()
+            mm_config = self.ctx.get_mm_config()
 
             for modality, items in mm_items.items():
                 if isinstance(items, (EmbeddingItems, DictEmbeddingItems)):

From 9f555096cd84f9e5ef5f097a8fc576f45d2604e9 Mon Sep 17 00:00:00 2001
From: Reagan Lee <“reaganjlee@gmail.com”>
Date: Sat, 7 Feb 2026 04:45:05 +0000
Subject: [PATCH 09/17] e2e
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Reagan Lee <“reaganjlee@gmail.com”>
---
 tests/entrypoints/llm/test_mm_embeds.py | 69 +++++++++++++++++++++++++
 1 file changed, 69 insertions(+)
 create mode 100644 tests/entrypoints/llm/test_mm_embeds.py

diff --git a/tests/entrypoints/llm/test_mm_embeds.py b/tests/entrypoints/llm/test_mm_embeds.py
new file mode 100644
index 000000000000..3d5f3fb3b100
--- /dev/null
+++ b/tests/entrypoints/llm/test_mm_embeds.py
@@ -0,0 +1,69 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import weakref
+
+import pytest
+
+from vllm import LLM, SamplingParams
+from vllm.assets.image import ImageAsset
+from vllm.distributed import cleanup_dist_env_and_memory
+
+MODEL = "llava-hf/llava-1.5-7b-hf"
+PROMPT = "USER: <image>\nDescribe this image briefly.\nASSISTANT:"
+TEXT_ONLY_PROMPT = "USER: What is 2 + 2?\nASSISTANT:"
+
+
+@pytest.fixture(scope="module")
+def llm():
+    """LLM with enable_mm_embeds=True and all modality limits zeroed out."""
+    llm = LLM(
+        model=MODEL,
+        max_model_len=2048,
+        enforce_eager=True,
+        gpu_memory_utilization=0.8,
+        enable_mm_embeds=True,
+        limit_mm_per_prompt={"image": 0},
+    )
+
+    yield weakref.proxy(llm)
+
+    del llm
+
+    cleanup_dist_env_and_memory()
+
+
+@pytest.mark.skip_global_cleanup
+def test_generate_with_embedding(llm: LLM):
+    """Pre-computed embedding produces tokens without hanging."""
+    embedding = ImageAsset("stop_sign").image_embeds
+    outputs = llm.generate(
+        {"prompt": PROMPT, "multi_modal_data": {"image": embedding}},
+        sampling_params=SamplingParams(max_tokens=32, temperature=0.0),
+    )
+    assert len(outputs) == 1
+    assert len(outputs[0].outputs[0].text) > 0
+
+
+@pytest.mark.skip_global_cleanup
+def test_raw_image_rejected(llm: LLM):
+    """Raw image input is still rejected when limit=0."""
+    raw_image = ImageAsset("stop_sign").pil_image
+    with pytest.raises(ValueError, match=r"At most 0 image\(s\)"):
+        llm.generate(
+            {"prompt": PROMPT, "multi_modal_data": {"image": raw_image}},
+            sampling_params=SamplingParams(max_tokens=16),
+        )
+
+
+@pytest.mark.skip_global_cleanup
+def test_text_only_prompt(llm: LLM):
+    """Text-only prompts still work under this config."""
+    outputs = llm.generate(
+        TEXT_ONLY_PROMPT,
+        sampling_params=SamplingParams(max_tokens=16, temperature=0.0),
+    )
+    assert len(outputs) == 1
+    assert len(outputs[0].outputs[0].text) > 0
+
+

From edd409d339bc7c654883c9762617dfaab3e23941 Mon Sep 17 00:00:00 2001
From: Reagan Lee <“reaganjlee@gmail.com”>
Date: Sat, 7 Feb 2026 05:14:42 +0000
Subject: [PATCH 10/17] ruff
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Reagan Lee <“reaganjlee@gmail.com”>
---
 vllm/multimodal/budget.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/vllm/multimodal/budget.py b/vllm/multimodal/budget.py
index e8d9c2287f26..2a5535d521b0 100644
--- a/vllm/multimodal/budget.py
+++ b/vllm/multimodal/budget.py
@@ -63,20 +63,21 @@ def __init__(
 
             self.cache = cache
             mm_config = model_config.get_multimodal_config()
-            enable_mm_embeds = (mm_config is not None
-                                and mm_config.enable_mm_embeds)
+            enable_mm_embeds = mm_config is not None and mm_config.enable_mm_embeds
 
             supported_mm_limits = processor.info.supported_mm_limits
             self.mm_limits = mm_limits = processor.info.allowed_mm_limits
 
             # Modalities that pass through the MM encoder tower
             tower_modalities = {
-                modality for modality in supported_mm_limits
+                modality
+                for modality in supported_mm_limits
                 if mm_limits.get(modality, 0) > 0
             }
             # Modalities that bypass the tower (pre-computed embeddings only)
             embed_only_modalities = {
-                modality for modality in supported_mm_limits
+                modality
+                for modality in supported_mm_limits
                 if enable_mm_embeds and mm_limits.get(modality, 0) == 0
             }
 
@@ -91,8 +92,7 @@ def __init__(
 
         if embed_only_modalities:
             logger.info(
-                "enable_mm_embeds is True; modalities handled as "
-                "embedding-only: %s",
+                "enable_mm_embeds is True; modalities handled as embedding-only: %s",
                 embed_only_modalities,
             )
 

From 6842ab11d7586669fbb3f1c8f7f88fcaab673641 Mon Sep 17 00:00:00 2001
From: Reagan Lee <“reaganjlee@gmail.com”>
Date: Sat, 7 Feb 2026 10:23:28 +0000
Subject: [PATCH 11/17] change
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Reagan Lee <“reaganjlee@gmail.com”>
---
 vllm/multimodal/processing/dummy_inputs.py | 3 ++-
 vllm/multimodal/processing/processor.py    | 3 ++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/vllm/multimodal/processing/dummy_inputs.py b/vllm/multimodal/processing/dummy_inputs.py
index b23e2b86cc20..1e33cc071ae9 100644
--- a/vllm/multimodal/processing/dummy_inputs.py
+++ b/vllm/multimodal/processing/dummy_inputs.py
@@ -95,7 +95,8 @@ def get_dummy_processor_inputs(
         """
         dummy_text = self.get_dummy_text(mm_counts)
         dummy_mm_data = self.get_dummy_mm_data(seq_len, mm_counts, mm_options)
-        dummy_mm_items = self.info.parse_mm_data(dummy_mm_data)
+        dummy_mm_items = self.info.parse_mm_data(dummy_mm_data,
+                                                 validate=False)
 
         tokenization_kwargs = {"truncation": False}
 
diff --git a/vllm/multimodal/processing/processor.py b/vllm/multimodal/processing/processor.py
index fe697c5ceab4..4c32035f608c 100644
--- a/vllm/multimodal/processing/processor.py
+++ b/vllm/multimodal/processing/processor.py
@@ -1395,7 +1395,8 @@ def _get_cache_missing_items(
                     missing_modality_data.append(data)
             mm_missing_data[modality] = missing_modality_data
 
-        mm_missing_items = self.info.parse_mm_data(mm_missing_data)
+        mm_missing_items = self.info.parse_mm_data(mm_missing_data,
+                                                     validate=False)
 
         return mm_is_cached, mm_missing_items
 

From e107e8214d64cb17f93ea11c956c054c1870b48d Mon Sep 17 00:00:00 2001
From: Reagan Lee <“reaganjlee@gmail.com”>
Date: Sat, 7 Feb 2026 10:24:53 +0000
Subject: [PATCH 12/17] rename
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Reagan Lee <“reaganjlee@gmail.com”>
---
 .../entrypoints/llm/{test_mm_embeds.py => test_mm_embeds_only.py} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename tests/entrypoints/llm/{test_mm_embeds.py => test_mm_embeds_only.py} (100%)

diff --git a/tests/entrypoints/llm/test_mm_embeds.py b/tests/entrypoints/llm/test_mm_embeds_only.py
similarity index 100%
rename from tests/entrypoints/llm/test_mm_embeds.py
rename to tests/entrypoints/llm/test_mm_embeds_only.py

From a8508a0b55a401ad3af3d144f73a96a510f4fea1 Mon Sep 17 00:00:00 2001
From: Reagan Lee <“reaganjlee@gmail.com”>
Date: Sat, 7 Feb 2026 19:09:20 +0000
Subject: [PATCH 13/17] change + precommit
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Reagan Lee <“reaganjlee@gmail.com”>
---
 tests/entrypoints/llm/test_mm_embeds_only.py | 2 --
 vllm/multimodal/budget.py                    | 2 +-
 vllm/multimodal/processing/dummy_inputs.py   | 3 +--
 vllm/multimodal/processing/processor.py      | 3 +--
 4 files changed, 3 insertions(+), 7 deletions(-)

diff --git a/tests/entrypoints/llm/test_mm_embeds_only.py b/tests/entrypoints/llm/test_mm_embeds_only.py
index 3d5f3fb3b100..13d0fd58b139 100644
--- a/tests/entrypoints/llm/test_mm_embeds_only.py
+++ b/tests/entrypoints/llm/test_mm_embeds_only.py
@@ -65,5 +65,3 @@ def test_text_only_prompt(llm: LLM):
     )
     assert len(outputs) == 1
     assert len(outputs[0].outputs[0].text) > 0
-
-
diff --git a/vllm/multimodal/budget.py b/vllm/multimodal/budget.py
index 2a5535d521b0..871874246425 100644
--- a/vllm/multimodal/budget.py
+++ b/vllm/multimodal/budget.py
@@ -91,7 +91,7 @@ def __init__(
             )
 
         if embed_only_modalities:
-            logger.info(
+            logger.info_once(
                 "enable_mm_embeds is True; modalities handled as embedding-only: %s",
                 embed_only_modalities,
             )
diff --git a/vllm/multimodal/processing/dummy_inputs.py b/vllm/multimodal/processing/dummy_inputs.py
index 1e33cc071ae9..a93fd2c2422a 100644
--- a/vllm/multimodal/processing/dummy_inputs.py
+++ b/vllm/multimodal/processing/dummy_inputs.py
@@ -95,8 +95,7 @@ def get_dummy_processor_inputs(
         """
         dummy_text = self.get_dummy_text(mm_counts)
         dummy_mm_data = self.get_dummy_mm_data(seq_len, mm_counts, mm_options)
-        dummy_mm_items = self.info.parse_mm_data(dummy_mm_data,
-                                                 validate=False)
+        dummy_mm_items = self.info.parse_mm_data(dummy_mm_data, validate=False)
 
         tokenization_kwargs = {"truncation": False}
 
diff --git a/vllm/multimodal/processing/processor.py b/vllm/multimodal/processing/processor.py
index 4c32035f608c..5f98cce3daf3 100644
--- a/vllm/multimodal/processing/processor.py
+++ b/vllm/multimodal/processing/processor.py
@@ -1395,8 +1395,7 @@ def _get_cache_missing_items(
                     missing_modality_data.append(data)
             mm_missing_data[modality] = missing_modality_data
 
-        mm_missing_items = self.info.parse_mm_data(mm_missing_data,
-                                                     validate=False)
+        mm_missing_items = self.info.parse_mm_data(mm_missing_data, validate=False)
 
         return mm_is_cached, mm_missing_items
 

From e33d53705abe18a4bd04f780942b8d7a6c8dc633 Mon Sep 17 00:00:00 2001
From: Reagan Lee <“reaganjlee@gmail.com”>
Date: Sat, 7 Feb 2026 19:35:58 +0000
Subject: [PATCH 14/17] mypy CI
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Reagan Lee <“reaganjlee@gmail.com”>
---
 vllm/multimodal/budget.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/vllm/multimodal/budget.py b/vllm/multimodal/budget.py
index 871874246425..0ffbcb776916 100644
--- a/vllm/multimodal/budget.py
+++ b/vllm/multimodal/budget.py
@@ -93,7 +93,7 @@ def __init__(
         if embed_only_modalities:
             logger.info_once(
                 "enable_mm_embeds is True; modalities handled as embedding-only: %s",
-                embed_only_modalities,
+                sorted(embed_only_modalities),
             )
 
         # Some models (e.g., Qwen3Omni with use_audio_in_video=True) share

From dd2a342a394c2841bc2a9c75b1e8b1fd14818232 Mon Sep 17 00:00:00 2001
From: Reagan Lee <“reaganjlee@gmail.com”>
Date: Sat, 7 Feb 2026 19:52:08 +0000
Subject: [PATCH 15/17] precommit again
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Reagan Lee <“reaganjlee@gmail.com”>
---
 vllm/multimodal/budget.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/vllm/multimodal/budget.py b/vllm/multimodal/budget.py
index 0ffbcb776916..a9bcfcbd4b8d 100644
--- a/vllm/multimodal/budget.py
+++ b/vllm/multimodal/budget.py
@@ -93,7 +93,7 @@ def __init__(
         if embed_only_modalities:
             logger.info_once(
                 "enable_mm_embeds is True; modalities handled as embedding-only: %s",
-                sorted(embed_only_modalities),
+                tuple(embed_only_modalities),
             )
 
         # Some models (e.g., Qwen3Omni with use_audio_in_video=True) share

From 034dca270115c41d4d996f06725fc39e82a43444 Mon Sep 17 00:00:00 2001
From: Reagan Lee <“reaganjlee@gmail.com”>
Date: Sun, 8 Feb 2026 09:00:57 +0000
Subject: [PATCH 16/17] RFC, change out unit test for more e2e-like
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Reagan Lee <“reaganjlee@gmail.com”>
---
 tests/multimodal/test_processing.py | 79 ++++++++++++++++++-----------
 1 file changed, 48 insertions(+), 31 deletions(-)

diff --git a/tests/multimodal/test_processing.py b/tests/multimodal/test_processing.py
index 316234ba59b4..2f6b4c53800e 100644
--- a/tests/multimodal/test_processing.py
+++ b/tests/multimodal/test_processing.py
@@ -901,7 +901,7 @@ def test_find_mm_placeholders(
 
 @pytest.mark.parametrize("model_id", ["llava-hf/llava-v1.6-mistral-7b-hf"])
 @pytest.mark.parametrize(
-    ("limit", "num_supported", "is_valid"),
+    ("num_images", "limit", "is_valid"),
     [
         (0, 0, True),
         (0, 1, True),
@@ -912,7 +912,7 @@ def test_find_mm_placeholders(
         (2, 2, True),
     ],
 )
-def test_limit_mm_per_prompt_dummy(model_id, limit, num_supported, is_valid):
+def test_limit_mm_per_prompt_apply(model_id, num_images, limit, is_valid):
     limit_mm_per_prompt = {"image": limit}
 
     model_config = ModelConfig(
@@ -921,33 +921,46 @@ def test_limit_mm_per_prompt_dummy(model_id, limit, num_supported, is_valid):
     )
 
     processor = MULTIMODAL_REGISTRY.create_processor(model_config)
-    processor.info.get_supported_mm_limits = lambda: {"image": num_supported}
+
+    rng = np.random.RandomState(0)
+    image = random_image(rng, min_wh=128, max_wh=256)
+    if num_images == 0:
+        mm_data = {}
+    elif num_images == 1:
+        mm_data = {"image": image}
+    else:
+        mm_data = {"image": [image] * num_images}
 
     exc_ctx = nullcontext() if is_valid else pytest.raises(ValueError, match="At most")
 
     with exc_ctx:
-        MULTIMODAL_REGISTRY.get_dummy_mm_inputs(
-            model_config,
-            mm_counts=limit_mm_per_prompt,
-            processor=processor,
+        processor.apply(
+            "<image>" * num_images,
+            mm_items=processor.info.parse_mm_data(mm_data),
+            hf_processor_mm_kwargs={},
         )
 
 
 @pytest.mark.parametrize("model_id", ["llava-hf/llava-v1.6-mistral-7b-hf"])
 @pytest.mark.parametrize(
-    ("num_images", "limit", "is_valid"),
+    ("user_limit", "supported_limit"),
     [
-        (0, 0, True),
-        (0, 1, True),
-        (1, 0, False),
-        (1, 1, True),
-        (1, 2, True),
-        (2, 1, False),
-        (2, 2, True),
+        (0, 0),
+        (0, 1),
+        (1, 0),  # user wants 1, model supports 0 → capped to 0
+        (1, 1),
+        (1, 2),
+        (2, 1),  # user wants 2, model supports 1 → capped to 1
+        (2, 2),
+        (5, 1),  # large user limit, low model support → capped to 1
+        (1, 5),
+        (10, 0),  # large user limit, no model support → capped to 0
     ],
 )
-def test_limit_mm_per_prompt_apply(model_id, num_images, limit, is_valid):
-    limit_mm_per_prompt = {"image": limit}
+def test_budget_caps_prevent_dummy_input_validation_failure(
+    model_id, user_limit, supported_limit
+):
+    limit_mm_per_prompt = {"image": user_limit}
 
     model_config = ModelConfig(
         model=model_id,
@@ -955,23 +968,27 @@ def test_limit_mm_per_prompt_apply(model_id, num_images, limit, is_valid):
     )
 
     processor = MULTIMODAL_REGISTRY.create_processor(model_config)
+    processor.info.get_supported_mm_limits = lambda: {"image": supported_limit}
 
-    rng = np.random.RandomState(0)
-    image = random_image(rng, min_wh=128, max_wh=256)
-    if num_images == 0:
-        mm_data = {}
-    elif num_images == 1:
-        mm_data = {"image": image}
-    else:
-        mm_data = {"image": [image] * num_images}
+    # This is what budget.py uses to derive mm_counts
+    allowed = processor.info.allowed_mm_limits
 
-    exc_ctx = nullcontext() if is_valid else pytest.raises(ValueError, match="At most")
+    assert allowed["image"] <= supported_limit, (
+        f"allowed_mm_limits['image']={allowed['image']} exceeds "
+        f"supported_limit={supported_limit}"
+    )
 
-    with exc_ctx:
-        processor.apply(
-            "<image>" * num_images,
-            mm_items=processor.info.parse_mm_data(mm_data),
-            hf_processor_mm_kwargs={},
+    assert allowed["image"] <= user_limit, (
+        f"allowed_mm_limits['image']={allowed['image']} exceeds user_limit={user_limit}"
+    )
+
+    assert allowed["image"] == min(user_limit, supported_limit)
+
+    if allowed["image"] > 0:
+        MULTIMODAL_REGISTRY.get_dummy_mm_inputs(
+            model_config,
+            mm_counts={"image": allowed["image"]},
+            processor=processor,
         )
 
 

From 4eae106902a3a0bf59b82127c98153eb6aa996af Mon Sep 17 00:00:00 2001
From: Reagan Lee <“reaganjlee@gmail.com”>
Date: Sun, 8 Feb 2026 09:18:17 +0000
Subject: [PATCH 17/17] respond
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Reagan Lee <“reaganjlee@gmail.com”>
---
 tests/multimodal/test_processing.py | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/tests/multimodal/test_processing.py b/tests/multimodal/test_processing.py
index 2f6b4c53800e..2ab20fe2cf8b 100644
--- a/tests/multimodal/test_processing.py
+++ b/tests/multimodal/test_processing.py
@@ -984,13 +984,6 @@ def test_budget_caps_prevent_dummy_input_validation_failure(
 
     assert allowed["image"] == min(user_limit, supported_limit)
 
-    if allowed["image"] > 0:
-        MULTIMODAL_REGISTRY.get_dummy_mm_inputs(
-            model_config,
-            mm_counts={"image": allowed["image"]},
-            processor=processor,
-        )
-
 
 class DummyProcessor:
     def __init__(self, a: int = 0, b: int = 0) -> None: