Add embedding input functionality for disabled modalities

tests/models/utils.py

@@ -278,6 +278,7 @@ def build_model_context(
     mm_processor_kwargs: dict[str, Any] | None = None,
     limit_mm_per_prompt: dict[str, int] | None = None,
     mm_processor_cache_gb: int = 0,
+    enable_mm_embeds: bool | None = None,
 ):
     """Creates an InputProcessingContext for a given model.
 
@@ -286,6 +287,8 @@ def build_model_context(
         mm_processor_kwargs: optional processor kwargs for to be leveraged
             in the input processor, mapper, dummy data creation, etc.
         limit_mm_per_prompt: Multimodal limits.
+        enable_mm_embeds: Whether to enable multimodal embedding processing.
+            If None, defaults to model_info.require_embed_inputs.
 
     Returns:
         InputProcessingContext for the model being considered.
@@ -296,6 +299,11 @@ def build_model_context(
 
     model_config_kwargs = model_config_kwargs or {}
     limit_mm_per_prompt = limit_mm_per_prompt or {}
+    enable_mm_embeds = (
+        model_info.require_embed_inputs
+        if enable_mm_embeds is None
+        else enable_mm_embeds
+    )
     model_config = ModelConfig(
         model_id,
         runner=runner,
@@ -311,7 +319,7 @@ def build_model_context(
         hf_overrides=model_info.hf_overrides,
         skip_tokenizer_init=model_info.require_embed_inputs,
         enable_prompt_embeds=model_info.require_embed_inputs,
-        enable_mm_embeds=model_info.require_embed_inputs,
+        enable_mm_embeds=enable_mm_embeds,
         enforce_eager=model_info.enforce_eager,
         **model_config_kwargs,
     )

tests/multimodal/test_registry.py

@@ -15,20 +15,58 @@
 
 
 @pytest.mark.parametrize(
-    "model_id,limit_mm_per_prompt,expected",
+    "model_id,limit_mm_per_prompt,enable_mm_embeds,expected",
     [
-        ("Qwen/Qwen2-0.5B-Instruct", {}, False),
-        ("Qwen/Qwen2.5-VL-3B-Instruct", {}, True),
-        ("Qwen/Qwen2.5-VL-3B-Instruct", {"image": 0, "video": 0}, False),
-        ("Qwen/Qwen2.5-VL-3B-Instruct", {"image": 0}, True),
+        ("Qwen/Qwen2-0.5B-Instruct", {}, False, False),
+        ("Qwen/Qwen2-0.5B-Instruct", {}, True, False),
+        ("Qwen/Qwen2.5-VL-3B-Instruct", {}, False, True),
+        ("Qwen/Qwen2.5-VL-3B-Instruct", {}, True, True),
+        ("Qwen/Qwen2.5-VL-3B-Instruct", {"image": 0, "video": 0}, False, False),
+        ("Qwen/Qwen2.5-VL-3B-Instruct", {"image": 0, "video": 0}, True, True),
+        ("Qwen/Qwen2.5-VL-3B-Instruct", {"image": 0}, True, True),
+        ("Qwen/Qwen2.5-VL-3B-Instruct", {"image": 0}, False, True),
+        ("Qwen/Qwen2.5-VL-3B-Instruct", {"image": 1, "video": 1}, True, True),
+        ("Qwen/Qwen2.5-VL-3B-Instruct", {"image": 1, "video": 1}, False, True),
     ],
 )
 @pytest.mark.core_model
-def test_supports_multimodal_inputs(model_id, limit_mm_per_prompt, expected):
+def test_supports_multimodal_inputs(
+    model_id, limit_mm_per_prompt, enable_mm_embeds, expected
+):
     """Test supports_multimodal_inputs returns correct boolean for various
-    configs."""
+    configs.
+
+    This test verifies that when enable_mm_embeds=True, the multimodal
+    processing pipeline is enabled even when all modality limits are 0,
+    allowing pre-computed embeddings to be processed while skipping
+    encoder weight loading for memory optimization.
+    """
     ctx = build_model_context(
         model_id,
         limit_mm_per_prompt=limit_mm_per_prompt,
+        enable_mm_embeds=enable_mm_embeds,
     )
     assert MULTIMODAL_REGISTRY.supports_multimodal_inputs(ctx.model_config) is expected
+
+
+@pytest.mark.core_model
+def test_encoder_weight_skipping_with_enable_mm_embeds():
+    """Integration test verifying encoder weights are not loaded when all
+    modality limits are 0, even with enable_mm_embeds=True."""
+    from vllm.v1.core.encoder_cache_manager import compute_encoder_cache_budget
+
+    ctx = build_model_context(
+        "Qwen/Qwen2.5-VL-3B-Instruct",
+        limit_mm_per_prompt={"image": 0, "video": 0},
+        enable_mm_embeds=True,
+    )
+
+    assert MULTIMODAL_REGISTRY.supports_multimodal_inputs(ctx.model_config) is True
+
+    compute_budget, cache_size = compute_encoder_cache_budget(
+        ctx.model_config, ctx.scheduler_config, MULTIMODAL_REGISTRY
+    )
+    assert compute_budget == 1, (
+        "Compute budget should be minimal (1) for embeddings-only mode"
+    )
+    assert cache_size == 1, "Cache size should be minimal (1) for embeddings-only mode"

vllm/multimodal/processing.py

@@ -1384,6 +1384,12 @@ def _to_mm_items(
                     )
 
         for modality, items in mm_items.items():
+            # When enable_mm_embeds=True, allow embedding inputs even if
+            # modality limits are zero (for memory optimization)
+            if mm_config.enable_mm_embeds and isinstance(
+                items, (EmbeddingItems, DictEmbeddingItems)
+            ):
+                continue
             self.validate_num_items(modality, len(items))
 
         return mm_items

vllm/multimodal/registry.py

@@ -118,17 +118,20 @@ def supports_multimodal_inputs(self, model_config: "ModelConfig") -> bool:
         """
         Checks if the model supports multimodal inputs.
         Returns True if the model is multimodal with any non-zero supported
-        modalities, otherwise returns False, effectively running in
-        text-only mode.
+        modalities OR if enable_mm_embeds is True (for pre-computed embeddings),
+        otherwise returns False, effectively running in text-only mode.
         """
         if not model_config.is_multimodal_model:
             return False
 
+        mm_config = model_config.get_multimodal_config()
+
+        if mm_config.enable_mm_embeds:
+            return True
+
         info = self._create_processing_info(model_config, tokenizer=None)
         supported_modalities = info.get_supported_mm_limits()
 
-        mm_config = model_config.get_multimodal_config()
-
         # Check if all supported modalities have limit == 0
         if all(
             mm_config.get_limit_per_prompt(modality) == 0

vllm/v1/core/encoder_cache_manager.py

@@ -277,6 +277,7 @@ def compute_encoder_budget(
         return compute_mm_encoder_budget(
             scheduler_config,
             max_tokens_by_modality,
+            model_config,
         )
 
     return compute_text_encoder_budget(scheduler_config)
@@ -302,6 +303,7 @@ def compute_text_encoder_budget(scheduler_config: "SchedulerConfig") -> tuple[in
 def compute_mm_encoder_budget(
     scheduler_config: "SchedulerConfig",
     max_tokens_by_modality: Mapping[str, int],
+    model_config: "ModelConfig | None" = None,
 ) -> tuple[int, int]:
     """Compute the encoder cache budget based on the model and scheduler
     configurations for a multimodal model.
@@ -310,6 +312,7 @@ def compute_mm_encoder_budget(
         scheduler_config: Scheduler configuration.
         max_tokens_by_modality: The maximum number of tokens for each
             non-text modality.
+        model_config: Model configuration. Used to check enable_mm_embeds flag.
 
     Returns:
         - Compute budget for encoder execution, measured in number of tokens
@@ -319,6 +322,20 @@ def compute_mm_encoder_budget(
     """
 
     if not max_tokens_by_modality:
+        # Check if enable_mm_embeds is True - if so, return minimal cache size
+        # to allow None value caching for memory optimization
+        if (
+            model_config is not None
+            and model_config.multimodal_config is not None
+            and model_config.multimodal_config.enable_mm_embeds
+        ):
+            logger.info(
+                "All non-text modalities are disabled but enable_mm_embeds=True. "
+                "Initializing encoder cache with minimal size to support "
+                "embedding inputs."
+            )
+            return 1, 1
+
         logger.warning(
             "All non-text modalities supported by the model have been "
             "explicitly disabled via limit_mm_per_prompt. Encoder cache will "

vllm/v1/worker/gpu_model_runner.py

@@ -2267,7 +2267,25 @@ def _gather_mm_embeddings(
 
                 mm_hash = mm_feature.identifier
                 encoder_output = self.encoder_cache.get(mm_hash, None)
-                assert encoder_output is not None, f"Encoder cache miss for {mm_hash}."
+
+                if encoder_output is None:
+                    mm_config = self.model_config.multimodal_config
+                    if (
+                        mm_config is not None
+                        and mm_config.enable_mm_embeds
+                        and "dummy_none" in self.encoder_cache
+                        and mm_hash not in self.encoder_cache
+                    ):
+                        # enable_mm_embeds is True and we have the dummy cache entry.
+                        # This edge case can occur when embeddings are provided
+                        # directly without going through the normal encoder caching
+                        # path. Skip this entry as the embeddings may have been
+                        # provided directly.
+                        continue
+                    else:
+                        assert encoder_output is not None, (
+                            f"Encoder cache miss for {mm_hash}."
+                        )
 
                 if (is_embed := pos_info.is_embed) is not None:
                     is_embed = is_embed[start_idx:end_idx]
@@ -4506,7 +4524,22 @@ def profile_run(self) -> None:
                 mm_budget = self.mm_budget
                 assert mm_budget is not None
 
-                if (encoder_budget := mm_budget.get_encoder_budget()) > 0:
+                encoder_budget = mm_budget.get_encoder_budget()
+                # Check if this is a minimal budget case (encoder_budget == 1)
+                # which occurs when all modalities are zero but enable_mm_embeds=True
+                if (
+                    encoder_budget == 1
+                    and mm_config is not None
+                    and mm_config.enable_mm_embeds
+                    and not mm_budget.max_tokens_by_modality
+                ):
+                    logger.info(
+                        "All modalities are disabled but enable_mm_embeds=True. "
+                        "Initializing encoder cache with None values for memory "
+                        "optimization."
+                    )
+                    self.encoder_cache["dummy_none"] = None
+                elif encoder_budget > 0:
                     # NOTE: Currently model is profiled with a single non-text
                     # modality with the max possible input tokens even when
                     # it supports multiple.

vllm/v1/worker/tpu_model_runner.py

@@ -1052,7 +1052,24 @@ def _gather_mm_embeddings(
 
                 mm_hash = mm_feature.identifier
                 encoder_output = self.encoder_cache.get(mm_hash, None)
-                assert encoder_output is not None, f"Encoder cache miss for {mm_hash}."
+                if encoder_output is None:
+                    mm_config = self.model_config.multimodal_config
+                    if (
+                        mm_config is not None
+                        and mm_config.enable_mm_embeds
+                        and "dummy_none" in self.encoder_cache
+                        and mm_hash not in self.encoder_cache
+                    ):
+                        # enable_mm_embeds is True and we have the dummy cache entry.
+                        # This edge case can occur when embeddings are provided
+                        # directly without going through the normal encoder caching
+                        # path. Skip this entry as the embeddings may have been
+                        # provided directly.
+                        continue
+                    else:
+                        assert encoder_output is not None, (
+                            f"Encoder cache miss for {mm_hash}."
+                        )
 
                 assert pos_info.is_embed is None, (
                     "Expected all positions to be contiguous and embeddings."
@@ -1726,7 +1743,22 @@ def profile_run(
                 assert mm_budget is not None
 
                 # TODO: handle encoder-decoder models once we support them.
-                if (encoder_budget := mm_budget.get_encoder_budget()) > 0:
+                encoder_budget = mm_budget.get_encoder_budget()
+                # Check if this is a minimal budget case (encoder_budget == 1)
+                # which occurs when all modalities are zero but enable_mm_embeds=True
+                if (
+                    encoder_budget == 1
+                    and mm_config is not None
+                    and mm_config.enable_mm_embeds
+                    and not mm_budget.max_tokens_by_modality
+                ):
+                    logger.info(
+                        "All modalities are disabled but enable_mm_embeds=True. "
+                        "Initializing encoder cache with None values for memory "
+                        "optimization."
+                    )
+                    self.encoder_cache["dummy_none"] = None
+                elif encoder_budget > 0:
                     # NOTE: Currently model is profiled with a single non-text
                     # modality with the max possible input tokens even when
                     # it supports multiple.

vllm/v1/worker/utils.py

@@ -54,6 +54,7 @@ def __init__(
         encoder_compute_budget, encoder_cache_size = compute_mm_encoder_budget(
             scheduler_config,
             max_tokens_by_modality,
+            model_config,
         )
 
         self.encoder_compute_budget = encoder_compute_budget