fix(vllm): ship codec-free OpenCV for image inputs

components/src/dynamo/common/multimodal/codec_errors.py

@@ -94,6 +94,38 @@ def video_decoder_missing(
     )
 
 
+def mistral_image_decoder_missing(
+    backend: str, cause: str | None = None
+) -> MissingMediaDecoderError:
+    """Build the error for still-image input on the Mistral tokenizer path.
+
+    Not a video case: ``mistral_common`` resizes every still image with
+    ``cv2.resize`` inside ``transform_image``
+    (``mistral_common.tokens.tokenizers.image``) before normalization, so a
+    model served with ``--tokenizer-mode mistral`` needs OpenCV for plain JPEG
+    and PNG input. NVDEC decodes video, never a still image, so unlike video
+    there is no hardware alternative here.
+
+    Upstream's own message recommends ``pip install mistral-common[opencv]``.
+    That is the wrong remedy for these images -- it resolves an unbounded
+    OpenCV version and pulls transitive dependencies into a pinned stack --
+    so this message names the validated, bounded spec instead.
+    """
+    return MissingMediaDecoderError(
+        _with_cause(
+            "Cannot process image input: this model tokenizes images through "
+            "mistral_common (--tokenizer-mode mistral), which resizes them with "
+            "OpenCV ('cv2'). This image deliberately does not ship OpenCV, and "
+            "NVDEC decodes video only, so there is no hardware alternative for "
+            "still images. To enable image input, "
+            + _install_hint(backend, "opencv-python-headless")
+            + ". A model that also ships a Hugging Face processor can instead "
+            "be served with --tokenizer-mode auto, which needs no OpenCV.",
+            cause,
+        )
+    )
+
+
 def audio_decoder_missing(
     backend: str, cause: str | None = None
 ) -> MissingMediaDecoderError:

components/src/dynamo/common/tests/multimodal/test_codec_errors.py

@@ -9,6 +9,7 @@
 from dynamo.common.multimodal.codec_errors import (
     MissingMediaDecoderError,
     audio_decoder_missing,
+    mistral_image_decoder_missing,
     video_decoder_missing,
 )
 from dynamo.common.utils.install_media_decoders import VALIDATED_SPECS
@@ -77,6 +78,29 @@ def test_audio_message_has_no_hardware_alternative():
     assert "install_media_decoders vllm" in msg
 
 
+def test_mistral_image_message_names_cv2_spec_and_installer():
+    err = mistral_image_decoder_missing("vllm")
+
+    msg = str(err)
+    assert isinstance(err, MissingMediaDecoderError)
+    assert "cv2" in msg
+    assert VALIDATED_SPECS["opencv-python-headless"] in msg
+    assert "install_media_decoders vllm" in msg
+    # Still images, so the message must not send the reader looking for a
+    # codec or for the NVDEC capability that cannot help here.
+    assert "NVDEC decodes video only" in msg
+
+
+def test_mistral_image_message_does_not_repeat_upstream_remedy():
+    """mistral_common's own ImportError suggests `pip install
+    mistral-common[opencv]`: unbounded version, full dependency resolution
+    against the image's pinned stack. Ours must not echo it."""
+    msg = str(mistral_image_decoder_missing("vllm"))
+
+    assert "mistral-common[opencv]" not in msg
+    assert "pip install --no-deps" in msg
+
+
 def test_cause_text_is_appended_and_optional(monkeypatch):
     """The underlying decoder text must survive wraps whose handlers ship
     only str(exc) to the client; absent a cause, no dangling suffix."""
@@ -89,6 +113,8 @@ def test_cause_text_is_appended_and_optional(monkeypatch):
     assert "decoder reported" not in str(without)
     audio = audio_decoder_missing("vllm", cause="Please install vllm[audio]")
     assert "(decoder reported: Please install vllm[audio])" in str(audio)
+    image = mistral_image_decoder_missing("vllm", cause="`opencv` is not installed")
+    assert "(decoder reported: `opencv` is not installed)" in str(image)
 
 
 def test_error_is_not_a_value_error():

components/src/dynamo/common/utils/install_media_decoders.py

@@ -14,15 +14,21 @@
 
 This module covers what is left:
 
-* compressed audio (AAC and friends), which NVDEC does not decode at all, and
+* compressed audio (AAC and friends), which NVDEC does not decode at all,
 * H.264/H.265 on hosts where NVDEC is unavailable -- a GPU with no video decode
-  engine, or a container not granted the ``video`` driver capability.
+  engine, or a container not granted the ``video`` driver capability, and
+* still images on the Mistral tokenizer path. This one is not a codec gap:
+  ``mistral_common`` resizes every image with ``cv2``, so a model served with
+  ``--tokenizer-mode mistral`` needs OpenCV for plain JPEG/PNG input even
+  though no video is involved.
 
 Each backend decodes such input through a specific Python package whose wheel
 bundles its own FFmpeg, so the support can be added by a plain ``pip install``
 -- no image rebuild:
 
 * vLLM video input    -> OpenCV (``cv2``),    package ``opencv-python-headless``
+* vLLM image input    -> OpenCV (``cv2``),    package ``opencv-python-headless``
+  (``--tokenizer-mode mistral`` only)
 * vLLM audio input    -> PyAV (``av``),       package ``av``
 * SGLang video input  -> decord (``decord``), package ``decord2``
 * TRT-LLM video input -> OpenCV (``cv2``),    package ``opencv-python-headless``

components/src/dynamo/vllm/main.py

@@ -2,6 +2,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
 import asyncio
+import importlib
 import json
 import logging
 import os
@@ -17,15 +18,17 @@
 from huggingface_hub import try_to_load_from_cache
 from huggingface_hub.utils import HFValidationError
 from prometheus_client import REGISTRY, CollectorRegistry, multiprocess
-from vllm.config import VllmConfig
+from vllm.config import ModelConfig, VllmConfig
 from vllm.distributed.kv_events import ZmqEventPublisher
+from vllm.multimodal import MULTIMODAL_REGISTRY
 from vllm.usage.usage_lib import UsageContext
 from vllm.v1.engine.async_llm import AsyncLLM
 from vllm.v1.metrics.prometheus import setup_multiprocess_prometheus
 
 from dynamo.common.config_dump import dump_config
 from dynamo.common.configuration.groups.router_args import build_router_config
 from dynamo.common.model_fetch import fetch_model
+from dynamo.common.multimodal.codec_errors import mistral_image_decoder_missing
 from dynamo.common.snapshot.lifecycle import elect_and_wake
 from dynamo.common.snapshot.restore_context import (
     parse_snapshot_restore_runtime_config,
@@ -439,6 +442,78 @@ def _resolve_video_token_id(vllm_config: VllmConfig) -> Optional[int]:
     return None
 
 
+def _model_accepts_image_input(model_config: ModelConfig) -> bool:
+    """Whether this model will actually tokenize an image.
+
+    The configured limit alone does not answer this. ``get_limit_per_prompt``
+    is defined for any modality name, and an unspecified one defaults to 999,
+    so a non-zero image limit is reported for a model that supports no images
+    at all -- an audio-only Mistral model such as Voxtral among them. The
+    modality set comes from the multimodal registry instead, where omitting a
+    modality means the model does not support it.
+
+    The limit is still the first question, because ``--limit-mm-per-prompt
+    image=0`` takes a supported modality out of vLLM's multimodal profiling,
+    and such a worker runs today without OpenCV.
+    """
+    mm_config = getattr(model_config, "multimodal_config", None)
+    get_limit_per_prompt = getattr(mm_config, "get_limit_per_prompt", None)
+    if callable(get_limit_per_prompt) and get_limit_per_prompt("image") == 0:
+        return False
+    try:
+        info = MULTIMODAL_REGISTRY.get_processing_info(model_config)
+        return "image" in info.supported_mm_limits
+    except Exception:
+        # Unanswerable, so assume no image input. Being wrong that way costs a
+        # deployment the late upstream error this check exists to replace.
+        # Being wrong the other way stops a worker that would have served.
+        logger.debug(
+            "Could not determine the supported modalities of %s; "
+            "skipping the image-decoder preflight.",
+            getattr(model_config, "model", "the model"),
+            exc_info=True,
+        )
+        return False
+
+
+def check_mistral_image_decoder(vllm_config: VllmConfig) -> None:
+    """Fail fast when a Mistral-tokenizer multimodal model has no OpenCV.
+
+    ``mistral_common`` resizes every still image with ``cv2`` before
+    normalization, and the runtime images deliberately ship no OpenCV. Without
+    this check the gap surfaces from inside vLLM's multimodal profiling run,
+    after the weights are already loaded, as an upstream ``ImportError``
+    recommending ``pip install mistral-common[opencv]`` -- which is not the
+    supported install for these images.
+
+    Only ``--tokenizer-mode mistral`` routes through ``mistral_common``. The
+    Hugging Face processor path imports no OpenCV, so it must not trip this.
+
+    "Multimodal" is not "takes images", and neither is a non-zero image limit:
+    see ``_model_accepts_image_input``. A worker that will never tokenize an
+    image starts without OpenCV today, and this check must not be what stops
+    it.
+
+    Every config attribute is read defensively: a guard that runs on every
+    startup must never be the thing that breaks startup if vLLM moves them.
+    A move is caught loudly instead, by the contract tests alongside this check.
+    """
+    model_config = vllm_config.model_config
+    if not getattr(model_config, "is_multimodal_model", False):
+        return
+    if getattr(model_config, "tokenizer_mode", None) != "mistral":
+        return
+    if not _model_accepts_image_input(model_config):
+        return
+    # Import rather than probe for the spec. An OpenCV whose native libraries
+    # are absent is perfectly discoverable and still raises on import, and that
+    # deployment needs the same fast failure as one with no cv2 at all.
+    try:
+        importlib.import_module("cv2")
+    except ImportError as exc:
+        raise mistral_image_decoder_missing("vllm", str(exc)) from exc
+
+
 def setup_kv_event_publisher(
     config: Config,
     generate_endpoint: Endpoint,
@@ -671,6 +746,9 @@ def setup_vllm_engine(
     # Taken from build_async_engine_client_from_engine_args()
     usage_context = UsageContext.OPENAI_API_SERVER
     vllm_config = engine_args.create_engine_config(usage_context=usage_context)
+    # Before AsyncLLM: the same gap found later, during vLLM's multimodal
+    # profiling run, costs a full model load first and reports the wrong remedy.
+    check_mistral_image_decoder(vllm_config)
     disable_hybrid_kv_cache_manager_for_incompatible_pd_connector(vllm_config)
     default_sampling_params = vllm_config.model_config.get_diff_sampling_param()