fix(vllm): ship codec-free OpenCV for image inputs

components/src/dynamo/common/multimodal/codec_errors.py

@@ -94,6 +94,38 @@ def video_decoder_missing(
     )
 
 
+def mistral_image_decoder_missing(
+    backend: str, cause: str | None = None
+) -> MissingMediaDecoderError:
+    """Build the error for still-image input on the Mistral tokenizer path.
+
+    Not a video case: ``mistral_common`` resizes every still image with
+    ``cv2.resize`` inside ``transform_image``
+    (``mistral_common.tokens.tokenizers.image``) before normalization, so a
+    model served with ``--tokenizer-mode mistral`` needs OpenCV for plain JPEG
+    and PNG input. NVDEC decodes video, never a still image, so unlike video
+    there is no hardware alternative here.
+
+    Upstream's own message recommends ``pip install mistral-common[opencv]``.
+    That is the wrong remedy for these images -- it resolves an unbounded
+    OpenCV version and pulls transitive dependencies into a pinned stack --
+    so this message names the validated, bounded spec instead.
+    """
+    return MissingMediaDecoderError(
+        _with_cause(
+            "Cannot process image input: this model tokenizes images through "
+            "mistral_common (--tokenizer-mode mistral), which resizes them with "
+            "OpenCV ('cv2'). This image deliberately does not ship OpenCV, and "
+            "NVDEC decodes video only, so there is no hardware alternative for "
+            "still images. To enable image input, "
+            + _install_hint(backend, "opencv-python-headless")
+            + ". A model that also ships a Hugging Face processor can instead "
+            "be served with --tokenizer-mode auto, which needs no OpenCV.",
+            cause,
+        )
+    )
+
+
 def audio_decoder_missing(
     backend: str, cause: str | None = None
 ) -> MissingMediaDecoderError:

components/src/dynamo/common/tests/multimodal/test_codec_errors.py

@@ -9,6 +9,7 @@
 from dynamo.common.multimodal.codec_errors import (
     MissingMediaDecoderError,
     audio_decoder_missing,
+    mistral_image_decoder_missing,
     video_decoder_missing,
 )
 from dynamo.common.utils.install_media_decoders import VALIDATED_SPECS
@@ -77,6 +78,29 @@ def test_audio_message_has_no_hardware_alternative():
     assert "install_media_decoders vllm" in msg
 
 
+def test_mistral_image_message_names_cv2_spec_and_installer():
+    err = mistral_image_decoder_missing("vllm")
+
+    msg = str(err)
+    assert isinstance(err, MissingMediaDecoderError)
+    assert "cv2" in msg
+    assert VALIDATED_SPECS["opencv-python-headless"] in msg
+    assert "install_media_decoders vllm" in msg
+    # Still images, so the message must not send the reader looking for a
+    # codec or for the NVDEC capability that cannot help here.
+    assert "NVDEC decodes video only" in msg
+
+
+def test_mistral_image_message_does_not_repeat_upstream_remedy():
+    """mistral_common's own ImportError suggests `pip install
+    mistral-common[opencv]`: unbounded version, full dependency resolution
+    against the image's pinned stack. Ours must not echo it."""
+    msg = str(mistral_image_decoder_missing("vllm"))
+
+    assert "mistral-common[opencv]" not in msg
+    assert "pip install --no-deps" in msg
+
+
 def test_cause_text_is_appended_and_optional(monkeypatch):
     """The underlying decoder text must survive wraps whose handlers ship
     only str(exc) to the client; absent a cause, no dangling suffix."""
@@ -89,6 +113,8 @@ def test_cause_text_is_appended_and_optional(monkeypatch):
     assert "decoder reported" not in str(without)
     audio = audio_decoder_missing("vllm", cause="Please install vllm[audio]")
     assert "(decoder reported: Please install vllm[audio])" in str(audio)
+    image = mistral_image_decoder_missing("vllm", cause="`opencv` is not installed")
+    assert "(decoder reported: `opencv` is not installed)" in str(image)
 
 
 def test_error_is_not_a_value_error():

components/src/dynamo/common/utils/install_media_decoders.py

@@ -14,15 +14,21 @@
 
 This module covers what is left:
 
-* compressed audio (AAC and friends), which NVDEC does not decode at all, and
+* compressed audio (AAC and friends), which NVDEC does not decode at all,
 * H.264/H.265 on hosts where NVDEC is unavailable -- a GPU with no video decode
-  engine, or a container not granted the ``video`` driver capability.
+  engine, or a container not granted the ``video`` driver capability, and
+* still images on the Mistral tokenizer path. This one is not a codec gap:
+  ``mistral_common`` resizes every image with ``cv2``, so a model served with
+  ``--tokenizer-mode mistral`` needs OpenCV for plain JPEG/PNG input even
+  though no video is involved.
 
 Each backend decodes such input through a specific Python package whose wheel
 bundles its own FFmpeg, so the support can be added by a plain ``pip install``
 -- no image rebuild:
 
 * vLLM video input    -> OpenCV (``cv2``),    package ``opencv-python-headless``
+* vLLM image input    -> OpenCV (``cv2``),    package ``opencv-python-headless``
+  (``--tokenizer-mode mistral`` only)
 * vLLM audio input    -> PyAV (``av``),       package ``av``
 * SGLang video input  -> decord (``decord``), package ``decord2``
 * TRT-LLM video input -> OpenCV (``cv2``),    package ``opencv-python-headless``

components/src/dynamo/vllm/main.py

@@ -2,6 +2,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
 import asyncio
+import importlib.util
 import json
 import logging
 import os
@@ -26,6 +27,7 @@
 from dynamo.common.config_dump import dump_config
 from dynamo.common.configuration.groups.router_args import build_router_config
 from dynamo.common.model_fetch import fetch_model
+from dynamo.common.multimodal.codec_errors import mistral_image_decoder_missing
 from dynamo.common.snapshot.lifecycle import elect_and_wake
 from dynamo.common.snapshot.restore_context import (
     parse_snapshot_restore_runtime_config,
@@ -439,6 +441,33 @@ def _resolve_video_token_id(vllm_config: VllmConfig) -> Optional[int]:
     return None
 
 
+def check_mistral_image_decoder(vllm_config: VllmConfig) -> None:
+    """Fail fast when a Mistral-tokenizer multimodal model has no OpenCV.
+
+    ``mistral_common`` resizes every still image with ``cv2`` before
+    normalization, and the runtime images deliberately ship no OpenCV. Without
+    this check the gap surfaces from inside vLLM's multimodal profiling run,
+    after the weights are already loaded, as an upstream ``ImportError``
+    recommending ``pip install mistral-common[opencv]`` -- which is not the
+    supported install for these images.
+
+    Only ``--tokenizer-mode mistral`` routes through ``mistral_common``. The
+    Hugging Face processor path imports no OpenCV, so it must not trip this.
+
+    Both config attributes are read defensively: a guard that runs on every
+    startup must never be the thing that breaks startup if vLLM moves them.
+    A move is caught loudly instead, by the contract test alongside this check.
+    """
+    model_config = vllm_config.model_config
+    if not getattr(model_config, "is_multimodal_model", False):
+        return
+    if getattr(model_config, "tokenizer_mode", None) != "mistral":
+        return
+    if importlib.util.find_spec("cv2") is not None:
+        return
+    raise mistral_image_decoder_missing("vllm")
+
+
 def setup_kv_event_publisher(
     config: Config,
     generate_endpoint: Endpoint,
@@ -671,6 +700,9 @@ def setup_vllm_engine(
     # Taken from build_async_engine_client_from_engine_args()
     usage_context = UsageContext.OPENAI_API_SERVER
     vllm_config = engine_args.create_engine_config(usage_context=usage_context)
+    # Before AsyncLLM: the same gap found later, during vLLM's multimodal
+    # profiling run, costs a full model load first and reports the wrong remedy.
+    check_mistral_image_decoder(vllm_config)
     disable_hybrid_kv_cache_manager_for_incompatible_pd_connector(vllm_config)
     default_sampling_params = vllm_config.model_config.get_diff_sampling_param()
 

components/src/dynamo/vllm/tests/test_vllm_mistral_image_preflight.py

@@ -0,0 +1,177 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Startup preflight for image input on the Mistral tokenizer path.
+
+The runtime images deliberately ship no OpenCV, but ``mistral_common`` resizes
+every still image with ``cv2``. Left alone, that gap surfaces from inside
+vLLM's multimodal profiling run, after the weights have loaded, as an upstream
+``ImportError`` pointing at an install these images do not support.
+"""
+
+import dataclasses
+import importlib
+import importlib.machinery
+import importlib.util
+import sys
+from types import ModuleType, SimpleNamespace
+
+import pytest
+
+from dynamo.common.multimodal.codec_errors import MissingMediaDecoderError
+from dynamo.common.utils.install_media_decoders import VALIDATED_SPECS
+
+pytestmark = [
+    pytest.mark.unit,
+    pytest.mark.vllm,
+    pytest.mark.core,
+    pytest.mark.gpu_0,
+    pytest.mark.xpu_1,
+    pytest.mark.profiled_vram_gib(0),
+    pytest.mark.timeout(180),
+    pytest.mark.pre_merge,
+]
+
+
+def _load_vllm_main() -> ModuleType:
+    """Load the entrypoint lazily, as the other vLLM unit tests do.
+
+    ``dynamo.vllm.main`` imports uvloop at module scope, which the lightweight
+    pre-commit collection environment omits.
+    """
+    return importlib.import_module("dynamo.vllm.main")
+
+
+def _vllm_config(*, multimodal: bool, tokenizer_mode: str) -> SimpleNamespace:
+    return SimpleNamespace(
+        model_config=SimpleNamespace(
+            is_multimodal_model=multimodal, tokenizer_mode=tokenizer_mode
+        )
+    )
+
+
+def _pin_cv2(monkeypatch, *, present: bool) -> None:
+    """Pin what ``find_spec("cv2")`` reports, in both directions.
+
+    ``importlib.util.find_spec`` consults ``sys.modules`` first: a ``None``
+    entry makes it return ``None``, and a module carrying a spec makes it
+    return that spec. Pinning both directions keeps the test meaningful on a
+    host that has opted into the documented install -- otherwise the failing
+    branch would never run there.
+    """
+    if present:
+        stub = ModuleType("cv2")
+        stub.__spec__ = importlib.machinery.ModuleSpec("cv2", loader=None)
+        monkeypatch.setitem(sys.modules, "cv2", stub)
+    else:
+        monkeypatch.setitem(sys.modules, "cv2", None)
+
+
+def test_mistral_multimodal_without_cv2_fails_before_the_engine_starts(monkeypatch):
+    vllm_main = _load_vllm_main()
+    _pin_cv2(monkeypatch, present=False)
+
+    with pytest.raises(MissingMediaDecoderError) as excinfo:
+        vllm_main.check_mistral_image_decoder(
+            _vllm_config(multimodal=True, tokenizer_mode="mistral")
+        )
+
+    msg = str(excinfo.value)
+    assert "cv2" in msg
+    # The bounded spec and the installer command are what make the failure
+    # actionable; upstream's own suggestion is the wrong one for these images.
+    assert VALIDATED_SPECS["opencv-python-headless"] in msg
+    assert "install_media_decoders vllm" in msg
+    assert "mistral-common[opencv]" not in msg
+
+
+@pytest.mark.parametrize(
+    "multimodal, tokenizer_mode",
+    [
+        # Text-only Mistral models tokenize no images.
+        (False, "mistral"),
+        # The Hugging Face processor path imports no OpenCV, so a multimodal
+        # model on it must still start without the install.
+        (True, "auto"),
+        (False, "auto"),
+    ],
+)
+def test_preflight_stays_silent_off_the_mistral_image_path(
+    monkeypatch, multimodal, tokenizer_mode
+):
+    vllm_main = _load_vllm_main()
+    _pin_cv2(monkeypatch, present=False)
+
+    assert (
+        vllm_main.check_mistral_image_decoder(
+            _vllm_config(multimodal=multimodal, tokenizer_mode=tokenizer_mode)
+        )
+        is None
+    )
+
+
+def test_preflight_passes_once_the_decoder_is_installed(monkeypatch):
+    """The documented remedy has to clear the check, or a worker could never
+    start after running it."""
+    vllm_main = _load_vllm_main()
+    _pin_cv2(monkeypatch, present=True)
+
+    assert (
+        vllm_main.check_mistral_image_decoder(
+            _vllm_config(multimodal=True, tokenizer_mode="mistral")
+        )
+        is None
+    )
+
+
+def test_preflight_tolerates_a_config_without_the_attributes(monkeypatch):
+    """A guard on the startup path must not itself break startup.
+
+    The preflight reads both attributes defensively, so a config object that
+    predates them -- or a vLLM release that moves them -- degrades to "do not
+    check" rather than raising AttributeError on every worker.
+    """
+    vllm_main = _load_vllm_main()
+    _pin_cv2(monkeypatch, present=False)
+
+    assert (
+        vllm_main.check_mistral_image_decoder(
+            SimpleNamespace(model_config=SimpleNamespace())
+        )
+        is None
+    )
+
+
+def test_vllm_model_config_still_exposes_the_attributes_the_preflight_reads():
+    """The loud half of that defensiveness.
+
+    Reading through ``getattr`` means a rename upstream would silently disable
+    the check. Pin the two names here so the rename fails CI instead.
+    """
+    model_config_cls = pytest.importorskip("vllm.config").ModelConfig
+
+    assert hasattr(model_config_cls, "is_multimodal_model")
+    assert "tokenizer_mode" in {
+        field.name for field in dataclasses.fields(model_config_cls)
+    }
+
+
+@pytest.mark.skipif(
+    importlib.util.find_spec("cv2") is not None,
+    reason="cv2 is installed here, so mistral_common's own guard cannot fire",
+)
+def test_upstream_still_image_encode_needs_cv2():
+    """Ground the preflight against upstream rather than against a mock.
+
+    ``mistral_common.tokens.tokenizers.image.transform_image`` is the single
+    site that asserts OpenCV is present, and it is reached for a plain still
+    image with no video anywhere. If upstream ever stops needing OpenCV here,
+    this test fails and the preflight can go.
+    """
+    pil_image = pytest.importorskip("PIL.Image")
+    image_tokenizer = pytest.importorskip("mistral_common.tokens.tokenizers.image")
+
+    with pytest.raises(ImportError) as excinfo:
+        image_tokenizer.transform_image(pil_image.new("RGB", (64, 64), "red"), (64, 64))
+
+    assert "opencv" in str(excinfo.value).lower()

docs/fern/pages/use-cases/multimodal-serving/additional-media-decoders.md

@@ -16,6 +16,7 @@ Installing an additional decoder package covers what remains:
 
 - **AAC and other compressed audio**, which NVDEC does not decode at all.
 - **H.264 and H.265 on hosts where NVDEC is unavailable** — no video decode engine on the GPU, or a container without the `video` capability.
+- **Still images on Mistral-tokenizer models**, described in [Still images on the Mistral tokenizer path](#still-images-on-the-mistral-tokenizer-path) below. This one is not about video at all.
 
 Each backend decodes such input through a specific Python package whose wheel bundles its own FFmpeg, so the support is added with a plain `pip install` — no image rebuild. **Nothing installs automatically.** There is no environment switch and no startup hook: an operator runs the install as a deliberate, visible step, so a deployment that broadens the image's codec surface says so in its Dockerfile, pod spec, or runbook.
 
@@ -27,12 +28,37 @@ Each backend decodes such input through a specific Python package whose wheel bu
 | Backend | Input | Package (validated version bounds) | Import |
 |---------|-------|------------------------------------|--------|
 | vLLM | video | `opencv-python-headless>=4.13.0.92,<5` | `cv2` |
+| vLLM | image, with `--tokenizer-mode mistral` | `opencv-python-headless>=4.13.0.92,<5` | `cv2` |
 | vLLM | audio | `av>=18.0.0,<19` | `av` |
 | SGLang | video | `decord2>=3.4.0,<4` | `decord` |
 | TensorRT-LLM | video | `opencv-python-headless>=4.13.0.92,<5` | `cv2` |
 
 The lower bound of each spec is the version validated against Dynamo's multimodal test suite; the upper bound excludes the next major release so an install cannot silently pick up an unvalidated version. PyNvVideoCodec is not in this list because the images already ship it — it is the NVDEC path, not a fallback, so there is nothing to install. torchcodec is left out because no Dynamo decode path imports it.
 
+## Still images on the Mistral tokenizer path
+
+Serving a multimodal Mistral-family model with `--tokenizer-mode mistral` tokenizes images through `mistral_common`, which resizes every image with OpenCV before normalizing it. That makes `opencv-python-headless` a requirement for plain JPEG and PNG input on this path — no video is involved, and NVDEC never decodes a still image, so there is no hardware alternative here.
+
+Without the install, the vLLM worker refuses to start and names the package:
+
+```text
+Cannot process image input: this model tokenizes images through mistral_common
+(--tokenizer-mode mistral), which resizes them with OpenCV ('cv2'). ...
+```
+
+Install the same validated spec as the vLLM rows above, with either command:
+
+```bash
+pip install --no-deps 'opencv-python-headless>=4.13.0.92,<5'
+# or
+python -m dynamo.common.utils.install_media_decoders vllm
+```
+
+> [!NOTE]
+> Upstream `mistral_common` raises ``ImportError: `opencv` is not installed`` and suggests `pip install mistral-common[opencv]`. Do not use that command here: it resolves an unbounded OpenCV version and pulls transitive dependencies into the image's pinned stack. The bounded, `--no-deps` install above is the supported one.
+
+The requirement follows the tokenizer mode, not the model. A model that also ships a Hugging Face processor can be served with `--tokenizer-mode auto` instead, which imports no OpenCV.
+
 ## Install with pip
 
 The table above is the contract; these commands are its direct translation, and work with the installer of your choice (`pip`, `uv pip`, ...):