fix(vllm): support unified multimodal routing

components/src/dynamo/common/multimodal/routing_utils.py

@@ -27,6 +27,17 @@ def pad_value_for_mm_hash(mm_hash: int) -> int:
     return MM_PAD_SHIFT_VALUE + (mm_hash & MM_PAD_HASH_MASK)
 
 
+def _is_embed_mask(mm_position: Any) -> list[bool] | None:
+    is_embed = getattr(mm_position, "is_embed", None)
+    if is_embed is None:
+        return None
+    if hasattr(is_embed, "detach"):
+        is_embed = is_embed.detach().cpu().tolist()
+    elif hasattr(is_embed, "tolist"):
+        is_embed = is_embed.tolist()
+    return [bool(value) for value in is_embed]
+
+
 def build_mm_routing_info_from_features(
     mm_features: list[Any],
     prompt_token_ids: list[int],
@@ -62,9 +73,16 @@ def build_mm_routing_info_from_features(
         pad = pad_value_for_mm_hash(mm_hash)
         start = f.mm_position.offset
         end = min(start + f.mm_position.length, n_tokens)
-        for i in range(start, end):
-            routing_token_ids[i] = pad
-        found = True
+        is_embed = _is_embed_mask(f.mm_position)
+        if is_embed is None:
+            for i in range(start, end):
+                routing_token_ids[i] = pad
+                found = True
+        else:
+            for rel_idx, use_pad in enumerate(is_embed[: end - start]):
+                if use_pad:
+                    routing_token_ids[start + rel_idx] = pad
+                    found = True
 
     if not found:
         return None

components/src/dynamo/common/tests/multimodal/test_routing_utils.py

@@ -34,12 +34,13 @@ def test_formula(self):
 class TestBuildMmRoutingInfoFromFeatures:
     """Tests for build_mm_routing_info_from_features (pad_value scheme)."""
 
-    def _make_feature(self, mm_hash, offset, length):
+    def _make_feature(self, mm_hash, offset, length, is_embed=None):
         feat = MagicMock()
         feat.mm_hash = mm_hash
         feat.mm_position = MagicMock()
         feat.mm_position.offset = offset
         feat.mm_position.length = length
+        feat.mm_position.is_embed = is_embed
         return feat
 
     def test_single_feature_substitutes_pad_value(self):
@@ -70,6 +71,25 @@ def test_two_features_each_get_own_pad_value(self):
         assert result["routing_token_ids"] == [p1, p1, 2, 3, p2, p2]
         assert result["block_mm_infos"] == []
 
+    def test_is_embed_mask_leaves_boundary_tokens_unchanged(self):
+        mm_hash = int("abcdef0123456789", 16)
+        features = [
+            self._make_feature(
+                "abcdef0123456789" + "0" * 48,
+                2,
+                4,
+                is_embed=[False, True, True, False],
+            )
+        ]
+        token_ids = list(range(8))
+
+        result = build_mm_routing_info_from_features(features, token_ids)
+
+        assert result is not None
+        pad = pad_value_for_mm_hash(mm_hash)
+        assert result["routing_token_ids"] == [0, 1, 2, pad, pad, 5, 6, 7]
+        assert result["block_mm_infos"] == []
+
     def test_no_features_returns_none(self):
         assert build_mm_routing_info_from_features([], [1, 2, 3]) is None
 

components/src/dynamo/frontend/tests/test_vllm_processor_unit.py

@@ -17,6 +17,13 @@
 
 from dynamo.frontend.prepost import _prepare_request
 
+# NOTE: dynamo.frontend.vllm_processor is imported lazily inside the tests that
+# need it (and via the vllm_processor_module fixture). Importing it at module
+# top level would run its `from vllm.tasks import ...` /
+# `from vllm.v1.engine.parallel_sampling import ...` imports during pytest
+# collection, which breaks the pytest-marker-report pre-commit hook (its vllm
+# stub list does not cover those submodules).
+
 # Needs vllm packages (gpu_1 container), but does not allocate GPU VRAM.
 pytestmark = [
     pytest.mark.unit,
@@ -29,6 +36,7 @@
 ]
 
 MODEL = "Qwen/Qwen3-0.6B"
+_DEFAULT_MM_DATA = object()
 
 TOOL_REQUEST = {
     "model": MODEL,
@@ -126,6 +134,171 @@ def test_no_tools_in_request(self, tokenizer):
         ), "No tools in request should produce None tools in template"
 
 
+class TestMultimodalFeatureMetadata:
+    def _feature(
+        self, modality, mm_hash, offset, length, data=_DEFAULT_MM_DATA, is_embed=None
+    ):
+        return SimpleNamespace(
+            modality=modality,
+            mm_hash=mm_hash,
+            data=data,
+            mm_position=SimpleNamespace(
+                offset=offset,
+                length=length,
+                is_embed=is_embed,
+            ),
+        )
+
+    def test_groups_hashes_and_placeholders_by_modality(self):
+        from dynamo.frontend.vllm_processor import _group_mm_feature_metadata
+
+        features = [
+            self._feature("image", "image_hash", 4, 8),
+            self._feature("audio", "audio_hash", 20, 6),
+        ]
+
+        (
+            flat_hashes,
+            flat_placeholders,
+            hashes_by_modality,
+            placeholders_by_modality,
+        ) = _group_mm_feature_metadata(features)
+
+        assert flat_hashes == []
+        assert flat_placeholders == []
+        assert hashes_by_modality == {
+            "image": ["image_hash"],
+            "audio": ["audio_hash"],
+        }
+        assert placeholders_by_modality == {
+            "image": [(4, 8)],
+            "audio": [(20, 6)],
+        }
+
+    def test_image_only_metadata_keeps_legacy_flat_fields(self):
+        from dynamo.frontend.vllm_processor import _group_mm_feature_metadata
+
+        features = [
+            self._feature("image", "image_hash_0", 4, 8),
+            self._feature("image", "image_hash_1", 20, 6),
+        ]
+
+        (
+            flat_hashes,
+            flat_placeholders,
+            hashes_by_modality,
+            placeholders_by_modality,
+        ) = _group_mm_feature_metadata(features)
+
+        assert flat_hashes == ["image_hash_0", "image_hash_1"]
+        assert flat_placeholders == [(4, 8), (20, 6)]
+        assert hashes_by_modality == {"image": ["image_hash_0", "image_hash_1"]}
+        assert placeholders_by_modality == {"image": [(4, 8), (20, 6)]}
+
+    def test_placeholder_metadata_preserves_is_embed_mask(self):
+        from dynamo.frontend.vllm_processor import _group_mm_feature_metadata
+
+        mask = [False, True, True, False]
+        features = [
+            self._feature("image", "image_hash", 4, 4, is_embed=mask),
+        ]
+
+        _, flat_placeholders, _, placeholders_by_modality = _group_mm_feature_metadata(
+            features
+        )
+
+        expected = {"offset": 4, "length": 4, "is_embed": mask}
+        assert flat_placeholders == [expected]
+        assert placeholders_by_modality == {"image": [expected]}
+
+    def test_missing_hash_skips_only_that_feature(self):
+        from dynamo.frontend.vllm_processor import _group_mm_feature_metadata
+
+        features = [
+            self._feature("image", "image_hash", 4, 8),
+            self._feature("image", None, 20, 8),
+        ]
+
+        assert _group_mm_feature_metadata(features) == (
+            ["image_hash"],
+            [(4, 8)],
+            {"image": ["image_hash"]},
+            {"image": [(4, 8)]},
+        )
+
+    def test_single_transfer_modality_rejects_mixed_features(self):
+        from dynamo.frontend.vllm_processor import _single_transfer_modality
+
+        assert (
+            _single_transfer_modality(
+                [
+                    self._feature("image", "image_hash", 4, 8),
+                    self._feature("audio", "audio_hash", 20, 6),
+                ]
+            )
+            is None
+        )
+        assert (
+            _single_transfer_modality(
+                [
+                    self._feature("image", "image_hash_0", 4, 8),
+                    self._feature("image", "image_hash_1", 20, 8),
+                ]
+            )
+            == "image"
+        )
+
+
+@pytest.mark.asyncio
+async def test_prepare_mm_routing_skips_single_modality_transfer_for_mixed_features(
+    vllm_processor_module,
+    monkeypatch,
+):
+    def fail_sender():
+        raise AssertionError("mixed-modality requests must not construct a sender")
+
+    monkeypatch.setattr(vllm_processor_module, "MmKwargsShmSender", fail_sender)
+
+    processor = vllm_processor_module.VllmProcessor.__new__(
+        vllm_processor_module.VllmProcessor
+    )
+    processor.block_size = 16
+    processor.nixl_mm_enabled = True
+    processor.use_shm_transfer = True
+    processor._sender = None
+
+    def feature(modality, mm_hash, offset, length):
+        return SimpleNamespace(
+            modality=modality,
+            mm_hash=mm_hash,
+            data=object(),
+            mm_position=SimpleNamespace(offset=offset, length=length),
+        )
+
+    vllm_preproc = SimpleNamespace(
+        prompt_token_ids=list(range(32)),
+        mm_features=[
+            feature("image", "a" * 64, 0, 16),
+            feature("audio", "b" * 64, 16, 8),
+        ],
+    )
+    dynamo_preproc = {}
+
+    mm_routing_info, cleanup_items, transferred = await processor._prepare_mm_routing(
+        vllm_preproc,
+        dynamo_preproc,
+    )
+
+    assert mm_routing_info is not None
+    assert cleanup_items == []
+    assert transferred is False
+    assert dynamo_preproc["extra_args"]["mm_hashes"] == []
+    assert dynamo_preproc["extra_args"]["mm_hashes_by_modality"] == {
+        "image": ["a" * 64],
+        "audio": ["b" * 64],
+    }
+
+
 class TestReasoningParserMetadata:
     def test_no_reasoning_parser_returns_none(self):
         from dynamo.frontend.vllm_processor import _build_reasoning_parser_metadata