fix(sglang): reject media for text-only engines

components/src/dynamo/sglang/request_handlers/llm/decode_handler.py

@@ -47,9 +47,11 @@
     IMAGE_URL_KEY,
     VIDEO_URL_KEY,
     build_disagg_mm_kwargs,
+    engine_consumes_media,
     extract_media_urls,
     extract_mm_hashes,
     raise_if_unextracted_multimodal,
+    reject_unconsumed_media,
 )
 from dynamo.sglang.request_utils import request_cache_salt
 from dynamo.sglang.thinking_budget import (
@@ -709,6 +711,9 @@ async def generate(
 
         if self.serving_mode == DisaggregationMode.DECODE:
             raise_if_unextracted_multimodal(request)
+            reject_unconsumed_media(
+                request, consumes_media=engine_consumes_media(self.engine)
+            )
 
             # Check if bootstrap_info is pre-computed in the request (from frontend)
             bootstrap_info = request.get("bootstrap_info")
@@ -784,6 +789,9 @@ async def generate(
                     yield out
         else:
             raise_if_unextracted_multimodal(request)
+            reject_unconsumed_media(
+                request, consumes_media=engine_consumes_media(self.engine)
+            )
 
             # Extract media URLs for multimodal requests. SGLang's mm_data_processor
             # handles loading/preprocessing, and the scheduler does vision encoding.

components/src/dynamo/sglang/request_handlers/llm/diffusion_handler.py

@@ -11,6 +11,7 @@
 from dynamo.sglang.args import Config
 from dynamo.sglang.publisher import DynamoSglangPublisher
 from dynamo.sglang.request_handlers.llm.decode_handler import DecodeWorkerHandler
+from dynamo.sglang.request_handlers.llm.mm_disagg_utils import reject_unconsumed_media
 from dynamo.sglang.thinking_budget import thinking_budget_requested
 
 
@@ -76,6 +77,9 @@ async def generate(
             f"input_tokens={len(request.get('token_ids', []))}"
         )
 
+        # The diffusion path never forwards media to the engine.
+        reject_unconsumed_media(request, consumes_media=False)
+
         # Get input parameters (tokens or text)
         input_param = self._get_input_param(request)
 

components/src/dynamo/sglang/request_handlers/llm/mm_disagg_utils.py

@@ -90,6 +90,35 @@ def raise_if_unextracted_multimodal(request: Dict[str, Any]) -> None:
     raise InvalidArgument(message)
 
 
+def engine_consumes_media(engine: Any) -> bool:
+    """Return False only when SGLang reports a text-only engine."""
+    tokenizer_manager = getattr(engine, "tokenizer_manager", None)
+    model_config = getattr(tokenizer_manager, "model_config", None)
+    return getattr(model_config, "is_multimodal", None) is not False
+
+
+def reject_unconsumed_media(request: Dict[str, Any], *, consumes_media: bool) -> None:
+    """Reject media the engine would drop instead of answering without it.
+
+    SGLang's Engine API skips the HTTP server's text-only media check and
+    silently discards media when the model is not multimodal.
+    """
+    if consumes_media:
+        return
+    content_types = {
+        content_type
+        for content_type, items in _multi_modal_data(request).items()
+        if content_type in _SUPPORTED_MULTIMODAL_CONTENT_TYPES and items
+    } | _raw_multimodal_content_types(request)
+    if not content_types:
+        return
+    content_type = min(content_types)
+    raise InvalidArgument(
+        f"Model does not accept {content_type.removesuffix('_url')} input; "
+        f"received unsupported content type '{content_type}'."
+    )
+
+
 def extract_media_urls(
     mm_data: Optional[Dict[str, Any]], media_key: str
 ) -> list[str] | None:

components/src/dynamo/sglang/request_handlers/llm/prefill_handler.py

@@ -31,7 +31,9 @@
 )
 from dynamo.sglang.request_handlers.llm.mm_disagg_utils import (
     build_disagg_mm_kwargs,
+    engine_consumes_media,
     raise_if_unextracted_multimodal,
+    reject_unconsumed_media,
 )
 from dynamo.sglang.request_utils import request_cache_salt
 from dynamo.sglang.thinking_budget import (
@@ -176,6 +178,9 @@ async def generate(
         # Prefill encodes the media so the KV it transfers carries the vision
         # context; decode extracts the same URLs to match the token layout.
         raise_if_unextracted_multimodal(inner_request)
+        reject_unconsumed_media(
+            inner_request, consumes_media=engine_consumes_media(self.engine)
+        )
         mm_kwargs = build_disagg_mm_kwargs(inner_request)
 
         routing = inner_request.get("routing") or {}

components/src/dynamo/sglang/tests/test_sglang_decode_handler.py

@@ -39,8 +39,10 @@
 )
 from dynamo.sglang.request_handlers.llm.mm_disagg_utils import (
     build_disagg_mm_kwargs,
+    engine_consumes_media,
     extract_media_urls,
     raise_if_unextracted_multimodal,
+    reject_unconsumed_media,
 )
 from dynamo.sglang.request_handlers.llm.prefill_handler import PrefillWorkerHandler
 from dynamo.sglang.request_handlers.multimodal.worker_handler import SglangUtils
@@ -1714,6 +1716,80 @@ def test_rejects_multimodal_cache_uuid(self):
                 }
             )
 
+    @pytest.mark.parametrize(
+        "engine, consumes_media",
+        [
+            (
+                SimpleNamespace(
+                    tokenizer_manager=SimpleNamespace(
+                        model_config=SimpleNamespace(is_multimodal=False)
+                    )
+                ),
+                False,
+            ),
+            (
+                SimpleNamespace(
+                    tokenizer_manager=SimpleNamespace(
+                        model_config=SimpleNamespace(is_multimodal=True)
+                    )
+                ),
+                True,
+            ),
+            (SimpleNamespace(), True),
+            (None, True),
+        ],
+        ids=["text_only", "multimodal", "no_model_config", "no_engine"],
+    )
+    def test_engine_consumes_media(self, engine, consumes_media):
+        assert engine_consumes_media(engine) is consumes_media
+
+    @pytest.mark.parametrize(
+        "media_request",
+        [
+            {
+                "token_ids": [1, 2, 3],
+                "multi_modal_data": {
+                    "image_url": [{"Url": "https://example.com/a.jpg"}]
+                },
+            },
+            {
+                "token_ids": [1, 2, 3],
+                "messages": [
+                    {
+                        "role": "user",
+                        "content": [
+                            {
+                                "type": "image_url",
+                                "image_url": {"url": "https://example.com/a.jpg"},
+                            }
+                        ],
+                    }
+                ],
+            },
+        ],
+        ids=["extracted", "raw_message"],
+    )
+    def test_rejects_media_the_engine_does_not_consume(self, media_request):
+        with pytest.raises(InvalidArgument, match="does not accept image input"):
+            reject_unconsumed_media(media_request, consumes_media=False)
+
+    def test_allows_media_the_engine_consumes(self):
+        reject_unconsumed_media(
+            {
+                "token_ids": [1, 2, 3],
+                "multi_modal_data": {
+                    "image_url": [{"Url": "https://example.com/a.jpg"}]
+                },
+            },
+            consumes_media=True,
+        )
+
+    def test_allows_text_and_empty_media_lists(self):
+        reject_unconsumed_media(
+            {"token_ids": [1, 2, 3], "multi_modal_data": {"image_url": []}},
+            consumes_media=False,
+        )
+
 
 def test_build_logprob_kwargs_allows_chosen_token_logprobs(monkeypatch):
     monkeypatch.delenv("DYN_SGL_ALLOW_TOP_LOGPROBS", raising=False)