fix(sglang): reject media for text-only engines

components/src/dynamo/frontend/sglang_processor.py

@@ -49,7 +49,9 @@
     nvext_extra_field_requested,
     random_uuid,
     read_jinja_chat_template,
+    reject_undeclared_input_modalities,
     resolve_chat_template,
+    runtime_input_modalities,
     worker_warmup,
 )
 
@@ -510,6 +512,7 @@ def __init__(
         preprocess_workers: int = 0,
         stream_interval: int = 1,
         default_thinking_mode: str | None = None,
+        input_modalities: frozenset[str] | None = None,
     ):
         self.tokenizer = tokenizer
         # Detect force_reasoning once from the chat template, matching
@@ -534,6 +537,7 @@ def __init__(
         self.debug_perf = debug_perf
         self.stream_interval = stream_interval
         self.default_thinking_mode = default_thinking_mode
+        self.input_modalities = input_modalities
         self.preprocess_pool = preprocess_pool
         if preprocess_pool is not None:
             self._worker_semaphore: asyncio.Semaphore | None = asyncio.Semaphore(
@@ -557,6 +561,9 @@ async def generator(
             logger.info("[perf] sglang generator enter: active_requests=%d", active)
 
         try:
+            reject_undeclared_input_modalities(
+                request.get("messages") or [], self.input_modalities
+            )
             if self.preprocess_pool is None:
                 async for item in self._generator_inner(request, context=context):
                     yield item
@@ -1110,6 +1117,7 @@ async def chat_engine_factory(
             preprocess_workers=preprocess_workers,
             stream_interval=self.stream_interval,
             default_thinking_mode=default_thinking_mode,
+            input_modalities=runtime_input_modalities(mdc.runtime_config()),
         )
         gen.exclude_tools_when_tool_choice_none = (
             self.config.exclude_tools_when_tool_choice_none

components/src/dynamo/frontend/tests/test_multimodal_utils.py

@@ -3,7 +3,12 @@
 
 import pytest
 
-from dynamo.frontend.utils import extract_mm_urls
+from dynamo.frontend.utils import (
+    extract_mm_urls,
+    reject_undeclared_input_modalities,
+    runtime_input_modalities,
+)
+from dynamo.llm.exceptions import InvalidArgument
 
 pytestmark = [
     pytest.mark.unit,
@@ -286,3 +291,50 @@ def test_rejects_media_parts_without_valid_url_or_uuid(part):
 
     with pytest.raises(ValueError, match=r"URL or uuid|non-empty string"):
         extract_mm_urls(messages)
+
+
+_IMAGE_MESSAGE = {
+    "role": "user",
+    "content": [
+        {"type": "text", "text": "What is this?"},
+        {"type": "image_url", "image_url": {"url": "https://example.com/cat.png"}},
+    ],
+}
+
+
+@pytest.mark.parametrize(
+    "runtime_config, expected",
+    [
+        (None, None),
+        ({}, None),
+        ({"runtime_data": {}}, None),
+        ({"runtime_data": {"input_modalities": "text"}}, None),
+        ({"runtime_data": {"input_modalities": [1]}}, None),
+        ({"runtime_data": {"input_modalities": ["text", 1]}}, None),
+        ({"runtime_data": {"input_modalities": []}}, frozenset()),
+        ({"runtime_data": {"input_modalities": ["text"]}}, frozenset({"text"})),
+    ],
+)
+def test_runtime_input_modalities(runtime_config, expected):
+    assert runtime_input_modalities(runtime_config) == expected
+
+
+def test_undeclared_input_modalities_accept_media():
+    reject_undeclared_input_modalities([_IMAGE_MESSAGE], None)
+
+
+def test_declared_input_modalities_accept_listed_media():
+    reject_undeclared_input_modalities([_IMAGE_MESSAGE], frozenset({"text", "image"}))
+
+
+def test_text_only_input_modalities_reject_image():
+    with pytest.raises(InvalidArgument, match="'image_url'"):
+        reject_undeclared_input_modalities([_IMAGE_MESSAGE], frozenset({"text"}))
+
+
+def test_text_only_input_modalities_accept_text():
+    messages = [
+        {"role": "user", "content": [{"type": "text", "text": "Hello"}]},
+        {"role": "user", "content": "Hello again"},
+    ]
+    reject_undeclared_input_modalities(messages, frozenset({"text"}))

components/src/dynamo/frontend/tests/test_sglang_processor_unit.py

@@ -4967,3 +4967,45 @@ def apply_chat_template(self, messages, **kwargs):
             reasoning_parser_name=None,
         )
         assert result.request.get("chat_template_kwargs", {}) == case.expected
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "modalities, rejects",
+    [(None, False), (frozenset({"text"}), True), (frozenset({"text", "image"}), False)],
+)
+async def test_input_modalities_guard_before_dispatch(tokenizer, modalities, rejects):
+    engine = FakeRoutedEngine(
+        items=[
+            {
+                "token_ids": tokenizer.encode("ok", add_special_tokens=False),
+                "finish_reason": "stop",
+            }
+        ]
+    )
+    processor = SglangProcessor(
+        tokenizer, engine, None, None, None, input_modalities=modalities
+    )
+    request = {
+        "model": MODEL,
+        "messages": [
+            {
+                "role": "user",
+                "content": [
+                    {"type": "text", "text": "Describe."},
+                    {
+                        "type": "image_url",
+                        "image_url": {"url": "https://example.invalid/image.png"},
+                    },
+                ],
+            }
+        ],
+    }
+    expected = (
+        pytest.raises(InvalidArgument, match="does not accept image")
+        if rejects
+        else nullcontext()
+    )
+    with expected:
+        assert [item async for item in processor.generator(request)]
+    assert len(engine.requests) == (0 if rejects else 1)

components/src/dynamo/frontend/utils.py

@@ -205,6 +205,58 @@ def extract_mm_urls(
     return mm_data or None, mm_uuids if has_user_uuid else None
 
 
+# Mirrors INPUT_MODALITIES_RUNTIME_KEY in lib/llm/src/local_model/runtime_config.rs.
+INPUT_MODALITIES_RUNTIME_KEY = "input_modalities"
+
+
+def runtime_input_modalities(
+    runtime_config: dict[str, Any] | None,
+) -> frozenset[str] | None:
+    """Read worker-declared input modalities; ``None`` when undeclared."""
+    if not isinstance(runtime_config, dict):
+        return None
+    runtime_data = runtime_config.get("runtime_data")
+    if not isinstance(runtime_data, dict):
+        return None
+    declared = runtime_data.get(INPUT_MODALITIES_RUNTIME_KEY)
+    if not isinstance(declared, list) or not all(
+        isinstance(value, str) for value in declared
+    ):
+        return None
+    return frozenset(declared)
+
+
+def reject_undeclared_input_modalities(
+    messages: list[dict[str, Any]],
+    input_modalities: frozenset[str] | None,
+) -> None:
+    """Reject media the worker did not declare, matching the Rust preprocessor.
+
+    Without this, a text-only chat template drops the media and the request
+    succeeds as though the model had seen it.
+    """
+    if input_modalities is None:
+        return
+    for msg in messages:
+        if not isinstance(msg, dict) or msg.get("role") not in ("user", "tool"):
+            continue
+        content = msg.get("content")
+        if not isinstance(content, list):
+            continue
+        for part in content:
+            if not isinstance(part, dict):
+                continue
+            part_type = part.get("type")
+            if part_type not in _MEDIA_CONTENT_TYPES:
+                continue
+            modality = part_type.removesuffix("_url")
+            if modality not in input_modalities:
+                raise InvalidArgument(
+                    f"Model does not accept {modality} input; "
+                    f"received unsupported content type '{part_type}'."
+                )
+
+
 def make_backend_error(engine_response: dict[str, Any]) -> dict[str, Any]:
     """Build an OpenAI-style error dict, guarding against None/missing message."""
     backend_msg = engine_response.get("message") or "unknown backend error"

components/src/dynamo/frontend/vllm_processor.py

@@ -53,7 +53,9 @@
     handle_engine_error,
     make_internal_error,
     random_uuid,
+    reject_undeclared_input_modalities,
     resolve_chat_template,
+    runtime_input_modalities,
 )
 
 logger = logging.getLogger(__name__)
@@ -419,6 +421,7 @@ def __init__(
         structural_tag_mode: str = "off",
         structural_tag_scope: str = "auto",
         structural_tag_schema: str = "auto",
+        input_modalities: frozenset[str] | None = None,
     ):
         self.tokenizer = tokenizer
         self.input_processor = input_processor
@@ -431,6 +434,7 @@ def __init__(
         self.enable_auto_tool_choice = enable_auto_tool_choice
         self.default_chat_template_kwargs = default_chat_template_kwargs
         self.default_thinking_mode = default_thinking_mode
+        self.input_modalities = input_modalities
         self.structural_tag_mode = structural_tag_mode
         self.structural_tag_scope = structural_tag_scope
         self.structural_tag_schema = structural_tag_schema
@@ -653,6 +657,7 @@ async def _generator_inner(
             )
 
         messages = request.get("messages") or []
+        reject_undeclared_input_modalities(messages, self.input_modalities)
         _normalize_vllm_image_parts(messages)
         # Preserve user cache UUIDs alongside URL-backed media. UUID-only image
         # slots are resolved by the worker-side vLLM processor cache.
@@ -1315,6 +1320,7 @@ async def chat_engine_factory(
             structural_tag_mode=structural_tag_mode,
             structural_tag_scope=structural_tag_scope,
             structural_tag_schema=structural_tag_schema,
+            input_modalities=runtime_input_modalities(mdc.runtime_config()),
         )
         gen.exclude_tools_when_tool_choice_none = (
             self.config.exclude_tools_when_tool_choice_none

components/src/dynamo/sglang/register.py

@@ -57,6 +57,8 @@
 
 SGLANG_HICACHE_MOONCAKE_RUNTIME_KEY = "sglang_hicache_mooncake"
 SPEC_DECODE_RUNTIME_KEY = "spec_decode"
+# Mirrors INPUT_MODALITIES_RUNTIME_KEY in lib/llm/src/local_model/runtime_config.rs.
+INPUT_MODALITIES_RUNTIME_KEY = "input_modalities"
 
 
 def _supports_engine_generate(
@@ -109,6 +111,21 @@ def _register_model_source_path(
     return server_args.model_path
 
 
+def _get_input_modalities(engine: Optional[sgl.Engine]) -> Optional[List[str]]:
+    """Declare only models known to reject all media.
+
+    ``is_multimodal`` cannot distinguish image, video and audio support, so
+    leave multimodal capabilities undeclared and let the engine validate them.
+    """
+    if engine is None:
+        return None
+    try:
+        is_multimodal = engine.tokenizer_manager.model_config.is_multimodal
+    except AttributeError:
+        return None
+    return ["text"] if is_multimodal is False else None
+
+
 def _build_media_decoder_and_fetcher():
     """Construct MediaDecoder/MediaFetcher for frontend-decoded multimodal.
 
@@ -558,6 +575,13 @@ async def get_runtime_config(
                 f"Failed to attach Mooncake HiCache runtime metadata to registration: {e}"
             )
 
+    input_modalities = _get_input_modalities(engine)
+    if input_modalities is not None:
+        runtime_config.set_engine_specific(
+            INPUT_MODALITIES_RUNTIME_KEY, json.dumps(input_modalities)
+        )
+        logging.info("Published SGLang input modalities: %s", input_modalities)
+
     if engine is None:
         return runtime_config
 

components/src/dynamo/sglang/tests/test_runtime_metadata.py

@@ -339,3 +339,26 @@ def fail_hicache_publish(self, key, value):
     assert (
         "Failed to attach native offloading capacity from SGLang HiCache" in caplog.text
     )
+
+
+@pytest.mark.parametrize(
+    "flag, expected", [(False, ["text"]), (True, None), (None, None), ("false", None)]
+)
+def test_input_modalities_only_declare_text_only_models(flag, expected):
+    # Keep registration imports lazy for pytest-marker-report's SGLang stubs.
+    from dynamo.sglang.register import _get_input_modalities
+
+    engine = SimpleNamespace(
+        tokenizer_manager=SimpleNamespace(
+            model_config=SimpleNamespace(is_multimodal=flag)
+        )
+    )
+    assert _get_input_modalities(engine) == expected
+
+
+def test_input_modalities_without_engine_metadata():
+    # Keep registration imports lazy for pytest-marker-report's SGLang stubs.
+    from dynamo.sglang.register import _get_input_modalities
+
+    assert _get_input_modalities(None) is None
+    assert _get_input_modalities(SimpleNamespace()) is None

lib/llm/src/discovery/watcher.rs

@@ -1921,6 +1921,7 @@ mod tests {
             "explicit-default",
             "overridden-policy",
             "needs",
+            "input-modalities",
         ] {
             let endpoint = drt
                 .namespace(difference)
@@ -1949,6 +1950,13 @@ mod tests {
                     });
                 }
                 "needs" => newcomer.needs = vec![vec![WorkerType::Encode]],
+                "input-modalities" => {
+                    newcomer.runtime_config.runtime_data.insert(
+                        crate::local_model::runtime_config::INPUT_MODALITIES_RUNTIME_KEY
+                            .to_string(),
+                        serde_json::json!(["text"]),
+                    );
+                }
                 _ => unreachable!(),
             }
             let manager = Arc::new(ModelManager::new());