yechank-nvidia · Oct 30, 2026 · Oct 30, 2026 · Nov 3, 2026 · Nov 3, 2026 · 2ez4bz
@@ -27,6 +27,74 @@ TensorRT LLM incorporates some key optimizations to enhance the performance of m
 
 Further optimizations are under development and will be updated as they become available.
 
+## Encoder attention backends
+
+`multimodal_config.encoder_attention_backend` selects a TensorRT-LLM attention backend for each logical encoder. The public values are `VANILLA` (PyTorch scaled dot product attention), `TRTLLM`, and `FLASHINFER`; values are case-insensitive. Image and video inputs share the `vision` encoder.
+
+```yaml
+multimodal_config:
+  encoder_attention_backend:
+    vision: FLASHINFER
+```
+
+The same setting is available in the Python API:
+
+```python
+from tensorrt_llm import LLM
+
+llm = LLM(
+    model="Qwen/Qwen3-VL-8B-Instruct",
+    multimodal_config={"encoder_attention_backend": {"vision": "VANILLA"}},
+)
+```
+
+Omitted entries preserve the model's existing encoder defaults, including inheritance from `attn_backend` where applicable. An encoder override leaves the language model's backend unchanged. Backend availability and supported tensor shapes still depend on the installed kernels and hardware; floating-point differences between backends are expected. `VANILLA` does not support `multimodal_config.encoder_cuda_graph`, so selecting it for an encoder with CUDA graphs enabled fails during configuration.
+
+The table covers every registered multimodal model implementation; model aliases share the same declaration. Image/video share `vision`, and `audio` names a separate encoder. "Not supported" means this option cannot change that encoder's attention backend; it does not mean the model or modality is unsupported. Gemma 4 Unified uses encoder-free projections, and Mllama's vision input is not implemented. The CPU registry test checks this table against each model's explicit `supported_encoder_attention_backends` declaration.
+
+<!-- encoder-attention-backends:start -->
+| Model | Logical encoder | Supported backends |
+| --- | --- | --- |
+| `Cosmos3EdgeModel` | `vision` | Not supported |
+| `Cosmos3Model` | `vision` | `VANILLA`, `TRTLLM`, `FLASHINFER` |
+| `DeepseekV41ForConditionalGeneration` | `vision` | Not supported |
+| `Exaone4_5_ForConditionalGeneration` | `vision` | `VANILLA`, `TRTLLM`, `FLASHINFER` |
+| `Gemma3VLM` | `vision` | Not supported |
+| `Gemma4ForConditionalGeneration` | `audio` | Not supported |
+| `Gemma4ForConditionalGeneration` | `vision` | `VANILLA`, `TRTLLM`, `FLASHINFER` |
+| `Gemma4UnifiedForConditionalGeneration` | None (encoder-free) | Not supported |
+| `Glm5NextVLM` | `vision` | `TRTLLM` |
+| `HCXVisionForCausalLM` | `vision` | Not supported |
+| `KimiK25ForConditionalGeneration` | `vision` | `VANILLA`, `TRTLLM`, `FLASHINFER` |
+| `KimiK3ForConditionalGeneration` | `vision` | `VANILLA`, `TRTLLM`, `FLASHINFER` |
+| `Llama4ForConditionalGeneration` | `vision` | Not supported |
+| `LlavaNextModel` | `vision` | Not supported |
+| `MiniCPMV4_6Model` | `vision` | `VANILLA`, `TRTLLM`, `FLASHINFER` |
+| `MiniMaxM3VLForConditionalGeneration` | `vision` | Not supported |
+| `Mistral3VLM` | `vision` | `VANILLA`, `TRTLLM`, `FLASHINFER` |
+| `MllamaForConditionalGeneration` | `vision` | Not supported |
+| `MuseSparkForConditionalGeneration` | `audio` | Not supported |
+| `MuseSparkForConditionalGeneration` | `vision` | Not supported |
+| `NemotronHMultimodalModel` | `audio` | Not supported |
+| `NemotronHMultimodalModel` | `vision` | `VANILLA`, `TRTLLM`, `FLASHINFER` |
+| `Phi4MMForCausalLM` | `audio` | Not supported |
+| `Phi4MMForCausalLM` | `vision` | Not supported |
+| `Qwen2VLModel` | `vision` | Not supported |
+| `Qwen2_5_VLModel` | `vision` | `VANILLA`, `TRTLLM`, `FLASHINFER` |
+| `Qwen3MoeVLModel` | `vision` | `VANILLA`, `TRTLLM`, `FLASHINFER` |
+| `Qwen3VLModel` | `vision` | `VANILLA`, `TRTLLM`, `FLASHINFER` |
+| `Qwen3_5MoeVLModel` | `vision` | `VANILLA`, `TRTLLM`, `FLASHINFER` |
+| `Qwen3_5VLModel` | `vision` | `VANILLA`, `TRTLLM`, `FLASHINFER` |
+| `Qwen4ExpForConditionalGeneration` | `vision` | `VANILLA`, `TRTLLM`, `FLASHINFER` |
+| `Step3p7VLForConditionalGeneration` | `vision` | `VANILLA`, `TRTLLM`, `FLASHINFER` |
+| `VilaModel` | `vision` | Not supported |
+| `WhisperForConditionalGeneration` | `audio` | Not supported |
+<!-- encoder-attention-backends:end -->
+
+For MiniCPM-V 4.6, the selection applies to its TensorRT-LLM attention layers; its fixed window-merger attention is unchanged. The Nemotron-H RADIO vision encoder defaults to `TRTLLM`, and `VANILLA` and `FLASHINFER` remain selectable.
+
+Supported entries use TensorRT-LLM's `Attention` and apply encoder overrides explicitly. Other encoders keep their fixed implementation, including Hugging Face attention, direct SDPA, and custom attention paths. Overrides for entries marked "Not supported" fail during initialization, even if the requested value is `VANILLA`. Unknown encoder names and unsupported backends also fail during initialization; Hugging Face `attn_implementation` names are not accepted. With `disable_mm_encoder=True`, encoders that the model skips do not use the setting, so text-only runs can reuse a serving configuration.
+
 ## Model Support Matrix
 
 Please refer to the latest multimodal [support matrix](../models/supported-models.md#multimodal-feature-support-matrix-pytorch-backend).

@@ -153,6 +153,14 @@ def generate_serve_schema() -> dict:
         {**per_layer_dynamic, "required": ["mode"]},
         {"type": ["string", "null"]},
     ]
+    # MultimodalConfig normalizes encoder backend names before Literal validation.
+    encoder_backend = schema["$defs"]["MultimodalConfig"]["properties"][
+        "encoder_attention_backend"
+    ]["anyOf"][0]["additionalProperties"]
+    alternatives = [
+        "".join(f"[{char}{char.lower()}]" for char in name) for name in encoder_backend.pop("enum")
+    ]
+    encoder_backend["pattern"] = "^(?:" + "|".join(alternatives) + r")$(?![\s\S])"
     # These object/Any annotations are Python API hooks, not arbitrary YAML mappings.
     # Update these restrictions when hooks are added or gain YAML construction support.
     for name in ("batched_logits_processor", "checkpoint_loader", "_mpi_session"):

@@ -12,7 +12,7 @@
 
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Any, NamedTuple, Optional, Tuple
+from typing import TYPE_CHECKING, Any, Dict, NamedTuple, Optional, Tuple
 
 import torch
 
@@ -57,6 +57,7 @@ class EngineChoices(NamedTuple):
     mamba_ssm_cache_dtype: torch.dtype  # the recurrent-state cache's dtype
     gdn_replay: bool  # TRTLLM_USE_GDN_REPLAY, which the engine reads for eligible MTP workloads
     prefill_cuda_graph_backend: str  # `prefill_cuda_graph_backend`: disabled, piecewise, breakable
+    encoder_attention_backend: Optional[Dict[str, str]]  # `multimodal_config` encoder backends
 
 
 def engine_choices(model_config: ModelConfig, llm_args: TorchLlmArgs) -> EngineChoices:
@@ -65,6 +66,7 @@ def engine_choices(model_config: ModelConfig, llm_args: TorchLlmArgs) -> EngineC
     from tensorrt_llm._torch.utils import is_gdn_replay_enabled
 
     kv_cache_config = llm_args.kv_cache_config
+    mm_config = llm_args.multimodal_config
     return EngineChoices(
         torch_dtype=model_config.torch_dtype,
         moe_backend=model_config.moe_backend_requested,
@@ -73,16 +75,20 @@ def engine_choices(model_config: ModelConfig, llm_args: TorchLlmArgs) -> EngineC
         ),
         gdn_replay=is_gdn_replay_enabled(),
         prefill_cuda_graph_backend=str(llm_args.prefill_cuda_graph_backend.value),
+        encoder_attention_backend=(
+            mm_config.encoder_attention_backend if mm_config is not None else None
+        ),
     )
 
 
 def _check_engine(
     spec_config: Optional[Any], kv_cache_quant_algo: Optional[Any], engine: EngineChoices
 ) -> None:
-    """Decline what the target's constructor or first forward would refuse.
+    """Decline what the target's constructor or first forward would refuse or ignore.
 
-    Without this ``auto`` claims the deployment and then fails mid-build instead of
-    falling back to the built-in implementation.
+    Without this ``auto`` claims the deployment and then fails mid-build, or runs
+    without a requested setting, instead of falling back to the built-in
+    implementation.
     """
     if spec_config is not None and (
         getattr(spec_config, "draft_len_schedule", None) is not None
@@ -116,6 +122,11 @@ def _check_engine(
             "which the target does not implement; 'breakable' runs the prefill step as a "
             "breakable CUDA graph"
         )
+    if engine.encoder_attention_backend:
+        raise NoTarget(
+            f"multimodal_config.encoder_attention_backend={engine.encoder_attention_backend}; "
+            "the target's vision tower runs its own attention and does not apply it"
+        )
     if spec_config is None:
         return
     draft = spec_config.max_draft_len

@@ -14,6 +14,7 @@
 # limitations under the License.
 
 import contextlib
+import copy
 import errno
 import json
 import os
@@ -22,7 +23,7 @@
 from dataclasses import dataclass, field
 from pathlib import Path
 from typing import (TYPE_CHECKING, Any, Dict, Generic, Iterator, List, Optional,
-                    Sequence, Tuple, TypeVar)
+                    Sequence, Tuple, TypeVar, get_args)
 
 import filelock
 import torch
@@ -39,14 +40,17 @@
                                  torch_dtype_to_binding)
 from tensorrt_llm.bindings import LayerType as LayerTypeCpp
 from tensorrt_llm.functional import AllReduceStrategy
+# yapf: disable
 # Callers import MoeLoadBalancerConfig from this module.
 from tensorrt_llm.llmapi.llm_args import MoeLoadBalancerConfig  # noqa: F401
 from tensorrt_llm.llmapi.llm_args import (CSA2SparseAttentionConfig,
                                           DeepSeekSparseAttentionConfig,
                                           DeepSeekV4SparseAttentionConfig,
+                                          EncoderAttentionBackend,
                                           KvCacheConfig,
                                           MoeLoadBalancerConfigType,
                                           MultimodalConfig)
+# yapf: enable
 from tensorrt_llm.logger import logger
 from tensorrt_llm.mapping import Mapping
 from tensorrt_llm.models.modeling_utils import QuantConfig
@@ -370,6 +374,57 @@ class ModelConfig(Generic[TConfig]):
     # implementation. Set by the model loader, read by `_resolve_class`.
     modeling_v2_target: Optional[type] = None
 
+    def get_encoder_attention_backend(self, encoder: str, default: str) -> str:
+        """Return the attention backend selected for a logical encoder.
+
+        Args:
+            encoder: Logical encoder name, such as ``"vision"`` or ``"audio"``.
+            default: Backend the model uses for this encoder when
+                ``multimodal_config.encoder_attention_backend`` omits it.
+
+        Returns:
+            The backend from ``multimodal_config.encoder_attention_backend``,
+            or ``default`` when the encoder has no entry.
+
+        Raises:
+            ValueError: If the selected backend is not a public encoder backend.
+        """
+        overrides = (self.multimodal_config.encoder_attention_backend
+                     if self.multimodal_config is not None else None)
+        if not overrides or encoder not in overrides:
+            return default
+        backend = overrides[encoder]
+        backends = get_args(EncoderAttentionBackend)
+        if backend not in backends:
+            raise ValueError(
+                f"Invalid attention backend {backend!r} for the {encoder} encoder; "
+                f"expected one of {', '.join(backends)}")
+        return backend
+
+    def with_encoder_attention_backend(self,
+                                       encoder: str,
+                                       default: str | None = None
+                                       ) -> "ModelConfig":
+        """Return a config whose ``attn_backend`` is the encoder's backend.
+
+        Args:
+            encoder: Logical encoder name, such as ``"vision"`` or ``"audio"``.
+            default: Backend the model uses for this encoder when no override
+                is set. ``None`` means the decoder's ``attn_backend``.
+
+        Returns:
+            ``self`` when the selected backend equals ``attn_backend``;
+            otherwise a shallow copy with only ``attn_backend`` changed, so the
+            decoder config is never modified.
+        """
+        backend = self.get_encoder_attention_backend(
+            encoder, self.attn_backend if default is None else default)
+        if backend == self.attn_backend:
+            return self
+        config = copy.copy(self)
+        object.__setattr__(config, "attn_backend", backend)
+        return config
+
     @property
     def kv_cache_layer_specs(self) -> Optional[Tuple[KVCacheLayerSpec, ...]]:
         """Return model-provided per-layer KV-cache geometry, if any."""

@@ -76,6 +76,8 @@ def get_preferred_media_io_kwargs(self) -> Dict[str, Dict[str, Any]]:
     Cosmos3InputProcessor, model_type="cosmos3_omni", placeholder_metadata=PLACEHOLDER_METADATA
 )
 class Cosmos3Model(Qwen3VLModel):
+    supported_encoder_attention_backends = Qwen3VLModel.supported_encoder_attention_backends
+
     def __init__(self, model_config: ModelConfig[PretrainedConfig], *args, **kwargs):
         omni_config = model_config.pretrained_config
         if omni_config is None:

@@ -1012,6 +1012,9 @@ def _edge_positional_embedding_params(config) -> PositionalEmbeddingParams:
     placeholder_metadata=PLACEHOLDER_METADATA,
 )
 class Cosmos3EdgeModel(Qwen3VLModelBase):
+    # The custom SigLIP path does not apply encoder overrides.
+    supported_encoder_attention_backends = {"vision": frozenset()}
+
     supports_encoder_cache = True
 
     def __init__(self, model_config: ModelConfig[PretrainedConfig], *args, **kwargs):

@@ -41,6 +41,9 @@ class DeepseekV41ForConditionalGeneration(MultimodalModelMixin, PreTrainedModel)
     Attention, Engram, and DSpark remain in the text model.
     """
 
+    # The vision encoder calls SDPA directly.
+    supported_encoder_attention_backends = {"vision": frozenset()}
+
     def __init__(self, model_config: ModelConfig) -> None:
         config = model_config.pretrained_config
         super().__init__(config)

@@ -192,6 +192,8 @@ class Exaone4_5_VisionModel(Qwen2VisionModelBase):
     ),
 )
 class Exaone4_5_ForConditionalGeneration(Qwen2VLModelBase):
+    supported_encoder_attention_backends = Qwen2VLModelBase.supported_encoder_attention_backends
+
     def __init__(
         self,
         model_config: ModelConfig[PretrainedConfig],

@@ -181,6 +181,9 @@ def forward(self, vision_outputs: torch.Tensor):
     ))
 class Gemma3VLM(PreTrainedModel):
 
+    # The native vision path does not apply encoder overrides.
+    supported_encoder_attention_backends = {"vision": frozenset()}
+
     lora_owner = DecoderLoraOwner(module_path="language_model")
 
     @classmethod

@@ -252,6 +252,8 @@ def get_mm_item_metadata(
 class Gemma4UnifiedForConditionalGeneration(Gemma4MultimodalModelBase):
     """Gemma 4 12B Unified model with encoder-free multimodal projections."""
 
+    supported_encoder_attention_backends = {}
+
     def __init__(self, model_config: ModelConfig):
         config = model_config.pretrained_config
         if config.image_token_id is None:

@@ -279,8 +279,9 @@ class Gemma4VisionAttention(Attention):
     """Bidirectional attention with q/k/v RMSNorms and 2D RoPE.
 
     Inherits ``Attention`` to participate in the TRT-LLM backend dispatch.
-    Vision sub_config is forced to ``attn_backend="TRTLLM"`` in
-    ``modeling_gemma4mm.get_sub_model_config`` (same pattern Qwen2.5-VL uses).
+    Vision defaults to ``attn_backend="TRTLLM"``; an explicit
+    ``multimodal_config.encoder_attention_backend['vision']`` selects another
+    supported backend in ``modeling_gemma4mm.get_sub_model_config``.
     Vision is context-only and runs through ``forward_impl`` with
     ``PredefinedAttentionMask.FULL`` and a tower-local ``attn_metadata``
     (``kv_cache_manager=None``).

@@ -985,6 +985,11 @@ def _normalize_vision_patches(
 class Gemma4MultimodalModelBase(MultimodalModelMixin, PreTrainedModel):
     """Shared multimodal encoder flow for Gemma4 conditional generation models."""
 
+    supported_encoder_attention_backends = {
+        "vision": frozenset({"VANILLA", "TRTLLM", "FLASHINFER"}),
+        "audio": frozenset(),  # Chunked local attention with relative-position bias.
+    }
+
     supports_encoder_cache = True
     supports_vision_gpu_normalize: ClassVar[bool] = True
     lora_owner = DecoderLoraOwner(module_path="language_model")
@@ -1246,7 +1251,11 @@ def get_sub_model_config(
         )
         pretrained_config = getattr(model_config.pretrained_config, name)
         quant_config = model_config.quant_config if name == "text_config" else None
-        attn_backend = model_config.attn_backend if name == "text_config" else "TRTLLM"
+        attn_backend = (
+            model_config.attn_backend
+            if name == "text_config"
+            else model_config.get_encoder_attention_backend(name.removesuffix("_config"), "TRTLLM")
+        )
         sub_config: ModelConfig = dataclasses.replace(
             model_config,
             pretrained_config=pretrained_config,
@@ -1381,6 +1390,10 @@ class Gemma4ForConditionalGeneration(Gemma4MultimodalModelBase):
     - mm_token_type_ids-based bidirectional masking
     """
 
+    # fmt: off
+    supported_encoder_attention_backends = Gemma4MultimodalModelBase.supported_encoder_attention_backends
+    # fmt: on
+
     build_mtp_draft_model_from_config = True
     supports_mm_encoder_item_scheduling = True
 

@@ -647,6 +647,7 @@ def __init__(
         vlm_base_model: Optional[type[nn.Module]] = None,
     ) -> None:
         super().__init__()
+        model_config = model_config.with_encoder_attention_backend("vision")
         self.model_config = model_config
         self.model_dtype = _text_dtype(model_config)
         # The tower is excluded from checkpoint quantization (BF16 in
@@ -1028,6 +1029,8 @@ class Glm5NextVLM(MultimodalModelMixin, PreTrainedModel):
     requests bypass the vision tower, which runs full attention without KV cache.
     """
 
+    supported_encoder_attention_backends = {"vision": frozenset({"TRTLLM"})}
+
     _supports_flash_attn = True
     _supports_sdpa = True
 

@@ -1,3 +1,6 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
 import copy
 import math
 import os
@@ -1031,6 +1034,9 @@ def forward(self, multimodal_params: List[MultimodalParams]):
     ))
 class HCXVisionForCausalLM(PreTrainedModel):
 
+    # The native vision path does not apply encoder overrides.
+    supported_encoder_attention_backends = {"vision": frozenset()}
+
     def __init__(self, model_config: ModelConfig):
         config = model_config.pretrained_config
         super().__init__(config)