[https://nvbugs/6405760][fix] Do not load the multimodal encoder for text-only trtllm-bench runs

tensorrt_llm/_torch/model_config.py

@@ -209,6 +209,10 @@ class ModelConfig(Generic[TConfig]):
     # If true, ONLY the vision encoder part of the full model is loaded/executed.
     mm_encoder_only: bool = False
 
+    # If true, the multimodal encoder of a multimodal checkpoint is NOT
+    # instantiated/loaded and the model serves text-only requests.
+    disable_mm_encoder: bool = False
+
     # Video pruning rate for VLM models (None = EVS disabled)
     video_pruning_rate: Optional[float] = None
 

tensorrt_llm/_torch/models/modeling_qwen3_5.py

@@ -34,7 +34,6 @@
 from ..pyexecutor.config_utils import get_qwen3_hybrid_layer_types
 from .checkpoints.base_weight_mapper import BaseWeightMapper
 from .checkpoints.hf.qwen3_5_weight_mapper import Qwen3_5MoeHfWeightMapper
-from .modeling_multimodal_utils import _is_mm_disagg
 from .modeling_qwen3_next import Qwen3NextForCausalLM
 from .modeling_qwen3vl import (
     Qwen3VisionModel,
@@ -691,7 +690,8 @@ def multimodal_data_device_paths(self) -> List[str]:
         ]
 
     def load_weights(self, weights: Dict[str, torch.Tensor], weight_mapper: BaseWeightMapper):
-        if not _is_mm_disagg():
+        # None under MM E/P disagg or disable_mm_encoder.
+        if self.mm_encoder is not None:
             self.mm_encoder.load_weights(weights)
 
         weight_mapper = Qwen3_5MoeHfWeightMapper()

tensorrt_llm/_torch/models/modeling_qwen3vl.py

@@ -1253,11 +1253,18 @@ def __init__(
         self.llm = AutoModelForCausalLM.from_config(llm_model_config)
 
         self.mm_encoder = None
-        # Normal workers own the encoder. MM E/P handoff uses attached embeddings.
-        if not _is_mm_disagg():
+        # Normal workers own the encoder. MM E/P handoff uses attached
+        # embeddings; disable_mm_encoder serves the checkpoint text-only and
+        # saves the encoder's GPU memory for the KV cache pool.
+        if not (_is_mm_disagg() or model_config.disable_mm_encoder):
             self.mm_encoder = Qwen3VisionModelBase(
                 copy.deepcopy(model_config), kwargs.get("vision_model_class", None)
             ).eval()
+        elif model_config.disable_mm_encoder:
+            logger.info(
+                f"{type(self).__name__}: multimodal encoder disabled "
+                "(disable_mm_encoder=True); serving text-only requests."
+            )
 
         self.use_deepstack = hasattr(config.vision_config, "deepstack_visual_indexes")
         self.deepstack_num_level = (

tensorrt_llm/_torch/pyexecutor/_util.py

@@ -584,6 +584,10 @@ def _create_dummy_encoder_inputs(self) -> List[MultimodalParams]:
                 self._profiling_stage_data,
                 dict) and not self._profiling_stage_data.get("enable_mm_reqs"):
             return []
+        # No local multimodal encoder (disable_mm_encoder or MM E/P disagg):
+        # nothing to profile.
+        if getattr(self._model_engine.model, "mm_encoder", object()) is None:
+            return []
         input_processor = self._model_engine.input_processor
         _, encoder_max_num_tokens = self._llm_args.get_encoder_runtime_sizes()
         # Modality-agnostic: the model declares each modality's per-item token

tensorrt_llm/_torch/pyexecutor/model_loader.py

@@ -1173,6 +1173,7 @@ def _load_and_validate_config(
             lora_config=self.lora_config,
             allreduce_strategy=self.llm_args.allreduce_strategy,
             mm_encoder_only=self.llm_args.mm_encoder_only,
+            disable_mm_encoder=self.llm_args.disable_mm_encoder,
             attn_backend=self.llm_args.attn_backend,
             moe_backend=self.llm_args.moe_config.backend,
             moe_disable_finalize_fusion=self.llm_args.moe_config.

tensorrt_llm/bench/benchmark/throughput.py

@@ -449,6 +449,16 @@ def throughput_command(
         kwargs = kwargs | runtime_config.get_llm_args()
         kwargs['skip_tokenizer_init'] = not no_skip_tokenizer_init
         kwargs['backend'] = options.backend
+        if (options.modality is None and options.backend == "pytorch"
+                and "disable_mm_encoder" not in kwargs
+                and not kwargs.get("mm_encoder_only", False)):
+            # Text-only benchmark: skip a multimodal checkpoint's encoder so
+            # its GPU memory goes to the KV cache pool instead. Text-only
+            # models ignore this flag. Overridable via extra_llm_api_options.
+            kwargs["disable_mm_encoder"] = True
+            logger.info(
+                "Text-only benchmark (--modality not set): the multimodal "
+                "encoder, if the model has one, will not be loaded.")
         if bench_env.telemetry_config is not None:
             kwargs["telemetry_config"] = bench_env.telemetry_config
 

tensorrt_llm/llmapi/llm_args.py

@@ -5092,6 +5092,16 @@ def validate_encoder_runtime_sizes(cls, v: Optional[int]) -> Optional[int]:
         "Only load/execute the vision encoder part of the full model. Defaults to False.",
         status="prototype")
 
+    disable_mm_encoder: bool = Field(
+        default=False,
+        description=
+        "Skip instantiating and loading the multimodal (e.g. vision) encoder "
+        "of a multimodal checkpoint and serve it text-only. Saves the "
+        "encoder's GPU memory (enlarging the KV cache pool) for workloads "
+        "that never send image/video/audio inputs; such requests are "
+        "rejected. Defaults to False.",
+        status="prototype")
+
     encode_only: bool = Field(
         default=False,
         description=
@@ -5273,6 +5283,11 @@ def validate_encoder_modes(self) -> 'TorchLlmArgs':
                 "Use encode_only=True for LLM.encode(), or use "
                 "MultimodalEncoder/mm_encoder_only for multimodal encoder "
                 "execution.")
+        if self.disable_mm_encoder and self.mm_encoder_only:
+            raise ValueError(
+                "disable_mm_encoder and mm_encoder_only are mutually "
+                "exclusive: one skips the multimodal encoder, the other runs "
+                "only the multimodal encoder.")
         return self
 
     @model_validator(mode="after")

tests/unittest/_torch/modeling/test_modeling_qwen3_5_vl.py

@@ -165,6 +165,31 @@ def test_qwen35_dense_vl_resolves_model_and_mapper(tmp_path: Path) -> None:
     )
 
 
+def test_qwen35_dense_vl_disable_mm_encoder_skips_vision_tower(
+    tmp_path: Path,
+) -> None:
+    """disable_mm_encoder must not instantiate the vision encoder (nvbug
+    6405760: the tower's GPU memory shrank the KV cache pool for text-only
+    serving of VLM checkpoints)."""
+    config = load_pretrained_config(str(_write_qwen35_dense_vl_config(tmp_path)))
+    model_config = ModelConfig(pretrained_config=config, disable_mm_encoder=True)
+    with torch.device("cuda"):
+        model = Qwen3_5VLModel(model_config)
+
+    assert model.mm_encoder is None
+    assert not any("mm_encoder" in name for name, _ in model.named_parameters())
+
+
+def test_qwen35_dense_vl_default_keeps_vision_tower(tmp_path: Path) -> None:
+    config = load_pretrained_config(str(_write_qwen35_dense_vl_config(tmp_path)))
+    model_config = ModelConfig(pretrained_config=config)
+    with torch.device("cuda"):
+        model = Qwen3_5VLModel(model_config)
+
+    assert model.mm_encoder is not None
+    assert any("mm_encoder" in name for name, _ in model.named_parameters())
+
+
 def test_qwen35_dense_vl_placeholder_metadata_registered() -> None:
     metadata = MULTIMODAL_PLACEHOLDER_REGISTRY.get_placeholder_metadata("qwen3_5")
 

tests/unittest/api_stability/references/llm.yaml

@@ -119,6 +119,10 @@ methods:
         annotation: bool
         default: False
         status: prototype
+      disable_mm_encoder:
+        annotation: bool
+        default: False
+        status: prototype
       encode_only:
         annotation: bool
         default: False