From 2eb41c68f6c65761b4cfcb1ce5e7b2dd6ef9cece Mon Sep 17 00:00:00 2001 From: Juice Date: Mon, 2 Nov 2026 18:07:34 +0000 Subject: [PATCH] Nemotron-H VL: run the RADIO vision tower on the TRTLLM attention backend FlashInfer's cuDNN ragged-prefill plan build for a not-yet-seen image shape can stall while the language model runs under CUDA graphs, freezing the server. Use the TRTLLM backend for the vision tower only, as other vision towers in the repo do. Signed-off-by: Juice Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01VSV19DmQuA2rhtusmvuxJx --- tensorrt_llm/_torch/models/modeling_nemotron_h_multimodal.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tensorrt_llm/_torch/models/modeling_nemotron_h_multimodal.py b/tensorrt_llm/_torch/models/modeling_nemotron_h_multimodal.py index 9d5dd7ff74b3..4913ab38bb19 100644 --- a/tensorrt_llm/_torch/models/modeling_nemotron_h_multimodal.py +++ b/tensorrt_llm/_torch/models/modeling_nemotron_h_multimodal.py @@ -667,6 +667,9 @@ def __init__(self, model_config: ModelConfig[transformers.PretrainedConfig]): self.vision_model = RADIOVisionModel( vision_model_config, disable_quantization=True, + # FlashInfer's cuDNN ragged-prefill plan build for a new image shape can stall + # indefinitely while the language model replays CUDA graphs, freezing the server. + vision_attn_backend="TRTLLM", encoder_cuda_graph_config=encoder_cuda_graph_config, )