diff --git a/tensorrt_llm/_torch/models/modeling_nemotron_h_multimodal.py b/tensorrt_llm/_torch/models/modeling_nemotron_h_multimodal.py
index 9d5dd7ff74b3..4913ab38bb19 100644
--- a/tensorrt_llm/_torch/models/modeling_nemotron_h_multimodal.py
+++ b/tensorrt_llm/_torch/models/modeling_nemotron_h_multimodal.py
@@ -667,6 +667,9 @@ def __init__(self, model_config: ModelConfig[transformers.PretrainedConfig]):
         self.vision_model = RADIOVisionModel(
             vision_model_config,
             disable_quantization=True,
+            # FlashInfer's cuDNN ragged-prefill plan build for a new image shape can stall
+            # indefinitely while the language model replays CUDA graphs, freezing the server.
+            vision_attn_backend="TRTLLM",
             encoder_cuda_graph_config=encoder_cuda_graph_config,
         )