From 8b4adfaca4e9b907e678df2fa4fb0298647a2b73 Mon Sep 17 00:00:00 2001 From: Pamela <179191831+pamelap-nvidia@users.noreply.github.com> Date: Mon, 18 Aug 2025 19:31:48 +0000 Subject: [PATCH 1/3] skip trtllm moe backend for sm120 Signed-off-by: Pamela <179191831+pamelap-nvidia@users.noreply.github.com> --- .../defs/accuracy/test_llm_api_pytorch.py | 26 +++++++++++++++++-- .../test_lists/qa/llm_function_rtx6kd.txt | 2 -- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch.py b/tests/integration/defs/accuracy/test_llm_api_pytorch.py index 22d04b26145..6a292a71172 100644 --- a/tests/integration/defs/accuracy/test_llm_api_pytorch.py +++ b/tests/integration/defs/accuracy/test_llm_api_pytorch.py @@ -1328,6 +1328,11 @@ def test_nvfp4_4gpus_online_eplb(self, fp8kv): @parametrize_with_ids("moe_backend", ["CUTLASS", "TRTLLM"]) def test_nvfp4(self, fp8kv, attention_dp, cuda_graph, overlap_scheduler, torch_compile, mtp_nextn, moe_backend): + if moe_backend == "TRTLLM" and (get_sm_version() == 120 + or get_sm_version() == 121): + pytest.skip( + "MOE TRTLLM backend does not support SM version 120 or 121") + kv_cache_config = KvCacheConfig(free_gpu_memory_fraction=0.75) torch_compile_config = TorchCompileConfig( enable_fullgraph=True, @@ -1375,8 +1380,10 @@ def test_nvfp4_4gpus(self, fp8kv, attention_dp, cuda_graph, torch_compile, mtp_nextn, moe_backend): if torch_compile and pp_size > 1: pytest.skip("PP with torch.compile is not supported yet.") - if moe_backend == "TRTLLM" and get_sm_version() == 120: - pytest.skip("MOE TRTLLM backend does not support SM version 120") + if moe_backend == "TRTLLM" and (get_sm_version() == 120 + or get_sm_version() == 121): + pytest.skip( + "MOE TRTLLM backend does not support SM version 120 or 121") kv_cache_config = KvCacheConfig(free_gpu_memory_fraction=0.75) # Picewise Cuda Graph cannot be enabled for nvfp4 attention dp. torch_compile_config = TorchCompileConfig( @@ -1591,6 +1598,11 @@ class TestDeepSeekR1(LlmapiAccuracyTestHarness): def test_nvfp4_multi_gpus(self, tp_size, pp_size, ep_size, mtp_nextn, fp8kv, attention_dp, cuda_graph, overlap_scheduler, max_batch_size, moe_backend): + if moe_backend == "TRTLLM" and (get_sm_version() == 120 + or get_sm_version() == 121): + pytest.skip( + "MOE TRTLLM backend does not support SM version 120 or 121") + kv_cache_config = KvCacheConfig(free_gpu_memory_fraction=0.70) pytorch_config = dict( disable_overlap_scheduler=not overlap_scheduler, @@ -2148,6 +2160,11 @@ def test_nvfp4( torch_compile, ): + if moe_backend == "TRTLLM" and (get_sm_version() == 120 + or get_sm_version() == 121): + pytest.skip( + "MOE TRTLLM backend does not support SM version 120 or 121") + torch_compile_config = TorchCompileConfig( enable_fullgraph=True, enable_piecewise_cuda_graph=cuda_graph and not attention_dp, @@ -2268,6 +2285,11 @@ def test_fp8(self, tp_size, pp_size, ep_size, attention_dp, cuda_graph, def test_nvfp4(self, tp_size, pp_size, ep_size, attention_dp, cuda_graph, overlap_scheduler, moe_backend, eagle3): + if moe_backend == "TRTLLM" and (get_sm_version() == 120 + or get_sm_version() == 121): + pytest.skip( + "MOE TRTLLM backend does not support SM version 120 or 121") + pytorch_config = dict( disable_overlap_scheduler=not overlap_scheduler, cuda_graph_config=CudaGraphConfig() if cuda_graph else None, diff --git a/tests/integration/test_lists/qa/llm_function_rtx6kd.txt b/tests/integration/test_lists/qa/llm_function_rtx6kd.txt index fbabac6b84f..d30cd857dea 100644 --- a/tests/integration/test_lists/qa/llm_function_rtx6kd.txt +++ b/tests/integration/test_lists/qa/llm_function_rtx6kd.txt @@ -24,8 +24,6 @@ accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[moe_backend=CUT accuracy/test_llm_api_pytorch.py::TestLlama3_1_8B::test_nvfp4 accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[latency_moe_cutlass-torch_compile=False] accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[latency_moe_cutlass-torch_compile=True] -accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[latency_moe_trtllm-torch_compile=False] -accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[latency_moe_trtllm-torch_compile=True] test_e2e.py::test_ptp_quickstart_advanced_mixed_precision test_e2e.py::test_ptp_quickstart_advanced[Llama3.1-8B-BF16-llama-3.1-model/Meta-Llama-3.1-8B] test_e2e.py::test_ptp_quickstart_advanced[Llama3.1-8B-FP8-llama-3.1-model/Llama-3.1-8B-Instruct-FP8] From 0565af133a0b5dd09a74a7e3ee5241608947a23c Mon Sep 17 00:00:00 2001 From: Pamela <179191831+pamelap-nvidia@users.noreply.github.com> Date: Wed, 20 Aug 2025 14:54:02 +0000 Subject: [PATCH 2/3] add checks in trtllm gen fused moe module Signed-off-by: Pamela <179191831+pamelap-nvidia@users.noreply.github.com> --- .../_torch/modules/fused_moe/fused_moe_trtllm_gen.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tensorrt_llm/_torch/modules/fused_moe/fused_moe_trtllm_gen.py b/tensorrt_llm/_torch/modules/fused_moe/fused_moe_trtllm_gen.py index 94e082a6670..f844f846c12 100644 --- a/tensorrt_llm/_torch/modules/fused_moe/fused_moe_trtllm_gen.py +++ b/tensorrt_llm/_torch/modules/fused_moe/fused_moe_trtllm_gen.py @@ -4,7 +4,7 @@ from ...distributed.ops import reducescatter from ...model_config import ModelConfig -from ...utils import Fp4QuantizedTensor +from ...utils import Fp4QuantizedTensor, get_sm_version from .interface import MoE, MoEWeightLoadingMode from .quantization import (DeepSeekFP8BlockScalesFusedMoEMethod, NVFP4TRTLLMGenFusedMoEMethod) @@ -68,6 +68,11 @@ def __init__( weight_loading_mode=weight_loading_mode, ) + sm_version = get_sm_version() + if sm_version >= 120: + raise NotImplementedError( + "TRTLLMGenFusedMoE does not support SM120 and above.") + assert not self.smart_router, "Smart router is not supported in TRTLLMGenFusedMoE." self.num_slots = self.num_experts From 03d315a3965aa89bed752470b5456d1c8e8620ba Mon Sep 17 00:00:00 2001 From: Pamela <179191831+pamelap-nvidia@users.noreply.github.com> Date: Wed, 20 Aug 2025 14:54:02 +0000 Subject: [PATCH 3/3] add checks in trtllm gen fused moe module Signed-off-by: Pamela <179191831+pamelap-nvidia@users.noreply.github.com> --- tensorrt_llm/_torch/modules/fused_moe/fused_moe_trtllm_gen.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tensorrt_llm/_torch/modules/fused_moe/fused_moe_trtllm_gen.py b/tensorrt_llm/_torch/modules/fused_moe/fused_moe_trtllm_gen.py index f844f846c12..2a8e1c30ea8 100644 --- a/tensorrt_llm/_torch/modules/fused_moe/fused_moe_trtllm_gen.py +++ b/tensorrt_llm/_torch/modules/fused_moe/fused_moe_trtllm_gen.py @@ -2,9 +2,11 @@ import torch +from tensorrt_llm._utils import get_sm_version + from ...distributed.ops import reducescatter from ...model_config import ModelConfig -from ...utils import Fp4QuantizedTensor, get_sm_version +from ...utils import Fp4QuantizedTensor from .interface import MoE, MoEWeightLoadingMode from .quantization import (DeepSeekFP8BlockScalesFusedMoEMethod, NVFP4TRTLLMGenFusedMoEMethod)