From 5a2dc861552b54159f5b0d38d179d5f7931b4e75 Mon Sep 17 00:00:00 2001 From: Ruodi Lu Date: Thu, 30 Jul 2026 05:55:23 +0000 Subject: [PATCH] [None][test] Add llm_perf_core coverage for DeepSeek-V4-Pro, GLM-5.2, MiniMax-M3 and Gemma-4 Four models that TensorRT LLM already supports had no coverage, or only partial coverage, in the QA perf core list: * DeepSeek-V4-Pro: only the DSpark speculative-decoding variant was listed, so the plain Pro checkpoint was untracked. Adds throughput and latency cases with the knobs from examples/configs/curated/deepseek-v4-pro-{throughput,latency}.yaml, at the 8K/1K fixed shape used for DeepSeek-V4 Pro in tech blog 26. * GLM-5.2: only glm_5_fp8 was listed. Adds the NVFP4 checkpoint with the CuteDSL MoE backend used by TestGLM52::test_nvfp4, mirroring the glm_5_fp8 ISL/OSL sweep so the two are directly comparable. Speculative decoding is intentionally left off so the sweep measures kernel time rather than MTP acceptance rate. * MiniMax-M3: the MXFP8 checkpoint is already covered at 4 GPUs. Adds the NVFP4 checkpoint (MXFP8 base layers with NVFP4 routed experts on CUTLASS) on the MSA sparse-attention path, at 4 GPUs and at the 8-GPU TP8/EP8 scale from docs/source/deployment-guide/deployment-guide-for-minimax-m3-on-trtllm.md. The MSA kernels are SM100/SM103 only, so the cases are restricted to B200/GB200/B300/GB300. The pattern is scoped to the NVFP4 label so the existing MXFP8 configuration is unchanged. ISL+OSL is kept at or below 2048 because the sparse path requires max_seq_len capped just above it, otherwise CUDA-graph capture fails on gigabyte-scale warmup temporaries. * Gemma-4: no perf coverage at all. Adds the NVFP4 MoE (26B-A4B) and dense (31B) checkpoints on a single GPU, reusing the existing gemma_4_26b_a4b_nvfp4 path entry and registering the 31B one alongside it. The FLASHINFER backend is selected by the model's own get_model_defaults(), so it is not repeated in the perf config. An 8000-token ISL is included to exercise the 1024-token sliding window. Coverage is limited to Blackwell because the model dispatches trtllm-gen cubins for all layers. Newer versions of these model families are not covered because the repository currently supports Kimi K2.5, Qwen3.5 and Nemotron 3. Signed-off-by: Ruodi Lu --- tests/integration/defs/perf/_model_paths.py | 6 + .../defs/perf/pytorch_model_config.py | 150 ++++++++++++++++++ tests/integration/defs/perf/test_perf.py | 1 + .../test_lists/qa/llm_perf_core.yml | 42 +++++ 4 files changed, 199 insertions(+) diff --git a/tests/integration/defs/perf/_model_paths.py b/tests/integration/defs/perf/_model_paths.py index c909a85ea3f6..0e40f784c9e5 100644 --- a/tests/integration/defs/perf/_model_paths.py +++ b/tests/integration/defs/perf/_model_paths.py @@ -46,6 +46,7 @@ "gemma_3_12b_it_fp4": "gemma/gemma-3-12b-it-fp4", "gemma_3_1b_it": "gemma/gemma-3-1b-it", "gemma_4_26b_a4b_nvfp4": "gemma/nvidia-Gemma-4-26B-A4B-NVFP4", + "gemma_4_31b_it_nvfp4": "gemma/nvidia-Gemma-4-31B-IT-NVFP4", "deepseek_r1_fp8": "DeepSeek-R1/DeepSeek-R1", "deepseek_r1_nvfp4": "DeepSeek-R1/DeepSeek-R1-FP4", "deepseek_r1_0528_fp8": "DeepSeek-R1/DeepSeek-R1-0528/", @@ -120,6 +121,11 @@ "deepseek_v4_pro_dspark": "DeepSeek-V4-Pro-DSpark", # GLM-5 FP8 (MoE) "glm_5_fp8": "GLM-5-FP8", + # GLM-5.2 NVFP4 (MoE, MLA + DSA on the DeepSeek-V3.2 code path) + "glm_5.2_fp4": "GLM-5.2-NVFP4", + # MiniMax-M3 NVFP4: MXFP8 base layers with NVFP4 routed experts + # (the MXFP8 checkpoint is registered as "minimax_m3_mxfp8" above). + "minimax_m3_fp4": "MiniMax-M3-NVFP4", # Kimi K2.5 NVFP4 (~1T MoE multimodal) "kimi_k2.5_fp4": "Kimi-K2.5-NVFP4", # Keys below are sanity-side aliases; some point to the same weights as diff --git a/tests/integration/defs/perf/pytorch_model_config.py b/tests/integration/defs/perf/pytorch_model_config.py index 00f1fe206a57..d0035576ae41 100644 --- a/tests/integration/defs/perf/pytorch_model_config.py +++ b/tests/integration/defs/perf/pytorch_model_config.py @@ -142,6 +142,156 @@ def get_model_yaml_config(model_label: str, }, } }, + # DeepSeek V4 Pro throughput knobs, from + # examples/configs/curated/deepseek-v4-pro-throughput.yaml (ADP + EP, + # small per-rank batch). MTP-1 matches the checkpoint's + # num_nextn_predict_layers and the curated throughput recipe. + { + 'patterns': [ + 'deepseek_v4_pro_fp4-bench-pytorch-float4-maxbs:32-maxnt:8448', + ], + 'config': { + 'enable_attention_dp': True, + 'enable_lm_head_tp_in_adp': True, + 'attention_dp_config': { + 'enable_balance': True, + }, + 'moe_config': { + 'backend': 'TRTLLM', + 'use_low_precision_moe_combine': True, + }, + 'max_seq_len': 9256, + 'kv_cache_config': { + 'dtype': 'fp8', + 'enable_block_reuse': False, + 'free_gpu_memory_fraction': 0.6, + 'tokens_per_block': 128, + }, + 'cuda_graph_config': { + 'enable_padding': True, + 'batch_sizes': [1, 2, 4, 8, 16, 24, 32], + }, + 'speculative_config': { + 'decoding_type': 'MTP', + 'max_draft_len': 1, + }, + 'stream_interval': 100, + 'num_postprocess_workers': 4, + } + }, + # DeepSeek V4 Pro latency knobs, from + # examples/configs/curated/deepseek-v4-pro-latency.yaml (pure TP, no + # attention DP, deeper MTP, larger KV fraction). + { + 'patterns': [ + 'deepseek_v4_pro_fp4-bench-pytorch-float4-maxbs:128-maxnt:8448', + ], + 'config': { + 'enable_attention_dp': False, + 'enable_lm_head_tp_in_adp': False, + 'moe_config': { + 'backend': 'TRTLLM', + 'use_low_precision_moe_combine': True, + }, + 'max_seq_len': 9256, + 'kv_cache_config': { + 'dtype': 'fp8', + 'enable_block_reuse': False, + 'free_gpu_memory_fraction': 0.9, + 'tokens_per_block': 128, + }, + 'cuda_graph_config': { + 'enable_padding': + True, + 'batch_sizes': [ + 1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, + 104, 112, 120, 128 + ], + }, + 'speculative_config': { + 'decoding_type': 'MTP', + 'max_draft_len': 3, + }, + 'stream_interval': 100, + 'num_postprocess_workers': 4, + } + }, + # GLM-5.2 NVFP4 reuses the DeepSeek-V3.2 MLA + DSA path with + # cross-layer indexer sharing; NVFP4 weights run on the CuteDSL MoE + # backend (see accuracy/test_llm_api_pytorch.py::TestGLM52). + # Spec decoding is intentionally left off so the sweep measures kernel + # time rather than MTP acceptance rate. + { + 'patterns': ['glm_5.2_fp4'], + 'config': { + 'enable_attention_dp': True, + 'enable_chunked_prefill': True, + 'moe_config': { + 'backend': 'CUTEDSL', + }, + 'max_seq_len': 10240, + 'kv_cache_config': { + 'free_gpu_memory_fraction': 0.7, + }, + 'cuda_graph_config': { + 'enable_padding': True, + 'max_batch_size': 128, + }, + } + }, + # MiniMax-M3 NVFP4 (MXFP8 base layers with NVFP4 routed experts, run on + # the CUTLASS MoE backend). Scoped to the NVFP4 checkpoint only; the + # MXFP8 checkpoint has its own block below and must not inherit the + # 'msa' implementation. + # The block-sparse attention path is mandatory (there is no dense + # fallback), it does not support KV-cache block reuse, and max_seq_len + # must be capped just above ISL+OSL -- left at the checkpoint default + # (1M) the CUDA-graph warmup decode allocates gigabyte-scale temporaries + # and capture fails with cudaErrorStreamCaptureUnsupported. Every NVFP4 + # case keeps ISL+OSL <= 2048. See + # docs/source/deployment-guide/deployment-guide-for-minimax-m3-on-trtllm.md + { + 'patterns': ['minimax_m3_fp4'], + 'config': { + 'trust_remote_code': True, + 'enable_attention_dp': True, + 'max_seq_len': 2560, + 'moe_config': { + 'backend': 'CUTLASS', + }, + 'sparse_attention_config': { + 'algorithm': 'minimax_m3', + # 'msa' is the fmha_sm100 path that the MiniMax-M3 perf + # work targets ('triton' is the reference implementation); + # it requires SM100/SM103, so these cases are Blackwell-only. + 'implementation': 'msa', + }, + 'kv_cache_config': { + 'enable_block_reuse': False, + 'free_gpu_memory_fraction': 0.6, + # The MSA path runs an FP8 KV cache. + 'dtype': 'fp8', + }, + 'stream_interval': 10, + 'num_postprocess_workers': 4, + } + }, + # Gemma 4 NVFP4: VSWA (1024-token sliding window) plus per-layer + # head_dim 256/512. The FLASHINFER backend is selected by the model's + # own get_model_defaults(), so it is not repeated here. KV-cache reuse + # is disabled to match the validated accuracy configuration. + { + 'patterns': ['gemma_4_'], + 'config': { + 'enable_chunked_prefill': True, + 'kv_cache_config': { + 'dtype': 'fp8', + 'enable_block_reuse': False, + 'enable_partial_reuse': False, + 'free_gpu_memory_fraction': 0.6, + }, + } + }, # DeepSeek R1 models with MTP speculative decoding { 'patterns': [ diff --git a/tests/integration/defs/perf/test_perf.py b/tests/integration/defs/perf/test_perf.py index 58e1be73cd35..512216ec3bce 100644 --- a/tests/integration/defs/perf/test_perf.py +++ b/tests/integration/defs/perf/test_perf.py @@ -61,6 +61,7 @@ "llama_v3.1_nemotron_ultra_253b_fp8", "kimi_k2_nvfp4", "kimi_k2.5_fp4", + "minimax_m3_fp4", "nemotron_3_super_120b_nvfp4", "nemotron_3_super_120b_nvfp4_mtp", "nemotron_3_ultra_550b_nvfp4", diff --git a/tests/integration/test_lists/qa/llm_perf_core.yml b/tests/integration/test_lists/qa/llm_perf_core.yml index ea30931f8b0a..d857795ca49b 100644 --- a/tests/integration/test_lists/qa/llm_perf_core.yml +++ b/tests/integration/test_lists/qa/llm_perf_core.yml @@ -209,6 +209,24 @@ llm_perf_core: #deepseek_v4_flash (post-trained, FP4+FP8 mixed, 4-GPU) - perf/test_perf.py::test_perf[deepseek_v4_flash-bench-pytorch-float4-maxbs:128-maxnt:4096-kv_frac:0.5-input_output_len:8192,1024-ep:4-tp:4-gpus:4] TIMEOUT(120) - perf/test_perf.py::test_perf[deepseek_v4_flash-bench-pytorch-float4-maxbs:128-maxnt:4096-kv_frac:0.5-input_output_len:1000,2000-ep:4-tp:4-gpus:4] TIMEOUT(120) + #minimax_m3_fp4 (block-sparse MoE, MXFP8 base + NVFP4 experts 233G, 4-GPU ep=4) + #mirrors accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_nvfp4[use_msa=True] (tp=ep=4). + #ISL+OSL is kept <= 2048 because the M3 sparse path caps max_seq_len for CUDA-graph capture. + - perf/test_perf.py::test_perf[minimax_m3_fp4-bench-pytorch-float4-maxbs:128-maxnt:4096-input_output_len:1024,1024-ep:4-tp:4-gpus:4] TIMEOUT(120) + - perf/test_perf.py::test_perf[minimax_m3_fp4-bench-pytorch-float4-maxbs:1-input_output_len:1024,1024-reqs:10-con:1-ep:4-tp:4-gpus:4] #min_latency + #gemma_4_26b_a4b_nvfp4 (Gemma4 MoE 26B-A4B NVFP4 18G, 1-GPU; VSWA sliding_window=1024) + - perf/test_perf.py::test_perf[gemma_4_26b_a4b_nvfp4-bench-pytorch-float4-input_output_len:128,128] + - perf/test_perf.py::test_perf[gemma_4_26b_a4b_nvfp4-bench-pytorch-float4-input_output_len:500,2000] + - perf/test_perf.py::test_perf[gemma_4_26b_a4b_nvfp4-bench-pytorch-float4-input_output_len:2000,500] + - perf/test_perf.py::test_perf[gemma_4_26b_a4b_nvfp4-bench-pytorch-float4-input_output_len:8000,1000] + - perf/test_perf.py::test_perf[gemma_4_26b_a4b_nvfp4-bench-pytorch-float4-maxbs:1-input_output_len:8000,1000-reqs:10-con:1] #min_latency + - perf/test_perf.py::test_perf[gemma_4_26b_a4b_nvfp4-bench-pytorch-float4-input_output_len:8000,1000-con:250] #max_throughput + #gemma_4_31b_it_nvfp4 (Gemma4 dense 31B NVFP4 31G, 1-GPU) + - perf/test_perf.py::test_perf[gemma_4_31b_it_nvfp4-bench-pytorch-float4-input_output_len:128,128] + - perf/test_perf.py::test_perf[gemma_4_31b_it_nvfp4-bench-pytorch-float4-input_output_len:500,2000] + - perf/test_perf.py::test_perf[gemma_4_31b_it_nvfp4-bench-pytorch-float4-input_output_len:2000,500] + - perf/test_perf.py::test_perf[gemma_4_31b_it_nvfp4-bench-pytorch-float4-maxbs:1-input_output_len:8000,1000-reqs:10-con:1] #min_latency + - perf/test_perf.py::test_perf[gemma_4_31b_it_nvfp4-bench-pytorch-float4-input_output_len:8000,1000-con:250] #max_throughput #llama_v3.1_nemotron_ultra_253b_fp8 (nemotron-nas FP8 241G, 4-GPU) - perf/test_perf.py::test_perf[llama_v3.1_nemotron_ultra_253b_fp8-bench-pytorch-float8-input_output_len:128,128-tp:4-gpus:4] - perf/test_perf.py::test_perf[llama_v3.1_nemotron_ultra_253b_fp8-bench-pytorch-float8-input_output_len:500,2000-tp:4-gpus:4] @@ -272,6 +290,12 @@ llm_perf_core: #deepseek_v4_flash (post-trained, FP4+FP8 mixed, 8-GPU) - perf/test_perf.py::test_perf[deepseek_v4_flash-bench-pytorch-float4-maxbs:128-maxnt:4096-kv_frac:0.5-input_output_len:8192,1024-ep:8-tp:8-gpus:8] TIMEOUT(120) - perf/test_perf.py::test_perf[deepseek_v4_flash-bench-pytorch-float4-maxbs:128-maxnt:4096-kv_frac:0.5-input_output_len:1000,2000-ep:8-tp:8-gpus:8] TIMEOUT(120) + #deepseek_v4_pro_fp4 (1.6T MoE, sparse MLA + mHC, FP4+FP8 mixed, 8-GPU) + #knobs from examples/configs/curated/deepseek-v4-pro-{throughput,latency}.yaml; + #8K/1K is the fixed-shape workload used in tech_blog26 for DeepSeek-V4 Pro. + - perf/test_perf.py::test_perf[deepseek_v4_pro_fp4-bench-pytorch-float4-maxbs:32-maxnt:8448-kv_frac:0.6-input_output_len:8192,1024-ep:8-tp:8-gpus:8] TIMEOUT(120) + - perf/test_perf.py::test_perf[deepseek_v4_pro_fp4-bench-pytorch-float4-maxbs:32-maxnt:8448-kv_frac:0.6-input_output_len:1000,2000-ep:8-tp:8-gpus:8] TIMEOUT(120) + - perf/test_perf.py::test_perf[deepseek_v4_pro_fp4-bench-pytorch-float4-maxbs:128-maxnt:8448-input_output_len:8192,1024-reqs:10-con:1-ep:8-tp:8-gpus:8] TIMEOUT(120) #min_latency #deepseek_v4_pro_dspark (DSpark speculative decoding, FP4+FP8 mixed, 8-GPU) - perf/test_perf.py::test_perf[deepseek_v4_pro_dspark-bench-pytorch-float4-maxbs:128-maxnt:9216-kv_frac:0.5-input_output_len:8192,1024-ep:8-tp:8-gpus:8] TIMEOUT(120) - perf/test_perf.py::test_perf[deepseek_v4_pro_dspark-bench-pytorch-float4-maxbs:128-maxnt:4096-kv_frac:0.5-input_output_len:1000,2000-ep:8-tp:8-gpus:8] TIMEOUT(120) @@ -291,6 +315,24 @@ llm_perf_core: - perf/test_perf.py::test_perf[glm_5_fp8-bench-pytorch-float8-input_output_len:1000,2000-ep:8-tp:8-gpus:8] TIMEOUT(120) - perf/test_perf.py::test_perf[glm_5_fp8-bench-pytorch-float8-maxbs:1-input_output_len:8000,1000-reqs:10-con:1-ep:8-tp:8-gpus:8] #min_latency - perf/test_perf.py::test_perf[glm_5_fp8-bench-pytorch-float8-maxbs:512-maxnt:2048-kv_frac:0.6-input_output_len:8000,1000-con:512-ep:8-tp:8-gpus:8] TIMEOUT(120) #max_throughput + #glm_5.2_fp4 (MoE NVFP4 433G, MLA + DSA on the DeepSeek-V3.2 path, 8-GPU ep=8) + - perf/test_perf.py::test_perf[glm_5.2_fp4-bench-pytorch-float4-input_output_len:128,128-ep:8-tp:8-gpus:8] + - perf/test_perf.py::test_perf[glm_5.2_fp4-bench-pytorch-float4-input_output_len:500,2000-ep:8-tp:8-gpus:8] TIMEOUT(120) + - perf/test_perf.py::test_perf[glm_5.2_fp4-bench-pytorch-float4-input_output_len:2000,500-ep:8-tp:8-gpus:8] TIMEOUT(120) + - perf/test_perf.py::test_perf[glm_5.2_fp4-bench-pytorch-float4-input_output_len:8000,1000-ep:8-tp:8-gpus:8] TIMEOUT(120) + - perf/test_perf.py::test_perf[glm_5.2_fp4-bench-pytorch-float4-input_output_len:1000,2000-ep:8-tp:8-gpus:8] TIMEOUT(120) + - perf/test_perf.py::test_perf[glm_5.2_fp4-bench-pytorch-float4-maxbs:1-input_output_len:8000,1000-reqs:10-con:1-ep:8-tp:8-gpus:8] #min_latency + - perf/test_perf.py::test_perf[glm_5.2_fp4-bench-pytorch-float4-maxbs:512-maxnt:2048-kv_frac:0.6-input_output_len:8000,1000-con:512-ep:8-tp:8-gpus:8] TIMEOUT(120) #max_throughput + #minimax_m3_fp4 (block-sparse MoE, 8-GPU ep=8 + attention DP, matching + #docs/source/deployment-guide/deployment-guide-for-minimax-m3-on-trtllm.md). + #The MXFP8 checkpoint is already covered at 4-GPU in condition 6. + #ISL+OSL stays <= 2048: the M3 sparse path needs max_seq_len capped just above it + #or CUDA-graph capture fails on gigabyte-scale warmup temporaries. + - perf/test_perf.py::test_perf[minimax_m3_fp4-bench-pytorch-float4-maxbs:256-maxnt:8192-input_output_len:1024,1024-ep:8-tp:8-gpus:8] TIMEOUT(120) + - perf/test_perf.py::test_perf[minimax_m3_fp4-bench-pytorch-float4-maxbs:256-maxnt:8192-input_output_len:2000,500-ep:8-tp:8-gpus:8] TIMEOUT(120) + - perf/test_perf.py::test_perf[minimax_m3_fp4-bench-pytorch-float4-maxbs:256-maxnt:8192-input_output_len:500,2000-ep:8-tp:8-gpus:8] TIMEOUT(120) + - perf/test_perf.py::test_perf[minimax_m3_fp4-bench-pytorch-float4-maxbs:1-input_output_len:1024,1024-reqs:10-con:1-ep:8-tp:8-gpus:8] #min_latency + - perf/test_perf.py::test_perf[minimax_m3_fp4-bench-pytorch-float4-maxbs:256-maxnt:8192-input_output_len:1024,1024-con:256-ep:8-tp:8-gpus:8] TIMEOUT(120) #max_throughput #deepseek_v3.2_fp4 (FP4 389G, 8-GPU ep=8) - perf/test_perf.py::test_perf[deepseek_r1_0528_fp4-bench-pytorch-float4-maxbs:384-maxnt:1536-input_output_len:1000,2000-reqs:10000-con:3072-ep:8-tp:8-gpus:8] TIMEOUT(120) #max throughput test - perf/test_perf.py::test_perf[deepseek_v3.2_fp4-bench-pytorch-float4-input_output_len:128,128-ep:8-tp:8-gpus:8]