diff --git a/tests/integration/defs/perf/_model_paths.py b/tests/integration/defs/perf/_model_paths.py index e69395611cec..522e9eb87dc2 100644 --- a/tests/integration/defs/perf/_model_paths.py +++ b/tests/integration/defs/perf/_model_paths.py @@ -22,12 +22,7 @@ "llama_v3.3_70b_instruct": "llama-3.3-models/Llama-3.3-70B-Instruct", "llama_v3.3_70b_instruct_fp8": "modelopt-hf-model-hub/Llama-3.3-70B-Instruct-fp8", "llama_v3.3_70b_instruct_fp4": "modelopt-hf-model-hub/Llama-3.3-70B-Instruct-fp4", - "llama_v3.1_nemotron_nano_8b_fp8": "Llama-3.1-Nemotron-Nano-8B-v1-FP8", - "llama_v3.3_nemotron_super_49b": "nemotron-nas/Llama-3_3-Nemotron-Super-49B-v1", - "llama_v3.3_nemotron_super_49b_fp8": "nemotron-nas/Llama-3_3-Nemotron-Super-49B-v1-FP8", "llama_v3.3_nemotron_super_49b_v1.5_fp8": "nemotron-nas/Llama-3_3-Nemotron-Super-49B-v1_5-FP8", - "llama_v3.1_nemotron_ultra_253b": "nemotron-nas/Llama-3_1-Nemotron-Ultra-253B-v1", - "llama_v3.1_nemotron_ultra_253b_fp8": "nemotron-nas/Llama-3_1-Nemotron-Ultra-253B-v1-FP8", "llama_v4_scout_17b_16e_instruct": "llama4-models/Llama-4-Scout-17B-16E-Instruct", "llama_v4_scout_17b_16e_instruct_fp8": "llama4-models/Llama-4-Scout-17B-16E-Instruct-FP8", "llama_v4_scout_17b_16e_instruct_fp4": "llama4-models/Llama-4-Scout-17B-16E-Instruct-FP4", @@ -42,8 +37,6 @@ "gemma_3_1b_it": "gemma/gemma-3-1b-it", "gemma_4_26b_a4b_nvfp4": "gemma/nvidia-Gemma-4-26B-A4B-NVFP4", "gemma_4_31b_it_nvfp4": "gemma/nvidia-Gemma-4-31B-IT-NVFP4", - "deepseek_r1_fp8": "DeepSeek-R1/DeepSeek-R1", - "deepseek_r1_nvfp4": "DeepSeek-R1/DeepSeek-R1-FP4", "deepseek_r1_0528_fp8": "DeepSeek-R1/DeepSeek-R1-0528/", "deepseek_r1_0528_fp4": "DeepSeek-R1/DeepSeek-R1-0528-FP4/", "deepseek_r1_0528_fp4_v2": "DeepSeek-R1/DeepSeek-R1-0528-FP4-v2/", @@ -85,9 +78,6 @@ # Nemotron-3-Nano-Omni-30B (text + image multimodal) "nemotron_3_nano_omni_nvfp4": "NVIDIA-Nemotron-3-Nano-Omni-30B-A3B-Reasoning-NVFP4", "nemotron_3_nano_omni_nvfp4_image": "NVIDIA-Nemotron-3-Nano-Omni-30B-A3B-Reasoning-NVFP4", - "kimi_k2_nvfp4": "Kimi-K2-Thinking-NVFP4", - # MiniMax M2.5 (FP8 block-scale, ~230B MoE) - "minimax_m2.5_fp8": "MiniMax-M2.5", # MiniMax M3 (block-sparse MoE, MXFP8 weights, BF16 activations + KV cache) "minimax_m3_mxfp8": "MiniMax-M3-MXFP8", # Qwen3.5 dense + MoE @@ -99,9 +89,6 @@ "qwen3.5_397b_a17b_fp4": "Qwen3.5-397B-A17B-NVFP4", # Qwen3.6 (GDN linear-attn MoE, NVFP4) "qwen3.6_35b_a3b_fp4": "Qwen3.6-35B-A3B-NVFP4", - # DeepSeek V3.2 (671B MoE) - "deepseek_v3.2_fp8": "DeepSeek-V3.2-hf", - "deepseek_v3.2_fp4": "DeepSeek-V3.2-NVFP4", # DeepSeek V4 "deepseek_v4_pro_fp4": "DeepSeek-V4-Pro", "deepseek_v4_flash": "DeepSeek-V4-Flash", diff --git a/tests/integration/defs/perf/pytorch_model_config.py b/tests/integration/defs/perf/pytorch_model_config.py index 8112b3cf17a4..1bb2b623fdf5 100644 --- a/tests/integration/defs/perf/pytorch_model_config.py +++ b/tests/integration/defs/perf/pytorch_model_config.py @@ -84,7 +84,7 @@ def get_model_yaml_config(model_label: str, pattern_configs = [ # Deepseek default cases { - 'patterns': ['deepseek_r1', 'kimi_k2_nvfp4'], + 'patterns': ['deepseek_r1'], 'config': { 'enable_attention_dp': True, } @@ -291,151 +291,10 @@ def get_model_yaml_config(model_label: str, }, } }, - # DeepSeek R1 models with MTP speculative decoding - { - 'patterns': [ - 'deepseek_r1-bench-pytorch-float16-maxbs:1-maxnt:8192-input_output_len:1000,2000-reqs:10-ep:4-gpus:8', - 'deepseek_r1_nvfp4-bench-pytorch-float16-maxbs:1-maxnt:8192-input_output_len:1000,2000-reqs:10-ep:4-tp:8-gpus:8' - ], - 'config': { - 'enable_attention_dp': True, - 'cuda_graph_config': {}, - 'speculative_config': { - 'decoding_type': 'MTP', - 'max_draft_len': 3 - } - } - }, - { - 'patterns': [ - 'deepseek_r1_nvfp4-bench-pytorch-float4-maxbs:32-maxnt:32768-input_output_len:8192,1024-reqs:20-con:1-ep:1-gpus:4' - ], - 'config': { - 'enable_iter_perf_stats': True, - 'print_iter_log': False, - 'cuda_graph_config': { - 'max_batch_size': 16, - 'enable_padding': False - }, - 'moe_config': { - 'max_num_tokens': 32768 - }, - 'speculative_config': { - 'decoding_type': 'MTP', - 'max_draft_len': 3 - }, - 'disable_overlap_scheduler': True, - 'enable_autotuner': True, - 'kv_cache_config': { - 'free_gpu_memory_fraction': 0.6, - 'enable_block_reuse': True, - 'enable_partial_reuse': False - }, - 'enable_chunked_prefill': True - } - }, - # DeepSeek R1 models with large batch sizes and cuda graph padding - { - 'patterns': [ - 'deepseek_r1_fp8-bench-pytorch-float16-maxbs:384-maxnt:1536-input_output_len:1000,2000-reqs:49152-con:3072-ep:8-gpus:8', - 'deepseek_r1_nvfp4-bench-pytorch-float16-maxbs:384-maxnt:1536-input_output_len:1000,2000-reqs:49152-con:3072-ep:8-gpus:8' - ], - 'config': { - 'enable_attention_dp': True, - 'cuda_graph_config': { - 'enable_padding': True, - 'batch_sizes': [1, 2, 4, 8, 16, 32, 64, 128, 256, 384] - } - } - }, - # DeepSeek R1 model with specific batch size 128 - { - 'patterns': - 'deepseek_r1_fp8-bench-pytorch-float16-maxbs:128-maxnt:1127-input_output_len:1000,2000-reqs:5120-con:1024-ep:8-gpus:8', - 'config': { - 'enable_attention_dp': True, - 'cuda_graph_config': { - 'batch_sizes': [128] - } - } - }, - # Deepseek R1 model with chunked prefill - { - 'patterns': [ - 'deepseek_r1_fp8-bench-pytorch-float8-maxbs:512-maxnt:2048-kv_frac:0.85-input_output_len:5000,500', - 'deepseek_r1_fp8-bench-pytorch-float8-maxbs:256-maxnt:1024-kv_frac:0.85-input_output_len:2000,2000', - 'deepseek_v3_lite_fp8-bench-pytorch-float8-maxbs:512-maxnt:2048-kv_frac:0.85-input_output_len:5000,500', - 'deepseek_v3_lite_nvfp4-bench-pytorch-float4-maxbs:512-maxnt:2048-kv_frac:0.85-input_output_len:5000,500', - 'deepseek_r1_nvfp4-bench-pytorch-float4-maxbs:512-maxnt:2048-kv_frac:0.85-input_output_len:5000,500', - 'deepseek_r1_nvfp4-bench-pytorch-float4-maxbs:256-maxnt:1024-kv_frac:0.85-input_output_len:2000,2000', - ], - 'config': { - 'enable_attention_dp': True, - 'enable_chunked_prefill': True, - } - }, - # Deepseek R1 NVFP4 with chunked prefill, large seq len, and fp8 KV cache - { - 'patterns': [ - 'deepseek_r1_nvfp4-bench-pytorch-float4-maxbs:32-maxnt:4096-kv_frac:0.80-input_output_len:8192,512-reqs:3000-ep:2-tp:4-gpus:4', - ], - 'config': { - 'enable_attention_dp': True, - 'enable_chunked_prefill': True, - 'max_num_tokens': 4096, - 'max_batch_size': 32, - 'max_seq_len': 81920, - 'kv_cache_config': { - 'dtype': 'fp8', - 'free_gpu_memory_fraction': 0.80, - 'enable_block_reuse': False, - }, - 'cuda_graph_config': { - 'enable_padding': True, - 'max_batch_size': 32, - }, - } - }, - # Deepseek R1 model with CUTLASS backend - { - 'patterns': [ - 'deepseek_r1_nvfp4-bench-pytorch-streaming-float4-maxbs:512-maxnt:5220-input_output_len:4000,2000', - ], - 'config': { - 'enable_attention_dp': True, - 'moe_config': { - 'backend': 'CUTLASS', - 'max_num_tokens': 3072, - }, - 'kv_cache_config': { - 'dtype': 'fp8', - 'free_gpu_memory_fraction': 0.5, - }, - 'cuda_graph_config': { - 'enable_padding': True, - 'batch_sizes': [1, 2, 4, 8, 16, 32, 64], - }, - } - }, - # Deepseek_v3_lite_cases - { - 'patterns': - 'deepseek_v3_lite_nvfp4-bench-pytorch-streaming-float4-maxbs:2048-maxnt:8192-input_output_len:256,256-reqs:200', - 'config': { - 'print_iter_log': True, - 'cuda_graph_config': { - 'enable_padding': True, - 'batch_sizes': [1, 512, 1024, 2048] - } - } - }, # Model-specific cases with attention_dp disabled to prevent hangs { 'patterns': [ 'deepseek_r1_distill_llama_70b', - 'llama_v3.1_nemotron_ultra_253b_fp8-bench-pytorch-float8', - 'llama_v3.3_nemotron_super_49b_fp8-bench-pytorch-float8', - 'llama_v3.3_nemotron_super_49b-bench-pytorch-bfloat16' ], 'config': { # True causes hang, needs model-specific fix. @@ -465,16 +324,6 @@ def get_model_yaml_config(model_label: str, }, } }, - # MiniMax-M2.5 FP8: every perf case must route MoE through attention DP. - # TP=8: intermediate_size=1536 is not block-scale divisible (1536/8=192, %128!=0). - # TP=4: trtllm-gen FP8 block-scale MoE kernel IMAs during CUDA-graph warmup - # on the 1536/4=384 N-shard (Blackwell B200/B300). - { - 'patterns': ['minimax_m2.5_fp8'], - 'config': { - 'enable_attention_dp': True, - } - }, # MiniMax-M3 MXFP8 block-sparse MoE: sparse backend, no KV reuse, trust_remote_code, capped max_seq_len to avoid the 1M-default CUDA-graph OOM. { 'patterns': ['minimax_m3_mxfp8'], diff --git a/tests/integration/defs/perf/test_perf.py b/tests/integration/defs/perf/test_perf.py index 512216ec3bce..9cc7fc6429a0 100644 --- a/tests/integration/defs/perf/test_perf.py +++ b/tests/integration/defs/perf/test_perf.py @@ -55,11 +55,6 @@ } TRUST_REMOTE_CODE_MODELS = { # these models require explicit trust_remote_code=True - "llama_v3.3_nemotron_super_49b", - "llama_v3.3_nemotron_super_49b_fp8", - "llama_v3.1_nemotron_ultra_253b", - "llama_v3.1_nemotron_ultra_253b_fp8", - "kimi_k2_nvfp4", "kimi_k2.5_fp4", "minimax_m3_fp4", "nemotron_3_super_120b_nvfp4", diff --git a/tests/integration/test_lists/qa/llm_perf_core.yml b/tests/integration/test_lists/qa/llm_perf_core.yml index e5dc74c7232e..bdf71033294d 100644 --- a/tests/integration/test_lists/qa/llm_perf_core.yml +++ b/tests/integration/test_lists/qa/llm_perf_core.yml @@ -10,9 +10,10 @@ llm_perf_core: # 5: GB200, B200, B300, GB300, RTX6000-Server test cases # 6: B200, GB200, B300, GB300 test cases # 7: B200, B300 test cases -# 8: H100, H20, B200, B300, RTX6000-Server test cases -# 9: H20, B200, B300, RTX6000-Server test cases -# 10: RTX-6000 Server test cases +# 8: B300 test cases +# 9: H100, H20, B200, B300, RTX6000-Server test cases +# 10: H20, B200, B300, RTX6000-Server test cases +# 11: RTX-6000 Server test cases # =============================================================================== @@ -24,16 +25,12 @@ llm_perf_core: system_gpu_count: gte: 4 tests: - - perf/test_perf.py::test_perf[llama_v3.1_8b_instruct_fp8-bench-pytorch-float8-input_output_len:128,128] - perf/test_perf.py::test_perf[qwen3.5_9b-bench-pytorch-bfloat16-input_output_len:128,128] - perf/test_perf.py::test_perf[qwen3.5_9b-bench-pytorch-bfloat16-input_output_len:500,2000] - perf/test_perf.py::test_perf[qwen3.5_9b-bench-pytorch-bfloat16-input_output_len:2000,500] - perf/test_perf.py::test_perf[qwen3.5_9b-bench-pytorch-bfloat16-input_output_len:1000,2000] - perf/test_perf.py::test_perf[qwen3.5_9b-bench-pytorch-bfloat16-input_output_len:8000,1000] - perf/test_perf.py::test_perf[qwen3_4b_eagle3-bench-pytorch-streaming-bfloat16-maxbs:4-kv_frac:0.6-input_output_len:500,100-reqs:200-con:4] - - perf/test_perf.py::test_perf[llama_v3.1_nemotron_nano_8b_fp8-bench-pytorch-float8-maxnt:5000-input_output_len:5000,500-reqs:8-con:1] - - perf/test_perf.py::test_perf[llama_v3.1_nemotron_nano_8b_fp8-bench-pytorch-float8-input_output_len:500,2000-reqs:8-con:1] - - perf/test_perf.py::test_perf[llama_v3.1_nemotron_nano_8b_fp8-bench-pytorch-float8-input_output_len:8000,1000-reqs:8-con:1] # test overlap scheduler # con:1 paired with a small model is an intentional design choice—it amplifies host-side overhead and simplifies execution timelines to the maximum extent. - perf/test_perf.py::test_perf[qwen3_0.6b-bench-pytorch-bfloat16-maxnt:2048-input_output_len:8000,1000-reqs:64-con:1-pp:4-gpus:4] @@ -53,15 +50,6 @@ llm_perf_core: - perf/test_perf.py::test_perf[nemotron_nano_12b_v2-bench-pytorch-bfloat16-maxbs:1-input_output_len:128,128-reqs:10-con:1] #min_latency #qwen3.5_27b (dense BF16 52G, 2-GPU) - perf/test_perf.py::test_perf[qwen3.5_27b-bench-pytorch-bfloat16-maxbs:1-input_output_len:8000,1000-reqs:10-con:1-tp:2-gpus:2] #min_latency - - perf/test_perf.py::test_perf[llama_v3.3_70b_instruct-bench-pytorch-streaming-bfloat16-input_output_len:128,128-gpus:4] - - perf/test_perf.py::test_perf[llama_v3.3_70b_instruct-bench-pytorch-bfloat16-input_output_len:128,128-gpus:4] - - perf/test_perf.py::test_perf[llama_v3.3_70b_instruct_fp8-bench-pytorch-streaming-float8-input_output_len:2000,200-gpus:8] - - perf/test_perf.py::test_perf[llama_v3.3_70b_instruct_fp8-bench-pytorch-float8-input_output_len:128,128-gpus:8] - #llama_v3.3_nemotron_super_49b (nemotron-nas arch, BF16 94G) - - perf/test_perf.py::test_perf[llama_v3.3_nemotron_super_49b-bench-pytorch-bfloat16-input_output_len:128,128-tp:4-gpus:4] - - perf/test_perf.py::test_perf[llama_v3.3_nemotron_super_49b-bench-pytorch-bfloat16-input_output_len:500,2000-tp:4-gpus:4] - - perf/test_perf.py::test_perf[llama_v3.3_nemotron_super_49b-bench-pytorch-bfloat16-input_output_len:2000,500-tp:4-gpus:4] - - perf/test_perf.py::test_perf[llama_v3.3_nemotron_super_49b-bench-pytorch-bfloat16-input_output_len:1000,2000-tp:4-gpus:4] # 3: H100, H20 test cases @@ -75,7 +63,6 @@ llm_perf_core: gpu_memory: gt: 80000 tests: - - perf/test_perf.py::test_perf[llama_v3.3_70b_instruct-bench-pytorch-bfloat16-input_output_len:2000,500-gpus:8] # gpt_oss_20b_fp4 (NVBug 5720470: MMHA vs XQA kernel regression) - perf/test_perf.py::test_perf[gpt_oss_20b_fp4-bench-pytorch-float4-maxbs:512-maxnt:8192-input_output_len:2000,200-con:64] - perf/test_perf.py::test_perf[gpt_oss_20b_fp4-bench-pytorch-float4-maxbs:512-maxnt:8192-input_output_len:128,128] @@ -89,17 +76,6 @@ llm_perf_core: - perf/test_perf.py::test_perf[qwen3.5_27b-bench-pytorch-bfloat16-input_output_len:8000,1000-tp:2-gpus:2] - perf/test_perf.py::test_perf[qwen3.5_27b-bench-pytorch-bfloat16-input_output_len:1000,2000-tp:2-gpus:2] - perf/test_perf.py::test_perf[qwen3.5_27b-bench-pytorch-bfloat16-input_output_len:8000,1000-con:250-tp:2-gpus:2] #max_throughput - #llama_v3.3_nemotron_super_49b (nemotron-nas BF16 94G, 2-GPU) - - perf/test_perf.py::test_perf[llama_v3.3_nemotron_super_49b-bench-pytorch-bfloat16-input_output_len:128,128-tp:2-gpus:2] - - perf/test_perf.py::test_perf[llama_v3.3_nemotron_super_49b-bench-pytorch-bfloat16-input_output_len:500,2000-tp:2-gpus:2] - - perf/test_perf.py::test_perf[llama_v3.3_nemotron_super_49b-bench-pytorch-bfloat16-input_output_len:2000,500-tp:2-gpus:2] - - perf/test_perf.py::test_perf[llama_v3.3_nemotron_super_49b-bench-pytorch-bfloat16-input_output_len:8000,1000-tp:2-gpus:2] - - perf/test_perf.py::test_perf[llama_v3.3_nemotron_super_49b-bench-pytorch-bfloat16-input_output_len:1000,2000-tp:2-gpus:2] - - perf/test_perf.py::test_perf[llama_v3.3_nemotron_super_49b-bench-pytorch-bfloat16-maxbs:1-input_output_len:8000,1000-reqs:10-con:1-tp:2-gpus:2] #min_latency - - perf/test_perf.py::test_perf[llama_v3.3_nemotron_super_49b-bench-pytorch-bfloat16-input_output_len:8000,1000-con:250-tp:2-gpus:2] #max_throughput - #llama_v3.3_nemotron_super_49b (nemotron-nas BF16 94G, 4-GPU): too slow on L40S PCIe-only interconnect - - perf/test_perf.py::test_perf[llama_v3.3_nemotron_super_49b-bench-pytorch-bfloat16-input_output_len:8000,1000-tp:4-gpus:4] - - perf/test_perf.py::test_perf[llama_v3.3_nemotron_super_49b-bench-pytorch-bfloat16-input_output_len:8000,1000-con:250-tp:4-gpus:4] #max_throughput # 4: H100, H20, GB200, B200, B300, GB300, RTX6000-Server test cases @@ -119,13 +95,7 @@ llm_perf_core: - perf/test_perf.py::test_perf[nemotron_nano_12b_v2-bench-pytorch-bfloat16-input_output_len:8000,1000-con:250] #max_throughput - perf/test_perf.py::test_perf[nemotron_nano_12b_v2-bench-pytorch-streaming-bfloat16-input_output_len:500,2000-con:250] #max_throughput streaming #qwen3.5_27b (dense BF16 52G, 1-GPU) - - perf/test_perf.py::test_perf[qwen3.5_27b-bench-pytorch-bfloat16-input_output_len:128,128] - - perf/test_perf.py::test_perf[qwen3.5_27b-bench-pytorch-bfloat16-input_output_len:500,2000] - - perf/test_perf.py::test_perf[qwen3.5_27b-bench-pytorch-bfloat16-input_output_len:2000,500] - - perf/test_perf.py::test_perf[qwen3.5_27b-bench-pytorch-bfloat16-input_output_len:8000,1000] - - perf/test_perf.py::test_perf[qwen3.5_27b-bench-pytorch-bfloat16-input_output_len:1000,2000] - perf/test_perf.py::test_perf[qwen3.5_27b-bench-pytorch-bfloat16-maxbs:1-input_output_len:8000,1000-reqs:10-con:1] #min_latency - - perf/test_perf.py::test_perf[qwen3.5_27b-bench-pytorch-bfloat16-input_output_len:8000,1000-con:250] #max_throughput #qwen3.5_35b_a3b_fp8 (MoE FP8 36G, 1-GPU) - perf/test_perf.py::test_perf[qwen3.5_35b_a3b_fp8-bench-pytorch-float8-input_output_len:128,128] - perf/test_perf.py::test_perf[qwen3.5_35b_a3b_fp8-bench-pytorch-float8-input_output_len:500,2000] @@ -134,16 +104,7 @@ llm_perf_core: - perf/test_perf.py::test_perf[qwen3.5_35b_a3b_fp8-bench-pytorch-float8-input_output_len:1000,2000] - perf/test_perf.py::test_perf[qwen3.5_35b_a3b_fp8-bench-pytorch-float8-maxbs:1-input_output_len:8000,1000-reqs:10-con:1] #min_latency - perf/test_perf.py::test_perf[qwen3.5_35b_a3b_fp8-bench-pytorch-float8-maxbs:512-input_output_len:8000,1000-con:256] #max_throughput - #llama_v3.3_nemotron_super_49b_fp8 (nemotron-nas FP8 49G, 2-GPU for safety) - - perf/test_perf.py::test_perf[llama_v3.3_nemotron_super_49b_fp8-bench-pytorch-float8-input_output_len:128,128-tp:2-gpus:2] - - perf/test_perf.py::test_perf[llama_v3.3_nemotron_super_49b_fp8-bench-pytorch-float8-input_output_len:500,2000-tp:2-gpus:2] - - perf/test_perf.py::test_perf[llama_v3.3_nemotron_super_49b_fp8-bench-pytorch-float8-input_output_len:2000,500-tp:2-gpus:2] - - perf/test_perf.py::test_perf[llama_v3.3_nemotron_super_49b_fp8-bench-pytorch-float8-input_output_len:8000,1000-tp:2-gpus:2] - - perf/test_perf.py::test_perf[llama_v3.3_nemotron_super_49b_fp8-bench-pytorch-float8-input_output_len:1000,2000-tp:2-gpus:2] - - perf/test_perf.py::test_perf[llama_v3.3_nemotron_super_49b_fp8-bench-pytorch-float8-maxbs:1-input_output_len:8000,1000-reqs:10-con:1-tp:2-gpus:2] #min_latency - - perf/test_perf.py::test_perf[llama_v3.3_nemotron_super_49b_fp8-bench-pytorch-float8-input_output_len:8000,1000-con:250-tp:2-gpus:2] #max_throughput #qwen3.5_122b_a10b (MoE BF16 234G, 4-GPU) - - perf/test_perf.py::test_perf[qwen3.5_122b_a10b-bench-pytorch-bfloat16-input_output_len:500,2000-ep:4-tp:4-gpus:4] - perf/test_perf.py::test_perf[qwen3.5_122b_a10b-bench-pytorch-bfloat16-input_output_len:2000,500-ep:4-tp:4-gpus:4] - perf/test_perf.py::test_perf[qwen3.5_122b_a10b-bench-pytorch-bfloat16-input_output_len:8000,1000-ep:4-tp:4-gpus:4] - perf/test_perf.py::test_perf[qwen3.5_122b_a10b-bench-pytorch-bfloat16-input_output_len:1000,2000-ep:4-tp:4-gpus:4] @@ -161,8 +122,6 @@ llm_perf_core: gte: 10.0 lte: 12.0 tests: - - perf/test_perf.py::test_perf[llama_v3.3_70b_instruct_fp8-bench-pytorch-streaming-float8-maxbs:256-input_output_len:512,32-gpus:2] - - perf/test_perf.py::test_perf[llama_v3.3_70b_instruct_fp4-bench-pytorch-float4-input_output_len:128,128-gpus:4] - perf/test_perf.py::test_perf[qwen3_235b_a22b_fp4-bench-pytorch-float4-input_output_len:1000,2000-con:512-ep:4-gpus:4] @@ -175,6 +134,13 @@ llm_perf_core: gte: 10.0 lte: 10.3 tests: + #qwen3.5_27b (dense BF16 52G, 1-GPU): maxbs 512 mamba states need >95G + - perf/test_perf.py::test_perf[qwen3.5_27b-bench-pytorch-bfloat16-input_output_len:128,128] + - perf/test_perf.py::test_perf[qwen3.5_27b-bench-pytorch-bfloat16-input_output_len:500,2000] + - perf/test_perf.py::test_perf[qwen3.5_27b-bench-pytorch-bfloat16-input_output_len:2000,500] + - perf/test_perf.py::test_perf[qwen3.5_27b-bench-pytorch-bfloat16-input_output_len:8000,1000] + - perf/test_perf.py::test_perf[qwen3.5_27b-bench-pytorch-bfloat16-input_output_len:1000,2000] + - perf/test_perf.py::test_perf[qwen3.5_27b-bench-pytorch-bfloat16-input_output_len:8000,1000-con:250] #max_throughput # deepseek_r1_0528_fp4 with chunked prefill - perf/test_perf.py::test_perf[deepseek_r1_0528_fp4-bench-pytorch-float4-maxbs:512-maxnt:2048-kv_frac:0.85-input_output_len:5000,500-reqs:200-ep:4-tp:4-gpus:4] TIMEOUT(120) - perf/test_perf.py::test_perf[deepseek_r1_0528_fp4-bench-pytorch-float4-maxbs:256-maxnt:1024-kv_frac:0.85-input_output_len:2000,2000-reqs:200-ep:4-tp:4-gpus:4] TIMEOUT(120) @@ -185,13 +151,6 @@ llm_perf_core: - perf/test_perf.py::test_perf[deepseek_r1_0528_fp4-bench-pytorch-float4-maxbs:1000-maxnt:5000-kv_frac:0.85-input_output_len:5000,500-reqs:20000-ep:4-tp:4-gpus:4] TIMEOUT(120) - perf/test_perf.py::test_perf[deepseek_r1_0528_fp4-bench-pytorch-float4-maxbs:512-input_output_len:128,128-ep:4-tp:4-gpus:4] - perf/test_perf.py::test_perf[deepseek_r1_0528_fp4-bench-pytorch-streaming-float4-maxbs:512-input_output_len:128,128-ep:4-tp:4-gpus:4] - #minimax_m2.5 (FP8 216G, 4-GPU) - - perf/test_perf.py::test_perf[minimax_m2.5_fp8-bench-pytorch-float8-input_output_len:128,128-tp:4-gpus:4] - - perf/test_perf.py::test_perf[minimax_m2.5_fp8-bench-pytorch-float8-input_output_len:500,2000-tp:4-gpus:4] - - perf/test_perf.py::test_perf[minimax_m2.5_fp8-bench-pytorch-float8-input_output_len:2000,500-tp:4-gpus:4] - - perf/test_perf.py::test_perf[minimax_m2.5_fp8-bench-pytorch-float8-input_output_len:8000,1000-tp:4-gpus:4] - - perf/test_perf.py::test_perf[minimax_m2.5_fp8-bench-pytorch-float8-maxbs:1-input_output_len:8000,1000-reqs:10-con:1-tp:4-gpus:4] #min_latency - - perf/test_perf.py::test_perf[minimax_m2.5_fp8-bench-pytorch-float8-maxbs:512-input_output_len:8000,1000-con:256-tp:4-gpus:4] #max_throughput #minimax_m3 (MXFP8 block-sparse MoE, 4-GPU) - perf/test_perf.py::test_perf[minimax_m3_mxfp8-bench-pytorch-float8-input_output_len:128,128-tp:4-gpus:4] - perf/test_perf.py::test_perf[minimax_m3_mxfp8-bench-pytorch-float8-input_output_len:500,2000-tp:4-gpus:4] @@ -199,10 +158,6 @@ llm_perf_core: - perf/test_perf.py::test_perf[minimax_m3_mxfp8-bench-pytorch-float8-input_output_len:8000,1000-tp:4-gpus:4] - perf/test_perf.py::test_perf[minimax_m3_mxfp8-bench-pytorch-float8-maxbs:1-input_output_len:8000,1000-reqs:10-con:1-tp:4-gpus:4] #min_latency - perf/test_perf.py::test_perf[minimax_m3_mxfp8-bench-pytorch-float8-maxbs:512-input_output_len:8000,1000-con:256-tp:4-gpus:4] #max_throughput - #deepseek_v3.2_fp4 (FP4 389G, 4-GPU) - - perf/test_perf.py::test_perf[deepseek_v3.2_fp4-bench-pytorch-float4-input_output_len:128,128-ep:4-tp:4-gpus:4] - - perf/test_perf.py::test_perf[deepseek_v3.2_fp4-bench-pytorch-float4-maxbs:1-input_output_len:8000,1000-reqs:10-con:1-ep:4-tp:4-gpus:4] #min_latency - - perf/test_perf.py::test_perf[deepseek_v3.2_fp4-bench-pytorch-float4-maxbs:512-kv_frac:0.85-input_output_len:8000,1000-con:512-ep:4-tp:4-gpus:4] TIMEOUT(120) #max_throughput #deepseek_v4_flash_base_fp8 (sparse-MLA MoE, FP8 block-scale, 4-GPU) - perf/test_perf.py::test_perf[deepseek_v4_flash_base_fp8-bench-pytorch-float8-input_output_len:8192,1024-ep:4-tp:4-gpus:4] TIMEOUT(120) - perf/test_perf.py::test_perf[deepseek_v4_flash_base_fp8-bench-pytorch-float8-input_output_len:1000,2000-ep:4-tp:4-gpus:4] TIMEOUT(120) @@ -227,14 +182,6 @@ llm_perf_core: - perf/test_perf.py::test_perf[gemma_4_31b_it_nvfp4-bench-pytorch-float4-input_output_len:2000,500] - perf/test_perf.py::test_perf[gemma_4_31b_it_nvfp4-bench-pytorch-float4-maxbs:1-input_output_len:8000,1000-reqs:10-con:1] #min_latency - perf/test_perf.py::test_perf[gemma_4_31b_it_nvfp4-bench-pytorch-float4-input_output_len:8000,1000-con:250] #max_throughput - #llama_v3.1_nemotron_ultra_253b_fp8 (nemotron-nas FP8 241G, 4-GPU) - - perf/test_perf.py::test_perf[llama_v3.1_nemotron_ultra_253b_fp8-bench-pytorch-float8-input_output_len:128,128-tp:4-gpus:4] - - perf/test_perf.py::test_perf[llama_v3.1_nemotron_ultra_253b_fp8-bench-pytorch-float8-input_output_len:500,2000-tp:4-gpus:4] - - perf/test_perf.py::test_perf[llama_v3.1_nemotron_ultra_253b_fp8-bench-pytorch-float8-input_output_len:2000,500-tp:4-gpus:4] - - perf/test_perf.py::test_perf[llama_v3.1_nemotron_ultra_253b_fp8-bench-pytorch-float8-input_output_len:8000,1000-tp:4-gpus:4] - - perf/test_perf.py::test_perf[llama_v3.1_nemotron_ultra_253b_fp8-bench-pytorch-float8-input_output_len:1000,2000-tp:4-gpus:4] - - perf/test_perf.py::test_perf[llama_v3.1_nemotron_ultra_253b_fp8-bench-pytorch-float8-maxbs:1-input_output_len:8000,1000-reqs:10-con:1-tp:4-gpus:4] #min_latency - - perf/test_perf.py::test_perf[llama_v3.1_nemotron_ultra_253b_fp8-bench-pytorch-float8-input_output_len:8000,1000-con:250-tp:4-gpus:4] #max_throughput #qwen3.5_397b_a17b_fp4 (MoE FP4 234G, 4-GPU) - perf/test_perf.py::test_perf[qwen3.5_397b_a17b_fp4-bench-pytorch-float4-input_output_len:128,128-ep:4-tp:4-gpus:4] - perf/test_perf.py::test_perf[qwen3.5_397b_a17b_fp4-bench-pytorch-float4-input_output_len:500,2000-ep:4-tp:4-gpus:4] @@ -272,8 +219,6 @@ llm_perf_core: gte: 10.0 lte: 10.3 tests: - - perf/test_perf.py::test_perf[kimi_k2_nvfp4-bench-pytorch-float4-maxbs:256-input_output_len:2000,500-ep:8-tp:8-gpus:8] - # gpt_oss_120b_fp4 - perf/test_perf.py::test_perf[gpt_oss_120b_fp4-bench-pytorch-float4-maxbs:720-maxnt:16384-input_output_len:1024,1024-reqs:1280-con:256-ep:8-tp:8-gpus:8] - perf/test_perf.py::test_perf[gpt_oss_120b_fp4-bench-pytorch-float4-maxbs:720-maxnt:16384-input_output_len:1024,1024-reqs:2560-con:512-ep:8-tp:8-gpus:8] @@ -297,7 +242,6 @@ llm_perf_core: - perf/test_perf.py::test_perf[deepseek_v4_pro_fp4-bench-pytorch-float4-maxbs:32-maxnt:8448-kv_frac:0.6-input_output_len:1000,2000-ep:8-tp:8-gpus:8] TIMEOUT(120) - perf/test_perf.py::test_perf[deepseek_v4_pro_fp4-bench-pytorch-float4-maxbs:128-maxnt:8448-input_output_len:8192,1024-reqs:10-con:1-ep:8-tp:8-gpus:8] TIMEOUT(120) #min_latency #deepseek_v4_pro_dspark (DSpark speculative decoding, FP4+FP8 mixed, 8-GPU) - - perf/test_perf.py::test_perf[deepseek_v4_pro_dspark-bench-pytorch-float4-maxbs:128-maxnt:9216-kv_frac:0.5-input_output_len:8192,1024-ep:8-tp:8-gpus:8] TIMEOUT(120) - perf/test_perf.py::test_perf[deepseek_v4_pro_dspark-bench-pytorch-float4-maxbs:128-maxnt:4096-kv_frac:0.5-input_output_len:1000,2000-ep:8-tp:8-gpus:8] TIMEOUT(120) #kimi_k2.5_fp4 (multimodal MoE FP4 553G, 8-GPU ep=8) - perf/test_perf.py::test_perf[kimi_k2.5_fp4-bench-pytorch-float4-input_output_len:128,128-ep:8-tp:8-gpus:8] @@ -333,15 +277,24 @@ llm_perf_core: - perf/test_perf.py::test_perf[minimax_m3_fp4-bench-pytorch-float4-maxbs:256-maxnt:8192-input_output_len:500,2000-ep:8-tp:8-gpus:8] TIMEOUT(120) - perf/test_perf.py::test_perf[minimax_m3_fp4-bench-pytorch-float4-maxbs:1-input_output_len:1024,1024-reqs:10-con:1-ep:8-tp:8-gpus:8] #min_latency - perf/test_perf.py::test_perf[minimax_m3_fp4-bench-pytorch-float4-maxbs:256-maxnt:8192-input_output_len:1024,1024-con:256-ep:8-tp:8-gpus:8] TIMEOUT(120) #max_throughput - #deepseek_v3.2_fp4 (FP4 389G, 8-GPU ep=8) - perf/test_perf.py::test_perf[deepseek_r1_0528_fp4-bench-pytorch-float4-maxbs:384-maxnt:1536-input_output_len:1000,2000-reqs:10000-con:3072-ep:8-tp:8-gpus:8] TIMEOUT(120) #max throughput test - - perf/test_perf.py::test_perf[deepseek_v3.2_fp4-bench-pytorch-float4-input_output_len:128,128-ep:8-tp:8-gpus:8] - - perf/test_perf.py::test_perf[deepseek_v3.2_fp4-bench-pytorch-float4-input_output_len:1000,2000-ep:8-tp:8-gpus:8] TIMEOUT(120) - - perf/test_perf.py::test_perf[deepseek_v3.2_fp4-bench-pytorch-float4-maxbs:1-input_output_len:8000,1000-reqs:10-con:1-ep:8-tp:8-gpus:8] #min_latency - - perf/test_perf.py::test_perf[deepseek_v3.2_fp4-bench-pytorch-float4-maxbs:384-maxnt:1536-input_output_len:8000,1000-con:3072-ep:8-tp:8-gpus:8] TIMEOUT(120) #max_throughput + - perf/test_perf.py::test_perf[qwen3.5_397b_a17b_fp8-bench-pytorch-float8-input_output_len:1000,2000-ep:8-tp:8-gpus:8] -# 8: H100, H20, B200, B300, RTX6000-Server test cases +# 8: B300 test cases +- condition: + ranges: + system_gpu_count: + gte: 8 + compute_capability: + gte: 10.3 + lte: 10.3 + tests: + #deepseek_v4_pro_dspark (DSpark speculative decoding, FP4+FP8 mixed, 8-GPU) + - perf/test_perf.py::test_perf[deepseek_v4_pro_dspark-bench-pytorch-float4-maxbs:128-maxnt:9216-kv_frac:0.5-input_output_len:8192,1024-ep:8-tp:8-gpus:8] TIMEOUT(120) + + +# 9: H100, H20, B200, B300, RTX6000-Server test cases - condition: ranges: system_gpu_count: @@ -350,37 +303,16 @@ llm_perf_core: gte: 9.0 lte: 12.0 tests: - #llama_v3.3_70b_instruct_fp8 - #pytorch backend - - perf/test_perf.py::test_perf[llama_v3.3_70b_instruct_fp8-bench-pytorch-float8-maxbs:4096-maxnt:20000-kv_frac:0.85-input_output_len:20000,2000-reqs:1000-tp:8-gpus:8] TIMEOUT(120) - #minimax_m2.5 (FP8 216G, 8-GPU) - use EP=8 + attention DP; TP=8 would split intermediate_size 1536/8=192 (not divisible by 128 block size) - - perf/test_perf.py::test_perf[minimax_m2.5_fp8-bench-pytorch-float8-input_output_len:128,128-ep:8-gpus:8] - - perf/test_perf.py::test_perf[minimax_m2.5_fp8-bench-pytorch-float8-input_output_len:500,2000-ep:8-gpus:8] - - perf/test_perf.py::test_perf[minimax_m2.5_fp8-bench-pytorch-float8-input_output_len:2000,500-ep:8-gpus:8] - - perf/test_perf.py::test_perf[minimax_m2.5_fp8-bench-pytorch-float8-input_output_len:8000,1000-ep:8-gpus:8] - - perf/test_perf.py::test_perf[minimax_m2.5_fp8-bench-pytorch-float8-input_output_len:1000,2000-ep:8-gpus:8] - - perf/test_perf.py::test_perf[minimax_m2.5_fp8-bench-pytorch-float8-maxbs:1-input_output_len:8000,1000-reqs:10-con:1-ep:8-gpus:8] #min_latency - - perf/test_perf.py::test_perf[minimax_m2.5_fp8-bench-pytorch-float8-maxbs:512-input_output_len:8000,1000-con:512-ep:8-gpus:8] #max_throughput - #llama_v3.1_nemotron_ultra_253b_fp8 (nemotron-nas FP8 241G, 8-GPU) - - perf/test_perf.py::test_perf[llama_v3.1_nemotron_ultra_253b_fp8-bench-pytorch-float8-input_output_len:128,128-tp:8-gpus:8] - - perf/test_perf.py::test_perf[llama_v3.1_nemotron_ultra_253b_fp8-bench-pytorch-float8-input_output_len:500,2000-tp:8-gpus:8] - - perf/test_perf.py::test_perf[llama_v3.1_nemotron_ultra_253b_fp8-bench-pytorch-float8-input_output_len:2000,500-tp:8-gpus:8] - - perf/test_perf.py::test_perf[llama_v3.1_nemotron_ultra_253b_fp8-bench-pytorch-float8-input_output_len:8000,1000-tp:8-gpus:8] - - perf/test_perf.py::test_perf[llama_v3.1_nemotron_ultra_253b_fp8-bench-pytorch-float8-input_output_len:1000,2000-tp:8-gpus:8] - - perf/test_perf.py::test_perf[llama_v3.1_nemotron_ultra_253b_fp8-bench-pytorch-float8-maxbs:1-input_output_len:8000,1000-reqs:10-con:1-tp:8-gpus:8] #min_latency - - perf/test_perf.py::test_perf[llama_v3.1_nemotron_ultra_253b_fp8-bench-pytorch-float8-input_output_len:8000,1000-con:250-tp:8-gpus:8] #max_throughput #qwen3.5_397b_a17b_fp8 (MoE FP8 380G, 8-GPU ep=8) - perf/test_perf.py::test_perf[qwen3.5_397b_a17b_fp8-bench-pytorch-float8-input_output_len:128,128-ep:8-tp:8-gpus:8] - perf/test_perf.py::test_perf[qwen3.5_397b_a17b_fp8-bench-pytorch-float8-input_output_len:500,2000-ep:8-tp:8-gpus:8] - perf/test_perf.py::test_perf[qwen3.5_397b_a17b_fp8-bench-pytorch-float8-input_output_len:2000,500-ep:8-tp:8-gpus:8] - perf/test_perf.py::test_perf[qwen3.5_397b_a17b_fp8-bench-pytorch-float8-input_output_len:8000,1000-ep:8-tp:8-gpus:8] - - perf/test_perf.py::test_perf[qwen3.5_397b_a17b_fp8-bench-pytorch-float8-input_output_len:1000,2000-ep:8-tp:8-gpus:8] - perf/test_perf.py::test_perf[qwen3.5_397b_a17b_fp8-bench-pytorch-float8-maxbs:1-input_output_len:8000,1000-reqs:10-con:1-ep:8-tp:8-gpus:8] #min_latency - perf/test_perf.py::test_perf[qwen3.5_397b_a17b_fp8-bench-pytorch-float8-maxbs:512-input_output_len:8000,1000-con:512-ep:8-tp:8-gpus:8] #max_throughput - - perf/test_perf.py::test_perf[qwen3.5_122b_a10b-bench-pytorch-bfloat16-input_output_len:128,128-ep:4-tp:4-gpus:4] -# 9: H20, B200, B300, RTX6000-Server test cases +# 10: H20, B200, B300, RTX6000-Server test cases - condition: ranges: system_gpu_count: @@ -394,9 +326,9 @@ llm_perf_core: # deepseek_r1_0528 - perf/test_perf.py::test_perf[deepseek_r1_0528_fp8-bench-pytorch-float8-input_output_len:8000,1000-reqs:20000-ep:8-tp:8-gpus:8] TIMEOUT(120) - perf/test_perf.py::test_perf[deepseek_r1_0528_fp8-bench-pytorch-float8-input_output_len:1000,2000-reqs:3000-ep:8-tp:8-gpus:8] TIMEOUT(100) - #deepseek_v3.2_fp8 (FP8 645G, 8-GPU ep=8) - - perf/test_perf.py::test_perf[deepseek_v3.2_fp8-bench-pytorch-float8-input_output_len:128,128-ep:8-tp:8-gpus:8] - - perf/test_perf.py::test_perf[deepseek_v3.2_fp8-bench-pytorch-float8-maxbs:384-maxnt:1536-input_output_len:8000,1000-con:3072-ep:8-tp:8-gpus:8] TIMEOUT(120) #max_throughput + #qwen3.5_122b_a10b (MoE BF16 234G, 4-GPU): OOM on 80G GPUs, needs >90G + - perf/test_perf.py::test_perf[qwen3.5_122b_a10b-bench-pytorch-bfloat16-input_output_len:128,128-ep:4-tp:4-gpus:4] + - perf/test_perf.py::test_perf[qwen3.5_122b_a10b-bench-pytorch-bfloat16-input_output_len:500,2000-ep:4-tp:4-gpus:4] #qwen3.5_397b_a17b_fp4 (MoE FP4 234G, 8-GPU ep=8) - perf/test_perf.py::test_perf[qwen3.5_397b_a17b_fp4-bench-pytorch-float4-input_output_len:128,128-ep:8-tp:8-gpus:8] - perf/test_perf.py::test_perf[qwen3.5_397b_a17b_fp4-bench-pytorch-float4-input_output_len:500,2000-ep:8-tp:8-gpus:8] @@ -405,16 +337,8 @@ llm_perf_core: - perf/test_perf.py::test_perf[qwen3.5_397b_a17b_fp4-bench-pytorch-float4-input_output_len:1000,2000-ep:8-tp:8-gpus:8] - perf/test_perf.py::test_perf[qwen3.5_397b_a17b_fp4-bench-pytorch-float4-maxbs:1-input_output_len:8000,1000-reqs:10-con:1-ep:8-tp:8-gpus:8] #min_latency - perf/test_perf.py::test_perf[qwen3.5_397b_a17b_fp4-bench-pytorch-float4-maxbs:512-input_output_len:8000,1000-con:512-ep:8-tp:8-gpus:8] #max_throughput - #llama_v3.1_nemotron_ultra_253b (nemotron-nas BF16 474G, 8-GPU) - - perf/test_perf.py::test_perf[llama_v3.1_nemotron_ultra_253b-bench-pytorch-bfloat16-input_output_len:128,128-tp:8-gpus:8] - - perf/test_perf.py::test_perf[llama_v3.1_nemotron_ultra_253b-bench-pytorch-bfloat16-input_output_len:500,2000-tp:8-gpus:8] - - perf/test_perf.py::test_perf[llama_v3.1_nemotron_ultra_253b-bench-pytorch-bfloat16-input_output_len:2000,500-tp:8-gpus:8] - - perf/test_perf.py::test_perf[llama_v3.1_nemotron_ultra_253b-bench-pytorch-bfloat16-input_output_len:8000,1000-tp:8-gpus:8] - - perf/test_perf.py::test_perf[llama_v3.1_nemotron_ultra_253b-bench-pytorch-bfloat16-input_output_len:1000,2000-tp:8-gpus:8] - - perf/test_perf.py::test_perf[llama_v3.1_nemotron_ultra_253b-bench-pytorch-bfloat16-maxbs:1-input_output_len:8000,1000-reqs:10-con:1-tp:8-gpus:8] #min_latency - - perf/test_perf.py::test_perf[llama_v3.1_nemotron_ultra_253b-bench-pytorch-bfloat16-input_output_len:8000,1000-con:250-tp:8-gpus:8] #max_throughput -# 10: RTX-6000 Server test cases +# 11: RTX-6000 Server test cases - condition: ranges: system_gpu_count: @@ -423,10 +347,6 @@ llm_perf_core: gte: 12.0 lte: 12.0 tests: - #llama_v3.3_70b - - perf/test_perf.py::test_perf[llama_v3.3_70b_instruct-bench-pytorch-bfloat16-maxbs:1-input_output_len:128,128-reqs:10-gpus:2] - - perf/test_perf.py::test_perf[llama_v3.3_70b_instruct_fp4-bench-pytorch-float4-input_output_len:128,128-tp:2-gpus:2] - - perf/test_perf.py::test_perf[llama_v3.3_70b_instruct_fp4-bench-pytorch-float4-maxbs:1024-maxnt:4096-kv_frac:0.85-input_output_len:8000,1000-reqs:3000-tp:8-gpus:8] - perf/test_perf.py::test_perf[qwen3_235b_a22b_fp4-bench-pytorch-float4-input_output_len:1000,2000-con:8-ep:8-tp:8-gpus:8] - perf/test_perf.py::test_perf[qwen3_235b_a22b_fp4-bench-pytorch-float4-input_output_len:1000,2000-con:512-ep:8-tp:8-gpus:8] # deepseek_r1_0528