diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index 27d2e1bef8e6..333035de824c 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -982,10 +982,12 @@ def getMultiGpuFileChanged(pipeline, testFilter, globalVars) "tests/integration/test_lists/test-db/l0_gb300.yml", "tests/integration/test_lists/test-db/l0_gb300_multi_gpus.yml", "tests/integration/test_lists/test-db/l0_gb300_multi_gpus_perf_sanity.yml", + "tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node1_gpu4.yml", + "tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node2_gpu8.yml", + "tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node8_gpu32.yml", "tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node1_gpu4.yml", "tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node2_gpu8.yml", "tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node4_gpu16.yml", - "tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node8_gpu32.yml", "tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_node2_gpu8.yml", "tests/integration/test_lists/test-db/l0_rtx_pro_6000.yml", "tests/integration/test_lists/test-db/l0_verl.yml", diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index ac9c1b83867f..b5fb633d9dbd 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -4075,10 +4075,11 @@ def launchTestJobs(pipeline, testFilter) // VisualGen PerfSanity post-merge test "DGX_B200-8_GPUs-PyTorch-VisualGen-PerfSanity-Post-Merge-1": ["auto:dgx-b200-flex", "l0_b200_visual_gen_perf_sanity", 1, 1, 8, 1, true], // PerfSanity post-merge tests - "DGX_B200-8_GPUs-PyTorch-PerfSanity-Post-Merge-1": ["auto:dgx-b200-flex", "l0_b200_multi_gpus_perf_sanity", 1, 4, 8, 1, true], - "DGX_B200-8_GPUs-PyTorch-PerfSanity-Post-Merge-2": ["auto:dgx-b200-flex", "l0_b200_multi_gpus_perf_sanity", 2, 4, 8, 1, true], - "DGX_B200-8_GPUs-PyTorch-PerfSanity-Post-Merge-3": ["auto:dgx-b200-flex", "l0_b200_multi_gpus_perf_sanity", 3, 4, 8, 1, true], - "DGX_B200-8_GPUs-PyTorch-PerfSanity-Post-Merge-4": ["auto:dgx-b200-flex", "l0_b200_multi_gpus_perf_sanity", 4, 4, 8, 1, true], + "DGX_B200-8_GPUs-PyTorch-PerfSanity-Post-Merge-1": ["auto:dgx-b200-flex", "l0_b200_multi_gpus_perf_sanity", 1, 5, 8, 1, true], + "DGX_B200-8_GPUs-PyTorch-PerfSanity-Post-Merge-2": ["auto:dgx-b200-flex", "l0_b200_multi_gpus_perf_sanity", 2, 5, 8, 1, true], + "DGX_B200-8_GPUs-PyTorch-PerfSanity-Post-Merge-3": ["auto:dgx-b200-flex", "l0_b200_multi_gpus_perf_sanity", 3, 5, 8, 1, true], + "DGX_B200-8_GPUs-PyTorch-PerfSanity-Post-Merge-4": ["auto:dgx-b200-flex", "l0_b200_multi_gpus_perf_sanity", 4, 5, 8, 1, true], + "DGX_B200-8_GPUs-PyTorch-PerfSanity-Post-Merge-5": ["auto:dgx-b200-flex", "l0_b200_multi_gpus_perf_sanity", 5, 5, 8, 1, true], ] // B200 PerfSanity post-merge disaggregated // 2 Nodes @@ -4139,15 +4140,18 @@ def launchTestJobs(pipeline, testFilter) "GB200-4_GPUs-PyTorch-PerfSanity-1": ["auto:gb200-x4", "l0_gb200_multi_gpus_perf_sanity", 1, 2, 4], "GB200-4_GPUs-PyTorch-PerfSanity-2": ["auto:gb200-x4", "l0_gb200_multi_gpus_perf_sanity", 2, 2, 4], // PerfSanity post-merge tests - "GB200-4_GPUs-PyTorch-PerfSanity-Post-Merge-1": ["auto:gb200-x4", "l0_gb200_multi_gpus_perf_sanity", 1, 7, 4], - "GB200-4_GPUs-PyTorch-PerfSanity-Post-Merge-2": ["auto:gb200-x4", "l0_gb200_multi_gpus_perf_sanity", 2, 7, 4], - "GB200-4_GPUs-PyTorch-PerfSanity-Post-Merge-3": ["auto:gb200-x4", "l0_gb200_multi_gpus_perf_sanity", 3, 7, 4], - "GB200-4_GPUs-PyTorch-PerfSanity-Post-Merge-4": ["auto:gb200-x4", "l0_gb200_multi_gpus_perf_sanity", 4, 7, 4], - "GB200-4_GPUs-PyTorch-PerfSanity-Post-Merge-5": ["auto:gb200-x4", "l0_gb200_multi_gpus_perf_sanity", 5, 7, 4], - "GB200-4_GPUs-PyTorch-PerfSanity-Post-Merge-6": ["auto:gb200-x4", "l0_gb200_multi_gpus_perf_sanity", 6, 7, 4], - "GB200-4_GPUs-PyTorch-PerfSanity-Post-Merge-7": ["auto:gb200-x4", "l0_gb200_multi_gpus_perf_sanity", 7, 7, 4], - "GB300-4_GPUs-PyTorch-PerfSanity-Post-Merge-1": ["auto:gb300-x4", "l0_gb300_multi_gpus_perf_sanity", 1, 2, 4], - "GB300-4_GPUs-PyTorch-PerfSanity-Post-Merge-2": ["auto:gb300-x4", "l0_gb300_multi_gpus_perf_sanity", 2, 2, 4], + "GB200-4_GPUs-PyTorch-PerfSanity-Post-Merge-1": ["auto:gb200-x4", "l0_gb200_multi_gpus_perf_sanity", 1, 9, 4], + "GB200-4_GPUs-PyTorch-PerfSanity-Post-Merge-2": ["auto:gb200-x4", "l0_gb200_multi_gpus_perf_sanity", 2, 9, 4], + "GB200-4_GPUs-PyTorch-PerfSanity-Post-Merge-3": ["auto:gb200-x4", "l0_gb200_multi_gpus_perf_sanity", 3, 9, 4], + "GB200-4_GPUs-PyTorch-PerfSanity-Post-Merge-4": ["auto:gb200-x4", "l0_gb200_multi_gpus_perf_sanity", 4, 9, 4], + "GB200-4_GPUs-PyTorch-PerfSanity-Post-Merge-5": ["auto:gb200-x4", "l0_gb200_multi_gpus_perf_sanity", 5, 9, 4], + "GB200-4_GPUs-PyTorch-PerfSanity-Post-Merge-6": ["auto:gb200-x4", "l0_gb200_multi_gpus_perf_sanity", 6, 9, 4], + "GB200-4_GPUs-PyTorch-PerfSanity-Post-Merge-7": ["auto:gb200-x4", "l0_gb200_multi_gpus_perf_sanity", 7, 9, 4], + "GB200-4_GPUs-PyTorch-PerfSanity-Post-Merge-8": ["auto:gb200-x4", "l0_gb200_multi_gpus_perf_sanity", 8, 9, 4], + "GB200-4_GPUs-PyTorch-PerfSanity-Post-Merge-9": ["auto:gb200-x4", "l0_gb200_multi_gpus_perf_sanity", 9, 9, 4], + "GB300-4_GPUs-PyTorch-PerfSanity-Post-Merge-1": ["auto:gb300-x4", "l0_gb300_multi_gpus_perf_sanity", 1, 3, 4], + "GB300-4_GPUs-PyTorch-PerfSanity-Post-Merge-2": ["auto:gb300-x4", "l0_gb300_multi_gpus_perf_sanity", 2, 3, 4], + "GB300-4_GPUs-PyTorch-PerfSanity-Post-Merge-3": ["auto:gb300-x4", "l0_gb300_multi_gpus_perf_sanity", 3, 3, 4], ] fullSet += SBSASlurmTestConfigs.keySet() @@ -4166,7 +4170,7 @@ def launchTestJobs(pipeline, testFilter) "GB200-8_GPUs-2_Nodes-PyTorch-PerfSanity-Node2-GPU8-Post-Merge", "auto:gb200-flex", "l0_gb200_multi_nodes_perf_sanity_node2_gpu8", - 7, + 9, 8, 2 ) @@ -4253,7 +4257,7 @@ def launchTestJobs(pipeline, testFilter) "GB200-36_GPUs-9_Nodes-PyTorch-Disagg-PerfSanity-CTX1-NODE1-GPU4-GEN1-NODE8-GPU32-Post-Merge", "auto:gb200-flex", "l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node8_gpu32", - 11, + 12, 36, 9 ) @@ -4282,7 +4286,7 @@ def launchTestJobs(pipeline, testFilter) "GB300-8_GPUs-2_Nodes-PyTorch-Disagg-PerfSanity-CTX1-NODE1-GPU4-GEN1-NODE1-GPU4-Post-Merge", "auto:gb300-flex", "l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node1_gpu4", - 4, + 3, 8, 2 ) @@ -4291,7 +4295,7 @@ def launchTestJobs(pipeline, testFilter) "GB300-12_GPUs-3_Nodes-PyTorch-Disagg-PerfSanity-CTX1-NODE1-GPU4-GEN1-NODE2-GPU8-Post-Merge", "auto:gb300-flex", "l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node2_gpu8", - 4, + 5, 12, 3 ) @@ -4309,7 +4313,35 @@ def launchTestJobs(pipeline, testFilter) "GB300-36_GPUs-9_Nodes-PyTorch-Disagg-PerfSanity-CTX1-NODE1-GPU4-GEN1-NODE8-GPU32-Post-Merge", "auto:gb300-flex", "l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node8_gpu32", - 4, + 2, + 36, + 9 + ) + // GB300 GLM-5 disaggregated (ctx DEP2) + // 2 Nodes + multiNodesSBSAConfigs += buildStageConfigs( + "GB300-8_GPUs-2_Nodes-PyTorch-Disagg-PerfSanity-CTX1-NODE1-GPU2-GEN1-NODE1-GPU4-Post-Merge", + "auto:gb300-flex", + "l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node1_gpu4", + 1, + 8, + 2 + ) + // 3 Nodes + multiNodesSBSAConfigs += buildStageConfigs( + "GB300-12_GPUs-3_Nodes-PyTorch-Disagg-PerfSanity-CTX1-NODE1-GPU2-GEN1-NODE2-GPU8-Post-Merge", + "auto:gb300-flex", + "l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node2_gpu8", + 5, + 12, + 3 + ) + // 9 Nodes + multiNodesSBSAConfigs += buildStageConfigs( + "GB300-36_GPUs-9_Nodes-PyTorch-Disagg-PerfSanity-CTX1-NODE1-GPU2-GEN1-NODE8-GPU32-Post-Merge", + "auto:gb300-flex", + "l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node8_gpu32", + 2, 36, 9 ) diff --git a/tests/integration/test_lists/qa/llm_perf_multinode.txt b/tests/integration/test_lists/qa/llm_perf_multinode.txt index a77c81e8f28b..9e9e0ee04a3e 100644 --- a/tests/integration/test_lists/qa/llm_perf_multinode.txt +++ b/tests/integration/test_lists/qa/llm_perf_multinode.txt @@ -53,7 +53,7 @@ perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_kimi-k25-thinking-fp4_1k1k_c perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_kimi-k25-thinking-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL] perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_kimi-k25-thinking-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-UCX] perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_kimi-k25-thinking-fp4_1k1k_con4_ctx1_dep4_gen1_tep4_eplb0_mtp0_ccb-NIXL] -perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_kimi-k25-thinking-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL] +perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_kimi-k25-thinking-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb416_mtp3_ccb-NIXL] perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-NIXL] perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-UCX] perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_kimi-k25-thinking-fp4_8k1k_con4_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] @@ -95,12 +95,22 @@ perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_deepseek-v32-fp4_8k1k_con1_c perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_deepseek-v32-fp4_8k1k_con4096_ctx1_dep4_gen1_dep32_eplb256_mtp0_ccb-NIXL] perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_deepseek-v32-fp4_8k1k_con4096_ctx1_dep4_gen1_dep32_eplb256_mtp0_ccb-UCX] +# GB300 GLM-5 +perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_glm-5-fp4_1k1k_con1_ctx1_dep2_gen1_tep4_eplb0_mtp3_ccb-NIXL] +perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_glm-5-fp4_1k1k_con4096_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-NIXL] +perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_glm-5-fp4_1k1k_con4096_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-UCX] +perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_glm-5-fp4_1k1k_con512_ctx1_dep2_gen1_dep32_eplb0_mtp3_ccb-NIXL] +perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_glm-5-fp4_8k1k_con1024_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-NIXL] +perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_glm-5-fp4_8k1k_con1024_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-UCX] +perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_glm-5-fp4_8k1k_con1_ctx1_dep2_gen1_tep8_eplb0_mtp3_ccb-NIXL] +perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_glm-5-fp4_8k1k_con512_ctx1_dep2_gen1_dep32_eplb0_mtp3_ccb-NIXL] + # GB300 Kimi-K2.5-Thinking perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_kimi-k25-thinking-fp4_1k1k_con2048_ctx1_dep4_gen1_dep32_eplb0_mtp0_ccb-NIXL] perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_kimi-k25-thinking-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL] perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_kimi-k25-thinking-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-UCX] perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_kimi-k25-thinking-fp4_1k1k_con4_ctx1_dep4_gen1_tep4_eplb0_mtp0_ccb-NIXL] -perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_kimi-k25-thinking-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL] +perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_kimi-k25-thinking-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb416_mtp3_ccb-NIXL] perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-NIXL] perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-UCX] perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_kimi-k25-thinking-fp4_8k1k_con4_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] diff --git a/tests/integration/test_lists/test-db/l0_b200_multi_gpus_perf_sanity.yml b/tests/integration/test_lists/test-db/l0_b200_multi_gpus_perf_sanity.yml index c026356682ae..854d8fae40b2 100644 --- a/tests/integration/test_lists/test-db/l0_b200_multi_gpus_perf_sanity.yml +++ b/tests/integration/test_lists/test-db/l0_b200_multi_gpus_perf_sanity.yml @@ -29,6 +29,9 @@ l0_b200_multi_gpus_perf_sanity: # deepseek-v32-fp4 - perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_v32_fp4_blackwell-v32_fp4_tep8_mtp3_8k1k] - perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_v32_fp4_blackwell-v32_fp4_dep8_mtp1_8k1k] TIMEOUT (90) + # glm-5-fp4 + - perf/test_perf_sanity.py::test_e2e[aggr_upload-glm5_fp4_blackwell-glm5_fp4_tep8_mtp3_8k1k] + - perf/test_perf_sanity.py::test_e2e[aggr_upload-glm5_fp4_blackwell-glm5_fp4_dep8_mtp1_8k1k] TIMEOUT (90) # gpt-oss-120b-fp4 - perf/test_perf_sanity.py::test_e2e[aggr_upload-gpt_oss_120b_fp4_blackwell-gpt_oss_fp4_tp2_mtp0_1k1k] - perf/test_perf_sanity.py::test_e2e[aggr_upload-gpt_oss_120b_fp4_blackwell-gpt_oss_fp4_tp1_mtp0_1k1k] diff --git a/tests/integration/test_lists/test-db/l0_gb200_multi_gpus_perf_sanity.yml b/tests/integration/test_lists/test-db/l0_gb200_multi_gpus_perf_sanity.yml index 1a55de21c3c5..961b34a93497 100644 --- a/tests/integration/test_lists/test-db/l0_gb200_multi_gpus_perf_sanity.yml +++ b/tests/integration/test_lists/test-db/l0_gb200_multi_gpus_perf_sanity.yml @@ -55,6 +55,9 @@ l0_gb200_multi_gpus_perf_sanity: - perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_v32_fp4_grace_blackwell-v32_fp4_dep4_mtp1_1k1k] - perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_v32_fp4_grace_blackwell-v32_fp4_tep4_mtp3_8k1k] - perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_v32_fp4_grace_blackwell-v32_fp4_dep4_mtp1_8k1k] + # glm-5-fp4 + - perf/test_perf_sanity.py::test_e2e[aggr_upload-glm5_fp4_grace_blackwell-glm5_fp4_tep4_mtp3_1k1k] + - perf/test_perf_sanity.py::test_e2e[aggr_upload-glm5_fp4_grace_blackwell-glm5_fp4_dep4_mtp1_1k1k] # gpt-oss-120b-fp4 - perf/test_perf_sanity.py::test_e2e[aggr_upload-gpt_oss_120b_fp4_grace_blackwell-gpt_oss_fp4_tp2_1k8k] - perf/test_perf_sanity.py::test_e2e[aggr_upload-gpt_oss_120b_fp4_grace_blackwell-gpt_oss_fp4_dep4_1k8k] @@ -95,7 +98,7 @@ l0_gb200_multi_gpus_perf_sanity: # kimi-k25-thinking-fp4 # - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_kimi-k25-thinking-fp4_1k1k_con2048_ctx1_dep4_gen1_dep32_eplb0_mtp0_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_kimi-k25-thinking-fp4_1k1k_con4_ctx1_dep4_gen1_tep4_eplb0_mtp0_ccb-NIXL] TIMEOUT (120) - # - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_kimi-k25-thinking-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL] TIMEOUT (120) + - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_kimi-k25-thinking-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb416_mtp3_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_kimi-k25-thinking-fp4_8k1k_con4_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (120) # qwen3-235b-fp4 diff --git a/tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node8_gpu32.yml b/tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node8_gpu32.yml index 5955e9b8b35c..8565ccfd9b7e 100644 --- a/tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node8_gpu32.yml +++ b/tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node8_gpu32.yml @@ -23,6 +23,6 @@ l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node8_gpu32: - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-v32-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-v32-fp4_8k1k_con4096_ctx1_dep4_gen1_dep32_eplb256_mtp0_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_kimi-k25-thinking-fp4_1k1k_con2048_ctx1_dep4_gen1_dep32_eplb0_mtp0_ccb-NIXL] TIMEOUT (120) - # - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_kimi-k25-thinking-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL] TIMEOUT (120) + - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_kimi-k25-thinking-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb416_mtp3_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_deepseek-v32-fp4_8k1k_con4096_ctx1_dep4_gen1_dep32_eplb256_mtp0_ccb-NIXL] TIMEOUT (120) diff --git a/tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_node2_gpu8.yml b/tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_node2_gpu8.yml index 501ecb45ebd7..46377d7a6420 100644 --- a/tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_node2_gpu8.yml +++ b/tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_node2_gpu8.yml @@ -22,6 +22,9 @@ l0_gb200_multi_nodes_perf_sanity_node2_gpu8: - perf/test_perf_sanity.py::test_e2e[aggr_upload-k25_thinking_fp4_2_nodes_grace_blackwell-k25_thinking_fp4_tep8_32k8k] - perf/test_perf_sanity.py::test_e2e[aggr_upload-k25_thinking_fp4_2_nodes_grace_blackwell-k25_thinking_fp4_dep8_32k8k] TIMEOUT (90) - perf/test_perf_sanity.py::test_e2e[aggr_upload-dynamo_deepseek_v32_fp4_2_nodes_grace_blackwell-dsv32_fp4_dep8_trtllm_lpc_mnnvl_8k1k] TIMEOUT (120) + # glm-5-fp4 + - perf/test_perf_sanity.py::test_e2e[aggr_upload-glm5_fp4_2_nodes_grace_blackwell-glm5_fp4_tep8_mtp3_8k1k] TIMEOUT (90) + - perf/test_perf_sanity.py::test_e2e[aggr_upload-glm5_fp4_2_nodes_grace_blackwell-glm5_fp4_dep8_mtp1_8k1k] TIMEOUT (90) # ctx only tests (disagg config) # deepseek-r1-fp4 - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-NIXL] TIMEOUT (120) diff --git a/tests/integration/test_lists/test-db/l0_gb300_multi_gpus_perf_sanity.yml b/tests/integration/test_lists/test-db/l0_gb300_multi_gpus_perf_sanity.yml index a2706e3ba79a..01cb11668fd8 100644 --- a/tests/integration/test_lists/test-db/l0_gb300_multi_gpus_perf_sanity.yml +++ b/tests/integration/test_lists/test-db/l0_gb300_multi_gpus_perf_sanity.yml @@ -21,8 +21,15 @@ l0_gb300_multi_gpus_perf_sanity: - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL] TIMEOUT (120) # kimi-k25-thinking-fp4 - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_kimi-k25-thinking-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL] TIMEOUT (120) + # - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_kimi-k25-thinking-fp4_1k1k_con2048_ctx1_dep4_gen1_dep32_eplb0_mtp0_ccb-NIXL] TIMEOUT (120) + - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_kimi-k25-thinking-fp4_1k1k_con4_ctx1_dep4_gen1_tep4_eplb0_mtp0_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-NIXL] TIMEOUT (120) - # deepseek-v32-fp4 - - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL] TIMEOUT (120) - - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_deepseek-v32-fp4_8k1k_con4096_ctx1_dep4_gen1_dep32_eplb256_mtp0_ccb-NIXL] TIMEOUT (120) - - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL] TIMEOUT (120) + - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_kimi-k25-thinking-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb416_mtp3_ccb-NIXL] TIMEOUT (120) + - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_kimi-k25-thinking-fp4_8k1k_con4_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (120) + # glm-5-fp4 + - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_glm-5-fp4_1k1k_con1_ctx1_dep2_gen1_tep4_eplb0_mtp3_ccb-NIXL] TIMEOUT (120) + - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_glm-5-fp4_1k1k_con512_ctx1_dep2_gen1_dep32_eplb0_mtp3_ccb-NIXL] TIMEOUT (120) + - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_glm-5-fp4_1k1k_con4096_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-NIXL] TIMEOUT (120) + - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_glm-5-fp4_8k1k_con1_ctx1_dep2_gen1_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (120) + - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_glm-5-fp4_8k1k_con512_ctx1_dep2_gen1_dep32_eplb0_mtp3_ccb-NIXL] TIMEOUT (120) + - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_glm-5-fp4_8k1k_con1024_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-NIXL] TIMEOUT (120) diff --git a/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node1_gpu4.yml b/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node1_gpu4.yml new file mode 100644 index 000000000000..95175326ac28 --- /dev/null +++ b/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node1_gpu4.yml @@ -0,0 +1,18 @@ +version: 0.0.1 +l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node1_gpu4: +- condition: + ranges: + # 1 ctx worker with each 1 node and 2 GPUs + # 1 gen worker with each 1 node and 4 GPUs + system_gpu_count: + gte: 8 + lte: 8 + wildcards: + gpu: + - '*gb300*' + terms: + stage: post_merge + backend: pytorch + tests: + # glm-5-fp4 + - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_glm-5-fp4_1k1k_con1_ctx1_dep2_gen1_tep4_eplb0_mtp3_ccb-NIXL] TIMEOUT (120) diff --git a/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node2_gpu8.yml b/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node2_gpu8.yml new file mode 100644 index 000000000000..0282983d1206 --- /dev/null +++ b/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node2_gpu8.yml @@ -0,0 +1,22 @@ +version: 0.0.1 +l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node2_gpu8: +- condition: + ranges: + # 1 ctx worker with each 1 node and 2 GPUs + # 1 gen worker with each 2 node and 8 GPUs + system_gpu_count: + gte: 12 + lte: 12 + wildcards: + gpu: + - '*gb300*' + terms: + stage: post_merge + backend: pytorch + tests: + # glm-5-fp4 + - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_glm-5-fp4_1k1k_con4096_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-NIXL] TIMEOUT (120) + - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_glm-5-fp4_8k1k_con1_ctx1_dep2_gen1_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (120) + - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_glm-5-fp4_8k1k_con1024_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-NIXL] TIMEOUT (120) + - perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_glm-5-fp4_1k1k_con4096_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-NIXL] TIMEOUT (120) + - perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_glm-5-fp4_8k1k_con1024_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-NIXL] TIMEOUT (120) diff --git a/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node8_gpu32.yml b/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node8_gpu32.yml new file mode 100644 index 000000000000..d587e9a12b61 --- /dev/null +++ b/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node8_gpu32.yml @@ -0,0 +1,19 @@ +version: 0.0.1 +l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node8_gpu32: +- condition: + ranges: + # 1 ctx worker with each 1 node and 2 GPUs + # 1 gen worker with each 8 node and 32 GPUs + system_gpu_count: + gte: 36 + lte: 36 + wildcards: + gpu: + - '*gb300*' + terms: + stage: post_merge + backend: pytorch + tests: + # glm-5-fp4 + - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_glm-5-fp4_1k1k_con512_ctx1_dep2_gen1_dep32_eplb0_mtp3_ccb-NIXL] TIMEOUT (120) + - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_glm-5-fp4_8k1k_con512_ctx1_dep2_gen1_dep32_eplb0_mtp3_ccb-NIXL] TIMEOUT (120) diff --git a/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node1_gpu4.yml b/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node1_gpu4.yml index 86dc7ffffd31..bc64fe529113 100644 --- a/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node1_gpu4.yml +++ b/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node1_gpu4.yml @@ -16,7 +16,6 @@ l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node1_gpu4: tests: # deepseek-r1-fp4 - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL] TIMEOUT (120) - # deepseek-v32-fp4 - - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL] TIMEOUT (120) + # kimi-k25-thinking-fp4 + - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_kimi-k25-thinking-fp4_1k1k_con4_ctx1_dep4_gen1_tep4_eplb0_mtp0_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL] TIMEOUT (120) - - perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL] TIMEOUT (120) diff --git a/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node2_gpu8.yml b/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node2_gpu8.yml index fdd32552da98..8b41e6984688 100644 --- a/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node2_gpu8.yml +++ b/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node2_gpu8.yml @@ -18,5 +18,7 @@ l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node2_gpu8: - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL] TIMEOUT (120) # kimi-k25-thinking-fp4 1k1k - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_kimi-k25-thinking-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL] TIMEOUT (120) + # kimi-k25-thinking-fp4 8k1k + - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_kimi-k25-thinking-fp4_8k1k_con4_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_kimi-k25-thinking-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL] TIMEOUT (120) diff --git a/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node8_gpu32.yml b/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node8_gpu32.yml index 08a437d6abb0..dac93c6f5de9 100644 --- a/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node8_gpu32.yml +++ b/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node8_gpu32.yml @@ -14,9 +14,6 @@ l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node8_gpu32: stage: post_merge backend: pytorch tests: - # deepseek-v32-fp4 8k1k - - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_deepseek-v32-fp4_8k1k_con4096_ctx1_dep4_gen1_dep32_eplb256_mtp0_ccb-NIXL] TIMEOUT (120) - # deepseek-v32-fp4 32k4k - - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL] TIMEOUT (120) - - perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_deepseek-v32-fp4_8k1k_con4096_ctx1_dep4_gen1_dep32_eplb256_mtp0_ccb-NIXL] TIMEOUT (120) - - perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL] TIMEOUT (120) + # kimi-k25-thinking-fp4 + - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_kimi-k25-thinking-fp4_1k1k_con2048_ctx1_dep4_gen1_dep32_eplb0_mtp0_ccb-NIXL] TIMEOUT (120) + - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_kimi-k25-thinking-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb416_mtp3_ccb-NIXL] TIMEOUT (120) diff --git a/tests/scripts/perf-sanity/aggregated/glm5_fp4_2_nodes_grace_blackwell.yaml b/tests/scripts/perf-sanity/aggregated/glm5_fp4_2_nodes_grace_blackwell.yaml new file mode 100644 index 000000000000..51486f63fb8a --- /dev/null +++ b/tests/scripts/perf-sanity/aggregated/glm5_fp4_2_nodes_grace_blackwell.yaml @@ -0,0 +1,74 @@ +metadata: + model_name: glm_5_nvfp4 + supported_gpus: + - GB200 +hardware: + gpus_per_node: 4 +server_configs: + # 8k1k configs - TEP8 with TRTLLM, MTP3 + - name: "glm5_fp4_tep8_mtp3_8k1k" + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3" + model_name: "glm_5_nvfp4" + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + max_batch_size: 4 + max_num_tokens: 256 + attn_backend: "TRTLLM" + enable_attention_dp: false + moe_config: + backend: 'TRTLLM' + cuda_graph_config: + enable_padding: true + max_batch_size: 4 + kv_cache_config: + dtype: 'fp8' + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + speculative_config: + decoding_type: 'MTP' + max_draft_len: 3 + client_configs: + - name: "con2_iter10_8k1k" + concurrency: 2 + iterations: 10 + isl: 8192 + osl: 1024 + backend: "openai" + dataset_file: + + # 8k1k configs - DEP8 with CUTLASS, MTP1 + - name: "glm5_fp4_dep8_mtp1_8k1k" + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1" + model_name: "glm_5_nvfp4" + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + max_batch_size: 64 + max_num_tokens: 12288 + attn_backend: "TRTLLM" + enable_attention_dp: true + attention_dp_config: + batching_wait_iters: 0 + enable_balance: true + timeout_iters: 60 + moe_config: + backend: 'CUTLASS' + cuda_graph_config: + enable_padding: true + max_batch_size: 64 + kv_cache_config: + dtype: 'fp8' + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + speculative_config: + decoding_type: 'MTP' + max_draft_len: 1 + client_configs: + - name: "con2_iter10_8k1k" + concurrency: 2 + iterations: 10 + isl: 8192 + osl: 1024 + backend: "openai" + dataset_file: diff --git a/tests/scripts/perf-sanity/aggregated/glm5_fp4_blackwell.yaml b/tests/scripts/perf-sanity/aggregated/glm5_fp4_blackwell.yaml new file mode 100644 index 000000000000..44d859240985 --- /dev/null +++ b/tests/scripts/perf-sanity/aggregated/glm5_fp4_blackwell.yaml @@ -0,0 +1,74 @@ +metadata: + model_name: glm_5_nvfp4 + supported_gpus: + - B200 +hardware: + gpus_per_node: 8 +server_configs: + # 8k1k configs - TEP8 with TRTLLM, MTP3 + - name: "glm5_fp4_tep8_mtp3_8k1k" + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3" + model_name: "glm_5_nvfp4" + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + max_batch_size: 4 + max_num_tokens: 256 + attn_backend: "TRTLLM" + enable_attention_dp: false + moe_config: + backend: 'TRTLLM' + cuda_graph_config: + enable_padding: true + max_batch_size: 4 + kv_cache_config: + dtype: 'fp8' + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + speculative_config: + decoding_type: 'MTP' + max_draft_len: 3 + client_configs: + - name: "con2_iter10_8k1k" + concurrency: 2 + iterations: 10 + isl: 8192 + osl: 1024 + backend: "openai" + dataset_file: + + # 8k1k configs - DEP8 with CUTLASS, MTP1 + - name: "glm5_fp4_dep8_mtp1_8k1k" + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1" + model_name: "glm_5_nvfp4" + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + max_batch_size: 64 + max_num_tokens: 12288 + attn_backend: "TRTLLM" + enable_attention_dp: true + attention_dp_config: + batching_wait_iters: 0 + enable_balance: true + timeout_iters: 60 + moe_config: + backend: 'CUTLASS' + cuda_graph_config: + enable_padding: true + max_batch_size: 64 + kv_cache_config: + dtype: 'fp8' + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + speculative_config: + decoding_type: 'MTP' + max_draft_len: 1 + client_configs: + - name: "con2_iter10_8k1k" + concurrency: 2 + iterations: 10 + isl: 8192 + osl: 1024 + backend: "openai" + dataset_file: diff --git a/tests/scripts/perf-sanity/aggregated/glm5_fp4_grace_blackwell.yaml b/tests/scripts/perf-sanity/aggregated/glm5_fp4_grace_blackwell.yaml new file mode 100644 index 000000000000..a0e1bffbb30e --- /dev/null +++ b/tests/scripts/perf-sanity/aggregated/glm5_fp4_grace_blackwell.yaml @@ -0,0 +1,74 @@ +metadata: + model_name: glm_5_nvfp4 + supported_gpus: + - GB200 +hardware: + gpus_per_node: 4 +server_configs: + # 1k1k configs - TEP4 with TRTLLM, MTP3 + - name: "glm5_fp4_tep4_mtp3_1k1k" + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3" + model_name: "glm_5_nvfp4" + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + max_batch_size: 4 + max_num_tokens: 1024 + attn_backend: "TRTLLM" + enable_attention_dp: false + moe_config: + backend: 'TRTLLM' + cuda_graph_config: + enable_padding: true + max_batch_size: 4 + kv_cache_config: + dtype: 'fp8' + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + speculative_config: + decoding_type: 'MTP' + max_draft_len: 3 + client_configs: + - name: "con2_iter10_1k1k" + concurrency: 2 + iterations: 10 + isl: 1024 + osl: 1024 + backend: "openai" + dataset_file: + + # 1k1k configs - DEP4 with CUTLASS, MTP1 + - name: "glm5_fp4_dep4_mtp1_1k1k" + server_env_var: "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1" + model_name: "glm_5_nvfp4" + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + max_batch_size: 256 + max_num_tokens: 8192 + attn_backend: "TRTLLM" + enable_attention_dp: true + attention_dp_config: + batching_wait_iters: 0 + enable_balance: true + timeout_iters: 60 + moe_config: + backend: 'CUTLASS' + cuda_graph_config: + enable_padding: true + max_batch_size: 256 + kv_cache_config: + dtype: 'fp8' + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + speculative_config: + decoding_type: 'MTP' + max_draft_len: 1 + client_configs: + - name: "con2_iter10_1k1k" + concurrency: 2 + iterations: 10 + isl: 1024 + osl: 1024 + backend: "openai" + dataset_file: diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_1k1k_con1_ctx1_dep4_gen1_tep4_eplb0_mtp3_ccb-NIXL.yaml similarity index 96% rename from tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml rename to tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_1k1k_con1_ctx1_dep4_gen1_tep4_eplb0_mtp3_ccb-NIXL.yaml index 82b59b7097b5..6678fdfc4f8b 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_1k1k_con1_ctx1_dep4_gen1_tep4_eplb0_mtp3_ccb-NIXL.yaml @@ -44,8 +44,8 @@ worker_config: print_iter_log: true max_batch_size: 128 max_num_tokens: 512 - tensor_parallel_size: 8 - moe_expert_parallel_size: 8 + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 pipeline_parallel_size: 1 context_parallel_size: 1 enable_attention_dp: false @@ -54,7 +54,7 @@ worker_config: max_batch_size: 128 kv_cache_config: enable_block_reuse: false - free_gpu_memory_fraction: 0.9 + free_gpu_memory_fraction: 0.85 dtype: fp8 moe_config: backend: TRTLLM diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb256_mtp1_ccb-NIXL.yaml similarity index 96% rename from tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml rename to tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb256_mtp1_ccb-NIXL.yaml index b4aea34d4745..a54e7d0335cf 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb256_mtp1_ccb-NIXL.yaml @@ -60,6 +60,9 @@ worker_config: moe_config: backend: CUTEDSL use_low_precision_moe_combine: true + load_balancer: + num_slots: 256 + layer_updates_per_iter: 1 cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_1k1k_con512_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_1k1k_con512_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml index 1afe92603861..e8bcff454293 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_1k1k_con512_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_1k1k_con512_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml @@ -43,7 +43,7 @@ worker_config: gen: print_iter_log: true max_batch_size: 16 - max_num_tokens: 128 + max_num_tokens: 64 tensor_parallel_size: 32 moe_expert_parallel_size: 32 pipeline_parallel_size: 1 diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_8k1k_con1024_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_8k1k_con1024_ctx1_dep4_gen1_dep8_eplb256_mtp1_ccb-NIXL.yaml similarity index 96% rename from tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_8k1k_con1024_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml rename to tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_8k1k_con1024_ctx1_dep4_gen1_dep8_eplb256_mtp1_ccb-NIXL.yaml index 24794f026fd5..37d162efda13 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_8k1k_con1024_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_8k1k_con1024_ctx1_dep4_gen1_dep8_eplb256_mtp1_ccb-NIXL.yaml @@ -60,6 +60,9 @@ worker_config: moe_config: backend: CUTEDSL use_low_precision_moe_combine: true + load_balancer: + num_slots: 256 + layer_updates_per_iter: 1 cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_8k1k_con512_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_8k1k_con512_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml index 7d7891f82571..876121e49a57 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_8k1k_con512_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_glm-5-fp4_8k1k_con512_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml @@ -43,7 +43,7 @@ worker_config: gen: print_iter_log: true max_batch_size: 16 - max_num_tokens: 128 + max_num_tokens: 64 tensor_parallel_size: 32 moe_expert_parallel_size: 32 pipeline_parallel_size: 1 diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_kimi-k25-thinking-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_kimi-k25-thinking-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb416_mtp3_ccb-NIXL.yaml similarity index 97% rename from tests/scripts/perf-sanity/disaggregated/gb200_kimi-k25-thinking-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml rename to tests/scripts/perf-sanity/disaggregated/gb200_kimi-k25-thinking-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb416_mtp3_ccb-NIXL.yaml index 38cd6e603fbd..0b376eb45b53 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_kimi-k25-thinking-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_kimi-k25-thinking-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb416_mtp3_ccb-NIXL.yaml @@ -60,6 +60,9 @@ worker_config: moe_config: backend: CUTEDSL use_low_precision_moe_combine: true + load_balancer: + num_slots: 416 + layer_updates_per_iter: 1 cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_glm-5-fp4_1k1k_con1_ctx1_dep2_gen1_tep4_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_glm-5-fp4_1k1k_con1_ctx1_dep2_gen1_tep4_eplb0_mtp3_ccb-NIXL.yaml new file mode 100644 index 000000000000..7e5aa487facc --- /dev/null +++ b/tests/scripts/perf-sanity/disaggregated/gb300_glm-5-fp4_1k1k_con1_ctx1_dep2_gen1_tep4_eplb0_mtp3_ccb-NIXL.yaml @@ -0,0 +1,93 @@ +metadata: + model_name: glm_5_nvfp4 + precision: fp4 + model_dir_name: GLM-5-NVFP4 + supported_gpus: + - GB300 + script_file: disaggr_torch.slurm + benchmark_type: 1k1k +slurm: + script_file: disaggr_torch.slurm + partition: + account: + job_time: 02:00:00 + job_name: unified-benchmark + extra_args: --gres=gpu:4 + numa_bind: true +benchmark: + mode: e2e + use_nv_sa_benchmark: false + multi_round: 10 + benchmark_ratio: 0.0 + streaming: true + concurrency_list: '1' + input_length: 1024 + output_length: 1024 + dataset_file: +hardware: + gpus_per_node: 4 + num_ctx_servers: 1 + num_gen_servers: 1 +environment: + container_mount: + container_image: + model_path: + trtllm_repo: '' + build_wheel: false + work_dir: + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 + server_env_var: TRTLLM_SERVER_DISABLE_GC=1 +profiling: + nsys_on: false +worker_config: + gen: + print_iter_log: true + max_batch_size: 128 + max_num_tokens: 512 + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + context_parallel_size: 1 + enable_attention_dp: false + cuda_graph_config: + enable_padding: true + max_batch_size: 128 + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + dtype: fp8 + moe_config: + backend: TRTLLM + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: NIXL + disable_overlap_scheduler: true + speculative_config: &id001 + decoding_type: MTP + num_nextn_predict_layers: 3 + num_postprocess_workers: 4 + stream_interval: 20 + allreduce_strategy: MNNVL + ctx: + print_iter_log: true + max_batch_size: 16 + max_num_tokens: 16384 + tensor_parallel_size: 2 + moe_expert_parallel_size: 2 + pipeline_parallel_size: 1 + context_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + cuda_graph_config: null + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + dtype: fp8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: NIXL + disable_overlap_scheduler: true + speculative_config: *id001 diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_glm-5-fp4_1k1k_con4096_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_glm-5-fp4_1k1k_con4096_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-NIXL.yaml new file mode 100644 index 000000000000..f6334f686d1c --- /dev/null +++ b/tests/scripts/perf-sanity/disaggregated/gb300_glm-5-fp4_1k1k_con4096_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-NIXL.yaml @@ -0,0 +1,97 @@ +metadata: + model_name: glm_5_nvfp4 + precision: fp4 + model_dir_name: GLM-5-NVFP4 + supported_gpus: + - GB300 + script_file: disaggr_torch.slurm + benchmark_type: 1k1k +slurm: + script_file: disaggr_torch.slurm + partition: + account: + job_time: 02:00:00 + job_name: unified-benchmark + extra_args: --gres=gpu:4 + numa_bind: true +benchmark: + mode: e2e + use_nv_sa_benchmark: false + multi_round: 5 + benchmark_ratio: 0.0 + streaming: true + concurrency_list: '4096' + input_length: 1024 + output_length: 1024 + dataset_file: +hardware: + gpus_per_node: 4 + num_ctx_servers: 1 + num_gen_servers: 1 +environment: + container_mount: + container_image: + model_path: + trtllm_repo: '' + build_wheel: false + work_dir: + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1 + server_env_var: TRTLLM_SERVER_DISABLE_GC=1 +profiling: + nsys_on: false +worker_config: + gen: + print_iter_log: true + max_batch_size: 512 + max_num_tokens: 1024 + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + context_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + cuda_graph_config: + enable_padding: true + max_batch_size: 512 + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + dtype: fp8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + load_balancer: + num_slots: 256 + layer_updates_per_iter: 1 + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: NIXL + disable_overlap_scheduler: true + speculative_config: &id001 + decoding_type: MTP + num_nextn_predict_layers: 1 + num_postprocess_workers: 4 + stream_interval: 20 + ctx: + print_iter_log: true + max_batch_size: 16 + max_num_tokens: 16384 + tensor_parallel_size: 2 + moe_expert_parallel_size: 2 + pipeline_parallel_size: 1 + context_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + cuda_graph_config: null + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + dtype: fp8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: NIXL + disable_overlap_scheduler: true + speculative_config: *id001 diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_glm-5-fp4_1k1k_con512_ctx1_dep2_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_glm-5-fp4_1k1k_con512_ctx1_dep2_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml new file mode 100644 index 000000000000..8ef0b3db0473 --- /dev/null +++ b/tests/scripts/perf-sanity/disaggregated/gb300_glm-5-fp4_1k1k_con512_ctx1_dep2_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml @@ -0,0 +1,94 @@ +metadata: + model_name: glm_5_nvfp4 + precision: fp4 + model_dir_name: GLM-5-NVFP4 + supported_gpus: + - GB300 + script_file: disaggr_torch.slurm + benchmark_type: 1k1k +slurm: + script_file: disaggr_torch.slurm + partition: + account: + job_time: 02:00:00 + job_name: unified-benchmark + extra_args: --gres=gpu:4 + numa_bind: true +benchmark: + mode: e2e + use_nv_sa_benchmark: false + multi_round: 10 + benchmark_ratio: 0.0 + streaming: true + concurrency_list: '512' + input_length: 1024 + output_length: 1024 + dataset_file: +hardware: + gpus_per_node: 4 + num_ctx_servers: 1 + num_gen_servers: 1 +environment: + container_mount: + container_image: + model_path: + trtllm_repo: '' + build_wheel: false + work_dir: + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 + server_env_var: TRTLLM_SERVER_DISABLE_GC=1 +profiling: + nsys_on: false +worker_config: + gen: + print_iter_log: true + max_batch_size: 16 + max_num_tokens: 64 + tensor_parallel_size: 32 + moe_expert_parallel_size: 32 + pipeline_parallel_size: 1 + context_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + cuda_graph_config: + enable_padding: true + max_batch_size: 16 + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + dtype: fp8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: NIXL + disable_overlap_scheduler: true + speculative_config: &id001 + decoding_type: MTP + num_nextn_predict_layers: 3 + num_postprocess_workers: 4 + stream_interval: 20 + ctx: + print_iter_log: true + max_batch_size: 16 + max_num_tokens: 16384 + tensor_parallel_size: 2 + moe_expert_parallel_size: 2 + pipeline_parallel_size: 1 + context_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + cuda_graph_config: null + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + dtype: fp8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: NIXL + disable_overlap_scheduler: true + speculative_config: *id001 diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_glm-5-fp4_8k1k_con1024_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_glm-5-fp4_8k1k_con1024_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-NIXL.yaml new file mode 100644 index 000000000000..22a64298673b --- /dev/null +++ b/tests/scripts/perf-sanity/disaggregated/gb300_glm-5-fp4_8k1k_con1024_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-NIXL.yaml @@ -0,0 +1,97 @@ +metadata: + model_name: glm_5_nvfp4 + precision: fp4 + model_dir_name: GLM-5-NVFP4 + supported_gpus: + - GB300 + script_file: disaggr_torch.slurm + benchmark_type: 8k1k +slurm: + script_file: disaggr_torch.slurm + partition: + account: + job_time: 02:00:00 + job_name: unified-benchmark + extra_args: --gres=gpu:4 + numa_bind: true +benchmark: + mode: e2e + use_nv_sa_benchmark: false + multi_round: 10 + benchmark_ratio: 0.0 + streaming: true + concurrency_list: '1024' + input_length: 8192 + output_length: 1024 + dataset_file: +hardware: + gpus_per_node: 4 + num_ctx_servers: 1 + num_gen_servers: 1 +environment: + container_mount: + container_image: + model_path: + trtllm_repo: '' + build_wheel: false + work_dir: + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1 + server_env_var: TRTLLM_SERVER_DISABLE_GC=1 +profiling: + nsys_on: false +worker_config: + gen: + print_iter_log: true + max_batch_size: 128 + max_num_tokens: 256 + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + context_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + cuda_graph_config: + enable_padding: true + max_batch_size: 128 + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + dtype: fp8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + load_balancer: + num_slots: 256 + layer_updates_per_iter: 1 + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: NIXL + disable_overlap_scheduler: true + speculative_config: &id001 + decoding_type: MTP + num_nextn_predict_layers: 1 + num_postprocess_workers: 4 + stream_interval: 20 + ctx: + print_iter_log: true + max_batch_size: 2 + max_num_tokens: 16384 + tensor_parallel_size: 2 + moe_expert_parallel_size: 2 + pipeline_parallel_size: 1 + context_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + cuda_graph_config: null + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + dtype: fp8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: NIXL + disable_overlap_scheduler: true + speculative_config: *id001 diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_glm-5-fp4_8k1k_con1_ctx1_dep2_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_glm-5-fp4_8k1k_con1_ctx1_dep2_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml new file mode 100644 index 000000000000..1e6c43b2ddf7 --- /dev/null +++ b/tests/scripts/perf-sanity/disaggregated/gb300_glm-5-fp4_8k1k_con1_ctx1_dep2_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml @@ -0,0 +1,93 @@ +metadata: + model_name: glm_5_nvfp4 + precision: fp4 + model_dir_name: GLM-5-NVFP4 + supported_gpus: + - GB300 + script_file: disaggr_torch.slurm + benchmark_type: 8k1k +slurm: + script_file: disaggr_torch.slurm + partition: + account: + job_time: 02:00:00 + job_name: unified-benchmark + extra_args: --gres=gpu:4 + numa_bind: true +benchmark: + mode: e2e + use_nv_sa_benchmark: false + multi_round: 10 + benchmark_ratio: 0.0 + streaming: true + concurrency_list: '1' + input_length: 8192 + output_length: 1024 + dataset_file: +hardware: + gpus_per_node: 4 + num_ctx_servers: 1 + num_gen_servers: 1 +environment: + container_mount: + container_image: + model_path: + trtllm_repo: '' + build_wheel: false + work_dir: + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 + server_env_var: TRTLLM_SERVER_DISABLE_GC=1 +profiling: + nsys_on: false +worker_config: + gen: + print_iter_log: true + max_batch_size: 128 + max_num_tokens: 512 + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + context_parallel_size: 1 + enable_attention_dp: false + cuda_graph_config: + enable_padding: true + max_batch_size: 128 + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + dtype: fp8 + moe_config: + backend: TRTLLM + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: NIXL + disable_overlap_scheduler: true + speculative_config: &id001 + decoding_type: MTP + num_nextn_predict_layers: 3 + num_postprocess_workers: 4 + stream_interval: 20 + allreduce_strategy: MNNVL + ctx: + print_iter_log: true + max_batch_size: 2 + max_num_tokens: 16384 + tensor_parallel_size: 2 + moe_expert_parallel_size: 2 + pipeline_parallel_size: 1 + context_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + cuda_graph_config: null + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + dtype: fp8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: NIXL + disable_overlap_scheduler: true + speculative_config: *id001 diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_glm-5-fp4_8k1k_con512_ctx1_dep2_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_glm-5-fp4_8k1k_con512_ctx1_dep2_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml new file mode 100644 index 000000000000..777987a01492 --- /dev/null +++ b/tests/scripts/perf-sanity/disaggregated/gb300_glm-5-fp4_8k1k_con512_ctx1_dep2_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml @@ -0,0 +1,94 @@ +metadata: + model_name: glm_5_nvfp4 + precision: fp4 + model_dir_name: GLM-5-NVFP4 + supported_gpus: + - GB300 + script_file: disaggr_torch.slurm + benchmark_type: 8k1k +slurm: + script_file: disaggr_torch.slurm + partition: + account: + job_time: 02:00:00 + job_name: unified-benchmark + extra_args: --gres=gpu:4 + numa_bind: true +benchmark: + mode: e2e + use_nv_sa_benchmark: false + multi_round: 10 + benchmark_ratio: 0.0 + streaming: true + concurrency_list: '512' + input_length: 8192 + output_length: 1024 + dataset_file: +hardware: + gpus_per_node: 4 + num_ctx_servers: 1 + num_gen_servers: 1 +environment: + container_mount: + container_image: + model_path: + trtllm_repo: '' + build_wheel: false + work_dir: + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 + server_env_var: TRTLLM_SERVER_DISABLE_GC=1 +profiling: + nsys_on: false +worker_config: + gen: + print_iter_log: true + max_batch_size: 16 + max_num_tokens: 64 + tensor_parallel_size: 32 + moe_expert_parallel_size: 32 + pipeline_parallel_size: 1 + context_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + cuda_graph_config: + enable_padding: true + max_batch_size: 16 + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + dtype: fp8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: NIXL + disable_overlap_scheduler: true + speculative_config: &id001 + decoding_type: MTP + num_nextn_predict_layers: 3 + num_postprocess_workers: 4 + stream_interval: 20 + ctx: + print_iter_log: true + max_batch_size: 2 + max_num_tokens: 16384 + tensor_parallel_size: 2 + moe_expert_parallel_size: 2 + pipeline_parallel_size: 1 + context_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + cuda_graph_config: null + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + dtype: fp8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: NIXL + disable_overlap_scheduler: true + speculative_config: *id001 diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_kimi-k25-thinking-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_kimi-k25-thinking-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb416_mtp3_ccb-NIXL.yaml similarity index 97% rename from tests/scripts/perf-sanity/disaggregated/gb300_kimi-k25-thinking-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml rename to tests/scripts/perf-sanity/disaggregated/gb300_kimi-k25-thinking-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb416_mtp3_ccb-NIXL.yaml index 725cda90bb36..26916bd61b00 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb300_kimi-k25-thinking-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb300_kimi-k25-thinking-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb416_mtp3_ccb-NIXL.yaml @@ -60,6 +60,9 @@ worker_config: moe_config: backend: CUTEDSL use_low_precision_moe_combine: true + load_balancer: + num_slots: 416 + layer_updates_per_iter: 1 cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL diff --git a/tests/scripts/perf/disaggregated/gb200_glm-5-fp4_1k1k_con1_ctx1_dep4_gen1_tep4_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_glm-5-fp4_1k1k_con1_ctx1_dep4_gen1_tep4_eplb0_mtp3_ccb-NIXL.yaml new file mode 100644 index 000000000000..4ed531013bec --- /dev/null +++ b/tests/scripts/perf/disaggregated/gb200_glm-5-fp4_1k1k_con1_ctx1_dep4_gen1_tep4_eplb0_mtp3_ccb-NIXL.yaml @@ -0,0 +1,104 @@ +metadata: + model_name: glm_5_nvfp4 + precision: fp4 + model_dir_name: GLM-5-NVFP4 + supported_gpus: + - GB200 + script_file: disaggr_torch.slurm + benchmark_type: 1k1k +slurm: + script_file: disaggr_torch.slurm + partition: + account: + job_time: 02:00:00 + job_name: unified-benchmark + extra_args: --gres=gpu:4 + numa_bind: true +benchmark: + mode: e2e + use_nv_sa_benchmark: false + multi_round: 10 + benchmark_ratio: 0.0 + streaming: true + concurrency_list: '1' + input_length: 1024 + output_length: 1024 + dataset_file: +hardware: + gpus_per_node: 4 + num_ctx_servers: 1 + num_gen_servers: 1 +environment: + container_mount: + container_image: + model_path: + trtllm_repo: '' + build_wheel: false + work_dir: + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 + server_env_var: TRTLLM_SERVER_DISABLE_GC=1 +profiling: + nsys_on: false +accuracy: + enable_accuracy_test: false + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml +worker_config: + gen: + print_iter_log: true + max_batch_size: 128 + max_num_tokens: 512 + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + context_parallel_size: 1 + enable_attention_dp: false + cuda_graph_config: + enable_padding: true + max_batch_size: 128 + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + dtype: fp8 + moe_config: + backend: TRTLLM + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: NIXL + disable_overlap_scheduler: true + speculative_config: &id001 + decoding_type: MTP + num_nextn_predict_layers: 3 + num_postprocess_workers: 4 + stream_interval: 20 + allreduce_strategy: MNNVL + ctx: + print_iter_log: true + max_batch_size: 16 + max_num_tokens: 16384 + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + context_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + cuda_graph_config: null + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + dtype: fp8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: NIXL + disable_overlap_scheduler: true + speculative_config: *id001 diff --git a/tests/scripts/perf/disaggregated/gb200_glm-5-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb256_mtp1_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_glm-5-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb256_mtp1_ccb-NIXL.yaml new file mode 100644 index 000000000000..899270e7d29f --- /dev/null +++ b/tests/scripts/perf/disaggregated/gb200_glm-5-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb256_mtp1_ccb-NIXL.yaml @@ -0,0 +1,108 @@ +metadata: + model_name: glm_5_nvfp4 + precision: fp4 + model_dir_name: GLM-5-NVFP4 + supported_gpus: + - GB200 + script_file: disaggr_torch.slurm + benchmark_type: 1k1k +slurm: + script_file: disaggr_torch.slurm + partition: + account: + job_time: 02:00:00 + job_name: unified-benchmark + extra_args: --gres=gpu:4 + numa_bind: true +benchmark: + mode: e2e + use_nv_sa_benchmark: false + multi_round: 5 + benchmark_ratio: 0.0 + streaming: true + concurrency_list: '4096' + input_length: 1024 + output_length: 1024 + dataset_file: +hardware: + gpus_per_node: 4 + num_ctx_servers: 1 + num_gen_servers: 1 +environment: + container_mount: + container_image: + model_path: + trtllm_repo: '' + build_wheel: false + work_dir: + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1 + server_env_var: TRTLLM_SERVER_DISABLE_GC=1 +profiling: + nsys_on: false +accuracy: + enable_accuracy_test: false + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml +worker_config: + gen: + print_iter_log: true + max_batch_size: 512 + max_num_tokens: 1024 + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + context_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + cuda_graph_config: + enable_padding: true + max_batch_size: 512 + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + dtype: fp8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + load_balancer: + num_slots: 256 + layer_updates_per_iter: 1 + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: NIXL + disable_overlap_scheduler: true + speculative_config: &id001 + decoding_type: MTP + num_nextn_predict_layers: 1 + num_postprocess_workers: 4 + stream_interval: 20 + ctx: + print_iter_log: true + max_batch_size: 16 + max_num_tokens: 16384 + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + context_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + cuda_graph_config: null + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + dtype: fp8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: NIXL + disable_overlap_scheduler: true + speculative_config: *id001 diff --git a/tests/scripts/perf/disaggregated/gb200_glm-5-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb256_mtp1_ccb-UCX.yaml b/tests/scripts/perf/disaggregated/gb200_glm-5-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb256_mtp1_ccb-UCX.yaml new file mode 100644 index 000000000000..00dced799780 --- /dev/null +++ b/tests/scripts/perf/disaggregated/gb200_glm-5-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb256_mtp1_ccb-UCX.yaml @@ -0,0 +1,108 @@ +metadata: + model_name: glm_5_nvfp4 + precision: fp4 + model_dir_name: GLM-5-NVFP4 + supported_gpus: + - GB200 + script_file: disaggr_torch.slurm + benchmark_type: 1k1k +slurm: + script_file: disaggr_torch.slurm + partition: + account: + job_time: 02:00:00 + job_name: unified-benchmark + extra_args: --gres=gpu:4 + numa_bind: true +benchmark: + mode: e2e + use_nv_sa_benchmark: false + multi_round: 5 + benchmark_ratio: 0.0 + streaming: true + concurrency_list: '4096' + input_length: 1024 + output_length: 1024 + dataset_file: +hardware: + gpus_per_node: 4 + num_ctx_servers: 1 + num_gen_servers: 1 +environment: + container_mount: + container_image: + model_path: + trtllm_repo: '' + build_wheel: false + work_dir: + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1 + server_env_var: TRTLLM_SERVER_DISABLE_GC=1 +profiling: + nsys_on: false +accuracy: + enable_accuracy_test: false + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml +worker_config: + gen: + print_iter_log: true + max_batch_size: 512 + max_num_tokens: 1024 + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + context_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + cuda_graph_config: + enable_padding: true + max_batch_size: 512 + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + dtype: fp8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + load_balancer: + num_slots: 256 + layer_updates_per_iter: 1 + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: UCX + disable_overlap_scheduler: true + speculative_config: &id001 + decoding_type: MTP + num_nextn_predict_layers: 1 + num_postprocess_workers: 4 + stream_interval: 20 + ctx: + print_iter_log: true + max_batch_size: 16 + max_num_tokens: 16384 + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + context_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + cuda_graph_config: null + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + dtype: fp8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: UCX + disable_overlap_scheduler: true + speculative_config: *id001 diff --git a/tests/scripts/perf/disaggregated/gb200_glm-5-fp4_1k1k_con512_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_glm-5-fp4_1k1k_con512_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml new file mode 100644 index 000000000000..916584e71cbb --- /dev/null +++ b/tests/scripts/perf/disaggregated/gb200_glm-5-fp4_1k1k_con512_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml @@ -0,0 +1,105 @@ +metadata: + model_name: glm_5_nvfp4 + precision: fp4 + model_dir_name: GLM-5-NVFP4 + supported_gpus: + - GB200 + script_file: disaggr_torch.slurm + benchmark_type: 1k1k +slurm: + script_file: disaggr_torch.slurm + partition: + account: + job_time: 02:00:00 + job_name: unified-benchmark + extra_args: --gres=gpu:4 + numa_bind: true +benchmark: + mode: e2e + use_nv_sa_benchmark: false + multi_round: 10 + benchmark_ratio: 0.0 + streaming: true + concurrency_list: '512' + input_length: 1024 + output_length: 1024 + dataset_file: +hardware: + gpus_per_node: 4 + num_ctx_servers: 1 + num_gen_servers: 1 +environment: + container_mount: + container_image: + model_path: + trtllm_repo: '' + build_wheel: false + work_dir: + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 + server_env_var: TRTLLM_SERVER_DISABLE_GC=1 +profiling: + nsys_on: false +accuracy: + enable_accuracy_test: false + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml +worker_config: + gen: + print_iter_log: true + max_batch_size: 16 + max_num_tokens: 64 + tensor_parallel_size: 32 + moe_expert_parallel_size: 32 + pipeline_parallel_size: 1 + context_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + cuda_graph_config: + enable_padding: true + max_batch_size: 16 + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + dtype: fp8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: NIXL + disable_overlap_scheduler: true + speculative_config: &id001 + decoding_type: MTP + num_nextn_predict_layers: 3 + num_postprocess_workers: 4 + stream_interval: 20 + ctx: + print_iter_log: true + max_batch_size: 16 + max_num_tokens: 16384 + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + context_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + cuda_graph_config: null + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + dtype: fp8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: NIXL + disable_overlap_scheduler: true + speculative_config: *id001 diff --git a/tests/scripts/perf/disaggregated/gb200_glm-5-fp4_8k1k_con1024_ctx1_dep4_gen1_dep8_eplb256_mtp1_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_glm-5-fp4_8k1k_con1024_ctx1_dep4_gen1_dep8_eplb256_mtp1_ccb-NIXL.yaml new file mode 100644 index 000000000000..d6132f177645 --- /dev/null +++ b/tests/scripts/perf/disaggregated/gb200_glm-5-fp4_8k1k_con1024_ctx1_dep4_gen1_dep8_eplb256_mtp1_ccb-NIXL.yaml @@ -0,0 +1,108 @@ +metadata: + model_name: glm_5_nvfp4 + precision: fp4 + model_dir_name: GLM-5-NVFP4 + supported_gpus: + - GB200 + script_file: disaggr_torch.slurm + benchmark_type: 8k1k +slurm: + script_file: disaggr_torch.slurm + partition: + account: + job_time: 02:00:00 + job_name: unified-benchmark + extra_args: --gres=gpu:4 + numa_bind: true +benchmark: + mode: e2e + use_nv_sa_benchmark: false + multi_round: 10 + benchmark_ratio: 0.0 + streaming: true + concurrency_list: '1024' + input_length: 8192 + output_length: 1024 + dataset_file: +hardware: + gpus_per_node: 4 + num_ctx_servers: 1 + num_gen_servers: 1 +environment: + container_mount: + container_image: + model_path: + trtllm_repo: '' + build_wheel: false + work_dir: + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1 + server_env_var: TRTLLM_SERVER_DISABLE_GC=1 +profiling: + nsys_on: false +accuracy: + enable_accuracy_test: false + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml +worker_config: + gen: + print_iter_log: true + max_batch_size: 128 + max_num_tokens: 256 + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + context_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + cuda_graph_config: + enable_padding: true + max_batch_size: 128 + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + dtype: fp8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + load_balancer: + num_slots: 256 + layer_updates_per_iter: 1 + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: NIXL + disable_overlap_scheduler: true + speculative_config: &id001 + decoding_type: MTP + num_nextn_predict_layers: 1 + num_postprocess_workers: 4 + stream_interval: 20 + ctx: + print_iter_log: true + max_batch_size: 2 + max_num_tokens: 16384 + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + context_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + cuda_graph_config: null + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + dtype: fp8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: NIXL + disable_overlap_scheduler: true + speculative_config: *id001 diff --git a/tests/scripts/perf/disaggregated/gb200_glm-5-fp4_8k1k_con1024_ctx1_dep4_gen1_dep8_eplb256_mtp1_ccb-UCX.yaml b/tests/scripts/perf/disaggregated/gb200_glm-5-fp4_8k1k_con1024_ctx1_dep4_gen1_dep8_eplb256_mtp1_ccb-UCX.yaml new file mode 100644 index 000000000000..19c11f4a1f66 --- /dev/null +++ b/tests/scripts/perf/disaggregated/gb200_glm-5-fp4_8k1k_con1024_ctx1_dep4_gen1_dep8_eplb256_mtp1_ccb-UCX.yaml @@ -0,0 +1,108 @@ +metadata: + model_name: glm_5_nvfp4 + precision: fp4 + model_dir_name: GLM-5-NVFP4 + supported_gpus: + - GB200 + script_file: disaggr_torch.slurm + benchmark_type: 8k1k +slurm: + script_file: disaggr_torch.slurm + partition: + account: + job_time: 02:00:00 + job_name: unified-benchmark + extra_args: --gres=gpu:4 + numa_bind: true +benchmark: + mode: e2e + use_nv_sa_benchmark: false + multi_round: 10 + benchmark_ratio: 0.0 + streaming: true + concurrency_list: '1024' + input_length: 8192 + output_length: 1024 + dataset_file: +hardware: + gpus_per_node: 4 + num_ctx_servers: 1 + num_gen_servers: 1 +environment: + container_mount: + container_image: + model_path: + trtllm_repo: '' + build_wheel: false + work_dir: + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1 + server_env_var: TRTLLM_SERVER_DISABLE_GC=1 +profiling: + nsys_on: false +accuracy: + enable_accuracy_test: false + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml +worker_config: + gen: + print_iter_log: true + max_batch_size: 128 + max_num_tokens: 256 + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + context_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + cuda_graph_config: + enable_padding: true + max_batch_size: 128 + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + dtype: fp8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + load_balancer: + num_slots: 256 + layer_updates_per_iter: 1 + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: UCX + disable_overlap_scheduler: true + speculative_config: &id001 + decoding_type: MTP + num_nextn_predict_layers: 1 + num_postprocess_workers: 4 + stream_interval: 20 + ctx: + print_iter_log: true + max_batch_size: 2 + max_num_tokens: 16384 + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + context_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + cuda_graph_config: null + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + dtype: fp8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: UCX + disable_overlap_scheduler: true + speculative_config: *id001 diff --git a/tests/scripts/perf/disaggregated/gb200_glm-5-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_glm-5-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml new file mode 100644 index 000000000000..221864404b62 --- /dev/null +++ b/tests/scripts/perf/disaggregated/gb200_glm-5-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml @@ -0,0 +1,104 @@ +metadata: + model_name: glm_5_nvfp4 + precision: fp4 + model_dir_name: GLM-5-NVFP4 + supported_gpus: + - GB200 + script_file: disaggr_torch.slurm + benchmark_type: 8k1k +slurm: + script_file: disaggr_torch.slurm + partition: + account: + job_time: 02:00:00 + job_name: unified-benchmark + extra_args: --gres=gpu:4 + numa_bind: true +benchmark: + mode: e2e + use_nv_sa_benchmark: false + multi_round: 10 + benchmark_ratio: 0.0 + streaming: true + concurrency_list: '1' + input_length: 8192 + output_length: 1024 + dataset_file: +hardware: + gpus_per_node: 4 + num_ctx_servers: 1 + num_gen_servers: 1 +environment: + container_mount: + container_image: + model_path: + trtllm_repo: '' + build_wheel: false + work_dir: + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 + server_env_var: TRTLLM_SERVER_DISABLE_GC=1 +profiling: + nsys_on: false +accuracy: + enable_accuracy_test: false + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml +worker_config: + gen: + print_iter_log: true + max_batch_size: 128 + max_num_tokens: 512 + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + context_parallel_size: 1 + enable_attention_dp: false + cuda_graph_config: + enable_padding: true + max_batch_size: 128 + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + dtype: fp8 + moe_config: + backend: TRTLLM + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: NIXL + disable_overlap_scheduler: true + speculative_config: &id001 + decoding_type: MTP + num_nextn_predict_layers: 3 + num_postprocess_workers: 4 + stream_interval: 20 + allreduce_strategy: MNNVL + ctx: + print_iter_log: true + max_batch_size: 2 + max_num_tokens: 16384 + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + context_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + cuda_graph_config: null + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + dtype: fp8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: NIXL + disable_overlap_scheduler: true + speculative_config: *id001 diff --git a/tests/scripts/perf/disaggregated/gb200_glm-5-fp4_8k1k_con512_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_glm-5-fp4_8k1k_con512_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml new file mode 100644 index 000000000000..81a45d042213 --- /dev/null +++ b/tests/scripts/perf/disaggregated/gb200_glm-5-fp4_8k1k_con512_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml @@ -0,0 +1,105 @@ +metadata: + model_name: glm_5_nvfp4 + precision: fp4 + model_dir_name: GLM-5-NVFP4 + supported_gpus: + - GB200 + script_file: disaggr_torch.slurm + benchmark_type: 8k1k +slurm: + script_file: disaggr_torch.slurm + partition: + account: + job_time: 02:00:00 + job_name: unified-benchmark + extra_args: --gres=gpu:4 + numa_bind: true +benchmark: + mode: e2e + use_nv_sa_benchmark: false + multi_round: 10 + benchmark_ratio: 0.0 + streaming: true + concurrency_list: '512' + input_length: 8192 + output_length: 1024 + dataset_file: +hardware: + gpus_per_node: 4 + num_ctx_servers: 1 + num_gen_servers: 1 +environment: + container_mount: + container_image: + model_path: + trtllm_repo: '' + build_wheel: false + work_dir: + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 + server_env_var: TRTLLM_SERVER_DISABLE_GC=1 +profiling: + nsys_on: false +accuracy: + enable_accuracy_test: false + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml +worker_config: + gen: + print_iter_log: true + max_batch_size: 16 + max_num_tokens: 64 + tensor_parallel_size: 32 + moe_expert_parallel_size: 32 + pipeline_parallel_size: 1 + context_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + cuda_graph_config: + enable_padding: true + max_batch_size: 16 + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + dtype: fp8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: NIXL + disable_overlap_scheduler: true + speculative_config: &id001 + decoding_type: MTP + num_nextn_predict_layers: 3 + num_postprocess_workers: 4 + stream_interval: 20 + ctx: + print_iter_log: true + max_batch_size: 2 + max_num_tokens: 16384 + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + context_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + cuda_graph_config: null + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + dtype: fp8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: NIXL + disable_overlap_scheduler: true + speculative_config: *id001 diff --git a/tests/scripts/perf/disaggregated/gb200_kimi-k25-thinking-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_kimi-k25-thinking-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb416_mtp3_ccb-NIXL.yaml similarity index 97% rename from tests/scripts/perf/disaggregated/gb200_kimi-k25-thinking-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml rename to tests/scripts/perf/disaggregated/gb200_kimi-k25-thinking-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb416_mtp3_ccb-NIXL.yaml index b3d5ea9ab114..2454a4bd8d9a 100644 --- a/tests/scripts/perf/disaggregated/gb200_kimi-k25-thinking-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_kimi-k25-thinking-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb416_mtp3_ccb-NIXL.yaml @@ -71,6 +71,9 @@ worker_config: moe_config: backend: CUTEDSL use_low_precision_moe_combine: true + load_balancer: + num_slots: 416 + layer_updates_per_iter: 1 cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL diff --git a/tests/scripts/perf/disaggregated/gb300_glm-5-fp4_1k1k_con1_ctx1_dep2_gen1_tep4_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb300_glm-5-fp4_1k1k_con1_ctx1_dep2_gen1_tep4_eplb0_mtp3_ccb-NIXL.yaml new file mode 100644 index 000000000000..58bc0b8bfa6c --- /dev/null +++ b/tests/scripts/perf/disaggregated/gb300_glm-5-fp4_1k1k_con1_ctx1_dep2_gen1_tep4_eplb0_mtp3_ccb-NIXL.yaml @@ -0,0 +1,104 @@ +metadata: + model_name: glm_5_nvfp4 + precision: fp4 + model_dir_name: GLM-5-NVFP4 + supported_gpus: + - GB300 + script_file: disaggr_torch.slurm + benchmark_type: 1k1k +slurm: + script_file: disaggr_torch.slurm + partition: + account: + job_time: 02:00:00 + job_name: unified-benchmark + extra_args: --gres=gpu:4 + numa_bind: true +benchmark: + mode: e2e + use_nv_sa_benchmark: false + multi_round: 10 + benchmark_ratio: 0.0 + streaming: true + concurrency_list: '1' + input_length: 1024 + output_length: 1024 + dataset_file: +hardware: + gpus_per_node: 4 + num_ctx_servers: 1 + num_gen_servers: 1 +environment: + container_mount: + container_image: + model_path: + trtllm_repo: '' + build_wheel: false + work_dir: + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 + server_env_var: TRTLLM_SERVER_DISABLE_GC=1 +profiling: + nsys_on: false +accuracy: + enable_accuracy_test: false + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml +worker_config: + gen: + print_iter_log: true + max_batch_size: 128 + max_num_tokens: 512 + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + context_parallel_size: 1 + enable_attention_dp: false + cuda_graph_config: + enable_padding: true + max_batch_size: 128 + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + dtype: fp8 + moe_config: + backend: TRTLLM + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: NIXL + disable_overlap_scheduler: true + speculative_config: &id001 + decoding_type: MTP + num_nextn_predict_layers: 3 + num_postprocess_workers: 4 + stream_interval: 20 + allreduce_strategy: MNNVL + ctx: + print_iter_log: true + max_batch_size: 16 + max_num_tokens: 16384 + tensor_parallel_size: 2 + moe_expert_parallel_size: 2 + pipeline_parallel_size: 1 + context_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + cuda_graph_config: null + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + dtype: fp8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: NIXL + disable_overlap_scheduler: true + speculative_config: *id001 diff --git a/tests/scripts/perf/disaggregated/gb300_glm-5-fp4_1k1k_con4096_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb300_glm-5-fp4_1k1k_con4096_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-NIXL.yaml new file mode 100644 index 000000000000..e5785b456ecb --- /dev/null +++ b/tests/scripts/perf/disaggregated/gb300_glm-5-fp4_1k1k_con4096_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-NIXL.yaml @@ -0,0 +1,108 @@ +metadata: + model_name: glm_5_nvfp4 + precision: fp4 + model_dir_name: GLM-5-NVFP4 + supported_gpus: + - GB300 + script_file: disaggr_torch.slurm + benchmark_type: 1k1k +slurm: + script_file: disaggr_torch.slurm + partition: + account: + job_time: 02:00:00 + job_name: unified-benchmark + extra_args: --gres=gpu:4 + numa_bind: true +benchmark: + mode: e2e + use_nv_sa_benchmark: false + multi_round: 5 + benchmark_ratio: 0.0 + streaming: true + concurrency_list: '4096' + input_length: 1024 + output_length: 1024 + dataset_file: +hardware: + gpus_per_node: 4 + num_ctx_servers: 1 + num_gen_servers: 1 +environment: + container_mount: + container_image: + model_path: + trtllm_repo: '' + build_wheel: false + work_dir: + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1 + server_env_var: TRTLLM_SERVER_DISABLE_GC=1 +profiling: + nsys_on: false +accuracy: + enable_accuracy_test: false + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml +worker_config: + gen: + print_iter_log: true + max_batch_size: 512 + max_num_tokens: 1024 + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + context_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + cuda_graph_config: + enable_padding: true + max_batch_size: 512 + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + dtype: fp8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + load_balancer: + num_slots: 256 + layer_updates_per_iter: 1 + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: NIXL + disable_overlap_scheduler: true + speculative_config: &id001 + decoding_type: MTP + num_nextn_predict_layers: 1 + num_postprocess_workers: 4 + stream_interval: 20 + ctx: + print_iter_log: true + max_batch_size: 16 + max_num_tokens: 16384 + tensor_parallel_size: 2 + moe_expert_parallel_size: 2 + pipeline_parallel_size: 1 + context_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + cuda_graph_config: null + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + dtype: fp8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: NIXL + disable_overlap_scheduler: true + speculative_config: *id001 diff --git a/tests/scripts/perf/disaggregated/gb300_glm-5-fp4_1k1k_con4096_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-UCX.yaml b/tests/scripts/perf/disaggregated/gb300_glm-5-fp4_1k1k_con4096_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-UCX.yaml new file mode 100644 index 000000000000..17c05eb03d04 --- /dev/null +++ b/tests/scripts/perf/disaggregated/gb300_glm-5-fp4_1k1k_con4096_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-UCX.yaml @@ -0,0 +1,108 @@ +metadata: + model_name: glm_5_nvfp4 + precision: fp4 + model_dir_name: GLM-5-NVFP4 + supported_gpus: + - GB300 + script_file: disaggr_torch.slurm + benchmark_type: 1k1k +slurm: + script_file: disaggr_torch.slurm + partition: + account: + job_time: 02:00:00 + job_name: unified-benchmark + extra_args: --gres=gpu:4 + numa_bind: true +benchmark: + mode: e2e + use_nv_sa_benchmark: false + multi_round: 5 + benchmark_ratio: 0.0 + streaming: true + concurrency_list: '4096' + input_length: 1024 + output_length: 1024 + dataset_file: +hardware: + gpus_per_node: 4 + num_ctx_servers: 1 + num_gen_servers: 1 +environment: + container_mount: + container_image: + model_path: + trtllm_repo: '' + build_wheel: false + work_dir: + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1 + server_env_var: TRTLLM_SERVER_DISABLE_GC=1 +profiling: + nsys_on: false +accuracy: + enable_accuracy_test: false + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml +worker_config: + gen: + print_iter_log: true + max_batch_size: 512 + max_num_tokens: 1024 + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + context_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + cuda_graph_config: + enable_padding: true + max_batch_size: 512 + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + dtype: fp8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + load_balancer: + num_slots: 256 + layer_updates_per_iter: 1 + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: UCX + disable_overlap_scheduler: true + speculative_config: &id001 + decoding_type: MTP + num_nextn_predict_layers: 1 + num_postprocess_workers: 4 + stream_interval: 20 + ctx: + print_iter_log: true + max_batch_size: 16 + max_num_tokens: 16384 + tensor_parallel_size: 2 + moe_expert_parallel_size: 2 + pipeline_parallel_size: 1 + context_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + cuda_graph_config: null + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + dtype: fp8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: UCX + disable_overlap_scheduler: true + speculative_config: *id001 diff --git a/tests/scripts/perf/disaggregated/gb300_glm-5-fp4_1k1k_con512_ctx1_dep2_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb300_glm-5-fp4_1k1k_con512_ctx1_dep2_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml new file mode 100644 index 000000000000..e041b5b9b538 --- /dev/null +++ b/tests/scripts/perf/disaggregated/gb300_glm-5-fp4_1k1k_con512_ctx1_dep2_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml @@ -0,0 +1,105 @@ +metadata: + model_name: glm_5_nvfp4 + precision: fp4 + model_dir_name: GLM-5-NVFP4 + supported_gpus: + - GB300 + script_file: disaggr_torch.slurm + benchmark_type: 1k1k +slurm: + script_file: disaggr_torch.slurm + partition: + account: + job_time: 02:00:00 + job_name: unified-benchmark + extra_args: --gres=gpu:4 + numa_bind: true +benchmark: + mode: e2e + use_nv_sa_benchmark: false + multi_round: 10 + benchmark_ratio: 0.0 + streaming: true + concurrency_list: '512' + input_length: 1024 + output_length: 1024 + dataset_file: +hardware: + gpus_per_node: 4 + num_ctx_servers: 1 + num_gen_servers: 1 +environment: + container_mount: + container_image: + model_path: + trtllm_repo: '' + build_wheel: false + work_dir: + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 + server_env_var: TRTLLM_SERVER_DISABLE_GC=1 +profiling: + nsys_on: false +accuracy: + enable_accuracy_test: false + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml +worker_config: + gen: + print_iter_log: true + max_batch_size: 16 + max_num_tokens: 64 + tensor_parallel_size: 32 + moe_expert_parallel_size: 32 + pipeline_parallel_size: 1 + context_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + cuda_graph_config: + enable_padding: true + max_batch_size: 16 + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + dtype: fp8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: NIXL + disable_overlap_scheduler: true + speculative_config: &id001 + decoding_type: MTP + num_nextn_predict_layers: 3 + num_postprocess_workers: 4 + stream_interval: 20 + ctx: + print_iter_log: true + max_batch_size: 16 + max_num_tokens: 16384 + tensor_parallel_size: 2 + moe_expert_parallel_size: 2 + pipeline_parallel_size: 1 + context_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + cuda_graph_config: null + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + dtype: fp8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: NIXL + disable_overlap_scheduler: true + speculative_config: *id001 diff --git a/tests/scripts/perf/disaggregated/gb300_glm-5-fp4_8k1k_con1024_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb300_glm-5-fp4_8k1k_con1024_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-NIXL.yaml new file mode 100644 index 000000000000..dcf12ce44b52 --- /dev/null +++ b/tests/scripts/perf/disaggregated/gb300_glm-5-fp4_8k1k_con1024_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-NIXL.yaml @@ -0,0 +1,108 @@ +metadata: + model_name: glm_5_nvfp4 + precision: fp4 + model_dir_name: GLM-5-NVFP4 + supported_gpus: + - GB300 + script_file: disaggr_torch.slurm + benchmark_type: 8k1k +slurm: + script_file: disaggr_torch.slurm + partition: + account: + job_time: 02:00:00 + job_name: unified-benchmark + extra_args: --gres=gpu:4 + numa_bind: true +benchmark: + mode: e2e + use_nv_sa_benchmark: false + multi_round: 10 + benchmark_ratio: 0.0 + streaming: true + concurrency_list: '1024' + input_length: 8192 + output_length: 1024 + dataset_file: +hardware: + gpus_per_node: 4 + num_ctx_servers: 1 + num_gen_servers: 1 +environment: + container_mount: + container_image: + model_path: + trtllm_repo: '' + build_wheel: false + work_dir: + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1 + server_env_var: TRTLLM_SERVER_DISABLE_GC=1 +profiling: + nsys_on: false +accuracy: + enable_accuracy_test: false + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml +worker_config: + gen: + print_iter_log: true + max_batch_size: 128 + max_num_tokens: 256 + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + context_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + cuda_graph_config: + enable_padding: true + max_batch_size: 128 + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + dtype: fp8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + load_balancer: + num_slots: 256 + layer_updates_per_iter: 1 + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: NIXL + disable_overlap_scheduler: true + speculative_config: &id001 + decoding_type: MTP + num_nextn_predict_layers: 1 + num_postprocess_workers: 4 + stream_interval: 20 + ctx: + print_iter_log: true + max_batch_size: 2 + max_num_tokens: 16384 + tensor_parallel_size: 2 + moe_expert_parallel_size: 2 + pipeline_parallel_size: 1 + context_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + cuda_graph_config: null + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + dtype: fp8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: NIXL + disable_overlap_scheduler: true + speculative_config: *id001 diff --git a/tests/scripts/perf/disaggregated/gb300_glm-5-fp4_8k1k_con1024_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-UCX.yaml b/tests/scripts/perf/disaggregated/gb300_glm-5-fp4_8k1k_con1024_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-UCX.yaml new file mode 100644 index 000000000000..51f008f1ae4a --- /dev/null +++ b/tests/scripts/perf/disaggregated/gb300_glm-5-fp4_8k1k_con1024_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-UCX.yaml @@ -0,0 +1,108 @@ +metadata: + model_name: glm_5_nvfp4 + precision: fp4 + model_dir_name: GLM-5-NVFP4 + supported_gpus: + - GB300 + script_file: disaggr_torch.slurm + benchmark_type: 8k1k +slurm: + script_file: disaggr_torch.slurm + partition: + account: + job_time: 02:00:00 + job_name: unified-benchmark + extra_args: --gres=gpu:4 + numa_bind: true +benchmark: + mode: e2e + use_nv_sa_benchmark: false + multi_round: 10 + benchmark_ratio: 0.0 + streaming: true + concurrency_list: '1024' + input_length: 8192 + output_length: 1024 + dataset_file: +hardware: + gpus_per_node: 4 + num_ctx_servers: 1 + num_gen_servers: 1 +environment: + container_mount: + container_image: + model_path: + trtllm_repo: '' + build_wheel: false + work_dir: + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1 + server_env_var: TRTLLM_SERVER_DISABLE_GC=1 +profiling: + nsys_on: false +accuracy: + enable_accuracy_test: false + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml +worker_config: + gen: + print_iter_log: true + max_batch_size: 128 + max_num_tokens: 256 + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + context_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + cuda_graph_config: + enable_padding: true + max_batch_size: 128 + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + dtype: fp8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + load_balancer: + num_slots: 256 + layer_updates_per_iter: 1 + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: UCX + disable_overlap_scheduler: true + speculative_config: &id001 + decoding_type: MTP + num_nextn_predict_layers: 1 + num_postprocess_workers: 4 + stream_interval: 20 + ctx: + print_iter_log: true + max_batch_size: 2 + max_num_tokens: 16384 + tensor_parallel_size: 2 + moe_expert_parallel_size: 2 + pipeline_parallel_size: 1 + context_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + cuda_graph_config: null + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + dtype: fp8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: UCX + disable_overlap_scheduler: true + speculative_config: *id001 diff --git a/tests/scripts/perf/disaggregated/gb300_glm-5-fp4_8k1k_con1_ctx1_dep2_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb300_glm-5-fp4_8k1k_con1_ctx1_dep2_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml new file mode 100644 index 000000000000..85f0c280a12d --- /dev/null +++ b/tests/scripts/perf/disaggregated/gb300_glm-5-fp4_8k1k_con1_ctx1_dep2_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml @@ -0,0 +1,104 @@ +metadata: + model_name: glm_5_nvfp4 + precision: fp4 + model_dir_name: GLM-5-NVFP4 + supported_gpus: + - GB300 + script_file: disaggr_torch.slurm + benchmark_type: 8k1k +slurm: + script_file: disaggr_torch.slurm + partition: + account: + job_time: 02:00:00 + job_name: unified-benchmark + extra_args: --gres=gpu:4 + numa_bind: true +benchmark: + mode: e2e + use_nv_sa_benchmark: false + multi_round: 10 + benchmark_ratio: 0.0 + streaming: true + concurrency_list: '1' + input_length: 8192 + output_length: 1024 + dataset_file: +hardware: + gpus_per_node: 4 + num_ctx_servers: 1 + num_gen_servers: 1 +environment: + container_mount: + container_image: + model_path: + trtllm_repo: '' + build_wheel: false + work_dir: + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 + server_env_var: TRTLLM_SERVER_DISABLE_GC=1 +profiling: + nsys_on: false +accuracy: + enable_accuracy_test: false + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml +worker_config: + gen: + print_iter_log: true + max_batch_size: 128 + max_num_tokens: 512 + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + context_parallel_size: 1 + enable_attention_dp: false + cuda_graph_config: + enable_padding: true + max_batch_size: 128 + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + dtype: fp8 + moe_config: + backend: TRTLLM + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: NIXL + disable_overlap_scheduler: true + speculative_config: &id001 + decoding_type: MTP + num_nextn_predict_layers: 3 + num_postprocess_workers: 4 + stream_interval: 20 + allreduce_strategy: MNNVL + ctx: + print_iter_log: true + max_batch_size: 2 + max_num_tokens: 16384 + tensor_parallel_size: 2 + moe_expert_parallel_size: 2 + pipeline_parallel_size: 1 + context_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + cuda_graph_config: null + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + dtype: fp8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: NIXL + disable_overlap_scheduler: true + speculative_config: *id001 diff --git a/tests/scripts/perf/disaggregated/gb300_glm-5-fp4_8k1k_con512_ctx1_dep2_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb300_glm-5-fp4_8k1k_con512_ctx1_dep2_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml new file mode 100644 index 000000000000..a04c6f4eaf94 --- /dev/null +++ b/tests/scripts/perf/disaggregated/gb300_glm-5-fp4_8k1k_con512_ctx1_dep2_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml @@ -0,0 +1,105 @@ +metadata: + model_name: glm_5_nvfp4 + precision: fp4 + model_dir_name: GLM-5-NVFP4 + supported_gpus: + - GB300 + script_file: disaggr_torch.slurm + benchmark_type: 8k1k +slurm: + script_file: disaggr_torch.slurm + partition: + account: + job_time: 02:00:00 + job_name: unified-benchmark + extra_args: --gres=gpu:4 + numa_bind: true +benchmark: + mode: e2e + use_nv_sa_benchmark: false + multi_round: 10 + benchmark_ratio: 0.0 + streaming: true + concurrency_list: '512' + input_length: 8192 + output_length: 1024 + dataset_file: +hardware: + gpus_per_node: 4 + num_ctx_servers: 1 + num_gen_servers: 1 +environment: + container_mount: + container_image: + model_path: + trtllm_repo: '' + build_wheel: false + work_dir: + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 + server_env_var: TRTLLM_SERVER_DISABLE_GC=1 +profiling: + nsys_on: false +accuracy: + enable_accuracy_test: false + env_var: + HF_HOME: + tasks: + gsm8k_local: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096 + extra_kwargs: + trust_remote_code: true + custom_config: tests/integration/lm_eval_configs/gsm8k_local.yaml +worker_config: + gen: + print_iter_log: true + max_batch_size: 16 + max_num_tokens: 64 + tensor_parallel_size: 32 + moe_expert_parallel_size: 32 + pipeline_parallel_size: 1 + context_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + cuda_graph_config: + enable_padding: true + max_batch_size: 16 + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + dtype: fp8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: NIXL + disable_overlap_scheduler: true + speculative_config: &id001 + decoding_type: MTP + num_nextn_predict_layers: 3 + num_postprocess_workers: 4 + stream_interval: 20 + ctx: + print_iter_log: true + max_batch_size: 2 + max_num_tokens: 16384 + tensor_parallel_size: 2 + moe_expert_parallel_size: 2 + pipeline_parallel_size: 1 + context_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + cuda_graph_config: null + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + dtype: fp8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: NIXL + disable_overlap_scheduler: true + speculative_config: *id001 diff --git a/tests/scripts/perf/disaggregated/gb300_kimi-k25-thinking-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb300_kimi-k25-thinking-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb416_mtp3_ccb-NIXL.yaml similarity index 97% rename from tests/scripts/perf/disaggregated/gb300_kimi-k25-thinking-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml rename to tests/scripts/perf/disaggregated/gb300_kimi-k25-thinking-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb416_mtp3_ccb-NIXL.yaml index b9e3f0e89ba5..9dbfa216ef05 100644 --- a/tests/scripts/perf/disaggregated/gb300_kimi-k25-thinking-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb300_kimi-k25-thinking-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb416_mtp3_ccb-NIXL.yaml @@ -71,6 +71,9 @@ worker_config: moe_config: backend: CUTEDSL use_low_precision_moe_combine: true + load_balancer: + num_slots: 416 + layer_updates_per_iter: 1 cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL