diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 6d762e93e4c6..90916ca36488 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -299,12 +299,9 @@ perf/test_perf_sanity.py::test_e2e[aggr_upload-k25_thinking_fp4_blackwell-k25_th perf/test_perf_sanity.py::test_e2e[aggr_upload-llama3_1_8b_fp8_ad_hopper-llama3_1_8b_ad_ws1_1k1k] SKIP (https://nvbugs/6244474) perf/test_perf_sanity.py::test_e2e[aggr_upload-super_ad_blackwell-super_ad_ws1_1k1k] SKIP (https://nvbugs/6153575) perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-NIXL] SKIP (https://nvbugs/6215844) -perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL] SKIP (https://nvbugs/6085022) perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-NIXL] SKIP (https://nvbugs/6179661) -perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL] SKIP (https://nvbugs/6085022) perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-b200_deepseek-r1-fp4_8k1k_con1536_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL] SKIP (https://nvbugs/6016528) perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-NIXL] SKIP (https://nvbugs/6221024) -perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL] SKIP (https://nvbugs/6085022) perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-v32-fp4_32k4k_con256_ctx1_dep8_gen1_dep8_eplb0_mtp0_ccb-NIXL] SKIP (https://nvbugs/6085022) perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-v32-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL] SKIP (https://nvbugs/6200257) perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-NIXL] SKIP (https://nvbugs/6221022) diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL.yaml index ba4a156fba23..d17172e8d66d 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL.yaml @@ -17,7 +17,7 @@ slurm: benchmark: mode: e2e use_nv_sa_benchmark: false - multi_round: 5 + multi_round: 2 benchmark_ratio: 0.0 streaming: true concurrency_list: '2048' @@ -69,6 +69,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 32832 backend: NIXL + kv_transfer_timeout_ms: 600000 disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP @@ -105,5 +106,6 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 32832 backend: NIXL + kv_transfer_timeout_ms: 600000 disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL.yaml index b41688f89aa9..b7a042d78e58 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL.yaml @@ -17,7 +17,7 @@ slurm: benchmark: mode: e2e use_nv_sa_benchmark: false - multi_round: 5 + multi_round: 2 benchmark_ratio: 0.0 streaming: true concurrency_list: '2048' @@ -69,6 +69,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 32832 backend: NIXL + kv_transfer_timeout_ms: 600000 disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP @@ -105,5 +106,6 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 32832 backend: NIXL + kv_transfer_timeout_ms: 600000 disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL.yaml index 38fc1c8e88f7..94340dd398ef 100644 --- a/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL.yaml @@ -17,7 +17,7 @@ slurm: benchmark: mode: e2e use_nv_sa_benchmark: false - multi_round: 5 + multi_round: 2 benchmark_ratio: 0.0 streaming: true concurrency_list: '2048' @@ -80,6 +80,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 32832 backend: NIXL + kv_transfer_timeout_ms: 600000 disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP @@ -116,5 +117,6 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 32832 backend: NIXL + kv_transfer_timeout_ms: 600000 disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-UCX.yaml b/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-UCX.yaml index 34974af25222..69919b355ed8 100644 --- a/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-UCX.yaml +++ b/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-UCX.yaml @@ -17,7 +17,7 @@ slurm: benchmark: mode: e2e use_nv_sa_benchmark: false - multi_round: 5 + multi_round: 2 benchmark_ratio: 0.0 streaming: true concurrency_list: '2048' @@ -80,6 +80,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 32832 backend: UCX + kv_transfer_timeout_ms: 600000 disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP @@ -116,5 +117,6 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 32832 backend: UCX + kv_transfer_timeout_ms: 600000 disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL.yaml index 9ee99dff719a..3f9961739b88 100644 --- a/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL.yaml @@ -17,7 +17,7 @@ slurm: benchmark: mode: e2e use_nv_sa_benchmark: false - multi_round: 5 + multi_round: 2 benchmark_ratio: 0.0 streaming: true concurrency_list: '2048' @@ -80,6 +80,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 32832 backend: NIXL + kv_transfer_timeout_ms: 600000 disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP @@ -116,5 +117,6 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 32832 backend: NIXL + kv_transfer_timeout_ms: 600000 disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-UCX.yaml b/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-UCX.yaml index eef32468c274..86ede92716e3 100644 --- a/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-UCX.yaml +++ b/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-UCX.yaml @@ -17,7 +17,7 @@ slurm: benchmark: mode: e2e use_nv_sa_benchmark: false - multi_round: 5 + multi_round: 2 benchmark_ratio: 0.0 streaming: true concurrency_list: '2048' @@ -80,6 +80,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 32832 backend: UCX + kv_transfer_timeout_ms: 600000 disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP @@ -116,5 +117,6 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 32832 backend: UCX + kv_transfer_timeout_ms: 600000 disable_overlap_scheduler: true speculative_config: *id001