diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 0baf89ec459f..8410937093fe 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -322,10 +322,8 @@ perf/test_perf_sanity.py::test_e2e[aggr_upload-super_ad_blackwell-super_ad_ws1_1 perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-NIXL] SKIP (https://nvbugs/6215844) perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL] SKIP (https://nvbugs/6280649) perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-NIXL] SKIP (https://nvbugs/6179661) -perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL] SKIP (https://nvbugs/6280649) perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL] SKIP (https://nvbugs/6280649) perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-NIXL] SKIP (https://nvbugs/6280649) -perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-NIXL] SKIP (https://nvbugs/6221024) perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-v32-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL] SKIP (https://nvbugs/6280649) perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-v32-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] SKIP (https://nvbugs/6280721) perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL] SKIP (https://nvbugs/6280649) diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml index 92cae48ef671..48c0ea7c40f6 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml @@ -63,6 +63,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 131104 backend: NIXL + kv_transfer_timeout_ms: 600000 disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP @@ -89,5 +90,6 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 131104 backend: NIXL + kv_transfer_timeout_ms: 600000 disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_32k4k_con256_ctx1_dep8_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_32k4k_con256_ctx1_dep8_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml index 09ed51589940..f5fd13315d4f 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_32k4k_con256_ctx1_dep8_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_32k4k_con256_ctx1_dep8_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml @@ -70,6 +70,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 120000 backend: NIXL + kv_transfer_timeout_ms: 600000 num_postprocess_workers: 8 stream_interval: 10 ctx: @@ -97,3 +98,4 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 120000 backend: NIXL + kv_transfer_timeout_ms: 600000 diff --git a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml index a1dc23f751dc..942ed2d44abe 100644 --- a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml @@ -74,6 +74,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 131104 backend: NIXL + kv_transfer_timeout_ms: 600000 disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP @@ -100,5 +101,6 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 131104 backend: NIXL + kv_transfer_timeout_ms: 600000 disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml index 388a98538a54..b967b8c64a3d 100644 --- a/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml @@ -75,6 +75,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 131104 backend: NIXL + kv_transfer_timeout_ms: 600000 disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP @@ -101,5 +102,6 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 131104 backend: NIXL + kv_transfer_timeout_ms: 600000 disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-UCX.yaml b/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-UCX.yaml index 68210344a755..e905b6b9759e 100644 --- a/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-UCX.yaml +++ b/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-UCX.yaml @@ -75,6 +75,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 131104 backend: UCX + kv_transfer_timeout_ms: 600000 disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP @@ -101,5 +102,6 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 131104 backend: UCX + kv_transfer_timeout_ms: 600000 disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_32k4k_con256_ctx1_dep8_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_32k4k_con256_ctx1_dep8_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml index 09ed51589940..f5fd13315d4f 100644 --- a/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_32k4k_con256_ctx1_dep8_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_32k4k_con256_ctx1_dep8_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml @@ -70,6 +70,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 120000 backend: NIXL + kv_transfer_timeout_ms: 600000 num_postprocess_workers: 8 stream_interval: 10 ctx: @@ -97,3 +98,4 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 120000 backend: NIXL + kv_transfer_timeout_ms: 600000