From 712761e1f4f91fab9802aaf7f8dedff778deddb8 Mon Sep 17 00:00:00 2001 From: Dmitry Tokarev Date: Tue, 21 Apr 2026 13:17:54 -0400 Subject: [PATCH 01/10] test(fault_tolerance): skip vllm prefill_cancel test pending engine fix test_request_cancellation_vllm_prefill_cancel[nats] and [tcp] have both failed in every one of the last 8 nightly vllm multi-gpu runs on main (16/16 consecutive failures). Failure signature is EngineDeadError: the vLLM engine crashes when the request is cancelled mid-prefill, so the expected "Aborted Prefill Request ID:" log line never appears and the 500ms poll times out. This is the same class of engine-level cancellation bug as the decode twin (test_request_cancellation_vllm_decode_cancel), which is already skipped under DYN-2606 (#8006). Mirror that skip here with a TODO placeholder until the Linear ticket ID is filled in. Co-Authored-By: Claude Opus 4.7 (1M context) Signed-off-by: Dmitry Tokarev --- tests/fault_tolerance/cancellation/test_vllm.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/fault_tolerance/cancellation/test_vllm.py b/tests/fault_tolerance/cancellation/test_vllm.py index 6d363f205d18..84043e08d105 100644 --- a/tests/fault_tolerance/cancellation/test_vllm.py +++ b/tests/fault_tolerance/cancellation/test_vllm.py @@ -396,6 +396,7 @@ def test_request_cancellation_vllm_decode_cancel( ) +@pytest.mark.skip(reason="Nightly CI failure: TODO()") @pytest.mark.timeout(150) # 3x average @pytest.mark.nightly @pytest.mark.gpu_2 From 32ce9e998e939d8f2f4427a6cd559fb36ef0049b Mon Sep 17 00:00:00 2001 From: Dmitry Tokarev Date: Tue, 21 Apr 2026 13:28:20 -0400 Subject: [PATCH 02/10] test(serve): skip qwen3-vl-2b e_pd/epd vLLM multimodal profiles MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three multimodal serve deployment tests have failed in every one of the last 7 nightly vllm amd64 runs on main (7/7 consistent): - test_serve_deployment[mm_e_pd_qwen3-vl-2b] (e_pd topology) - test_serve_deployment[mm_epd_qwen3-vl-2b] (epd topology) - test_serve_deployment[mm_epd_qwen3-vl-2b-video] (epd topology) Failure is identical in all three: vLLM v1 engine core initialization crashes during startup of Qwen/Qwen3-VL-2B-Instruct under the disagg launch scripts (disagg_multimodal_e_pd.sh, disagg_multimodal_epd.sh), so the health-check URL never comes up and the test times out. RuntimeError: Engine core initialization failed. See root cause above. at vllm/v1/engine/utils.py:1057 wait_for_engine_startup The matching agg and p_d topology tests pass, so this is specific to the e_pd/epd scripts. Same class of vLLM engine-init crash as #8411 for mm_agg_qwen2-audio-7b — not flaky, needs real investigation. Ticket placeholder left on each skipped TopologyConfig. Co-Authored-By: Claude Opus 4.7 (1M context) Signed-off-by: Dmitry Tokarev --- tests/serve/multimodal_profiles/vllm.py | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/tests/serve/multimodal_profiles/vllm.py b/tests/serve/multimodal_profiles/vllm.py index db25472de5c4..f5339d780799 100644 --- a/tests/serve/multimodal_profiles/vllm.py +++ b/tests/serve/multimodal_profiles/vllm.py @@ -29,12 +29,24 @@ profiled_vram_gib=9.6, ), "e_pd": TopologyConfig( - marks=[pytest.mark.pre_merge], + marks=[ + pytest.mark.skip( + reason="vLLM engine core init fails on disagg e_pd. " + "TODO()" + ), + pytest.mark.pre_merge, + ], timeout_s=340, single_gpu=True, ), "epd": TopologyConfig( - marks=[pytest.mark.pre_merge], + marks=[ + pytest.mark.skip( + reason="vLLM engine core init fails on disagg epd. " + "TODO()" + ), + pytest.mark.pre_merge, + ], timeout_s=300, single_gpu=True, ), @@ -56,7 +68,13 @@ delayed_start=60, ), "epd": TopologyConfig( - marks=[pytest.mark.pre_merge], + marks=[ + pytest.mark.skip( + reason="vLLM engine core init fails on disagg epd. " + "TODO()" + ), + pytest.mark.pre_merge, + ], timeout_s=600, delayed_start=60, single_gpu=True, From e4e89963efe7f649ec5e45c449ba63b68612bdbe Mon Sep 17 00:00:00 2001 From: Dmitry Tokarev Date: Tue, 21 Apr 2026 13:32:32 -0400 Subject: [PATCH 03/10] test(serve): link qwen3-vl-2b disagg skips to OPS-4445 Co-Authored-By: Claude Opus 4.7 (1M context) Signed-off-by: Dmitry Tokarev --- tests/serve/multimodal_profiles/vllm.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/serve/multimodal_profiles/vllm.py b/tests/serve/multimodal_profiles/vllm.py index f5339d780799..dd203339f215 100644 --- a/tests/serve/multimodal_profiles/vllm.py +++ b/tests/serve/multimodal_profiles/vllm.py @@ -32,7 +32,7 @@ marks=[ pytest.mark.skip( reason="vLLM engine core init fails on disagg e_pd. " - "TODO()" + "https://linear.app/nvidia/issue/OPS-4445" ), pytest.mark.pre_merge, ], @@ -43,7 +43,7 @@ marks=[ pytest.mark.skip( reason="vLLM engine core init fails on disagg epd. " - "TODO()" + "https://linear.app/nvidia/issue/OPS-4445" ), pytest.mark.pre_merge, ], @@ -71,7 +71,7 @@ marks=[ pytest.mark.skip( reason="vLLM engine core init fails on disagg epd. " - "TODO()" + "https://linear.app/nvidia/issue/OPS-4445" ), pytest.mark.pre_merge, ], From c1cc2fa45d69b4001f6e5056a7c248cd04ab2962 Mon Sep 17 00:00:00 2001 From: Dmitry Tokarev Date: Tue, 21 Apr 2026 13:40:55 -0400 Subject: [PATCH 04/10] test(fault_tolerance): convert vllm migration xfails to skips MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three vllm migration tests are marked @xfail(strict=False) for known architectural limitations: - test_request_migration_vllm_prefill (prefill migration unsupported) - test_request_migration_vllm_kv_transfer (request_id/KV cache reuse) - test_request_migration_vllm_decode (same request_id/KV cache reuse) The last seven nightly vllm multi-gpu runs logged 144 XFAIL results across these three tests with 0 XPASS — every non-skipped parameter combination fails exactly as predicted by the xfail reason. Running them anyway still incurs the per-parameterisation GPU setup cost and generates noise in the nightly report. Convert to @skip with the same reason strings. No functional change expected: if any of these start passing, the skip reason is the signal to re-enable, same as xfail(strict=False) would have been. Co-Authored-By: Claude Opus 4.7 (1M context) Signed-off-by: Dmitry Tokarev --- tests/fault_tolerance/migration/test_vllm.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/tests/fault_tolerance/migration/test_vllm.py b/tests/fault_tolerance/migration/test_vllm.py index f92457ff3297..c2dce82ca779 100644 --- a/tests/fault_tolerance/migration/test_vllm.py +++ b/tests/fault_tolerance/migration/test_vllm.py @@ -271,7 +271,7 @@ def test_request_migration_vllm_aggregated( ) -@pytest.mark.xfail(strict=False, reason="Prefill migration not yet supported") +@pytest.mark.skip(reason="Prefill migration not yet supported") @pytest.mark.timeout(350) # 3x average @pytest.mark.nightly def test_request_migration_vllm_prefill( @@ -346,8 +346,7 @@ def test_request_migration_vllm_prefill( ) -@pytest.mark.xfail( - strict=False, +@pytest.mark.skip( reason=( "Migration reuses the same request_id for vLLM, but the prefill worker's " "KV cache still holds the request due to delay_free_blocks in disaggregated mode. " @@ -430,8 +429,7 @@ def test_request_migration_vllm_kv_transfer( ) -@pytest.mark.xfail( - strict=False, +@pytest.mark.skip( reason=( "Migration reuses the same request_id for vLLM, but the prefill worker's " "KV cache still holds the request due to delay_free_blocks in disaggregated mode. " From d634588d6da10843dd1d122cd8bce3b28c876b3b Mon Sep 17 00:00:00 2001 From: Dmitry Tokarev Date: Tue, 21 Apr 2026 14:12:56 -0400 Subject: [PATCH 05/10] test(serve): allow http URLs in multimodal_agg_llava + aggregated_toolcalling Two legacy VLLMConfig entries in tests/serve/test_vllm.py started failing in today's nightly vllm amd64 job with: Exception: Failed to load image from http://...: http:// URLs are not allowed; set DYN_MM_ALLOW_INTERNAL=1 to enable The strict URL validator landed in #8282 and correctly plumbed the env var through the factory for VLLM_MULTIMODAL_PROFILES-generated configs in tests/utils/multimodal.py, but missed these two hand- authored configs that still use http:// URLs (cocodataset HTTP image and the local test fixture server at localhost:8806). Set DYN_MM_ALLOW_INTERNAL=1 on both configs so they can load http:// payloads like before, matching what the factory already does for the mm_* generated configs. Co-Authored-By: Claude Opus 4.7 (1M context) Signed-off-by: Dmitry Tokarev --- tests/serve/test_vllm.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/serve/test_vllm.py b/tests/serve/test_vllm.py index 2a83bdf4bf4a..bd074f1e009f 100644 --- a/tests/serve/test_vllm.py +++ b/tests/serve/test_vllm.py @@ -422,6 +422,7 @@ class VLLMConfig(EngineConfig): ], model="llava-hf/llava-1.5-7b-hf", script_args=["--model", "llava-hf/llava-1.5-7b-hf"], + env={"DYN_MM_ALLOW_INTERNAL": "1"}, delayed_start=0, timeout=360, request_payloads=[ @@ -471,6 +472,7 @@ class VLLMConfig(EngineConfig): "--dyn-tool-call-parser", "hermes", ], + env={"DYN_MM_ALLOW_INTERNAL": "1"}, delayed_start=0, timeout=600, request_payloads=[ From c145464d7b271b963f1294c6a355c269bf0ddb92 Mon Sep 17 00:00:00 2001 From: Dmitry Tokarev Date: Tue, 21 Apr 2026 14:25:51 -0400 Subject: [PATCH 06/10] Apply suggestion from @dmitry-tokarev-nv Signed-off-by: Dmitry Tokarev --- tests/fault_tolerance/cancellation/test_vllm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/fault_tolerance/cancellation/test_vllm.py b/tests/fault_tolerance/cancellation/test_vllm.py index 84043e08d105..6001b51f2723 100644 --- a/tests/fault_tolerance/cancellation/test_vllm.py +++ b/tests/fault_tolerance/cancellation/test_vllm.py @@ -396,7 +396,7 @@ def test_request_cancellation_vllm_decode_cancel( ) -@pytest.mark.skip(reason="Nightly CI failure: TODO()") +@pytest.mark.skip(reason="Nightly CI failure: OPS-4448") @pytest.mark.timeout(150) # 3x average @pytest.mark.nightly @pytest.mark.gpu_2 From b3ac5ce25edfae55ce84ef43c67cb0c86af47f5f Mon Sep 17 00:00:00 2001 From: Dmitry Tokarev Date: Tue, 21 Apr 2026 14:50:34 -0400 Subject: [PATCH 07/10] test(fault_tolerance): extend sglang migration skip to TCP transport (OPS-4446) PR #8404 skipped only the NATS variant of test_request_migration_sglang_aggregated[migration_enabled-max_seq_len_disabled-worker_failure-chat-stream-*] because the TCP variant was stable at the time. Starting from the first post-merge run after that skip landed, the TCP variant began failing with the same symptom (AssertionError: Delay before response > 6 secs, got ~13 secs at utils.py:417) in every run: 24733651809, 24733908729, 24734216001, 24735128282, and onwards. The cause is ordering: when NATS ran first, it warmed the path; with NATS skipped, TCP now pays the cold-start cost itself. Drop the request_plane == "nats" clause so both transports skip, and also replace the stale TODO() comment with the OPS-4446 reference that the skip reason string already uses. Co-Authored-By: Claude Opus 4.7 (1M context) Signed-off-by: Dmitry Tokarev --- tests/fault_tolerance/migration/test_sglang.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/tests/fault_tolerance/migration/test_sglang.py b/tests/fault_tolerance/migration/test_sglang.py index 9314e7d3d316..1299160818d5 100644 --- a/tests/fault_tolerance/migration/test_sglang.py +++ b/tests/fault_tolerance/migration/test_sglang.py @@ -235,20 +235,18 @@ def test_request_migration_sglang_aggregated( stream: True for streaming, False for non-streaming """ - # TODO(): Flaky on NATS transport — first-token delay routinely - # exceeds the 6s threshold in utils.validate_response. Other parameter - # combinations (including the TCP variant) are stable. + # OPS-4446: first-token delay routinely exceeds the 6s threshold in + # utils.validate_response for this parameter combination. Originally only + # the NATS variant tripped; once the NATS skip landed, the TCP variant + # started failing the same way (now bears the cold-start cost first). if ( migration_limit == 3 and migration_max_seq_len is None and immediate_kill is True and request_api == "chat" and stream is True - and request.getfixturevalue("request_plane") == "nats" ): - pytest.skip( - "Flaky on NATS transport: first-token delay > 6s threshold. OPS-4446" - ) + pytest.skip("Flaky: first-token delay > 6s threshold. OPS-4446") # Step 1: Start the frontend with DynamoFrontendProcess( From 2dabd67f33c29acdad5cb9a29c5d6d10269fcb34 Mon Sep 17 00:00:00 2001 From: Dmitry Tokarev Date: Tue, 21 Apr 2026 15:19:56 -0400 Subject: [PATCH 08/10] test(trtllm): skip nightly failures, convert cancellation xfails to skips MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two concerns in the trtllm nightly `Test cuda13.1 (amd64)` job: 1) Convert xfails → skips in tests/fault_tolerance/cancellation/test_trtllm.py The module-level xfail (pytestmark, strict=True, "Cancellation is temporarily disabled") and the function-level xfail on test_request_cancellation_trtllm_kv_transfer_cancel (strict=False, "Test fails only on CI") have both produced only XFAIL (0 XPASS) across the last 7 nightlies, matching the vllm migration pattern in commit c1cc2fa45d. Switch to @pytest.mark.skip with the same reason text to stop burning GPU on tests that are definitionally expected to fail. 2) Skip 4 persistent trtllm failures All four have failed in the last 6+ nightlies with the same family of trtllm engine-crash errors (`AssertionError: assert 'error' == 'ok'`, `RuntimeError: Executor worker returned error`, main server dying with exit 1 during health-check): - test_gms_basic_quiesce_resume_trtllm - test_gms_read_only_import_trtllm - test_gms_shadow_engine_failover_trtllm - test_deployment[disaggregated_same_gpu-2] Marked with TODO() placeholder — to be swapped for the real Linear ticket(s) before merge. Likely share a single ticket given the identical engine-crash family. Co-Authored-By: Claude Opus 4.7 (1M context) Signed-off-by: Dmitry Tokarev --- tests/fault_tolerance/cancellation/test_trtllm.py | 4 ++-- tests/gpu_memory_service/test_quiesce_resume.py | 2 ++ tests/gpu_memory_service/test_shadow_failover.py | 1 + tests/serve/test_trtllm.py | 1 + 4 files changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/fault_tolerance/cancellation/test_trtllm.py b/tests/fault_tolerance/cancellation/test_trtllm.py index 43a0794e0146..4ea014dfdcef 100644 --- a/tests/fault_tolerance/cancellation/test_trtllm.py +++ b/tests/fault_tolerance/cancellation/test_trtllm.py @@ -40,7 +40,7 @@ pytest.mark.model(FAULT_TOLERANCE_MODEL_NAME), pytest.mark.nightly, pytest.mark.parametrize("request_plane", ["nats", "tcp"], indirect=True), - pytest.mark.xfail(reason="Cancellation is temporarily disabled", strict=True), + pytest.mark.skip(reason="Cancellation is temporarily disabled"), ] @@ -473,7 +473,7 @@ def test_request_cancellation_trtllm_prefill_cancel( ) -@pytest.mark.xfail(reason="Test fails only on CI", strict=False) +@pytest.mark.skip(reason="Test fails only on CI") @pytest.mark.timeout(195) # 3x average def test_request_cancellation_trtllm_kv_transfer_cancel( request, runtime_services_dynamic_ports, predownload_models diff --git a/tests/gpu_memory_service/test_quiesce_resume.py b/tests/gpu_memory_service/test_quiesce_resume.py index 11152fe353c6..056bdd676312 100644 --- a/tests/gpu_memory_service/test_quiesce_resume.py +++ b/tests/gpu_memory_service/test_quiesce_resume.py @@ -131,6 +131,7 @@ def test_gms_basic_quiesce_resume_sglang( # --------------------------------------------------------------------------- +@pytest.mark.skip(reason="Nightly CI failure: TODO()") @pytest.mark.trtllm @pytest.mark.e2e @pytest.mark.gpu_1 @@ -177,6 +178,7 @@ def test_gms_basic_quiesce_resume_trtllm( ) +@pytest.mark.skip(reason="Nightly CI failure: TODO()") @pytest.mark.trtllm @pytest.mark.e2e @pytest.mark.gpu_1 diff --git a/tests/gpu_memory_service/test_shadow_failover.py b/tests/gpu_memory_service/test_shadow_failover.py index d1f4bf2b53e1..bdbe26565977 100644 --- a/tests/gpu_memory_service/test_shadow_failover.py +++ b/tests/gpu_memory_service/test_shadow_failover.py @@ -302,6 +302,7 @@ def _trtllm_quiesce( return ws +@pytest.mark.skip(reason="Nightly CI failure: TODO()") @pytest.mark.trtllm @pytest.mark.e2e @pytest.mark.gpu_1 diff --git a/tests/serve/test_trtllm.py b/tests/serve/test_trtllm.py index f8459143a3ed..a9b902d6f4af 100644 --- a/tests/serve/test_trtllm.py +++ b/tests/serve/test_trtllm.py @@ -138,6 +138,7 @@ class TRTLLMConfig(EngineConfig): directory=trtllm_dir, script_name="disagg_same_gpu.sh", marks=[ + pytest.mark.skip(reason="Nightly CI failure: TODO()"), pytest.mark.gpu_1, # 1 GPU(s) used, peak 6.6 GiB pytest.mark.pre_merge, pytest.mark.trtllm, From 8084d1bf7052ae3251b831c39de0906ddcc558ce Mon Sep 17 00:00:00 2001 From: Dmitry Tokarev Date: Tue, 21 Apr 2026 15:32:59 -0400 Subject: [PATCH 09/10] test(trtllm): link skipped nightly failures to OPS-4450 Co-Authored-By: Claude Opus 4.7 (1M context) Signed-off-by: Dmitry Tokarev --- tests/gpu_memory_service/test_quiesce_resume.py | 4 ++-- tests/gpu_memory_service/test_shadow_failover.py | 2 +- tests/serve/test_trtllm.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/gpu_memory_service/test_quiesce_resume.py b/tests/gpu_memory_service/test_quiesce_resume.py index 056bdd676312..1390450cee6b 100644 --- a/tests/gpu_memory_service/test_quiesce_resume.py +++ b/tests/gpu_memory_service/test_quiesce_resume.py @@ -131,7 +131,7 @@ def test_gms_basic_quiesce_resume_sglang( # --------------------------------------------------------------------------- -@pytest.mark.skip(reason="Nightly CI failure: TODO()") +@pytest.mark.skip(reason="Nightly CI failure: https://linear.app/nvidia/issue/OPS-4450") @pytest.mark.trtllm @pytest.mark.e2e @pytest.mark.gpu_1 @@ -178,7 +178,7 @@ def test_gms_basic_quiesce_resume_trtllm( ) -@pytest.mark.skip(reason="Nightly CI failure: TODO()") +@pytest.mark.skip(reason="Nightly CI failure: https://linear.app/nvidia/issue/OPS-4450") @pytest.mark.trtllm @pytest.mark.e2e @pytest.mark.gpu_1 diff --git a/tests/gpu_memory_service/test_shadow_failover.py b/tests/gpu_memory_service/test_shadow_failover.py index bdbe26565977..388407b70de2 100644 --- a/tests/gpu_memory_service/test_shadow_failover.py +++ b/tests/gpu_memory_service/test_shadow_failover.py @@ -302,7 +302,7 @@ def _trtllm_quiesce( return ws -@pytest.mark.skip(reason="Nightly CI failure: TODO()") +@pytest.mark.skip(reason="Nightly CI failure: https://linear.app/nvidia/issue/OPS-4450") @pytest.mark.trtllm @pytest.mark.e2e @pytest.mark.gpu_1 diff --git a/tests/serve/test_trtllm.py b/tests/serve/test_trtllm.py index a9b902d6f4af..8da06f870c33 100644 --- a/tests/serve/test_trtllm.py +++ b/tests/serve/test_trtllm.py @@ -138,7 +138,7 @@ class TRTLLMConfig(EngineConfig): directory=trtllm_dir, script_name="disagg_same_gpu.sh", marks=[ - pytest.mark.skip(reason="Nightly CI failure: TODO()"), + pytest.mark.skip(reason="Nightly CI failure: https://linear.app/nvidia/issue/OPS-4450"), pytest.mark.gpu_1, # 1 GPU(s) used, peak 6.6 GiB pytest.mark.pre_merge, pytest.mark.trtllm, From 02d348a5a72ec3a77f1b66516f164de0e9de91dc Mon Sep 17 00:00:00 2001 From: Dmitry Tokarev Date: Tue, 21 Apr 2026 16:06:57 -0400 Subject: [PATCH 10/10] formatting Signed-off-by: Dmitry Tokarev --- tests/serve/test_trtllm.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/serve/test_trtllm.py b/tests/serve/test_trtllm.py index 8da06f870c33..3b3e8ddd06dc 100644 --- a/tests/serve/test_trtllm.py +++ b/tests/serve/test_trtllm.py @@ -138,7 +138,9 @@ class TRTLLMConfig(EngineConfig): directory=trtllm_dir, script_name="disagg_same_gpu.sh", marks=[ - pytest.mark.skip(reason="Nightly CI failure: https://linear.app/nvidia/issue/OPS-4450"), + pytest.mark.skip( + reason="Nightly CI failure: https://linear.app/nvidia/issue/OPS-4450" + ), pytest.mark.gpu_1, # 1 GPU(s) used, peak 6.6 GiB pytest.mark.pre_merge, pytest.mark.trtllm,