From d7d0e97e4ca36f36cfae999c38aeeed5fc8e2e78 Mon Sep 17 00:00:00 2001 From: "svc-glamr@nvidia.com" Date: Wed, 9 Sep 2026 23:45:16 +0000 Subject: [PATCH 1/3] fix(sglang): disable both CUDA graphs in the elastic-EP scale-up manifest SGLang asserts that decode and prefill CUDA graphs are both disabled before it permits elastic-EP runtime scale-up, so the SGLangDecodeWorker in tests/fault_tolerance/deploy/templates/sglang/sglang_elastic_ep.yaml crashed on startup and never reached Ready. --moe-a2a-backend nixl disables the prefill side as a side effect; nothing disables decode. Pass --cuda-graph-backend-decode disabled and --cuda-graph-backend-prefill disabled explicitly, in preference to the deprecated blanket --disable-cuda-graph, and record the precondition in the manifest's header REQUIREMENTS block. The joining group launched by run_sglang_elastic_ep_scale_up_test.sh hits the same assertion, and its launch knobs are declared to match the manifest, so it gets the same two flags. The joiner still passes no --node-rank 1, which SGLang asserts on separately for --elastic-ep-join-mode scale; that is untouched here. Signed-off-by: svc-glamr@nvidia.com --- .../sglang/run_sglang_elastic_ep_scale_up_test.sh | 1 + .../deploy/templates/sglang/sglang_elastic_ep.yaml | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/tests/fault_tolerance/deploy/templates/sglang/run_sglang_elastic_ep_scale_up_test.sh b/tests/fault_tolerance/deploy/templates/sglang/run_sglang_elastic_ep_scale_up_test.sh index daf35fe38863..38cc21e93745 100755 --- a/tests/fault_tolerance/deploy/templates/sglang/run_sglang_elastic_ep_scale_up_test.sh +++ b/tests/fault_tolerance/deploy/templates/sglang/run_sglang_elastic_ep_scale_up_test.sh @@ -167,6 +167,7 @@ launch_joiner() { --elastic-ep-backend mooncake --mooncake-ib-device $IB_DEVICE \ --enable-eplb --ep-num-redundant-experts 24 \ --elastic-ep-initial-size 4 --max-ep-size 8 \ + --cuda-graph-backend-decode disabled --cuda-graph-backend-prefill disabled \ --elastic-ep-join-mode scale --elastic-ep-join-rank-offset $rank_offset \ --dist-init-addr $DIST_INIT_ADDR \ --host 127.0.0.1 --port $((30000 + rank_offset)) \ diff --git a/tests/fault_tolerance/deploy/templates/sglang/sglang_elastic_ep.yaml b/tests/fault_tolerance/deploy/templates/sglang/sglang_elastic_ep.yaml index 4f7a5c281be4..936a293fdbee 100644 --- a/tests/fault_tolerance/deploy/templates/sglang/sglang_elastic_ep.yaml +++ b/tests/fault_tolerance/deploy/templates/sglang/sglang_elastic_ep.yaml @@ -31,6 +31,11 @@ # --mooncake-ib-device to the node's IB device list before applying. # * DYN_SYSTEM_ENABLED=true so the leader exposes the :9090 system server that # serves /engine/control/*. +# * BOTH the decode and the prefill CUDA graph disabled: SGLang asserts this +# before it permits elastic-EP runtime scale-up, and the worker exits on +# startup otherwise. --moe-a2a-backend nixl disables the prefill side on its +# own, as a side effect, but nothing disables the decode side, so both are +# passed explicitly below (--cuda-graph-backend-decode/-prefill disabled). # # The joining group is NOT launched by the operator yet (that orchestration is # tracked separately). The accompanying run_sglang_elastic_ep_scale_up_test.sh @@ -128,6 +133,11 @@ spec: - "4" - --max-ep-size - "8" + # Elastic-EP scale-up is asserted to run with both CUDA graphs off. + - --cuda-graph-backend-decode + - disabled + - --cuda-graph-backend-prefill + - disabled # Stable rendezvous address the joining group dials on scale-up. # Sourced from SGLANG_ELASTIC_SCALE_DIST_INIT (single source of truth). - --dist-init-addr From 2a5e4af11bdd88a64e310db0010121630020588c Mon Sep 17 00:00:00 2001 From: "svc-glamr@nvidia.com" Date: Thu, 10 Sep 2026 00:15:01 +0000 Subject: [PATCH 2/3] chore: remove factory working notes Signed-off-by: svc-glamr@nvidia.com --- .../deploy/templates/sglang/sglang_elastic_ep.yaml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/tests/fault_tolerance/deploy/templates/sglang/sglang_elastic_ep.yaml b/tests/fault_tolerance/deploy/templates/sglang/sglang_elastic_ep.yaml index 936a293fdbee..3606873b6216 100644 --- a/tests/fault_tolerance/deploy/templates/sglang/sglang_elastic_ep.yaml +++ b/tests/fault_tolerance/deploy/templates/sglang/sglang_elastic_ep.yaml @@ -31,11 +31,8 @@ # --mooncake-ib-device to the node's IB device list before applying. # * DYN_SYSTEM_ENABLED=true so the leader exposes the :9090 system server that # serves /engine/control/*. -# * BOTH the decode and the prefill CUDA graph disabled: SGLang asserts this -# before it permits elastic-EP runtime scale-up, and the worker exits on -# startup otherwise. --moe-a2a-backend nixl disables the prefill side on its -# own, as a side effect, but nothing disables the decode side, so both are -# passed explicitly below (--cuda-graph-backend-decode/-prefill disabled). +# * BOTH the decode and the prefill CUDA graph disabled, passed explicitly +# below: SGLang asserts this for elastic-EP scale-up and exits on startup. # # The joining group is NOT launched by the operator yet (that orchestration is # tracked separately). The accompanying run_sglang_elastic_ep_scale_up_test.sh From fcf86d9ed6f61ab66b4e916833a420bbbe2cd3b4 Mon Sep 17 00:00:00 2001 From: "svc-glamr@nvidia.com" Date: Thu, 10 Sep 2026 00:28:20 +0000 Subject: [PATCH 3/3] fix(sglang): drop duplicate CUDA-graph note in elastic-EP template The inline note above the CUDA-graph arguments restated the requirement already listed in the file's REQUIREMENTS header. Keep the header entry as the single statement of it. Signed-off-by: svc-glamr@nvidia.com --- .../deploy/templates/sglang/sglang_elastic_ep.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/fault_tolerance/deploy/templates/sglang/sglang_elastic_ep.yaml b/tests/fault_tolerance/deploy/templates/sglang/sglang_elastic_ep.yaml index 3606873b6216..50f753ea2076 100644 --- a/tests/fault_tolerance/deploy/templates/sglang/sglang_elastic_ep.yaml +++ b/tests/fault_tolerance/deploy/templates/sglang/sglang_elastic_ep.yaml @@ -130,7 +130,6 @@ spec: - "4" - --max-ep-size - "8" - # Elastic-EP scale-up is asserted to run with both CUDA graphs off. - --cuda-graph-backend-decode - disabled - --cuda-graph-backend-prefill