From 758bd164a5b0003e83d609acdad8262a8df28126 Mon Sep 17 00:00:00 2001 From: Banruo Liu Date: Wed, 23 Sep 2026 22:54:57 -0500 Subject: [PATCH 1/6] test(sglang): add EAGLE3 speculative decoding e2e Add an aggregated SGLang speculative decoding launch script and a nightly serve test, mirroring the existing vLLM aggregated_spec_decoding case. - launch/agg_spec_decoding.sh: Qwen3-8B with the Tengyunw/qwen3_8b_eagle3 draft model (both ungated), metrics enabled. - SGLangSpecDecodeMetricsPayload: asserts speculation actually ran (sglang:spec_verify_calls_total > 0) and that draft tokens are being accepted (generation_tokens_total / spec_verify_calls_total > 1.2), using cumulative counters rather than the windowed spec_accept_length gauge. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Banruo Liu --- .../sglang/launch/agg_spec_decoding.sh | 48 +++++++++++++++++++ tests/serve/test_sglang.py | 44 +++++++++++++++++ tests/utils/payloads.py | 41 ++++++++++++++++ 3 files changed, 133 insertions(+) create mode 100755 examples/backends/sglang/launch/agg_spec_decoding.sh diff --git a/examples/backends/sglang/launch/agg_spec_decoding.sh b/examples/backends/sglang/launch/agg_spec_decoding.sh new file mode 100755 index 000000000000..b095d0fd66eb --- /dev/null +++ b/examples/backends/sglang/launch/agg_spec_decoding.sh @@ -0,0 +1,48 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +set -e +trap 'echo Cleaning up...; kill 0' EXIT + +SCRIPT_DIR="$(dirname "$(readlink -f "$0")")" +source "$SCRIPT_DIR/../../../common/launch_utils.sh" +source "$SCRIPT_DIR/../../../common/gpu_utils.sh" # build_sglang_gpu_mem_args + +# When the serve test sets requested_sglang_kv_tokens, this emits a +# --max-total-tokens cap so VRAM is bounded (GPU-size-independent). Empty for +# manual runs. +GPU_MEM_ARGS=$(build_sglang_gpu_mem_args) + +MODEL="Qwen/Qwen3-8B" +DRAFT_MODEL="Tengyunw/qwen3_8b_eagle3" +HTTP_PORT="${DYN_HTTP_PORT:-8000}" +print_launch_banner "Launching Speculative Decoding (1 GPU)" "$MODEL" "$HTTP_PORT" + +# --------------------------- +# 1. Frontend (Ingress) +# --------------------------- +python3 -m dynamo.frontend & + + +# --------------------------- +# 2. Speculative Main Worker +# --------------------------- +# This runs the main model with an EAGLE3 draft model for speculative decoding. +# --enable-metrics exposes sglang:spec_* metrics on the worker system port. +DYN_SYSTEM_PORT=${DYN_SYSTEM_PORT:-8081} \ +python3 -m dynamo.sglang \ + --model-path "$MODEL" \ + --served-model-name "$MODEL" \ + --page-size 16 \ + --tp 1 \ + --trust-remote-code \ + --enable-metrics \ + --speculative-algorithm EAGLE3 \ + --speculative-draft-model-path "$DRAFT_MODEL" \ + --speculative-num-steps 3 \ + --speculative-eagle-topk 1 \ + --speculative-num-draft-tokens 4 \ + $GPU_MEM_ARGS "$@" & + +# Exit on first worker failure; kill 0 in the EXIT trap tears down the rest +wait_any_exit diff --git a/tests/serve/test_sglang.py b/tests/serve/test_sglang.py index a4b7f028f80f..56e701c403ca 100644 --- a/tests/serve/test_sglang.py +++ b/tests/serve/test_sglang.py @@ -56,6 +56,7 @@ ResponsesPayload, ResponsesStreamPayload, SGLangDisaggRouterMetricsPayload, + SGLangSpecDecodeMetricsPayload, VideoGenerationPayload, ) from tests.utils.port_utils import allocate_contiguous_ports, deallocate_ports @@ -154,6 +155,49 @@ class SGLangConfig(EngineConfig): metric_payload_default(min_num_requests=6, backend="sglang"), ], ), + # Speculative decoding: Qwen3-8B main model with an EAGLE3 draft model + # (see launch/agg_spec_decoding.sh). Both repos are ungated. + # Nightly-only: the 8B base plus EAGLE3 draft model is intentionally outside pre-merge CI. + "aggregated_spec_decoding": SGLangConfig( + name="aggregated_spec_decoding", + directory=sglang_dir, + script_name="agg_spec_decoding.sh", + marks=[ + pytest.mark.gpu_1, + # Also predownload the EAGLE3 draft: CI workers run HF_HUB_OFFLINE=True + # and only the base cfg.model is auto-registered, so the draft repo + # can't be resolved offline without this. + pytest.mark.model("Tengyunw/qwen3_8b_eagle3"), + # Measured peak ~20.6 GiB on H200 (8B weights + EAGLE3 draft + + # capped KV + CUDA graphs). + pytest.mark.profiled_vram_gib(21.0), + pytest.mark.requested_sglang_kv_tokens(4096), + pytest.mark.timeout(300), # ~3x ~65s (H200, models pre-cached) + pytest.mark.nightly, + ], + model="Qwen/Qwen3-8B", + env={}, + frontend_port=DefaultPort.FRONTEND.value, + request_payloads=[ + # 3 x up to 1000 tokens: enough decode steps for a stable tokens/verify ratio. + chat_payload_default(), + chat_payload( + "What is the capital of France? Answer in one word.", + repeat_count=1, + expected_response=["Paris"], + temperature=0.0, + max_tokens=16, + extra_body={"chat_template_args": {"enable_thinking": False}}, + ), + SGLangSpecDecodeMetricsPayload( + body={}, + repeat_count=1, + expected_log=[], + expected_response=[], + min_num_requests=4, + ), + ], + ), "disaggregated": SGLangConfig( name="disaggregated", directory=sglang_dir, diff --git a/tests/utils/payloads.py b/tests/utils/payloads.py index 4919cd5966cc..d0372100aaea 100644 --- a/tests/utils/payloads.py +++ b/tests/utils/payloads.py @@ -2149,6 +2149,47 @@ def _get_backend_specific_checks(self) -> list[MetricCheck]: return checks +@dataclass +class SGLangSpecDecodeMetricsPayload(SGLangMetricsPayload): + """Metrics validation for an SGLang worker running speculative decoding. + + Both checks use cumulative counters rather than the windowed + ``sglang:spec_accept_length`` gauge, which only refreshes on SGLang's decode + log interval and can be stale after short requests. + """ + + # Loose floor on tokens per verify step; a working EAGLE3 draft measures ~2.2. + min_tokens_per_verify: float = 1.2 + + @staticmethod + def _sum_counter(name: str, content: str) -> float: + pattern = rf"^{re.escape(name)}(?:\{{[^}}]*\}})?\s+([\d.eE+-]+)" + values = re.findall(pattern, content, re.MULTILINE) + if not values: + raise AssertionError(f"Metric '{name}' not found in metrics output") + return sum(float(v) for v in values) + + def validate(self, response: Any, content: str) -> None: + super().validate(response, content) + + verify_calls = self._sum_counter("sglang:spec_verify_calls_total", content) + if verify_calls <= 0: + raise AssertionError( + "sglang:spec_verify_calls_total is 0; speculative decoding did not run" + ) + logger.info(f"SUCCESS: sglang:spec_verify_calls_total = {verify_calls}") + + generated = self._sum_counter("sglang:generation_tokens_total", content) + tokens_per_verify = generated / verify_calls + if tokens_per_verify <= self.min_tokens_per_verify: + raise AssertionError( + f"{generated} generated tokens over {verify_calls} verify calls is " + f"{tokens_per_verify:.2f} tokens/verify, expected > " + f"{self.min_tokens_per_verify}; draft tokens are not being accepted" + ) + logger.info(f"SUCCESS: {tokens_per_verify:.2f} tokens per verify step") + + @dataclass class SGLangDisaggMetricsPayload(SGLangMetricsPayload): """Metrics validation for SGLang disaggregated workers. From 7da6dc19b684be83ea3d6387cdff06c7a347bdf9 Mon Sep 17 00:00:00 2001 From: Banruo Liu Date: Wed, 23 Sep 2026 23:43:23 -0500 Subject: [PATCH 2/6] test(sglang): add core marker and trim launcher banners Address review feedback on the SGLang speculative decoding e2e: - Add the required component marker (core) to aggregated_spec_decoding. - Drop section banners in agg_spec_decoding.sh that only restate the commands; keep the --enable-metrics note. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Banruo Liu --- examples/backends/sglang/launch/agg_spec_decoding.sh | 8 -------- tests/serve/test_sglang.py | 1 + 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/examples/backends/sglang/launch/agg_spec_decoding.sh b/examples/backends/sglang/launch/agg_spec_decoding.sh index b095d0fd66eb..2ef5901fa8d2 100755 --- a/examples/backends/sglang/launch/agg_spec_decoding.sh +++ b/examples/backends/sglang/launch/agg_spec_decoding.sh @@ -18,16 +18,8 @@ DRAFT_MODEL="Tengyunw/qwen3_8b_eagle3" HTTP_PORT="${DYN_HTTP_PORT:-8000}" print_launch_banner "Launching Speculative Decoding (1 GPU)" "$MODEL" "$HTTP_PORT" -# --------------------------- -# 1. Frontend (Ingress) -# --------------------------- python3 -m dynamo.frontend & - -# --------------------------- -# 2. Speculative Main Worker -# --------------------------- -# This runs the main model with an EAGLE3 draft model for speculative decoding. # --enable-metrics exposes sglang:spec_* metrics on the worker system port. DYN_SYSTEM_PORT=${DYN_SYSTEM_PORT:-8081} \ python3 -m dynamo.sglang \ diff --git a/tests/serve/test_sglang.py b/tests/serve/test_sglang.py index 56e701c403ca..7b7b238130cf 100644 --- a/tests/serve/test_sglang.py +++ b/tests/serve/test_sglang.py @@ -163,6 +163,7 @@ class SGLangConfig(EngineConfig): directory=sglang_dir, script_name="agg_spec_decoding.sh", marks=[ + pytest.mark.core, pytest.mark.gpu_1, # Also predownload the EAGLE3 draft: CI workers run HF_HUB_OFFLINE=True # and only the base cfg.model is auto-registered, so the draft repo From 53d02de665d4bdd0db9ec823bc496ea589f5a6a4 Mon Sep 17 00:00:00 2001 From: Banruo Liu Date: Thu, 24 Sep 2026 11:28:22 -0500 Subject: [PATCH 3/6] test(sglang): add EAGLE3 disagg KV-router e2e Add test_router_decisions_sglang_disagg_eagle: the disagg router decisions scenario with EAGLE3 speculation on both the prefill and decode workers (Qwen3-8B + Tengyunw/qwen3_8b_eagle3, the pair from agg_spec_decoding.sh). On top of the existing same-prefill-worker and true-disagg checks it asserts: - nvext.timing.kv_hit_rate > 0 on each shared-prefix request, so the prefill worker's EAGLE (bigram) KV event hashes match the router's hashes. - The decode worker ran EAGLE verify steps and accepted draft tokens after the KV and speculative state transfer (SGLangSpecDecodeMetricsPayload). Harness: SGLangProcess takes extra_args, _test_router_decisions_disagg takes require_kv_hit, and run_disagg_router_decisions_test takes a post_check hook that runs while the workers are still up. Also fix and re-enable test_router_decisions_sglang_disagg, which failed for two reasons the EAGLE variant also hit: - Disagg workers now pass --host 0.0.0.0, as launch/disagg.sh does. SGLang binds the KV bootstrap server to --host (default 127.0.0.1) while dynamo.sglang advertises the auto-detected local IP, so decode workers got connection refused and every request aborted. - Both SGLang disagg tests pass enable_bootstrap=True. SGLang always takes the bootstrap path, where decode can emit its first token before prefill completion is recorded, so kv_transfer_estimated_latency_ms can be 0. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Banruo Liu --- tests/router/common.py | 12 +++ tests/router/e2e_harness.py | 9 ++ tests/router/test_router_e2e_with_sglang.py | 112 +++++++++++++++++++- 3 files changed, 132 insertions(+), 1 deletion(-) diff --git a/tests/router/common.py b/tests/router/common.py index b0708f69e44c..ceb3d6ed9f15 100644 --- a/tests/router/common.py +++ b/tests/router/common.py @@ -2259,6 +2259,7 @@ def _test_router_decisions_disagg( request_plane: str = "nats", router_aic_config: Optional[dict[str, Any]] = None, enable_bootstrap: bool = False, + require_kv_hit: bool = False, ): """Validate KV cache prefix reuse in disaggregated prefill-decode setup via HTTP frontend. @@ -2281,6 +2282,10 @@ def _test_router_decisions_disagg( test_payload: Base test payload to send to /v1/chat/completions store_backend: Storage backend to use ("etcd" or "file"). Defaults to "etcd". router_aic_config: Optional AIC router perf-model config for frontend KV routing. + require_kv_hit: If True, also assert the router predicted a KV cache hit + (nvext.timing.kv_hit_rate > 0) for requests 2-4, which share a prefix + with the previous request. This catches worker/router block-hash + mismatches that routing to the same worker alone would not. Raises: AssertionError: If prefill_worker_ids differ across requests (prefix reuse failure) @@ -2392,6 +2397,13 @@ async def send_progressive_requests(): timing_info is not None ), f"Request {i + 1}: Expected timing info in final chunk, got None" verify_response_timing(timing_info, disagg=not enable_bootstrap) + if require_kv_hit and i > 0: + kv_hit_rate = timing_info.get("kv_hit_rate") + assert kv_hit_rate is not None and kv_hit_rate > 0, ( + f"Request {i + 1}: expected kv_hit_rate > 0 on a shared " + f"prefix, got {kv_hit_rate}; worker KV event hashes " + f"may not match the router's" + ) # Small delay between requests await asyncio.sleep(1) diff --git a/tests/router/e2e_harness.py b/tests/router/e2e_harness.py index a1ba23b1dc04..e9fd68bc8fee 100644 --- a/tests/router/e2e_harness.py +++ b/tests/router/e2e_harness.py @@ -423,7 +423,14 @@ def run_disagg_router_decisions_test( | None = None, test_payload: dict[str, Any] | None = None, test_kwargs: dict[str, Any] | None = None, + post_check: Callable[[Any, Any], None] | None = None, ): + """Run the disagg router decisions test against prefill and decode workers. + + ``post_check(prefill_workers, decode_workers)``, if set, runs after the + routing assertions while the workers are still up (e.g. to scrape their + /metrics). + """ shared_namespace = f"test-namespace-{generate_random_suffix()}" frontend_port = allocate_frontend_ports(request, 1)[0] @@ -447,6 +454,8 @@ def run_test(prefill_workers, decode_workers): request_plane=request_plane, **(test_kwargs or {}), ) + if post_check is not None: + post_check(prefill_workers, decode_workers) if worker_context_factory is not None: with worker_context_factory(shared_namespace) as workers: diff --git a/tests/router/test_router_e2e_with_sglang.py b/tests/router/test_router_e2e_with_sglang.py index 5903fb3d426e..9f552d9b72fd 100644 --- a/tests/router/test_router_e2e_with_sglang.py +++ b/tests/router/test_router_e2e_with_sglang.py @@ -10,9 +10,11 @@ from typing import Any, Dict, Optional import pytest +import requests from tests.router.e2e_harness import ( ManagedEngineProcessMixin, + build_test_payload, run_basic_router_test, run_disagg_router_decisions_test, run_indexers_sync_test, @@ -22,6 +24,7 @@ from tests.utils.constants import DynamoPortRange from tests.utils.gpu_args import build_gpu_mem_args from tests.utils.managed_process import ManagedProcess +from tests.utils.payloads import SGLangSpecDecodeMetricsPayload from tests.utils.port_utils import ( allocate_contiguous_ports, allocate_port, @@ -94,6 +97,7 @@ def __init__( (see tests/README.md "SGLang KV tokens") - context_length: Maximum sequence length (optional) - disable_cuda_graph: Disable CUDA graphs (default: False) + - extra_args: Additional dynamo.sglang CLI flags (optional) num_workers: Number of SGLang worker processes single_gpu: If True, all workers share GPU 0 data_parallel_size: If set, enables this many data-parallel ranks per worker process. @@ -139,6 +143,7 @@ def __init__( max_total_tokens = sglang_args.get("max_total_tokens") context_length = sglang_args.get("context_length") disable_cuda_graph = sglang_args.get("disable_cuda_graph", False) + extra_args = sglang_args.get("extra_args", []) # Resolved memory budget, for startup logs (mirrors the command flags). mem_budget = ( f"max_total_tokens={max_total_tokens}, mem_frac=0.9" @@ -207,6 +212,10 @@ def __init__( if disaggregation_mode is not None: command.extend(["--disaggregation-mode", disaggregation_mode]) command.extend(["--disaggregation-transfer-backend", "nixl"]) + # The KV bootstrap server binds to --host (default 127.0.0.1), + # but dynamo.sglang advertises the auto-detected local IP to + # decode workers, so bind all interfaces like launch/disagg.sh. + command.extend(["--host", "0.0.0.0"]) if data_parallel_size is not None: # Add DP configuration @@ -226,6 +235,7 @@ def __init__( kv_events_port = self._kv_event_ports[worker_idx * kv_event_rank_span] kv_events_config = f'{{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:{kv_events_port}"}}' command.extend(["--kv-events-config", kv_events_config]) + command.extend(extra_args) # Each SGLang worker needs a unique DYN_SYSTEM_PORT to avoid conflicts. # Ports are dynamically allocated for xdist-safe parallel execution. @@ -372,7 +382,6 @@ def test_router_decisions_sglang_dp( ) -@pytest.mark.skip(reason="Nightly CI failure: https://linear.app/nvidia/issue/DYN-2603") @pytest.mark.e2e @pytest.mark.model(MODEL_NAME) @pytest.mark.gpu_2 @@ -406,6 +415,107 @@ def test_router_decisions_sglang_disagg( "gpu_start_index": 1, "disaggregation_mode": "decode", }, + # See test_router_decisions_sglang_disagg_eagle. + test_kwargs={"enable_bootstrap": True}, + ) + + +# EAGLE3 speculative decoding: same pair as launch/agg_spec_decoding.sh, both +# ungated. An 8B base is needed because the test requires a real EAGLE3 draft +# that accepts tokens, and there is no smaller ungated base+draft pair. Nightly +# only, on H100-class GPUs: the two prefill workers share GPU0 (peak ~36 GiB). +EAGLE_MODEL_NAME = "Qwen/Qwen3-8B" +EAGLE_DRAFT_MODEL_NAME = "Tengyunw/qwen3_8b_eagle3" +SGLANG_EAGLE_ARGS: Dict[str, Any] = { + **SGLANG_ARGS, + "model": EAGLE_MODEL_NAME, + "extra_args": [ + "--enable-metrics", + "--speculative-algorithm", + "EAGLE3", + "--speculative-draft-model-path", + EAGLE_DRAFT_MODEL_NAME, + "--speculative-num-steps", + "3", + "--speculative-eagle-topk", + "1", + "--speculative-num-draft-tokens", + "4", + ], +} + + +def _assert_decode_workers_speculated(prefill_workers, decode_workers) -> None: + """Assert every decode worker ran EAGLE verify steps and accepted draft tokens.""" + for port in decode_workers._system_ports: + response = requests.get(f"http://localhost:{port}/metrics", timeout=10) + response.raise_for_status() + content = response.text + SGLangSpecDecodeMetricsPayload( + body={}, + repeat_count=1, + expected_log=[], + expected_response=[], + min_num_requests=4, + ).validate(None, content) + + +@pytest.mark.e2e +@pytest.mark.model(EAGLE_MODEL_NAME) +@pytest.mark.model(EAGLE_DRAFT_MODEL_NAME) +@pytest.mark.h100 +@pytest.mark.gpu_2 +@pytest.mark.nightly +@pytest.mark.requested_sglang_kv_tokens(2048) +@pytest.mark.parametrize("request_plane", ["nats"], indirect=True) +# ~3x ~96s (2x H200, models pre-cached); peak ~36 GiB on the shared prefill GPU. +@pytest.mark.timeout(300) +def test_router_decisions_sglang_disagg_eagle( + request, + runtime_services_dynamic_ports, + predownload_models, + set_ucx_tls_no_mm, + request_plane, +): + """EAGLE3 on both prefill and decode workers, behind the KV router. + + On top of the disagg routing checks, this asserts that: + * the router predicts a KV hit on each shared-prefix request, i.e. the + prefill worker's EAGLE (bigram) KV event hashes match the router's + * the decode worker keeps speculating after the KV and EAGLE state + transfer, and accepts draft tokens + """ + run_disagg_router_decisions_test( + engine_process_cls=SGLangProcess, + engine_args_name="sglang_args", + engine_args=SGLANG_EAGLE_ARGS, + request=request, + request_plane=request_plane, + model_name=EAGLE_MODEL_NAME, + block_size=PAGE_SIZE, + num_prefill_workers=2, + num_decode_workers=1, + prefill_process_kwargs={ + "single_gpu": True, + "gpu_start_index": 0, + "disaggregation_mode": "prefill", + }, + decode_process_kwargs={ + "single_gpu": True, + "gpu_start_index": 1, + "disaggregation_mode": "decode", + }, + # Enough decode steps for a stable tokens-per-verify ratio. + test_payload={ + **build_test_payload(EAGLE_MODEL_NAME), + "max_tokens": 128, + "temperature": 0.0, + }, + # SGLang always takes the bootstrap path: decode can emit its first + # token before prefill completion is recorded, so the estimated KV + # transfer latency may be 0. + test_kwargs={"enable_bootstrap": True, "require_kv_hit": True}, + post_check=_assert_decode_workers_speculated, ) From 4b68050fa370b496dc58ea1e9f86f6f426b1d0cc Mon Sep 17 00:00:00 2001 From: Banruo Liu Date: Thu, 24 Sep 2026 14:01:06 -0500 Subject: [PATCH 4/6] test(sglang): tighten EAGLE3 disagg router test per self-review - Make SGLANG_EAGLE_ARGS["extra_args"] an immutable tuple (it lives in a module-level dict) and default SGLangProcess extra_args to (). - Type-annotate the decode-worker metrics post_check. - Document which nightly lane runs the test: it is marked h100 (peak ~36 GiB on the shared prefill GPU) and, like the other gpu_2 tests in the file, has no profiled_vram_gib, so it runs in sglang-h100-test's sequential stage. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Banruo Liu --- tests/router/test_router_e2e_with_sglang.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/tests/router/test_router_e2e_with_sglang.py b/tests/router/test_router_e2e_with_sglang.py index 9f552d9b72fd..0398a8e3c5da 100644 --- a/tests/router/test_router_e2e_with_sglang.py +++ b/tests/router/test_router_e2e_with_sglang.py @@ -143,7 +143,7 @@ def __init__( max_total_tokens = sglang_args.get("max_total_tokens") context_length = sglang_args.get("context_length") disable_cuda_graph = sglang_args.get("disable_cuda_graph", False) - extra_args = sglang_args.get("extra_args", []) + extra_args = sglang_args.get("extra_args", ()) # Resolved memory budget, for startup logs (mirrors the command flags). mem_budget = ( f"max_total_tokens={max_total_tokens}, mem_frac=0.9" @@ -422,14 +422,16 @@ def test_router_decisions_sglang_disagg( # EAGLE3 speculative decoding: same pair as launch/agg_spec_decoding.sh, both # ungated. An 8B base is needed because the test requires a real EAGLE3 draft -# that accepts tokens, and there is no smaller ungated base+draft pair. Nightly -# only, on H100-class GPUs: the two prefill workers share GPU0 (peak ~36 GiB). +# that accepts tokens, and there is no smaller ungated base+draft pair. The two +# prefill workers share GPU0 (peak ~36 GiB), so the test is marked h100 and runs +# in the nightly sglang-h100-test lane. Like the other gpu_2 tests here it has no +# profiled_vram_gib, so it runs in that lane's sequential stage. EAGLE_MODEL_NAME = "Qwen/Qwen3-8B" EAGLE_DRAFT_MODEL_NAME = "Tengyunw/qwen3_8b_eagle3" SGLANG_EAGLE_ARGS: Dict[str, Any] = { **SGLANG_ARGS, "model": EAGLE_MODEL_NAME, - "extra_args": [ + "extra_args": ( "--enable-metrics", "--speculative-algorithm", "EAGLE3", @@ -441,11 +443,13 @@ def test_router_decisions_sglang_disagg( "1", "--speculative-num-draft-tokens", "4", - ], + ), } -def _assert_decode_workers_speculated(prefill_workers, decode_workers) -> None: +def _assert_decode_workers_speculated( + prefill_workers: SGLangProcess, decode_workers: SGLangProcess +) -> None: """Assert every decode worker ran EAGLE verify steps and accepted draft tokens.""" for port in decode_workers._system_ports: response = requests.get(f"http://localhost:{port}/metrics", timeout=10) From 123c9b05d47680326de6bc7dd71b424a35ed20c5 Mon Sep 17 00:00:00 2001 From: Banruo Liu Date: Thu, 24 Sep 2026 16:17:12 -0500 Subject: [PATCH 5/6] test(sglang): explain why SGLang disagg tests skip the KV transfer latency check Spell out the SGLang bootstrap flow step by step and why the decode first token can race ahead of the recorded prefill completion, which makes kv_transfer_estimated_latency_ms exactly 0 and the > 0 check flaky. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Banruo Liu --- tests/router/test_router_e2e_with_sglang.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/tests/router/test_router_e2e_with_sglang.py b/tests/router/test_router_e2e_with_sglang.py index f5ba76a5b0a5..b5525a013909 100644 --- a/tests/router/test_router_e2e_with_sglang.py +++ b/tests/router/test_router_e2e_with_sglang.py @@ -560,9 +560,21 @@ def test_router_decisions_sglang_disagg_eagle( "max_tokens": 128, "temperature": 0.0, }, - # SGLang always takes the bootstrap path: decode can emit its first - # token before prefill completion is recorded, so the estimated KV - # transfer latency may be 0. + # enable_bootstrap=True skips the kv_transfer_estimated_latency_ms > 0 + # check, which does not hold for SGLang. SGLang disagg always uses the + # bootstrap flow: + # 1. The router sends the request to a prefill worker. + # 2. The prefill worker replies at once with its bootstrap address, + # before prefill has finished. + # 3. The router dispatches the request to a decode worker, so + # prefill and decode now run in parallel. + # 4. The decode worker pulls the KV cache from the prefill worker and + # emits its first token. + # 5. The prefill worker's response stream ends. Only now does the + # router record prefill completion. + # The latency is (decode first token time - prefill completion time), + # clamped at 0. Steps 4 and 5 race, and when the first token arrives + # first the latency is exactly 0, so the > 0 check fails intermittently. test_kwargs={"enable_bootstrap": True, "require_kv_hit": True}, post_check=_assert_decode_workers_speculated, ) From 656aafa0d910e53608ba23669df5e0e5d7eca315 Mon Sep 17 00:00:00 2001 From: Banruo Liu Date: Thu, 24 Sep 2026 16:55:25 -0500 Subject: [PATCH 6/6] test(sglang): drop cross-reference comment in SGLang disagg test Co-Authored-By: Claude Opus 5.5 Signed-off-by: Banruo Liu --- tests/router/test_router_e2e_with_sglang.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/router/test_router_e2e_with_sglang.py b/tests/router/test_router_e2e_with_sglang.py index b5525a013909..f8dc29a08061 100644 --- a/tests/router/test_router_e2e_with_sglang.py +++ b/tests/router/test_router_e2e_with_sglang.py @@ -460,7 +460,6 @@ def test_router_decisions_sglang_disagg( "gpu_start_index": 1, "disaggregation_mode": "decode", }, - # See test_router_decisions_sglang_disagg_eagle. test_kwargs={"enable_bootstrap": True}, )