Skip to content
40 changes: 40 additions & 0 deletions examples/backends/sglang/launch/agg_spec_decoding.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
#!/bin/bash
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
set -e
trap 'echo Cleaning up...; kill 0' EXIT

SCRIPT_DIR="$(dirname "$(readlink -f "$0")")"
source "$SCRIPT_DIR/../../../common/launch_utils.sh"
source "$SCRIPT_DIR/../../../common/gpu_utils.sh" # build_sglang_gpu_mem_args

# When the serve test sets requested_sglang_kv_tokens, this emits a
# --max-total-tokens cap so VRAM is bounded (GPU-size-independent). Empty for
# manual runs.
GPU_MEM_ARGS=$(build_sglang_gpu_mem_args)

MODEL="Qwen/Qwen3-8B"
DRAFT_MODEL="Tengyunw/qwen3_8b_eagle3"
HTTP_PORT="${DYN_HTTP_PORT:-8000}"
print_launch_banner "Launching Speculative Decoding (1 GPU)" "$MODEL" "$HTTP_PORT"

python3 -m dynamo.frontend &

# --enable-metrics exposes sglang:spec_* metrics on the worker system port.
DYN_SYSTEM_PORT=${DYN_SYSTEM_PORT:-8081} \
python3 -m dynamo.sglang \
--model-path "$MODEL" \
--served-model-name "$MODEL" \
--page-size 16 \
--tp 1 \
--trust-remote-code \
--enable-metrics \
--speculative-algorithm EAGLE3 \
--speculative-draft-model-path "$DRAFT_MODEL" \
--speculative-num-steps 3 \
--speculative-eagle-topk 1 \
--speculative-num-draft-tokens 4 \
$GPU_MEM_ARGS "$@" &

# Exit on first worker failure; kill 0 in the EXIT trap tears down the rest
Comment thread
livingshade marked this conversation as resolved.
Resolved
wait_any_exit
12 changes: 12 additions & 0 deletions tests/router/common.py
Original file line number Diff line number Diff line change
Expand Up @@ -2259,6 +2259,7 @@ def _test_router_decisions_disagg(
request_plane: str = "nats",
router_aic_config: Optional[dict[str, Any]] = None,
enable_bootstrap: bool = False,
require_kv_hit: bool = False,
):
"""Validate KV cache prefix reuse in disaggregated prefill-decode setup via HTTP frontend.

Expand All @@ -2281,6 +2282,10 @@ def _test_router_decisions_disagg(
test_payload: Base test payload to send to /v1/chat/completions
store_backend: Storage backend to use ("etcd" or "file"). Defaults to "etcd".
router_aic_config: Optional AIC router perf-model config for frontend KV routing.
require_kv_hit: If True, also assert the router predicted a KV cache hit
(nvext.timing.kv_hit_rate > 0) for requests 2-4, which share a prefix
with the previous request. This catches worker/router block-hash
mismatches that routing to the same worker alone would not.

Raises:
AssertionError: If prefill_worker_ids differ across requests (prefix reuse failure)
Expand Down Expand Up @@ -2392,6 +2397,13 @@ async def send_progressive_requests():
timing_info is not None
), f"Request {i + 1}: Expected timing info in final chunk, got None"
verify_response_timing(timing_info, disagg=not enable_bootstrap)
if require_kv_hit and i > 0:
kv_hit_rate = timing_info.get("kv_hit_rate")
assert kv_hit_rate is not None and kv_hit_rate > 0, (
f"Request {i + 1}: expected kv_hit_rate > 0 on a shared "
f"prefix, got {kv_hit_rate}; worker KV event hashes "
f"may not match the router's"
)

# Small delay between requests
await asyncio.sleep(1)
Expand Down
9 changes: 9 additions & 0 deletions tests/router/e2e_harness.py
Original file line number Diff line number Diff line change
Expand Up @@ -423,7 +423,14 @@ def run_disagg_router_decisions_test(
| None = None,
test_payload: dict[str, Any] | None = None,
test_kwargs: dict[str, Any] | None = None,
post_check: Callable[[Any, Any], None] | None = None,
):
"""Run the disagg router decisions test against prefill and decode workers.

``post_check(prefill_workers, decode_workers)``, if set, runs after the
routing assertions while the workers are still up (e.g. to scrape their
/metrics).
"""
shared_namespace = f"test-namespace-{generate_random_suffix()}"
frontend_port = allocate_frontend_ports(request, 1)[0]

Expand All @@ -447,6 +454,8 @@ def run_test(prefill_workers, decode_workers):
request_plane=request_plane,
**(test_kwargs or {}),
)
if post_check is not None:
post_check(prefill_workers, decode_workers)

if worker_context_factory is not None:
with worker_context_factory(shared_namespace) as workers:
Expand Down
128 changes: 127 additions & 1 deletion tests/router/test_router_e2e_with_sglang.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,11 @@
from typing import Any, Dict, Optional

import pytest
import requests

from tests.router.e2e_harness import (
ManagedEngineProcessMixin,
build_test_payload,
run_basic_router_test,
run_cache_salt_isolation_test,
run_disagg_router_decisions_test,
Expand All @@ -23,6 +25,7 @@
from tests.utils.constants import DynamoPortRange
from tests.utils.gpu_args import build_gpu_mem_args
from tests.utils.managed_process import ManagedProcess, check_health_ready
from tests.utils.payloads import SGLangSpecDecodeMetricsPayload
from tests.utils.port_utils import (
allocate_contiguous_ports,
allocate_port,
Expand Down Expand Up @@ -95,6 +98,7 @@ def __init__(
(see tests/README.md "SGLang KV tokens")
- context_length: Maximum sequence length (optional)
- disable_cuda_graph: Disable CUDA graphs (default: False)
- extra_args: Additional dynamo.sglang CLI flags (optional)
num_workers: Number of SGLang worker processes
single_gpu: If True, all workers share GPU 0
data_parallel_size: If set, enables this many data-parallel ranks per worker process.
Expand Down Expand Up @@ -148,6 +152,7 @@ def __init__(
max_total_tokens = sglang_args.get("max_total_tokens")
context_length = sglang_args.get("context_length")
disable_cuda_graph = sglang_args.get("disable_cuda_graph", False)
extra_args = sglang_args.get("extra_args", ())
# Resolved memory budget, for startup logs (mirrors the command flags).
mem_budget = (
f"max_total_tokens={max_total_tokens}, mem_frac=0.9"
Expand Down Expand Up @@ -216,6 +221,10 @@ def __init__(
if disaggregation_mode is not None:
command.extend(["--disaggregation-mode", disaggregation_mode])
command.extend(["--disaggregation-transfer-backend", "nixl"])
# The KV bootstrap server binds to --host (default 127.0.0.1),
# but dynamo.sglang advertises the auto-detected local IP to
# decode workers, so bind all interfaces like launch/disagg.sh.
command.extend(["--host", "0.0.0.0"])

if data_parallel_size is not None:
# Add DP configuration
Expand All @@ -235,6 +244,7 @@ def __init__(
kv_events_port = self._kv_event_ports[worker_idx * kv_event_rank_span]
kv_events_config = f'{{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:{kv_events_port}"}}'
command.extend(["--kv-events-config", kv_events_config])
command.extend(extra_args)

nccl_port = self._nccl_ports[worker_idx]
command.extend(["--nccl-port", str(nccl_port)])
Expand Down Expand Up @@ -417,7 +427,6 @@ def test_router_decisions_sglang_dp(
)


@pytest.mark.skip(reason="Nightly CI failure: https://linear.app/nvidia/issue/DYN-2603")
@pytest.mark.e2e
@pytest.mark.model(MODEL_NAME)
@pytest.mark.gpu_2
Expand Down Expand Up @@ -451,6 +460,123 @@ def test_router_decisions_sglang_disagg(
"gpu_start_index": 1,
"disaggregation_mode": "decode",
},
# See test_router_decisions_sglang_disagg_eagle.
Comment thread
livingshade marked this conversation as resolved.
Resolved
test_kwargs={"enable_bootstrap": True},
)


# EAGLE3 speculative decoding: same pair as launch/agg_spec_decoding.sh, both
# ungated. An 8B base is needed because the test requires a real EAGLE3 draft
# that accepts tokens, and there is no smaller ungated base+draft pair. The two
# prefill workers share GPU0 (peak ~36 GiB), so the test is marked h100 and runs
# in the nightly sglang-h100-test lane. Like the other gpu_2 tests here it has no
# profiled_vram_gib, so it runs in that lane's sequential stage.
EAGLE_MODEL_NAME = "Qwen/Qwen3-8B"
EAGLE_DRAFT_MODEL_NAME = "Tengyunw/qwen3_8b_eagle3"
SGLANG_EAGLE_ARGS: Dict[str, Any] = {
**SGLANG_ARGS,
"model": EAGLE_MODEL_NAME,
"extra_args": (
"--enable-metrics",
"--speculative-algorithm",
"EAGLE3",
"--speculative-draft-model-path",
EAGLE_DRAFT_MODEL_NAME,
"--speculative-num-steps",
"3",
"--speculative-eagle-topk",
"1",
"--speculative-num-draft-tokens",
"4",
),
}


def _assert_decode_workers_speculated(
prefill_workers: SGLangProcess, decode_workers: SGLangProcess
) -> None:
"""Assert every decode worker ran EAGLE verify steps and accepted draft tokens."""
for port in decode_workers._system_ports:
response = requests.get(f"http://localhost:{port}/metrics", timeout=10)
response.raise_for_status()
content = response.text
SGLangSpecDecodeMetricsPayload(
Comment thread
livingshade marked this conversation as resolved.
Resolved
body={},
repeat_count=1,
expected_log=[],
expected_response=[],
min_num_requests=4,
).validate(None, content)


@pytest.mark.e2e
@pytest.mark.model(EAGLE_MODEL_NAME)
@pytest.mark.model(EAGLE_DRAFT_MODEL_NAME)
@pytest.mark.h100
@pytest.mark.gpu_2
@pytest.mark.nightly
@pytest.mark.requested_sglang_kv_tokens(2048)
@pytest.mark.parametrize("request_plane", ["nats"], indirect=True)
# ~3x ~96s (2x H200, models pre-cached); peak ~36 GiB on the shared prefill GPU.
@pytest.mark.timeout(300)
def test_router_decisions_sglang_disagg_eagle(
request,
runtime_services_dynamic_ports,
predownload_models,
set_ucx_tls_no_mm,
request_plane,
):
"""EAGLE3 on both prefill and decode workers, behind the KV router.

On top of the disagg routing checks, this asserts that:
* the router predicts a KV hit on each shared-prefix request, i.e. the
prefill worker's EAGLE (bigram) KV event hashes match the router's
* the decode worker keeps speculating after the KV and EAGLE state
transfer, and accepts draft tokens
"""
run_disagg_router_decisions_test(
engine_process_cls=SGLangProcess,
engine_args_name="sglang_args",
engine_args=SGLANG_EAGLE_ARGS,
request=request,
request_plane=request_plane,
model_name=EAGLE_MODEL_NAME,
block_size=PAGE_SIZE,
num_prefill_workers=2,
num_decode_workers=1,
prefill_process_kwargs={
"single_gpu": True,
"gpu_start_index": 0,
"disaggregation_mode": "prefill",
},
decode_process_kwargs={
"single_gpu": True,
"gpu_start_index": 1,
"disaggregation_mode": "decode",
},
# Enough decode steps for a stable tokens-per-verify ratio.
test_payload={
**build_test_payload(EAGLE_MODEL_NAME),
"max_tokens": 128,
"temperature": 0.0,
},
# enable_bootstrap=True skips the kv_transfer_estimated_latency_ms > 0
# check, which does not hold for SGLang. SGLang disagg always uses the
# bootstrap flow:
# 1. The router sends the request to a prefill worker.
# 2. The prefill worker replies at once with its bootstrap address,
# before prefill has finished.
# 3. The router dispatches the request to a decode worker, so
# prefill and decode now run in parallel.
# 4. The decode worker pulls the KV cache from the prefill worker and
# emits its first token.
# 5. The prefill worker's response stream ends. Only now does the
# router record prefill completion.
# The latency is (decode first token time - prefill completion time),
# clamped at 0. Steps 4 and 5 race, and when the first token arrives
# first the latency is exactly 0, so the > 0 check fails intermittently.
test_kwargs={"enable_bootstrap": True, "require_kv_hit": True},
post_check=_assert_decode_workers_speculated,
)


Expand Down
45 changes: 45 additions & 0 deletions tests/serve/test_sglang.py
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@
ResponsesPayload,
ResponsesStreamPayload,
SGLangDisaggRouterMetricsPayload,
SGLangSpecDecodeMetricsPayload,
VideoGenerationPayload,
)
from tests.utils.port_utils import allocate_contiguous_ports, deallocate_ports
Expand Down Expand Up @@ -154,6 +155,50 @@ class SGLangConfig(EngineConfig):
metric_payload_default(min_num_requests=6, backend="sglang"),
],
),
# Speculative decoding: Qwen3-8B main model with an EAGLE3 draft model
# (see launch/agg_spec_decoding.sh). Both repos are ungated.
# Nightly-only: the 8B base plus EAGLE3 draft model is intentionally outside pre-merge CI.
"aggregated_spec_decoding": SGLangConfig(
name="aggregated_spec_decoding",
directory=sglang_dir,
script_name="agg_spec_decoding.sh",
marks=[
pytest.mark.core,
pytest.mark.gpu_1,
# Also predownload the EAGLE3 draft: CI workers run HF_HUB_OFFLINE=True
# and only the base cfg.model is auto-registered, so the draft repo
# can't be resolved offline without this.
pytest.mark.model("Tengyunw/qwen3_8b_eagle3"),
# Measured peak ~20.6 GiB on H200 (8B weights + EAGLE3 draft +
# capped KV + CUDA graphs).
pytest.mark.profiled_vram_gib(21.0),
pytest.mark.requested_sglang_kv_tokens(4096),
pytest.mark.timeout(300), # ~3x ~65s (H200, models pre-cached)
pytest.mark.nightly,
],
model="Qwen/Qwen3-8B",
env={},
frontend_port=DefaultPort.FRONTEND.value,
request_payloads=[
# 3 x up to 1000 tokens: enough decode steps for a stable tokens/verify ratio.
chat_payload_default(),
chat_payload(
Comment thread
livingshade marked this conversation as resolved.
Resolved
"What is the capital of France? Answer in one word.",
repeat_count=1,
expected_response=["Paris"],
temperature=0.0,
max_tokens=16,
extra_body={"chat_template_args": {"enable_thinking": False}},
),
SGLangSpecDecodeMetricsPayload(
body={},
repeat_count=1,
expected_log=[],
expected_response=[],
min_num_requests=4,
),
],
),
"disaggregated": SGLangConfig(
name="disaggregated",
directory=sglang_dir,
Expand Down
41 changes: 41 additions & 0 deletions tests/utils/payloads.py
Original file line number Diff line number Diff line change
Expand Up @@ -2149,6 +2149,47 @@ def _get_backend_specific_checks(self) -> list[MetricCheck]:
return checks


@dataclass
class SGLangSpecDecodeMetricsPayload(SGLangMetricsPayload):
"""Metrics validation for an SGLang worker running speculative decoding.

Both checks use cumulative counters rather than the windowed
``sglang:spec_accept_length`` gauge, which only refreshes on SGLang's decode
log interval and can be stale after short requests.
"""

# Loose floor on tokens per verify step; a working EAGLE3 draft measures ~2.2.
min_tokens_per_verify: float = 1.2

@staticmethod
def _sum_counter(name: str, content: str) -> float:
pattern = rf"^{re.escape(name)}(?:\{{[^}}]*\}})?\s+([\d.eE+-]+)"
values = re.findall(pattern, content, re.MULTILINE)
if not values:
raise AssertionError(f"Metric '{name}' not found in metrics output")
return sum(float(v) for v in values)

def validate(self, response: Any, content: str) -> None:
super().validate(response, content)

verify_calls = self._sum_counter("sglang:spec_verify_calls_total", content)
if verify_calls <= 0:
raise AssertionError(
"sglang:spec_verify_calls_total is 0; speculative decoding did not run"
)
logger.info(f"SUCCESS: sglang:spec_verify_calls_total = {verify_calls}")

generated = self._sum_counter("sglang:generation_tokens_total", content)
tokens_per_verify = generated / verify_calls
if tokens_per_verify <= self.min_tokens_per_verify:
raise AssertionError(
f"{generated} generated tokens over {verify_calls} verify calls is "
f"{tokens_per_verify:.2f} tokens/verify, expected > "
f"{self.min_tokens_per_verify}; draft tokens are not being accepted"
)
logger.info(f"SUCCESS: {tokens_per_verify:.2f} tokens per verify step")
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved


@dataclass
class SGLangDisaggMetricsPayload(SGLangMetricsPayload):
"""Metrics validation for SGLang disaggregated workers.
Expand Down
Loading