From aec71b449891e6cf30fc8a27ffa18c7ad8a1cfb8 Mon Sep 17 00:00:00 2001 From: Lin Hu Date: Thu, 20 Aug 2026 14:14:34 +0800 Subject: [PATCH 1/3] feat(sglang): carry router hints to SGLang workers Ports the router-hint contract to the SGLang backend so a worker running the KVCR HiCache storage backend can act as both a hint target and a hint source, matching what the vLLM backend already does. Registration advertises the three runtime keys the router requires: the router_hint capability, the worker_type role, and a per-global-DP-rank map of KVCR peer control endpoints. All three go out together -- router_hint_metadata_ for_dp_rank returns None without worker_type, and router_hint_for_selection matches a hint source to its target on that role, so publishing a subset registers a worker the router silently never hints to. Endpoints come from the KVCR backend's --hicache-storage-backend-extra-config rather than vLLM's secondary_tiers. SGLang builds a storage backend in every attention rank's scheduler process, so one DP rank owns a block of control ports and the map strides by tp_size // dp_size; this must stay in step with _rank_port_offset on the SGLang side. Configured-for-hints but unadvertisable raises at registration instead of leaving a worker whose every peer fetch fails. On the request path, a hint arriving under extra_args.kv_transfer_params is forwarded to async_generate as kv_router_hint. An SGLang install without that kwarg warns once and recomputes the prefix, so a missing hint degrades rather than failing the request. Validation: unit tests cover the three published keys, the disaggregation-mode to worker_type mapping, DP-rank port striding (single-node and multinode), advertise-host handling, the no-op and raising paths, and hint extraction on both a current and a legacy engine. Signed-off-by: Lin Hu --- components/src/dynamo/sglang/_compat.py | 44 +++ components/src/dynamo/sglang/register.py | 14 + .../request_handlers/llm/decode_handler.py | 3 + .../request_handlers/llm/prefill_handler.py | 3 +- components/src/dynamo/sglang/router_hints.py | 190 ++++++++++ .../sglang/tests/test_sglang_router_hints.py | 330 ++++++++++++++++++ 6 files changed, 583 insertions(+), 1 deletion(-) create mode 100644 components/src/dynamo/sglang/router_hints.py create mode 100644 components/src/dynamo/sglang/tests/test_sglang_router_hints.py diff --git a/components/src/dynamo/sglang/_compat.py b/components/src/dynamo/sglang/_compat.py index d644143adf57..3da68933af27 100644 --- a/components/src/dynamo/sglang/_compat.py +++ b/components/src/dynamo/sglang/_compat.py @@ -28,6 +28,21 @@ logger = logging.getLogger(__name__) +# Where the KV router stashes a request's hints. Mirrors the vLLM handler's +# private copies of the same wire keys (dynamo/vllm/handlers.py); the Rust side +# owns the canonical definitions in lib/kv-router/src/router_hint.rs. +_KV_TRANSFER_PARAMS_EXTRA_ARGS_KEY = "kv_transfer_params" +_ROUTER_HINT_EXTRA_ARGS_KEY = "router_hint" + + +@lru_cache(maxsize=1) +def _warn_router_hint_unsupported() -> None: + logger.warning( + "Dropping the router KV-reuse hint because SGLang Engine.async_generate " + "does not accept kv_router_hint; requests will recompute prefixes a peer " + "already holds. Upgrade SGLang to enable hint-driven KV reuse." + ) + @lru_cache(maxsize=1) def _warn_require_reasoning_unsupported() -> None: @@ -173,9 +188,38 @@ def require_reasoning_kwargs(engine: Any, request: Mapping[str, Any]) -> dict[st return kwargs +def router_hint_kwargs(engine: Any, request: Mapping[str, Any]) -> dict[str, Any]: + """Build the optional SGLang per-request router KV-reuse hint argument. + + The KV router attaches its hint under ``extra_args.kv_transfer_params``; + SGLang consumes it as the ``kv_router_hint`` async_generate kwarg, which is + threaded down to the KVCR HiCache storage backend so it can pull the prefix + from the named peer. An engine without that kwarg just recomputes the + prefix, so a missing hint degrades rather than failing the request. + """ + extra_args = request.get("extra_args") + if not isinstance(extra_args, Mapping): + return {} + kv_transfer_params = extra_args.get(_KV_TRANSFER_PARAMS_EXTRA_ARGS_KEY) + if not isinstance(kv_transfer_params, Mapping): + return {} + router_hint = kv_transfer_params.get(_ROUTER_HINT_EXTRA_ARGS_KEY) + if not isinstance(router_hint, Mapping): + return {} + + kwargs = filter_supported_async_generate_kwargs( + engine, + {"kv_router_hint": dict(router_hint)}, + ) + if "kv_router_hint" not in kwargs: + _warn_router_hint_unsupported() + return kwargs + + __all__ = [ "ensure_sglang_tensor_image_size", "ensure_sglang_top_level_exports", "filter_supported_async_generate_kwargs", "require_reasoning_kwargs", + "router_hint_kwargs", ] diff --git a/components/src/dynamo/sglang/register.py b/components/src/dynamo/sglang/register.py index c681b72447bb..daaea4b95038 100644 --- a/components/src/dynamo/sglang/register.py +++ b/components/src/dynamo/sglang/register.py @@ -32,10 +32,12 @@ get_hicache_native_offloading_capacity, get_spec_decode_runtime_data, kv_event_block_size, + local_dp_rank_bounds, model_card_dp_rank_bounds, runtime_capacity, ) from dynamo.sglang.engine_generate import SGLANG_GENERATE_CAPABILITY +from dynamo.sglang.router_hints import enable_router_hint_support SGLANG_HICACHE_MOONCAKE_RUNTIME_KEY = "sglang_hicache_mooncake" SPEC_DECODE_RUNTIME_KEY = "spec_decode" @@ -491,6 +493,18 @@ async def get_runtime_config( f"Failed to attach SGLang spec decode runtime metadata: {e}" ) + # KVCR HiCache backend: advertise this worker as a router-hint KV source. + # Local bounds, not model-card bounds: the KVCR control channel is per + # scheduler process, so only the ranks this node actually runs are dialable. + enable_router_hint_support( + runtime_config=runtime_config, + server_args=server_args, + extra_config=_parse_hicache_storage_extra_config( + server_args.hicache_storage_backend_extra_config + ), + dp_bounds=local_dp_rank_bounds(server_args), + ) + mooncake_runtime_data = _get_mooncake_runtime_data(server_args) if mooncake_runtime_data is not None: try: diff --git a/components/src/dynamo/sglang/request_handlers/llm/decode_handler.py b/components/src/dynamo/sglang/request_handlers/llm/decode_handler.py index 1697c0ecd60f..1dbc5490639e 100644 --- a/components/src/dynamo/sglang/request_handlers/llm/decode_handler.py +++ b/components/src/dynamo/sglang/request_handlers/llm/decode_handler.py @@ -19,6 +19,7 @@ from dynamo.sglang._compat import ( filter_supported_async_generate_kwargs, require_reasoning_kwargs, + router_hint_kwargs, ) from dynamo.sglang.args import Config from dynamo.sglang.engine_generate import ( @@ -465,6 +466,7 @@ async def generate( sampling_params=sampling_params, stream=True, **require_reasoning_kwargs(self.engine, request), + **router_hint_kwargs(self.engine, request), **self._routed_experts_kwargs, bootstrap_host=bootstrap_info["bootstrap_host"], bootstrap_port=bootstrap_info["bootstrap_port"], @@ -537,6 +539,7 @@ async def generate( sampling_params=sampling_params, stream=True, **require_reasoning_kwargs(self.engine, request), + **router_hint_kwargs(self.engine, request), **self._routed_experts_kwargs, **mm_hashes_kwargs, external_trace_header=trace_header, diff --git a/components/src/dynamo/sglang/request_handlers/llm/prefill_handler.py b/components/src/dynamo/sglang/request_handlers/llm/prefill_handler.py index 28e1675432d7..d9e6c47df488 100644 --- a/components/src/dynamo/sglang/request_handlers/llm/prefill_handler.py +++ b/components/src/dynamo/sglang/request_handlers/llm/prefill_handler.py @@ -10,7 +10,7 @@ from dynamo._core import Context from dynamo.health_check import HEALTH_CHECK_KEY -from dynamo.sglang._compat import require_reasoning_kwargs +from dynamo.sglang._compat import require_reasoning_kwargs, router_hint_kwargs from dynamo.sglang.args import Config from dynamo.sglang.engine_generate import ( build_native_generate_request, @@ -191,6 +191,7 @@ async def generate( sampling_params=sampling_params, stream=True, **require_reasoning_kwargs(self.engine, inner_request), + **router_hint_kwargs(self.engine, inner_request), bootstrap_host=bootstrap_host, bootstrap_port=bootstrap_port, bootstrap_room=bootstrap_room, diff --git a/components/src/dynamo/sglang/router_hints.py b/components/src/dynamo/sglang/router_hints.py new file mode 100644 index 000000000000..842632fe7f36 --- /dev/null +++ b/components/src/dynamo/sglang/router_hints.py @@ -0,0 +1,190 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Router-hint capability advertisement for the SGLang backend. + +Mirrors ``dynamo/vllm/router_hints.py``: a worker must advertise the +``router_hint`` runtime capability, its topology role, and a per-DP-rank map of +KV source control endpoints, or the router treats it as hint-incapable and +silently emits no hints. + +The only difference from the vLLM version is where the endpoints come from. +vLLM reads ``kv_transfer_config.kv_connector_extra_config.secondary_tiers[]``; +SGLang has no secondary tiers, so the equivalent values live in the KVCR +HiCache storage backend's extra-config JSON +(``--hicache-storage-backend-extra-config``), whose ``control_host`` / +``control_port`` / ``control_advertise_host`` fields are what the KVCR store +binds its ZMQ peer control channel to. +""" + +from __future__ import annotations + +import json +import logging +from typing import Any, Optional + +from dynamo.common.constants import ( + ROUTER_HINT_RUNTIME_CAPABILITY_KEY, + ROUTER_HINT_SOURCE_CONTROL_ENDPOINTS_RUNTIME_KEY, + ROUTER_HINT_WORKER_TYPE_RUNTIME_KEY, +) + +# The HiCache storage backend that speaks the router-hint protocol. +_KVCR_BACKEND_NAME = "kvcr" + +# Hosts that identify a bind-any wildcard rather than a reachable peer address. +_UNROUTABLE_HOSTS = frozenset({"0.0.0.0", "::"}) + +# SGLang's disaggregation_mode spelled as the router's worker_type vocabulary. +# SGLang says "null" for a non-disaggregated engine; the router says +# "aggregated". The router only matches a hint source to a target when both +# report the same role, so this mapping has to agree with the vLLM backend's +# ``_router_hint_worker_type``. +_WORKER_TYPE_BY_DISAGGREGATION_MODE = { + "null": "aggregated", + "prefill": "prefill", + "decode": "decode", +} + + +def _router_hint_worker_type(server_args: Any) -> Optional[str]: + """Normalize SGLang's disaggregation mode into router-hint runtime metadata.""" + mode = getattr(server_args, "disaggregation_mode", None) or "null" + return _WORKER_TYPE_BY_DISAGGREGATION_MODE.get(mode) + + +def _dp_port_stride(server_args: Any) -> int: + """How many KVCR control ports one attention-DP rank of this engine owns. + + Unlike vLLM, where the KV secondary tier lives in the one scheduler process + per DP rank, SGLang builds a HiCache storage backend in *every* attention + rank's scheduler process. All of them read the same configured base port and + offset it by their own rank coordinate, so a DP rank occupies a whole block + of ports rather than a single one, and the next DP rank starts after that + block. + + The block size is the number of schedulers per DP rank, which SGLang spells + as ``attn_cp_size * attn_tp_size`` and which reduces to ``tp_size // + dp_size`` (``attn_tp_size = tp_size // dp_size // attn_cp_size``). Without + attention DP there is a single DP group and the stride is unused, but 1 is + also the honest answer: the whole engine is one block. + + Must stay in step with ``_rank_port_offset`` in SGLang's + ``mem_cache/storage/kvcr/kvcr_store.py``; a mismatch does not fail, it makes + peers dial the wrong rank and silently fetch the wrong attention shard. + """ + dp_size = getattr(server_args, "dp_size", 1) or 1 + if not getattr(server_args, "enable_dp_attention", False) or dp_size <= 1: + return 1 + tp_size = getattr(server_args, "tp_size", 1) or 1 + return max(tp_size // dp_size, 1) + + +def _source_control_endpoint( + extra_config: dict[str, Any], port_offset: int = 0 +) -> Optional[str]: + """Peer-reachable ZMQ control endpoint, or None if not advertisable. + + ``port_offset`` locates one DP rank's port block relative to the configured + base port (see :func:`_dp_port_stride`). The endpoint names that block's + first port, i.e. the DP rank's first attention rank; the consuming backend + adds its own within-DP-group offset, since the router has no TP concept and + cannot resolve that half itself. + + A wildcard bind host is not something a peer can dial, so it must be paired + with an explicit ``control_advertise_host``. An ephemeral port (0) cannot be + advertised either -- the bound port is only known inside the scheduler + process, so registration has nothing to publish. + """ + try: + control_port = int(extra_config.get("control_port")) + port_offset + except (TypeError, ValueError): + return None + if control_port <= 0: + return None + host = extra_config.get("control_advertise_host") or extra_config.get( + "control_host" + ) + if not isinstance(host, str) or not host or host in _UNROUTABLE_HOSTS: + return None + return f"tcp://{host}:{control_port}" + + +def _source_control_endpoints( + extra_config: dict[str, Any], dp_bounds: tuple[int, int], dp_port_stride: int +) -> Optional[dict[str, str]]: + """Per-global-DP-rank endpoint map, or None if any rank is unresolvable. + + The router keys hint sources by ``(worker_id, dp_rank)``, so the map is + keyed by *global* rank while the port offset follows the *local* rank -- on + a multinode engine this node only owns its own slice of the global range, + but each node numbers its ports from the same base. It is all-or-nothing: a + partial map would let the router select a rank no peer can dial. + """ + dp_start, dp_end = dp_bounds + endpoints: dict[str, str] = {} + for local_dp_rank in range(dp_end - dp_start): + endpoint = _source_control_endpoint( + extra_config, local_dp_rank * dp_port_stride + ) + if endpoint is None: + return None + endpoints[str(dp_start + local_dp_rank)] = endpoint + return endpoints + + +def enable_router_hint_support( + runtime_config: Any, + server_args: Any, + extra_config: dict[str, Any], + dp_bounds: tuple[int, int], +) -> None: + """Advertise router-hint capability when this worker runs the KVCR backend. + + No-op unless the KVCR HiCache storage backend is selected and configured to + consume hints -- a worker without a remote-capable KV source has nothing to + serve a peer from. Raises when the backend is configured for hints but its + control endpoints are not advertisable, since that combination would + register a worker the router selects as a source but no peer can dial. + + All three runtime keys are set together: the router requires all of them, so + publishing a subset produces a worker it silently never hints to. + """ + if getattr(server_args, "hicache_storage_backend", None) != _KVCR_BACKEND_NAME: + return + if not extra_config.get("enable_remote_hint"): + return + + worker_type = _router_hint_worker_type(server_args) + if worker_type is None: + return + + endpoints = _source_control_endpoints( + extra_config, dp_bounds, _dp_port_stride(server_args) + ) + if endpoints is None: + raise ValueError( + "router_hint support requires advertisable source control endpoints " + "for all managed DP ranks; set control_advertise_host (or a " + "non-wildcard control_host) and a positive control_port in " + "--hicache-storage-backend-extra-config" + ) + + # set_engine_specific expects JSON text; the Rust side accepts a truthy + # string for the capability flag but parses the other two as JSON values. + runtime_config.set_engine_specific( + ROUTER_HINT_RUNTIME_CAPABILITY_KEY, json.dumps(True) + ) + runtime_config.set_engine_specific( + ROUTER_HINT_WORKER_TYPE_RUNTIME_KEY, json.dumps(worker_type) + ) + runtime_config.set_engine_specific( + ROUTER_HINT_SOURCE_CONTROL_ENDPOINTS_RUNTIME_KEY, + json.dumps(endpoints), + ) + logging.info( + "Advertised router_hint capability (worker_type=%s) with source control " + "endpoints %s", + worker_type, + endpoints, + ) diff --git a/components/src/dynamo/sglang/tests/test_sglang_router_hints.py b/components/src/dynamo/sglang/tests/test_sglang_router_hints.py new file mode 100644 index 000000000000..dd95a6d581eb --- /dev/null +++ b/components/src/dynamo/sglang/tests/test_sglang_router_hints.py @@ -0,0 +1,330 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import json +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pytest + +from dynamo.common.constants import ( + ROUTER_HINT_RUNTIME_CAPABILITY_KEY, + ROUTER_HINT_SOURCE_CONTROL_ENDPOINTS_RUNTIME_KEY, + ROUTER_HINT_WORKER_TYPE_RUNTIME_KEY, +) +from dynamo.sglang._compat import router_hint_kwargs +from dynamo.sglang.router_hints import enable_router_hint_support + +pytestmark = [ + pytest.mark.unit, + pytest.mark.sglang, + pytest.mark.unified, + pytest.mark.gpu_0, + pytest.mark.pre_merge, +] + + +def _server_args(**overrides): + args = {"hicache_storage_backend": "kvcr"} + args.update(overrides) + return SimpleNamespace(**args) + + +def _kvcr_config(**overrides): + config = { + "enable_remote_hint": True, + "control_host": "0.0.0.0", + "control_advertise_host": "127.0.0.1", + "control_port": 25000, + } + config.update(overrides) + return config + + +def _published(runtime_config): + return dict(call.args for call in runtime_config.set_engine_specific.call_args_list) + + +def test_publishes_single_dp_rank_endpoint(): + runtime_config = MagicMock() + + enable_router_hint_support( + runtime_config=runtime_config, + server_args=_server_args(), + extra_config=_kvcr_config(), + dp_bounds=(0, 1), + ) + + assert _published(runtime_config) == { + ROUTER_HINT_RUNTIME_CAPABILITY_KEY: json.dumps(True), + ROUTER_HINT_WORKER_TYPE_RUNTIME_KEY: json.dumps("aggregated"), + ROUTER_HINT_SOURCE_CONTROL_ENDPOINTS_RUNTIME_KEY: json.dumps( + {"0": "tcp://127.0.0.1:25000"} + ), + } + + +@pytest.mark.parametrize( + "disaggregation_mode,expected_worker_type", + [ + (None, "aggregated"), + ("null", "aggregated"), + ("prefill", "prefill"), + ("decode", "decode"), + ], +) +def test_publishes_worker_type_from_disaggregation_mode( + disaggregation_mode, expected_worker_type +): + """The router only hints between workers reporting the same role. + + ``router_hint_metadata_for_dp_rank`` (lib/llm/src/local_model/runtime_config.rs) + returns None without this key, and ``router_hint_for_selection`` + (lib/llm/src/kv_router.rs) then matches source to target on it. A worker + that omits it is invisible to the router as both hint target and hint + source, so hints silently never fire. SGLang spells a non-disaggregated + engine "null"; the router's vocabulary calls it "aggregated". + """ + runtime_config = MagicMock() + + enable_router_hint_support( + runtime_config=runtime_config, + server_args=_server_args(disaggregation_mode=disaggregation_mode), + extra_config=_kvcr_config(), + dp_bounds=(0, 1), + ) + + assert _published(runtime_config)[ + ROUTER_HINT_WORKER_TYPE_RUNTIME_KEY + ] == json.dumps(expected_worker_type) + + +def test_publishes_one_endpoint_per_dp_rank(): + """One DP rank per port when each rank owns a single scheduler (attn_tp=1).""" + runtime_config = MagicMock() + + enable_router_hint_support( + runtime_config=runtime_config, + server_args=_server_args( + enable_dp_attention=True, dp_size=4, tp_size=4, nnodes=1, node_rank=0 + ), + extra_config=_kvcr_config(control_advertise_host="worker-a"), + dp_bounds=(0, 4), + ) + + assert json.loads( + _published(runtime_config)[ROUTER_HINT_SOURCE_CONTROL_ENDPOINTS_RUNTIME_KEY] + ) == { + "0": "tcp://worker-a:25000", + "1": "tcp://worker-a:25001", + "2": "tcp://worker-a:25002", + "3": "tcp://worker-a:25003", + } + + +def test_dp_ranks_are_strided_by_the_schedulers_each_one_owns(): + """A DP rank owns a *block* of ports, one per attention rank under it. + + SGLang builds a KVCR store in every attention rank's scheduler process, all + offsetting the same configured base port by their own rank coordinate. At + DP=2/TP=4 that is four schedulers laid out as ports P..P+3, so DP rank 1 + starts at P+2, not P+1. Advertising a bare DP rank here (which is what the + vLLM module correctly does, since it has one scheduler per DP rank) would + name a port belonging to DP rank 0's second attention rank -- and because + KVCR block keys are token hashes carrying no rank identity, the peer would + accept the wrong attention shard instead of erroring. + """ + runtime_config = MagicMock() + + enable_router_hint_support( + runtime_config=runtime_config, + server_args=_server_args( + enable_dp_attention=True, dp_size=2, tp_size=4, nnodes=1, node_rank=0 + ), + extra_config=_kvcr_config(control_advertise_host="worker-a"), + dp_bounds=(0, 2), + ) + + assert json.loads( + _published(runtime_config)[ROUTER_HINT_SOURCE_CONTROL_ENDPOINTS_RUNTIME_KEY] + ) == { + "0": "tcp://worker-a:25000", + "1": "tcp://worker-a:25002", + } + + +def test_multinode_keys_global_dp_ranks_but_strides_from_the_local_base(): + """Each node numbers its own ports from the same configured base port. + + The map key is the router-visible global rank, but the offset is the local + one -- node 1 of a DP=4/TP=8 engine binds the same ports as node 0, on its + own host. Striding by the global rank would advertise ports nothing binds. + """ + runtime_config = MagicMock() + + enable_router_hint_support( + runtime_config=runtime_config, + server_args=_server_args( + enable_dp_attention=True, dp_size=4, tp_size=8, nnodes=2, node_rank=1 + ), + extra_config=_kvcr_config(control_advertise_host="worker-b"), + dp_bounds=(2, 4), + ) + + assert json.loads( + _published(runtime_config)[ROUTER_HINT_SOURCE_CONTROL_ENDPOINTS_RUNTIME_KEY] + ) == { + "2": "tcp://worker-b:25000", + "3": "tcp://worker-b:25002", + } + + +@pytest.mark.parametrize( + "server_args_overrides", + [ + # Plain --tp-size: one DP group, so every scheduler is under DP rank 0 + # and the backend's own tp_rank offset already spans the whole engine. + {"enable_dp_attention": False, "dp_size": 1, "tp_size": 8}, + # --dp-size without --enable-dp-attention is replicated, not attention + # DP: the schedulers still form a single rank space per engine, and + # `local_dp_rank_bounds` reports a single rank for it. + {"enable_dp_attention": False, "dp_size": 4, "tp_size": 8}, + ], +) +def test_no_attention_dp_advertises_the_base_port_unstrided(server_args_overrides): + """Guards the TP-only path that was validated on hardware.""" + runtime_config = MagicMock() + + enable_router_hint_support( + runtime_config=runtime_config, + server_args=_server_args(**server_args_overrides), + extra_config=_kvcr_config(control_advertise_host="worker-a"), + dp_bounds=(0, 1), + ) + + assert json.loads( + _published(runtime_config)[ROUTER_HINT_SOURCE_CONTROL_ENDPOINTS_RUNTIME_KEY] + ) == {"0": "tcp://worker-a:25000"} + + +def test_advertise_host_overrides_wildcard_bind_host(): + runtime_config = MagicMock() + + enable_router_hint_support( + runtime_config=runtime_config, + server_args=_server_args(), + extra_config=_kvcr_config(control_host="::"), + dp_bounds=(0, 1), + ) + + assert ( + json.loads( + _published(runtime_config)[ROUTER_HINT_SOURCE_CONTROL_ENDPOINTS_RUNTIME_KEY] + )["0"] + == "tcp://127.0.0.1:25000" + ) + + +@pytest.mark.parametrize( + "server_args,extra_config", + [ + # A different HiCache backend cannot serve a hint at all. + (_server_args(hicache_storage_backend="mooncake"), _kvcr_config()), + (_server_args(hicache_storage_backend=None), _kvcr_config()), + # KVCR without remote hints is local-only; advertising it would route + # hints to a worker that will not act on them. + (_server_args(), _kvcr_config(enable_remote_hint=False)), + (_server_args(), {}), + # A disaggregation mode the router has no worker_type for. Publishing + # the other two keys without it registers a worker the router filters + # out anyway, so say nothing rather than half-advertise. + (_server_args(disaggregation_mode="encode"), _kvcr_config()), + ], +) +def test_publishes_nothing_when_not_a_hint_source(server_args, extra_config): + runtime_config = MagicMock() + + enable_router_hint_support( + runtime_config=runtime_config, + server_args=server_args, + extra_config=extra_config, + dp_bounds=(0, 1), + ) + + runtime_config.set_engine_specific.assert_not_called() + + +@pytest.mark.parametrize( + "overrides", + [ + # A wildcard bind with no advertise host is not dialable by a peer. + {"control_advertise_host": None, "control_host": "0.0.0.0"}, + {"control_advertise_host": None, "control_host": "::"}, + {"control_advertise_host": None, "control_host": ""}, + # An ephemeral port is only known inside the scheduler process. + {"control_port": 0}, + {"control_port": None}, + {"control_port": "not-a-port"}, + ], +) +def test_raises_when_endpoint_is_not_advertisable(overrides): + """Configured for hints but unreachable is a misconfiguration, not a no-op. + + Silently skipping would leave the worker running while every peer fetch + fails, which is far harder to diagnose than failing registration. + """ + with pytest.raises(ValueError, match="advertisable source control endpoints"): + enable_router_hint_support( + runtime_config=MagicMock(), + server_args=_server_args(), + extra_config=_kvcr_config(**overrides), + dp_bounds=(0, 1), + ) + + +class _Engine: + """Stand-in whose async_generate accepts kv_router_hint.""" + + async def async_generate(self, *, kv_router_hint=None, **kwargs): + return None + + +class _LegacyEngine: + """An older SGLang install with no kv_router_hint kwarg.""" + + async def async_generate(self, *, sampling_params=None): + return None + + +_HINT = {"source_control_endpoint": "tcp://peer:25000", "block_hashes": [1, 2]} + + +def test_router_hint_kwargs_extracts_hint_from_kv_transfer_params(): + request = {"extra_args": {"kv_transfer_params": {"router_hint": _HINT}}} + + assert router_hint_kwargs(_Engine(), request) == {"kv_router_hint": _HINT} + + +@pytest.mark.parametrize( + "request_payload", + [ + {}, + {"extra_args": None}, + {"extra_args": {}}, + {"extra_args": {"kv_transfer_params": None}}, + {"extra_args": {"kv_transfer_params": {}}}, + # A non-dict hint is not something the backend can parse. + {"extra_args": {"kv_transfer_params": {"router_hint": "nope"}}}, + ], +) +def test_router_hint_kwargs_is_empty_without_a_hint(request_payload): + assert router_hint_kwargs(_Engine(), request_payload) == {} + + +def test_router_hint_kwargs_dropped_when_engine_cannot_accept_it(): + """A stale SGLang recomputes the prefix rather than failing the request.""" + request = {"extra_args": {"kv_transfer_params": {"router_hint": _HINT}}} + + assert router_hint_kwargs(_LegacyEngine(), request) == {} From 629bd08f5f7fbd7c5f0f8e11de1c90b591cd3780 Mon Sep 17 00:00:00 2001 From: linhu-nv Date: Fri, 21 Aug 2026 11:18:24 +0800 Subject: [PATCH 2/3] fix(sglang): address router hint review feedback - Read hicache_storage_backend_extra_config defensively. The attribute is evaluated before enable_router_hint_support can short-circuit on a non-KVCR backend, so a server_args without the field aborted registration entirely. Matches how _get_mooncake_runtime_data reads the same field. - Range-check the control port after the DP-rank offset. A base port near the top of the range carries the last rank past 65535, which registers a worker the router selects as a hint source but no peer can dial. - Advertise control_advertise_host only. control_host is the bind address and is legitimately a wildcard, so falling back to it publishes whichever of the two happens to be set. SGLang's own config validation is being changed to match; leaving them out of step would have dynamo reject a config the backend accepts, with no diagnostic tying the two together. Signed-off-by: linhu-nv --- components/src/dynamo/sglang/register.py | 2 +- components/src/dynamo/sglang/router_hints.py | 28 +++++------ .../sglang/tests/test_sglang_router_hints.py | 48 ++++++++++++++++--- 3 files changed, 56 insertions(+), 22 deletions(-) diff --git a/components/src/dynamo/sglang/register.py b/components/src/dynamo/sglang/register.py index daaea4b95038..e9fedb454ca9 100644 --- a/components/src/dynamo/sglang/register.py +++ b/components/src/dynamo/sglang/register.py @@ -500,7 +500,7 @@ async def get_runtime_config( runtime_config=runtime_config, server_args=server_args, extra_config=_parse_hicache_storage_extra_config( - server_args.hicache_storage_backend_extra_config + getattr(server_args, "hicache_storage_backend_extra_config", None) ), dp_bounds=local_dp_rank_bounds(server_args), ) diff --git a/components/src/dynamo/sglang/router_hints.py b/components/src/dynamo/sglang/router_hints.py index 842632fe7f36..054149e3a1e1 100644 --- a/components/src/dynamo/sglang/router_hints.py +++ b/components/src/dynamo/sglang/router_hints.py @@ -12,9 +12,9 @@ vLLM reads ``kv_transfer_config.kv_connector_extra_config.secondary_tiers[]``; SGLang has no secondary tiers, so the equivalent values live in the KVCR HiCache storage backend's extra-config JSON -(``--hicache-storage-backend-extra-config``), whose ``control_host`` / -``control_port`` / ``control_advertise_host`` fields are what the KVCR store -binds its ZMQ peer control channel to. +(``--hicache-storage-backend-extra-config``), whose ``control_advertise_host`` +and ``control_port`` fields describe the ZMQ peer control channel the KVCR store +binds. """ from __future__ import annotations @@ -91,20 +91,20 @@ def _source_control_endpoint( adds its own within-DP-group offset, since the router has no TP concept and cannot resolve that half itself. - A wildcard bind host is not something a peer can dial, so it must be paired - with an explicit ``control_advertise_host``. An ephemeral port (0) cannot be - advertised either -- the bound port is only known inside the scheduler - process, so registration has nothing to publish. + Only ``control_advertise_host`` names the address peers dial; + ``control_host`` is the bind address and is legitimately a wildcard, so it + is not a fallback. An ephemeral port (0) cannot be advertised either -- the + bound port is only known inside the scheduler process, so registration has + nothing to publish. The offset port is range-checked too, since a base port + near the top of the range can carry the last DP rank past 65535. """ try: control_port = int(extra_config.get("control_port")) + port_offset except (TypeError, ValueError): return None - if control_port <= 0: + if not 0 < control_port <= 65535: return None - host = extra_config.get("control_advertise_host") or extra_config.get( - "control_host" - ) + host = extra_config.get("control_advertise_host") if not isinstance(host, str) or not host or host in _UNROUTABLE_HOSTS: return None return f"tcp://{host}:{control_port}" @@ -165,9 +165,9 @@ def enable_router_hint_support( if endpoints is None: raise ValueError( "router_hint support requires advertisable source control endpoints " - "for all managed DP ranks; set control_advertise_host (or a " - "non-wildcard control_host) and a positive control_port in " - "--hicache-storage-backend-extra-config" + "for all managed DP ranks; set control_advertise_host to a " + "peer-reachable address and a control_port that keeps every rank " + "within 1..65535 in --hicache-storage-backend-extra-config" ) # set_engine_specific expects JSON text; the Rust side accepts a truthy diff --git a/components/src/dynamo/sglang/tests/test_sglang_router_hints.py b/components/src/dynamo/sglang/tests/test_sglang_router_hints.py index dd95a6d581eb..0b31c1034d9f 100644 --- a/components/src/dynamo/sglang/tests/test_sglang_router_hints.py +++ b/components/src/dynamo/sglang/tests/test_sglang_router_hints.py @@ -209,13 +209,41 @@ def test_no_attention_dp_advertises_the_base_port_unstrided(server_args_override ) == {"0": "tcp://worker-a:25000"} -def test_advertise_host_overrides_wildcard_bind_host(): +def test_raises_when_striding_carries_a_later_rank_past_the_port_range(): + """The base port being dialable does not make the whole block dialable. + + Only the last rank's port leaves the range here, and it is the one the + router would hand out for that DP rank. + """ + with pytest.raises(ValueError, match="advertisable source control endpoints"): + enable_router_hint_support( + runtime_config=MagicMock(), + server_args=_server_args( + enable_dp_attention=True, dp_size=4, tp_size=4, nnodes=1, node_rank=0 + ), + extra_config=_kvcr_config( + control_advertise_host="worker-a", control_port=65534 + ), + dp_bounds=(0, 4), + ) + + +def test_advertises_the_advertise_host_not_the_bind_host(): + """The bind host is not an address a peer can dial, even when it is routable. + + ``control_host`` names the interface to bind, and a worker commonly binds a + wildcard while advertising one reachable address. Reading the bind host + would publish whichever of the two happens to be set, which is only right by + coincidence. + """ runtime_config = MagicMock() enable_router_hint_support( runtime_config=runtime_config, server_args=_server_args(), - extra_config=_kvcr_config(control_host="::"), + extra_config=_kvcr_config( + control_host="10.0.0.1", control_advertise_host="worker-a" + ), dp_bounds=(0, 1), ) @@ -223,7 +251,7 @@ def test_advertise_host_overrides_wildcard_bind_host(): json.loads( _published(runtime_config)[ROUTER_HINT_SOURCE_CONTROL_ENDPOINTS_RUNTIME_KEY] )["0"] - == "tcp://127.0.0.1:25000" + == "tcp://worker-a:25000" ) @@ -259,14 +287,20 @@ def test_publishes_nothing_when_not_a_hint_source(server_args, extra_config): @pytest.mark.parametrize( "overrides", [ - # A wildcard bind with no advertise host is not dialable by a peer. - {"control_advertise_host": None, "control_host": "0.0.0.0"}, - {"control_advertise_host": None, "control_host": "::"}, - {"control_advertise_host": None, "control_host": ""}, + # No advertise host at all: the bind host is not a substitute, however + # routable it looks. + {"control_advertise_host": None}, + {"control_advertise_host": None, "control_host": "10.0.0.1"}, + {"control_advertise_host": ""}, + # A wildcard is a bind address, so it cannot be advertised either. + {"control_advertise_host": "0.0.0.0"}, + {"control_advertise_host": "::"}, # An ephemeral port is only known inside the scheduler process. {"control_port": 0}, {"control_port": None}, {"control_port": "not-a-port"}, + # Every rank's port has to stay dialable, not just the base. + {"control_port": 65536}, ], ) def test_raises_when_endpoint_is_not_advertisable(overrides): From 7cd76903b72ad6c8f2b38dbf34520b3de81464bc Mon Sep 17 00:00:00 2001 From: linhu-nv Date: Thu, 27 Aug 2026 14:45:38 +0800 Subject: [PATCH 3/3] fix(sglang): send the hint as the versioned kv_hints field SGLang renamed the request field to kv_hints and now parses it as a KV-hint envelope, so the old kwarg name is silently dropped by the supported-kwarg filter and every hint is lost with only a warning. Forward the router's hint under the new name. The value still passes through untouched: SGLang reads the envelope and picks out the actions it implements, so this adapter needs no further change once the router wraps its hint in one. --- components/src/dynamo/sglang/_compat.py | 21 +++++++++++++------ .../sglang/tests/test_sglang_router_hints.py | 8 +++---- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/components/src/dynamo/sglang/_compat.py b/components/src/dynamo/sglang/_compat.py index 3da68933af27..469a1ece4d04 100644 --- a/components/src/dynamo/sglang/_compat.py +++ b/components/src/dynamo/sglang/_compat.py @@ -34,12 +34,17 @@ _KV_TRANSFER_PARAMS_EXTRA_ARGS_KEY = "kv_transfer_params" _ROUTER_HINT_EXTRA_ARGS_KEY = "router_hint" +# The async_generate kwarg SGLang reads the hint from. Named for the KV-hint +# envelope (protocol 0.1) rather than for one hint shape, since SGLang parses +# the field as a list of independently versioned actions. +_SGLANG_HINT_KWARG = "kv_hints" + @lru_cache(maxsize=1) def _warn_router_hint_unsupported() -> None: logger.warning( "Dropping the router KV-reuse hint because SGLang Engine.async_generate " - "does not accept kv_router_hint; requests will recompute prefixes a peer " + f"does not accept {_SGLANG_HINT_KWARG}; requests will recompute prefixes a peer " "already holds. Upgrade SGLang to enable hint-driven KV reuse." ) @@ -192,10 +197,14 @@ def router_hint_kwargs(engine: Any, request: Mapping[str, Any]) -> dict[str, Any """Build the optional SGLang per-request router KV-reuse hint argument. The KV router attaches its hint under ``extra_args.kv_transfer_params``; - SGLang consumes it as the ``kv_router_hint`` async_generate kwarg, which is - threaded down to the KVCR HiCache storage backend so it can pull the prefix - from the named peer. An engine without that kwarg just recomputes the + SGLang consumes it as the ``kv_hints`` async_generate kwarg, which is + threaded down to the HiCache storage backends so one of them can pull the + prefix from the named peer. An engine without that kwarg just recomputes the prefix, so a missing hint degrades rather than failing the request. + + The value is forwarded untouched: SGLang reads the KV-hint envelope and + picks out the actions it implements, so this adapter needs no change when + the router starts wrapping its hint in one. """ extra_args = request.get("extra_args") if not isinstance(extra_args, Mapping): @@ -209,9 +218,9 @@ def router_hint_kwargs(engine: Any, request: Mapping[str, Any]) -> dict[str, Any kwargs = filter_supported_async_generate_kwargs( engine, - {"kv_router_hint": dict(router_hint)}, + {_SGLANG_HINT_KWARG: dict(router_hint)}, ) - if "kv_router_hint" not in kwargs: + if _SGLANG_HINT_KWARG not in kwargs: _warn_router_hint_unsupported() return kwargs diff --git a/components/src/dynamo/sglang/tests/test_sglang_router_hints.py b/components/src/dynamo/sglang/tests/test_sglang_router_hints.py index 0b31c1034d9f..3e85a4329596 100644 --- a/components/src/dynamo/sglang/tests/test_sglang_router_hints.py +++ b/components/src/dynamo/sglang/tests/test_sglang_router_hints.py @@ -319,14 +319,14 @@ def test_raises_when_endpoint_is_not_advertisable(overrides): class _Engine: - """Stand-in whose async_generate accepts kv_router_hint.""" + """Stand-in whose async_generate accepts kv_hints.""" - async def async_generate(self, *, kv_router_hint=None, **kwargs): + async def async_generate(self, *, kv_hints=None, **kwargs): return None class _LegacyEngine: - """An older SGLang install with no kv_router_hint kwarg.""" + """An older SGLang install with no kv_hints kwarg.""" async def async_generate(self, *, sampling_params=None): return None @@ -338,7 +338,7 @@ async def async_generate(self, *, sampling_params=None): def test_router_hint_kwargs_extracts_hint_from_kv_transfer_params(): request = {"extra_args": {"kv_transfer_params": {"router_hint": _HINT}}} - assert router_hint_kwargs(_Engine(), request) == {"kv_router_hint": _HINT} + assert router_hint_kwargs(_Engine(), request) == {"kv_hints": _HINT} @pytest.mark.parametrize(