Skip to content
Merged
Show file tree
Hide file tree
Changes from 8 commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
8b35d31
feat(sidecar): launch headless sidecars for local KV publishers
jain-ria Sep 15, 2026
51907f8
fix(sidecar): remove provider readiness handshake
jain-ria Sep 15, 2026
ca0971e
fix(sidecar): tighten coverage and report logical KV block sizes
jain-ria Sep 16, 2026
9969f50
test(sidecar): remove expanded controller test and trim setup
jain-ria Sep 16, 2026
c6c49ac
feat(grpc): scope follower discovery to node-local metadata
jain-ria Sep 16, 2026
c50b251
fix(kv-events): support explicit loopback publisher binds
jain-ria Sep 17, 2026
8eaffcf
test(grpc): trim redundant metadata coverage
jain-ria Sep 17, 2026
e676e58
fix(kv-events): advertise absolute filesystem IPC endpoints
jain-ria Sep 22, 2026
1d3c0d3
refactor: share server metadata across HTTP and native gRPC
jain-ria Sep 23, 2026
e1155ad
test: remove dedicated follower metadata test module
jain-ria Sep 23, 2026
39859ba
Merge main into codex/sglang-local-kv-sidecars
jain-ria Sep 29, 2026
13bbd63
docs: simplify shared server metadata docstring
jain-ria Sep 29, 2026
9001399
test(grpc): avoid Python wrapper in metadata service test
jain-ria Sep 29, 2026
a07f3cd
Merge branch 'main' into codex/sglang-local-kv-sidecars
jain-ria Sep 30, 2026
0b18034
Merge main and adapt follower metadata to shared protobuf bindings
jain-ria Sep 30, 2026
4a8efff
Merge branch 'main' into codex/sglang-local-kv-sidecars
ishandhanani Oct 1, 2026
6049848
fix(grpc): align Rust frontend streams with shared default stubs
jain-ria Oct 5, 2026
cbd9ea1
Merge main to satisfy CI minimum base requirement
jain-ria Oct 5, 2026
d9e2a4a
Merge branch 'main' into codex/sglang-local-kv-sidecars
ishandhanani Oct 5, 2026
6c6e1cc
Merge branch 'main' into codex/sglang-local-kv-sidecars
jain-ria Oct 6, 2026
047eb45
Merge branch 'main' into codex/sglang-local-kv-sidecars
jain-ria Oct 6, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion python/sglang/srt/arg_groups/fields/serving.py
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,8 @@ class Serving(msgspec.Struct):
Optional[int],
"Port for the native gRPC server, started alongside HTTP. Setting this "
"(or SGLANG_GRPC_PORT) enables the native gRPC server; it is off by "
"default. In legacy --smg-grpc-mode this is the SMG server port and "
"default. Follower nodes expose GetServerInfo only. "
"In legacy --smg-grpc-mode this is the SMG server port and "
"defaults to --port + 10000.",
] = None
# Env-only (SGLANG_GRPC_WORKER_THREADS); a field so the projection sees it.
Expand Down
89 changes: 80 additions & 9 deletions python/sglang/srt/disaggregation/kv_events.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import atexit
import enum
import logging
import os
import queue
import threading
import time
Expand Down Expand Up @@ -330,6 +331,10 @@ def publish(self, events: EventBatch) -> None:
def shutdown(self) -> None:
"""Shutdown the publisher."""

def describe_local_source(self, block_size: int) -> Optional[dict[str, Any]]:
"""Describe a source accessible to a separate local process, if any."""
return None


class NullEventPublisher(EventPublisher):
"""No-op implementation (default when disabled)."""
Expand All @@ -350,7 +355,7 @@ class ZmqEventPublisher(EventPublisher):
----------
endpoint:
PUB address. Use ``tcp://*:5557`` to bind or ``tcp://host:5557`` to
connect.
connect, unless overridden by ``bind``.
replay_endpoint:
Optional ROUTER address for replay requests. When given, subscribers can
request missed batches by sending the starting sequence number as an
Expand All @@ -363,6 +368,9 @@ class ZmqEventPublisher(EventPublisher):
Maximum number of events to buffer in memory.
topic:
Topic to publish events to.
bind:
Whether to bind the PUB socket. When unset, preserve the endpoint-based
heuristic: bind wildcard, IPC and inproc addresses; connect otherwise.
"""

SHUTDOWN_TIMEOUT: float = 1.0
Expand All @@ -377,6 +385,7 @@ def __init__(
hwm: int = 100_000,
max_queue_size: int = 100_000,
topic: str = "",
bind: Optional[bool] = None,
) -> None:
# Storage
self._event_queue = Queue[Optional[EventBatch]](maxsize=max_queue_size)
Expand All @@ -385,9 +394,12 @@ def __init__(
# ZMQ sockets
self._ctx = zmq.Context.instance()
self._pub: Optional[zmq.Socket] = None
self._local_pub_endpoint: Optional[str] = None
self._local_replay_endpoint: Optional[str] = None
self._replay: Optional[zmq.Socket] = None
self._dp_rank = attn_dp_rank
self._endpoint = self.offset_endpoint_port(endpoint, self._dp_rank)
self._bind = bind
self._replay_endpoint = self.offset_endpoint_port(
replay_endpoint, self._dp_rank
)
Expand Down Expand Up @@ -416,6 +428,43 @@ def publish(self, events: EventBatch) -> None:
events.attn_dp_rank = self._dp_rank
self._event_queue.put(events)

def describe_local_source(self, block_size: int) -> Optional[dict[str, Any]]:
"""Report the bound socket, not a port reconstructed from global DP size.

Connect-style publishers and inproc sockets have no subscribable local
endpoint. They keep working as before, but are not advertised to external
processes. Wildcard binds are reachable on loopback from this node.
"""

def local_endpoint(endpoint: Optional[str]) -> Optional[str]:
if endpoint is None:
return None
if endpoint.startswith("ipc://"):
return endpoint
Comment thread
jain-ria marked this conversation as resolved.
Resolved
if parse_advertisable_tcp(endpoint) is None:
return None
address = NetworkAddress.parse(endpoint[len("tcp://") :])
host = address.host
if host in ("*", "0.0.0.0"):
host = "127.0.0.1"
elif host == "::":
host = "::1"
return NetworkAddress(host, address.port).to_tcp()

endpoint = local_endpoint(self._local_pub_endpoint)
if endpoint is None:
return None
source = {
"dp_rank": self._dp_rank,
"endpoint": endpoint,
"topic": self._topic_bytes.decode("utf-8"),
"block_size": block_size,
}
replay_endpoint = local_endpoint(self._local_replay_endpoint)
if replay_endpoint is not None:
source["replay_endpoint"] = replay_endpoint
return source

def shutdown(self) -> None:
"""Stop the publisher thread and clean up resources."""
self._running = False
Expand Down Expand Up @@ -454,23 +503,27 @@ def _socket_setup(self) -> None:
if self._pub is None:
self._pub = self._ctx.socket(zmq.PUB)
self._pub.set_hwm(self._hwm)
# Heuristic: bind if wildcard / * present, else connect.
# Default heuristic: bind if wildcard / * present, else connect.
# bind stable, connect volatile convention.
# ``0.0.0.0`` is the IPv4 bind-all wildcard alongside ``*``
# and ``::``; ``/server_info`` advertises it as a wildcard,
# so the publisher must bind it for the advertised endpoint
# to actually be listening.
if (
"*" in self._endpoint
or "::" in self._endpoint
or "0.0.0.0" in self._endpoint
or self._endpoint.startswith("ipc://")
or self._endpoint.startswith("inproc://")
):
should_bind = self._bind
if should_bind is None:
should_bind = (
"*" in self._endpoint
or "::" in self._endpoint
or "0.0.0.0" in self._endpoint
or self._endpoint.startswith("ipc://")
or self._endpoint.startswith("inproc://")
)
if should_bind:
logger.debug(
f"ZmqEventPublisher socket publisher_endpoint bind to {self._endpoint}"
)
self._pub.bind(self._endpoint)
self._local_pub_endpoint = self._bound_endpoint(self._pub)
else:
self._pub.connect(self._endpoint)

Expand All @@ -484,6 +537,19 @@ def _socket_setup(self) -> None:
f"ZmqEventPublisher socket replay_endpoint bind to {self._replay_endpoint}"
)
self._replay.bind(self._replay_endpoint)
self._local_replay_endpoint = self._bound_endpoint(self._replay)

@staticmethod
def _bound_endpoint(socket: zmq.Socket) -> str:
endpoint = socket.getsockopt_string(zmq.LAST_ENDPOINT)
if endpoint.startswith("ipc://"):
path = endpoint[len("ipc://") :]
# Capture the publisher's working directory at bind time so a
# separate process can use the address. Abstract IPC names are
# independent of the filesystem and must remain unchanged.
if not path.startswith("@"):
endpoint = f"ipc://{os.path.abspath(path)}"
return endpoint

def _publisher_thread(self) -> None:
"""Background thread that processes the event queue."""
Expand Down Expand Up @@ -590,6 +656,11 @@ class KVEventsConfig(BaseModel):
"""The zmq endpoint to use for publishing kv events.
"""

bind: Optional[bool] = None
"""Bind (true) or connect (false) the PUB socket. When unset, infer from
the endpoint as before. Use true to bind a concrete address such as loopback.
"""

replay_endpoint: Optional[str] = None
"""The zmq endpoint to use for replaying kv events.
"""
Expand Down
53 changes: 44 additions & 9 deletions python/sglang/srt/entrypoints/engine.py
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,12 @@ class SchedulerInitResult:
wait_for_ready: Callable[[], None] = lambda: None
block_until_scheduler_exits: Callable[[], None] = lambda: None
engine_info_bootstrap_server: Optional[Any] = None
grpc_server: Optional[Any] = None

def stop_grpc_server(self) -> None:
if self.grpc_server is not None:
self.grpc_server.shutdown()
self.grpc_server = None


def init_tokenizer_manager(
Expand Down Expand Up @@ -937,6 +943,17 @@ def _launch_scheduler_processes(

def wait_for_ready():
infos = _wait_for_scheduler_ready(scheduler_pipe_readers, scheduler_procs)
if any("kv_event_sources" in info for info in infos):
# Both gRPC entrypoints consume the first scheduler info. Keep
# the sources from every local scheduler, not just the first.
infos[0]["kv_event_sources"] = sorted(
(
source
for info in infos
for source in info.get("kv_event_sources", [])
),
key=lambda source: source["dp_rank"],
)
scheduler_infos.extend(infos)
if use_dp_controller:
for info in infos:
Expand Down Expand Up @@ -1159,6 +1176,20 @@ def _launch_subprocesses(
# Non-zero-rank nodes do not run tokenizer processes.
scheduler_init_result.wait_for_ready()

from sglang.srt.entrypoints.grpc_metadata import start_follower_grpc_server

try:
scheduler_init_result.grpc_server = start_follower_grpc_server(
server_args, scheduler_init_result.scheduler_infos[0]
)
except BaseException:
# Engine.__init__ has not received these handles yet. Do not
# leave GPU workers behind if binding the metadata port fails.
for proc in scheduler_procs or []:
kill_process_tree(proc.pid, wait_timeout=60)
cls._terminate_weight_cache_daemons(weight_cache_daemon_procs)
raise

if os.getenv("SGLANG_BLOCK_NONZERO_RANK_CHILDREN") == "0":
# When using `Engine` as a Python API, we don't want to block here.
return (
Expand All @@ -1174,14 +1205,16 @@ def _launch_subprocesses(
rust_server_owns_base_port = (
envs.SGLANG_RUST_SERVER.get() and node_hosts_rust_server()
)
if not rust_server_owns_base_port:
launch_dummy_health_check_server(
get_serving().host,
get_serving().port,
get_observability().enable_metrics,
)

scheduler_init_result.block_until_scheduler_exits()
try:
if not rust_server_owns_base_port:
launch_dummy_health_check_server(
get_serving().host,
get_serving().port,
get_observability().enable_metrics,
)
scheduler_init_result.block_until_scheduler_exits()
finally:
scheduler_init_result.stop_grpc_server()
return (
None,
None,
Expand Down Expand Up @@ -1275,6 +1308,9 @@ def shutdown(self):
its GPU context so the caller can immediately reallocate on the same
device."""
try:
scheduler_init_result = getattr(self, "_scheduler_init_result", None)
if scheduler_init_result is not None:
scheduler_init_result.stop_grpc_server()
if (
self.tokenizer_manager is not None
and self.tokenizer_manager._subprocess_watchdog is not None
Expand Down Expand Up @@ -1665,7 +1701,6 @@ def save_sharded_model(self, **kwargs):


def _set_envs_and_config(server_args: ServerArgs):

cfg = resolving_view(server_args)
# Set global environments
# MNNVL fabric (GB200/GB300) multi-node: cross-node NVLink needs NCCL's
Expand Down
24 changes: 5 additions & 19 deletions python/sglang/srt/entrypoints/grpc_bridge.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,12 +16,8 @@

from sglang.srt.arg_groups.overrides import resolving_view
from sglang.srt.configs.embedding_model_spec import resolved_embedding_plan
from sglang.srt.runtime_context import (
describe_kv_events_publisher,
get_lora,
get_serving,
)
from sglang.srt.utils.msgspec_utils import msgspec_to_builtins
from sglang.srt.entrypoints.grpc_metadata import get_server_info_json
from sglang.srt.runtime_context import get_lora, get_serving

logger = logging.getLogger(__name__)

Expand Down Expand Up @@ -235,9 +231,7 @@ def _get_openai_serving(self):
return self._openai_serving_classes

from sglang.srt.entrypoints.openai.serving_chat import OpenAIServingChat
from sglang.srt.entrypoints.openai.serving_classify import (
OpenAIServingClassify,
)
from sglang.srt.entrypoints.openai.serving_classify import OpenAIServingClassify
from sglang.srt.entrypoints.openai.serving_completions import (
OpenAIServingCompletion,
)
Expand Down Expand Up @@ -424,17 +418,9 @@ def get_model_info(self) -> str:
return json.dumps(result, default=str)

def get_server_info(self) -> str:
result: Dict[str, Any] = self.tokenizer_manager.server_args.resolved_dict()
# `resolved_dict` answers with what resolution decided; the launch
# command answers with what was asked for, and the two are not
# derivable from each other. The HTTP and in-process readbacks both
# carry it, so this one does too.
result["launch_command"] = self.tokenizer_manager.server_args.launch_command
result.update(self.scheduler_info)
result["kv_events"] = describe_kv_events_publisher(
self.tokenizer_manager.server_args
return get_server_info_json(
self.tokenizer_manager.server_args, self.scheduler_info
)
return json.dumps(msgspec_to_builtins(result), default=str)

def health_check(self) -> bool:
from sglang.srt.managers.tokenizer_manager import ServerStatus
Expand Down
45 changes: 45 additions & 0 deletions python/sglang/srt/entrypoints/grpc_metadata.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
"""Server metadata shared by the full and follower-only native gRPC servers."""
Comment thread
jain-ria marked this conversation as resolved.
Resolved

import json
from typing import Any, Dict

from sglang.srt.runtime_context import describe_kv_events_publisher, get_serving
from sglang.srt.utils.msgspec_utils import msgspec_to_builtins


def get_server_info_json(server_args, scheduler_info: Dict[str, Any]) -> str:
Comment thread
jain-ria marked this conversation as resolved.
Resolved
"""Preserve the leader's server-info schema on both node roles.

``kv_event_sources`` comes from scheduler readiness and lists only actual
node-local publishers (an empty list means none). Each entry carries the
global DP rank, local subscription endpoint, topic and logical block size.
The existing ``kv_events`` descriptor remains unchanged for older clients;
its global DP count must not be used to infer node-local ownership.
``node_rank``, ``nnodes``, ``dp_size`` and ``dist_init_addr`` are already in
the resolved server args, including the shared engine rendezvous address.
"""
result = server_args.resolved_dict()
result["launch_command"] = server_args.launch_command
result.update(scheduler_info)
result["kv_events"] = describe_kv_events_publisher(server_args)
return json.dumps(msgspec_to_builtins(result), default=str)


def start_follower_grpc_server(server_args, scheduler_info: Dict[str, Any]):
"""Expose GetServerInfo only, without constructing a TokenizerManager.

The snapshot is taken after scheduler readiness. This does not launch a
sidecar or enable inference/control RPCs.
"""
serving = get_serving()
if serving.grpc_port is None or serving.smg_grpc_mode or serving.grpc_mode:
return None

from sglang.srt.rust_extensions import load_rust_extension

grpc_native = load_rust_extension("sglang.srt.rust_extensions._grpc")
return grpc_native.start_metadata_server(
host=serving.host,
port=serving.grpc_port,
server_info_json=get_server_info_json(server_args, scheduler_info),
)
Loading
Loading