Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 11 additions & 18 deletions python/sglang/srt/disaggregation/ascend/conn.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,10 @@

from sglang.srt.disaggregation.ascend.transfer_engine import AscendTransferEngine
from sglang.srt.disaggregation.base.conn import StateType
from sglang.srt.disaggregation.common.utils import group_concurrent_contiguous
from sglang.srt.disaggregation.common.utils import (
group_concurrent_contiguous,
submit_transfer_calls,
)
from sglang.srt.disaggregation.mooncake.conn import (
MooncakeKVBootstrapServer,
MooncakeKVManager,
Expand Down Expand Up @@ -197,28 +200,18 @@ def process_layers(layers_params: List[Tuple[int, int, int]]) -> int:
return self._transfer_data(mooncake_session_id, transfer_blocks)

if self.enable_custom_mem_pool:
futures = [
executor.submit(
process_layer,
src_ptr,
dst_ptr,
item_len,
)
for (src_ptr, dst_ptr, item_len) in layers_params
]
for future in concurrent.futures.as_completed(futures):
status = future.result()
if status != 0:
for f in futures:
f.cancel()
return status
return submit_transfer_calls(
executor,
[
(process_layer, (src_ptr, dst_ptr, item_len))
for (src_ptr, dst_ptr, item_len) in layers_params
],
)
else:
# Combining all layers' params in one batch transfer is more efficient
# compared to using multiple threads
return process_layers(layers_params)

return 0

def _is_generic_kvcache_state_type(self, st) -> bool:
# DSV4 per-pool components also use the page-indexed send path.
return (
Expand Down
56 changes: 56 additions & 0 deletions python/sglang/srt/disaggregation/base/conn.py
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,16 @@ class KVArgs:
total_kv_layers: int


class KVTransferBarrierEscalation(RuntimeError):
"""Raised when KV pages cannot be proven idle and must not be reused.

Deliberately distinct from a backend fault: the poll path contains ordinary
exceptions and releases the request, but this one has to reach the scheduler.
Restarting is the only way to reclaim these pages safely, and the scheduler's
top-level handler already turns an exception into engine teardown.
"""


class KVPoll:
Failed = 0
Bootstrapping = 1
Expand Down Expand Up @@ -180,6 +190,39 @@ def abort(self):
"""
pass

def advance_failure_quiescence(self) -> bool:
"""
Drive this request's transfer ownership barrier and report progress.

Mooncake-style backends hand raw KV pointers to native code (RDMA, a
transfer executor, CUDA staging copies) that outlives the Python call
which started it. A request that becomes terminal while that work runs
would let the allocator hand the same pages to another request.

Called on every poll of a request that has failed, before the scheduler
is allowed to observe ``KVPoll.Failed``. The first call closes the
barrier -- no new transfer work may be admitted for this request -- and
each call returns ``True`` once no native work can still touch its KV
pages. Backends without out-of-band transfer work keep the default.

Implementations must be idempotent, must not block the scheduler loop,
and must not fail silently: a transport or peer that never confirms
quiescence must be reported after a bounded wait, and must eventually
raise ``KVTransferBarrierEscalation`` rather than pin an unbounded
number of requests -- a restart reclaims every withheld page safely,
while releasing without proof can corrupt another request's KV.
"""
return True

def is_failure_quiescing(self) -> bool:
"""
Whether this request's ownership barrier has been closed.

Once it has, the request is logically failed on every rank, so all of
them converge on releasing it together.
"""
return False


class BaseKVReceiver(ABC):
@abstractmethod
Expand Down Expand Up @@ -239,6 +282,19 @@ def abort(self):
"""
pass

def advance_failure_quiescence(self) -> bool:
"""See ``BaseKVSender.advance_failure_quiescence``.

On the decode side the KV pages are the *destination* of the prefill's
writes, so quiescence additionally requires confirmation from every
prefill rank that was handed this request's page indices.
"""
return True

def is_failure_quiescing(self) -> bool:
"""See ``BaseKVSender.is_failure_quiescing``."""
return False


class BaseKVBootstrapServer(ABC):
@abstractmethod
Expand Down
26 changes: 20 additions & 6 deletions python/sglang/srt/disaggregation/common/conn.py
Original file line number Diff line number Diff line change
Expand Up @@ -786,13 +786,27 @@ def _connect(self, endpoint: str, is_ipv6: bool = False):
return sock

def _send_multipart_locked(
self, endpoint: str, parts: List[bytes], is_ipv6: bool = False
self,
endpoint: str,
parts: List[bytes],
is_ipv6: bool = False,
flags: int = 0,
):
# Cached sockets are shared across sender threads and zmq sockets are
# not thread-safe; serialize sends per endpoint.
sock = self._connect(endpoint, is_ipv6=is_ipv6)
with self._socket_send_locks[endpoint]:
sock.send_multipart(parts)
"""Send one message, serialized per endpoint.

Cached sockets are shared across sender threads and zmq sockets are not
thread-safe. *flags* accepts ``zmq.NOBLOCK`` for messages the caller
retries, so one unreachable peer cannot hold this lock for SNDTIMEO.
"""
# The endpoint lock must cover connection validation as well as send.
# Otherwise one thread can receive the cached socket from _connect(),
# then another can observe a disconnect and close/replace that socket
# before the first thread acquires this lock.
with self._socket_lock:
send_lock = self._socket_send_locks.setdefault(endpoint, threading.Lock())
with send_lock:
sock = self._connect(endpoint, is_ipv6=is_ipv6)
sock.send_multipart(parts, flags)

def get_mha_kv_ptrs_with_pp(
self, src_kv_ptrs: List[int], dst_kv_ptrs: List[int]
Expand Down
44 changes: 39 additions & 5 deletions python/sglang/srt/disaggregation/common/staging_handler.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
from typing import TYPE_CHECKING, List, Optional, Tuple

import torch
import zmq

logger = logging.getLogger(__name__)

Expand Down Expand Up @@ -94,6 +95,11 @@ def __init__(
# before unregister runs, but release_room still needs it.
self._room_to_receiver: dict = {}
self._wm_subscribers: dict = {}
# Subscriber keys whose last watermark send hit backpressure. Watermarks
# are cumulative, so resending the current value later is always correct
# -- but it has to actually happen, or a prefill waiting on staging space
# never learns the ring advanced.
self._wm_retry: set = set()
# room -> chunk_idx -> [(page_start, num_pages, writer_id)] fan-in
# arrivals; handler-owned so room teardown can purge them.
self._writer_counts: dict = {}
Expand Down Expand Up @@ -442,22 +448,50 @@ def _free_and_send_watermark(
) -> None:
"""Free a staging allocation and broadcast watermark to all prefills."""
self.staging_allocator.free(alloc_id)
post_wm = self.staging_allocator.get_watermark()
room = decode_req.req.bootstrap_room
self._broadcast_watermark(self.staging_allocator.get_watermark())

def retry_pending_watermarks(self) -> None:
"""Resend the current watermark to subscribers that were backpressured.

Driven from the scheduler loop next to advance_scatter, so a dropped
watermark costs one poll of latency instead of stalling a prefill that is
waiting for staging space which is already free.
"""
if not self._wm_retry:
return
self._broadcast_watermark(self.staging_allocator.get_watermark(), retry=True)

def _broadcast_watermark(self, post_wm, retry: bool = False) -> None:
wm_round, wm_tail = post_wm
wm_round_b = str(wm_round).encode("ascii")
wm_tail_b = str(wm_tail).encode("ascii")
for _key, (receiver, session_id) in list(self._wm_subscribers.items()):
if retry:
targets = [
(key, entry)
for key, entry in list(self._wm_subscribers.items())
if key in self._wm_retry
]
else:
targets = list(self._wm_subscribers.items())
self._wm_retry.difference_update(key for key, _entry in targets)

for key, (receiver, session_id) in targets:
sid_b = session_id.encode("ascii")
for bootstrap_info in receiver.bootstrap_infos:
# Reached from the scheduler loop (advance_scatter, and
# release_room on every teardown) over sockets that set no ZMQ
# send timeout, so it must never wait for a peer.
try:
sock, lock = receiver._connect_to_bootstrap_server(bootstrap_info)
with lock:
sock.send_multipart(
[b"WATERMARK", wm_round_b, wm_tail_b, sid_b]
[b"WATERMARK", wm_round_b, wm_tail_b, sid_b], zmq.NOBLOCK
)
except Exception:
pass
# Retry the current value later rather than dropping it: this
# may be the last release, with no later broadcast to
# supersede it.
self._wm_retry.add(key)


def is_watermark_ready(
Expand Down
65 changes: 64 additions & 1 deletion python/sglang/srt/disaggregation/common/utils.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
import concurrent.futures
import ctypes
import dataclasses
import struct
import threading
from collections import deque
from typing import List, Optional, Tuple, Union
from typing import Callable, List, Optional, Sequence, Tuple, Union

import numpy as np
import numpy.typing as npt
Expand All @@ -25,6 +26,10 @@ class TransferKVChunk:
prefill_aux_index: Optional[int]
state_indices: Optional[List]
chunk_id: Optional[int] = None
# Identity of the request that queued this chunk, for backends that must not
# let a recycled bootstrap_room attach queued work to a new request. Opaque
# to the queue; see MooncakeKVManager.try_lease_chunk.
owner: Optional[object] = None
num_kv_tokens: Optional[int] = None
trace_ctx: Union[TraceReqContext, TraceNullContext] = dataclasses.field(
default_factory=TraceNullContext
Expand Down Expand Up @@ -133,6 +138,64 @@ def group_concurrent_contiguous(
return src_groups, dst_groups


def drain_transfer_futures(futures: Sequence[concurrent.futures.Future]) -> int:
"""Wait for every transfer future to leave the running state.

Returns the first non-zero transfer status, or re-raises the first
exception. Unlike a plain ``as_completed`` loop that returns on the first
error, this never returns while a sibling future may still be reading or
writing KV pages: ``Future.cancel()`` is a no-op once a future is running,
so returning early would let the caller release pages that are still being
transferred.
"""
first_status = 0
first_exception = None
for future in concurrent.futures.as_completed(futures):
try:
status = future.result()
except concurrent.futures.CancelledError:
continue
except BaseException as e: # noqa: BLE001 - re-raised after draining
if first_exception is None:
first_exception = e
_cancel_pending(futures)
continue
if status != 0 and first_status == 0:
first_status = status
_cancel_pending(futures)
# as_completed() already yielded every future, so this only reaps the ones
# cancelled above; it is what makes the "no work in flight" guarantee hold.
concurrent.futures.wait(futures)
if first_exception is not None:
raise first_exception
return first_status


def submit_transfer_calls(
executor: concurrent.futures.Executor,
calls: Sequence[Tuple[Callable[..., int], tuple]],
) -> int:
"""Submit transfer work and drain it via ``drain_transfer_futures``.

A failure part-way through submission still drains the futures that were
accepted, so the caller never regains KV page ownership early.
"""
futures: List[concurrent.futures.Future] = []
try:
for fn, args in calls:
futures.append(executor.submit(fn, *args))
except BaseException:
_cancel_pending(futures)
concurrent.futures.wait(futures)
raise
return drain_transfer_futures(futures)


def _cancel_pending(futures: Sequence[concurrent.futures.Future]) -> None:
for future in futures:
future.cancel()


@dataclasses.dataclass(frozen=True)
class DCPTokenTransferPlan:
src_token_indices: npt.NDArray[np.int64]
Expand Down
19 changes: 19 additions & 0 deletions python/sglang/srt/disaggregation/decode.py
Original file line number Diff line number Diff line change
Expand Up @@ -791,6 +791,10 @@ def _update_handshake_waiters(

if poll == KVPoll.Bootstrapping:
pass
elif poll == KVPoll.Transferring:
# A backend can defer a terminal transition while it releases
# ownership of the request's KV pages; keep waiting.
pass
elif poll == KVPoll.WaitingForInput:
decode_req.waiting_for_input = True
decode_req.req.time_stats.set_bootstrap_done_time()
Expand Down Expand Up @@ -2040,6 +2044,21 @@ def pop_transferred(self, rids_to_check: Optional[List[str]] = None) -> List[Req
continue

hicache_restore_status = decode_req.hicache_restore_status
if (
hicache_restore_status == HiCacheRestoreResult.FAILED
and poll != KVPoll.Failed
):
# The restore failed, but that says nothing about the KV transfer:
# a prefill rank may still be writing into this request's pages.
# Fail it through the same ownership barrier as a transfer failure
# instead of releasing the pages here, and pick it up below once
# the barrier reports Failed.
decode_req.kv_receiver.abort()
decode_req.kv_receiver.kv_mgr.record_failure(
decode_req.req.bootstrap_room,
"HiCache restore failed for this request",
)
continue
if (
poll == KVPoll.Failed
or hicache_restore_status == HiCacheRestoreResult.FAILED
Expand Down
8 changes: 8 additions & 0 deletions python/sglang/srt/disaggregation/decode_hicache_mixin.py
Original file line number Diff line number Diff line change
Expand Up @@ -166,6 +166,14 @@ def poll(self) -> KVPoll:
return KVPoll.Transferring
return poll

# The transfer ownership barrier belongs to the wrapped receiver, which owns
# the KV pages; this wrapper only gates Success on HiCache restore.
def advance_failure_quiescence(self) -> bool:
return self.decode_req.kv_receiver.advance_failure_quiescence()

def is_failure_quiescing(self) -> bool:
return self.decode_req.kv_receiver.is_failure_quiescing()


class DecodeHiCacheTransferMixin:
"""HiCache hooks for ``DecodeTransferQueue``: drive restore state machine."""
Expand Down
Loading
Loading