From 91a02e1f35f9630bf5ec5ddbd316f4b5b076ccaf Mon Sep 17 00:00:00 2001 From: Kastan Day Date: Tue, 28 Jul 2026 09:44:29 -0700 Subject: [PATCH 01/10] fix(disaggregation): don't release KV pages while Mooncake transfers run --- .../sglang/srt/disaggregation/ascend/conn.py | 29 +- python/sglang/srt/disaggregation/base/conn.py | 54 + .../sglang/srt/disaggregation/common/conn.py | 19 + .../disaggregation/common/staging_handler.py | 44 +- .../sglang/srt/disaggregation/common/utils.py | 65 +- python/sglang/srt/disaggregation/decode.py | 19 + .../disaggregation/decode_hicache_mixin.py | 8 + .../srt/disaggregation/mooncake/conn.py | 1109 ++++++++++-- python/sglang/srt/disaggregation/prefill.py | 8 +- python/sglang/srt/disaggregation/utils.py | 104 +- python/sglang/srt/environ.py | 29 + python/sglang/srt/managers/scheduler.py | 23 +- .../test_transfer_ownership_barrier.py | 1608 +++++++++++++++++ 13 files changed, 2952 insertions(+), 167 deletions(-) create mode 100644 test/registered/unit/disaggregation/test_transfer_ownership_barrier.py diff --git a/python/sglang/srt/disaggregation/ascend/conn.py b/python/sglang/srt/disaggregation/ascend/conn.py index 2162a49bd04c..80840b3a915b 100644 --- a/python/sglang/srt/disaggregation/ascend/conn.py +++ b/python/sglang/srt/disaggregation/ascend/conn.py @@ -8,7 +8,10 @@ from sglang.srt.disaggregation.ascend.transfer_engine import AscendTransferEngine from sglang.srt.disaggregation.base.conn import StateType -from sglang.srt.disaggregation.common.utils import group_concurrent_contiguous +from sglang.srt.disaggregation.common.utils import ( + group_concurrent_contiguous, + submit_transfer_calls, +) from sglang.srt.disaggregation.mooncake.conn import ( MooncakeKVBootstrapServer, MooncakeKVManager, @@ -186,28 +189,18 @@ def process_layers(layers_params: List[Tuple[int, int, int]]) -> int: return self._transfer_data(mooncake_session_id, transfer_blocks) if self.enable_custom_mem_pool: - futures = [ - executor.submit( - process_layer, - src_ptr, - dst_ptr, - item_len, - ) - for (src_ptr, dst_ptr, item_len) in layers_params - ] - for future in concurrent.futures.as_completed(futures): - status = future.result() - if status != 0: - for f in futures: - f.cancel() - return status + return submit_transfer_calls( + executor, + [ + (process_layer, (src_ptr, dst_ptr, item_len)) + for (src_ptr, dst_ptr, item_len) in layers_params + ], + ) else: # Combining all layers' params in one batch transfer is more efficient # compared to using multiple threads return process_layers(layers_params) - return 0 - def _is_generic_kvcache_state_type(self, st) -> bool: # DSV4 per-pool components also use the page-indexed send path. return ( diff --git a/python/sglang/srt/disaggregation/base/conn.py b/python/sglang/srt/disaggregation/base/conn.py index 2d028a629f7e..6a5070244f04 100644 --- a/python/sglang/srt/disaggregation/base/conn.py +++ b/python/sglang/srt/disaggregation/base/conn.py @@ -85,6 +85,16 @@ class KVArgs: total_kv_layers: int +class KVTransferBarrierEscalation(RuntimeError): + """Raised when KV pages cannot be proven idle and must not be reused. + + Deliberately distinct from a backend fault: the poll path contains ordinary + exceptions and releases the request, but this one has to reach the scheduler. + Restarting is the only way to reclaim these pages safely, and the scheduler's + top-level handler already turns an exception into engine teardown. + """ + + class KVPoll: Failed = 0 Bootstrapping = 1 @@ -178,6 +188,37 @@ def abort(self): """ pass + def advance_failure_quiescence(self) -> bool: + """ + Drive this request's transfer ownership barrier and report progress. + + Mooncake-style backends hand raw KV pointers to native code (RDMA, a + transfer executor, CUDA staging copies) that outlives the Python call + which started it. A request that becomes terminal while that work runs + would let the allocator hand the same pages to another request. + + Called on every poll of a request that has failed, before the scheduler + is allowed to observe ``KVPoll.Failed``. The first call closes the + barrier -- no new transfer work may be admitted for this request -- and + each call returns ``True`` once no native work can still touch its KV + pages. Backends without out-of-band transfer work keep the default. + + Implementations must be idempotent, must not block the scheduler loop, + and must be *bounded*: a transport or peer that never confirms + quiescence has to eventually return ``True`` rather than pinning the + request, and its pages, forever. + """ + return True + + def is_failure_quiescing(self) -> bool: + """ + Whether this request's ownership barrier has been closed. + + Once it has, the request is logically failed on every rank, so all of + them converge on releasing it together. + """ + return False + class BaseKVReceiver(ABC): @abstractmethod @@ -237,6 +278,19 @@ def abort(self): """ pass + def advance_failure_quiescence(self) -> bool: + """See ``BaseKVSender.advance_failure_quiescence``. + + On the decode side the KV pages are the *destination* of the prefill's + writes, so quiescence additionally requires confirmation from every + prefill rank that was handed this request's page indices. + """ + return True + + def is_failure_quiescing(self) -> bool: + """See ``BaseKVSender.is_failure_quiescing``.""" + return False + class BaseKVBootstrapServer(ABC): @abstractmethod diff --git a/python/sglang/srt/disaggregation/common/conn.py b/python/sglang/srt/disaggregation/common/conn.py index a7bfcd438baf..15510fdfffbe 100644 --- a/python/sglang/srt/disaggregation/common/conn.py +++ b/python/sglang/srt/disaggregation/common/conn.py @@ -202,6 +202,7 @@ def __init__( self.request_status: Dict[int, KVPoll] = {} self._socket_cache: Dict[str, zmq.Socket] = {} self._monitor_cache: Dict[str, zmq.Socket] = {} + self._socket_send_locks: Dict[str, threading.Lock] = {} self._socket_lock = threading.Lock() self.failure_records: Dict[int, str] = {} self.failure_lock = threading.Lock() @@ -726,8 +727,26 @@ def _connect(self, endpoint: str, is_ipv6: bool = False): self._monitor_cache[endpoint] = sock.get_monitor_socket( zmq.EVENT_DISCONNECTED ) + self._socket_send_locks.setdefault(endpoint, threading.Lock()) return sock + def _send_multipart_locked( + self, + endpoint: str, + parts: List[bytes], + is_ipv6: bool = False, + flags: int = 0, + ): + """Send one message, serialized per endpoint. + + Cached sockets are shared across sender threads and zmq sockets are not + thread-safe. *flags* accepts ``zmq.NOBLOCK`` for messages the caller + retries, so one unreachable peer cannot hold this lock for SNDTIMEO. + """ + sock = self._connect(endpoint, is_ipv6=is_ipv6) + with self._socket_send_locks[endpoint]: + sock.send_multipart(parts, flags) + def get_mha_kv_ptrs_with_pp( self, src_kv_ptrs: List[int], dst_kv_ptrs: List[int] ) -> Tuple[List[int], List[int], List[int], List[int], int]: diff --git a/python/sglang/srt/disaggregation/common/staging_handler.py b/python/sglang/srt/disaggregation/common/staging_handler.py index b2ef46ad1620..4bbaa3156a54 100644 --- a/python/sglang/srt/disaggregation/common/staging_handler.py +++ b/python/sglang/srt/disaggregation/common/staging_handler.py @@ -15,6 +15,7 @@ from typing import TYPE_CHECKING, List, Optional, Tuple import torch +import zmq logger = logging.getLogger(__name__) @@ -87,6 +88,11 @@ def __init__( # before unregister runs, but release_room still needs it. self._room_to_receiver: dict = {} self._wm_subscribers: dict = {} + # Subscriber keys whose last watermark send hit backpressure. Watermarks + # are cumulative, so resending the current value later is always correct + # -- but it has to actually happen, or a prefill waiting on staging space + # never learns the ring advanced. + self._wm_retry: set = set() def register_wm_subscriber(self, receiver, session_id: str) -> None: """Register a prefill's bootstrap connection for watermark broadcasts.""" @@ -419,22 +425,50 @@ def _free_and_send_watermark( ) -> None: """Free a staging allocation and broadcast watermark to all prefills.""" self.staging_allocator.free(alloc_id) - post_wm = self.staging_allocator.get_watermark() - room = decode_req.req.bootstrap_room + self._broadcast_watermark(self.staging_allocator.get_watermark()) + + def retry_pending_watermarks(self) -> None: + """Resend the current watermark to subscribers that were backpressured. + + Driven from the scheduler loop next to advance_scatter, so a dropped + watermark costs one poll of latency instead of stalling a prefill that is + waiting for staging space which is already free. + """ + if not self._wm_retry: + return + self._broadcast_watermark(self.staging_allocator.get_watermark(), retry=True) + + def _broadcast_watermark(self, post_wm, retry: bool = False) -> None: wm_round, wm_tail = post_wm wm_round_b = str(wm_round).encode("ascii") wm_tail_b = str(wm_tail).encode("ascii") - for _key, (receiver, session_id) in list(self._wm_subscribers.items()): + if retry: + targets = [ + (key, entry) + for key, entry in list(self._wm_subscribers.items()) + if key in self._wm_retry + ] + else: + targets = list(self._wm_subscribers.items()) + self._wm_retry.difference_update(key for key, _entry in targets) + + for key, (receiver, session_id) in targets: sid_b = session_id.encode("ascii") for bootstrap_info in receiver.bootstrap_infos: + # Reached from the scheduler loop (advance_scatter, and + # release_room on every teardown) over sockets that set no ZMQ + # send timeout, so it must never wait for a peer. try: sock, lock = receiver._connect_to_bootstrap_server(bootstrap_info) with lock: sock.send_multipart( - [b"WATERMARK", wm_round_b, wm_tail_b, sid_b] + [b"WATERMARK", wm_round_b, wm_tail_b, sid_b], zmq.NOBLOCK ) except Exception: - pass + # Retry the current value later rather than dropping it: this + # may be the last release, with no later broadcast to + # supersede it. + self._wm_retry.add(key) def is_watermark_ready( diff --git a/python/sglang/srt/disaggregation/common/utils.py b/python/sglang/srt/disaggregation/common/utils.py index e4571f2d96fe..953ee731d2d9 100644 --- a/python/sglang/srt/disaggregation/common/utils.py +++ b/python/sglang/srt/disaggregation/common/utils.py @@ -1,9 +1,10 @@ +import concurrent.futures import ctypes import dataclasses import struct import threading from collections import deque -from typing import List, Optional, Tuple, Union +from typing import Callable, List, Optional, Sequence, Tuple, Union import numpy as np import numpy.typing as npt @@ -25,6 +26,10 @@ class TransferKVChunk: prefill_aux_index: Optional[int] state_indices: Optional[List] chunk_id: Optional[int] = None + # Identity of the request that queued this chunk, for backends that must not + # let a recycled bootstrap_room attach queued work to a new request. Opaque + # to the queue; see MooncakeKVManager.try_lease_chunk. + owner: Optional[object] = None trace_ctx: Union[TraceReqContext, TraceNullContext] = dataclasses.field( default_factory=TraceNullContext ) @@ -127,3 +132,61 @@ def group_concurrent_contiguous( dst_groups = [g.tolist() for g in dst_groups] return src_groups, dst_groups + + +def drain_transfer_futures(futures: Sequence[concurrent.futures.Future]) -> int: + """Wait for every transfer future to leave the running state. + + Returns the first non-zero transfer status, or re-raises the first + exception. Unlike a plain ``as_completed`` loop that returns on the first + error, this never returns while a sibling future may still be reading or + writing KV pages: ``Future.cancel()`` is a no-op once a future is running, + so returning early would let the caller release pages that are still being + transferred. + """ + first_status = 0 + first_exception = None + for future in concurrent.futures.as_completed(futures): + try: + status = future.result() + except concurrent.futures.CancelledError: + continue + except BaseException as e: # noqa: BLE001 - re-raised after draining + if first_exception is None: + first_exception = e + _cancel_pending(futures) + continue + if status != 0 and first_status == 0: + first_status = status + _cancel_pending(futures) + # as_completed() already yielded every future, so this only reaps the ones + # cancelled above; it is what makes the "no work in flight" guarantee hold. + concurrent.futures.wait(futures) + if first_exception is not None: + raise first_exception + return first_status + + +def submit_transfer_calls( + executor: concurrent.futures.Executor, + calls: Sequence[Tuple[Callable[..., int], tuple]], +) -> int: + """Submit transfer work and drain it via ``drain_transfer_futures``. + + A failure part-way through submission still drains the futures that were + accepted, so the caller never regains KV page ownership early. + """ + futures: List[concurrent.futures.Future] = [] + try: + for fn, args in calls: + futures.append(executor.submit(fn, *args)) + except BaseException: + _cancel_pending(futures) + concurrent.futures.wait(futures) + raise + return drain_transfer_futures(futures) + + +def _cancel_pending(futures: Sequence[concurrent.futures.Future]) -> None: + for future in futures: + future.cancel() diff --git a/python/sglang/srt/disaggregation/decode.py b/python/sglang/srt/disaggregation/decode.py index bbb30ef8baa4..431e7b973b48 100644 --- a/python/sglang/srt/disaggregation/decode.py +++ b/python/sglang/srt/disaggregation/decode.py @@ -749,6 +749,10 @@ def _update_handshake_waiters( if poll == KVPoll.Bootstrapping: pass + elif poll == KVPoll.Transferring: + # A backend can defer a terminal transition while it releases + # ownership of the request's KV pages; keep waiting. + pass elif poll == KVPoll.WaitingForInput: decode_req.waiting_for_input = True decode_req.req.time_stats.set_bootstrap_done_time() @@ -1937,6 +1941,21 @@ def pop_transferred(self, rids_to_check: Optional[List[str]] = None) -> List[Req continue hicache_restore_status = decode_req.hicache_restore_status + if ( + hicache_restore_status == HiCacheRestoreResult.FAILED + and poll != KVPoll.Failed + ): + # The restore failed, but that says nothing about the KV transfer: + # a prefill rank may still be writing into this request's pages. + # Fail it through the same ownership barrier as a transfer failure + # instead of releasing the pages here, and pick it up below once + # the barrier reports Failed. + decode_req.kv_receiver.abort() + decode_req.kv_receiver.kv_mgr.record_failure( + decode_req.req.bootstrap_room, + "HiCache restore failed for this request", + ) + continue if ( poll == KVPoll.Failed or hicache_restore_status == HiCacheRestoreResult.FAILED diff --git a/python/sglang/srt/disaggregation/decode_hicache_mixin.py b/python/sglang/srt/disaggregation/decode_hicache_mixin.py index fbdc08d64c8a..f89b652db605 100644 --- a/python/sglang/srt/disaggregation/decode_hicache_mixin.py +++ b/python/sglang/srt/disaggregation/decode_hicache_mixin.py @@ -166,6 +166,14 @@ def poll(self) -> KVPoll: return KVPoll.Transferring return poll + # The transfer ownership barrier belongs to the wrapped receiver, which owns + # the KV pages; this wrapper only gates Success on HiCache restore. + def advance_failure_quiescence(self) -> bool: + return self.decode_req.kv_receiver.advance_failure_quiescence() + + def is_failure_quiescing(self) -> bool: + return self.decode_req.kv_receiver.is_failure_quiescing() + class DecodeHiCacheTransferMixin: """HiCache hooks for ``DecodeTransferQueue``: drive restore state machine.""" diff --git a/python/sglang/srt/disaggregation/mooncake/conn.py b/python/sglang/srt/disaggregation/mooncake/conn.py index 3a92fd2a5865..0e3fa69d6eed 100644 --- a/python/sglang/srt/disaggregation/mooncake/conn.py +++ b/python/sglang/srt/disaggregation/mooncake/conn.py @@ -2,20 +2,27 @@ import concurrent.futures import dataclasses +import itertools import logging import os +import queue import struct import threading import time -from collections import defaultdict -from typing import List, Optional, Tuple, Union +from collections import OrderedDict, defaultdict +from typing import Dict, List, Optional, Tuple, Union import numpy as np import numpy.typing as npt import zmq from prometheus_client import Counter -from sglang.srt.disaggregation.base.conn import KVArgs, KVPoll, StateType +from sglang.srt.disaggregation.base.conn import ( + KVArgs, + KVPoll, + KVTransferBarrierEscalation, + StateType, +) from sglang.srt.disaggregation.common.conn import ( CommonKVBootstrapServer, CommonKVManager, @@ -36,6 +43,7 @@ TransferKVChunk, group_concurrent_contiguous, pack_int_lists, + submit_transfer_calls, unpack_int_lists, ) from sglang.srt.disaggregation.mooncake.utils import ( @@ -47,7 +55,7 @@ compute_mamba_state_slice_byte_blocks, ) from sglang.srt.distributed.parallel_state import get_mooncake_transfer_engine -from sglang.srt.environ import envs +from sglang.srt.environ import TransferBarrierLevel, envs from sglang.srt.observability.mooncake_trace import ( MooncakeRequestStage, mooncake_trace_func, @@ -68,6 +76,164 @@ "Number of mooncake_session_ids un-blacklisted via probe.", ) +TRANSFER_QUIESCE_TIMEOUTS = Counter( + "sglang:kv_transfer_quiesce_timeouts_total", + "Requests released while KV transfer work may still have been in flight.", +) + +# How often a decode rank re-sends an unacknowledged abort notification. +ABORT_RETRY_INTERVAL_S = 0.5 +# How often the bookkeeping thread re-checks drained rooms and sweeps lifetimes. +TRANSFER_BOOKKEEPING_INTERVAL_S = 0.01 +# How often that thread runs the (cheap) room-lifetime sweep. +ROOM_SWEEP_INTERVAL_S = 1.0 +# How long an ABORT_ACK may stay owed to a peer that is not reading its socket. +ABORT_ACK_MAX_AGE_S = 60.0 +# Hard cap on owed ABORT_ACKs, so an abort storm against a dead peer cannot grow +# the pending list, or the per-tick sweep over it, without bound. +MAX_PENDING_ABORT_ACKS = 4096 +# Soft cap on tracked room lifetimes. Reaching it means the periodic sweep is not +# keeping up, so an insert also does a *bounded* scan for reclaimable rooms. +MAX_TRACKED_ROOMS = 4096 +# Entries examined per emergency scan, so an insert can never become O(n). +MAX_EMERGENCY_SCAN = 256 + + +class RoomTransferLifetime: + """Ownership barrier for one bootstrap room's KV pages on this rank. + + Mooncake transfers hand raw KV pointers to native code (RDMA, the transfer + executor, CUDA staging copies). Those readers and writers outlive the + Python call that started them, so a request that becomes terminal while + they run would let the allocator hand the same pages to another request. + + Every piece of native transfer work holds a *lease*. ``close()`` stops new + leases from being handed out; the room is *quiesced* once it is closed and + every outstanding lease has been returned. Only then is it safe to release + the room's KV pages. + + Abort tokens minted by decode peers are recorded here so that a late or + duplicated abort for a recycled room cannot close a live room. + """ + + __slots__ = ("_cond", "_leases", "_open", "_abort_tokens", "created_at", "_claimed") + + def __init__(self) -> None: + self._cond = threading.Condition() + self._leases = 0 + self._open = True + self._abort_tokens: set = set() + self.created_at = time.monotonic() + # True once a local sender has taken responsibility for the room. An + # unclaimed room was created by decode metadata alone, so nothing will + # ever release it and the sweep must. + self._claimed = False + + def try_lease(self) -> bool: + """Take a lease, or return False if the room no longer admits work.""" + with self._cond: + if not self._open: + return False + self._leases += 1 + return True + + def end_lease(self) -> None: + with self._cond: + self._leases -= 1 + if self._leases == 0: + self._cond.notify_all() + + def close(self) -> None: + """Stop admitting transfer work (idempotent).""" + with self._cond: + self._open = False + if self._leases == 0: + self._cond.notify_all() + + def is_closed(self) -> bool: + with self._cond: + return not self._open + + def is_quiesced(self) -> bool: + with self._cond: + return not self._open and self._leases == 0 + + def outstanding_leases(self) -> int: + with self._cond: + return self._leases + + def claim(self) -> None: + with self._cond: + self._claimed = True + + def is_claimed(self) -> bool: + with self._cond: + return self._claimed + + def is_reclaimable(self) -> bool: + """Whether forgetting this room can no longer affect a live transfer. + + Safe in both cases: a quiesced room admits no work and has none running, + and an unclaimed room has no sender, so it can never take a lease (only + ``add_transfer_request`` enqueues work, and only a sender calls it). + """ + with self._cond: + return (not self._open and self._leases == 0) or not self._claimed + + def wait_quiesced(self, timeout: Optional[float]) -> bool: + """Block until quiesced. Returns False if *timeout* elapsed first.""" + with self._cond: + return self._cond.wait_for( + lambda: not self._open and self._leases == 0, timeout + ) + + def add_abort_token(self, token: bytes) -> None: + if not token: + return + with self._cond: + self._abort_tokens.add(token) + + def authorizes_abort(self, token: bytes) -> bool: + """Whether *token* may close this room. + + A tokenless abort comes from a peer that predates the token protocol and + is honoured unconditionally, as before. + + A room that has not received any decode metadata yet has no tokens to + compare against, and accepts the abort. That is deliberate, and it is the + safe direction of the trade: an abort can legitimately arrive before this + rank has any metadata for the room (the decode gave up during bootstrap), + and dropping it would leave this rank free to transfer into pages the + decode has already released -- the corruption this barrier exists to + prevent. Accepting instead risks one spurious request failure if a + recycled bootstrap_room draws a delayed abort from its previous occupant, + which is availability rather than correctness, and requires a collision in + a 64-bit space. Distinguishing the two cases would need a generation tag + on every room-scoped message; that is not worth a wire change here. + """ + if not token: + return True + with self._cond: + return not self._abort_tokens or token in self._abort_tokens + + +@dataclasses.dataclass +class _PendingAbortAck: + """An ABORT_ACK owed to a decode rank once its room has drained.""" + + room: int + endpoint: str + dst_port: int + token: bytes + lifetime: Optional[RoomTransferLifetime] + # Give up after this: a peer that has not drained its socket for this long is + # gone, and it has its own barrier deadline. Retrying forever would grow the + # pending list, and the sweep over it, without bound. + deadline: float = 0.0 + + def key(self) -> Tuple[int, str, int, bytes]: + return (self.room, self.endpoint, self.dst_port, self.token) + # decode @dataclasses.dataclass @@ -82,6 +248,9 @@ class TransferInfo: required_dst_info_num: int is_dummy: bool decode_prefix_len: Optional[int] = None + # Nonce this decode rank will require us to echo in ABORT_ACK. Empty when the + # peer predates the ownership-barrier protocol. + abort_token: bytes = b"" # Note: always put the optional staging field at the final (it will be set through 'STAGING_RSP' pkg when needed) staging: Optional[StagingTransferInfo] = None @@ -110,6 +279,7 @@ def from_zmq(cls, msg: List[bytes]): decode_prefix_len=( int(msg[8].decode("ascii")) if len(msg) > 8 and msg[8] != b"" else None ), + abort_token=msg[9] if len(msg) > 9 else b"", ) @@ -179,11 +349,26 @@ def __init__( is_mla_backend: Optional[bool] = False, ): super().__init__(args, disaggregation_mode, server_args, is_mla_backend) + self.transfer_barrier = TransferBarrierLevel( + envs.SGLANG_DISAGGREGATION_TRANSFER_BARRIER.get() + ) + # WARN releases after this long without proof; STRICT never does. + self.quiesce_timeout = envs.SGLANG_DISAGGREGATION_QUIESCE_TIMEOUT.get() + self.max_unquiesced = envs.SGLANG_DISAGGREGATION_MAX_UNQUIESCED.get() + self._unquiesced_rooms: set = set() + self._unquiesced_lock = threading.Lock() self.init_engine() self.register_buffer_to_engine() self.enable_staging = envs.SGLANG_DISAGG_STAGING_BUFFER.get() self.enable_trace = server_args.enable_trace if self.disaggregation_mode == DisaggregationMode.PREFILL: + # room -> ownership barrier for that room's KV pages. Shares its + # lifecycle with self.transfer_infos: created when the room is first + # seen (by the sender or by decode metadata, whichever comes first) + # and dropped by MooncakeKVSender.clear(). + self._room_lifetimes: OrderedDict[int, RoomTransferLifetime] = OrderedDict() + self._room_lifetimes_lock = threading.Lock() + self._start_transfer_bookkeeping() self.start_prefill_thread() self.session_failures = defaultdict(int) self.failed_sessions = set() @@ -246,6 +431,10 @@ def __init__( daemon=True, ).start() elif self.disaggregation_mode == DisaggregationMode.DECODE: + # room -> live receiver. Used to route ABORT_ACKs and to drop late + # transport messages for rooms that are tearing down. + self._receivers: Dict[int, MooncakeKVReceiver] = {} + self._receivers_lock = threading.Lock() self._staging_ctx = DecodeStagingContext() if self.enable_staging else None if self.enable_staging: self._init_staging_allocator() @@ -253,6 +442,415 @@ def __init__( self._chunk_writer_counts: dict = defaultdict(lambda: defaultdict(list)) self.start_decode_thread() + # ------------------------------------------------------------------ + # Transfer ownership barrier (prefill side) + # ------------------------------------------------------------------ + + def _room_lifetime( + self, room: int, *, create: bool + ) -> Optional[RoomTransferLifetime]: + with self._room_lifetimes_lock: + lifetime = self._room_lifetimes.get(room) + if lifetime is None and create: + # Reclaim before inserting, so the entry being created can never + # be the one that gets evicted. + if len(self._room_lifetimes) >= MAX_TRACKED_ROOMS: + self._emergency_reclaim_rooms_locked() + lifetime = self._room_lifetimes[room] = RoomTransferLifetime() + return lifetime + + def _retire_rooms_locked(self, rooms: List[int]) -> None: + """Forget every piece of state keyed by these rooms, atomically. + + Must happen under ``_room_lifetimes_lock`` and in one step: dropping the + lifetime first and the rest afterwards would let a request that draws the + same bootstrap_room in between have its own destinations and status + deleted, and leaving them behind would let it inherit the old decode's + addresses. + """ + for room in rooms: + del self._room_lifetimes[room] + self.transfer_infos.pop(room, None) + self.req_to_decode_prefix_len.pop(room, None) + self.request_status.pop(room, None) + + def _emergency_reclaim_rooms_locked(self) -> None: + """Bounded reclaim when the periodic sweep is not keeping up. + + Scans at most ``MAX_EMERGENCY_SCAN`` of the oldest entries so an insert + can never degrade to O(tracked rooms); the sweep does the rest. + """ + # islice over the live view keeps this O(MAX_EMERGENCY_SCAN); the keys are + # copied out first because the dict cannot be mutated while iterating. + candidates = [ + room + for room, lifetime in itertools.islice( + self._room_lifetimes.items(), MAX_EMERGENCY_SCAN + ) + if lifetime.is_reclaimable() + ] + self._retire_rooms_locked(candidates) + if not candidates: + logger.warning_once( + "Tracking more than %d Mooncake bootstrap rooms with none " + "reclaimable; KV metadata may be arriving for rooms this rank " + "never serves.", + MAX_TRACKED_ROOMS, + ) + + def _sweep_room_lifetimes(self) -> int: + """Reclaim rooms that no local sender will ever release. + + Entries younger than the bootstrap timeout are always kept: that is the + window in which a sender may still appear and claim the room, and in + which a tombstone must keep rejecting a late abort. After it, a quiesced + or never-claimed room is dead weight. + """ + cutoff = time.monotonic() - self._room_sweep_ttl + with self._room_lifetimes_lock: + stale = [ + room + for room, lifetime in self._room_lifetimes.items() + if lifetime.created_at <= cutoff and lifetime.is_reclaimable() + ] + self._retire_rooms_locked(stale) + if stale: + logger.debug("Reclaimed %d abandoned Mooncake rooms", len(stale)) + return len(stale) + + def _forget_room_lifetime(self, room: int) -> None: + with self._room_lifetimes_lock: + self._room_lifetimes.pop(room, None) + + def try_lease_room(self, room: int) -> Optional[RoomTransferLifetime]: + """Take a transfer lease on *room*, or None if it admits no more work. + + Deliberately does not create a missing lifetime: an absent room has + already been released, so admitting work would reintroduce the + use-after-free this barrier exists to prevent. Callers must return the + lease with ``end_lease()`` from a ``finally`` block. + """ + lifetime = self._room_lifetime(room, create=False) + if lifetime is None or not lifetime.try_lease(): + return None + return lifetime + + def try_lease_chunk( + self, kv_chunk: TransferKVChunk + ) -> Optional[RoomTransferLifetime]: + """Lease the room for *kv_chunk*, matching on request identity. + + A bootstrap_room is recycled, so a chunk that outlived its request must + not attach to whichever request holds that room now: its indices name the + old request's pages and its destinations the old decode's. The chunk + carries the lifetime it was queued against, and only that exact object + may be leased. + """ + owner = kv_chunk.owner + lifetime = self._room_lifetime(kv_chunk.room, create=False) + if lifetime is None or (owner is not None and owner is not lifetime): + return None + if not lifetime.try_lease(): + return None + return lifetime + + def open_room_transfers(self, room: int) -> bool: + """Claim *room*'s ownership barrier for a local sender. + + Returns False when the room has already been closed, which happens when + a decode abort overtakes this rank's sender: transferring then would + write into pages the decode instance has released. + """ + lifetime = self._room_lifetime(room, create=True) + lifetime.claim() + return not lifetime.is_closed() + + def close_room_transfers(self, room: int) -> Optional[RoomTransferLifetime]: + """Stop admitting transfers for *room* (no-op if already released).""" + lifetime = self._room_lifetime(room, create=False) + if lifetime is not None: + lifetime.close() + return lifetime + + def _close_room_for_abort( + self, room: int, token: bytes + ) -> Optional[RoomTransferLifetime]: + """Close *room* on behalf of a decode abort, if the token authorizes it. + + The lifetime is created when missing so that an abort which overtakes + the prefill sender still leaves a tombstone; the sender then fails the + request instead of transferring into pages the decode has released. + """ + lifetime = self._room_lifetime(room, create=False) + if lifetime is not None and not lifetime.authorizes_abort(token): + logger.debug( + "Ignoring abort for room %s: token was not issued for this room", + room, + ) + return None + if lifetime is None: + lifetime = self._room_lifetime(room, create=True) + lifetime.close() + return lifetime + + def _start_transfer_bookkeeping(self) -> None: + """Start the single thread that owns deferred ABORT_ACKs and room GC. + + An ABORT_ACK promises the decode rank that this rank can no longer touch + the room's pages, so it may only be sent once the room has drained. + Waiting for that on the bootstrap thread would stall KV metadata, and + waiting on a thread per abort does not survive an abort storm, so one + periodic thread does both jobs. + """ + self._abort_ack_queue: queue.Queue[_PendingAbortAck] = queue.Queue() + self._abort_ack_pending: set = set() + self._abort_ack_lock = threading.Lock() + self._room_sweep_ttl = max(1.0, float(self.bootstrap_timeout)) + threading.Thread( + target=self._transfer_bookkeeping_loop, + name="MooncakeTransferBookkeeping", + daemon=True, + ).start() + + def request_abort_ack( + self, + lifetime: Optional[RoomTransferLifetime], + room: int, + decode_ip: str, + decode_port: int, + token: bytes, + ) -> None: + pending = _PendingAbortAck( + room, + decode_ip, + decode_port, + token, + lifetime, + deadline=time.monotonic() + ABORT_ACK_MAX_AGE_S, + ) + with self._abort_ack_lock: + if pending.key() in self._abort_ack_pending: + return + if len(self._abort_ack_pending) >= MAX_PENDING_ABORT_ACKS: + logger.warning_once( + "More than %d ABORT_ACKs are owed; dropping new ones. Their " + "decode peers fall back to their own barrier policy.", + MAX_PENDING_ABORT_ACKS, + ) + return + self._abort_ack_pending.add(pending.key()) + self._abort_ack_queue.put(pending) + + def _transfer_bookkeeping_loop(self) -> None: + waiting: List[_PendingAbortAck] = [] + next_sweep = time.monotonic() + ROOM_SWEEP_INTERVAL_S + while True: + # This thread is the only one that can honour an ABORT_ACK, and a + # decode peer that never gets one waits out its whole quiescence + # deadline. It must therefore never exit, whatever goes wrong. + try: + waiting = self._drain_abort_acks(waiting) + now = time.monotonic() + if now >= next_sweep: + next_sweep = now + ROOM_SWEEP_INTERVAL_S + self._sweep_room_lifetimes() + except Exception: + # Belt and braces: _drain_abort_acks already isolates individual + # entries, but this thread is the only one that can honour an + # ABORT_ACK, so it must outlive any bug in here. + logger.exception( + "Mooncake transfer bookkeeping iteration failed; continuing" + ) + time.sleep(TRANSFER_BOOKKEEPING_INTERVAL_S) + + def _drain_abort_acks( + self, waiting: List[_PendingAbortAck] + ) -> List[_PendingAbortAck]: + """Send every owed ACK whose room has drained; return those still owed.""" + try: + # Poll quickly while ACKs are owed, otherwise just often enough to + # keep the room sweep running. + pending = self._abort_ack_queue.get( + timeout=( + TRANSFER_BOOKKEEPING_INTERVAL_S + if waiting + else ROOM_SWEEP_INTERVAL_S + ) + ) + except queue.Empty: + pass + else: + waiting.append(pending) + + still_waiting = [] + now = time.monotonic() + for pending in waiting: + # Isolated per entry: a single unsatisfiable ACK must not starve the + # ones behind it, which would strand their peers until they time out. + try: + if now >= pending.deadline: + logger.warning( + "Giving up on the ABORT_ACK owed to %s:%s for room %s", + pending.endpoint, + pending.dst_port, + pending.room, + ) + self._retire_abort_ack(pending) + elif ( + pending.lifetime is not None and not pending.lifetime.is_quiesced() + ): + still_waiting.append(pending) + elif not self._send_abort_ack(pending): + # The peer's socket is backed up. Retry on the next tick + # rather than blocking every other peer's ACK behind it. + still_waiting.append(pending) + except Exception: + logger.exception( + "Dropping unsendable ABORT_ACK for room %s; its decode peer " + "will fall back to its own barrier policy", + pending.room, + ) + self._retire_abort_ack(pending) + return still_waiting + + def _retire_abort_ack(self, pending: _PendingAbortAck) -> None: + with self._abort_ack_lock: + self._abort_ack_pending.discard(pending.key()) + + def _send_abort_ack(self, pending: _PendingAbortAck) -> bool: + """Try to send one ACK without blocking. False means "retry later".""" + try: + self._send_manager_message( + pending.endpoint, + pending.dst_port, + [ + b"ABORT_ACK", + str(pending.room).encode("ascii"), + pending.token, + ], + nonblocking=True, + ) + except zmq.Again: + return False + except Exception as e: + # A broken endpoint must not owe an ACK forever; the decode peer + # falls back to its own quiescence deadline. + logger.debug("Failed to send ABORT_ACK for room %s: %s", pending.room, e) + else: + logger.debug( + "Sent ABORT_ACK for room %s to %s:%s", + pending.room, + pending.endpoint, + pending.dst_port, + ) + self._retire_abort_ack(pending) + return True + + # ------------------------------------------------------------------ + # Live receiver registry (decode side) + # ------------------------------------------------------------------ + + def register_receiver(self, receiver: MooncakeKVReceiver) -> bool: + """Claim *receiver*'s room. False if the room is still owned elsewhere.""" + with self._receivers_lock: + if ( + self._receivers.setdefault(receiver.bootstrap_room, receiver) + is receiver + ): + return True + return False + + def unregister_receiver(self, receiver: MooncakeKVReceiver) -> None: + with self._receivers_lock: + if self._receivers.get(receiver.bootstrap_room) is receiver: + del self._receivers[receiver.bootstrap_room] + + def _is_tearing_down(self, room: int) -> bool: + """Whether *room* is releasing ownership, so late messages must be dropped.""" + with self._receivers_lock: + receiver = self._receivers.get(room) + return receiver is not None and receiver.is_failure_quiescing() + + def _handle_abort_ack(self, msg: List[bytes]) -> None: + room = int(msg[1].decode("ascii")) + # Peers that predate the ownership barrier reply without echoing the + # token; such an ACK carries no drain guarantee (see record_abort_ack). + token = msg[2] if len(msg) > 2 else None + logger.debug("Received ABORT_ACK for room %s", room) + with self._receivers_lock: + receiver = self._receivers.get(room) + if receiver is not None: + receiver.record_abort_ack(token) + + # ------------------------------------------------------------------ + + def release_without_proof(self, room: int, reason: str) -> bool: + """Whether *room* may release its KV pages without proof of quiescence. + + WARN trades a narrow, loud corruption window for liveness. STRICT refuses, + and fails the worker once too many requests are stuck, because a restart + releases every page safely while a silent overwrite does not. + """ + TRANSFER_QUIESCE_TIMEOUTS.inc() + if self.transfer_barrier != TransferBarrierLevel.STRICT: + logger.error( + "Releasing KV pages for bootstrap_room=%s without proof that " + "transfer work stopped: %s. An in-flight write may still land in " + "them. Set SGLANG_DISAGGREGATION_TRANSFER_BARRIER=2 to refuse " + "instead.", + room, + reason, + ) + return True + + with self._unquiesced_lock: + self._unquiesced_rooms.add(room) + stuck = len(self._unquiesced_rooms) + logger.error( + "Withholding KV pages for bootstrap_room=%s: %s (%d/%d requests " + "stuck). STRICT will not reuse a page it cannot prove is idle.", + room, + reason, + stuck, + self.max_unquiesced, + ) + if stuck >= self.max_unquiesced: + # Unrecoverable: the only way to reclaim these pages safely is to + # stop using this address space entirely. The scheduler's top-level + # handler turns this into engine teardown. + raise KVTransferBarrierEscalation( + f"{stuck} KV transfers cannot be proven quiesced " + f"(SGLANG_DISAGGREGATION_MAX_UNQUIESCED={self.max_unquiesced}); " + "refusing to reuse their pages. Restarting this worker is the " + "only safe way to reclaim them. Check for an unreachable or " + "wedged peer." + ) + return False + + def forget_unquiesced(self, room: int) -> None: + with self._unquiesced_lock: + self._unquiesced_rooms.discard(room) + + def _send_manager_message( + self, + remote: str, + dst_port: int, + parts: List[bytes], + nonblocking: bool = False, + ) -> None: + """Send one multipart message to a peer's manager socket. + + *nonblocking* raises ``zmq.Again`` instead of waiting for a backed-up + peer; use it for messages the caller retries. + """ + na = NetworkAddress(remote, dst_port) + self._send_multipart_locked( + na.to_tcp(), + parts, + is_ipv6=na.is_ipv6, + flags=zmq.NOBLOCK if nonblocking else 0, + ) + def init_engine(self): self.engine = get_mooncake_transfer_engine() @@ -383,11 +981,9 @@ def _try_create_staging_strategy(self, staging_buffer): def _send_chunk_ready(self, req, chunk_idx, kv_chunk, prefill_unique_rank): """Notify decode that a non-last staging chunk RDMA is complete.""" - na = NetworkAddress(req.endpoint, req.dst_port) - self._connect( - na.to_tcp(), - is_ipv6=na.is_ipv6, - ).send_multipart( + self._send_manager_message( + req.endpoint, + req.dst_port, [ b"CHUNK_READY", str(req.room).encode("ascii"), @@ -396,7 +992,7 @@ def _send_chunk_ready(self, req, chunk_idx, kv_chunk, prefill_unique_rank): str(len(kv_chunk.prefill_kv_indices)).encode("ascii"), req.mooncake_session_id.encode("ascii"), str(prefill_unique_rank).encode("ascii"), - ] + ], ) def _do_staging_transfer( @@ -707,22 +1303,13 @@ def process_layers(layers_params: List[Tuple[int, int, int]]) -> int: return self._transfer_data(mooncake_session_id, transfer_blocks) if self.enable_custom_mem_pool: - futures = [ - executor.submit( - process_layer, - src_ptr, - dst_ptr, - item_len, - ) - for (src_ptr, dst_ptr, item_len) in layers_params - ] - for future in concurrent.futures.as_completed(futures): - status = future.result() - if status != 0: - for f in futures: - f.cancel() - return status - return 0 + return submit_transfer_calls( + executor, + [ + (process_layer, (src_ptr, dst_ptr, item_len)) + for (src_ptr, dst_ptr, item_len) in layers_params + ], + ) else: # Combining all layers' params in one batch transfer is more efficient # compared to using multiple threads @@ -858,24 +1445,15 @@ def process_layer_tp_aware(src_layer_ptr, dst_layer_ptr): mooncake_session_id, src_addr_list, dst_addr_list, length_list ) - futures = [] - for i in range(layers_current_pp_stage): - futures.append( - executor.submit(process_layer_tp_aware, src_k_ptrs[i], dst_k_ptrs[i]) - ) - for i in range(layers_current_pp_stage): - futures.append( - executor.submit(process_layer_tp_aware, src_v_ptrs[i], dst_v_ptrs[i]) - ) - - for future in concurrent.futures.as_completed(futures): - status = future.result() - if status != 0: - for f in futures: - f.cancel() - return status - - return 0 + calls = [ + (process_layer_tp_aware, (src_k_ptrs[i], dst_k_ptrs[i])) + for i in range(layers_current_pp_stage) + ] + calls += [ + (process_layer_tp_aware, (src_v_ptrs[i], dst_v_ptrs[i])) + for i in range(layers_current_pp_stage) + ] + return submit_transfer_calls(executor, calls) def send_aux( self, @@ -935,10 +1513,9 @@ def send_aux_data_to_endpoint( aux_index: int, data: bytes, ): - na = NetworkAddress(remote, dst_port) - socket = self._connect(na.to_tcp(), is_ipv6=na.is_ipv6) - - socket.send_multipart( + self._send_manager_message( + remote, + dst_port, [ MooncakeKVManager.AUX_DATA_HEADER, str(room).encode("ascii"), @@ -946,7 +1523,7 @@ def send_aux_data_to_endpoint( str(aux_index).encode("ascii"), struct.pack(">I", len(data)), data, - ] + ], ) def _handle_aux_data(self, msg: List[bytes]): @@ -1338,13 +1915,14 @@ def _send_mamba_state_slice( def sync_status_to_decode_endpoint( self, remote: str, dst_port: int, room: int, status: int, prefill_rank: int ): - na = NetworkAddress(remote, dst_port) - self._connect(na.to_tcp(), is_ipv6=na.is_ipv6).send_multipart( + self._send_manager_message( + remote, + dst_port, [ str(room).encode("ascii"), str(status).encode("ascii"), str(prefill_rank).encode("ascii"), - ] + ], ) def transfer_worker( @@ -1363,6 +1941,7 @@ def transfer_worker( ) while True: + lifetime = None try: kv_chunk: TransferKVChunk = queue.get() if self.enable_trace: @@ -1372,8 +1951,13 @@ def transfer_worker( MooncakeRequestStage.MOONCAKE_WORKER_SEND.level, ) + # Hold the room's transfer lease for the whole chunk. Until it is + # returned the scheduler cannot observe the request as terminal, + # so these KV pages stay owned by this transfer. + lifetime = self.try_lease_chunk(kv_chunk) if ( - kv_chunk.room not in self.request_status + lifetime is None + or kv_chunk.room not in self.request_status or self.check_status(kv_chunk.room) == KVPoll.Failed ): logger.debug( @@ -1598,6 +2182,9 @@ def transfer_worker( raise RuntimeError( f"Transfer thread failed because of {e}. Prefill instance with bootstrap_port={self.bootstrap_port} is dead." ) + finally: + if lifetime is not None: + lifetime.end_lease() def start_prefill_thread(self): def bootstrap_thread(): @@ -1624,41 +2211,7 @@ def bootstrap_thread(): continue # Decode-side abort notification: mark room as failed and ACK if room == "ABORT": - room_to_be_aborted = int(waiting_req_bytes[1].decode("ascii")) - decode_ip = waiting_req_bytes[2].decode("ascii") - decode_port = int(waiting_req_bytes[3].decode("ascii")) - # No need to abort the room if it has already succeeded - if ( - room_to_be_aborted in self.request_status - and self.check_status(room_to_be_aborted) != KVPoll.Success - ): - self.update_status(room_to_be_aborted, KVPoll.Failed) - logger.debug( - f"Received abort notification for room {room_to_be_aborted}, " - f"marked as Failed" - ) - else: - logger.debug( - f"Received abort notification for room {room_to_be_aborted}, " - f"ignoring (already completed or unknown)" - ) - # Send ACK back to decode endpoint - try: - na = NetworkAddress(decode_ip, decode_port) - self._connect(na.to_tcp(), is_ipv6=na.is_ipv6).send_multipart( - [ - b"ABORT_ACK", - str(room_to_be_aborted).encode("ascii"), - ] - ) - logger.debug( - f"Sent ABORT_ACK for room {room_to_be_aborted} to " - f"{decode_ip}:{decode_port}" - ) - except Exception as e: - logger.debug( - f"Failed to send ABORT_ACK for room {room_to_be_aborted}: {e}" - ) + self._handle_abort_notification(waiting_req_bytes) continue mooncake_session_id = waiting_req_bytes[3].decode("ascii") if room == "None": @@ -1675,29 +2228,75 @@ def bootstrap_thread(): ) continue else: - required_dst_info_num = int(waiting_req_bytes[7].decode("ascii")) - room = int(room) - if room not in self.transfer_infos: - self.transfer_infos[room] = {} - - self.transfer_infos[room][mooncake_session_id] = ( - TransferInfo.from_zmq(waiting_req_bytes) - ) - # NOTE: after bootstrapping we can mark the req as waiting for input - if len(self.transfer_infos[room]) == required_dst_info_num: - self.resolve_kv_replica_factor(self.transfer_infos[room]) - self.req_to_decode_prefix_len[room] = next( - ( - info.decode_prefix_len - for info in self.transfer_infos[room].values() - if info.decode_prefix_len is not None - ), - 0, - ) - self.update_status(room, KVPoll.WaitingForInput) + self._handle_bootstrap_metadata(waiting_req_bytes) threading.Thread(target=bootstrap_thread).start() + def _handle_abort_notification(self, msg: List[bytes]) -> None: + """Handle an ABORT from a decode rank (runs on the bootstrap thread).""" + room = int(msg[1].decode("ascii")) + decode_ip = msg[2].decode("ascii") + decode_port = int(msg[3].decode("ascii")) + # Peers that predate the ownership barrier send no token; their aborts + # are honoured unconditionally, as before. + abort_token = msg[4] if len(msg) > 4 else b"" + + lifetime = self._close_room_for_abort(room, abort_token) + # No need to abort the room if it has already succeeded + if ( + lifetime is not None + and room in self.request_status + and self.check_status(room) != KVPoll.Success + ): + self.update_status(room, KVPoll.Failed) + logger.debug( + f"Received abort notification for room {room}, marked as Failed" + ) + else: + logger.debug( + f"Received abort notification for room {room}, " + f"ignoring (already completed or unknown)" + ) + # The ACK tells decode its pages are safe to reuse, so the pump sends it + # once this room's transfers have drained. + self.request_abort_ack(lifetime, room, decode_ip, decode_port, abort_token) + + def _handle_bootstrap_metadata(self, msg: List[bytes]) -> bool: + """Record a decode rank's KV destinations (runs on the bootstrap thread). + + Returns False if the metadata was dropped. + """ + room = int(msg[0].decode("ascii")) + mooncake_session_id = msg[3].decode("ascii") + required_dst_info_num = int(msg[7].decode("ascii")) + transfer_info = TransferInfo.from_zmq(msg) + + # Metadata can arrive before this rank creates the sender, so the room's + # ownership barrier is created by whichever side gets here first. A closed + # barrier means the room was aborted or released, and accepting + # destinations for it would let us transfer into pages decode has freed. + lifetime = self._room_lifetime(room, create=True) + if lifetime.is_closed(): + logger.debug("Dropping KV metadata for room %s: transfers are closed", room) + return False + lifetime.add_abort_token(transfer_info.abort_token) + + infos = self.transfer_infos.setdefault(room, {}) + infos[mooncake_session_id] = transfer_info + # NOTE: after bootstrapping we can mark the req as waiting for input + if len(infos) == required_dst_info_num: + self.resolve_kv_replica_factor(infos) + self.req_to_decode_prefix_len[room] = next( + ( + info.decode_prefix_len + for info in infos.values() + if info.decode_prefix_len is not None + ), + 0, + ) + self.update_status(room, KVPoll.WaitingForInput) + return True + def start_decode_thread(self): def decode_thread(): while True: @@ -1709,10 +2308,18 @@ def decode_thread(): # Staging: prefill notifies a chunk written to staging buffer if msg[0] == b"CHUNK_READY": room = int(msg[1].decode("ascii")) + if self._is_tearing_down(room): + continue chunk_idx = int(msg[2].decode("ascii")) page_start = int(msg[3].decode("ascii")) num_pages = int(msg[4].decode("ascii")) - session_id = msg[5].decode("ascii") + # Prefer the prefill's unique rank id when present so that + # writers are counted per rank rather than per session. + writer_id = ( + msg[6].decode("ascii") + if len(msg) > 6 + else msg[5].decode("ascii") + ) handler = self._staging_handler assert ( handler is not None @@ -1722,21 +2329,21 @@ def decode_thread(): chunk_idx, page_start, num_pages, - session_id, + writer_id, self._chunk_writer_counts, ) continue # Staging: prefill pre-requests staging allocation before forward if msg[0] == b"STAGING_REQ": + if self._is_tearing_down(int(msg[1].decode("ascii"))): + continue self._handle_staging_req(msg) continue # Prefill acknowledges abort notification if msg[0] == b"ABORT_ACK": - # TODO(shangming): use this info to implement the deferred release mechanism if needed - ack_aborted_room = int(msg[1].decode("ascii")) - logger.debug(f"Received ABORT_ACK for room {ack_aborted_room}") + self._handle_abort_ack(msg) continue bootstrap_room, status, prefill_rank = msg @@ -1744,6 +2351,9 @@ def decode_thread(): bootstrap_room = int(bootstrap_room.decode("ascii")) prefill_rank = int(prefill_rank.decode("ascii")) + if self._is_tearing_down(bootstrap_room): + continue + if status == KVPoll.Success: if bootstrap_room in self.request_status: self.prefill_response_tracker[bootstrap_room].add(prefill_rank) @@ -1792,6 +2402,18 @@ def add_transfer_request( ) return + lifetime = self._room_lifetime(bootstrap_room, create=False) + if lifetime is None or lifetime.is_closed(): + # The room was aborted (possibly by a decode notification that + # overtook this rank's own bookkeeping) or already released. + logger.debug( + "Room %s no longer admits transfers, dropping chunk", bootstrap_room + ) + return + # Stamp the chunk with this request's identity so a recycled room cannot + # pick it up (see try_lease_chunk). + chunk_owner = lifetime + if bootstrap_room not in self.transfer_infos: # This means that the current rank is a dummy rank for this request, # and it has already been marked as success, so there is no need to @@ -1816,6 +2438,7 @@ def add_transfer_request( is_last_chunk=is_last_chunk, prefill_aux_index=aux_index, state_indices=state_indices, + owner=chunk_owner, trace_ctx=trace_ctx, ) ) @@ -1889,6 +2512,17 @@ def __init__( ) self.conclude_state = None self.init_time = time.time() + self._quiescing = False + self._quiesce_deadline = float("inf") + self._quiesce_timed_out = False + # Join the room's ownership barrier, which decode metadata may already + # have created, and fail fast if an abort got here first. + if not self.kv_mgr.open_room_transfers(self.bootstrap_room): + self.kv_mgr.record_failure( + self.bootstrap_room, + "Aborted by the decode instance before KV transfer started.", + ) + self.kv_mgr.update_status(self.bootstrap_room, KVPoll.Failed) self._init_trace_ctx() @mooncake_trace_func(MooncakeRequestStage.MOONCAKE_SEND) @@ -1938,6 +2572,52 @@ def poll(self) -> KVPoll: else: return self.conclude_state + # ------------------------------------------------------------------ + # Transfer ownership barrier + # + # A failed request keeps its KV pages until this rank's transfer workers + # have returned. ``poll()`` still reports the logical state; the scheduler + # defers the terminal transition via ``advance_failure_quiescence`` (see + # ``disaggregation.utils.poll_and_all_reduce``). + # ------------------------------------------------------------------ + + def _close_barrier(self) -> None: + """Stop admitting transfer work for this room (idempotent).""" + if self._quiescing: + return + self._quiescing = True + self._quiesce_deadline = time.monotonic() + self.kv_mgr.quiesce_timeout + self.kv_mgr.close_room_transfers(self.bootstrap_room) + + def is_failure_quiescing(self) -> bool: + return self._quiescing + + def advance_failure_quiescence(self) -> bool: + self._close_barrier() + if self.kv_mgr.transfer_barrier == TransferBarrierLevel.OFF: + return True + lifetime = self.kv_mgr._room_lifetime(self.bootstrap_room, create=False) + if lifetime is None or lifetime.is_quiesced(): + self.kv_mgr.forget_unquiesced(self.bootstrap_room) + return True + if self._quiesce_timed_out: + # WARN already decided to release; do not re-report it every poll. + return True + if time.monotonic() < self._quiesce_deadline: + return False + released = self.kv_mgr.release_without_proof( + self.bootstrap_room, + f"{lifetime.outstanding_leases()} transfer worker(s) still hold it " + f"after {self.kv_mgr.quiesce_timeout:.1f}s", + ) + self._quiesce_timed_out = released + return released + + def clear(self) -> None: + super().clear() + self.kv_mgr.forget_unquiesced(self.bootstrap_room) + self.kv_mgr._forget_room_lifetime(self.bootstrap_room) + def failure_exception(self): # Explicitly set the status to failure since this request has failed in another rank if self.conclude_state is None: @@ -1971,6 +2651,9 @@ def _init_trace_ctx(self): def abort(self): super().abort() + # Close the barrier now rather than on the next poll, so no further + # chunks are queued for a request the scheduler has given up on. + self._close_barrier() self.trace_ctx.abort(abort_info={"reason": "Aborted"}) self.trace_ctx.trace_req_finish() @@ -1984,7 +2667,32 @@ def __init__( ): self.session_id = mgr.get_session_id() self.init_time = None + # Ownership barrier state. Only meaningful once send_metadata() has told + # a prefill rank where to write: before that no peer can touch our pages. + self._metadata_sent = False + self._quiescing = False + self._quiesce_complete = False + self._quiesce_deadline = float("inf") + self._abort_targets: List[Tuple[dict, bytes]] = [] + self._expected_abort_acks: set = set() + self._received_abort_acks: set = set() + self._peer_lacks_barrier = False + self._last_abort_send = float("-inf") + self._abort_lock = threading.Lock() super().__init__(mgr, bootstrap_addr, bootstrap_room) + # Per-room state (ABORT_ACK routing, staging teardown) may only be + # touched by the receiver that owns the room. A second live receiver for + # the same bootstrap_room means the room numbers collided upstream: + # everything keyed by room is then ambiguous, so say so loudly rather + # than corrupt the owner's state. Abort nonces are per receiver, so this + # one still terminates -- via its quiescence deadline. + self._owns_room = mgr.register_receiver(self) + if not self._owns_room: + logger.error( + "bootstrap_room %s is already in use by an unfinished KV transfer; " + "colliding bootstrap rooms make per-room state ambiguous", + self.bootstrap_room, + ) def _register_kv_args(self) -> bool: for bootstrap_info in self.bootstrap_infos: @@ -2090,7 +2798,10 @@ def send_metadata( self.bootstrap_room, self.bootstrap_infos, self ) - for bootstrap_info in self.bootstrap_infos: + # From here on a prefill rank may write into our KV pages, so the pages + # must not be released until every peer confirms it has stopped. + self._metadata_sent = True + for bootstrap_info, abort_token in self._abort_targets_snapshot(): sock, lock = self._connect_to_bootstrap_server(bootstrap_info) is_dummy = bootstrap_info["is_dummy"] try: @@ -2110,6 +2821,7 @@ def send_metadata( ), str(self.required_dst_info_num).encode("ascii"), str(decode_prefix_len or 0).encode("ascii"), + abort_token, ] ) except zmq.ZMQError: @@ -2136,6 +2848,175 @@ def poll(self) -> KVPoll: return status + # ------------------------------------------------------------------ + # Transfer ownership barrier + # + # Our KV pages are the *destination* of the prefill's RDMA writes, so we + # cannot release them on the strength of a local decision alone: every + # prefill rank we handed indices to has to confirm it has stopped writing. + # ------------------------------------------------------------------ + + def abort(self): + super().abort() + # Stop admitting staging work now rather than on the next poll. + self._close_barrier() + + def _close_barrier(self) -> None: + """Stop admitting transfer work for this room (idempotent).""" + if self._quiescing: + return + self._quiescing = True + self._quiesce_deadline = time.monotonic() + self.kv_mgr.quiesce_timeout + + def is_failure_quiescing(self) -> bool: + return self._quiescing + + def advance_failure_quiescence(self) -> bool: + self._close_barrier() + if self._quiesce_complete or not self._metadata_sent: + # No prefill rank was ever handed this request's page indices, so + # nothing can be writing into them. + return True + if self.kv_mgr.transfer_barrier == TransferBarrierLevel.OFF: + self._quiesce_complete = True + return True + if not self._owns_room: + # Another live receiver owns this bootstrap_room, so our ABORT_ACKs + # are routed to it and proof can never reach us. Do not pretend to + # wait for it; the collision itself is already reported as an error. + return self._conclude_without_proof( + f"bootstrap_room {self.bootstrap_room} is owned by another " + "receiver, so its acknowledgements are unroutable" + ) + if self._peers_quiesced(): + self._quiesce_complete = True + self.kv_mgr.forget_unquiesced(self.bootstrap_room) + return True + if time.monotonic() < self._quiesce_deadline: + return False + with self._abort_lock: + missing = len(self._expected_abort_acks - self._received_abort_acks) + total = len(self._expected_abort_acks) + return self._conclude_without_proof( + f"{missing} of {total} prefill ranks unacknowledged after " + f"{self.kv_mgr.quiesce_timeout:.1f}s" + ) + + def _conclude_without_proof(self, reason: str) -> bool: + released = self.kv_mgr.release_without_proof(self.bootstrap_room, reason) + if released: + self._quiesce_complete = True + return released + + def _peers_quiesced(self) -> bool: + self._send_abort_notification() + with self._abort_lock: + if self._peer_lacks_barrier: + # A peer that acknowledges without echoing a nonce does so before + # its transfers drain, so its ACK is not proof. WARN accepts that + # and keeps a rolling upgrade working; STRICT cannot, by + # definition, so such a peer never satisfies the barrier. + return self.kv_mgr.transfer_barrier != TransferBarrierLevel.STRICT + return self._expected_abort_acks <= self._received_abort_acks + + def _abort_targets_snapshot(self) -> List[Tuple[dict, bytes]]: + """Per-peer abort nonces, minted once and reused across retries. + + One nonce per prefill rank lets an ABORT_ACK identify *which* peer has + drained, and prevents a stale ACK for a recycled room from satisfying + this request. + """ + with self._abort_lock: + if not self._abort_targets and self.bootstrap_infos: + self._abort_targets = [ + (bootstrap_info, os.urandom(16).hex().encode("ascii")) + for bootstrap_info in self.bootstrap_infos + ] + self._expected_abort_acks = { + token for _info, token in self._abort_targets + } + return list(self._abort_targets) + + def record_abort_ack(self, token: Optional[bytes]) -> None: + with self._abort_lock: + if not token: + # A prefill that predates the ownership barrier acknowledges the + # notification without echoing a nonce, and does so before its + # transfers have drained. It cannot promise quiescence, so stop + # waiting rather than stalling until the deadline. + self._peer_lacks_barrier = True + logger.warning_once( + "Prefill peer acknowledged an abort without a transfer-quiescence " + "token, so it acknowledges before its transfers drain and cannot " + "prove quiescence. Upgrade the prefill instances; until then this " + "request's pages are released on the " + "SGLANG_DISAGGREGATION_TRANSFER_BARRIER policy rather than on " + "proof, and STRICT will refuse to release them at all." + ) + return + if token in self._expected_abort_acks: + self._received_abort_acks.add(token) + + def _send_abort_notification(self): + """(Re-)notify prefill ranks that have not acknowledged the abort.""" + targets = self._abort_targets_snapshot() + if not targets: + return + now = time.monotonic() + with self._abort_lock: + if now - self._last_abort_send < ABORT_RETRY_INTERVAL_S: + return + self._last_abort_send = now + targets = [ + (info, token) + for info, token in targets + if token not in self._received_abort_acks + ] + + for bootstrap_info, token in targets: + # Best-effort notification to prefill side that this request was + # aborted. This runs on the scheduler loop, and these sockets have no + # send timeout, so it must never wait for a peer: zmq.NOBLOCK turns a + # backed-up or dead prefill into a retry on the next poll instead of + # an unbounded stall of the whole engine. + try: + sock, lock = self._connect_to_bootstrap_server(bootstrap_info) + with lock: + sock.send_multipart( + [ + b"ABORT", + str(self.bootstrap_room).encode("ascii"), + self.kv_mgr.local_ip.encode("ascii"), + str(self.kv_mgr.rank_port).encode("ascii"), + token, + ], + zmq.NOBLOCK, + ) + except zmq.Again: + # Retried by the next _send_abort_notification() tick. + with self._abort_lock: + self._last_abort_send = float("-inf") + except Exception as e: + logger.debug( + "Failed to send abort notification for room %s: %s", + self.bootstrap_room, + e, + ) + + def clear(self) -> None: + if not self._owns_room: + # Everything CommonKVReceiver.clear() drops is keyed by + # bootstrap_room and therefore belongs to the receiver that owns it. + # Tearing it down here would fail the owner's live request as well. + logger.debug( + "Skipping shared cleanup for bootstrap_room %s: owned elsewhere", + self.bootstrap_room, + ) + return + super().clear() + self.kv_mgr.forget_unquiesced(self.bootstrap_room) + self.kv_mgr.unregister_receiver(self) + def failure_exception(self): if self.conclude_state is None: self.conclude_state = KVPoll.Failed diff --git a/python/sglang/srt/disaggregation/prefill.py b/python/sglang/srt/disaggregation/prefill.py index 3e95298e2add..6d13a5b6e0a1 100644 --- a/python/sglang/srt/disaggregation/prefill.py +++ b/python/sglang/srt/disaggregation/prefill.py @@ -427,6 +427,10 @@ def pop_bootstrapped( bootstrapped_reqs.append(req) indices_to_remove.add(i) req.time_stats.set_wait_queue_entry_time() + elif poll == KVPoll.Transferring: + # A backend can defer a terminal transition while it releases + # ownership of the request's KV pages; keep waiting. + continue else: raise RuntimeError( f"Unexpected poll state {poll} for req {req.rid} in pop_bootstrapped" @@ -986,7 +990,9 @@ def handle_pending_bootstrap(self: Scheduler, req: Req, poll: KVPoll) -> bool: if poll == KVPoll.Failed: self.handle_bootstrap_failure(req) return False - elif poll == KVPoll.Bootstrapping: + elif poll in (KVPoll.Bootstrapping, KVPoll.Transferring): + # Transferring here means the backend is still releasing ownership of + # a failed request's KV pages; it will report Failed once done. return False elif poll == KVPoll.WaitingForInput: if should_force_retry(req): # test hook diff --git a/python/sglang/srt/disaggregation/utils.py b/python/sglang/srt/disaggregation/utils.py index cbb4abf75bb9..815b3912c0ca 100644 --- a/python/sglang/srt/disaggregation/utils.py +++ b/python/sglang/srt/disaggregation/utils.py @@ -1,5 +1,6 @@ from __future__ import annotations +import logging import os import random from collections import deque @@ -13,9 +14,12 @@ from sglang.srt.configs.model_config import get_dsa_index_topk from sglang.srt.disaggregation.base import KVPoll +from sglang.srt.disaggregation.base.conn import KVTransferBarrierEscalation from sglang.srt.environ import envs from sglang.srt.utils import is_hip, is_npu +logger = logging.getLogger(__name__) + if TYPE_CHECKING: from sglang.srt.disaggregation.base.conn import KVArgs, StateType from sglang.srt.disaggregation.common.conn import ( @@ -99,11 +103,19 @@ def _get_failure_prob() -> float: def _poll_with_failure_injection(pollers) -> List[int]: if (failure_prob := _get_failure_prob()) > 0: - return [ + polls = [ int(KVPoll.Failed) if random.random() < failure_prob else int(poller.poll()) for poller in pollers ] - return [int(poller.poll()) for poller in pollers] + else: + polls = [int(poller.poll()) for poller in pollers] + # A request whose ownership barrier has closed is logically failed even if + # its backend still reports an earlier state, so every rank agrees to fail + # it (and therefore to release it together). + return [ + int(KVPoll.Failed) if poller.is_failure_quiescing() else poll + for poller, poll in zip(pollers, polls) + ] def _is_fake_transfer(req: Req, server_args: ServerArgs) -> bool: @@ -131,6 +143,64 @@ def _apply_metadata_gate(polls, decode_reqs, metadata_buffers, server_args) -> N polls[i] = int(KVPoll.Transferring) +def _reduce_polls(polls: List[int], groups) -> List[int]: + tensor_to_reduce = torch.tensor(polls, dtype=torch.uint8, device="cpu") + for group in groups: + dist.all_reduce(tensor_to_reduce, op=dist.ReduceOp.MIN, group=group) + return tensor_to_reduce.tolist() + + +def _advance_quiescence(poller) -> int: + """Ask one poller whether its KV pages are safe to release. + + Backend faults are contained here on purpose. This runs inside the scheduler + loop, and the barrier is a safety optimisation: letting a bug in it take the + whole engine down, or pin a request forever, is worse than releasing one + request's pages early and saying so. + + A KVTransferBarrierEscalation is not such a fault. It is the barrier + reporting that pages cannot be proven idle and must not be reused, so it is + propagated: swallowing it here would silently turn the strictest policy into + the most permissive one. + """ + try: + return int(poller.advance_failure_quiescence()) + except KVTransferBarrierEscalation: + raise + except Exception: + logger.exception( + "Transfer quiescence check failed; releasing the request's KV pages" + ) + return 1 + + +def _gate_failures_on_quiescence(pollers, polls: List[int], groups) -> List[int]: + """Hold back ``KVPoll.Failed`` until the KV pages are safe to release. + + A failed transfer may still have native work reading or writing its KV + pages (see ``BaseKVSender.advance_failure_quiescence``). Reporting the + failure lets the scheduler return those pages to the allocator, so the + terminal state is deferred -- as ``Transferring`` -- while any rank in the + group is still holding ownership. + + Both collectives are entered on exactly the same iterations on every rank, + because the decision is made from the already-reduced poll values. + """ + failed = [poll == int(KVPoll.Failed) for poll in polls] + if not any(failed): + return polls + + quiesced = [ + _advance_quiescence(poller) if is_failed else 1 + for poller, is_failed in zip(pollers, failed) + ] + quiesced = _reduce_polls(quiesced, groups) + return [ + int(KVPoll.Transferring) if is_failed and not ok else poll + for poll, is_failed, ok in zip(polls, failed, quiesced) + ] + + def poll_and_all_reduce( pollers, gloo_group: dist.ProcessGroup, @@ -148,9 +218,8 @@ def poll_and_all_reduce( and server_args is not None ): _apply_metadata_gate(polls, decode_reqs, metadata_buffers, server_args) - tensor_to_reduce = torch.tensor(polls, dtype=torch.uint8, device="cpu") - dist.all_reduce(tensor_to_reduce, op=dist.ReduceOp.MIN, group=gloo_group) - return tensor_to_reduce.tolist() + groups = [gloo_group] + return _gate_failures_on_quiescence(pollers, _reduce_polls(polls, groups), groups) def poll_and_all_reduce_attn_cp_tp_group( @@ -158,19 +227,12 @@ def poll_and_all_reduce_attn_cp_tp_group( attn_cp_cpu_group: dist.ProcessGroup, attn_tp_cpu_group: dist.ProcessGroup, ): + polls = _poll_with_failure_injection(pollers) # First sync across attn-tp ranks so all TP participants for a given (dp, cp) - # shard observe the same status transitions. - polls = poll_and_all_reduce(pollers, attn_tp_cpu_group) - - # Then sync across attn-cp ranks, so all TPxCP participants in one DP shard - # converge to the same global status. - tensor_to_reduce = torch.tensor(polls, dtype=torch.uint8, device="cpu") - dist.all_reduce( - tensor_to_reduce, - op=dist.ReduceOp.MIN, - group=attn_cp_cpu_group, - ) - return tensor_to_reduce.tolist() + # shard observe the same status transitions, then across attn-cp ranks so all + # TPxCP participants in one DP shard converge to the same global status. + groups = [attn_tp_cpu_group, attn_cp_cpu_group] + return _gate_failures_on_quiescence(pollers, _reduce_polls(polls, groups), groups) def poll_and_all_reduce_with_staging( @@ -181,6 +243,7 @@ def poll_and_all_reduce_with_staging( server_args: Optional[ServerArgs] = None, ): """Staging-aware polling: advance scatter, demote incomplete transfers, all_reduce.""" + staging_handler.retry_pending_watermarks() for decode_req in decode_reqs: if decode_req.kv_receiver.require_staging and not staging_handler.is_done( decode_req @@ -199,9 +262,10 @@ def poll_and_all_reduce_with_staging( # Apply metadata gate on the decode requests to downgrade Success → Transferring for requests whose metadata hasn't landed. if metadata_buffers is not None and server_args is not None: _apply_metadata_gate(raw_polls, decode_reqs, metadata_buffers, server_args) - poll_tensor = torch.tensor(raw_polls, dtype=torch.uint8, device="cpu") - dist.all_reduce(poll_tensor, op=dist.ReduceOp.MIN, group=gloo_group) - return poll_tensor.tolist() + groups = [gloo_group] + return _gate_failures_on_quiescence( + receivers, _reduce_polls(raw_polls, groups), groups + ) ######################### diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index b4ff61057eb6..8c675dda8f82 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -210,6 +210,23 @@ class ToolStrictLevel(IntEnum): PARAMETER = 2 +class TransferBarrierLevel(IntEnum): + """How strictly KV pages are withheld from reuse while a transfer may run. + + OFF: no barrier; a failed request releases its pages immediately (legacy). + WARN: wait for proof that transfer work stopped, but release anyway after + SGLANG_DISAGGREGATION_QUIESCE_TIMEOUT, logging an error and counting it. + A peer too old to prove quiescence is also released, once, loudly. + STRICT: never release without proof. A request that cannot get it stays + non-terminal, and the worker fails once too many accumulate, so the + restart releases the pages safely rather than corrupting a peer. + """ + + OFF = 0 + WARN = 1 + STRICT = 2 + + class InvariantCheckLevel(IntEnum): """Signal level for value/index validity checks (see invariants.py). @@ -396,6 +413,18 @@ class Envs: SGLANG_DISAGGREGATION_HEARTBEAT_INTERVAL = EnvFloat(5.0) SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE = EnvInt(2) SGLANG_DISAGGREGATION_WAITING_TIMEOUT = EnvInt(300) + # Whether a failed KV transfer may hand its KV pages back to the allocator + # before the transfer is known to have stopped. See TransferBarrierLevel. + SGLANG_DISAGGREGATION_TRANSFER_BARRIER = EnvInt(TransferBarrierLevel.WARN) + # WARN only: upper bound, in seconds, on how long a failed transfer holds its + # pages before they are released anyway. Keep this above the manager socket's + # 30s ZMQ send timeout, because a transfer holds its room while notifying the + # peer; a shorter value reports spurious timeouts whenever a decode stops + # reading. STRICT ignores it. + SGLANG_DISAGGREGATION_QUIESCE_TIMEOUT = EnvFloat(60.0) + # STRICT only: how many requests may be stuck waiting for proof before the + # worker gives up and fails, so that a restart releases the pages safely. + SGLANG_DISAGGREGATION_MAX_UNQUIESCED = EnvInt(256) SGLANG_DISAGGREGATION_NIXL_BACKEND = EnvStr("UCX") SGLANG_DISAGGREGATION_NIXL_BACKEND_PARAMS = EnvStr("{}") SGLANG_DISAGG_PREFILL_EARLY_SEND_CACHED_PREFIX = EnvBool(True) diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index 892c20305de1..bcaa1b4a434f 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -2705,19 +2705,26 @@ def process_pending_chunked_abort(self) -> None: prepare_abort(req, "Aborted") req.time_stats.trace_ctx.abort(abort_info={"reason": "Aborted"}) req.to_finish = None - if self.disaggregation_mode == DisaggregationMode.PREFILL: - req.disagg_kv_sender.abort() - maybe_release_metadata_buffer( - req, self.req_to_metadata_buffer_idx_allocator - ) - req.pending_bootstrap = False if self.enable_hicache_storage: self.tree_cache.release_aborted_request(req.rid) - release_kv_cache(req, self.tree_cache, is_insert=False) + + if self.disaggregation_mode == DisaggregationMode.PREFILL: + # Earlier chunks may already be in flight to the decode instance, + # which reads this request's KV pages directly. Park the request in + # the inflight queue instead of releasing here: that queue polls the + # sender and releases the pages once the transfer has quiesced. + req.disagg_kv_sender.abort() + # Its partial prefix must not enter the radix cache, matching the + # is_insert=False release this path used to do inline. + req.skip_radix_cache_insert = True + if req not in self.disagg_prefill_inflight_queue: + self.disagg_prefill_inflight_queue.append(req) + else: + release_kv_cache(req, self.tree_cache, is_insert=False) + self.ipc_channels.send_to_tokenizer.send_output(AbortReq(rid=req.rid), req) self.chunked_req = None self._pending_chunked_abort_req = None - self.ipc_channels.send_to_tokenizer.send_output(AbortReq(rid=req.rid), req) logger.debug(f"Abort chunked prefill request. {req.rid=}") def _build_hisparse_decode_batch(self, reqs): diff --git a/test/registered/unit/disaggregation/test_transfer_ownership_barrier.py b/test/registered/unit/disaggregation/test_transfer_ownership_barrier.py new file mode 100644 index 000000000000..e27613e0038c --- /dev/null +++ b/test/registered/unit/disaggregation/test_transfer_ownership_barrier.py @@ -0,0 +1,1608 @@ +"""Tests for the KV transfer ownership barrier. + +The barrier keeps a failed request's KV pages allocated until no native +transfer work can still read or write them. The properties under test are: + +* pages stay owned while transfer work is in flight (safety); +* ownership is always released eventually, even if a peer never answers + (liveness -- a wedged transport must not pin pages forever); +* the deferral only happens where the scheduler can handle it, and never + weakens the failure diagnostics the operator sees. +""" + +import concurrent.futures +import threading +import time +import unittest +from collections import OrderedDict +from types import SimpleNamespace +from unittest.mock import Mock, patch + +import numpy as np +import zmq + +from sglang.srt.disaggregation.base.conn import ( + KVPoll, + KVTransferBarrierEscalation, +) +from sglang.srt.disaggregation.common.staging_handler import DecodeStagingHandler +from sglang.srt.disaggregation.common.utils import ( + FastQueue, + TransferKVChunk, + drain_transfer_futures, + submit_transfer_calls, +) +from sglang.srt.disaggregation.mooncake.conn import ( + ABORT_RETRY_INTERVAL_S, + MAX_EMERGENCY_SCAN, + MooncakeKVManager, + MooncakeKVReceiver, + MooncakeKVSender, + RoomTransferLifetime, + TransferInfo, +) +from sglang.srt.disaggregation.utils import DisaggregationMode, poll_and_all_reduce +from sglang.srt.environ import TransferBarrierLevel +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(est_time=6, suite="base-a-test-cpu") + + +ROOM = 7 + + +def make_prefill_manager( + quiesce_timeout=30.0, room_sweep_ttl=300.0, barrier=TransferBarrierLevel.WARN +): + """A prefill-side manager with only its ownership-barrier state wired up.""" + mgr = MooncakeKVManager.__new__(MooncakeKVManager) + mgr.disaggregation_mode = DisaggregationMode.PREFILL + mgr.quiesce_timeout = quiesce_timeout + mgr.transfer_barrier = barrier + mgr.max_unquiesced = 256 + mgr._unquiesced_rooms = set() + mgr._unquiesced_lock = threading.Lock() + mgr.bootstrap_timeout = 300 + mgr._room_sweep_ttl = room_sweep_ttl + mgr._room_lifetimes = OrderedDict() + mgr._room_lifetimes_lock = threading.Lock() + mgr._endpoint_send_locks = {} + mgr._endpoint_send_locks_lock = threading.Lock() + mgr.request_status = {} + mgr.transfer_infos = {} + mgr.req_to_decode_prefix_len = {} + mgr.check_status = lambda room: mgr.request_status[room] + mgr.update_status = lambda room, status: mgr.request_status.__setitem__( + room, status + ) + mgr.record_failure = Mock() + mgr.enable_trace = False + mgr.enable_staging = False + mgr.bootstrap_port = 0 + # Minimal KVArgs surface the transfer worker dereferences. + mgr.kv_args = SimpleNamespace( + kv_data_ptrs=[0x1000], + kv_layer_ids=[0], + state_layer_ids=[], + state_data_ptrs=[], + ) + mgr._get_dsa_cache_transfer_skip_flags = Mock(return_value=(False, False)) + return mgr + + +def make_sender(mgr, room=ROOM, status=KVPoll.WaitingForInput): + """A prefill sender bound to *mgr*, bypassing bootstrap/registration.""" + sender = MooncakeKVSender.__new__(MooncakeKVSender) + sender.kv_mgr = mgr + sender.bootstrap_room = room + sender.conclude_state = None + sender._quiescing = False + sender._quiesce_deadline = float("inf") + sender._quiesce_timed_out = False + sender.trace_ctx = SimpleNamespace( + abort=Mock(), + trace_req_finish=Mock(), + copy_for_thread=Mock(return_value=Mock()), + ) + mgr.open_room_transfers(room) + if status is not None: + mgr.request_status[room] = status + return sender + + +def make_receiver(mgr, room=ROOM, bootstrap_infos=None, metadata_sent=True): + """A decode receiver bound to *mgr*, bypassing bootstrap/registration.""" + receiver = MooncakeKVReceiver.__new__(MooncakeKVReceiver) + receiver.kv_mgr = mgr + receiver.bootstrap_room = room + receiver.conclude_state = None + receiver.require_staging = False + receiver.bootstrap_infos = bootstrap_infos + receiver._metadata_sent = metadata_sent + receiver._quiescing = False + receiver._quiesce_complete = False + receiver._quiesce_deadline = float("inf") + receiver._abort_targets = [] + receiver._expected_abort_acks = set() + receiver._received_abort_acks = set() + receiver._peer_lacks_barrier = False + receiver._last_abort_send = float("-inf") + receiver._owns_room = True + receiver._abort_lock = threading.Lock() + receiver._connect_to_bootstrap_server = Mock( + side_effect=lambda info: (Mock(), threading.Lock()) + ) + return receiver + + +def make_decode_manager( + quiesce_timeout=30.0, staging_handler=None, barrier=TransferBarrierLevel.WARN +): + mgr = MooncakeKVManager.__new__(MooncakeKVManager) + mgr.quiesce_timeout = quiesce_timeout + mgr._receivers = {} + mgr._receivers_lock = threading.Lock() + mgr._staging_handler = staging_handler + mgr.transfer_barrier = barrier + mgr.max_unquiesced = 256 + mgr._unquiesced_rooms = set() + mgr._unquiesced_lock = threading.Lock() + mgr.request_status = {} + mgr.local_ip = "127.0.0.1" + mgr.rank_port = 1234 + mgr.record_failure = Mock() + mgr.update_status = Mock() + return mgr + + +class TestPageReuseRace(unittest.TestCase): + """The reproducer: a transfer must never outlive its pages' ownership. + + This is the whole bug, expressed without a GPU, a network, or a second + process. It drives the real transfer_worker with an engine that blocks inside + the write, then asks the real poll path what the scheduler would be told. + Reporting a terminal state is what lets the scheduler hand the pages back to + the allocator, so a terminal state while the engine is still writing means a + request allocated those pages next has its KV silently overwritten. + + On a build without the barrier this asserts Failed at [T3] and fails. + """ + + def test_terminal_state_is_withheld_while_a_write_is_in_flight(self): + entered_write = threading.Event() + release_write = threading.Event() + + mgr = make_prefill_manager() + mgr.request_status[ROOM] = KVPoll.WaitingForInput + mgr.transfer_infos = { + ROOM: { + "session:1": SimpleNamespace( + room=ROOM, + endpoint="127.0.0.1", + dst_port=1, + mooncake_session_id="session:1", + dst_kv_indices=np.array([0], dtype=np.int32), + is_dummy=False, + ) + } + } + mgr.decode_kv_args_table = { + "session:1": SimpleNamespace( + dst_attn_tp_size=1, + dst_kv_ptrs=[0xDEAD0000], + dst_kv_layer_ids=[0], + dst_state_layer_ids=[], + ) + } + mgr.session_lock, mgr.failed_sessions = threading.Lock(), set() + mgr.is_mla_backend, mgr.is_hybrid_mla_backend = True, False + mgr.attn_tp_size = mgr.attn_cp_size = mgr.pp_size = 1 + mgr.attn_tp_rank = mgr.attn_cp_rank = mgr.pp_rank = mgr.attn_dp_rank = 0 + mgr.transfer_queues = [FastQueue()] + + def blocking_write(*_args, **_kwargs): + entered_write.set() + release_write.wait(5) + return 0 + + mgr.send_kvcache = blocking_write + sender = make_sender(mgr) + + threading.Thread( + target=mgr.transfer_worker, + args=(mgr.transfer_queues[0], Mock()), + daemon=True, + ).start() + mgr.add_transfer_request( + ROOM, np.array([0], dtype=np.int32), slice(0, 1), False + ) + + # [T1] a worker is inside a write that targets the decode's KV pages + self.assertTrue(entered_write.wait(5), "worker never entered the transfer") + + # [T2] the request is aborted while that write is still in flight + sender.abort() + + # [T3] the scheduler must not be told this request is over + self.assertEqual( + self._poll(sender), + KVPoll.Transferring, + "a terminal state here lets the allocator hand these pages to another " + "request while the transfer is still writing to them", + ) + + # [T4] once the write returns, the request may conclude + release_write.set() + for _ in range(500): + if self._poll(sender) == KVPoll.Failed: + break + time.sleep(0.01) + self.assertEqual(self._poll(sender), KVPoll.Failed) + + def test_a_queued_chunk_cannot_attach_to_a_recycled_room(self): + """The other half of the race, reported on the upstream discussion. + + A chunk still sitting in the transfer queue names the *old* request's + pages and the *old* decode's destinations. bootstrap_room is recycled, so + matching on the room alone would let that chunk be transferred on behalf + of whichever request holds the room now. + """ + mgr = make_prefill_manager() + first = make_sender(mgr) + stale = TransferKVChunk( + room=ROOM, + prefill_kv_indices=np.array([0], dtype=np.int32), + index_slice=slice(0, 1), + is_last_chunk=False, + prefill_aux_index=None, + state_indices=None, + owner=mgr._room_lifetime(ROOM, create=False), + trace_ctx=Mock(), + ) + first.clear() # the request concludes, the room is released + + # a new request draws the same bootstrap_room + make_sender(mgr) + self.assertIsNotNone(mgr._room_lifetime(ROOM, create=False)) + self.assertIsNone( + mgr.try_lease_chunk(stale), + "the stale chunk must not lease the new request's room", + ) + + @staticmethod + def _poll(sender): + with patch( + "sglang.srt.disaggregation.utils.dist.all_reduce", + side_effect=lambda tensor, **kw: None, + ): + return poll_and_all_reduce([sender], object())[0] + + +class TestRoomTransferLifetime(unittest.TestCase): + """The primitive that decides when a room's pages may be reused.""" + + def test_open_room_is_not_quiesced_and_admits_leases(self): + lifetime = RoomTransferLifetime() + self.assertFalse(lifetime.is_quiesced()) + self.assertTrue(lifetime.try_lease()) + lifetime.end_lease() + self.assertFalse(lifetime.is_quiesced()) + + def test_close_stops_admission_and_waits_for_outstanding_lease(self): + lifetime = RoomTransferLifetime() + self.assertTrue(lifetime.try_lease()) + lifetime.close() + + self.assertFalse(lifetime.try_lease()) + self.assertFalse(lifetime.is_quiesced()) + self.assertFalse(lifetime.wait_quiesced(timeout=0.01)) + + lifetime.end_lease() + self.assertTrue(lifetime.is_quiesced()) + self.assertTrue(lifetime.wait_quiesced(timeout=1)) + + def test_waiter_wakes_when_the_last_lease_is_returned(self): + lifetime = RoomTransferLifetime() + self.assertTrue(lifetime.try_lease()) + lifetime.close() + quiesced = threading.Event() + + waiter = threading.Thread( + target=lambda: quiesced.set() if lifetime.wait_quiesced(2) else None + ) + waiter.start() + self.assertFalse(quiesced.wait(0.05)) + lifetime.end_lease() + self.assertTrue(quiesced.wait(1)) + waiter.join() + + def test_abort_token_authorization(self): + lifetime = RoomTransferLifetime() + # A room with no registered decode metadata accepts any abort: nothing + # can be in flight, and refusing would drop the abort entirely. + self.assertTrue(lifetime.authorizes_abort(b"unknown")) + lifetime.add_abort_token(b"mine") + self.assertTrue(lifetime.authorizes_abort(b"mine")) + self.assertFalse(lifetime.authorizes_abort(b"someone-elses")) + # Peers that predate the token protocol send none; honour those aborts. + self.assertTrue(lifetime.authorizes_abort(b"")) + + +class TestExecutorDrain(unittest.TestCase): + """Failing early must not hand pages back while siblings still run.""" + + def test_failure_waits_for_a_running_sibling(self): + sibling_running = threading.Event() + release_sibling = threading.Event() + drained = threading.Event() + result = [] + + def sibling(): + sibling_running.set() + release_sibling.wait() + return 0 + + with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor: + futures = [executor.submit(lambda: -1), executor.submit(sibling)] + self.assertTrue(sibling_running.wait(1)) + waiter = threading.Thread( + target=lambda: ( + result.append(drain_transfer_futures(futures)), + drained.set(), + ) + ) + waiter.start() + self.assertFalse(drained.wait(0.05), "returned while a sibling still ran") + release_sibling.set() + self.assertTrue(drained.wait(1)) + waiter.join() + + self.assertEqual(result, [-1], "the first failing status must be reported") + + def test_partial_submission_failure_still_drains_accepted_work(self): + running = threading.Event() + release = threading.Event() + returned = threading.Event() + errors = [] + + def blocked(): + running.set() + release.wait() + return 0 + + class FailsOnSecondSubmit: + def __init__(self): + self.pool = concurrent.futures.ThreadPoolExecutor(max_workers=1) + self.calls = 0 + + def submit(self, fn, *args): + self.calls += 1 + if self.calls == 2: + raise RuntimeError("submit failed") + return self.pool.submit(fn, *args) + + executor = FailsOnSecondSubmit() + + def submit_all(): + try: + submit_transfer_calls(executor, [(blocked, ()), (lambda: 0, ())]) + except Exception as error: # noqa: BLE001 - recorded for assertion + errors.append(error) + finally: + returned.set() + + waiter = threading.Thread(target=submit_all) + waiter.start() + self.assertTrue(running.wait(1)) + self.assertFalse(returned.wait(0.05), "raised while accepted work still ran") + release.set() + self.assertTrue(returned.wait(1)) + waiter.join() + executor.pool.shutdown() + self.assertEqual(str(errors[0]), "submit failed") + + def test_worker_exception_propagates_after_draining(self): + with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor: + futures = [ + executor.submit(lambda: (_ for _ in ()).throw(ValueError("boom"))), + executor.submit(lambda: 0), + ] + with self.assertRaises(ValueError): + drain_transfer_futures(futures) + + +class TestPrefillOwnership(unittest.TestCase): + """The prefill's pages are the transfer source.""" + + def test_transfer_worker_holds_the_lease_across_the_native_transfer(self): + entered_transfer = threading.Event() + release_transfer = threading.Event() + + mgr = make_prefill_manager() + mgr.request_status[ROOM] = KVPoll.WaitingForInput + mgr.transfer_infos[ROOM] = { + "session:1": SimpleNamespace( + room=ROOM, + endpoint="127.0.0.1", + dst_port=1, + mooncake_session_id="session:1", + dst_kv_indices=np.array([0], dtype=np.int32), + is_dummy=False, + ) + } + mgr.decode_kv_args_table = { + "session:1": SimpleNamespace( + dst_attn_tp_size=1, + dst_kv_ptrs=[1], + dst_kv_layer_ids=[0], + dst_state_layer_ids=[], + ) + } + mgr.session_lock = threading.Lock() + mgr.failed_sessions = set() + mgr.is_mla_backend = True + mgr.is_hybrid_mla_backend = False + mgr.attn_tp_size, mgr.attn_tp_rank = 1, 0 + mgr.attn_cp_size, mgr.attn_cp_rank = 1, 0 + mgr.pp_size, mgr.pp_rank = 1, 0 + mgr.attn_dp_rank = 0 + + def blocking_send_kvcache(*_args, **_kwargs): + entered_transfer.set() + release_transfer.wait() + return 0 + + mgr.send_kvcache = blocking_send_kvcache + lifetime = mgr._room_lifetime(ROOM, create=True) + sender = make_sender(mgr) + + queue = FastQueue() + threading.Thread( + target=mgr.transfer_worker, args=(queue, Mock()), daemon=True + ).start() + queue.put( + TransferKVChunk( + room=ROOM, + prefill_kv_indices=np.array([0], dtype=np.int32), + index_slice=slice(0, 1), + is_last_chunk=False, + prefill_aux_index=None, + state_indices=None, + trace_ctx=Mock(), + ) + ) + self.assertTrue(entered_transfer.wait(1)) + + # An abort now must not let the scheduler reclaim the pages. + sender._close_barrier() + self.assertTrue(lifetime.is_closed()) + self.assertFalse(sender.advance_failure_quiescence()) + + release_transfer.set() + for _ in range(200): + if sender.advance_failure_quiescence(): + break + time.sleep(0.01) + self.assertTrue(sender.advance_failure_quiescence()) + + def test_closed_room_admits_no_new_chunks(self): + mgr = make_prefill_manager() + sender = make_sender(mgr) + mgr.transfer_queues = [FastQueue()] + sender._close_barrier() + + mgr.add_transfer_request( + ROOM, np.array([0], dtype=np.int32), slice(0, 1), False + ) + self.assertEqual(len(mgr.transfer_queues[0]._buf), 0) + + def test_released_room_cannot_be_resurrected_by_a_queued_chunk(self): + mgr = make_prefill_manager() + make_sender(mgr) + mgr._forget_room_lifetime(ROOM) + # try_lease_room deliberately refuses to recreate the lifetime, so a + # chunk that outlived the request cannot start a transfer into pages + # that have already been handed back to the allocator. + self.assertIsNone(mgr.try_lease_room(ROOM)) + self.assertNotIn(ROOM, mgr._room_lifetimes) + + def test_quiescence_is_bounded_when_a_transfer_never_returns(self): + mgr = make_prefill_manager(quiesce_timeout=0.05) + sender = make_sender(mgr) + lifetime = mgr._room_lifetime(ROOM, create=False) + self.assertTrue(lifetime.try_lease()) # never returned + + with patch( + "sglang.srt.disaggregation.mooncake.conn.TRANSFER_QUIESCE_TIMEOUTS" + ) as metric: + self.assertFalse(sender.advance_failure_quiescence()) + time.sleep(0.06) + self.assertTrue( + sender.advance_failure_quiescence(), + "a wedged transfer must not pin the request forever", + ) + metric.inc.assert_called_once_with() + # The timeout is reported once, not on every subsequent poll. + self.assertTrue(sender.advance_failure_quiescence()) + metric.inc.assert_called_once_with() + + def test_bootstrapping_sender_is_immediately_quiesced(self): + # Nothing has been submitted, so the request can fail without any wait. + # This is what keeps KVPoll.Transferring out of the bootstrap queues. + mgr = make_prefill_manager() + sender = make_sender(mgr, status=KVPoll.Bootstrapping) + self.assertTrue(sender.advance_failure_quiescence()) + + def test_clear_forgets_the_room(self): + mgr = make_prefill_manager() + sender = make_sender(mgr) + sender.clear() + self.assertNotIn(ROOM, mgr._room_lifetimes) + self.assertNotIn(ROOM, mgr.request_status) + + def test_tracked_rooms_are_bounded_without_evicting_a_live_room(self): + mgr = make_prefill_manager() + live = mgr._room_lifetime(-1, create=True) + mgr.open_room_transfers(-1) + self.assertTrue(live.try_lease()) + with patch("sglang.srt.disaggregation.mooncake.conn.MAX_TRACKED_ROOMS", 4): + for room in range(200): + mgr._close_room_for_abort(room, b"") + # ... and rooms known only from decode metadata, which no local + # sender will ever release, must be bounded too. + for room in range(1000, 1200): + mgr._room_lifetime(room, create=True) + self.assertLessEqual(len(mgr._room_lifetimes), 8) + self.assertIn( + -1, mgr._room_lifetimes, "a claimed, leased room must never be evicted" + ) + + def test_emergency_reclaim_cost_does_not_grow_with_the_map(self): + # A saturated map of non-reclaimable rooms must not make every new room + # O(tracked rooms) on the bootstrap thread. + mgr = make_prefill_manager() + with patch("sglang.srt.disaggregation.mooncake.conn.MAX_TRACKED_ROOMS", 8): + for room in range(400): + lifetime = mgr._room_lifetime(room, create=True) + mgr.open_room_transfers(room) + self.assertTrue(lifetime.try_lease()) + probe = Mock(side_effect=lambda: False) + with patch.object(RoomTransferLifetime, "is_reclaimable", probe): + mgr._room_lifetime(99999, create=True) + self.assertLessEqual( + probe.call_count, + MAX_EMERGENCY_SCAN, + "the reclaim scan must be bounded per insert", + ) + + def test_sweep_reclaims_rooms_no_sender_will_release(self): + mgr = make_prefill_manager(room_sweep_ttl=0.0) + for room in range(5): + mgr._room_lifetime(room, create=True) # decode metadata only + mgr.open_room_transfers(3) # this one has a local sender + self.assertEqual(mgr._sweep_room_lifetimes(), 4) + self.assertEqual(list(mgr._room_lifetimes), [3]) + + def test_sweep_keeps_young_rooms_so_a_late_sender_still_finds_them(self): + mgr = make_prefill_manager(room_sweep_ttl=300.0) + mgr._room_lifetime(1, create=True) + self.assertEqual(mgr._sweep_room_lifetimes(), 0) + self.assertIn(1, mgr._room_lifetimes) + + def test_barrier_can_be_disabled(self): + mgr = make_prefill_manager(barrier=TransferBarrierLevel.OFF) + sender = make_sender(mgr) + lifetime = mgr._room_lifetime(ROOM, create=False) + self.assertTrue(lifetime.try_lease()) + with patch( + "sglang.srt.disaggregation.mooncake.conn.TRANSFER_QUIESCE_TIMEOUTS" + ) as metric: + self.assertTrue(sender.advance_failure_quiescence()) + metric.inc.assert_not_called() + + +class TestPrefillAbortProtocol(unittest.TestCase): + """Aborts arrive out of band and may overtake this rank's own bookkeeping.""" + + def _bootstrap_thread_state(self, mgr): + mgr._start_transfer_bookkeeping() + mgr._send_manager_message = Mock() + mgr.session_lock = threading.Lock() + mgr.failed_sessions = set() + mgr.session_failures = {} + mgr.decode_kv_args_table = {} + mgr.resolve_kv_replica_factor = Mock() + mgr._staging_ctx = None + return mgr + + @staticmethod + def _metadata_msg(room=ROOM, session=b"session:1", token=b"token", required=1): + return [ + str(room).encode("ascii"), + b"127.0.0.1", + b"1", + session, + np.array([0], dtype=np.int32).tobytes(), + b"0", + b"", + str(required).encode("ascii"), + b"0", + token, + ] + + def test_metadata_arriving_before_the_sender_is_applied_directly(self): + # Decode and prefill are separate processes with no ordering between + # them; metadata for a room this rank has not seen yet must still take + # effect (KV manager status is monotonic, so the later sender keeps it). + mgr = self._bootstrap_thread_state(make_prefill_manager()) + self.assertTrue(mgr._handle_bootstrap_metadata(self._metadata_msg())) + self.assertEqual(mgr.request_status[ROOM], KVPoll.WaitingForInput) + self.assertIn("session:1", mgr.transfer_infos[ROOM]) + self.assertTrue( + mgr._room_lifetime(ROOM, create=False).authorizes_abort(b"token") + ) + + def test_abort_arriving_before_the_sender_fails_the_request(self): + mgr = self._bootstrap_thread_state(make_prefill_manager()) + mgr._handle_abort_notification( + [b"ABORT", str(ROOM).encode("ascii"), b"127.0.0.1", b"1"] + ) + # A sender created afterwards must not transfer: open_room_transfers is + # what MooncakeKVSender.__init__ consults before accepting the request. + self.assertFalse(mgr.open_room_transfers(ROOM)) + # Metadata that arrives afterwards must not revive the room either. + self.assertFalse(mgr._handle_bootstrap_metadata(self._metadata_msg())) + self.assertNotIn(ROOM, mgr.transfer_infos) + + def test_abort_notification_is_acknowledged_even_for_an_unknown_room(self): + # Otherwise a decode rank waits out its whole timeout for a room this + # prefill has never heard of. + mgr = self._bootstrap_thread_state(make_prefill_manager()) + sent = [] + mgr._send_manager_message = lambda ip, port, parts, **kw: sent.append(parts) + mgr._handle_abort_notification([b"ABORT", b"4242", b"127.0.0.1", b"1", b"tok"]) + for _ in range(200): + if sent: + break + time.sleep(0.01) + self.assertEqual(sent, [[b"ABORT_ACK", b"4242", b"tok"]]) + + def test_a_foreign_token_cannot_close_a_live_room(self): + mgr = self._bootstrap_thread_state(make_prefill_manager()) + mgr._handle_bootstrap_metadata(self._metadata_msg(token=b"real")) + self.assertIsNone(mgr._close_room_for_abort(ROOM, b"stale")) + self.assertFalse(mgr._room_lifetime(ROOM, create=False).is_closed()) + self.assertIsNotNone(mgr._close_room_for_abort(ROOM, b"real")) + self.assertTrue(mgr._room_lifetime(ROOM, create=False).is_closed()) + + def test_abort_ack_is_withheld_until_the_room_drains(self): + mgr = make_prefill_manager() + mgr._start_transfer_bookkeeping() + sent = [] + mgr._send_manager_message = lambda ip, port, parts, **kw: sent.append(parts) + + lifetime = mgr._room_lifetime(ROOM, create=True) + self.assertTrue(lifetime.try_lease()) + lifetime.close() + + mgr.request_abort_ack(lifetime, ROOM, "127.0.0.1", 1, b"token") + mgr.request_abort_ack(lifetime, ROOM, "127.0.0.1", 1, b"token") + time.sleep(0.05) + self.assertEqual(sent, [], "ACK must not promise a drain that has not happened") + + lifetime.end_lease() + for _ in range(200): + if sent: + break + time.sleep(0.01) + self.assertEqual(sent, [[b"ABORT_ACK", str(ROOM).encode("ascii"), b"token"]]) + + def test_one_pump_thread_serves_every_pending_ack(self): + mgr = make_prefill_manager() + before = threading.active_count() + mgr._start_transfer_bookkeeping() + mgr._send_manager_message = Mock() + lifetimes = [] + for room in range(64): + lifetime = mgr._room_lifetime(room, create=True) + self.assertTrue(lifetime.try_lease()) + lifetime.close() + lifetimes.append(lifetime) + mgr.request_abort_ack(lifetime, room, "127.0.0.1", 1, b"t") + time.sleep(0.05) + self.assertLessEqual( + threading.active_count() - before, + 2, + "aborts must not spawn a thread each", + ) + for lifetime in lifetimes: + lifetime.end_lease() + + +class TestNoUnboundedWaits(unittest.TestCase): + """Nothing on the ownership path may wait on a peer without a bound. + + The barrier runs inside the scheduler loop and inside the one thread that + honours ABORT_ACKs, so a wedged peer must degrade, never stall. + """ + + def test_abort_notification_never_blocks_the_scheduler_loop(self): + # These sockets have no ZMQ send timeout, so a backed-up peer would + # otherwise stall the whole decode engine indefinitely. + mgr = make_decode_manager() + receiver = make_receiver(mgr, bootstrap_infos=[{"rank": 0}]) + used_nonblocking = [] + + class WouldBlock: + def send_multipart(self, parts, flags=0): + used_nonblocking.append(bool(flags & zmq.NOBLOCK)) + raise zmq.Again() + + receiver._connect_to_bootstrap_server = lambda info: ( + WouldBlock(), + threading.Lock(), + ) + + started = time.monotonic() + self.assertFalse(receiver.advance_failure_quiescence()) + self.assertLess(time.monotonic() - started, 1.0) + self.assertEqual(used_nonblocking, [True]) + # The dropped notification must be retried, not silently lost. + self.assertEqual(receiver._last_abort_send, float("-inf")) + + def test_bookkeeping_thread_survives_a_faulty_entry(self): + # It is the only thread that can honour an ABORT_ACK: if it dies, every + # decode peer of this instance waits out its full deadline forever. + mgr = make_prefill_manager() + mgr._start_transfer_bookkeeping() + thread = self._bookkeeping_thread() + sent = [] + mgr._send_manager_message = lambda ip, port, parts, **kw: sent.append(parts) + + class Faulty(RoomTransferLifetime): + def is_quiesced(self): + raise RuntimeError("backend bug") + + mgr.request_abort_ack(Faulty(), 1, "127.0.0.1", 1, b"bad") + healthy = RoomTransferLifetime() + healthy.close() + mgr.request_abort_ack(healthy, 2, "127.0.0.1", 1, b"good") + + for _ in range(200): + if sent: + break + time.sleep(0.01) + self.assertTrue(thread.is_alive()) + self.assertEqual( + sent, + [[b"ABORT_ACK", b"2", b"good"]], + "a faulty entry must not starve the ones behind it", + ) + + def test_a_backed_up_peer_does_not_delay_other_peers(self): + mgr = make_prefill_manager() + mgr._start_transfer_bookkeeping() + delivered, stuck = [], {"on": True} + + def send(ip, port, parts, nonblocking=False): + self.assertTrue(nonblocking, "ACKs must not block on a peer") + if parts[1] == b"1" and stuck["on"]: + raise zmq.Again() + delivered.append(parts[1]) + + mgr._send_manager_message = send + for room in (1, 2, 3): + lifetime = RoomTransferLifetime() + lifetime.close() + mgr.request_abort_ack(lifetime, room, "127.0.0.1", 1, b"t") + + for _ in range(200): + if len(delivered) >= 2: + break + time.sleep(0.01) + self.assertEqual(sorted(delivered), [b"2", b"3"]) + + stuck["on"] = False + for _ in range(200): + if len(delivered) >= 3: + break + time.sleep(0.01) + self.assertEqual( + sorted(delivered), + [b"1", b"2", b"3"], + "the ACK must be retried, not dropped", + ) + + def test_a_backend_fault_releases_the_request_instead_of_the_engine(self): + # The barrier is a safety optimisation: a bug in it must not take down + # the scheduler, nor pin a request's pages forever. + poller = SimpleNamespace( + poll=Mock(return_value=KVPoll.Failed), + advance_failure_quiescence=Mock(side_effect=RuntimeError("backend bug")), + is_failure_quiescing=Mock(return_value=True), + ) + with patch( + "sglang.srt.disaggregation.utils.dist.all_reduce", + side_effect=lambda tensor, **kw: None, + ): + self.assertEqual(poll_and_all_reduce([poller], object()), [KVPoll.Failed]) + + @staticmethod + def _bookkeeping_thread(): + return next( + t + for t in threading.enumerate() + if t.name == "MooncakeTransferBookkeeping" and t.is_alive() + ) + + +class TestStagingComposition(unittest.TestCase): + """Staging teardown is upstream's (release_room); this PR gates when it runs. + + release_room() drains the scatter stream and frees the room's reservations on + every teardown, which covers local CUDA work. It cannot know about a remote + prefill still writing into those buffers over RDMA -- that is what the ACK + gate provides, by making the terminal transition (and therefore teardown) + unreachable until every peer has confirmed. + """ + + def _handler(self): + allocator = SimpleNamespace( + free=Mock(), + get_watermark=Mock(return_value=(0, 0)), + _scatter_stream=None, + ) + staged = SimpleNamespace( + chunk_staging_infos=[(11, 0, 0, 128, 1), (12, 128, 0, 256, 1)], + prefill_info=SimpleNamespace(attn_tp_size=2), + bootstrap_infos=[], + _connect_to_bootstrap_server=Mock(), + ) + decode_req = SimpleNamespace( + kv_receiver=staged, req=SimpleNamespace(bootstrap_room=ROOM) + ) + handler = DecodeStagingHandler( + kv_manager=SimpleNamespace( + _staging_ctx=SimpleNamespace( + room_receivers={ROOM: staged}, room_bootstrap={ROOM: []} + ) + ), + staging_allocator=allocator, + kv_buffer_info={}, + decode_tp=1, + total_kv_heads=8, + tp_rank=0, + scheduler=SimpleNamespace(), + ) + handler.register_decode_req(ROOM, decode_req) + return handler, staged, allocator + + def test_teardown_releases_reservations(self): + # Upstream's behaviour, asserted here because this PR depends on it: the + # barrier no longer frees staging itself. + handler, staged, allocator = self._handler() + handler.unregister_decode_req(ROOM) + self.assertEqual( + sorted(call.args[0] for call in allocator.free.call_args_list), [11, 12] + ) + self.assertEqual(staged.chunk_staging_infos, [(-1, -1, 0, -1, 0)] * 2) + + def test_teardown_is_unreachable_until_peers_confirm(self): + # The composition that makes release_room safe against a remote writer. + mgr = make_decode_manager() + receiver = make_receiver(mgr, bootstrap_infos=[{"rank": 0}]) + self.assertFalse( + receiver.advance_failure_quiescence(), + "the request must not go terminal, so unregister_decode_req cannot run", + ) + receiver.record_abort_ack(next(iter(receiver._expected_abort_acks))) + self.assertTrue(receiver.advance_failure_quiescence()) + + def test_watermark_broadcast_never_blocks_the_scheduler_loop(self): + # release_room broadcasts a watermark from the scheduler loop over sockets + # with no send timeout, so it must not wait for a peer. + handler, _staged, _allocator = self._handler() + flags_used = [] + + class WouldBlock: + def send_multipart(self, parts, flags=0): + flags_used.append(bool(flags & zmq.NOBLOCK)) + raise zmq.Again() + + handler.register_wm_subscriber( + SimpleNamespace( + bootstrap_infos=[{"rank": 0}], + _connect_to_bootstrap_server=lambda info: ( + WouldBlock(), + threading.Lock(), + ), + ), + "session", + ) + started = time.monotonic() + handler.unregister_decode_req(ROOM) + self.assertLess(time.monotonic() - started, 1.0) + self.assertTrue(flags_used and all(flags_used)) + + +class TestDecodeOwnership(unittest.TestCase): + """The decode's pages are the transfer destination, so peers must confirm.""" + + def test_receiver_without_metadata_is_immediately_quiesced(self): + # This is what makes an abort during bootstrap/preallocation terminal in + # one step: no prefill rank has been told where to write. + mgr = make_decode_manager() + receiver = make_receiver(mgr, metadata_sent=False) + self.assertTrue(receiver.advance_failure_quiescence()) + + def test_receiver_waits_for_every_peer_then_releases(self): + mgr = make_decode_manager() + receiver = make_receiver(mgr, bootstrap_infos=[{"rank": 0}, {"rank": 1}]) + + self.assertFalse(receiver.advance_failure_quiescence()) + tokens = sorted(receiver._expected_abort_acks) + self.assertEqual(len(tokens), 2, "one nonce per prefill rank") + + receiver.record_abort_ack(tokens[0]) + self.assertFalse(receiver.advance_failure_quiescence()) + receiver.record_abort_ack(tokens[1]) + self.assertTrue(receiver.advance_failure_quiescence()) + + def test_abort_notifications_carry_a_stable_token_and_are_retried(self): + mgr = make_decode_manager() + sockets = {0: Mock(), 1: Mock()} + receiver = make_receiver(mgr, bootstrap_infos=[{"rank": 0}, {"rank": 1}]) + receiver._connect_to_bootstrap_server = lambda info: ( + sockets[info["rank"]], + threading.Lock(), + ) + + clock = [0.0] + with patch( + "sglang.srt.disaggregation.mooncake.conn.time.monotonic", + side_effect=lambda: clock[0], + ): + receiver._send_abort_notification() + tokens = { + rank: socket.send_multipart.call_args.args[0][4] + for rank, socket in sockets.items() + } + clock[0] = ABORT_RETRY_INTERVAL_S / 2 + receiver._send_abort_notification() # rate limited + receiver.record_abort_ack(tokens[0]) + clock[0] = ABORT_RETRY_INTERVAL_S * 2 + receiver._send_abort_notification() # retry rank 1 only + + self.assertEqual(sockets[0].send_multipart.call_count, 1) + self.assertEqual(sockets[1].send_multipart.call_count, 2) + self.assertEqual( + sockets[1].send_multipart.call_args_list[1].args[0][4], + tokens[1], + "a retry must reuse the nonce so the ACK still matches", + ) + + def test_quiescence_is_bounded_when_a_peer_never_acknowledges(self): + mgr = make_decode_manager(quiesce_timeout=0.05) + receiver = make_receiver(mgr, bootstrap_infos=[{"rank": 0}]) + + with patch( + "sglang.srt.disaggregation.mooncake.conn.TRANSFER_QUIESCE_TIMEOUTS" + ) as metric: + self.assertFalse(receiver.advance_failure_quiescence()) + time.sleep(0.06) + self.assertTrue( + receiver.advance_failure_quiescence(), + "a dead prefill must not pin decode pages forever", + ) + metric.inc.assert_called_once_with() + + def test_legacy_prefill_ack_stops_the_wait(self): + # A prefill that predates the barrier ACKs immediately and without a + # nonce. It cannot promise a drain, so waiting for it is pointless; + # degrade to the previous behaviour instead of stalling. + mgr = make_decode_manager() + receiver = make_receiver(mgr, bootstrap_infos=[{"rank": 0}]) + self.assertFalse(receiver.advance_failure_quiescence()) + receiver.record_abort_ack(None) + self.assertTrue(receiver.advance_failure_quiescence()) + + def test_stale_ack_for_another_request_is_ignored(self): + mgr = make_decode_manager() + receiver = make_receiver(mgr, bootstrap_infos=[{"rank": 0}]) + receiver.advance_failure_quiescence() + receiver.record_abort_ack(b"nonce-from-a-previous-request") + self.assertFalse(receiver.advance_failure_quiescence()) + + def test_abort_ack_routes_to_the_registered_receiver_only(self): + mgr = make_decode_manager() + receiver = make_receiver(mgr, bootstrap_infos=[{"rank": 0}]) + receiver.record_abort_ack = Mock() + mgr._receivers[ROOM] = receiver + + mgr._handle_abort_ack([b"ABORT_ACK", str(ROOM).encode("ascii"), b"tok"]) + receiver.record_abort_ack.assert_called_once_with(b"tok") + # Unknown rooms must not raise on the transport thread. + mgr._handle_abort_ack([b"ABORT_ACK", b"999", b"tok"]) + + def test_only_one_receiver_owns_a_room(self): + mgr = make_decode_manager() + first = make_receiver(mgr) + self.assertTrue(mgr.register_receiver(first)) + second = make_receiver(mgr) + self.assertFalse( + mgr.register_receiver(second), + "colliding rooms must not let a second receiver take over", + ) + self.assertIs(mgr._receivers[ROOM], first) + + # The loser must not evict the owner's registration on cleanup. + mgr.unregister_receiver(second) + self.assertIs(mgr._receivers[ROOM], first) + mgr.unregister_receiver(first) + self.assertTrue(mgr.register_receiver(second)) + + def test_a_receiver_that_lost_the_room_touches_no_shared_state(self): + handler = Mock() + mgr = make_decode_manager(staging_handler=handler) + loser = make_receiver(mgr, metadata_sent=False) + loser._owns_room = False + self.assertTrue(loser.advance_failure_quiescence()) + handler.begin_abort.assert_not_called() + loser.clear = MooncakeKVReceiver.clear.__get__(loser) + handler.forget_abort.assert_not_called() + + def test_late_transport_messages_are_dropped_while_tearing_down(self): + mgr = make_decode_manager() + receiver = make_receiver(mgr) + mgr._receivers[ROOM] = receiver + self.assertFalse(mgr._is_tearing_down(ROOM)) + receiver._close_barrier() + self.assertTrue(mgr._is_tearing_down(ROOM)) + self.assertFalse(mgr._is_tearing_down(999)) + + +class TestBarrierLevels(unittest.TestCase): + """What happens when proof of quiescence never arrives. + + WARN trades a narrow, loud corruption window for liveness; STRICT refuses to + reuse a page it cannot prove is idle, and escalates instead. + """ + + def _stuck_sender(self, barrier, max_unquiesced=256): + mgr = make_prefill_manager(quiesce_timeout=0.01, barrier=barrier) + mgr.max_unquiesced = max_unquiesced + sender = make_sender(mgr) + lifetime = mgr._room_lifetime(ROOM, create=False) + self.assertTrue(lifetime.try_lease()) # a worker that never returns + return mgr, sender + + @staticmethod + def _arm_and_expire(sender): + """Arm the deadline, then let it pass.""" + with patch("sglang.srt.disaggregation.mooncake.conn.TRANSFER_QUIESCE_TIMEOUTS"): + sender.advance_failure_quiescence() + time.sleep(0.02) + + def test_warn_releases_once_and_loudly(self): + mgr, sender = self._stuck_sender(TransferBarrierLevel.WARN) + self.assertFalse(sender.advance_failure_quiescence()) + time.sleep(0.02) + with patch( + "sglang.srt.disaggregation.mooncake.conn.TRANSFER_QUIESCE_TIMEOUTS" + ) as metric: + self.assertTrue(sender.advance_failure_quiescence()) + self.assertTrue(sender.advance_failure_quiescence()) + metric.inc.assert_called_once_with() + + def test_strict_never_releases_without_proof(self): + mgr, sender = self._stuck_sender(TransferBarrierLevel.STRICT) + self._arm_and_expire(sender) + with patch("sglang.srt.disaggregation.mooncake.conn.TRANSFER_QUIESCE_TIMEOUTS"): + for _ in range(5): + self.assertFalse( + sender.advance_failure_quiescence(), + "STRICT must not hand back a page it cannot prove is idle", + ) + self.assertIn( + ROOM, mgr._unquiesced_rooms, "the stuck room must be tracked for escalation" + ) + + def test_strict_escalates_rather_than_leaking_forever(self): + # Withholding pages is only safe while there are few of them; past the + # cap the worker fails so a restart reclaims them all at once. + mgr, sender = self._stuck_sender(TransferBarrierLevel.STRICT, max_unquiesced=2) + self._arm_and_expire(sender) + with patch("sglang.srt.disaggregation.mooncake.conn.TRANSFER_QUIESCE_TIMEOUTS"): + sender.advance_failure_quiescence() + mgr._unquiesced_rooms.add(999) # a second stuck request + with self.assertRaises(RuntimeError) as cm: + mgr.release_without_proof(1234, "third stuck request") + self.assertIn("cannot be proven quiesced", str(cm.exception)) + + def test_off_restores_legacy_behaviour(self): + mgr, sender = self._stuck_sender(TransferBarrierLevel.OFF) + with patch( + "sglang.srt.disaggregation.mooncake.conn.TRANSFER_QUIESCE_TIMEOUTS" + ) as metric: + self.assertTrue(sender.advance_failure_quiescence()) + metric.inc.assert_not_called() + + def test_a_receiver_that_lost_its_room_does_not_wait_for_unroutable_proof(self): + mgr = make_decode_manager() + receiver = make_receiver(mgr, bootstrap_infos=[{"rank": 0}]) + receiver._owns_room = False + with patch("sglang.srt.disaggregation.mooncake.conn.TRANSFER_QUIESCE_TIMEOUTS"): + self.assertTrue( + receiver.advance_failure_quiescence(), + "ACKs are routed to the owning receiver, so this one can never " + "receive proof and must not wait for it", + ) + + +class TestStrictCannotBeDowngraded(unittest.TestCase): + """STRICT must not degrade into the most permissive policy by accident.""" + + def test_escalation_reaches_the_scheduler(self): + # The poll path contains backend faults so a bug cannot kill the engine. + # The escalation is not a bug: swallowing it would release exactly the + # pages STRICT exists to withhold. + poller = SimpleNamespace( + poll=Mock(return_value=KVPoll.Failed), + advance_failure_quiescence=Mock( + side_effect=KVTransferBarrierEscalation("cannot prove idle") + ), + is_failure_quiescing=Mock(return_value=True), + ) + with patch( + "sglang.srt.disaggregation.utils.dist.all_reduce", + side_effect=lambda tensor, **kw: None, + ): + with self.assertRaises(KVTransferBarrierEscalation): + poll_and_all_reduce([poller], object()) + + def test_an_ordinary_backend_fault_is_still_contained(self): + poller = SimpleNamespace( + poll=Mock(return_value=KVPoll.Failed), + advance_failure_quiescence=Mock(side_effect=RuntimeError("backend bug")), + is_failure_quiescing=Mock(return_value=True), + ) + with patch( + "sglang.srt.disaggregation.utils.dist.all_reduce", + side_effect=lambda tensor, **kw: None, + ): + self.assertEqual(poll_and_all_reduce([poller], object()), [KVPoll.Failed]) + + def test_strict_does_not_accept_a_legacy_ack_as_proof(self): + # A peer that predates the barrier acknowledges before draining, so its + # ACK proves nothing. WARN accepts it to keep a rolling upgrade working; + # STRICT cannot, by definition. + for level, accepted in ( + (TransferBarrierLevel.WARN, True), + (TransferBarrierLevel.STRICT, False), + ): + with self.subTest(level=level): + mgr = make_decode_manager(barrier=level) + receiver = make_receiver(mgr, bootstrap_infos=[{"rank": 0}]) + receiver.advance_failure_quiescence() + receiver.record_abort_ack(None) + self.assertEqual(receiver._peers_quiesced(), accepted) + + +class TestRoomStateRetirement(unittest.TestCase): + """Reclaiming a room must retire everything keyed by it.""" + + def _room_with_metadata(self, mgr, room=ROOM): + mgr.session_lock = threading.Lock() + mgr.failed_sessions, mgr.session_failures = set(), {} + mgr.decode_kv_args_table = {} + mgr.resolve_kv_replica_factor = Mock() + mgr._handle_bootstrap_metadata( + [ + str(room).encode("ascii"), + b"127.0.0.1", + b"1", + b"session:1", + np.array([0], dtype=np.int32).tobytes(), + b"0", + b"", + b"1", + b"0", + b"token", + ] + ) + + def test_emergency_reclaim_retires_destinations_and_status_too(self): + # Same hazard as the sweep, via the other reclaim path: a request drawing + # a reclaimed room must not inherit the old decode's addresses. + mgr = make_prefill_manager() + self._room_with_metadata(mgr) + self.assertIn(ROOM, mgr.transfer_infos) + with patch("sglang.srt.disaggregation.mooncake.conn.MAX_TRACKED_ROOMS", 1): + mgr._room_lifetime(ROOM + 1, create=True) + self.assertNotIn(ROOM, mgr._room_lifetimes) + self.assertNotIn(ROOM, mgr.transfer_infos) + self.assertNotIn(ROOM, mgr.request_status) + self.assertNotIn(ROOM, mgr.req_to_decode_prefix_len) + + def test_retirement_is_atomic_against_a_new_generation(self): + # The lifetime and the rest of the room's state must go in one step: a + # request created in between would have its own metadata deleted. + mgr = make_prefill_manager(room_sweep_ttl=0.0) + self._room_with_metadata(mgr) + observed = {} + real_retire = mgr._retire_rooms_locked + + def retire(rooms): + observed["held"] = mgr._room_lifetimes_lock.locked() + real_retire(rooms) + + mgr._retire_rooms_locked = retire + self.assertEqual(mgr._sweep_room_lifetimes(), 1) + self.assertTrue(observed["held"], "retirement must run under the lifetime lock") + + def test_sweep_retires_destinations_and_status_together(self): + # Otherwise a later request drawing the same bootstrap_room inherits the + # old decode's addresses and a stale status. + mgr = make_prefill_manager(room_sweep_ttl=0.0) + mgr.session_lock = threading.Lock() + mgr.failed_sessions, mgr.session_failures = set(), {} + mgr.decode_kv_args_table = {} + mgr.resolve_kv_replica_factor = Mock() + mgr._handle_bootstrap_metadata( + [ + str(ROOM).encode("ascii"), + b"127.0.0.1", + b"1", + b"session:1", + np.array([0], dtype=np.int32).tobytes(), + b"0", + b"", + b"1", + b"0", + b"token", + ] + ) + self.assertIn(ROOM, mgr.transfer_infos) + self.assertIn(ROOM, mgr.request_status) + + self.assertEqual(mgr._sweep_room_lifetimes(), 1) + self.assertNotIn(ROOM, mgr._room_lifetimes) + self.assertNotIn(ROOM, mgr.transfer_infos) + self.assertNotIn(ROOM, mgr.request_status) + self.assertNotIn(ROOM, mgr.req_to_decode_prefix_len) + + +class TestRoomCollisionIsolation(unittest.TestCase): + """A receiver that lost a room collision must not damage the owner.""" + + def test_the_loser_does_not_tear_down_shared_state(self): + mgr = make_decode_manager() + owner = make_receiver(mgr) + self.assertTrue(mgr.register_receiver(owner)) + loser = make_receiver(mgr) + loser._owns_room = mgr.register_receiver(loser) + self.assertFalse(loser._owns_room) + + mgr.request_status[ROOM] = KVPoll.WaitingForInput + mgr.required_prefill_response_num_table = {ROOM: 1} + mgr.prefill_response_tracker = {ROOM: set()} + + loser.clear() + + self.assertIn(ROOM, mgr.request_status, "the owner's status must survive") + self.assertIn(ROOM, mgr.required_prefill_response_num_table) + self.assertIs(mgr._receivers[ROOM], owner) + + +class TestWatermarkDelivery(unittest.TestCase): + """A dropped watermark must be retried; it can gate a prefill's progress.""" + + def _handler(self, socket): + allocator = SimpleNamespace( + free=Mock(), get_watermark=Mock(return_value=(3, 128)), _scatter_stream=None + ) + handler = DecodeStagingHandler( + kv_manager=SimpleNamespace( + _staging_ctx=SimpleNamespace(room_receivers={}, room_bootstrap={}) + ), + staging_allocator=allocator, + kv_buffer_info={}, + decode_tp=1, + total_kv_heads=8, + tp_rank=0, + scheduler=SimpleNamespace(), + ) + handler.register_wm_subscriber( + SimpleNamespace( + bootstrap_infos=[{"rank": 0}], + _connect_to_bootstrap_server=lambda info: (socket, threading.Lock()), + ), + "session", + ) + return handler + + def test_a_backpressured_watermark_is_resent(self): + state = {"blocked": True} + sent = [] + + class Socket: + def send_multipart(self, parts, flags=0): + if state["blocked"]: + raise zmq.Again() + sent.append(parts) + + handler = self._handler(Socket()) + handler._free_and_send_watermark( + 1, SimpleNamespace(req=SimpleNamespace(bootstrap_room=ROOM)) + ) + self.assertEqual(sent, [], "the send was backpressured") + self.assertTrue(handler._wm_retry, "the subscriber must be remembered") + + state["blocked"] = False + handler.retry_pending_watermarks() + self.assertEqual(len(sent), 1, "the current watermark must be resent") + self.assertEqual(sent[0][1:3], [b"3", b"128"]) + self.assertFalse(handler._wm_retry) + + def test_retry_is_free_when_nothing_is_pending(self): + sent = [] + + class Socket: + def send_multipart(self, parts, flags=0): + sent.append(parts) + + handler = self._handler(Socket()) + handler.retry_pending_watermarks() + self.assertEqual(sent, []) + + +class TestOwedAckBookkeeping(unittest.TestCase): + """A peer that never drains its socket must not accumulate work forever.""" + + def test_an_undeliverable_ack_is_eventually_given_up_on(self): + mgr = make_prefill_manager() + mgr._start_transfer_bookkeeping() + mgr._send_manager_message = Mock(side_effect=zmq.Again()) + lifetime = RoomTransferLifetime() + lifetime.close() + mgr.request_abort_ack(lifetime, ROOM, "127.0.0.1", 1, b"t") + with mgr._abort_ack_lock: + self.assertEqual(len(mgr._abort_ack_pending), 1) + # Expire it rather than waiting out ABORT_ACK_MAX_AGE_S. + with patch("sglang.srt.disaggregation.mooncake.conn.ABORT_ACK_MAX_AGE_S", 0.0): + mgr.request_abort_ack(lifetime, ROOM + 1, "127.0.0.1", 1, b"t2") + for _ in range(300): + with mgr._abort_ack_lock: + if len(mgr._abort_ack_pending) < 2: + break + time.sleep(0.01) + with mgr._abort_ack_lock: + self.assertLess(len(mgr._abort_ack_pending), 2) + + def test_owed_acks_are_capped(self): + mgr = make_prefill_manager() + mgr._abort_ack_queue = __import__("queue").Queue() + mgr._abort_ack_pending = set() + mgr._abort_ack_lock = threading.Lock() + with patch("sglang.srt.disaggregation.mooncake.conn.MAX_PENDING_ABORT_ACKS", 4): + for room in range(50): + mgr.request_abort_ack(None, room, "127.0.0.1", 1, b"t") + self.assertLessEqual(len(mgr._abort_ack_pending), 4) + + +class TestPollGating(unittest.TestCase): + """How the deferral is surfaced to the scheduler.""" + + @staticmethod + def _reduce_min(tensor, **_kwargs): + return None # single rank: MIN over one rank is the identity + + def test_failed_is_withheld_until_quiesced_then_reported(self): + poller = SimpleNamespace( + poll=Mock(return_value=KVPoll.Failed), + advance_failure_quiescence=Mock(side_effect=[False, True]), + is_failure_quiescing=Mock(return_value=True), + ) + with patch( + "sglang.srt.disaggregation.utils.dist.all_reduce", + side_effect=self._reduce_min, + ): + self.assertEqual( + poll_and_all_reduce([poller], object()), [KVPoll.Transferring] + ) + self.assertEqual(poll_and_all_reduce([poller], object()), [KVPoll.Failed]) + + def test_healthy_polls_take_no_extra_collective(self): + poller = SimpleNamespace( + poll=Mock(return_value=KVPoll.WaitingForInput), + advance_failure_quiescence=Mock(), + is_failure_quiescing=Mock(return_value=False), + ) + with patch( + "sglang.srt.disaggregation.utils.dist.all_reduce", + side_effect=self._reduce_min, + ) as all_reduce: + self.assertEqual( + poll_and_all_reduce([poller], object()), [KVPoll.WaitingForInput] + ) + self.assertEqual(all_reduce.call_count, 1) + poller.advance_failure_quiescence.assert_not_called() + + def test_a_quiescing_rank_forces_the_group_to_fail(self): + # Otherwise one rank could release pages while another kept transferring. + poller = SimpleNamespace( + poll=Mock(return_value=KVPoll.WaitingForInput), + advance_failure_quiescence=Mock(return_value=True), + is_failure_quiescing=Mock(return_value=True), + ) + with patch( + "sglang.srt.disaggregation.utils.dist.all_reduce", + side_effect=self._reduce_min, + ): + self.assertEqual(poll_and_all_reduce([poller], object()), [KVPoll.Failed]) + + def test_bootstrap_queue_tolerates_a_deferred_terminal_state(self): + # Regression guard: pop_bootstrapped and handle_pending_bootstrap raise on + # unexpected poll states, so the deferral must be an accepted state there. + from sglang.srt.disaggregation.prefill import ( + SchedulerDisaggregationPrefillMixin, + ) + + handle = SchedulerDisaggregationPrefillMixin.handle_pending_bootstrap + self.assertFalse( + handle(SimpleNamespace(), SimpleNamespace(rid="r"), KVPoll.Transferring) + ) + + def test_hicache_wrapper_delegates_the_barrier(self): + from sglang.srt.disaggregation.decode_hicache_mixin import ( + HiCacheRestoreGatedKVReceiver, + ) + + receiver = SimpleNamespace( + poll=Mock(return_value=KVPoll.Failed), + advance_failure_quiescence=Mock(return_value=False), + is_failure_quiescing=Mock(return_value=True), + ) + gated = HiCacheRestoreGatedKVReceiver( + SimpleNamespace(kv_receiver=receiver, hicache_restore_status=None) + ) + self.assertFalse(gated.advance_failure_quiescence()) + self.assertTrue(gated.is_failure_quiescing()) + + +class TestFailureDiagnostics(unittest.TestCase): + """Releasing pages safely must not cost the operator the real reason.""" + + def test_propagated_failures_keep_their_attribution(self): + mgr = make_decode_manager() + receiver = make_receiver(mgr, bootstrap_infos=[{"rank": 0}]) + # A failure reported by the prefill (or another rank) drives quiescence, + # but must not be relabelled as a local user abort. + receiver.advance_failure_quiescence() + mgr.record_failure.assert_not_called() + + def test_user_abort_still_records_its_reason(self): + mgr = make_decode_manager() + receiver = make_receiver(mgr, bootstrap_infos=[{"rank": 0}]) + receiver.abort_notified = False + MooncakeKVReceiver.abort(receiver) + mgr.record_failure.assert_called_once() + self.assertIn("Aborted by AbortReq", mgr.record_failure.call_args.args[1]) + + +class TestChunkedPrefillAbort(unittest.TestCase): + """Aborting a chunked prefill must not free pages an earlier chunk is reading.""" + + @staticmethod + def _scheduler(req, mode): + return SimpleNamespace( + _pending_chunked_abort_req=req, + chunked_req=req, + enable_hicache_storage=False, + tree_cache=Mock(), + disaggregation_mode=mode, + disagg_prefill_inflight_queue=[], + ipc_channels=SimpleNamespace( + send_to_tokenizer=SimpleNamespace(send_output=Mock()) + ), + ) + + @staticmethod + def _chunked_req(): + return SimpleNamespace( + rid="chunked", + disagg_kv_sender=Mock(), + time_stats=SimpleNamespace(trace_ctx=Mock()), + to_finish=object(), + skip_radix_cache_insert=False, + ) + + def _run(self, mode): + from sglang.srt.managers import scheduler as scheduler_mod + + req = self._chunked_req() + scheduler = self._scheduler(req, mode) + with patch.object( + scheduler_mod, "prepare_abort" + ) as prepare_abort, patch.object( + scheduler_mod, "release_kv_cache" + ) as release_kv_cache: + scheduler_mod.Scheduler.process_pending_chunked_abort(scheduler) + return req, scheduler, prepare_abort, release_kv_cache + + def test_prefill_defers_release_to_the_inflight_queue(self): + req, scheduler, prepare_abort, release_kv_cache = self._run( + DisaggregationMode.PREFILL + ) + prepare_abort.assert_called_once() + req.disagg_kv_sender.abort.assert_called_once_with() + release_kv_cache.assert_not_called() + self.assertEqual(scheduler.disagg_prefill_inflight_queue, [req]) + self.assertTrue( + req.skip_radix_cache_insert, + "the aborted partial prefix must not enter the radix cache", + ) + self.assertIsNone(scheduler.chunked_req) + self.assertIsNone(scheduler._pending_chunked_abort_req) + + def test_non_disagg_release_is_unchanged(self): + req, scheduler, _prepare_abort, release_kv_cache = self._run( + DisaggregationMode.DECODE + ) + release_kv_cache.assert_called_once() + self.assertEqual(release_kv_cache.call_args.kwargs, {"is_insert": False}) + self.assertEqual(scheduler.disagg_prefill_inflight_queue, []) + scheduler.ipc_channels.send_to_tokenizer.send_output.assert_called_once() + + def test_deferral_is_idempotent(self): + from sglang.srt.managers import scheduler as scheduler_mod + + req = self._chunked_req() + scheduler = self._scheduler(req, DisaggregationMode.PREFILL) + with patch.object(scheduler_mod, "prepare_abort"), patch.object( + scheduler_mod, "release_kv_cache" + ): + scheduler_mod.Scheduler.process_pending_chunked_abort(scheduler) + scheduler._pending_chunked_abort_req = req + scheduler.chunked_req = req + scheduler_mod.Scheduler.process_pending_chunked_abort(scheduler) + self.assertEqual(scheduler.disagg_prefill_inflight_queue, [req]) + + +class TestTransferInfoWire(unittest.TestCase): + """The abort nonce is an optional trailing frame.""" + + def test_metadata_without_a_token_parses_as_legacy(self): + msg = [ + b"7", + b"127.0.0.1", + b"1", + b"session", + np.array([0], dtype=np.int32).tobytes(), + b"0", + b"", + b"1", + b"0", + ] + info = TransferInfo.from_zmq(msg) + self.assertEqual(info.room, 7) + self.assertEqual(info.abort_token, b"") + + def test_metadata_with_a_token_parses_it(self): + msg = [ + b"7", + b"127.0.0.1", + b"1", + b"session", + np.array([0], dtype=np.int32).tobytes(), + b"0", + b"", + b"1", + b"0", + b"nonce", + ] + self.assertEqual(TransferInfo.from_zmq(msg).abort_token, b"nonce") + + +if __name__ == "__main__": + unittest.main() From b674046ef715a1da9175a89aaa60e17b39b1405d Mon Sep 17 00:00:00 2001 From: Kastan Day Date: Tue, 28 Jul 2026 09:44:29 -0700 Subject: [PATCH 02/10] fix(disaggregation): harden transfer barrier coordination --- .../srt/disaggregation/mooncake/conn.py | 76 +++++++++++++++---- python/sglang/srt/disaggregation/utils.py | 34 +++++++-- .../test_transfer_ownership_barrier.py | 67 +++++++++++++++- 3 files changed, 152 insertions(+), 25 deletions(-) diff --git a/python/sglang/srt/disaggregation/mooncake/conn.py b/python/sglang/srt/disaggregation/mooncake/conn.py index 0e3fa69d6eed..2bfdf6bd73e0 100644 --- a/python/sglang/srt/disaggregation/mooncake/conn.py +++ b/python/sglang/srt/disaggregation/mooncake/conn.py @@ -171,11 +171,11 @@ def is_claimed(self) -> bool: return self._claimed def is_reclaimable(self) -> bool: - """Whether forgetting this room can no longer affect a live transfer. + """Whether this room has no active local transfer ownership. - Safe in both cases: a quiesced room admits no work and has none running, - and an unclaimed room has no sender, so it can never take a lease (only - ``add_transfer_request`` enqueues work, and only a sender calls it). + A quiesced room admits no work and has none running. An unclaimed room + has no sender *yet*, so callers must additionally preserve it for the + bootstrap grace period in which a metadata-late sender may still arrive. """ with self._cond: return (not self._open and self._leases == 0) or not self._claimed @@ -478,23 +478,26 @@ def _emergency_reclaim_rooms_locked(self) -> None: """Bounded reclaim when the periodic sweep is not keeping up. Scans at most ``MAX_EMERGENCY_SCAN`` of the oldest entries so an insert - can never degrade to O(tracked rooms); the sweep does the rest. + can never degrade to O(tracked rooms); the sweep does the rest. The + normal bootstrap grace period still applies: the cap is soft when every + candidate may still be waiting for its local sender. """ # islice over the live view keeps this O(MAX_EMERGENCY_SCAN); the keys are # copied out first because the dict cannot be mutated while iterating. + cutoff = time.monotonic() - self._room_sweep_ttl candidates = [ room for room, lifetime in itertools.islice( self._room_lifetimes.items(), MAX_EMERGENCY_SCAN ) - if lifetime.is_reclaimable() + if lifetime.created_at <= cutoff and lifetime.is_reclaimable() ] self._retire_rooms_locked(candidates) if not candidates: logger.warning_once( - "Tracking more than %d Mooncake bootstrap rooms with none " - "reclaimable; KV metadata may be arriving for rooms this rank " - "never serves.", + "Tracking more than %d Mooncake bootstrap rooms with none old " + "enough and reclaimable; allowing the soft cap to grow while " + "metadata-first rooms remain inside their bootstrap window.", MAX_TRACKED_ROOMS, ) @@ -2679,20 +2682,43 @@ def __init__( self._peer_lacks_barrier = False self._last_abort_send = float("-inf") self._abort_lock = threading.Lock() - super().__init__(mgr, bootstrap_addr, bootstrap_room) # Per-room state (ABORT_ACK routing, staging teardown) may only be # touched by the receiver that owns the room. A second live receiver for # the same bootstrap_room means the room numbers collided upstream: - # everything keyed by room is then ambiguous, so say so loudly rather - # than corrupt the owner's state. Abort nonces are per receiver, so this - # one still terminates -- via its quiescence deadline. + # everything keyed by room is then ambiguous. Claim before the common + # initializer writes any room-keyed state so the loser cannot change the + # owner's status or address tracking. + self.bootstrap_room = bootstrap_room + self.bootstrap_addr = bootstrap_addr + self.kv_mgr = mgr self._owns_room = mgr.register_receiver(self) if not self._owns_room: + self.conclude_state = KVPoll.Failed + self.require_staging = False + self.init_time = None + self.abort_notified = False + self._room_collision_error = ( + f"bootstrap_room {self.bootstrap_room} is already in use by an " + "unfinished KV transfer" + ) logger.error( - "bootstrap_room %s is already in use by an unfinished KV transfer; " - "colliding bootstrap rooms make per-room state ambiguous", - self.bootstrap_room, + "%s; rejecting the colliding receiver before it can touch " + "room-keyed state", + self._room_collision_error, ) + return + try: + super().__init__(mgr, bootstrap_addr, bootstrap_room) + except Exception: + # Do not strand the room registration if construction fails before + # the receiver becomes operational. + mgr.unregister_receiver(self) + raise + + def init(self, prefill_dp_rank: int): + if not self._owns_room: + return + super().init(prefill_dp_rank) def _register_kv_args(self) -> bool: for bootstrap_info in self.bootstrap_infos: @@ -2781,6 +2807,11 @@ def send_metadata( state_indices: Optional[List] = None, decode_prefix_len: Optional[int] = None, ): + if not self._owns_room: + # The owner is the only receiver allowed to publish destinations for + # a room. Sending the loser's addresses would redirect the owner's + # transfer into unrelated pages. + return if self.bootstrap_infos is None: self.kv_mgr.record_failure( self.bootstrap_room, @@ -2857,6 +2888,12 @@ def poll(self) -> KVPoll: # ------------------------------------------------------------------ def abort(self): + if not self._owns_room: + # CommonKVReceiver.abort() writes failure state keyed only by room, + # which belongs to the winning receiver. + self.conclude_state = KVPoll.Failed + self._close_barrier() + return super().abort() # Stop admitting staging work now rather than on the next poll. self._close_barrier() @@ -3023,6 +3060,13 @@ def failure_exception(self): self.clear() + if not self._owns_room: + raise KVTransferError( + self.bootstrap_room, + self._room_collision_error, + is_from_another_rank=False, + ) + with self.kv_mgr.failure_lock: failure_reason = self.kv_mgr.failure_records.pop(self.bootstrap_room, None) is_propagated = failure_reason is None diff --git a/python/sglang/srt/disaggregation/utils.py b/python/sglang/srt/disaggregation/utils.py index 815b3912c0ca..b19a6e86f30f 100644 --- a/python/sglang/srt/disaggregation/utils.py +++ b/python/sglang/srt/disaggregation/utils.py @@ -190,14 +190,34 @@ def _gate_failures_on_quiescence(pollers, polls: List[int], groups) -> List[int] if not any(failed): return polls - quiesced = [ - _advance_quiescence(poller) if is_failed else 1 - for poller, is_failed in zip(pollers, failed) - ] - quiesced = _reduce_polls(quiesced, groups) + # Three ordered states let MIN carry both ownership and escalation through + # the same collective: + # 0 = a rank requires worker teardown, 1 = still owns pages, 2 = quiesced. + # In particular, a local KVTransferBarrierEscalation must not escape before + # peers enter this collective or they will wait forever. + local_escalation = None + quiescence_states = [] + for poller, is_failed in zip(pollers, failed): + if not is_failed: + quiescence_states.append(2) + continue + try: + quiescence_states.append(1 + _advance_quiescence(poller)) + except KVTransferBarrierEscalation as exc: + local_escalation = exc + quiescence_states.append(0) + + quiescence_states = _reduce_polls(quiescence_states, groups) + if any(state == 0 for state in quiescence_states): + if local_escalation is not None: + raise local_escalation + raise KVTransferBarrierEscalation( + "A peer cannot prove that KV transfers are quiesced; escalating on " + "every participant after transfer-barrier coordination." + ) return [ - int(KVPoll.Transferring) if is_failed and not ok else poll - for poll, is_failed, ok in zip(polls, failed, quiesced) + int(KVPoll.Transferring) if is_failed and state == 1 else poll + for poll, is_failed, state in zip(polls, failed, quiescence_states) ] diff --git a/test/registered/unit/disaggregation/test_transfer_ownership_barrier.py b/test/registered/unit/disaggregation/test_transfer_ownership_barrier.py index e27613e0038c..ca7e1eb493a5 100644 --- a/test/registered/unit/disaggregation/test_transfer_ownership_barrier.py +++ b/test/registered/unit/disaggregation/test_transfer_ownership_barrier.py @@ -547,11 +547,11 @@ def test_tracked_rooms_are_bounded_without_evicting_a_live_room(self): self.assertTrue(live.try_lease()) with patch("sglang.srt.disaggregation.mooncake.conn.MAX_TRACKED_ROOMS", 4): for room in range(200): - mgr._close_room_for_abort(room, b"") + mgr._close_room_for_abort(room, b"").created_at = 0 # ... and rooms known only from decode metadata, which no local # sender will ever release, must be bounded too. for room in range(1000, 1200): - mgr._room_lifetime(room, create=True) + mgr._room_lifetime(room, create=True).created_at = 0 self.assertLessEqual(len(mgr._room_lifetimes), 8) self.assertIn( -1, mgr._room_lifetimes, "a claimed, leased room must never be evicted" @@ -1156,9 +1156,36 @@ def test_escalation_reaches_the_scheduler(self): with patch( "sglang.srt.disaggregation.utils.dist.all_reduce", side_effect=lambda tensor, **kw: None, + ) as all_reduce: + with self.assertRaises(KVTransferBarrierEscalation): + poll_and_all_reduce([poller], object()) + self.assertEqual( + all_reduce.call_count, + 2, + "a local escalation must be coordinated before it reaches the scheduler", + ) + + def test_a_peer_escalation_reaches_every_scheduler(self): + poller = SimpleNamespace( + poll=Mock(return_value=KVPoll.Failed), + advance_failure_quiescence=Mock(return_value=False), + is_failure_quiescing=Mock(return_value=True), + ) + collective = Mock() + + def reduce(tensor, **_kwargs): + collective() + if collective.call_count == 2: + # The local rank is still waiting, but a peer encoded escalation + # into the coordinated quiescence state. + tensor.fill_(0) + + with patch( + "sglang.srt.disaggregation.utils.dist.all_reduce", side_effect=reduce ): with self.assertRaises(KVTransferBarrierEscalation): poll_and_all_reduce([poller], object()) + self.assertEqual(collective.call_count, 2) def test_an_ordinary_backend_fault_is_still_contained(self): poller = SimpleNamespace( @@ -1217,6 +1244,7 @@ def test_emergency_reclaim_retires_destinations_and_status_too(self): mgr = make_prefill_manager() self._room_with_metadata(mgr) self.assertIn(ROOM, mgr.transfer_infos) + mgr._room_lifetimes[ROOM].created_at = 0 with patch("sglang.srt.disaggregation.mooncake.conn.MAX_TRACKED_ROOMS", 1): mgr._room_lifetime(ROOM + 1, create=True) self.assertNotIn(ROOM, mgr._room_lifetimes) @@ -1224,6 +1252,18 @@ def test_emergency_reclaim_retires_destinations_and_status_too(self): self.assertNotIn(ROOM, mgr.request_status) self.assertNotIn(ROOM, mgr.req_to_decode_prefix_len) + def test_emergency_reclaim_preserves_live_metadata_first_room(self): + mgr = make_prefill_manager(room_sweep_ttl=300.0) + self._room_with_metadata(mgr) + + with patch("sglang.srt.disaggregation.mooncake.conn.MAX_TRACKED_ROOMS", 1): + mgr._room_lifetime(ROOM + 1, create=True) + + self.assertIn(ROOM, mgr._room_lifetimes) + self.assertIn(ROOM, mgr.transfer_infos) + self.assertIn(ROOM, mgr.request_status) + self.assertEqual(len(mgr._room_lifetimes), 2, "the tracked-room cap is soft") + def test_retirement_is_atomic_against_a_new_generation(self): # The lifetime and the rest of the room's state must go in one step: a # request created in between would have its own metadata deleted. @@ -1275,6 +1315,29 @@ def test_sweep_retires_destinations_and_status_together(self): class TestRoomCollisionIsolation(unittest.TestCase): """A receiver that lost a room collision must not damage the owner.""" + def test_constructor_rejects_loser_before_shared_state_or_metadata(self): + mgr = make_decode_manager() + mgr.get_session_id = Mock(side_effect=["owner", "loser"]) + mgr.addr_to_rooms_tracker = {"prefill:1": set()} + mgr.required_prefill_response_num_table = {} + mgr.prefill_response_tracker = {} + + owner = MooncakeKVReceiver(mgr, "prefill:1", ROOM) + mgr.update_status.reset_mock() + loser = MooncakeKVReceiver(mgr, "prefill:1", ROOM) + + self.assertIs(mgr._receivers[ROOM], owner) + self.assertEqual(loser.poll(), KVPoll.Failed) + self.assertFalse(loser._owns_room) + mgr.update_status.assert_not_called() + self.assertEqual(mgr.addr_to_rooms_tracker["prefill:1"], {ROOM}) + + loser.init(0) + loser.send_metadata(np.array([123], dtype=np.int32)) + loser.abort() + mgr.update_status.assert_not_called() + self.assertIs(mgr._receivers[ROOM], owner) + def test_the_loser_does_not_tear_down_shared_state(self): mgr = make_decode_manager() owner = make_receiver(mgr) From dabde23704bb85467a1c30152e854a1e182a6091 Mon Sep 17 00:00:00 2001 From: Kastan Day Date: Tue, 28 Jul 2026 09:44:29 -0700 Subject: [PATCH 03/10] fix(disaggregation): harden mixed-version quiescence --- .../srt/disaggregation/mooncake/conn.py | 50 +++++++++++-------- .../test_transfer_ownership_barrier.py | 34 ++++++++++--- 2 files changed, 58 insertions(+), 26 deletions(-) diff --git a/python/sglang/srt/disaggregation/mooncake/conn.py b/python/sglang/srt/disaggregation/mooncake/conn.py index 2bfdf6bd73e0..ddf3779cec48 100644 --- a/python/sglang/srt/disaggregation/mooncake/conn.py +++ b/python/sglang/srt/disaggregation/mooncake/conn.py @@ -78,7 +78,7 @@ TRANSFER_QUIESCE_TIMEOUTS = Counter( "sglang:kv_transfer_quiesce_timeouts_total", - "Requests released while KV transfer work may still have been in flight.", + "Requests whose KV transfer quiescence deadline expired.", ) # How often a decode rank re-sends an unacknowledged abort notification. @@ -794,8 +794,8 @@ def release_without_proof(self, room: int, reason: str) -> bool: and fails the worker once too many requests are stuck, because a restart releases every page safely while a silent overwrite does not. """ - TRANSFER_QUIESCE_TIMEOUTS.inc() if self.transfer_barrier != TransferBarrierLevel.STRICT: + TRANSFER_QUIESCE_TIMEOUTS.inc() logger.error( "Releasing KV pages for bootstrap_room=%s without proof that " "transfer work stopped: %s. An in-flight write may still land in " @@ -807,16 +807,19 @@ def release_without_proof(self, room: int, reason: str) -> bool: return True with self._unquiesced_lock: + first_timeout = room not in self._unquiesced_rooms self._unquiesced_rooms.add(room) stuck = len(self._unquiesced_rooms) - logger.error( - "Withholding KV pages for bootstrap_room=%s: %s (%d/%d requests " - "stuck). STRICT will not reuse a page it cannot prove is idle.", - room, - reason, - stuck, - self.max_unquiesced, - ) + if first_timeout: + TRANSFER_QUIESCE_TIMEOUTS.inc() + logger.error( + "Withholding KV pages for bootstrap_room=%s: %s (%d/%d requests " + "stuck). STRICT will not reuse a page it cannot prove is idle.", + room, + reason, + stuck, + self.max_unquiesced, + ) if stuck >= self.max_unquiesced: # Unrecoverable: the only way to reclaim these pages safely is to # stop using this address space entirely. The scheduler's top-level @@ -2950,10 +2953,17 @@ def _peers_quiesced(self) -> bool: with self._abort_lock: if self._peer_lacks_barrier: # A peer that acknowledges without echoing a nonce does so before - # its transfers drain, so its ACK is not proof. WARN accepts that - # and keeps a rolling upgrade working; STRICT cannot, by - # definition, so such a peer never satisfies the barrier. - return self.kv_mgr.transfer_barrier != TransferBarrierLevel.STRICT + # its transfers drain, so its ACK is not proof. WARN may credit + # that compatibility exception to one peer, but it must still + # receive every other peer's token. Tokenless ACKs carry no source + # identity and may be retries, so crediting more than one would + # let duplicates impersonate token-aware peers. STRICT cannot + # accept any proof-less peer. + return ( + self.kv_mgr.transfer_barrier != TransferBarrierLevel.STRICT + and len(self._received_abort_acks) + 1 + >= len(self._expected_abort_acks) + ) return self._expected_abort_acks <= self._received_abort_acks def _abort_targets_snapshot(self) -> List[Tuple[dict, bytes]]: @@ -2979,16 +2989,16 @@ def record_abort_ack(self, token: Optional[bytes]) -> None: if not token: # A prefill that predates the ownership barrier acknowledges the # notification without echoing a nonce, and does so before its - # transfers have drained. It cannot promise quiescence, so stop - # waiting rather than stalling until the deadline. + # transfers have drained. It cannot promise quiescence. WARN can + # credit one such compatibility exception while still waiting + # for every token-aware peer; STRICT waits for proof or escalates. self._peer_lacks_barrier = True logger.warning_once( "Prefill peer acknowledged an abort without a transfer-quiescence " "token, so it acknowledges before its transfers drain and cannot " - "prove quiescence. Upgrade the prefill instances; until then this " - "request's pages are released on the " - "SGLANG_DISAGGREGATION_TRANSFER_BARRIER policy rather than on " - "proof, and STRICT will refuse to release them at all." + "prove quiescence. Upgrade the prefill instances; WARN will still " + "wait for every token-aware peer before crediting one legacy peer, " + "and STRICT will refuse to release without proof." ) return if token in self._expected_abort_acks: diff --git a/test/registered/unit/disaggregation/test_transfer_ownership_barrier.py b/test/registered/unit/disaggregation/test_transfer_ownership_barrier.py index ca7e1eb493a5..3d4f60c7ca2b 100644 --- a/test/registered/unit/disaggregation/test_transfer_ownership_barrier.py +++ b/test/registered/unit/disaggregation/test_transfer_ownership_barrier.py @@ -1006,6 +1006,23 @@ def test_legacy_prefill_ack_stops_the_wait(self): receiver.record_abort_ack(None) self.assertTrue(receiver.advance_failure_quiescence()) + def test_legacy_ack_does_not_satisfy_a_token_aware_peer(self): + mgr = make_decode_manager() + receiver = make_receiver( + mgr, bootstrap_infos=[{"rank": "legacy"}, {"rank": "current"}] + ) + self.assertFalse(receiver.advance_failure_quiescence()) + tokens = {info["rank"]: token for info, token in receiver._abort_targets} + + receiver.record_abort_ack(None) + self.assertFalse( + receiver.advance_failure_quiescence(), + "one legacy ACK must not satisfy another rank's outstanding token", + ) + + receiver.record_abort_ack(tokens["current"]) + self.assertTrue(receiver.advance_failure_quiescence()) + def test_stale_ack_for_another_request_is_ignored(self): mgr = make_decode_manager() receiver = make_receiver(mgr, bootstrap_infos=[{"rank": 0}]) @@ -1097,12 +1114,17 @@ def test_warn_releases_once_and_loudly(self): def test_strict_never_releases_without_proof(self): mgr, sender = self._stuck_sender(TransferBarrierLevel.STRICT) self._arm_and_expire(sender) - with patch("sglang.srt.disaggregation.mooncake.conn.TRANSFER_QUIESCE_TIMEOUTS"): - for _ in range(5): - self.assertFalse( - sender.advance_failure_quiescence(), - "STRICT must not hand back a page it cannot prove is idle", - ) + with patch( + "sglang.srt.disaggregation.mooncake.conn.TRANSFER_QUIESCE_TIMEOUTS" + ) as metric: + with patch("sglang.srt.disaggregation.mooncake.conn.logger.error") as error: + for _ in range(5): + self.assertFalse( + sender.advance_failure_quiescence(), + "STRICT must not hand back a page it cannot prove is idle", + ) + metric.inc.assert_called_once_with() + error.assert_called_once() self.assertIn( ROOM, mgr._unquiesced_rooms, "the stuck room must be tracked for escalation" ) From 75b6726c097b825d15ff19bbaa9c2626e9fa7633 Mon Sep 17 00:00:00 2001 From: Kastan Day Date: Tue, 28 Jul 2026 09:44:29 -0700 Subject: [PATCH 04/10] fix(disaggregation): close transfer lifecycle races --- .../sglang/srt/disaggregation/common/conn.py | 10 +- .../srt/disaggregation/mooncake/conn.py | 69 +++++++----- python/sglang/srt/disaggregation/prefill.py | 5 +- .../test_transfer_ownership_barrier.py | 102 ++++++++++++++++++ 4 files changed, 155 insertions(+), 31 deletions(-) diff --git a/python/sglang/srt/disaggregation/common/conn.py b/python/sglang/srt/disaggregation/common/conn.py index 15510fdfffbe..945fc9111c63 100644 --- a/python/sglang/srt/disaggregation/common/conn.py +++ b/python/sglang/srt/disaggregation/common/conn.py @@ -743,8 +743,14 @@ def _send_multipart_locked( thread-safe. *flags* accepts ``zmq.NOBLOCK`` for messages the caller retries, so one unreachable peer cannot hold this lock for SNDTIMEO. """ - sock = self._connect(endpoint, is_ipv6=is_ipv6) - with self._socket_send_locks[endpoint]: + # The endpoint lock must cover connection validation as well as send. + # Otherwise one thread can receive the cached socket from _connect(), + # then another can observe a disconnect and close/replace that socket + # before the first thread acquires this lock. + with self._socket_lock: + send_lock = self._socket_send_locks.setdefault(endpoint, threading.Lock()) + with send_lock: + sock = self._connect(endpoint, is_ipv6=is_ipv6) sock.send_multipart(parts, flags) def get_mha_kv_ptrs_with_pp( diff --git a/python/sglang/srt/disaggregation/mooncake/conn.py b/python/sglang/srt/disaggregation/mooncake/conn.py index ddf3779cec48..72016137faca 100644 --- a/python/sglang/srt/disaggregation/mooncake/conn.py +++ b/python/sglang/srt/disaggregation/mooncake/conn.py @@ -450,14 +450,20 @@ def _room_lifetime( self, room: int, *, create: bool ) -> Optional[RoomTransferLifetime]: with self._room_lifetimes_lock: - lifetime = self._room_lifetimes.get(room) - if lifetime is None and create: - # Reclaim before inserting, so the entry being created can never - # be the one that gets evicted. - if len(self._room_lifetimes) >= MAX_TRACKED_ROOMS: - self._emergency_reclaim_rooms_locked() - lifetime = self._room_lifetimes[room] = RoomTransferLifetime() - return lifetime + return self._room_lifetime_locked(room, create=create) + + def _room_lifetime_locked( + self, room: int, *, create: bool + ) -> Optional[RoomTransferLifetime]: + """Return a room lifetime while the caller holds its generation lock.""" + lifetime = self._room_lifetimes.get(room) + if lifetime is None and create: + # Reclaim before inserting, so the entry being created can never be + # the one that gets evicted. + if len(self._room_lifetimes) >= MAX_TRACKED_ROOMS: + self._emergency_reclaim_rooms_locked() + lifetime = self._room_lifetimes[room] = RoomTransferLifetime() + return lifetime def _retire_rooms_locked(self, rooms: List[int]) -> None: """Forget every piece of state keyed by these rooms, atomically. @@ -2281,26 +2287,33 @@ def _handle_bootstrap_metadata(self, msg: List[bytes]) -> bool: # ownership barrier is created by whichever side gets here first. A closed # barrier means the room was aborted or released, and accepting # destinations for it would let us transfer into pages decode has freed. - lifetime = self._room_lifetime(room, create=True) - if lifetime.is_closed(): - logger.debug("Dropping KV metadata for room %s: transfers are closed", room) - return False - lifetime.add_abort_token(transfer_info.abort_token) - - infos = self.transfer_infos.setdefault(room, {}) - infos[mooncake_session_id] = transfer_info - # NOTE: after bootstrapping we can mark the req as waiting for input - if len(infos) == required_dst_info_num: - self.resolve_kv_replica_factor(infos) - self.req_to_decode_prefix_len[room] = next( - ( - info.decode_prefix_len - for info in infos.values() - if info.decode_prefix_len is not None - ), - 0, - ) - self.update_status(room, KVPoll.WaitingForInput) + # Lifetime validation and every room-keyed write are one generation + # transaction. The sweeper takes this same lock, so it cannot retire the + # lifetime between validation and publication and leave orphan metadata + # that a later generation could inherit. + with self._room_lifetimes_lock: + lifetime = self._room_lifetime_locked(room, create=True) + if lifetime.is_closed(): + logger.debug( + "Dropping KV metadata for room %s: transfers are closed", room + ) + return False + lifetime.add_abort_token(transfer_info.abort_token) + + infos = self.transfer_infos.setdefault(room, {}) + infos[mooncake_session_id] = transfer_info + # NOTE: after bootstrapping we can mark the req as waiting for input + if len(infos) == required_dst_info_num: + self.resolve_kv_replica_factor(infos) + self.req_to_decode_prefix_len[room] = next( + ( + info.decode_prefix_len + for info in infos.values() + if info.decode_prefix_len is not None + ), + 0, + ) + self.update_status(room, KVPoll.WaitingForInput) return True def start_decode_thread(self): diff --git a/python/sglang/srt/disaggregation/prefill.py b/python/sglang/srt/disaggregation/prefill.py index 6d13a5b6e0a1..eb5b13e1ce77 100644 --- a/python/sglang/srt/disaggregation/prefill.py +++ b/python/sglang/srt/disaggregation/prefill.py @@ -978,7 +978,10 @@ def handle_bootstrap_failure(self: Scheduler, req: Req) -> None: release_kv_cache(req, self.tree_cache) maybe_release_metadata_buffer(req, self.req_to_metadata_buffer_idx_allocator) req.pending_bootstrap = False - prepare_abort(req, error_message, status_code=HTTPStatus.INTERNAL_SERVER_ERROR) + if not isinstance(req.finished_reason, FINISH_ABORT): + prepare_abort( + req, error_message, status_code=HTTPStatus.INTERNAL_SERVER_ERROR + ) self.output_streamer.stream_output([req], req.return_logprob) if self.metrics_reporter.enable_metrics: self.metrics_collector.increment_bootstrap_failed_reqs() diff --git a/test/registered/unit/disaggregation/test_transfer_ownership_barrier.py b/test/registered/unit/disaggregation/test_transfer_ownership_barrier.py index 3d4f60c7ca2b..5f1dff20c319 100644 --- a/test/registered/unit/disaggregation/test_transfer_ownership_barrier.py +++ b/test/registered/unit/disaggregation/test_transfer_ownership_barrier.py @@ -1286,6 +1286,37 @@ def test_emergency_reclaim_preserves_live_metadata_first_room(self): self.assertIn(ROOM, mgr.request_status) self.assertEqual(len(mgr._room_lifetimes), 2, "the tracked-room cap is soft") + def test_metadata_publication_holds_the_lifetime_generation_lock(self): + mgr = make_prefill_manager() + mgr.session_lock = threading.Lock() + mgr.failed_sessions, mgr.session_failures = set(), {} + mgr.decode_kv_args_table = {} + mgr.resolve_kv_replica_factor = Mock() + + def assert_generation_locked(): + self.assertTrue( + mgr._room_lifetimes_lock.locked(), + "room metadata must publish under the lifetime generation lock", + ) + + class GenerationGuardedDict(dict): + def setdefault(self, *args, **kwargs): + assert_generation_locked() + return super().setdefault(*args, **kwargs) + + def __setitem__(self, key, value): + assert_generation_locked() + return super().__setitem__(key, value) + + mgr.transfer_infos = GenerationGuardedDict() + mgr.req_to_decode_prefix_len = GenerationGuardedDict() + mgr.request_status = GenerationGuardedDict() + + self._room_with_metadata(mgr) + self.assertIn(ROOM, mgr._room_lifetimes) + self.assertIn(ROOM, mgr.transfer_infos) + self.assertIn(ROOM, mgr.request_status) + def test_retirement_is_atomic_against_a_new_generation(self): # The lifetime and the rest of the room's state must go in one step: a # request created in between would have its own metadata deleted. @@ -1379,6 +1410,41 @@ def test_the_loser_does_not_tear_down_shared_state(self): self.assertIs(mgr._receivers[ROOM], owner) +class TestEndpointSendSerialization(unittest.TestCase): + def test_connection_validation_and_send_share_the_endpoint_lock(self): + mgr = MooncakeKVManager.__new__(MooncakeKVManager) + mgr._socket_lock = threading.Lock() + mgr._socket_send_locks = {} + endpoint = "tcp://127.0.0.1:1" + observed = [] + + class Socket: + def send_multipart(self, parts, flags=0): + observed.append( + ( + "send", + mgr._socket_send_locks[endpoint].locked(), + parts, + flags, + ) + ) + + def connect(*_args, **_kwargs): + observed.append(("connect", mgr._socket_send_locks[endpoint].locked())) + return Socket() + + mgr._connect = Mock(side_effect=connect) + mgr._send_multipart_locked(endpoint, [b"status"], flags=zmq.NOBLOCK) + + self.assertEqual( + observed, + [ + ("connect", True), + ("send", True, [b"status"], zmq.NOBLOCK), + ], + ) + + class TestWatermarkDelivery(unittest.TestCase): """A dropped watermark must be retried; it can gate a prefill's progress.""" @@ -1653,6 +1719,42 @@ def test_deferral_is_idempotent(self): scheduler_mod.Scheduler.process_pending_chunked_abort(scheduler) self.assertEqual(scheduler.disagg_prefill_inflight_queue, [req]) + def test_pending_bootstrap_failure_preserves_user_abort(self): + from sglang.srt.disaggregation import prefill as prefill_mod + from sglang.srt.managers.schedule_batch import FINISH_ABORT + + user_abort = FINISH_ABORT("Aborted") + req = SimpleNamespace( + rid="chunked", + bootstrap_room=ROOM, + disagg_kv_sender=Mock(), + time_stats=SimpleNamespace(trace_ctx=Mock()), + req_pool_idx=None, + kv=None, + mamba_pool_idx=None, + metadata_buffer_index=-1, + pending_bootstrap=True, + finished_reason=user_abort, + return_logprob=False, + ) + scheduler = SimpleNamespace( + ps=SimpleNamespace(tp_rank=0), + tree_cache=Mock(), + req_to_metadata_buffer_idx_allocator=Mock(), + output_streamer=SimpleNamespace(stream_output=Mock()), + metrics_reporter=SimpleNamespace(enable_metrics=False), + enable_hicache_storage=False, + ) + + with patch.object(prefill_mod, "prepare_abort") as prepare_abort: + prefill_mod.SchedulerDisaggregationPrefillMixin.handle_bootstrap_failure( + scheduler, req + ) + + prepare_abort.assert_not_called() + self.assertIs(req.finished_reason, user_abort) + self.assertFalse(req.pending_bootstrap) + class TestTransferInfoWire(unittest.TestCase): """The abort nonce is an optional trailing frame.""" From 3d1af6c7d7c6335493e14d5f64fd7c02dea14591 Mon Sep 17 00:00:00 2001 From: Kastan Day Date: Tue, 28 Jul 2026 09:44:29 -0700 Subject: [PATCH 05/10] fix(disaggregation): track metadata exposure per peer --- .../srt/disaggregation/mooncake/conn.py | 68 ++++++++++++------- .../test_transfer_ownership_barrier.py | 45 +++++++++++- 2 files changed, 88 insertions(+), 25 deletions(-) diff --git a/python/sglang/srt/disaggregation/mooncake/conn.py b/python/sglang/srt/disaggregation/mooncake/conn.py index 72016137faca..dd7960e07d96 100644 --- a/python/sglang/srt/disaggregation/mooncake/conn.py +++ b/python/sglang/srt/disaggregation/mooncake/conn.py @@ -2845,32 +2845,42 @@ def send_metadata( self.bootstrap_room, self.bootstrap_infos, self ) - # From here on a prefill rank may write into our KV pages, so the pages - # must not be released until every peer confirms it has stopped. - self._metadata_sent = True for bootstrap_info, abort_token in self._abort_targets_snapshot(): sock, lock = self._connect_to_bootstrap_server(bootstrap_info) is_dummy = bootstrap_info["is_dummy"] try: with lock: - sock.send_multipart( - [ - str(self.bootstrap_room).encode("ascii"), - self.kv_mgr.local_ip.encode("ascii"), - str(self.kv_mgr.rank_port).encode("ascii"), - self.session_id.encode("ascii"), - kv_indices.tobytes() if not is_dummy else b"", - str(aux_index).encode("ascii") if not is_dummy else b"", - ( - pack_int_lists(state_indices, "i") - if not is_dummy and state_indices - else b"" - ), - str(self.required_dst_info_num).encode("ascii"), - str(decode_prefix_len or 0).encode("ascii"), - abort_token, - ] - ) + # Register exposure before send so an abort racing a + # successful send cannot receive and discard an early ACK. + # ZeroMQ delivers multipart messages atomically, so a send + # that raises exposed no complete metadata and is removed. + self._record_metadata_exposure(abort_token) + try: + sock.send_multipart( + [ + str(self.bootstrap_room).encode("ascii"), + self.kv_mgr.local_ip.encode("ascii"), + str(self.kv_mgr.rank_port).encode("ascii"), + self.session_id.encode("ascii"), + kv_indices.tobytes() if not is_dummy else b"", + ( + str(aux_index).encode("ascii") + if not is_dummy + else b"" + ), + ( + pack_int_lists(state_indices, "i") + if not is_dummy and state_indices + else b"" + ), + str(self.required_dst_info_num).encode("ascii"), + str(decode_prefix_len or 0).encode("ascii"), + abort_token, + ] + ) + except Exception: + self._discard_metadata_exposure(abort_token) + raise except zmq.ZMQError: self.kv_mgr.record_failure( self.bootstrap_room, @@ -2992,11 +3002,21 @@ def _abort_targets_snapshot(self) -> List[Tuple[dict, bytes]]: (bootstrap_info, os.urandom(16).hex().encode("ascii")) for bootstrap_info in self.bootstrap_infos ] - self._expected_abort_acks = { - token for _info, token in self._abort_targets - } return list(self._abort_targets) + def _record_metadata_exposure(self, token: bytes) -> None: + """Require an ACK from a peer that may now write into this room.""" + with self._abort_lock: + self._expected_abort_acks.add(token) + self._metadata_sent = True + + def _discard_metadata_exposure(self, token: bytes) -> None: + """Undo an exposure whose atomic multipart send did not complete.""" + with self._abort_lock: + self._expected_abort_acks.discard(token) + self._received_abort_acks.discard(token) + self._metadata_sent = bool(self._expected_abort_acks) + def record_abort_ack(self, token: Optional[bytes]) -> None: with self._abort_lock: if not token: diff --git a/test/registered/unit/disaggregation/test_transfer_ownership_barrier.py b/test/registered/unit/disaggregation/test_transfer_ownership_barrier.py index 5f1dff20c319..e97052ac8fbd 100644 --- a/test/registered/unit/disaggregation/test_transfer_ownership_barrier.py +++ b/test/registered/unit/disaggregation/test_transfer_ownership_barrier.py @@ -118,7 +118,7 @@ def make_receiver(mgr, room=ROOM, bootstrap_infos=None, metadata_sent=True): receiver.conclude_state = None receiver.require_staging = False receiver.bootstrap_infos = bootstrap_infos - receiver._metadata_sent = metadata_sent + receiver._metadata_sent = False receiver._quiescing = False receiver._quiesce_complete = False receiver._quiesce_deadline = float("inf") @@ -132,6 +132,13 @@ def make_receiver(mgr, room=ROOM, bootstrap_infos=None, metadata_sent=True): receiver._connect_to_bootstrap_server = Mock( side_effect=lambda info: (Mock(), threading.Lock()) ) + if metadata_sent: + targets = receiver._abort_targets_snapshot() + if targets: + for _info, token in targets: + receiver._record_metadata_exposure(token) + else: + receiver._metadata_sent = True return receiver @@ -981,6 +988,42 @@ def test_abort_notifications_carry_a_stable_token_and_are_retried(self): "a retry must reuse the nonce so the ACK still matches", ) + def test_failed_metadata_send_does_not_expect_unexposed_peers(self): + mgr = make_decode_manager() + mgr.enable_staging = False + receiver = make_receiver( + mgr, + bootstrap_infos=[ + { + "rank": rank, + "rank_ip": "127.0.0.1", + "rank_port": rank + 1, + "is_dummy": False, + } + for rank in range(3) + ], + metadata_sent=False, + ) + receiver.session_id = "decode" + receiver.required_dst_info_num = 3 + sockets = {rank: Mock() for rank in range(3)} + sockets[1].send_multipart.side_effect = zmq.ZMQError() + connected = [] + + def connect(info): + connected.append(info["rank"]) + return sockets[info["rank"]], threading.Lock() + + receiver._connect_to_bootstrap_server = connect + receiver.send_metadata(np.array([123], dtype=np.int32)) + + tokens = {info["rank"]: token for info, token in receiver._abort_targets} + self.assertEqual(connected, [0, 1]) + self.assertEqual(receiver._expected_abort_acks, {tokens[0]}) + self.assertTrue(receiver._metadata_sent) + self.assertNotIn(tokens[1], receiver._expected_abort_acks) + self.assertNotIn(tokens[2], receiver._expected_abort_acks) + def test_quiescence_is_bounded_when_a_peer_never_acknowledges(self): mgr = make_decode_manager(quiesce_timeout=0.05) receiver = make_receiver(mgr, bootstrap_infos=[{"rank": 0}]) From 5da0f821f8efeddc341ff37556b91334a15e3e9c Mon Sep 17 00:00:00 2001 From: Kastan Day Date: Mon, 10 Aug 2026 09:33:03 -0700 Subject: [PATCH 06/10] Enforce the transfer ownership barrier unconditionally and condense its tests --- python/sglang/srt/disaggregation/base/conn.py | 8 +- .../srt/disaggregation/mooncake/conn.py | 186 ++--- python/sglang/srt/disaggregation/utils.py | 4 +- python/sglang/srt/environ.py | 29 - .../test_transfer_ownership_barrier.py | 756 ++++-------------- 5 files changed, 237 insertions(+), 746 deletions(-) diff --git a/python/sglang/srt/disaggregation/base/conn.py b/python/sglang/srt/disaggregation/base/conn.py index 6a5070244f04..f2f4e25b0d56 100644 --- a/python/sglang/srt/disaggregation/base/conn.py +++ b/python/sglang/srt/disaggregation/base/conn.py @@ -204,9 +204,11 @@ def advance_failure_quiescence(self) -> bool: pages. Backends without out-of-band transfer work keep the default. Implementations must be idempotent, must not block the scheduler loop, - and must be *bounded*: a transport or peer that never confirms - quiescence has to eventually return ``True`` rather than pinning the - request, and its pages, forever. + and must not fail silently: a transport or peer that never confirms + quiescence must be reported after a bounded wait, and must eventually + raise ``KVTransferBarrierEscalation`` rather than pin an unbounded + number of requests -- a restart reclaims every withheld page safely, + while releasing without proof can corrupt another request's KV. """ return True diff --git a/python/sglang/srt/disaggregation/mooncake/conn.py b/python/sglang/srt/disaggregation/mooncake/conn.py index dd7960e07d96..d233d01d14e9 100644 --- a/python/sglang/srt/disaggregation/mooncake/conn.py +++ b/python/sglang/srt/disaggregation/mooncake/conn.py @@ -55,7 +55,7 @@ compute_mamba_state_slice_byte_blocks, ) from sglang.srt.distributed.parallel_state import get_mooncake_transfer_engine -from sglang.srt.environ import TransferBarrierLevel, envs +from sglang.srt.environ import envs from sglang.srt.observability.mooncake_trace import ( MooncakeRequestStage, mooncake_trace_func, @@ -81,6 +81,16 @@ "Requests whose KV transfer quiescence deadline expired.", ) +# How long a failed request may wait for proof of transfer quiescence before +# the wait is reported as an error. The pages are never released without proof; +# this only bounds how long the condition stays silent. Keep it above the +# manager socket's 30s ZMQ send timeout, because a transfer holds its room +# while notifying the peer; a shorter value reports spurious timeouts whenever +# a decode stops reading. +QUIESCE_TIMEOUT_S = 60.0 +# How many requests may be stuck without proof of quiescence before the worker +# fails, so that a restart releases every withheld page safely at once. +MAX_UNQUIESCED_ROOMS = 256 # How often a decode rank re-sends an unacknowledged abort notification. ABORT_RETRY_INTERVAL_S = 0.5 # How often the bookkeeping thread re-checks drained rooms and sweeps lifetimes. @@ -180,13 +190,6 @@ def is_reclaimable(self) -> bool: with self._cond: return (not self._open and self._leases == 0) or not self._claimed - def wait_quiesced(self, timeout: Optional[float]) -> bool: - """Block until quiesced. Returns False if *timeout* elapsed first.""" - with self._cond: - return self._cond.wait_for( - lambda: not self._open and self._leases == 0, timeout - ) - def add_abort_token(self, token: bytes) -> None: if not token: return @@ -196,19 +199,18 @@ def add_abort_token(self, token: bytes) -> None: def authorizes_abort(self, token: bytes) -> bool: """Whether *token* may close this room. - A tokenless abort comes from a peer that predates the token protocol and - is honoured unconditionally, as before. + A tokenless abort cannot be authenticated, and is honoured anyway: + closing the room only fails one request (availability), while dropping + the abort would leave this rank free to transfer into pages the decode + has already released (the corruption this barrier exists to prevent). A room that has not received any decode metadata yet has no tokens to - compare against, and accepts the abort. That is deliberate, and it is the - safe direction of the trade: an abort can legitimately arrive before this - rank has any metadata for the room (the decode gave up during bootstrap), - and dropping it would leave this rank free to transfer into pages the - decode has already released -- the corruption this barrier exists to - prevent. Accepting instead risks one spurious request failure if a - recycled bootstrap_room draws a delayed abort from its previous occupant, - which is availability rather than correctness, and requires a collision in - a 64-bit space. Distinguishing the two cases would need a generation tag + compare against, and accepts the abort for the same fail-safe reason: + an abort can legitimately arrive before this rank has any metadata for + the room (the decode gave up during bootstrap). Accepting risks one + spurious request failure if a recycled bootstrap_room draws a delayed + abort from its previous occupant, which requires a collision in a + 64-bit space. Distinguishing the two cases would need a generation tag on every room-scoped message; that is not worth a wire change here. """ if not token: @@ -349,12 +351,6 @@ def __init__( is_mla_backend: Optional[bool] = False, ): super().__init__(args, disaggregation_mode, server_args, is_mla_backend) - self.transfer_barrier = TransferBarrierLevel( - envs.SGLANG_DISAGGREGATION_TRANSFER_BARRIER.get() - ) - # WARN releases after this long without proof; STRICT never does. - self.quiesce_timeout = envs.SGLANG_DISAGGREGATION_QUIESCE_TIMEOUT.get() - self.max_unquiesced = envs.SGLANG_DISAGGREGATION_MAX_UNQUIESCED.get() self._unquiesced_rooms: set = set() self._unquiesced_lock = threading.Lock() self.init_engine() @@ -643,7 +639,7 @@ def request_abort_ack( if len(self._abort_ack_pending) >= MAX_PENDING_ABORT_ACKS: logger.warning_once( "More than %d ABORT_ACKs are owed; dropping new ones. Their " - "decode peers fall back to their own barrier policy.", + "decode peers keep withholding the pages and escalate.", MAX_PENDING_ABORT_ACKS, ) return @@ -716,7 +712,7 @@ def _drain_abort_acks( except Exception: logger.exception( "Dropping unsendable ABORT_ACK for room %s; its decode peer " - "will fall back to its own barrier policy", + "keeps withholding the pages and escalates", pending.room, ) self._retire_abort_ack(pending) @@ -793,51 +789,37 @@ def _handle_abort_ack(self, msg: List[bytes]) -> None: # ------------------------------------------------------------------ - def release_without_proof(self, room: int, reason: str) -> bool: - """Whether *room* may release its KV pages without proof of quiescence. + def report_unquiesced(self, room: int, reason: str) -> None: + """Record that *room*'s KV pages cannot be proven idle. - WARN trades a narrow, loud corruption window for liveness. STRICT refuses, - and fails the worker once too many requests are stuck, because a restart - releases every page safely while a silent overwrite does not. + The pages are never released without proof: a silent overwrite is worse + than a stuck request. Once too many requests are stuck the worker fails + instead, because a restart releases every withheld page safely at once. """ - if self.transfer_barrier != TransferBarrierLevel.STRICT: - TRANSFER_QUIESCE_TIMEOUTS.inc() - logger.error( - "Releasing KV pages for bootstrap_room=%s without proof that " - "transfer work stopped: %s. An in-flight write may still land in " - "them. Set SGLANG_DISAGGREGATION_TRANSFER_BARRIER=2 to refuse " - "instead.", - room, - reason, - ) - return True - with self._unquiesced_lock: - first_timeout = room not in self._unquiesced_rooms + first_report = room not in self._unquiesced_rooms self._unquiesced_rooms.add(room) stuck = len(self._unquiesced_rooms) - if first_timeout: + if first_report: TRANSFER_QUIESCE_TIMEOUTS.inc() logger.error( "Withholding KV pages for bootstrap_room=%s: %s (%d/%d requests " - "stuck). STRICT will not reuse a page it cannot prove is idle.", + "stuck). A page that cannot be proven idle is never reused.", room, reason, stuck, - self.max_unquiesced, + MAX_UNQUIESCED_ROOMS, ) - if stuck >= self.max_unquiesced: + if stuck >= MAX_UNQUIESCED_ROOMS: # Unrecoverable: the only way to reclaim these pages safely is to # stop using this address space entirely. The scheduler's top-level # handler turns this into engine teardown. raise KVTransferBarrierEscalation( f"{stuck} KV transfers cannot be proven quiesced " - f"(SGLANG_DISAGGREGATION_MAX_UNQUIESCED={self.max_unquiesced}); " - "refusing to reuse their pages. Restarting this worker is the " - "only safe way to reclaim them. Check for an unreachable or " - "wedged peer." + f"(MAX_UNQUIESCED_ROOMS={MAX_UNQUIESCED_ROOMS}); refusing to " + "reuse their pages. Restarting this worker is the only safe " + "way to reclaim them. Check for an unreachable or wedged peer." ) - return False def forget_unquiesced(self, room: int) -> None: with self._unquiesced_lock: @@ -2533,7 +2515,6 @@ def __init__( self.init_time = time.time() self._quiescing = False self._quiesce_deadline = float("inf") - self._quiesce_timed_out = False # Join the room's ownership barrier, which decode metadata may already # have created, and fail fast if an abort got here first. if not self.kv_mgr.open_room_transfers(self.bootstrap_room): @@ -2605,7 +2586,7 @@ def _close_barrier(self) -> None: if self._quiescing: return self._quiescing = True - self._quiesce_deadline = time.monotonic() + self.kv_mgr.quiesce_timeout + self._quiesce_deadline = time.monotonic() + QUIESCE_TIMEOUT_S self.kv_mgr.close_room_transfers(self.bootstrap_room) def is_failure_quiescing(self) -> bool: @@ -2613,24 +2594,17 @@ def is_failure_quiescing(self) -> bool: def advance_failure_quiescence(self) -> bool: self._close_barrier() - if self.kv_mgr.transfer_barrier == TransferBarrierLevel.OFF: - return True lifetime = self.kv_mgr._room_lifetime(self.bootstrap_room, create=False) if lifetime is None or lifetime.is_quiesced(): self.kv_mgr.forget_unquiesced(self.bootstrap_room) return True - if self._quiesce_timed_out: - # WARN already decided to release; do not re-report it every poll. - return True - if time.monotonic() < self._quiesce_deadline: - return False - released = self.kv_mgr.release_without_proof( - self.bootstrap_room, - f"{lifetime.outstanding_leases()} transfer worker(s) still hold it " - f"after {self.kv_mgr.quiesce_timeout:.1f}s", - ) - self._quiesce_timed_out = released - return released + if time.monotonic() >= self._quiesce_deadline: + self.kv_mgr.report_unquiesced( + self.bootstrap_room, + f"{lifetime.outstanding_leases()} transfer worker(s) still " + f"hold it after {QUIESCE_TIMEOUT_S:.1f}s", + ) + return False def clear(self) -> None: super().clear() @@ -2695,7 +2669,6 @@ def __init__( self._abort_targets: List[Tuple[dict, bytes]] = [] self._expected_abort_acks: set = set() self._received_abort_acks: set = set() - self._peer_lacks_barrier = False self._last_abort_send = float("-inf") self._abort_lock = threading.Lock() # Per-room state (ABORT_ACK routing, staging teardown) may only be @@ -2929,7 +2902,7 @@ def _close_barrier(self) -> None: if self._quiescing: return self._quiescing = True - self._quiesce_deadline = time.monotonic() + self.kv_mgr.quiesce_timeout + self._quiesce_deadline = time.monotonic() + QUIESCE_TIMEOUT_S def is_failure_quiescing(self) -> bool: return self._quiescing @@ -2940,53 +2913,34 @@ def advance_failure_quiescence(self) -> bool: # No prefill rank was ever handed this request's page indices, so # nothing can be writing into them. return True - if self.kv_mgr.transfer_barrier == TransferBarrierLevel.OFF: - self._quiesce_complete = True - return True if not self._owns_room: # Another live receiver owns this bootstrap_room, so our ABORT_ACKs - # are routed to it and proof can never reach us. Do not pretend to - # wait for it; the collision itself is already reported as an error. - return self._conclude_without_proof( + # are routed to it and proof can never reach us. Report immediately + # instead of pretending to wait for it. + self.kv_mgr.report_unquiesced( + self.bootstrap_room, f"bootstrap_room {self.bootstrap_room} is owned by another " - "receiver, so its acknowledgements are unroutable" + "receiver, so its acknowledgements are unroutable", ) + return False if self._peers_quiesced(): self._quiesce_complete = True self.kv_mgr.forget_unquiesced(self.bootstrap_room) return True - if time.monotonic() < self._quiesce_deadline: - return False - with self._abort_lock: - missing = len(self._expected_abort_acks - self._received_abort_acks) - total = len(self._expected_abort_acks) - return self._conclude_without_proof( - f"{missing} of {total} prefill ranks unacknowledged after " - f"{self.kv_mgr.quiesce_timeout:.1f}s" - ) - - def _conclude_without_proof(self, reason: str) -> bool: - released = self.kv_mgr.release_without_proof(self.bootstrap_room, reason) - if released: - self._quiesce_complete = True - return released + if time.monotonic() >= self._quiesce_deadline: + with self._abort_lock: + missing = len(self._expected_abort_acks - self._received_abort_acks) + total = len(self._expected_abort_acks) + self.kv_mgr.report_unquiesced( + self.bootstrap_room, + f"{missing} of {total} prefill ranks unacknowledged after " + f"{QUIESCE_TIMEOUT_S:.1f}s", + ) + return False def _peers_quiesced(self) -> bool: self._send_abort_notification() with self._abort_lock: - if self._peer_lacks_barrier: - # A peer that acknowledges without echoing a nonce does so before - # its transfers drain, so its ACK is not proof. WARN may credit - # that compatibility exception to one peer, but it must still - # receive every other peer's token. Tokenless ACKs carry no source - # identity and may be retries, so crediting more than one would - # let duplicates impersonate token-aware peers. STRICT cannot - # accept any proof-less peer. - return ( - self.kv_mgr.transfer_barrier != TransferBarrierLevel.STRICT - and len(self._received_abort_acks) + 1 - >= len(self._expected_abort_acks) - ) return self._expected_abort_acks <= self._received_abort_acks def _abort_targets_snapshot(self) -> List[Tuple[dict, bytes]]: @@ -3021,17 +2975,15 @@ def record_abort_ack(self, token: Optional[bytes]) -> None: with self._abort_lock: if not token: # A prefill that predates the ownership barrier acknowledges the - # notification without echoing a nonce, and does so before its - # transfers have drained. It cannot promise quiescence. WARN can - # credit one such compatibility exception while still waiting - # for every token-aware peer; STRICT waits for proof or escalates. - self._peer_lacks_barrier = True + # notification without echoing our nonce, and does so before its + # transfers have drained, so its ACK proves nothing. Mixed-version + # PD deployments are unsupported: this rank keeps withholding the + # request's pages and eventually escalates. logger.warning_once( - "Prefill peer acknowledged an abort without a transfer-quiescence " - "token, so it acknowledges before its transfers drain and cannot " - "prove quiescence. Upgrade the prefill instances; WARN will still " - "wait for every token-aware peer before crediting one legacy peer, " - "and STRICT will refuse to release without proof." + "Prefill peer acknowledged an abort without echoing a " + "transfer-quiescence token; it acknowledges before its " + "transfers drain, which proves nothing. Mixed-version PD " + "deployments are unsupported: upgrade the prefill instances." ) return if token in self._expected_abort_acks: diff --git a/python/sglang/srt/disaggregation/utils.py b/python/sglang/srt/disaggregation/utils.py index b19a6e86f30f..952e78483442 100644 --- a/python/sglang/srt/disaggregation/utils.py +++ b/python/sglang/srt/disaggregation/utils.py @@ -160,8 +160,8 @@ def _advance_quiescence(poller) -> int: A KVTransferBarrierEscalation is not such a fault. It is the barrier reporting that pages cannot be proven idle and must not be reused, so it is - propagated: swallowing it here would silently turn the strictest policy into - the most permissive one. + propagated: swallowing it here would silently release exactly the pages the + barrier refused to. """ try: return int(poller.advance_failure_quiescence()) diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index 8c675dda8f82..b4ff61057eb6 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -210,23 +210,6 @@ class ToolStrictLevel(IntEnum): PARAMETER = 2 -class TransferBarrierLevel(IntEnum): - """How strictly KV pages are withheld from reuse while a transfer may run. - - OFF: no barrier; a failed request releases its pages immediately (legacy). - WARN: wait for proof that transfer work stopped, but release anyway after - SGLANG_DISAGGREGATION_QUIESCE_TIMEOUT, logging an error and counting it. - A peer too old to prove quiescence is also released, once, loudly. - STRICT: never release without proof. A request that cannot get it stays - non-terminal, and the worker fails once too many accumulate, so the - restart releases the pages safely rather than corrupting a peer. - """ - - OFF = 0 - WARN = 1 - STRICT = 2 - - class InvariantCheckLevel(IntEnum): """Signal level for value/index validity checks (see invariants.py). @@ -413,18 +396,6 @@ class Envs: SGLANG_DISAGGREGATION_HEARTBEAT_INTERVAL = EnvFloat(5.0) SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE = EnvInt(2) SGLANG_DISAGGREGATION_WAITING_TIMEOUT = EnvInt(300) - # Whether a failed KV transfer may hand its KV pages back to the allocator - # before the transfer is known to have stopped. See TransferBarrierLevel. - SGLANG_DISAGGREGATION_TRANSFER_BARRIER = EnvInt(TransferBarrierLevel.WARN) - # WARN only: upper bound, in seconds, on how long a failed transfer holds its - # pages before they are released anyway. Keep this above the manager socket's - # 30s ZMQ send timeout, because a transfer holds its room while notifying the - # peer; a shorter value reports spurious timeouts whenever a decode stops - # reading. STRICT ignores it. - SGLANG_DISAGGREGATION_QUIESCE_TIMEOUT = EnvFloat(60.0) - # STRICT only: how many requests may be stuck waiting for proof before the - # worker gives up and fails, so that a restart releases the pages safely. - SGLANG_DISAGGREGATION_MAX_UNQUIESCED = EnvInt(256) SGLANG_DISAGGREGATION_NIXL_BACKEND = EnvStr("UCX") SGLANG_DISAGGREGATION_NIXL_BACKEND_PARAMS = EnvStr("{}") SGLANG_DISAGG_PREFILL_EARLY_SEND_CACHED_PREFIX = EnvBool(True) diff --git a/test/registered/unit/disaggregation/test_transfer_ownership_barrier.py b/test/registered/unit/disaggregation/test_transfer_ownership_barrier.py index e97052ac8fbd..8ef6afba4cd1 100644 --- a/test/registered/unit/disaggregation/test_transfer_ownership_barrier.py +++ b/test/registered/unit/disaggregation/test_transfer_ownership_barrier.py @@ -4,8 +4,8 @@ transfer work can still read or write them. The properties under test are: * pages stay owned while transfer work is in flight (safety); -* ownership is always released eventually, even if a peer never answers - (liveness -- a wedged transport must not pin pages forever); +* proof that never arrives is loud and bounded: reported after a deadline, + escalated once too many requests are stuck -- but never a silent release; * the deferral only happens where the scheduler can handle it, and never weakens the failure diagnostics the operator sees. """ @@ -34,7 +34,6 @@ ) from sglang.srt.disaggregation.mooncake.conn import ( ABORT_RETRY_INTERVAL_S, - MAX_EMERGENCY_SCAN, MooncakeKVManager, MooncakeKVReceiver, MooncakeKVSender, @@ -42,32 +41,24 @@ TransferInfo, ) from sglang.srt.disaggregation.utils import DisaggregationMode, poll_and_all_reduce -from sglang.srt.environ import TransferBarrierLevel from sglang.test.ci.ci_register import register_cpu_ci -register_cpu_ci(est_time=6, suite="base-a-test-cpu") +register_cpu_ci(est_time=5, suite="base-a-test-cpu") ROOM = 7 -def make_prefill_manager( - quiesce_timeout=30.0, room_sweep_ttl=300.0, barrier=TransferBarrierLevel.WARN -): +def make_prefill_manager(room_sweep_ttl=300.0): """A prefill-side manager with only its ownership-barrier state wired up.""" mgr = MooncakeKVManager.__new__(MooncakeKVManager) mgr.disaggregation_mode = DisaggregationMode.PREFILL - mgr.quiesce_timeout = quiesce_timeout - mgr.transfer_barrier = barrier - mgr.max_unquiesced = 256 mgr._unquiesced_rooms = set() mgr._unquiesced_lock = threading.Lock() mgr.bootstrap_timeout = 300 mgr._room_sweep_ttl = room_sweep_ttl mgr._room_lifetimes = OrderedDict() mgr._room_lifetimes_lock = threading.Lock() - mgr._endpoint_send_locks = {} - mgr._endpoint_send_locks_lock = threading.Lock() mgr.request_status = {} mgr.transfer_infos = {} mgr.req_to_decode_prefix_len = {} @@ -98,7 +89,6 @@ def make_sender(mgr, room=ROOM, status=KVPoll.WaitingForInput): sender.conclude_state = None sender._quiescing = False sender._quiesce_deadline = float("inf") - sender._quiesce_timed_out = False sender.trace_ctx = SimpleNamespace( abort=Mock(), trace_req_finish=Mock(), @@ -125,7 +115,6 @@ def make_receiver(mgr, room=ROOM, bootstrap_infos=None, metadata_sent=True): receiver._abort_targets = [] receiver._expected_abort_acks = set() receiver._received_abort_acks = set() - receiver._peer_lacks_barrier = False receiver._last_abort_send = float("-inf") receiver._owns_room = True receiver._abort_lock = threading.Lock() @@ -142,16 +131,11 @@ def make_receiver(mgr, room=ROOM, bootstrap_infos=None, metadata_sent=True): return receiver -def make_decode_manager( - quiesce_timeout=30.0, staging_handler=None, barrier=TransferBarrierLevel.WARN -): +def make_decode_manager(staging_handler=None): mgr = MooncakeKVManager.__new__(MooncakeKVManager) - mgr.quiesce_timeout = quiesce_timeout mgr._receivers = {} mgr._receivers_lock = threading.Lock() mgr._staging_handler = staging_handler - mgr.transfer_barrier = barrier - mgr.max_unquiesced = 256 mgr._unquiesced_rooms = set() mgr._unquiesced_lock = threading.Lock() mgr.request_status = {} @@ -267,6 +251,8 @@ def test_a_queued_chunk_cannot_attach_to_a_recycled_room(self): trace_ctx=Mock(), ) first.clear() # the request concludes, the room is released + self.assertNotIn(ROOM, mgr._room_lifetimes) + self.assertNotIn(ROOM, mgr.request_status) # a new request draws the same bootstrap_room make_sender(mgr) @@ -288,40 +274,16 @@ def _poll(sender): class TestRoomTransferLifetime(unittest.TestCase): """The primitive that decides when a room's pages may be reused.""" - def test_open_room_is_not_quiesced_and_admits_leases(self): - lifetime = RoomTransferLifetime() - self.assertFalse(lifetime.is_quiesced()) - self.assertTrue(lifetime.try_lease()) - lifetime.end_lease() - self.assertFalse(lifetime.is_quiesced()) - - def test_close_stops_admission_and_waits_for_outstanding_lease(self): + def test_lease_lifecycle(self): lifetime = RoomTransferLifetime() self.assertTrue(lifetime.try_lease()) lifetime.close() - self.assertFalse(lifetime.try_lease()) - self.assertFalse(lifetime.is_quiesced()) - self.assertFalse(lifetime.wait_quiesced(timeout=0.01)) + self.assertFalse(lifetime.try_lease(), "a closed room admits no work") + self.assertFalse(lifetime.is_quiesced(), "a lease is still outstanding") lifetime.end_lease() self.assertTrue(lifetime.is_quiesced()) - self.assertTrue(lifetime.wait_quiesced(timeout=1)) - - def test_waiter_wakes_when_the_last_lease_is_returned(self): - lifetime = RoomTransferLifetime() - self.assertTrue(lifetime.try_lease()) - lifetime.close() - quiesced = threading.Event() - - waiter = threading.Thread( - target=lambda: quiesced.set() if lifetime.wait_quiesced(2) else None - ) - waiter.start() - self.assertFalse(quiesced.wait(0.05)) - lifetime.end_lease() - self.assertTrue(quiesced.wait(1)) - waiter.join() def test_abort_token_authorization(self): lifetime = RoomTransferLifetime() @@ -331,7 +293,7 @@ def test_abort_token_authorization(self): lifetime.add_abort_token(b"mine") self.assertTrue(lifetime.authorizes_abort(b"mine")) self.assertFalse(lifetime.authorizes_abort(b"someone-elses")) - # Peers that predate the token protocol send none; honour those aborts. + # A tokenless abort cannot be authenticated; honour it, fail-safe. self.assertTrue(lifetime.authorizes_abort(b"")) @@ -421,77 +383,6 @@ def test_worker_exception_propagates_after_draining(self): class TestPrefillOwnership(unittest.TestCase): """The prefill's pages are the transfer source.""" - def test_transfer_worker_holds_the_lease_across_the_native_transfer(self): - entered_transfer = threading.Event() - release_transfer = threading.Event() - - mgr = make_prefill_manager() - mgr.request_status[ROOM] = KVPoll.WaitingForInput - mgr.transfer_infos[ROOM] = { - "session:1": SimpleNamespace( - room=ROOM, - endpoint="127.0.0.1", - dst_port=1, - mooncake_session_id="session:1", - dst_kv_indices=np.array([0], dtype=np.int32), - is_dummy=False, - ) - } - mgr.decode_kv_args_table = { - "session:1": SimpleNamespace( - dst_attn_tp_size=1, - dst_kv_ptrs=[1], - dst_kv_layer_ids=[0], - dst_state_layer_ids=[], - ) - } - mgr.session_lock = threading.Lock() - mgr.failed_sessions = set() - mgr.is_mla_backend = True - mgr.is_hybrid_mla_backend = False - mgr.attn_tp_size, mgr.attn_tp_rank = 1, 0 - mgr.attn_cp_size, mgr.attn_cp_rank = 1, 0 - mgr.pp_size, mgr.pp_rank = 1, 0 - mgr.attn_dp_rank = 0 - - def blocking_send_kvcache(*_args, **_kwargs): - entered_transfer.set() - release_transfer.wait() - return 0 - - mgr.send_kvcache = blocking_send_kvcache - lifetime = mgr._room_lifetime(ROOM, create=True) - sender = make_sender(mgr) - - queue = FastQueue() - threading.Thread( - target=mgr.transfer_worker, args=(queue, Mock()), daemon=True - ).start() - queue.put( - TransferKVChunk( - room=ROOM, - prefill_kv_indices=np.array([0], dtype=np.int32), - index_slice=slice(0, 1), - is_last_chunk=False, - prefill_aux_index=None, - state_indices=None, - trace_ctx=Mock(), - ) - ) - self.assertTrue(entered_transfer.wait(1)) - - # An abort now must not let the scheduler reclaim the pages. - sender._close_barrier() - self.assertTrue(lifetime.is_closed()) - self.assertFalse(sender.advance_failure_quiescence()) - - release_transfer.set() - for _ in range(200): - if sender.advance_failure_quiescence(): - break - time.sleep(0.01) - self.assertTrue(sender.advance_failure_quiescence()) - def test_closed_room_admits_no_new_chunks(self): mgr = make_prefill_manager() sender = make_sender(mgr) @@ -513,26 +404,6 @@ def test_released_room_cannot_be_resurrected_by_a_queued_chunk(self): self.assertIsNone(mgr.try_lease_room(ROOM)) self.assertNotIn(ROOM, mgr._room_lifetimes) - def test_quiescence_is_bounded_when_a_transfer_never_returns(self): - mgr = make_prefill_manager(quiesce_timeout=0.05) - sender = make_sender(mgr) - lifetime = mgr._room_lifetime(ROOM, create=False) - self.assertTrue(lifetime.try_lease()) # never returned - - with patch( - "sglang.srt.disaggregation.mooncake.conn.TRANSFER_QUIESCE_TIMEOUTS" - ) as metric: - self.assertFalse(sender.advance_failure_quiescence()) - time.sleep(0.06) - self.assertTrue( - sender.advance_failure_quiescence(), - "a wedged transfer must not pin the request forever", - ) - metric.inc.assert_called_once_with() - # The timeout is reported once, not on every subsequent poll. - self.assertTrue(sender.advance_failure_quiescence()) - metric.inc.assert_called_once_with() - def test_bootstrapping_sender_is_immediately_quiesced(self): # Nothing has been submitted, so the request can fail without any wait. # This is what keeps KVPoll.Transferring out of the bootstrap queues. @@ -540,13 +411,6 @@ def test_bootstrapping_sender_is_immediately_quiesced(self): sender = make_sender(mgr, status=KVPoll.Bootstrapping) self.assertTrue(sender.advance_failure_quiescence()) - def test_clear_forgets_the_room(self): - mgr = make_prefill_manager() - sender = make_sender(mgr) - sender.clear() - self.assertNotIn(ROOM, mgr._room_lifetimes) - self.assertNotIn(ROOM, mgr.request_status) - def test_tracked_rooms_are_bounded_without_evicting_a_live_room(self): mgr = make_prefill_manager() live = mgr._room_lifetime(-1, create=True) @@ -564,48 +428,15 @@ def test_tracked_rooms_are_bounded_without_evicting_a_live_room(self): -1, mgr._room_lifetimes, "a claimed, leased room must never be evicted" ) - def test_emergency_reclaim_cost_does_not_grow_with_the_map(self): - # A saturated map of non-reclaimable rooms must not make every new room - # O(tracked rooms) on the bootstrap thread. - mgr = make_prefill_manager() - with patch("sglang.srt.disaggregation.mooncake.conn.MAX_TRACKED_ROOMS", 8): - for room in range(400): - lifetime = mgr._room_lifetime(room, create=True) - mgr.open_room_transfers(room) - self.assertTrue(lifetime.try_lease()) - probe = Mock(side_effect=lambda: False) - with patch.object(RoomTransferLifetime, "is_reclaimable", probe): - mgr._room_lifetime(99999, create=True) - self.assertLessEqual( - probe.call_count, - MAX_EMERGENCY_SCAN, - "the reclaim scan must be bounded per insert", - ) - - def test_sweep_reclaims_rooms_no_sender_will_release(self): - mgr = make_prefill_manager(room_sweep_ttl=0.0) - for room in range(5): - mgr._room_lifetime(room, create=True) # decode metadata only - mgr.open_room_transfers(3) # this one has a local sender - self.assertEqual(mgr._sweep_room_lifetimes(), 4) - self.assertEqual(list(mgr._room_lifetimes), [3]) - - def test_sweep_keeps_young_rooms_so_a_late_sender_still_finds_them(self): + def test_sweep_reclaims_abandoned_rooms_but_keeps_young_and_claimed_ones(self): mgr = make_prefill_manager(room_sweep_ttl=300.0) - mgr._room_lifetime(1, create=True) - self.assertEqual(mgr._sweep_room_lifetimes(), 0) - self.assertIn(1, mgr._room_lifetimes) + mgr._room_lifetime(1, create=True).created_at = 0 # abandoned: reclaim + mgr._room_lifetime(2, create=True) # young: a sender may still arrive + mgr.open_room_transfers(3) # claimed: its sender releases it + mgr._room_lifetime(3, create=False).created_at = 0 - def test_barrier_can_be_disabled(self): - mgr = make_prefill_manager(barrier=TransferBarrierLevel.OFF) - sender = make_sender(mgr) - lifetime = mgr._room_lifetime(ROOM, create=False) - self.assertTrue(lifetime.try_lease()) - with patch( - "sglang.srt.disaggregation.mooncake.conn.TRANSFER_QUIESCE_TIMEOUTS" - ) as metric: - self.assertTrue(sender.advance_failure_quiescence()) - metric.inc.assert_not_called() + self.assertEqual(mgr._sweep_room_lifetimes(), 1) + self.assertEqual(sorted(mgr._room_lifetimes), [2, 3]) class TestPrefillAbortProtocol(unittest.TestCase): @@ -704,27 +535,6 @@ def test_abort_ack_is_withheld_until_the_room_drains(self): time.sleep(0.01) self.assertEqual(sent, [[b"ABORT_ACK", str(ROOM).encode("ascii"), b"token"]]) - def test_one_pump_thread_serves_every_pending_ack(self): - mgr = make_prefill_manager() - before = threading.active_count() - mgr._start_transfer_bookkeeping() - mgr._send_manager_message = Mock() - lifetimes = [] - for room in range(64): - lifetime = mgr._room_lifetime(room, create=True) - self.assertTrue(lifetime.try_lease()) - lifetime.close() - lifetimes.append(lifetime) - mgr.request_abort_ack(lifetime, room, "127.0.0.1", 1, b"t") - time.sleep(0.05) - self.assertLessEqual( - threading.active_count() - before, - 2, - "aborts must not spawn a thread each", - ) - for lifetime in lifetimes: - lifetime.end_lease() - class TestNoUnboundedWaits(unittest.TestCase): """Nothing on the ownership path may wait on a peer without a bound. @@ -843,95 +653,6 @@ def _bookkeeping_thread(): ) -class TestStagingComposition(unittest.TestCase): - """Staging teardown is upstream's (release_room); this PR gates when it runs. - - release_room() drains the scatter stream and frees the room's reservations on - every teardown, which covers local CUDA work. It cannot know about a remote - prefill still writing into those buffers over RDMA -- that is what the ACK - gate provides, by making the terminal transition (and therefore teardown) - unreachable until every peer has confirmed. - """ - - def _handler(self): - allocator = SimpleNamespace( - free=Mock(), - get_watermark=Mock(return_value=(0, 0)), - _scatter_stream=None, - ) - staged = SimpleNamespace( - chunk_staging_infos=[(11, 0, 0, 128, 1), (12, 128, 0, 256, 1)], - prefill_info=SimpleNamespace(attn_tp_size=2), - bootstrap_infos=[], - _connect_to_bootstrap_server=Mock(), - ) - decode_req = SimpleNamespace( - kv_receiver=staged, req=SimpleNamespace(bootstrap_room=ROOM) - ) - handler = DecodeStagingHandler( - kv_manager=SimpleNamespace( - _staging_ctx=SimpleNamespace( - room_receivers={ROOM: staged}, room_bootstrap={ROOM: []} - ) - ), - staging_allocator=allocator, - kv_buffer_info={}, - decode_tp=1, - total_kv_heads=8, - tp_rank=0, - scheduler=SimpleNamespace(), - ) - handler.register_decode_req(ROOM, decode_req) - return handler, staged, allocator - - def test_teardown_releases_reservations(self): - # Upstream's behaviour, asserted here because this PR depends on it: the - # barrier no longer frees staging itself. - handler, staged, allocator = self._handler() - handler.unregister_decode_req(ROOM) - self.assertEqual( - sorted(call.args[0] for call in allocator.free.call_args_list), [11, 12] - ) - self.assertEqual(staged.chunk_staging_infos, [(-1, -1, 0, -1, 0)] * 2) - - def test_teardown_is_unreachable_until_peers_confirm(self): - # The composition that makes release_room safe against a remote writer. - mgr = make_decode_manager() - receiver = make_receiver(mgr, bootstrap_infos=[{"rank": 0}]) - self.assertFalse( - receiver.advance_failure_quiescence(), - "the request must not go terminal, so unregister_decode_req cannot run", - ) - receiver.record_abort_ack(next(iter(receiver._expected_abort_acks))) - self.assertTrue(receiver.advance_failure_quiescence()) - - def test_watermark_broadcast_never_blocks_the_scheduler_loop(self): - # release_room broadcasts a watermark from the scheduler loop over sockets - # with no send timeout, so it must not wait for a peer. - handler, _staged, _allocator = self._handler() - flags_used = [] - - class WouldBlock: - def send_multipart(self, parts, flags=0): - flags_used.append(bool(flags & zmq.NOBLOCK)) - raise zmq.Again() - - handler.register_wm_subscriber( - SimpleNamespace( - bootstrap_infos=[{"rank": 0}], - _connect_to_bootstrap_server=lambda info: ( - WouldBlock(), - threading.Lock(), - ), - ), - "session", - ) - started = time.monotonic() - handler.unregister_decode_req(ROOM) - self.assertLess(time.monotonic() - started, 1.0) - self.assertTrue(flags_used and all(flags_used)) - - class TestDecodeOwnership(unittest.TestCase): """The decode's pages are the transfer destination, so peers must confirm.""" @@ -989,6 +710,8 @@ def test_abort_notifications_carry_a_stable_token_and_are_retried(self): ) def test_failed_metadata_send_does_not_expect_unexposed_peers(self): + # A peer whose metadata send failed never learned where to write, so + # waiting for its ACK would withhold the pages forever. mgr = make_decode_manager() mgr.enable_staging = False receiver = make_receiver( @@ -1024,55 +747,21 @@ def connect(info): self.assertNotIn(tokens[1], receiver._expected_abort_acks) self.assertNotIn(tokens[2], receiver._expected_abort_acks) - def test_quiescence_is_bounded_when_a_peer_never_acknowledges(self): - mgr = make_decode_manager(quiesce_timeout=0.05) - receiver = make_receiver(mgr, bootstrap_infos=[{"rank": 0}]) - - with patch( - "sglang.srt.disaggregation.mooncake.conn.TRANSFER_QUIESCE_TIMEOUTS" - ) as metric: - self.assertFalse(receiver.advance_failure_quiescence()) - time.sleep(0.06) - self.assertTrue( - receiver.advance_failure_quiescence(), - "a dead prefill must not pin decode pages forever", - ) - metric.inc.assert_called_once_with() - - def test_legacy_prefill_ack_stops_the_wait(self): - # A prefill that predates the barrier ACKs immediately and without a - # nonce. It cannot promise a drain, so waiting for it is pointless; - # degrade to the previous behaviour instead of stalling. + def test_unauthorized_acks_are_not_proof(self): mgr = make_decode_manager() receiver = make_receiver(mgr, bootstrap_infos=[{"rank": 0}]) self.assertFalse(receiver.advance_failure_quiescence()) - receiver.record_abort_ack(None) - self.assertTrue(receiver.advance_failure_quiescence()) - - def test_legacy_ack_does_not_satisfy_a_token_aware_peer(self): - mgr = make_decode_manager() - receiver = make_receiver( - mgr, bootstrap_infos=[{"rank": "legacy"}, {"rank": "current"}] - ) - self.assertFalse(receiver.advance_failure_quiescence()) - tokens = {info["rank"]: token for info, token in receiver._abort_targets} + # A nonce minted for a previous occupant of this recycled room. + receiver.record_abort_ack(b"nonce-from-a-previous-request") + # A tokenless ACK from a peer that predates the barrier: it acknowledges + # before its transfers drain, so it proves nothing. receiver.record_abort_ack(None) - self.assertFalse( - receiver.advance_failure_quiescence(), - "one legacy ACK must not satisfy another rank's outstanding token", - ) + self.assertFalse(receiver.advance_failure_quiescence()) - receiver.record_abort_ack(tokens["current"]) + receiver.record_abort_ack(next(iter(receiver._expected_abort_acks))) self.assertTrue(receiver.advance_failure_quiescence()) - def test_stale_ack_for_another_request_is_ignored(self): - mgr = make_decode_manager() - receiver = make_receiver(mgr, bootstrap_infos=[{"rank": 0}]) - receiver.advance_failure_quiescence() - receiver.record_abort_ack(b"nonce-from-a-previous-request") - self.assertFalse(receiver.advance_failure_quiescence()) - def test_abort_ack_routes_to_the_registered_receiver_only(self): mgr = make_decode_manager() receiver = make_receiver(mgr, bootstrap_infos=[{"rank": 0}]) @@ -1084,33 +773,6 @@ def test_abort_ack_routes_to_the_registered_receiver_only(self): # Unknown rooms must not raise on the transport thread. mgr._handle_abort_ack([b"ABORT_ACK", b"999", b"tok"]) - def test_only_one_receiver_owns_a_room(self): - mgr = make_decode_manager() - first = make_receiver(mgr) - self.assertTrue(mgr.register_receiver(first)) - second = make_receiver(mgr) - self.assertFalse( - mgr.register_receiver(second), - "colliding rooms must not let a second receiver take over", - ) - self.assertIs(mgr._receivers[ROOM], first) - - # The loser must not evict the owner's registration on cleanup. - mgr.unregister_receiver(second) - self.assertIs(mgr._receivers[ROOM], first) - mgr.unregister_receiver(first) - self.assertTrue(mgr.register_receiver(second)) - - def test_a_receiver_that_lost_the_room_touches_no_shared_state(self): - handler = Mock() - mgr = make_decode_manager(staging_handler=handler) - loser = make_receiver(mgr, metadata_sent=False) - loser._owns_room = False - self.assertTrue(loser.advance_failure_quiescence()) - handler.begin_abort.assert_not_called() - loser.clear = MooncakeKVReceiver.clear.__get__(loser) - handler.forget_abort.assert_not_called() - def test_late_transport_messages_are_dropped_while_tearing_down(self): mgr = make_decode_manager() receiver = make_receiver(mgr) @@ -1120,97 +782,86 @@ def test_late_transport_messages_are_dropped_while_tearing_down(self): self.assertTrue(mgr._is_tearing_down(ROOM)) self.assertFalse(mgr._is_tearing_down(999)) + def test_user_abort_still_records_its_reason(self): + # Releasing pages safely must not cost the operator the real reason. + mgr = make_decode_manager() + receiver = make_receiver(mgr, bootstrap_infos=[{"rank": 0}]) + receiver.abort_notified = False + MooncakeKVReceiver.abort(receiver) + mgr.record_failure.assert_called_once() + self.assertIn("Aborted by AbortReq", mgr.record_failure.call_args.args[1]) + -class TestBarrierLevels(unittest.TestCase): +class TestStrictBarrier(unittest.TestCase): """What happens when proof of quiescence never arrives. - WARN trades a narrow, loud corruption window for liveness; STRICT refuses to - reuse a page it cannot prove is idle, and escalates instead. + The pages are never released without proof: the wait is reported after the + quiescence deadline, and once too many requests are stuck the worker fails, + so a restart reclaims every withheld page safely at once. """ - def _stuck_sender(self, barrier, max_unquiesced=256): - mgr = make_prefill_manager(quiesce_timeout=0.01, barrier=barrier) - mgr.max_unquiesced = max_unquiesced + def test_prefill_pages_are_withheld_when_a_transfer_never_returns(self): + mgr = make_prefill_manager() sender = make_sender(mgr) lifetime = mgr._room_lifetime(ROOM, create=False) self.assertTrue(lifetime.try_lease()) # a worker that never returns - return mgr, sender - - @staticmethod - def _arm_and_expire(sender): - """Arm the deadline, then let it pass.""" - with patch("sglang.srt.disaggregation.mooncake.conn.TRANSFER_QUIESCE_TIMEOUTS"): - sender.advance_failure_quiescence() - time.sleep(0.02) - - def test_warn_releases_once_and_loudly(self): - mgr, sender = self._stuck_sender(TransferBarrierLevel.WARN) - self.assertFalse(sender.advance_failure_quiescence()) - time.sleep(0.02) - with patch( - "sglang.srt.disaggregation.mooncake.conn.TRANSFER_QUIESCE_TIMEOUTS" - ) as metric: - self.assertTrue(sender.advance_failure_quiescence()) - self.assertTrue(sender.advance_failure_quiescence()) - metric.inc.assert_called_once_with() - def test_strict_never_releases_without_proof(self): - mgr, sender = self._stuck_sender(TransferBarrierLevel.STRICT) - self._arm_and_expire(sender) - with patch( - "sglang.srt.disaggregation.mooncake.conn.TRANSFER_QUIESCE_TIMEOUTS" - ) as metric: - with patch("sglang.srt.disaggregation.mooncake.conn.logger.error") as error: - for _ in range(5): - self.assertFalse( - sender.advance_failure_quiescence(), - "STRICT must not hand back a page it cannot prove is idle", - ) + with patch("sglang.srt.disaggregation.mooncake.conn.QUIESCE_TIMEOUT_S", 0.0): + with patch( + "sglang.srt.disaggregation.mooncake.conn.TRANSFER_QUIESCE_TIMEOUTS" + ) as metric: + with patch( + "sglang.srt.disaggregation.mooncake.conn.logger.error" + ) as error: + for _ in range(5): + self.assertFalse( + sender.advance_failure_quiescence(), + "a page that cannot be proven idle must not be reused", + ) metric.inc.assert_called_once_with() error.assert_called_once() self.assertIn( ROOM, mgr._unquiesced_rooms, "the stuck room must be tracked for escalation" ) - def test_strict_escalates_rather_than_leaking_forever(self): - # Withholding pages is only safe while there are few of them; past the - # cap the worker fails so a restart reclaims them all at once. - mgr, sender = self._stuck_sender(TransferBarrierLevel.STRICT, max_unquiesced=2) - self._arm_and_expire(sender) - with patch("sglang.srt.disaggregation.mooncake.conn.TRANSFER_QUIESCE_TIMEOUTS"): - sender.advance_failure_quiescence() - mgr._unquiesced_rooms.add(999) # a second stuck request - with self.assertRaises(RuntimeError) as cm: - mgr.release_without_proof(1234, "third stuck request") - self.assertIn("cannot be proven quiesced", str(cm.exception)) - - def test_off_restores_legacy_behaviour(self): - mgr, sender = self._stuck_sender(TransferBarrierLevel.OFF) - with patch( - "sglang.srt.disaggregation.mooncake.conn.TRANSFER_QUIESCE_TIMEOUTS" - ) as metric: - self.assertTrue(sender.advance_failure_quiescence()) - metric.inc.assert_not_called() + # Proof arriving late still recovers the request. + lifetime.end_lease() + self.assertTrue(sender.advance_failure_quiescence()) + self.assertNotIn(ROOM, mgr._unquiesced_rooms) - def test_a_receiver_that_lost_its_room_does_not_wait_for_unroutable_proof(self): + def test_decode_pages_are_withheld_when_a_peer_never_acknowledges(self): mgr = make_decode_manager() receiver = make_receiver(mgr, bootstrap_infos=[{"rank": 0}]) - receiver._owns_room = False - with patch("sglang.srt.disaggregation.mooncake.conn.TRANSFER_QUIESCE_TIMEOUTS"): - self.assertTrue( - receiver.advance_failure_quiescence(), - "ACKs are routed to the owning receiver, so this one can never " - "receive proof and must not wait for it", - ) + with patch("sglang.srt.disaggregation.mooncake.conn.QUIESCE_TIMEOUT_S", 0.0): + with patch( + "sglang.srt.disaggregation.mooncake.conn.TRANSFER_QUIESCE_TIMEOUTS" + ) as metric: + for _ in range(3): + self.assertFalse( + receiver.advance_failure_quiescence(), + "a dead prefill must not cause a silent release", + ) + metric.inc.assert_called_once_with() + self.assertIn(ROOM, mgr._unquiesced_rooms) -class TestStrictCannotBeDowngraded(unittest.TestCase): - """STRICT must not degrade into the most permissive policy by accident.""" + def test_escalation_once_too_many_requests_are_stuck(self): + # Withholding pages is only safe while there are few of them; past the + # cap the worker fails so a restart reclaims them all at once. + mgr = make_prefill_manager() + with patch("sglang.srt.disaggregation.mooncake.conn.MAX_UNQUIESCED_ROOMS", 2): + with patch( + "sglang.srt.disaggregation.mooncake.conn.TRANSFER_QUIESCE_TIMEOUTS" + ): + mgr.report_unquiesced(1, "first stuck request") + with self.assertRaises(KVTransferBarrierEscalation) as cm: + mgr.report_unquiesced(2, "second stuck request") + self.assertIn("cannot be proven quiesced", str(cm.exception)) def test_escalation_reaches_the_scheduler(self): # The poll path contains backend faults so a bug cannot kill the engine. # The escalation is not a bug: swallowing it would release exactly the - # pages STRICT exists to withhold. + # pages the barrier exists to withhold. poller = SimpleNamespace( poll=Mock(return_value=KVPoll.Failed), advance_failure_quiescence=Mock( @@ -1252,33 +903,6 @@ def reduce(tensor, **_kwargs): poll_and_all_reduce([poller], object()) self.assertEqual(collective.call_count, 2) - def test_an_ordinary_backend_fault_is_still_contained(self): - poller = SimpleNamespace( - poll=Mock(return_value=KVPoll.Failed), - advance_failure_quiescence=Mock(side_effect=RuntimeError("backend bug")), - is_failure_quiescing=Mock(return_value=True), - ) - with patch( - "sglang.srt.disaggregation.utils.dist.all_reduce", - side_effect=lambda tensor, **kw: None, - ): - self.assertEqual(poll_and_all_reduce([poller], object()), [KVPoll.Failed]) - - def test_strict_does_not_accept_a_legacy_ack_as_proof(self): - # A peer that predates the barrier acknowledges before draining, so its - # ACK proves nothing. WARN accepts it to keep a rolling upgrade working; - # STRICT cannot, by definition. - for level, accepted in ( - (TransferBarrierLevel.WARN, True), - (TransferBarrierLevel.STRICT, False), - ): - with self.subTest(level=level): - mgr = make_decode_manager(barrier=level) - receiver = make_receiver(mgr, bootstrap_infos=[{"rank": 0}]) - receiver.advance_failure_quiescence() - receiver.record_abort_ack(None) - self.assertEqual(receiver._peers_quiesced(), accepted) - class TestRoomStateRetirement(unittest.TestCase): """Reclaiming a room must retire everything keyed by it.""" @@ -1303,33 +927,46 @@ def _room_with_metadata(self, mgr, room=ROOM): ] ) - def test_emergency_reclaim_retires_destinations_and_status_too(self): - # Same hazard as the sweep, via the other reclaim path: a request drawing - # a reclaimed room must not inherit the old decode's addresses. - mgr = make_prefill_manager() + def test_sweep_retires_destinations_and_status_together(self): + # Otherwise a later request drawing the same bootstrap_room inherits the + # old decode's addresses and a stale status. + mgr = make_prefill_manager(room_sweep_ttl=0.0) self._room_with_metadata(mgr) self.assertIn(ROOM, mgr.transfer_infos) - mgr._room_lifetimes[ROOM].created_at = 0 - with patch("sglang.srt.disaggregation.mooncake.conn.MAX_TRACKED_ROOMS", 1): - mgr._room_lifetime(ROOM + 1, create=True) + self.assertIn(ROOM, mgr.request_status) + + self.assertEqual(mgr._sweep_room_lifetimes(), 1) self.assertNotIn(ROOM, mgr._room_lifetimes) self.assertNotIn(ROOM, mgr.transfer_infos) self.assertNotIn(ROOM, mgr.request_status) self.assertNotIn(ROOM, mgr.req_to_decode_prefix_len) - def test_emergency_reclaim_preserves_live_metadata_first_room(self): + def test_emergency_reclaim_spares_young_rooms_then_retires_old_ones_fully(self): + # Same hazard as the sweep, via the reclaim that runs when the tracked- + # room cap is hit on insert. mgr = make_prefill_manager(room_sweep_ttl=300.0) self._room_with_metadata(mgr) with patch("sglang.srt.disaggregation.mooncake.conn.MAX_TRACKED_ROOMS", 1): + # Young, metadata-first room: a sender may still arrive, so the cap + # is soft and the room survives. mgr._room_lifetime(ROOM + 1, create=True) - - self.assertIn(ROOM, mgr._room_lifetimes) - self.assertIn(ROOM, mgr.transfer_infos) - self.assertIn(ROOM, mgr.request_status) - self.assertEqual(len(mgr._room_lifetimes), 2, "the tracked-room cap is soft") + self.assertIn(ROOM, mgr.transfer_infos) + self.assertEqual(len(mgr._room_lifetimes), 2, "the cap is soft") + + # Once aged out, the room is retired with everything keyed by it: a + # request drawing it later must not inherit the old decode's + # addresses or a stale status. + mgr._room_lifetimes[ROOM].created_at = 0 + mgr._room_lifetime(ROOM + 2, create=True) + self.assertNotIn(ROOM, mgr._room_lifetimes) + self.assertNotIn(ROOM, mgr.transfer_infos) + self.assertNotIn(ROOM, mgr.request_status) + self.assertNotIn(ROOM, mgr.req_to_decode_prefix_len) def test_metadata_publication_holds_the_lifetime_generation_lock(self): + # Otherwise the sweeper can retire the lifetime between validation and + # publication, leaving orphan metadata for a later generation to inherit. mgr = make_prefill_manager() mgr.session_lock = threading.Lock() mgr.failed_sessions, mgr.session_failures = set(), {} @@ -1360,53 +997,6 @@ def __setitem__(self, key, value): self.assertIn(ROOM, mgr.transfer_infos) self.assertIn(ROOM, mgr.request_status) - def test_retirement_is_atomic_against_a_new_generation(self): - # The lifetime and the rest of the room's state must go in one step: a - # request created in between would have its own metadata deleted. - mgr = make_prefill_manager(room_sweep_ttl=0.0) - self._room_with_metadata(mgr) - observed = {} - real_retire = mgr._retire_rooms_locked - - def retire(rooms): - observed["held"] = mgr._room_lifetimes_lock.locked() - real_retire(rooms) - - mgr._retire_rooms_locked = retire - self.assertEqual(mgr._sweep_room_lifetimes(), 1) - self.assertTrue(observed["held"], "retirement must run under the lifetime lock") - - def test_sweep_retires_destinations_and_status_together(self): - # Otherwise a later request drawing the same bootstrap_room inherits the - # old decode's addresses and a stale status. - mgr = make_prefill_manager(room_sweep_ttl=0.0) - mgr.session_lock = threading.Lock() - mgr.failed_sessions, mgr.session_failures = set(), {} - mgr.decode_kv_args_table = {} - mgr.resolve_kv_replica_factor = Mock() - mgr._handle_bootstrap_metadata( - [ - str(ROOM).encode("ascii"), - b"127.0.0.1", - b"1", - b"session:1", - np.array([0], dtype=np.int32).tobytes(), - b"0", - b"", - b"1", - b"0", - b"token", - ] - ) - self.assertIn(ROOM, mgr.transfer_infos) - self.assertIn(ROOM, mgr.request_status) - - self.assertEqual(mgr._sweep_room_lifetimes(), 1) - self.assertNotIn(ROOM, mgr._room_lifetimes) - self.assertNotIn(ROOM, mgr.transfer_infos) - self.assertNotIn(ROOM, mgr.request_status) - self.assertNotIn(ROOM, mgr.req_to_decode_prefix_len) - class TestRoomCollisionIsolation(unittest.TestCase): """A receiver that lost a room collision must not damage the owner.""" @@ -1452,6 +1042,25 @@ def test_the_loser_does_not_tear_down_shared_state(self): self.assertIn(ROOM, mgr.required_prefill_response_num_table) self.assertIs(mgr._receivers[ROOM], owner) + # Unregistration is owner-checked in both directions. + mgr.unregister_receiver(loser) + self.assertIs(mgr._receivers[ROOM], owner) + mgr.unregister_receiver(owner) + self.assertTrue(mgr.register_receiver(loser)) + + def test_a_lost_room_reports_instead_of_waiting_for_unroutable_proof(self): + # ACKs are routed to the owning receiver, so the loser can never receive + # proof; pretending to wait out the deadline would just hide that. + mgr = make_decode_manager() + receiver = make_receiver(mgr, bootstrap_infos=[{"rank": 0}]) + receiver._owns_room = False + with patch( + "sglang.srt.disaggregation.mooncake.conn.TRANSFER_QUIESCE_TIMEOUTS" + ) as metric: + self.assertFalse(receiver.advance_failure_quiescence()) + metric.inc.assert_called_once_with() + self.assertIn(ROOM, mgr._unquiesced_rooms, "reported without waiting") + class TestEndpointSendSerialization(unittest.TestCase): def test_connection_validation_and_send_share_the_endpoint_lock(self): @@ -1491,7 +1100,19 @@ def connect(*_args, **_kwargs): class TestWatermarkDelivery(unittest.TestCase): """A dropped watermark must be retried; it can gate a prefill's progress.""" - def _handler(self, socket): + def test_a_backpressured_watermark_is_resent_and_never_blocks(self): + state = {"blocked": True} + sent, flags_used = [], [] + + class Socket: + def send_multipart(self, parts, flags=0): + # Sent from the scheduler loop over sockets with no ZMQ send + # timeout, so it must never wait for a peer. + flags_used.append(bool(flags & zmq.NOBLOCK)) + if state["blocked"]: + raise zmq.Again() + sent.append(parts) + allocator = SimpleNamespace( free=Mock(), get_watermark=Mock(return_value=(3, 128)), _scatter_stream=None ) @@ -1509,23 +1130,14 @@ def _handler(self, socket): handler.register_wm_subscriber( SimpleNamespace( bootstrap_infos=[{"rank": 0}], - _connect_to_bootstrap_server=lambda info: (socket, threading.Lock()), + _connect_to_bootstrap_server=lambda info: (Socket(), threading.Lock()), ), "session", ) - return handler - - def test_a_backpressured_watermark_is_resent(self): - state = {"blocked": True} - sent = [] - class Socket: - def send_multipart(self, parts, flags=0): - if state["blocked"]: - raise zmq.Again() - sent.append(parts) + handler.retry_pending_watermarks() + self.assertEqual(flags_used, [], "nothing pending, nothing sent") - handler = self._handler(Socket()) handler._free_and_send_watermark( 1, SimpleNamespace(req=SimpleNamespace(bootstrap_room=ROOM)) ) @@ -1537,17 +1149,7 @@ def send_multipart(self, parts, flags=0): self.assertEqual(len(sent), 1, "the current watermark must be resent") self.assertEqual(sent[0][1:3], [b"3", b"128"]) self.assertFalse(handler._wm_retry) - - def test_retry_is_free_when_nothing_is_pending(self): - sent = [] - - class Socket: - def send_multipart(self, parts, flags=0): - sent.append(parts) - - handler = self._handler(Socket()) - handler.retry_pending_watermarks() - self.assertEqual(sent, []) + self.assertTrue(all(flags_used), "every attempt must be non-blocking") class TestOwedAckBookkeeping(unittest.TestCase): @@ -1664,26 +1266,6 @@ def test_hicache_wrapper_delegates_the_barrier(self): self.assertTrue(gated.is_failure_quiescing()) -class TestFailureDiagnostics(unittest.TestCase): - """Releasing pages safely must not cost the operator the real reason.""" - - def test_propagated_failures_keep_their_attribution(self): - mgr = make_decode_manager() - receiver = make_receiver(mgr, bootstrap_infos=[{"rank": 0}]) - # A failure reported by the prefill (or another rank) drives quiescence, - # but must not be relabelled as a local user abort. - receiver.advance_failure_quiescence() - mgr.record_failure.assert_not_called() - - def test_user_abort_still_records_its_reason(self): - mgr = make_decode_manager() - receiver = make_receiver(mgr, bootstrap_infos=[{"rank": 0}]) - receiver.abort_notified = False - MooncakeKVReceiver.abort(receiver) - mgr.record_failure.assert_called_once() - self.assertIn("Aborted by AbortReq", mgr.record_failure.call_args.args[1]) - - class TestChunkedPrefillAbort(unittest.TestCase): """Aborting a chunked prefill must not free pages an earlier chunk is reading.""" @@ -1725,6 +1307,8 @@ def _run(self, mode): return req, scheduler, prepare_abort, release_kv_cache def test_prefill_defers_release_to_the_inflight_queue(self): + from sglang.srt.managers import scheduler as scheduler_mod + req, scheduler, prepare_abort, release_kv_cache = self._run( DisaggregationMode.PREFILL ) @@ -1739,6 +1323,15 @@ def test_prefill_defers_release_to_the_inflight_queue(self): self.assertIsNone(scheduler.chunked_req) self.assertIsNone(scheduler._pending_chunked_abort_req) + # Processing the same abort again must not enqueue the request twice. + scheduler._pending_chunked_abort_req = req + scheduler.chunked_req = req + with patch.object(scheduler_mod, "prepare_abort"), patch.object( + scheduler_mod, "release_kv_cache" + ): + scheduler_mod.Scheduler.process_pending_chunked_abort(scheduler) + self.assertEqual(scheduler.disagg_prefill_inflight_queue, [req]) + def test_non_disagg_release_is_unchanged(self): req, scheduler, _prepare_abort, release_kv_cache = self._run( DisaggregationMode.DECODE @@ -1748,20 +1341,6 @@ def test_non_disagg_release_is_unchanged(self): self.assertEqual(scheduler.disagg_prefill_inflight_queue, []) scheduler.ipc_channels.send_to_tokenizer.send_output.assert_called_once() - def test_deferral_is_idempotent(self): - from sglang.srt.managers import scheduler as scheduler_mod - - req = self._chunked_req() - scheduler = self._scheduler(req, DisaggregationMode.PREFILL) - with patch.object(scheduler_mod, "prepare_abort"), patch.object( - scheduler_mod, "release_kv_cache" - ): - scheduler_mod.Scheduler.process_pending_chunked_abort(scheduler) - scheduler._pending_chunked_abort_req = req - scheduler.chunked_req = req - scheduler_mod.Scheduler.process_pending_chunked_abort(scheduler) - self.assertEqual(scheduler.disagg_prefill_inflight_queue, [req]) - def test_pending_bootstrap_failure_preserves_user_abort(self): from sglang.srt.disaggregation import prefill as prefill_mod from sglang.srt.managers.schedule_batch import FINISH_ABORT @@ -1800,9 +1379,7 @@ def test_pending_bootstrap_failure_preserves_user_abort(self): class TestTransferInfoWire(unittest.TestCase): - """The abort nonce is an optional trailing frame.""" - - def test_metadata_without_a_token_parses_as_legacy(self): + def test_abort_token_is_an_optional_trailing_frame(self): msg = [ b"7", b"127.0.0.1", @@ -1816,22 +1393,11 @@ def test_metadata_without_a_token_parses_as_legacy(self): ] info = TransferInfo.from_zmq(msg) self.assertEqual(info.room, 7) - self.assertEqual(info.abort_token, b"") - - def test_metadata_with_a_token_parses_it(self): - msg = [ - b"7", - b"127.0.0.1", - b"1", - b"session", - np.array([0], dtype=np.int32).tobytes(), - b"0", - b"", - b"1", - b"0", + self.assertEqual(info.abort_token, b"", "a peer may omit the token frame") + self.assertEqual( + TransferInfo.from_zmq(msg + [b"nonce"]).abort_token, b"nonce", - ] - self.assertEqual(TransferInfo.from_zmq(msg).abort_token, b"nonce") + ) if __name__ == "__main__": From 807f1a07094a5cdcea8f61e01cfef1ca8fe06d85 Mon Sep 17 00:00:00 2001 From: Kastan Day Date: Mon, 10 Aug 2026 16:36:04 -0700 Subject: [PATCH 07/10] fix(disaggregation): drain DCP sibling layer transfers before reporting a failed send --- .../srt/disaggregation/mooncake/conn.py | 18 ++--- .../test_transfer_ownership_barrier.py | 68 +++++++++++++++++++ 2 files changed, 75 insertions(+), 11 deletions(-) diff --git a/python/sglang/srt/disaggregation/mooncake/conn.py b/python/sglang/srt/disaggregation/mooncake/conn.py index f31fbbcaaebe..0153932c3c4a 100644 --- a/python/sglang/srt/disaggregation/mooncake/conn.py +++ b/python/sglang/srt/disaggregation/mooncake/conn.py @@ -1479,17 +1479,13 @@ def process_layer(src_ptr: int, dst_ptr: int, token_item_len: int) -> int: ) if self.enable_custom_mem_pool: - futures = [ - executor.submit(process_layer, src_ptr, dst_ptr, token_item_len) - for src_ptr, dst_ptr, token_item_len in layers_params - ] - for future in concurrent.futures.as_completed(futures): - status = future.result() - if status != 0: - for pending in futures: - pending.cancel() - return status - return 0 + return submit_transfer_calls( + executor, + [ + (process_layer, (src_ptr, dst_ptr, token_item_len)) + for (src_ptr, dst_ptr, token_item_len) in layers_params + ], + ) transfer_blocks = [] for src_ptr, dst_ptr, token_item_len in layers_params: diff --git a/test/registered/unit/disaggregation/test_transfer_ownership_barrier.py b/test/registered/unit/disaggregation/test_transfer_ownership_barrier.py index 2ed3b2047978..70aa40cbc906 100644 --- a/test/registered/unit/disaggregation/test_transfer_ownership_barrier.py +++ b/test/registered/unit/disaggregation/test_transfer_ownership_barrier.py @@ -382,6 +382,74 @@ def test_worker_exception_propagates_after_draining(self): with self.assertRaises(ValueError): drain_transfer_futures(futures) + def test_dcp_send_failure_waits_for_a_running_sibling(self): + # Per-call-site guard: send_kvcache_dcp's custom-mem-pool branch must + # route through submit_transfer_calls like the other send paths. An + # early return on the first failed layer would end the worker's room + # lease while a sibling layer is still writing into decode's pages. + mgr = MooncakeKVManager.__new__(MooncakeKVManager) + mgr.enable_custom_mem_pool = True + mgr.kv_args = SimpleNamespace( + page_size=1, kv_data_ptrs=[0x1, 0x2], kv_layer_ids=[] + ) + mgr.get_mla_kv_ptrs_with_pp = lambda src, dst: (src, dst, None) + + sibling_running = threading.Event() + release_sibling = threading.Event() + returned = threading.Event() + result = [] + FAILING_LAYER_PTR = 0x1 + + def transfer_data(session_id, blocks): + if blocks[0][0] == FAILING_LAYER_PTR: + # Fail only once the sibling layer is genuinely mid-write, so + # cancelling a still-pending future cannot pass this trivially. + sibling_running.wait(5) + return -1 + sibling_running.set() + release_sibling.wait(5) + return 0 + + mgr._transfer_data = transfer_data + plan = SimpleNamespace( + src_token_indices=np.array([0]), dst_token_indices=np.array([0]) + ) + with patch( + "sglang.srt.disaggregation.mooncake.conn.build_dcp_token_transfer_plan", + return_value=plan, + ), concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor: + + def send(): + result.append( + mgr.send_kvcache_dcp( + "session", + np.array([0], dtype=np.int32), + [0x11, 0x22], + np.array([0], dtype=np.int32), + dcp_token_item_lens=[16, 16], + dst_dcp_size=1, + dst_dcp_rank=0, + src_page_offset=0, + decode_prefix_len=0, + num_kv_tokens=1, + executor=executor, + dst_layer_ids=[], + ) + ) + returned.set() + + sender = threading.Thread(target=send) + sender.start() + self.assertTrue(sibling_running.wait(1)) + self.assertFalse( + returned.wait(0.05), "returned while a sibling layer still ran" + ) + release_sibling.set() + self.assertTrue(returned.wait(1)) + sender.join() + + self.assertEqual(result, [-1], "the failing layer status must be reported") + class TestPrefillOwnership(unittest.TestCase): """The prefill's pages are the transfer source.""" From 5dd59d753f41b06dda6619d8ff4f4cd475c01bb1 Mon Sep 17 00:00:00 2001 From: Kastan Day Date: Mon, 10 Aug 2026 16:36:04 -0700 Subject: [PATCH 08/10] test(disaggregation): cover the decode HiCache restore-failure reroute --- .../test_transfer_ownership_barrier.py | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/test/registered/unit/disaggregation/test_transfer_ownership_barrier.py b/test/registered/unit/disaggregation/test_transfer_ownership_barrier.py index 70aa40cbc906..57105521d383 100644 --- a/test/registered/unit/disaggregation/test_transfer_ownership_barrier.py +++ b/test/registered/unit/disaggregation/test_transfer_ownership_barrier.py @@ -1370,6 +1370,53 @@ def test_hicache_wrapper_delegates_the_barrier(self): self.assertFalse(gated.advance_failure_quiescence()) self.assertTrue(gated.is_failure_quiescing()) + @patch("sglang.srt.disaggregation.decode.release_kv_cache") + @patch("sglang.srt.disaggregation.decode.poll_and_all_reduce") + def test_failed_restore_aborts_through_the_barrier_instead_of_releasing( + self, mock_poll, mock_release + ): + # A failed HiCache restore says nothing about the KV transfer: a + # prefill rank may still be writing into this request's pages. The + # request must be failed through the ownership barrier (abort, then + # wait for KVPoll.Failed) -- never released directly from the queue. + from sglang.srt.disaggregation.decode import ( + DecodeTransferQueue, + HiCacheRestoreResult, + ) + + decode_req = SimpleNamespace( + req=SimpleNamespace(rid="r1", bootstrap_room=ROOM, return_logprob=False), + kv_receiver=Mock(), + metadata_buffer_index=3, + hicache_restore_status=HiCacheRestoreResult.FAILED, + ) + queue = DecodeTransferQueue.__new__(DecodeTransferQueue) + queue.queue = [decode_req] + queue.enable_staging = False + queue.gloo_group = Mock() + queue.req_to_metadata_buffer_idx_allocator = Mock() + queue.tp_rank = 0 + queue.tree_cache = Mock() + queue.metadata_buffers = SimpleNamespace(bootstrap_room=[None] * 4) + queue.spec_algorithm = Mock() + queue.spec_algorithm.is_none.return_value = True + queue._clean_hicache_prefetch_resources = Mock() + queue._process_hicache_local_restores = Mock() + queue.scheduler = Mock() + queue.scheduler.enable_decode_hicache = True + queue.scheduler.enable_hisparse = False + queue.scheduler.metrics_reporter.enable_metrics = False + mock_poll.return_value = [KVPoll.Transferring] + + transferred = queue.pop_transferred() + + self.assertEqual(transferred, []) + self.assertEqual(queue.queue, [decode_req], "request must stay queued") + decode_req.kv_receiver.abort.assert_called_once() + decode_req.kv_receiver.kv_mgr.record_failure.assert_called_once() + mock_release.assert_not_called() + queue.req_to_metadata_buffer_idx_allocator.free.assert_not_called() + class TestChunkedPrefillAbort(unittest.TestCase): """Aborting a chunked prefill must not free pages an earlier chunk is reading.""" From b87d3160e28112317083d5d92f4e6977f8056966 Mon Sep 17 00:00:00 2001 From: Kastan Day Date: Mon, 10 Aug 2026 20:50:41 -0700 Subject: [PATCH 09/10] fix(disaggregation): quarantine rooms whose engine call failed before counting them as drained --- .../srt/disaggregation/mooncake/conn.py | 75 ++++++++++-- .../test_transfer_ownership_barrier.py | 109 ++++++++++++++++++ 2 files changed, 173 insertions(+), 11 deletions(-) diff --git a/python/sglang/srt/disaggregation/mooncake/conn.py b/python/sglang/srt/disaggregation/mooncake/conn.py index 0153932c3c4a..f6d2e36ba7d6 100644 --- a/python/sglang/srt/disaggregation/mooncake/conn.py +++ b/python/sglang/srt/disaggregation/mooncake/conn.py @@ -93,6 +93,16 @@ # How many requests may be stuck without proof of quiescence before the worker # fails, so that a restart releases every withheld page safely at once. MAX_UNQUIESCED_ROOMS = 256 +# An error return from a Mooncake engine call is NOT proof that its RDMA work +# stopped: the engine has no cancellation API, its sync wrapper's deadline +# return deliberately leaves the batch running (a known issue acknowledged in +# the engine's own source), and a failed batch is resubmitted internally so an +# earlier generation's writes can still land after the call returns. Posted +# work drains within the QP retransmit horizon times the engine's software +# re-posts, so a room whose engine call failed only counts as drained after +# this additional quarantine. Success returns need none: COMPLETED requires a +# terminal completion for every slice. +ENGINE_FAILURE_QUARANTINE_S = 30.0 # How often a decode rank re-sends an unacknowledged abort notification. ABORT_RETRY_INTERVAL_S = 0.5 # How often the bookkeeping thread re-checks drained rooms and sweeps lifetimes. @@ -124,17 +134,28 @@ class RoomTransferLifetime: every outstanding lease has been returned. Only then is it safe to release the room's KV pages. - The proof this provides is at the transfer-engine API boundary: a lease is - returned when the engine call returns, so the guarantee is exactly as - strong as the engine's contract that a returned call -- success or error -- - leaves no posted RDMA work that can still land. Verifying quiescence below - that boundary is the engine's job, not this barrier's. + The proof this provides is at the transfer-engine API boundary, and that + contract is asymmetric. A success return means every slice reached a + terminal completion, so no posted RDMA work can still land. An error + return proves nothing: the engine has no cancellation, and its deadline + and retry paths return with work still posted or queued for re-posting. + A lease returned by a *failed* engine call therefore only counts toward + quiescence after ``ENGINE_FAILURE_QUARANTINE_S``, which bounds the + engine's residual drain. Abort tokens minted by decode peers are recorded here so that a late or duplicated abort for a recycled room cannot close a live room. """ - __slots__ = ("_cond", "_leases", "_open", "_abort_tokens", "created_at", "_claimed") + __slots__ = ( + "_cond", + "_leases", + "_open", + "_abort_tokens", + "created_at", + "_claimed", + "_quarantine_until", + ) def __init__(self) -> None: self._cond = threading.Condition() @@ -146,6 +167,9 @@ def __init__(self) -> None: # unclaimed room was created by decode metadata alone, so nothing will # ever release it and the sweep must. self._claimed = False + # Monotonic deadline before which the room must not report quiesced, + # armed whenever an engine call for this room returns an error. + self._quarantine_until = 0.0 def try_lease(self) -> bool: """Take a lease, or return False if the room no longer admits work.""" @@ -172,9 +196,29 @@ def is_closed(self) -> bool: with self._cond: return not self._open + def quarantine(self, duration_s: float) -> None: + """Withhold quiescence for *duration_s* from now. + + Called when an engine call for this room returns an error: the engine + cannot cancel posted RDMA work, so the returned lease alone is not + proof of drain and the room must stay owned while the engine's + residual work runs out. + """ + with self._cond: + self._quarantine_until = max( + self._quarantine_until, time.monotonic() + duration_s + ) + + def _quiesced_locked(self) -> bool: + return ( + not self._open + and self._leases == 0 + and time.monotonic() >= self._quarantine_until + ) + def is_quiesced(self) -> bool: with self._cond: - return not self._open and self._leases == 0 + return self._quiesced_locked() def outstanding_leases(self) -> int: with self._cond: @@ -191,12 +235,13 @@ def is_claimed(self) -> bool: def is_reclaimable(self) -> bool: """Whether this room has no active local transfer ownership. - A quiesced room admits no work and has none running. An unclaimed room - has no sender *yet*, so callers must additionally preserve it for the - bootstrap grace period in which a metadata-late sender may still arrive. + A quiesced room admits no work and has none running (including any + engine-failure quarantine). An unclaimed room has no sender *yet*, so + callers must additionally preserve it for the bootstrap grace period + in which a metadata-late sender may still arrive. """ with self._cond: - return (not self._open and self._leases == 0) or not self._claimed + return self._quiesced_locked() or not self._claimed def add_abort_token(self, token: bytes) -> None: if not token: @@ -2307,6 +2352,9 @@ def transfer_worker( executor, ) if ret != 0: + # A failed engine call can leave RDMA work running; + # keep the room owned while the engine drains. + lifetime.quarantine(ENGINE_FAILURE_QUARANTINE_S) with self.session_lock: self.session_failures[req.mooncake_session_id] += 1 # Failures should never happen if the session is not dead, if the session fails once, mark it as failed @@ -2339,6 +2387,7 @@ def transfer_worker( target_rank_registration_info, ) if state_rc != 0: + lifetime.quarantine(ENGINE_FAILURE_QUARANTINE_S) with self.session_lock: self.session_failures[ req.mooncake_session_id @@ -2434,6 +2483,10 @@ def transfer_worker( self._staging_ctx.prefetched_rooms.discard(kv_chunk.room) except Exception as e: + # An exception may have interrupted an engine call mid-flight; + # its RDMA work can still be running, so quarantine the room. + if lifetime is not None: + lifetime.quarantine(ENGINE_FAILURE_QUARANTINE_S) # NOTE(shangming): Remove this when we make sure the transfer thread is bug-free raise RuntimeError( f"Transfer thread failed because of {e}. Prefill instance with bootstrap_port={self.bootstrap_port} is dead." diff --git a/test/registered/unit/disaggregation/test_transfer_ownership_barrier.py b/test/registered/unit/disaggregation/test_transfer_ownership_barrier.py index 57105521d383..0487bfcc7401 100644 --- a/test/registered/unit/disaggregation/test_transfer_ownership_barrier.py +++ b/test/registered/unit/disaggregation/test_transfer_ownership_barrier.py @@ -233,6 +233,90 @@ def blocking_write(*_args, **_kwargs): time.sleep(0.01) self.assertEqual(self._poll(sender), KVPoll.Failed) + def test_failed_engine_call_quarantines_the_room(self): + # An error return from the engine is NOT proof its RDMA work stopped: + # Mooncake has no cancellation, and its deadline/retry paths return + # with work still posted. The room must therefore stay owned for the + # quarantine window even though every lease has been returned. + mgr = make_prefill_manager() + mgr.request_status[ROOM] = KVPoll.WaitingForInput + mgr.transfer_infos = { + ROOM: { + "session:1": SimpleNamespace( + room=ROOM, + endpoint="127.0.0.1", + dst_port=1, + mooncake_session_id="session:1", + dst_kv_indices=np.array([0], dtype=np.int32), + dst_device_kv_indices=None, + is_dummy=False, + ) + } + } + mgr.decode_kv_args_table = { + "session:1": SimpleNamespace( + dst_attn_tp_size=1, + dst_kv_ptrs=[0xDEAD0000], + dst_kv_layer_ids=[0], + dst_state_layer_ids=[], + requires_dcp_relayout=False, + ) + } + mgr.session_lock, mgr.failed_sessions = threading.Lock(), set() + mgr.session_failures = defaultdict(int) + mgr.is_mla_backend, mgr.is_hybrid_mla_backend = True, False + mgr.attn_tp_size = mgr.attn_cp_size = mgr.pp_size = 1 + mgr.attn_tp_rank = mgr.attn_cp_rank = mgr.pp_rank = mgr.attn_dp_rank = 0 + mgr.transfer_queues = [FastQueue()] + mgr.sync_status_to_decode_endpoint = Mock() + + failed_send = threading.Event() + + def failing_write(*_args, **_kwargs): + failed_send.set() + return -1 + + mgr.send_kvcache = failing_write + sender = make_sender(mgr) + + with patch( + "sglang.srt.disaggregation.mooncake.conn.ENGINE_FAILURE_QUARANTINE_S", + 0.3, + ): + threading.Thread( + target=mgr.transfer_worker, + args=(mgr.transfer_queues[0], Mock()), + daemon=True, + ).start() + mgr.add_transfer_request( + ROOM, np.array([0], dtype=np.int32), slice(0, 1), False + ) + self.assertTrue(failed_send.wait(5), "worker never attempted the send") + sender.abort() + + # Leases drain almost immediately (the failed call returned), but + # the scheduler must still not observe a terminal state: the + # engine's work may still be landing in the decode's pages. + deadline = time.monotonic() + 1 + while time.monotonic() < deadline: + lifetime = mgr._room_lifetime(ROOM, create=False) + if lifetime is not None and lifetime.outstanding_leases() == 0: + break + time.sleep(0.01) + self.assertEqual( + self._poll(sender), + KVPoll.Transferring, + "a failed engine call must not count as drained before the " + "quarantine expires", + ) + + # Once the quarantine elapses, the failure may conclude. + for _ in range(500): + if self._poll(sender) == KVPoll.Failed: + break + time.sleep(0.01) + self.assertEqual(self._poll(sender), KVPoll.Failed) + def test_a_queued_chunk_cannot_attach_to_a_recycled_room(self): """The other half of the race, reported on the upstream discussion. @@ -299,6 +383,31 @@ def test_abort_token_authorization(self): # A tokenless abort cannot be authenticated; honour it, fail-safe. self.assertTrue(lifetime.authorizes_abort(b"")) + def test_quarantine_withholds_quiescence_after_a_failed_engine_call(self): + # An error-returning engine call is not proof of drain: the engine has + # no cancellation, so its work can still land after the lease returns. + lifetime = RoomTransferLifetime() + lifetime.claim() + self.assertTrue(lifetime.try_lease()) + lifetime.quarantine(0.2) + lifetime.end_lease() + lifetime.close() + + self.assertFalse( + lifetime.is_quiesced(), + "drained leases alone must not prove quiescence during quarantine", + ) + self.assertFalse( + lifetime.is_reclaimable(), + "the sweep must not retire a quarantined room either", + ) + + deadline = time.monotonic() + 2 + while time.monotonic() < deadline and not lifetime.is_quiesced(): + time.sleep(0.02) + self.assertTrue(lifetime.is_quiesced()) + self.assertTrue(lifetime.is_reclaimable()) + class TestExecutorDrain(unittest.TestCase): """Failing early must not hand pages back while siblings still run.""" From 25c9d6857df377101215e2953a637557e3b604df Mon Sep 17 00:00:00 2001 From: Kastan Day Date: Tue, 11 Aug 2026 10:37:44 -0700 Subject: [PATCH 10/10] fix(disaggregation): keep the manager socket readers alive through poisoned messages --- .../srt/disaggregation/mooncake/conn.py | 259 ++++++++++-------- .../test_transfer_ownership_barrier.py | 63 +++++ 2 files changed, 206 insertions(+), 116 deletions(-) diff --git a/python/sglang/srt/disaggregation/mooncake/conn.py b/python/sglang/srt/disaggregation/mooncake/conn.py index f6d2e36ba7d6..a347e8b21be5 100644 --- a/python/sglang/srt/disaggregation/mooncake/conn.py +++ b/python/sglang/srt/disaggregation/mooncake/conn.py @@ -2495,59 +2495,71 @@ def transfer_worker( if lifetime is not None: lifetime.end_lease() + def _handle_prefill_message(self, waiting_req_bytes) -> None: + """Dispatch one message from the prefill-side manager socket.""" + room = waiting_req_bytes[0].decode("ascii") + # Staging: decode reports consumption watermark back to prefill + if room == "WATERMARK": + from sglang.srt.disaggregation.common.staging_handler import ( + handle_watermark_msg, + ) + + handle_watermark_msg(self._staging_ctx, waiting_req_bytes) + return + # Staging: decode replies with allocated staging offset + if room == "STAGING_RSP": + from sglang.srt.disaggregation.common.staging_handler import ( + handle_staging_rsp, + ) + + handle_staging_rsp(waiting_req_bytes, self.transfer_infos) + return + # Decode-side abort notification: mark room as failed and ACK + if room == "ABORT": + self._handle_abort_notification(waiting_req_bytes) + return + mooncake_session_id = waiting_req_bytes[3].decode("ascii") + if room == "None": + decode_kv_args = KVArgsRegisterInfo.from_zmq(waiting_req_bytes) + decode_kv_args.requires_dcp_relayout = self.requires_dcp_relayout( + decode_kv_args.dst_dcp_size, + decode_kv_args.dst_dcp_rank, + ) + if decode_kv_args.requires_dcp_relayout: + decode_kv_args.dcp_token_item_lens = self.prepare_dcp_token_item_lens( + [decode_kv_args.dst_kv_item_len] * len(self.kv_args.kv_item_lens) + ) + self.decode_kv_args_table[mooncake_session_id] = decode_kv_args + with self.session_lock: + if mooncake_session_id in self.failed_sessions: + self.failed_sessions.remove(mooncake_session_id) + if mooncake_session_id in self.session_failures: + del self.session_failures[mooncake_session_id] + logger.debug(f"Register KVArgs from {mooncake_session_id} successfully") + return + else: + self._handle_bootstrap_metadata(waiting_req_bytes) + def start_prefill_thread(self): def bootstrap_thread(): """This thread recvs pre-alloc notification from the decode engine""" # KVPoll.Bootstrapping -> KVPoll.WaitingForInput while True: waiting_req_bytes = self.server_socket.recv_multipart() - room = waiting_req_bytes[0].decode("ascii") - # Staging: decode reports consumption watermark back to prefill - if room == "WATERMARK": - from sglang.srt.disaggregation.common.staging_handler import ( - handle_watermark_msg, - ) - - handle_watermark_msg(self._staging_ctx, waiting_req_bytes) - continue - # Staging: decode replies with allocated staging offset - if room == "STAGING_RSP": - from sglang.srt.disaggregation.common.staging_handler import ( - handle_staging_rsp, - ) - - handle_staging_rsp(waiting_req_bytes, self.transfer_infos) - continue - # Decode-side abort notification: mark room as failed and ACK - if room == "ABORT": - self._handle_abort_notification(waiting_req_bytes) - continue - mooncake_session_id = waiting_req_bytes[3].decode("ascii") - if room == "None": - decode_kv_args = KVArgsRegisterInfo.from_zmq(waiting_req_bytes) - decode_kv_args.requires_dcp_relayout = self.requires_dcp_relayout( - decode_kv_args.dst_dcp_size, - decode_kv_args.dst_dcp_rank, - ) - if decode_kv_args.requires_dcp_relayout: - decode_kv_args.dcp_token_item_lens = ( - self.prepare_dcp_token_item_lens( - [decode_kv_args.dst_kv_item_len] - * len(self.kv_args.kv_item_lens) - ) - ) - self.decode_kv_args_table[mooncake_session_id] = decode_kv_args - with self.session_lock: - if mooncake_session_id in self.failed_sessions: - self.failed_sessions.remove(mooncake_session_id) - if mooncake_session_id in self.session_failures: - del self.session_failures[mooncake_session_id] - logger.debug( - f"Register KVArgs from {mooncake_session_id} successfully" + # Sole reader of this socket, and the only thread that can + # receive a decode ABORT: if it dies, this rank never closes + # aborted rooms and never ACKs, so every decode peer withholds + # its pages until MAX_UNQUIESCED_ROOMS tears it down. The loop + # must outlive any bug in a handler; a poisoned message is + # dropped loudly, never fatal. recv stays outside the guard so + # a closed socket still ends the thread at shutdown. + try: + self._handle_prefill_message(waiting_req_bytes) + except Exception: + logger.exception( + "Mooncake prefill-side message handling failed; " + "dropping message and continuing" ) - continue - else: - self._handle_bootstrap_metadata(waiting_req_bytes) threading.Thread(target=bootstrap_thread).start() @@ -2623,83 +2635,98 @@ def _handle_bootstrap_metadata(self, msg: List[bytes]) -> bool: self.update_status(room, KVPoll.WaitingForInput) return True - def start_decode_thread(self): - def decode_thread(): - while True: - msg = self.server_socket.recv_multipart() - if msg[0] == MooncakeKVManager.AUX_DATA_HEADER: - self._handle_aux_data(msg) - continue + def _handle_decode_message(self, msg) -> None: + """Dispatch one message from the decode-side manager socket.""" + if msg[0] == MooncakeKVManager.AUX_DATA_HEADER: + self._handle_aux_data(msg) + return - # Staging: prefill notifies a chunk written to staging buffer - if msg[0] == b"CHUNK_READY": - room = int(msg[1].decode("ascii")) - if self._is_tearing_down(room): - continue - chunk_idx = int(msg[2].decode("ascii")) - page_start = int(msg[3].decode("ascii")) - num_pages = int(msg[4].decode("ascii")) - # Prefer the prefill's unique rank id when present so that - # writers are counted per rank rather than per session. - writer_id = ( - msg[6].decode("ascii") - if len(msg) > 6 - else msg[5].decode("ascii") - ) - handler = self._staging_handler - assert ( - handler is not None - ), "CHUNK_READY received before staging handler initialized" - handler.handle_chunk_arrived( - room, - chunk_idx, - page_start, - num_pages, - writer_id, - ) - continue + # Staging: prefill notifies a chunk written to staging buffer + if msg[0] == b"CHUNK_READY": + room = int(msg[1].decode("ascii")) + if self._is_tearing_down(room): + return + chunk_idx = int(msg[2].decode("ascii")) + page_start = int(msg[3].decode("ascii")) + num_pages = int(msg[4].decode("ascii")) + # Prefer the prefill's unique rank id when present so that + # writers are counted per rank rather than per session. + writer_id = ( + msg[6].decode("ascii") if len(msg) > 6 else msg[5].decode("ascii") + ) + handler = self._staging_handler + assert ( + handler is not None + ), "CHUNK_READY received before staging handler initialized" + handler.handle_chunk_arrived( + room, + chunk_idx, + page_start, + num_pages, + writer_id, + ) + return - # Staging: prefill pre-requests staging allocation before forward - if msg[0] == b"STAGING_REQ": - if self._is_tearing_down(int(msg[1].decode("ascii"))): - continue - self._handle_staging_req(msg) - continue + # Staging: prefill pre-requests staging allocation before forward + if msg[0] == b"STAGING_REQ": + if self._is_tearing_down(int(msg[1].decode("ascii"))): + return + self._handle_staging_req(msg) + return - # Prefill acknowledges abort notification - if msg[0] == b"ABORT_ACK": - self._handle_abort_ack(msg) - continue + # Prefill acknowledges abort notification + if msg[0] == b"ABORT_ACK": + self._handle_abort_ack(msg) + return - bootstrap_room, status, prefill_rank = msg - status = int(status.decode("ascii")) - bootstrap_room = int(bootstrap_room.decode("ascii")) - prefill_rank = int(prefill_rank.decode("ascii")) + bootstrap_room, status, prefill_rank = msg + status = int(status.decode("ascii")) + bootstrap_room = int(bootstrap_room.decode("ascii")) + prefill_rank = int(prefill_rank.decode("ascii")) - if self._is_tearing_down(bootstrap_room): - continue + if self._is_tearing_down(bootstrap_room): + return - if status == KVPoll.Success: - if bootstrap_room in self.request_status: - self.prefill_response_tracker[bootstrap_room].add(prefill_rank) - expected_response_num = ( - self.required_prefill_response_num_table[bootstrap_room] - ) - arrived_response_num = len( - self.prefill_response_tracker[bootstrap_room] - ) - if arrived_response_num == expected_response_num: - if self.enable_staging: - handler = self._staging_handler - if handler.is_staging_room(bootstrap_room): - handler.submit_last_scatter_async(bootstrap_room) - self.update_status(bootstrap_room, KVPoll.Success) - elif status == KVPoll.Failed: - self.record_failure( - bootstrap_room, - "Failed to get kvcache from prefill instance, it might be dead", + if status == KVPoll.Success: + if bootstrap_room in self.request_status: + self.prefill_response_tracker[bootstrap_room].add(prefill_rank) + expected_response_num = self.required_prefill_response_num_table[ + bootstrap_room + ] + arrived_response_num = len( + self.prefill_response_tracker[bootstrap_room] + ) + if arrived_response_num == expected_response_num: + if self.enable_staging: + handler = self._staging_handler + if handler.is_staging_room(bootstrap_room): + handler.submit_last_scatter_async(bootstrap_room) + self.update_status(bootstrap_room, KVPoll.Success) + elif status == KVPoll.Failed: + self.record_failure( + bootstrap_room, + "Failed to get kvcache from prefill instance, it might be dead", + ) + self.update_status(bootstrap_room, status) + + def start_decode_thread(self): + def decode_thread(): + while True: + msg = self.server_socket.recv_multipart() + # Sole reader of this socket, and the only thread that can + # record an ABORT_ACK: if it dies, no abort is ever proven + # quiesced and every aborted room withholds its pages until + # MAX_UNQUIESCED_ROOMS tears the engine down. The loop must + # outlive any bug in a handler; a poisoned message is dropped + # loudly, never fatal. recv stays outside the guard so a + # closed socket still ends the thread at shutdown. + try: + self._handle_decode_message(msg) + except Exception: + logger.exception( + "Mooncake decode-side message handling failed; " + "dropping message and continuing" ) - self.update_status(bootstrap_room, status) threading.Thread(target=decode_thread).start() self._start_heartbeat_checker_thread() diff --git a/test/registered/unit/disaggregation/test_transfer_ownership_barrier.py b/test/registered/unit/disaggregation/test_transfer_ownership_barrier.py index 0487bfcc7401..f7b9b014b728 100644 --- a/test/registered/unit/disaggregation/test_transfer_ownership_barrier.py +++ b/test/registered/unit/disaggregation/test_transfer_ownership_barrier.py @@ -777,6 +777,69 @@ def is_quiesced(self): "a faulty entry must not starve the ones behind it", ) + def test_decode_reader_survives_a_poisoned_message(self): + # The decode reader is the only thread that can record an ABORT_ACK; + # a message that killed it would leave every subsequently aborted room + # withholding its pages until escalation. A poisoned message must be + # dropped, never fatal. + mgr = make_decode_manager() + done = threading.Event() + messages = iter( + [ + [b"7", b"4", b"0", b"unexpected-extra-frame"], + [b"7", b"0", b"0"], # well-formed Failed status + ] + ) + + def recv_multipart(): + try: + return next(messages) + except StopIteration: + done.set() + raise zmq.ZMQError() # ends the reader, as a closed socket would + + mgr.server_socket = SimpleNamespace(recv_multipart=recv_multipart) + mgr._start_heartbeat_checker_thread = Mock() + mgr.start_decode_thread() + + self.assertTrue(done.wait(5), "the reader died on the poisoned message") + mgr.record_failure.assert_called_once() + mgr.update_status.assert_called_once_with(7, KVPoll.Failed) + + def test_prefill_reader_survives_a_poisoned_message(self): + # The bootstrap reader is the only thread that can receive a decode + # ABORT; a message that killed it would leave every peer waiting out + # its whole quiescence deadline. A poisoned message must be dropped. + mgr = make_prefill_manager() + mgr._start_transfer_bookkeeping() + mgr._send_manager_message = lambda *args, **kwargs: None + mgr.open_room_transfers(ROOM) + mgr.request_status[ROOM] = KVPoll.WaitingForInput + done = threading.Event() + messages = iter( + [ + [b"ABORT"], # poisoned: missing every frame after the header + [b"ABORT", str(ROOM).encode("ascii"), b"127.0.0.1", b"1", b""], + ] + ) + + def recv_multipart(): + try: + return next(messages) + except StopIteration: + done.set() + raise zmq.ZMQError() + + mgr.server_socket = SimpleNamespace(recv_multipart=recv_multipart) + mgr.start_prefill_thread() + + self.assertTrue(done.wait(5), "the reader died on the poisoned message") + lifetime = mgr._room_lifetime(ROOM, create=False) + self.assertIsNotNone(lifetime) + self.assertTrue( + lifetime.is_closed(), "the well-formed abort behind it must still land" + ) + def test_a_backed_up_peer_does_not_delay_other_peers(self): mgr = make_prefill_manager() mgr._start_transfer_bookkeeping()