Skip to content
Closed
Show file tree
Hide file tree
Changes from 97 commits
Commits
Show all changes
104 commits
Select commit Hold shift + click to select a range
8b7f4bf
Add DSpark PD hidden transfer support
zhangxiaolei123456 Jul 15, 2026
6a762cf
Align DSpark hidden prefill with decode prefix
zhangxiaolei123456 Jul 15, 2026
a79986e
Harden DSpark PD prefill bootstrap failures
zhangxiaolei123456 Jul 15, 2026
264557a
Fix DSpark PD hidden debug env
zhangxiaolei123456 Jul 15, 2026
45b151b
Guard missing future indices in overlap spec extras
zhangxiaolei123456 Jul 15, 2026
7a953fe
Make DSpark PP hidden slices explicit
zhangxiaolei123456 Jul 15, 2026
e0ae4d3
Seed DSpark disagg draft input in overlap
zhangxiaolei123456 Jul 15, 2026
d9b5dbf
Pad DSpark prefill tail on draft input merge
zhangxiaolei123456 Jul 15, 2026
c526e8e
Propagate DSpark hidden capture layers to forward batch
zhangxiaolei123456 Jul 15, 2026
d3886fe
Carry DSpark hidden outputs through PP prefill
zhangxiaolei123456 Jul 15, 2026
9ffef48
Enable DSpark hidden capture in PP disagg prefill
zhangxiaolei123456 Jul 15, 2026
ef5ab3a
Route DSpark hidden metadata by prefill PP rank
zhangxiaolei123456 Jul 15, 2026
547ec64
Defer DSpark hidden transfer until rows are ready
zhangxiaolei123456 Jul 15, 2026
74aab38
Wait for DSpark metadata before prefill admission
zhangxiaolei123456 Jul 15, 2026
f035f9d
Require matching radix cache policy for DSpark PD
zhangxiaolei123456 Jul 15, 2026
72999df
Preserve PP consensus for DSpark prefill admission
zhangxiaolei123456 Jul 15, 2026
4a294b7
Keep DSpark PP prefill forward mode consistent
zhangxiaolei123456 Jul 15, 2026
9566f30
Limit DSpark PP capture to local target layers
zhangxiaolei123456 Jul 15, 2026
88c8db4
Gate PP prefill admission on local bootstrap finalize
zhangxiaolei123456 Jul 15, 2026
129b6df
Fail fast on invalid DSpark hidden transfer
zhangxiaolei123456 Jul 15, 2026
126a135
Warn and skip invalid DSpark hidden
zhangxiaolei123456 Jul 15, 2026
851fa3b
Relax DSpark hidden prealloc throttling
zhangxiaolei123456 Jul 15, 2026
c37a784
Use packetized DSpark hidden receive buffers
zhangxiaolei123456 Jul 15, 2026
0d039f1
Packetize DSpark hidden transfer state
zhangxiaolei123456 Jul 15, 2026
f01f281
Disable DSpark hidden request-count throttling
zhangxiaolei123456 Jul 15, 2026
f9ad078
Prune DSpark hidden transfer knobs
zhangxiaolei123456 Jul 15, 2026
25a9331
Tune DSpark hidden transfer defaults
zhangxiaolei123456 Jul 15, 2026
fbcb238
Move DSpark hidden transfer buffers to GPU
zhangxiaolei123456 Jul 15, 2026
fbbe252
Default DSpark hidden transfer to one-shot
zhangxiaolei123456 Jul 15, 2026
3d1170b
Add DSpark hidden transfer status logs
zhangxiaolei123456 Jul 15, 2026
a3a1344
Log DSpark hidden transfer timing
zhangxiaolei123456 Jul 15, 2026
3d0c607
Log DSpark PP bootstrap admission
zhangxiaolei123456 Jul 15, 2026
c6c5e33
Defer DSpark hidden pool allocation
zhangxiaolei123456 Jul 15, 2026
0fdf3a9
Remove verbose DSpark PP bootstrap log
zhangxiaolei123456 Jul 15, 2026
76f4875
Reduce DSpark prefill admission logs
zhangxiaolei123456 Jul 15, 2026
fea4afa
Revert deferred DSpark hidden allocation
zhangxiaolei123456 Jul 15, 2026
d289b64
Decouple DSpark hidden credits from PP consensus
zhangxiaolei123456 Jul 16, 2026
7c1a102
Fix DSpark PD scheduling and transfer backpressure
zhangxiaolei123456 Jul 16, 2026
ac59825
Remove DSpark PD info logs
zhangxiaolei123456 Jul 16, 2026
d2a91db
Add PD warmup stall instrumentation
zhangxiaolei123456 Jul 16, 2026
ed4defc
Trace PP bootstrap consensus progress
zhangxiaolei123456 Jul 16, 2026
dda43cc
Preserve single-stage PP bootstrap consensus
zhangxiaolei123456 Jul 16, 2026
0a4ca17
Remove PD warmup debug instrumentation
zhangxiaolei123456 Jul 16, 2026
0e2dc8e
Synchronize DSpark hidden source event with PP launch
zhangxiaolei123456 Jul 16, 2026
6a4dc82
Simplify DSpark PP bootstrap consensus
zhangxiaolei123456 Jul 16, 2026
5d87f84
Ensure DSpark hidden capture reaches PD transfer
zhangxiaolei123456 Jul 16, 2026
990923c
Restore DSpark hidden transfer fail-fast
zhangxiaolei123456 Jul 16, 2026
5fede8c
Fix DSpark hidden capture selection in mixed batches
zhangxiaolei123456 Jul 16, 2026
c7b8a6f
Disable optimistic prefill for DSpark hidden transfer
zhangxiaolei123456 Jul 16, 2026
884659d
Gate DSpark hidden PP admission on local finalize
zhangxiaolei123456 Jul 16, 2026
c5ca696
Probe DSpark hidden credits before PP admission
zhangxiaolei123456 Jul 16, 2026
396f6da
Snapshot PP batch seq lens for DSpark hidden writes
zhangxiaolei123456 Jul 16, 2026
5928c5d
Optimize DSpark hidden row writes and send indices
zhangxiaolei123456 Jul 16, 2026
4abe1e4
Add DSpark hidden PD timing diagnostics
zhangxiaolei123456 Jul 16, 2026
7141238
Release DSpark hidden rows after hidden transfer
zhangxiaolei123456 Jul 16, 2026
1401efc
Prioritize DSpark hidden transfer before final KV
zhangxiaolei123456 Jul 16, 2026
5917075
Trace DSpark hidden worker queue and event wait
zhangxiaolei123456 Jul 16, 2026
617a06a
Release DSpark hidden rows in transfer worker
zhangxiaolei123456 Jul 16, 2026
76c1f32
Trace Mooncake final transfer phases
zhangxiaolei123456 Jul 16, 2026
79f6c78
Trace DSpark release consensus latency
zhangxiaolei123456 Jul 16, 2026
0b6542a
Trace decode release arrival latency
zhangxiaolei123456 Jul 16, 2026
9cda8b1
Trace decode release callbacks
zhangxiaolei123456 Jul 16, 2026
a40fe1f
Trace DSpark prefill queue stages
zhangxiaolei123456 Jul 16, 2026
fc7415b
Trace prefill PP release callbacks
zhangxiaolei123456 Jul 16, 2026
99ff8f1
Throttle DSpark PD event timing logs
zhangxiaolei123456 Jul 16, 2026
9fc0f76
Trace DSpark decode commit latency
zhangxiaolei123456 Jul 16, 2026
b7d03f1
Remove DSpark PD timing diagnostics
zhangxiaolei123456 Jul 16, 2026
bfed593
Optimize DSpark decode hidden receive path
zhangxiaolei123456 Jul 16, 2026
a04a7f3
Fix decode prefill info retry import
zhangxiaolei123456 Jul 16, 2026
6e99f0f
Address DSpark PD review feedback
zhangxiaolei123456 Jul 16, 2026
26d04db
Merge remote-tracking branch 'upstream/main' into deepseev_v4_dpsark_…
zhangxiaolei123456 Jul 16, 2026
a576b8d
Fix DSpark draft global token metadata
zhangxiaolei123456 Jul 16, 2026
4d8d542
Slice DSpark hidden rows by prefill PP rank
zhangxiaolei123456 Jul 16, 2026
6cae666
Fix DSpark prefill PP rank lookup
zhangxiaolei123456 Jul 16, 2026
2090292
Harden DSpark hidden row release
zhangxiaolei123456 Jul 16, 2026
7e44bda
Simplify DSpark prefill PP release consensus
zhangxiaolei123456 Jul 16, 2026
5ad2ab1
Avoid DSpark hidden credit head-of-line blocking
zhangxiaolei123456 Jul 16, 2026
cf96ffb
Release DSpark hidden rows on transfer abort
zhangxiaolei123456 Jul 16, 2026
ccde191
Synchronize DSpark hidden source event before abort release
zhangxiaolei123456 Jul 16, 2026
e0855a5
Use one-shot DSpark hidden transfer
zhangxiaolei123456 Jul 16, 2026
cef9555
Reuse larger DSpark hidden receive pages
zhangxiaolei123456 Jul 16, 2026
b849ddc
Use fixed DSpark hidden receive row pool
zhangxiaolei123456 Jul 17, 2026
4ba8e46
Remove DSpark hidden receive prewarm call
zhangxiaolei123456 Jul 17, 2026
2bfc5e4
Match PP proxy tensors by microbatch metadata
zhangxiaolei123456 Jul 17, 2026
26fef6c
Fail fast on PP proxy metadata mismatch
zhangxiaolei123456 Jul 17, 2026
386dc32
Release DSpark prefill tail hidden after draft input
zhangxiaolei123456 Jul 17, 2026
ef647ff
Avoid extra DSpark hidden tail copies
zhangxiaolei123456 Jul 17, 2026
b68ef35
Fail fast on disagg prefill PP proxy divergence
zhangxiaolei123456 Jul 17, 2026
52a7461
Send empty PP proxy metadata for idle batches
zhangxiaolei123456 Jul 17, 2026
7cb01d1
Skip PP proxy divergence check on first rank
zhangxiaolei123456 Jul 17, 2026
1e9252f
Remove PP proxy metadata pyobj handshake
zhangxiaolei123456 Jul 17, 2026
f85f348
Revert PP proxy metadata diagnostics
zhangxiaolei123456 Jul 17, 2026
c59b01a
Keep PP bootstrap admission deterministic
zhangxiaolei123456 Jul 17, 2026
8608569
Allow PP bootstrap consensus filtering
zhangxiaolei123456 Jul 17, 2026
12370d1
Minimize PP scheduler DSpark changes
zhangxiaolei123456 Jul 17, 2026
0f4273c
DSpark streaming hidden inject (#640)
zhangxiaolei123456 Jul 17, 2026
f96b07b
Fix DSpark PD streaming hidden ownership (#641)
zhangxiaolei123456 Jul 18, 2026
28b0747
Refactor DSpark disagg (#645)
zhangxiaolei123456 Jul 20, 2026
17562a2
mooncake pd hidden cleanup (#649)
zhangxiaolei123456 Jul 21, 2026
9d61d03
Align DSpark PD hidden changes with project rules (#651)
zhangxiaolei123456 Jul 21, 2026
32990a0
schedule dspark slim (#652)
zhangxiaolei123456 Jul 21, 2026
6c3d430
Reduce DFlash spec input conflict with main
zhangxiaolei123456 Jul 21, 2026
6c67f78
Merge branch 'main' into deepseev_v4_dpsark_pd_dev
zhangxiaolei123456 Jul 21, 2026
c96e68b
Keep scheduler aligned with main for disaggregation init (#653)
zhangxiaolei123456 Jul 21, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions python/sglang/srt/disaggregation/base/conn.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@ class StateType(str, enum.Enum):
SWA_RING = "swa_ring"
# DeepSeek-V4 online C128 request-scoped state.
C128_STATE = "c128_state"
# DSpark target aux hidden rows used to bootstrap decode-side draft KV.
DSPARK_HIDDEN = "dspark_hidden"


@dataclasses.dataclass
Expand Down Expand Up @@ -201,6 +203,7 @@ def send_metadata(
aux_index: Optional[int] = None,
state_indices: Optional[List] = None,
decode_prefix_len: Optional[int] = None,
spec_metadata: Optional[dict] = None,
):
"""
Notify the prefill server about the kv indices, aux index, and state_indices.
Expand Down
40 changes: 40 additions & 0 deletions python/sglang/srt/disaggregation/common/conn.py
Original file line number Diff line number Diff line change
Expand Up @@ -200,6 +200,7 @@ def __init__(
self.register_to_bootstrap()
self.transfer_infos = {}
self.req_to_decode_prefix_len: Dict[int, int] = {}
self.req_to_dspark_hidden_meta: Dict[int, dict] = {}
self.decode_kv_args_table = {}
self.pp_group = get_pp_group()
# If a timeout happens on the prefill side, it means prefill instances
Expand Down Expand Up @@ -244,6 +245,39 @@ def __init__(
f"Unsupported DisaggregationMode: {self.disaggregation_mode}"
)

def supports_dspark_hidden_streaming(self) -> bool:
return False

def mark_dspark_hidden_request_done(
self,
bootstrap_room: int,
state_indices: Optional[List] = None,
) -> None:
"""Mark the hidden-transfer portion of a request done.

Backends that support streaming hidden transfer override this to release
their source window independently from KV request completion.
"""
del bootstrap_room, state_indices
return None

def pop_dspark_hidden_request_done(self, bootstrap_room: int) -> bool:
"""Consume a hidden-request-done event for early source-window release."""
del bootstrap_room
return False

# Backward-compatible aliases for backend-specific implementations that have
# not yet migrated to the request-level naming.
def mark_dspark_hidden_done(
self,
bootstrap_room: int,
state_indices: Optional[List] = None,
) -> None:
self.mark_dspark_hidden_request_done(bootstrap_room, state_indices)

def pop_dspark_hidden_done(self, bootstrap_room: int) -> bool:
return self.pop_dspark_hidden_request_done(bootstrap_room)

def check_status(self, bootstrap_room: int) -> KVPoll:
return self.request_status[bootstrap_room]

Expand Down Expand Up @@ -1148,6 +1182,8 @@ def clear(self) -> None:
self.kv_mgr.request_status.pop(self.bootstrap_room, None)
if hasattr(self.kv_mgr, "req_to_decode_prefix_len"):
self.kv_mgr.req_to_decode_prefix_len.pop(self.bootstrap_room, None)
if hasattr(self.kv_mgr, "req_to_dspark_hidden_meta"):
self.kv_mgr.req_to_dspark_hidden_meta.pop(self.bootstrap_room, None)
if hasattr(self.kv_mgr, "transfer_infos"):
self.kv_mgr.transfer_infos.pop(self.bootstrap_room, None)

Expand Down Expand Up @@ -1284,6 +1320,9 @@ def _get_bootstrap_info_from_server(
response = requests.get(url, timeout=5)
if response.status_code == 200:
bootstrap_info = response.json()
bootstrap_info["target_cp_rank"] = int(prefill_cp_rank)
bootstrap_info["target_tp_rank"] = int(target_tp_rank)
bootstrap_info["target_pp_rank"] = int(target_pp_rank)
return bootstrap_info
else:
logger.error(
Expand Down Expand Up @@ -1361,6 +1400,7 @@ def send_metadata(
aux_index: Optional[int] = None,
state_indices: Optional[List[int]] = None,
decode_prefix_len: Optional[int] = None,
spec_metadata: Optional[dict] = None,
):
raise NotImplementedError

Expand Down
118 changes: 117 additions & 1 deletion python/sglang/srt/disaggregation/common/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
import struct
import threading
from collections import deque
from typing import List, Optional, Tuple, Union
from typing import Any, List, Optional, Tuple, Union

import numpy as np
import numpy.typing as npt
Expand All @@ -25,11 +25,127 @@ class TransferKVChunk:
prefill_aux_index: Optional[int]
state_indices: Optional[List]
chunk_id: Optional[int] = None
kv_sent: bool = False
dspark_hidden_packet_idx: int = 0
dspark_hidden_sent: bool = False
dspark_hidden_ready_sent: bool = False
dspark_hidden_ack_ready: bool = False
dspark_hidden_ack_expected_count: int = 0
dspark_hidden_ack_timed_out: bool = False
dspark_hidden_start: Optional[int] = None
dspark_hidden_row_len: int = 0
dspark_hidden_is_last_chunk: bool = False
dspark_hidden_release_indices: Optional[List[int]] = None
enqueue_time: float = 0.0
source_event: Optional[Any] = None
trace_ctx: Union[TraceReqContext, TraceNullContext] = dataclasses.field(
default_factory=TraceNullContext
)


@dataclasses.dataclass
class DSparkHiddenChunk:
"""Transport-neutral DSpark hidden chunk descriptor."""

room: int
prefill_rank: int
hidden_start: int
row_len: int
is_last_hidden_chunk: bool
dst_indices: List[int]
ack_host: Optional[str] = None
ack_port: Optional[int] = None


@dataclasses.dataclass
class DSparkHiddenRequestState:
"""Decode-side request state for hidden transfer, separate from KV status."""

enabled: bool = False
streaming: bool = False
start: int = 0
next_start: int = 0
end: int = 0
hidden_done: bool = True
kv_done: bool = False

@classmethod
def disabled(cls) -> "DSparkHiddenRequestState":
return cls()

@classmethod
def full(cls, start: int, end: int) -> "DSparkHiddenRequestState":
return cls(
enabled=True,
streaming=False,
start=int(start),
next_start=int(start),
end=int(end),
hidden_done=True,
)

@classmethod
def streaming_state(cls, start: int, end: int) -> "DSparkHiddenRequestState":
return cls(
enabled=True,
streaming=True,
start=int(start),
next_start=int(start),
end=int(end),
hidden_done=False,
)

def reset(self) -> None:
self.enabled = False
self.streaming = False
self.start = 0
self.next_start = 0
self.end = 0
self.hidden_done = True
self.kv_done = False

def mark_kv_done(self) -> None:
self.kv_done = True

def mark_hidden_done(self) -> None:
self.hidden_done = True

def hidden_request_done(self) -> bool:
return self.hidden_done

def kv_request_done(self) -> bool:
return self.kv_done

def request_done(self) -> bool:
return self.kv_request_done() and self.hidden_request_done()

def accept_chunk(
self, chunk: DSparkHiddenChunk, *, defer_hidden_done: bool = False
) -> str:
"""Return accepted/future/stale for a streaming hidden chunk."""
hidden_start = int(chunk.hidden_start)
if hidden_start > self.next_start:
return "future"
if hidden_start < self.next_start:
return "stale"
next_start = hidden_start + int(chunk.row_len)
if next_start > self.end:
raise RuntimeError(
"DSpark streaming hidden chunk exceeds request range: "
f"next_start={next_start}, expected_end={self.end}"
)
if chunk.is_last_hidden_chunk:
if next_start != self.end:
raise RuntimeError(
"DSpark streaming hidden ended at an unexpected offset: "
f"next_start={next_start}, expected_end={self.end}"
)
if not defer_hidden_done:
self.mark_hidden_done()
self.next_start = next_start
return "accepted"


def pack_list_of_buffers(buffers: List[bytes]) -> bytes:
if not buffers:
return b""
Expand Down
Loading
Loading