Skip to content
Open
Show file tree
Hide file tree
Changes from 9 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 15 additions & 1 deletion python/sglang/srt/beam_search/coordinator.py
Original file line number Diff line number Diff line change
Expand Up @@ -250,7 +250,11 @@ def _collect_stop_token_ids(req: Req, user_params) -> List[int]:
return sorted(stop_ids)

def maybe_select_and_relay(
self, batch: ScheduleBatch, batch_result, chunked_req: Optional[Req] = None
self,
batch: ScheduleBatch,
batch_result,
chunked_req: Optional[Req] = None,
skip_rows: Optional[set[int]] = None,
) -> None:
"""Per-forward relay hook: overwrite beam rows' relayed tokens with
joint-selected ones. O(1) when no beam group is live."""
Expand All @@ -275,6 +279,7 @@ def maybe_select_and_relay(
group = req.beam_group
if (
group is None
or (skip_rows is not None and i in skip_rows)
or group.state != BeamGroupState.DECODING
or group.num_generated > 0
or req is chunked_req # mid-chunk leader: no selection yet
Expand Down Expand Up @@ -507,6 +512,15 @@ def retire_group(self, req: Req) -> None:
group.final_results = []
self._retire_group(group)

def abort_group_preserving_finish_reason(self, req: Req) -> None:
group = req.beam_group
if group is None or group.retired:
return
finished_reason = req.finished_reason
self._abort_group(group)
if finished_reason is not None:
req.finished_reason = finished_reason

def _free_member_rows(self, group: BeamGroup) -> None:
# Staged orphans are unreachable from every row, so the group-wide
# dedup free below would miss them.
Expand Down
69 changes: 54 additions & 15 deletions python/sglang/srt/disaggregation/prefill.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@
MetadataBuffers,
ReqToMetadataIdxAllocator,
TransferBackend,
abort_kv_transfer,
build_kv_layer_ids,
build_staging_slot_metadata,
get_dsv4_c128_state_indices,
Expand All @@ -63,6 +64,10 @@
Req,
ScheduleBatch,
)
from sglang.srt.managers.utils import (
complete_mm_embedding_validations,
decode_mm_embedding_errors,
)
from sglang.srt.mem_cache.common import (
kv_to_page_indices,
kv_to_page_num,
Expand All @@ -78,11 +83,7 @@
SchedulerStageMetricsRecorder,
scheduler_stage_method,
)
from sglang.srt.runtime_context import (
get_disagg,
get_parallel,
get_schedule,
)
from sglang.srt.runtime_context import get_disagg, get_parallel, get_schedule
from sglang.srt.utils import is_npu

if TYPE_CHECKING:
Expand Down Expand Up @@ -717,6 +718,12 @@ def process_batch_result_disagg_prefill(

if copy_done is not None:
copy_done.synchronize()
mm_embedding_error_tensor = result.mm_embedding_errors
if mm_embedding_error_tensor is not None:
complete_mm_embedding_validations(batch.reqs, mm_embedding_error_tensor)
decoded_mm_embedding_errors = decode_mm_embedding_errors(
mm_embedding_error_tensor
)
auxiliary_output_starts = (
self.batch_result_processor.snapshot_auxiliary_output_starts(batch, result)
)
Expand All @@ -730,7 +737,7 @@ def process_batch_result_disagg_prefill(

logprob_pt = 0
aborted_reqs: List[Req] = []
assert batch.spec_info is result.next_draft_input
assert decoded_mm_embedding_errors or batch.spec_info is result.next_draft_input
draft_input = result.next_draft_input
draft_hidden_states_cpu = None
draft_dsa_topk_indices_cpu = None
Expand Down Expand Up @@ -758,9 +765,29 @@ def advance_logprob_pt(i: int, req: Req) -> None:
if extend_logprob_start_len < extend_input_len:
logprob_pt += extend_input_len - extend_logprob_start_len

mm_embedding_errors = {
req_idx: (expected, actual)
for req_idx, expected, actual in decoded_mm_embedding_errors
}
for i, (req, next_token_id) in enumerate(
zip(batch.reqs, next_token_ids, strict=True)
):
mm_embedding_error = mm_embedding_errors.get(i)
if mm_embedding_error is not None:
self.defer_mm_embedding_abort(req, *mm_embedding_error)
if mm_embedding_error is not None or req.mm_embedding_abort_pending:
advance_logprob_pt(i, req)
if req.inflight_middle_chunks > 0:
req.inflight_middle_chunks -= 1
req.time_stats.set_last_chunked_prefill_finish_time()
if (
req.inflight_middle_chunks <= 0
and req.mm_embedding_validation_count == 0
):
self.finish_disagg_mm_embedding_abort(req)
aborted_reqs.append(req)
continue

if req.inflight_middle_chunks <= 0:
req.time_stats.set_prefill_finished_time()

Expand Down Expand Up @@ -907,6 +934,23 @@ def advance_logprob_pt(i: int, req: Req) -> None:
dp_cooperation_info=batch.dp_cooperation_info,
)

def finish_disagg_mm_embedding_abort(self: Scheduler, req: Req) -> None:
if not req.mm_embedding_abort_pending:
return
if req.disagg_kv_sender is not None:
try:
req.disagg_kv_sender.clear()
except Exception:
logger.exception("Failed to clear KV sender for request %s", req.rid)
maybe_release_metadata_buffer(req, self.req_to_metadata_buffer_idx_allocator)
req.pending_bootstrap = False
self.disagg_prefill_inflight_queue = [
inflight_req
for inflight_req in self.disagg_prefill_inflight_queue
if inflight_req is not req
]
self.batch_result_processor._finish_deferred_mm_embedding_abort(req)

@scheduler_stage_method(SCHEDULER_STAGE_PROCESS_QUEUE)
def process_disagg_prefill_inflight_queue(
self: Scheduler, rids_to_check: Optional[List[str]] = None
Expand Down Expand Up @@ -1063,14 +1107,7 @@ def _retire_aborted_prefill_result(self: Scheduler, req: Req) -> bool:
# A bootstrap failure or earlier abort already retired it.
return False

sender = req.disagg_kv_sender
if sender is not None:
try:
sender.abort()
except Exception:
# Transport notification is best effort; local ownership must
# still be released or the next idle invariant check will fail.
logger.exception("Failed to notify KV sender of abort for %s", req.rid)
abort_kv_transfer(req)

if req.to_finish is not None and not req.finished():
req.update_finish_state()
Expand Down Expand Up @@ -1168,7 +1205,7 @@ def process_prefill_chunk(
req.extend_range.end,
len(req.origin_input_ids),
)
else:
elif req.mm_embedding_validation_count == 0:
self.send_kv_chunk(req)

if self.chunked_req is not None:
Expand All @@ -1186,6 +1223,8 @@ def process_prefill_chunk(
running_batch.batch_is_full = False

def maybe_send_cached_prefix_chunk(self: Scheduler, req: Req) -> None:
if req.mm_embedding_validation_count > 0:
return
if not envs.SGLANG_DISAGG_PREFILL_EARLY_SEND_CACHED_PREFIX.get():
return

Expand Down
15 changes: 12 additions & 3 deletions python/sglang/srt/disaggregation/utils.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
from __future__ import annotations

import logging
import random
from collections import deque
from contextlib import nullcontext
Expand All @@ -22,9 +23,7 @@
from sglang.srt.configs.model_config import get_dsa_index_topk
from sglang.srt.disaggregation.base import KVPoll
from sglang.srt.environ import envs
from sglang.srt.runtime_context import (
get_disagg,
)
from sglang.srt.runtime_context import get_disagg
from sglang.srt.utils import is_hip, is_npu

if TYPE_CHECKING:
Expand All @@ -47,6 +46,16 @@
#########################
FAKE_BOOTSTRAP_HOST = "2.2.2.2"
_IS_HIP = is_hip()
logger = logging.getLogger(__name__)


def abort_kv_transfer(req: Req) -> None:
"""notify the sender without letting transport errors bypass local teardown"""
if req.disagg_kv_sender is not None:
try:
req.disagg_kv_sender.abort()
except Exception:
logger.exception("Failed to notify KV sender of abort for %s", req.rid)


def poll_and_all_reduce_pp(
Expand Down
Loading
Loading