From 16898da24c3dc9663b95f6dd2fd014f837be693a Mon Sep 17 00:00:00 2001 From: AMD-yanfeiwang Date: Sat, 27 Jun 2026 03:52:07 +0000 Subject: [PATCH] fix(hicache): make is_load_back_event_done a pure per-rank poll (no TP collective) is_load_back_event_done() conditionally called loading_check() (a TP all_reduce) only on ranks whose local load-back finish_event had already fired. It is invoked per-request from the decode load-back path, so the number of calls and the local finish_event.query() result diverge across TP ranks; the conditional collective therefore desynchronized the TP all_reduce sequence and could deadlock under TP > 1 (manifesting as a small number of requests stuck forever with errors=0 and idle GPUs). The load ack-reaping collective is already issued once per scheduler iteration at a rank-invariant point (check_hicache_events -> loading_check), so this per-request helper only needs to report local GPU-event readiness. Drop the collective and make it a pure finish_event.query(). This mirrors the existing rank-invariant handling in writing_check(). --- python/sglang/srt/mem_cache/hiradix_cache.py | 24 +++++++++++++++----- 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/python/sglang/srt/mem_cache/hiradix_cache.py b/python/sglang/srt/mem_cache/hiradix_cache.py index 65e185413887..4e7a7a2a80bb 100644 --- a/python/sglang/srt/mem_cache/hiradix_cache.py +++ b/python/sglang/srt/mem_cache/hiradix_cache.py @@ -963,18 +963,30 @@ def loading_check(self): finish_count -= 1 def is_load_back_event_done(self, consumer_index: int) -> bool: - """Return True after the local load-back event is complete.""" + """Return True after the local load-back event is complete. + + This is a pure per-rank GPU-event poll and must NOT issue any TP + collective. It is called per-request from the decode load-back path, + and both the number of calls and the result of the local + ``finish_event.query()`` differ across TP ranks. Previously this + conditionally called ``loading_check()`` (which runs an ``all_reduce``) + only on the subset of ranks whose local event had already fired, so the + TP collective sequence diverged and the ``all_reduce`` could deadlock + under TP > 1. + + The ack-reaping collective is already issued exactly once per scheduler + iteration at a rank-invariant point (``check_hicache_events`` -> + ``loading_check``), so dropping it here is safe: load-back acks are still + reaped in lockstep, and this function only reports local readiness for + per-request scheduling. + """ if consumer_index < 0: return True finish_event = self.cache_controller.layer_done_counter.events[ consumer_index ].finish_event - if not finish_event.query(): - return False - - self.loading_check() - return True + return finish_event.query() def evictable_size(self): return self.evictable_size_