[Fix]: Adjust FutureMap's token_id_bufs Size to Prevent ChunkedPrefill's next_token_ids from Overwriting Previous Prefill Requests' next_token_id

python/sglang/srt/managers/overlap_utils.py

@@ -33,14 +33,22 @@ class FutureMap:
     def __init__(
         self,
         max_running_requests: int,
+        chunked_prefill_size: int,
+        context_len: int,
         device: torch.device,
         spec_algo: Optional[SpeculativeAlgorithm] = None,
     ):
         self.future_ct = 0
-        # A factor of 3 is used to avoid collision in the circular buffer.
-        self.future_limit = max_running_requests * 3
-        # A factor of 5 is used to ensure the buffer is large enough.
-        self.future_buffer_len = max_running_requests * 5
+        # For long-text requests in chunked_prefill computation mode, the input is split into multiple chunks. Each chunk generates
+        # a next_token_id stored in the buffer. When processing multiple requests concurrently, there is a possibility that 
+        # subsequently processed chunks may overwrite the next_token_ids generated during the prefill stage of earlier requests. 
+        # These overwritten IDs serve as the first input_ids in the decode phase, potentially causing errors. Therefore, the 
+        # buffer size should be determined based on both the text length limit and the chunked_prefill_size to ensure it can 
+        # accommodate all possible next_token_ids during concurrent processing
+        # Additionally, to avoid collisions in the circular buffer, the future_limit must be set to a value greater than or equal to three.
+        self.future_limit = max_running_requests * max(3, (context_len + chunked_prefill_size -1)//chunked_prefill_size) if chunked_prefill_size else 3 * max_running_requests
+        # Adding 2 * max_running_requests to future_limit ensures the buffer is sufficiently large.
+        self.future_buffer_len = self.future_limit + 2 * max_running_requests
         self.device = device
         self.spec_algo = spec_algo
         self.buf_initialized = False

python/sglang/srt/managers/scheduler.py

@@ -951,7 +951,7 @@ def init_overlap(self):
         ).stream(self.copy_stream)
 
         self.future_map = FutureMap(
-            self.max_running_requests, self.device, self.spec_algorithm
+            self.max_running_requests, self.chunked_prefill_size, self.model_config.context_len, self.device, self.spec_algorithm
         )
         self.batch_record_buf = [None] * 2
         self.batch_record_ct = 0