From 47fb11bc025540158a5243865ed14e8cd3ae0bd3 Mon Sep 17 00:00:00 2001 From: namantiwari2002 Date: Tue, 26 May 2026 23:44:31 +0900 Subject: [PATCH] [Bugfix] Skip CUDA-graph padded positions in eagle draft step kernel During speculative decoding draft iterations, the eagle_step_slot_mapping_metadata_kernel increments seq_lens for all positions in the batch. CUDA-graph padded positions (beyond the actual request count) have seq_lens == 0 and block_table entries filled with -1. Incrementing their seq_lens from 0 to 1 causes the attention kernel to index into block_table with the -1 entry, triggering a cudaErrorIllegalAddress / illegal memory access. The fix checks seq_lens before incrementing: if seq_len == 0, the position is treated as padding (PADDING_SLOT_ID for slot_mapping, 0 for clamped_positions) and seq_lens is left at 0. This manifests under concurrent load with async scheduling enabled, when CUDA-graph batch padding creates positions that the original kernel would incorrectly activate. Signed-off-by: Kshitij --- vllm/v1/spec_decode/utils.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/vllm/v1/spec_decode/utils.py b/vllm/v1/spec_decode/utils.py index e046f0136152..3d6446bc482e 100644 --- a/vllm/v1/spec_decode/utils.py +++ b/vllm/v1/spec_decode/utils.py @@ -57,8 +57,17 @@ def eagle_step_slot_mapping_metadata_kernel( tl.store(out_slot_mapping_ptr + req_idx, PAD_ID) return - # Load current position and increment - position = tl.load(positions_ptr + req_idx) + # CUDA-graph padded positions (beyond the actual request count) have + # seq_lens == 0 and block_table entries of -1. Incrementing their + # seq_lens from 0 to 1 would cause the attention kernel to read + # block_table[-1], triggering an illegal memory access. + seq_len = tl.load(seq_lens_ptr + req_idx) + if seq_len == 0: + tl.store(out_slot_mapping_ptr + req_idx, PAD_ID) + tl.store(out_clamped_positions_ptr + req_idx, 0) + return + + position = tl.load(positions_ptr + req_idx) # Load current position new_position = position + 1 # Check bounds and compute clamped position @@ -75,7 +84,6 @@ def eagle_step_slot_mapping_metadata_kernel( slot_id = tl.where(exceeds_max, PAD_ID, slot_id) # Update seq_lens: +1 normally, or 1 if exceeded - seq_len = tl.load(seq_lens_ptr + req_idx) new_seq_len = tl.where(exceeds_max, 1, seq_len + 1) new_seq_len = tl.minimum(new_seq_len, max_model_len)