diff --git a/csrc/libtorch_stable/persistent_topk.cuh b/csrc/libtorch_stable/persistent_topk.cuh index 9ae0fd4bcbaa..dabf0df887a5 100644 --- a/csrc/libtorch_stable/persistent_topk.cuh +++ b/csrc/libtorch_stable/persistent_topk.cuh @@ -233,12 +233,14 @@ __device__ __noinline__ void histogram_2048_topk( if (pair_suffix >= TopK && (pair_suffix - h0) < TopK) { decode_smem[SBASE + sTHR] = 2 * tx; + if (h0 > DBUF) decode_smem[SBASE + sBUF0] = DBUF + 1; } { const int right_suf = pair_suffix - h0; const int next_suf = pair_suffix - pair_sum; if (right_suf >= TopK && next_suf < TopK) { decode_smem[SBASE + sTHR] = 2 * tx + 1; + if (pair_sum - h0 > DBUF) decode_smem[SBASE + sBUF0] = DBUF + 1; } } __syncthreads(); @@ -250,7 +252,9 @@ __device__ __noinline__ void histogram_2048_topk( const int sOUT_abs = SBASE + sOUT; const int sBUF0_abs = SBASE + sBUF0; - { + // Skip a collection pass when the histogram already proves that the + // threshold bin cannot fit in the fixed-size buffer. + if (__builtin_expect(decode_smem[sBUF0_abs] <= DBUF, 1)) { const uint32_t uthr = static_cast(threshold); int item = 0; const int n_vec_iters = (n_vec + kThreadsPerBlock - 1) / kThreadsPerBlock; @@ -305,6 +309,11 @@ __device__ __noinline__ void histogram_2048_topk( // If all buffered elements fit, output them all (common for short seqs) const int raw_buf0 = decode_smem[SBASE + sBUF0]; + if (raw_buf0 > DBUF) { + topk_histogram_4096::exact_topk_rescan( + logits, output_indices, static_cast(seq_len), decode_smem); + return; + } if (raw_buf0 <= remaining_k) { const int nb = (raw_buf0 < DBUF) ? raw_buf0 : DBUF; const int base = decode_smem[SBASE + sOUT]; @@ -351,6 +360,11 @@ __device__ __noinline__ void histogram_2048_topk( const int dst = src ^ 1; const int raw_buf = decode_smem[SBASE + sBUF0 + src]; + if (raw_buf > DBUF) { + topk_histogram_4096::exact_topk_rescan( + logits, output_indices, static_cast(seq_len), decode_smem); + return; + } const int num_buffered = (raw_buf < DBUF) ? raw_buf : DBUF; compute_suffix_sum(); @@ -475,7 +489,11 @@ __device__ __noinline__ void histogram_256_topk( if (thread_id < RADIX && shared_histogram[0][thread_id] > remaining_k && shared_histogram[0][thread_id + 1] <= remaining_k) { shared_threshold_bin = thread_id; - shared_buffered_count[0] = 0; + const int threshold_bin_count = shared_histogram[0][thread_id] - + shared_histogram[0][thread_id + 1]; + shared_buffered_count[0] = threshold_bin_count > MAX_BUFFERED_ITEMS + ? MAX_BUFFERED_ITEMS + 1 + : 0; shared_output_count = 0; } __syncthreads(); @@ -501,19 +519,21 @@ __device__ __noinline__ void histogram_256_topk( } __syncthreads(); - for (int idx = thread_id; idx < seq_len; idx += kThreadsPerBlock) { - const float logit_value = logits[idx + logits_offset]; - const int bin = convert_to_uint8(logit_value); - if (bin > threshold_bin) { - const int output_pos = atomicAdd(&shared_output_count, 1); - output_indices[output_pos] = idx; - } else if (bin == threshold_bin) { - const int buffer_pos = atomicAdd(&shared_buffered_count[0], 1); - if (__builtin_expect(buffer_pos < MAX_BUFFERED_ITEMS, 1)) { - buffered_indices[0][buffer_pos] = idx; - const uint32_t fp32_bits = convert_to_uint32_v2(logit_value); - const int next_bin = (fp32_bits >> 24) & 0xFF; - atomicAdd(&shared_histogram[0][next_bin], 1); + if (__builtin_expect(shared_buffered_count[0] <= MAX_BUFFERED_ITEMS, 1)) { + for (int idx = thread_id; idx < seq_len; idx += kThreadsPerBlock) { + const float logit_value = logits[idx + logits_offset]; + const int bin = convert_to_uint8(logit_value); + if (bin > threshold_bin) { + const int output_pos = atomicAdd(&shared_output_count, 1); + output_indices[output_pos] = idx; + } else if (bin == threshold_bin) { + const int buffer_pos = atomicAdd(&shared_buffered_count[0], 1); + if (__builtin_expect(buffer_pos < MAX_BUFFERED_ITEMS, 1)) { + buffered_indices[0][buffer_pos] = idx; + const uint32_t fp32_bits = convert_to_uint32_v2(logit_value); + const int next_bin = (fp32_bits >> 24) & 0xFF; + atomicAdd(&shared_histogram[0][next_bin], 1); + } } } } @@ -524,6 +544,12 @@ __device__ __noinline__ void histogram_256_topk( const int src_buffer = pass % 2; const int dst_buffer = src_buffer ^ 1; const int raw_buffered = shared_buffered_count[src_buffer]; + if (raw_buffered > MAX_BUFFERED_ITEMS) { + topk_histogram_4096::exact_topk_rescan( + logits + logits_offset, output_indices, + static_cast(seq_len), medium_smem); + return; + } const int num_buffered = (raw_buffered < MAX_BUFFERED_ITEMS) ? raw_buffered : MAX_BUFFERED_ITEMS; @@ -1001,21 +1027,25 @@ struct vec_t { template struct FilteredTopKTraits; -// Specialization for float (32-bit): coarse histogram uses FP16 high 8 bits, 4 -// refinement rounds +// Specialization for float (32-bit): use the high 11 ordered FP16 bits for +// coarse selection, matching TensorRT-LLM's production radix selector. The +// wider first stage sharply reduces candidate-buffer overflow before the exact +// FP32 refinement rounds. template <> struct FilteredTopKTraits { using OrderedType = uint32_t; + static constexpr int COARSE_BITS = 11; + static constexpr int COARSE_RADIX = 1 << COARSE_BITS; static constexpr int NUM_REFINE_ROUNDS = 4; static constexpr int FIRST_REFINE_SHIFT = 24; - __device__ __forceinline__ static uint8_t ToCoarseKey(float x) { - // Convert to FP16 representation and extract high 8 bits + __device__ __forceinline__ static uint16_t ToCoarseKey(float x) { + // Convert to an ordered FP16 representation and extract its high 11 bits. __half h = __float2half_rn(x); uint16_t bits = __half_as_ushort(h); uint16_t key = (bits & 0x8000) ? static_cast(~bits) : static_cast(bits | 0x8000); - return static_cast(key >> 8); + return static_cast(key >> (16 - COARSE_BITS)); } __device__ __forceinline__ static OrderedType ToOrdered(float x) { @@ -1048,6 +1078,20 @@ __global__ void __launch_bounds__(FILTERED_TOPK_BLOCK_THREADS) constexpr uint32_t BLOCK_SIZE = FILTERED_TOPK_BLOCK_THREADS; constexpr int RADIX = 256; constexpr int SMEM_INPUT_SIZE = FILTERED_TOPK_SMEM_INPUT_SIZE; + using Traits = FilteredTopKTraits; + constexpr int COARSE_RADIX = Traits::COARSE_RADIX; + static_assert(COARSE_RADIX == 2 * BLOCK_SIZE); + using CoarseScan = cub::BlockScan; + struct CoarseSmem { + int histogram[COARSE_RADIX + 1]; + typename CoarseScan::TempStorage scan; + }; + static_assert(sizeof(CoarseSmem) <= FILTERED_TOPK_SMEM_DYNAMIC); + + // The long path's coarse histogram and candidate double-buffer are never + // live at the same time. Overlay them in the existing 128 KiB dynamic + // allocation so widening the histogram does not increase per-CTA SMEM. + extern __shared__ __align__(16) uint8_t dynamic_smem[]; const uint32_t bid = blockIdx.x; const int tx = threadIdx.x; @@ -1069,13 +1113,12 @@ __global__ void __launch_bounds__(FILTERED_TOPK_BLOCK_THREADS) // Short path if (length <= 32768) { - extern __shared__ uint8_t _smem_reg[]; if constexpr (UsePredicatedShortLoads) { hist4096::histogram_4096_topk_predicated(score, dst, length, - _smem_reg); + dynamic_smem); } else { hist4096::histogram_4096_topk(score, dst, length, - _smem_reg); + dynamic_smem); } return; } @@ -1089,14 +1132,17 @@ __global__ void __launch_bounds__(FILTERED_TOPK_BLOCK_THREADS) auto& s_histogram = s_histogram_buf[0]; - // Dynamic shared memory for input double buffer - extern __shared__ int s_input_idx[][SMEM_INPUT_SIZE]; + auto* coarse_smem = reinterpret_cast(dynamic_smem); + auto* s_coarse_histogram = coarse_smem->histogram; + auto* s_input_idx = + reinterpret_cast(dynamic_smem); - using Traits = FilteredTopKTraits; int topk = top_k; - // Stage 1: 8-bit coarse histogram with vectorized loads - if (tx < RADIX + 1) s_histogram[tx] = 0; + // Stage 1: 11-bit ordered-FP16 histogram with vectorized loads. + for (int bin = tx; bin < COARSE_RADIX + 1; bin += BLOCK_SIZE) { + s_coarse_histogram[bin] = 0; + } __syncthreads(); vec_t score_vec; @@ -1109,18 +1155,36 @@ __global__ void __launch_bounds__(FILTERED_TOPK_BLOCK_THREADS) #pragma unroll for (int j = 0; j < VEC_SIZE; ++j) { const auto bin = Traits::ToCoarseKey(score_vec[j]); - atomicAdd(&s_histogram[bin], 1); + atomicAdd(&s_coarse_histogram[bin], 1); } } // Handle tail for (int i = aligned_length + tx; i < length; i += BLOCK_SIZE) { const auto bin = Traits::ToCoarseKey(score[i]); - atomicAdd(&s_histogram[bin], 1); + atomicAdd(&s_coarse_histogram[bin], 1); } __syncthreads(); - // Suffix sum - const auto run_cumsum = [&]() { + // One block scan covers two contiguous bins per thread. Converting its + // ascending exclusive prefix into suffix counts avoids eleven Hillis-Steele + // rounds over the 2048-bin coarse histogram. + const int coarse_bin0 = 2 * tx; + const int coarse_count0 = s_coarse_histogram[coarse_bin0]; + const int coarse_count1 = s_coarse_histogram[coarse_bin0 + 1]; + const int coarse_local_sum = coarse_count0 + coarse_count1; + int coarse_lower_prefix; + int coarse_total; + CoarseScan(coarse_smem->scan) + .ExclusiveSum(coarse_local_sum, coarse_lower_prefix, coarse_total); + __syncthreads(); + s_coarse_histogram[coarse_bin0] = coarse_total - coarse_lower_prefix; + s_coarse_histogram[coarse_bin0 + 1] = + coarse_total - coarse_lower_prefix - coarse_count0; + if (tx == 0) s_coarse_histogram[COARSE_RADIX] = 0; + __syncthreads(); + + // Suffix sum for the later 8-bit exact-refinement histograms. + const auto run_refine_cumsum = [&]() { #pragma unroll 8 for (int i = 0; i < 8; ++i) { if (tx < RADIX) { @@ -1136,16 +1200,19 @@ __global__ void __launch_bounds__(FILTERED_TOPK_BLOCK_THREADS) } }; - run_cumsum(); - if (tx < RADIX && s_histogram[tx] > topk && s_histogram[tx + 1] <= topk) { - s_threshold_bin_id = tx; - s_num_input[0] = 0; - s_counter = 0; + for (int i = 0; i < 2; ++i) { + const int bin = coarse_bin0 + i; + if (s_coarse_histogram[bin] > topk && + s_coarse_histogram[bin + 1] <= topk) { + s_threshold_bin_id = bin; + s_num_input[0] = 0; + s_counter = 0; + } } __syncthreads(); const auto threshold_bin = s_threshold_bin_id; - topk -= s_histogram[threshold_bin + 1]; + topk -= s_coarse_histogram[threshold_bin + 1]; constexpr int NUM_ROUNDS = Traits::NUM_REFINE_ROUNDS; constexpr int FIRST_SHIFT = Traits::FIRST_REFINE_SHIFT; @@ -1217,10 +1284,15 @@ __global__ void __launch_bounds__(FILTERED_TOPK_BLOCK_THREADS) const auto r_idx = round % 2; const auto _raw_num_input = s_num_input[r_idx]; + if (_raw_num_input > SMEM_INPUT_SIZE) { + hist4096::exact_topk_rescan( + score, dst, length, s_input_idx); + return; + } const auto num_input = (_raw_num_input < SMEM_INPUT_SIZE) ? _raw_num_input : SMEM_INPUT_SIZE; - run_cumsum(); + run_refine_cumsum(); if (tx < RADIX && s_histogram[tx] > topk && s_histogram[tx + 1] <= topk) { s_threshold_bin_id = tx; s_num_input[r_idx ^ 1] = 0; diff --git a/csrc/libtorch_stable/topk_histogram_4096.cuh b/csrc/libtorch_stable/topk_histogram_4096.cuh index 5f9f823a3399..eea4fabd77b8 100644 --- a/csrc/libtorch_stable/topk_histogram_4096.cuh +++ b/csrc/libtorch_stable/topk_histogram_4096.cuh @@ -7,6 +7,7 @@ #include #include +#include #include namespace vllm { @@ -73,6 +74,207 @@ __device__ __forceinline__ auto convert_to_uint32_v2(float x) -> uint32_t { return (bits & 0x80000000u) ? ~bits : (bits | 0x80000000u); } +template +__device__ __forceinline__ void for_each_score( + const float* __restrict__ scores, uint32_t length, Visit visit) { + static_assert(VecSize == 1 || VecSize == 2 || VecSize == 4); + const uint32_t tx = threadIdx.x; + + if constexpr (VecSize == 1) { + for (uint32_t idx = tx; idx < length; idx += BlockSize) { + visit(idx, scores[idx]); + } + } else { + const uint32_t aligned_length = length - length % VecSize; + + for (uint32_t base = tx * VecSize; base < aligned_length; + base += BlockSize * VecSize) { + if constexpr (VecSize == 4) { + const float4 values = + *reinterpret_cast(scores + base); + visit(base, values.x); + visit(base + 1, values.y); + visit(base + 2, values.z); + visit(base + 3, values.w); + } else { + const float2 values = + *reinterpret_cast(scores + base); + visit(base, values.x); + visit(base + 1, values.y); + } + } + + for (uint32_t idx = aligned_length + tx; idx < length; idx += BlockSize) { + visit(idx, scores[idx]); + } + } +} + +enum class PivotRelation : uint8_t { kGreater, kEqual }; + +template +__device__ __forceinline__ void collect_pivot_matches( + const float* __restrict__ scores, int32_t* __restrict__ output, + uint32_t length, uint32_t pivot, uint32_t* output_counter) { + for_each_score( + scores, length, [&](uint32_t idx, float score) { + const uint32_t ordered = convert_to_uint32_v2(score); + bool matches; + if constexpr (Relation == PivotRelation::kGreater) { + matches = ordered > pivot; + } else { + matches = ordered == pivot; + } + if (matches) { + const uint32_t pos = atomicAdd(output_counter, 1); + if (pos < TopK) output[pos] = static_cast(idx); + } + }); +} + +// Exact, bounded-memory fallback for candidate-buffer overflow. Each radix +// round rescans the full row, so correctness does not depend on how many +// values share a coarse histogram bin. This is intentionally slower than the +// buffered paths and is entered only after an overflow is detected. +template +__device__ void exact_topk_rescan(const float* __restrict__ scores, + int32_t* __restrict__ output, + uint32_t length, void* _smem) { + static_assert(BlockSize >= RADIX); + static_assert(VecSize == 1 || VecSize == 2 || VecSize == 4); + constexpr uint32_t kHistogramBins = UseWideRadix ? 2048 : RADIX; + constexpr uint32_t kRadixRounds = UseWideRadix ? 3 : 4; + static_assert(!UseWideRadix || kHistogramBins % BlockSize == 0); + using BlockScan = cub::BlockScan; + struct ExactSmem { + uint32_t histogram[kHistogramBins]; + typename BlockScan::TempStorage scan; + uint32_t prefix; + uint32_t remaining; + uint32_t output_counter; + }; + + auto* smem = static_cast(_smem); + const uint32_t tx = threadIdx.x; + + if (tx == 0) { + smem->prefix = 0; + smem->remaining = TopK; + } + __syncthreads(); + +#pragma unroll + for (uint32_t round = 0; round < kRadixRounds; ++round) { + for (uint32_t bin = tx; bin < kHistogramBins; bin += BlockSize) { + smem->histogram[bin] = 0; + } + __syncthreads(); + + uint32_t shift; + uint32_t digit_mask; + uint32_t prefix_mask; + if constexpr (UseWideRadix) { + shift = (round == 0) ? 21 : ((round == 1) ? 10 : 0); + digit_mask = (round < 2) ? 0x7FFu : 0x3FFu; + prefix_mask = + (round == 0) ? 0u : ((round == 1) ? 0xFFE00000u : 0xFFFFFC00u); + } else { + shift = 24 - round * 8; + digit_mask = 0xFFu; + prefix_mask = (round == 0) ? 0u : (~0u << (32 - round * 8)); + } + const uint32_t prefix = smem->prefix; + + for_each_score( + scores, length, [&](uint32_t, float score) { + const uint32_t ordered = convert_to_uint32_v2(score); + if ((ordered & prefix_mask) == prefix) { + atomicAdd(&smem->histogram[(ordered >> shift) & digit_mask], 1); + } + }); + __syncthreads(); + + if constexpr (UseWideRadix) { + constexpr uint32_t kBinsPerThread = kHistogramBins / BlockSize; + uint32_t counts[kBinsPerThread]; + uint32_t local_sum = 0; +#pragma unroll + for (uint32_t i = 0; i < kBinsPerThread; ++i) { + counts[i] = smem->histogram[tx * kBinsPerThread + i]; + local_sum += counts[i]; + } + + uint32_t lower_prefix; + uint32_t total; + BlockScan(smem->scan).ExclusiveSum(local_sum, lower_prefix, total); + uint32_t count_above = total - lower_prefix - local_sum; + const uint32_t remaining = smem->remaining; +#pragma unroll + for (int i = static_cast(kBinsPerThread) - 1; i >= 0; --i) { + const uint32_t count = counts[i]; + if (count_above < remaining && count_above + count >= remaining) { + const uint32_t bin = tx * kBinsPerThread + i; + smem->remaining = remaining - count_above; + smem->prefix |= bin << shift; + } + count_above += count; + } + } else if (tx == 0) { + uint32_t count_above = 0; + for (int bin = RADIX - 1; bin >= 0; --bin) { + const uint32_t count = smem->histogram[bin]; + if (count_above + count >= smem->remaining) { + smem->remaining -= count_above; + smem->prefix |= static_cast(bin) << shift; + break; + } + count_above += count; + } + } + __syncthreads(); + } + + const uint32_t pivot = smem->prefix; + if constexpr (FuseOutputScan) { + static_assert(VecSize == 1); + if (tx == 0) { + smem->output_counter = 0; + smem->histogram[0] = 0; + } + __syncthreads(); + + const uint32_t equal_count = smem->remaining; + const uint32_t equal_base = TopK - equal_count; + for_each_score( + scores, length, [&](uint32_t idx, float score) { + const uint32_t ordered = convert_to_uint32_v2(score); + if (ordered > pivot) { + const uint32_t pos = atomicAdd(&smem->output_counter, 1); + output[pos] = static_cast(idx); + } else if (ordered == pivot) { + const uint32_t pos = atomicAdd(&smem->histogram[0], 1); + if (pos < equal_count) { + output[equal_base + pos] = static_cast(idx); + } + } + }); + __syncthreads(); + } else { + if (tx == 0) smem->output_counter = 0; + __syncthreads(); + + collect_pivot_matches( + scores, output, length, pivot, &smem->output_counter); + __syncthreads(); + + collect_pivot_matches( + scores, output, length, pivot, &smem->output_counter); + __syncthreads(); + } +} + // Converts each score to a 12-bit bin (FP16 sign-magnitude -> top 12 bits -> // bin 0-4095) template @@ -446,6 +648,13 @@ __device__ void histogram_4096_topk(const float* __restrict__ scores, const auto [thr_bin, num_above, num_equal] = smem->match; const bool need_tie = (num_equal + num_above > TopK); + // The scatter below intentionally stores at most TopK candidates even when + // the union is larger (kMaxTies for small K), so TopK is the real capacity. + if (need_tie && num_equal > TopK) { + exact_topk_rescan(scores, output, length, _smem); + return; + } + done = false; #pragma unroll for (uint32_t v = 0; v < VECS_PER_THREAD && !done; v++) { diff --git a/tests/kernels/test_top_k_per_row.py b/tests/kernels/test_top_k_per_row.py index a1e27d2e50a3..97b8f38dc698 100644 --- a/tests/kernels/test_top_k_per_row.py +++ b/tests/kernels/test_top_k_per_row.py @@ -1265,6 +1265,47 @@ def test_persistent_topk_reused_group_after_short_row() -> None: assert set(indices[target_row].cpu().tolist()) == set(expected.cpu().tolist()) +@pytest.mark.skipif(not current_platform.is_cuda(), reason="This test requires CUDA") +@pytest.mark.parametrize("top_k", [512, 1024, 2048]) +@pytest.mark.parametrize( + ("num_rows", "seq_len"), + [ + pytest.param(33, 16384, id="filtered_short"), + pytest.param(8, 8192, id="persistent_decode"), + pytest.param(8, 12288, id="persistent_medium"), + pytest.param(33, 32769, id="filtered_large"), + ], +) +@torch.inference_mode() +def test_persistent_topk_candidate_buffer_overflow( + top_k: int, num_rows: int, seq_len: int +) -> None: + """Narrow score distributions must not truncate threshold candidates.""" + generator = torch.Generator(device="cpu").manual_seed(0) + logits = ( + 1.0 + torch.rand(num_rows, seq_len, generator=generator) * 0.01 + ).cuda() + lengths = torch.full((num_rows,), seq_len, dtype=torch.int32, device="cuda") + indices = torch.empty((num_rows, top_k), dtype=torch.int32, device="cuda") + + _run_topk_backend( + "persistent_topk", logits, lengths, indices, top_k, seq_len + ) + torch.accelerator.synchronize() + + assert torch.all((indices >= 0) & (indices < seq_len)) + sorted_indices = indices.sort(dim=1).values + assert torch.all(sorted_indices[:, 1:] != sorted_indices[:, :-1]) + + selected = torch.gather(logits, 1, indices.long()).sort( + dim=1, descending=True + ).values + expected = torch.topk(logits, top_k, dim=1).values.sort( + dim=1, descending=True + ).values + torch.testing.assert_close(selected, expected, rtol=0, atol=0) + + @pytest.mark.skipif(not current_platform.is_cuda(), reason="This test requires CUDA") @pytest.mark.parametrize("top_k", [512, 2048]) @pytest.mark.parametrize("backend", WORKSPACE_TOPK_BACKENDS)