Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 14 additions & 3 deletions tests/v1/sample/test_logprobs.py
Original file line number Diff line number Diff line change
Expand Up @@ -1630,21 +1630,32 @@ def test_token_logprobs_large_batch_int64_row_offset():
)


@pytest.mark.parametrize("split_topk", [False, True])
@pytest.mark.parametrize("num_prompt_logprobs", [-1, 3])
def test_chunked_prompt_logprobs_match_unchunked(num_prompt_logprobs: int):
"""Chunks written into preallocated outputs must match one unchunked pass."""
def test_chunked_prompt_logprobs_match_unchunked(
num_prompt_logprobs: int, split_topk: bool, monkeypatch
):
"""Chunks, and top-k blocks within them, written into preallocated outputs
must match one unchunked pass."""
if not current_platform.is_cuda_alike():
pytest.skip("the logprob kernels are Triton")
from vllm.v1.worker.gpu.sample import prompt_logprob
from vllm.v1.worker.gpu.sample.logprob import compute_topk_scores
from vllm.v1.worker.gpu.sample.prompt_logprob import (
CHUNK_SIZE,
compute_prompt_logprobs_with_chunking,
)

if split_topk:
# Blocks of 100 rows at full vocab, straddling the chunk boundaries.
monkeypatch.setattr(prompt_logprob, "TOPK_MAX_ELEMENTS", 100 * 513)
device = torch.device(current_platform.device_type)
vocab_size = 512
num_tokens = 2 * CHUNK_SIZE + 3
logits = torch.randn(num_tokens, vocab_size, device=device)
# Distinct values per row, since topk's order among ties can vary with the
# number of rows ranked together.
logits = torch.rand(num_tokens, vocab_size, device=device).argsort(dim=-1)
logits = logits.float() / vocab_size
prompt_token_ids = torch.randint(vocab_size, (num_tokens,), device=device)

token_ids, logprobs, ranks = compute_prompt_logprobs_with_chunking(
Expand Down
46 changes: 27 additions & 19 deletions vllm/v1/worker/gpu/sample/prompt_logprob.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,9 @@
)

CHUNK_SIZE = 1024
# Cap rows x k per torch.topk call, whose workspace grows with both (~5.5 GiB
# for 1024 rows at a 152k vocab). Only splits chunks when k is very large.
TOPK_MAX_ELEMENTS = 64 << 20


@dataclass
Expand Down Expand Up @@ -308,25 +311,30 @@ def compute_prompt_logprobs_with_chunking(
if num_prompt_logprobs == -1
else num_prompt_logprobs
)
result = compute_topk_scores(
prompt_logits,
requested_num,
prompt_token_ids[start_idx:end_idx],
logits_mode=logits_mode,
)
# Return token_ids as int32 to reduce peak memory usage.
if num_tokens <= CHUNK_SIZE:
logprob_token_ids = result.logprob_token_ids.to(torch.int32)
return logprob_token_ids, result.logprobs, result.selected_token_ranks
if token_ids is None:
size = num_tokens, result.logprobs.shape[1]
token_ids = prompt_token_ids.new_empty(size, dtype=torch.int32)
scores = result.logprobs.new_empty(size)
ranks = result.selected_token_ranks.new_empty(num_tokens)
assert scores is not None and ranks is not None
token_ids[start_idx:end_idx] = result.logprob_token_ids
scores[start_idx:end_idx] = result.logprobs
ranks[start_idx:end_idx] = result.selected_token_ranks
rows_per_topk = max(1, TOPK_MAX_ELEMENTS // (requested_num + 1))
for block_start in range(0, prompt_logits.shape[0], rows_per_topk):
logits = prompt_logits[block_start : block_start + rows_per_topk]
Comment thread
njhill marked this conversation as resolved.
Resolved
num_logits = len(logits)
rows = slice(start_idx + block_start, start_idx + block_start + num_logits)
result = compute_topk_scores(
logits, requested_num, prompt_token_ids[rows], logits_mode=logits_mode
)
# Return token_ids as int32 to reduce peak memory usage.
if num_logits == num_tokens:
logprob_token_ids = result.logprob_token_ids.to(torch.int32)
return logprob_token_ids, result.logprobs, result.selected_token_ranks
if token_ids is None:
size = num_tokens, result.logprobs.shape[1]
token_ids = prompt_token_ids.new_empty(size, dtype=torch.int32)
scores = result.logprobs.new_empty(size)
ranks = result.selected_token_ranks.new_empty(num_tokens)
assert scores is not None and ranks is not None
token_ids[rows] = result.logprob_token_ids
scores[rows] = result.logprobs
ranks[rows] = result.selected_token_ranks
# Free each block's results before ranking the next.
del result
del prompt_logits, logits
Comment thread
njhill marked this conversation as resolved.
Resolved

assert token_ids is not None and scores is not None and ranks is not None
return token_ids, scores, ranks