Skip to content
Merged
44 changes: 37 additions & 7 deletions vllm/model_executor/layers/sparse_attn_indexer.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Custom Sparse Attention Indexer layers."""

import math

import torch
import torch.distributed as dist

Expand Down Expand Up @@ -46,6 +48,7 @@
mqa_logits_use_mx_scales,
)
from vllm.utils.import_utils import has_cutedsl
from vllm.utils.math_utils import round_up
from vllm.utils.torch_utils import (
LayerNameType,
_encode_layer_name,
Expand Down Expand Up @@ -478,6 +481,10 @@ def sparse_attn_indexer(
)
assert candidate_block_size > 0

# DeepSelect ranks SM100's BF16 decode logits directly; other top-k
# backends and the DCP merge read an FP32 copy.
fp32_decode_logits = topk_backend != "deep_select" or dcp_world_size != 1

# assert isinstance(attn_metadata, dict)
if not isinstance(attn_metadata, dict):
# Reserve workspace for indexer during profiling run
Expand All @@ -498,11 +505,34 @@ def sparse_attn_indexer(
profile_specs.extend(gather_spec * 2)
current_workspace_manager().get_simultaneous(*profile_specs)

# Dummy allocation to simulate for peak logits tensor memory during inference.
# FP8 elements so elements == bytes
max_logits_elems = envs.VLLM_SPARSE_INDEXER_MAX_LOGITS_MB * 1024 * 1024
# Decode rows per step: each decode is padded to 1 + speculative tokens.
# The profiling run sets the vLLM config; other dummy runs may not.
max_decode_tokens = 0
if (cfg := get_current_vllm_config_or_none()) is not None:
max_decode_tokens = min(
cfg.scheduler_config.max_num_seqs,
cfg.scheduler_config.max_num_batched_tokens,
) * (1 + cfg.num_speculative_tokens)

# Dummy allocation for the peak logits memory during inference: the
# prefill chunks' budget, or the decode logits, which eager and
# piecewise steps allocate outside the CUDA graph pools. DeepGEMM emits
# them in BF16 on SM100 and FP32 elsewhere, each row holding whole KV
# splits at a 1024-byte stride; the FP32 copy keeps that stride.
if current_platform.is_device_capability_family(100):
logits_dtype, split_kv = torch.bfloat16, 384
else:
logits_dtype, split_kv = torch.float32, 256
row_align = math.lcm(split_kv, 1024 // logits_dtype.itemsize)
elem_bytes = logits_dtype.itemsize
if logits_dtype != torch.float32 and fp32_decode_logits:
elem_bytes += torch.float32.itemsize
max_logits_bytes = max(
envs.VLLM_SPARSE_INDEXER_MAX_LOGITS_MB * 1024 * 1024,
max_decode_tokens * round_up(max_model_len, row_align) * elem_bytes,
)
_ = torch.empty(
max_logits_elems, dtype=torch.uint8, device=hidden_states.device
max_logits_bytes, dtype=torch.uint8, device=hidden_states.device
)

# The dummy run returns before exchange_topk executes, so reserve its
Expand Down Expand Up @@ -887,9 +917,9 @@ def sparse_attn_indexer(
max_model_len=max_model_len,
indices=decode_metadata.indices,
)
# Keeping bf16 halves top-k/candidate reads; other top-k backends and
# DCP need fp32. SM100 decode is varlen: one context length per row.
if topk_backend != "deep_select" or dcp_world_size != 1:
# Keeping bf16 halves top-k/candidate reads. SM100 decode is varlen:
# one context length per row.
if fp32_decode_logits:
logits = as_fp32_logits(logits, seq_lens.view(-1))
num_rows = logits.shape[0]
if candidate_blocks is not None:
Expand Down
Loading