diff --git a/tests/evals/gsm8k/configs/GLM-5.2-NVFP4-TP1-PCP4-DCP4-EP.yaml b/tests/evals/gsm8k/configs/GLM-5.2-NVFP4-TP1-PCP4-DCP4-EP.yaml index b9d2675431e7..8f3e9ce918ef 100644 --- a/tests/evals/gsm8k/configs/GLM-5.2-NVFP4-TP1-PCP4-DCP4-EP.yaml +++ b/tests/evals/gsm8k/configs/GLM-5.2-NVFP4-TP1-PCP4-DCP4-EP.yaml @@ -3,10 +3,13 @@ accuracy_threshold: 0.90 num_questions: 1319 num_fewshot: 5 max_concurrency: 100 +# Preserve KV-cache headroom after CUDA graph memory profiling and bound the +# DCP-aware FlashInfer workspace to 2.39 GiB at the 4,096-token scheduler cap. server_args: >- --compilation-config '{"cudagraph_mode":"PIECEWISE"}' + --gpu-memory-utilization 0.96 --max-model-len 4096 - --max-num-batched-tokens 32768 + --max-num-batched-tokens 4096 --safetensors-load-strategy prefetch --moe-backend flashinfer_cutlass --prefill-context-parallel-size 4 diff --git a/tests/evals/gsm8k/configs/GLM-5.2-NVFP4-TP2-PCP2-EP.yaml b/tests/evals/gsm8k/configs/GLM-5.2-NVFP4-TP2-PCP2-EP.yaml index 72f7fdf1ae91..bc5c11fa53e2 100644 --- a/tests/evals/gsm8k/configs/GLM-5.2-NVFP4-TP2-PCP2-EP.yaml +++ b/tests/evals/gsm8k/configs/GLM-5.2-NVFP4-TP2-PCP2-EP.yaml @@ -3,8 +3,10 @@ accuracy_threshold: 0.90 num_questions: 1319 num_fewshot: 5 max_concurrency: 100 +# Leave headroom for FlashInfer CUTLASS autotuning after KV-cache allocation. server_args: >- --compilation-config '{"cudagraph_mode":"NONE"}' + --gpu-memory-utilization 0.85 --max-model-len 4096 --max-num-batched-tokens 32768 --safetensors-load-strategy prefetch diff --git a/tests/v1/attention/test_dcp_a2a_pack_mask.py b/tests/v1/attention/test_dcp_a2a_pack_mask.py index 36032e7a69a0..cff1228baf03 100644 --- a/tests/v1/attention/test_dcp_a2a_pack_mask.py +++ b/tests/v1/attention/test_dcp_a2a_pack_mask.py @@ -13,7 +13,7 @@ mask_dcp_empty_shards_, ) -pytestmark = pytest.mark.skipif( +requires_accelerator = pytest.mark.skipif( not current_platform.is_cuda_alike(), reason="needs a CUDA or ROCm device for the Triton pack kernel", ) @@ -44,6 +44,7 @@ def _pack(out, lse, world_size, h_per_rank, head_dim, seq_lens, query_start_loc) @pytest.mark.parametrize("world_size,h_per_rank", [(8, 2), (8, 16), (4, 4), (2, 1)]) @pytest.mark.parametrize("tokens_per_req", [1, 2, 3]) @pytest.mark.parametrize("num_pad_rows", [0, 2]) +@requires_accelerator @pytest.mark.parametrize( "seq_lens_list", [ @@ -97,6 +98,7 @@ def test_fused_mask_matches_eager( @pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16]) @pytest.mark.parametrize("seed", range(8)) +@requires_accelerator def test_fused_mask_matches_eager_ragged(dtype, seed): """Non-uniform per-request query lengths. @@ -137,6 +139,7 @@ def test_fused_mask_matches_eager_ragged(dtype, seed): ) +@requires_accelerator def test_mask_disabled_is_unmasked(): """Passing no seq_lens/query_start_loc must leave the LSE untouched.""" torch.manual_seed(0) @@ -149,3 +152,14 @@ def test_mask_disabled_is_unmasked(): packed = _pack(out, lse, world_size, h_per_rank, head_dim, None, None) assert torch.isfinite(packed.float()).all() + + +def test_eager_mask_handles_rank_with_no_local_sequences(): + """Padded graph rows are all empty when a DCP rank has no sequences.""" + lse = torch.randn(8, 4, dtype=torch.float32) + seq_lens = torch.empty(0, dtype=torch.int32) + query_start_loc = torch.tensor([0], dtype=torch.int32) + + mask_dcp_empty_shards_(lse, seq_lens, query_start_loc) + + assert torch.isneginf(lse).all() diff --git a/tests/v1/attention/test_flashinfer_sparse_mla_workspace.py b/tests/v1/attention/test_flashinfer_sparse_mla_workspace.py new file mode 100644 index 000000000000..5b63c2c609f9 --- /dev/null +++ b/tests/v1/attention/test_flashinfer_sparse_mla_workspace.py @@ -0,0 +1,166 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""CPU-only tests for FlashInfer sparse MLA backend constraints.""" + +from types import SimpleNamespace +from unittest.mock import patch + +import pytest +import torch + +from vllm.v1.attention.backends.mla.flashinfer_mla_sparse import ( + _DEFAULT_WORKSPACE_BUFFER_SIZE, + FlashInferMLASparseTRTLLMBackend, + _required_workspace_bytes, + compute_trtllm_sparse_mla_workspace_bytes, +) + +# Reporter config from vllm#50781: GLM-5.2 (64 q heads), TP=8, DCP=8, +# max_num_batched_tokens=16384, FlashInfer 0.6.14. +_REPORTER_HEADS_PER_RANK = 64 // 8 +_REPORTER_DCP = 8 +_REPORTER_MAX_TOKENS = 16384 +# From the reported crash: trtllm-gen requested exactly this many bytes for +# trtllm_gen_softmax_workspace after an 8 MiB counter carve had left +# 404,750,336 of the 413,138,944-byte default buffer. +_REPORTER_OBSERVED_SOFTMAX_BYTES = 1_611_661_312 +_REPORTER_OBSERVED_PRECARVE_BYTES = 8_388_608 + + +def test_reporter_softmax_carve_is_reproduced_byte_exactly(): + # The crashing step scheduled 12288 tokens (a single 12K-token request); + # the softmax slab formula must reproduce the kernel's request exactly: + # 8 * (8 heads/rank * 8 dcp) * 12288 tokens * 256 + 1 MiB guard. + softmax_bytes = compute_trtllm_sparse_mla_workspace_bytes( + base_workspace_bytes=0, + dcp_world_size=_REPORTER_DCP, + num_heads_per_rank=_REPORTER_HEADS_PER_RANK, + max_num_batched_tokens=12288, + ) + assert softmax_bytes == _REPORTER_OBSERVED_SOFTMAX_BYTES + + +def test_reporter_config_covers_observed_overflow(): + computed = compute_trtllm_sparse_mla_workspace_bytes( + base_workspace_bytes=_DEFAULT_WORKSPACE_BUFFER_SIZE, + dcp_world_size=_REPORTER_DCP, + num_heads_per_rank=_REPORTER_HEADS_PER_RANK, + max_num_batched_tokens=_REPORTER_MAX_TOKENS, + ) + assert computed >= ( + _REPORTER_OBSERVED_SOFTMAX_BYTES + _REPORTER_OBSERVED_PRECARVE_BYTES + ) + + +def test_non_dcp_size_is_unchanged(): + computed = compute_trtllm_sparse_mla_workspace_bytes( + base_workspace_bytes=_DEFAULT_WORKSPACE_BUFFER_SIZE, + dcp_world_size=1, + num_heads_per_rank=128, + max_num_batched_tokens=65536, + ) + assert computed == _DEFAULT_WORKSPACE_BUFFER_SIZE + + +def test_combined_pcp_dcp_rejects_flashinfer_sparse(): + config = SimpleNamespace( + parallel_config=SimpleNamespace( + prefill_context_parallel_size=4, + decode_context_parallel_size=4, + ), + model_config=None, + ) + with patch("vllm.config.get_current_vllm_config", return_value=config): + reason = FlashInferMLASparseTRTLLMBackend.supports_combination( + head_size=576, + dtype=torch.bfloat16, + kv_cache_dtype="fp8", + block_size=64, + use_mla=True, + has_sink=False, + use_sparse=True, + use_mm_prefix=False, + device_capability=SimpleNamespace(major=10), + ) + + assert reason is not None + assert "use FLASHMLA_SPARSE" in reason + + +@pytest.mark.parametrize("pcp_size,dcp_size", [(4, 1), (1, 4)]) +def test_single_context_parallel_mode_allows_flashinfer_sparse( + pcp_size: int, dcp_size: int +): + config = SimpleNamespace( + parallel_config=SimpleNamespace( + prefill_context_parallel_size=pcp_size, + decode_context_parallel_size=dcp_size, + ), + model_config=None, + ) + with patch("vllm.config.get_current_vllm_config", return_value=config): + reason = FlashInferMLASparseTRTLLMBackend.supports_combination( + head_size=576, + dtype=torch.bfloat16, + kv_cache_dtype="fp8", + block_size=64, + use_mla=True, + has_sink=False, + use_sparse=True, + use_mm_prefix=False, + device_capability=SimpleNamespace(major=10), + ) + + assert reason is None + + +def test_default_constant_matches_envs_default(monkeypatch): + from vllm import envs + + monkeypatch.delenv("VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE", raising=False) + assert _DEFAULT_WORKSPACE_BUFFER_SIZE == envs.VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE + + +def test_env_unset_returns_computed(monkeypatch): + monkeypatch.delenv("VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE", raising=False) + required = _required_workspace_bytes( + dcp_world_size=_REPORTER_DCP, + num_heads_per_rank=_REPORTER_HEADS_PER_RANK, + max_num_batched_tokens=_REPORTER_MAX_TOKENS, + ) + assert required == compute_trtllm_sparse_mla_workspace_bytes( + _DEFAULT_WORKSPACE_BUFFER_SIZE, + _REPORTER_DCP, + _REPORTER_HEADS_PER_RANK, + _REPORTER_MAX_TOKENS, + ) + + +def test_env_override_below_computed_is_respected_with_warning(monkeypatch): + override = 100 * 1024 * 1024 + monkeypatch.setenv("VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE", str(override)) + with patch( + "vllm.v1.attention.backends.mla.flashinfer_mla_sparse.logger" + ) as mock_logger: + required = _required_workspace_bytes( + dcp_world_size=_REPORTER_DCP, + num_heads_per_rank=_REPORTER_HEADS_PER_RANK, + max_num_batched_tokens=_REPORTER_MAX_TOKENS, + ) + assert required == override + mock_logger.warning_once.assert_called_once() + + +def test_env_override_above_computed_is_respected_without_warning(monkeypatch): + override = 8 * 1024 * 1024 * 1024 + monkeypatch.setenv("VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE", str(override)) + with patch( + "vllm.v1.attention.backends.mla.flashinfer_mla_sparse.logger" + ) as mock_logger: + required = _required_workspace_bytes( + dcp_world_size=_REPORTER_DCP, + num_heads_per_rank=_REPORTER_HEADS_PER_RANK, + max_num_batched_tokens=_REPORTER_MAX_TOKENS, + ) + assert required == override + mock_logger.warning_once.assert_not_called() diff --git a/tests/v1/attention/test_indexer_dcp_localize.py b/tests/v1/attention/test_indexer_dcp_localize.py index eab4759640a6..812bce39d889 100644 --- a/tests/v1/attention/test_indexer_dcp_localize.py +++ b/tests/v1/attention/test_indexer_dcp_localize.py @@ -307,6 +307,23 @@ def test_get_dcp_local_seq_lens_preserves_mtp_bounds_shape(): torch.testing.assert_close(actual, expected) +def test_get_dcp_local_seq_lens_reuses_device_rank_tensor(monkeypatch): + seq_lens = torch.tensor([0, 1, 7, 8, 17], dtype=torch.int32) + world = 4 + rank = 2 + rank_tensor = torch.tensor(rank, dtype=torch.int32) + expected = get_dcp_local_seq_lens(seq_lens, world, rank, 2) + + def fail_tensor_construction(*args, **kwargs): + raise AssertionError("rank tensor must be reused") + + with monkeypatch.context() as patch: + patch.setattr(torch, "tensor", fail_tensor_construction) + actual = get_dcp_local_seq_lens(seq_lens, world, rank_tensor, 2) + + torch.testing.assert_close(actual, expected) + + def test_get_dcp_local_seq_lens_must_run_after_decode_expansion(): world = 2 rank = 1 diff --git a/tests/v1/attention/test_indexer_deepseek_v4_slot_mapping.py b/tests/v1/attention/test_indexer_deepseek_v4_slot_mapping.py index 87df1e409cf3..f3b4b7f5ba24 100644 --- a/tests/v1/attention/test_indexer_deepseek_v4_slot_mapping.py +++ b/tests/v1/attention/test_indexer_deepseek_v4_slot_mapping.py @@ -179,6 +179,7 @@ def test_indexer_warmup_normalizes_zero_compress_ratios(): ), parallel_config=SimpleNamespace( decode_context_parallel_size=1, + prefill_context_parallel_size=1, cp_kv_cache_interleave_size=1, ), ) @@ -193,6 +194,37 @@ def test_indexer_warmup_normalizes_zero_compress_ratios(): } +def test_indexer_warmup_includes_pcp_normalized_dcp_key(monkeypatch): + monkeypatch.setattr( + "vllm.v1.attention.backends.mla.indexer.get_dcp_group", + lambda: SimpleNamespace(rank_in_group=2), + ) + config = SimpleNamespace( + scheduler_config=SimpleNamespace(max_num_batched_tokens=8), + model_config=SimpleNamespace( + hf_text_config=SimpleNamespace(compress_ratios=[32], index_kpool=32) + ), + parallel_config=SimpleNamespace( + decode_context_parallel_size=4, + prefill_context_parallel_size=4, + cp_kv_cache_interleave_size=1, + ), + ) + + keys = BuildPrefillChunkMetadataKernel().get_warmup_keys(config) + + # Triton's compile key normalizes generic i32 values to 2 and divisible + # i32 values (including zero) to 16. + assert {(key.dcp_rank, key.dcp_world) for key in keys} == {(2, 2), (16, 1)} + assert { + ( + key.input_variant.is_aligned("uncompressed_seq_lens"), + key.input_variant.is_aligned("cu_compressed_seq_lens"), + ) + for key in keys + } == {(False, False), (False, True), (True, False), (True, True)} + + def test_compressed_slot_mapping_warmup_includes_index_kpool(): config = SimpleNamespace( cache_config=SimpleNamespace(block_size=256), diff --git a/vllm/v1/attention/backends/mla/flashinfer_mla_sparse.py b/vllm/v1/attention/backends/mla/flashinfer_mla_sparse.py index ae2e51284015..ba10a2e91cea 100644 --- a/vllm/v1/attention/backends/mla/flashinfer_mla_sparse.py +++ b/vllm/v1/attention/backends/mla/flashinfer_mla_sparse.py @@ -113,6 +113,16 @@ def supports_combination( from vllm.config import get_current_vllm_config vllm_config = get_current_vllm_config() + parallel_config = vllm_config.parallel_config + if ( + parallel_config.prefill_context_parallel_size > 1 + and parallel_config.decode_context_parallel_size > 1 + ): + return ( + "FLASHINFER_MLA_SPARSE does not support combined PCP+DCP; " + "use FLASHMLA_SPARSE, which gathers each DCP KV shard before " + "running the rank-local PCP prefill queries" + ) if kv_cache_dtype == "fp8_ds_mla": return ( "FLASHINFER_MLA_SPARSE SM10 does not support fp8_ds_mla kv-cache dtype" @@ -265,6 +275,31 @@ class FlashInferMLASparseTRTLLMMetadataBuilder(FlashInferMLASparseMetadataBuilde _cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.ALWAYS hisparse_supports_multi_token_decode: ClassVar[bool] = True + def __init__( + self, + kv_cache_spec: AttentionSpec, + layer_names: list[str], + vllm_config: VllmConfig, + device: torch.device, + ) -> None: + super().__init__(kv_cache_spec, layer_names, vllm_config, device) + + # Under DCP the workspace must hold the trtllm-gen softmax-stats slab. + # The buffer address is baked into CUDA graphs, so it has to reach its + # final size here, before the first forward/capture (no lazy regrow). + if self.dcp_world_size > 1: + num_q_heads = vllm_config.model_config.get_num_attention_heads( + vllm_config.parallel_config + ) + _get_workspace_buffer( + device, + _required_workspace_bytes( + self.dcp_world_size, + num_q_heads, + vllm_config.scheduler_config.max_num_batched_tokens, + ), + ) + def _build_req_id_per_token( self, common_attn_metadata: "CommonAttentionMetadata", @@ -275,14 +310,98 @@ def _build_req_id_per_token( # Global workspace buffer (lazily initialized) _fi_sparse_workspace: torch.Tensor | None = None +# trtllm-gen carves a softmax-stats slab from the workspace whenever LSE is +# requested (FlashInfer csrc/trtllm_fmha_kernel_launcher.cu, unchanged from +# v0.6.14 through current main): +# sizeof(float2) * num_qo_heads * batch_size * round_up(max_q_len, 256) +# + 1 MiB guard +# forward_mqa always passes q_len == 1, so each (head, token) pair costs +# round_up(1, 256) == 256 slots. +_TRTLLM_GEN_SOFTMAX_STAT_BYTES = 8 # sizeof(float2) +_TRTLLM_GEN_SOFTMAX_SLOTS_PER_TOKEN = 256 # round_up(q_len=1, 256) +_TRTLLM_GEN_SOFTMAX_GUARD_BYTES = 1024 * 1024 + +# Keep in sync with the VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE default in +# vllm/envs.py. +_DEFAULT_WORKSPACE_BUFFER_SIZE = 394 * 1024 * 1024 + + +def compute_trtllm_sparse_mla_workspace_bytes( + base_workspace_bytes: int, + dcp_world_size: int, + num_heads_per_rank: int, + max_num_batched_tokens: int, +) -> int: + """Workspace bytes needed by the trtllm-gen sparse MLA decode kernel. + + Under DCP the query is all-gathered across the DCP group in the head dim + and the kernel is asked for LSE, so trtllm-gen carves the softmax-stats + slab described above from the workspace before its (batch-independent) + counter and scratch regions. ``base_workspace_bytes`` must stay available + for those regions, so the slab is added on top of it (see #50781). + + Without DCP no LSE is requested, no slab is carved, and the base size is + returned unchanged. + """ + if dcp_world_size <= 1: + return base_workspace_bytes + softmax_bytes = ( + _TRTLLM_GEN_SOFTMAX_STAT_BYTES + * (num_heads_per_rank * dcp_world_size) + * max_num_batched_tokens + * _TRTLLM_GEN_SOFTMAX_SLOTS_PER_TOKEN + + _TRTLLM_GEN_SOFTMAX_GUARD_BYTES + ) + return base_workspace_bytes + softmax_bytes + + +def _required_workspace_bytes( + dcp_world_size: int, + num_heads_per_rank: int, + max_num_batched_tokens: int, +) -> int: + """Resolve the workspace size, honoring an explicit env override.""" + computed = compute_trtllm_sparse_mla_workspace_bytes( + _DEFAULT_WORKSPACE_BUFFER_SIZE, + dcp_world_size, + num_heads_per_rank, + max_num_batched_tokens, + ) + if not envs.is_set("VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE"): + return computed + env_bytes = envs.VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE + if env_bytes < computed: + logger.warning_once( + "VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE=%d is below the %d bytes " + "computed for the sparse MLA workspace with dcp_world_size=%d, " + "%d heads/rank and max_num_batched_tokens=%d. Respecting the " + "override, but the trtllm-gen kernel may crash with a workspace " + "overflow; set VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE=%d or unset " + "it to use the computed size.", + env_bytes, + computed, + dcp_world_size, + num_heads_per_rank, + max_num_batched_tokens, + computed, + ) + return env_bytes + -def _get_workspace_buffer(device: torch.device) -> torch.Tensor: +def _get_workspace_buffer( + device: torch.device, min_bytes: int | None = None +) -> torch.Tensor: global _fi_sparse_workspace - if _fi_sparse_workspace is None: + required = ( + min_bytes + if min_bytes is not None + else envs.VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE + ) + if _fi_sparse_workspace is None or _fi_sparse_workspace.numel() < required: # FlashInfer's CuteDSL MLA-decode tactic requires an int8 workspace; # the trtllm-gen path views it as uint8, so int8 is safe for all backends. _fi_sparse_workspace = torch.zeros( - envs.VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE, + required, dtype=torch.int8, device=device, ) diff --git a/vllm/v1/attention/backends/mla/indexer.py b/vllm/v1/attention/backends/mla/indexer.py index 2321cf8d13d4..28bd450da1ca 100644 --- a/vllm/v1/attention/backends/mla/indexer.py +++ b/vllm/v1/attention/backends/mla/indexer.py @@ -10,7 +10,7 @@ from vllm.config import VllmConfig from vllm.distributed import get_dcp_group, get_pcp_group from vllm.logger import init_logger -from vllm.model_executor.warmup.jit_warmup import kernel_launcher +from vllm.model_executor.warmup.jit_warmup import kernel_launcher, zip_inputs from vllm.model_executor.warmup.jit_warmup_triton_helper import ( LaunchSpec, TritonPointerInputVariant, @@ -493,6 +493,14 @@ def get_warmup_keys(self, vllm_config: VllmConfig) -> list[CompileKey]: dcp_world = parallel_config.decode_context_parallel_size dcp_interleave = parallel_config.cp_kv_cache_interleave_size dcp_rank = get_dcp_group().rank_in_group if dcp_world > 1 else 0 + dcp_cases = [dict(DCP_RANK=dcp_rank, DCP_WORLD=dcp_world)] + if parallel_config.prefill_context_parallel_size > 1 and dcp_world > 1: + # PCP+DCP prefill constructs a global PCP chunk plan and therefore + # dispatches this kernel with already-localized row starts. The + # runtime key is normalized to rank 0/world 1 in that path; warm it + # explicitly so one PCP rank cannot enter a collective while + # another rank is still compiling the first real inference batch. + dcp_cases.append(dict(DCP_RANK=0, DCP_WORLD=1)) compress_ratios = tuple( dict.fromkeys( max(1, int(ratio)) @@ -506,17 +514,33 @@ def get_warmup_keys(self, vllm_config: VllmConfig) -> list[CompileKey]: if index_kpool and index_kpool > 1 and index_kpool not in compress_ratios: compress_ratios = compress_ratios + (index_kpool,) return self._trace_dispatch(self.dispatch)( + zip_inputs(*dcp_cases), # Cover Triton's divisible, exact-one, and generic i32 classes. query_slice_start=(0, 1, 2), query_slice_stop=(1, 2 * max_tokens - 1, 2 * max_tokens), - DCP_RANK=dcp_rank, - DCP_WORLD=dcp_world, DCP_INTERLEAVE=dcp_interleave, BLOCK_SIZE=self.BLOCK_SIZE, COMPRESS_RATIO=list(compress_ratios), + # PCP's global cumulative lengths are the second row of one packed + # allocation, so their pointer is not always 16-byte aligned. + # Prefill chunking can independently unalign uncompressed lengths. input_variant=( - TritonPointerInputVariant.from_alignment(uncompressed_seq_lens=True), - TritonPointerInputVariant.from_alignment(uncompressed_seq_lens=False), + TritonPointerInputVariant.from_alignment( + uncompressed_seq_lens=True, + cu_compressed_seq_lens=True, + ), + TritonPointerInputVariant.from_alignment( + uncompressed_seq_lens=True, + cu_compressed_seq_lens=False, + ), + TritonPointerInputVariant.from_alignment( + uncompressed_seq_lens=False, + cu_compressed_seq_lens=True, + ), + TritonPointerInputVariant.from_alignment( + uncompressed_seq_lens=False, + cu_compressed_seq_lens=False, + ), ), ) @@ -527,7 +551,9 @@ def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]: uncompressed_seq_lens=compile_key.input_variant.pointer( "uncompressed_seq_lens", torch.int32 ), - cu_compressed_seq_lens=int32_ptr, + cu_compressed_seq_lens=compile_key.input_variant.pointer( + "cu_compressed_seq_lens", torch.int32 + ), row_start_cu_compressed_seq_lens=int32_ptr, token_to_seq=int32_ptr, cu_compressed_seq_len_ks=int32_ptr, @@ -836,6 +862,12 @@ def __init__(self, *args, block_table_width: int, **kwargs) -> None: dtype=torch.int32, device=self.device, ) + # Materialize the rank on device during builder initialization. Creating + # this scalar in build() would introduce a GPU<->CPU sync in the decode + # hot path. + self.dcp_rank_tensor = torch.tensor( + self.dcp_rank, dtype=torch.int32, device=self.device + ) self.expanded_block_table_buffer = torch.zeros( (scheduler_config.max_num_batched_tokens, block_table_width), dtype=torch.int32, @@ -889,7 +921,7 @@ def _dcp_localize_decode_seq_lens( local_seq_lens = get_dcp_local_seq_lens( seq_lens, self.dcp_world_size, - self.dcp_rank, + self.dcp_rank_tensor, self.cp_kv_cache_interleave_size, ) if seq_lens_is_buffer_view: diff --git a/vllm/v1/attention/backends/utils.py b/vllm/v1/attention/backends/utils.py index a3adf70d2588..e656343a9189 100644 --- a/vllm/v1/attention/backends/utils.py +++ b/vllm/v1/attention/backends/utils.py @@ -1094,7 +1094,7 @@ def compute_causal_conv1d_metadata( def get_dcp_local_seq_lens( seq_lens: torch.Tensor, dcp_size: int = 1, - dcp_rank: int | None = None, + dcp_rank: int | torch.Tensor | None = None, cp_kv_cache_interleave_size: int = 1, ) -> torch.Tensor: """While using dcp, kv_cache size stored on each rank may be different, @@ -1112,6 +1112,12 @@ def get_dcp_local_seq_lens( dcp_size, ) seq_lens_tiled = seq_lens_i32.unsqueeze(-1) + elif isinstance(dcp_rank, torch.Tensor): + assert dcp_rank.dtype == torch.int32 + assert dcp_rank.device == seq_lens.device + assert dcp_rank.numel() == 1 + rank_offsets = dcp_rank + seq_lens_tiled = seq_lens_i32 else: rank_offsets = torch.tensor(dcp_rank, dtype=torch.int32, device=seq_lens.device) seq_lens_tiled = seq_lens_i32 diff --git a/vllm/v1/attention/ops/dcp.py b/vllm/v1/attention/ops/dcp.py index bd20c9020d3e..58ac1e14a09d 100644 --- a/vllm/v1/attention/ops/dcp.py +++ b/vllm/v1/attention/ops/dcp.py @@ -77,6 +77,13 @@ def mask_dcp_empty_shards_( return assert seq_lens is not None and query_start_loc is not None + # A DCP rank can receive no local sequences during CUDA graph warmup even + # though the padded LSE buffer still has rows. In that case every row is an + # empty shard; avoid indexing the empty seq_lens tensor below. + if seq_lens.shape[0] == 0: + lse.fill_(float("-inf")) + return + row_indices = torch.arange( lse.shape[0], device=lse.device, dtype=query_start_loc.dtype )