Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 26 additions & 0 deletions tests/v1/attention/test_sparse_mla_backends.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
B12xMLASparseBackend,
B12xMLASparseImpl,
B12xMLASparseMetadataBuilder,
_global_causal_lens_for_ckv_gather,
)
from vllm.v1.attention.backends.mla.flashinfer_mla_sparse import (
FlashInferMLASparseTRTLLMBackend,
Expand Down Expand Up @@ -73,6 +74,31 @@
DEVICE_TYPE = current_platform.device_type


@pytest.mark.parametrize(
("global_seq_lens", "query_start_loc", "req_ids", "expected"),
[
([8], [0, 4], [0, 0, 0, 0], [5, 6, 7, 8]),
([5, 10], [0, 2, 5], [0, 0, 1, 1, 1], [4, 5, 8, 9, 10]),
],
)
def test_ckv_gather_uses_global_per_token_causal_lens(
global_seq_lens: list[int],
query_start_loc: list[int],
req_ids: list[int],
expected: list[int],
) -> None:
device = torch.device(DEVICE_TYPE)
result = _global_causal_lens_for_ckv_gather(
torch.tensor(global_seq_lens, dtype=torch.int32, device=device),
torch.tensor(query_start_loc, dtype=torch.int32, device=device),
torch.tensor(req_ids, dtype=torch.int32, device=device),
len(req_ids),
)

assert result.dtype == torch.int32
assert result.tolist() == expected


@pytest.mark.parametrize(
("batch_spec", "local_seq_lens", "expected_req_ids", "expected_token_lens"),
[
Expand Down
59 changes: 59 additions & 0 deletions vllm/distributed/parallel_state.py
Original file line number Diff line number Diff line change
Expand Up @@ -1415,6 +1415,22 @@ def get_dcp_group() -> GroupCoordinator:
return _DCP


_QUERY_SPLIT: GroupCoordinator | None = None


def get_query_split_group() -> GroupCoordinator:
assert _QUERY_SPLIT is not None, "query split group is not initialized"
return _QUERY_SPLIT


_DCP_CKV_PREFETCH: GroupCoordinator | None = None


def get_dcp_ckv_prefetch_group() -> GroupCoordinator:
assert _DCP_CKV_PREFETCH is not None, "DCP ckv prefetch group is not initialized"
return _DCP_CKV_PREFETCH


_PP: GroupCoordinator | None = None


Expand Down Expand Up @@ -1878,6 +1894,39 @@ def initialize_model_parallel(
group_name="dcp",
)

# Build the query-split groups for the indexer query split (Fix A).
# Ranks sharing the same dcp_rank (position within their DCP group)
# form a query-split group. At TP=8/DCP=2 the DCP groups are
# {0,1},{2,3},{4,5},{6,7} and the query-split groups are
# {0,2,4,6} (dcp_rank=0) and {1,3,5,7} (dcp_rank=1).
global _QUERY_SPLIT
assert _QUERY_SPLIT is None, "query split group is already initialized"
if decode_context_model_parallel_size > 1 and envs.VLLM_DCP_QUERY_SPLIT:
query_split_ranks: list[list[int]] = []
for dcp_rank_idx in range(decode_context_model_parallel_size):
query_split_ranks.append([grp[dcp_rank_idx] for grp in group_ranks])
_QUERY_SPLIT = init_model_parallel_group(
Comment on lines +1904 to +1908

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🩺 Stability & Availability | 🔴 Critical | ⚡ Quick win

Keep query-split groups inside each TP cohort.

group_ranks contains DCP groups from every DP/PP/PCP cohort. Collecting each DCP position across the entire list merges unrelated replicas. For DP2 with TP8/DCP8, this even creates two-rank query groups instead of the intended singleton no-op, risking incorrect outputs or collective hangs.

Proposed fix
-        query_split_ranks: list[list[int]] = []
-        for dcp_rank_idx in range(decode_context_model_parallel_size):
-            query_split_ranks.append([grp[dcp_rank_idx] for grp in group_ranks])
+        query_split_source = local_all_ranks if enable_elastic_ep else all_ranks
+        dcp_groups_per_tp = (
+            tensor_model_parallel_size // decode_context_model_parallel_size
+        )
+        query_split_ranks = (
+            query_split_source.reshape(
+                -1,
+                dcp_groups_per_tp,
+                decode_context_model_parallel_size,
+            )
+            .transpose(1, 2)
+            .reshape(-1, dcp_groups_per_tp)
+            .tolist()
+        )
📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
if decode_context_model_parallel_size > 1 and envs.VLLM_DCP_QUERY_SPLIT:
query_split_ranks: list[list[int]] = []
for dcp_rank_idx in range(decode_context_model_parallel_size):
query_split_ranks.append([grp[dcp_rank_idx] for grp in group_ranks])
_QUERY_SPLIT = init_model_parallel_group(
if decode_context_model_parallel_size > 1 and envs.VLLM_DCP_QUERY_SPLIT:
query_split_source = local_all_ranks if enable_elastic_ep else all_ranks
dcp_groups_per_tp = (
tensor_model_parallel_size // decode_context_model_parallel_size
)
query_split_ranks = (
query_split_source.reshape(
-1,
dcp_groups_per_tp,
decode_context_model_parallel_size,
)
.transpose(1, 2)
.reshape(-1, dcp_groups_per_tp)
.tolist()
)
_QUERY_SPLIT = init_model_parallel_group(
🤖 Prompt for AI Agents
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.

In `@vllm/distributed/parallel_state.py` around lines 1904 - 1908, Update the
query-split construction in the decode-context parallel initialization block so
ranks are grouped only within each TP cohort, not across all entries in
group_ranks. Partition or iterate group_ranks by the appropriate TP cohort
boundaries, then collect each DCP position within each cohort; preserve
singleton groups as the intended no-op for configurations such as DP2 with
TP8/DCP8.

query_split_ranks,
get_world_group().local_rank,
backend,
group_name="query_split",
)

# A dedicated communicator over the DCP ranks for the transient ckv
# prefetch gather (Fix B). The prefetch runs on a side stream and would
# otherwise share the DCP communicator with the indexer's DCP top-k
# merge on the default stream; concurrent collectives on one NCCL
# communicator from two streams is unsupported. Same ranks as ``_DCP``.
global _DCP_CKV_PREFETCH
assert _DCP_CKV_PREFETCH is None, "DCP ckv prefetch group is already initialized"
if decode_context_model_parallel_size > 1 and envs.VLLM_B12X_MLA_CKV_GATHER:
_DCP_CKV_PREFETCH = init_model_parallel_group(
group_ranks,
get_world_group().local_rank,
backend,
group_name="dcp_ckv_prefetch",
)

global _PCP
assert _PCP is None, "prefill context parallel group is already initialized"
group_ranks = (
Expand Down Expand Up @@ -2103,6 +2152,16 @@ def destroy_model_parallel():
_DCP.destroy()
_DCP = None

global _QUERY_SPLIT
if _QUERY_SPLIT:
_QUERY_SPLIT.destroy()
_QUERY_SPLIT = None

global _DCP_CKV_PREFETCH
if _DCP_CKV_PREFETCH:
_DCP_CKV_PREFETCH.destroy()
_DCP_CKV_PREFETCH = None

global _PCP
if _PCP:
_PCP.destroy()
Expand Down
16 changes: 16 additions & 0 deletions vllm/envs.py
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,10 @@
VLLM_DCP_A2A_LARGE_BACKEND: Literal["ag_rs", "a2a"] = "ag_rs"
VLLM_DCP_SHARD_DRAFT: str | None = None
VLLM_DCP_GLOBAL_TOPK: bool = True
VLLM_DCP_QUERY_SPLIT: bool = False
VLLM_B12X_MLA_CKV_GATHER: bool = False
VLLM_B12X_MLA_CKV_GATHER_MIN_TOKENS: int = 16
VLLM_B12X_MLA_CKV_GATHER_MAX_TOKENS: int = 524288
VLLM_MINIMAX_M3_ENABLE_TORCH_COMPILE: bool = False
VLLM_B12X_CUDAGRAPH_PIECEWISE_PREWARM: bool = False
VLLM_B12X_MOE_FORCE_MODELOPT_PREP: bool = False
Expand Down Expand Up @@ -1119,6 +1123,18 @@ def _resolve_rust_frontend_path() -> str | None:
"VLLM_DCP_GLOBAL_TOPK": lambda: (
os.getenv("VLLM_DCP_GLOBAL_TOPK", "1").lower() in ("1", "true", "yes", "on")
),
"VLLM_DCP_QUERY_SPLIT": lambda: (
os.getenv("VLLM_DCP_QUERY_SPLIT", "0").lower() in ("1", "true", "yes", "on")
),
"VLLM_B12X_MLA_CKV_GATHER": lambda: (
os.getenv("VLLM_B12X_MLA_CKV_GATHER", "0").lower() in ("1", "true", "yes", "on")
),
"VLLM_B12X_MLA_CKV_GATHER_MIN_TOKENS": lambda: int(
os.getenv("VLLM_B12X_MLA_CKV_GATHER_MIN_TOKENS", "16")
),
"VLLM_B12X_MLA_CKV_GATHER_MAX_TOKENS": lambda: int(
os.getenv("VLLM_B12X_MLA_CKV_GATHER_MAX_TOKENS", "524288")
),
# Diagnostic flag retained for local experiments. MiniMax M3 compile is
# fail-closed in the model until the no-break path is validated.
"VLLM_MINIMAX_M3_ENABLE_TORCH_COMPILE": lambda: bool(
Expand Down
63 changes: 38 additions & 25 deletions vllm/model_executor/layers/attention/mla_attention.py
Original file line number Diff line number Diff line change
Expand Up @@ -187,8 +187,8 @@
return curr_o @ W_O
"""

import os
import functools
import os
from abc import abstractmethod
from dataclasses import dataclass
from enum import Enum
Expand Down Expand Up @@ -943,13 +943,22 @@ def forward_impl(
dcp_use_a2a = False
project_before_merge = False
workspace_gather_used = False
ckv_gather_used = False
if self.impl.dcp_world_size > 1:
if not self.impl.can_return_lse_for_decode:
raise NotImplementedError(
f"{type(self.impl).__name__} cannot use DCP because it "
"does not return decode softmax LSE."
)
self.impl.need_to_return_lse_for_decode = True
ckv_gather_selector = getattr(
self.impl, "dcp_prefill_ckv_gather_eligible", None
)
ckv_gather_used = bool(
callable(ckv_gather_selector)
and ckv_gather_selector(attn_metadata, num_mqa_tokens)
)
if not ckv_gather_used:
if not self.impl.can_return_lse_for_decode:
raise NotImplementedError(
f"{type(self.impl).__name__} cannot use DCP because it "
"does not return decode softmax LSE."
)
self.impl.need_to_return_lse_for_decode = True
if (
fp8_attention
and isinstance(mqa_q, torch.Tensor)
Expand All @@ -974,7 +983,9 @@ def forward_impl(
)
# The project-before path currently targets eager AG/RS
# prefill. A2A retains its established merge-then-project path.
project_before_merge = use_ondemand_w_uv and not dcp_use_a2a
project_before_merge = (
use_ondemand_w_uv and not dcp_use_a2a and not ckv_gather_used
)
workspace_gather_eligible = _can_use_b12x_dcp_prefill_workspace(
enabled=envs.VLLM_B12X_MLA_DCP_GATHER_IN_WORKSPACE,
project_before_merge=project_before_merge,
Expand All @@ -988,11 +999,18 @@ def forward_impl(
backend_name=self.attn_backend.get_name(),
is_capturing=torch.cuda.is_current_stream_capturing(),
)
if isinstance(mqa_q, tuple) and not workspace_gather_eligible:
# Ordinary communicators require one contiguous query.
if (
isinstance(mqa_q, tuple)
and not workspace_gather_eligible
and not ckv_gather_used
):
mqa_q = torch.cat(mqa_q, dim=-1)
# mqa_q do allgather in head dim.
if dcp_use_b12x:
if ckv_gather_used:
logger.info_once(
"Keeping local query heads for transient full-CKV "
"B12X sparse MLA prefill"
)
elif dcp_use_b12x:
mqa_q = dcp_b12x_all_gather_heads(
mqa_q,
get_dcp_group(),
Expand Down Expand Up @@ -1027,10 +1045,14 @@ def forward_impl(
# call decode attn
if not is_sparse_impl:
assert attn_metadata.decode is not None
if ckv_gather_used:
ckv_setter = getattr(self.impl, "set_ckv_current_chunk_kv", None)
if callable(ckv_setter):
ckv_setter(k_c_normed, k_pe)
attn_out, lse = self.impl.forward_mqa(mqa_q, kv_cache, attn_metadata, self) # type: ignore[attr-defined]

# correct dcp attn_out with lse.
if self.impl.dcp_world_size > 1:
if self.impl.dcp_world_size > 1 and not ckv_gather_used:
if lse is None:
raise RuntimeError(
f"{type(self.impl).__name__} did not return decode "
Expand Down Expand Up @@ -1355,24 +1377,17 @@ def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec:
and num_hidden_layers is not None
and layer_id >= int(num_hidden_layers)
)
model_type = getattr(
vllm_config.model_config.hf_config, "model_type", None
)
model_type = getattr(vllm_config.model_config.hf_config, "model_type", None)
speculative_config = getattr(vllm_config, "speculative_config", None)
target_model_config = getattr(
speculative_config, "target_model_config", None
)
target_model_config = getattr(speculative_config, "target_model_config", None)
target_model_type = (
getattr(target_model_config.hf_config, "model_type", None)
if target_model_config is not None
else None
)
glm_model_or_mtp = bool(
model_type == "glm_moe_dsa"
or (
model_type == "deepseek_mtp"
and target_model_type == "glm_moe_dsa"
)
or (model_type == "deepseek_mtp" and target_model_type == "glm_moe_dsa")
)
glm_fp8_rope = bool(
os.environ.get("KV_FP8_ROPE", "0") == "1"
Expand Down Expand Up @@ -1490,8 +1505,6 @@ def _v_up_proj_bmm_chunked(
self._v_up_proj_bmm(x[start:end], out[start:end], w_uv)




def unified_mla_kv_cache_update(
kv_c_normed: torch.Tensor,
k_pe: torch.Tensor,
Expand Down
Loading
Loading