Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
28ead89
tp-lm-head-all-to-all, replacing the allgather + scatter under dp-att…
SYChen123 Jul 24, 2026
f6bb877
Merge branch 'main' into lm-head-opt
JustinTong0323 Jul 28, 2026
e33bed9
Pass flattened input tensor to all_to_all_single according to the pyn…
SYChen123 Jul 29, 2026
b55f4a0
Use triton attention backend in the test.
SYChen123 Jul 29, 2026
80d4aea
Merge branch 'main' into lm-head-opt
SYChen123 Jul 29, 2026
162a4c3
Merge upstream/main into lm-head-opt
JustinTong0323 Aug 1, 2026
bd42543
Merge branch 'main' into lm-head-opt
SYChen123 Aug 2, 2026
6f64241
fix lint
SYChen123 Aug 2, 2026
530859c
Merge branch 'main' into lm-head-opt
JustinTong0323 Aug 4, 2026
ccc758a
fix server args namespace in logit_processor which causes test_global…
SYChen123 Aug 4, 2026
9845be9
make default tp-lmhead-all2all for decode node + pure DP parallel + d…
SYChen123 Aug 6, 2026
91e689a
Add resolvable field in test_model_overrides for enable_tp_lm_head_al…
SYChen123 Aug 6, 2026
4e88f8f
Merge branch 'main' into lm-head-opt
SYChen123 Aug 6, 2026
dd386ec
Merge branch 'main' into lm-head-opt
SYChen123 Aug 7, 2026
55048c8
Merge remote-tracking branch 'origin/main' into lm-head-opt
SYChen123 Aug 10, 2026
1ff8e27
Add runtime check to guarantee that the LMHead weight is not replicat…
SYChen123 Aug 11, 2026
68e0a86
Merge branch 'main' into lm-head-opt
SYChen123 Aug 11, 2026
f59682d
Merge branch 'main' into lm-head-opt
Fridge003 Aug 12, 2026
f0057f3
Add pre-warmup for tp-lmhead-all2all before calculating available gpu…
SYChen123 Aug 13, 2026
2e18ca7
Fix deepgemm reading incorrect device during precompilation with intr…
SYChen123 Aug 14, 2026
0e12bf5
Merge branch 'main' into lm-head-opt
SYChen123 Aug 14, 2026
2bdb9fd
Merge branch 'main' into lm-head-opt
Fridge003 Aug 17, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 18 additions & 1 deletion python/sglang/srt/arg_groups/overrides.py
Original file line number Diff line number Diff line change
Expand Up @@ -1980,11 +1980,28 @@ def _data_parallelism_defaults(view: Any) -> dict:
@register_post_process
def _dp_lm_head_validation(view: Any) -> dict:
"""Read-only validation pass: dp-attention is a prerequisite for the
dp LM head. Reads the mid-resolution values through the view."""
dp LM head and the TP LM-head all-to-all path. Reads the mid-resolution
values through the view."""
if view.enable_dp_lm_head:
assert (
view.enable_dp_attention
), "Please enable dp attention when setting enable_dp_lm_head. "
if view.enable_tp_lm_head_all_to_all:
assert view.enable_dp_attention, (
"Please enable dp attention when setting " "enable_tp_lm_head_all_to_all."
)
assert not view.enable_dp_lm_head, (
"--enable-tp-lm-head-all-to-all uses a TP-sharded LM head and is "
"incompatible with --enable-dp-lm-head."
)
assert view.tp_size == view.dp_size, (
"--enable-tp-lm-head-all-to-all currently requires tp_size == "
f"dp_size, got tp_size={view.tp_size}, dp_size={view.dp_size}."
)
assert view.attn_cp_size == 1, (
"--enable-tp-lm-head-all-to-all currently requires "
f"attn_cp_size == 1, got {view.attn_cp_size}."
)
return {}


Expand Down
74 changes: 71 additions & 3 deletions python/sglang/srt/layers/logits_processor.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
from sglang.kernels.ops.activation.softcap import (
softcap_inplace_logits as fused_softcap,
)
from sglang.srt.distributed import get_tp_group
from sglang.srt.distributed.device_communicators import triton_symm_mem_ag
from sglang.srt.layers.dp_attention import (
DpPaddingMode,
Expand Down Expand Up @@ -67,6 +68,26 @@
}


def _reassemble_tp_lm_head_all_to_all_output(
all_to_all_output: torch.Tensor, tp_size: int
) -> torch.Tensor:
"""Convert source-major all-to-all output to row-major full-vocab logits.

Each source TP rank contributes ``[local_rows, vocab_shard]`` for this
destination DP rank. ``all_to_all_single`` concatenates those contributions
along dim 0, while the sampler expects the vocab shards concatenated along
dim 1.
"""
assert all_to_all_output.shape[0] % tp_size == 0
local_rows = all_to_all_output.shape[0] // tp_size
vocab_shard = all_to_all_output.shape[1]
return (
all_to_all_output.view(tp_size, local_rows, vocab_shard)
.permute(1, 0, 2)
.reshape(local_rows, tp_size * vocab_shard)
)


def _has_lm_head_runtime_attrs(lm_head, attr_names: Tuple[str, ...]) -> bool:
return all(hasattr(lm_head, attr_name) for attr_name in attr_names)

Expand Down Expand Up @@ -346,6 +367,7 @@ def __init__(
self.vocab_size = config.vocab_size
self.logit_scale = logit_scale
self.use_attn_tp_group = get_server_args().enable_dp_lm_head
self.use_tp_lm_head_all_to_all = get_server_args().enable_tp_lm_head_all_to_all
self.use_fp32_lm_head = get_server_args().enable_fp32_lm_head
if self.use_attn_tp_group:
self.attn_tp_size = get_parallel().attn_tp_size
Expand Down Expand Up @@ -703,15 +725,22 @@ def _get_logits(
if self.logit_scale is not None:
logits.mul_(self.logit_scale)

used_tp_lm_head_all_to_all = False
if self.do_tensor_parallel_all_gather:
if self.use_attn_tp_group:
logits = self._gather_attn_tp_logits(logits)
elif self._can_use_tp_lm_head_all_to_all(
logits, local_hidden_states, logits_metadata
):
logits = self._tp_lm_head_all_to_all(logits)
used_tp_lm_head_all_to_all = True
else:
logits = self._logits_gatherer(logits)

logits = self._scatter_dp_attn_logits(
logits, local_hidden_states, logits_metadata
)
if not used_tp_lm_head_all_to_all:
logits = self._scatter_dp_attn_logits(
logits, local_hidden_states, logits_metadata
)

logits = self._copy_logits_to_buffer(
logits, logits_metadata, use_buffer=use_logits_buffer
Expand Down Expand Up @@ -814,6 +843,45 @@ def _gather_attn_tp_logits(self, logits: torch.Tensor) -> torch.Tensor:
)
return global_logits

def _can_use_tp_lm_head_all_to_all(
self,
logits: torch.Tensor,
local_hidden_states: torch.Tensor,
logits_metadata: LogitsMetadata,
) -> bool:
if not self.use_tp_lm_head_all_to_all:
return False

# Every participant must make the same collective choice. Decode CUDA
# graphs omit CPU counts and fill every GPU count with the same padded
# bucket size. Eager batches carry the same global CPU count list on
# every rank, so they are also safe when all entries are equal.
global_counts_cpu = logits_metadata.global_num_tokens_for_logprob_cpu
is_equal_padded_graph_layout = global_counts_cpu is None and (
logits_metadata.global_num_tokens_for_logprob_gpu is not None
)
is_equal_eager_layout = (
global_counts_cpu is not None
and len(global_counts_cpu) == get_parallel().tp_size
and len(global_counts_cpu) > 0
and all(count == global_counts_cpu[0] for count in global_counts_cpu)
)
if not (is_equal_padded_graph_layout or is_equal_eager_layout):
return False

tp_size = get_parallel().tp_size
local_rows = local_hidden_states.shape[0]
return local_rows > 0 and logits.shape[0] == local_rows * tp_size

def _tp_lm_head_all_to_all(self, logits: torch.Tensor) -> torch.Tensor:
"""Exchange only the row block owned by each destination DP rank."""
logits = logits.contiguous()
all_to_all_output = torch.empty_like(logits)
get_tp_group().all_to_all_single(all_to_all_output, logits)
Comment thread
JustinTong0323 marked this conversation as resolved.
Resolved
return _reassemble_tp_lm_head_all_to_all_output(
all_to_all_output, get_parallel().tp_size
)

def _scatter_dp_attn_logits(
self,
logits: torch.Tensor,
Expand Down
9 changes: 9 additions & 0 deletions python/sglang/srt/server_args.py
Original file line number Diff line number Diff line change
Expand Up @@ -1129,6 +1129,15 @@ class ServerArgs:
),
NS("parallel"),
] = False
enable_tp_lm_head_all_to_all: A[
bool,
"Use all-to-all instead of TP all-gather followed by DP scatter for the "
"TP-sharded LM head under DP attention. This experimental path requires "
"tp_size == dp_size, attn_cp_size == 1, and --enable-dp-lm-head to be "
"disabled. Batches without an equal padded row count fall back to the "
"existing all-gather path.",
NS("parallel"),
] = False
enable_attn_tp_input_scattered: A[
bool,
"Allow input of attention to be scattered when only using tensor parallelism, to reduce the computational load of operations such as qkv latent.",
Expand Down
Loading