Skip to content

feat(Qwen3.5): hybrid linear attention support PP+PD #19254

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
zhangxiaolei123456 wants to merge 22 commits into from
Closed

feat(Qwen3.5): hybrid linear attention support PP+PD #19254

Changes from 1 commit
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
4ad99d7
Update qwen3_5.py
zhangxiaolei123456 Feb 24, 2026
109a473
Update qwen3_5.py
zhangxiaolei123456 Feb 24, 2026
f32e00f
Update qwen3_5.py
zhangxiaolei123456 Feb 24, 2026
9b4a7e0
Update qwen3_5.py
zhangxiaolei123456 Feb 24, 2026
ecb9d4b
Update qwen3_5.py
zhangxiaolei123456 Feb 24, 2026
23c41c1
Merge branch 'main' into main_qwen3.5_0224_fix_pp
zhangxiaolei123456 Feb 26, 2026
7481e12
Merge branch 'main' into main_qwen3.5_0224_fix_pp
zhangxiaolei123456 Feb 28, 2026
bdce5d1
Update qwen3_5.py
zhangxiaolei123456 Feb 28, 2026
0321307
Update model_runner_kv_cache_mixin.py
zhangxiaolei123456 Feb 28, 2026
8f6e92d
Update memory_pool.py
zhangxiaolei123456 Feb 28, 2026
16f58f3
Update memory_pool.py
zhangxiaolei123456 Feb 28, 2026
7b30482
Update prefill.py
zhangxiaolei123456 Feb 28, 2026
ba908f4
Update decode.py
zhangxiaolei123456 Feb 28, 2026
b20f236
Update model_runner_kv_cache_mixin.py
zhangxiaolei123456 Feb 28, 2026
e3483f0
Update model_runner_kv_cache_mixin.py
zhangxiaolei123456 Feb 28, 2026
cfad179
Update conn.py
zhangxiaolei123456 Feb 28, 2026
c8f6059
Update conn.py
zhangxiaolei123456 Feb 28, 2026
4822b6a
Create E=512,N=512,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=…
zhangxiaolei123456 Feb 28, 2026
58712ed
Update memory_pool.py
zhangxiaolei123456 Feb 28, 2026
b3675e2
Update conn.py
zhangxiaolei123456 Feb 28, 2026
1acc611
Update model_runner_kv_cache_mixin.py
zhangxiaolei123456 Feb 28, 2026
6ab83da
Update qwen3_vl.py
zhangxiaolei123456 Feb 28, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 34 additions & 1 deletion python/sglang/srt/models/qwen3_5.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,7 @@
from sglang.srt.models.qwen3_vl import Qwen3VLForConditionalGeneration

# Utils
from sglang.srt.layers.utils import get_layer_id
from sglang.srt.utils import add_prefix, is_cuda, is_npu, make_layers, set_weight_attrs
from sglang.srt.utils.hf_transformers_utils import get_processor

Expand Down Expand Up @@ -672,6 +673,8 @@ def __init__(
org_num_embeddings=config.vocab_size,
enable_tp=not is_dp_attention_enabled(),
)
else:
self.embed_tokens = PPMissingLayer()

# Decoder layers
def get_layer(idx: int, prefix: str):
Expand All @@ -689,9 +692,11 @@ def get_layer(idx: int, prefix: str):
alt_stream=alt_stream,
)

self.layers = make_layers(
self.layers, self.start_layer, self.end_layer= make_layers(
config.num_hidden_layers,
get_layer,
pp_rank=self.pp_group.rank_in_group,
pp_size=self.pp_group.world_size,
prefix=f"{prefix}.layers",
)

Expand Down Expand Up @@ -789,6 +794,13 @@ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
name = name.replace(r"model.language_model.", r"model.")
if ".self_attn." in name:
name = name.replace(".self_attn", "")
layer_id = get_layer_id(name)
if (
layer_id is not None
and hasattr(self, "start_layer")
and (layer_id < self.start_layer or layer_id >= self.end_layer)
):
continue
Comment on lines +807 to +813

@gemini-code-assist gemini-code-assist Bot Feb 24, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

This logic to skip loading weights for layers not on the current pipeline parallel rank is duplicated in three other load_weights methods in this file (at lines 925, 1091, and 1241). To improve maintainability and reduce code duplication, consider extracting this logic into a single helper function.


for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
Expand Down Expand Up @@ -910,6 +922,13 @@ def load_fused_expert_weights(
name = name.replace(r"model.language_model.", r"model.")
if ".self_attn." in name:
name = name.replace(".self_attn", "")
layer_id = get_layer_id(name)
if (
layer_id is not None
and hasattr(self, "start_layer")
and (layer_id < self.start_layer or layer_id >= self.end_layer)
):
continue

for param_name, weight_name, shard_id in stacked_params_mapping:
if "experts.gate_up_proj" in name or "experts.down_proj" in name:
Expand Down Expand Up @@ -1069,6 +1088,13 @@ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
name = name.replace(r"model.language_model.", r"model.")
if ".self_attn." in name:
name = name.replace(".self_attn", "")
layer_id = get_layer_id(name)
if (
layer_id is not None
and hasattr(self.model, "start_layer")
and (layer_id < self.model.start_layer or layer_id >= self.model.end_layer)
):
continue

for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
Expand Down Expand Up @@ -1212,6 +1238,13 @@ def load_fused_expert_weights(
name = name.replace(r"model.language_model.", r"model.")
if ".self_attn." in name:
name = name.replace(".self_attn", "")
layer_id = get_layer_id(name)
if (
layer_id is not None
and hasattr(self.model, "start_layer")
and (layer_id < self.model.start_layer or layer_id >= self.model.end_layer)
):
continue

for param_name, weight_name, shard_id in stacked_params_mapping:
if name.endswith("experts.gate_up_proj") or name.endswith(
Expand Down
Loading