Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
109 changes: 89 additions & 20 deletions atom/plugin/register.py
Original file line number Diff line number Diff line change
Expand Up @@ -410,8 +410,10 @@ def _is_dsv4_pro_runner(runner) -> bool:
def _draft_extend_graph_enabled(runner) -> bool:
if _env_flag("ATOM_SGLANG_V4_DISABLE_DRAFT_EXTEND_CG"):
return False
return _env_flag("ATOM_SGLANG_V4_ENABLE_DRAFT_EXTEND_CG") or (
_is_dsv4_nextn_runner(runner) and _is_dsv4_flash_runner(runner)
return (
_env_flag("ATOM_SGLANG_V4_ENABLE_DRAFT_EXTEND_CG")
or (_is_dsv4_nextn_runner(runner) and _is_dsv4_flash_runner(runner))
or _is_qwen4_exp_nextn_runner(runner)
)

def _target_verify_graph_enabled() -> bool:
Expand Down Expand Up @@ -523,6 +525,13 @@ def _replay(self, forward_batch):
setattr(EAGLEDraftCudaGraphRunner, draft_internal_method, _replay)

def replay(self, forward_batch):
if _is_qwen4_exp_nextn_runner(getattr(self, "model_runner", None)):
from atom.plugin.sglang.qwen4_exp_bridge import (
stage_draft_graph_mrope,
)

stage_draft_graph_mrope(self.buffers.mrope_positions, forward_batch)
return original_draft_replay(self, forward_batch)
if not _is_dsv4_or_glm52_nextn_runner(getattr(self, "model_runner", None)):
return original_draft_replay(self, forward_batch)
if _env_flag("ATOM_SGLANG_V4_DISABLE_DRAFT_CG"):
Expand Down Expand Up @@ -606,16 +615,25 @@ def can_run(self, forward_batch):
return False
return _dsv4_draft_extend_graph_layout_ok(self, forward_batch)

def replay(self, forward_batch):
# SGLang 0.5.20's ``execute(forward_batch, select_index)`` passes an
# extra argument; forward whatever the runner API passes.
def replay(self, forward_batch, *args, **kwargs):
model_runner = getattr(self, "model_runner", None)
if _is_glm52_nextn_runner(model_runner):
if not _draft_extend_graph_enabled(model_runner):
raise RuntimeError(
"GLM-5.2 draft-extend cuda graph replay was disabled"
)
return original_extend_replay(self, forward_batch)
return original_extend_replay(self, forward_batch, *args, **kwargs)
if _is_qwen4_exp_nextn_runner(model_runner):
from atom.plugin.sglang.qwen4_exp_bridge import (
stage_draft_graph_mrope,
)

stage_draft_graph_mrope(self.buffers.mrope_positions, forward_batch)
return original_extend_replay(self, forward_batch, *args, **kwargs)
if not _is_dsv4_nextn_runner(model_runner):
return original_extend_replay(self, forward_batch)
return original_extend_replay(self, forward_batch, *args, **kwargs)
if not _draft_extend_graph_enabled(getattr(self, "model_runner", None)):
raise RuntimeError(
"DSV4 draft-extend cuda graph replay was disabled after capture; "
Expand Down Expand Up @@ -651,7 +669,7 @@ def replay(self, forward_batch):
:num_tokens
],
)
out = original_extend_replay(self, forward_batch)
out = original_extend_replay(self, forward_batch, *args, **kwargs)
# EAGLE V2 consumes draft-extend logits with a fixed
# `seq * speculative_num_draft_tokens + offset` layout.
# SGLang's runner trims to the actual compact token count,
Expand Down Expand Up @@ -695,6 +713,46 @@ def replay(self, forward_batch):
setattr(EAGLEDraftExtendCudaGraphRunner, extend_replay_method, replay)
EAGLEDraftExtendCudaGraphRunner._atom_dsv4_replay_patched = True

if not getattr(EagleDraftWorker, "_atom_qwen4_draft_mrope_patched", False):
original_draft_forward = EagleDraftWorker.draft_forward

def draft_forward(self, forward_batch):
# EAGLE advances only the 1-D positions after each draft decode
# forward, and the eager / graph runners hand the model a copy of
# the batch. Advance the caller's mRoPE rows (the Flash drafter's
# RoPE input) alongside; inside a draft graph capture the add is
# recorded on the graph's mRoPE buffer.
runner = getattr(self, "draft_runner", None)
if (
not _is_qwen4_exp_nextn_runner(runner)
or forward_batch.forward_mode.is_idle()
or getattr(forward_batch, "mrope_positions", None) is None
):
return original_draft_forward(self, forward_batch)
from atom.plugin.sglang.qwen4_exp_bridge import (
advance_draft_decode_mrope,
)

shadowed = vars(runner).get("forward")
runner_forward = runner.forward

def forward(step_batch, *args, **kwargs):
out = runner_forward(step_batch, *args, **kwargs)
advance_draft_decode_mrope(forward_batch)
return out

runner.forward = forward
try:
return original_draft_forward(self, forward_batch)
finally:
if shadowed is None:
del runner.forward
else:
runner.forward = shadowed

EagleDraftWorker.draft_forward = draft_forward
EagleDraftWorker._atom_qwen4_draft_mrope_patched = True

if not getattr(EagleDraftWorker, "_atom_dsv4_draft_extend_accept_patched", False):
original_draft_extend_for_decode = EagleDraftWorker._draft_extend_for_decode

Expand Down Expand Up @@ -860,13 +918,11 @@ def init_cuda_graphs(self):
backend.get_cuda_graph_seq_len_fill_value = (
lambda value=GLM52_GRAPH_SEQ_LEN_CAPACITY: value
)
# Flash draft stays eager: mRoPE positions are [3, N], same as
# Native EagleProposer skipping mid-step graphs.
skip_all_draft_graphs = (
_env_flag("ATOM_SGLANG_V4_DISABLE_DRAFT_CG")
and not _draft_extend_graph_enabled(draft_runner)
and _is_dsv4_or_glm52_nextn_runner(draft_runner)
) or _is_qwen4_exp_nextn_runner(draft_runner)
)
original_capture_cuda_graphs = None
if skip_all_draft_graphs:
original_capture_cuda_graphs = self._capture_cuda_graphs
Expand Down Expand Up @@ -898,9 +954,13 @@ def _skip_draft_cuda_graphs(_self=self):
getattr(self, "draft_runner", None)
):
self.cuda_graph_runner = None
# The Flash drafter also takes this plugin-built draft-extend
# runner (its [3, N] mRoPE input is staged on replay by
# stage_draft_graph_mrope); SGLang's own HIP gate only admits
# aiter multi-step / DSV4 / DSA backends.
supports_plugin_graph = _is_dsv4_or_glm52_nextn_runner(
getattr(self, "draft_runner", None)
)
) or _is_qwen4_exp_nextn_runner(getattr(self, "draft_runner", None))
if (
getattr(self, "cuda_graph_runner_for_draft_extend", None)
is not None
Expand All @@ -924,24 +984,31 @@ def _skip_draft_cuda_graphs(_self=self):
or 1
),
)
for backend in (
getattr(
getattr(self, "draft_runner", None), "attn_backend", None
),
getattr(self, "draft_extend_attn_backend", None),
# DSV4 / GLM-5.2 pad rows need a long fill; Flash QSA keeps
# the backend default.
if _is_dsv4_or_glm52_nextn_runner(
getattr(self, "draft_runner", None)
):
if backend is not None and hasattr(
backend, "_cuda_graph_seq_len_fill_value"
for backend in (
getattr(
getattr(self, "draft_runner", None),
"attn_backend",
None,
),
getattr(self, "draft_extend_attn_backend", None),
):
backend._cuda_graph_seq_len_fill_value = seq_len_fill
if backend is not None and hasattr(
backend, "_cuda_graph_seq_len_fill_value"
):
backend._cuda_graph_seq_len_fill_value = seq_len_fill
self.cuda_graph_runner_for_draft_extend = (
EAGLEDraftExtendCudaGraphRunner(self)
)
elif supports_plugin_graph:
self.cuda_graph_runner_for_draft_extend = None
except Exception as exc:
logger.warning(
"Failed to enable DSV4 draft-extend cuda graph in ATOM plugin: %s",
"Failed to enable draft-extend cuda graph in ATOM plugin: %s",
exc,
)
return ret
Expand Down Expand Up @@ -1123,8 +1190,10 @@ def register_ops_to_sglang(atom_config: Config) -> None:
_keep_atom_backend_for_qwen_qsa()
_patch_sglang_dsv4_draft_backends()
patch_sglang_eagle3_runtime_compat()
_patch_sglang_dsv4_spec_cuda_graph()
# Legacy SGLang: the argmax patch replaces EagleDraftWorker.draft_forward,
# so install it before the spec-graph patch wraps draft_forward.
_patch_sglang_eagle_v2_draft_argmax()
_patch_sglang_dsv4_spec_cuda_graph()


def set_attn_cls() -> None:
Expand Down
61 changes: 58 additions & 3 deletions atom/plugin/sglang/qwen4_exp_bridge.py
Original file line number Diff line number Diff line change
Expand Up @@ -234,10 +234,51 @@ def bind_graph_ple(


_DECODE_GRAPH = _Qwen4ExpDecodeGraphBuffers()
# Draft stays eager (Flash mRoPE) but must not write the buffers baked into
# The draft must not write the buffers baked into
# the target verify graph. A second set still keeps page tables and seq_lens
# on device, so a draft step does not allocate or sync seq_lens back to host.
_DRAFT_QSA = _Qwen4ExpDecodeGraphBuffers()
# Non-None marker returned to the draft runner's out-of-graph hook
# before the first draft forward has populated _DRAFT_QSA.last_qsa.
_DRAFT_GRAPH_METADATA_IN_GRAPH = object()


def _is_draft_decode_batch(forward_batch: Any) -> bool:
"""Draft-runner decode batch (EAGLE draft input)."""
spec = getattr(forward_batch, "spec_info", None)
return type(spec).__name__ == "EagleDraftInput"


def stage_draft_graph_mrope(mrope_buffer: torch.Tensor, forward_batch: Any) -> None:
"""Stage the live mRoPE rows into a draft graph runner's ``[3, N]`` buffer.

SGLang's EAGLE draft decode / draft-extend runners capture
``forward_batch.mrope_positions`` from that buffer but refresh only the
1-D ``positions`` on replay, and the buffer is shared by name and shape
with other runners. The Flash drafter reads the 3-row positions
(``_lm_positions_for_runtime``), so copy SGLang's live rows in (they carry
multimodal offsets), or broadcast the 1-D positions when SGLang computed
none. Rows past the batch are zeroed, as SGLang zeroes padded ``positions``.
"""
live = getattr(forward_batch, "mrope_positions", None)
if live is None:
live = forward_batch.positions.unsqueeze(0).expand(3, -1)
num_tokens = int(live.shape[-1])
mrope_buffer[:, :num_tokens].copy_(live)
mrope_buffer[:, num_tokens:].zero_()


def advance_draft_decode_mrope(forward_batch: Any) -> None:
"""Advance mRoPE in step with EAGLE ``draft_forward``'s ``positions.add_(1)``.

SGLang moves only the 1-D positions after each draft decode step, so the
3-row positions the Flash drafter reads would stay at step 0. The plugin's
``draft_forward`` wrapper calls this on the caller's batch after every draft
decode forward; inside a draft decode graph capture the add is recorded.
"""
mrope = getattr(forward_batch, "mrope_positions", None)
if mrope is not None:
mrope.add_(1)


def _is_draft_forward() -> bool:
Expand Down Expand Up @@ -918,11 +959,14 @@ def _serving_context_len(atom_config: Any, indexer_budget: int) -> int:


def _ensure_draft_qsa(device: torch.device, block_size: int, ctx_len: int) -> None:
"""Allocate the draft page tables once, at serving capacity."""
"""Allocate the draft page tables once, at serving and graph capacity."""
args = _server_args()
max_bs = 32
if args is not None:
max_bs = max(int(getattr(args, "max_running_requests", 32) or 32), 1)
# The draft decode graph captures up to the decode bucket, which exceeds
# the 32-row default when --max-running-requests is omitted.
max_bs = max(max_bs, _get_cuda_graph_max_bs())
ctx = max(int(ctx_len), 1)
tokens_per_req = _get_qsa_graph_max_tokens_per_req()
pages = max(1, (ctx + block_size - 1) // block_size)
Expand Down Expand Up @@ -1601,6 +1645,12 @@ def prepare_qwen4_exp_decode_graph_metadata(
return None
if atom_config is None or not _is_qwen4_exp_config(atom_config):
return None
# A Flash draft decode graph rebuilds its QSA metadata into
# _DRAFT_QSA inside the graph (positions / out_cache_loc change per draft
# step). Do not fill the target's _DECODE_GRAPH buffers from the draft
# runner's out-of-graph hook: TARGET_VERIFY replays from them.
if mode.is_decode_or_idle() and _is_draft_decode_batch(forward_batch):
return _DRAFT_QSA.last_qsa or _DRAFT_GRAPH_METADATA_IN_GRAPH

tokens_per_req = _get_qsa_tokens_per_req(forward_batch)
positions = getattr(forward_batch, "positions", None)
Expand Down Expand Up @@ -1653,7 +1703,12 @@ def qwen4_exp_metadata_from_forward_batch(
that storage in place; there is no extra PLE copy.
"""
bind_qsa_caches(model, forward_batch, atom_config)
if _is_capturing() and _DECODE_GRAPH.last_qsa is not None:
# Draft graphs build into _DRAFT_QSA inside the graph instead.
if (
_is_capturing()
and not _is_draft_forward()
Comment thread
zovonoir marked this conversation as resolved.
Resolved
and _DECODE_GRAPH.last_qsa is not None
):
ple = _DECODE_GRAPH.last_ple
if ple is None:
ple = _build_decode_graph_ple_metadata(
Expand Down
92 changes: 92 additions & 0 deletions tests/plugin/test_sglang_qwen4_exp_mtp.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,10 @@
)
from atom.plugin.sglang.qwen4_exp_bridge import (
_query_start_loc,
advance_draft_decode_mrope,
bind_qsa_caches,
prepare_qwen4_exp_decode_graph_metadata,
stage_draft_graph_mrope,
)


Expand Down Expand Up @@ -462,3 +465,92 @@ def install(obj):
assert experts.expert_map is fused_map
assert experts.expert_map.shape[0] == num_routed + 2
assert int(experts.expert_map[-2]) == local_num


def test_draft_graph_mrope_broadcasts_text_positions_and_zeroes_padding():
"""SGLang computed no mRoPE: stage the 1-D positions on all three rows."""
buf = torch.full((3, 6), 99, dtype=torch.int64)
batch = SimpleNamespace(
mrope_positions=None, positions=torch.tensor([64, 7, 300], dtype=torch.int64)
)
stage_draft_graph_mrope(buf, batch)
assert buf[:, :3].tolist() == [[64, 7, 300]] * 3
assert buf[:, 3:].eq(0).all()


def test_draft_graph_mrope_keeps_multimodal_offsets():
"""Live rows carry the image delta; staging must not rebuild them from positions."""
buf = torch.full((3, 4), 99, dtype=torch.int64)
live = torch.tensor([[10, 11], [12, 13], [14, 15]], dtype=torch.int64)
batch = SimpleNamespace(
mrope_positions=live, positions=torch.tensor([500, 501], dtype=torch.int64)
)
stage_draft_graph_mrope(buf, batch)
assert torch.equal(buf[:, :2], live)
assert buf[:, 2:].eq(0).all()


def test_draft_decode_mrope_tracks_positions_across_steps():
"""EAGLE draft_forward: forward, then positions.add_(1); mRoPE must follow."""
positions = torch.tensor([64, 7], dtype=torch.int64)
delta = torch.tensor([-1000, 0], dtype=torch.int64)
batch = SimpleNamespace(
positions=positions,
mrope_positions=(positions + delta).unsqueeze(0).repeat(3, 1),
)
for _ in range(3):
assert torch.equal(batch.mrope_positions - batch.positions, delta.expand(3, -1))
advance_draft_decode_mrope(batch)
batch.positions.add_(1)
advance_draft_decode_mrope(SimpleNamespace(mrope_positions=None))


class EagleDraftInput:
pass


class _DecodeMode:
@staticmethod
def is_decode_or_idle():
return True

@staticmethod
def is_target_verify():
return False


def test_draft_decode_hook_never_builds_target_graph_metadata():
"""Flash draft decode batches stop before the target's _DECODE_GRAPH fill."""
flash = SimpleNamespace(hf_config=SimpleNamespace(model_type="qwen4_exp_text"))
draft = SimpleNamespace(forward_mode=_DecodeMode(), spec_info=EagleDraftInput())
assert prepare_qwen4_exp_decode_graph_metadata(draft, atom_config=flash) is not None


def test_draft_decode_hook_ignores_other_models():
"""Another EAGLE drafter on the ATOM backend keeps its normal metadata path."""
other = SimpleNamespace(
hf_config=SimpleNamespace(model_type="qwen3_5_moe", architectures=["X"])
)
draft = SimpleNamespace(forward_mode=_DecodeMode(), spec_info=EagleDraftInput())
assert prepare_qwen4_exp_decode_graph_metadata(draft, atom_config=other) is None


def test_draft_qsa_covers_the_graph_bucket_without_a_request_limit(monkeypatch):
"""No --max-running-requests: the 32-row default must grow to the capture bucket."""
from atom.plugin.sglang import qwen4_exp_bridge as bridge

args = SimpleNamespace(
max_running_requests=None,
cuda_graph_config=SimpleNamespace(decode=SimpleNamespace(bs=[1, 8, 64])),
)
seen = {}
monkeypatch.setattr(bridge, "_server_args", lambda: args)
monkeypatch.setattr(bridge, "_get_qsa_graph_max_tokens_per_req", lambda: 5)
monkeypatch.setattr(
bridge,
"_DRAFT_QSA",
SimpleNamespace(allocate_once=lambda **kw: seen.update(kw)),
)
bridge._ensure_draft_qsa(torch.device("cpu"), block_size=64, ctx_len=8192)
assert seen["max_bs"] == 64
assert seen["max_tokens"] == 64 * 5
Loading