diff --git a/.agents/skills/cookbook-add-model/references/authoring-reference.md b/.agents/skills/cookbook-add-model/references/authoring-reference.md index d98d2d4feda3..1e86fcf09dbc 100644 --- a/.agents/skills/cookbook-add-model/references/authoring-reference.md +++ b/.agents/skills/cookbook-add-model/references/authoring-reference.md @@ -134,7 +134,7 @@ Each cell describes one verified (or auto-estimated) launch recipe. - DO NOT include `--host` / `--port` literally — use `{{HOST_IP}}` / `{{PORT}}` placeholders so users can override through the Env modal. - Order flags as: `--model-path` first (after any `--trust-remote-code`), - then parallelism (`--tp`, `--dp`, `--enable-dp-attention`), then MoE + then parallelism (`--tp`, `--attn-dp-size`), then MoE flags, then tuning knobs, with `--host` / `--port` last. The playground engine assumes this ordering when inserting overrides (its anchors target `--model-path` / `--tp` / etc., and inserts before the `--host` tail). @@ -171,7 +171,7 @@ schemas (full reference in the `_playground.jsx` header): | Axis key | Widget | Use when | |---|---|---| -| `attention` | TP / CP / DP-Attention sub-knobs (DP-Attention is a combined knob: its value is the DP degree AND toggles `--enable-dp-attention`) | Model exposes parallelism knobs in its cells (§2.2) and you want users to override them. | +| `attention` | TP / CP / DP-Attention sub-knobs (DP-Attention is a single knob: its value is the attention DP size, emitted as `--attn-dp-size N`) | Model exposes parallelism knobs in its cells (§2.2) and you want users to override them. | | `moe` | Backend select (incl. MegaMoE) + EP knob; picking the MegaMoE backend reveals a Quantization sub-select (W4A8/W4A4) | Model is MoE and supports multiple `--moe-*-backend` choices. For Blackwell MoE kernel-fusion, give the `megamoe` backend option a `requiresHw` (and optional `excludesStrategy`) gate, then add a sibling `megamoeQuant` block (`{stripEnv, options}`): W4A8 = `NUM_MAX` only, W4A4 adds the FP4-activations env vars; both strip the DeepEP dispatch env. | | `parsers` | Multi-toggle | Model has reasoning / tool-call parsers. | | `speculative` | Single-select chip group | Model has spec-decoding presets you want to expose. | @@ -208,7 +208,7 @@ schemas (full reference in the `_playground.jsx` header): knobs/values that are disabled under the live facts, so stale picks never emit a blocked combination. Interleave prefill-CP + DP-Attention is deliberately NOT grayed (combined support is planned upstream even though - current releases assert `dp_size == 1` for interleave) — the engine shows + current releases assert `attn_dp_size == 1` for interleave) — the engine shows a warning hint below the command box instead. - The CP knob emits `--attn-cp-size N --enable-prefill-cp --cp-strategy S`, where S is: an optional `{ id: "cpStrategy", values: [null, "interleave", diff --git a/.agents/skills/cookbook-add-model/templates/config.jsx.tmpl b/.agents/skills/cookbook-add-model/templates/config.jsx.tmpl index dfb5e537adc7..83baf0a717ae 100644 --- a/.agents/skills/cookbook-add-model/templates/config.jsx.tmpl +++ b/.agents/skills/cookbook-add-model/templates/config.jsx.tmpl @@ -176,7 +176,7 @@ sgl-eval run gsm8k \\ playgroundFeatures: { // ----- Card: "Attention Parallelism" ----- KEEP if the model exposes TP/CP/DP - // knobs. DP-Attention is a combined knob: value = DP degree AND toggles `--enable-dp-attention`. + // knobs. DP-Attention is a single knob: value = attention DP size, emitted as `--attn-dp-size N`. attention: { knobs: [ { id: "tp", label: "TP", values: [ @@ -357,8 +357,7 @@ sgl-eval run gsm8k \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 4", - "--dp 4", - "--enable-dp-attention", + "--attn-dp-size 4", "--moe-a2a-backend deepep", "--host {{HOST_IP}}", "--port {{PORT}}", @@ -403,8 +402,7 @@ sgl-eval run gsm8k \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 16", - "--dp 16", - "--enable-dp-attention", + "--attn-dp-size 16", "--moe-a2a-backend deepep", "--host {{HOST_IP}}", "--port {{PORT}}", diff --git a/.agents/skills/cookbook-migrate-model/SKILL.md b/.agents/skills/cookbook-migrate-model/SKILL.md index ae9b55dad95f..26c5314894c9 100644 --- a/.agents/skills/cookbook-migrate-model/SKILL.md +++ b/.agents/skills/cookbook-migrate-model/SKILL.md @@ -118,7 +118,7 @@ that **changes other parts of the command** (TP, mem) must (the Playground can't do coupled changes), and so does a toggle the legacy page itself labels with operating-point words — e.g. a `dpattention` radio whose options are subtitled "Low Latency" / "High Throughput" (GLM-5.1 / Kimi-K2.6 pattern) — -even when its flags are uncoupled (`--dp N --enable-dp-attention` is a pure +even when its flags are uncoupled (`--attn-dp-size N` is a pure flag add). Any other toggle that only adds/removes its own flags becomes a Playground axis with the flags baked into cells when the legacy default was ON — EXCEPT parsers: diff --git a/.agents/skills/cookbook-migrate-model/references/dimension-mapping.md b/.agents/skills/cookbook-migrate-model/references/dimension-mapping.md index 3e34ae790321..90d35ffc8cce 100644 --- a/.agents/skills/cookbook-migrate-model/references/dimension-mapping.md +++ b/.agents/skills/cookbook-migrate-model/references/dimension-mapping.md @@ -12,7 +12,7 @@ this file is about the *mapping decisions*. | hardware radio | `match.hw` | Catalog ids as-is. Off-catalog hardware → `config.hardware` entry — e.g. A100 `{id:"a100", label:"A100", vram:"80GB", vendor:"nvidia"}` (merges into the NVIDIA row), Xeon `{id:"xeon", label:"Xeon", vram:"host RAM", vendor:"intel"}` (engine renders a new INTEL row; any vendor key works). A merged chip like GLM-5's "MI300X/MI325X" splits into two ids with duplicated cells (cells are denormalized by design). | | model-size / model-name radio | `variants` | One variant per deployable checkpoint family; single `{id:"default"}` when there's no variant axis (then `modelNames` keys drop the variant half). | | quantization radio | `quantizations` | Real precision ids (`bf16`/`fp8`/`fp4`/`int4`/…). One `fp4` id even when checkpoints differ per vendor — route via `hw\|variant\|quant` triple keys in `modelNames` (NVFP4 on Blackwell vs AMD MXFP4 is the precedent); per-hw greying falls out of which cells exist. | -| toggle that **couples** with other parts of the command (changes TP/mem/EP), OR one the legacy page labels with **operating-point words** | `strategies` | The Playground applies pure flag diffs — it cannot do coupled changes. Example: Qwen3.5's MTP toggle bumps TP on three H100 combos → strategies `low-latency` (MTP on) / `high-throughput` (MTP off). **Naming counts like coupling**: GLM-5.1's / Kimi-K2.6's `dpattention` adds only `--dp N --enable-dp-attention` (uncoupled), but its options are subtitled "Low Latency" / "High Throughput" — the page's own named operating-point split → strategies; a flag-only spec toggle riding alongside it stays a Playground axis and bakes per its legacy default. GPU-count radios (GLM-4.7, MiniMax-M2.5/2.7) → budget-tier strategies with the legacy SUPPORT matrix preserved by which cells exist. Strategy count follows the page's operating points: 1 → `balanced`, 2 → `low-latency`+`high-throughput`, 3 → the full trio (§4). | +| toggle that **couples** with other parts of the command (changes TP/mem/EP), OR one the legacy page labels with **operating-point words** | `strategies` | The Playground applies pure flag diffs — it cannot do coupled changes. Example: Qwen3.5's MTP toggle bumps TP on three H100 combos → strategies `low-latency` (MTP on) / `high-throughput` (MTP off). **Naming counts like coupling**: GLM-5.1's / Kimi-K2.6's `dpattention` adds only `--attn-dp-size N` (uncoupled), but its options are subtitled "Low Latency" / "High Throughput" — the page's own named operating-point split → strategies; a flag-only spec toggle riding alongside it stays a Playground axis and bakes per its legacy default. GPU-count radios (GLM-4.7, MiniMax-M2.5/2.7) → budget-tier strategies with the legacy SUPPORT matrix preserved by which cells exist. Strategy count follows the page's operating points: 1 → `balanced`, 2 → `low-latency`+`high-throughput`, 3 → the full trio (§4). | | toggle that only adds/removes its own flags | Playground axis (+ bake, EXCEPT parsers and accuracy-degrading flags) | **Parsers (`--reasoning-parser` / `--tool-call-parser`) are NEVER baked into cells** — Deployment commands ship without them regardless of the legacy default or the measured command; the `parsers` axis adds them on top (DSv4 convention; cells mirror the legacy generator's parsers-OFF output). Accuracy-degrading toggles are never baked either — §2 caveats (axis-only, accuracy-safe cells). Other flag-only toggles: legacy default ON → bake into cells AND declare the axis so users can strip (red strikethrough); default OFF → keep cells clean, axis preset only. MTP/EAGLE presets → `speculative` axis; dp-attention → a strategy when the legacy page labels it as the operating-point split or when coupled (see the row above), else `attention.dpAttn`. **EVERY legacy control survives as an interactive control** (a dimension or a Playground axis), never a tips-only mention — but a model-specific control is **config DATA, not engine code**: the axis handler reads options/flags/env/gating straight from `config.playgroundFeatures` (MegaMoE W4A4 is entirely DSv4 config data on the existing `moe` axis — no per-model engine edit). A control that fits an existing axis's data schema is therefore pure config, full stop. A **titled single-select that strips a flag family** (e.g. Nemotron3's "KV Cache DType" `--kv-cache-dtype`) is covered by the merged generic **`flagSelects`** axis → **config-only**: declare a `flagSelects` list of `{ id, title, stripPrefixes, options }` (see the Qwen3.5 mamba example), **no engine PR**. Only a control whose *shape* `flagSelects` still can't express would need a new ONE-TIME generic primitive (never a model-named handler) on a prior engine PR; the backward-compat reasoning (opt-in per key, not in the opt-out set) is in engine-axis.md. | | per-combo hidden option (e.g. spec hidden on Xeon) | absent cells | Don't create cells for combos the legacy widget couldn't produce; the engine greys them automatically. `# Error:` pseudo-commands → no cell + explanation in §2 tips and/or a chip `disable`/`disableReason`. | | coupled secondary knob (e.g. mamba cache V1/V2) | cells + Playground axis | Bake the correct value per cell following the legacy coupling (Qwen3.5: MTP ⇒ `--mamba-radix-cache-strategy extra_buffer` on NVIDIA; AMD/Xeon ⇒ V1/no flag) and document the coupling in §2 tips — AND surface the knob as a Playground axis like every other legacy feature (row above; add the axis when none fits). Baking alone is NOT enough — the every-feature rule supersedes the pilot's cells+prose-only treatment of Qwen3.5's mamba knob (retrofit pending). The mamba knob is the same single-select shape as KV Cache DType, so it rides the merged generic **`flagSelects`** axis — Qwen3.6 / Qwen3-Coder-Next declare it purely in config (a `flagSelects` block), **no engine PR**. | @@ -30,7 +30,7 @@ this file is about the *mapping decisions*. | (absent) | append `--host {{HOST_IP}}`, `--port {{PORT}}` to every cell | | `--nnodes N --node-rank … --dist-init-addr …` literals | delete; `match.nodes: "multi-N"` + `nodesOptions` entry — the engine injects the trio after the last parallelism anchor plus the multi-node header comment | | env-var command prefixes | verbatim into `cell.env[]` (never drop/normalize) | -| flag order as emitted | re-sort to canonical: `--trust-remote-code` → `--model-path` → parallelism (`--tp`/`--dp`/`--enable-dp-attention`/EP) → MoE → tuning → `--host`/`--port` (Playground insert anchors assume this). Keep the legacy relative order within the tuning span so commands stay eyeball-diffable. | +| flag order as emitted | re-sort to canonical: `--trust-remote-code` → `--model-path` → parallelism (`--tp`/`--attn-dp-size`/EP) → MoE → tuning → `--host`/`--port` (Playground insert anchors assume this). Keep the legacy relative order within the tuning span so commands stay eyeball-diffable. | Caveats discovered in the pilot: - The Playground `moe.ep` knob only understands `--ep` — normalize a legacy diff --git a/.agents/skills/sglang-runtime-context/SKILL.md b/.agents/skills/sglang-runtime-context/SKILL.md index 6733281bc2b9..ea02a3086c4b 100644 --- a/.agents/skills/sglang-runtime-context/SKILL.md +++ b/.agents/skills/sglang-runtime-context/SKILL.md @@ -227,7 +227,7 @@ slack (`>= len(...) - 1`) or compared key names instead of value sources. read-through over the canonical getters, so they answer with the live process groups. Everything else — `tp_size`, `pp_size`, `attn_cp_size`, `dcp_size`, `moe_dp_size` included, alongside config-only leaves like `nccl_port`, -`enable_dp_attention`, `dp_size`, `ep_size`, `dwdp_size` — is answered from the +`dp_size`, `ep_size`, `dwdp_size` — is answered from the published `parallel` bag. Reading a leaf before publish raises a `ValueError` naming the namespace; an unknown name is an `AttributeError`. diff --git a/benchmark/deepseek_v3/README.md b/benchmark/deepseek_v3/README.md index 14c67c8cb01d..141e9d60368c 100644 --- a/benchmark/deepseek_v3/README.md +++ b/benchmark/deepseek_v3/README.md @@ -47,7 +47,7 @@ Add [performance optimization options](#performance-optimization-options) as nee [MLA optimizations](https://lmsys.org/blog/2024-09-04-sglang-v0-3/#deepseek-multi-head-latent-attention-mla-throughput-optimizations) are enabled by default. Here are some optional optimizations can be enabled as needed. -- [Data Parallelism Attention](https://lmsys.org/blog/2024-12-04-sglang-v0-4/#data-parallelism-attention-for-deepseek-models): For high QPS scenarios, add the `--enable-dp-attention` argument to boost throughput. +- [Data Parallelism Attention](https://lmsys.org/blog/2024-12-04-sglang-v0-4/#data-parallelism-attention-for-deepseek-models): For high QPS scenarios, add the `--attn-dp-size ` argument (e.g. equal to `--tp`) to boost throughput. - [Torch.compile Optimization](https://lmsys.org/blog/2024-09-04-sglang-v0-3/#torchcompile-latency-optimizations): Add `--enable-torch-compile` argument to enable it. This will take some time while server starts. The maximum batch size for torch.compile optimization can be controlled with `--torch-compile-max-bs`. It's recommended to set it between `1` and `8`. (e.g., `--torch-compile-max-bs 8`) ### Usage: Chat with DeepSeek diff --git a/docs/cookbook/autoregressive/DeepSeek/DeepSeek-Math-V2.mdx b/docs/cookbook/autoregressive/DeepSeek/DeepSeek-Math-V2.mdx index a89618b8703e..2ee3ff173e21 100644 --- a/docs/cookbook/autoregressive/DeepSeek/DeepSeek-Math-V2.mdx +++ b/docs/cookbook/autoregressive/DeepSeek/DeepSeek-Math-V2.mdx @@ -52,7 +52,7 @@ This section provides deployment configurations optimized for different hardware **DP Attention:** - Enable DP attention for high-throughput scenarios -- The `--dp` value commonly matches the `--tp` value +- The `--attn-dp-size` value commonly matches the `--tp` value - Trade-off: Higher throughput at the cost of slightly increased latency ## 4. Model Invocation diff --git a/docs/cookbook/autoregressive/DeepSeek/DeepSeek-OCR-2.mdx b/docs/cookbook/autoregressive/DeepSeek/DeepSeek-OCR-2.mdx index 3aa56939e200..e80e8de4133b 100644 --- a/docs/cookbook/autoregressive/DeepSeek/DeepSeek-OCR-2.mdx +++ b/docs/cookbook/autoregressive/DeepSeek/DeepSeek-OCR-2.mdx @@ -192,8 +192,6 @@ sglang serve \ --enable-multimodal \ --tp 1 \ --ep 1 \ - --dp 1 \ - --enable-dp-attention \ --host 0.0.0.0 \ --port 30000 ``` diff --git a/docs/cookbook/autoregressive/DeepSeek/DeepSeek-OCR.mdx b/docs/cookbook/autoregressive/DeepSeek/DeepSeek-OCR.mdx index ef8bc0caab5c..f999fa73be94 100644 --- a/docs/cookbook/autoregressive/DeepSeek/DeepSeek-OCR.mdx +++ b/docs/cookbook/autoregressive/DeepSeek/DeepSeek-OCR.mdx @@ -184,8 +184,6 @@ python3 -m sglang.launch_server \ --model-path deepseek-ai/DeepSeek-OCR \ --tp 1 \ --ep 1 \ - --dp 1 \ - --enable-dp-attention \ --dtype float16 \ --host 0.0.0.0 \ --port 8000 diff --git a/docs/cookbook/autoregressive/DeepSeek/DeepSeek-R1.mdx b/docs/cookbook/autoregressive/DeepSeek/DeepSeek-R1.mdx index 436c7add6b38..3b6b3583e07c 100644 --- a/docs/cookbook/autoregressive/DeepSeek/DeepSeek-R1.mdx +++ b/docs/cookbook/autoregressive/DeepSeek/DeepSeek-R1.mdx @@ -95,7 +95,7 @@ DeepSeek-R1 shares the same MoE architecture as DeepSeek-V3, so the same hardwar python3 -m sglang.compile_deep_gemm --model deepseek-ai/DeepSeek-R1 --tp 8 --trust-remote-code ``` -**Data Parallelism Attention (`--enable-dp-attention`):** Recommended for high-throughput scenarios. Use `--enable-dp-attention --tp 8 --dp 8` on a single 8-GPU node. +**Data Parallelism Attention (`--attn-dp-size`):** Recommended for high-throughput scenarios. Use `--tp 8 --attn-dp-size 8` on a single 8-GPU node. **NCCL timeout:** If model loading is slow, increase: `--dist-timeout 3600`. diff --git a/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V3.mdx b/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V3.mdx index a69839f1e1b9..b0f22db90b15 100644 --- a/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V3.mdx +++ b/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V3.mdx @@ -75,7 +75,7 @@ python3 -m sglang.compile_deep_gemm --model deepseek-ai/DeepSeek-V3 --tp 8 --tru ``` DeepGEMM is enabled by default on Hopper/Blackwell and can be disabled with `SGLANG_ENABLE_JIT_DEEPGEMM=0`. -**Data Parallelism Attention (`--enable-dp-attention`):** Recommended for high-throughput scenarios with large batch sizes. Reduces KV-cache duplication across TP ranks. Use `--enable-dp-attention --tp 8 --dp 8` on a single 8-GPU node. Not recommended for low-latency, small-batch workloads. +**Data Parallelism Attention (`--attn-dp-size`):** Recommended for high-throughput scenarios with large batch sizes. Reduces KV-cache duplication across TP ranks. Use `--tp 8 --attn-dp-size 8` on a single 8-GPU node. Not recommended for low-latency, small-batch workloads. **NCCL timeout:** If model loading is slow and you hit an NCCL timeout, increase it: `--dist-timeout 3600`. @@ -454,8 +454,7 @@ We use SGLang's built-in benchmarking tool to conduct performance evaluation on python3 -m sglang.launch_server \ --model-path deepseek-ai/DeepSeek-V3 \ --tp 8 \ - --dp 8 \ - --enable-dp-attention \ + --attn-dp-size 8 \ --speculative-algorithm EAGLE \ --speculative-num-steps 3 \ --speculative-eagle-topk 1 \ @@ -529,8 +528,7 @@ python3 -m sglang.launch_server \ --model-path deepseek-ai/DeepSeek-V3 \ --tp 8 \ --ep 8 \ - --dp 8 \ - --enable-dp-attention \ + --attn-dp-size 8 \ --host 0.0.0.0 \ --port 8000 ``` diff --git a/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V3_1.mdx b/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V3_1.mdx index 0dcad01acdd4..67e4bcdd9460 100644 --- a/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V3_1.mdx +++ b/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V3_1.mdx @@ -73,7 +73,7 @@ python3 -m sglang.compile_deep_gemm --model deepseek-ai/DeepSeek-V3.1 --tp 8 --t ``` DeepGEMM is enabled by default on Hopper/Blackwell and can be disabled with `SGLANG_ENABLE_JIT_DEEPGEMM=0`. -**Data Parallelism Attention (`--enable-dp-attention`):** Recommended for high-throughput scenarios with large batch sizes. Reduces KV-cache duplication across TP ranks. Use `--enable-dp-attention --tp 8 --dp 8` on a single 8-GPU node. Not recommended for low-latency, small-batch workloads. +**Data Parallelism Attention (`--attn-dp-size`):** Recommended for high-throughput scenarios with large batch sizes. Reduces KV-cache duplication across TP ranks. Use `--tp 8 --attn-dp-size 8` on a single 8-GPU node. Not recommended for low-latency, small-batch workloads. **NCCL timeout:** If model loading is slow and you hit an NCCL timeout, increase it: `--dist-timeout 3600`. diff --git a/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V3_2.mdx b/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V3_2.mdx index f609d2705a05..6860fd1a0fbc 100644 --- a/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V3_2.mdx +++ b/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V3_2.mdx @@ -173,7 +173,7 @@ DeepSeek-V3.2 and DeepSeek-V3.2-Exp support tool calling capabilities. But they python3 -m sglang.launch_server \ --model-path deepseek-ai/DeepSeek-V3.2-Speciale \ --trust-remote-code \ - --tp-size 8 --dp-size 8 --enable-dp-attention \ + --tp-size 8 --attn-dp-size 8 \ --reasoning-parser deepseek-v3 ``` @@ -357,8 +357,7 @@ SGLang implements Multi-Token Prediction (MTP) for DeepSeek V3.2 based on [EAGLE **With DP Attention:** ```bash Command -python -m sglang.launch_server --model deepseek-ai/DeepSeek-V3.2-Exp --tp 8 --dp 8 \ - --enable-dp-attention \ +python -m sglang.launch_server --model deepseek-ai/DeepSeek-V3.2-Exp --tp 8 --attn-dp-size 8 \ --speculative-algorithm EAGLE \ --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 ``` @@ -394,8 +393,7 @@ python -m sglang.launch_server \ --host $LOCAL_IP \ --port $PORT \ --tp 8 \ - --dp 8 \ - --enable-dp-attention \ + --attn-dp-size 8 \ --dist-init-addr ${HOST}:${DIST_PORT} \ --trust-remote-code \ --disaggregation-bootstrap-port 8998 \ @@ -411,8 +409,7 @@ python -m sglang.launch_server \ --host $LOCAL_IP \ --port $PORT \ --tp 8 \ - --dp 8 \ - --enable-dp-attention \ + --attn-dp-size 8 \ --dist-init-addr ${HOST}:${DIST_PORT} \ --trust-remote-code \ --mem-fraction-static 0.9 @@ -435,7 +432,7 @@ For production deployments (RBG / LWS-based, DeepEP EP parallelism), see [multi_ Enable prefill context parallelism (CP) on CUDA with `--enable-prefill-cp --cp-strategy interleave` for long-sequence workloads. -Zigzag prefill CP (`--cp-strategy zigzag`) is temporarily unavailable for DeepSeek V3.2, GLM-5, GLM-5.1, GLM-5.2, and GLM-5.3. Use `interleave` for these models and keep `--dp 1`; interleave DSA CP does not support `--dp` greater than 1. +Zigzag prefill CP (`--cp-strategy zigzag`) is temporarily unavailable for DeepSeek V3.2, GLM-5, GLM-5.1, GLM-5.2, and GLM-5.3. Use `interleave` for these models and keep `--attn-dp-size 1`; interleave DSA CP does not support `--attn-dp-size` greater than 1. **Interleave** (`--cp-strategy interleave`): Distributes tokens by `token_idx % cp_size`. It supports fused MoE, FP8 KV cache, and multi-batch prefill. @@ -462,7 +459,7 @@ export SGLANG_SET_CPU_AFFINITY=1 sglang_args=$(echo serve \ --model-path $MAPPED_MODEL_PATH \ --nccl-init $MASTER_ADDR:$MASTER_PORT --nnodes 2 --node-rank $RANK --tp 16 \ - --dp 2 --enable-dp-attention --page-size 64 \ + --attn-dp-size 2 --page-size 64 \ --trust-remote-code --host "0.0.0.0" --port 30000 \ --log-requests \ --context-length 65536 --max-running-requests 128 \ @@ -478,7 +475,7 @@ sglang_args=($sglang_args) sglang "${sglang_args[@]}" 2>&1 | tee $LOG_DIR/$RANK.log ``` -**CP + PP + EP + DP** +**CP + PP + EP** `CP` is currently enabled with `PP=2` on Hopper platform and we can reduce TP=16 to TP=8 from standalone deployment: @@ -486,7 +483,7 @@ sglang "${sglang_args[@]}" 2>&1 | tee $LOG_DIR/$RANK.log # verified on Hopper platform sglang_args=$(echo serve \ --model-path $MAPPED_MODEL_PATH \ - --nccl-init $MASTER_ADDR:$MASTER_PORT --nnodes 2 --node-rank $RANK --tp 8 --pp-size 2 --dp 1 --enable-dp-attention \ + --nccl-init $MASTER_ADDR:$MASTER_PORT --nnodes 2 --node-rank $RANK --tp 8 --pp-size 2 \ --moe-a2a-backend deepep --ep-size 16 \ --page-size 128 \ --chunked-prefill-size 16384 \ @@ -502,7 +499,7 @@ sglang_args=$(echo serve \ --context-length 65536 \ --allow-auto-truncate --enable-metrics \ --tool-call-parser deepseekv32 --reasoning-parser deepseek-v3 \ - --served-model-name DeepSeek-V3.2-dsa-pp-cp-ep-dp + --served-model-name DeepSeek-V3.2-dsa-pp-cp-ep ) sglang_args=($sglang_args) @@ -512,14 +509,10 @@ sglang "${sglang_args[@]}" 2>&1 | tee $LOG_DIR/$RANK.log **fp8 KV + CP + PP** -FP8 KV reduces the memory footprint. For DeepSeek V3.2, combine it with interleave CP and PP while keeping `--dp 1`: +FP8 KV reduces the memory footprint. For DeepSeek V3.2, combine it with interleave CP and PP, without attention DP: ```shell Command # verified in Hopper platform -dp_config=" \ - --dp 1 --enable-dp-attention \ -" - cp_config=" \ --enable-prefill-cp \ --cp-strategy interleave \ @@ -529,7 +522,6 @@ cp_config=" \ sglang_args=$(echo serve \ --model-path $MAPPED_MODEL_PATH \ --nccl-init $MASTER_ADDR:$MASTER_PORT --nnodes 2 --node-rank $RANK --tp 8 --pp-size 2 --pp-async-batch-depth 1 \ - $dp_config \ --trust-remote-code --host "0.0.0.0" --port 30000 \ --log-requests \ --context-length 65536 --max-running-requests 128 \ @@ -641,8 +633,7 @@ sglang serve \ --model-path deepseek-ai/DeepSeek-V3.2-Exp \ --tp 8 \ --ep 8 \ - --dp 8 \ - --enable-dp-attention \ + --attn-dp-size 8 \ --host 0.0.0.0 \ --port 30000 ``` @@ -875,7 +866,7 @@ export SGLANG_SET_CPU_AFFINITY=1 sglang_args=$(echo serve \ --model-path $MAPPED_MODEL_PATH \ --nccl-init $MASTER_ADDR:$MASTER_PORT --nnodes 2 --node-rank $RANK --tp 16 \ - --dp 2 --enable-dp-attention --page-size 64 \ + --attn-dp-size 2 --page-size 64 \ --trust-remote-code --host "0.0.0.0" --port 30000 \ --log-requests \ --context-length 65536 --max-running-requests 128 \ diff --git a/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V4.mdx b/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V4.mdx index 4297e86b75c1..6702a46bc19e 100644 --- a/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V4.mdx +++ b/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V4.mdx @@ -390,7 +390,7 @@ HiCache and MegaMoE are **not** supported on RTX PRO 6000. - **Model checkpoints** — for correct accuracy, the FP4 model uses the stock `deepseek-ai/DeepSeek-V4-{Flash,Pro}`, and the FP8 model uses the repackaged `sgl-project/DeepSeek-V4-{Flash,Pro}-FP8`. - **Supported models** — **MI300X** supports DeepSeek-V4-Flash in FP8; **MI355X** supports DeepSeek-V4-Flash / Pro in both FP4 and FP8. All recipes run single-node. -- **TP / DP setting (MI355X)** — both TP=4 and TP=8 are supported. At low concurrency we recommend **TP-only**; at high concurrency use **TP + DP** (balanced / high-throughput recipes). The verified MI355X DP recipes additionally set `--dp 8 --enable-dp-attention --enable-dp-attention-local-control-broadcast --tokenizer-worker-num 8 --stream-interval 20 --prefill-decode-interval 10`. +- **TP / DP setting (MI355X)** — both TP=4 and TP=8 are supported. At low concurrency we recommend **TP-only**; at high concurrency use **TP + DP** (balanced / high-throughput recipes). The verified MI355X DP recipes additionally set `--attn-dp-size 8 --enable-dp-attention-local-control-broadcast --tokenizer-worker-num 8 --stream-interval 20 --prefill-decode-interval 10`. - **MTP** — speculative decoding on the original Flash / Pro checkpoints; add `--speculative-algorithm EAGLE --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4`. The 0813 Pro Official checkpoint keeps its MTP head too, so the same flags work there as a fallback to DSpark. - **DSpark (MI355X Pro Official 0813)** — the 0813 checkpoint bundles the DSpark draft head. Prefer it over EAGLE on 0813 for ordinary serving: enable `--speculative-algorithm DSPARK` (low-latency) and see the [MI355X agentic recipe](#3-7-agentic-long-context-with-hicache-dram-offload-mi355x-fp4-dspark) for the long-context path. DSpark also runs under **PD disaggregation** — see the Pro Official pairs in [§3.8](#3-8-pd-disaggregation-on-mi355x-mori-io). - **Kernels** — uses the Unified KV attention and the flydsl MoE. @@ -408,7 +408,7 @@ export SGLANG_MORI_RECV_BOUND=1 sglang serve \ --model-path {MODEL_PATH} \ - --tp 8 --dp 8 --enable-dp-attention \ + --tp 8 --attn-dp-size 8 \ --ep-size 8 --moe-a2a-backend mori --deepep-mode normal \ ``` @@ -769,8 +769,7 @@ python3 -m sglang.launch_server \ --model-path deepseek-ai/DeepSeek-V4-Pro-0813 \ --trust-remote-code \ --tp 8 \ - --dp 8 \ - --enable-dp-attention \ + --attn-dp-size 8 \ --enable-dp-lm-head \ --ep-size 8 \ --moe-a2a-backend megamoe \ @@ -792,7 +791,7 @@ python3 -m sglang.launch_server \ --hicache-mem-layout page_first_direct ``` -`--chunked-prefill-size` is a global budget divided by `--dp`, so this keeps 6144 tokens per rank. +`--chunked-prefill-size` is a global budget divided by `--attn-dp-size`, so this keeps 6144 tokens per rank. **Disaggregated 1P1D (DEP8 prefill / DEP8 decode), concurrency 64–128.** Prefill, on one 8-GPU node: ```bash Command @@ -804,8 +803,7 @@ python3 -m sglang.launch_server \ --model-path deepseek-ai/DeepSeek-V4-Pro-0813 \ --trust-remote-code \ --tp 8 \ - --dp 8 \ - --enable-dp-attention \ + --attn-dp-size 8 \ --enable-dp-lm-head \ --ep-size 8 \ --moe-dense-tp-size 1 \ @@ -841,8 +839,7 @@ python3 -m sglang.launch_server \ --model-path deepseek-ai/DeepSeek-V4-Pro-0813 \ --trust-remote-code \ --tp 8 \ - --dp 8 \ - --enable-dp-attention \ + --attn-dp-size 8 \ --enable-dp-lm-head \ --ep-size 8 \ --moe-dense-tp-size 1 \ @@ -941,8 +938,7 @@ python3 -m sglang.launch_server \ --model-path deepseek-ai/DeepSeek-V4-Pro-0813 \ --trust-remote-code \ --tp 8 \ - --dp 8 \ - --enable-dp-attention \ + --attn-dp-size 8 \ --enable-dp-lm-head \ --enable-prefill-delayer \ --enable-dp-attention-local-control-broadcast \ @@ -971,7 +967,7 @@ python3 -m sglang.launch_server \ --watchdog-timeout 3600 ``` -`--chunked-prefill-size` is a global budget divided by `--dp`, so this keeps 8192 tokens per rank. `--enable-dp-lm-head` is required for DSpark under DP attention. Front the DP ranks with `sglang_router --policy cache_aware` so multi-turn sessions land on the rank holding the longest radix/HiCache prefix; above concurrency 160 add `--balance-abs-threshold 32`. +`--chunked-prefill-size` is a global budget divided by `--attn-dp-size`, so this keeps 8192 tokens per rank. `--enable-dp-lm-head` is required for DSpark under DP attention. Front the DP ranks with `sglang_router --policy cache_aware` so multi-turn sessions land on the rank holding the longest radix/HiCache prefix; above concurrency 160 add `--balance-abs-threshold 32`. **DP8 EP8 MegaMoE, concurrency 128–384:** ```bash Command @@ -996,9 +992,8 @@ python3 -m sglang.launch_server \ --model-path deepseek-ai/DeepSeek-V4-Pro-0813 \ --trust-remote-code \ --tp 8 \ - --dp 8 \ + --attn-dp-size 8 \ --ep 8 \ - --enable-dp-attention \ --enable-dp-lm-head \ --enable-prefill-delayer \ --enable-dp-attention-local-control-broadcast \ diff --git a/docs/cookbook/autoregressive/GLM/GLM-5.1.mdx b/docs/cookbook/autoregressive/GLM/GLM-5.1.mdx index f8a3308efa73..c069d84c9bb0 100644 --- a/docs/cookbook/autoregressive/GLM/GLM-5.1.mdx +++ b/docs/cookbook/autoregressive/GLM/GLM-5.1.mdx @@ -101,7 +101,7 @@ import { GLM51Deployment } from '/src/snippets/autoregressive/glm-51-deployment. - **AMD GPUs**: BF16 and FP8 checkpoints run on MI300X/MI325X/MI355X at tp=8. On MI355X (gfx950), the MXFP4 checkpoint `amd/GLM-5.1-MXFP4` is also supported at tp=4 with `--kv-cache-dtype fp8_e4m3`. All AMD paths pass `--dsa-prefill-backend tilelang --dsa-decode-backend tilelang`, `--chunked-prefill-size 131072`, and `--watchdog-timeout 1200` (20 minutes for weight loading). FP8 uses approximately half the memory of BF16 (~89 GB/GPU vs ~175 GB/GPU). EAGLE speculative decoding is supported on AMD GPUs: MI300X/MI325X (gfx942) and MI355X (gfx950), but it **requires `--disable-custom-all-reduce`** — the aiter custom all-reduce kernel deadlocks during EAGLE verify at high concurrency, so without this flag the server will hang. - For other configuration tips (MTP, DSA kernel, Context Parallel, HiSparse, NVFP4, Index Cache), see the [DeepSeek-V3.2 cookbook page](../DeepSeek/DeepSeek-V3_2). GLM-5.1 and DeepSeek-V3.2 share the same model structure, so the optimization techniques are common. -- **Prefill CP on CUDA**: Zigzag (`--cp-strategy zigzag`) is temporarily unavailable for GLM-5.1. Use `--enable-prefill-cp --cp-strategy interleave` with `--dp 1`. +- **Prefill CP on CUDA**: Zigzag (`--cp-strategy zigzag`) is temporarily unavailable for GLM-5.1. Use `--enable-prefill-cp --cp-strategy interleave` with `--attn-dp-size 1`. - Use `--json-model-override-args '{"index_topk_pattern": "FFSFSSSFSSFFFSSSFFFSFSSSSSSFFSFFSFFSSFFFFFFSFFFFFSFFSSSSSSFSFFFSFSSSFSFFSFFSSS"}'` to enable the [IndexCache](https://github.com/THUDM/IndexCache) method for GLM-5.1. This can improve serving efficiency with only a small accuracy loss. If you are running rigorous accuracy evaluations, do not enable this feature. diff --git a/docs/cookbook/autoregressive/GLM/GLM-5.2.mdx b/docs/cookbook/autoregressive/GLM/GLM-5.2.mdx index eed0bfc2e657..0388ea2bc4bd 100644 --- a/docs/cookbook/autoregressive/GLM/GLM-5.2.mdx +++ b/docs/cookbook/autoregressive/GLM/GLM-5.2.mdx @@ -255,7 +255,7 @@ For the full setup (streaming, tool-use, count_tokens, persisting env in `~/.cla ### 3.5 Context Parallelism -Zigzag prefill CP (`--cp-strategy zigzag`) is temporarily unavailable for GLM-5.2. For prefill CP on CUDA, use `interleave` with `--dp 1` as shown below. +Zigzag prefill CP (`--cp-strategy zigzag`) is temporarily unavailable for GLM-5.2. For prefill CP on CUDA, use `interleave` with `--attn-dp-size 1` as shown below. Prefill context parallelism can help with reduction of TTFT under long context. To enable prefill context parallelism for GLM 5.2, please append the following arguments: diff --git a/docs/cookbook/autoregressive/GLM/GLM-5.3.mdx b/docs/cookbook/autoregressive/GLM/GLM-5.3.mdx index bd6b6760b912..38dfe435eb66 100644 --- a/docs/cookbook/autoregressive/GLM/GLM-5.3.mdx +++ b/docs/cookbook/autoregressive/GLM/GLM-5.3.mdx @@ -243,7 +243,7 @@ For the full setup (streaming, tool-use, count_tokens, persisting env in `~/.cla ### 3.5 Context Parallelism -Zigzag prefill CP (`--cp-strategy zigzag`) is temporarily unavailable for GLM-5.3. For prefill CP on CUDA, use `interleave` with `--dp 1` as shown below. +Zigzag prefill CP (`--cp-strategy zigzag`) is temporarily unavailable for GLM-5.3. For prefill CP on CUDA, use `interleave` with `--attn-dp-size 1` as shown below. Prefill context parallelism can help with reduction of TTFT under long context. To enable prefill context parallelism for GLM 5.3, please append the following arguments: diff --git a/docs/cookbook/autoregressive/GLM/GLM-5.mdx b/docs/cookbook/autoregressive/GLM/GLM-5.mdx index 791ebb418990..9c480d552255 100644 --- a/docs/cookbook/autoregressive/GLM/GLM-5.mdx +++ b/docs/cookbook/autoregressive/GLM/GLM-5.mdx @@ -94,7 +94,7 @@ import { GLM5Deployment } from '/src/snippets/autoregressive/glm-5-deployment.js - **AMD GPUs**: Use `--dsa-prefill-backend tilelang --dsa-decode-backend tilelang` for the DSA attention backend. Add `--chunked-prefill-size 131072` and `--watchdog-timeout 1200` (20 minutes for weight loading). EAGLE speculative decoding is not currently supported on AMD for GLM-5. - For other configuration tips (MTP, DSA kernel, Context Parallel, HiSparse, NVFP4, Index Cache), see the [DeepSeek-V3.2 cookbook page](../DeepSeek/DeepSeek-V3_2). GLM-5 and DeepSeek-V3.2 share the same model structure, so the optimization techniques are common. -- **Prefill CP on CUDA**: Zigzag (`--cp-strategy zigzag`) is temporarily unavailable for GLM-5. Use `--enable-prefill-cp --cp-strategy interleave` with `--dp 1`. +- **Prefill CP on CUDA**: Zigzag (`--cp-strategy zigzag`) is temporarily unavailable for GLM-5. Use `--enable-prefill-cp --cp-strategy interleave` with `--attn-dp-size 1`. - Use `--json-model-override-args '{"index_topk_pattern": "FFSFSSSFSSFFFSSSFFFSFSSSSSSFFSFFSFFSSFFFFFFSFFFFFSFFSSSSSSFSFFFSFSSSFSFFSFFSSS"}'` for GLM-5-FP8 if you want to enable the [IndexCache](https://github.com/THUDM/IndexCache) method. This feature is supported through [this PR](https://github.com/sgl-project/sglang/pull/21405) and introduces only a small accuracy loss. However, if you are running rigorous accuracy evaluations, it is not recommended to enable this feature. diff --git a/docs/cookbook/autoregressive/Moonshotai/Kimi-K2.5.mdx b/docs/cookbook/autoregressive/Moonshotai/Kimi-K2.5.mdx index ecf6f37be6e3..2eddfd6f7742 100644 --- a/docs/cookbook/autoregressive/Moonshotai/Kimi-K2.5.mdx +++ b/docs/cookbook/autoregressive/Moonshotai/Kimi-K2.5.mdx @@ -40,7 +40,7 @@ import { KimiK25Deployment } from '/src/snippets/autoregressive/kimi-k25-deploym - **Memory**: Requires GPUs with >=140GB each. Supported platforms: H200 (8x, TP=8), B300 (8x, TP=8), GB300 (4x, TP=4), MI300X/MI325X (4x, TP=4), MI350X/MI355X (4x, TP=4). Use `--context-length 128000` to conserve memory. - **AMD GPU TP Constraint**: On AMD GPUs, TP must be <= 4 (not 8). Kimi-K2.5 has 64 attention heads; the AITER MLA kernel requires `heads_per_gpu % 16 == 0`. With TP=4, each GPU gets 16 heads (valid). With TP=8, each GPU gets 8 heads (invalid). - **AMD Docker Image**: Use `lmsysorg/sglang:v0.5.9-rocm700-mi35x` for MI350X/MI355X and `lmsysorg/sglang:v0.5.9-rocm700-mi30x` for MI300X/MI325X. The ROCm 7.2 images (`rocm720`) have an AITER compatibility issue. -- **DP Attention**: Enable with `--dp --enable-dp-attention` for production throughput. A common choice is to set `--dp` equal to `--tp`, but this is not required. +- **DP Attention**: Enable with `--attn-dp-size ` for production throughput. A common choice is to set `--attn-dp-size` equal to `--tp`, but this is not required. - **Reasoning Parser**: Add `--reasoning-parser kimi_k2` to separate thinking and content in model outputs. - **Tool Call Parser**: Add `--tool-call-parser kimi_k2` for structured tool calls. diff --git a/docs/cookbook/autoregressive/Moonshotai/Kimi-K2.6.mdx b/docs/cookbook/autoregressive/Moonshotai/Kimi-K2.6.mdx index 696859a10180..df3aba605c0e 100644 --- a/docs/cookbook/autoregressive/Moonshotai/Kimi-K2.6.mdx +++ b/docs/cookbook/autoregressive/Moonshotai/Kimi-K2.6.mdx @@ -94,7 +94,7 @@ import { KimiK26Deployment } from '/src/snippets/autoregressive/kimi-k26-deploym - **NVFP4**: Use `nvidia/Kimi-K2.6-NVFP4` with `--quantization modelopt_fp4` on NVIDIA Blackwell. Use `tp=8` on B300 and `tp=4` on GB300. - **AMD GPU TP Constraint**: On AMD GPUs, TP must be ≤ 4 (not 8). Kimi-K2.6 has 64 attention heads; the AITER MLA kernel requires `heads_per_gpu % 16 == 0`. With TP=4, each GPU gets 16 heads (valid). With TP=8, each GPU gets 8 heads (invalid). - **AMD Docker Image**: Use `lmsysorg/sglang:v0.5.9-rocm700-mi35x` for MI350X/MI355X and `lmsysorg/sglang:v0.5.9-rocm700-mi30x` for MI300X/MI325X. -- **DP Attention**: Enable with `--dp --enable-dp-attention` for production throughput. A common choice is to set `--dp` equal to `--tp`, but this is not required. +- **DP Attention**: Enable with `--attn-dp-size ` for production throughput. A common choice is to set `--attn-dp-size` equal to `--tp`, but this is not required. - **Reasoning Parser**: Add `--reasoning-parser kimi_k2` to separate thinking and content in model outputs. - **Tool Call Parser**: Add `--tool-call-parser kimi_k2` for structured tool calls. - **AMD FP8 KV Cache**: On AMD platforms the generator adds `--kv-cache-dtype fp8_e4m3` by default and sets `--mem-fraction-static 0.8` to fit the INT4 weights plus KV cache. FP8 KV cache trades a small amount of accuracy for memory; omit the flag if you observe accuracy regressions on your workload. diff --git a/docs/cookbook/autoregressive/Moonshotai/Kimi-K2.7-Code.mdx b/docs/cookbook/autoregressive/Moonshotai/Kimi-K2.7-Code.mdx index 6d2b93eac7b7..4c1841b61924 100644 --- a/docs/cookbook/autoregressive/Moonshotai/Kimi-K2.7-Code.mdx +++ b/docs/cookbook/autoregressive/Moonshotai/Kimi-K2.7-Code.mdx @@ -94,7 +94,7 @@ import { KimiK27CodeDeployment } from '/src/snippets/autoregressive/kimi-k27-cod - **Transformers Version**: The model card requires `transformers>=4.57.1,<5.0.0`. - **AMD GPU TP Constraint**: On AMD GPUs, TP must be ≤ 4 (not 8). Kimi-K2.7-Code has 64 attention heads; the AITER MLA kernel requires `heads_per_gpu % 16 == 0`. With TP=4, each GPU gets 16 heads (valid). With TP=8, each GPU gets 8 heads (invalid). - **AMD Docker Image**: Use `lmsysorg/sglang:v0.5.9-rocm700-mi35x` for MI350X/MI355X and `lmsysorg/sglang:v0.5.9-rocm700-mi30x` for MI300X/MI325X. -- **DP Attention**: Enable with `--dp --enable-dp-attention` for production throughput. A common choice is to set `--dp` equal to `--tp`, but this is not required. +- **DP Attention**: Enable with `--attn-dp-size ` for production throughput. A common choice is to set `--attn-dp-size` equal to `--tp`, but this is not required. - **Reasoning Parser**: Add `--reasoning-parser kimi_k2` to separate thinking and content in model outputs. - **Tool Call Parser**: Add `--tool-call-parser kimi_k2` for structured tool calls. - **AMD FP8 KV Cache**: On AMD platforms the generator adds `--kv-cache-dtype fp8_e4m3` by default and sets `--mem-fraction-static 0.8` to fit the INT4 weights plus KV cache. FP8 KV cache trades a small amount of accuracy for memory; omit the flag if you observe accuracy regressions on your workload. diff --git a/docs/cookbook/autoregressive/Moonshotai/Kimi-K2.mdx b/docs/cookbook/autoregressive/Moonshotai/Kimi-K2.mdx index af58a3e85489..767b283c5ee8 100644 --- a/docs/cookbook/autoregressive/Moonshotai/Kimi-K2.mdx +++ b/docs/cookbook/autoregressive/Moonshotai/Kimi-K2.mdx @@ -36,7 +36,7 @@ This section provides a progressive guide from quick deployment to performance o - **Memory**: Requires 8 GPUs with ≥140GB each (H200/B200). Use `--context-length 128000` to conserve memory. - **Expert Parallelism (EP)**: Use `--ep` for better MoE throughput. See [EP docs](../../../docs/advanced_features/expert_parallelism). -- **Data Parallel (DP)**: Enable with `--dp 4 --enable-dp-attention` for production throughput. +- **Data Parallel (DP)**: Enable with `--attn-dp-size 4` for production throughput. - **KV Cache**: Use `--kv-cache-dtype fp8_e4m3` to reduce memory by 50% (CUDA 11.8+). - **Reasoning Parser**: Add `--reasoning-parser kimi_k2` for Kimi-K2-Thinking to separate thinking and content. - **Tool Call Parser**: Add `--tool-call-parser kimi_k2` for structured tool calls. @@ -359,8 +359,7 @@ We use SGLang's built-in benchmarking tool to conduct performance evaluation on python3 -m sglang.launch_server \ --model-path moonshotai/Kimi-K2-Instruct \ --tp 8 \ - --dp 4 \ - --enable-dp-attention \ + --attn-dp-size 4 \ --trust-remote-code \ --host 0.0.0.0 \ --port 8000 @@ -427,9 +426,8 @@ Max ITL (ms): 19.94 python3 -m sglang.launch_server \ --model-path moonshotai/Kimi-K2-Instruct \ --tp 8 \ - --dp 4 \ + --attn-dp-size 4 \ --ep 4 \ - --enable-dp-attention \ --trust-remote-code \ --host 0.0.0.0 \ --port 8000 diff --git a/docs/cookbook/autoregressive/Moonshotai/Kimi-K3.mdx b/docs/cookbook/autoregressive/Moonshotai/Kimi-K3.mdx index 6dd8da8b4fe3..00aa1a17235a 100644 --- a/docs/cookbook/autoregressive/Moonshotai/Kimi-K3.mdx +++ b/docs/cookbook/autoregressive/Moonshotai/Kimi-K3.mdx @@ -437,7 +437,7 @@ Two presets come out of this, at `N = 8k` GPUs: | **Peak Capacity (+DCP8)** — `dp = k` + `--dcp-size 8` | Deduplicates the attention-TP group's MLA KV: concurrency ceiling +72% at the same engine throughput, ~1.8× ITL. | Context ≥ ~16K, or per-replica concurrency past 128. | - **Radix cache** is independent of the preset: for prefix-free traffic (offline batch, evals) switch it off (Playground's **Prefix Cache** card) — one state slot per request instead of 4–5. -- The fully data-parallel extreme (`--dp-size` = GPU count, attention-TP 1) — the shape behind the 64-GPU sweep's ~3K tok/s per GPU — is not a preset: 288 GB GPUs only, radix forced off, no head-to-head against the preset shape. +- The fully data-parallel extreme (`--attn-dp-size` = GPU count, attention-TP 1) — the shape behind the 64-GPU sweep's ~3K tok/s per GPU — is not a preset: 288 GB GPUs only, radix forced off, no head-to-head against the preset shape. The Peak Throughput preset at 32 GPUs on B200/B300 (4 nodes × 8; every node runs the same command with its own `--node-rank`). On GB200/GB300 the same 32-GPU shape uses 8 nodes × 4, and the Playground emits `--nnodes 8`: @@ -447,7 +447,7 @@ sglang serve \ --trust-remote-code \ --model-path moonshotai/Kimi-K3 \ --tp-size 32 --ep-size 32 \ - --enable-dp-attention --dp-size 4 --enable-dp-lm-head \ + --attn-dp-size 4 --enable-dp-lm-head \ --nnodes 4 --node-rank --dist-init-addr :20000 \ --moe-a2a-backend megamoe --moe-runner-backend deep_gemm \ --kv-cache-dtype fp8_e4m3 \ @@ -460,7 +460,7 @@ sglang serve \ Scale by holding the per-replica shape fixed and moving only the replica count; pool sizing rides the calculator-driven `--mamba-full-memory-ratio`, which folds in DP, DCP, precision, and speculation: -| GPUs | B200/B300 nodes | GB200/GB300 nodes | `--tp-size` / `--ep-size` | `--dp-size` | +| GPUs | B200/B300 nodes | GB200/GB300 nodes | `--tp-size` / `--ep-size` | `--attn-dp-size` | |---|---|---|---|---| | 16 | 2×8 | 4×4 | 16 | 2 | | 32 | 4×8 | 8×4 | 32 | 4 | diff --git a/docs/cookbook/autoregressive/NVIDIA/Nemotron3-Ultra.mdx b/docs/cookbook/autoregressive/NVIDIA/Nemotron3-Ultra.mdx index 68af9b0ac802..55a2912027f5 100644 --- a/docs/cookbook/autoregressive/NVIDIA/Nemotron3-Ultra.mdx +++ b/docs/cookbook/autoregressive/NVIDIA/Nemotron3-Ultra.mdx @@ -86,7 +86,7 @@ The generator only emits a runnable command for combinations that NVIDIA / SGLan - **DP attention**: - By default the attention layers are tensor-parallel (sharded across all TP ranks). Enabling DP attention (the toggle above, or `--dp --enable-dp-attention`) instead runs attention as `N` data-parallel groups: each DP rank serves its own slice of the requests with its own KV cache. `--dp` must divide `--tp`. + By default the attention layers are tensor-parallel (sharded across all TP ranks). Enabling DP attention (the toggle above, or `--attn-dp-size `) instead runs attention as `N` data-parallel groups: each DP rank serves its own slice of the requests with its own KV cache. `--attn-dp-size` must divide `--tp`. - **Expert parallel (EP)**: diff --git a/docs/cookbook/autoregressive/Poolside/Laguna-XS.2.mdx b/docs/cookbook/autoregressive/Poolside/Laguna-XS.2.mdx index eeddd0d7cb5d..62d9eef229ac 100644 --- a/docs/cookbook/autoregressive/Poolside/Laguna-XS.2.mdx +++ b/docs/cookbook/autoregressive/Poolside/Laguna-XS.2.mdx @@ -84,7 +84,7 @@ import { LagunaXS2Deployment } from '/src/snippets/autoregressive/laguna-xs2-dep - **Quantization**: NVFP4 requires Blackwell (B200 / B300); BF16 and FP8 run on either H200 or B200. FP8's first launch triggers a multi-session DeepGEMM JIT pre-compile (~10-20 min); pre-warm with `python3 -m sglang.compile_deep_gemm --model poolside/Laguna-XS.2-FP8` to avoid that cost on every restart. - **Reasoning parser** (`--reasoning-parser poolside_v1`): Splits `...` segments into `reasoning_content` so `content` holds only the final answer. Disable only if you want the raw `` tags in `content`. - **Tool call parser** (`--tool-call-parser poolside_v1`): Required for OpenAI-compatible tool-call streaming. Disable only for chat-only deployments. -- **DP attention**: For higher-throughput deployments, enable the DP-Attention toggle — it emits `--dp --enable-dp-attention` with `--dp` matching `--tp` (tune independently if needed). +- **DP attention**: For higher-throughput deployments, enable the DP-Attention toggle — it emits `--attn-dp-size ` with `--attn-dp-size` matching `--tp` (tune independently if needed). - **Thinking default**: Thinking is **off by default** at the model level. Opt in per request with `extra_body={"chat_template_kwargs": {"enable_thinking": True}}`. ## 4. Model Invocation diff --git a/docs/cookbook/autoregressive/Qwen/Qwen3-Coder.mdx b/docs/cookbook/autoregressive/Qwen/Qwen3-Coder.mdx index 9a0d8974da44..a743f7ba3eb3 100644 --- a/docs/cookbook/autoregressive/Qwen/Qwen3-Coder.mdx +++ b/docs/cookbook/autoregressive/Qwen/Qwen3-Coder.mdx @@ -51,7 +51,7 @@ This section provides deployment configurations verified on AMD MI300X, MI325X, **NVIDIA (B200/GB200):** * **GB200 Parallelism**: Use `--tp 4 --ep 4` on GB200. B200 uses the default NVIDIA settings generated above. * **NVFP4 Quantization**: Requires `--quantization modelopt_fp4` and uses a different model path (`nvidia/Qwen3-Coder-...`). -* **DP Attention**: NVFP4 configuration supports `--enable-dp-attention` for improved throughput. +* **DP Attention**: NVFP4 configuration supports DP attention (`--attn-dp-size `) for improved throughput. **Intel Xeon CPU:** * Please refer to the `Notes` part in the serving engine launching section in [the SGLang CPU server document](../../../docs/hardware-platforms/cpu_server#launch-of-the-serving-engine) to better understand how to configure the arguments, especially for TP (tensor parallel) and NUMA binding settings. diff --git a/docs/cookbook/autoregressive/Xiaomi/MiMo-V2.5.mdx b/docs/cookbook/autoregressive/Xiaomi/MiMo-V2.5.mdx index 53067a9a0f37..dffd8cda2913 100644 --- a/docs/cookbook/autoregressive/Xiaomi/MiMo-V2.5.mdx +++ b/docs/cookbook/autoregressive/Xiaomi/MiMo-V2.5.mdx @@ -76,12 +76,12 @@ import { MiMoV25Deployment } from '/src/snippets/autoregressive/mimo-v25-deploym - EAGLE speculative decoding (3 steps, topk=1) typically yields a 2–3× decode speedup. Requires `--enable-multi-layer-eagle` (both Hopper and Blackwell). See §5.4 for acceptance-rate behavior on natural text vs random prompts. **MiMo-V2.5 (310B):** -- The checkpoint has a TP=4-interleaved fused `qkv_proj`; attention-TP per DP group **must** be 4. Use `--dp = TP / 4`; for TP > 4 this also requires DP-attention. Total GPUs must be a multiple of 4. A bare `--tp 8` without `--dp 2` will fail to load with `MiMoV2 fused qkv_proj checkpoint is TP=4-interleaved; got attention tp_size=8`. -- Single-node deployments: H100/H200 8× GPUs (`--tp 8 --dp 2`), B200 4× GPUs (`--tp 4`, dp=1, no DP-attn flag needed), GB300 4× GPUs (`--tp 4`, single NVL4 node). FP8 quantization. +- The checkpoint has a TP=4-interleaved fused `qkv_proj`; attention-TP per DP group **must** be 4. Use `--attn-dp-size = TP / 4`; for TP > 4 this means DP-attention is on. Total GPUs must be a multiple of 4. A bare `--tp 8` without `--attn-dp-size 2` will fail to load with `MiMoV2 fused qkv_proj checkpoint is TP=4-interleaved; got attention tp_size=8`. +- Single-node deployments: H100/H200 8× GPUs (`--tp 8 --attn-dp-size 2`), B200 4× GPUs (`--tp 4`, attention DP size 1, no DP-attn flag needed), GB300 4× GPUs (`--tp 4`, single NVL4 node). FP8 quantization. - On Blackwell, pass `--attention-backend fa4`: MiMoV2's asymmetric KV (`head_dim` 192 / `v_head_dim` 128) fails on the SM100 default `trtllm_mha`, which requires equal K/V widths. - On Blackwell, pass `--mm-attention-backend fa4` for the V2.5 vision encoder. The checkpoint config requests FlashAttention-3 internally, but SGLang rejects FA3 on Blackwell and expects FA4 for multimodal attention. - On Blackwell, pass `--moe-runner-backend flashinfer_trtllm`; the default `auto` falls through to the triton fused-MoE runner, ~12% slower at bs=1 decode. -- `--enable-dp-lm-head` and `--mm-enable-dp-encoder` are required whenever `--enable-dp-attention` is on, to keep LM head and encoder sharding consistent. +- `--enable-dp-lm-head` and `--mm-enable-dp-encoder` are required whenever `--attn-dp-size` is greater than 1, to keep LM head and encoder sharding consistent. - EAGLE MTP uses the checkpoint's MTP weights. Enable with `--speculative-algorithm EAGLE` and `--enable-multi-layer-eagle` (both Hopper and Blackwell). - **Multimodal**: Supports image, video, and audio understanding; see Section 4.3 for invocation examples. @@ -547,7 +547,7 @@ Max ITL (ms): 1600.00 - Hardware: NVIDIA H200 GPU (8×) - Model: `XiaomiMiMo/MiMo-V2.5` (FP8) -- Tensor Parallelism: 8 (DP-attention with `--dp 2`) +- Tensor Parallelism: 8 (DP-attention with `--attn-dp-size 2`) - Recipe: Balanced (DP-attn + EAGLE MTP) - sglang version: `0.0.0.dev1+g7d99af439` (`lmsysorg/sglang:dev-mimo-v2.5`) @@ -777,7 +777,7 @@ Reference numbers from the [day0 enablement PR](https://github.com/sgl-project/s - Hardware: NVIDIA Hopper GPU (2 nodes × 8 GPUs, GPU SKU intentionally not disclosed) - Model: `XiaomiMiMo/MiMo-V2.5-Pro` (FP8) -- Parallelism: `--tp 16 --dp 2 --ep 16 --moe-dense-tp-size 1 --enable-dp-attention` +- Parallelism: `--tp 16 --attn-dp-size 2 --ep 16 --moe-dense-tp-size 1` - Recipe: Hopper EP16 (DeepEP + EAGLE multi-layer MTP) #### 5.5.1 Long-Context Prefill Throughput diff --git a/docs/cookbook/autoregressive/Xiaomi/MiMo-V2.6.mdx b/docs/cookbook/autoregressive/Xiaomi/MiMo-V2.6.mdx index 475f269eaf47..2ae5966dde99 100644 --- a/docs/cookbook/autoregressive/Xiaomi/MiMo-V2.6.mdx +++ b/docs/cookbook/autoregressive/Xiaomi/MiMo-V2.6.mdx @@ -113,7 +113,7 @@ The shared prefill settings are `--chunked-prefill-size 49152 --max-prefill-toke The base recipe uses `--speculative-algorithm DFLASH --speculative-num-draft-tokens 8`. Supply the DFlash checkpoint paired with the selected target model; the draft path is independently editable and does not have to be a subdirectory of the target checkpoint. -At the pinned commit, DFlash on NVIDIA GPUs requires PP=1 and rejects DP-attention. Do not add `--enable-dp-attention` or copy EAGLE-specific settings from the V2.5 cookbook. +At the pinned commit, DFlash on NVIDIA GPUs requires PP=1 and rejects DP-attention. Do not set `--attn-dp-size` above 1 or copy EAGLE-specific settings from the V2.5 cookbook. ## 3. Reasoning and tool calling diff --git a/docs/cookbook/base/reference/server_arguments.mdx b/docs/cookbook/base/reference/server_arguments.mdx index 737a1b35f4d0..d08442308b87 100644 --- a/docs/cookbook/base/reference/server_arguments.mdx +++ b/docs/cookbook/base/reference/server_arguments.mdx @@ -37,10 +37,15 @@ This guide explains the parallelism configuration fields used in SGLang model co `--ep-size`, `--expert-parallel-size`, `--ep` Expert Parallelism - distributes MoE experts + + `attn_dp_size` + `--attn-dp-size`, `--attention-data-parallel-size` + Attention Data Parallelism - data-parallel attention groups inside the TP group, TP for FFN (hybrid) + `enable_dp_attention` `--enable-dp-attention` - DP for attention, TP for FFN (hybrid) + Deprecated. Use --attn-dp-size <dp-size> instead of --dp-size <dp-size> --enable-dp-attention. diff --git a/docs/docs/advanced_features/dcp.mdx b/docs/docs/advanced_features/dcp.mdx index 262b69c4c295..d88bcb588e0e 100644 --- a/docs/docs/advanced_features/dcp.mdx +++ b/docs/docs/advanced_features/dcp.mdx @@ -17,7 +17,7 @@ On Kimi K3, DCP applies only to MLA layers. Request-indexed KDA state is unchang | `--dcp-size N` | Enables DCP and widens virtual capacity and page size by `N`. Alias: `--decode-context-parallel-size`. | | `--dcp-comm-backend` | Selects the MLA partial-output merge: `ag_rs`, `a2a`, or `fi_a2a`. Unset resolves to `fi_a2a` where the DCP group shares an MNNVL domain on Blackwell, otherwise `a2a` on CUDA/ROCm. | | `--dcp-replicate-q-proj` | Removes the query all-gather for supported MLA weights. Use `--no-dcp-replicate-q-proj` to disable a model-specific default. | -| `--enable-dp-attention` | Composes only when DCP groups nest inside attention TP. | +| `--attn-dp-size N` | Composes only when DCP groups nest inside attention TP. | A DCP group must fit inside one attention-TP group and one attention-DP replica: @@ -26,7 +26,7 @@ attn_tp_size = tp_size / attn_dp_size attn_tp_size % dcp_size == 0 ``` -For example, `TP=64, DP=4, DCP=16` creates four valid 16-rank attention replicas. `DCP=32` would cross replica boundaries and is invalid. +For example, `TP=64, attention DP=4, DCP=16` creates four valid 16-rank attention replicas. `DCP=32` would cross replica boundaries and is invalid. Startup currently checks only `tp_size % dcp_size == 0`, not the stronger condition above. Until that check is fixed, DPA+DCP deployments must enforce containment when choosing the topology. @@ -52,8 +52,7 @@ sglang serve \ --model-path moonshotai/Kimi-K3 \ --tp-size 32 \ --ep-size 32 \ - --enable-dp-attention \ - --dp-size 4 \ + --attn-dp-size 4 \ --dcp-size 8 \ --host 0.0.0.0 \ --port 30000 diff --git a/docs/docs/advanced_features/dp_dpa_smg_guide.mdx b/docs/docs/advanced_features/dp_dpa_smg_guide.mdx index 6fc3fc5fe141..896b533d769e 100644 --- a/docs/docs/advanced_features/dp_dpa_smg_guide.mdx +++ b/docs/docs/advanced_features/dp_dpa_smg_guide.mdx @@ -80,14 +80,13 @@ For MoE models like DeepSeek, DPA is **often** paired with Expert Parallelism (E python -m sglang.launch_server \ --model-path deepseek-ai/DeepSeek-V3 \ --tp 8 \ - --dp-size 8 \ + --attn-dp-size 8 \ --ep 8 \ - --enable-dp-attention \ --moe-a2a-backend deepep \ --moe-runner-backend deep_gemm ``` -> **Note**: `--dp-size` must be explicitly set when using `--enable-dp-attention`. If `dp_size` is 1 (default), DPA will be disabled. +> **Note**: `--attn-dp-size` sets the number of attention data-parallel groups inside the TP group. If `attn_dp_size` is 1 (default), DPA is disabled. For detailed EP configuration (DeepEP, Two-Batch Overlap, EPLB), see [Expert Parallelism](/docs/advanced_features/expert_parallelism). @@ -106,21 +105,28 @@ DPA supports the following model architectures: For models like Llama, with standard GQA, standard DP, or TP is typically recommended. -To enable DPA, add `--enable-dp-attention` to your server launch command. +To enable DPA, add `--attn-dp-size N` (N > 1) to your server launch command. ### Activation Logic -DPA is enabled explicitly via server arguments (CLI or config). You must set both `--dp-size` and `--enable-dp-attention`: +DPA is enabled explicitly via server arguments (CLI or config) with `--attn-dp-size` (alias `--attention-data-parallel-size`). It creates N attention data-parallel groups inside the TP group, one scheduler each, while the FFN stays tensor parallel: ```bash python -m sglang.launch_server \ --model-path deepseek-ai/DeepSeek-V3 \ --tp 8 \ - --dp-size 8 \ - --enable-dp-attention + --attn-dp-size 8 ``` -**Important**: `--dp-size` must be greater than 1 for DPA to work. When `dp_size == 1` (default), `--enable-dp-attention` is automatically disabled. The constraint `tp_size % dp_size == 0` must also be satisfied. +**Important**: `--attn-dp-size` must be greater than 1 for DPA to work. When `attn_dp_size == 1` (default), DPA is disabled. The constraint `tp_size % attn_dp_size == 0` must also be satisfied. `--dp-size` only sets the number of data-parallel replicas of the model; combining replicas (`--dp-size` > 1) with DPA is not supported. + +> **Deprecated**: `--enable-dp-attention` still works: `--dp-size N --enable-dp-attention` resolves to `--attn-dp-size N --dp-size 1` and prints a deprecation warning. Use `--attn-dp-size N` instead. + +**Elastic EP scale joiners.** A rank that joins an existing DPA deployment through elastic EP scale-up runs DPA even when it is launched with `--attn-dp-size 1`: its own attention DP group is one rank wide. + +**DP rank count.** The number of DP ranks (one scheduler each) is `dp_size × attn_dp_size`; one of the two is always 1. +- `/server_info` reports `dp_size` (replicas) and `attn_dp_size` separately, so under DPA its `dp_size` is 1. Its deprecated `enable_dp_attention` field is always `false`; read `attn_dp_size` instead. +- Wire formats that predate `--attn-dp-size` keep the key `dp_size` for the DP rank count: KV-event descriptors, the PD prefill server info, and load snapshots. ### TP LMHead with all-to-all @@ -135,8 +141,7 @@ python -m sglang.launch_server \ --model-path MODEL_PATH \ --disaggregation-mode decode \ --tp 8 \ - --dp-size 8 \ - --enable-dp-attention + --attn-dp-size 8 ``` Prefill / colocated workers will fallback to TP LMHead with allgather, as well as other parallel strategies. More details can be found in [PR #32313](https://github.com/sgl-project/sglang/pull/32313). @@ -505,7 +510,7 @@ For detailed metrics and monitoring setup, see [SGLang Model Gateway Documentati Cache-aware routing, high availability - DPA (--dp-size N --enable-dp-attention) + DPA (--attn-dp-size N) DeepSeek/MLA models Eliminates KV cache duplication, improved throughput @@ -518,7 +523,7 @@ For detailed metrics and monitoring setup, see [SGLang Model Gateway Documentati **Recommended production setup for DeepSeek:** -1. Enable **DPA** for attention layers (`--dp-size 8 --enable-dp-attention`) +1. Enable **DPA** for attention layers (`--attn-dp-size 8`) 2. Enable **EP** for MoE layers (`--ep 8 --moe-a2a-backend deepep`) 3. Use **SMG** with **cache_aware** policy diff --git a/docs/docs/advanced_features/expert_parallelism.mdx b/docs/docs/advanced_features/expert_parallelism.mdx index 26e38cf017e1..077884fd9f17 100644 --- a/docs/docs/advanced_features/expert_parallelism.mdx +++ b/docs/docs/advanced_features/expert_parallelism.mdx @@ -72,7 +72,7 @@ SGLang's EP integrates diverse, highly efficient backends for different use case DeepEP and Mooncake backends support two modes for token dispatch: `normal` mode (optimized for prefill workloads with high throughput) and `low_latency` mode (optimized for decode workloads with low latency and CUDA Graph compatibility). MORI backend only supports `normal` mode now. NIXL-EP and PPLX currently operate in low-latency mode with CUDA Graph support (PPLX reuses the DeepEP low-latency masked expert-compute path). Users are recommended to set `--deepep-mode auto` to enable automatic dispatch mode switching during runtime. Setting `--deepep-mode normal` or `--deepep-mode low_latency` is useful for debugging or development purposes. -Currently, DeepEP, Mooncake, NIXL-EP, `ascend_fuseep`, `pplx` and MORI only support cases where `ep_size = tp_size`. For hybrid EP and TP (i.e., `ep_size < tp_size`), only the `none` backend (All-Reduce or All-Gather-based dispatching) is supported. Note that `pplx` additionally requires `--enable-dp-attention` with at least 2 DP groups (i.e., `tp_size / attention_tp_size > 1`); otherwise pplx-kernels' AllToAll cannot be constructed. +Currently, DeepEP, Mooncake, NIXL-EP, `ascend_fuseep`, `pplx` and MORI only support cases where `ep_size = tp_size`. For hybrid EP and TP (i.e., `ep_size < tp_size`), only the `none` backend (All-Reduce or All-Gather-based dispatching) is supported. Note that `pplx` additionally requires DP attention with at least 2 attention DP groups (i.e., `--attn-dp-size` > 1, so `tp_size / attention_tp_size > 1`); otherwise pplx-kernels' AllToAll cannot be constructed. ### Backends for MoE Computation diff --git a/docs/docs/advanced_features/hisparse_guide.mdx b/docs/docs/advanced_features/hisparse_guide.mdx index 54c412bafd23..2fe612f2114c 100644 --- a/docs/docs/advanced_features/hisparse_guide.mdx +++ b/docs/docs/advanced_features/hisparse_guide.mdx @@ -137,7 +137,7 @@ python3 -m sglang.launch_server \ --port 8000 --host 0.0.0.0 \ --context-length 81920 \ --chunked-prefill-size 65536 \ - --tp-size 8 --dp-size 8 --enable-dp-attention \ + --tp-size 8 --attn-dp-size 8 \ --mem-fraction-static 0.85 \ --disaggregation-mode prefill \ --disaggregation-ib-device mlx5_0,mlx5_1,mlx5_2,mlx5_3 \ @@ -152,7 +152,7 @@ python3 -m sglang.launch_server \ --trust-remote-code \ --port 8000 --host 0.0.0.0 \ --context-length 81920 \ - --tp-size 8 --dp-size 8 --enable-dp-attention \ + --tp-size 8 --attn-dp-size 8 \ --mem-fraction-static 0.85 \ --disable-radix-cache \ --disaggregation-mode decode \ diff --git a/docs/docs/advanced_features/hyperparameter_tuning.mdx b/docs/docs/advanced_features/hyperparameter_tuning.mdx index 327e2efce995..db3952a872cc 100644 --- a/docs/docs/advanced_features/hyperparameter_tuning.mdx +++ b/docs/docs/advanced_features/hyperparameter_tuning.mdx @@ -78,5 +78,5 @@ Data parallelism is better for throughput. When there is enough GPU memory, alwa - `torch.compile` accelerates small models on small batch sizes. You can enable it with `--enable-torch-compile`. - Try other quantization (e.g. FP8 quantization with `--quantization fp8`) -- Try other parallelism strategies (e.g. [expert parallelism](https://lmsys.org/blog/2025-05-05-large-scale-ep/)) or DP attention for deepseek models (with `--enable-dp-attention --dp-size 8`). +- Try other parallelism strategies (e.g. [expert parallelism](https://lmsys.org/blog/2025-05-05-large-scale-ep/)) or DP attention for deepseek models (with `--attn-dp-size 8`). - If the workload has many shared prefixes, try `--schedule-policy lpm`. Here, `lpm` stands for longest prefix match. It reorders requests to encourage more cache hits but introduces more scheduling overhead. diff --git a/docs/docs/advanced_features/lora.mdx b/docs/docs/advanced_features/lora.mdx index 3d318ab3b4de..2cfcdc081820 100644 --- a/docs/docs/advanced_features/lora.mdx +++ b/docs/docs/advanced_features/lora.mdx @@ -422,8 +422,7 @@ The following NVIDIA CUDA example uses four attention-DP ranks: python3 -m sglang.launch_server \ --model-path MODEL_PATH \ --tp 4 \ - --dp-size 4 \ - --enable-dp-attention \ + --attn-dp-size 4 \ --enable-lora \ --lora-backend triton \ --lora-paths '{"lora_name":"ADAPTER_NAME","lora_path":"ADAPTER_PATH","pinned":true}' @@ -431,7 +430,7 @@ python3 -m sglang.launch_server \ Keep these constraints in mind: -- `--dp-size` must equal `--tp-size`, so every attention DP rank owns its whole token batch. The server rejects other layouts at startup. +- `--attn-dp-size` must equal `--tp-size`, so every attention DP rank owns its whole token batch. The server rejects other layouts at startup. - All adapters must be GPU-pinned (`pinned: true`), including dynamically loaded adapters. You can pin at most `--max-loras-per-batch - 1` adapters; the remaining slot is reserved for the base model. - Decode CUDA graphs are supported. LoRA prefill runs eagerly when DPA is enabled. - `--max-loras-per-batch` limits the union of adapters active across all DP ranks in a synchronized forward, not the adapters on one rank alone. diff --git a/docs/docs/advanced_features/pd_disaggregation.mdx b/docs/docs/advanced_features/pd_disaggregation.mdx index 7fd878bc94c6..5ce40a82e4e5 100644 --- a/docs/docs/advanced_features/pd_disaggregation.mdx +++ b/docs/docs/advanced_features/pd_disaggregation.mdx @@ -132,8 +132,7 @@ python -m sglang.launch_server \ --nnodes 2 \ --node-rank 0 \ --tp-size 16 \ - --dp-size 8 \ - --enable-dp-attention \ + --attn-dp-size 8 \ --moe-a2a-backend deepep \ --mem-fraction-static 0.8 # prefill 1 @@ -148,8 +147,7 @@ python -m sglang.launch_server \ --nnodes 2 \ --node-rank 1 \ --tp-size 16 \ - --dp-size 8 \ - --enable-dp-attention \ + --attn-dp-size 8 \ --moe-a2a-backend deepep \ --mem-fraction-static 0.8 # decode 0 @@ -164,8 +162,7 @@ python -m sglang.launch_server \ --nnodes 2 \ --node-rank 0 \ --tp-size 16 \ - --dp-size 8 \ - --enable-dp-attention \ + --attn-dp-size 8 \ --moe-a2a-backend deepep \ --mem-fraction-static 0.8 \ --max-running-requests 128 @@ -181,8 +178,7 @@ python -m sglang.launch_server \ --nnodes 2 \ --node-rank 1 \ --tp-size 16 \ - --dp-size 8 \ - --enable-dp-attention \ + --attn-dp-size 8 \ --moe-a2a-backend deepep \ --mem-fraction-static 0.8 \ --max-running-requests 128 @@ -348,8 +344,7 @@ python -m sglang.launch_server \ --disaggregation-mode decode \ --port 30001 \ --tp 4 \ - --dp 4 \ - --enable-dp-attention \ + --attn-dp-size 4 \ --trust-remote-code \ --disaggregation-ib-device mlx5_3,mlx5_4 @@ -413,8 +408,7 @@ python -m sglang.launch_server \ --nnodes 2 \ --node-rank 0 \ --tp-size 16 \ - --dp-size 8 \ - --enable-dp-attention \ + --attn-dp-size 8 \ --moe-a2a-backend deepep \ --mem-fraction-static 0.8 # prefill 1 @@ -429,8 +423,7 @@ python -m sglang.launch_server \ --nnodes 2 \ --node-rank 1 \ --tp-size 16 \ - --dp-size 8 \ - --enable-dp-attention \ + --attn-dp-size 8 \ --moe-a2a-backend deepep \ --mem-fraction-static 0.8 # decode 0 @@ -445,8 +438,7 @@ python -m sglang.launch_server \ --nnodes 2 \ --node-rank 0 \ --tp-size 16 \ - --dp-size 8 \ - --enable-dp-attention \ + --attn-dp-size 8 \ --moe-a2a-backend deepep \ --mem-fraction-static 0.8 \ --max-running-requests 128 @@ -462,8 +454,7 @@ python -m sglang.launch_server \ --nnodes 2 \ --node-rank 1 \ --tp-size 16 \ - --dp-size 8 \ - --enable-dp-attention \ + --attn-dp-size 8 \ --moe-a2a-backend deepep \ --mem-fraction-static 0.8 \ --max-running-requests 128 @@ -706,7 +697,7 @@ do --disaggregation-transfer-backend ascend --max-running-requests 32 --context-length 8192 --disable-radix-cache \ --chunked-prefill-size -1 --max-prefill-tokens 10240 --moe-a2a-backend deepep --deepep-mode normal \ --speculative-algorithm NEXTN --speculative-num-steps 1 --speculative-eagle-topk 1 --speculative-num-draft-tokens 2 \ - --dp-size 8 --enable-dp-attention --disable-shared-experts-fusion --dtype bfloat16 + --attn-dp-size 8 --disable-shared-experts-fusion --dtype bfloat16 NODE_RANK=$i break fi @@ -730,9 +721,9 @@ do export HCCL_SOCKET_IFNAME=data0.3001 export GLOO_SOCKET_IFNAME=data0.3001 python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode decode --host ${D_IP[$i]} \ - --port 8001 --trust-remote-code --nnodes 1 --node-rank 0 --tp-size 16 --dp-size 16 \ + --port 8001 --trust-remote-code --nnodes 1 --node-rank 0 --tp-size 16 --attn-dp-size 16 \ --mem-fraction-static 0.8 --max-running-requests 448 --attention-backend ascend --device npu --quantization modelslim \ - --moe-a2a-backend deepep --enable-dp-attention --deepep-mode low_latency --enable-dp-lm-head \ + --moe-a2a-backend deepep --deepep-mode low_latency --enable-dp-lm-head \ --cuda-graph-bs-decode 2 4 6 8 10 12 14 16 18 20 22 24 26 28 --disaggregation-transfer-backend ascend --watchdog-timeout 9000 --context-length 8192 \ --speculative-algorithm NEXTN --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 --disable-shared-experts-fusion --dtype bfloat16 --tokenizer-worker-num 4 \ --load-balance-method round_robin diff --git a/docs/docs/advanced_features/server_arguments.mdx b/docs/docs/advanced_features/server_arguments.mdx index 15f0543c4b7c..8f4a52b5e008 100644 --- a/docs/docs/advanced_features/server_arguments.mdx +++ b/docs/docs/advanced_features/server_arguments.mdx @@ -1220,7 +1220,13 @@ Combining `--enable-response-store` with `--disaggregation-mode=prefill` or `dec `--data-parallel-size`
`--dp-size` - The data parallelism size. + The number of data-parallel replicas of the model. + `1` + Type: int + + + `--attention-data-parallel-size`
`--attn-dp-size` + The attention data parallelism size: the number of data-parallel attention groups inside the TP group, one scheduler each, while the FFN stays tensor parallel. `1` Type: int @@ -2564,8 +2570,8 @@ Combining `--enable-response-store` with `--disaggregation-mode=prefill` or `dec `--enable-dp-attention` - Enabling data parallelism for attention and tensor parallelism for FFN. The dp size should be equal to the tp size. Currently DeepSeek-V2 and Qwen 2/3 MoE models are supported. - `False` + Deprecated. Use --attn-dp-size <dp-size> instead of --dp-size <dp-size> --enable-dp-attention. + False bool flag (set to enable) diff --git a/docs/docs/advanced_features/speculative_decoding.mdx b/docs/docs/advanced_features/speculative_decoding.mdx index 727d091bf657..c3dff0450299 100644 --- a/docs/docs/advanced_features/speculative_decoding.mdx +++ b/docs/docs/advanced_features/speculative_decoding.mdx @@ -100,21 +100,21 @@ SGLang provides several speculative decoding options, including EAGLE-2/EAGLE-3, DFlash draft model (linear block verification) Yes --speculative-algorithm DFLASH + --speculative-draft-model-path ... - No --enable-dp-attention; pp_size == 1; disables overlap scheduler & mixed chunked prefill + No DP attention (--attn-dp-size > 1); pp_size == 1; disables overlap scheduler & mixed chunked prefill STANDALONE Smaller draft LLM (token-level) Yes --speculative-algorithm STANDALONE + --speculative-draft-model-path ... - Does not support --enable-dp-attention + Does not support DP attention (--attn-dp-size > 1) NGRAM Ngram cache from previous tokens No --speculative-algorithm NGRAM - CUDA-only; no --enable-dp-attention; disables overlap scheduler & mixed chunked prefill + CUDA-only; no DP attention (--attn-dp-size > 1); disables overlap scheduler & mixed chunked prefill @@ -678,7 +678,7 @@ Relevant parameters: -> **Note:** Standalone speculative decoding currently **does not support** `--enable-dp-attention`. +> **Note:** Standalone speculative decoding currently **does not support** DP attention (`--attn-dp-size` > 1). ```bash Command python3 -m sglang.launch_server \ @@ -815,7 +815,7 @@ Enable it with: Notes: - Ngram speculative decoding **only supports CUDA**. -- It currently **does not support** `--enable-dp-attention`. +- It currently **does not support** DP attention (`--attn-dp-size` > 1). - It disables the overlap scheduler and mixed chunked prefill. - If `--speculative-ngram-max-bfs-breadth > 1` (thus `speculative_eagle_topk > 1`) and `page_size > 1`, use `--attention-backend flashinfer`; otherwise the server will error. - Optional: set `SGLANG_NGRAM_FORCE_GREEDY_VERIFY=True` to force greedy verification. diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_r1.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_r1.mdx index 6c2a8bb081d3..1767d1fbd0fc 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_r1.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_r1.mdx @@ -125,8 +125,7 @@ do --speculative-num-steps 1 \ --speculative-eagle-topk 1 \ --speculative-num-draft-tokens 2 \ - --dp-size 8 \ - --enable-dp-attention \ + --attn-dp-size 8 \ --disable-shared-experts-fusion \ --dtype bfloat16 \ --reasoning-parser deepseek-r1 \ @@ -160,12 +159,11 @@ do --port 8001 \ --nnodes 1 \ --tp-size 16 \ - --dp-size 16 \ + --attn-dp-size 16 \ --mem-fraction-static 0.9 \ --max-running-requests 416 \ --quantization modelslim \ --moe-a2a-backend deepep \ - --enable-dp-attention \ --deepep-mode low_latency \ --enable-dp-lm-head \ --cuda-graph-bs-decode 2 4 6 8 10 12 14 16 18 20 22 24 26 \ @@ -302,8 +300,7 @@ python3 -m sglang.launch_server \ --max-prefill-tokens 3000 \ --moe-a2a-backend deepep \ --deepep-mode auto \ - --enable-dp-attention \ - --dp-size 16 \ + --attn-dp-size 16 \ --enable-dp-lm-head \ --speculative-algorithm NEXTN \ --speculative-num-steps 3 \ @@ -430,8 +427,7 @@ do --speculative-num-steps 1 \ --speculative-eagle-topk 1 \ --speculative-num-draft-tokens 2 \ - --dp-size 2 \ - --enable-dp-attention \ + --attn-dp-size 2 \ --disable-shared-experts-fusion \ --dtype bfloat16 \ --enable-attn-tp-input-scattered \ @@ -468,12 +464,11 @@ do --node-rank $i \ --nnodes 2 \ --tp-size 32 \ - --dp-size 16 \ + --attn-dp-size 16 \ --mem-fraction-static 0.75 \ --max-running-requests 32 \ --quantization modelslim \ --moe-a2a-backend deepep \ - --enable-dp-attention \ --deepep-mode low_latency \ --enable-dp-lm-head \ --moe-dense-tp 1 \ @@ -636,8 +631,7 @@ do --speculative-num-steps 1 \ --speculative-eagle-topk 1 \ --speculative-num-draft-tokens 2 \ - --dp-size 4 \ - --enable-dp-attention \ + --attn-dp-size 4 \ --disable-shared-experts-fusion \ --dtype bfloat16 \ --enable-attn-tp-input-scattered \ @@ -675,12 +669,11 @@ do --node-rank $i \ --nnodes 2 \ --tp-size 32 \ - --dp-size 32 \ + --attn-dp-size 32 \ --mem-fraction-static 0.82 \ --max-running-requests 1024 \ --quantization modelslim \ --moe-a2a-backend ascend_fuseep \ - --enable-dp-attention \ --deepep-mode low_latency \ --moe-dense-tp 1 \ --cuda-graph-bs-decode 2 4 6 8 10 12 14 16 18 20 22 24 26 28 30 32 \ @@ -838,8 +831,7 @@ do --speculative-num-steps 1 \ --speculative-eagle-topk 1 \ --speculative-num-draft-tokens 2 \ - --dp-size 2 \ - --enable-dp-attention \ + --attn-dp-size 2 \ --disable-shared-experts-fusion \ --dtype bfloat16 \ --enable-attn-tp-input-scattered \ @@ -876,12 +868,11 @@ do --node-rank $i \ --nnodes 2 \ --tp-size 32 \ - --dp-size 16 \ + --attn-dp-size 16 \ --mem-fraction-static 0.75 \ --max-running-requests 32 \ --quantization modelslim \ --moe-a2a-backend deepep \ - --enable-dp-attention \ --deepep-mode low_latency \ --enable-dp-lm-head \ --moe-dense-tp 1 \ @@ -1040,8 +1031,7 @@ do --speculative-num-steps 1 \ --speculative-eagle-topk 1 \ --speculative-num-draft-tokens 2 \ - --dp-size 2 \ - --enable-dp-attention \ + --attn-dp-size 2 \ --disable-shared-experts-fusion \ --dtype bfloat16 \ --enable-attn-tp-input-scattered \ @@ -1078,12 +1068,11 @@ do --node-rank $i \ --nnodes 2 \ --tp-size 32 \ - --dp-size 16 \ + --attn-dp-size 16 \ --mem-fraction-static 0.75 \ --max-running-requests 32 \ --quantization modelslim \ --moe-a2a-backend deepep \ - --enable-dp-attention \ --deepep-mode low_latency \ --enable-dp-lm-head \ --moe-dense-tp 1 \ @@ -1241,8 +1230,7 @@ do --speculative-num-steps 1 \ --speculative-eagle-topk 1 \ --speculative-num-draft-tokens 2 \ - --dp-size 2 \ - --enable-dp-attention \ + --attn-dp-size 2 \ --disable-shared-experts-fusion \ --dtype bfloat16 \ --enable-attn-tp-input-scattered \ @@ -1279,12 +1267,11 @@ do --node-rank $i \ --nnodes 2 \ --tp-size 32 \ - --dp-size 8 \ + --attn-dp-size 8 \ --mem-fraction-static 0.75 \ --max-running-requests 32 \ --quantization modelslim \ --moe-a2a-backend deepep \ - --enable-dp-attention \ --deepep-mode low_latency \ --enable-dp-lm-head \ --moe-dense-tp 1 \ diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_v3_2.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_v3_2.mdx index eedc6777bd8a..b8a4cdb65adf 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_v3_2.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_v3_2.mdx @@ -159,10 +159,9 @@ do --node-rank $i \ --nnodes 2 \ --tp 32 \ - --dp 8 \ + --attn-dp-size 8 \ --ep 32 \ --moe-dense-tp-size 1 \ - --enable-dp-attention \ --enable-dp-lm-head \ --watchdog-timeout 9000 \ --mem-fraction-static 0.79 \ @@ -356,10 +355,9 @@ do --node-rank $i \ --nnodes 2 \ --tp 32 \ - --dp 8 \ + --attn-dp-size 8 \ --ep 32 \ --moe-dense-tp-size 1 \ - --enable-dp-attention \ --enable-dp-lm-head \ --watchdog-timeout 9000 \ --mem-fraction-static 0.79 \ @@ -552,10 +550,9 @@ do --node-rank $i \ --nnodes 2 \ --tp 32 \ - --dp 8 \ + --attn-dp-size 8 \ --ep 32 \ --moe-dense-tp-size 1 \ - --enable-dp-attention \ --enable-dp-lm-head \ --watchdog-timeout 9000 \ --mem-fraction-static 0.79 \ diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_v4_flash.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_v4_flash.mdx index 6e8ff2e502cf..390945be625b 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_v4_flash.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_v4_flash.mdx @@ -131,8 +131,7 @@ do --max-prefill-tokens 70000 \ --chunked-prefill-size -1 \ --max-running-requests 112 \ - --dp-size 16 \ - --enable-dp-attention \ + --attn-dp-size 16 \ --moe-a2a-backend deepep \ --deepep-mode normal \ --quantization modelslim \ @@ -177,8 +176,7 @@ do --chunked-prefill-size 32768 \ --disaggregation-transfer-backend ascend \ --max-running-requests 896 \ - --dp-size 16 \ - --enable-dp-attention \ + --attn-dp-size 16 \ --moe-a2a-backend deepep \ --deepep-mode auto \ --quantization modelslim \ @@ -319,8 +317,7 @@ python3 -m sglang.launch_server \ --mem-fraction-static 0.7 \ --chunked-prefill-size 131072 \ --max-running-requests 64 \ - --dp-size 16 \ - --enable-dp-attention \ + --attn-dp-size 16 \ --moe-a2a-backend deepep \ --deepep-mode auto \ --quantization modelslim \ @@ -444,8 +441,7 @@ python3 -m sglang.launch_server \ --mem-fraction-static 0.7 \ --chunked-prefill-size 131072 \ --max-running-requests 160 \ - --dp-size 16 \ - --enable-dp-attention \ + --attn-dp-size 16 \ --moe-a2a-backend deepep \ --deepep-mode auto \ --quantization modelslim \ diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/glm_5_1.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/glm_5_1.mdx index 276fa9080a15..16a12899ed7b 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/glm_5_1.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/glm_5_1.mdx @@ -105,8 +105,7 @@ do --attention-backend ascend \ --device npu \ --tp-size 32 \ - --dp-size 16 \ - --enable-dp-attention \ + --attn-dp-size 16 \ --chunked-prefill-size 65536 \ --max-prefill-tokens 280000 \ --trust-remote-code \ @@ -294,9 +293,8 @@ do --trust-remote-code \ --tp-size 32 \ --nnodes 2 \ - --dp-size 32 \ + --attn-dp-size 32 \ --ep-size 32 \ - --enable-dp-attention \ --mem-fraction-static 0.85 \ --max-running-requests 32 \ --attention-backend ascend \ @@ -454,8 +452,7 @@ do --disable-shared-experts-fusion \ --disable-cuda-graph \ --dtype bfloat16 \ - --dp-size 4 \ - --enable-dp-attention \ + --attn-dp-size 4 \ --load-balance-method round_robin \ --enable-prefill-cp \ --cp-strategy zigzag \ @@ -492,9 +489,8 @@ do --node-rank $i \ --tp-size 32 \ --nnodes 2 \ - --dp-size 32 \ + --attn-dp-size 32 \ --ep-size 32 \ - --enable-dp-attention \ --mem-fraction-static 0.87 \ --max-running-requests 96 \ --attention-backend ascend \ @@ -702,8 +698,7 @@ do --node-rank $i \ --tp-size 32 \ --nnodes 2 \ - --dp-size 32 \ - --enable-dp-attention \ + --attn-dp-size 32 \ --ep-size 32 \ --mem-fraction-static 0.85 \ --max-running-requests 32 \ @@ -903,8 +898,7 @@ do --node-rank $i \ --tp-size 32 \ --nnodes 2 \ - --dp-size 32 \ - --enable-dp-attention \ + --attn-dp-size 32 \ --ep-size 32 \ --mem-fraction-static 0.85 \ --max-running-requests 32 \ @@ -1062,8 +1056,7 @@ do --disable-shared-experts-fusion \ --disable-cuda-graph \ --dtype bfloat16 \ - --dp-size 4 \ - --enable-dp-attention \ + --attn-dp-size 4 \ --load-balance-method round_robin \ --enable-prefill-cp \ --cp-strategy zigzag \ @@ -1100,9 +1093,8 @@ do --node-rank $i \ --tp-size 32 \ --nnodes 2 \ - --dp-size 32 \ + --attn-dp-size 32 \ --ep-size 32 \ - --enable-dp-attention \ --mem-fraction-static 0.87 \ --max-running-requests 96 \ --attention-backend ascend \ @@ -1308,9 +1300,8 @@ do --node-rank $i \ --tp-size 32 \ --nnodes 2 \ - --dp-size 32 \ + --attn-dp-size 32 \ --ep-size 32 \ - --enable-dp-attention \ --mem-fraction-static 0.865 \ --max-running-requests 96 \ --attention-backend ascend \ @@ -1516,9 +1507,8 @@ do --node-rank $i \ --tp-size 32 \ --nnodes 2 \ - --dp-size 32 \ + --attn-dp-size 32 \ --ep-size 32 \ - --enable-dp-attention \ --mem-fraction-static 0.85 \ --max-running-requests 320 \ --attention-backend ascend \ diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/glm_5_2.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/glm_5_2.mdx index f74bfc74549d..1d3ceb13e7c0 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/glm_5_2.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/glm_5_2.mdx @@ -159,8 +159,7 @@ do --port 8001 \ --trust-remote-code \ --tp-size 16 \ - --dp-size 16 \ - --enable-dp-attention \ + --attn-dp-size 16 \ --ep-size 16 \ --mem-fraction-static 0.895 \ --max-running-requests 128 \ diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/kimi_k2_6.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/kimi_k2_6.mdx index 5419b1792a21..9356e08d79a4 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/kimi_k2_6.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/kimi_k2_6.mdx @@ -119,8 +119,7 @@ do --enable-multimodal \ --mm-attention-backend ascend_attn \ --sampling-backend ascend \ - --enable-dp-attention \ - --dp-size 32 \ + --attn-dp-size 32 \ --moe-a2a-backend deepep \ --deepep-mode auto \ --cuda-graph-bs-decode 1 \ @@ -295,8 +294,6 @@ do --tp-size 16 \ --mem-fraction-static 0.82 \ --max-running-requests 2 \ - --enable-dp-attention \ - --dp-size 1 \ --enable-dp-lm-head \ --disable-radix-cache \ --enable-multimodal \ @@ -493,8 +490,6 @@ do --tp-size 16 \ --mem-fraction-static 0.82 \ --max-running-requests 2 \ - --enable-dp-attention \ - --dp-size 1 \ --enable-dp-lm-head \ --disable-radix-cache \ --enable-multimodal \ @@ -690,8 +685,6 @@ do --tp-size 16 \ --mem-fraction-static 0.82 \ --max-running-requests 16 \ - --enable-dp-attention \ - --dp-size 1 \ --enable-dp-lm-head \ --disable-radix-cache \ --enable-multimodal \ @@ -883,8 +876,7 @@ do --tp-size 16 \ --mem-fraction-static 0.82 \ --max-running-requests 2 \ - --enable-dp-attention \ - --dp-size 2 \ + --attn-dp-size 2 \ --enable-dp-lm-head \ --disable-radix-cache \ --enable-multimodal \ @@ -1019,8 +1011,7 @@ python3 -m sglang.launch_server \ --enable-multimodal \ --mm-attention-backend ascend_attn \ --sampling-backend ascend \ - --enable-dp-attention \ - --dp-size 16 \ + --attn-dp-size 16 \ --moe-a2a-backend deepep \ --deepep-mode auto \ --cuda-graph-bs-decode 1 2 4 8 9 10 11 \ @@ -1129,8 +1120,7 @@ python3 -m sglang.launch_server \ --enable-multimodal \ --mm-attention-backend ascend_attn \ --sampling-backend ascend \ - --enable-dp-attention \ - --dp-size 16 \ + --attn-dp-size 16 \ --moe-a2a-backend deepep \ --deepep-mode auto \ --cuda-graph-bs-decode 1 2 3 4 \ @@ -1244,8 +1234,7 @@ python3 -m sglang.launch_server \ --enable-multimodal \ --mm-attention-backend ascend_attn \ --sampling-backend ascend \ - --enable-dp-attention \ - --dp-size 16 \ + --attn-dp-size 16 \ --moe-a2a-backend deepep \ --deepep-mode auto \ --cuda-graph-bs-decode 1 2 3 4 5 \ @@ -1357,8 +1346,7 @@ python3 -m sglang.launch_server \ --enable-multimodal \ --mm-attention-backend ascend_attn \ --sampling-backend ascend \ - --enable-dp-attention \ - --dp-size 16 \ + --attn-dp-size 16 \ --moe-a2a-backend deepep \ --deepep-mode auto \ --cuda-graph-bs-decode 1 2 4 8 11 12 13 \ diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/mimo_v2_5_pro.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/mimo_v2_5_pro.mdx index c18f549457b3..720474797084 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/mimo_v2_5_pro.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/mimo_v2_5_pro.mdx @@ -82,7 +82,7 @@ python3 -m sglang.launch_server \ --disaggregation-mode prefill --disaggregation-transfer-backend ascend \ --disaggregation-bootstrap-port 8996 \ --disable-piecewise-cuda-graph \ - --dp-size 2 --enable-dp-attention --enable-dp-lm-head \ + --attn-dp-size 2 --enable-dp-lm-head \ --moe-a2a-backend deepep --deepep-mode normal ``` @@ -110,7 +110,7 @@ python3 -m sglang.launch_server \ --disaggregation-mode decode --disaggregation-transfer-backend ascend \ --disaggregation-bootstrap-port 8996 \ --moe-a2a-backend deepep --deepep-mode low_latency \ - --dp-size 2 --enable-dp-attention --enable-dp-lm-head \ + --attn-dp-size 2 --enable-dp-lm-head \ --speculative-algorithm DFLASH \ --speculative-num-draft-tokens 8 ``` diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/mimo_v2_flash.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/mimo_v2_flash.mdx index 8599e2c4f605..377e53899133 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/mimo_v2_flash.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/mimo_v2_flash.mdx @@ -166,8 +166,7 @@ do --speculative-num-draft-tokens 4 \ --enable-multi-layer-eagle \ --disable-radix-cache \ - --dp-size 2 \ - --enable-dp-attention \ + --attn-dp-size 2 \ --enable-dp-lm-head \ --moe-a2a-backend deepep \ --deepep-mode low_latency @@ -350,8 +349,7 @@ do --speculative-num-draft-tokens 4 \ --enable-multi-layer-eagle \ --disable-radix-cache \ - --dp-size 2 \ - --enable-dp-attention \ + --attn-dp-size 2 \ --enable-dp-lm-head \ --moe-a2a-backend deepep \ --deepep-mode low_latency @@ -534,8 +532,7 @@ do --speculative-num-draft-tokens 4 \ --enable-multi-layer-eagle \ --disable-radix-cache \ - --dp-size 2 \ - --enable-dp-attention \ + --attn-dp-size 2 \ --enable-dp-lm-head \ --moe-a2a-backend deepep \ --deepep-mode low_latency @@ -718,8 +715,7 @@ do --speculative-num-draft-tokens 4 \ --enable-multi-layer-eagle \ --disable-radix-cache \ - --dp-size 2 \ - --enable-dp-attention \ + --attn-dp-size 2 \ --enable-dp-lm-head \ --moe-a2a-backend deepep \ --deepep-mode low_latency diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/minimax_m2_5.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/minimax_m2_5.mdx index 4fdf7a210e79..e63a327eac77 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/minimax_m2_5.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/minimax_m2_5.mdx @@ -312,8 +312,7 @@ python3 -m sglang.launch_server \ --model-path $MODEL_PATH \ --host 127.0.0.1 --port 6688 \ --tp-size 16 \ - --dp-size 2 \ - --enable-dp-attention \ + --attn-dp-size 2 \ --mem-fraction-static 0.65 \ --max-running-requests 4 \ --reasoning-parser minimax-append-think \ @@ -422,8 +421,7 @@ python3 -m sglang.launch_server \ --model-path $MODEL_PATH \ --host 127.0.0.1 --port 6688 \ --tp-size 16 \ - --enable-dp-attention \ - --dp-size 16 \ + --attn-dp-size 16 \ --mem-fraction-static 0.53 \ --max-running-requests 96 \ --disable-radix-cache \ @@ -527,8 +525,7 @@ python3 -m sglang.launch_server \ --model-path $MODEL_PATH \ --host 127.0.0.1 --port 6688 \ --tp-size 16 \ - --enable-dp-attention \ - --dp-size 16 \ + --attn-dp-size 16 \ --mem-fraction-static 0.75 \ --max-running-requests 320 \ --disable-radix-cache \ diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_235b_a22b.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_235b_a22b.mdx index 6c3a3f49a18b..d877d34dddc5 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_235b_a22b.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_235b_a22b.mdx @@ -204,8 +204,7 @@ python3 -m sglang.launch_server \ --speculative-num-draft-tokens 4 \ --speculative-draft-model-quantization unquant \ --tp 16 \ - --dp-size 16 \ - --enable-dp-attention \ + --attn-dp-size 16 \ --enable-dp-lm-head \ --mem-fraction-static 0.8 \ --cuda-graph-bs-decode 1 2 4 8 16 20 24 26 27 \ diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_30b_a3b.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_30b_a3b.mdx index d18fb5389325..6a821b99ae01 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_30b_a3b.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_30b_a3b.mdx @@ -105,8 +105,7 @@ python3 -m sglang.launch_server \ --speculative-eagle-topk 1 \ --speculative-num-draft-tokens 8 \ --tp-size 2 \ - --enable-dp-attention \ - --dp-size 2 \ + --attn-dp-size 2 \ --mem-fraction-static 0.85 \ --cuda-graph-bs-decode 1 2 4 8 16 20 24 28 32 36 40 44 48 52 56 60 64 68 72 76 80 84 \ --dtype bfloat16 \ diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_5_397b.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_5_397b.mdx index 767fd2f5a910..3848c6548074 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_5_397b.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_5_397b.mdx @@ -455,8 +455,7 @@ python3 -m sglang.launch_server \ --mm-attention-backend ascend_attn \ --dtype bfloat16 \ --mamba-ssm-dtype bfloat16 \ - --dp-size 4 \ - --enable-dp-attention \ + --attn-dp-size 4 \ --enable-dp-lm-head \ --speculative-algorithm NEXTN \ --speculative-num-steps 3 \ @@ -572,8 +571,7 @@ python3 -m sglang.launch_server \ --mm-attention-backend ascend_attn \ --dtype bfloat16 \ --mamba-ssm-dtype bfloat16 \ - --dp-size 4 \ - --enable-dp-attention \ + --attn-dp-size 4 \ --enable-dp-lm-head \ --speculative-algorithm NEXTN \ --speculative-num-steps 3 \ @@ -688,8 +686,7 @@ python3 -m sglang.launch_server \ --mm-attention-backend ascend_attn \ --dtype bfloat16 \ --mamba-ssm-dtype bfloat16 \ - --dp-size 8 \ - --enable-dp-attention \ + --attn-dp-size 8 \ --enable-dp-lm-head \ --speculative-algorithm NEXTN \ --speculative-num-steps 3 \ @@ -804,8 +801,7 @@ python3 -m sglang.launch_server \ --mm-attention-backend ascend_attn \ --dtype bfloat16 \ --mamba-ssm-dtype bfloat16 \ - --dp-size 8 \ - --enable-dp-attention \ + --attn-dp-size 8 \ --enable-dp-lm-head \ --speculative-algorithm NEXTN \ --speculative-num-steps 3 \ @@ -921,8 +917,7 @@ python3 -m sglang.launch_server \ --mm-attention-backend ascend_attn \ --dtype bfloat16 \ --mamba-ssm-dtype bfloat16 \ - --dp-size 2 \ - --enable-dp-attention \ + --attn-dp-size 2 \ --enable-dp-lm-head \ --speculative-algorithm NEXTN \ --speculative-num-steps 3 \ @@ -1038,8 +1033,7 @@ python3 -m sglang.launch_server \ --mm-attention-backend ascend_attn \ --dtype bfloat16 \ --mamba-ssm-dtype bfloat16 \ - --dp-size 2 \ - --enable-dp-attention \ + --attn-dp-size 2 \ --enable-dp-lm-head \ --speculative-algorithm NEXTN \ --speculative-num-steps 3 \ @@ -1149,8 +1143,7 @@ python3 -m sglang.launch_server \ --mm-attention-backend ascend_attn \ --dtype bfloat16 \ --mamba-ssm-dtype bfloat16 \ - --dp-size 2 \ - --enable-dp-attention \ + --attn-dp-size 2 \ --enable-dp-lm-head \ --speculative-algorithm NEXTN \ --speculative-num-steps 3 \ diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_next_80b_a3b_instruct.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_next_80b_a3b_instruct.mdx index f0f6e8e68aae..f875fe1181aa 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_next_80b_a3b_instruct.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_next_80b_a3b_instruct.mdx @@ -228,8 +228,7 @@ python3 -m sglang.launch_server \ --speculative-num-draft-tokens 4 \ --speculative-draft-model-quantization unquant \ --speculative-draft-model-path $DRAFT_MODEL_PATH \ - --dp-size 2 \ - --enable-dp-attention \ + --attn-dp-size 2 \ --enable-dp-lm-head \ --moe-a2a-backend deepep \ --deepep-mode auto \ @@ -337,8 +336,7 @@ python3 -m sglang.launch_server \ --max-prefill-tokens 28672 \ --context-length 81920 \ --max-total-tokens 122304 \ - --dp-size 2 \ - --enable-dp-attention \ + --attn-dp-size 2 \ --enable-dp-lm-head \ --speculative-algorithm NEXTN \ --speculative-num-steps 3 \ diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/deepseek_v4_flash.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/deepseek_v4_flash.mdx index bffa9c5330e1..4f4000d7eb45 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/deepseek_v4_flash.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/deepseek_v4_flash.mdx @@ -31,7 +31,7 @@ recommended to use v0.5.16 or a later version. | Speculative Decoding | `--speculative-algorithm EAGLE \`
`--speculative-num-steps 2 \`
`--speculative-eagle-topk 1 \`
`--speculative-num-draft-tokens 3` | | Overlap Schedule | `export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1` | | DP LM Head | `--enable-dp-lm-head` | -| DP Attention | `--enable-dp-attention` | +| DP Attention | `--attn-dp-size 16` | The values in the **Example usage** column are for illustration only. Adjust them according to your hardware, deployment @@ -218,7 +218,7 @@ python3 -m sglang.launch_server \ --max-prefill-tokens 80000 \ --chunked-prefill-size 131072 \ --max-running-requests 160 \ - --dp-size 8 --enable-dp-attention \ + --attn-dp-size 8 \ --moe-a2a-backend deepep --deepep-mode auto \ --quantization modelslim --enable-dp-lm-head \ --kv-cache-dtype bfloat16 diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/glm_5_2.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/glm_5_2.mdx index 4a18b869c20f..7fa82659bf3a 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/glm_5_2.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/glm_5_2.mdx @@ -384,7 +384,7 @@ do --disaggregation-transfer-backend ascend --max-running-requests 64 \ --served-model-name glm-5 --chunked-prefill-size 524288 --max-prefill-tokens 180000 --moe-a2a-backend deepep --deepep-mode normal \ --disable-shared-experts-fusion --disable-cuda-graph --dtype bfloat16 \ - --dp-size 4 --enable-dp-attention \ + --attn-dp-size 4 \ --load-balance-method round_robin \ --enable-dp-lm-head --moe-dense-tp 1 \ --speculative-draft-model-quantization unquant \ @@ -418,9 +418,9 @@ do export SGLANG_NPU_USE_MULTI_STREAM=1 python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode decode --host ${D_IP[$i]} \ - --port 8003 --trust-remote-code --nnodes 1 --node-rank $i --tp-size 16 --dp-size 16 --ep-size 16 \ + --port 8003 --trust-remote-code --nnodes 1 --node-rank $i --tp-size 16 --attn-dp-size 16 --ep-size 16 \ --mem-fraction-static 0.8 --max-running-requests 128 --attention-backend ascend --device npu --quantization modelslim \ - --served-model-name glm-5 --moe-a2a-backend deepep --enable-dp-attention --deepep-mode low_latency \ + --served-model-name glm-5 --moe-a2a-backend deepep --deepep-mode low_latency \ --cuda-graph-max-bs-decode 4 --disaggregation-transfer-backend ascend --watchdog-timeout 9000 --context-length 180000 \ --tokenizer-worker-num 4 --disable-shared-experts-fusion --dtype bfloat16 --load-balance-method round_robin \ --speculative-draft-model-quantization unquant \ diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/mimo_v2_flash.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/mimo_v2_flash.mdx index a497850d5514..11f47b94cd33 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/mimo_v2_flash.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/mimo_v2_flash.mdx @@ -30,7 +30,7 @@ v0.5.16 or a later version. | Speculative Decoding | `--speculative-algorithm EAGLE \`
`--speculative-num-steps 3 \`
`--speculative-eagle-topk 1 \`
`--speculative-num-draft-tokens 4 \`
`--enable-multi-layer-eagle` | | Overlap Schedule | `export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0` | | DP LM Head | `--enable-dp-lm-head` | -| DP Attention | `--enable-dp-attention` | +| DP Attention | `--attn-dp-size 2` | The values in the **Example usage** column are for illustration only. Adjust them according to your hardware, deployment diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_8_max.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_8_max.mdx index 4c398e08c539..dfed0ada1745 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_8_max.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_8_max.mdx @@ -24,7 +24,7 @@ source. | Feature | Example usage | |-------------------------------|-----------------------------------------------------------------------------------------------| | Tensor Parallelism | `--tp-size 64` | -| Data Parallelism | `--enable-dp-attention \`
`--dp-size 4` | +| DP Attention | `--attn-dp-size 4` | | Expert Parallelism | `--moe-a2a-backend deepep \`
`--deepep-mode auto` | | Quantization | `--quantization modelslim` | | Chunked Prefill | auto based on device memory, or set explicit value;
disable with `--chunked-prefill-size -1`; e.g., `--chunked-prefill-size 8192` | @@ -136,7 +136,7 @@ quay.io/ascend/sglang:main-cann9.0.0-a3 Multi-node deployment distributes the model across multiple Ascend A3 Series nodes using tensor parallelism while keeping prefill and decode on the same nodes (PD mixed mode), suitable for scenarios that need more device memory than a single node can provide. The validated configuration deploys the W4A8 quantized checkpoint on 4 nodes with `--tp-size 64` -(16 dies per node), DP attention (`--dp-size 4`), and DeepEP in `auto` mode. +(16 dies per node), DP attention (`--attn-dp-size 4`), and DeepEP in `auto` mode. Modify the IP addresses of the four nodes, then run the same script on all four nodes. Each node determines its own rank by matching the local IP address against `IPS`. @@ -211,7 +211,7 @@ do --quantization modelslim \ --dtype bfloat16 \ --tp-size 64 \ - --enable-dp-attention --dp-size 4 --enable-dp-lm-head \ + --attn-dp-size 4 --enable-dp-lm-head \ --mem-fraction-static 0.8 \ --chunked-prefill-size 8192 \ --cuda-graph-bs-decode 16 \ diff --git a/docs/docs/hardware-platforms/ascend-npus/optimization/parameter_tuning.mdx b/docs/docs/hardware-platforms/ascend-npus/optimization/parameter_tuning.mdx index 2b5c45fa05fa..0e2ab1f1780f 100644 --- a/docs/docs/hardware-platforms/ascend-npus/optimization/parameter_tuning.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/optimization/parameter_tuning.mdx @@ -218,8 +218,8 @@ These arguments and environment variables are critical for tuning decode perform - `--dp-size` / `--data-parallel-size` - Sets the data parallelism degree for the decode server. With DP attention enabled, attention layers are sharded across DP ranks while FFN/MoE layers use tensor parallelism. Higher values create more independent decode instances, increasing throughput through parallel request processing. Choose a value that divides evenly into your total card count, with remaining cards used for TP/EP. + `--attn-dp-size` / `--attention-data-parallel-size` + Sets the attention DP size for the decode server: the number of data-parallel attention groups inside the TP group. This is a key optimization for decode throughput — attention is DP-sharded to reduce KV cache duplication, while FFN/MoE layers remain TP-sharded to leverage expert parallelism. Best suited for MoE models where attention is not the compute bottleneck. Higher values create more independent decode instances, increasing throughput through parallel request processing. Choose a value that divides the TP size evenly. `8` @@ -227,11 +227,6 @@ These arguments and environment variables are critical for tuning decode perform Sets the expert parallelism degree. For MoE models, this distributes experts across cards, reducing per-card expert loading overhead and enabling all-to-all dispatch. The code default is `1`; set explicitly for MoE models. The optimal value depends on your model's expert count and architecture. DeepSeek V3.2 with 256 routed experts uses `ep=32`. For models with fewer experts, use a proportionally smaller value. `32` - - `--enable-dp-attention` - Enables data parallelism for attention layers while keeping tensor parallelism for FFN/MoE layers. This is a key optimization for decode throughput — attention is DP-sharded to reduce KV cache duplication, while MoE layers remain TP-sharded to leverage expert parallelism. Best suited for MoE models where attention is not the compute bottleneck. - Enabled - `--enable-dp-lm-head` Enables vocabulary parallelism across the DP attention group, sharding the LM head weight across ranks. Each rank only computes logits for its vocabulary shard, avoiding a costly all-gather of logits across the DP group. This is essential when DP attention is enabled to maintain throughput. @@ -727,9 +722,9 @@ The symbols used are defined as follows: Splits model weights across multiple NPU devices so that large models can be loaded and run cooperatively. Each device holds a shard of every weight tensor and communicates via HCCL all-reduce. This is the primary mechanism for deploying models that exceed single-device memory. -### Data Parallelism (`--dp-size`, `--enable-dp-attention`) +### Data Parallelism (`--dp-size`, `--attn-dp-size`) -Replicates the model across independent device groups to increase throughput by processing multiple requests in parallel. With --enable-dp-attention, the attention layers are replicated across DP ranks while the FFN/MoE layers remain tensor-parallel/expert-parallel. This reduces the communication overhead of attention layers in TP (which is significant during small-batch decode) while keeping the heavy compute and memory demands of MoE layers distributed. +`--dp-size` replicates the model across independent device groups to increase throughput by processing multiple requests in parallel. `--attn-dp-size` instead runs attention data parallel inside the TP group: the attention layers are replicated across attention DP ranks while the FFN/MoE layers remain tensor-parallel/expert-parallel. This reduces the communication overhead of attention layers in TP (which is significant during small-batch decode) while keeping the heavy compute and memory demands of MoE layers distributed. Model replicas (`--dp-size` > 1) cannot be combined with attention DP. ### Expert Parallelism (`--ep-size`) @@ -779,7 +774,7 @@ On Ascend, this is primarily controlled via environment variable SGLANG_ENABLE_O ### DP LM Head (`--enable-dp-lm-head`) -Shards the language model head (the final vocabulary projection layer) across DP ranks to reduce memory consumption and improve decode throughput. Without this, each DP rank holds a full copy of the LM head, which is wasteful for large vocabularies. Typically used together with `--enable-dp-attention`. +Shards the language model head (the final vocabulary projection layer) across DP ranks to reduce memory consumption and improve decode throughput. Without this, each DP rank holds a full copy of the LM head, which is wasteful for large vocabularies. Typically used together with `--attn-dp-size`. ### MLAPO (`SGLANG_NPU_USE_MLAPO=1`) diff --git a/docs/docs/hardware-platforms/ascend-npus/reference/support_features.mdx b/docs/docs/hardware-platforms/ascend-npus/reference/support_features.mdx index d66366ba923c..e2586903fe34 100644 --- a/docs/docs/hardware-platforms/ascend-npus/reference/support_features.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/reference/support_features.mdx @@ -1080,6 +1080,12 @@ click [Server Arguments](../../../advanced_features/server_arguments). Type: int A2/A3 Series + + `--attention-data-parallel-size`
`--attn-dp-size` + `1` + Type: int + A2/A3 Series + `--load-balance-method` `auto` @@ -2190,12 +2196,6 @@ If the value is int8, you must also set the environment variable:DEEP_NORMAL_MOD bool flag
(set to enable) A2/A3 Series - - `--enable-dp-attention` - `False` - bool flag
(set to enable) - A2/A3 Series - `--enable-dp-attention-local-control-broadcast` `False` diff --git a/docs/docs/references/multi_node_deployment/lws_pd/lws_pd_deploy.mdx b/docs/docs/references/multi_node_deployment/lws_pd/lws_pd_deploy.mdx index 380b7c545490..5d4135dcce21 100644 --- a/docs/docs/references/multi_node_deployment/lws_pd/lws_pd_deploy.mdx +++ b/docs/docs/references/multi_node_deployment/lws_pd/lws_pd_deploy.mdx @@ -63,8 +63,7 @@ spec: # - --deepep-config # - /home/aiges/tuned/tuned_8sms.json - --enable-dp-lm-head - - --enable-dp-attention - - --dp-size + - --attn-dp-size - "16" - --disable-radix-cache - --moe-a2a-backend @@ -210,8 +209,7 @@ spec: # - --deepep-config # - /home/aiges/tuned/tuned_8sms.json - --enable-dp-lm-head - - --enable-dp-attention - - --dp-size + - --attn-dp-size - "16" - --disable-radix-cache - --moe-a2a-backend @@ -370,9 +368,8 @@ spec: - "262144" - --page-size - "64" - - --enable-dp-attention - --enable-dp-lm-head - - --dp-size + - --attn-dp-size - "16" - --moe-a2a-backend - deepep @@ -503,10 +500,9 @@ spec: - "262144" - --page-size - "64" - - --enable-dp-attention - --enable-dp-lm-head #- --enable-two-batch-overlap - - --dp-size + - --attn-dp-size - "16" - --moe-a2a-backend - deepep diff --git a/docs/docs/references/multi_node_deployment/rbg_pd/deepseekv32_pd.mdx b/docs/docs/references/multi_node_deployment/rbg_pd/deepseekv32_pd.mdx index c0fa4228e9b0..86dad0f357c2 100644 --- a/docs/docs/references/multi_node_deployment/rbg_pd/deepseekv32_pd.mdx +++ b/docs/docs/references/multi_node_deployment/rbg_pd/deepseekv32_pd.mdx @@ -69,8 +69,7 @@ spec: - --eplb-algorithm - deepseek - --enable-dp-lm-head - - --enable-dp-attention - - --dp-size + - --attn-dp-size - "8" - --moe-a2a-backend - deepep @@ -267,9 +266,8 @@ spec: - "29" - --page-size - "64" - - --enable-dp-attention - --enable-dp-lm-head - - --dp-size + - --attn-dp-size - "8" - --moe-a2a-backend - deepep @@ -338,9 +336,8 @@ spec: - "29" - --page-size - "64" - - --enable-dp-attention - --enable-dp-lm-head - - --dp-size + - --attn-dp-size - "32" - --moe-a2a-backend - "deepep" diff --git a/docs/src/snippets/_deployment.jsx b/docs/src/snippets/_deployment.jsx index a70ed97ba96b..842a335194f8 100644 --- a/docs/src/snippets/_deployment.jsx +++ b/docs/src/snippets/_deployment.jsx @@ -810,7 +810,8 @@ export const Deployment = ({ config, benchmarks }) => { // Insert the multi-node trio after the last parallelism flag, // falling back to right after --model-path. const PARALLELISM_ANCHORS = new Set([ - "--enable-dp-attention", "--dp-size", "--dp", "--tp-size", "--tp", + "--attn-dp-size", "--attention-data-parallel-size", + "--dp-size", "--dp", "--tp-size", "--tp", "--sp-degree", "--ulysses-degree", "--ring-degree", ]); let i = flags.reduce( diff --git a/docs/src/snippets/_kimi_k3_mamba_ratio_calculator.jsx b/docs/src/snippets/_kimi_k3_mamba_ratio_calculator.jsx index 97a0b2baf938..6be4e81f0d6c 100644 --- a/docs/src/snippets/_kimi_k3_mamba_ratio_calculator.jsx +++ b/docs/src/snippets/_kimi_k3_mamba_ratio_calculator.jsx @@ -67,7 +67,7 @@ export const KimiK3MambaRatioCalculator = () => { const hasFlag = (name) => flags.some((f) => f.split(/[\s=]/)[0] === name); const tp = Number(flagArg("--tp-size")) || 8; - const dp = hasFlag("--enable-dp-attention") ? Number(flagArg("--dp-size")) || 1 : 1; + const dp = Number(flagArg("--attn-dp-size")) || 1; // KDA state and the replicated MLA KV both live per attention-TP group. // PP needs no term: it splits the layers of both pools equally. const attnTp = Math.max(1, Math.round(tp / dp)); @@ -231,9 +231,9 @@ export const KimiK3MambaRatioCalculator = () => { : `DSPARK (D = ${block + 1})`; const derivedChips = [ // With DP attention on, show the whole topology so a large-scale preset is - // visibly understood: total GPUs = DP replicas x attention-TP group width. + // visibly understood: total GPUs = attention DP size x attention-TP group width. dp > 1 - ? `${tp} GPUs = DP ${dp} × Attention TP ${attnTp}` + ? `${tp} GPUs = Attention DP ${dp} × Attention TP ${attnTp}` : `Attention TP ${attnTp}`, `DCP ${dcp}`, `KV ${kvDtype === "fp8_e4m3" ? "FP8" : "BF16"}`, diff --git a/docs/src/snippets/_playground.jsx b/docs/src/snippets/_playground.jsx index d8188aa95e03..6dbe91819899 100644 --- a/docs/src/snippets/_playground.jsx +++ b/docs/src/snippets/_playground.jsx @@ -360,6 +360,7 @@ export const Playground = ({ config }) => { const TP_HEADS = ["--tp-size", "--tp", "--tensor-parallel-size"]; const EP_HEADS = ["--ep-size", "--ep", "--expert-parallel-size"]; const DP_HEADS = ["--dp-size", "--dp", "--data-parallel-size"]; + const ATTN_DP_HEADS = ["--attn-dp-size", "--attention-data-parallel-size"]; const parseIntFlagAny = (flags, heads) => { for (const head of heads) { const n = parseIntFlag(flags, head); @@ -370,22 +371,25 @@ export const Playground = ({ config }) => { const flagSpelling = (flags, heads, fallback) => heads.find((head) => (flags || []).some((f) => f.split(/[\s=]/)[0] === head)) || fallback; + // DP-Attention is on when the attention DP size is above 1 (1 = off). + const dpAttnOnIn = (flags) => (parseIntFlagAny(flags, ATTN_DP_HEADS) ?? 1) > 1; // Insertion-anchor sets (priority-ordered; each includes siblings so // insertion still works in partial cells). const ANCHOR_NEAR_MODEL_PATH = ["--model-path"]; const ANCHOR_NEAR_TP = ["--tp-size", "--tp", "--model-path"]; const ANCHOR_NEAR_DP = ["--dp-size", "--dp", "--tp-size", "--tp", "--model-path"]; - const ANCHOR_NEAR_DPATTN = ["--enable-dp-attention", "--dp-size", "--dp", "--tp-size", "--tp", "--model-path"]; + const ANCHOR_NEAR_DPATTN = [...ATTN_DP_HEADS, "--dp-size", "--dp", "--tp-size", "--tp", "--model-path"]; const ANCHOR_NEAR_MOE = ["--moe-a2a-backend", "--moe-runner-backend", - "--enable-dp-attention", "--dp-size", "--dp", "--tp-size", "--tp", "--model-path"]; + ...ATTN_DP_HEADS, "--dp-size", "--dp", "--tp-size", "--tp", "--model-path"]; // Helper bundle passed to every axis handler. const helpers = { matchConstraint, evaluateChip, findEntry, isHidden, stripFlagsByFirstToken, stripEnvByPrefix, insertBeforeTail, insertAfter, parseIntFlag, hasFlag, findFlagArg, - TP_HEADS, EP_HEADS, DP_HEADS, parseIntFlagAny, flagSpelling, + TP_HEADS, EP_HEADS, DP_HEADS, ATTN_DP_HEADS, parseIntFlagAny, flagSpelling, + dpAttnOnIn, ANCHOR_NEAR_MODEL_PATH, ANCHOR_NEAR_TP, ANCHOR_NEAR_DP, ANCHOR_NEAR_DPATTN, ANCHOR_NEAR_MOE, }; @@ -452,27 +456,23 @@ export const Playground = ({ config }) => { const AXIS_HANDLERS = { // ---- Axis: Attention Parallelism ---------------------------------------- - // TP / CP / DP-Attention sub-knobs; `null` = inherit. DP-Attention is - // combined: a numeric value emits `--dp N --enable-dp-attention`, `false` - // strips both. An optional `cpStrategy` knob (values from --cp-strategy: - // "zigzag" / "interleave") picks the CP layout; without it the strategy - // baked in the base is preserved, defaulting to "interleave" (the legacy - // knob's round-robin-split). + // TP / CP / DP-Attention sub-knobs; `null` = inherit. DP-Attention's + // value is the attention DP size: a numeric value emits + // `--attn-dp-size N`, `false` strips it. An optional `cpStrategy` knob + // (values from --cp-strategy: "zigzag" / "interleave") picks the CP + // layout; without it the strategy baked in the base is preserved, + // defaulting to "interleave" (the legacy knob's round-robin-split). attention: { initState: () => ({ tp: null, cp: null, cpStrategy: null, dpAttn: null }), - // DP-Attention: `--dp N --enable-dp-attention` → N; neither → false; - // bare `--enable-dp-attention` → 1. CP: any enable spelling → - // `--attn-cp-size N` (bare enable → 2, the legacy convention), plus the - // baked strategy (legacy mode flags mapped to zigzag/interleave). + // DP-Attention: `--attn-dp-size N` → N; absent → false. CP: any enable + // spelling → `--attn-cp-size N` (bare enable → 2, the legacy + // convention), plus the baked strategy (legacy mode flags mapped to + // zigzag/interleave). deriveFromBase: (cell, fc, h) => { const flags = (cell && cell.flags) || []; - const dpVal = h.parseIntFlagAny(flags, h.DP_HEADS); - const hasDpAttn = h.hasFlag(flags, "--enable-dp-attention"); - let dpAttn; - if (dpVal !== null) dpAttn = dpVal; - else if (hasDpAttn) dpAttn = 1; - else dpAttn = false; + const dpVal = h.parseIntFlagAny(flags, h.ATTN_DP_HEADS); + const dpAttn = dpVal !== null ? dpVal : false; const cpSize = h.parseIntFlag(flags, "--attn-cp-size"); return { tp: h.parseIntFlagAny(flags, h.TP_HEADS), @@ -488,7 +488,7 @@ export const Playground = ({ config }) => { const knobEntry = (id) => (fc.knobs || []).find((k) => k.id === id) || {}; const factsNow = () => ({ ...(sel || {}), - dpAttnOn: h.hasFlag(flags, "--enable-dp-attention"), + dpAttnOn: h.dpAttnOnIn(flags), cpOn: cpEnabledIn(flags), cpStrategy: bakedCpStrategy(flags) || "interleave", effTp: h.parseIntFlagAny(flags, h.TP_HEADS), @@ -503,8 +503,7 @@ export const Playground = ({ config }) => { if (knobEntry("cp").freeSize) return null; const dpIntent = (value.dpAttn !== null && value.dpAttn !== undefined) ? value.dpAttn - : (h.hasFlag(flags, "--enable-dp-attention") - ? (h.parseIntFlagAny(flags, h.DP_HEADS) ?? 1) : false); + : (h.parseIntFlagAny(flags, h.ATTN_DP_HEADS) ?? false); if (typeof dpIntent === "number" && dpIntent > 1) return null; return h.parseIntFlagAny(flags, h.TP_HEADS); }; @@ -524,7 +523,7 @@ export const Playground = ({ config }) => { && h.evaluateChip(e, facts).disabled); }; // NOTE: interleave prefill-CP + DP-Attention currently fails the - // runtime's dp_size == 1 assert, but combined support is planned + // runtime's attn_dp_size == 1 assert, but combined support is planned // upstream — the combination is allowed here (with a warning hint // below the command box) rather than banned. @@ -567,7 +566,7 @@ export const Playground = ({ config }) => { (r) => r && h.matchConstraint(factsNow(), r.when)); if (dpForced) { flags = h.stripFlagsByFirstToken(flags, [ - ...h.DP_HEADS, "--enable-dp-attention", + ...h.ATTN_DP_HEADS, "--enable-dp-attention-local-control-broadcast", ]); // The cell's DP-only env would otherwise outlive the flags it tunes. @@ -579,22 +578,21 @@ export const Playground = ({ config }) => { && !blocked("dpAttn", value.dpAttn)) { // Capture the spelling before stripping — the TP/EP handlers do the // same, and a lookup on the stripped array always hits the fallback. - const dpHead = h.flagSpelling(flags, h.DP_HEADS, "--dp-size"); + const dpHead = h.flagSpelling(flags, h.ATTN_DP_HEADS, "--attn-dp-size"); // The local-control-broadcast companion only means anything with DP // attention on, so it has to go down with it — stripping just - // `--enable-dp-attention` would leave it orphaned in the command. + // `--attn-dp-size` would leave it orphaned in the command. // apply re-seeds from the base cell, so this restores it when the // cell had it and the user is only re-sizing DP rather than disabling. const hadLocalBroadcast = h.hasFlag(flags, "--enable-dp-attention-local-control-broadcast"); flags = h.stripFlagsByFirstToken(flags, [ - ...h.DP_HEADS, "--enable-dp-attention", + ...h.ATTN_DP_HEADS, "--enable-dp-attention-local-control-broadcast", ]); if (typeof value.dpAttn === "number" && value.dpAttn > 0) { flags = h.insertAfter(flags, h.ANCHOR_NEAR_TP, [ `${dpHead} ${value.dpAttn}`, - "--enable-dp-attention", ...(hadLocalBroadcast ? ["--enable-dp-attention-local-control-broadcast"] : []), ]); @@ -608,7 +606,7 @@ export const Playground = ({ config }) => { if (!knobs.length) return null; const setKnob = (k, v) => setValue({ ...value, [k]: v }); // Interleave prefill-CP + DP-Attention is deliberately NOT grayed: - // current releases assert dp_size == 1 for interleave, but combined + // current releases assert attn_dp_size == 1 for interleave, but combined // support is planned upstream — a warning hint below the command box // covers it instead. const labelFor = (knob) => (c) => { @@ -930,7 +928,7 @@ export const Playground = ({ config }) => { const picked = (fc.options || []).find((p) => p.id === value); if (picked && h.evaluateChip(picked, { ...sel, - dpAttnOn: h.hasFlag(flags, "--enable-dp-attention"), + dpAttnOn: h.dpAttnOnIn(flags), }).disabled) { return { flags, env }; } @@ -1414,7 +1412,7 @@ export const Playground = ({ config }) => { // flags rather than the Deploy dims — the attention axis runs first. // A role override with `allowTp` is a TP-only shape validated as-is. if (fc.requiresDpAttention && !(roleOverride && roleOverride.allowTp) - && !flags.some((f) => f.split(/[\s=]/)[0] === "--enable-dp-attention")) { + && !h.dpAttnOnIn(flags)) { return { flags, env }; } flags = h.stripFlagsByFirstToken(flags, HICACHE_HEADS); @@ -1517,7 +1515,7 @@ export const Playground = ({ config }) => { apply: ({ flags, env, value, fc, sel, h, derived }) => { const evalBase = { ...(sel || {}), - dpAttnOn: h.hasFlag(flags, "--enable-dp-attention"), + dpAttnOn: h.dpAttnOnIn(flags), pdMode: h.findFlagArg(flags, "--disaggregation-mode") || "off", }; for (const spec of (fc || [])) { @@ -1701,7 +1699,7 @@ export const Playground = ({ config }) => { if (multinode && !f.some((x) => x.startsWith("--nnodes"))) { // Insert the multi-node trio after the last parallelism flag (matches // _deployment.jsx so untouched-base output is byte-identical). - const PARALLELISM_ANCHORS = ["--enable-dp-attention", "--dp-size", "--dp", "--tp-size", "--tp"]; + const PARALLELISM_ANCHORS = [...ATTN_DP_HEADS, "--dp-size", "--dp", "--tp-size", "--tp"]; let at = -1; for (const anchor of PARALLELISM_ANCHORS) { at = f.findIndex((x) => x.split(/[\s=]/)[0] === anchor); @@ -2418,7 +2416,7 @@ export const Playground = ({ config }) => { ? (staleExplicit("dpAttn", attnDelta.dpAttn) ? null : attnDelta.dpAttn) : (attnDerived.dpAttn !== undefined ? attnDerived.dpAttn : null); const dpAttnOn = (effDpAttn === true) - || (typeof effDpAttn === "number" && effDpAttn > 0); + || (typeof effDpAttn === "number" && effDpAttn > 1); // Runtime derivation attn_cp_size = tp/dp: with DP-Attention off, the only // enable-able CP size is TP. With DP-Attention on, sizes are NOT gated — // CP + DP-Attention is an allowed experiment covered by a warning hint @@ -2542,11 +2540,11 @@ export const Playground = ({ config }) => { // Interleave prefill-CP + DP-Attention hint on the EFFECTIVE command: // deliberately allowed (combined support is planned upstream), but current - // releases assert dp_size == 1 for the interleave layout at startup. + // releases assert attn_dp_size == 1 for the interleave layout at startup. const pgCpDpHint = cpEnabledIn(pgFlagsLatest) && (bakedCpStrategy(pgFlagsLatest) || "interleave") === "interleave" - && pgFlagsLatest.some((f) => f.split(/[\s=]/)[0] === "--enable-dp-attention"); + && dpAttnOnIn(pgFlagsLatest); // Submission snippets: proposed cell + existing cell at the same match. const proposedCellSnippet = baseCell @@ -2822,7 +2820,7 @@ export const Playground = ({ config }) => { )} {pgCpDpHint && (
- ⚠️ Interleave prefill-CP together with DP-Attention: current SGLang releases assert dp_size == 1 for the interleave layout, so this command fails at startup. Combined CP + DP-Attention support is planned upstream — keep one of the two off until it lands. + ⚠️ Interleave prefill-CP together with DP-Attention: current SGLang releases assert attn_dp_size == 1 for the interleave layout, so this command fails at startup. Combined CP + DP-Attention support is planned upstream — keep one of the two off until it lands.
)} diff --git a/docs/src/snippets/autoregressive/deepseek-math-v2-deployment.jsx b/docs/src/snippets/autoregressive/deepseek-math-v2-deployment.jsx index 95726df1e895..23f24cb6a0e3 100644 --- a/docs/src/snippets/autoregressive/deepseek-math-v2-deployment.jsx +++ b/docs/src/snippets/autoregressive/deepseek-math-v2-deployment.jsx @@ -51,9 +51,9 @@ export const DeepSeekMathV2Deployment = () => { // TP setting cmd += ` \\\n --tp ${tpValue}`; - // DP Attention: --dp matches --tp + // DP Attention: --attn-dp-size matches --tp if (values.dpattention === 'enabled') { - cmd += ` \\\n --dp ${tpValue} \\\n --enable-dp-attention`; + cmd += ` \\\n --attn-dp-size ${tpValue}`; } // EP setting (commonly matches tp for MoE models) diff --git a/docs/src/snippets/autoregressive/deepseek-ocr-deployment.jsx b/docs/src/snippets/autoregressive/deepseek-ocr-deployment.jsx index e80f979aa3a2..bdf9e36819e4 100644 --- a/docs/src/snippets/autoregressive/deepseek-ocr-deployment.jsx +++ b/docs/src/snippets/autoregressive/deepseek-ocr-deployment.jsx @@ -24,7 +24,6 @@ export const DeepSeekOCRDeployment = () => { type: 'checkbox', items: [ { id: 'tp', label: 'TP', subtitle: 'Tensor Parallel', default: true, required: true }, - { id: 'dp', label: 'DP', subtitle: 'Data Parallel', default: false, disabledWhen: (v) => v.hardware === 'xeon', disabledReason: 'Intel Xeon CPUs only support Tensor Parallel (TP)' }, { id: 'ep', label: 'EP', subtitle: 'Expert Parallel', default: false, disabledWhen: (v) => v.hardware === 'xeon', disabledReason: 'Intel Xeon CPUs only support Tensor Parallel (TP)' } ] } @@ -117,11 +116,6 @@ export const DeepSeekOCRDeployment = () => { cmd += ` \\\n --tp 1`; } - // DP strategy - if (strategyArray.includes('dp')) { - cmd += ` \\\n --dp 1 \\\n --enable-dp-attention`; - } - // EP strategy if (strategyArray.includes('ep')) { cmd += ` \\\n --ep 1`; diff --git a/docs/src/snippets/autoregressive/deepseek-ocr-v2-deployment.jsx b/docs/src/snippets/autoregressive/deepseek-ocr-v2-deployment.jsx index 2e7cb06701d3..dd82ae0e8d61 100644 --- a/docs/src/snippets/autoregressive/deepseek-ocr-v2-deployment.jsx +++ b/docs/src/snippets/autoregressive/deepseek-ocr-v2-deployment.jsx @@ -26,7 +26,6 @@ export const DeepSeekOCR2Deployment = () => { type: 'checkbox', items: [ { id: 'tp', label: 'TP', subtitle: 'Tensor Parallel', default: true, required: true }, - { id: 'dp', label: 'DP', subtitle: 'Data Parallel', default: false, disabledWhen: (v) => v.hardware === 'xeon' || v.hardware === 'arc_b', disabledReason: 'Only Tensor Parallel (TP) is supported on this hardware' }, { id: 'ep', label: 'EP', subtitle: 'Expert Parallel', default: false, disabledWhen: (v) => v.hardware === 'xeon' || v.hardware === 'arc_b', disabledReason: 'Only Tensor Parallel (TP) is supported on this hardware' } ] }, @@ -52,10 +51,6 @@ export const DeepSeekOCR2Deployment = () => { cmd += ` \\\n --tp 1`; } - if (strategyArray.includes('dp')) { - cmd += ` \\\n --dp 1 \\\n --enable-dp-attention`; - } - if (strategyArray.includes('ep')) { cmd += ` \\\n --ep 1`; } diff --git a/docs/src/snippets/autoregressive/deepseek-r1-basic-deployment.jsx b/docs/src/snippets/autoregressive/deepseek-r1-basic-deployment.jsx index 1d51eaf2fdb6..3353074f7882 100644 --- a/docs/src/snippets/autoregressive/deepseek-r1-basic-deployment.jsx +++ b/docs/src/snippets/autoregressive/deepseek-r1-basic-deployment.jsx @@ -116,7 +116,7 @@ export const DeepSeekR1BasicDeployment = () => { command += isXeon ? ' \\\n --tp 6' : ' \\\n --tp 8'; } if (strategyValues.includes('dp')) { - command += ' \\\n --dp 8 \\\n --enable-dp-attention'; + command += ' \\\n --attn-dp-size 8'; } if (strategyValues.includes('ep')) { command += ' \\\n --ep 8'; diff --git a/docs/src/snippets/autoregressive/deepseek-v3-deployment.jsx b/docs/src/snippets/autoregressive/deepseek-v3-deployment.jsx index 5b9788946208..4733499aa79a 100644 --- a/docs/src/snippets/autoregressive/deepseek-v3-deployment.jsx +++ b/docs/src/snippets/autoregressive/deepseek-v3-deployment.jsx @@ -151,7 +151,7 @@ export const DeepSeekV3Deployment = () => { cmd += ` --model-path ${modelPath}`; if (strategyArray.includes('tp')) cmd += isXeon ? ' \\\n --tp 6' : ' \\\n --tp 8'; - if (strategyArray.includes('dp')) cmd += ' \\\n --dp 8 \\\n --enable-dp-attention'; + if (strategyArray.includes('dp')) cmd += ' \\\n --attn-dp-size 8'; if (strategyArray.includes('ep')) cmd += ' \\\n --ep 8'; if (strategyArray.includes('mtp')) { cmd += ' \\\n --speculative-algorithm EAGLE \\\n --speculative-num-steps 3 \\\n --speculative-eagle-topk 1 \\\n --speculative-num-draft-tokens 4'; diff --git a/docs/src/snippets/autoregressive/deepseek-v31-deployment.jsx b/docs/src/snippets/autoregressive/deepseek-v31-deployment.jsx index 4f75db97ccf3..a60a3c0dbc88 100644 --- a/docs/src/snippets/autoregressive/deepseek-v31-deployment.jsx +++ b/docs/src/snippets/autoregressive/deepseek-v31-deployment.jsx @@ -148,7 +148,7 @@ export const DeepSeekV31Deployment = () => { // TP is mandatory cmd += isXeon ? ` \\\n --tp 6` : ` \\\n --tp 8`; if (strategyArray.includes('dp')) { - cmd += ` \\\n --dp 8 \\\n --enable-dp-attention`; + cmd += ` \\\n --attn-dp-size 8`; } if (strategyArray.includes('ep')) { cmd += ` \\\n --ep 8`; diff --git a/docs/src/snippets/autoregressive/deepseek-v32-deployment.jsx b/docs/src/snippets/autoregressive/deepseek-v32-deployment.jsx index 4e4ece37beb1..a7617130c77a 100644 --- a/docs/src/snippets/autoregressive/deepseek-v32-deployment.jsx +++ b/docs/src/snippets/autoregressive/deepseek-v32-deployment.jsx @@ -219,7 +219,7 @@ export const DeepSeekV32Deployment = () => { const epSize = 8; cmd += ` \\\n --tp ${tpSize}`; if (strategyArray.includes('dp')) { - cmd += ` \\\n --dp ${dpSize} \\\n --enable-dp-attention`; + cmd += ` \\\n --attn-dp-size ${dpSize}`; } if (strategyArray.includes('ep')) { cmd += ` \\\n --ep ${epSize}`; diff --git a/docs/src/snippets/autoregressive/ernie-45-deployment.jsx b/docs/src/snippets/autoregressive/ernie-45-deployment.jsx index 9adf1adf4815..b024bb0aa5a4 100644 --- a/docs/src/snippets/autoregressive/ernie-45-deployment.jsx +++ b/docs/src/snippets/autoregressive/ernie-45-deployment.jsx @@ -55,7 +55,7 @@ export const Ernie45Deployment = () => { } if (strategyArray.includes('dp') && modelsize === '300b') { - cmd += ` \\\n --dp ${dpValue} \\\n --enable-dp-attention`; + cmd += ` \\\n --attn-dp-size ${dpValue}`; } if (strategyArray.includes('ep') && modelsize === '300b') { diff --git a/docs/src/snippets/autoregressive/glm-45-deployment.jsx b/docs/src/snippets/autoregressive/glm-45-deployment.jsx index 1e7cdf4450c7..1ef8dfd0093a 100644 --- a/docs/src/snippets/autoregressive/glm-45-deployment.jsx +++ b/docs/src/snippets/autoregressive/glm-45-deployment.jsx @@ -121,7 +121,7 @@ export const GLM45Deployment = () => { // Strategy-specific parameters if (strategyArray.includes('dp')) { - cmd += ` \\\n --dp 8 \\\n --enable-dp-attention`; + cmd += ` \\\n --attn-dp-size 8`; } if (strategyArray.includes('ep')) { cmd += ` \\\n --ep 8`; diff --git a/docs/src/snippets/autoregressive/glm-46-deployment.jsx b/docs/src/snippets/autoregressive/glm-46-deployment.jsx index f931a5f2e9da..fbfd1fd1ff75 100644 --- a/docs/src/snippets/autoregressive/glm-46-deployment.jsx +++ b/docs/src/snippets/autoregressive/glm-46-deployment.jsx @@ -132,7 +132,7 @@ export const GLM46Deployment = () => { // Strategy-specific parameters if (strategyArray.includes('dp')) { - cmd += ` \\\n --dp 8 \\\n --enable-dp-attention`; + cmd += ` \\\n --attn-dp-size 8`; if (hardware === 'b300') { cmd += ` \\\n --cuda-graph-max-bs-decode 256`; } diff --git a/docs/src/snippets/autoregressive/glm-47-deployment.jsx b/docs/src/snippets/autoregressive/glm-47-deployment.jsx index 9c6fbf802984..3a39b3a5f425 100644 --- a/docs/src/snippets/autoregressive/glm-47-deployment.jsx +++ b/docs/src/snippets/autoregressive/glm-47-deployment.jsx @@ -176,7 +176,7 @@ export const GLM47Deployment = () => { cmd += ` \\\n --max-context-length 8192 \\\n --mem-fraction-static 0.9`; } if (strategyArray.includes('dp')) { - cmd += ` \\\n --dp 8 \\\n --enable-dp-attention`; + cmd += ` \\\n --attn-dp-size 8`; } if (strategyArray.includes('ep')) { cmd += ` \\\n --ep 8`; diff --git a/docs/src/snippets/autoregressive/glm-47-flash-deployment.jsx b/docs/src/snippets/autoregressive/glm-47-flash-deployment.jsx index 4f5200544b76..1f6dceeb3c34 100644 --- a/docs/src/snippets/autoregressive/glm-47-flash-deployment.jsx +++ b/docs/src/snippets/autoregressive/glm-47-flash-deployment.jsx @@ -23,7 +23,6 @@ export const GLM47FlashDeployment = () => { type: 'checkbox', items: [ { id: 'tp', label: 'TP', subtitle: 'Tensor Parallel', default: true, required: true }, - { id: 'dp', label: 'DP', subtitle: 'Data Parallel', default: false }, { id: 'mtp', label: 'MTP', subtitle: 'Multi-token Prediction', default: false } ] }, @@ -113,9 +112,6 @@ export const GLM47FlashDeployment = () => { } // Strategy-specific parameters - if (strategyArray.includes('dp')) { - cmd += ` \\\n --dp 1 \\\n --enable-dp-attention`; - } if (strategyArray.includes('mtp')) { if (hardware === 'b200') { diff --git a/docs/src/snippets/autoregressive/glm-5-deployment.jsx b/docs/src/snippets/autoregressive/glm-5-deployment.jsx index 1346c07c4ea7..4d0654cac56f 100644 --- a/docs/src/snippets/autoregressive/glm-5-deployment.jsx +++ b/docs/src/snippets/autoregressive/glm-5-deployment.jsx @@ -199,7 +199,7 @@ export const GLM5Deployment = () => { } if (values.dpattention === 'enabled') { - cmd += ` \\\n --dp ${tpValue} \\\n --enable-dp-attention`; + cmd += ` \\\n --attn-dp-size ${tpValue}`; if (hardware === 'b300') { cmd += ' \\\n --cuda-graph-max-bs-decode 256'; } diff --git a/docs/src/snippets/autoregressive/glm-51-deployment.jsx b/docs/src/snippets/autoregressive/glm-51-deployment.jsx index 3b65d1a2c5bd..29a157be13a2 100644 --- a/docs/src/snippets/autoregressive/glm-51-deployment.jsx +++ b/docs/src/snippets/autoregressive/glm-51-deployment.jsx @@ -166,7 +166,7 @@ export const GLM51Deployment = () => { } if (values.dpattention === 'enabled') { - cmd += ` \\\n --dp ${tpValue} \\\n --enable-dp-attention`; + cmd += ` \\\n --attn-dp-size ${tpValue}`; } if (values.reasoning === 'enabled') cmd += ' \\\n --reasoning-parser glm45'; if (values.toolcall === 'enabled') cmd += ' \\\n --tool-call-parser glm47'; diff --git a/docs/src/snippets/autoregressive/kimi-k2-deployment.jsx b/docs/src/snippets/autoregressive/kimi-k2-deployment.jsx index 04ccff38f1d8..e6fe4753394f 100644 --- a/docs/src/snippets/autoregressive/kimi-k2-deployment.jsx +++ b/docs/src/snippets/autoregressive/kimi-k2-deployment.jsx @@ -75,7 +75,7 @@ export const KimiK2Deployment = () => { const strategyArray = Array.isArray(strategy) ? strategy : []; cmd += ` \\\n --tp 8`; if (strategyArray.includes('dp')) { - cmd += ` \\\n --dp 4 \\\n --enable-dp-attention`; + cmd += ` \\\n --attn-dp-size 4`; } if (strategyArray.includes('ep')) { cmd += ` \\\n --ep 4`; diff --git a/docs/src/snippets/autoregressive/kimi-k25-deployment.jsx b/docs/src/snippets/autoregressive/kimi-k25-deployment.jsx index 982fd443def4..ad21e8df6a3b 100644 --- a/docs/src/snippets/autoregressive/kimi-k25-deployment.jsx +++ b/docs/src/snippets/autoregressive/kimi-k25-deployment.jsx @@ -176,9 +176,9 @@ export const KimiK25Deployment = () => { cmd += ` \\\n --tp ${tpValue}`; cmd += ' \\\n --trust-remote-code'; - // DP Attention: --dp matches --tp + // DP Attention: --attn-dp-size matches --tp if (values.dpattention === 'enabled') { - cmd += ` \\\n --dp ${tpValue} \\\n --enable-dp-attention`; + cmd += ` \\\n --attn-dp-size ${tpValue}`; } // Reasoning parser diff --git a/docs/src/snippets/autoregressive/kimi-k26-deployment.jsx b/docs/src/snippets/autoregressive/kimi-k26-deployment.jsx index 3e7b998bc0a3..251155c3077a 100644 --- a/docs/src/snippets/autoregressive/kimi-k26-deployment.jsx +++ b/docs/src/snippets/autoregressive/kimi-k26-deployment.jsx @@ -175,7 +175,7 @@ export const KimiK26Deployment = () => { cmd += ' \\\n --trust-remote-code'; if (dpattention === 'enabled') { - cmd += ` \\\n --dp ${tpValue} \\\n --enable-dp-attention`; + cmd += ` \\\n --attn-dp-size ${tpValue}`; } if (reasoning === 'enabled') { diff --git a/docs/src/snippets/autoregressive/kimi-k27-code-deployment.jsx b/docs/src/snippets/autoregressive/kimi-k27-code-deployment.jsx index f654300232c3..4d3cff8386de 100644 --- a/docs/src/snippets/autoregressive/kimi-k27-code-deployment.jsx +++ b/docs/src/snippets/autoregressive/kimi-k27-code-deployment.jsx @@ -179,7 +179,7 @@ export const KimiK27CodeDeployment = () => { cmd += ' \\\n --trust-remote-code'; if (dpattention === 'enabled') { - cmd += ` \\\n --dp ${tpValue} \\\n --enable-dp-attention`; + cmd += ` \\\n --attn-dp-size ${tpValue}`; } if (reasoning === 'enabled') { diff --git a/docs/src/snippets/autoregressive/laguna-xs2-deployment.jsx b/docs/src/snippets/autoregressive/laguna-xs2-deployment.jsx index 47aef9906964..048e71f0b02d 100644 --- a/docs/src/snippets/autoregressive/laguna-xs2-deployment.jsx +++ b/docs/src/snippets/autoregressive/laguna-xs2-deployment.jsx @@ -127,8 +127,7 @@ export const LagunaXS2Deployment = () => { if (dpAttention === 'enabled') { lines[lines.length - 1] += ' \\'; - lines.push(` --dp ${tp} \\`); - lines.push(' --enable-dp-attention'); + lines.push(` --attn-dp-size ${tp}`); } if (reasoning === 'enabled') { diff --git a/docs/src/snippets/autoregressive/llama31-deployment.jsx b/docs/src/snippets/autoregressive/llama31-deployment.jsx index 2914e34de915..26b8472ff5a7 100644 --- a/docs/src/snippets/autoregressive/llama31-deployment.jsx +++ b/docs/src/snippets/autoregressive/llama31-deployment.jsx @@ -235,7 +235,6 @@ export const Llama31Deployment = () => { // NVIDIA-specific optimizations if (!isAMD && !isXeon && hardware !== 'arc_b') { if (optimization === 'throughput') { - args.push(`--enable-dp-attention`); args.push(`--mem-fraction-static 0.85`); } else if (optimization === 'latency') { args.push(`--speculative-algorithm EAGLE3`); diff --git a/docs/src/snippets/autoregressive/mimo-v2-flash-deployment.jsx b/docs/src/snippets/autoregressive/mimo-v2-flash-deployment.jsx index 936bc4a2f1a4..1fdd810f6d4c 100644 --- a/docs/src/snippets/autoregressive/mimo-v2-flash-deployment.jsx +++ b/docs/src/snippets/autoregressive/mimo-v2-flash-deployment.jsx @@ -112,7 +112,7 @@ export const MiMoV2FlashDeployment = () => { // DP settings if (!isMI355X && strategyArray.includes('dp')) { - cmd += ` \\\n --dp-size 2 \\\n --enable-dp-attention`; + cmd += ` \\\n --attn-dp-size 2`; } // Performance Optimizations diff --git a/docs/src/snippets/autoregressive/mimo-v25-deployment.jsx b/docs/src/snippets/autoregressive/mimo-v25-deployment.jsx index 5611d2016a0a..9add5be8f928 100644 --- a/docs/src/snippets/autoregressive/mimo-v25-deployment.jsx +++ b/docs/src/snippets/autoregressive/mimo-v25-deployment.jsx @@ -138,14 +138,14 @@ export const MiMoV25Deployment = () => { const c = {}; // Both checkpoints are TP-interleaved (Pro: 8, base: 4), so attention-TP per // DP group must equal that factor. When the spec carries dp>1 (Pro/Hopper - // tp=16, base tp=8) DP-attention with `--dp = tp/factor` is required; without + // tp=16, base tp=8) DP-attention with `--attn-dp-size = tp/factor` is required; without // it a bare `--tp` gives attn_tp = tp != factor and the loader rejects the // checkpoint. When no dp>1 (Pro/Blackwell tp=8, base tp=4) it's a single // attention group and DP-attention must stay off. if (spec && !jax) { const factor = isPro ? 8 : 4; if (spec.dp > 1) { - c.dpAttention = { force: "enabled", reason: `Checkpoint is TP=${factor}-interleaved; DP-attention is required (--dp = tp/${factor} = ${spec.dp}).` }; + c.dpAttention = { force: "enabled", reason: `Checkpoint is TP=${factor}-interleaved; DP-attention is required (--attn-dp-size = tp/${factor} = ${spec.dp}).` }; } else { c.dpAttention = { force: "disabled", reason: `Single attention group on this hardware (tp=${factor}, no dp-attention).` }; } @@ -243,7 +243,8 @@ export const MiMoV25Deployment = () => { // sgl-jax conventions: // - `--tp-size` is always the total JAX device count; per-DP TP is // derived automatically as tp/dp. - // - No `--enable-dp-attention` flag — DP attention is the default + // - No separate attention-DP flag (SGLang's `--attn-dp-size`) — + // sgl-jax's `--dp-size` is DP attention by default // (FFN layers auto-pick EP-split for MoE, attn-TP-split for dense). const isV7x = hardware === "tpu-v7x"; const useEp = expertParallelism === "enabled"; @@ -309,8 +310,7 @@ export const MiMoV25Deployment = () => { flags.push(` --tp ${tp}`); if (useDpAttn) { - flags.push(` --dp ${dpSize}`); - flags.push(" --enable-dp-attention"); + flags.push(` --attn-dp-size ${dpSize}`); if (!isPro) { flags.push(" --enable-dp-lm-head"); flags.push(" --mm-enable-dp-encoder"); diff --git a/docs/src/snippets/autoregressive/nemotron3-ultra-deployment.jsx b/docs/src/snippets/autoregressive/nemotron3-ultra-deployment.jsx index 537f2b54c47f..b8f444d786a8 100644 --- a/docs/src/snippets/autoregressive/nemotron3-ultra-deployment.jsx +++ b/docs/src/snippets/autoregressive/nemotron3-ultra-deployment.jsx @@ -30,7 +30,7 @@ export const Nemotron3UltraDeployment = () => { [...new Set(VERIFIED_CONFIGS.filter((c) => c.model === model && c.hardware === hardware).map((c) => c.tp))]; // DP attention is verified at dp=2 for BF16, and dp in {2,4,8} for NVFP4. SGLang - // requires tp_size % dp_size == 0, so dp is capped at both the selected TP and the + // requires tp_size % attn_dp_size == 0, so dp is capped at both the selected TP and the // max verified TP for this model+hardware (whichever is smaller). const dpCandidatesForModel = (model) => (model === 'bf16' ? ['2'] : ['2', '4', '8']); @@ -128,12 +128,12 @@ export const Nemotron3UltraDeployment = () => { }; }); }, - // dp_size must divide tp_size; only emit when the selected DP is valid for the current TP. + // attn_dp_size must divide tp_size; only emit when the selected DP is valid for the current TP. commandRule: (value, state) => value && value !== 'disabled' && dpCandidatesForModel(state.model).includes(value) && Number(value) <= Number(state.tp) - ? `--dp ${value} \\\n --enable-dp-attention` + ? `--attn-dp-size ${value}` : null }, mtp: { diff --git a/docs/src/snippets/autoregressive/qwen3-coder-deployment.jsx b/docs/src/snippets/autoregressive/qwen3-coder-deployment.jsx index 9ba7e2887648..6aa6e8e0288b 100644 --- a/docs/src/snippets/autoregressive/qwen3-coder-deployment.jsx +++ b/docs/src/snippets/autoregressive/qwen3-coder-deployment.jsx @@ -119,11 +119,6 @@ export const Qwen3CoderDeployment = () => { cmd += ` \\\n --ep 2`; } - // DP attention setting - if (quantization === 'nvfp4') { - cmd += ` \\\n --enable-dp-attention`; - } - // MOE runner backend for NVIDIA if (isNvidia) { if (quantization === 'nvfp4') { diff --git a/docs/src/snippets/configs/Qwen/qwen3.8.jsx b/docs/src/snippets/configs/Qwen/qwen3.8.jsx index b0f0ce4fc000..337e890d0043 100644 --- a/docs/src/snippets/configs/Qwen/qwen3.8.jsx +++ b/docs/src/snippets/configs/Qwen/qwen3.8.jsx @@ -367,9 +367,8 @@ export const config = { "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp-size 16", - "--dp-size 4", + "--attn-dp-size 4", "--ep-size 16", - "--enable-dp-attention", "--enable-dp-lm-head", "--enable-dp-attention-local-control-broadcast", "--moe-dense-tp-size 1", @@ -452,9 +451,8 @@ export const config = { "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp-size 16", - "--dp-size 4", + "--attn-dp-size 4", "--ep-size 16", - "--enable-dp-attention", "--enable-dp-lm-head", "--enable-dp-attention-local-control-broadcast", "--moe-dense-tp-size 1", @@ -540,8 +538,7 @@ export const config = { "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp-size 16", - "--dp-size 16", - "--enable-dp-attention", + "--attn-dp-size 16", "--enable-dp-lm-head", "--enable-dp-attention-local-control-broadcast", "--moe-dense-tp-size 1", diff --git a/docs/src/snippets/configs/deepseek-ai/deepseek-v4.jsx b/docs/src/snippets/configs/deepseek-ai/deepseek-v4.jsx index a985eff0c404..e237814df0dc 100644 --- a/docs/src/snippets/configs/deepseek-ai/deepseek-v4.jsx +++ b/docs/src/snippets/configs/deepseek-ai/deepseek-v4.jsx @@ -235,11 +235,11 @@ sgl-eval run mmmu_pro \\ playgroundFeatures: { // ----- Card 1: "Attention Parallelism" ----- - // DP-Attention is a combined knob: value is the DP degree AND toggles `--enable-dp-attention`. + // DP-Attention is a single knob: value is the attention DP size, emitted as `--attn-dp-size N`. // CP sizes auto-gate in the engine to the runtime derivation // attn_cp_size = tp/dp (a user-passed --attn-cp-size is overridden). // CP is single-machine only (tp_size <= 8). Interleave CP + DP-Attention - // currently fails the runtime's dp_size == 1 assert but is allowed here + // currently fails the runtime's attn_dp_size == 1 assert but is allowed here // with a warning (combined support is planned upstream). No `cpStrategy` // knob: DeepSeek-V4 supports only interleave (the runtime rejects zigzag). attention: { @@ -263,7 +263,7 @@ sgl-eval run mmmu_pro \\ // The low-latency and balanced PD roles run TP-only. That is what lets // their decode ladders run to the full ceiling (8, 32 on Pro Official // low-latency, and 96): the ceiling - // is server-wide and floor-divided by attn_dp_size, so only at dp_size + // is server-wide and floor-divided by attn_dp_size, so only at attn_dp_size // 1 is it also the per-rank batch. Forced rather than left to the // reader, because switching DP on would cut the slots per rank without // changing either flag in the command — the ladder would still read 96 @@ -889,8 +889,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 8", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--moe-a2a-backend deepep", "--deepep-config '{\"normal_dispatch\":{\"num_sms\":96},\"normal_combine\":{\"num_sms\":96}}'", "--host {{HOST_IP}}", @@ -905,8 +904,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 4", - "--dp 4", - "--enable-dp-attention", + "--attn-dp-size 4", "--moe-a2a-backend deepep", "--speculative-algorithm EAGLE", "--speculative-num-steps 1", @@ -927,8 +925,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 8", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--moe-a2a-backend megamoe", "--host {{HOST_IP}}", "--port {{PORT}}", @@ -944,8 +941,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 4", - "--dp 4", - "--enable-dp-attention", + "--attn-dp-size 4", "--moe-a2a-backend megamoe", "--host {{HOST_IP}}", "--port {{PORT}}", @@ -982,8 +978,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 8", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--moe-a2a-backend megamoe", "--chunked-prefill-size 32768", "--swa-full-tokens-ratio 0.1", @@ -1007,8 +1002,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 8", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--moe-a2a-backend megamoe", "--mem-fraction-static 0.835", "--cuda-graph-max-bs-decode 544", @@ -1063,8 +1057,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 4", - "--dp 4", - "--enable-dp-attention", + "--attn-dp-size 4", "--moe-a2a-backend deepep", "--deepep-config '{\"normal_dispatch\":{\"num_sms\":96},\"normal_combine\":{\"num_sms\":96}}'", "--host {{HOST_IP}}", @@ -1079,8 +1072,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 4", - "--dp 4", - "--enable-dp-attention", + "--attn-dp-size 4", "--moe-a2a-backend deepep", "--speculative-algorithm EAGLE", "--speculative-num-steps 1", @@ -1101,8 +1093,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 4", - "--dp 4", - "--enable-dp-attention", + "--attn-dp-size 4", "--moe-a2a-backend megamoe", "--host {{HOST_IP}}", "--port {{PORT}}", @@ -1118,8 +1109,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 4", - "--dp 4", - "--enable-dp-attention", + "--attn-dp-size 4", "--moe-a2a-backend megamoe", "--host {{HOST_IP}}", "--port {{PORT}}", @@ -1154,8 +1144,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 8", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--moe-runner-backend flashinfer_mxfp4", "--disable-flashinfer-autotune", "--chunked-prefill-size 32768", @@ -1181,8 +1170,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 8", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--moe-a2a-backend megamoe", "--mem-fraction-static 0.835", "--cuda-graph-max-bs-decode 544", @@ -1408,8 +1396,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 4", - "--dp 4", - "--enable-dp-attention", + "--attn-dp-size 4", "--moe-a2a-backend deepep", "--deepep-config '{\"normal_dispatch\":{\"num_sms\":96},\"normal_combine\":{\"num_sms\":96}}'", "--host {{HOST_IP}}", @@ -1424,8 +1411,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 4", - "--dp 4", - "--enable-dp-attention", + "--attn-dp-size 4", "--moe-a2a-backend deepep", "--speculative-algorithm EAGLE", "--speculative-num-steps 1", @@ -1446,8 +1432,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 4", - "--dp 4", - "--enable-dp-attention", + "--attn-dp-size 4", "--moe-a2a-backend megamoe", "--host {{HOST_IP}}", "--port {{PORT}}", @@ -1463,8 +1448,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 4", - "--dp 4", - "--enable-dp-attention", + "--attn-dp-size 4", "--moe-a2a-backend megamoe", "--host {{HOST_IP}}", "--port {{PORT}}", @@ -1507,8 +1491,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 8", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--moe-a2a-backend deepep", "--speculative-algorithm EAGLE", "--speculative-num-steps 1", @@ -1533,8 +1516,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 8", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--moe-a2a-backend megamoe", "--mem-fraction-static 0.78", "--cuda-graph-max-bs-decode 64", @@ -1673,8 +1655,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 4", - "--dp 4", - "--enable-dp-attention", + "--attn-dp-size 4", "--moe-a2a-backend deepep", "--deepep-config '{\"normal_dispatch\":{\"num_sms\":96},\"normal_combine\":{\"num_sms\":96}}'", "--host {{HOST_IP}}", @@ -1689,8 +1670,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 4", - "--dp 4", - "--enable-dp-attention", + "--attn-dp-size 4", "--moe-a2a-backend deepep", "--speculative-algorithm EAGLE", "--speculative-num-steps 1", @@ -1711,8 +1691,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 4", - "--dp 4", - "--enable-dp-attention", + "--attn-dp-size 4", "--moe-a2a-backend megamoe", "--host {{HOST_IP}}", "--port {{PORT}}", @@ -1728,8 +1707,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 4", - "--dp 4", - "--enable-dp-attention", + "--attn-dp-size 4", "--moe-a2a-backend megamoe", "--host {{HOST_IP}}", "--port {{PORT}}", @@ -1764,8 +1742,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 4", - "--dp 4", - "--enable-dp-attention", + "--attn-dp-size 4", "--moe-a2a-backend deepep", "--speculative-algorithm EAGLE", "--speculative-num-steps 1", @@ -1789,8 +1766,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 4", - "--dp 4", - "--enable-dp-attention", + "--attn-dp-size 4", "--moe-a2a-backend megamoe", "--mem-fraction-static 0.9", "--cuda-graph-max-bs-decode 128", @@ -1833,8 +1809,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 4", - "--dp 4", - "--enable-dp-attention", + "--attn-dp-size 4", "--moe-a2a-backend deepep", "--deepep-config '{\"normal_dispatch\":{\"num_sms\":96},\"normal_combine\":{\"num_sms\":96}}'", "--host {{HOST_IP}}", @@ -1855,8 +1830,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 4", - "--dp 4", - "--enable-dp-attention", + "--attn-dp-size 4", "--moe-a2a-backend megamoe", "--mem-fraction-static 0.9", "--cuda-graph-max-bs-decode 128", @@ -1898,8 +1872,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 8", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--moe-a2a-backend megamoe", "--chunked-prefill-size 32768", "--swa-full-tokens-ratio 0.1", @@ -1917,8 +1890,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 8", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--moe-a2a-backend megamoe", "--mem-fraction-static 0.835", "--cuda-graph-max-bs-decode 544", @@ -1962,8 +1934,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 8", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--moe-runner-backend flashinfer_mxfp4", "--disable-flashinfer-autotune", "--chunked-prefill-size 32768", @@ -1983,8 +1954,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 8", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--moe-a2a-backend megamoe", "--mem-fraction-static 0.835", "--cuda-graph-max-bs-decode 544", @@ -2028,8 +1998,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 8", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--moe-a2a-backend deepep", "--mem-fraction-static 0.78", "--cuda-graph-max-bs-decode 64", @@ -2046,8 +2015,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 8", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--moe-a2a-backend megamoe", "--mem-fraction-static 0.78", "--cuda-graph-max-bs-decode 64", @@ -2195,8 +2163,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 8", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--enable-dp-attention-local-control-broadcast", "--tokenizer-worker-num 8", "--stream-interval 20", @@ -2223,8 +2190,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 8", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--enable-dp-attention-local-control-broadcast", "--tokenizer-worker-num 8", "--stream-interval 20", @@ -2356,8 +2322,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 4", - "--dp 4", - "--enable-dp-attention", + "--attn-dp-size 4", "--moe-a2a-backend deepep", "--speculative-algorithm EAGLE", "--speculative-num-steps 1", @@ -2381,8 +2346,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 4", - "--dp 4", - "--enable-dp-attention", + "--attn-dp-size 4", "--moe-a2a-backend deepep", "--cuda-graph-max-bs-decode 128", "--max-running-requests 256", @@ -2402,8 +2366,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 16", - "--dp 16", - "--enable-dp-attention", + "--attn-dp-size 16", "--moe-a2a-backend deepep", "--cuda-graph-max-bs-decode 8", "--max-running-requests 32", @@ -2427,8 +2390,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 16", - "--dp 16", - "--enable-dp-attention", + "--attn-dp-size 16", "--moe-a2a-backend deepep", "--speculative-algorithm EAGLE", "--speculative-num-steps 1", @@ -2452,8 +2414,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 16", - "--dp 16", - "--enable-dp-attention", + "--attn-dp-size 16", "--moe-a2a-backend deepep", "--mem-fraction-static 0.88", "--cuda-graph-max-bs-decode 128", @@ -2857,8 +2818,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 8", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--enable-prefill-delayer", "--prefill-delayer-max-delay-ms 5000", "--attention-backend dsv4", @@ -2890,8 +2850,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 8", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--enable-prefill-delayer", "--prefill-delayer-max-delay-ms 5000", "--attention-backend dsv4", @@ -2983,8 +2942,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 8", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--enable-dp-attention-local-control-broadcast", "--tokenizer-worker-num 8", "--stream-interval 20", @@ -3019,8 +2977,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 8", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--enable-dp-attention-local-control-broadcast", "--tokenizer-worker-num 8", "--stream-interval 20", @@ -3059,8 +3016,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 8", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--enable-dp-attention-local-control-broadcast", "--tokenizer-worker-num 8", "--stream-interval 20", @@ -3095,8 +3051,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 8", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--enable-dp-attention-local-control-broadcast", "--tokenizer-worker-num 8", "--stream-interval 20", @@ -3166,8 +3121,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 8", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--enable-dp-attention-local-control-broadcast", "--tokenizer-worker-num 8", "--stream-interval 20", @@ -3206,8 +3160,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 8", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--enable-dp-attention-local-control-broadcast", "--tokenizer-worker-num 8", "--stream-interval 20", @@ -3278,8 +3231,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 8", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--enable-dp-attention-local-control-broadcast", "--tokenizer-worker-num 8", "--stream-interval 20", @@ -3318,8 +3270,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 8", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--enable-dp-attention-local-control-broadcast", "--tokenizer-worker-num 8", "--stream-interval 20", @@ -3389,8 +3340,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 8", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--enable-dp-attention-local-control-broadcast", "--tokenizer-worker-num 8", "--stream-interval 20", @@ -3429,8 +3379,7 @@ sgl-eval run mmmu_pro \\ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 8", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--enable-dp-attention-local-control-broadcast", "--tokenizer-worker-num 8", "--stream-interval 20", @@ -3606,8 +3555,7 @@ sgl-eval run mmmu_pro \\ flags: [ "--model-path {{MODEL_NAME}}", "--tp 4", - "--dp 4", - "--enable-dp-attention", + "--attn-dp-size 4", "--moe-a2a-backend deepep", "--mem-fraction-static 0.85", "--deepep-config '{\"normal_dispatch\":{\"num_sms\":96},\"normal_combine\":{\"num_sms\":96}}'", @@ -3625,8 +3573,7 @@ sgl-eval run mmmu_pro \\ flags: [ "--model-path {{MODEL_NAME}}", "--tp 4", - "--dp 4", - "--enable-dp-attention", + "--attn-dp-size 4", "--moe-a2a-backend megamoe", "--mem-fraction-static 0.85", "--host {{HOST_IP}}", @@ -3661,8 +3608,7 @@ sgl-eval run mmmu_pro \\ flags: [ "--model-path {{MODEL_NAME}}", "--tp 4", - "--dp 4", - "--enable-dp-attention", + "--attn-dp-size 4", "--moe-a2a-backend deepep", "--mem-fraction-static 0.85", "--deepep-config '{\"normal_dispatch\":{\"num_sms\":96},\"normal_combine\":{\"num_sms\":96}}'", @@ -3681,8 +3627,7 @@ sgl-eval run mmmu_pro \\ flags: [ "--model-path {{MODEL_NAME}}", "--tp 4", - "--dp 4", - "--enable-dp-attention", + "--attn-dp-size 4", "--moe-a2a-backend megamoe", "--mem-fraction-static 0.85", "--host {{HOST_IP}}", @@ -3713,8 +3658,7 @@ sgl-eval run mmmu_pro \\ flags: [ "--model-path {{MODEL_NAME}}", "--tp 4", - "--dp 4", - "--enable-dp-attention", + "--attn-dp-size 4", "--moe-a2a-backend deepep", "--mem-fraction-static 0.85", "--deepep-config '{\"normal_dispatch\":{\"num_sms\":96},\"normal_combine\":{\"num_sms\":96}}'", @@ -3733,8 +3677,7 @@ sgl-eval run mmmu_pro \\ flags: [ "--model-path {{MODEL_NAME}}", "--tp 4", - "--dp 4", - "--enable-dp-attention", + "--attn-dp-size 4", "--moe-a2a-backend megamoe", "--mem-fraction-static 0.85", "--host {{HOST_IP}}", @@ -3763,8 +3706,7 @@ sgl-eval run mmmu_pro \\ flags: [ "--model-path {{MODEL_NAME}}", "--tp 4", - "--dp 4", - "--enable-dp-attention", + "--attn-dp-size 4", "--moe-a2a-backend deepep", "--mem-fraction-static 0.85", "--deepep-config '{\"normal_dispatch\":{\"num_sms\":96},\"normal_combine\":{\"num_sms\":96}}'", @@ -3782,8 +3724,7 @@ sgl-eval run mmmu_pro \\ flags: [ "--model-path {{MODEL_NAME}}", "--tp 4", - "--dp 4", - "--enable-dp-attention", + "--attn-dp-size 4", "--moe-a2a-backend megamoe", "--mem-fraction-static 0.85", "--host {{HOST_IP}}", diff --git a/docs/src/snippets/configs/moonshotai/kimi-k3.jsx b/docs/src/snippets/configs/moonshotai/kimi-k3.jsx index 3585e5d7163d..0c821ee34862 100644 --- a/docs/src/snippets/configs/moonshotai/kimi-k3.jsx +++ b/docs/src/snippets/configs/moonshotai/kimi-k3.jsx @@ -364,7 +364,7 @@ export const config = { config.isNpuHw(s) || ["h100", "h200", "mi350x", "mi355x"].includes(s.hw) || config.isPipelined(s) || - !!config.flagOf(config.cellFor(s), "--enable-dp-attention"), + config.sizeOf(config.cellFor(s), "--attn-dp-size") > 1, disableReason: (s) => config.isNpuHw(s) ? "Only DSPARK is supported on this recipe." @@ -536,7 +536,7 @@ export const config = { playgroundFeatures: { // ----- Card: "Attention Parallelism" ----- - // DP-Attention is a combined knob: value = DP degree AND toggles `--enable-dp-attention`. + // DP-Attention is a single knob: value = attention DP size, emitted as `--attn-dp-size N`. // K3's MLA latent KV is TP-replicated, so DP-attention RAISES per-GPU KV pressure — // dp=8/attn_tp=1 OOMs on a single node; use dp=2/attn_tp=4. No CP knob: K3 uses // decode context parallel (`--dcp-size`), a different lever from prefill `--attn-cp-size`. @@ -602,12 +602,6 @@ export const config = { { id: "dpAttn", label: "DP-Attention", values: [ null, - { - // 950PR/DT Series only: the recipe enables DP-Attention at dp=1 - // (attn-TP 32), unlike A3's dp=4. - value: 1, - hide: { hw: ["b300", "gb300", "b200", "gb200", "h200", "h100", "mi350x", "mi355x", "a3"] }, - }, { value: false, get disable() { return [ @@ -615,10 +609,6 @@ export const config = { when: { hw: ["a3"] }, reason: "Only DP-Attention=4 (2 on Prefill) is supported on this recipe.", }, - { - when: { hw: ["a5"] }, - reason: "Only DP-Attention=1 is supported on this recipe.", - }, ]; }, }, { @@ -632,7 +622,7 @@ export const config = { }, { when: { hw: ["a5"] }, - reason: "Only DP-Attention=1 is supported on this recipe.", + reason: "This recipe runs without DP-Attention.", }, ]; }, }, @@ -641,7 +631,7 @@ export const config = { get disable() { return [ { when: { hw: ["a5"] }, - reason: "Only DP-Attention=1 is supported on this recipe.", + reason: "This recipe runs without DP-Attention.", }, { when: { hw: ["a3"], pdMode: ["prefill"] }, @@ -658,7 +648,7 @@ export const config = { }, { when: { hw: ["a5"] }, - reason: "Only DP-Attention=1 is supported on this recipe.", + reason: "This recipe runs without DP-Attention.", }, { when: { hw: ["b300", "gb300"] }, @@ -680,7 +670,7 @@ export const config = { }, { when: { hw: ["a5"] }, - reason: "Only DP-Attention=1 is supported on this recipe.", + reason: "This recipe runs without DP-Attention.", }, { when: { hw: ["b300", "gb300"] }, @@ -1058,7 +1048,7 @@ export const config = { stripPrefixes: [ "--tp-size", "--tp", "--tensor-parallel-size", "--ep-size", "--ep", "--expert-parallel-size", - "--enable-dp-attention", "--dp-size", "--enable-dp-lm-head", + "--attn-dp-size", "--enable-dp-lm-head", "--dcp-size", "--dcp-comm-backend", // Every B200 Unified cell carries --pp-size 2; left standing it // multiplies against the preset's --tp-size for a world size the @@ -1083,7 +1073,7 @@ export const config = { const nnodes = n / gpusPerNode; return [ `--tp-size ${n}`, `--ep-size ${n}`, - ...(dp > 1 ? ["--enable-dp-attention", `--dp-size ${dp}`, "--enable-dp-lm-head"] : []), + ...(dp > 1 ? [`--attn-dp-size ${dp}`, "--enable-dp-lm-head"] : []), ...(nnodes > 1 ? [`--nnodes ${nnodes}`, "--node-rank {{NODE_RANK}}", "--dist-init-addr {{NODE0_IP}}:20000"] : []), "--moe-a2a-backend megamoe", "--moe-runner-backend deep_gemm", "--kv-cache-dtype fp8_e4m3", "--mamba-ssm-dtype bfloat16", @@ -1115,7 +1105,7 @@ export const config = { const nnodes = n / gpusPerNode; return [ `--tp-size ${n}`, `--ep-size ${n}`, - ...(dp > 1 ? ["--enable-dp-attention", `--dp-size ${dp}`, "--enable-dp-lm-head"] : []), + ...(dp > 1 ? [`--attn-dp-size ${dp}`, "--enable-dp-lm-head"] : []), "--dcp-size 8", ...(nnodes > 1 ? [`--nnodes ${nnodes}`, "--node-rank {{NODE_RANK}}", "--dist-init-addr {{NODE0_IP}}:20000"] : []), "--moe-a2a-backend megamoe", "--moe-runner-backend deep_gemm", @@ -2452,8 +2442,7 @@ export const config = { "--model-path {{MODEL_NAME}}", "--tp-size 16", "--dcp-size 8", - "--dp-size 2", - "--enable-dp-attention", + "--attn-dp-size 2", "--ep-size 16", "--mem-fraction-static 0.85", "--disaggregation-decode-extra-slots 16", @@ -2507,8 +2496,7 @@ export const config = { "--quantization modelslim", "--dtype bfloat16", "--tp-size 64", - "--enable-dp-attention", - "--dp-size 4", + "--attn-dp-size 4", "--enable-dp-lm-head", "--enable-shared-experts-attn-tp", "--enable-dense-mlp-attn-tp", @@ -2578,8 +2566,7 @@ export const config = { "--dtype bfloat16", "--tp-size 16", "--pp-size 2", - "--dp-size 2", - "--enable-dp-attention", + "--attn-dp-size 2", "--enable-dp-lm-head", "--enable-shared-experts-attn-tp", "--enable-dense-mlp-attn-tp", @@ -2646,8 +2633,7 @@ export const config = { "--dtype bfloat16", "--tp-size 64", "--pp-size 1", - "--dp-size 4", - "--enable-dp-attention", + "--attn-dp-size 4", "--enable-dp-lm-head", "--enable-shared-experts-attn-tp", "--enable-dense-mlp-attn-tp", @@ -2672,8 +2658,8 @@ export const config = { // Ascend 950PR/DT Series: 4 nodes × 8 cards, one rank per card (TP32). // Unified PD, Balanced, DSPARK-only, with the product-line kernels armed // per env: FIAS V2 BSND for the DSpark target-verify/draft attention - // paths and the fine-grained dual-stream MoE overlap. DP-attention runs - // at dp=1 (attn-TP 32), and the shared experts / dense MLP shard across + // paths and the fine-grained dual-stream MoE overlap. It runs without + // attention DP (attn-TP 32), and the shared experts / dense MLP shard across // attention-TP through the server flags (--shared-experts-tp-size 4). // Checkpoint: the official Moonshot MXFP4 build (moonshotai/Kimi-K3, // fetched from ModelScope by SGLANG_USE_MODELSCOPE=1 above). Its routed @@ -2729,7 +2715,6 @@ export const config = { "--device npu", "--dtype bfloat16", "--tp-size 32", - "--enable-dp-attention", "--enable-dp-lm-head", "--mem-fraction-static 0.9", "--chunked-prefill-size 8192", @@ -2800,7 +2785,6 @@ export const config = { "--device npu", "--dtype bfloat16", "--tp-size 32", - "--enable-dp-attention", "--enable-dp-lm-head", "--mem-fraction-static 0.85", "--chunked-prefill-size 8192", @@ -2872,7 +2856,6 @@ export const config = { "--device npu", "--dtype bfloat16", "--tp-size 32", - "--enable-dp-attention", "--enable-dp-lm-head", "--mem-fraction-static 0.85", "--chunked-prefill-size 8192", diff --git a/docs/src/snippets/configs/rednote/dots3-note.jsx b/docs/src/snippets/configs/rednote/dots3-note.jsx index 8e85f19cd2f9..3c6d9f3edf0b 100644 --- a/docs/src/snippets/configs/rednote/dots3-note.jsx +++ b/docs/src/snippets/configs/rednote/dots3-note.jsx @@ -135,9 +135,8 @@ export const config = { flags: [ "--model-path {{MODEL_NAME}}", "--trust-remote-code", - "--enable-dp-attention", "--tp 8", - "--dp 8", + "--attn-dp-size 8", "--ep 8", "--moe-dense-tp-size 1", "--moe-a2a-backend deepep", @@ -184,9 +183,8 @@ export const config = { flags: [ "--model-path {{MODEL_NAME}}", "--trust-remote-code", - "--enable-dp-attention", "--tp 8", - "--dp 8", + "--attn-dp-size 8", "--ep 8", "--moe-dense-tp-size 1", "--moe-a2a-backend deepep", @@ -233,9 +231,8 @@ export const config = { flags: [ "--model-path {{MODEL_NAME}}", "--trust-remote-code", - "--enable-dp-attention", "--tp 8", - "--dp 8", + "--attn-dp-size 8", "--ep 8", "--moe-dense-tp-size 1", "--moe-a2a-backend deepep", @@ -282,9 +279,8 @@ export const config = { flags: [ "--model-path {{MODEL_NAME}}", "--trust-remote-code", - "--enable-dp-attention", "--tp 8", - "--dp 8", + "--attn-dp-size 8", "--ep 8", "--moe-dense-tp-size 1", "--moe-a2a-backend deepep", diff --git a/docs/src/snippets/configs/zai-org/glm-5.2.jsx b/docs/src/snippets/configs/zai-org/glm-5.2.jsx index 44cfb82ed022..9fa84902fee0 100644 --- a/docs/src/snippets/configs/zai-org/glm-5.2.jsx +++ b/docs/src/snippets/configs/zai-org/glm-5.2.jsx @@ -114,7 +114,7 @@ sgl-eval run aime25 \\ // CP sizes auto-gate in the engine to the runtime derivation // attn_cp_size = tp/dp (a user-passed --attn-cp-size is overridden). // CP is single-machine only (tp_size <= 8). Interleave CP + DP-Attention - // currently fails the runtime's dp_size == 1 assert but is allowed here + // currently fails the runtime's attn_dp_size == 1 assert but is allowed here // with a warning (combined support is planned upstream). // Strategy knob: interleave (ex round-robin-split) is the layout verified // here and the default; zigzag (ex in-seq-split) is exposed as an @@ -280,8 +280,7 @@ sgl-eval run aime25 \\ flags: [ "--model-path {{MODEL_NAME}}", "--tp 8", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--moe-a2a-backend deepep", "--speculative-algorithm EAGLE", "--speculative-num-steps 1", @@ -303,8 +302,7 @@ sgl-eval run aime25 \\ flags: [ "--model-path {{MODEL_NAME}}", "--tp 8", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--moe-a2a-backend deepep", "--mem-fraction-static 0.85", "--max-running-requests 256", @@ -339,8 +337,7 @@ sgl-eval run aime25 \\ flags: [ "--model-path {{MODEL_NAME}}", "--tp 8", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--moe-a2a-backend deepep", "--speculative-algorithm EAGLE", "--speculative-num-steps 1", @@ -362,8 +359,7 @@ sgl-eval run aime25 \\ flags: [ "--model-path {{MODEL_NAME}}", "--tp 8", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--moe-a2a-backend deepep", "--mem-fraction-static 0.85", "--max-running-requests 256", @@ -401,8 +397,7 @@ sgl-eval run aime25 \\ flags: [ "--model-path {{MODEL_NAME}}", "--tp 4", - "--dp 4", - "--enable-dp-attention", + "--attn-dp-size 4", "--moe-a2a-backend deepep", "--speculative-algorithm EAGLE", "--speculative-num-steps 1", @@ -425,8 +420,7 @@ sgl-eval run aime25 \\ flags: [ "--model-path {{MODEL_NAME}}", "--tp 4", - "--dp 4", - "--enable-dp-attention", + "--attn-dp-size 4", "--moe-a2a-backend deepep", "--mem-fraction-static 0.85", "--host {{HOST_IP}}", @@ -462,8 +456,7 @@ sgl-eval run aime25 \\ flags: [ "--model-path {{MODEL_NAME}}", "--tp 8", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--moe-a2a-backend deepep", "--speculative-algorithm EAGLE", "--speculative-num-steps 1", @@ -483,8 +476,7 @@ sgl-eval run aime25 \\ flags: [ "--model-path {{MODEL_NAME}}", "--tp 8", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--moe-a2a-backend deepep", "--mem-fraction-static 0.85", "--max-running-requests 256", @@ -728,8 +720,7 @@ sgl-eval run aime25 \\ "--model-path {{MODEL_NAME}}", "--tp 8", "--quantization modelopt_fp4", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", // Shorter draft (MTP 2-1-3) than low-latency's 5-1-6: at this concurrency the // verify overhead of a long draft outweighs the accept-length gain. "--speculative-algorithm EAGLE", @@ -752,8 +743,7 @@ sgl-eval run aime25 \\ "--model-path {{MODEL_NAME}}", "--tp 8", "--quantization modelopt_fp4", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--chunked-prefill-size 32768", "--mem-fraction-static 0.92", "--max-running-requests 512", @@ -789,8 +779,7 @@ sgl-eval run aime25 \\ flags: [ "--model-path {{MODEL_NAME}}", "--tp 8", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--quantization modelopt_fp4", // Shorter draft (MTP 2-1-3) than low-latency's 5-1-6: at this concurrency the // verify overhead of a long draft outweighs the accept-length gain. @@ -816,8 +805,7 @@ sgl-eval run aime25 \\ flags: [ "--model-path {{MODEL_NAME}}", "--tp 8", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--quantization modelopt_fp4", "--max-running-requests 1024", "--chunked-prefill-size 8192", @@ -855,8 +843,7 @@ sgl-eval run aime25 \\ "--model-path {{MODEL_NAME}}", "--tp 4", "--quantization modelopt_fp4", - "--dp 4", - "--enable-dp-attention", + "--attn-dp-size 4", // Shorter draft (MTP 2-1-3) than low-latency's 5-1-6: at this concurrency the // verify overhead of a long draft outweighs the accept-length gain. "--speculative-algorithm EAGLE", @@ -878,8 +865,7 @@ sgl-eval run aime25 \\ "--model-path {{MODEL_NAME}}", "--tp 4", "--quantization modelopt_fp4", - "--dp 4", - "--enable-dp-attention", + "--attn-dp-size 4", "--chunked-prefill-size 8192", "--mem-fraction-static 0.92", "--max-running-requests 512", diff --git a/docs/src/snippets/configs/zai-org/glm-5.3.jsx b/docs/src/snippets/configs/zai-org/glm-5.3.jsx index 08062133577f..02c8d3bf5e6d 100644 --- a/docs/src/snippets/configs/zai-org/glm-5.3.jsx +++ b/docs/src/snippets/configs/zai-org/glm-5.3.jsx @@ -143,7 +143,7 @@ sgl-eval run aime25 \\ // CP sizes auto-gate in the engine to the runtime derivation // attn_cp_size = tp/dp (a user-passed --attn-cp-size is overridden). // CP is single-machine only (tp_size <= 8). Interleave CP + DP-Attention - // currently fails the runtime's dp_size == 1 assert but is allowed here + // currently fails the runtime's attn_dp_size == 1 assert but is allowed here // with a warning (combined support is planned upstream). // Strategy knob: interleave (ex round-robin-split) is the default; // zigzag (ex in-seq-split) is exposed as an @@ -317,8 +317,7 @@ sgl-eval run aime25 \\ flags: [ "--model-path {{MODEL_NAME}}", "--tp 8", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--moe-a2a-backend deepep", "--speculative-algorithm EAGLE", "--speculative-num-steps 1", @@ -360,8 +359,7 @@ sgl-eval run aime25 \\ flags: [ "--model-path {{MODEL_NAME}}", "--tp 8", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--moe-a2a-backend deepep", "--speculative-algorithm EAGLE", "--speculative-num-steps 1", @@ -405,8 +403,7 @@ sgl-eval run aime25 \\ flags: [ "--model-path {{MODEL_NAME}}", "--tp 4", - "--dp 4", - "--enable-dp-attention", + "--attn-dp-size 4", "--moe-a2a-backend deepep", "--speculative-algorithm EAGLE", "--speculative-num-steps 1", @@ -448,8 +445,7 @@ sgl-eval run aime25 \\ flags: [ "--model-path {{MODEL_NAME}}", "--tp 8", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--moe-a2a-backend deepep", "--speculative-algorithm EAGLE", "--speculative-num-steps 1", @@ -698,8 +694,7 @@ sgl-eval run aime25 \\ "--model-path {{MODEL_NAME}}", "--tp 8", "--quantization modelopt_fp4", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--speculative-algorithm EAGLE", "--speculative-num-steps 2", "--speculative-eagle-topk 1", @@ -719,8 +714,7 @@ sgl-eval run aime25 \\ "--model-path {{MODEL_NAME}}", "--tp 8", "--quantization modelopt_fp4", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--chunked-prefill-size 32768", "--mem-fraction-static 0.92", "--max-running-requests 512", @@ -757,8 +751,7 @@ sgl-eval run aime25 \\ "--model-path {{MODEL_NAME}}", "--tp 8", "--quantization modelopt_fp4", - "--dp 8", - "--enable-dp-attention", + "--attn-dp-size 8", "--speculative-algorithm EAGLE", "--speculative-num-steps 2", "--speculative-eagle-topk 1", @@ -799,8 +792,7 @@ sgl-eval run aime25 \\ "--model-path {{MODEL_NAME}}", "--tp 4", "--quantization modelopt_fp4", - "--dp 4", - "--enable-dp-attention", + "--attn-dp-size 4", "--speculative-algorithm EAGLE", "--speculative-num-steps 2", "--speculative-eagle-topk 1", diff --git a/python/sglang/compile_deep_gemm.py b/python/sglang/compile_deep_gemm.py index 65d809442dcc..8a8ae9cd928e 100644 --- a/python/sglang/compile_deep_gemm.py +++ b/python/sglang/compile_deep_gemm.py @@ -65,7 +65,7 @@ async def warm_up_compile( disaggregation_mode: str, tokenizer_manager: TokenizerManager ): print("\nGenerate warm up request for compiling DeepGEMM...\n") - dp_size = get_parallel().dp_size + dp_size = get_parallel().num_dp_ranks base_ids = [0, 1, 2, 3] sampling_params = { "temperature": 0.0, diff --git a/python/sglang/srt/arg_groups/argparse_actions.py b/python/sglang/srt/arg_groups/argparse_actions.py index 6b2b31e800c2..c63052444e36 100644 --- a/python/sglang/srt/arg_groups/argparse_actions.py +++ b/python/sglang/srt/arg_groups/argparse_actions.py @@ -42,11 +42,11 @@ def print_deprecated_warning(message: str): # * an old valued flag replaced by a renamed valued flag # -> `DeprecatedAliasStoreAction`. # -# Only the second has a registration today (`--disable-cuda-graph`). The other -# three are kept because the shapes recur -- this package has retired flags of -# every one of them -- and the fiddly parts (`nargs=0` on a boolean, where the -# const goes, warn-and-continue versus `parser.error`) are what a -# reimplementation gets wrong. Pass `new_flag=` so the warning tells the +# Only the second has registrations today (`--disable-cuda-graph`, +# `--enable-dp-attention`). The other three are kept because the shapes recur +# -- this package has retired flags of every one of them -- and the fiddly +# parts (`nargs=0` on a boolean, where the const goes, warn-and-continue versus +# `parser.error`) are what a reimplementation gets wrong. Pass `new_flag=` so the warning tells the # operator what to switch to; that pointer is the whole point. diff --git a/python/sglang/srt/arg_groups/cuda_graph_hook.py b/python/sglang/srt/arg_groups/cuda_graph_hook.py index 1aeee2839b60..0d3f35402477 100644 --- a/python/sglang/srt/arg_groups/cuda_graph_hook.py +++ b/python/sglang/srt/arg_groups/cuda_graph_hook.py @@ -23,7 +23,7 @@ with_phase, ) from sglang.srt.platforms import current_platform -from sglang.srt.runtime_context import get_platform +from sglang.srt.runtime_context import attn_dp_enabled_of, get_platform from sglang.srt.utils.common import ( is_cpu, is_mps, @@ -186,7 +186,7 @@ def disable_tc_piecewise_cudagraph_if_incompatible(server_args: Any): "model-arch blacklist", lambda: model_config_of(server_args).is_piecewise_cuda_graph_disabled_model, ), - ("DP attention", lambda: resolved_view(server_args).enable_dp_attention), + ("DP attention", lambda: attn_dp_enabled_of(resolved_view(server_args))), ("full torch.compile mode", lambda: cfg.enable_torch_compile), ("pipeline parallelism (pp_size > 1)", lambda: cfg.pp_size > 1), ( diff --git a/python/sglang/srt/arg_groups/deepseek_v4_hook.py b/python/sglang/srt/arg_groups/deepseek_v4_hook.py index b109483f7887..9a83ee1f22f9 100644 --- a/python/sglang/srt/arg_groups/deepseek_v4_hook.py +++ b/python/sglang/srt/arg_groups/deepseek_v4_hook.py @@ -11,7 +11,7 @@ run_post_process_pass, ) from sglang.srt.environ import envs -from sglang.srt.runtime_context import get_platform +from sglang.srt.runtime_context import attn_dp_enabled_of, get_platform, num_dp_ranks_of from sglang.srt.utils.common import is_gfx95_supported, is_npu if TYPE_CHECKING: @@ -123,10 +123,18 @@ def validate_deepseek_v4_cp(server_args: ServerArgs) -> None: "reindexes with interleave order." ) + # DeepSeek-V4 CP runs data-parallel groups as attention DP. + assert not (cfg.attn_dp_size > 1 and cfg.dp_size > 1), ( + f"--dp-size {cfg.dp_size} with --attn-dp-size {cfg.attn_dp_size}: " + "data-parallel replicas combined with attention data parallelism " + "are not supported." + ) + attn_dp_size = cfg.attn_dp_size * cfg.dp_size declare_resolution( server_args, "validate_deepseek_v4_cp", - enable_dp_attention=True, + attn_dp_size=attn_dp_size, + dp_size=1, ) declare_resolution( server_args, @@ -136,10 +144,10 @@ def validate_deepseek_v4_cp(server_args: ServerArgs) -> None: declare_resolution( server_args, "validate_deepseek_v4_cp", - attn_cp_size=cfg.tp_size // cfg.dp_size, + attn_cp_size=cfg.tp_size // attn_dp_size, ) if not is_npu(): - assert cfg.dp_size == 1, ( + assert attn_dp_size == 1, ( "For round-robin split mode, dp attention is not supported." ) assert cfg.tp_size <= 8, ( @@ -161,7 +169,7 @@ def validate_deepseek_v4_cp(server_args: ServerArgs) -> None: logger.warning( f"Enable Context Parallel for DeepSeekV4, " f"strategy={cfg.cp_strategy}, " - f"dp_size={cfg.dp_size}, moe_dense_tp_size={cfg.moe_dense_tp_size}, " + f"attn_dp_size={attn_dp_size}, moe_dense_tp_size={cfg.moe_dense_tp_size}, " f"attn_cp_size={cfg.attn_cp_size}, ep_size={cfg.ep_size}, tp_size={cfg.tp_size}" ) @@ -193,7 +201,7 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None: "prefill CUDA graphs", cfg.cuda_graph_config.prefill.backend != Backend.DISABLED, ), - ("DP attention", cfg.enable_dp_attention), + ("DP attention", attn_dp_enabled_of(cfg)), ("context parallelism", cfg.attn_cp_size > 1), ("external cache linker", cfg.enable_unified_cache_external_linker), ("unified memory", cfg.enable_unified_memory), @@ -246,8 +254,8 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None: if ( read_ragged_verify_mode() is not RaggedVerifyMode.STATIC or cfg.disaggregation_transfer_backend != "mooncake" - or cfg.dp_size != 1 - or cfg.enable_dp_attention + or num_dp_ranks_of(cfg) != 1 + or attn_dp_enabled_of(cfg) or cfg.attn_cp_size != 1 or cfg.dcp_size != 1 ): @@ -285,7 +293,7 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None: cfg.cuda_graph_config.prefill.backend != Backend.DISABLED, ), # input_ids_global is a DP-wide gather, so the tail slice cannot apply. - ("DP attention", cfg.enable_dp_attention), + ("DP attention", attn_dp_enabled_of(cfg)), ) for feature, enabled in incompatible: if enabled: diff --git a/python/sglang/srt/arg_groups/expert_pack_hook.py b/python/sglang/srt/arg_groups/expert_pack_hook.py index ecb4aa5c90eb..f6a10e666403 100644 --- a/python/sglang/srt/arg_groups/expert_pack_hook.py +++ b/python/sglang/srt/arg_groups/expert_pack_hook.py @@ -40,6 +40,7 @@ def handle_expert_pack(server_args: Any) -> None: parallelism = ( ("tensor", "--tp-size", cfg.tp_size), ("data", "--dp-size", cfg.dp_size), + ("attention data", "--attn-dp-size", cfg.attn_dp_size), ("expert", "--ep-size", cfg.ep_size), ) for label, option, size in parallelism: diff --git a/python/sglang/srt/arg_groups/field_order.py b/python/sglang/srt/arg_groups/field_order.py index 20c35da68093..d38e3983953d 100644 --- a/python/sglang/srt/arg_groups/field_order.py +++ b/python/sglang/srt/arg_groups/field_order.py @@ -79,6 +79,7 @@ "pp_max_micro_batch_size", "pp_async_batch_depth", "dp_size", + "attn_dp_size", "load_balance_method", "attn_cp_size", "moe_dp_size", diff --git a/python/sglang/srt/arg_groups/fields/parallel.py b/python/sglang/srt/arg_groups/fields/parallel.py index 3e0e28e6d4c8..edf20e3d410d 100644 --- a/python/sglang/srt/arg_groups/fields/parallel.py +++ b/python/sglang/srt/arg_groups/fields/parallel.py @@ -73,8 +73,19 @@ class Parallel(msgspec.Struct): dp_size: A[ int, Arg( - help="The data parallelism size.", + help="The number of data-parallel replicas of the model.", aliases=["--data-parallel-size"], + resolvable=True, + ), + ] = 1 + attn_dp_size: A[ + int, + Arg( + help="The attention data parallelism size: the number of " + "data-parallel attention groups inside the TP group, one scheduler " + "each, while the FFN stays tensor parallel.", + aliases=["--attention-data-parallel-size"], + resolvable=True, ), ] = 1 load_balance_method: A[ @@ -159,14 +170,9 @@ class Parallel(msgspec.Struct): bool, "Enable attention tensor-parallel weight slicing during decode under context parallel (cp_size>1). Slices the replicated attention linears to the local CP partition, eliminating redundant decode GEMMs.", ] = False - # DP attention - enable_dp_attention: A[ - bool, - Arg( - help="Enabling data parallelism for attention and tensor parallelism for FFN. The dp size should be equal to the tp size. Currently DeepSeek-V2 and Qwen 2/3 MoE models are supported.", - resolvable=True, - ), - ] = False + # Deprecated spelling of `attn_dp_size`: `--dp-size N --enable-dp-attention` + # resolves to `attn_dp_size = N`, `dp_size = 1`. + enable_dp_attention: A[bool, Arg(no_cli=True, resolvable=True)] = False enable_dp_attention_local_control_broadcast: A[ bool, "With DP-attention, send control messages to every DP group leader and broadcast within attn_tp_group instead of the full tp_group. Eliminates a costly all-ranks gloo sync on every scheduler iteration.", @@ -184,7 +190,7 @@ class Parallel(msgspec.Struct): help="Use all-to-all instead of TP all-gather followed by DP scatter " "for the TP-sharded LM head under DP attention. By default this is " "enabled only on decode-only PD nodes with pure DP attention " - "(tp_size == dp_size > 1 and attn_cp_size == 1), and disabled on " + "(tp_size == attn_dp_size > 1 and attn_cp_size == 1), and disabled on " "prefill-only and colocated nodes. Pass " "--no-enable-tp-lm-head-all-to-all to opt out. The path is " "incompatible with --enable-dp-lm-head; batches without an equal " @@ -282,10 +288,18 @@ class Parallel(msgspec.Struct): doc="Attention tensor-parallel width: `tp_size` divided by the " "attention-DP and attention-CP dimensions.", ) - attn_dp_size = Derived( - fn="sglang.srt.runtime_context.attn_dp_size_of", - doc="Attention data-parallel width: `dp_size` when DP attention is " - "on, otherwise one.", + attn_dp_enabled = Derived( + fn="sglang.srt.runtime_context.attn_dp_enabled_of", + doc="Whether attention runs data parallel: `attn_dp_size` is wider " + "than one, or this process is an elastic EP scale joiner, which joins " + "an attention-DP deployment with a group of its own that may be one " + "rank wide.", + ) + num_dp_ranks = Derived( + fn="sglang.srt.runtime_context.num_dp_ranks_of", + doc="How many data-parallel ranks the deployment serves with, one " + "scheduler each: `dp_size * attn_dp_size`. Elastic EP scale-up widens " + "it as ranks join.", ) attn_dcp_size = Derived( fn="sglang.srt.runtime_context.attn_dcp_size_of", diff --git a/python/sglang/srt/arg_groups/kv_cache_hook.py b/python/sglang/srt/arg_groups/kv_cache_hook.py index 5db8194b9dda..196d8fdfd94d 100644 --- a/python/sglang/srt/arg_groups/kv_cache_hook.py +++ b/python/sglang/srt/arg_groups/kv_cache_hook.py @@ -18,7 +18,7 @@ from sglang.srt.configs.hybrid_arch import mambaish_config from sglang.srt.environ import envs from sglang.srt.model_executor.cuda_graph_config import Backend -from sglang.srt.runtime_context import get_platform +from sglang.srt.runtime_context import attn_dp_enabled_of, get_platform logger = logging.getLogger(__name__) @@ -450,7 +450,7 @@ def handle_cache_compatibility(server_args: Any) -> None: raise NotImplementedError( "LMCacheUnifiedRadixCache does not yet support speculative decoding" ) - if cfg.enable_dp_attention: + if attn_dp_enabled_of(cfg): raise NotImplementedError( "LMCacheUnifiedRadixCache does not yet support DP attention" ) diff --git a/python/sglang/srt/arg_groups/layernorm_sp_hook.py b/python/sglang/srt/arg_groups/layernorm_sp_hook.py index ddc14ddbafd0..025a642c8041 100644 --- a/python/sglang/srt/arg_groups/layernorm_sp_hook.py +++ b/python/sglang/srt/arg_groups/layernorm_sp_hook.py @@ -7,6 +7,7 @@ from sglang.srt.server_args import ServerArgs from sglang.srt.arg_groups.overrides import model_config_of, resolving_view +from sglang.srt.runtime_context import attn_dp_enabled_of logger = logging.getLogger(__name__) @@ -24,7 +25,7 @@ def handle_layernorm_sp(server_args: ServerArgs) -> None: validate_layernorm_sp( architecture=architectures[0] if architectures else None, tp_size=cfg.tp_size, - enable_dp_attention=cfg.enable_dp_attention, + attn_dp_enabled=attn_dp_enabled_of(cfg), speculative_algorithm=cfg.speculative_algorithm, ) @@ -33,7 +34,7 @@ def validate_layernorm_sp( *, architecture: Optional[str], tp_size: int, - enable_dp_attention: bool, + attn_dp_enabled: bool, speculative_algorithm: Optional[str], ) -> None: """Fail loud for unsupported / incompatible configs. Callers gate on the flag.""" @@ -49,9 +50,9 @@ def validate_layernorm_sp( "--enable-layernorm-sp requires tp_size > 1: there is no sequence to " "shard across a single TP rank." ) - if enable_dp_attention: + if attn_dp_enabled: raise ValueError( - "--enable-layernorm-sp is not compatible with --enable-dp-attention: " + "--enable-layernorm-sp is not compatible with attention DP: " "SP shards the sequence across the full TP group, which under DP " "attention spans data-parallel groups holding different sequences." ) diff --git a/python/sglang/srt/arg_groups/mega_moe_hook.py b/python/sglang/srt/arg_groups/mega_moe_hook.py index d45ae3caff1f..654c956bf3bc 100644 --- a/python/sglang/srt/arg_groups/mega_moe_hook.py +++ b/python/sglang/srt/arg_groups/mega_moe_hook.py @@ -13,7 +13,7 @@ ) from sglang.srt.connector import ConnectorType from sglang.srt.environ import envs -from sglang.srt.runtime_context import get_platform +from sglang.srt.runtime_context import attn_dp_enabled_of, get_platform from sglang.srt.utils.common import parse_connector_type logger = logging.getLogger(__name__) @@ -143,11 +143,11 @@ def validate_mega_moe_token_budget(server_args: ServerArgs, model_label: str) -> local_chunked_prefill_size = ( cfg.chunked_prefill_size + token_partition_size - 1 ) // token_partition_size - elif cfg.enable_dp_attention: - token_partition_size = cfg.dp_size - token_partition_name = "dp_size" + elif attn_dp_enabled_of(cfg): + token_partition_size = cfg.attn_dp_size + token_partition_name = "attn_dp_size" token_alignment = max( - cfg.tp_size // cfg.dp_size // cfg.attn_cp_size, + cfg.tp_size // cfg.attn_dp_size // cfg.attn_cp_size, 1, ) local_chunked_prefill_size = cfg.chunked_prefill_size // token_partition_size diff --git a/python/sglang/srt/arg_groups/memory_hook.py b/python/sglang/srt/arg_groups/memory_hook.py index aed841a57265..ae2ac83e84f8 100644 --- a/python/sglang/srt/arg_groups/memory_hook.py +++ b/python/sglang/srt/arg_groups/memory_hook.py @@ -17,7 +17,7 @@ ) from sglang.srt.environ import envs from sglang.srt.model_executor.cuda_graph_config import Backend, Phase -from sglang.srt.runtime_context import get_platform +from sglang.srt.runtime_context import attn_dp_enabled_of, get_platform from sglang.srt.utils.common import get_device_memory_capacity logger = logging.getLogger(__name__) @@ -361,14 +361,14 @@ def reserve_for_graph_mb(server_args: Any) -> float: reserved_mem += decode_cuda_graph_config.max_bs * 2 if ( - resolved_view(server_args).enable_dp_attention + attn_dp_enabled_of(resolved_view(server_args)) and cfg.disaggregation_mode != "prefill" ): # DP attention needs more padding for some operations, and much more for large # cuda graph max bs (torch allocator / implementation inefficiencies). - reserved_mem += decode_cuda_graph_config.max_bs * cfg.dp_size * 3 + reserved_mem += decode_cuda_graph_config.max_bs * cfg.attn_dp_size * 3 if decode_cuda_graph_config.max_bs > 300: - reserved_mem += decode_cuda_graph_config.max_bs * cfg.dp_size * 1.5 + reserved_mem += decode_cuda_graph_config.max_bs * cfg.attn_dp_size * 1.5 if ( cfg.disaggregation_mode != "decode" diff --git a/python/sglang/srt/arg_groups/model_hook.py b/python/sglang/srt/arg_groups/model_hook.py index da31f4a1d254..1fe46d046751 100644 --- a/python/sglang/srt/arg_groups/model_hook.py +++ b/python/sglang/srt/arg_groups/model_hook.py @@ -37,7 +37,11 @@ from sglang.srt.environ import envs from sglang.srt.hardware_backend.mlx.runtime import use_mlx from sglang.srt.model_executor.cuda_graph_config import Backend, Phase, with_phase -from sglang.srt.runtime_context import derive_attention_widths, get_platform +from sglang.srt.runtime_context import ( + attn_dp_enabled_of, + derive_attn_tp_size, + get_platform, +) from sglang.srt.utils.common import ( get_quantization_config, is_mps, @@ -290,15 +294,14 @@ def handle_model_specific_adjustments(server_args: Any): ) else: # Pure TP and partial DP Attention mode is active for DSA, logging a warning - if cfg.dp_size < cfg.tp_size: - _, attn_tp_size = derive_attention_widths( - tp_size=cfg.tp_size, - attn_cp_size=cfg.attn_cp_size, - dp_size=cfg.dp_size, - enable_dp_attention=cfg.enable_dp_attention, - ) + attn_tp_size = derive_attn_tp_size( + tp_size=cfg.tp_size, + attn_cp_size=cfg.attn_cp_size, + attn_dp_size=cfg.attn_dp_size, + ) + if attn_tp_size > 1: logger.warning( - f"DSA with TP mode is active, dp_size={cfg.dp_size}, tp_size={cfg.tp_size}, " + f"DSA with TP mode is active, attn_dp_size={cfg.attn_dp_size}, tp_size={cfg.tp_size}, " f"attn_tp_size={attn_tp_size}, attention weights will be sharded across {attn_tp_size} ranks." ) @@ -410,7 +413,7 @@ def handle_model_specific_adjustments(server_args: Any): if is_deepseek_dsa(hf_config) and not envs.SGLANG_OPT_USE_TOPK_V2.is_set(): # Prefer HIP top-k by default while honoring an explicit selection. envs.SGLANG_OPT_USE_TOPK_V2.set(False) - if not resolved_view(server_args).enable_dp_attention and cfg.nnodes == 1: + if not attn_dp_enabled_of(resolved_view(server_args)) and cfg.nnodes == 1: # TODO (Hubert): Put this back later # server_args.enable_aiter_allreduce_fusion = True @@ -514,7 +517,7 @@ def handle_model_specific_adjustments(server_args: Any): quant_method = get_quantization_config(hf_config) is_mxfp4_quant_format = quant_method == "mxfp4" if ( - not resolved_view(server_args).enable_dp_attention + not attn_dp_enabled_of(resolved_view(server_args)) and cfg.nnodes == 1 and get_platform().is_hip ): @@ -543,8 +546,11 @@ def handle_model_specific_adjustments(server_args: Any): if model_arch == "MiMoV2ForCausalLM" and not cfg.encoder_only: expected_attn_tp_size = get_mimo_v2_fused_qkv_expected_tp_size(hf_config) view = resolved_view(server_args) - attn_dp_size = cfg.dp_size if view.enable_dp_attention else 1 - effective_attn_tp_size = cfg.tp_size // attn_dp_size // view.attn_cp_size + effective_attn_tp_size = derive_attn_tp_size( + tp_size=cfg.tp_size, + attn_cp_size=view.attn_cp_size, + attn_dp_size=cfg.attn_dp_size, + ) if ( expected_attn_tp_size is not None and expected_attn_tp_size % effective_attn_tp_size != 0 @@ -555,10 +561,9 @@ def handle_model_specific_adjustments(server_args: Any): "qkv_proj weights are " f"TP={expected_attn_tp_size}-interleaved; got " f"{effective_attn_tp_size} " - f"(tp_size={cfg.tp_size}, dp_size={cfg.dp_size}, " - f"enable_dp_attention={view.enable_dp_attention}, " + f"(tp_size={cfg.tp_size}, attn_dp_size={cfg.attn_dp_size}, " f"attn_cp_size={view.attn_cp_size}). " - "Set --tp, --dp, --enable-dp-attention, and " + "Set --tp, --attn-dp-size, and " "--attention-context-parallel-size so the effective " f"attention TP size is {expected_attn_tp_size}." ) diff --git a/python/sglang/srt/arg_groups/model_overrides/deepseek_v2.py b/python/sglang/srt/arg_groups/model_overrides/deepseek_v2.py index 23568e15d1cc..87b19a5c8dbb 100644 --- a/python/sglang/srt/arg_groups/model_overrides/deepseek_v2.py +++ b/python/sglang/srt/arg_groups/model_overrides/deepseek_v2.py @@ -85,7 +85,15 @@ def _deepseek_family_overrides(server_args: Any, hf_config: Any) -> dict: logger.warning( "Context parallel feature is still under experiment. It has only been verified on Hopper platform." ) - overrides["enable_dp_attention"] = True + # DSA CP runs data-parallel groups as attention DP. + assert not (cfg.attn_dp_size > 1 and cfg.dp_size > 1), ( + f"--dp-size {cfg.dp_size} with --attn-dp-size {cfg.attn_dp_size}: " + "data-parallel replicas combined with attention data parallelism " + "are not supported." + ) + attn_dp_size = cfg.attn_dp_size * cfg.dp_size + overrides["attn_dp_size"] = attn_dp_size + overrides["dp_size"] = 1 overrides["moe_dense_tp_size"] = 1 if cfg.cp_strategy == "zigzag": overrides["moe_a2a_backend"] = "deepep" @@ -95,7 +103,7 @@ def _deepseek_family_overrides(server_args: Any, hf_config: Any) -> dict: "moe_a2a_backend=deepep, ep_size=tp_size, batch_size=1." ) else: - assert cfg.dp_size == 1, ( + assert attn_dp_size == 1, ( "interleave DSA CP does not support DP attention." ) assert cfg.tp_size <= 8, ( @@ -104,11 +112,11 @@ def _deepseek_family_overrides(server_args: Any, hf_config: Any) -> dict: # Note(kpham-sgl): Keep attn_tp_size == 1 under DSA CP. # The DSA / MLA CP gather and reduce-scatter # (the dsa_cp_* helpers in adapters/context_parallel.py) assume it. - attn_cp_size = cfg.tp_size // cfg.dp_size + attn_cp_size = cfg.tp_size // attn_dp_size overrides["attn_cp_size"] = attn_cp_size logger.warning( "Enabled DSA context parallel: " - f"strategy={cfg.cp_strategy}, dp_size={cfg.dp_size}, " + f"strategy={cfg.cp_strategy}, attn_dp_size={attn_dp_size}, " f"moe_dense_tp_size={overrides['moe_dense_tp_size']}, " f"ep_size={overrides.get('ep_size', cfg.ep_size)}, tp_size={cfg.tp_size}, " f"attn_cp_size={attn_cp_size}, " @@ -158,7 +166,15 @@ def _deepseek_family_overrides(server_args: Any, hf_config: Any) -> dict: "MLA prefill context parallel is still experimental. " "Verified on Hopper with the fa3 backend." ) - overrides["enable_dp_attention"] = True + # MLA CP runs data-parallel groups as attention DP. + assert not (cfg.attn_dp_size > 1 and cfg.dp_size > 1), ( + f"--dp-size {cfg.dp_size} with --attn-dp-size {cfg.attn_dp_size}: " + "data-parallel replicas combined with attention data parallelism " + "are not supported." + ) + attn_dp_size = cfg.attn_dp_size * cfg.dp_size + overrides["attn_dp_size"] = attn_dp_size + overrides["dp_size"] = 1 # TODO(kpham-sgl) Supports moe_dense_tp_size != 1. overrides["moe_dense_tp_size"] = 1 overrides["moe_a2a_backend"] = "deepep" @@ -169,11 +185,11 @@ def _deepseek_family_overrides(server_args: Any, hf_config: Any) -> dict: # FIXME(kpham-sgl): Keep attn_tp_size == 1 under MLA CP. # The DSA / MLA CP gather and reduce-scatter # (the dsa_cp_* helpers in adapters/context_parallel.py) assume it. - attn_cp_size = cfg.tp_size // cfg.dp_size + attn_cp_size = cfg.tp_size // attn_dp_size overrides["attn_cp_size"] = attn_cp_size logger.warning( f"Enable Context Parallel opt for MLA, " - f"Setting dp_size == {cfg.dp_size} and " + f"Setting attn_dp_size == {attn_dp_size} and " f"attn_cp_size == {attn_cp_size}, " f"moe_dense_tp_size == {overrides['moe_dense_tp_size']}, " f"ep_size == {overrides['ep_size']}, " diff --git a/python/sglang/srt/arg_groups/model_overrides/iquest_q1.py b/python/sglang/srt/arg_groups/model_overrides/iquest_q1.py index 7e8f1608e5df..b0b9c09ebb60 100644 --- a/python/sglang/srt/arg_groups/model_overrides/iquest_q1.py +++ b/python/sglang/srt/arg_groups/model_overrides/iquest_q1.py @@ -5,6 +5,7 @@ is_attention_backend_not_set, resolving_view, ) +from sglang.srt.runtime_context import attn_dp_enabled_of @_register_for("IQuestQ1ForCausalLM", "IQuestQ1MTP") @@ -33,7 +34,7 @@ def _iquest_q1_overrides(server_args: Any, hf_config: Any) -> dict: ) if cfg.enable_multi_layer_eagle: raise ValueError("IQuest Q1 MTP drafting does not support multi-layer EAGLE.") - if cfg.enable_dp_attention: + if attn_dp_enabled_of(cfg): raise ValueError("IQuest Q1 does not support DP attention.") if cfg.pp_size > 1: raise ValueError("IQuest Q1 does not support pipeline parallelism.") diff --git a/python/sglang/srt/arg_groups/model_overrides/minicpm.py b/python/sglang/srt/arg_groups/model_overrides/minicpm.py index 3790c30d6e60..ed03eb303ff0 100644 --- a/python/sglang/srt/arg_groups/model_overrides/minicpm.py +++ b/python/sglang/srt/arg_groups/model_overrides/minicpm.py @@ -8,13 +8,13 @@ resolving_view, ) from sglang.srt.environ import envs -from sglang.srt.runtime_context import get_platform +from sglang.srt.runtime_context import attn_dp_enabled_of, get_platform @_register_for("MiniCPMForCausalLM", "MiniCPMSALAForCausalLM") def _minicpm_sala_overrides(server_args: Any, hf_config: Any) -> dict: cfg = resolving_view(server_args) - if cfg.enable_dp_attention: + if attn_dp_enabled_of(cfg): raise ValueError("MiniCPM does not support DP attention") has_sparse_attention = getattr(hf_config, "has_minicpm_sparse_attention", False) has_hybrid_attention = has_sparse_attention or getattr( diff --git a/python/sglang/srt/arg_groups/moe_hook.py b/python/sglang/srt/arg_groups/moe_hook.py index 4306ac25eb09..13fd2b2a686c 100644 --- a/python/sglang/srt/arg_groups/moe_hook.py +++ b/python/sglang/srt/arg_groups/moe_hook.py @@ -29,7 +29,7 @@ from sglang.srt.connector import ConnectorType from sglang.srt.environ import envs from sglang.srt.model_executor.cuda_graph_config import Backend, Phase, with_phase -from sglang.srt.runtime_context import get_platform +from sglang.srt.runtime_context import attn_dp_enabled_of, get_platform from sglang.srt.utils.common import is_sm100_supported, parse_connector_type logger = logging.getLogger(__name__) @@ -269,8 +269,8 @@ def handle_a2a_moe(server_args: Any): if a2a_backend == "flashinfer_megamoe": validate_flashinfer_megamoe_model(server_args) validate_flashinfer_megamoe_envs() - assert cfg.enable_dp_attention and cfg.dp_size == cfg.tp_size, ( - "FlashInfer MegaMOE is only supported with dp_size == tp_size and --enable-dp-attention" + assert attn_dp_enabled_of(cfg) and cfg.attn_dp_size == cfg.tp_size, ( + "FlashInfer MegaMOE is only supported with --attn-dp-size equal to --tp-size" ) if resolved_view(server_args).moe_runner_backend == "auto": declare_resolution( @@ -395,11 +395,8 @@ def handle_a2a_moe(server_args: Any): ) if a2a_now == "flashinfer": - assert ( - resolved_view(server_args).enable_dp_attention - and cfg.dp_size == cfg.tp_size - ), ( - "Flashinfer MoE A2A is only supported with dp_size == tp_size and --enable-dp-attention" + assert attn_dp_enabled_of(cfg) and cfg.attn_dp_size == cfg.tp_size, ( + "Flashinfer MoE A2A is only supported with --attn-dp-size equal to --tp-size" ) if cfg.deepep_mode != "auto": logger.warning("--deepep-mode is ignored for Flashinfer MoE A2A") @@ -491,12 +488,12 @@ def handle_a2a_moe(server_args: Any): deepep_mode="low_latency", ) logger.warning("auto set deepep_mode=`low_latency` for PPLX EP") - # pplx-kernels' AllToAll needs numDPGroups (== attention dp_size) > 1; + # pplx-kernels' AllToAll needs numDPGroups (== attn_dp_size) > 1; # without DP attention numDPGroups == 1 and construction fails deep in # the kernel. This also implies ep_size >= 2. - assert resolved_view(server_args).enable_dp_attention and cfg.dp_size >= 2, ( - "moe_a2a_backend='pplx' requires --enable-dp-attention with at " - "least 2 DP groups (--dp-size >= 2)." + assert cfg.attn_dp_size >= 2, ( + "moe_a2a_backend='pplx' requires attention DP with at least 2 " + "groups (--attn-dp-size >= 2)." ) # pplx runs the masked DeepGEMM expert path (sm_90a): reject other # runners and resolve auto -> deep_gemm. Unquantized bf16 pplx needs @@ -581,8 +578,7 @@ def validate_deepep_v2_dispatch_token_budget(server_args: Any) -> None: graph_bs = decode_config.max_bs or 0 if view.max_running_requests is not None: - attn_dp_size = view.dp_size if view.enable_dp_attention else 1 - per_rank_pool_bs = max(1, view.max_running_requests // attn_dp_size) + per_rank_pool_bs = max(1, view.max_running_requests // view.attn_dp_size) graph_bs = min(graph_bs, per_rank_pool_bs) tokens_per_req = ( max_speculative_num_draft_tokens(server_args) or 1 diff --git a/python/sglang/srt/arg_groups/overrides.py b/python/sglang/srt/arg_groups/overrides.py index a2de0bc2937b..05eaa3693e84 100644 --- a/python/sglang/srt/arg_groups/overrides.py +++ b/python/sglang/srt/arg_groups/overrides.py @@ -62,8 +62,10 @@ from sglang.srt.environ import envs from sglang.srt.model_executor.cuda_graph_config import Backend from sglang.srt.runtime_context import ( + attn_dp_enabled_of, get_context, get_platform, + num_dp_ranks_of, ) from sglang.srt.utils.common import ( get_quantization_config, @@ -931,7 +933,7 @@ def _flashinfer_allreduce_fusion_auto_enable(view: Any) -> dict: and not prefer_custom_dsv41 and (get_platform().is_sm90 or get_platform().is_sm100) and view.tp_size > 1 - and not view.enable_dp_attention + and not attn_dp_enabled_of(view) and (view.nnodes == 1 or get_platform().is_sm100) and view.moe_a2a_backend == "none" ): @@ -1304,8 +1306,8 @@ def _page_size_default(view: Any) -> dict: @register_post_process def _data_parallelism_defaults(view: Any) -> dict: - if view.dp_size == 1 and view.ep_join_mode != "scale": - return {"enable_dp_attention": False, "enable_dp_lm_head": False} + if num_dp_ranks_of(view) == 1 and view.ep_join_mode != "scale": + return {"enable_dp_lm_head": False} return {} @@ -1349,9 +1351,8 @@ def _tp_lm_head_all_to_all_default(view: Any) -> dict: enable = ( view.disaggregation_mode == "decode" - and view.enable_dp_attention - and view.dp_size > 1 - and view.tp_size == view.dp_size + and view.attn_dp_size > 1 + and view.tp_size == view.attn_dp_size and view.attn_cp_size == 1 and not view.enable_dp_lm_head ) @@ -1362,20 +1363,21 @@ def _tp_lm_head_all_to_all_default(view: Any) -> dict: def _dp_lm_head_validation(view: Any) -> dict: """Require DP attention for DP LM head and TP LM-head all-to-all.""" if view.enable_dp_lm_head: - assert view.enable_dp_attention, ( - "Please enable dp attention when setting enable_dp_lm_head. " + assert attn_dp_enabled_of(view), ( + "--enable-dp-lm-head requires attention DP (--attn-dp-size)." ) if view.enable_tp_lm_head_all_to_all: - assert view.enable_dp_attention, ( - "Please enable dp attention when setting enable_tp_lm_head_all_to_all." + assert attn_dp_enabled_of(view), ( + "--enable-tp-lm-head-all-to-all requires attention DP (--attn-dp-size)." ) assert not view.enable_dp_lm_head, ( "--enable-tp-lm-head-all-to-all uses a TP-sharded LM head and is " "incompatible with --enable-dp-lm-head." ) - assert view.tp_size == view.dp_size, ( + assert view.tp_size == view.attn_dp_size, ( "--enable-tp-lm-head-all-to-all currently requires tp_size == " - f"dp_size, got tp_size={view.tp_size}, dp_size={view.dp_size}." + f"attn_dp_size, got tp_size={view.tp_size}, " + f"attn_dp_size={view.attn_dp_size}." ) assert view.attn_cp_size == 1, ( "--enable-tp-lm-head-all-to-all currently requires " diff --git a/python/sglang/srt/arg_groups/parallel_hook.py b/python/sglang/srt/arg_groups/parallel_hook.py index 1a396935fbf1..06101adfd9cc 100644 --- a/python/sglang/srt/arg_groups/parallel_hook.py +++ b/python/sglang/srt/arg_groups/parallel_hook.py @@ -23,12 +23,40 @@ from sglang.srt.connector import ConnectorType from sglang.srt.environ import envs from sglang.srt.model_executor.cuda_graph_config import Backend, Phase, with_phase -from sglang.srt.runtime_context import get_platform +from sglang.srt.runtime_context import ( + attn_dp_enabled_of, + derive_attn_tp_size, + get_platform, +) from sglang.srt.utils.common import parse_connector_type logger = logging.getLogger(__name__) +def handle_deprecated_dp_attention(server_args: Any): + """Turn the deprecated `--dp-size N --enable-dp-attention` into + `--attn-dp-size N`. + + Runs before any handler reads the DP layout. The old spelling counted + attention-DP groups with `dp_size`; that width moves to `attn_dp_size`, and + `dp_size` counts replicas again. + """ + cfg = resolving_view(server_args) + if cfg.enable_dp_attention: + if cfg.attn_dp_size != 1: + raise ValueError( + "--enable-dp-attention is the deprecated spelling of " + "--attn-dp-size; pass --attn-dp-size alone." + ) + declare_resolution( + server_args, + "_handle_deprecated_dp_attention", + attn_dp_size=cfg.dp_size, + dp_size=1, + enable_dp_attention=False, + ) + + def _boundary_parallelism_overrides(cfg, model_type: str) -> dict: """Resolve the token-row modes the model's forward actually enters.""" nemotron = model_type in ("nemotron_h", "nemotron_h_puzzle") @@ -108,8 +136,8 @@ def handle_context_parallelism(server_args: Any): assert cfg.tp_size % view.attn_cp_size == 0, ( "tp_size must be divisible by attn_cp_size" ) - assert cfg.tp_size % (cfg.dp_size * view.attn_cp_size) == 0, ( - "tp_size must be divisible by dp_size * attn_cp_size" + assert cfg.tp_size % (cfg.attn_dp_size * view.attn_cp_size) == 0, ( + "tp_size must be divisible by attn_dp_size * attn_cp_size" ) assert not cfg.enable_aiter_allreduce_fusion, ( @@ -155,14 +183,11 @@ def handle_shared_experts_tp(server_args: Any): if size is None: return - from sglang.srt.runtime_context import derive_attention_widths - view = resolved_view(server_args) - _, attn_tp_size = derive_attention_widths( + attn_tp_size = derive_attn_tp_size( tp_size=cfg.tp_size, attn_cp_size=view.attn_cp_size, - dp_size=cfg.dp_size, - enable_dp_attention=view.enable_dp_attention, + attn_dp_size=cfg.attn_dp_size, ) if size < 1 or attn_tp_size % size != 0: raise ValueError( @@ -224,7 +249,7 @@ def handle_decode_context_parallelism(server_args: Any): def handle_data_parallelism(server_args: Any): - # The dp_size==1 resets moved to the resolution pipeline + # The resets without attention DP live in the resolution pipeline # (arg_groups/overrides.py: _data_parallelism_defaults). from sglang.srt.arg_groups.cuda_graph_hook import ( generate_prefill_cuda_graph_batch_sizes, @@ -232,6 +257,15 @@ def handle_data_parallelism(server_args: Any): cfg = resolving_view(server_args) + if cfg.attn_dp_size < 1: + raise ValueError(f"--attn-dp-size must be positive (got {cfg.attn_dp_size}).") + if cfg.attn_dp_size > 1 and cfg.dp_size > 1: + raise ValueError( + f"--dp-size {cfg.dp_size} with --attn-dp-size {cfg.attn_dp_size}: " + "data-parallel replicas combined with attention data parallelism " + "are not supported." + ) + run_post_process_pass(server_args, _data_parallelism_defaults) if cfg.mm_enable_dp_encoder: @@ -252,18 +286,18 @@ def handle_data_parallelism(server_args: Any): cfg.tp_size, ) - if resolved_view(server_args).enable_dp_attention: + if attn_dp_enabled_of(resolved_view(server_args)): declare_resolution( server_args, "_handle_data_parallelism", schedule_conservativeness=cfg.schedule_conservativeness * 0.3, ) - assert cfg.tp_size % cfg.dp_size == 0 + assert cfg.tp_size % cfg.attn_dp_size == 0 original_chunked_prefill_size = cfg.chunked_prefill_size declare_resolution( server_args, "_handle_data_parallelism", - chunked_prefill_size=cfg.chunked_prefill_size // cfg.dp_size, + chunked_prefill_size=cfg.chunked_prefill_size // cfg.attn_dp_size, ) logger.warning( f"DP attention is enabled. chunked prefill size is adjusted " @@ -367,12 +401,8 @@ def handle_dwdp(server_args: Any): declare_resolution( server_args, "_handle_dwdp", - dp_size=cfg.dwdp_size, - ) - declare_resolution( - server_args, - "_handle_dwdp", - enable_dp_attention=True, + attn_dp_size=cfg.dwdp_size, + dp_size=1, ) declare_resolution( server_args, "_handle_dwdp", enable_dp_attention_local_control_broadcast=True @@ -413,7 +443,7 @@ def handle_dwdp(server_args: Any): logger.info( f"DWDP enabled: dwdp_size={cfg.dwdp_size}, " - f"auto-forced dp_size={cfg.dp_size}, ep_size={cfg.dwdp_size}, " + f"auto-forced attn_dp_size={cfg.attn_dp_size}, ep_size={cfg.dwdp_size}, " f"moe_dense_tp_size=1, moe_a2a_backend=none, " f"dp_attention_local_control_broadcast=True, " f"enable_dp_lm_head=True, SCHEDULER_SKIP_ALL_GATHER=True, " @@ -582,10 +612,10 @@ def handle_elastic_ep(server_args: Any): assert not cfg.enable_pdmux, ( "Elastic EP CUDA graph recapture does not support PDMux." ) - assert resolved.enable_dp_attention, ( - "Elastic EP scale-up requires --enable-dp-attention; without it " - "the TP group is not equivalent to WORLD and the post-scale " - "collective path is invalid." + assert attn_dp_enabled_of(resolved), ( + "Elastic EP scale-up requires attention DP (--attn-dp-size); " + "without it the TP group is not equivalent to WORLD and the " + "post-scale collective path is invalid." ) assert resolved.enable_dp_lm_head, ( "Elastic EP scale-up requires --enable-dp-lm-head so output " @@ -604,9 +634,9 @@ def handle_elastic_ep(server_args: Any): f"(got ep_size={resolved.ep_size}, tp_size={cfg.tp_size}); EP, TP " "and the attention DP group must all coincide with WORLD." ) - assert cfg.dp_size == cfg.tp_size, ( - "Elastic EP scale-up requires dp_size == tp_size " - f"(got dp_size={cfg.dp_size}, tp_size={cfg.tp_size})." + assert cfg.attn_dp_size == cfg.tp_size, ( + "Elastic EP scale-up requires attn_dp_size == tp_size " + f"(got attn_dp_size={cfg.attn_dp_size}, tp_size={cfg.tp_size})." ) assert resolved.moe_a2a_backend == "nixl", ( "Elastic EP scale-up requires --moe-a2a-backend nixl " diff --git a/python/sglang/srt/arg_groups/pd_disaggregation_hook.py b/python/sglang/srt/arg_groups/pd_disaggregation_hook.py index b15044b25210..f7d619d2f3f5 100644 --- a/python/sglang/srt/arg_groups/pd_disaggregation_hook.py +++ b/python/sglang/srt/arg_groups/pd_disaggregation_hook.py @@ -12,6 +12,7 @@ resolving_view, ) from sglang.srt.environ import envs +from sglang.srt.runtime_context import attn_dp_enabled_of, num_dp_ranks_of if TYPE_CHECKING: from sglang.srt.server_args import ServerArgs @@ -102,7 +103,7 @@ def handle_pd_disaggregation(server_args: ServerArgs) -> None: f"(--speculative-algorithm {cfg.speculative_algorithm})" ) - if resolved_view(server_args).enable_dp_attention: + if attn_dp_enabled_of(resolved_view(server_args)): logger.warning( "EXPERIMENTAL: Decode radix cache with DP attention. " "Requires prefix-aware DP rank routing for optimal cache hits." @@ -128,7 +129,7 @@ def handle_pd_disaggregation(server_args: ServerArgs) -> None: if cfg.disaggregation_decode_extra_slots is None: extra_slots = 0 if cfg.max_running_requests is not None: - per_worker = cfg.max_running_requests // max(1, cfg.dp_size) + per_worker = cfg.max_running_requests // num_dp_ranks_of(cfg) if per_worker <= 32: extra_slots = per_worker * 2 declare_resolution( @@ -160,8 +161,8 @@ def handle_pd_disaggregation(server_args: ServerArgs) -> None: if cfg.enable_pd_role_switch: view = resolved_view(server_args) unsupported = [] - if view.enable_dp_attention: - unsupported.append("DP attention (--enable-dp-attention)") + if attn_dp_enabled_of(view): + unsupported.append(f"attention DP (--attn-dp-size {view.attn_dp_size})") if view.ep_size > 1: unsupported.append(f"expert parallelism (--ep-size {view.ep_size})") if view.moe_a2a_backend != "none": diff --git a/python/sglang/srt/arg_groups/pipeline.py b/python/sglang/srt/arg_groups/pipeline.py index 3963f548c128..615d48ddb089 100644 --- a/python/sglang/srt/arg_groups/pipeline.py +++ b/python/sglang/srt/arg_groups/pipeline.py @@ -38,6 +38,11 @@ def run_resolution_pipeline(server_args: Any) -> None: cfg = resolving_view(server_args) + from sglang.srt.arg_groups.parallel_hook import handle_deprecated_dp_attention + + # Before any handler reads the DP layout. + run_hook(handle_deprecated_dp_attention, server_args) + from sglang.srt.arg_groups.mega_moe_hook import handle_mega_moe run_hook(handle_mega_moe, server_args) diff --git a/python/sglang/srt/arg_groups/resolution_hooks.py b/python/sglang/srt/arg_groups/resolution_hooks.py index 9422d8025e09..4b0cf78c305a 100644 --- a/python/sglang/srt/arg_groups/resolution_hooks.py +++ b/python/sglang/srt/arg_groups/resolution_hooks.py @@ -13,6 +13,7 @@ # to every invocation of that name, including repeated steps. _OVERRIDABLE_HOOKS: FrozenSet[str] = frozenset( { + "handle_deprecated_dp_attention", "handle_mega_moe", "handle_return_hidden_states_mode", "handle_media_url_security", diff --git a/python/sglang/srt/arg_groups/speculative_hook.py b/python/sglang/srt/arg_groups/speculative_hook.py index d2f497396dbb..18a84bd4cb45 100644 --- a/python/sglang/srt/arg_groups/speculative_hook.py +++ b/python/sglang/srt/arg_groups/speculative_hook.py @@ -17,7 +17,7 @@ ) from sglang.srt.environ import envs from sglang.srt.platforms import current_platform -from sglang.srt.runtime_context import get_platform +from sglang.srt.runtime_context import attn_dp_enabled_of, get_platform if TYPE_CHECKING: from sglang.srt.server_args import ServerArgs @@ -271,7 +271,7 @@ def _handle_dflash(server_args: ServerArgs) -> None: ) # DFLASH + dp attention is validated on NPU only. - if cfg.enable_dp_attention and not cfg.device == "npu": + if attn_dp_enabled_of(cfg) and not cfg.device == "npu": raise ValueError( "Currently DFLASH speculative decoding does not support dp " "attention on non-NPU devices." @@ -538,7 +538,7 @@ def _handle_uno(server_args: ServerArgs) -> None: if (cfg.tp_size, cfg.pp_size) != (1, 1): raise ValueError("UNO requires TP=PP=1.") - if cfg.enable_dp_attention or cfg.attn_cp_size != 1: + if attn_dp_enabled_of(cfg) or cfg.attn_cp_size != 1: raise ValueError("UNO does not support DP attention or context parallelism.") if cfg.enable_lora or cfg.lora_paths: raise ValueError("UNO does not support public Multi-LoRA serving.") @@ -588,8 +588,7 @@ def _handle_dspark(server_args: ServerArgs) -> None: "DSpark speculative decoding only supports CUDA or NPU device." ) - # dp_size==1 with dp_attention is a degenerate flag under DSV4 CP; skip DP-only checks. - if cfg.enable_dp_attention and cfg.dp_size > 1: + if cfg.attn_dp_size > 1: if not cfg.enable_dp_lm_head: raise ValueError("DSpark with dp attention requires --enable-dp-lm-head.") if not _is_npu and cfg.moe_a2a_backend not in ("none", "megamoe", "mori"): @@ -963,9 +962,8 @@ def _handle_iquest_q1_mtp_draft(server_args: ServerArgs) -> bool: def _handle_eagle_family(server_args: ServerArgs) -> None: cfg = resolving_view(server_args) - if ( - cfg.speculative_algorithm == "STANDALONE" - and resolved_view(server_args).enable_dp_attention + if cfg.speculative_algorithm == "STANDALONE" and attn_dp_enabled_of( + resolved_view(server_args) ): # TODO: support dp attention for standalone speculative decoding raise ValueError( @@ -1254,7 +1252,7 @@ def _handle_ngram(server_args: ServerArgs) -> None: "and produces incorrect results for paged attention backends. " "This combination is only supported for the 'flashinfer' backend." ) - if view.enable_dp_attention: + if attn_dp_enabled_of(view): # TODO: support dp attention for ngram speculative decoding raise ValueError( "Currently ngram speculative decoding does not support dp attention." diff --git a/python/sglang/srt/arg_groups/validation_hook.py b/python/sglang/srt/arg_groups/validation_hook.py index 388a61867559..582329a8774b 100644 --- a/python/sglang/srt/arg_groups/validation_hook.py +++ b/python/sglang/srt/arg_groups/validation_hook.py @@ -18,7 +18,7 @@ parse_ib_device_config, ) from sglang.srt.environ import envs -from sglang.srt.runtime_context import get_platform +from sglang.srt.runtime_context import attn_dp_enabled_of, get_platform, num_dp_ranks_of from sglang.srt.utils.common import torch_release from sglang.srt.utils.runai_utils import is_runai_obj_uri @@ -72,8 +72,9 @@ def check_pipeline_parallel_compat(cfg: Any) -> None: # Every stage rebuilds the same verify input from the relayed # per-request state, so all stages must see the same batch. # DP attention partitions it per DP rank. - assert not cfg.enable_dp_attention, ( - "SGLANG_ENABLE_PP_SPEC is not compatible with --enable-dp-attention" + assert not attn_dp_enabled_of(cfg), ( + "SGLANG_ENABLE_PP_SPEC is not compatible with attention DP " + "(--attn-dp-size)" ) else: assert cfg.disaggregation_mode == "prefill", ( @@ -114,8 +115,9 @@ def check_server_args(server_args: Any): if cfg.pp_size > 1: check_pipeline_parallel_compat(cfg) - assert not (cfg.dp_size > 1 and cfg.nnodes != 1 and not cfg.enable_dp_attention), ( - "multi-node data parallel is not supported unless dp attention!" + assert not (cfg.dp_size > 1 and cfg.nnodes != 1), ( + "multi-node data-parallel replicas are not supported; use attention DP " + "(--attn-dp-size) across nodes" ) assert cfg.base_gpu_id >= 0, "base_gpu_id must be non-negative" @@ -375,7 +377,7 @@ def check_load_publish_args(server_args: Any): _, reason = resolve_load_pub_range( kv_endpoint=cfg.endpoint, replay_endpoint=cfg.replay_endpoint, - dp_size=server_cfg.dp_size, + dp_size=num_dp_ranks_of(server_cfg), load_publish_endpoint=mode, ) if reason: @@ -528,10 +530,10 @@ def check_two_batch_overlap(server_args: Any): if ( cfg.enable_two_batch_overlap and cfg.moe_a2a_backend == "none" - and not cfg.enable_dp_attention + and not attn_dp_enabled_of(cfg) ): raise ValueError( "When enabling two batch overlap without an EP a2a backend " - "(moe_a2a_backend='none'), --enable-dp-attention is required " + "(moe_a2a_backend='none'), attention DP (--attn-dp-size) is required " "(DeepSeek-V4 non-EP DP TBO path)." ) diff --git a/python/sglang/srt/debug_utils/dumper.py b/python/sglang/srt/debug_utils/dumper.py index 6bffc95e6ed8..094a4614da78 100644 --- a/python/sglang/srt/debug_utils/dumper.py +++ b/python/sglang/srt/debug_utils/dumper.py @@ -1744,7 +1744,7 @@ def collect_parallel_info(self) -> dict: try: parallel = get_parallel() - info["enable_dp_attention"] = self._dp_attn.is_dp_attention_enabled() + info["attn_dp_enabled"] = self._dp_attn.is_dp_attention_enabled() info["attn_tp_rank"] = parallel.attn_tp_rank info["attn_tp_size"] = parallel.attn_tp_size info["attn_dp_rank"] = parallel.attn_dp_rank diff --git a/python/sglang/srt/disaggregation/common/conn.py b/python/sglang/srt/disaggregation/common/conn.py index e56796d5c028..7ff8b3997ac7 100644 --- a/python/sglang/srt/disaggregation/common/conn.py +++ b/python/sglang/srt/disaggregation/common/conn.py @@ -200,9 +200,7 @@ def __init__( self.dcp_rank = parallel.attn_dcp_rank self.attn_dp_size = parallel.attn_dp_size self.attn_dp_rank = parallel.attn_dp_rank - self.system_dp_size = ( - 1 if get_parallel().enable_dp_attention else get_parallel().dp_size - ) + self.system_dp_size = get_parallel().dp_size self.system_dp_rank = ( self.kv_args.system_dp_rank if self.kv_args.system_dp_rank else 0 ) @@ -1545,11 +1543,12 @@ def __init__( return self.kv_mgr.update_status(self.bootstrap_room, KVPoll.Bootstrapping) - if get_parallel().dp_size > 1 and not req_has_disagg_prefill_dp_rank: + if get_parallel().num_dp_ranks > 1 and not req_has_disagg_prefill_dp_rank: if get_parallel().load_balance_method != "follow_bootstrap_room": self._register_prefill_dp_rank() elif ( - self.kv_mgr.attn_dp_rank != self.bootstrap_room % get_parallel().dp_size + self.kv_mgr.attn_dp_rank + != self.bootstrap_room % get_parallel().num_dp_ranks ): # follow_bootstrap_room was overridden by external routed_dp_rank if envs.SGLANG_DISAGGREGATION_FORCE_QUERY_PREFILL_DP_RANK.get(): @@ -1560,7 +1559,7 @@ def __init__( f"follow_bootstrap_room conflict: dispatched to dp_rank " f"{self.kv_mgr.attn_dp_rank} but bootstrap_room " f"{self.bootstrap_room} implies dp_rank " - f"{self.bootstrap_room % get_parallel().dp_size}. " + f"{self.bootstrap_room % get_parallel().num_dp_ranks}. " f"Set SGLANG_DISAGGREGATION_FORCE_QUERY_PREFILL_DP_RANK=1 " f"to allow mixed routing.", ) diff --git a/python/sglang/srt/disaggregation/encoder/http_server.py b/python/sglang/srt/disaggregation/encoder/http_server.py index f38f881b614c..29053ef0c43d 100644 --- a/python/sglang/srt/disaggregation/encoder/http_server.py +++ b/python/sglang/srt/disaggregation/encoder/http_server.py @@ -209,7 +209,7 @@ def launch_server(server_args: ServerArgs): configure_logger(server_args, prefix=" encode_server") publish(server_args, role="encoder") - if get_parallel().dp_size > 1: + if get_parallel().num_dp_ranks > 1: dp_dispatcher = launch_dp_runtime(server_args) # runtime initializes multiprocess metrics before spawning; # HTTP only exposes their endpoint. diff --git a/python/sglang/srt/disaggregation/encoder/runtime.py b/python/sglang/srt/disaggregation/encoder/runtime.py index 3fab75f4322c..a45e240523c3 100644 --- a/python/sglang/srt/disaggregation/encoder/runtime.py +++ b/python/sglang/srt/disaggregation/encoder/runtime.py @@ -1744,10 +1744,10 @@ def launch_local_runtime(server_args: ServerArgs) -> EncoderRuntime: This function owns backend construction only. HTTP/gRPC middleware, service registration, and network serving remain Transport concerns. """ - if get_parallel().dp_size > 1: + if get_parallel().num_dp_ranks > 1: raise ValueError( - "launch_local_runtime requires --dp-size 1; got " - f"dp_size={get_parallel().dp_size}." + "launch_local_runtime requires a single DP rank; got " + f"num_dp_ranks={get_parallel().num_dp_ranks}." ) # Set up prometheus metrics. @@ -1827,8 +1827,8 @@ def launch_dp_runtime(server_args: ServerArgs) -> DPDispatcher: "Encoder DP mode requires --dp-size > 1 and --tp-size 1; got " f"dp_size={get_parallel().dp_size}, tp_size={get_parallel().tp_size}." ) - dp_size = get_parallel().dp_size - logger.info(f"Launching encoder in DP mode: dp_size={dp_size}") + num_dp_ranks = get_parallel().num_dp_ranks + logger.info(f"Launching encoder in DP mode: num_dp_ranks={num_dp_ranks}") # DP mode: workers (subprocesses) write metrics to the shared multiproc dir; # the main process exposes the aggregated /metrics endpoint. @@ -1837,7 +1837,7 @@ def launch_dp_runtime(server_args: ServerArgs) -> DPDispatcher: ctx = mp.get_context("spawn") ipc_prefix = random_uuid() - async_zmq_ctx = zmq.asyncio.Context(dp_size + 1) + async_zmq_ctx = zmq.asyncio.Context(num_dp_ranks + 1) result_path = f"ipc:///tmp/{ipc_prefix}_dp_result" result_socket = get_zmq_socket(async_zmq_ctx, zmq.PULL, result_path, True) @@ -1845,13 +1845,13 @@ def launch_dp_runtime(server_args: ServerArgs) -> DPDispatcher: get_zmq_socket( async_zmq_ctx, zmq.PUSH, f"ipc:///tmp/{ipc_prefix}_dp_dispatch_{r}", True ) - for r in range(dp_size) + for r in range(num_dp_ranks) ] release_sockets: List[zmq.asyncio.Socket] = [ get_zmq_socket( async_zmq_ctx, zmq.PUSH, f"ipc:///tmp/{ipc_prefix}_dp_release_{r}", True ) - for r in range(dp_size) + for r in range(num_dp_ranks) ] worker_processes: List[mp.Process] = [] @@ -1867,7 +1867,7 @@ def _kill_workers(): # exception (atexit holds the list ref and reads it at exit time). atexit.register(_kill_workers) - for dp_rank in range(dp_size): + for dp_rank in range(num_dp_ranks): gpu_id = get_device().base_gpu_id + dp_rank # Pin the device parent-side around spawn (same convention as the # scheduler launcher and DP controller) so the child inherits @@ -1894,7 +1894,7 @@ def _kill_workers(): if get_observability().extra_metric_labels: labels.update(get_observability().extra_metric_labels) return DPDispatcher( - dp_size, + num_dp_ranks, dispatch_sockets, release_sockets, result_socket, diff --git a/python/sglang/srt/distributed/bootstrap.py b/python/sglang/srt/distributed/bootstrap.py index 61e05396950d..2f025b88eca1 100644 --- a/python/sglang/srt/distributed/bootstrap.py +++ b/python/sglang/srt/distributed/bootstrap.py @@ -79,14 +79,11 @@ def _bind_threads_if_cpu(*, device: str) -> "Optional[List[int]]": from sglang.srt.utils import numa_utils parallel = get_parallel() - # With --enable-dp-attention, dp partitions the existing TP group rather - # than spawning additional processes, so dp_size must not be multiplied - # into the process count here (unlike regular DP, where dp_size * tp_size * - # pp_size is the true worker count). - dp_size = 1 if parallel.enable_dp_attention else parallel.dp_size + # Attention DP partitions the existing TP group rather than spawning + # additional processes, so only the replicas multiply the process count. return numa_utils.init_threads_binding( numa_index=get_device().gpu_id, - world_size=dp_size * parallel.tp_size * parallel.pp_size, + world_size=parallel.dp_size * parallel.tp_size * parallel.pp_size, ) diff --git a/python/sglang/srt/entrypoints/engine.py b/python/sglang/srt/entrypoints/engine.py index af49cbe0cbb7..d167a2d93dad 100644 --- a/python/sglang/srt/entrypoints/engine.py +++ b/python/sglang/srt/entrypoints/engine.py @@ -394,15 +394,15 @@ def _resolve_routed_dp_rank( routed_dp_rank = data_parallel_rank if routed_dp_rank is not None: - dp_size = get_parallel().dp_size - if dp_size <= 1 and routed_dp_rank == 0: + num_dp_ranks = get_parallel().num_dp_ranks + if num_dp_ranks <= 1 and routed_dp_rank == 0: logger.debug( - f"routed_dp_rank={routed_dp_rank} is ignored because dp_size={dp_size}" + f"routed_dp_rank={routed_dp_rank} is ignored because num_dp_ranks={num_dp_ranks}" ) return None - if routed_dp_rank < 0 or routed_dp_rank >= dp_size: + if routed_dp_rank < 0 or routed_dp_rank >= num_dp_ranks: raise ValueError( - f"routed_dp_rank={routed_dp_rank} out of range [0, {dp_size})" + f"routed_dp_rank={routed_dp_rank} out of range [0, {num_dp_ranks})" ) logger.debug(f"routed_dp_rank: {routed_dp_rank}") @@ -886,7 +886,7 @@ def _launch_scheduler_processes( """ scheduler_procs = [] use_dp_controller = ( - get_parallel().dp_size > 1 or get_exec().moe.ep_join_mode == "scale" + get_parallel().num_dp_ranks > 1 or get_exec().moe.ep_join_mode == "scale" ) if not use_dp_controller: diff --git a/python/sglang/srt/entrypoints/http_server.py b/python/sglang/srt/entrypoints/http_server.py index 3258cfb9e974..6dc6e215de60 100644 --- a/python/sglang/srt/entrypoints/http_server.py +++ b/python/sglang/srt/entrypoints/http_server.py @@ -2249,7 +2249,7 @@ async def send_request(session: aiohttp.ClientSession, dp_rank: int) -> int: return await asyncio.gather( *( send_request(session, dp_rank) - for dp_rank in range(get_parallel().dp_size) + for dp_rank in range(get_parallel().num_dp_ranks) ) ) @@ -2313,9 +2313,11 @@ def _execute_server_warmup(server_args: ServerArgs): }, } if get_serving().skip_tokenizer_init: - json_data["input_ids"] = [[10, 11, 12] for _ in range(get_parallel().dp_size)] + json_data["input_ids"] = [ + [10, 11, 12] for _ in range(get_parallel().num_dp_ranks) + ] # TODO Workaround the bug that embedding errors for list of size 1 - if get_parallel().dp_size == 1: + if get_parallel().num_dp_ranks == 1: json_data["input_ids"] = json_data["input_ids"][0] elif ( is_vlm @@ -2359,9 +2361,11 @@ def _execute_server_warmup(server_args: ServerArgs): "temperature": 0.0, } else: - json_data["text"] = ["The capital city of France is"] * get_parallel().dp_size + json_data["text"] = [ + "The capital city of France is" + ] * get_parallel().num_dp_ranks # TODO Workaround the bug that embedding errors for list of size 1 - if get_parallel().dp_size == 1: + if get_parallel().num_dp_ranks == 1: json_data["text"] = json_data["text"][0] # Config debug dumping @@ -2402,7 +2406,7 @@ def _execute_server_warmup(server_args: ServerArgs): if not failed_status_codes: logger.info( "Disaggregation warmup requests completed for all %s DP ranks", - get_parallel().dp_size, + get_parallel().num_dp_ranks, ) logger.info("End of disaggregation warmup") else: diff --git a/python/sglang/srt/entrypoints/v1_loads.py b/python/sglang/srt/entrypoints/v1_loads.py index 1d466e432054..c105f310d34b 100644 --- a/python/sglang/srt/entrypoints/v1_loads.py +++ b/python/sglang/srt/entrypoints/v1_loads.py @@ -49,8 +49,8 @@ def _num_accelerators_per_dp_rank() -> int: """ parallel = get_parallel() num_accelerators = parallel.tp_size * parallel.pp_size - if parallel.enable_dp_attention: - num_accelerators //= parallel.dp_size + if parallel.attn_dp_enabled: + num_accelerators //= parallel.num_dp_ranks return num_accelerators diff --git a/python/sglang/srt/hardware_backend/npu/graph_runner/npu_graph_runner.py b/python/sglang/srt/hardware_backend/npu/graph_runner/npu_graph_runner.py index 7ea16d9e2ba8..abbe52d50949 100644 --- a/python/sglang/srt/hardware_backend/npu/graph_runner/npu_graph_runner.py +++ b/python/sglang/srt/hardware_backend/npu/graph_runner/npu_graph_runner.py @@ -283,7 +283,7 @@ def execute( # Refresh the static DP token buffers bound by the captured # graph (stale values misalign dp-gather segments across # ranks); mirror the capture-side uniform - # [padded_num_tokens] * dp_size. + # [padded_num_tokens] * num_dp_ranks. if self.require_mlp_tp_gather: _padded_num_tokens = bs * self.captured_req_width self.buffers.global_num_tokens_gpu.fill_(_padded_num_tokens) diff --git a/python/sglang/srt/layers/attention/aiter_backend.py b/python/sglang/srt/layers/attention/aiter_backend.py index a9ae9476b5af..349d1cf5ab52 100755 --- a/python/sglang/srt/layers/attention/aiter_backend.py +++ b/python/sglang/srt/layers/attention/aiter_backend.py @@ -525,7 +525,7 @@ def _pool_is_vec5d(pool): 16 if _gathered_num_head < 16 else _gathered_num_head ) - self.enable_dp_attention = is_dp_attention_enabled() + self.attn_dp_enabled = is_dp_attention_enabled() self.qo_indptr_ = torch.zeros( (max_bs + 1,), dtype=torch.int32, device=model_runner.device ) @@ -543,7 +543,7 @@ def _pool_is_vec5d(pool): # need to fall back to non-persist # only use mla_ps_kernel when fp8 kv_cache # for non-fp8 kv_cache on tp8, use non-persist kernel to avoid performance degradation - # head_num=16 (tp8 perf issue), head_num=128 (unsupported, like tp1 or --enable-dp-attention with tp8-dp8) + # head_num=16 (tp8 perf issue), head_num=128 (unsupported, like tp1 or tp8 with --attn-dp-size 8) # Native 16-head persist is slow on TP8; keep disabled unless zero-pad # (e.g. Kimi K3 h12 -> qh16) where persist ASM is the fast path. if ( @@ -628,7 +628,7 @@ def make_mla_decode_meta_data_buffer(self, max_seqlen_qo, batch_size): nhead = self.mla_kernel_num_head_padded dtype = self.kv_cache_dtype - if self.enable_dp_attention: + if self.attn_dp_enabled: gpu = torch.cuda.current_device() device_properties = torch.cuda.get_device_properties(gpu) cu_num = device_properties.multi_processor_count diff --git a/python/sglang/srt/layers/attention/flashattention_backend.py b/python/sglang/srt/layers/attention/flashattention_backend.py index 64562b9de0a1..a8fceedfcf86 100644 --- a/python/sglang/srt/layers/attention/flashattention_backend.py +++ b/python/sglang/srt/layers/attention/flashattention_backend.py @@ -63,7 +63,7 @@ def _should_disable_scheduler_metadata_precompute() -> bool: - return bool(get_parallel().enable_prefill_cp or get_parallel().enable_dp_attention) + return bool(get_parallel().enable_prefill_cp or get_parallel().attn_dp_enabled) @dataclass diff --git a/python/sglang/srt/layers/dp_attention.py b/python/sglang/srt/layers/dp_attention.py index bf6c1e8f6743..521f0d964adb 100644 --- a/python/sglang/srt/layers/dp_attention.py +++ b/python/sglang/srt/layers/dp_attention.py @@ -23,7 +23,7 @@ from sglang.srt.environ import envs from sglang.srt.runtime_context import ( derive_attention_ranks, - derive_attention_widths, + derive_attn_tp_size, get_device, get_exec, get_flags, @@ -51,13 +51,7 @@ def deployment_attn_dp_size() -> int: the target's replicas. """ parallel = get_parallel() - attn_dp_size, _ = derive_attention_widths( - tp_size=parallel.tp_size, - attn_cp_size=parallel.attn_cp_size, - dp_size=parallel.dp_size, - enable_dp_attention=parallel.enable_dp_attention, - ) - return attn_dp_size + return parallel.num_dp_ranks if parallel.attn_dp_enabled else 1 def dp_gather_width() -> int: @@ -69,7 +63,7 @@ def dp_gather_width() -> int: """ parallel = get_parallel() if world_dp_gather_enabled(): - return parallel.dp_size + return parallel.num_dp_ranks if get_flags().dp.scoped_gather_slot is not None: return deployment_attn_dp_size() return parallel.attn_dp_size @@ -127,7 +121,7 @@ def update_dp_attention_post_scale(new_dp_size: int, new_dp_rank: int): """ get_flags().dp.use_world_group_for_gather = True logger.debug( - "[Elastic EP] dp_attention switched to WORLD: dp_size=%d dp_rank=%d", + "[Elastic EP] dp_attention switched to WORLD: num_dp_ranks=%d dp_rank=%d", new_dp_size, new_dp_rank, ) @@ -154,7 +148,7 @@ def is_sum_len(self): def get_dp_padding_mode( cls, is_extend_in_batch, global_num_tokens: List[int] ) -> DpPaddingMode: - dp_size = dp_gather_width() + gather_width = dp_gather_width() # (trangdough) pplx-kernels a2a is a symmetric collective: every EP rank # must dispatch the same number of tokens or the device-side handshake @@ -169,11 +163,11 @@ def get_dp_padding_mode( if moe_a2a_backend.is_deepep_v2() and envs.SGLANG_DEEPEP_V2_FORCE_MAX_LEN.get(): return DpPaddingMode.MAX_LEN - # When is_extend_in_batch and dp_size > 1, use SUM_LEN to avoid padding - # overhead from uneven token distribution. - # For dp_size=1, max_len equals sum_len, so prefer MAX_LEN mode + # When is_extend_in_batch and the gather spans several DP ranks, use + # SUM_LEN to avoid padding overhead from uneven token distribution. + # Over one rank, max_len equals sum_len, so prefer MAX_LEN mode # to enable symmetric memory optimization (needed for DSA CP, etc.). - if is_extend_in_batch and dp_size > 1: + if is_extend_in_batch and gather_width > 1: # Hybrid-SSM models materialize idle ranks via the MAX_LEN # fabricated-row conversion; other models keep mainline SUM_LEN. if get_flags().dp.max_len_with_idle and min(global_num_tokens) == 0: @@ -184,7 +178,7 @@ def get_dp_padding_mode( # prefer MAX_LEN when communication cost is equal to enable symmetric memory max_len = max(global_num_tokens) sum_len = sum(global_num_tokens) - if sum_len * 2 >= max_len * dp_size: + if sum_len * 2 >= max_len * gather_width: return cls.MAX_LEN else: return cls.SUM_LEN @@ -405,25 +399,19 @@ def is_dp_max_padding() -> bool: def compute_dp_attention_world_info( - enable_dp_attention, tp_rank, tp_size, dp_size, attn_cp_size: int = 1 + tp_rank, tp_size, attn_dp_size, attn_cp_size: int = 1 ): """This rank's place in the attention topology, plus the widths it sits in. - The widths come from `derive_attention_widths`; what this adds is the two - ranks, which are per-process and so are not among the widths + The attention-TP width comes from `derive_attn_tp_size`; what this adds is + the two ranks, which are per-process and so are not among the widths `override_permanently` records. """ - attn_dp_size, attn_tp_size = derive_attention_widths( - tp_size=tp_size, - attn_cp_size=attn_cp_size, - dp_size=dp_size, - enable_dp_attention=enable_dp_attention, + attn_tp_size = derive_attn_tp_size( + tp_size=tp_size, attn_cp_size=attn_cp_size, attn_dp_size=attn_dp_size ) attn_tp_rank, attn_dp_rank = derive_attention_ranks( - tp_rank=tp_rank, - attn_tp_size=attn_tp_size, - attn_cp_size=attn_cp_size, - enable_dp_attention=enable_dp_attention, + tp_rank=tp_rank, attn_tp_size=attn_tp_size, attn_cp_size=attn_cp_size ) return attn_tp_rank, attn_tp_size, attn_dp_rank, attn_dp_size @@ -431,7 +419,7 @@ def compute_dp_attention_world_info( def initialize_dp_attention_flags(server_args: ServerArgs): """Initialize DP runtime flags without changing the worker's placement.""" dp = get_flags().dp - dp.enabled = get_parallel().enable_dp_attention + dp.enabled = get_parallel().attn_dp_enabled if get_exec().moe.elastic_ep_backend is not None and get_parallel().max_ep_size: if ep_scale_joiner_of(resolving_view(server_args)): @@ -443,10 +431,9 @@ def initialize_dp_attention(server_args: ServerArgs): initialize_dp_attention_flags(server_args) parallel = get_parallel() _, _, attn_dp_rank, attn_dp_size = compute_dp_attention_world_info( - parallel.enable_dp_attention, parallel.tp_rank, parallel.tp_size, - parallel.dp_size, + parallel.attn_dp_size, parallel.attn_cp_size, ) parallel.override_permanently(attn_dp_size=attn_dp_size, attn_dp_rank=attn_dp_rank) @@ -652,7 +639,7 @@ def _dp_gather_via_all_gather( # Variable-length DP-MoE gather (reference https://github.com/ROCm/ATOM/pull/930): instead of padding every # rank to max_len (all_gather) or all-reducing a sum_len zero-buffer (all_reduce), # gather exactly sum(per-rank tokens) via all_gatherv. Env-gated; only the simple -# tp_size==dp_size (attn_tp_size==1) case is supported for now (e.g. tp8dp8). +# tp_size==attn_dp_size (attn_tp_size==1) case is supported for now (e.g. tp8, attn_dp8). _USE_DP_GATHERV = get_bool_env_var("SGLANG_DP_USE_GATHERV") _DP_GATHER_FP8_GROUP = 128 @@ -725,7 +712,7 @@ def mask_dp_pad_moe_topk_ids(topk_ids: torch.Tensor) -> None: """Set MAX_LEN pad rows' (post-translation, local) topk_ids to -1 in place. Under dp-attention MAX_LEN padding the gathered MoE buffer is - [dp_size * max_len, hidden] with rank r's real rows at + [num_dp_ranks * max_len, hidden] with rank r's real rows at [r*max_len, r*max_len + global_num_tokens[r]); the pad rows carry stale hidden values, run the router, and get dispatched into experts whose outputs are then discarded by the post-reorder scatter — pure wasted @@ -794,7 +781,7 @@ def is_dp_gatherv_active() -> bool: """Variable-length DP-MoE gather/scatter (all_gatherv + reduce_scatterv) is enabled and applicable to the CURRENT forward. Requires: - env SGLANG_DP_USE_GATHERV (default off), - - supported layout (attn_tp_size==1, tp_size==dp_size), + - supported layout (attn_tp_size==1, tp_size==attn_dp_size), - SUM_LEN padding mode. The gatherv pair (all_gatherv + reduce_scatterv) is only valid under SUM_LEN; under MAX_LEN the buffer is equal-padded and the gather/combine use all_gather / (aiter) reduce_scatter instead. Reading the @@ -1004,7 +991,7 @@ def can_use_dp_reduce_scatter() -> bool: return True parallel = get_parallel() - return parallel.tp_size == parallel.dp_size * parallel.attn_tp_size + return parallel.tp_size == parallel.num_dp_ranks * parallel.attn_tp_size def dp_reduce_scatter_tensor(output: torch.Tensor, input: torch.Tensor): diff --git a/python/sglang/srt/layers/layer_boundary/construction.py b/python/sglang/srt/layers/layer_boundary/construction.py index 1b9f0e695814..334b64b7fbf8 100644 --- a/python/sglang/srt/layers/layer_boundary/construction.py +++ b/python/sglang/srt/layers/layer_boundary/construction.py @@ -175,7 +175,7 @@ def __init__( self._speculative_algo = SpeculativeAlgorithm.from_string( get_spec().speculative_algorithm ) - self._publish_lora_layout = get_parallel().enable_dp_attention and bool( + self._publish_lora_layout = get_parallel().attn_dp_enabled and bool( get_lora().enable_lora ) self._next_input_rows = None diff --git a/python/sglang/srt/layers/logits_processor.py b/python/sglang/srt/layers/logits_processor.py index 77b47df9c9d3..31cd9d2723c1 100644 --- a/python/sglang/srt/layers/logits_processor.py +++ b/python/sglang/srt/layers/logits_processor.py @@ -84,7 +84,7 @@ # None outside a FlashInfer autotune pass; inside one, whether that pass runs the # LM head. Not-None means the forward's output is discarded -- attention backends # read that via get_in_autotune_dummy_run() to skip a cross-node exchange. -# Skipping the LM head skips its [batch * dp_size, vocab] all-gather, which OOMs +# Skipping the LM head skips its [batch * num_dp_ranks, vocab] all-gather, which OOMs # under DP attention with a tight mem_fraction_static. _autotune_run_lm_head: Optional[bool] = None @@ -189,7 +189,7 @@ def should_apply_lm_head_quant_method(lm_head, quant_method) -> bool: # FlashInfer autotune skips the unprofiled LM-head all-gather; its -# [batch * dp_size, vocab] output can OOM under tight DP-attention memory. +# [batch * num_dp_ranks, vocab] output can OOM under tight DP-attention memory. _in_autotune_dummy_run = False diff --git a/python/sglang/srt/layers/moe/moe_runner/flashinfer_cutedsl.py b/python/sglang/srt/layers/moe/moe_runner/flashinfer_cutedsl.py index efdb3cc36027..ff312750af0d 100644 --- a/python/sglang/srt/layers/moe/moe_runner/flashinfer_cutedsl.py +++ b/python/sglang/srt/layers/moe/moe_runner/flashinfer_cutedsl.py @@ -340,9 +340,9 @@ def ensure_cutedsl_wrapper(layer: torch.nn.Module) -> None: # A2A path: bounded by the dispatcher's own workspace limit. max_num_tokens = dispatcher.max_num_tokens * getattr(dispatcher, "ep_size", 1) else: - # Standard allgather path: the MoE sees up to dp_size local forwards - # gathered together, so scale the per-rank forward bound by dp_size. - max_num_tokens = get_parallel().dp_size * cutedsl_moe_max_num_tokens() + # Standard allgather path: the MoE sees up to num_dp_ranks local forwards + # gathered together, so scale the per-rank forward bound by num_dp_ranks. + max_num_tokens = get_parallel().num_dp_ranks * cutedsl_moe_max_num_tokens() top_k = layer.top_k if layer.top_k is not None else layer.moe_runner_config.top_k # inference_mode(False) ensures the wrapper's pre-allocated CUDA-graph # buffers are normal tensors. This call typically happens inside diff --git a/python/sglang/srt/lora/lora_manager.py b/python/sglang/srt/lora/lora_manager.py index dae118d8082b..4ef61510f92a 100644 --- a/python/sglang/srt/lora/lora_manager.py +++ b/python/sglang/srt/lora/lora_manager.py @@ -99,9 +99,9 @@ def __init__( self.enable_lora_overlap_loading: Optional[bool] = ( get_lora().enable_lora_overlap_loading ) - self.enable_dp_attention: bool = get_parallel().enable_dp_attention + self.attn_dp_enabled: bool = get_parallel().attn_dp_enabled if ( - self.enable_dp_attention + self.attn_dp_enabled and self.enable_lora_overlap_loading and not LoRAMemoryPool.supports_dp_attention_overlap_loading ): @@ -112,11 +112,11 @@ def __init__( ) # LoRA routing knows only DP-local and TP-global token layouts. A wider # attention TP/CP group can scatter one DP rank's tokens across ranks. - if self.enable_dp_attention and ( + if self.attn_dp_enabled and ( self.attn_tp_size != 1 or get_parallel().attn_cp_size != 1 ): raise ValueError( - "LoRA with DP attention requires --dp-size equal to --tp-size " + "LoRA with DP attention requires --attn-dp-size equal to --tp-size " f"(got attention TP size {self.attn_tp_size} and attention CP " f"size {get_parallel().attn_cp_size})" ) @@ -138,7 +138,7 @@ def __init__( device=self.device, server_args=server_args, ) - if self.enable_dp_attention and not self.lora_backend.supports_dp_attention: + if self.attn_dp_enabled and not self.lora_backend.supports_dp_attention: raise ValueError( f"LoRA backend {lora_backend!r} does not support DP attention" ) @@ -164,9 +164,9 @@ def init_cuda_graph_batch_info( num_tokens_per_req=num_tokens_per_req, ) max_moe_tokens = max_bs_in_cuda_graph * num_tokens_per_req - if self.enable_dp_attention: + if self.attn_dp_enabled: self.lora_backend.init_dp_attention_cuda_graph_batch_info(max_moe_tokens) - max_moe_tokens *= get_parallel().dp_size + max_moe_tokens *= get_parallel().num_dp_ranks if self.lora_backend.resize_cuda_graph_moe_buffers(max_moe_tokens): logger.info( "Right-sized shared MoE LoRA CUDA graph buffers " @@ -211,10 +211,7 @@ def supports_prefill_cuda_graph(self) -> bool: check_cuda_graph_backend, ) - if ( - self.enable_dp_attention - or not self.lora_backend.supports_prefill_cuda_graph - ): + if self.attn_dp_enabled or not self.lora_backend.supports_prefill_cuda_graph: return False if self.lora_backend.is_moe_lora: return check_cuda_graph_backend( @@ -237,7 +234,7 @@ def can_use_prefill_cuda_graph(self, forward_batch: ForwardBatch) -> bool: return False # DP attention: per-rank eligibility could diverge across ranks and # desync collectives; keep LoRA prefill eager. - if self.enable_dp_attention: + if self.attn_dp_enabled: return False # Decode-CUDA-graph extend modes (TARGET_VERIFY, DLLM_EXTEND) are # owned by the decode static batch info path. @@ -333,7 +330,7 @@ def validate_new_adapter(self, lora_config: LoRAConfig, lora_ref: LoRARef): """ Validate if an adapter can be loaded into the current LoRA memory pool and generate error if it is incompatible. """ - assert not self.enable_dp_attention or lora_ref.pinned, ( + assert not self.attn_dp_enabled or lora_ref.pinned, ( "DP-attention LoRA requires pinned adapters" ) if lora_config.lora_added_tokens_size > 0: @@ -506,7 +503,7 @@ def prepare_lora_batch(self, forward_batch: ForwardBatch): ) active_lora_ids = set(forward_batch.lora_ids) - if self.enable_dp_attention: + if self.attn_dp_enabled: get_forward().set("lora_batch_layout", LoRABatchLayout.DP_LOCAL) gathered_lora_ids = get_parallel().tp_group.all_gather_object( forward_batch.lora_ids @@ -551,7 +548,7 @@ def prepare_lora_batch(self, forward_batch: ForwardBatch): self.lora_backend.batch_info.has_active_lora = any( lora_ranks[wi] > 0 for wi in weight_indices ) - if self.enable_dp_attention: + if self.attn_dp_enabled: self.lora_backend.prepare_global_lora_batch(forward_batch) if forward_batch.forward_mode.is_idle(): # Idle ranks must join the global routing collectives, but @@ -563,9 +560,7 @@ def _use_cuda_graph_batch(self, forward_batch: ForwardBatch) -> bool: hasattr(self, "max_bs_in_cuda_graph") and forward_batch.batch_size <= self.max_bs_in_cuda_graph and forward_batch.forward_mode.is_cuda_graph() - and ( - not self.enable_dp_attention or forward_batch.can_run_decode_cuda_graph - ) + and (not self.attn_dp_enabled or forward_batch.can_run_decode_cuda_graph) ) def prepare_lora_token_segments( @@ -1194,7 +1189,7 @@ def init_lora_cuda_graph_moe_buffers( # num_draft_tokens per request, and the buffers below are per-token, so # they must be sized in tokens rather than requests. max_tokens = max_bs * (get_spec().speculative_num_draft_tokens or 1) - if get_parallel().enable_dp_attention: + if get_parallel().attn_dp_enabled: max_tokens *= get_parallel().attn_dp_size max_loras = get_lora().max_loras_per_batch for module in model.modules(): diff --git a/python/sglang/srt/lora/mem_pool.py b/python/sglang/srt/lora/mem_pool.py index 506a469c3498..7a3e8200059d 100644 --- a/python/sglang/srt/lora/mem_pool.py +++ b/python/sglang/srt/lora/mem_pool.py @@ -180,7 +180,7 @@ def __init__( ) # Per-expert MoE weights are sharded by `moe_tp_size`, NOT the outer - # `tp_size`: `moe_tp_size = tp_size // ep_size // dp_size`, so under + # `tp_size`: `moe_tp_size = tp_size // ep_size // moe_dp_size`, so under # e.g. `--tp 4 --ep 4` each rank holds full-width expert weights # (`moe_tp_size == 1`). Sizing per-expert LoRA buffers by `tp_size` # here would yield a 4x-narrower inner dim than the adapter weight @@ -190,7 +190,7 @@ def __init__( self.moe_tp_size, self.moe_tp_rank = _get_moe_tp_context() # Attention projections shard along the attention TP group, which - # under `--enable-dp-attention` is `attn_tp_size = tp_size // dp_size`. + # under attention DP is `attn_tp_size = tp_size // attn_dp_size`. # The corresponding LoRA wrappers slice weights by the base layer's # attn_tp-local rank, so the buffer shapes must match that shard. self.attn_tp_size: int = attn_tp_size @@ -273,7 +273,7 @@ def _effective_tp_size(self, module_name: str) -> int: """TP width the module's weights are actually sharded along: routed MoE experts shard by `moe_tp_size` (shared experts by the outer `tp_size` at EP=1), attention projections by `attn_tp_size` (smaller - than the outer `tp_size` under `--enable-dp-attention`), everything + than the outer `tp_size` under attention DP), everything else by the outer `tp_size`.""" if self.is_moe_module(module_name) and not self.is_shared_moe_module( module_name diff --git a/python/sglang/srt/lora/utils.py b/python/sglang/srt/lora/utils.py index f4310e1f0020..9d21463ee5b8 100644 --- a/python/sglang/srt/lora/utils.py +++ b/python/sglang/srt/lora/utils.py @@ -373,7 +373,7 @@ def get_target_module_name(full_module_name: str, target_modules: Set[str]) -> s *DSA_INDEXER_LORA_NAMES, ] # Attention-projection LoRA modules shard on the attention-TP group, which -# under `--enable-dp-attention` is `attn_tp_size = tp_size // dp_size` rather +# under attention DP is `attn_tp_size = tp_size // attn_dp_size` rather # than the outer TP size. in_proj / in_proj_qkvz (linear-attention hybrids) # belong here too: their layers are built on the attn-TP group (mamba.py and # qwen3_5.py take tp_size/tp_rank from attn_tp when dp attention is enabled). diff --git a/python/sglang/srt/managers/data_parallel_controller.py b/python/sglang/srt/managers/data_parallel_controller.py index 8c1b940c9cb0..0396ab2c11d6 100644 --- a/python/sglang/srt/managers/data_parallel_controller.py +++ b/python/sglang/srt/managers/data_parallel_controller.py @@ -102,11 +102,11 @@ def from_str(cls, method: str): class DPBudget: - def __init__(self, dp_size: int): - self.dp_size = dp_size - self.total_requests = [0] * dp_size - self.total_tokens = [0] * dp_size - self.last_timestamp = [0.0] * dp_size + def __init__(self, num_dp_ranks: int): + self.num_dp_ranks = num_dp_ranks + self.total_requests = [0] * num_dp_ranks + self.total_tokens = [0] * num_dp_ranks + self.last_timestamp = [0.0] * num_dp_ranks def update_budget(self, loads): """Update budget from shm snapshots, skipping stale reads.""" @@ -125,7 +125,7 @@ def dispatch(self, method: LoadBalanceMethod, estimated_tokens: int = 0): elif method == LoadBalanceMethod.TOTAL_TOKENS: # Use total_requests as a tie-breaker when total_tokens are equal target_rank = min( - range(self.dp_size), + range(self.num_dp_ranks), key=lambda i: (self.total_tokens[i], self.total_requests[i]), ) else: @@ -155,7 +155,7 @@ def __init__( self.run_scheduler_process_func = run_scheduler_process_func # Init inter-process communication - self.context = zmq.Context(1 + get_parallel().dp_size) + self.context = zmq.Context(1 + get_parallel().num_dp_ranks) if get_parallel().node_rank == 0: self.recv_from_tokenizer = get_zmq_socket( self.context, zmq.PULL, port_args.scheduler_input_ipc_name, False @@ -175,8 +175,10 @@ def __init__( LoadBalanceMethod.TOTAL_TOKENS, ) - self.launch_dp_size: int = get_parallel().dp_size - self.max_dp_size: int = get_parallel().max_ep_size or get_parallel().dp_size + self.launch_dp_size: int = get_parallel().num_dp_ranks + self.max_dp_size: int = ( + get_parallel().max_ep_size or get_parallel().num_dp_ranks + ) assert self.max_dp_size >= self.launch_dp_size, ( f"--max-ep-size ({self.max_dp_size}) must be >= " f"--dp ({self.launch_dp_size})." @@ -186,7 +188,7 @@ def __init__( self.max_dp_size - self.launch_dp_size ) - self.dp_budget = DPBudget(get_parallel().dp_size) + self.dp_budget = DPBudget(get_parallel().num_dp_ranks) self.load_snapshot_reader = create_load_snapshot_reader( port_args, caller="DataParallelController", @@ -203,7 +205,7 @@ def __init__( self._active_workers: list[int] = list(range(self.launch_dp_size)) self._active_count_cache: int = self.launch_dp_size - if get_parallel().enable_dp_attention: + if get_parallel().attn_dp_enabled: self.launch_dp_attention_schedulers(server_args, port_args) # When local control broadcast is enabled, send control messages to # every DP group leader (attn_tp_rank=0) so each leader broadcasts @@ -374,7 +376,7 @@ def launch_dp_schedulers(self, server_args, port_args): threads = [] sockets = [] ready_events = [] - for dp_rank in range(get_parallel().dp_size): + for dp_rank in range(get_parallel().num_dp_ranks): tmp_port_args = PortArgs.init_new(server_args) tmp_port_args.tokenizer_ipc_name = port_args.tokenizer_ipc_name tmp_port_args.detokenizer_ipc_name = port_args.detokenizer_ipc_name @@ -577,7 +579,7 @@ def launch_dp_attention_schedulers( bind_count = ( self.max_dp_size if get_exec().moe.elastic_ep_backend is not None - else get_parallel().dp_size + else get_parallel().num_dp_ranks ) for slot in range(bind_count): worker_port, worker_socket = get_zmq_socket_on_host( @@ -607,7 +609,7 @@ def launch_tensor_parallel_group( dp_rank: int | None, worker_ports: list[int] | None = None, ): - if not get_parallel().enable_dp_attention: + if not get_parallel().attn_dp_enabled: logger.info(f"Launch DP{dp_rank} starting at GPU #{base_gpu_id}.") memory_saver_adapter = TorchMemorySaverAdapter.create( @@ -639,13 +641,12 @@ def launch_tensor_parallel_group( for tp_rank in tp_rank_range: rank_port_args = port_args - if get_parallel().enable_dp_attention: + if get_parallel().attn_dp_enabled: # dp attention has different sharding logic _, _, dp_rank, _ = compute_dp_attention_world_info( - get_parallel().enable_dp_attention, tp_rank, get_parallel().tp_size, - get_parallel().dp_size, + get_parallel().attn_dp_size, get_parallel().attn_cp_size, ) # compute zmq ports for this dp rank diff --git a/python/sglang/srt/managers/load_snapshot.py b/python/sglang/srt/managers/load_snapshot.py index 968f438f8d81..7830129f0839 100644 --- a/python/sglang/srt/managers/load_snapshot.py +++ b/python/sglang/srt/managers/load_snapshot.py @@ -71,7 +71,7 @@ def should_use_zmq() -> bool: ``SGLANG_LOAD_SNAPSHOT_USE_ZMQ`` forces zmq mode for testing. """ return ( - get_parallel().enable_dp_attention and get_parallel().nnodes > 1 + get_parallel().attn_dp_enabled and get_parallel().nnodes > 1 ) or envs.SGLANG_LOAD_SNAPSHOT_USE_ZMQ.get() @@ -120,10 +120,10 @@ def zmq_reader_owner(caller: str) -> bool: return False if caller == "DataParallelController": return ( - get_parallel().dp_size > 1 + get_parallel().num_dp_ranks > 1 and get_parallel().load_balance_method.lower() in _LOAD_AWARE_METHODS ) - if get_parallel().dp_size > 1 and ( + if get_parallel().num_dp_ranks > 1 and ( get_parallel().load_balance_method.lower() in _LOAD_AWARE_METHODS ): return False @@ -659,9 +659,9 @@ def create_load_snapshot_reader(port_args, caller: str): ``"MultiTokenizerRouter"`` -- determines who binds the zmq PULL socket when zmq mode is active. """ - dp_size = get_parallel().dp_size + num_dp_ranks = get_parallel().num_dp_ranks if zmq_reader_owner(caller): return ZmqShmLoadSnapshotReader( - _zmq_addr_for(port_args), shm_path_for(port_args.instance_id), dp_size + _zmq_addr_for(port_args), shm_path_for(port_args.instance_id), num_dp_ranks ) - return ShmLoadSnapshotReader(shm_path_for(port_args.instance_id), dp_size) + return ShmLoadSnapshotReader(shm_path_for(port_args.instance_id), num_dp_ranks) diff --git a/python/sglang/srt/managers/prefill_delayer.py b/python/sglang/srt/managers/prefill_delayer.py index da93dbfaa049..f7d990574d56 100644 --- a/python/sglang/srt/managers/prefill_delayer.py +++ b/python/sglang/srt/managers/prefill_delayer.py @@ -106,9 +106,9 @@ def __init__( f"queue_trigger_enabled={self._queue_trigger_enabled}" ) parallel = get_parallel() - self.dp_size = parallel.dp_size - self.enable_dp_attention = parallel.enable_dp_attention - dp_size_dim = self.dp_size if self.enable_dp_attention else 1 + self.num_dp_ranks = parallel.num_dp_ranks + self.attn_dp_enabled = parallel.attn_dp_enabled + dp_size_dim = self.num_dp_ranks if self.attn_dp_enabled else 1 # Mirror scheduler_dp_attn_mixin's NCCL all-gather path: when the # env flag is on (or overlap scheduling is disabled), ride the NCCL @@ -238,10 +238,10 @@ def _negotiate_should_allow_prefill_pure( **wait_info, ) - if not self.enable_dp_attention: + if not self.attn_dp_enabled: max_running_requests = ( - max_running_requests + self.dp_size - 1 - ) // self.dp_size + max_running_requests + self.num_dp_ranks - 1 + ) // self.num_dp_ranks global_running_batch_max = int(global_running_batch.max().item()) global_max_prefill_bs_max = int(global_max_prefill_bs.max().item()) @@ -411,10 +411,10 @@ def _estimate_attempted_prefill_bs( waiting_queue_len: int, ) -> int: local_max_running_requests = max_running_requests - if not self._prefill_delayer.enable_dp_attention: + if not self._prefill_delayer.attn_dp_enabled: local_max_running_requests = ( - max_running_requests + self._prefill_delayer.dp_size - 1 - ) // self._prefill_delayer.dp_size + max_running_requests + self._prefill_delayer.num_dp_ranks - 1 + ) // self._prefill_delayer.num_dp_ranks # The delayer negotiates before PrefillAdder materializes can_run_list, # so a rejected pass has no exact batch size. This upper bound is exact diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index b35a56451c49..9cd644285b57 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -513,7 +513,7 @@ def __init__( self.max_recv_per_poll = envs.SGLANG_SCHEDULER_MAX_RECV_PER_POLL.get() self.max_new_tokens_limit = envs.SGLANG_MAX_NEW_TOKENS_LIMIT.get() self.enable_hisparse = get_memory().enable_hisparse - self.enable_dp_attention = get_parallel().enable_dp_attention + self.attn_dp_enabled = get_parallel().attn_dp_enabled self.enable_unified_memory = get_memory().enable_unified_memory # Init model configs @@ -626,7 +626,7 @@ def __init__( token_to_kv_pool_allocator=self.token_to_kv_pool_allocator, tp_group=( self.attn_tp_cpu_group - if self.enable_dp_attention + if self.attn_dp_enabled else self.tp_cpu_group ), tree_cache=self.tree_cache, @@ -820,7 +820,7 @@ def init_ipc_channels(self, port_args: PortArgs): try: self.load_snapshot_writer = create_load_snapshot_writer( port_args, - get_parallel().dp_size, + get_parallel().num_dp_ranks, dp_rank, publish_interval=get_observability().load_snapshot_publish_interval, ) @@ -1213,9 +1213,7 @@ def init_model_worker(self): # the base TP group. Entry rank is the local rank 0 in that group. # Use the CPU (gloo) group to broadcast VLM Python objects and avoid CUDA # stream/device coupling (#11910). - self.dp_tp_group = ( - self.attn_tp_group if self.enable_dp_attention else self.tp_group - ) + self.dp_tp_group = self.attn_tp_group if self.attn_dp_enabled else self.tp_group self.dp_tp_cpu_group = self.dp_tp_group.cpu_group self.pad_input_ids_func = self.tp_worker.get_pad_input_ids_func() @@ -4647,9 +4645,7 @@ def run_batch( return ret def _maybe_report_active_ranks(self) -> None: - if not ( - self.enable_dp_attention and get_exec().moe.elastic_ep_backend is not None - ): + if not (self.attn_dp_enabled and get_exec().moe.elastic_ep_backend is not None): return from sglang.srt.elastic_ep.elastic_ep import ElasticEPStateManager @@ -5219,7 +5215,6 @@ def get_internal_state(self, recv_req: GetInternalStateReq): # readback reflects values changed via /set_internal_state, not startup. ret = get_context().resolved_server_args_dict() ret["world_size"] = compute_world_size( - enable_dp_attention=get_parallel().enable_dp_attention, dp_size=get_parallel().dp_size, tp_size=get_parallel().tp_size, pp_size=get_parallel().pp_size, diff --git a/python/sglang/srt/managers/scheduler_components/dp_attn.py b/python/sglang/srt/managers/scheduler_components/dp_attn.py index ea64aea930bf..a7c57c4292a9 100644 --- a/python/sglang/srt/managers/scheduler_components/dp_attn.py +++ b/python/sglang/srt/managers/scheduler_components/dp_attn.py @@ -65,47 +65,47 @@ def _spec_input_cuda_graph_compatible( return spec_info is None or spec_info.cuda_graph_compatible -def _resolve_elastic_world_dp_size( - dp_size: int, +def _resolve_elastic_world_num_dp_ranks( + num_dp_ranks: int, *, group: torch.distributed.ProcessGroup, local_num_tokens: int, local_forward_mode: int, ) -> int: if not world_dp_gather_enabled(): - return dp_size + return num_dp_ranks from sglang.srt.elastic_ep.elastic_ep import ElasticEPStateManager - live_dp_size = dp_gather_width() + live_num_dp_ranks = dp_gather_width() effective_ep_size = ElasticEPStateManager.get_effective_ep_size() # Query live membership because elastic joins can expand WORLD. world_size = torch.distributed.get_world_size(group) - if live_dp_size != effective_ep_size: + if live_num_dp_ranks != effective_ep_size: raise RuntimeError( - "[Elastic EP] WORLD MLP sync dp_size is out of sync: " + "[Elastic EP] WORLD MLP sync num_dp_ranks is out of sync: " f"rank={torch.distributed.get_rank(group)} " - f"live_dp_size={live_dp_size} " + f"live_num_dp_ranks={live_num_dp_ranks} " f"effective_ep_size={effective_ep_size} " - f"world_size={world_size} server_args_dp_size={dp_size} " + f"world_size={world_size} configured_num_dp_ranks={num_dp_ranks} " f"local_num_tokens={local_num_tokens} " f"local_forward_mode={local_forward_mode}" ) - if live_dp_size > world_size: + if live_num_dp_ranks > world_size: raise RuntimeError( - "[Elastic EP] WORLD MLP sync dp_size exceeds WORLD size: " + "[Elastic EP] WORLD MLP sync num_dp_ranks exceeds WORLD size: " f"rank={torch.distributed.get_rank(group)} " - f"live_dp_size={live_dp_size} world_size={world_size} " + f"live_num_dp_ranks={live_num_dp_ranks} world_size={world_size} " f"effective_ep_size={effective_ep_size}" ) - return live_dp_size + return live_num_dp_ranks @dataclass class MLPSyncBatchInfo: - dp_size: int + num_dp_ranks: int tp_size: int cp_size: int @@ -186,7 +186,7 @@ def all_gather( # is a no-op, and the masked fallback writes below would then read and # write the same storage. global_info_tensor = fallback_tensor.repeat( - self.dp_size, self.tp_size * self.cp_size, 1 + self.num_dp_ranks, self.tp_size * self.cp_size, 1 ) if use_all_reduce: @@ -211,7 +211,7 @@ def all_gather( ) tp_info = global_info_tensor.view( - self.dp_size * self.tp_size * self.cp_size, info_width + self.num_dp_ranks * self.tp_size * self.cp_size, info_width ) num_ranks_in_tp_info = tp_info.shape[0] if device == "cpu": @@ -296,7 +296,7 @@ def _update_gather_batch( ) -def should_skip_scheduler_all_gather(dp_size: int) -> bool: +def should_skip_scheduler_all_gather(num_dp_ranks: int) -> bool: """Return whether scheduler metadata is already local and rank-invariant. With one attention-DP rank there is no cross-DP state to reconcile. The @@ -307,7 +307,7 @@ def should_skip_scheduler_all_gather(dp_size: int) -> bool: DP1. """ - return dp_size == 1 or envs.SGLANG_SCHEDULER_SKIP_ALL_GATHER.get() + return num_dp_ranks == 1 or envs.SGLANG_SCHEDULER_SKIP_ALL_GATHER.get() def _local_decode_cuda_graph_vote( @@ -421,7 +421,7 @@ def prepare_mlp_sync_batch_raw( dwdp: bool = False, ): parallel = get_parallel() - dp_size = parallel.dp_size + num_dp_ranks = parallel.num_dp_ranks attn_tp_size = parallel.attn_tp_size tp_group = parallel.tp_group # Check if other DP workers have running batches @@ -498,16 +498,16 @@ def prepare_mlp_sync_batch_raw( local_can_run_tbo, local_forward_mode = tbo_preparer.prepare_all_gather(local_batch) if use_world_group: - dp_size = _resolve_elastic_world_dp_size( - dp_size, + num_dp_ranks = _resolve_elastic_world_num_dp_ranks( + num_dp_ranks, group=group, local_num_tokens=num_tokens, local_forward_mode=local_forward_mode, ) - skip_all_gather = should_skip_scheduler_all_gather(dp_size) + skip_all_gather = should_skip_scheduler_all_gather(num_dp_ranks) mlp_sync_info = MLPSyncBatchInfo( - dp_size=dp_size, + num_dp_ranks=num_dp_ranks, tp_size=attn_tp_size, cp_size=parallel.attn_cp_size, num_tokens=num_tokens, @@ -521,7 +521,7 @@ def prepare_mlp_sync_batch_raw( prefill_cuda_graph_max_prefix_len=prefill_cuda_graph_max_prefix_len, ) - if dp_size == 1: + if num_dp_ranks == 1: mlp_sync_info.finalize_local() elif not skip_all_gather: mlp_sync_info.all_gather( @@ -540,7 +540,7 @@ def prepare_mlp_sync_batch_raw( # Decide whether to emit idle batch if skip_all_gather: # Skip idle batch when attn-dp=1 (and always under DWDP: ranks run independently) - need_idle_batch = not dwdp and dp_size > 1 + need_idle_batch = not dwdp and num_dp_ranks > 1 else: need_idle_batch = max(mlp_sync_info.global_num_tokens) > 0 diff --git a/python/sglang/srt/managers/scheduler_components/dynamic_chunk_sizer.py b/python/sglang/srt/managers/scheduler_components/dynamic_chunk_sizer.py index 318675488d41..ea373c12a531 100644 --- a/python/sglang/srt/managers/scheduler_components/dynamic_chunk_sizer.py +++ b/python/sglang/srt/managers/scheduler_components/dynamic_chunk_sizer.py @@ -191,8 +191,7 @@ def _profile_prefill_latency(self) -> Tuple[List[int], List[float]]: if is_dp_attention_enabled(): # Profiling runs one request on this rank; other DP ranks report 0. - dp_size = get_parallel().attn_dp_size - global_num_tokens = [0] * dp_size + global_num_tokens = [0] * get_parallel().attn_dp_size dp_rank = get_parallel().attn_dp_rank global_num_tokens[dp_rank] = current_seq_len batch.global_num_tokens = global_num_tokens diff --git a/python/sglang/srt/managers/scheduler_components/load_publisher.py b/python/sglang/srt/managers/scheduler_components/load_publisher.py index 50a526a79e1a..bf3d34718365 100644 --- a/python/sglang/srt/managers/scheduler_components/load_publisher.py +++ b/python/sglang/srt/managers/scheduler_components/load_publisher.py @@ -19,7 +19,7 @@ co-hosted neighbor's KV publisher binds. The port comes from `resolve_load_pub_range` (the same function `/server_info` advertises with, so the two cannot drift). With `auto`, a worker's ZMQ footprint is -`2 * dp_size` ports after its KV base (`2 * dp_size + 1` with the +`2 * num_dp_ranks` ports after its KV base (`2 * num_dp_ranks + 1` with the conventional adjacent replay), so co-hosted workers must space their KV bases that far apart or move the range with an explicit address. """ @@ -164,7 +164,7 @@ def __init__( resolved, reason = resolve_load_pub_range( kv_endpoint=cfg.endpoint, replay_endpoint=cfg.replay_endpoint, - dp_size=get_parallel().dp_size, + dp_size=get_parallel().num_dp_ranks, load_publish_endpoint=load_publish_endpoint, ) if resolved is None: diff --git a/python/sglang/srt/managers/scheduler_components/profiler_manager.py b/python/sglang/srt/managers/scheduler_components/profiler_manager.py index 2791c3a4b4ab..27814e104f70 100644 --- a/python/sglang/srt/managers/scheduler_components/profiler_manager.py +++ b/python/sglang/srt/managers/scheduler_components/profiler_manager.py @@ -285,7 +285,7 @@ def _merge_profile_traces(self) -> str: if get_parallel().tp_rank != 0: return "" - if get_parallel().dp_size > 1 and get_parallel().dp_rank != 0: + if get_parallel().num_dp_ranks > 1 and get_parallel().dp_rank != 0: return "" if get_parallel().pp_size > 1 and get_parallel().pp_rank != 0: return "" @@ -340,7 +340,7 @@ def _stop_profile( filename_parts = [self.profile_id, f"TP-{get_parallel().tp_rank}"] # Only add other ranks if parallelism is enabled (size > 1) - if get_parallel().dp_size > 1: + if get_parallel().num_dp_ranks > 1: filename_parts.append(f"DP-{get_parallel().dp_rank}") if get_parallel().pp_size > 1: filename_parts.append(f"PP-{get_parallel().pp_rank}") diff --git a/python/sglang/srt/managers/scheduler_components/recv_skipper.py b/python/sglang/srt/managers/scheduler_components/recv_skipper.py index 364723b6c2fe..546b56d7f753 100644 --- a/python/sglang/srt/managers/scheduler_components/recv_skipper.py +++ b/python/sglang/srt/managers/scheduler_components/recv_skipper.py @@ -34,7 +34,7 @@ def derive_forward_mode(gathered_modes: List[int]) -> Optional[ForwardMode]: return ForwardMode.DECODE def __init__(self): - self._use_synced_mode = get_parallel().enable_dp_attention + self._use_synced_mode = get_parallel().attn_dp_enabled self._counter = 0 self._threshold = get_schedule().scheduler_recv_interval # All can be tuned if needed diff --git a/python/sglang/srt/managers/scheduler_components/request_receiver.py b/python/sglang/srt/managers/scheduler_components/request_receiver.py index 007cdf0edeb5..f188b547d80b 100644 --- a/python/sglang/srt/managers/scheduler_components/request_receiver.py +++ b/python/sglang/srt/managers/scheduler_components/request_receiver.py @@ -173,7 +173,7 @@ def _broadcast_reqs_across_ranks( rank 0 and would overwrite every DP group's aborts but the first. """ local_reqs = local_reqs or [] - if get_parallel().enable_dp_attention: + if get_parallel().attn_dp_enabled: if get_parallel().attn_tp_rank == 0 and get_parallel().attn_cp_rank == 0: work_reqs, control_reqs = self._split_work_and_control_reqs(recv_reqs) work_reqs.extend(local_reqs) @@ -266,7 +266,7 @@ def _finalize_shm_features(self, recv_reqs: Optional[List]) -> None: # 1. wait until every rank has opened the shared feature segments parallel = get_parallel() - if parallel.enable_dp_attention: + if parallel.attn_dp_enabled: if parallel.attn_tp_size > 1: barrier(group=self.attn_tp_cpu_group) if parallel.attn_cp_size > 1: @@ -290,7 +290,7 @@ def _finalize_shm_features(self, recv_reqs: Optional[List]) -> None: failed[index] = 1 # 3. all ranks reject the same requests before entering model collectives - if parallel.enable_dp_attention: + if parallel.attn_dp_enabled: if parallel.attn_tp_size > 1: all_reduce(failed, op=ReduceOp.MAX, group=self.attn_tp_cpu_group) if parallel.attn_cp_size > 1: diff --git a/python/sglang/srt/managers/tokenizer_control_mixin.py b/python/sglang/srt/managers/tokenizer_control_mixin.py index f5051295bd33..f62afbae8ed6 100644 --- a/python/sglang/srt/managers/tokenizer_control_mixin.py +++ b/python/sglang/srt/managers/tokenizer_control_mixin.py @@ -186,7 +186,7 @@ def init_communicators(self: TokenizerManager): mode = spec[2] if len(spec) > 2 else "queueing" comm = FanOutCommunicator( self._dispatch_to_scheduler, - get_parallel().dp_size, + get_parallel().num_dp_ranks, mode, ) setattr(self, f"{name}_communicator", comm) @@ -195,7 +195,7 @@ def init_communicators(self: TokenizerManager): def update_control_communicator_fan_out(self: TokenizerManager, worker_count: int): primary_group_control = ( - get_parallel().enable_dp_attention + get_parallel().attn_dp_enabled and not get_parallel().enable_dp_attention_local_control_broadcast ) if primary_group_control: @@ -444,8 +444,8 @@ async def init_weights_update_group( request: Optional[fastapi.Request] = None, ) -> Tuple[bool, str]: self.auto_create_handle_loop() - assert get_parallel().dp_size == 1 or get_parallel().enable_dp_attention, ( - "dp_size must be 1 or dp attention must be enabled for update weights from distributed" + assert get_parallel().dp_size == 1, ( + "data-parallel replicas are not supported for update weights from distributed" ) results = await self.init_weights_update_group_communicator(obj) @@ -457,8 +457,8 @@ async def destroy_weights_update_group( request: Optional[fastapi.Request] = None, ) -> Tuple[bool, str]: self.auto_create_handle_loop() - assert get_parallel().dp_size == 1 or get_parallel().enable_dp_attention, ( - "dp_size must be 1 or dp attention must be enabled for destroy parameter update group" + assert get_parallel().dp_size == 1, ( + "data-parallel replicas are not supported for destroy parameter update group" ) results = await self.destroy_weights_update_group_communicator(obj) @@ -502,8 +502,8 @@ async def update_weights_from_distributed( request: Optional[fastapi.Request] = None, ) -> Tuple[bool, str]: self.auto_create_handle_loop() - assert get_parallel().dp_size == 1 or get_parallel().enable_dp_attention, ( - "dp_size must be 1 or dp attention must be enabled for update weights from distributed" + assert get_parallel().dp_size == 1, ( + "data-parallel replicas are not supported for update weights from distributed" ) if obj.abort_all_requests: @@ -535,8 +535,8 @@ async def init_weights_send_group_for_remote_instance( ) -> Tuple[bool, str]: self.auto_create_handle_loop() # TODO: support DP - assert get_parallel().dp_size == 1, ( - "dp_size must be 1 for init_weights_send_group_for_remote_instance" + assert get_parallel().num_dp_ranks == 1, ( + "init_weights_send_group_for_remote_instance requires a single DP rank" ) result = ( await self.init_weights_send_group_for_remote_instance_communicator(obj) @@ -550,8 +550,8 @@ async def send_weights_to_remote_instance( ) -> Tuple[bool, str]: self.auto_create_handle_loop() # TODO: support DP - assert get_parallel().dp_size == 1, ( - "dp_size must be 1 for send_weights_to_remote_instance" + assert get_parallel().num_dp_ranks == 1, ( + "send_weights_to_remote_instance requires a single DP rank" ) result = (await self.send_weights_to_remote_instance_communicator(obj))[0] return result.success, result.message @@ -562,8 +562,8 @@ async def update_weights_from_tensor( request: Optional[fastapi.Request] = None, ) -> Tuple[bool, str]: self.auto_create_handle_loop() - assert get_parallel().dp_size == 1 or get_parallel().enable_dp_attention, ( - "dp_size must be 1 or dp attention must be enabled for update weights from tensor" + assert get_parallel().dp_size == 1, ( + "data-parallel replicas are not supported for update weights from tensor" ) if obj.abort_all_requests: @@ -600,8 +600,8 @@ async def update_weights_from_ipc( self.auto_create_handle_loop() try: # For now, we only support single data parallel instance - assert get_parallel().dp_size == 1 or get_parallel().enable_dp_attention, ( - "dp_size must be 1 or dp attention must be enabled for update weights from IPC" + assert get_parallel().dp_size == 1, ( + "data-parallel replicas are not supported for update weights from IPC" ) logger.info("Starting IPC weight update") @@ -665,8 +665,8 @@ async def load_lora_adapter( "LoRA is not enabled. Please set `--enable-lora` to enable LoRA." ) - assert get_parallel().dp_size == 1 or get_parallel().enable_dp_attention, ( - "dp_size must be 1 or dp attention must be enabled for dynamic lora loading" + assert get_parallel().dp_size == 1, ( + "data-parallel replicas are not supported for dynamic lora loading" ) logger.info( "Start load Lora adapter. Lora name=%s, path=%s", @@ -751,8 +751,8 @@ async def load_lora_adapter_from_tensors( "LoRA is not enabled. Please set `--enable-lora` to enable LoRA." ) - assert get_parallel().dp_size == 1 or get_parallel().enable_dp_attention, ( - "dp_size must be 1 or dp attention must be enabled for dynamic lora loading" + assert get_parallel().dp_size == 1, ( + "data-parallel replicas are not supported for dynamic lora loading" ) logger.info( "Start load Lora adapter from tensors. Lora name=%s", @@ -841,8 +841,8 @@ async def unload_lora_adapter( "lora_name must be provided to unload LoRA adapter" ) - assert get_parallel().dp_size == 1 or get_parallel().enable_dp_attention, ( - "dp_size must be 1 or dp attention must be enabled for dynamic lora loading" + assert get_parallel().dp_size == 1, ( + "data-parallel replicas are not supported for dynamic lora loading" ) logger.info( "Start unload Lora adapter. Lora name=%s", @@ -862,7 +862,7 @@ async def get_weights_by_name( self.auto_create_handle_loop() results = await self.get_weights_by_name_communicator(obj) all_parameters = [r.parameter for r in results] - if get_parallel().dp_size == 1: + if get_parallel().num_dp_ranks == 1: return all_parameters[0] else: return all_parameters diff --git a/python/sglang/srt/managers/tokenizer_manager.py b/python/sglang/srt/managers/tokenizer_manager.py index 989b7a2faa8b..797e8a6180f6 100644 --- a/python/sglang/srt/managers/tokenizer_manager.py +++ b/python/sglang/srt/managers/tokenizer_manager.py @@ -488,7 +488,7 @@ def __init__( self.server_args = server_args assert_published(server_args, role="tokenizer") self.startup_time: Optional[Dict[str, Any]] = None - self.elastic_worker_count = get_parallel().dp_size + self.elastic_worker_count = get_parallel().num_dp_ranks self.elastic_pending_ep_size = None self.elastic_scale_phase = "idle" self.elastic_last_error = None @@ -872,14 +872,14 @@ async def generate_request( ) if isinstance(obj, GenerateReqInput) and obj.routed_dp_rank is not None: - dp_size = self.elastic_worker_count - if dp_size <= 1 and obj.routed_dp_rank == 0: + num_dp_ranks = self.elastic_worker_count + if num_dp_ranks <= 1 and obj.routed_dp_rank == 0: logger.debug( - f"routed_dp_rank={obj.routed_dp_rank} is ignored because dp_size={dp_size}" + f"routed_dp_rank={obj.routed_dp_rank} is ignored because num_dp_ranks={num_dp_ranks}" ) - elif obj.routed_dp_rank < 0 or obj.routed_dp_rank >= dp_size: + elif obj.routed_dp_rank < 0 or obj.routed_dp_rank >= num_dp_ranks: raise ValueError( - f"routed_dp_rank={obj.routed_dp_rank} out of range [0, {dp_size})" + f"routed_dp_rank={obj.routed_dp_rank} out of range [0, {num_dp_ranks})" ) self._init_req_state(obj, request) @@ -1670,7 +1670,7 @@ def _should_use_batch_tokenization(self, batch_size, requests) -> bool: return batch_size > 0 and ( get_serving().enable_tokenizer_batch_encode or ( - (not get_parallel().enable_dp_attention) + (not get_parallel().attn_dp_enabled) and (not self._batch_has_text(batch_size, requests)) ) ) diff --git a/python/sglang/srt/mem_cache/kv_cache_builder.py b/python/sglang/srt/mem_cache/kv_cache_builder.py index ec75dd81ded7..b03769326a7f 100644 --- a/python/sglang/srt/mem_cache/kv_cache_builder.py +++ b/python/sglang/srt/mem_cache/kv_cache_builder.py @@ -359,7 +359,7 @@ def build_kv_cache( ), is_eagle=spec_algorithm.is_eagle(), tp_cache_group=( - attn_tp_cpu_group if get_parallel().enable_dp_attention else tp_cpu_group + attn_tp_cpu_group if get_parallel().attn_dp_enabled else tp_cpu_group ), attn_cp_cache_group=attn_cp_cpu_group, attn_tp_cache_group=attn_tp_cpu_group, diff --git a/python/sglang/srt/mem_cache/storage/flexkv/README.md b/python/sglang/srt/mem_cache/storage/flexkv/README.md index 23e0daa76f86..11b1f326ee93 100644 --- a/python/sglang/srt/mem_cache/storage/flexkv/README.md +++ b/python/sglang/srt/mem_cache/storage/flexkv/README.md @@ -299,7 +299,7 @@ Supported: `TransferManagerOnRemote` via the same ZMQ channel used for GPU registration. * **CP** (`attn_cp_size > 1`) — sync handled symmetrically with TP. -* **DP attention** (`enable_dp_attention=True`) — the inner +* **DP attention** (`--attn-dp-size` > 1) — the inner `attn_tp_size` is what FlexKV uses for register-side routing. --- diff --git a/python/sglang/srt/mem_cache/storage/flexkv/flexkv_connector.py b/python/sglang/srt/mem_cache/storage/flexkv/flexkv_connector.py index 64141c4bf5d8..1cec682dd029 100644 --- a/python/sglang/srt/mem_cache/storage/flexkv/flexkv_connector.py +++ b/python/sglang/srt/mem_cache/storage/flexkv/flexkv_connector.py @@ -2,7 +2,7 @@ The public surface is small (see "Public API" below). The class owns: -* the FlexKV ``KVManager`` (server-client mode when ``dp_size > 1`` or +* the FlexKV ``KVManager`` (server-client mode when ``num_dp_ranks > 1`` or multi-instance; in-process otherwise — handled by FlexKV itself); * the per-rank ``KVTPClient`` that registers this rank's GPU KV cache with the FlexKV TransferManager; diff --git a/python/sglang/srt/model_executor/cpu_graph_runner.py b/python/sglang/srt/model_executor/cpu_graph_runner.py index 3f055ee49e14..bbdfad923144 100644 --- a/python/sglang/srt/model_executor/cpu_graph_runner.py +++ b/python/sglang/srt/model_executor/cpu_graph_runner.py @@ -619,7 +619,7 @@ def __init__(self, model_runner: ModelRunner): self.speculative_algorithm = get_spec().speculative_algorithm self.enable_profile_cuda_graph = get_exec().graph.enable_profile_cuda_graph self.tp_size = get_parallel().tp_size - self.dp_size = get_parallel().dp_size + self.num_dp_ranks = get_parallel().num_dp_ranks self.pp_size = get_parallel().pp_size self.capture_forward_mode = ForwardMode.DECODE @@ -644,7 +644,7 @@ def __init__(self, model_runner: ModelRunner): "CPUGraphRunner does not support speculative inference yet." ) - assert self.dp_size == 1, "CPUGraphRunner does not support DP yet." + assert self.num_dp_ranks == 1, "CPUGraphRunner does not support DP yet." assert self.pp_size == 1, "CPUGraphRunner does not support PP yet." # Batch sizes to capture diff --git a/python/sglang/srt/model_executor/cuda_graph_buffer_registry.py b/python/sglang/srt/model_executor/cuda_graph_buffer_registry.py index f14aa7a9af3f..a461e6e625fa 100644 --- a/python/sglang/srt/model_executor/cuda_graph_buffer_registry.py +++ b/python/sglang/srt/model_executor/cuda_graph_buffer_registry.py @@ -530,7 +530,7 @@ def build_decode_registry( require_mlp_tp_gather: bool = False, # Per-bucket attn-TP sharded (SP) predicate; defaults to replicated. attn_tp_sharded_fn: Callable[[int], bool] = lambda num_tokens: False, - dp_size: int = 1, + num_dp_ranks: int = 1, register_global_num_tokens: bool = True, share_pool: bool = True, source: Optional[Any] = None, @@ -700,7 +700,7 @@ def _global_num_tokens_post_fill(buf, fb, ctx): buf.fill_(ctx.padded_num_tokens) _global_shape = ( - (lambda _bs, _mt: (dp_size,)) + (lambda _bs, _mt: (num_dp_ranks,)) if require_mlp_tp_gather else (lambda _bs, _mt: (1,)) ) @@ -1048,7 +1048,7 @@ def build_eager_registry( register_global_num_tokens=False, require_gathered_buffer=False, require_mlp_tp_gather=False, - dp_size=get_parallel().dp_size, + num_dp_ranks=get_parallel().num_dp_ranks, share_pool=True, source=None, ) diff --git a/python/sglang/srt/model_executor/forward_batch_info.py b/python/sglang/srt/model_executor/forward_batch_info.py index 6733618299ed..2917e755b425 100644 --- a/python/sglang/srt/model_executor/forward_batch_info.py +++ b/python/sglang/srt/model_executor/forward_batch_info.py @@ -1186,7 +1186,7 @@ def init_new( # Init lora information if ( model_runner.lora_manager is not None - and not model_runner.lora_manager.enable_dp_attention + and not model_runner.lora_manager.attn_dp_enabled ): # In the non-LoRA overlap loading case, we fetch LoRA adapters into the memory pool # as a batch, right before running the batch diff --git a/python/sglang/srt/model_executor/model_runner.py b/python/sglang/srt/model_executor/model_runner.py index b66e633704cc..937cd160aeb7 100644 --- a/python/sglang/srt/model_executor/model_runner.py +++ b/python/sglang/srt/model_executor/model_runner.py @@ -530,7 +530,7 @@ def _initialize_elastic_ep_joiner(self) -> None: new_dp_size=join_effective_ep_size, new_dp_rank=global_ep_rank, ) - get_context().override("elastic_ep.scale_join", dp_size=join_effective_ep_size) + get_parallel().override_permanently(num_dp_ranks=join_effective_ep_size) if self.eplb_manager is not None: self.eplb_manager.disable_rebalance( "EPLB rebalance is disabled while elastic EP scale-up " @@ -965,7 +965,7 @@ def maybe_init_hisparse_coordinator(self): device=self.device, tp_group=( get_parallel().attn_tp_group.cpu_group - if get_parallel().enable_dp_attention + if get_parallel().attn_dp_enabled else get_parallel().tp_group.cpu_group ), host_to_device_ratio=hisparse_cfg.host_to_device_ratio, @@ -1628,7 +1628,7 @@ def _prepare_eager_forward_batch(self, forward_batch: ForwardBatch) -> None: forward_batch.prepare_mlp_sync_batch(self) else: forward_batch.prepare_attn_tp_scatter_input(self) - if self.lora_manager is not None and self.lora_manager.enable_dp_attention: + if self.lora_manager is not None and self.lora_manager.attn_dp_enabled: self.lora_manager.prepare_lora_batch(forward_batch) # Derive the LOCAL num_token_non_padded from the GLOBAL scalar. sharded is @@ -2217,7 +2217,7 @@ def _finalize_scale_up( new_dp_size=target_size, new_dp_rank=self._elastic_global_rank(), ) - get_context().override("elastic_ep.scale", dp_size=target_size) + get_parallel().override_permanently(num_dp_ranks=target_size) recapture_cuda_graph = self._elastic_cuda_graph_enabled() if recapture_cuda_graph: diff --git a/python/sglang/srt/model_executor/model_runner_components/startup_weight_load.py b/python/sglang/srt/model_executor/model_runner_components/startup_weight_load.py index 856281b73a63..8388da240346 100644 --- a/python/sglang/srt/model_executor/model_runner_components/startup_weight_load.py +++ b/python/sglang/srt/model_executor/model_runner_components/startup_weight_load.py @@ -82,7 +82,7 @@ class StartupWeightLoadOptions: attn_cp_size: int dcp_size: int pp_size: int - dp_size: int + num_dp_ranks: int ep_size: int cpu_offload_gb: int offload_group_size: int @@ -123,7 +123,7 @@ def from_published_config( attn_cp_size=get_parallel().attn_cp_size, dcp_size=get_parallel().dcp_size, pp_size=get_parallel().pp_size, - dp_size=get_parallel().dp_size, + num_dp_ranks=get_parallel().num_dp_ranks, ep_size=get_parallel().ep_size, cpu_offload_gb=get_exec().offload.cpu_offload_gb, offload_group_size=get_exec().offload.offload_group_size, @@ -369,7 +369,7 @@ def _get_unsupported_reason( "decode context parallelism is not supported", ), (options.pp_size != 1, "pipeline parallelism is not supported"), - (options.dp_size != 1, "data parallelism is not supported"), + (options.num_dp_ranks != 1, "data parallelism is not supported"), (options.ep_size != 1, "expert parallelism is not supported"), (options.cpu_offload_gb > 0, "CPU offload is not supported"), ( diff --git a/python/sglang/srt/model_executor/runner/base_runner.py b/python/sglang/srt/model_executor/runner/base_runner.py index b7d5ebdefda9..f82ed5afb1ab 100644 --- a/python/sglang/srt/model_executor/runner/base_runner.py +++ b/python/sglang/srt/model_executor/runner/base_runner.py @@ -162,9 +162,11 @@ def _allocate_decode_buffers( encoder_lens = None if require_mlp_tp_gather: - global_num_tokens_gpu = torch.zeros((parallel.dp_size,), dtype=torch.int32) + global_num_tokens_gpu = torch.zeros( + (parallel.num_dp_ranks,), dtype=torch.int32 + ) global_num_tokens_for_logprob_gpu = torch.zeros( - (parallel.dp_size,), dtype=torch.int32 + (parallel.num_dp_ranks,), dtype=torch.int32 ) else: global_num_tokens_gpu = torch.zeros((1,), dtype=torch.int32) @@ -226,8 +228,8 @@ def __init__(self, model_runner: ModelRunner) -> None: self.model_runner = model_runner self.device = model_runner.device self.device_module = torch.get_device_module(self.device) - # elastic-EP scale-up rewrites dp_size on the published config - self.dp_size = get_parallel().dp_size + # elastic-EP scale-up widens num_dp_ranks on the published config + self.num_dp_ranks = get_parallel().num_dp_ranks self.pp_size = get_parallel().pp_size self.enable_pdmux = get_disagg().enable_pdmux self.return_hidden_states_mode = ( @@ -613,7 +615,7 @@ def _dummy_run( assert require_mlp_tp_gather_ or require_attn_tp_gather_ if require_mlp_tp_gather_: - global_num_tokens_cpu = [num_tokens] * get_parallel().dp_size + global_num_tokens_cpu = [num_tokens] * get_parallel().num_dp_ranks elif require_attn_tp_gather_: global_num_tokens_cpu = [num_tokens] else: diff --git a/python/sglang/srt/model_executor/runner/decode_cuda_graph_runner.py b/python/sglang/srt/model_executor/runner/decode_cuda_graph_runner.py index 1846c7b3b221..a89b012f9404 100644 --- a/python/sglang/srt/model_executor/runner/decode_cuda_graph_runner.py +++ b/python/sglang/srt/model_executor/runner/decode_cuda_graph_runner.py @@ -260,7 +260,7 @@ def __init__( self.require_mlp_tp_gather or self.require_attn_tp_gather ) self.require_mlp_sync = ( - get_parallel().enable_dp_attention or self.require_gathered_buffer + get_parallel().attn_dp_enabled or self.require_gathered_buffer ) self.enable_two_batch_overlap = get_exec().overlap.enable_two_batch_overlap self.use_ngram_embedding = model_runner.ngram_embedding_manager.enabled @@ -414,7 +414,7 @@ def __init__( self.model_runner.model_config.vocab_size, rows=logits_buffer_rows ), dtype=self.model_runner.model_config.dtype, - dp_size=self.dp_size, + num_dp_ranks=self.num_dp_ranks, pp_size=self.pp_size, is_encoder_decoder=self.is_encoder_decoder, require_mlp_tp_gather=self.require_mlp_tp_gather, @@ -458,7 +458,7 @@ def __init__( enable_prefill_cp=self.enable_prefill_cp, require_mlp_tp_gather=self.require_mlp_tp_gather, attn_tp_sharded_fn=self.model_runner.attn_tp_sequence_sharded, - dp_size=self.dp_size, + num_dp_ranks=self.num_dp_ranks, source=self.buffers, ) @@ -584,7 +584,7 @@ def _capture_graph_size(self, *, bs: int, num_tokens: int) -> int: def _global_num_tokens_for_graph(self, num_tokens: int) -> Optional[list[int]]: if self.require_mlp_tp_gather: - return [num_tokens] * self.dp_size + return [num_tokens] * self.num_dp_ranks if self.require_attn_tp_gather: return [num_tokens] return None @@ -1427,7 +1427,7 @@ def load_batch( ) if ( self.model_runner.lora_manager is not None - and self.model_runner.lora_manager.enable_dp_attention + and self.model_runner.lora_manager.attn_dp_enabled ): self.model_runner.lora_manager.prepare_lora_batch( cast(ForwardBatch, fb_view) diff --git a/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py b/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py index 2def2fd4aa3d..d0678daa8c1b 100644 --- a/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py +++ b/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py @@ -311,7 +311,7 @@ def __init__(self, model_runner: ModelRunner): self.prefill_backend_name = prefill_config.backend self.prefer_eager_mixed_prefill = ( self.prefill_backend_name == Backend.BREAKABLE - and get_parallel().enable_dp_attention + and get_parallel().attn_dp_enabled and getattr( model_runner.attn_backend, "prefer_eager_mixed_prefill_under_dp_attention", @@ -1432,7 +1432,7 @@ def _slot(name): return registry.get_slot(name).slice_for(bs, num_tokens) if self.require_mlp_tp_gather: - global_num_tokens_cpu = [num_tokens] * self.dp_size + global_num_tokens_cpu = [num_tokens] * self.num_dp_ranks elif self.require_attn_tp_gather: global_num_tokens_cpu = [num_tokens] else: diff --git a/python/sglang/srt/model_executor/runner_utils/buffers.py b/python/sglang/srt/model_executor/runner_utils/buffers.py index ac7b53a3ef4a..e0d173d51582 100644 --- a/python/sglang/srt/model_executor/runner_utils/buffers.py +++ b/python/sglang/srt/model_executor/runner_utils/buffers.py @@ -136,7 +136,7 @@ def create( hidden_size: int, next_token_logits_buffer: torch.Tensor, dtype: torch.dtype, - dp_size: int, + num_dp_ranks: int, pp_size: int, is_encoder_decoder: bool, require_mlp_tp_gather: bool, @@ -207,9 +207,9 @@ def create( encoder_lens = None if require_mlp_tp_gather: - global_num_tokens_gpu = torch.zeros((dp_size,), dtype=torch.int32) + global_num_tokens_gpu = torch.zeros((num_dp_ranks,), dtype=torch.int32) global_num_tokens_for_logprob_gpu = torch.zeros( - (dp_size,), dtype=torch.int32 + (num_dp_ranks,), dtype=torch.int32 ) else: global_num_tokens_gpu = torch.zeros((1,), dtype=torch.int32) diff --git a/python/sglang/srt/models/deepseek_v4.py b/python/sglang/srt/models/deepseek_v4.py index 5268254c7741..53461b33eca2 100644 --- a/python/sglang/srt/models/deepseek_v4.py +++ b/python/sglang/srt/models/deepseek_v4.py @@ -3896,9 +3896,9 @@ def _run_moe_ffn_dp_sync( # Applies to BOTH prefill and decode: the shared expert is a per-token MLP, # so computing it on this rank's local tokens (M_local rows) is identical to # computing it on the gathered global buffer (M_global rows) and keeping the - # local slice -- but costs 1/dp_size the rows. With a replicated (TP1) shared + # local slice -- but costs 1/attn_dp_size the rows. With a replicated (TP1) shared # expert this cancels the TP1 "full-dim" cost in decode (M_local * dim == - # M_global * dim/tp), so decode no longer pays the ~dp_size x penalty. + # M_global * dim/tp), so decode no longer pays the ~attn_dp_size x penalty. _shared_local = None _do_shared_local = ( _SHARED_EXPERT_LOCAL diff --git a/python/sglang/srt/models/zaya.py b/python/sglang/srt/models/zaya.py index 539828ef7f3d..a8fe43d8e890 100644 --- a/python/sglang/srt/models/zaya.py +++ b/python/sglang/srt/models/zaya.py @@ -842,7 +842,7 @@ def __init__( assert attn_tp_size == self.tp_size, ( f"ZAYA1 head-parallel attention requires the attention TP group " f"({attn_tp_size}) to equal the global TP group ({self.tp_size}); " - "DP attention (enable_dp_attention) is not supported for ZAYA1." + "attention DP is not supported for ZAYA1." ) assert self.num_q_heads_full % self.tp_size == 0, ( f"num_attention_heads ({self.num_q_heads_full}) must be divisible " diff --git a/python/sglang/srt/ray/data_parallel_controller.py b/python/sglang/srt/ray/data_parallel_controller.py index 7522f84c5467..e2fdfce8589b 100644 --- a/python/sglang/srt/ray/data_parallel_controller.py +++ b/python/sglang/srt/ray/data_parallel_controller.py @@ -77,7 +77,7 @@ def launch_dp_schedulers(self, server_args: ServerArgs, port_args: PortArgs): sockets = [] dp_port_args_list = [] - for dp_rank in range(get_parallel().dp_size): + for dp_rank in range(get_parallel().num_dp_ranks): tmp_port_args = PortArgs.init_new(server_args) tmp_port_args.tokenizer_ipc_name = port_args.tokenizer_ipc_name tmp_port_args.detokenizer_ipc_name = port_args.detokenizer_ipc_name @@ -101,7 +101,7 @@ def launch_dp_schedulers(self, server_args: ServerArgs, port_args: PortArgs): sock.close() # Create actors for each DP rank sequentially - for dp_rank in range(get_parallel().dp_size): + for dp_rank in range(get_parallel().num_dp_ranks): self._launch_ray_tp_group(server_args, dp_port_args_list[dp_rank], dp_rank) def launch_dp_attention_schedulers( @@ -112,7 +112,7 @@ def launch_dp_attention_schedulers( # rank-0 node IP instead of tcp://* to avoid exposing unauthenticated # ZMQ sockets (CVE-2026-3060). worker_ports = [] - for dp_rank in range(get_parallel().dp_size): + for dp_rank in range(get_parallel().num_dp_ranks): worker_port, worker_socket = get_zmq_socket_on_host( self.context, zmq.PUSH, host=self.rank0_node_ip ) @@ -158,12 +158,11 @@ def _launch_ray_tp_group( ) parallel = get_parallel() - if parallel.enable_dp_attention: + if parallel.attn_dp_enabled: _, _, actual_dp_rank, _ = compute_dp_attention_world_info( - parallel.enable_dp_attention, tp_rank, parallel.tp_size, - parallel.dp_size, + parallel.attn_dp_size, parallel.attn_cp_size, ) rank_port_args = PortArgs.init_new( @@ -230,12 +229,11 @@ def _launch_ray_tp_group( bundle_idx = bundle_indices[global_rank] - if get_parallel().enable_dp_attention: + if get_parallel().attn_dp_enabled: _, _, actual_dp_rank, _ = compute_dp_attention_world_info( - get_parallel().enable_dp_attention, tp_rank, get_parallel().tp_size, - get_parallel().dp_size, + get_parallel().attn_dp_size, get_parallel().attn_cp_size, ) rank_port_args = PortArgs.init_new( diff --git a/python/sglang/srt/ray/engine.py b/python/sglang/srt/ray/engine.py index 0d6b8e3d53a2..ced74e717293 100644 --- a/python/sglang/srt/ray/engine.py +++ b/python/sglang/srt/ray/engine.py @@ -112,7 +112,6 @@ def _compute_world_size() -> int: will hold the process groups, so there is nothing live to ask. """ return compute_world_size( - enable_dp_attention=get_parallel().enable_dp_attention, dp_size=get_parallel().dp_size, tp_size=get_parallel().tp_size, pp_size=get_parallel().pp_size, @@ -274,10 +273,10 @@ def _launch_scheduler_processes( ) parallel = get_parallel() - if parallel.enable_dp_attention: + if parallel.attn_dp_enabled: total_gpus = parallel.tp_size * parallel.pp_size else: - total_gpus = parallel.dp_size * parallel.tp_size * parallel.pp_size + total_gpus = parallel.num_dp_ranks * parallel.tp_size * parallel.pp_size nnodes = parallel.nnodes gpus_per_node = total_gpus // nnodes @@ -318,7 +317,7 @@ def _launch_scheduler_processes( rank0_bundle_idx = int(indices_str.split(",")[0]) if indices_str else 0 rank0_node_ip = _get_bundle_node_ip(pg, rank0_bundle_idx) - if get_parallel().dp_size == 1: + if get_parallel().num_dp_ranks == 1: dist_init_addr = f"{rank0_node_ip}:{port_args.nccl_port}" logger.info(f"dist_init_addr: {dist_init_addr}") @@ -448,17 +447,17 @@ def _launch_dp_scheduler_processes( ) parallel = get_parallel() - if parallel.enable_dp_attention: + if parallel.attn_dp_enabled: # DP attention folds DP into TP — total GPUs = tp_size * pp_size total_gpus = parallel.tp_size * parallel.pp_size else: - total_gpus = parallel.dp_size * parallel.tp_size * parallel.pp_size + total_gpus = parallel.num_dp_ranks * parallel.tp_size * parallel.pp_size gpus_per_node = total_gpus // parallel.nnodes logger.info( f"Ray DP cluster: {parallel.nnodes} nodes, " f"{gpus_per_node} GPUs/node, dp_size={parallel.dp_size}, " - f"tp_size={parallel.tp_size}, pp_size={parallel.pp_size}, " - f"enable_dp_attention={parallel.enable_dp_attention}" + f"attn_dp_size={parallel.attn_dp_size}, " + f"tp_size={parallel.tp_size}, pp_size={parallel.pp_size}" ) # Declared on the record itself so `PortArgs.init_new()` can compute diff --git a/python/sglang/srt/runtime_context.py b/python/sglang/srt/runtime_context.py index 5742a0bd7fe3..d58c8ccb3d61 100644 --- a/python/sglang/srt/runtime_context.py +++ b/python/sglang/srt/runtime_context.py @@ -70,16 +70,14 @@ def _parallel_fields() -> frozenset: return frozenset(_parallel_config_leaves() | derived) -def derive_attention_widths( - *, tp_size: int, attn_cp_size: int, dp_size: int, enable_dp_attention: bool -) -> tuple: - """Return (attn_dp_size, attn_tp_size) from the configured widths.""" - attn_dp_size = dp_size if enable_dp_attention else 1 - return attn_dp_size, tp_size // attn_dp_size // attn_cp_size +def derive_attn_tp_size(*, tp_size: int, attn_cp_size: int, attn_dp_size: int) -> int: + """Return the attention-TP width: what is left of the TP group after the + attention-DP and attention-CP dimensions.""" + return tp_size // attn_dp_size // attn_cp_size def derive_attention_ranks( - *, tp_rank: int, attn_tp_size: int, attn_cp_size: int, enable_dp_attention: bool + *, tp_rank: int, attn_tp_size: int, attn_cp_size: int ) -> tuple: """Return (attn_tp_rank, attn_dp_rank) for a process at ``tp_rank``. @@ -87,11 +85,11 @@ def derive_attention_ranks( tp_rank = (attn_dp_rank * attn_cp_size + attn_cp_rank) * attn_tp_size + attn_tp_rank + + Without attention DP the TP group is one attention-DP group, so + ``attn_dp_rank`` is zero. """ - attn_tp_rank = tp_rank % attn_tp_size - if not enable_dp_attention: - return attn_tp_rank, 0 - return attn_tp_rank, tp_rank // (attn_tp_size * attn_cp_size) + return tp_rank % attn_tp_size, tp_rank // (attn_tp_size * attn_cp_size) def spawn_world_rank(server_args, *, tp_rank: int, pp_rank: int) -> int: @@ -149,13 +147,9 @@ def derive_parallel_widths( """Derive attention and MoE widths and DCP settings from configuration.""" return { "attn_dp_size": attn_dp_size, - # `attn_dp_size` already accounts for disabled DP attention. - "attn_tp_size": derive_attention_widths( - tp_size=tp_size, - attn_cp_size=attn_cp_size, - dp_size=attn_dp_size, - enable_dp_attention=True, - )[1], + "attn_tp_size": derive_attn_tp_size( + tp_size=tp_size, attn_cp_size=attn_cp_size, attn_dp_size=attn_dp_size + ), "moe_ep_size": moe_ep_size, "moe_tp_size": tp_size // moe_ep_size // moe_dp_size, "dcp_enabled": dcp_enabled, @@ -165,16 +159,10 @@ def derive_parallel_widths( def parallel_widths_of(cfg: Any) -> dict: """Return derived parallel settings from resolved configuration.""" - attn_dp_size, _ = derive_attention_widths( - tp_size=cfg.tp_size, - attn_cp_size=cfg.attn_cp_size, - dp_size=cfg.dp_size, - enable_dp_attention=cfg.enable_dp_attention, - ) return derive_parallel_widths( tp_size=cfg.tp_size, attn_cp_size=cfg.attn_cp_size, - attn_dp_size=attn_dp_size, + attn_dp_size=cfg.attn_dp_size, moe_ep_size=cfg.ep_size, moe_dp_size=cfg.moe_dp_size, dcp_size=cfg.dcp_size, @@ -200,9 +188,23 @@ def attn_tp_size_of(cfg: Any): return parallel_widths_of(cfg)["attn_tp_size"] -def attn_dp_size_of(cfg: Any): - """`attn_dp_size`, computed at publish. See `parallel_widths_of`.""" - return parallel_widths_of(cfg)["attn_dp_size"] +def attn_dp_enabled_of(cfg: Any) -> bool: + """`attn_dp_enabled`, computed at publish. + + A scale joiner's own group can be one rank wide, but it joins an + attention-DP deployment and runs its DP-attention paths. + """ + from sglang.srt.arg_groups.model_override_base import ep_scale_joiner_of + + return cfg.attn_dp_size > 1 or ep_scale_joiner_of(cfg) + + +def num_dp_ranks_of(cfg: Any) -> int: + """`num_dp_ranks`, computed at publish: replicas times attention-DP groups. + + DP attention does not run inside replicas, so one of the two is one. + """ + return cfg.dp_size * cfg.attn_dp_size def attn_dcp_size_of(cfg: Any): @@ -1560,7 +1562,6 @@ def _attention_ranks(parallel, tp_rank: int) -> dict: tp_rank=tp_rank, attn_tp_size=parallel.attn_tp_size, attn_cp_size=parallel.attn_cp_size, - enable_dp_attention=parallel.enable_dp_attention, ) return {"attn_tp_rank": attn_tp_rank, "attn_dp_rank": attn_dp_rank} @@ -2119,7 +2120,8 @@ def describe_kv_events_publisher(server_args: Any) -> Optional[dict]: "endpoint_port_base": port, "topic": cfg.topic, "block_size": kv_event_block_size_of(resolved), - "dp_size": resolved.dp_size, + # The wire key predates attention DP; it counts DP ranks. + "dp_size": num_dp_ranks_of(resolved), } # Load range, from the same resolver SchedulerLoadPublisher binds # with (so the two can't drift). The decline reason is logged once at @@ -2127,7 +2129,7 @@ def describe_kv_events_publisher(server_args: Any) -> Optional[dict]: resolved_range, _reason = resolve_load_pub_range( kv_endpoint=cfg.endpoint, replay_endpoint=cfg.replay_endpoint, - dp_size=resolved.dp_size, + dp_size=num_dp_ranks_of(resolved), load_publish_endpoint=resolved.load_publish_endpoint, ) if resolved_range is not None: diff --git a/python/sglang/srt/rust_server/server.py b/python/sglang/srt/rust_server/server.py index 47c2396b39d0..06f89f778962 100644 --- a/python/sglang/srt/rust_server/server.py +++ b/python/sglang/srt/rust_server/server.py @@ -120,7 +120,9 @@ def launch(cls, scheduler: Scheduler) -> RustServer: os.environ.setdefault("TOKENIZERS_PARALLELISM", "false") # Preserve the DP startup log; ports use node-local offsets. - dp_rank = get_parallel().attn_dp_rank if get_parallel().dp_size > 1 else None + dp_rank = ( + get_parallel().attn_dp_rank if get_parallel().num_dp_ranks > 1 else None + ) if get_exec().moe.is_ep_scale_joiner: # The joining TP group is entirely local to this node. tp_size_per_node = get_parallel().tp_size @@ -180,7 +182,9 @@ def launch(cls, scheduler: Scheduler) -> RustServer: # Under DP every rank runs its own server on its own port, so the rank is # what tells two otherwise identical startup lines apart. dp_note = ( - "" if dp_rank is None else f" (DP rank {dp_rank}/{get_parallel().dp_size})" + "" + if dp_rank is None + else f" (DP rank {dp_rank}/{get_parallel().num_dp_ranks})" ) logger.info( "SGLANG_RUST_SERVER enabled, Rust server listen on %s%s", diff --git a/python/sglang/srt/server_args.py b/python/sglang/srt/server_args.py index a49b2300990c..bcd2371510e5 100644 --- a/python/sglang/srt/server_args.py +++ b/python/sglang/srt/server_args.py @@ -43,7 +43,12 @@ resolving_view, ) from sglang.srt.environ import envs -from sglang.srt.runtime_context import get_platform, publish +from sglang.srt.runtime_context import ( + attn_dp_enabled_of, + get_platform, + num_dp_ranks_of, + publish, +) from sglang.srt.speculative.decoupled_spec_io import DecoupledSpecIpcConfig from sglang.srt.utils.network import NetworkAddress, get_free_port, wait_port_available @@ -345,6 +350,15 @@ def add_cli_args(parser: argparse.ArgumentParser): new_flag="--cuda-graph-backend-{decode,prefill}=disabled", help="Deprecated. Use --cuda-graph-backend-{decode,prefill}=disabled instead.", ) + # `enable_dp_attention` is `no_cli=True` too; resolution turns it into + # `attn_dp_size`. + parser.add_argument( + "--enable-dp-attention", + action=DeprecatedStoreTrueAction, + new_flag="--attn-dp-size ", + help="Deprecated. Use --attn-dp-size instead of " + "--dp-size --enable-dp-attention.", + ) parser.add_argument( "--enable-flashinfer-allreduce-fusion", action="store_true", @@ -488,16 +502,15 @@ def resolve_encoder_transfer_backend( return "zmq_to_scheduler" -def compute_world_size( - *, enable_dp_attention: bool, dp_size: int, tp_size: int, pp_size: int -) -> int: +def compute_world_size(*, dp_size: int, tp_size: int, pp_size: int) -> int: """Total GPU count across all data-parallel replicas. - Takes the values rather than a config object: the two sizes are the widths + Takes the values rather than a config object: the sizes are the widths the launch asked for, which the Ray driver needs before any process group exists, and passing a context would hand it the live groups instead. + Attention DP runs inside the TP group, so it adds no GPUs. """ - return (1 if enable_dp_attention else dp_size) * tp_size * pp_size + return dp_size * tp_size * pp_size def m3_fp8_attn_gemm_enabled(args) -> bool: @@ -706,7 +719,7 @@ def init_new( rank=int(server_args.decoupled_spec_rank), ) - if not cfg.enable_dp_attention: + if not attn_dp_enabled_of(cfg): # Normal case, use IPC within a single node return PortArgs( tokenizer_ipc_name=f"ipc://{tempfile.NamedTemporaryFile(delete=False).name}", @@ -732,12 +745,12 @@ def init_new( dist_init_host = na.host dist_init_port = na.port - # Reserve port_base+0..NUM_DERIVED_PORTS-1 (6 fixed ports + dp_size - # rust-path slots); derive from server_args only (never dp_rank) so + # Reserve port_base+0..NUM_DERIVED_PORTS-1 (6 fixed ports + one + # rust-path slot per DP rank); derive from server_args only (never dp_rank) so # every init_new call agrees, decrementing below dist_init_port on # overflow. is_rust_server = envs.SGLANG_RUST_SERVER.get() - NUM_DERIVED_PORTS = 6 if not is_rust_server else 6 + cfg.dp_size + NUM_DERIVED_PORTS = 6 if not is_rust_server else 6 + num_dp_ranks_of(cfg) if ep_scale_joiner_of(resolving_view(server_args)): port_base = server_args.port + ZMQ_TCP_PORT_DELTA if port_base + NUM_DERIVED_PORTS > 65535: diff --git a/python/sglang/srt/speculative/adaptive_spec_params.py b/python/sglang/srt/speculative/adaptive_spec_params.py index 6d3353499640..268ceaeafa77 100644 --- a/python/sglang/srt/speculative/adaptive_spec_params.py +++ b/python/sglang/srt/speculative/adaptive_spec_params.py @@ -16,6 +16,7 @@ resolved_view, resolving_view, ) +from sglang.srt.runtime_context import attn_dp_enabled_of from sglang.srt.utils import log_info_on_rank0 if TYPE_CHECKING: @@ -66,9 +67,9 @@ def adaptive_unsupported_reason(server_args: ServerArgs) -> str | None: f"speculative_eagle_topk={cfg.speculative_eagle_topk} " "(only topk=1 is supported)" ) - if resolved_view(server_args).enable_dp_attention: + if attn_dp_enabled_of(resolved_view(server_args)): return ( - "enable_dp_attention=True is not supported " + "attention DP is not supported " "(adaptive tier decisions are not synchronized across DP ranks)" ) if resolved_view(server_args).enable_multi_layer_eagle: diff --git a/python/sglang/srt/speculative/dflash_worker_v2.py b/python/sglang/srt/speculative/dflash_worker_v2.py index 50edb5bf5b59..c5b1cdabcd4e 100644 --- a/python/sglang/srt/speculative/dflash_worker_v2.py +++ b/python/sglang/srt/speculative/dflash_worker_v2.py @@ -400,14 +400,14 @@ def __init__( # spec broadcasts must stay within the attn-TP group. self._tp_sync = SpecTpSync( get_parallel().attn_tp_group - if get_parallel().enable_dp_attention + if get_parallel().attn_dp_enabled else get_parallel().tp_group ) # Under dp attention, the draft worker runs on the per-DP attn-TP # group, independent of idle peer DP ranks; it is built and run under # the same placement. - self.draft_owns_attention = get_parallel().enable_dp_attention + self.draft_owns_attention = get_parallel().attn_dp_enabled # Inside the draft scope the context answers the draft's narrowed rank. self._target_tp_rank = get_parallel().tp_rank with draft_pp_context(), draft_tp_context(self.draft_owns_attention): @@ -642,7 +642,7 @@ def init_cuda_graphs(self): "device graph capture.", type(current_platform).__name__, ) - if get_parallel().enable_dp_attention and capture_decode_cuda_graph: + if get_parallel().attn_dp_enabled and capture_decode_cuda_graph: # Idle DP ranks skip the draft step, so they cannot join a # shared graph capture/replay; keep the draft eager under dp # attention. @@ -1292,7 +1292,7 @@ def _cache_full_embed_weight(self) -> None: the full embedding once during init and keep it replicated, making the draft block-id lookup a collective-free on-device F.embedding. """ - if not get_parallel().enable_dp_attention: + if not get_parallel().attn_dp_enabled: return tp_group = get_parallel().tp_group @@ -2332,7 +2332,7 @@ def forward_batch_generation( # verify forward (IDLE mode) so its cross-DP collectives stay in # lockstep with the active DP group; the draft block's # collectives are within-rank and skipped. - if get_parallel().enable_dp_attention: + if get_parallel().attn_dp_enabled: idle_verify_input = DFlashVerifyInput( draft_token=torch.empty((0,), dtype=torch.long, device=self.device), positions=torch.empty((0,), dtype=torch.int64, device=self.device), @@ -2723,7 +2723,7 @@ def forward_batch_generation( # their DP-gather segment offsets disagree. Fall back to eager verify # whenever any DP rank is idle this round. if ( - get_parallel().enable_dp_attention + get_parallel().attn_dp_enabled and verify_forward_batch.original_global_num_tokens_cpu is not None and min(verify_forward_batch.original_global_num_tokens_cpu) == 0 ): @@ -2732,7 +2732,7 @@ def forward_batch_generation( # Mixed-round guard: an extend rank's raw token counts disagree with # the verify batch's spec-scaled counts on the DP-gather layout; run # eager, symmetric with the idle guard above. - if get_parallel().enable_dp_attention and batch.is_extend_in_batch: + if get_parallel().attn_dp_enabled and batch.is_extend_in_batch: verify_forward_batch.can_run_decode_cuda_graph = False target_out = self.target_worker.forward_batch_generation( diff --git a/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py b/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py index ec34bf5789a7..3470e7e32ef9 100644 --- a/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py +++ b/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py @@ -162,7 +162,7 @@ def __init__( self._draft_is_moe = draft_is_deepseek_v4() self._draft_dp_context_enabled = ( - get_parallel().enable_dp_attention and not self._draft_is_moe + get_parallel().attn_dp_enabled and not self._draft_is_moe ) self._is_pd_prefill = get_disagg().disaggregation_mode == "prefill" self._decode_graph_allowed = ( @@ -170,7 +170,7 @@ def __init__( and not self._is_pd_prefill ) if ( - get_parallel().enable_dp_attention + get_parallel().attn_dp_enabled and self._draft_is_moe and get_parallel().attn_tp_size > 1 ): @@ -229,9 +229,7 @@ def __init__( parallel = get_parallel() self._tp_sync = SpecTpSync( - parallel.attn_tp_group - if parallel.enable_dp_attention - else parallel.tp_group + parallel.attn_tp_group if parallel.attn_dp_enabled else parallel.tp_group ) self._draft_graph_group = ( parallel.attn_tp_group @@ -291,13 +289,13 @@ def __init__( tp_sync=self._tp_sync, ) if ( - get_parallel().enable_dp_attention + get_parallel().attn_dp_enabled and not self._draft_is_moe and self._verify_planner.is_compact_mode and self._decode_graph_allowed ): raise ValueError( - "DSpark dense-draft compact verify under --enable-dp-attention does not " + "DSpark dense-draft compact verify under attention DP does not " "yet support cuda graph (idle DP groups cannot join the token-keyed " "compact graph). Re-run with --disable-cuda-graph (eager is lossless), " "or use SGLANG_RAGGED_VERIFY_MODE=static. The dsv4 (MoE) draft supports " @@ -318,7 +316,7 @@ def __init__( mask_token_id=self._mask_token_id, draft_block_spec_info=self._draft_block_spec_info, tp_sync=self._tp_sync, - dp_moe_sync=self._draft_is_moe and get_parallel().enable_dp_attention, + dp_moe_sync=self._draft_is_moe and get_parallel().attn_dp_enabled, ) self._verify_epilogue = None target_is_dsv41 = ( @@ -333,7 +331,7 @@ def __init__( target_is_dsv41 and self._verify_planner.mode_value == "static" and self._draft_is_moe - and not get_parallel().enable_dp_attention + and not get_parallel().attn_dp_enabled and get_parallel().pp_size == 1 ) # ROCm (V4.1 target only): inside a HIP graph the accept-site TP broadcasts @@ -625,7 +623,7 @@ def _forward_prefill( pp_proxy_tensors: Optional[PPProxyTensors] = None, ) -> GenerationBatchResult: if batch.forward_mode.is_idle(): - if get_parallel().enable_dp_attention: + if get_parallel().attn_dp_enabled: self.target_worker.forward_batch_generation( batch, capture_hidden_mode=CaptureHiddenMode.FULL ) @@ -741,7 +739,7 @@ def _idle_verify_ragged_layout(self, batch: ScheduleBatch): def _dp_verify_tier_num_tokens(self, batch: ScheduleBatch) -> Optional[int]: if not ( self._draft_is_moe - and get_parallel().enable_dp_attention + and get_parallel().attn_dp_enabled and batch.global_num_tokens is not None and self._verify_planner.is_compact_mode ): @@ -790,7 +788,7 @@ def _forward_decode( global_num_reqs = ( max(batch.global_num_tokens) if self._draft_is_moe - and get_parallel().enable_dp_attention + and get_parallel().attn_dp_enabled and batch.global_num_tokens is not None else None ) @@ -806,7 +804,7 @@ def _forward_decode( if batch.forward_mode.is_idle(): self._observers.note_idle_decode_step() - if get_parallel().enable_dp_attention: + if get_parallel().attn_dp_enabled: if self._draft_is_moe: self._proposer.run_idle_participation(batch) if coordination_plan is not None: diff --git a/python/sglang/srt/speculative/eagle_worker_v2.py b/python/sglang/srt/speculative/eagle_worker_v2.py index 4a8973bffc9a..8da5d34fc1be 100644 --- a/python/sglang/srt/speculative/eagle_worker_v2.py +++ b/python/sglang/srt/speculative/eagle_worker_v2.py @@ -186,8 +186,7 @@ def __init__( # Use the same attention topology during draft construction and execution. self.draft_owns_attention = ( - get_parallel().enable_dp_attention - and self.speculative_algorithm.is_eagle3() + get_parallel().attn_dp_enabled and self.speculative_algorithm.is_eagle3() ) with ( draft_tp_context(self.draft_owns_attention), diff --git a/python/sglang/srt/speculative/multi_layer_eagle_draft_extend_cuda_graph_runner.py b/python/sglang/srt/speculative/multi_layer_eagle_draft_extend_cuda_graph_runner.py index 3965f2bee172..def5bcf94a1a 100644 --- a/python/sglang/srt/speculative/multi_layer_eagle_draft_extend_cuda_graph_runner.py +++ b/python/sglang/srt/speculative/multi_layer_eagle_draft_extend_cuda_graph_runner.py @@ -155,7 +155,7 @@ def __init__(self, eagle_worker: MultiLayerEagleDraftWorker, step: int): # Fields the parent's capture() reads: self.device = model_runner.device self.device_module = torch.get_device_module(self.device) - self.dp_size = get_parallel().dp_size + self.num_dp_ranks = get_parallel().num_dp_ranks self.pp_size = get_parallel().pp_size self.enable_torch_compile = get_flags().capture.enable_torch_compile self.disable_padding = get_exec().graph.disable_cuda_graph_padding @@ -283,8 +283,8 @@ def get_forward_batch(self, bs: int) -> ForwardBatch: next_token_logits_buffer = buffers.next_token_logits_buffer[:num_tokens] if self.require_mlp_tp_gather: - global_num_tokens_cpu = [num_tokens] * self.dp_size - global_num_tokens_for_logprob_cpu = [num_tokens] * self.dp_size + global_num_tokens_cpu = [num_tokens] * self.num_dp_ranks + global_num_tokens_for_logprob_cpu = [num_tokens] * self.num_dp_ranks elif self.require_attn_tp_gather: global_num_tokens_cpu = [num_tokens] # DRAFT_EXTEND_V2 produces logits for all tokens, not bs (see mlp branch above) @@ -678,10 +678,12 @@ def _allocate_buffers(self) -> MultiLayerEagleDraftExtendInputBuffers: if self.require_gathered_buffer: if self.require_mlp_tp_gather: - dp_size = runner.dp_size - global_num_tokens_gpu = torch.zeros((dp_size,), dtype=torch.int32) + num_dp_ranks = runner.num_dp_ranks + global_num_tokens_gpu = torch.zeros( + (num_dp_ranks,), dtype=torch.int32 + ) global_num_tokens_for_logprob_gpu = torch.zeros( - (dp_size,), dtype=torch.int32 + (num_dp_ranks,), dtype=torch.int32 ) else: global_num_tokens_gpu = torch.zeros((1,), dtype=torch.int32) diff --git a/python/sglang/srt/state_capturer/routed_experts.py b/python/sglang/srt/state_capturer/routed_experts.py index fab5c4d9207e..ef989b822ae2 100644 --- a/python/sglang/srt/state_capturer/routed_experts.py +++ b/python/sglang/srt/state_capturer/routed_experts.py @@ -71,14 +71,14 @@ def __init__( topk_size = model_config.hf_text_config.num_experts_per_tok num_layers = model_config.hf_text_config.num_hidden_layers - # Scale by dp_size so the buffer covers the full DP-concatenated batch. + # Scale by num_dp_ranks so the buffer covers the full DP-concatenated batch. # _get_local_slice indexes into [attention_dp_rank * cuda_graph_batch, ...) # and otherwise overflows on dp_rank > 0 when max_running_requests > # chunked_prefill_size. # FIXME: spec decoding's num_verify_tokens is still not accounted for. max_batch_size = max( - get_schedule().chunked_prefill_size * get_parallel().dp_size, - max_running_requests * get_parallel().dp_size, + get_schedule().chunked_prefill_size * get_parallel().num_dp_ranks, + max_running_requests * get_parallel().num_dp_ranks, ) super().__init__( diff --git a/python/sglang/srt/utils/common.py b/python/sglang/srt/utils/common.py index e2de7af75b41..cacf1fb243cf 100644 --- a/python/sglang/srt/utils/common.py +++ b/python/sglang/srt/utils/common.py @@ -3985,9 +3985,11 @@ def require_mlp_tp_gather(*, moe_a2a_backend=None): elif not isinstance(moe_a2a_backend, MoeA2ABackend): moe_a2a_backend = MoeA2ABackend(moe_a2a_backend) - # elastic-EP scale-up rewrites dp_size on the published config - if get_parallel().enable_dp_attention: - assert get_parallel().dp_size > 1, "dp_size must be greater than 1" + # elastic-EP scale-up widens num_dp_ranks on the published config + if get_parallel().attn_dp_enabled: + assert get_parallel().num_dp_ranks > 1, ( + "attention DP needs more than one DP rank" + ) if get_exec().moe.elastic_ep_backend is not None: from sglang.srt.elastic_ep.elastic_ep import ( elastic_expanded_world_enabled, @@ -4029,7 +4031,7 @@ def require_mlp_tp_gather(*, moe_a2a_backend=None): else: return ( get_parallel().moe_dense_tp_size - > get_parallel().tp_size // get_parallel().dp_size + > get_parallel().tp_size // get_parallel().num_dp_ranks ) else: return False @@ -4053,8 +4055,8 @@ def require_attn_tp_gather(): not get_moe_a2a_backend().is_none() or get_parallel().moe_dense_tp_size is not None ): - if get_parallel().enable_dp_attention: - return get_parallel().dp_size < get_parallel().tp_size + if get_parallel().attn_dp_enabled: + return get_parallel().num_dp_ranks < get_parallel().tp_size else: return True else: @@ -4067,7 +4069,7 @@ def require_gathered_buffer(): def require_mlp_sync(): - return get_parallel().enable_dp_attention or require_gathered_buffer() + return get_parallel().attn_dp_enabled or require_gathered_buffer() def get_cuda_graph_batch_size_alignment() -> int: diff --git a/python/sglang/srt/utils/cuda_vmm_transport_utils.py b/python/sglang/srt/utils/cuda_vmm_transport_utils.py index 67c5ee1a784a..e00d36cb6d37 100644 --- a/python/sglang/srt/utils/cuda_vmm_transport_utils.py +++ b/python/sglang/srt/utils/cuda_vmm_transport_utils.py @@ -192,8 +192,8 @@ def _contains_tensor_container(value) -> bool: def get_vmm_feature_consumer_count() -> int: - if get_parallel().enable_dp_attention: - return get_parallel().tp_size // get_parallel().dp_size + if get_parallel().attn_dp_enabled: + return get_parallel().tp_size // get_parallel().num_dp_ranks return get_parallel().tp_size diff --git a/python/sglang/srt/utils/offloader.py b/python/sglang/srt/utils/offloader.py index 100e116398a5..17bf994d92b3 100644 --- a/python/sglang/srt/utils/offloader.py +++ b/python/sglang/srt/utils/offloader.py @@ -210,7 +210,7 @@ def __init__( mode: str, ): parallel = get_parallel() - dp_rank, dp_size = parallel.dp_rank, parallel.dp_size + dp_rank, num_dp_ranks = parallel.dp_rank, parallel.num_dp_ranks self.group_size = group_size self.num_in_group = num_in_group self.prefetch_step = prefetch_step @@ -224,7 +224,7 @@ def __init__( set_naive_distributed( NaiveDistributed( rank=dp_rank, - world_size=dp_size, + world_size=num_dp_ranks, rendezvous=f"/tmp/{run_id}", ) ) diff --git a/python/sglang/srt/utils/profile_utils.py b/python/sglang/srt/utils/profile_utils.py index a7d6ab08f322..1e116da44378 100644 --- a/python/sglang/srt/utils/profile_utils.py +++ b/python/sglang/srt/utils/profile_utils.py @@ -357,7 +357,7 @@ def stop(self): filename_parts = [self.profile_id, f"TP-{get_parallel().tp_rank}"] # Only add other ranks if parallelism is enabled (size > 1) - if get_parallel().dp_size > 1: + if get_parallel().num_dp_ranks > 1: filename_parts.append(f"DP-{get_parallel().dp_rank}") if get_parallel().pp_size > 1: filename_parts.append(f"PP-{get_parallel().pp_rank}") diff --git a/python/sglang/srt/weight_cache/daemon.py b/python/sglang/srt/weight_cache/daemon.py index 2fe9a1648090..a40af383b857 100644 --- a/python/sglang/srt/weight_cache/daemon.py +++ b/python/sglang/srt/weight_cache/daemon.py @@ -162,10 +162,8 @@ def __init__( self.tp_rank = tp_rank self.pp_size = cfg.pp_size self.pp_rank = pp_rank - self.dp_size = cfg.dp_size self.ep_size = cfg.ep_size self.moe_dp_size = cfg.moe_dp_size - self.enable_dp_attention = cfg.enable_dp_attention self.enable_dp_lm_head = cfg.enable_dp_lm_head self.attn_cp_size = cfg.attn_cp_size self.moe_dense_tp_size = cfg.moe_dense_tp_size @@ -338,12 +336,12 @@ def load(self): tp_rank=self.tp_rank, pp_size=self.pp_size, pp_rank=self.pp_rank, - dp_size=self.dp_size, + dp_size=get_parallel().dp_size, ep_size=self.ep_size, moe_dp_size=self.moe_dp_size, moe_dp_rank=moe_dp_rank, moe_ep_rank=moe_ep_rank, - enable_dp_attention=self.enable_dp_attention, + attn_dp_size=get_parallel().attn_dp_size, enable_dp_lm_head=self.enable_dp_lm_head, attn_cp_size=self.attn_cp_size, moe_dense_tp_size=self.moe_dense_tp_size, diff --git a/python/sglang/srt/weight_cache/ipc_loader.py b/python/sglang/srt/weight_cache/ipc_loader.py index ef5a09bec0c3..6bf6a9305b4b 100644 --- a/python/sglang/srt/weight_cache/ipc_loader.py +++ b/python/sglang/srt/weight_cache/ipc_loader.py @@ -515,8 +515,6 @@ def _fetch_from_cache(self, model_config, device_config) -> Optional[dict]: moe_dp_rank = parallel.moe_dp_rank moe_ep_rank = parallel.moe_ep_rank - dp_size = get_parallel().dp_size - quant_method, quant_config = self._resolve_engine_quant(model_config) engine_config = CacheConfig( @@ -530,12 +528,12 @@ def _fetch_from_cache(self, model_config, device_config) -> Optional[dict]: tp_rank=tp_rank, pp_size=pp_size, pp_rank=pp_rank, - dp_size=dp_size, + dp_size=parallel.dp_size, ep_size=ep_size, moe_dp_size=moe_dp_size, moe_dp_rank=moe_dp_rank, moe_ep_rank=moe_ep_rank, - enable_dp_attention=parallel.enable_dp_attention, + attn_dp_size=parallel.attn_dp_size, enable_dp_lm_head=parallel.enable_dp_lm_head, attn_cp_size=parallel.attn_cp_size, moe_dense_tp_size=parallel.moe_dense_tp_size, diff --git a/python/sglang/srt/weight_cache/protocol.py b/python/sglang/srt/weight_cache/protocol.py index 1ae09eb40e1b..46cfe588d82a 100644 --- a/python/sglang/srt/weight_cache/protocol.py +++ b/python/sglang/srt/weight_cache/protocol.py @@ -39,7 +39,7 @@ class CacheConfig(msgspec.Struct): moe_dp_size: int moe_dp_rank: int moe_ep_rank: int - enable_dp_attention: bool + attn_dp_size: int enable_dp_lm_head: bool attn_cp_size: int moe_dense_tp_size: Optional[int] diff --git a/python/sglang/test/cache_consistency_jitter.py b/python/sglang/test/cache_consistency_jitter.py index 4289c93c47f7..eed04357a41a 100644 --- a/python/sglang/test/cache_consistency_jitter.py +++ b/python/sglang/test/cache_consistency_jitter.py @@ -190,7 +190,7 @@ def get_jitter_engine( engine_kwargs.setdefault("schedule_conservativeness", 0.05) if ( - engine_kwargs.get("enable_dp_attention") + engine_kwargs.get("attn_dp_size", 1) > 1 and "dist_init_addr" not in engine_kwargs ): import portpicker diff --git a/python/sglang/test/kits/attention_unittest/attention_methods/dense_attention.py b/python/sglang/test/kits/attention_unittest/attention_methods/dense_attention.py index e08dbb0d1adb..3b0f392c5b88 100644 --- a/python/sglang/test/kits/attention_unittest/attention_methods/dense_attention.py +++ b/python/sglang/test/kits/attention_unittest/attention_methods/dense_attention.py @@ -367,7 +367,6 @@ def __init__( dllm_algorithm=None, dllm_algorithm_config=None, dp_size=1, - enable_dp_attention=False, enable_deterministic_inference=False, enable_mis=False, is_embedding=False, diff --git a/python/sglang/test/kits/attention_unittest/attention_methods/dsa_attention.py b/python/sglang/test/kits/attention_unittest/attention_methods/dsa_attention.py index 0458f1f265e9..336c23698ac3 100644 --- a/python/sglang/test/kits/attention_unittest/attention_methods/dsa_attention.py +++ b/python/sglang/test/kits/attention_unittest/attention_methods/dsa_attention.py @@ -341,7 +341,6 @@ def __init__( dsa_prefill_backend=dsa_prefill_backend, device=device, enable_deterministic_inference=False, - enable_dp_attention=False, enable_prefill_cp=False, enable_mis=False, is_embedding=False, diff --git a/python/sglang/test/kits/attention_unittest/attention_methods/dsv4_attention.py b/python/sglang/test/kits/attention_unittest/attention_methods/dsv4_attention.py index e07aa633ffde..7ddef6953ed9 100644 --- a/python/sglang/test/kits/attention_unittest/attention_methods/dsv4_attention.py +++ b/python/sglang/test/kits/attention_unittest/attention_methods/dsv4_attention.py @@ -364,7 +364,6 @@ def __init__( disaggregation_mode=None, dp_size=1, enable_deterministic_inference=False, - enable_dp_attention=False, enable_mis=False, is_embedding=False, kv_cache_dtype="auto", diff --git a/python/sglang/test/kits/attention_unittest/attention_methods/mla_attention.py b/python/sglang/test/kits/attention_unittest/attention_methods/mla_attention.py index ae558bda2ea7..d535c3d0acfa 100644 --- a/python/sglang/test/kits/attention_unittest/attention_methods/mla_attention.py +++ b/python/sglang/test/kits/attention_unittest/attention_methods/mla_attention.py @@ -271,7 +271,6 @@ def __init__( dllm_algorithm=None, dllm_algorithm_config=None, dp_size=1, - enable_dp_attention=False, enable_deterministic_inference=False, enable_mis=False, flashinfer_mla_disable_ragged=True, diff --git a/python/sglang/test/kv_canary/consts.py b/python/sglang/test/kv_canary/consts.py index 3a16ca1cf069..3712e812afef 100644 --- a/python/sglang/test/kv_canary/consts.py +++ b/python/sglang/test/kv_canary/consts.py @@ -23,9 +23,8 @@ "--trust-remote-code", "--tp", "4", - "--dp", + "--attn-dp-size", "4", - "--enable-dp-attention", "--moe-a2a-backend", "deepep", "--cuda-graph-max-bs-decode", diff --git a/python/sglang/test/runners.py b/python/sglang/test/runners.py index 363daf9ddcf0..757880e21be1 100644 --- a/python/sglang/test/runners.py +++ b/python/sglang/test/runners.py @@ -581,7 +581,7 @@ def __init__( context_length: Optional[int] = None, max_total_tokens: Optional[int] = None, page_size: Optional[int] = None, - dp_size: int = 1, + attn_dp_size: int = 1, tokenizer_path: Optional[str] = None, mem_fraction_static: float = 0.65, trust_remote_code: bool = False, @@ -607,7 +607,6 @@ def __init__( ): self.model_type = model_type self.is_generation = model_type == "generation" - enable_dp_attention = dp_size > 1 spec_kwargs = {} if speculative_draft_model_path: @@ -645,8 +644,7 @@ def __init__( context_length=context_length, max_total_tokens=max_total_tokens, page_size=page_size, - enable_dp_attention=enable_dp_attention, - dp_size=dp_size, + attn_dp_size=attn_dp_size, tokenizer_path=tokenizer_path, disable_overlap_schedule=disable_overlap_schedule, cuda_graph_max_bs_decode=cuda_graph_max_bs_decode, diff --git a/python/sglang/test/server_fixtures/dsa_mtp_fixture.py b/python/sglang/test/server_fixtures/dsa_mtp_fixture.py index 0a958bc71c9b..693d98094640 100644 --- a/python/sglang/test/server_fixtures/dsa_mtp_fixture.py +++ b/python/sglang/test/server_fixtures/dsa_mtp_fixture.py @@ -3,7 +3,7 @@ Variants combine `DsaMtpServerBase` (server lifecycle) with `DsaMtpEvalConfigDefaults` (shared eval thresholds/params), `GSM8KMixin` and `SpecDecodingMixin`, then set `model` and per-variant -overrides (`enable_dp_attention`, `mem_fraction_static`, `bs_1_speed_thres`). +overrides (`attn_dp_size`, `mem_fraction_static`, `bs_1_speed_thres`). Example: class TestDsv32DP( @@ -13,7 +13,7 @@ class TestDsv32DP( SpecDecodingMixin, ): model = "deepseek-ai/DeepSeek-V3.2" - enable_dp_attention = True + attn_dp_size = 8 bs_1_speed_thres = 90 The base itself is NOT a runnable test (no `test_*` methods until a subclass @@ -50,9 +50,8 @@ class DsaMtpServerBase(CustomTestCase): # Subclasses must set `model`; the others have sensible defaults. model: str = "" tp_size: int = 8 - dp_size: int = 8 + attn_dp_size: int = 1 mem_fraction_static: float = 0.7 - enable_dp_attention: bool = False extra_server_args = () # EAGLE MTP config (fixed across DSA-MTP variants). @@ -65,8 +64,8 @@ class DsaMtpServerBase(CustomTestCase): def get_server_args(cls): assert cls.model, f"{cls.__name__} must set `model`" args = ["--trust-remote-code", "--tp", str(cls.tp_size)] - if cls.enable_dp_attention: - args += ["--dp", str(cls.dp_size), "--enable-dp-attention"] + if cls.attn_dp_size > 1: + args += ["--attn-dp-size", str(cls.attn_dp_size)] args += [ "--speculative-algorithm", cls.speculative_algorithm, diff --git a/scripts/ci/cuda/warmup_deep_gemm.py b/scripts/ci/cuda/warmup_deep_gemm.py index edf59679e2de..f53b6eff9a02 100644 --- a/scripts/ci/cuda/warmup_deep_gemm.py +++ b/scripts/ci/cuda/warmup_deep_gemm.py @@ -43,21 +43,19 @@ # cache key includes per-rank N/K (depends on tp/dp/ep) — must match each # model's `other_args` in test/registered/ or warmed shapes won't be hit. FALLBACK_ARGS: Dict[str, List[str]] = { - "deepseek-ai/DeepSeek-V3.2": ["--dp", "8", "--enable-dp-attention"], - "zai-org/GLM-5-FP8": ["--dp", "8", "--enable-dp-attention"], + "deepseek-ai/DeepSeek-V3.2": ["--attn-dp-size", "8"], + "zai-org/GLM-5-FP8": ["--attn-dp-size", "8"], "XiaomiMiMo/MiMo-V2-Flash": [ - "--dp", + "--attn-dp-size", "2", - "--enable-dp-attention", "--attention-backend", "fa3", ], # --mm-enable-dp-encoder is required: without it DP0 runs the vision # encoder alone and DP1 deadlocks at the next collective. "XiaomiMiMo/MiMo-V2.5": [ - "--dp", + "--attn-dp-size", "2", - "--enable-dp-attention", "--mm-enable-dp-encoder", "--attention-backend", "fa3", diff --git a/scripts/ci/slurm/launch_mi355x.sh b/scripts/ci/slurm/launch_mi355x.sh index 9a6488be60af..e9231d50cdaa 100755 --- a/scripts/ci/slurm/launch_mi355x.sh +++ b/scripts/ci/slurm/launch_mi355x.sh @@ -541,10 +541,10 @@ IONIC_EOF # and the generated argv is byte-identical. Oren's wide-EP recipes set decode # EP/DP=16 while prefill stays EP8. PREFILL_DPEP="" -(( PDP > 1 )) && PREFILL_DPEP="$PREFILL_DPEP --enable-dp-attention --dp-size $PDP" +(( PDP > 1 )) && PREFILL_DPEP="$PREFILL_DPEP --attn-dp-size $PDP" (( PEP > 1 )) && PREFILL_DPEP="$PREFILL_DPEP --ep-size $PEP" DECODE_DPEP="" -(( DDP > 1 )) && DECODE_DPEP="$DECODE_DPEP --enable-dp-attention --dp-size $DDP" +(( DDP > 1 )) && DECODE_DPEP="$DECODE_DPEP --attn-dp-size $DDP" (( DEP > 1 )) && DECODE_DPEP="$DECODE_DPEP --ep-size $DEP" # Flags shared by both roles (a2a backend, mtp). --max-total-tokens stays here for # non-wide recipes; wide recipes carry a per-role prefill_max_total via wide_ep. @@ -688,10 +688,10 @@ DENV_ARG=""; [[ -n "$DENV" ]] && DENV_ARG=" $DENV" # port and stays under the 32768 ephemeral floor. These recipes place at most # one single-node engine per node, so two engines cannot collide. # -# Pinning is safe here because --dp-size is only emitted together with -# --enable-dp-attention, and that path gives every DP rank the same nccl port. -# launch_dp_schedulers(), which needs a distinct port per worker, is never -# reached by these recipes. +# Pinning is safe here because these recipes only emit attention DP +# (--attn-dp-size), never --dp-size replicas, and the attention-DP path gives +# every DP rank the same nccl port. launch_dp_schedulers(), which needs a +# distinct port per worker, is never reached by these recipes. PNCCL_ARG="" DNCCL_ARG="" if [[ "$CLUSTER" == "spur" ]]; then diff --git a/scripts/ci/slurm/nightly-configs.yaml b/scripts/ci/slurm/nightly-configs.yaml index 8290206352aa..5149c42542df 100644 --- a/scripts/ci/slurm/nightly-configs.yaml +++ b/scripts/ci/slurm/nightly-configs.yaml @@ -125,7 +125,7 @@ dsv4pro-fp4-mi355x-sglang: # * -dp8ep8 : DP-attention 8 + narrow within-node EP8 # * -dp8ep8-mtp : DP8 + narrow EP8 + EAGLE MTP # launch_mi355x.sh reads expert-/data-parallel-size and the `mtp:` section to -# append --ep-size / --enable-dp-attention --dp-size / --speculative-* flags. +# append --ep-size / --attn-dp-size / --speculative-* flags. dsv4flash-fp8-mi355x-mtp-sglang: model: sgl-project/DeepSeek-V4-Flash-FP8 diff --git a/sgl-model-gateway/src/core/steps/worker/local/discover_dp.rs b/sgl-model-gateway/src/core/steps/worker/local/discover_dp.rs index 58d1a855b037..f7cc8147c474 100644 --- a/sgl-model-gateway/src/core/steps/worker/local/discover_dp.rs +++ b/sgl-model-gateway/src/core/steps/worker/local/discover_dp.rs @@ -19,7 +19,7 @@ pub async fn get_dp_info(url: &str, api_key: Option<&str>) -> Result, pub tp_size: Option, pub dp_size: Option, + pub attn_dp_size: Option, pub load_balance_method: Option, pub disaggregation_mode: Option, pub version: Option, @@ -43,6 +44,16 @@ pub struct ServerInfo { pub max_num_reqs: Option, } +impl ServerInfo { + /// How many DP ranks the server runs, one scheduler each: its replicas + /// times its attention-DP groups. A server that predates `attn_dp_size` + /// counts attention-DP groups in `dp_size` and reports no `attn_dp_size`. + pub fn num_dp_ranks(&self) -> Option { + self.dp_size + .map(|dp_size| dp_size * self.attn_dp_size.unwrap_or(1)) + } +} + /// Model information returned from /model_info endpoint. #[derive(Debug, Clone, Deserialize, Serialize)] pub struct ModelInfo { @@ -328,8 +339,8 @@ impl StepExecutor for DiscoverMetadataStep { if let Some(tp_size) = server_info.tp_size { labels.insert("tp_size".to_string(), tp_size.to_string()); } - if let Some(dp_size) = server_info.dp_size { - labels.insert("dp_size".to_string(), dp_size.to_string()); + if let Some(num_dp_ranks) = server_info.num_dp_ranks() { + labels.insert("dp_size".to_string(), num_dp_ranks.to_string()); } if let Some(load_balance_method) = server_info.load_balance_method { labels.insert("load_balance_method".to_string(), load_balance_method); @@ -400,3 +411,30 @@ impl StepExecutor for DiscoverMetadataStep { true } } + +#[cfg(test)] +mod server_info_tests { + use super::ServerInfo; + + fn parse(json: &str) -> ServerInfo { + serde_json::from_str(json).expect("valid server info") + } + + #[test] + fn dp_ranks_multiply_replicas_by_attention_dp_groups() { + let info = parse(r#"{"dp_size": 1, "attn_dp_size": 8}"#); + assert_eq!(info.num_dp_ranks(), Some(8)); + } + + #[test] + fn a_server_without_attention_dp_size_counts_dp_size() { + let info = parse(r#"{"dp_size": 4}"#); + assert_eq!(info.num_dp_ranks(), Some(4)); + } + + #[test] + fn no_dp_size_means_no_rank_count() { + let info = parse(r#"{"attn_dp_size": 2}"#); + assert_eq!(info.num_dp_ranks(), None); + } +} diff --git a/test/manual/8-gpu-models/test_dsa_models_basic.py b/test/manual/8-gpu-models/test_dsa_models_basic.py index 672d365c4949..e9aaa35b82db 100644 --- a/test/manual/8-gpu-models/test_dsa_models_basic.py +++ b/test/manual/8-gpu-models/test_dsa_models_basic.py @@ -26,9 +26,8 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "8", - "--dp", + "--attn-dp-size", "8", - "--enable-dp-attention", "--model-loader-extra-config", '{"enable_multithread_load": true, "num_threads": 64}', ] @@ -141,9 +140,8 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "8", - "--dp", + "--attn-dp-size", "8", - "--enable-dp-attention", "--model-loader-extra-config", '{"enable_multithread_load": true, "num_threads": 64}', ] diff --git a/test/manual/ascend/llm_models/test_npu_mimo_v2_5_pro_fp4_dflash.py b/test/manual/ascend/llm_models/test_npu_mimo_v2_5_pro_fp4_dflash.py index 0e1d9649154f..7eb153674e7f 100644 --- a/test/manual/ascend/llm_models/test_npu_mimo_v2_5_pro_fp4_dflash.py +++ b/test/manual/ascend/llm_models/test_npu_mimo_v2_5_pro_fp4_dflash.py @@ -36,9 +36,8 @@ class TestMiMoV25ProFP4GraphWithDFlash(GSM8KAscendMixin, CustomTestCase): MIMO_V2_5_PRO_FP4_DFLASH_DRAFT_WEIGHTS_PATH, "--speculative-num-draft-tokens", "8", - "--dp-size", + "--attn-dp-size", "2", - "--enable-dp-attention", "--enable-dp-lm-head", ] diff --git a/test/manual/ascend/llm_models/test_npu_mimo_v2_5_w8a8.py b/test/manual/ascend/llm_models/test_npu_mimo_v2_5_w8a8.py index e7108883176f..4e54ccf43593 100644 --- a/test/manual/ascend/llm_models/test_npu_mimo_v2_5_w8a8.py +++ b/test/manual/ascend/llm_models/test_npu_mimo_v2_5_w8a8.py @@ -38,9 +38,8 @@ class TestMiMoV25W8A8GraphWithMTP(GSM8KAscendMixin, CustomTestCase): "modelslim", "--speculative-draft-model-quantization", "unquant", - "--dp-size", + "--attn-dp-size", "2", - "--enable-dp-attention", "--enable-dp-lm-head", ] diff --git a/test/manual/ascend/test_ascend_deepseek_mtp.py b/test/manual/ascend/test_ascend_deepseek_mtp.py index 482426b375ab..85ac1a45348e 100644 --- a/test/manual/ascend/test_ascend_deepseek_mtp.py +++ b/test/manual/ascend/test_ascend_deepseek_mtp.py @@ -38,9 +38,8 @@ def setUpClass(cls): 32768, "--tp-size", 16, - "--dp-size", + "--attn-dp-size", 2, - "--enable-dp-attention", "--speculative-algorithm", "NEXTN", "--speculative-num-steps", diff --git a/test/manual/attention/test_trtllm_mla_backend.py b/test/manual/attention/test_trtllm_mla_backend.py index 193ba8919578..966122a0f6c7 100755 --- a/test/manual/attention/test_trtllm_mla_backend.py +++ b/test/manual/attention/test_trtllm_mla_backend.py @@ -327,7 +327,6 @@ class TestTRTLLMMLA(CustomTestCase): def setUpClass(cls): """Set up global server args for testing.""" server_args = ServerArgs(model_path="dummy") - server_args.enable_dp_attention = False set_global_server_args_for_scheduler(server_args) @classmethod diff --git a/test/manual/chunked_prefill/test_e2e_dp_attention.py b/test/manual/chunked_prefill/test_e2e_dp_attention.py index e5c018d27a2a..63391c266180 100644 --- a/test/manual/chunked_prefill/test_e2e_dp_attention.py +++ b/test/manual/chunked_prefill/test_e2e_dp_attention.py @@ -13,8 +13,7 @@ class TestChunkedFeatureDPAttention(ChunkedTestBase): "--trust-remote-code", "--tp", "2", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "2", ] diff --git a/test/manual/distributed/test_dp_attention_archived.py b/test/manual/distributed/test_dp_attention_archived.py index 40f7ff7b1028..1907cbe274e4 100644 --- a/test/manual/distributed/test_dp_attention_archived.py +++ b/test/manual/distributed/test_dp_attention_archived.py @@ -51,8 +51,7 @@ def setUpClass(cls): DEFAULT_MODEL_NAME_FOR_TEST_MLA_NEXTN, "--tp-size", "2", - "--enable-dp-attention", - "--dp-size", + "--attn-dp-size", "2", ] if not is_in_amd_ci(): diff --git a/test/manual/distributed/test_dp_attention_large.py b/test/manual/distributed/test_dp_attention_large.py index 7186a713334f..4dc7b55dba25 100644 --- a/test/manual/distributed/test_dp_attention_large.py +++ b/test/manual/distributed/test_dp_attention_large.py @@ -42,8 +42,7 @@ def setUpClass(cls): other_args=[ "--trust-remote-code", "--tp=4", - "--enable-dp-attention", - "--dp=2", + "--attn-dp-size=2", ], ) @@ -89,8 +88,7 @@ def setUpClass(cls): "--speculative-draft-model-path", DEFAULT_MODEL_NAME_FOR_TEST_MLA_NEXTN, "--tp-size=4", - "--enable-dp-attention", - "--dp-size=2", + "--attn-dp-size=2", ] if not is_in_amd_ci(): other_args += ["--mem-frac", "0.7"] @@ -151,8 +149,7 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "4", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "2", ], ) diff --git a/test/manual/dsv4/test_b200_flash.py b/test/manual/dsv4/test_b200_flash.py index 30df91e80608..81aa095067a0 100644 --- a/test/manual/dsv4/test_b200_flash.py +++ b/test/manual/dsv4/test_b200_flash.py @@ -43,9 +43,8 @@ class TestB200FlashBalanced(DSV4FlashAime25TestBase): "--trust-remote-code", "--tp", "4", - "--dp", + "--attn-dp-size", "4", - "--enable-dp-attention", "--moe-a2a-backend", "deepep", "--speculative-algorithm", @@ -68,9 +67,8 @@ class TestB200FlashMaxThroughput(DSV4FlashAime25TestBase): "--trust-remote-code", "--tp", "4", - "--dp", + "--attn-dp-size", "4", - "--enable-dp-attention", "--moe-a2a-backend", "deepep", "--deepep-config", diff --git a/test/manual/dsv4/test_b200_pro.py b/test/manual/dsv4/test_b200_pro.py index f95f29020b13..2d01932000b7 100644 --- a/test/manual/dsv4/test_b200_pro.py +++ b/test/manual/dsv4/test_b200_pro.py @@ -45,9 +45,8 @@ class TestB200ProBalanced(DSV4ProAime25TestBase): "--trust-remote-code", "--tp", "8", - "--dp", + "--attn-dp-size", "8", - "--enable-dp-attention", "--moe-a2a-backend", "deepep", "--speculative-algorithm", @@ -76,9 +75,8 @@ class TestB200ProMaxThroughput(DSV4ProAime25TestBase): "--trust-remote-code", "--tp", "8", - "--dp", + "--attn-dp-size", "8", - "--enable-dp-attention", "--moe-a2a-backend", "deepep", "--mem-fraction-static", diff --git a/test/manual/dsv4/test_b300_flash.py b/test/manual/dsv4/test_b300_flash.py index 422299718caa..3c2d31f7c291 100644 --- a/test/manual/dsv4/test_b300_flash.py +++ b/test/manual/dsv4/test_b300_flash.py @@ -45,9 +45,8 @@ class TestB300FlashBalanced(DSV4FlashAime25TestBase): "--trust-remote-code", "--tp", "4", - "--dp", + "--attn-dp-size", "4", - "--enable-dp-attention", "--moe-a2a-backend", "deepep", "--speculative-algorithm", @@ -70,9 +69,8 @@ class TestB300FlashMaxThroughput(DSV4FlashAime25TestBase): "--trust-remote-code", "--tp", "4", - "--dp", + "--attn-dp-size", "4", - "--enable-dp-attention", "--moe-a2a-backend", "deepep", "--deepep-config", diff --git a/test/manual/dsv4/test_b300_pro.py b/test/manual/dsv4/test_b300_pro.py index 0825e851055e..f992c997713a 100644 --- a/test/manual/dsv4/test_b300_pro.py +++ b/test/manual/dsv4/test_b300_pro.py @@ -47,9 +47,8 @@ class TestB300ProBalanced(DSV4ProAime25TestBase): "--trust-remote-code", "--tp", "8", - "--dp", + "--attn-dp-size", "8", - "--enable-dp-attention", "--moe-a2a-backend", "deepep", "--speculative-algorithm", @@ -78,9 +77,8 @@ class TestB300ProMaxThroughput(DSV4ProAime25TestBase): "--trust-remote-code", "--tp", "8", - "--dp", + "--attn-dp-size", "8", - "--enable-dp-attention", "--moe-a2a-backend", "deepep", "--mem-fraction-static", diff --git a/test/manual/dsv4/test_dsv4_flash_mtp_dp4.py b/test/manual/dsv4/test_dsv4_flash_mtp_dp4.py index 5274621b244c..7f9400cf32fc 100644 --- a/test/manual/dsv4/test_dsv4_flash_mtp_dp4.py +++ b/test/manual/dsv4/test_dsv4_flash_mtp_dp4.py @@ -61,9 +61,8 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "4", - "--dp", + "--attn-dp-size", "4", - "--enable-dp-attention", "--moe-a2a-backend", "deepep", "--cuda-graph-max-bs-decode", diff --git a/test/manual/dsv4/test_dsv4_flash_sanity_dp4.py b/test/manual/dsv4/test_dsv4_flash_sanity_dp4.py index 1546bbc541cc..64e5873bb83f 100644 --- a/test/manual/dsv4/test_dsv4_flash_sanity_dp4.py +++ b/test/manual/dsv4/test_dsv4_flash_sanity_dp4.py @@ -60,9 +60,8 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "4", - "--dp", + "--attn-dp-size", "4", - "--enable-dp-attention", "--moe-a2a-backend", "deepep", "--cuda-graph-max-bs-decode", @@ -98,7 +97,7 @@ def setUpClass(cls): "4", "--ep", "4", - # No --enable-dp-attention by design: covers TP-attn path. + # No --attn-dp-size by design: covers TP-attn path. "--moe-a2a-backend", "deepep", "--cuda-graph-max-bs-decode", @@ -132,9 +131,8 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "4", - "--dp", + "--attn-dp-size", "4", - "--enable-dp-attention", "--moe-a2a-backend", "deepep", "--chunked-prefill-size", diff --git a/test/manual/dsv4/test_dsv4_pd_disagg_nixl.py b/test/manual/dsv4/test_dsv4_pd_disagg_nixl.py index ac84120248fd..d10c701b0ea0 100644 --- a/test/manual/dsv4/test_dsv4_pd_disagg_nixl.py +++ b/test/manual/dsv4/test_dsv4_pd_disagg_nixl.py @@ -54,9 +54,8 @@ def start_prefill(cls): "0", "--tp", "4", - "--dp", + "--attn-dp-size", "4", - "--enable-dp-attention", "--moe-a2a-backend", "deepep", "--deepep-config", @@ -96,9 +95,8 @@ def start_decode(cls): "4", "--tp", "4", - "--dp", + "--attn-dp-size", "4", - "--enable-dp-attention", "--moe-a2a-backend", "deepep", "--deepep-config", diff --git a/test/manual/dsv4/test_dsv4_swa_radix_retract.py b/test/manual/dsv4/test_dsv4_swa_radix_retract.py index 7db064d58e7e..bdc8fa041325 100644 --- a/test/manual/dsv4/test_dsv4_swa_radix_retract.py +++ b/test/manual/dsv4/test_dsv4_swa_radix_retract.py @@ -64,9 +64,8 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "4", - "--dp", + "--attn-dp-size", "4", - "--enable-dp-attention", "--moe-a2a-backend", "deepep", "--cuda-graph-max-bs-decode", diff --git a/test/manual/dsv4/test_gb300_flash.py b/test/manual/dsv4/test_gb300_flash.py index e19abae2f276..b3459983fe3f 100644 --- a/test/manual/dsv4/test_gb300_flash.py +++ b/test/manual/dsv4/test_gb300_flash.py @@ -43,9 +43,8 @@ class TestGB300FlashBalanced(DSV4FlashAime25TestBase): "--trust-remote-code", "--tp", "4", - "--dp", + "--attn-dp-size", "4", - "--enable-dp-attention", "--moe-a2a-backend", "deepep", "--speculative-algorithm", @@ -68,9 +67,8 @@ class TestGB300FlashMaxThroughput(DSV4FlashAime25TestBase): "--trust-remote-code", "--tp", "4", - "--dp", + "--attn-dp-size", "4", - "--enable-dp-attention", "--moe-a2a-backend", "deepep", "--deepep-config", diff --git a/test/manual/dsv4/test_gb300_pro.py b/test/manual/dsv4/test_gb300_pro.py index 363296c477be..11edaaa7b9eb 100644 --- a/test/manual/dsv4/test_gb300_pro.py +++ b/test/manual/dsv4/test_gb300_pro.py @@ -47,9 +47,8 @@ class TestGB300ProBalanced(DSV4ProAime25TestBase): "--trust-remote-code", "--tp", "4", - "--dp", + "--attn-dp-size", "4", - "--enable-dp-attention", "--moe-a2a-backend", "deepep", "--speculative-algorithm", @@ -78,9 +77,8 @@ class TestGB300ProMaxThroughput(DSV4ProAime25TestBase): "--trust-remote-code", "--tp", "4", - "--dp", + "--attn-dp-size", "4", - "--enable-dp-attention", "--moe-a2a-backend", "deepep", "--mem-fraction-static", diff --git a/test/manual/dsv4/test_h200_fp8_flash.py b/test/manual/dsv4/test_h200_fp8_flash.py index 684272b9d0e5..7051f3276e3b 100644 --- a/test/manual/dsv4/test_h200_fp8_flash.py +++ b/test/manual/dsv4/test_h200_fp8_flash.py @@ -40,9 +40,8 @@ class TestH200Fp8FlashBalanced(DSV4FlashAime25TestBase): "--trust-remote-code", "--tp", "4", - "--dp", + "--attn-dp-size", "4", - "--enable-dp-attention", "--moe-a2a-backend", "deepep", "--speculative-algorithm", @@ -72,9 +71,8 @@ class TestH200Fp8FlashMaxThroughput(DSV4FlashAime25TestBase): "--trust-remote-code", "--tp", "4", - "--dp", + "--attn-dp-size", "4", - "--enable-dp-attention", "--moe-a2a-backend", "deepep", "--cuda-graph-max-bs-decode", diff --git a/test/manual/dsv4/test_h200_fp8_pro.py b/test/manual/dsv4/test_h200_fp8_pro.py index 3e724a4b7100..ee4b5f5044a9 100644 --- a/test/manual/dsv4/test_h200_fp8_pro.py +++ b/test/manual/dsv4/test_h200_fp8_pro.py @@ -38,9 +38,8 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "16", - "--dp", + "--attn-dp-size", "16", - "--enable-dp-attention", *multinode_args(2), "--moe-a2a-backend", "deepep", @@ -72,9 +71,8 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "16", - "--dp", + "--attn-dp-size", "16", - "--enable-dp-attention", *multinode_args(2), "--moe-a2a-backend", "deepep", @@ -106,9 +104,8 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "16", - "--dp", + "--attn-dp-size", "16", - "--enable-dp-attention", *multinode_args(2), "--moe-a2a-backend", "deepep", diff --git a/test/manual/ep/test_elastic_recover.py b/test/manual/ep/test_elastic_recover.py index beab45b92e08..4cfefda5e7e3 100644 --- a/test/manual/ep/test_elastic_recover.py +++ b/test/manual/ep/test_elastic_recover.py @@ -69,7 +69,7 @@ def _server_args(node_rank: int, port: int, recover: bool = False) -> list[str]: "--trust-remote-code", "--tp", str(EP_SIZE), - "--dp", + "--attn-dp-size", str(EP_SIZE), "--nnodes", "2", @@ -79,7 +79,6 @@ def _server_args(node_rank: int, port: int, recover: bool = False) -> list[str]: DIST_INIT_ADDR, "--random-seed", str(RANDOM_SEED), - "--enable-dp-attention", "--enable-dp-lm-head", "--elastic-ep-backend", "mooncake", diff --git a/test/manual/ep/test_elastic_scale.py b/test/manual/ep/test_elastic_scale.py index 092e639bf9f4..8cc30557d1b5 100644 --- a/test/manual/ep/test_elastic_scale.py +++ b/test/manual/ep/test_elastic_scale.py @@ -136,9 +136,8 @@ def _scale_up_common_args( "low_latency", "--tp", str(tp_size), - "--dp", + "--attn-dp-size", str(tp_size), - "--enable-dp-attention", "--enable-dp-lm-head", "--elastic-ep-backend", "mooncake", diff --git a/test/manual/ep/test_eplb.py b/test/manual/ep/test_eplb.py index f0c5cce2e4aa..cce1a29e992a 100755 --- a/test/manual/ep/test_eplb.py +++ b/test/manual/ep/test_eplb.py @@ -60,9 +60,8 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "2", - "--dp", + "--attn-dp-size", "2", - "--enable-dp-attention", *get_a2a_backend_args(), "--disable-cuda-graph", "--enable-eplb", @@ -117,11 +116,10 @@ def test_save_expert_distribution_and_init_expert_location(self): model_path=DEFAULT_MLA_MODEL_NAME_FOR_TEST, trust_remote_code=True, ep_num_redundant_experts=4, - enable_dp_attention=True, disable_cuda_graph=True, expert_distribution_recorder_mode="stat", tp_size=2, - dp_size=2, + attn_dp_size=2, log_level="info", ) engine_kwargs.update(get_a2a_backend_kwargs()) diff --git a/test/manual/ep/test_eplb_mori.py b/test/manual/ep/test_eplb_mori.py index 5710e4d99564..88c78e976d3e 100644 --- a/test/manual/ep/test_eplb_mori.py +++ b/test/manual/ep/test_eplb_mori.py @@ -48,9 +48,8 @@ def wait_all_ports_release(base_url, timeout_s=60): "8", "--ep-size", "8", - "--dp-size", + "--attn-dp-size", "8", - "--enable-dp-attention", "--moe-a2a-backend", "mori", "--trust-remote-code", diff --git a/test/manual/ep/test_moe_deepep.py b/test/manual/ep/test_moe_deepep.py index ce7074b81718..d2304aeef50a 100644 --- a/test/manual/ep/test_moe_deepep.py +++ b/test/manual/ep/test_moe_deepep.py @@ -64,9 +64,8 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "2", - "--dp", + "--attn-dp-size", "2", - "--enable-dp-attention", "--moe-a2a-backend", "deepep", "--deepep-mode", diff --git a/test/manual/ep/test_mooncake_expert_backup.py b/test/manual/ep/test_mooncake_expert_backup.py index 5298ed31e44b..820adea46352 100644 --- a/test/manual/ep/test_mooncake_expert_backup.py +++ b/test/manual/ep/test_mooncake_expert_backup.py @@ -36,8 +36,7 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "4", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "4", "--elastic-ep-backend", "mooncake", diff --git a/test/manual/ep/test_nixl_ep.py b/test/manual/ep/test_nixl_ep.py index 6341fe9bc38a..81a97cf39bba 100644 --- a/test/manual/ep/test_nixl_ep.py +++ b/test/manual/ep/test_nixl_ep.py @@ -33,7 +33,7 @@ "--mem-fraction-static", "0.78", ] -DP_ATTN = ["--dp", "8", "--enable-dp-attention"] +DP_ATTN = ["--attn-dp-size", "8"] ELASTIC_NIXL = [ "--elastic-ep-backend", "nixl", diff --git a/test/manual/ep/test_pplx_small.py b/test/manual/ep/test_pplx_small.py index 3e4c3c4143e9..ecf14a70d0b2 100644 --- a/test/manual/ep/test_pplx_small.py +++ b/test/manual/ep/test_pplx_small.py @@ -28,8 +28,7 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "4", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "4", "--moe-a2a-backend", "pplx", @@ -43,7 +42,7 @@ def setUpClass(cls): "0.5", ], # Per-rank dispatch cap must cover the per-rank prefill chunk - # (chunked_prefill_size // dp_size = 8192 // 4 = 2048 on H100). + # (chunked_prefill_size // attn_dp_size = 8192 // 4 = 2048 on H100). env={"SGLANG_PPLX_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "4096"}, ) @@ -99,8 +98,7 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "4", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "2", "--moe-a2a-backend", "pplx", @@ -114,7 +112,7 @@ def setUpClass(cls): "0.5", ], # Per-rank dispatch cap must cover the per-rank prefill chunk - # (chunked_prefill_size // dp_size = 8192 // 2 = 4096 on H100). + # (chunked_prefill_size // attn_dp_size = 8192 // 2 = 4096 on H100). env={"SGLANG_PPLX_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "4096"}, ) diff --git a/test/manual/hicache/test_npu_kimi_k3_hicache.py b/test/manual/hicache/test_npu_kimi_k3_hicache.py index 444a96e06216..86c221cc858c 100644 --- a/test/manual/hicache/test_npu_kimi_k3_hicache.py +++ b/test/manual/hicache/test_npu_kimi_k3_hicache.py @@ -28,8 +28,7 @@ class TestKimiK3MixedWithHiCacheL2(GSM8KAscendMixin, CustomTestCase): "bfloat16", "--tp-size", "64", - "--enable-dp-attention", - "--dp-size", + "--attn-dp-size", "4", "--enable-dp-lm-head", "--moe-a2a-backend", diff --git a/test/manual/nightly/test_deepseek_v32_perf.py b/test/manual/nightly/test_deepseek_v32_perf.py index bb64468c97c8..85f9ee1d01b1 100644 --- a/test/manual/nightly/test_deepseek_v32_perf.py +++ b/test/manual/nightly/test_deepseek_v32_perf.py @@ -24,9 +24,8 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "8", - "--dp", + "--attn-dp-size", "8", - "--enable-dp-attention", "--model-loader-extra-config", '{"enable_multithread_load": true}', ], @@ -37,9 +36,8 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "8", - "--dp", + "--attn-dp-size", "8", - "--enable-dp-attention", "--speculative-algorithm", "EAGLE", "--speculative-num-steps", @@ -60,9 +58,8 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "8", - "--dp", + "--attn-dp-size", "8", - "--enable-dp-attention", "--attention-backend", "dsa", "--dsa-prefill-backend", diff --git a/test/manual/quant/test_deepseek_v32_fp4_4gpu.py b/test/manual/quant/test_deepseek_v32_fp4_4gpu.py index 773029805a1a..d56542bda4cd 100644 --- a/test/manual/quant/test_deepseek_v32_fp4_4gpu.py +++ b/test/manual/quant/test_deepseek_v32_fp4_4gpu.py @@ -24,9 +24,8 @@ def setUpClass(cls): other_args = [ "--tp", "4", - "--dp", + "--attn-dp-size", "4", - "--enable-dp-attention", "--moe-runner-backend", "flashinfer_trtllm", "--quantization", diff --git a/test/manual/test_cross_node_scheduler_info_sync.py b/test/manual/test_cross_node_scheduler_info_sync.py index f6e5a835fb73..2704f91b5db3 100755 --- a/test/manual/test_cross_node_scheduler_info_sync.py +++ b/test/manual/test_cross_node_scheduler_info_sync.py @@ -44,7 +44,7 @@ class TestCase: name: str tp_size: int - dp_size: int + attn_dp_size: int nnodes: int gpus_per_node: int expected_ranks: int @@ -55,7 +55,7 @@ class TestCase: "tp4_nodes2": TestCase( name="tp4_nodes2", tp_size=4, - dp_size=1, + attn_dp_size=1, nnodes=2, gpus_per_node=2, expected_ranks=4, @@ -64,20 +64,20 @@ class TestCase: "dp2_single_node": TestCase( name="dp2_single_node", tp_size=2, - dp_size=2, + attn_dp_size=2, nnodes=1, gpus_per_node=2, expected_ranks=2, - extra_args=["--enable-dp-attention", "--dp", "2", "--attention-backend", "fa3"], + extra_args=["--attn-dp-size", "2", "--attention-backend", "fa3"], ), "dp2_tp2_nodes2": TestCase( name="dp2_tp2_nodes2", tp_size=4, - dp_size=2, + attn_dp_size=2, nnodes=2, gpus_per_node=2, expected_ranks=4, - extra_args=["--enable-dp-attention", "--dp", "2", "--attention-backend", "fa3"], + extra_args=["--attn-dp-size", "2", "--attention-backend", "fa3"], ), } diff --git a/test/manual/test_kv_events.py b/test/manual/test_kv_events.py index 01607eb6257c..55c5e05783dc 100644 --- a/test/manual/test_kv_events.py +++ b/test/manual/test_kv_events.py @@ -48,9 +48,6 @@ def test_kv_events_enabled(self): 32, "--cuda-graph-max-bs-decode", 2, - "--enable-dp-attention", - "--dp-size", - 1, ], ) @@ -201,8 +198,7 @@ def test_kv_events_attn_dp(self): 64, "--cuda-graph-max-bs-decode", 4, - "--enable-dp-attention", - "--dp-size", + "--attn-dp-size", 2, "--tp-size", 2, diff --git a/test/manual/test_ray_engine.py b/test/manual/test_ray_engine.py index e53f343bca04..cd98f0428794 100644 --- a/test/manual/test_ray_engine.py +++ b/test/manual/test_ray_engine.py @@ -101,12 +101,8 @@ def shutdown(self): self.engine.shutdown() self.engine = None - enable_dp_attention = (extra_kwargs or {}).get("enable_dp_attention", False) - if enable_dp_attention: - # DP attention folds DP into TP — total GPUs = tp_size * pp_size - total_gpus = tp_size * pp_size - else: - total_gpus = dp_size * tp_size * pp_size + # Attention DP runs inside the TP group, so only replicas add GPUs. + total_gpus = dp_size * tp_size * pp_size pg = placement_group( [{"CPU": 1, "GPU": total_gpus}], strategy="STRICT_PACK", @@ -292,14 +288,14 @@ def test_batch_generate_dp2(self): # --------------------------------------------------------------------------- -# Tests: Offline DP Attention (dp=2, tp=2) +# Tests: Offline DP Attention (attn_dp_size=2, tp=2) # --------------------------------------------------------------------------- @unittest.skipUnless(_has_ray, "ray is not installed") @unittest.skipUnless(_NUM_GPUS >= 2, "requires at least 2 GPUs") class TestRayEngineOfflineDPAttention(unittest.TestCase): - """Test Ray engine with dp_size=2, tp_size=2, enable_dp_attention=True.""" + """Test Ray engine with attn_dp_size=2, tp_size=2.""" @classmethod def setUpClass(cls): @@ -307,10 +303,9 @@ def setUpClass(cls): ray.init(log_to_driver=True, runtime_env=_RAY_RUNTIME_ENV) cls.actor, cls.pg = _create_engine_on_pg( tp_size=2, - dp_size=2, model=_DP_ATTN_MODEL, extra_kwargs={ - "enable_dp_attention": True, + "attn_dp_size": 2, "disable_cuda_graph": True, "port": 31500, }, @@ -630,7 +625,7 @@ def test_bundle_indices_skip_bundle(self): del os.environ["SGLANG_RAY_BUNDLE_INDICES"] def test_custom_pg_dp_attention(self): - """Test custom placement_group with enable_dp_attention=True.""" + """Test custom placement_group with tp_size=2 on the DP-attention model.""" from sglang.srt.ray.engine import RayEngine pg = placement_group([{"GPU": 1}] * 2, strategy="STRICT_PACK") @@ -641,7 +636,6 @@ def test_custom_pg_dp_attention(self): tp_size=2, placement_group=pg, use_ray=True, - enable_dp_attention=True, ) result = engine.generate("The capital of France is", _SAMPLING_PARAMS) diff --git a/test/manual/test_two_batch_overlap.py b/test/manual/test_two_batch_overlap.py index 2786aa6f9055..5ea72ccdd1e8 100644 --- a/test/manual/test_two_batch_overlap.py +++ b/test/manual/test_two_batch_overlap.py @@ -34,9 +34,8 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "2", - "--dp", + "--attn-dp-size", "2", - "--enable-dp-attention", "--moe-a2a-backend", "deepep", "--deepep-mode", @@ -135,9 +134,8 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "2", - "--dp", + "--attn-dp-size", "2", - "--enable-dp-attention", "--moe-a2a-backend", "deepep", "--deepep-mode", diff --git a/test/manual/test_w4a8_deepseek_v3.py b/test/manual/test_w4a8_deepseek_v3.py index e005679854c6..096f075783e2 100644 --- a/test/manual/test_w4a8_deepseek_v3.py +++ b/test/manual/test_w4a8_deepseek_v3.py @@ -135,9 +135,8 @@ def setUpClass(cls): "deepep", "--deepep-mode", "normal", - "--dp", + "--attn-dp-size", "8", - "--enable-dp-attention", "--moe-runner-backend", "cutlass", ] @@ -191,9 +190,8 @@ def setUpClass(cls): "auto", "--deepep-dispatcher-output-dtype", "bf16", - "--dp", + "--attn-dp-size", "8", - "--enable-dp-attention", "--moe-runner-backend", "cutlass", "--speculative-algorithm", diff --git a/test/registered/amd/accuracy/mi30x/test_deepseek_r1_eval_amd.py b/test/registered/amd/accuracy/mi30x/test_deepseek_r1_eval_amd.py index d2740823cda1..a335b8479b6e 100644 --- a/test/registered/amd/accuracy/mi30x/test_deepseek_r1_eval_amd.py +++ b/test/registered/amd/accuracy/mi30x/test_deepseek_r1_eval_amd.py @@ -114,9 +114,8 @@ def get_display_name(self) -> str: other_args=[ "--chunked-prefill-size", "131072", - "--dp-size", + "--attn-dp-size", "8", - "--enable-dp-attention", "--mem-fraction-static", "0.85", "--trust-remote-code", diff --git a/test/registered/amd/accuracy/mi30x/test_deepseek_v32_dp_eval_amd.py b/test/registered/amd/accuracy/mi30x/test_deepseek_v32_dp_eval_amd.py index ab1f9b7a14fd..179d194961ac 100644 --- a/test/registered/amd/accuracy/mi30x/test_deepseek_v32_dp_eval_amd.py +++ b/test/registered/amd/accuracy/mi30x/test_deepseek_v32_dp_eval_amd.py @@ -50,9 +50,8 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "8", - "--dp", + "--attn-dp-size", "8", - "--enable-dp-attention", "--attention-backend", "aiter", "--chunked-prefill-size", diff --git a/test/registered/amd/accuracy/mi35x/test_deepseek_v32_dp_eval_mi35x.py b/test/registered/amd/accuracy/mi35x/test_deepseek_v32_dp_eval_mi35x.py index 2ca6033f9a22..fd45a55b261c 100644 --- a/test/registered/amd/accuracy/mi35x/test_deepseek_v32_dp_eval_mi35x.py +++ b/test/registered/amd/accuracy/mi35x/test_deepseek_v32_dp_eval_mi35x.py @@ -49,9 +49,8 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "8", - "--dp", + "--attn-dp-size", "8", - "--enable-dp-attention", "--model-loader-extra-config", '{"enable_multithread_load": true}', "--dsa-prefill-backend", diff --git a/test/registered/amd/test_deepseek_v32_basic.py b/test/registered/amd/test_deepseek_v32_basic.py index 71610f4ecf79..9c01179a218c 100644 --- a/test/registered/amd/test_deepseek_v32_basic.py +++ b/test/registered/amd/test_deepseek_v32_basic.py @@ -30,9 +30,8 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "8", - "--dp", + "--attn-dp-size", "8", - "--enable-dp-attention", "--model-loader-extra-config", '{"enable_multithread_load": true}', ] diff --git a/test/registered/amd/test_deepseek_v32_mtp.py b/test/registered/amd/test_deepseek_v32_mtp.py index e3a8f4fed08f..74cf614ac761 100644 --- a/test/registered/amd/test_deepseek_v32_mtp.py +++ b/test/registered/amd/test_deepseek_v32_mtp.py @@ -36,9 +36,8 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "8", - "--dp", + "--attn-dp-size", "8", - "--enable-dp-attention", "--speculative-algorithm", "EAGLE", "--speculative-num-steps", diff --git a/test/registered/amd/test_deepseek_v4_flash_fp8_tbo.py b/test/registered/amd/test_deepseek_v4_flash_fp8_tbo.py index 331566ff97df..88c8a24de7a1 100644 --- a/test/registered/amd/test_deepseek_v4_flash_fp8_tbo.py +++ b/test/registered/amd/test_deepseek_v4_flash_fp8_tbo.py @@ -6,7 +6,7 @@ TBO here is the DP-attention TP-MoE variant (moe_a2a_backend='none'): it overlaps one micro-batch's DP all_gatherv (pre-MoE gather) + reduce_scatterv (post-MoE combine) with the other micro-batch's attention + expert compute (prefill only). -Enabled purely via `--enable-dp-attention` + `--enable-two-batch-overlap` (no opt-in +Enabled purely via `--attn-dp-size` + `--enable-two-batch-overlap` (no opt-in env). This test guards that TBO does not regress GSM8K accuracy and that the DP TBO server launches + runs to completion (exercises op_gather/op_moe/op_combine and the event+ref combine-buffer lifetime that fixed the reserved-memory OOM at mem0.9). @@ -93,9 +93,8 @@ def setUpClass(cls): "8", # DP attention + TBO: non-EP DP TP-MoE two-batch-overlap. DP TBO is # selected because moe_a2a_backend stays 'none'; no opt-in env needed. - "--dp", + "--attn-dp-size", "8", - "--enable-dp-attention", "--enable-prefill-delayer", "--enable-two-batch-overlap", "--disable-radix-cache", @@ -113,7 +112,7 @@ def setUpClass(cls): "0.90", "--swa-full-tokens-ratio", "0.15", - # global chunk; DP-attention divides by dp_size=8 -> 8192/rank. + # global chunk; DP-attention divides by attn_dp_size=8 -> 8192/rank. "--chunked-prefill-size", "65536", "--disable-shared-experts-fusion", diff --git a/test/registered/amd/test_deepseek_v4_pro_fp4_dspark.py b/test/registered/amd/test_deepseek_v4_pro_fp4_dspark.py index cfc73385c64e..2077891b24f2 100644 --- a/test/registered/amd/test_deepseek_v4_pro_fp4_dspark.py +++ b/test/registered/amd/test_deepseek_v4_pro_fp4_dspark.py @@ -183,9 +183,8 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "8", - "--dp", + "--attn-dp-size", "8", - "--enable-dp-attention", "--enable-dp-lm-head", "--enable-prefill-delayer", "--disable-radix-cache", diff --git a/test/registered/amd/test_deepseek_v4_pro_fp4_tbo.py b/test/registered/amd/test_deepseek_v4_pro_fp4_tbo.py index 458f70f2503f..797ede0ba2ae 100644 --- a/test/registered/amd/test_deepseek_v4_pro_fp4_tbo.py +++ b/test/registered/amd/test_deepseek_v4_pro_fp4_tbo.py @@ -6,7 +6,7 @@ TBO here is the DP-attention TP-MoE variant (moe_a2a_backend='none'): it overlaps one micro-batch's DP all_gatherv (pre-MoE gather) + reduce_scatterv (post-MoE combine) with the other micro-batch's attention + expert compute (prefill only). -Enabled purely via `--enable-dp-attention` + `--enable-two-batch-overlap` (no opt-in +Enabled purely via `--attn-dp-size` + `--enable-two-batch-overlap` (no opt-in env). This test guards that TBO does not regress GSM8K accuracy and that the DP TBO server launches + runs (exercises op_gather/op_moe/op_combine and the event+ref combine-buffer lifetime that fixed the reserved-memory OOM at mem0.9). @@ -83,9 +83,8 @@ def setUpClass(cls): "8", # DP attention + TBO: non-EP DP TP-MoE two-batch-overlap. DP TBO is # selected because moe_a2a_backend stays 'none'; no opt-in env needed. - "--dp", + "--attn-dp-size", "8", - "--enable-dp-attention", "--enable-prefill-delayer", "--enable-two-batch-overlap", "--disable-radix-cache", @@ -103,7 +102,7 @@ def setUpClass(cls): "0.90", "--swa-full-tokens-ratio", "0.15", - # global chunk; DP-attention divides by dp_size=8 -> 8192/rank. + # global chunk; DP-attention divides by attn_dp_size=8 -> 8192/rank. "--chunked-prefill-size", "65536", "--disable-shared-experts-fusion", diff --git a/test/registered/amd/test_deepseek_v4_pro_fp4_tbo_mtp.py b/test/registered/amd/test_deepseek_v4_pro_fp4_tbo_mtp.py index 97876b7699e8..ad82b2938336 100644 --- a/test/registered/amd/test_deepseek_v4_pro_fp4_tbo_mtp.py +++ b/test/registered/amd/test_deepseek_v4_pro_fp4_tbo_mtp.py @@ -70,9 +70,8 @@ def setUpClass(cls): "8", # DP attention + TBO: non-EP DP TP-MoE two-batch-overlap. DP TBO is # selected because moe_a2a_backend stays 'none'; no opt-in env needed. - "--dp", + "--attn-dp-size", "8", - "--enable-dp-attention", "--enable-prefill-delayer", "--enable-two-batch-overlap", "--disable-radix-cache", @@ -101,7 +100,7 @@ def setUpClass(cls): "0.90", "--swa-full-tokens-ratio", "0.15", - # global chunk; DP-attention divides by dp_size=8 -> 8192/rank. + # global chunk; DP-attention divides by attn_dp_size=8 -> 8192/rank. "--chunked-prefill-size", "65536", "--disable-shared-experts-fusion", diff --git a/test/registered/amd/test_moriep_small.py b/test/registered/amd/test_moriep_small.py index 5f407552d929..ab7ec35a47c5 100644 --- a/test/registered/amd/test_moriep_small.py +++ b/test/registered/amd/test_moriep_small.py @@ -49,9 +49,8 @@ def wait_all_ports_release(base_url, timeout_s=60): "8", "--ep-size", "8", - "--dp-size", + "--attn-dp-size", "8", - "--enable-dp-attention", "--moe-a2a-backend", "mori", "--trust-remote-code", diff --git a/test/registered/backends/test_flashinfer_nvfp4_online_moe_backend.py b/test/registered/backends/test_flashinfer_nvfp4_online_moe_backend.py index aadfefb40a1c..f6bb9d0459f7 100644 --- a/test/registered/backends/test_flashinfer_nvfp4_online_moe_backend.py +++ b/test/registered/backends/test_flashinfer_nvfp4_online_moe_backend.py @@ -142,9 +142,8 @@ class TestFlashinferCuteDSLMoeBackendNvFp4OnlineW4A16( "1", "--speculative-num-draft-tokens", "4", - "--dp-size", + "--attn-dp-size", "4", - "--enable-dp-attention", "--enable-dp-lm-head", "--moe-a2a-backend", "flashinfer", diff --git a/test/registered/backends/test_flashinfer_trtllm_gen_moe_backend.py b/test/registered/backends/test_flashinfer_trtllm_gen_moe_backend.py index 13a10ed3c3b2..b89d74e96855 100644 --- a/test/registered/backends/test_flashinfer_trtllm_gen_moe_backend.py +++ b/test/registered/backends/test_flashinfer_trtllm_gen_moe_backend.py @@ -170,8 +170,7 @@ def setUpClass(cls): other_args=[ "--quantization", "mxfp8", - "--enable-dp-attention", - "--dp-size", + "--attn-dp-size", "4", "--tp-size", "4", diff --git a/test/registered/cp/test_deepseek_v4_flash_fp4_b200_cp.py b/test/registered/cp/test_deepseek_v4_flash_fp4_b200_cp.py index 812053168789..34a008401e23 100644 --- a/test/registered/cp/test_deepseek_v4_flash_fp4_b200_cp.py +++ b/test/registered/cp/test_deepseek_v4_flash_fp4_b200_cp.py @@ -68,7 +68,6 @@ def setUpClass(cls): "4", "--attn-cp-size", "4", - "--enable-dp-attention", "--moe-a2a-backend", "deepep", "--speculative-algorithm", @@ -119,7 +118,6 @@ def setUpClass(cls): "4", "--attn-cp-size", "4", - "--enable-dp-attention", "--moe-a2a-backend", "megamoe", "--enable-w4a4-mxfp4-megamoe", diff --git a/test/registered/cpu/test_intel_amx_attention_backend_a.py b/test/registered/cpu/test_intel_amx_attention_backend_a.py index 23a27c8269d3..bb8671f23355 100644 --- a/test/registered/cpu/test_intel_amx_attention_backend_a.py +++ b/test/registered/cpu/test_intel_amx_attention_backend_a.py @@ -82,8 +82,7 @@ def setUpClass(cls): "--disable-overlap-schedule", "--tp", "2", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "2", ] cls.process = popen_launch_server( diff --git a/test/registered/debug_utils/comparator/test_entrypoint.py b/test/registered/debug_utils/comparator/test_entrypoint.py index 422c38cd1409..8a034b6c6687 100644 --- a/test/registered/debug_utils/comparator/test_entrypoint.py +++ b/test/registered/debug_utils/comparator/test_entrypoint.py @@ -4550,7 +4550,7 @@ def _sglang_dp_attn_parallel_info(*, tp_rank: int) -> dict: "moe_tp_size": 2, "moe_dp_rank": 0, "moe_dp_size": 1, - "enable_dp_attention": True, + "attn_dp_enabled": True, "attn_tp_rank": 0, "attn_tp_size": 1, "attn_dp_rank": tp_rank, diff --git a/test/registered/debug_utils/test_dumper.py b/test/registered/debug_utils/test_dumper.py index 9423c0877045..14c65a453278 100644 --- a/test/registered/debug_utils/test_dumper.py +++ b/test/registered/debug_utils/test_dumper.py @@ -2423,7 +2423,7 @@ def test_step_and_non_intrusive_hooks(self, tmp_path): "moe_tp_size", "moe_dp_rank", "moe_dp_size", - "enable_dp_attention", + "attn_dp_enabled", "attn_tp_rank", "attn_tp_size", "attn_dp_rank", diff --git a/test/registered/debug_utils/test_engine_dumper_comparator_e2e.py b/test/registered/debug_utils/test_engine_dumper_comparator_e2e.py index 7615d095740c..510a0d678228 100644 --- a/test/registered/debug_utils/test_engine_dumper_comparator_e2e.py +++ b/test/registered/debug_utils/test_engine_dumper_comparator_e2e.py @@ -197,7 +197,7 @@ def test_dp_attention(self, tmp_path: Path) -> None: _run_e2e_scenario( tmp_path=tmp_path, target_tp=BASELINE_TP, - extra_target_server_args=["--dp", "2", "--enable-dp-attention"], + extra_target_server_args=["--attn-dp-size", "2"], target_patch_config_yaml=PATCH_CONFIG_DP_ATTENTION_YAML, extra_comparator_args=[ "--token-aligner", diff --git a/test/registered/disaggregation/test_disaggregation_dp_attention.py b/test/registered/disaggregation/test_disaggregation_dp_attention.py index 50a8a4f1c516..69d80b3150e3 100644 --- a/test/registered/disaggregation/test_disaggregation_dp_attention.py +++ b/test/registered/disaggregation/test_disaggregation_dp_attention.py @@ -54,9 +54,8 @@ def start_prefill(cls): cls.bootstrap_port, "--tp", str(cls.PREFILL_DP_SIZE), - "--dp", + "--attn-dp-size", str(cls.PREFILL_DP_SIZE), - "--enable-dp-attention", "--load-balance-method", cls.LOAD_BALANCE_METHOD, ] @@ -80,9 +79,8 @@ def start_decode(cls): cls.bootstrap_port, "--tp", str(cls.DECODE_DP_SIZE), - "--dp", + "--attn-dp-size", str(cls.DECODE_DP_SIZE), - "--enable-dp-attention", "--base-gpu-id", str(cls.PREFILL_DP_SIZE), "--load-balance-method", diff --git a/test/registered/disaggregation/test_disaggregation_dsv4.py b/test/registered/disaggregation/test_disaggregation_dsv4.py index 4d5722f549fe..aa5caa7db38c 100644 --- a/test/registered/disaggregation/test_disaggregation_dsv4.py +++ b/test/registered/disaggregation/test_disaggregation_dsv4.py @@ -66,9 +66,8 @@ def start_prefill(cls): cls.prefill_nccl_port, "--tp", 4, - "--dp", + "--attn-dp-size", 4, - "--enable-dp-attention", "--moe-a2a-backend", "deepep", "--deepep-config", @@ -102,9 +101,8 @@ def start_decode(cls): cls.decode_nccl_port, "--tp", 4, - "--dp", + "--attn-dp-size", 4, - "--enable-dp-attention", "--base-gpu-id", 4, "--moe-a2a-backend", diff --git a/test/registered/disaggregation/test_disaggregation_dwdp_gpt_oss.py b/test/registered/disaggregation/test_disaggregation_dwdp_gpt_oss.py index 4ea69b8ea63d..6b12303fc2c3 100644 --- a/test/registered/disaggregation/test_disaggregation_dwdp_gpt_oss.py +++ b/test/registered/disaggregation/test_disaggregation_dwdp_gpt_oss.py @@ -35,9 +35,8 @@ class TestDisaggregationDWDPGptOss(PDDisaggregationServerBase): "0.85", ] extra_decode_args = [ - "--dp", + "--attn-dp-size", str(NUM_DECODE_GPUS), - "--enable-dp-attention", "--disable-flashinfer-autotune", "--mem-fraction-static", "0.85", diff --git a/test/registered/disaggregation/test_disaggregation_hybrid_attention.py b/test/registered/disaggregation/test_disaggregation_hybrid_attention.py index 3a47277b7678..bc1a4b190d58 100644 --- a/test/registered/disaggregation/test_disaggregation_hybrid_attention.py +++ b/test/registered/disaggregation/test_disaggregation_hybrid_attention.py @@ -210,9 +210,8 @@ def start_decode(cls): cls.bootstrap_port, "--tp", "2", - "--dp", + "--attn-dp-size", "2", - "--enable-dp-attention", "--enable-dp-lm-head", "--base-gpu-id", "2", diff --git a/test/registered/dp_attn/test_dp_attention.py b/test/registered/dp_attn/test_dp_attention.py index 7e7ea2b6f669..4dbfeb59d3e8 100644 --- a/test/registered/dp_attn/test_dp_attention.py +++ b/test/registered/dp_attn/test_dp_attention.py @@ -54,8 +54,7 @@ def setUpClass(cls): "4", "--tp", "2", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "2", ], ) @@ -72,7 +71,7 @@ class TestDPAttentionGatherv( ): """Exercise the variable-length all_gatherv + reduce_scatterv DP-MoE path (SGLANG_DP_USE_GATHERV=1). The path only activates for the - attn_tp_size == 1, tp_size == dp_size layout, which tp2 + dp2 satisfies. + attn_tp_size == 1, tp_size == attn_dp_size layout, which tp2 + attn_dp2 satisfies. Without this test the gatherv/reduce_scatterv code is never exercised by CI (it is gated behind the env var, default off). gsm8k must stay correct since the change is a pure communication reorg, not a numerics change.""" @@ -92,8 +91,7 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "2", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "2", "--chunked-prefill-size", "256", @@ -124,8 +122,7 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "2", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "2", "--enable-mixed-chunk", "--chunked-prefill-size", @@ -157,8 +154,7 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "2", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "2", "--max-total-tokens", "4500", @@ -194,8 +190,7 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "2", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "2", ], ) diff --git a/test/registered/dp_attn/test_dp_attention_bcg_kl.py b/test/registered/dp_attn/test_dp_attention_bcg_kl.py index 0b561874646e..b16e39894970 100644 --- a/test/registered/dp_attn/test_dp_attention_bcg_kl.py +++ b/test/registered/dp_attn/test_dp_attention_bcg_kl.py @@ -220,9 +220,8 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "2", - "--dp", + "--attn-dp-size", "2", - "--enable-dp-attention", "--enable-deterministic-inference", "--attention-backend", cls.attention_backend, @@ -287,7 +286,7 @@ def _wait_for_prefill_graph_replay( def test_prefill_and_decode_cache_hit_kl_is_zero(self): server_info = requests.get(self.base_url + "/server_info", timeout=30).json() self.assertFalse(server_info["disable_radix_cache"]) - self.assertTrue(server_info["enable_dp_attention"]) + self.assertEqual(server_info["attn_dp_size"], 2) self.assertTrue(server_info["enable_deterministic_inference"]) self.assertEqual(server_info["attention_backend"], self.attention_backend) self.assertEqual( diff --git a/test/registered/e2e/dp_attn/test_dp_attention_local_control_broadcast.py b/test/registered/e2e/dp_attn/test_dp_attention_local_control_broadcast.py index 556d986d3272..43f058f5acba 100644 --- a/test/registered/e2e/dp_attn/test_dp_attention_local_control_broadcast.py +++ b/test/registered/e2e/dp_attn/test_dp_attention_local_control_broadcast.py @@ -49,9 +49,8 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "4", - "--dp", + "--attn-dp-size", "2", - "--enable-dp-attention", "--enable-dp-attention-local-control-broadcast", ], ) diff --git a/test/registered/e2e/models/test_deepseek_v4_flash_fp4_b200.py b/test/registered/e2e/models/test_deepseek_v4_flash_fp4_b200.py index 4958d2f2c314..742ecfdff803 100644 --- a/test/registered/e2e/models/test_deepseek_v4_flash_fp4_b200.py +++ b/test/registered/e2e/models/test_deepseek_v4_flash_fp4_b200.py @@ -133,9 +133,8 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "4", - "--dp", + "--attn-dp-size", "4", - "--enable-dp-attention", "--moe-a2a-backend", "deepep", "--deepep-config", @@ -180,9 +179,8 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "4", - "--dp", + "--attn-dp-size", "4", - "--enable-dp-attention", "--enable-mixed-chunk", "--cuda-graph-backend-prefill", "breakable", diff --git a/test/registered/e2e/models/test_deepseek_v4_flash_fp4_b200_trtllm.py b/test/registered/e2e/models/test_deepseek_v4_flash_fp4_b200_trtllm.py index 80173c0e51b4..ffa620c11328 100644 --- a/test/registered/e2e/models/test_deepseek_v4_flash_fp4_b200_trtllm.py +++ b/test/registered/e2e/models/test_deepseek_v4_flash_fp4_b200_trtllm.py @@ -122,9 +122,8 @@ def setUpClass(cls): "trtllm", "--tp", "4", - "--dp", + "--attn-dp-size", "4", - "--enable-dp-attention", "--enable-mixed-chunk", "--cuda-graph-backend-prefill", "breakable", diff --git a/test/registered/e2e/models/test_deepseek_v4_flash_fp4_megamoe_b200.py b/test/registered/e2e/models/test_deepseek_v4_flash_fp4_megamoe_b200.py index ac3f47d8718e..0e565b7478cf 100644 --- a/test/registered/e2e/models/test_deepseek_v4_flash_fp4_megamoe_b200.py +++ b/test/registered/e2e/models/test_deepseek_v4_flash_fp4_megamoe_b200.py @@ -61,9 +61,8 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "4", - "--dp", + "--attn-dp-size", "4", - "--enable-dp-attention", "--enable-dp-lm-head", "--moe-a2a-backend", "megamoe", @@ -103,9 +102,8 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "4", - "--dp", + "--attn-dp-size", "4", - "--enable-dp-attention", "--enable-dp-lm-head", "--moe-a2a-backend", "megamoe", diff --git a/test/registered/e2e/models/test_deepseek_v4_flash_fp8_h200.py b/test/registered/e2e/models/test_deepseek_v4_flash_fp8_h200.py index 4c956f2457ca..3e0b5a03e085 100644 --- a/test/registered/e2e/models/test_deepseek_v4_flash_fp8_h200.py +++ b/test/registered/e2e/models/test_deepseek_v4_flash_fp8_h200.py @@ -53,9 +53,8 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "4", - "--dp", + "--attn-dp-size", "4", - "--enable-dp-attention", "--moe-a2a-backend", "deepep", "--speculative-algorithm", @@ -114,9 +113,8 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "4", - "--dp", + "--attn-dp-size", "4", - "--enable-dp-attention", "--moe-a2a-backend", "deepep", "--moe-runner-backend", @@ -141,10 +139,11 @@ def setUpClass(cls): env={ "SGLANG_DSV4_FP4_EXPERTS": "0", # INVARIANT: this per-rank cap MUST equal - # chunked_prefill_size / dp_size (= 8192 / 4 = 2048), the per-rank - # prefill bound under --enable-dp-attention. If you change - # --chunked-prefill-size or --dp above, update this to match — - # otherwise MegaMoE falls back whenever a prefill chunk exceeds it. + # chunked_prefill_size / attn_dp_size (= 8192 / 4 = 2048), the + # per-rank prefill bound under attention DP. If you change + # --chunked-prefill-size or --attn-dp-size above, update this + # to match — otherwise MegaMoE falls back whenever a prefill + # chunk exceeds it. "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "2048", }, ) diff --git a/test/registered/e2e/models/test_dsa_glm52_dp_mtp.py b/test/registered/e2e/models/test_dsa_glm52_dp_mtp.py index 85d8a0c8f9b6..cb8bf2f2b3be 100644 --- a/test/registered/e2e/models/test_dsa_glm52_dp_mtp.py +++ b/test/registered/e2e/models/test_dsa_glm52_dp_mtp.py @@ -20,7 +20,7 @@ class TestGLM52DPMTP( ): model = "zai-org/GLM-5.2-FP8" mem_fraction_static = 0.88 - enable_dp_attention = True + attn_dp_size = 8 bs_1_speed_thres = 70 diff --git a/test/registered/e2e/models/test_dsa_glm52_hisparse.py b/test/registered/e2e/models/test_dsa_glm52_hisparse.py index 5cfdd34dd7ba..bd5add5e6b39 100644 --- a/test/registered/e2e/models/test_dsa_glm52_hisparse.py +++ b/test/registered/e2e/models/test_dsa_glm52_hisparse.py @@ -24,9 +24,8 @@ class TestGLM52HiSparse(DefaultServerBase, GSM8KMixin): "--trust-remote-code", "--tp", "8", - "--dp", + "--attn-dp-size", "8", - "--enable-dp-attention", "--page-size", "64", "--max-running-requests", diff --git a/test/registered/e2e/models/test_dsa_glm52_nvfp4_dp_mtp.py b/test/registered/e2e/models/test_dsa_glm52_nvfp4_dp_mtp.py index db33bda82a94..be03a67492d7 100644 --- a/test/registered/e2e/models/test_dsa_glm52_nvfp4_dp_mtp.py +++ b/test/registered/e2e/models/test_dsa_glm52_nvfp4_dp_mtp.py @@ -20,9 +20,8 @@ class TestGLM52NVFP4DPMTP( ): model = "nvidia/GLM-5.2-NVFP4" tp_size = 4 - dp_size = 4 + attn_dp_size = 4 mem_fraction_static = 0.88 - enable_dp_attention = True bs_1_speed_thres = 180 gsm8k_accuracy_thres = 0.92 extra_server_args = [ diff --git a/test/registered/e2e/models/test_dsa_glm52_pd_mtp_cp_layersplit.py b/test/registered/e2e/models/test_dsa_glm52_pd_mtp_cp_layersplit.py index 37611054b128..c4b5ddb91ebf 100644 --- a/test/registered/e2e/models/test_dsa_glm52_pd_mtp_cp_layersplit.py +++ b/test/registered/e2e/models/test_dsa_glm52_pd_mtp_cp_layersplit.py @@ -107,9 +107,8 @@ class TestGLM52DSACacheLayerSplitMixedMoE(TestGLM52DSACacheLayerSplit): gsm8k_num_threads = 32 extra_decode_args = TestGLM52DSACacheLayerSplit.extra_decode_args + [ - "--dp-size", + "--attn-dp-size", "4", - "--enable-dp-attention", "--enable-dp-lm-head", "--ep-size", "4", diff --git a/test/registered/e2e/models/test_glm53_flash_b200.py b/test/registered/e2e/models/test_glm53_flash_b200.py index 8dd646ec225b..4e93d154ffc2 100644 --- a/test/registered/e2e/models/test_glm53_flash_b200.py +++ b/test/registered/e2e/models/test_glm53_flash_b200.py @@ -105,8 +105,7 @@ class TestGLM53FlashB200HighThroughput( gsm8k_num_shots = 20 server_args = [ *COMMON_SERVER_ARGS, - "--enable-dp-attention", - "--dp-size", + "--attn-dp-size", "4", "--cuda-graph-backend-prefill", "breakable", diff --git a/test/registered/e2e/models/test_glm53_flash_h200.py b/test/registered/e2e/models/test_glm53_flash_h200.py index 1349e8910b88..1f9e0747fac3 100644 --- a/test/registered/e2e/models/test_glm53_flash_h200.py +++ b/test/registered/e2e/models/test_glm53_flash_h200.py @@ -104,8 +104,7 @@ class TestGLM53FlashH200HighThroughput( gsm8k_num_shots = 20 server_args = [ *COMMON_SERVER_ARGS, - "--enable-dp-attention", - "--dp-size", + "--attn-dp-size", "8", "--mm-enable-dp-encoder", ] diff --git a/test/registered/e2e/models/test_gpt_oss_4gpu_mxfp4.py b/test/registered/e2e/models/test_gpt_oss_4gpu_mxfp4.py index 8ffe7dda982c..5582276896c3 100644 --- a/test/registered/e2e/models/test_gpt_oss_4gpu_mxfp4.py +++ b/test/registered/e2e/models/test_gpt_oss_4gpu_mxfp4.py @@ -32,11 +32,10 @@ def test_mxfp4_120b_dpa(self): other_args=[ "--tp", "4", - "--dp", + "--attn-dp-size", "4", "--ep", "4", - "--enable-dp-attention", "--moe-a2a-backend", "none", "--cuda-graph-max-bs-decode", diff --git a/test/registered/e2e/models/test_mimo_v2.py b/test/registered/e2e/models/test_mimo_v2.py index 2e3ae99f1107..46dca802d32c 100644 --- a/test/registered/e2e/models/test_mimo_v2.py +++ b/test/registered/e2e/models/test_mimo_v2.py @@ -10,9 +10,8 @@ MIMO_V2_OTHER_ARGS = [ "--tp", "8", - "--dp", + "--attn-dp-size", "2", - "--enable-dp-attention", "--mm-enable-dp-encoder", "--attention-backend", "fa3", diff --git a/test/registered/e2e/models/test_mimo_v2_flash.py b/test/registered/e2e/models/test_mimo_v2_flash.py index a4e0704abdd6..cd823a300077 100644 --- a/test/registered/e2e/models/test_mimo_v2_flash.py +++ b/test/registered/e2e/models/test_mimo_v2_flash.py @@ -16,9 +16,8 @@ class TestMiMoV2Flash(GSM8KMixin, SpecDecodingMixin, DefaultServerBase): other_args = [ "--tp", "4", - "--dp", + "--attn-dp-size", "2", - "--enable-dp-attention", "--trust-remote-code", "--attention-backend", "fa4" if is_blackwell() else "fa3", diff --git a/test/registered/e2e/models_large/test_deepseek_v32_indexcache.py b/test/registered/e2e/models_large/test_deepseek_v32_indexcache.py index 8e6365fec840..94f7004eb05a 100644 --- a/test/registered/e2e/models_large/test_deepseek_v32_indexcache.py +++ b/test/registered/e2e/models_large/test_deepseek_v32_indexcache.py @@ -27,7 +27,6 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", "--model-loader-extra-config", '{"enable_multithread_load": true, "num_threads": 64}', "--json-model-override-args", diff --git a/test/registered/e2e/models_large/test_deepseek_v3_cutedsl_4gpu.py b/test/registered/e2e/models_large/test_deepseek_v3_cutedsl_4gpu.py index 26905799a8d3..7b4ea6f7dc0e 100644 --- a/test/registered/e2e/models_large/test_deepseek_v3_cutedsl_4gpu.py +++ b/test/registered/e2e/models_large/test_deepseek_v3_cutedsl_4gpu.py @@ -39,13 +39,12 @@ def setUpClass(cls): "1024", "--tp", "4", - "--dp", + "--attn-dp-size", "4", "--ep", "4", "--moe-dense-tp-size", "1", - "--enable-dp-attention", "--nccl-port", str(NCCL_PORT_BASE), "--quantization", diff --git a/test/registered/e2e/models_large/test_glm52_fp8.py b/test/registered/e2e/models_large/test_glm52_fp8.py index 6130fc4e75d5..b551d4af2f58 100644 --- a/test/registered/e2e/models_large/test_glm52_fp8.py +++ b/test/registered/e2e/models_large/test_glm52_fp8.py @@ -32,7 +32,7 @@ class TestGlm52Fp8(unittest.TestCase): """GLM-5.2 FP8 on H200/B200 (8-GPU, tp=8).""" def test_glm52_fp8(self): - dp_args = ["--dp=8", "--enable-dp-attention"] + dp_args = ["--attn-dp-size=8"] variants = [ ModelLaunchSettings( diff --git a/test/registered/e2e/models_large/test_kimi_k25.py b/test/registered/e2e/models_large/test_kimi_k25.py index e3b6b114dc26..439b66b96ba4 100644 --- a/test/registered/e2e/models_large/test_kimi_k25.py +++ b/test/registered/e2e/models_large/test_kimi_k25.py @@ -31,8 +31,7 @@ def test_kimi_k25(self): ] dp_attn_args = [ - "--dp=8", - "--enable-dp-attention", + "--attn-dp-size=8", ] variants = [ diff --git a/test/registered/e2e/models_large/test_minimax_m25.py b/test/registered/e2e/models_large/test_minimax_m25.py index 900a71b77d0b..fbcda5b318be 100644 --- a/test/registered/e2e/models_large/test_minimax_m25.py +++ b/test/registered/e2e/models_large/test_minimax_m25.py @@ -31,8 +31,7 @@ def test_minimax_m25(self): "--reasoning-parser=minimax-append-think", ] dp_attn_args = base_args + [ - "--enable-dp-attention", - "--dp=8", + "--attn-dp-size=8", ] variants = [ diff --git a/test/registered/e2e/models_large/test_nvidia_nemotron_3_super_nvfp4.py b/test/registered/e2e/models_large/test_nvidia_nemotron_3_super_nvfp4.py index 257b62b2f141..43f5a878f9e2 100644 --- a/test/registered/e2e/models_large/test_nvidia_nemotron_3_super_nvfp4.py +++ b/test/registered/e2e/models_large/test_nvidia_nemotron_3_super_nvfp4.py @@ -30,9 +30,8 @@ ] DP_ATTENTION_EP_ARGS = [ - "--dp-size", + "--attn-dp-size", "4", - "--enable-dp-attention", "--enable-dp-lm-head", "--ep-size", "4", diff --git a/test/registered/e2e/models_large/test_qwen35.py b/test/registered/e2e/models_large/test_qwen35.py index 24a222859663..bc7178c058ab 100644 --- a/test/registered/e2e/models_large/test_qwen35.py +++ b/test/registered/e2e/models_large/test_qwen35.py @@ -31,7 +31,7 @@ def test_qwen35(self): "--tool-call-parser=qwen3_coder", "--mem-fraction-static=0.8", ] - dp_args = ["--dp=8", "--enable-dp-attention"] + dp_args = ["--attn-dp-size=8"] mtp_args = [ "--speculative-algorithm=EAGLE", "--speculative-num-steps=3", diff --git a/test/registered/e2e/pp/test_pp_spec.py b/test/registered/e2e/pp/test_pp_spec.py index 10e37461e694..2d6bac6222e1 100644 --- a/test/registered/e2e/pp/test_pp_spec.py +++ b/test/registered/e2e/pp/test_pp_spec.py @@ -198,9 +198,7 @@ def test_gate_on_rejects_unsupported_combinations(self): # DP attention partitions the batch per DP rank, so the stages # would no longer rebuild the same verify tree. with self.assertRaises(AssertionError): - self._server_args( - tp_size=2, dp_size=2, enable_dp_attention=True - ).check_server_args() + self._server_args(tp_size=2, attn_dp_size=2).check_server_args() # Adaptive spec changes num_draft_tokens at runtime, which the # relay slices results with. with self.assertRaises(AssertionError): diff --git a/test/registered/ep/test_deepep_large.py b/test/registered/ep/test_deepep_large.py index 5acfafa7709f..051b24fb8299 100644 --- a/test/registered/ep/test_deepep_large.py +++ b/test/registered/ep/test_deepep_large.py @@ -34,8 +34,7 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "8", "--moe-dense-tp-size", "1", @@ -108,9 +107,8 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "8", - "--dp", + "--attn-dp-size", "8", - "--enable-dp-attention", "--enable-two-batch-overlap", "--moe-a2a-backend", "deepep", diff --git a/test/registered/ep/test_deepep_small.py b/test/registered/ep/test_deepep_small.py index cb526f8298a1..2031624df46a 100644 --- a/test/registered/ep/test_deepep_small.py +++ b/test/registered/ep/test_deepep_small.py @@ -32,8 +32,7 @@ def setUpClass(cls): "--trust-remote-code", "--tp-size", "4", - "--enable-dp-attention", - "--dp-size", + "--attn-dp-size", "4", "--moe-a2a-backend", "deepep", @@ -121,8 +120,7 @@ def setUpClass(cls): "--trust-remote-code", "--tp-size", "4", - "--enable-dp-attention", - "--dp-size", + "--attn-dp-size", "4", "--moe-dense-tp-size", "1", @@ -173,8 +171,7 @@ def setUpClass(cls): other_args=[ "--tp-size", "4", - "--enable-dp-attention", - "--dp-size", + "--attn-dp-size", "4", "--enable-two-batch-overlap", "--moe-a2a-backend", diff --git a/test/registered/ep/test_deepep_small_extra.py b/test/registered/ep/test_deepep_small_extra.py index 3b0a2c6b4a37..e5c4d3fe5a73 100644 --- a/test/registered/ep/test_deepep_small_extra.py +++ b/test/registered/ep/test_deepep_small_extra.py @@ -29,8 +29,7 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "4", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "2", "--moe-a2a-backend", "deepep", diff --git a/test/registered/ep/test_eplb_no_a2a.py b/test/registered/ep/test_eplb_no_a2a.py index ea3b3850895b..2f451901f348 100644 --- a/test/registered/ep/test_eplb_no_a2a.py +++ b/test/registered/ep/test_eplb_no_a2a.py @@ -99,8 +99,7 @@ class TestEPLBNoA2ADPAttention(TestEPLBNoA2A): the expert-weight migration.""" extra_args = [ - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "2", ] rebalance_num_iterations = "50" diff --git a/test/registered/ep/test_flashinfer_a2a.py b/test/registered/ep/test_flashinfer_a2a.py index a8bd95cd7ee1..379cbe00efd3 100644 --- a/test/registered/ep/test_flashinfer_a2a.py +++ b/test/registered/ep/test_flashinfer_a2a.py @@ -43,9 +43,8 @@ def setUpClass(cls): "4", "--ep", "4", - "--dp", + "--attn-dp-size", "4", - "--enable-dp-attention", "--nccl-port", str(NCCL_PORT_BASE), "--moe-a2a-backend", @@ -96,9 +95,8 @@ def setUpClass(cls): "4", "--ep", "4", - "--dp", + "--attn-dp-size", "4", - "--enable-dp-attention", "--nccl-port", str(NCCL_PORT_BASE + 1), "--moe-a2a-backend", @@ -157,9 +155,8 @@ def setUpClass(cls): "4", "--ep", "4", - "--dp", + "--attn-dp-size", "4", - "--enable-dp-attention", "--nccl-port", str(NCCL_PORT_BASE + 2), "--moe-a2a-backend", diff --git a/test/registered/ep/test_mooncake_ep_small.py b/test/registered/ep/test_mooncake_ep_small.py index 26a6916f9c5d..46e7670244e5 100644 --- a/test/registered/ep/test_mooncake_ep_small.py +++ b/test/registered/ep/test_mooncake_ep_small.py @@ -93,8 +93,7 @@ def test_gsm8k(self): class TestPureDP(TestTP): extra_args = [ - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "4", ] @@ -135,8 +134,7 @@ def test_gsm8k_fault_2(self): @unittest.skipIf(is_in_ci(), "To reduce the CI execution time.") class TestHybridDPTP(TestPureDP): extra_args = [ - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "2", ] diff --git a/test/registered/ep/test_routed_experts_dp_readback.py b/test/registered/ep/test_routed_experts_dp_readback.py index c8a99053243b..d946ab4a135a 100644 --- a/test/registered/ep/test_routed_experts_dp_readback.py +++ b/test/registered/ep/test_routed_experts_dp_readback.py @@ -71,11 +71,10 @@ def setUpClass(cls): ), "--tp", "2", - "--dp", + "--attn-dp-size", "2", "--ep", "2", - "--enable-dp-attention", "--enable-return-routed-experts", "--disable-cuda-graph", "--disable-radix-cache", diff --git a/test/registered/ep/test_tbo_shared_experts_fusion.py b/test/registered/ep/test_tbo_shared_experts_fusion.py index dd9cc0a7135e..52ce20b5229d 100644 --- a/test/registered/ep/test_tbo_shared_experts_fusion.py +++ b/test/registered/ep/test_tbo_shared_experts_fusion.py @@ -30,8 +30,7 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "8", "--moe-dense-tp-size", "1", diff --git a/test/registered/gb300/test_deepseek_v4_pro_fp4_balanced.py b/test/registered/gb300/test_deepseek_v4_pro_fp4_balanced.py index 873d8c2259fa..be1cfe563be1 100644 --- a/test/registered/gb300/test_deepseek_v4_pro_fp4_balanced.py +++ b/test/registered/gb300/test_deepseek_v4_pro_fp4_balanced.py @@ -15,9 +15,8 @@ BALANCED_ARGS = [ "--trust-remote-code", - "--dp", + "--attn-dp-size", "4", - "--enable-dp-attention", "--moe-a2a-backend", "deepep", "--speculative-algorithm", diff --git a/test/registered/gb300/test_deepseek_v4_pro_fp4_high_throughput.py b/test/registered/gb300/test_deepseek_v4_pro_fp4_high_throughput.py index 800f35cc6e9f..e154f39cbf75 100644 --- a/test/registered/gb300/test_deepseek_v4_pro_fp4_high_throughput.py +++ b/test/registered/gb300/test_deepseek_v4_pro_fp4_high_throughput.py @@ -14,9 +14,8 @@ HIGH_THROUGHPUT_ARGS = [ "--trust-remote-code", - "--dp", + "--attn-dp-size", "4", - "--enable-dp-attention", "--moe-a2a-backend", "megamoe", "--mem-fraction-static", diff --git a/test/registered/gb300/test_glm52_nvfp4.py b/test/registered/gb300/test_glm52_nvfp4.py index 0be117022a66..358d3ea55de1 100644 --- a/test/registered/gb300/test_glm52_nvfp4.py +++ b/test/registered/gb300/test_glm52_nvfp4.py @@ -52,9 +52,7 @@ def test_glm52_nvfp4(self): ModelLaunchSettings( MODEL_PATH, tp_size=4, - extra_args=COMMON_ARGS - + ["--dp-size=4", "--enable-dp-attention"] - + DP_MTP_ARGS, + extra_args=COMMON_ARGS + ["--attn-dp-size=4"] + DP_MTP_ARGS, variant="TP4+DP4+DPA+MTP", ), ] diff --git a/test/registered/gb300/test_kimi_k25_nvfp4_dp.py b/test/registered/gb300/test_kimi_k25_nvfp4_dp.py index 9554584c040e..7469a9d1f7c5 100644 --- a/test/registered/gb300/test_kimi_k25_nvfp4_dp.py +++ b/test/registered/gb300/test_kimi_k25_nvfp4_dp.py @@ -48,9 +48,7 @@ def test_kimi_k25_nvfp4_dp(self): ModelLaunchSettings( MODEL_PATH, tp_size=4, - extra_args=COMMON_ARGS - + ["--dp-size=4", "--enable-dp-attention"] - + DP_EAGLE_ARGS, + extra_args=COMMON_ARGS + ["--attn-dp-size=4"] + DP_EAGLE_ARGS, variant="TP4+DP4+DPA+EAGLE3", ) ], diff --git a/test/registered/gb300/test_qwen35_fp8_dp.py b/test/registered/gb300/test_qwen35_fp8_dp.py index 380c713f8a99..e62234096be1 100644 --- a/test/registered/gb300/test_qwen35_fp8_dp.py +++ b/test/registered/gb300/test_qwen35_fp8_dp.py @@ -45,9 +45,7 @@ def test_qwen35_fp8_dp(self): ModelLaunchSettings( MODEL_PATH, tp_size=4, - extra_args=COMMON_ARGS - + ["--dp-size=4", "--enable-dp-attention"] - + DP_MTP_ARGS, + extra_args=COMMON_ARGS + ["--attn-dp-size=4"] + DP_MTP_ARGS, variant="TP4+DP4+DPA+MTP", ) ], diff --git a/test/registered/kernels/ops/attention/test_dsa_indexer.py b/test/registered/kernels/ops/attention/test_dsa_indexer.py index c5999d23b179..254ad92ed7f4 100644 --- a/test/registered/kernels/ops/attention/test_dsa_indexer.py +++ b/test/registered/kernels/ops/attention/test_dsa_indexer.py @@ -278,7 +278,6 @@ class TestDSAIndexer(CustomTestCase): def setUpClass(cls): """Set up global server args for testing.""" server_args = ServerArgs(model_path="dummy") - server_args.enable_dp_attention = False server_args.dsa_prefill_backend = "flashmla_sparse" server_args.dsa_decode_backend = "flashmla_sparse" set_global_server_args_for_scheduler(server_args) diff --git a/test/registered/mock_model/test_e2e_tp.py b/test/registered/mock_model/test_e2e_tp.py index 030a6a01636a..10f2721a868a 100644 --- a/test/registered/mock_model/test_e2e_tp.py +++ b/test/registered/mock_model/test_e2e_tp.py @@ -23,9 +23,8 @@ def test_tp_lm_head_all_to_all_cuda_graph(self) -> None: extra_server_args=[ "--tp", "2", - "--dp", + "--attn-dp-size", "2", - "--enable-dp-attention", "--enable-tp-lm-head-all-to-all", "--cuda-graph-max-bs-decode", "4", diff --git a/test/registered/model_loading/test_prefetch_checkpoints_multi_gpu.py b/test/registered/model_loading/test_prefetch_checkpoints_multi_gpu.py index 373a293c2222..11ead88f2011 100644 --- a/test/registered/model_loading/test_prefetch_checkpoints_multi_gpu.py +++ b/test/registered/model_loading/test_prefetch_checkpoints_multi_gpu.py @@ -22,8 +22,7 @@ def setUpClass(cls): cls.engine = sgl.Engine( model_path="Qwen/Qwen1.5-MoE-A2.7B-Chat", tp_size=4, - dp_size=4, - enable_dp_attention=True, + attn_dp_size=4, disable_radix_cache=True, weight_loader_prefetch_checkpoints=True, cuda_graph_max_bs_decode=1, diff --git a/test/registered/model_loading/test_weight_cache_daemon.py b/test/registered/model_loading/test_weight_cache_daemon.py index 22dbe4726aa3..7e6bcbd1e967 100644 --- a/test/registered/model_loading/test_weight_cache_daemon.py +++ b/test/registered/model_loading/test_weight_cache_daemon.py @@ -352,11 +352,10 @@ class TestWeightCacheDaemonQwen3MoeDP(TestWeightCacheDaemonTP2): """Qwen3 with static attention DP through the existing IPC fixture.""" daemon_args = [ - "--dp", + "--attn-dp-size", "2", "--ep-size", "1", - "--enable-dp-attention", "--enable-dp-lm-head", "--random-seed", "42", diff --git a/test/registered/moe/test_flashinfer_a2a_cutedsl_v2.py b/test/registered/moe/test_flashinfer_a2a_cutedsl_v2.py index 7fa1636ef1f3..253c85fac407 100644 --- a/test/registered/moe/test_flashinfer_a2a_cutedsl_v2.py +++ b/test/registered/moe/test_flashinfer_a2a_cutedsl_v2.py @@ -46,9 +46,8 @@ def setUpClass(cls): "4", "--ep-size", "4", - "--dp", + "--attn-dp-size", "4", - "--enable-dp-attention", "--enable-dp-lm-head", "--moe-runner-backend", "flashinfer_cutedsl", diff --git a/test/registered/moe/test_flashinfer_a2a_cutlass.py b/test/registered/moe/test_flashinfer_a2a_cutlass.py index 1e4cddc9247c..9fafc7ffa31f 100644 --- a/test/registered/moe/test_flashinfer_a2a_cutlass.py +++ b/test/registered/moe/test_flashinfer_a2a_cutlass.py @@ -50,9 +50,8 @@ def setUpClass(cls): "4", "--ep-size", "4", - "--dp", + "--attn-dp-size", "4", - "--enable-dp-attention", "--enable-dp-lm-head", "--moe-runner-backend", "flashinfer_cutlass", diff --git a/test/registered/moe/test_hybrid_dp_ep_tp_mtp.py b/test/registered/moe/test_hybrid_dp_ep_tp_mtp.py index 946891b48bec..a3902d731485 100644 --- a/test/registered/moe/test_hybrid_dp_ep_tp_mtp.py +++ b/test/registered/moe/test_hybrid_dp_ep_tp_mtp.py @@ -67,8 +67,7 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "4", ], ) @@ -104,8 +103,7 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "8", ], ) @@ -178,8 +176,7 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "4", "--moe-dense-tp-size", "1", @@ -217,8 +214,7 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "8", "--moe-dense-tp-size", "1", @@ -256,8 +252,7 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "4", "--enable-dp-lm-head", ], @@ -294,8 +289,7 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "8", "--enable-dp-lm-head", ], @@ -332,8 +326,7 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "4", "--moe-dense-tp-size", "1", @@ -372,8 +365,7 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "8", "--moe-dense-tp-size", "1", @@ -450,8 +442,7 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "4", "--moe-a2a-backend", "deepep", @@ -491,8 +482,7 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "8", "--moe-a2a-backend", "deepep", @@ -572,8 +562,7 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "4", "--moe-dense-tp-size", "1", @@ -615,8 +604,7 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "8", "--moe-dense-tp-size", "1", @@ -658,8 +646,7 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "4", "--enable-dp-lm-head", "--moe-a2a-backend", @@ -700,8 +687,7 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "8", "--enable-dp-lm-head", "--moe-a2a-backend", @@ -742,8 +728,7 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "4", "--moe-dense-tp-size", "1", @@ -786,8 +771,7 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "8", "--moe-dense-tp-size", "1", @@ -866,8 +850,7 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "4", "--ep", "8", @@ -905,8 +888,7 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "8", "--ep", "8", @@ -982,8 +964,7 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "4", "--moe-dense-tp-size", "1", @@ -1023,8 +1004,7 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "8", "--moe-dense-tp-size", "1", @@ -1064,8 +1044,7 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "4", "--enable-dp-lm-head", "--ep", @@ -1104,8 +1083,7 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "8", "--enable-dp-lm-head", "--ep", @@ -1144,8 +1122,7 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "4", "--moe-dense-tp-size", "1", @@ -1186,8 +1163,7 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "8", "--moe-dense-tp-size", "1", @@ -1272,8 +1248,7 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "4", "--speculative-algo", "EAGLE", @@ -1319,8 +1294,7 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "8", "--speculative-algo", "EAGLE", @@ -1412,8 +1386,7 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "4", "--moe-dense-tp-size", "1", @@ -1461,8 +1434,7 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "8", "--moe-dense-tp-size", "1", @@ -1510,8 +1482,7 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "4", "--enable-dp-lm-head", "--speculative-algo", @@ -1558,8 +1529,7 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "8", "--enable-dp-lm-head", "--speculative-algo", @@ -1606,8 +1576,7 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "4", "--moe-dense-tp-size", "1", @@ -1656,8 +1625,7 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "8", "--moe-dense-tp-size", "1", @@ -1756,8 +1724,7 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "4", "--moe-a2a-backend", "deepep", @@ -1809,8 +1776,7 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "8", "--moe-a2a-backend", "deepep", @@ -1914,8 +1880,7 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "4", "--moe-dense-tp-size", "1", @@ -1969,8 +1934,7 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "8", "--moe-dense-tp-size", "1", @@ -2024,8 +1988,7 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "4", "--enable-dp-lm-head", "--moe-a2a-backend", @@ -2078,8 +2041,7 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "8", "--enable-dp-lm-head", "--moe-a2a-backend", @@ -2132,8 +2094,7 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "4", "--moe-dense-tp-size", "1", @@ -2188,8 +2149,7 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "8", "--moe-dense-tp-size", "1", @@ -2290,8 +2250,7 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "4", "--ep", "8", @@ -2339,8 +2298,7 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "8", "--ep", "8", @@ -2436,8 +2394,7 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "4", "--moe-dense-tp-size", "1", @@ -2487,8 +2444,7 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "8", "--moe-dense-tp-size", "1", @@ -2538,8 +2494,7 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "4", "--enable-dp-lm-head", "--ep", @@ -2588,8 +2543,7 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "8", "--enable-dp-lm-head", "--ep", @@ -2638,8 +2592,7 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "4", "--moe-dense-tp-size", "1", @@ -2690,8 +2643,7 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "8", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "8", "--moe-dense-tp-size", "1", diff --git a/test/registered/npu/accuracy/deepseek_v4_flash/test_npu_deepseek_v4_flash_fp8_4p_gpqa_a5.py b/test/registered/npu/accuracy/deepseek_v4_flash/test_npu_deepseek_v4_flash_fp8_4p_gpqa_a5.py index 5549d6a5a023..0445bf4fcdba 100644 --- a/test/registered/npu/accuracy/deepseek_v4_flash/test_npu_deepseek_v4_flash_fp8_4p_gpqa_a5.py +++ b/test/registered/npu/accuracy/deepseek_v4_flash/test_npu_deepseek_v4_flash_fp8_4p_gpqa_a5.py @@ -86,8 +86,7 @@ "auto", "--enable-dp-lm-head", "--disable-radix-cache", - "--enable-dp-attention", - "--dp-size", + "--attn-dp-size", 4, "--reasoning-parser", "deepseek-v4", diff --git a/test/registered/npu/accuracy/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_gpqa.py b/test/registered/npu/accuracy/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_gpqa.py index de817f847028..f5b869ffb2e7 100644 --- a/test/registered/npu/accuracy/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_gpqa.py +++ b/test/registered/npu/accuracy/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_gpqa.py @@ -82,9 +82,8 @@ 131072, "--max-running-requests", 192, - "--dp-size", + "--attn-dp-size", 16, - "--enable-dp-attention", "--moe-a2a-backend", "deepep", "--deepep-mode", diff --git a/test/registered/npu/accuracy/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms_aime26.py b/test/registered/npu/accuracy/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms_aime26.py index 502e521f9cb2..87d49529263c 100644 --- a/test/registered/npu/accuracy/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms_aime26.py +++ b/test/registered/npu/accuracy/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms_aime26.py @@ -106,9 +106,8 @@ 32, "--nnodes", 2, - "--dp-size", + "--attn-dp-size", 32, - "--enable-dp-attention", "--ep-size", 32, "--mem-fraction-static", diff --git a/test/registered/npu/accuracy/glm5_2/test_npu_glm_5_2_w8a8_16p_gpqa.py b/test/registered/npu/accuracy/glm5_2/test_npu_glm_5_2_w8a8_16p_gpqa.py index 1ef75f3b85f6..76e801b7e24a 100644 --- a/test/registered/npu/accuracy/glm5_2/test_npu_glm_5_2_w8a8_16p_gpqa.py +++ b/test/registered/npu/accuracy/glm5_2/test_npu_glm_5_2_w8a8_16p_gpqa.py @@ -42,9 +42,8 @@ 32, "--nnodes", 2, - "--dp-size", + "--attn-dp-size", 8, - "--enable-dp-attention", "--chunked-prefill-size", 65536, "--max-prefill-tokens", diff --git a/test/registered/npu/accuracy/kimi_k2_6/test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms_aime25.py b/test/registered/npu/accuracy/kimi_k2_6/test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms_aime25.py index 177e68f20c1a..4a472e322b14 100644 --- a/test/registered/npu/accuracy/kimi_k2_6/test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms_aime25.py +++ b/test/registered/npu/accuracy/kimi_k2_6/test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms_aime25.py @@ -57,8 +57,7 @@ "ascend_attn", "--sampling-backend", "ascend", - "--enable-dp-attention", - "--dp-size", + "--attn-dp-size", 32, "--moe-a2a-backend", "deepep", diff --git a/test/registered/npu/accuracy/kimi_k3/test_npu_kimi_k3_w4a8_32p_gpqa.py b/test/registered/npu/accuracy/kimi_k3/test_npu_kimi_k3_w4a8_32p_gpqa.py index 9ff5dd16f5a6..23c7b7ba0687 100644 --- a/test/registered/npu/accuracy/kimi_k3/test_npu_kimi_k3_w4a8_32p_gpqa.py +++ b/test/registered/npu/accuracy/kimi_k3/test_npu_kimi_k3_w4a8_32p_gpqa.py @@ -60,8 +60,7 @@ "bfloat16", "--tp-size", 64, - "--enable-dp-attention", - "--dp-size", + "--attn-dp-size", 4, "--enable-dp-lm-head", "--enable-shared-experts-attn-tp", diff --git a/test/registered/npu/accuracy/minimax_m2_5/test_npu_minimax_m2_5_w8a8_8p_in3k5_out1k5_50ms_gpqa.py b/test/registered/npu/accuracy/minimax_m2_5/test_npu_minimax_m2_5_w8a8_8p_in3k5_out1k5_50ms_gpqa.py index bd9d19385185..e960839ec2f2 100644 --- a/test/registered/npu/accuracy/minimax_m2_5/test_npu_minimax_m2_5_w8a8_8p_in3k5_out1k5_50ms_gpqa.py +++ b/test/registered/npu/accuracy/minimax_m2_5/test_npu_minimax_m2_5_w8a8_8p_in3k5_out1k5_50ms_gpqa.py @@ -34,8 +34,7 @@ MINIMAX_M2_5_HIGH_THROUGHPUT_OTHER_ARGS = [ "--tp-size", 16, - "--enable-dp-attention", - "--dp-size", + "--attn-dp-size", 16, "--mem-fraction-static", 0.75, diff --git a/test/registered/npu/accuracy/qwen3_5_9b/test_npu_qwen3_5_9b_bf16_1p_gsm8k.py b/test/registered/npu/accuracy/qwen3_5_9b/test_npu_qwen3_5_9b_bf16_1p_gsm8k.py index 66d9820f7aa3..2aef537a1ffb 100644 --- a/test/registered/npu/accuracy/qwen3_5_9b/test_npu_qwen3_5_9b_bf16_1p_gsm8k.py +++ b/test/registered/npu/accuracy/qwen3_5_9b/test_npu_qwen3_5_9b_bf16_1p_gsm8k.py @@ -30,7 +30,6 @@ "ascend", "--device", "npu", - "--enable-dp-attention", "--chunked-prefill-size", 4096, "--max-prefill-tokens", diff --git a/test/registered/npu/accuracy/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16_aime25.py b/test/registered/npu/accuracy/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16_aime25.py index 68d57b758e6e..a75b1c312e77 100644 --- a/test/registered/npu/accuracy/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16_aime25.py +++ b/test/registered/npu/accuracy/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16_aime25.py @@ -62,9 +62,8 @@ 81920, "--max-total-tokens", 122304, - "--dp-size", + "--attn-dp-size", 2, - "--enable-dp-attention", "--enable-dp-lm-head", "--speculative-algorithm", "NEXTN", diff --git a/test/registered/npu/basic_function/dp_attn/test_npu_dp_attention.py b/test/registered/npu/basic_function/dp_attn/test_npu_dp_attention.py index 960fc4206562..f93f67d65b2f 100644 --- a/test/registered/npu/basic_function/dp_attn/test_npu_dp_attention.py +++ b/test/registered/npu/basic_function/dp_attn/test_npu_dp_attention.py @@ -56,8 +56,7 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "4", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "2", "--attention-backend", "ascend", @@ -94,8 +93,7 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "2", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "2", "--enable-mixed-chunk", "--chunked-prefill-size", @@ -128,8 +126,7 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "2", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "2", "--max-total-tokens", "4500", @@ -166,8 +163,7 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "2", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "2", "--attention-backend", "ascend", diff --git a/test/registered/npu/basic_function/parallel_strategy/data_parallelism/test_npu_load_balance_method.py b/test/registered/npu/basic_function/parallel_strategy/data_parallelism/test_npu_load_balance_method.py index 157d0089a4b4..2fbc3c1f43ae 100644 --- a/test/registered/npu/basic_function/parallel_strategy/data_parallelism/test_npu_load_balance_method.py +++ b/test/registered/npu/basic_function/parallel_strategy/data_parallelism/test_npu_load_balance_method.py @@ -40,8 +40,7 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "16", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "2", "--enable-torch-compile", "--torch-compile-max-bs", diff --git a/test/registered/npu/basic_function/parallel_strategy/data_parallelism/test_npu_load_balance_method_pd_disaggregation.py b/test/registered/npu/basic_function/parallel_strategy/data_parallelism/test_npu_load_balance_method_pd_disaggregation.py index ac2e2b2c2eb7..6b1509e1849e 100644 --- a/test/registered/npu/basic_function/parallel_strategy/data_parallelism/test_npu_load_balance_method_pd_disaggregation.py +++ b/test/registered/npu/basic_function/parallel_strategy/data_parallelism/test_npu_load_balance_method_pd_disaggregation.py @@ -70,8 +70,7 @@ def start_prefill(cls): "prefill", "--tp-size", "2", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "2", "--load-balance-method", cls.prefill_load_balance_method, @@ -107,8 +106,7 @@ def start_decode(cls): 2, "--tp-size", "2", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "2", "--load-balance-method", cls.decode_load_balance_method, diff --git a/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_auto_qwen3_480b.py b/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_auto_qwen3_480b.py index 3f45e0a9c13f..b6dea5f3248c 100644 --- a/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_auto_qwen3_480b.py +++ b/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_auto_qwen3_480b.py @@ -51,9 +51,8 @@ class TestDeepEpQwen(GSM8KAscendMixin, TestMMLU, CustomTestCase): "auto", "--tp-size", 16, - "--dp-size", + "--attn-dp-size", 4, - "--enable-dp-attention", "--enable-dp-lm-head", "--mem-fraction-static", 0.7, diff --git a/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_low_latency_qwen3_480b.py b/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_low_latency_qwen3_480b.py index 8663a582d44a..fe26a763b68e 100644 --- a/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_low_latency_qwen3_480b.py +++ b/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_low_latency_qwen3_480b.py @@ -52,9 +52,8 @@ class TestDeepEpQwen(GSM8KAscendMixin, TestMMLU, CustomTestCase): "low_latency", "--tp-size", 16, - "--dp-size", + "--attn-dp-size", 4, - "--enable-dp-attention", "--enable-dp-lm-head", "--mem-fraction-static", 0.7, diff --git a/test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_1p1d_16p_in8k_out1k_50ms.py b/test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_1p1d_16p_in8k_out1k_50ms.py index 6323f2fd13be..3139f8730516 100644 --- a/test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_1p1d_16p_in8k_out1k_50ms.py +++ b/test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_1p1d_16p_in8k_out1k_50ms.py @@ -109,9 +109,8 @@ 131072, "--max-running-requests", 112, - "--dp-size", + "--attn-dp-size", 16, - "--enable-dp-attention", "--moe-a2a-backend", "deepep", "--deepep-mode", @@ -157,9 +156,8 @@ "ascend", "--max-running-requests", 896, - "--dp-size", + "--attn-dp-size", 16, - "--enable-dp-attention", "--moe-a2a-backend", "deepep", "--deepep-mode", diff --git a/test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_in8k_out1k_50ms.py b/test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_in8k_out1k_50ms.py index 85897ab9b6be..b725ae10aa2c 100644 --- a/test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_in8k_out1k_50ms.py +++ b/test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_in8k_out1k_50ms.py @@ -73,9 +73,8 @@ 131072, "--max-running-requests", 192, - "--dp-size", + "--attn-dp-size", 16, - "--enable-dp-attention", "--moe-a2a-backend", "deepep", "--deepep-mode", diff --git a/test/registered/npu/performance/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms.py b/test/registered/npu/performance/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms.py index a8577e70809b..26c88e01551c 100644 --- a/test/registered/npu/performance/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms.py +++ b/test/registered/npu/performance/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms.py @@ -106,9 +106,8 @@ 32, "--nnodes", 2, - "--dp-size", + "--attn-dp-size", 32, - "--enable-dp-attention", "--ep-size", 32, "--mem-fraction-static", diff --git a/test/registered/npu/performance/qwen3_5_397b/test_npu_qwen3_5_397b_w4a8_8p_in3k5_out1k5_50ms.py b/test/registered/npu/performance/qwen3_5_397b/test_npu_qwen3_5_397b_w4a8_8p_in3k5_out1k5_50ms.py index 467f085c69fa..280a3fe63863 100644 --- a/test/registered/npu/performance/qwen3_5_397b/test_npu_qwen3_5_397b_w4a8_8p_in3k5_out1k5_50ms.py +++ b/test/registered/npu/performance/qwen3_5_397b/test_npu_qwen3_5_397b_w4a8_8p_in3k5_out1k5_50ms.py @@ -84,9 +84,8 @@ "bfloat16", "--mamba-ssm-dtype", "bfloat16", - "--dp-size", + "--attn-dp-size", 8, - "--enable-dp-attention", "--enable-dp-lm-head", "--speculative-algorithm", "NEXTN", diff --git a/test/registered/npu/rl/indexer_replay/test_npu_return_indexer_topk_deepseek_v4.py b/test/registered/npu/rl/indexer_replay/test_npu_return_indexer_topk_deepseek_v4.py index 21389a9492fe..cb494a318d45 100644 --- a/test/registered/npu/rl/indexer_replay/test_npu_return_indexer_topk_deepseek_v4.py +++ b/test/registered/npu/rl/indexer_replay/test_npu_return_indexer_topk_deepseek_v4.py @@ -81,9 +81,8 @@ def setUpClass(cls): "0.72", "--tp-size", "2", - "--dp-size", + "--attn-dp-size", "2", - "--enable-dp-attention", "--enable-dp-lm-head", "--kv-cache-dtype", "auto", diff --git a/test/registered/npu/rl/indexer_replay/test_npu_return_indexer_topk_glm52.py b/test/registered/npu/rl/indexer_replay/test_npu_return_indexer_topk_glm52.py index 3cc9b955277f..8caef443fd50 100644 --- a/test/registered/npu/rl/indexer_replay/test_npu_return_indexer_topk_glm52.py +++ b/test/registered/npu/rl/indexer_replay/test_npu_return_indexer_topk_glm52.py @@ -56,9 +56,8 @@ def setUpClass(cls): "0.56", "--tp-size", "2", - "--dp-size", + "--attn-dp-size", "2", - "--enable-dp-attention", "--enable-dp-lm-head", "--disable-cuda-graph", "--enable-return-indexer-topk", diff --git a/test/registered/observability/test_metrics.py b/test/registered/observability/test_metrics.py index 4013c63bd868..cba73727b58a 100644 --- a/test/registered/observability/test_metrics.py +++ b/test/registered/observability/test_metrics.py @@ -85,7 +85,7 @@ def _verify_metrics_extra(metrics): self.assertIn("1", num_prefill_ranks_values) self._execute_core( - other_args=["--tp", "2", "--dp", "2", "--enable-dp-attention"], + other_args=["--tp", "2", "--attn-dp-size", "2"], verify_metrics_extra=_verify_metrics_extra, expect_mfu_metrics=True, enable_mfu_metrics=True, diff --git a/test/registered/pp/test_pp_parallel_compat.py b/test/registered/pp/test_pp_parallel_compat.py index ceb431b43e39..f371549a76c4 100644 --- a/test/registered/pp/test_pp_parallel_compat.py +++ b/test/registered/pp/test_pp_parallel_compat.py @@ -93,16 +93,15 @@ class TestQwen3MoePPxCP(_Qwen3MoePPCompatMixin, CustomTestCase): class TestQwen3MoePPxDP(_Qwen3MoePPCompatMixin, CustomTestCase): - """PP x DP: pp_size=2 x dp_size=2 attention (tp_size=2).""" + """PP x DP: pp_size=2 x attn_dp_size=2 (tp_size=2).""" parallel_args = [ "--tp-size", "2", "--pp-size", "2", - "--dp-size", + "--attn-dp-size", "2", - "--enable-dp-attention", ] diff --git a/test/registered/pp/test_pp_single_node.py b/test/registered/pp/test_pp_single_node.py index a652048c8c3a..ab80d69fb625 100644 --- a/test/registered/pp/test_pp_single_node.py +++ b/test/registered/pp/test_pp_single_node.py @@ -117,8 +117,7 @@ def setUpClass(cls): "2", "--pp-size", "2", - "--enable-dp-attention", - "--dp", + "--attn-dp-size", "2", ], ) diff --git a/test/registered/radix_cache/unified_radix_tree/test_unified_radix_cache_kl_mimo.py b/test/registered/radix_cache/unified_radix_tree/test_unified_radix_cache_kl_mimo.py index d70cb6712e0d..9d458d62b7eb 100644 --- a/test/registered/radix_cache/unified_radix_tree/test_unified_radix_cache_kl_mimo.py +++ b/test/registered/radix_cache/unified_radix_tree/test_unified_radix_cache_kl_mimo.py @@ -54,9 +54,8 @@ def setUpClass(cls): "--enable-multimodal", "--tp", "8", - "--dp", + "--attn-dp-size", "2", - "--enable-dp-attention", "--mm-enable-dp-encoder", "--attention-backend", "fa3", diff --git a/test/registered/rl/test_return_indexer_topk.py b/test/registered/rl/test_return_indexer_topk.py index eb0c24096e3c..612f3b5e743a 100644 --- a/test/registered/rl/test_return_indexer_topk.py +++ b/test/registered/rl/test_return_indexer_topk.py @@ -52,9 +52,8 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "8", - "--dp", + "--attn-dp-size", "8", - "--enable-dp-attention", "--enable-return-indexer-topk", # Cap KV pool so the indexer-topk host buffer (488 KB / token for # V3.2) stays bounded; with the default ~600k tokens × 8 procs the diff --git a/test/registered/rl/test_return_routed_experts.py b/test/registered/rl/test_return_routed_experts.py index fc54a43f32f8..a1664d174681 100644 --- a/test/registered/rl/test_return_routed_experts.py +++ b/test/registered/rl/test_return_routed_experts.py @@ -44,8 +44,8 @@ class TestReturnRoutedExperts(CustomTestCase): under DeepEP a2a + attn_tp_size > 1, across overlap/cuda-graph/radix optimisations. - Both servers run ``--tp 4 --dp 2 --enable-dp-attention --moe-a2a-backend - deepep`` so attn_tp_size=2 and the all-gather hot path in + Both servers run ``--tp 4 --attn-dp-size 2 --moe-a2a-backend deepep`` + so attn_tp_size=2 and the all-gather hot path in RoutedExpertsCapturer.capture is hit on every step. Baseline disables overlap/cuda-graph/radix to give a deterministic ground truth; reference leaves them on. If the gather were skipping a rank or racing against the @@ -59,9 +59,8 @@ def setUpClass(cls): "--enable-deterministic-inference", "--tp", 4, - "--dp", + "--attn-dp-size", 2, - "--enable-dp-attention", "--moe-a2a-backend", "deepep", # Force normal-mode dispatch: deepep auto routes decode through diff --git a/test/registered/rl/test_update_weights_from_disk_blackwell.py b/test/registered/rl/test_update_weights_from_disk_blackwell.py index a70286dcb9e0..34463dd50f11 100644 --- a/test/registered/rl/test_update_weights_from_disk_blackwell.py +++ b/test/registered/rl/test_update_weights_from_disk_blackwell.py @@ -207,9 +207,8 @@ class TestServerUpdateWeightsFromDiskMXFP8(UpdateWeightsFromDiskBase, CustomTest "other_args": ( "--tp-size", "4", - "--dp-size", + "--attn-dp-size", "4", - "--enable-dp-attention", "--fp8-gemm-backend", "flashinfer_trtllm", "--moe-runner-backend", @@ -286,9 +285,8 @@ class TestServerUpdateWeightsFromDiskNVFP4W4A16CuteDSL( "other_args": ( "--tp-size", "4", - "--dp-size", + "--attn-dp-size", "4", - "--enable-dp-attention", "--ep-size", "4", "--fp4-gemm-backend", diff --git a/test/registered/rl/test_weight_version_spans.py b/test/registered/rl/test_weight_version_spans.py index f5fa8d4fe3f8..9afd9291af6a 100644 --- a/test/registered/rl/test_weight_version_spans.py +++ b/test/registered/rl/test_weight_version_spans.py @@ -48,9 +48,8 @@ def setUpClass(cls): "--trust-remote-code", "--tp-size", "2", - "--dp-size", + "--attn-dp-size", "2", - "--enable-dp-attention", "--speculative-algorithm", "EAGLE", "--speculative-draft-model-path", diff --git a/test/registered/spec/dspark/test_dspark_dp_spec_prefill_coordination.py b/test/registered/spec/dspark/test_dspark_dp_spec_prefill_coordination.py index 8e7fbcb47fe0..c8e3b19243a6 100644 --- a/test/registered/spec/dspark/test_dspark_dp_spec_prefill_coordination.py +++ b/test/registered/spec/dspark/test_dspark_dp_spec_prefill_coordination.py @@ -40,9 +40,8 @@ def setUpClass(cls): "deepseek-ai/dspark_qwen3_14b_block7", "--tp-size", "2", - "--dp-size", + "--attn-dp-size", "2", - "--enable-dp-attention", "--enable-dp-lm-head", "--attention-backend", "trtllm_mha" if is_sm100_supported() else "fa3", diff --git a/test/registered/spec/dspark/test_dspark_draft_path_default.py b/test/registered/spec/dspark/test_dspark_draft_path_default.py index b0da41feb86e..c852e837f8fc 100644 --- a/test/registered/spec/dspark/test_dspark_draft_path_default.py +++ b/test/registered/spec/dspark/test_dspark_draft_path_default.py @@ -100,9 +100,8 @@ def _dp_server_args(self, *, moe_a2a_backend: str) -> ServerArgs: server_args = _make_dspark_server_args( model_path=_BUNDLED_MODEL_PATH, hf_config=_bundled_hf_config() ) - server_args.enable_dp_attention = True server_args.enable_dp_lm_head = True - server_args.dp_size = 2 + server_args.attn_dp_size = 2 server_args.tp_size = 2 server_args.moe_a2a_backend = moe_a2a_backend return server_args diff --git a/test/registered/spec/eagle/test_dp_spec_prefill_coordination.py b/test/registered/spec/eagle/test_dp_spec_prefill_coordination.py index 5bd05c26d88f..aca2864e0853 100644 --- a/test/registered/spec/eagle/test_dp_spec_prefill_coordination.py +++ b/test/registered/spec/eagle/test_dp_spec_prefill_coordination.py @@ -39,9 +39,8 @@ def setUpClass(cls): DEFAULT_DRAFT_MODEL_EAGLE_DP_ATTN, "--tp-size", "2", - "--dp-size", + "--attn-dp-size", "2", - "--enable-dp-attention", "--enable-dp-lm-head", "--moe-dense-tp-size", "1", diff --git a/test/registered/spec/eagle/test_dp_spec_prefill_coordination_mixed.py b/test/registered/spec/eagle/test_dp_spec_prefill_coordination_mixed.py index bf28b3b8b6f8..c91ea9bfaa0a 100644 --- a/test/registered/spec/eagle/test_dp_spec_prefill_coordination_mixed.py +++ b/test/registered/spec/eagle/test_dp_spec_prefill_coordination_mixed.py @@ -65,9 +65,8 @@ def setUpClass(cls): DEFAULT_DRAFT_MODEL_EAGLE_DP_ATTN, "--tp-size", "2", - "--dp-size", + "--attn-dp-size", "2", - "--enable-dp-attention", "--enable-dp-lm-head", "--moe-dense-tp-size", "1", diff --git a/test/registered/spec/eagle/test_eagle_dp_attention.py b/test/registered/spec/eagle/test_eagle_dp_attention.py index 1669fd58db62..52a0b3e1872c 100644 --- a/test/registered/spec/eagle/test_eagle_dp_attention.py +++ b/test/registered/spec/eagle/test_eagle_dp_attention.py @@ -44,9 +44,8 @@ def setUpClass(cls): DEFAULT_DRAFT_MODEL_EAGLE_DP_ATTN, "--tp-size", "2", - "--dp-size", + "--attn-dp-size", "2", - "--enable-dp-attention", "--enable-dp-lm-head", "--moe-dense-tp-size", "1", diff --git a/test/registered/unit/disaggregation/test_pd_role_switch.py b/test/registered/unit/disaggregation/test_pd_role_switch.py index 629af34bcf0d..adea1355394a 100644 --- a/test/registered/unit/disaggregation/test_pd_role_switch.py +++ b/test/registered/unit/disaggregation/test_pd_role_switch.py @@ -307,7 +307,8 @@ def _sa(self, **kw): disaggregation_mode="prefill", disaggregation_decode_host_receive_threshold=0.0, enable_pd_role_switch=True, - enable_dp_attention=False, + attn_dp_size=1, + ep_join_mode=None, ep_size=1, moe_a2a_backend="none", pp_size=1, @@ -331,8 +332,8 @@ def test_pure_tp_role_switch_accepted(self): def test_reject_dp_attention(self): with self.assertRaises(ValueError) as ctx: - self._run(self._sa(enable_dp_attention=True)) - self.assertIn("DP attention", str(ctx.exception)) + self._run(self._sa(attn_dp_size=2)) + self.assertIn("attention DP", str(ctx.exception)) def test_reject_expert_parallelism(self): with self.assertRaises(ValueError) as ctx: diff --git a/test/registered/unit/entrypoints/test_rust_server_dp_ports.py b/test/registered/unit/entrypoints/test_rust_server_dp_ports.py index a834d7b08b50..55b0aa785efd 100644 --- a/test/registered/unit/entrypoints/test_rust_server_dp_ports.py +++ b/test/registered/unit/entrypoints/test_rust_server_dp_ports.py @@ -13,7 +13,7 @@ @pytest.mark.parametrize( - "nnodes,tp_size,dp_size,ep_join_mode,ranks,expected", + "nnodes,tp_size,attn_dp_size,ep_join_mode,ranks,expected", [ (2, 4, 4, None, (0, 1, 2, 3), [0, 1, 0, 1]), (4, 4, 2, None, (0, 2), [0, 0]), @@ -22,14 +22,13 @@ ids=["multiple-listeners-per-node", "dp-spans-nodes", "scale-joiner"], ) def test_dp_leaders_reuse_node_local_ports( - nnodes, tp_size, dp_size, ep_join_mode, ranks, expected + nnodes, tp_size, attn_dp_size, ep_join_mode, ranks, expected ): with ( get_context().override_server_args( nnodes=nnodes, tp_size=tp_size, - dp_size=dp_size, - enable_dp_attention=True, + attn_dp_size=attn_dp_size, ep_join_mode=ep_join_mode, host="0.0.0.0", port=30000, @@ -70,8 +69,7 @@ def test_node_listener_placement(pp_size, expected, node_rank): node_rank=node_rank, tp_size=4, pp_size=pp_size, - dp_size=2, - enable_dp_attention=True, + attn_dp_size=2, attn_cp_size=2, ): assert node_hosts_rust_server() == expected[node_rank] @@ -82,8 +80,6 @@ def test_scale_joiner_hosts_listener(): nnodes=2, node_rank=1, tp_size=1, - dp_size=1, - enable_dp_attention=True, ep_join_mode="scale", ): assert node_hosts_rust_server() diff --git a/test/registered/unit/entrypoints/test_v1_loads_aggregate.py b/test/registered/unit/entrypoints/test_v1_loads_aggregate.py index ad3ee9247e30..5f9be76017fb 100644 --- a/test/registered/unit/entrypoints/test_v1_loads_aggregate.py +++ b/test/registered/unit/entrypoints/test_v1_loads_aggregate.py @@ -47,7 +47,6 @@ def __init__(self, reader, dp_size: int): self.elastic_worker_count = dp_size self.server_args = SimpleNamespace( dp_size=dp_size, - enable_dp_attention=False, nnodes=1, ) @@ -64,7 +63,7 @@ def __init__( tp_size=1, dp_size=1, pp_size=1, - enable_dp_attention=False, + attn_dp_size=1, ): from sglang.srt.runtime_context import get_context @@ -78,7 +77,7 @@ def __init__( tp_size=tp_size, dp_size=dp_size, pp_size=pp_size, - enable_dp_attention=enable_dp_attention, + attn_dp_size=attn_dp_size, ) self._override.install() @@ -136,8 +135,7 @@ def test_dp_attention_reports_accelerators_per_dp_rank(self): manager = _FakeHttpTokenizerManager( [LoadSnapshot(dp_rank=rank) for rank in range(8)], tp_size=8, - dp_size=8, - enable_dp_attention=True, + attn_dp_size=8, ) self.addCleanup(manager.restore) diff --git a/test/registered/unit/hardware_backend/mlx/test_attn_dp_request_capacity.py b/test/registered/unit/hardware_backend/mlx/test_attn_dp_request_capacity.py index e77137506d76..2ace85fb8166 100644 --- a/test/registered/unit/hardware_backend/mlx/test_attn_dp_request_capacity.py +++ b/test/registered/unit/hardware_backend/mlx/test_attn_dp_request_capacity.py @@ -41,8 +41,8 @@ def _arch(*, hybrid: bool): def _stub_for_initialize( test, *, - dp_size: int, - attn_dp_size: int, + dp_size: int = 1, + attn_dp_size: int = 1, max_running_requests: int = 8, max_mamba_cache_size: int | None = None, pool_size: int = 64, @@ -53,7 +53,7 @@ def _stub_for_initialize( override = get_context().override_server_args( tp_size=attn_dp_size, dp_size=dp_size, - enable_dp_attention=attn_dp_size > 1, + attn_dp_size=attn_dp_size, enable_memory_saver=False, max_running_requests=max_running_requests, max_mamba_cache_size=max_mamba_cache_size, @@ -91,14 +91,14 @@ def _initialize_stub(stub, *, hybrid: bool = False): class TestAttentionDpRequestCapacity(CustomTestCase): def test_pure_dp_replica_retains_full_request_limit(self): stub = _initialize_stub( - _stub_for_initialize(self, dp_size=4, attn_dp_size=1), + _stub_for_initialize(self, dp_size=4), ) self.assertEqual(stub.max_running_requests, 8) self.assertEqual(stub.req_to_token_pool.size, 8) def test_attention_dp_partitions_request_limit(self): stub = _initialize_stub( - _stub_for_initialize(self, dp_size=4, attn_dp_size=4), + _stub_for_initialize(self, attn_dp_size=4), ) self.assertEqual(stub.max_running_requests, 2) self.assertEqual(stub.req_to_token_pool.size, 2) @@ -107,7 +107,6 @@ def test_attention_dp_partitions_explicit_auxiliary_state_limit(self): stub = _initialize_stub( _stub_for_initialize( self, - dp_size=4, attn_dp_size=4, max_running_requests=8, max_mamba_cache_size=4 * RATIO, @@ -122,7 +121,6 @@ def test_attention_dp_partitions_explicit_auxiliary_state_limit(self): def test_attention_dp_auxiliary_error_reports_global_cli_units(self): stub = _stub_for_initialize( self, - dp_size=4, attn_dp_size=4, max_running_requests=8, max_mamba_cache_size=4 * RATIO - 1, diff --git a/test/registered/unit/hardware_backend/mlx/test_max_running_requests.py b/test/registered/unit/hardware_backend/mlx/test_max_running_requests.py index 974988edf30c..fdd1d43d0fb3 100644 --- a/test/registered/unit/hardware_backend/mlx/test_max_running_requests.py +++ b/test/registered/unit/hardware_backend/mlx/test_max_running_requests.py @@ -60,8 +60,7 @@ def _published(stub): return get_context().override_server_args( # One attention replica per TP rank gives the stub's attention-DP width. tp_size=stub._attn_dp_size, - dp_size=stub._attn_dp_size, - enable_dp_attention=stub._attn_dp_size > 1, + attn_dp_size=stub._attn_dp_size, max_running_requests=stub._max_running_requests, max_mamba_cache_size=stub._max_mamba_cache_size, disable_radix_cache=stub._disable_radix_cache, diff --git a/test/registered/unit/hardware_backend/mlx/test_metal_profiler.py b/test/registered/unit/hardware_backend/mlx/test_metal_profiler.py index b559d4206843..128a7fa46d27 100644 --- a/test/registered/unit/hardware_backend/mlx/test_metal_profiler.py +++ b/test/registered/unit/hardware_backend/mlx/test_metal_profiler.py @@ -217,7 +217,9 @@ def test_start_profile_success_with_mock_capture(self): torch.mps.profiler, "metal_capture", return_value=capture_ctx ), mock_patch("torch.distributed.barrier"), - get_parallel().override(tp_rank=0, dp_size=1, pp_size=1, moe_ep_size=1), + get_parallel().override( + tp_rank=0, num_dp_ranks=1, pp_size=1, moe_ep_size=1 + ), ): result = mgr._start_profile() self.assertTrue(result.success, result.message) diff --git a/test/registered/unit/layer_boundary/test_declared_attention_cp.py b/test/registered/unit/layer_boundary/test_declared_attention_cp.py index 10c7e09911b8..72a3cfbd7faf 100644 --- a/test/registered/unit/layer_boundary/test_declared_attention_cp.py +++ b/test/registered/unit/layer_boundary/test_declared_attention_cp.py @@ -84,7 +84,7 @@ def as_rank(self, cp, collectives, moe_group=None): tp_rank=cp, attn_dp_size=1, attn_dp_rank=0, - enable_dp_attention=False, + attn_dp_enabled=False, attn_tp_size=1, attn_tp_rank=0, attn_cp_size=CP_SIZE, diff --git a/test/registered/unit/layer_boundary/test_declared_decoder_boundary.py b/test/registered/unit/layer_boundary/test_declared_decoder_boundary.py index c244cc9296a3..968d1f5c57f7 100644 --- a/test/registered/unit/layer_boundary/test_declared_decoder_boundary.py +++ b/test/registered/unit/layer_boundary/test_declared_decoder_boundary.py @@ -83,7 +83,7 @@ def parallel_of(*, attn_dp, attn_tp, attn_cp=1, **overrides): moe_tp_size=attn_dp * attn_cp * attn_tp, moe_dp_size=1, dwdp_size=1, - enable_dp_attention=attn_dp > 1, + attn_dp_enabled=attn_dp > 1, enable_attn_tp_input_scattered=False, tp_group=SimpleNamespace( name="tp", ranks=list(range(attn_dp * attn_cp * attn_tp)) diff --git a/test/registered/unit/layer_boundary/test_dp_attention_cp_gather.py b/test/registered/unit/layer_boundary/test_dp_attention_cp_gather.py index ac11493e79ba..52c18d114474 100644 --- a/test/registered/unit/layer_boundary/test_dp_attention_cp_gather.py +++ b/test/registered/unit/layer_boundary/test_dp_attention_cp_gather.py @@ -188,7 +188,7 @@ def as_rank(dp, cp, tp, all_reduce): attn_cp_rank=cp, attn_tp_rank=tp, attn_dp_size=DP_SIZE, - enable_dp_attention=True, + attn_dp_enabled=True, attn_cp_size=CP_SIZE, attn_tp_size=attn_tp_size, tp_size=tp_size, diff --git a/test/registered/unit/layer_boundary/test_layernorm_sp.py b/test/registered/unit/layer_boundary/test_layernorm_sp.py index 468687a0c1b4..1cd549c634dd 100644 --- a/test/registered/unit/layer_boundary/test_layernorm_sp.py +++ b/test/registered/unit/layer_boundary/test_layernorm_sp.py @@ -103,7 +103,7 @@ class TestLayerNormSPValidation(CustomTestCase): VALID = dict( architecture="Qwen3ForCausalLM", tp_size=2, - enable_dp_attention=False, + attn_dp_enabled=False, speculative_algorithm=None, ) @@ -119,8 +119,8 @@ def test_rejects_tp_size_one(self): validate_layernorm_sp(**{**self.VALID, "tp_size": 1}) def test_rejects_dp_attention(self): - with self.assertRaisesRegex(ValueError, "dp-attention"): - validate_layernorm_sp(**{**self.VALID, "enable_dp_attention": True}) + with self.assertRaisesRegex(ValueError, "attention DP"): + validate_layernorm_sp(**{**self.VALID, "attn_dp_enabled": True}) def test_rejects_speculative(self): with self.assertRaisesRegex(ValueError, "speculative"): diff --git a/test/registered/unit/layer_boundary/test_nemotron_h_aux_capture.py b/test/registered/unit/layer_boundary/test_nemotron_h_aux_capture.py index 11ee2d7511e3..6a4fea2876e2 100644 --- a/test/registered/unit/layer_boundary/test_nemotron_h_aux_capture.py +++ b/test/registered/unit/layer_boundary/test_nemotron_h_aux_capture.py @@ -96,9 +96,7 @@ def reduce_in_place(x): ) inputs = torch.tensor([[0.3, -0.5, 0.7, 1.1], [-0.4, 0.9, 0.2, -0.6]]) with ( - get_context().override_server_args( - tp_size=tp, enable_dp_attention=dp_enabled - ), + get_context().override_server_args(tp_size=tp), get_flags().dp.override(enabled=dp_enabled), get_parallel().override( attn_tp_group=group, diff --git a/test/registered/unit/layer_boundary/test_nemotron_h_mtp_reduction.py b/test/registered/unit/layer_boundary/test_nemotron_h_mtp_reduction.py index 0ff3ad10b5ff..9ad4e97b86b1 100644 --- a/test/registered/unit/layer_boundary/test_nemotron_h_mtp_reduction.py +++ b/test/registered/unit/layer_boundary/test_nemotron_h_mtp_reduction.py @@ -34,9 +34,7 @@ def test_attention_partial_is_reduced_once(self): reduce = Mock(side_effect=lambda x: x * tp) group = SimpleNamespace(all_reduce=reduce) with ( - get_context().override_server_args( - tp_size=tp, enable_dp_attention=True - ), + get_context().override_server_args(tp_size=tp), get_flags().dp.override(enabled=True), get_parallel().override( attn_tp_group=group, diff --git a/test/registered/unit/layers/attention/test_flashattention_pa_swa_prefill_lens_size.py b/test/registered/unit/layers/attention/test_flashattention_pa_swa_prefill_lens_size.py index 774484717a0c..e18cf01dc581 100644 --- a/test/registered/unit/layers/attention/test_flashattention_pa_swa_prefill_lens_size.py +++ b/test/registered/unit/layers/attention/test_flashattention_pa_swa_prefill_lens_size.py @@ -68,7 +68,6 @@ def _make_prefill_aware_swa_runner( chunked_prefill_size=8192, disable_radix_cache=False, enable_prefill_cp=False, - enable_dp_attention=False, ) token_to_kv_pool = token_to_kv_pool if token_to_kv_pool is not None else object() token_to_kv_pool_allocator = object() diff --git a/test/registered/unit/lora/test_mem_pool_ep_unit.py b/test/registered/unit/lora/test_mem_pool_ep_unit.py index 7f73273bc908..2848a806486e 100644 --- a/test/registered/unit/lora/test_mem_pool_ep_unit.py +++ b/test/registered/unit/lora/test_mem_pool_ep_unit.py @@ -1126,8 +1126,8 @@ def _pool( pool.max_loras_per_batch = 2 pool.tp_size = tp_size pool.tp_rank = 0 - # Without --enable-dp-attention the attention TP group equals the - # outer TP group. + # Without attention DP the attention TP group equals the outer TP + # group. pool.attn_tp_size = tp_size pool.moe_ep_size = ep_size pool.moe_ep_rank = ep_rank @@ -1232,8 +1232,8 @@ class TestAttnModulesShardByAttnTp(unittest.TestCase): """Regression: attention-module LoRA buffers must shard by `attn_tp_size`, not the outer `tp_size`. - Under `--enable-dp-attention` attention layers are built on the attn_tp - group (`attn_tp_size = tp_size // dp_size`), so e.g. MLA `o_proj` holds an + Under `--attn-dp-size` attention layers are built on the attn_tp group + (`attn_tp_size = tp_size // attn_dp_size`), so e.g. MLA `o_proj` holds an attn_tp-local input shard. Sizing the LoRA buffer by the outer `tp_size` would make it narrower than the slice produced by `RowParallelLinearWithLoRA.slice_lora_a_weights` (which slices by the base @@ -1437,7 +1437,7 @@ class TestRowParallelLoraAShardsByBaseLinear(unittest.TestCase): """Regression: dense row-parallel LoRA-A buffers must match the base linear's real input shard. - Under `--enable-dp-attention --moe-dense-tp-size 1` the dense-MLP and + Under `--attn-dp-size N --moe-dense-tp-size 1` the dense-MLP and shared-expert `down_proj` are fully replicated (K is the full intermediate size) although the outer `tp_size` is large. Dividing by `tp_size` undersized the buffer and `sgemm_lora_a_fwd` failed its `x.shape[-1] == K` assertion. diff --git a/test/registered/unit/lora/test_triton_dp_attention_unit.py b/test/registered/unit/lora/test_triton_dp_attention_unit.py index 2d22bf67f375..e6352cf00398 100644 --- a/test/registered/unit/lora/test_triton_dp_attention_unit.py +++ b/test/registered/unit/lora/test_triton_dp_attention_unit.py @@ -106,8 +106,8 @@ def prepare_lora_batch( class TestDPAttentionBackendContract(CustomTestCase): def test_manager_requires_global_routing_only_for_dp_attention(self): """DPA must not proceed with local-only routing; non-DPA remains supported.""" - for enable_dp_attention in (False, True): - with self.subTest(enable_dp_attention=enable_dp_attention): + for attn_dp_enabled in (False, True): + with self.subTest(attn_dp_enabled=attn_dp_enabled): backend = _LocalOnlyLoRABackend( max_loras_per_batch=3, device=torch.device("cpu") ) @@ -115,7 +115,7 @@ def test_manager_requires_global_routing_only_for_dp_attention(self): pool.max_loras_per_batch = 3 pool.uid_to_buffer_id = {None: 0, "adapter": 1} manager = LoRAManager.__new__(LoRAManager) - manager.enable_dp_attention = enable_dp_attention + manager.attn_dp_enabled = attn_dp_enabled manager.max_loras_per_batch = 3 manager.num_pinned_loras = 1 manager.lora_backend = backend @@ -146,7 +146,7 @@ def test_manager_requires_global_routing_only_for_dp_attention(self): NotImplementedError, "_LocalOnlyLoRABackend.*prepare_global_lora_batch", ) - if enable_dp_attention + if attn_dp_enabled else nullcontext() ) with ( @@ -190,12 +190,12 @@ def test_manager_rejects_dp_attention_with_multi_rank_attention_groups( ): with ( get_context().override_server_args( - enable_dp_attention=True, enable_lora_overlap_loading=False + tp_size=4, attn_dp_size=2, enable_lora_overlap_loading=False ) as args, get_parallel().override( tp_size=4, attn_dp_size=2, moe_tp_size=4, **attention_widths ), - pytest.raises(ValueError, match="requires --dp-size equal to --tp-size"), + pytest.raises(ValueError, match="requires --attn-dp-size equal to --tp-size"), ): LoRAManager( base_model=torch.nn.Linear(2, 2), @@ -219,7 +219,7 @@ def test_communicator_publishes_layout_at_each_transition( monkeypatch.setattr( module, "get_parallel", - lambda: SimpleNamespace(enable_dp_attention=publish_lora_layout), + lambda: SimpleNamespace(attn_dp_enabled=publish_lora_layout), ) # Start from TP_GLOBAL so an unpublished transition is distinguishable. initial = LoRABatchLayout.TP_GLOBAL @@ -285,7 +285,7 @@ def test_communicator_publishes_layout_at_each_transition( @pytest.mark.parametrize("can_run_decode_cuda_graph", [False, True]) def test_manager_uses_current_dp_cuda_graph_eligibility(can_run_decode_cuda_graph): manager = LoRAManager.__new__(LoRAManager) - manager.enable_dp_attention = True + manager.attn_dp_enabled = True manager.max_bs_in_cuda_graph = 4 tokens = torch.zeros(2, dtype=torch.int64) forward_batch = ForwardBatch( diff --git a/test/registered/unit/lora/test_uno_lora_targets.py b/test/registered/unit/lora/test_uno_lora_targets.py index 9610fb7e0210..65c500f0368e 100644 --- a/test/registered/unit/lora/test_uno_lora_targets.py +++ b/test/registered/unit/lora/test_uno_lora_targets.py @@ -235,7 +235,7 @@ def test_manager_preflights_targets_before_wrapping(self): def test_manager_rejects_uno_with_dp_attention(self): with ( get_context().override_server_args( - enable_dp_attention=True, enable_lora_overlap_loading=False + tp_size=2, attn_dp_size=2, enable_lora_overlap_loading=False ) as args, get_parallel().override(attn_tp_size=1), self.assertRaisesRegex( diff --git a/test/registered/unit/managers/scheduler_components/test_dp_attn.py b/test/registered/unit/managers/scheduler_components/test_dp_attn.py index b70882092300..461effc9ce08 100644 --- a/test/registered/unit/managers/scheduler_components/test_dp_attn.py +++ b/test/registered/unit/managers/scheduler_components/test_dp_attn.py @@ -18,10 +18,10 @@ class TestDPAttnSchedulerMetadata(CustomTestCase): def test_skip_all_gather_policy(self): with envs.SGLANG_SCHEDULER_SKIP_ALL_GATHER.override(False): - self.assertTrue(dp_attn.should_skip_scheduler_all_gather(dp_size=1)) - self.assertFalse(dp_attn.should_skip_scheduler_all_gather(dp_size=2)) + self.assertTrue(dp_attn.should_skip_scheduler_all_gather(num_dp_ranks=1)) + self.assertFalse(dp_attn.should_skip_scheduler_all_gather(num_dp_ranks=2)) with envs.SGLANG_SCHEDULER_SKIP_ALL_GATHER.override(True): - self.assertTrue(dp_attn.should_skip_scheduler_all_gather(dp_size=2)) + self.assertTrue(dp_attn.should_skip_scheduler_all_gather(num_dp_ranks=2)) def test_dp1_skip_preserves_local_tbo_metadata(self): batch = SimpleNamespace( @@ -44,7 +44,7 @@ def test_dp1_skip_preserves_local_tbo_metadata(self): dp_attn, "get_parallel", return_value=SimpleNamespace( - dp_size=1, + num_dp_ranks=1, attn_tp_size=4, attn_cp_size=1, tp_group=SimpleNamespace( diff --git a/test/registered/unit/managers/test_auxiliary_output.py b/test/registered/unit/managers/test_auxiliary_output.py index c6c12a272e77..7545473fa6e6 100644 --- a/test/registered/unit/managers/test_auxiliary_output.py +++ b/test/registered/unit/managers/test_auxiliary_output.py @@ -532,7 +532,7 @@ def test_pdmux_split_prefill_schedules_auxiliary_output_copy(): scheduler.future_map = object() scheduler._relay_forward_payload = Mock() scheduler.device_module = SimpleNamespace(Event=Mock(return_value=copy_done)) - scheduler.enable_dp_attention = False + scheduler.attn_dp_enabled = False batch = SimpleNamespace( forward_mode=SimpleNamespace( is_prebuilt=lambda: False, diff --git a/test/registered/unit/managers/test_data_parallel_controller.py b/test/registered/unit/managers/test_data_parallel_controller.py index c17ad1f712ca..1e735c7c3d2b 100644 --- a/test/registered/unit/managers/test_data_parallel_controller.py +++ b/test/registered/unit/managers/test_data_parallel_controller.py @@ -44,14 +44,14 @@ def _load(**overrides) -> LoadSnapshot: return msgspec.structs.replace(_BASE_LOAD, **overrides) -def _make_controller(dp_size: int) -> DataParallelController: +def _make_controller(num_dp_ranks: int) -> DataParallelController: """Bypass __init__; inject only the attrs dispatch methods read.""" ctl = DataParallelController.__new__(DataParallelController) - ctl.workers = [MagicMock(name=f"worker_{i}") for i in range(dp_size)] - ctl.status = [True] * dp_size - ctl._active_workers = list(range(dp_size)) + ctl.workers = [MagicMock(name=f"worker_{i}") for i in range(num_dp_ranks)] + ctl.status = [True] * num_dp_ranks + ctl._active_workers = list(range(num_dp_ranks)) ctl.round_robin_counter = 0 - ctl.dp_budget = DPBudget(dp_size=dp_size) + ctl.dp_budget = DPBudget(num_dp_ranks=num_dp_ranks) return ctl @@ -66,7 +66,7 @@ def _req(routed_dp_rank=None, bootstrap_room=None, input_ids=None): class TestDPBudgetUpdateBudget(CustomTestCase): def test_maps_running_plus_waiting_to_total_requests(self): - budget = DPBudget(dp_size=2) + budget = DPBudget(num_dp_ranks=2) budget.update_budget( [ _load(dp_rank=0, timestamp=1.0, num_running_reqs=3, num_waiting_reqs=2), @@ -76,7 +76,7 @@ def test_maps_running_plus_waiting_to_total_requests(self): self.assertEqual(budget.total_requests, [5, 6]) def test_maps_num_total_tokens_not_num_used_tokens(self): - budget = DPBudget(dp_size=2) + budget = DPBudget(num_dp_ranks=2) budget.update_budget( [ _load( @@ -90,7 +90,7 @@ def test_maps_num_total_tokens_not_num_used_tokens(self): self.assertEqual(budget.total_tokens, [150, 80]) def test_partial_update_only_affects_reported_rank(self): - budget = DPBudget(dp_size=3) + budget = DPBudget(num_dp_ranks=3) budget.update_budget( [ _load( @@ -123,7 +123,7 @@ class TestDPBudgetDispatch(CustomTestCase): """DPBudget.dispatch picks a rank from current state and updates counters.""" def test_total_requests_dispatch_picks_min_and_increments(self): - budget = DPBudget(dp_size=3) + budget = DPBudget(num_dp_ranks=3) budget.total_requests = [4, 2, 7] rank = budget.dispatch(LoadBalanceMethod.TOTAL_REQUESTS) self.assertEqual(rank, 1) @@ -134,7 +134,7 @@ def test_total_requests_dispatch_picks_min_and_increments(self): ) def test_total_tokens_dispatch_applies_estimated_tokens(self): - budget = DPBudget(dp_size=3) + budget = DPBudget(num_dp_ranks=3) budget.total_tokens = [100, 50, 200] budget.total_requests = [0, 0, 0] rank = budget.dispatch(LoadBalanceMethod.TOTAL_TOKENS, estimated_tokens=30) @@ -151,7 +151,7 @@ def test_total_tokens_dispatch_applies_estimated_tokens(self): ) def test_total_tokens_tie_breaks_on_total_requests(self): - budget = DPBudget(dp_size=3) + budget = DPBudget(num_dp_ranks=3) budget.total_tokens = [50, 50, 50] budget.total_requests = [4, 2, 7] rank = budget.dispatch(LoadBalanceMethod.TOTAL_TOKENS, estimated_tokens=10) @@ -162,14 +162,14 @@ def test_total_tokens_tie_breaks_on_total_requests(self): def test_dispatch_returns_none_for_methods_not_handled(self): """Round-robin and follow_bootstrap_room dispatch elsewhere; DPBudget only handles the load-aware variants.""" - budget = DPBudget(dp_size=3) + budget = DPBudget(num_dp_ranks=3) self.assertIsNone(budget.dispatch(LoadBalanceMethod.ROUND_ROBIN)) self.assertIsNone(budget.dispatch(LoadBalanceMethod.FOLLOW_BOOTSTRAP_ROOM)) class TestRoundRobinScheduler(CustomTestCase): def test_cycles_through_active_workers_in_order(self): - ctl = _make_controller(dp_size=4) + ctl = _make_controller(num_dp_ranks=4) for _ in range(8): ctl.round_robin_scheduler(_req()) # 8 reqs across 4 active workers — 2 each, in round-robin order @@ -177,14 +177,14 @@ def test_cycles_through_active_workers_in_order(self): self.assertEqual(worker.send_pyobj.call_count, 2, f"worker {i} call count") def test_first_dispatch_picks_worker_zero(self): - ctl = _make_controller(dp_size=4) + ctl = _make_controller(num_dp_ranks=4) ctl.round_robin_scheduler(_req()) ctl.workers[0].send_pyobj.assert_called_once() for i in (1, 2, 3): ctl.workers[i].send_pyobj.assert_not_called() def test_skips_inactive_workers(self): - ctl = _make_controller(dp_size=4) + ctl = _make_controller(num_dp_ranks=4) ctl.status[1] = False ctl.status[3] = False for _ in range(6): @@ -197,7 +197,7 @@ def test_skips_inactive_workers(self): def test_routed_dp_rank_bypasses_counter(self): """External dp-rank routing must not advance the counter.""" - ctl = _make_controller(dp_size=4) + ctl = _make_controller(num_dp_ranks=4) ctl.round_robin_scheduler(_req(routed_dp_rank=2)) ctl.workers[2].send_pyobj.assert_called_once() self.assertEqual( @@ -212,7 +212,7 @@ def test_routed_dp_rank_bypasses_counter(self): class TestFollowBootstrapRoomScheduler(CustomTestCase): def test_dispatches_by_bootstrap_room_modulo(self): - ctl = _make_controller(dp_size=4) + ctl = _make_controller(num_dp_ranks=4) for room, expected_rank in [ (0, 0), (1, 1), @@ -225,12 +225,12 @@ def test_dispatches_by_bootstrap_room_modulo(self): ctl.workers[expected_rank].send_pyobj.assert_called() def test_requires_bootstrap_room(self): - ctl = _make_controller(dp_size=4) + ctl = _make_controller(num_dp_ranks=4) with self.assertRaises(AssertionError): ctl.follow_bootstrap_room_scheduler(_req(bootstrap_room=None)) def test_routed_dp_rank_bypasses_bootstrap_room(self): - ctl = _make_controller(dp_size=4) + ctl = _make_controller(num_dp_ranks=4) ctl.follow_bootstrap_room_scheduler(_req(routed_dp_rank=3, bootstrap_room=1)) ctl.workers[3].send_pyobj.assert_called_once() ctl.workers[1].send_pyobj.assert_not_called() @@ -238,7 +238,7 @@ def test_routed_dp_rank_bypasses_bootstrap_room(self): class TestTotalRequestsScheduler(CustomTestCase): def test_dispatches_to_min_request_worker(self): - ctl = _make_controller(dp_size=4) + ctl = _make_controller(num_dp_ranks=4) ctl.dp_budget.total_requests = [5, 3, 1, 4] ctl.total_requests_scheduler(_req()) ctl.workers[2].send_pyobj.assert_called_once() @@ -251,7 +251,7 @@ def test_dispatches_to_min_request_worker(self): ) def test_routed_dp_rank_bypasses_budget(self): - ctl = _make_controller(dp_size=4) + ctl = _make_controller(num_dp_ranks=4) ctl.dp_budget.total_requests = [5, 3, 1, 4] ctl.total_requests_scheduler(_req(routed_dp_rank=0)) ctl.workers[0].send_pyobj.assert_called_once() @@ -271,7 +271,7 @@ class TestStatusAwarenessInconsistency(CustomTestCase): this test will fail and force a reviewer to confirm intent.""" def test_total_requests_ignores_status(self): - ctl = _make_controller(dp_size=4) + ctl = _make_controller(num_dp_ranks=4) # Worker 2 is the global minimum AND marked inactive. ctl.dp_budget.total_requests = [5, 3, 1, 4] ctl.status[2] = False @@ -282,15 +282,15 @@ def test_total_requests_ignores_status(self): class TestRefreshLoadBudgetThrottle(CustomTestCase): @staticmethod - def _controller_with_reader(dp_size, snapshots): - ctl = _make_controller(dp_size) + def _controller_with_reader(num_dp_ranks, snapshots): + ctl = _make_controller(num_dp_ranks) ctl.load_snapshot_reader = MagicMock() ctl.load_snapshot_reader.read_all.return_value = snapshots return ctl def test_throttled_refresh_spreads_a_burst_across_ranks(self): idle = [_load(dp_rank=i, timestamp=1.0, num_total_tokens=0) for i in range(4)] - ctl = self._controller_with_reader(dp_size=4, snapshots=idle) + ctl = self._controller_with_reader(num_dp_ranks=4, snapshots=idle) # A refresh stamp in the future keeps every call inside the window, so # the burst runs entirely on speculative counters. ctl._last_refresh_time = time.perf_counter() + 3600.0 @@ -315,7 +315,7 @@ def test_refresh_outside_window_overwrites_speculative_increments(self): _load(dp_rank=0, timestamp=2.0, num_total_tokens=10), _load(dp_rank=1, timestamp=2.0, num_total_tokens=20), ] - ctl = self._controller_with_reader(dp_size=2, snapshots=reported) + ctl = self._controller_with_reader(num_dp_ranks=2, snapshots=reported) ctl._last_refresh_time = 0.0 # window has long passed ctl.dp_budget.total_tokens = [999, 999] @@ -331,7 +331,7 @@ def test_refresh_outside_window_overwrites_speculative_increments(self): def test_unchanged_snapshot_does_not_reset_the_burst(self): frozen = [_load(dp_rank=i, timestamp=1.0, num_total_tokens=0) for i in range(2)] - ctl = self._controller_with_reader(dp_size=2, snapshots=frozen) + ctl = self._controller_with_reader(num_dp_ranks=2, snapshots=frozen) ctl._last_refresh_time = 0.0 ctl.refresh_load_budget() # adopts timestamp 1.0 diff --git a/test/registered/unit/managers/test_load_snapshot_backends.py b/test/registered/unit/managers/test_load_snapshot_backends.py index e93bbfcb8f9a..0b2bfbf1fd43 100644 --- a/test/registered/unit/managers/test_load_snapshot_backends.py +++ b/test/registered/unit/managers/test_load_snapshot_backends.py @@ -7,6 +7,7 @@ from types import SimpleNamespace from unittest import mock +from sglang.srt.environ import envs from sglang.srt.managers.load_snapshot import ( LoadSnapshot, ShmLoadSnapshotReader, @@ -263,7 +264,7 @@ def _publish(self, **fields): self.addCleanup(override.restore) def test_shm_mode(self): - self._publish(enable_dp_attention=False, nnodes=1, dp_size=1) + self._publish(nnodes=1, dp_size=1) port_args = SimpleNamespace(instance_id="test_shm_factory") writer = create_load_snapshot_writer(port_args, dp_size=1, dp_rank=0) self.assertIsInstance(writer, ShmLoadSnapshotWriter) @@ -278,7 +279,7 @@ def test_shm_mode(self): os.unlink(path) def test_zmq_mode_via_env(self): - self._publish(enable_dp_attention=False, nnodes=1, dp_size=1) + self._publish(nnodes=1, dp_size=1) port_args = SimpleNamespace(instance_id="test_zmq_factory") os.environ["SGLANG_LOAD_SNAPSHOT_USE_ZMQ"] = "1" try: @@ -292,7 +293,7 @@ def test_zmq_mode_via_env(self): del os.environ["SGLANG_LOAD_SNAPSHOT_USE_ZMQ"] def test_should_use_zmq_multinode_dp_attention(self): - self._publish(enable_dp_attention=True, nnodes=2, dp_size=2) + self._publish(attn_dp_size=2, nnodes=2) self.assertTrue(should_use_zmq()) @@ -304,10 +305,9 @@ class TestZmqReaderOwner(CustomTestCase): def _owners(self, **overrides): """Publish a config and return the callers that claim the socket.""" fields = dict( - enable_dp_attention=True, nnodes=2, node_rank=0, - dp_size=1, + attn_dp_size=1, load_balance_method="round_robin", tokenizer_worker_num=1, ) @@ -320,38 +320,43 @@ def _owners(self, **overrides): override.restore() def test_zmq_disabled_no_owner(self): - self.assertEqual(self._owners(enable_dp_attention=False, nnodes=1), set()) + self.assertEqual(self._owners(nnodes=1), set()) def test_non_zero_node_rank_no_owner(self): self.assertEqual( - self._owners(node_rank=1, dp_size=4, tokenizer_worker_num=8), set() + self._owners(node_rank=1, attn_dp_size=4, tokenizer_worker_num=8), set() ) def test_tokenizer_manager_owns_when_dp1(self): - self.assertEqual(self._owners(dp_size=1), {"TokenizerManager"}) + # A single DP rank runs zmq mode only when it is forced. + with envs.SGLANG_LOAD_SNAPSHOT_USE_ZMQ.override(True): + self.assertEqual(self._owners(attn_dp_size=1), {"TokenizerManager"}) def test_multi_tokenizer_router_owns_in_multi_tokenizer_dp1(self): - self.assertEqual( - self._owners(dp_size=1, tokenizer_worker_num=8), {"MultiTokenizerRouter"} - ) + with envs.SGLANG_LOAD_SNAPSHOT_USE_ZMQ.override(True): + self.assertEqual( + self._owners(attn_dp_size=1, tokenizer_worker_num=8), + {"MultiTokenizerRouter"}, + ) def test_multi_tokenizer_router_owns_in_multi_tokenizer_round_robin(self): self.assertEqual( - self._owners(dp_size=4, tokenizer_worker_num=8), {"MultiTokenizerRouter"} + self._owners(attn_dp_size=4, tokenizer_worker_num=8), + {"MultiTokenizerRouter"}, ) def test_data_parallel_controller_owns_load_aware(self): for method in ("total_tokens", "total_requests"): self.assertEqual( self._owners( - dp_size=4, tokenizer_worker_num=8, load_balance_method=method + attn_dp_size=4, tokenizer_worker_num=8, load_balance_method=method ), {"DataParallelController"}, ) def test_tokenizer_manager_owns_dp4_round_robin(self): self.assertEqual( - self._owners(dp_size=4, tokenizer_worker_num=1), {"TokenizerManager"} + self._owners(attn_dp_size=4, tokenizer_worker_num=1), {"TokenizerManager"} ) def test_the_controller_answers_within_its_audited_namespaces(self): @@ -365,10 +370,9 @@ def test_the_controller_answers_within_its_audited_namespaces(self): import sglang.srt.runtime_context as rc fields = dict( - enable_dp_attention=True, nnodes=2, node_rank=0, - dp_size=4, + attn_dp_size=4, load_balance_method="total_tokens", tokenizer_worker_num=8, ) @@ -382,17 +386,19 @@ def test_the_controller_answers_within_its_audited_namespaces(self): self.assertTrue(zmq_reader_owner("DataParallelController")) def test_at_most_one_owner_across_configs(self): - for dp_size in (1, 4): - for tw in (1, 8): - for method in ("round_robin", "total_tokens", "total_requests"): - for node_rank in (0, 1): - owners = self._owners( - dp_size=dp_size, - tokenizer_worker_num=tw, - load_balance_method=method, - node_rank=node_rank, - ) - self.assertLessEqual(len(owners), 1, owners) + # Forced zmq mode puts a single DP rank in the sweep too. + with envs.SGLANG_LOAD_SNAPSHOT_USE_ZMQ.override(True): + for attn_dp_size in (1, 4): + for tw in (1, 8): + for method in ("round_robin", "total_tokens", "total_requests"): + for node_rank in (0, 1): + owners = self._owners( + attn_dp_size=attn_dp_size, + tokenizer_worker_num=tw, + load_balance_method=method, + node_rank=node_rank, + ) + self.assertLessEqual(len(owners), 1, owners) class TestZmqAddr(CustomTestCase): diff --git a/test/registered/unit/managers/test_loadstat_wire.py b/test/registered/unit/managers/test_loadstat_wire.py index 02a55f6643c3..15bf6d3e0f65 100644 --- a/test/registered/unit/managers/test_loadstat_wire.py +++ b/test/registered/unit/managers/test_loadstat_wire.py @@ -153,8 +153,7 @@ def test_pure_dp_keys_the_load_port_by_dp_rank(self): def test_dp_attention_keys_the_load_port_by_attn_dp_rank(self): _, open_sock = self._build( tp_size=4, - dp_size=4, - enable_dp_attention=True, + attn_dp_size=4, ranks={"world_rank": 3, "dp_rank": 0}, ) open_sock.assert_called_once_with("tcp://*:5564") diff --git a/test/registered/unit/managers/test_mm_shm_error_consensus.py b/test/registered/unit/managers/test_mm_shm_error_consensus.py index c7fee26d6205..26e647043c7d 100644 --- a/test/registered/unit/managers/test_mm_shm_error_consensus.py +++ b/test/registered/unit/managers/test_mm_shm_error_consensus.py @@ -121,7 +121,7 @@ def _run_consensus_rank(rank: int, world_size: int, init_file: str) -> None: ) try: req = _request(_failed_pointer() if rank == 1 else _successful_pointer()) - parallel = SimpleNamespace(enable_dp_attention=False, tp_size=world_size) + parallel = SimpleNamespace(attn_dp_enabled=False, tp_size=world_size) receiver = _receiver() object.__setattr__(receiver, "tp_cpu_group", torch.distributed.group.WORLD) with ( @@ -169,7 +169,7 @@ def _run_image_receiver(rank, init_file, pipe): ), patch( "sglang.srt.managers.scheduler_components.request_receiver.get_parallel", - return_value=SimpleNamespace(enable_dp_attention=False, tp_size=2), + return_value=SimpleNamespace(attn_dp_enabled=False, tp_size=2), ), ): for base in [30, 90]: @@ -372,7 +372,7 @@ def test_real_gloo_group_propagates_one_rank_failure(self): def test_local_materialization_failure_becomes_request_error(self): req = _request(_failed_pointer()) - parallel = SimpleNamespace(enable_dp_attention=False, tp_size=1) + parallel = SimpleNamespace(attn_dp_enabled=False, tp_size=1) with ( patch( @@ -396,7 +396,7 @@ def test_local_materialization_failure_becomes_request_error(self): def test_peer_failure_rejects_the_local_request(self): req = _request(torch.zeros(1)) - parallel = SimpleNamespace(enable_dp_attention=False, tp_size=2) + parallel = SimpleNamespace(attn_dp_enabled=False, tp_size=2) def inject_peer_failure(mask, **kwargs): mask.fill_(1) @@ -428,7 +428,7 @@ def test_batched_requests_only_reject_the_failed_item(self): failed_req = _request(torch.zeros(1), rid="failed") healthy_req = _request(torch.zeros(1), rid="healthy") batch = BatchTokenizedEmbeddingReqInput(batch=[failed_req, healthy_req]) - parallel = SimpleNamespace(enable_dp_attention=False, tp_size=1) + parallel = SimpleNamespace(attn_dp_enabled=False, tp_size=1) def materialize(req): if req.rid == "failed": diff --git a/test/registered/unit/managers/test_pp_cp_rank_offsets.py b/test/registered/unit/managers/test_pp_cp_rank_offsets.py index fed3110607de..b837ce0facbe 100644 --- a/test/registered/unit/managers/test_pp_cp_rank_offsets.py +++ b/test/registered/unit/managers/test_pp_cp_rank_offsets.py @@ -36,9 +36,8 @@ def _published_topology(): ranks={"world_rank": 12, "dp_rank": 1}, tp_size=8, pp_size=2, - dp_size=2, + attn_dp_size=2, attn_cp_size=2, - enable_dp_attention=True, ) @@ -63,7 +62,6 @@ def _make_receiver() -> SchedulerRequestReceiver: attn_cp_cpu_group=attn_cp_group, world_group=world_group, server_args=SimpleNamespace( - enable_dp_attention=True, enable_dp_attention_local_control_broadcast=False, ), model_config=SimpleNamespace(is_multimodal=False), @@ -81,7 +79,7 @@ def test_local_control_skips_full_tp_broadcast_for_decode_dp(self): receiver = _make_receiver() control_req = SimpleNamespace(kind="control") parallel = SimpleNamespace( - enable_dp_attention=True, + attn_dp_enabled=True, enable_dp_attention_local_control_broadcast=True, attn_tp_rank=0, attn_cp_rank=0, @@ -115,7 +113,7 @@ def test_default_control_uses_full_tp_broadcast(self): receiver = _make_receiver() control_req = SimpleNamespace(kind="control") parallel = SimpleNamespace( - enable_dp_attention=True, + attn_dp_enabled=True, enable_dp_attention_local_control_broadcast=False, attn_tp_rank=0, attn_cp_rank=0, diff --git a/test/registered/unit/managers/test_prefill_delayer_gather.py b/test/registered/unit/managers/test_prefill_delayer_gather.py index 8cadc7334b52..9c5177716d57 100644 --- a/test/registered/unit/managers/test_prefill_delayer_gather.py +++ b/test/registered/unit/managers/test_prefill_delayer_gather.py @@ -13,10 +13,8 @@ TOPOLOGIES = { "tp": dict(tp_size=4), "attn_cp": dict(tp_size=4, attn_cp_size=2), - "dp_attention": dict(tp_size=4, dp_size=2, enable_dp_attention=True), - "dp_attention_attn_cp": dict( - tp_size=8, dp_size=2, attn_cp_size=2, enable_dp_attention=True - ), + "dp_attention": dict(tp_size=4, attn_dp_size=2), + "dp_attention_attn_cp": dict(tp_size=8, attn_dp_size=2, attn_cp_size=2), } @@ -55,7 +53,7 @@ def test_gather_returns_the_first_rank_of_each_dp_group(self): waiting_queue_len=5, ) parallel = get_parallel() - dp_groups = parallel.dp_size if parallel.enable_dp_attention else 1 + dp_groups = parallel.num_dp_ranks if parallel.attn_dp_enabled else 1 ranks_per_dp_group = parallel.tp_size // dp_groups local = torch.tensor([1, 0, 3, 4, 5]) expected = torch.stack( diff --git a/test/registered/unit/managers/test_prefill_delayer_high_watermark.py b/test/registered/unit/managers/test_prefill_delayer_high_watermark.py index 2631f4e5ba3e..3cb6bba4cb90 100644 --- a/test/registered/unit/managers/test_prefill_delayer_high_watermark.py +++ b/test/registered/unit/managers/test_prefill_delayer_high_watermark.py @@ -33,8 +33,8 @@ def test_rejected_non_empty_attempt_advances_window(self): self.assertEqual(tracker.observe_attempt(10), 10) delayer = MagicMock() - delayer.enable_dp_attention = True - delayer.dp_size = 1 + delayer.attn_dp_enabled = True + delayer.num_dp_ranks = 1 delayer._metrics_collector = None delayer._debug_log_enabled = False delayer._negotiate_should_allow_prefill.return_value = _NegotiateOutput( @@ -72,8 +72,8 @@ def test_print_rejected_attempt_estimates_after_unusual_peak(self): self.assertEqual(tracker.observe_attempt(100), 100) delayer = MagicMock() - delayer.enable_dp_attention = True - delayer.dp_size = 1 + delayer.attn_dp_enabled = True + delayer.num_dp_ranks = 1 delayer._metrics_collector = None delayer._debug_log_enabled = False delayer._negotiate_should_allow_prefill.return_value = _NegotiateOutput( diff --git a/test/registered/unit/managers/test_scheduler_internal_state_world_size.py b/test/registered/unit/managers/test_scheduler_internal_state_world_size.py index a36b6a27cabf..741fb868d065 100644 --- a/test/registered/unit/managers/test_scheduler_internal_state_world_size.py +++ b/test/registered/unit/managers/test_scheduler_internal_state_world_size.py @@ -8,46 +8,45 @@ from sglang.srt.managers.io_struct import GetInternalStateReq from sglang.srt.managers.scheduler import Scheduler -from sglang.srt.runtime_context import get_context +from sglang.srt.runtime_context import get_context, get_parallel from sglang.srt.server_args import compute_world_size register_cpu_ci(est_time=12, suite="base-a-test-cpu") -def _shape( - *, tp_size: int, pp_size: int, dp_size: int, enable_dp_attention: bool -) -> dict: - """The four `parallel` leaves the world size is computed from.""" - return { - "tp_size": tp_size, - "pp_size": pp_size, - "dp_size": dp_size, - "enable_dp_attention": enable_dp_attention, - } +def _shape(*, tp_size: int, pp_size: int, dp_size: int) -> dict: + """The three `parallel` leaves the world size is computed from.""" + return {"tp_size": tp_size, "pp_size": pp_size, "dp_size": dp_size} class TestComputeWorldSize(unittest.TestCase): def test_a_single_gpu_server_holds_one_gpu(self): """The default shape has to come out as one, or every consumer is off by a factor.""" - shape = _shape(tp_size=1, pp_size=1, dp_size=1, enable_dp_attention=False) + shape = _shape(tp_size=1, pp_size=1, dp_size=1) self.assertEqual(compute_world_size(**shape), 1) def test_tensor_and_pipeline_stages_multiply(self): """Each (pp_rank, tp_rank) pair is its own scheduler process on its own gpu.""" - shape = _shape(tp_size=2, pp_size=3, dp_size=1, enable_dp_attention=False) + shape = _shape(tp_size=2, pp_size=3, dp_size=1) self.assertEqual(compute_world_size(**shape), 6) def test_plain_data_parallel_replicas_each_hold_their_own_gpus(self): """Without dp attention every replica launches a full tensor-parallel group of its own.""" - shape = _shape(tp_size=2, pp_size=1, dp_size=2, enable_dp_attention=False) + shape = _shape(tp_size=2, pp_size=1, dp_size=2) self.assertEqual(compute_world_size(**shape), 4) def test_data_parallel_attention_shares_the_tensor_parallel_gpus(self): """With dp attention the dp ranks live inside the tensor-parallel world, not beside it.""" - shape = _shape(tp_size=4, pp_size=1, dp_size=2, enable_dp_attention=True) + with get_context().override_server_args(tp_size=4, attn_dp_size=2): + parallel = get_parallel() + shape = _shape( + tp_size=parallel.tp_size, + pp_size=parallel.pp_size, + dp_size=parallel.dp_size, + ) self.assertEqual(compute_world_size(**shape), 4) @@ -88,7 +87,7 @@ def _get_internal_state(self, shape: dict) -> dict: def test_the_internal_state_reports_the_whole_server(self): """A consumer sizing an external fleet reads the gpus the server occupies, not the declared sizes.""" - shape = _shape(tp_size=2, pp_size=1, dp_size=2, enable_dp_attention=False) + shape = _shape(tp_size=2, pp_size=1, dp_size=2) internal_state = self._get_internal_state(shape) diff --git a/test/registered/unit/managers/test_scheduler_recv_skipper.py b/test/registered/unit/managers/test_scheduler_recv_skipper.py index e912ffabef6c..72abd82d91bf 100644 --- a/test/registered/unit/managers/test_scheduler_recv_skipper.py +++ b/test/registered/unit/managers/test_scheduler_recv_skipper.py @@ -15,12 +15,13 @@ register_cpu_ci(est_time=10, suite="base-a-test-cpu") -def _publish(case, interval, enable_dp_attention=False): +def _publish(case, interval, attn_dp_size=1): """Publish the config the skipper reads; the double is a published config, not an injected object.""" override = get_context().override_server_args( scheduler_recv_interval=interval, - enable_dp_attention=enable_dp_attention, + tp_size=attn_dp_size, + attn_dp_size=attn_dp_size, ) override.install() case.addCleanup(override.restore) @@ -40,8 +41,8 @@ def test_disabled_at_default_interval(self): self.assertIsNone(SchedulerRecvSkipper.maybe_create()) def test_enabled_under_dp_attention(self): - # Regression: the constructor used to assert `not enable_dp_attention`. - _publish(self, 50, enable_dp_attention=True) + # Regression: the constructor used to reject DP attention. + _publish(self, 50, attn_dp_size=2) skipper = SchedulerRecvSkipper.maybe_create() self.assertIsNotNone(skipper) @@ -69,12 +70,12 @@ def test_prefill_triggers_recv_immediately(self): def test_dp_uses_synced_mode_not_local(self): # Local EXTEND (weight 1000) must be ignored in favor of the synced # DECODE (weight 1); a recv here would mean the local mode leaked in. - _publish(self, 50, enable_dp_attention=True) + _publish(self, 50, attn_dp_size=2) skipper = SchedulerRecvSkipper.maybe_create() self.assertFalse(skipper.handle(_batch(ForwardMode.EXTEND, ForwardMode.DECODE))) def test_dp_synced_extend_triggers_recv(self): - _publish(self, 50, enable_dp_attention=True) + _publish(self, 50, attn_dp_size=2) skipper = SchedulerRecvSkipper.maybe_create() self.assertTrue(skipper.handle(_batch(ForwardMode.IDLE, ForwardMode.EXTEND))) diff --git a/test/registered/unit/model_executor/model_runner_components/test_startup_weight_load.py b/test/registered/unit/model_executor/model_runner_components/test_startup_weight_load.py index 2163f5a3d238..a4c7908c4089 100644 --- a/test/registered/unit/model_executor/model_runner_components/test_startup_weight_load.py +++ b/test/registered/unit/model_executor/model_runner_components/test_startup_weight_load.py @@ -65,7 +65,7 @@ def _make_options(**overrides): attn_cp_size=1, dcp_size=1, pp_size=1, - dp_size=1, + num_dp_ranks=1, ep_size=1, cpu_offload_gb=0, offload_group_size=-1, diff --git a/test/registered/unit/model_executor/test_mlp_sync_pad_unpad.py b/test/registered/unit/model_executor/test_mlp_sync_pad_unpad.py index 68449056aa68..395791b2b787 100644 --- a/test/registered/unit/model_executor/test_mlp_sync_pad_unpad.py +++ b/test/registered/unit/model_executor/test_mlp_sync_pad_unpad.py @@ -115,7 +115,7 @@ def test_init_mlp_sync_metadata_scales_speculative_request_width(self): def test_draft_graph_gate_has_an_independent_dp_vote(self): sync_info = MLPSyncBatchInfo( - dp_size=1, + num_dp_ranks=1, tp_size=1, cp_size=1, num_tokens=1, @@ -306,8 +306,7 @@ class TestDraftScopeMlpSync(CustomTestCase): def setUp(self): override = get_context().override_server_args( tp_size=4, - dp_size=4, - enable_dp_attention=True, + attn_dp_size=4, cuda_graph_config=CudaGraphConfig(prefill=PhaseConfig(bs=[])), ) override.install() diff --git a/test/registered/unit/model_loader/test_weight_cache_protocol.py b/test/registered/unit/model_loader/test_weight_cache_protocol.py index f7e8d6b1c236..cbf8fec665d2 100644 --- a/test/registered/unit/model_loader/test_weight_cache_protocol.py +++ b/test/registered/unit/model_loader/test_weight_cache_protocol.py @@ -66,7 +66,7 @@ def _make_cache_config(**overrides) -> CacheConfig: moe_dp_size=1, moe_dp_rank=0, moe_ep_rank=0, - enable_dp_attention=False, + attn_dp_size=1, enable_dp_lm_head=False, attn_cp_size=1, moe_dense_tp_size=None, @@ -173,7 +173,7 @@ def test_any_field_difference_breaks_match(self): ("tp_rank", 1), ("moe_dp_rank", 1), ("moe_ep_rank", 1), - ("enable_dp_attention", True), + ("attn_dp_size", 2), ("moe_dense_tp_size", 1), ("moe_a2a_backend", "mooncake"), ("dtype", "torch.bfloat16"), @@ -294,10 +294,9 @@ def test_spawn_forwards_complete_server_args_without_projection(self): model_path="/models/demo", tp_size=8, pp_size=1, - dp_size=8, + attn_dp_size=8, ep_size=8, moe_dp_size=2, - enable_dp_attention=True, enable_dp_lm_head=True, attn_cp_size=2, moe_dense_tp_size=1, diff --git a/test/registered/unit/multimodal/rust/shared/test_rust_server_extension.py b/test/registered/unit/multimodal/rust/shared/test_rust_server_extension.py index 42bb14029bff..40ac50860469 100644 --- a/test/registered/unit/multimodal/rust/shared/test_rust_server_extension.py +++ b/test/registered/unit/multimodal/rust/shared/test_rust_server_extension.py @@ -45,7 +45,7 @@ def _start_multimodal(self, scheduler): return_value=SimpleNamespace( nnodes=1, pp_size=1, - dp_size=2, + num_dp_ranks=2, attn_dp_rank=1, tp_size=2, tp_rank=1, diff --git a/test/registered/unit/server_args/test_attn_dp_size.py b/test/registered/unit/server_args/test_attn_dp_size.py new file mode 100644 index 000000000000..6a70050f6e2a --- /dev/null +++ b/test/registered/unit/server_args/test_attn_dp_size.py @@ -0,0 +1,166 @@ +"""--attn-dp-size sets the attention data-parallel width; the deprecated +`--dp-size N --enable-dp-attention` spelling resolves to `--attn-dp-size N`.""" + +import argparse +import json +import logging +import os +import shutil +import tempfile +import unittest + +from sglang.srt.arg_groups.overrides import resolution_result +from sglang.srt.runtime_context import get_context, get_parallel +from sglang.srt.server_args import ServerArgs +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase, published_topology + +register_cpu_ci(est_time=20, suite="base-a-test-cpu") + +# A real config.json, so resolution runs past the dummy-model early return. +_MINI_CONFIG = { + "architectures": ["LlamaForCausalLM"], + "model_type": "llama", + "hidden_size": 16, + "intermediate_size": 32, + "num_attention_heads": 8, + "num_key_value_heads": 8, + "num_hidden_layers": 2, + "vocab_size": 128, + "max_position_embeddings": 2048, +} + + +class TestAttnDpSize(CustomTestCase): + def setUp(self): + super().setUp() + environment = dict(os.environ) + + def restore(): + os.environ.clear() + os.environ.update(environment) + + self.addCleanup(restore) + self.path = tempfile.mkdtemp(prefix="attn_dp_size_") + self.addCleanup(shutil.rmtree, self.path, ignore_errors=True) + with open(os.path.join(self.path, "config.json"), "w") as handle: + json.dump(_MINI_CONFIG, handle) + + def resolve(self, **fields): + server_args = ServerArgs( + model_path=self.path, device="cuda", random_seed=42, **fields + ) + server_args.resolve_once() + return server_args + + def layout(self, server_args): + return tuple( + resolution_result(server_args, name) + for name in ("dp_size", "attn_dp_size", "enable_dp_attention") + ) + + def test_the_deprecated_spelling_moves_the_width_to_attn_dp_size(self): + for tp_size, width in ((2, 2), (4, 4), (4, 2), (8, 4)): + with self.subTest(tp_size=tp_size, width=width): + legacy = self.resolve( + tp_size=tp_size, dp_size=width, enable_dp_attention=True + ) + new = self.resolve(tp_size=tp_size, attn_dp_size=width) + self.assertEqual(self.layout(legacy), (1, width, False)) + self.assertEqual(legacy.resolved_dict(), new.resolved_dict()) + + def test_without_attention_dp_the_width_is_one(self): + for fields in ( + {"tp_size": 2}, + {"tp_size": 2, "dp_size": 2}, + # One DP group under the deprecated flag is no attention DP. + {"tp_size": 2, "dp_size": 1, "enable_dp_attention": True}, + ): + with self.subTest(**fields): + server_args = self.resolve(**fields) + self.assertEqual( + self.layout(server_args), (fields.get("dp_size", 1), 1, False) + ) + + def test_rejected_layouts(self): + for fields, message in ( + ( + {"tp_size": 4, "dp_size": 2, "attn_dp_size": 2}, + "replicas combined with attention data parallelism", + ), + ( + {"tp_size": 4, "attn_dp_size": 2, "enable_dp_attention": True}, + "pass --attn-dp-size alone", + ), + ({"tp_size": 2, "attn_dp_size": 0}, "must be positive"), + ): + with self.subTest(**fields): + with self.assertRaisesRegex(ValueError, message): + self.resolve(**fields) + + def test_both_cli_spellings_parse(self): + parser = argparse.ArgumentParser() + ServerArgs.add_cli_args(parser) + for flag in ("--attn-dp-size", "--attention-data-parallel-size"): + with self.subTest(flag=flag): + raw = parser.parse_args(["--model-path", self.path, flag, "2"]) + self.assertEqual(ServerArgs.from_cli_args(raw).attn_dp_size, 2) + + def test_the_deprecated_flag_warns_with_its_replacement(self): + parser = argparse.ArgumentParser() + ServerArgs.add_cli_args(parser) + argv = ["--model-path", self.path, "--dp-size", "2", "--enable-dp-attention"] + with self.assertLogs( + "sglang.srt.arg_groups.argparse_actions", "WARNING" + ) as logs: + raw = parser.parse_args(argv) + self.assertEqual(len(logs.output), 1) + self.assertIn("--attn-dp-size", logs.output[0]) + self.assertTrue(ServerArgs.from_cli_args(raw).enable_dp_attention) + + +class TestPublishedAttnDpSize(CustomTestCase): + """What the runtime reads: the attention-DP switch and the DP rank count.""" + + def test_the_published_layout(self): + for fields, (attn_dp_size, num_dp_ranks) in ( + ({"tp_size": 4, "attn_dp_size": 2}, (2, 2)), + ({"tp_size": 4, "dp_size": 2, "enable_dp_attention": True}, (2, 2)), + ({"tp_size": 2, "dp_size": 2}, (1, 2)), + ({"tp_size": 2}, (1, 1)), + ): + with self.subTest(**fields): + with published_topology(**fields): + parallel = get_parallel() + self.assertEqual(parallel.attn_dp_size, attn_dp_size) + self.assertEqual(parallel.num_dp_ranks, num_dp_ranks) + self.assertEqual(parallel.attn_dp_enabled, attn_dp_size > 1) + self.assertEqual( + parallel.attn_tp_size, fields["tp_size"] // attn_dp_size + ) + + def test_a_scale_joiner_runs_attention_dp_at_width_one(self): + with published_topology(tp_size=1, attn_dp_size=1, ep_join_mode="scale"): + parallel = get_parallel() + self.assertEqual(parallel.attn_dp_size, 1) + self.assertEqual(parallel.num_dp_ranks, 1) + self.assertTrue(parallel.attn_dp_enabled) + + def test_an_overridden_layout_publishes_the_same_leaves(self): + for fields, width in ( + ({"tp_size": 4, "attn_dp_size": 4}, 4), + ({"tp_size": 4, "attn_dp_size": 2}, 2), + ({"tp_size": 4, "dp_size": 2}, 1), + ): + with self.subTest(**fields): + with get_context().override_server_args(**fields): + self.assertEqual(get_parallel().attn_dp_size, width) + self.assertEqual(get_parallel().attn_dp_enabled, width > 1) + self.assertEqual( + get_parallel().num_dp_ranks, fields.get("dp_size", width) + ) + + +if __name__ == "__main__": + logging.basicConfig() + unittest.main() diff --git a/test/registered/unit/server_args/test_resolution_declarations.py b/test/registered/unit/server_args/test_resolution_declarations.py index 0fb39a9a2161..7704a257e395 100644 --- a/test/registered/unit/server_args/test_resolution_declarations.py +++ b/test/registered/unit/server_args/test_resolution_declarations.py @@ -58,7 +58,7 @@ "speculative_eagle_topk": 1, "speculative_num_draft_tokens": 4, }, - {"dp_size": 2, "tp_size": 2, "enable_dp_attention": True}, + {"attn_dp_size": 2, "tp_size": 2}, {"enable_hierarchical_cache": True}, {"disaggregation_mode": "prefill"}, {"enable_lora": True, "max_lora_rank": 16}, @@ -77,6 +77,7 @@ "_speculative_draft_quantization_explicitly_set", "allowed_media_domains", "attention_backend", + "attn_dp_size", "chunked_prefill_size", "cuda_graph_config", "custom_weight_loader", @@ -84,7 +85,6 @@ "disable_cuda_graph", "disaggregation_ib_device", "dp_size", - "enable_dp_attention", "enable_dp_attention_local_control_broadcast", "enable_dp_lm_head", "enable_flashinfer_allreduce_fusion", @@ -761,7 +761,7 @@ def test_no_entry_changes_after_it_is_recorded(self): ("disaggregation", {"disaggregation_mode": "prefill"}), ("deterministic", {"enable_deterministic_inference": True}), ("speculative", {"speculative_algorithm": "EAGLE"}), - ("dp_attention", {"tp_size": 2, "dp_size": 2, "enable_dp_attention": True}), + ("dp_attention", {"tp_size": 2, "attn_dp_size": 2}), ): with self.subTest(shape=label): server_args, recorded = self._resolve_recording_each_entry(**supplied) diff --git a/test/registered/unit/server_args/test_resolution_is_reproducible.py b/test/registered/unit/server_args/test_resolution_is_reproducible.py index 4ed946b1ce55..b55b6438c6b6 100644 --- a/test/registered/unit/server_args/test_resolution_is_reproducible.py +++ b/test/registered/unit/server_args/test_resolution_is_reproducible.py @@ -501,9 +501,8 @@ def _resolved(self): server_args = ServerArgs( model_path=config_dir, device="cuda", - dp_size=2, + attn_dp_size=2, tp_size=2, - enable_dp_attention=True, random_seed=42, ) server_args.resolve_once() diff --git a/test/registered/unit/server_args/test_server_args.py b/test/registered/unit/server_args/test_server_args.py index 4949abe03aa6..be2872e2f837 100644 --- a/test/registered/unit/server_args/test_server_args.py +++ b/test/registered/unit/server_args/test_server_args.py @@ -1523,8 +1523,7 @@ def _make_args( moe_a2a_backend="flashinfer", moe_runner_backend=runner_backend, flashinfer_a2a_dispatch_type=dispatch_type, - enable_dp_attention=True, - dp_size=4, + attn_dp_size=4, tp_size=4, ) server_args._model_config = SimpleNamespace(nvfp4_moe_meta=None) @@ -1678,8 +1677,7 @@ def _make_args( quantization=quantization, moe_a2a_backend="flashinfer_megamoe", moe_runner_backend="flashinfer_megamoe", - enable_dp_attention=True, - dp_size=4, + attn_dp_size=4, tp_size=4, ) server_args._model_config = SimpleNamespace( @@ -1766,7 +1764,6 @@ def test_init_new_standard_case(self, mock_temp_file): server_args = ServerArgs(model_path="dummy") server_args.port = 30000 server_args.nccl_port = None - server_args.enable_dp_attention = False port_args = PortArgs.init_new(server_args) @@ -1781,7 +1778,6 @@ def test_init_new_builds_decoupled_spec_ipc_config(self, mock_temp_file): server_args = ServerArgs(model_path="dummy") server_args.nccl_port = None - server_args.enable_dp_attention = False server_args.decoupled_spec_role = "verifier" server_args.decoupled_spec_bind_endpoint = "ipc:///tmp/v" server_args.decoupled_spec_connect_endpoints = ["ipc:///tmp/d"] @@ -1804,7 +1800,6 @@ def test_init_new_no_decoupled_config_when_role_null(self, mock_temp_file): server_args = ServerArgs(model_path="dummy") server_args.nccl_port = None - server_args.enable_dp_attention = False # decoupled_spec_role defaults to "null" port_args = PortArgs.init_new(server_args) @@ -1814,7 +1809,6 @@ def test_init_new_no_decoupled_config_when_role_null(self, mock_temp_file): def test_init_new_decoupled_role_requires_endpoints(self): server_args = ServerArgs(model_path="dummy") server_args.nccl_port = None - server_args.enable_dp_attention = False server_args.decoupled_spec_role = "drafter" # endpoints intentionally left as their None defaults @@ -1826,7 +1820,7 @@ def test_init_new_with_single_node_dp_attention(self): server_args = ServerArgs(model_path="dummy") server_args.port = 30000 server_args.nccl_port = None - server_args.enable_dp_attention = True + server_args.attn_dp_size = 2 server_args.nnodes = 1 server_args.dist_init_addr = None @@ -1843,7 +1837,7 @@ def test_init_new_with_dp_rank(self): server_args = ServerArgs(model_path="dummy") server_args.port = 30000 server_args.nccl_port = None - server_args.enable_dp_attention = True + server_args.attn_dp_size = 4 server_args.nnodes = 1 server_args.dist_init_addr = "192.168.1.1:25000" @@ -1861,7 +1855,7 @@ def test_init_new_with_ipv4_address(self): server_args.port = 30000 server_args.nccl_port = None - server_args.enable_dp_attention = True + server_args.attn_dp_size = 2 server_args.nnodes = 2 server_args.dist_init_addr = "192.168.1.1:25000" @@ -1879,7 +1873,7 @@ def test_init_new_with_malformed_ipv4_address(self): server_args.port = 30000 server_args.nccl_port = None - server_args.enable_dp_attention = True + server_args.attn_dp_size = 2 server_args.nnodes = 2 server_args.dist_init_addr = "192.168.1.1" @@ -1893,7 +1887,7 @@ def test_init_new_with_malformed_ipv4_address_invalid_port(self): server_args.port = 30000 server_args.nccl_port = None - server_args.enable_dp_attention = True + server_args.attn_dp_size = 2 server_args.nnodes = 2 server_args.dist_init_addr = "192.168.1.1:abc" @@ -2582,7 +2576,8 @@ def test_pp_spec_env_gate_allows_aggregate_and_rejects_pd(self): speculative_algorithm="EAGLE", disaggregation_mode="null", speculative_adaptive=False, - enable_dp_attention=False, + attn_dp_size=1, + ep_join_mode=None, ) with patch.object( validation_hook.envs.SGLANG_ENABLE_PP_SPEC, "get", return_value=True @@ -3249,8 +3244,7 @@ def test_dp_attention_divides_max_running_requests_per_rank(self): disaggregation_mode="decode", max_running_requests=256, tp_size=8, - dp_size=8, - enable_dp_attention=True, + attn_dp_size=8, ) with envs.SGLANG_DEEPEP_V2_NUM_MAX_DISPATCH_TOKENS_PER_RANK.override(128): validate_deepep_v2_dispatch_token_budget(args) @@ -3261,7 +3255,6 @@ def test_tp_only_max_running_requests_is_not_divided(self): max_running_requests=256, tp_size=8, dp_size=1, - enable_dp_attention=False, ) with envs.SGLANG_DEEPEP_V2_NUM_MAX_DISPATCH_TOKENS_PER_RANK.override(128): with self.assertRaisesRegex(ValueError, "decode CUDA graph"): @@ -3282,8 +3275,7 @@ def test_speculative_decode_width_is_included(self): speculative_algorithm="EAGLE", speculative_num_draft_tokens=8, max_running_requests=256, - dp_size=8, - enable_dp_attention=True, + attn_dp_size=8, ) with envs.SGLANG_DEEPEP_V2_NUM_MAX_DISPATCH_TOKENS_PER_RANK.override(128): with self.assertRaisesRegex(ValueError, "tokens/request=8"): @@ -3296,8 +3288,7 @@ def test_adaptive_speculative_uses_widest_candidate(self): speculative_num_draft_tokens=4, speculative_adaptive=True, max_running_requests=128, - dp_size=8, - enable_dp_attention=True, + attn_dp_size=8, ) with patch( "sglang.srt.arg_groups.moe_hook.max_speculative_num_draft_tokens", @@ -3663,7 +3654,7 @@ class TestTwoBatchOverlapBackend(CustomTestCase): With no EP a2a backend (moe_a2a_backend='none'), --enable-two-batch-overlap is only valid on the DeepSeek-V4 non-EP DP TP-MoE path (overlapping the DP all_gatherv / reduce_scatterv with the other ubatch's compute), which - requires --enable-dp-attention. This replaced the removed opt-in + requires attention DP (--attn-dp-size). This replaced the removed opt-in SGLANG_ENABLE_DP_TBO env: enabling DP TBO now needs no extra flag. dummy-model short-circuits __post_init__, so the guard handler is invoked @@ -3673,26 +3664,25 @@ def _args(self, **overrides): args = ServerArgs(model_path="dummy") args.enable_two_batch_overlap = True args.moe_a2a_backend = "none" - args.enable_dp_attention = False for key, value in overrides.items(): setattr(args, key, value) return args def test_no_a2a_without_dp_attention_raises(self): - args = self._args(enable_dp_attention=False) - with self.assertRaisesRegex(ValueError, "enable-dp-attention"): + args = self._args() + with self.assertRaisesRegex(ValueError, "attn-dp-size"): check_two_batch_overlap(args) def test_no_a2a_with_dp_attention_ok(self): - # DP TBO path is valid: --enable-dp-attention + --enable-two-batch-overlap + # DP TBO path is valid: --attn-dp-size + --enable-two-batch-overlap # with a2a backend 'none' must NOT raise (no SGLANG_ENABLE_DP_TBO needed). - args = self._args(enable_dp_attention=True) + args = self._args(attn_dp_size=2) check_two_batch_overlap(args) def test_ep_a2a_backend_ok_without_dp_attention(self): # EP a2a path (e.g. deepep) overlaps dispatch/combine; the guard does not # require dp-attention there. - args = self._args(moe_a2a_backend="deepep", enable_dp_attention=False) + args = self._args(moe_a2a_backend="deepep") check_two_batch_overlap(args) @@ -3951,9 +3941,8 @@ def _resolve(self, **kwargs): # have settled by then; the dummy-model pipeline itself returns early. server_args = ServerArgs( model_path="dummy", - enable_dp_attention=True, + attn_dp_size=2, tp_size=2, - dp_size=2, chunked_prefill_size=8192, cuda_graph_config=CudaGraphConfig( prefill=PhaseConfig(backend=Backend.DISABLED) diff --git a/test/registered/unit/spec/test_draft_tp_context.py b/test/registered/unit/spec/test_draft_tp_context.py index d7ef2c33599e..1e1b6795c688 100644 --- a/test/registered/unit/spec/test_draft_tp_context.py +++ b/test/registered/unit/spec/test_draft_tp_context.py @@ -21,9 +21,7 @@ class TestDraftTpContext(CustomTestCase): """Draft work enters the attention-TP placement only when it owns attention.""" def setUp(self): - topology = published_topology( - tp_size=4, dp_size=2, enable_dp_attention=True, ranks=dict(dp_rank=0) - ) + topology = published_topology(tp_size=4, attn_dp_size=2, ranks=dict(dp_rank=0)) topology.__enter__() self.addCleanup(topology.__exit__, None, None, None) self.whole_tp = _group(world_size=4, rank=0) diff --git a/test/registered/unit/test_model_overrides.py b/test/registered/unit/test_model_overrides.py index 9036047bc8ee..e77d007ddc6b 100644 --- a/test/registered/unit/test_model_overrides.py +++ b/test/registered/unit/test_model_overrides.py @@ -62,6 +62,7 @@ def config(self, **options): cp_strategy=None, tp_size=4, dp_size=1, + attn_dp_size=1, enable_aiter_allreduce_fusion=False, ), **options, @@ -171,6 +172,8 @@ def test_server_args_whitelist_is_exactly_the_migrated_fields(self): "page_size", "moe_runner_backend", "quantization", + "dp_size", + "attn_dp_size", "enable_dp_attention", "enable_attn_tp_input_scattered", "enable_dp_lm_head", @@ -589,7 +592,7 @@ def _minicpm_overrides( prefill_attention_backend=None, decode_attention_backend=None, disaggregation_mode="null", - enable_dp_attention=False, + attn_dp_size=1, enable_hierarchical_cache=False, ): args = SimpleNamespace( @@ -597,7 +600,8 @@ def _minicpm_overrides( prefill_attention_backend=prefill_attention_backend, decode_attention_backend=decode_attention_backend, disaggregation_mode=disaggregation_mode, - enable_dp_attention=enable_dp_attention, + attn_dp_size=attn_dp_size, + ep_join_mode=None, enable_hierarchical_cache=enable_hierarchical_cache, ) mixer_types = [] @@ -651,7 +655,7 @@ def test_minicpm_rejects_dp_attention(self): ): self._minicpm_overrides( architecture, - enable_dp_attention=True, + attn_dp_size=2, ) def test_minicpm_rejects_hierarchical_cache_for_hybrid_models(self): @@ -685,7 +689,8 @@ def test_minicpm_overrides_use_config_capabilities(self): prefill_attention_backend=None, decode_attention_backend=None, disaggregation_mode="null", - enable_dp_attention=False, + attn_dp_size=1, + ep_join_mode=None, enable_hierarchical_cache=False, ) config = SimpleNamespace( @@ -2387,7 +2392,8 @@ def _view(arch="Qwen3MoeForCausalLM", **kw): defaults = dict( flashinfer_allreduce_fusion_backend=None, tp_size=2, - enable_dp_attention=False, + attn_dp_size=1, + ep_join_mode=None, nnodes=1, moe_a2a_backend="none", enforce_disable_flashinfer_allreduce_fusion=False, @@ -2423,9 +2429,7 @@ def _view(arch="Qwen3MoeForCausalLM", **kw): _flashinfer_allreduce_fusion_auto_enable(_view(tp_size=1)), {} ) self.assertEqual( - _flashinfer_allreduce_fusion_auto_enable( - _view(enable_dp_attention=True) - ), + _flashinfer_allreduce_fusion_auto_enable(_view(attn_dp_size=2)), {}, ) self.assertEqual( @@ -3408,16 +3412,26 @@ def test_data_parallelism_and_a2a_passes(self): self.assertEqual( _data_parallelism_defaults( - ResolvedView(SimpleNamespace(dp_size=1, ep_join_mode=None)) + ResolvedView( + SimpleNamespace(dp_size=1, attn_dp_size=1, ep_join_mode=None) + ) ), - {"enable_dp_attention": False, "enable_dp_lm_head": False}, - ) - self.assertEqual( - _data_parallelism_defaults( - ResolvedView(SimpleNamespace(dp_size=2, ep_join_mode=None)) - ), - {}, + {"enable_dp_lm_head": False}, ) + for dp_size, attn_dp_size in ((2, 1), (1, 2)): + with self.subTest(dp_size=dp_size, attn_dp_size=attn_dp_size): + self.assertEqual( + _data_parallelism_defaults( + ResolvedView( + SimpleNamespace( + dp_size=dp_size, + attn_dp_size=attn_dp_size, + ep_join_mode=None, + ) + ) + ), + {}, + ) self.assertEqual( _a2a_ep_size( @@ -3500,6 +3514,7 @@ def _args(**kw): cp_strategy="zigzag", tp_size=8, dp_size=1, + attn_dp_size=1, ep_size=1, moe_a2a_backend="none", kv_cache_dtype="auto", @@ -3511,21 +3526,23 @@ def _args(**kw): { "attention_backend": "dsa", "page_size": 64, - "enable_dp_attention": True, + "attn_dp_size": 1, + "dp_size": 1, "moe_dense_tp_size": 1, "moe_a2a_backend": "deepep", "ep_size": 8, "attn_cp_size": 8, }, ) - # interleave CP with dp>1 must assert + # interleave CP with attention DP must assert with self.assertRaises(AssertionError): _deepseek_family_overrides( _args( enable_prefill_cp=True, cp_strategy="interleave", tp_size=8, - dp_size=2, + dp_size=1, + attn_dp_size=2, ), None, ) diff --git a/test/registered/unit/test_runtime_context.py b/test/registered/unit/test_runtime_context.py index 5aedb9dc6fe9..462806429bd2 100644 --- a/test/registered/unit/test_runtime_context.py +++ b/test/registered/unit/test_runtime_context.py @@ -370,38 +370,38 @@ def test_it_matches_the_topology_init_for_every_shape(self): from sglang.srt.layers.dp_attention import compute_dp_attention_world_info shapes = [ - (8, 1, 1, False), - (8, 2, 1, True), - (8, 4, 1, True), - (8, 2, 2, True), - (16, 4, 2, True), + (8, 1, 1), + (8, 2, 1), + (8, 4, 1), + (8, 2, 2), + (16, 4, 2), ] - for tp_size, dp_size, attn_cp_size, dp_attn in shapes: + for tp_size, attn_dp_size, attn_cp_size in shapes: for tp_rank in range(tp_size): reset_context() publish( ServerArgs( model_path="dummy", tp_size=tp_size, - dp_size=dp_size, + attn_dp_size=attn_dp_size, attn_cp_size=attn_cp_size, - enable_dp_attention=dp_attn, ), role="test", ranks=SpawnRanks(world_rank=tp_rank), ) want_tp, _, want_dp, _ = compute_dp_attention_world_info( - dp_attn, tp_rank, tp_size, dp_size, attn_cp_size + tp_rank, tp_size, attn_dp_size, attn_cp_size + ) + msg = ( + f"tp={tp_size} attn_dp={attn_dp_size} cp={attn_cp_size} " + f"rank={tp_rank}" ) - msg = f"tp={tp_size} dp={dp_size} cp={attn_cp_size} rank={tp_rank}" self.assertEqual(get_parallel().attn_tp_rank, want_tp, msg) self.assertEqual(get_parallel().attn_dp_rank, want_dp, msg) def test_the_rank_reads_without_a_process_group(self): publish( - ServerArgs( - model_path="dummy", tp_size=8, dp_size=2, enable_dp_attention=True - ), + ServerArgs(model_path="dummy", tp_size=8, attn_dp_size=2), role="test", ranks=SpawnRanks(world_rank=5), ) @@ -485,9 +485,7 @@ def restore(): self.addCleanup(restore) publish( - ServerArgs( - model_path="dummy", tp_size=8, dp_size=8, enable_dp_attention=True - ), + ServerArgs(model_path="dummy", tp_size=8, attn_dp_size=8), role="test", ranks=SpawnRanks(world_rank=3), ) @@ -512,9 +510,7 @@ def test_a_scale_up_writes_no_width(self): self.addCleanup(setattr, dp_flags, "use_world_group_for_gather", saved_gather) self.addCleanup(reset_context) publish( - ServerArgs( - model_path="dummy", tp_size=8, dp_size=8, enable_dp_attention=True - ), + ServerArgs(model_path="dummy", tp_size=8, attn_dp_size=8), role="test", ranks=SpawnRanks(world_rank=3), ) @@ -630,9 +626,7 @@ def test_a_width_read_compiles_into_the_graph(self): reset_context() self.addCleanup(reset_context) publish( - ServerArgs( - model_path="dummy", tp_size=8, dp_size=2, enable_dp_attention=True - ), + ServerArgs(model_path="dummy", tp_size=8, attn_dp_size=2), role="test", ) @@ -1945,9 +1939,7 @@ def test_the_published_configuration_decides_the_widths(self): reset_context() self.addCleanup(reset_context) publish( - ServerArgs( - model_path="dummy", tp_size=8, dp_size=2, enable_dp_attention=True - ), + ServerArgs(model_path="dummy", tp_size=8, attn_dp_size=2), role="test", ) self.assertEqual(get_parallel().attn_tp_size, 4) @@ -2068,9 +2060,7 @@ def test_the_group_is_never_consulted(self): side_effect=AssertionError("the group must not be consulted"), ): publish( - ServerArgs( - model_path="dummy", tp_size=8, dp_size=2, enable_dp_attention=True - ), + ServerArgs(model_path="dummy", tp_size=8, attn_dp_size=2), role="test", ) self.assertEqual(get_parallel().attn_tp_size, 4) @@ -2108,9 +2098,9 @@ def test_the_rank_helper_agrees_with_the_override(self): widths from the same derivation `override_permanently`'s callers use.""" from sglang.srt.layers.dp_attention import compute_dp_attention_world_info - for tp_size, dp_size, attn_cp_size in ((8, 2, 1), (8, 2, 2), (16, 4, 2)): + for tp_size, dp_width, attn_cp_size in ((8, 2, 1), (8, 2, 2), (16, 4, 2)): _, attn_tp_size, _, attn_dp_size = compute_dp_attention_world_info( - True, 0, tp_size, dp_size, attn_cp_size + 0, tp_size, dp_width, attn_cp_size ) widths = derive_parallel_widths( tp_size=tp_size, @@ -2127,7 +2117,7 @@ def test_the_rank_helper_agrees_with_the_override(self): def test_recomputing_from_published_leaves_matches_the_publish_bag(self): shapes = ( dict(tp_size=8), - dict(tp_size=8, dp_size=2, enable_dp_attention=True), + dict(tp_size=8, attn_dp_size=2), dict(tp_size=8, ep_size=4, moe_dp_size=2), dict(tp_size=8, dcp_size=8), ) @@ -2151,9 +2141,7 @@ def test_recomputing_from_published_leaves_matches_the_publish_bag(self): recomputed = derive_parallel_widths( tp_size=parallel.tp_size, attn_cp_size=parallel.attn_cp_size, - attn_dp_size=( - parallel.dp_size if parallel.enable_dp_attention else 1 - ), + attn_dp_size=parallel.attn_dp_size, moe_ep_size=parallel.ep_size, moe_dp_size=parallel.moe_dp_size, dcp_size=parallel.dcp_size, @@ -2244,7 +2232,10 @@ def test_a_computed_name_names_the_function_that_computes_it(self): self.assertTrue(callable(getattr(importlib.import_module(module), attr))) def test_the_arithmetic_produces_nothing_that_is_not_declared(self): - from sglang.srt.runtime_context import _derived_widths + from sglang.srt.runtime_context import ( + _derived_widths, + _parallel_config_leaves, + ) produced = set( derive_parallel_widths( @@ -2257,7 +2248,10 @@ def test_the_arithmetic_produces_nothing_that_is_not_declared(self): dcp_enabled=False, ) ) - self.assertEqual(produced - set(_derived_widths()), set()) + # `attn_dp_size` is a configured leaf the widths carry along, so a + # process with nothing published can stamp the whole layout at once. + self.assertIn("attn_dp_size", _parallel_config_leaves()) + self.assertEqual(produced - set(_derived_widths()), {"attn_dp_size"}) def test_a_declared_quotient_is_not_a_record_field(self): """It has no operator input to preserve, and the record is what crosses @@ -2441,9 +2435,7 @@ def _publish_square(self): reset_context() self.addCleanup(reset_context) publish( - ServerArgs( - model_path="dummy", tp_size=4, dp_size=2, enable_dp_attention=True - ), + ServerArgs(model_path="dummy", tp_size=4, attn_dp_size=2), role="scheduler", ranks=SpawnRanks(world_rank=3, dp_rank=1), ) @@ -2585,8 +2577,7 @@ def test_the_layer_phase_leaves_a_stated_placement_alone(self): ServerArgs( model_path="dummy", tp_size=4, - dp_size=2, - enable_dp_attention=True, + attn_dp_size=2, device="cpu", ), role="test", @@ -2630,17 +2621,17 @@ def test_encoder_cp_mlp_reduces_over_attention_tp(self): server_args = ServerArgs( model_path="dummy", device="cpu", - tp_size=4, + tp_size=8, + attn_dp_size=2, attn_cp_size=2, - enable_dp_attention=True, ) publish(server_args, role="test", ranks=SpawnRanks(world_rank=0)) # Identical shards contribute equally; reducing across CP replicas # would double the output even though the layer uses attention TP. tp_group = SimpleNamespace( - world_size=4, + world_size=8, rank_in_group=0, - all_reduce=Mock(side_effect=lambda x: x * 4), + all_reduce=Mock(side_effect=lambda x: x * 8), ) attn_tp_group = SimpleNamespace( world_size=2, @@ -2779,9 +2770,7 @@ def test_the_tensor_swap_states_the_draft_has_no_attention_replica(self): reset_context() self.addCleanup(reset_context) publish( - ServerArgs( - model_path="dummy", tp_size=4, dp_size=2, enable_dp_attention=True - ), + ServerArgs(model_path="dummy", tp_size=4, attn_dp_size=2), role="scheduler", ranks=SpawnRanks(world_rank=0, dp_rank=0), ) @@ -2798,14 +2787,14 @@ def test_the_tensor_swap_states_the_draft_has_no_attention_replica(self): self.assertEqual(parallel.attn_dp_rank, 0) self.assertEqual(parallel.attn_cp_size, 1) self.assertEqual(parallel.attn_cp_rank, 0) - # The scope leaves the deployment's replica count alone. - self.assertEqual(parallel.dp_size, 2) + # The scope leaves the deployment's DP rank count alone. + self.assertEqual(parallel.num_dp_ranks, 2) self.assertEqual( parallel.tp_size, parallel.attn_tp_size * parallel.attn_dp_size * parallel.attn_cp_size, ) self.assertEqual(get_parallel().attn_dp_size, 2) - self.assertEqual(get_parallel().dp_size, 2) + self.assertEqual(get_parallel().num_dp_ranks, 2) def test_a_full_width_swap_leaves_the_attention_layout_alone(self): from sglang.srt.distributed import parallel_state @@ -2813,9 +2802,7 @@ def test_a_full_width_swap_leaves_the_attention_layout_alone(self): reset_context() self.addCleanup(reset_context) publish( - ServerArgs( - model_path="dummy", tp_size=4, dp_size=2, enable_dp_attention=True - ), + ServerArgs(model_path="dummy", tp_size=4, attn_dp_size=2), role="scheduler", ranks=SpawnRanks(world_rank=0, dp_rank=0), ) @@ -2828,7 +2815,7 @@ def test_a_full_width_swap_leaves_the_attention_layout_alone(self): self.assertEqual(parallel.tp_size, 4) self.assertEqual(parallel.attn_dp_size, 2) self.assertEqual(parallel.attn_tp_size, 2) - self.assertEqual(parallel.dp_size, 2) + self.assertEqual(parallel.num_dp_ranks, 2) def test_a_narrowed_draft_still_indexes_the_targets_dp_sync(self): """A draft scope that owns its attention answers the DP gather width and @@ -2839,9 +2826,7 @@ def test_a_narrowed_draft_still_indexes_the_targets_dp_sync(self): reset_context() self.addCleanup(reset_context) publish( - ServerArgs( - model_path="dummy", tp_size=4, dp_size=4, enable_dp_attention=True - ), + ServerArgs(model_path="dummy", tp_size=4, attn_dp_size=4), role="scheduler", ranks=SpawnRanks(world_rank=0, dp_rank=0), ) @@ -2866,9 +2851,7 @@ def _draft_scope_at_slot_two(self): reset_context() self.addCleanup(reset_context) publish( - ServerArgs( - model_path="dummy", tp_size=4, dp_size=4, enable_dp_attention=True - ), + ServerArgs(model_path="dummy", tp_size=4, attn_dp_size=4), role="scheduler", ranks=SpawnRanks(world_rank=0, dp_rank=0), ) diff --git a/test/registered/unit/test_server_args_migration.py b/test/registered/unit/test_server_args_migration.py index 0371f11a0385..4a5270c081b0 100644 --- a/test/registered/unit/test_server_args_migration.py +++ b/test/registered/unit/test_server_args_migration.py @@ -178,9 +178,10 @@ def backends(argv): def test_each_deprecation_shape_does_its_job(self): """The four `Deprecated*Action` shapes, on a throwaway parser. - Only `DeprecatedStoreTrueAction` has a registration in `ServerArgs` - today (`--disable-cuda-graph`, covered above). The other three are kept - for the next flag that needs them, which makes this the only thing + Only `DeprecatedStoreTrueAction` has registrations in `ServerArgs` + today (`--disable-cuda-graph`, covered above, and + `--enable-dp-attention`). The other three are kept for the next flag + that needs them, which makes this the only thing standing between them and silent rot -- and it doubles as the worked example of which shape to reach for. """ diff --git a/test/registered/unit/utils/test_weight_checker.py b/test/registered/unit/utils/test_weight_checker.py index 2df370879dc7..874ea3b30ce1 100644 --- a/test/registered/unit/utils/test_weight_checker.py +++ b/test/registered/unit/utils/test_weight_checker.py @@ -761,8 +761,7 @@ def setUp(self): self, published_topology( tp_size=4, - dp_size=2, - enable_dp_attention=True, + attn_dp_size=2, ranks={"world_rank": 2, "dp_rank": 1}, ), ) diff --git a/test/registered/utils/test_network_address.py b/test/registered/utils/test_network_address.py index d3ca555b0daa..9600182aa04c 100644 --- a/test/registered/utils/test_network_address.py +++ b/test/registered/utils/test_network_address.py @@ -246,7 +246,7 @@ def test_init_new_with_ipv6_address(self): server_args = ServerArgs(model_path="dummy") server_args.port = 30000 server_args.nccl_port = None - server_args.enable_dp_attention = True + server_args.attn_dp_size = 2 server_args.nnodes = 2 server_args.dist_init_addr = "[2001:db8::1]:25000" @@ -265,7 +265,7 @@ def test_init_new_with_invalid_ipv6_address(self): server_args = ServerArgs(model_path="dummy") server_args.port = 30000 server_args.nccl_port = None - server_args.enable_dp_attention = True + server_args.attn_dp_size = 2 server_args.nnodes = 2 server_args.dist_init_addr = "[invalid-ipv6]:25000" @@ -277,7 +277,7 @@ def test_init_new_with_malformed_ipv6_address_missing_bracket(self): server_args = ServerArgs(model_path="dummy") server_args.port = 30000 server_args.nccl_port = None - server_args.enable_dp_attention = True + server_args.attn_dp_size = 2 server_args.nnodes = 2 server_args.dist_init_addr = "[2001:db8::1:25000" @@ -289,7 +289,7 @@ def test_init_new_with_malformed_ipv6_address_missing_port(self): server_args = ServerArgs(model_path="dummy") server_args.port = 30000 server_args.nccl_port = None - server_args.enable_dp_attention = True + server_args.attn_dp_size = 2 server_args.nnodes = 2 server_args.dist_init_addr = "[2001:db8::1]" @@ -301,7 +301,7 @@ def test_init_new_with_malformed_ipv6_address_invalid_port(self): server_args = ServerArgs(model_path="dummy") server_args.port = 30000 server_args.nccl_port = None - server_args.enable_dp_attention = True + server_args.attn_dp_size = 2 server_args.nnodes = 2 server_args.dist_init_addr = "[2001:db8::1]:abcde" @@ -313,7 +313,7 @@ def test_init_new_with_malformed_ipv6_address_wrong_separator(self): server_args = ServerArgs(model_path="dummy") server_args.port = 30000 server_args.nccl_port = None - server_args.enable_dp_attention = True + server_args.attn_dp_size = 2 server_args.nnodes = 2 server_args.dist_init_addr = "[2001:db8::1]#25000" diff --git a/test/registered/xpu/test_dsa_indexer_xpu.py b/test/registered/xpu/test_dsa_indexer_xpu.py index 292545e3b358..a05a326b54be 100644 --- a/test/registered/xpu/test_dsa_indexer_xpu.py +++ b/test/registered/xpu/test_dsa_indexer_xpu.py @@ -236,7 +236,6 @@ class TestDSAIndexerXPU(CustomTestCase): @classmethod def setUpClass(cls): server_args = ServerArgs(model_path="dummy") - server_args.enable_dp_attention = False server_args.dsa_prefill_backend = "intel_xpu" server_args.dsa_decode_backend = "intel_xpu" server_args.dsa_topk_backend = "torch"