Skip to content

Add DSV4 B200 disaggregated Dynamo SGLang STP configuration / 新增 DSV4 B200 分离式 Dynamo SGLang STP 配置 #2560

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
cquil11 merged 15 commits into from
Sep 1, 2026
Merged

Add DSV4 B200 disaggregated Dynamo SGLang STP configuration / 新增 DSV4 B200 分离式 Dynamo SGLang STP 配置 #2560

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
050e955
feat(config): add DSV4 B200 Dynamo-SGLang STP matrix
RohitNagraj Aug 11, 2026
202bd2d
chore(changelog): link PR #2560
RohitNagraj Aug 11, 2026
09a6533
Merge branch 'main' into dsv4-fp4-b200-dynamo-sglang-nscale
xinli-sw Aug 12, 2026
7ccc381
fix(config): align DSV4 B200 STP benchmark inputs
RohitNagraj Aug 12, 2026
4a92c35
fix(config): switch DSV4 B200 STP to NIXL
RohitNagraj Aug 17, 2026
f36ce4f
chore: sync latest main
RohitNagraj Aug 17, 2026
da25e66
fix(config): disable chat templates for DSV4 B200 STP
RohitNagraj Aug 18, 2026
15d1eaa
chore: merge main and resolve conflicts
RohitNagraj Aug 27, 2026
747a600
chore(config): refresh DSV4 B200 SGLang recipes
RohitNagraj Aug 31, 2026
c89ce06
fix(config): refresh DSV4 B200 SGLang image
RohitNagraj Aug 31, 2026
6365712
fix(config): align DSV4 B200 recipe images
RohitNagraj Aug 31, 2026
65f33da
fix(config): route DSV4 B200 SGLang to NScale
cquil11 Aug 31, 2026
99d1561
Merge branch 'main' into dsv4-fp4-b200-dynamo-sglang-nscale
RohitNagraj Aug 31, 2026
a430c17
fix(config): enable B200 power telemetry
RohitNagraj Aug 31, 2026
23ff29f
chore: merge main into PR #2560 and preserve changelog history
cquil11 Sep 1, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
154 changes: 154 additions & 0 deletions ...arks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-1p1d-dep8-dep8.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,154 @@
name: "disagg-b200-1p1d-dep8-dep8-c512"


model:
path: "deepseek-v4-pro"
container: "lmsysorg/sglang:nightly-dev-cu13-20260821-f825d729"
precision: "fp4"

dynamo:
hash: "86f84b9486905c2bfaa4b756c9fa39f04ecf5ebd"
install: true

sbatch_directives:
cpus-per-task: "144"
mem: "0"

resources:
gpu_type: "b200"
gpus_per_node: 8
prefill_nodes: 1
prefill_workers: 1
gpus_per_prefill: 8
decode_nodes: 1
decode_workers: 1
gpus_per_decode: 8

health_check:
max_attempts: 240
frontend:
type: dynamo
enable_multiple_frontends: false
env:
DYN_ROUTER_LOAD_BLOCK_SIZE: "1"
args:
router-mode: "kv"
router-kv-overlap-score-weight: 0
router-queue-threshold: 64
router-temperature: 0.5
no-kv-events: true

backend:
type: sglang

prefill_environment:
PYTHONUNBUFFERED: "1"
SGLANG_RADIX_FORCE_MISS: "1"
SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1"
SGLANG_DEFAULT_THINKING: "1"
SGLANG_DSV4_REASONING_EFFORT: "max"
SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1"
SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1"
SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192"
SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1"
SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1"
SGLANG_OPT_USE_ONLINE_COMPRESS: "1"
SGLANG_OPT_FP8_WO_A_GEMM: "1"
SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000"
SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000"
SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1"
DYN_SKIP_SGLANG_LOG_FORMATTING: "1"
SGLANG_LOG_FORWARD_ITERS: "1"
SGLANG_LOG_MS: "1"
SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60"

decode_environment:
PYTHONUNBUFFERED: "1"
SGLANG_RADIX_FORCE_MISS: "1"
SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1"
SGLANG_DEFAULT_THINKING: "1"
SGLANG_DSV4_REASONING_EFFORT: "max"
SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1"
SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1"
SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "1280"
SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1"
SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1"
SGLANG_OPT_USE_ONLINE_COMPRESS: "1"
SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: "8"
SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000"
SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000"
SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1"
DYN_SKIP_SGLANG_LOG_FORMATTING: "1"
SGLANG_LOG_FORWARD_ITERS: "1"
SGLANG_LOG_MS: "1"
SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60"

sglang_config:
prefill:
served-model-name: "deepseek-ai/DeepSeek-V4-Pro"
trust-remote-code: true
watchdog-timeout: 86400
skip-tokenizer-init: true
stream-interval: 60

tensor-parallel-size: 8
data-parallel-size: 8
expert-parallel-size: 8

enable-dp-attention: true
moe-a2a-backend: "megamoe"
moe-dense-tp-size: 1

disaggregation-mode: "prefill"
disaggregation-transfer-backend: nixl

mem-fraction-static: 0.80
max-running-requests: 1024
chunked-prefill-size: 65536

decode:
served-model-name: "deepseek-ai/DeepSeek-V4-Pro"
trust-remote-code: true
watchdog-timeout: 86400
skip-tokenizer-init: true
stream-interval: 60

load-balance-method: "total_requests"
moe-a2a-backend: "megamoe"

disaggregation-mode: "decode"
disaggregation-transfer-backend: nixl
disaggregation-decode-polling-interval: 8

mem-fraction-static: 0.94
swa-full-tokens-ratio: 0.056
context-length: 9216
tensor-parallel-size: 8
data-parallel-size: 8
expert-parallel-size: 8
enable-dp-attention: true
enable-dp-lm-head: true
max-running-requests: 21504
cuda-graph-max-bs: 1280


benchmark:
type: "sa-bench"
isl: 8192
osl: 1024
concurrencies: "256x512"
req_rate: "inf"
use_chat_template: false
Comment thread
cursor[bot] marked this conversation as resolved.

telemetry:
enabled: true
provider: dcgm-power
default_frequency: 1.0
storage_subdir: power
required: true
startup_timeout_seconds: 120
request_timeout_seconds: 2
collector_join_timeout_seconds: 10
dcgm_exporter:
container_image: dcgm-exporter
port: 9401
146 changes: 146 additions & 0 deletions ...marks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-1p1d-dep8-tp8.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,146 @@
name: "disagg-b200-1p1d-dep8-tp8-4-c64"

@claude claude Bot Aug 11, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🟡 The name: field in disagg-b200-1p1d-dep8-tp8.yaml is "disagg-b200-1p1d-dep8-tp8-4-c64", with a stray -4- segment that doesn't match this recipe's topology (1 prefill + 1 decode = 2 nodes) or the naming convention used by every other new B200 recipe in this PR (e.g. disagg-b200-1p4d-dep8-tp8-c64, disagg-b200-1p1d-tp8-tp8-c1). It looks like a copy-paste leftover from a GB200-style template where that suffix denotes total node count; harmless at runtime since launch_b200-nscale-slurm.sh overwrites the name via sed before srtctl reads it, but worth cleaning up for consistency.

Extended reasoning...

The name: field on line 1 of disagg-b200-1p1d-dep8-tp8.yaml reads "disagg-b200-1p1d-dep8-tp8-4-c64". Every other new B200 recipe added in this PR follows a clean <topology>-c<concurrency> naming pattern with no extra numeric segment: disagg-b200-1p1d-dep8-dep8.yaml -> disagg-b200-1p1d-dep8-dep8-c512, disagg-b200-1p1d-tp8-tp8.yaml -> disagg-b200-1p1d-tp8-tp8-c1, disagg-b200-1p4d-dep8-tp8.yaml -> disagg-b200-1p4d-dep8-tp8-c64, and so on through the 3p2d/4p2d/5p2d/6p2d files. This file alone inserts a stray -4- between the topology and the concurrency suffix.\n\nOn the existing GB200 recipes in this codebase, that kind of numeric segment is a real, intentional convention: it denotes total node count (prefill_nodes + decode_nodes), e.g. a GB200 disagg-gb200-1p4d-dep8-tp8-10-c64.yaml-style file uses -10- because it sums to 10 nodes. Applying that same mental model here breaks down: this recipe's resources: block sets prefill_nodes: 1 and decode_nodes: 1, i.e. 2 total nodes, not 4. So -4- isn't just inconsistent with sibling files in this PR, it's also the wrong number under the very convention it appears to be borrowed from — strong evidence this is a copy-paste leftover from a GB200-style template rather than a deliberate choice.\n\nProof by walkthrough: (1) open disagg-b200-1p1d-dep8-tp8.yaml, line 1 is name: \"disagg-b200-1p1d-dep8-tp8-4-c64\"\". (2) Check resources: in the same file: prefill_nodes: 1, decode_nodes: 1 -> topology is 1p1d, 2 nodes total. (3) Compare to the sibling disagg-b200-1p4d-dep8-tp8.yaml in the same PR, which has prefill_nodes: 1, decode_nodes: 4 (5 nodes total) and is named disagg-b200-1p4d-dep8-tp8-c64 — no numeric node-count segment despite actually having more nodes. (4) There is therefore no consistent rule under which -4- belongs in the 1p1d file's name.\n\nWhy nothing currently prevents or catches this: recipe names are free-form YAML string values with no schema validation, and nothing in configs/nvidia-master.yaml or the sweep matrix reads the name: field for routing — the matrix entry in nvidia-master.yaml references the recipe purely by its CONFIG_FILE path, not by the embedded name. And at runtime, runners/launch_b200-nscale-slurm.sh executes sed -i \"s/^name:.*/name: \\\"\\\\"/\" \"\\"\" before \"srtctl apply\"\" ever runs, unconditionally replacing whatever is on that line with the runner-generated name. So this typo has zero effect on job submission, GPU allocation, or benchmark results — it is purely a checked-in metadata string that a human reading the repo (or grepping recipe names) would see and find confusing.\n\nFix is trivial: change line 1 to name: "disagg-b200-1p1d-dep8-tp8-c64"`, matching the convention of every sibling file added in this PR. Given the complete absence of functional impact (confirmed by tracing the sed override in the launcher) and the triviality of the fix, this is cosmetic — nit severity, not a merge blocker.



model:
path: "deepseek-v4-pro"
container: "lmsysorg/sglang:nightly-dev-cu13-20260821-f825d729"
precision: "fp4"

dynamo:
hash: "86f84b9486905c2bfaa4b756c9fa39f04ecf5ebd"
install: true

sbatch_directives:
cpus-per-task: "144"
mem: "0"

resources:
gpu_type: "b200"
gpus_per_node: 8
prefill_nodes: 1
prefill_workers: 1
gpus_per_prefill: 8
decode_nodes: 1
decode_workers: 1
gpus_per_decode: 8

health_check:
max_attempts: 240

frontend:
type: dynamo
enable_multiple_frontends: false
env:
DYN_ROUTER_LOAD_BLOCK_SIZE: "1"
args:
router-mode: "kv"
router-kv-overlap-score-weight: 0
router-queue-threshold: 64
router-temperature: 0.5
no-kv-events: true

backend:
type: sglang

prefill_environment:
PYTHONUNBUFFERED: "1"
SGLANG_RADIX_FORCE_MISS: "1"
SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1"
SGLANG_DEFAULT_THINKING: "1"
SGLANG_DSV4_REASONING_EFFORT: "max"
SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1"
SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1"
SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192"
SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1"
SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1"
SGLANG_OPT_USE_ONLINE_COMPRESS: "1"
SGLANG_OPT_FP8_WO_A_GEMM: "1"
SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000"
SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000"
SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1"
DYN_SKIP_SGLANG_LOG_FORMATTING: "1"
SGLANG_LOG_FORWARD_ITERS: "1"
SGLANG_LOG_MS: "1"
SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60"

decode_environment:
PYTHONUNBUFFERED: "1"
SGLANG_OPT_USE_ONLINE_COMPRESS: "1"
SGLANG_RADIX_FORCE_MISS: "1"
SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1"
SGLANG_DEFAULT_THINKING: "1"
SGLANG_DSV4_REASONING_EFFORT: "max"
SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1"
SGLANG_OPT_USE_JIT_NORM: "1"
SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1"
SGLANG_OPT_USE_TOPK_V2: "1"
SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000"
SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000"
SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1"

sglang_config:
prefill:
served-model-name: "deepseek-ai/DeepSeek-V4-Pro"
trust-remote-code: true
watchdog-timeout: 86400
skip-tokenizer-init: true
stream-interval: 60

tensor-parallel-size: 8
data-parallel-size: 8
expert-parallel-size: 8

enable-dp-attention: true
moe-a2a-backend: "megamoe"
moe-dense-tp-size: 1

disaggregation-mode: "prefill"
disaggregation-transfer-backend: nixl

mem-fraction-static: 0.80
max-running-requests: 1024
chunked-prefill-size: 65536

decode:
served-model-name: "deepseek-ai/DeepSeek-V4-Pro"
model-path: "/model/"
trust-remote-code: true
disable-radix-cache: true

disaggregation-mode: "decode"
disaggregation-transfer-backend: nixl

tensor-parallel-size: 8
data-parallel-size: 1
expert-parallel-size: 1

moe-runner-backend: "flashinfer_mxfp4"
disable-flashinfer-autotune: true

mem-fraction-static: 0.9
max-running-requests: 1024
cuda-graph-max-bs: 512
swa-full-tokens-ratio: 0.1
context-length: 16384


benchmark:
type: "sa-bench"
isl: 8192
osl: 1024
concurrencies: "32x64"
req_rate: "inf"
use_chat_template: false

telemetry:
enabled: true
provider: dcgm-power
default_frequency: 1.0
storage_subdir: power
required: true
startup_timeout_seconds: 120
request_timeout_seconds: 2
collector_join_timeout_seconds: 10
dcgm_exporter:
container_image: dcgm-exporter
port: 9401
Loading
Loading