Skip to content

Add B300 DSV4 SGLang recipes #281

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
RohitNagraj wants to merge 4 commits into from
Closed

Add B300 DSV4 SGLang recipes #281

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
b487e86
feat(recipes): add DSV4 B300 SGLang disagg configs
RohitNagraj Jul 27, 2026
2065f6e
refactor(recipes): use compact B300 DSV4 matrices
RohitNagraj Jul 27, 2026
cb94d59
fix(recipes): enable CUDA UCX transport for DSV4
RohitNagraj Jul 28, 2026
5bde771
fix: align DSV4 STP input formatting
RohitNagraj Jul 29, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
235 changes: 235 additions & 0 deletions recipes/dsv4-pro/sglang/b300-fp4/8k1k-mtp.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,235 @@
base:
name: dsv4-pro-b300-disagg-8k1k-1p1d-dep4-dep8-mtp
model:
path: deepseek-v4-pro
container: lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4
precision: fp4
dynamo:
hash: 41882ae9b07232eed4850fb1daf8c958abb2556a
install: true
sbatch_directives:
cpus-per-task: '144'
mem: '0'
resources:
gpu_type: b300
gpus_per_node: 8
prefill_nodes: 1
prefill_workers: 1
gpus_per_prefill: 4
decode_nodes: 1
decode_workers: 1
gpus_per_decode: 8
frontend:
type: dynamo
enable_multiple_frontends: true
num_additional_frontends: 8
backend:
type: sglang
prefill_environment:
PYTHONUNBUFFERED: '1'
SGLANG_RADIX_FORCE_MISS: '1'
SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1'
SGLANG_DEFAULT_THINKING: '1'
SGLANG_DSV4_REASONING_EFFORT: max
SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1'
SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: '1'
SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: '1'
SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: '1'
SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '9216'
NCCL_MNNVL_ENABLE: '1'
NCCL_CUMEM_ENABLE: '1'
MC_FORCE_MNNVL: '1'
UCX_TLS: 'cuda_copy,rc'
SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000'
SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000'
SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1'
decode_environment:
PYTHONUNBUFFERED: '1'
SGLANG_RADIX_FORCE_MISS: '1'
SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1'
SGLANG_DEFAULT_THINKING: '1'
SGLANG_DSV4_REASONING_EFFORT: max
SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1'
SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: '1'
SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: '1'
SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '2048'
NCCL_MNNVL_ENABLE: '1'
NCCL_CUMEM_ENABLE: '1'
MC_FORCE_MNNVL: '1'
UCX_TLS: 'cuda_copy,rc'
SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000'
SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000'
SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1'
SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: '0'
sglang_config:
prefill:
served-model-name: deepseek-ai/DeepSeek-V4-Pro
model-path: /model/
trust-remote-code: true
tool-call-parser: deepseekv4
disaggregation-mode: prefill
watchdog-timeout: 86400
disaggregation-transfer-backend: nixl
tensor-parallel-size: 4
data-parallel-size: 4
expert-parallel-size: 4
enable-dp-attention: true
enable-dp-lm-head: true
moe-a2a-backend: megamoe
mem-fraction-static: 0.9
max-running-requests: 256
cuda-graph-max-bs: 256
chunked-prefill-size: 32768
stream-interval: 60
decode:
served-model-name: deepseek-ai/DeepSeek-V4-Pro
model-path: /model/
trust-remote-code: true
tool-call-parser: deepseekv4
disaggregation-mode: decode
watchdog-timeout: 86400
disaggregation-transfer-backend: nixl
tensor-parallel-size: 8
data-parallel-size: 8
expert-parallel-size: 8
enable-dp-attention: true
enable-dp-lm-head: true
moe-a2a-backend: megamoe
speculative-algo: EAGLE
speculative-num-steps: 3
speculative-eagle-topk: 1
speculative-num-draft-tokens: 4
mem-fraction-static: 0.94
max-running-requests: 3072
cuda-graph-max-bs: 256
swa-full-tokens-ratio: 0.15
context-length: 16384
stream-interval: 60
benchmark:
type: sa-bench
isl: 8192
osl: 1024
random_range_ratio: 0.8
concurrencies: '256'
req_rate: inf
use_chat_template: true
custom_tokenizer: sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer
identity:
model:
repo: deepseek-ai/DeepSeek-V4-Pro
revision: 0366e4e064385807ea86b088a5c6c878ff23343b
container:
image: lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4@sha256:86ea65a66c45e6597292986068523ca7738e2e949c31bfd93330db9870e1a930
frameworks:
dynamo: 41882ae9b07232eed4850fb1daf8c958abb2556a
sglang: 9462c303a5c5349488e84b2c346e9b5a066de21c

zip_override_dep4_dep8:
name:
- dsv4-pro-b300-disagg-8k1k-1p1d-dep4-dep8-mtp
- dsv4-pro-b300-disagg-8k1k-1p2d-dep4-dep8-mtp
- dsv4-pro-b300-disagg-8k1k-2p1d-dep4-dep8-mtp
- dsv4-pro-b300-disagg-8k1k-4p1d-dep4-dep8-mtp
- dsv4-pro-b300-disagg-8k1k-6p1d-dep4-dep8-mtp
- dsv4-pro-b300-disagg-8k1k-8p1d-dep4-dep8-mtp
resources:
prefill_nodes: [1, 1, 1, 2, 3, 4]
prefill_workers: [1, 1, 2, 4, 6, 8]
decode_nodes: [1, 2, 1, 1, 1, 1]
decode_workers: [1, 2, 1, 1, 1, 1]
backend:
decode_environment:
SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: ['2048', '2048', '4096', '4096', '4096', '4096']
sglang_config:
prefill:
max-running-requests: [256, 256, 512, 1024, 1024, 1024]
cuda-graph-max-bs: [256, 256, 512, 1024, 1024, 1024]
decode:
speculative-num-steps: [3, 3, 3, 3, 2, 1]
speculative-num-draft-tokens: [4, 4, 4, 4, 3, 2]
mem-fraction-static: [0.94, 0.94, 0.9, 0.9, 0.9, 0.85]
max-running-requests: [3072, 3072, 3072, 3072, 4096, 8192]
cuda-graph-max-bs: [256, 256, 512, 1024, 1024, 1280]
swa-full-tokens-ratio: [0.15, 0.15, 0.15, 0.15, 0.15, 0.1]
context-length: [16384, 16384, 16384, 16384, 9216, 9216]
benchmark:
concurrencies: ['256', '256', '512', '1024', '2048', '3072']

override_tp4_tp4:
name: dsv4-pro-b300-disagg-8k1k-1p1d-tp4-tp4-mtp
resources:
gpus_per_decode: 4
backend:
prefill_environment:
SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: null
SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: null
SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: null
SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: null
decode_environment:
SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: null
SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: null
SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: null
SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: null
sglang_config:
prefill:
tensor-parallel-size: 4
data-parallel-size: 1
expert-parallel-size: 1
enable-dp-attention: null
enable-dp-lm-head: null
moe-a2a-backend: null
moe-runner-backend: flashinfer_mxfp4
disable-flashinfer-autotune: true
max-running-requests: 8
cuda-graph-max-bs: 8
stream-interval: null
decode:
tensor-parallel-size: 4
data-parallel-size: 1
expert-parallel-size: 1
enable-dp-attention: null
enable-dp-lm-head: null
moe-a2a-backend: null
moe-runner-backend: flashinfer_mxfp4
disable-flashinfer-autotune: true
mem-fraction-static: 0.9
max-running-requests: 8
cuda-graph-max-bs: 8
swa-full-tokens-ratio: 0.1
stream-interval: null
benchmark:
concurrencies: '1'

override_dep4_tp4:
name: dsv4-pro-b300-disagg-8k1k-1p6d-dep4-tp4-mtp
resources:
decode_nodes: 3
decode_workers: 6
gpus_per_decode: 4
backend:
decode_environment:
SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: null
SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: null
SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: null
SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: null
sglang_config:
prefill:
max-running-requests: 128
cuda-graph-max-bs: 128
stream-interval: null
decode:
tensor-parallel-size: 4
data-parallel-size: 1
expert-parallel-size: 1
enable-dp-attention: null
enable-dp-lm-head: null
moe-a2a-backend: null
moe-runner-backend: flashinfer_mxfp4
disable-flashinfer-autotune: true
mem-fraction-static: 0.9
max-running-requests: 128
cuda-graph-max-bs: 128
swa-full-tokens-ratio: 0.1
stream-interval: null
benchmark:
concurrencies: 8x32x64
Loading
Loading