Skip to content

fix sgl b200/b300 dpsk-v4 script #1158

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Qiaolin-Yu merged 13 commits into from
Apr 26, 2026
+101 −91
Merged

fix sgl b200/b300 dpsk-v4 script #1158

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
3a2e459
fix sgl b200/b300 script
Qiaolin-Yu Apr 25, 2026
3141900
ping docker image
Qiaolin-Yu Apr 25, 2026
ef41995
fix
Qiaolin-Yu Apr 26, 2026
7d107e8
fix
Qiaolin-Yu Apr 26, 2026
485cb8e
fix
Qiaolin-Yu Apr 26, 2026
289a855
Merge branch 'main' into main
Qiaolin-Yu Apr 26, 2026
7a4d415
tune swa-full-tokens-ratio
Qiaolin-Yu Apr 26, 2026
0f975c8
high concurrency
Qiaolin-Yu Apr 26, 2026
68e40f8
change perflog
Qiaolin-Yu Apr 26, 2026
c09b03f
Merge branch 'main' into main
Qiaolin-Yu Apr 26, 2026
e8e4810
upd
Qiaolin-Yu Apr 26, 2026
cbde82d
fix hang
Qiaolin-Yu Apr 26, 2026
7273032
remove useless points
Qiaolin-Yu Apr 26, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 45 additions & 9 deletions benchmarks/single_node/dsv4_fp4_b200.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,35 +49,71 @@ DEEPEP_CONFIG='{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}

if [[ $CONC -le 32 ]]; then
RECIPE=low-latency
export SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT=1
# common optimizations
export SGLANG_OPT_USE_JIT_NORM=1
export SGLANG_OPT_USE_JIT_INDEXER_METADATA=1
export SGLANG_OPT_USE_TOPK_V2=1
export SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2=1
RECIPE_FLAGS=(
--moe-runner-backend flashinfer_mxfp4
--chunked-prefill-size 4096
--chunked-prefill-size 8192
--disable-flashinfer-autotune
--mem-fraction-static 0.82
--mem-fraction-static 0.90
--max-running-requests 32
--swa-full-tokens-ratio 0.1
)
elif [[ $CONC -le 128 ]]; then
RECIPE=balanced
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=256
export SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT=1
# common optimizations
export SGLANG_OPT_USE_JIT_NORM=1
export SGLANG_OPT_USE_JIT_INDEXER_METADATA=1
export SGLANG_OPT_USE_TOPK_V2=1
export SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2=1
# MoE EP related flags
export SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE=1
export SGLANG_OPT_FIX_HASH_MEGA_MOE=1
export SGLANG_OPT_USE_FAST_MASK_EP=1
export SGLANG_OPT_FIX_MEGA_MOE_MEMORY=1
export SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK=4096
export SGLANG_OPT_FIX_NEXTN_MEGA_MOE=1
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=0
RECIPE_FLAGS=(
--dp-size "$TP"
--enable-dp-attention
--moe-a2a-backend deepep
--deepep-config "$DEEPEP_CONFIG"
--mem-fraction-static 0.82
--cuda-graph-max-bs 64
--mem-fraction-static 0.83
--max-running-requests 128
--chunked-prefill-size 32768
--swa-full-tokens-ratio 0.1
)
else
RECIPE=max-throughput
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=256
export SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT=1
# common optimizations
export SGLANG_OPT_USE_JIT_NORM=1
export SGLANG_OPT_USE_JIT_INDEXER_METADATA=1
export SGLANG_OPT_USE_TOPK_V2=1
export SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2=1
# MoE EP related flags
export SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE=1
export SGLANG_OPT_FIX_HASH_MEGA_MOE=1
export SGLANG_OPT_USE_FAST_MASK_EP=1
export SGLANG_OPT_FIX_MEGA_MOE_MEMORY=1
export SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK=4096
export SGLANG_OPT_FIX_NEXTN_MEGA_MOE=1
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=0
RECIPE_FLAGS=(
--dp-size "$TP"
--enable-dp-attention
--moe-a2a-backend deepep
--deepep-config "$DEEPEP_CONFIG"
--mem-fraction-static 0.82
--cuda-graph-max-bs 64
--max-running-requests 256
--mem-fraction-static 0.90
--max-running-requests 512
--chunked-prefill-size 32768
--swa-full-tokens-ratio 0.1
)
fi
echo "Recipe: $RECIPE (CONC=$CONC)"
Expand Down
59 changes: 45 additions & 14 deletions benchmarks/single_node/dsv4_fp4_b300_sglang.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,11 +25,6 @@ nvidia-smi

export SGLANG_JIT_DEEPGEMM_PRECOMPILE=0

# The deepseek-v4 sglang images (lmsysorg/sglang:deepseek-v4-blackwell and its
# B300 forks) bake CUDA_VISIBLE_DEVICES=4,5,6,7 into their ENV, which masks half
# of the 8 GPUs Slurm allocates us. Clear it so TP=8 can bind to all ranks.
unset CUDA_VISIBLE_DEVICES

# TODO(Cam): the deepseek-v4 sglang images install sglang editable at
# /workspace/sglang/python; prior sglang tags used /sgl-workspace/sglang.
# The runner mounts our repo at a non-/workspace path for these images so the
Expand Down Expand Up @@ -59,35 +54,71 @@ DEEPEP_CONFIG='{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}

if [[ $CONC -le 32 ]]; then
RECIPE=low-latency
export SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT=1
# common optimizations
export SGLANG_OPT_USE_JIT_NORM=1
export SGLANG_OPT_USE_JIT_INDEXER_METADATA=1
export SGLANG_OPT_USE_TOPK_V2=1
export SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2=1
RECIPE_FLAGS=(
--moe-runner-backend flashinfer_mxfp4
--chunked-prefill-size 4096
--chunked-prefill-size 8192
--disable-flashinfer-autotune
--mem-fraction-static 0.82
--mem-fraction-static 0.90
--max-running-requests 32
--swa-full-tokens-ratio 0.1
)
elif [[ $CONC -le 128 ]]; then
RECIPE=balanced
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=256
export SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT=1
# common optimizations
export SGLANG_OPT_USE_JIT_NORM=1
export SGLANG_OPT_USE_JIT_INDEXER_METADATA=1
export SGLANG_OPT_USE_TOPK_V2=1
export SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2=1
# MoE EP related flags
export SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE=1
export SGLANG_OPT_FIX_HASH_MEGA_MOE=1
export SGLANG_OPT_USE_FAST_MASK_EP=1
export SGLANG_OPT_FIX_MEGA_MOE_MEMORY=1
export SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK=4096
export SGLANG_OPT_FIX_NEXTN_MEGA_MOE=1
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=0
RECIPE_FLAGS=(
--dp-size "$TP"
--enable-dp-attention
--moe-a2a-backend deepep
--deepep-config "$DEEPEP_CONFIG"
--mem-fraction-static 0.82
--cuda-graph-max-bs 64
--mem-fraction-static 0.83
--max-running-requests 128
--chunked-prefill-size 32768
--swa-full-tokens-ratio 0.1
)
else
RECIPE=max-throughput
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=256
export SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT=1
# common optimizations
export SGLANG_OPT_USE_JIT_NORM=1
export SGLANG_OPT_USE_JIT_INDEXER_METADATA=1
export SGLANG_OPT_USE_TOPK_V2=1
export SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2=1
# MoE EP related flags
export SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE=1
export SGLANG_OPT_FIX_HASH_MEGA_MOE=1
export SGLANG_OPT_USE_FAST_MASK_EP=1
export SGLANG_OPT_FIX_MEGA_MOE_MEMORY=1
export SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK=4096
export SGLANG_OPT_FIX_NEXTN_MEGA_MOE=1
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=0
RECIPE_FLAGS=(
--dp-size "$TP"
--enable-dp-attention
--moe-a2a-backend deepep
--deepep-config "$DEEPEP_CONFIG"
--mem-fraction-static 0.82
--cuda-graph-max-bs 64
--max-running-requests 256
--mem-fraction-static 0.90
--max-running-requests 512
--chunked-prefill-size 32768
--swa-full-tokens-ratio 0.1
)
fi
echo "Recipe: $RECIPE (CONC=$CONC)"
Expand Down
Loading