diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b200_sglang_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_b200_sglang_mtp.sh index 884705c148..182ec81bf0 100755 --- a/benchmarks/single_node/agentic/dsv4_fp4_b200_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_b200_sglang_mtp.sh @@ -101,14 +101,19 @@ fi PARALLEL_ARGS=(--tp "$TP") METRICS_ARGS=(--enable-metrics --enable-cache-report) +MODEL_ARGS=() CHUNKED_PREFILL_SIZE=8192 +SWA_FULL_TOKENS_RATIO=0.1 if [ "$DP_ATTENTION" = "true" ]; then DEEPEP_CONFIG='{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' + MODEL_ARGS+=(--enable-deepseek-v4-fp4-indexer) export SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE=1 + export SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS=1 + export SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND=1 export SGLANG_OPT_FIX_HASH_MEGA_MOE=1 export SGLANG_OPT_USE_FAST_MASK_EP=1 export SGLANG_OPT_FIX_MEGA_MOE_MEMORY=1 - export SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK=4096 + export SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK=8192 export SGLANG_OPT_FIX_NEXTN_MEGA_MOE=1 export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=0 PARALLEL_ARGS+=( @@ -123,7 +128,9 @@ if [ "$DP_ATTENTION" = "true" ]; then --moe-a2a-backend deepep --deepep-config "$DEEPEP_CONFIG" ) - CHUNKED_PREFILL_SIZE=32768 + # This is the global budget across all eight DP-attention ranks. + CHUNKED_PREFILL_SIZE=65536 + SWA_FULL_TOKENS_RATIO=0.02 else PARALLEL_ARGS+=( --moe-runner-backend flashinfer_mxfp4 @@ -131,7 +138,6 @@ else ) fi -MODEL_ARGS=() # The B200-specialized image deadlocks immediately after weight loading when # forced through the B300 compressed-attention/page-size overrides. # DeepGEMM's DSv4 indexer needs a multi-GiB temporary allocation at long @@ -186,7 +192,7 @@ SGLANG_CMD=( --trust-remote-code "${PARALLEL_ARGS[@]}" --mem-fraction-static "$MEM_FRACTION_STATIC" - --swa-full-tokens-ratio 0.1 + --swa-full-tokens-ratio "$SWA_FULL_TOKENS_RATIO" --max-running-requests "$MAX_RUNNING_REQUESTS" --cuda-graph-max-bs "$CUDA_GRAPH_MAX_BS" --chunked-prefill-size "$CHUNKED_PREFILL_SIZE" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 7290e3c3db..a995310234 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -941,9 +941,9 @@ dsv4-fp4-b200-sglang-agentic-hicache-mtp: - dram-utilization: 0.80 search-space: - { tp: 8, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 3, 4, 5] } - - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: hicache }, spec-decoding: mtp, conc-list: [8, 10, 16, 32, 40, 44] } - - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [16, 24, 32, 38, 44, 48, 50, 52], router: { name: sglang-router, version: "0.3.2" } } - - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, spec-decoding: mtp, conc-list: [16, 32, 38, 44, 50, 56, 64, 66, 68], router: { name: sglang-router, version: "0.3.2" } } + - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: hicache }, spec-decoding: mtp, conc-list: [8, 10, 16, 32] } + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [16], router: { name: sglang-router, version: "0.3.2" } } + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, spec-decoding: mtp, conc-list: [16, 40, 48], router: { name: sglang-router, version: "0.3.2" } } dsv4-fp4-b200-vllm: image: vllm/vllm-openai:v0.25.0 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 85cb52f060..538a800d95 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6153,3 +6153,11 @@ - "Add MiniMax-M3 NVFP4 B300 single-node TensorRT-LLM AgentX with EAGLE3-GQA speculative decoding (3 draft tokens)." - "Use the nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc23.post1 image." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2658 + +- config-keys: + - dsv4-fp4-b200-sglang-agentic-hicache-mtp + scenario-type: + - agentic-coding + description: + - "Update the B200 SGLang AgentX HiCache MTP DEP8 configuration." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2656