diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm_dspark.sh b/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm_dspark.sh new file mode 100755 index 0000000000..77b8743410 --- /dev/null +++ b/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm_dspark.sh @@ -0,0 +1,349 @@ +#!/usr/bin/env bash +set -eo pipefail +set -x + +# Agentic trace replay benchmark for DeepSeek-V4-Pro-0813 FP4 on B200 using +# vLLM, with DSpark speculative decoding (num_speculative_tokens=6): synthetic +# acceptance length 3.77 for throughput, real target verification for the +# EVAL_ONLY eval. +# +# This DSpark recipe keeps the established engine args and agentic AIPerf rig, +# with two speculative-decoding behaviors: +# --speculative-config: synthetic acceptance length 3.77 (throughput) vs real DSpark (EVAL_ONLY); see the SPEC_CONFIG block +# cudagraph capture sizes expressed in TOKENS (see the capture block below). +# +# DSpark is unreachable on this image under the FlashInfer MLA sparse layout, so +# this recipe uses the FlashMLA sparse DeepSeek-V4 backend instead. It otherwise +# keeps FP8 KV cache, an FP4 indexer cache, mega-MoE, long-prefill chunking, and +# CUDA graphs with every decode batch captured explicitly. +# +# Required env vars: +# MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR +# +# DEP8 offloads KV to host DRAM with KV_OFFLOAD_BACKEND=vllm-simple. + +source "$(dirname "$0")/../../benchmark_lib.sh" + +check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION + +DCP_SIZE="${DCP_SIZE:-1}" +PCP_SIZE="${PCP_SIZE:-1}" +VLLM_CP_ARGS=() +if [ "$DCP_SIZE" -gt 1 ]; then + VLLM_CP_ARGS+=(--decode-context-parallel-size "$DCP_SIZE") +fi +if [ "$PCP_SIZE" -gt 1 ]; then + VLLM_CP_ARGS+=(--prefill-context-parallel-size "$PCP_SIZE") +fi + +GPU_COUNT=$((TP * PCP_SIZE)) +if [[ ! "$GPU_COUNT" =~ ^[1-9][0-9]*$ ]]; then + echo "Error: GPU_COUNT must be a positive integer, got '$GPU_COUNT'" >&2 + exit 1 +fi +export GPU_COUNT + +# Under DP-attention the DP world size equals TP, and the scheduler's global +# running-sequence budget is 2*CONC. Split that budget evenly across DP ranks. +if [ "$DP_ATTENTION" = "true" ] && [ $((2 * CONC % TP)) -ne 0 ]; then + echo "Error: DEP requires 2*CONC divisible by TP, got CONC='$CONC' and TP='$TP'" >&2 + exit 1 +fi + +if [[ -n "$SLURM_JOB_ID" ]]; then + echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" +fi + +# `hf download` creates the target dir if missing and is itself idempotent. +# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE +# Either way, MODEL_PATH is what the server is launched with. +if [[ -n "$MODEL_PATH" ]]; then + if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then + hf download "$MODEL" --local-dir "$MODEL_PATH" + fi +else + hf download "$MODEL" + export MODEL_PATH="$MODEL" +fi +nvidia-smi + +# ---- Resolve traces and install deps ---------------------------------------- +resolve_trace_source +install_agentic_deps + +# vllm-project/router expands the one HTTP backend into one logical worker per +# DP rank and sends X-data-parallel-rank on forwarded requests. aiperf's +# X-Correlation-ID is stable for every turn of a conversation; alias it to the +# router's preferred X-Session-ID header. +USE_VLLM_ROUTER=false +VLLM_BACKEND_PORT="$PORT" +if [ "$DP_ATTENTION" = "true" ]; then + USE_VLLM_ROUTER=true + VLLM_BACKEND_PORT=$((PORT + 1)) + VLLM_ROUTER_VERSION=0.1.14 + VLLM_ROUTER_POLICY=consistent_hash + VLLM_ROUTER_METRICS_PORT=$((PORT + 10000)) + export AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID=1 + agentic_pip_install --quiet "vllm-router==$VLLM_ROUTER_VERSION" +fi + +# AIPerf automatically scrapes the public endpoint's /metrics URL. That is the +# vLLM engine for pure TP, but the native router for DP-attention. Explicitly +# add the engine endpoint so every topology captures vLLM metrics; AIPerf +# deduplicates it against the automatic endpoint in pure-TP runs. +export AIPERF_SERVER_METRICS_URLS="http://localhost:${VLLM_BACKEND_PORT}/metrics" +export AIPERF_REQUIRED_SERVER_METRIC_PREFIX="vllm:" + +# DeepSeek-V4-Pro weights are large; engine startup can exceed default 600s. +export VLLM_ENGINE_READY_TIMEOUT_S=3600 + +# vllm-project/vllm#43447 keeps local SWA prefix-cache tails sparsely, while +# vllm-project/vllm#44774 applies the same reachability policy to Mooncake's +# store mask. +export VLLM_PREFIX_CACHE_RETENTION_INTERVAL=32768 +export VLLM_USE_V2_MODEL_RUNNER=1 +export VLLM_USE_RUST_FRONTEND=1 +export VLLM_DSV4_MEGA_FP8_COMBINE=1 +export VLLM_RPC_TIMEOUT=600000 + +# ---- Server config ---------------------------------------------------------- +SERVER_LOG="$RESULT_DIR/server.log" +ROUTER_LOG="$RESULT_DIR/router.log" +MOONCAKE_MASTER_LOG="$RESULT_DIR/mooncake_master.log" +mkdir -p "$RESULT_DIR" + +SERVER_PID="" +ROUTER_PID="" +MOONCAKE_MASTER_PID="" + +OFFLOAD_ARGS=() +case "$KV_OFFLOAD_BACKEND" in + "") + require_agentic_kv_offload_none + ;; + vllm-simple) + require_agentic_kv_offload_backend vllm-simple + CPU_BYTES_PER_RANK=$(( TOTAL_CPU_DRAM_GB * 1000 * 1000 * 1000 / GPU_COUNT )) + # Identical prefixes must hash to identical block keys across DP ranks. + export PYTHONHASHSEED=42 + OFFLOAD_CONFIG=$(cat </dev/null + + MOONCAKE_MASTER_PORT=$((PORT + 12000)) + MOONCAKE_CONFIG_PATH="$RESULT_DIR/mooncake_config.json" + cat > "$MOONCAKE_CONFIG_PATH" < "$MOONCAKE_MASTER_LOG" 2>&1 & + MOONCAKE_MASTER_PID=$! + sleep 2 + if ! kill -0 "$MOONCAKE_MASTER_PID" 2>/dev/null; then + echo "Mooncake master died during startup." >&2 + cat "$MOONCAKE_MASTER_LOG" >&2 + exit 1 + fi + unset VLLM_USE_SIMPLE_KV_OFFLOAD + OFFLOAD_ARGS=( + --kv-transfer-config + '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true}}' + ) + ;; + *) + echo "Error: unsupported B200 KV_OFFLOAD_BACKEND='$KV_OFFLOAD_BACKEND'" >&2 + exit 1 + ;; +esac + +PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1) +MODE_ARGS=() +if [ "$DP_ATTENTION" = "true" ]; then + PARALLEL_ARGS=(--tensor-parallel-size 1 --data-parallel-size "$TP") + export PYTORCH_ALLOC_CONF=expandable_segments:True +fi + +if [ "$EP_SIZE" -gt 1 ]; then + MODE_ARGS+=( + --enable-expert-parallel + --enable-ep-weight-filter + --moe-backend deep_gemm_amxf4_mega_moe + ) +else + # The 0813 checkpoint is MXFP4. Select the MoE kernel explicitly rather than + # relying on the implicit default, matching the Kimi-K3 MXFP4 vLLM DSpark + # recipe (benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh). + MODE_ARGS+=(--moe-backend auto) +fi +if [ "$DP_ATTENTION" = "true" ]; then + # Keep B200's profiled activation footprint within the 0.90 GPU-memory + # budget; 16K prefill batches leave too little DeepGEMM runtime headroom. + MODE_ARGS+=( + --prefill-schedule-interval 8 + --long-prefill-token-threshold 512 + --max-num-batched-tokens 8192 + ) +fi + +# AgentX concurrency counts live session trees. Subagent fan-out can push the +# global instantaneous request count above CONC, so retain 2x global headroom +# while avoiding the old 8x over-allocation of that budget on every DEP rank. +if [ "$DP_ATTENTION" = "true" ]; then + MAX_NUM_SEQS=$((2 * CONC / TP)) +else + MAX_NUM_SEQS=$((2 * CONC)) +fi + +# DSpark: cudagraph capture sizes are in TOKENS. With num_speculative_tokens=N, +# every uniform decode batch of S seqs verifies S*(1+N) tokens, so capture the +# explicit multiples (1+N), 2*(1+N), ..., MAX_NUM_SEQS*(1+N). vLLM rounds +# configured sizes up to multiples of (1+N) and deduplicates them; a plain +# 1..MAX_NUM_SEQS list would cover only MAX_NUM_SEQS/(1+N) decode sequences. +NUM_SPEC_TOKENS=6 +TOKENS_PER_SEQ=$((1 + NUM_SPEC_TOKENS)) +# Throughput pins synthetic DSpark acceptance to the committed golden AL +# (thinking_on, num_speculative_tokens=6 -> 3.77, +# golden_al_distribution/dsv4-pro-0813-dspark.yaml). The EVAL_ONLY accuracy run +# uses real target verification instead because synthetic acceptance bypasses +# verification. +if [ "${EVAL_ONLY:-false}" = "true" ]; then + SPEC_CONFIG="{\"method\": \"dspark\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS, \"draft_sample_method\": \"probabilistic\"}" +else + SPEC_CONFIG="{\"method\": \"dspark\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS, \"draft_sample_method\": \"probabilistic\", \"rejection_sample_method\": \"synthetic\", \"synthetic_acceptance_length\": 3.77}" +fi +CAPTURE_SIZE_LIST=() +for ((num_seqs = 1; num_seqs <= MAX_NUM_SEQS; num_seqs++)); do + CAPTURE_SIZE_LIST+=("$((num_seqs * TOKENS_PER_SEQ))") +done +# TP additionally captures piecewise graphs for the mixed prefill/decode batches, +# on top of the full graphs for the uniform decode batches above. Piecewise +# capture requires the compiled graph, so no "mode":0 on that path. The DEP arm +# keeps decode-only capture. The decode multiples and the piecewise sizes overlap +# once MAX_NUM_SEQS*(1+N) passes 100, so sort and de-duplicate. +if [ "$DP_ATTENTION" != "true" ]; then + CAPTURE_SIZE_LIST+=(100 200 300 400 500) +fi +CUDA_GRAPH_CAPTURE_SIZES=$(printf '%s\n' "${CAPTURE_SIZE_LIST[@]}" | sort -n -u | paste -sd, -) +if [ "$DP_ATTENTION" != "true" ]; then + COMPILATION_CONFIG="{\"cudagraph_mode\":\"FULL_AND_PIECEWISE\",\"cudagraph_capture_sizes\":[${CUDA_GRAPH_CAPTURE_SIZES}]}" +else + COMPILATION_CONFIG="{\"cudagraph_mode\":\"FULL_DECODE_ONLY\",\"cudagraph_capture_sizes\":[${CUDA_GRAPH_CAPTURE_SIZES}],\"mode\":0}" +fi + +echo "Starting vllm server..." +export TORCH_CUDA_ARCH_LIST="10.0" +export PYTHONNOUSERSITE=1 +export VLLM_FLOAT32_MATMUL_PRECISION=high +GPU_MEMORY_UTILIZATION="${GPU_MEMORY_UTILIZATION:-0.90}" + +{ set +x; } 2>/dev/null +VLLM_CMD=( + vllm serve "$MODEL_PATH" --served-model-name "$MODEL" + --host 0.0.0.0 + --port "$VLLM_BACKEND_PORT" + --trust-remote-code + --kv-cache-dtype fp8 + --block-size 256 + --max-model-len 1048576 + --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" + --numa-bind + --enable-cumem-allocator + --no-enable-flashinfer-autotune + --tokenizer-mode deepseek_v4 + --tool-call-parser deepseek_v4 + --enable-auto-tool-choice + --reasoning-parser deepseek_v4 + --attention-config '{"backend":"FLASHMLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + --speculative-config "$SPEC_CONFIG" + --no-disable-hybrid-kv-cache-manager + --disable-uvicorn-access-log + --compilation-config "$COMPILATION_CONFIG" + --max-num-seqs "$MAX_NUM_SEQS" + "${PARALLEL_ARGS[@]}" + "${VLLM_CP_ARGS[@]}" + "${MODE_ARGS[@]}" + "${OFFLOAD_ARGS[@]}" +) +printf '%q ' "${VLLM_CMD[@]}" | tee "$RESULT_DIR/vllm_command.txt" +printf '\n' | tee -a "$RESULT_DIR/vllm_command.txt" +"${VLLM_CMD[@]}" > "$SERVER_LOG" 2>&1 & +SERVER_PID=$! +echo "Server PID: $SERVER_PID" + +wait_for_server_ready --port "$VLLM_BACKEND_PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" + +if [ "$USE_VLLM_ROUTER" = "true" ]; then + echo "Starting native vLLM router on port $PORT for $TP DP ranks..." + vllm-router \ + --worker-urls "http://localhost:$VLLM_BACKEND_PORT" \ + --policy "$VLLM_ROUTER_POLICY" \ + --intra-node-data-parallel-size "$TP" \ + --host 0.0.0.0 \ + --port "$PORT" \ + --prometheus-host 127.0.0.1 \ + --prometheus-port "$VLLM_ROUTER_METRICS_PORT" \ + --request-timeout-secs 14400 \ + --disable-retries > "$ROUTER_LOG" 2>&1 & + ROUTER_PID=$! + echo "Router PID: $ROUTER_PID" + wait_for_server_ready --port "$PORT" --server-log "$ROUTER_LOG" --server-pid "$ROUTER_PID" +fi + +if [ "${EVAL_ONLY}" = "true" ]; then + run_eval --port "$PORT" +else + build_replay_cmd "$RESULT_DIR" + run_agentic_replay_and_write_outputs "$RESULT_DIR" +fi diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index bc1e7f5170..adc08bfd3e 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -978,6 +978,23 @@ dsv4-fp4-b200-vllm-agentic-mtp: # DEP8 SimpleCPU + MTP - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, spec-decoding: mtp, conc-list: [32, 64, 96, 128, 160, 196], router: { name: vllm-router, version: "0.1.14" } } +dsv4-fp4-b200-vllm-agentic-dspark: + # Native DSpark draft heads on the 0813 checkpoint, agentic coding only. + # Throughput pins synthetic acceptance to the committed thinking-on golden AL + # for num_speculative_tokens=6; EVAL_ONLY uses real DSpark verification. + image: vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-426e59f + model: deepseek-ai/DeepSeek-V4-Pro-0813 + model-prefix: dsv4 + runner: cluster:b200-nscale + precision: fp4 + framework: vllm + multinode: false + scenarios: + agentic-coding: + - dram-utilization: 0.80 + search-space: + - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, spec-decoding: draft_model, conc-list: [1, 4, 8, 14] } + dsv4-fp4-b200-trt: image: ghcr.io#semianalysisai/trtllm-deepseek-v4:feat-deepseek_v4-c185066 model: deepseek-ai/DeepSeek-V4-Pro diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 792e237fba..adb8adefac 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -7138,3 +7138,14 @@ - "Pin vLLM nightly-9ea8f3ffc354901b740f0b31988900897b7221d7, Dynamo 1.5.0.dev20260908, and NVIDIA/srt-slurm d50ee7280c33d469df8708e363e23be2456e94fb." - "Use Dynamo-native MiniMax parsing, EAGLE3 with FlashInfer, FULL_AND_PIECEWISE CUDA graphs, and wait for the OpenAI chat route before AgentX trace replay." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2925 + +- config-keys: + - dsv4-fp4-b200-vllm-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Add a DeepSeek-V4-Pro FP4 B200 vLLM AgentX configuration for the 0813 checkpoint with native DSpark draft heads." + - "Use the committed thinking-on golden synthetic acceptance target for throughput and real DSpark verification for evaluation." + - "Select the FlashMLA sparse DeepSeek-V4 attention backend and route draft-model jobs to the DSpark-specific benchmark script." + - "Use the official vLLM image as shipped with CPU DRAM key-value cache offloading." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2946 diff --git a/runners/launch_b200-nscale-compat.sh b/runners/launch_b200-nscale-compat.sh index 8ac37aadeb..58f3c96f3b 100644 --- a/runners/launch_b200-nscale-compat.sh +++ b/runners/launch_b200-nscale-compat.sh @@ -509,6 +509,11 @@ else # with multiple inference engines can coexist; fall back to the historical # name without an engine suffix (`_trt` for trt, bare for everyone else). BENCH_BASE="benchmarks/single_node/${SCENARIO_SUBDIR}${EXP_NAME%%_*}_${PRECISION}_b200" + # A DSpark path cannot share the native-MTP script on the same engine because + # their attention backends and speculative configurations differ. + if [[ "$SPEC_DECODING" == "draft_model" && -f "${BENCH_BASE}_${FRAMEWORK}_dspark.sh" ]]; then + SPEC_SUFFIX="_dspark" + fi BENCH_SCRIPT="${BENCH_BASE}_${FRAMEWORK}${SPEC_SUFFIX}.sh" if [[ ! -f "$BENCH_SCRIPT" ]]; then BENCH_SCRIPT="${BENCH_BASE}${FRAMEWORK_SUFFIX}${SPEC_SUFFIX}.sh"