diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-1p1d-tp4-tp4-2-c1_mooncake_store.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-1p1d-tp4-tp4-2-c1_mooncake_store.yaml new file mode 100644 index 0000000000..d246d7f131 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-1p1d-tp4-tp4-2-c1_mooncake_store.yaml @@ -0,0 +1,173 @@ +name: "disagg-gb300-1p1d-tp4-tp4-2-c1-mooncake-kvcache" + +# 8k/1k high-throughput topology for the wideep DSV4-Pro setup. +# +# Schema/values come from PR #1213 (513cbef) — that PR introduced the +# `dsv4-pro-gb300-fp4` upstream-style recipe with two `zip_override` +# variants (wideep [0] / narrow_ep [1]) and `backend.benchmark`. Our +# pinned srtctl (NVIDIA/srt-slurm @ sa-submission-q2-2026) doesn't +# support either: `zip_override_*_hightpt` rejects with `Unknown field` +# and `benchmark` only validates at top level. So this file inlines the +# wideep [0] override and lifts `benchmark` back out — same operational +# values, schema the pinned srtctl will accept. +# +# Other adjustments back to the InferenceX cluster shape: container & +# model.path restored to the aliases mapped in launch_gb300.sh's +# srtslurm.yaml (`lmsysorg/sglang:deepseek-v4-grace-blackwell` and +# `deepseek-v4-pro`); `dynamo.install: true` added so the container +# (which has no dynamo baked in) installs from the pinned hash. +# +# Cluster-specific items NOT inlined (require InferenceX-side equivalents): +# - slurm.partition (the source cluster uses `hpc-mid`) +# - frontend.nginx_container (yangminl's `nginx-1.27.4.sqsh` path) +# - extra_mount: yangminl/sglang-patched/sglang. Earlier diff analysis +# showed only `expert_location_dispatch.py` topk_ids int32 cast is an +# active runtime diff vs container sglang; other patched files are +# env-gated dead code under the same SGLANG_OPT_* flags this yaml +# already sets. +# +# DG-related env intentionally diverged (DG cache path is host-specific): +# - SGLANG_DG_CACHE_DIR=/configs/deepgemm_cache (yangminl host) +# - SGLANG_JIT_DEEPGEMM_PRECOMPILE=0 (yangminl uses prebuilt cache) +# This yaml uses SGLANG_JIT_DEEPGEMM_FAST_WARMUP=1 instead. + +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260624-b2c8f7a2" + precision: "fp4" + +# See ../1k1k/disagg-gb200-1p1d-dep8-tep8.yaml for the dynamo pin +# rationale. Hash bumped from PR #1213 to track the dynamo-sglang dsv4 +# dev branch. +dynamo: + hash: "81d0555ee23519cea80a42b4fe824e30368b7300" + install: true + +slurm: + time_limit: "03:00:00" + +# Match yangminl's working all-dynamo.yaml on the source cluster: +# cpus-per-task=144 — without this slurm hands out 1 CPU/task, which +# turns the dynamo `hash:` cold source build (~500 rust crates, +# ravif/exr/zip/pyo3 stack) into a 30+ min serial compile. With 144 +# cargo finishes in ~5 min. +# mem=0 — slurm's "give the whole node's memory"; needed +# for sglang loading 671B FP4 weights + dynamo build at the same +# time without OOM. +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +# Topology: 7 prefill (TP=4 / DP=4 / EP=4 / 1 node each) + 1 decode +# (TP=8 / DP=8 / EP=8 / 2 nodes). 9 nodes total. +resources: + gpu_type: "gb300" + gpus_per_node: 4 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 4 + decode_nodes: 1 + decode_workers: 1 + gpus_per_decode: 4 + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + +backend: + type: sglang + + # Dedicated mooncake master configuration for kvcache-enabled runs. + mooncake_kv_store: + env: + MOONCAKE_PROTOCOL: "rdma" # remain this even in MNNVL-only + MOONCAKE_DEVICE: "" + MOONCAKE_GLOBAL_SEGMENT_SIZE: "800gb" + MC_FORCE_MNNVL: "1" + + prefill_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + + decode_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + # is single-node only and corrupts results in 2-node decode setups. + + sglang_config: + prefill: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + disable-radix-cache: true + + disaggregation-mode: "prefill" + disaggregation-transfer-backend: mooncake + + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + + moe-runner-backend: "flashinfer_mxfp4" + disable-flashinfer-autotune: true + + mem-fraction-static: 0.90 + max-running-requests: 512 + cuda-graph-max-bs: 512 + chunked-prefill-size: 32768 + + decode: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + disable-radix-cache: true + + disaggregation-mode: "decode" + disaggregation-transfer-backend: mooncake + + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + + moe-runner-backend: "flashinfer_mxfp4" + disable-flashinfer-autotune: true + + mem-fraction-static: 0.9 + max-running-requests: 1024 + cuda-graph-max-bs: 512 + swa-full-tokens-ratio: 0.1 + context-length: 16384 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "1" + req_rate: "inf" + use_chat_template: false + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_1_mooncake_store.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_1_mooncake_store.yaml new file mode 100644 index 0000000000..250cbf6ab3 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_1_mooncake_store.yaml @@ -0,0 +1,183 @@ +# Qwen3.5-397B-A17B-NVFP4 Disaggregated 6P1D wide-EP +# Prefill: 6 workers @ TP4/DP4/EP4 with DP-attn (per-node, DEP4) +# Decode: 1 worker @ TP16/DP16/EP16 with DP-attn + TBO (DEP16, 4 nodes) +# Total: 40 GB300 GPUs (6*4 + 4*4); 8k1k concurrency 5120. +# +# Values taken from ni_experiment_config of pareto row +# qwen3.5-6p_dep4x1d_dep16-fia2a-tbo-cc5120-dynamo-tot-mooncake +# (sa-qwen-3.5-8k1k-fp4-baseline-mid-pareto study). + +name: "gb300-fp4-qwen3.5_8k1k_maxtpt_1_mooncake_kvcache" + +model: + path: "qwen3.5-fp4" + container: "dynamo-sglang" + precision: "fp4" + +dynamo: + version: "1.1.0" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 2 + nginx_container: nginx + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + prefill_nodes: 6 + prefill_workers: 6 + decode_nodes: 4 + decode_workers: 1 + +backend: + type: sglang + + # Dedicated mooncake master configuration for kvcache-enabled runs. + mooncake_kv_store: + env: + MOONCAKE_PROTOCOL: "rdma" # remain this even in MNNVL-only + MOONCAKE_DEVICE: "" + MOONCAKE_GLOBAL_SEGMENT_SIZE: "800gb" + MC_FORCE_MNNVL: "1" + + prefill_environment: + NO_COLOR: "1" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + SGLANG_ENABLE_SPEC_V2: "1" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_ENABLE_JIT_DEEPGEMM: "true" + SGLANG_ENABLE_FLASHINFER_GEMM: "true" + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: "cutlass" + FLASHINFER_DISABLE_VERSION_CHECK: "1" + SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + + decode_environment: + NO_COLOR: "1" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + SGLANG_ENABLE_SPEC_V2: "1" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + MC_FORCE_MNNVL: "1" + MC_TE_METRIC: "true" + SGLANG_ENABLE_JIT_DEEPGEMM: "true" + SGLANG_ENABLE_FLASHINFER_GEMM: "true" + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: "cutlass" + SGLANG_MOE_NVFP4_DISPATCH: "1" + SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: "1" + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: "1" + SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: "1" + FLASHINFER_DISABLE_VERSION_CHECK: "1" + SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "1024" + SGLANG_HEALTH_CHECK_TIMEOUT: "1800" + SGLANG_HEALTH_STARTING_OK: "1" + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" + + sglang_config: + prefill: + served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4" + model-path: "/model/" + trust-remote-code: true + + quantization: "modelopt_fp4" + kv-cache-dtype: "fp8_e4m3" + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + + mamba-scheduler-strategy: "no_buffer" + mamba-track-interval: 2048 + mamba-ssm-dtype: "bfloat16" + + disaggregation-mode: "prefill" + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + disaggregation-transfer-backend: "mooncake" + + mem-fraction-static: 0.8 + max-total-tokens: 128000 + chunked-prefill-size: 65536 + load-balance-method: "round_robin" + watchdog-timeout: 1000000 + log-level: "info" + page-size: 64 + + attention-backend: "trtllm_mha" + moe-runner-backend: "flashinfer_trtllm" + linear-attn-decode-backend: "flashinfer" + + decode: + served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4" + model-path: "/model/" + trust-remote-code: true + + quantization: "modelopt_fp4" + kv-cache-dtype: "fp8_e4m3" + + tensor-parallel-size: 16 + data-parallel-size: 16 + expert-parallel-size: 16 + enable-dp-attention: true + enable-dp-lm-head: true + enable-two-batch-overlap: true + + mamba-scheduler-strategy: "no_buffer" + mamba-track-interval: 128 + mamba-ssm-dtype: "bfloat16" + + disaggregation-mode: "decode" + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + disaggregation-transfer-backend: "mooncake" + + chunked-prefill-size: 5120 + max-mamba-cache-size: 5120 + max-total-tokens: 3200000 + max-running-requests: 5120 + mem-fraction-static: 0.8 + watchdog-timeout: 1000000 + page-size: 64 + + attention-backend: "trtllm_mha" + moe-runner-backend: "flashinfer_cutedsl" + moe-a2a-backend: "flashinfer" + disable-shared-experts-fusion: true + linear-attn-decode-backend: "flashinfer" + + decode-log-interval: 50 + stream-interval: 50 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "5120" + req_rate: "inf" + random_range_ratio: 0.8 diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 74eb03c530..6fa3ac8b05 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -13270,3 +13270,60 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-2p1d-dep8-dep8: tp: 8 ep: 8 dp-attn: true + +dsv4-fp4-gb300-dynamo-sglang-mooncake-store: + image: lmsysorg/sglang:nightly-dev-cu13-20260624-b2c8f7a2 + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: gb300 + precision: fp4 + framework: dynamo-sglang + multinode: true + disagg: true + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + - conc-list: [1] + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb300-1p1d-tp4-tp4-2-c1_mooncake_store.yaml" + decode: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + +qwen3.5-fp4-gb300-dynamo-sglang-mooncake-store: + image: lmsysorg/sglang:nightly-dev-cu13-20260624-b2c8f7a2 + model: nvidia/Qwen3.5-397B-A17B-NVFP4 + model-prefix: qwen3.5 + runner: gb300 + precision: fp4 + framework: dynamo-sglang + multinode: true + disagg: true + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + - spec-decoding: "none" + conc-list: [5120] + prefill: + num-worker: 6 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_1_mooncake_store.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true diff --git a/perf-changelog.yaml b/perf-changelog.yaml index f01199eb1a..3f41475120 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4519,6 +4519,16 @@ - "Update SGLang ROCm image from lmsysorg/sglang-rocm:v0.5.12.post1-rocm720-mi35x-20260528 to lmsysorg/sglang:v0.5.14-rocm720-mi35x" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2067 +- config-keys: + - dsv4-fp4-gb300-dynamo-sglang-mooncake-store + - qwen3.5-fp4-gb300-dynamo-sglang-mooncake-store + description: + - "Add mooncake store-enabled variants for DeepSeek V4 and Qwen 3.5 on GB300, using dedicated _mooncake_store.yaml recipe files with mooncake_kv_store backend (RDMA protocol, 800GB global segment size, MNNVL forced)." + - "Configure mooncake-specific environment variables." + - "Wire launcher routing so CONFIG_FILE values ending with _mooncake_store.yaml use the srt-slurm main branch; legacy recipes continue on their previous branch flow." + - "Pin the DeepSeek V4 mooncake recipe to sglang nightly-dev-cu13-20260624-b2c8f7a2 (the inherited 20260520 nightly was garbage-collected from Docker Hub); Qwen 3.5 mooncake already uses this build." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2085 + - config-keys: - kimik2.5-fp4-mi355x-vllm description: diff --git a/runners/launch_gb300-nv.sh b/runners/launch_gb300-nv.sh index aaf48134fa..9765677e3c 100644 --- a/runners/launch_gb300-nv.sh +++ b/runners/launch_gb300-nv.sh @@ -69,6 +69,13 @@ elif [[ $MODEL_PREFIX == "qwen3.5" && $PRECISION == "fp4" ]]; then # Qwen3.5 sglang recipes (qwen3.5-fp4). export MODEL_PATH=/scratch/models/Qwen3.5-397B-A17B-NVFP4 export SRT_SLURM_MODEL_PREFIX="qwen3.5-fp4" + # The mooncake_store variant's weights live on shared NFS (/data) rather + # than node-local /scratch/models, which isn't populated with Qwen3.5 on + # the compute nodes. /data is mounted on all compute nodes (cf. the + # MiniMax /data/models recipes above). + if [[ "${CONFIG_FILE:-}" == *"_mooncake_store.yaml" ]]; then + export MODEL_PATH=/data/home/sa-shared/models/Qwen3.5-397B-A17B-NVFP4 + fi else echo "Unsupported model: $MODEL_PREFIX-$PRECISION. Supported models are: dsr1-fp4, dsr1-fp8, dsv4-fp4, glm5-fp4, glm5-fp8, minimaxm2.5-fp4, minimaxm2.5-fp8, kimik2.5-fp4, qwen3.5-fp4" exit 1 @@ -164,6 +171,20 @@ elif [[ $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "glm5" ]]; then mkdir -p recipes/sglang/glm5/gb300-fp4 cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp4" recipes/sglang/glm5/gb300-fp4 fi +elif [[ $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "dsv4" && "${CONFIG_FILE:-}" == *"_mooncake_store.yaml" ]]; then + # Mooncake-kvcache variants use main; legacy DSV4 recipes keep default flow. + git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" + cd "$SRT_REPO_DIR" + git checkout main + mkdir -p recipes/sglang/deepseek-v4 + cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4" recipes/sglang/deepseek-v4 +elif [[ $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "qwen3.5" && "${CONFIG_FILE:-}" == *"_mooncake_store.yaml" ]]; then + # Mooncake-kvcache variants use main; legacy Qwen3.5 recipes keep submission branch. + git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" + cd "$SRT_REPO_DIR" + git checkout main + mkdir -p recipes/sglang/qwen3.5 + cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5" recipes/sglang/qwen3.5 elif [[ $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "qwen3.5" ]]; then # Overlay our version-controlled Qwen3.5 recipes onto the submission branch. git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR"