Add mooncake store for dsv4 and qwen35

benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-1p1d-tp4-tp4-2-c1_mooncake_store.yaml

@@ -0,0 +1,173 @@
+name: "disagg-gb300-1p1d-tp4-tp4-2-c1-mooncake-kvcache"
+
+# 8k/1k high-throughput topology for the wideep DSV4-Pro setup.
+#
+# Schema/values come from PR #1213 (513cbef) — that PR introduced the
+# `dsv4-pro-gb300-fp4` upstream-style recipe with two `zip_override`
+# variants (wideep [0] / narrow_ep [1]) and `backend.benchmark`. Our
+# pinned srtctl (NVIDIA/srt-slurm @ sa-submission-q2-2026) doesn't
+# support either: `zip_override_*_hightpt` rejects with `Unknown field`
+# and `benchmark` only validates at top level. So this file inlines the
+# wideep [0] override and lifts `benchmark` back out — same operational
+# values, schema the pinned srtctl will accept.
+#
+# Other adjustments back to the InferenceX cluster shape: container &
+# model.path restored to the aliases mapped in launch_gb300.sh's
+# srtslurm.yaml (`lmsysorg/sglang:deepseek-v4-grace-blackwell` and
+# `deepseek-v4-pro`); `dynamo.install: true` added so the container
+# (which has no dynamo baked in) installs from the pinned hash.
+#
+# Cluster-specific items NOT inlined (require InferenceX-side equivalents):
+#   - slurm.partition (the source cluster uses `hpc-mid`)
+#   - frontend.nginx_container (yangminl's `nginx-1.27.4.sqsh` path)
+#   - extra_mount: yangminl/sglang-patched/sglang. Earlier diff analysis
+#     showed only `expert_location_dispatch.py` topk_ids int32 cast is an
+#     active runtime diff vs container sglang; other patched files are
+#     env-gated dead code under the same SGLANG_OPT_* flags this yaml
+#     already sets.
+#
+# DG-related env intentionally diverged (DG cache path is host-specific):
+#   - SGLANG_DG_CACHE_DIR=/configs/deepgemm_cache (yangminl host)
+#   - SGLANG_JIT_DEEPGEMM_PRECOMPILE=0 (yangminl uses prebuilt cache)
+#   This yaml uses SGLANG_JIT_DEEPGEMM_FAST_WARMUP=1 instead.
+
+model:
+  path: "deepseek-v4-pro"
+  container: "lmsysorg/sglang:nightly-dev-cu13-20260624-b2c8f7a2"
+  precision: "fp4"
+
+# See ../1k1k/disagg-gb200-1p1d-dep8-tep8.yaml for the dynamo pin
+# rationale. Hash bumped from PR #1213 to track the dynamo-sglang dsv4
+# dev branch.
+dynamo:
+  hash: "81d0555ee23519cea80a42b4fe824e30368b7300"
+  install: true
+
+slurm:
+  time_limit: "03:00:00"
+
+# Match yangminl's working all-dynamo.yaml on the source cluster:
+#   cpus-per-task=144  — without this slurm hands out 1 CPU/task, which
+#     turns the dynamo `hash:` cold source build (~500 rust crates,
+#     ravif/exr/zip/pyo3 stack) into a 30+ min serial compile. With 144
+#     cargo finishes in ~5 min.
+#   mem=0              — slurm's "give the whole node's memory"; needed
+#     for sglang loading 671B FP4 weights + dynamo build at the same
+#     time without OOM.
+sbatch_directives:
+  cpus-per-task: "144"
+  mem: "0"
+
+# Topology: 7 prefill (TP=4 / DP=4 / EP=4 / 1 node each) + 1 decode
+# (TP=8 / DP=8 / EP=8 / 2 nodes). 9 nodes total.
+resources:
+  gpu_type: "gb300"
+  gpus_per_node: 4
+  prefill_nodes: 1
+  prefill_workers: 1
+  gpus_per_prefill: 4
+  decode_nodes: 1
+  decode_workers: 1
+  gpus_per_decode: 4
+
+frontend:
+  type: dynamo
+  enable_multiple_frontends: true
+  num_additional_frontends: 8
+
+backend:
+  type: sglang
+
+  # Dedicated mooncake master configuration for kvcache-enabled runs.
+  mooncake_kv_store:
+    env:
+      MOONCAKE_PROTOCOL: "rdma" # remain this even in MNNVL-only
+      MOONCAKE_DEVICE: ""
+      MOONCAKE_GLOBAL_SEGMENT_SIZE: "800gb"
+      MC_FORCE_MNNVL: "1"
+
+  prefill_environment:
+    PYTHONUNBUFFERED: "1"
+    SGLANG_RADIX_FORCE_MISS: "1"
+    SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1"
+    SGLANG_DEFAULT_THINKING: "1"
+    SGLANG_DSV4_REASONING_EFFORT: "max"
+    SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1"
+    SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1"
+    NCCL_MNNVL_ENABLE: "1"
+    NCCL_CUMEM_ENABLE: "1"
+    SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True"
+    MC_FORCE_MNNVL: "1"
+    SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000"
+    SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000"
+    SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1"
+
+  decode_environment:
+    PYTHONUNBUFFERED: "1"
+    SGLANG_RADIX_FORCE_MISS: "1"
+    SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1"
+    SGLANG_DEFAULT_THINKING: "1"
+    SGLANG_DSV4_REASONING_EFFORT: "max"
+    SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1"
+    SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1"
+    NCCL_MNNVL_ENABLE: "1"
+    NCCL_CUMEM_ENABLE: "1"
+    SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True"
+    MC_FORCE_MNNVL: "1"
+    SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000"
+    SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000"
+    SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1"
+    # is single-node only and corrupts results in 2-node decode setups.
+
+  sglang_config:
+    prefill:
+      served-model-name: "deepseek-ai/DeepSeek-V4-Pro"
+      model-path: "/model/"
+      trust-remote-code: true
+      disable-radix-cache: true
+
+      disaggregation-mode: "prefill"
+      disaggregation-transfer-backend: mooncake
+
+      tensor-parallel-size: 4
+      data-parallel-size:   1
+      expert-parallel-size: 1
+
+      moe-runner-backend: "flashinfer_mxfp4"
+      disable-flashinfer-autotune: true
+
+      mem-fraction-static: 0.90
+      max-running-requests: 512
+      cuda-graph-max-bs: 512
+      chunked-prefill-size: 32768
+
+    decode:
+      served-model-name: "deepseek-ai/DeepSeek-V4-Pro"
+      model-path: "/model/"
+      trust-remote-code: true
+      disable-radix-cache: true
+
+      disaggregation-mode: "decode"
+      disaggregation-transfer-backend: mooncake
+
+      tensor-parallel-size: 4
+      data-parallel-size:   1
+      expert-parallel-size: 1
+
+      moe-runner-backend: "flashinfer_mxfp4"
+      disable-flashinfer-autotune: true
+
+      mem-fraction-static: 0.9
+      max-running-requests: 1024
+      cuda-graph-max-bs:    512
+      swa-full-tokens-ratio: 0.1
+      context-length: 16384
+
+benchmark:
+  type: "sa-bench"
+  isl: 8192
+  osl: 1024
+  concurrencies: "1"
+  req_rate: "inf"
+  use_chat_template: false
+  custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer"

benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_1_mooncake_store.yaml

@@ -0,0 +1,183 @@
+# Qwen3.5-397B-A17B-NVFP4 Disaggregated 6P1D wide-EP
+# Prefill: 6 workers @ TP4/DP4/EP4 with DP-attn (per-node, DEP4)
+# Decode:  1 worker  @ TP16/DP16/EP16 with DP-attn + TBO (DEP16, 4 nodes)
+# Total: 40 GB300 GPUs (6*4 + 4*4); 8k1k concurrency 5120.
+#
+# Values taken from ni_experiment_config of pareto row
+# qwen3.5-6p_dep4x1d_dep16-fia2a-tbo-cc5120-dynamo-tot-mooncake
+# (sa-qwen-3.5-8k1k-fp4-baseline-mid-pareto study).
+
+name: "gb300-fp4-qwen3.5_8k1k_maxtpt_1_mooncake_kvcache"
+
+model:
+  path: "qwen3.5-fp4"
+  container: "dynamo-sglang"
+  precision: "fp4"
+
+dynamo:
+  version: "1.1.0"
+
+frontend:
+  type: dynamo
+  enable_multiple_frontends: true
+  num_additional_frontends: 2
+  nginx_container: nginx
+
+resources:
+  gpu_type: "gb300"
+  gpus_per_node: 4
+  prefill_nodes: 6
+  prefill_workers: 6
+  decode_nodes: 4
+  decode_workers: 1
+
+backend:
+  type: sglang
+
+  # Dedicated mooncake master configuration for kvcache-enabled runs.
+  mooncake_kv_store:
+    env:
+      MOONCAKE_PROTOCOL: "rdma" # remain this even in MNNVL-only
+      MOONCAKE_DEVICE: ""
+      MOONCAKE_GLOBAL_SEGMENT_SIZE: "800gb"
+      MC_FORCE_MNNVL: "1"
+
+  prefill_environment:
+    NO_COLOR: "1"
+    TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800"
+    SGLANG_ENABLE_SPEC_V2: "1"
+    PYTHONUNBUFFERED: "1"
+    NCCL_MNNVL_ENABLE: "1"
+    NCCL_CUMEM_ENABLE: "1"
+    NCCL_NVLS_ENABLE: "1"
+    MC_FORCE_MNNVL: "1"
+    SGLANG_ENABLE_JIT_DEEPGEMM: "true"
+    SGLANG_ENABLE_FLASHINFER_GEMM: "true"
+    SGLANG_FLASHINFER_FP4_GEMM_BACKEND: "cutlass"
+    FLASHINFER_DISABLE_VERSION_CHECK: "1"
+    SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache"
+    FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache"
+    SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000"
+    SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000"
+    SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000"
+    SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True"
+    SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0"
+    SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1"
+
+  decode_environment:
+    NO_COLOR: "1"
+    TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800"
+    SGLANG_ENABLE_SPEC_V2: "1"
+    PYTHONUNBUFFERED: "1"
+    NCCL_MNNVL_ENABLE: "1"
+    NCCL_CUMEM_ENABLE: "1"
+    NCCL_NVLS_ENABLE: "1"
+    MC_FORCE_MNNVL: "1"
+    MC_TE_METRIC: "true"
+    SGLANG_ENABLE_JIT_DEEPGEMM: "true"
+    SGLANG_ENABLE_FLASHINFER_GEMM: "true"
+    SGLANG_FLASHINFER_FP4_GEMM_BACKEND: "cutlass"
+    SGLANG_MOE_NVFP4_DISPATCH: "1"
+    SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: "1"
+    SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: "1"
+    SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: "1"
+    FLASHINFER_DISABLE_VERSION_CHECK: "1"
+    SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache"
+    FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache"
+    SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000"
+    SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000"
+    SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000"
+    SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000"
+    SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1"
+    SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True"
+    SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0"
+    SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1"
+    SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "1024"
+    SGLANG_HEALTH_CHECK_TIMEOUT: "1800"
+    SGLANG_HEALTH_STARTING_OK: "1"
+    SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0"
+
+  sglang_config:
+    prefill:
+      served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4"
+      model-path: "/model/"
+      trust-remote-code: true
+
+      quantization: "modelopt_fp4"
+      kv-cache-dtype: "fp8_e4m3"
+
+      tensor-parallel-size: 4
+      data-parallel-size: 4
+      expert-parallel-size: 4
+      enable-dp-attention: true
+      enable-dp-lm-head: true
+
+      mamba-scheduler-strategy: "no_buffer"
+      mamba-track-interval: 2048
+      mamba-ssm-dtype: "bfloat16"
+
+      disaggregation-mode: "prefill"
+      disable-radix-cache: true
+      disaggregation-bootstrap-port: 31000
+      disaggregation-transfer-backend: "mooncake"
+
+      mem-fraction-static: 0.8
+      max-total-tokens: 128000
+      chunked-prefill-size: 65536
+      load-balance-method: "round_robin"
+      watchdog-timeout: 1000000
+      log-level: "info"
+      page-size: 64
+
+      attention-backend: "trtllm_mha"
+      moe-runner-backend: "flashinfer_trtllm"
+      linear-attn-decode-backend: "flashinfer"
+
+    decode:
+      served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4"
+      model-path: "/model/"
+      trust-remote-code: true
+
+      quantization: "modelopt_fp4"
+      kv-cache-dtype: "fp8_e4m3"
+
+      tensor-parallel-size: 16
+      data-parallel-size: 16
+      expert-parallel-size: 16
+      enable-dp-attention: true
+      enable-dp-lm-head: true
+      enable-two-batch-overlap: true
+
+      mamba-scheduler-strategy: "no_buffer"
+      mamba-track-interval: 128
+      mamba-ssm-dtype: "bfloat16"
+
+      disaggregation-mode: "decode"
+      disable-radix-cache: true
+      disaggregation-bootstrap-port: 31000
+      disaggregation-transfer-backend: "mooncake"
+
+      chunked-prefill-size: 5120
+      max-mamba-cache-size: 5120
+      max-total-tokens: 3200000
+      max-running-requests: 5120
+      mem-fraction-static: 0.8
+      watchdog-timeout: 1000000
+      page-size: 64
+
+      attention-backend: "trtllm_mha"
+      moe-runner-backend: "flashinfer_cutedsl"
+      moe-a2a-backend: "flashinfer"
+      disable-shared-experts-fusion: true
+      linear-attn-decode-backend: "flashinfer"
+
+      decode-log-interval: 50
+      stream-interval: 50
+
+benchmark:
+  type: "sa-bench"
+  isl: 8192
+  osl: 1024
+  concurrencies: "5120"
+  req_rate: "inf"
+  random_range_ratio: 0.8