From b487e86db324e501224ce548638b71e1e56814a8 Mon Sep 17 00:00:00 2001 From: Rohit Pujar Nagraj Date: Mon, 27 Jul 2026 13:54:46 -0700 Subject: [PATCH 1/4] feat(recipes): add DSV4 B300 SGLang disagg configs Signed-off-by: Rohit Pujar Nagraj --- .../disagg-b300-1p1d-dep4-dep8-mtp.yaml | 142 ++++++++++++++++ .../disagg/disagg-b300-1p1d-dep4-dep8.yaml | 153 ++++++++++++++++++ .../disagg/disagg-b300-1p1d-tp4-tp4-mtp.yaml | 133 +++++++++++++++ .../8k1k/disagg/disagg-b300-1p1d-tp4-tp4.yaml | 128 +++++++++++++++ .../disagg-b300-1p2d-dep4-dep8-mtp.yaml | 143 ++++++++++++++++ .../disagg/disagg-b300-1p6d-dep4-tp4-mtp.yaml | 137 ++++++++++++++++ .../disagg-b300-2p1d-dep4-dep8-mtp.yaml | 143 ++++++++++++++++ .../disagg/disagg-b300-2p1d-dep4-dep8.yaml | 153 ++++++++++++++++++ .../disagg-b300-4p1d-dep4-dep8-mtp.yaml | 143 ++++++++++++++++ .../disagg/disagg-b300-4p1d-dep4-dep8.yaml | 153 ++++++++++++++++++ .../disagg-b300-6p1d-dep4-dep8-mtp.yaml | 143 ++++++++++++++++ .../disagg/disagg-b300-6p1d-dep4-dep8.yaml | 153 ++++++++++++++++++ .../disagg-b300-8p1d-dep4-dep8-mtp.yaml | 143 ++++++++++++++++ 13 files changed, 1867 insertions(+) create mode 100644 recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-1p1d-dep4-dep8-mtp.yaml create mode 100644 recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-1p1d-dep4-dep8.yaml create mode 100644 recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-1p1d-tp4-tp4-mtp.yaml create mode 100644 recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-1p1d-tp4-tp4.yaml create mode 100644 recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-1p2d-dep4-dep8-mtp.yaml create mode 100644 recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-1p6d-dep4-tp4-mtp.yaml create mode 100644 recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-2p1d-dep4-dep8-mtp.yaml create mode 100644 recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-2p1d-dep4-dep8.yaml create mode 100644 recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-4p1d-dep4-dep8-mtp.yaml create mode 100644 recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-4p1d-dep4-dep8.yaml create mode 100644 recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-6p1d-dep4-dep8-mtp.yaml create mode 100644 recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-6p1d-dep4-dep8.yaml create mode 100644 recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-8p1d-dep4-dep8-mtp.yaml diff --git a/recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-1p1d-dep4-dep8-mtp.yaml b/recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-1p1d-dep4-dep8-mtp.yaml new file mode 100644 index 000000000..a2a058fab --- /dev/null +++ b/recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-1p1d-dep4-dep8-mtp.yaml @@ -0,0 +1,142 @@ +name: "dsv4-pro-b300-disagg-8k1k-1p1d-dep4-dep8-mtp" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + +dynamo: + hash: "41882ae9b07232eed4850fb1daf8c958abb2556a" + install: true + +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4" + precision: "fp4" + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "b300" + gpus_per_node: 8 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 4 + decode_nodes: 1 + decode_workers: 1 + gpus_per_decode: 8 + +backend: + type: sglang + + prefill_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + + decode_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "2048" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" + + sglang_config: + prefill: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + + disaggregation-mode: "prefill" + watchdog-timeout: 86400 + disaggregation-transfer-backend: nixl + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + + enable-dp-attention: true + enable-dp-lm-head: true + moe-a2a-backend: "megamoe" + mem-fraction-static: 0.9 + max-running-requests: 256 + cuda-graph-max-bs: 256 + chunked-prefill-size: 32768 + stream-interval: 60 + + decode: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + + disaggregation-mode: "decode" + watchdog-timeout: 86400 + disaggregation-transfer-backend: nixl + + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + + enable-dp-attention: true + enable-dp-lm-head: true + moe-a2a-backend: "megamoe" + + speculative-algo: "EAGLE" + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + mem-fraction-static: 0.94 + max-running-requests: 3072 + cuda-graph-max-bs: 256 + swa-full-tokens-ratio: 0.15 + context-length: 16384 + stream-interval: 60 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + random_range_ratio: 0.8 + concurrencies: "256" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + revision: "0366e4e064385807ea86b088a5c6c878ff23343b" + container: + image: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4@sha256:86ea65a66c45e6597292986068523ca7738e2e949c31bfd93330db9870e1a930" + frameworks: + dynamo: "41882ae9b07232eed4850fb1daf8c958abb2556a" + sglang: "9462c303a5c5349488e84b2c346e9b5a066de21c" diff --git a/recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-1p1d-dep4-dep8.yaml b/recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-1p1d-dep4-dep8.yaml new file mode 100644 index 000000000..5a1dcdf2c --- /dev/null +++ b/recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-1p1d-dep4-dep8.yaml @@ -0,0 +1,153 @@ +name: "dsv4-pro-b300-disagg-8k1k-1p1d-dep4-dep8" + +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4" + precision: "fp4" + +dynamo: + hash: "41882ae9b07232eed4850fb1daf8c958abb2556a" + install: true + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "b300" + gpus_per_node: 8 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 4 + decode_nodes: 1 + decode_workers: 1 + gpus_per_decode: 8 + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + env: + args: + router-mode: "kv" + router-kv-overlap-score-weight: 0 + router-queue-threshold: 64 + router-temperature: 0.5 + no-kv-events: true + +backend: + type: sglang + + prefill_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_LOG_FORWARD_ITERS: "1" + SGLANG_LOG_MS: "1" + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" + + decode_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "1280" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: "8" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_LOG_FORWARD_ITERS: "1" + SGLANG_LOG_MS: "1" + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" + + sglang_config: + prefill: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + trust-remote-code: true + watchdog-timeout: 86400 + skip-tokenizer-init: true + stream-interval: 60 + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + + enable-dp-attention: true + moe-a2a-backend: "megamoe" + moe-dense-tp-size: 1 + + disaggregation-mode: "prefill" + disaggregation-transfer-backend: nixl + enable-dp-lm-head: true + + mem-fraction-static: 0.90 + max-running-requests: 512 + cuda-graph-max-bs: 512 + chunked-prefill-size: 32768 + + decode: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + trust-remote-code: true + watchdog-timeout: 86400 + skip-tokenizer-init: true + stream-interval: 60 + + moe-a2a-backend: "megamoe" + moe-dense-tp-size: 1 + + disaggregation-mode: "decode" + disaggregation-transfer-backend: nixl + disaggregation-decode-polling-interval: 8 + + mem-fraction-static: 0.94 + swa-full-tokens-ratio: 0.20 + context-length: 9216 + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + max-running-requests: 18432 + cuda-graph-max-bs: 1280 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "256" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + revision: "0366e4e064385807ea86b088a5c6c878ff23343b" + container: + image: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4@sha256:86ea65a66c45e6597292986068523ca7738e2e949c31bfd93330db9870e1a930" + frameworks: + dynamo: "41882ae9b07232eed4850fb1daf8c958abb2556a" + sglang: "9462c303a5c5349488e84b2c346e9b5a066de21c" diff --git a/recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-1p1d-tp4-tp4-mtp.yaml b/recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-1p1d-tp4-tp4-mtp.yaml new file mode 100644 index 000000000..60f87a01b --- /dev/null +++ b/recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-1p1d-tp4-tp4-mtp.yaml @@ -0,0 +1,133 @@ +name: "dsv4-pro-b300-disagg-8k1k-1p1d-tp4-tp4-mtp" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + +dynamo: + hash: "41882ae9b07232eed4850fb1daf8c958abb2556a" + install: true + +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4" + precision: "fp4" + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "b300" + gpus_per_node: 8 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 4 + decode_nodes: 1 + decode_workers: 1 + gpus_per_decode: 4 + +backend: + type: sglang + + prefill_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + + decode_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + + sglang_config: + prefill: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + + disaggregation-mode: "prefill" + watchdog-timeout: 86400 + disaggregation-transfer-backend: nixl + + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + + moe-runner-backend: "flashinfer_mxfp4" + disable-flashinfer-autotune: true + + mem-fraction-static: 0.9 + max-running-requests: 8 + cuda-graph-max-bs: 8 + chunked-prefill-size: 32768 + + + decode: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + + disaggregation-mode: "decode" + watchdog-timeout: 86400 + disaggregation-transfer-backend: nixl + + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + + moe-runner-backend: "flashinfer_mxfp4" + disable-flashinfer-autotune: true + + speculative-algo: "EAGLE" + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + mem-fraction-static: 0.9 + max-running-requests: 8 + cuda-graph-max-bs: 8 + swa-full-tokens-ratio: 0.1 + context-length: 16384 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + random_range_ratio: 0.8 + concurrencies: "1" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + revision: "0366e4e064385807ea86b088a5c6c878ff23343b" + container: + image: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4@sha256:86ea65a66c45e6597292986068523ca7738e2e949c31bfd93330db9870e1a930" + frameworks: + dynamo: "41882ae9b07232eed4850fb1daf8c958abb2556a" + sglang: "9462c303a5c5349488e84b2c346e9b5a066de21c" diff --git a/recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-1p1d-tp4-tp4.yaml b/recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-1p1d-tp4-tp4.yaml new file mode 100644 index 000000000..c9b876751 --- /dev/null +++ b/recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-1p1d-tp4-tp4.yaml @@ -0,0 +1,128 @@ +name: "dsv4-pro-b300-disagg-8k1k-1p1d-tp4-tp4" + +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4" + precision: "fp4" + +dynamo: + hash: "41882ae9b07232eed4850fb1daf8c958abb2556a" + install: true + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "b300" + gpus_per_node: 8 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 4 + decode_nodes: 1 + decode_workers: 1 + gpus_per_decode: 4 + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + +backend: + type: sglang + + prefill_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + + decode_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + + sglang_config: + prefill: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + disable-radix-cache: true + + disaggregation-mode: "prefill" + watchdog-timeout: 86400 + disaggregation-transfer-backend: nixl + + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + + moe-runner-backend: "flashinfer_mxfp4" + disable-flashinfer-autotune: true + + mem-fraction-static: 0.90 + max-running-requests: 32 + cuda-graph-max-bs: 32 + chunked-prefill-size: 32768 + + + decode: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + disable-radix-cache: true + + disaggregation-mode: "decode" + watchdog-timeout: 86400 + disaggregation-transfer-backend: nixl + + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + + moe-runner-backend: "flashinfer_mxfp4" + disable-flashinfer-autotune: true + + mem-fraction-static: 0.9 + max-running-requests: 64 + cuda-graph-max-bs: 64 + swa-full-tokens-ratio: 0.1 + context-length: 16384 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + random_range_ratio: 0.8 + concurrencies: "1x8x16x32" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + revision: "0366e4e064385807ea86b088a5c6c878ff23343b" + container: + image: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4@sha256:86ea65a66c45e6597292986068523ca7738e2e949c31bfd93330db9870e1a930" + frameworks: + dynamo: "41882ae9b07232eed4850fb1daf8c958abb2556a" + sglang: "9462c303a5c5349488e84b2c346e9b5a066de21c" diff --git a/recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-1p2d-dep4-dep8-mtp.yaml b/recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-1p2d-dep4-dep8-mtp.yaml new file mode 100644 index 000000000..278a2390f --- /dev/null +++ b/recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-1p2d-dep4-dep8-mtp.yaml @@ -0,0 +1,143 @@ +name: "dsv4-pro-b300-disagg-8k1k-1p2d-dep4-dep8-mtp" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + +dynamo: + hash: "41882ae9b07232eed4850fb1daf8c958abb2556a" + install: true + +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4" + precision: "fp4" + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "b300" + gpus_per_node: 8 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 4 + decode_nodes: 2 + decode_workers: 2 + gpus_per_decode: 8 + +backend: + type: sglang + + prefill_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + + decode_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "2048" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" + + sglang_config: + prefill: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + + disaggregation-mode: "prefill" + watchdog-timeout: 86400 + disaggregation-transfer-backend: nixl + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + + enable-dp-attention: true + enable-dp-lm-head: true + moe-a2a-backend: "megamoe" + + mem-fraction-static: 0.9 + max-running-requests: 256 + cuda-graph-max-bs: 256 + chunked-prefill-size: 32768 + stream-interval: 60 + + decode: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + + disaggregation-mode: "decode" + watchdog-timeout: 86400 + disaggregation-transfer-backend: nixl + + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + + enable-dp-attention: true + enable-dp-lm-head: true + moe-a2a-backend: "megamoe" + + speculative-algo: "EAGLE" + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + mem-fraction-static: 0.94 + max-running-requests: 3072 + cuda-graph-max-bs: 256 + swa-full-tokens-ratio: 0.15 + context-length: 16384 + stream-interval: 60 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + random_range_ratio: 0.8 + concurrencies: "256" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + revision: "0366e4e064385807ea86b088a5c6c878ff23343b" + container: + image: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4@sha256:86ea65a66c45e6597292986068523ca7738e2e949c31bfd93330db9870e1a930" + frameworks: + dynamo: "41882ae9b07232eed4850fb1daf8c958abb2556a" + sglang: "9462c303a5c5349488e84b2c346e9b5a066de21c" diff --git a/recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-1p6d-dep4-tp4-mtp.yaml b/recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-1p6d-dep4-tp4-mtp.yaml new file mode 100644 index 000000000..b3f0a4c11 --- /dev/null +++ b/recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-1p6d-dep4-tp4-mtp.yaml @@ -0,0 +1,137 @@ +name: "dsv4-pro-b300-disagg-8k1k-1p6d-dep4-tp4-mtp" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + +dynamo: + hash: "41882ae9b07232eed4850fb1daf8c958abb2556a" + install: true + +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4" + precision: "fp4" + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "b300" + gpus_per_node: 8 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 4 + decode_nodes: 3 + decode_workers: 6 + gpus_per_decode: 4 + +backend: + type: sglang + + prefill_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + + decode_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + + sglang_config: + prefill: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + + disaggregation-mode: "prefill" + watchdog-timeout: 86400 + disaggregation-transfer-backend: nixl + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + + enable-dp-attention: true + enable-dp-lm-head: true + moe-a2a-backend: "megamoe" + + mem-fraction-static: 0.9 + max-running-requests: 128 + cuda-graph-max-bs: 128 + chunked-prefill-size: 32768 + + decode: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + + disaggregation-mode: "decode" + watchdog-timeout: 86400 + disaggregation-transfer-backend: nixl + + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + + moe-runner-backend: "flashinfer_mxfp4" + disable-flashinfer-autotune: true + + speculative-algo: "EAGLE" + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + mem-fraction-static: 0.9 + max-running-requests: 128 + cuda-graph-max-bs: 128 + swa-full-tokens-ratio: 0.1 + context-length: 16384 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + random_range_ratio: 0.8 + concurrencies: "8x32x64" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + revision: "0366e4e064385807ea86b088a5c6c878ff23343b" + container: + image: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4@sha256:86ea65a66c45e6597292986068523ca7738e2e949c31bfd93330db9870e1a930" + frameworks: + dynamo: "41882ae9b07232eed4850fb1daf8c958abb2556a" + sglang: "9462c303a5c5349488e84b2c346e9b5a066de21c" diff --git a/recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-2p1d-dep4-dep8-mtp.yaml b/recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-2p1d-dep4-dep8-mtp.yaml new file mode 100644 index 000000000..2804bb6fe --- /dev/null +++ b/recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-2p1d-dep4-dep8-mtp.yaml @@ -0,0 +1,143 @@ +name: "dsv4-pro-b300-disagg-8k1k-2p1d-dep4-dep8-mtp" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + +dynamo: + hash: "41882ae9b07232eed4850fb1daf8c958abb2556a" + install: true + +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4" + precision: "fp4" + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "b300" + gpus_per_node: 8 + prefill_nodes: 1 + prefill_workers: 2 + gpus_per_prefill: 4 + decode_nodes: 1 + decode_workers: 1 + gpus_per_decode: 8 + +backend: + type: sglang + + prefill_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + + decode_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "4096" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" + + sglang_config: + prefill: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + + disaggregation-mode: "prefill" + watchdog-timeout: 86400 + disaggregation-transfer-backend: nixl + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + + enable-dp-attention: true + enable-dp-lm-head: true + moe-a2a-backend: "megamoe" + + mem-fraction-static: 0.9 + max-running-requests: 512 + cuda-graph-max-bs: 512 + chunked-prefill-size: 32768 + stream-interval: 60 + + decode: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + + disaggregation-mode: "decode" + watchdog-timeout: 86400 + disaggregation-transfer-backend: nixl + + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + + enable-dp-attention: true + enable-dp-lm-head: true + moe-a2a-backend: "megamoe" + + speculative-algo: "EAGLE" + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + mem-fraction-static: 0.9 + max-running-requests: 3072 + cuda-graph-max-bs: 512 + swa-full-tokens-ratio: 0.15 + context-length: 16384 + stream-interval: 60 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + random_range_ratio: 0.8 + concurrencies: "512" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + revision: "0366e4e064385807ea86b088a5c6c878ff23343b" + container: + image: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4@sha256:86ea65a66c45e6597292986068523ca7738e2e949c31bfd93330db9870e1a930" + frameworks: + dynamo: "41882ae9b07232eed4850fb1daf8c958abb2556a" + sglang: "9462c303a5c5349488e84b2c346e9b5a066de21c" diff --git a/recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-2p1d-dep4-dep8.yaml b/recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-2p1d-dep4-dep8.yaml new file mode 100644 index 000000000..a373aae5d --- /dev/null +++ b/recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-2p1d-dep4-dep8.yaml @@ -0,0 +1,153 @@ +name: "dsv4-pro-b300-disagg-8k1k-2p1d-dep4-dep8" + +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4" + precision: "fp4" + +dynamo: + hash: "41882ae9b07232eed4850fb1daf8c958abb2556a" + install: true + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "b300" + gpus_per_node: 8 + prefill_nodes: 1 + prefill_workers: 2 + gpus_per_prefill: 4 + decode_nodes: 1 + decode_workers: 1 + gpus_per_decode: 8 + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + env: + args: + router-mode: "kv" + router-kv-overlap-score-weight: 0 + router-queue-threshold: 64 + router-temperature: 0.5 + no-kv-events: true + +backend: + type: sglang + + prefill_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_LOG_FORWARD_ITERS: "1" + SGLANG_LOG_MS: "1" + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" + + decode_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "1280" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: "8" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_LOG_FORWARD_ITERS: "1" + SGLANG_LOG_MS: "1" + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" + + sglang_config: + prefill: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + trust-remote-code: true + watchdog-timeout: 86400 + skip-tokenizer-init: true + stream-interval: 60 + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + + enable-dp-attention: true + moe-a2a-backend: "megamoe" + moe-dense-tp-size: 1 + + disaggregation-mode: "prefill" + disaggregation-transfer-backend: nixl + enable-dp-lm-head: true + + mem-fraction-static: 0.90 + max-running-requests: 512 + cuda-graph-max-bs: 512 + chunked-prefill-size: 32768 + + decode: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + trust-remote-code: true + watchdog-timeout: 86400 + skip-tokenizer-init: true + stream-interval: 60 + + moe-a2a-backend: "megamoe" + moe-dense-tp-size: 1 + + disaggregation-mode: "decode" + disaggregation-transfer-backend: nixl + disaggregation-decode-polling-interval: 8 + + mem-fraction-static: 0.94 + swa-full-tokens-ratio: 0.20 + context-length: 9216 + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + max-running-requests: 18432 + cuda-graph-max-bs: 1280 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "768" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + revision: "0366e4e064385807ea86b088a5c6c878ff23343b" + container: + image: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4@sha256:86ea65a66c45e6597292986068523ca7738e2e949c31bfd93330db9870e1a930" + frameworks: + dynamo: "41882ae9b07232eed4850fb1daf8c958abb2556a" + sglang: "9462c303a5c5349488e84b2c346e9b5a066de21c" diff --git a/recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-4p1d-dep4-dep8-mtp.yaml b/recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-4p1d-dep4-dep8-mtp.yaml new file mode 100644 index 000000000..4b257ffed --- /dev/null +++ b/recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-4p1d-dep4-dep8-mtp.yaml @@ -0,0 +1,143 @@ +name: "dsv4-pro-b300-disagg-8k1k-4p1d-dep4-dep8-mtp" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + +dynamo: + hash: "41882ae9b07232eed4850fb1daf8c958abb2556a" + install: true + +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4" + precision: "fp4" + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "b300" + gpus_per_node: 8 + prefill_nodes: 2 + prefill_workers: 4 + gpus_per_prefill: 4 + decode_nodes: 1 + decode_workers: 1 + gpus_per_decode: 8 + +backend: + type: sglang + + prefill_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + + decode_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "4096" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" + + sglang_config: + prefill: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + + disaggregation-mode: "prefill" + watchdog-timeout: 86400 + disaggregation-transfer-backend: nixl + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + + enable-dp-attention: true + enable-dp-lm-head: true + moe-a2a-backend: "megamoe" + + mem-fraction-static: 0.9 + max-running-requests: 1024 + cuda-graph-max-bs: 1024 + chunked-prefill-size: 32768 + stream-interval: 60 + + decode: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + + disaggregation-mode: "decode" + watchdog-timeout: 86400 + disaggregation-transfer-backend: nixl + + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + + enable-dp-attention: true + enable-dp-lm-head: true + moe-a2a-backend: "megamoe" + + speculative-algo: "EAGLE" + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + mem-fraction-static: 0.9 + max-running-requests: 3072 + cuda-graph-max-bs: 1024 + swa-full-tokens-ratio: 0.15 + context-length: 16384 + stream-interval: 60 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + random_range_ratio: 0.8 + concurrencies: "1024" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + revision: "0366e4e064385807ea86b088a5c6c878ff23343b" + container: + image: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4@sha256:86ea65a66c45e6597292986068523ca7738e2e949c31bfd93330db9870e1a930" + frameworks: + dynamo: "41882ae9b07232eed4850fb1daf8c958abb2556a" + sglang: "9462c303a5c5349488e84b2c346e9b5a066de21c" diff --git a/recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-4p1d-dep4-dep8.yaml b/recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-4p1d-dep4-dep8.yaml new file mode 100644 index 000000000..af6e22e37 --- /dev/null +++ b/recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-4p1d-dep4-dep8.yaml @@ -0,0 +1,153 @@ +name: "dsv4-pro-b300-disagg-8k1k-4p1d-dep4-dep8" + +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4" + precision: "fp4" + +dynamo: + hash: "41882ae9b07232eed4850fb1daf8c958abb2556a" + install: true + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "b300" + gpus_per_node: 8 + prefill_nodes: 2 + prefill_workers: 4 + gpus_per_prefill: 4 + decode_nodes: 1 + decode_workers: 1 + gpus_per_decode: 8 + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + env: + args: + router-mode: "kv" + router-kv-overlap-score-weight: 0 + router-queue-threshold: 64 + router-temperature: 0.5 + no-kv-events: true + +backend: + type: sglang + + prefill_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_LOG_FORWARD_ITERS: "1" + SGLANG_LOG_MS: "1" + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" + + decode_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "1280" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: "8" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_LOG_FORWARD_ITERS: "1" + SGLANG_LOG_MS: "1" + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" + + sglang_config: + prefill: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + trust-remote-code: true + watchdog-timeout: 86400 + skip-tokenizer-init: true + stream-interval: 60 + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + + enable-dp-attention: true + moe-a2a-backend: "megamoe" + moe-dense-tp-size: 1 + + disaggregation-mode: "prefill" + disaggregation-transfer-backend: nixl + enable-dp-lm-head: true + + mem-fraction-static: 0.90 + max-running-requests: 512 + cuda-graph-max-bs: 512 + chunked-prefill-size: 32768 + + decode: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + trust-remote-code: true + watchdog-timeout: 86400 + skip-tokenizer-init: true + stream-interval: 60 + + moe-a2a-backend: "megamoe" + moe-dense-tp-size: 1 + + disaggregation-mode: "decode" + disaggregation-transfer-backend: nixl + disaggregation-decode-polling-interval: 8 + + mem-fraction-static: 0.94 + swa-full-tokens-ratio: 0.20 + context-length: 9216 + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + max-running-requests: 18432 + cuda-graph-max-bs: 1280 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "2048" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + revision: "0366e4e064385807ea86b088a5c6c878ff23343b" + container: + image: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4@sha256:86ea65a66c45e6597292986068523ca7738e2e949c31bfd93330db9870e1a930" + frameworks: + dynamo: "41882ae9b07232eed4850fb1daf8c958abb2556a" + sglang: "9462c303a5c5349488e84b2c346e9b5a066de21c" diff --git a/recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-6p1d-dep4-dep8-mtp.yaml b/recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-6p1d-dep4-dep8-mtp.yaml new file mode 100644 index 000000000..e2148bcb9 --- /dev/null +++ b/recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-6p1d-dep4-dep8-mtp.yaml @@ -0,0 +1,143 @@ +name: "dsv4-pro-b300-disagg-8k1k-6p1d-dep4-dep8-mtp" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + +dynamo: + hash: "41882ae9b07232eed4850fb1daf8c958abb2556a" + install: true + +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4" + precision: "fp4" + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "b300" + gpus_per_node: 8 + prefill_nodes: 3 + prefill_workers: 6 + gpus_per_prefill: 4 + decode_nodes: 1 + decode_workers: 1 + gpus_per_decode: 8 + +backend: + type: sglang + + prefill_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + + decode_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "4096" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" + + sglang_config: + prefill: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + + disaggregation-mode: "prefill" + watchdog-timeout: 86400 + disaggregation-transfer-backend: nixl + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + + enable-dp-attention: true + enable-dp-lm-head: true + moe-a2a-backend: "megamoe" + + mem-fraction-static: 0.9 + max-running-requests: 1024 + cuda-graph-max-bs: 1024 + chunked-prefill-size: 32768 + stream-interval: 60 + + decode: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + + disaggregation-mode: "decode" + watchdog-timeout: 86400 + disaggregation-transfer-backend: nixl + + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + + enable-dp-attention: true + enable-dp-lm-head: true + moe-a2a-backend: "megamoe" + + speculative-algo: "EAGLE" + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 + + mem-fraction-static: 0.9 + max-running-requests: 4096 + cuda-graph-max-bs: 1024 + swa-full-tokens-ratio: 0.15 + context-length: 9216 + stream-interval: 60 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + random_range_ratio: 0.8 + concurrencies: "2048" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + revision: "0366e4e064385807ea86b088a5c6c878ff23343b" + container: + image: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4@sha256:86ea65a66c45e6597292986068523ca7738e2e949c31bfd93330db9870e1a930" + frameworks: + dynamo: "41882ae9b07232eed4850fb1daf8c958abb2556a" + sglang: "9462c303a5c5349488e84b2c346e9b5a066de21c" diff --git a/recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-6p1d-dep4-dep8.yaml b/recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-6p1d-dep4-dep8.yaml new file mode 100644 index 000000000..7fa0c42c7 --- /dev/null +++ b/recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-6p1d-dep4-dep8.yaml @@ -0,0 +1,153 @@ +name: "dsv4-pro-b300-disagg-8k1k-6p1d-dep4-dep8" + +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4" + precision: "fp4" + +dynamo: + hash: "41882ae9b07232eed4850fb1daf8c958abb2556a" + install: true + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "b300" + gpus_per_node: 8 + prefill_nodes: 3 + prefill_workers: 6 + gpus_per_prefill: 4 + decode_nodes: 1 + decode_workers: 1 + gpus_per_decode: 8 + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + env: + args: + router-mode: "kv" + router-kv-overlap-score-weight: 0 + router-queue-threshold: 64 + router-temperature: 0.5 + no-kv-events: true + +backend: + type: sglang + + prefill_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_LOG_FORWARD_ITERS: "1" + SGLANG_LOG_MS: "1" + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" + + decode_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "1280" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: "8" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_LOG_FORWARD_ITERS: "1" + SGLANG_LOG_MS: "1" + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" + + sglang_config: + prefill: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + trust-remote-code: true + watchdog-timeout: 86400 + skip-tokenizer-init: true + stream-interval: 60 + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + + enable-dp-attention: true + moe-a2a-backend: "megamoe" + moe-dense-tp-size: 1 + + disaggregation-mode: "prefill" + disaggregation-transfer-backend: nixl + enable-dp-lm-head: true + + mem-fraction-static: 0.90 + max-running-requests: 512 + cuda-graph-max-bs: 512 + chunked-prefill-size: 32768 + + decode: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + trust-remote-code: true + watchdog-timeout: 86400 + skip-tokenizer-init: true + stream-interval: 60 + + moe-a2a-backend: "megamoe" + moe-dense-tp-size: 1 + + disaggregation-mode: "decode" + disaggregation-transfer-backend: nixl + disaggregation-decode-polling-interval: 8 + + mem-fraction-static: 0.94 + swa-full-tokens-ratio: 0.20 + context-length: 9216 + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + max-running-requests: 18432 + cuda-graph-max-bs: 1280 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "3072" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + revision: "0366e4e064385807ea86b088a5c6c878ff23343b" + container: + image: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4@sha256:86ea65a66c45e6597292986068523ca7738e2e949c31bfd93330db9870e1a930" + frameworks: + dynamo: "41882ae9b07232eed4850fb1daf8c958abb2556a" + sglang: "9462c303a5c5349488e84b2c346e9b5a066de21c" diff --git a/recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-8p1d-dep4-dep8-mtp.yaml b/recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-8p1d-dep4-dep8-mtp.yaml new file mode 100644 index 000000000..4449061c4 --- /dev/null +++ b/recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-8p1d-dep4-dep8-mtp.yaml @@ -0,0 +1,143 @@ +name: "dsv4-pro-b300-disagg-8k1k-8p1d-dep4-dep8-mtp" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + +dynamo: + hash: "41882ae9b07232eed4850fb1daf8c958abb2556a" + install: true + +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4" + precision: "fp4" + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "b300" + gpus_per_node: 8 + prefill_nodes: 4 + prefill_workers: 8 + gpus_per_prefill: 4 + decode_nodes: 1 + decode_workers: 1 + gpus_per_decode: 8 + +backend: + type: sglang + + prefill_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + + decode_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "4096" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" + + sglang_config: + prefill: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + + disaggregation-mode: "prefill" + watchdog-timeout: 86400 + disaggregation-transfer-backend: nixl + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + + enable-dp-attention: true + enable-dp-lm-head: true + moe-a2a-backend: "megamoe" + + mem-fraction-static: 0.9 + max-running-requests: 1024 + cuda-graph-max-bs: 1024 + chunked-prefill-size: 32768 + stream-interval: 60 + + decode: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + + disaggregation-mode: "decode" + watchdog-timeout: 86400 + disaggregation-transfer-backend: nixl + + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + + enable-dp-attention: true + enable-dp-lm-head: true + moe-a2a-backend: "megamoe" + + speculative-algo: "EAGLE" + speculative-num-steps: 1 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 2 + + mem-fraction-static: 0.85 + max-running-requests: 8192 + cuda-graph-max-bs: 1280 + swa-full-tokens-ratio: 0.1 + context-length: 9216 + stream-interval: 60 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + random_range_ratio: 0.8 + concurrencies: "3072" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + revision: "0366e4e064385807ea86b088a5c6c878ff23343b" + container: + image: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4@sha256:86ea65a66c45e6597292986068523ca7738e2e949c31bfd93330db9870e1a930" + frameworks: + dynamo: "41882ae9b07232eed4850fb1daf8c958abb2556a" + sglang: "9462c303a5c5349488e84b2c346e9b5a066de21c" From 2065f6e7e798b48c0dc8d20a33c1f75427659dec Mon Sep 17 00:00:00 2001 From: Rohit Pujar Nagraj Date: Mon, 27 Jul 2026 14:59:30 -0700 Subject: [PATCH 2/4] refactor(recipes): use compact B300 DSV4 matrices Signed-off-by: Rohit Pujar Nagraj --- .../dsv4-pro/sglang/b300-fp4/8k1k-mtp.yaml | 233 ++++++++++++++++++ recipes/dsv4-pro/sglang/b300-fp4/8k1k.yaml | 215 ++++++++++++++++ .../disagg-b300-1p1d-dep4-dep8-mtp.yaml | 142 ----------- .../disagg/disagg-b300-1p1d-dep4-dep8.yaml | 153 ------------ .../disagg/disagg-b300-1p1d-tp4-tp4-mtp.yaml | 133 ---------- .../8k1k/disagg/disagg-b300-1p1d-tp4-tp4.yaml | 128 ---------- .../disagg-b300-1p2d-dep4-dep8-mtp.yaml | 143 ----------- .../disagg/disagg-b300-1p6d-dep4-tp4-mtp.yaml | 137 ---------- .../disagg-b300-2p1d-dep4-dep8-mtp.yaml | 143 ----------- .../disagg/disagg-b300-2p1d-dep4-dep8.yaml | 153 ------------ .../disagg-b300-4p1d-dep4-dep8-mtp.yaml | 143 ----------- .../disagg/disagg-b300-4p1d-dep4-dep8.yaml | 153 ------------ .../disagg-b300-6p1d-dep4-dep8-mtp.yaml | 143 ----------- .../disagg/disagg-b300-6p1d-dep4-dep8.yaml | 153 ------------ .../disagg-b300-8p1d-dep4-dep8-mtp.yaml | 143 ----------- 15 files changed, 448 insertions(+), 1867 deletions(-) create mode 100644 recipes/dsv4-pro/sglang/b300-fp4/8k1k-mtp.yaml create mode 100644 recipes/dsv4-pro/sglang/b300-fp4/8k1k.yaml delete mode 100644 recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-1p1d-dep4-dep8-mtp.yaml delete mode 100644 recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-1p1d-dep4-dep8.yaml delete mode 100644 recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-1p1d-tp4-tp4-mtp.yaml delete mode 100644 recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-1p1d-tp4-tp4.yaml delete mode 100644 recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-1p2d-dep4-dep8-mtp.yaml delete mode 100644 recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-1p6d-dep4-tp4-mtp.yaml delete mode 100644 recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-2p1d-dep4-dep8-mtp.yaml delete mode 100644 recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-2p1d-dep4-dep8.yaml delete mode 100644 recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-4p1d-dep4-dep8-mtp.yaml delete mode 100644 recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-4p1d-dep4-dep8.yaml delete mode 100644 recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-6p1d-dep4-dep8-mtp.yaml delete mode 100644 recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-6p1d-dep4-dep8.yaml delete mode 100644 recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-8p1d-dep4-dep8-mtp.yaml diff --git a/recipes/dsv4-pro/sglang/b300-fp4/8k1k-mtp.yaml b/recipes/dsv4-pro/sglang/b300-fp4/8k1k-mtp.yaml new file mode 100644 index 000000000..b6f55da7e --- /dev/null +++ b/recipes/dsv4-pro/sglang/b300-fp4/8k1k-mtp.yaml @@ -0,0 +1,233 @@ +base: + name: dsv4-pro-b300-disagg-8k1k-1p1d-dep4-dep8-mtp + model: + path: deepseek-v4-pro + container: lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4 + precision: fp4 + dynamo: + hash: 41882ae9b07232eed4850fb1daf8c958abb2556a + install: true + sbatch_directives: + cpus-per-task: '144' + mem: '0' + resources: + gpu_type: b300 + gpus_per_node: 8 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 4 + decode_nodes: 1 + decode_workers: 1 + gpus_per_decode: 8 + frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + backend: + type: sglang + prefill_environment: + PYTHONUNBUFFERED: '1' + SGLANG_RADIX_FORCE_MISS: '1' + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' + SGLANG_DEFAULT_THINKING: '1' + SGLANG_DSV4_REASONING_EFFORT: max + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '9216' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + MC_FORCE_MNNVL: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' + decode_environment: + PYTHONUNBUFFERED: '1' + SGLANG_RADIX_FORCE_MISS: '1' + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' + SGLANG_DEFAULT_THINKING: '1' + SGLANG_DSV4_REASONING_EFFORT: max + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '2048' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + MC_FORCE_MNNVL: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: '0' + sglang_config: + prefill: + served-model-name: deepseek-ai/DeepSeek-V4-Pro + model-path: /model/ + trust-remote-code: true + tool-call-parser: deepseekv4 + disaggregation-mode: prefill + watchdog-timeout: 86400 + disaggregation-transfer-backend: nixl + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + moe-a2a-backend: megamoe + mem-fraction-static: 0.9 + max-running-requests: 256 + cuda-graph-max-bs: 256 + chunked-prefill-size: 32768 + stream-interval: 60 + decode: + served-model-name: deepseek-ai/DeepSeek-V4-Pro + model-path: /model/ + trust-remote-code: true + tool-call-parser: deepseekv4 + disaggregation-mode: decode + watchdog-timeout: 86400 + disaggregation-transfer-backend: nixl + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + moe-a2a-backend: megamoe + speculative-algo: EAGLE + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + mem-fraction-static: 0.94 + max-running-requests: 3072 + cuda-graph-max-bs: 256 + swa-full-tokens-ratio: 0.15 + context-length: 16384 + stream-interval: 60 + benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + random_range_ratio: 0.8 + concurrencies: '256' + req_rate: inf + use_chat_template: true + custom_tokenizer: sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer + identity: + model: + repo: deepseek-ai/DeepSeek-V4-Pro + revision: 0366e4e064385807ea86b088a5c6c878ff23343b + container: + image: lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4@sha256:86ea65a66c45e6597292986068523ca7738e2e949c31bfd93330db9870e1a930 + frameworks: + dynamo: 41882ae9b07232eed4850fb1daf8c958abb2556a + sglang: 9462c303a5c5349488e84b2c346e9b5a066de21c + +zip_override_dep4_dep8: + name: + - dsv4-pro-b300-disagg-8k1k-1p1d-dep4-dep8-mtp + - dsv4-pro-b300-disagg-8k1k-1p2d-dep4-dep8-mtp + - dsv4-pro-b300-disagg-8k1k-2p1d-dep4-dep8-mtp + - dsv4-pro-b300-disagg-8k1k-4p1d-dep4-dep8-mtp + - dsv4-pro-b300-disagg-8k1k-6p1d-dep4-dep8-mtp + - dsv4-pro-b300-disagg-8k1k-8p1d-dep4-dep8-mtp + resources: + prefill_nodes: [1, 1, 1, 2, 3, 4] + prefill_workers: [1, 1, 2, 4, 6, 8] + decode_nodes: [1, 2, 1, 1, 1, 1] + decode_workers: [1, 2, 1, 1, 1, 1] + backend: + decode_environment: + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: ['2048', '2048', '4096', '4096', '4096', '4096'] + sglang_config: + prefill: + max-running-requests: [256, 256, 512, 1024, 1024, 1024] + cuda-graph-max-bs: [256, 256, 512, 1024, 1024, 1024] + decode: + speculative-num-steps: [3, 3, 3, 3, 2, 1] + speculative-num-draft-tokens: [4, 4, 4, 4, 3, 2] + mem-fraction-static: [0.94, 0.94, 0.9, 0.9, 0.9, 0.85] + max-running-requests: [3072, 3072, 3072, 3072, 4096, 8192] + cuda-graph-max-bs: [256, 256, 512, 1024, 1024, 1280] + swa-full-tokens-ratio: [0.15, 0.15, 0.15, 0.15, 0.15, 0.1] + context-length: [16384, 16384, 16384, 16384, 9216, 9216] + benchmark: + concurrencies: ['256', '256', '512', '1024', '2048', '3072'] + +override_tp4_tp4: + name: dsv4-pro-b300-disagg-8k1k-1p1d-tp4-tp4-mtp + resources: + gpus_per_decode: 4 + backend: + prefill_environment: + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: null + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: null + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: null + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: null + decode_environment: + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: null + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: null + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: null + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: null + sglang_config: + prefill: + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: null + enable-dp-lm-head: null + moe-a2a-backend: null + moe-runner-backend: flashinfer_mxfp4 + disable-flashinfer-autotune: true + max-running-requests: 8 + cuda-graph-max-bs: 8 + stream-interval: null + decode: + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: null + enable-dp-lm-head: null + moe-a2a-backend: null + moe-runner-backend: flashinfer_mxfp4 + disable-flashinfer-autotune: true + mem-fraction-static: 0.9 + max-running-requests: 8 + cuda-graph-max-bs: 8 + swa-full-tokens-ratio: 0.1 + stream-interval: null + benchmark: + concurrencies: '1' + +override_dep4_tp4: + name: dsv4-pro-b300-disagg-8k1k-1p6d-dep4-tp4-mtp + resources: + decode_nodes: 3 + decode_workers: 6 + gpus_per_decode: 4 + backend: + decode_environment: + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: null + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: null + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: null + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: null + sglang_config: + prefill: + max-running-requests: 128 + cuda-graph-max-bs: 128 + stream-interval: null + decode: + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: null + enable-dp-lm-head: null + moe-a2a-backend: null + moe-runner-backend: flashinfer_mxfp4 + disable-flashinfer-autotune: true + mem-fraction-static: 0.9 + max-running-requests: 128 + cuda-graph-max-bs: 128 + swa-full-tokens-ratio: 0.1 + stream-interval: null + benchmark: + concurrencies: 8x32x64 diff --git a/recipes/dsv4-pro/sglang/b300-fp4/8k1k.yaml b/recipes/dsv4-pro/sglang/b300-fp4/8k1k.yaml new file mode 100644 index 000000000..344850704 --- /dev/null +++ b/recipes/dsv4-pro/sglang/b300-fp4/8k1k.yaml @@ -0,0 +1,215 @@ +base: + name: dsv4-pro-b300-disagg-8k1k-1p1d-dep4-dep8 + model: + path: deepseek-v4-pro + container: lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4 + precision: fp4 + dynamo: + hash: 41882ae9b07232eed4850fb1daf8c958abb2556a + install: true + sbatch_directives: + cpus-per-task: '144' + mem: '0' + resources: + gpu_type: b300 + gpus_per_node: 8 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 4 + decode_nodes: 1 + decode_workers: 1 + gpus_per_decode: 8 + frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + env: null + args: + router-mode: kv + router-kv-overlap-score-weight: 0 + router-queue-threshold: 64 + router-temperature: 0.5 + no-kv-events: true + backend: + type: sglang + prefill_environment: + PYTHONUNBUFFERED: '1' + SGLANG_RADIX_FORCE_MISS: '1' + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' + SGLANG_DEFAULT_THINKING: '1' + SGLANG_DSV4_REASONING_EFFORT: max + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '8192' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: '1' + SGLANG_OPT_USE_ONLINE_COMPRESS: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + MC_FORCE_MNNVL: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_LOG_FORWARD_ITERS: '1' + SGLANG_LOG_MS: '1' + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' + decode_environment: + PYTHONUNBUFFERED: '1' + SGLANG_RADIX_FORCE_MISS: '1' + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' + SGLANG_DEFAULT_THINKING: '1' + SGLANG_DSV4_REASONING_EFFORT: max + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '1280' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: '1' + SGLANG_OPT_USE_ONLINE_COMPRESS: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' + MC_FORCE_MNNVL: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_LOG_FORWARD_ITERS: '1' + SGLANG_LOG_MS: '1' + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' + sglang_config: + prefill: + served-model-name: deepseek-ai/DeepSeek-V4-Pro + trust-remote-code: true + watchdog-timeout: 86400 + skip-tokenizer-init: true + stream-interval: 60 + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + moe-a2a-backend: megamoe + moe-dense-tp-size: 1 + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + enable-dp-lm-head: true + mem-fraction-static: 0.9 + max-running-requests: 512 + cuda-graph-max-bs: 512 + chunked-prefill-size: 32768 + decode: + served-model-name: deepseek-ai/DeepSeek-V4-Pro + trust-remote-code: true + watchdog-timeout: 86400 + skip-tokenizer-init: true + stream-interval: 60 + moe-a2a-backend: megamoe + moe-dense-tp-size: 1 + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + disaggregation-decode-polling-interval: 8 + mem-fraction-static: 0.94 + swa-full-tokens-ratio: 0.2 + context-length: 9216 + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + max-running-requests: 18432 + cuda-graph-max-bs: 1280 + benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: '256' + req_rate: inf + use_chat_template: true + custom_tokenizer: sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer + identity: + model: + repo: deepseek-ai/DeepSeek-V4-Pro + revision: 0366e4e064385807ea86b088a5c6c878ff23343b + container: + image: lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4@sha256:86ea65a66c45e6597292986068523ca7738e2e949c31bfd93330db9870e1a930 + frameworks: + dynamo: 41882ae9b07232eed4850fb1daf8c958abb2556a + sglang: 9462c303a5c5349488e84b2c346e9b5a066de21c + +zip_override_dep4_dep8: + name: + - dsv4-pro-b300-disagg-8k1k-1p1d-dep4-dep8 + - dsv4-pro-b300-disagg-8k1k-2p1d-dep4-dep8 + - dsv4-pro-b300-disagg-8k1k-4p1d-dep4-dep8 + - dsv4-pro-b300-disagg-8k1k-6p1d-dep4-dep8 + resources: + prefill_nodes: [1, 1, 2, 3] + prefill_workers: [1, 2, 4, 6] + benchmark: + concurrencies: ['256', '768', '2048', '3072'] + +override_tp4_tp4: + name: dsv4-pro-b300-disagg-8k1k-1p1d-tp4-tp4 + resources: + gpus_per_decode: 4 + frontend: + env: null + args: null + backend: + prefill_environment: + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: null + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: null + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: null + SGLANG_OPT_USE_ONLINE_COMPRESS: null + DYN_SKIP_SGLANG_LOG_FORMATTING: null + SGLANG_LOG_FORWARD_ITERS: null + SGLANG_LOG_MS: null + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: null + decode_environment: + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: null + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: null + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: null + SGLANG_OPT_USE_ONLINE_COMPRESS: null + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: null + DYN_SKIP_SGLANG_LOG_FORMATTING: null + SGLANG_LOG_FORWARD_ITERS: null + SGLANG_LOG_MS: null + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: null + sglang_config: + prefill: + model-path: /model/ + skip-tokenizer-init: null + stream-interval: null + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: null + enable-dp-lm-head: null + moe-a2a-backend: null + moe-dense-tp-size: null + moe-runner-backend: flashinfer_mxfp4 + disable-flashinfer-autotune: true + disable-radix-cache: true + max-running-requests: 32 + cuda-graph-max-bs: 32 + decode: + model-path: /model/ + skip-tokenizer-init: null + stream-interval: null + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: null + enable-dp-lm-head: null + moe-a2a-backend: null + moe-dense-tp-size: null + moe-runner-backend: flashinfer_mxfp4 + disable-flashinfer-autotune: true + disable-radix-cache: true + disaggregation-decode-polling-interval: null + mem-fraction-static: 0.9 + swa-full-tokens-ratio: 0.1 + context-length: 16384 + max-running-requests: 64 + cuda-graph-max-bs: 64 + benchmark: + random_range_ratio: 0.8 + concurrencies: 1x8x16x32 diff --git a/recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-1p1d-dep4-dep8-mtp.yaml b/recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-1p1d-dep4-dep8-mtp.yaml deleted file mode 100644 index a2a058fab..000000000 --- a/recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-1p1d-dep4-dep8-mtp.yaml +++ /dev/null @@ -1,142 +0,0 @@ -name: "dsv4-pro-b300-disagg-8k1k-1p1d-dep4-dep8-mtp" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 8 - -dynamo: - hash: "41882ae9b07232eed4850fb1daf8c958abb2556a" - install: true - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4" - precision: "fp4" - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_nodes: 1 - decode_workers: 1 - gpus_per_decode: 8 - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "2048" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 - - disaggregation-mode: "prefill" - watchdog-timeout: 86400 - disaggregation-transfer-backend: nixl - - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - - enable-dp-attention: true - enable-dp-lm-head: true - moe-a2a-backend: "megamoe" - mem-fraction-static: 0.9 - max-running-requests: 256 - cuda-graph-max-bs: 256 - chunked-prefill-size: 32768 - stream-interval: 60 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 - - disaggregation-mode: "decode" - watchdog-timeout: 86400 - disaggregation-transfer-backend: nixl - - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - - enable-dp-attention: true - enable-dp-lm-head: true - moe-a2a-backend: "megamoe" - - speculative-algo: "EAGLE" - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - - mem-fraction-static: 0.94 - max-running-requests: 3072 - cuda-graph-max-bs: 256 - swa-full-tokens-ratio: 0.15 - context-length: 16384 - stream-interval: 60 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - random_range_ratio: 0.8 - concurrencies: "256" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - revision: "0366e4e064385807ea86b088a5c6c878ff23343b" - container: - image: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4@sha256:86ea65a66c45e6597292986068523ca7738e2e949c31bfd93330db9870e1a930" - frameworks: - dynamo: "41882ae9b07232eed4850fb1daf8c958abb2556a" - sglang: "9462c303a5c5349488e84b2c346e9b5a066de21c" diff --git a/recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-1p1d-dep4-dep8.yaml b/recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-1p1d-dep4-dep8.yaml deleted file mode 100644 index 5a1dcdf2c..000000000 --- a/recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-1p1d-dep4-dep8.yaml +++ /dev/null @@ -1,153 +0,0 @@ -name: "dsv4-pro-b300-disagg-8k1k-1p1d-dep4-dep8" - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4" - precision: "fp4" - -dynamo: - hash: "41882ae9b07232eed4850fb1daf8c958abb2556a" - install: true - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_nodes: 1 - decode_workers: 1 - gpus_per_decode: 8 - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 8 - env: - args: - router-mode: "kv" - router-kv-overlap-score-weight: 0 - router-queue-threshold: 64 - router-temperature: 0.5 - no-kv-events: true - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "1280" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: "8" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - - enable-dp-attention: true - moe-a2a-backend: "megamoe" - moe-dense-tp-size: 1 - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: nixl - enable-dp-lm-head: true - - mem-fraction-static: 0.90 - max-running-requests: 512 - cuda-graph-max-bs: 512 - chunked-prefill-size: 32768 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - moe-a2a-backend: "megamoe" - moe-dense-tp-size: 1 - - disaggregation-mode: "decode" - disaggregation-transfer-backend: nixl - disaggregation-decode-polling-interval: 8 - - mem-fraction-static: 0.94 - swa-full-tokens-ratio: 0.20 - context-length: 9216 - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - enable-dp-attention: true - enable-dp-lm-head: true - max-running-requests: 18432 - cuda-graph-max-bs: 1280 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "256" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - revision: "0366e4e064385807ea86b088a5c6c878ff23343b" - container: - image: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4@sha256:86ea65a66c45e6597292986068523ca7738e2e949c31bfd93330db9870e1a930" - frameworks: - dynamo: "41882ae9b07232eed4850fb1daf8c958abb2556a" - sglang: "9462c303a5c5349488e84b2c346e9b5a066de21c" diff --git a/recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-1p1d-tp4-tp4-mtp.yaml b/recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-1p1d-tp4-tp4-mtp.yaml deleted file mode 100644 index 60f87a01b..000000000 --- a/recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-1p1d-tp4-tp4-mtp.yaml +++ /dev/null @@ -1,133 +0,0 @@ -name: "dsv4-pro-b300-disagg-8k1k-1p1d-tp4-tp4-mtp" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 8 - -dynamo: - hash: "41882ae9b07232eed4850fb1daf8c958abb2556a" - install: true - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4" - precision: "fp4" - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_nodes: 1 - decode_workers: 1 - gpus_per_decode: 4 - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 - - disaggregation-mode: "prefill" - watchdog-timeout: 86400 - disaggregation-transfer-backend: nixl - - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - - moe-runner-backend: "flashinfer_mxfp4" - disable-flashinfer-autotune: true - - mem-fraction-static: 0.9 - max-running-requests: 8 - cuda-graph-max-bs: 8 - chunked-prefill-size: 32768 - - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 - - disaggregation-mode: "decode" - watchdog-timeout: 86400 - disaggregation-transfer-backend: nixl - - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - - moe-runner-backend: "flashinfer_mxfp4" - disable-flashinfer-autotune: true - - speculative-algo: "EAGLE" - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - - mem-fraction-static: 0.9 - max-running-requests: 8 - cuda-graph-max-bs: 8 - swa-full-tokens-ratio: 0.1 - context-length: 16384 - - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - random_range_ratio: 0.8 - concurrencies: "1" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - revision: "0366e4e064385807ea86b088a5c6c878ff23343b" - container: - image: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4@sha256:86ea65a66c45e6597292986068523ca7738e2e949c31bfd93330db9870e1a930" - frameworks: - dynamo: "41882ae9b07232eed4850fb1daf8c958abb2556a" - sglang: "9462c303a5c5349488e84b2c346e9b5a066de21c" diff --git a/recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-1p1d-tp4-tp4.yaml b/recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-1p1d-tp4-tp4.yaml deleted file mode 100644 index c9b876751..000000000 --- a/recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-1p1d-tp4-tp4.yaml +++ /dev/null @@ -1,128 +0,0 @@ -name: "dsv4-pro-b300-disagg-8k1k-1p1d-tp4-tp4" - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4" - precision: "fp4" - -dynamo: - hash: "41882ae9b07232eed4850fb1daf8c958abb2556a" - install: true - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_nodes: 1 - decode_workers: 1 - gpus_per_decode: 4 - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 8 - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - disable-radix-cache: true - - disaggregation-mode: "prefill" - watchdog-timeout: 86400 - disaggregation-transfer-backend: nixl - - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - - moe-runner-backend: "flashinfer_mxfp4" - disable-flashinfer-autotune: true - - mem-fraction-static: 0.90 - max-running-requests: 32 - cuda-graph-max-bs: 32 - chunked-prefill-size: 32768 - - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - disable-radix-cache: true - - disaggregation-mode: "decode" - watchdog-timeout: 86400 - disaggregation-transfer-backend: nixl - - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - - moe-runner-backend: "flashinfer_mxfp4" - disable-flashinfer-autotune: true - - mem-fraction-static: 0.9 - max-running-requests: 64 - cuda-graph-max-bs: 64 - swa-full-tokens-ratio: 0.1 - context-length: 16384 - - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - random_range_ratio: 0.8 - concurrencies: "1x8x16x32" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - revision: "0366e4e064385807ea86b088a5c6c878ff23343b" - container: - image: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4@sha256:86ea65a66c45e6597292986068523ca7738e2e949c31bfd93330db9870e1a930" - frameworks: - dynamo: "41882ae9b07232eed4850fb1daf8c958abb2556a" - sglang: "9462c303a5c5349488e84b2c346e9b5a066de21c" diff --git a/recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-1p2d-dep4-dep8-mtp.yaml b/recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-1p2d-dep4-dep8-mtp.yaml deleted file mode 100644 index 278a2390f..000000000 --- a/recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-1p2d-dep4-dep8-mtp.yaml +++ /dev/null @@ -1,143 +0,0 @@ -name: "dsv4-pro-b300-disagg-8k1k-1p2d-dep4-dep8-mtp" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 8 - -dynamo: - hash: "41882ae9b07232eed4850fb1daf8c958abb2556a" - install: true - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4" - precision: "fp4" - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_nodes: 2 - decode_workers: 2 - gpus_per_decode: 8 - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "2048" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 - - disaggregation-mode: "prefill" - watchdog-timeout: 86400 - disaggregation-transfer-backend: nixl - - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - - enable-dp-attention: true - enable-dp-lm-head: true - moe-a2a-backend: "megamoe" - - mem-fraction-static: 0.9 - max-running-requests: 256 - cuda-graph-max-bs: 256 - chunked-prefill-size: 32768 - stream-interval: 60 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 - - disaggregation-mode: "decode" - watchdog-timeout: 86400 - disaggregation-transfer-backend: nixl - - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - - enable-dp-attention: true - enable-dp-lm-head: true - moe-a2a-backend: "megamoe" - - speculative-algo: "EAGLE" - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - - mem-fraction-static: 0.94 - max-running-requests: 3072 - cuda-graph-max-bs: 256 - swa-full-tokens-ratio: 0.15 - context-length: 16384 - stream-interval: 60 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - random_range_ratio: 0.8 - concurrencies: "256" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - revision: "0366e4e064385807ea86b088a5c6c878ff23343b" - container: - image: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4@sha256:86ea65a66c45e6597292986068523ca7738e2e949c31bfd93330db9870e1a930" - frameworks: - dynamo: "41882ae9b07232eed4850fb1daf8c958abb2556a" - sglang: "9462c303a5c5349488e84b2c346e9b5a066de21c" diff --git a/recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-1p6d-dep4-tp4-mtp.yaml b/recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-1p6d-dep4-tp4-mtp.yaml deleted file mode 100644 index b3f0a4c11..000000000 --- a/recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-1p6d-dep4-tp4-mtp.yaml +++ /dev/null @@ -1,137 +0,0 @@ -name: "dsv4-pro-b300-disagg-8k1k-1p6d-dep4-tp4-mtp" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 8 - -dynamo: - hash: "41882ae9b07232eed4850fb1daf8c958abb2556a" - install: true - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4" - precision: "fp4" - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_nodes: 3 - decode_workers: 6 - gpus_per_decode: 4 - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 - - disaggregation-mode: "prefill" - watchdog-timeout: 86400 - disaggregation-transfer-backend: nixl - - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - - enable-dp-attention: true - enable-dp-lm-head: true - moe-a2a-backend: "megamoe" - - mem-fraction-static: 0.9 - max-running-requests: 128 - cuda-graph-max-bs: 128 - chunked-prefill-size: 32768 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 - - disaggregation-mode: "decode" - watchdog-timeout: 86400 - disaggregation-transfer-backend: nixl - - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - - moe-runner-backend: "flashinfer_mxfp4" - disable-flashinfer-autotune: true - - speculative-algo: "EAGLE" - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - - mem-fraction-static: 0.9 - max-running-requests: 128 - cuda-graph-max-bs: 128 - swa-full-tokens-ratio: 0.1 - context-length: 16384 - - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - random_range_ratio: 0.8 - concurrencies: "8x32x64" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - revision: "0366e4e064385807ea86b088a5c6c878ff23343b" - container: - image: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4@sha256:86ea65a66c45e6597292986068523ca7738e2e949c31bfd93330db9870e1a930" - frameworks: - dynamo: "41882ae9b07232eed4850fb1daf8c958abb2556a" - sglang: "9462c303a5c5349488e84b2c346e9b5a066de21c" diff --git a/recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-2p1d-dep4-dep8-mtp.yaml b/recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-2p1d-dep4-dep8-mtp.yaml deleted file mode 100644 index 2804bb6fe..000000000 --- a/recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-2p1d-dep4-dep8-mtp.yaml +++ /dev/null @@ -1,143 +0,0 @@ -name: "dsv4-pro-b300-disagg-8k1k-2p1d-dep4-dep8-mtp" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 8 - -dynamo: - hash: "41882ae9b07232eed4850fb1daf8c958abb2556a" - install: true - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4" - precision: "fp4" - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 1 - prefill_workers: 2 - gpus_per_prefill: 4 - decode_nodes: 1 - decode_workers: 1 - gpus_per_decode: 8 - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "4096" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 - - disaggregation-mode: "prefill" - watchdog-timeout: 86400 - disaggregation-transfer-backend: nixl - - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - - enable-dp-attention: true - enable-dp-lm-head: true - moe-a2a-backend: "megamoe" - - mem-fraction-static: 0.9 - max-running-requests: 512 - cuda-graph-max-bs: 512 - chunked-prefill-size: 32768 - stream-interval: 60 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 - - disaggregation-mode: "decode" - watchdog-timeout: 86400 - disaggregation-transfer-backend: nixl - - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - - enable-dp-attention: true - enable-dp-lm-head: true - moe-a2a-backend: "megamoe" - - speculative-algo: "EAGLE" - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - - mem-fraction-static: 0.9 - max-running-requests: 3072 - cuda-graph-max-bs: 512 - swa-full-tokens-ratio: 0.15 - context-length: 16384 - stream-interval: 60 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - random_range_ratio: 0.8 - concurrencies: "512" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - revision: "0366e4e064385807ea86b088a5c6c878ff23343b" - container: - image: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4@sha256:86ea65a66c45e6597292986068523ca7738e2e949c31bfd93330db9870e1a930" - frameworks: - dynamo: "41882ae9b07232eed4850fb1daf8c958abb2556a" - sglang: "9462c303a5c5349488e84b2c346e9b5a066de21c" diff --git a/recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-2p1d-dep4-dep8.yaml b/recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-2p1d-dep4-dep8.yaml deleted file mode 100644 index a373aae5d..000000000 --- a/recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-2p1d-dep4-dep8.yaml +++ /dev/null @@ -1,153 +0,0 @@ -name: "dsv4-pro-b300-disagg-8k1k-2p1d-dep4-dep8" - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4" - precision: "fp4" - -dynamo: - hash: "41882ae9b07232eed4850fb1daf8c958abb2556a" - install: true - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 1 - prefill_workers: 2 - gpus_per_prefill: 4 - decode_nodes: 1 - decode_workers: 1 - gpus_per_decode: 8 - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 8 - env: - args: - router-mode: "kv" - router-kv-overlap-score-weight: 0 - router-queue-threshold: 64 - router-temperature: 0.5 - no-kv-events: true - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "1280" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: "8" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - - enable-dp-attention: true - moe-a2a-backend: "megamoe" - moe-dense-tp-size: 1 - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: nixl - enable-dp-lm-head: true - - mem-fraction-static: 0.90 - max-running-requests: 512 - cuda-graph-max-bs: 512 - chunked-prefill-size: 32768 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - moe-a2a-backend: "megamoe" - moe-dense-tp-size: 1 - - disaggregation-mode: "decode" - disaggregation-transfer-backend: nixl - disaggregation-decode-polling-interval: 8 - - mem-fraction-static: 0.94 - swa-full-tokens-ratio: 0.20 - context-length: 9216 - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - enable-dp-attention: true - enable-dp-lm-head: true - max-running-requests: 18432 - cuda-graph-max-bs: 1280 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "768" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - revision: "0366e4e064385807ea86b088a5c6c878ff23343b" - container: - image: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4@sha256:86ea65a66c45e6597292986068523ca7738e2e949c31bfd93330db9870e1a930" - frameworks: - dynamo: "41882ae9b07232eed4850fb1daf8c958abb2556a" - sglang: "9462c303a5c5349488e84b2c346e9b5a066de21c" diff --git a/recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-4p1d-dep4-dep8-mtp.yaml b/recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-4p1d-dep4-dep8-mtp.yaml deleted file mode 100644 index 4b257ffed..000000000 --- a/recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-4p1d-dep4-dep8-mtp.yaml +++ /dev/null @@ -1,143 +0,0 @@ -name: "dsv4-pro-b300-disagg-8k1k-4p1d-dep4-dep8-mtp" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 8 - -dynamo: - hash: "41882ae9b07232eed4850fb1daf8c958abb2556a" - install: true - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4" - precision: "fp4" - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 2 - prefill_workers: 4 - gpus_per_prefill: 4 - decode_nodes: 1 - decode_workers: 1 - gpus_per_decode: 8 - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "4096" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 - - disaggregation-mode: "prefill" - watchdog-timeout: 86400 - disaggregation-transfer-backend: nixl - - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - - enable-dp-attention: true - enable-dp-lm-head: true - moe-a2a-backend: "megamoe" - - mem-fraction-static: 0.9 - max-running-requests: 1024 - cuda-graph-max-bs: 1024 - chunked-prefill-size: 32768 - stream-interval: 60 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 - - disaggregation-mode: "decode" - watchdog-timeout: 86400 - disaggregation-transfer-backend: nixl - - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - - enable-dp-attention: true - enable-dp-lm-head: true - moe-a2a-backend: "megamoe" - - speculative-algo: "EAGLE" - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - - mem-fraction-static: 0.9 - max-running-requests: 3072 - cuda-graph-max-bs: 1024 - swa-full-tokens-ratio: 0.15 - context-length: 16384 - stream-interval: 60 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - random_range_ratio: 0.8 - concurrencies: "1024" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - revision: "0366e4e064385807ea86b088a5c6c878ff23343b" - container: - image: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4@sha256:86ea65a66c45e6597292986068523ca7738e2e949c31bfd93330db9870e1a930" - frameworks: - dynamo: "41882ae9b07232eed4850fb1daf8c958abb2556a" - sglang: "9462c303a5c5349488e84b2c346e9b5a066de21c" diff --git a/recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-4p1d-dep4-dep8.yaml b/recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-4p1d-dep4-dep8.yaml deleted file mode 100644 index af6e22e37..000000000 --- a/recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-4p1d-dep4-dep8.yaml +++ /dev/null @@ -1,153 +0,0 @@ -name: "dsv4-pro-b300-disagg-8k1k-4p1d-dep4-dep8" - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4" - precision: "fp4" - -dynamo: - hash: "41882ae9b07232eed4850fb1daf8c958abb2556a" - install: true - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 2 - prefill_workers: 4 - gpus_per_prefill: 4 - decode_nodes: 1 - decode_workers: 1 - gpus_per_decode: 8 - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 8 - env: - args: - router-mode: "kv" - router-kv-overlap-score-weight: 0 - router-queue-threshold: 64 - router-temperature: 0.5 - no-kv-events: true - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "1280" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: "8" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - - enable-dp-attention: true - moe-a2a-backend: "megamoe" - moe-dense-tp-size: 1 - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: nixl - enable-dp-lm-head: true - - mem-fraction-static: 0.90 - max-running-requests: 512 - cuda-graph-max-bs: 512 - chunked-prefill-size: 32768 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - moe-a2a-backend: "megamoe" - moe-dense-tp-size: 1 - - disaggregation-mode: "decode" - disaggregation-transfer-backend: nixl - disaggregation-decode-polling-interval: 8 - - mem-fraction-static: 0.94 - swa-full-tokens-ratio: 0.20 - context-length: 9216 - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - enable-dp-attention: true - enable-dp-lm-head: true - max-running-requests: 18432 - cuda-graph-max-bs: 1280 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "2048" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - revision: "0366e4e064385807ea86b088a5c6c878ff23343b" - container: - image: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4@sha256:86ea65a66c45e6597292986068523ca7738e2e949c31bfd93330db9870e1a930" - frameworks: - dynamo: "41882ae9b07232eed4850fb1daf8c958abb2556a" - sglang: "9462c303a5c5349488e84b2c346e9b5a066de21c" diff --git a/recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-6p1d-dep4-dep8-mtp.yaml b/recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-6p1d-dep4-dep8-mtp.yaml deleted file mode 100644 index e2148bcb9..000000000 --- a/recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-6p1d-dep4-dep8-mtp.yaml +++ /dev/null @@ -1,143 +0,0 @@ -name: "dsv4-pro-b300-disagg-8k1k-6p1d-dep4-dep8-mtp" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 8 - -dynamo: - hash: "41882ae9b07232eed4850fb1daf8c958abb2556a" - install: true - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4" - precision: "fp4" - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 3 - prefill_workers: 6 - gpus_per_prefill: 4 - decode_nodes: 1 - decode_workers: 1 - gpus_per_decode: 8 - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "4096" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 - - disaggregation-mode: "prefill" - watchdog-timeout: 86400 - disaggregation-transfer-backend: nixl - - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - - enable-dp-attention: true - enable-dp-lm-head: true - moe-a2a-backend: "megamoe" - - mem-fraction-static: 0.9 - max-running-requests: 1024 - cuda-graph-max-bs: 1024 - chunked-prefill-size: 32768 - stream-interval: 60 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 - - disaggregation-mode: "decode" - watchdog-timeout: 86400 - disaggregation-transfer-backend: nixl - - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - - enable-dp-attention: true - enable-dp-lm-head: true - moe-a2a-backend: "megamoe" - - speculative-algo: "EAGLE" - speculative-num-steps: 2 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 3 - - mem-fraction-static: 0.9 - max-running-requests: 4096 - cuda-graph-max-bs: 1024 - swa-full-tokens-ratio: 0.15 - context-length: 9216 - stream-interval: 60 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - random_range_ratio: 0.8 - concurrencies: "2048" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - revision: "0366e4e064385807ea86b088a5c6c878ff23343b" - container: - image: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4@sha256:86ea65a66c45e6597292986068523ca7738e2e949c31bfd93330db9870e1a930" - frameworks: - dynamo: "41882ae9b07232eed4850fb1daf8c958abb2556a" - sglang: "9462c303a5c5349488e84b2c346e9b5a066de21c" diff --git a/recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-6p1d-dep4-dep8.yaml b/recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-6p1d-dep4-dep8.yaml deleted file mode 100644 index 7fa0c42c7..000000000 --- a/recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-6p1d-dep4-dep8.yaml +++ /dev/null @@ -1,153 +0,0 @@ -name: "dsv4-pro-b300-disagg-8k1k-6p1d-dep4-dep8" - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4" - precision: "fp4" - -dynamo: - hash: "41882ae9b07232eed4850fb1daf8c958abb2556a" - install: true - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 3 - prefill_workers: 6 - gpus_per_prefill: 4 - decode_nodes: 1 - decode_workers: 1 - gpus_per_decode: 8 - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 8 - env: - args: - router-mode: "kv" - router-kv-overlap-score-weight: 0 - router-queue-threshold: 64 - router-temperature: 0.5 - no-kv-events: true - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "1280" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: "8" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - - enable-dp-attention: true - moe-a2a-backend: "megamoe" - moe-dense-tp-size: 1 - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: nixl - enable-dp-lm-head: true - - mem-fraction-static: 0.90 - max-running-requests: 512 - cuda-graph-max-bs: 512 - chunked-prefill-size: 32768 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - moe-a2a-backend: "megamoe" - moe-dense-tp-size: 1 - - disaggregation-mode: "decode" - disaggregation-transfer-backend: nixl - disaggregation-decode-polling-interval: 8 - - mem-fraction-static: 0.94 - swa-full-tokens-ratio: 0.20 - context-length: 9216 - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - enable-dp-attention: true - enable-dp-lm-head: true - max-running-requests: 18432 - cuda-graph-max-bs: 1280 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "3072" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - revision: "0366e4e064385807ea86b088a5c6c878ff23343b" - container: - image: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4@sha256:86ea65a66c45e6597292986068523ca7738e2e949c31bfd93330db9870e1a930" - frameworks: - dynamo: "41882ae9b07232eed4850fb1daf8c958abb2556a" - sglang: "9462c303a5c5349488e84b2c346e9b5a066de21c" diff --git a/recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-8p1d-dep4-dep8-mtp.yaml b/recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-8p1d-dep4-dep8-mtp.yaml deleted file mode 100644 index 4449061c4..000000000 --- a/recipes/dsv4-pro/sglang/b300-fp4/8k1k/disagg/disagg-b300-8p1d-dep4-dep8-mtp.yaml +++ /dev/null @@ -1,143 +0,0 @@ -name: "dsv4-pro-b300-disagg-8k1k-8p1d-dep4-dep8-mtp" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 8 - -dynamo: - hash: "41882ae9b07232eed4850fb1daf8c958abb2556a" - install: true - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4" - precision: "fp4" - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 4 - prefill_workers: 8 - gpus_per_prefill: 4 - decode_nodes: 1 - decode_workers: 1 - gpus_per_decode: 8 - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "4096" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 - - disaggregation-mode: "prefill" - watchdog-timeout: 86400 - disaggregation-transfer-backend: nixl - - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - - enable-dp-attention: true - enable-dp-lm-head: true - moe-a2a-backend: "megamoe" - - mem-fraction-static: 0.9 - max-running-requests: 1024 - cuda-graph-max-bs: 1024 - chunked-prefill-size: 32768 - stream-interval: 60 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 - - disaggregation-mode: "decode" - watchdog-timeout: 86400 - disaggregation-transfer-backend: nixl - - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - - enable-dp-attention: true - enable-dp-lm-head: true - moe-a2a-backend: "megamoe" - - speculative-algo: "EAGLE" - speculative-num-steps: 1 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 2 - - mem-fraction-static: 0.85 - max-running-requests: 8192 - cuda-graph-max-bs: 1280 - swa-full-tokens-ratio: 0.1 - context-length: 9216 - stream-interval: 60 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - random_range_ratio: 0.8 - concurrencies: "3072" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - revision: "0366e4e064385807ea86b088a5c6c878ff23343b" - container: - image: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4@sha256:86ea65a66c45e6597292986068523ca7738e2e949c31bfd93330db9870e1a930" - frameworks: - dynamo: "41882ae9b07232eed4850fb1daf8c958abb2556a" - sglang: "9462c303a5c5349488e84b2c346e9b5a066de21c" From cb94d592dfd03a7f3f1ede9f0a3bbbb8a8452ab7 Mon Sep 17 00:00:00 2001 From: Rohit Pujar Nagraj Date: Tue, 28 Jul 2026 00:57:40 -0700 Subject: [PATCH 3/4] fix(recipes): enable CUDA UCX transport for DSV4 Signed-off-by: Rohit Pujar Nagraj --- recipes/dsv4-pro/sglang/b300-fp4/8k1k-mtp.yaml | 2 ++ recipes/dsv4-pro/sglang/b300-fp4/8k1k.yaml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/recipes/dsv4-pro/sglang/b300-fp4/8k1k-mtp.yaml b/recipes/dsv4-pro/sglang/b300-fp4/8k1k-mtp.yaml index b6f55da7e..ff06814b7 100644 --- a/recipes/dsv4-pro/sglang/b300-fp4/8k1k-mtp.yaml +++ b/recipes/dsv4-pro/sglang/b300-fp4/8k1k-mtp.yaml @@ -39,6 +39,7 @@ base: NCCL_MNNVL_ENABLE: '1' NCCL_CUMEM_ENABLE: '1' MC_FORCE_MNNVL: '1' + UCX_TLS: 'cuda_copy,rc' SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' @@ -55,6 +56,7 @@ base: NCCL_MNNVL_ENABLE: '1' NCCL_CUMEM_ENABLE: '1' MC_FORCE_MNNVL: '1' + UCX_TLS: 'cuda_copy,rc' SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' diff --git a/recipes/dsv4-pro/sglang/b300-fp4/8k1k.yaml b/recipes/dsv4-pro/sglang/b300-fp4/8k1k.yaml index 344850704..65aff7db5 100644 --- a/recipes/dsv4-pro/sglang/b300-fp4/8k1k.yaml +++ b/recipes/dsv4-pro/sglang/b300-fp4/8k1k.yaml @@ -46,6 +46,7 @@ base: NCCL_MNNVL_ENABLE: '1' NCCL_CUMEM_ENABLE: '1' MC_FORCE_MNNVL: '1' + UCX_TLS: 'cuda_copy,rc' SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' @@ -68,6 +69,7 @@ base: NCCL_CUMEM_ENABLE: '1' SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' MC_FORCE_MNNVL: '1' + UCX_TLS: 'cuda_copy,rc' SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' From 5bde771f6a29468c71df107600da5c03b5ca4186 Mon Sep 17 00:00:00 2001 From: Rohit Pujar Nagraj Date: Tue, 28 Jul 2026 23:52:53 -0700 Subject: [PATCH 4/4] fix: align DSV4 STP input formatting Signed-off-by: Rohit Pujar Nagraj --- recipes/dsv4-pro/sglang/b300-fp4/8k1k.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes/dsv4-pro/sglang/b300-fp4/8k1k.yaml b/recipes/dsv4-pro/sglang/b300-fp4/8k1k.yaml index 65aff7db5..b7e918897 100644 --- a/recipes/dsv4-pro/sglang/b300-fp4/8k1k.yaml +++ b/recipes/dsv4-pro/sglang/b300-fp4/8k1k.yaml @@ -124,7 +124,7 @@ base: osl: 1024 concurrencies: '256' req_rate: inf - use_chat_template: true + use_chat_template: false custom_tokenizer: sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer identity: model: