From 4937e535b2ced4ff77207600243b7e1fafcb8693 Mon Sep 17 00:00:00 2001 From: ishandhanani Date: Wed, 4 Feb 2026 17:53:33 -0800 Subject: [PATCH 01/10] init mtp --- .github/configs/nvidia-master.yaml | 110 +++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) diff --git a/.github/configs/nvidia-master.yaml b/.github/configs/nvidia-master.yaml index a5cad52066..36233b67c3 100644 --- a/.github/configs/nvidia-master.yaml +++ b/.github/configs/nvidia-master.yaml @@ -1652,6 +1652,116 @@ gptoss-fp4-h100-vllm: - { tp: 4, conc-start: 4, conc-end: 64 } - { tp: 8, conc-start: 4, conc-end: 16 } +dsr1-fp8-h100-dynamo-sglang: + image: lmsysorg/sglang:v0.5.8-cu130 + model: deepseek-ai/DeepSeek-R1-0528 + model-prefix: dsr1 + runner: h100-multinode-slurm + precision: fp8 + framework: dynamo-sglang + multinode: true + disagg: true + seq-len-configs: + - isl: 1024 + osl: 1024 + search-space: + # MTP: Max throughput TEP (1 prefill, 2 decode) + - spec-decoding: "mtp" + conc-list: [1, 2, 4, 8, 16, 32, 64, 128] + prefill: + num-worker: 1 + tp: 16 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/h100/1k1k/mtp/h100-fp8-1p2d-max-tp-mtp.yaml" + decode: + num-worker: 2 + tp: 16 + ep: 1 + dp-attn: false + # MTP: Max throughput DEP (1 prefill, 1 decode, dp-attention) + - spec-decoding: "mtp" + conc-list: [1, 2, 4, 8, 16, 32, 64] + prefill: + num-worker: 1 + tp: 16 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/h100/1k1k/mtp/h100-fp8-1p1d-max-dep-mtp.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + - isl: 1024 + osl: 8192 + search-space: + # MTP: Max throughput TEP (1 prefill, 2 decode) + - spec-decoding: "mtp" + conc-list: [1, 2, 4, 8, 16, 32, 64] + prefill: + num-worker: 1 + tp: 16 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/h100/1k8k/mtp/h100-fp8-1p2d-max-tp-mtp.yaml" + decode: + num-worker: 2 + tp: 16 + ep: 1 + dp-attn: false + # MTP: Max throughput DEP (1 prefill, 1 decode, dp-attention) + - spec-decoding: "mtp" + conc-list: [1, 2, 4, 8, 16, 32, 64] + prefill: + num-worker: 1 + tp: 16 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/h100/1k8k/mtp/h100-fp8-1p1d-max-dep-mtp.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + - isl: 8192 + osl: 1024 + search-space: + # MTP: Max throughput TEP (1 prefill, 1 decode) + - spec-decoding: "mtp" + conc-list: [1, 2, 4, 8, 16, 32, 64, 128] + prefill: + num-worker: 1 + tp: 16 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/h100/8k1k/mtp/h100-fp8-1p1d-max-tp-mtp.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 1 + dp-attn: false + # MTP: Max throughput DEP (1 prefill, 1 decode, dp-attention) + - spec-decoding: "mtp" + conc-list: [1, 2, 4, 8, 16, 32, 64] + prefill: + num-worker: 1 + tp: 16 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/h100/8k1k/mtp/h100-fp8-1p1d-max-dep-mtp.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + gptoss-fp4-h200-trt: image: nvcr.io#nvidia/tensorrt-llm/release:gpt-oss-dev model: openai/gpt-oss-120b From 603daca88e081c80a3cdbb29b59030b2fc64eda7 Mon Sep 17 00:00:00 2001 From: "claude[bot]" <41898282+claude[bot]@users.noreply.github.com> Date: Thu, 5 Feb 2026 15:34:45 +0000 Subject: [PATCH 02/10] docs: add perf-changelog entry for H100 MTP SGLang configs Add changelog entry for dsr1-fp8-h100-dynamo-sglang configuration with MTP disaggregated multinode support. Co-authored-by: Cameron Quilici --- perf-changelog.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 813c173e6e..69e7ec784e 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -367,3 +367,12 @@ description: - "Fix model_prefix argument in yaml configs" pr-link: https://github.com/InferenceMAX/InferenceMAX/pull/646 + +- config-keys: + - dsr1-fp8-h100-dynamo-sglang + description: + - "Add DeepSeek-R1 FP8 H100 Dynamo SGLang MTP disaggregated multinode configurations" + - "Image: lmsysorg/sglang:v0.5.8-cu130" + - "1k1k, 1k8k, 8k1k sequence lengths with MTP speculative decoding" + - "Two modes per seq-len: Max throughput TEP (1P2D) and Max throughput DEP (1P1D with dp-attention)" + pr-link: https://github.com/InferenceMAX/InferenceMAX/pull/644 From 161fec16f4d7579215fe74455e4d1c10aeaf3d9a Mon Sep 17 00:00:00 2001 From: Kedar Potdar Date: Sun, 8 Feb 2026 10:17:35 -0800 Subject: [PATCH 03/10] Fix perf-changelog whitespace to satisfy process_changelog (additions-only) --- perf-changelog.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index f74f910de0..04c610f0a7 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -408,7 +408,7 @@ - "Without this arg, MTP acceptance rates are artificially high for DeepSeek with MTP" pr-link: https://github.com/InferenceMAX/InferenceMAX/pull/647 -- config-keys: +- config-keys: - dsr1-fp8-b200-sglang-mtp description: - "Add MTP (Multi-Token Prediction) support for DeepSeek R1 FP8 B200 SGLang using EAGLE speculative decoding" @@ -475,7 +475,7 @@ - "Refactor to use CONFIG_FILE-based srt-slurm recipes instead of inline parameters" - "Add 1k1k configurations: low-latency (1P2D), mid-curve (4P8D), max-tpt (4P12D)" - "Add 8k1k configurations: low-latency (1P4D), mid-curve (6P12D), max-tpt (10P8D)" - pr-link: https://github.com/InferenceMAX/InferenceMAX/pull/633 + pr-link: https://github.com/InferenceMAX/InferenceMAX/pull/633 - config-keys: - dsr1-fp8-h100-dynamo-sglang-mtp From 78f6aa6dc341cf20172afcdbcb4e61ca1977b236 Mon Sep 17 00:00:00 2001 From: Kedar Potdar Date: Sun, 8 Feb 2026 10:19:11 -0800 Subject: [PATCH 04/10] typo --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 04c610f0a7..7b8c3484fb 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -478,7 +478,7 @@ pr-link: https://github.com/InferenceMAX/InferenceMAX/pull/633 - config-keys: - - dsr1-fp8-h100-dynamo-sglang-mtp + - dsr1-fp8-h100-dynamo-sglang description: - "Add DeepSeek-R1 FP8 H100 Dynamo SGLang MTP disaggregated multinode configurations" - "Image: lmsysorg/sglang:v0.5.8-cu130" From 4a3dec7b36cc008a5c587d864c4275d4071f32fa Mon Sep 17 00:00:00 2001 From: ishandhanani Date: Mon, 9 Feb 2026 10:53:56 -0800 Subject: [PATCH 05/10] Add SGLang support to H100 runner - Add framework-aware model path and container configuration - Map SGLang container to /mnt/nfs/lustre/containers/lmsysorg_sglang_v0.5.8.post1-cu130.sqsh - Add nginx container for SGLang at /mnt/nfs/lustre/containers/nginx_1.27.4.sqsh - Update srtslurm.yaml with dynamo-sglang and nginx-sqsh entries --- runners/launch_h100-dgxc-slurm.sh | 57 +++++++++++++++++++++++-------- 1 file changed, 42 insertions(+), 15 deletions(-) diff --git a/runners/launch_h100-dgxc-slurm.sh b/runners/launch_h100-dgxc-slurm.sh index d461e4a940..2112cc1d4f 100644 --- a/runners/launch_h100-dgxc-slurm.sh +++ b/runners/launch_h100-dgxc-slurm.sh @@ -2,6 +2,31 @@ set -x +# MODEL_PATH: Override with pre-downloaded paths on H100 runner +# The yaml files specify HuggingFace model IDs for portability, but we use +# local paths to avoid repeated downloading on the shared H100 cluster. +if [[ $FRAMEWORK == "dynamo-sglang" ]]; then + if [[ $MODEL_PREFIX == "dsr1" && $PRECISION == "fp8" ]]; then + export MODEL_PATH="/mnt/numa1/shared/models/dsr1-fp8" + export SRT_SLURM_MODEL_PREFIX="dsr1-fp8" + else + echo "Unsupported model prefix/precision for dynamo-sglang: $MODEL_PREFIX/$PRECISION" + exit 1 + fi +elif [[ $FRAMEWORK == "dynamo-trt" ]]; then + if [[ $MODEL_PREFIX == "dsr1" && $PRECISION == "fp8" ]]; then + export MODEL_PATH="/mnt/numa1/shared/models/dsr1-fp8" + export SERVED_MODEL_NAME="DeepSeek-R1-0528" + export SRT_SLURM_MODEL_PREFIX="DeepSeek-R1-0528" + else + echo "Unsupported model prefix/precision for dynamo-trt: $MODEL_PREFIX/$PRECISION" + exit 1 + fi +else + echo "Unsupported framework: $FRAMEWORK. Supported frameworks are: dynamo-trt, dynamo-sglang" + exit 1 +fi + echo "Cloning srt-slurm repository..." SRT_REPO_DIR="srt-slurm" if [ -d "$SRT_REPO_DIR" ]; then @@ -31,25 +56,24 @@ echo "Configs available at: $SRT_REPO_DIR/" export SLURM_PARTITION="hpc-gpu-1" export SLURM_ACCOUNT="customer" -# Convert IMAGE to srt-slurm format (nvcr.io/ -> nvcr.io#) -CONTAINER_KEY=$(echo "$IMAGE" | sed 's|nvcr.io/|nvcr.io#|') - -# Map container image to local squash file -SQUASH_FILE="/mnt/nfs/sa-shared/containers/$(echo "$IMAGE" | sed 's|nvcr.io/||' | sed 's/[\/:@#]/+/g').sqsh" - -if [[ $MODEL_PREFIX == "dsr1" && $PRECISION == "fp8" ]]; then - export MODEL_PATH="/mnt/numa1/shared/models/dsr1-fp8" - export SERVED_MODEL_NAME="DeepSeek-R1-0528" - export SRT_SLURM_MODEL_PREFIX="DeepSeek-R1-0528" -else - echo "Unsupported model prefix: $MODEL_PREFIX. Supported prefixes are: DeepSeek-R1-0528" - exit 1 +# Map container images to local squash files based on framework +NGINX_SQUASH_FILE="/mnt/nfs/lustre/containers/nginx_1.27.4.sqsh" + +if [[ $FRAMEWORK == "dynamo-sglang" ]]; then + # SGLang container mapping + SQUASH_FILE="/mnt/nfs/lustre/containers/lmsysorg_sglang_v0.5.8.post1-cu130.sqsh" + CONTAINER_KEY="lmsysorg/sglang:v0.5.8-cu130" +elif [[ $FRAMEWORK == "dynamo-trt" ]]; then + # TRT-LLM container mapping - convert IMAGE to srt-slurm format (nvcr.io/ -> nvcr.io#) + CONTAINER_KEY=$(echo "$IMAGE" | sed 's|nvcr.io/|nvcr.io#|') + SQUASH_FILE="/mnt/nfs/sa-shared/containers/$(echo "$IMAGE" | sed 's|nvcr.io/||' | sed 's/[\/:@#]/+/g').sqsh" fi export ISL="$ISL" export OSL="$OSL" -# Create srtslurm.yaml for srtctl +# Create srtslurm.yaml for srtctl (used by both frameworks) +SRTCTL_ROOT="${GITHUB_WORKSPACE}/${SRT_REPO_DIR}" echo "Creating srtslurm.yaml configuration..." cat > srtslurm.yaml < Date: Tue, 10 Feb 2026 10:27:53 -0800 Subject: [PATCH 06/10] Add STP configs alongside MTP for H100 SGLang - Add STP entries (without spec-decoding) for all ISL/OSL combinations - Keep MTP entries (with spec-decoding: mtp) as before - Now both STP and MTP will run when testing dsr1-fp8-h100-dynamo-sglang --- .github/configs/nvidia-master.yaml | 84 ++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/.github/configs/nvidia-master.yaml b/.github/configs/nvidia-master.yaml index 011287dac3..759c4e9c6a 100644 --- a/.github/configs/nvidia-master.yaml +++ b/.github/configs/nvidia-master.yaml @@ -2526,6 +2526,34 @@ dsr1-fp8-h100-dynamo-sglang: - isl: 1024 osl: 1024 search-space: + # STP: Max throughput TEP (1 prefill, 2 decode) + - conc-list: [1, 2, 4, 8, 16, 32, 64, 128] + prefill: + num-worker: 1 + tp: 16 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/h100/1k1k/stp/h100-fp8-1p2d-max-tp.yaml" + decode: + num-worker: 2 + tp: 16 + ep: 1 + dp-attn: false + # STP: Max throughput DEP (1 prefill, 1 decode, dp-attention) + - conc-list: [1, 2, 4, 8, 16, 32, 64] + prefill: + num-worker: 1 + tp: 16 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/h100/1k1k/stp/h100-fp8-1p1d-max-dep.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true # MTP: Max throughput TEP (1 prefill, 2 decode) - spec-decoding: "mtp" conc-list: [1, 2, 4, 8, 16, 32, 64, 128] @@ -2559,6 +2587,34 @@ dsr1-fp8-h100-dynamo-sglang: - isl: 1024 osl: 8192 search-space: + # STP: Max throughput TEP (1 prefill, 2 decode) + - conc-list: [1, 2, 4, 8, 16, 32] + prefill: + num-worker: 1 + tp: 16 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/h100/1k8k/stp/h100-fp8-1p2d-max-tp.yaml" + decode: + num-worker: 2 + tp: 16 + ep: 1 + dp-attn: false + # STP: Max throughput DEP (1 prefill, 1 decode, dp-attention) + - conc-list: [1, 2, 4, 8] + prefill: + num-worker: 1 + tp: 16 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/h100/1k8k/stp/h100-fp8-1p1d-max-dep.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true # MTP: Max throughput TEP (1 prefill, 2 decode) - spec-decoding: "mtp" conc-list: [1, 2, 4, 8, 16, 32, 64] @@ -2592,6 +2648,34 @@ dsr1-fp8-h100-dynamo-sglang: - isl: 8192 osl: 1024 search-space: + # STP: Max throughput TEP (1 prefill, 1 decode) + - conc-list: [1, 2, 4, 8, 16, 32, 64, 128] + prefill: + num-worker: 1 + tp: 16 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/h100/8k1k/stp/h100-fp8-1p1d-max-tp.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 1 + dp-attn: false + # STP: Max throughput DEP (1 prefill, 1 decode, dp-attention) + - conc-list: [1, 2, 4, 8, 16, 32, 64] + prefill: + num-worker: 1 + tp: 16 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/h100/8k1k/stp/h100-fp8-1p1d-max-dep.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true # MTP: Max throughput TEP (1 prefill, 1 decode) - spec-decoding: "mtp" conc-list: [1, 2, 4, 8, 16, 32, 64, 128] From 2b1ea7521123e89f2383d61d55b7238432c7e5d9 Mon Sep 17 00:00:00 2001 From: ishandhanani Date: Tue, 10 Feb 2026 10:42:01 -0800 Subject: [PATCH 07/10] Restore B200 MTP SGLang config lost in merge The dsr1-fp8-b200-dynamo-sglang-mtp config was accidentally removed during merge conflict resolution. --- .github/configs/nvidia-master.yaml | 152 +++++++++++++++++++++++++++++ 1 file changed, 152 insertions(+) diff --git a/.github/configs/nvidia-master.yaml b/.github/configs/nvidia-master.yaml index 759c4e9c6a..a22ed6ae8e 100644 --- a/.github/configs/nvidia-master.yaml +++ b/.github/configs/nvidia-master.yaml @@ -6131,3 +6131,155 @@ dsr1-fp8-b200-dynamo-sglang: tp: 8 ep: 8 dp-attn: true + +dsr1-fp8-b200-dynamo-sglang-mtp: + image: lmsysorg/sglang:v0.5.8.post1-cu130-amd64 + model: deepseek-ai/DeepSeek-R1-0528 + model-prefix: dsr1 + runner: b200-multinode-slurm + precision: fp8 + framework: dynamo-sglang + multinode: true + disagg: true + seq-len-configs: + - isl: 1024 + osl: 1024 + search-space: + # MTP low-latency: 1P1D + - spec-decoding: "mtp" + conc-list: [4, 64] + prefill: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/b200-fp8/1k1k/mtp/low-latency-tep8-1p1d.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: false + # MTP low-latency: 1P3D + - spec-decoding: "mtp" + conc-list: [4, 8, 16, 32] + prefill: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/b200-fp8/1k1k/mtp/low-latency-tep8-1p3d.yaml" + decode: + num-worker: 3 + tp: 8 + ep: 8 + dp-attn: false + # MTP max-tpt: 1P5D + - spec-decoding: "mtp" + conc-list: [512, 4096] + prefill: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/b200-fp8/1k1k/mtp/max-tpt-dep8-1p5d.yaml" + decode: + num-worker: 5 + tp: 8 + ep: 8 + dp-attn: true + # MTP max-tpt: 2P5D + - spec-decoding: "mtp" + conc-list: [1024, 2048, 4096] + prefill: + num-worker: 2 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/b200-fp8/1k1k/mtp/max-tpt-dep8-2p5d.yaml" + decode: + num-worker: 5 + tp: 8 + ep: 8 + dp-attn: true + - isl: 8192 + osl: 1024 + search-space: + # MTP low-latency: 1P1D + - spec-decoding: "mtp" + conc-list: [16, 32, 64] + prefill: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/b200-fp8/8k1k/mtp/low-latency-tep8-1p1d.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: false + # MTP low-latency: 1P4D + - spec-decoding: "mtp" + conc-list: [8, 256] + prefill: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/b200-fp8/8k1k/mtp/low-latency-tep8-1p4d.yaml" + decode: + num-worker: 4 + tp: 8 + ep: 8 + dp-attn: false + # MTP low-latency: 1P6D + - spec-decoding: "mtp" + conc-list: [4, 8, 16, 256] + prefill: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/b200-fp8/8k1k/mtp/low-latency-tep8-1p6d.yaml" + decode: + num-worker: 6 + tp: 8 + ep: 8 + dp-attn: false + # MTP max-tpt: 1P1D + - spec-decoding: "mtp" + conc-list: [256] + prefill: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/b200-fp8/8k1k/mtp/max-tpt-dep8-1p1d.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + # MTP max-tpt: 2P1D + - spec-decoding: "mtp" + conc-list: [128, 512] + prefill: + num-worker: 2 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/b200-fp8/8k1k/mtp/max-tpt-dep8-2p1d.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true From 75e7bf66f49f222cf52598ced641a9e1fff9dffe Mon Sep 17 00:00:00 2001 From: ishandhanani Date: Tue, 10 Feb 2026 10:45:04 -0800 Subject: [PATCH 08/10] Add perf-changelog entry for H100 STP SGLang (PR #643) --- perf-changelog.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index b16be6d8a2..5433e3c6fd 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -494,6 +494,15 @@ - "9 recipes: 4x 1k1k + 5x 8k1k, low-latency and max-throughput with EAGLE speculative decoding" pr-link: https://github.com/InferenceMAX/InferenceMAX/pull/667 +- config-keys: + - dsr1-fp8-h100-dynamo-sglang + description: + - "Add DeepSeek-R1 FP8 H100 Dynamo SGLang STP disaggregated multinode configurations" + - "Image: lmsysorg/sglang:v0.5.8-cu130" + - "1k1k, 1k8k, 8k1k sequence lengths" + - "Two modes per seq-len: Max throughput TEP (1P2D) and Max throughput DEP (1P1D with dp-attention)" + pr-link: https://github.com/InferenceMAX/InferenceMAX/pull/643 + - config-keys: - dsr1-fp8-h100-dynamo-sglang description: From bf0ceb64428a7827a37830bf78e955ba26868958 Mon Sep 17 00:00:00 2001 From: ishandhanani Date: Tue, 10 Feb 2026 10:48:53 -0800 Subject: [PATCH 09/10] Comment out H100 SGLang STP configs for MTP-only CI testing --- .github/configs/nvidia-master.yaml | 168 ++++++++++++++--------------- 1 file changed, 84 insertions(+), 84 deletions(-) diff --git a/.github/configs/nvidia-master.yaml b/.github/configs/nvidia-master.yaml index a22ed6ae8e..c98650d99e 100644 --- a/.github/configs/nvidia-master.yaml +++ b/.github/configs/nvidia-master.yaml @@ -2526,34 +2526,34 @@ dsr1-fp8-h100-dynamo-sglang: - isl: 1024 osl: 1024 search-space: - # STP: Max throughput TEP (1 prefill, 2 decode) - - conc-list: [1, 2, 4, 8, 16, 32, 64, 128] - prefill: - num-worker: 1 - tp: 16 - ep: 1 - dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/h100/1k1k/stp/h100-fp8-1p2d-max-tp.yaml" - decode: - num-worker: 2 - tp: 16 - ep: 1 - dp-attn: false - # STP: Max throughput DEP (1 prefill, 1 decode, dp-attention) - - conc-list: [1, 2, 4, 8, 16, 32, 64] - prefill: - num-worker: 1 - tp: 16 - ep: 1 - dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/h100/1k1k/stp/h100-fp8-1p1d-max-dep.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true + # # STP: Max throughput TEP (1 prefill, 2 decode) + # - conc-list: [1, 2, 4, 8, 16, 32, 64, 128] + # prefill: + # num-worker: 1 + # tp: 16 + # ep: 1 + # dp-attn: false + # additional-settings: + # - "CONFIG_FILE=recipes/h100/1k1k/stp/h100-fp8-1p2d-max-tp.yaml" + # decode: + # num-worker: 2 + # tp: 16 + # ep: 1 + # dp-attn: false + # # STP: Max throughput DEP (1 prefill, 1 decode, dp-attention) + # - conc-list: [1, 2, 4, 8, 16, 32, 64] + # prefill: + # num-worker: 1 + # tp: 16 + # ep: 1 + # dp-attn: false + # additional-settings: + # - "CONFIG_FILE=recipes/h100/1k1k/stp/h100-fp8-1p1d-max-dep.yaml" + # decode: + # num-worker: 1 + # tp: 16 + # ep: 16 + # dp-attn: true # MTP: Max throughput TEP (1 prefill, 2 decode) - spec-decoding: "mtp" conc-list: [1, 2, 4, 8, 16, 32, 64, 128] @@ -2587,34 +2587,34 @@ dsr1-fp8-h100-dynamo-sglang: - isl: 1024 osl: 8192 search-space: - # STP: Max throughput TEP (1 prefill, 2 decode) - - conc-list: [1, 2, 4, 8, 16, 32] - prefill: - num-worker: 1 - tp: 16 - ep: 1 - dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/h100/1k8k/stp/h100-fp8-1p2d-max-tp.yaml" - decode: - num-worker: 2 - tp: 16 - ep: 1 - dp-attn: false - # STP: Max throughput DEP (1 prefill, 1 decode, dp-attention) - - conc-list: [1, 2, 4, 8] - prefill: - num-worker: 1 - tp: 16 - ep: 1 - dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/h100/1k8k/stp/h100-fp8-1p1d-max-dep.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true + # # STP: Max throughput TEP (1 prefill, 2 decode) + # - conc-list: [1, 2, 4, 8, 16, 32] + # prefill: + # num-worker: 1 + # tp: 16 + # ep: 1 + # dp-attn: false + # additional-settings: + # - "CONFIG_FILE=recipes/h100/1k8k/stp/h100-fp8-1p2d-max-tp.yaml" + # decode: + # num-worker: 2 + # tp: 16 + # ep: 1 + # dp-attn: false + # # STP: Max throughput DEP (1 prefill, 1 decode, dp-attention) + # - conc-list: [1, 2, 4, 8] + # prefill: + # num-worker: 1 + # tp: 16 + # ep: 1 + # dp-attn: false + # additional-settings: + # - "CONFIG_FILE=recipes/h100/1k8k/stp/h100-fp8-1p1d-max-dep.yaml" + # decode: + # num-worker: 1 + # tp: 16 + # ep: 16 + # dp-attn: true # MTP: Max throughput TEP (1 prefill, 2 decode) - spec-decoding: "mtp" conc-list: [1, 2, 4, 8, 16, 32, 64] @@ -2648,34 +2648,34 @@ dsr1-fp8-h100-dynamo-sglang: - isl: 8192 osl: 1024 search-space: - # STP: Max throughput TEP (1 prefill, 1 decode) - - conc-list: [1, 2, 4, 8, 16, 32, 64, 128] - prefill: - num-worker: 1 - tp: 16 - ep: 1 - dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/h100/8k1k/stp/h100-fp8-1p1d-max-tp.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 1 - dp-attn: false - # STP: Max throughput DEP (1 prefill, 1 decode, dp-attention) - - conc-list: [1, 2, 4, 8, 16, 32, 64] - prefill: - num-worker: 1 - tp: 16 - ep: 1 - dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/h100/8k1k/stp/h100-fp8-1p1d-max-dep.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true + # # STP: Max throughput TEP (1 prefill, 1 decode) + # - conc-list: [1, 2, 4, 8, 16, 32, 64, 128] + # prefill: + # num-worker: 1 + # tp: 16 + # ep: 1 + # dp-attn: false + # additional-settings: + # - "CONFIG_FILE=recipes/h100/8k1k/stp/h100-fp8-1p1d-max-tp.yaml" + # decode: + # num-worker: 1 + # tp: 16 + # ep: 1 + # dp-attn: false + # # STP: Max throughput DEP (1 prefill, 1 decode, dp-attention) + # - conc-list: [1, 2, 4, 8, 16, 32, 64] + # prefill: + # num-worker: 1 + # tp: 16 + # ep: 1 + # dp-attn: false + # additional-settings: + # - "CONFIG_FILE=recipes/h100/8k1k/stp/h100-fp8-1p1d-max-dep.yaml" + # decode: + # num-worker: 1 + # tp: 16 + # ep: 16 + # dp-attn: true # MTP: Max throughput TEP (1 prefill, 1 decode) - spec-decoding: "mtp" conc-list: [1, 2, 4, 8, 16, 32, 64, 128] From 883d8fee9d6274857e60b111566b638358f68756 Mon Sep 17 00:00:00 2001 From: ishandhanani Date: Wed, 11 Feb 2026 11:53:41 -0800 Subject: [PATCH 10/10] Increase H100 SLURM time limit from 4h to 6h --- runners/launch_h100-dgxc-slurm.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runners/launch_h100-dgxc-slurm.sh b/runners/launch_h100-dgxc-slurm.sh index 2112cc1d4f..0d6f718bfa 100644 --- a/runners/launch_h100-dgxc-slurm.sh +++ b/runners/launch_h100-dgxc-slurm.sh @@ -81,7 +81,7 @@ cat > srtslurm.yaml <