diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp4/agentic/agg-gb200-tp2ep2-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp4/agentic/agg-gb200-tp2ep2-mtp.yaml new file mode 100644 index 0000000000..716ac5d4cc --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp4/agentic/agg-gb200-tp2ep2-mtp.yaml @@ -0,0 +1,78 @@ +name: qwen35-gb200-sglang-agentic-mtp-agg-tp2ep2 + +model: { path: qwen3.5-fp4, container: dynamo-sglang, precision: fp4 } +identity: + model: { repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 } + container: { image: lmsysorg/sglang:v0.5.17-cu130 } +slurm: { time_limit: "8:00:00" } +health_check: { max_attempts: 2160, interval_seconds: 10 } +resources: { gpu_type: gb200, gpus_per_node: 4, agg_nodes: 1, agg_workers: 1, gpus_per_agg: 2 } +infra: { nats_max_payload_mb: 8 } +frontend: + type: sglang + args: + worker-startup-timeout-secs: 3600 + +backend: + type: sglang + aggregated_environment: + PYTHONNOUSERSITE: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + SGLANG_ENABLE_FLASHINFER_GEMM: "true" + SGLANG_ENABLE_SPEC_V2: "1" + SGL_ENABLE_JIT_DEEPGEMM: "false" + TORCH_CUDA_ARCH_LIST: "10.0" + sglang_config: + aggregated: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + model-path: /model/ + trust-remote-code: true + tensor-parallel-size: 2 + data-parallel-size: 1 + expert-parallel-size: 2 + enable-symm-mem: true + quantization: modelopt_fp4 + fp4-gemm-backend: flashinfer_cutlass + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 8192 + attention-backend: trtllm_mha + linear-attn-decode-backend: flashinfer + moe-runner-backend: flashinfer_trtllm + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + cuda-graph-max-bs: 64 + max-running-requests: 40 + max-prefill-tokens: 16384 + chunked-prefill-size: 16384 + mem-fraction-static: 0.85 + max-mamba-cache-size: 200 + allow-auto-truncate: true + stream-interval: 50 + scheduler-recv-interval: 10 + mamba-max-states-per-path: 1 + enable-metrics: true + enable-cache-report: true + +sbatch_directives: { mem: "0", cpus-per-task: "144" } +srun_options: { mem: "0", container-remap-root: "" } +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "false" + TP: "2" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp4/agentic/agg-gb200-tp4-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp4/agentic/agg-gb200-tp4-mtp-hicache.yaml new file mode 100644 index 0000000000..917f7e5765 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp4/agentic/agg-gb200-tp4-mtp-hicache.yaml @@ -0,0 +1,85 @@ +name: qwen35-gb200-sglang-agentic-mtp-agg-tp4-hicache + +model: { path: qwen3.5-fp4, container: dynamo-sglang, precision: fp4 } +identity: + model: { repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 } + container: { image: lmsysorg/sglang:v0.5.17-cu130 } +slurm: { time_limit: "8:00:00" } +health_check: { max_attempts: 2160, interval_seconds: 10 } +resources: { gpu_type: gb200, gpus_per_node: 4, agg_nodes: 1, agg_workers: 1, gpus_per_agg: 4 } +infra: { nats_max_payload_mb: 8 } +frontend: + type: sglang + args: + worker-startup-timeout-secs: 3600 + +backend: + type: sglang + aggregated_environment: + PYTHONNOUSERSITE: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + SGLANG_ENABLE_FLASHINFER_GEMM: "true" + SGLANG_ENABLE_SPEC_V2: "1" + SGL_ENABLE_JIT_DEEPGEMM: "false" + TORCH_CUDA_ARCH_LIST: "10.0" + sglang_config: + aggregated: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + model-path: /model/ + trust-remote-code: true + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-symm-mem: true + quantization: modelopt_fp4 + fp4-gemm-backend: flashinfer_cutlass + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 8192 + attention-backend: trtllm_mha + linear-attn-decode-backend: flashinfer + moe-runner-backend: flashinfer_trtllm + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + cuda-graph-max-bs: 64 + max-running-requests: 160 + max-prefill-tokens: 16384 + chunked-prefill-size: 16384 + mem-fraction-static: 0.78 + max-mamba-cache-size: 360 + allow-auto-truncate: true + stream-interval: 50 + scheduler-recv-interval: 10 + tokenizer-worker-num: 6 + page-size: 64 + enable-hierarchical-cache: true + hicache-ratio: 0.70 + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-write-policy: write_back + mamba-max-states-per-path: 1 + enable-metrics: true + enable-cache-report: true + +sbatch_directives: { mem: "0", cpus-per-task: "144" } +srun_options: { mem: "0", container-remap-root: "" } +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "false" + TP: "4" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp4/agentic/agg-gb200-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp4/agentic/agg-gb200-tp4-mtp.yaml new file mode 100644 index 0000000000..c85a63efb9 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp4/agentic/agg-gb200-tp4-mtp.yaml @@ -0,0 +1,92 @@ +name: qwen35-gb200-sglang-agentic-mtp-agg-tp4 + +model: + path: qwen3.5-fp4 + container: dynamo-sglang + precision: fp4 + +identity: + model: { repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 } + container: { image: lmsysorg/sglang:v0.5.17-cu130 } + +slurm: { time_limit: "8:00:00" } +health_check: { max_attempts: 2160, interval_seconds: 10 } + +resources: + gpu_type: gb200 + gpus_per_node: 4 + agg_nodes: 1 + agg_workers: 1 + gpus_per_agg: 4 + +infra: { nats_max_payload_mb: 8 } +frontend: + type: sglang + args: + worker-startup-timeout-secs: 3600 + +backend: + type: sglang + aggregated_environment: + PYTHONNOUSERSITE: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + SGLANG_ENABLE_FLASHINFER_GEMM: "true" + SGLANG_ENABLE_SPEC_V2: "1" + SGL_ENABLE_JIT_DEEPGEMM: "false" + TORCH_CUDA_ARCH_LIST: "10.0" + sglang_config: + aggregated: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + model-path: /model/ + trust-remote-code: true + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-symm-mem: true + quantization: modelopt_fp4 + fp4-gemm-backend: flashinfer_cutlass + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 8192 + attention-backend: trtllm_mha + linear-attn-decode-backend: flashinfer + moe-runner-backend: flashinfer_trtllm + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + cuda-graph-max-bs: 64 + max-running-requests: 160 + max-prefill-tokens: 16384 + chunked-prefill-size: 16384 + mem-fraction-static: 0.80 + max-mamba-cache-size: 360 + allow-auto-truncate: true + stream-interval: 50 + scheduler-recv-interval: 10 + tokenizer-worker-num: 6 + mamba-max-states-per-path: 1 + enable-metrics: true + enable-cache-report: true + +sbatch_directives: { mem: "0", cpus-per-task: "144" } +srun_options: { mem: "0", container-remap-root: "" } + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "false" + TP: "4" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 007cc48ed7..92e8f1646e 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7194,6 +7194,64 @@ qwen3.5-fp4-b200-sglang-agentic-mtp: - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 14] } - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [16, 18, 20, 22, 24, 28, 32] } +qwen3.5-fp4-gb200-dynamo-sglang-agentic-mtp: + image: lmsysorg/sglang:v0.5.17-cu130 + model: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + model-prefix: qwen3.5 + runner: cluster:gb200-nv + precision: fp4 + framework: dynamo-sglang + multinode: true + disagg: false + scenarios: + agentic-coding: + - dram-utilization: 0.70 + search-space: + # Measured Pareto points from the complete fast sweep. TP4 covers the + # low-latency branch; HiCache extends the high-throughput branch. + - spec-decoding: mtp + kv-offloading: none + conc-list: [1, 4, 8, 12, 16, 20, 32, 40] + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=3.39" + - "CONFIG_FILE=recipes/sglang/qwen3.5/gb200-fp4/agentic/agg-gb200-tp4-mtp.yaml" + decode: { num-worker: 0, tp: 4, ep: 1, dp-attn: false } + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: hicache } + conc-list: [48, 56, 64, 72] + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=3.39" + - "CONFIG_FILE=recipes/sglang/qwen3.5/gb200-fp4/agentic/agg-gb200-tp4-mtp-hicache.yaml" + decode: { num-worker: 0, tp: 4, ep: 1, dp-attn: false } + # TP2/EP2 supplies the middle of the measured frontier with half the + # GPU count of TP4. + - spec-decoding: mtp + kv-offloading: none + conc-list: [4, 8, 16, 20, 24] + prefill: + num-worker: 1 + tp: 2 + ep: 2 + dp-attn: false + additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=3.39" + - "CONFIG_FILE=recipes/sglang/qwen3.5/gb200-fp4/agentic/agg-gb200-tp2ep2-mtp.yaml" + decode: { num-worker: 0, tp: 2, ep: 2, dp-attn: false } + minimaxm3-fp4-b300-vllm-agentic-mtp: image: vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7 model: nvidia/MiniMax-M3-NVFP4 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 3aac6d1586..bb655ee911 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5845,3 +5845,14 @@ description: - "Add MiniMax-M3 MXFP8 AgentX on H100 with vLLM v0.27.1, EAGLE3 golden AL 2.78, resident TP8 c1-c5, and Mooncake DRAM offload c6/c8." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2564 + +- config-keys: + - qwen3.5-fp4-gb200-dynamo-sglang-agentic-mtp + description: + - "Add the measured GB200 Qwen3.5 NVFP4 AgentX Pareto frontier across aggregate TP4, TP2/EP2, and TP4 HiCache with K=3 MTP." + - "Use stable SGLang v0.5.17-cu130 and NVIDIA/srt-slurm v1.0.45." + - "Enable SGLang metrics/cache reporting and require nonempty sglang: server metrics before publishing results." + - "Use K=3 golden synthetic acceptance length 3.39 for throughput; eval retains real MTP verification." + - "Give the released SGLang router one hour to register the 28-minute-loading backend and route AgentX sessions by correlation ID." + - "Bound hybrid-state pools to the measured resident and HiCache operating ranges instead of reserving capacity that cannot fit beside target and MTP weights." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2568 diff --git a/runners/launch_gb200-nv.sh b/runners/launch_gb200-nv.sh index 34d7c4c806..660ae078d5 100755 --- a/runners/launch_gb200-nv.sh +++ b/runners/launch_gb200-nv.sh @@ -178,6 +178,9 @@ if [[ $FRAMEWORK == "dynamo-sglang" ]]; then elif [[ $MODEL_PREFIX == "qwen3.5" && $PRECISION == "fp8" ]]; then export MODEL_PATH="/mnt/lustre01/models/Qwen3.5-397B-A17B-FP8" export SRT_SLURM_MODEL_PREFIX="qwen3.5-fp8" + elif [[ $MODEL_PREFIX == "qwen3.5" && $PRECISION == "fp4" ]]; then + export MODEL_PATH="/mnt/lustre01/models/Qwen3.5-397B-A17B-NVFP4-V2" + export SRT_SLURM_MODEL_PREFIX="qwen3.5-fp4" elif [[ $MODEL_PREFIX == "glm5.1" && $PRECISION == "fp4" ]]; then # SRT_SLURM_MODEL_PREFIX matches the model.path alias ("glm-5-fp4") # in our GLM-5.1 sglang recipes. @@ -388,8 +391,20 @@ if [ -d "$SRT_REPO_DIR" ]; then rm -rf "$SRT_REPO_DIR" fi -# TODO(CJQ): make first class upon srt-slurm upstream refactor -if [[ "$IS_AGENTIC" == "1" ]]; then +# Qwen3.5 FP4 AgentX uses the latest released srt-slurm. v1.0.45 injects +# the aggregate logical-worker Prometheus endpoint into custom benchmarks. +if [[ "$IS_AGENTIC" == "1" && "$MODEL_PREFIX" == "qwen3.5" && "$PRECISION" == "fp4" && "$FRAMEWORK" == "dynamo-sglang" ]]; then + git clone --branch v1.0.45 --single-branch https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" + cd "$SRT_REPO_DIR" + test "$(git rev-parse HEAD)" = "9d8d92b20c350a5d42f0709f5a0b64e30eb37d33" || { + echo "Error: NVIDIA/srt-slurm v1.0.45 resolved to an unexpected commit" >&2 + exit 1 + } + mkdir -p recipes/sglang/qwen3.5/gb200-fp4/agentic + cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp4/agentic" \ + recipes/sglang/qwen3.5/gb200-fp4/agentic +# TODO(CJQ): migrate the remaining Agentic model paths to released srt-slurm. +elif [[ "$IS_AGENTIC" == "1" ]]; then # Agentic multi-node pins cquil11/srt-slurm-nv revisions that provide: # - BenchmarkType.CUSTOM + benchmark.command + benchmark.env # (the hook that hands off to benchmarks/multi_node/agentic_srt.sh) diff --git a/runners/synthetic_injectors/__init__.py b/runners/synthetic_injectors/__init__.py index 5bc4d8a366..8836a46b4a 100644 --- a/runners/synthetic_injectors/__init__.py +++ b/runners/synthetic_injectors/__init__.py @@ -35,4 +35,4 @@ def get_injector(framework): # Import backends after register/get_injector are defined so each module can # call register() at import time. Add new frameworks (sglang, trtllm, ...) here. -from . import vllm # noqa: E402,F401 +from . import sglang, vllm # noqa: E402,F401 diff --git a/runners/synthetic_injectors/sglang.py b/runners/synthetic_injectors/sglang.py new file mode 100644 index 0000000000..48fb1706aa --- /dev/null +++ b/runners/synthetic_injectors/sglang.py @@ -0,0 +1,34 @@ +"""SGLang synthetic-acceptance backend for srt-slurm recipes.""" + +import re +import sys + +from . import register + +_SPEC_STEPS_RE = re.compile(r"(?m)^\s+speculative-num-steps:\s*([0-9]+)\s*$") +_ENV_BLOCK_RE = re.compile(r"(?m)^( (?:aggregated|prefill|decode)_environment:\s*)$") + + +def spec_tokens_from_recipe(text): + """Read SGLang's speculative step count from the recipe.""" + match = _SPEC_STEPS_RE.search(text) + return int(match.group(1)) if match else None + + +def rewrite(content, al, log): + """Add throughput-only golden-acceptance variables to each worker role.""" + if "SGLANG_SIMULATE_ACC_LEN" in content: + raise ValueError("recipe already contains SGLANG_SIMULATE_ACC_* variables") + + variables = ( + f'\n SGLANG_SIMULATE_ACC_LEN: "{al:g}"' + '\n SGLANG_SIMULATE_ACC_METHOD: "match-expected"' + '\n SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token"' + ) + rewritten, count = _ENV_BLOCK_RE.subn(lambda match: match.group(1) + variables, content) + if count: + log(f"Added SGLANG_SIMULATE_ACC_* to {count} worker environment block(s)") + return rewritten, count + + +register("dynamo-sglang", sys.modules[__name__])