From 693e911d089233d53fdca6dd364d689357b005c1 Mon Sep 17 00:00:00 2001 From: Frankie Siino Date: Wed, 12 Aug 2026 05:45:28 +0000 Subject: [PATCH 01/26] Fix eval container profiling dependency Signed-off-by: Frankie Siino --- benchmarks/nemotron_3.5_super/build_eval_container.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/benchmarks/nemotron_3.5_super/build_eval_container.sh b/benchmarks/nemotron_3.5_super/build_eval_container.sh index bde782c4956..3e87855d091 100644 --- a/benchmarks/nemotron_3.5_super/build_eval_container.sh +++ b/benchmarks/nemotron_3.5_super/build_eval_container.sh @@ -49,8 +49,12 @@ git checkout $NEMO_GYM_GIT_REF # See the script for more information. python benchmarks/nemotron_3.5_super/downgrade_python.py + uv sync --active -uv pip install "\$ray_dependency" +# gym eval prepare imports nemo_gym.profiling at startup. In this Python 3.12 build, +# uv sync --active can leave gprof2dot absent, so install the uv.lock version and verify the import. +uv pip install "\$ray_dependency" "gprof2dot==2025.4.14" +python -c 'import gprof2dot' ######################################## # START Benchmark specific preparation From e367f3497bd6cf7ce6af612153d3d4d033d91a0b Mon Sep 17 00:00:00 2001 From: Frankie Siino Date: Wed, 12 Aug 2026 05:45:51 +0000 Subject: [PATCH 02/26] Harden external vllm jobs Signed-off-by: Frankie Siino --- .../sbatch_external_vllm.sh | 33 +++++++++++++++---- 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/benchmarks/nemotron_3.5_super/sbatch_external_vllm.sh b/benchmarks/nemotron_3.5_super/sbatch_external_vllm.sh index 0f10064093a..a1864535c88 100644 --- a/benchmarks/nemotron_3.5_super/sbatch_external_vllm.sh +++ b/benchmarks/nemotron_3.5_super/sbatch_external_vllm.sh @@ -9,6 +9,12 @@ MODEL=$MODEL CONTAINER=$CONTAINER MOUNTS=$MOUNTS VLLM_CONFIG=$VLLM_CONFIG +SBATCH_EXCLUDE=${SBATCH_EXCLUDE:-} + +SBATCH_EXCLUDE_ARGS=() +if [[ -n "$SBATCH_EXCLUDE" ]]; then + SBATCH_EXCLUDE_ARGS+=(--exclude="$SBATCH_EXCLUDE") +fi should_run_eval=$(( $# > 0 )) if (( should_run_eval )); then @@ -106,6 +112,23 @@ export UCX_RNDV_THRESH=0 source "$VLLM_CONFIG" +wait_for_vllm_health() { + local role=\$1 + local url=\$2 + local pid=\$3 + + until curl -fs "\$url" >/dev/null; do + if ! kill -0 "\$pid" 2>/dev/null; then + local status=0 + wait "\$pid" || status=\$? + (( status != 0 )) || status=1 + echo "ERROR: \$role vLLM process exited before becoming healthy (status=\$status)." >&2 + return "\$status" + fi + sleep 5 + done +} + this_node_hostname=\$(hostname) # Split nodes here by index if (( SLURM_PROCID == 0 )); then @@ -126,12 +149,8 @@ if (( SLURM_PROCID == 0 )); then prefill_pid=\$! trap 'kill "\$prefill_pid" 2>/dev/null || true' EXIT - until curl -fs "http://\$PREFILL_HEAD:$PREFILL_SERVER_PORT/health" >/dev/null; do - sleep 5 - done - until curl -fs "http://\$DECODE_HEAD:$DECODE_SERVER_PORT/health" >/dev/null; do - sleep 5 - done + wait_for_vllm_health "prefill" "http://\$PREFILL_HEAD:$PREFILL_SERVER_PORT/health" "\$prefill_pid" + wait_for_vllm_health "decode" "http://\$DECODE_HEAD:$DECODE_SERVER_PORT/health" "\$prefill_pid" # --intra-node-data-parallel-size must match the data-parallel-size-local above. # Set a super long request timeout since some reasoning requests may take a long time to generate. @@ -201,6 +220,7 @@ DECODE_HEAD="\${nodes[$NUM_PREFILL_NODES]}" PREFILL_HEAD="\$PREFILL_HEAD" \ DECODE_HEAD="\$DECODE_HEAD" \ srun --nodes=$NUM_NODES --ntasks=$NUM_NODES --ntasks-per-node=1 \ + --kill-on-bad-exit=1 \ --container-image=$CONTAINER \ --container-name=container-on-node \ --container-mounts=$MOUNTS \ @@ -257,6 +277,7 @@ VLLM_PD_BATCH_COMMAND="$batch_command" \ sbatch \ --nodes=$NUM_NODES \ --time=04:00:00 \ + "${SBATCH_EXCLUDE_ARGS[@]}" \ --job-name=gym-$EXPERIMENT_NAME-$USER \ --output=slurm-logs/%j-%x.log \ --ntasks-per-node=1 \ From 7c5ecb919b104eda6dae7c91b66cfd17df128e48 Mon Sep 17 00:00:00 2001 From: Frankie Siino Date: Wed, 12 Aug 2026 05:46:26 +0000 Subject: [PATCH 03/26] Init Qwen config Signed-off-by: Frankie Siino --- .../vllm_configs/qwen3.5-122b-a10b.sh | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 benchmarks/nemotron_3.5_super/vllm_configs/qwen3.5-122b-a10b.sh diff --git a/benchmarks/nemotron_3.5_super/vllm_configs/qwen3.5-122b-a10b.sh b/benchmarks/nemotron_3.5_super/vllm_configs/qwen3.5-122b-a10b.sh new file mode 100644 index 00000000000..4bbea1ed33d --- /dev/null +++ b/benchmarks/nemotron_3.5_super/vllm_configs/qwen3.5-122b-a10b.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +VLLM_COMMON_ARGS=( + --trust-remote-code + --gpu-memory-utilization 0.9 + --distributed-executor-backend mp + --data-parallel-backend mp + --data-parallel-size-local 1 + --tensor-parallel-size 4 + --max-model-len 262144 + --enable-auto-tool-choice + --tool-call-parser qwen3_coder + --reasoning-parser qwen3 + --mm-encoder-tp-mode data + --enable-chunked-prefill + --enable-prefix-caching + --kv-cache-dtype fp8 + --no-disable-hybrid-kv-cache-manager + --no-async-scheduling + --block-size 128 + --mamba-cache-mode align + --mamba-ssm-cache-dtype float32 + --model-loader-extra-config '{"enable_multithread_load": true, "num_threads": 96}' + --enable-expert-parallel + --max-num-batched-tokens 33920 + --max-num-seqs 512 +) From dbd666d0080f0834a9b78e0e9bd0340b30a49f79 Mon Sep 17 00:00:00 2001 From: Frankie Siino Date: Wed, 12 Aug 2026 06:42:31 +0000 Subject: [PATCH 04/26] Fix concurrent Tau2 data setup Signed-off-by: Frankie Siino --- responses_api_agents/tau2/source.py | 85 +++++++++++++++---- .../tau2/tests/test_source.py | 57 +++++++++++++ 2 files changed, 125 insertions(+), 17 deletions(-) create mode 100644 responses_api_agents/tau2/tests/test_source.py diff --git a/responses_api_agents/tau2/source.py b/responses_api_agents/tau2/source.py index 11311eb4359..57aea9ec8f6 100644 --- a/responses_api_agents/tau2/source.py +++ b/responses_api_agents/tau2/source.py @@ -16,6 +16,10 @@ import os import shutil +import time +import uuid +from collections.abc import Iterator +from contextlib import contextmanager from pathlib import Path from subprocess import run from tempfile import TemporaryDirectory @@ -26,6 +30,9 @@ DEFAULT_TAU2_BENCH_REPO_URL = "https://github.com/bxyu-nvidia/tau2-bench" DEFAULT_TAU2_BENCH_REF = "bxyu/nemo_gym_stable" +TAU2_DATA_LOCK_POLL_INTERVAL_SECONDS = 5.0 +TAU2_DATA_LOCK_STALE_SECONDS = 3600.0 +TAU2_DATA_LOCK_TIMEOUT_SECONDS = 3600.0 def get_tau2_bench_repo_url() -> str: @@ -57,6 +64,49 @@ def _ignore_generated_tau2_data(directory: str, names: list[str]) -> set[str]: return set() +@contextmanager +def _tau2_data_lock(data_dir: Path) -> Iterator[None]: + """Serialize Tau2 data setup across processes and nodes sharing a filesystem.""" + + lock_path = data_dir.parent / f".{data_dir.name}.lockdir" + owner_path = lock_path / "owner" + owner = uuid.uuid4().hex + started_at = time.monotonic() + + while True: + try: + lock_path.mkdir(exist_ok=False) + try: + owner_path.write_text(owner) + except OSError: + shutil.rmtree(lock_path, ignore_errors=True) + raise + break + except FileExistsError: + try: + lock_age = time.time() - lock_path.stat().st_mtime + except FileNotFoundError: + continue + + if lock_age > TAU2_DATA_LOCK_STALE_SECONDS: + shutil.rmtree(lock_path, ignore_errors=True) + continue + + if time.monotonic() - started_at >= TAU2_DATA_LOCK_TIMEOUT_SECONDS: + raise TimeoutError(f"Timed out waiting for Tau2 data setup lock at {lock_path}") + time.sleep(TAU2_DATA_LOCK_POLL_INTERVAL_SECONDS) + + try: + yield + finally: + try: + is_owner = owner_path.read_text() == owner + except FileNotFoundError: + is_owner = False + if is_owner: + shutil.rmtree(lock_path, ignore_errors=True) + + def ensure_tau2_data_dir(data_dir: Path) -> Path: """Ensure ``data_dir`` contains Tau data, including Tau3 banking_knowledge. @@ -66,27 +116,28 @@ def ensure_tau2_data_dir(data_dir: Path) -> Path: ``NEMO_GYM_TAU2_BENCH_REF``. """ - if _has_banking_knowledge_data(data_dir): - return data_dir - data_dir.parent.mkdir(parents=True, exist_ok=True) - repo_url = get_tau2_bench_repo_url() - ref = get_tau2_bench_ref() + with _tau2_data_lock(data_dir): + if _has_banking_knowledge_data(data_dir): + return data_dir + + repo_url = get_tau2_bench_repo_url() + ref = get_tau2_bench_ref() - with TemporaryDirectory(prefix="tau2-bench-", dir=data_dir.parent) as tmp_dir: - checkout_dir = Path(tmp_dir) / "tau2-bench" - run(["git", "clone", repo_url, str(checkout_dir)], check=True) - run(["git", "-C", str(checkout_dir), "checkout", ref], check=True) + with TemporaryDirectory(prefix="tau2-bench-", dir=data_dir.parent) as tmp_dir: + checkout_dir = Path(tmp_dir) / "tau2-bench" + run(["git", "clone", repo_url, str(checkout_dir)], check=True) + run(["git", "-C", str(checkout_dir), "checkout", ref], check=True) - source_data_dir = checkout_dir / "data" - if not source_data_dir.exists(): - raise RuntimeError(f"Tau2 data directory not found at {source_data_dir}") + source_data_dir = checkout_dir / "data" + if not source_data_dir.exists(): + raise RuntimeError(f"Tau2 data directory not found at {source_data_dir}") - if data_dir.exists(): - shutil.rmtree(data_dir) - shutil.copytree(source_data_dir, data_dir, ignore=_ignore_generated_tau2_data) + if data_dir.exists(): + shutil.rmtree(data_dir) + shutil.copytree(source_data_dir, data_dir, ignore=_ignore_generated_tau2_data) - if not _has_banking_knowledge_data(data_dir): - raise RuntimeError(f"Tau2 data copied from {repo_url}@{ref} is missing banking_knowledge assets") + if not _has_banking_knowledge_data(data_dir): + raise RuntimeError(f"Tau2 data copied from {repo_url}@{ref} is missing banking_knowledge assets") return data_dir diff --git a/responses_api_agents/tau2/tests/test_source.py b/responses_api_agents/tau2/tests/test_source.py new file mode 100644 index 00000000000..9cc5a3750e0 --- /dev/null +++ b/responses_api_agents/tau2/tests/test_source.py @@ -0,0 +1,57 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import threading +import time +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from unittest.mock import patch + +from responses_api_agents.tau2 import source + + +def _write_tau2_source_data(data_dir: Path) -> None: + banking_dir = data_dir / "tau2" / "domains" / "banking_knowledge" + banking_dir.mkdir(parents=True) + (banking_dir / "db.json").write_text("{}") + for directory in ("documents", "prompts", "tasks"): + (banking_dir / directory).mkdir() + + +def test_concurrent_initialization_fetches_tau2_data_once(tmp_path: Path) -> None: + data_dir = tmp_path / "tau2_data" + partial_banking_dir = data_dir / "tau2" / "domains" / "banking_knowledge" + (partial_banking_dir / "documents").mkdir(parents=True) + (partial_banking_dir / "tasks").mkdir() + + start_barrier = threading.Barrier(2) + clone_count = 0 + clone_count_lock = threading.Lock() + + def fake_run(command: list[str], *, check: bool) -> None: + nonlocal clone_count + assert check + if command[1] != "clone": + return + + with clone_count_lock: + clone_count += 1 + time.sleep(0.1) + _write_tau2_source_data(Path(command[-1]) / "data") + + def initialize() -> Path: + start_barrier.wait() + return source.ensure_tau2_data_dir(data_dir) + + with ( + patch.object(source, "run", side_effect=fake_run), + patch.object(source, "TAU2_DATA_LOCK_POLL_INTERVAL_SECONDS", 0.01), + ThreadPoolExecutor(max_workers=2) as executor, + ): + futures = [executor.submit(initialize) for _ in range(2)] + results = [future.result() for future in futures] + + assert results == [data_dir, data_dir] + assert clone_count == 1 + assert source._has_banking_knowledge_data(data_dir) + assert not (tmp_path / ".tau2_data.lockdir").exists() From 900d18174448d7853eb1db7e160e96edc886a460 Mon Sep 17 00:00:00 2001 From: Frankie Siino Date: Wed, 12 Aug 2026 11:19:52 -0700 Subject: [PATCH 05/26] Fail fast during Super 3.5 eval Signed-off-by: Frankie Siino --- benchmarks/nemotron_3.5_super/sbatch_external_vllm.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/benchmarks/nemotron_3.5_super/sbatch_external_vllm.sh b/benchmarks/nemotron_3.5_super/sbatch_external_vllm.sh index a1864535c88..1999786985f 100644 --- a/benchmarks/nemotron_3.5_super/sbatch_external_vllm.sh +++ b/benchmarks/nemotron_3.5_super/sbatch_external_vllm.sh @@ -42,6 +42,8 @@ PREFILL_DP_RPC_PORT=13345 DECODE_DP_RPC_PORT=13346 EVAL_COMMAND=$(cat < Date: Wed, 12 Aug 2026 11:37:23 -0700 Subject: [PATCH 06/26] Make Super 3.5 vllm launcher portable Signed-off-by: Frankie Siino --- .../build-super-vl-rl-v0251-thin.sh | 6 +-- .../sbatch_external_vllm.sh | 37 ++++++++++++++++--- 2 files changed, 34 insertions(+), 9 deletions(-) diff --git a/benchmarks/nemotron_3.5_super/build-super-vl-rl-v0251-thin.sh b/benchmarks/nemotron_3.5_super/build-super-vl-rl-v0251-thin.sh index 00c56001a74..5c794b46391 100644 --- a/benchmarks/nemotron_3.5_super/build-super-vl-rl-v0251-thin.sh +++ b/benchmarks/nemotron_3.5_super/build-super-vl-rl-v0251-thin.sh @@ -1,8 +1,8 @@ #!/usr/bin/env bash -# Canonical Super VL RL v0.25.1 SQSH build for GB200/B200. +# Canonical Super VL RL v0.25.1 SQSH build for Blackwell GPUs. # # Usage: -# SLURM_ACCOUNT= ./build-super-vl-rl-v0251-thin.sh /path/image.sqsh +# CUDA_ARCH= SLURM_ACCOUNT= ./build-super-vl-rl-v0251-thin.sh /path/image.sqsh # # The build always resolves the latest heads of the team release branches, # verifies the published TRTLLM-GEN cubin manifest, and rebuilds the small @@ -21,7 +21,7 @@ export FMHA_MANIFEST_SHA256="${FMHA_MANIFEST_SHA256:-03e0f29f970de40b0fd3c6025a1 export FMHA_MANIFEST_ENTRIES="${FMHA_MANIFEST_ENTRIES:-19227}" VLLM_VERSION=0.25.1 # setuptools-scm cannot infer a version from a branch clone -CUDA_ARCH=10.0a # Blackwell (GB200/B200); 9.0a for H100 +export CUDA_ARCH="${CUDA_ARCH:-10.0a}" # 10.0a for GB200/B200; 10.3a for GB300; 9.0a for H100 BUILD_ROOT=/opt/super-vl-rl ############################################################################### diff --git a/benchmarks/nemotron_3.5_super/sbatch_external_vllm.sh b/benchmarks/nemotron_3.5_super/sbatch_external_vllm.sh index 1999786985f..0bcb6df293b 100644 --- a/benchmarks/nemotron_3.5_super/sbatch_external_vllm.sh +++ b/benchmarks/nemotron_3.5_super/sbatch_external_vllm.sh @@ -10,6 +10,10 @@ CONTAINER=$CONTAINER MOUNTS=$MOUNTS VLLM_CONFIG=$VLLM_CONFIG SBATCH_EXCLUDE=${SBATCH_EXCLUDE:-} +SBATCH_TIME=${SBATCH_TIME:-04:00:00} +# auto uses NUM_NODES when sbatch advertises --segment; none omits it; a positive integer overrides it. +VLLM_SLURM_SEGMENT=${VLLM_SLURM_SEGMENT:-auto} +export UCX_NET_DEVICES="${UCX_NET_DEVICES:-mlx5_0:1}" SBATCH_EXCLUDE_ARGS=() if [[ -n "$SBATCH_EXCLUDE" ]]; then @@ -104,10 +108,10 @@ export VLLM_SSM_CONV_STATE_LAYOUT=DS export VLLM_USE_FASTOKENS=1 # NIXL uses UCX for cross-node KV transfer. Explicitly enable UCX's CUDA -# transports and the GB200 InfiniBand interface; otherwise UCX treats VRAM as -# host memory and NIXL KV-cache registration fails with NIXL_ERR_BACKEND. +# transports and the selected InfiniBand/RoCE interface; otherwise UCX treats +# VRAM as host memory and NIXL KV-cache registration fails with NIXL_ERR_BACKEND. export UCX_TLS=rc_x,rc,cuda_copy,cuda_ipc -export UCX_NET_DEVICES=mlx5_0:1 +export UCX_NET_DEVICES="\${UCX_NET_DEVICES:-mlx5_0:1}" export UCX_IB_ADDR_TYPE=eth export UCX_RNDV_SCHEME=get_zcopy export UCX_RNDV_THRESH=0 @@ -212,6 +216,25 @@ EOF ) NUM_NODES=$((NUM_PREFILL_NODES + NUM_DECODE_NODES)) +SBATCH_SEGMENT_ARGS=() +case "$VLLM_SLURM_SEGMENT" in + auto) + sbatch_help=$(sbatch --help 2>&1) + if [[ "$sbatch_help" == *"--segment"* ]]; then + SBATCH_SEGMENT_ARGS+=(--segment="$NUM_NODES") + fi + ;; + none) + ;; + *) + if [[ ! "$VLLM_SLURM_SEGMENT" =~ ^[1-9][0-9]*$ ]]; then + echo "ERROR: VLLM_SLURM_SEGMENT must be auto, none, or a positive integer." >&2 + exit 2 + fi + SBATCH_SEGMENT_ARGS+=(--segment="$VLLM_SLURM_SEGMENT") + ;; +esac + batch_command=$(cat < 0 otherwise the engine will hang on the second or third engine step. +# Some Slurm deployments need --segment > 0 to avoid engine hangs, while +# upstream Slurm does not provide the option. Auto-detection preserves that +# behavior and omits the argument when the local sbatch does not advertise it. VLLM_PD_WORKLOAD="$command" \ EVAL_COMMAND="$EVAL_COMMAND" \ VLLM_PD_BATCH_COMMAND="$batch_command" \ sbatch \ --nodes=$NUM_NODES \ - --time=04:00:00 \ + --time="$SBATCH_TIME" \ "${SBATCH_EXCLUDE_ARGS[@]}" \ + "${SBATCH_SEGMENT_ARGS[@]}" \ --job-name=gym-$EXPERIMENT_NAME-$USER \ --output=slurm-logs/%j-%x.log \ --ntasks-per-node=1 \ --exclusive \ - --segment=$NUM_NODES \ --wrap 'exec bash -lc "$VLLM_PD_BATCH_COMMAND"' From 474b4efd657e54c14d40a7424bbae403b7f101e2 Mon Sep 17 00:00:00 2001 From: Frankie Siino Date: Thu, 13 Aug 2026 06:17:27 +0000 Subject: [PATCH 07/26] Stabilize Qwen3.5 vLLM serving under load Disable prefix caching, reduce the maximum sequence count from 512 to 256, match CUDA-graph capture to 256, and enforce eager execution to isolate the decode stalls observed during full-suite evaluation. Signed-off-by: Frankie Siino --- .../nemotron_3.5_super/vllm_configs/qwen3.5-122b-a10b.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/benchmarks/nemotron_3.5_super/vllm_configs/qwen3.5-122b-a10b.sh b/benchmarks/nemotron_3.5_super/vllm_configs/qwen3.5-122b-a10b.sh index 4bbea1ed33d..73b0b3f7440 100644 --- a/benchmarks/nemotron_3.5_super/vllm_configs/qwen3.5-122b-a10b.sh +++ b/benchmarks/nemotron_3.5_super/vllm_configs/qwen3.5-122b-a10b.sh @@ -13,15 +13,18 @@ VLLM_COMMON_ARGS=( --reasoning-parser qwen3 --mm-encoder-tp-mode data --enable-chunked-prefill - --enable-prefix-caching + --no-enable-prefix-caching --kv-cache-dtype fp8 + --enable-expert-parallel --no-disable-hybrid-kv-cache-manager --no-async-scheduling + # Isolate compiled/CUDA-graph execution while retaining the required MoE weight layout. + --enforce-eager --block-size 128 --mamba-cache-mode align --mamba-ssm-cache-dtype float32 --model-loader-extra-config '{"enable_multithread_load": true, "num_threads": 96}' - --enable-expert-parallel --max-num-batched-tokens 33920 - --max-num-seqs 512 + --max-num-seqs 256 + --max-cudagraph-capture-size 256 ) From c11be3707c34f79a81713e0c242ccf5163713b9a Mon Sep 17 00:00:00 2001 From: Frankie Siino Date: Thu, 13 Aug 2026 18:33:59 +0000 Subject: [PATCH 08/26] Add opt-in eval concurrency and safe node exclusions Pass num_samples_in_parallel only when explicitly configured. Validate the value and filter stale exclusions against Slurm's current node inventory before submission. Signed-off-by: Frankie Siino --- .../sbatch_external_vllm.sh | 56 ++++++++++++++++++- 1 file changed, 54 insertions(+), 2 deletions(-) diff --git a/benchmarks/nemotron_3.5_super/sbatch_external_vllm.sh b/benchmarks/nemotron_3.5_super/sbatch_external_vllm.sh index 0bcb6df293b..bdce440a4a8 100644 --- a/benchmarks/nemotron_3.5_super/sbatch_external_vllm.sh +++ b/benchmarks/nemotron_3.5_super/sbatch_external_vllm.sh @@ -11,13 +11,65 @@ MOUNTS=$MOUNTS VLLM_CONFIG=$VLLM_CONFIG SBATCH_EXCLUDE=${SBATCH_EXCLUDE:-} SBATCH_TIME=${SBATCH_TIME:-04:00:00} +# Optionally bound full-rollout concurrency after measuring adapter and model-server backpressure. +# When unset, preserve Gym's existing configured/default behavior. +NUM_SAMPLES_IN_PARALLEL=${NUM_SAMPLES_IN_PARALLEL:-} +NUM_SAMPLES_IN_PARALLEL_ARG="" # auto uses NUM_NODES when sbatch advertises --segment; none omits it; a positive integer overrides it. VLLM_SLURM_SEGMENT=${VLLM_SLURM_SEGMENT:-auto} export UCX_NET_DEVICES="${UCX_NET_DEVICES:-mlx5_0:1}" +if [[ -n "$NUM_SAMPLES_IN_PARALLEL" ]]; then + if [[ ! "$NUM_SAMPLES_IN_PARALLEL" =~ ^[1-9][0-9]*$ ]]; then + echo "ERROR: NUM_SAMPLES_IN_PARALLEL must be a positive integer; got '$NUM_SAMPLES_IN_PARALLEL'." >&2 + exit 1 + fi + NUM_SAMPLES_IN_PARALLEL_ARG="++num_samples_in_parallel=$NUM_SAMPLES_IN_PARALLEL" +fi + SBATCH_EXCLUDE_ARGS=() if [[ -n "$SBATCH_EXCLUDE" ]]; then - SBATCH_EXCLUDE_ARGS+=(--exclude="$SBATCH_EXCLUDE") + # Exclusion lists can outlive cluster topology changes. Revalidate immediately before + # submission so stale node names do not make sbatch reject the entire job. + if ! requested_excludes=$(scontrol show hostnames "$SBATCH_EXCLUDE"); then + echo "ERROR: Could not expand SBATCH_EXCLUDE='$SBATCH_EXCLUDE'." >&2 + exit 1 + fi + + if ! cluster_nodes=$(scontrol show nodes --oneliner); then + echo "ERROR: Could not query Slurm's node inventory." >&2 + exit 1 + fi + + declare -A cluster_node_set=() + while IFS= read -r node_record; do + node=${node_record#NodeName=} + node=${node%% *} + [[ -n "$node" ]] && cluster_node_set["$node"]=1 + done <<< "$cluster_nodes" + + valid_excludes=() + stale_excludes=() + while IFS= read -r node; do + [[ -z "$node" ]] && continue + if [[ -n "${cluster_node_set[$node]:-}" ]]; then + valid_excludes+=("$node") + else + stale_excludes+=("$node") + fi + done <<< "$requested_excludes" + + if (( ${#stale_excludes[@]} )); then + stale_excludes_csv=$(IFS=,; echo "${stale_excludes[*]}") + echo "WARNING: Ignoring exclusions absent from Slurm's current node inventory: $stale_excludes_csv" >&2 + fi + if (( ${#valid_excludes[@]} )); then + valid_excludes_csv=$(IFS=,; echo "${valid_excludes[*]}") + echo "Using ${#valid_excludes[@]} valid node exclusions." + SBATCH_EXCLUDE_ARGS+=(--exclude="$valid_excludes_csv") + else + echo "WARNING: No requested node exclusions exist in Slurm's current node inventory." >&2 + fi fi should_run_eval=$(( $# > 0 )) @@ -72,7 +124,7 @@ gym eval run \ ++policy_base_url=http://\$(getent hosts "\$PREFILL_HEAD" | awk 'NR == 1 {print \$1}'):$ROUTER_SERVER_PORT/v1 \ ++policy_api_key=dummy_api_key \ ++policy_model_name=$MODEL \ - ++upload_rollouts_to_wandb=false \ + ++upload_rollouts_to_wandb=false $NUM_SAMPLES_IN_PARALLEL_ARG \ ++global_aiohttp_connector_limit_per_host=16384 \ ++port_range_low=63000 \ ++port_range_high=64000 From a7284080e031bc3958412a265c3b0499473429d6 Mon Sep 17 00:00:00 2001 From: Frankie Siino Date: Thu, 13 Aug 2026 18:34:10 +0000 Subject: [PATCH 09/26] Enable Qwen3.5 CUDA graphs with BF16 KV cache Remove FP8 KV caching and eager execution while retaining the 256 sequence and capture limits. Bound each request to four images and one video. Signed-off-by: Frankie Siino --- .../nemotron_3.5_super/vllm_configs/qwen3.5-122b-a10b.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/benchmarks/nemotron_3.5_super/vllm_configs/qwen3.5-122b-a10b.sh b/benchmarks/nemotron_3.5_super/vllm_configs/qwen3.5-122b-a10b.sh index 73b0b3f7440..79040a3d998 100644 --- a/benchmarks/nemotron_3.5_super/vllm_configs/qwen3.5-122b-a10b.sh +++ b/benchmarks/nemotron_3.5_super/vllm_configs/qwen3.5-122b-a10b.sh @@ -14,12 +14,9 @@ VLLM_COMMON_ARGS=( --mm-encoder-tp-mode data --enable-chunked-prefill --no-enable-prefix-caching - --kv-cache-dtype fp8 --enable-expert-parallel --no-disable-hybrid-kv-cache-manager --no-async-scheduling - # Isolate compiled/CUDA-graph execution while retaining the required MoE weight layout. - --enforce-eager --block-size 128 --mamba-cache-mode align --mamba-ssm-cache-dtype float32 @@ -27,4 +24,5 @@ VLLM_COMMON_ARGS=( --max-num-batched-tokens 33920 --max-num-seqs 256 --max-cudagraph-capture-size 256 + --limit-mm-per-prompt '{"image":4,"video":1}' ) From d25862685d7c6a3d7260f7547f464bdb84a8c0a4 Mon Sep 17 00:00:00 2001 From: Frankie Siino Date: Fri, 14 Aug 2026 03:24:24 +0000 Subject: [PATCH 10/26] Use fixed-scale FP8 KV cache for Qwen3.5 Store attention KV blocks in FP8 to reduce cache memory traffic and preserve decode throughput. Disable runtime scale calculation because Qwen3.5 hybrid-cache warmup produced invalid inferred scales. Signed-off-by: Frankie Siino --- benchmarks/nemotron_3.5_super/vllm_configs/qwen3.5-122b-a10b.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/benchmarks/nemotron_3.5_super/vllm_configs/qwen3.5-122b-a10b.sh b/benchmarks/nemotron_3.5_super/vllm_configs/qwen3.5-122b-a10b.sh index 79040a3d998..f81403f9856 100644 --- a/benchmarks/nemotron_3.5_super/vllm_configs/qwen3.5-122b-a10b.sh +++ b/benchmarks/nemotron_3.5_super/vllm_configs/qwen3.5-122b-a10b.sh @@ -14,6 +14,8 @@ VLLM_COMMON_ARGS=( --mm-encoder-tp-mode data --enable-chunked-prefill --no-enable-prefix-caching + --kv-cache-dtype fp8 + --no-calculate-kv-scales --enable-expert-parallel --no-disable-hybrid-kv-cache-manager --no-async-scheduling From 715fbda621e581ee54a0ab124163dd618cb0580b Mon Sep 17 00:00:00 2001 From: Frankie Siino Date: Fri, 14 Aug 2026 03:24:33 +0000 Subject: [PATCH 11/26] Add independent decode replicas to the vLLM launcher Allow each decode node to run a local TP/EP replica instead of forming a cross- node DP/EP group. Register every replica with the PD router and add an opt-in decode policy for balanced variable-length eval traffic. Preserve the existing coupled and consistent-hash defaults. Signed-off-by: Frankie Siino --- .../sbatch_external_vllm.sh | 67 ++++++++++++++++++- 1 file changed, 65 insertions(+), 2 deletions(-) diff --git a/benchmarks/nemotron_3.5_super/sbatch_external_vllm.sh b/benchmarks/nemotron_3.5_super/sbatch_external_vllm.sh index bdce440a4a8..726d6425170 100644 --- a/benchmarks/nemotron_3.5_super/sbatch_external_vllm.sh +++ b/benchmarks/nemotron_3.5_super/sbatch_external_vllm.sh @@ -15,6 +15,12 @@ SBATCH_TIME=${SBATCH_TIME:-04:00:00} # When unset, preserve Gym's existing configured/default behavior. NUM_SAMPLES_IN_PARALLEL=${NUM_SAMPLES_IN_PARALLEL:-} NUM_SAMPLES_IN_PARALLEL_ARG="" +# coupled preserves the existing multi-node DP/EP deployment. independent runs one +# TP/EP replica per decode node and registers every replica with the router. +VLLM_DECODE_MODE=${VLLM_DECODE_MODE:-coupled} +# Optional PD-router override for decode replicas. When unset, the main +# consistent-hash policy remains in effect for both prefill and decode. +VLLM_DECODE_POLICY=${VLLM_DECODE_POLICY:-} # auto uses NUM_NODES when sbatch advertises --segment; none omits it; a positive integer overrides it. VLLM_SLURM_SEGMENT=${VLLM_SLURM_SEGMENT:-auto} export UCX_NET_DEVICES="${UCX_NET_DEVICES:-mlx5_0:1}" @@ -27,6 +33,24 @@ if [[ -n "$NUM_SAMPLES_IN_PARALLEL" ]]; then NUM_SAMPLES_IN_PARALLEL_ARG="++num_samples_in_parallel=$NUM_SAMPLES_IN_PARALLEL" fi +case "$VLLM_DECODE_MODE" in + coupled|independent) + ;; + *) + echo "ERROR: VLLM_DECODE_MODE must be coupled or independent; got '$VLLM_DECODE_MODE'." >&2 + exit 1 + ;; +esac + +case "$VLLM_DECODE_POLICY" in + ""|random|round_robin|cache_aware|power_of_two|consistent_hash) + ;; + *) + echo "ERROR: VLLM_DECODE_POLICY must be random, round_robin, cache_aware, power_of_two, consistent_hash, or empty; got '$VLLM_DECODE_POLICY'." >&2 + exit 1 + ;; +esac + SBATCH_EXCLUDE_ARGS=() if [[ -n "$SBATCH_EXCLUDE" ]]; then # Exclusion lists can outlive cluster topology changes. Revalidate immediately before @@ -151,6 +175,8 @@ set -euo pipefail # Input arguments and validation PREFILL_HEAD=\$PREFILL_HEAD DECODE_HEAD=\$DECODE_HEAD +DECODE_HOSTS_CSV=\$DECODE_HOSTS_CSV +IFS=, read -r -a DECODE_HOSTS <<< "\$DECODE_HOSTS_CSV" # Nemotron's three-read Mamba SSM state must use the dimension-sequence layout when KV transfer is enabled. # Not used when the model has no Mamba layers. @@ -208,7 +234,25 @@ if (( SLURM_PROCID == 0 )); then trap 'kill "\$prefill_pid" 2>/dev/null || true' EXIT wait_for_vllm_health "prefill" "http://\$PREFILL_HEAD:$PREFILL_SERVER_PORT/health" "\$prefill_pid" - wait_for_vllm_health "decode" "http://\$DECODE_HEAD:$DECODE_SERVER_PORT/health" "\$prefill_pid" + + decode_router_args=() + if [[ "$VLLM_DECODE_MODE" == independent ]]; then + for decode_host in "\${DECODE_HOSTS[@]}"; do + wait_for_vllm_health \ + "decode replica \$decode_host" \ + "http://\$decode_host:$DECODE_SERVER_PORT/health" \ + "\$prefill_pid" + decode_router_args+=(--decode "http://\$decode_host:$DECODE_SERVER_PORT") + done + else + wait_for_vllm_health "decode" "http://\$DECODE_HEAD:$DECODE_SERVER_PORT/health" "\$prefill_pid" + decode_router_args+=(--decode "http://\$DECODE_HEAD:$DECODE_SERVER_PORT") + fi + + decode_policy_args=() + if [[ -n "$VLLM_DECODE_POLICY" ]]; then + decode_policy_args+=(--decode-policy "$VLLM_DECODE_POLICY") + fi # --intra-node-data-parallel-size must match the data-parallel-size-local above. # Set a super long request timeout since some reasoning requests may take a long time to generate. @@ -216,7 +260,8 @@ if (( SLURM_PROCID == 0 )); then --policy consistent_hash \ --vllm-pd-disaggregation \ --prefill http://\$PREFILL_HEAD:$PREFILL_SERVER_PORT \ - --decode http://\$DECODE_HEAD:$DECODE_SERVER_PORT \ + "\${decode_router_args[@]}" \ + "\${decode_policy_args[@]}" \ --host \$PREFILL_HEAD \ --port $ROUTER_SERVER_PORT \ --intra-node-data-parallel-size 1 \ @@ -234,6 +279,21 @@ elif (( SLURM_PROCID < $NUM_PREFILL_NODES )); then --data-parallel-rpc-port $PREFILL_DP_RPC_PORT \ --kv-transfer-config \ '{"kv_connector":"NixlConnector","kv_role":"kv_producer","kv_load_failure_policy":"fail"}' +elif [[ "$VLLM_DECODE_MODE" == independent ]]; then + # Each decode node is a complete TP/EP replica. This prevents expert-parallel + # collectives from crossing nodes while still letting the router use every replica. + VLLM_NIXL_SIDE_CHANNEL_HOST=\$this_node_hostname \ + VLLM_NIXL_SIDE_CHANNEL_PORT=$DECODE_VLLM_NIXL_SIDE_CHANNEL_PORT \ + vllm serve "$MODEL" "\${VLLM_COMMON_ARGS[@]}" \ + --host \$this_node_hostname \ + --port $DECODE_SERVER_PORT \ + --data-parallel-size 1 \ + --data-parallel-address \$this_node_hostname \ + --data-parallel-rpc-port $DECODE_DP_RPC_PORT \ + --api-server-count 1 \ + --kv-transfer-config \ + '{"kv_connector":"NixlConnector","kv_role":"kv_consumer","kv_load_failure_policy":"fail"}' \ + --compilation-config '{"cudagraph_mode":"FULL_DECODE_ONLY"}' elif (( SLURM_PROCID == NUM_PREFILL_NODES )); then # Decode head @@ -293,9 +353,12 @@ set -euo pipefail nodes=(\$(scontrol show hostnames "\$SLURM_JOB_NODELIST")) PREFILL_HEAD="\${nodes[0]}" DECODE_HEAD="\${nodes[$NUM_PREFILL_NODES]}" +decode_nodes=("\${nodes[@]:$NUM_PREFILL_NODES:$NUM_DECODE_NODES}") +DECODE_HOSTS_CSV=\$(IFS=,; echo "\${decode_nodes[*]}") PREFILL_HEAD="\$PREFILL_HEAD" \ DECODE_HEAD="\$DECODE_HEAD" \ +DECODE_HOSTS_CSV="\$DECODE_HOSTS_CSV" \ srun --nodes=$NUM_NODES --ntasks=$NUM_NODES --ntasks-per-node=1 \ --kill-on-bad-exit=1 \ --container-image=$CONTAINER \ From c4cea5bf1f45622d8e1b3dba872850be9cc46c6b Mon Sep 17 00:00:00 2001 From: Frankie Siino Date: Fri, 14 Aug 2026 03:25:02 +0000 Subject: [PATCH 12/26] Add a vLLM decode metrics summarizer Parse combined Slurm logs from coupled or independent decode deployments and report synchronized aggregate throughput, output tokens per second per request, waiting requests, and GPU KV-cache usage. Signed-off-by: Frankie Siino --- .../summarize_vllm_metrics.py | 227 ++++++++++++++++++ 1 file changed, 227 insertions(+) create mode 100644 benchmarks/nemotron_3.5_super/summarize_vllm_metrics.py diff --git a/benchmarks/nemotron_3.5_super/summarize_vllm_metrics.py b/benchmarks/nemotron_3.5_super/summarize_vllm_metrics.py new file mode 100644 index 00000000000..21ebdbfebfc --- /dev/null +++ b/benchmarks/nemotron_3.5_super/summarize_vllm_metrics.py @@ -0,0 +1,227 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Summarize decode-tier vLLM metrics from a combined Slurm log.""" + +from __future__ import annotations + +import argparse +import re +from collections import defaultdict +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path + + +API_SERVER_RE = re.compile(r"\(APIServer pid=(?P\d+)\).*?non-default args: (?P.*)") +PORT_RE = re.compile(r"'port': (?P\d+)") +DP_SIZE_RE = re.compile(r"'data_parallel_size': (?P\d+)") +KV_ROLE_RE = re.compile(r"kv_role='(?P[^']+)'") +METRIC_RE = re.compile( + r"\(APIServer pid=(?P\d+)\).*?INFO " + r"(?P\d{2}-\d{2}) (?P