test: validate upstream-native vLLM Router topologies

.github/workflows/e2e-tests.yml

@@ -261,6 +261,7 @@ jobs:
                   MULTI_AGENTIC_EVAL=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if x.get('scenario-type') == 'agentic-coding' and 'prefill' in x and x.get('run-eval', False)]))" | score_matrix multi-agentic-eval)
                   SINGLE=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' not in x and x.get('scenario-type') != 'agentic-coding' and not x.get('eval-only', False)]))" | score_matrix single)
                   MULTI=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' in x and x.get('scenario-type') != 'agentic-coding' and not x.get('eval-only', False)]))" | score_matrix multi)
+                  EVALS=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' not in x and x.get('scenario-type') != 'agentic-coding' and x.get('run-eval', False)]))" | score_matrix eval)
                   MULTI_EVAL=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' in x and x.get('scenario-type') != 'agentic-coding' and x.get('run-eval', False)]))" | score_matrix multi-eval)
                   {
                     echo "agentic-config=$AGENTIC"

.github/workflows/run-sweep.yml

@@ -249,7 +249,7 @@ jobs:
                       --model 'claude-fable-5'
                       --max-turns 8
                       --allowedTools "Read,Glob,Grep,Bash(git diff:*)"
-                      --json-schema '{"type":"object","properties":{"criteria":{"type":"array","items":{"type":"string","enum":["multi-node","agentic","eval-only","fp4","mtp","eagle","eagle3","sglang","vllm","dynamo-sglang","dynamo-vllm","glm5","glm5.1","kimik2.5","kimik3","dsv4","minimaxm3","qwen3.5","dsr1","checklist-complete","patchwork"]},"uniqueItems":true},"reason":{"type":"string"}},"required":["criteria","reason"]}'
+                      --json-schema '{"type":"object","properties":{"criteria":{"type":"array","items":{"type":"string","enum":["multi-node","agentic","eval-only","fp4","mtp","eagle","eagle3","sglang","vllm","vllm-router","dynamo-sglang","dynamo-vllm","glm5","glm5.1","kimik2.5","kimik3","dsv4","minimaxm3","qwen3.5","dsr1","checklist-complete","patchwork"]},"uniqueItems":true},"reason":{"type":"string"}},"required":["criteria","reason"]}'
                   prompt: |
                       Inspect this Actions run's change range.
 
@@ -264,7 +264,7 @@ jobs:
                       - eval-only: evaluation without throughput measurement
                       - fp4: FP4 precision
                       - mtp, eagle, eagle3: speculative decoding method
-                      - sglang, vllm, dynamo-vllm: runtime framework
+                      - sglang, vllm, vllm-router, dynamo-vllm: runtime framework
                       - model criterion: matching configured model family
                       - checklist-complete: PR checklist is satisfied
                       - patchwork: modified upstream engine or runtime source

benchmarks/benchmark_lib.sh

@@ -67,8 +67,10 @@ require_agentic_kv_offload_backend() {
     esac
 }
 
-# Agentic replays must use the model's native context limit. Ignore inherited
-# workflow or shell overrides so neither the server nor AIPerf applies a cap.
+# Agentic replays normally use the model's native context limit. Ignore the
+# workflow's generated MAX_MODEL_LEN so it cannot accidentally cap a native
+# AgentX recipe. Recipes that intentionally serve below the native limit can
+# opt in with AIPERF_MAX_CONTEXT_LENGTH.
 _benchmark_caller="${BASH_SOURCE[1]:-}"
 if [[ "$_benchmark_caller" == */agentic/* ||
       "$_benchmark_caller" == */agentic_*.sh ||
@@ -1973,6 +1975,7 @@ build_replay_cmd() {
     local result_dir="$1"
     local duration="$DURATION"
     local warmup_requests_per_lane="${AIPERF_WARMUP_REQUESTS_PER_LANE:-10}"
+    local max_context_length="${AIPERF_MAX_CONTEXT_LENGTH:-${MAX_MODEL_LEN:-}}"
 
     # Fast mode minimizes setup by advancing each trajectory lane only once
     # and shortens profiling to 20 minutes.
@@ -2092,8 +2095,12 @@ build_replay_cmd() {
     # server. The WEKA corpus contains a few very long parent/subagent traces;
     # if we mmap and replay them against a smaller-context server they become
     # deterministic 4xxs and can still pressure the engine while queued.
-    if [ -n "${MAX_MODEL_LEN:-}" ] && [ "$MAX_MODEL_LEN" != "0" ]; then
-        REPLAY_CMD+=" --max-context-length $MAX_MODEL_LEN"
+    if [ -n "$max_context_length" ] && [ "$max_context_length" != "0" ]; then
+        if ! [[ "$max_context_length" =~ ^[1-9][0-9]*$ ]]; then
+            echo "Error: max context length must be a positive integer, got '$max_context_length'" >&2
+            return 1
+        fi
+        REPLAY_CMD+=" --max-context-length $max_context_length"
     fi
     # Default --num-dataset-entries is 100; the with-subagents Weka corpus
     # has 393. Cap at 393 so all unique traces are loaded (the loader treats

benchmarks/multi_node/srt-slurm-recipes/configs/native-vllm-router-deps.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+python3 -m pip install --no-cache-dir --upgrade "vllm-router==0.1.15"
+command -v vllm-router >/dev/null
+python3 - <<'PY'
+from importlib.metadata import version
+
+assert version("vllm-router") == "0.1.15"
+print(f"vllm-router {version('vllm-router')}")
+PY

benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp4/agentic/agg-gb200-2x-tp2ep2-router-regression.yaml

@@ -0,0 +1,87 @@
+name: qwen35-gb200-sglang-router-regression-agg-2x-tp2ep2
+
+model: { path: qwen3.5-fp4, container: lmsysorg/sglang:nightly-dev-20260818-c0b6474b, precision: fp4 }
+identity:
+  model: { repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 }
+  container: { image: lmsysorg/sglang:nightly-dev-20260818-c0b6474b }
+slurm: { time_limit: "2:00:00" }
+health_check: { max_attempts: 720, interval_seconds: 10 }
+resources: { gpu_type: gb200, gpus_per_node: 4, agg_nodes: 1, agg_workers: 2, gpus_per_agg: 2 }
+infra: { nats_max_payload_mb: 8 }
+frontend:
+  type: sglang
+  args:
+    worker-startup-timeout-secs: 3600
+
+backend:
+  type: sglang
+  aggregated_environment:
+    PYTHONNOUSERSITE: "1"
+    NCCL_CUMEM_ENABLE: "1"
+    NCCL_MNNVL_ENABLE: "1"
+    NCCL_NVLS_ENABLE: "1"
+    SGLANG_ENABLE_FLASHINFER_GEMM: "true"
+    SGLANG_ENABLE_SPEC_V2: "1"
+    SGL_ENABLE_JIT_DEEPGEMM: "false"
+    TORCH_CUDA_ARCH_LIST: "10.0"
+  sglang_config:
+    aggregated:
+      served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4-V2
+      model-path: /model/
+      trust-remote-code: true
+      tensor-parallel-size: 2
+      data-parallel-size: 1
+      expert-parallel-size: 2
+      enable-dp-attention: false
+      enable-dp-lm-head: false
+      enable-symm-mem: false
+      quantization: modelopt_fp4
+      fp4-gemm-backend: flashinfer_cutlass
+      kv-cache-dtype: fp8_e4m3
+      mamba-ssm-dtype: bfloat16
+      mamba-radix-cache-strategy: extra_buffer_lazy
+      mamba-track-interval: 1048576
+      attention-backend: trtllm_mha
+      linear-attn-prefill-backend: flashinfer
+      linear-attn-decode-backend: flashinfer
+      moe-runner-backend: flashinfer_trtllm
+      speculative-moe-runner-backend: flashinfer_trtllm
+      speculative-algorithm: NEXTN
+      speculative-num-steps: 3
+      speculative-eagle-topk: 1
+      speculative-num-draft-tokens: 4
+      enable-linear-replayssm-spec: true
+      linear-replayssm-cache-len: 8
+      cuda-graph-max-bs: 64
+      max-running-requests: 80
+      max-prefill-tokens: 16384
+      chunked-prefill-size: 16384
+      mem-fraction-static: 0.85
+      max-mamba-cache-size: 320
+      allow-auto-truncate: true
+      stream-interval: 50
+      scheduler-recv-interval: 10
+      mamba-max-states-per-path: 1
+      weight-loader-prefetch-checkpoints: true
+      weight-loader-prefetch-num-threads: 4
+      weight-loader-drop-cache-after-load: true
+      enable-metrics: true
+      enable-cache-report: true
+
+sbatch_directives: { mem: "0", cpus-per-task: "144" }
+srun_options: { mem: "0", container-remap-root: "" }
+benchmark:
+  type: custom
+  command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
+  env:
+    INFMAX_CONTAINER_WORKSPACE: /infmax-workspace
+    RESULT_DIR: /logs/agentic
+    PORT: "8000"
+    IS_MULTINODE: "false"
+    TP: "2"
+    AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true"
+    AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0"
+    WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k
+    AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache
+    HF_HUB_CACHE: /hf_hub_cache
+    AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:"

benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp4/agentic/disagg-gb200-1p1d-tp2ep2-router-regression.yaml

@@ -0,0 +1,102 @@
+name: qwen35-gb200-sglang-router-regression-disagg-1p1d-tp2ep2
+
+model: { path: qwen3.5-fp4, container: lmsysorg/sglang:nightly-dev-20260818-c0b6474b, precision: fp4 }
+identity:
+  model: { repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 }
+  container: { image: lmsysorg/sglang:nightly-dev-20260818-c0b6474b }
+slurm: { time_limit: "2:00:00" }
+health_check: { max_attempts: 720, interval_seconds: 10 }
+resources:
+  gpu_type: gb200
+  gpus_per_node: 4
+  prefill_nodes: 1
+  prefill_workers: 1
+  gpus_per_prefill: 2
+  decode_nodes: 0
+  decode_workers: 1
+  gpus_per_decode: 2
+infra: { nats_max_payload_mb: 8 }
+frontend:
+  type: sglang
+  args:
+    worker-startup-timeout-secs: 3600
+
+backend:
+  type: sglang
+  prefill_environment: &worker_environment
+    PYTHONNOUSERSITE: "1"
+    PYTHONUNBUFFERED: "1"
+    NCCL_CUMEM_ENABLE: "1"
+    NCCL_MNNVL_ENABLE: "1"
+    NCCL_NVLS_ENABLE: "1"
+    SGLANG_ENABLE_FLASHINFER_GEMM: "true"
+    SGLANG_ENABLE_SPEC_V2: "1"
+    SGL_ENABLE_JIT_DEEPGEMM: "false"
+    SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000"
+    SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000"
+    SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000"
+    SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0"
+    TORCH_CUDA_ARCH_LIST: "10.0"
+  decode_environment: *worker_environment
+  sglang_config:
+    prefill: &worker_config
+      served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4-V2
+      model-path: /model/
+      trust-remote-code: true
+      tensor-parallel-size: 2
+      data-parallel-size: 1
+      expert-parallel-size: 2
+      enable-dp-attention: false
+      enable-dp-lm-head: false
+      enable-symm-mem: false
+      quantization: modelopt_fp4
+      fp4-gemm-backend: flashinfer_cutlass
+      kv-cache-dtype: fp8_e4m3
+      mamba-ssm-dtype: bfloat16
+      attention-backend: trtllm_mha
+      linear-attn-prefill-backend: flashinfer
+      linear-attn-decode-backend: flashinfer
+      moe-runner-backend: flashinfer_trtllm
+      speculative-moe-runner-backend: flashinfer_trtllm
+      speculative-algorithm: NEXTN
+      speculative-num-steps: 3
+      speculative-eagle-topk: 1
+      speculative-num-draft-tokens: 4
+      cuda-graph-max-bs: 64
+      max-running-requests: 80
+      max-prefill-tokens: 16384
+      chunked-prefill-size: 16384
+      mem-fraction-static: 0.80
+      max-mamba-cache-size: 320
+      allow-auto-truncate: true
+      stream-interval: 50
+      scheduler-recv-interval: 10
+      mamba-max-states-per-path: 1
+      weight-loader-prefetch-checkpoints: true
+      weight-loader-prefetch-num-threads: 4
+      weight-loader-drop-cache-after-load: true
+      disaggregation-transfer-backend: nixl
+      enable-metrics: true
+      enable-cache-report: true
+    decode:
+      <<: *worker_config
+      disable-radix-cache: true
+      mem-fraction-static: 0.80
+
+sbatch_directives: { mem: "0", cpus-per-task: "144" }
+srun_options: { mem: "0", container-remap-root: "" }
+benchmark:
+  type: custom
+  command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
+  env:
+    INFMAX_CONTAINER_WORKSPACE: /infmax-workspace
+    RESULT_DIR: /logs/agentic
+    PORT: "8000"
+    IS_MULTINODE: "true"
+    TP: "2"
+    AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true"
+    AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0"
+    WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k
+    AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache
+    HF_HUB_CACHE: /hf_hub_cache
+    AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:"

benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-r1/agentic/agg-gb200-dep4-vllm-router-agentic.yaml

@@ -0,0 +1,82 @@
+name: "dsr1-gb200-vllm-router-agg-dep4-agentic"
+
+model:
+  path: "dsr1-nvfp4"
+  container: "vllm/vllm-openai:v0.25.1"
+  precision: "fp4"
+
+setup_script: native-vllm-router-deps.sh
+
+slurm:
+  time_limit: "4:00:00"
+
+health_check:
+  max_attempts: 720
+  interval_seconds: 10
+
+resources:
+  gpu_type: "gb200"
+  gpus_per_node: 4
+  agg_nodes: 1
+  agg_workers: 1
+  gpus_per_agg: 4
+
+infra:
+  etcd_nats_dedicated_node: false
+
+frontend:
+  type: vllm-router
+  enable_multiple_frontends: false
+  args:
+    policy: consistent_hash
+
+backend:
+  type: vllm
+  connector: null
+  dp_launch_mode: per_node
+  aggregated_environment:
+    VLLM_ENGINE_READY_TIMEOUT_S: "3600"
+    VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800"
+    VLLM_LOG_STATS_INTERVAL: "1"
+    VLLM_USE_V2_MODEL_RUNNER: "1"
+    VLLM_USE_FLASHINFER_MOE_FP4: "1"
+  vllm_config:
+    aggregated:
+      served-model-name: "nvidia/DeepSeek-R1-0528-NVFP4-v2"
+      kv-cache-dtype: "fp8"
+      tensor-parallel-size: 1
+      pipeline-parallel-size: 1
+      data-parallel-size: 4
+      data-parallel-rpc-port: 13345
+      enable-expert-parallel: true
+      enable-prefix-caching: true
+      enable-prompt-tokens-details: true
+      max-model-len: 163840
+      max-num-seqs: 128
+      max-num-batched-tokens: 8192
+      trust-remote-code: true
+      block-size: 128
+      gpu-memory-utilization: 0.85
+      stream-interval: 10
+
+sbatch_directives:
+  cpus-per-task: "144"
+  mem: "0"
+
+srun_options:
+  container-remap-root: ""
+
+benchmark:
+  type: custom
+  command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
+  env:
+    INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace"
+    RESULT_DIR: "/logs/agentic"
+    PORT: "8000"
+    IS_MULTINODE: "true"
+    AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:"
+    AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: "1"
+    AIPERF_MAX_CONTEXT_LENGTH: "163840"
+    AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache"
+    HF_HUB_CACHE: "/hf_hub_cache"
+    WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126_256k"