diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-kvoffload.yaml index 3ff9e31924..d40942f792 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-kvoffload.yaml @@ -87,6 +87,7 @@ backend: sglang_config: aggregated: served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + enable-metrics: true trust-remote-code: true stream-interval: 50 watchdog-timeout: 1000000 @@ -139,11 +140,10 @@ benchmark: # in process_agentic_result.py instead. TP must match sglang_config tp-size. IS_MULTINODE: "false" TP: "4" - # Enable dynamo conv-aware routing (per-session affinity so multi-turn - # KV cache is reused across turns). Previously opted out because the - # frontend 400'd on aiperf's nvext.session_control actions; re-enabled - # to test with the current build. - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "1" + # Bind each conversation through the supported Dynamo session header. + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" # Container-side path of the aiperf mmap dataset cache; host-side mount # wired via launch_gb300-*.sh srtslurm.yaml default_mounts. Without it, # aiperf re-tokenizes + re-writes the dataset mmap on every run. @@ -151,4 +151,3 @@ benchmark: # Persistent HF hub cache (also via default_mounts) so the trace dataset # isn't re-downloaded each run; overrides the workflow-level HF_HUB_CACHE. HF_HUB_CACHE: "/hf_hub_cache" - diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-12p4d-dep8-dep16-c1024-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-12p4d-dep8-dep16-c1024-mtp-kvoffload.yaml index 715b0c631e..7ed2dd5d2b 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-12p4d-dep8-dep16-c1024-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-12p4d-dep8-dep16-c1024-mtp-kvoffload.yaml @@ -46,6 +46,9 @@ frontend: num_additional_frontends: 4 env: DYN_ROUTER_TEMPERATURE: "10000000" + # AgentX warmup can legitimately keep the single wide decode worker busy + # for longer than Dynamo's 10-second TCP request-plane default. + DYN_TCP_REQUEST_TIMEOUT: "60" PIP_BREAK_SYSTEM_PACKAGES: "1" args: router-mode: "kv" @@ -129,6 +132,7 @@ backend: prefill: host: 0.0.0.0 served-model-name: deepseek-ai/DeepSeek-V4-Pro + enable-metrics: true model-path: /model/ trust-remote-code: true watchdog-timeout: 86400 @@ -163,6 +167,7 @@ backend: decode: host: 0.0.0.0 served-model-name: deepseek-ai/DeepSeek-V4-Pro + enable-metrics: true model-path: /model/ trust-remote-code: true watchdog-timeout: 86400 @@ -206,7 +211,8 @@ benchmark: RESULT_DIR: /logs/agentic PORT: "8000" IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "1" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" HF_HUB_CACHE: "/hf_hub_cache" - diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c8-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c8-mtp-kvoffload.yaml index dc91c255ce..2ac8d01b52 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c8-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c8-mtp-kvoffload.yaml @@ -129,6 +129,7 @@ backend: prefill: host: 0.0.0.0 served-model-name: deepseek-ai/DeepSeek-V4-Pro + enable-metrics: true model-path: /model/ trust-remote-code: true watchdog-timeout: 86400 @@ -163,6 +164,7 @@ backend: decode: host: 0.0.0.0 served-model-name: deepseek-ai/DeepSeek-V4-Pro + enable-metrics: true model-path: /model/ trust-remote-code: true watchdog-timeout: 86400 @@ -205,7 +207,8 @@ benchmark: RESULT_DIR: /logs/agentic PORT: "8000" IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "1" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" HF_HUB_CACHE: "/hf_hub_cache" - diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c256-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c256-mtp-kvoffload.yaml index 6df695b914..60d0fadda1 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c256-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c256-mtp-kvoffload.yaml @@ -129,6 +129,7 @@ backend: prefill: host: 0.0.0.0 served-model-name: deepseek-ai/DeepSeek-V4-Pro + enable-metrics: true model-path: /model/ trust-remote-code: true watchdog-timeout: 86400 @@ -163,6 +164,7 @@ backend: decode: host: 0.0.0.0 served-model-name: deepseek-ai/DeepSeek-V4-Pro + enable-metrics: true model-path: /model/ trust-remote-code: true watchdog-timeout: 86400 @@ -206,7 +208,8 @@ benchmark: RESULT_DIR: /logs/agentic PORT: "8000" IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "1" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" HF_HUB_CACHE: "/hf_hub_cache" - diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c256-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c256-mtp-kvoffload.yaml index 107a787ba9..25bdf62713 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c256-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c256-mtp-kvoffload.yaml @@ -129,6 +129,7 @@ backend: prefill: host: 0.0.0.0 served-model-name: deepseek-ai/DeepSeek-V4-Pro + enable-metrics: true model-path: /model/ trust-remote-code: true watchdog-timeout: 86400 @@ -163,6 +164,7 @@ backend: decode: host: 0.0.0.0 served-model-name: deepseek-ai/DeepSeek-V4-Pro + enable-metrics: true model-path: /model/ trust-remote-code: true watchdog-timeout: 86400 @@ -206,7 +208,8 @@ benchmark: RESULT_DIR: /logs/agentic PORT: "8000" IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "1" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" HF_HUB_CACHE: "/hf_hub_cache" - diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c512-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c512-mtp-kvoffload.yaml index 27a4e42d22..3cb306c684 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c512-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c512-mtp-kvoffload.yaml @@ -129,6 +129,7 @@ backend: prefill: host: 0.0.0.0 served-model-name: deepseek-ai/DeepSeek-V4-Pro + enable-metrics: true model-path: /model/ trust-remote-code: true watchdog-timeout: 86400 @@ -163,6 +164,7 @@ backend: decode: host: 0.0.0.0 served-model-name: deepseek-ai/DeepSeek-V4-Pro + enable-metrics: true model-path: /model/ trust-remote-code: true watchdog-timeout: 86400 @@ -206,7 +208,8 @@ benchmark: RESULT_DIR: /logs/agentic PORT: "8000" IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "1" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" HF_HUB_CACHE: "/hf_hub_cache" - diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 641d92ce58..471981f60c 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5697,3 +5697,15 @@ - "Use the official lmsysorg/sglang:v0.5.16-rocm720-mi30x image, enable SGLang prompt-cache reporting, and pass the backend Prometheus metrics endpoint explicitly to AIPerf" - "Allow up to 30 minutes for healthy AgentX responses admitted near the end of the measurement window to drain before AIPerf finalizes profile metric coverage" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2528 + +- config-keys: + - dsv4-fp4-gb300-dynamo-sglang-agentic-agg + - dsv4-fp4-gb300-dynamo-sglang-agentic-disagg + scenario-type: + - agentic-coding + description: + - "Refresh submission with up to date AgentX harness." + - "Collect every SGLang worker's metrics through AgentX and fail when required sglang: series are absent." + - "Enable SGLang metrics on every aggregate, prefill, and decode engine." + - "Use supported header-based Dynamo session routing with the in-repo AIPerf build." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2520 diff --git a/runners/launch_gb300-nv.sh b/runners/launch_gb300-nv.sh index d955d18ebc..e90c8cad2e 100644 --- a/runners/launch_gb300-nv.sh +++ b/runners/launch_gb300-nv.sh @@ -206,13 +206,15 @@ if [[ "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5" \ recipes/sglang/qwen3.5 elif [[ "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "dsv4" ]]; then - # DSv4 GB300 sglang agentic: NVIDIA/srt-slurm v1.0.10 has the nginx - # client_max_body_size fix (>1 MiB agentic warmup bodies), the - # session-affinity frontend, and the BenchmarkType.CUSTOM / extra_mount - # schema these recipes need. + # DSv4 GB300 SGLang agentic uses NVIDIA/srt-slurm v1.0.38. In addition to + # the nginx body-size fix, session-affinity frontend, and custom benchmark + # schema required by these recipes, this release injects every logical + # SGLang worker leader's /metrics URL into AIPERF_SERVER_METRICS_URLS. + # AgentX forwards that list to aiperf's --server-metrics argument so its + # trace artifacts include backend metrics for every engine. git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" - git checkout v1.0.10 + git checkout v1.0.38 mkdir -p recipes/sglang/deepseek-v4/agentic cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic" \ recipes/sglang/deepseek-v4/agentic