From 045fee2ab4f2134b188c37bab7676819b1a825e0 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 7 Aug 2026 00:11:29 -0500 Subject: [PATCH 1/3] perf(agentx): refresh dsv4-b200-vllm-agentic-mtp harness --- perf-changelog.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 4a2145e445..6a74d9ab28 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5526,4 +5526,12 @@ - "Enable SGLang backend metrics on every aggregate, prefill, and decode engine, and fail before publishing a partial trace artifact if required sglang: metrics are absent." - "Record the recipes' active HiCache host-DRAM tier and, for disaggregated points, Dynamo router commit 5a638087 in nvidia-master metadata so generated artifacts no longer report kv_offloading=none, allocated_cpu_dram_gb=0, or a null router." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2477 + +- config-keys: + - dsv4-fp4-b200-vllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Refresh submission with up to date AgentX harness." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2516 From 06cc78e3367331c0cb940263017b8841c5e8cae9 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 7 Aug 2026 01:17:19 -0500 Subject: [PATCH 2/3] fix(agentx): collect B200 vLLM backend metrics --- benchmarks/single_node/agentic/dsv4_fp4_b200_vllm_mtp.sh | 7 +++++++ perf-changelog.yaml | 1 + 2 files changed, 8 insertions(+) diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm_mtp.sh index 85ffd64609..8152c7091e 100755 --- a/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm_mtp.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm_mtp.sh @@ -93,6 +93,13 @@ if [ "$DP_ATTENTION" = "true" ]; then agentic_pip_install --quiet "vllm-router==$VLLM_ROUTER_VERSION" fi +# AIPerf automatically scrapes the public endpoint's /metrics URL. That is the +# vLLM engine for pure TP, but the native router for DP-attention. Explicitly +# add the engine endpoint so every topology captures vLLM metrics; AIPerf +# deduplicates it against the automatic endpoint in pure-TP runs. +export AIPERF_SERVER_METRICS_URLS="http://localhost:${VLLM_BACKEND_PORT}/metrics" +export AIPERF_REQUIRED_SERVER_METRIC_PREFIX="vllm:" + # DeepSeek-V4-Pro weights are large; engine startup can exceed default 600s. export VLLM_ENGINE_READY_TIMEOUT_S=3600 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 6a74d9ab28..c0f11658f9 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5533,5 +5533,6 @@ - agentic-coding description: - "Refresh submission with up to date AgentX harness." + - "Capture the vLLM backend Prometheus endpoint for every topology and require vllm: metrics before publishing AgentX results." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2516 From dd5a7eca02da4f75d3c6a69c82b3dcbf0d3063e7 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 7 Aug 2026 10:22:25 -0500 Subject: [PATCH 3/3] fix(agentx): pin concurrency-safe AIPerf --- utils/aiperf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/aiperf b/utils/aiperf index 743fb82ab7..67a42758b3 160000 --- a/utils/aiperf +++ b/utils/aiperf @@ -1 +1 @@ -Subproject commit 743fb82ab798dc898fce98d0321db61e7564942a +Subproject commit 67a42758b366589951cdecc06087e0a6cf88603d