From d24c585286a2d31d9634559ddfe6945e817b7108 Mon Sep 17 00:00:00 2001 From: Chenfei Zhang Date: Wed, 26 Aug 2026 23:06:41 -0700 Subject: [PATCH 01/14] [None][test] Add AgentX DeepSeek-V4-Pro-DSpark perf-sanity lanes on GB300 Adds the AgentX agentic trace-replay workload to the perf-sanity system and two DeepSeek-V4-Pro-DSpark lanes that use it on GB300 aws-cmh. AgentX replays a recorded multi-turn conversation corpus for a fixed wall-clock duration rather than a fixed prompt count, so it cannot reuse benchmark_serving (there is no ISL/OSL/num-prompts to give it). A new client, tests/integration/defs/perf/agentx_client.py, adapts the pinned agentx-harness build of aiperf to the harness: it installs the pinned build, proves the endpoint serves one real token before spending GPU-hours, runs `aiperf profile`, and re-emits the export in benchmark_serving's exact report format so the existing metric scanner, DB upload and regression check all keep working unchanged. Wiring in test_perf_sanity.py: - New optional `benchmark_client` key on a disaggregated benchmark_config selects the client. Any unrecognised value is rejected at parse time rather than silently falling through to the default client. - The value is uploaded as the `s_benchmark_client` baseline match key so a differently-driven lane forms its own population. It is emitted as "" (not "default") for the built-in client, and benchmark_data_matches treats absent and empty as equal, so pre-existing baselines keep matching. - AgentX lanes are exempt from the spec-decoding `al` hard-fail: `al` derives from a TRT-LLM-specific per-response field aiperf does not propagate, and both lanes pin the accepted length with TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS (already a match key), so `al` would restate a configured constant rather than measure anything. - force_num_accepted_tokens_from_env_str now parses a fractional value, which the runtime accepts and both lanes use. Both lanes are e2e only. gen_only is deliberately left to a follow-up: the trace's median ISL is ~101K tokens, so the gen_only fill gate would need far more gen KV resident than 8x GB300 can hold and would never open. AgentX gen_only needs the gen_only_no_context flavor, which implies different node counts and its own stages. Lanes: - 24 GPUs / 6 nodes: ctx2 dep8 + gen1 dep8, DSpark draft len 3 - 40 GPUs / 10 nodes: ctx3 dep8 + gen1 dep16, DSpark draft len 5 Validated end-to-end against a real 6-node aws-cmh run: submission_valid true, TTFT coverage 0.9963, ITL coverage 1.0, 990 requests over 1820s. The generated argv is byte-identical to the reference script's, and all 16 metric lines parse under the harness's existing regexes. Signed-off-by: Chenfei Zhang --- jenkins/L0_Test.groovy | 22 + tests/integration/defs/.test_durations | 2 + .../defs/perf/README_test_perf_sanity.md | 35 ++ tests/integration/defs/perf/agentx_client.py | 538 ++++++++++++++++++ .../integration/defs/perf/test_perf_sanity.py | 79 ++- ...sanity_ctx2_node2_gpu8_gen1_node2_gpu8.yml | 23 + ...anity_ctx3_node2_gpu8_gen1_node4_gpu16.yml | 21 + ...dep8_gen1_dep8_eplb0_dspark3_ccb-NIXL.yaml | 148 +++++ ...ep8_gen1_dep16_eplb0_dspark5_ccb-NIXL.yaml | 148 +++++ 9 files changed, 1014 insertions(+), 2 deletions(-) create mode 100644 tests/integration/defs/perf/agentx_client.py create mode 100644 tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx2_node2_gpu8_gen1_node2_gpu8.yml create mode 100644 tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx3_node2_gpu8_gen1_node4_gpu16.yml create mode 100644 tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v4-pro-dspark_agentx_con1156_ctx2_dep8_gen1_dep8_eplb0_dspark3_ccb-NIXL.yaml create mode 100644 tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v4-pro-dspark_agentx_con1456_ctx3_dep8_gen1_dep16_eplb0_dspark5_ccb-NIXL.yaml diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 603788b1b626..2bc190304316 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -6557,6 +6557,28 @@ def launchTestJobs(pipeline, testFilter, globalVars) // Nemotron-Ultra-V3 con9832 (8k64k) and con1197 (50k2k) are ctx_only-only: // their full 68-/72-GPU e2e+gen_only disagg topologies are intentionally not // created; the ctx_only ids run in the 4-GPU multi_gpus post-merge stage. + // GB300 DeepSeek-V4-Pro-DSpark, AgentX agentic trace replay. + // These lanes replay a ~1M-token multi-turn conversation trace for a fixed + // wall-clock duration instead of a fixed prompt count, so they are pinned to + // aws-cmh where the DSpark checkpoint and the trace corpus are staged. + // 6 Nodes: ctx2 (2 nodes, 8 GPUs each) + gen1 (2 nodes, 8 GPUs) = 24 GPUs + multiNodesSBSAConfigs += buildStageConfigs( + "GB300-24_GPUs-6_Nodes-PyTorch-Disagg-PerfSanity-AgentX-CTX2-NODE2-GPU8-GEN1-NODE2-GPU8-Post-Merge", + "gb300-flex-aws-cmh", + "l0_gb300_multi_nodes_perf_sanity_ctx2_node2_gpu8_gen1_node2_gpu8", + 1, + 24, + 6 + ) + // 10 Nodes: ctx3 (2 nodes, 8 GPUs each) + gen1 (4 nodes, 16 GPUs) = 40 GPUs + multiNodesSBSAConfigs += buildStageConfigs( + "GB300-40_GPUs-10_Nodes-PyTorch-Disagg-PerfSanity-AgentX-CTX3-NODE2-GPU8-GEN1-NODE4-GPU16-Post-Merge", + "gb300-flex-aws-cmh", + "l0_gb300_multi_nodes_perf_sanity_ctx3_node2_gpu8_gen1_node4_gpu16", + 1, + 40, + 10 + ) multiNodesSBSAConfigs = cbtsResizeSplits(multiNodesSBSAConfigs) fullSet += multiNodesSBSAConfigs.keySet() diff --git a/tests/integration/defs/.test_durations b/tests/integration/defs/.test_durations index 4b3dba9d99dd..4fba998faf46 100644 --- a/tests/integration/defs/.test_durations +++ b/tests/integration/defs/.test_durations @@ -930,6 +930,8 @@ "perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_qwen3-235b-fp4_8k1k_con1024_ctx1_tp1_gen1_dep8_eplb0_mtp0_ccb-NIXL]": 2103.162375, "perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL]": 7666.37925, "perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL]": 2542.9724666666666, + "perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_deepseek-v4-pro-dspark_agentx_con1156_ctx2_dep8_gen1_dep8_eplb0_dspark3_ccb-NIXL]": 5400.0, + "perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_deepseek-v4-pro-dspark_agentx_con1456_ctx3_dep8_gen1_dep16_eplb0_dspark5_ccb-NIXL]": 5400.0, "perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_glm-5-fp4_8k1k_con1024_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-NIXL]": 3729.2565, "perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_nemotron-ultra-v3-fp4_50k2k_con12_ctx1_dep4_gen6_tep4_eplb0_mtp6_ccb-NIXL]": 1731.7142, "perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_nemotron-ultra-v3-fp4_50k2k_con178_ctx5_dep4_gen1_dep4_eplb0_mtp6_ccb-NIXL]": 1023.3791666666666, diff --git a/tests/integration/defs/perf/README_test_perf_sanity.md b/tests/integration/defs/perf/README_test_perf_sanity.md index 25c07594452c..6883bb829aca 100644 --- a/tests/integration/defs/perf/README_test_perf_sanity.md +++ b/tests/integration/defs/perf/README_test_perf_sanity.md @@ -117,6 +117,37 @@ pass. `process_and_upload_test_results` therefore looks history up against a baseline branch — `PERF_BASELINE_BRANCH`, default `main` — using a lookup-only copy of the data. The uploaded documents keep their true `s_branch`. +`ClientConfig.to_match_keys()` includes `s_benchmark_client`, so a lane driven by a +different load generator forms its own baseline population instead of being compared +against `benchmark_serving` history. The built-in client uploads `""` (not `"default"`) +for this field, and `benchmark_data_matches` treats absent and empty as equal, so every +baseline recorded before the field existed keeps matching. + +## Benchmark Clients + +A disaggregated `benchmark_config` may select which load generator drives the lane via +the optional `benchmark_client` key. Any value other than the two below is rejected at +config-parse time rather than silently falling through to the default client. + +| `benchmark_client` | Client | Notes | +|---|---|---| +| *(omitted / `""`)* | `tensorrt_llm/serve/scripts/benchmark_serving.py` | Default. Fixed request count from `dataset_file` + `concurrency_list`. | +| `agentx` | `perf/agentx_client.py` | AgentX agentic multi-turn trace replay (a fork of `aiperf`). | + +**AgentX specifics**: +- Duration-bounded, not count-bounded: the run replays a conversation trace for + `AGENTX_DURATION` seconds, so `dataset_file` names an `aiperf` dataset loader (fetched + from HF), *not* a path — `get_dataset_dir` must not be applied to it. +- `concurrency_list` is the whole-cluster total and is passed through un-multiplied. +- Tunables are passed through `client_env_var` (`AGENTX_MAX_CTX`, `AGENTX_DURATION`, + `AGENTX_WARMUP_PER_LANE`). Artifacts land in + `{test_output_dir}/agentx.{server_idx}.{client_idx}/concurrency_{N}/`. +- `al` (Mean Avg Decoded Tokens per Iter) is exempt from the spec-decoding hard-fail for + agentx lanes: it derives from a TRT-LLM-specific per-response field that `aiperf` does + not propagate. This loses no signal as long as the lane pins the accepted length with + `TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS` (already a match key), which makes `al` a + restatement of a configured constant rather than a measurement. + ## Overview - Run performance sanity benchmarks across multiple model configs @@ -191,6 +222,10 @@ There are two modes for perf sanity tests: aggregated (aggr) and disaggregated ( **Use Case**: Disaggregated architecture where model runs across multiple nodes with separate context (prefill) and generation (decode) servers. +**Optional `benchmark_config` keys**: `benchmark_client` selects a non-default load +generator (see [Benchmark Clients](#benchmark-clients)); `client_env_var` passes +client-side environment variables through to it. + ## Test Case Formats In each test db yml file (with keyword `perf_sanity`), there are four test types: diff --git a/tests/integration/defs/perf/agentx_client.py b/tests/integration/defs/perf/agentx_client.py new file mode 100644 index 000000000000..d37634031060 --- /dev/null +++ b/tests/integration/defs/perf/agentx_client.py @@ -0,0 +1,538 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""AgentX benchmark client for perf-sanity disaggregated lanes. + +AgentX is an agentic, multi-turn *trace replay* workload. It is driven by +``aiperf`` from the SemiAnalysis ``agentx-harness`` fork, whose +``inferencex-agentx-mvp`` scenario replays a recorded conversation corpus for a +fixed **wall-clock duration** rather than a fixed prompt count. That single +difference is why it cannot reuse +``tensorrt_llm.serve.scripts.benchmark_serving``: there is no ISL/OSL/num-prompts +to give it, and the run length is time-bounded. + +This module is the adapter between the perf-sanity harness and that client. It + +1. installs the pinned ``agentx-harness`` build (see ``AGENTX_AIPERF_REF``), +2. verifies the endpoint really serves one token before spending GPU-hours, +3. runs ``aiperf profile``, and +4. translates ``profile_export_aiperf.json`` into the exact stdout lines that + ``PERF_METRIC_LOG_QUERIES`` in ``test_perf_sanity.py`` already scans for. + +Step 4 is the reason this is a separate process rather than harness code: by +emitting ``benchmark_serving``'s own report format verbatim, the perf-sanity +metric scanner, database upload and regression check all keep working unchanged. + +Two deliberate deviations from the upstream ``run_benchmark_agentx.sh``: + +* **Concurrency is passed through, not multiplied.** The reference script + computes ``concurrency * num_gen_servers`` because its YAML states per-server + concurrency. perf-sanity's ``concurrency_list`` is already the whole-cluster + total (it feeds ``--max-concurrency`` directly in the default client), so + scaling it here would silently over-drive the lane. +* **Readiness is re-verified, briefly.** The harness has already waited for the + disagg server, so the long poll is redundant; the one-token probe is kept + because it is the only cheap check that proves the ctx->gen KV path works. + Without it a broken KV transfer surfaces an hour later as a client-side + "failed request threshold" error, sending the reader to the wrong logs. + +All tuning knobs are read from the environment with the same names and defaults +as the reference script, so a lane is configured purely through the YAML's +``environment.client_env_var``. +""" + +import argparse +import json +import os +import subprocess +import sys +import threading +import time +import urllib.error +import urllib.request + +# Pinned agentx-harness build. It installs *as* ``aiperf`` (shadowing any +# PyPI aiperf), so the version is pinned by commit rather than by release. +DEFAULT_AIPERF_REF = ( + "git+https://github.com/SemiAnalysisAI/agentx-harness.git" + "@754356e9a39acc6cc6afb242d123bb57c3fb6f75" +) + +# aiperf's CLI/serialization/UI dependencies. The TRT-LLM container already +# carries its ML stack (numpy/pandas/torch/huggingface-hub/aiohttp); installing +# only this set has been verified to add packages without reinstalling the +# pinned scientific stack. +AIPERF_EXTRA_DEPS = [ + "cyclopts", + "ruamel.yaml", + "msgspec", + "orjson", + "kaleido", + "dash", + "dash-bootstrap-components", + "textual", + "starlette-compress", + "setproctitle", + "jmespath", + "aiofiles", + "ffmpeg-python", + "pydantic-settings", + "prometheus_client", + "fastapi", + "soundfile", + "plotly", + "seaborn", + "tiktoken", + "tqdm", + "uvicorn[standard]", + "uvloop", + "crick", + "zstandard", +] + +# The container ships datasets 3.x, which cannot parse the trace corpus schema +# ("Feature type 'Json' not found"). 4.x+ is required. +MIN_DATASETS_MAJOR = 4 + +# PyExecutor's HangDetector is hard-coded at 300 s with no env override, so a +# multi-minute pip install with zero requests in flight idles the GEN worker +# into MPI_Abort -- which then presents as a *client* connection failure. A +# best-effort 1-token request on this period keeps the executor awake. +HEARTBEAT_PERIOD_S = 60 + +# Mapping from perf-sanity's report labels to aiperf metric fields. +# +# Two of these are easy to get wrong and are worth stating explicitly, because +# both names exist in the export and differ by ~47x: +# * ``inter_chunk_latency`` is pooled over every streamed chunk (count == total +# chunks) -- that is ITL. +# * ``inter_token_latency`` is already averaged per request (count == request +# count) -- that is TPOT. +# ``user_throughput`` likewise maps to ``e2e_output_token_throughput`` (which +# includes TTFT, matching benchmark_serving's definition) and NOT to the +# similarly-named ``output_token_throughput_per_user`` (decode-only). +# +# aiperf reports every latency in ms and every throughput in tokens/sec or +# requests/sec, which is what the labels below claim, so no unit conversion is +# applied. ``_read_metric`` asserts the unit to keep that assumption honest. +THROUGHPUT_METRICS = [ + ("Request throughput (req/s)", "request_throughput", "requests/sec"), + ("Output token throughput (tok/s)", "output_token_throughput", "tokens/sec"), + ("Total Token throughput (tok/s)", "total_token_throughput", "tokens/sec"), + ("User throughput (tok/s)", "e2e_output_token_throughput", "tokens/sec/user"), +] + +# (label stem, aiperf field). Each contributes Mean/Median/P99 lines. +LATENCY_METRICS = [ + ("TTFT", "time_to_first_token"), + ("ITL", "inter_chunk_latency"), + ("TPOT", "inter_token_latency"), + ("E2EL", "request_latency"), +] + +# benchmark_serving's report format. Reproduced exactly so the existing +# regex scanner needs no change; see _print_metric for why width matters. +_REPORT_FORMAT = "{:<40} {:<10.2f}" + + +def _log(msg: str) -> None: + print(f"[agentx] {msg}", flush=True) + + +def _print_metric(label: str, value: float) -> None: + """Emit one metric line in benchmark_serving's exact report format. + + ``PERF_METRIC_LOG_QUERIES`` requires at least one whitespace character + between the colon and the number. The 40-char field supplies it only while + ``len(label) + 1 < 40``, which holds for every label here (longest is 30); + a longer label would silently stop matching, hence the assert. + """ + stem = f"{label}:" + assert len(stem) < 40, f"label {label!r} too long to leave a separator" + print(_REPORT_FORMAT.format(stem, value), flush=True) + + +def _env_int(name: str, default: int) -> int: + return int(os.environ.get(name, default)) + + +def _env_float(name: str, default: float) -> float: + return float(os.environ.get(name, default)) + + +def aiperf_is_usable() -> bool: + """Return whether the ambient interpreter already has a working agentx stack. + + Checks importability *and* the datasets major version, since a container + with aiperf but datasets 3.x fails only later, when loading the corpus. + """ + probe = ( + "import aiperf, cyclopts, datasets, transformers, pydantic, msgspec, orjson; " + f"import sys; sys.exit(0 if int(datasets.__version__.split('.')[0]) >= {MIN_DATASETS_MAJOR} else 1)" + ) + return ( + subprocess.call( + [sys.executable, "-c", probe], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL + ) + == 0 + ) + + +def post_one_token(url: str, model: str, timeout: float) -> "str | None": + """POST a 1-token non-streaming completion. Returns the body, or None on failure.""" + payload = json.dumps( + { + "model": model, + "messages": [{"role": "user", "content": "hi"}], + "max_tokens": 1, + "stream": False, + } + ).encode() + req = urllib.request.Request( + f"{url}/v1/chat/completions", + data=payload, + headers={"Content-Type": "application/json"}, + method="POST", + ) + try: + with urllib.request.urlopen(req, timeout=timeout) as resp: + return resp.read().decode(errors="replace") + except (urllib.error.URLError, OSError, TimeoutError): + return None + + +class _Heartbeat: + """Keeps the GEN worker's hang detector at bay during a long pip install.""" + + def __init__(self, url: str, model: str): + self._url = url + self._model = model + self._stop = threading.Event() + self._thread = threading.Thread(target=self._run, daemon=True) + + def _run(self) -> None: + while not self._stop.wait(HEARTBEAT_PERIOD_S): + # Best-effort by construction: post_one_token swallows every + # network error, so the heartbeat can never fail the run. + post_one_token(self._url, self._model, timeout=60) + + def __enter__(self) -> "_Heartbeat": + self._thread.start() + _log(f"heartbeat armed ({HEARTBEAT_PERIOD_S}s) to hold off the 300s hang detector") + return self + + def __exit__(self, *exc_info: object) -> None: + self._stop.set() + self._thread.join(timeout=HEARTBEAT_PERIOD_S + 30) + + +def _pip(*args: str) -> None: + subprocess.check_call([sys.executable, "-m", "pip", "install", *args]) + + +def ensure_aiperf(url: str, model: str) -> None: + """Install the pinned agentx build unless a usable one is already present.""" + if os.environ.get("AGENTX_FORCE_REINSTALL", "0") == "1": + _log("AGENTX_FORCE_REINSTALL=1 -> reinstalling unconditionally") + elif aiperf_is_usable(): + _log("aiperf + deps already usable; skipping install") + return + + ref = os.environ.get("AGENTX_AIPERF_REF", DEFAULT_AIPERF_REF) + with _Heartbeat(url, model): + _log(f"installing aiperf (agentx) from {ref}") + _pip("--force-reinstall", "--no-deps", f"aiperf @ {ref}") + _pip(*AIPERF_EXTRA_DEPS) + _pip("-U", "datasets") + _pip("transformers==5.8.1") + # aiperf needs a newer pydantic than the container pins, but its own + # dependency resolution would drag in a fresh ML stack; pin the pair + # directly with --no-deps instead. + _pip("--force-reinstall", "--no-deps", "pydantic==2.12.5", "pydantic-core==2.41.5") + + if not aiperf_is_usable(): + raise RuntimeError( + "aiperf is still not importable after install; refusing to start the " + "benchmark (see the pip output above)" + ) + + +def verify_endpoint(url: str, model: str) -> None: + """Prove the endpoint serves one real token before spending GPU-hours. + + The harness has already waited for the disagg server, so this is a short + confirmation rather than the reference script's 5400 s poll. It exists + because a broken ctx->gen KV path answers /health perfectly well and only + fails once real traffic arrives -- an hour later, as a client-side error. + """ + timeout = _env_int("AGENTX_READY_TIMEOUT", 900) + interval = _env_int("AGENTX_READY_INTERVAL", 15) + deadline = time.monotonic() + timeout + last = None + while time.monotonic() < deadline: + last = post_one_token(url, model, timeout=300) + if last and '"choices"' in last: + _log("1-token completion OK -- the disagg path is live") + return + _log(f"not serving yet (response head: {str(last)[:200]})") + time.sleep(interval) + raise RuntimeError( + f"{url} could not complete ONE token within {timeout}s. This is a " + f"SERVER-side failure: inspect the ctx/gen worker logs and the disagg " + f"server log, not the client. Suspect the ctx->gen KV transfer " + f"(on GB300/NVL72 check UCX_TLS contains no 'rc'). " + f"Last response: {str(last)[:500]}" + ) + + +def build_aiperf_cmd(args: argparse.Namespace, url: str, artifact_dir: str) -> "list[str]": + """Assemble the ``aiperf profile`` argv. + + Note ``--url`` carries scheme+host+port only; the request path is a separate + ``--endpoint``. That is why this client takes ``--host``/``--port`` and + composes them itself instead of having them appended to the argv. + """ + cmd = [ + "aiperf", + "profile", + "--scenario", + "inferencex-agentx-mvp", + "-m", + args.model, + "--tokenizer-trust-remote-code", + "--url", + url, + "--endpoint", + "/v1/chat/completions", + "--endpoint-type", + "chat", + "--streaming", + "--public-dataset", + args.dataset, + "--max-context-length", + str(_env_int("AGENTX_MAX_CTX", 1048576)), + "--trajectory-start-min-ratio", + str(_env_float("AGENTX_MIN_RATIO", 0.25)), + "--trajectory-start-max-ratio", + str(_env_float("AGENTX_MAX_RATIO", 0.75)), + "--failed-request-threshold", + str(_env_float("AGENTX_FAILED_THRESH", 0.1)), + "--use-server-token-count", + "--trace-idle-gap-cap-seconds", + "300", + "--no-gpu-telemetry", + "--random-seed", + str(_env_int("AGENTX_SEED", 42)), + "--benchmark-duration", + str(_env_int("AGENTX_DURATION", 3600)), + "--concurrency", + str(args.concurrency), + "--artifact-dir", + artifact_dir, + "--ui", + "simple", + ] + + # --benchmark-duration is always set; adding --request-count makes the run + # stop at whichever limit is hit first, which is only ever used for smoke + # tests. Left unset in CI so the lane is purely duration-bounded. + mult = os.environ.get("AGENTX_REQUEST_COUNT_MULT") + if mult: + count = max(args.concurrency, args.concurrency * int(mult)) + _log(f"request-count = max({args.concurrency}, {mult} x {args.concurrency}) = {count}") + cmd += ["--request-count", str(count)] + + # aiperf runs warmup as a distinct phase reported under a separate + # top-level `warmup_metrics` key, so these requests never pollute the + # measured window. <= 0 omits the flag entirely. + warmup = _env_int("AGENTX_WARMUP_PER_LANE", 10) + if warmup > 0: + cmd += ["--warmup-requests-per-lane", str(warmup)] + + # Plain os.environ.get with a default, mirroring the reference script's + # `${VAR-default}`: an explicitly empty value must disable the flag, which + # is how a production (>=900 s) run drops --unsafe-override. + extra = os.environ.get("AGENTX_EXTRA_ARGS", "--unsafe-override") + if extra.strip(): + cmd += extra.split() + return cmd + + +def _read_metric(export: dict, field: str, stat: str, expect_unit: "str | None" = None) -> float: + """Read one statistic out of the aiperf export, failing loudly if absent. + + A silently-missing metric would leave the perf-sanity scanner short a line, + which it reports as a vague "metrics are missing" much later; and a unit + change would corrupt a baseline without any error at all. Both are checked + here, at the point where the assumption is actually made. + """ + entry = export.get(field) + if not isinstance(entry, dict): + raise RuntimeError(f"aiperf export has no metric {field!r}") + if expect_unit is not None and entry.get("unit") != expect_unit: + raise RuntimeError( + f"aiperf metric {field!r} changed unit: expected {expect_unit!r}, " + f"got {entry.get('unit')!r}. Refusing to upload a mis-scaled value." + ) + if stat not in entry: + raise RuntimeError(f"aiperf metric {field!r} has no {stat!r} (keys: {sorted(entry)})") + return float(entry[stat]) + + +def check_run_health(export: dict) -> None: + """Fail unless the export shows a valid, complete, error-free measurement. + + Deliberately gates on *content*, never on aiperf's exit status: a run that + served nothing but HTTP 500s can still exit 0, and conversely the artifacts + are flushed before the process tears down. ``submission_valid`` is the + scenario's own verdict and is the single most important field here -- it + goes false when the measured window does not actually cover the requested + duration, which is exactly the failure mode that makes a short run look + like a fast one. + """ + metadata = export.get("metadata") or {} + + if export.get("was_cancelled"): + raise RuntimeError("aiperf reports was_cancelled=true; the measurement is incomplete") + + errors = export.get("error_summary") or [] + if errors: + raise RuntimeError(f"aiperf reported request errors: {json.dumps(errors)[:2000]}") + + # Report coverage before the verdict: when submission_valid is false this + # is the number that says why, so it must be in the log either way. + for phase in metadata.get("metric_duration_coverage") or []: + _log( + "coverage[{}]: ttft={:.4f} itl={:.4f} required={} expected_duration={}s".format( + phase.get("phase_name"), + phase.get("ttft_ratio", float("nan")), + phase.get("inter_token_latency_ratio", float("nan")), + phase.get("required_ratio"), + phase.get("expected_duration_seconds"), + ) + ) + + if "submission_valid" not in metadata: + raise RuntimeError( + "aiperf export has no metadata.submission_valid; the agentx scenario " + "did not complete (schema_version=" + f"{export.get('schema_version')!r})" + ) + if not metadata["submission_valid"]: + raise RuntimeError( + "aiperf reports submission_valid=false: the measured window does not " + "cover the requested duration, so these numbers are not comparable to " + "a baseline. See the coverage line above." + ) + + requests = _read_metric(export, "request_count", "avg") + duration = _read_metric(export, "benchmark_duration", "avg", "sec") + if requests <= 0: + raise RuntimeError("aiperf completed 0 requests") + _log(f"health OK: {int(requests)} requests over {duration:.1f}s, submission_valid=true") + + +def report_metrics(export: dict) -> None: + """Print every PERF_METRIC_LOG_QUERIES line from the aiperf export.""" + for label, field, unit in THROUGHPUT_METRICS: + _print_metric(label, _read_metric(export, field, "avg", unit)) + for stem, field in LATENCY_METRICS: + for prefix, stat in (("Mean", "avg"), ("Median", "p50"), ("P99", "p99")): + _print_metric(f"{prefix} {stem} (ms)", _read_metric(export, field, stat, "ms")) + + +def parse_args(argv: "list[str] | None" = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--model", required=True, help="Model path or HF id served by the endpoint") + parser.add_argument( + "--concurrency", + required=True, + type=int, + help="Total in-flight requests. Passed through to aiperf unscaled.", + ) + parser.add_argument("--dataset", required=True, help="aiperf --public-dataset loader name") + parser.add_argument("--host", required=True) + parser.add_argument("--port", required=True) + parser.add_argument( + "--artifact-dir", + # The harness exports this so artifacts land in the test output + # directory and are collected with the rest of the lane's logs. + default=os.environ.get("TRTLLM_AGENTX_ARTIFACT_DIR", "agentx_artifacts"), + help="Parent directory for aiperf artifacts; a concurrency_ subdir is created in it.", + ) + return parser.parse_args(argv) + + +def _run(args: argparse.Namespace, artifact_dir: str) -> int: + url = f"http://{args.host}:{args.port}" + # Custom tokenizers (e.g. deepseek_v4) are loaded in aiperf's worker + # processes, which do not inherit a CLI trust flag. + os.environ["HF_HUB_TRUST_REMOTE_CODE"] = "1" + + ensure_aiperf(url, args.model) + verify_endpoint(url, args.model) + + cmd = build_aiperf_cmd(args, url, artifact_dir) + _log("running: " + " ".join(cmd)) + completed = subprocess.run(cmd) + + export_path = os.path.join(artifact_dir, "profile_export_aiperf.json") + if not os.path.exists(export_path): + # No export at all means aiperf died before writing results; its exit + # status is the only signal left, so surface it. + raise RuntimeError( + f"aiperf exited {completed.returncode} and wrote no {export_path}. " + f"See the aiperf output above and {artifact_dir}/logs." + ) + if completed.returncode != 0: + # An export exists, so the health checks below are more informative + # than the status; log it and let them make the call. + _log(f"warning: aiperf exited {completed.returncode} but wrote an export; validating it") + + with open(export_path) as f: + export = json.load(f) + + check_run_health(export) + report_metrics(export) + return 0 + + +def main(argv: "list[str] | None" = None) -> int: + args = parse_args(argv) + # aiperf writes its files flat into --artifact-dir, so give each + # concurrency its own directory to keep exports from overwriting. + artifact_dir = os.path.abspath( + os.path.join(args.artifact_dir, f"concurrency_{args.concurrency}") + ) + os.makedirs(artifact_dir, exist_ok=True) + try: + return _run(args, artifact_dir) + except Exception as exc: + # The perf-sanity harness invokes this via subprocess.check_output, + # which discards stdout when the exit status is non-zero -- so a + # failure would otherwise lose every diagnostic printed above. Persist + # the reason next to aiperf's own logs, which are collected with the + # test output. + reason = f"{type(exc).__name__}: {exc}" + _log(f"FAILED -- {reason}") + try: + with open(os.path.join(artifact_dir, "agentx_client_error.txt"), "w") as f: + f.write(reason + "\n") + except OSError: + pass + raise + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/integration/defs/perf/test_perf_sanity.py b/tests/integration/defs/perf/test_perf_sanity.py index 6fe156d597ea..7ee98aea0bb2 100644 --- a/tests/integration/defs/perf/test_perf_sanity.py +++ b/tests/integration/defs/perf/test_perf_sanity.py @@ -54,6 +54,10 @@ "H200": "h200", } +# benchmark_client value selecting the AgentX trace-replay client +# (agentx_client.py). Any other non-empty value is rejected at parse time. +AGENTX_BENCHMARK_CLIENT = "agentx" + BENCH_SERVING_REPO = "https://github.com/kedarpotdar-nv/bench_serving.git" BENCH_SERVING_COMMIT = "f3ea022a5780de5d0babc5fffa53634e2023d28f" BENCH_SERVING_DIR = "/tmp/bench_serving" @@ -647,9 +651,16 @@ def force_num_accepted_tokens_from_env_str(env_vars: str) -> int: """Extract TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS from a space-separated KEY=val env-var string. Returns 0 when not set. + + The runtime accepts a fractional value (see get_force_num_accepted_tokens_float + in tensorrt_llm), so parse as float first and truncate. The return value only + feeds the l_force_num_accepted_tokens baseline match key, which is a long, so + two lanes differing solely in the fractional part share a match identity -- + acceptable today because every such lane also differs in concurrency and + parallelism, but widen the key if that ever stops holding. """ val = to_env_dict(env_vars).get("TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS") - return int(val) if val is not None else 0 + return int(float(val)) if val is not None else 0 def add_host_port_to_cmd(cmd: List[str], host: str, port: int) -> List[str]: @@ -1099,6 +1110,11 @@ def __init__( self.model_path = "" self.dataset_file = client_config_data.get("dataset_file", "") self.use_nv_sa_benchmark = client_config_data.get("use_nv_sa_benchmark", False) + # Which load generator drives the lane. "" selects the built-in + # benchmark_serving client; "agentx" selects the trace-replay client in + # agentx_client.py. The empty default is load-bearing for baseline + # matching -- see the s_benchmark_client note in to_db_data. + self.benchmark_client = client_config_data.get("benchmark_client", "") self.env_vars = env_vars # spec_decoding flag is retained for DB matching (b_eos column). --ignore-eos # is now always passed; output-length stability with spec decoding comes from @@ -1123,11 +1139,39 @@ def to_cmd(self) -> List[str]: model_dir = get_model_dir(self.model_name) self.model_path = model_dir if os.path.exists(model_dir) else self.model_name - if self.use_nv_sa_benchmark: + if self.benchmark_client == AGENTX_BENCHMARK_CLIENT: + return self._to_agentx_cmd() + elif self.use_nv_sa_benchmark: return self._to_sa_benchmark_cmd() else: return self._to_default_benchmark_cmd() + def _to_agentx_cmd(self) -> List[str]: + """Generate AgentX benchmark command (aiperf trace replay). + + AgentX replays a recorded conversation corpus for a fixed wall-clock + duration, so it takes neither a prompt count nor ISL/OSL; every other + knob comes from AGENTX_* env vars set in the lane's client_env_var. The + dataset name is passed through verbatim rather than resolved to a path + because it names an aiperf loader (which fetches from HF), not a file -- + so get_dataset_dir must not be applied to it. + """ + if not self.dataset_file: + raise ValueError( + f"Client {self.name} uses benchmark_client={AGENTX_BENCHMARK_CLIENT} but sets no " + "dataset_file; the agentx scenario has no default corpus." + ) + return [ + "python", + os.path.join(os.path.dirname(os.path.abspath(__file__)), "agentx_client.py"), + "--model", + self.model_path, + "--concurrency", + str(self.concurrency), + "--dataset", + self.dataset_file, + ] + def _to_sa_benchmark_cmd(self) -> List[str]: """Generate SA benchmark command (bench_serving repo).""" bench_script = ensure_bench_serving_repo() @@ -1235,6 +1279,11 @@ def to_db_data(self) -> dict: "b_streaming": self.streaming, "b_trust_remote_code": self.trust_remote_code, "b_use_nv_sa_benchmark": self.use_nv_sa_benchmark, + # Match key. Deliberately uploaded as "" (not "default") for the + # built-in client: benchmark_data_matches treats absent and empty + # as equal, so every baseline recorded before this field existed + # keeps matching. Only named clients form their own population. + "s_benchmark_client": self.benchmark_client, "b_eos": self.spec_decoding, "s_client_log_link": "", "s_client_env_vars": self.env_vars, @@ -1834,6 +1883,12 @@ def run_cmd(self, server_idx: int) -> List[str]: bench_env = copy.deepcopy(os.environ) if client_config: bench_env.update(client_config.to_env()) + # Keep aiperf's artifacts (its own logs included) with + # the rest of the lane's output; ignored by other + # clients. + bench_env["TRTLLM_AGENTX_ARTIFACT_DIR"] = os.path.join( + self.test_output_dir, f"agentx.{server_idx}.{client_idx}" + ) output = _run_benchmark_with_log( client_cmd_with_port, bench_env, @@ -2248,6 +2303,15 @@ def _parse_disagg_config_file(self, config_file_path: str, config_file: str): osl = 1 if benchmark_mode == "ctx_only" else benchmark.get("output_length", 1024) dataset_file = "" if benchmark_mode == "ctx_only" else benchmark.get("dataset_file", "") use_nv_sa_benchmark = benchmark.get("use_nv_sa_benchmark", False) + benchmark_client = benchmark.get("benchmark_client", "") + if benchmark_client not in ("", AGENTX_BENCHMARK_CLIENT): + # There is no schema validation on these yamls, so an unrecognised + # value would otherwise fall through to the default client and + # quietly measure the wrong workload. + raise ValueError( + f"Unknown benchmark_client {benchmark_client!r}; " + f"expected '' or {AGENTX_BENCHMARK_CLIENT!r}." + ) if benchmark_mode == "ctx_only": spec_decoding = bool(ctx_server_config.spec_decoding_type) @@ -2276,6 +2340,7 @@ def _parse_disagg_config_file(self, config_file_path: str, config_file: str): "streaming": benchmark.get("streaming", True), "dataset_file": dataset_file, "use_nv_sa_benchmark": use_nv_sa_benchmark, + "benchmark_client": benchmark_client, "accuracy_config": accuracy_data, "only_run_accuracy": only_run_accuracy, } @@ -2564,9 +2629,19 @@ def check_test_failure(self): # Spec-decoding tests must report 'Mean Avg Decoded Tokens per Iter' # (parsed as 'al'). If the field is missing the test fails here so the # data is never uploaded to OpenSearch. + # AgentX is exempt: 'al' comes from TRT-LLM's non-standard + # avg_decoded_tokens_per_iter response field, which aiperf does + # not propagate. It is not a real loss of signal, because every + # agentx lane pins the accepted length with + # TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS -- already captured + # as the l_force_num_accepted_tokens match key -- so 'al' would + # be a restatement of a configured constant rather than a + # measurement. Revisit if an agentx lane ever runs spec decoding + # without forcing the accepted count. if ( client_idx < len(client_configs) and client_configs[client_idx].spec_decoding + and client_configs[client_idx].benchmark_client != AGENTX_BENCHMARK_CLIENT and "al" not in metrics ): error_msg += ( diff --git a/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx2_node2_gpu8_gen1_node2_gpu8.yml b/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx2_node2_gpu8_gen1_node2_gpu8.yml new file mode 100644 index 000000000000..f382c09eb0b6 --- /dev/null +++ b/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx2_node2_gpu8_gen1_node2_gpu8.yml @@ -0,0 +1,23 @@ +version: 0.0.1 +l0_gb300_multi_nodes_perf_sanity_ctx2_node2_gpu8_gen1_node2_gpu8: +- condition: + ranges: + # 2 ctx workers each with 2 nodes and 8 GPUs + # 1 gen worker with 2 nodes and 8 GPUs + system_gpu_count: + gte: 24 + lte: 24 + wildcards: + gpu: + - '*gb300*' + terms: + stage: post_merge + backend: pytorch + tests: + # deepseek-v4-pro-dspark agentx con1156. + # e2e only: the agentx trace's median ISL is ~101K tokens, so the gen_only + # fill gate (which needs `concurrency` requests resident before it opens, + # clamped to the 512-request admission capacity) would demand far more gen KV + # than 8x GB300 can hold and would never open. gen_only for agentx needs the + # gen_only_no_context flavor and is deliberately left to a follow-up. + - perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_deepseek-v4-pro-dspark_agentx_con1156_ctx2_dep8_gen1_dep8_eplb0_dspark3_ccb-NIXL] TIMEOUT (210) diff --git a/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx3_node2_gpu8_gen1_node4_gpu16.yml b/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx3_node2_gpu8_gen1_node4_gpu16.yml new file mode 100644 index 000000000000..485ac2030b10 --- /dev/null +++ b/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx3_node2_gpu8_gen1_node4_gpu16.yml @@ -0,0 +1,21 @@ +version: 0.0.1 +l0_gb300_multi_nodes_perf_sanity_ctx3_node2_gpu8_gen1_node4_gpu16: +- condition: + ranges: + # 3 ctx workers each with 2 nodes and 8 GPUs + # 1 gen worker with 4 nodes and 16 GPUs + system_gpu_count: + gte: 40 + lte: 40 + wildcards: + gpu: + - '*gb300*' + terms: + stage: post_merge + backend: pytorch + tests: + # deepseek-v4-pro-dspark agentx con1456. + # e2e only -- see the sibling ctx2_node2_gpu8_gen1_node2_gpu8 list for why + # agentx gen_only needs the gen_only_no_context flavor and is left to a + # follow-up. + - perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_deepseek-v4-pro-dspark_agentx_con1456_ctx3_dep8_gen1_dep16_eplb0_dspark5_ccb-NIXL] TIMEOUT (210) diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v4-pro-dspark_agentx_con1156_ctx2_dep8_gen1_dep8_eplb0_dspark3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v4-pro-dspark_agentx_con1156_ctx2_dep8_gen1_dep8_eplb0_dspark3_ccb-NIXL.yaml new file mode 100644 index 000000000000..8239e73d860a --- /dev/null +++ b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v4-pro-dspark_agentx_con1156_ctx2_dep8_gen1_dep8_eplb0_dspark3_ccb-NIXL.yaml @@ -0,0 +1,148 @@ +metadata: + model_name: deepseek_v4_pro_dspark + precision: fp4 + model_dir_name: DeepSeek-V4-Pro-DSpark + supported_gpus: + - GB300 + script_file: disaggr_torch.slurm + benchmark_type: agentx +slurm: + script_file: disaggr_torch.slurm + partition: + account: + # AgentX replays a 1M-context multi-turn trace for AGENTX_DURATION seconds + # (3600) on top of a DeepSeek-V4-Pro weight load, so this needs more than the + # 02:00:00 the fixed-shape 8k1k lanes use. + job_time: 04:00:00 + job_name: unified-benchmark + extra_args: --gres=gpu:4 + numa_bind: true +benchmark: + mode: e2e + use_nv_sa_benchmark: false + benchmark_client: agentx + multi_round: 1 + benchmark_ratio: 0.8 + streaming: true + concurrency_list: '1156' + input_length: 990016 + output_length: 6579 + # AgentX resolves this as a HuggingFace dataset name + # (semianalysisai/cc-traces-weka-062126, public + Apache-2.0), NOT a local + # path, so it is a literal name rather than the placeholder the + # random-synthetic lanes use. + dataset_file: semianalysis_cc_traces_weka_062126 +hardware: + gpus_per_node: 4 + num_ctx_servers: 2 + num_gen_servers: 1 +environment: + container_mount: + container_image: + model_path: + trtllm_repo: '' + build_wheel: false + work_dir: + # AGENTX_* are consumed by the agentx client in test_perf_sanity.py. Values + # are space-separated KEY=val, so none of them may contain a space. + client_env_var: AGENTX_MAX_CTX=996579 AGENTX_DURATION=3600 AGENTX_WARMUP_PER_LANE=3 + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 ENROOT_ALLOW_DEV=yes NCCL_GRAPH_MIXING_SUPPORT=0 MIMALLOC_ARENA_RESERVE=0 OMP_NUM_THREADS=1 TRTLLM_PINNED_WEIGHT_STAGING=1 + ctx_worker_env_var: PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True CUDA_SCALE_LAUNCH_QUEUES=4x TLLM_INDEXER_MQA_LOGITS_ELEM_BUDGET=1073741824 TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2.08 + gen_worker_env_var: TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2.08 + server_env_var: TRTLLM_SERVER_DISABLE_GC=1 +profiling: + nsys_on: false +worker_config: + gen: + print_iter_log: true + max_batch_size: 64 + max_num_tokens: 256 + max_seq_len: 996595 + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + cuda_graph_config: + enable_padding: true + batch_sizes: [1, 2, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64] + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + dtype: fp8 + tokens_per_block: 128 + event_buffer_max_size: 0 + avg_seq_len: 200000 + host_cache_size: 0 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + stream_interval: 20 + num_postprocess_workers: 4 + speculative_config: + decoding_type: DSpark + max_draft_len: 3 + block_size: 3 + # Resolved against llm_models_root() by generate_extra_llm_api_config(). + # DSpark ships its draft inside the target checkpoint, so this is the same + # directory as the target model. + speculative_model: DeepSeek-V4-Pro-DSpark + custom_tokenizer: deepseek_v4 + return_perf_metrics: true + perf_metrics_max_requests: 100000 + perf_metrics_output_dir: perf_metrics/gen + sparse_attention_config: + algorithm: deepseek_v4 + enable_heuristic_topk: true + num_serve_frontends: 4 + ctx: + print_iter_log: true + max_batch_size: 256 + max_num_tokens: 8192 + max_seq_len: 990016 + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + enable_attention_dp: true + cuda_graph_config: null + disable_overlap_scheduler: false + kv_cache_config: + # Conversation-scoped reuse is the point of the agentic workload: each + # replayed turn re-sends the whole prior conversation. + enable_block_reuse: true + free_gpu_memory_fraction: 0.8 + dtype: fp8 + tokens_per_block: 128 + event_buffer_max_size: 0 + block_reuse_config: + policy: per_conversation + max_num_turns: 5 + pool_ratio: [0.55, 0.22, 0.23] + host_cache_size: 193273528320 + moe_config: + backend: CUTEDSL + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + custom_tokenizer: deepseek_v4 + return_perf_metrics: true + perf_metrics_max_requests: 100000 + perf_metrics_output_dir: perf_metrics/ctx + enable_chunked_prefill: true + attention_dp_config: + kv_cache_routing_conversation_affinity: true + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + num_serve_frontends: 8 +ctx_router_config: + type: conversation +server_config_extra: + gen_strip_message_history: true + gen_tokids_ctxbytes: true + server_keep_alive_timeout: 3600 + num_workers: 4 diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v4-pro-dspark_agentx_con1456_ctx3_dep8_gen1_dep16_eplb0_dspark5_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v4-pro-dspark_agentx_con1456_ctx3_dep8_gen1_dep16_eplb0_dspark5_ccb-NIXL.yaml new file mode 100644 index 000000000000..fdaa6601db60 --- /dev/null +++ b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v4-pro-dspark_agentx_con1456_ctx3_dep8_gen1_dep16_eplb0_dspark5_ccb-NIXL.yaml @@ -0,0 +1,148 @@ +metadata: + model_name: deepseek_v4_pro_dspark + precision: fp4 + model_dir_name: DeepSeek-V4-Pro-DSpark + supported_gpus: + - GB300 + script_file: disaggr_torch.slurm + benchmark_type: agentx +slurm: + script_file: disaggr_torch.slurm + partition: + account: + # AgentX replays a 1M-context multi-turn trace for AGENTX_DURATION seconds + # (3600) on top of a DeepSeek-V4-Pro weight load, so this needs more than the + # 02:00:00 the fixed-shape 8k1k lanes use. + job_time: 04:00:00 + job_name: unified-benchmark + extra_args: --gres=gpu:4 + numa_bind: true +benchmark: + mode: e2e + use_nv_sa_benchmark: false + benchmark_client: agentx + multi_round: 1 + benchmark_ratio: 0.8 + streaming: true + concurrency_list: '1456' + input_length: 990016 + output_length: 6579 + # AgentX resolves this as a HuggingFace dataset name + # (semianalysisai/cc-traces-weka-062126, public + Apache-2.0), NOT a local + # path, so it is a literal name rather than the placeholder the + # random-synthetic lanes use. + dataset_file: semianalysis_cc_traces_weka_062126 +hardware: + gpus_per_node: 4 + num_ctx_servers: 3 + num_gen_servers: 1 +environment: + container_mount: + container_image: + model_path: + trtllm_repo: '' + build_wheel: false + work_dir: + # AGENTX_* are consumed by the agentx client in test_perf_sanity.py. Values + # are space-separated KEY=val, so none of them may contain a space. + client_env_var: AGENTX_MAX_CTX=996579 AGENTX_DURATION=3600 AGENTX_WARMUP_PER_LANE=3 + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 ENROOT_ALLOW_DEV=yes NCCL_GRAPH_MIXING_SUPPORT=0 MIMALLOC_ARENA_RESERVE=0 OMP_NUM_THREADS=1 TRTLLM_PINNED_WEIGHT_STAGING=1 + ctx_worker_env_var: PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True CUDA_SCALE_LAUNCH_QUEUES=4x TLLM_INDEXER_MQA_LOGITS_ELEM_BUDGET=1073741824 TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2.74 + gen_worker_env_var: TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2.74 + server_env_var: TRTLLM_SERVER_DISABLE_GC=1 +profiling: + nsys_on: false +worker_config: + gen: + print_iter_log: true + max_batch_size: 32 + max_num_tokens: 192 + max_seq_len: 996595 + tensor_parallel_size: 16 + moe_expert_parallel_size: 16 + pipeline_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + cuda_graph_config: + enable_padding: true + batch_sizes: [1, 2, 4, 8, 16, 24, 32] + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.7 + dtype: fp8 + tokens_per_block: 128 + event_buffer_max_size: 0 + avg_seq_len: 200000 + host_cache_size: 0 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + stream_interval: 20 + num_postprocess_workers: 4 + speculative_config: + decoding_type: DSpark + max_draft_len: 5 + block_size: 5 + # Resolved against llm_models_root() by generate_extra_llm_api_config(). + # DSpark ships its draft inside the target checkpoint, so this is the same + # directory as the target model. + speculative_model: DeepSeek-V4-Pro-DSpark + custom_tokenizer: deepseek_v4 + return_perf_metrics: true + perf_metrics_max_requests: 100000 + perf_metrics_output_dir: perf_metrics/gen + sparse_attention_config: + algorithm: deepseek_v4 + enable_heuristic_topk: true + num_serve_frontends: 4 + ctx: + print_iter_log: true + max_batch_size: 256 + max_num_tokens: 8192 + max_seq_len: 990016 + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + enable_attention_dp: true + cuda_graph_config: null + disable_overlap_scheduler: false + kv_cache_config: + # Conversation-scoped reuse is the point of the agentic workload: each + # replayed turn re-sends the whole prior conversation. + enable_block_reuse: true + free_gpu_memory_fraction: 0.8 + dtype: fp8 + tokens_per_block: 128 + event_buffer_max_size: 0 + block_reuse_config: + policy: per_conversation + max_num_turns: 5 + pool_ratio: [0.55, 0.22, 0.23] + host_cache_size: 193273528320 + moe_config: + backend: CUTEDSL + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + custom_tokenizer: deepseek_v4 + return_perf_metrics: true + perf_metrics_max_requests: 100000 + perf_metrics_output_dir: perf_metrics/ctx + enable_chunked_prefill: true + attention_dp_config: + kv_cache_routing_conversation_affinity: true + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + num_serve_frontends: 8 +ctx_router_config: + type: conversation +server_config_extra: + gen_strip_message_history: true + gen_tokids_ctxbytes: true + server_keep_alive_timeout: 3600 + num_workers: 4 From 3c7e2d82b757c76379454e60bb52229de20eb182 Mon Sep 17 00:00:00 2001 From: Chenfei Zhang Date: Thu, 27 Aug 2026 01:45:08 -0700 Subject: [PATCH 02/14] [None][test] Harden AgentX lane result validation Two review findings on the AgentX perf-sanity lanes. Do not validate an export left by a previous run. agentx_client reused whatever profile_export_aiperf.json was already in the artifact directory, so when aiperf crashed but a stale export was present the run reported the previous invocation's metrics as a success. Reachable whenever the directory is reused: a retried lane, or a local rerun against the same --output-dir. Remove the export before launching aiperf so the existence check afterwards is a statement about this run. Condition the 'al' hard-fail exemption on the accepted count actually being pinned. The exemption is sound only because every agentx lane sets TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS, which made 'al' a restatement of a configured constant; that precondition was only a comment. Derive it from ServerConfig.force_num_accepted_tokens instead, so an agentx lane running spec decoding without forcing the accepted count still fails. Both shipped lanes resolve force_num_accepted_tokens=2 and stay exempt; with the variable stripped the same lane is no longer exempt. Signed-off-by: Chenfei Zhang --- tests/integration/defs/perf/agentx_client.py | 11 +++++++++- .../integration/defs/perf/test_perf_sanity.py | 21 ++++++++++++++++--- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/tests/integration/defs/perf/agentx_client.py b/tests/integration/defs/perf/agentx_client.py index d37634031060..57afa27d60ac 100644 --- a/tests/integration/defs/perf/agentx_client.py +++ b/tests/integration/defs/perf/agentx_client.py @@ -484,10 +484,19 @@ def _run(args: argparse.Namespace, artifact_dir: str) -> int: verify_endpoint(url, args.model) cmd = build_aiperf_cmd(args, url, artifact_dir) + export_path = os.path.join(artifact_dir, "profile_export_aiperf.json") + # Drop any export left by an earlier invocation that reused this directory + # (a retried lane, or a local rerun pointed at the same --output-dir). + # Without this, a crashed aiperf would be validated against -- and would + # report -- the previous run's numbers, turning a failure into a green + # result carrying stale metrics. Removing it first makes the existence + # check below a statement about *this* run. + if os.path.exists(export_path): + _log(f"removing export from a previous run: {export_path}") + os.remove(export_path) _log("running: " + " ".join(cmd)) completed = subprocess.run(cmd) - export_path = os.path.join(artifact_dir, "profile_export_aiperf.json") if not os.path.exists(export_path): # No export at all means aiperf died before writing results; its exit # status is the only signal left, so surface it. diff --git a/tests/integration/defs/perf/test_perf_sanity.py b/tests/integration/defs/perf/test_perf_sanity.py index 7ee98aea0bb2..c8a8c4358728 100644 --- a/tests/integration/defs/perf/test_perf_sanity.py +++ b/tests/integration/defs/perf/test_perf_sanity.py @@ -2636,12 +2636,27 @@ def check_test_failure(self): # TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS -- already captured # as the l_force_num_accepted_tokens match key -- so 'al' would # be a restatement of a configured constant rather than a - # measurement. Revisit if an agentx lane ever runs spec decoding - # without forcing the accepted count. + # measurement. The exemption is conditioned on that forcing + # actually being in effect rather than merely documented, so an + # agentx lane that ever runs spec decoding without pinning the + # accepted count still hard-fails here. + agentx_al_exempt = False + if ( + client_idx < len(client_configs) + and client_configs[client_idx].benchmark_client == AGENTX_BENCHMARK_CLIENT + ): + server_entry = self.server_configs[server_idx] + # disagg stores (ctx, gen, disagg); aggregated stores one config. + candidates = ( + server_entry if isinstance(server_entry, tuple) else (server_entry,) + ) + agentx_al_exempt = any( + getattr(c, "force_num_accepted_tokens", 0) for c in candidates + ) if ( client_idx < len(client_configs) and client_configs[client_idx].spec_decoding - and client_configs[client_idx].benchmark_client != AGENTX_BENCHMARK_CLIENT + and not agentx_al_exempt and "al" not in metrics ): error_msg += ( From c632da05dfdcefd1aa68bf3152f7d2865c2f9320 Mon Sep 17 00:00:00 2001 From: Chenfei Zhang Date: Thu, 27 Aug 2026 07:51:46 -0700 Subject: [PATCH 03/14] [None][test] Support multi-frontend disagg workers in perf-sanity The AgentX DeepSeek-V4-Pro-DSpark lanes set num_serve_frontends (ctx=8, gen=4) to keep the HTTP frontend from bottlenecking at concurrency 1156/1456. trtllm-serve rejects that outright under the harness's launch scheme: Invalid value: port 0 and --report_addr are only supported with a single serving frontend, but num_serve_frontends=8. The rejection is correct. Attached frontends re-exec the command line verbatim, so with port 0 each binds its own kernel-assigned port instead of sharing one, and each republishes its address, leaving the reader with whichever wrote last. So pick the port on this side when, and only when, the worker config asks for more than one frontend. Reserving it by binding and closing would reopen the window the port-0 scheme exists to close, so the socket stays bound for the process lifetime instead: multi-frontend servers bind with SO_REUSEPORT, and Linux lets same-uid SO_REUSEPORT sockets share a port provided the first binder set the flag. The reservation is therefore transparent to the server while still refusing a plain bind() from anything else on the node, and it is deliberately never listen()ed, since only listening sockets join the kernel's accept load-balancing group -- a bound-only socket holds the port without ever swallowing a request. The harness then publishes host:port on the worker's behalf, atomically, so the disagg server's hostname-file reader is untouched. Publishing before the server is up matches what it replaces rather than loosening it: launch_server publishes at bind time, well before it builds the engine, so a reader could always observe a worker that was still loading weights, and the disagg server's readiness wait is what covers that. Single-frontend configs keep the port-0 path byte for byte; of the 196 ctx/gen role configs under tests/scripts/perf-sanity/disaggregated, only the 4 belonging to these two lanes take the new branch. Signed-off-by: Chenfei Zhang --- .../integration/defs/perf/test_perf_sanity.py | 121 ++++++++++++++++-- 1 file changed, 110 insertions(+), 11 deletions(-) diff --git a/tests/integration/defs/perf/test_perf_sanity.py b/tests/integration/defs/perf/test_perf_sanity.py index c8a8c4358728..b2d16d3e66c3 100644 --- a/tests/integration/defs/perf/test_perf_sanity.py +++ b/tests/integration/defs/perf/test_perf_sanity.py @@ -24,6 +24,7 @@ import shutil import socket import subprocess +import tempfile import time from typing import Dict, List, NamedTuple, Optional, Tuple @@ -668,6 +669,77 @@ def add_host_port_to_cmd(cmd: List[str], host: str, port: int) -> List[str]: return cmd + ["--host", host, "--port", str(port)] +# Ports reserved for multi-frontend servers. Module-level so a reservation is +# never garbage-collected: closing the socket would release the port and reopen +# the very race the reservation exists to close. +_RESERVED_PORT_SOCKETS: List[socket.socket] = [] + + +def reserve_multi_frontend_port(host: str) -> int: + """Reserve a port for a server that runs several HTTP frontends. + + trtllm-serve rejects port 0 / --report_addr when num_serve_frontends > 1: + the extra frontends re-exec the command line verbatim, so with port 0 each + would bind its *own* kernel-assigned port instead of sharing one, and each + would republish its address, leaving the reader with whichever wrote last. + The port therefore has to be chosen on this side. + + Choosing it by binding and closing would reopen exactly the window the port-0 + scheme was introduced to remove -- anything on the node could take the port + between the probe and the server's bind. So the socket stays bound instead. + In multi-frontend mode every frontend binds with SO_REUSEPORT (see + launch_server), and Linux lets same-uid SO_REUSEPORT sockets share a port + provided the *first* binder set the flag, which this one does. So the + reservation is transparent to the server while still refusing a plain bind() + from any unrelated process on the node. + + The socket is deliberately never listen()ed: only *listening* SO_REUSEPORT + sockets join the kernel's accept load-balancing group, so a bound-only socket + holds the port without ever swallowing a request. + """ + # Mirror launch_server's family choice; a reservation in a different address + # family than the server's bind would not share the port. + addr_info = socket.getaddrinfo(host, 0, socket.AF_UNSPEC, socket.SOCK_STREAM) + family = ( + socket.AF_INET6 + if addr_info and all(info[0] == socket.AF_INET6 for info in addr_info) + else socket.AF_INET + ) + sock = socket.socket(family, socket.SOCK_STREAM) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) + sock.bind((host, 0)) + port = sock.getsockname()[1] + _RESERVED_PORT_SOCKETS.append(sock) + print_info(f"Reserved multi-frontend port {host}:{port} (holding SO_REUSEPORT socket)") + return port + + +def publish_addr_file(path: str, host: str, port: int) -> None: + """Write "host:port" to *path* the way trtllm-serve's --report_addr does. + + Used when this side picked the port (multi-frontend), so the disagg server's + hostname-file reader needs no special case. The write is atomic + (temp file in the same directory, then rename) with a ".tmp" suffix: the + reader counts only ".txt" entries, and a partial read on the shared + filesystem these tests coordinate through would be parsed as a URL. + """ + parent = os.path.dirname(os.path.abspath(path)) + os.makedirs(parent, exist_ok=True) + # Bracket IPv6 literals so the value is a usable URL authority: readers build + # "http:///..." from it verbatim. + reported_host = f"[{host}]" if ":" in host else host + fd, tmp_path = tempfile.mkstemp(dir=parent, prefix=os.path.basename(path) + ".", suffix=".tmp") + try: + with os.fdopen(fd, "w") as addr_file: + addr_file.write(f"{reported_host}:{port}\n") + addr_file.flush() + os.fsync(addr_file.fileno()) + os.replace(tmp_path, path) + finally: + if os.path.exists(tmp_path): + os.unlink(tmp_path) + + def _run_benchmark_with_log(cmd: List[str], env: Dict[str, str], log_path: str) -> str: """Run a benchmark while streaming its combined output to an artifact log.""" benchmark_env = env.copy() @@ -826,6 +898,12 @@ def __init__(self, server_config_data: dict, env_vars: str = ""): k: v for k, v in server_config_data.items() if k not in exclude_keys } + # Not a recognized field, so it rides through in extra_llm_api_config_data + # to the engine. Read it out here too: K > 1 HTTP frontends against one + # executor is incompatible with the port-0 launch scheme, so the launcher + # has to know the count before it builds the command line. + self.num_serve_frontends = self.extra_llm_api_config_data.get("num_serve_frontends", 1) + def to_cmd( self, output_dir: str, numa_bind: bool = False, disagg_serving_type: str = "" ) -> List[str]: @@ -1741,10 +1819,6 @@ def run_cmd(self, server_idx: int) -> List[str]: self.server_configs[server_idx] if server_idx < len(self.server_configs) else None ) if "CTX" in self.disagg_serving_type or "GEN" in self.disagg_serving_type: - # port 0 + --report_addr: the worker binds a kernel-assigned port - # and publishes host:port itself, so no port is reserved here and - # left unbound while anything on the node could take it. The disagg - # server reads these files to build its config, exactly as before. hostname_file = self._hostname_file(server_idx) is_ctx = "CTX" in self.disagg_serving_type server_cmd = ctx_cmd if is_ctx else gen_cmd @@ -1754,10 +1828,36 @@ def run_cmd(self, server_idx: int) -> List[str]: config_idx = server_cmd.index("--config") + 1 self._wait_for_config_file(server_cmd[config_idx]) - server_cmd = add_host_port_to_cmd(server_cmd, self.hostname, 0) + [ - "--report_addr", - hostname_file, - ] + worker_cfg = None + if configs_for_idx is not None: + ctx_cfg, gen_cfg, _ = configs_for_idx + worker_cfg = ctx_cfg if is_ctx else gen_cfg + num_frontends = getattr(worker_cfg, "num_serve_frontends", 1) or 1 + + if num_frontends > 1: + # trtllm-serve refuses port 0 / --report_addr with several + # frontends (each would bind a different port), so reserve the + # port here and publish it on the worker's behalf; the disagg + # server's hostname-file reader is unchanged. + # + # Publishing before the server is up matches the semantics this + # replaces rather than loosening them: launch_server publishes at + # *bind* time, well before it constructs the engine, so a reader + # has always been able to see the address of a worker that is + # still loading weights. The disagg server's readiness wait is + # what covers that, and it is untouched here. + worker_port = reserve_multi_frontend_port(self.hostname) + server_cmd = add_host_port_to_cmd(server_cmd, self.hostname, worker_port) + publish_addr_file(hostname_file, self.hostname, worker_port) + else: + # port 0 + --report_addr: the worker binds a kernel-assigned port + # and publishes host:port itself, so no port is reserved here and + # left unbound while anything on the node could take it. The disagg + # server reads these files to build its config, exactly as before. + server_cmd = add_host_port_to_cmd(server_cmd, self.hostname, 0) + [ + "--report_addr", + hostname_file, + ] try: print_info( f"Starting server. disagg_serving_type: {self.disagg_serving_type} cmd is {server_cmd}" @@ -1767,9 +1867,8 @@ def run_cmd(self, server_idx: int) -> List[str]: f"trtllm-serve.{self.disagg_serving_type}.{server_idx}.log", ) worker_env = copy.deepcopy(os.environ) - if configs_for_idx is not None: - ctx_cfg, gen_cfg, _ = configs_for_idx - worker_env.update((ctx_cfg if is_ctx else gen_cfg).to_env()) + if worker_cfg is not None: + worker_env.update(worker_cfg.to_env()) with open(server_file_path, "w") as server_ctx: server_proc = subprocess.Popen( server_cmd, From 987ec1415e799e19cc330a767c28c5b4e361e6f7 Mon Sep 17 00:00:00 2001 From: Chenfei Zhang Date: Thu, 27 Aug 2026 11:39:06 -0700 Subject: [PATCH 04/14] [None][test] Omit d_al on upload when acceptance length is unreported The AgentX 'al' exemption was only half applied. check_test_failure stopped hard-failing an AgentX spec-decoding lane that reports no acceptance length (aiperf does not propagate TRT-LLM's non-standard avg_decoded_tokens_per_iter field), but add_perf_metric_value still indexed metrics['al'] unconditionally whenever spec_decoding=True. A GB300 AgentX lane therefore completed a full 2h47m measurement, produced a valid aiperf export, and then died with KeyError: 'al' while uploading -- losing the row for a run that had succeeded. Omit the column instead. This cannot hide a real gap: check_test_failure() raises RuntimeError and runs immediately before upload_test_results_to_database(), so any non-exempt spec-decoding run whose 'al' is missing has already failed and never reaches this code. Reaching it without 'al' means the run was exempt by design. Omitted rather than defaulted on purpose: typeCheckForOpenSearchDB rejects None for a d_ key and would fail the whole row, and substituting 0.0 would corrupt the spec-decoding baseline population. Signed-off-by: Chenfei Zhang --- .../integration/defs/perf/test_perf_sanity.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/tests/integration/defs/perf/test_perf_sanity.py b/tests/integration/defs/perf/test_perf_sanity.py index b2d16d3e66c3..7d66848cbe2b 100644 --- a/tests/integration/defs/perf/test_perf_sanity.py +++ b/tests/integration/defs/perf/test_perf_sanity.py @@ -547,8 +547,10 @@ def add_perf_metric_value( """Populate `new_data` with per-test perf metrics from `metrics`. - Always copies every key in PERF_METRIC_LOG_QUERIES as `d_`. - - Adds `d_al` only when spec_decoding=True; non-spec rows omit it so - OpenSearch baselines don't blend the two populations. + - Adds `d_al` only when spec_decoding=True *and* the value was parsed; + non-spec rows omit it so OpenSearch baselines don't blend the two + populations, and spec rows exempted from reporting it (AgentX) omit it + rather than failing the upload. - Adds the `d_*_gen_worker_per_iter_device_step_time` family only for the disagg gen_only mode (the only mode that emits them). Of these the mean and the median are regression-gated (GEN_ONLY_REGRESSION_METRICS); the @@ -563,7 +565,17 @@ def add_perf_metric_value( for metric_name in PERF_METRIC_LOG_QUERIES: new_data[f"d_{metric_name}"] = metrics[metric_name] if spec_decoding: - new_data["d_al"] = metrics["al"] + # 'al' is legitimately absent for AgentX lanes: aiperf does not propagate + # TRT-LLM's non-standard avg_decoded_tokens_per_iter field. Omit the + # column instead of raising -- check_test_failure runs immediately before + # upload and has already hard-failed any non-exempt spec-decoding run + # whose 'al' is missing, so reaching here without it means the run is + # exempt by design. Omitted rather than defaulted: typeCheckForOpenSearchDB + # rejects None for a d_ key (losing the whole row), and a substituted 0.0 + # would corrupt the spec-decoding baseline population. + al = metrics.get("al") + if al is not None: + new_data["d_al"] = al if benchmark_mode == "gen_only": for metric_name in GEN_ONLY_DEVICE_STEP_TIME_METRICS: value = metrics.get(metric_name) From ef9491a2507d0e0f427e6ab96c5061b64b997c69 Mon Sep 17 00:00:00 2001 From: Chenfei Zhang Date: Fri, 28 Aug 2026 01:44:38 -0700 Subject: [PATCH 05/14] [None][test] Drop inert config keys from the AgentX perf-sanity lanes The trailing ctx_router_config / server_config_extra block was carried over from the bench-trtllm-disagg harness, where both keys are live: its submit.py turns ctx_router_config into context_servers.router and merges server_config_extra into the generated disagg server config. perf-sanity has no equivalent. test_perf_sanity.py builds the disagg server config as a literal dict of hostname, port, backend, internal_request_auth_key, context_servers and generation_servers, and never reads a router key, so these settings were silently ignored -- the server_config.0.yaml generated by a passing run contains none of them. Removing them so the file no longer describes behaviour that is not configured. This leaves a real gap against the reference recipe, whose generated server config does carry context_servers.router.type=conversation plus the four server_config_extra keys. Closing that gap needs perf-sanity harness support for a router key, not a config-file edit, so it is left for a follow-up. Also drops the explanatory comments; the two lanes now match the top-level shape of the other disaggregated perf-sanity configs. Signed-off-by: Chenfei Zhang --- ...dep8_gen1_dep8_eplb0_dspark3_ccb-NIXL.yaml | 21 ------------------- ...ep8_gen1_dep16_eplb0_dspark5_ccb-NIXL.yaml | 21 ------------------- 2 files changed, 42 deletions(-) diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v4-pro-dspark_agentx_con1156_ctx2_dep8_gen1_dep8_eplb0_dspark3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v4-pro-dspark_agentx_con1156_ctx2_dep8_gen1_dep8_eplb0_dspark3_ccb-NIXL.yaml index 8239e73d860a..14ab5103aaa2 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v4-pro-dspark_agentx_con1156_ctx2_dep8_gen1_dep8_eplb0_dspark3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v4-pro-dspark_agentx_con1156_ctx2_dep8_gen1_dep8_eplb0_dspark3_ccb-NIXL.yaml @@ -10,9 +10,6 @@ slurm: script_file: disaggr_torch.slurm partition: account: - # AgentX replays a 1M-context multi-turn trace for AGENTX_DURATION seconds - # (3600) on top of a DeepSeek-V4-Pro weight load, so this needs more than the - # 02:00:00 the fixed-shape 8k1k lanes use. job_time: 04:00:00 job_name: unified-benchmark extra_args: --gres=gpu:4 @@ -27,10 +24,6 @@ benchmark: concurrency_list: '1156' input_length: 990016 output_length: 6579 - # AgentX resolves this as a HuggingFace dataset name - # (semianalysisai/cc-traces-weka-062126, public + Apache-2.0), NOT a local - # path, so it is a literal name rather than the placeholder the - # random-synthetic lanes use. dataset_file: semianalysis_cc_traces_weka_062126 hardware: gpus_per_node: 4 @@ -43,8 +36,6 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - # AGENTX_* are consumed by the agentx client in test_perf_sanity.py. Values - # are space-separated KEY=val, so none of them may contain a space. client_env_var: AGENTX_MAX_CTX=996579 AGENTX_DURATION=3600 AGENTX_WARMUP_PER_LANE=3 worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 ENROOT_ALLOW_DEV=yes NCCL_GRAPH_MIXING_SUPPORT=0 MIMALLOC_ARENA_RESERVE=0 OMP_NUM_THREADS=1 TRTLLM_PINNED_WEIGHT_STAGING=1 ctx_worker_env_var: PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True CUDA_SCALE_LAUNCH_QUEUES=4x TLLM_INDEXER_MQA_LOGITS_ELEM_BUDGET=1073741824 TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2.08 @@ -87,9 +78,6 @@ worker_config: decoding_type: DSpark max_draft_len: 3 block_size: 3 - # Resolved against llm_models_root() by generate_extra_llm_api_config(). - # DSpark ships its draft inside the target checkpoint, so this is the same - # directory as the target model. speculative_model: DeepSeek-V4-Pro-DSpark custom_tokenizer: deepseek_v4 return_perf_metrics: true @@ -111,8 +99,6 @@ worker_config: cuda_graph_config: null disable_overlap_scheduler: false kv_cache_config: - # Conversation-scoped reuse is the point of the agentic workload: each - # replayed turn re-sends the whole prior conversation. enable_block_reuse: true free_gpu_memory_fraction: 0.8 dtype: fp8 @@ -139,10 +125,3 @@ worker_config: scheduler_config: capacity_scheduler_policy: MAX_UTILIZATION num_serve_frontends: 8 -ctx_router_config: - type: conversation -server_config_extra: - gen_strip_message_history: true - gen_tokids_ctxbytes: true - server_keep_alive_timeout: 3600 - num_workers: 4 diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v4-pro-dspark_agentx_con1456_ctx3_dep8_gen1_dep16_eplb0_dspark5_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v4-pro-dspark_agentx_con1456_ctx3_dep8_gen1_dep16_eplb0_dspark5_ccb-NIXL.yaml index fdaa6601db60..3b3be5dec8bb 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v4-pro-dspark_agentx_con1456_ctx3_dep8_gen1_dep16_eplb0_dspark5_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v4-pro-dspark_agentx_con1456_ctx3_dep8_gen1_dep16_eplb0_dspark5_ccb-NIXL.yaml @@ -10,9 +10,6 @@ slurm: script_file: disaggr_torch.slurm partition: account: - # AgentX replays a 1M-context multi-turn trace for AGENTX_DURATION seconds - # (3600) on top of a DeepSeek-V4-Pro weight load, so this needs more than the - # 02:00:00 the fixed-shape 8k1k lanes use. job_time: 04:00:00 job_name: unified-benchmark extra_args: --gres=gpu:4 @@ -27,10 +24,6 @@ benchmark: concurrency_list: '1456' input_length: 990016 output_length: 6579 - # AgentX resolves this as a HuggingFace dataset name - # (semianalysisai/cc-traces-weka-062126, public + Apache-2.0), NOT a local - # path, so it is a literal name rather than the placeholder the - # random-synthetic lanes use. dataset_file: semianalysis_cc_traces_weka_062126 hardware: gpus_per_node: 4 @@ -43,8 +36,6 @@ environment: trtllm_repo: '' build_wheel: false work_dir: - # AGENTX_* are consumed by the agentx client in test_perf_sanity.py. Values - # are space-separated KEY=val, so none of them may contain a space. client_env_var: AGENTX_MAX_CTX=996579 AGENTX_DURATION=3600 AGENTX_WARMUP_PER_LANE=3 worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 ENROOT_ALLOW_DEV=yes NCCL_GRAPH_MIXING_SUPPORT=0 MIMALLOC_ARENA_RESERVE=0 OMP_NUM_THREADS=1 TRTLLM_PINNED_WEIGHT_STAGING=1 ctx_worker_env_var: PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True CUDA_SCALE_LAUNCH_QUEUES=4x TLLM_INDEXER_MQA_LOGITS_ELEM_BUDGET=1073741824 TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2.74 @@ -87,9 +78,6 @@ worker_config: decoding_type: DSpark max_draft_len: 5 block_size: 5 - # Resolved against llm_models_root() by generate_extra_llm_api_config(). - # DSpark ships its draft inside the target checkpoint, so this is the same - # directory as the target model. speculative_model: DeepSeek-V4-Pro-DSpark custom_tokenizer: deepseek_v4 return_perf_metrics: true @@ -111,8 +99,6 @@ worker_config: cuda_graph_config: null disable_overlap_scheduler: false kv_cache_config: - # Conversation-scoped reuse is the point of the agentic workload: each - # replayed turn re-sends the whole prior conversation. enable_block_reuse: true free_gpu_memory_fraction: 0.8 dtype: fp8 @@ -139,10 +125,3 @@ worker_config: scheduler_config: capacity_scheduler_policy: MAX_UTILIZATION num_serve_frontends: 8 -ctx_router_config: - type: conversation -server_config_extra: - gen_strip_message_history: true - gen_tokids_ctxbytes: true - server_keep_alive_timeout: 3600 - num_workers: 4 From feeae5420ecba4129520c1d4b19a0eb23e4ee993 Mon Sep 17 00:00:00 2001 From: Chenfei Zhang Date: Fri, 28 Aug 2026 03:04:47 -0700 Subject: [PATCH 06/14] [None][test] Support disagg router and extra server config keys in perf-sanity Follow-up to the previous commit, which dropped ctx_router_config and server_config_extra from the AgentX lanes because perf-sanity ignored them. This adds the harness support they were missing, under the same key names bench-trtllm-disagg/trtllm-disagg-benchmark uses, so a recipe can be carried over unchanged: router_config -> router on both roles (its submit.py:429-430) ctx_router_config -> router on context_servers (submit.py:440-441) gen_router_config -> router on generation_servers (submit.py:432-433) server_config_extra -> merged into the server config (submit.py:1038-1040) Precedence follows that harness: a generic router_config applies to both roles, a role-specific key overrides it for its role, and server_config_extra is merged last so it wins over everything above it. One deliberate divergence: perf-sanity rejects a server_config_extra carrying port, hostname, internal_request_auth_key, context_servers or generation_servers. bench-trtllm-disagg pins a fixed port and a static url list, whereas perf-sanity binds port 0 and has each server report its bound address back through --report_addr, then discovers the urls from the hostname files the tasks exchange. Overriding those keys would surface as a startup hang or as a benchmark run against the wrong endpoint, a long way from the cause, so it raises instead. All four keys are optional and default to None, and a config that sets none of them produces the same server config as before, so existing lanes are unaffected. Signed-off-by: Chenfei Zhang --- .../integration/defs/perf/test_perf_sanity.py | 65 +++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/tests/integration/defs/perf/test_perf_sanity.py b/tests/integration/defs/perf/test_perf_sanity.py index 7d66848cbe2b..678aa6dcf47f 100644 --- a/tests/integration/defs/perf/test_perf_sanity.py +++ b/tests/integration/defs/perf/test_perf_sanity.py @@ -1400,6 +1400,10 @@ def __init__( hardware: dict, server_env_var: str, internal_request_auth_key: str | None = None, + router_config: dict | None = None, + ctx_router_config: dict | None = None, + gen_router_config: dict | None = None, + server_config_extra: dict | None = None, ): self.name = name self.disagg_serving_type = disagg_serving_type @@ -1411,6 +1415,10 @@ def __init__( self.hardware = hardware self.server_env_var = server_env_var self.internal_request_auth_key = internal_request_auth_key + self.router_config = router_config + self.ctx_router_config = ctx_router_config + self.gen_router_config = gen_router_config + self.server_config_extra = server_config_extra self.num_ctx_servers = hardware.get("num_ctx_servers", 0) self.num_gen_servers = hardware.get("num_gen_servers", 0) @@ -1559,6 +1567,12 @@ class DisaggTestCmds(NamedTuple): # disagg, only rank-0 pytest goes through this path; multi-rank workers # receive env via SLURM env propagation set up by submit.py. server_configs: List[Tuple["ServerConfig", "ServerConfig", "DisaggConfig"]] = [] + # Disagg-server-level keys, named as in bench-trtllm-disagg. A generic + # router applies to both roles; a role-specific one overrides it. + router_config: Optional[dict] = None + ctx_router_config: Optional[dict] = None + gen_router_config: Optional[dict] = None + server_config_extra: Optional[dict] = None def _hostnames_dir(self, server_idx: int) -> str: """Directory the disagg tasks exchange bound addresses through. @@ -1637,6 +1651,43 @@ def _generate_disagg_server_config(self, server_idx: int) -> str: "urls": gen_hostnames, }, } + # Router selection, mirroring bench-trtllm-disagg's submit.py: a generic + # router applies to both roles and a role-specific one overrides it for + # that role, e.g. ctx_router_config={"type": "conversation"} puts a + # conversation router only on the context servers. Deep-copied because + # trtllm-serve pops keys out of this dict while parsing it. + if self.router_config: + server_config["context_servers"]["router"] = copy.deepcopy(self.router_config) + server_config["generation_servers"]["router"] = copy.deepcopy(self.router_config) + if self.ctx_router_config: + server_config["context_servers"]["router"] = copy.deepcopy(self.ctx_router_config) + if self.gen_router_config: + server_config["generation_servers"]["router"] = copy.deepcopy(self.gen_router_config) + + if self.server_config_extra: + # Merged last, as bench-trtllm-disagg does, so it wins over + # everything above. The reserved keys are the exception: the harness + # owns them, not the config file. port must stay 0 so the server + # binds a kernel-assigned port and reports it via --report_addr, and + # the url lists are discovered from the hostname files above. + # Overriding either surfaces as a hang or a benchmark against the + # wrong endpoint, a long way from the cause, so reject it here. + reserved = { + "port", + "hostname", + "internal_request_auth_key", + "context_servers", + "generation_servers", + } + clobbered = sorted(reserved & set(self.server_config_extra)) + if clobbered: + raise RuntimeError( + f"server_config_extra may not override harness-owned keys {clobbered}: " + "the port is kernel-assigned and reported back via --report_addr, and " + "the server urls are discovered at runtime." + ) + server_config.update(copy.deepcopy(self.server_config_extra)) + config_path = os.path.join(self.test_output_dir, f"server_config.{server_idx}.yaml") with open(config_path, "w") as f: yaml.dump(server_config, f) @@ -2332,6 +2383,12 @@ def _parse_disagg_config_file(self, config_file_path: str, config_file: str): server_env_var = environment.get("server_env_var", "") client_env_var = environment.get("client_env_var", "") internal_request_auth_key = self._resolve_internal_request_auth_key(config) + # Optional disagg-server-level keys, same names as bench-trtllm-disagg's + # sweep config so a recipe can be carried over unchanged. + router_config = config.get("router_config", None) + ctx_router_config = config.get("ctx_router_config", None) + gen_router_config = config.get("gen_router_config", None) + server_config_extra = config.get("server_config_extra", None) # Parse concurrency_list - can be string or list concurrency_str = benchmark.get("concurrency_list", "1") @@ -2404,6 +2461,10 @@ def _parse_disagg_config_file(self, config_file_path: str, config_file: str): hardware=hardware, server_env_var=server_env_var, internal_request_auth_key=internal_request_auth_key, + router_config=router_config, + ctx_router_config=ctx_router_config, + gen_router_config=gen_router_config, + server_config_extra=server_config_extra, ) # server_configs is a list with one element (tuple of ctx, gen, disagg config) @@ -2604,6 +2665,10 @@ def _get_disagg_commands(self, output_dir: str, test_output_dir: str): test_output_dir=test_output_dir, model_name=disagg_config.model_name, internal_request_auth_key=disagg_config.internal_request_auth_key, + router_config=disagg_config.router_config, + ctx_router_config=disagg_config.ctx_router_config, + gen_router_config=disagg_config.gen_router_config, + server_config_extra=disagg_config.server_config_extra, client_configs=self.server_client_configs, server_configs=list(self.server_configs), ) From dedb501f51a9cdb0d0d21977aa408ab88afaae2c Mon Sep 17 00:00:00 2001 From: Chenfei Zhang Date: Fri, 28 Aug 2026 03:18:32 -0700 Subject: [PATCH 07/14] [None][test] Restore the disagg router config on the AgentX perf-sanity lanes The previous two commits removed these keys because perf-sanity ignored them, then taught the harness to honour them. This puts them back, so both AgentX lanes now match the routing and disagg-server settings of the reference bench-trtllm-disagg recipe they were derived from: ctx_router_config.type: conversation A conversation router on the context servers only. This is the piece the lanes were missing: the ctx worker_config already sets attention_dp_config.kv_cache_routing_conversation_affinity and a per_conversation block reuse policy, but with the default round_robin router the frontend spread a conversation's turns across ctx servers, so the affinity had nothing to act on. Every ConversationRouter argument has a default, so the bare type is sufficient. server_config_extra gen_strip_message_history and gen_tokids_ctxbytes change what the disagg service forwards to the generation server; server_keep_alive_timeout raises the frontend keep-alive from its default of 10s to 3600s, matching the 3600s AGENTX_DURATION; num_workers runs the disagg frontend as a delegating fleet rather than a single self-contained server. All four are DisaggServerConfig fields, read by openai_disagg_service.py and serve.py. These change the configuration under test, so the perf numbers currently in the PR description do not describe this config; they are being re-measured. Signed-off-by: Chenfei Zhang --- ...con1156_ctx2_dep8_gen1_dep8_eplb0_dspark3_ccb-NIXL.yaml | 7 +++++++ ...on1456_ctx3_dep8_gen1_dep16_eplb0_dspark5_ccb-NIXL.yaml | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v4-pro-dspark_agentx_con1156_ctx2_dep8_gen1_dep8_eplb0_dspark3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v4-pro-dspark_agentx_con1156_ctx2_dep8_gen1_dep8_eplb0_dspark3_ccb-NIXL.yaml index 14ab5103aaa2..f0f758d358ed 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v4-pro-dspark_agentx_con1156_ctx2_dep8_gen1_dep8_eplb0_dspark3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v4-pro-dspark_agentx_con1156_ctx2_dep8_gen1_dep8_eplb0_dspark3_ccb-NIXL.yaml @@ -43,6 +43,13 @@ environment: server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false +ctx_router_config: + type: conversation +server_config_extra: + gen_strip_message_history: true + gen_tokids_ctxbytes: true + server_keep_alive_timeout: 3600 + num_workers: 4 worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v4-pro-dspark_agentx_con1456_ctx3_dep8_gen1_dep16_eplb0_dspark5_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v4-pro-dspark_agentx_con1456_ctx3_dep8_gen1_dep16_eplb0_dspark5_ccb-NIXL.yaml index 3b3be5dec8bb..36887fe4a350 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v4-pro-dspark_agentx_con1456_ctx3_dep8_gen1_dep16_eplb0_dspark5_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v4-pro-dspark_agentx_con1456_ctx3_dep8_gen1_dep16_eplb0_dspark5_ccb-NIXL.yaml @@ -43,6 +43,13 @@ environment: server_env_var: TRTLLM_SERVER_DISABLE_GC=1 profiling: nsys_on: false +ctx_router_config: + type: conversation +server_config_extra: + gen_strip_message_history: true + gen_tokids_ctxbytes: true + server_keep_alive_timeout: 3600 + num_workers: 4 worker_config: gen: print_iter_log: true From 1a1630480c7a3b4f23f2b73cbf9119fab706c449 Mon Sep 17 00:00:00 2001 From: Chenfei Zhang Date: Sat, 29 Aug 2026 23:44:05 -0700 Subject: [PATCH 08/14] [None][test] Reject disagg fleet keys in perf-sanity and drop num_workers num_workers > 1, or a disagg_coordinator_url, puts trtllm-serve disaggregated into fleet mode. trtllm-serve refuses to combine that with the port-0 plus report_addr address discovery perf-sanity depends on, and the refusal is correct: a fleet hands one port to N SO_REUSEPORT workers, so with port 0 each worker would get a different kernel-assigned port and the published address would serve only 1/N of requests. Observed on aws-cmh GB300 (job 3370429): the AgentX 24-GPU lane failed 31s in with "DISAGG_SERVER server exited unexpectedly with code 2". That message names neither the key nor the mechanism; the only real signal was a click.BadParameter buried in the captured server log. So reject the combination during config generation, where the offending key can be named and the fix stated, and drop num_workers from the two AgentX lanes. num_workers: 1 stays legal. ctx_router_config and the three remaining server_config_extra keys are unaffected, and were confirmed on the same run to reach the generated server_config: router type conversation appears under context_servers only, with the extras at top level and the harness-owned port, hostname, auth key and discovered urls intact. Signed-off-by: Chenfei Zhang --- .../integration/defs/perf/test_perf_sanity.py | 23 +++++++++++++++++++ ...dep8_gen1_dep8_eplb0_dspark3_ccb-NIXL.yaml | 1 - ...ep8_gen1_dep16_eplb0_dspark5_ccb-NIXL.yaml | 1 - 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/tests/integration/defs/perf/test_perf_sanity.py b/tests/integration/defs/perf/test_perf_sanity.py index 678aa6dcf47f..ddae7fe91fc6 100644 --- a/tests/integration/defs/perf/test_perf_sanity.py +++ b/tests/integration/defs/perf/test_perf_sanity.py @@ -1686,6 +1686,29 @@ def _generate_disagg_server_config(self, server_idx: int) -> str: "the port is kernel-assigned and reported back via --report_addr, and " "the server urls are discovered at runtime." ) + # Distinct from the above: these keys override nothing, but they put + # trtllm-serve into fleet mode, which it refuses to combine with the + # port-0 + --report_addr discovery this harness depends on (see + # tensorrt_llm/commands/serve.py, "single self-contained + # disaggregated server"). A fleet hands one port to N SO_REUSEPORT + # workers; with port 0 each would get a *different* kernel-assigned + # port, so the reported address would serve 1/N of requests. + # Rejected here so the cause is named at config time instead of + # surfacing ~30s later as "DISAGG_SERVER server exited unexpectedly + # with code 2", which points nowhere near this key. + fleet_keys = sorted( + {"num_workers", "disagg_coordinator_url"} & set(self.server_config_extra) + ) + if fleet_keys and ( + self.server_config_extra.get("num_workers", 1) > 1 + or self.server_config_extra.get("disagg_coordinator_url") + ): + raise RuntimeError( + f"server_config_extra sets {fleet_keys}, which selects a disaggregated " + "server fleet. perf-sanity binds port 0 and discovers the address via " + "--report_addr, and trtllm-serve rejects that combination. Remove the " + "key, or teach the harness to bind a fixed port first." + ) server_config.update(copy.deepcopy(self.server_config_extra)) config_path = os.path.join(self.test_output_dir, f"server_config.{server_idx}.yaml") diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v4-pro-dspark_agentx_con1156_ctx2_dep8_gen1_dep8_eplb0_dspark3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v4-pro-dspark_agentx_con1156_ctx2_dep8_gen1_dep8_eplb0_dspark3_ccb-NIXL.yaml index f0f758d358ed..4adc07545af2 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v4-pro-dspark_agentx_con1156_ctx2_dep8_gen1_dep8_eplb0_dspark3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v4-pro-dspark_agentx_con1156_ctx2_dep8_gen1_dep8_eplb0_dspark3_ccb-NIXL.yaml @@ -49,7 +49,6 @@ server_config_extra: gen_strip_message_history: true gen_tokids_ctxbytes: true server_keep_alive_timeout: 3600 - num_workers: 4 worker_config: gen: print_iter_log: true diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v4-pro-dspark_agentx_con1456_ctx3_dep8_gen1_dep16_eplb0_dspark5_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v4-pro-dspark_agentx_con1456_ctx3_dep8_gen1_dep16_eplb0_dspark5_ccb-NIXL.yaml index 36887fe4a350..ee32a067e580 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v4-pro-dspark_agentx_con1456_ctx3_dep8_gen1_dep16_eplb0_dspark5_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v4-pro-dspark_agentx_con1456_ctx3_dep8_gen1_dep16_eplb0_dspark5_ccb-NIXL.yaml @@ -49,7 +49,6 @@ server_config_extra: gen_strip_message_history: true gen_tokids_ctxbytes: true server_keep_alive_timeout: 3600 - num_workers: 4 worker_config: gen: print_iter_log: true From 84d99e31f508a572720b7865f54e96597f407d00 Mon Sep 17 00:00:00 2001 From: Chenfei Zhang Date: Sun, 30 Aug 2026 17:34:51 -0700 Subject: [PATCH 09/14] [None][test] Keep the 40-GPU agentx lane out of perf-sanity CI The AgentX DSpark trace-replay lanes were registered in two sizes: 24 GPUs / 6 nodes (ctx2+gen1) and 40 GPUs / 10 nodes (ctx3+gen1). A post-merge run of the larger one holds 10 GB300 nodes for the lane's full wall-clock-bounded replay, which is more capacity than this coverage is worth: the 24-GPU lane already exercises the same model, trace corpus, conversation-router path and disagg topology, and the 40-GPU case differs only in ctx/gen worker counts. So de-register the 40-GPU case while keeping its config: - drop its test-db list (nothing selects it, so no stage runs it) - drop its buildStageConfigs declaration - drop its .test_durations entry Its config YAML is deliberately left in place. Config YAMLs under tests/scripts/perf-sanity/disaggregated/ produce test ids just by existing -- PERF_SANITY_TEST_CASES globs the directory at import time -- so the case stays collectable and runnable on demand via jenkins/scripts/perf/local/submit.py. Only automatic CI selection goes away, which is the part that costs GPUs. The surviving 24-GPU stage keeps testCount=1 against a one-test block, as the disagg e2e path requires (submit.py raises unless the split group resolves to exactly one case). Signed-off-by: Chenfei Zhang --- jenkins/L0_Test.groovy | 9 -------- tests/integration/defs/.test_durations | 1 - ...anity_ctx3_node2_gpu8_gen1_node4_gpu16.yml | 21 ------------------- 3 files changed, 31 deletions(-) delete mode 100644 tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx3_node2_gpu8_gen1_node4_gpu16.yml diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 2bc190304316..3911a2e9151d 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -6570,15 +6570,6 @@ def launchTestJobs(pipeline, testFilter, globalVars) 24, 6 ) - // 10 Nodes: ctx3 (2 nodes, 8 GPUs each) + gen1 (4 nodes, 16 GPUs) = 40 GPUs - multiNodesSBSAConfigs += buildStageConfigs( - "GB300-40_GPUs-10_Nodes-PyTorch-Disagg-PerfSanity-AgentX-CTX3-NODE2-GPU8-GEN1-NODE4-GPU16-Post-Merge", - "gb300-flex-aws-cmh", - "l0_gb300_multi_nodes_perf_sanity_ctx3_node2_gpu8_gen1_node4_gpu16", - 1, - 40, - 10 - ) multiNodesSBSAConfigs = cbtsResizeSplits(multiNodesSBSAConfigs) fullSet += multiNodesSBSAConfigs.keySet() diff --git a/tests/integration/defs/.test_durations b/tests/integration/defs/.test_durations index 4fba998faf46..30d5b9d69b81 100644 --- a/tests/integration/defs/.test_durations +++ b/tests/integration/defs/.test_durations @@ -931,7 +931,6 @@ "perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL]": 7666.37925, "perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL]": 2542.9724666666666, "perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_deepseek-v4-pro-dspark_agentx_con1156_ctx2_dep8_gen1_dep8_eplb0_dspark3_ccb-NIXL]": 5400.0, - "perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_deepseek-v4-pro-dspark_agentx_con1456_ctx3_dep8_gen1_dep16_eplb0_dspark5_ccb-NIXL]": 5400.0, "perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_glm-5-fp4_8k1k_con1024_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-NIXL]": 3729.2565, "perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_nemotron-ultra-v3-fp4_50k2k_con12_ctx1_dep4_gen6_tep4_eplb0_mtp6_ccb-NIXL]": 1731.7142, "perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_nemotron-ultra-v3-fp4_50k2k_con178_ctx5_dep4_gen1_dep4_eplb0_mtp6_ccb-NIXL]": 1023.3791666666666, diff --git a/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx3_node2_gpu8_gen1_node4_gpu16.yml b/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx3_node2_gpu8_gen1_node4_gpu16.yml deleted file mode 100644 index 485ac2030b10..000000000000 --- a/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx3_node2_gpu8_gen1_node4_gpu16.yml +++ /dev/null @@ -1,21 +0,0 @@ -version: 0.0.1 -l0_gb300_multi_nodes_perf_sanity_ctx3_node2_gpu8_gen1_node4_gpu16: -- condition: - ranges: - # 3 ctx workers each with 2 nodes and 8 GPUs - # 1 gen worker with 4 nodes and 16 GPUs - system_gpu_count: - gte: 40 - lte: 40 - wildcards: - gpu: - - '*gb300*' - terms: - stage: post_merge - backend: pytorch - tests: - # deepseek-v4-pro-dspark agentx con1456. - # e2e only -- see the sibling ctx2_node2_gpu8_gen1_node2_gpu8 list for why - # agentx gen_only needs the gen_only_no_context flavor and is left to a - # follow-up. - - perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_deepseek-v4-pro-dspark_agentx_con1456_ctx3_dep8_gen1_dep16_eplb0_dspark5_ccb-NIXL] TIMEOUT (210) From b735880d061b90c27c576d959a4b4247ef05c5fd Mon Sep 17 00:00:00 2001 From: Chenfei Zhang Date: Sun, 30 Aug 2026 21:26:08 -0700 Subject: [PATCH 10/14] [None][test] align agentx perf-sanity docs with test-case-name matching PR #18408 re-keyed perf-sanity case identity onto s_test_case_name (plus GPU type, runtime and branch), deleting the per-server/per-client match key tuples. This branch had added s_benchmark_client as a client match key so an agentx lane would form its own baseline population; that key is now redundant, because a disaggregated case name embeds its config stem and the agentx stems contain 'agentx', so those lanes are already separated by name. The match key itself was dropped during the rebase. This commit updates the four comments and doc paragraphs that still described the old behaviour: - README: the case-separation paragraph now explains separation by s_test_case_name, and notes s_benchmark_client is uploaded for reporting only. - README: TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS is no longer described as a match key, since l_force_num_accepted_tokens is not one after #18408. - test_perf_sanity.py: the ClientConfig and to_db_data comments no longer call the empty default load-bearing for matching. No behaviour change; s_benchmark_client is still emitted to the database. Signed-off-by: Chenfei Zhang --- .../defs/perf/README_test_perf_sanity.md | 14 +++++----- .../integration/defs/perf/test_perf_sanity.py | 27 ++++++++++--------- 2 files changed, 22 insertions(+), 19 deletions(-) diff --git a/tests/integration/defs/perf/README_test_perf_sanity.md b/tests/integration/defs/perf/README_test_perf_sanity.md index 6883bb829aca..7bd773bd28fc 100644 --- a/tests/integration/defs/perf/README_test_perf_sanity.md +++ b/tests/integration/defs/perf/README_test_perf_sanity.md @@ -117,11 +117,13 @@ pass. `process_and_upload_test_results` therefore looks history up against a baseline branch — `PERF_BASELINE_BRANCH`, default `main` — using a lookup-only copy of the data. The uploaded documents keep their true `s_branch`. -`ClientConfig.to_match_keys()` includes `s_benchmark_client`, so a lane driven by a -different load generator forms its own baseline population instead of being compared -against `benchmark_serving` history. The built-in client uploads `""` (not `"default"`) -for this field, and `benchmark_data_matches` treats absent and empty as equal, so every -baseline recorded before the field existed keeps matching. +Case identity is keyed on `s_test_case_name` (plus GPU type, runtime and branch), and a +disaggregated case name embeds its config stem — so an agentx lane, whose stem contains +`agentx`, already forms its own baseline population without needing an extra match key. +`s_benchmark_client` is still uploaded, as a reportable record of which load generator +drove the lane, but it does not participate in matching. The built-in client uploads `""` +(not `"default"`) so the column reads consistently against records written before the +field existed. ## Benchmark Clients @@ -145,7 +147,7 @@ config-parse time rather than silently falling through to the default client. - `al` (Mean Avg Decoded Tokens per Iter) is exempt from the spec-decoding hard-fail for agentx lanes: it derives from a TRT-LLM-specific per-response field that `aiperf` does not propagate. This loses no signal as long as the lane pins the accepted length with - `TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS` (already a match key), which makes `al` a + `TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS` (set per-yaml), which makes `al` a restatement of a configured constant rather than a measurement. ## Overview diff --git a/tests/integration/defs/perf/test_perf_sanity.py b/tests/integration/defs/perf/test_perf_sanity.py index ddae7fe91fc6..f99eaeee2535 100644 --- a/tests/integration/defs/perf/test_perf_sanity.py +++ b/tests/integration/defs/perf/test_perf_sanity.py @@ -666,11 +666,11 @@ def force_num_accepted_tokens_from_env_str(env_vars: str) -> int: Returns 0 when not set. The runtime accepts a fractional value (see get_force_num_accepted_tokens_float - in tensorrt_llm), so parse as float first and truncate. The return value only - feeds the l_force_num_accepted_tokens baseline match key, which is a long, so - two lanes differing solely in the fractional part share a match identity -- - acceptable today because every such lane also differs in concurrency and - parallelism, but widen the key if that ever stops holding. + in tensorrt_llm), so parse as float first and truncate. The return value is + uploaded as l_force_num_accepted_tokens, which is a long, so a fractional + setting is not preserved in the record. It is reported rather than matched on: + case identity is keyed on the test case name, so two lanes differing solely in + the fractional part are already separate cases by name. """ val = to_env_dict(env_vars).get("TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS") return int(float(val)) if val is not None else 0 @@ -1202,8 +1202,8 @@ def __init__( self.use_nv_sa_benchmark = client_config_data.get("use_nv_sa_benchmark", False) # Which load generator drives the lane. "" selects the built-in # benchmark_serving client; "agentx" selects the trace-replay client in - # agentx_client.py. The empty default is load-bearing for baseline - # matching -- see the s_benchmark_client note in to_db_data. + # agentx_client.py. Reported only -- see the s_benchmark_client note in + # to_db_data for why it is not a match key. self.benchmark_client = client_config_data.get("benchmark_client", "") self.env_vars = env_vars # spec_decoding flag is retained for DB matching (b_eos column). --ignore-eos @@ -1369,10 +1369,11 @@ def to_db_data(self) -> dict: "b_streaming": self.streaming, "b_trust_remote_code": self.trust_remote_code, "b_use_nv_sa_benchmark": self.use_nv_sa_benchmark, - # Match key. Deliberately uploaded as "" (not "default") for the - # built-in client: benchmark_data_matches treats absent and empty - # as equal, so every baseline recorded before this field existed - # keeps matching. Only named clients form their own population. + # Reported, not matched. Case identity is keyed on s_test_case_name + # (plus GPU type, runtime, branch), and a disagg case name embeds its + # config stem, so an agentx lane already forms its own population by + # name. Uploaded as "" (not "default") for the built-in client so the + # column reads consistently against records written before this field. "s_benchmark_client": self.benchmark_client, "b_eos": self.spec_decoding, "s_client_log_link": "", @@ -2832,8 +2833,8 @@ def check_test_failure(self): # avg_decoded_tokens_per_iter response field, which aiperf does # not propagate. It is not a real loss of signal, because every # agentx lane pins the accepted length with - # TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS -- already captured - # as the l_force_num_accepted_tokens match key -- so 'al' would + # TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS -- recorded on the + # case as l_force_num_accepted_tokens -- so 'al' would # be a restatement of a configured constant rather than a # measurement. The exemption is conditioned on that forcing # actually being in effect rather than merely documented, so an From 7d429920102012b584023bb3e72ef6f9afd46a51 Mon Sep 17 00:00:00 2001 From: Chenfei Zhang Date: Sun, 30 Aug 2026 22:37:19 -0700 Subject: [PATCH 11/14] [None][test] add AgentX GB300 lanes to the QA disagg perf list Registers both AgentX DeepSeek-V4-Pro-DSpark trace-replay cases in tests/integration/test_lists/qa/llm_perf_disagg.yml, under the existing GB300 condition block. The 40-GPU case is included here even though it is deliberately not registered in post-merge CI: the QA list is run on a daily/release schedule rather than per-merge, so listing it does not add recurring 10-node cost to every merge. Timeouts are set above this file's usual 120 minutes because the lanes replay a trace for a fixed AGENTX_DURATION=3600 s wall-clock window on top of server startup and model load. The 24-GPU case measured 9124 s (152 min) of pytest time on aws-cmh, so it gets 210, matching the value already used for it in the CI test-db. The 40-GPU case has not been run; it uses the same replay duration but brings up four workers across ten nodes, so it gets 240 pending a measurement. The ids use the non-_upload spelling, matching every other entry in this file. Signed-off-by: Chenfei Zhang --- tests/integration/test_lists/qa/llm_perf_disagg.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/integration/test_lists/qa/llm_perf_disagg.yml b/tests/integration/test_lists/qa/llm_perf_disagg.yml index 0a86b3199735..e22ddd7db046 100644 --- a/tests/integration/test_lists/qa/llm_perf_disagg.yml +++ b/tests/integration/test_lists/qa/llm_perf_disagg.yml @@ -53,6 +53,9 @@ llm_perf_disagg: - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_deepseek-v4-pro-fp4_8k1k_con1229_ctx7_dep4_gen1_dep8_eplb384_mtp3_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_deepseek-v4-pro-fp4_8k1k_con666_ctx8_dep4_gen1_dep32_eplb384_mtp3_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_deepseek-v4-pro-fp4_8k1k_con2_ctx1_dep4_gen5_tep4_eplb0_mtp3_ccb-NIXL] TIMEOUT (120) + # GB300 DeepSeek-V4-Pro-DSpark (AgentX agentic trace replay) + - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_deepseek-v4-pro-dspark_agentx_con1156_ctx2_dep8_gen1_dep8_eplb0_dspark3_ccb-NIXL] TIMEOUT (210) + - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_deepseek-v4-pro-dspark_agentx_con1456_ctx3_dep8_gen1_dep16_eplb0_dspark5_ccb-NIXL] TIMEOUT (240) # GB300 Kimi-K2.5-Thinking - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_kimi-k25-thinking-fp4_8k1k_con4_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (120) From aa417dde6f6ed84b0b2136c963991447e00a33e4 Mon Sep 17 00:00:00 2001 From: Chenfei Zhang Date: Sun, 30 Aug 2026 22:48:46 -0700 Subject: [PATCH 12/14] [None][test] add AgentX GB300 disagg configs to the QA config folder The two AgentX DeepSeek-V4-Pro-DSpark lanes were listed in tests/integration/test_lists/qa/llm_perf_disagg.yml but their configs existed only under tests/scripts/perf-sanity/disaggregated/. PERF_SANITY_TEST_CASES globs a single directory, DISAGG_CONFIG_FOLDER, so a QA run pointed at tests/scripts/perf/disaggregated/ could not collect them. 31 of the 33 stems in the QA disagg list already resolve in that folder; these two were the only exceptions. The copies are byte-identical to the perf-sanity ones. QA variants of other cases often add a top-level accuracy: block (37 of 54 files, only 2 with enable_accuracy_test: true) and a larger multi_round, but neither applies here: the AgentX client replays a trace corpus for a fixed wall-clock window, so multi_round is not the workload knob, and there is no lm_eval task defined for this model. Signed-off-by: Chenfei Zhang --- ...dep8_gen1_dep8_eplb0_dspark3_ccb-NIXL.yaml | 133 ++++++++++++++++++ ...ep8_gen1_dep16_eplb0_dspark5_ccb-NIXL.yaml | 133 ++++++++++++++++++ 2 files changed, 266 insertions(+) create mode 100644 tests/scripts/perf/disaggregated/gb300_deepseek-v4-pro-dspark_agentx_con1156_ctx2_dep8_gen1_dep8_eplb0_dspark3_ccb-NIXL.yaml create mode 100644 tests/scripts/perf/disaggregated/gb300_deepseek-v4-pro-dspark_agentx_con1456_ctx3_dep8_gen1_dep16_eplb0_dspark5_ccb-NIXL.yaml diff --git a/tests/scripts/perf/disaggregated/gb300_deepseek-v4-pro-dspark_agentx_con1156_ctx2_dep8_gen1_dep8_eplb0_dspark3_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb300_deepseek-v4-pro-dspark_agentx_con1156_ctx2_dep8_gen1_dep8_eplb0_dspark3_ccb-NIXL.yaml new file mode 100644 index 000000000000..4adc07545af2 --- /dev/null +++ b/tests/scripts/perf/disaggregated/gb300_deepseek-v4-pro-dspark_agentx_con1156_ctx2_dep8_gen1_dep8_eplb0_dspark3_ccb-NIXL.yaml @@ -0,0 +1,133 @@ +metadata: + model_name: deepseek_v4_pro_dspark + precision: fp4 + model_dir_name: DeepSeek-V4-Pro-DSpark + supported_gpus: + - GB300 + script_file: disaggr_torch.slurm + benchmark_type: agentx +slurm: + script_file: disaggr_torch.slurm + partition: + account: + job_time: 04:00:00 + job_name: unified-benchmark + extra_args: --gres=gpu:4 + numa_bind: true +benchmark: + mode: e2e + use_nv_sa_benchmark: false + benchmark_client: agentx + multi_round: 1 + benchmark_ratio: 0.8 + streaming: true + concurrency_list: '1156' + input_length: 990016 + output_length: 6579 + dataset_file: semianalysis_cc_traces_weka_062126 +hardware: + gpus_per_node: 4 + num_ctx_servers: 2 + num_gen_servers: 1 +environment: + container_mount: + container_image: + model_path: + trtllm_repo: '' + build_wheel: false + work_dir: + client_env_var: AGENTX_MAX_CTX=996579 AGENTX_DURATION=3600 AGENTX_WARMUP_PER_LANE=3 + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 ENROOT_ALLOW_DEV=yes NCCL_GRAPH_MIXING_SUPPORT=0 MIMALLOC_ARENA_RESERVE=0 OMP_NUM_THREADS=1 TRTLLM_PINNED_WEIGHT_STAGING=1 + ctx_worker_env_var: PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True CUDA_SCALE_LAUNCH_QUEUES=4x TLLM_INDEXER_MQA_LOGITS_ELEM_BUDGET=1073741824 TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2.08 + gen_worker_env_var: TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2.08 + server_env_var: TRTLLM_SERVER_DISABLE_GC=1 +profiling: + nsys_on: false +ctx_router_config: + type: conversation +server_config_extra: + gen_strip_message_history: true + gen_tokids_ctxbytes: true + server_keep_alive_timeout: 3600 +worker_config: + gen: + print_iter_log: true + max_batch_size: 64 + max_num_tokens: 256 + max_seq_len: 996595 + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + cuda_graph_config: + enable_padding: true + batch_sizes: [1, 2, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64] + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + dtype: fp8 + tokens_per_block: 128 + event_buffer_max_size: 0 + avg_seq_len: 200000 + host_cache_size: 0 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + stream_interval: 20 + num_postprocess_workers: 4 + speculative_config: + decoding_type: DSpark + max_draft_len: 3 + block_size: 3 + speculative_model: DeepSeek-V4-Pro-DSpark + custom_tokenizer: deepseek_v4 + return_perf_metrics: true + perf_metrics_max_requests: 100000 + perf_metrics_output_dir: perf_metrics/gen + sparse_attention_config: + algorithm: deepseek_v4 + enable_heuristic_topk: true + num_serve_frontends: 4 + ctx: + print_iter_log: true + max_batch_size: 256 + max_num_tokens: 8192 + max_seq_len: 990016 + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + enable_attention_dp: true + cuda_graph_config: null + disable_overlap_scheduler: false + kv_cache_config: + enable_block_reuse: true + free_gpu_memory_fraction: 0.8 + dtype: fp8 + tokens_per_block: 128 + event_buffer_max_size: 0 + block_reuse_config: + policy: per_conversation + max_num_turns: 5 + pool_ratio: [0.55, 0.22, 0.23] + host_cache_size: 193273528320 + moe_config: + backend: CUTEDSL + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + custom_tokenizer: deepseek_v4 + return_perf_metrics: true + perf_metrics_max_requests: 100000 + perf_metrics_output_dir: perf_metrics/ctx + enable_chunked_prefill: true + attention_dp_config: + kv_cache_routing_conversation_affinity: true + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + num_serve_frontends: 8 diff --git a/tests/scripts/perf/disaggregated/gb300_deepseek-v4-pro-dspark_agentx_con1456_ctx3_dep8_gen1_dep16_eplb0_dspark5_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb300_deepseek-v4-pro-dspark_agentx_con1456_ctx3_dep8_gen1_dep16_eplb0_dspark5_ccb-NIXL.yaml new file mode 100644 index 000000000000..ee32a067e580 --- /dev/null +++ b/tests/scripts/perf/disaggregated/gb300_deepseek-v4-pro-dspark_agentx_con1456_ctx3_dep8_gen1_dep16_eplb0_dspark5_ccb-NIXL.yaml @@ -0,0 +1,133 @@ +metadata: + model_name: deepseek_v4_pro_dspark + precision: fp4 + model_dir_name: DeepSeek-V4-Pro-DSpark + supported_gpus: + - GB300 + script_file: disaggr_torch.slurm + benchmark_type: agentx +slurm: + script_file: disaggr_torch.slurm + partition: + account: + job_time: 04:00:00 + job_name: unified-benchmark + extra_args: --gres=gpu:4 + numa_bind: true +benchmark: + mode: e2e + use_nv_sa_benchmark: false + benchmark_client: agentx + multi_round: 1 + benchmark_ratio: 0.8 + streaming: true + concurrency_list: '1456' + input_length: 990016 + output_length: 6579 + dataset_file: semianalysis_cc_traces_weka_062126 +hardware: + gpus_per_node: 4 + num_ctx_servers: 3 + num_gen_servers: 1 +environment: + container_mount: + container_image: + model_path: + trtllm_repo: '' + build_wheel: false + work_dir: + client_env_var: AGENTX_MAX_CTX=996579 AGENTX_DURATION=3600 AGENTX_WARMUP_PER_LANE=3 + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 ENROOT_ALLOW_DEV=yes NCCL_GRAPH_MIXING_SUPPORT=0 MIMALLOC_ARENA_RESERVE=0 OMP_NUM_THREADS=1 TRTLLM_PINNED_WEIGHT_STAGING=1 + ctx_worker_env_var: PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True CUDA_SCALE_LAUNCH_QUEUES=4x TLLM_INDEXER_MQA_LOGITS_ELEM_BUDGET=1073741824 TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2.74 + gen_worker_env_var: TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2.74 + server_env_var: TRTLLM_SERVER_DISABLE_GC=1 +profiling: + nsys_on: false +ctx_router_config: + type: conversation +server_config_extra: + gen_strip_message_history: true + gen_tokids_ctxbytes: true + server_keep_alive_timeout: 3600 +worker_config: + gen: + print_iter_log: true + max_batch_size: 32 + max_num_tokens: 192 + max_seq_len: 996595 + tensor_parallel_size: 16 + moe_expert_parallel_size: 16 + pipeline_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + cuda_graph_config: + enable_padding: true + batch_sizes: [1, 2, 4, 8, 16, 24, 32] + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.7 + dtype: fp8 + tokens_per_block: 128 + event_buffer_max_size: 0 + avg_seq_len: 200000 + host_cache_size: 0 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + stream_interval: 20 + num_postprocess_workers: 4 + speculative_config: + decoding_type: DSpark + max_draft_len: 5 + block_size: 5 + speculative_model: DeepSeek-V4-Pro-DSpark + custom_tokenizer: deepseek_v4 + return_perf_metrics: true + perf_metrics_max_requests: 100000 + perf_metrics_output_dir: perf_metrics/gen + sparse_attention_config: + algorithm: deepseek_v4 + enable_heuristic_topk: true + num_serve_frontends: 4 + ctx: + print_iter_log: true + max_batch_size: 256 + max_num_tokens: 8192 + max_seq_len: 990016 + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + enable_attention_dp: true + cuda_graph_config: null + disable_overlap_scheduler: false + kv_cache_config: + enable_block_reuse: true + free_gpu_memory_fraction: 0.8 + dtype: fp8 + tokens_per_block: 128 + event_buffer_max_size: 0 + block_reuse_config: + policy: per_conversation + max_num_turns: 5 + pool_ratio: [0.55, 0.22, 0.23] + host_cache_size: 193273528320 + moe_config: + backend: CUTEDSL + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + custom_tokenizer: deepseek_v4 + return_perf_metrics: true + perf_metrics_max_requests: 100000 + perf_metrics_output_dir: perf_metrics/ctx + enable_chunked_prefill: true + attention_dp_config: + kv_cache_routing_conversation_affinity: true + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + num_serve_frontends: 8 From 2c17e0d94213581fd9a7ff51566b0e23742b28f1 Mon Sep 17 00:00:00 2001 From: Chenfei Zhang Date: Sun, 30 Aug 2026 23:01:22 -0700 Subject: [PATCH 13/14] [None][doc] document running the AgentX perf-sanity lane with local/submit.py Appends a short walkthrough to jenkins/scripts/perf/local/README.md for running an AgentX trace-replay lane by hand. The existing README documents the flags; this covers the three things needed on top of them -- the environment variables, the submit command, and how to read the result: - EXTRA_CONTAINER_EXPORTS is read at generation time and spliced into the four per-role env prefixes, so it must be exported before submit.py - submit.py appends its own HF_HOME=/tmp/hf_home after that splice for the worker roles, so HF_HUB_CACHE and HF_DATASETS_CACHE should be set too to make the result independent of splice order - dataset_file is an aiperf --public-dataset loader name, not a path - AGENTX_* knobs come from client_env_var in the config yaml, not from the shell and not from a submit.py flag - the verdict is submission_valid plus duration coverage in profile_export_aiperf.json, not the pytest exit status Headings are demoted one level so the file keeps a single H1. Signed-off-by: Chenfei Zhang --- jenkins/scripts/perf/local/README.md | 103 +++++++++++++++++++++++++++ 1 file changed, 103 insertions(+) diff --git a/jenkins/scripts/perf/local/README.md b/jenkins/scripts/perf/local/README.md index d11e9d7b2297..b8730295a17c 100644 --- a/jenkins/scripts/perf/local/README.md +++ b/jenkins/scripts/perf/local/README.md @@ -99,3 +99,106 @@ python3 submit.py --test-list "perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb2 --mounts $mounts \ --llm-models-root $llm_models_path ``` + +--- + +## Running the AgentX perf-sanity lane + +The AgentX client replays a recorded multi-turn conversation corpus for a fixed +wall-clock window (`AGENTX_DURATION`, default 3600 s) rather than a fixed number of +fixed-shape prompts, so `isl`/`osl`/`iterations` in the config are descriptive and +the run is judged on duration coverage, not a request count. Example test id +(`_upload` is stripped for local runs, so nothing reaches OpenSearch): + +``` +perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_deepseek-v4-pro-dspark_agentx_con1156_ctx2_dep8_gen1_dep8_eplb0_dspark3_ccb-NIXL] +``` + +### Environment variables + +`submit.py` reads `EXTRA_CONTAINER_EXPORTS` -- a `;`-separated `KEY=VALUE` list -- +at **generation time** and splices it into the four per-role env prefixes +(`CTX_WORKER_ENV_VARS`, `GEN_WORKER_ENV_VARS`, `SERVER_ENV_VARS`, +`BENCHMARK_ENV_VARS`). Export it before running `submit.py`, not at `sbatch` time: + +```bash +export EXTRA_CONTAINER_EXPORTS="HF_HOME=$hf_cache;HF_HUB_CACHE=$hf_cache/hub;HF_DATASETS_CACHE=$hf_cache/datasets;PIP_CACHE_DIR=$work_dir/.pip-cache" +``` + +Set all three HF variables, not just `HF_HOME`: `submit.py` appends its own +`HF_HOME=/tmp/hf_home` **after** the splice for the ctx and gen worker roles, and +the later assignment wins. `HF_HUB_CACHE` and `HF_DATASETS_CACHE` are never +overridden and outrank `HF_HOME` in `huggingface_hub` / `datasets`, making the +result independent of splice order. + +`dataset_file` in the config is an aiperf `--public-dataset` loader name, not a +path, and is fetched from Hugging Face at runtime. Either warm the cache above +before submitting, or confirm the compute nodes can reach the HF CDN. + +The `AGENTX_*` knobs are not `submit.py` flags -- they come from `client_env_var` +in the config YAML, so exporting them in your shell has no effect. Edit the YAML +to change one. `AGENTX_DURATION` is the main cost knob; keep `AGENTX_SEED` +(default 42) fixed when comparing runs. + +```yaml +client_env_var: 'AGENTX_MAX_CTX=996579 AGENTX_DURATION=3600 AGENTX_WARMUP_PER_LANE=3' +``` + +### Generate and submit + +```bash +python3 submit.py \ + --test-list "perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_deepseek-v4-pro-dspark_agentx_con1156_ctx2_dep8_gen1_dep8_eplb0_dspark3_ccb-NIXL]" \ + --draft-launch-sh $trtllm/jenkins/scripts/perf/disaggregated/slurm_launch_draft.sh \ + --launch-sh $work_dir/slurm_launch.sh \ + --install-sh $trtllm/jenkins/scripts/perf/local/slurm_install.sh \ + --run-sh $trtllm/jenkins/scripts/perf/local/slurm_run.sh \ + --llm-src $trtllm \ + --work-dir $work_dir \ + --llm-models-root $llm_models_path \ + --partition $partition \ + --account $account \ + --job-name agentx_test \ + --image $image \ + --mounts $mounts \ + --install-mode wheel \ + --wheel-path $wheel \ + --cluster-name $cluster + +cd $work_dir && sbatch slurm_launch.sh +``` + +`--cluster-name` selects the UCX and env rules in `cluster_env.py`; an unmatched +(cluster, GPU) pair falls through to a catch-all that pins no transport, changing +performance silently. Use a fresh `--work-dir` every run -- a reused disaggregated +work directory reads stale hostname files and hangs. This lane takes 6 nodes / +24 GPUs and roughly 2.5 h including the wheel install, so allow a 4 h limit. + +### Viewing the perf results + +Artifacts land under `$work_dir//`: + +| Path | Contents | +|---|---| +| `agentx.0.0/concurrency_1156/profile_export_aiperf.json` | metrics, percentiles, `submission_valid`, duration coverage (client log alongside in `logs/aiperf.log`) | +| `trtllm-benchmark.0.0.log` | human-readable metric block | +| `{ctx,gen}_server_*.log`, `disagg_server.log` | server logs | + +Gate on the export rather than on the pytest exit status: + +```bash +python3 -c " +import json +d = json.load(open('$work_dir//agentx.0.0/concurrency_1156/profile_export_aiperf.json')) +m = d['metadata'] +print('submission_valid', m.get('submission_valid')) +print('was_cancelled ', m.get('was_cancelled')) +print('errors ', m.get('error_summary')) +for p in m.get('metric_duration_coverage') or []: + print('coverage', p) +" +``` + +Coverage reports the TTFT and ITL sample ratios against the requested window, so a +run that only partly covered it is detectable instead of quietly averaging a +truncated segment. A green pytest summary alone is not a passing stage. From e7c667925c45eb1d31dbf80a6c3bb354e5b1e184 Mon Sep 17 00:00:00 2001 From: Chenfei Zhang Date: Sun, 30 Aug 2026 23:33:48 -0700 Subject: [PATCH 14/14] [None][test] add the 24-GPU AgentX GB300 case to the QA multinode perf list Adds the AgentX DeepSeek-V4-Pro-DSpark 24-GPU lane to tests/integration/test_lists/qa/llm_perf_multinode.txt, in a new # GB300 DeepSeek-V4-Pro-DSpark section after the existing # GB300 DeepSeek-V4-Pro group. Matches the conventions of that file: bare test id in the disagg-e2e- form with no _upload suffix and no TIMEOUT marker (the file carries none), grouped under a section header whose platform tag is what .claude/agents/perf-test-sync.md parses to classify the case. The 40-GPU lane is deliberately left out of this list to limit GPU hours; its config stays in the repo and it remains listed in llm_perf_disagg.yml. Signed-off-by: Chenfei Zhang --- tests/integration/test_lists/qa/llm_perf_multinode.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/integration/test_lists/qa/llm_perf_multinode.txt b/tests/integration/test_lists/qa/llm_perf_multinode.txt index 8021fdac6f81..e11cadae2a6a 100644 --- a/tests/integration/test_lists/qa/llm_perf_multinode.txt +++ b/tests/integration/test_lists/qa/llm_perf_multinode.txt @@ -45,6 +45,9 @@ perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_deepseek-v4-pro-fp4_8k1k_con perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_deepseek-v4-pro-fp4_8k1k_con666_ctx8_dep4_gen1_dep32_eplb384_mtp3_ccb-NIXL] perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_deepseek-v4-pro-fp4_8k1k_con2_ctx1_dep4_gen5_tep4_eplb0_mtp3_ccb-NIXL] +# GB300 DeepSeek-V4-Pro-DSpark (AgentX agentic trace replay) +perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_deepseek-v4-pro-dspark_agentx_con1156_ctx2_dep8_gen1_dep8_eplb0_dspark3_ccb-NIXL] + # GB300 GLM-5 perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_glm-5-fp4_8k1k_con1024_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-NIXL] perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_glm-5-fp4_8k1k_con1_ctx1_dep2_gen1_tep8_eplb0_mtp3_ccb-NIXL]