diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 603788b1b626..3911a2e9151d 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -6557,6 +6557,19 @@ def launchTestJobs(pipeline, testFilter, globalVars) // Nemotron-Ultra-V3 con9832 (8k64k) and con1197 (50k2k) are ctx_only-only: // their full 68-/72-GPU e2e+gen_only disagg topologies are intentionally not // created; the ctx_only ids run in the 4-GPU multi_gpus post-merge stage. + // GB300 DeepSeek-V4-Pro-DSpark, AgentX agentic trace replay. + // These lanes replay a ~1M-token multi-turn conversation trace for a fixed + // wall-clock duration instead of a fixed prompt count, so they are pinned to + // aws-cmh where the DSpark checkpoint and the trace corpus are staged. + // 6 Nodes: ctx2 (2 nodes, 8 GPUs each) + gen1 (2 nodes, 8 GPUs) = 24 GPUs + multiNodesSBSAConfigs += buildStageConfigs( + "GB300-24_GPUs-6_Nodes-PyTorch-Disagg-PerfSanity-AgentX-CTX2-NODE2-GPU8-GEN1-NODE2-GPU8-Post-Merge", + "gb300-flex-aws-cmh", + "l0_gb300_multi_nodes_perf_sanity_ctx2_node2_gpu8_gen1_node2_gpu8", + 1, + 24, + 6 + ) multiNodesSBSAConfigs = cbtsResizeSplits(multiNodesSBSAConfigs) fullSet += multiNodesSBSAConfigs.keySet() diff --git a/jenkins/scripts/perf/local/README.md b/jenkins/scripts/perf/local/README.md index d11e9d7b2297..b8730295a17c 100644 --- a/jenkins/scripts/perf/local/README.md +++ b/jenkins/scripts/perf/local/README.md @@ -99,3 +99,106 @@ python3 submit.py --test-list "perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb2 --mounts $mounts \ --llm-models-root $llm_models_path ``` + +--- + +## Running the AgentX perf-sanity lane + +The AgentX client replays a recorded multi-turn conversation corpus for a fixed +wall-clock window (`AGENTX_DURATION`, default 3600 s) rather than a fixed number of +fixed-shape prompts, so `isl`/`osl`/`iterations` in the config are descriptive and +the run is judged on duration coverage, not a request count. Example test id +(`_upload` is stripped for local runs, so nothing reaches OpenSearch): + +``` +perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_deepseek-v4-pro-dspark_agentx_con1156_ctx2_dep8_gen1_dep8_eplb0_dspark3_ccb-NIXL] +``` + +### Environment variables + +`submit.py` reads `EXTRA_CONTAINER_EXPORTS` -- a `;`-separated `KEY=VALUE` list -- +at **generation time** and splices it into the four per-role env prefixes +(`CTX_WORKER_ENV_VARS`, `GEN_WORKER_ENV_VARS`, `SERVER_ENV_VARS`, +`BENCHMARK_ENV_VARS`). Export it before running `submit.py`, not at `sbatch` time: + +```bash +export EXTRA_CONTAINER_EXPORTS="HF_HOME=$hf_cache;HF_HUB_CACHE=$hf_cache/hub;HF_DATASETS_CACHE=$hf_cache/datasets;PIP_CACHE_DIR=$work_dir/.pip-cache" +``` + +Set all three HF variables, not just `HF_HOME`: `submit.py` appends its own +`HF_HOME=/tmp/hf_home` **after** the splice for the ctx and gen worker roles, and +the later assignment wins. `HF_HUB_CACHE` and `HF_DATASETS_CACHE` are never +overridden and outrank `HF_HOME` in `huggingface_hub` / `datasets`, making the +result independent of splice order. + +`dataset_file` in the config is an aiperf `--public-dataset` loader name, not a +path, and is fetched from Hugging Face at runtime. Either warm the cache above +before submitting, or confirm the compute nodes can reach the HF CDN. + +The `AGENTX_*` knobs are not `submit.py` flags -- they come from `client_env_var` +in the config YAML, so exporting them in your shell has no effect. Edit the YAML +to change one. `AGENTX_DURATION` is the main cost knob; keep `AGENTX_SEED` +(default 42) fixed when comparing runs. + +```yaml +client_env_var: 'AGENTX_MAX_CTX=996579 AGENTX_DURATION=3600 AGENTX_WARMUP_PER_LANE=3' +``` + +### Generate and submit + +```bash +python3 submit.py \ + --test-list "perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_deepseek-v4-pro-dspark_agentx_con1156_ctx2_dep8_gen1_dep8_eplb0_dspark3_ccb-NIXL]" \ + --draft-launch-sh $trtllm/jenkins/scripts/perf/disaggregated/slurm_launch_draft.sh \ + --launch-sh $work_dir/slurm_launch.sh \ + --install-sh $trtllm/jenkins/scripts/perf/local/slurm_install.sh \ + --run-sh $trtllm/jenkins/scripts/perf/local/slurm_run.sh \ + --llm-src $trtllm \ + --work-dir $work_dir \ + --llm-models-root $llm_models_path \ + --partition $partition \ + --account $account \ + --job-name agentx_test \ + --image $image \ + --mounts $mounts \ + --install-mode wheel \ + --wheel-path $wheel \ + --cluster-name $cluster + +cd $work_dir && sbatch slurm_launch.sh +``` + +`--cluster-name` selects the UCX and env rules in `cluster_env.py`; an unmatched +(cluster, GPU) pair falls through to a catch-all that pins no transport, changing +performance silently. Use a fresh `--work-dir` every run -- a reused disaggregated +work directory reads stale hostname files and hangs. This lane takes 6 nodes / +24 GPUs and roughly 2.5 h including the wheel install, so allow a 4 h limit. + +### Viewing the perf results + +Artifacts land under `$work_dir//`: + +| Path | Contents | +|---|---| +| `agentx.0.0/concurrency_1156/profile_export_aiperf.json` | metrics, percentiles, `submission_valid`, duration coverage (client log alongside in `logs/aiperf.log`) | +| `trtllm-benchmark.0.0.log` | human-readable metric block | +| `{ctx,gen}_server_*.log`, `disagg_server.log` | server logs | + +Gate on the export rather than on the pytest exit status: + +```bash +python3 -c " +import json +d = json.load(open('$work_dir//agentx.0.0/concurrency_1156/profile_export_aiperf.json')) +m = d['metadata'] +print('submission_valid', m.get('submission_valid')) +print('was_cancelled ', m.get('was_cancelled')) +print('errors ', m.get('error_summary')) +for p in m.get('metric_duration_coverage') or []: + print('coverage', p) +" +``` + +Coverage reports the TTFT and ITL sample ratios against the requested window, so a +run that only partly covered it is detectable instead of quietly averaging a +truncated segment. A green pytest summary alone is not a passing stage. diff --git a/tests/integration/defs/.test_durations b/tests/integration/defs/.test_durations index 4b3dba9d99dd..30d5b9d69b81 100644 --- a/tests/integration/defs/.test_durations +++ b/tests/integration/defs/.test_durations @@ -930,6 +930,7 @@ "perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_qwen3-235b-fp4_8k1k_con1024_ctx1_tp1_gen1_dep8_eplb0_mtp0_ccb-NIXL]": 2103.162375, "perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL]": 7666.37925, "perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL]": 2542.9724666666666, + "perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_deepseek-v4-pro-dspark_agentx_con1156_ctx2_dep8_gen1_dep8_eplb0_dspark3_ccb-NIXL]": 5400.0, "perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_glm-5-fp4_8k1k_con1024_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-NIXL]": 3729.2565, "perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_nemotron-ultra-v3-fp4_50k2k_con12_ctx1_dep4_gen6_tep4_eplb0_mtp6_ccb-NIXL]": 1731.7142, "perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_nemotron-ultra-v3-fp4_50k2k_con178_ctx5_dep4_gen1_dep4_eplb0_mtp6_ccb-NIXL]": 1023.3791666666666, diff --git a/tests/integration/defs/perf/README_test_perf_sanity.md b/tests/integration/defs/perf/README_test_perf_sanity.md index 25c07594452c..7bd773bd28fc 100644 --- a/tests/integration/defs/perf/README_test_perf_sanity.md +++ b/tests/integration/defs/perf/README_test_perf_sanity.md @@ -117,6 +117,39 @@ pass. `process_and_upload_test_results` therefore looks history up against a baseline branch — `PERF_BASELINE_BRANCH`, default `main` — using a lookup-only copy of the data. The uploaded documents keep their true `s_branch`. +Case identity is keyed on `s_test_case_name` (plus GPU type, runtime and branch), and a +disaggregated case name embeds its config stem — so an agentx lane, whose stem contains +`agentx`, already forms its own baseline population without needing an extra match key. +`s_benchmark_client` is still uploaded, as a reportable record of which load generator +drove the lane, but it does not participate in matching. The built-in client uploads `""` +(not `"default"`) so the column reads consistently against records written before the +field existed. + +## Benchmark Clients + +A disaggregated `benchmark_config` may select which load generator drives the lane via +the optional `benchmark_client` key. Any value other than the two below is rejected at +config-parse time rather than silently falling through to the default client. + +| `benchmark_client` | Client | Notes | +|---|---|---| +| *(omitted / `""`)* | `tensorrt_llm/serve/scripts/benchmark_serving.py` | Default. Fixed request count from `dataset_file` + `concurrency_list`. | +| `agentx` | `perf/agentx_client.py` | AgentX agentic multi-turn trace replay (a fork of `aiperf`). | + +**AgentX specifics**: +- Duration-bounded, not count-bounded: the run replays a conversation trace for + `AGENTX_DURATION` seconds, so `dataset_file` names an `aiperf` dataset loader (fetched + from HF), *not* a path — `get_dataset_dir` must not be applied to it. +- `concurrency_list` is the whole-cluster total and is passed through un-multiplied. +- Tunables are passed through `client_env_var` (`AGENTX_MAX_CTX`, `AGENTX_DURATION`, + `AGENTX_WARMUP_PER_LANE`). Artifacts land in + `{test_output_dir}/agentx.{server_idx}.{client_idx}/concurrency_{N}/`. +- `al` (Mean Avg Decoded Tokens per Iter) is exempt from the spec-decoding hard-fail for + agentx lanes: it derives from a TRT-LLM-specific per-response field that `aiperf` does + not propagate. This loses no signal as long as the lane pins the accepted length with + `TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS` (set per-yaml), which makes `al` a + restatement of a configured constant rather than a measurement. + ## Overview - Run performance sanity benchmarks across multiple model configs @@ -191,6 +224,10 @@ There are two modes for perf sanity tests: aggregated (aggr) and disaggregated ( **Use Case**: Disaggregated architecture where model runs across multiple nodes with separate context (prefill) and generation (decode) servers. +**Optional `benchmark_config` keys**: `benchmark_client` selects a non-default load +generator (see [Benchmark Clients](#benchmark-clients)); `client_env_var` passes +client-side environment variables through to it. + ## Test Case Formats In each test db yml file (with keyword `perf_sanity`), there are four test types: diff --git a/tests/integration/defs/perf/agentx_client.py b/tests/integration/defs/perf/agentx_client.py new file mode 100644 index 000000000000..57afa27d60ac --- /dev/null +++ b/tests/integration/defs/perf/agentx_client.py @@ -0,0 +1,547 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""AgentX benchmark client for perf-sanity disaggregated lanes. + +AgentX is an agentic, multi-turn *trace replay* workload. It is driven by +``aiperf`` from the SemiAnalysis ``agentx-harness`` fork, whose +``inferencex-agentx-mvp`` scenario replays a recorded conversation corpus for a +fixed **wall-clock duration** rather than a fixed prompt count. That single +difference is why it cannot reuse +``tensorrt_llm.serve.scripts.benchmark_serving``: there is no ISL/OSL/num-prompts +to give it, and the run length is time-bounded. + +This module is the adapter between the perf-sanity harness and that client. It + +1. installs the pinned ``agentx-harness`` build (see ``AGENTX_AIPERF_REF``), +2. verifies the endpoint really serves one token before spending GPU-hours, +3. runs ``aiperf profile``, and +4. translates ``profile_export_aiperf.json`` into the exact stdout lines that + ``PERF_METRIC_LOG_QUERIES`` in ``test_perf_sanity.py`` already scans for. + +Step 4 is the reason this is a separate process rather than harness code: by +emitting ``benchmark_serving``'s own report format verbatim, the perf-sanity +metric scanner, database upload and regression check all keep working unchanged. + +Two deliberate deviations from the upstream ``run_benchmark_agentx.sh``: + +* **Concurrency is passed through, not multiplied.** The reference script + computes ``concurrency * num_gen_servers`` because its YAML states per-server + concurrency. perf-sanity's ``concurrency_list`` is already the whole-cluster + total (it feeds ``--max-concurrency`` directly in the default client), so + scaling it here would silently over-drive the lane. +* **Readiness is re-verified, briefly.** The harness has already waited for the + disagg server, so the long poll is redundant; the one-token probe is kept + because it is the only cheap check that proves the ctx->gen KV path works. + Without it a broken KV transfer surfaces an hour later as a client-side + "failed request threshold" error, sending the reader to the wrong logs. + +All tuning knobs are read from the environment with the same names and defaults +as the reference script, so a lane is configured purely through the YAML's +``environment.client_env_var``. +""" + +import argparse +import json +import os +import subprocess +import sys +import threading +import time +import urllib.error +import urllib.request + +# Pinned agentx-harness build. It installs *as* ``aiperf`` (shadowing any +# PyPI aiperf), so the version is pinned by commit rather than by release. +DEFAULT_AIPERF_REF = ( + "git+https://github.com/SemiAnalysisAI/agentx-harness.git" + "@754356e9a39acc6cc6afb242d123bb57c3fb6f75" +) + +# aiperf's CLI/serialization/UI dependencies. The TRT-LLM container already +# carries its ML stack (numpy/pandas/torch/huggingface-hub/aiohttp); installing +# only this set has been verified to add packages without reinstalling the +# pinned scientific stack. +AIPERF_EXTRA_DEPS = [ + "cyclopts", + "ruamel.yaml", + "msgspec", + "orjson", + "kaleido", + "dash", + "dash-bootstrap-components", + "textual", + "starlette-compress", + "setproctitle", + "jmespath", + "aiofiles", + "ffmpeg-python", + "pydantic-settings", + "prometheus_client", + "fastapi", + "soundfile", + "plotly", + "seaborn", + "tiktoken", + "tqdm", + "uvicorn[standard]", + "uvloop", + "crick", + "zstandard", +] + +# The container ships datasets 3.x, which cannot parse the trace corpus schema +# ("Feature type 'Json' not found"). 4.x+ is required. +MIN_DATASETS_MAJOR = 4 + +# PyExecutor's HangDetector is hard-coded at 300 s with no env override, so a +# multi-minute pip install with zero requests in flight idles the GEN worker +# into MPI_Abort -- which then presents as a *client* connection failure. A +# best-effort 1-token request on this period keeps the executor awake. +HEARTBEAT_PERIOD_S = 60 + +# Mapping from perf-sanity's report labels to aiperf metric fields. +# +# Two of these are easy to get wrong and are worth stating explicitly, because +# both names exist in the export and differ by ~47x: +# * ``inter_chunk_latency`` is pooled over every streamed chunk (count == total +# chunks) -- that is ITL. +# * ``inter_token_latency`` is already averaged per request (count == request +# count) -- that is TPOT. +# ``user_throughput`` likewise maps to ``e2e_output_token_throughput`` (which +# includes TTFT, matching benchmark_serving's definition) and NOT to the +# similarly-named ``output_token_throughput_per_user`` (decode-only). +# +# aiperf reports every latency in ms and every throughput in tokens/sec or +# requests/sec, which is what the labels below claim, so no unit conversion is +# applied. ``_read_metric`` asserts the unit to keep that assumption honest. +THROUGHPUT_METRICS = [ + ("Request throughput (req/s)", "request_throughput", "requests/sec"), + ("Output token throughput (tok/s)", "output_token_throughput", "tokens/sec"), + ("Total Token throughput (tok/s)", "total_token_throughput", "tokens/sec"), + ("User throughput (tok/s)", "e2e_output_token_throughput", "tokens/sec/user"), +] + +# (label stem, aiperf field). Each contributes Mean/Median/P99 lines. +LATENCY_METRICS = [ + ("TTFT", "time_to_first_token"), + ("ITL", "inter_chunk_latency"), + ("TPOT", "inter_token_latency"), + ("E2EL", "request_latency"), +] + +# benchmark_serving's report format. Reproduced exactly so the existing +# regex scanner needs no change; see _print_metric for why width matters. +_REPORT_FORMAT = "{:<40} {:<10.2f}" + + +def _log(msg: str) -> None: + print(f"[agentx] {msg}", flush=True) + + +def _print_metric(label: str, value: float) -> None: + """Emit one metric line in benchmark_serving's exact report format. + + ``PERF_METRIC_LOG_QUERIES`` requires at least one whitespace character + between the colon and the number. The 40-char field supplies it only while + ``len(label) + 1 < 40``, which holds for every label here (longest is 30); + a longer label would silently stop matching, hence the assert. + """ + stem = f"{label}:" + assert len(stem) < 40, f"label {label!r} too long to leave a separator" + print(_REPORT_FORMAT.format(stem, value), flush=True) + + +def _env_int(name: str, default: int) -> int: + return int(os.environ.get(name, default)) + + +def _env_float(name: str, default: float) -> float: + return float(os.environ.get(name, default)) + + +def aiperf_is_usable() -> bool: + """Return whether the ambient interpreter already has a working agentx stack. + + Checks importability *and* the datasets major version, since a container + with aiperf but datasets 3.x fails only later, when loading the corpus. + """ + probe = ( + "import aiperf, cyclopts, datasets, transformers, pydantic, msgspec, orjson; " + f"import sys; sys.exit(0 if int(datasets.__version__.split('.')[0]) >= {MIN_DATASETS_MAJOR} else 1)" + ) + return ( + subprocess.call( + [sys.executable, "-c", probe], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL + ) + == 0 + ) + + +def post_one_token(url: str, model: str, timeout: float) -> "str | None": + """POST a 1-token non-streaming completion. Returns the body, or None on failure.""" + payload = json.dumps( + { + "model": model, + "messages": [{"role": "user", "content": "hi"}], + "max_tokens": 1, + "stream": False, + } + ).encode() + req = urllib.request.Request( + f"{url}/v1/chat/completions", + data=payload, + headers={"Content-Type": "application/json"}, + method="POST", + ) + try: + with urllib.request.urlopen(req, timeout=timeout) as resp: + return resp.read().decode(errors="replace") + except (urllib.error.URLError, OSError, TimeoutError): + return None + + +class _Heartbeat: + """Keeps the GEN worker's hang detector at bay during a long pip install.""" + + def __init__(self, url: str, model: str): + self._url = url + self._model = model + self._stop = threading.Event() + self._thread = threading.Thread(target=self._run, daemon=True) + + def _run(self) -> None: + while not self._stop.wait(HEARTBEAT_PERIOD_S): + # Best-effort by construction: post_one_token swallows every + # network error, so the heartbeat can never fail the run. + post_one_token(self._url, self._model, timeout=60) + + def __enter__(self) -> "_Heartbeat": + self._thread.start() + _log(f"heartbeat armed ({HEARTBEAT_PERIOD_S}s) to hold off the 300s hang detector") + return self + + def __exit__(self, *exc_info: object) -> None: + self._stop.set() + self._thread.join(timeout=HEARTBEAT_PERIOD_S + 30) + + +def _pip(*args: str) -> None: + subprocess.check_call([sys.executable, "-m", "pip", "install", *args]) + + +def ensure_aiperf(url: str, model: str) -> None: + """Install the pinned agentx build unless a usable one is already present.""" + if os.environ.get("AGENTX_FORCE_REINSTALL", "0") == "1": + _log("AGENTX_FORCE_REINSTALL=1 -> reinstalling unconditionally") + elif aiperf_is_usable(): + _log("aiperf + deps already usable; skipping install") + return + + ref = os.environ.get("AGENTX_AIPERF_REF", DEFAULT_AIPERF_REF) + with _Heartbeat(url, model): + _log(f"installing aiperf (agentx) from {ref}") + _pip("--force-reinstall", "--no-deps", f"aiperf @ {ref}") + _pip(*AIPERF_EXTRA_DEPS) + _pip("-U", "datasets") + _pip("transformers==5.8.1") + # aiperf needs a newer pydantic than the container pins, but its own + # dependency resolution would drag in a fresh ML stack; pin the pair + # directly with --no-deps instead. + _pip("--force-reinstall", "--no-deps", "pydantic==2.12.5", "pydantic-core==2.41.5") + + if not aiperf_is_usable(): + raise RuntimeError( + "aiperf is still not importable after install; refusing to start the " + "benchmark (see the pip output above)" + ) + + +def verify_endpoint(url: str, model: str) -> None: + """Prove the endpoint serves one real token before spending GPU-hours. + + The harness has already waited for the disagg server, so this is a short + confirmation rather than the reference script's 5400 s poll. It exists + because a broken ctx->gen KV path answers /health perfectly well and only + fails once real traffic arrives -- an hour later, as a client-side error. + """ + timeout = _env_int("AGENTX_READY_TIMEOUT", 900) + interval = _env_int("AGENTX_READY_INTERVAL", 15) + deadline = time.monotonic() + timeout + last = None + while time.monotonic() < deadline: + last = post_one_token(url, model, timeout=300) + if last and '"choices"' in last: + _log("1-token completion OK -- the disagg path is live") + return + _log(f"not serving yet (response head: {str(last)[:200]})") + time.sleep(interval) + raise RuntimeError( + f"{url} could not complete ONE token within {timeout}s. This is a " + f"SERVER-side failure: inspect the ctx/gen worker logs and the disagg " + f"server log, not the client. Suspect the ctx->gen KV transfer " + f"(on GB300/NVL72 check UCX_TLS contains no 'rc'). " + f"Last response: {str(last)[:500]}" + ) + + +def build_aiperf_cmd(args: argparse.Namespace, url: str, artifact_dir: str) -> "list[str]": + """Assemble the ``aiperf profile`` argv. + + Note ``--url`` carries scheme+host+port only; the request path is a separate + ``--endpoint``. That is why this client takes ``--host``/``--port`` and + composes them itself instead of having them appended to the argv. + """ + cmd = [ + "aiperf", + "profile", + "--scenario", + "inferencex-agentx-mvp", + "-m", + args.model, + "--tokenizer-trust-remote-code", + "--url", + url, + "--endpoint", + "/v1/chat/completions", + "--endpoint-type", + "chat", + "--streaming", + "--public-dataset", + args.dataset, + "--max-context-length", + str(_env_int("AGENTX_MAX_CTX", 1048576)), + "--trajectory-start-min-ratio", + str(_env_float("AGENTX_MIN_RATIO", 0.25)), + "--trajectory-start-max-ratio", + str(_env_float("AGENTX_MAX_RATIO", 0.75)), + "--failed-request-threshold", + str(_env_float("AGENTX_FAILED_THRESH", 0.1)), + "--use-server-token-count", + "--trace-idle-gap-cap-seconds", + "300", + "--no-gpu-telemetry", + "--random-seed", + str(_env_int("AGENTX_SEED", 42)), + "--benchmark-duration", + str(_env_int("AGENTX_DURATION", 3600)), + "--concurrency", + str(args.concurrency), + "--artifact-dir", + artifact_dir, + "--ui", + "simple", + ] + + # --benchmark-duration is always set; adding --request-count makes the run + # stop at whichever limit is hit first, which is only ever used for smoke + # tests. Left unset in CI so the lane is purely duration-bounded. + mult = os.environ.get("AGENTX_REQUEST_COUNT_MULT") + if mult: + count = max(args.concurrency, args.concurrency * int(mult)) + _log(f"request-count = max({args.concurrency}, {mult} x {args.concurrency}) = {count}") + cmd += ["--request-count", str(count)] + + # aiperf runs warmup as a distinct phase reported under a separate + # top-level `warmup_metrics` key, so these requests never pollute the + # measured window. <= 0 omits the flag entirely. + warmup = _env_int("AGENTX_WARMUP_PER_LANE", 10) + if warmup > 0: + cmd += ["--warmup-requests-per-lane", str(warmup)] + + # Plain os.environ.get with a default, mirroring the reference script's + # `${VAR-default}`: an explicitly empty value must disable the flag, which + # is how a production (>=900 s) run drops --unsafe-override. + extra = os.environ.get("AGENTX_EXTRA_ARGS", "--unsafe-override") + if extra.strip(): + cmd += extra.split() + return cmd + + +def _read_metric(export: dict, field: str, stat: str, expect_unit: "str | None" = None) -> float: + """Read one statistic out of the aiperf export, failing loudly if absent. + + A silently-missing metric would leave the perf-sanity scanner short a line, + which it reports as a vague "metrics are missing" much later; and a unit + change would corrupt a baseline without any error at all. Both are checked + here, at the point where the assumption is actually made. + """ + entry = export.get(field) + if not isinstance(entry, dict): + raise RuntimeError(f"aiperf export has no metric {field!r}") + if expect_unit is not None and entry.get("unit") != expect_unit: + raise RuntimeError( + f"aiperf metric {field!r} changed unit: expected {expect_unit!r}, " + f"got {entry.get('unit')!r}. Refusing to upload a mis-scaled value." + ) + if stat not in entry: + raise RuntimeError(f"aiperf metric {field!r} has no {stat!r} (keys: {sorted(entry)})") + return float(entry[stat]) + + +def check_run_health(export: dict) -> None: + """Fail unless the export shows a valid, complete, error-free measurement. + + Deliberately gates on *content*, never on aiperf's exit status: a run that + served nothing but HTTP 500s can still exit 0, and conversely the artifacts + are flushed before the process tears down. ``submission_valid`` is the + scenario's own verdict and is the single most important field here -- it + goes false when the measured window does not actually cover the requested + duration, which is exactly the failure mode that makes a short run look + like a fast one. + """ + metadata = export.get("metadata") or {} + + if export.get("was_cancelled"): + raise RuntimeError("aiperf reports was_cancelled=true; the measurement is incomplete") + + errors = export.get("error_summary") or [] + if errors: + raise RuntimeError(f"aiperf reported request errors: {json.dumps(errors)[:2000]}") + + # Report coverage before the verdict: when submission_valid is false this + # is the number that says why, so it must be in the log either way. + for phase in metadata.get("metric_duration_coverage") or []: + _log( + "coverage[{}]: ttft={:.4f} itl={:.4f} required={} expected_duration={}s".format( + phase.get("phase_name"), + phase.get("ttft_ratio", float("nan")), + phase.get("inter_token_latency_ratio", float("nan")), + phase.get("required_ratio"), + phase.get("expected_duration_seconds"), + ) + ) + + if "submission_valid" not in metadata: + raise RuntimeError( + "aiperf export has no metadata.submission_valid; the agentx scenario " + "did not complete (schema_version=" + f"{export.get('schema_version')!r})" + ) + if not metadata["submission_valid"]: + raise RuntimeError( + "aiperf reports submission_valid=false: the measured window does not " + "cover the requested duration, so these numbers are not comparable to " + "a baseline. See the coverage line above." + ) + + requests = _read_metric(export, "request_count", "avg") + duration = _read_metric(export, "benchmark_duration", "avg", "sec") + if requests <= 0: + raise RuntimeError("aiperf completed 0 requests") + _log(f"health OK: {int(requests)} requests over {duration:.1f}s, submission_valid=true") + + +def report_metrics(export: dict) -> None: + """Print every PERF_METRIC_LOG_QUERIES line from the aiperf export.""" + for label, field, unit in THROUGHPUT_METRICS: + _print_metric(label, _read_metric(export, field, "avg", unit)) + for stem, field in LATENCY_METRICS: + for prefix, stat in (("Mean", "avg"), ("Median", "p50"), ("P99", "p99")): + _print_metric(f"{prefix} {stem} (ms)", _read_metric(export, field, stat, "ms")) + + +def parse_args(argv: "list[str] | None" = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--model", required=True, help="Model path or HF id served by the endpoint") + parser.add_argument( + "--concurrency", + required=True, + type=int, + help="Total in-flight requests. Passed through to aiperf unscaled.", + ) + parser.add_argument("--dataset", required=True, help="aiperf --public-dataset loader name") + parser.add_argument("--host", required=True) + parser.add_argument("--port", required=True) + parser.add_argument( + "--artifact-dir", + # The harness exports this so artifacts land in the test output + # directory and are collected with the rest of the lane's logs. + default=os.environ.get("TRTLLM_AGENTX_ARTIFACT_DIR", "agentx_artifacts"), + help="Parent directory for aiperf artifacts; a concurrency_ subdir is created in it.", + ) + return parser.parse_args(argv) + + +def _run(args: argparse.Namespace, artifact_dir: str) -> int: + url = f"http://{args.host}:{args.port}" + # Custom tokenizers (e.g. deepseek_v4) are loaded in aiperf's worker + # processes, which do not inherit a CLI trust flag. + os.environ["HF_HUB_TRUST_REMOTE_CODE"] = "1" + + ensure_aiperf(url, args.model) + verify_endpoint(url, args.model) + + cmd = build_aiperf_cmd(args, url, artifact_dir) + export_path = os.path.join(artifact_dir, "profile_export_aiperf.json") + # Drop any export left by an earlier invocation that reused this directory + # (a retried lane, or a local rerun pointed at the same --output-dir). + # Without this, a crashed aiperf would be validated against -- and would + # report -- the previous run's numbers, turning a failure into a green + # result carrying stale metrics. Removing it first makes the existence + # check below a statement about *this* run. + if os.path.exists(export_path): + _log(f"removing export from a previous run: {export_path}") + os.remove(export_path) + _log("running: " + " ".join(cmd)) + completed = subprocess.run(cmd) + + if not os.path.exists(export_path): + # No export at all means aiperf died before writing results; its exit + # status is the only signal left, so surface it. + raise RuntimeError( + f"aiperf exited {completed.returncode} and wrote no {export_path}. " + f"See the aiperf output above and {artifact_dir}/logs." + ) + if completed.returncode != 0: + # An export exists, so the health checks below are more informative + # than the status; log it and let them make the call. + _log(f"warning: aiperf exited {completed.returncode} but wrote an export; validating it") + + with open(export_path) as f: + export = json.load(f) + + check_run_health(export) + report_metrics(export) + return 0 + + +def main(argv: "list[str] | None" = None) -> int: + args = parse_args(argv) + # aiperf writes its files flat into --artifact-dir, so give each + # concurrency its own directory to keep exports from overwriting. + artifact_dir = os.path.abspath( + os.path.join(args.artifact_dir, f"concurrency_{args.concurrency}") + ) + os.makedirs(artifact_dir, exist_ok=True) + try: + return _run(args, artifact_dir) + except Exception as exc: + # The perf-sanity harness invokes this via subprocess.check_output, + # which discards stdout when the exit status is non-zero -- so a + # failure would otherwise lose every diagnostic printed above. Persist + # the reason next to aiperf's own logs, which are collected with the + # test output. + reason = f"{type(exc).__name__}: {exc}" + _log(f"FAILED -- {reason}") + try: + with open(os.path.join(artifact_dir, "agentx_client_error.txt"), "w") as f: + f.write(reason + "\n") + except OSError: + pass + raise + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/integration/defs/perf/test_perf_sanity.py b/tests/integration/defs/perf/test_perf_sanity.py index 6fe156d597ea..f99eaeee2535 100644 --- a/tests/integration/defs/perf/test_perf_sanity.py +++ b/tests/integration/defs/perf/test_perf_sanity.py @@ -24,6 +24,7 @@ import shutil import socket import subprocess +import tempfile import time from typing import Dict, List, NamedTuple, Optional, Tuple @@ -54,6 +55,10 @@ "H200": "h200", } +# benchmark_client value selecting the AgentX trace-replay client +# (agentx_client.py). Any other non-empty value is rejected at parse time. +AGENTX_BENCHMARK_CLIENT = "agentx" + BENCH_SERVING_REPO = "https://github.com/kedarpotdar-nv/bench_serving.git" BENCH_SERVING_COMMIT = "f3ea022a5780de5d0babc5fffa53634e2023d28f" BENCH_SERVING_DIR = "/tmp/bench_serving" @@ -542,8 +547,10 @@ def add_perf_metric_value( """Populate `new_data` with per-test perf metrics from `metrics`. - Always copies every key in PERF_METRIC_LOG_QUERIES as `d_`. - - Adds `d_al` only when spec_decoding=True; non-spec rows omit it so - OpenSearch baselines don't blend the two populations. + - Adds `d_al` only when spec_decoding=True *and* the value was parsed; + non-spec rows omit it so OpenSearch baselines don't blend the two + populations, and spec rows exempted from reporting it (AgentX) omit it + rather than failing the upload. - Adds the `d_*_gen_worker_per_iter_device_step_time` family only for the disagg gen_only mode (the only mode that emits them). Of these the mean and the median are regression-gated (GEN_ONLY_REGRESSION_METRICS); the @@ -558,7 +565,17 @@ def add_perf_metric_value( for metric_name in PERF_METRIC_LOG_QUERIES: new_data[f"d_{metric_name}"] = metrics[metric_name] if spec_decoding: - new_data["d_al"] = metrics["al"] + # 'al' is legitimately absent for AgentX lanes: aiperf does not propagate + # TRT-LLM's non-standard avg_decoded_tokens_per_iter field. Omit the + # column instead of raising -- check_test_failure runs immediately before + # upload and has already hard-failed any non-exempt spec-decoding run + # whose 'al' is missing, so reaching here without it means the run is + # exempt by design. Omitted rather than defaulted: typeCheckForOpenSearchDB + # rejects None for a d_ key (losing the whole row), and a substituted 0.0 + # would corrupt the spec-decoding baseline population. + al = metrics.get("al") + if al is not None: + new_data["d_al"] = al if benchmark_mode == "gen_only": for metric_name in GEN_ONLY_DEVICE_STEP_TIME_METRICS: value = metrics.get(metric_name) @@ -647,9 +664,16 @@ def force_num_accepted_tokens_from_env_str(env_vars: str) -> int: """Extract TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS from a space-separated KEY=val env-var string. Returns 0 when not set. + + The runtime accepts a fractional value (see get_force_num_accepted_tokens_float + in tensorrt_llm), so parse as float first and truncate. The return value is + uploaded as l_force_num_accepted_tokens, which is a long, so a fractional + setting is not preserved in the record. It is reported rather than matched on: + case identity is keyed on the test case name, so two lanes differing solely in + the fractional part are already separate cases by name. """ val = to_env_dict(env_vars).get("TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS") - return int(val) if val is not None else 0 + return int(float(val)) if val is not None else 0 def add_host_port_to_cmd(cmd: List[str], host: str, port: int) -> List[str]: @@ -657,6 +681,77 @@ def add_host_port_to_cmd(cmd: List[str], host: str, port: int) -> List[str]: return cmd + ["--host", host, "--port", str(port)] +# Ports reserved for multi-frontend servers. Module-level so a reservation is +# never garbage-collected: closing the socket would release the port and reopen +# the very race the reservation exists to close. +_RESERVED_PORT_SOCKETS: List[socket.socket] = [] + + +def reserve_multi_frontend_port(host: str) -> int: + """Reserve a port for a server that runs several HTTP frontends. + + trtllm-serve rejects port 0 / --report_addr when num_serve_frontends > 1: + the extra frontends re-exec the command line verbatim, so with port 0 each + would bind its *own* kernel-assigned port instead of sharing one, and each + would republish its address, leaving the reader with whichever wrote last. + The port therefore has to be chosen on this side. + + Choosing it by binding and closing would reopen exactly the window the port-0 + scheme was introduced to remove -- anything on the node could take the port + between the probe and the server's bind. So the socket stays bound instead. + In multi-frontend mode every frontend binds with SO_REUSEPORT (see + launch_server), and Linux lets same-uid SO_REUSEPORT sockets share a port + provided the *first* binder set the flag, which this one does. So the + reservation is transparent to the server while still refusing a plain bind() + from any unrelated process on the node. + + The socket is deliberately never listen()ed: only *listening* SO_REUSEPORT + sockets join the kernel's accept load-balancing group, so a bound-only socket + holds the port without ever swallowing a request. + """ + # Mirror launch_server's family choice; a reservation in a different address + # family than the server's bind would not share the port. + addr_info = socket.getaddrinfo(host, 0, socket.AF_UNSPEC, socket.SOCK_STREAM) + family = ( + socket.AF_INET6 + if addr_info and all(info[0] == socket.AF_INET6 for info in addr_info) + else socket.AF_INET + ) + sock = socket.socket(family, socket.SOCK_STREAM) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) + sock.bind((host, 0)) + port = sock.getsockname()[1] + _RESERVED_PORT_SOCKETS.append(sock) + print_info(f"Reserved multi-frontend port {host}:{port} (holding SO_REUSEPORT socket)") + return port + + +def publish_addr_file(path: str, host: str, port: int) -> None: + """Write "host:port" to *path* the way trtllm-serve's --report_addr does. + + Used when this side picked the port (multi-frontend), so the disagg server's + hostname-file reader needs no special case. The write is atomic + (temp file in the same directory, then rename) with a ".tmp" suffix: the + reader counts only ".txt" entries, and a partial read on the shared + filesystem these tests coordinate through would be parsed as a URL. + """ + parent = os.path.dirname(os.path.abspath(path)) + os.makedirs(parent, exist_ok=True) + # Bracket IPv6 literals so the value is a usable URL authority: readers build + # "http:///..." from it verbatim. + reported_host = f"[{host}]" if ":" in host else host + fd, tmp_path = tempfile.mkstemp(dir=parent, prefix=os.path.basename(path) + ".", suffix=".tmp") + try: + with os.fdopen(fd, "w") as addr_file: + addr_file.write(f"{reported_host}:{port}\n") + addr_file.flush() + os.fsync(addr_file.fileno()) + os.replace(tmp_path, path) + finally: + if os.path.exists(tmp_path): + os.unlink(tmp_path) + + def _run_benchmark_with_log(cmd: List[str], env: Dict[str, str], log_path: str) -> str: """Run a benchmark while streaming its combined output to an artifact log.""" benchmark_env = env.copy() @@ -815,6 +910,12 @@ def __init__(self, server_config_data: dict, env_vars: str = ""): k: v for k, v in server_config_data.items() if k not in exclude_keys } + # Not a recognized field, so it rides through in extra_llm_api_config_data + # to the engine. Read it out here too: K > 1 HTTP frontends against one + # executor is incompatible with the port-0 launch scheme, so the launcher + # has to know the count before it builds the command line. + self.num_serve_frontends = self.extra_llm_api_config_data.get("num_serve_frontends", 1) + def to_cmd( self, output_dir: str, numa_bind: bool = False, disagg_serving_type: str = "" ) -> List[str]: @@ -1099,6 +1200,11 @@ def __init__( self.model_path = "" self.dataset_file = client_config_data.get("dataset_file", "") self.use_nv_sa_benchmark = client_config_data.get("use_nv_sa_benchmark", False) + # Which load generator drives the lane. "" selects the built-in + # benchmark_serving client; "agentx" selects the trace-replay client in + # agentx_client.py. Reported only -- see the s_benchmark_client note in + # to_db_data for why it is not a match key. + self.benchmark_client = client_config_data.get("benchmark_client", "") self.env_vars = env_vars # spec_decoding flag is retained for DB matching (b_eos column). --ignore-eos # is now always passed; output-length stability with spec decoding comes from @@ -1123,11 +1229,39 @@ def to_cmd(self) -> List[str]: model_dir = get_model_dir(self.model_name) self.model_path = model_dir if os.path.exists(model_dir) else self.model_name - if self.use_nv_sa_benchmark: + if self.benchmark_client == AGENTX_BENCHMARK_CLIENT: + return self._to_agentx_cmd() + elif self.use_nv_sa_benchmark: return self._to_sa_benchmark_cmd() else: return self._to_default_benchmark_cmd() + def _to_agentx_cmd(self) -> List[str]: + """Generate AgentX benchmark command (aiperf trace replay). + + AgentX replays a recorded conversation corpus for a fixed wall-clock + duration, so it takes neither a prompt count nor ISL/OSL; every other + knob comes from AGENTX_* env vars set in the lane's client_env_var. The + dataset name is passed through verbatim rather than resolved to a path + because it names an aiperf loader (which fetches from HF), not a file -- + so get_dataset_dir must not be applied to it. + """ + if not self.dataset_file: + raise ValueError( + f"Client {self.name} uses benchmark_client={AGENTX_BENCHMARK_CLIENT} but sets no " + "dataset_file; the agentx scenario has no default corpus." + ) + return [ + "python", + os.path.join(os.path.dirname(os.path.abspath(__file__)), "agentx_client.py"), + "--model", + self.model_path, + "--concurrency", + str(self.concurrency), + "--dataset", + self.dataset_file, + ] + def _to_sa_benchmark_cmd(self) -> List[str]: """Generate SA benchmark command (bench_serving repo).""" bench_script = ensure_bench_serving_repo() @@ -1235,6 +1369,12 @@ def to_db_data(self) -> dict: "b_streaming": self.streaming, "b_trust_remote_code": self.trust_remote_code, "b_use_nv_sa_benchmark": self.use_nv_sa_benchmark, + # Reported, not matched. Case identity is keyed on s_test_case_name + # (plus GPU type, runtime, branch), and a disagg case name embeds its + # config stem, so an agentx lane already forms its own population by + # name. Uploaded as "" (not "default") for the built-in client so the + # column reads consistently against records written before this field. + "s_benchmark_client": self.benchmark_client, "b_eos": self.spec_decoding, "s_client_log_link": "", "s_client_env_vars": self.env_vars, @@ -1261,6 +1401,10 @@ def __init__( hardware: dict, server_env_var: str, internal_request_auth_key: str | None = None, + router_config: dict | None = None, + ctx_router_config: dict | None = None, + gen_router_config: dict | None = None, + server_config_extra: dict | None = None, ): self.name = name self.disagg_serving_type = disagg_serving_type @@ -1272,6 +1416,10 @@ def __init__( self.hardware = hardware self.server_env_var = server_env_var self.internal_request_auth_key = internal_request_auth_key + self.router_config = router_config + self.ctx_router_config = ctx_router_config + self.gen_router_config = gen_router_config + self.server_config_extra = server_config_extra self.num_ctx_servers = hardware.get("num_ctx_servers", 0) self.num_gen_servers = hardware.get("num_gen_servers", 0) @@ -1420,6 +1568,12 @@ class DisaggTestCmds(NamedTuple): # disagg, only rank-0 pytest goes through this path; multi-rank workers # receive env via SLURM env propagation set up by submit.py. server_configs: List[Tuple["ServerConfig", "ServerConfig", "DisaggConfig"]] = [] + # Disagg-server-level keys, named as in bench-trtllm-disagg. A generic + # router applies to both roles; a role-specific one overrides it. + router_config: Optional[dict] = None + ctx_router_config: Optional[dict] = None + gen_router_config: Optional[dict] = None + server_config_extra: Optional[dict] = None def _hostnames_dir(self, server_idx: int) -> str: """Directory the disagg tasks exchange bound addresses through. @@ -1498,6 +1652,66 @@ def _generate_disagg_server_config(self, server_idx: int) -> str: "urls": gen_hostnames, }, } + # Router selection, mirroring bench-trtllm-disagg's submit.py: a generic + # router applies to both roles and a role-specific one overrides it for + # that role, e.g. ctx_router_config={"type": "conversation"} puts a + # conversation router only on the context servers. Deep-copied because + # trtllm-serve pops keys out of this dict while parsing it. + if self.router_config: + server_config["context_servers"]["router"] = copy.deepcopy(self.router_config) + server_config["generation_servers"]["router"] = copy.deepcopy(self.router_config) + if self.ctx_router_config: + server_config["context_servers"]["router"] = copy.deepcopy(self.ctx_router_config) + if self.gen_router_config: + server_config["generation_servers"]["router"] = copy.deepcopy(self.gen_router_config) + + if self.server_config_extra: + # Merged last, as bench-trtllm-disagg does, so it wins over + # everything above. The reserved keys are the exception: the harness + # owns them, not the config file. port must stay 0 so the server + # binds a kernel-assigned port and reports it via --report_addr, and + # the url lists are discovered from the hostname files above. + # Overriding either surfaces as a hang or a benchmark against the + # wrong endpoint, a long way from the cause, so reject it here. + reserved = { + "port", + "hostname", + "internal_request_auth_key", + "context_servers", + "generation_servers", + } + clobbered = sorted(reserved & set(self.server_config_extra)) + if clobbered: + raise RuntimeError( + f"server_config_extra may not override harness-owned keys {clobbered}: " + "the port is kernel-assigned and reported back via --report_addr, and " + "the server urls are discovered at runtime." + ) + # Distinct from the above: these keys override nothing, but they put + # trtllm-serve into fleet mode, which it refuses to combine with the + # port-0 + --report_addr discovery this harness depends on (see + # tensorrt_llm/commands/serve.py, "single self-contained + # disaggregated server"). A fleet hands one port to N SO_REUSEPORT + # workers; with port 0 each would get a *different* kernel-assigned + # port, so the reported address would serve 1/N of requests. + # Rejected here so the cause is named at config time instead of + # surfacing ~30s later as "DISAGG_SERVER server exited unexpectedly + # with code 2", which points nowhere near this key. + fleet_keys = sorted( + {"num_workers", "disagg_coordinator_url"} & set(self.server_config_extra) + ) + if fleet_keys and ( + self.server_config_extra.get("num_workers", 1) > 1 + or self.server_config_extra.get("disagg_coordinator_url") + ): + raise RuntimeError( + f"server_config_extra sets {fleet_keys}, which selects a disaggregated " + "server fleet. perf-sanity binds port 0 and discovers the address via " + "--report_addr, and trtllm-serve rejects that combination. Remove the " + "key, or teach the harness to bind a fixed port first." + ) + server_config.update(copy.deepcopy(self.server_config_extra)) + config_path = os.path.join(self.test_output_dir, f"server_config.{server_idx}.yaml") with open(config_path, "w") as f: yaml.dump(server_config, f) @@ -1692,10 +1906,6 @@ def run_cmd(self, server_idx: int) -> List[str]: self.server_configs[server_idx] if server_idx < len(self.server_configs) else None ) if "CTX" in self.disagg_serving_type or "GEN" in self.disagg_serving_type: - # port 0 + --report_addr: the worker binds a kernel-assigned port - # and publishes host:port itself, so no port is reserved here and - # left unbound while anything on the node could take it. The disagg - # server reads these files to build its config, exactly as before. hostname_file = self._hostname_file(server_idx) is_ctx = "CTX" in self.disagg_serving_type server_cmd = ctx_cmd if is_ctx else gen_cmd @@ -1705,10 +1915,36 @@ def run_cmd(self, server_idx: int) -> List[str]: config_idx = server_cmd.index("--config") + 1 self._wait_for_config_file(server_cmd[config_idx]) - server_cmd = add_host_port_to_cmd(server_cmd, self.hostname, 0) + [ - "--report_addr", - hostname_file, - ] + worker_cfg = None + if configs_for_idx is not None: + ctx_cfg, gen_cfg, _ = configs_for_idx + worker_cfg = ctx_cfg if is_ctx else gen_cfg + num_frontends = getattr(worker_cfg, "num_serve_frontends", 1) or 1 + + if num_frontends > 1: + # trtllm-serve refuses port 0 / --report_addr with several + # frontends (each would bind a different port), so reserve the + # port here and publish it on the worker's behalf; the disagg + # server's hostname-file reader is unchanged. + # + # Publishing before the server is up matches the semantics this + # replaces rather than loosening them: launch_server publishes at + # *bind* time, well before it constructs the engine, so a reader + # has always been able to see the address of a worker that is + # still loading weights. The disagg server's readiness wait is + # what covers that, and it is untouched here. + worker_port = reserve_multi_frontend_port(self.hostname) + server_cmd = add_host_port_to_cmd(server_cmd, self.hostname, worker_port) + publish_addr_file(hostname_file, self.hostname, worker_port) + else: + # port 0 + --report_addr: the worker binds a kernel-assigned port + # and publishes host:port itself, so no port is reserved here and + # left unbound while anything on the node could take it. The disagg + # server reads these files to build its config, exactly as before. + server_cmd = add_host_port_to_cmd(server_cmd, self.hostname, 0) + [ + "--report_addr", + hostname_file, + ] try: print_info( f"Starting server. disagg_serving_type: {self.disagg_serving_type} cmd is {server_cmd}" @@ -1718,9 +1954,8 @@ def run_cmd(self, server_idx: int) -> List[str]: f"trtllm-serve.{self.disagg_serving_type}.{server_idx}.log", ) worker_env = copy.deepcopy(os.environ) - if configs_for_idx is not None: - ctx_cfg, gen_cfg, _ = configs_for_idx - worker_env.update((ctx_cfg if is_ctx else gen_cfg).to_env()) + if worker_cfg is not None: + worker_env.update(worker_cfg.to_env()) with open(server_file_path, "w") as server_ctx: server_proc = subprocess.Popen( server_cmd, @@ -1834,6 +2069,12 @@ def run_cmd(self, server_idx: int) -> List[str]: bench_env = copy.deepcopy(os.environ) if client_config: bench_env.update(client_config.to_env()) + # Keep aiperf's artifacts (its own logs included) with + # the rest of the lane's output; ignored by other + # clients. + bench_env["TRTLLM_AGENTX_ARTIFACT_DIR"] = os.path.join( + self.test_output_dir, f"agentx.{server_idx}.{client_idx}" + ) output = _run_benchmark_with_log( client_cmd_with_port, bench_env, @@ -2166,6 +2407,12 @@ def _parse_disagg_config_file(self, config_file_path: str, config_file: str): server_env_var = environment.get("server_env_var", "") client_env_var = environment.get("client_env_var", "") internal_request_auth_key = self._resolve_internal_request_auth_key(config) + # Optional disagg-server-level keys, same names as bench-trtllm-disagg's + # sweep config so a recipe can be carried over unchanged. + router_config = config.get("router_config", None) + ctx_router_config = config.get("ctx_router_config", None) + gen_router_config = config.get("gen_router_config", None) + server_config_extra = config.get("server_config_extra", None) # Parse concurrency_list - can be string or list concurrency_str = benchmark.get("concurrency_list", "1") @@ -2238,6 +2485,10 @@ def _parse_disagg_config_file(self, config_file_path: str, config_file: str): hardware=hardware, server_env_var=server_env_var, internal_request_auth_key=internal_request_auth_key, + router_config=router_config, + ctx_router_config=ctx_router_config, + gen_router_config=gen_router_config, + server_config_extra=server_config_extra, ) # server_configs is a list with one element (tuple of ctx, gen, disagg config) @@ -2248,6 +2499,15 @@ def _parse_disagg_config_file(self, config_file_path: str, config_file: str): osl = 1 if benchmark_mode == "ctx_only" else benchmark.get("output_length", 1024) dataset_file = "" if benchmark_mode == "ctx_only" else benchmark.get("dataset_file", "") use_nv_sa_benchmark = benchmark.get("use_nv_sa_benchmark", False) + benchmark_client = benchmark.get("benchmark_client", "") + if benchmark_client not in ("", AGENTX_BENCHMARK_CLIENT): + # There is no schema validation on these yamls, so an unrecognised + # value would otherwise fall through to the default client and + # quietly measure the wrong workload. + raise ValueError( + f"Unknown benchmark_client {benchmark_client!r}; " + f"expected '' or {AGENTX_BENCHMARK_CLIENT!r}." + ) if benchmark_mode == "ctx_only": spec_decoding = bool(ctx_server_config.spec_decoding_type) @@ -2276,6 +2536,7 @@ def _parse_disagg_config_file(self, config_file_path: str, config_file: str): "streaming": benchmark.get("streaming", True), "dataset_file": dataset_file, "use_nv_sa_benchmark": use_nv_sa_benchmark, + "benchmark_client": benchmark_client, "accuracy_config": accuracy_data, "only_run_accuracy": only_run_accuracy, } @@ -2428,6 +2689,10 @@ def _get_disagg_commands(self, output_dir: str, test_output_dir: str): test_output_dir=test_output_dir, model_name=disagg_config.model_name, internal_request_auth_key=disagg_config.internal_request_auth_key, + router_config=disagg_config.router_config, + ctx_router_config=disagg_config.ctx_router_config, + gen_router_config=disagg_config.gen_router_config, + server_config_extra=disagg_config.server_config_extra, client_configs=self.server_client_configs, server_configs=list(self.server_configs), ) @@ -2564,9 +2829,34 @@ def check_test_failure(self): # Spec-decoding tests must report 'Mean Avg Decoded Tokens per Iter' # (parsed as 'al'). If the field is missing the test fails here so the # data is never uploaded to OpenSearch. + # AgentX is exempt: 'al' comes from TRT-LLM's non-standard + # avg_decoded_tokens_per_iter response field, which aiperf does + # not propagate. It is not a real loss of signal, because every + # agentx lane pins the accepted length with + # TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS -- recorded on the + # case as l_force_num_accepted_tokens -- so 'al' would + # be a restatement of a configured constant rather than a + # measurement. The exemption is conditioned on that forcing + # actually being in effect rather than merely documented, so an + # agentx lane that ever runs spec decoding without pinning the + # accepted count still hard-fails here. + agentx_al_exempt = False + if ( + client_idx < len(client_configs) + and client_configs[client_idx].benchmark_client == AGENTX_BENCHMARK_CLIENT + ): + server_entry = self.server_configs[server_idx] + # disagg stores (ctx, gen, disagg); aggregated stores one config. + candidates = ( + server_entry if isinstance(server_entry, tuple) else (server_entry,) + ) + agentx_al_exempt = any( + getattr(c, "force_num_accepted_tokens", 0) for c in candidates + ) if ( client_idx < len(client_configs) and client_configs[client_idx].spec_decoding + and not agentx_al_exempt and "al" not in metrics ): error_msg += ( diff --git a/tests/integration/test_lists/qa/llm_perf_disagg.yml b/tests/integration/test_lists/qa/llm_perf_disagg.yml index 0a86b3199735..e22ddd7db046 100644 --- a/tests/integration/test_lists/qa/llm_perf_disagg.yml +++ b/tests/integration/test_lists/qa/llm_perf_disagg.yml @@ -53,6 +53,9 @@ llm_perf_disagg: - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_deepseek-v4-pro-fp4_8k1k_con1229_ctx7_dep4_gen1_dep8_eplb384_mtp3_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_deepseek-v4-pro-fp4_8k1k_con666_ctx8_dep4_gen1_dep32_eplb384_mtp3_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_deepseek-v4-pro-fp4_8k1k_con2_ctx1_dep4_gen5_tep4_eplb0_mtp3_ccb-NIXL] TIMEOUT (120) + # GB300 DeepSeek-V4-Pro-DSpark (AgentX agentic trace replay) + - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_deepseek-v4-pro-dspark_agentx_con1156_ctx2_dep8_gen1_dep8_eplb0_dspark3_ccb-NIXL] TIMEOUT (210) + - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_deepseek-v4-pro-dspark_agentx_con1456_ctx3_dep8_gen1_dep16_eplb0_dspark5_ccb-NIXL] TIMEOUT (240) # GB300 Kimi-K2.5-Thinking - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_kimi-k25-thinking-fp4_8k1k_con4_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (120) diff --git a/tests/integration/test_lists/qa/llm_perf_multinode.txt b/tests/integration/test_lists/qa/llm_perf_multinode.txt index 8021fdac6f81..e11cadae2a6a 100644 --- a/tests/integration/test_lists/qa/llm_perf_multinode.txt +++ b/tests/integration/test_lists/qa/llm_perf_multinode.txt @@ -45,6 +45,9 @@ perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_deepseek-v4-pro-fp4_8k1k_con perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_deepseek-v4-pro-fp4_8k1k_con666_ctx8_dep4_gen1_dep32_eplb384_mtp3_ccb-NIXL] perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_deepseek-v4-pro-fp4_8k1k_con2_ctx1_dep4_gen5_tep4_eplb0_mtp3_ccb-NIXL] +# GB300 DeepSeek-V4-Pro-DSpark (AgentX agentic trace replay) +perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_deepseek-v4-pro-dspark_agentx_con1156_ctx2_dep8_gen1_dep8_eplb0_dspark3_ccb-NIXL] + # GB300 GLM-5 perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_glm-5-fp4_8k1k_con1024_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-NIXL] perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb300_glm-5-fp4_8k1k_con1_ctx1_dep2_gen1_tep8_eplb0_mtp3_ccb-NIXL] diff --git a/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx2_node2_gpu8_gen1_node2_gpu8.yml b/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx2_node2_gpu8_gen1_node2_gpu8.yml new file mode 100644 index 000000000000..f382c09eb0b6 --- /dev/null +++ b/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx2_node2_gpu8_gen1_node2_gpu8.yml @@ -0,0 +1,23 @@ +version: 0.0.1 +l0_gb300_multi_nodes_perf_sanity_ctx2_node2_gpu8_gen1_node2_gpu8: +- condition: + ranges: + # 2 ctx workers each with 2 nodes and 8 GPUs + # 1 gen worker with 2 nodes and 8 GPUs + system_gpu_count: + gte: 24 + lte: 24 + wildcards: + gpu: + - '*gb300*' + terms: + stage: post_merge + backend: pytorch + tests: + # deepseek-v4-pro-dspark agentx con1156. + # e2e only: the agentx trace's median ISL is ~101K tokens, so the gen_only + # fill gate (which needs `concurrency` requests resident before it opens, + # clamped to the 512-request admission capacity) would demand far more gen KV + # than 8x GB300 can hold and would never open. gen_only for agentx needs the + # gen_only_no_context flavor and is deliberately left to a follow-up. + - perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_deepseek-v4-pro-dspark_agentx_con1156_ctx2_dep8_gen1_dep8_eplb0_dspark3_ccb-NIXL] TIMEOUT (210) diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v4-pro-dspark_agentx_con1156_ctx2_dep8_gen1_dep8_eplb0_dspark3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v4-pro-dspark_agentx_con1156_ctx2_dep8_gen1_dep8_eplb0_dspark3_ccb-NIXL.yaml new file mode 100644 index 000000000000..4adc07545af2 --- /dev/null +++ b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v4-pro-dspark_agentx_con1156_ctx2_dep8_gen1_dep8_eplb0_dspark3_ccb-NIXL.yaml @@ -0,0 +1,133 @@ +metadata: + model_name: deepseek_v4_pro_dspark + precision: fp4 + model_dir_name: DeepSeek-V4-Pro-DSpark + supported_gpus: + - GB300 + script_file: disaggr_torch.slurm + benchmark_type: agentx +slurm: + script_file: disaggr_torch.slurm + partition: + account: + job_time: 04:00:00 + job_name: unified-benchmark + extra_args: --gres=gpu:4 + numa_bind: true +benchmark: + mode: e2e + use_nv_sa_benchmark: false + benchmark_client: agentx + multi_round: 1 + benchmark_ratio: 0.8 + streaming: true + concurrency_list: '1156' + input_length: 990016 + output_length: 6579 + dataset_file: semianalysis_cc_traces_weka_062126 +hardware: + gpus_per_node: 4 + num_ctx_servers: 2 + num_gen_servers: 1 +environment: + container_mount: + container_image: + model_path: + trtllm_repo: '' + build_wheel: false + work_dir: + client_env_var: AGENTX_MAX_CTX=996579 AGENTX_DURATION=3600 AGENTX_WARMUP_PER_LANE=3 + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 ENROOT_ALLOW_DEV=yes NCCL_GRAPH_MIXING_SUPPORT=0 MIMALLOC_ARENA_RESERVE=0 OMP_NUM_THREADS=1 TRTLLM_PINNED_WEIGHT_STAGING=1 + ctx_worker_env_var: PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True CUDA_SCALE_LAUNCH_QUEUES=4x TLLM_INDEXER_MQA_LOGITS_ELEM_BUDGET=1073741824 TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2.08 + gen_worker_env_var: TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2.08 + server_env_var: TRTLLM_SERVER_DISABLE_GC=1 +profiling: + nsys_on: false +ctx_router_config: + type: conversation +server_config_extra: + gen_strip_message_history: true + gen_tokids_ctxbytes: true + server_keep_alive_timeout: 3600 +worker_config: + gen: + print_iter_log: true + max_batch_size: 64 + max_num_tokens: 256 + max_seq_len: 996595 + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + cuda_graph_config: + enable_padding: true + batch_sizes: [1, 2, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64] + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + dtype: fp8 + tokens_per_block: 128 + event_buffer_max_size: 0 + avg_seq_len: 200000 + host_cache_size: 0 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + stream_interval: 20 + num_postprocess_workers: 4 + speculative_config: + decoding_type: DSpark + max_draft_len: 3 + block_size: 3 + speculative_model: DeepSeek-V4-Pro-DSpark + custom_tokenizer: deepseek_v4 + return_perf_metrics: true + perf_metrics_max_requests: 100000 + perf_metrics_output_dir: perf_metrics/gen + sparse_attention_config: + algorithm: deepseek_v4 + enable_heuristic_topk: true + num_serve_frontends: 4 + ctx: + print_iter_log: true + max_batch_size: 256 + max_num_tokens: 8192 + max_seq_len: 990016 + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + enable_attention_dp: true + cuda_graph_config: null + disable_overlap_scheduler: false + kv_cache_config: + enable_block_reuse: true + free_gpu_memory_fraction: 0.8 + dtype: fp8 + tokens_per_block: 128 + event_buffer_max_size: 0 + block_reuse_config: + policy: per_conversation + max_num_turns: 5 + pool_ratio: [0.55, 0.22, 0.23] + host_cache_size: 193273528320 + moe_config: + backend: CUTEDSL + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + custom_tokenizer: deepseek_v4 + return_perf_metrics: true + perf_metrics_max_requests: 100000 + perf_metrics_output_dir: perf_metrics/ctx + enable_chunked_prefill: true + attention_dp_config: + kv_cache_routing_conversation_affinity: true + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + num_serve_frontends: 8 diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v4-pro-dspark_agentx_con1456_ctx3_dep8_gen1_dep16_eplb0_dspark5_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v4-pro-dspark_agentx_con1456_ctx3_dep8_gen1_dep16_eplb0_dspark5_ccb-NIXL.yaml new file mode 100644 index 000000000000..ee32a067e580 --- /dev/null +++ b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v4-pro-dspark_agentx_con1456_ctx3_dep8_gen1_dep16_eplb0_dspark5_ccb-NIXL.yaml @@ -0,0 +1,133 @@ +metadata: + model_name: deepseek_v4_pro_dspark + precision: fp4 + model_dir_name: DeepSeek-V4-Pro-DSpark + supported_gpus: + - GB300 + script_file: disaggr_torch.slurm + benchmark_type: agentx +slurm: + script_file: disaggr_torch.slurm + partition: + account: + job_time: 04:00:00 + job_name: unified-benchmark + extra_args: --gres=gpu:4 + numa_bind: true +benchmark: + mode: e2e + use_nv_sa_benchmark: false + benchmark_client: agentx + multi_round: 1 + benchmark_ratio: 0.8 + streaming: true + concurrency_list: '1456' + input_length: 990016 + output_length: 6579 + dataset_file: semianalysis_cc_traces_weka_062126 +hardware: + gpus_per_node: 4 + num_ctx_servers: 3 + num_gen_servers: 1 +environment: + container_mount: + container_image: + model_path: + trtllm_repo: '' + build_wheel: false + work_dir: + client_env_var: AGENTX_MAX_CTX=996579 AGENTX_DURATION=3600 AGENTX_WARMUP_PER_LANE=3 + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 ENROOT_ALLOW_DEV=yes NCCL_GRAPH_MIXING_SUPPORT=0 MIMALLOC_ARENA_RESERVE=0 OMP_NUM_THREADS=1 TRTLLM_PINNED_WEIGHT_STAGING=1 + ctx_worker_env_var: PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True CUDA_SCALE_LAUNCH_QUEUES=4x TLLM_INDEXER_MQA_LOGITS_ELEM_BUDGET=1073741824 TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2.74 + gen_worker_env_var: TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2.74 + server_env_var: TRTLLM_SERVER_DISABLE_GC=1 +profiling: + nsys_on: false +ctx_router_config: + type: conversation +server_config_extra: + gen_strip_message_history: true + gen_tokids_ctxbytes: true + server_keep_alive_timeout: 3600 +worker_config: + gen: + print_iter_log: true + max_batch_size: 32 + max_num_tokens: 192 + max_seq_len: 996595 + tensor_parallel_size: 16 + moe_expert_parallel_size: 16 + pipeline_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + cuda_graph_config: + enable_padding: true + batch_sizes: [1, 2, 4, 8, 16, 24, 32] + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.7 + dtype: fp8 + tokens_per_block: 128 + event_buffer_max_size: 0 + avg_seq_len: 200000 + host_cache_size: 0 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + stream_interval: 20 + num_postprocess_workers: 4 + speculative_config: + decoding_type: DSpark + max_draft_len: 5 + block_size: 5 + speculative_model: DeepSeek-V4-Pro-DSpark + custom_tokenizer: deepseek_v4 + return_perf_metrics: true + perf_metrics_max_requests: 100000 + perf_metrics_output_dir: perf_metrics/gen + sparse_attention_config: + algorithm: deepseek_v4 + enable_heuristic_topk: true + num_serve_frontends: 4 + ctx: + print_iter_log: true + max_batch_size: 256 + max_num_tokens: 8192 + max_seq_len: 990016 + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + enable_attention_dp: true + cuda_graph_config: null + disable_overlap_scheduler: false + kv_cache_config: + enable_block_reuse: true + free_gpu_memory_fraction: 0.8 + dtype: fp8 + tokens_per_block: 128 + event_buffer_max_size: 0 + block_reuse_config: + policy: per_conversation + max_num_turns: 5 + pool_ratio: [0.55, 0.22, 0.23] + host_cache_size: 193273528320 + moe_config: + backend: CUTEDSL + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + custom_tokenizer: deepseek_v4 + return_perf_metrics: true + perf_metrics_max_requests: 100000 + perf_metrics_output_dir: perf_metrics/ctx + enable_chunked_prefill: true + attention_dp_config: + kv_cache_routing_conversation_affinity: true + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + num_serve_frontends: 8 diff --git a/tests/scripts/perf/disaggregated/gb300_deepseek-v4-pro-dspark_agentx_con1156_ctx2_dep8_gen1_dep8_eplb0_dspark3_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb300_deepseek-v4-pro-dspark_agentx_con1156_ctx2_dep8_gen1_dep8_eplb0_dspark3_ccb-NIXL.yaml new file mode 100644 index 000000000000..4adc07545af2 --- /dev/null +++ b/tests/scripts/perf/disaggregated/gb300_deepseek-v4-pro-dspark_agentx_con1156_ctx2_dep8_gen1_dep8_eplb0_dspark3_ccb-NIXL.yaml @@ -0,0 +1,133 @@ +metadata: + model_name: deepseek_v4_pro_dspark + precision: fp4 + model_dir_name: DeepSeek-V4-Pro-DSpark + supported_gpus: + - GB300 + script_file: disaggr_torch.slurm + benchmark_type: agentx +slurm: + script_file: disaggr_torch.slurm + partition: + account: + job_time: 04:00:00 + job_name: unified-benchmark + extra_args: --gres=gpu:4 + numa_bind: true +benchmark: + mode: e2e + use_nv_sa_benchmark: false + benchmark_client: agentx + multi_round: 1 + benchmark_ratio: 0.8 + streaming: true + concurrency_list: '1156' + input_length: 990016 + output_length: 6579 + dataset_file: semianalysis_cc_traces_weka_062126 +hardware: + gpus_per_node: 4 + num_ctx_servers: 2 + num_gen_servers: 1 +environment: + container_mount: + container_image: + model_path: + trtllm_repo: '' + build_wheel: false + work_dir: + client_env_var: AGENTX_MAX_CTX=996579 AGENTX_DURATION=3600 AGENTX_WARMUP_PER_LANE=3 + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 ENROOT_ALLOW_DEV=yes NCCL_GRAPH_MIXING_SUPPORT=0 MIMALLOC_ARENA_RESERVE=0 OMP_NUM_THREADS=1 TRTLLM_PINNED_WEIGHT_STAGING=1 + ctx_worker_env_var: PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True CUDA_SCALE_LAUNCH_QUEUES=4x TLLM_INDEXER_MQA_LOGITS_ELEM_BUDGET=1073741824 TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2.08 + gen_worker_env_var: TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2.08 + server_env_var: TRTLLM_SERVER_DISABLE_GC=1 +profiling: + nsys_on: false +ctx_router_config: + type: conversation +server_config_extra: + gen_strip_message_history: true + gen_tokids_ctxbytes: true + server_keep_alive_timeout: 3600 +worker_config: + gen: + print_iter_log: true + max_batch_size: 64 + max_num_tokens: 256 + max_seq_len: 996595 + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + cuda_graph_config: + enable_padding: true + batch_sizes: [1, 2, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64] + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + dtype: fp8 + tokens_per_block: 128 + event_buffer_max_size: 0 + avg_seq_len: 200000 + host_cache_size: 0 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + stream_interval: 20 + num_postprocess_workers: 4 + speculative_config: + decoding_type: DSpark + max_draft_len: 3 + block_size: 3 + speculative_model: DeepSeek-V4-Pro-DSpark + custom_tokenizer: deepseek_v4 + return_perf_metrics: true + perf_metrics_max_requests: 100000 + perf_metrics_output_dir: perf_metrics/gen + sparse_attention_config: + algorithm: deepseek_v4 + enable_heuristic_topk: true + num_serve_frontends: 4 + ctx: + print_iter_log: true + max_batch_size: 256 + max_num_tokens: 8192 + max_seq_len: 990016 + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + enable_attention_dp: true + cuda_graph_config: null + disable_overlap_scheduler: false + kv_cache_config: + enable_block_reuse: true + free_gpu_memory_fraction: 0.8 + dtype: fp8 + tokens_per_block: 128 + event_buffer_max_size: 0 + block_reuse_config: + policy: per_conversation + max_num_turns: 5 + pool_ratio: [0.55, 0.22, 0.23] + host_cache_size: 193273528320 + moe_config: + backend: CUTEDSL + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + custom_tokenizer: deepseek_v4 + return_perf_metrics: true + perf_metrics_max_requests: 100000 + perf_metrics_output_dir: perf_metrics/ctx + enable_chunked_prefill: true + attention_dp_config: + kv_cache_routing_conversation_affinity: true + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + num_serve_frontends: 8 diff --git a/tests/scripts/perf/disaggregated/gb300_deepseek-v4-pro-dspark_agentx_con1456_ctx3_dep8_gen1_dep16_eplb0_dspark5_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb300_deepseek-v4-pro-dspark_agentx_con1456_ctx3_dep8_gen1_dep16_eplb0_dspark5_ccb-NIXL.yaml new file mode 100644 index 000000000000..ee32a067e580 --- /dev/null +++ b/tests/scripts/perf/disaggregated/gb300_deepseek-v4-pro-dspark_agentx_con1456_ctx3_dep8_gen1_dep16_eplb0_dspark5_ccb-NIXL.yaml @@ -0,0 +1,133 @@ +metadata: + model_name: deepseek_v4_pro_dspark + precision: fp4 + model_dir_name: DeepSeek-V4-Pro-DSpark + supported_gpus: + - GB300 + script_file: disaggr_torch.slurm + benchmark_type: agentx +slurm: + script_file: disaggr_torch.slurm + partition: + account: + job_time: 04:00:00 + job_name: unified-benchmark + extra_args: --gres=gpu:4 + numa_bind: true +benchmark: + mode: e2e + use_nv_sa_benchmark: false + benchmark_client: agentx + multi_round: 1 + benchmark_ratio: 0.8 + streaming: true + concurrency_list: '1456' + input_length: 990016 + output_length: 6579 + dataset_file: semianalysis_cc_traces_weka_062126 +hardware: + gpus_per_node: 4 + num_ctx_servers: 3 + num_gen_servers: 1 +environment: + container_mount: + container_image: + model_path: + trtllm_repo: '' + build_wheel: false + work_dir: + client_env_var: AGENTX_MAX_CTX=996579 AGENTX_DURATION=3600 AGENTX_WARMUP_PER_LANE=3 + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 ENROOT_ALLOW_DEV=yes NCCL_GRAPH_MIXING_SUPPORT=0 MIMALLOC_ARENA_RESERVE=0 OMP_NUM_THREADS=1 TRTLLM_PINNED_WEIGHT_STAGING=1 + ctx_worker_env_var: PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True CUDA_SCALE_LAUNCH_QUEUES=4x TLLM_INDEXER_MQA_LOGITS_ELEM_BUDGET=1073741824 TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2.74 + gen_worker_env_var: TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2.74 + server_env_var: TRTLLM_SERVER_DISABLE_GC=1 +profiling: + nsys_on: false +ctx_router_config: + type: conversation +server_config_extra: + gen_strip_message_history: true + gen_tokids_ctxbytes: true + server_keep_alive_timeout: 3600 +worker_config: + gen: + print_iter_log: true + max_batch_size: 32 + max_num_tokens: 192 + max_seq_len: 996595 + tensor_parallel_size: 16 + moe_expert_parallel_size: 16 + pipeline_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + cuda_graph_config: + enable_padding: true + batch_sizes: [1, 2, 4, 8, 16, 24, 32] + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.7 + dtype: fp8 + tokens_per_block: 128 + event_buffer_max_size: 0 + avg_seq_len: 200000 + host_cache_size: 0 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + stream_interval: 20 + num_postprocess_workers: 4 + speculative_config: + decoding_type: DSpark + max_draft_len: 5 + block_size: 5 + speculative_model: DeepSeek-V4-Pro-DSpark + custom_tokenizer: deepseek_v4 + return_perf_metrics: true + perf_metrics_max_requests: 100000 + perf_metrics_output_dir: perf_metrics/gen + sparse_attention_config: + algorithm: deepseek_v4 + enable_heuristic_topk: true + num_serve_frontends: 4 + ctx: + print_iter_log: true + max_batch_size: 256 + max_num_tokens: 8192 + max_seq_len: 990016 + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + enable_attention_dp: true + cuda_graph_config: null + disable_overlap_scheduler: false + kv_cache_config: + enable_block_reuse: true + free_gpu_memory_fraction: 0.8 + dtype: fp8 + tokens_per_block: 128 + event_buffer_max_size: 0 + block_reuse_config: + policy: per_conversation + max_num_turns: 5 + pool_ratio: [0.55, 0.22, 0.23] + host_cache_size: 193273528320 + moe_config: + backend: CUTEDSL + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + custom_tokenizer: deepseek_v4 + return_perf_metrics: true + perf_metrics_max_requests: 100000 + perf_metrics_output_dir: perf_metrics/ctx + enable_chunked_prefill: true + attention_dp_config: + kv_cache_routing_conversation_affinity: true + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + num_serve_frontends: 8