From b1a40b4d9ec21f84ca83d02dbc6a3a3642cf7075 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Wed, 7 Oct 2026 22:28:01 -0500 Subject: [PATCH 01/14] refactor(srt): compose AgentX recipes from fragments and set power on master rows AgentX recipes, single- and multi-node, become fragments like the fixed-sequence ones. configs/srt-recipes/agentic-{single,multi}.yaml supply the client command; the binder writes the master model, image, precision and identity, and the launcher hands multi-node clients RESULT_DIR, AIPERF_DATASET_MMAP_CACHE_DIR and HF_HUB_CACHE from the volumes it mounts (agentic-volume-mounts replaces lanes._AGENTIC_CACHES). Dead keys and copies of the workflow environment leave the fragments; a fragment that sets a bound key fails before submission. A multinode search-space row with power: true replaces recipe telemetry blocks: the shared telemetry-dcgm.yaml is merged with the cluster's power-exporter-port and benchmark.concurrencies is bound after variant selection, so bundles can measure power. POWER_LANES now judges the row's field instead of reading recipe text, and inject_concurrencies is gone. Selected zip variants drop null-valued keys as srtctl apply does, and h200-dgxc GLM-5.2 FP8 AgentX keeps serving the Hub snapshot. --- .../workflows/benchmark-multinode-tmpl.yml | 1 + .../multi_node/srt-slurm-recipes/RECIPES.md | 19 ++- .../agentx/disagg-lmcache-dspark.yaml | 25 --- .../sglang/b200-fp4/agentx/agg-variants.yaml | 17 +- .../b200-fp4/agentx/disagg-variants.yaml | 18 +- .../agentx/agg-gb200-tp8-c1-mtp-hicache.yaml | 18 +- .../agentx/agg-gb200-tp8-c4-mtp-hicache.yaml | 18 +- ...0-1p1d-dep16-dep32-c256-mtp-kvoffload.yaml | 18 +- ...00-1p1d-dep8-dep16-c128-mtp-kvoffload.yaml | 19 +-- ...200-1p1d-dep8-dep16-c64-mtp-kvoffload.yaml | 19 +-- ...-2p1d-dep16-dep32-c1024-mtp-kvoffload.yaml | 18 +- ...-2p1d-dep16-dep32-c1280-mtp-kvoffload.yaml | 18 +- ...0-2p1d-dep16-dep32-c768-mtp-kvoffload.yaml | 18 +- .../sglang/gb300-fp4/agentx/agg-variants.yaml | 17 +- .../gb300-fp4/agentx/disagg-variants.yaml | 18 +- .../agentx/agg-tp8-mtp-kvoffload.yaml | 30 +--- .../mi355x-fp4/agentx/disagg-umbp-dspark.yaml | 28 --- .../gb300-fp4/agentx/disagg-variants.yaml | 43 +---- .../gb200-fp4/agentx/agg-mtp-variants.yaml | 19 +-- .../gb200-fp4/agentx/agg-mtp3-variants.yaml | 21 +-- .../gb200-fp4/agentx/disagg-mtp-variants.yaml | 17 +- .../agentx/disagg-mtp3-variants.yaml | 19 +-- .../vllm/gb300-fp4/agentx/agg-variants.yaml | 21 +-- .../agentx/disagg-1p4d-dep4-tp8-c4-mtp.yaml | 37 +--- .../gb300-fp4/agentx/disagg-mtp-variants.yaml | 19 +-- .../gb300-fp4/agentx/disagg-variants.yaml | 21 --- .../sglang/gb300-fp4/agentx/agg-variants.yaml | 17 +- .../gb300-fp4/agentx/disagg-variants.yaml | 34 +--- .../sglang/b200-fp4/agentx/agg-variants.yaml | 16 +- .../b200-fp4/agentx/disagg-variants.yaml | 16 +- .../gb200-fp4/agentx/agg-mtp-variants.yaml | 16 +- .../glm5.2/sglang/gb200-fp4/agentx/agg.yaml | 27 +-- .../agentx/disagg-dep8-mtp-variants.yaml | 16 +- .../gb200-fp4/agentx/disagg-mtp-variants.yaml | 16 +- .../gb300-fp4/agentx/agg-tp8-c1-mtp.yaml | 15 +- .../gb300-fp4/agentx/agg-tp8-c2-mtp.yaml | 15 +- .../gb300-fp4/agentx/agg-tp8-c4-mtp.yaml | 15 +- .../disagg-1p2d-p-dep4-d-tp4-c48-mtp.yaml | 15 +- .../disagg-1p4d-p-dep4-d-tp4-c45-mtp.yaml | 15 +- .../disagg-1p6d-p-dep4-d-tp4-c45-mtp.yaml | 15 +- .../disagg-2p1d-p-dep8-d-dep16-c192-mtp.yaml | 15 +- .../h200-fp8/agentx/agg-h200-tp8-mtp-c12.yaml | 28 +-- .../h200-fp8/agentx/agg-h200-tp8-mtp-c16.yaml | 28 +-- .../h200-fp8/agentx/agg-h200-tp8-mtp-c2.yaml | 28 +-- .../h200-fp8/agentx/agg-h200-tp8-mtp-c4.yaml | 28 +-- .../h200-fp8/agentx/agg-h200-tp8-mtp-c8.yaml | 28 +-- ...disagg-1p1d-pcp8-tp8-dp8-mtp6-hicache.yaml | 26 --- .../agentx/disagg-2p2d-pcp8-tp8-dp8-mtp.yaml | 28 --- .../gb300-fp4/agentx/disagg-variants.yaml | 15 -- .../agentx/disagg-1p1d-tp8-mtp.yaml | 10 -- .../agentx/agg-tp8pp2-mooncake-c1.yaml | 31 +--- .../agentx/agg-tp8pp2-mooncake-c14.yaml | 31 +--- .../agentx/agg-tp8pp2-mooncake-c24.yaml | 31 +--- .../agentx/agg-tp8pp2-mooncake-c32.yaml | 31 +--- .../agentx/agg-tp8pp2-mooncake-c4.yaml | 31 +--- .../agentx/agg-tp8pp2-mooncake-c48.yaml | 31 +--- .../agentx/agg-tp8pp2-mooncake-c56.yaml | 31 +--- .../agentx/agg-tp8pp2-mooncake-c72.yaml | 31 +--- .../agentx/agg-tp8pp2-mooncake-c8.yaml | 31 +--- .../agentx/agg-tp8pp2-mooncake-c80.yaml | 31 +--- .../agentx/agg-tp8pp2-mooncake-c96.yaml | 31 +--- .../agg-dcp16-dspark4-maxseq2-mooncake.yaml | 32 +--- .../agentx/agg-dcp16-nospec-mooncake.yaml | 32 +--- .../agentx/agg-tp8pp2-mooncake-c16.yaml | 32 +--- .../agentx/agg-tp8pp2-mooncake-c32.yaml | 32 +--- .../agentx/agg-tp8pp2-mooncake-c48.yaml | 32 +--- .../agentx/agg-tp8pp2-mooncake-c72.yaml | 32 +--- .../agentx/agg-tp8pp2-mooncake-c96.yaml | 32 +--- .../vllm/gb200-fp4/agentx/agg-variants.yaml | 19 +-- .../agentx/agg-dcp8-dspark4-mooncake.yaml | 32 +--- .../agg-dcp8-dspark7-maxseq2-mooncake.yaml | 32 +--- ...isagg-1p1d-dcp8-dcp8-dspark4-mooncake.yaml | 32 +--- ...isagg-1p2d-dcp8-dcp8-dspark4-mooncake.yaml | 32 +--- ...isagg-1p3d-dcp8-dcp8-dspark4-mooncake.yaml | 32 +--- ...isagg-1p3d-dcp8-dcp8-dspark7-mooncake.yaml | 32 +--- .../agentx/agg-tp16dp2ep32-latency.yaml | 29 +--- .../agentx/agg-tp8dp4ep32-balanced.yaml | 29 +--- .../agentx/agg-tp8dp4ep32-vllm-simple.yaml | 29 +--- .../trtllm/gb200-fp4/agentx/agg-variants.yaml | 43 ----- .../agentx/agg-nightly-native-variants.yaml | 17 +- .../gb200-fp4/agentx/disagg-variants.yaml | 17 +- .../gb300-fp4/agentx/disagg-variants.yaml | 19 +-- ...sagg-1p1d-p-tp4-d-tp4-hicache-c16-mtp.yaml | 29 ---- ...sagg-1p1d-p-tp4-d-tp4-hicache-c24-mtp.yaml | 29 ---- ...sagg-1p1d-p-tp4-d-tp4-hicache-c32-mtp.yaml | 29 ---- ...4-d-tp4-hicache-c48-write-through-mtp.yaml | 29 ---- ...4-d-tp4-hicache-c64-write-through-mtp.yaml | 29 ---- ...isagg-1p1d-p-tp4-d-tp4-hicache-c8-mtp.yaml | 29 ---- .../b300-fp8/agentx/disagg-variants.yaml | 44 +---- .../sglang/gb200-fp4/agentx/agg-variants.yaml | 16 +- ...sagg-1p1d-p-tp4-d-tp4-b128-c1x2x8-mtp.yaml | 12 -- ...tp8-ep8-d-tp8-ep8-b1024-c32x48x80-mtp.yaml | 12 -- .../8k1k/disagg-1p1d-tp4-tp4-stp.yaml | 12 -- ...4-dp4-d-tp16-ep16-dp16-b1024-c480-mtp.yaml | 12 -- .../8k1k/disagg-4p1d-dep4-dep16-stp.yaml | 12 -- ...4-dp4-d-tp16-ep16-dp16-b1024-c768-mtp.yaml | 12 -- ...-dp4-d-tp16-ep16-dp16-b1024-c1280-mtp.yaml | 12 -- ...-dp4-d-tp16-ep16-dp16-b2048-c1344-mtp.yaml | 12 -- ...d-tp16-ep16-dp16-b2048-c1920x2304-mtp.yaml | 12 -- ...p4-d-tp4-b128-c1x4x8x16x32x64x256-stp.yaml | 14 -- ...-dp4-d-tp16-ep16-dp16-b4096-c2048-stp.yaml | 12 -- ...-dp4-d-tp16-ep16-dp16-b5120-c5120-stp.yaml | 12 -- ...-dp4-d-tp16-ep16-dp16-b5120-c5120-stp.yaml | 12 -- .../gb300-fp4/agentx/agg-pareto-variants.yaml | 12 -- .../sglang/gb300-fp4/agentx/agg-variants.yaml | 12 -- .../agentx/disagg-pp-pareto-variants.yaml | 13 -- .../gb300-fp4/agentx/disagg-variants.yaml | 13 -- ...agg-1p1d-p-tp4-d-tp4-b1024-c1x2x8-mtp.yaml | 12 -- ...tp8-ep8-d-tp8-ep8-b1024-c32x48x80-mtp.yaml | 12 -- .../8k1k/disagg-1p1d-tp4-tp4-stp.yaml | 14 -- ...4-dp4-d-tp16-ep16-dp16-b1024-c480-mtp.yaml | 12 -- .../8k1k/disagg-4p1d-dep4-dep16-stp.yaml | 12 -- ...4-dp4-d-tp16-ep16-dp16-b1024-c768-mtp.yaml | 12 -- ...-dp4-d-tp16-ep16-dp16-b1024-c1280-mtp.yaml | 12 -- ...-dp4-d-tp16-ep16-dp16-b2048-c1344-mtp.yaml | 12 -- ...d-tp16-ep16-dp16-b2048-c1920x2304-mtp.yaml | 12 -- ...p4-c1-b1-mtp-hicache-nightly-20260913.yaml | 12 -- ...4-c44-b1-mtp-hicache-nightly-20260913.yaml | 12 -- ...4-c44-b2-mtp-hicache-nightly-20260913.yaml | 12 -- ...4-c44-b4-mtp-hicache-nightly-20260913.yaml | 12 -- ...p8-c1-b1-mtp-hicache-nightly-20260913.yaml | 13 -- ...ep4-c100-mtp-hicache-nightly-20260913.yaml | 13 -- ...ep4-c200-mtp-hicache-nightly-20260913.yaml | 13 -- ...ep4-c600-mtp-hicache-nightly-20260913.yaml | 13 -- ...p4-c1000-mtp-hicache-nightly-20260913.yaml | 13 -- .../gb300-fp4/agentx/disagg-variants.yaml | 19 +-- .../dsv4/atom/mi355x-fp4-mtp/agentic.yaml | 7 - .../dsv4/sglang/b200-fp4-mtp/agentic.yaml | 7 - .../dsv4/sglang/b300-fp4-eagle/agentic.yaml | 7 - .../dsv4/sglang/b300-fp4-mtp/agentic.yaml | 7 - .../dsv4/sglang/mi355x-fp4-mtp/agentic.yaml | 7 - .../dsv4/vllm/b200-fp4-mtp/agentic.yaml | 7 - .../dsv4/vllm/b300-fp4-mtp/agentic.yaml | 7 - .../dsv4/vllm/mi355x-fp4-mtp/agentic.yaml | 7 - .../sglang/b200-fp4-mtp/agentic.yaml | 7 - .../sglang/b300-fp4-mtp/agentic.yaml | 7 - .../sglang/gb200-fp4-mtp/agentic.yaml | 7 - .../sglang/gb300-fp4-mtp/agentic.yaml | 7 - .../sglang/h100-fp4-mtp/agentic.yaml | 7 - .../sglang/h200-fp4-mtp/agentic.yaml | 7 - .../sglang/mi355x-fp4-mtp/agentic.yaml | 9 - .../dsv41flash/vllm/b200-fp4-mtp/agentic.yaml | 7 - .../dsv41flash/vllm/b300-fp4-mtp/agentic.yaml | 7 - .../vllm/gb200-fp4-mtp/agentic.yaml | 7 - .../vllm/gb300-fp4-mtp/agentic.yaml | 7 - .../dsv41flash/vllm/h100-fp4-mtp/agentic.yaml | 7 - .../dsv41flash/vllm/h200-fp4-mtp/agentic.yaml | 7 - .../vllm/mi300x-fp4-mtp/agentic.yaml | 9 - .../vllm/mi325x-fp4-mtp/agentic.yaml | 9 - .../vllm/mi355x-fp4-mtp/agentic.yaml | 12 -- .../glm5.2/atom/mi355x-fp4-mtp/agentic.yaml | 7 - .../glm5.2/sglang/b200-fp4-mtp/agentic.yaml | 7 - .../glm5.2/sglang/b200-fp8-mtp/agentic.yaml | 7 - .../glm5.2/sglang/b300-fp4-mtp/agentic.yaml | 7 - .../glm5.2/sglang/b300-fp8-mtp/agentic.yaml | 7 - .../glm5.2/sglang/mi325x-fp8-mtp/agentic.yaml | 9 - .../glm5.2/sglang/mi355x-fp4-mtp/agentic.yaml | 7 - .../glm5.2/sglang/mi355x-fp8-mtp/agentic.yaml | 7 - .../kimik3/atom/mi355x-fp4-mtp/agentic.yaml | 7 - .../kimik3/vllm/b300-fp4-mtp/agentic.yaml | 7 - .../kimik3/vllm/mi355x-fp4-mtp/agentic.yaml | 7 - .../atom/mi355x-fp4-mtp/agentic.yaml | 7 - .../trtllm/b200-fp4-mtp/agentic.yaml | 7 - .../trtllm/b300-fp4-mtp/agentic.yaml | 7 - .../minimaxm3/vllm/b200-fp4-mtp/agentic.yaml | 7 - .../minimaxm3/vllm/b300-fp4-mtp/agentic.yaml | 7 - .../minimaxm3/vllm/h100-fp8-mtp/agentic.yaml | 8 - .../minimaxm3/vllm/h200-fp8-mtp/agentic.yaml | 8 - .../vllm/mi300x-fp8-mtp/agentic.yaml | 7 - .../vllm/mi325x-fp8-mtp/agentic.yaml | 7 - .../qwen3.5/sglang/b200-fp4-mtp/agentic.yaml | 9 - .../qwen3.5/sglang/b200-fp8-mtp/agentic.yaml | 9 - .../qwen3.5/sglang/b300-fp4-mtp/agentic.yaml | 9 - .../qwen3.5/sglang/b300-fp8-mtp/agentic.yaml | 9 - .../qwen3.5/sglang/h100-fp8-mtp/agentic.yaml | 12 -- .../qwen3.5/sglang/h100-fp8/agentic.yaml | 12 -- .../qwen3.5/sglang/h200-fp8-mtp/agentic.yaml | 14 -- .../sglang/mi300x-fp8-mtp/agentic.yaml | 9 - .../sglang/mi325x-fp8-mtp/agentic.yaml | 9 - .../sglang/mi355x-fp4-mtp/agentic.yaml | 9 - .../sglang/mi355x-fp8-mtp/agentic.yaml | 9 - .../sglang/b200-fp4-mtp/agentic.yaml | 9 - .../sglang/b300-fp4-mtp/agentic.yaml | 9 - .../sglang/h200-fp8-mtp/agentic.yaml | 11 -- inferencex-e2e/configs/CONFIGS.md | 7 +- inferencex-e2e/configs/amd-master.yaml | 5 +- inferencex-e2e/configs/nvidia-master.yaml | 68 ++++++++ inferencex-e2e/configs/runners.yaml | 11 ++ .../configs/srt-recipes/agentic-multi.yaml | 3 + .../configs/srt-recipes/agentic-single.yaml | 3 + .../configs/srt-recipes/telemetry-dcgm.yaml | 12 ++ inferencex-e2e/infx/clusters/slurm.py | 19 ++- .../infx/launch/drivers/srt/__init__.py | 42 +++-- .../infx/launch/drivers/srt/config.py | 42 ++++- .../infx/launch/drivers/srt/lanes.py | 14 +- .../infx/launch/drivers/srt/models.py | 54 ++---- .../infx/launch/drivers/srt/power.py | 71 ++------ .../infx/launch/drivers/srt/recipe.py | 38 +---- .../infx/launch/drivers/srt/submit.py | 15 +- inferencex-e2e/infx/launch/request.py | 1 + inferencex-e2e/infx/matrix/generate.py | 5 + inferencex-e2e/infx/matrix/validation.py | 9 + inferencex-e2e/infx/srt_slurm/single_node.py | 43 ++--- .../infx/srt_slurm/synthetic_acceptance.py | 28 ++- inferencex-e2e/infx/srt_slurm/workload.py | 159 +++++++++++++----- .../infx/tests/launch/test_launch_policy.py | 158 +++++++---------- .../infx/tests/launch/test_srt_config.py | 2 +- .../infx/tests/launch/test_srt_driver.py | 63 +++---- .../infx/tests/launch/test_srt_policy.py | 51 ++---- .../tests/launch/test_srt_recipe_edits.py | 29 +--- .../infx/tests/matrix/test_validation.py | 13 ++ .../tests/srt_slurm/test_srt_single_node.py | 31 ++-- .../infx/tests/srt_slurm/test_workload.py | 137 +++++++++++---- 213 files changed, 767 insertions(+), 3489 deletions(-) create mode 100644 inferencex-e2e/configs/srt-recipes/agentic-multi.yaml create mode 100644 inferencex-e2e/configs/srt-recipes/agentic-single.yaml create mode 100644 inferencex-e2e/configs/srt-recipes/telemetry-dcgm.yaml diff --git a/.github/workflows/benchmark-multinode-tmpl.yml b/.github/workflows/benchmark-multinode-tmpl.yml index 920a2945f9..42337135d1 100644 --- a/.github/workflows/benchmark-multinode-tmpl.yml +++ b/.github/workflows/benchmark-multinode-tmpl.yml @@ -145,6 +145,7 @@ env: EVAL_SUITE: ${{ inputs.eval-suite }} EVAL_CONC: ${{ inputs.eval-conc }} EVAL_LIMIT: ${{ inputs.eval-limit }} + POWER: ${{ fromJSON(inputs.config).power && '1' || '0' }} REQUIRE_POWER: ${{ inputs.require-power && '1' || '0' }} POWER_PRODUCER_SHA: ${{ inputs.power-producer-sha }} # Agentic-coding env. Fixed-seq-len jobs leave these empty. diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/RECIPES.md b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/RECIPES.md index e35f7cd043..ffe76d7e3c 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/RECIPES.md +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/RECIPES.md @@ -20,7 +20,7 @@ qwen3.5/trtllm/gb300-fp4/agentx/disagg-variants.yaml - Workloads are `1k1k`, `8k1k`, or `agentx`. Existing bundles spanning several fixed sequence lengths use `fixed-seq-len`; keep their override selectors intact. - Use lowercase, hyphen-separated filenames beginning with `agg` or `disagg`. Include topology and the settings that distinguish sibling recipes, such as parallelism, batch size, concurrency, MTP, offload, or cache configuration. Avoid dates, numbered latency/throughput labels, and repeating the model or hardware already in the path. - In topology names, `1p4d` denotes prefill/decode worker counts, not necessarily physical nodes. Role-qualified `p-tp4` and `d-tp8` identify prefill/decode TP; `b` denotes batch size and `c` concurrency. The YAML is authoritative for runtime settings. -- Name override bundles `*-variants.yaml`. Multi-node AgentX recipes that differ only per configuration share one bundle per master-config entry, usually `agg-variants.yaml` or `disagg-variants.yaml`: `base` holds the shared settings and each former recipe becomes a named `override_` block holding only its differences (plain overrides, not `zip_override_*`). Master rows select one with `srt-recipe: .yaml:override_`, relative to the entry's `srt-recipe-dir`. Recipes read as text by a launcher, such as power recipes with top-level `telemetry:`, stay standalone. Keep distinct sweep entry files separate even when their contents match: recipe paths participate in eval grouping. The Qwen3.5 `*-stp-sweep.yaml` and `*-mtp-sweep.yaml` pair preserves that existing distinction. +- Name override bundles `*-variants.yaml`. Multi-node AgentX recipes that differ only per configuration share one bundle per master-config entry, usually `agg-variants.yaml` or `disagg-variants.yaml`: `base` holds the shared settings and each former recipe becomes a named `override_` block holding only its differences (plain overrides, not `zip_override_*`). Master rows select one with `srt-recipe: .yaml:override_`, relative to the entry's `srt-recipe-dir`. Keep distinct sweep entry files separate even when their contents match: recipe paths participate in eval grouping. The Qwen3.5 `*-stp-sweep.yaml` and `*-mtp-sweep.yaml` pair preserves that existing distinction. - Update `srt-recipe-dir`, `srt-recipe`, and `eval-srt-recipe` references in master configs, launcher path rules, workflow filters, and local documentation together when moving a file. Preserve upstream source URLs as provenance and leave historical performance-changelog entries unchanged. No aliases for the old layout are provided. Shared runtime assets stay under `configs/` beside the model directories; they are not standalone recipes. The four files in `configs/dsv4-moe-load-balancer-configs/` are copied verbatim from NVIDIA/srt-slurm commit `deb1dfd9934398664f92d194169c183e009da83b`, preserving the EPLB initial expert assignments formerly used by the DSV4 TRT recipes; no checked-in recipe currently references them. The srt driver ([`infx/launch/drivers/srt/checkout.py`](../../../infx/launch/drivers/srt/checkout.py)) stages them into the job checkout's `configs/` directory for the recipes' bind mounts. Keeping a recipe in this tree does not activate it; the master configs determine the benchmark matrix. @@ -40,7 +40,8 @@ Recipes use `schema: 2`, `engine`, and `roles`. Each worker role owns its node c | `roles.prefill.args.tp-size` (SGLang) | `prefill.tp` | | `roles.prefill.args.ep-size` (SGLang) | `prefill.ep` | | `roles.prefill.args.enable-dp-attention` | `prefill.dp-attn` | -| `benchmark.concurrencies` | `conc-list` (AgentX recipes; fixed-sequence recipes are bound) | +| `benchmark.concurrencies` | `conc-list` (bound for `power: true` rows) | +| `telemetry:` | search-space `power: true` | | Recipe directory, relative to this tree | entry-level `srt-recipe-dir` | | Recipe file, optionally with a `base`, `override_`, or `zip_override_[]` selector | search-space `srt-recipe`; `eval-srt-recipe` for eval-only real verification | @@ -48,15 +49,15 @@ Keep the recipe and master configuration synchronized. The launcher executes the All referenced recipes must be checked in: srt-slurm 2 ships curated examples instead of the historical `recipes/` archive. The initial migration restores 204 previously external recipes and two still-referenced AgentX recipes from InferenceX history. Master-config paths follow the layout above; existing override selectors are preserved. -## Fixed-sequence fragments +## Recipe fragments -Fixed-sequence recipes, single- and multi-node, are fragments: native srt-slurm YAML holding only recipe-specific settings. At launch the fragment is composed and bound: +Active recipes, fixed-sequence and AgentX, single- and multi-node, are fragments: native srt-slurm YAML holding only recipe-specific settings. At launch the fragment is composed and bound: -1. The workload's shared block, [`configs/srt-recipes/fixed-sequence-single.yaml`](../../../configs/srt-recipes/fixed-sequence-single.yaml) or [`fixed-sequence-multi.yaml`](../../../configs/srt-recipes/fixed-sequence-multi.yaml), is merged under the fragment (under `base` for bundles). The fragment wins: mappings merge and lists replace. +1. The lane's shared block in [`configs/srt-recipes/`](../../../configs/srt-recipes) (`fixed-sequence-{single,multi}.yaml` or `agentic-{single,multi}.yaml`) is merged under the fragment (under `base` for bundles). A multi-node row with `power: true` also gets [`telemetry-dcgm.yaml`](../../../configs/srt-recipes/telemetry-dcgm.yaml), its exporter on the cluster's `srt-slurm.power-exporter-port`; the lane must allow power (`POWER_LANES`). The fragment wins: mappings merge and lists replace, so a fragment may keep telemetry tunables such as `collector_join_timeout_seconds`. 2. The row's selector picks the variant. A single-node variant may name its `benchmark.env.CONC`, which keeps that concurrency paired with its tuning. -3. The binder ([`workload.py`](../../../infx/srt_slurm/workload.py)) writes the matrix point into the selected recipe: `model.path: hf:`, `model.container: `, `model.precision`, `benchmark.env.ISL`/`OSL`, `identity.container.image` and `identity.model.repo` when the fragment declares `identity.container`/`identity.model`, and `benchmark.concurrencies` when telemetry is enabled. Single-node recipes also get `MODEL`, `CONC`, `RANDOM_RANGE_RATIO` and `USE_CHAT_TEMPLATE` (`true` exactly when the recipe speculates). The multi-node client reads `CONC_LIST` from the job environment. +3. The binder ([`workload.py`](../../../infx/srt_slurm/workload.py)) writes the matrix point into the selected recipe: `model.path: hf:`, `model.container: `, `model.precision`, `identity.container.image` (as a registry reference) and `identity.model.repo` when the fragment declares `identity.container`/`identity.model`, and `benchmark.concurrencies` when telemetry is enabled. Fixed-sequence recipes also get `benchmark.env.ISL`/`OSL`; single-node recipes get `MODEL` and `CONC`, and fixed-sequence ones `RANDOM_RANGE_RATIO` and `USE_CHAT_TEMPLATE` (`true` exactly when the recipe speculates). Multi-node AgentX recipes get the client's `RESULT_DIR`, `AIPERF_DATASET_MMAP_CACHE_DIR` and `HF_HUB_CACHE` from the launcher, which derives them from the volumes it mounts (`volume-mounts`, `agentic-volume-mounts`, lane mounts); a fragment that sets `HF_HOME` keeps its own cache. The multi-node client reads `CONC_LIST`, `CONC`, `MODEL` and the other matrix inputs from the job environment, so fragments do not copy them; single-node AgentX points get them as runtime arguments. -A fragment that sets any of these bound keys, or `benchmark.env.CONC_LIST`, fails before submission, even when the value matches. `hf:` resolves to the cluster's staged checkpoint (`models.entries`) unless a `models.OVERRIDES` row serves the Hub snapshot, and the master image to its staged container. A multi-node point with neither checkpoint nor Hub row fails before submission. AgentX recipes stay complete and are not composed. +A fragment that sets any of these bound keys, `telemetry.enabled`, `benchmark.env.CONC_LIST`, or an AgentX client path (`RESULT_DIR`, `AGENTIC_OUTPUT_DIR`, `HF_HUB_CACHE`, `HUGGINGFACE_HUB_CACHE`, and on multi-node `AIPERF_DATASET_MMAP_CACHE_DIR`) fails before submission, even when the value matches. `hf:` resolves to the cluster's staged checkpoint (`models.entries`) unless a `models.OVERRIDES` row serves the Hub snapshot, and the master image to its staged container. A multi-node fixed-sequence point with neither checkpoint nor Hub row fails before submission. Inspect what the launcher submits without a cluster: @@ -64,7 +65,7 @@ Inspect what the launcher submits without a cluster: uv run --extra recipes infx generate --config-key 'dsr1-fp8-h200-*' --output-dir /tmp/recipes ``` -It writes one bound recipe per fixed-sequence point, validated by the pinned srtctl, plus a `manifest.json` that maps each file to its matrix point. Launch-time edits such as the job name, health-check floor and runtime `--set` values are not applied. +It writes one bound recipe per fixed-sequence or AgentX point, validated by the pinned srtctl, plus a `manifest.json` that maps each file to its matrix point and to the one cluster its runner label schedules on (`null` when the label spans several). Multi-node points get the binder inputs the launcher derives from that cluster: the DCGM telemetry block on its `power-exporter-port` when the lane allows the row's `power: true`, and the AgentX client paths; a point that needs them fails when its runner label does not name one cluster or the lane refuses its power. Launch-time edits such as the job name, health-check floor and runtime `--set` values are not applied. ## Migration and validation @@ -85,7 +86,7 @@ The initial migration also resolves compatibility issues that `srtctl migrate` c - SGLang Model Gateway recipes use `frontend.type: sglang-router`; in v2.36.0, `sglang` selects a direct worker without a router. - Duplicate YAML keys retain the value selected by the former PyYAML loader. -- DCGM telemetry uses `collect_interval_ms: 1000` instead of `provider` and `default_frequency`. The collector derives its shutdown budget; an explicit ten-second budget is too short for the current validator. Dedicated discovery-service placement is preserved from the original recipes. The pinned upstream runtime rejects telemetry with dedicated infrastructure nodes; this remains a power compatibility blocker rather than changing the original topology to satisfy validation. H200 custom recipes declare a default concurrency that the launcher replaces before submission. +- DCGM telemetry uses `collect_interval_ms: 1000` instead of `provider` and `default_frequency`. The collector derives its shutdown budget; an explicit ten-second budget is too short for the current validator. Dedicated discovery-service placement is preserved from the original recipes. The pinned upstream runtime rejects telemetry with dedicated infrastructure nodes; this remains a power compatibility blocker rather than changing the original topology to satisfy validation. The binder sets `benchmark.concurrencies` from the point for power rows. - DeepSeek-V4 vLLM benchmarks use the supported `custom_tokenizer` loader. Retired `warmup_req_rate: inf` fields are removed; the current upstream client uses its fixed warmup rate of 250 requests per second. - The power reader accepts both generations of samples CSV while validating utilization values and continuing to compute board energy from watts. - Post-eval selection uses native `post_eval.command` and `post_eval.passthrough_env` with [`srt_eval.sh`](../srt_eval.sh). TRT AgentX recipes declare their existing Dynamo fork with `dynamo.source.git`; launchers no longer rewrite the srt-slurm source. diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/atom/mi355x-fp4/agentx/disagg-lmcache-dspark.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/atom/mi355x-fp4/agentx/disagg-lmcache-dspark.yaml index 803838ecf3..9b5def664e 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/atom/mi355x-fp4/agentx/disagg-lmcache-dspark.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/atom/mi355x-fp4/agentx/disagg-lmcache-dspark.yaml @@ -7,10 +7,6 @@ base: schema: 2 name: mi355x-dsv4-pro-0813-atom-agentx-lmcache - model: - path: DeepSeek-V4-Pro-0813 - container: rocm/atom-dev:nightly_202609221542 - precision: fp4 slurm: time_limit: "08:00:00" resources: @@ -87,16 +83,8 @@ base: max_attempts: 720 interval_seconds: 5 benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - MODEL: deepseek-ai/DeepSeek-V4-Pro-0813 - # The MI355X launcher collects results from the job workspace. - RESULT_DIR: /infmax-workspace/LOGS/agentic - AGENTIC_OUTPUT_DIR: /infmax-workspace AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "atom:" - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache/hub TOKENIZERS_PARALLELISM: "false" TRANSFORMERS_VERBOSITY: error @@ -118,9 +106,6 @@ override_tp8_c1: args: max-num-seqs: 2 cudagraph-capture-sizes: '[1,2]' - benchmark: - env: - CONC: '1' override_tp8_c16: frontend: @@ -136,9 +121,6 @@ override_tp8_c16: args: max-num-seqs: 32 cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32]' - benchmark: - env: - CONC: '16' override_dpa8_c64: frontend: @@ -167,9 +149,6 @@ override_dpa8_c64: max-num-seqs: 128 cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16]' enable-dp-attention: true - benchmark: - env: - CONC: '64' override_dpa8_c128: frontend: @@ -198,9 +177,6 @@ override_dpa8_c128: max-num-seqs: 256 cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32]' enable-dp-attention: true - benchmark: - env: - CONC: '128' override_dpa8_c256_lmcache: frontend: @@ -248,6 +224,5 @@ override_dpa8_c256_lmcache: enable-dp-attention: true benchmark: env: - CONC: '256' # dp_sticky keys sessions on the correlation id. AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1' diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/agg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/agg-variants.yaml index efa41b3216..656eba3707 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/agg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/agg-variants.yaml @@ -6,15 +6,9 @@ schema: 2 base: # B200 AgentX aggregate topology: one TP8 worker uses one eight-GPU node # and serves both prefill and decode with bundled DSpark and HiCache. - model: - path: "deepseek-v4-pro-0813" - container: "dynamo-sglang" - precision: "fp4" identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro-0813" - container: - image: "lmsysorg/sglang:nightly-dev-20260916-c9a8fba9" + model: {} + container: {} frameworks: dynamo: "1.5.0.dev20260914" dynamo: @@ -111,19 +105,12 @@ base: mem: "0" container-remap-root: "" benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" IS_MULTINODE: "false" TP: "8" AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" override_c1: name: "agg-b200-tp8-c1-mtp-hicache" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/disagg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/disagg-variants.yaml index 7711a95165..63db76c96a 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/disagg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/disagg-variants.yaml @@ -4,15 +4,9 @@ schema: 2 base: - model: - path: "deepseek-v4-pro-0813" - container: "dynamo-sglang" - precision: "fp4" identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro-0813" - container: - image: "lmsysorg/sglang:nightly-dev-20260916-c9a8fba9" + model: {} + container: {} frameworks: dynamo: "1.5.0.dev20260914" dynamo: @@ -205,18 +199,10 @@ base: mem: "0" container-remap-root: "" benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" # B200 AgentX SGLang disaggregated recipe for DeepSeek-V4-Pro-0813 # (1P x DEP8 / 1D x DEP8, bundled DSpark + HiCache KV offload), tuned for concurrency 64. diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/agg-gb200-tp8-c1-mtp-hicache.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/agg-gb200-tp8-c1-mtp-hicache.yaml index 3a200c1518..74473f5bb3 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/agg-gb200-tp8-c1-mtp-hicache.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/agg-gb200-tp8-c1-mtp-hicache.yaml @@ -5,16 +5,9 @@ name: "agg-gb200-tp8-c1-mtp-hicache" # and serves both prefill and decode with bundled DSpark speculation (K=6) # and HiCache. -model: - path: "deepseek-v4-pro-0813" - container: "dynamo-sglang" - precision: "fp4" - identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro-0813" - container: - image: "lmsysorg/sglang:nightly-dev-20260916-c9a8fba9" + model: {} + container: {} frameworks: dynamo: "1.5.0.dev20260910" @@ -119,12 +112,7 @@ srun_options: container-remap-root: "" benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" IS_MULTINODE: "false" TP: "8" PP_SIZE: "1" @@ -132,5 +120,3 @@ benchmark: AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/agg-gb200-tp8-c4-mtp-hicache.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/agg-gb200-tp8-c4-mtp-hicache.yaml index 18bd91a5d9..6551bffc47 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/agg-gb200-tp8-c4-mtp-hicache.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/agg-gb200-tp8-c4-mtp-hicache.yaml @@ -5,16 +5,9 @@ name: "agg-gb200-tp8-c4-mtp-hicache" # and serves both prefill and decode with bundled DSpark speculation (K=6) # and HiCache. -model: - path: "deepseek-v4-pro-0813" - container: "dynamo-sglang" - precision: "fp4" - identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro-0813" - container: - image: "lmsysorg/sglang:nightly-dev-20260916-c9a8fba9" + model: {} + container: {} frameworks: dynamo: "1.5.0.dev20260910" @@ -119,12 +112,7 @@ srun_options: container-remap-root: "" benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" IS_MULTINODE: "false" TP: "8" PP_SIZE: "1" @@ -132,5 +120,3 @@ benchmark: AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-1p1d-dep16-dep32-c256-mtp-kvoffload.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-1p1d-dep16-dep32-c256-mtp-kvoffload.yaml index 541934f0af..ee9cfab440 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-1p1d-dep16-dep32-c256-mtp-kvoffload.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-1p1d-dep16-dep32-c256-mtp-kvoffload.yaml @@ -1,14 +1,8 @@ schema: 2 name: disagg-gb200-1p1d-dep16-dep32-c256-mtp-kvoffload -model: - path: deepseek-v4-pro-0813 - container: dynamo-sglang - precision: fp4 identity: - model: - repo: deepseek-ai/DeepSeek-V4-Pro-0813 - container: - image: lmsysorg/sglang:nightly-dev-20260916-c9a8fba9 + model: {} + container: {} dynamo: install: true source: @@ -262,19 +256,11 @@ srun_options: mem: '0' container-remap-root: '' benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh # The replay client holds ~300 GB of host memory; on the default head node it # shares prefill_0's node and OOMs it. Put it on the etcd/nats node instead. placement: node: dedicated env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-1p1d-dep8-dep16-c128-mtp-kvoffload.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-1p1d-dep8-dep16-c128-mtp-kvoffload.yaml index a44d935fda..d7320736de 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-1p1d-dep8-dep16-c128-mtp-kvoffload.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-1p1d-dep8-dep16-c128-mtp-kvoffload.yaml @@ -6,16 +6,9 @@ name: "disagg-gb200-1p1d-dep8-dep16-c128-mtp-kvoffload" # concurrency 128. # The DEP8 prefill worker spans two nodes; the DEP16 decode worker spans four. -model: - path: "deepseek-v4-pro-0813" - container: "dynamo-sglang" - precision: "fp4" - identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro-0813" - container: - image: "lmsysorg/sglang:nightly-dev-20260916-c9a8fba9" + model: {} + container: {} frameworks: dynamo: "1.5.0.dev20260910" @@ -278,19 +271,11 @@ srun_options: container-remap-root: "" benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh # The replay client holds ~300 GB of host memory; on the default head node it # shares prefill_0's node and OOMs it. Put it on the etcd/nats node instead. placement: node: dedicated env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-1p1d-dep8-dep16-c64-mtp-kvoffload.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-1p1d-dep8-dep16-c64-mtp-kvoffload.yaml index f0181c16ad..b462d6bebb 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-1p1d-dep8-dep16-c64-mtp-kvoffload.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-1p1d-dep8-dep16-c64-mtp-kvoffload.yaml @@ -6,16 +6,9 @@ name: "disagg-gb200-1p1d-dep8-dep16-c64-mtp-kvoffload" # concurrency 64. # The DEP8 prefill worker spans two nodes; the DEP16 decode worker spans four. -model: - path: "deepseek-v4-pro-0813" - container: "dynamo-sglang" - precision: "fp4" - identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro-0813" - container: - image: "lmsysorg/sglang:nightly-dev-20260916-c9a8fba9" + model: {} + container: {} frameworks: dynamo: "1.5.0.dev20260910" @@ -278,19 +271,11 @@ srun_options: container-remap-root: "" benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh # The replay client holds ~300 GB of host memory; on the default head node it # shares prefill_0's node and OOMs it. Put it on the etcd/nats node instead. placement: node: dedicated env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-2p1d-dep16-dep32-c1024-mtp-kvoffload.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-2p1d-dep16-dep32-c1024-mtp-kvoffload.yaml index f1672f9e54..8da3f30de6 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-2p1d-dep16-dep32-c1024-mtp-kvoffload.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-2p1d-dep16-dep32-c1024-mtp-kvoffload.yaml @@ -1,14 +1,8 @@ schema: 2 name: disagg-gb200-2p1d-dep16-dep32-c1024-mtp-kvoffload -model: - path: deepseek-v4-pro-0813 - container: dynamo-sglang - precision: fp4 identity: - model: - repo: deepseek-ai/DeepSeek-V4-Pro-0813 - container: - image: lmsysorg/sglang:nightly-dev-20260916-c9a8fba9 + model: {} + container: {} dynamo: install: true source: @@ -262,19 +256,11 @@ srun_options: mem: '0' container-remap-root: '' benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh # The replay client holds ~300 GB of host memory; on the default head node it # shares prefill_0's node and OOMs it. Put it on the etcd/nats node instead. placement: node: dedicated env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-2p1d-dep16-dep32-c1280-mtp-kvoffload.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-2p1d-dep16-dep32-c1280-mtp-kvoffload.yaml index ac2ed56b08..1078d341f3 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-2p1d-dep16-dep32-c1280-mtp-kvoffload.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-2p1d-dep16-dep32-c1280-mtp-kvoffload.yaml @@ -1,14 +1,8 @@ schema: 2 name: disagg-gb200-2p1d-dep16-dep32-c1280-mtp-kvoffload -model: - path: deepseek-v4-pro-0813 - container: dynamo-sglang - precision: fp4 identity: - model: - repo: deepseek-ai/DeepSeek-V4-Pro-0813 - container: - image: lmsysorg/sglang:nightly-dev-20260916-c9a8fba9 + model: {} + container: {} dynamo: install: true source: @@ -262,19 +256,11 @@ srun_options: mem: '0' container-remap-root: '' benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh # The replay client holds ~300 GB of host memory; on the default head node it # shares prefill_0's node and OOMs it. Put it on the etcd/nats node instead. placement: node: dedicated env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-2p1d-dep16-dep32-c768-mtp-kvoffload.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-2p1d-dep16-dep32-c768-mtp-kvoffload.yaml index ff4cd0a15f..82140cb681 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-2p1d-dep16-dep32-c768-mtp-kvoffload.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/disagg-gb200-2p1d-dep16-dep32-c768-mtp-kvoffload.yaml @@ -1,14 +1,8 @@ schema: 2 name: disagg-gb200-2p1d-dep16-dep32-c768-mtp-kvoffload -model: - path: deepseek-v4-pro-0813 - container: dynamo-sglang - precision: fp4 identity: - model: - repo: deepseek-ai/DeepSeek-V4-Pro-0813 - container: - image: lmsysorg/sglang:nightly-dev-20260916-c9a8fba9 + model: {} + container: {} dynamo: install: true source: @@ -262,19 +256,11 @@ srun_options: mem: '0' container-remap-root: '' benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh # The replay client holds ~300 GB of host memory; on the default head node it # shares prefill_0's node and OOMs it. Put it on the etcd/nats node instead. placement: node: dedicated env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/agg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/agg-variants.yaml index 8cf0b086da..6418b5c4eb 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/agg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/agg-variants.yaml @@ -4,15 +4,9 @@ schema: 2 base: - model: - path: "deepseek-v4-pro-0813" - container: "dynamo-sglang" - precision: "fp4" identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro-0813" - container: - image: "lmsysorg/sglang:v0.5.19-cu130@sha256:d6e7288627be8b02be88e4bba38e73f6d50e2826869f753c13a4c4385ab3eda9" + model: {} + container: {} dynamo: install: true source: @@ -99,12 +93,7 @@ base: mem: "0" container-remap-root: "" benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" IS_MULTINODE: "false" # The AgentX power path checks the GPU topology (TP from each variant) before replay. PP_SIZE: "1" @@ -112,8 +101,6 @@ base: AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" # Low-latency AgentX aggregate topology: one TP8 worker spans two # four-GPU GB300 nodes and serves both prefill and decode with DSpark K=6. diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-variants.yaml index c7512f7adc..a3c557c962 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-variants.yaml @@ -4,15 +4,9 @@ schema: 2 base: - model: - path: "deepseek-v4-pro-0813" - container: "dynamo-sglang" - precision: "fp4" identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro-0813" - container: - image: "lmsysorg/sglang:nightly-dev-cu13-20260916-c9a8fba9" + model: {} + container: {} dynamo: install: true source: @@ -269,22 +263,14 @@ base: mem: "0" container-remap-root: "" benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh # The replay client holds ~300 GB of host memory; on the default head node it # shares prefill_0's node and OOMs it. Put it on the etcd/nats node instead. placement: node: dedicated env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" # Agentic-coding SGLang disaggregated Pareto recipe for DeepSeek-V4-Pro on GB300 # (2P x DEP8 / 4D x DEP16, DSpark K=6 + Mooncake external-linker KV offload), tuned for concurrency 480 (also run at 120). diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/h200-fp8/agentx/agg-tp8-mtp-kvoffload.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/h200-fp8/agentx/agg-tp8-mtp-kvoffload.yaml index 0f98de347a..ee1df4bf6e 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/h200-fp8/agentx/agg-tp8-mtp-kvoffload.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/h200-fp8/agentx/agg-tp8-mtp-kvoffload.yaml @@ -5,16 +5,9 @@ name: "agg-h200-tp8-mtp-kvoffload" # DSpark K=6. The recipe is topology-invariant; configs/nvidia-master.yaml # varies only concurrency to produce the latency/throughput Pareto curve. -model: - path: "deepseek-v4-pro-0813" - container: "dynamo-sglang" - precision: "fp8" - identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro-0813" - container: - image: "lmsysorg/sglang:nightly-dev-20260914-4358a161" + model: {} + container: {} dynamo: install: true @@ -113,28 +106,11 @@ srun_options: container-remap-root: "" telemetry: - enabled: true - collect_interval_ms: 1000 - storage_subdir: power required: false - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 - command: "dcgm-exporter --collect-interval=100 --address :{port} -f /configs/dcgm-counters-noprof.csv" benchmark: - type: custom - # Replaced with CONC_LIST by the launcher before submission. - concurrencies: [1] - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" # Aggregated serving uses one TP8 worker for both prefill and decode. - IS_MULTINODE: "true" TP: "8" AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" @@ -142,5 +118,3 @@ benchmark: # the AIPerf export, instead of publishing a trace artifact with no backend # series behind it. AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/mi355x-fp4/agentx/disagg-umbp-dspark.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/mi355x-fp4/agentx/disagg-umbp-dspark.yaml index eb46cf57a9..b3b5ccaf67 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/mi355x-fp4/agentx/disagg-umbp-dspark.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/mi355x-fp4/agentx/disagg-umbp-dspark.yaml @@ -5,10 +5,6 @@ schema: 2 base: name: mi355x-dsv4-pro-0813-agentx-umbp - model: - path: DeepSeek-V4-Pro-0813 - container: lmsysorg/sglang-rocm:v0.5.21-rocm724-mi35x-20261001 - precision: fp4 slurm: time_limit: "08:00:00" resources: @@ -209,19 +205,11 @@ base: max_attempts: 720 interval_seconds: 5 benchmark: - type: custom container_image: lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260907 - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - # The MI355X launcher collects results from the job workspace. - RESULT_DIR: /infmax-workspace/LOGS/agentic - AGENTIC_OUTPUT_DIR: /infmax-workspace AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache/hub TOKENIZERS_PARALLELISM: "false" TRANSFORMERS_VERBOSITY: error - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 # One variant per point; each point runs in its own allocation. Admission is # 2x CONC on both roles. Decode captures graphs up to 2x CONC, or CONC/4 per @@ -262,9 +250,6 @@ override_c4: - 8 speculative-dspark-block-size: 6 speculative-num-draft-tokens: 7 - benchmark: - env: - CONC: '4' override_c16: roles: @@ -320,9 +305,6 @@ override_c16: - 32 speculative-dspark-block-size: 6 speculative-num-draft-tokens: 7 - benchmark: - env: - CONC: '16' override_c32: roles: @@ -410,9 +392,6 @@ override_c32: - 64 speculative-dspark-block-size: 6 speculative-num-draft-tokens: 7 - benchmark: - env: - CONC: '32' override_c48: roles: @@ -532,9 +511,6 @@ override_c48: - 96 speculative-dspark-block-size: 3 speculative-num-draft-tokens: 4 - benchmark: - env: - CONC: '48' override_c192: roles: @@ -672,7 +648,6 @@ override_c192: speculative-num-draft-tokens: 4 benchmark: env: - CONC: '192' AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID: 'true' override_c256: @@ -827,7 +802,6 @@ override_c256: speculative-num-draft-tokens: 4 benchmark: env: - CONC: '256' AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID: 'true' # MegaMoE prefill on the DP-attention arm, with the fp8 unified KV pool, at @@ -1029,7 +1003,6 @@ override_c384: speculative-num-draft-tokens: 4 benchmark: env: - CONC: '384' AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID: 'true' override_c512: @@ -1256,5 +1229,4 @@ override_c512: speculative-num-draft-tokens: 4 benchmark: env: - CONC: '512' AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID: 'true' diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-variants.yaml index 353df44501..a4f0285ae6 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-variants.yaml @@ -4,15 +4,9 @@ schema: 2 base: - model: - path: deepseek-ai/DeepSeek-V4-Pro - container: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc24 - precision: fp4 identity: - model: - repo: deepseek-ai/DeepSeek-V4-Pro - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + model: {} + container: {} frameworks: tensorrt_llm: 1.3.0rc24 dynamo: @@ -160,26 +154,11 @@ base: placement: node: first_decode benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - MODEL: deepseek-ai/DeepSeek-V4-Pro - MODEL_PREFIX: dsv4 - FRAMEWORK: dynamo-trt - PRECISION: fp4 - DURATION: '3600' - KV_OFFLOADING: dram - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 SERVED_MODEL_NAME: DeepSeek-V4-Pro placement: node: last_decode @@ -219,9 +198,6 @@ override_1p4d_dep4_tep8_c4_b1: frontend: env: CONC: '4' - benchmark: - env: - CONC: '4' override_1p6d_dep4_tep4_c24_b4: name: dynamo-disagg-gb300-1p6d-dep4-tep4-c24-b4-mtp @@ -258,9 +234,6 @@ override_1p6d_dep4_tep4_c24_b4: frontend: env: CONC: '24' - benchmark: - env: - CONC: '24' override_1p1d_dep8_dep32_c388_b4: name: dynamo-disagg-gb300-1p1d-dep8-dep32-c388-b4-mtp @@ -284,9 +257,6 @@ override_1p1d_dep8_dep32_c388_b4: frontend: env: CONC: '388' - benchmark: - env: - CONC: '388' override_2p1d_dep8_dep32_c736_b8: name: dynamo-disagg-gb300-2p1d-dep8-dep32-c736-b8-mtp @@ -311,9 +281,6 @@ override_2p1d_dep8_dep32_c736_b8: frontend: env: CONC: '736' - benchmark: - env: - CONC: '736' override_3p1d_dep8_dep16_c1152_b32: name: dynamo-disagg-gb300-3p1d-dep8-dep16-c1152-b32-mtp @@ -344,9 +311,6 @@ override_3p1d_dep8_dep16_c1152_b32: frontend: env: CONC: '1152' - benchmark: - env: - CONC: '1152' override_5p1d_dep8_dep16_c2626_b96: name: dynamo-disagg-gb300-5p1d-dep8-dep16-c2626-b96-mtp @@ -393,6 +357,3 @@ override_5p1d_dep8_dep16_c2626_b96: frontend: env: CONC: '2626' - benchmark: - env: - CONC: '2626' diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-mtp-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-mtp-variants.yaml index c2a6430112..8c9d21dab9 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-mtp-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-mtp-variants.yaml @@ -4,15 +4,9 @@ schema: 2 base: - model: - path: deepseek-v4-pro - container: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106 - precision: fp4 identity: - model: - repo: deepseek-ai/DeepSeek-V4-Pro - container: - image: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106 + model: {} + container: {} frameworks: dynamo: "1.3.1" dynamo: @@ -94,18 +88,9 @@ base: srun_options: container-remap-root: "" benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" override_tp8: diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-mtp3-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-mtp3-variants.yaml index 81410efe2d..55f3cb2053 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-mtp3-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-mtp3-variants.yaml @@ -4,15 +4,9 @@ schema: 2 base: - model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - precision: "fp4" identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" + model: {} + container: {} frameworks: dynamo: "1.2.1" dynamo: @@ -128,19 +122,8 @@ base: srun_options: container-remap-root: "" benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - # Keep aggregate workers in the multinode result schema so ingestion uses - # the zero decode-worker count instead of duplicating TP into P and D. - IS_MULTINODE: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" # GB200 AgentX aggregate topology: one TP8 worker spans two four-GPU # nodes and serves both prefill and decode at concurrency 4. Keep at least diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-mtp-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-mtp-variants.yaml index 5ff4e6f3e8..383bfefd6d 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-mtp-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-mtp-variants.yaml @@ -4,13 +4,9 @@ schema: 2 base: - model: - path: deepseek-v4-pro - container: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106 - precision: fp4 identity: - model: {repo: deepseek-ai/DeepSeek-V4-Pro} - container: {image: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106} + model: {} + container: {} frameworks: {dynamo: "1.3.1"} dynamo: install: true @@ -141,18 +137,9 @@ base: sbatch_directives: {cpus-per-task: "144", mem: "0"} srun_options: {container-remap-root: ""} benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" # Long AgentX prefills can exceed Dynamo's request-plane default while diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-mtp3-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-mtp3-variants.yaml index c6308fadc7..2200ee586c 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-mtp3-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-mtp3-variants.yaml @@ -4,15 +4,9 @@ schema: 2 base: - model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - precision: "fp4" identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" + model: {} + container: {} frameworks: dynamo: "1.3.0.dev20260720" dynamo: @@ -180,20 +174,11 @@ base: srun_options: container-remap-root: "" benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" # Avoid concurrent readers observing a mismatched mmap data/index pair. AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" # GB200 AgentX MTP3 topology: one DEP8 prefill worker feeds one # DEP8 decode worker at concurrency 128. diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/agg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/agg-variants.yaml index b3fa74c8c5..210e4a39e1 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/agg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/agg-variants.yaml @@ -4,15 +4,9 @@ schema: 2 base: - model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - precision: "fp4" identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" + model: {} + container: {} frameworks: dynamo: "1.2.1" dynamo: @@ -124,19 +118,8 @@ base: srun_options: container-remap-root: "" benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - # Keep aggregate workers in the multinode result schema so ingestion uses - # the zero decode-worker count instead of duplicating TP into P and D. - IS_MULTINODE: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" # GB300 AgentX aggregate topology: one TP4 worker occupies one four-GPU node # and serves both prefill and decode at concurrency 8. Size max-num-seqs at diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p4d-dep4-tp8-c4-mtp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p4d-dep4-tp8-c4-mtp.yaml index 02c44990af..98e3bcb3f8 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p4d-dep4-tp8-c4-mtp.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p4d-dep4-tp8-c4-mtp.yaml @@ -6,16 +6,10 @@ name: "svf-vllm-disagg-gb300-1p4d-dep4-tp8-c4-mtp-agentic" # GB300 high-interactivity AgentX MTP3 topology: one DEP4 prefill worker # feeds four TP8 decode workers at concurrency 4 through NIXL. -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - precision: "fp4" identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" + model: {} + container: {} frameworks: dynamo: "1.4.0" @@ -188,35 +182,12 @@ srun_options: container-remap-root: "" benchmark: - # The AgentX client uses localhost:8000, so colocate it with the Dynamo - # frontend launched on the first decode node. - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - MODEL: "deepseek-ai/DeepSeek-V4-Pro" - MODEL_PREFIX: "dsv4" SERVED_MODEL_NAME: "deepseek-ai/DeepSeek-V4-Pro" - FRAMEWORK: "dynamo-vllm" - PRECISION: "fp4" - CONC: "4" - DURATION: "3600" RUNNER_TYPE: "gb300" - IMAGE: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - SPEC_DECODING: "mtp" - DISAGG: "true" OFFLOADING: "none" - KV_OFFLOADING: "none" TP: "8" PREFILL_TP: "1" - PREFILL_NUM_WORKERS: "1" - PREFILL_EP: "4" - DECODE_TP: "8" - DECODE_NUM_WORKERS: "4" - DECODE_EP: "1" EP_SIZE: "1" AIPERF_MAX_OSL: "none" AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" @@ -224,11 +195,9 @@ benchmark: AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" # Avoid concurrent readers observing a mismatched mmap data/index pair. AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" NUM_DATASET_ENTRIES: "393" HF_WEKA_DATASET: "semianalysisai/cc-traces-weka-062126" PUBLIC_DATASET: "semianalysis_cc_traces_weka_062126" + # Colocate the AgentX client with the Dynamo frontend on the first decode node. placement: node: first_decode diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-mtp-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-mtp-variants.yaml index a333f36d0b..8fa84d383e 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-mtp-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-mtp-variants.yaml @@ -4,15 +4,9 @@ schema: 2 base: - model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - precision: "fp4" identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" + model: {} + container: {} dynamo: install: true environment: @@ -178,20 +172,11 @@ base: srun_options: container-remap-root: "" benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" # Avoid concurrent readers observing a mismatched mmap data/index pair. AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" # Derived from PR #2302's DEP8/DEP32 c256 recipe. # Halve the prefill/decode topology while preserving its conservative diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-variants.yaml index 2978805ee7..00281d1424 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-variants.yaml @@ -4,10 +4,6 @@ schema: 2 base: - model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:v0.21.0-ubuntu2404" - precision: "fp4" dynamo: install: true source: @@ -126,23 +122,6 @@ base: # remap us to uid 0 inside the container. srt-slurm renders empty-string # values as flag-only srun args (see core/slurm.py:250). container-remap-root: "" - benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" - # Container-side path of the aiperf mmap dataset cache; the host-side - # mount is wired via launch_gb300-*.sh's srtslurm.yaml default_mounts. - # Without this, aiperf re-tokenizes + re-writes ~65 GB of mmap files - # per dataset on every run. - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - # Persistent HF hub cache (also wired via default_mounts) so the trace - # dataset isn't re-downloaded on every run. Overrides the workflow-level - # HF_HUB_CACHE=/mnt/hf_hub_cache, which doesn't exist on these nodes. - HF_HUB_CACHE: "/hf_hub_cache" # Agentic-coding variant of dsv4/vllm/gb300-fp4/8k1k/disagg-1p6d-dep4-tp4-stp.yaml. # Topology is identical (1 prefill DEP=4 + 6 decode TP=4, 28 GPUs across 7 diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/agg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/agg-variants.yaml index 4534849c7f..40b043930a 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/agg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/agg-variants.yaml @@ -22,13 +22,8 @@ schema: 2 base: name: dsv41flash-fp4-gb300-dynamo-sglang-agentx-agg - model: - path: hf:deepseek-ai/DeepSeek-V4.1-Flash - container: lmsysorg/sglang:nightly-dev-20260928-81f27fb3@sha256:d9e4917808cfaa4b3be033a0c85a3a73d71eb93c17accbfa2ac3e96f551f3a33 - precision: fp4 identity: - model: - repo: deepseek-ai/DeepSeek-V4.1-Flash + model: {} dynamo: install: true source: @@ -111,19 +106,11 @@ base: srun_options: container-remap-root: "" benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" IS_MULTINODE: "false" - MODEL: deepseek-ai/DeepSeek-V4.1-Flash AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache # Warmup can leave bytes unacknowledged past AIPerf's 30 s default. AIPERF_HTTP_TCP_USER_TIMEOUT: "900000" TP: "2" @@ -148,7 +135,6 @@ override_tp4_c1: SGLANG_RAGGED_VERIFY_MODE: static benchmark: env: - CONC: "1" TP: "4" AGENTIC_WARMUP_GRACE_PERIOD: "1800" @@ -167,5 +153,4 @@ override_tp2_c1: SGLANG_RAGGED_VERIFY_MODE: static benchmark: env: - CONC: "1" AGENTIC_WARMUP_GRACE_PERIOD: "1800" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/disagg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/disagg-variants.yaml index 98a9a5a03f..9970ca10bc 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/disagg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/disagg-variants.yaml @@ -28,13 +28,8 @@ schema: 2 base: name: dsv41flash-fp4-gb300-dynamo-sglang-agentx-disagg - model: - path: hf:deepseek-ai/DeepSeek-V4.1-Flash - container: lmsysorg/sglang:nightly-dev-20260928-81f27fb3@sha256:d9e4917808cfaa4b3be033a0c85a3a73d71eb93c17accbfa2ac3e96f551f3a33 - precision: fp4 identity: - model: - repo: deepseek-ai/DeepSeek-V4.1-Flash + model: {} dynamo: install: true source: @@ -164,19 +159,10 @@ base: srun_options: container-remap-root: "" benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" - MODEL: deepseek-ai/DeepSeek-V4.1-Flash AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache AIPERF_HTTP_TCP_USER_TIMEOUT: "900000" AGENTIC_WARMUP_GRACE_PERIOD: "3600" @@ -187,9 +173,6 @@ base: # 4.09 s P90 TTFT. override_1p1d_c96: name: disagg-gb300-1p1d-tp2ep2-c96 - benchmark: - env: - CONC: "96" # 1 prefill TP2/EP2 + 2 decode TP2/EP2 (6 GPUs). Two decode workers halve the sessions # per decode worker at a given concurrency, which sets the interactivity band. The @@ -212,9 +195,6 @@ override_1p2d_c48: args: max-running-requests: 64 cuda-graph-max-bs-decode: 64 - benchmark: - env: - CONC: "48" override_1p2d_c64: name: disagg-gb300-1p2d-tp2ep2-c64 @@ -227,9 +207,6 @@ override_1p2d_c64: args: max-running-requests: 64 cuda-graph-max-bs-decode: 64 - benchmark: - env: - CONC: "64" # Spread layout (one decode worker per node, 3 nodes total), 8 sessions per decode # worker: 19,305 tokens/s/GPU @ 331.6 tok/s/user, P90 TTFT 0.69 s (GSM8K 0.9704) - @@ -247,9 +224,6 @@ override_1p2d_spread_c16: args: max-running-requests: 64 cuda-graph-max-bs-decode: 64 - benchmark: - env: - CONC: "16" # Throughput end. The prefill worker keeps its KV prefix cache in a host-memory tier # (SGLang HiCache: ratio 1 = one host copy of the device pool, write-back, direct I/O), @@ -271,9 +245,6 @@ override_1p1d_hicache_c160: args: max-running-requests: 192 cuda-graph-max-bs-decode: 192 - benchmark: - env: - CONC: "160" # Two HiCache prefill workers (one per node) feeding one decode worker (6 GPUs, 3 nodes), # 128 sessions per prefill worker, decode admission and graph batch 256, session-affinity @@ -299,6 +270,3 @@ override_2p1d_hicache_c256: args: max-running-requests: 256 cuda-graph-max-bs-decode: 256 - benchmark: - env: - CONC: "256" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/agg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/agg-variants.yaml index 2d86f69c78..92c729778e 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/agg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/agg-variants.yaml @@ -4,16 +4,10 @@ schema: 2 base: - model: - path: glm-5.2-fp4 - container: dynamo-sglang - precision: fp4 identity: model: - repo: nvidia/GLM-5.2-NVFP4 revision: aec724e8c7b8ee9db3b48c01c320f63f9cdaf8aa - container: - image: lmsysorg/sglang:nightly-dev-20260910-00840301 + container: {} frameworks: dynamo: 1.5.0.dev20260909 sglang: 0.0.0.dev1+g008403017 @@ -78,17 +72,9 @@ base: enable-metrics: true enable-cache-report: true benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache health_check: max_attempts: 1440 interval_seconds: 10 diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/disagg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/disagg-variants.yaml index bc5b0dd7a5..4a6dafca9c 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/disagg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/disagg-variants.yaml @@ -4,16 +4,10 @@ schema: 2 base: - model: - path: glm-5.2-fp4 - container: dynamo-sglang - precision: fp4 identity: model: - repo: nvidia/GLM-5.2-NVFP4 revision: aec724e8c7b8ee9db3b48c01c320f63f9cdaf8aa - container: - image: lmsysorg/sglang:nightly-dev-20260910-00840301 + container: {} frameworks: dynamo: 1.5.0.dev20260909 sglang: 0.0.0.dev1+g008403017 @@ -148,19 +142,11 @@ base: max_attempts: 1440 interval_seconds: 10 benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache services: - name: etcd type: etcd diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/agg-mtp-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/agg-mtp-variants.yaml index 59357ed961..415e3dcdd2 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/agg-mtp-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/agg-mtp-variants.yaml @@ -4,16 +4,10 @@ schema: 2 base: - model: - path: glm-5.2-fp4 - container: dynamo-sglang - precision: fp4 identity: model: - repo: nvidia/GLM-5.2-NVFP4 revision: aec724e8c7b8ee9db3b48c01c320f63f9cdaf8aa - container: - image: lmsysorg/sglang:nightly-dev-cu13-20260805-211ee642 + container: {} frameworks: dynamo: 71eb001e17fa73c742f0afe1a6ed96836cb135fd sglang: nightly-dev-cu13-20260805-211ee642 @@ -100,19 +94,11 @@ base: max_attempts: 1440 interval_seconds: 10 benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache services: - name: etcd type: etcd diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/agg.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/agg.yaml index eae2807daf..748baac49a 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/agg.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/agg.yaml @@ -1,15 +1,9 @@ schema: 2 name: gb200-fp4-glm5.2-agentx-agg -model: - path: glm-5.2-fp4 - container: dynamo-sglang - precision: fp4 identity: model: - repo: nvidia/GLM-5.2-NVFP4 revision: aec724e8c7b8ee9db3b48c01c320f63f9cdaf8aa - container: - image: lmsysorg/sglang:v0.5.17-cu130 + container: {} frameworks: dynamo: 71eb001e17fa73c742f0afe1a6ed96836cb135fd sglang: 0.5.17 @@ -102,32 +96,13 @@ health_check: max_attempts: 1440 interval_seconds: 10 telemetry: - enabled: true - collect_interval_ms: 1000 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 collector_join_timeout_seconds: 12 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 - command: "dcgm-exporter --collect-interval=100 --address :{port} -f /configs/dcgm-counters-noprof.csv" benchmark: - type: custom - concurrencies: [1] - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache services: - name: etcd type: etcd diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-dep8-mtp-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-dep8-mtp-variants.yaml index 2525c8b0b9..1e4a55c8cb 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-dep8-mtp-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-dep8-mtp-variants.yaml @@ -4,16 +4,10 @@ schema: 2 base: - model: - path: glm-5.2-fp4 - container: lmsysorg/sglang:v0.5.18-cu130@sha256:db37df8aded8fa169f7264404f3c893672f8046609a4a43d15bedcb4f63d692a - precision: fp4 identity: model: - repo: nvidia/GLM-5.2-NVFP4 revision: aec724e8c7b8ee9db3b48c01c320f63f9cdaf8aa - container: - image: lmsysorg/sglang:v0.5.18-cu130@sha256:db37df8aded8fa169f7264404f3c893672f8046609a4a43d15bedcb4f63d692a + container: {} frameworks: dynamo: 71eb001e17fa73c742f0afe1a6ed96836cb135fd sglang: 0.5.18 @@ -170,19 +164,11 @@ base: max_attempts: 1440 interval_seconds: 10 benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache services: - name: etcd type: etcd diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-mtp-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-mtp-variants.yaml index 5a479c2665..29bc4a89d8 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-mtp-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-mtp-variants.yaml @@ -1,16 +1,10 @@ schema: 2 base: name: gb200-fp4-glm5.2-agentx - model: - path: glm-5.2-fp4 - container: lmsysorg/sglang:v0.5.18-cu130@sha256:db37df8aded8fa169f7264404f3c893672f8046609a4a43d15bedcb4f63d692a - precision: fp4 identity: model: - repo: nvidia/GLM-5.2-NVFP4 revision: aec724e8c7b8ee9db3b48c01c320f63f9cdaf8aa - container: - image: lmsysorg/sglang:v0.5.18-cu130@sha256:db37df8aded8fa169f7264404f3c893672f8046609a4a43d15bedcb4f63d692a + container: {} frameworks: dynamo: 71eb001e17fa73c742f0afe1a6ed96836cb135fd sglang: 0.5.18 @@ -181,19 +175,11 @@ base: max_attempts: 1440 interval_seconds: 10 benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache services: - name: etcd type: etcd diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb300-fp4/agentx/agg-tp8-c1-mtp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb300-fp4/agentx/agg-tp8-c1-mtp.yaml index f9497d8cb1..4bbaa1d92c 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb300-fp4/agentx/agg-tp8-c1-mtp.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb300-fp4/agentx/agg-tp8-c1-mtp.yaml @@ -5,13 +5,8 @@ # one job per concurrency from the master-config conc-list into srt_agentic.sh. schema: 2 name: agg-tp8-c1-mtp -model: - path: glm-5.2-fp4 - container: dynamo-sglang - precision: fp4 identity: - container: - image: lmsysorg/sglang:nightly-dev-20260914-4358a161 + container: {} resources: gpu_type: gb300 gpus_per_node: 4 @@ -82,16 +77,8 @@ health_check: max_attempts: 1440 interval_seconds: 10 benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache services: - name: etcd type: etcd diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb300-fp4/agentx/agg-tp8-c2-mtp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb300-fp4/agentx/agg-tp8-c2-mtp.yaml index 67541e5308..23d265fb65 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb300-fp4/agentx/agg-tp8-c2-mtp.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb300-fp4/agentx/agg-tp8-c2-mtp.yaml @@ -1,12 +1,7 @@ schema: 2 name: agg-tp8-c2-mtp -model: - path: glm-5.2-fp4 - container: dynamo-sglang - precision: fp4 identity: - container: - image: lmsysorg/sglang:nightly-dev-20260914-4358a161 + container: {} resources: gpu_type: gb300 gpus_per_node: 4 @@ -77,16 +72,8 @@ health_check: max_attempts: 1440 interval_seconds: 10 benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache services: - name: etcd type: etcd diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb300-fp4/agentx/agg-tp8-c4-mtp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb300-fp4/agentx/agg-tp8-c4-mtp.yaml index d031960d39..f08da71e25 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb300-fp4/agentx/agg-tp8-c4-mtp.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb300-fp4/agentx/agg-tp8-c4-mtp.yaml @@ -1,12 +1,7 @@ schema: 2 name: agg-tp8-c4-mtp -model: - path: glm-5.2-fp4 - container: dynamo-sglang - precision: fp4 identity: - container: - image: lmsysorg/sglang:nightly-dev-20260914-4358a161 + container: {} resources: gpu_type: gb300 gpus_per_node: 4 @@ -77,16 +72,8 @@ health_check: max_attempts: 1440 interval_seconds: 10 benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache services: - name: etcd type: etcd diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb300-fp4/agentx/disagg-1p2d-p-dep4-d-tp4-c48-mtp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb300-fp4/agentx/disagg-1p2d-p-dep4-d-tp4-c48-mtp.yaml index de869049ff..7ffea81cf0 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb300-fp4/agentx/disagg-1p2d-p-dep4-d-tp4-c48-mtp.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb300-fp4/agentx/disagg-1p2d-p-dep4-d-tp4-c48-mtp.yaml @@ -1,12 +1,7 @@ schema: 2 name: disagg-1p2d-p-dep4-d-tp4-c48-mtp -model: - path: glm-5.2-fp4 - container: dynamo-sglang - precision: fp4 identity: - container: - image: lmsysorg/sglang:nightly-dev-20260914-4358a161 + container: {} resources: gpu_type: gb300 gpus_per_node: 4 @@ -143,16 +138,8 @@ health_check: max_attempts: 1440 interval_seconds: 10 benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache services: - name: etcd type: etcd diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb300-fp4/agentx/disagg-1p4d-p-dep4-d-tp4-c45-mtp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb300-fp4/agentx/disagg-1p4d-p-dep4-d-tp4-c45-mtp.yaml index d0226c0783..36d30844d2 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb300-fp4/agentx/disagg-1p4d-p-dep4-d-tp4-c45-mtp.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb300-fp4/agentx/disagg-1p4d-p-dep4-d-tp4-c45-mtp.yaml @@ -1,12 +1,7 @@ schema: 2 name: disagg-1p4d-p-dep4-d-tp4-c45-mtp -model: - path: glm-5.2-fp4 - container: dynamo-sglang - precision: fp4 identity: - container: - image: lmsysorg/sglang:nightly-dev-20260914-4358a161 + container: {} resources: gpu_type: gb300 gpus_per_node: 4 @@ -143,16 +138,8 @@ health_check: max_attempts: 1440 interval_seconds: 10 benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache services: - name: etcd type: etcd diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb300-fp4/agentx/disagg-1p6d-p-dep4-d-tp4-c45-mtp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb300-fp4/agentx/disagg-1p6d-p-dep4-d-tp4-c45-mtp.yaml index 6000cb23e0..4a489ce6e6 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb300-fp4/agentx/disagg-1p6d-p-dep4-d-tp4-c45-mtp.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb300-fp4/agentx/disagg-1p6d-p-dep4-d-tp4-c45-mtp.yaml @@ -1,12 +1,7 @@ schema: 2 name: disagg-1p6d-p-dep4-d-tp4-c45-mtp -model: - path: glm-5.2-fp4 - container: dynamo-sglang - precision: fp4 identity: - container: - image: lmsysorg/sglang:nightly-dev-20260914-4358a161 + container: {} resources: gpu_type: gb300 gpus_per_node: 4 @@ -143,16 +138,8 @@ health_check: max_attempts: 1440 interval_seconds: 10 benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache services: - name: etcd type: etcd diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb300-fp4/agentx/disagg-2p1d-p-dep8-d-dep16-c192-mtp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb300-fp4/agentx/disagg-2p1d-p-dep8-d-dep16-c192-mtp.yaml index 356093e0ef..80d76ba7cb 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb300-fp4/agentx/disagg-2p1d-p-dep8-d-dep16-c192-mtp.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb300-fp4/agentx/disagg-2p1d-p-dep8-d-dep16-c192-mtp.yaml @@ -1,12 +1,7 @@ schema: 2 name: disagg-2p1d-p-dep8-d-dep16-c192-mtp -model: - path: glm-5.2-fp4 - container: dynamo-sglang - precision: fp4 identity: - container: - image: lmsysorg/sglang:nightly-dev-20260914-4358a161 + container: {} resources: gpu_type: gb300 gpus_per_node: 4 @@ -156,16 +151,8 @@ health_check: max_attempts: 1440 interval_seconds: 10 benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache services: - name: etcd type: etcd diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/agg-h200-tp8-mtp-c12.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/agg-h200-tp8-mtp-c12.yaml index 36e09ed79f..748ea0a3ea 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/agg-h200-tp8-mtp-c12.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/agg-h200-tp8-mtp-c12.yaml @@ -5,17 +5,10 @@ name: agg-h200-tp8-mtp-c12 # committed GLM-5.2 golden thinking-on acceptance length for three speculative # steps, and max-running-requests is twice concurrency 12. -model: - path: hf:zai-org/GLM-5.2-FP8 - container: lmsysorg/sglang:nightly-dev-20260914-4358a161 - precision: fp8 - identity: model: - repo: zai-org/GLM-5.2-FP8 revision: f33c6dc501ee5a2c7e35155653b1b1abbc320951 - container: - image: lmsysorg/sglang:nightly-dev-20260914-4358a161 + container: {} frameworks: dynamo: 1.5.0.dev20260913 sglang: 4358a1617cad734fc37bf53d9e5c2092bd43d075 @@ -109,33 +102,14 @@ health_check: interval_seconds: 10 telemetry: - enabled: true - collect_interval_ms: 1000 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 collector_join_timeout_seconds: 12 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 - command: "dcgm-exporter --collect-interval=100 --address :{port} -f /configs/dcgm-counters-noprof.csv" benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" AIPERF_HTTP_TCP_USER_TIMEOUT: "900000" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 sbatch_directives: mem: "0" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/agg-h200-tp8-mtp-c16.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/agg-h200-tp8-mtp-c16.yaml index efdcbb08a9..4d87ac8387 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/agg-h200-tp8-mtp-c16.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/agg-h200-tp8-mtp-c16.yaml @@ -5,17 +5,10 @@ name: agg-h200-tp8-mtp-c16 # committed GLM-5.2 golden thinking-on acceptance length for three speculative # steps, and max-running-requests is twice concurrency 16. -model: - path: hf:zai-org/GLM-5.2-FP8 - container: lmsysorg/sglang:nightly-dev-20260914-4358a161 - precision: fp8 - identity: model: - repo: zai-org/GLM-5.2-FP8 revision: f33c6dc501ee5a2c7e35155653b1b1abbc320951 - container: - image: lmsysorg/sglang:nightly-dev-20260914-4358a161 + container: {} frameworks: dynamo: 1.5.0.dev20260913 sglang: 4358a1617cad734fc37bf53d9e5c2092bd43d075 @@ -109,33 +102,14 @@ health_check: interval_seconds: 10 telemetry: - enabled: true - collect_interval_ms: 1000 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 collector_join_timeout_seconds: 12 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 - command: "dcgm-exporter --collect-interval=100 --address :{port} -f /configs/dcgm-counters-noprof.csv" benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" AIPERF_HTTP_TCP_USER_TIMEOUT: "900000" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 sbatch_directives: mem: "0" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/agg-h200-tp8-mtp-c2.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/agg-h200-tp8-mtp-c2.yaml index 3da791fd12..22ef82af17 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/agg-h200-tp8-mtp-c2.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/agg-h200-tp8-mtp-c2.yaml @@ -5,17 +5,10 @@ name: agg-h200-tp8-mtp-c2 # committed GLM-5.2 golden thinking-on acceptance length for three speculative # steps, and max-running-requests is twice concurrency 2. -model: - path: hf:zai-org/GLM-5.2-FP8 - container: lmsysorg/sglang:nightly-dev-20260914-4358a161 - precision: fp8 - identity: model: - repo: zai-org/GLM-5.2-FP8 revision: f33c6dc501ee5a2c7e35155653b1b1abbc320951 - container: - image: lmsysorg/sglang:nightly-dev-20260914-4358a161 + container: {} frameworks: dynamo: 1.5.0.dev20260913 sglang: 4358a1617cad734fc37bf53d9e5c2092bd43d075 @@ -109,33 +102,14 @@ health_check: interval_seconds: 10 telemetry: - enabled: true - collect_interval_ms: 1000 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 collector_join_timeout_seconds: 12 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 - command: "dcgm-exporter --collect-interval=100 --address :{port} -f /configs/dcgm-counters-noprof.csv" benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" AIPERF_HTTP_TCP_USER_TIMEOUT: "900000" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 sbatch_directives: mem: "0" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/agg-h200-tp8-mtp-c4.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/agg-h200-tp8-mtp-c4.yaml index 86ee3ac54f..3dea52a419 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/agg-h200-tp8-mtp-c4.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/agg-h200-tp8-mtp-c4.yaml @@ -5,17 +5,10 @@ name: agg-h200-tp8-mtp-c4 # committed GLM-5.2 golden thinking-on acceptance length for three speculative # steps, and max-running-requests is twice concurrency 4. -model: - path: hf:zai-org/GLM-5.2-FP8 - container: lmsysorg/sglang:nightly-dev-20260914-4358a161 - precision: fp8 - identity: model: - repo: zai-org/GLM-5.2-FP8 revision: f33c6dc501ee5a2c7e35155653b1b1abbc320951 - container: - image: lmsysorg/sglang:nightly-dev-20260914-4358a161 + container: {} frameworks: dynamo: 1.5.0.dev20260913 sglang: 4358a1617cad734fc37bf53d9e5c2092bd43d075 @@ -109,33 +102,14 @@ health_check: interval_seconds: 10 telemetry: - enabled: true - collect_interval_ms: 1000 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 collector_join_timeout_seconds: 12 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 - command: "dcgm-exporter --collect-interval=100 --address :{port} -f /configs/dcgm-counters-noprof.csv" benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" AIPERF_HTTP_TCP_USER_TIMEOUT: "900000" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 sbatch_directives: mem: "0" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/agg-h200-tp8-mtp-c8.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/agg-h200-tp8-mtp-c8.yaml index 7f81fde575..a59bd1e215 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/agg-h200-tp8-mtp-c8.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/agg-h200-tp8-mtp-c8.yaml @@ -5,17 +5,10 @@ name: agg-h200-tp8-mtp-c8 # committed GLM-5.2 golden thinking-on acceptance length for three speculative # steps, and max-running-requests is twice concurrency 8. -model: - path: hf:zai-org/GLM-5.2-FP8 - container: lmsysorg/sglang:nightly-dev-20260914-4358a161 - precision: fp8 - identity: model: - repo: zai-org/GLM-5.2-FP8 revision: f33c6dc501ee5a2c7e35155653b1b1abbc320951 - container: - image: lmsysorg/sglang:nightly-dev-20260914-4358a161 + container: {} frameworks: dynamo: 1.5.0.dev20260913 sglang: 4358a1617cad734fc37bf53d9e5c2092bd43d075 @@ -109,33 +102,14 @@ health_check: interval_seconds: 10 telemetry: - enabled: true - collect_interval_ms: 1000 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 collector_join_timeout_seconds: 12 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 - command: "dcgm-exporter --collect-interval=100 --address :{port} -f /configs/dcgm-counters-noprof.csv" benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" AIPERF_HTTP_TCP_USER_TIMEOUT: "900000" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 sbatch_directives: mem: "0" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/disagg-1p1d-pcp8-tp8-dp8-mtp6-hicache.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/disagg-1p1d-pcp8-tp8-dp8-mtp6-hicache.yaml index 4b8c3220f2..2e1f780829 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/disagg-1p1d-pcp8-tp8-dp8-mtp6-hicache.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/disagg-1p1d-pcp8-tp8-dp8-mtp6-hicache.yaml @@ -4,11 +4,6 @@ name: "disagg-h200-1p1d-pcp8-tp8-dp8-mtp6-hicache" # 1P1D GLM-5.2 FP8 AgentX topology on H200 with EAGLE MTP and prefill-side # hierarchical KV cache. The decode-side KV cache remains GPU-resident. -model: - path: "hf:zai-org/GLM-5.2-FP8" - container: "lmsysorg/sglang:v0.5.16-cu130" - precision: "fp8" - dynamo: install: true source: @@ -145,29 +140,8 @@ srun_options: mem: "0" container-remap-root: "" -telemetry: - enabled: true - collect_interval_ms: 1000 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 - command: "dcgm-exporter --collect-interval=100 --address :{port} -f /configs/dcgm-counters-noprof.csv" - benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/disagg-2p2d-pcp8-tp8-dp8-mtp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/disagg-2p2d-pcp8-tp8-dp8-mtp.yaml index 52438bfbb6..d38e10a227 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/disagg-2p2d-pcp8-tp8-dp8-mtp.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/disagg-2p2d-pcp8-tp8-dp8-mtp.yaml @@ -6,11 +6,6 @@ name: "disagg-h200-2p2d-pcp8-tp8-dp8-mtp" # cache remains GPU-resident. MTP uses the committed GLM-5.2 golden thinking-on # acceptance length for three speculative steps. -model: - path: "hf:zai-org/GLM-5.2-FP8" - container: "lmsysorg/sglang:v0.5.16-cu130" - precision: "fp8" - dynamo: install: true source: @@ -144,31 +139,8 @@ srun_options: mem: "0" container-remap-root: "" -telemetry: - enabled: true - collect_interval_ms: 1000 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 - command: "dcgm-exporter --collect-interval=100 --address :{port} -f /configs/dcgm-counters-noprof.csv" - benchmark: - type: custom - # Replaced with CONC_LIST by the launcher before submission. - concurrencies: [1] - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-variants.yaml index 379fb95af2..501a239064 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-variants.yaml @@ -4,10 +4,6 @@ schema: 2 base: - model: - path: nvidia/GLM-5.2-NVFP4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc28.dev202609210001 - precision: fp4 dynamo: install: true request_plane: tcp @@ -191,25 +187,14 @@ base: placement: node: head benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: "0" SERVED_MODEL_NAME: GLM-5.2-NVFP4 MAX_MODEL_LEN: '1048576' AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - OPENAI_API_KEY: EMPTY - KV_OFFLOADING: dram placement: node: dedicated diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.3/tilert/mi355x-fp8/agentx/disagg-1p1d-tp8-mtp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.3/tilert/mi355x-fp8/agentx/disagg-1p1d-tp8-mtp.yaml index 2ccafb3e5b..eaa23f6d5b 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.3/tilert/mi355x-fp8/agentx/disagg-1p1d-tp8-mtp.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.3/tilert/mi355x-fp8/agentx/disagg-1p1d-tp8-mtp.yaml @@ -1,9 +1,5 @@ schema: 2 name: glm5.3-mi355x-tilert-agentx -model: - path: GLM-5.3 - container: ghcr.io/tile-ai/tilert-rocm-decode:0.1.6 - precision: fp8 slurm: time_limit: "08:00:00" resources: @@ -99,16 +95,10 @@ health_check: max_attempts: 2160 interval_seconds: 5 benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - CONC: "1" MODEL: /model SERVED_MODEL_NAME: glm5_2 AIPERF_MAX_CONTEXT_LENGTH: "1048576" - RESULT_DIR: /infmax-workspace/LOGS/agentic - AGENTIC_OUTPUT_DIR: /infmax-workspace - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" HF_HOME: /logs/hf_cache HF_HUB_OFFLINE: "" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c1.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c1.yaml index e903c51b6b..a739fcb75d 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c1.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c1.yaml @@ -1,16 +1,9 @@ schema: 2 name: "kimik3-vllm-agg-b200-tp8pp2-mooncake-c1-agentic" -model: - path: "kimik3" - container: "vllm/vllm-openai:nightly-ac9126e58aa7bbab1856ba6593ba4d5003fea516" - precision: "fp4" - identity: - model: - repo: "moonshotai/Kimi-K3" - container: - image: "vllm/vllm-openai:nightly-ac9126e58aa7bbab1856ba6593ba4d5003fea516" + model: {} + container: {} dynamo: install: false @@ -114,31 +107,11 @@ srun_options: container-remap-root: "" telemetry: - enabled: true - collect_interval_ms: 1000 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 collector_join_timeout_seconds: 12 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 - command: "dcgm-exporter --collect-interval=100 --address :{port} -f /configs/dcgm-counters-noprof.csv" benchmark: - type: custom - concurrencies: [1] - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c14.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c14.yaml index 363ce48ad5..685a0486b6 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c14.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c14.yaml @@ -1,16 +1,9 @@ schema: 2 name: "kimik3-vllm-agg-b200-tp8pp2-mooncake-c14-agentic" -model: - path: "kimik3" - container: "vllm/vllm-openai:nightly-ac9126e58aa7bbab1856ba6593ba4d5003fea516" - precision: "fp4" - identity: - model: - repo: "moonshotai/Kimi-K3" - container: - image: "vllm/vllm-openai:nightly-ac9126e58aa7bbab1856ba6593ba4d5003fea516" + model: {} + container: {} dynamo: install: false @@ -114,31 +107,11 @@ srun_options: container-remap-root: "" telemetry: - enabled: true - collect_interval_ms: 1000 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 collector_join_timeout_seconds: 12 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 - command: "dcgm-exporter --collect-interval=100 --address :{port} -f /configs/dcgm-counters-noprof.csv" benchmark: - type: custom - concurrencies: [14] - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c24.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c24.yaml index f365fcd137..f93a44b717 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c24.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c24.yaml @@ -1,16 +1,9 @@ schema: 2 name: "kimik3-vllm-agg-b200-tp8pp2-mooncake-c24-agentic" -model: - path: "kimik3" - container: "vllm/vllm-openai:nightly-ac9126e58aa7bbab1856ba6593ba4d5003fea516" - precision: "fp4" - identity: - model: - repo: "moonshotai/Kimi-K3" - container: - image: "vllm/vllm-openai:nightly-ac9126e58aa7bbab1856ba6593ba4d5003fea516" + model: {} + container: {} dynamo: install: false @@ -114,31 +107,11 @@ srun_options: container-remap-root: "" telemetry: - enabled: true - collect_interval_ms: 1000 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 collector_join_timeout_seconds: 12 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 - command: "dcgm-exporter --collect-interval=100 --address :{port} -f /configs/dcgm-counters-noprof.csv" benchmark: - type: custom - concurrencies: [24] - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c32.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c32.yaml index 56b28836e7..8a85607222 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c32.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c32.yaml @@ -1,16 +1,9 @@ schema: 2 name: "kimik3-vllm-agg-b200-tp8pp2-mooncake-c32-agentic" -model: - path: "kimik3" - container: "vllm/vllm-openai:nightly-ac9126e58aa7bbab1856ba6593ba4d5003fea516" - precision: "fp4" - identity: - model: - repo: "moonshotai/Kimi-K3" - container: - image: "vllm/vllm-openai:nightly-ac9126e58aa7bbab1856ba6593ba4d5003fea516" + model: {} + container: {} dynamo: install: false @@ -114,31 +107,11 @@ srun_options: container-remap-root: "" telemetry: - enabled: true - collect_interval_ms: 1000 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 collector_join_timeout_seconds: 12 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 - command: "dcgm-exporter --collect-interval=100 --address :{port} -f /configs/dcgm-counters-noprof.csv" benchmark: - type: custom - concurrencies: [32] - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c4.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c4.yaml index b89d85ec28..51c49b6fe7 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c4.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c4.yaml @@ -1,16 +1,9 @@ schema: 2 name: "kimik3-vllm-agg-b200-tp8pp2-mooncake-c4-agentic" -model: - path: "kimik3" - container: "vllm/vllm-openai:nightly-ac9126e58aa7bbab1856ba6593ba4d5003fea516" - precision: "fp4" - identity: - model: - repo: "moonshotai/Kimi-K3" - container: - image: "vllm/vllm-openai:nightly-ac9126e58aa7bbab1856ba6593ba4d5003fea516" + model: {} + container: {} dynamo: install: false @@ -114,31 +107,11 @@ srun_options: container-remap-root: "" telemetry: - enabled: true - collect_interval_ms: 1000 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 collector_join_timeout_seconds: 12 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 - command: "dcgm-exporter --collect-interval=100 --address :{port} -f /configs/dcgm-counters-noprof.csv" benchmark: - type: custom - concurrencies: [4] - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c48.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c48.yaml index 490b381449..939f082257 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c48.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c48.yaml @@ -1,16 +1,9 @@ schema: 2 name: "kimik3-vllm-agg-b200-tp8pp2-mooncake-c48-agentic" -model: - path: "kimik3" - container: "vllm/vllm-openai:nightly-ac9126e58aa7bbab1856ba6593ba4d5003fea516" - precision: "fp4" - identity: - model: - repo: "moonshotai/Kimi-K3" - container: - image: "vllm/vllm-openai:nightly-ac9126e58aa7bbab1856ba6593ba4d5003fea516" + model: {} + container: {} dynamo: install: false @@ -114,31 +107,11 @@ srun_options: container-remap-root: "" telemetry: - enabled: true - collect_interval_ms: 1000 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 collector_join_timeout_seconds: 12 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 - command: "dcgm-exporter --collect-interval=100 --address :{port} -f /configs/dcgm-counters-noprof.csv" benchmark: - type: custom - concurrencies: [48] - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c56.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c56.yaml index 361ad8d7df..b9f8c68431 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c56.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c56.yaml @@ -1,16 +1,9 @@ schema: 2 name: "kimik3-vllm-agg-b200-tp8pp2-mooncake-c56-agentic" -model: - path: "kimik3" - container: "vllm/vllm-openai:nightly-ac9126e58aa7bbab1856ba6593ba4d5003fea516" - precision: "fp4" - identity: - model: - repo: "moonshotai/Kimi-K3" - container: - image: "vllm/vllm-openai:nightly-ac9126e58aa7bbab1856ba6593ba4d5003fea516" + model: {} + container: {} dynamo: install: false @@ -114,31 +107,11 @@ srun_options: container-remap-root: "" telemetry: - enabled: true - collect_interval_ms: 1000 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 collector_join_timeout_seconds: 12 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 - command: "dcgm-exporter --collect-interval=100 --address :{port} -f /configs/dcgm-counters-noprof.csv" benchmark: - type: custom - concurrencies: [56] - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c72.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c72.yaml index 8ac094e160..cb666cb58f 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c72.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c72.yaml @@ -1,16 +1,9 @@ schema: 2 name: "kimik3-vllm-agg-b200-tp8pp2-mooncake-c72-agentic" -model: - path: "kimik3" - container: "vllm/vllm-openai:nightly-ac9126e58aa7bbab1856ba6593ba4d5003fea516" - precision: "fp4" - identity: - model: - repo: "moonshotai/Kimi-K3" - container: - image: "vllm/vllm-openai:nightly-ac9126e58aa7bbab1856ba6593ba4d5003fea516" + model: {} + container: {} dynamo: install: false @@ -114,31 +107,11 @@ srun_options: container-remap-root: "" telemetry: - enabled: true - collect_interval_ms: 1000 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 collector_join_timeout_seconds: 12 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 - command: "dcgm-exporter --collect-interval=100 --address :{port} -f /configs/dcgm-counters-noprof.csv" benchmark: - type: custom - concurrencies: [72] - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c8.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c8.yaml index e599cc11d9..4d342809ae 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c8.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c8.yaml @@ -1,16 +1,9 @@ schema: 2 name: "kimik3-vllm-agg-b200-tp8pp2-mooncake-c8-agentic" -model: - path: "kimik3" - container: "vllm/vllm-openai:nightly-ac9126e58aa7bbab1856ba6593ba4d5003fea516" - precision: "fp4" - identity: - model: - repo: "moonshotai/Kimi-K3" - container: - image: "vllm/vllm-openai:nightly-ac9126e58aa7bbab1856ba6593ba4d5003fea516" + model: {} + container: {} dynamo: install: false @@ -114,31 +107,11 @@ srun_options: container-remap-root: "" telemetry: - enabled: true - collect_interval_ms: 1000 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 collector_join_timeout_seconds: 12 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 - command: "dcgm-exporter --collect-interval=100 --address :{port} -f /configs/dcgm-counters-noprof.csv" benchmark: - type: custom - concurrencies: [8] - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c80.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c80.yaml index 883e6b3970..4ac947e16e 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c80.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c80.yaml @@ -1,16 +1,9 @@ schema: 2 name: "kimik3-vllm-agg-b200-tp8pp2-mooncake-c80-agentic" -model: - path: "kimik3" - container: "vllm/vllm-openai:nightly-ac9126e58aa7bbab1856ba6593ba4d5003fea516" - precision: "fp4" - identity: - model: - repo: "moonshotai/Kimi-K3" - container: - image: "vllm/vllm-openai:nightly-ac9126e58aa7bbab1856ba6593ba4d5003fea516" + model: {} + container: {} dynamo: install: false @@ -114,31 +107,11 @@ srun_options: container-remap-root: "" telemetry: - enabled: true - collect_interval_ms: 1000 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 collector_join_timeout_seconds: 12 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 - command: "dcgm-exporter --collect-interval=100 --address :{port} -f /configs/dcgm-counters-noprof.csv" benchmark: - type: custom - concurrencies: [80] - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c96.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c96.yaml index 7f0f26dae5..79b11d98d6 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c96.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c96.yaml @@ -1,16 +1,9 @@ schema: 2 name: "kimik3-vllm-agg-b200-tp8pp2-mooncake-c96-agentic" -model: - path: "kimik3" - container: "vllm/vllm-openai:nightly-ac9126e58aa7bbab1856ba6593ba4d5003fea516" - precision: "fp4" - identity: - model: - repo: "moonshotai/Kimi-K3" - container: - image: "vllm/vllm-openai:nightly-ac9126e58aa7bbab1856ba6593ba4d5003fea516" + model: {} + container: {} dynamo: install: false @@ -113,31 +106,11 @@ srun_options: container-remap-root: "" telemetry: - enabled: true - collect_interval_ms: 1000 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 collector_join_timeout_seconds: 12 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 - command: "dcgm-exporter --collect-interval=100 --address :{port} -f /configs/dcgm-counters-noprof.csv" benchmark: - type: custom - concurrencies: [96] - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dcp16-dspark4-maxseq2-mooncake.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dcp16-dspark4-maxseq2-mooncake.yaml index 0ad4c659ef..12b7c9ce4a 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dcp16-dspark4-maxseq2-mooncake.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dcp16-dspark4-maxseq2-mooncake.yaml @@ -2,16 +2,9 @@ schema: 2 name: "kimi-k3-vllm-agg-gb200-dcp16-dspark4-maxseq2-mooncake-agentic" -model: - path: "kimi-k3" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-75c2eef" - precision: "fp4" - identity: - model: - repo: "moonshotai/Kimi-K3" - container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-75c2eef" + model: {} + container: {} frameworks: dynamo: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" @@ -144,36 +137,15 @@ srun_options: container-remap-root: "" telemetry: - enabled: true - collect_interval_ms: 1000 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 collector_join_timeout_seconds: 12 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 - command: "dcgm-exporter --collect-interval=100 --address :{port} -f /configs/dcgm-counters-noprof.csv" benchmark: - type: custom placement: node: head - concurrencies: [1, 2, 4, 8, 16] - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - ENABLE_AGENTX_POWER: "1" REQUIRE_POWER: "1" - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" AIPERF_HTTP_TCP_USER_TIMEOUT: "900000" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dcp16-nospec-mooncake.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dcp16-nospec-mooncake.yaml index 173c0128ba..4ae9f39e08 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dcp16-nospec-mooncake.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dcp16-nospec-mooncake.yaml @@ -2,16 +2,9 @@ schema: 2 name: "kimi-k3-vllm-agg-gb200-dcp16-nospec-mooncake-agentic" -model: - path: "kimi-k3" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-75c2eef" - precision: "fp4" - identity: - model: - repo: "moonshotai/Kimi-K3" - container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-75c2eef" + model: {} + container: {} frameworks: dynamo: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" @@ -140,36 +133,15 @@ srun_options: container-remap-root: "" telemetry: - enabled: true - collect_interval_ms: 1000 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 collector_join_timeout_seconds: 12 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 - command: "dcgm-exporter --collect-interval=100 --address :{port} -f /configs/dcgm-counters-noprof.csv" benchmark: - type: custom placement: node: head - concurrencies: [8, 40, 48] - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - ENABLE_AGENTX_POWER: "1" REQUIRE_POWER: "1" - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" AIPERF_HTTP_TCP_USER_TIMEOUT: "900000" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c16.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c16.yaml index 09c18c0017..fec3a725a6 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c16.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c16.yaml @@ -1,16 +1,9 @@ schema: 2 name: "kimik3-vllm-agg-gb200-tp8pp2-mooncake-c16-agentic" -model: - path: "kimi-k3" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-728d3ad" - precision: "fp4" - identity: - model: - repo: "moonshotai/Kimi-K3" - container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-728d3ad" + model: {} + container: {} dynamo: install: false @@ -113,36 +106,15 @@ srun_options: container-remap-root: "" telemetry: - enabled: true - collect_interval_ms: 1000 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 collector_join_timeout_seconds: 12 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 - command: "dcgm-exporter --collect-interval=100 --address :{port} -f /configs/dcgm-counters-noprof.csv" benchmark: - type: custom placement: node: head - concurrencies: [16] - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - ENABLE_AGENTX_POWER: "1" REQUIRE_POWER: "1" - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" AIPERF_SERVER_METRICS_URLS: "http://localhost:8000/metrics" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c32.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c32.yaml index 9d9e543fb1..000d53c5dd 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c32.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c32.yaml @@ -1,16 +1,9 @@ schema: 2 name: "kimik3-vllm-agg-gb200-tp8pp2-mooncake-c32-agentic" -model: - path: "kimi-k3" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-728d3ad" - precision: "fp4" - identity: - model: - repo: "moonshotai/Kimi-K3" - container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-728d3ad" + model: {} + container: {} dynamo: install: false @@ -113,36 +106,15 @@ srun_options: container-remap-root: "" telemetry: - enabled: true - collect_interval_ms: 1000 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 collector_join_timeout_seconds: 12 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 - command: "dcgm-exporter --collect-interval=100 --address :{port} -f /configs/dcgm-counters-noprof.csv" benchmark: - type: custom placement: node: head - concurrencies: [32] - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - ENABLE_AGENTX_POWER: "1" REQUIRE_POWER: "1" - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" AIPERF_SERVER_METRICS_URLS: "http://localhost:8000/metrics" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c48.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c48.yaml index d9dfbd4c25..4b8f04652a 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c48.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c48.yaml @@ -1,16 +1,9 @@ schema: 2 name: "kimik3-vllm-agg-gb200-tp8pp2-mooncake-c48-agentic" -model: - path: "kimi-k3" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-728d3ad" - precision: "fp4" - identity: - model: - repo: "moonshotai/Kimi-K3" - container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-728d3ad" + model: {} + container: {} dynamo: install: false @@ -112,36 +105,15 @@ srun_options: container-remap-root: "" telemetry: - enabled: true - collect_interval_ms: 1000 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 collector_join_timeout_seconds: 12 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 - command: "dcgm-exporter --collect-interval=100 --address :{port} -f /configs/dcgm-counters-noprof.csv" benchmark: - type: custom placement: node: head - concurrencies: [48] - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - ENABLE_AGENTX_POWER: "1" REQUIRE_POWER: "1" - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" AIPERF_SERVER_METRICS_URLS: "http://localhost:8000/metrics" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c72.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c72.yaml index 758cf040df..57e894f523 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c72.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c72.yaml @@ -1,16 +1,9 @@ schema: 2 name: "kimik3-vllm-agg-gb200-tp8pp2-mooncake-c72-agentic" -model: - path: "kimi-k3" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-728d3ad" - precision: "fp4" - identity: - model: - repo: "moonshotai/Kimi-K3" - container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-728d3ad" + model: {} + container: {} dynamo: install: false @@ -112,36 +105,15 @@ srun_options: container-remap-root: "" telemetry: - enabled: true - collect_interval_ms: 1000 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 collector_join_timeout_seconds: 12 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 - command: "dcgm-exporter --collect-interval=100 --address :{port} -f /configs/dcgm-counters-noprof.csv" benchmark: - type: custom placement: node: head - concurrencies: [72] - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - ENABLE_AGENTX_POWER: "1" REQUIRE_POWER: "1" - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" AIPERF_SERVER_METRICS_URLS: "http://localhost:8000/metrics" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c96.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c96.yaml index bd4af535ec..17e0743f63 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c96.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c96.yaml @@ -1,16 +1,9 @@ schema: 2 name: "kimik3-vllm-agg-gb200-tp8pp2-mooncake-c96-agentic" -model: - path: "kimi-k3" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-728d3ad" - precision: "fp4" - identity: - model: - repo: "moonshotai/Kimi-K3" - container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-728d3ad" + model: {} + container: {} dynamo: install: false @@ -112,36 +105,15 @@ srun_options: container-remap-root: "" telemetry: - enabled: true - collect_interval_ms: 1000 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 collector_join_timeout_seconds: 12 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 - command: "dcgm-exporter --collect-interval=100 --address :{port} -f /configs/dcgm-counters-noprof.csv" benchmark: - type: custom placement: node: head - concurrencies: [96] - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - ENABLE_AGENTX_POWER: "1" REQUIRE_POWER: "1" - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" AIPERF_SERVER_METRICS_URLS: "http://localhost:8000/metrics" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-variants.yaml index 5a38f180c0..e07b7ca885 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-variants.yaml @@ -4,15 +4,9 @@ schema: 2 base: - model: - path: "kimi-k3" - container: "vllm/vllm-openai:kimi-k3" - precision: "fp4" identity: - model: - repo: "moonshotai/Kimi-K3" - container: - image: "vllm/vllm-openai:kimi-k3" + model: {} + container: {} frameworks: dynamo: "1.3.0" dynamo: @@ -113,19 +107,10 @@ base: srun_options: container-remap-root: "" benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" # Day-0 GB200 translation of the official latency-oriented multi_node_tp # profile. TP16 spans four GB200 nodes and uses the official FP8 KV cache. diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/agg-dcp8-dspark4-mooncake.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/agg-dcp8-dspark4-mooncake.yaml index bfab748b71..15bfa64e43 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/agg-dcp8-dspark4-mooncake.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/agg-dcp8-dspark4-mooncake.yaml @@ -1,16 +1,9 @@ schema: 2 name: "kimi-k3-vllm-agg-gb300-dcp8-dspark4-mooncake-agentic" -model: - path: "moonshotai/Kimi-K3" - container: "vllm/vllm-openai:nightly-dc36fcce902a63eab06c1b93a5c4a5ee178a0c56" - precision: "fp4" - identity: - model: - repo: "moonshotai/Kimi-K3" - container: - image: "vllm/vllm-openai:nightly-dc36fcce902a63eab06c1b93a5c4a5ee178a0c56" + model: {} + container: {} frameworks: dynamo: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" @@ -131,36 +124,15 @@ srun_options: container-remap-root: "" telemetry: - enabled: true - collect_interval_ms: 1000 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 collector_join_timeout_seconds: 12 - dcgm_exporter: - container_image: dcgm-exporter - port: 19401 - command: "dcgm-exporter --collect-interval=100 --address :{port} -f /configs/dcgm-counters-noprof.csv" benchmark: - type: custom - concurrencies: [48, 52, 56] - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - ENABLE_AGENTX_POWER: "1" REQUIRE_POWER: "1" - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" AIPERF_HTTP_TCP_USER_TIMEOUT: "900000" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" placement: node: head diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/agg-dcp8-dspark7-maxseq2-mooncake.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/agg-dcp8-dspark7-maxseq2-mooncake.yaml index 58aa8ae8cd..f7cf792b91 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/agg-dcp8-dspark7-maxseq2-mooncake.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/agg-dcp8-dspark7-maxseq2-mooncake.yaml @@ -1,16 +1,9 @@ schema: 2 name: "kimi-k3-vllm-agg-gb300-dcp8-dspark7-maxseq2-mooncake-agentic" -model: - path: "moonshotai/Kimi-K3" - container: "vllm/vllm-openai:nightly-dc36fcce902a63eab06c1b93a5c4a5ee178a0c56" - precision: "fp4" - identity: - model: - repo: "moonshotai/Kimi-K3" - container: - image: "vllm/vllm-openai:nightly-dc36fcce902a63eab06c1b93a5c4a5ee178a0c56" + model: {} + container: {} frameworks: dynamo: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" @@ -129,32 +122,11 @@ srun_options: container-remap-root: "" telemetry: - enabled: true - collect_interval_ms: 1000 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 collector_join_timeout_seconds: 12 - dcgm_exporter: - container_image: dcgm-exporter - port: 19401 - command: "dcgm-exporter --collect-interval=100 --address :{port} -f /configs/dcgm-counters-noprof.csv" benchmark: - type: custom - concurrencies: [1, 4] - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - ENABLE_AGENTX_POWER: "1" REQUIRE_POWER: "1" - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" placement: node: head diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p1d-dcp8-dcp8-dspark4-mooncake.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p1d-dcp8-dcp8-dspark4-mooncake.yaml index 46641fcb11..66b252bf46 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p1d-dcp8-dcp8-dspark4-mooncake.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p1d-dcp8-dcp8-dspark4-mooncake.yaml @@ -1,16 +1,9 @@ schema: 2 name: "kimi-k3-vllm-disagg-gb300-1p1d-dcp8-dcp8-dspark4-mooncake-agentic" -model: - path: "moonshotai/Kimi-K3" - container: "vllm/vllm-openai:nightly-dc36fcce902a63eab06c1b93a5c4a5ee178a0c56" - precision: "fp4" - identity: - model: - repo: "moonshotai/Kimi-K3" - container: - image: "vllm/vllm-openai:nightly-dc36fcce902a63eab06c1b93a5c4a5ee178a0c56" + model: {} + container: {} frameworks: dynamo: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" @@ -173,34 +166,13 @@ srun_options: container-remap-root: "" telemetry: - enabled: true - collect_interval_ms: 1000 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 collector_join_timeout_seconds: 12 - dcgm_exporter: - container_image: dcgm-exporter - port: 19401 - command: "dcgm-exporter --collect-interval=100 --address :{port} -f /configs/dcgm-counters-noprof.csv" benchmark: - type: custom placement: node: head - concurrencies: [48, 52, 56] - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - ENABLE_AGENTX_POWER: "1" REQUIRE_POWER: "1" - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p2d-dcp8-dcp8-dspark4-mooncake.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p2d-dcp8-dcp8-dspark4-mooncake.yaml index 2f5fc7809f..096fb1b01a 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p2d-dcp8-dcp8-dspark4-mooncake.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p2d-dcp8-dcp8-dspark4-mooncake.yaml @@ -1,16 +1,9 @@ schema: 2 name: "kimi-k3-vllm-disagg-gb300-1p2d-dcp8-dcp8-dspark4-mooncake-agentic" -model: - path: "moonshotai/Kimi-K3" - container: "vllm/vllm-openai:nightly-dc36fcce902a63eab06c1b93a5c4a5ee178a0c56" - precision: "fp4" - identity: - model: - repo: "moonshotai/Kimi-K3" - container: - image: "vllm/vllm-openai:nightly-dc36fcce902a63eab06c1b93a5c4a5ee178a0c56" + model: {} + container: {} frameworks: dynamo: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" @@ -173,34 +166,13 @@ srun_options: container-remap-root: "" telemetry: - enabled: true - collect_interval_ms: 1000 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 collector_join_timeout_seconds: 12 - dcgm_exporter: - container_image: dcgm-exporter - port: 19401 - command: "dcgm-exporter --collect-interval=100 --address :{port} -f /configs/dcgm-counters-noprof.csv" benchmark: - type: custom placement: node: head - concurrencies: [32, 48, 64] - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - ENABLE_AGENTX_POWER: "1" REQUIRE_POWER: "1" - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p3d-dcp8-dcp8-dspark4-mooncake.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p3d-dcp8-dcp8-dspark4-mooncake.yaml index a0fac39310..0e87cae609 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p3d-dcp8-dcp8-dspark4-mooncake.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p3d-dcp8-dcp8-dspark4-mooncake.yaml @@ -1,16 +1,9 @@ schema: 2 name: "kimi-k3-vllm-disagg-gb300-1p3d-dcp8-dcp8-dspark4-mooncake-agentic" -model: - path: "moonshotai/Kimi-K3" - container: "vllm/vllm-openai:nightly-dc36fcce902a63eab06c1b93a5c4a5ee178a0c56" - precision: "fp4" - identity: - model: - repo: "moonshotai/Kimi-K3" - container: - image: "vllm/vllm-openai:nightly-dc36fcce902a63eab06c1b93a5c4a5ee178a0c56" + model: {} + container: {} frameworks: dynamo: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" @@ -173,34 +166,13 @@ srun_options: container-remap-root: "" telemetry: - enabled: true - collect_interval_ms: 1000 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 collector_join_timeout_seconds: 12 - dcgm_exporter: - container_image: dcgm-exporter - port: 19401 - command: "dcgm-exporter --collect-interval=100 --address :{port} -f /configs/dcgm-counters-noprof.csv" benchmark: - type: custom placement: node: head - concurrencies: [32, 48] - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - ENABLE_AGENTX_POWER: "1" REQUIRE_POWER: "1" - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p3d-dcp8-dcp8-dspark7-mooncake.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p3d-dcp8-dcp8-dspark7-mooncake.yaml index e89794f552..51c43c64cc 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p3d-dcp8-dcp8-dspark7-mooncake.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p3d-dcp8-dcp8-dspark7-mooncake.yaml @@ -1,16 +1,9 @@ schema: 2 name: "kimi-k3-vllm-disagg-gb300-1p3d-dcp8-dcp8-dspark7-mooncake-agentic" -model: - path: "moonshotai/Kimi-K3" - container: "vllm/vllm-openai:nightly-dc36fcce902a63eab06c1b93a5c4a5ee178a0c56" - precision: "fp4" - identity: - model: - repo: "moonshotai/Kimi-K3" - container: - image: "vllm/vllm-openai:nightly-dc36fcce902a63eab06c1b93a5c4a5ee178a0c56" + model: {} + container: {} frameworks: dynamo: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" @@ -173,34 +166,13 @@ srun_options: container-remap-root: "" telemetry: - enabled: true - collect_interval_ms: 1000 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 collector_join_timeout_seconds: 12 - dcgm_exporter: - container_image: dcgm-exporter - port: 19401 - command: "dcgm-exporter --collect-interval=100 --address :{port} -f /configs/dcgm-counters-noprof.csv" benchmark: - type: custom placement: node: head - concurrencies: [1] - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - ENABLE_AGENTX_POWER: "1" REQUIRE_POWER: "1" - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/h200-fp4/agentx/agg-tp16dp2ep32-latency.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/h200-fp4/agentx/agg-tp16dp2ep32-latency.yaml index 0d9e535e4c..3af8669c39 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/h200-fp4/agentx/agg-tp16dp2ep32-latency.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/h200-fp4/agentx/agg-tp16dp2ep32-latency.yaml @@ -1,17 +1,10 @@ schema: 2 name: "kimik3-vllm-agg-h200-tp16dp2ep32-latency-agentic" -model: - path: "kimik3" - container: "vllm/vllm-openai:kimi-k3" - precision: "fp4" - identity: model: - repo: "moonshotai/Kimi-K3" revision: "9f62e4e9fffbd0a83ddd60e1c209d828994b3569" - container: - image: "vllm/vllm-openai:kimi-k3" + container: {} frameworks: vllm: "0.1.dev19262+gb6bbf29dd.d20260727" @@ -83,28 +76,8 @@ srun_options: container-remap-root: "" telemetry: - enabled: true - collect_interval_ms: 1000 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 collector_join_timeout_seconds: 12 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 - command: "dcgm-exporter --collect-interval=100 --address :{port} -f /configs/dcgm-counters-noprof.csv" benchmark: - type: custom - concurrencies: [1, 2, 3, 4, 5, 6, 7, 8, 10, 12] - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/h200-fp4/agentx/agg-tp8dp4ep32-balanced.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/h200-fp4/agentx/agg-tp8dp4ep32-balanced.yaml index ada72514f2..e4634bd00f 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/h200-fp4/agentx/agg-tp8dp4ep32-balanced.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/h200-fp4/agentx/agg-tp8dp4ep32-balanced.yaml @@ -1,17 +1,10 @@ schema: 2 name: "kimik3-vllm-agg-h200-tp8dp4ep32-balanced-agentic" -model: - path: "kimik3" - container: "vllm/vllm-openai:kimi-k3" - precision: "fp4" - identity: model: - repo: "moonshotai/Kimi-K3" revision: "9f62e4e9fffbd0a83ddd60e1c209d828994b3569" - container: - image: "vllm/vllm-openai:kimi-k3" + container: {} frameworks: vllm: "0.1.dev19262+gb6bbf29dd.d20260727" @@ -84,28 +77,8 @@ srun_options: container-remap-root: "" telemetry: - enabled: true - collect_interval_ms: 1000 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 collector_join_timeout_seconds: 12 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 - command: "dcgm-exporter --collect-interval=100 --address :{port} -f /configs/dcgm-counters-noprof.csv" benchmark: - type: custom - concurrencies: [1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 14, 16] - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/h200-fp4/agentx/agg-tp8dp4ep32-vllm-simple.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/h200-fp4/agentx/agg-tp8dp4ep32-vllm-simple.yaml index 6bec983914..fc6b3f6133 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/h200-fp4/agentx/agg-tp8dp4ep32-vllm-simple.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/h200-fp4/agentx/agg-tp8dp4ep32-vllm-simple.yaml @@ -1,17 +1,10 @@ schema: 2 name: "kimik3-vllm-agg-h200-tp8dp4ep32-vllm-simple-agentic" -model: - path: "kimik3" - container: "vllm/vllm-openai:kimi-k3" - precision: "fp4" - identity: model: - repo: "moonshotai/Kimi-K3" revision: "9f62e4e9fffbd0a83ddd60e1c209d828994b3569" - container: - image: "vllm/vllm-openai:kimi-k3" + container: {} frameworks: vllm: "0.1.dev19262+gb6bbf29dd.d20260727" @@ -85,28 +78,8 @@ srun_options: container-remap-root: "" telemetry: - enabled: true - collect_interval_ms: 1000 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 collector_join_timeout_seconds: 12 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 - command: "dcgm-exporter --collect-interval=100 --address :{port} -f /configs/dcgm-counters-noprof.csv" benchmark: - type: custom - concurrencies: [8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32] - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-variants.yaml index 735ae0b008..d78fef2a09 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-variants.yaml @@ -4,10 +4,6 @@ schema: 2 base: - model: - path: minimax-m3-nvfp4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc23.post1 - precision: fp4 dynamo: install: true source: @@ -113,32 +109,14 @@ base: router-mode: kv no-kv-events: true benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: '0' SERVED_MODEL_NAME: nvidia/MiniMax-M3-NVFP4 MAX_MODEL_LEN: '1048576' AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - OPENAI_API_KEY: EMPTY - KV_OFFLOADING: dram - KV_OFFLOAD_BACKEND: native KV_OFFLOAD_BACKEND_METADATA: '{"name":"native"}' - TOTAL_CPU_DRAM_GB: '550' - MODEL: nvidia/MiniMax-M3-NVFP4 - MODEL_PREFIX: minimaxm3 - FRAMEWORK: dynamo-trt - PRECISION: fp4 - DURATION: '3600' # MiniMax-M3 NVFP4, TensorRT-LLM aggregated TP4 + EAGLE3-GQA (3 draft tokens), one GB200 node, Dynamo frontend. # Engine config is the max_batch_size=5 member of the TP4 recipe set. @@ -158,9 +136,6 @@ override_c5_b5: - 3 - 4 - 5 - benchmark: - env: - CONC: '5' # MiniMax-M3 NVFP4, TensorRT-LLM aggregated TP4 + EAGLE3-GQA (3 draft tokens), one GB200 node, Dynamo frontend. # Engine config is the max_batch_size=10 member of the TP4 recipe set. @@ -185,9 +160,6 @@ override_c10_b10: - 8 - 9 - 10 - benchmark: - env: - CONC: '10' # MiniMax-M3 NVFP4, TensorRT-LLM aggregated TP4 + EAGLE3-GQA (3 draft tokens), one GB200 node, Dynamo frontend. # Engine config is the max_batch_size=15 member of the TP4 recipe set. @@ -217,9 +189,6 @@ override_c15_b15: - 13 - 14 - 15 - benchmark: - env: - CONC: '15' # MiniMax-M3 NVFP4, TensorRT-LLM aggregated TP4 + EAGLE3-GQA (3 draft tokens), one GB200 node, Dynamo frontend. # Engine config is the max_batch_size=20 member of the TP4 recipe set. @@ -254,9 +223,6 @@ override_c20_b20: - 18 - 19 - 20 - benchmark: - env: - CONC: '20' # MiniMax-M3 NVFP4, TensorRT-LLM aggregated TP4 + EAGLE3-GQA (3 draft tokens), one GB200 node, Dynamo frontend. # Engine config is the max_batch_size=25 member of the TP4 recipe set. @@ -291,9 +257,6 @@ override_c25_b25: - 21 - 23 - 25 - benchmark: - env: - CONC: '25' # MiniMax-M3 NVFP4, TensorRT-LLM aggregated TP4 + EAGLE3-GQA (3 draft tokens), one GB200 node, Dynamo frontend. # Engine config is the max_batch_size=30 member of the TP4 recipe set. @@ -328,9 +291,6 @@ override_c30_b30: - 24 - 27 - 30 - benchmark: - env: - CONC: '30' # MiniMax-M3 NVFP4, TensorRT-LLM aggregated TP4 + EAGLE3-GQA (3 draft tokens), one GB200 node, Dynamo frontend. # Engine config is the max_batch_size=40 member of the TP4 recipe set. @@ -369,6 +329,3 @@ override_c40_b40: - 36 - 38 - 40 - benchmark: - env: - CONC: '40' diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-nightly-native-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-nightly-native-variants.yaml index 1e65fda9de..08bd1fddb9 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-nightly-native-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-nightly-native-variants.yaml @@ -4,13 +4,9 @@ schema: 2 base: - model: - path: "minimax-m3-nvfp4" - container: "vllm/vllm-openai:nightly-9ea8f3ffc354901b740f0b31988900897b7221d7" - precision: "fp4" identity: - model: {repo: "nvidia/MiniMax-M3-NVFP4"} - container: {image: "vllm/vllm-openai:nightly-9ea8f3ffc354901b740f0b31988900897b7221d7"} + model: {} + container: {} frameworks: {dynamo: "1.5.0.dev20260908"} dynamo: {install: true, source: {pypi: "1.5.0.dev20260908"}} environment: {ETCD_LEASE_TTL: "7200"} @@ -88,21 +84,12 @@ base: sbatch_directives: {cpus-per-task: "144", mem: "0"} srun_options: {container-remap-root: ""} benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" AIPERF_EXTRA_INPUTS: "thinking:true" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" override_tp4: name: "minimax-m3-vllm-agg-gb200-tp4-agentic-nightly-native" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-variants.yaml index 67870f9257..6280fd1aef 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-variants.yaml @@ -4,13 +4,9 @@ schema: 2 base: - model: - path: "minimax-m3-nvfp4" - container: "vllm/vllm-openai@sha256:b9104b7ef3048e42f79fba9ab5da06e5aff8164aca9968692ec2f569aaaf34c6" - precision: "fp4" identity: - model: {repo: "nvidia/MiniMax-M3-NVFP4"} - container: {image: "vllm/vllm-openai@sha256:b9104b7ef3048e42f79fba9ab5da06e5aff8164aca9968692ec2f569aaaf34c6"} + model: {} + container: {} frameworks: {dynamo: "1.5.0.dev20260819"} dynamo: {install: true, source: {wheel: "1.5.0.dev20260819"}} environment: {PYTHONHASHSEED: "0"} @@ -175,13 +171,7 @@ base: sbatch_directives: {cpus-per-task: "144", mem: "0"} srun_options: {container-remap-root: ""} benchmark: - type: custom - command: "bash /infmax-workspace/benchmarks/srt_agentic.sh" env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" AGENTIC_WARMUP_GRACE_PERIOD: "1800" AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" @@ -189,9 +179,6 @@ base: AIPERF_EXTRA_INPUTS: "thinking:true" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" override_1p1d_tp8_c1: name: "minimax-m3-vllm-disagg-gb200-1p1d-tp4-tp8-c1-agentic" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-variants.yaml index 51e581e71b..fbf86ee2db 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-variants.yaml @@ -4,15 +4,9 @@ schema: 2 base: - model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - precision: "fp4" identity: - model: - repo: "nvidia/MiniMax-M3-NVFP4" - container: - image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + model: {} + container: {} frameworks: dynamo: "1.4.0.dev20260730" dynamo: @@ -168,19 +162,10 @@ base: srun_options: container-remap-root: "" benchmark: - type: custom - command: "bash /infmax-workspace/benchmarks/srt_agentic.sh" env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" AIPERF_EXTRA_INPUTS: "thinking:true" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" override_1p1d_tep4_tp4_c1: name: "minimax-m3-vllm-disagg-gb300-1p1d-tep4-tp4-c1-fp4-agentic" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c16-mtp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c16-mtp.yaml index c7ca2cfa1c..c9d08af1a8 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c16-mtp.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c16-mtp.yaml @@ -1,10 +1,6 @@ # Experimental; retain only measured improvements over the current Pareto frontier. schema: 2 name: qwen35-b200-fp8-agentx-disagg-1p1d-p-tp4-d-tp4-hicache-c16-mtp -model: - path: qwen3.5-fp8 - container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 - precision: fp8 engine: sglang dynamo: install: true @@ -142,35 +138,10 @@ srun_options: mem: '0' container-remap-root: '' telemetry: - enabled: true - collect_interval_ms: 1000 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 collector_join_timeout_seconds: 12 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 - command: "dcgm-exporter --collect-interval=100 --address :{port} -f /configs/dcgm-counters-noprof.csv" benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - concurrencies: - - 16 env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - SRT_MEASUREMENT_WINDOW_BENCHMARK_TYPE: custom - SRT_MEASUREMENT_WINDOW_CONCURRENCIES: '16' - SRT_MEASUREMENT_WINDOW_RESULT_ROOT: /logs - PORT: '8000' - IS_MULTINODE: 'true' - EVAL_FRAMEWORK: lm-eval - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k AIPERF_MAX_CONTEXT_LENGTH: '262144' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c24-mtp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c24-mtp.yaml index 7646e9db61..79190ebf1d 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c24-mtp.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c24-mtp.yaml @@ -1,10 +1,6 @@ # Experimental; retain only measured improvements over the current Pareto frontier. schema: 2 name: qwen35-b200-fp8-agentx-disagg-1p1d-p-tp4-d-tp4-hicache-c24-mtp -model: - path: qwen3.5-fp8 - container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 - precision: fp8 engine: sglang dynamo: install: true @@ -142,35 +138,10 @@ srun_options: mem: '0' container-remap-root: '' telemetry: - enabled: true - collect_interval_ms: 1000 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 collector_join_timeout_seconds: 12 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 - command: "dcgm-exporter --collect-interval=100 --address :{port} -f /configs/dcgm-counters-noprof.csv" benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - concurrencies: - - 24 env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - SRT_MEASUREMENT_WINDOW_BENCHMARK_TYPE: custom - SRT_MEASUREMENT_WINDOW_CONCURRENCIES: '24' - SRT_MEASUREMENT_WINDOW_RESULT_ROOT: /logs - PORT: '8000' - IS_MULTINODE: 'true' - EVAL_FRAMEWORK: lm-eval - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k AIPERF_MAX_CONTEXT_LENGTH: '262144' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c32-mtp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c32-mtp.yaml index c974dec6e5..8cbcbeff0a 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c32-mtp.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c32-mtp.yaml @@ -1,10 +1,6 @@ # Fast replay improves the published frontier; canonical sweep qualification is pending. schema: 2 name: qwen35-b200-fp8-agentx-disagg-1p1d-p-tp4-d-tp4-hicache-c32-mtp -model: - path: qwen3.5-fp8 - container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 - precision: fp8 engine: sglang dynamo: install: true @@ -142,35 +138,10 @@ srun_options: mem: '0' container-remap-root: '' telemetry: - enabled: true - collect_interval_ms: 1000 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 collector_join_timeout_seconds: 12 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 - command: "dcgm-exporter --collect-interval=100 --address :{port} -f /configs/dcgm-counters-noprof.csv" benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - concurrencies: - - 32 env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - SRT_MEASUREMENT_WINDOW_BENCHMARK_TYPE: custom - SRT_MEASUREMENT_WINDOW_CONCURRENCIES: '32' - SRT_MEASUREMENT_WINDOW_RESULT_ROOT: /logs - PORT: '8000' - IS_MULTINODE: 'true' - EVAL_FRAMEWORK: lm-eval - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k AIPERF_MAX_CONTEXT_LENGTH: '262144' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c48-write-through-mtp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c48-write-through-mtp.yaml index 6e20f62ee1..d48edf78ca 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c48-write-through-mtp.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c48-write-through-mtp.yaml @@ -1,10 +1,6 @@ # Experimental; retain only measured improvements over the current Pareto frontier. schema: 2 name: qwen35-b200-fp8-agentx-disagg-1p1d-p-tp4-d-tp4-hicache-c48-write-through-mtp -model: - path: qwen3.5-fp8 - container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 - precision: fp8 engine: sglang dynamo: install: true @@ -143,35 +139,10 @@ srun_options: mem: '0' container-remap-root: '' telemetry: - enabled: true - collect_interval_ms: 1000 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 collector_join_timeout_seconds: 12 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 - command: "dcgm-exporter --collect-interval=100 --address :{port} -f /configs/dcgm-counters-noprof.csv" benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - concurrencies: - - 48 env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - SRT_MEASUREMENT_WINDOW_BENCHMARK_TYPE: custom - SRT_MEASUREMENT_WINDOW_CONCURRENCIES: '48' - SRT_MEASUREMENT_WINDOW_RESULT_ROOT: /logs - PORT: '8000' - IS_MULTINODE: 'true' - EVAL_FRAMEWORK: lm-eval - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k AIPERF_MAX_CONTEXT_LENGTH: '262144' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c64-write-through-mtp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c64-write-through-mtp.yaml index 9cd1f0c73c..68396a77dc 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c64-write-through-mtp.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c64-write-through-mtp.yaml @@ -1,10 +1,6 @@ # Short replay improves the published frontier; full official qualification remains required. schema: 2 name: qwen35-b200-fp8-agentx-disagg-1p1d-p-tp4-d-tp4-hicache-c64-write-through-mtp -model: - path: qwen3.5-fp8 - container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 - precision: fp8 engine: sglang dynamo: install: true @@ -143,35 +139,10 @@ srun_options: mem: '0' container-remap-root: '' telemetry: - enabled: true - collect_interval_ms: 1000 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 collector_join_timeout_seconds: 12 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 - command: "dcgm-exporter --collect-interval=100 --address :{port} -f /configs/dcgm-counters-noprof.csv" benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - concurrencies: - - 64 env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - SRT_MEASUREMENT_WINDOW_BENCHMARK_TYPE: custom - SRT_MEASUREMENT_WINDOW_CONCURRENCIES: '64' - SRT_MEASUREMENT_WINDOW_RESULT_ROOT: /logs - PORT: '8000' - IS_MULTINODE: 'true' - EVAL_FRAMEWORK: lm-eval - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k AIPERF_MAX_CONTEXT_LENGTH: '262144' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c8-mtp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c8-mtp.yaml index d0d2d11f4a..dc505ea367 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c8-mtp.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c8-mtp.yaml @@ -1,10 +1,6 @@ # Experimental; retain only measured improvements over the current Pareto frontier. schema: 2 name: qwen35-b200-fp8-agentx-disagg-1p1d-p-tp4-d-tp4-hicache-c8-mtp -model: - path: qwen3.5-fp8 - container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 - precision: fp8 engine: sglang dynamo: install: true @@ -142,35 +138,10 @@ srun_options: mem: '0' container-remap-root: '' telemetry: - enabled: true - collect_interval_ms: 1000 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 collector_join_timeout_seconds: 12 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 - command: "dcgm-exporter --collect-interval=100 --address :{port} -f /configs/dcgm-counters-noprof.csv" benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - concurrencies: - - 8 env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - SRT_MEASUREMENT_WINDOW_BENCHMARK_TYPE: custom - SRT_MEASUREMENT_WINDOW_CONCURRENCIES: '8' - SRT_MEASUREMENT_WINDOW_RESULT_ROOT: /logs - PORT: '8000' - IS_MULTINODE: 'true' - EVAL_FRAMEWORK: lm-eval - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k AIPERF_MAX_CONTEXT_LENGTH: '262144' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-variants.yaml index cffec07d8e..fc6f386e10 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-variants.yaml @@ -4,16 +4,10 @@ schema: 2 base: - model: - path: qwen3.5-fp8 - container: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 - precision: fp8 identity: model: - repo: Qwen/Qwen3.5-397B-A17B-FP8 revision: ea5b4f81096f3901c91dea97f81324302495781d - container: - image: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 + container: {} dynamo: install: true source: @@ -161,17 +155,8 @@ base: sbatch_directives: mem: '0' benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' ENABLE_AGENTX_POWER: '0' - IS_MULTINODE: 'true' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' @@ -196,9 +181,6 @@ override_tp4_tp4_c4: mem-fraction-static: 0.8 max-running-requests: 32 cuda-graph-max-bs-decode: 16 - benchmark: - concurrencies: - - 4 # Colocated prefill and decode with prefill HiCache. override_tp4_tp4_c12: @@ -220,9 +202,6 @@ override_tp4_tp4_c12: mem-fraction-static: 0.8 max-running-requests: 32 cuda-graph-max-bs-decode: 16 - benchmark: - concurrencies: - - 12 # Colocated TP4 prefill and decode with prefill HiCache. override_tp4_tp4_c24: @@ -244,9 +223,6 @@ override_tp4_tp4_c24: mem-fraction-static: 0.8 max-running-requests: 32 cuda-graph-max-bs-decode: 32 - benchmark: - concurrencies: - - 24 # Colocated prefill and decode with prefill HiCache. override_tp4ep4_tp4_c32: @@ -268,9 +244,6 @@ override_tp4ep4_tp4_c32: mem-fraction-static: 0.8 max-running-requests: 32 cuda-graph-max-bs-decode: 32 - benchmark: - concurrencies: - - 32 # Colocated TP2+EP2 prefill and decode with write-through HiCache. override_tp2ep2_tp2ep2_c32: @@ -280,9 +253,6 @@ override_tp2ep2_tp2ep2_c32: args: mem-fraction-static: 0.88 cuda-graph-max-bs-decode: 32 - benchmark: - concurrencies: - - 32 # Colocated TP2+EP2 prefill and decode with write-through HiCache. override_tp2ep2_tp2ep2_c40: @@ -292,9 +262,6 @@ override_tp2ep2_tp2ep2_c40: args: mem-fraction-static: 0.88 cuda-graph-max-bs-decode: 48 - benchmark: - concurrencies: - - 40 # Colocated TP2+EP2 prefill and decode with write-through HiCache. override_tp2ep2_tp2ep2_c44: @@ -304,9 +271,6 @@ override_tp2ep2_tp2ep2_c44: args: mem-fraction-static: 0.88 cuda-graph-max-bs-decode: 48 - benchmark: - concurrencies: - - 44 # Colocated TP2+EP2 prefill and decode with write-through HiCache. override_tp2ep2_tp2ep2_c48: @@ -316,9 +280,6 @@ override_tp2ep2_tp2ep2_c48: args: mem-fraction-static: 0.92 cuda-graph-max-bs-decode: 48 - benchmark: - concurrencies: - - 48 # Colocated TP2+EP2 prefill and decode with write-through HiCache. override_tp2ep2_tp2ep2_c56_replayssm: @@ -329,6 +290,3 @@ override_tp2ep2_tp2ep2_c56_replayssm: mem-fraction-static: 0.92 enable-linear-replayssm-spec: true cuda-graph-max-bs-decode: 48 - benchmark: - concurrencies: - - 56 diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-variants.yaml index 6333e2619c..84e02bc7ac 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-variants.yaml @@ -4,13 +4,9 @@ schema: 2 base: - model: - path: qwen3.5-fp4 - container: lmsysorg/sglang:nightly-dev-20260818-c0b6474b - precision: fp4 identity: - model: {repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2} - container: {image: lmsysorg/sglang:nightly-dev-20260818-c0b6474b} + model: {} + container: {} slurm: {time_limit: "8:00:00"} health_check: {max_attempts: 2160, interval_seconds: 10} resources: @@ -84,20 +80,12 @@ base: sbatch_directives: {mem: "0", cpus-per-task: "144"} srun_options: {mem: "0", container-remap-root: ""} benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" IS_MULTINODE: "false" PP_SIZE: "1" PCP_SIZE: "1" AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" AIPERF_EXTRA_INPUTS: '{"chat_template_kwargs":{"enable_thinking":true},"presence_penalty":0,"temperature":0.6,"top_k":20,"top_p":0.95}' diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-1p1d-p-tp4-d-tp4-b128-c1x2x8-mtp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-1p1d-p-tp4-d-tp4-b128-c1x2x8-mtp.yaml index 4a2d542ae7..5f557c75af 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-1p1d-p-tp4-d-tp4-b128-c1x2x8-mtp.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-1p1d-p-tp4-d-tp4-b128-c1x2x8-mtp.yaml @@ -129,15 +129,3 @@ roles: decode-log-interval: 1 stream-interval: 50 disaggregation-mode: "decode" - -telemetry: - enabled: true - collect_interval_ms: 1000 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 - command: "dcgm-exporter --collect-interval=100 --address :{port} -f /configs/dcgm-counters-noprof.csv" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-1p1d-p-tp8-ep8-d-tp8-ep8-b1024-c32x48x80-mtp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-1p1d-p-tp8-ep8-d-tp8-ep8-b1024-c32x48x80-mtp.yaml index b92942964b..7b05b74839 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-1p1d-p-tp8-ep8-d-tp8-ep8-b1024-c32x48x80-mtp.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-1p1d-p-tp8-ep8-d-tp8-ep8-b1024-c32x48x80-mtp.yaml @@ -141,15 +141,3 @@ roles: stream-interval: 50 disaggregation-mode: "decode" disaggregation-transfer-backend: "mooncake" - -telemetry: - enabled: true - collect_interval_ms: 1000 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 - command: "dcgm-exporter --collect-interval=100 --address :{port} -f /configs/dcgm-counters-noprof.csv" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-1p1d-tp4-tp4-stp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-1p1d-tp4-tp4-stp.yaml index 471d134f17..1a9ae0ecff 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-1p1d-tp4-tp4-stp.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-1p1d-tp4-tp4-stp.yaml @@ -108,15 +108,3 @@ roles: decode-log-interval: 1 stream-interval: 50 disaggregation-mode: "decode" - -telemetry: - enabled: true - collect_interval_ms: 1000 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 - command: "dcgm-exporter --collect-interval=100 --address :{port} -f /configs/dcgm-counters-noprof.csv" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-3p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b1024-c480-mtp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-3p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b1024-c480-mtp.yaml index e689e8f1d5..889d2812c2 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-3p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b1024-c480-mtp.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-3p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b1024-c480-mtp.yaml @@ -156,15 +156,3 @@ roles: decode-log-interval: 1 stream-interval: 50 - -telemetry: - enabled: true - collect_interval_ms: 1000 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 - command: "dcgm-exporter --collect-interval=100 --address :{port} -f /configs/dcgm-counters-noprof.csv" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-4p1d-dep4-dep16-stp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-4p1d-dep4-dep16-stp.yaml index b307fa8798..24cc0ccb49 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-4p1d-dep4-dep16-stp.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-4p1d-dep4-dep16-stp.yaml @@ -144,15 +144,3 @@ roles: decode-log-interval: 1 stream-interval: 50 - -telemetry: - enabled: true - collect_interval_ms: 1000 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 - command: "dcgm-exporter --collect-interval=100 --address :{port} -f /configs/dcgm-counters-noprof.csv" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-4p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b1024-c768-mtp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-4p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b1024-c768-mtp.yaml index 55f93a5205..9264dfd2fe 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-4p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b1024-c768-mtp.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-4p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b1024-c768-mtp.yaml @@ -156,15 +156,3 @@ roles: decode-log-interval: 1 stream-interval: 50 - -telemetry: - enabled: true - collect_interval_ms: 1000 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 - command: "dcgm-exporter --collect-interval=100 --address :{port} -f /configs/dcgm-counters-noprof.csv" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-6p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b1024-c1280-mtp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-6p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b1024-c1280-mtp.yaml index 3e66627afb..4cf983e4eb 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-6p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b1024-c1280-mtp.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-6p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b1024-c1280-mtp.yaml @@ -156,15 +156,3 @@ roles: decode-log-interval: 1 stream-interval: 50 - -telemetry: - enabled: true - collect_interval_ms: 1000 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 - command: "dcgm-exporter --collect-interval=100 --address :{port} -f /configs/dcgm-counters-noprof.csv" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-7p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b2048-c1344-mtp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-7p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b2048-c1344-mtp.yaml index a19f04bf9c..d655226c23 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-7p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b2048-c1344-mtp.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-7p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b2048-c1344-mtp.yaml @@ -156,15 +156,3 @@ roles: decode-log-interval: 1 stream-interval: 50 - -telemetry: - enabled: true - collect_interval_ms: 1000 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 - command: "dcgm-exporter --collect-interval=100 --address :{port} -f /configs/dcgm-counters-noprof.csv" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-8p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b2048-c1920x2304-mtp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-8p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b2048-c1920x2304-mtp.yaml index ced3d2d344..ac4c0b1889 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-8p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b2048-c1920x2304-mtp.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-8p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b2048-c1920x2304-mtp.yaml @@ -156,15 +156,3 @@ roles: decode-log-interval: 1 stream-interval: 50 - -telemetry: - enabled: true - collect_interval_ms: 1000 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 - command: "dcgm-exporter --collect-interval=100 --address :{port} -f /configs/dcgm-counters-noprof.csv" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/8k1k/disagg-1p1d-p-tp4-d-tp4-b128-c1x4x8x16x32x64x256-stp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/8k1k/disagg-1p1d-p-tp4-d-tp4-b128-c1x4x8x16x32x64x256-stp.yaml index f31452a552..8f86149755 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/8k1k/disagg-1p1d-p-tp4-d-tp4-b128-c1x4x8x16x32x64x256-stp.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/8k1k/disagg-1p1d-p-tp4-d-tp4-b128-c1x4x8x16x32x64x256-stp.yaml @@ -162,17 +162,3 @@ roles: page-size: 64 watchdog-timeout: 1000000 decode-log-interval: 50 - -telemetry: - enabled: true - collect_interval_ms: 1000 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - dcgm_exporter: - container_image: dcgm-exporter - # 9401 is already bound by the cluster-level exporter on im-gb300 nodes; - # use a port outside that range. - port: 19401 - command: "dcgm-exporter --collect-interval=100 --address :{port} -f /configs/dcgm-counters-noprof.csv" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/8k1k/disagg-5p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b4096-c2048-stp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/8k1k/disagg-5p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b4096-c2048-stp.yaml index c0d65a0fa4..bbbb3535fd 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/8k1k/disagg-5p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b4096-c2048-stp.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/8k1k/disagg-5p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b4096-c2048-stp.yaml @@ -164,15 +164,3 @@ roles: decode-log-interval: 50 stream-interval: 50 - -telemetry: - enabled: true - collect_interval_ms: 1000 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - dcgm_exporter: - container_image: dcgm-exporter - port: 19401 - command: "dcgm-exporter --collect-interval=100 --address :{port} -f /configs/dcgm-counters-noprof.csv" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/8k1k/disagg-6p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b5120-c5120-stp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/8k1k/disagg-6p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b5120-c5120-stp.yaml index fb0c170c66..ad969244d5 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/8k1k/disagg-6p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b5120-c5120-stp.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/8k1k/disagg-6p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b5120-c5120-stp.yaml @@ -162,15 +162,3 @@ roles: decode-log-interval: 50 stream-interval: 50 - -telemetry: - enabled: true - collect_interval_ms: 1000 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - dcgm_exporter: - container_image: dcgm-exporter - port: 19401 - command: "dcgm-exporter --collect-interval=100 --address :{port} -f /configs/dcgm-counters-noprof.csv" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/8k1k/disagg-7p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b5120-c5120-stp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/8k1k/disagg-7p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b5120-c5120-stp.yaml index f066c07dcc..44461a1b59 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/8k1k/disagg-7p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b5120-c5120-stp.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/8k1k/disagg-7p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b5120-c5120-stp.yaml @@ -162,15 +162,3 @@ roles: decode-log-interval: 50 stream-interval: 50 - -telemetry: - enabled: true - collect_interval_ms: 1000 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - dcgm_exporter: - container_image: dcgm-exporter - port: 19401 - command: "dcgm-exporter --collect-interval=100 --address :{port} -f /configs/dcgm-counters-noprof.csv" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-pareto-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-pareto-variants.yaml index 5ed9aaa501..21338332b6 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-pareto-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-pareto-variants.yaml @@ -4,10 +4,6 @@ schema: 2 base: - model: - path: qwen3.5-fp4 - container: lmsysorg/sglang:nightly-dev-cu13-20260831-bb5e6198 - precision: fp4 slurm: time_limit: '8:00:00' health_check: @@ -116,17 +112,9 @@ base: mem: '0' container-remap-root: '' benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' IS_MULTINODE: 'false' TP: '2' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-variants.yaml index 6c6314f3e8..b7e1ad122f 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-variants.yaml @@ -4,10 +4,6 @@ schema: 2 base: - model: - path: qwen3.5-fp4 - container: dynamo-sglang - precision: fp4 slurm: time_limit: '8:00:00' health_check: @@ -75,19 +71,11 @@ base: mem: '0' container-remap-root: '' benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' IS_MULTINODE: 'false' TP: '2' PP_SIZE: '1' PCP_SIZE: '1' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-pp-pareto-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-pp-pareto-variants.yaml index cf21ca15f3..af10c50291 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-pp-pareto-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-pp-pareto-variants.yaml @@ -4,10 +4,6 @@ schema: 2 base: - model: - path: qwen3.5-fp4 - container: lmsysorg/sglang:nightly-dev-cu13-20260831-bb5e6198 - precision: fp4 dynamo: install: true source: @@ -227,16 +223,7 @@ base: mem: '0' container-remap-root: '' benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-variants.yaml index 557e76fcb4..b162d7f147 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-variants.yaml @@ -4,10 +4,6 @@ schema: 2 base: - model: - path: qwen3.5-fp4 - container: dynamo-sglang - precision: fp4 dynamo: install: true source: @@ -194,16 +190,7 @@ base: mem: '0' container-remap-root: '' benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-1p1d-p-tp4-d-tp4-b1024-c1x2x8-mtp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-1p1d-p-tp4-d-tp4-b1024-c1x2x8-mtp.yaml index 5d1a47cc90..8cab47c7b5 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-1p1d-p-tp4-d-tp4-b1024-c1x2x8-mtp.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-1p1d-p-tp4-d-tp4-b1024-c1x2x8-mtp.yaml @@ -129,15 +129,3 @@ roles: decode-log-interval: 1 stream-interval: 50 disaggregation-mode: "decode" - -telemetry: - enabled: true - collect_interval_ms: 1000 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - dcgm_exporter: - container_image: dcgm-exporter - port: 19401 - command: "dcgm-exporter --collect-interval=100 --address :{port} -f /configs/dcgm-counters-noprof.csv" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-1p1d-p-tp8-ep8-d-tp8-ep8-b1024-c32x48x80-mtp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-1p1d-p-tp8-ep8-d-tp8-ep8-b1024-c32x48x80-mtp.yaml index feb96f5f6f..e67823d43a 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-1p1d-p-tp8-ep8-d-tp8-ep8-b1024-c32x48x80-mtp.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-1p1d-p-tp8-ep8-d-tp8-ep8-b1024-c32x48x80-mtp.yaml @@ -141,15 +141,3 @@ roles: stream-interval: 50 disaggregation-mode: "decode" disaggregation-transfer-backend: "mooncake" - -telemetry: - enabled: true - collect_interval_ms: 1000 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - dcgm_exporter: - container_image: dcgm-exporter - port: 19401 - command: "dcgm-exporter --collect-interval=100 --address :{port} -f /configs/dcgm-counters-noprof.csv" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-1p1d-tp4-tp4-stp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-1p1d-tp4-tp4-stp.yaml index 885ab17d8a..3726af8901 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-1p1d-tp4-tp4-stp.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-1p1d-tp4-tp4-stp.yaml @@ -108,17 +108,3 @@ roles: decode-log-interval: 1 stream-interval: 50 disaggregation-mode: "decode" - -telemetry: - enabled: true - collect_interval_ms: 1000 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - dcgm_exporter: - container_image: dcgm-exporter - # 9401 is already bound by the cluster-level exporter on im-gb300 nodes; - # use a port outside that range. - port: 19401 - command: "dcgm-exporter --collect-interval=100 --address :{port} -f /configs/dcgm-counters-noprof.csv" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-3p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b1024-c480-mtp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-3p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b1024-c480-mtp.yaml index df4c5a0406..c5bbfeecd0 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-3p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b1024-c480-mtp.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-3p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b1024-c480-mtp.yaml @@ -157,15 +157,3 @@ roles: decode-log-interval: 1 stream-interval: 50 - -telemetry: - enabled: true - collect_interval_ms: 1000 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - dcgm_exporter: - container_image: dcgm-exporter - port: 19401 - command: "dcgm-exporter --collect-interval=100 --address :{port} -f /configs/dcgm-counters-noprof.csv" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-4p1d-dep4-dep16-stp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-4p1d-dep4-dep16-stp.yaml index 3a6ba087e2..b56132b6d5 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-4p1d-dep4-dep16-stp.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-4p1d-dep4-dep16-stp.yaml @@ -143,15 +143,3 @@ roles: decode-log-interval: 1 stream-interval: 50 - -telemetry: - enabled: true - collect_interval_ms: 1000 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - dcgm_exporter: - container_image: dcgm-exporter - port: 19401 - command: "dcgm-exporter --collect-interval=100 --address :{port} -f /configs/dcgm-counters-noprof.csv" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-4p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b1024-c768-mtp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-4p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b1024-c768-mtp.yaml index 168cbf92a7..99c7fd8e2c 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-4p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b1024-c768-mtp.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-4p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b1024-c768-mtp.yaml @@ -157,15 +157,3 @@ roles: decode-log-interval: 1 stream-interval: 50 - -telemetry: - enabled: true - collect_interval_ms: 1000 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - dcgm_exporter: - container_image: dcgm-exporter - port: 19401 - command: "dcgm-exporter --collect-interval=100 --address :{port} -f /configs/dcgm-counters-noprof.csv" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-6p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b1024-c1280-mtp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-6p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b1024-c1280-mtp.yaml index 50e9292602..0a351e4540 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-6p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b1024-c1280-mtp.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-6p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b1024-c1280-mtp.yaml @@ -157,15 +157,3 @@ roles: decode-log-interval: 1 stream-interval: 50 - -telemetry: - enabled: true - collect_interval_ms: 1000 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - dcgm_exporter: - container_image: dcgm-exporter - port: 19401 - command: "dcgm-exporter --collect-interval=100 --address :{port} -f /configs/dcgm-counters-noprof.csv" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-7p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b2048-c1344-mtp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-7p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b2048-c1344-mtp.yaml index 971abbe239..8955f0aa2a 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-7p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b2048-c1344-mtp.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-7p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b2048-c1344-mtp.yaml @@ -157,15 +157,3 @@ roles: decode-log-interval: 1 stream-interval: 50 - -telemetry: - enabled: true - collect_interval_ms: 1000 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - dcgm_exporter: - container_image: dcgm-exporter - port: 19401 - command: "dcgm-exporter --collect-interval=100 --address :{port} -f /configs/dcgm-counters-noprof.csv" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-8p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b2048-c1920x2304-mtp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-8p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b2048-c1920x2304-mtp.yaml index 3b49fa99da..a6bc9b37ec 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-8p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b2048-c1920x2304-mtp.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-8p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b2048-c1920x2304-mtp.yaml @@ -157,15 +157,3 @@ roles: decode-log-interval: 1 stream-interval: 50 - -telemetry: - enabled: true - collect_interval_ms: 1000 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - dcgm_exporter: - container_image: dcgm-exporter - port: 19401 - command: "dcgm-exporter --collect-interval=100 --address :{port} -f /configs/dcgm-counters-noprof.csv" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/agentx/agg-gb300-tp4-c1-b1-mtp-hicache-nightly-20260913.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/agentx/agg-gb300-tp4-c1-b1-mtp-hicache-nightly-20260913.yaml index 6c98616771..ad1ac902f6 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/agentx/agg-gb300-tp4-c1-b1-mtp-hicache-nightly-20260913.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/agentx/agg-gb300-tp4-c1-b1-mtp-hicache-nightly-20260913.yaml @@ -1,9 +1,5 @@ schema: 2 name: agg-gb300-tp4-c1-b1-mtp-hicache-nightly-20260913 -model: - path: qwen3.5-fp8 - container: lmsysorg/sglang:nightly-dev-cu13-20260913-fa663e72 - precision: fp8 slurm: time_limit: '8:00:00' health_check: @@ -107,16 +103,8 @@ srun_options: mem: '0' container-remap-root: '' benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' IS_MULTINODE: 'false' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/agentx/agg-gb300-tp4-c44-b1-mtp-hicache-nightly-20260913.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/agentx/agg-gb300-tp4-c44-b1-mtp-hicache-nightly-20260913.yaml index 0b11e0470c..88f63e37f2 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/agentx/agg-gb300-tp4-c44-b1-mtp-hicache-nightly-20260913.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/agentx/agg-gb300-tp4-c44-b1-mtp-hicache-nightly-20260913.yaml @@ -1,9 +1,5 @@ schema: 2 name: agg-gb300-tp4-c44-b1-mtp-hicache-nightly-20260913 -model: - path: qwen3.5-fp8 - container: lmsysorg/sglang:nightly-dev-cu13-20260913-fa663e72 - precision: fp8 slurm: time_limit: '8:00:00' health_check: @@ -107,16 +103,8 @@ srun_options: mem: '0' container-remap-root: '' benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' IS_MULTINODE: 'false' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/agentx/agg-gb300-tp4-c44-b2-mtp-hicache-nightly-20260913.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/agentx/agg-gb300-tp4-c44-b2-mtp-hicache-nightly-20260913.yaml index 1f08bb4189..4ddfb8d3d9 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/agentx/agg-gb300-tp4-c44-b2-mtp-hicache-nightly-20260913.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/agentx/agg-gb300-tp4-c44-b2-mtp-hicache-nightly-20260913.yaml @@ -1,9 +1,5 @@ schema: 2 name: agg-gb300-tp4-c44-b2-mtp-hicache-nightly-20260913 -model: - path: qwen3.5-fp8 - container: lmsysorg/sglang:nightly-dev-cu13-20260913-fa663e72 - precision: fp8 slurm: time_limit: '8:00:00' health_check: @@ -107,16 +103,8 @@ srun_options: mem: '0' container-remap-root: '' benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' IS_MULTINODE: 'false' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/agentx/agg-gb300-tp4-c44-b4-mtp-hicache-nightly-20260913.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/agentx/agg-gb300-tp4-c44-b4-mtp-hicache-nightly-20260913.yaml index 54e91ce380..76a8bff705 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/agentx/agg-gb300-tp4-c44-b4-mtp-hicache-nightly-20260913.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/agentx/agg-gb300-tp4-c44-b4-mtp-hicache-nightly-20260913.yaml @@ -1,9 +1,5 @@ schema: 2 name: agg-gb300-tp4-c44-b4-mtp-hicache-nightly-20260913 -model: - path: qwen3.5-fp8 - container: lmsysorg/sglang:nightly-dev-cu13-20260913-fa663e72 - precision: fp8 slurm: time_limit: '8:00:00' health_check: @@ -107,16 +103,8 @@ srun_options: mem: '0' container-remap-root: '' benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' IS_MULTINODE: 'false' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/agentx/agg-gb300-tp8-c1-b1-mtp-hicache-nightly-20260913.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/agentx/agg-gb300-tp8-c1-b1-mtp-hicache-nightly-20260913.yaml index 09ba0c53ed..2acfcf3c62 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/agentx/agg-gb300-tp8-c1-b1-mtp-hicache-nightly-20260913.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/agentx/agg-gb300-tp8-c1-b1-mtp-hicache-nightly-20260913.yaml @@ -1,9 +1,5 @@ schema: 2 name: agg-gb300-tp8-c1-b1-mtp-hicache-nightly-20260913 -model: - path: qwen3.5-fp8 - container: lmsysorg/sglang:nightly-dev-cu13-20260913-fa663e72 - precision: fp8 slurm: time_limit: '8:00:00' health_check: @@ -107,16 +103,7 @@ srun_options: mem: '0' container-remap-root: '' benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/agentx/disagg-gb300-1p1d-pp4-dep4-c100-mtp-hicache-nightly-20260913.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/agentx/disagg-gb300-1p1d-pp4-dep4-c100-mtp-hicache-nightly-20260913.yaml index a3f46626c1..5b8bf79a10 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/agentx/disagg-gb300-1p1d-pp4-dep4-c100-mtp-hicache-nightly-20260913.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/agentx/disagg-gb300-1p1d-pp4-dep4-c100-mtp-hicache-nightly-20260913.yaml @@ -1,9 +1,5 @@ schema: 2 name: disagg-gb300-1p1d-pp4-dep4-c100-mtp-hicache-nightly-20260913 -model: - path: qwen3.5-fp8 - container: lmsysorg/sglang:nightly-dev-cu13-20260913-fa663e72 - precision: fp8 dynamo: install: true source: @@ -231,16 +227,7 @@ srun_options: mem: '0' container-remap-root: '' benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/agentx/disagg-gb300-1p1d-pp4-dep4-c200-mtp-hicache-nightly-20260913.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/agentx/disagg-gb300-1p1d-pp4-dep4-c200-mtp-hicache-nightly-20260913.yaml index 49d3b5113a..c2c0d332a9 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/agentx/disagg-gb300-1p1d-pp4-dep4-c200-mtp-hicache-nightly-20260913.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/agentx/disagg-gb300-1p1d-pp4-dep4-c200-mtp-hicache-nightly-20260913.yaml @@ -1,9 +1,5 @@ schema: 2 name: disagg-gb300-1p1d-pp4-dep4-c200-mtp-hicache-nightly-20260913 -model: - path: qwen3.5-fp8 - container: lmsysorg/sglang:nightly-dev-cu13-20260913-fa663e72 - precision: fp8 dynamo: install: true source: @@ -231,16 +227,7 @@ srun_options: mem: '0' container-remap-root: '' benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/agentx/disagg-gb300-3p2d-pp4-dep4-c600-mtp-hicache-nightly-20260913.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/agentx/disagg-gb300-3p2d-pp4-dep4-c600-mtp-hicache-nightly-20260913.yaml index ec3a9b7ef4..2a27950d05 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/agentx/disagg-gb300-3p2d-pp4-dep4-c600-mtp-hicache-nightly-20260913.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/agentx/disagg-gb300-3p2d-pp4-dep4-c600-mtp-hicache-nightly-20260913.yaml @@ -1,9 +1,5 @@ schema: 2 name: disagg-gb300-3p2d-pp4-dep4-c600-mtp-hicache-nightly-20260913 -model: - path: qwen3.5-fp8 - container: lmsysorg/sglang:nightly-dev-cu13-20260913-fa663e72 - precision: fp8 dynamo: install: true source: @@ -231,16 +227,7 @@ srun_options: mem: '0' container-remap-root: '' benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/agentx/disagg-gb300-5p3d-pp4-dep4-c1000-mtp-hicache-nightly-20260913.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/agentx/disagg-gb300-5p3d-pp4-dep4-c1000-mtp-hicache-nightly-20260913.yaml index d5f6575aef..adee471e86 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/agentx/disagg-gb300-5p3d-pp4-dep4-c1000-mtp-hicache-nightly-20260913.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/agentx/disagg-gb300-5p3d-pp4-dep4-c1000-mtp-hicache-nightly-20260913.yaml @@ -1,9 +1,5 @@ schema: 2 name: disagg-gb300-5p3d-pp4-dep4-c1000-mtp-hicache-nightly-20260913 -model: - path: qwen3.5-fp8 - container: lmsysorg/sglang:nightly-dev-cu13-20260913-fa663e72 - precision: fp8 dynamo: install: true source: @@ -231,16 +227,7 @@ srun_options: mem: '0' container-remap-root: '' benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-variants.yaml index ab164ce4c3..56beda647d 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-variants.yaml @@ -4,20 +4,14 @@ schema: 2 base: - model: - path: qwen3.5-fp4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - precision: fp4 dynamo: install: true source: git: https://github.com/cquil11/dynamo.git rev: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 identity: - model: - repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + model: {} + container: {} frameworks: tensorrt_llm: 1.3.0rc24 slurm: @@ -188,19 +182,10 @@ base: active-prefill-tokens-threshold: None active-prefill-tokens-threshold-frac: None benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k SERVED_MODEL_NAME: Qwen3.5-397B-A17B-NVFP4-V2 override_1p7d_dep4_tep8_c7_b1: diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv4/atom/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv4/atom/mi355x-fp4-mtp/agentic.yaml index f6dd120075..acd654392e 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv4/atom/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv4/atom/mi355x-fp4-mtp/agentic.yaml @@ -5,10 +5,6 @@ base: schema: 2 name: dsv4-fp4-mi355x-atom-agentic - model: - path: hf:deepseek-ai/DeepSeek-V4-Pro-0813 - container: rocm/atom-dev:nightly_202610071700 - precision: fp4 resources: gpu_type: mi355x gpus_per_node: 8 @@ -50,10 +46,7 @@ base: ATOM_PROFILER_MORE: '0' ATOM_PROFILER_TIMEOUT: '1200' benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - MODEL: deepseek-ai/DeepSeek-V4-Pro-0813 # Long AgentX stalls exceed AIPerf's 30 s TCP_USER_TIMEOUT default. AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' AIPERF_TIMING_CANCEL_DRAIN_TIMEOUT: '300' diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/b200-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/b200-fp4-mtp/agentic.yaml index 36bba702cb..e7cc7837db 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/b200-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/b200-fp4-mtp/agentic.yaml @@ -4,10 +4,6 @@ base: schema: 2 name: dsv4-fp4-b200-sglang-agentic - model: - path: hf:deepseek-ai/DeepSeek-V4-Pro-0813 - container: lmsysorg/sglang:v0.5.19-cu130 - precision: fp4 resources: gpu_type: b200 gpus_per_node: 8 @@ -61,10 +57,7 @@ base: # The image's first ptxas on the legacy search path. TRITON_PTXAS_PATH: /usr/local/cuda-13.0/bin/ptxas benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - MODEL: deepseek-ai/DeepSeek-V4-Pro-0813 # Warmup can leave bytes unacknowledged past AIPerf's 30 s default. AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/b300-fp4-eagle/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/b300-fp4-eagle/agentic.yaml index bdcd7c61de..b48240e55e 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/b300-fp4-eagle/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/b300-fp4-eagle/agentic.yaml @@ -5,10 +5,6 @@ base: schema: 2 name: dsv4-fp4-b300-sglang-agentic-eagle - model: - path: hf:deepseek-ai/DeepSeek-V4-Pro - container: lmsysorg/sglang:dev-cu13-nightly-09242@sha256:c47bbe7448050608b4e08669c742df4835a45a95bf6b19f2c4e51a7f6609e9d5 - precision: fp4 resources: gpu_type: b300 gpus_per_node: 8 @@ -64,10 +60,7 @@ base: # Triton compiles with the image's CUDA ptxas. TRITON_PTXAS_PATH: /usr/local/cuda/bin/ptxas benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - MODEL: deepseek-ai/DeepSeek-V4-Pro # Warmup can leave bytes unacknowledged past AIPerf's 30 s default. AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/b300-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/b300-fp4-mtp/agentic.yaml index 3beb2ef09c..0fa153fcae 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/b300-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/b300-fp4-mtp/agentic.yaml @@ -4,10 +4,6 @@ base: schema: 2 name: dsv4-fp4-b300-sglang-agentic - model: - path: hf:deepseek-ai/DeepSeek-V4-Pro-0813 - container: lmsysorg/sglang:nightly-dev-20260901-07c8f729 - precision: fp4 resources: gpu_type: b300 gpus_per_node: 8 @@ -64,10 +60,7 @@ base: # Triton compiles with the image's CUDA ptxas. TRITON_PTXAS_PATH: /usr/local/cuda/bin/ptxas benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - MODEL: deepseek-ai/DeepSeek-V4-Pro-0813 # Warmup can leave bytes unacknowledged past AIPerf's 30 s default. AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/mi355x-fp4-mtp/agentic.yaml index e0e0b24703..02726ed842 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/mi355x-fp4-mtp/agentic.yaml @@ -7,10 +7,6 @@ base: schema: 2 name: dsv4-fp4-mi355x-sglang-agentic - model: - path: hf:deepseek-ai/DeepSeek-V4-Pro-0813 - container: lmsysorg/sglang-rocm:v0.5.21-rocm720-mi35x-20261007 - precision: fp4 resources: gpu_type: mi355x gpus_per_node: 8 @@ -78,10 +74,7 @@ base: SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: '1' GPU_MAX_HW_QUEUES: '2' benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - MODEL: deepseek-ai/DeepSeek-V4-Pro-0813 # Warmup can leave bytes unacknowledged past AIPerf's 30 s default. AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/b200-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/b200-fp4-mtp/agentic.yaml index 44c6285845..e09414adf6 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/b200-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/b200-fp4-mtp/agentic.yaml @@ -4,10 +4,6 @@ base: schema: 2 name: dsv4-fp4-b200-vllm-agentic - model: - path: hf:deepseek-ai/DeepSeek-V4-Pro-0813 - container: vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-591bb95 - precision: fp4 resources: gpu_type: b200 gpus_per_node: 8 @@ -59,10 +55,7 @@ base: PYTHONNOUSERSITE: '1' VLLM_FLOAT32_MATMUL_PRECISION: high benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - MODEL: deepseek-ai/DeepSeek-V4-Pro-0813 AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' # One variant per point. Graphs capture every decode batch of up to diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/b300-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/b300-fp4-mtp/agentic.yaml index a5197bf4e8..30db70e108 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/b300-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/b300-fp4-mtp/agentic.yaml @@ -3,10 +3,6 @@ base: schema: 2 name: dsv4-fp4-b300-vllm-agentic - model: - path: hf:deepseek-ai/DeepSeek-V4-Pro-0813 - container: vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-20ebb28 - precision: fp4 resources: gpu_type: b300 gpus_per_node: 8 @@ -58,10 +54,7 @@ base: PYTHONNOUSERSITE: '1' VLLM_FLOAT32_MATMUL_PRECISION: high benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - MODEL: deepseek-ai/DeepSeek-V4-Pro-0813 AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' # Native single-node jobs mount the persistent HF hub cache here. AIPERF_DATASET_MMAP_CACHE_DIR: /mnt/hf_hub_cache/aiperf_mmap_cache diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/mi355x-fp4-mtp/agentic.yaml index 47d2f7616e..67421dbee1 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/mi355x-fp4-mtp/agentic.yaml @@ -5,10 +5,6 @@ base: schema: 2 name: dsv4-fp4-mi355x-vllm-agentic - model: - path: hf:deepseek-ai/DeepSeek-V4-Pro-0813 - container: vllm/vllm-openai-rocm:nightly-e9757321527ca1ecd514c07c1418dd2c53da3d19 - precision: fp4 resources: gpu_type: mi355x gpus_per_node: 8 @@ -72,10 +68,7 @@ base: # vLLM only clamps torch threads after weight loading. OMP_NUM_THREADS: '1' benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - MODEL: deepseek-ai/DeepSeek-V4-Pro-0813 AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/b200-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/b200-fp4-mtp/agentic.yaml index c5a02c8ff2..b9164e1ccb 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/b200-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/b200-fp4-mtp/agentic.yaml @@ -3,10 +3,6 @@ base: schema: 2 name: dsv41flash-fp4-b200-sglang-agentic - model: - path: hf:deepseek-ai/DeepSeek-V4.1-Flash - container: lmsysorg/sglang:nightly-dev-cu13-20261004-295a53c9@sha256:7cfd4dd8e0e902d2a3c0560565963c4e172091e1586aaed15ff5a0a131394d34 - precision: fp4 resources: gpu_type: b200 gpus_per_node: 8 @@ -49,10 +45,7 @@ base: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1' SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - MODEL: deepseek-ai/DeepSeek-V4.1-Flash # Warmup can leave bytes unacknowledged past AIPerf's 30 s default. AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/b300-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/b300-fp4-mtp/agentic.yaml index 9645801cc9..0dca9aae27 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/b300-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/b300-fp4-mtp/agentic.yaml @@ -1,10 +1,6 @@ base: schema: 2 name: dsv41flash-fp4-b300-sglang-agentic - model: - path: hf:deepseek-ai/DeepSeek-V4.1-Flash - container: lmsysorg/sglang:nightly-dev-cu13-20261004-295a53c9@sha256:7cfd4dd8e0e902d2a3c0560565963c4e172091e1586aaed15ff5a0a131394d34 - precision: fp4 identity: model: repo: deepseek-ai/DeepSeek-V4.1-Flash @@ -59,10 +55,7 @@ base: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1' SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - MODEL: deepseek-ai/DeepSeek-V4.1-Flash # Warmup can leave bytes unacknowledged past AIPerf's 30 s default. AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb200-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb200-fp4-mtp/agentic.yaml index 560eb34bee..184dbfc412 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb200-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb200-fp4-mtp/agentic.yaml @@ -3,10 +3,6 @@ base: schema: 2 name: dsv41flash-fp4-gb200-sglang-agentic - model: - path: hf:deepseek-ai/DeepSeek-V4.1-Flash - container: lmsysorg/sglang:nightly-dev-cu13-20260923-06008c17@sha256:5921361fcf358cdde4df1968c941c14157f418613b099ad7f3e5aeed6427ae15 - precision: fp4 resources: gpu_type: gb200 gpus_per_node: 4 @@ -50,10 +46,7 @@ base: max_attempts: 1440 interval_seconds: 5 benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - MODEL: deepseek-ai/DeepSeek-V4.1-Flash # Warmup can leave bytes unacknowledged past AIPerf's 30 s default. AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic.yaml index 3796366014..1a09f4f788 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic.yaml @@ -3,10 +3,6 @@ base: schema: 2 name: dsv41flash-fp4-gb300-sglang-agentic - model: - path: hf:deepseek-ai/DeepSeek-V4.1-Flash - container: lmsysorg/sglang:nightly-dev-cu13-20261012-e6072935@sha256:b5c488694d74c13587af6057b90220498c4869571c8e1e3c43dd6f43ae721306 - precision: fp4 resources: gpu_type: gb300 gpus_per_node: 4 @@ -57,10 +53,7 @@ base: max_attempts: 1440 interval_seconds: 5 benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - MODEL: deepseek-ai/DeepSeek-V4.1-Flash # Warmup can leave bytes unacknowledged past AIPerf's 30 s default. AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/h100-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/h100-fp4-mtp/agentic.yaml index 041fb4849f..38fa613f01 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/h100-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/h100-fp4-mtp/agentic.yaml @@ -9,10 +9,6 @@ base: schema: 2 name: dsv41flash-fp4-h100-sglang-agentic - model: - path: hf:deepseek-ai/DeepSeek-V4.1-Flash - container: lmsysorg/sglang:nightly-dev-cu13-20260922-582389ce@sha256:0e1b14e302619a42ef5581b87db6804651d4f946301cf543d74a3a1eb1c33b40 - precision: fp4 resources: gpu_type: h100 gpus_per_node: 8 @@ -73,10 +69,7 @@ base: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1' SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - MODEL: deepseek-ai/DeepSeek-V4.1-Flash # Warmup can leave bytes unacknowledged past AIPerf's 30 s default. AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/h200-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/h200-fp4-mtp/agentic.yaml index a5d86251de..c12d1e82c2 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/h200-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/h200-fp4-mtp/agentic.yaml @@ -4,10 +4,6 @@ base: schema: 2 name: dsv41flash-fp4-h200-sglang-agentic - model: - path: hf:deepseek-ai/DeepSeek-V4.1-Flash - container: lmsysorg/sglang:nightly-dev-cu13-20260923-06008c17 - precision: fp4 resources: gpu_type: h200 gpus_per_node: 8 @@ -69,10 +65,7 @@ base: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1' SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - MODEL: deepseek-ai/DeepSeek-V4.1-Flash # Warmup can leave bytes unacknowledged past AIPerf's 30 s default. AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/mi355x-fp4-mtp/agentic.yaml index c944810c36..5bac021ede 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/mi355x-fp4-mtp/agentic.yaml @@ -3,10 +3,6 @@ base: schema: 2 name: dsv41flash-fp4-mi355x-sglang-agentic - model: - path: hf:deepseek-ai/DeepSeek-V4.1-Flash - container: lmsysorg/sglang-rocm:v0.5.21-rocm720-mi35x-20261012 - precision: fp4 resources: gpu_type: mi355x gpus_per_node: 8 @@ -60,12 +56,7 @@ base: HSA_NO_SCRATCH_RECLAIM: '0' GPU_MAX_HW_QUEUES: '2' benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - MODEL: deepseek-ai/DeepSeek-V4.1-Flash - # Full-context corpus for this 1M-context recipe. - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 # Warmup can leave bytes unacknowledged past AIPerf's 30 s default. AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/b200-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/b200-fp4-mtp/agentic.yaml index 4e98740488..73b12273f6 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/b200-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/b200-fp4-mtp/agentic.yaml @@ -4,10 +4,6 @@ base: schema: 2 name: dsv41flash-fp4-b200-vllm-agentic - model: - path: hf:deepseek-ai/DeepSeek-V4.1-Flash - container: vllm/vllm-openai:nightly-dev-x86_64-cu130-ac9126e58aa7 - precision: fp4 resources: gpu_type: b200 gpus_per_node: 8 @@ -57,10 +53,7 @@ base: PYTHONUNBUFFERED: '1' VLLM_ENGINE_READY_TIMEOUT_S: '7200' benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - MODEL: deepseek-ai/DeepSeek-V4.1-Flash AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' # Native zip expansion creates one variant per concurrency. diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/b300-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/b300-fp4-mtp/agentic.yaml index 3016ebd728..08633e328d 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/b300-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/b300-fp4-mtp/agentic.yaml @@ -4,10 +4,6 @@ base: schema: 2 name: dsv41flash-fp4-b300-vllm-agentic - model: - path: hf:deepseek-ai/DeepSeek-V4.1-Flash - container: vllm/vllm-openai:nightly-dev-x86_64-cu130-ac9126e58aa7 - precision: fp4 resources: gpu_type: b300 gpus_per_node: 8 @@ -56,10 +52,7 @@ base: PYTHONUNBUFFERED: '1' VLLM_ENGINE_READY_TIMEOUT_S: '7200' benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - MODEL: deepseek-ai/DeepSeek-V4.1-Flash AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' # Native zip expansion creates one variant per concurrency. diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/gb200-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/gb200-fp4-mtp/agentic.yaml index 52dc7a14a5..8e61642c11 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/gb200-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/gb200-fp4-mtp/agentic.yaml @@ -4,10 +4,6 @@ base: schema: 2 name: dsv41flash-fp4-gb200-vllm-agentic - model: - path: hf:deepseek-ai/DeepSeek-V4.1-Flash - container: vllm/vllm-openai:nightly-dev-arm64-cu130-ac9126e58aa7 - precision: fp4 resources: gpu_type: gb200 gpus_per_node: 4 @@ -57,10 +53,7 @@ base: PYTHONUNBUFFERED: '1' VLLM_ENGINE_READY_TIMEOUT_S: '7200' benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - MODEL: deepseek-ai/DeepSeek-V4.1-Flash AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' # Native zip expansion creates one variant per concurrency. diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/gb300-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/gb300-fp4-mtp/agentic.yaml index 1d3be2acdf..216dc6edc8 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/gb300-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/gb300-fp4-mtp/agentic.yaml @@ -4,10 +4,6 @@ base: schema: 2 name: dsv41flash-fp4-gb300-vllm-agentic - model: - path: hf:deepseek-ai/DeepSeek-V4.1-Flash - container: vllm/vllm-openai:nightly-dev-arm64-cu130-ac9126e58aa7 - precision: fp4 resources: gpu_type: gb300 gpus_per_node: 4 @@ -56,10 +52,7 @@ base: PYTHONUNBUFFERED: '1' VLLM_ENGINE_READY_TIMEOUT_S: '7200' benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - MODEL: deepseek-ai/DeepSeek-V4.1-Flash AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' # Native zip expansion creates one variant per concurrency. diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/h100-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/h100-fp4-mtp/agentic.yaml index d9a4abce41..41fe46c57f 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/h100-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/h100-fp4-mtp/agentic.yaml @@ -9,10 +9,6 @@ base: schema: 2 name: dsv41flash-fp4-h100-vllm-agentic - model: - path: hf:deepseek-ai/DeepSeek-V4.1-Flash - container: vllm/vllm-openai:nightly-cd10ed6f9f6b37a8ace9cf380007e66fe12ec0c3 - precision: fp4 resources: gpu_type: h100 gpus_per_node: 8 @@ -59,10 +55,7 @@ base: PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True PYTHONUNBUFFERED: '1' benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - MODEL: deepseek-ai/DeepSeek-V4.1-Flash AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' # One variant per point. The scheduler batch is 2x CONC (AgentX subagent diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/h200-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/h200-fp4-mtp/agentic.yaml index 2818ab0c17..5659f6a4cc 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/h200-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/h200-fp4-mtp/agentic.yaml @@ -4,10 +4,6 @@ base: schema: 2 name: dsv41flash-fp4-h200-vllm-agentic - model: - path: hf:deepseek-ai/DeepSeek-V4.1-Flash - container: vllm/vllm-openai:nightly-cd10ed6f9f6b37a8ace9cf380007e66fe12ec0c3 - precision: fp4 resources: gpu_type: h200 gpus_per_node: 8 @@ -52,10 +48,7 @@ base: VLLM_USE_RUST_FRONTEND: '1' PYTHONUNBUFFERED: '1' benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - MODEL: deepseek-ai/DeepSeek-V4.1-Flash AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' # One variant per point. Graph capture starts at 64 tokens (DSpark5 verification diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi300x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi300x-fp4-mtp/agentic.yaml index f2f1956224..95138e7833 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi300x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi300x-fp4-mtp/agentic.yaml @@ -5,10 +5,6 @@ base: schema: 2 name: dsv41flash-fp4-mi300x-vllm-agentic - model: - path: hf:deepseek-ai/DeepSeek-V4.1-Flash - container: vllm/vllm-openai-rocm:nightly-rocm100-3df4ae153eb385e27b52f26c81f8edb9e20b9984 - precision: fp4 resources: gpu_type: mi300x gpus_per_node: 8 @@ -71,12 +67,7 @@ base: VLLM_USE_RUST_FRONTEND: '1' PYTHONUNBUFFERED: '1' benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - MODEL: deepseek-ai/DeepSeek-V4.1-Flash - # Full-context corpus for this 1M-context recipe. - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' # One variant per point. There is no TP2 arm: a 192 GB card measured negative KV diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi325x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi325x-fp4-mtp/agentic.yaml index ec9b792ede..1caab526b6 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi325x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi325x-fp4-mtp/agentic.yaml @@ -5,10 +5,6 @@ base: schema: 2 name: dsv41flash-fp4-mi325x-vllm-agentic - model: - path: hf:deepseek-ai/DeepSeek-V4.1-Flash - container: vllm/vllm-openai-rocm:nightly-rocm100-3df4ae153eb385e27b52f26c81f8edb9e20b9984 - precision: fp4 resources: gpu_type: mi325x gpus_per_node: 8 @@ -71,12 +67,7 @@ base: VLLM_USE_RUST_FRONTEND: '1' PYTHONUNBUFFERED: '1' benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - MODEL: deepseek-ai/DeepSeek-V4.1-Flash - # Full-context corpus for this 1M-context recipe. - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' # One variant per point. TP2 leaves ~145 GiB of offloaded weights per rank, so it diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi355x-fp4-mtp/agentic.yaml index a7d2e49174..c422d72df2 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi355x-fp4-mtp/agentic.yaml @@ -4,13 +4,6 @@ base: schema: 2 name: dsv41flash-fp4-mi355x-vllm-agentic - model: - path: hf:deepseek-ai/DeepSeek-V4.1-Flash - # ROCm 10.0 nightly of 2026-10-13, the first to carry both - # vllm-project/vllm#59039 (decoder SWA bounded replay on ROCm) and #61082 - # (mono decode on all 40 backbone layers, switched by --monokernel). - container: vllm/vllm-openai-rocm:nightly-rocm100-5555a2d8f7116b2b4877f66516f4128176cf722a - precision: fp4 resources: gpu_type: mi355x gpus_per_node: 8 @@ -76,12 +69,7 @@ base: VLLM_USE_RUST_FRONTEND: '1' PYTHONUNBUFFERED: '1' benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - MODEL: deepseek-ai/DeepSeek-V4.1-Flash - # Full-context corpus for this 1M-context recipe. - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' # One variant per point, concurrency 1 through 64 on both TP arms (c128 and diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4-mtp/agentic.yaml index 5c6f598100..c0c3ce8802 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4-mtp/agentic.yaml @@ -4,10 +4,6 @@ base: schema: 2 name: glm5.2-fp4-mi355x-atom-agentic - model: - path: hf:amd/GLM-5.2-MXFP4 - container: rocm/atom-dev:nightly_202609211553 - precision: fp4 resources: gpu_type: mi355x gpus_per_node: 8 @@ -41,10 +37,7 @@ base: AITER_QUICK_REDUCE_QUANTIZATION: INT4 AITER_USE_FLYDSL_MOE_SORTING: '1' benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - MODEL: amd/GLM-5.2-MXFP4 AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'atom:' AIPERF_APPLY_CHAT_TEMPLATE: 'true' diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4-mtp/agentic.yaml index 8add44971e..b3489e9a61 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4-mtp/agentic.yaml @@ -4,10 +4,6 @@ base: schema: 2 name: glm5.2-fp4-b200-sglang-agentic - model: - path: hf:nvidia/GLM-5.2-NVFP4 - container: lmsysorg/sglang:nightly-dev-cu13-20260901-07c8f729 - precision: fp4 resources: gpu_type: b200 gpus_per_node: 8 @@ -58,10 +54,7 @@ base: # Outlast AIPerf's pooled keep-alive connections. SGLANG_TIMEOUT_KEEP_ALIVE: '900' benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - MODEL: nvidia/GLM-5.2-NVFP4 # Warmup can leave bytes unacknowledged past AIPerf's 30 s default. AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b200-fp8-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b200-fp8-mtp/agentic.yaml index 5aa940cc33..d9e4bc84a4 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b200-fp8-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b200-fp8-mtp/agentic.yaml @@ -4,10 +4,6 @@ base: schema: 2 name: glm5.2-fp8-b200-sglang-agentic - model: - path: hf:zai-org/GLM-5.2-FP8 - container: lmsysorg/sglang:nightly-dev-cu13-20260908-20ca564b - precision: fp8 resources: gpu_type: b200 gpus_per_node: 8 @@ -58,10 +54,7 @@ base: # Outlast AIPerf's pooled keep-alive connections. SGLANG_TIMEOUT_KEEP_ALIVE: '900' benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - MODEL: zai-org/GLM-5.2-FP8 # Warmup can leave bytes unacknowledged past AIPerf's 30 s default. AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b300-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b300-fp4-mtp/agentic.yaml index 621dd5438a..2950811b02 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b300-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b300-fp4-mtp/agentic.yaml @@ -5,10 +5,6 @@ base: schema: 2 name: glm5.2-fp4-b300-sglang-agentic - model: - path: hf:nvidia/GLM-5.2-NVFP4 - container: lmsysorg/sglang:nightly-dev-cu13-20260901-07c8f729 - precision: fp4 resources: gpu_type: b300 gpus_per_node: 8 @@ -58,10 +54,7 @@ base: # Outlast AIPerf's pooled connections past Uvicorn's keep-alive. SGLANG_TIMEOUT_KEEP_ALIVE: '900' benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - MODEL: nvidia/GLM-5.2-NVFP4 # Warmup can leave bytes unacknowledged past AIPerf's 30 s default. AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b300-fp8-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b300-fp8-mtp/agentic.yaml index 00a9aec092..223a56c14d 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b300-fp8-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b300-fp8-mtp/agentic.yaml @@ -5,10 +5,6 @@ base: schema: 2 name: glm5.2-fp8-b300-sglang-agentic - model: - path: hf:zai-org/GLM-5.2-FP8 - container: lmsysorg/sglang@sha256:2d53500cb72184020c59fe16aab3512d53b1aa8ee6195bfb2aaf5855d58099df - precision: fp8 resources: gpu_type: b300 gpus_per_node: 8 @@ -60,10 +56,7 @@ base: # Keep FlashInfer, Triton and JIT caches allocation-local. SGLANG_CACHE_DIR: /tmp/sglang-cache benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - MODEL: zai-org/GLM-5.2-FP8 # Warmup can leave bytes unacknowledged past AIPerf's 30 s default. AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/mi325x-fp8-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/mi325x-fp8-mtp/agentic.yaml index f329216a2d..5ec8404933 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/mi325x-fp8-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/mi325x-fp8-mtp/agentic.yaml @@ -3,10 +3,6 @@ base: schema: 2 name: glm5.2-fp8-mi325x-sglang-agentic - model: - path: hf:zai-org/GLM-5.2-FP8 - container: lmsysorg/sglang:v0.5.19-rocm720-mi30x - precision: fp8 resources: gpu_type: mi325x gpus_per_node: 8 @@ -58,12 +54,7 @@ base: SGLANG_DSA_FUSE_TOPK: 'false' SGLANG_OPT_USE_TOPK_V2: 'false' benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - MODEL: zai-org/GLM-5.2-FP8 - # GLM-5.2 natively supports 1M context, so replay the complete corpus. - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' # Long responses admitted near the end of the measurement window can take diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/mi355x-fp4-mtp/agentic.yaml index 06ef42e6a0..a323fc2127 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/mi355x-fp4-mtp/agentic.yaml @@ -2,10 +2,6 @@ base: schema: 2 name: glm5.2-fp4-mi355x-sglang-agentic - model: - path: hf:amd/GLM-5.2-MXFP4 - container: lmsysorg/sglang-rocm:v0.5.20-rocm720-mi35x-20260930 - precision: fp4 resources: gpu_type: mi355x gpus_per_node: 8 @@ -48,10 +44,7 @@ base: # The DSA indexer uses the top-k v2 kernel. SGLANG_OPT_USE_TOPK_V2: 'true' benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - MODEL: amd/GLM-5.2-MXFP4 # Warmup can leave bytes unacknowledged past AIPerf's 30 s default. AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/mi355x-fp8-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/mi355x-fp8-mtp/agentic.yaml index 28a4b3cc63..9059951aa5 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/mi355x-fp8-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/mi355x-fp8-mtp/agentic.yaml @@ -3,10 +3,6 @@ base: schema: 2 name: glm5.2-fp8-mi355x-sglang-agentic - model: - path: hf:zai-org/GLM-5.2-FP8 - container: lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260917 - precision: fp8 resources: gpu_type: mi355x gpus_per_node: 8 @@ -49,10 +45,7 @@ base: # The DSA indexer uses the precompiled HIP Top-K op. SGLANG_OPT_USE_TOPK_V2: 'false' benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - MODEL: zai-org/GLM-5.2-FP8 # Warmup can leave bytes unacknowledged past AIPerf's 30 s default. AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3/atom/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3/atom/mi355x-fp4-mtp/agentic.yaml index 6f40b5636a..a161ec1850 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3/atom/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3/atom/mi355x-fp4-mtp/agentic.yaml @@ -8,10 +8,6 @@ base: schema: 2 name: kimik3-fp4-mi355x-atom-agentic - model: - path: hf:moonshotai/Kimi-K3 - container: rocm/atom-dev:nightly_202609251613 - precision: fp4 resources: gpu_type: mi355x gpus_per_node: 8 @@ -73,10 +69,7 @@ base: # FlyDSL FP8 prefill attention; ATOM defaults it off. ATOM_USE_FLYDSL_FP8_PREFILL_ATTN: '1' benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - MODEL: moonshotai/Kimi-K3 HTTP_PROXY: '' HTTPS_PROXY: '' http_proxy: '' diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3/vllm/b300-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3/vllm/b300-fp4-mtp/agentic.yaml index e8d5e7039b..015ae6930b 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3/vllm/b300-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3/vllm/b300-fp4-mtp/agentic.yaml @@ -4,10 +4,6 @@ base: schema: 2 name: kimik3-fp4-b300-vllm-agentic - model: - path: hf:moonshotai/Kimi-K3 - container: vllm/vllm-openai:nightly-dev-x86_64-cu13-3696c77 - precision: fp4 resources: gpu_type: b300 gpus_per_node: 8 @@ -99,10 +95,7 @@ base: WITH_NVIDIA_PEERMEM: '0' VLLM_MOONCAKE_LOAD_RECV_THREADS: '4' benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - MODEL: moonshotai/Kimi-K3 AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' KV_OFFLOADING: dram diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3/vllm/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3/vllm/mi355x-fp4-mtp/agentic.yaml index e7d2c47351..429f4b8265 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3/vllm/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3/vllm/mi355x-fp4-mtp/agentic.yaml @@ -9,10 +9,6 @@ base: schema: 2 name: kimik3-fp4-mi355x-vllm-agentic - model: - path: hf:moonshotai/Kimi-K3 - container: vllm/vllm-openai-rocm:nightly-rocm100-81198e97ba7eee2a22540caaa756b7fdddcb4d93 - precision: fp4 resources: gpu_type: mi355x gpus_per_node: 8 @@ -72,10 +68,7 @@ base: # Route DCP multi-token verify through the AITER ASM round-robin decode. VLLM_ROCM_AITER_MLA_DCP_VERIFY: asm benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - MODEL: moonshotai/Kimi-K3 # Long agentic turns against a 1M context are prefill-bound on the server. AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/atom/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/atom/mi355x-fp4-mtp/agentic.yaml index 60520959b6..080eb0bf9e 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/atom/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/atom/mi355x-fp4-mtp/agentic.yaml @@ -5,10 +5,6 @@ base: schema: 2 name: minimaxm3-fp4-mi355x-atom-agentic - model: - path: hf:amd/MiniMax-M3-MXFP4 - container: rocm/atom-dev:nightly_202610070333 - precision: fp4 resources: gpu_type: mi355x gpus_per_node: 8 @@ -50,10 +46,7 @@ base: ATOM_PA_FLYDSL: '1' ATOM_PA_FLYDSL_PLAN: '1' benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - MODEL: amd/MiniMax-M3-MXFP4 AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'atom:' AIPERF_APPLY_CHAT_TEMPLATE: 'true' diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b200-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b200-fp4-mtp/agentic.yaml index f48684350d..78ce051d4f 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b200-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b200-fp4-mtp/agentic.yaml @@ -4,10 +4,6 @@ base: schema: 2 name: minimaxm3-fp4-b200-trtllm-agentic - model: - path: hf:nvidia/MiniMax-M3-NVFP4 - container: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc23.post1 - precision: fp4 resources: gpu_type: b200 gpus_per_node: 8 @@ -96,10 +92,7 @@ base: TRTLLM_SERVE_ENABLE_MSGSPEC: '1' TRTLLM_TORCH_COMPILE_CONTEXT_ONLY: '1' benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - MODEL: nvidia/MiniMax-M3-NVFP4 AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization KV_OFFLOADING: dram diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b300-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b300-fp4-mtp/agentic.yaml index 8f2a74bbdd..a82f9802f8 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b300-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b300-fp4-mtp/agentic.yaml @@ -3,10 +3,6 @@ base: schema: 2 name: minimaxm3-fp4-b300-trtllm-agentic - model: - path: hf:nvidia/MiniMax-M3-NVFP4 - container: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc23.post1 - precision: fp4 resources: gpu_type: b300 gpus_per_node: 8 @@ -93,10 +89,7 @@ base: max_attempts: 1440 interval_seconds: 5 benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - MODEL: nvidia/MiniMax-M3-NVFP4 # trtllm-serve exposes Prometheus text at /prometheus/metrics. AIPERF_SERVER_METRICS_URLS: http://localhost:8000/prometheus/metrics AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b200-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b200-fp4-mtp/agentic.yaml index c176e47fd5..b73b19c951 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b200-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b200-fp4-mtp/agentic.yaml @@ -3,10 +3,6 @@ base: schema: 2 name: minimaxm3-fp4-b200-vllm-agentic - model: - path: hf:nvidia/MiniMax-M3-NVFP4 - container: vllm/vllm-openai:nightly-af7f9488c2210d67e1033ecdc845b087ee7fe92b - precision: fp4 resources: gpu_type: b200 gpus_per_node: 8 @@ -55,10 +51,7 @@ base: VLLM_FLOAT32_MATMUL_PRECISION: high VLLM_FLASHINFER_ALLREDUCE_BACKEND: trtllm benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - MODEL: nvidia/MiniMax-M3-NVFP4 AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' # One variant per point. DRAM points give SimpleCPUOffload the whole host budget diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4-mtp/agentic.yaml index 5abfebead9..fff694c9d5 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4-mtp/agentic.yaml @@ -3,10 +3,6 @@ base: schema: 2 name: minimaxm3-fp4-b300-vllm-agentic - model: - path: hf:nvidia/MiniMax-M3-NVFP4 - container: vllm/vllm-openai:nightly-5555a2d8f7116b2b4877f66516f4128176cf722a@sha256:e5c9e6bb82f645e8fca04e4d395743ee5ed473a62b98942d65f83c35d0c69301 - precision: fp4 resources: gpu_type: b300 gpus_per_node: 8 @@ -54,10 +50,7 @@ base: VLLM_FLOAT32_MATMUL_PRECISION: high VLLM_FLASHINFER_ALLREDUCE_BACKEND: trtllm benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - MODEL: nvidia/MiniMax-M3-NVFP4 AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' # One variant per point. DRAM points give SimpleCPUOffload the whole host budget diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/h100-fp8-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/h100-fp8-mtp/agentic.yaml index dc2aa8f058..e4dfe115f0 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/h100-fp8-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/h100-fp8-mtp/agentic.yaml @@ -3,10 +3,6 @@ base: schema: 2 name: minimaxm3-fp8-h100-vllm-agentic - model: - path: hf:MiniMaxAI/MiniMax-M3-MXFP8 - container: vllm/vllm-openai:nightly-ddd6fbca148a867aad1fcab7ec72f582b9977db4 - precision: fp8 resources: gpu_type: h100 gpus_per_node: 8 @@ -56,12 +52,8 @@ base: PYTHONNOUSERSITE: '1' VLLM_ENGINE_READY_TIMEOUT_S: '7200' benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - MODEL: MiniMaxAI/MiniMax-M3-MXFP8 AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 # One variant per point. The scheduler batch is 2x CONC and graph capture covers # it at four tokens per sequence (three drafts). diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/h200-fp8-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/h200-fp8-mtp/agentic.yaml index ae9ff17d5f..f244d9ae5f 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/h200-fp8-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/h200-fp8-mtp/agentic.yaml @@ -3,10 +3,6 @@ base: schema: 2 name: minimaxm3-fp8-h200-vllm-agentic - model: - path: hf:MiniMaxAI/MiniMax-M3-MXFP8 - container: vllm/vllm-openai:nightly-1a001d584244bd21829a5e5bd4a376ea141c0ce3 - precision: fp8 resources: gpu_type: h200 gpus_per_node: 8 @@ -61,12 +57,8 @@ base: # Atomic reduction for the Marlin MXFP8 MoE path. VLLM_MARLIN_USE_ATOMIC_ADD: '1' benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - MODEL: MiniMaxAI/MiniMax-M3-MXFP8 AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 # One variant per point. The scheduler batch is 2x CONC and graph capture covers # it at four tokens per sequence (three drafts). diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/mi300x-fp8-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/mi300x-fp8-mtp/agentic.yaml index ad1cb5bd46..f6ef776145 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/mi300x-fp8-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/mi300x-fp8-mtp/agentic.yaml @@ -3,10 +3,6 @@ base: schema: 2 name: minimaxm3-fp8-mi300x-vllm-agentic - model: - path: hf:MiniMaxAI/MiniMax-M3-MXFP8 - container: vllm/vllm-openai-rocm:v0.30.0 - precision: fp8 resources: gpu_type: mi300x gpus_per_node: 8 @@ -63,10 +59,7 @@ base: NCCL_MIN_NCHANNELS: '112' GPU_MAX_HW_QUEUES: '2' benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - MODEL: MiniMaxAI/MiniMax-M3-MXFP8 AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' AIPERF_APPLY_CHAT_TEMPLATE: 'true' diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/mi325x-fp8-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/mi325x-fp8-mtp/agentic.yaml index 43e25d9eec..1789ac13f1 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/mi325x-fp8-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/mi325x-fp8-mtp/agentic.yaml @@ -3,10 +3,6 @@ base: schema: 2 name: minimaxm3-fp8-mi325x-vllm-agentic - model: - path: hf:MiniMaxAI/MiniMax-M3-MXFP8 - container: vllm/vllm-openai-rocm:v0.27.1 - precision: fp8 resources: gpu_type: mi325x gpus_per_node: 8 @@ -55,10 +51,7 @@ base: VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: '1800' VLLM_USE_BREAKABLE_CUDAGRAPH: '0' benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - MODEL: MiniMaxAI/MiniMax-M3-MXFP8 AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' AIPERF_APPLY_CHAT_TEMPLATE: 'true' diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp4-mtp/agentic.yaml index f31bfa382b..f37d7c0cfb 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp4-mtp/agentic.yaml @@ -3,10 +3,6 @@ base: schema: 2 name: qwen3.5-fp4-b200-sglang-agentic - model: - path: hf:nvidia/Qwen3.5-397B-A17B-NVFP4 - container: lmsysorg/sglang:v0.5.16-cu130 - precision: fp4 resources: gpu_type: b200 gpus_per_node: 8 @@ -56,12 +52,7 @@ base: # Outlast AIPerf's 300 s client connection pool. SGLANG_TIMEOUT_KEEP_ALIVE: '1800' benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - MODEL: nvidia/Qwen3.5-397B-A17B-NVFP4 - # 256k-capped corpus, within this model's context. - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' # One variant per point. Admission is 2x CONC and the graph batch is CONC, diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8-mtp/agentic.yaml index 817cebae91..8bc4c43b5c 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8-mtp/agentic.yaml @@ -3,10 +3,6 @@ base: schema: 2 name: qwen3.5-fp8-b200-sglang-agentic - model: - path: hf:Qwen/Qwen3.5-397B-A17B-FP8 - container: lmsysorg/sglang:nightly-dev-cu13-20260914-4358a161 - precision: fp8 resources: gpu_type: b200 gpus_per_node: 8 @@ -57,12 +53,7 @@ base: # Outlast AIPerf's 300 s client connection pool. SGLANG_TIMEOUT_KEEP_ALIVE: '1800' benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - MODEL: Qwen/Qwen3.5-397B-A17B-FP8 - # 256k-capped corpus, within this model's context. - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' # One variant per point. Admission is 2x CONC and the decode graph batch is diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4-mtp/agentic.yaml index 0c2d2370cd..fd6be3598c 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4-mtp/agentic.yaml @@ -2,10 +2,6 @@ base: schema: 2 name: qwen3.5-fp4-b300-sglang-agentic - model: - path: hf:nvidia/Qwen3.5-397B-A17B-NVFP4 - container: lmsysorg/sglang:v0.5.16-cu130 - precision: fp4 resources: gpu_type: b300 gpus_per_node: 8 @@ -55,13 +51,8 @@ base: # Outlast AIPerf's 300 s client connection pool. SGLANG_TIMEOUT_KEEP_ALIVE: '1800' benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - MODEL: nvidia/Qwen3.5-397B-A17B-NVFP4 AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' - # 256k-capped corpus, within this model's context. - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k # One variant per point. Admission is 2x CONC and the graph batch is CONC, # capped at 64. TP4 tokenizes with six workers to keep 256k warmups inside the diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8-mtp/agentic.yaml index 274f545f90..25bf4e09ea 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8-mtp/agentic.yaml @@ -2,10 +2,6 @@ base: schema: 2 name: qwen3.5-fp8-b300-sglang-agentic - model: - path: hf:Qwen/Qwen3.5-397B-A17B-FP8 - container: lmsysorg/sglang:v0.5.16-cu130 - precision: fp8 resources: gpu_type: b300 gpus_per_node: 8 @@ -54,13 +50,8 @@ base: # Outlast AIPerf's 300 s client connection pool. SGLANG_TIMEOUT_KEEP_ALIVE: '1800' benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - MODEL: Qwen/Qwen3.5-397B-A17B-FP8 AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' - # 256k-capped corpus, within this model's context. - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k # One variant per point. Admission is 2x CONC and the graph batch is CONC, # capped at 64. TP4 tokenizes with six workers to keep 256k warmups inside the diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h100-fp8-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h100-fp8-mtp/agentic.yaml index f9bf8e5180..2f9305071c 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h100-fp8-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h100-fp8-mtp/agentic.yaml @@ -3,10 +3,6 @@ base: schema: 2 name: qwen3.5-fp8-h100-sglang-agentic-mtp - model: - path: hf:Qwen/Qwen3.5-397B-A17B-FP8 - container: lmsysorg/sglang:nightly-dev-cu13-20260914-4358a161 - precision: fp8 resources: gpu_type: h100 gpus_per_node: 8 @@ -47,14 +43,6 @@ base: env: PYTHONNOUSERSITE: '1' SGLANG_ENABLE_SPEC_V2: '1' - benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - MODEL: Qwen/Qwen3.5-397B-A17B-FP8 - # H100 caps max_model_len at 131k; the 256k-capped with-subagents corpus - # keeps the rejection rate far below the unfiltered ~1M-token corpus. - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k override_gpu_kv: benchmark: diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h100-fp8/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h100-fp8/agentic.yaml index e5dbb259a0..599975ef43 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h100-fp8/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h100-fp8/agentic.yaml @@ -3,10 +3,6 @@ base: schema: 2 name: qwen3.5-fp8-h100-sglang-agentic - model: - path: hf:Qwen/Qwen3.5-397B-A17B-FP8 - container: lmsysorg/sglang:v0.5.12-cu130 - precision: fp8 resources: gpu_type: h100 gpus_per_node: 8 @@ -44,14 +40,6 @@ base: enable-metrics: true env: PYTHONNOUSERSITE: '1' - benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - MODEL: Qwen/Qwen3.5-397B-A17B-FP8 - # H100 caps max_model_len at 131k; the 256k-capped with-subagents corpus - # keeps the rejection rate far below the unfiltered ~1M-token corpus. - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k override_gpu_kv: benchmark: diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h200-fp8-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h200-fp8-mtp/agentic.yaml index 67e3146165..1c9b68eb88 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h200-fp8-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h200-fp8-mtp/agentic.yaml @@ -3,10 +3,6 @@ base: schema: 2 name: qwen3.5-fp8-h200-sglang-agentic - model: - path: hf:Qwen/Qwen3.5-397B-A17B-FP8 - container: lmsysorg/sglang:v0.5.19-cu130@sha256:d6e7288627be8b02be88e4bba38e73f6d50e2826869f753c13a4c4385ab3eda9 - precision: fp8 resources: gpu_type: h200 gpus_per_node: 8 @@ -46,14 +42,6 @@ base: env: PYTHONNOUSERSITE: '1' SGLANG_ENABLE_SPEC_V2: '1' - benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - MODEL: Qwen/Qwen3.5-397B-A17B-FP8 - # 256k-capped with-subagents corpus: the unfiltered corpus has requests up - # to ~1M tokens, beyond this model's TP8 context. - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k override_gpu_kv: benchmark: @@ -77,8 +65,6 @@ override_hicache: TOTAL_CPU_DRAM_GB: '1234' # The same HiCache arm with the experts TP-sharded (EP1) on a later nightly. override_hicache_ep1: - model: - container: lmsysorg/sglang:nightly-dev-cu13-20260916-c9a8fba9 roles: agg: args: diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi300x-fp8-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi300x-fp8-mtp/agentic.yaml index 3c3cded953..1061c47218 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi300x-fp8-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi300x-fp8-mtp/agentic.yaml @@ -3,10 +3,6 @@ base: schema: 2 name: qwen3.5-fp8-mi300x-sglang-agentic - model: - path: hf:Qwen/Qwen3.5-397B-A17B-FP8 - container: lmsysorg/sglang-rocm:v0.5.16-rocm720-mi30x-20260730 - precision: fp8 resources: gpu_type: mi300x gpus_per_node: 8 @@ -54,12 +50,7 @@ base: SGLANG_ENABLE_SPEC_V2: '1' SGLANG_TIMEOUT_KEEP_ALIVE: '1800' benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - MODEL: Qwen/Qwen3.5-397B-A17B-FP8 - # 256k-capped corpus: the unfiltered corpus exceeds this model's context. - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' # One variant per point. AgentX concurrency counts live session trees, so diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi325x-fp8-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi325x-fp8-mtp/agentic.yaml index 96e1d7b329..b67c8ebb9c 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi325x-fp8-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi325x-fp8-mtp/agentic.yaml @@ -3,10 +3,6 @@ base: schema: 2 name: qwen3.5-fp8-mi325x-sglang-agentic - model: - path: hf:Qwen/Qwen3.5-397B-A17B-FP8 - container: lmsysorg/sglang:v0.5.16-rocm720-mi30x - precision: fp8 resources: gpu_type: mi325x gpus_per_node: 8 @@ -58,12 +54,7 @@ base: SGLANG_USE_AITER_UNIFIED_ATTN: '1' SGLANG_TIMEOUT_KEEP_ALIVE: '1800' benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - MODEL: Qwen/Qwen3.5-397B-A17B-FP8 - # 256k-capped corpus: the unfiltered corpus exceeds this model's context. - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' AIPERF_APPLY_CHAT_TEMPLATE: 'true' diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4-mtp/agentic.yaml index 3617049091..23283b17e9 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4-mtp/agentic.yaml @@ -3,10 +3,6 @@ base: schema: 2 name: qwen3.5-fp4-mi355x-sglang-agentic - model: - path: hf:amd/Qwen3.5-397B-A17B-MXFP4-AttnFP8-V2 - container: lmsysorg/sglang-rocm:v0.5.20-rocm720-mi35x-20260927 - precision: fp4 resources: gpu_type: mi355x gpus_per_node: 8 @@ -55,12 +51,7 @@ base: ROCM_QUICK_REDUCE_QUANTIZATION: INT4 SGLANG_TIMEOUT_KEEP_ALIVE: '1800' benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - MODEL: amd/Qwen3.5-397B-A17B-MXFP4-AttnFP8-V2 - # 256k-capped corpus within this model's context. - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' AIPERF_APPLY_CHAT_TEMPLATE: 'true' diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp8-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp8-mtp/agentic.yaml index 8c86408daf..fbadb24839 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp8-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp8-mtp/agentic.yaml @@ -3,10 +3,6 @@ base: schema: 2 name: qwen3.5-fp8-mi355x-sglang-agentic - model: - path: hf:Qwen/Qwen3.5-397B-A17B-FP8 - container: lmsysorg/sglang-rocm:v0.5.20-rocm720-mi35x-20260929 - precision: fp8 resources: gpu_type: mi355x gpus_per_node: 8 @@ -60,12 +56,7 @@ base: # Outlast AIPerf's 300 s client connection pool. SGLANG_TIMEOUT_KEEP_ALIVE: '1800' benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - MODEL: Qwen/Qwen3.5-397B-A17B-FP8 - # 256k-capped corpus within this model's context. - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' AIPERF_APPLY_CHAT_TEMPLATE: 'true' diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/b200-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/b200-fp4-mtp/agentic.yaml index f355e0b994..537ae89a32 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/b200-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/b200-fp4-mtp/agentic.yaml @@ -3,10 +3,6 @@ base: schema: 2 name: qwen3.8next-fp4-b200-sglang-agentic - model: - path: hf:RadixArk/Qwen3.8-Flash-Next-NVFP4 - container: lmsysorg/sglang:qwen38flashnext - precision: fp4 resources: gpu_type: b200 gpus_per_node: 8 @@ -53,12 +49,7 @@ base: # Outlast AIPerf's 300 s client connection pool. SGLANG_TIMEOUT_KEEP_ALIVE: '1800' benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - MODEL: RadixArk/Qwen3.8-Flash-Next-NVFP4 - # 256k-capped corpus, within this model's context. - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' # One variant per point. NEXTN resets an unset admission cap to 48, so it is diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/b300-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/b300-fp4-mtp/agentic.yaml index 52879777ed..c3530d6798 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/b300-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/b300-fp4-mtp/agentic.yaml @@ -3,10 +3,6 @@ base: schema: 2 name: qwen3.8next-fp4-b300-sglang-agentic - model: - path: hf:RadixArk/Qwen3.8-Flash-Next-NVFP4 - container: lmsysorg/sglang:qwen38flashnext - precision: fp4 resources: gpu_type: b300 gpus_per_node: 8 @@ -60,13 +56,8 @@ base: max_attempts: 1440 interval_seconds: 5 benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - MODEL: RadixArk/Qwen3.8-Flash-Next-NVFP4 AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' - # 256k-capped corpus, within this model's context. - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k # One variant per point: admission is 2x CONC and the graph batch is CONC. NEXTN # resets an unset admission limit to 48, so it stays explicit. diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/h200-fp8-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/h200-fp8-mtp/agentic.yaml index e5786b73e5..ee529a4cb8 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/h200-fp8-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/h200-fp8-mtp/agentic.yaml @@ -2,10 +2,6 @@ base: schema: 2 name: qwen3.8next-fp8-h200-sglang-agentic - model: - path: hf:Qwen/Qwen3.8-Flash-Next-FP8 - container: lmsysorg/sglang:qwen38flashnext - precision: fp8 resources: gpu_type: h200 gpus_per_node: 8 @@ -45,13 +41,6 @@ base: env: PYTHONNOUSERSITE: '1' SGLANG_ENABLE_SPEC_V2: '1' - benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh - env: - MODEL: Qwen/Qwen3.8-Flash-Next-FP8 - # 256k-capped with-subagents corpus, within this model's context. - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k # One variant per point: admission is 2x CONC and the graph batch is CONC. override_c1: diff --git a/inferencex-e2e/configs/CONFIGS.md b/inferencex-e2e/configs/CONFIGS.md index 6a163537c2..dbdc40abbb 100644 --- a/inferencex-e2e/configs/CONFIGS.md +++ b/inferencex-e2e/configs/CONFIGS.md @@ -144,7 +144,8 @@ jobs to 3600 seconds. Reusable workflow callers may override the `duration` input. Notes: -- The fields above are the common ones, not the full schema. The Pydantic models in [`infx/matrix/validation.py`](../infx/matrix/validation.py) are the authoritative contract; they also accept fields such as `spec-decoding`, `num-nodes`, and `require-power`, and they reject any field they do not define, which fails matrix generation. +- The fields above are the common ones, not the full schema. The Pydantic models in [`infx/matrix/validation.py`](../infx/matrix/validation.py) are the authoritative contract; they also accept fields such as `spec-decoding`, `num-nodes`, `require-power` and `power`, and they reject any field they do not define, which fails matrix generation. +- A multinode search-space entry with `power: true` measures DCGM power: its rows carry `power: true`, the launcher merges the shared telemetry block into the recipe and rejects lanes without a power rule. - srt-slurm entries require `srt-recipe-dir`. Every row requires `srt-recipe: [:]` under it (selector: `base`, `override_` or `zip_override_[]`). Multinode rows may add `eval-srt-recipe`. Validation rejects missing files or variants, and `CONFIG_FILE`/`EVAL_CONFIG_FILE` settings. Only `llmd-vllm` keeps `CONFIG_FILE`. Generated rows carry the path from `inferencex-e2e/`. - Setting the fields above only guarantees that their values are passed as environment variables to benchmark scripts. Single-node jobs receive `PP_SIZE`, `DCP_SIZE`, and `PCP_SIZE`. Multinode jobs receive `PREFILL_PP_SIZE`, `PREFILL_DCP_SIZE`, `PREFILL_PCP_SIZE`, `DECODE_PP_SIZE`, `DECODE_DCP_SIZE`, and `DECODE_PCP_SIZE`. Actually using those variables is an implementation detail of the benchmark Bash script. @@ -242,7 +243,9 @@ schema; unknown keys fail. import step. Without `slurm.squash`, every job starts from the registry image, which Pyxis imports inside the job. - `slurm.srt-slurm` is the cluster's srt-slurm profile: `volume-mounts` maps - volumes every job mounts to container paths, `mounts` does the same for host + volumes every job mounts to container paths, `agentic-volume-mounts` those multi-node + AgentX jobs add (world-writable), `power-exporter-port` is the DCGM exporter port of + power points, `mounts` does the same for host paths outside the volumes (devices), `env` is the environment of the srt-slurm launch, `container-aliases` and `nginx-aliases` name the recipe containers that resolve to the main image and to the staged nginx, and `outputs`, diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index 054647ac37..fcca9a0fae 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -1512,8 +1512,9 @@ dsv4-fp4-mi355x-sglang-agentic-mtp: # Two GPUs per server doubles the servers per node and is the layout that # decides whether DSv4.1-Flash is throughput- or interactivity-bound here. dsv41flash-fp4-mi355x-vllm-agentic-dspark: - # ROCm 10.0 nightly carrying vllm-project/vllm#59039 and #61082 (see - # agentic.yaml). + # ROCm 10.0 nightly of 2026-10-13, the first to carry both + # vllm-project/vllm#59039 (decoder SWA bounded replay on ROCm) and #61082 + # (mono decode on all 40 backbone layers, switched by --monokernel). image: vllm/vllm-openai-rocm:nightly-rocm100-5555a2d8f7116b2b4877f66516f4128176cf722a model: deepseek-ai/DeepSeek-V4.1-Flash model-prefix: dsv41flash diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index fd97e2f75a..df7b8a9418 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -1383,6 +1383,7 @@ kimik3-fp4-h200-vllm-agentic-latency: conc-list: [1, 2, 3, 4, 5, 6, 7, 8, 10, 12] num-nodes: 4 srt-recipe: agg-tp16dp2ep32-latency.yaml + power: true worker: num-worker: 2 tp: 16 @@ -1409,6 +1410,7 @@ kimik3-fp4-h200-vllm-agentic-balanced: conc-list: [1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 14, 16] num-nodes: 4 srt-recipe: agg-tp8dp4ep32-balanced.yaml + power: true worker: num-worker: 4 tp: 8 @@ -1436,6 +1438,7 @@ kimik3-fp4-h200-vllm-agentic-simple: conc-list: [8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32] num-nodes: 4 srt-recipe: agg-tp8dp4ep32-vllm-simple.yaml + power: true worker: num-worker: 4 tp: 8 @@ -1632,6 +1635,7 @@ dsv4-fp8-h200-dynamo-sglang-agentic-agg: kv-offload-backend: { name: hicache } num-nodes: 1 srt-recipe: agg-tp8-mtp-kvoffload.yaml + power: true worker: num-worker: 1 tp: 8 @@ -3970,6 +3974,7 @@ qwen3.5-fp8-gb200-dynamo-sglang: - spec-decoding: "none" conc-list: [1, 2, 4, 8, 16, 32, 64, 128] srt-recipe: disagg-1p1d-tp4-tp4-stp.yaml + power: true prefill: num-worker: 1 tp: 4 @@ -3984,6 +3989,7 @@ qwen3.5-fp8-gb200-dynamo-sglang: - spec-decoding: "none" conc-list: [1024] srt-recipe: disagg-4p1d-dep4-dep16-stp.yaml + power: true prefill: num-worker: 4 tp: 4 @@ -4078,6 +4084,7 @@ qwen3.5-fp4-gb300-dynamo-sglang: kv-p2p-transfer: mooncake conc-list: [1, 4, 8, 16, 32, 64, 256] srt-recipe: disagg-1p1d-p-tp4-d-tp4-b128-c1x4x8x16x32x64x256-stp.yaml + power: true prefill: num-worker: 1 tp: 4 @@ -4094,6 +4101,7 @@ qwen3.5-fp4-gb300-dynamo-sglang: kv-p2p-transfer: nixl conc-list: [2048] srt-recipe: disagg-5p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b4096-c2048-stp.yaml + power: true prefill: num-worker: 5 tp: 4 @@ -4110,6 +4118,7 @@ qwen3.5-fp4-gb300-dynamo-sglang: kv-p2p-transfer: mooncake conc-list: [5120] srt-recipe: disagg-6p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b5120-c5120-stp.yaml + power: true prefill: num-worker: 6 tp: 4 @@ -4126,6 +4135,7 @@ qwen3.5-fp4-gb300-dynamo-sglang: kv-p2p-transfer: mooncake conc-list: [5120] srt-recipe: disagg-7p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b5120-c5120-stp.yaml + power: true prefill: num-worker: 7 tp: 4 @@ -6374,6 +6384,7 @@ kimik3-fp4-gb200-dynamo-vllm-agentic-dspark-mooncake-dcp16-agg: conc-list: [1, 2, 4, 8, 16] num-nodes: 4 srt-recipe: agg-dcp16-dspark4-maxseq2-mooncake.yaml + power: true worker: num-worker: 1 tp: 16 @@ -6400,6 +6411,7 @@ kimik3-fp4-gb200-dynamo-vllm-agentic-mooncake-dcp16-agg: conc-list: [8, 40, 48] num-nodes: 4 srt-recipe: agg-dcp16-nospec-mooncake.yaml + power: true worker: num-worker: 1 tp: 16 @@ -6535,6 +6547,7 @@ kimik3-fp4-gb200-dynamo-vllm-agentic-dspark-mooncake-tp8pp2: conc-list: [16] num-nodes: 4 srt-recipe: agg-tp8pp2-mooncake-c16.yaml + power: true worker: num-worker: 1 tp: 8 @@ -6548,6 +6561,7 @@ kimik3-fp4-gb200-dynamo-vllm-agentic-dspark-mooncake-tp8pp2: conc-list: [32] num-nodes: 4 srt-recipe: agg-tp8pp2-mooncake-c32.yaml + power: true worker: num-worker: 1 tp: 8 @@ -6560,6 +6574,7 @@ kimik3-fp4-gb200-dynamo-vllm-agentic-dspark-mooncake-tp8pp2: conc-list: [48] num-nodes: 4 srt-recipe: agg-tp8pp2-mooncake-c48.yaml + power: true worker: num-worker: 1 tp: 8 @@ -6572,6 +6587,7 @@ kimik3-fp4-gb200-dynamo-vllm-agentic-dspark-mooncake-tp8pp2: conc-list: [72] num-nodes: 4 srt-recipe: agg-tp8pp2-mooncake-c72.yaml + power: true worker: num-worker: 1 tp: 8 @@ -6584,6 +6600,7 @@ kimik3-fp4-gb200-dynamo-vllm-agentic-dspark-mooncake-tp8pp2: conc-list: [96] num-nodes: 4 srt-recipe: agg-tp8pp2-mooncake-c96.yaml + power: true worker: num-worker: 1 tp: 8 @@ -6854,6 +6871,7 @@ qwen3.5-fp8-gb300-dynamo-sglang: - spec-decoding: "none" conc-list: [1, 2, 4, 8, 16, 32, 64, 128] srt-recipe: disagg-1p1d-tp4-tp4-stp.yaml + power: true prefill: num-worker: 1 tp: 4 @@ -6868,6 +6886,7 @@ qwen3.5-fp8-gb300-dynamo-sglang: - spec-decoding: "none" conc-list: [1024] srt-recipe: disagg-4p1d-dep4-dep16-stp.yaml + power: true prefill: num-worker: 4 tp: 4 @@ -6916,6 +6935,7 @@ glm5.2-fp8-h200-dynamo-sglang-agentic-mtp-2p2d: conc-list: [8] kv-offloading: none srt-recipe: disagg-2p2d-pcp8-tp8-dp8-mtp.yaml + power: true prefill: num-worker: 2 tp: 1 @@ -6931,6 +6951,7 @@ glm5.2-fp8-h200-dynamo-sglang-agentic-mtp-2p2d: conc-list: [12] kv-offloading: none srt-recipe: disagg-2p2d-pcp8-tp8-dp8-mtp.yaml + power: true prefill: num-worker: 2 tp: 1 @@ -6946,6 +6967,7 @@ glm5.2-fp8-h200-dynamo-sglang-agentic-mtp-2p2d: conc-list: [16] kv-offloading: none srt-recipe: disagg-2p2d-pcp8-tp8-dp8-mtp.yaml + power: true prefill: num-worker: 2 tp: 1 @@ -6982,6 +7004,7 @@ glm5.2-fp8-h200-dynamo-sglang-agentic-mtp-1p1d-hicache: kv-offloading: dram kv-offload-backend: { name: hicache } srt-recipe: disagg-1p1d-pcp8-tp8-dp8-mtp6-hicache.yaml + power: true prefill: num-worker: 1 tp: 1 @@ -6998,6 +7021,7 @@ glm5.2-fp8-h200-dynamo-sglang-agentic-mtp-1p1d-hicache: kv-offloading: dram kv-offload-backend: { name: hicache } srt-recipe: disagg-1p1d-pcp8-tp8-dp8-mtp6-hicache.yaml + power: true prefill: num-worker: 1 tp: 1 @@ -7014,6 +7038,7 @@ glm5.2-fp8-h200-dynamo-sglang-agentic-mtp-1p1d-hicache: kv-offloading: dram kv-offload-backend: { name: hicache } srt-recipe: disagg-1p1d-pcp8-tp8-dp8-mtp6-hicache.yaml + power: true prefill: num-worker: 1 tp: 1 @@ -7050,6 +7075,7 @@ glm5.2-fp8-h200-dynamo-sglang-agentic-mtp-agg: kv-offload-backend: { name: hicache } num-nodes: 1 srt-recipe: agg-h200-tp8-mtp-c2.yaml + power: true worker: num-worker: 1 tp: 8 @@ -7061,6 +7087,7 @@ glm5.2-fp8-h200-dynamo-sglang-agentic-mtp-agg: kv-offload-backend: { name: hicache } num-nodes: 1 srt-recipe: agg-h200-tp8-mtp-c4.yaml + power: true worker: num-worker: 1 tp: 8 @@ -7072,6 +7099,7 @@ glm5.2-fp8-h200-dynamo-sglang-agentic-mtp-agg: kv-offload-backend: { name: hicache } num-nodes: 1 srt-recipe: agg-h200-tp8-mtp-c8.yaml + power: true worker: num-worker: 1 tp: 8 @@ -7083,6 +7111,7 @@ glm5.2-fp8-h200-dynamo-sglang-agentic-mtp-agg: kv-offload-backend: { name: hicache } num-nodes: 1 srt-recipe: agg-h200-tp8-mtp-c12.yaml + power: true worker: num-worker: 1 tp: 8 @@ -7094,6 +7123,7 @@ glm5.2-fp8-h200-dynamo-sglang-agentic-mtp-agg: kv-offload-backend: { name: hicache } num-nodes: 1 srt-recipe: agg-h200-tp8-mtp-c16.yaml + power: true worker: num-worker: 1 tp: 8 @@ -7334,6 +7364,7 @@ glm5.2-fp4-gb200-dynamo-sglang-agentic-agg: num-nodes: 2 # Shared deployment GPU power telemetry. srt-recipe: agg.yaml + power: true worker: num-worker: 1 tp: 8 @@ -7897,6 +7928,7 @@ qwen3.5-fp8-gb200-dynamo-sglang-mtp: - spec-decoding: "mtp" conc-list: [1, 2, 8] srt-recipe: disagg-1p1d-p-tp4-d-tp4-b128-c1x2x8-mtp.yaml + power: true prefill: num-worker: 1 tp: 4 @@ -7911,6 +7943,7 @@ qwen3.5-fp8-gb200-dynamo-sglang-mtp: - spec-decoding: "mtp" conc-list: [32, 48, 80] srt-recipe: disagg-1p1d-p-tp8-ep8-d-tp8-ep8-b1024-c32x48x80-mtp.yaml + power: true prefill: num-worker: 1 tp: 8 @@ -7925,6 +7958,7 @@ qwen3.5-fp8-gb200-dynamo-sglang-mtp: - spec-decoding: "mtp" conc-list: [480] srt-recipe: disagg-3p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b1024-c480-mtp.yaml + power: true prefill: num-worker: 3 tp: 4 @@ -7939,6 +7973,7 @@ qwen3.5-fp8-gb200-dynamo-sglang-mtp: - spec-decoding: "mtp" conc-list: [768] srt-recipe: disagg-4p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b1024-c768-mtp.yaml + power: true prefill: num-worker: 4 tp: 4 @@ -7953,6 +7988,7 @@ qwen3.5-fp8-gb200-dynamo-sglang-mtp: - spec-decoding: "mtp" conc-list: [1280] srt-recipe: disagg-6p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b1024-c1280-mtp.yaml + power: true prefill: num-worker: 6 tp: 4 @@ -7967,6 +8003,7 @@ qwen3.5-fp8-gb200-dynamo-sglang-mtp: - spec-decoding: "mtp" conc-list: [1344] srt-recipe: disagg-7p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b2048-c1344-mtp.yaml + power: true prefill: num-worker: 7 tp: 4 @@ -7981,6 +8018,7 @@ qwen3.5-fp8-gb200-dynamo-sglang-mtp: - spec-decoding: "mtp" conc-list: [1920, 2304] srt-recipe: disagg-8p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b2048-c1920x2304-mtp.yaml + power: true prefill: num-worker: 8 tp: 4 @@ -8017,6 +8055,7 @@ kimik3-fp4-gb300-dynamo-vllm-agentic-dspark-mooncake-dcp8-disagg: kv-offload-backend: { name: mooncake, version: "0.3.13.post1" } conc-list: [48, 56] srt-recipe: disagg-1p1d-dcp8-dcp8-dspark4-mooncake.yaml + power: true prefill: num-worker: 1 tp: 8 @@ -8029,6 +8068,7 @@ kimik3-fp4-gb300-dynamo-vllm-agentic-dspark-mooncake-dcp8-disagg: kv-offload-backend: { name: mooncake, version: "0.3.13.post1" } conc-list: [48, 64] srt-recipe: disagg-1p2d-dcp8-dcp8-dspark4-mooncake.yaml + power: true prefill: num-worker: 1 tp: 8 @@ -8041,6 +8081,7 @@ kimik3-fp4-gb300-dynamo-vllm-agentic-dspark-mooncake-dcp8-disagg: kv-offload-backend: { name: mooncake, version: "0.3.13.post1" } conc-list: [1] srt-recipe: disagg-1p3d-dcp8-dcp8-dspark7-mooncake.yaml + power: true prefill: num-worker: 1 tp: 8 @@ -8053,6 +8094,7 @@ kimik3-fp4-gb300-dynamo-vllm-agentic-dspark-mooncake-dcp8-disagg: kv-offload-backend: { name: mooncake, version: "0.3.13.post1" } conc-list: [48] srt-recipe: disagg-1p3d-dcp8-dcp8-dspark4-mooncake.yaml + power: true prefill: num-worker: 1 tp: 8 @@ -8081,6 +8123,7 @@ kimik3-fp4-gb300-dynamo-vllm-agentic-dspark-mooncake-dcp8-agg: conc-list: [1, 4] num-nodes: 2 srt-recipe: agg-dcp8-dspark7-maxseq2-mooncake.yaml + power: true worker: num-worker: 1 tp: 8 @@ -8093,6 +8136,7 @@ kimik3-fp4-gb300-dynamo-vllm-agentic-dspark-mooncake-dcp8-agg: conc-list: [48, 52, 56] num-nodes: 2 srt-recipe: agg-dcp8-dspark4-mooncake.yaml + power: true worker: num-worker: 1 tp: 8 @@ -8133,6 +8177,7 @@ kimik3-fp4-b200-dynamo-vllm-agentic-dspark: conc-list: [1] num-nodes: 2 srt-recipe: agg-tp8pp2-mooncake-c1.yaml + power: true worker: num-worker: 1 tp: 8 @@ -8145,6 +8190,7 @@ kimik3-fp4-b200-dynamo-vllm-agentic-dspark: conc-list: [4] num-nodes: 2 srt-recipe: agg-tp8pp2-mooncake-c4.yaml + power: true worker: num-worker: 1 tp: 8 @@ -8157,6 +8203,7 @@ kimik3-fp4-b200-dynamo-vllm-agentic-dspark: conc-list: [8] num-nodes: 2 srt-recipe: agg-tp8pp2-mooncake-c8.yaml + power: true worker: num-worker: 1 tp: 8 @@ -8169,6 +8216,7 @@ kimik3-fp4-b200-dynamo-vllm-agentic-dspark: conc-list: [14] num-nodes: 2 srt-recipe: agg-tp8pp2-mooncake-c14.yaml + power: true worker: num-worker: 1 tp: 8 @@ -8181,6 +8229,7 @@ kimik3-fp4-b200-dynamo-vllm-agentic-dspark: conc-list: [24] num-nodes: 2 srt-recipe: agg-tp8pp2-mooncake-c24.yaml + power: true worker: num-worker: 1 tp: 8 @@ -8193,6 +8242,7 @@ kimik3-fp4-b200-dynamo-vllm-agentic-dspark: conc-list: [48] num-nodes: 2 srt-recipe: agg-tp8pp2-mooncake-c48.yaml + power: true worker: num-worker: 1 tp: 8 @@ -8204,6 +8254,7 @@ kimik3-fp4-b200-dynamo-vllm-agentic-dspark: conc-list: [96] num-nodes: 2 srt-recipe: agg-tp8pp2-mooncake-c96.yaml + power: true worker: num-worker: 1 tp: 8 @@ -8216,6 +8267,7 @@ kimik3-fp4-b200-dynamo-vllm-agentic-dspark: conc-list: [32] num-nodes: 2 srt-recipe: agg-tp8pp2-mooncake-c32.yaml + power: true worker: num-worker: 1 tp: 8 @@ -8228,6 +8280,7 @@ kimik3-fp4-b200-dynamo-vllm-agentic-dspark: conc-list: [56] num-nodes: 2 srt-recipe: agg-tp8pp2-mooncake-c56.yaml + power: true worker: num-worker: 1 tp: 8 @@ -8240,6 +8293,7 @@ kimik3-fp4-b200-dynamo-vllm-agentic-dspark: conc-list: [72] num-nodes: 2 srt-recipe: agg-tp8pp2-mooncake-c72.yaml + power: true worker: num-worker: 1 tp: 8 @@ -8252,6 +8306,7 @@ kimik3-fp4-b200-dynamo-vllm-agentic-dspark: conc-list: [80] num-nodes: 2 srt-recipe: agg-tp8pp2-mooncake-c80.yaml + power: true worker: num-worker: 1 tp: 8 @@ -8280,6 +8335,7 @@ qwen3.5-fp8-gb300-dynamo-sglang-mtp: - spec-decoding: "mtp" conc-list: [1, 2, 8] srt-recipe: disagg-1p1d-p-tp4-d-tp4-b1024-c1x2x8-mtp.yaml + power: true prefill: num-worker: 1 tp: 4 @@ -8294,6 +8350,7 @@ qwen3.5-fp8-gb300-dynamo-sglang-mtp: - spec-decoding: "mtp" conc-list: [32, 48, 80] srt-recipe: disagg-1p1d-p-tp8-ep8-d-tp8-ep8-b1024-c32x48x80-mtp.yaml + power: true prefill: num-worker: 1 tp: 8 @@ -8308,6 +8365,7 @@ qwen3.5-fp8-gb300-dynamo-sglang-mtp: - spec-decoding: "mtp" conc-list: [480] srt-recipe: disagg-3p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b1024-c480-mtp.yaml + power: true prefill: num-worker: 3 tp: 4 @@ -8322,6 +8380,7 @@ qwen3.5-fp8-gb300-dynamo-sglang-mtp: - spec-decoding: "mtp" conc-list: [768] srt-recipe: disagg-4p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b1024-c768-mtp.yaml + power: true prefill: num-worker: 4 tp: 4 @@ -8336,6 +8395,7 @@ qwen3.5-fp8-gb300-dynamo-sglang-mtp: - spec-decoding: "mtp" conc-list: [1280] srt-recipe: disagg-6p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b1024-c1280-mtp.yaml + power: true prefill: num-worker: 6 tp: 4 @@ -8350,6 +8410,7 @@ qwen3.5-fp8-gb300-dynamo-sglang-mtp: - spec-decoding: "mtp" conc-list: [1344] srt-recipe: disagg-7p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b2048-c1344-mtp.yaml + power: true prefill: num-worker: 7 tp: 4 @@ -8364,6 +8425,7 @@ qwen3.5-fp8-gb300-dynamo-sglang-mtp: - spec-decoding: "mtp" conc-list: [1920, 2304] srt-recipe: disagg-8p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b2048-c1920x2304-mtp.yaml + power: true prefill: num-worker: 8 tp: 4 @@ -8765,6 +8827,7 @@ qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp: conc-list: - 8 srt-recipe: disagg-1p1d-p-tp4-d-tp4-hicache-c8-mtp.yaml + power: true prefill: num-worker: 1 tp: 4 @@ -8782,6 +8845,7 @@ qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp: conc-list: - 16 srt-recipe: disagg-1p1d-p-tp4-d-tp4-hicache-c16-mtp.yaml + power: true prefill: num-worker: 1 tp: 4 @@ -8799,6 +8863,7 @@ qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp: conc-list: - 24 srt-recipe: disagg-1p1d-p-tp4-d-tp4-hicache-c24-mtp.yaml + power: true prefill: num-worker: 1 tp: 4 @@ -8816,6 +8881,7 @@ qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp: conc-list: - 32 srt-recipe: disagg-1p1d-p-tp4-d-tp4-hicache-c32-mtp.yaml + power: true prefill: num-worker: 1 tp: 4 @@ -8833,6 +8899,7 @@ qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp: conc-list: - 48 srt-recipe: disagg-1p1d-p-tp4-d-tp4-hicache-c48-write-through-mtp.yaml + power: true prefill: num-worker: 1 tp: 4 @@ -8850,6 +8917,7 @@ qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp: conc-list: - 64 srt-recipe: disagg-1p1d-p-tp4-d-tp4-hicache-c64-write-through-mtp.yaml + power: true prefill: num-worker: 1 tp: 4 diff --git a/inferencex-e2e/configs/runners.yaml b/inferencex-e2e/configs/runners.yaml index 51153f6d88..3129966952 100644 --- a/inferencex-e2e/configs/runners.yaml +++ b/inferencex-e2e/configs/runners.yaml @@ -518,6 +518,7 @@ clusters: volume-mounts: aiperf-cache: /aiperf_mmap_cache hf-hub-cache: /hf_hub_cache + power-exporter-port: 9401 b200-cw: gpus-per-node: 8 arch: x86_64 @@ -611,6 +612,10 @@ clusters: single-node-models: staged container-aliases: [dynamo-vllm, dynamo-sglang, dynamo-trtllm, sglang-v0.5.11-cu130] nginx-aliases: [nginx-sqsh] + agentic-volume-mounts: + aiperf-cache: /aiperf_mmap_cache + hf-hub-cache: /hf_hub_cache + power-exporter-port: 9401 b300-dsxe: gpus-per-node: 8 available-cpu-dram-mib: 3_977_095 @@ -668,6 +673,7 @@ clusters: single-node-models: staged container-aliases: [dynamo-trtllm, dynamo-sglang, dynamo-vllm] nginx-aliases: [nginx-sqsh] + power-exporter-port: 9401 gb200-nv: gpus-per-node: 4 available-cpu-dram-mib: 860_160 @@ -719,6 +725,10 @@ clusters: container-aliases: [dynamo-trtllm, dynamo-sglang] nginx-aliases: [nginx-sqsh] shared-run-root: /mnt/lustre01/users-public/sa-shared/gha-runs + agentic-volume-mounts: + aiperf-cache: /aiperf_mmap_cache + hf-hub-cache: /hf_hub_cache + power-exporter-port: 9401 gb300-nv: gpus-per-node: 4 available-cpu-dram-mib: 860_160 @@ -773,6 +783,7 @@ clusters: aiperf-cache: /aiperf_mmap_cache hf-hub-cache: /hf_hub_cache dynamo-wheels: /configs/dynamo-wheels + power-exporter-port: 19401 mi300x-amd: gpus-per-node: 8 available-cpu-dram-mib: 1_547_820 diff --git a/inferencex-e2e/configs/srt-recipes/agentic-multi.yaml b/inferencex-e2e/configs/srt-recipes/agentic-multi.yaml new file mode 100644 index 0000000000..ae8ca6b43e --- /dev/null +++ b/inferencex-e2e/configs/srt-recipes/agentic-multi.yaml @@ -0,0 +1,3 @@ +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh diff --git a/inferencex-e2e/configs/srt-recipes/agentic-single.yaml b/inferencex-e2e/configs/srt-recipes/agentic-single.yaml new file mode 100644 index 0000000000..ae8ca6b43e --- /dev/null +++ b/inferencex-e2e/configs/srt-recipes/agentic-single.yaml @@ -0,0 +1,3 @@ +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh diff --git a/inferencex-e2e/configs/srt-recipes/telemetry-dcgm.yaml b/inferencex-e2e/configs/srt-recipes/telemetry-dcgm.yaml new file mode 100644 index 0000000000..f6327f349f --- /dev/null +++ b/inferencex-e2e/configs/srt-recipes/telemetry-dcgm.yaml @@ -0,0 +1,12 @@ +# Merged under the fragment of a point whose master row sets `power: true`. The launcher +# adds the cluster's power-exporter-port and stages the dcgm-exporter image. +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + command: "dcgm-exporter --collect-interval=100 --address :{port} -f /configs/dcgm-counters-noprof.csv" diff --git a/inferencex-e2e/infx/clusters/slurm.py b/inferencex-e2e/infx/clusters/slurm.py index 72bafde63e..ee66733e8e 100644 --- a/inferencex-e2e/infx/clusters/slurm.py +++ b/inferencex-e2e/infx/clusters/slurm.py @@ -173,7 +173,11 @@ class SrtSlurmSettings(Record): shared_run_root: HostPath | None = Field(default=None, alias="shared-run-root") uv_cache_root: HostPath | None = Field(default=None, alias="uv-cache-root") volume_mounts: dict[str, str] = Field(default_factory=dict, alias="volume-mounts") + agentic_volume_mounts: dict[str, str] = Field( + default_factory=dict, alias="agentic-volume-mounts" + ) mounts: dict[str, str] = Field(default_factory=dict) + power_exporter_port: int | None = Field(default=None, alias="power-exporter-port", gt=0) extra: dict[str, Any] = Field(default_factory=dict) @@ -211,9 +215,18 @@ def _gres_template(cls, gres: str | None) -> str | None: @model_validator(mode="after") def _mounted_volumes_exist(self) -> Self: """srt-slurm volume mounts must name declared volumes.""" - mounted = self.srt_slurm.volume_mounts if self.srt_slurm is not None else {} - if unknown := sorted(mounted.keys() - self.volumes.keys()): - raise ValueError(f"srt-slurm.volume-mounts names unknown volumes: {unknown}") + srt = self.srt_slurm + mounts = ( + {} + if srt is None + else { + "volume-mounts": srt.volume_mounts, + "agentic-volume-mounts": srt.agentic_volume_mounts, + } + ) + for field, mounted in mounts.items(): + if unknown := sorted(mounted.keys() - self.volumes.keys()): + raise ValueError(f"srt-slurm.{field} names unknown volumes: {unknown}") return self @model_validator(mode="after") diff --git a/inferencex-e2e/infx/launch/drivers/srt/__init__.py b/inferencex-e2e/infx/launch/drivers/srt/__init__.py index bb85cf47e3..36d1d45dd8 100644 --- a/inferencex-e2e/infx/launch/drivers/srt/__init__.py +++ b/inferencex-e2e/infx/launch/drivers/srt/__init__.py @@ -19,7 +19,7 @@ from infx.launch import policy from infx.launch.backends.base import BackendError from infx.launch.backends.slurm import srtctl_job_name -from infx.launch.context import Launch +from infx.launch.context import Launch, LaunchError from infx.launch.drivers.srt import collect, config, lanes, models, power, submit from infx.launch.drivers.srt.checkout import ( checkout_dir, @@ -28,7 +28,12 @@ prepare_checkout, run_setup, ) -from infx.launch.drivers.srt.recipe import eval_overrides, prepare_recipe, staged_recipe +from infx.launch.drivers.srt.recipe import ( + eval_overrides, + prepare_recipe, + recipe_mirror_path, + staged_recipe, +) from infx.launch.drivers.srt.run import SrtRun, require, slurm_backend from infx.launch.request import ( BATCH_REENTRY_ENV, @@ -130,10 +135,12 @@ def run_multinode(launch: Launch) -> int: lanes.check_request(lane, request) srt_recipe = lanes.srt_recipe(request) staged = staged_recipe(srt_recipe) - decision = power.resolve_power(launch.cluster.id, launch.path, request) + if not recipe_mirror_path(request.workspace, srt_recipe).is_file(): + raise LaunchError(f"{srt_recipe} is not in the recipe mirror") + decision = power.decide_power(launch.cluster.id, launch.path, request) model = models.checkpoint(launch.cluster, request) served = models.served_path(launch.cluster, request, model) - model_paths = models.model_paths(launch.cluster, request, srt_recipe, served) + model_paths = models.model_paths(launch.cluster, request, served) run = SrtRun.create(launch, request, models.job_env(launch.cluster, request, served)) preflight = run.srt.preflight and not (model_paths and model and model.node_local) if request.framework == "tilert": @@ -152,15 +159,13 @@ def run_multinode(launch: Launch) -> int: infmax = compute_workspace(run, checkout, shared=shared) run.env["INFMAX_WORKSPACE"] = str(infmax) - # The binder writes a fixed-sequence recipe's telemetry concurrencies. - conc_list = request.env.get("CONC_LIST", "") if decision.dcgm and request.is_agentic else None job_name = srtctl_job_name(request.runner_name) - prepare_recipe(checkout.root, staged, job_name, run.srt.dist_timeout_s, conc_list) - recipe = staged - if not request.is_agentic: - recipe = str(checkout.root / BOUND_RECIPE) - if rc := submit.bind_recipe(run, checkout, staged, recipe): - return rc + prepare_recipe(checkout.root, staged, job_name, run.srt.dist_timeout_s) + recipe = str(checkout.root / BOUND_RECIPE) + client_env = config.agentic_client_env(run, lane) if request.is_agentic else {} + power_port = run.srt.power_exporter_port if decision.dcgm else None + if rc := submit.bind_recipe(run, checkout, staged, recipe, power_port, client_env): + return rc arguments = submit.multinode_arguments(run, lane, recipe, overrides, preflight=preflight) manifest = run.workspace / submit.MULTINODE_SUBMISSION submitted = submit.Submitted(manifest=manifest) @@ -202,11 +207,14 @@ def volume(where: str, cluster_id: str, name: str) -> None: problems.append(f"{where}: no srt-slurm.shared-run-root") if srt.default_time_limit is not None and (lane.time_limit or lane.long_time_limit): problems.append(f"{where}: its time limits are shadowed by default-time-limit") - problems.extend( - f"POWER_LANES[{cluster_id!r}, {path}]: no SRT_LANES row" - for cluster_id, path in power.POWER_LANES - if scoped(cluster_id) and (cluster_id, path) not in lanes.SRT_LANES - ) + for cluster_id, path in power.POWER_LANES: + where = f"POWER_LANES[{cluster_id!r}, {path}]" + if not scoped(cluster_id): + continue + if (cluster_id, path) not in lanes.SRT_LANES: + problems.append(f"{where}: no SRT_LANES row") + elif (srt := _srt_settings(clusters.get(cluster_id))) and srt.power_exporter_port is None: + problems.append(f"{where}: no srt-slurm.power-exporter-port") for cluster_id, overrides in models.OVERRIDES.items(): if scoped(cluster_id) and profile("OVERRIDES", cluster_id) is not None: entries = clusters[cluster_id].models.entries diff --git a/inferencex-e2e/infx/launch/drivers/srt/config.py b/inferencex-e2e/infx/launch/drivers/srt/config.py index 569156c4c2..6fade32c1a 100644 --- a/inferencex-e2e/infx/launch/drivers/srt/config.py +++ b/inferencex-e2e/infx/launch/drivers/srt/config.py @@ -184,19 +184,49 @@ def create_volume_mounts(run: SrtRun) -> None: _create_dir(volume_path(run.cluster, name)) +def _lane_volumes(run: SrtRun, lane: SrtLane) -> list[tuple[str, str | None, bool]]: + """(volume, container target, world-writable) of each volume a multi-node job adds.""" + request = run.request + volumes = ( + [(name, target, True) for name, target in run.srt.agentic_volume_mounts.items()] + if request.is_agentic + else [] + ) + volumes += [(m.volume, m.target, m.world_writable) for m in lane.mounts if m.when(request)] + return volumes + + def lane_mounts(run: SrtRun, lane: SrtLane) -> list[tuple[str, str]]: - """The (host, container) mounts the lane adds for this request; their hosts are created.""" + """The (host, container) mounts a multi-node job adds for this request; hosts are created.""" mounts: list[tuple[str, str]] = [] - for mount in lane.mounts: - if mount.when(run.request): - host = volume_path(run.cluster, mount.volume) - _create_dir(host, world_writable=mount.world_writable) - mounts.append((str(host), mount.target or str(host))) + for volume, target, world_writable in _lane_volumes(run, lane): + host = volume_path(run.cluster, volume) + _create_dir(host, world_writable=world_writable) + mounts.append((str(host), target or str(host))) if run.request.framework == "tilert": mounts.append((str(run.workspace), "/infmax-workspace")) return mounts +# Each AgentX client cache is where the job mounts one of its volumes, else a directory +# inside the container. +_CLIENT_CACHES = { + "AIPERF_DATASET_MMAP_CACHE_DIR": (("aiperf-cache",), "/aiperf_mmap_cache"), + "HF_HUB_CACHE": (("hf-hub-cache", "shared-hf-hub-cache"), "/hf_hub_cache"), +} + + +def agentic_client_env(run: SrtRun, lane: SrtLane) -> dict[str, str]: + """The result and cache paths a multi-node AgentX job hands its benchmark client.""" + targets = dict(run.srt.volume_mounts) + for volume, target, _ in _lane_volumes(run, lane): + targets[volume] = target or str(volume_path(run.cluster, volume)) + env = {"RESULT_DIR": lane.agentic_result_dir} + for name, (volumes, unmounted) in _CLIENT_CACHES.items(): + env[name] = next((targets[volume] for volume in volumes if volume in targets), unmounted) + return env + + def write_lane_config( run: SrtRun, lane: SrtLane, diff --git a/inferencex-e2e/infx/launch/drivers/srt/lanes.py b/inferencex-e2e/infx/launch/drivers/srt/lanes.py index 1f042563f5..37aa08a500 100644 --- a/inferencex-e2e/infx/launch/drivers/srt/lanes.py +++ b/inferencex-e2e/infx/launch/drivers/srt/lanes.py @@ -41,19 +41,14 @@ class SrtLane: time_limit: str | None = None long_time_limit: str | None = None long_time: Match | None = None + # Where the AgentX client writes its per-concurrency artifacts in the container. + agentic_result_dir: str = "/logs/agentic" _DYNAMO = any_of("dynamo-sglang", "dynamo-trt", "dynamo-vllm") -_AGENTIC = Match(agentic=True) -_AGENTIC_CACHES = ( - LaneMount(_AGENTIC, "aiperf-cache", "/aiperf_mmap_cache", world_writable=True), - LaneMount(_AGENTIC, "hf-hub-cache", "/hf_hub_cache", world_writable=True), -) SRT_LANES: dict[tuple[str, LaunchPath], SrtLane] = { - ("b200-nscale", LaunchPath.SRT_NATIVE): SrtLane( - mounts=_AGENTIC_CACHES, - ), + ("b200-nscale", LaunchPath.SRT_NATIVE): SrtLane(), ("b200-nscale", LaunchPath.SRT_MULTI): SrtLane( frameworks=_DYNAMO, rejects=( @@ -62,14 +57,12 @@ class SrtLane: "multinode dsv4 supports only dynamo-vllm", ), ), - mounts=_AGENTIC_CACHES, ), ("b300-dsxe", LaunchPath.SRT_MULTI): SrtLane(frameworks=_DYNAMO), ("gb200-nv", LaunchPath.SRT_MULTI): SrtLane( frameworks=_DYNAMO, setup_scripts={"dynamo-sglang": "install-torchao.sh"}, mounts=( - *_AGENTIC_CACHES, LaneMount( Match(any_of("glm5.2"), any_of("fp4"), any_of("dynamo-sglang"), agentic=True), "dynamo-wheels", @@ -110,6 +103,7 @@ class SrtLane: "roles.decode.args.ep-dispatch-algorithm", ), time_limit="01:00:00", + agentic_result_dir="/infmax-workspace/LOGS/agentic", ), } diff --git a/inferencex-e2e/infx/launch/drivers/srt/models.py b/inferencex-e2e/infx/launch/drivers/srt/models.py index 2ce3c62341..027d910fa0 100644 --- a/inferencex-e2e/infx/launch/drivers/srt/models.py +++ b/inferencex-e2e/infx/launch/drivers/srt/models.py @@ -3,8 +3,7 @@ MODEL resolves to its ``models.entries`` record, keyed by its basename or ``@``, a node-local copy first; ``OVERRIDES`` holds the exceptions. An srt-slurm job serves that checkpoint unless the point's additional-settings name a -MODEL_PATH. Every ``model.path`` alias of its recipe (anything but an ``hf:`` id or -absolute path), and the ``hf:`` a fixed-sequence recipe is bound to, maps to it. +MODEL_PATH: the ``hf:`` its recipe is bound to maps to it. """ from __future__ import annotations @@ -14,12 +13,9 @@ from pathlib import Path from typing import TYPE_CHECKING -import yaml - from infx.clusters.slurm import slurm_settings from infx.launch.context import LaunchError from infx.launch.drivers.srt.config import volume_path -from infx.launch.drivers.srt.recipe import recipe_mirror_path from infx.launch.policy import Match, any_of, point_settings if TYPE_CHECKING: @@ -91,6 +87,10 @@ class Override: Match(any_of("dsr1"), any_of("fp8"), any_of("dynamo-trt")), served_name="DeepSeek-R1-0528", ), + Override( + Match(any_of("glm5.2"), any_of("fp8"), any_of("dynamo-sglang"), agentic=True), + hub=True, + ), ), "mi355x-amds": ( # The cluster stages neither checkpoint; these lanes load the Hub snapshot. @@ -164,50 +164,18 @@ def served_path(cluster: Cluster, request: LaunchRequest, model: Checkpoint | No return None -def recipe_aliases(recipe: Path) -> set[str]: - """The aliases a recipe's ``model.path`` names, in any variant of an override bundle. - - A launch serves one MODEL, so every variant's alias maps to the same checkpoint. - """ - raw = yaml.safe_load(recipe.read_text()) - blocks = raw.values() if isinstance(raw, dict) and "base" in raw else [raw] - paths: set[object] = set() - for block in blocks: - model = block.get("model") if isinstance(block, dict) else None - value = model.get("path") if isinstance(model, dict) else None - paths.update(value if isinstance(value, list) else [value]) - return {path for path in paths if isinstance(path, str) and not path.startswith(("hf:", "/"))} - - -def model_paths( - cluster: Cluster, - request: LaunchRequest, - srt_recipe: str, - served: str | None, -) -> dict[str, str]: - """srtslurm.yaml ``model_paths``: recipe aliases and a bound ``hf:`` to ``served``.""" - recipe = recipe_mirror_path(request.workspace, srt_recipe) - if not recipe.is_file(): - raise LaunchError(f"{srt_recipe} is not in the recipe mirror: {recipe}") - aliases = recipe_aliases(recipe) - if aliases and served is None: - raise LaunchError( - f"cluster {cluster.id!r} stages no checkpoint for MODEL={request.model}, " - f"which recipe aliases {sorted(aliases)} name" - ) - # A fixed-sequence recipe is bound to hf:; only a hub lane loads it from the Hub. - bound = not request.is_agentic and not _override(cluster, request, "hub") +def model_paths(cluster: Cluster, request: LaunchRequest, served: str | None) -> dict[str, str]: + """srtslurm.yaml ``model_paths``: the bound ``hf:`` to ``served``.""" + if not request.model or _override(cluster, request, "hub"): + return {} if served is None: - if bound: + if not request.is_agentic: raise LaunchError( f"cluster {cluster.id!r} stages no checkpoint for MODEL={request.model}, " f"which the fixed-sequence recipe binds as hf:{request.model}" ) return {} - paths = dict.fromkeys(sorted(aliases), served) - if bound: - paths[f"hf:{request.model}"] = served - return paths + return {f"hf:{request.model}": served} def job_env(cluster: Cluster, request: LaunchRequest, served: str | None) -> dict[str, str]: diff --git a/inferencex-e2e/infx/launch/drivers/srt/power.py b/inferencex-e2e/infx/launch/drivers/srt/power.py index e8e372a481..4991081b5e 100644 --- a/inferencex-e2e/infx/launch/drivers/srt/power.py +++ b/inferencex-e2e/infx/launch/drivers/srt/power.py @@ -1,43 +1,23 @@ """DCGM power eligibility of the srt-slurm multi-node lanes. -A recipe asks for DCGM power in its top-level ``telemetry:`` mapping; each lane lists the -requests and recipes that may have it. +A matrix point asks for DCGM power with its master ``power`` field; each lane lists the +requests that may have it. """ from __future__ import annotations -import fnmatch from dataclasses import dataclass from typing import TYPE_CHECKING -import yaml - from infx.launch.context import LaunchError -from infx.launch.drivers.srt.recipe import RECIPES_MIRROR, recipe_mirror_path, recipe_relpath from infx.launch.policy import LaunchPath, Match, any_of if TYPE_CHECKING: - from infx.launch.request import LaunchRequest, MultiNodeRequest - -MIRROR = f"{RECIPES_MIRROR.as_posix()}/" - - -def recipe_enables_dcgm_power(text: str) -> bool: - """Whether the recipe's top-level ``telemetry`` is ``enabled: true`` with a ``dcgm_exporter``.""" - try: - recipe = yaml.safe_load(text) - except yaml.YAMLError: - return False - telemetry = recipe.get("telemetry") if isinstance(recipe, dict) else None - return ( - isinstance(telemetry, dict) - and "dcgm_exporter" in telemetry - and telemetry.get("enabled") is True - ) + from infx.launch.request import MultiNodeRequest class PowerPolicyError(LaunchError): - """The recipe enables DCGM power on a lane that does not support it.""" + """The point asks for DCGM power on a lane that does not support it.""" @dataclass(frozen=True) @@ -46,7 +26,6 @@ class PowerRule: when: Match agentx: bool - recipe_glob: str | None = None adapter: bool = False @@ -57,7 +36,6 @@ class PowerLane: rules: tuple[PowerRule, ...] error: str agentic_error: str | None = None - eval_recipe_when_eval_only: bool = False POWER_LANES: dict[tuple[str, LaunchPath], PowerLane] = { @@ -66,24 +44,21 @@ class PowerLane: PowerRule( Match(any_of("glm5.2"), any_of("fp4"), any_of("dynamo-sglang"), agentic=True), agentx=True, - recipe_glob=f"{MIRROR}glm5.2/sglang/gb200-fp4/agentx/agg.yaml", ), PowerRule( Match(any_of("kimik3"), any_of("fp4"), any_of("dynamo-vllm"), agentic=True), agentx=True, - recipe_glob=f"{MIRROR}kimik3/vllm/gb200-fp4/agentx/*", ), PowerRule(Match(frameworks=any_of("dynamo-sglang"), agentic=False), agentx=False), ), error="dcgm-power requires dynamo-sglang or the supported Kimi-K3 AgentX route", - agentic_error="AgentX dcgm-power requires the GLM-5.2 aggregate or supported Kimi-K3 recipe", + agentic_error="AgentX dcgm-power requires GLM-5.2 FP4 dynamo-sglang or Kimi-K3 FP4 dynamo-vllm", ), ("gb300-nv", LaunchPath.SRT_MULTI): PowerLane( rules=( PowerRule( Match(any_of("kimik3"), any_of("fp4"), any_of("dynamo-vllm"), agentic=True), agentx=True, - recipe_glob=f"{MIRROR}kimik3/vllm/*/agentx/*", ), PowerRule(Match(frameworks=any_of("dynamo-sglang")), agentx=False), ), @@ -120,7 +95,6 @@ class PowerLane: ), ), error="B200 nscale dcgm-power requires a supported fixed-sequence lane or Kimi-K3 AgentX vLLM", - eval_recipe_when_eval_only=True, ), ("b200-nscale", LaunchPath.SRT_MULTI): PowerLane( rules=( @@ -134,7 +108,6 @@ class PowerLane: ), ), error="B200 Nscale dcgm-power requires fixed-sequence DSV4 FP4 dynamo-vllm or Qwen3.5 FP8 AgentX dynamo-sglang", - eval_recipe_when_eval_only=True, ), ("h200-dgxc", LaunchPath.SRT_MULTI): PowerLane( rules=( @@ -166,38 +139,18 @@ class PowerDecision: NO_POWER = PowerDecision(dcgm=False, agentx=False) -def decide_power( - cluster_id: str, path: LaunchPath, *, dcgm: bool, request: LaunchRequest, recipe: str -) -> PowerDecision: - """Apply the lane's rules to the inspected ``recipe`` path when it enables dcgm. +def decide_power(cluster_id: str, path: LaunchPath, request: MultiNodeRequest) -> PowerDecision: + """Apply the lane's rules when the point's ``power`` field is on. Raises ``PowerPolicyError`` for a combination the lane does not allow. """ - lane = POWER_LANES.get((cluster_id, path)) - if not dcgm or lane is None: + if not request.power: return NO_POWER + lane = POWER_LANES.get((cluster_id, path)) + if lane is None: + raise PowerPolicyError(f"cluster {cluster_id!r} measures no DCGM power on {path}") for rule in lane.rules: - if rule.when(request) and ( - rule.recipe_glob is None or fnmatch.fnmatchcase(recipe, rule.recipe_glob) - ): + if rule.when(request): return PowerDecision(dcgm=True, agentx=rule.agentx, adapter=rule.adapter) message = lane.agentic_error if request.is_agentic and lane.agentic_error else lane.error raise PowerPolicyError(message) - - -def resolve_power(cluster_id: str, path: LaunchPath, request: MultiNodeRequest) -> PowerDecision: - """Detect dcgm in the workspace copy of the recipe the lane inspects, and decide. - - A recipe missing from the workspace stays non-power. - """ - lane = POWER_LANES.get((cluster_id, path)) - if lane is None: - return NO_POWER - srt_recipe = request.srt_recipe - if lane.eval_recipe_when_eval_only and request.eval_only and request.eval_srt_recipe: - srt_recipe = request.eval_srt_recipe - mirror = recipe_mirror_path(request.workspace, srt_recipe) - dcgm = mirror.is_file() and recipe_enables_dcgm_power(mirror.read_text()) - return decide_power( - cluster_id, path, dcgm=dcgm, request=request, recipe=recipe_relpath(srt_recipe) - ) diff --git a/inferencex-e2e/infx/launch/drivers/srt/recipe.py b/inferencex-e2e/infx/launch/drivers/srt/recipe.py index 1c5e9d75fe..d9733b7369 100644 --- a/inferencex-e2e/infx/launch/drivers/srt/recipe.py +++ b/inferencex-e2e/infx/launch/drivers/srt/recipe.py @@ -1,21 +1,17 @@ -"""The workspace recipe mirror, and the edits multi-node lanes make to the job's recipe copy. +"""The workspace recipe mirror, and the text edits multi-node lanes make to the job's copy. -Only the disposable copy staged in the job's srt-slurm checkout is edited. Text edits keep -its comments; concurrency injection rewrites it through YAML, which drops them. +Only the disposable copy staged in the job's srt-slurm checkout is edited; the edits keep +its comments. """ from __future__ import annotations import fnmatch import re -from collections.abc import Sequence from pathlib import Path from typing import TYPE_CHECKING -import yaml - from infx.launch.context import LaunchError -from infx.srt_slurm.workload import parse_concurrencies if TYPE_CHECKING: from infx.launch.drivers.srt.lanes import SrtLane @@ -80,13 +76,7 @@ def add_dist_timeout(text: str, seconds: int) -> str: return "".join(lines) -def prepare_recipe( - checkout: Path, - staged: str, - job_name: str, - dist_timeout_s: int | None, - conc_list: str | None, -) -> None: +def prepare_recipe(checkout: Path, staged: str, job_name: str, dist_timeout_s: int | None) -> None: """Edit the checkout's staged copy of the recipe for this job.""" config_path = checkout / recipe_relpath(staged) if not config_path.is_file(): @@ -95,11 +85,6 @@ def prepare_recipe( if dist_timeout_s is not None: text = add_dist_timeout(text, dist_timeout_s) config_path.write_text(text) - if conc_list is not None: - try: - inject_concurrencies(config_path, parse_concurrencies(conc_list)) - except ValueError as error: - raise LaunchError(str(error)) from error def strip_forced_acceptance(recipes: Path) -> None: @@ -124,18 +109,3 @@ def eval_overrides(recipes: Path, lane: SrtLane, request: LaunchRequest) -> list overrides += ["--unset", key] return overrides - -def inject_concurrencies(recipe_path: Path, concurrencies: Sequence[int]) -> None: - """Set the recipe's top-level ``benchmark.concurrencies``. - - Raises ``ValueError``, leaving the file untouched, for unreadable YAML or no top-level - ``benchmark`` mapping (override bundles keep theirs under ``base``). - """ - try: - recipe = yaml.safe_load(recipe_path.read_text(encoding="utf-8")) - except (OSError, yaml.YAMLError) as error: - raise ValueError(f"failed to load recipe {recipe_path}: {error}") from error - if not isinstance(recipe, dict) or not isinstance(recipe.get("benchmark"), dict): - raise ValueError(f"recipe {recipe_path} must contain a benchmark mapping") - recipe["benchmark"]["concurrencies"] = list(concurrencies) - recipe_path.write_text(yaml.safe_dump(recipe, sort_keys=False), encoding="utf-8") diff --git a/inferencex-e2e/infx/launch/drivers/srt/submit.py b/inferencex-e2e/infx/launch/drivers/srt/submit.py index c9ea7eb02f..79ef0f6b09 100644 --- a/inferencex-e2e/infx/launch/drivers/srt/submit.py +++ b/inferencex-e2e/infx/launch/drivers/srt/submit.py @@ -78,9 +78,20 @@ def bound_arguments(arguments: Path) -> tuple[str, list[str]]: return selected, runtime_args -def bind_recipe(run: SrtRun, checkout: Checkout, staged: str, output: str) -> int: - """Compose and bind the staged fixed-sequence fragment into ``output``.""" +def bind_recipe( + run: SrtRun, + checkout: Checkout, + staged: str, + output: str, + power_port: int | None, + client_env: Mapping[str, str], +) -> int: + """Compose and bind the staged fragment into ``output``, with telemetry if ``power_port``.""" bind = [str(checkout.venv / "bin/python"), "-m", "infx.srt_slurm.workload", staged, output] + if power_port is not None: + bind += ["--power-port", str(power_port)] + for name, value in client_env.items(): + bind += ["--client-env", f"{name}={value}"] return proc.run(bind, env=run.env, cwd=checkout.root).returncode diff --git a/inferencex-e2e/infx/launch/request.py b/inferencex-e2e/infx/launch/request.py index b41747cca1..88150eaa61 100644 --- a/inferencex-e2e/infx/launch/request.py +++ b/inferencex-e2e/infx/launch/request.py @@ -149,6 +149,7 @@ class MultiNodeRequest(SrtRequest): srt_recipe: str = Field(alias="SRT_RECIPE") eval_srt_recipe: str | None = Field(None, alias="EVAL_SRT_RECIPE") + power: OneFlag = Field(False, alias="POWER") class SingleNodeRequest(SrtRequest): diff --git a/inferencex-e2e/infx/matrix/generate.py b/inferencex-e2e/infx/matrix/generate.py index a2dee231c5..32fc8ab4b2 100644 --- a/inferencex-e2e/infx/matrix/generate.py +++ b/inferencex-e2e/infx/matrix/generate.py @@ -1076,6 +1076,8 @@ def _fixed_sequence_entries( } ) entry.update(srt_recipe_fields(config, benchmark)) + if benchmark.get(Fields.POWER.value): + entry[Fields.POWER.value] = True entry.update( { Fields.EXP_NAME.value: f"{model_code}_{seq_len_to_str(isl, osl)}", @@ -1178,6 +1180,7 @@ def _agentic_entries( Fields.CONC.value: conc, } ) + exp_name = multinode_agentic_exp_name(model_code, prefill, decode, conc, offload_suffix) else: entry.update( @@ -1198,6 +1201,8 @@ def _agentic_entries( + (f"_spec-{spec_decoding}" if spec_decoding != "none" else "") ) entry.update(srt_recipe_fields(config, benchmark)) + if benchmark.get(Fields.POWER.value): + entry[Fields.POWER.value] = True entry.update( { Fields.KV_OFFLOADING.value: kv_offloading, diff --git a/inferencex-e2e/infx/matrix/validation.py b/inferencex-e2e/infx/matrix/validation.py index e2f08db533..eb76b9c6c8 100644 --- a/inferencex-e2e/infx/matrix/validation.py +++ b/inferencex-e2e/infx/matrix/validation.py @@ -89,6 +89,7 @@ class Fields(Enum): NODE_COUNT = "node-count" DURATION = "duration" REQUIRE_POWER = "require-power" + POWER = "power" # Matrix entry fields CONC = "conc" @@ -295,6 +296,7 @@ class MultiNodeMatrixEntry(BaseModel): isl: int osl: int require_power: bool = Field(default=False, alias=Fields.REQUIRE_POWER.value, strict=True) + power: bool = Field(default=False, strict=True) prefill: WorkerConfig decode: WorkerConfig conc: list[int] @@ -397,6 +399,7 @@ class MultiNodeAgenticMatrixEntry(BaseModel): default=None, alias=Fields.EVAL_SRT_RECIPE.value, min_length=1 ) node_count: int = Field(alias=Fields.NODE_COUNT.value, gt=0, strict=True) + power: bool = Field(default=False, strict=True) prefill: WorkerConfig decode: WorkerConfig conc: list[int] @@ -632,6 +635,7 @@ class MultiNodeSearchSpaceEntry(BaseModel): eval_srt_recipe: RecipeReference | None = Field( default=None, alias=Fields.EVAL_SRT_RECIPE.value ) + power: bool = Field(default=False, strict=True) num_nodes: int | None = Field(default=None, alias=Fields.NUM_NODES.value, gt=0, strict=True) router: ComponentMetadata | None = None kv_p2p_transfer: str | None = Field( @@ -693,6 +697,7 @@ class AgenticCodingSearchSpaceEntry(BaseModel): eval_srt_recipe: RecipeReference | None = Field( default=None, alias=Fields.EVAL_SRT_RECIPE.value ) + power: bool = Field(default=False, strict=True) pp: int = Field(default=1, gt=0, strict=True) dcp_size: int = Field(default=1, alias=Fields.DCP_SIZE.value, gt=0, strict=True) pcp_size: int = Field(default=1, alias=Fields.PCP_SIZE.value, gt=0, strict=True) @@ -751,6 +756,10 @@ def validate_topology_fields(self) -> Self: f"Single-node agentic search-space entries must specify " f"{Fields.KV_OFFLOADING.value}" ) + if self.power: + raise ValueError( + f"Only multinode entries measure DCGM power; remove {Fields.POWER.value}" + ) _validate_tp_context_topology(self) if has_aggregate_worker or has_complete_multinode: explicitly_single_node_fields = { diff --git a/inferencex-e2e/infx/srt_slurm/single_node.py b/inferencex-e2e/infx/srt_slurm/single_node.py index 72ed966a27..7b01a39634 100644 --- a/inferencex-e2e/infx/srt_slurm/single_node.py +++ b/inferencex-e2e/infx/srt_slurm/single_node.py @@ -62,25 +62,20 @@ def select_recipe( ) -> tuple[str, dict[str, Any]]: """Resolve a matrix point to one native variant, never submit an entire sweep. - Fixed-sequence variants are composed with the shared block under ``root`` (default: - this repository) and bound first; AgentX recipes are used as written. + Variants are composed with the shared block under ``root`` (default: this + repository) and bound before they are validated. """ path, _, selector = config.partition(":") - fixed = environment["IS_AGENTIC"] == "0" - raw = ( - compose_recipe(Path(path), multinode=False, root=root or repository_root()) - if fixed - else yaml.safe_load(Path(path).read_text()) + agentic = environment["IS_AGENTIC"] == "1" + raw = compose_recipe( + Path(path), agentic=agentic, multinode=False, root=root or repository_root() ) - if not isinstance(raw, dict): - raise ValueError("Recipe must be a mapping") recipes = selected_recipes(raw, selector or None) matches = [] errors = [] for name, recipe in recipes: try: - if fixed: - recipe = bind_workload(recipe, environment, multinode=False) + recipe = bind_workload(recipe, environment, agentic=agentic, multinode=False) validate_recipe(recipe, environment) except ValueError as exc: errors.append(f"{name}: {exc}") @@ -149,7 +144,7 @@ def validate_recipe(recipe: dict[str, Any], environment: Mapping[str, str]) -> N def runtime_arguments(config: str, environment: Mapping[str, str]) -> list[str]: """Bind only runtime-owned values after validating the selected recipe.""" - _, recipe = select_recipe(config, environment) + select_recipe(config, environment) for name in ("RUN_EVAL", "EVAL_ONLY", "DP_ATTENTION"): if environment[name] not in {"true", "false"}: raise ValueError(f"{name} must be true or false") @@ -168,23 +163,14 @@ def runtime_arguments(config: str, environment: Mapping[str, str]) -> list[str]: for key, value in options.items(): overrides += ["--set", f"srun_options.{key}={json.dumps(value)}"] agentic = environment["IS_AGENTIC"] == "1" - names = [ - "CONC", - "RESULT_FILENAME", - "RUN_EVAL", - "EVAL_ONLY", - "FRAMEWORK", - ] + # The binder already wrote CONC into the selected variant. + names = ["RESULT_FILENAME", "RUN_EVAL", "EVAL_ONLY", "FRAMEWORK"] if agentic: names += ["MODEL_PREFIX", "PRECISION", "DURATION", "TP", "PP_SIZE", "PCP_SIZE"] for name in names: value = environment[name] if not value: raise ValueError(f"Missing runtime input: {name}") - # Native --set broadcasts into zip groups. CONC already matched above; - # replacing its list could collapse the selected variant's index. - if name == "CONC" and name in recipe["benchmark"]["env"]: - continue overrides += ["--set", f"benchmark.env.{name}={json.dumps(value)}"] if agentic: # The aggregated result lands where fixed-sequence results do. @@ -230,13 +216,12 @@ def main() -> None: parsed = parser.parse_args() try: if parsed.command == "prepare": - config, recipe = select_recipe(parsed.recipe, os.environ) + _, recipe = select_recipe(parsed.recipe, os.environ) arguments = runtime_arguments(parsed.recipe, os.environ) - if os.environ["IS_AGENTIC"] == "0": - # srtctl gets the bound variant, never the fragment. - config = str(parsed.output.with_name(BOUND_RECIPE)) - Path(config).write_text(yaml.safe_dump(recipe, sort_keys=False)) - parsed.output.write_bytes("\0".join([config, *arguments, ""]).encode()) + # srtctl gets the bound variant, never the fragment. + config = parsed.output.with_name(BOUND_RECIPE) + config.write_text(yaml.safe_dump(recipe, sort_keys=False)) + parsed.output.write_bytes("\0".join([str(config), *arguments, ""]).encode()) else: print("\n".join(submission_fields(parsed.manifest))) except (OSError, ValueError, KeyError, TypeError, yaml.YAMLError) as exc: diff --git a/inferencex-e2e/infx/srt_slurm/synthetic_acceptance.py b/inferencex-e2e/infx/srt_slurm/synthetic_acceptance.py index 19ba42a60d..4698cdd820 100644 --- a/inferencex-e2e/infx/srt_slurm/synthetic_acceptance.py +++ b/inferencex-e2e/infx/srt_slurm/synthetic_acceptance.py @@ -195,10 +195,21 @@ def build_overrides( return overrides +def _without_nulls(value: Any) -> Any: + if isinstance(value, dict): + return {key: _without_nulls(item) for key, item in value.items() if item is not None} + return value + + def selected_recipes( raw: dict[str, Any], selector: str | None ) -> list[tuple[str | None, dict[str, Any]]]: - """Delegate expansion to SRT, retaining its selector for each submission.""" + """Delegate expansion to SRT, retaining its selector for each submission. + + ``srtctl apply`` merges a zip variant over its base as an override, which drops the + variant's null-valued keys (``connector: null`` falls back to the engine default); + the selected zip variants match that. + """ if "base" not in raw: if selector is not None: raise ValueError("recipe selector requires an override-format recipe") @@ -206,10 +217,11 @@ def selected_recipes( from srtctl.core.config import generate_override_configs selected = generate_override_configs(raw, selector=selector) - if ( - selector == "base" - or (selector and selector.startswith("override_") and not any(c in selector for c in "*?")) - or (selector and re.fullmatch(r"zip_override_[\w-]+\[\d+\]", selector)) + zip_variant = selector and re.fullmatch(r"zip_override_[\w-]+\[\d+\]", selector) + if zip_variant: + return [(selector, _without_nulls(selected[0][1]))] + if selector == "base" or ( + selector and selector.startswith("override_") and not any(c in selector for c in "*?") ): return [(selector, selected[0][1])] keys = sorted(k for k in raw if k.startswith("override_")) + sorted( @@ -220,8 +232,10 @@ def selected_recipes( if selector is not None and not fnmatch.fnmatch(key, selector): continue for index, (_, recipe) in enumerate(generate_override_configs(raw, selector=key)): - name = f"{key}[{index}]" if key.startswith("zip_override_") else key - result.append((name, recipe)) + if key.startswith("zip_override_"): + result.append((f"{key}[{index}]", _without_nulls(recipe))) + else: + result.append((key, recipe)) return result diff --git a/inferencex-e2e/infx/srt_slurm/workload.py b/inferencex-e2e/infx/srt_slurm/workload.py index 201127f065..c0f0ef7509 100644 --- a/inferencex-e2e/infx/srt_slurm/workload.py +++ b/inferencex-e2e/infx/srt_slurm/workload.py @@ -1,7 +1,8 @@ -"""Compose fixed-sequence recipe fragments and bind the master-owned workload into them. +"""Compose srt-slurm recipe fragments and bind the master-owned workload into them. -A fragment holds only recipe-specific srt-slurm settings. Its lane's shared block is merged -under it, a variant is selected, and the binder then writes the matrix point's values. +A fragment holds only recipe-specific srt-slurm settings. Its lane's shared block, and the +DCGM telemetry block for a point that measures power, is merged under it, a variant is +selected, and the binder then writes the matrix point's and the launcher's values. """ from __future__ import annotations @@ -18,27 +19,37 @@ from infx.config import repository_root from infx.srt_slurm.synthetic_acceptance import selected_recipes, spec_parameters +# Keyed by (agentic, multinode). SHARED_BLOCKS = { - False: Path("configs/srt-recipes/fixed-sequence-single.yaml"), - True: Path("configs/srt-recipes/fixed-sequence-multi.yaml"), + (False, False): Path("configs/srt-recipes/fixed-sequence-single.yaml"), + (False, True): Path("configs/srt-recipes/fixed-sequence-multi.yaml"), + (True, False): Path("configs/srt-recipes/agentic-single.yaml"), + (True, True): Path("configs/srt-recipes/agentic-multi.yaml"), } # The bound variant srtctl gets. BOUND_RECIPE = "recipe.yaml" -# The binder writes these, or the workflow exports them to the benchmark client. -BOUND_KEYS = ( +TELEMETRY_BLOCK = Path("configs/srt-recipes/telemetry-dcgm.yaml") +# The binder writes these, the master power field owns telemetry, and the launcher hands +# the benchmark client its cache and result paths. +_POINT_KEYS = ( ("model", "path"), ("model", "container"), ("model", "precision"), ("identity", "model", "repo"), ("identity", "container", "image"), ("benchmark", "concurrencies"), - *( - ("benchmark", "env", name) - for name in ( - "MODEL", "ISL", "OSL", "CONC", "CONC_LIST", "RANDOM_RANGE_RATIO", "USE_CHAT_TEMPLATE", - ) - ), -) # fmt: skip + ("telemetry", "enabled"), + ("telemetry", "dcgm_exporter", "port"), +) +_FIXED_ENV = ("MODEL", "ISL", "OSL", "CONC", "CONC_LIST", "RANDOM_RANGE_RATIO", "USE_CHAT_TEMPLATE") +_AGENTIC_ENV = ("CONC", "CONC_LIST", "RESULT_DIR", "AGENTIC_OUTPUT_DIR", "HF_HUB_CACHE", + "HUGGINGFACE_HUB_CACHE") # fmt: skip +BOUND_ENV = { + (False, False): _FIXED_ENV, + (False, True): _FIXED_ENV, + (True, False): ("MODEL", *_AGENTIC_ENV), + (True, True): (*_AGENTIC_ENV, "AIPERF_DATASET_MMAP_CACHE_DIR"), +} def merge_blocks(shared: Mapping[str, Any], fragment: Mapping[str, Any]) -> dict[str, Any]: @@ -64,8 +75,9 @@ def _lookup(block: Any, key: tuple[str, ...]) -> tuple[bool, Any]: return True, block -def check_fragment(raw: Mapping[str, Any], source: Path, *, multinode: bool) -> None: +def check_fragment(raw: Mapping[str, Any], source: Path, *, agentic: bool, multinode: bool) -> None: """Reject a fragment that sets a bound key, even to the value the binder would write.""" + keys = (*_POINT_KEYS, *(("benchmark", "env", name) for name in BOUND_ENV[agentic, multinode])) if "base" in raw: blocks = [(name, block) for name, block in raw.items() if name != "schema"] else: @@ -74,24 +86,38 @@ def check_fragment(raw: Mapping[str, Any], source: Path, *, multinode: bool) -> for name, block in blocks: # A single-node variant may name its point, pairing that concurrency with its tuning. conc_allowed = not multinode and name not in (None, "base") - for key in BOUND_KEYS: + for key in keys: present, value = _lookup(block, key) if present and not (conc_allowed and key == ("benchmark", "env", "CONC")): found.append(f"{'.'.join(filter(None, (name, *key)))} (= {value!r})") if found: raise ValueError( - f"{source}: remove {', '.join(found)} from the fragment; " - "they are bound from the matrix point" + f"{source}: remove {', '.join(found)} from the fragment; the launcher binds them" ) -def compose_recipe(path: Path, *, multinode: bool, root: Path) -> dict[str, Any]: - """The fragment at ``path`` over its lane's shared block, under ``base`` for bundles.""" +def _block(path: Path) -> dict[str, Any]: + block = yaml.safe_load(path.read_text()) + if not isinstance(block, dict): + raise ValueError(f"{path}: shared block must be a mapping") + return block + + +def compose_recipe( + path: Path, *, agentic: bool, multinode: bool, root: Path, power_port: int | None = None +) -> dict[str, Any]: + """The fragment at ``path`` over its lane's shared block, under ``base`` for bundles. + + ``power_port`` adds the DCGM telemetry block with its exporter on that port. + """ raw = yaml.safe_load(path.read_text()) if not isinstance(raw, dict): raise ValueError(f"{path}: recipe must be a mapping") - check_fragment(raw, path, multinode=multinode) - shared = yaml.safe_load((root / SHARED_BLOCKS[multinode]).read_text()) + check_fragment(raw, path, agentic=agentic, multinode=multinode) + shared = _block(root / SHARED_BLOCKS[agentic, multinode]) + if power_port is not None: + exporter = {"telemetry": {"dcgm_exporter": {"port": power_port}}} + shared = merge_blocks(shared, merge_blocks(_block(root / TELEMETRY_BLOCK), exporter)) if "base" in raw: return {**raw, "base": merge_blocks(shared, raw["base"])} return merge_blocks(shared, raw) @@ -123,15 +149,20 @@ def parse_concurrencies(conc_list: str) -> list[int]: def bind_workload( - recipe: Mapping[str, Any], environment: Mapping[str, str], *, multinode: bool + recipe: Mapping[str, Any], + environment: Mapping[str, str], + *, + agentic: bool, + multinode: bool, + client_env: Mapping[str, str] | None = None, ) -> dict[str, Any]: - """Write the matrix point's model, image, precision, lengths and concurrency. + """Write the matrix point's model, image, precision, concurrency and, for fixed sequences, + lengths; ``client_env`` holds the launcher's benchmark client paths. Call after variant selection: binding a zip group would detach its concurrency from the tuning it pairs with. """ image, model = _required(environment, "IMAGE"), _required(environment, "MODEL") - lengths = {name: _positive(_required(environment, name), name) for name in ("ISL", "OSL")} # schema, name and model lead the written recipe. bound = {key: deepcopy(value) for key, value in recipe.items() if key in ("schema", "name")} bound["model"] = { @@ -145,10 +176,14 @@ def bind_workload( if isinstance(identity.get("model"), dict): identity["model"]["repo"] = model if isinstance(identity.get("container"), dict): - identity["container"]["image"] = image + # Identity names the registry reference, not Pyxis's registry#path spelling. + identity["container"]["image"] = image.replace("#", "/", 1) benchmark = bound.setdefault("benchmark", {}) workload = benchmark.setdefault("env", {}) - workload.update((name, str(value)) for name, value in lengths.items()) + if not agentic: + workload.update( + (name, str(_positive(_required(environment, name), name))) for name in ("ISL", "OSL") + ) if multinode: # The client reads CONC_LIST from the job environment the workflow exports. concurrencies = parse_concurrencies(_required(environment, "CONC_LIST")) @@ -156,17 +191,23 @@ def bind_workload( conc = _positive(_required(environment, "CONC"), "CONC") if "CONC" in workload and str(workload["CONC"]) != str(conc): raise ValueError(f"CONC: recipe {workload['CONC']} != point {conc}") - engine = bound["engine"] - speculates = spec_parameters( - bound["roles"]["agg"], engine["type"] if isinstance(engine, Mapping) else engine - ) - workload.update( - MODEL=model, - CONC=str(conc), - RANDOM_RANGE_RATIO=_required(environment, "RANDOM_RANGE_RATIO"), - USE_CHAT_TEMPLATE="true" if speculates else "false", - ) + workload.update(MODEL=model, CONC=str(conc)) + if not agentic: + engine = bound["engine"] + speculates = spec_parameters( + bound["roles"]["agg"], engine["type"] if isinstance(engine, Mapping) else engine + ) + workload.update( + RANDOM_RANGE_RATIO=_required(environment, "RANDOM_RANGE_RATIO"), + USE_CHAT_TEMPLATE="true" if speculates else "false", + ) concurrencies = [conc] + # A fragment that sets HF_HOME keeps its own Hugging Face cache layout. + workload.update( + (name, value) + for name, value in (client_env or {}).items() + if not (name == "HF_HUB_CACHE" and "HF_HOME" in workload) + ) # srtctl derives power-telemetry windows from benchmark.concurrencies. if (bound.get("telemetry") or {}).get("enabled") is True: benchmark["concurrencies"] = concurrencies @@ -174,25 +215,59 @@ def bind_workload( def bind_multinode( - recipe: str, environment: Mapping[str, str], *, root: Path + recipe: str, + environment: Mapping[str, str], + *, + root: Path, + power_port: int | None = None, + client_env: Mapping[str, str] | None = None, ) -> tuple[str | None, dict[str, Any]]: """Bind the one variant ``recipe`` (``fragment[:selector]``) selects; return its name too.""" + agentic = environment.get("IS_AGENTIC") == "1" path, _, selector = recipe.partition(":") - composed = compose_recipe(Path(path), multinode=True, root=root) + composed = compose_recipe( + Path(path), agentic=agentic, multinode=True, root=root, power_port=power_port + ) variants = selected_recipes(composed, selector or None) if len(variants) != 1: raise ValueError(f"{recipe} selects {len(variants)} variants, not one") name, selected = variants[0] - return name, bind_workload(selected, environment, multinode=True) + return name, bind_workload( + selected, environment, agentic=agentic, multinode=True, client_env=client_env + ) + + +def _assignment(text: str) -> tuple[str, str]: + name, separator, value = text.partition("=") + if not separator or not name: + raise argparse.ArgumentTypeError(f"expected NAME=VALUE, got {text!r}") + return name, value def main(argv: list[str] | None = None) -> None: - parser = argparse.ArgumentParser(description="Bind one multi-node fixed-sequence recipe") + parser = argparse.ArgumentParser(description="Bind one multi-node recipe variant") parser.add_argument("recipe", help="fragment[:selector]") parser.add_argument("output", type=Path) + parser.add_argument( + "--power-port", type=int, help="merge the DCGM telemetry block, its exporter on this port" + ) + parser.add_argument( + "--client-env", + type=_assignment, + action="append", + default=[], + metavar="NAME=VALUE", + help="a launcher-owned benchmark client setting", + ) args = parser.parse_args(argv) try: - _, bound = bind_multinode(args.recipe, os.environ, root=repository_root()) + _, bound = bind_multinode( + args.recipe, + os.environ, + root=repository_root(), + power_port=args.power_port, + client_env=dict(args.client_env), + ) except (OSError, ValueError, KeyError, TypeError, yaml.YAMLError) as error: parser.error(str(error)) args.output.write_text(yaml.safe_dump(bound, sort_keys=False)) diff --git a/inferencex-e2e/infx/tests/launch/test_launch_policy.py b/inferencex-e2e/infx/tests/launch/test_launch_policy.py index 6f36a58d96..5b4398a386 100644 --- a/inferencex-e2e/infx/tests/launch/test_launch_policy.py +++ b/inferencex-e2e/infx/tests/launch/test_launch_policy.py @@ -1,5 +1,4 @@ -"""Power eligibility, dcgm detection, allocation-time policy, and the tables' agreement -with the cluster inventory.""" +"""Power eligibility, allocation-time policy, and the tables' agreement with the cluster inventory.""" import json @@ -10,13 +9,8 @@ from infx.launch.__main__ import launch from infx.launch.context import LaunchError from infx.launch.drivers import check_tables -from infx.launch.drivers.srt import models -from infx.launch.drivers.srt.power import ( - PowerPolicyError, - decide_power, - recipe_enables_dcgm_power, - resolve_power, -) +from infx.launch.drivers.srt import models, power +from infx.launch.drivers.srt.power import PowerPolicyError, decide_power from infx.launch.lifecycle import Lifecycle from infx.launch.policy import LaunchPath, Match, salloc_time_limit from infx.launch.request import LaunchRequest, MultiNodeRequest @@ -29,114 +23,80 @@ def _request(**env): return LaunchRequest.from_env({"RUNNER_NAME": "r_0", **env}) -MIRROR = "benchmarks/multi_node/srt-slurm-recipes" -GB200_GLM = f"{MIRROR}/glm5.2/sglang/gb200-fp4/agentx/agg.yaml" -KIMI_GB200 = f"{MIRROR}/kimik3/vllm/gb200-fp4/agentx/k.yaml" -KIMI_GB300 = f"{MIRROR}/kimik3/vllm/gb300-fp4/agentx/deep/k.yaml" -OTHER = f"{MIRROR}/other.yaml" +def _multinode(**env): + return MultiNodeRequest.from_env({ + "RUNNER_NAME": "r_0", "IMAGE": "i", "SPEC_DECODING": "none", "RESULT_FILENAME": "r", + "RUN_EVAL": "false", "EVAL_ONLY": "false", "GITHUB_WORKSPACE": "/ws", "SRT_RECIPE": "r.yaml", + **env, + }) + CASES = [ - ("gb200-nv", MULTI, "1", "glm5.2", "fp4", "dynamo-sglang", GB200_GLM, AGENTX), - ("gb200-nv", MULTI, "1", "glm5.2", "fp4", "dynamo-sglang", OTHER, ERR), - ("gb200-nv", MULTI, "1", "kimik3", "fp4", "dynamo-vllm", KIMI_GB200, AGENTX), - ("gb200-nv", MULTI, "1", "kimik3", "fp4", "dynamo-vllm", KIMI_GB300, ERR), - ("gb200-nv", MULTI, "0", "dsr1", "fp8", "dynamo-sglang", OTHER, DCGM), - ("gb200-nv", MULTI, "0", "dsv4", "fp4", "dynamo-vllm", OTHER, ERR), - ("gb300-nv", MULTI, "1", "kimik3", "fp4", "dynamo-vllm", KIMI_GB300, AGENTX), - ("gb300-nv", MULTI, "1", "kimik3", "fp4", "dynamo-vllm", OTHER, ERR), - ("gb300-nv", MULTI, "1", "glm5.2", "fp4", "dynamo-sglang", OTHER, DCGM), - ("gb300-nv", MULTI, "0", "qwen3.5", "fp8", "dynamo-trt", OTHER, ERR), - ("b300-dsxe", MULTI, "0", "dsv4", "fp4", "dynamo-vllm", OTHER, DCGM), - ("b300-dsxe", MULTI, "0", "dsv4", "fp8", "dynamo-sglang", OTHER, ERR), - ("b300-dsxe", MULTI, "1", "dsv4", "fp4", "dynamo-sglang", OTHER, ERR), - ("b200-nscale", NATIVE, "1", "kimik3", "fp4", "dynamo-vllm", OTHER, AGENTX), - ("b200-nscale", NATIVE, "0", "dsv4", "fp4", "dynamo-sglang", OTHER, DCGM), - ("b200-nscale", NATIVE, "0", "glm5.2", "fp4", "dynamo-vllm", OTHER, ERR), - ("b200-nscale", MULTI, "1", "qwen3.5", "fp8", "dynamo-sglang", OTHER, AGENTX), - ("b200-nscale", MULTI, "1", "kimik3", "fp4", "dynamo-vllm", OTHER, ERR), - ("b200-nscale", MULTI, "0", "dsv4", "fp4", "dynamo-vllm", OTHER, DCGM), - ("b200-nscale", MULTI, "0", "dsv4", "fp4", "dynamo-sglang", OTHER, ERR), - ("h200-dgxc", MULTI, "1", "kimik3", "fp4", "vllm", OTHER, AGENTX), - ("h200-dgxc", MULTI, "1", "dsv4", "fp8", "dynamo-sglang", OTHER, ADAPTER), - ("h200-dgxc", MULTI, "0", "dsv4", "fp8", "dynamo-sglang", OTHER, ERR), - ("h200-dgxc", MULTI, "1", "kimik3", "fp4", "dynamo-vllm", OTHER, ERR), + ("gb200-nv", MULTI, "1", "glm5.2", "fp4", "dynamo-sglang", AGENTX), + ("gb200-nv", MULTI, "1", "kimik3", "fp4", "dynamo-vllm", AGENTX), + ("gb200-nv", MULTI, "0", "dsr1", "fp8", "dynamo-sglang", DCGM), + ("gb200-nv", MULTI, "0", "dsv4", "fp4", "dynamo-vllm", ERR), + ("gb300-nv", MULTI, "1", "kimik3", "fp4", "dynamo-vllm", AGENTX), + ("gb300-nv", MULTI, "1", "glm5.2", "fp4", "dynamo-sglang", DCGM), + ("gb300-nv", MULTI, "0", "qwen3.5", "fp8", "dynamo-trt", ERR), + ("b300-dsxe", MULTI, "0", "dsv4", "fp4", "dynamo-vllm", DCGM), + ("b300-dsxe", MULTI, "0", "dsv4", "fp8", "dynamo-sglang", ERR), + ("b300-dsxe", MULTI, "1", "dsv4", "fp4", "dynamo-sglang", ERR), + ("b200-nscale", NATIVE, "1", "kimik3", "fp4", "dynamo-vllm", AGENTX), + ("b200-nscale", NATIVE, "0", "dsv4", "fp4", "dynamo-sglang", DCGM), + ("b200-nscale", NATIVE, "0", "glm5.2", "fp4", "dynamo-vllm", ERR), + ("b200-nscale", MULTI, "1", "qwen3.5", "fp8", "dynamo-sglang", AGENTX), + ("b200-nscale", MULTI, "1", "kimik3", "fp4", "dynamo-vllm", ERR), + ("b200-nscale", MULTI, "0", "dsv4", "fp4", "dynamo-vllm", DCGM), + ("b200-nscale", MULTI, "0", "dsv4", "fp4", "dynamo-sglang", ERR), + ("h200-dgxc", MULTI, "1", "kimik3", "fp4", "vllm", AGENTX), + ("h200-dgxc", MULTI, "1", "dsv4", "fp8", "dynamo-sglang", ADAPTER), + ("h200-dgxc", MULTI, "0", "dsv4", "fp8", "dynamo-sglang", ERR), + ("h200-dgxc", MULTI, "1", "kimik3", "fp4", "dynamo-vllm", ERR), + ("h100-dgxc", MULTI, "0", "dsr1", "fp8", "dynamo-sglang", ERR), ] @pytest.mark.parametrize( - ("cluster", "path", "agentic", "prefix", "precision", "framework", "recipe", "expected"), CASES + ("cluster", "path", "agentic", "prefix", "precision", "framework", "expected"), CASES ) def test_power_eligibility_of_each_cluster_lane( - cluster, path, agentic, prefix, precision, framework, recipe, expected + cluster, path, agentic, prefix, precision, framework, expected ): - request = _request( - IS_AGENTIC=agentic, MODEL_PREFIX=prefix, PRECISION=precision, FRAMEWORK=framework - ) + env = dict(IS_AGENTIC=agentic, MODEL_PREFIX=prefix, PRECISION=precision, FRAMEWORK=framework) + if agentic == "1": + env.update(CONC="4", CONC_LIST="4", IS_MULTINODE="true") if expected == ERR: with pytest.raises(PowerPolicyError): - decide_power(cluster, path, dcgm=True, request=request, recipe=recipe) + decide_power(cluster, path, _multinode(**env, POWER="1")) return - decision = decide_power(cluster, path, dcgm=True, request=request, recipe=recipe) + decision = decide_power(cluster, path, _multinode(**env, POWER="1")) assert (decision.dcgm, decision.agentx, decision.adapter) == ( True, expected == AGENTX, expected == ADAPTER, ) # fmt: skip - assert not decide_power(cluster, path, dcgm=False, request=request, recipe=recipe).dcgm + assert not decide_power(cluster, path, _multinode(**env, POWER="0")).dcgm def test_gb200_reports_agentic_misses_distinctly(): - request = _request(IS_AGENTIC="1", MODEL_PREFIX="dsv4", PRECISION="fp4", FRAMEWORK="dynamo-sglang") - with pytest.raises(PowerPolicyError, match="AgentX dcgm-power requires the GLM-5.2"): - decide_power("gb200-nv", MULTI, dcgm=True, request=request, recipe=OTHER) - - -@pytest.mark.parametrize( - ("text", "enabled"), - [ - ("telemetry:\n dcgm_exporter:\n image: x\n enabled: true\n", True), - ("telemetry:\n dcgm_exporter:\nbenchmark:\n enabled: true\n", False), - ("benchmark:\n dcgm_exporter:\n enabled: true\n", False), - ("telemetry:\n dcgm_exporter:\n enabled: true\n", False), - ("telemetry: [unclosed\n enabled: true\n", False), - ], -) -def test_dcgm_detection_is_scoped_to_telemetry_block(text, enabled): - assert recipe_enables_dcgm_power(text) is enabled - - -def _mirror(tmp_path, rel, text): - path = tmp_path / "benchmarks/multi_node/srt-slurm-recipes" / rel - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(text) - - -POWER_RECIPE = "telemetry:\n dcgm_exporter:\n enabled: true\n" - - -def _multinode(**env): - return MultiNodeRequest.from_env({ - "RUNNER_NAME": "r_0", "IMAGE": "i", "SPEC_DECODING": "none", "RESULT_FILENAME": "r", - "RUN_EVAL": "false", **env, - }) - - -def test_nscale_eval_only_inspects_eval_recipe(tmp_path): - _mirror(tmp_path, "dsv4/eval.yaml", POWER_RECIPE) - _mirror(tmp_path, "dsv4/bench.yaml", "model: {}\n") - env = dict( - GITHUB_WORKSPACE=str(tmp_path), IS_AGENTIC="0", MODEL_PREFIX="dsv4", PRECISION="fp4", - FRAMEWORK="dynamo-vllm", SRT_RECIPE=f"{MIRROR}/dsv4/bench.yaml:zip", - EVAL_SRT_RECIPE=f"{MIRROR}/dsv4/eval.yaml", - ) - assert resolve_power("b200-nscale", MULTI, _multinode(**env, EVAL_ONLY="true")).dcgm - assert not resolve_power("b200-nscale", MULTI, _multinode(**env, EVAL_ONLY="false")).dcgm - assert not resolve_power("b300-dsxe", MULTI, _multinode(**env, EVAL_ONLY="true")).dcgm - - -def test_recipe_missing_from_the_workspace_stays_non_power(tmp_path): - request = _multinode(GITHUB_WORKSPACE=str(tmp_path), SRT_RECIPE=f"{MIRROR}/missing.yaml", - IS_AGENTIC="0", FRAMEWORK="dynamo-trt", MODEL_PREFIX="m", - PRECISION="fp8", EVAL_ONLY="false") - assert not resolve_power("gb300-nv", MULTI, request).dcgm + request = _multinode(IS_AGENTIC="1", MODEL_PREFIX="dsv4", PRECISION="fp4", + FRAMEWORK="dynamo-sglang", CONC="4", CONC_LIST="4", POWER="1") # fmt: skip + with pytest.raises(PowerPolicyError, match="AgentX dcgm-power requires GLM-5.2"): + decide_power("gb200-nv", MULTI, request) + + +def test_a_power_lane_needs_its_clusters_exporter_port(monkeypatch): + labels = {"cluster:c": ["c_0"]} + srt = {"network-interface": ""} + record = {**RECORD, "slurm": {**RECORD["slurm"], "srt-slurm": srt}} + clusters = load_inventory({"labels": labels, "clusters": {"c": record}}).clusters + monkeypatch.setitem(drivers.srt.lanes.SRT_LANES, ("c", MULTI), drivers.srt.lanes.SrtLane()) + monkeypatch.setitem(power.POWER_LANES, ("c", MULTI), power.PowerLane(rules=(), error="x")) + assert drivers.srt.table_problems(clusters, "c") == [ + "POWER_LANES['c', srt-multi]: no srt-slurm.power-exporter-port" + ] + srt["power-exporter-port"] = 9401 + clusters = load_inventory({"labels": labels, "clusters": {"c": record}}).clusters + assert drivers.srt.table_problems(clusters, "c") == [] AGENTX_FLASH = dict(MODEL_PREFIX="dsv41flash", FRAMEWORK="sglang", IS_MULTINODE="false", diff --git a/inferencex-e2e/infx/tests/launch/test_srt_config.py b/inferencex-e2e/infx/tests/launch/test_srt_config.py index 273ae27422..bda8be9f6d 100644 --- a/inferencex-e2e/infx/tests/launch/test_srt_config.py +++ b/inferencex-e2e/infx/tests/launch/test_srt_config.py @@ -181,7 +181,7 @@ def test_multinode_jobs_wait_at_least_the_health_floor_for_their_server(tmp_path recipe.parent.mkdir() recipe.write_text(yaml.safe_dump({"schema": 2, "name": "r", **({"health_check": health} if health else {})})) - prepare_recipe(tmp_path, "recipes/r.yaml", "job", None, None) + prepare_recipe(tmp_path, "recipes/r.yaml", "job", None) resolved = resolve_config_with_defaults(yaml.safe_load(recipe.read_text()), render(cluster(), job())) assert resolved["health_check"] == effective diff --git a/inferencex-e2e/infx/tests/launch/test_srt_driver.py b/inferencex-e2e/infx/tests/launch/test_srt_driver.py index 0f42416f59..b0c92045bd 100644 --- a/inferencex-e2e/infx/tests/launch/test_srt_driver.py +++ b/inferencex-e2e/infx/tests/launch/test_srt_driver.py @@ -41,12 +41,6 @@ "args": {"tensor-parallel-size": 4, "data-parallel-size": 1, "max-running-requests": 32}, }}, } # fmt: skip -AGENTX_RECIPE = { - **POINT_RECIPE, - "model": {"path": "hf:test/model", "container": "test:tag", "precision": "fp8"}, - "benchmark": {"type": "custom", "command": "bash /infmax-workspace/benchmarks/srt_agentic.sh", - "env": {"MODEL": "test/model"}}, -} # fmt: skip POINT_ENV = { "FRAMEWORK": "sglang", "MODEL": "test/model", "IMAGE": "test:tag", "PRECISION": "fp8", @@ -70,15 +64,7 @@ max_attempts: 100 interval_seconds: 5 """ -LANE_RECIPE = LANE_FRAGMENT + """model: - path: "alias" - container: "test:tag" - precision: "fp8" -benchmark: - type: sa-bench - concurrencies: [4] -""" -POWER_TELEMETRY = "telemetry:\n dcgm_exporter:\n image: dcgm\n enabled: true\n" +POWER_TELEMETRY = "telemetry:\n collector_join_timeout_seconds: 12\n" MIRROR = "benchmarks/multi_node/srt-slurm-recipes" @@ -193,7 +179,7 @@ def test_single_node_failed_allocation_fails_the_launch(harness): "lab-b": dict( lane=SrtLane(shared_run_root=(Match(),)), env=dict(FRAMEWORK="dynamo-vllm", IS_AGENTIC="1", ISL="0", OSL="0", CONC="4", FAKE_RESULTS="agentic"), - recipe=LANE_RECIPE, paths={"alias": "models/model"}, preflight=True, tag=None, setup_script=None, served=None, + paths={"hf:org/Model": "models/model"}, preflight=True, tag=None, setup_script=None, served=None, dist_timeout=False, time="10", mounts=(), staging="registry", shared_checkout=True, ), } # fmt: skip @@ -280,15 +266,18 @@ def test_multinode_lane_stages_workflow_artifacts(harness, monkeypatch, cluster_ assert staged["name"] == call["env"]["RUNNER_NAME"] == f"inferencex-{runner}" dist = {"dist-timeout": 1800} if lab["dist_timeout"] else {} assert staged["roles"]["prefill"]["args"] == {"tensor-parallel-size": 8, "watchdog-timeout": 600, **dist} - if "recipe" in lab: - assert submitted == "recipes/test/lane.yaml" - else: - # The fixed-sequence fragment is bound after the lane edits its staged copy. - bound = yaml.safe_load(Path(submitted).read_text()) - assert (bound["name"], bound["roles"], bound["model"]) == ( - staged["name"], staged["roles"], - {"path": "hf:org/Model", "container": "test:tag", "precision": "fp8"}, - ) + # The fragment is composed and bound after the lane edits its staged copy. + bound = yaml.safe_load(Path(submitted).read_text()) + assert (bound["name"], bound["roles"], bound["model"]) == ( + staged["name"], staged["roles"], + {"path": "hf:org/Model", "container": "test:tag", "precision": "fp8"}, + ) + if env["IS_AGENTIC"] == "1": + # lab-b mounts no cache volume, so the client keeps its caches in the container. + assert bound["benchmark"]["env"] == { + "RESULT_DIR": "/logs/agentic", "AIPERF_DATASET_MMAP_CACHE_DIR": "/aiperf_mmap_cache", + "HF_HUB_CACHE": "/hf_hub_cache", + } config = srtslurm(checkout) assert config["model_paths"] == {name: str(tmp / path) for name, path in lab["paths"].items()} @@ -317,10 +306,10 @@ def test_power_lane_stages_provenance_and_validates_each_concurrency( ) adapter.chmod(0o755) env = lane_env( - harness, "h200-dgxc", LANE_RECIPE + POWER_TELEMETRY, + harness, "h200-dgxc", LANE_FRAGMENT + POWER_TELEMETRY, MODEL_PREFIX=model_prefix, PRECISION=precision, FRAMEWORK=framework, MODEL=model, IS_AGENTIC="1", ISL="0", OSL="0", CONC="4", CONC_LIST="4", FAKE_RESULTS="agentic", - REQUIRE_POWER=require_power, INFERENCEX_RESULTS_PYTHON=str(adapter), + REQUIRE_POWER=require_power, INFERENCEX_RESULTS_PYTHON=str(adapter), POWER="1", ) # fmt: skip assert_ok(launch(env, harness.config, harness.workspace)) @@ -333,8 +322,10 @@ def test_power_lane_stages_provenance_and_validates_each_concurrency( assert provenance.endswith(f" {exporter}\n") assert (workspace / "LOGS/power/exporter-image.sha256").read_text() == provenance assert (workspace / "LOGS/power/power-producer-sha.txt").read_text() == commit + "\n" - staged = yaml.safe_load((checkout / "recipes/test/lane.yaml").read_text()) - assert staged["benchmark"]["concurrencies"] == [4] + bound = yaml.safe_load((checkout / "recipe.yaml").read_text()) + assert bound["benchmark"]["concurrencies"] == [4] + assert bound["telemetry"]["collector_join_timeout_seconds"] == 12 + assert (bound["telemetry"]["enabled"], bound["telemetry"]["dcgm_exporter"]["port"]) == (True, 9401) runs = lines(harness.logs, "adapter") assert [run.split("--result-dir ")[1].split()[0].rsplit("/", 1)[1] for run in runs] == ["conc_4"] @@ -390,7 +381,7 @@ def test_sigterm_while_streaming_cancels_the_job_and_exits_143(harness, shape): def test_b300_flash_agentx_reenters_inside_a_batch_allocation(harness): runner_temp = harness.tmp / "runner-temp" runner_temp.mkdir() - (harness.workspace / "recipe.yaml").write_text(yaml.safe_dump({"base": AGENTX_RECIPE})) + (harness.workspace / "recipe.yaml").write_text(yaml.safe_dump({"base": POINT_RECIPE})) env = { **harness.env, **POINT_ENV, "RUNNER_NAME": runner_for("b300-dsxe"), "MODEL_PREFIX": "dsv41flash", "PRECISION": "fp8", "IS_AGENTIC": "1", "DURATION": "600", "RUNNER_TEMP": str(runner_temp), @@ -433,7 +424,7 @@ def test_eval_only_runs_the_eval_recipe_with_real_verification(harness): (mirror / "trtllm/forced.yaml").write_text( "roles:\n decode:\n env:\n TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: 2\n KEEP: 1\n" ) - (mirror / "eval.yaml").write_text(LANE_RECIPE) + (mirror / "eval.yaml").write_text(LANE_FRAGMENT) # What the in-container eval staged: its own batch record, but not every workflow input. staged = { "eval_suite": "kimi_tool_call_schema", "recipe_fingerprint": "", "conc": 4, @@ -450,7 +441,7 @@ def test_eval_only_runs_the_eval_recipe_with_real_verification(harness): [call] = srtctl_calls(harness.logs) argv = call["argv"] - assert argv[argv.index("--file") + 1] == "recipes/test/eval.yaml" + assert Path(argv[argv.index("--file") + 1]).name == "recipe.yaml" assert "frontend.placement.node=head" in argv checkout = Path(call["cwd"]) assert (checkout / "recipes/test/trtllm/forced.yaml").read_text() == ( @@ -520,13 +511,11 @@ def test_a_setup_failure_without_a_bad_archive_is_not_retried(harness): @pytest.mark.parametrize(("cluster_id", "env", "message"), [ ("h100-dgxc", dict(MODEL_PREFIX="dsr1", PRECISION="fp8", FRAMEWORK="dynamo-vllm"), "Unsupported framework"), ("b200-nscale", dict(MODEL_PREFIX="dsv4", PRECISION="fp4", FRAMEWORK="dynamo-trt"), "only dynamo-vllm"), - # Only complete (AgentX) recipes name model aliases; fragments are bound to hf:. - ("gb300-nv", dict(MODEL_PREFIX="llama", PRECISION="fp8", FRAMEWORK="dynamo-sglang", IS_AGENTIC="1", - CONC="4"), "stages no checkpoint"), + ("h200-dgxc", dict(MODEL_PREFIX="dsr1", PRECISION="fp8", FRAMEWORK="dynamo-sglang", + SRT_RECIPE=f"{MIRROR}/test/missing.yaml"), "is not in the recipe mirror"), ]) # fmt: skip def test_unsupported_multinode_requests_fail_before_any_setup(harness, cluster_id, env, message): - recipe = LANE_RECIPE if env.get("IS_AGENTIC") == "1" else LANE_FRAGMENT - result = launch(lane_env(harness, cluster_id, recipe, MODEL="m", **env), harness.config, harness.workspace) + result = launch(lane_env(harness, cluster_id, MODEL="m", **env), harness.config, harness.workspace) assert result.returncode == 1 assert message in result.stderr assert lines(harness.logs, "git") == [] diff --git a/inferencex-e2e/infx/tests/launch/test_srt_policy.py b/inferencex-e2e/infx/tests/launch/test_srt_policy.py index 16ba9ad036..46b5111cb3 100644 --- a/inferencex-e2e/infx/tests/launch/test_srt_policy.py +++ b/inferencex-e2e/infx/tests/launch/test_srt_policy.py @@ -118,45 +118,26 @@ def test_a_points_own_model_path_is_what_its_job_serves(tmp_path, monkeypatch): assert job_env(c, point, served)["MODEL_PATH"] == "/point/m" -BUNDLE = """base: - model: {path: alias-a} -override_x: - model: {path: alias-b} -zip_override_y: - model: {path: [alias-c, "hf:org/M"]} -""" - - -@pytest.mark.parametrize(("recipe", "model", "agentic", "paths"), [ - (BUNDLE, "org/M", "1", {"alias-a": "nvme/m", "alias-b": "nvme/m", "alias-c": "nvme/m"}), - ("model: {path: /abs/m}\n", "org/Unstaged", "1", {}), - ("model: {path: alias-a}\n", "org/Unstaged", "1", LaunchError), - # A fixed-sequence fragment is bound to hf:, which needs a staged checkpoint. - ("roles: {}\n", "org/M", "0", {"hf:org/M": "nvme/m"}), - ("roles: {}\n", "org/Unstaged", "0", LaunchError), - # A hub lane loads the Hub snapshot, staged or not. - ("roles: {}\n", "hub/S", "0", {}), - ("roles: {}\n", "hub/Unstaged", "0", {}), +@pytest.mark.parametrize(("model", "agentic", "overrides", "paths"), [ + ("org/M", "0", (), {"hf:org/M": "nvme/m"}), + # A fixed-sequence point needs a staged checkpoint unless its lane loads the Hub snapshot. + ("org/Unstaged", "0", (), LaunchError), + ("org/Unstaged", "1", (), {}), + ("org/M", "0", (Override(Match(), hub=True),), {}), + ("org/Unstaged", "0", (Override(Match(), hub=True),), {}), ]) # fmt: skip -def test_every_recipe_alias_maps_to_the_checkpoint_and_literals_pass_through(tmp_path, monkeypatch, recipe, model, agentic, paths): - monkeypatch.setitem(models.OVERRIDES, "c", (Override(Match(model_glob="hub/*"), hub=True),)) - mirror = tmp_path / "ws/benchmarks/multi_node/srt-slurm-recipes/r.yaml" - mirror.parent.mkdir(parents=True) - mirror.write_text(recipe) - c = cluster(tmp_path) - point = request(MODEL=model, IS_AGENTIC=agentic, GITHUB_WORKSPACE=str(tmp_path / "ws")) +def test_the_bound_model_maps_to_its_checkpoint_unless_served_from_the_hub( + tmp_path, monkeypatch, model, agentic, overrides, paths +): + monkeypatch.setitem(models.OVERRIDES, "c", overrides) + c, point = cluster(tmp_path), request(MODEL=model, IS_AGENTIC=agentic) + served = served_path(c, point, checkpoint(c, point)) if paths is LaunchError: with pytest.raises(LaunchError, match="stages no checkpoint"): - model_paths(c, point, f"{MIRROR}/r.yaml:override_x", served_path(c, point, checkpoint(c, point))) + model_paths(c, point, served) return - resolved = model_paths(c, point, f"{MIRROR}/r.yaml:override_x", served_path(c, point, checkpoint(c, point))) - assert resolved == {alias: str(tmp_path / path) for alias, path in paths.items()} - - -def test_missing_recipe_fails_before_model_resolution(tmp_path): - c, point = cluster(tmp_path), request(MODEL="org/M", GITHUB_WORKSPACE=str(tmp_path)) - with pytest.raises(LaunchError, match="not in the recipe mirror"): - model_paths(c, point, f"{MIRROR}/missing.yaml", "/m") + resolved = model_paths(c, point, served) + assert resolved == {name: str(tmp_path / path) for name, path in paths.items()} def test_matching_single_node_points_read_the_shared_hub_cache(tmp_path, monkeypatch): diff --git a/inferencex-e2e/infx/tests/launch/test_srt_recipe_edits.py b/inferencex-e2e/infx/tests/launch/test_srt_recipe_edits.py index 599bf6ac42..99be67f150 100644 --- a/inferencex-e2e/infx/tests/launch/test_srt_recipe_edits.py +++ b/inferencex-e2e/infx/tests/launch/test_srt_recipe_edits.py @@ -1,14 +1,8 @@ """Edits the multi-node lanes apply to the staged recipe copy.""" -import pytest import yaml -from infx.launch.drivers.srt.recipe import ( - add_dist_timeout, - inject_concurrencies, - raise_health_attempts, - rename_job, -) +from infx.launch.drivers.srt.recipe import add_dist_timeout, raise_health_attempts, rename_job RECIPE = """name: "upstream" roles: @@ -47,24 +41,3 @@ def test_dist_timeout_follows_each_role_watchdog_timeout(): edited = yaml.safe_load(add_dist_timeout(RECIPE, 1800)) assert edited["roles"]["prefill"]["args"] == {"watchdog-timeout": 600, "dist-timeout": 1800, "tp": 4} assert add_dist_timeout("watchdog-timeout: 1\n", 1800) == "watchdog-timeout: 1\n" - - -def test_injection_replaces_only_the_benchmark_concurrencies(tmp_path): - recipe = tmp_path / "recipe.yaml" - recipe.write_text("name: x\nbenchmark:\n type: sa-bench\n concurrencies: 1x2x4\n") - inject_concurrencies(recipe, [4, 16]) - assert yaml.safe_load(recipe.read_text()) == { - "name": "x", "benchmark": {"type": "sa-bench", "concurrencies": [4, 16]}, - } # fmt: skip - - -@pytest.mark.parametrize(("text", "message"), [ - ("base:\n benchmark:\n concurrencies: [1]\n", "benchmark mapping"), - ("benchmark: [\n", "failed to load"), -]) # fmt: skip -def test_an_unusable_recipe_is_left_untouched(tmp_path, text, message): - recipe = tmp_path / "recipe.yaml" - recipe.write_text(text) - with pytest.raises(ValueError, match=message): - inject_concurrencies(recipe, [4]) - assert recipe.read_text() == text diff --git a/inferencex-e2e/infx/tests/matrix/test_validation.py b/inferencex-e2e/infx/tests/matrix/test_validation.py index f95dc9f7e2..1cf7ed2ffd 100644 --- a/inferencex-e2e/infx/tests/matrix/test_validation.py +++ b/inferencex-e2e/infx/tests/matrix/test_validation.py @@ -1471,3 +1471,16 @@ def test_validation_runs_by_default(self, tmp_path): with pytest.raises(ValueError) as exc_info: load_runner_file(str(runner_file)) assert "must be a list" in str(exc_info.value) + + +@pytest.mark.parametrize(("topology", "accepted"), [ + ({"worker": {"num-worker": 1, "tp": 8, "ep": 1, "dp-attn": False}}, True), + ({"tp": 8, "kv-offloading": "none"}, False), +]) # fmt: skip +def test_only_multinode_agentic_rows_measure_power(topology, accepted): + row = {"srt-recipe": "r.yaml", "conc-list": [4], "power": True, **topology} + if accepted: + assert AgenticCodingSearchSpaceEntry(**row).power is True + else: + with pytest.raises(ValidationError, match="Only multinode entries measure DCGM power"): + AgenticCodingSearchSpaceEntry(**row) diff --git a/inferencex-e2e/infx/tests/srt_slurm/test_srt_single_node.py b/inferencex-e2e/infx/tests/srt_slurm/test_srt_single_node.py index b191ccafaf..41fedf3fc0 100644 --- a/inferencex-e2e/infx/tests/srt_slurm/test_srt_single_node.py +++ b/inferencex-e2e/infx/tests/srt_slurm/test_srt_single_node.py @@ -18,10 +18,11 @@ @pytest.fixture def point(tmp_path, monkeypatch): - """A fixed-sequence fragment, the point binding its first variant, and the shared block.""" - shared = tmp_path / "configs/srt-recipes/fixed-sequence-single.yaml" - shared.parent.mkdir(parents=True) - shared.write_text(yaml.safe_dump({"benchmark": {"type": "custom", "command": "bash client.sh"}})) + """A fixed-sequence fragment, the point binding its first variant, and the shared blocks.""" + for workload, client in (("fixed-sequence", "client.sh"), ("agentic", "srt_agentic.sh")): + shared = tmp_path / f"configs/srt-recipes/{workload}-single.yaml" + shared.parent.mkdir(parents=True, exist_ok=True) + shared.write_text(yaml.safe_dump({"benchmark": {"type": "custom", "command": f"bash {client}"}})) monkeypatch.setenv("INFERENCEX_REPOSITORY_ROOT", str(tmp_path)) recipe = { "engine": "sglang", @@ -76,17 +77,19 @@ def test_mismatched_point_fails_before_submission(point, field, value, message): runtime_arguments(f"{path}:base", {**env, field: value}) -def test_agentx_recipes_are_validated_as_written(point): +def test_agentx_fragments_are_composed_and_bound_to_the_point(point): path, recipe, env = point - complete = { - **recipe, "model": {"path": "hf:test/model", "container": "test:tag", "precision": "fp8"}, - "benchmark": {"type": "custom", "command": "bash srt_agentic.sh", "env": {"MODEL": "test/model"}}, - } # fmt: skip - path.write_text(yaml.safe_dump({"base": complete})) - agentic = {**env, "IS_AGENTIC": "1"} - assert select_recipe(f"{path}:base", agentic) == (f"{path}:base", complete) - with pytest.raises(ValueError, match="image"): - select_recipe(f"{path}:base", {**agentic, "IMAGE": "other:tag"}) + path.write_text(yaml.safe_dump({"base": recipe, "override_c2": {"benchmark": {"env": {"CONC": "2"}}}})) + agentic = {**env, "IS_AGENTIC": "1", "IMAGE": "other:tag"} + selected, bound = select_recipe(str(path), agentic) + assert selected == f"{path}:override_c2" + assert bound["model"] == {"path": "hf:test/model", "container": "other:tag", "precision": "fp8"} + assert bound["benchmark"] == { + "type": "custom", "command": "bash srt_agentic.sh", "env": {"CONC": "2", "MODEL": "test/model"}, + } + path.write_text(yaml.safe_dump({"base": {**recipe, "model": {"container": "stale:tag"}}})) + with pytest.raises(ValueError, match=r"remove base\.model\.container"): + select_recipe(f"{path}:base", agentic) def test_ambiguous_native_variants_are_rejected(point): diff --git a/inferencex-e2e/infx/tests/srt_slurm/test_workload.py b/inferencex-e2e/infx/tests/srt_slurm/test_workload.py index 8eeb0b73f3..33d1c001b2 100644 --- a/inferencex-e2e/infx/tests/srt_slurm/test_workload.py +++ b/inferencex-e2e/infx/tests/srt_slurm/test_workload.py @@ -1,4 +1,4 @@ -"""Composing fixed-sequence fragments with their shared block, and binding the matrix point.""" +"""Composing recipe fragments with their shared blocks, and binding the matrix point.""" import os import subprocess @@ -8,24 +8,33 @@ import pytest import yaml +from infx.bench.agentic.run import Plan +from infx.srt_slurm.synthetic_acceptance import selected_recipes from infx.srt_slurm.workload import bind_workload, compose_recipe, parse_concurrencies ROOT = Path(__file__).resolve().parents[3] +sys.path.insert(0, str(ROOT / "utils/srt-slurm/src")) MULTI_ENV = { "IMAGE": "registry/image:2", "MODEL": "org/model", "PRECISION": "fp8", "ISL": "8192", "OSL": "1024", "CONC_LIST": "4 16", } # fmt: skip +AGENTX_ENV = {"IMAGE": "nvcr.io#org/image:3", "MODEL": "org/model", "PRECISION": "fp4", "CONC_LIST": "8"} +CLIENT_ENV = {"RESULT_DIR": "/logs/agentic", "HF_HUB_CACHE": "/hf_hub_cache"} @pytest.fixture def project(tmp_path): """A project whose shared blocks set the client and a default env value.""" - for lane in ("single", "multi"): - shared = tmp_path / f"configs/srt-recipes/fixed-sequence-{lane}.yaml" - shared.parent.mkdir(parents=True, exist_ok=True) - shared.write_text(yaml.safe_dump({"benchmark": { - "type": "custom", "command": f"bash {lane}.sh", "env": {"TOKENIZER": "/shared"}, - }, "health_check": {"max_attempts": 10, "interval_seconds": 5}})) + for workload in ("fixed-sequence", "agentic"): + for lane in ("single", "multi"): + shared = tmp_path / f"configs/srt-recipes/{workload}-{lane}.yaml" + shared.parent.mkdir(parents=True, exist_ok=True) + shared.write_text(yaml.safe_dump({"benchmark": { + "type": "custom", "command": f"bash {workload}-{lane}.sh", "env": {"TOKENIZER": "/shared"}, + }, "health_check": {"max_attempts": 10, "interval_seconds": 5}})) + (tmp_path / "configs/srt-recipes/telemetry-dcgm.yaml").write_text(yaml.safe_dump({"telemetry": { + "enabled": True, "required": True, "dcgm_exporter": {"container_image": "dcgm-exporter"}, + }})) # fmt: skip return tmp_path @@ -43,7 +52,7 @@ def test_fragment_values_win_over_shared_defaults_and_lists_replace(project): "services": [{"name": "etcd"}], }) - assert compose_recipe(path, multinode=True, root=project) == { + assert compose_recipe(path, agentic=False, multinode=True, root=project) == { "name": "plain", "benchmark": { "type": "custom", "command": "bash own.sh --trust-remote-code", @@ -62,27 +71,30 @@ def test_bundles_take_the_shared_block_under_base_and_keep_their_variants(projec "zip_override_conc": {"benchmark": {"env": {"CONC": ["2", "4"]}}}, }) - assert compose_recipe(path, multinode=False, root=project) == { + assert compose_recipe(path, agentic=False, multinode=False, root=project) == { "base": {"name": "bundle", "args": [3], "benchmark": {"type": "custom"}}, "zip_override_conc": {"benchmark": {"env": {"CONC": ["2", "4"]}}}, } -@pytest.mark.parametrize(("data", "multinode", "reported"), [ - ({"model": {"container": "other/image:1"}}, True, "model.container (= 'other/image:1')"), - ({"base": {"benchmark": {"env": {"ISL": "8192"}}}}, False, "base.benchmark.env.ISL (= '8192')"), - ({"base": {}, "zip_override_c": {"benchmark": {"env": {"CONC": ["4"]}}}}, True, +@pytest.mark.parametrize(("data", "agentic", "multinode", "reported"), [ + ({"model": {"container": "other/image:1"}}, False, True, "model.container (= 'other/image:1')"), + ({"base": {"benchmark": {"env": {"ISL": "8192"}}}}, False, False, "base.benchmark.env.ISL (= '8192')"), + ({"base": {}, "zip_override_c": {"benchmark": {"env": {"CONC": ["4"]}}}}, False, True, "zip_override_c.benchmark.env.CONC (= ['4'])"), - ({"base": {}, "override_c": {"benchmark": {"concurrencies": [4]}}}, False, + ({"base": {}, "override_c": {"benchmark": {"concurrencies": [4]}}}, False, False, "override_c.benchmark.concurrencies (= [4])"), + ({"telemetry": {"enabled": True}}, False, True, "telemetry.enabled (= True)"), + ({"benchmark": {"env": {"HF_HUB_CACHE": "/elsewhere"}}}, True, True, + "benchmark.env.HF_HUB_CACHE (= '/elsewhere')"), + ({"base": {"benchmark": {"env": {"RESULT_DIR": "/logs"}}}}, True, False, + "base.benchmark.env.RESULT_DIR (= '/logs')"), ]) # fmt: skip -def test_a_fragment_that_sets_a_bound_key_is_rejected(project, data, multinode, reported): +def test_a_fragment_that_sets_a_bound_key_is_rejected(project, data, agentic, multinode, reported): path = fragment(project, data) with pytest.raises(ValueError) as error: - compose_recipe(path, multinode=multinode, root=project) - assert str(error.value) == ( - f"{path}: remove {reported} from the fragment; they are bound from the matrix point" - ) + compose_recipe(path, agentic=agentic, multinode=multinode, root=project) + assert str(error.value) == f"{path}: remove {reported} from the fragment; the launcher binds them" def test_multinode_binding_writes_the_point_and_leaves_the_client_its_job_environment(): @@ -93,7 +105,7 @@ def test_multinode_binding_writes_the_point_and_leaves_the_client_its_job_enviro "benchmark": {"type": "custom", "env": {"TOKENIZER": "/model"}}, } # fmt: skip - bound = bind_workload(recipe, MULTI_ENV, multinode=True) + bound = bind_workload(recipe, MULTI_ENV, agentic=False, multinode=True) assert bound == { "schema": 2, "name": "job", @@ -105,16 +117,64 @@ def test_multinode_binding_writes_the_point_and_leaves_the_client_its_job_enviro "identity": {"container": {"image": "registry/image:2"}, "frameworks": {"sglang": "0.5"}}, "benchmark": {"type": "custom", "env": {"TOKENIZER": "/model", "ISL": "8192", "OSL": "1024"}}, } # fmt: skip - assert "identity" not in bind_workload({}, MULTI_ENV, multinode=True) + assert "identity" not in bind_workload({}, MULTI_ENV, agentic=False, multinode=True) -@pytest.mark.parametrize(("telemetry", "concurrencies"), [ - ({"enabled": True, "dcgm_exporter": {"port": 9401}}, [4, 16]), - ({"enabled": False}, None), -]) # fmt: skip -def test_telemetry_recipes_get_the_points_concurrencies(telemetry, concurrencies): - bound = bind_workload({"telemetry": telemetry}, MULTI_ENV, multinode=True) - assert bound["benchmark"].get("concurrencies") == concurrencies +def test_agentx_binding_writes_the_launchers_client_paths_without_lengths(): + recipe = {"identity": {"container": {}}, "benchmark": {"env": {"AIPERF_X": "1"}}} + + bound = bind_workload(recipe, AGENTX_ENV, agentic=True, multinode=True, client_env=CLIENT_ENV) + + assert bound == { + "model": {"path": "hf:org/model", "container": "nvcr.io#org/image:3", "precision": "fp4"}, + "identity": {"container": {"image": "nvcr.io/org/image:3"}}, + "benchmark": {"env": {"AIPERF_X": "1", **CLIENT_ENV}}, + } + own_layout = {"benchmark": {"env": {"HF_HOME": "/logs/hf"}}} + bound = bind_workload(own_layout, AGENTX_ENV, agentic=True, multinode=True, client_env=CLIENT_ENV) + assert bound["benchmark"]["env"] == {"HF_HOME": "/logs/hf", "RESULT_DIR": "/logs/agentic"} + + +def test_a_power_point_gets_telemetry_and_its_concurrencies_on_a_bundle_variant(project): + path = fragment(project, { + "base": {"name": "bundle", "telemetry": {"collector_join_timeout_seconds": 12}}, + "override_c8": {"roles": {"agg": {"nodes": 2}}}, + }) + + composed = compose_recipe(path, agentic=True, multinode=True, root=project, power_port=19401) + [(_, variant)] = selected_recipes(composed, "override_c8") + bound = bind_workload(variant, AGENTX_ENV, agentic=True, multinode=True) + + assert bound["telemetry"] == { + "collector_join_timeout_seconds": 12, "enabled": True, "required": True, + "dcgm_exporter": {"container_image": "dcgm-exporter", "port": 19401}, + } # fmt: skip + assert bound["benchmark"]["concurrencies"] == [8] + unpowered = compose_recipe(path, agentic=True, multinode=True, root=project)["base"] + assert "concurrencies" not in bind_workload(unpowered, AGENTX_ENV, agentic=True, multinode=True)[ + "benchmark" + ] + + +def test_a_bound_agentx_point_satisfies_the_benchmark_client(project, tmp_path): + path = fragment(project, {"benchmark": {"env": {"AIPERF_LIVE_FAILED_REQUEST_THRESHOLD": "0.25"}}}) + composed = compose_recipe(path, agentic=True, multinode=True, root=project) + bound = bind_workload(composed, AGENTX_ENV, agentic=True, multinode=True, client_env=CLIENT_ENV) + job = { + "RESULT_FILENAME": "r", "EVAL_ONLY": "false", "IS_MULTINODE": "true", "PRECISION": "fp4", + "MODEL": "org/model", "MODEL_PREFIX": "dsv4", "FRAMEWORK": "dynamo-vllm", "CONC": "8", + "CONC_LIST": "8", "DURATION": "3600", "AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS": "3600", + "AIPERF_EXPERIMENTAL_FAST": "0", "AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID": "false", + "AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING": "1", "ENABLE_AGENTX_POWER": "0", "REQUIRE_POWER": "0", + "IS_AGENTIC": "1", "KV_OFFLOADING": "none", "SRT_FRONTEND_HOST": "10.0.0.1", + "SRT_FRONTEND_PORT": "8000", + } # fmt: skip + + plan = Plan.from_env({**job, **bound["benchmark"]["env"]}) + + assert plan.result_dir == Path("/logs/agentic/conc_8") + assert plan.replay.url == "http://10.0.0.1:8000" + assert plan.replay.live_failed_request_threshold == "0.25" @pytest.mark.parametrize(("environment", "message"), [ @@ -124,7 +184,7 @@ def test_telemetry_recipes_get_the_points_concurrencies(telemetry, concurrencies ]) # fmt: skip def test_a_malformed_point_is_rejected_before_binding(environment, message): with pytest.raises(ValueError, match=message): - bind_workload({}, environment, multinode=True) + bind_workload({}, environment, agentic=False, multinode=True) def test_conc_list_must_be_canonical_positive_integers(): @@ -144,19 +204,28 @@ def test_the_multinode_binder_writes_the_one_selected_variant(project, tmp_path) env = {**os.environ, **MULTI_ENV, "INFERENCEX_REPOSITORY_ROOT": str(project), "PYTHONPATH": os.pathsep.join([str(ROOT), str(ROOT / "utils/srt-slurm/src")])} # fmt: skip - def bind(selector: str) -> subprocess.CompletedProcess[str]: + def bind(*arguments: str) -> subprocess.CompletedProcess[str]: return subprocess.run( - [sys.executable, "-m", "infx.srt_slurm.workload", f"{path}{selector}", str(output)], + [sys.executable, "-m", "infx.srt_slurm.workload", *arguments], env=env, capture_output=True, text=True, check=False, ) - assert bind(":override_wide").returncode == 0 + assert bind(f"{path}:override_wide", str(output)).returncode == 0 bound = yaml.safe_load(output.read_text()) assert (bound["name"], bound["roles"], bound["benchmark"]["command"]) == ( - "bundle_wide", {"decode": {"nodes": 4}}, "bash multi.sh", + "bundle_wide", {"decode": {"nodes": 4}}, "bash fixed-sequence-multi.sh", ) output.unlink() - result = bind(":override_*") + result = bind(f"{path}:override_*", str(output)) assert result.returncode == 2 assert "selects 2 variants, not one" in result.stderr assert not output.exists() + env["IS_AGENTIC"] = "1" + arguments = ("--power-port", "9401", "--client-env", "RESULT_DIR=/logs/agentic") + assert bind(f"{path}:override_narrow", str(output), *arguments).returncode == 0 + bound = yaml.safe_load(output.read_text()) + assert (bound["benchmark"]["command"], bound["benchmark"]["concurrencies"]) == ( + "bash agentic-multi.sh", [4, 16], + ) + assert bound["benchmark"]["env"]["RESULT_DIR"] == "/logs/agentic" + assert bound["telemetry"]["dcgm_exporter"]["port"] == 9401 From b5eae9cbe4d62272da299045a09b22c56a80a721 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Thu, 8 Oct 2026 10:15:58 -0500 Subject: [PATCH 02/14] feat(srt): generate the bound recipes of AgentX points too infx generate now binds every srt-slurm point of the selected keys, AgentX as well as fixed-sequence, single- and multi-node, through the binder the launcher runs. The workflow environment it reproduces gains the AgentX inputs (IS_AGENTIC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, THINKING_MODE) and the multi-node request fields. A multi-node point takes the binder inputs the launcher derives from the one cluster its runner label schedules on: launch_path, the lane's POWER_LANES decision and the cluster's power-exporter-port give the DCGM telemetry block and bound concurrencies, and config.agentic_client_env, which no longer needs an SrtRun, gives the AgentX client paths. Such a point fails when its label spans several clusters or the lane refuses its power. The manifest records the cluster. --- inferencex-e2e/infx/cli.py | 2 +- .../infx/launch/drivers/srt/__init__.py | 4 +- .../infx/launch/drivers/srt/config.py | 22 +-- inferencex-e2e/infx/srt_slurm/generate.py | 131 +++++++++++++-- .../infx/tests/srt_slurm/test_generate.py | 152 +++++++++++++++--- 5 files changed, 265 insertions(+), 46 deletions(-) diff --git a/inferencex-e2e/infx/cli.py b/inferencex-e2e/infx/cli.py index 54e26b6f1e..6ad09e8691 100644 --- a/inferencex-e2e/infx/cli.py +++ b/inferencex-e2e/infx/cli.py @@ -15,7 +15,7 @@ def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(prog="infx") commands = parser.add_subparsers(dest="command", required=True) generate = commands.add_parser( - "generate", help="Write the bound fixed-sequence srt-slurm recipes of master-config points" + "generate", help="Write the bound srt-slurm recipes of master-config points" ) generate.add_argument( "--config-key", action="append", required=True, help="master-config key or glob; repeatable" diff --git a/inferencex-e2e/infx/launch/drivers/srt/__init__.py b/inferencex-e2e/infx/launch/drivers/srt/__init__.py index 36d1d45dd8..0b0c74875a 100644 --- a/inferencex-e2e/infx/launch/drivers/srt/__init__.py +++ b/inferencex-e2e/infx/launch/drivers/srt/__init__.py @@ -162,7 +162,9 @@ def run_multinode(launch: Launch) -> int: job_name = srtctl_job_name(request.runner_name) prepare_recipe(checkout.root, staged, job_name, run.srt.dist_timeout_s) recipe = str(checkout.root / BOUND_RECIPE) - client_env = config.agentic_client_env(run, lane) if request.is_agentic else {} + client_env = ( + config.agentic_client_env(run.cluster, run.srt, lane, request) if request.is_agentic else {} + ) power_port = run.srt.power_exporter_port if decision.dcgm else None if rc := submit.bind_recipe(run, checkout, staged, recipe, power_port, client_env): return rc diff --git a/inferencex-e2e/infx/launch/drivers/srt/config.py b/inferencex-e2e/infx/launch/drivers/srt/config.py index 6fade32c1a..79dd8e8614 100644 --- a/inferencex-e2e/infx/launch/drivers/srt/config.py +++ b/inferencex-e2e/infx/launch/drivers/srt/config.py @@ -22,11 +22,12 @@ if TYPE_CHECKING: from infx.clusters import Cluster - from infx.clusters.slurm import SlurmSettings + from infx.clusters.slurm import SlurmSettings, SrtSlurmSettings from infx.launch.drivers.srt.checkout import Checkout from infx.launch.drivers.srt.lanes import SrtLane from infx.launch.drivers.srt.power import PowerDecision from infx.launch.drivers.srt.run import SrtRun + from infx.launch.request import SrtRequest NGINX_IMAGE = "nginx:1.27.4" DCGM_EXPORTER_IMAGE = "nvcr.io/nvidia/k8s/dcgm-exporter:4.6.0-4.8.3-distroless" @@ -184,11 +185,12 @@ def create_volume_mounts(run: SrtRun) -> None: _create_dir(volume_path(run.cluster, name)) -def _lane_volumes(run: SrtRun, lane: SrtLane) -> list[tuple[str, str | None, bool]]: +def _lane_volumes( + srt: SrtSlurmSettings, lane: SrtLane, request: SrtRequest +) -> list[tuple[str, str | None, bool]]: """(volume, container target, world-writable) of each volume a multi-node job adds.""" - request = run.request volumes = ( - [(name, target, True) for name, target in run.srt.agentic_volume_mounts.items()] + [(name, target, True) for name, target in srt.agentic_volume_mounts.items()] if request.is_agentic else [] ) @@ -199,7 +201,7 @@ def _lane_volumes(run: SrtRun, lane: SrtLane) -> list[tuple[str, str | None, boo def lane_mounts(run: SrtRun, lane: SrtLane) -> list[tuple[str, str]]: """The (host, container) mounts a multi-node job adds for this request; hosts are created.""" mounts: list[tuple[str, str]] = [] - for volume, target, world_writable in _lane_volumes(run, lane): + for volume, target, world_writable in _lane_volumes(run.srt, lane, run.request): host = volume_path(run.cluster, volume) _create_dir(host, world_writable=world_writable) mounts.append((str(host), target or str(host))) @@ -216,11 +218,13 @@ def lane_mounts(run: SrtRun, lane: SrtLane) -> list[tuple[str, str]]: } -def agentic_client_env(run: SrtRun, lane: SrtLane) -> dict[str, str]: +def agentic_client_env( + cluster: Cluster, srt: SrtSlurmSettings, lane: SrtLane, request: SrtRequest +) -> dict[str, str]: """The result and cache paths a multi-node AgentX job hands its benchmark client.""" - targets = dict(run.srt.volume_mounts) - for volume, target, _ in _lane_volumes(run, lane): - targets[volume] = target or str(volume_path(run.cluster, volume)) + targets = dict(srt.volume_mounts) + for volume, target, _ in _lane_volumes(srt, lane, request): + targets[volume] = target or str(volume_path(cluster, volume)) env = {"RESULT_DIR": lane.agentic_result_dir} for name, (volumes, unmounted) in _CLIENT_CACHES.items(): env[name] = next((targets[volume] for volume in volumes if volume in targets), unmounted) diff --git a/inferencex-e2e/infx/srt_slurm/generate.py b/inferencex-e2e/infx/srt_slurm/generate.py index 753ceacbde..d4eb74b133 100644 --- a/inferencex-e2e/infx/srt_slurm/generate.py +++ b/inferencex-e2e/infx/srt_slurm/generate.py @@ -1,4 +1,4 @@ -"""Write the bound fixed-sequence srt-slurm recipes of master-config points for inspection.""" +"""Write the bound srt-slurm recipes of master-config points for inspection.""" from __future__ import annotations @@ -9,36 +9,54 @@ from collections.abc import Mapping from copy import deepcopy from pathlib import Path -from typing import Any +from typing import TYPE_CHECKING, Any import yaml +from infx.clusters import load_inventory +from infx.clusters.slurm import slurm_settings from infx.matrix.generate import expand_config_keys, generate_config_matrix from infx.matrix.validation import config_root, load_config_files, load_runner_file from infx.srt_slurm.single_node import select_recipe from infx.srt_slurm.synthetic_acceptance import build_overrides from infx.srt_slurm.workload import bind_multinode +if TYPE_CHECKING: + from infx.clusters import Cluster, RunnerInventory + # What benchmark-tmpl.yml and benchmark-multinode-tmpl.yml export. RANDOM_RANGE_RATIO = "0.8" +THINKING_MODE = "thinking_on" def _environment(point: Mapping[str, Any]) -> dict[str, str]: """The workflow environment the launcher reads for ``point``.""" + agentic = point.get("scenario-type") == "agentic-coding" environment = { "IMAGE": point["image"], "MODEL": point["model"], + "MODEL_PREFIX": point["model-prefix"], "PRECISION": point["precision"], "FRAMEWORK": point["framework"], "SPEC_DECODING": point["spec-decoding"], - "IS_AGENTIC": "0", - "ISL": str(point["isl"]), - "OSL": str(point["osl"]), + "IS_AGENTIC": "1" if agentic else "0", + "ISL": "0" if agentic else str(point["isl"]), + "OSL": "0" if agentic else str(point["osl"]), "RANDOM_RANGE_RATIO": RANDOM_RANGE_RATIO, + "THINKING_MODE": THINKING_MODE, + "RUN_EVAL": "false", "EVAL_ONLY": "false", } if "prefill" in point: - environment["CONC_LIST"] = " ".join(map(str, point["conc"])) + environment.update( + IS_MULTINODE="true", + SRT_RECIPE=point["srt-recipe"], + POWER="1" if point.get("power") else "0", + CONC_LIST=" ".join(map(str, point["conc"])), + ) + if agentic: + # A multi-node AgentX job serves its one concurrency. + environment["CONC"] = str(point["conc"][0]) return environment return { **environment, @@ -50,17 +68,87 @@ def _environment(point: Mapping[str, Any]) -> dict[str, str]: "PCP_SIZE": str(point["pcp-size"]), "DP_ATTENTION": str(point["dp-attn"]).lower(), "GPU_COUNT": str(point["tp"] * point["pp"] * point["pcp-size"]), + "KV_OFFLOADING": str(point["kv-offloading"]) if agentic else "", + "TOTAL_CPU_DRAM_GB": str(point["total-cpu-dram-gb"]) if agentic else "0", } +def _placement(inventory: RunnerInventory, label: str) -> tuple[str, Cluster] | None: + """A runner of ``label`` and its cluster, when every runner of ``label`` is on one cluster.""" + runners = inventory.labels.get(label, []) + if len({inventory.cluster_for(runner).id for runner in runners}) != 1: + return None + return runners[0], inventory.cluster_for(runners[0]) + + +def _binder_inputs( + point: Mapping[str, Any], + environment: Mapping[str, str], + placement: tuple[str, Cluster] | None, + root: Path, +) -> tuple[int | None, dict[str, str]]: + """The DCGM exporter port and AgentX client paths the launcher binds a multi-node point with. + + Raises ``ValueError`` when they depend on a cluster the runner label does not single out, + or the cluster's lanes refuse the point. + """ + agentic = environment["IS_AGENTIC"] == "1" + if "prefill" not in point or not (agentic or point.get("power")): + return None, {} + if placement is None: + raise ValueError( + f"runner {point['runner']!r} must schedule on one cluster to bind the cluster's " + "power telemetry and AgentX client paths" + ) + from infx.launch.context import LaunchError + from infx.launch.drivers.srt import config, lanes, power + from infx.launch.policy import launch_path + from infx.launch.request import MultiNodeRequest + + runner, cluster = placement + srt = slurm_settings(cluster).srt_slurm + if srt is None: + raise ValueError(f"cluster {cluster.id!r} has no slurm.srt-slurm settings") + # The job reads these; the binder inputs depend on none of them. + runtime = { + "RUNNER_NAME": runner, + "GITHUB_WORKSPACE": str(root), + "RESULT_FILENAME": point["exp-name"], + } + request = MultiNodeRequest.from_env({**environment, **runtime}) + try: + path = launch_path(cluster.id, request) + lane = lanes.srt_lane(cluster.id, path) + decision = power.decide_power(cluster.id, path, request) + client_env = config.agentic_client_env(cluster, srt, lane, request) if agentic else {} + except LaunchError as error: + raise ValueError(f"cluster {cluster.id!r}: {error}") from error + return (srt.power_exporter_port if decision.dcgm else None), client_env + + def _bound_variant( - point: Mapping[str, Any], environment: Mapping[str, str], root: Path + point: Mapping[str, Any], + environment: Mapping[str, str], + root: Path, + *, + power_port: int | None, + client_env: Mapping[str, str], ) -> tuple[str | None, dict[str, Any]]: - """The variant the launcher submits for ``point``, bound.""" + """The variant the launcher submits for ``point``, bound. + + Multi-node variants get the DCGM telemetry block on ``power_port`` and the client + paths ``client_env``, the binder inputs the launcher computes. + """ if "prefill" not in point: selected, recipe = select_recipe(str(root / point["srt-recipe"]), environment, root=root) return selected.partition(":")[2] or None, recipe - return bind_multinode(str(root / point["srt-recipe"]), environment, root=root) + return bind_multinode( + str(root / point["srt-recipe"]), + environment, + root=root, + power_port=power_port, + client_env=client_env, + ) def _apply_acceptance_and_validate( @@ -92,7 +180,11 @@ def _apply_acceptance_and_validate( def generate_recipes( *, config_keys: list[str], config_files: list[Path], runner_file: Path, output: Path ) -> dict[str, Any]: - """Bind every fixed-sequence point of ``config_keys``; write recipes and a manifest.""" + """Bind every srt-slurm point of ``config_keys``; write recipes and a manifest. + + A multi-node point is bound with its cluster's DCGM exporter port and AgentX client + paths; the manifest names that cluster, the one its runner label schedules on. + """ if output.exists() and (not output.is_dir() or any(output.iterdir())): raise ValueError(f"Output directory must be empty: {output}") files = [str(path) for path in config_files] @@ -104,23 +196,30 @@ def generate_recipes( sys.path.insert(0, str(source)) master = load_config_files(files) runners = load_runner_file(str(runner_file)) + inventory = load_inventory(runner_file) recipes: dict[str, dict[str, Any]] = {} records = [] for key in expand_config_keys(config_keys, master): - points = generate_config_matrix( - [key], master, runners, scenario_types=["fixed-seq-len"], eval_mode="none", root=root - ) + points = generate_config_matrix([key], master, runners, eval_mode="none", root=root) + points = [point for point in points if point.get("srt-recipe")] if not points: - raise ValueError(f"{key} has no fixed-sequence points") + raise ValueError(f"{key} has no srt-slurm points") for point in points: environment = _environment(point) - variant, recipe = _bound_variant(point, environment, root) + placement = _placement(inventory, point["runner"]) + power_port, client_env = _binder_inputs(point, environment, placement, root) + variant, recipe = _bound_variant( + point, environment, root, power_port=power_port, client_env=client_env + ) _apply_acceptance_and_validate(recipe, environment, point["srt-recipe"]) identity = json.dumps({"point": point, "variant": variant}, sort_keys=True) digest = hashlib.sha256(identity.encode()).hexdigest()[:12] name = f"{re.sub(r'[^A-Za-z0-9_.-]', '_', key)}-{digest}.yaml" recipes[name] = recipe - records.append({"file": name, "config-key": key, "variant": variant, "matrix": point}) + records.append({ + "file": name, "config-key": key, "variant": variant, + "cluster": placement[1].id if placement else None, "matrix": point, + }) # fmt: skip output.mkdir(parents=True, exist_ok=True) for name, recipe in recipes.items(): (output / name).write_text(yaml.safe_dump(recipe, sort_keys=False)) diff --git a/inferencex-e2e/infx/tests/srt_slurm/test_generate.py b/inferencex-e2e/infx/tests/srt_slurm/test_generate.py index 19f5d6bfb4..77a7bc3103 100644 --- a/inferencex-e2e/infx/tests/srt_slurm/test_generate.py +++ b/inferencex-e2e/infx/tests/srt_slurm/test_generate.py @@ -1,4 +1,4 @@ -"""``infx generate``: bound fixed-sequence recipes of master-config points, validated by srtctl.""" +"""``infx generate``: bound srt-slurm recipes of master-config points, validated by srtctl.""" import json from pathlib import Path @@ -7,6 +7,8 @@ import yaml from infx.cli import main +from infx.launch.drivers.srt import lanes, power +from infx.launch.policy import LaunchPath, Match ROOT = Path(__file__).resolve().parents[3] FRAGMENT = { @@ -20,43 +22,111 @@ }, "zip_override_conc": {"benchmark": {"env": {"CONC": ["2", "4"]}}}, } # fmt: skip -MASTER = {"fixture-sglang": { - "image": "org/image:1", "model": "org/model", "model-prefix": "dsr1", "precision": "fp8", - "framework": "sglang", "runner": "fixture", "multinode": False, "srt-recipe-dir": "fixture", - "scenarios": {"fixed-seq-len": [{"isl": 1024, "osl": 128, "search-space": [ - {"tp": 4, "conc-start": 2, "conc-end": 4, "spec-decoding": "mtp", "srt-recipe": "recipe.yaml"}, - ]}]}, -}} # fmt: skip +RECIPE = { + "schema": 2, "name": "agentic", "engine": "sglang", + "resources": {"gpu_type": "h200", "gpus_per_node": 8}, +} # fmt: skip +ROLE = {"num-worker": 1, "tp": 8, "ep": 1, "dp-attn": False} +MASTER = { + "fixture-sglang": { + "image": "org/image:1", "model": "org/model", "model-prefix": "dsr1", "precision": "fp8", + "framework": "sglang", "runner": "fixture", "multinode": False, "srt-recipe-dir": "fixture", + "scenarios": {"fixed-seq-len": [{"isl": 1024, "osl": 128, "search-space": [ + {"tp": 4, "conc-start": 2, "conc-end": 4, "spec-decoding": "mtp", "srt-recipe": "recipe.yaml"}, + ]}]}, + }, + "agentx-single": { + "image": "org/image:1", "model": "org/model", "model-prefix": "dsr1", "precision": "fp8", + "framework": "sglang", "runner": "cluster:fixture", "multinode": False, + "srt-recipe-dir": "fixture", "scenarios": {"agentic-coding": [{"search-space": [ + {"tp": 4, "conc-list": [2], "kv-offloading": "none", "srt-recipe": "agentic.yaml"}, + ]}]}, + }, + "agentx-multi": { + "image": "org/image:1", "model": "org/model", "model-prefix": "dsr1", "precision": "fp8", + "framework": "dynamo-sglang", "runner": "cluster:fixture", "multinode": True, + "disagg": True, "kv-p2p-transfer": "nixl", "srt-recipe-dir": "fixture", + "scenarios": {"agentic-coding": [{"search-space": [{ + "prefill": ROLE, "decode": ROLE, "conc-list": [8], "srt-recipe": "agentic.yaml", + "power": True, + }]}]}, + }, +} # fmt: skip +SHARED = { + "fixed-sequence-single.yaml": {"benchmark": {"type": "custom", "command": "bash client.sh"}}, + "agentic-single.yaml": {"benchmark": {"type": "custom", "command": "bash srt_agentic.sh"}}, + "agentic-multi.yaml": {"benchmark": {"type": "custom", "command": "bash srt_agentic.sh"}}, + "telemetry-dcgm.yaml": {"telemetry": {"enabled": True, "dcgm_exporter": { + "container_image": "dcgm-exporter", "command": "dcgm-exporter --address :{port}", + }}}, +} # fmt: skip @pytest.fixture def project(tmp_path): - (tmp_path / "configs/srt-recipes").mkdir(parents=True) - (tmp_path / "configs/srt-recipes/fixed-sequence-single.yaml").write_text( - yaml.safe_dump({"benchmark": {"type": "custom", "command": "bash client.sh"}}) - ) + for name, block in SHARED.items(): + (tmp_path / "configs/srt-recipes" / name).parent.mkdir(parents=True, exist_ok=True) + (tmp_path / "configs/srt-recipes" / name).write_text(yaml.safe_dump(block)) (tmp_path / "configs/master.yaml").write_text(yaml.safe_dump(MASTER)) (tmp_path / "configs/runners.yaml").write_text(yaml.safe_dump({ "labels": {"fixture": ["fixture_0"], "cluster:fixture": ["fixture_0"]}, - "clusters": {"fixture": {"gpus-per-node": 8, "arch": "x86_64", "scheduler": "slurm", - "slurm": {"partition": "p", "exclusive": True}}}, + "clusters": {"fixture": {"gpus-per-node": 8, "arch": "x86_64", "scheduler": "slurm", "slurm": { + "partition": "p", "exclusive": True, + "volumes": {"hf-hub-cache": {"path": "/srv/hf"}, "aiperf-cache": {"path": "/srv/aiperf"}}, + "srt-slurm": { + "network-interface": "eth0", "power-exporter-port": 9555, + "volume-mounts": {"hf-hub-cache": "/hf"}, + "agentic-volume-mounts": {"aiperf-cache": "/aiperf"}, + }, + }}}, })) # fmt: skip - recipe = tmp_path / "benchmarks/single_node/srt-slurm-recipes/fixture/recipe.yaml" - recipe.parent.mkdir(parents=True) - recipe.write_text(yaml.safe_dump(FRAGMENT)) + recipes = { + "single_node/srt-slurm-recipes/fixture/recipe.yaml": FRAGMENT, + "single_node/srt-slurm-recipes/fixture/agentic.yaml": { + **RECIPE, "frontend": {"type": "sglang", "enable_multiple_frontends": False}, + "roles": {"agg": {"nodes": 1, "workers": 1, "gpus": 4, "args": {"tensor-parallel-size": 4}}}, + }, + "multi_node/srt-slurm-recipes/fixture/agentic.yaml": { + **RECIPE, "frontend": {"type": "sglang-router"}, "roles": { + role: {"nodes": 1, "workers": 1, "gpus": 8, "args": {"tensor-parallel-size": 8}} + for role in ("prefill", "decode") + }, + }, + } # fmt: skip + for path, data in recipes.items(): + (tmp_path / "benchmarks" / path).parent.mkdir(parents=True, exist_ok=True) + (tmp_path / "benchmarks" / path).write_text(yaml.safe_dump(data)) (tmp_path / "utils/srt-slurm").mkdir(parents=True) (tmp_path / "utils/srt-slurm/src").symlink_to(ROOT / "utils/srt-slurm/src") return tmp_path -def generate(project: Path, output: Path) -> int: +@pytest.fixture +def power_lane(monkeypatch): + """The fixture cluster's multi-node lane, measuring AgentX power.""" + monkeypatch.setitem( + lanes.SRT_LANES, ("fixture", LaunchPath.SRT_MULTI), lanes.SrtLane(agentic_result_dir="/results") + ) + rules = (power.PowerRule(Match(agentic=True), agentx=True),) + monkeypatch.setitem( + power.POWER_LANES, ("fixture", LaunchPath.SRT_MULTI), power.PowerLane(rules, error="no power") + ) + + +def generate(project: Path, output: Path, key: str = "fixture-*") -> int: return main([ - "generate", "--config-key", "fixture-*", "--output-dir", str(output), + "generate", "--config-key", key, "--output-dir", str(output), "--config-file", str(project / "configs/master.yaml"), "--runner-config", str(project / "configs/runners.yaml"), ]) # fmt: skip +def written(output: Path) -> list[tuple[dict, dict]]: + """Each manifest record with the recipe it names.""" + records = json.loads((output / "manifest.json").read_text())["recipes"] + return [(record, yaml.safe_load((output / record["file"]).read_text())) for record in records] + + def test_each_point_gets_its_bound_variant_and_a_manifest_entry(project, tmp_path): output = tmp_path / "out" assert generate(project, output) == 0 @@ -76,6 +146,50 @@ def test_each_point_gets_its_bound_variant_and_a_manifest_entry(project, tmp_pat assert recipe["roles"]["agg"]["env"] == {} +def test_a_single_node_agentx_point_gets_the_agentic_client_and_no_sequence_lengths( + project, tmp_path +): + output = tmp_path / "out" + assert generate(project, output, "agentx-single") == 0 + + [(record, recipe)] = written(output) + assert (record["variant"], record["cluster"]) == (None, "fixture") + assert recipe["model"] == {"path": "hf:org/model", "container": "org/image:1", "precision": "fp8"} + assert recipe["benchmark"] == { + "type": "custom", "command": "bash srt_agentic.sh", "env": {"MODEL": "org/model", "CONC": "2"}, + } # fmt: skip + + +def test_a_multinode_agentx_power_point_gets_its_clusters_exporter_port_and_client_paths( + project, power_lane, tmp_path +): + output = tmp_path / "out" + assert generate(project, output, "agentx-multi") == 0 + + [(record, recipe)] = written(output) + assert record["cluster"] == "fixture" + assert recipe["telemetry"] == {"enabled": True, "dcgm_exporter": { + "container_image": "dcgm-exporter", "command": "dcgm-exporter --address :{port}", "port": 9555, + }} # fmt: skip + # The lane's result directory, the AgentX-only aiperf mount and the cluster-wide Hub cache. + assert recipe["benchmark"]["env"] == { + "RESULT_DIR": "/results", "AIPERF_DATASET_MMAP_CACHE_DIR": "/aiperf", "HF_HUB_CACHE": "/hf", + } # fmt: skip + assert recipe["benchmark"]["concurrencies"] == [8] + + +def test_a_power_point_its_lane_refuses_fails_without_writing( + project, power_lane, monkeypatch, tmp_path, capsys +): + refusing = power.PowerLane(rules=(), error="this lane measures no AgentX power") + monkeypatch.setitem(power.POWER_LANES, ("fixture", LaunchPath.SRT_MULTI), refusing) + output = tmp_path / "out" + with pytest.raises(SystemExit): + generate(project, output, "agentx-multi") + assert "this lane measures no AgentX power" in capsys.readouterr().err + assert not output.exists() + + def test_output_must_be_new_or_empty(project, tmp_path, capsys): output = tmp_path / "out" output.mkdir() From b46382438fdbc38982504a66f895bd1ae333b29b Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Thu, 8 Oct 2026 11:14:43 -0500 Subject: [PATCH 03/14] refactor(srt): compose the DSv4.1-Flash ATOM AgentX recipe from a fragment so binding it fails on its model block and benchmark MODEL. Like the other AgentX fragments it drops them, the client command its shared block supplies, and WEKA_LOADER_OVERRIDE, which names the corpus dsv41flash replays by default. Every point binds to main's selected variant without that override. --- .../dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml index 39f78b50d4..5a209ef4ce 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml @@ -5,10 +5,6 @@ base: schema: 2 name: dsv41flash-fp4-mi355x-atom-agentic - model: - path: hf:deepseek-ai/DeepSeek-V4.1-Flash - container: rocm/atom-dev:nightly_202610121516 - precision: fp4 resources: gpu_type: mi355x gpus_per_node: 8 @@ -52,12 +48,7 @@ base: ATOM_DISABLE_MMAP: 'true' AITER_LOG_LEVEL: WARNING benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - MODEL: deepseek-ai/DeepSeek-V4.1-Flash - # Full-context corpus for this 1M-context recipe. - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 # The upstream one-hour profile warms five requests per lane. AIPERF_WARMUP_REQUESTS_PER_LANE: '5' # Long AgentX stalls exceed AIPerf's 30 s TCP_USER_TIMEOUT default. From cb352ec85cc408dc32b7f6b5eb165b7e890e25a3 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 9 Oct 2026 12:20:50 -0500 Subject: [PATCH 04/14] refactor(srt): deslop AgentX composition The multi-node binder reads IS_AGENTIC instead of defaulting a missing value to fixed-sequence, model_paths drops a guard the binder makes redundant, and RECIPES.md points at the bound-key tables instead of copying them. --- .../benchmarks/multi_node/srt-slurm-recipes/RECIPES.md | 2 +- inferencex-e2e/infx/launch/drivers/srt/models.py | 2 +- inferencex-e2e/infx/launch/drivers/srt/recipe.py | 1 - inferencex-e2e/infx/srt_slurm/workload.py | 2 +- inferencex-e2e/infx/tests/srt_slurm/test_workload.py | 2 +- 5 files changed, 4 insertions(+), 5 deletions(-) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/RECIPES.md b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/RECIPES.md index ffe76d7e3c..564a604c82 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/RECIPES.md +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/RECIPES.md @@ -57,7 +57,7 @@ Active recipes, fixed-sequence and AgentX, single- and multi-node, are fragments 2. The row's selector picks the variant. A single-node variant may name its `benchmark.env.CONC`, which keeps that concurrency paired with its tuning. 3. The binder ([`workload.py`](../../../infx/srt_slurm/workload.py)) writes the matrix point into the selected recipe: `model.path: hf:`, `model.container: `, `model.precision`, `identity.container.image` (as a registry reference) and `identity.model.repo` when the fragment declares `identity.container`/`identity.model`, and `benchmark.concurrencies` when telemetry is enabled. Fixed-sequence recipes also get `benchmark.env.ISL`/`OSL`; single-node recipes get `MODEL` and `CONC`, and fixed-sequence ones `RANDOM_RANGE_RATIO` and `USE_CHAT_TEMPLATE` (`true` exactly when the recipe speculates). Multi-node AgentX recipes get the client's `RESULT_DIR`, `AIPERF_DATASET_MMAP_CACHE_DIR` and `HF_HUB_CACHE` from the launcher, which derives them from the volumes it mounts (`volume-mounts`, `agentic-volume-mounts`, lane mounts); a fragment that sets `HF_HOME` keeps its own cache. The multi-node client reads `CONC_LIST`, `CONC`, `MODEL` and the other matrix inputs from the job environment, so fragments do not copy them; single-node AgentX points get them as runtime arguments. -A fragment that sets any of these bound keys, `telemetry.enabled`, `benchmark.env.CONC_LIST`, or an AgentX client path (`RESULT_DIR`, `AGENTIC_OUTPUT_DIR`, `HF_HUB_CACHE`, `HUGGINGFACE_HUB_CACHE`, and on multi-node `AIPERF_DATASET_MMAP_CACHE_DIR`) fails before submission, even when the value matches. `hf:` resolves to the cluster's staged checkpoint (`models.entries`) unless a `models.OVERRIDES` row serves the Hub snapshot, and the master image to its staged container. A multi-node fixed-sequence point with neither checkpoint nor Hub row fails before submission. +A fragment that sets a key the binder or launcher writes (`_POINT_KEYS`, `BOUND_ENV` in `workload.py`; single-node variants may still name `CONC`) fails before submission, even with the bound value. `hf:` resolves to the cluster's staged checkpoint (`models.entries`) unless a `models.OVERRIDES` row serves the Hub snapshot, and the master image to its staged container. A multi-node fixed-sequence point with neither checkpoint nor Hub row fails before submission. Inspect what the launcher submits without a cluster: diff --git a/inferencex-e2e/infx/launch/drivers/srt/models.py b/inferencex-e2e/infx/launch/drivers/srt/models.py index 027d910fa0..126e983887 100644 --- a/inferencex-e2e/infx/launch/drivers/srt/models.py +++ b/inferencex-e2e/infx/launch/drivers/srt/models.py @@ -166,7 +166,7 @@ def served_path(cluster: Cluster, request: LaunchRequest, model: Checkpoint | No def model_paths(cluster: Cluster, request: LaunchRequest, served: str | None) -> dict[str, str]: """srtslurm.yaml ``model_paths``: the bound ``hf:`` to ``served``.""" - if not request.model or _override(cluster, request, "hub"): + if _override(cluster, request, "hub"): return {} if served is None: if not request.is_agentic: diff --git a/inferencex-e2e/infx/launch/drivers/srt/recipe.py b/inferencex-e2e/infx/launch/drivers/srt/recipe.py index d9733b7369..afd95cc3ab 100644 --- a/inferencex-e2e/infx/launch/drivers/srt/recipe.py +++ b/inferencex-e2e/infx/launch/drivers/srt/recipe.py @@ -108,4 +108,3 @@ def eval_overrides(recipes: Path, lane: SrtLane, request: LaunchRequest) -> list for key in lane.eval_unsets: overrides += ["--unset", key] return overrides - diff --git a/inferencex-e2e/infx/srt_slurm/workload.py b/inferencex-e2e/infx/srt_slurm/workload.py index c0f0ef7509..32323cb3f2 100644 --- a/inferencex-e2e/infx/srt_slurm/workload.py +++ b/inferencex-e2e/infx/srt_slurm/workload.py @@ -223,7 +223,7 @@ def bind_multinode( client_env: Mapping[str, str] | None = None, ) -> tuple[str | None, dict[str, Any]]: """Bind the one variant ``recipe`` (``fragment[:selector]``) selects; return its name too.""" - agentic = environment.get("IS_AGENTIC") == "1" + agentic = environment["IS_AGENTIC"] == "1" path, _, selector = recipe.partition(":") composed = compose_recipe( Path(path), agentic=agentic, multinode=True, root=root, power_port=power_port diff --git a/inferencex-e2e/infx/tests/srt_slurm/test_workload.py b/inferencex-e2e/infx/tests/srt_slurm/test_workload.py index 33d1c001b2..4992e79d1c 100644 --- a/inferencex-e2e/infx/tests/srt_slurm/test_workload.py +++ b/inferencex-e2e/infx/tests/srt_slurm/test_workload.py @@ -201,7 +201,7 @@ def test_the_multinode_binder_writes_the_one_selected_variant(project, tmp_path) "override_narrow": {"roles": {"decode": {"nodes": 2}}}, }) output = tmp_path / "bound.yaml" - env = {**os.environ, **MULTI_ENV, "INFERENCEX_REPOSITORY_ROOT": str(project), + env = {**os.environ, **MULTI_ENV, "IS_AGENTIC": "0", "INFERENCEX_REPOSITORY_ROOT": str(project), "PYTHONPATH": os.pathsep.join([str(ROOT), str(ROOT / "utils/srt-slurm/src")])} # fmt: skip def bind(*arguments: str) -> subprocess.CompletedProcess[str]: From 0710e68a7a17953bb6562a66133014d0db028451 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 9 Oct 2026 13:12:07 -0500 Subject: [PATCH 05/14] refactor(srt): drop checks and env placeholders the binder makes dead select_recipe binds each variant before validate_recipe sees it, and the binder writes model.path, model.container, model.precision and benchmark.env.MODEL from the same environment, so those four comparisons can never fail. infx generate set KV_OFFLOADING and TOTAL_CPU_DRAM_GB to empty or zero placeholders for fixed-sequence points that nothing reads; it now sets them only for AgentX points. --- inferencex-e2e/infx/srt_slurm/generate.py | 7 +++++-- inferencex-e2e/infx/srt_slurm/single_node.py | 4 ---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/inferencex-e2e/infx/srt_slurm/generate.py b/inferencex-e2e/infx/srt_slurm/generate.py index d4eb74b133..fb0fad930b 100644 --- a/inferencex-e2e/infx/srt_slurm/generate.py +++ b/inferencex-e2e/infx/srt_slurm/generate.py @@ -47,6 +47,11 @@ def _environment(point: Mapping[str, Any]) -> dict[str, str]: "RUN_EVAL": "false", "EVAL_ONLY": "false", } + if agentic: + environment.update( + KV_OFFLOADING=str(point["kv-offloading"]), + TOTAL_CPU_DRAM_GB=str(point["total-cpu-dram-gb"]), + ) if "prefill" in point: environment.update( IS_MULTINODE="true", @@ -68,8 +73,6 @@ def _environment(point: Mapping[str, Any]) -> dict[str, str]: "PCP_SIZE": str(point["pcp-size"]), "DP_ATTENTION": str(point["dp-attn"]).lower(), "GPU_COUNT": str(point["tp"] * point["pp"] * point["pcp-size"]), - "KV_OFFLOADING": str(point["kv-offloading"]) if agentic else "", - "TOTAL_CPU_DRAM_GB": str(point["total-cpu-dram-gb"]) if agentic else "0", } diff --git a/inferencex-e2e/infx/srt_slurm/single_node.py b/inferencex-e2e/infx/srt_slurm/single_node.py index 7b01a39634..66ecaec20c 100644 --- a/inferencex-e2e/infx/srt_slurm/single_node.py +++ b/inferencex-e2e/infx/srt_slurm/single_node.py @@ -107,16 +107,12 @@ def validate_recipe(recipe: dict[str, Any], environment: Mapping[str, str]) -> N agentic = environment["IS_AGENTIC"] == "1" expected = { "engine": (engine, SINGLE_NODE_ENGINES[environment["FRAMEWORK"]]), - "model": (recipe["model"]["path"], f"hf:{environment['MODEL']}"), - "image": (recipe["model"]["container"], environment["IMAGE"]), - "precision": (recipe["model"]["precision"], environment["PRECISION"]), **parallelism_constraints(engine, args, environment), "gpus": (role["gpus"], int(environment["GPU_COUNT"])), "nodes": (role["nodes"], 1), "workers": (role["workers"], 1), "roles": (set(recipe["roles"]), {"agg"}), "benchmark type": (benchmark["type"], "custom"), - "benchmark MODEL": (workload["MODEL"], environment["MODEL"]), # draft_model names a bundled or separate draft; its recipes speculate natively. "SPEC_DECODING": ( speculation, From 7eb436917962416e8d117aace223f57d2bb1a4a6 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 9 Oct 2026 13:12:17 -0500 Subject: [PATCH 06/14] refactor(srt): share the multi-node binder inputs with infx generate The launcher and infx generate each derived the DCGM exporter port and the AgentX client paths from the cluster, lane, request and power decision, and infx generate must match the launcher. Both now call config.binder_inputs. infx generate imports the launcher modules at the top: they import nothing that imports it back. --- inferencex-e2e/infx/launch/drivers/srt/__init__.py | 5 +---- inferencex-e2e/infx/launch/drivers/srt/config.py | 12 ++++++++++++ inferencex-e2e/infx/srt_slurm/generate.py | 11 +++++------ 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/inferencex-e2e/infx/launch/drivers/srt/__init__.py b/inferencex-e2e/infx/launch/drivers/srt/__init__.py index 0b0c74875a..70d32a3c24 100644 --- a/inferencex-e2e/infx/launch/drivers/srt/__init__.py +++ b/inferencex-e2e/infx/launch/drivers/srt/__init__.py @@ -162,10 +162,7 @@ def run_multinode(launch: Launch) -> int: job_name = srtctl_job_name(request.runner_name) prepare_recipe(checkout.root, staged, job_name, run.srt.dist_timeout_s) recipe = str(checkout.root / BOUND_RECIPE) - client_env = ( - config.agentic_client_env(run.cluster, run.srt, lane, request) if request.is_agentic else {} - ) - power_port = run.srt.power_exporter_port if decision.dcgm else None + power_port, client_env = config.binder_inputs(run.cluster, run.srt, lane, request, decision) if rc := submit.bind_recipe(run, checkout, staged, recipe, power_port, client_env): return rc arguments = submit.multinode_arguments(run, lane, recipe, overrides, preflight=preflight) diff --git a/inferencex-e2e/infx/launch/drivers/srt/config.py b/inferencex-e2e/infx/launch/drivers/srt/config.py index 79dd8e8614..f2d48040ff 100644 --- a/inferencex-e2e/infx/launch/drivers/srt/config.py +++ b/inferencex-e2e/infx/launch/drivers/srt/config.py @@ -231,6 +231,18 @@ def agentic_client_env( return env +def binder_inputs( + cluster: Cluster, + srt: SrtSlurmSettings, + lane: SrtLane, + request: SrtRequest, + decision: PowerDecision, +) -> tuple[int | None, dict[str, str]]: + """The DCGM exporter port and AgentX client paths a multi-node recipe is bound with.""" + client_env = agentic_client_env(cluster, srt, lane, request) if request.is_agentic else {} + return (srt.power_exporter_port if decision.dcgm else None), client_env + + def write_lane_config( run: SrtRun, lane: SrtLane, diff --git a/inferencex-e2e/infx/srt_slurm/generate.py b/inferencex-e2e/infx/srt_slurm/generate.py index fb0fad930b..b250a16345 100644 --- a/inferencex-e2e/infx/srt_slurm/generate.py +++ b/inferencex-e2e/infx/srt_slurm/generate.py @@ -15,6 +15,10 @@ from infx.clusters import load_inventory from infx.clusters.slurm import slurm_settings +from infx.launch.context import LaunchError +from infx.launch.drivers.srt import config, lanes, power +from infx.launch.policy import launch_path +from infx.launch.request import MultiNodeRequest from infx.matrix.generate import expand_config_keys, generate_config_matrix from infx.matrix.validation import config_root, load_config_files, load_runner_file from infx.srt_slurm.single_node import select_recipe @@ -103,10 +107,6 @@ def _binder_inputs( f"runner {point['runner']!r} must schedule on one cluster to bind the cluster's " "power telemetry and AgentX client paths" ) - from infx.launch.context import LaunchError - from infx.launch.drivers.srt import config, lanes, power - from infx.launch.policy import launch_path - from infx.launch.request import MultiNodeRequest runner, cluster = placement srt = slurm_settings(cluster).srt_slurm @@ -123,10 +123,9 @@ def _binder_inputs( path = launch_path(cluster.id, request) lane = lanes.srt_lane(cluster.id, path) decision = power.decide_power(cluster.id, path, request) - client_env = config.agentic_client_env(cluster, srt, lane, request) if agentic else {} + return config.binder_inputs(cluster, srt, lane, request, decision) except LaunchError as error: raise ValueError(f"cluster {cluster.id!r}: {error}") from error - return (srt.power_exporter_port if decision.dcgm else None), client_env def _bound_variant( From 49ce6aaffc5642f34bd28db1c4be73281ef281e1 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 9 Oct 2026 13:12:17 -0500 Subject: [PATCH 07/14] test(srt): drop binder tests that echo their inputs or repeat launcher tests The bound AgentX client test asserted values it passed in, and the CLI test's power and client-env tail repeated the srt driver tests that run the same module with those flags. The fragment-rejection test no longer pins the explanatory suffix, and an unused constant goes. --- .../infx/tests/launch/test_srt_policy.py | 1 - .../infx/tests/srt_slurm/test_workload.py | 36 +++---------------- 2 files changed, 4 insertions(+), 33 deletions(-) diff --git a/inferencex-e2e/infx/tests/launch/test_srt_policy.py b/inferencex-e2e/infx/tests/launch/test_srt_policy.py index 46b5111cb3..26146c4339 100644 --- a/inferencex-e2e/infx/tests/launch/test_srt_policy.py +++ b/inferencex-e2e/infx/tests/launch/test_srt_policy.py @@ -47,7 +47,6 @@ def cluster(tmp_path, single_node_models: str = "staged") -> Cluster: MULTI = dict(IS_MULTINODE="true") SINGLE = dict(IS_MULTINODE="false") -MIRROR = "benchmarks/multi_node/srt-slurm-recipes" @pytest.mark.parametrize(("cluster_id", "env", "path"), [ diff --git a/inferencex-e2e/infx/tests/srt_slurm/test_workload.py b/inferencex-e2e/infx/tests/srt_slurm/test_workload.py index 4992e79d1c..0ad6375142 100644 --- a/inferencex-e2e/infx/tests/srt_slurm/test_workload.py +++ b/inferencex-e2e/infx/tests/srt_slurm/test_workload.py @@ -1,6 +1,7 @@ """Composing recipe fragments with their shared blocks, and binding the matrix point.""" import os +import re import subprocess import sys from pathlib import Path @@ -8,7 +9,6 @@ import pytest import yaml -from infx.bench.agentic.run import Plan from infx.srt_slurm.synthetic_acceptance import selected_recipes from infx.srt_slurm.workload import bind_workload, compose_recipe, parse_concurrencies @@ -92,9 +92,8 @@ def test_bundles_take_the_shared_block_under_base_and_keep_their_variants(projec ]) # fmt: skip def test_a_fragment_that_sets_a_bound_key_is_rejected(project, data, agentic, multinode, reported): path = fragment(project, data) - with pytest.raises(ValueError) as error: + with pytest.raises(ValueError, match=re.escape(f"remove {reported} from the fragment")): compose_recipe(path, agentic=agentic, multinode=multinode, root=project) - assert str(error.value) == f"{path}: remove {reported} from the fragment; the launcher binds them" def test_multinode_binding_writes_the_point_and_leaves_the_client_its_job_environment(): @@ -156,27 +155,6 @@ def test_a_power_point_gets_telemetry_and_its_concurrencies_on_a_bundle_variant( ] -def test_a_bound_agentx_point_satisfies_the_benchmark_client(project, tmp_path): - path = fragment(project, {"benchmark": {"env": {"AIPERF_LIVE_FAILED_REQUEST_THRESHOLD": "0.25"}}}) - composed = compose_recipe(path, agentic=True, multinode=True, root=project) - bound = bind_workload(composed, AGENTX_ENV, agentic=True, multinode=True, client_env=CLIENT_ENV) - job = { - "RESULT_FILENAME": "r", "EVAL_ONLY": "false", "IS_MULTINODE": "true", "PRECISION": "fp4", - "MODEL": "org/model", "MODEL_PREFIX": "dsv4", "FRAMEWORK": "dynamo-vllm", "CONC": "8", - "CONC_LIST": "8", "DURATION": "3600", "AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS": "3600", - "AIPERF_EXPERIMENTAL_FAST": "0", "AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID": "false", - "AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING": "1", "ENABLE_AGENTX_POWER": "0", "REQUIRE_POWER": "0", - "IS_AGENTIC": "1", "KV_OFFLOADING": "none", "SRT_FRONTEND_HOST": "10.0.0.1", - "SRT_FRONTEND_PORT": "8000", - } # fmt: skip - - plan = Plan.from_env({**job, **bound["benchmark"]["env"]}) - - assert plan.result_dir == Path("/logs/agentic/conc_8") - assert plan.replay.url == "http://10.0.0.1:8000" - assert plan.replay.live_failed_request_threshold == "0.25" - - @pytest.mark.parametrize(("environment", "message"), [ ({**MULTI_ENV, "IMAGE": ""}, "Missing workload input: IMAGE"), ({**MULTI_ENV, "OSL": "1k"}, "OSL must be a positive integer: '1k'"), @@ -221,11 +199,5 @@ def bind(*arguments: str) -> subprocess.CompletedProcess[str]: assert "selects 2 variants, not one" in result.stderr assert not output.exists() env["IS_AGENTIC"] = "1" - arguments = ("--power-port", "9401", "--client-env", "RESULT_DIR=/logs/agentic") - assert bind(f"{path}:override_narrow", str(output), *arguments).returncode == 0 - bound = yaml.safe_load(output.read_text()) - assert (bound["benchmark"]["command"], bound["benchmark"]["concurrencies"]) == ( - "bash agentic-multi.sh", [4, 16], - ) - assert bound["benchmark"]["env"]["RESULT_DIR"] == "/logs/agentic" - assert bound["telemetry"]["dcgm_exporter"]["port"] == 9401 + assert bind(f"{path}:override_narrow", str(output)).returncode == 0 + assert yaml.safe_load(output.read_text())["benchmark"]["command"] == "bash agentic-multi.sh" From d37b018204bd5e554d1b8a2afa358f6e89a22a27 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 9 Oct 2026 13:12:26 -0500 Subject: [PATCH 08/14] docs(srt): condense infx generate guidance RECIPES.md states the infx generate binder inputs in fewer words and drops a sentence the power-row table already makes. CONFIGS.md puts mounts back beside volume-mounts, which it describes, and the GLM-5.2 GB200 power comment moves onto the power field. --- .../benchmarks/multi_node/srt-slurm-recipes/RECIPES.md | 4 ++-- inferencex-e2e/configs/CONFIGS.md | 8 ++++---- inferencex-e2e/configs/nvidia-master.yaml | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/RECIPES.md b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/RECIPES.md index 564a604c82..94cc115500 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/RECIPES.md +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/RECIPES.md @@ -65,7 +65,7 @@ Inspect what the launcher submits without a cluster: uv run --extra recipes infx generate --config-key 'dsr1-fp8-h200-*' --output-dir /tmp/recipes ``` -It writes one bound recipe per fixed-sequence or AgentX point, validated by the pinned srtctl, plus a `manifest.json` that maps each file to its matrix point and to the one cluster its runner label schedules on (`null` when the label spans several). Multi-node points get the binder inputs the launcher derives from that cluster: the DCGM telemetry block on its `power-exporter-port` when the lane allows the row's `power: true`, and the AgentX client paths; a point that needs them fails when its runner label does not name one cluster or the lane refuses its power. Launch-time edits such as the job name, health-check floor and runtime `--set` values are not applied. +It writes one bound recipe per fixed-sequence or AgentX point, validated by the pinned srtctl, plus a `manifest.json` that maps each file to its matrix point and to the one cluster its runner label schedules on (`null` when the label spans several). Multi-node points take their cluster's binder inputs (DCGM exporter port, AgentX client paths); a point that needs them fails unless its runner label names one cluster and the lane accepts the row's `power`. Launch-time edits such as the job name, health-check floor and runtime `--set` values are not applied. ## Migration and validation @@ -86,7 +86,7 @@ The initial migration also resolves compatibility issues that `srtctl migrate` c - SGLang Model Gateway recipes use `frontend.type: sglang-router`; in v2.36.0, `sglang` selects a direct worker without a router. - Duplicate YAML keys retain the value selected by the former PyYAML loader. -- DCGM telemetry uses `collect_interval_ms: 1000` instead of `provider` and `default_frequency`. The collector derives its shutdown budget; an explicit ten-second budget is too short for the current validator. Dedicated discovery-service placement is preserved from the original recipes. The pinned upstream runtime rejects telemetry with dedicated infrastructure nodes; this remains a power compatibility blocker rather than changing the original topology to satisfy validation. The binder sets `benchmark.concurrencies` from the point for power rows. +- DCGM telemetry uses `collect_interval_ms: 1000` instead of `provider` and `default_frequency`. The collector derives its shutdown budget; an explicit ten-second budget is too short for the current validator. Dedicated discovery-service placement is preserved from the original recipes. The pinned upstream runtime rejects telemetry with dedicated infrastructure nodes; this remains a power compatibility blocker rather than changing the original topology to satisfy validation. - DeepSeek-V4 vLLM benchmarks use the supported `custom_tokenizer` loader. Retired `warmup_req_rate: inf` fields are removed; the current upstream client uses its fixed warmup rate of 250 requests per second. - The power reader accepts both generations of samples CSV while validating utilization values and continuing to compute board energy from watts. - Post-eval selection uses native `post_eval.command` and `post_eval.passthrough_env` with [`srt_eval.sh`](../srt_eval.sh). TRT AgentX recipes declare their existing Dynamo fork with `dynamo.source.git`; launchers no longer rewrite the srt-slurm source. diff --git a/inferencex-e2e/configs/CONFIGS.md b/inferencex-e2e/configs/CONFIGS.md index dbdc40abbb..d429285f0f 100644 --- a/inferencex-e2e/configs/CONFIGS.md +++ b/inferencex-e2e/configs/CONFIGS.md @@ -243,10 +243,10 @@ schema; unknown keys fail. import step. Without `slurm.squash`, every job starts from the registry image, which Pyxis imports inside the job. - `slurm.srt-slurm` is the cluster's srt-slurm profile: `volume-mounts` maps - volumes every job mounts to container paths, `agentic-volume-mounts` those multi-node - AgentX jobs add (world-writable), `power-exporter-port` is the DCGM exporter port of - power points, `mounts` does the same for host - paths outside the volumes (devices), `env` is the environment of the srt-slurm + volumes every job mounts to container paths, `mounts` does the same for host + paths outside the volumes (devices), `agentic-volume-mounts` adds world-writable + volumes for multi-node AgentX jobs, `power-exporter-port` is the DCGM exporter port + of power points, `env` is the environment of the srt-slurm launch, `container-aliases` and `nginx-aliases` name the recipe containers that resolve to the main image and to the staged nginx, and `outputs`, `shared-run-root` and `uv-cache-root` are the directories the srt-slurm launcher diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index df7b8a9418..35d240f718 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -7362,8 +7362,8 @@ glm5.2-fp4-gb200-dynamo-sglang-agentic-agg: kv-offloading: dram kv-offload-backend: { name: hicache } num-nodes: 2 - # Shared deployment GPU power telemetry. srt-recipe: agg.yaml + # Shared deployment GPU power telemetry. power: true worker: num-worker: 1 From c5a4d90874f175c8c55be5a54fa987bfb80d1190 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 9 Oct 2026 13:12:26 -0500 Subject: [PATCH 09/14] docs: drop recipe image and telemetry rules the binder now owns Recipes may no longer set model.container, identity.container.image or telemetry.enabled, yet the Klaud prompt and docs still asked for matching images, nuke.md still ran a script that rewrites model.container (it now fails on every recipe), and the power docs still gated energy collection on recipe telemetry. They now say the binder writes the master image and a power: true master row enables DCGM telemetry. --- .agents/skills/debug-runs/SKILL.md | 7 +-- .claude/commands/add-model-hardware.md | 8 ++-- .claude/commands/nuke.md | 46 +------------------- .github/AGENT_OPERATIONS.md | 2 +- .github/klaud-candidate-prompt.md | 7 +-- AGENTS.md | 2 +- inferencex-e2e/docs/klaud.md | 2 +- inferencex-e2e/docs/results-and-ingestion.md | 2 +- 8 files changed, 18 insertions(+), 58 deletions(-) diff --git a/.agents/skills/debug-runs/SKILL.md b/.agents/skills/debug-runs/SKILL.md index 0bb77eb3ec..f116fc78a7 100644 --- a/.agents/skills/debug-runs/SKILL.md +++ b/.agents/skills/debug-runs/SKILL.md @@ -111,9 +111,10 @@ Steps: of `python -m infx.launch run` (`inferencex-e2e/infx/launch/drivers/srt/`): it binds the matrix row's `srt-recipe` (`inferencex-e2e/benchmarks/single_node/srt-slurm-recipes//`) with `python -m infx.srt_slurm.single_node prepare`, renders the job-local `srtslurm.yaml` - from the cluster's record, then submits it through srtctl. Read the recipe for the image - (`model.container`), server args and env, and the cluster record for mounts and the job env - (`IMAGE`, `TP`, `PRECISION`, `SPEC_DECODING`, `CONC`, …). On Slurm clusters, use + from the cluster's record, then submits it through srtctl. Read the bound `recipe.yaml` that + `prepare` writes for the image (`model.container`, the master `image`), server args and env, + and the cluster record for mounts and the job env (`IMAGE`, `TP`, `PRECISION`, + `SPEC_DECODING`, `CONC`, …). On Slurm clusters, use `salloc` or `srun` with the squash image. On the **bare-metal `-tw` pools, use `docker run`** on the node directly without `srun`. 3. **Always diff against a working node or working SKU** for reference. Most node failures diff --git a/.claude/commands/add-model-hardware.md b/.claude/commands/add-model-hardware.md index 65006a3280..ba1f9ea077 100644 --- a/.claude/commands/add-model-hardware.md +++ b/.claude/commands/add-model-hardware.md @@ -98,10 +98,10 @@ adjusted for this hardware's quirks." Copy the sibling recipe (`base:` plus one `override_*` variant per matrix point) and adjust. Engine flags are `roles.agg.args` keys without the leading `--`; env vars go in `roles.agg.env`; each variant names its `CONC` (and `KV_OFFLOADING` for AgentX) in -`benchmark.env`. Fixed-sequence recipes are fragments. Omit `model.path`/`container`/`precision`, -`benchmark.type`/`command` and the client env, which the shared block and binder supply (see -`RECIPES.md`). AgentX recipes need `model.container` equal to the master-config `image`. Things -that vary and must be checked against the sibling / +`benchmark.env`. Recipes, fixed-sequence and AgentX, are fragments. Omit +`model.path`/`container`/`precision`, `benchmark.type`/`command` and the client env, which the +shared block and binder supply (see `RECIPES.md`). Things that vary and must be checked against +the sibling / the model's `recipes.vllm.ai` page: - **Mandatory model flags** (carry from the sibling): block size, parser flags (`--tool-call-parser` / `--reasoning-parser`), `--language-model-only` for text-only sweeps, diff --git a/.claude/commands/nuke.md b/.claude/commands/nuke.md index 2c5556421e..e780faf915 100644 --- a/.claude/commands/nuke.md +++ b/.claude/commands/nuke.md @@ -68,7 +68,8 @@ Each PR triggers a full GPU sweep, so surface the total PR count explicitly. ## Step 4 — create one PR per family Use these helpers (write them to /tmp) for precise, per-config-key edits. A blind -`sed` is unsafe because the same old tag appears under many keys. +`sed` is unsafe because the same old tag appears under many keys. Only the master `image:` +changes: the srt-slurm binder writes it into the recipe. `/tmp/edit_image.py`: ```python @@ -104,48 +105,6 @@ block += [" description:", f' - "{desc}"', " pr-link: PRLINK_PLACEHOLDER"] open(f,'w').write(content + '\n' + '\n'.join(block) + '\n') ``` -`/tmp/edit_recipe_container.py` (single-node runs go through the srt-slurm -recipe each master search-space row names in `srt-recipe:`, and -`inferencex-e2e/infx/srt_slurm/single_node.py` rejects the run unless the recipe's -`model.container` equals the master `image:` exactly): -```python -#!/usr/bin/env python3 -# Usage: edit_recipe_container.py [key2 ...] -import os, re, sys -f, new_image, keys = sys.argv[1], sys.argv[2], sys.argv[3:] -# srt-recipe names a file under the key's srt-recipe-dir, below this root -root = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(f))), "benchmarks/single_node/srt-slurm-recipes") -master = open(f).read().split('\n') -recipes = set() -for key in keys: - kre = re.compile(r'^' + re.escape(key) + r':\s*$') - start = next((i for i,l in enumerate(master) if kre.match(l)), None) - if start is None: sys.exit(f"ERROR: key not found: {key}") - directory, names = None, set() - for j in range(start+1, len(master)): - if re.match(r'^[A-Za-z0-9._-]+:\s*$', master[j]): break # next top-level key - d = re.match(r'^\s+srt-recipe-dir:\s*(\S+)\s*$', master[j]) - if d: directory = d.group(1) - names.update(re.findall(r'(? {new_image}"); break - if not hit: sys.exit(f"ERROR: no model.container in {r}") - open(r, 'w').write('\n'.join(lines)) -``` - -If another key's `srt-recipe-dir` plus `srt-recipe` resolves to the same recipe file, stop and ask the user: -bumping it would break that other key's `model.container == image` check. - For each family, run strictly sequentially because git checkouts can't be parallel: ```bash @@ -153,7 +112,6 @@ git checkout main -q && git reset --hard origin/main -q branch="klaud/-" git checkout -b "$branch" -q python3 /tmp/edit_image.py [-mtp] -python3 /tmp/edit_recipe_container.py [-mtp] python3 /tmp/append_changelog.py inferencex-e2e/perf-changelog.yaml "" [-mtp] git add -A git commit -q -m "[Klaud Cold] Update [ (+mtp)] to " diff --git a/.github/AGENT_OPERATIONS.md b/.github/AGENT_OPERATIONS.md index 00daad47d3..43b5c8c8d9 100644 --- a/.github/AGENT_OPERATIONS.md +++ b/.github/AGENT_OPERATIONS.md @@ -74,7 +74,7 @@ Multinode disaggregated results add `prefill_gpu_energy_j`, `decode_gpu_energy_j Every power result, valid or invalid, carries `power_metric_schema_version`. Version 2 defines each unprefixed `joules_per_*` field as whole-deployment GPU-board energy over the named denominator; role-scoped energy uses the explicit `prefill_*` / `decode_*` keys. Rows without the field predate the whole-deployment switch and their unprefixed joules are not comparable across topologies. -For srt-slurm recipes, `telemetry.enabled: true` with `telemetry.dcgm_exporter` enables official energy collection. The Git submodule pointer at `inferencex-e2e/utils/srt-slurm` is the source of truth for every srt-slurm job, including TileRT. CI derives `POWER_PRODUCER_SHA` from the launcher stamp. The aggregate-power and AgentX power tests validate telemetry and provenance. These local tests do not prove hardware power collection. Eligible recipe-gated `dynamo-sglang` dcgm-power lanes are validated. +A multinode master row with `power: true` enables official energy collection: the launcher merges `inferencex-e2e/configs/srt-recipes/telemetry-dcgm.yaml` into its srt-slurm recipe. The Git submodule pointer at `inferencex-e2e/utils/srt-slurm` is the source of truth for every srt-slurm job, including TileRT. CI derives `POWER_PRODUCER_SHA` from the launcher stamp. The aggregate-power and AgentX power tests validate telemetry and provenance. These local tests do not prove hardware power collection. Eligible `dynamo-sglang` dcgm-power lanes are validated. Power audit artifacts are named `power_audit_` and contain the multinode `power_validation__*.json` sidecars. They are uploaded even when validation fails. diff --git a/.github/klaud-candidate-prompt.md b/.github/klaud-candidate-prompt.md index 00258416ff..cbfb9d2646 100644 --- a/.github/klaud-candidate-prompt.md +++ b/.github/klaud-candidate-prompt.md @@ -34,9 +34,10 @@ Check the exact selected launch path for existing engine patches as well as prop Edit only the family's master image and its already referenced, unshared recipe's source-backed compatibility flags/environment or srt-slurm YAML image/backend settings. -Match model.container and identity.container.image to the master. Preserve model, precision, -topology, speculation, workload/dataset, duration, resources, recipe references, all points -and default evals. No broad tuning, shared code/workflows or other-family edits. +Recipes never set model.container or identity.container.image; the binder writes both from +the master image. Preserve model, precision, topology, speculation, workload/dataset, +duration, resources, recipe references, all points and default evals. No broad tuning, +shared code/workflows or other-family edits. ZERO runtime engine/serving-stack patching: no source/site-packages/container rewrites, overlays, monkey patches or forked/rebuilt engine wheels, including pre-existing patches. If the shipped stack needs one, report incompatible. Preserve nodes:N, MTP chat templates, diff --git a/AGENTS.md b/AGENTS.md index 7f45b9c89d..ce4f7ab8cd 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -111,7 +111,7 @@ Deleting a test that fails these questions needs no replacement. Do not preserve - Every change that can affect benchmark performance and every recipe addition or modification requires a new `inferencex-e2e/perf-changelog.yaml` entry. The file is append-only and byte-sensitive. Preserve all existing bytes and separator whitespace, and append only at the tail. - One PR maps to one `inferencex-e2e/perf-changelog.yaml` block. By convention, a PR with a perf changelog entry is a change that affects inference performance or otherwise requires a rerun to collect up-to-date results. That block may list one or more `config-keys` to run, but do not add a second block for the same PR, even when the change evolved across commits. Revise the PR's existing block so it describes the final state of the change. - New `inferencex-e2e/perf-changelog.yaml` entries must be English-only. Do not add Chinese translations or bilingual descriptions; the bilingual README and GitHub-content rules do not apply to these entries. Leave historical entries unchanged. -- Multi-node srt-slurm changes update the recipe YAML and matching master config together. Fixed-sequence recipes are fragments that get the master image, model, precision and lengths from the binder; for AgentX image bumps, `model.container` must equal `image`. +- Multi-node srt-slurm changes update the recipe YAML and matching master config together. srt-slurm recipes, fixed-sequence and AgentX, are fragments that get the master image, model, precision (and fixed-sequence lengths) from the binder. - Every speculative fixed-sequence benchmark renders prompts with the chat template: the binder (`inferencex-e2e/infx/srt_slurm/workload.py::bind_workload`) sets `benchmark.env.USE_CHAT_TEMPLATE: "true"` for single-node recipes that speculate, which `python3 -m infx.bench fixed-seq` (run by `srt_fixed_sequence.sh`) turns into `--use-chat-template` for the benchmark client. - Benchmarks create no new directories under `/workspace`. Root containers must not leave root-owned files in shared AMD runner workspaces. - Generated configuration is not runtime proof. Run the narrowest local check, then the applicable smoke, sweep, or eval procedure from the [task routing table](inferencex-e2e/docs/index.md#task-routing). diff --git a/inferencex-e2e/docs/klaud.md b/inferencex-e2e/docs/klaud.md index 699c8efcf7..637c49f876 100644 --- a/inferencex-e2e/docs/klaud.md +++ b/inferencex-e2e/docs/klaud.md @@ -98,7 +98,7 @@ The app also has useful **unpublished UI readers**: `/api/unofficial-run` normal ## Change scope and PR policy -The authoritative [candidate instructions](../../.github/klaud-candidate-prompt.md) permit only the selected master image and source-backed compatibility flags/environment in its already referenced, unshared single-node recipe or srt-slurm YAML. Preserve model, precision, topology, speculation, workload/dataset, duration, resources, all points and evals. `model.container` and `identity.container.image` must match. No broad tuning or shared script/launcher/library/workflow edits by Klaud. ZERO runtime engine/serving-stack patching, including existing patches on the selected path: no source/container/site-packages rewrites, overlays, monkey patches or rebuilt/forked wheels. Use an unmodified supported image or report incompatible. +The authoritative [candidate instructions](../../.github/klaud-candidate-prompt.md) permit only the selected master image and source-backed compatibility flags/environment in its already referenced, unshared single-node recipe or srt-slurm YAML. Preserve model, precision, topology, speculation, workload/dataset, duration, resources, all points and evals. No broad tuning or shared script/launcher/library/workflow edits by Klaud. ZERO runtime engine/serving-stack patching, including existing patches on the selected path: no source/container/site-packages rewrites, overlays, monkey patches or rebuilt/forked wheels. Use an unmodified supported image or report incompatible. After smoke benchmarks and representative evals pass, append the changelog at its physical tail without altering prior bytes, push the exact head, generate/check the full matrix and recheck capacity. Keep DRAFT and apply `full-sweep-fail-fast` as the sole sweep label. A final failure stays draft without sweep labels before repair. Only `finish` marks ready after verified results and final reporting; readiness triggers reviews but no new sweep. Klaud then records the verified final run with `/use`; maintainers retain review, staging and merge authority. diff --git a/inferencex-e2e/docs/results-and-ingestion.md b/inferencex-e2e/docs/results-and-ingestion.md index 7d6b2c3a8f..c04261111e 100644 --- a/inferencex-e2e/docs/results-and-ingestion.md +++ b/inferencex-e2e/docs/results-and-ingestion.md @@ -186,7 +186,7 @@ Multinode runs retain the deployment telemetry under `LOGS/power/` and per-concurrency window/validation files under `LOGS/agentic/` in their `power_audit_` artifact. Available audits and AgentX aggregates upload even when a benchmark fails. Missing files do not establish power support: a -multinode recipe also needs `telemetry` enabled so the pinned srt-slurm exports +multinode row also needs `power: true` so the pinned srt-slurm exports `SRT_MEASUREMENT_WINDOW_DIR` to its custom benchmark command; InferenceX derives the result root and concurrency from that directory and the replay itself. When that measurement-window contract is absent, the aggregate records From f1123d17e4cee68da9d53183482ee03a789ce0af Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 9 Oct 2026 13:39:46 -0500 Subject: [PATCH 10/14] refactor(srt): compose main's new GB300 and B300 AgentX recipes from fragments #3816, #3840 and #3744 added full recipes: the Qwen3.8 GB300 and Qwen3.5 GB300 DEP16 recipes carry model blocks, the client command, benchmark MODEL and the client paths the launcher hands multi-node points, and both new recipes and the DSv4.1-Flash B300 one set the identity repo and image the binder writes. Like the other AgentX fragments they drop them, and WEKA_LOADER_OVERRIDE, which names both models' default corpus. The Qwen3.8 recipe keeps the TOTAL_CPU_DRAM_GB and KV_OFFLOADING the launcher checks against the point. Each point binds to main's selected variant, with the Qwen3.5 model alias bound as its Hub id like the other GB300 Qwen3.5 AgentX points. --- .../agentx/disagg-dep16-mtp6-variants.yaml | 13 ------------- .../dsv41flash/sglang/b300-fp4-mtp/agentic.yaml | 4 +--- .../qwen3.8next/sglang/gb300-fp4-mtp/agentic.yaml | 12 +----------- 3 files changed, 2 insertions(+), 27 deletions(-) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/agentx/disagg-dep16-mtp6-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/agentx/disagg-dep16-mtp6-variants.yaml index fdf57870bf..9788b09f7b 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/agentx/disagg-dep16-mtp6-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/agentx/disagg-dep16-mtp6-variants.yaml @@ -1,9 +1,5 @@ base: schema: 2 - model: - path: qwen3.5-fp8 - container: lmsysorg/sglang:nightly-dev-cu13-20260913-fa663e72 - precision: fp8 dynamo: install: true source: @@ -227,16 +223,7 @@ base: mem: '0' container-remap-root: '' benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/b300-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/b300-fp4-mtp/agentic.yaml index 0dca9aae27..227b2632bb 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/b300-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/b300-fp4-mtp/agentic.yaml @@ -3,10 +3,8 @@ base: name: dsv41flash-fp4-b300-sglang-agentic identity: model: - repo: deepseek-ai/DeepSeek-V4.1-Flash revision: dba1be0a40aa45a94ad051997016db3960a90277 - container: - image: lmsysorg/sglang:nightly-dev-cu13-20261004-295a53c9@sha256:7cfd4dd8e0e902d2a3c0560565963c4e172091e1586aaed15ff5a0a131394d34 + container: {} frameworks: sglang: 0.0.0.dev1+g295a53c90 sglang-router: 0.3.2 diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/gb300-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/gb300-fp4-mtp/agentic.yaml index d92632b1a2..09a41bf18c 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/gb300-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/gb300-fp4-mtp/agentic.yaml @@ -6,14 +6,8 @@ base: name: qwen3.8next-fp4-gb300-sglang-agentic-stock-v0521 identity: model: - repo: nvidia/Qwen3.8-Flash-Next-NVFP4 revision: fc694b54fb0174e0913e6adf86691ef85a4ead47 - container: - image: lmsysorg/sglang:v0.5.21-cu130@sha256:1f176b3a1dd1205d2a462ab80ea9fc17769a32e53cb6f3c690efa2dc2c384977 - model: - path: hf:nvidia/Qwen3.8-Flash-Next-NVFP4 - container: lmsysorg/sglang:v0.5.21-cu130@sha256:1f176b3a1dd1205d2a462ab80ea9fc17769a32e53cb6f3c690efa2dc2c384977 - precision: fp4 + container: {} resources: gpu_type: gb300 gpus_per_node: 4 @@ -104,12 +98,8 @@ base: mem: '0' container-remap-root: '' benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: - MODEL: nvidia/Qwen3.8-Flash-Next-NVFP4 AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k # Native single-node binding verifies these against the generated point. TOTAL_CPU_DRAM_GB: '225' KV_OFFLOADING: dram From e7a076e59ba8acc76d955be559e5102ec1102f1e Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Mon, 12 Oct 2026 11:28:13 -0500 Subject: [PATCH 11/14] refactor(srt): share one agentic.yaml block between the AgentX lanes agentic-single.yaml and agentic-multi.yaml were byte-identical. Single- and multi-node AgentX points now compose with configs/srt-recipes/agentic.yaml; the bound recipes are unchanged. --- .../multi_node/srt-slurm-recipes/RECIPES.md | 2 +- .../configs/srt-recipes/agentic-single.yaml | 3 --- .../{agentic-multi.yaml => agentic.yaml} | 0 inferencex-e2e/infx/srt_slurm/workload.py | 4 ++-- .../infx/tests/srt_slurm/test_generate.py | 3 +-- .../infx/tests/srt_slurm/test_srt_single_node.py | 4 ++-- .../infx/tests/srt_slurm/test_workload.py | 15 +++++++-------- 7 files changed, 13 insertions(+), 18 deletions(-) delete mode 100644 inferencex-e2e/configs/srt-recipes/agentic-single.yaml rename inferencex-e2e/configs/srt-recipes/{agentic-multi.yaml => agentic.yaml} (100%) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/RECIPES.md b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/RECIPES.md index 94cc115500..a65f05741f 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/RECIPES.md +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/RECIPES.md @@ -53,7 +53,7 @@ All referenced recipes must be checked in: srt-slurm 2 ships curated examples in Active recipes, fixed-sequence and AgentX, single- and multi-node, are fragments: native srt-slurm YAML holding only recipe-specific settings. At launch the fragment is composed and bound: -1. The lane's shared block in [`configs/srt-recipes/`](../../../configs/srt-recipes) (`fixed-sequence-{single,multi}.yaml` or `agentic-{single,multi}.yaml`) is merged under the fragment (under `base` for bundles). A multi-node row with `power: true` also gets [`telemetry-dcgm.yaml`](../../../configs/srt-recipes/telemetry-dcgm.yaml), its exporter on the cluster's `srt-slurm.power-exporter-port`; the lane must allow power (`POWER_LANES`). The fragment wins: mappings merge and lists replace, so a fragment may keep telemetry tunables such as `collector_join_timeout_seconds`. +1. The lane's shared block in [`configs/srt-recipes/`](../../../configs/srt-recipes) (`fixed-sequence-{single,multi}.yaml` or `agentic.yaml`) is merged under the fragment (under `base` for bundles). A multi-node row with `power: true` also gets [`telemetry-dcgm.yaml`](../../../configs/srt-recipes/telemetry-dcgm.yaml), its exporter on the cluster's `srt-slurm.power-exporter-port`; the lane must allow power (`POWER_LANES`). The fragment wins: mappings merge and lists replace, so a fragment may keep telemetry tunables such as `collector_join_timeout_seconds`. 2. The row's selector picks the variant. A single-node variant may name its `benchmark.env.CONC`, which keeps that concurrency paired with its tuning. 3. The binder ([`workload.py`](../../../infx/srt_slurm/workload.py)) writes the matrix point into the selected recipe: `model.path: hf:`, `model.container: `, `model.precision`, `identity.container.image` (as a registry reference) and `identity.model.repo` when the fragment declares `identity.container`/`identity.model`, and `benchmark.concurrencies` when telemetry is enabled. Fixed-sequence recipes also get `benchmark.env.ISL`/`OSL`; single-node recipes get `MODEL` and `CONC`, and fixed-sequence ones `RANDOM_RANGE_RATIO` and `USE_CHAT_TEMPLATE` (`true` exactly when the recipe speculates). Multi-node AgentX recipes get the client's `RESULT_DIR`, `AIPERF_DATASET_MMAP_CACHE_DIR` and `HF_HUB_CACHE` from the launcher, which derives them from the volumes it mounts (`volume-mounts`, `agentic-volume-mounts`, lane mounts); a fragment that sets `HF_HOME` keeps its own cache. The multi-node client reads `CONC_LIST`, `CONC`, `MODEL` and the other matrix inputs from the job environment, so fragments do not copy them; single-node AgentX points get them as runtime arguments. diff --git a/inferencex-e2e/configs/srt-recipes/agentic-single.yaml b/inferencex-e2e/configs/srt-recipes/agentic-single.yaml deleted file mode 100644 index ae8ca6b43e..0000000000 --- a/inferencex-e2e/configs/srt-recipes/agentic-single.yaml +++ /dev/null @@ -1,3 +0,0 @@ -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/srt_agentic.sh diff --git a/inferencex-e2e/configs/srt-recipes/agentic-multi.yaml b/inferencex-e2e/configs/srt-recipes/agentic.yaml similarity index 100% rename from inferencex-e2e/configs/srt-recipes/agentic-multi.yaml rename to inferencex-e2e/configs/srt-recipes/agentic.yaml diff --git a/inferencex-e2e/infx/srt_slurm/workload.py b/inferencex-e2e/infx/srt_slurm/workload.py index 32323cb3f2..f408984c39 100644 --- a/inferencex-e2e/infx/srt_slurm/workload.py +++ b/inferencex-e2e/infx/srt_slurm/workload.py @@ -23,8 +23,8 @@ SHARED_BLOCKS = { (False, False): Path("configs/srt-recipes/fixed-sequence-single.yaml"), (False, True): Path("configs/srt-recipes/fixed-sequence-multi.yaml"), - (True, False): Path("configs/srt-recipes/agentic-single.yaml"), - (True, True): Path("configs/srt-recipes/agentic-multi.yaml"), + (True, False): Path("configs/srt-recipes/agentic.yaml"), + (True, True): Path("configs/srt-recipes/agentic.yaml"), } # The bound variant srtctl gets. BOUND_RECIPE = "recipe.yaml" diff --git a/inferencex-e2e/infx/tests/srt_slurm/test_generate.py b/inferencex-e2e/infx/tests/srt_slurm/test_generate.py index 77a7bc3103..c75f7bb092 100644 --- a/inferencex-e2e/infx/tests/srt_slurm/test_generate.py +++ b/inferencex-e2e/infx/tests/srt_slurm/test_generate.py @@ -54,8 +54,7 @@ } # fmt: skip SHARED = { "fixed-sequence-single.yaml": {"benchmark": {"type": "custom", "command": "bash client.sh"}}, - "agentic-single.yaml": {"benchmark": {"type": "custom", "command": "bash srt_agentic.sh"}}, - "agentic-multi.yaml": {"benchmark": {"type": "custom", "command": "bash srt_agentic.sh"}}, + "agentic.yaml": {"benchmark": {"type": "custom", "command": "bash srt_agentic.sh"}}, "telemetry-dcgm.yaml": {"telemetry": {"enabled": True, "dcgm_exporter": { "container_image": "dcgm-exporter", "command": "dcgm-exporter --address :{port}", }}}, diff --git a/inferencex-e2e/infx/tests/srt_slurm/test_srt_single_node.py b/inferencex-e2e/infx/tests/srt_slurm/test_srt_single_node.py index 41fedf3fc0..81ca064173 100644 --- a/inferencex-e2e/infx/tests/srt_slurm/test_srt_single_node.py +++ b/inferencex-e2e/infx/tests/srt_slurm/test_srt_single_node.py @@ -19,8 +19,8 @@ @pytest.fixture def point(tmp_path, monkeypatch): """A fixed-sequence fragment, the point binding its first variant, and the shared blocks.""" - for workload, client in (("fixed-sequence", "client.sh"), ("agentic", "srt_agentic.sh")): - shared = tmp_path / f"configs/srt-recipes/{workload}-single.yaml" + for block, client in (("fixed-sequence-single", "client.sh"), ("agentic", "srt_agentic.sh")): + shared = tmp_path / f"configs/srt-recipes/{block}.yaml" shared.parent.mkdir(parents=True, exist_ok=True) shared.write_text(yaml.safe_dump({"benchmark": {"type": "custom", "command": f"bash {client}"}})) monkeypatch.setenv("INFERENCEX_REPOSITORY_ROOT", str(tmp_path)) diff --git a/inferencex-e2e/infx/tests/srt_slurm/test_workload.py b/inferencex-e2e/infx/tests/srt_slurm/test_workload.py index 0ad6375142..741d3c280b 100644 --- a/inferencex-e2e/infx/tests/srt_slurm/test_workload.py +++ b/inferencex-e2e/infx/tests/srt_slurm/test_workload.py @@ -25,13 +25,12 @@ @pytest.fixture def project(tmp_path): """A project whose shared blocks set the client and a default env value.""" - for workload in ("fixed-sequence", "agentic"): - for lane in ("single", "multi"): - shared = tmp_path / f"configs/srt-recipes/{workload}-{lane}.yaml" - shared.parent.mkdir(parents=True, exist_ok=True) - shared.write_text(yaml.safe_dump({"benchmark": { - "type": "custom", "command": f"bash {workload}-{lane}.sh", "env": {"TOKENIZER": "/shared"}, - }, "health_check": {"max_attempts": 10, "interval_seconds": 5}})) + for block in ("fixed-sequence-single", "fixed-sequence-multi", "agentic"): + shared = tmp_path / f"configs/srt-recipes/{block}.yaml" + shared.parent.mkdir(parents=True, exist_ok=True) + shared.write_text(yaml.safe_dump({"benchmark": { + "type": "custom", "command": f"bash {block}.sh", "env": {"TOKENIZER": "/shared"}, + }, "health_check": {"max_attempts": 10, "interval_seconds": 5}})) (tmp_path / "configs/srt-recipes/telemetry-dcgm.yaml").write_text(yaml.safe_dump({"telemetry": { "enabled": True, "required": True, "dcgm_exporter": {"container_image": "dcgm-exporter"}, }})) # fmt: skip @@ -200,4 +199,4 @@ def bind(*arguments: str) -> subprocess.CompletedProcess[str]: assert not output.exists() env["IS_AGENTIC"] = "1" assert bind(f"{path}:override_narrow", str(output)).returncode == 0 - assert yaml.safe_load(output.read_text())["benchmark"]["command"] == "bash agentic-multi.sh" + assert yaml.safe_load(output.read_text())["benchmark"]["command"] == "bash agentic.sh" From ad7805b4c17b4d4258702f0f98f4520a206ee4a8 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Tue, 13 Oct 2026 14:36:46 -0500 Subject: [PATCH 12/14] test(srt): reject a single-node variant sized for another DRAM budget The launcher rejects a variant whose TOTAL_CPU_DRAM_GB differs from the point's, the check that ties a recipe's literal host pools to the row's dram-utilization. No test covered it. --- .../infx/tests/srt_slurm/test_srt_single_node.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/inferencex-e2e/infx/tests/srt_slurm/test_srt_single_node.py b/inferencex-e2e/infx/tests/srt_slurm/test_srt_single_node.py index 81ca064173..41ebe13a2c 100644 --- a/inferencex-e2e/infx/tests/srt_slurm/test_srt_single_node.py +++ b/inferencex-e2e/infx/tests/srt_slurm/test_srt_single_node.py @@ -92,6 +92,17 @@ def test_agentx_fragments_are_composed_and_bound_to_the_point(point): select_recipe(f"{path}:base", agentic) +def test_a_variant_with_another_dram_budget_is_rejected(point): + path, recipe, env = point + path.write_text(yaml.safe_dump({"base": recipe, "override_c2_dram": {"benchmark": {"env": { + "CONC": "2", "KV_OFFLOADING": "dram", "TOTAL_CPU_DRAM_GB": "865", + }}}})) # fmt: skip + dram = {**env, "IS_AGENTIC": "1", "KV_OFFLOADING": "dram", "TOTAL_CPU_DRAM_GB": "865"} + assert select_recipe(str(path), dram)[0] == f"{path}:override_c2_dram" + with pytest.raises(ValueError, match=r"TOTAL_CPU_DRAM_GB: recipe/matrix '865' != '1731'"): + select_recipe(str(path), {**dram, "TOTAL_CPU_DRAM_GB": "1731"}) + + def test_ambiguous_native_variants_are_rejected(point): path, recipe, env = point path.write_text(yaml.safe_dump({"base": recipe, "override_first": {}, "override_second": {}})) From ee90781013f7a001fda4993afb391d6ecebc47e3 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Tue, 13 Oct 2026 15:17:31 -0500 Subject: [PATCH 13/14] fix(srt): bind the AgentX client's IS_MULTINODE and power switch Aggregated multi-node AgentX recipes pinned IS_MULTINODE=false so the client would not measure power, and so published a single-node result shape for jobs that span nodes (the DSv4 GB300 TP8 agg point runs on two). The client's conc_N naming had to accept the workflow's CONC_LIST to keep collecting them. The Qwen3.5 B300 disagg recipe pinned ENABLE_AGENTX_POWER=0 instead. The binder now writes both into every multi-node AgentX recipe from the launcher's own state: IS_MULTINODE from the request, and ENABLE_AGENTX_POWER=1 exactly when the power decision collects AgentX or adapter power. A fragment that sets either fails composition. The pins are deleted, and so are the TP, PP_SIZE, PCP_SIZE and EP_SIZE copies: only single-node points read them, and multi-node results take their topology from the workflow's PREFILL_* and DECODE_* env. The client names results per concurrency on IS_MULTINODE alone. The DSv4 GB300 vLLM 1P4D recipe also pinned RUNNER_TYPE=gb300 and PREFILL_TP=1. Its prefill worker is DEP4 (tensor-parallel-size 1, data-parallel-size 4, expert parallel) on four GPUs, which the master row records as tp 4, ep 4, dp-attn like its DEP4 siblings; the pin published prefill_tp 1 and counted one prefill GPU instead of four. Those pins and the unread OFFLOADING copy are deleted, so the row publishes its runner label and topology like its siblings. The client's power mode, result directory and file name are unchanged for every point, except that the 192 multi-node points without power switch from an invalid power verdict (multinode_power_contract_missing) to none. The 35 formerly pinned points publish is_multinode true with prefill and decode topology. --- .../multi_node/srt-slurm-recipes/RECIPES.md | 4 +-- .../sglang/b200-fp4/agentx/agg-variants.yaml | 2 -- .../agentx/agg-gb200-tp8-c1-mtp-hicache.yaml | 4 --- .../agentx/agg-gb200-tp8-c4-mtp-hicache.yaml | 4 --- .../sglang/gb300-fp4/agentx/agg-variants.yaml | 7 ---- .../agentx/agg-tp8-mtp-kvoffload.yaml | 2 -- .../agentx/disagg-1p4d-dep4-tp8-c4-mtp.yaml | 5 --- .../sglang/gb300-fp4/agentx/agg-variants.yaml | 5 --- .../b300-fp8/agentx/disagg-variants.yaml | 1 - .../sglang/gb200-fp4/agentx/agg-variants.yaml | 21 ----------- .../gb300-fp4/agentx/agg-pareto-variants.yaml | 6 ---- .../sglang/gb300-fp4/agentx/agg-variants.yaml | 4 --- ...p4-c1-b1-mtp-hicache-nightly-20260913.yaml | 4 --- ...4-c44-b1-mtp-hicache-nightly-20260913.yaml | 4 --- ...4-c44-b2-mtp-hicache-nightly-20260913.yaml | 4 --- ...4-c44-b4-mtp-hicache-nightly-20260913.yaml | 4 --- ...p8-c1-b1-mtp-hicache-nightly-20260913.yaml | 1 - inferencex-e2e/docs/results-and-ingestion.md | 9 +++-- inferencex-e2e/infx/bench/agentic/run.py | 8 ++--- .../infx/launch/drivers/srt/config.py | 15 ++++++-- inferencex-e2e/infx/srt_slurm/generate.py | 2 +- inferencex-e2e/infx/srt_slurm/workload.py | 10 ++++-- .../infx/tests/bench/test_agentic_command.py | 13 ++----- .../infx/tests/launch/test_srt_config.py | 35 +++++++++++++++++-- .../infx/tests/launch/test_srt_driver.py | 6 ++-- .../infx/tests/srt_slurm/test_generate.py | 18 +++++++++- .../infx/tests/srt_slurm/test_workload.py | 5 +++ 27 files changed, 94 insertions(+), 109 deletions(-) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/RECIPES.md b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/RECIPES.md index a65f05741f..017876e5fa 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/RECIPES.md +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/RECIPES.md @@ -55,7 +55,7 @@ Active recipes, fixed-sequence and AgentX, single- and multi-node, are fragments 1. The lane's shared block in [`configs/srt-recipes/`](../../../configs/srt-recipes) (`fixed-sequence-{single,multi}.yaml` or `agentic.yaml`) is merged under the fragment (under `base` for bundles). A multi-node row with `power: true` also gets [`telemetry-dcgm.yaml`](../../../configs/srt-recipes/telemetry-dcgm.yaml), its exporter on the cluster's `srt-slurm.power-exporter-port`; the lane must allow power (`POWER_LANES`). The fragment wins: mappings merge and lists replace, so a fragment may keep telemetry tunables such as `collector_join_timeout_seconds`. 2. The row's selector picks the variant. A single-node variant may name its `benchmark.env.CONC`, which keeps that concurrency paired with its tuning. -3. The binder ([`workload.py`](../../../infx/srt_slurm/workload.py)) writes the matrix point into the selected recipe: `model.path: hf:`, `model.container: `, `model.precision`, `identity.container.image` (as a registry reference) and `identity.model.repo` when the fragment declares `identity.container`/`identity.model`, and `benchmark.concurrencies` when telemetry is enabled. Fixed-sequence recipes also get `benchmark.env.ISL`/`OSL`; single-node recipes get `MODEL` and `CONC`, and fixed-sequence ones `RANDOM_RANGE_RATIO` and `USE_CHAT_TEMPLATE` (`true` exactly when the recipe speculates). Multi-node AgentX recipes get the client's `RESULT_DIR`, `AIPERF_DATASET_MMAP_CACHE_DIR` and `HF_HUB_CACHE` from the launcher, which derives them from the volumes it mounts (`volume-mounts`, `agentic-volume-mounts`, lane mounts); a fragment that sets `HF_HOME` keeps its own cache. The multi-node client reads `CONC_LIST`, `CONC`, `MODEL` and the other matrix inputs from the job environment, so fragments do not copy them; single-node AgentX points get them as runtime arguments. +3. The binder ([`workload.py`](../../../infx/srt_slurm/workload.py)) writes the matrix point into the selected recipe: `model.path: hf:`, `model.container: `, `model.precision`, `identity.container.image` (as a registry reference) and `identity.model.repo` when the fragment declares `identity.container`/`identity.model`, and `benchmark.concurrencies` when telemetry is enabled. Fixed-sequence recipes also get `benchmark.env.ISL`/`OSL`; single-node recipes get `MODEL` and `CONC`, and fixed-sequence ones `RANDOM_RANGE_RATIO` and `USE_CHAT_TEMPLATE` (`true` exactly when the recipe speculates). Multi-node AgentX recipes get the client's `RESULT_DIR`, `AIPERF_DATASET_MMAP_CACHE_DIR` and `HF_HUB_CACHE` from the launcher, which derives them from the volumes it mounts (`volume-mounts`, `agentic-volume-mounts`, lane mounts); a fragment that sets `HF_HOME` keeps its own cache. They also get `IS_MULTINODE: "true"` and `ENABLE_AGENTX_POWER`, `"1"` exactly when the launcher collects the point's AgentX power (a `power: true` row whose lane rule collects AgentX or adapter power) and `"0"` otherwise. The multi-node client reads `CONC_LIST`, `CONC`, `MODEL` and the other matrix inputs, prefill and decode topology included, from the job environment, so fragments do not copy them (`TP`, `PP_SIZE`, `PCP_SIZE` and `EP_SIZE` describe single-node points only); single-node AgentX points get them as runtime arguments. A fragment that sets a key the binder or launcher writes (`_POINT_KEYS`, `BOUND_ENV` in `workload.py`; single-node variants may still name `CONC`) fails before submission, even with the bound value. `hf:` resolves to the cluster's staged checkpoint (`models.entries`) unless a `models.OVERRIDES` row serves the Hub snapshot, and the master image to its staged container. A multi-node fixed-sequence point with neither checkpoint nor Hub row fails before submission. @@ -65,7 +65,7 @@ Inspect what the launcher submits without a cluster: uv run --extra recipes infx generate --config-key 'dsr1-fp8-h200-*' --output-dir /tmp/recipes ``` -It writes one bound recipe per fixed-sequence or AgentX point, validated by the pinned srtctl, plus a `manifest.json` that maps each file to its matrix point and to the one cluster its runner label schedules on (`null` when the label spans several). Multi-node points take their cluster's binder inputs (DCGM exporter port, AgentX client paths); a point that needs them fails unless its runner label names one cluster and the lane accepts the row's `power`. Launch-time edits such as the job name, health-check floor and runtime `--set` values are not applied. +It writes one bound recipe per fixed-sequence or AgentX point, validated by the pinned srtctl, plus a `manifest.json` that maps each file to its matrix point and to the one cluster its runner label schedules on (`null` when the label spans several). Multi-node points take their cluster's binder inputs (DCGM exporter port, AgentX client env); a point that needs them fails unless its runner label names one cluster and the lane accepts the row's `power`. Launch-time edits such as the job name, health-check floor and runtime `--set` values are not applied. ## Migration and validation diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/agg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/agg-variants.yaml index 656eba3707..4d75efed20 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/agg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/agg-variants.yaml @@ -106,8 +106,6 @@ base: container-remap-root: "" benchmark: env: - IS_MULTINODE: "false" - TP: "8" AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/agg-gb200-tp8-c1-mtp-hicache.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/agg-gb200-tp8-c1-mtp-hicache.yaml index 74473f5bb3..282cae40f6 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/agg-gb200-tp8-c1-mtp-hicache.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/agg-gb200-tp8-c1-mtp-hicache.yaml @@ -113,10 +113,6 @@ srun_options: benchmark: env: - IS_MULTINODE: "false" - TP: "8" - PP_SIZE: "1" - PCP_SIZE: "1" AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/agg-gb200-tp8-c4-mtp-hicache.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/agg-gb200-tp8-c4-mtp-hicache.yaml index 6551bffc47..455bfdcd7d 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/agg-gb200-tp8-c4-mtp-hicache.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/agg-gb200-tp8-c4-mtp-hicache.yaml @@ -113,10 +113,6 @@ srun_options: benchmark: env: - IS_MULTINODE: "false" - TP: "8" - PP_SIZE: "1" - PCP_SIZE: "1" AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/agg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/agg-variants.yaml index 6418b5c4eb..4e90219f8b 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/agg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/agg-variants.yaml @@ -94,10 +94,6 @@ base: container-remap-root: "" benchmark: env: - IS_MULTINODE: "false" - # The AgentX power path checks the GPU topology (TP from each variant) before replay. - PP_SIZE: "1" - PCP_SIZE: "1" AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" @@ -114,6 +110,3 @@ override_tp8: max-running-requests: 4 cuda-graph-max-bs-decode: 4 tp-size: 8 - benchmark: - env: - TP: "8" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/h200-fp8/agentx/agg-tp8-mtp-kvoffload.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/h200-fp8/agentx/agg-tp8-mtp-kvoffload.yaml index ee1df4bf6e..36d374a43e 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/h200-fp8/agentx/agg-tp8-mtp-kvoffload.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/h200-fp8/agentx/agg-tp8-mtp-kvoffload.yaml @@ -110,8 +110,6 @@ telemetry: benchmark: env: - # Aggregated serving uses one TP8 worker for both prefill and decode. - TP: "8" AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" # Fail the run loudly if the aggregate engine's sglang: metrics never reach diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p4d-dep4-tp8-c4-mtp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p4d-dep4-tp8-c4-mtp.yaml index 98e3bcb3f8..88310f19fb 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p4d-dep4-tp8-c4-mtp.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p4d-dep4-tp8-c4-mtp.yaml @@ -184,11 +184,6 @@ srun_options: benchmark: env: SERVED_MODEL_NAME: "deepseek-ai/DeepSeek-V4-Pro" - RUNNER_TYPE: "gb300" - OFFLOADING: "none" - TP: "8" - PREFILL_TP: "1" - EP_SIZE: "1" AIPERF_MAX_OSL: "none" AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/agg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/agg-variants.yaml index 40b043930a..0131b25c51 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/agg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/agg-variants.yaml @@ -107,15 +107,11 @@ base: container-remap-root: "" benchmark: env: - IS_MULTINODE: "false" AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" # Warmup can leave bytes unacknowledged past AIPerf's 30 s default. AIPERF_HTTP_TCP_USER_TIMEOUT: "900000" - TP: "2" - PP_SIZE: "1" - PCP_SIZE: "1" # Lowest-latency arm: one pure TP4 (EP1) worker on all four GPUs at c1, the standalone # recipe's override_tp4_c1 behind the Dynamo frontend: static ragged verify, Engram @@ -135,7 +131,6 @@ override_tp4_c1: SGLANG_RAGGED_VERIFY_MODE: static benchmark: env: - TP: "4" AGENTIC_WARMUP_GRACE_PERIOD: "1800" # Two-GPU c1 arm: the base TP2/EP2 worker with the Engram tables in host DRAM, static diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-variants.yaml index fc6f386e10..ad81c2c1db 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-variants.yaml @@ -156,7 +156,6 @@ base: mem: '0' benchmark: env: - ENABLE_AGENTX_POWER: '0' AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-variants.yaml index 84e02bc7ac..5f6e402b1c 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-variants.yaml @@ -81,9 +81,6 @@ base: srun_options: {mem: "0", container-remap-root: ""} benchmark: env: - IS_MULTINODE: "false" - PP_SIZE: "1" - PCP_SIZE: "1" AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" @@ -102,9 +99,6 @@ override_tp4: mamba-track-interval: 8192 mem-fraction-static: 0.80 max-mamba-cache-size: 360 - benchmark: - env: - TP: "4" override_tp4_parity: name: qwen35-gb200-dynsg-agentic-mtp-agg-tp4-k5-parity @@ -121,9 +115,6 @@ override_tp4_parity: linear-replayssm-cache-len: 8 mem-fraction-static: 0.80 max-mamba-cache-size: 360 - benchmark: - env: - TP: "4" override_tp4_no_symm: name: qwen35-gb200-dynsg-agentic-mtp-agg-tp4-k5-no-symm @@ -137,9 +128,6 @@ override_tp4_no_symm: mamba-track-interval: 8192 mem-fraction-static: 0.80 max-mamba-cache-size: 360 - benchmark: - env: - TP: "4" override_tp2ep2_hicache: name: qwen35-gb200-dynsg-agentic-mtp-agg-tp2ep2-hicache-k5 @@ -168,9 +156,6 @@ override_tp2ep2_hicache: hicache-mem-layout: page_first_direct hicache-write-policy: write_through_selective watchdog-timeout: 1000000 - benchmark: - env: - TP: "2" override_tp2ep2_hicache_cap48: name: qwen35-gb200-dynsg-agentic-mtp-agg-tp2ep2-hicache-cap48 @@ -201,9 +186,6 @@ override_tp2ep2_hicache_cap48: hicache-mem-layout: page_first_direct hicache-write-policy: write_through_selective watchdog-timeout: 1000000 - benchmark: - env: - TP: "2" override_tp2ep2_hicache_k3_baseline: name: qwen35-gb200-dynsg-agentic-mtp-agg-tp2ep2-hicache-k3-baseline @@ -233,6 +215,3 @@ override_tp2ep2_hicache_k3_baseline: hicache-mem-layout: page_first_direct hicache-write-policy: write_through_selective watchdog-timeout: 1000000 - benchmark: - env: - TP: "2" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-pareto-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-pareto-variants.yaml index 21338332b6..e977173467 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-pareto-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-pareto-variants.yaml @@ -113,8 +113,6 @@ base: container-remap-root: '' benchmark: env: - IS_MULTINODE: 'false' - TP: '2' AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' @@ -167,7 +165,3 @@ override_tp8_c7_b1: moe-runner-backend: flashinfer_cutedsl speculative-moe-runner-backend: flashinfer_cutedsl disable-prefill-cuda-graph: true - benchmark: - env: - IS_MULTINODE: 'true' - TP: '8' diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-variants.yaml index b7e1ad122f..db044e68ff 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-variants.yaml @@ -72,10 +72,6 @@ base: container-remap-root: '' benchmark: env: - IS_MULTINODE: 'false' - TP: '2' - PP_SIZE: '1' - PCP_SIZE: '1' AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/agentx/agg-gb300-tp4-c1-b1-mtp-hicache-nightly-20260913.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/agentx/agg-gb300-tp4-c1-b1-mtp-hicache-nightly-20260913.yaml index ad1ac902f6..e9221c0f70 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/agentx/agg-gb300-tp4-c1-b1-mtp-hicache-nightly-20260913.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/agentx/agg-gb300-tp4-c1-b1-mtp-hicache-nightly-20260913.yaml @@ -104,10 +104,6 @@ srun_options: container-remap-root: '' benchmark: env: - IS_MULTINODE: 'false' AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' - TP: '4' - PP_SIZE: '1' - PCP_SIZE: '1' diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/agentx/agg-gb300-tp4-c44-b1-mtp-hicache-nightly-20260913.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/agentx/agg-gb300-tp4-c44-b1-mtp-hicache-nightly-20260913.yaml index 88f63e37f2..1eb0027c7a 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/agentx/agg-gb300-tp4-c44-b1-mtp-hicache-nightly-20260913.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/agentx/agg-gb300-tp4-c44-b1-mtp-hicache-nightly-20260913.yaml @@ -104,10 +104,6 @@ srun_options: container-remap-root: '' benchmark: env: - IS_MULTINODE: 'false' AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' - TP: '4' - PP_SIZE: '1' - PCP_SIZE: '1' diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/agentx/agg-gb300-tp4-c44-b2-mtp-hicache-nightly-20260913.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/agentx/agg-gb300-tp4-c44-b2-mtp-hicache-nightly-20260913.yaml index 4ddfb8d3d9..2bf552b042 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/agentx/agg-gb300-tp4-c44-b2-mtp-hicache-nightly-20260913.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/agentx/agg-gb300-tp4-c44-b2-mtp-hicache-nightly-20260913.yaml @@ -104,10 +104,6 @@ srun_options: container-remap-root: '' benchmark: env: - IS_MULTINODE: 'false' AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' - TP: '4' - PP_SIZE: '1' - PCP_SIZE: '1' diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/agentx/agg-gb300-tp4-c44-b4-mtp-hicache-nightly-20260913.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/agentx/agg-gb300-tp4-c44-b4-mtp-hicache-nightly-20260913.yaml index 76a8bff705..678953ed66 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/agentx/agg-gb300-tp4-c44-b4-mtp-hicache-nightly-20260913.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/agentx/agg-gb300-tp4-c44-b4-mtp-hicache-nightly-20260913.yaml @@ -104,10 +104,6 @@ srun_options: container-remap-root: '' benchmark: env: - IS_MULTINODE: 'false' AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' - TP: '4' - PP_SIZE: '1' - PCP_SIZE: '1' diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/agentx/agg-gb300-tp8-c1-b1-mtp-hicache-nightly-20260913.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/agentx/agg-gb300-tp8-c1-b1-mtp-hicache-nightly-20260913.yaml index 2acfcf3c62..91d4250b47 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/agentx/agg-gb300-tp8-c1-b1-mtp-hicache-nightly-20260913.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/agentx/agg-gb300-tp8-c1-b1-mtp-hicache-nightly-20260913.yaml @@ -107,4 +107,3 @@ benchmark: AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' - TP: '8' diff --git a/inferencex-e2e/docs/results-and-ingestion.md b/inferencex-e2e/docs/results-and-ingestion.md index c04261111e..2b4353e389 100644 --- a/inferencex-e2e/docs/results-and-ingestion.md +++ b/inferencex-e2e/docs/results-and-ingestion.md @@ -179,9 +179,12 @@ The aggregate artifact matches the `bmk_*` collection pattern and therefore also Server logs are separate `server_logs_` artifacts. The app uses the fully stripped suffix fallback so AgentX rows can find a server log even though the log artifact has no `agentic_` prefix. -AgentX power applies only to replays with `IS_MULTINODE=true`. Single-node -replays, including aggregated srt-slurm recipes that set `IS_MULTINODE: false`, -publish no power fields or verdict, whatever `ENABLE_AGENTX_POWER` says. +AgentX power applies only to replays with `IS_MULTINODE=true`; single-node replays +publish no power fields or verdict, whatever `ENABLE_AGENTX_POWER` says. The binder +writes both into every srt-slurm multi-node AgentX recipe: `IS_MULTINODE=true`, and +`ENABLE_AGENTX_POWER=1` exactly when the launcher collects the point's AgentX power +(a `power: true` row whose lane rule collects AgentX or adapter power). Other +multi-node points get `ENABLE_AGENTX_POWER=0` and publish no power fields or verdict. Multinode runs retain the deployment telemetry under `LOGS/power/` and per-concurrency window/validation files under `LOGS/agentic/` in their `power_audit_` artifact. Available audits and AgentX aggregates diff --git a/inferencex-e2e/infx/bench/agentic/run.py b/inferencex-e2e/infx/bench/agentic/run.py index ae2309e143..d0099aeffb 100644 --- a/inferencex-e2e/infx/bench/agentic/run.py +++ b/inferencex-e2e/infx/bench/agentic/run.py @@ -27,7 +27,8 @@ "IS_MULTINODE", "PRECISION", ) -# Only srt-slurm's multi-node telemetry measures AgentX power; single-node points publish none. +# Only srt-slurm's multi-node telemetry measures AgentX power: its binder sets +# ENABLE_AGENTX_POWER to 1 exactly when the launcher collects it. Single-node points publish none. POWER_SWITCHES = ("ENABLE_AGENTX_POWER", "REQUIRE_POWER") # The finished profile's accepted error fraction; a recipe's live abort threshold # (AIPERF_LIVE_FAILED_REQUEST_THRESHOLD) does not move it. @@ -62,9 +63,8 @@ def from_env(cls, env: Mapping[str, str]) -> Plan: switches = inputs.require(*POWER_SWITCHES, env=env) if multinode else {} result_dir = Path(values["RESULT_DIR"]).absolute() result_filename = values["RESULT_FILENAME"] - if multinode or env.get("CONC_LIST"): - # Multi-node collection globs one ${RESULT_FILENAME}_conc*.json per point. Some - # multi-node recipes pin IS_MULTINODE=false, so the workflow's CONC_LIST counts too. + if multinode: + # Multi-node collection globs one ${RESULT_FILENAME}_conc*.json per point. result_dir /= f"conc_{values['CONC']}" result_filename += f"_conc{values['CONC']}" replay = ReplayConfig.from_env(env, result_dir) diff --git a/inferencex-e2e/infx/launch/drivers/srt/config.py b/inferencex-e2e/infx/launch/drivers/srt/config.py index f2d48040ff..c7d5f762fa 100644 --- a/inferencex-e2e/infx/launch/drivers/srt/config.py +++ b/inferencex-e2e/infx/launch/drivers/srt/config.py @@ -238,9 +238,18 @@ def binder_inputs( request: SrtRequest, decision: PowerDecision, ) -> tuple[int | None, dict[str, str]]: - """The DCGM exporter port and AgentX client paths a multi-node recipe is bound with.""" - client_env = agentic_client_env(cluster, srt, lane, request) if request.is_agentic else {} - return (srt.power_exporter_port if decision.dcgm else None), client_env + """The DCGM exporter port and AgentX client env a multi-node recipe is bound with. + + The client marks power windows exactly when the launcher collects the point's AgentX power. + """ + port = srt.power_exporter_port if decision.dcgm else None + if not request.is_agentic: + return port, {} + return port, { + **agentic_client_env(cluster, srt, lane, request), + "IS_MULTINODE": str(request.is_multinode).lower(), + "ENABLE_AGENTX_POWER": "1" if decision.agentx or decision.adapter else "0", + } def write_lane_config( diff --git a/inferencex-e2e/infx/srt_slurm/generate.py b/inferencex-e2e/infx/srt_slurm/generate.py index b250a16345..71e9cf09d6 100644 --- a/inferencex-e2e/infx/srt_slurm/generate.py +++ b/inferencex-e2e/infx/srt_slurm/generate.py @@ -94,7 +94,7 @@ def _binder_inputs( placement: tuple[str, Cluster] | None, root: Path, ) -> tuple[int | None, dict[str, str]]: - """The DCGM exporter port and AgentX client paths the launcher binds a multi-node point with. + """The DCGM exporter port and AgentX client env the launcher binds a multi-node point with. Raises ``ValueError`` when they depend on a cluster the runner label does not single out, or the cluster's lanes refuse the point. diff --git a/inferencex-e2e/infx/srt_slurm/workload.py b/inferencex-e2e/infx/srt_slurm/workload.py index f408984c39..60a7d28183 100644 --- a/inferencex-e2e/infx/srt_slurm/workload.py +++ b/inferencex-e2e/infx/srt_slurm/workload.py @@ -30,7 +30,8 @@ BOUND_RECIPE = "recipe.yaml" TELEMETRY_BLOCK = Path("configs/srt-recipes/telemetry-dcgm.yaml") # The binder writes these, the master power field owns telemetry, and the launcher hands -# the benchmark client its cache and result paths. +# the benchmark client its cache and result paths and, multi-node, IS_MULTINODE and the +# AgentX power switch. _POINT_KEYS = ( ("model", "path"), ("model", "container"), @@ -48,7 +49,12 @@ (False, False): _FIXED_ENV, (False, True): _FIXED_ENV, (True, False): ("MODEL", *_AGENTIC_ENV), - (True, True): (*_AGENTIC_ENV, "AIPERF_DATASET_MMAP_CACHE_DIR"), + (True, True): ( + *_AGENTIC_ENV, + "AIPERF_DATASET_MMAP_CACHE_DIR", + "IS_MULTINODE", + "ENABLE_AGENTX_POWER", + ), } diff --git a/inferencex-e2e/infx/tests/bench/test_agentic_command.py b/inferencex-e2e/infx/tests/bench/test_agentic_command.py index 080ca3a6e4..8efd291427 100644 --- a/inferencex-e2e/infx/tests/bench/test_agentic_command.py +++ b/inferencex-e2e/infx/tests/bench/test_agentic_command.py @@ -166,17 +166,8 @@ def test_standalone_srt_slurm_point_needs_no_kv_offload_declaration(tmp_path): id="single-node-publishes-no-power", ), pytest.param( - # Multi-node recipes that pin IS_MULTINODE=false still run under the multi-node - # workflow, which collects ${RESULT_FILENAME}_conc*.json. - {"CONC_LIST": "8", "KV_OFFLOADING": "dram", "KV_OFFLOAD_BACKEND": "lmcache", - "TOTAL_CPU_DRAM_GB": "2400"}, - 0, - ["replay", "aggregate agentx_conc8", "analyze", "validate"], - {f"conc_8/{name}" for name in REPLAYED}, - id="conc-list-point", - ), - pytest.param( - {**WINDOW, "ENABLE_AGENTX_POWER": "0"}, + {**WINDOW, "ENABLE_AGENTX_POWER": "0", "KV_OFFLOADING": "dram", + "KV_OFFLOAD_BACKEND": "lmcache", "TOTAL_CPU_DRAM_GB": "2400"}, 0, ["replay", "aggregate agentx_conc8", "analyze", "validate"], {f"conc_8/{name}" for name in REPLAYED}, diff --git a/inferencex-e2e/infx/tests/launch/test_srt_config.py b/inferencex-e2e/infx/tests/launch/test_srt_config.py index bda8be9f6d..9758eb977b 100644 --- a/inferencex-e2e/infx/tests/launch/test_srt_config.py +++ b/inferencex-e2e/infx/tests/launch/test_srt_config.py @@ -10,15 +10,18 @@ import yaml from infx.clusters import Cluster, load_inventory +from infx.clusters.slurm import slurm_settings from infx.launch.backends.slurm import SlurmBackend from infx.launch.context import Launch from infx.launch.drivers.srt.checkout import Checkout, compute_workspace -from infx.launch.drivers.srt.config import SrtJob, pyxis_spelling, render, write +from infx.launch.drivers.srt.config import SrtJob, binder_inputs, pyxis_spelling, render, write +from infx.launch.drivers.srt.lanes import SrtLane +from infx.launch.drivers.srt.power import NO_POWER, PowerDecision from infx.launch.drivers.srt.recipe import HEALTH_ATTEMPTS, prepare_recipe from infx.launch.drivers.srt.run import SrtRun from infx.launch.lifecycle import Lifecycle from infx.launch.policy import LaunchPath -from infx.launch.request import SrtRequest +from infx.launch.request import MultiNodeRequest, SrtRequest ROOT = Path(__file__).resolve().parents[3] sys.path.insert(0, str(ROOT / "utils/srt-slurm/src")) @@ -229,3 +232,31 @@ def rsync(source: Path, staging: Path, *, exclude: tuple[str, ...]) -> Path: assert staged == tmp_path / "runs/infmax-workspace-1-1-abc" assert sorted(str(p.relative_to(staged)) for p in staged.rglob("*") if p.is_file()) == sorted(kept) + + +@pytest.mark.parametrize(("agentic", "decision", "expected"), [ + ("1", NO_POWER, (None, "true", "0")), + # DCGM alone: the launcher collects no AgentX power, so the client marks no window. + ("1", PowerDecision(dcgm=True, agentx=False), (9401, "true", "0")), + ("1", PowerDecision(dcgm=True, agentx=True), (9401, "true", "1")), + ("1", PowerDecision(dcgm=True, agentx=False, adapter=True), (9401, "true", "1")), + ("0", PowerDecision(dcgm=True, agentx=False), (9401, None, None)), +]) # fmt: skip +def test_the_agentx_client_measures_power_exactly_when_the_launcher_collects_it( + tmp_path, agentic, decision, expected +): + record = cluster(srt={"power-exporter-port": 9401}) + request = MultiNodeRequest.from_env({ + "RUNNER_NAME": "c_0", "GITHUB_WORKSPACE": str(tmp_path), "IMAGE": "i", + "FRAMEWORK": "dynamo-sglang", "MODEL_PREFIX": "m", "PRECISION": "fp8", + "SPEC_DECODING": "none", "RESULT_FILENAME": "r", "IS_AGENTIC": agentic, "RUN_EVAL": "false", + "EVAL_ONLY": "false", "IS_MULTINODE": "true", "SRT_RECIPE": "r.yaml", "CONC": "4", + "CONC_LIST": "4", + }) # fmt: skip + + port, client_env = binder_inputs( + record, slurm_settings(record).srt_slurm, SrtLane(), request, decision + ) + + switches = (client_env.get("IS_MULTINODE"), client_env.get("ENABLE_AGENTX_POWER")) + assert (port, *switches) == expected diff --git a/inferencex-e2e/infx/tests/launch/test_srt_driver.py b/inferencex-e2e/infx/tests/launch/test_srt_driver.py index b0c92045bd..f63065baf6 100644 --- a/inferencex-e2e/infx/tests/launch/test_srt_driver.py +++ b/inferencex-e2e/infx/tests/launch/test_srt_driver.py @@ -273,10 +273,11 @@ def test_multinode_lane_stages_workflow_artifacts(harness, monkeypatch, cluster_ {"path": "hf:org/Model", "container": "test:tag", "precision": "fp8"}, ) if env["IS_AGENTIC"] == "1": - # lab-b mounts no cache volume, so the client keeps its caches in the container. + # lab-b mounts no cache volume, so the client keeps its caches in the container, and + # measures no power. assert bound["benchmark"]["env"] == { "RESULT_DIR": "/logs/agentic", "AIPERF_DATASET_MMAP_CACHE_DIR": "/aiperf_mmap_cache", - "HF_HUB_CACHE": "/hf_hub_cache", + "HF_HUB_CACHE": "/hf_hub_cache", "IS_MULTINODE": "true", "ENABLE_AGENTX_POWER": "0", } config = srtslurm(checkout) @@ -324,6 +325,7 @@ def test_power_lane_stages_provenance_and_validates_each_concurrency( assert (workspace / "LOGS/power/power-producer-sha.txt").read_text() == commit + "\n" bound = yaml.safe_load((checkout / "recipe.yaml").read_text()) assert bound["benchmark"]["concurrencies"] == [4] + assert bound["benchmark"]["env"]["ENABLE_AGENTX_POWER"] == "1" assert bound["telemetry"]["collector_join_timeout_seconds"] == 12 assert (bound["telemetry"]["enabled"], bound["telemetry"]["dcgm_exporter"]["port"]) == (True, 9401) diff --git a/inferencex-e2e/infx/tests/srt_slurm/test_generate.py b/inferencex-e2e/infx/tests/srt_slurm/test_generate.py index c75f7bb092..d68f3a6530 100644 --- a/inferencex-e2e/infx/tests/srt_slurm/test_generate.py +++ b/inferencex-e2e/infx/tests/srt_slurm/test_generate.py @@ -170,13 +170,29 @@ def test_a_multinode_agentx_power_point_gets_its_clusters_exporter_port_and_clie assert recipe["telemetry"] == {"enabled": True, "dcgm_exporter": { "container_image": "dcgm-exporter", "command": "dcgm-exporter --address :{port}", "port": 9555, }} # fmt: skip - # The lane's result directory, the AgentX-only aiperf mount and the cluster-wide Hub cache. + # The lane's result directory, the AgentX-only aiperf mount, the cluster-wide Hub cache and + # the launcher's power decision. assert recipe["benchmark"]["env"] == { "RESULT_DIR": "/results", "AIPERF_DATASET_MMAP_CACHE_DIR": "/aiperf", "HF_HUB_CACHE": "/hf", + "IS_MULTINODE": "true", "ENABLE_AGENTX_POWER": "1", } # fmt: skip assert recipe["benchmark"]["concurrencies"] == [8] +def test_a_multinode_agentx_point_without_power_turns_its_clients_power_off( + project, power_lane, tmp_path +): + master = project / "configs/master.yaml" + config = yaml.safe_load(master.read_text()) + del config["agentx-multi"]["scenarios"]["agentic-coding"][0]["search-space"][0]["power"] + master.write_text(yaml.safe_dump(config)) + output = tmp_path / "out" + assert generate(project, output, "agentx-multi") == 0 + + [(_, recipe)] = written(output) + assert recipe["benchmark"]["env"]["ENABLE_AGENTX_POWER"] == "0" + + def test_a_power_point_its_lane_refuses_fails_without_writing( project, power_lane, monkeypatch, tmp_path, capsys ): diff --git a/inferencex-e2e/infx/tests/srt_slurm/test_workload.py b/inferencex-e2e/infx/tests/srt_slurm/test_workload.py index 741d3c280b..e8d913ee3f 100644 --- a/inferencex-e2e/infx/tests/srt_slurm/test_workload.py +++ b/inferencex-e2e/infx/tests/srt_slurm/test_workload.py @@ -88,6 +88,11 @@ def test_bundles_take_the_shared_block_under_base_and_keep_their_variants(projec "benchmark.env.HF_HUB_CACHE (= '/elsewhere')"), ({"base": {"benchmark": {"env": {"RESULT_DIR": "/logs"}}}}, True, False, "base.benchmark.env.RESULT_DIR (= '/logs')"), + ({"base": {}, "override_tp8": {"benchmark": {"env": { + "IS_MULTINODE": "false", "ENABLE_AGENTX_POWER": "0", + }}}}, True, True, + "override_tp8.benchmark.env.IS_MULTINODE (= 'false'), " + "override_tp8.benchmark.env.ENABLE_AGENTX_POWER (= '0')"), ]) # fmt: skip def test_a_fragment_that_sets_a_bound_key_is_rejected(project, data, agentic, multinode, reported): path = fragment(project, data) From 8d8141598e4e099faaef7a75d9975607497b9c6c Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Tue, 13 Oct 2026 19:16:30 -0500 Subject: [PATCH 14/14] refactor(srt): stop pinning workflow env in recipe fragments Fragments still set values the workflow exports from the matrix point or from runtime_settings.sh. The Kimi-K3 GB200 and GB300 fragments pinned REQUIRE_POWER=1, two TensorRT-LLM fragments pinned MAX_MODEL_LEN, MiniMax-M3 GB200 TRT copied KV_OFFLOAD_BACKEND_METADATA, the DSv4 GB300 frontends copied MODEL, MODEL_PREFIX, FRAMEWORK, PRECISION, DURATION, KV_OFFLOADING and CONC, and GLM-5.3 TileRT set MODEL=/model, which it published as the model. The binder now writes REQUIRE_POWER into every multi-node AgentX recipe: 1 when the workflow requires power or the lane's power rule does. The gb200-nv and gb300-nv Kimi-K3 AgentX rules require it, which keeps the pinned points strict and no others. BOUND_ENV rejects every name the workflow exports from the matrix point. MAX_MODEL_LEN goes (the AgentX client never reads it; evals take the job's), and so do the frontend copies, which the Dynamo frontend never reads. GLM-5.3 names its local checkpoint as the client's TOKENIZER, which the replay reads before MODEL, so it publishes zai-org/GLM-5.3 and keeps its tokenizer. Copies that equal the workflow's value go too: VLLM_ENGINE_READY_TIMEOUT_S 3600, PYTHONDONTWRITEBYTECODE, AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS 3600 and the MORI, NCCL and SGLang settings runtime_settings.sh exports for sglang-disagg. Every process keeps its effective value through the job environment. Fragments keep values that override the workflow's on purpose: the Dynamo routing and session settings, longer engine-ready timeouts, MORI_IO_QP_MAX_SEND_WR 32767 and single-node variant labels. --- .../multi_node/srt-slurm-recipes/RECIPES.md | 4 +-- .../mi355x-fp4/8k1k/disagg-variants.yaml | 8 ------ .../mi355x-fp8/8k1k/disagg-variants.yaml | 8 ------ .../agentx/disagg-lmcache-dspark.yaml | 1 - .../mi355x-fp4/agentx/disagg-umbp-dspark.yaml | 10 ------- .../gb300-fp4/agentx/disagg-variants.yaml | 24 ---------------- .../gb200-fp4/agentx/agg-mtp-variants.yaml | 1 - .../gb200-fp4/agentx/agg-mtp3-variants.yaml | 1 - .../gb200-fp4/agentx/disagg-mtp-variants.yaml | 2 -- .../agentx/disagg-mtp3-variants.yaml | 2 -- .../vllm/gb300-fp4/agentx/agg-variants.yaml | 1 - .../agentx/disagg-1p4d-dep4-tp8-c4-mtp.yaml | 8 ------ .../gb300-fp4/agentx/disagg-mtp-variants.yaml | 2 -- .../gb300-fp4/agentx/disagg-variants.yaml | 1 - .../agentx/disagg-1p1d-tp8-mtp.yaml | 3 +- .../agentx/agg-tp8pp2-mooncake-c1.yaml | 1 - .../agentx/agg-tp8pp2-mooncake-c14.yaml | 1 - .../agentx/agg-tp8pp2-mooncake-c24.yaml | 1 - .../agentx/agg-tp8pp2-mooncake-c32.yaml | 1 - .../agentx/agg-tp8pp2-mooncake-c4.yaml | 1 - .../agentx/agg-tp8pp2-mooncake-c48.yaml | 1 - .../agentx/agg-tp8pp2-mooncake-c56.yaml | 1 - .../agentx/agg-tp8pp2-mooncake-c72.yaml | 1 - .../agentx/agg-tp8pp2-mooncake-c8.yaml | 1 - .../agentx/agg-tp8pp2-mooncake-c80.yaml | 1 - .../agentx/agg-tp8pp2-mooncake-c96.yaml | 1 - .../agg-dcp16-dspark4-maxseq2-mooncake.yaml | 2 -- .../agentx/agg-dcp16-nospec-mooncake.yaml | 2 -- .../agentx/agg-tp8pp2-mooncake-c16.yaml | 2 -- .../agentx/agg-tp8pp2-mooncake-c32.yaml | 2 -- .../agentx/agg-tp8pp2-mooncake-c48.yaml | 2 -- .../agentx/agg-tp8pp2-mooncake-c72.yaml | 2 -- .../agentx/agg-tp8pp2-mooncake-c96.yaml | 2 -- .../agentx/agg-dcp8-dspark4-mooncake.yaml | 2 -- .../agg-dcp8-dspark7-maxseq2-mooncake.yaml | 2 -- ...isagg-1p1d-dcp8-dcp8-dspark4-mooncake.yaml | 2 -- ...isagg-1p2d-dcp8-dcp8-dspark4-mooncake.yaml | 2 -- ...isagg-1p3d-dcp8-dcp8-dspark4-mooncake.yaml | 2 -- ...isagg-1p3d-dcp8-dcp8-dspark7-mooncake.yaml | 2 -- .../trtllm/gb200-fp4/agentx/agg-variants.yaml | 2 -- .../gb200-fp4/agentx/disagg-variants.yaml | 3 -- .../gb300-fp4/agentx/disagg-variants.yaml | 2 -- .../mi355x-fp4/8k1k/disagg-1p1d-tp8.yaml | 4 --- .../8k1k/disagg-1p1d-p-tp4-d-tp8.yaml | 4 --- .../dsv41flash/vllm/h100-fp4-mtp/agentic.yaml | 1 - .../vllm/mi300x-fp4-mtp/agentic.yaml | 1 - .../vllm/mi325x-fp4-mtp/agentic.yaml | 1 - .../vllm/mi355x-fp4-mtp/agentic.yaml | 1 - .../kimik3/vllm/b300-fp4-mtp/agentic.yaml | 1 - .../minimaxm3/vllm/b200-fp4-mtp/agentic.yaml | 1 - .../minimaxm3/vllm/b300-fp4-mtp/agentic.yaml | 1 - .../vllm/mi300x-fp8-mtp/agentic.yaml | 1 - .../vllm/mi325x-fp8-mtp/agentic.yaml | 1 - inferencex-e2e/docs/results-and-ingestion.md | 3 ++ inferencex-e2e/infx/bench/agentic/replay.py | 5 ++-- .../infx/launch/drivers/srt/config.py | 4 ++- .../infx/launch/drivers/srt/power.py | 10 +++++-- inferencex-e2e/infx/srt_slurm/workload.py | 28 +++++++++++++++---- .../infx/tests/bench/test_agentic_replay.py | 5 ++++ .../infx/tests/launch/test_launch_policy.py | 2 ++ .../infx/tests/launch/test_srt_config.py | 17 +++++++---- .../infx/tests/launch/test_srt_driver.py | 3 ++ .../infx/tests/srt_slurm/test_generate.py | 2 +- .../infx/tests/srt_slurm/test_workload.py | 3 ++ 64 files changed, 68 insertions(+), 151 deletions(-) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/RECIPES.md b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/RECIPES.md index 017876e5fa..d43336e159 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/RECIPES.md +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/RECIPES.md @@ -55,9 +55,9 @@ Active recipes, fixed-sequence and AgentX, single- and multi-node, are fragments 1. The lane's shared block in [`configs/srt-recipes/`](../../../configs/srt-recipes) (`fixed-sequence-{single,multi}.yaml` or `agentic.yaml`) is merged under the fragment (under `base` for bundles). A multi-node row with `power: true` also gets [`telemetry-dcgm.yaml`](../../../configs/srt-recipes/telemetry-dcgm.yaml), its exporter on the cluster's `srt-slurm.power-exporter-port`; the lane must allow power (`POWER_LANES`). The fragment wins: mappings merge and lists replace, so a fragment may keep telemetry tunables such as `collector_join_timeout_seconds`. 2. The row's selector picks the variant. A single-node variant may name its `benchmark.env.CONC`, which keeps that concurrency paired with its tuning. -3. The binder ([`workload.py`](../../../infx/srt_slurm/workload.py)) writes the matrix point into the selected recipe: `model.path: hf:`, `model.container: `, `model.precision`, `identity.container.image` (as a registry reference) and `identity.model.repo` when the fragment declares `identity.container`/`identity.model`, and `benchmark.concurrencies` when telemetry is enabled. Fixed-sequence recipes also get `benchmark.env.ISL`/`OSL`; single-node recipes get `MODEL` and `CONC`, and fixed-sequence ones `RANDOM_RANGE_RATIO` and `USE_CHAT_TEMPLATE` (`true` exactly when the recipe speculates). Multi-node AgentX recipes get the client's `RESULT_DIR`, `AIPERF_DATASET_MMAP_CACHE_DIR` and `HF_HUB_CACHE` from the launcher, which derives them from the volumes it mounts (`volume-mounts`, `agentic-volume-mounts`, lane mounts); a fragment that sets `HF_HOME` keeps its own cache. They also get `IS_MULTINODE: "true"` and `ENABLE_AGENTX_POWER`, `"1"` exactly when the launcher collects the point's AgentX power (a `power: true` row whose lane rule collects AgentX or adapter power) and `"0"` otherwise. The multi-node client reads `CONC_LIST`, `CONC`, `MODEL` and the other matrix inputs, prefill and decode topology included, from the job environment, so fragments do not copy them (`TP`, `PP_SIZE`, `PCP_SIZE` and `EP_SIZE` describe single-node points only); single-node AgentX points get them as runtime arguments. +3. The binder ([`workload.py`](../../../infx/srt_slurm/workload.py)) writes the matrix point into the selected recipe: `model.path: hf:`, `model.container: `, `model.precision`, `identity.container.image` (as a registry reference) and `identity.model.repo` when the fragment declares `identity.container`/`identity.model`, and `benchmark.concurrencies` when telemetry is enabled. Fixed-sequence recipes also get `benchmark.env.ISL`/`OSL`; single-node recipes get `MODEL` and `CONC`, and fixed-sequence ones `RANDOM_RANGE_RATIO` and `USE_CHAT_TEMPLATE` (`true` exactly when the recipe speculates). Multi-node AgentX recipes get the client's `RESULT_DIR`, `AIPERF_DATASET_MMAP_CACHE_DIR` and `HF_HUB_CACHE` from the launcher, which derives them from the volumes it mounts (`volume-mounts`, `agentic-volume-mounts`, lane mounts); a fragment that sets `HF_HOME` keeps its own cache. They also get `IS_MULTINODE: "true"` and `ENABLE_AGENTX_POWER`, `"1"` exactly when the launcher collects the point's AgentX power (a `power: true` row whose lane rule collects AgentX or adapter power) and `"0"` otherwise, and `REQUIRE_POWER`, `"1"` when the workflow requires power or the lane rule marks the point `require` (GB200 and GB300 Kimi-K3 AgentX), so the client fails a point whose power is invalid. The multi-node client reads `CONC_LIST`, `CONC`, `MODEL` and the other matrix inputs, prefill and decode topology included, from the job environment, so fragments do not copy them (`TP`, `PP_SIZE`, `PCP_SIZE` and `EP_SIZE` describe single-node points only); single-node AgentX points get them as runtime arguments. -A fragment that sets a key the binder or launcher writes (`_POINT_KEYS`, `BOUND_ENV` in `workload.py`; single-node variants may still name `CONC`) fails before submission, even with the bound value. `hf:` resolves to the cluster's staged checkpoint (`models.entries`) unless a `models.OVERRIDES` row serves the Hub snapshot, and the master image to its staged container. A multi-node fixed-sequence point with neither checkpoint nor Hub row fails before submission. +A fragment that sets a key the binder or launcher writes, or that the workflow exports from the matrix point (`_POINT_KEYS`, `BOUND_ENV` in `workload.py`; single-node variants may still name `CONC`), fails before submission, even with the bound value. Values `benchmarks/runtime_settings.sh` and the multi-node `FRAMEWORK` settings export reach every process through the job environment, so a fragment sets one only to override it (an AgentX client `TOKENIZER` names a local checkpoint while `MODEL` stays the published Hugging Face id). `hf:` resolves to the cluster's staged checkpoint (`models.entries`) unless a `models.OVERRIDES` row serves the Hub snapshot, and the master image to its staged container. A multi-node fixed-sequence point with neither checkpoint nor Hub row fails before submission. Inspect what the launcher submits without a cluster: diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/mi355x-fp4/8k1k/disagg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/mi355x-fp4/8k1k/disagg-variants.yaml index daf7d1cfef..281f81cc69 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/mi355x-fp4/8k1k/disagg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/mi355x-fp4/8k1k/disagg-variants.yaml @@ -36,10 +36,6 @@ base: NCCL_SOCKET_IFNAME: eno0 MORI_RDMA_TC: '104' MORI_IO_TC: '104' - MORI_IO_SQ_BACKOFF_TIMEOUT_US: '50000' - MORI_IO_QP_MAX_SEND_WR: '16384' - MORI_IO_QP_MAX_CQE: '32768' - MORI_IO_QP_MAX_SGE: '2' MORI_SHMEM_MODE: ISOLATION MORI_ENABLE_SDMA: 'false' MORI_EP_LAUNCH_CONFIG_MODE: AUTO @@ -94,10 +90,6 @@ base: NCCL_SOCKET_IFNAME: eno0 MORI_RDMA_TC: '104' MORI_IO_TC: '104' - MORI_IO_SQ_BACKOFF_TIMEOUT_US: '50000' - MORI_IO_QP_MAX_SEND_WR: '16384' - MORI_IO_QP_MAX_CQE: '32768' - MORI_IO_QP_MAX_SGE: '2' MORI_SHMEM_MODE: ISOLATION MORI_ENABLE_SDMA: 'false' MORI_EP_LAUNCH_CONFIG_MODE: AUTO diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/mi355x-fp8/8k1k/disagg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/mi355x-fp8/8k1k/disagg-variants.yaml index 70ffa1a805..de1991a36a 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/mi355x-fp8/8k1k/disagg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/mi355x-fp8/8k1k/disagg-variants.yaml @@ -36,10 +36,6 @@ base: NCCL_SOCKET_IFNAME: eno0 MORI_RDMA_TC: '104' MORI_IO_TC: '104' - MORI_IO_SQ_BACKOFF_TIMEOUT_US: '50000' - MORI_IO_QP_MAX_SEND_WR: '16384' - MORI_IO_QP_MAX_CQE: '32768' - MORI_IO_QP_MAX_SGE: '2' MORI_SHMEM_MODE: ISOLATION SGLANG_USE_AITER: '1' AITER_LOG_LEVEL: ERROR @@ -89,10 +85,6 @@ base: NCCL_SOCKET_IFNAME: eno0 MORI_RDMA_TC: '104' MORI_IO_TC: '104' - MORI_IO_SQ_BACKOFF_TIMEOUT_US: '50000' - MORI_IO_QP_MAX_SEND_WR: '16384' - MORI_IO_QP_MAX_CQE: '32768' - MORI_IO_QP_MAX_SGE: '2' MORI_SHMEM_MODE: ISOLATION SGLANG_USE_AITER: '1' AITER_LOG_LEVEL: ERROR diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/atom/mi355x-fp4/agentx/disagg-lmcache-dspark.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/atom/mi355x-fp4/agentx/disagg-lmcache-dspark.yaml index 9b5def664e..53039c538e 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/atom/mi355x-fp4/agentx/disagg-lmcache-dspark.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/atom/mi355x-fp4/agentx/disagg-lmcache-dspark.yaml @@ -28,7 +28,6 @@ base: gpus: 8 env: &environment PYTHONUNBUFFERED: "1" - PYTHONDONTWRITEBYTECODE: "1" SAFETENSORS_FAST_GPU: "1" VLLM_LOG_LEVEL: WARNING ATOM_LOG_LEVEL: WARNING diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/mi355x-fp4/agentx/disagg-umbp-dspark.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/mi355x-fp4/agentx/disagg-umbp-dspark.yaml index b3b5ccaf67..bc0ad3270c 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/mi355x-fp4/agentx/disagg-umbp-dspark.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/mi355x-fp4/agentx/disagg-umbp-dspark.yaml @@ -86,11 +86,8 @@ base: HF_HOME: /hf_hub_cache HF_HUB_CACHE: /hf_hub_cache/hub HUGGINGFACE_HUB_CACHE: /hf_hub_cache/hub - PYTHONDONTWRITEBYTECODE: "1" PYTHONUNBUFFERED: "1" PYTHONPATH: /sgl-workspace/aiter - TORCH_NCCL_BLOCKING_WAIT: "1" - NCCL_BLOCKING_WAIT: "1" NCCL_IB_HCA: rdma3,rdma0,rdma2,rdma1,rdma7,rdma4,rdma6,rdma5 GLOO_SOCKET_IFNAME: eno0 NCCL_SOCKET_IFNAME: eno0 @@ -108,7 +105,6 @@ base: SGLANG_USE_AITER: "1" SGLANG_USE_ROCM700A: "0" SGLANG_HACK_FLASHMLA_BACKEND: unified_kv_triton - SGLANG_OPT_USE_AITER_INDEXER: "true" SGLANG_OPT_FP8_WO_A_FUSED_INVROPE: "1" SGLANG_OPT_USE_AITER_BATCHED_GEMM: "1" AITER_BF16_FP8_MOE_BOUND: "0" @@ -116,14 +112,9 @@ base: IBDEVICES: rdma3,rdma0,rdma2,rdma1,rdma7,rdma4,rdma6,rdma5 MORI_RDMA_TC: "104" MORI_IO_TC: "104" - MORI_IO_SQ_BACKOFF_TIMEOUT_US: "500000" MORI_IO_QP_MAX_SEND_WR: "32767" - MORI_IO_QP_MAX_CQE: "32768" - MORI_IO_QP_MAX_SGE: "2" - MORI_IO_TC_DISABLE: "0" MORI_SHMEM_MODE: ISOLATION MORI_EP_LAUNCH_CONFIG_MODE: AUTO - MORI_APP_LOG_LEVEL: WARNING SGLANG_MORI_COMBINE_DTYPE: auto SGLANG_MORI_QP_PER_TRANSFER: "4" SGLANG_MORI_NUM_WORKERS: "4" @@ -136,7 +127,6 @@ base: SGLANG_ENABLE_OVERLAP_PLAN_STREAM: "0" SGLANG_LOG_MS: "true" SGLANG_DISAGGREGATION_NUM_PRE_ALLOCATE_REQS: "32" - SGLANG_ROUTER_STDOUT_LOGS: "0" UMBP_STANDALONE_ADDRESS: unix:///logs/umbp.sock args: &server served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-variants.yaml index a4f0285ae6..eeed345ebe 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-variants.yaml @@ -133,12 +133,6 @@ base: type: dynamo enable_multiple_frontends: false env: - MODEL: deepseek-ai/DeepSeek-V4-Pro - MODEL_PREFIX: dsv4 - FRAMEWORK: dynamo-trt - PRECISION: fp4 - DURATION: '3600' - KV_OFFLOADING: dram ETCD_LEASE_TTL: '120' DYN_ROUTER_QUEUE_THRESHOLD: None DYN_TOKENIZER_CACHE: '1' @@ -195,9 +189,6 @@ override_1p4d_dep4_tep8_c4_b1: backend: TRTLLM moe_expert_parallel_size: 8 tensor_parallel_size: 8 - frontend: - env: - CONC: '4' override_1p6d_dep4_tep4_c24_b4: name: dynamo-disagg-gb300-1p6d-dep4-tep4-c24-b4-mtp @@ -231,9 +222,6 @@ override_1p6d_dep4_tep4_c24_b4: backend: TRTLLM moe_expert_parallel_size: 4 tensor_parallel_size: 4 - frontend: - env: - CONC: '24' override_1p1d_dep8_dep32_c388_b4: name: dynamo-disagg-gb300-1p1d-dep8-dep32-c388-b4-mtp @@ -254,9 +242,6 @@ override_1p1d_dep8_dep32_c388_b4: max_num_tokens: 16 moe_expert_parallel_size: 32 tensor_parallel_size: 32 - frontend: - env: - CONC: '388' override_2p1d_dep8_dep32_c736_b8: name: dynamo-disagg-gb300-2p1d-dep8-dep32-c736-b8-mtp @@ -278,9 +263,6 @@ override_2p1d_dep8_dep32_c736_b8: max_num_tokens: 32 moe_expert_parallel_size: 32 tensor_parallel_size: 32 - frontend: - env: - CONC: '736' override_3p1d_dep8_dep16_c1152_b32: name: dynamo-disagg-gb300-3p1d-dep8-dep16-c1152-b32-mtp @@ -308,9 +290,6 @@ override_3p1d_dep8_dep16_c1152_b32: max_num_tokens: 128 moe_expert_parallel_size: 16 tensor_parallel_size: 16 - frontend: - env: - CONC: '1152' override_5p1d_dep8_dep16_c2626_b96: name: dynamo-disagg-gb300-5p1d-dep8-dep16-c2626-b96-mtp @@ -354,6 +333,3 @@ override_5p1d_dep8_dep16_c2626_b96: max_num_tokens: 384 moe_expert_parallel_size: 16 tensor_parallel_size: 16 - frontend: - env: - CONC: '2626' diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-mtp-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-mtp-variants.yaml index 8c9d21dab9..c0b002be17 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-mtp-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-mtp-variants.yaml @@ -53,7 +53,6 @@ base: gpus: 8 env: HF_HUB_CACHE: /hf_hub_cache - VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_RPC_TIMEOUT: "600000" VLLM_LOG_STATS_INTERVAL: "1" VLLM_SERVER_DEV_MODE: "1" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-mtp3-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-mtp3-variants.yaml index 55f3cb2053..b4f487d8f8 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-mtp3-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-mtp3-variants.yaml @@ -72,7 +72,6 @@ base: HF_HUB_CACHE: "/hf_hub_cache" HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_RPC_TIMEOUT: "600000" VLLM_LOG_STATS_INTERVAL: "1" TILELANG_CLEANUP_TEMP_FILES: "1" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-mtp-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-mtp-variants.yaml index 383bfefd6d..0ab4058ed7 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-mtp-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-mtp-variants.yaml @@ -49,7 +49,6 @@ base: gpus: 8 env: HF_HUB_CACHE: /hf_hub_cache - VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_RPC_TIMEOUT: "600000" VLLM_LOG_STATS_INTERVAL: "1" VLLM_SERVER_DEV_MODE: "1" @@ -93,7 +92,6 @@ base: gpus: 8 env: HF_HUB_CACHE: /hf_hub_cache - VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_RPC_TIMEOUT: "600000" VLLM_LOG_STATS_INTERVAL: "1" VLLM_SERVER_DEV_MODE: "1" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-mtp3-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-mtp3-variants.yaml index 2200ee586c..f3db08445e 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-mtp3-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-mtp3-variants.yaml @@ -70,7 +70,6 @@ base: HF_HUB_CACHE: "/hf_hub_cache" HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_RPC_TIMEOUT: "600000" VLLM_LOG_STATS_INTERVAL: "1" VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" @@ -125,7 +124,6 @@ base: HF_HUB_CACHE: "/hf_hub_cache" HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_RPC_TIMEOUT: "600000" VLLM_LOG_STATS_INTERVAL: "1" VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/agg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/agg-variants.yaml index 210e4a39e1..cff7bb2313 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/agg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/agg-variants.yaml @@ -70,7 +70,6 @@ base: HF_HUB_CACHE: "/hf_hub_cache" HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_RPC_TIMEOUT: "600000" VLLM_LOG_STATS_INTERVAL: "1" TILELANG_CLEANUP_TEMP_FILES: "1" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p4d-dep4-tp8-c4-mtp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p4d-dep4-tp8-c4-mtp.yaml index 88310f19fb..1c1acd2fc2 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p4d-dep4-tp8-c4-mtp.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p4d-dep4-tp8-c4-mtp.yaml @@ -24,7 +24,6 @@ environment: HF_HUB_CACHE: "/hf_hub_cache" HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_RPC_TIMEOUT: "600000" VLLM_LOG_STATS_INTERVAL: "1" VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" @@ -76,13 +75,6 @@ frontend: type: dynamo enable_multiple_frontends: false env: - MODEL: "deepseek-ai/DeepSeek-V4-Pro" - MODEL_PREFIX: "dsv4" - FRAMEWORK: "dynamo-vllm" - PRECISION: "fp4" - CONC: "4" - DURATION: "3600" - KV_OFFLOADING: "none" ETCD_LEASE_TTL: "120" DYN_ROUTER_QUEUE_THRESHOLD: "None" DYN_TOKENIZER_CACHE: "1" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-mtp-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-mtp-variants.yaml index 8fa84d383e..cbec10cb41 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-mtp-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-mtp-variants.yaml @@ -66,7 +66,6 @@ base: HF_HUB_CACHE: "/hf_hub_cache" HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_RPC_TIMEOUT: "600000" VLLM_LOG_STATS_INTERVAL: "1" VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" @@ -122,7 +121,6 @@ base: HF_HUB_CACHE: "/hf_hub_cache" HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_RPC_TIMEOUT: "600000" VLLM_LOG_STATS_INTERVAL: "1" VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-variants.yaml index 501a239064..cda61b4a13 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-variants.yaml @@ -192,7 +192,6 @@ base: AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: "0" SERVED_MODEL_NAME: GLM-5.2-NVFP4 - MAX_MODEL_LEN: '1048576' AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' placement: diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.3/tilert/mi355x-fp8/agentx/disagg-1p1d-tp8-mtp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.3/tilert/mi355x-fp8/agentx/disagg-1p1d-tp8-mtp.yaml index eaa23f6d5b..e78bd69b1b 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.3/tilert/mi355x-fp8/agentx/disagg-1p1d-tp8-mtp.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.3/tilert/mi355x-fp8/agentx/disagg-1p1d-tp8-mtp.yaml @@ -9,7 +9,6 @@ setup_script: glm5.3-tilert-rocm.sh environment: TILERT_VERSION: "0.1.6.post3" RDMAV_FORK_SAFE: "1" - PYTHONDONTWRITEBYTECODE: "1" NCCL_SOCKET_IFNAME: eno0 GLOO_SOCKET_IFNAME: eno0 NCCL_IB_HCA: rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7 @@ -96,7 +95,7 @@ health_check: interval_seconds: 5 benchmark: env: - MODEL: /model + TOKENIZER: /model SERVED_MODEL_NAME: glm5_2 AIPERF_MAX_CONTEXT_LENGTH: "1048576" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c1.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c1.yaml index a739fcb75d..d9ec93530d 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c1.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c1.yaml @@ -43,7 +43,6 @@ roles: VLLM_USE_DIRECT_DCP_A2A: "1" VLLM_USE_DIRECT_DCP_Q_GATHER: "1" VLLM_USE_DIRECT_DCP_KV_GATHER: "1" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" VLLM_USE_V2_MODEL_RUNNER: "1" VLLM_SERVER_DEV_MODE: "1" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c14.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c14.yaml index 685a0486b6..cca7e52ccc 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c14.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c14.yaml @@ -43,7 +43,6 @@ roles: VLLM_USE_DIRECT_DCP_A2A: "1" VLLM_USE_DIRECT_DCP_Q_GATHER: "1" VLLM_USE_DIRECT_DCP_KV_GATHER: "1" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" VLLM_USE_V2_MODEL_RUNNER: "1" VLLM_SERVER_DEV_MODE: "1" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c24.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c24.yaml index f93a44b717..0ed4f7b5db 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c24.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c24.yaml @@ -43,7 +43,6 @@ roles: VLLM_USE_DIRECT_DCP_A2A: "1" VLLM_USE_DIRECT_DCP_Q_GATHER: "1" VLLM_USE_DIRECT_DCP_KV_GATHER: "1" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" VLLM_USE_V2_MODEL_RUNNER: "1" VLLM_SERVER_DEV_MODE: "1" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c32.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c32.yaml index 8a85607222..ca52456925 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c32.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c32.yaml @@ -43,7 +43,6 @@ roles: VLLM_USE_DIRECT_DCP_A2A: "1" VLLM_USE_DIRECT_DCP_Q_GATHER: "1" VLLM_USE_DIRECT_DCP_KV_GATHER: "1" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" VLLM_USE_V2_MODEL_RUNNER: "1" VLLM_SERVER_DEV_MODE: "1" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c4.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c4.yaml index 51c49b6fe7..d96fb574a4 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c4.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c4.yaml @@ -43,7 +43,6 @@ roles: VLLM_USE_DIRECT_DCP_A2A: "1" VLLM_USE_DIRECT_DCP_Q_GATHER: "1" VLLM_USE_DIRECT_DCP_KV_GATHER: "1" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" VLLM_USE_V2_MODEL_RUNNER: "1" VLLM_SERVER_DEV_MODE: "1" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c48.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c48.yaml index 939f082257..b7331204c7 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c48.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c48.yaml @@ -43,7 +43,6 @@ roles: VLLM_USE_DIRECT_DCP_A2A: "1" VLLM_USE_DIRECT_DCP_Q_GATHER: "1" VLLM_USE_DIRECT_DCP_KV_GATHER: "1" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" VLLM_USE_V2_MODEL_RUNNER: "1" VLLM_SERVER_DEV_MODE: "1" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c56.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c56.yaml index b9f8c68431..9532262c72 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c56.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c56.yaml @@ -43,7 +43,6 @@ roles: VLLM_USE_DIRECT_DCP_A2A: "1" VLLM_USE_DIRECT_DCP_Q_GATHER: "1" VLLM_USE_DIRECT_DCP_KV_GATHER: "1" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" VLLM_USE_V2_MODEL_RUNNER: "1" VLLM_SERVER_DEV_MODE: "1" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c72.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c72.yaml index cb666cb58f..93ac90d4fc 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c72.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c72.yaml @@ -43,7 +43,6 @@ roles: VLLM_USE_DIRECT_DCP_A2A: "1" VLLM_USE_DIRECT_DCP_Q_GATHER: "1" VLLM_USE_DIRECT_DCP_KV_GATHER: "1" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" VLLM_USE_V2_MODEL_RUNNER: "1" VLLM_SERVER_DEV_MODE: "1" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c8.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c8.yaml index 4d342809ae..eeebae0698 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c8.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c8.yaml @@ -43,7 +43,6 @@ roles: VLLM_USE_DIRECT_DCP_A2A: "1" VLLM_USE_DIRECT_DCP_Q_GATHER: "1" VLLM_USE_DIRECT_DCP_KV_GATHER: "1" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" VLLM_USE_V2_MODEL_RUNNER: "1" VLLM_SERVER_DEV_MODE: "1" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c80.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c80.yaml index 4ac947e16e..b2106d0e35 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c80.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c80.yaml @@ -43,7 +43,6 @@ roles: VLLM_USE_DIRECT_DCP_A2A: "1" VLLM_USE_DIRECT_DCP_Q_GATHER: "1" VLLM_USE_DIRECT_DCP_KV_GATHER: "1" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" VLLM_USE_V2_MODEL_RUNNER: "1" VLLM_SERVER_DEV_MODE: "1" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c96.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c96.yaml index 79b11d98d6..e2e6e22b07 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c96.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c96.yaml @@ -43,7 +43,6 @@ roles: VLLM_USE_DIRECT_DCP_A2A: "1" VLLM_USE_DIRECT_DCP_Q_GATHER: "1" VLLM_USE_DIRECT_DCP_KV_GATHER: "1" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" VLLM_USE_V2_MODEL_RUNNER: "1" VLLM_SERVER_DEV_MODE: "1" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dcp16-dspark4-maxseq2-mooncake.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dcp16-dspark4-maxseq2-mooncake.yaml index 12b7c9ce4a..b3afda8ff3 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dcp16-dspark4-maxseq2-mooncake.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dcp16-dspark4-maxseq2-mooncake.yaml @@ -80,7 +80,6 @@ roles: VLLM_USE_DIRECT_DCP_A2A: "0" VLLM_USE_DIRECT_DCP_Q_GATHER: "0" VLLM_USE_DIRECT_DCP_KV_GATHER: "0" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_RPC_TIMEOUT: "600000" VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: "0" @@ -143,7 +142,6 @@ benchmark: placement: node: head env: - REQUIRE_POWER: "1" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dcp16-nospec-mooncake.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dcp16-nospec-mooncake.yaml index 4ae9f39e08..333fb119fd 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dcp16-nospec-mooncake.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dcp16-nospec-mooncake.yaml @@ -80,7 +80,6 @@ roles: VLLM_USE_DIRECT_DCP_A2A: "0" VLLM_USE_DIRECT_DCP_Q_GATHER: "0" VLLM_USE_DIRECT_DCP_KV_GATHER: "0" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_RPC_TIMEOUT: "600000" VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: "0" @@ -139,7 +138,6 @@ benchmark: placement: node: head env: - REQUIRE_POWER: "1" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c16.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c16.yaml index fec3a725a6..b5f532392e 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c16.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c16.yaml @@ -43,7 +43,6 @@ roles: VLLM_USE_DIRECT_DCP_A2A: "1" VLLM_USE_DIRECT_DCP_Q_GATHER: "1" VLLM_USE_DIRECT_DCP_KV_GATHER: "1" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" VLLM_USE_V2_MODEL_RUNNER: "1" VLLM_SERVER_DEV_MODE: "1" @@ -112,7 +111,6 @@ benchmark: placement: node: head env: - REQUIRE_POWER: "1" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" AIPERF_SERVER_METRICS_URLS: "http://localhost:8000/metrics" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c32.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c32.yaml index 000d53c5dd..3e461deccb 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c32.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c32.yaml @@ -43,7 +43,6 @@ roles: VLLM_USE_DIRECT_DCP_A2A: "1" VLLM_USE_DIRECT_DCP_Q_GATHER: "1" VLLM_USE_DIRECT_DCP_KV_GATHER: "1" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" VLLM_USE_V2_MODEL_RUNNER: "1" VLLM_SERVER_DEV_MODE: "1" @@ -112,7 +111,6 @@ benchmark: placement: node: head env: - REQUIRE_POWER: "1" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" AIPERF_SERVER_METRICS_URLS: "http://localhost:8000/metrics" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c48.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c48.yaml index 4b8f04652a..b0ee1b07c4 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c48.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c48.yaml @@ -43,7 +43,6 @@ roles: VLLM_USE_DIRECT_DCP_A2A: "1" VLLM_USE_DIRECT_DCP_Q_GATHER: "1" VLLM_USE_DIRECT_DCP_KV_GATHER: "1" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" VLLM_USE_V2_MODEL_RUNNER: "1" VLLM_SERVER_DEV_MODE: "1" @@ -111,7 +110,6 @@ benchmark: placement: node: head env: - REQUIRE_POWER: "1" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" AIPERF_SERVER_METRICS_URLS: "http://localhost:8000/metrics" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c72.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c72.yaml index 57e894f523..4755fec2db 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c72.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c72.yaml @@ -43,7 +43,6 @@ roles: VLLM_USE_DIRECT_DCP_A2A: "1" VLLM_USE_DIRECT_DCP_Q_GATHER: "1" VLLM_USE_DIRECT_DCP_KV_GATHER: "1" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" VLLM_USE_V2_MODEL_RUNNER: "1" VLLM_SERVER_DEV_MODE: "1" @@ -111,7 +110,6 @@ benchmark: placement: node: head env: - REQUIRE_POWER: "1" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" AIPERF_SERVER_METRICS_URLS: "http://localhost:8000/metrics" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c96.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c96.yaml index 17e0743f63..991440af71 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c96.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c96.yaml @@ -43,7 +43,6 @@ roles: VLLM_USE_DIRECT_DCP_A2A: "1" VLLM_USE_DIRECT_DCP_Q_GATHER: "1" VLLM_USE_DIRECT_DCP_KV_GATHER: "1" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" VLLM_USE_V2_MODEL_RUNNER: "1" VLLM_SERVER_DEV_MODE: "1" @@ -111,7 +110,6 @@ benchmark: placement: node: head env: - REQUIRE_POWER: "1" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" AIPERF_SERVER_METRICS_URLS: "http://localhost:8000/metrics" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/agg-dcp8-dspark4-mooncake.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/agg-dcp8-dspark4-mooncake.yaml index 15bfa64e43..77a441c60d 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/agg-dcp8-dspark4-mooncake.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/agg-dcp8-dspark4-mooncake.yaml @@ -77,7 +77,6 @@ roles: VLLM_USE_DIRECT_DCP_Q_GATHER: "1" VLLM_USE_DIRECT_DCP_KV_GATHER: "1" ETCD_LEASE_TTL: "120" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_RPC_TIMEOUT: "600000" VLLM_USE_RUST_FRONTEND: "1" VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" @@ -128,7 +127,6 @@ telemetry: benchmark: env: - REQUIRE_POWER: "1" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/agg-dcp8-dspark7-maxseq2-mooncake.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/agg-dcp8-dspark7-maxseq2-mooncake.yaml index f7cf792b91..9a310407e8 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/agg-dcp8-dspark7-maxseq2-mooncake.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/agg-dcp8-dspark7-maxseq2-mooncake.yaml @@ -74,7 +74,6 @@ roles: VLLM_USE_DIRECT_DCP_Q_GATHER: "1" VLLM_USE_DIRECT_DCP_KV_GATHER: "1" ETCD_LEASE_TTL: "120" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_RPC_TIMEOUT: "600000" VLLM_USE_V2_MODEL_RUNNER: "1" VLLM_USE_RUST_FRONTEND: "1" @@ -126,7 +125,6 @@ telemetry: benchmark: env: - REQUIRE_POWER: "1" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" placement: node: head diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p1d-dcp8-dcp8-dspark4-mooncake.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p1d-dcp8-dcp8-dspark4-mooncake.yaml index 66b252bf46..0341182508 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p1d-dcp8-dcp8-dspark4-mooncake.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p1d-dcp8-dcp8-dspark4-mooncake.yaml @@ -75,7 +75,6 @@ roles: VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT: "0" DYN_REQUEST_PLANE: "tcp" ETCD_LEASE_TTL: "600" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_RPC_TIMEOUT: "600000" TILELANG_CLEANUP_TEMP_FILES: "1" VLLM_USE_NCCL_SYMM_MEM: "0" @@ -172,7 +171,6 @@ benchmark: placement: node: head env: - REQUIRE_POWER: "1" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p2d-dcp8-dcp8-dspark4-mooncake.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p2d-dcp8-dcp8-dspark4-mooncake.yaml index 096fb1b01a..7a98fe849e 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p2d-dcp8-dcp8-dspark4-mooncake.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p2d-dcp8-dcp8-dspark4-mooncake.yaml @@ -75,7 +75,6 @@ roles: VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT: "0" DYN_REQUEST_PLANE: "tcp" ETCD_LEASE_TTL: "600" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_RPC_TIMEOUT: "600000" TILELANG_CLEANUP_TEMP_FILES: "1" VLLM_USE_NCCL_SYMM_MEM: "0" @@ -172,7 +171,6 @@ benchmark: placement: node: head env: - REQUIRE_POWER: "1" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p3d-dcp8-dcp8-dspark4-mooncake.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p3d-dcp8-dcp8-dspark4-mooncake.yaml index 0e87cae609..93caa9b2e1 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p3d-dcp8-dcp8-dspark4-mooncake.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p3d-dcp8-dcp8-dspark4-mooncake.yaml @@ -75,7 +75,6 @@ roles: VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT: "0" DYN_REQUEST_PLANE: "tcp" ETCD_LEASE_TTL: "600" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_RPC_TIMEOUT: "600000" TILELANG_CLEANUP_TEMP_FILES: "1" VLLM_USE_NCCL_SYMM_MEM: "0" @@ -172,7 +171,6 @@ benchmark: placement: node: head env: - REQUIRE_POWER: "1" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p3d-dcp8-dcp8-dspark7-mooncake.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p3d-dcp8-dcp8-dspark7-mooncake.yaml index 51c43c64cc..b82ce62b75 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p3d-dcp8-dcp8-dspark7-mooncake.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p3d-dcp8-dcp8-dspark7-mooncake.yaml @@ -75,7 +75,6 @@ roles: VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT: "0" DYN_REQUEST_PLANE: "tcp" ETCD_LEASE_TTL: "600" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_RPC_TIMEOUT: "600000" TILELANG_CLEANUP_TEMP_FILES: "1" VLLM_USE_NCCL_SYMM_MEM: "0" @@ -172,7 +171,6 @@ benchmark: placement: node: head env: - REQUIRE_POWER: "1" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-variants.yaml index d78fef2a09..3a53ca670b 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-variants.yaml @@ -113,10 +113,8 @@ base: AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: '0' SERVED_MODEL_NAME: nvidia/MiniMax-M3-NVFP4 - MAX_MODEL_LEN: '1048576' AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - KV_OFFLOAD_BACKEND_METADATA: '{"name":"native"}' # MiniMax-M3 NVFP4, TensorRT-LLM aggregated TP4 + EAGLE3-GQA (3 draft tokens), one GB200 node, Dynamo frontend. # Engine config is the max_batch_size=5 member of the TP4 recipe set. diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-variants.yaml index 6280fd1aef..f0d50e6034 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-variants.yaml @@ -67,7 +67,6 @@ base: HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" TRANSFORMERS_CACHE: "/hf_hub_cache" DYN_TCP_CONNECT_TIMEOUT: "120" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_FLOAT32_MATMUL_PRECISION: "high" VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" @@ -127,7 +126,6 @@ base: HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" TRANSFORMERS_CACHE: "/hf_hub_cache" DYN_TCP_CONNECT_TIMEOUT: "120" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_FLOAT32_MATMUL_PRECISION: "high" VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" VLLM_USE_NCCL_SYMM_MEM: "0" @@ -175,7 +173,6 @@ base: AGENTIC_WARMUP_GRACE_PERIOD: "1800" AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "3600" AIPERF_EXTRA_INPUTS: "thinking:true" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-variants.yaml index fbf86ee2db..2d1fd6b822 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-variants.yaml @@ -70,7 +70,6 @@ base: HF_HUB_CACHE: "/hf_hub_cache" HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" DYN_TCP_CONNECT_TIMEOUT: "120" VLLM_FLOAT32_MATMUL_PRECISION: "high" VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" @@ -120,7 +119,6 @@ base: HF_HUB_CACHE: "/hf_hub_cache" HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" DYN_TCP_CONNECT_TIMEOUT: "120" VLLM_FLOAT32_MATMUL_PRECISION: "high" VLLM_USE_NCCL_SYMM_MEM: "0" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4/8k1k/disagg-1p1d-tp8.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4/8k1k/disagg-1p1d-tp8.yaml index 227ef824ae..c9175872b0 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4/8k1k/disagg-1p1d-tp8.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4/8k1k/disagg-1p1d-tp8.yaml @@ -46,10 +46,6 @@ roles: NCCL_SOCKET_IFNAME: "eno0" MORI_RDMA_TC: "104" MORI_IO_TC: "104" - MORI_IO_SQ_BACKOFF_TIMEOUT_US: "50000" - MORI_IO_QP_MAX_SEND_WR: "16384" - MORI_IO_QP_MAX_CQE: "32768" - MORI_IO_QP_MAX_SGE: "2" MORI_SHMEM_MODE: "ISOLATION" SGLANG_USE_AITER: "1" AITER_LOG_LEVEL: "ERROR" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp8/8k1k/disagg-1p1d-p-tp4-d-tp8.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp8/8k1k/disagg-1p1d-p-tp4-d-tp8.yaml index 7731e7e4e3..aebf8db25a 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp8/8k1k/disagg-1p1d-p-tp4-d-tp8.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp8/8k1k/disagg-1p1d-p-tp4-d-tp8.yaml @@ -52,10 +52,6 @@ roles: NCCL_SOCKET_IFNAME: "eno0" MORI_RDMA_TC: "104" MORI_IO_TC: "104" - MORI_IO_SQ_BACKOFF_TIMEOUT_US: "50000" - MORI_IO_QP_MAX_SEND_WR: "16384" - MORI_IO_QP_MAX_CQE: "32768" - MORI_IO_QP_MAX_SGE: "2" MORI_SHMEM_MODE: "ISOLATION" SGLANG_USE_AITER: "1" AITER_LOG_LEVEL: "ERROR" diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/h100-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/h100-fp4-mtp/agentic.yaml index 41fe46c57f..8bd5e32e77 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/h100-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/h100-fp4-mtp/agentic.yaml @@ -48,7 +48,6 @@ base: gpu-memory-utilization: 0.92 disable-uvicorn-access-log: true env: - VLLM_ENGINE_READY_TIMEOUT_S: '3600' VLLM_USE_RUST_FRONTEND: '1' VLLM_USE_V2_MODEL_RUNNER: '1' # The indexer buffer is large enough that allocator fragmentation costs a KV block. diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi300x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi300x-fp4-mtp/agentic.yaml index 95138e7833..3f4be8b885 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi300x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi300x-fp4-mtp/agentic.yaml @@ -63,7 +63,6 @@ base: AITER_TRITON_LOG_LEVEL: ERROR VLLM_USE_BREAKABLE_CUDAGRAPH: '0' OMP_NUM_THREADS: '1' - VLLM_ENGINE_READY_TIMEOUT_S: '3600' VLLM_USE_RUST_FRONTEND: '1' PYTHONUNBUFFERED: '1' benchmark: diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi325x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi325x-fp4-mtp/agentic.yaml index 1caab526b6..def8329e20 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi325x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi325x-fp4-mtp/agentic.yaml @@ -63,7 +63,6 @@ base: AITER_TRITON_LOG_LEVEL: ERROR VLLM_USE_BREAKABLE_CUDAGRAPH: '0' OMP_NUM_THREADS: '1' - VLLM_ENGINE_READY_TIMEOUT_S: '3600' VLLM_USE_RUST_FRONTEND: '1' PYTHONUNBUFFERED: '1' benchmark: diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi355x-fp4-mtp/agentic.yaml index c422d72df2..52f1125e17 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi355x-fp4-mtp/agentic.yaml @@ -65,7 +65,6 @@ base: # graphs keep FULL_AND_PIECEWISE capture working. VLLM_USE_BREAKABLE_CUDAGRAPH: '1' OMP_NUM_THREADS: '1' - VLLM_ENGINE_READY_TIMEOUT_S: '3600' VLLM_USE_RUST_FRONTEND: '1' PYTHONUNBUFFERED: '1' benchmark: diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3/vllm/b300-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3/vllm/b300-fp4-mtp/agentic.yaml index 015ae6930b..dc6ee2a42a 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3/vllm/b300-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3/vllm/b300-fp4-mtp/agentic.yaml @@ -78,7 +78,6 @@ base: VLLM_USE_DIRECT_DCP_A2A: '1' VLLM_USE_DIRECT_DCP_Q_GATHER: '1' VLLM_USE_DIRECT_DCP_KV_GATHER: '1' - VLLM_ENGINE_READY_TIMEOUT_S: '3600' VLLM_RPC_TIMEOUT: '600000' VLLM_PREFIX_CACHE_RETENTION_INTERVAL: '0' VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0' diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b200-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b200-fp4-mtp/agentic.yaml index b73b19c951..734ac80bdf 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b200-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b200-fp4-mtp/agentic.yaml @@ -47,7 +47,6 @@ base: speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASHINFER","use_local_argmax_reduction":true}' env: PYTHONNOUSERSITE: '1' - VLLM_ENGINE_READY_TIMEOUT_S: '3600' VLLM_FLOAT32_MATMUL_PRECISION: high VLLM_FLASHINFER_ALLREDUCE_BACKEND: trtllm benchmark: diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4-mtp/agentic.yaml index fff694c9d5..a7bef4e0d6 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4-mtp/agentic.yaml @@ -46,7 +46,6 @@ base: speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASHINFER","use_local_argmax_reduction":true}' env: PYTHONNOUSERSITE: '1' - VLLM_ENGINE_READY_TIMEOUT_S: '3600' VLLM_FLOAT32_MATMUL_PRECISION: high VLLM_FLASHINFER_ALLREDUCE_BACKEND: trtllm benchmark: diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/mi300x-fp8-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/mi300x-fp8-mtp/agentic.yaml index f6ef776145..e6d0175cbe 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/mi300x-fp8-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/mi300x-fp8-mtp/agentic.yaml @@ -50,7 +50,6 @@ base: speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"TRITON_ATTN"}' env: PYTHONNOUSERSITE: '1' - VLLM_ENGINE_READY_TIMEOUT_S: '3600' VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: '1800' VLLM_USE_BREAKABLE_CUDAGRAPH: '0' VLLM_ROCM_USE_AITER: '1' diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/mi325x-fp8-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/mi325x-fp8-mtp/agentic.yaml index 1789ac13f1..3932982d9f 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/mi325x-fp8-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/mi325x-fp8-mtp/agentic.yaml @@ -47,7 +47,6 @@ base: stream-interval: 20 env: PYTHONNOUSERSITE: '1' - VLLM_ENGINE_READY_TIMEOUT_S: '3600' VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: '1800' VLLM_USE_BREAKABLE_CUDAGRAPH: '0' benchmark: diff --git a/inferencex-e2e/docs/results-and-ingestion.md b/inferencex-e2e/docs/results-and-ingestion.md index 2b4353e389..6b2980d2c8 100644 --- a/inferencex-e2e/docs/results-and-ingestion.md +++ b/inferencex-e2e/docs/results-and-ingestion.md @@ -185,6 +185,9 @@ writes both into every srt-slurm multi-node AgentX recipe: `IS_MULTINODE=true`, `ENABLE_AGENTX_POWER=1` exactly when the launcher collects the point's AgentX power (a `power: true` row whose lane rule collects AgentX or adapter power). Other multi-node points get `ENABLE_AGENTX_POWER=0` and publish no power fields or verdict. +`REQUIRE_POWER` is bound beside them: `1` when the workflow requires power or the +lane's power rule does (GB200 and GB300 Kimi-K3 AgentX), so the client fails a point +whose power is invalid. Multinode runs retain the deployment telemetry under `LOGS/power/` and per-concurrency window/validation files under `LOGS/agentic/` in their `power_audit_` artifact. Available audits and AgentX aggregates diff --git a/inferencex-e2e/infx/bench/agentic/replay.py b/inferencex-e2e/infx/bench/agentic/replay.py index 76df8aae12..34e4963d50 100644 --- a/inferencex-e2e/infx/bench/agentic/replay.py +++ b/inferencex-e2e/infx/bench/agentic/replay.py @@ -36,7 +36,8 @@ class ReplayConfig: model: str """Served name sent as ``--model``.""" tokenizer: str - """Hugging Face id; a wire name is not necessarily a valid repo id.""" + """``TOKENIZER`` (a recipe's checkpoint path), else the Hugging Face id; a wire name is not + necessarily a valid repo id.""" concurrency: int duration: int # None keeps the agentx scenario's default. @@ -80,7 +81,7 @@ def from_env(cls, env: Mapping[str, str], result_dir: Path) -> ReplayConfig: return cls( url=_server_url(env), model=inputs.optional("SERVED_MODEL_NAME", env) or values["MODEL"], - tokenizer=values["MODEL"], + tokenizer=inputs.optional("TOKENIZER", env) or values["MODEL"], concurrency=inputs.parse_positive_int("CONC", values["CONC"]), duration=duration, warmup_requests_per_lane=warmup, diff --git a/inferencex-e2e/infx/launch/drivers/srt/config.py b/inferencex-e2e/infx/launch/drivers/srt/config.py index c7d5f762fa..0ed17c2cce 100644 --- a/inferencex-e2e/infx/launch/drivers/srt/config.py +++ b/inferencex-e2e/infx/launch/drivers/srt/config.py @@ -240,7 +240,8 @@ def binder_inputs( ) -> tuple[int | None, dict[str, str]]: """The DCGM exporter port and AgentX client env a multi-node recipe is bound with. - The client marks power windows exactly when the launcher collects the point's AgentX power. + The client marks power windows exactly when the launcher collects the point's AgentX power, + and fails invalid power when the workflow or the lane's power rule requires it. """ port = srt.power_exporter_port if decision.dcgm else None if not request.is_agentic: @@ -249,6 +250,7 @@ def binder_inputs( **agentic_client_env(cluster, srt, lane, request), "IS_MULTINODE": str(request.is_multinode).lower(), "ENABLE_AGENTX_POWER": "1" if decision.agentx or decision.adapter else "0", + "REQUIRE_POWER": "1" if decision.require or request.require_power else "0", } diff --git a/inferencex-e2e/infx/launch/drivers/srt/power.py b/inferencex-e2e/infx/launch/drivers/srt/power.py index 4991081b5e..61d2270970 100644 --- a/inferencex-e2e/infx/launch/drivers/srt/power.py +++ b/inferencex-e2e/infx/launch/drivers/srt/power.py @@ -22,11 +22,12 @@ class PowerPolicyError(LaunchError): @dataclass(frozen=True) class PowerRule: - """One allowed power combination.""" + """One allowed power combination; ``require`` makes the client fail invalid AgentX power.""" when: Match agentx: bool adapter: bool = False + require: bool = False @dataclass(frozen=True) @@ -48,6 +49,7 @@ class PowerLane: PowerRule( Match(any_of("kimik3"), any_of("fp4"), any_of("dynamo-vllm"), agentic=True), agentx=True, + require=True, ), PowerRule(Match(frameworks=any_of("dynamo-sglang"), agentic=False), agentx=False), ), @@ -59,6 +61,7 @@ class PowerLane: PowerRule( Match(any_of("kimik3"), any_of("fp4"), any_of("dynamo-vllm"), agentic=True), agentx=True, + require=True, ), PowerRule(Match(frameworks=any_of("dynamo-sglang")), agentx=False), ), @@ -134,6 +137,7 @@ class PowerDecision: dcgm: bool agentx: bool adapter: bool = False + require: bool = False NO_POWER = PowerDecision(dcgm=False, agentx=False) @@ -151,6 +155,8 @@ def decide_power(cluster_id: str, path: LaunchPath, request: MultiNodeRequest) - raise PowerPolicyError(f"cluster {cluster_id!r} measures no DCGM power on {path}") for rule in lane.rules: if rule.when(request): - return PowerDecision(dcgm=True, agentx=rule.agentx, adapter=rule.adapter) + return PowerDecision( + dcgm=True, agentx=rule.agentx, adapter=rule.adapter, require=rule.require + ) message = lane.agentic_error if request.is_agentic and lane.agentic_error else lane.error raise PowerPolicyError(message) diff --git a/inferencex-e2e/infx/srt_slurm/workload.py b/inferencex-e2e/infx/srt_slurm/workload.py index 60a7d28183..b57e2d083f 100644 --- a/inferencex-e2e/infx/srt_slurm/workload.py +++ b/inferencex-e2e/infx/srt_slurm/workload.py @@ -31,7 +31,7 @@ TELEMETRY_BLOCK = Path("configs/srt-recipes/telemetry-dcgm.yaml") # The binder writes these, the master power field owns telemetry, and the launcher hands # the benchmark client its cache and result paths and, multi-node, IS_MULTINODE and the -# AgentX power switch. +# AgentX power switches. _POINT_KEYS = ( ("model", "path"), ("model", "container"), @@ -45,12 +45,29 @@ _FIXED_ENV = ("MODEL", "ISL", "OSL", "CONC", "CONC_LIST", "RANDOM_RANGE_RATIO", "USE_CHAT_TEMPLATE") _AGENTIC_ENV = ("CONC", "CONC_LIST", "RESULT_DIR", "AGENTIC_OUTPUT_DIR", "HF_HUB_CACHE", "HUGGINGFACE_HUB_CACHE") # fmt: skip +# The workflow exports the matrix point to every job, so no fragment pins it. A single-node +# variant may still label its point with CONC, SPEC_DECODING, KV_OFFLOADING and +# TOTAL_CPU_DRAM_GB, which the launcher checks against the point. +_WORKFLOW_ENV = ( + "IMAGE", "MODEL_PREFIX", "FRAMEWORK", "PRECISION", "DISAGG", "DURATION", "MAX_MODEL_LEN", + "KV_OFFLOAD_BACKEND", "KV_OFFLOAD_BACKEND_METADATA", "ROUTER_METADATA", "KV_P2P_TRANSFER", + "POWER", "REQUIRE_POWER", "IS_AGENTIC", "SCENARIO_TYPE", "RUN_EVAL", "EVAL_ONLY", +) # fmt: skip +_MULTINODE_ENV = ( + "SPEC_DECODING", "KV_OFFLOADING", "TOTAL_CPU_DRAM_GB", "PREFILL_DWDP_SIZE", + *(f"{role}_{field}" for role in ("PREFILL", "DECODE") + for field in ("NUM_WORKERS", "TP", "PP_SIZE", "DCP_SIZE", "PCP_SIZE", "EP", "DP_ATTN")), +) # fmt: skip +_SINGLE_NODE_ENV = ("TP", "EP_SIZE", "PP_SIZE", "DCP_SIZE", "PCP_SIZE", "DP_ATTENTION", "GPU_COUNT") BOUND_ENV = { - (False, False): _FIXED_ENV, - (False, True): _FIXED_ENV, - (True, False): ("MODEL", *_AGENTIC_ENV), + (False, False): (*_FIXED_ENV, *_WORKFLOW_ENV, *_SINGLE_NODE_ENV), + (False, True): (*_FIXED_ENV, *_WORKFLOW_ENV, *_MULTINODE_ENV), + (True, False): ("MODEL", *_AGENTIC_ENV, *_WORKFLOW_ENV, *_SINGLE_NODE_ENV), (True, True): ( + "MODEL", *_AGENTIC_ENV, + *_WORKFLOW_ENV, + *_MULTINODE_ENV, "AIPERF_DATASET_MMAP_CACHE_DIR", "IS_MULTINODE", "ENABLE_AGENTX_POWER", @@ -98,7 +115,8 @@ def check_fragment(raw: Mapping[str, Any], source: Path, *, agentic: bool, multi found.append(f"{'.'.join(filter(None, (name, *key)))} (= {value!r})") if found: raise ValueError( - f"{source}: remove {', '.join(found)} from the fragment; the launcher binds them" + f"{source}: remove {', '.join(found)} from the fragment; " + "the launcher or the workflow sets them" ) diff --git a/inferencex-e2e/infx/tests/bench/test_agentic_replay.py b/inferencex-e2e/infx/tests/bench/test_agentic_replay.py index e9c1841d23..4ea3f81c57 100644 --- a/inferencex-e2e/infx/tests/bench/test_agentic_replay.py +++ b/inferencex-e2e/infx/tests/bench/test_agentic_replay.py @@ -45,6 +45,11 @@ def test_default_point_leaves_the_replay_policy_to_the_scenario(): ] # fmt: skip +def test_a_recipe_tokenizer_replaces_the_model_id_as_tokenizer_only(): + argv = _argv(TOKENIZER="/model", SERVED_MODEL_NAME="glm5_2") + assert (_value(argv, "--model"), _value(argv, "--tokenizer")) == ("glm5_2", "/model") + + def test_opt_in_inputs_add_their_flags_in_place(): argv = _argv( SERVED_MODEL_NAME="DeepSeek-V4-Pro", diff --git a/inferencex-e2e/infx/tests/launch/test_launch_policy.py b/inferencex-e2e/infx/tests/launch/test_launch_policy.py index 5b4398a386..3dd54fe0b6 100644 --- a/inferencex-e2e/infx/tests/launch/test_launch_policy.py +++ b/inferencex-e2e/infx/tests/launch/test_launch_policy.py @@ -74,6 +74,8 @@ def test_power_eligibility_of_each_cluster_lane( assert (decision.dcgm, decision.agentx, decision.adapter) == ( True, expected == AGENTX, expected == ADAPTER, ) # fmt: skip + # GB200 and GB300 Kimi-K3 AgentX clients fail a point whose power is invalid. + assert decision.require is (prefix == "kimik3" and cluster in ("gb200-nv", "gb300-nv")) assert not decide_power(cluster, path, _multinode(**env, POWER="0")).dcgm diff --git a/inferencex-e2e/infx/tests/launch/test_srt_config.py b/inferencex-e2e/infx/tests/launch/test_srt_config.py index 9758eb977b..59133d29d8 100644 --- a/inferencex-e2e/infx/tests/launch/test_srt_config.py +++ b/inferencex-e2e/infx/tests/launch/test_srt_config.py @@ -235,12 +235,13 @@ def rsync(source: Path, staging: Path, *, exclude: tuple[str, ...]) -> Path: @pytest.mark.parametrize(("agentic", "decision", "expected"), [ - ("1", NO_POWER, (None, "true", "0")), + ("1", NO_POWER, (None, "true", "0", "0")), # DCGM alone: the launcher collects no AgentX power, so the client marks no window. - ("1", PowerDecision(dcgm=True, agentx=False), (9401, "true", "0")), - ("1", PowerDecision(dcgm=True, agentx=True), (9401, "true", "1")), - ("1", PowerDecision(dcgm=True, agentx=False, adapter=True), (9401, "true", "1")), - ("0", PowerDecision(dcgm=True, agentx=False), (9401, None, None)), + ("1", PowerDecision(dcgm=True, agentx=False), (9401, "true", "0", "0")), + ("1", PowerDecision(dcgm=True, agentx=True), (9401, "true", "1", "0")), + ("1", PowerDecision(dcgm=True, agentx=True, require=True), (9401, "true", "1", "1")), + ("1", PowerDecision(dcgm=True, agentx=False, adapter=True), (9401, "true", "1", "0")), + ("0", PowerDecision(dcgm=True, agentx=False), (9401, None, None, None)), ]) # fmt: skip def test_the_agentx_client_measures_power_exactly_when_the_launcher_collects_it( tmp_path, agentic, decision, expected @@ -258,5 +259,9 @@ def test_the_agentx_client_measures_power_exactly_when_the_launcher_collects_it( record, slurm_settings(record).srt_slurm, SrtLane(), request, decision ) - switches = (client_env.get("IS_MULTINODE"), client_env.get("ENABLE_AGENTX_POWER")) + switches = ( + client_env.get("IS_MULTINODE"), + client_env.get("ENABLE_AGENTX_POWER"), + client_env.get("REQUIRE_POWER"), + ) assert (port, *switches) == expected diff --git a/inferencex-e2e/infx/tests/launch/test_srt_driver.py b/inferencex-e2e/infx/tests/launch/test_srt_driver.py index f63065baf6..6564002766 100644 --- a/inferencex-e2e/infx/tests/launch/test_srt_driver.py +++ b/inferencex-e2e/infx/tests/launch/test_srt_driver.py @@ -278,6 +278,7 @@ def test_multinode_lane_stages_workflow_artifacts(harness, monkeypatch, cluster_ assert bound["benchmark"]["env"] == { "RESULT_DIR": "/logs/agentic", "AIPERF_DATASET_MMAP_CACHE_DIR": "/aiperf_mmap_cache", "HF_HUB_CACHE": "/hf_hub_cache", "IS_MULTINODE": "true", "ENABLE_AGENTX_POWER": "0", + "REQUIRE_POWER": "0", } config = srtslurm(checkout) @@ -326,6 +327,8 @@ def test_power_lane_stages_provenance_and_validates_each_concurrency( bound = yaml.safe_load((checkout / "recipe.yaml").read_text()) assert bound["benchmark"]["concurrencies"] == [4] assert bound["benchmark"]["env"]["ENABLE_AGENTX_POWER"] == "1" + # The client fails invalid power when the workflow requires it; h200-dgxc rules never do. + assert bound["benchmark"]["env"]["REQUIRE_POWER"] == require_power assert bound["telemetry"]["collector_join_timeout_seconds"] == 12 assert (bound["telemetry"]["enabled"], bound["telemetry"]["dcgm_exporter"]["port"]) == (True, 9401) diff --git a/inferencex-e2e/infx/tests/srt_slurm/test_generate.py b/inferencex-e2e/infx/tests/srt_slurm/test_generate.py index d68f3a6530..8cfec51183 100644 --- a/inferencex-e2e/infx/tests/srt_slurm/test_generate.py +++ b/inferencex-e2e/infx/tests/srt_slurm/test_generate.py @@ -174,7 +174,7 @@ def test_a_multinode_agentx_power_point_gets_its_clusters_exporter_port_and_clie # the launcher's power decision. assert recipe["benchmark"]["env"] == { "RESULT_DIR": "/results", "AIPERF_DATASET_MMAP_CACHE_DIR": "/aiperf", "HF_HUB_CACHE": "/hf", - "IS_MULTINODE": "true", "ENABLE_AGENTX_POWER": "1", + "IS_MULTINODE": "true", "ENABLE_AGENTX_POWER": "1", "REQUIRE_POWER": "0", } # fmt: skip assert recipe["benchmark"]["concurrencies"] == [8] diff --git a/inferencex-e2e/infx/tests/srt_slurm/test_workload.py b/inferencex-e2e/infx/tests/srt_slurm/test_workload.py index e8d913ee3f..9f42d89787 100644 --- a/inferencex-e2e/infx/tests/srt_slurm/test_workload.py +++ b/inferencex-e2e/infx/tests/srt_slurm/test_workload.py @@ -93,6 +93,9 @@ def test_bundles_take_the_shared_block_under_base_and_keep_their_variants(projec }}}}, True, True, "override_tp8.benchmark.env.IS_MULTINODE (= 'false'), " "override_tp8.benchmark.env.ENABLE_AGENTX_POWER (= '0')"), + # The workflow exports the matrix point; a fragment pins none of it. + ({"benchmark": {"env": {"REQUIRE_POWER": "1", "MODEL": "/model"}}}, True, True, + "benchmark.env.MODEL (= '/model'), benchmark.env.REQUIRE_POWER (= '1')"), ]) # fmt: skip def test_a_fragment_that_sets_a_bound_key_is_rejected(project, data, agentic, multinode, reported): path = fragment(project, data)