From 8c8fd424934b6462f6afbc7e99bee86ae6d11b64 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Mon, 6 Jul 2026 13:05:43 -0500 Subject: [PATCH 1/5] feat: add heterogeneous disaggregated hardware metadata --- .../workflows/benchmark-multinode-tmpl.yml | 12 ++ .github/workflows/e2e-tests.yml | 6 + .github/workflows/run-sweep.yml | 6 + configs/CONFIGS.md | 9 ++ configs/amd-master.yaml | 45 ++++++ .../amd-minimaxm2.5-m2.7-master.yaml | 3 + .../nvidia-minimaxm2.5-m2.7-master.yaml | 24 +++ configs/nvidia-master.yaml | 147 ++++++++++++++++++ utils/matrix_logic/generate_sweep_configs.py | 10 ++ .../test_generate_sweep_configs.py | 4 + utils/matrix_logic/test_validation.py | 26 ++++ utils/matrix_logic/validation.py | 39 +++++ utils/process_result.py | 11 +- utils/test_process_result.py | 4 + 14 files changed, 344 insertions(+), 2 deletions(-) diff --git a/.github/workflows/benchmark-multinode-tmpl.yml b/.github/workflows/benchmark-multinode-tmpl.yml index c20b7cc356..4277e40c13 100644 --- a/.github/workflows/benchmark-multinode-tmpl.yml +++ b/.github/workflows/benchmark-multinode-tmpl.yml @@ -39,6 +39,16 @@ on: disagg: required: true type: string + prefill-hardware: + description: "GPU SKU used by prefill workers" + required: false + type: string + default: "" + decode-hardware: + description: "GPU SKU used by decode workers" + required: false + type: string + default: "" max-model-len: required: true @@ -144,6 +154,8 @@ env: CONC_LIST: ${{ join(fromJson(inputs.conc-list), ' ') }} SPEC_DECODING: ${{ inputs.spec-decoding }} DISAGG: ${{ inputs.disagg }} + PREFILL_HARDWARE: ${{ inputs.prefill-hardware }} + DECODE_HARDWARE: ${{ inputs.decode-hardware }} RUN_EVAL: ${{ inputs.run-eval }} EVAL_ONLY: ${{ inputs.eval-only }} EVAL_CONC: ${{ inputs.eval-conc }} diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index f05c3ef99b..536222ee77 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -106,6 +106,8 @@ jobs: conc-list: ${{ toJson(matrix.config.conc) }} spec-decoding: ${{ matrix.config.spec-decoding }} disagg: ${{ matrix.config.disagg }} + prefill-hardware: ${{ matrix.config.hardware.prefill }} + decode-hardware: ${{ matrix.config.hardware.decode }} prefill-num-worker: ${{ matrix.config.prefill.num-worker }} prefill-tp: ${{ matrix.config.prefill.tp }} @@ -145,6 +147,8 @@ jobs: conc-list: ${{ toJson(matrix.config.conc) }} spec-decoding: ${{ matrix.config.spec-decoding }} disagg: ${{ matrix.config.disagg }} + prefill-hardware: ${{ matrix.config.hardware.prefill }} + decode-hardware: ${{ matrix.config.hardware.decode }} prefill-num-worker: ${{ matrix.config.prefill.num-worker }} prefill-tp: ${{ matrix.config.prefill.tp }} @@ -221,6 +225,8 @@ jobs: conc-list: ${{ toJson(matrix.config.conc) }} spec-decoding: ${{ matrix.config.spec-decoding }} disagg: ${{ matrix.config.disagg }} + prefill-hardware: ${{ matrix.config.hardware.prefill }} + decode-hardware: ${{ matrix.config.hardware.decode }} prefill-num-worker: ${{ matrix.config.prefill.num-worker }} prefill-tp: ${{ matrix.config.prefill.tp }} prefill-ep: ${{ matrix.config.prefill.ep }} diff --git a/.github/workflows/run-sweep.yml b/.github/workflows/run-sweep.yml index 4ba50ef91d..8a15e4d852 100644 --- a/.github/workflows/run-sweep.yml +++ b/.github/workflows/run-sweep.yml @@ -369,6 +369,8 @@ jobs: conc-list: ${{ toJson(matrix.config.conc) }} spec-decoding: ${{ matrix.config.spec-decoding }} disagg: ${{ matrix.config.disagg }} + prefill-hardware: ${{ matrix.config.hardware.prefill }} + decode-hardware: ${{ matrix.config.hardware.decode }} prefill-num-worker: ${{ matrix.config.prefill.num-worker }} prefill-tp: ${{ matrix.config.prefill.tp }} @@ -531,6 +533,8 @@ jobs: conc-list: '[${{ matrix.config.conc }}]' spec-decoding: ${{ matrix.config.spec-decoding }} disagg: ${{ matrix.config.disagg }} + prefill-hardware: ${{ matrix.config.hardware.prefill }} + decode-hardware: ${{ matrix.config.hardware.decode }} prefill-num-worker: ${{ matrix.config.prefill.num-worker }} prefill-tp: ${{ matrix.config.prefill.tp }} prefill-ep: ${{ matrix.config.prefill.ep }} @@ -618,6 +622,8 @@ jobs: conc-list: ${{ toJson(matrix.config.conc) }} spec-decoding: ${{ matrix.config.spec-decoding }} disagg: ${{ matrix.config.disagg }} + prefill-hardware: ${{ matrix.config.hardware.prefill }} + decode-hardware: ${{ matrix.config.hardware.decode }} prefill-num-worker: ${{ matrix.config.prefill.num-worker }} prefill-tp: ${{ matrix.config.prefill.tp }} prefill-ep: ${{ matrix.config.prefill.ep }} diff --git a/configs/CONFIGS.md b/configs/CONFIGS.md index f46c4efda1..279eecfcc7 100644 --- a/configs/CONFIGS.md +++ b/configs/CONFIGS.md @@ -12,6 +12,11 @@ entry-name: runner: string precision: string framework: string + multinode: true + disagg: true + hardware: + prefill: string + decode: string scenarios: fixed-seq-len: - isl: int @@ -41,6 +46,10 @@ The below list describes what each field is: fleet. - `precision`: The precision to run the benchmark. Again, this is used to find which script to run in `benchmarks/`. - `framework`: The framework (serving runtime) to serve the benchmark, e.g., `vllm`, `sglang`, `trt`. +- `hardware`: Required when `multinode: true` and `disagg: true`. `prefill` and + `decode` identify the GPU SKU used by each worker pool (for example, `b200` + and `h100`). These values are benchmark metadata: they flow into generated + matrix entries and aggregate results, but do not affect runner scheduling. - `scenarios`: A dictionary of benchmark scenario types. At least one must be specified. Currently supported: - `fixed-seq-len`: Fixed input/output sequence length benchmarks. Each entry must have: - `isl`: An integer representing the input sequence length, e.g., `1024` diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index dd85cd81db..f26ae54504 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -358,6 +358,9 @@ qwen3.5-fp8-mi355x-sglang-disagg: framework: sglang-disagg multinode: true disagg: true + hardware: + prefill: mi355x + decode: mi355x scenarios: fixed-seq-len: - isl: 1024 @@ -485,6 +488,9 @@ qwen3.5-fp4-mi355x-sglang-disagg: framework: sglang-disagg multinode: true disagg: true + hardware: + prefill: mi355x + decode: mi355x scenarios: fixed-seq-len: - isl: 1024 @@ -598,6 +604,9 @@ glm5-fp8-mi355x-sglang-disagg: framework: sglang-disagg multinode: true disagg: true + hardware: + prefill: mi355x + decode: mi355x scenarios: fixed-seq-len: - isl: 1024 @@ -973,6 +982,9 @@ dsr1-fp8-mi355x-sglang-disagg: framework: sglang-disagg multinode: true disagg: true + hardware: + prefill: mi355x + decode: mi355x scenarios: fixed-seq-len: - isl: 1024 @@ -1127,6 +1139,9 @@ dsr1-fp8-mi355x-sglang-disagg-mtp: framework: sglang-disagg multinode: true disagg: true + hardware: + prefill: mi355x + decode: mi355x scenarios: fixed-seq-len: - isl: 1024 @@ -1281,6 +1296,9 @@ kimik2.5-fp4-mi355x-vllm-disagg: framework: vllm-disagg multinode: true disagg: true + hardware: + prefill: mi355x + decode: mi355x scenarios: fixed-seq-len: - isl: 1024 @@ -1335,6 +1353,9 @@ dsr1-fp4-mi355x-sglang-disagg: framework: sglang-disagg multinode: true disagg: true + hardware: + prefill: mi355x + decode: mi355x scenarios: fixed-seq-len: - isl: 1024 @@ -1563,6 +1584,9 @@ dsr1-fp4-mi355x-sglang-disagg-1k1k-mtp: framework: sglang-disagg multinode: true disagg: true + hardware: + prefill: mi355x + decode: mi355x scenarios: fixed-seq-len: - isl: 1024 @@ -1674,6 +1698,9 @@ dsr1-fp4-mi355x-sglang-disagg-8k1k-mtp: framework: sglang-disagg multinode: true disagg: true + hardware: + prefill: mi355x + decode: mi355x scenarios: fixed-seq-len: - isl: 8192 @@ -2161,6 +2188,9 @@ dsr1-fp4-mi355x-sglang-disagg-mtp: framework: sglang-disagg multinode: true disagg: true + hardware: + prefill: mi355x + decode: mi355x scenarios: fixed-seq-len: - isl: 1024 @@ -2413,6 +2443,9 @@ dsv4-fp4-mi355x-atom-disagg: framework: atom-disagg multinode: true disagg: true + hardware: + prefill: mi355x + decode: mi355x scenarios: fixed-seq-len: # 1P1D DPA+TP8 @@ -2532,6 +2565,9 @@ minimaxm3-fp4-mi355x-vllm-disagg: framework: vllm-disagg multinode: true disagg: true + hardware: + prefill: mi355x + decode: mi355x scenarios: fixed-seq-len: - isl: 8192 @@ -2720,6 +2756,9 @@ minimaxm3-fp8-mi355x-atom-disagg: framework: atom-disagg multinode: true disagg: true + hardware: + prefill: mi355x + decode: mi355x scenarios: fixed-seq-len: - isl: 8192 @@ -2771,6 +2810,9 @@ minimaxm3-fp4-mi355x-atom-disagg: framework: atom-disagg multinode: true disagg: true + hardware: + prefill: mi355x + decode: mi355x scenarios: fixed-seq-len: - isl: 8192 @@ -2946,6 +2988,9 @@ minimaxm3-fp8-mi355x-vllm-disagg: framework: vllm-disagg multinode: true disagg: true + hardware: + prefill: mi355x + decode: mi355x scenarios: fixed-seq-len: - isl: 1024 diff --git a/configs/deprecated/amd-minimaxm2.5-m2.7-master.yaml b/configs/deprecated/amd-minimaxm2.5-m2.7-master.yaml index a92bea3aa6..127d57ce98 100644 --- a/configs/deprecated/amd-minimaxm2.5-m2.7-master.yaml +++ b/configs/deprecated/amd-minimaxm2.5-m2.7-master.yaml @@ -219,6 +219,9 @@ minimaxm2.5-fp8-mi355x-vllm-disagg: framework: vllm-disagg multinode: true disagg: true + hardware: + prefill: mi355x + decode: mi355x scenarios: fixed-seq-len: - isl: 1024 diff --git a/configs/deprecated/nvidia-minimaxm2.5-m2.7-master.yaml b/configs/deprecated/nvidia-minimaxm2.5-m2.7-master.yaml index 50be2d4e8d..6118c54426 100644 --- a/configs/deprecated/nvidia-minimaxm2.5-m2.7-master.yaml +++ b/configs/deprecated/nvidia-minimaxm2.5-m2.7-master.yaml @@ -333,6 +333,9 @@ minimaxm2.5-fp4-gb300-dynamo-vllm: framework: dynamo-vllm multinode: true disagg: true + hardware: + prefill: gb300 + decode: gb300 scenarios: fixed-seq-len: - isl: 1024 @@ -507,6 +510,9 @@ minimaxm2.5-fp8-gb200-dynamo-vllm: framework: dynamo-vllm multinode: true disagg: true + hardware: + prefill: gb200 + decode: gb200 scenarios: fixed-seq-len: - isl: 1024 @@ -643,6 +649,9 @@ minimaxm2.5-fp8-b200-dynamo-vllm: framework: dynamo-vllm multinode: true disagg: true + hardware: + prefill: b200 + decode: b200 scenarios: fixed-seq-len: - isl: 1024 @@ -791,6 +800,9 @@ minimaxm2.5-fp4-gb200-dynamo-vllm: framework: dynamo-vllm multinode: true disagg: true + hardware: + prefill: gb200 + decode: gb200 scenarios: fixed-seq-len: - isl: 1024 @@ -1030,6 +1042,9 @@ minimaxm2.5-fp4-b200-dynamo-vllm: framework: dynamo-vllm multinode: true disagg: true + hardware: + prefill: b200 + decode: b200 scenarios: fixed-seq-len: - isl: 1024 @@ -1269,6 +1284,9 @@ minimaxm2.5-fp4-b300-dynamo-vllm: framework: dynamo-vllm multinode: true disagg: true + hardware: + prefill: b300 + decode: b300 scenarios: fixed-seq-len: - isl: 1024 @@ -1482,6 +1500,9 @@ minimaxm2.5-fp8-b300-dynamo-vllm: framework: dynamo-vllm multinode: true disagg: true + hardware: + prefill: b300 + decode: b300 scenarios: fixed-seq-len: - isl: 1024 @@ -1695,6 +1716,9 @@ minimaxm2.5-fp8-gb300-dynamo-vllm: framework: dynamo-vllm multinode: true disagg: true + hardware: + prefill: gb300 + decode: gb300 scenarios: fixed-seq-len: - isl: 1024 diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 74eb03c530..6ca25f7e43 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7,6 +7,9 @@ dsr1-fp4-b200-dynamo-trt: framework: dynamo-trt multinode: true disagg: true + hardware: + prefill: b200 + decode: b200 scenarios: fixed-seq-len: - isl: 1024 @@ -394,6 +397,9 @@ dsr1-fp8-b200-dynamo-trt: framework: dynamo-trt multinode: true disagg: true + hardware: + prefill: b200 + decode: b200 scenarios: fixed-seq-len: - isl: 1024 @@ -844,6 +850,9 @@ dsr1-fp4-b300-dynamo-trt: framework: dynamo-trt multinode: true disagg: true + hardware: + prefill: b300 + decode: b300 scenarios: fixed-seq-len: - isl: 1024 @@ -1257,6 +1266,9 @@ dsr1-fp8-b300-dynamo-trt: framework: dynamo-trt multinode: true disagg: true + hardware: + prefill: b300 + decode: b300 scenarios: fixed-seq-len: # 1k1k MTP configs @@ -2342,6 +2354,9 @@ glm5-fp4-gb300-dynamo-trt: framework: dynamo-trt multinode: true disagg: true + hardware: + prefill: gb300 + decode: gb300 scenarios: fixed-seq-len: - isl: 1024 @@ -3440,6 +3455,9 @@ dsr1-fp8-h200-dynamo-trt: framework: dynamo-trt multinode: true disagg: true + hardware: + prefill: h200 + decode: h200 scenarios: fixed-seq-len: - isl: 1024 @@ -3984,6 +4002,9 @@ dsr1-fp8-h100-dynamo-trt: framework: dynamo-trt multinode: true disagg: true + hardware: + prefill: h100 + decode: h100 scenarios: fixed-seq-len: - isl: 1024 @@ -4540,6 +4561,9 @@ dsr1-fp8-h100-dynamo-sglang: framework: dynamo-sglang multinode: true disagg: true + hardware: + prefill: h100 + decode: h100 scenarios: fixed-seq-len: - isl: 1024 @@ -4758,6 +4782,9 @@ dsr1-fp4-gb200-dynamo-trt: framework: dynamo-trt multinode: true disagg: true + hardware: + prefill: gb200 + decode: gb200 scenarios: fixed-seq-len: - isl: 1024 @@ -5115,6 +5142,9 @@ dsr1-fp8-gb200-dynamo-trt: framework: dynamo-trt multinode: true disagg: true + hardware: + prefill: gb200 + decode: gb200 scenarios: fixed-seq-len: # 1k1k MTP configs @@ -5543,6 +5573,9 @@ dsr1-fp8-gb200-dynamo-sglang: framework: dynamo-sglang multinode: true disagg: true + hardware: + prefill: gb200 + decode: gb200 scenarios: fixed-seq-len: - isl: 1024 @@ -5672,6 +5705,9 @@ dsr1-fp8-gb300-dynamo-sglang: framework: dynamo-sglang multinode: true disagg: true + hardware: + prefill: gb300 + decode: gb300 scenarios: fixed-seq-len: - isl: 1024 @@ -5785,6 +5821,9 @@ dsr1-fp4-gb200-dynamo-sglang: framework: dynamo-sglang multinode: true disagg: true + hardware: + prefill: gb200 + decode: gb200 scenarios: fixed-seq-len: # 1k1k configurations @@ -5900,6 +5939,9 @@ dsr1-fp4-gb300-dynamo-trt: framework: dynamo-trt multinode: true disagg: true + hardware: + prefill: gb300 + decode: gb300 scenarios: fixed-seq-len: - isl: 1024 @@ -6327,6 +6369,9 @@ dsr1-fp4-gb300-dynamo-sglang: framework: dynamo-sglang multinode: true disagg: true + hardware: + prefill: gb300 + decode: gb300 scenarios: fixed-seq-len: # 1k1k configurations @@ -6442,6 +6487,9 @@ dsr1-fp8-gb300-dynamo-trt: framework: dynamo-trt multinode: true disagg: true + hardware: + prefill: gb300 + decode: gb300 scenarios: fixed-seq-len: - isl: 1024 @@ -6854,6 +6902,9 @@ gptoss-fp4-gb200-dynamo-trt: framework: dynamo-trt multinode: true disagg: true + hardware: + prefill: gb200 + decode: gb200 scenarios: fixed-seq-len: - isl: 1024 @@ -7125,6 +7176,9 @@ dsr1-fp8-h200-dynamo-sglang: framework: dynamo-sglang multinode: true disagg: true + hardware: + prefill: h200 + decode: h200 scenarios: fixed-seq-len: - isl: 1024 @@ -7382,6 +7436,9 @@ dsr1-fp4-b200-dynamo-sglang: framework: dynamo-sglang multinode: true disagg: true + hardware: + prefill: b200 + decode: b200 scenarios: fixed-seq-len: - isl: 1024 @@ -7518,6 +7575,9 @@ dsr1-fp8-b200-dynamo-sglang: framework: dynamo-sglang multinode: true disagg: true + hardware: + prefill: b200 + decode: b200 scenarios: fixed-seq-len: - isl: 1024 @@ -7689,6 +7749,9 @@ dsr1-fp8-b200-dynamo-sglang-mtp: framework: dynamo-sglang multinode: true disagg: true + hardware: + prefill: b200 + decode: b200 scenarios: fixed-seq-len: - isl: 1024 @@ -7889,6 +7952,9 @@ dsr1-fp4-b200-dynamo-sglang-mtp: framework: dynamo-sglang multinode: true disagg: true + hardware: + prefill: b200 + decode: b200 scenarios: fixed-seq-len: - isl: 1024 @@ -8054,6 +8120,9 @@ kimik2.5-fp4-gb200-dynamo-trt: framework: dynamo-trt multinode: true disagg: true + hardware: + prefill: gb200 + decode: gb200 scenarios: fixed-seq-len: - isl: 1024 @@ -8425,6 +8494,9 @@ kimik2.5-fp4-gb300-dynamo-trt: framework: dynamo-trt multinode: true disagg: true + hardware: + prefill: gb300 + decode: gb300 scenarios: fixed-seq-len: - isl: 1024 @@ -8768,6 +8840,9 @@ kimik2.5-fp4-gb200-dynamo-vllm: framework: dynamo-vllm multinode: true disagg: true + hardware: + prefill: gb200 + decode: gb200 scenarios: fixed-seq-len: - isl: 1024 @@ -8890,6 +8965,9 @@ dsv4-fp4-b200-dynamo-vllm: framework: dynamo-vllm multinode: true disagg: true + hardware: + prefill: b200 + decode: b200 scenarios: fixed-seq-len: - isl: 8192 @@ -8972,6 +9050,9 @@ dsv4-fp4-gb200-dynamo-vllm: framework: dynamo-vllm multinode: true disagg: true + hardware: + prefill: gb200 + decode: gb200 scenarios: fixed-seq-len: - isl: 8192 @@ -9072,6 +9153,9 @@ dsv4-fp4-gb200-dynamo-vllm-mtp2: framework: dynamo-vllm multinode: true disagg: true + hardware: + prefill: gb200 + decode: gb200 scenarios: fixed-seq-len: - isl: 8192 @@ -9152,6 +9236,9 @@ dsv4-fp4-gb200-dynamo-sglang: framework: dynamo-sglang multinode: true disagg: true + hardware: + prefill: gb200 + decode: gb200 scenarios: fixed-seq-len: - isl: 8192 @@ -9279,6 +9366,9 @@ qwen3.5-fp8-gb200-dynamo-sglang: framework: dynamo-sglang multinode: true disagg: true + hardware: + prefill: gb200 + decode: gb200 scenarios: fixed-seq-len: - isl: 1024 @@ -9389,6 +9479,9 @@ dsv4-fp4-gb200-dynamo-sglang-mtp: framework: dynamo-sglang multinode: true disagg: true + hardware: + prefill: gb200 + decode: gb200 scenarios: fixed-seq-len: - isl: 8192 @@ -9524,6 +9617,9 @@ dsv4-fp4-b300-dynamo-vllm: framework: dynamo-vllm multinode: true disagg: true + hardware: + prefill: b300 + decode: b300 scenarios: fixed-seq-len: - isl: 8192 @@ -9580,6 +9676,9 @@ dsv4-fp4-gb300-dynamo-vllm: framework: dynamo-vllm multinode: true disagg: true + hardware: + prefill: gb300 + decode: gb300 scenarios: fixed-seq-len: - isl: 8192 @@ -9673,6 +9772,9 @@ dsv4-fp4-gb300-dynamo-trt: framework: dynamo-trt multinode: true disagg: true + hardware: + prefill: gb300 + decode: gb300 scenarios: fixed-seq-len: - isl: 1024 @@ -10069,6 +10171,9 @@ dsv4-fp4-gb300-dynamo-trt-mtp: framework: dynamo-trt multinode: true disagg: true + hardware: + prefill: gb300 + decode: gb300 scenarios: fixed-seq-len: - isl: 1024 @@ -10492,6 +10597,9 @@ dsv4-fp4-gb300-dynamo-sglang: framework: dynamo-sglang multinode: true disagg: true + hardware: + prefill: gb300 + decode: gb300 scenarios: fixed-seq-len: - isl: 8192 @@ -10606,6 +10714,9 @@ glm5-fp8-b200-dynamo-sglang: framework: dynamo-sglang multinode: true disagg: true + hardware: + prefill: b200 + decode: b200 scenarios: fixed-seq-len: - isl: 1024 @@ -10820,6 +10931,9 @@ dsv4-fp4-gb300-dynamo-sglang-mtp: framework: dynamo-sglang multinode: true disagg: true + hardware: + prefill: gb300 + decode: gb300 scenarios: fixed-seq-len: - isl: 8192 @@ -11015,6 +11129,9 @@ qwen3.5-fp4-gb300-dynamo-sglang: framework: dynamo-sglang multinode: true disagg: true + hardware: + prefill: gb300 + decode: gb300 scenarios: fixed-seq-len: - isl: 8192 @@ -11095,6 +11212,9 @@ glm5-fp4-gb300-dynamo-sglang: framework: dynamo-sglang multinode: true disagg: true + hardware: + prefill: gb300 + decode: gb300 scenarios: fixed-seq-len: # ---------- 8k1k high-throughput (wide-EP TP=32 decode) ---------- @@ -11305,6 +11425,9 @@ glm5-fp8-gb300-dynamo-sglang: framework: dynamo-sglang multinode: true disagg: true + hardware: + prefill: gb300 + decode: gb300 scenarios: fixed-seq-len: # ---------- 8k1k high-throughput (wide-EP decode) ---------- @@ -11560,6 +11683,9 @@ dsv4-fp4-gb300-dynamo-vllm-agentic: framework: dynamo-vllm multinode: true disagg: true + hardware: + prefill: gb300 + decode: gb300 scenarios: agentic-coding: - search-space: @@ -11645,6 +11771,9 @@ minimaxm3-fp8-b300-dynamo-vllm: framework: dynamo-vllm multinode: true disagg: true + hardware: + prefill: b300 + decode: b300 scenarios: fixed-seq-len: - isl: 1024 @@ -11861,6 +11990,9 @@ minimaxm3-fp4-b300-dynamo-vllm: framework: dynamo-vllm multinode: true disagg: true + hardware: + prefill: b300 + decode: b300 scenarios: fixed-seq-len: - isl: 1024 @@ -12077,6 +12209,9 @@ minimaxm3-fp8-gb300-dynamo-vllm: framework: dynamo-vllm multinode: true disagg: true + hardware: + prefill: gb300 + decode: gb300 scenarios: fixed-seq-len: - isl: 1024 @@ -12350,6 +12485,9 @@ minimaxm3-fp8-gb200-dynamo-vllm: framework: dynamo-vllm multinode: true disagg: true + hardware: + prefill: gb200 + decode: gb200 scenarios: fixed-seq-len: - isl: 1024 @@ -12916,6 +13054,9 @@ kimik2.5-fp4-gb300-dynamo-vllm: framework: dynamo-vllm multinode: true disagg: true + hardware: + prefill: gb300 + decode: gb300 scenarios: fixed-seq-len: - isl: 1024 @@ -13168,6 +13309,9 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-3p2d-tep8-tp8: framework: dynamo-vllm multinode: true disagg: true + hardware: + prefill: gb200 + decode: gb200 scenarios: agentic-coding: - search-space: @@ -13224,6 +13368,9 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-2p1d-dep8-dep8: framework: dynamo-vllm multinode: true disagg: true + hardware: + prefill: gb200 + decode: gb200 scenarios: agentic-coding: - search-space: diff --git a/utils/matrix_logic/generate_sweep_configs.py b/utils/matrix_logic/generate_sweep_configs.py index 545eb9f969..2702cb7e68 100644 --- a/utils/matrix_logic/generate_sweep_configs.py +++ b/utils/matrix_logic/generate_sweep_configs.py @@ -349,6 +349,7 @@ def generate_full_sweep(args, all_config_data, runner_data): is_multinode = val.get(Fields.MULTINODE.value, False) # Get disagg value, defaulting to False if not specified disagg = val.get(Fields.DISAGG.value, False) + hardware = val.get(Fields.HARDWARE.value) scenarios = val[Fields.SCENARIOS.value] scenario_filter = set(args.scenario_type) if getattr(args, 'scenario_type', None) else None @@ -457,6 +458,8 @@ def generate_full_sweep(args, all_config_data, runner_data): Fields.DISAGG.value: disagg, Fields.RUN_EVAL.value: False, # Default, may be overridden by mark_eval_entries } + if hardware is not None: + entry[Fields.HARDWARE.value] = hardware validate_matrix_entry(entry, is_multinode) matrix_values.append(entry) @@ -654,6 +657,8 @@ def generate_full_sweep(args, all_config_data, runner_data): Fields.DISAGG.value: disagg, Fields.SCENARIO_TYPE.value: "agentic-coding", } + if hardware is not None: + entry[Fields.HARDWARE.value] = hardware validate_agentic_matrix_entry(entry) matrix_values.append(entry) else: @@ -731,6 +736,7 @@ def generate_test_config_sweep(args, all_config_data, runner_data=None): if not runners_for_entry: continue disagg = val.get(Fields.DISAGG.value, False) + hardware = val.get(Fields.HARDWARE.value) # Build seq-len filter if --seq-lens was provided seq_lens_filter = None @@ -797,6 +803,8 @@ def generate_test_config_sweep(args, all_config_data, runner_data=None): Fields.DISAGG.value: disagg, Fields.RUN_EVAL.value: False, } + if hardware is not None: + entry[Fields.HARDWARE.value] = hardware matrix_values.append(validate_matrix_entry(entry, is_multinode=True)) else: # Single-node config @@ -921,6 +929,8 @@ def generate_test_config_sweep(args, all_config_data, runner_data=None): Fields.DISAGG.value: disagg, Fields.SCENARIO_TYPE.value: "agentic-coding", } + if hardware is not None: + entry[Fields.HARDWARE.value] = hardware matrix_values.append(validate_agentic_matrix_entry(entry)) else: for conc in conc_values: diff --git a/utils/matrix_logic/test_generate_sweep_configs.py b/utils/matrix_logic/test_generate_sweep_configs.py index 0e2c64d95e..17994b7991 100644 --- a/utils/matrix_logic/test_generate_sweep_configs.py +++ b/utils/matrix_logic/test_generate_sweep_configs.py @@ -68,6 +68,7 @@ def sample_multinode_config(): "runner": "gb200", "multinode": True, "disagg": True, + "hardware": {"prefill": "gb200", "decode": "h100"}, "scenarios": { "fixed-seq-len": [ @@ -981,6 +982,7 @@ def test_multinode_entry_structure(self, sample_multinode_config, sample_runner_ assert entry["prefill"]["num-worker"] == 5 assert entry["decode"]["num-worker"] == 1 assert entry["disagg"] is True + assert entry["hardware"] == {"prefill": "gb200", "decode": "h100"} def test_multinode_conc_as_list(self, sample_multinode_config, sample_runner_config, full_sweep_args_multi_node): """Multinode conc should be passed as list.""" @@ -1998,6 +2000,7 @@ def test_multinode_agentic_groups_concurrencies_per_search_entry(self): "runner": "gb200", "multinode": True, "disagg": True, + "hardware": {"prefill": "gb200", "decode": "h100"}, "scenarios": { "agentic-coding": [ { @@ -2146,6 +2149,7 @@ def test_node_type_filters_apply_to_agentic_configs( "runner": "cluster:gb200-nv", "multinode": True, "disagg": True, + "hardware": {"prefill": "gb200", "decode": "h100"}, "scenarios": { "agentic-coding": [{ "search-space": [ diff --git a/utils/matrix_logic/test_validation.py b/utils/matrix_logic/test_validation.py index 7e794331f5..cea10c4f1b 100644 --- a/utils/matrix_logic/test_validation.py +++ b/utils/matrix_logic/test_validation.py @@ -90,6 +90,7 @@ def valid_multinode_matrix_entry(): "max-model-len": 2248, "exp-name": "dsr1_1k1k", "disagg": True, + "hardware": {"prefill": "gb200", "decode": "h100"}, "run-eval": False, } @@ -132,6 +133,7 @@ def valid_multinode_master_config(): "runner": "gb200", "multinode": True, "disagg": True, + "hardware": {"prefill": "gb200", "decode": "h100"}, "scenarios": { "fixed-seq-len": [ @@ -213,6 +215,7 @@ def test_key_fields_exist(self): assert Fields.SPEC_DECODING.value == "spec-decoding" assert Fields.PREFILL.value == "prefill" assert Fields.DECODE.value == "decode" + assert Fields.HARDWARE.value == "hardware" # ============================================================================= @@ -466,6 +469,20 @@ def test_valid_entry(self, valid_multinode_matrix_entry): assert entry.model == "deepseek-r1-fp4" assert entry.conc == [2150] assert entry.disagg is True + assert entry.hardware.prefill == "gb200" + assert entry.hardware.decode == "h100" + + def test_disagg_requires_hardware(self, valid_multinode_matrix_entry): + """Disaggregated matrix entries must identify both hardware pools.""" + del valid_multinode_matrix_entry["hardware"] + with pytest.raises(Exception, match="hardware.*required"): + MultiNodeMatrixEntry(**valid_multinode_matrix_entry) + + def test_non_disagg_rejects_hardware(self, valid_multinode_matrix_entry): + """Hardware pool metadata is scoped to disaggregated entries.""" + valid_multinode_matrix_entry["disagg"] = False + with pytest.raises(Exception, match="hardware.*only be set"): + MultiNodeMatrixEntry(**valid_multinode_matrix_entry) def test_prefill_decode_worker_configs(self, valid_multinode_matrix_entry): """Prefill and decode should be WorkerConfig objects.""" @@ -817,6 +834,14 @@ def test_multinode_master_config(self, valid_multinode_master_config): assert config.model_prefix == "dsr1" assert config.runner == "gb200" assert config.disagg is True + assert config.hardware.prefill == "gb200" + assert config.hardware.decode == "h100" + + def test_disagg_master_config_requires_hardware(self, valid_multinode_master_config): + """Disaggregated master configs must identify prefill and decode hardware.""" + del valid_multinode_master_config["hardware"] + with pytest.raises(Exception, match="hardware.*required"): + MultiNodeMasterConfigEntry(**valid_multinode_master_config) def test_single_node_cannot_have_multinode_true(self, valid_single_node_master_config): """Single node config must have multinode=False.""" @@ -873,6 +898,7 @@ def test_multinode_agentic_master_config_requires_cluster_runner(self): "runner": "b200-multinode", "multinode": True, "disagg": True, + "hardware": {"prefill": "b200", "decode": "b200"}, "scenarios": { "agentic-coding": [ { diff --git a/utils/matrix_logic/validation.py b/utils/matrix_logic/validation.py index 1f4986272a..0f9a5628f2 100644 --- a/utils/matrix_logic/validation.py +++ b/utils/matrix_logic/validation.py @@ -23,6 +23,7 @@ class Fields(Enum): PRECISION = 'precision' FRAMEWORK = 'framework' RUNNER = 'runner' + HARDWARE = 'hardware' SCENARIOS = 'scenarios' MULTINODE = 'multinode' @@ -124,6 +125,29 @@ class WorkerConfig(BaseModel): default=[], alias=Fields.ADDITIONAL_SETTINGS.value) +class DisaggregatedHardwareConfig(BaseModel): + """Hardware SKUs assigned to each side of a disaggregated deployment.""" + model_config = ConfigDict(extra='forbid', populate_by_name=True) + + prefill: str = Field(min_length=1) + decode: str = Field(min_length=1) + + +def _validate_disaggregated_hardware(self): + """Require explicit prefill/decode hardware exactly when disaggregation is enabled.""" + if self.disagg and self.hardware is None: + raise ValueError( + f"'{Fields.HARDWARE.value}' is required when " + f"'{Fields.DISAGG.value}' is true" + ) + if not self.disagg and self.hardware is not None: + raise ValueError( + f"'{Fields.HARDWARE.value}' can only be set when " + f"'{Fields.DISAGG.value}' is true" + ) + return self + + class MultiNodeMatrixEntry(BaseModel): """Pydantic model for validating multinode matrix entry structure. This validates the input that should be expected to .github/workflows/benchmark-multinode-tmpl.yml""" @@ -146,6 +170,7 @@ class MultiNodeMatrixEntry(BaseModel): max_model_len: int = Field(alias=Fields.MAX_MODEL_LEN.value) exp_name: str = Field(alias=Fields.EXP_NAME.value) disagg: bool + hardware: Optional[DisaggregatedHardwareConfig] = None run_eval: bool = Field(alias=Fields.RUN_EVAL.value) eval_only: bool = Field(alias=Fields.EVAL_ONLY.value, default=False) eval_conc: Optional[int] = Field(default=None, alias=Fields.EVAL_CONC.value) @@ -153,6 +178,10 @@ class MultiNodeMatrixEntry(BaseModel): default=False, alias=Fields.EVAL_ALL_CONCS.value ) + @model_validator(mode='after') + def validate_disaggregated_hardware(self): + return _validate_disaggregated_hardware(self) + class SingleNodeAgenticMatrixEntry(BaseModel): """Pydantic model for validating single-node agentic coding matrix entries.""" @@ -204,8 +233,13 @@ class MultiNodeAgenticMatrixEntry(BaseModel): duration: int = Field(alias=Fields.DURATION.value) exp_name: str = Field(alias=Fields.EXP_NAME.value) disagg: bool + hardware: Optional[DisaggregatedHardwareConfig] = None scenario_type: str = Field(alias=Fields.SCENARIO_TYPE.value) + @model_validator(mode='after') + def validate_disaggregated_hardware(self): + return _validate_disaggregated_hardware(self) + AgenticMatrixEntry = Union[SingleNodeAgenticMatrixEntry, MultiNodeAgenticMatrixEntry] @@ -521,8 +555,13 @@ class MultiNodeMasterConfigEntry(BaseModel): runner: str multinode: Literal[True] disagg: bool = Field(default=False) + hardware: Optional[DisaggregatedHardwareConfig] = None scenarios: MultiNodeScenarios + @model_validator(mode='after') + def validate_disaggregated_hardware(self): + return _validate_disaggregated_hardware(self) + @model_validator(mode='after') def validate_agentic_runner(self): _validate_agentic_runner_is_cluster(self.runner, self.scenarios) diff --git a/utils/process_result.py b/utils/process_result.py index 5fb0594739..33ae4c1ed0 100644 --- a/utils/process_result.py +++ b/utils/process_result.py @@ -62,8 +62,12 @@ def get_required_env_vars(required_vars): # TODO: Eventually will have to have a separate condition in here for multinode disagg and # multinode agg. For now, just assume that multinode implies disagg. - multinode_env = get_required_env_vars(['PREFILL_GPUS', 'DECODE_GPUS', 'PREFILL_NUM_WORKERS', 'PREFILL_TP', - 'PREFILL_EP', 'PREFILL_DP_ATTN', 'DECODE_NUM_WORKERS', 'DECODE_TP', 'DECODE_EP', 'DECODE_DP_ATTN']) + multinode_vars = ['PREFILL_GPUS', 'DECODE_GPUS', 'PREFILL_NUM_WORKERS', 'PREFILL_TP', + 'PREFILL_EP', 'PREFILL_DP_ATTN', 'DECODE_NUM_WORKERS', 'DECODE_TP', + 'DECODE_EP', 'DECODE_DP_ATTN'] + if disagg: + multinode_vars.extend(['PREFILL_HARDWARE', 'DECODE_HARDWARE']) + multinode_env = get_required_env_vars(multinode_vars) prefill_gpus = int(multinode_env['PREFILL_GPUS']) decode_gpus = int(multinode_env['DECODE_GPUS']) prefill_num_workers = int(multinode_env['PREFILL_NUM_WORKERS']) @@ -101,6 +105,9 @@ def get_required_env_vars(required_vars): 'output_tput_per_gpu': float(bmk_result['output_throughput']) / output_tput_denominator, 'input_tput_per_gpu': (float(bmk_result['total_token_throughput']) - float(bmk_result['output_throughput'])) / prefill_gpus, } + if disagg: + multi_node_data['prefill_hw'] = multinode_env['PREFILL_HARDWARE'] + multi_node_data['decode_hw'] = multinode_env['DECODE_HARDWARE'] data = data | multi_node_data else: diff --git a/utils/test_process_result.py b/utils/test_process_result.py index 4037689eaf..016d41d371 100644 --- a/utils/test_process_result.py +++ b/utils/test_process_result.py @@ -82,6 +82,8 @@ def multinode_env_vars(base_env_vars): "DECODE_TP": "8", "DECODE_EP": "8", "DECODE_DP_ATTN": "true", + "PREFILL_HARDWARE": "gb200", + "DECODE_HARDWARE": "h100", } @@ -232,6 +234,8 @@ def test_multinode_processing(self, tmp_path, sample_benchmark_result, multinode assert output_data["decode_num_workers"] == 1 assert output_data["num_prefill_gpu"] == 20 assert output_data["num_decode_gpu"] == 8 + assert output_data["prefill_hw"] == "gb200" + assert output_data["decode_hw"] == "h100" # Verify throughput calculations total_gpus = 20 + 8 # prefill + decode From 78657e2103b50c25bbac7a21317a130bc2e6c38b Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Mon, 6 Jul 2026 14:31:31 -0500 Subject: [PATCH 2/5] fix: make heterogeneous hardware metadata optional --- configs/CONFIGS.md | 11 +- configs/amd-master.yaml | 45 ------ .../amd-minimaxm2.5-m2.7-master.yaml | 3 - .../nvidia-minimaxm2.5-m2.7-master.yaml | 24 --- configs/nvidia-master.yaml | 147 ------------------ utils/matrix_logic/test_validation.py | 28 ++-- utils/matrix_logic/validation.py | 27 ---- utils/process_result.py | 14 +- utils/test_process_result.py | 25 +++ 9 files changed, 57 insertions(+), 267 deletions(-) diff --git a/configs/CONFIGS.md b/configs/CONFIGS.md index 279eecfcc7..51ebd429b6 100644 --- a/configs/CONFIGS.md +++ b/configs/CONFIGS.md @@ -14,7 +14,7 @@ entry-name: framework: string multinode: true disagg: true - hardware: + hardware: # optional; omit for homogeneous hardware prefill: string decode: string scenarios: @@ -46,10 +46,11 @@ The below list describes what each field is: fleet. - `precision`: The precision to run the benchmark. Again, this is used to find which script to run in `benchmarks/`. - `framework`: The framework (serving runtime) to serve the benchmark, e.g., `vllm`, `sglang`, `trt`. -- `hardware`: Required when `multinode: true` and `disagg: true`. `prefill` and - `decode` identify the GPU SKU used by each worker pool (for example, `b200` - and `h100`). These values are benchmark metadata: they flow into generated - matrix entries and aggregate results, but do not affect runner scheduling. +- `hardware`: Optional metadata for heterogeneous disaggregated deployments. + When present, both `prefill` and `decode` are required and identify the GPU + SKU used by each worker pool (for example, `b200` and `h100`). Omit this block + for homogeneous hardware. These values flow into generated matrix entries + and aggregate results, but do not affect runner scheduling. - `scenarios`: A dictionary of benchmark scenario types. At least one must be specified. Currently supported: - `fixed-seq-len`: Fixed input/output sequence length benchmarks. Each entry must have: - `isl`: An integer representing the input sequence length, e.g., `1024` diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index f26ae54504..dd85cd81db 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -358,9 +358,6 @@ qwen3.5-fp8-mi355x-sglang-disagg: framework: sglang-disagg multinode: true disagg: true - hardware: - prefill: mi355x - decode: mi355x scenarios: fixed-seq-len: - isl: 1024 @@ -488,9 +485,6 @@ qwen3.5-fp4-mi355x-sglang-disagg: framework: sglang-disagg multinode: true disagg: true - hardware: - prefill: mi355x - decode: mi355x scenarios: fixed-seq-len: - isl: 1024 @@ -604,9 +598,6 @@ glm5-fp8-mi355x-sglang-disagg: framework: sglang-disagg multinode: true disagg: true - hardware: - prefill: mi355x - decode: mi355x scenarios: fixed-seq-len: - isl: 1024 @@ -982,9 +973,6 @@ dsr1-fp8-mi355x-sglang-disagg: framework: sglang-disagg multinode: true disagg: true - hardware: - prefill: mi355x - decode: mi355x scenarios: fixed-seq-len: - isl: 1024 @@ -1139,9 +1127,6 @@ dsr1-fp8-mi355x-sglang-disagg-mtp: framework: sglang-disagg multinode: true disagg: true - hardware: - prefill: mi355x - decode: mi355x scenarios: fixed-seq-len: - isl: 1024 @@ -1296,9 +1281,6 @@ kimik2.5-fp4-mi355x-vllm-disagg: framework: vllm-disagg multinode: true disagg: true - hardware: - prefill: mi355x - decode: mi355x scenarios: fixed-seq-len: - isl: 1024 @@ -1353,9 +1335,6 @@ dsr1-fp4-mi355x-sglang-disagg: framework: sglang-disagg multinode: true disagg: true - hardware: - prefill: mi355x - decode: mi355x scenarios: fixed-seq-len: - isl: 1024 @@ -1584,9 +1563,6 @@ dsr1-fp4-mi355x-sglang-disagg-1k1k-mtp: framework: sglang-disagg multinode: true disagg: true - hardware: - prefill: mi355x - decode: mi355x scenarios: fixed-seq-len: - isl: 1024 @@ -1698,9 +1674,6 @@ dsr1-fp4-mi355x-sglang-disagg-8k1k-mtp: framework: sglang-disagg multinode: true disagg: true - hardware: - prefill: mi355x - decode: mi355x scenarios: fixed-seq-len: - isl: 8192 @@ -2188,9 +2161,6 @@ dsr1-fp4-mi355x-sglang-disagg-mtp: framework: sglang-disagg multinode: true disagg: true - hardware: - prefill: mi355x - decode: mi355x scenarios: fixed-seq-len: - isl: 1024 @@ -2443,9 +2413,6 @@ dsv4-fp4-mi355x-atom-disagg: framework: atom-disagg multinode: true disagg: true - hardware: - prefill: mi355x - decode: mi355x scenarios: fixed-seq-len: # 1P1D DPA+TP8 @@ -2565,9 +2532,6 @@ minimaxm3-fp4-mi355x-vllm-disagg: framework: vllm-disagg multinode: true disagg: true - hardware: - prefill: mi355x - decode: mi355x scenarios: fixed-seq-len: - isl: 8192 @@ -2756,9 +2720,6 @@ minimaxm3-fp8-mi355x-atom-disagg: framework: atom-disagg multinode: true disagg: true - hardware: - prefill: mi355x - decode: mi355x scenarios: fixed-seq-len: - isl: 8192 @@ -2810,9 +2771,6 @@ minimaxm3-fp4-mi355x-atom-disagg: framework: atom-disagg multinode: true disagg: true - hardware: - prefill: mi355x - decode: mi355x scenarios: fixed-seq-len: - isl: 8192 @@ -2988,9 +2946,6 @@ minimaxm3-fp8-mi355x-vllm-disagg: framework: vllm-disagg multinode: true disagg: true - hardware: - prefill: mi355x - decode: mi355x scenarios: fixed-seq-len: - isl: 1024 diff --git a/configs/deprecated/amd-minimaxm2.5-m2.7-master.yaml b/configs/deprecated/amd-minimaxm2.5-m2.7-master.yaml index 127d57ce98..a92bea3aa6 100644 --- a/configs/deprecated/amd-minimaxm2.5-m2.7-master.yaml +++ b/configs/deprecated/amd-minimaxm2.5-m2.7-master.yaml @@ -219,9 +219,6 @@ minimaxm2.5-fp8-mi355x-vllm-disagg: framework: vllm-disagg multinode: true disagg: true - hardware: - prefill: mi355x - decode: mi355x scenarios: fixed-seq-len: - isl: 1024 diff --git a/configs/deprecated/nvidia-minimaxm2.5-m2.7-master.yaml b/configs/deprecated/nvidia-minimaxm2.5-m2.7-master.yaml index 6118c54426..50be2d4e8d 100644 --- a/configs/deprecated/nvidia-minimaxm2.5-m2.7-master.yaml +++ b/configs/deprecated/nvidia-minimaxm2.5-m2.7-master.yaml @@ -333,9 +333,6 @@ minimaxm2.5-fp4-gb300-dynamo-vllm: framework: dynamo-vllm multinode: true disagg: true - hardware: - prefill: gb300 - decode: gb300 scenarios: fixed-seq-len: - isl: 1024 @@ -510,9 +507,6 @@ minimaxm2.5-fp8-gb200-dynamo-vllm: framework: dynamo-vllm multinode: true disagg: true - hardware: - prefill: gb200 - decode: gb200 scenarios: fixed-seq-len: - isl: 1024 @@ -649,9 +643,6 @@ minimaxm2.5-fp8-b200-dynamo-vllm: framework: dynamo-vllm multinode: true disagg: true - hardware: - prefill: b200 - decode: b200 scenarios: fixed-seq-len: - isl: 1024 @@ -800,9 +791,6 @@ minimaxm2.5-fp4-gb200-dynamo-vllm: framework: dynamo-vllm multinode: true disagg: true - hardware: - prefill: gb200 - decode: gb200 scenarios: fixed-seq-len: - isl: 1024 @@ -1042,9 +1030,6 @@ minimaxm2.5-fp4-b200-dynamo-vllm: framework: dynamo-vllm multinode: true disagg: true - hardware: - prefill: b200 - decode: b200 scenarios: fixed-seq-len: - isl: 1024 @@ -1284,9 +1269,6 @@ minimaxm2.5-fp4-b300-dynamo-vllm: framework: dynamo-vllm multinode: true disagg: true - hardware: - prefill: b300 - decode: b300 scenarios: fixed-seq-len: - isl: 1024 @@ -1500,9 +1482,6 @@ minimaxm2.5-fp8-b300-dynamo-vllm: framework: dynamo-vllm multinode: true disagg: true - hardware: - prefill: b300 - decode: b300 scenarios: fixed-seq-len: - isl: 1024 @@ -1716,9 +1695,6 @@ minimaxm2.5-fp8-gb300-dynamo-vllm: framework: dynamo-vllm multinode: true disagg: true - hardware: - prefill: gb300 - decode: gb300 scenarios: fixed-seq-len: - isl: 1024 diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 6ca25f7e43..74eb03c530 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7,9 +7,6 @@ dsr1-fp4-b200-dynamo-trt: framework: dynamo-trt multinode: true disagg: true - hardware: - prefill: b200 - decode: b200 scenarios: fixed-seq-len: - isl: 1024 @@ -397,9 +394,6 @@ dsr1-fp8-b200-dynamo-trt: framework: dynamo-trt multinode: true disagg: true - hardware: - prefill: b200 - decode: b200 scenarios: fixed-seq-len: - isl: 1024 @@ -850,9 +844,6 @@ dsr1-fp4-b300-dynamo-trt: framework: dynamo-trt multinode: true disagg: true - hardware: - prefill: b300 - decode: b300 scenarios: fixed-seq-len: - isl: 1024 @@ -1266,9 +1257,6 @@ dsr1-fp8-b300-dynamo-trt: framework: dynamo-trt multinode: true disagg: true - hardware: - prefill: b300 - decode: b300 scenarios: fixed-seq-len: # 1k1k MTP configs @@ -2354,9 +2342,6 @@ glm5-fp4-gb300-dynamo-trt: framework: dynamo-trt multinode: true disagg: true - hardware: - prefill: gb300 - decode: gb300 scenarios: fixed-seq-len: - isl: 1024 @@ -3455,9 +3440,6 @@ dsr1-fp8-h200-dynamo-trt: framework: dynamo-trt multinode: true disagg: true - hardware: - prefill: h200 - decode: h200 scenarios: fixed-seq-len: - isl: 1024 @@ -4002,9 +3984,6 @@ dsr1-fp8-h100-dynamo-trt: framework: dynamo-trt multinode: true disagg: true - hardware: - prefill: h100 - decode: h100 scenarios: fixed-seq-len: - isl: 1024 @@ -4561,9 +4540,6 @@ dsr1-fp8-h100-dynamo-sglang: framework: dynamo-sglang multinode: true disagg: true - hardware: - prefill: h100 - decode: h100 scenarios: fixed-seq-len: - isl: 1024 @@ -4782,9 +4758,6 @@ dsr1-fp4-gb200-dynamo-trt: framework: dynamo-trt multinode: true disagg: true - hardware: - prefill: gb200 - decode: gb200 scenarios: fixed-seq-len: - isl: 1024 @@ -5142,9 +5115,6 @@ dsr1-fp8-gb200-dynamo-trt: framework: dynamo-trt multinode: true disagg: true - hardware: - prefill: gb200 - decode: gb200 scenarios: fixed-seq-len: # 1k1k MTP configs @@ -5573,9 +5543,6 @@ dsr1-fp8-gb200-dynamo-sglang: framework: dynamo-sglang multinode: true disagg: true - hardware: - prefill: gb200 - decode: gb200 scenarios: fixed-seq-len: - isl: 1024 @@ -5705,9 +5672,6 @@ dsr1-fp8-gb300-dynamo-sglang: framework: dynamo-sglang multinode: true disagg: true - hardware: - prefill: gb300 - decode: gb300 scenarios: fixed-seq-len: - isl: 1024 @@ -5821,9 +5785,6 @@ dsr1-fp4-gb200-dynamo-sglang: framework: dynamo-sglang multinode: true disagg: true - hardware: - prefill: gb200 - decode: gb200 scenarios: fixed-seq-len: # 1k1k configurations @@ -5939,9 +5900,6 @@ dsr1-fp4-gb300-dynamo-trt: framework: dynamo-trt multinode: true disagg: true - hardware: - prefill: gb300 - decode: gb300 scenarios: fixed-seq-len: - isl: 1024 @@ -6369,9 +6327,6 @@ dsr1-fp4-gb300-dynamo-sglang: framework: dynamo-sglang multinode: true disagg: true - hardware: - prefill: gb300 - decode: gb300 scenarios: fixed-seq-len: # 1k1k configurations @@ -6487,9 +6442,6 @@ dsr1-fp8-gb300-dynamo-trt: framework: dynamo-trt multinode: true disagg: true - hardware: - prefill: gb300 - decode: gb300 scenarios: fixed-seq-len: - isl: 1024 @@ -6902,9 +6854,6 @@ gptoss-fp4-gb200-dynamo-trt: framework: dynamo-trt multinode: true disagg: true - hardware: - prefill: gb200 - decode: gb200 scenarios: fixed-seq-len: - isl: 1024 @@ -7176,9 +7125,6 @@ dsr1-fp8-h200-dynamo-sglang: framework: dynamo-sglang multinode: true disagg: true - hardware: - prefill: h200 - decode: h200 scenarios: fixed-seq-len: - isl: 1024 @@ -7436,9 +7382,6 @@ dsr1-fp4-b200-dynamo-sglang: framework: dynamo-sglang multinode: true disagg: true - hardware: - prefill: b200 - decode: b200 scenarios: fixed-seq-len: - isl: 1024 @@ -7575,9 +7518,6 @@ dsr1-fp8-b200-dynamo-sglang: framework: dynamo-sglang multinode: true disagg: true - hardware: - prefill: b200 - decode: b200 scenarios: fixed-seq-len: - isl: 1024 @@ -7749,9 +7689,6 @@ dsr1-fp8-b200-dynamo-sglang-mtp: framework: dynamo-sglang multinode: true disagg: true - hardware: - prefill: b200 - decode: b200 scenarios: fixed-seq-len: - isl: 1024 @@ -7952,9 +7889,6 @@ dsr1-fp4-b200-dynamo-sglang-mtp: framework: dynamo-sglang multinode: true disagg: true - hardware: - prefill: b200 - decode: b200 scenarios: fixed-seq-len: - isl: 1024 @@ -8120,9 +8054,6 @@ kimik2.5-fp4-gb200-dynamo-trt: framework: dynamo-trt multinode: true disagg: true - hardware: - prefill: gb200 - decode: gb200 scenarios: fixed-seq-len: - isl: 1024 @@ -8494,9 +8425,6 @@ kimik2.5-fp4-gb300-dynamo-trt: framework: dynamo-trt multinode: true disagg: true - hardware: - prefill: gb300 - decode: gb300 scenarios: fixed-seq-len: - isl: 1024 @@ -8840,9 +8768,6 @@ kimik2.5-fp4-gb200-dynamo-vllm: framework: dynamo-vllm multinode: true disagg: true - hardware: - prefill: gb200 - decode: gb200 scenarios: fixed-seq-len: - isl: 1024 @@ -8965,9 +8890,6 @@ dsv4-fp4-b200-dynamo-vllm: framework: dynamo-vllm multinode: true disagg: true - hardware: - prefill: b200 - decode: b200 scenarios: fixed-seq-len: - isl: 8192 @@ -9050,9 +8972,6 @@ dsv4-fp4-gb200-dynamo-vllm: framework: dynamo-vllm multinode: true disagg: true - hardware: - prefill: gb200 - decode: gb200 scenarios: fixed-seq-len: - isl: 8192 @@ -9153,9 +9072,6 @@ dsv4-fp4-gb200-dynamo-vllm-mtp2: framework: dynamo-vllm multinode: true disagg: true - hardware: - prefill: gb200 - decode: gb200 scenarios: fixed-seq-len: - isl: 8192 @@ -9236,9 +9152,6 @@ dsv4-fp4-gb200-dynamo-sglang: framework: dynamo-sglang multinode: true disagg: true - hardware: - prefill: gb200 - decode: gb200 scenarios: fixed-seq-len: - isl: 8192 @@ -9366,9 +9279,6 @@ qwen3.5-fp8-gb200-dynamo-sglang: framework: dynamo-sglang multinode: true disagg: true - hardware: - prefill: gb200 - decode: gb200 scenarios: fixed-seq-len: - isl: 1024 @@ -9479,9 +9389,6 @@ dsv4-fp4-gb200-dynamo-sglang-mtp: framework: dynamo-sglang multinode: true disagg: true - hardware: - prefill: gb200 - decode: gb200 scenarios: fixed-seq-len: - isl: 8192 @@ -9617,9 +9524,6 @@ dsv4-fp4-b300-dynamo-vllm: framework: dynamo-vllm multinode: true disagg: true - hardware: - prefill: b300 - decode: b300 scenarios: fixed-seq-len: - isl: 8192 @@ -9676,9 +9580,6 @@ dsv4-fp4-gb300-dynamo-vllm: framework: dynamo-vllm multinode: true disagg: true - hardware: - prefill: gb300 - decode: gb300 scenarios: fixed-seq-len: - isl: 8192 @@ -9772,9 +9673,6 @@ dsv4-fp4-gb300-dynamo-trt: framework: dynamo-trt multinode: true disagg: true - hardware: - prefill: gb300 - decode: gb300 scenarios: fixed-seq-len: - isl: 1024 @@ -10171,9 +10069,6 @@ dsv4-fp4-gb300-dynamo-trt-mtp: framework: dynamo-trt multinode: true disagg: true - hardware: - prefill: gb300 - decode: gb300 scenarios: fixed-seq-len: - isl: 1024 @@ -10597,9 +10492,6 @@ dsv4-fp4-gb300-dynamo-sglang: framework: dynamo-sglang multinode: true disagg: true - hardware: - prefill: gb300 - decode: gb300 scenarios: fixed-seq-len: - isl: 8192 @@ -10714,9 +10606,6 @@ glm5-fp8-b200-dynamo-sglang: framework: dynamo-sglang multinode: true disagg: true - hardware: - prefill: b200 - decode: b200 scenarios: fixed-seq-len: - isl: 1024 @@ -10931,9 +10820,6 @@ dsv4-fp4-gb300-dynamo-sglang-mtp: framework: dynamo-sglang multinode: true disagg: true - hardware: - prefill: gb300 - decode: gb300 scenarios: fixed-seq-len: - isl: 8192 @@ -11129,9 +11015,6 @@ qwen3.5-fp4-gb300-dynamo-sglang: framework: dynamo-sglang multinode: true disagg: true - hardware: - prefill: gb300 - decode: gb300 scenarios: fixed-seq-len: - isl: 8192 @@ -11212,9 +11095,6 @@ glm5-fp4-gb300-dynamo-sglang: framework: dynamo-sglang multinode: true disagg: true - hardware: - prefill: gb300 - decode: gb300 scenarios: fixed-seq-len: # ---------- 8k1k high-throughput (wide-EP TP=32 decode) ---------- @@ -11425,9 +11305,6 @@ glm5-fp8-gb300-dynamo-sglang: framework: dynamo-sglang multinode: true disagg: true - hardware: - prefill: gb300 - decode: gb300 scenarios: fixed-seq-len: # ---------- 8k1k high-throughput (wide-EP decode) ---------- @@ -11683,9 +11560,6 @@ dsv4-fp4-gb300-dynamo-vllm-agentic: framework: dynamo-vllm multinode: true disagg: true - hardware: - prefill: gb300 - decode: gb300 scenarios: agentic-coding: - search-space: @@ -11771,9 +11645,6 @@ minimaxm3-fp8-b300-dynamo-vllm: framework: dynamo-vllm multinode: true disagg: true - hardware: - prefill: b300 - decode: b300 scenarios: fixed-seq-len: - isl: 1024 @@ -11990,9 +11861,6 @@ minimaxm3-fp4-b300-dynamo-vllm: framework: dynamo-vllm multinode: true disagg: true - hardware: - prefill: b300 - decode: b300 scenarios: fixed-seq-len: - isl: 1024 @@ -12209,9 +12077,6 @@ minimaxm3-fp8-gb300-dynamo-vllm: framework: dynamo-vllm multinode: true disagg: true - hardware: - prefill: gb300 - decode: gb300 scenarios: fixed-seq-len: - isl: 1024 @@ -12485,9 +12350,6 @@ minimaxm3-fp8-gb200-dynamo-vllm: framework: dynamo-vllm multinode: true disagg: true - hardware: - prefill: gb200 - decode: gb200 scenarios: fixed-seq-len: - isl: 1024 @@ -13054,9 +12916,6 @@ kimik2.5-fp4-gb300-dynamo-vllm: framework: dynamo-vllm multinode: true disagg: true - hardware: - prefill: gb300 - decode: gb300 scenarios: fixed-seq-len: - isl: 1024 @@ -13309,9 +13168,6 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-3p2d-tep8-tp8: framework: dynamo-vllm multinode: true disagg: true - hardware: - prefill: gb200 - decode: gb200 scenarios: agentic-coding: - search-space: @@ -13368,9 +13224,6 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-2p1d-dep8-dep8: framework: dynamo-vllm multinode: true disagg: true - hardware: - prefill: gb200 - decode: gb200 scenarios: agentic-coding: - search-space: diff --git a/utils/matrix_logic/test_validation.py b/utils/matrix_logic/test_validation.py index cea10c4f1b..b62f868144 100644 --- a/utils/matrix_logic/test_validation.py +++ b/utils/matrix_logic/test_validation.py @@ -472,16 +472,16 @@ def test_valid_entry(self, valid_multinode_matrix_entry): assert entry.hardware.prefill == "gb200" assert entry.hardware.decode == "h100" - def test_disagg_requires_hardware(self, valid_multinode_matrix_entry): - """Disaggregated matrix entries must identify both hardware pools.""" + def test_disagg_allows_omitted_hardware(self, valid_multinode_matrix_entry): + """Homogeneous disaggregated entries may omit hardware metadata.""" del valid_multinode_matrix_entry["hardware"] - with pytest.raises(Exception, match="hardware.*required"): - MultiNodeMatrixEntry(**valid_multinode_matrix_entry) + entry = MultiNodeMatrixEntry(**valid_multinode_matrix_entry) + assert entry.hardware is None - def test_non_disagg_rejects_hardware(self, valid_multinode_matrix_entry): - """Hardware pool metadata is scoped to disaggregated entries.""" - valid_multinode_matrix_entry["disagg"] = False - with pytest.raises(Exception, match="hardware.*only be set"): + def test_hardware_requires_prefill_and_decode(self, valid_multinode_matrix_entry): + """Heterogeneous hardware metadata must identify both worker pools.""" + del valid_multinode_matrix_entry["hardware"]["decode"] + with pytest.raises(Exception, match="decode"): MultiNodeMatrixEntry(**valid_multinode_matrix_entry) def test_prefill_decode_worker_configs(self, valid_multinode_matrix_entry): @@ -837,10 +837,16 @@ def test_multinode_master_config(self, valid_multinode_master_config): assert config.hardware.prefill == "gb200" assert config.hardware.decode == "h100" - def test_disagg_master_config_requires_hardware(self, valid_multinode_master_config): - """Disaggregated master configs must identify prefill and decode hardware.""" + def test_disagg_master_config_allows_omitted_hardware(self, valid_multinode_master_config): + """Homogeneous disaggregated master configs may omit hardware metadata.""" del valid_multinode_master_config["hardware"] - with pytest.raises(Exception, match="hardware.*required"): + config = MultiNodeMasterConfigEntry(**valid_multinode_master_config) + assert config.hardware is None + + def test_master_hardware_requires_prefill_and_decode(self, valid_multinode_master_config): + """Heterogeneous master configs must identify both worker pools.""" + del valid_multinode_master_config["hardware"]["decode"] + with pytest.raises(Exception, match="decode"): MultiNodeMasterConfigEntry(**valid_multinode_master_config) def test_single_node_cannot_have_multinode_true(self, valid_single_node_master_config): diff --git a/utils/matrix_logic/validation.py b/utils/matrix_logic/validation.py index 0f9a5628f2..5b5b2334b3 100644 --- a/utils/matrix_logic/validation.py +++ b/utils/matrix_logic/validation.py @@ -133,21 +133,6 @@ class DisaggregatedHardwareConfig(BaseModel): decode: str = Field(min_length=1) -def _validate_disaggregated_hardware(self): - """Require explicit prefill/decode hardware exactly when disaggregation is enabled.""" - if self.disagg and self.hardware is None: - raise ValueError( - f"'{Fields.HARDWARE.value}' is required when " - f"'{Fields.DISAGG.value}' is true" - ) - if not self.disagg and self.hardware is not None: - raise ValueError( - f"'{Fields.HARDWARE.value}' can only be set when " - f"'{Fields.DISAGG.value}' is true" - ) - return self - - class MultiNodeMatrixEntry(BaseModel): """Pydantic model for validating multinode matrix entry structure. This validates the input that should be expected to .github/workflows/benchmark-multinode-tmpl.yml""" @@ -178,10 +163,6 @@ class MultiNodeMatrixEntry(BaseModel): default=False, alias=Fields.EVAL_ALL_CONCS.value ) - @model_validator(mode='after') - def validate_disaggregated_hardware(self): - return _validate_disaggregated_hardware(self) - class SingleNodeAgenticMatrixEntry(BaseModel): """Pydantic model for validating single-node agentic coding matrix entries.""" @@ -236,10 +217,6 @@ class MultiNodeAgenticMatrixEntry(BaseModel): hardware: Optional[DisaggregatedHardwareConfig] = None scenario_type: str = Field(alias=Fields.SCENARIO_TYPE.value) - @model_validator(mode='after') - def validate_disaggregated_hardware(self): - return _validate_disaggregated_hardware(self) - AgenticMatrixEntry = Union[SingleNodeAgenticMatrixEntry, MultiNodeAgenticMatrixEntry] @@ -558,10 +535,6 @@ class MultiNodeMasterConfigEntry(BaseModel): hardware: Optional[DisaggregatedHardwareConfig] = None scenarios: MultiNodeScenarios - @model_validator(mode='after') - def validate_disaggregated_hardware(self): - return _validate_disaggregated_hardware(self) - @model_validator(mode='after') def validate_agentic_runner(self): _validate_agentic_runner_is_cluster(self.runner, self.scenarios) diff --git a/utils/process_result.py b/utils/process_result.py index 33ae4c1ed0..c4801edec1 100644 --- a/utils/process_result.py +++ b/utils/process_result.py @@ -65,9 +65,13 @@ def get_required_env_vars(required_vars): multinode_vars = ['PREFILL_GPUS', 'DECODE_GPUS', 'PREFILL_NUM_WORKERS', 'PREFILL_TP', 'PREFILL_EP', 'PREFILL_DP_ATTN', 'DECODE_NUM_WORKERS', 'DECODE_TP', 'DECODE_EP', 'DECODE_DP_ATTN'] - if disagg: - multinode_vars.extend(['PREFILL_HARDWARE', 'DECODE_HARDWARE']) multinode_env = get_required_env_vars(multinode_vars) + prefill_hardware = os.environ.get('PREFILL_HARDWARE', '') + decode_hardware = os.environ.get('DECODE_HARDWARE', '') + if bool(prefill_hardware) != bool(decode_hardware): + raise ValueError( + "PREFILL_HARDWARE and DECODE_HARDWARE must be specified together." + ) prefill_gpus = int(multinode_env['PREFILL_GPUS']) decode_gpus = int(multinode_env['DECODE_GPUS']) prefill_num_workers = int(multinode_env['PREFILL_NUM_WORKERS']) @@ -105,9 +109,9 @@ def get_required_env_vars(required_vars): 'output_tput_per_gpu': float(bmk_result['output_throughput']) / output_tput_denominator, 'input_tput_per_gpu': (float(bmk_result['total_token_throughput']) - float(bmk_result['output_throughput'])) / prefill_gpus, } - if disagg: - multi_node_data['prefill_hw'] = multinode_env['PREFILL_HARDWARE'] - multi_node_data['decode_hw'] = multinode_env['DECODE_HARDWARE'] + if prefill_hardware: + multi_node_data['prefill_hw'] = prefill_hardware + multi_node_data['decode_hw'] = decode_hardware data = data | multi_node_data else: diff --git a/utils/test_process_result.py b/utils/test_process_result.py index 016d41d371..daf02f6470 100644 --- a/utils/test_process_result.py +++ b/utils/test_process_result.py @@ -243,6 +243,31 @@ def test_multinode_processing(self, tmp_path, sample_benchmark_result, multinode assert output_data["output_tput_per_gpu"] == pytest.approx(12000.0 / 8) # decode gpus assert output_data["input_tput_per_gpu"] == pytest.approx((15000.5 - 12000.0) / 20) # prefill gpus + def test_homogeneous_multinode_omits_hardware_fields( + self, tmp_path, sample_benchmark_result, multinode_env_vars + ): + """Absent hardware metadata should preserve homogeneous result output.""" + multinode_env_vars.pop("PREFILL_HARDWARE") + multinode_env_vars.pop("DECODE_HARDWARE") + + result = run_script(tmp_path, multinode_env_vars, sample_benchmark_result) + + assert result.returncode == 0, f"Script failed: {result.stderr}" + output_data = json.loads(result.stdout) + assert "prefill_hw" not in output_data + assert "decode_hw" not in output_data + + def test_partial_hardware_metadata_fails( + self, tmp_path, sample_benchmark_result, multinode_env_vars + ): + """Prefill and decode hardware must always be provided together.""" + multinode_env_vars.pop("DECODE_HARDWARE") + + result = run_script(tmp_path, multinode_env_vars, sample_benchmark_result) + + assert result.returncode != 0 + assert "PREFILL_HARDWARE and DECODE_HARDWARE" in result.stderr + def test_missing_base_env_vars(self, tmp_path, sample_benchmark_result): """Test that missing base env vars causes failure.""" result_file = tmp_path / "benchmark_result.json" From 4a80449c566ab1e10b22420c6059099942de9ef2 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Mon, 6 Jul 2026 14:37:10 -0500 Subject: [PATCH 3/5] fix: restrict disaggregation to multinode configs --- configs/CONFIGS.md | 2 ++ utils/matrix_logic/test_validation.py | 12 ++++++++++++ utils/matrix_logic/validation.py | 4 ++-- 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/configs/CONFIGS.md b/configs/CONFIGS.md index 51ebd429b6..bc47a09999 100644 --- a/configs/CONFIGS.md +++ b/configs/CONFIGS.md @@ -46,6 +46,8 @@ The below list describes what each field is: fleet. - `precision`: The precision to run the benchmark. Again, this is used to find which script to run in `benchmarks/`. - `framework`: The framework (serving runtime) to serve the benchmark, e.g., `vllm`, `sglang`, `trt`. +- `disagg`: Enables disaggregated serving and may only be `true` when + `multinode` is also `true`. - `hardware`: Optional metadata for heterogeneous disaggregated deployments. When present, both `prefill` and `decode` are required and identify the GPU SKU used by each worker pool (for example, `b200` and `h100`). Omit this block diff --git a/utils/matrix_logic/test_validation.py b/utils/matrix_logic/test_validation.py index b62f868144..d418d46a7f 100644 --- a/utils/matrix_logic/test_validation.py +++ b/utils/matrix_logic/test_validation.py @@ -321,6 +321,12 @@ def test_extra_field_forbidden(self, valid_single_node_matrix_entry): with pytest.raises(Exception): SingleNodeMatrixEntry(**valid_single_node_matrix_entry) + def test_disagg_requires_multinode(self, valid_single_node_matrix_entry): + """Single-node matrix entries cannot enable disaggregation.""" + valid_single_node_matrix_entry["disagg"] = True + with pytest.raises(Exception, match="disagg"): + SingleNodeMatrixEntry(**valid_single_node_matrix_entry) + # ============================================================================= # Test Agentic Matrix Entries @@ -866,6 +872,12 @@ def test_disagg_default_false(self, valid_single_node_master_config): config = SingleNodeMasterConfigEntry(**valid_single_node_master_config) assert config.disagg is False + def test_disagg_requires_multinode(self, valid_single_node_master_config): + """Single-node master configs cannot enable disaggregation.""" + valid_single_node_master_config["disagg"] = True + with pytest.raises(Exception, match="disagg"): + SingleNodeMasterConfigEntry(**valid_single_node_master_config) + def test_single_node_agentic_master_config_requires_cluster_runner(self): """Single-node agentic configs must pin an exact cluster label.""" config = { diff --git a/utils/matrix_logic/validation.py b/utils/matrix_logic/validation.py index 5b5b2334b3..439f6c1d6c 100644 --- a/utils/matrix_logic/validation.py +++ b/utils/matrix_logic/validation.py @@ -108,7 +108,7 @@ class SingleNodeMatrixEntry(BaseModel): conc: Union[int, List[int]] max_model_len: int = Field(alias=Fields.MAX_MODEL_LEN.value) exp_name: str = Field(alias=Fields.EXP_NAME.value) - disagg: bool + disagg: Literal[False] run_eval: bool = Field(alias=Fields.RUN_EVAL.value) eval_only: bool = Field(alias=Fields.EVAL_ONLY.value, default=False) @@ -511,7 +511,7 @@ class SingleNodeMasterConfigEntry(BaseModel): framework: str runner: str multinode: Literal[False] - disagg: bool = Field(default=False) + disagg: Literal[False] = Field(default=False) scenarios: SingleNodeScenarios @model_validator(mode='after') From fa48e274e4eb17af59e2c79023b7bdda9bcab032 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Mon, 6 Jul 2026 14:43:45 -0500 Subject: [PATCH 4/5] refactor: colocate hardware with worker configs --- .github/workflows/e2e-tests.yml | 12 ++--- .github/workflows/run-sweep.yml | 12 ++--- configs/CONFIGS.md | 42 +++++++++++++---- utils/matrix_logic/generate_sweep_configs.py | 10 ---- .../test_generate_sweep_configs.py | 16 +++---- utils/matrix_logic/test_validation.py | 46 ++++++++++++------- utils/matrix_logic/validation.py | 32 +++++++++---- utils/test_process_result.py | 5 +- 8 files changed, 108 insertions(+), 67 deletions(-) diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 536222ee77..cdb7247b30 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -106,8 +106,8 @@ jobs: conc-list: ${{ toJson(matrix.config.conc) }} spec-decoding: ${{ matrix.config.spec-decoding }} disagg: ${{ matrix.config.disagg }} - prefill-hardware: ${{ matrix.config.hardware.prefill }} - decode-hardware: ${{ matrix.config.hardware.decode }} + prefill-hardware: ${{ matrix.config.prefill.hardware }} + decode-hardware: ${{ matrix.config.decode.hardware }} prefill-num-worker: ${{ matrix.config.prefill.num-worker }} prefill-tp: ${{ matrix.config.prefill.tp }} @@ -147,8 +147,8 @@ jobs: conc-list: ${{ toJson(matrix.config.conc) }} spec-decoding: ${{ matrix.config.spec-decoding }} disagg: ${{ matrix.config.disagg }} - prefill-hardware: ${{ matrix.config.hardware.prefill }} - decode-hardware: ${{ matrix.config.hardware.decode }} + prefill-hardware: ${{ matrix.config.prefill.hardware }} + decode-hardware: ${{ matrix.config.decode.hardware }} prefill-num-worker: ${{ matrix.config.prefill.num-worker }} prefill-tp: ${{ matrix.config.prefill.tp }} @@ -225,8 +225,8 @@ jobs: conc-list: ${{ toJson(matrix.config.conc) }} spec-decoding: ${{ matrix.config.spec-decoding }} disagg: ${{ matrix.config.disagg }} - prefill-hardware: ${{ matrix.config.hardware.prefill }} - decode-hardware: ${{ matrix.config.hardware.decode }} + prefill-hardware: ${{ matrix.config.prefill.hardware }} + decode-hardware: ${{ matrix.config.decode.hardware }} prefill-num-worker: ${{ matrix.config.prefill.num-worker }} prefill-tp: ${{ matrix.config.prefill.tp }} prefill-ep: ${{ matrix.config.prefill.ep }} diff --git a/.github/workflows/run-sweep.yml b/.github/workflows/run-sweep.yml index 8a15e4d852..99b632c33d 100644 --- a/.github/workflows/run-sweep.yml +++ b/.github/workflows/run-sweep.yml @@ -369,8 +369,8 @@ jobs: conc-list: ${{ toJson(matrix.config.conc) }} spec-decoding: ${{ matrix.config.spec-decoding }} disagg: ${{ matrix.config.disagg }} - prefill-hardware: ${{ matrix.config.hardware.prefill }} - decode-hardware: ${{ matrix.config.hardware.decode }} + prefill-hardware: ${{ matrix.config.prefill.hardware }} + decode-hardware: ${{ matrix.config.decode.hardware }} prefill-num-worker: ${{ matrix.config.prefill.num-worker }} prefill-tp: ${{ matrix.config.prefill.tp }} @@ -533,8 +533,8 @@ jobs: conc-list: '[${{ matrix.config.conc }}]' spec-decoding: ${{ matrix.config.spec-decoding }} disagg: ${{ matrix.config.disagg }} - prefill-hardware: ${{ matrix.config.hardware.prefill }} - decode-hardware: ${{ matrix.config.hardware.decode }} + prefill-hardware: ${{ matrix.config.prefill.hardware }} + decode-hardware: ${{ matrix.config.decode.hardware }} prefill-num-worker: ${{ matrix.config.prefill.num-worker }} prefill-tp: ${{ matrix.config.prefill.tp }} prefill-ep: ${{ matrix.config.prefill.ep }} @@ -622,8 +622,8 @@ jobs: conc-list: ${{ toJson(matrix.config.conc) }} spec-decoding: ${{ matrix.config.spec-decoding }} disagg: ${{ matrix.config.disagg }} - prefill-hardware: ${{ matrix.config.hardware.prefill }} - decode-hardware: ${{ matrix.config.hardware.decode }} + prefill-hardware: ${{ matrix.config.prefill.hardware }} + decode-hardware: ${{ matrix.config.decode.hardware }} prefill-num-worker: ${{ matrix.config.prefill.num-worker }} prefill-tp: ${{ matrix.config.prefill.tp }} prefill-ep: ${{ matrix.config.prefill.ep }} diff --git a/configs/CONFIGS.md b/configs/CONFIGS.md index bc47a09999..0ede2e0b77 100644 --- a/configs/CONFIGS.md +++ b/configs/CONFIGS.md @@ -12,11 +12,6 @@ entry-name: runner: string precision: string framework: string - multinode: true - disagg: true - hardware: # optional; omit for homogeneous hardware - prefill: string - decode: string scenarios: fixed-seq-len: - isl: int @@ -33,6 +28,33 @@ entry-name: - { tp: int, conc-start: int, conc-end: int } - ... ``` + +Heterogeneous disaggregated search-space entries declare hardware on each +worker pool. Omit both `hardware` fields for homogeneous hardware: + +```yaml +multinode: true +disagg: true +scenarios: + fixed-seq-len: + - isl: 1024 + osl: 1024 + search-space: + - conc-list: [64] + prefill: + hardware: b200 + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: false + decode: + hardware: h100 + num-worker: 2 + tp: 8 + ep: 8 + dp-attn: false +``` + Note: while not required, `entry-name` typically takes the format `---`. The below list describes what each field is: @@ -48,11 +70,11 @@ The below list describes what each field is: - `framework`: The framework (serving runtime) to serve the benchmark, e.g., `vllm`, `sglang`, `trt`. - `disagg`: Enables disaggregated serving and may only be `true` when `multinode` is also `true`. -- `hardware`: Optional metadata for heterogeneous disaggregated deployments. - When present, both `prefill` and `decode` are required and identify the GPU - SKU used by each worker pool (for example, `b200` and `h100`). Omit this block - for homogeneous hardware. These values flow into generated matrix entries - and aggregate results, but do not affect runner scheduling. +- `hardware`: Optional metadata within each `prefill` and `decode` worker block + for heterogeneous disaggregated deployments. If one worker declares a GPU + SKU, the other must also declare one. Omit both fields for homogeneous + hardware. These values flow into aggregate results but do not affect runner + scheduling. - `scenarios`: A dictionary of benchmark scenario types. At least one must be specified. Currently supported: - `fixed-seq-len`: Fixed input/output sequence length benchmarks. Each entry must have: - `isl`: An integer representing the input sequence length, e.g., `1024` diff --git a/utils/matrix_logic/generate_sweep_configs.py b/utils/matrix_logic/generate_sweep_configs.py index 2702cb7e68..545eb9f969 100644 --- a/utils/matrix_logic/generate_sweep_configs.py +++ b/utils/matrix_logic/generate_sweep_configs.py @@ -349,7 +349,6 @@ def generate_full_sweep(args, all_config_data, runner_data): is_multinode = val.get(Fields.MULTINODE.value, False) # Get disagg value, defaulting to False if not specified disagg = val.get(Fields.DISAGG.value, False) - hardware = val.get(Fields.HARDWARE.value) scenarios = val[Fields.SCENARIOS.value] scenario_filter = set(args.scenario_type) if getattr(args, 'scenario_type', None) else None @@ -458,8 +457,6 @@ def generate_full_sweep(args, all_config_data, runner_data): Fields.DISAGG.value: disagg, Fields.RUN_EVAL.value: False, # Default, may be overridden by mark_eval_entries } - if hardware is not None: - entry[Fields.HARDWARE.value] = hardware validate_matrix_entry(entry, is_multinode) matrix_values.append(entry) @@ -657,8 +654,6 @@ def generate_full_sweep(args, all_config_data, runner_data): Fields.DISAGG.value: disagg, Fields.SCENARIO_TYPE.value: "agentic-coding", } - if hardware is not None: - entry[Fields.HARDWARE.value] = hardware validate_agentic_matrix_entry(entry) matrix_values.append(entry) else: @@ -736,7 +731,6 @@ def generate_test_config_sweep(args, all_config_data, runner_data=None): if not runners_for_entry: continue disagg = val.get(Fields.DISAGG.value, False) - hardware = val.get(Fields.HARDWARE.value) # Build seq-len filter if --seq-lens was provided seq_lens_filter = None @@ -803,8 +797,6 @@ def generate_test_config_sweep(args, all_config_data, runner_data=None): Fields.DISAGG.value: disagg, Fields.RUN_EVAL.value: False, } - if hardware is not None: - entry[Fields.HARDWARE.value] = hardware matrix_values.append(validate_matrix_entry(entry, is_multinode=True)) else: # Single-node config @@ -929,8 +921,6 @@ def generate_test_config_sweep(args, all_config_data, runner_data=None): Fields.DISAGG.value: disagg, Fields.SCENARIO_TYPE.value: "agentic-coding", } - if hardware is not None: - entry[Fields.HARDWARE.value] = hardware matrix_values.append(validate_agentic_matrix_entry(entry)) else: for conc in conc_values: diff --git a/utils/matrix_logic/test_generate_sweep_configs.py b/utils/matrix_logic/test_generate_sweep_configs.py index 17994b7991..5a78dedfef 100644 --- a/utils/matrix_logic/test_generate_sweep_configs.py +++ b/utils/matrix_logic/test_generate_sweep_configs.py @@ -68,7 +68,6 @@ def sample_multinode_config(): "runner": "gb200", "multinode": True, "disagg": True, - "hardware": {"prefill": "gb200", "decode": "h100"}, "scenarios": { "fixed-seq-len": [ @@ -79,6 +78,7 @@ def sample_multinode_config(): { "conc-list": [2150], "prefill": { + "hardware": "gb200", "num-worker": 5, "tp": 4, "ep": 4, @@ -89,6 +89,7 @@ def sample_multinode_config(): ], }, "decode": { + "hardware": "h100", "num-worker": 1, "tp": 8, "ep": 8, @@ -982,7 +983,8 @@ def test_multinode_entry_structure(self, sample_multinode_config, sample_runner_ assert entry["prefill"]["num-worker"] == 5 assert entry["decode"]["num-worker"] == 1 assert entry["disagg"] is True - assert entry["hardware"] == {"prefill": "gb200", "decode": "h100"} + assert entry["prefill"]["hardware"] == "gb200" + assert entry["decode"]["hardware"] == "h100" def test_multinode_conc_as_list(self, sample_multinode_config, sample_runner_config, full_sweep_args_multi_node): """Multinode conc should be passed as list.""" @@ -2000,15 +2002,14 @@ def test_multinode_agentic_groups_concurrencies_per_search_entry(self): "runner": "gb200", "multinode": True, "disagg": True, - "hardware": {"prefill": "gb200", "decode": "h100"}, "scenarios": { "agentic-coding": [ { "search-space": [ { "conc-list": [16, 32, 64, 128, 256], - "prefill": {"num-worker": 2, "tp": 8, "ep": 8, "dp-attn": False}, - "decode": {"num-worker": 1, "tp": 8, "ep": 1, "dp-attn": False}, + "prefill": {"hardware": "gb200", "num-worker": 2, "tp": 8, "ep": 8, "dp-attn": False}, + "decode": {"hardware": "h100", "num-worker": 1, "tp": 8, "ep": 1, "dp-attn": False}, } ], } @@ -2149,14 +2150,13 @@ def test_node_type_filters_apply_to_agentic_configs( "runner": "cluster:gb200-nv", "multinode": True, "disagg": True, - "hardware": {"prefill": "gb200", "decode": "h100"}, "scenarios": { "agentic-coding": [{ "search-space": [ { "conc-list": [16, 32], - "prefill": {"num-worker": 2, "tp": 8, "ep": 8, "dp-attn": False}, - "decode": {"num-worker": 1, "tp": 8, "ep": 1, "dp-attn": False}, + "prefill": {"hardware": "gb200", "num-worker": 2, "tp": 8, "ep": 8, "dp-attn": False}, + "decode": {"hardware": "h100", "num-worker": 1, "tp": 8, "ep": 1, "dp-attn": False}, }, ], }], diff --git a/utils/matrix_logic/test_validation.py b/utils/matrix_logic/test_validation.py index d418d46a7f..b60c99f6f3 100644 --- a/utils/matrix_logic/test_validation.py +++ b/utils/matrix_logic/test_validation.py @@ -65,6 +65,7 @@ def valid_multinode_matrix_entry(): "isl": 1024, "osl": 1024, "prefill": { + "hardware": "gb200", "num-worker": 5, "tp": 4, "ep": 4, @@ -75,6 +76,7 @@ def valid_multinode_matrix_entry(): ], }, "decode": { + "hardware": "h100", "num-worker": 1, "tp": 8, "ep": 8, @@ -90,7 +92,6 @@ def valid_multinode_matrix_entry(): "max-model-len": 2248, "exp-name": "dsr1_1k1k", "disagg": True, - "hardware": {"prefill": "gb200", "decode": "h100"}, "run-eval": False, } @@ -133,7 +134,6 @@ def valid_multinode_master_config(): "runner": "gb200", "multinode": True, "disagg": True, - "hardware": {"prefill": "gb200", "decode": "h100"}, "scenarios": { "fixed-seq-len": [ @@ -143,6 +143,7 @@ def valid_multinode_master_config(): "search-space": [ { "prefill": { + "hardware": "gb200", "num-worker": 5, "tp": 4, "ep": 4, @@ -153,6 +154,7 @@ def valid_multinode_master_config(): ], }, "decode": { + "hardware": "h100", "num-worker": 1, "tp": 8, "ep": 8, @@ -475,19 +477,24 @@ def test_valid_entry(self, valid_multinode_matrix_entry): assert entry.model == "deepseek-r1-fp4" assert entry.conc == [2150] assert entry.disagg is True - assert entry.hardware.prefill == "gb200" - assert entry.hardware.decode == "h100" + assert entry.prefill.hardware == "gb200" + assert entry.decode.hardware == "h100" def test_disagg_allows_omitted_hardware(self, valid_multinode_matrix_entry): """Homogeneous disaggregated entries may omit hardware metadata.""" - del valid_multinode_matrix_entry["hardware"] + del valid_multinode_matrix_entry["prefill"]["hardware"] + del valid_multinode_matrix_entry["decode"]["hardware"] entry = MultiNodeMatrixEntry(**valid_multinode_matrix_entry) - assert entry.hardware is None + assert entry.prefill.hardware is None + assert entry.decode.hardware is None - def test_hardware_requires_prefill_and_decode(self, valid_multinode_matrix_entry): + @pytest.mark.parametrize("missing_worker", ["prefill", "decode"]) + def test_hardware_requires_prefill_and_decode( + self, valid_multinode_matrix_entry, missing_worker + ): """Heterogeneous hardware metadata must identify both worker pools.""" - del valid_multinode_matrix_entry["hardware"]["decode"] - with pytest.raises(Exception, match="decode"): + del valid_multinode_matrix_entry[missing_worker]["hardware"] + with pytest.raises(Exception, match="both.*prefill.*decode"): MultiNodeMatrixEntry(**valid_multinode_matrix_entry) def test_prefill_decode_worker_configs(self, valid_multinode_matrix_entry): @@ -840,19 +847,25 @@ def test_multinode_master_config(self, valid_multinode_master_config): assert config.model_prefix == "dsr1" assert config.runner == "gb200" assert config.disagg is True - assert config.hardware.prefill == "gb200" - assert config.hardware.decode == "h100" + search_entry = config.scenarios.fixed_seq_len[0].search_space[0] + assert search_entry.prefill.hardware == "gb200" + assert search_entry.decode.hardware == "h100" def test_disagg_master_config_allows_omitted_hardware(self, valid_multinode_master_config): """Homogeneous disaggregated master configs may omit hardware metadata.""" - del valid_multinode_master_config["hardware"] + search_entry = valid_multinode_master_config["scenarios"]["fixed-seq-len"][0]["search-space"][0] + del search_entry["prefill"]["hardware"] + del search_entry["decode"]["hardware"] config = MultiNodeMasterConfigEntry(**valid_multinode_master_config) - assert config.hardware is None + validated_entry = config.scenarios.fixed_seq_len[0].search_space[0] + assert validated_entry.prefill.hardware is None + assert validated_entry.decode.hardware is None def test_master_hardware_requires_prefill_and_decode(self, valid_multinode_master_config): """Heterogeneous master configs must identify both worker pools.""" - del valid_multinode_master_config["hardware"]["decode"] - with pytest.raises(Exception, match="decode"): + search_entry = valid_multinode_master_config["scenarios"]["fixed-seq-len"][0]["search-space"][0] + del search_entry["decode"]["hardware"] + with pytest.raises(Exception, match="both.*prefill.*decode"): MultiNodeMasterConfigEntry(**valid_multinode_master_config) def test_single_node_cannot_have_multinode_true(self, valid_single_node_master_config): @@ -916,7 +929,6 @@ def test_multinode_agentic_master_config_requires_cluster_runner(self): "runner": "b200-multinode", "multinode": True, "disagg": True, - "hardware": {"prefill": "b200", "decode": "b200"}, "scenarios": { "agentic-coding": [ { @@ -925,12 +937,14 @@ def test_multinode_agentic_master_config_requires_cluster_runner(self): "spec-decoding": "none", "conc-list": [1], "prefill": { + "hardware": "b200", "num-worker": 1, "tp": 4, "ep": 4, "dp-attn": True, }, "decode": { + "hardware": "b200", "num-worker": 1, "tp": 8, "ep": 8, diff --git a/utils/matrix_logic/validation.py b/utils/matrix_logic/validation.py index 439f6c1d6c..3de1f4d606 100644 --- a/utils/matrix_logic/validation.py +++ b/utils/matrix_logic/validation.py @@ -121,16 +121,19 @@ class WorkerConfig(BaseModel): tp: int ep: int dp_attn: bool = Field(alias=Fields.DP_ATTN.value) + hardware: Optional[str] = Field(default=None, min_length=1) additional_settings: Optional[List[str]] = Field( default=[], alias=Fields.ADDITIONAL_SETTINGS.value) -class DisaggregatedHardwareConfig(BaseModel): - """Hardware SKUs assigned to each side of a disaggregated deployment.""" - model_config = ConfigDict(extra='forbid', populate_by_name=True) - - prefill: str = Field(min_length=1) - decode: str = Field(min_length=1) +def _validate_worker_hardware_pair(self): + """Require prefill and decode workers to declare hardware together.""" + if bool(self.prefill.hardware) != bool(self.decode.hardware): + raise ValueError( + f"'{Fields.HARDWARE.value}' must be specified for both " + f"'{Fields.PREFILL.value}' and '{Fields.DECODE.value}', or neither" + ) + return self class MultiNodeMatrixEntry(BaseModel): @@ -155,7 +158,6 @@ class MultiNodeMatrixEntry(BaseModel): max_model_len: int = Field(alias=Fields.MAX_MODEL_LEN.value) exp_name: str = Field(alias=Fields.EXP_NAME.value) disagg: bool - hardware: Optional[DisaggregatedHardwareConfig] = None run_eval: bool = Field(alias=Fields.RUN_EVAL.value) eval_only: bool = Field(alias=Fields.EVAL_ONLY.value, default=False) eval_conc: Optional[int] = Field(default=None, alias=Fields.EVAL_CONC.value) @@ -163,6 +165,10 @@ class MultiNodeMatrixEntry(BaseModel): default=False, alias=Fields.EVAL_ALL_CONCS.value ) + @model_validator(mode='after') + def validate_worker_hardware_pair(self): + return _validate_worker_hardware_pair(self) + class SingleNodeAgenticMatrixEntry(BaseModel): """Pydantic model for validating single-node agentic coding matrix entries.""" @@ -214,9 +220,12 @@ class MultiNodeAgenticMatrixEntry(BaseModel): duration: int = Field(alias=Fields.DURATION.value) exp_name: str = Field(alias=Fields.EXP_NAME.value) disagg: bool - hardware: Optional[DisaggregatedHardwareConfig] = None scenario_type: str = Field(alias=Fields.SCENARIO_TYPE.value) + @model_validator(mode='after') + def validate_worker_hardware_pair(self): + return _validate_worker_hardware_pair(self) + AgenticMatrixEntry = Union[SingleNodeAgenticMatrixEntry, MultiNodeAgenticMatrixEntry] @@ -378,6 +387,10 @@ class MultiNodeSearchSpaceEntry(BaseModel): def validate_conc_fields(self): return _validate_conc_fields(self) + @model_validator(mode='after') + def validate_worker_hardware_pair(self): + return _validate_worker_hardware_pair(self) + class SingleNodeSeqLenConfig(BaseModel): """Single node sequence length configuration.""" @@ -444,6 +457,8 @@ def validate_topology_fields(self): f"Single-node agentic search-space entries must specify " f"{Fields.KV_OFFLOADING.value}" ) + if has_complete_multinode: + _validate_worker_hardware_pair(self) return self class AgenticCodingConfig(BaseModel): @@ -532,7 +547,6 @@ class MultiNodeMasterConfigEntry(BaseModel): runner: str multinode: Literal[True] disagg: bool = Field(default=False) - hardware: Optional[DisaggregatedHardwareConfig] = None scenarios: MultiNodeScenarios @model_validator(mode='after') diff --git a/utils/test_process_result.py b/utils/test_process_result.py index daf02f6470..127e2512f4 100644 --- a/utils/test_process_result.py +++ b/utils/test_process_result.py @@ -257,11 +257,12 @@ def test_homogeneous_multinode_omits_hardware_fields( assert "prefill_hw" not in output_data assert "decode_hw" not in output_data + @pytest.mark.parametrize("missing_var", ["PREFILL_HARDWARE", "DECODE_HARDWARE"]) def test_partial_hardware_metadata_fails( - self, tmp_path, sample_benchmark_result, multinode_env_vars + self, tmp_path, sample_benchmark_result, multinode_env_vars, missing_var ): """Prefill and decode hardware must always be provided together.""" - multinode_env_vars.pop("DECODE_HARDWARE") + multinode_env_vars.pop(missing_var) result = run_script(tmp_path, multinode_env_vars, sample_benchmark_result) From 255e2e8426288bd385cdea71fd7e607b9628dabd Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Mon, 6 Jul 2026 14:50:53 -0500 Subject: [PATCH 5/5] fix: surface heterogeneous hardware in agentic results --- .../aggregation/process_agentic_result.py | 9 +++ .../test_process_agentic_result.py | 67 +++++++++++++++++++ 2 files changed, 76 insertions(+) diff --git a/utils/agentic/aggregation/process_agentic_result.py b/utils/agentic/aggregation/process_agentic_result.py index 83211f4f33..d87f452a86 100644 --- a/utils/agentic/aggregation/process_agentic_result.py +++ b/utils/agentic/aggregation/process_agentic_result.py @@ -67,6 +67,12 @@ def _gpu_shape() -> tuple[dict[str, Any], int, int, int, str]: decode_tp = env_int("DECODE_TP") decode_ep = env_int("DECODE_EP", 1) decode_dp_attention = os.environ.get("DECODE_DP_ATTN", "false") + prefill_hardware = os.environ.get("PREFILL_HARDWARE", "") + decode_hardware = os.environ.get("DECODE_HARDWARE", "") + if bool(prefill_hardware) != bool(decode_hardware): + raise SystemExit( + "PREFILL_HARDWARE and DECODE_HARDWARE must be specified together." + ) num_prefill_gpu = prefill_num_workers * prefill_tp num_decode_gpu = decode_num_workers * decode_tp num_gpus = num_prefill_gpu + num_decode_gpu @@ -91,6 +97,9 @@ def _gpu_shape() -> tuple[dict[str, Any], int, int, int, str]: "num_decode_gpu": num_decode_gpu, } ) + if prefill_hardware: + fields["prefill_hw"] = prefill_hardware + fields["decode_hw"] = decode_hardware return fields, num_gpus, tp, ep, dp_attention diff --git a/utils/agentic/aggregation/test_process_agentic_result.py b/utils/agentic/aggregation/test_process_agentic_result.py index a2bac8ea52..37a10ffc1e 100644 --- a/utils/agentic/aggregation/test_process_agentic_result.py +++ b/utils/agentic/aggregation/test_process_agentic_result.py @@ -26,6 +26,7 @@ load_aggregate, load_records, ) +from utils.agentic.aggregation.process_agentic_result import _gpu_shape from utils.agentic.aggregation.server_metrics import ( compute_server_metrics, load_server_metrics, @@ -308,6 +309,8 @@ def _run_processor( env_overrides: dict[str, str] | None = None, ) -> dict: env = os.environ.copy() + env.pop("PREFILL_HARDWARE", None) + env.pop("DECODE_HARDWARE", None) env.update( { "RESULT_DIR": str(result_dir), @@ -453,6 +456,70 @@ def test_processor_surfaces_allocated_cpu_dram(tmp_path: Path): assert agg["allocated_cpu_dram_gb"] == 2400 +def test_multinode_processor_surfaces_heterogeneous_hardware(tmp_path: Path): + result_dir = _write_fixture(tmp_path) + agg = _run_processor( + result_dir, + tmp_path / "out", + env_overrides={ + "IS_MULTINODE": "true", + "DISAGG": "true", + "PREFILL_NUM_WORKERS": "1", + "PREFILL_TP": "8", + "PREFILL_EP": "8", + "PREFILL_DP_ATTN": "false", + "PREFILL_HARDWARE": "b200", + "DECODE_NUM_WORKERS": "2", + "DECODE_TP": "8", + "DECODE_EP": "8", + "DECODE_DP_ATTN": "false", + "DECODE_HARDWARE": "h100", + }, + ) + + assert agg["prefill_hw"] == "b200" + assert agg["decode_hw"] == "h100" + + +def test_multinode_processor_omits_homogeneous_hardware(tmp_path: Path): + result_dir = _write_fixture(tmp_path) + agg = _run_processor( + result_dir, + tmp_path / "out", + env_overrides={ + "IS_MULTINODE": "true", + "DISAGG": "true", + "PREFILL_NUM_WORKERS": "1", + "PREFILL_TP": "8", + "DECODE_NUM_WORKERS": "2", + "DECODE_TP": "8", + }, + ) + + assert "prefill_hw" not in agg + assert "decode_hw" not in agg + + +@pytest.mark.parametrize( + ("present_var", "missing_var"), + [ + ("PREFILL_HARDWARE", "DECODE_HARDWARE"), + ("DECODE_HARDWARE", "PREFILL_HARDWARE"), + ], +) +def test_multinode_processor_rejects_one_sided_hardware( + monkeypatch: pytest.MonkeyPatch, + present_var: str, + missing_var: str, +): + monkeypatch.setenv("IS_MULTINODE", "true") + monkeypatch.setenv(present_var, "b200") + monkeypatch.delenv(missing_var, raising=False) + + with pytest.raises(SystemExit, match="must be specified together"): + _gpu_shape() + + def test_processor_surfaces_request_accounting(tmp_path: Path): result_dir = tmp_path / "results" artifact = result_dir / "aiperf_artifacts"