diff --git a/.github/workflows/benchmark-multinode-tmpl.yml b/.github/workflows/benchmark-multinode-tmpl.yml index c20b7cc356..4277e40c13 100644 --- a/.github/workflows/benchmark-multinode-tmpl.yml +++ b/.github/workflows/benchmark-multinode-tmpl.yml @@ -39,6 +39,16 @@ on: disagg: required: true type: string + prefill-hardware: + description: "GPU SKU used by prefill workers" + required: false + type: string + default: "" + decode-hardware: + description: "GPU SKU used by decode workers" + required: false + type: string + default: "" max-model-len: required: true @@ -144,6 +154,8 @@ env: CONC_LIST: ${{ join(fromJson(inputs.conc-list), ' ') }} SPEC_DECODING: ${{ inputs.spec-decoding }} DISAGG: ${{ inputs.disagg }} + PREFILL_HARDWARE: ${{ inputs.prefill-hardware }} + DECODE_HARDWARE: ${{ inputs.decode-hardware }} RUN_EVAL: ${{ inputs.run-eval }} EVAL_ONLY: ${{ inputs.eval-only }} EVAL_CONC: ${{ inputs.eval-conc }} diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index f05c3ef99b..cdb7247b30 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -106,6 +106,8 @@ jobs: conc-list: ${{ toJson(matrix.config.conc) }} spec-decoding: ${{ matrix.config.spec-decoding }} disagg: ${{ matrix.config.disagg }} + prefill-hardware: ${{ matrix.config.prefill.hardware }} + decode-hardware: ${{ matrix.config.decode.hardware }} prefill-num-worker: ${{ matrix.config.prefill.num-worker }} prefill-tp: ${{ matrix.config.prefill.tp }} @@ -145,6 +147,8 @@ jobs: conc-list: ${{ toJson(matrix.config.conc) }} spec-decoding: ${{ matrix.config.spec-decoding }} disagg: ${{ matrix.config.disagg }} + prefill-hardware: ${{ matrix.config.prefill.hardware }} + decode-hardware: ${{ matrix.config.decode.hardware }} prefill-num-worker: ${{ matrix.config.prefill.num-worker }} prefill-tp: ${{ matrix.config.prefill.tp }} @@ -221,6 +225,8 @@ jobs: conc-list: ${{ toJson(matrix.config.conc) }} spec-decoding: ${{ matrix.config.spec-decoding }} disagg: ${{ matrix.config.disagg }} + prefill-hardware: ${{ matrix.config.prefill.hardware }} + decode-hardware: ${{ matrix.config.decode.hardware }} prefill-num-worker: ${{ matrix.config.prefill.num-worker }} prefill-tp: ${{ matrix.config.prefill.tp }} prefill-ep: ${{ matrix.config.prefill.ep }} diff --git a/.github/workflows/run-sweep.yml b/.github/workflows/run-sweep.yml index 4ba50ef91d..99b632c33d 100644 --- a/.github/workflows/run-sweep.yml +++ b/.github/workflows/run-sweep.yml @@ -369,6 +369,8 @@ jobs: conc-list: ${{ toJson(matrix.config.conc) }} spec-decoding: ${{ matrix.config.spec-decoding }} disagg: ${{ matrix.config.disagg }} + prefill-hardware: ${{ matrix.config.prefill.hardware }} + decode-hardware: ${{ matrix.config.decode.hardware }} prefill-num-worker: ${{ matrix.config.prefill.num-worker }} prefill-tp: ${{ matrix.config.prefill.tp }} @@ -531,6 +533,8 @@ jobs: conc-list: '[${{ matrix.config.conc }}]' spec-decoding: ${{ matrix.config.spec-decoding }} disagg: ${{ matrix.config.disagg }} + prefill-hardware: ${{ matrix.config.prefill.hardware }} + decode-hardware: ${{ matrix.config.decode.hardware }} prefill-num-worker: ${{ matrix.config.prefill.num-worker }} prefill-tp: ${{ matrix.config.prefill.tp }} prefill-ep: ${{ matrix.config.prefill.ep }} @@ -618,6 +622,8 @@ jobs: conc-list: ${{ toJson(matrix.config.conc) }} spec-decoding: ${{ matrix.config.spec-decoding }} disagg: ${{ matrix.config.disagg }} + prefill-hardware: ${{ matrix.config.prefill.hardware }} + decode-hardware: ${{ matrix.config.decode.hardware }} prefill-num-worker: ${{ matrix.config.prefill.num-worker }} prefill-tp: ${{ matrix.config.prefill.tp }} prefill-ep: ${{ matrix.config.prefill.ep }} diff --git a/configs/CONFIGS.md b/configs/CONFIGS.md index f46c4efda1..0ede2e0b77 100644 --- a/configs/CONFIGS.md +++ b/configs/CONFIGS.md @@ -28,6 +28,33 @@ entry-name: - { tp: int, conc-start: int, conc-end: int } - ... ``` + +Heterogeneous disaggregated search-space entries declare hardware on each +worker pool. Omit both `hardware` fields for homogeneous hardware: + +```yaml +multinode: true +disagg: true +scenarios: + fixed-seq-len: + - isl: 1024 + osl: 1024 + search-space: + - conc-list: [64] + prefill: + hardware: b200 + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: false + decode: + hardware: h100 + num-worker: 2 + tp: 8 + ep: 8 + dp-attn: false +``` + Note: while not required, `entry-name` typically takes the format `---`. The below list describes what each field is: @@ -41,6 +68,13 @@ The below list describes what each field is: fleet. - `precision`: The precision to run the benchmark. Again, this is used to find which script to run in `benchmarks/`. - `framework`: The framework (serving runtime) to serve the benchmark, e.g., `vllm`, `sglang`, `trt`. +- `disagg`: Enables disaggregated serving and may only be `true` when + `multinode` is also `true`. +- `hardware`: Optional metadata within each `prefill` and `decode` worker block + for heterogeneous disaggregated deployments. If one worker declares a GPU + SKU, the other must also declare one. Omit both fields for homogeneous + hardware. These values flow into aggregate results but do not affect runner + scheduling. - `scenarios`: A dictionary of benchmark scenario types. At least one must be specified. Currently supported: - `fixed-seq-len`: Fixed input/output sequence length benchmarks. Each entry must have: - `isl`: An integer representing the input sequence length, e.g., `1024` diff --git a/utils/agentic/aggregation/process_agentic_result.py b/utils/agentic/aggregation/process_agentic_result.py index 83211f4f33..d87f452a86 100644 --- a/utils/agentic/aggregation/process_agentic_result.py +++ b/utils/agentic/aggregation/process_agentic_result.py @@ -67,6 +67,12 @@ def _gpu_shape() -> tuple[dict[str, Any], int, int, int, str]: decode_tp = env_int("DECODE_TP") decode_ep = env_int("DECODE_EP", 1) decode_dp_attention = os.environ.get("DECODE_DP_ATTN", "false") + prefill_hardware = os.environ.get("PREFILL_HARDWARE", "") + decode_hardware = os.environ.get("DECODE_HARDWARE", "") + if bool(prefill_hardware) != bool(decode_hardware): + raise SystemExit( + "PREFILL_HARDWARE and DECODE_HARDWARE must be specified together." + ) num_prefill_gpu = prefill_num_workers * prefill_tp num_decode_gpu = decode_num_workers * decode_tp num_gpus = num_prefill_gpu + num_decode_gpu @@ -91,6 +97,9 @@ def _gpu_shape() -> tuple[dict[str, Any], int, int, int, str]: "num_decode_gpu": num_decode_gpu, } ) + if prefill_hardware: + fields["prefill_hw"] = prefill_hardware + fields["decode_hw"] = decode_hardware return fields, num_gpus, tp, ep, dp_attention diff --git a/utils/agentic/aggregation/test_process_agentic_result.py b/utils/agentic/aggregation/test_process_agentic_result.py index a2bac8ea52..37a10ffc1e 100644 --- a/utils/agentic/aggregation/test_process_agentic_result.py +++ b/utils/agentic/aggregation/test_process_agentic_result.py @@ -26,6 +26,7 @@ load_aggregate, load_records, ) +from utils.agentic.aggregation.process_agentic_result import _gpu_shape from utils.agentic.aggregation.server_metrics import ( compute_server_metrics, load_server_metrics, @@ -308,6 +309,8 @@ def _run_processor( env_overrides: dict[str, str] | None = None, ) -> dict: env = os.environ.copy() + env.pop("PREFILL_HARDWARE", None) + env.pop("DECODE_HARDWARE", None) env.update( { "RESULT_DIR": str(result_dir), @@ -453,6 +456,70 @@ def test_processor_surfaces_allocated_cpu_dram(tmp_path: Path): assert agg["allocated_cpu_dram_gb"] == 2400 +def test_multinode_processor_surfaces_heterogeneous_hardware(tmp_path: Path): + result_dir = _write_fixture(tmp_path) + agg = _run_processor( + result_dir, + tmp_path / "out", + env_overrides={ + "IS_MULTINODE": "true", + "DISAGG": "true", + "PREFILL_NUM_WORKERS": "1", + "PREFILL_TP": "8", + "PREFILL_EP": "8", + "PREFILL_DP_ATTN": "false", + "PREFILL_HARDWARE": "b200", + "DECODE_NUM_WORKERS": "2", + "DECODE_TP": "8", + "DECODE_EP": "8", + "DECODE_DP_ATTN": "false", + "DECODE_HARDWARE": "h100", + }, + ) + + assert agg["prefill_hw"] == "b200" + assert agg["decode_hw"] == "h100" + + +def test_multinode_processor_omits_homogeneous_hardware(tmp_path: Path): + result_dir = _write_fixture(tmp_path) + agg = _run_processor( + result_dir, + tmp_path / "out", + env_overrides={ + "IS_MULTINODE": "true", + "DISAGG": "true", + "PREFILL_NUM_WORKERS": "1", + "PREFILL_TP": "8", + "DECODE_NUM_WORKERS": "2", + "DECODE_TP": "8", + }, + ) + + assert "prefill_hw" not in agg + assert "decode_hw" not in agg + + +@pytest.mark.parametrize( + ("present_var", "missing_var"), + [ + ("PREFILL_HARDWARE", "DECODE_HARDWARE"), + ("DECODE_HARDWARE", "PREFILL_HARDWARE"), + ], +) +def test_multinode_processor_rejects_one_sided_hardware( + monkeypatch: pytest.MonkeyPatch, + present_var: str, + missing_var: str, +): + monkeypatch.setenv("IS_MULTINODE", "true") + monkeypatch.setenv(present_var, "b200") + monkeypatch.delenv(missing_var, raising=False) + + with pytest.raises(SystemExit, match="must be specified together"): + _gpu_shape() + + def test_processor_surfaces_request_accounting(tmp_path: Path): result_dir = tmp_path / "results" artifact = result_dir / "aiperf_artifacts" diff --git a/utils/matrix_logic/test_generate_sweep_configs.py b/utils/matrix_logic/test_generate_sweep_configs.py index 0e2c64d95e..5a78dedfef 100644 --- a/utils/matrix_logic/test_generate_sweep_configs.py +++ b/utils/matrix_logic/test_generate_sweep_configs.py @@ -78,6 +78,7 @@ def sample_multinode_config(): { "conc-list": [2150], "prefill": { + "hardware": "gb200", "num-worker": 5, "tp": 4, "ep": 4, @@ -88,6 +89,7 @@ def sample_multinode_config(): ], }, "decode": { + "hardware": "h100", "num-worker": 1, "tp": 8, "ep": 8, @@ -981,6 +983,8 @@ def test_multinode_entry_structure(self, sample_multinode_config, sample_runner_ assert entry["prefill"]["num-worker"] == 5 assert entry["decode"]["num-worker"] == 1 assert entry["disagg"] is True + assert entry["prefill"]["hardware"] == "gb200" + assert entry["decode"]["hardware"] == "h100" def test_multinode_conc_as_list(self, sample_multinode_config, sample_runner_config, full_sweep_args_multi_node): """Multinode conc should be passed as list.""" @@ -2004,8 +2008,8 @@ def test_multinode_agentic_groups_concurrencies_per_search_entry(self): "search-space": [ { "conc-list": [16, 32, 64, 128, 256], - "prefill": {"num-worker": 2, "tp": 8, "ep": 8, "dp-attn": False}, - "decode": {"num-worker": 1, "tp": 8, "ep": 1, "dp-attn": False}, + "prefill": {"hardware": "gb200", "num-worker": 2, "tp": 8, "ep": 8, "dp-attn": False}, + "decode": {"hardware": "h100", "num-worker": 1, "tp": 8, "ep": 1, "dp-attn": False}, } ], } @@ -2151,8 +2155,8 @@ def test_node_type_filters_apply_to_agentic_configs( "search-space": [ { "conc-list": [16, 32], - "prefill": {"num-worker": 2, "tp": 8, "ep": 8, "dp-attn": False}, - "decode": {"num-worker": 1, "tp": 8, "ep": 1, "dp-attn": False}, + "prefill": {"hardware": "gb200", "num-worker": 2, "tp": 8, "ep": 8, "dp-attn": False}, + "decode": {"hardware": "h100", "num-worker": 1, "tp": 8, "ep": 1, "dp-attn": False}, }, ], }], diff --git a/utils/matrix_logic/test_validation.py b/utils/matrix_logic/test_validation.py index 7e794331f5..b60c99f6f3 100644 --- a/utils/matrix_logic/test_validation.py +++ b/utils/matrix_logic/test_validation.py @@ -65,6 +65,7 @@ def valid_multinode_matrix_entry(): "isl": 1024, "osl": 1024, "prefill": { + "hardware": "gb200", "num-worker": 5, "tp": 4, "ep": 4, @@ -75,6 +76,7 @@ def valid_multinode_matrix_entry(): ], }, "decode": { + "hardware": "h100", "num-worker": 1, "tp": 8, "ep": 8, @@ -141,6 +143,7 @@ def valid_multinode_master_config(): "search-space": [ { "prefill": { + "hardware": "gb200", "num-worker": 5, "tp": 4, "ep": 4, @@ -151,6 +154,7 @@ def valid_multinode_master_config(): ], }, "decode": { + "hardware": "h100", "num-worker": 1, "tp": 8, "ep": 8, @@ -213,6 +217,7 @@ def test_key_fields_exist(self): assert Fields.SPEC_DECODING.value == "spec-decoding" assert Fields.PREFILL.value == "prefill" assert Fields.DECODE.value == "decode" + assert Fields.HARDWARE.value == "hardware" # ============================================================================= @@ -318,6 +323,12 @@ def test_extra_field_forbidden(self, valid_single_node_matrix_entry): with pytest.raises(Exception): SingleNodeMatrixEntry(**valid_single_node_matrix_entry) + def test_disagg_requires_multinode(self, valid_single_node_matrix_entry): + """Single-node matrix entries cannot enable disaggregation.""" + valid_single_node_matrix_entry["disagg"] = True + with pytest.raises(Exception, match="disagg"): + SingleNodeMatrixEntry(**valid_single_node_matrix_entry) + # ============================================================================= # Test Agentic Matrix Entries @@ -466,6 +477,25 @@ def test_valid_entry(self, valid_multinode_matrix_entry): assert entry.model == "deepseek-r1-fp4" assert entry.conc == [2150] assert entry.disagg is True + assert entry.prefill.hardware == "gb200" + assert entry.decode.hardware == "h100" + + def test_disagg_allows_omitted_hardware(self, valid_multinode_matrix_entry): + """Homogeneous disaggregated entries may omit hardware metadata.""" + del valid_multinode_matrix_entry["prefill"]["hardware"] + del valid_multinode_matrix_entry["decode"]["hardware"] + entry = MultiNodeMatrixEntry(**valid_multinode_matrix_entry) + assert entry.prefill.hardware is None + assert entry.decode.hardware is None + + @pytest.mark.parametrize("missing_worker", ["prefill", "decode"]) + def test_hardware_requires_prefill_and_decode( + self, valid_multinode_matrix_entry, missing_worker + ): + """Heterogeneous hardware metadata must identify both worker pools.""" + del valid_multinode_matrix_entry[missing_worker]["hardware"] + with pytest.raises(Exception, match="both.*prefill.*decode"): + MultiNodeMatrixEntry(**valid_multinode_matrix_entry) def test_prefill_decode_worker_configs(self, valid_multinode_matrix_entry): """Prefill and decode should be WorkerConfig objects.""" @@ -817,6 +847,26 @@ def test_multinode_master_config(self, valid_multinode_master_config): assert config.model_prefix == "dsr1" assert config.runner == "gb200" assert config.disagg is True + search_entry = config.scenarios.fixed_seq_len[0].search_space[0] + assert search_entry.prefill.hardware == "gb200" + assert search_entry.decode.hardware == "h100" + + def test_disagg_master_config_allows_omitted_hardware(self, valid_multinode_master_config): + """Homogeneous disaggregated master configs may omit hardware metadata.""" + search_entry = valid_multinode_master_config["scenarios"]["fixed-seq-len"][0]["search-space"][0] + del search_entry["prefill"]["hardware"] + del search_entry["decode"]["hardware"] + config = MultiNodeMasterConfigEntry(**valid_multinode_master_config) + validated_entry = config.scenarios.fixed_seq_len[0].search_space[0] + assert validated_entry.prefill.hardware is None + assert validated_entry.decode.hardware is None + + def test_master_hardware_requires_prefill_and_decode(self, valid_multinode_master_config): + """Heterogeneous master configs must identify both worker pools.""" + search_entry = valid_multinode_master_config["scenarios"]["fixed-seq-len"][0]["search-space"][0] + del search_entry["decode"]["hardware"] + with pytest.raises(Exception, match="both.*prefill.*decode"): + MultiNodeMasterConfigEntry(**valid_multinode_master_config) def test_single_node_cannot_have_multinode_true(self, valid_single_node_master_config): """Single node config must have multinode=False.""" @@ -835,6 +885,12 @@ def test_disagg_default_false(self, valid_single_node_master_config): config = SingleNodeMasterConfigEntry(**valid_single_node_master_config) assert config.disagg is False + def test_disagg_requires_multinode(self, valid_single_node_master_config): + """Single-node master configs cannot enable disaggregation.""" + valid_single_node_master_config["disagg"] = True + with pytest.raises(Exception, match="disagg"): + SingleNodeMasterConfigEntry(**valid_single_node_master_config) + def test_single_node_agentic_master_config_requires_cluster_runner(self): """Single-node agentic configs must pin an exact cluster label.""" config = { @@ -881,12 +937,14 @@ def test_multinode_agentic_master_config_requires_cluster_runner(self): "spec-decoding": "none", "conc-list": [1], "prefill": { + "hardware": "b200", "num-worker": 1, "tp": 4, "ep": 4, "dp-attn": True, }, "decode": { + "hardware": "b200", "num-worker": 1, "tp": 8, "ep": 8, diff --git a/utils/matrix_logic/validation.py b/utils/matrix_logic/validation.py index 1f4986272a..3de1f4d606 100644 --- a/utils/matrix_logic/validation.py +++ b/utils/matrix_logic/validation.py @@ -23,6 +23,7 @@ class Fields(Enum): PRECISION = 'precision' FRAMEWORK = 'framework' RUNNER = 'runner' + HARDWARE = 'hardware' SCENARIOS = 'scenarios' MULTINODE = 'multinode' @@ -107,7 +108,7 @@ class SingleNodeMatrixEntry(BaseModel): conc: Union[int, List[int]] max_model_len: int = Field(alias=Fields.MAX_MODEL_LEN.value) exp_name: str = Field(alias=Fields.EXP_NAME.value) - disagg: bool + disagg: Literal[False] run_eval: bool = Field(alias=Fields.RUN_EVAL.value) eval_only: bool = Field(alias=Fields.EVAL_ONLY.value, default=False) @@ -120,10 +121,21 @@ class WorkerConfig(BaseModel): tp: int ep: int dp_attn: bool = Field(alias=Fields.DP_ATTN.value) + hardware: Optional[str] = Field(default=None, min_length=1) additional_settings: Optional[List[str]] = Field( default=[], alias=Fields.ADDITIONAL_SETTINGS.value) +def _validate_worker_hardware_pair(self): + """Require prefill and decode workers to declare hardware together.""" + if bool(self.prefill.hardware) != bool(self.decode.hardware): + raise ValueError( + f"'{Fields.HARDWARE.value}' must be specified for both " + f"'{Fields.PREFILL.value}' and '{Fields.DECODE.value}', or neither" + ) + return self + + class MultiNodeMatrixEntry(BaseModel): """Pydantic model for validating multinode matrix entry structure. This validates the input that should be expected to .github/workflows/benchmark-multinode-tmpl.yml""" @@ -153,6 +165,10 @@ class MultiNodeMatrixEntry(BaseModel): default=False, alias=Fields.EVAL_ALL_CONCS.value ) + @model_validator(mode='after') + def validate_worker_hardware_pair(self): + return _validate_worker_hardware_pair(self) + class SingleNodeAgenticMatrixEntry(BaseModel): """Pydantic model for validating single-node agentic coding matrix entries.""" @@ -206,6 +222,10 @@ class MultiNodeAgenticMatrixEntry(BaseModel): disagg: bool scenario_type: str = Field(alias=Fields.SCENARIO_TYPE.value) + @model_validator(mode='after') + def validate_worker_hardware_pair(self): + return _validate_worker_hardware_pair(self) + AgenticMatrixEntry = Union[SingleNodeAgenticMatrixEntry, MultiNodeAgenticMatrixEntry] @@ -367,6 +387,10 @@ class MultiNodeSearchSpaceEntry(BaseModel): def validate_conc_fields(self): return _validate_conc_fields(self) + @model_validator(mode='after') + def validate_worker_hardware_pair(self): + return _validate_worker_hardware_pair(self) + class SingleNodeSeqLenConfig(BaseModel): """Single node sequence length configuration.""" @@ -433,6 +457,8 @@ def validate_topology_fields(self): f"Single-node agentic search-space entries must specify " f"{Fields.KV_OFFLOADING.value}" ) + if has_complete_multinode: + _validate_worker_hardware_pair(self) return self class AgenticCodingConfig(BaseModel): @@ -500,7 +526,7 @@ class SingleNodeMasterConfigEntry(BaseModel): framework: str runner: str multinode: Literal[False] - disagg: bool = Field(default=False) + disagg: Literal[False] = Field(default=False) scenarios: SingleNodeScenarios @model_validator(mode='after') diff --git a/utils/process_result.py b/utils/process_result.py index 5fb0594739..c4801edec1 100644 --- a/utils/process_result.py +++ b/utils/process_result.py @@ -62,8 +62,16 @@ def get_required_env_vars(required_vars): # TODO: Eventually will have to have a separate condition in here for multinode disagg and # multinode agg. For now, just assume that multinode implies disagg. - multinode_env = get_required_env_vars(['PREFILL_GPUS', 'DECODE_GPUS', 'PREFILL_NUM_WORKERS', 'PREFILL_TP', - 'PREFILL_EP', 'PREFILL_DP_ATTN', 'DECODE_NUM_WORKERS', 'DECODE_TP', 'DECODE_EP', 'DECODE_DP_ATTN']) + multinode_vars = ['PREFILL_GPUS', 'DECODE_GPUS', 'PREFILL_NUM_WORKERS', 'PREFILL_TP', + 'PREFILL_EP', 'PREFILL_DP_ATTN', 'DECODE_NUM_WORKERS', 'DECODE_TP', + 'DECODE_EP', 'DECODE_DP_ATTN'] + multinode_env = get_required_env_vars(multinode_vars) + prefill_hardware = os.environ.get('PREFILL_HARDWARE', '') + decode_hardware = os.environ.get('DECODE_HARDWARE', '') + if bool(prefill_hardware) != bool(decode_hardware): + raise ValueError( + "PREFILL_HARDWARE and DECODE_HARDWARE must be specified together." + ) prefill_gpus = int(multinode_env['PREFILL_GPUS']) decode_gpus = int(multinode_env['DECODE_GPUS']) prefill_num_workers = int(multinode_env['PREFILL_NUM_WORKERS']) @@ -101,6 +109,9 @@ def get_required_env_vars(required_vars): 'output_tput_per_gpu': float(bmk_result['output_throughput']) / output_tput_denominator, 'input_tput_per_gpu': (float(bmk_result['total_token_throughput']) - float(bmk_result['output_throughput'])) / prefill_gpus, } + if prefill_hardware: + multi_node_data['prefill_hw'] = prefill_hardware + multi_node_data['decode_hw'] = decode_hardware data = data | multi_node_data else: diff --git a/utils/test_process_result.py b/utils/test_process_result.py index 4037689eaf..127e2512f4 100644 --- a/utils/test_process_result.py +++ b/utils/test_process_result.py @@ -82,6 +82,8 @@ def multinode_env_vars(base_env_vars): "DECODE_TP": "8", "DECODE_EP": "8", "DECODE_DP_ATTN": "true", + "PREFILL_HARDWARE": "gb200", + "DECODE_HARDWARE": "h100", } @@ -232,6 +234,8 @@ def test_multinode_processing(self, tmp_path, sample_benchmark_result, multinode assert output_data["decode_num_workers"] == 1 assert output_data["num_prefill_gpu"] == 20 assert output_data["num_decode_gpu"] == 8 + assert output_data["prefill_hw"] == "gb200" + assert output_data["decode_hw"] == "h100" # Verify throughput calculations total_gpus = 20 + 8 # prefill + decode @@ -239,6 +243,32 @@ def test_multinode_processing(self, tmp_path, sample_benchmark_result, multinode assert output_data["output_tput_per_gpu"] == pytest.approx(12000.0 / 8) # decode gpus assert output_data["input_tput_per_gpu"] == pytest.approx((15000.5 - 12000.0) / 20) # prefill gpus + def test_homogeneous_multinode_omits_hardware_fields( + self, tmp_path, sample_benchmark_result, multinode_env_vars + ): + """Absent hardware metadata should preserve homogeneous result output.""" + multinode_env_vars.pop("PREFILL_HARDWARE") + multinode_env_vars.pop("DECODE_HARDWARE") + + result = run_script(tmp_path, multinode_env_vars, sample_benchmark_result) + + assert result.returncode == 0, f"Script failed: {result.stderr}" + output_data = json.loads(result.stdout) + assert "prefill_hw" not in output_data + assert "decode_hw" not in output_data + + @pytest.mark.parametrize("missing_var", ["PREFILL_HARDWARE", "DECODE_HARDWARE"]) + def test_partial_hardware_metadata_fails( + self, tmp_path, sample_benchmark_result, multinode_env_vars, missing_var + ): + """Prefill and decode hardware must always be provided together.""" + multinode_env_vars.pop(missing_var) + + result = run_script(tmp_path, multinode_env_vars, sample_benchmark_result) + + assert result.returncode != 0 + assert "PREFILL_HARDWARE and DECODE_HARDWARE" in result.stderr + def test_missing_base_env_vars(self, tmp_path, sample_benchmark_result): """Test that missing base env vars causes failure.""" result_file = tmp_path / "benchmark_result.json"