diff --git a/.github/AGENT_OPERATIONS.md b/.github/AGENT_OPERATIONS.md index 1652ac87fb..0ef9acd1ea 100644 --- a/.github/AGENT_OPERATIONS.md +++ b/.github/AGENT_OPERATIONS.md @@ -92,7 +92,7 @@ For multi-node configurations, `--all-evals` creates one eval job per engine top Single-node fixed-sequence results may include `power_valid`, `avg_power_w`, `avg_total_gpu_power_w`, `total_gpu_energy_j`, and joules per query/input/output/total token. Invalid telemetry records `power_valid: 0` without energy metrics and fails only with `REQUIRE_POWER=1`. -Multinode disaggregated results add `prefill_gpu_energy_j`, `decode_gpu_energy_j`, `prefill_joules_per_input_token`, and `decode_joules_per_output_token`. Role energy covers the full formal benchmark window, not kernel-level phases. +Multinode disaggregated results add `prefill_gpu_energy_j`, `decode_gpu_energy_j`, `prefill_avg_power_w`, `decode_avg_power_w`, `prefill_joules_per_input_token`, and `decode_joules_per_output_token`. Role energy covers the full formal benchmark window, not kernel-level phases, and the role watts are that energy divided by the same window and by the role's GPU count. For srt-slurm recipes, `telemetry: {provider: dcgm-power}` enables official energy collection. `runners/launch_gb200-nv.sh` and `runners/launch_gb300-nv.sh` are the source of truth for `POWER_SRT_SLURM_PIN`. CI derives `POWER_PRODUCER_SHA` from the launcher stamp. `utils/test_gb200_power_official_contract.py` and `utils/test_gb300_power_official_contract.py` enforce the recipe/launcher contract. Only `PRECISION=fp8` dcgm-power lanes are validated. diff --git a/utils/aggregate_power_multinode.py b/utils/aggregate_power_multinode.py index 7460eecd20..cabed59d6d 100644 --- a/utils/aggregate_power_multinode.py +++ b/utils/aggregate_power_multinode.py @@ -15,6 +15,9 @@ Role energy semantics: ``prefill_gpu_energy_j`` / ``decode_gpu_energy_j`` are the board-level energy of that role's GPUs integrated over the FULL formal serving window. They are not kernel-level prefill/decode phase energies. +``prefill_avg_power_w`` / ``decode_avg_power_w`` divide that role energy by the +same full window and by the role's GPU count, so they are the mean board draw +of that role's GPUs across the whole serving window, not a phase power. Ordinary benchmark runs are best-effort: invalid telemetry records ``power_valid=0`` (and no energy metrics) in the aggregate plus a validation @@ -108,6 +111,8 @@ ROLE_METRIC_KEYS = ( "prefill_gpu_energy_j", "decode_gpu_energy_j", + "prefill_avg_power_w", + "decode_avg_power_w", "prefill_joules_per_input_token", "decode_joules_per_output_token", ) @@ -1062,11 +1067,13 @@ def validate_and_integrate( } if prefill_gpus > 0: metrics["prefill_gpu_energy_j"] = role_energy["prefill"] + metrics["prefill_avg_power_w"] = role_energy["prefill"] / duration_s / prefill_gpus metrics["prefill_joules_per_input_token"] = ( role_energy["prefill"] / benchmark.total_input_tokens ) if decode_gpus > 0: metrics["decode_gpu_energy_j"] = role_energy["decode"] + metrics["decode_avg_power_w"] = role_energy["decode"] / duration_s / decode_gpus metrics["decode_joules_per_output_token"] = ( role_energy["decode"] / benchmark.total_output_tokens ) diff --git a/utils/process_result.py b/utils/process_result.py index 79e378cd09..a8bdc8cca6 100644 --- a/utils/process_result.py +++ b/utils/process_result.py @@ -46,6 +46,8 @@ def get_optional_component_metadata(env_var): _MULTINODE_ROLE_METRIC_KEYS = ( "prefill_gpu_energy_j", "decode_gpu_energy_j", + "prefill_avg_power_w", + "decode_avg_power_w", "prefill_joules_per_input_token", "decode_joules_per_output_token", ) diff --git a/utils/test_aggregate_power_multinode.py b/utils/test_aggregate_power_multinode.py index fdc2796430..e4c78a1156 100644 --- a/utils/test_aggregate_power_multinode.py +++ b/utils/test_aggregate_power_multinode.py @@ -239,6 +239,8 @@ def test_emits_all_metrics_exactly(self, tmp_path): assert agg["joules_per_total_token"] == round(84000 / 36864, 6) assert agg["prefill_gpu_energy_j"] == 48000.0 assert agg["decode_gpu_energy_j"] == 36000.0 + assert agg["prefill_avg_power_w"] == 400.0 + assert agg["decode_avg_power_w"] == 300.0 assert agg["prefill_joules_per_input_token"] == round(48000 / 32768, 6) assert agg["decode_joules_per_output_token"] == round(36000 / 4096, 6) @@ -258,6 +260,23 @@ def test_emits_all_metrics_exactly(self, tmp_path): } assert set(sidecar["per_gpu_energy_j"]) == set(sidecar["per_gpu_role"]) + def test_role_watts_close_over_the_whole_deployment(self, tmp_path): + """Role watts weighted by their GPU counts must reproduce the whole- + deployment watts -- the topology gate makes the role partition + exhaustive. Catches dividing by the total device count.""" + + def ramp(host, idx, ts): + if (host, idx) == ("node-d", 0): + return 300.0 + (ts - FIRST_TS) + return dict(((h, i), w) for h, i, _r, _g, w in DEVICES)[(host, idx)] + + pkg = build_package(tmp_path, power_fn=ramp) + assert pkg.run() == 0 + agg = pkg.agg() + weighted = 2 * agg["prefill_avg_power_w"] + 2 * agg["decode_avg_power_w"] + assert weighted == pytest.approx(agg["avg_total_gpu_power_w"], abs=1e-3) + assert weighted / 4 == pytest.approx(agg["avg_power_w"], abs=1e-3) + def test_strict_mode_passes_on_valid_package(self, tmp_path): pkg = build_package(tmp_path) assert pkg.run(require_power=True) == 0 @@ -444,6 +463,8 @@ def test_none_het_groups_valid_for_non_het_deployments(self, tmp_path): assert agg["power_valid"] == 1 assert agg["prefill_gpu_energy_j"] == 48000.0 assert agg["decode_gpu_energy_j"] == 36000.0 + assert agg["prefill_avg_power_w"] == 400.0 + assert agg["decode_avg_power_w"] == 300.0 def test_mixed_none_and_real_het_groups_rejected(self, tmp_path): pkg = build_package(tmp_path) diff --git a/utils/test_process_result.py b/utils/test_process_result.py index 1142a117f8..4d5219010f 100644 --- a/utils/test_process_result.py +++ b/utils/test_process_result.py @@ -1224,6 +1224,8 @@ def test_valid_package_patches_role_energy(self, tmp_path, power_env): assert agg["power_valid"] == 1 assert agg["prefill_gpu_energy_j"] == 48000.0 assert agg["decode_gpu_energy_j"] == 36000.0 + assert agg["prefill_avg_power_w"] == 400.0 + assert agg["decode_avg_power_w"] == 300.0 assert (tmp_path / "power_validation_benchmark_result.json").is_file() def test_missing_package_is_best_effort(self, tmp_path, power_env):