Emit per-role average power on multinode disaggregated runs / 多节点分离式运行输出每角色平均功率

.github/AGENT_OPERATIONS.md

@@ -92,7 +92,7 @@ For multi-node configurations, `--all-evals` creates one eval job per engine top
 
 Single-node fixed-sequence results may include `power_valid`, `avg_power_w`, `avg_total_gpu_power_w`, `total_gpu_energy_j`, and joules per query/input/output/total token. Invalid telemetry records `power_valid: 0` without energy metrics and fails only with `REQUIRE_POWER=1`.
 
-Multinode disaggregated results add `prefill_gpu_energy_j`, `decode_gpu_energy_j`, `prefill_joules_per_input_token`, and `decode_joules_per_output_token`. Role energy covers the full formal benchmark window, not kernel-level phases.
+Multinode disaggregated results add `prefill_gpu_energy_j`, `decode_gpu_energy_j`, `prefill_avg_power_w`, `decode_avg_power_w`, `prefill_joules_per_input_token`, and `decode_joules_per_output_token`. Role energy covers the full formal benchmark window, not kernel-level phases, and the role watts are that energy divided by the same window and by the role's GPU count.
 
 For srt-slurm recipes, `telemetry: {provider: dcgm-power}` enables official energy collection. `runners/launch_gb200-nv.sh` and `runners/launch_gb300-nv.sh` are the source of truth for `POWER_SRT_SLURM_PIN`. CI derives `POWER_PRODUCER_SHA` from the launcher stamp. `utils/test_gb200_power_official_contract.py` and `utils/test_gb300_power_official_contract.py` enforce the recipe/launcher contract. Only `PRECISION=fp8` dcgm-power lanes are validated.
 

utils/aggregate_power_multinode.py

@@ -15,6 +15,9 @@
 Role energy semantics: ``prefill_gpu_energy_j`` / ``decode_gpu_energy_j`` are
 the board-level energy of that role's GPUs integrated over the FULL formal
 serving window. They are not kernel-level prefill/decode phase energies.
+``prefill_avg_power_w`` / ``decode_avg_power_w`` divide that role energy by the
+same full window and by the role's GPU count, so they are the mean board draw
+of that role's GPUs across the whole serving window, not a phase power.
 
 Ordinary benchmark runs are best-effort: invalid telemetry records
 ``power_valid=0`` (and no energy metrics) in the aggregate plus a validation
@@ -108,6 +111,8 @@
 ROLE_METRIC_KEYS = (
     "prefill_gpu_energy_j",
     "decode_gpu_energy_j",
+    "prefill_avg_power_w",
+    "decode_avg_power_w",
     "prefill_joules_per_input_token",
     "decode_joules_per_output_token",
 )
@@ -1062,11 +1067,13 @@ def validate_and_integrate(
     }
     if prefill_gpus > 0:
         metrics["prefill_gpu_energy_j"] = role_energy["prefill"]
+        metrics["prefill_avg_power_w"] = role_energy["prefill"] / duration_s / prefill_gpus
         metrics["prefill_joules_per_input_token"] = (
             role_energy["prefill"] / benchmark.total_input_tokens
         )
     if decode_gpus > 0:
         metrics["decode_gpu_energy_j"] = role_energy["decode"]
+        metrics["decode_avg_power_w"] = role_energy["decode"] / duration_s / decode_gpus
         metrics["decode_joules_per_output_token"] = (
             role_energy["decode"] / benchmark.total_output_tokens
         )

utils/process_result.py

@@ -46,6 +46,8 @@ def get_optional_component_metadata(env_var):
 _MULTINODE_ROLE_METRIC_KEYS = (
     "prefill_gpu_energy_j",
     "decode_gpu_energy_j",
+    "prefill_avg_power_w",
+    "decode_avg_power_w",
     "prefill_joules_per_input_token",
     "decode_joules_per_output_token",
 )

utils/test_aggregate_power_multinode.py

@@ -239,6 +239,8 @@ def test_emits_all_metrics_exactly(self, tmp_path):
         assert agg["joules_per_total_token"] == round(84000 / 36864, 6)
         assert agg["prefill_gpu_energy_j"] == 48000.0
         assert agg["decode_gpu_energy_j"] == 36000.0
+        assert agg["prefill_avg_power_w"] == 400.0
+        assert agg["decode_avg_power_w"] == 300.0
         assert agg["prefill_joules_per_input_token"] == round(48000 / 32768, 6)
         assert agg["decode_joules_per_output_token"] == round(36000 / 4096, 6)
 
@@ -258,6 +260,23 @@ def test_emits_all_metrics_exactly(self, tmp_path):
         }
         assert set(sidecar["per_gpu_energy_j"]) == set(sidecar["per_gpu_role"])
 
+    def test_role_watts_close_over_the_whole_deployment(self, tmp_path):
+        """Role watts weighted by their GPU counts must reproduce the whole-
+        deployment watts -- the topology gate makes the role partition
+        exhaustive. Catches dividing by the total device count."""
+
+        def ramp(host, idx, ts):
+            if (host, idx) == ("node-d", 0):
+                return 300.0 + (ts - FIRST_TS)
+            return dict(((h, i), w) for h, i, _r, _g, w in DEVICES)[(host, idx)]
+
+        pkg = build_package(tmp_path, power_fn=ramp)
+        assert pkg.run() == 0
+        agg = pkg.agg()
+        weighted = 2 * agg["prefill_avg_power_w"] + 2 * agg["decode_avg_power_w"]
+        assert weighted == pytest.approx(agg["avg_total_gpu_power_w"], abs=1e-3)
+        assert weighted / 4 == pytest.approx(agg["avg_power_w"], abs=1e-3)
+
     def test_strict_mode_passes_on_valid_package(self, tmp_path):
         pkg = build_package(tmp_path)
         assert pkg.run(require_power=True) == 0
@@ -444,6 +463,8 @@ def test_none_het_groups_valid_for_non_het_deployments(self, tmp_path):
         assert agg["power_valid"] == 1
         assert agg["prefill_gpu_energy_j"] == 48000.0
         assert agg["decode_gpu_energy_j"] == 36000.0
+        assert agg["prefill_avg_power_w"] == 400.0
+        assert agg["decode_avg_power_w"] == 300.0
 
     def test_mixed_none_and_real_het_groups_rejected(self, tmp_path):
         pkg = build_package(tmp_path)

utils/test_process_result.py

@@ -1224,6 +1224,8 @@ def test_valid_package_patches_role_energy(self, tmp_path, power_env):
         assert agg["power_valid"] == 1
         assert agg["prefill_gpu_energy_j"] == 48000.0
         assert agg["decode_gpu_energy_j"] == 36000.0
+        assert agg["prefill_avg_power_w"] == 400.0
+        assert agg["decode_avg_power_w"] == 300.0
         assert (tmp_path / "power_validation_benchmark_result.json").is_file()
 
     def test_missing_package_is_best_effort(self, tmp_path, power_env):