[Power] feat: extend dcgm energy lanes to gb dsv4/qwen3.5 fp4 / 扩展 dcgm 能耗采集到 gb dsv4 与 qwen3.5 fp4

benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-1p1d-dep8-dep16-6-c512.yaml

@@ -154,3 +154,16 @@ benchmark:
   concurrencies: "512"
   req_rate: "inf"
   use_chat_template: false
+
+telemetry:
+  enabled: true
+  provider: dcgm-power
+  default_frequency: 1.0
+  storage_subdir: power
+  required: true
+  startup_timeout_seconds: 120
+  request_timeout_seconds: 2
+  collector_join_timeout_seconds: 10
+  dcgm_exporter:
+    container_image: dcgm-exporter
+    port: 9401

benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-1p1d-tp8-tp8-4-c1.yaml

@@ -115,3 +115,16 @@ benchmark:
   concurrencies: "1"
   req_rate: "inf"
   use_chat_template: false
+
+telemetry:
+  enabled: true
+  provider: dcgm-power
+  default_frequency: 1.0
+  storage_subdir: power
+  required: true
+  startup_timeout_seconds: 120
+  request_timeout_seconds: 2
+  collector_join_timeout_seconds: 10
+  dcgm_exporter:
+    container_image: dcgm-exporter
+    port: 9401

benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-1p2d-dep8-dep16-10-c256.yaml

@@ -154,3 +154,16 @@ benchmark:
   concurrencies: "256"
   req_rate: "inf"
   use_chat_template: false
+
+telemetry:
+  enabled: true
+  provider: dcgm-power
+  default_frequency: 1.0
+  storage_subdir: power
+  required: true
+  startup_timeout_seconds: 120
+  request_timeout_seconds: 2
+  collector_join_timeout_seconds: 10
+  dcgm_exporter:
+    container_image: dcgm-exporter
+    port: 9401

benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-1p1d-dep4-dep16-5-c1024.yaml

@@ -180,3 +180,18 @@ benchmark:
   req_rate: "inf"
   use_chat_template: false
   custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer"
+
+telemetry:
+  enabled: true
+  provider: dcgm-power
+  default_frequency: 1.0
+  storage_subdir: power
+  required: true
+  startup_timeout_seconds: 120
+  request_timeout_seconds: 2
+  collector_join_timeout_seconds: 10
+  dcgm_exporter:
+    container_image: dcgm-exporter
+    # 9401 is already bound by the cluster-level exporter on im-gb300 nodes;
+    # use a port outside that range.
+    port: 19401

benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-1p1d-tp4-tp4-2-c1.yaml

@@ -163,3 +163,18 @@ benchmark:
   req_rate: "inf"
   use_chat_template: false
   custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer"
+
+telemetry:
+  enabled: true
+  provider: dcgm-power
+  default_frequency: 1.0
+  storage_subdir: power
+  required: true
+  startup_timeout_seconds: 120
+  request_timeout_seconds: 2
+  collector_join_timeout_seconds: 10
+  dcgm_exporter:
+    container_image: dcgm-exporter
+    # 9401 is already bound by the cluster-level exporter on im-gb300 nodes;
+    # use a port outside that range.
+    port: 19401

benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_0.yaml

@@ -173,3 +173,18 @@ benchmark:
   concurrencies: "1x4x8x16x32x64x256"
   req_rate: "inf"
   random_range_ratio: 0.8
+
+telemetry:
+  enabled: true
+  provider: dcgm-power
+  default_frequency: 1.0
+  storage_subdir: power
+  required: true
+  startup_timeout_seconds: 120
+  request_timeout_seconds: 2
+  collector_join_timeout_seconds: 10
+  dcgm_exporter:
+    container_image: dcgm-exporter
+    # 9401 is already bound by the cluster-level exporter on im-gb300 nodes;
+    # use a port outside that range.
+    port: 19401

runners/launch_gb200-nv.sh

@@ -305,12 +305,13 @@
 ' "$_RECIPE_SRC"; then
     USES_DCGM_POWER=1
 fi
 
-# Note (wenyao): the producer pin descends from the fp8 srt-slurm lineage
-# (cargo/maturin bootstrap); a non-fp8 power recipe would silently clone the
-# wrong lineage, so fail fast instead.
-if [[ "$USES_DCGM_POWER" == "1" && "$PRECISION" != "fp8" ]]; then
-    echo "Error: dcgm-power lanes are only validated for PRECISION=fp8, got: $PRECISION" >&2
+# Note (wenyao): the producer pin follows the srt-slurm main lineage that the
+# dynamo-sglang lanes run on (fp8 validated end-to-end, fp4 recipes
+# parse-verified against the pin); other frameworks clone diverging refs
+# (aflowers branch, sa-submission), so fail fast for them instead.
+if [[ "$USES_DCGM_POWER" == "1" && "$FRAMEWORK" != "dynamo-sglang" ]]; then
+    echo "Error: dcgm-power lanes are only validated for FRAMEWORK=dynamo-sglang, got: $FRAMEWORK" >&2
     exit 1
 fi
 
@@ -429,10 +430,19 @@
     mkdir -p recipes/vllm/deepseek-v4
     cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4" recipes/vllm/deepseek-v4
 elif [[ $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "dsv4" ]]; then
-    # Stay on NVIDIA/srt-slurm:main (default) — submission branch no
-    # longer needed; overlay our hand-rolled DSV4 sglang recipes onto it.
-    git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR"
-    cd "$SRT_REPO_DIR"
+    if [[ "$USES_DCGM_POWER" == "1" ]]; then
+        git clone "$POWER_SRT_SLURM_URL" "$SRT_REPO_DIR"
+        cd "$SRT_REPO_DIR"
+        git checkout "$POWER_SRT_SLURM_PIN" || exit 1
+        # The power lane must run the exact pinned producer SHA, never a moving branch.
+        test "$(git rev-parse HEAD)" = "$POWER_SRT_SLURM_PIN" || { echo "Error: srt-slurm HEAD does not match POWER_SRT_SLURM_PIN=$POWER_SRT_SLURM_PIN" >&2; exit 1; }
+        git rev-parse HEAD > "$GITHUB_WORKSPACE/power-producer-sha.txt"
+    else
+        # Stay on NVIDIA/srt-slurm:main (default) — submission branch no
+        # longer needed; overlay our hand-rolled DSV4 sglang recipes onto it.
+        git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR"
+        cd "$SRT_REPO_DIR"
+    fi
     mkdir -p recipes/sglang/deepseek-v4
     cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4" recipes/sglang/deepseek-v4
 elif [[ $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "glm5.1" ]]; then

runners/launch_gb300-nv.sh

@@ -146,11 +146,12 @@ if [[ -n "$CONFIG_FILE" && -f "$_RECIPE_SRC" ]] && awk '
     USES_DCGM_POWER=1
 fi
 
-# Note (wenyao): the producer pin descends from the fp8 v1.0.25 lineage
-# (cargo/maturin bootstrap); an fp4 power recipe would silently skip the
-# sa-submission branch it needs, so fail fast instead.
-if [[ "$USES_DCGM_POWER" == "1" && "$PRECISION" != "fp8" ]]; then
-    echo "Error: dcgm-power lanes are only validated for PRECISION=fp8, got: $PRECISION" >&2
+# Note (wenyao): the producer pin follows the srt-slurm main lineage that the
+# dynamo-sglang lanes run on (fp8 validated end-to-end, fp4 recipes
+# parse-verified against the pin); other frameworks clone diverging refs
+# (aflowers branch, sa-submission), so fail fast for them instead.
+if [[ "$USES_DCGM_POWER" == "1" && "$FRAMEWORK" != "dynamo-sglang" ]]; then
+    echo "Error: dcgm-power lanes are only validated for FRAMEWORK=dynamo-sglang, got: $FRAMEWORK" >&2
     exit 1
 fi
 
@@ -294,6 +295,19 @@ elif [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "kimik2.5" && $PRECISION
     git checkout main
     mkdir -p recipes/vllm/kimi-k2.5-fp4
     cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.5-fp4" recipes/vllm/kimi-k2.5-fp4
+elif [[ $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "dsv4" && "$USES_DCGM_POWER" == "1" ]]; then
+    # Note (wenyao): only the power lane routes here — non-power dsv4 sglang
+    # keeps the generic fallthrough untouched. The overlay is required: the
+    # telemetry-tagged DSV4 recipes are version-controlled in-repo only,
+    # neither upstream ref ships them.
+    git clone "$POWER_SRT_SLURM_URL" "$SRT_REPO_DIR"
+    cd "$SRT_REPO_DIR"
+    git checkout "$POWER_SRT_SLURM_PIN" || exit 1
+    # The power lane must run the exact pinned producer SHA, never a moving branch.
+    test "$(git rev-parse HEAD)" = "$POWER_SRT_SLURM_PIN" || { echo "Error: srt-slurm HEAD does not match POWER_SRT_SLURM_PIN=$POWER_SRT_SLURM_PIN" >&2; exit 1; }
+    git rev-parse HEAD > "$GITHUB_WORKSPACE/power-producer-sha.txt"
+    mkdir -p recipes/sglang/deepseek-v4
+    cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4" recipes/sglang/deepseek-v4
 elif [[ $FRAMEWORK == "dynamo-trt" && $MODEL_PREFIX == "dsv4" ]]; then
     # DSv4 dynamo-trt recipes use the HuggingFace model ID as model.path,
     # so override SRT_SLURM_MODEL_PREFIX to match the recipe's model path key.

utils/test_gb200_power_official_contract.py

@@ -61,7 +61,7 @@ def assert_recipe_driven_detection(launcher):
     assert "t && /^  enabled: true$/        { e = 1 }" in launcher
     assert "USES_DCGM_POWER=1" in launcher
     assert (
-        'if [[ "$USES_DCGM_POWER" == "1" && "$PRECISION" != "fp8" ]]; then'
+        'if [[ "$USES_DCGM_POWER" == "1" && "$FRAMEWORK" != "dynamo-sglang" ]]; then'
         in launcher
     )
     # Exporter provisioning, pinned clone, srtslurm.yaml injection, and audit
@@ -149,14 +149,36 @@ def test_pin_literal_lives_only_in_the_two_launchers():
     ]
 
 
-def test_exactly_two_recipes_opt_into_dcgm_power():
+# Every recipe that opts into dcgm-power, with the exporter port its cluster
+# requires (im-gb300 nodes already bind 9401; gb200 keeps the default).
+POWER_RECIPES = {
+    "benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-1p1d-dep8-dep16-6-c512.yaml": 9401,
+    "benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-1p1d-tp8-tp8-4-c1.yaml": 9401,
+    "benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-1p2d-dep8-dep16-10-c256.yaml": 9401,
+    "benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-1p1d-dep4-dep16-5-c1024.yaml": 19401,
+    "benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-1p1d-tp4-tp4-2-c1.yaml": 19401,
+    "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/1p1d-tp4-tp4.yaml": 9401,
+    "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_0.yaml": 19401,
+    "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/1p1d-tp4-tp4.yaml": 19401,
+}
+
+
+def test_exactly_the_declared_recipes_opt_into_dcgm_power():
     hits = git_grep_lines(
         "-lF", "provider: dcgm-power", "--", "benchmarks/multi_node/srt-slurm-recipes"
     )
-    assert sorted(hits) == [
-        "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/1p1d-tp4-tp4.yaml",
-        "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/1p1d-tp4-tp4.yaml",
-    ]
+    assert sorted(hits) == sorted(POWER_RECIPES)
+
+
+def test_every_power_recipe_declares_the_same_telemetry_contract():
+    for path, port in POWER_RECIPES.items():
+        telemetry = yaml.safe_load((REPO_ROOT / path).read_text())["telemetry"]
+        assert telemetry["enabled"] is True, path
+        assert telemetry["provider"] == "dcgm-power", path
+        assert telemetry["default_frequency"] == 1.0, path
+        assert telemetry["required"] is True, path
+        assert telemetry["dcgm_exporter"]["container_image"] == "dcgm-exporter", path
+        assert telemetry["dcgm_exporter"]["port"] == port, path
 
 
 def test_lane_and_pin_have_no_env_override_backdoor():