feat(power-agent): per-node power-cap enforcement DaemonSet

.github/actions/build-deploy-component/action.yml

@@ -2,11 +2,11 @@
 # SPDX-License-Identifier: Apache-2.0
 
 name: 'Build Deploy Component'
-description: 'Build and push a deploy component container (operator or snapshot)'
+description: 'Build and push a deploy component container (operator, snapshot, or power-agent)'
 
 inputs:
   component:
-    description: 'Component to build: operator or snapshot'
+    description: 'Component to build: operator, snapshot, or power-agent'
     required: true
   image_tag:
     description: 'Image tag to apply to built images'
@@ -122,10 +122,18 @@ runs:
           fi
         done <<< "${{ inputs.extra_build_args }}"
 
+        # operator and snapshot both consume the named `snapshot` build context
+        # (deploy/operator/Dockerfile COPYs --from=snapshot; the snapshot image
+        # IS that tree). power-agent is self-contained, so omit it there.
+        BUILD_CONTEXT_FLAG=""
+        if [[ "${{ inputs.component }}" != "power-agent" ]]; then
+          BUILD_CONTEXT_FLAG="--build-context snapshot=../snapshot"
+        fi
+
         docker buildx build --push \
             --platform ${{ steps.settings.outputs.build_platforms }} \
             "${BUILD_ARGS[@]}" \
-            --build-context snapshot=../snapshot \
+            ${BUILD_CONTEXT_FLAG} \
             ${TARGET_FLAG} \
             ${TAGGING_FLAGS} -f Dockerfile .
 

.github/actions/changed-files/action.yml

@@ -16,6 +16,9 @@ outputs:
   snapshot:
     description: 'Whether snapshot-agent files changed'
     value: ${{ steps.filter.outputs.snapshot_any_modified }}
+  power_agent:
+    description: 'Whether power-agent files changed'
+    value: ${{ steps.filter.outputs.power_agent_any_modified }}
   deploy:
     description: 'Whether deploy files changed'
     value: ${{ steps.filter.outputs.deploy_any_modified }}
@@ -121,6 +124,7 @@ runs:
         echo "core: ${{ steps.filter.outputs.core_any_modified }}"
         echo "operator: ${{ steps.filter.outputs.operator_any_modified }}"
         echo "snapshot: ${{ steps.filter.outputs.snapshot_any_modified }}"
+        echo "power_agent: ${{ steps.filter.outputs.power_agent_any_modified }}"
         echo "deploy: ${{ steps.filter.outputs.deploy_any_modified }}"
         echo "planner: ${{ steps.filter.outputs.planner_any_modified }}"
         echo "vllm: ${{ steps.filter.outputs.vllm_any_modified }}"
@@ -138,6 +142,7 @@ runs:
         echo "core: ${{ steps.filter.outputs.core_all_modified_files }}"
         echo "operator: ${{ steps.filter.outputs.operator_all_modified_files }}"
         echo "snapshot: ${{ steps.filter.outputs.snapshot_all_modified_files }}"
+        echo "power_agent: ${{ steps.filter.outputs.power_agent_all_modified_files }}"
         echo "deploy: ${{ steps.filter.outputs.deploy_all_modified_files }}"
         echo "planner: ${{ steps.filter.outputs.planner_all_modified_files }}"
         echo "vllm: ${{ steps.filter.outputs.vllm_all_modified_files }}"
@@ -153,7 +158,7 @@ runs:
       shell: bash
       run: |
         # Combine all filter-specific files into one list
-        COVERED_FILES=$(echo "${{ steps.filter.outputs.docs_all_modified_files }} ${{ steps.filter.outputs.examples_all_modified_files }} ${{ steps.filter.outputs.ignore_all_modified_files }} ${{ steps.filter.outputs.ci_all_modified_files }} ${{ steps.filter.outputs.core_all_modified_files }} ${{ steps.filter.outputs.operator_all_modified_files }} ${{ steps.filter.outputs.snapshot_all_modified_files }} ${{ steps.filter.outputs.deploy_all_modified_files }} ${{ steps.filter.outputs.planner_all_modified_files }} ${{ steps.filter.outputs.vllm_all_modified_files }} ${{ steps.filter.outputs.sglang_all_modified_files }} ${{ steps.filter.outputs.trtllm_all_modified_files }} ${{ steps.filter.outputs.sample_all_modified_files }} ${{ steps.filter.outputs.frontend_all_modified_files }} ${{ steps.filter.outputs.benchmarks_all_modified_files }} ${{ steps.filter.outputs.efa_all_modified_files }} ${{ steps.filter.outputs.rust_all_modified_files }}" | tr ' ' '\n' | grep -v '^$' | sort -u)
+        COVERED_FILES=$(echo "${{ steps.filter.outputs.docs_all_modified_files }} ${{ steps.filter.outputs.examples_all_modified_files }} ${{ steps.filter.outputs.ignore_all_modified_files }} ${{ steps.filter.outputs.ci_all_modified_files }} ${{ steps.filter.outputs.core_all_modified_files }} ${{ steps.filter.outputs.operator_all_modified_files }} ${{ steps.filter.outputs.snapshot_all_modified_files }} ${{ steps.filter.outputs.power_agent_all_modified_files }} ${{ steps.filter.outputs.deploy_all_modified_files }} ${{ steps.filter.outputs.planner_all_modified_files }} ${{ steps.filter.outputs.vllm_all_modified_files }} ${{ steps.filter.outputs.sglang_all_modified_files }} ${{ steps.filter.outputs.trtllm_all_modified_files }} ${{ steps.filter.outputs.sample_all_modified_files }} ${{ steps.filter.outputs.frontend_all_modified_files }} ${{ steps.filter.outputs.benchmarks_all_modified_files }} ${{ steps.filter.outputs.efa_all_modified_files }} ${{ steps.filter.outputs.rust_all_modified_files }}" | tr ' ' '\n' | grep -v '^$' | sort -u)
 
         # Get all modified files
         ALL_FILES=$(echo "${{ steps.filter.outputs.all_all_modified_files }}" | tr ' ' '\n' | grep -v '^$' | sort -u)

.github/filters.yaml

@@ -4,6 +4,7 @@
 #   core     -> dynamo build-test, all backend builds (vllm/sglang/trtllm)
 #   operator -> operator build and test
 #   snapshot -> snapshot-agent build and test
+#   power_agent -> power-agent image build
 #   deploy   -> deploy tests
 #   planner  -> dynamo build-test
 #   vllm     -> vllm build and test
@@ -134,6 +135,13 @@ snapshot:
   - 'deploy/snapshot/**'
   - 'deploy/snapshot/.*'
 
+power_agent:
+  - *ci
+  - 'deploy/power-agent/**'
+  - 'deploy/helm/charts/power-agent/**'
+  - '!**/*.md'
+  - '!**/*.rst'
+
 deploy:
   - *ci
   - 'deploy/helm/**'
@@ -150,6 +158,9 @@ planner:
   - 'components/src/dynamo/global_planner/**'
   - 'components/src/dynamo/profiler/**'
   - 'components/src/dynamo/global_router/**'
+  # Power-aware planner (PR #9369): per-node power agent component
+  - 'deploy/power-agent/**'
+  - 'deploy/helm/charts/power-agent/**'
 
 vllm:
   - 'container/deps/requirements.vllm.txt'

.github/workflows/post-merge-ci.yml

@@ -116,6 +116,70 @@ jobs:
           extra_tags: |
             ${{ secrets.AZURE_ACR_HOSTNAME }}/ai-dynamo/dynamo:${{ github.sha }}-snapshot-agent
 
+# ============================================================================
+# Power Agent
+# ============================================================================
+
+  power-agent:
+    name: Power Agent
+    if: github.event_name != 'workflow_dispatch'
+    runs-on: prod-default-v2
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+      - name: Build and push power agent
+        uses: ./.github/actions/build-deploy-component
+        with:
+          component: power-agent
+          target: runtime  # push the slim shipped stage, not the test stage
+          image_tag: ${{ github.sha }}-power-agent
+          builder_name: b-${{ github.run_id }}-${{ github.run_attempt }}
+          aws_default_region: ${{ secrets.AWS_DEFAULT_REGION }}
+          aws_account_id: ${{ secrets.AWS_ACCOUNT_ID }}
+          azure_acr_hostname: ${{ secrets.AZURE_ACR_HOSTNAME }}
+          azure_acr_user: ${{ secrets.AZURE_ACR_USER }}
+          azure_acr_password: ${{ secrets.AZURE_ACR_PASSWORD }}
+          extra_tags: |
+            ${{ secrets.AZURE_ACR_HOSTNAME }}/ai-dynamo/dynamo:${{ github.sha }}-power-agent
+      - name: Refresh BuildKit builder
+        # The runtime build above can leave the remote BuildKit connection stale;
+        # re-establish it (re-routing only if unhealthy) before the test build.
+        uses: ./.github/actions/builder-refresher
+        with:
+          builder_name: b-${{ github.run_id }}-${{ github.run_attempt }}
+          flavor: general
+          arch: linux/amd64
+      - name: Run unit tests inside the container image
+        # Build the Dockerfile `test` stage (FROM runtime): its RUN pytest runs
+        # the suite against the exact shipped image (Python 3.12 + baked deps),
+        # validating the container itself rather than a pip-installed runner env.
+        # Reuses the builder + ECR Docker Hub mirror the build step set up above
+        # (runtime layers are a cache hit); no --push/--load runs pytest without
+        # exporting the never-shipped test image.
+        env:
+          ECR_HOSTNAME: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_DEFAULT_REGION }}.amazonaws.com
+        run: |
+          docker buildx build --builder b-${{ github.run_id }}-${{ github.run_attempt }} \
+            --target test \
+            --build-arg DOCKER_PROXY=${ECR_HOSTNAME}/dockerhub/ \
+            -f deploy/power-agent/Dockerfile deploy/power-agent
+      - name: Set up Helm
+        uses: azure/setup-helm@b9e51907a09c216f16ebe8536097933489208112  # v4.3.0
+        with:
+          version: 'v3.17.3'  # match deploy/operator/Makefile HELM_VERSION
+      - name: Lint and render (production + dev)
+        run: |
+          CHART=deploy/helm/charts/power-agent
+          helm lint "$CHART" --set image.tag=ci-test
+          # Production DaemonSet render (default daemonset.enabled=true).
+          helm template power-agent "$CHART" --set image.tag=ci-test >/dev/null
+          # Dev-pod render: uses dev.image.* and must NOT require the production
+          # image.tag (regression guard for the daemonset-gated validation).
+          helm template power-agent "$CHART" \
+            --set daemonset.enabled=false \
+            --set dev.enabled=true \
+            --set dev.nodeName=ci-node >/dev/null
+
   # ============================================================================
   # BUILD JOBS
   # ============================================================================
@@ -1177,7 +1241,7 @@ jobs:
     name: Clean K8s builder if exists
     runs-on: prod-default-small-v2
     if: always()
-    needs: [vllm-build, sglang-build, trtllm-build, vllm-dev-build, sglang-dev-build, trtllm-dev-build, vllm-efa-build, sglang-efa-build, trtllm-efa-build, operator, snapshot-agent, snapshot-placeholder-vllm, snapshot-placeholder-sglang, frontend-build, planner-build]
+    needs: [vllm-build, sglang-build, trtllm-build, vllm-dev-build, sglang-dev-build, trtllm-dev-build, vllm-efa-build, sglang-efa-build, trtllm-efa-build, operator, snapshot-agent, snapshot-placeholder-vllm, snapshot-placeholder-sglang, power-agent, frontend-build, planner-build]
     steps:
     - name: Checkout repository
       uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2

.github/workflows/pr.yaml

@@ -27,6 +27,7 @@ jobs:
       planner: ${{ steps.changes.outputs.planner }}
       operator: ${{ steps.changes.outputs.operator }}
       snapshot: ${{ steps.changes.outputs.snapshot }}
+      power_agent: ${{ steps.changes.outputs.power_agent }}
       deploy: ${{ steps.changes.outputs.deploy }}
       vllm: ${{ steps.changes.outputs.vllm }}
       sglang: ${{ steps.changes.outputs.sglang }}
@@ -59,6 +60,7 @@ jobs:
       - snapshot-agent
       - snapshot-placeholder-vllm
       - snapshot-placeholder-sglang
+      - power-agent
       - vllm-build
       - vllm-dev-build
       - vllm-test
@@ -203,6 +205,67 @@ jobs:
           extra_tags: |
             ${{ secrets.AZURE_ACR_HOSTNAME }}/ai-dynamo/dynamo:${{ github.sha }}-snapshot-agent
 
+# ============================================================================
+# Power Agent
+# ============================================================================
+
+  power-agent:
+    needs: changed-files
+    if: needs.changed-files.outputs.power_agent == 'true'
+    name: Power Agent
+    runs-on: prod-default-v2
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        # Do not use fetch-depth: 0 — it fetches all 1600+ branches (~3 min)
+      - name: Build and push power agent
+        uses: ./.github/actions/build-deploy-component
+        with:
+          component: power-agent
+          target: runtime  # push the slim shipped stage, not the test stage
+          image_tag: ${{ github.sha }}-power-agent
+          builder_name: ${{ needs.changed-files.outputs.builder_name }}
+          aws_default_region: ${{ secrets.AWS_DEFAULT_REGION }}
+          aws_account_id: ${{ secrets.AWS_ACCOUNT_ID }}
+      - name: Refresh BuildKit builder
+        # The runtime build above can leave the remote BuildKit connection stale;
+        # re-establish it (re-routing only if unhealthy) before the test build.
+        uses: ./.github/actions/builder-refresher
+        with:
+          builder_name: ${{ needs.changed-files.outputs.builder_name }}
+          flavor: general
+          arch: linux/amd64
+      - name: Run unit tests inside the container image
+        # Build the Dockerfile `test` stage (FROM runtime): its RUN pytest runs
+        # the suite against the exact shipped image (Python 3.12 + baked deps),
+        # validating the container itself rather than a pip-installed runner env.
+        # Reuses the builder + ECR Docker Hub mirror the build step set up above
+        # (runtime layers are a cache hit); no --push/--load runs pytest without
+        # exporting the never-shipped test image.
+        env:
+          ECR_HOSTNAME: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_DEFAULT_REGION }}.amazonaws.com
+        run: |
+          docker buildx build --builder ${{ needs.changed-files.outputs.builder_name }} \
+            --target test \
+            --build-arg DOCKER_PROXY=${ECR_HOSTNAME}/dockerhub/ \
+            -f deploy/power-agent/Dockerfile deploy/power-agent
+      - name: Set up Helm
+        uses: azure/setup-helm@b9e51907a09c216f16ebe8536097933489208112  # v4.3.0
+        with:
+          version: 'v3.17.3'  # match deploy/operator/Makefile HELM_VERSION
+      - name: Lint and render (production + dev)
+        run: |
+          CHART=deploy/helm/charts/power-agent
+          helm lint "$CHART" --set image.tag=ci-test
+          # Production DaemonSet render (default daemonset.enabled=true).
+          helm template power-agent "$CHART" --set image.tag=ci-test >/dev/null
+          # Dev-pod render: uses dev.image.* and must NOT require the production
+          # image.tag (regression guard for the daemonset-gated validation).
+          helm template power-agent "$CHART" \
+            --set daemonset.enabled=false \
+            --set dev.enabled=true \
+            --set dev.nodeName=ci-node >/dev/null
+
 # ============================================================================
 # FRAMEWORK PIPELINES (Build → Test → Copy)
 # ============================================================================
@@ -1196,6 +1259,7 @@ jobs:
       - snapshot-agent
       - snapshot-placeholder-vllm
       - snapshot-placeholder-sglang
+      - power-agent
       - planner-test
       - vllm-copy-to-acr
       - vllm-multi-gpu-test

deploy/helm/charts/power-agent/.helmignore

@@ -0,0 +1,26 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/
+# test files
+tests/
+test-values.yaml

deploy/helm/charts/power-agent/Chart.yaml

@@ -0,0 +1,25 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+apiVersion: v2
+name: power-agent
+description: |
+  Per-node NVML power-cap enforcement DaemonSet for Dynamo. Watches worker
+  pods for the dynamo.nvidia.com/gpu-power-limit annotation, parses
+  /proc/{pid}/cgroup to recover pod UIDs, and calls
+  nvmlDeviceSetPowerManagementLimit per physical GPU.
+
+  Replaces the previous raw DaemonSet, RBAC, and dev-pod manifests.
+type: application
+version: 1.0.0
+appVersion: "1.0.0"
+keywords:
+  - nvidia
+  - dynamo
+  - power
+  - nvml
+  - gpu
+home: https://github.com/ai-dynamo/dynamo
+sources:
+  - https://github.com/ai-dynamo/dynamo
+maintainers:
+  - name: NVIDIA

deploy/helm/charts/power-agent/templates/NOTES.txt

@@ -0,0 +1,60 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+Power Agent {{ .Chart.AppVersion }} installed as release "{{ .Release.Name }}" in namespace "{{ .Release.Namespace }}".
+
+{{- if .Values.daemonset.enabled }}
+
+Verify the DaemonSet is rolling out on every GPU node:
+
+  kubectl rollout status daemonset/{{ include "power-agent.fullname" . }} -n {{ .Release.Namespace }}
+  kubectl get pods -n {{ .Release.Namespace }} -l app.kubernetes.io/name=power-agent -o wide
+
+Tail logs:
+
+  kubectl logs -n {{ .Release.Namespace }} -l app.kubernetes.io/name=power-agent --tail=50
+
+Check Prometheus metrics (port {{ .Values.agent.prometheusPort }} on each pod):
+
+  - dynamo_power_agent_applied_limit_watts{gpu="N"}             # cap applied per GPU
+  - dynamo_power_agent_multi_pod_gpu_total{disposition=...}      # multi-pod conflict counts
+  - dynamo_power_agent_safe_default_applied_total                # safe-default fallbacks
+  - dynamo_power_agent_apply_failures_total                      # NVML apply failures
+
+RBAC scope: {{ if eq (include "power-agent.effectiveNamespaceRestricted" .) "true" }}namespace-scoped (Role + RoleBinding in {{ .Release.Namespace }}){{ else }}cluster-wide (ClusterRole + ClusterRoleBinding){{ end }}.
+
+{{- end }}
+
+{{- if .Values.dev.enabled }}
+
+DEV MODE: a single Pod (not a DaemonSet) was created, pinned to node "{{ .Values.dev.nodeName }}".
+
+╭─ PREREQUISITE — the script ConfigMap MUST exist before this Pod can start ─╮
+│                                                                            │
+│   If you have not done so already, create it from your local checkout:     │
+│                                                                            │
+│     kubectl create configmap {{ .Values.dev.scriptConfigMap }} \
+│       --from-file=power_agent.py=deploy/power-agent/power_agent.py \
+│       -n {{ .Release.Namespace }}
+│                                                                            │
+│   Update flow (after editing power_agent.py locally):                      │
+│                                                                            │
+│     kubectl create configmap {{ .Values.dev.scriptConfigMap }} \
+│       --from-file=power_agent.py=deploy/power-agent/power_agent.py \
+│       -n {{ .Release.Namespace }} --dry-run=client -o yaml \
+│       | kubectl apply -f -                                                 │
+│     kubectl delete pod {{ include "power-agent.fullname" . }}-dev -n {{ .Release.Namespace }}
+│                                                                            │
+│   Without this ConfigMap the Pod stays Pending with                        │
+│   `MountVolume.SetUp failed for volume "script"`.                          │
+│                                                                            │
+╰────────────────────────────────────────────────────────────────────────────╯
+
+Verify:
+
+  kubectl get pod {{ include "power-agent.fullname" . }}-dev -n {{ .Release.Namespace }} -o wide
+  kubectl logs -f {{ include "power-agent.fullname" . }}-dev -n {{ .Release.Namespace }}
+
+Dev mode forced namespace-scoped RBAC ({{ if .Values.dev.namespaceRestrictedOverride }}overridden via dev.namespaceRestrictedOverride=true → cluster-wide{{ else }}Role + RoleBinding in {{ .Release.Namespace }}; agent runs with --namespace={{ .Release.Namespace }}{{ end }}).
+
+{{- end }}