diff --git a/.github/actions/build-deploy-component/action.yml b/.github/actions/build-deploy-component/action.yml index 20ec7b8c45be..9ee843c9b84f 100644 --- a/.github/actions/build-deploy-component/action.yml +++ b/.github/actions/build-deploy-component/action.yml @@ -2,11 +2,11 @@ # SPDX-License-Identifier: Apache-2.0 name: 'Build Deploy Component' -description: 'Build and push a deploy component container (operator or snapshot)' +description: 'Build and push a deploy component container (operator, snapshot, or power-agent)' inputs: component: - description: 'Component to build: operator or snapshot' + description: 'Component to build: operator, snapshot, or power-agent' required: true image_tag: description: 'Image tag to apply to built images' @@ -122,10 +122,18 @@ runs: fi done <<< "${{ inputs.extra_build_args }}" + # operator and snapshot both consume the named `snapshot` build context + # (deploy/operator/Dockerfile COPYs --from=snapshot; the snapshot image + # IS that tree). power-agent is self-contained, so omit it there. + BUILD_CONTEXT_FLAG="" + if [[ "${{ inputs.component }}" != "power-agent" ]]; then + BUILD_CONTEXT_FLAG="--build-context snapshot=../snapshot" + fi + docker buildx build --push \ --platform ${{ steps.settings.outputs.build_platforms }} \ "${BUILD_ARGS[@]}" \ - --build-context snapshot=../snapshot \ + ${BUILD_CONTEXT_FLAG} \ ${TARGET_FLAG} \ ${TAGGING_FLAGS} -f Dockerfile . diff --git a/.github/actions/changed-files/action.yml b/.github/actions/changed-files/action.yml index c943b18f09eb..adeac1be41e1 100644 --- a/.github/actions/changed-files/action.yml +++ b/.github/actions/changed-files/action.yml @@ -16,6 +16,9 @@ outputs: snapshot: description: 'Whether snapshot-agent files changed' value: ${{ steps.filter.outputs.snapshot_any_modified }} + power_agent: + description: 'Whether power-agent files changed' + value: ${{ steps.filter.outputs.power_agent_any_modified }} deploy: description: 'Whether deploy files changed' value: ${{ steps.filter.outputs.deploy_any_modified }} @@ -121,6 +124,7 @@ runs: echo "core: ${{ steps.filter.outputs.core_any_modified }}" echo "operator: ${{ steps.filter.outputs.operator_any_modified }}" echo "snapshot: ${{ steps.filter.outputs.snapshot_any_modified }}" + echo "power_agent: ${{ steps.filter.outputs.power_agent_any_modified }}" echo "deploy: ${{ steps.filter.outputs.deploy_any_modified }}" echo "planner: ${{ steps.filter.outputs.planner_any_modified }}" echo "vllm: ${{ steps.filter.outputs.vllm_any_modified }}" @@ -138,6 +142,7 @@ runs: echo "core: ${{ steps.filter.outputs.core_all_modified_files }}" echo "operator: ${{ steps.filter.outputs.operator_all_modified_files }}" echo "snapshot: ${{ steps.filter.outputs.snapshot_all_modified_files }}" + echo "power_agent: ${{ steps.filter.outputs.power_agent_all_modified_files }}" echo "deploy: ${{ steps.filter.outputs.deploy_all_modified_files }}" echo "planner: ${{ steps.filter.outputs.planner_all_modified_files }}" echo "vllm: ${{ steps.filter.outputs.vllm_all_modified_files }}" @@ -153,7 +158,7 @@ runs: shell: bash run: | # Combine all filter-specific files into one list - COVERED_FILES=$(echo "${{ steps.filter.outputs.docs_all_modified_files }} ${{ steps.filter.outputs.examples_all_modified_files }} ${{ steps.filter.outputs.ignore_all_modified_files }} ${{ steps.filter.outputs.ci_all_modified_files }} ${{ steps.filter.outputs.core_all_modified_files }} ${{ steps.filter.outputs.operator_all_modified_files }} ${{ steps.filter.outputs.snapshot_all_modified_files }} ${{ steps.filter.outputs.deploy_all_modified_files }} ${{ steps.filter.outputs.planner_all_modified_files }} ${{ steps.filter.outputs.vllm_all_modified_files }} ${{ steps.filter.outputs.sglang_all_modified_files }} ${{ steps.filter.outputs.trtllm_all_modified_files }} ${{ steps.filter.outputs.sample_all_modified_files }} ${{ steps.filter.outputs.frontend_all_modified_files }} ${{ steps.filter.outputs.benchmarks_all_modified_files }} ${{ steps.filter.outputs.efa_all_modified_files }} ${{ steps.filter.outputs.rust_all_modified_files }}" | tr ' ' '\n' | grep -v '^$' | sort -u) + COVERED_FILES=$(echo "${{ steps.filter.outputs.docs_all_modified_files }} ${{ steps.filter.outputs.examples_all_modified_files }} ${{ steps.filter.outputs.ignore_all_modified_files }} ${{ steps.filter.outputs.ci_all_modified_files }} ${{ steps.filter.outputs.core_all_modified_files }} ${{ steps.filter.outputs.operator_all_modified_files }} ${{ steps.filter.outputs.snapshot_all_modified_files }} ${{ steps.filter.outputs.power_agent_all_modified_files }} ${{ steps.filter.outputs.deploy_all_modified_files }} ${{ steps.filter.outputs.planner_all_modified_files }} ${{ steps.filter.outputs.vllm_all_modified_files }} ${{ steps.filter.outputs.sglang_all_modified_files }} ${{ steps.filter.outputs.trtllm_all_modified_files }} ${{ steps.filter.outputs.sample_all_modified_files }} ${{ steps.filter.outputs.frontend_all_modified_files }} ${{ steps.filter.outputs.benchmarks_all_modified_files }} ${{ steps.filter.outputs.efa_all_modified_files }} ${{ steps.filter.outputs.rust_all_modified_files }}" | tr ' ' '\n' | grep -v '^$' | sort -u) # Get all modified files ALL_FILES=$(echo "${{ steps.filter.outputs.all_all_modified_files }}" | tr ' ' '\n' | grep -v '^$' | sort -u) diff --git a/.github/filters.yaml b/.github/filters.yaml index d6445d5ef71f..2949adddb3f6 100644 --- a/.github/filters.yaml +++ b/.github/filters.yaml @@ -4,6 +4,7 @@ # core -> dynamo build-test, all backend builds (vllm/sglang/trtllm) # operator -> operator build and test # snapshot -> snapshot-agent build and test +# power_agent -> power-agent image build # deploy -> deploy tests # planner -> dynamo build-test # vllm -> vllm build and test @@ -134,6 +135,13 @@ snapshot: - 'deploy/snapshot/**' - 'deploy/snapshot/.*' +power_agent: + - *ci + - 'deploy/power-agent/**' + - 'deploy/helm/charts/power-agent/**' + - '!**/*.md' + - '!**/*.rst' + deploy: - *ci - 'deploy/helm/**' @@ -150,6 +158,9 @@ planner: - 'components/src/dynamo/global_planner/**' - 'components/src/dynamo/profiler/**' - 'components/src/dynamo/global_router/**' + # Power-aware planner (PR #9369): per-node power agent component + - 'deploy/power-agent/**' + - 'deploy/helm/charts/power-agent/**' vllm: - 'container/deps/requirements.vllm.txt' diff --git a/.github/workflows/post-merge-ci.yml b/.github/workflows/post-merge-ci.yml index 12652761c44e..e89c3b09595f 100644 --- a/.github/workflows/post-merge-ci.yml +++ b/.github/workflows/post-merge-ci.yml @@ -116,6 +116,70 @@ jobs: extra_tags: | ${{ secrets.AZURE_ACR_HOSTNAME }}/ai-dynamo/dynamo:${{ github.sha }}-snapshot-agent +# ============================================================================ +# Power Agent +# ============================================================================ + + power-agent: + name: Power Agent + if: github.event_name != 'workflow_dispatch' + runs-on: prod-default-v2 + steps: + - name: Checkout code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - name: Build and push power agent + uses: ./.github/actions/build-deploy-component + with: + component: power-agent + target: runtime # push the slim shipped stage, not the test stage + image_tag: ${{ github.sha }}-power-agent + builder_name: b-${{ github.run_id }}-${{ github.run_attempt }} + aws_default_region: ${{ secrets.AWS_DEFAULT_REGION }} + aws_account_id: ${{ secrets.AWS_ACCOUNT_ID }} + azure_acr_hostname: ${{ secrets.AZURE_ACR_HOSTNAME }} + azure_acr_user: ${{ secrets.AZURE_ACR_USER }} + azure_acr_password: ${{ secrets.AZURE_ACR_PASSWORD }} + extra_tags: | + ${{ secrets.AZURE_ACR_HOSTNAME }}/ai-dynamo/dynamo:${{ github.sha }}-power-agent + - name: Refresh BuildKit builder + # The runtime build above can leave the remote BuildKit connection stale; + # re-establish it (re-routing only if unhealthy) before the test build. + uses: ./.github/actions/builder-refresher + with: + builder_name: b-${{ github.run_id }}-${{ github.run_attempt }} + flavor: general + arch: linux/amd64 + - name: Run unit tests inside the container image + # Build the Dockerfile `test` stage (FROM runtime): its RUN pytest runs + # the suite against the exact shipped image (Python 3.12 + baked deps), + # validating the container itself rather than a pip-installed runner env. + # Reuses the builder + ECR Docker Hub mirror the build step set up above + # (runtime layers are a cache hit); no --push/--load runs pytest without + # exporting the never-shipped test image. + env: + ECR_HOSTNAME: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_DEFAULT_REGION }}.amazonaws.com + run: | + docker buildx build --builder b-${{ github.run_id }}-${{ github.run_attempt }} \ + --target test \ + --build-arg DOCKER_PROXY=${ECR_HOSTNAME}/dockerhub/ \ + -f deploy/power-agent/Dockerfile deploy/power-agent + - name: Set up Helm + uses: azure/setup-helm@b9e51907a09c216f16ebe8536097933489208112 # v4.3.0 + with: + version: 'v3.17.3' # match deploy/operator/Makefile HELM_VERSION + - name: Lint and render (production + dev) + run: | + CHART=deploy/helm/charts/power-agent + helm lint "$CHART" --set image.tag=ci-test + # Production DaemonSet render (default daemonset.enabled=true). + helm template power-agent "$CHART" --set image.tag=ci-test >/dev/null + # Dev-pod render: uses dev.image.* and must NOT require the production + # image.tag (regression guard for the daemonset-gated validation). + helm template power-agent "$CHART" \ + --set daemonset.enabled=false \ + --set dev.enabled=true \ + --set dev.nodeName=ci-node >/dev/null + # ============================================================================ # BUILD JOBS # ============================================================================ @@ -1177,7 +1241,7 @@ jobs: name: Clean K8s builder if exists runs-on: prod-default-small-v2 if: always() - needs: [vllm-build, sglang-build, trtllm-build, vllm-dev-build, sglang-dev-build, trtllm-dev-build, vllm-efa-build, sglang-efa-build, trtllm-efa-build, operator, snapshot-agent, snapshot-placeholder-vllm, snapshot-placeholder-sglang, frontend-build, planner-build] + needs: [vllm-build, sglang-build, trtllm-build, vllm-dev-build, sglang-dev-build, trtllm-dev-build, vllm-efa-build, sglang-efa-build, trtllm-efa-build, operator, snapshot-agent, snapshot-placeholder-vllm, snapshot-placeholder-sglang, power-agent, frontend-build, planner-build] steps: - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 0b2d3c3397a9..e5d949c7921b 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -27,6 +27,7 @@ jobs: planner: ${{ steps.changes.outputs.planner }} operator: ${{ steps.changes.outputs.operator }} snapshot: ${{ steps.changes.outputs.snapshot }} + power_agent: ${{ steps.changes.outputs.power_agent }} deploy: ${{ steps.changes.outputs.deploy }} vllm: ${{ steps.changes.outputs.vllm }} sglang: ${{ steps.changes.outputs.sglang }} @@ -59,6 +60,7 @@ jobs: - snapshot-agent - snapshot-placeholder-vllm - snapshot-placeholder-sglang + - power-agent - vllm-build - vllm-dev-build - vllm-test @@ -203,6 +205,67 @@ jobs: extra_tags: | ${{ secrets.AZURE_ACR_HOSTNAME }}/ai-dynamo/dynamo:${{ github.sha }}-snapshot-agent +# ============================================================================ +# Power Agent +# ============================================================================ + + power-agent: + needs: changed-files + if: needs.changed-files.outputs.power_agent == 'true' + name: Power Agent + runs-on: prod-default-v2 + steps: + - name: Checkout code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + # Do not use fetch-depth: 0 — it fetches all 1600+ branches (~3 min) + - name: Build and push power agent + uses: ./.github/actions/build-deploy-component + with: + component: power-agent + target: runtime # push the slim shipped stage, not the test stage + image_tag: ${{ github.sha }}-power-agent + builder_name: ${{ needs.changed-files.outputs.builder_name }} + aws_default_region: ${{ secrets.AWS_DEFAULT_REGION }} + aws_account_id: ${{ secrets.AWS_ACCOUNT_ID }} + - name: Refresh BuildKit builder + # The runtime build above can leave the remote BuildKit connection stale; + # re-establish it (re-routing only if unhealthy) before the test build. + uses: ./.github/actions/builder-refresher + with: + builder_name: ${{ needs.changed-files.outputs.builder_name }} + flavor: general + arch: linux/amd64 + - name: Run unit tests inside the container image + # Build the Dockerfile `test` stage (FROM runtime): its RUN pytest runs + # the suite against the exact shipped image (Python 3.12 + baked deps), + # validating the container itself rather than a pip-installed runner env. + # Reuses the builder + ECR Docker Hub mirror the build step set up above + # (runtime layers are a cache hit); no --push/--load runs pytest without + # exporting the never-shipped test image. + env: + ECR_HOSTNAME: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_DEFAULT_REGION }}.amazonaws.com + run: | + docker buildx build --builder ${{ needs.changed-files.outputs.builder_name }} \ + --target test \ + --build-arg DOCKER_PROXY=${ECR_HOSTNAME}/dockerhub/ \ + -f deploy/power-agent/Dockerfile deploy/power-agent + - name: Set up Helm + uses: azure/setup-helm@b9e51907a09c216f16ebe8536097933489208112 # v4.3.0 + with: + version: 'v3.17.3' # match deploy/operator/Makefile HELM_VERSION + - name: Lint and render (production + dev) + run: | + CHART=deploy/helm/charts/power-agent + helm lint "$CHART" --set image.tag=ci-test + # Production DaemonSet render (default daemonset.enabled=true). + helm template power-agent "$CHART" --set image.tag=ci-test >/dev/null + # Dev-pod render: uses dev.image.* and must NOT require the production + # image.tag (regression guard for the daemonset-gated validation). + helm template power-agent "$CHART" \ + --set daemonset.enabled=false \ + --set dev.enabled=true \ + --set dev.nodeName=ci-node >/dev/null + # ============================================================================ # FRAMEWORK PIPELINES (Build → Test → Copy) # ============================================================================ @@ -1196,6 +1259,7 @@ jobs: - snapshot-agent - snapshot-placeholder-vllm - snapshot-placeholder-sglang + - power-agent - planner-test - vllm-copy-to-acr - vllm-multi-gpu-test diff --git a/deploy/helm/charts/power-agent/.helmignore b/deploy/helm/charts/power-agent/.helmignore new file mode 100644 index 000000000000..68f3fdcc6325 --- /dev/null +++ b/deploy/helm/charts/power-agent/.helmignore @@ -0,0 +1,26 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ +# test files +tests/ +test-values.yaml diff --git a/deploy/helm/charts/power-agent/Chart.yaml b/deploy/helm/charts/power-agent/Chart.yaml new file mode 100644 index 000000000000..16827d1fe171 --- /dev/null +++ b/deploy/helm/charts/power-agent/Chart.yaml @@ -0,0 +1,25 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +apiVersion: v2 +name: power-agent +description: | + Per-node NVML power-cap enforcement DaemonSet for Dynamo. Watches worker + pods for the dynamo.nvidia.com/gpu-power-limit annotation, parses + /proc/{pid}/cgroup to recover pod UIDs, and calls + nvmlDeviceSetPowerManagementLimit per physical GPU. + + Replaces the previous raw DaemonSet, RBAC, and dev-pod manifests. +type: application +version: 1.0.0 +appVersion: "1.0.0" +keywords: + - nvidia + - dynamo + - power + - nvml + - gpu +home: https://github.com/ai-dynamo/dynamo +sources: + - https://github.com/ai-dynamo/dynamo +maintainers: + - name: NVIDIA diff --git a/deploy/helm/charts/power-agent/templates/NOTES.txt b/deploy/helm/charts/power-agent/templates/NOTES.txt new file mode 100644 index 000000000000..3476498afb29 --- /dev/null +++ b/deploy/helm/charts/power-agent/templates/NOTES.txt @@ -0,0 +1,60 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +Power Agent {{ .Chart.AppVersion }} installed as release "{{ .Release.Name }}" in namespace "{{ .Release.Namespace }}". + +{{- if .Values.daemonset.enabled }} + +Verify the DaemonSet is rolling out on every GPU node: + + kubectl rollout status daemonset/{{ include "power-agent.fullname" . }} -n {{ .Release.Namespace }} + kubectl get pods -n {{ .Release.Namespace }} -l app.kubernetes.io/name=power-agent -o wide + +Tail logs: + + kubectl logs -n {{ .Release.Namespace }} -l app.kubernetes.io/name=power-agent --tail=50 + +Check Prometheus metrics (port {{ .Values.agent.prometheusPort }} on each pod): + + - dynamo_power_agent_applied_limit_watts{gpu="N"} # cap applied per GPU + - dynamo_power_agent_multi_pod_gpu_total{disposition=...} # multi-pod conflict counts + - dynamo_power_agent_safe_default_applied_total # safe-default fallbacks + - dynamo_power_agent_apply_failures_total # NVML apply failures + +RBAC scope: {{ if eq (include "power-agent.effectiveNamespaceRestricted" .) "true" }}namespace-scoped (Role + RoleBinding in {{ .Release.Namespace }}){{ else }}cluster-wide (ClusterRole + ClusterRoleBinding){{ end }}. + +{{- end }} + +{{- if .Values.dev.enabled }} + +DEV MODE: a single Pod (not a DaemonSet) was created, pinned to node "{{ .Values.dev.nodeName }}". + +╭─ PREREQUISITE — the script ConfigMap MUST exist before this Pod can start ─╮ +│ │ +│ If you have not done so already, create it from your local checkout: │ +│ │ +│ kubectl create configmap {{ .Values.dev.scriptConfigMap }} \ +│ --from-file=power_agent.py=deploy/power-agent/power_agent.py \ +│ -n {{ .Release.Namespace }} +│ │ +│ Update flow (after editing power_agent.py locally): │ +│ │ +│ kubectl create configmap {{ .Values.dev.scriptConfigMap }} \ +│ --from-file=power_agent.py=deploy/power-agent/power_agent.py \ +│ -n {{ .Release.Namespace }} --dry-run=client -o yaml \ +│ | kubectl apply -f - │ +│ kubectl delete pod {{ include "power-agent.fullname" . }}-dev -n {{ .Release.Namespace }} +│ │ +│ Without this ConfigMap the Pod stays Pending with │ +│ `MountVolume.SetUp failed for volume "script"`. │ +│ │ +╰────────────────────────────────────────────────────────────────────────────╯ + +Verify: + + kubectl get pod {{ include "power-agent.fullname" . }}-dev -n {{ .Release.Namespace }} -o wide + kubectl logs -f {{ include "power-agent.fullname" . }}-dev -n {{ .Release.Namespace }} + +Dev mode forced namespace-scoped RBAC ({{ if .Values.dev.namespaceRestrictedOverride }}overridden via dev.namespaceRestrictedOverride=true → cluster-wide{{ else }}Role + RoleBinding in {{ .Release.Namespace }}; agent runs with --namespace={{ .Release.Namespace }}{{ end }}). + +{{- end }} diff --git a/deploy/helm/charts/power-agent/templates/_helpers.tpl b/deploy/helm/charts/power-agent/templates/_helpers.tpl new file mode 100644 index 000000000000..380da7f63d8b --- /dev/null +++ b/deploy/helm/charts/power-agent/templates/_helpers.tpl @@ -0,0 +1,148 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +{{/* +Expand the name of the chart. +*/}} +{{- define "power-agent.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +*/}} +{{- define "power-agent.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "power-agent.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "power-agent.labels" -}} +helm.sh/chart: {{ include "power-agent.chart" . }} +{{ include "power-agent.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/component: power-agent +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "power-agent.selectorLabels" -}} +app.kubernetes.io/name: {{ include "power-agent.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "power-agent.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "power-agent.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} + +{{/* +Validate that the production image.tag is set — but only when the DaemonSet +is the thing being installed. Dev-pod mode uses dev.image.* instead (validated +by validateDevImageTag), so gating on daemonset.enabled avoids forcing a dummy +--set image.tag on dev-only installs. The :latest fallback was rejected on +PR #9682 review (CodeRabbit comment on daemonset.yaml:58). Pin a release tag or +sha256:digest at install time: + --set image.tag=v1.0.0 + --set image.tag=sha256:abc... +*/}} +{{- define "power-agent.validateImageTag" -}} +{{- if and .Values.daemonset.enabled (not .Values.image.tag) -}} +{{- fail "image.tag is required when daemonset.enabled (pin to a release tag or sha256:digest; :latest is not supported)" -}} +{{- end -}} +{{- end -}} + +{{/* +Validate that dev.image.tag is set when dev-pod mode is enabled. Mirror of +validateImageTag for the dev iteration image; keeps dev installs from silently +falling back to a mutable tag. +*/}} +{{- define "power-agent.validateDevImageTag" -}} +{{- if and .Values.dev.enabled (not .Values.dev.image.tag) -}} +{{- fail "dev.image.tag is required when dev.enabled (pin the dev iteration image; :latest is not supported)" -}} +{{- end -}} +{{- end -}} + +{{/* +Build an image reference, choosing the correct separator for the tag. + +A sha256 digest must be joined with "@" (repo@sha256:...); only a named tag +uses ":". Hard-coding ":" produces an invalid "repo:sha256:..." reference when +image.tag is a digest. Since values.yaml explicitly allows pinning to either a +release tag or a sha256 digest, detect the digest form and switch separators. +Per PR #9682 @sttts review (daemonset.yaml:64). + +Call with a dict, e.g.: + {{ include "power-agent.imageRef" (dict "repository" .Values.image.repository "tag" .Values.image.tag) }} +*/}} +{{- define "power-agent.imageRef" -}} +{{- $sep := ternary "@" ":" (hasPrefix "sha256:" .tag) -}} +{{- printf "%s%s%s" .repository $sep .tag -}} +{{- end -}} + +{{/* +Validate that production DaemonSet mode and in-cluster dev-pod mode are +not both enabled, and that dev mode has a pinned nodeName. Surfaces at +`helm install` / `helm template` time, not as two competing Pods at runtime. +*/}} +{{- define "power-agent.validateMutex" -}} +{{- if and .Values.daemonset.enabled .Values.dev.enabled -}} +{{- fail "daemonset.enabled and dev.enabled are mutually exclusive. Set exactly one." -}} +{{- end -}} +{{- if and .Values.dev.enabled (not .Values.dev.nodeName) -}} +{{- fail "dev.enabled requires dev.nodeName (the GPU node to pin the dev pod to)." -}} +{{- end -}} +{{- end -}} + +{{/* +Effective RBAC scope. + +Dev mode pins to one node and one namespace, so cluster-wide pod-listing +RBAC would be excessive. The agent's --namespace CLI flag +(power_agent.py:541-546) already constrains its pod queries to a single +namespace when set; the dev-pod template passes --namespace=$(POD_NAMESPACE) +via the downward API. This helper makes the RBAC default match the agent's +actual reach in dev mode. + +An operator can still opt back into cluster-wide RBAC in dev mode by +setting --set dev.namespaceRestrictedOverride=true. Without that flag, +the dev-mode default wins regardless of rbac.namespaceRestricted. + +Returns a Go-string "true" / "false" because Helm template conditionals +compare against the boolean values produced by YAML, but downstream +templates use it in `eq ... "true"` form for clarity. +*/}} +{{- define "power-agent.effectiveNamespaceRestricted" -}} +{{- if and .Values.dev.enabled (not .Values.dev.namespaceRestrictedOverride) -}} +true +{{- else -}} +{{- .Values.rbac.namespaceRestricted | toString -}} +{{- end -}} +{{- end -}} diff --git a/deploy/helm/charts/power-agent/templates/daemonset.yaml b/deploy/helm/charts/power-agent/templates/daemonset.yaml new file mode 100644 index 000000000000..73707c78d9c5 --- /dev/null +++ b/deploy/helm/charts/power-agent/templates/daemonset.yaml @@ -0,0 +1,125 @@ +{{- include "power-agent.validateImageTag" . -}} +{{- include "power-agent.validateMutex" . -}} +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +{{- if .Values.daemonset.enabled }} +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ include "power-agent.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "power-agent.labels" . | nindent 4 }} +spec: + selector: + matchLabels: + {{- include "power-agent.selectorLabels" . | nindent 6 }} + updateStrategy: + {{- toYaml .Values.daemonset.updateStrategy | nindent 4 }} + template: + metadata: + labels: + {{- include "power-agent.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: power-agent + {{- with .Values.daemonset.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.daemonset.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ include "power-agent.serviceAccountName" . }} + # hostPID is required to read /proc/{pid}/cgroup for the host PIDs + # returned by nvmlDeviceGetComputeRunningProcesses. See power_agent.py + # _extract_pod_uid_from_cgroup. + hostPID: true + # SIGTERM handler restores default TGP on managed GPUs before exit. + terminationGracePeriodSeconds: 30 + # NVIDIA Container Toolkit injects libnvidia-ml.so via the `nvidia` + # runtime class. Pairing with NVIDIA_VISIBLE_DEVICES=all in the + # container env exposes every GPU on the node WITHOUT consuming a + # `nvidia.com/gpu` resource claim — the same pattern DCGM Exporter + # uses. Required for nvmlInit() to succeed. + runtimeClassName: {{ .Values.runtimeClassName | quote }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.daemonset.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: power-agent + image: "{{ include "power-agent.imageRef" (dict "repository" .Values.image.repository "tag" .Values.image.tag) }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: + - python + - /app/power_agent.py + - --safe-default-watts=$(SAFE_DEFAULT_WATTS) + - --node-name=$(NODE_NAME) + {{- if eq (include "power-agent.effectiveNamespaceRestricted" .) "true" }} + # Namespace-scoped RBAC is in effect (Role + RoleBinding instead of + # ClusterRole + ClusterRoleBinding — see role.yaml). The agent's + # default pod listing (`list_pod_for_all_namespaces`, power_agent.py:444) + # would 403 against the namespace-only token; pass --namespace so it + # uses `list_namespaced_pod` (:439) instead. Mirrors the dev-pod + # template's wiring (dev-pod.yaml:57). + - --namespace=$(POD_NAMESPACE) + {{- end }} + - --prometheus-port={{ .Values.agent.prometheusPort }} + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + {{- if eq (include "power-agent.effectiveNamespaceRestricted" .) "true" }} + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + {{- end }} + - name: SAFE_DEFAULT_WATTS + value: {{ .Values.agent.safeDefaultWatts | quote }} + {{- range $k, $v := .Values.env }} + - name: {{ $k }} + value: {{ $v | quote }} + {{- end }} + securityContext: + # NVML power-cap writes require root, and the agent persists + # managed-GPU UUID state to a hostPath — non-root images cannot + # write there. See plan §1.2 / power_agent.py:apply_cap. + privileged: true + readOnlyRootFilesystem: false + runAsUser: 0 + ports: + - name: metrics + containerPort: {{ .Values.agent.prometheusPort }} + protocol: TCP + resources: + {{- toYaml .Values.resources | nindent 12 }} + volumeMounts: + - name: proc + mountPath: /host/proc + readOnly: true + - name: state + mountPath: /var/lib/dynamo-power-agent + volumes: + - name: proc + hostPath: + path: /proc + - name: state + hostPath: + path: {{ .Values.state.hostPath | quote }} + type: DirectoryOrCreate +{{- end }} diff --git a/deploy/helm/charts/power-agent/templates/dev-pod.yaml b/deploy/helm/charts/power-agent/templates/dev-pod.yaml new file mode 100644 index 000000000000..e4fcf1ef096e --- /dev/null +++ b/deploy/helm/charts/power-agent/templates/dev-pod.yaml @@ -0,0 +1,102 @@ +{{- include "power-agent.validateDevImageTag" . -}} +{{- include "power-agent.validateMutex" . -}} +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# In-cluster dev-iteration harness. Renders a single Pod that mounts +# power_agent.py from an externally-created ConfigMap, pinned to one GPU +# node (dev.nodeName) and one namespace (Release.Namespace). +# +# PREREQUISITE (verified by validateMutex; failure mode is a Pending Pod +# with `MountVolume.SetUp failed`): +# +# kubectl create configmap {{ .Values.dev.scriptConfigMap }} \ +# --from-file=power_agent.py=deploy/power-agent/power_agent.py \ +# -n {{ .Release.Namespace }} + +{{- if .Values.dev.enabled }} +apiVersion: v1 +kind: Pod +metadata: + name: {{ include "power-agent.fullname" . }}-dev + namespace: {{ .Release.Namespace }} + labels: + {{- include "power-agent.labels" . | nindent 4 }} + role: dev +spec: + serviceAccountName: {{ include "power-agent.serviceAccountName" . }} + hostPID: true # /proc/{pid}/cgroup readback + restartPolicy: Never # let it stay Failed if it crashes so we can read logs + terminationGracePeriodSeconds: 30 # SIGTERM handler restores TGP + runtimeClassName: {{ .Values.runtimeClassName | quote }} + nodeName: {{ .Values.dev.nodeName | quote }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 4 }} + {{- end }} + containers: + - name: power-agent + image: "{{ include "power-agent.imageRef" (dict "repository" .Values.dev.image.repository "tag" .Values.dev.image.tag) }}" + imagePullPolicy: {{ .Values.dev.image.pullPolicy }} + command: ["bash", "-c"] + args: + - | + set -euo pipefail + echo "[dev-pod] Installing python deps (kubernetes, prometheus-client)…" + pip install --quiet --no-input kubernetes prometheus-client + echo "[dev-pod] Starting Power Agent on node=$NODE_NAME, namespace=$POD_NAMESPACE…" + # --namespace=$POD_NAMESPACE constrains agent pod-list to this + # namespace; matches the namespace-scoped RBAC dev mode installs. + exec python3 /scripts/power_agent.py \ + --safe-default-watts=$SAFE_DEFAULT_WATTS \ + --node-name=$NODE_NAME \ + --namespace=$POD_NAMESPACE \ + --prometheus-port={{ .Values.agent.prometheusPort }} + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: SAFE_DEFAULT_WATTS + value: {{ .Values.agent.safeDefaultWatts | quote }} + {{- range $k, $v := .Values.env }} + - name: {{ $k }} + value: {{ $v | quote }} + {{- end }} + securityContext: + privileged: true + readOnlyRootFilesystem: false + runAsUser: 0 + ports: + - name: metrics + containerPort: {{ .Values.agent.prometheusPort }} + protocol: TCP + resources: + {{- toYaml .Values.dev.resources | nindent 8 }} + volumeMounts: + - name: script + mountPath: /scripts + readOnly: true + - name: state + mountPath: /var/lib/dynamo-power-agent + volumes: + - name: script + configMap: + name: {{ .Values.dev.scriptConfigMap }} + items: + - key: power_agent.py + path: power_agent.py + mode: 0o755 + - name: state + hostPath: + path: {{ .Values.state.hostPath | quote }} + type: DirectoryOrCreate +{{- end }} diff --git a/deploy/helm/charts/power-agent/templates/role.yaml b/deploy/helm/charts/power-agent/templates/role.yaml new file mode 100644 index 000000000000..459fbbb6624b --- /dev/null +++ b/deploy/helm/charts/power-agent/templates/role.yaml @@ -0,0 +1,40 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +{{- if .Values.rbac.create }} +{{- $effectiveRestricted := include "power-agent.effectiveNamespaceRestricted" . }} +{{- if eq $effectiveRestricted "true" }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "power-agent.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "power-agent.labels" . | nindent 4 }} +rules: + # The agent lists pods on its own node, filtered by spec.nodeName, to map + # GPU process PIDs back to pod UIDs and read the per-pod power-limit + # annotation. When namespace-scoped RBAC is in effect (production mode + # with rbac.namespaceRestricted=true, OR dev mode), the agent is invoked + # with --namespace={{ .Release.Namespace }} via the downward API so its + # list call targets `list_namespaced_pod` (power_agent.py:438-442) + # instead of the cluster-wide default; see daemonset.yaml + dev-pod.yaml. + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch"] +{{- else }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "power-agent.fullname" . }} + labels: + {{- include "power-agent.labels" . | nindent 4 }} +rules: + # Cluster-wide pod read: the production DS sees worker pods in any + # namespace on its node (GPU workloads may be spread across tenant + # namespaces). Verbs limited to read-only — the agent never mutates pods. + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch"] +{{- end }} +{{- end }} diff --git a/deploy/helm/charts/power-agent/templates/rolebinding.yaml b/deploy/helm/charts/power-agent/templates/rolebinding.yaml new file mode 100644 index 000000000000..c02178150147 --- /dev/null +++ b/deploy/helm/charts/power-agent/templates/rolebinding.yaml @@ -0,0 +1,38 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +{{- if .Values.rbac.create }} +{{- $effectiveRestricted := include "power-agent.effectiveNamespaceRestricted" . }} +{{- if eq $effectiveRestricted "true" }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "power-agent.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "power-agent.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "power-agent.fullname" . }} +subjects: + - kind: ServiceAccount + name: {{ include "power-agent.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +{{- else }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "power-agent.fullname" . }} + labels: + {{- include "power-agent.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "power-agent.fullname" . }} +subjects: + - kind: ServiceAccount + name: {{ include "power-agent.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +{{- end }} +{{- end }} diff --git a/deploy/helm/charts/power-agent/templates/serviceaccount.yaml b/deploy/helm/charts/power-agent/templates/serviceaccount.yaml new file mode 100644 index 000000000000..23960aa776c0 --- /dev/null +++ b/deploy/helm/charts/power-agent/templates/serviceaccount.yaml @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +{{- if .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "power-agent.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "power-agent.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- with .Values.imagePullSecrets }} +imagePullSecrets: + {{- toYaml . | nindent 2 }} +{{- end }} +{{- end }} diff --git a/deploy/helm/charts/power-agent/values.yaml b/deploy/helm/charts/power-agent/values.yaml new file mode 100644 index 000000000000..f2ac32ac074f --- /dev/null +++ b/deploy/helm/charts/power-agent/values.yaml @@ -0,0 +1,167 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Power Agent — per-node NVML power-cap enforcement DaemonSet. + +image: + repository: nvcr.io/nvidia/ai-dynamo/power-agent + # REQUIRED for DaemonSet mode (daemonset.enabled, the default). Pin to a + # release tag or sha256 digest. No default to avoid the `:latest` + # reproducibility footgun CodeRabbit flagged on PR #9682. `helm install` + # fails fast at template time if this is empty while daemonset.enabled. + # Dev-pod mode (dev.enabled) ignores this and uses dev.image.* instead. + tag: "" + pullPolicy: IfNotPresent + +imagePullSecrets: [] +# Example for NGC: +# - name: ngc-secret + +agent: + # Per-SKU safety floor. The agent falls back to this when: + # - A multi-pod-per-GPU conflict cannot agree on a single cap. + # - A pod is missing the dynamo.nvidia.com/gpu-power-limit annotation + # but is running on a GPU the agent previously managed. + # Recommended: ~70% of the SKU's TDP. + # H200 SXM → 500 H100 SXM → 490 A100 SXM (80GB) → 280 + safeDefaultWatts: 500 + # NOTE: the agent's reconcile interval is currently a hardcoded module + # constant (RECONCILE_INTERVAL_S = 15 in power_agent.py). It is NOT + # exposed as a CLI flag, so plumbing it through values.yaml would create + # a dead knob. If a future power_agent.py revision adds a + # --reconcile-interval flag, add the matching key here at that time. + prometheusPort: 9100 + +# NVIDIA container toolkit needs both runtimeClassName: nvidia AND +# NVIDIA_VISIBLE_DEVICES=all to inject libnvidia-ml.so AND expose every GPU +# WITHOUT consuming a nvidia.com/gpu resource claim. Same pattern DCGM +# Exporter uses. Privileged container bypasses the unprivileged-visibility +# edge case (accept-nvidia-visible-devices-envvar-when-unprivileged=false). +runtimeClassName: nvidia +env: + NVIDIA_VISIBLE_DEVICES: "all" + NVIDIA_DRIVER_CAPABILITIES: "compute,utility" + +nodeSelector: + nvidia.com/gpu.present: "true" + +tolerations: + - key: nvidia.com/gpu + operator: Exists + effect: NoSchedule + +resources: + # Steady-state RSS of the agent (pynvml + kubernetes client + prometheus + # exporter) is ~80 MiB on an idle 8-GPU node. The previous 128Mi limit + # left only ~48 MiB of headroom — enough to break under any of the + # workloads we expect operators to run alongside the agent: + # - kubectl exec --container=power-agent -- python -c "..." for + # ad-hoc inspection (re-importing the kubernetes client in the + # same cgroup pushes RSS over 128 MiB; OOM-kills observed twice + # during 2026-05-21 live testing on AKS dpp-dev-env) + # - the agent's transient allocations during multi-pod conflict + # resolution on a node with O(50) pods (UID→annotation dict + + # PID→UUID map both held in memory) + # 256Mi gives ~3× the steady-state RSS with safe room for both + # patterns; on a fleet of 1000 nodes the extra 128MiB per node is + # 125GiB — negligible compared to the cost of a silently OOM-killed + # power-cap controller. + requests: + cpu: "50m" + memory: "96Mi" + limits: + cpu: "200m" + memory: "256Mi" + +# Persisted UUID-gated cold-start state (managed_gpus.json). +state: + hostPath: /var/lib/dynamo-power-agent + +serviceAccount: + create: true + name: "" # auto-generated from fullname if empty + annotations: {} + +rbac: + create: true + # true → Role + RoleBinding; agent only sees pods in {{ .Release.Namespace }}. + # The DaemonSet's command is also invoked with + # `--namespace=$(POD_NAMESPACE)` (downward API → metadata.namespace) + # so power_agent.py uses `list_namespaced_pod` instead of its + # default `list_pod_for_all_namespaces`. Without that flag the + # agent would 403 on every reconcile against the namespace-only + # token. See templates/daemonset.yaml and power_agent.py:438-446. + # false → ClusterRole + ClusterRoleBinding; agent sees all pods on its + # node across every namespace. `--namespace` is NOT passed. + # Default matches today's raw-YAML behaviour (cluster-scoped). Dev mode + # overrides this to true automatically — see dev.namespaceRestrictedOverride. + namespaceRestricted: false + +# Production DaemonSet mode. Mutually exclusive with dev.enabled. +daemonset: + enabled: true + + # Conservative default by design: when a Power Agent restarts, the GPUs + # on its node have no NVML cap enforcement until it comes back. If the + # planner just emitted new annotations expecting them to apply, that + # propagation stalls. `maxUnavailable: 1` ensures only one node's worth + # of GPUs ever loses enforcement at a time. + # + # Override for large fleets: on clusters with hundreds of GPU nodes, + # a 1-at-a-time rollout (~30s/pod × N nodes) gets slow. Operators can + # opt into faster rollouts with: + # --set daemonset.updateStrategy.rollingUpdate.maxUnavailable=10% + # (or any percentage / integer). The trade-off is more concurrent + # enforcement gaps during a rollout; safe when the planner's reconcile + # interval (15s) is much shorter than the rollout duration anyway. + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + + podLabels: {} + podAnnotations: {} + affinity: {} + +# In-cluster dev-iteration mode. Renders a single Pod (not a DaemonSet) +# that mounts power_agent.py from a ConfigMap. The caller is responsible +# for creating the ConfigMap separately BEFORE `helm install`: +# +# kubectl create configmap dynamo-power-agent-script \ +# --from-file=power_agent.py=deploy/power-agent/power_agent.py \ +# -n $NAMESPACE +# +# Without this ConfigMap the dev pod will stay Pending with a +# `MountVolume.SetUp failed for volume "script"` event. +# +# Mutually exclusive with daemonset.enabled (chart fails to render if both +# are true). dev.nodeName is required when dev.enabled=true. +dev: + enabled: false + # Required when enabled=true. Pin to the GPU node hosting the workload pod + # being annotated. Find a candidate with: + # kubectl get pods -n $NAMESPACE -l nvidia.com/dynamo-component-type=worker \ + # -o jsonpath='{.items[*].spec.nodeName}' + nodeName: "" + # Name of an externally-created ConfigMap holding power_agent.py at + # key "power_agent.py". See top-of-section recipe. + scriptConfigMap: dynamo-power-agent-script + # The dev pod uses vllm-runtime (already cached on every GPU node by + # the quickstart DGDs) which ships with pynvml; the agent pip-installs + # kubernetes + prometheus-client at container start. + image: + repository: nvcr.io/nvidia/ai-dynamo/vllm-runtime + tag: "1.0.1" + pullPolicy: IfNotPresent + resources: + requests: + cpu: "50m" + memory: "128Mi" + limits: + cpu: "500m" + memory: "512Mi" + # Set to true ONLY if you genuinely need cluster-wide pod-listing RBAC + # while iterating on power_agent.py (e.g., testing cross-namespace + # multi-pod conflict resolution). Default: false — dev mode forces + # namespace-scoped RBAC for principle of least privilege. + namespaceRestrictedOverride: false diff --git a/deploy/power-agent/Dockerfile b/deploy/power-agent/Dockerfile new file mode 100644 index 000000000000..22904ae35945 --- /dev/null +++ b/deploy/power-agent/Dockerfile @@ -0,0 +1,78 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Power Agent image — per-node NVML power-cap enforcement DaemonSet. +# Referenced by deploy/helm/charts/power-agent/values.yaml (image.repository) +# and built by the `power-agent` CI job (.github/workflows/pr.yaml, +# post-merge-ci.yml). +# +# Two stages: +# * `runtime` — the slim, shipped image (built/pushed with `--target runtime`). +# * `test` — `FROM runtime` + pytest + tests; its `RUN pytest` executes the +# unit suite *inside the shipped image layers* so CI validates +# the container itself (same Python 3.12 + same baked deps), not +# a separately pip-installed runner env. Never pushed. +# +# # shipped image: +# docker build --target runtime -t power-agent:dev -f deploy/power-agent/Dockerfile \ +# deploy/power-agent +# # run unit tests inside the image (CI `power-agent-test` job): +# docker build --target test -f deploy/power-agent/Dockerfile deploy/power-agent + +# DOCKER_PROXY lets CI route the base image through the ECR Docker Hub mirror +# (e.g. .dkr.ecr..amazonaws.com/dockerhub/). Empty by default so +# `docker build` works unchanged for local/dev use straight from Docker Hub. +ARG DOCKER_PROXY= + +# Slim base — the NVIDIA Container Toolkit injects libnvidia-ml.so at runtime +# via runtimeClassName: nvidia (set by the Helm chart), so the image does NOT +# bake pynvml's native dependency. nvmlInit() resolves the host driver lib. +# +# Keep the explicit `library/` namespace: ECR's Docker Hub pull-through cache +# does not auto-expand bare official-image names (`python` -> `library/python`) +# the way a direct Docker Hub pull does, so the proxied tag fails to pull +# without it. `library/python` is also a valid fully-qualified reference when +# DOCKER_PROXY is empty, so local/dev builds are unaffected. +FROM ${DOCKER_PROXY}library/python:3.12-slim-bookworm AS runtime + +ENV PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + PIP_NO_CACHE_DIR=1 \ + PIP_DISABLE_PIP_VERSION_CHECK=1 + +# Runtime deps. These match power_agent.py's three optional imports: +# pynvml (NVML bindings), kubernetes (pod listing), prometheus_client (metrics). +RUN pip install --no-cache-dir \ + pynvml==12.0.* \ + kubernetes==30.* \ + prometheus-client==0.20.* + +WORKDIR /app +COPY power_agent.py /app/power_agent.py + +# NVML power-cap writes require root, and the agent persists managed-GPU UUID +# state to a hostPath volume. The Helm chart also sets runAsUser: 0 / privileged. +USER 0 + +# The Helm chart's `command:` owns the real argv (it injects --safe-default-watts, +# --node-name, --prometheus-port, etc.). This ENTRYPOINT is for operators who run +# the image directly outside the chart (dev / debugging). +ENTRYPOINT ["python", "/app/power_agent.py"] +CMD ["--help"] + +# --------------------------------------------------------------------------- +# test stage — CI-only. Builds FROM the shipped `runtime` image so the unit +# suite runs against the exact Python 3.12 + baked pynvml/kubernetes/ +# prometheus-client versions that ship, then fails the build if any test fails. +# Not referenced by `--target runtime`, so it never lands in the pushed image. +# --------------------------------------------------------------------------- +FROM runtime AS test + +RUN pip install --no-cache-dir pytest + +# pytest.ini sets pythonpath=. so `from power_agent import …` resolves against +# /app (WORKDIR) where power_agent.py already lives in the runtime layer. +COPY pytest.ini /app/pytest.ini +COPY tests/ /app/tests/ + +RUN python -m pytest tests -v diff --git a/deploy/power-agent/power_agent.py b/deploy/power-agent/power_agent.py new file mode 100755 index 000000000000..e58bf56ff607 --- /dev/null +++ b/deploy/power-agent/power_agent.py @@ -0,0 +1,787 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Power Agent DaemonSet — Phase 1 implementation. + +Runs as a privileged DaemonSet (hostPID: true) on each GPU node. Every 15s: + 1. Lists pods on this node via the K8s API. + 2. For each physical GPU: nvmlDeviceGetComputeRunningProcesses() → PID list. + 3. For each PID: reads /proc/{pid}/cgroup → extracts pod UID. + 4. Looks up the pod's dynamo.nvidia.com/gpu-power-limit annotation. + 5. Calls nvmlDeviceSetPowerManagementLimit(handle, watts × 1000). + +Scope is opt-in: the agent only ever caps a GPU whose pod carries the +dynamo.nvidia.com/gpu-power-limit annotation (set by the planner on +prefill/decode worker pods). A GPU running only unannotated pods — a +non-Dynamo workload, or a Dynamo worker not yet annotated — that the agent +never capped is left at its hardware default and untouched. If the agent had +previously capped that GPU and the opted-in pod is now gone (a non-managed +workload reuses it, or the planner removed the annotation), the cap is +released back to default so it does not strand on the new tenant. See +``_build_uid_to_annotation`` and ``_release_managed_gpu``. + +SIGTERM handler: restores default TDP on all managed GPUs before shutdown. +Cold-start orphan recovery: UUID-gated (persisted to /var/lib/dynamo-power-agent/). +""" + +import argparse +import json +import logging +import os +import re +import signal +import threading +from typing import Optional + +# Kubernetes and NVML — imported lazily with clear error messages +try: + import pynvml +except ImportError: + pynvml = None # type: ignore + +try: + from kubernetes import client as k8s_client + from kubernetes import config as k8s_config + from kubernetes.config.config_exception import ConfigException +except ImportError: + k8s_client = None # type: ignore + k8s_config = None # type: ignore + ConfigException = Exception # type: ignore + +try: + from prometheus_client import Counter, Gauge, start_http_server + + _PROMETHEUS_AVAILABLE = True +except ImportError: + _PROMETHEUS_AVAILABLE = False + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(name)s %(message)s", +) +logger = logging.getLogger("power_agent") + +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- + +POWER_ANNOTATION_KEY = "dynamo.nvidia.com/gpu-power-limit" +RECONCILE_INTERVAL_S = 15 +_MANAGED_STATE_PATH = "/var/lib/dynamo-power-agent/managed_gpus.json" + +# --------------------------------------------------------------------------- +# cgroup pod-UID extraction +# Handles cgroup v1 (multi-line) and v2 (single-line), systemd / cgroupfs +# drivers, Guaranteed / Burstable / BestEffort QoS, cri-containerd / cri-o. +# --------------------------------------------------------------------------- + +_SYSTEMD_RE = re.compile( + r"kubepods-(?:burstable-|besteffort-)?pod([a-fA-F0-9_]+)\.slice" +) +_CGROUPFS_RE = re.compile( + r"/kubepods(?:/burstable|/besteffort)?/pod([a-fA-F0-9-]+)(?:/|$)" +) + + +def _extract_pod_uid_from_cgroup(pid: int) -> Optional[str]: + """Recover the pod UID from /proc/{pid}/cgroup. + + Iterates lines because cgroup v1 has one line per controller hierarchy + while cgroup v2 has a single unified line. Uses .search() so wrapper + segments (cri-containerd, cri-o, dockershim) don't defeat the match. + Returns None for non-K8s processes — callers skip them silently. + """ + try: + with open(f"/proc/{pid}/cgroup") as f: + lines = f.read().splitlines() + except OSError: + return None + for line in lines: + m = _SYSTEMD_RE.search(line) + if m: + # systemd encodes dashes as underscores in the pod-UID segment + return m.group(1).replace("_", "-") + m = _CGROUPFS_RE.search(line) + if m: + return m.group(1) + return None # non-K8s process — skip + + +# --------------------------------------------------------------------------- +# Persistent managed-GPU state (UUID-gated orphan recovery) +# --------------------------------------------------------------------------- + +_previously_managed: set[str] = set() + + +def _load_previously_managed_gpus() -> set[str]: + """Load the persisted set of UUIDs this agent previously capped. + + Defensive parsing — corrupt / malformed state files must never crash + the agent's startup. Per PR #9682 CodeRabbit review, this catches a + superset of the original (FileNotFoundError, JSONDecodeError) cases: + + * OSError (PermissionError, IsADirectoryError, I/O errors) — disk + problems on the host volume should NOT brick the agent. + * Non-dict JSON root — a file with a top-level list / int / string + / null would have crashed ``.get(...)`` with ``AttributeError``. + * Non-list ``managed_uuids`` — a misshapen value would have crashed + ``set(non_iterable)`` with ``TypeError``. + + Returning empty means we lose the orphan-recovery opportunity for + this restart, which is strictly better than CrashLoopBackOff with + no caps actuated. + """ + try: + with open(_MANAGED_STATE_PATH) as f: + payload = json.load(f) + except (OSError, json.JSONDecodeError) as e: + logger.warning("Failed to read managed GPU state: %s", e) + return set() + if not isinstance(payload, dict): + logger.warning( + "Managed GPU state has non-dict root: %s", type(payload).__name__ + ) + return set() + uuids = payload.get("managed_uuids", []) + if not isinstance(uuids, list): + logger.warning("managed_uuids is not a list: %s", type(uuids).__name__) + return set() + return set(uuids) + + +def _persist_managed_gpus(uuids: set[str]) -> None: + os.makedirs(os.path.dirname(_MANAGED_STATE_PATH), exist_ok=True) + tmp = _MANAGED_STATE_PATH + ".tmp" + with open(tmp, "w") as f: + json.dump({"managed_uuids": sorted(uuids)}, f) + os.replace(tmp, _MANAGED_STATE_PATH) # atomic rename + + +def _nvml_uuid(handle) -> str: + """Return the GPU UUID as ``str`` regardless of pynvml major version. + + The legacy ``pynvml`` package (NVIDIA bindings) returns ``bytes`` and + callers ``.decode("ascii")`` themselves. ``nvidia-ml-py`` (the + officially supported successor and what newer pip releases install + under the name ``pynvml``) returns ``str`` directly, and an + unconditional ``.decode()`` raises ``AttributeError``. Callers must + go through this helper. + """ + uuid = pynvml.nvmlDeviceGetUUID(handle) + return uuid.decode("ascii") if isinstance(uuid, bytes) else uuid + + +def _record_managed_gpu_uuid(handle) -> None: + """Called from _apply_cap() after every successful NVML write.""" + uuid = _nvml_uuid(handle) + if uuid not in _previously_managed: + _previously_managed.add(uuid) + _persist_managed_gpus(_previously_managed) + + +# --------------------------------------------------------------------------- +# Prometheus metrics +# --------------------------------------------------------------------------- + + +class _NoopMetric: + def labels(self, **_): + return self + + def set(self, _): + pass + + def inc(self, _=1): + pass + + +class PowerAgentMetrics: + def __init__(self, prometheus_port: int = 0) -> None: + if _PROMETHEUS_AVAILABLE and prometheus_port > 0: + self.applied_limit_watts = Gauge( + "dynamo_power_agent_applied_limit_watts", + "NVML cap currently applied per physical GPU (watts).", + labelnames=("gpu",), + ) + self.multi_pod_gpu_total = Counter( + "dynamo_power_agent_multi_pod_gpu_total", + "Times a physical GPU had multiple pods (agree or conflict).", + labelnames=("disposition",), + ) + self.apply_failures_total = Counter( + "dynamo_power_agent_apply_failures_total", + "Times the agent failed to set an NVML power cap.", + ) + self.safe_default_applied_total = Counter( + "dynamo_power_agent_safe_default_applied_total", + "Times the safe-default cap was used (conflict or cold-start parse failure).", + ) + self.cap_clamped_total = Counter( + "dynamo_power_agent_cap_clamped_total", + "Times a requested cap was clamped to SKU NVML constraints.", + labelnames=("direction",), + ) + try: + start_http_server(prometheus_port) + logger.info( + "Prometheus metrics server started on port %d", prometheus_port + ) + except Exception as e: + logger.warning("Failed to start Prometheus server: %s", e) + else: + noop = _NoopMetric() + self.applied_limit_watts = noop + self.multi_pod_gpu_total = noop + self.apply_failures_total = noop + self.safe_default_applied_total = noop + self.cap_clamped_total = noop + + +# --------------------------------------------------------------------------- +# NVML helpers +# --------------------------------------------------------------------------- + + +def _clamp_to_constraints( + handle, requested_w: int, gpu_idx: int, metrics: PowerAgentMetrics +) -> int: + """Clamp `requested_w` to the SKU-defined NVML power-cap range.""" + try: + min_mw, max_mw = pynvml.nvmlDeviceGetPowerManagementLimitConstraints(handle) + except pynvml.NVMLError: + return requested_w + min_w, max_w = min_mw // 1000, max_mw // 1000 + if requested_w < min_w: + logger.warning( + "Requested cap %d W below SKU min %d W on GPU %d; clamping up.", + requested_w, + min_w, + gpu_idx, + ) + metrics.cap_clamped_total.labels(direction="min").inc() + return min_w + if requested_w > max_w: + logger.warning( + "Requested cap %d W above SKU max %d W on GPU %d; clamping down.", + requested_w, + max_w, + gpu_idx, + ) + metrics.cap_clamped_total.labels(direction="max").inc() + return max_w + return requested_w + + +_managed_gpu_indices: set[int] = set() + + +def _apply_cap( + handle, gpu_idx: int, requested_w: int, metrics: PowerAgentMetrics +) -> None: + """Apply NVML power cap. All writes go through here.""" + effective_w = _clamp_to_constraints(handle, requested_w, gpu_idx, metrics) + try: + pynvml.nvmlDeviceSetPowerManagementLimit(handle, effective_w * 1000) + _managed_gpu_indices.add(gpu_idx) + _record_managed_gpu_uuid(handle) + metrics.applied_limit_watts.labels(gpu=str(gpu_idx)).set(effective_w) + except pynvml.NVMLError as e: + logger.error( + "nvmlDeviceSetPowerManagementLimit GPU %d → %d W failed: %s", + gpu_idx, + effective_w, + e, + ) + metrics.apply_failures_total.inc() + + +def _release_managed_gpu(handle, gpu_idx: int) -> None: + """Restore default TGP on a GPU we previously capped, and unmanage it. + + Runtime counterpart to ``_handle_sigterm`` / ``_restore_orphaned_gpus_on_startup``. + Invoked from steady-state reconcile when a GPU we previously capped is now + running only unannotated / non-K8s processes — i.e. the opted-in pod is gone + and a non-managed workload owns the GPU (or the planner removed the + annotation to release it). Without this, the agent's last cap would strand + on the reused GPU until the next agent shutdown (startup orphan recovery + skips busy GPUs), silently throttling the new tenant. This implements the + "planner owns cap lifecycle via annotation removal/update" contract at + runtime. + + Eligibility is UUID-gated so caps set by other tooling are never touched. + A GPU is "ours" if it is in ``_managed_gpu_indices`` (capped in THIS process) + OR its UUID is in the persisted ``_previously_managed`` set (capped in a + prior process). The latter is essential across restarts: ``_managed_gpu_indices`` + is in-memory and empty after a restart, while ``_previously_managed`` is + loaded from disk — without it, a GPU capped before the restart and now busy + with only unannotated work would keep the stale cap (startup orphan recovery + only restores *idle* GPUs). + + The idle case (no processes at all) is intentionally NOT handled here; + ``_reconcile_gpu``'s ``not procs`` branch keeps the cap for a briefly-exited + worker that will return to the same GPU. + """ + try: + uuid = _nvml_uuid(handle) + except Exception as e: + logger.warning( + "Failed to read UUID for GPU %d during release check: %s", gpu_idx, e + ) + return + if gpu_idx not in _managed_gpu_indices and uuid not in _previously_managed: + return # not a GPU this agent capped — leave it alone (UUID-gating) + try: + default_mw = pynvml.nvmlDeviceGetPowerManagementDefaultLimit(handle) + current_mw = pynvml.nvmlDeviceGetPowerManagementLimit(handle) + if current_mw < default_mw: + pynvml.nvmlDeviceSetPowerManagementLimit(handle, default_mw) + logger.info( + "Released cap on GPU %d (%d W → %d W): previously managed, now " + "running only unannotated/non-K8s processes.", + gpu_idx, + current_mw // 1000, + default_mw // 1000, + ) + except Exception as e: + # Leave the GPU in the managed set so a later cycle retries the release. + logger.warning("Failed to release cap on GPU %d: %s", gpu_idx, e) + return + _managed_gpu_indices.discard(gpu_idx) + if uuid in _previously_managed: + _previously_managed.discard(uuid) + _persist_managed_gpus(_previously_managed) + + +# --------------------------------------------------------------------------- +# SIGTERM handler +# --------------------------------------------------------------------------- + +_shutdown = threading.Event() + + +def _handle_sigterm(signum, frame): + logger.info( + "SIGTERM received — restoring default TGP on managed GPUs and shutting down." + ) + for gpu_idx in list(_managed_gpu_indices): + try: + handle = pynvml.nvmlDeviceGetHandleByIndex(gpu_idx) + default_mw = pynvml.nvmlDeviceGetPowerManagementDefaultLimit(handle) + pynvml.nvmlDeviceSetPowerManagementLimit(handle, default_mw) + logger.info( + "Restored GPU %d to default TGP (%d W)", gpu_idx, default_mw // 1000 + ) + except Exception as e: + logger.exception("Failed to restore TGP on GPU %d: %s", gpu_idx, e) + try: + pynvml.nvmlShutdown() + except Exception: + # We MUST proceed to ``_shutdown.set()`` so the run loop unblocks + # and the container exits cleanly — re-raising here would leave + # the agent hung on SIGTERM. But silently dropping the failure + # made shutdown-time NVML faults impossible to diagnose from pod + # logs (PR #9682 CodeRabbit review). ``logger.exception`` writes + # the full traceback at ERROR level so operators can correlate + # with driver / hostengine events. + logger.exception("nvmlShutdown raised; proceeding with agent exit anyway.") + _shutdown.set() + + +# --------------------------------------------------------------------------- +# Orphan cap restoration on startup (UUID-gated) +# --------------------------------------------------------------------------- + + +def _restore_orphaned_gpus_on_startup(device_count: int) -> None: + """Restore default TDP only on GPUs this agent previously capped AND that are now idle. + + UUID-gating prevents touching caps applied by other workflows (different DGD, + manual nvidia-smi -pl, vendor firmware defaults). + """ + global _previously_managed + _previously_managed = _load_previously_managed_gpus() + for gpu_idx in range(device_count): + try: + handle = pynvml.nvmlDeviceGetHandleByIndex(gpu_idx) + uuid = _nvml_uuid(handle) + if uuid not in _previously_managed: + continue + procs = pynvml.nvmlDeviceGetComputeRunningProcesses(handle) + if procs: + continue # workload running — let normal reconcile handle it + current_mw = pynvml.nvmlDeviceGetPowerManagementLimit(handle) + default_mw = pynvml.nvmlDeviceGetPowerManagementDefaultLimit(handle) + if current_mw < default_mw: + pynvml.nvmlDeviceSetPowerManagementLimit(handle, default_mw) + logger.info( + "Restored orphaned cap on idle GPU %d (%d W → %d W).", + gpu_idx, + current_mw // 1000, + default_mw // 1000, + ) + _previously_managed.discard(uuid) + except Exception as e: + logger.warning("orphan-restore failed for GPU %d: %s", gpu_idx, e) + _persist_managed_gpus(_previously_managed) + + +# --------------------------------------------------------------------------- +# Multi-pod-per-GPU policy +# --------------------------------------------------------------------------- + + +def _resolve_cap_for_gpu( + gpu_idx: int, + pod_annotations: list[tuple[str, Optional[str]]], + safe_default_watts: int, + metrics: PowerAgentMetrics, +) -> int: + """Determine the NVML cap to apply for a GPU given the pod annotations on it. + + Policy (``None`` annotations are filtered out before conflict resolution): + - 1 pod with annotation → use that value. + - 2+ pods, all agree → use agreed value, WARNING (multi-pod is misconfig). + - 2+ pods, conflict → use safe_default_watts, ERROR. + - No parseable annotation → use safe_default_watts, ERROR. + + Mixed ``None`` + valid (e.g. ``[None, "480"]``) is intentionally lenient: + the ``None`` pod is dropped and the single remaining value (480) is + applied (this still fires the "all agree" multi-pod WARNING because + ``len(pod_annotations) > 1``). We apply the only constraint available + rather than fail-closed to safe_default — we do not penalise an annotated + pod for a co-located unannotated one. Multi-pod-per-GPU remains an + unsupported topology regardless. Per PR #9682 @sttts review. + + Returns the cap in watts. + """ + values = [v for _, v in pod_annotations if v is not None] + if not values: + logger.error( + "GPU %d: no parseable annotation on any pod; applying safe default (%d W).", + gpu_idx, + safe_default_watts, + ) + metrics.apply_failures_total.inc() + metrics.safe_default_applied_total.inc() + return safe_default_watts + + unique = set(values) + if len(pod_annotations) > 1: + if len(unique) == 1: + logger.warning( + "GPU %d: %d pods all agree on cap %s W (multi-pod-per-GPU is unsupported topology).", + gpu_idx, + len(pod_annotations), + values[0], + ) + metrics.multi_pod_gpu_total.labels(disposition="agree").inc() + else: + logger.error( + "GPU %d: %d pods with conflicting caps %s; applying safe default (%d W).", + gpu_idx, + len(pod_annotations), + sorted(unique), + safe_default_watts, + ) + metrics.multi_pod_gpu_total.labels(disposition="conflict").inc() + metrics.safe_default_applied_total.inc() + return safe_default_watts + + try: + return int(values[0]) + except (ValueError, TypeError): + logger.error( + "GPU %d: annotation value %r is not an integer; applying safe default (%d W).", + gpu_idx, + values[0], + safe_default_watts, + ) + metrics.apply_failures_total.inc() + metrics.safe_default_applied_total.inc() + return safe_default_watts + + +# --------------------------------------------------------------------------- +# Main reconcile loop +# --------------------------------------------------------------------------- + + +class PowerAgent: + def __init__( + self, + safe_default_watts: int, + node_name: Optional[str] = None, + k8s_namespace: Optional[str] = None, + prometheus_port: int = 0, + ) -> None: + self.safe_default_watts = safe_default_watts + self.node_name = node_name or os.environ.get("NODE_NAME", "") + self.k8s_namespace = k8s_namespace + self.metrics = PowerAgentMetrics(prometheus_port) + + if pynvml is None: + raise RuntimeError("pynvml is required — install pynvml or nvidia-ml-py") + if k8s_client is None: + raise RuntimeError("kubernetes Python SDK is required — install kubernetes") + + pynvml.nvmlInit() + self.device_count = pynvml.nvmlDeviceGetCount() + logger.info( + "NVML initialized. %d GPU(s) found on this node.", self.device_count + ) + + _restore_orphaned_gpus_on_startup(self.device_count) + + # K8s client + try: + k8s_config.load_incluster_config() + except ConfigException: + k8s_config.load_kube_config() + self._core_v1 = k8s_client.CoreV1Api() + + def _list_pods_on_node(self) -> Optional[list]: + """List all pods scheduled on this node. + + Returns the pod list on success (an empty list is a *valid* success + result, meaning this node genuinely hosts no pods), or ``None`` to + signal that the listing FAILED (API error). + + The ``None`` sentinel is deliberate and load-bearing: callers MUST + distinguish "the API call failed" from "this node has zero pods". + Returning ``[]`` for both would let a transient apiserver error look + identical to an empty node, silently re-deriving every GPU's cap from + a zero-pod view. ``reconcile_once`` keys its fail-safe (skip the cycle, + freeze each GPU at its last-known-good cap) off this ``None`` — so do + NOT collapse the failure path back to ``[]``. + """ + try: + field_selector = ( + f"spec.nodeName={self.node_name}" if self.node_name else None + ) + # TODO(#9682 follow-up): this polls a full pod LIST per agent every + # RECONCILE_INTERVAL_S. Even with the node field-selector that is one + # apiserver request per node per cycle, so aggregate request rate + # grows linearly with cluster size (~N/interval LISTs/s fleet-wide: + # ~66/s at 1000 nodes, ~330/s at 5000). It will not surface in tests + # or small clusters, only at production scale. The real fix is a + # watch/informer-backed local pod cache (one initial LIST + a + # streamed watch per node, as kubelet does) so steady-state cost is + # N idle watch connections instead of N LISTs every cycle. Tracked + # for a follow-up PR; see PR #9682 @sttts review. + # + # Interim mitigation: resource_version="0" lets the apiserver serve + # the LIST from its watch cache instead of reading through to etcd, + # which relieves etcd pressure (it does NOT change the request-rate + # shape). The tradeoff is "Any" list consistency: the result may be + # slightly stale and is not a quorum-consistent "most recent" read + # (https://kubernetes.io/docs/reference/using-api/api-concepts/#semantics-for-list-and-watch). + # That is acceptable for this MVP because reconcile is periodic, live + # GPU ownership is still checked from host PIDs each cycle, and a + # stale pod view delays convergence rather than changing the + # failure-path contract. + if self.k8s_namespace: + result = self._core_v1.list_namespaced_pod( + namespace=self.k8s_namespace, + field_selector=field_selector, + resource_version="0", + ) + else: + result = self._core_v1.list_pod_for_all_namespaces( + field_selector=field_selector, + resource_version="0", + ) + return result.items + except Exception as e: + # Explicit failure result — see the contract in the docstring. + # Returning None (not []) is what keeps the reconcile fail-safe. + logger.warning("Failed to list pods on node: %s", e) + return None + + def _build_uid_to_annotation(self, pods: list) -> dict[str, Optional[str]]: + """Map pod UID → power-limit annotation value, for opted-in pods only. + + Scope-by-annotation-key: a pod is in scope **only** if it actually + carries ``POWER_ANNOTATION_KEY``. Pods without the key are omitted + from the map entirely. + + This omission is load-bearing on shared/multi-tenant nodes. + ``_reconcile_gpu`` decides whether a GPU is managed by testing + ``uid in uid_to_annotation``; if an unannotated pod were added here + with a ``None`` value, a GPU running only that pod would still build a + non-empty ``pod_annotations`` and fall through to the "no parseable + annotation → safe default" branch in ``_resolve_cap_for_gpu`` — i.e. + the agent would silently power-cap a co-located non-Dynamo workload (or + a Dynamo worker the planner has not yet annotated). Gating on key + presence is what keeps the agent from touching GPUs it was never asked + to manage. The planner is the sole writer of this key and stamps it + only on prefill/decode worker pods. Do NOT reintroduce unannotated pods + with a ``None`` value. + + A pod that carries the key but with a malformed/empty value IS kept + (value as-is) so the safe-default fail-safe still applies to a + genuinely-managed pod whose annotation is broken. + """ + result: dict[str, Optional[str]] = {} + for pod in pods: + annotations = pod.metadata.annotations or {} + if POWER_ANNOTATION_KEY in annotations: + result[pod.metadata.uid] = annotations[POWER_ANNOTATION_KEY] + return result + + def reconcile_once(self) -> None: + """Run one reconcile cycle: list pods, map PIDs→UIDs, apply caps.""" + pods = self._list_pods_on_node() + if pods is None: + # Fail-safe: the pod listing failed (API error), so we have no + # trustworthy view of which pods own which GPUs this cycle. We + # deliberately SKIP the reconcile rather than proceed with an + # empty view — skipping freezes each GPU at its last-known-good + # cap until the next successful cycle, which is strictly safer + # than un-capping or re-deriving caps from a zero-pod snapshot. + # The cap state lives on the GPU (NVML) and the agent's managed + # set, so a skipped cycle loses nothing. + logger.warning( + "Pod listing unavailable this cycle; skipping reconcile to " + "preserve last-known-good caps." + ) + return + uid_to_annotation = self._build_uid_to_annotation(pods) + + for gpu_idx in range(self.device_count): + try: + self._reconcile_gpu(gpu_idx, uid_to_annotation) + except Exception as e: + logger.error("Reconcile failed for GPU %d: %s", gpu_idx, e) + + def _reconcile_gpu( + self, + gpu_idx: int, + uid_to_annotation: dict[str, Optional[str]], + ) -> None: + handle = pynvml.nvmlDeviceGetHandleByIndex(gpu_idx) + procs = pynvml.nvmlDeviceGetComputeRunningProcesses(handle) + if not procs: + # No process on this GPU this cycle. We deliberately DO NOT restore + # the cap to default TDP here — caps are persistent by design while + # the GPU is idle. A managed worker may exit briefly (OOM, + # reschedule) and return to the same GPU; restoring during that gap + # would violate the planner's power budget. An idle previously- + # managed GPU is restored by ``_handle_sigterm`` (agent shutdown) + # and ``_restore_orphaned_gpus_on_startup`` (at agent start). A + # previously-managed GPU that is *busy* with only unannotated work + # is released below via ``_release_managed_gpu`` (the opted-in pod + # is gone, so a stale cap must not strand on the new tenant). Per + # PR #9682 @sttts review. + return + + # Deduplicate by pod UID before building ``pod_annotations``. A + # single pod commonly runs multiple GPU processes (one per rank + # in a TP/PP/EP topology, helper workers, profilers, etc.); the + # pre-fix code emitted one entry per PID and would treat a + # one-pod / two-PID GPU as if two pods were colocated. That + # both fired the spurious "multi-pod-per-GPU" WARNING and, when + # the pod's annotation was missing/invalid, took the + # conflict-resolution branch in ``_resolve_cap_for_gpu`` (since + # ``len(pod_annotations) > 1`` was true), incorrectly applying + # safe_default + bumping multi_pod_gpu_total. Per PR #9682 + # CodeRabbit review. + seen_uids: set[str] = set() + pod_annotations: list[tuple[str, Optional[str]]] = [] + for proc in procs: + uid = _extract_pod_uid_from_cgroup(proc.pid) + if uid is None: + continue # non-K8s process — skip + if uid in seen_uids: + continue # already counted this pod via an earlier PID + if uid in uid_to_annotation: # opted-in: carries POWER_ANNOTATION_KEY + seen_uids.add(uid) + pod_annotations.append((uid, uid_to_annotation[uid])) + + if not pod_annotations: + # No opted-in pod owns this GPU (every process is either non-K8s or + # belongs to a pod without POWER_ANNOTATION_KEY). Two sub-cases, + # both handled by _release_managed_gpu's UUID-gated eligibility: + # * never managed by us → left at hardware default (the scope + # boundary — see _build_uid_to_annotation). + # * previously managed by us (this process OR a prior one, via the + # persisted UUID set) → the opted-in pod is gone and a + # non-managed workload now runs here, so release our cap rather + # than strand it on the new tenant until shutdown. + # The idle case (no processes) is handled by the `not procs` branch + # above, which keeps the cap for a briefly-exited worker. + _release_managed_gpu(handle, gpu_idx) + return + + cap_w = _resolve_cap_for_gpu( + gpu_idx, pod_annotations, self.safe_default_watts, self.metrics + ) + _apply_cap(handle, gpu_idx, cap_w, self.metrics) + + def run(self) -> None: + """Main reconcile loop. Blocks until SIGTERM.""" + signal.signal(signal.SIGTERM, _handle_sigterm) + signal.signal(signal.SIGINT, _handle_sigterm) + + logger.info( + "Power Agent started. Node=%s, safe_default=%dW, interval=%ds", + self.node_name or "(all)", + self.safe_default_watts, + RECONCILE_INTERVAL_S, + ) + + while not _shutdown.is_set(): + try: + self.reconcile_once() + except Exception as e: + logger.exception("Unexpected error in reconcile loop: %s", e) + _shutdown.wait(timeout=RECONCILE_INTERVAL_S) + + logger.info("Power Agent shut down.") + + +# --------------------------------------------------------------------------- +# Entry point +# --------------------------------------------------------------------------- + + +def main() -> None: + parser = argparse.ArgumentParser(description="Dynamo Power Agent DaemonSet") + parser.add_argument( + "--safe-default-watts", + type=int, + required=True, + help="Per-GPU fail-closed cap (watts) applied when annotation parsing fails.", + ) + parser.add_argument( + "--node-name", + type=str, + default=os.environ.get("NODE_NAME", ""), + help="K8s node name (defaults to NODE_NAME env var).", + ) + parser.add_argument( + "--namespace", + type=str, + default=None, + help="Restrict pod watch to this K8s namespace. Default: all namespaces.", + ) + parser.add_argument( + "--prometheus-port", + type=int, + default=int(os.environ.get("PROMETHEUS_PORT", "0")), + help="Port for Prometheus metrics (0 = disabled).", + ) + args = parser.parse_args() + + agent = PowerAgent( + safe_default_watts=args.safe_default_watts, + node_name=args.node_name, + k8s_namespace=args.namespace, + prometheus_port=args.prometheus_port, + ) + agent.run() + + +if __name__ == "__main__": + main() diff --git a/deploy/power-agent/pytest.ini b/deploy/power-agent/pytest.ini new file mode 100644 index 000000000000..3b0a5cc244c0 --- /dev/null +++ b/deploy/power-agent/pytest.ini @@ -0,0 +1,17 @@ +# Standalone pytest config for the power-agent deployable unit. +# +# Power-agent lives under deploy/ as a self-contained image + chart. Keeping +# this file here makes deploy/power-agent the pytest rootdir, so the +# component's tests do NOT inherit +# the monorepo-root pyproject.toml [tool.pytest.ini_options] — that config sets +# filterwarnings=error with TRTLLM/pydantic-specific filters (and extra plugins) +# that are absent from the power-agent CI image, which installs only +# pytest + pynvml + kubernetes + prometheus-client (see .github/workflows/pr.yaml +# and post-merge-ci.yml power-agent job). +[pytest] +testpaths = tests +# rootdir is deploy/power-agent (this file); "." puts it on sys.path so +# `from power_agent import …` resolves both locally and inside the container +# image's test stage (WORKDIR /app, where power_agent.py + this file live). +pythonpath = . +addopts = -ra --strict-markers --strict-config diff --git a/deploy/power-agent/tests/__init__.py b/deploy/power-agent/tests/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/deploy/power-agent/tests/test_annotation_scope.py b/deploy/power-agent/tests/test_annotation_scope.py new file mode 100644 index 000000000000..54e365e18adc --- /dev/null +++ b/deploy/power-agent/tests/test_annotation_scope.py @@ -0,0 +1,208 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Opt-in scope: the agent only caps GPUs owned by annotated pods (PR #9682 review). + +`_build_uid_to_annotation` is scope-by-annotation-key: a pod is in scope only +if it carries ``dynamo.nvidia.com/gpu-power-limit``. A GPU running only +unannotated pods — a co-located non-Dynamo workload, or a Dynamo worker the +planner has not yet annotated — must be left at its hardware default and never +written, instead of being silently capped to the safe default. + +A pod that *does* carry the key but with a malformed value stays in scope so +the safe-default fail-safe still protects a genuinely-managed pod. +""" + +import types +import unittest +from unittest.mock import MagicMock, patch + +import power_agent +from power_agent import POWER_ANNOTATION_KEY, PowerAgent + +SAFE_DEFAULT = 500 + + +def _pod(uid: str, annotations): + """Fake K8s pod object exposing ``metadata.uid`` / ``metadata.annotations``.""" + return types.SimpleNamespace( + metadata=types.SimpleNamespace(uid=uid, annotations=annotations) + ) + + +def _proc(pid: int): + return types.SimpleNamespace(pid=pid) + + +def _make_agent(device_count: int = 1) -> PowerAgent: + """Build a PowerAgent without touching NVML / K8s (bypass __init__).""" + agent = object.__new__(PowerAgent) + agent.node_name = "node-under-test" + agent.k8s_namespace = None + agent.device_count = device_count + agent.safe_default_watts = SAFE_DEFAULT + agent.metrics = MagicMock() + return agent + + +class TestBuildUidToAnnotationScope(unittest.TestCase): + def test_unannotated_pods_are_omitted(self): + agent = _make_agent() + pods = [ + _pod("annotated", {POWER_ANNOTATION_KEY: "480"}), + _pod("no-annotations-at-all", None), + _pod("other-annotations", {"team.example.com/foo": "bar"}), + ] + mapping = agent._build_uid_to_annotation(pods) + self.assertEqual(mapping, {"annotated": "480"}) + self.assertNotIn("no-annotations-at-all", mapping) + self.assertNotIn("other-annotations", mapping) + + def test_malformed_value_stays_in_scope(self): + """Key present but value broken/empty → kept, so the fail-safe still fires.""" + agent = _make_agent() + pods = [ + _pod("bad", {POWER_ANNOTATION_KEY: "not-a-number"}), + _pod("empty", {POWER_ANNOTATION_KEY: ""}), + ] + mapping = agent._build_uid_to_annotation(pods) + self.assertEqual(mapping, {"bad": "not-a-number", "empty": ""}) + + +class TestReconcileScope(unittest.TestCase): + def setUp(self): + power_agent._managed_gpu_indices.clear() + power_agent._previously_managed.clear() + + def tearDown(self): + power_agent._managed_gpu_indices.clear() + power_agent._previously_managed.clear() + + def test_unannotated_gpu_active_pod_is_left_untouched(self): + """A GPU whose only live process belongs to an unannotated pod gets + NO NVML write — not even the safe default.""" + agent = _make_agent(device_count=1) + pods = [_pod("bystander", {"team.example.com/foo": "bar"})] + uid_to_annotation = agent._build_uid_to_annotation(pods) + + with patch("power_agent.pynvml") as mock_nvml, patch( + "power_agent._extract_pod_uid_from_cgroup", return_value="bystander" + ), patch("power_agent._apply_cap") as mock_apply: + mock_nvml.nvmlDeviceGetHandleByIndex.return_value = "handle-0" + mock_nvml.nvmlDeviceGetComputeRunningProcesses.return_value = [_proc(1234)] + + agent._reconcile_gpu(0, uid_to_annotation) + + mock_apply.assert_not_called() + + def test_annotated_gpu_active_pod_is_capped(self): + """Happy path still works: an annotated pod's GPU is capped to its value.""" + agent = _make_agent(device_count=1) + pods = [_pod("worker", {POWER_ANNOTATION_KEY: "480"})] + uid_to_annotation = agent._build_uid_to_annotation(pods) + + with patch("power_agent.pynvml") as mock_nvml, patch( + "power_agent._extract_pod_uid_from_cgroup", return_value="worker" + ), patch("power_agent._apply_cap") as mock_apply: + mock_nvml.nvmlDeviceGetHandleByIndex.return_value = "handle-0" + mock_nvml.nvmlDeviceGetComputeRunningProcesses.return_value = [_proc(1234)] + + agent._reconcile_gpu(0, uid_to_annotation) + + mock_apply.assert_called_once() + # _apply_cap(handle, gpu_idx, cap_w, metrics) + args = mock_apply.call_args.args + self.assertEqual(args[1], 0) + self.assertEqual(args[2], 480) + + +class TestReleaseOnReuse(unittest.TestCase): + """A previously-managed GPU now running only unannotated work is released + back to default, instead of stranding a stale cap on the new tenant.""" + + def setUp(self): + power_agent._managed_gpu_indices.clear() + power_agent._previously_managed.clear() + + def tearDown(self): + power_agent._managed_gpu_indices.clear() + power_agent._previously_managed.clear() + + def _run_reconcile_with_unannotated_pod(self, current_mw, default_mw): + agent = _make_agent(device_count=1) + pods = [_pod("bystander", {"team.example.com/foo": "bar"})] + uid_to_annotation = agent._build_uid_to_annotation(pods) + + with patch("power_agent.pynvml") as mock_nvml, patch( + "power_agent._extract_pod_uid_from_cgroup", return_value="bystander" + ), patch("power_agent._apply_cap") as mock_apply, patch( + "power_agent._persist_managed_gpus" + ): + mock_nvml.nvmlDeviceGetHandleByIndex.return_value = "handle-0" + mock_nvml.nvmlDeviceGetComputeRunningProcesses.return_value = [_proc(1234)] + mock_nvml.nvmlDeviceGetPowerManagementDefaultLimit.return_value = default_mw + mock_nvml.nvmlDeviceGetPowerManagementLimit.return_value = current_mw + mock_nvml.nvmlDeviceGetUUID.return_value = "GPU-A" + + agent._reconcile_gpu(0, uid_to_annotation) + + return mock_nvml, mock_apply + + def test_previously_managed_gpu_is_released_to_default(self): + power_agent._managed_gpu_indices.add(0) + power_agent._previously_managed.add("GPU-A") + + mock_nvml, mock_apply = self._run_reconcile_with_unannotated_pod( + current_mw=400_000, default_mw=700_000 + ) + + # Restored to default, never re-capped, and unmanaged. + mock_nvml.nvmlDeviceSetPowerManagementLimit.assert_called_once_with( + "handle-0", 700_000 + ) + mock_apply.assert_not_called() + self.assertNotIn(0, power_agent._managed_gpu_indices) + self.assertNotIn("GPU-A", power_agent._previously_managed) + + def test_previously_managed_across_restart_is_released(self): + """After a restart `_managed_gpu_indices` is empty; the persisted UUID + set is the only signal. A busy GPU we capped before the restart must + still be released (startup orphan recovery skips busy GPUs).""" + # No _managed_gpu_indices entry (cleared on restart); only persisted UUID. + power_agent._previously_managed.add("GPU-A") + + mock_nvml, mock_apply = self._run_reconcile_with_unannotated_pod( + current_mw=400_000, default_mw=700_000 + ) + + mock_nvml.nvmlDeviceSetPowerManagementLimit.assert_called_once_with( + "handle-0", 700_000 + ) + mock_apply.assert_not_called() + self.assertNotIn("GPU-A", power_agent._previously_managed) + + def test_never_managed_gpu_is_not_touched(self): + # Neither in _managed_gpu_indices nor _previously_managed → not ours. + mock_nvml, mock_apply = self._run_reconcile_with_unannotated_pod( + current_mw=400_000, default_mw=700_000 + ) + + mock_nvml.nvmlDeviceSetPowerManagementLimit.assert_not_called() + mock_apply.assert_not_called() + + def test_release_unmanages_even_if_already_at_default(self): + """If the cap was already cleared externally, still drop it from the + managed set (no redundant NVML write).""" + power_agent._managed_gpu_indices.add(0) + power_agent._previously_managed.add("GPU-A") + + mock_nvml, _ = self._run_reconcile_with_unannotated_pod( + current_mw=700_000, default_mw=700_000 + ) + + mock_nvml.nvmlDeviceSetPowerManagementLimit.assert_not_called() + self.assertNotIn(0, power_agent._managed_gpu_indices) + + +if __name__ == "__main__": + unittest.main() diff --git a/deploy/power-agent/tests/test_apply_cap.py b/deploy/power-agent/tests/test_apply_cap.py new file mode 100644 index 000000000000..f4daa07bb8e2 --- /dev/null +++ b/deploy/power-agent/tests/test_apply_cap.py @@ -0,0 +1,297 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Unit tests for the NVML cap application path. + +Covers: + - PowerAgent._build_uid_to_annotation: pod UID → annotation value mapping + - _clamp_to_constraints: SKU min/max clamping before any NVML write + - _apply_cap: milliwatt conversion, managed-index tracking, metric update, + NVMLError handling, and the clamping→NVML write integration + - _nvml_uuid: handles both bytes (legacy pynvml) and str (nvidia-ml-py) +""" + +import unittest +from unittest.mock import MagicMock, patch + +import power_agent +from power_agent import ( + POWER_ANNOTATION_KEY, + PowerAgent, + _apply_cap, + _clamp_to_constraints, + _nvml_uuid, +) + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _make_agent() -> PowerAgent: + """Return a raw PowerAgent instance, bypassing __init__ (needs NVML + K8s).""" + return object.__new__(PowerAgent) + + +def _make_pod(uid: str, annotation_value=None) -> MagicMock: + """Return a minimal mock pod object.""" + pod = MagicMock() + pod.metadata.uid = uid + pod.metadata.annotations = ( + {POWER_ANNOTATION_KEY: annotation_value} if annotation_value is not None else {} + ) + return pod + + +def _make_nvml( + min_mw: int = 100_000, + max_mw: int = 700_000, + uuid: bytes = b"GPU-test-uuid-0000", +) -> MagicMock: + """Return a mock pynvml module with sensible defaults.""" + mock = MagicMock() + mock.NVMLError = Exception + mock.nvmlDeviceGetPowerManagementLimitConstraints.return_value = (min_mw, max_mw) + mock.nvmlDeviceGetUUID.return_value = uuid + return mock + + +# --------------------------------------------------------------------------- +# _build_uid_to_annotation +# --------------------------------------------------------------------------- + + +class TestBuildUidToAnnotation(unittest.TestCase): + def test_pod_with_annotation_present(self): + agent = _make_agent() + pod = _make_pod("uid-1", "300") + result = agent._build_uid_to_annotation([pod]) + self.assertEqual(result, {"uid-1": "300"}) + + def test_pod_without_annotation_is_omitted(self): + """Opt-in scope: a pod missing the key is omitted (not mapped to None), + so a GPU running only that pod is never managed. Per PR #9682 @sttts.""" + agent = _make_agent() + pod = _make_pod("uid-1") # annotation dict present but key absent + result = agent._build_uid_to_annotation([pod]) + self.assertEqual(result, {}) + + def test_pod_with_annotations_field_none_is_omitted(self): + """pod.metadata.annotations is None (no annotations block) → omitted.""" + agent = _make_agent() + pod = MagicMock() + pod.metadata.uid = "uid-1" + pod.metadata.annotations = None + result = agent._build_uid_to_annotation([pod]) + self.assertEqual(result, {}) + + def test_multiple_pods_correct_mapping(self): + agent = _make_agent() + pods = [ + _make_pod("uid-1", "300"), + _make_pod("uid-2", "480"), + _make_pod("uid-3"), # no annotation → omitted from the map + ] + result = agent._build_uid_to_annotation(pods) + self.assertEqual(result, {"uid-1": "300", "uid-2": "480"}) + + def test_empty_pod_list_returns_empty_dict(self): + agent = _make_agent() + self.assertEqual(agent._build_uid_to_annotation([]), {}) + + def test_annotation_with_unrelated_keys_ignored(self): + """Other annotation keys on the pod are not included.""" + agent = _make_agent() + pod = MagicMock() + pod.metadata.uid = "uid-1" + pod.metadata.annotations = { + "some.other/key": "irrelevant", + POWER_ANNOTATION_KEY: "350", + } + result = agent._build_uid_to_annotation([pod]) + self.assertEqual(result, {"uid-1": "350"}) + + +# --------------------------------------------------------------------------- +# _clamp_to_constraints +# --------------------------------------------------------------------------- + + +class TestClampToConstraints(unittest.TestCase): + def setUp(self): + self.handle = MagicMock() + self.metrics = MagicMock() + + def test_within_range_passes_through(self): + mock_nvml = _make_nvml(min_mw=200_000, max_mw=700_000) + with patch.object(power_agent, "pynvml", mock_nvml): + result = _clamp_to_constraints(self.handle, 400, 0, self.metrics) + self.assertEqual(result, 400) + self.metrics.cap_clamped_total.labels.assert_not_called() + + def test_below_min_clamped_up(self): + mock_nvml = _make_nvml(min_mw=200_000, max_mw=700_000) + with patch.object(power_agent, "pynvml", mock_nvml): + result = _clamp_to_constraints(self.handle, 100, 0, self.metrics) + self.assertEqual(result, 200) + self.metrics.cap_clamped_total.labels.assert_called_once_with(direction="min") + self.metrics.cap_clamped_total.labels.return_value.inc.assert_called_once() + + def test_above_max_clamped_down(self): + mock_nvml = _make_nvml(min_mw=200_000, max_mw=700_000) + with patch.object(power_agent, "pynvml", mock_nvml): + result = _clamp_to_constraints(self.handle, 900, 0, self.metrics) + self.assertEqual(result, 700) + self.metrics.cap_clamped_total.labels.assert_called_once_with(direction="max") + self.metrics.cap_clamped_total.labels.return_value.inc.assert_called_once() + + def test_exactly_at_min_not_clamped(self): + mock_nvml = _make_nvml(min_mw=200_000, max_mw=700_000) + with patch.object(power_agent, "pynvml", mock_nvml): + result = _clamp_to_constraints(self.handle, 200, 0, self.metrics) + self.assertEqual(result, 200) + self.metrics.cap_clamped_total.labels.assert_not_called() + + def test_exactly_at_max_not_clamped(self): + mock_nvml = _make_nvml(min_mw=200_000, max_mw=700_000) + with patch.object(power_agent, "pynvml", mock_nvml): + result = _clamp_to_constraints(self.handle, 700, 0, self.metrics) + self.assertEqual(result, 700) + self.metrics.cap_clamped_total.labels.assert_not_called() + + def test_nvml_error_returns_requested_unchanged(self): + """If GetPowerManagementLimitConstraints fails, pass the value through.""" + mock_nvml = MagicMock() + mock_nvml.NVMLError = Exception + mock_nvml.nvmlDeviceGetPowerManagementLimitConstraints.side_effect = Exception( + "nvml" + ) + with patch.object(power_agent, "pynvml", mock_nvml): + result = _clamp_to_constraints(self.handle, 300, 0, self.metrics) + self.assertEqual(result, 300) + self.metrics.cap_clamped_total.labels.assert_not_called() + + +# --------------------------------------------------------------------------- +# _apply_cap +# --------------------------------------------------------------------------- + + +class TestApplyCap(unittest.TestCase): + def setUp(self): + power_agent._managed_gpu_indices.clear() + power_agent._previously_managed.clear() + + def test_calls_nvml_with_milliwatts(self): + """Watts must be converted to milliwatts before the NVML call.""" + mock_nvml = _make_nvml() + handle = MagicMock() + metrics = MagicMock() + with patch.object(power_agent, "pynvml", mock_nvml): + with patch("power_agent._persist_managed_gpus"): + _apply_cap(handle, 0, 300, metrics) + mock_nvml.nvmlDeviceSetPowerManagementLimit.assert_called_once_with( + handle, 300_000 + ) + + def test_adds_gpu_to_managed_indices(self): + mock_nvml = _make_nvml() + handle = MagicMock() + metrics = MagicMock() + with patch.object(power_agent, "pynvml", mock_nvml): + with patch("power_agent._persist_managed_gpus"): + _apply_cap(handle, 2, 300, metrics) + self.assertIn(2, power_agent._managed_gpu_indices) + + def test_updates_applied_limit_gauge(self): + """metrics.applied_limit_watts.labels(gpu=...).set(watts) must be called.""" + mock_nvml = _make_nvml() + handle = MagicMock() + metrics = MagicMock() + with patch.object(power_agent, "pynvml", mock_nvml): + with patch("power_agent._persist_managed_gpus"): + _apply_cap(handle, 1, 450, metrics) + metrics.applied_limit_watts.labels.assert_called_once_with(gpu="1") + metrics.applied_limit_watts.labels.return_value.set.assert_called_once_with(450) + + def test_nvml_error_increments_failure_counter_and_does_not_raise(self): + mock_nvml = _make_nvml() + mock_nvml.nvmlDeviceSetPowerManagementLimit.side_effect = Exception("nvml fail") + handle = MagicMock() + metrics = MagicMock() + with patch.object(power_agent, "pynvml", mock_nvml): + _apply_cap(handle, 0, 300, metrics) # must not propagate + metrics.apply_failures_total.inc.assert_called_once() + metrics.applied_limit_watts.labels.assert_not_called() + + def test_nvml_error_does_not_add_to_managed_indices(self): + mock_nvml = _make_nvml() + mock_nvml.nvmlDeviceSetPowerManagementLimit.side_effect = Exception("nvml fail") + handle = MagicMock() + metrics = MagicMock() + with patch.object(power_agent, "pynvml", mock_nvml): + _apply_cap(handle, 3, 300, metrics) + self.assertNotIn(3, power_agent._managed_gpu_indices) + + def test_clamp_applied_before_nvml_call(self): + """Requested 900 W but SKU max is 700 W → NVML receives 700_000 mW.""" + mock_nvml = _make_nvml(min_mw=100_000, max_mw=700_000) + handle = MagicMock() + metrics = MagicMock() + with patch.object(power_agent, "pynvml", mock_nvml): + with patch("power_agent._persist_managed_gpus"): + _apply_cap(handle, 0, 900, metrics) + mock_nvml.nvmlDeviceSetPowerManagementLimit.assert_called_once_with( + handle, 700_000 + ) + + def test_clamp_below_min_uses_min(self): + """Requested 50 W but SKU min is 100 W → NVML receives 100_000 mW.""" + mock_nvml = _make_nvml(min_mw=100_000, max_mw=700_000) + handle = MagicMock() + metrics = MagicMock() + with patch.object(power_agent, "pynvml", mock_nvml): + with patch("power_agent._persist_managed_gpus"): + _apply_cap(handle, 0, 50, metrics) + mock_nvml.nvmlDeviceSetPowerManagementLimit.assert_called_once_with( + handle, 100_000 + ) + + +# --------------------------------------------------------------------------- +# _nvml_uuid: pynvml package-version compatibility shim +# --------------------------------------------------------------------------- + + +class TestNvmlUuid(unittest.TestCase): + """Regression guard for `'str' object has no attribute 'decode'` crash. + + Legacy ``pynvml`` (NVIDIA bindings, deprecated) returns ``bytes`` from + ``nvmlDeviceGetUUID``; ``nvidia-ml-py`` (the supported successor that + pip's ``pynvml`` now installs by default) returns ``str``. An + unconditional ``.decode("ascii")`` crashes the orphan-restore loop on + every iteration on hosts running the new bindings — this test pins the + shim so neither variant breaks. + """ + + def test_returns_str_when_nvml_returns_bytes(self): + mock_nvml = MagicMock() + mock_nvml.nvmlDeviceGetUUID.return_value = b"GPU-bytes-uuid-0000" + handle = MagicMock() + with patch.object(power_agent, "pynvml", mock_nvml): + result = _nvml_uuid(handle) + self.assertEqual(result, "GPU-bytes-uuid-0000") + self.assertIsInstance(result, str) + + def test_returns_str_when_nvml_returns_str(self): + mock_nvml = MagicMock() + mock_nvml.nvmlDeviceGetUUID.return_value = "GPU-str-uuid-0000" + handle = MagicMock() + with patch.object(power_agent, "pynvml", mock_nvml): + result = _nvml_uuid(handle) + self.assertEqual(result, "GPU-str-uuid-0000") + self.assertIsInstance(result, str) + + +if __name__ == "__main__": + unittest.main() diff --git a/deploy/power-agent/tests/test_cgroup_parser.py b/deploy/power-agent/tests/test_cgroup_parser.py new file mode 100644 index 000000000000..adee5b08a370 --- /dev/null +++ b/deploy/power-agent/tests/test_cgroup_parser.py @@ -0,0 +1,123 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Unit tests for _extract_pod_uid_from_cgroup. + +Covers all QoS × driver × runtime combinations documented in §6.5: + - cgroup v1 multi-line, cgroup v2 single-line + - systemd / cgroupfs drivers + - Guaranteed / Burstable / BestEffort QoS + - cri-containerd / cri-o wrapper segments + - non-K8s process (returns None) +""" + +import unittest +from unittest.mock import mock_open, patch + +from power_agent import _extract_pod_uid_from_cgroup + + +class TestCgroupParser(unittest.TestCase): + def _parse(self, content: str): + with patch("builtins.open", mock_open(read_data=content)): + return _extract_pod_uid_from_cgroup(12345) + + # ---- cgroup v1 (multi-line, one per controller) ---- + + def test_v1_systemd_guaranteed(self): + # systemd encodes dashes as underscores in the pod-UID segment + content = ( + "11:devices:/kubepods.slice/kubepods-podcafe0001_dead_0002_beef_0003.slice/" + "cri-containerd-abc.scope\n" + "10:memory:/kubepods.slice/kubepods-podcafe0001_dead_0002_beef_0003.slice/" + "cri-containerd-abc.scope\n" + ) + uid = self._parse(content) + self.assertEqual(uid, "cafe0001-dead-0002-beef-0003") + + def test_v1_systemd_burstable(self): + content = ( + "11:devices:/kubepods.slice/kubepods-burstable.slice/" + "kubepods-burstable-podcafe0001_dead_0002_beef_0003.slice/" + "cri-containerd-abc.scope\n" + ) + uid = self._parse(content) + self.assertEqual(uid, "cafe0001-dead-0002-beef-0003") + + def test_v1_systemd_besteffort(self): + content = ( + "10:memory:/kubepods.slice/kubepods-besteffort.slice/" + "kubepods-besteffort-podcafe0001_dead_0002_beef_0003.slice/" + "cri-containerd-abc.scope\n" + ) + uid = self._parse(content) + self.assertEqual(uid, "cafe0001-dead-0002-beef-0003") + + def test_v1_cgroupfs_guaranteed(self): + content = ( + "11:devices:/kubepods/podcafe0001-dead-0002-beef-0003/abc123\n" + "10:memory:/kubepods/podcafe0001-dead-0002-beef-0003/abc123\n" + ) + uid = self._parse(content) + self.assertEqual(uid, "cafe0001-dead-0002-beef-0003") + + def test_v1_cgroupfs_burstable(self): + content = ( + "10:memory:/kubepods/burstable/podcafe0001-dead-0002-beef-0003/abc123\n" + ) + uid = self._parse(content) + self.assertEqual(uid, "cafe0001-dead-0002-beef-0003") + + def test_v1_cgroupfs_besteffort(self): + content = ( + "10:memory:/kubepods/besteffort/podcafe0001-dead-0002-beef-0003/abc123\n" + ) + uid = self._parse(content) + self.assertEqual(uid, "cafe0001-dead-0002-beef-0003") + + # ---- cgroup v2 (single unified line) ---- + + def test_v2_systemd(self): + content = ( + "0::/kubepods.slice/kubepods-podcafe0001_dead_0002_beef_0003.slice/" + "cri-containerd-abc.scope\n" + ) + uid = self._parse(content) + self.assertEqual(uid, "cafe0001-dead-0002-beef-0003") + + def test_v2_cgroupfs(self): + content = "0::/kubepods/podcafe0001-dead-0002-beef-0003/cri-containerd-abc\n" + uid = self._parse(content) + self.assertEqual(uid, "cafe0001-dead-0002-beef-0003") + + # ---- non-K8s process (no pod slice in cgroup) ---- + + def test_non_k8s_process(self): + content = "11:devices:/system.slice/kubelet.service\n" + uid = self._parse(content) + self.assertIsNone(uid) + + def test_empty_cgroup(self): + uid = self._parse("") + self.assertIsNone(uid) + + def test_os_error_returns_none(self): + with patch("builtins.open", side_effect=OSError("no file")): + uid = _extract_pod_uid_from_cgroup(99999) + self.assertIsNone(uid) + + # ---- first matching line wins (v1 multi-line) ---- + + def test_first_matching_line_wins(self): + """When multiple lines match, the first one should be returned.""" + content = ( + "0::/not-k8s/something\n" + "10:memory:/kubepods/podAAAAAAAA-BBBB-CCCC-DDDD-EEEEEEEEEEEE/abc\n" + "9:cpu:/kubepods/pod11111111-2222-3333-4444-555555555555/abc\n" + ) + uid = self._parse(content) + self.assertEqual(uid, "AAAAAAAA-BBBB-CCCC-DDDD-EEEEEEEEEEEE") + + +if __name__ == "__main__": + unittest.main() diff --git a/deploy/power-agent/tests/test_multi_pod_policy.py b/deploy/power-agent/tests/test_multi_pod_policy.py new file mode 100644 index 000000000000..df04ab29900b --- /dev/null +++ b/deploy/power-agent/tests/test_multi_pod_policy.py @@ -0,0 +1,136 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Unit tests for multi-pod-per-GPU cap resolution policy (§6.5). + +Cases: + - 1 pod: apply pod annotation + - 2+ pods, all agree: apply agreed value + WARNING + - 2+ pods, conflict: apply safe_default + ERROR + - Parse failure on warm GPU (prior cap in effect): keep prior cap (no NVML call) + - Parse failure on cold GPU (no prior cap): apply safe_default +""" + +import unittest + +from power_agent import _resolve_cap_for_gpu + + +class _FakeMetrics: + def __init__(self): + self.multi_pod_agree = 0 + self.multi_pod_conflict = 0 + self.apply_failures = 0 + self.safe_default_applied = 0 + + @property + def multi_pod_gpu_total(self): + class _L: + def __init__(self, m): + self._m = m + + def labels(self, disposition): + class _C: + def __init__(self, m, d): + self._m, self._d = m, d + + def inc(self): + if self._d == "agree": + self._m.multi_pod_agree += 1 + else: + self._m.multi_pod_conflict += 1 + + return _C(self._m, disposition) + + return _L(self) + + @property + def apply_failures_total(self): + class _C: + def __init__(self, m): + self._m = m + + def inc(self): + self._m.apply_failures += 1 + + return _C(self) + + @property + def safe_default_applied_total(self): + class _C: + def __init__(self, m): + self._m = m + + def inc(self): + self._m.safe_default_applied += 1 + + return _C(self) + + +SAFE_DEFAULT = 500 + + +class TestMultiPodPolicy(unittest.TestCase): + def setUp(self): + self.m = _FakeMetrics() + + def test_single_pod_apply_annotation(self): + cap = _resolve_cap_for_gpu(0, [("uid-1", "480")], SAFE_DEFAULT, self.m) + self.assertEqual(cap, 480) + self.assertEqual(self.m.multi_pod_agree, 0) + self.assertEqual(self.m.multi_pod_conflict, 0) + self.assertEqual(self.m.safe_default_applied, 0) + + def test_two_pods_agree(self): + cap = _resolve_cap_for_gpu( + 0, [("uid-1", "480"), ("uid-2", "480")], SAFE_DEFAULT, self.m + ) + self.assertEqual(cap, 480) + self.assertEqual(self.m.multi_pod_agree, 1) + self.assertEqual(self.m.multi_pod_conflict, 0) + self.assertEqual(self.m.safe_default_applied, 0) + + def test_two_pods_conflict(self): + cap = _resolve_cap_for_gpu( + 0, [("uid-1", "480"), ("uid-2", "350")], SAFE_DEFAULT, self.m + ) + self.assertEqual(cap, SAFE_DEFAULT) + self.assertEqual(self.m.multi_pod_conflict, 1) + self.assertEqual(self.m.safe_default_applied, 1) + + def test_no_parseable_annotation(self): + cap = _resolve_cap_for_gpu( + 0, [("uid-1", None), ("uid-2", None)], SAFE_DEFAULT, self.m + ) + self.assertEqual(cap, SAFE_DEFAULT) + self.assertEqual(self.m.apply_failures, 1) + self.assertEqual(self.m.safe_default_applied, 1) + + def test_invalid_annotation_value(self): + cap = _resolve_cap_for_gpu(0, [("uid-1", "not-a-number")], SAFE_DEFAULT, self.m) + self.assertEqual(cap, SAFE_DEFAULT) + self.assertEqual(self.m.apply_failures, 1) + self.assertEqual(self.m.safe_default_applied, 1) + + def test_three_pods_all_agree(self): + cap = _resolve_cap_for_gpu( + 0, + [("uid-1", "300"), ("uid-2", "300"), ("uid-3", "300")], + SAFE_DEFAULT, + self.m, + ) + self.assertEqual(cap, 300) + self.assertEqual(self.m.multi_pod_agree, 1) + + def test_mixed_none_and_valid(self): + """One pod has no annotation, one has a valid annotation — treated as no-annotation pod.""" + cap = _resolve_cap_for_gpu( + 0, [("uid-1", None), ("uid-2", "480")], SAFE_DEFAULT, self.m + ) + # values list = ["480"], single value → apply 480. multi_pod_agree fires. + self.assertEqual(cap, 480) + self.assertEqual(self.m.multi_pod_agree, 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/deploy/power-agent/tests/test_reconcile_failsafe.py b/deploy/power-agent/tests/test_reconcile_failsafe.py new file mode 100644 index 000000000000..a6aade186650 --- /dev/null +++ b/deploy/power-agent/tests/test_reconcile_failsafe.py @@ -0,0 +1,113 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Reconcile fail-safe on pod-listing failure (PR #9682 review follow-up). + +`_list_pods_on_node` uses an *explicit* result contract so the failure path +cannot silently drift into the empty-node path: + + * success → a (possibly empty) list of pods + * failure → ``None`` + +`reconcile_once` keys its fail-safe off that ``None``: on a listing failure it +SKIPS the cycle, leaving every GPU at its last-known-good cap, rather than +proceeding with an empty pod view that would re-derive caps from a zero-pod +snapshot. An empty list (node genuinely has no pods) is NOT a failure and must +still drive a normal reconcile pass. +""" + +import unittest +from unittest.mock import MagicMock + +from power_agent import PowerAgent + + +def _make_agent(core_v1, device_count: int = 2) -> PowerAgent: + """Build a PowerAgent without touching NVML / K8s (bypass __init__).""" + agent = object.__new__(PowerAgent) + agent._core_v1 = core_v1 + agent.node_name = "node-under-test" + agent.k8s_namespace = None # exercise the list_pod_for_all_namespaces path + agent.device_count = device_count + return agent + + +class TestListPodsExplicitResult(unittest.TestCase): + def test_returns_none_on_api_error(self): + """An apiserver error yields the explicit ``None`` sentinel, never [].""" + core_v1 = MagicMock() + core_v1.list_pod_for_all_namespaces.side_effect = RuntimeError("boom") + agent = _make_agent(core_v1) + + self.assertIsNone(agent._list_pods_on_node()) + + def test_returns_empty_list_when_node_has_no_pods(self): + """A genuinely empty node returns ``[]`` — distinct from failure.""" + core_v1 = MagicMock() + core_v1.list_pod_for_all_namespaces.return_value = MagicMock(items=[]) + agent = _make_agent(core_v1) + + result = agent._list_pods_on_node() + self.assertIsNotNone(result) + self.assertEqual(result, []) + + def test_cluster_scoped_list_served_from_watch_cache(self): + """The cluster-scoped LIST passes resource_version="0" so the apiserver + can serve it from its watch cache instead of an etcd read (PR #9682 + scale mitigation).""" + core_v1 = MagicMock() + core_v1.list_pod_for_all_namespaces.return_value = MagicMock(items=[]) + agent = _make_agent(core_v1) + + agent._list_pods_on_node() + + _, kwargs = core_v1.list_pod_for_all_namespaces.call_args + self.assertEqual(kwargs.get("resource_version"), "0") + core_v1.list_namespaced_pod.assert_not_called() + + def test_namespaced_list_served_from_watch_cache(self): + """The namespace-scoped LIST also passes resource_version="0".""" + core_v1 = MagicMock() + core_v1.list_namespaced_pod.return_value = MagicMock(items=[]) + agent = _make_agent(core_v1) + agent.k8s_namespace = "dynamo" + + agent._list_pods_on_node() + + _, kwargs = core_v1.list_namespaced_pod.call_args + self.assertEqual(kwargs.get("resource_version"), "0") + self.assertEqual(kwargs.get("namespace"), "dynamo") + core_v1.list_pod_for_all_namespaces.assert_not_called() + + +class TestReconcileFailSafe(unittest.TestCase): + def test_listing_failure_skips_reconcile_and_preserves_caps(self): + """On listing failure, ``reconcile_once`` must not touch any GPU — + no per-GPU reconcile runs, so existing caps are frozen in place.""" + core_v1 = MagicMock() + core_v1.list_pod_for_all_namespaces.side_effect = RuntimeError("boom") + agent = _make_agent(core_v1, device_count=4) + agent._reconcile_gpu = MagicMock() + + agent.reconcile_once() + + agent._reconcile_gpu.assert_not_called() + + def test_empty_node_still_reconciles_every_gpu(self): + """An empty (but successful) listing is NOT a failure: reconcile + proceeds normally and still visits every GPU, instead of skipping the + whole cycle as it does on a listing failure. (Each GPU then early- + returns in ``_reconcile_gpu`` since no listed pod owns it — no + safe-default is applied on a genuinely empty node.)""" + core_v1 = MagicMock() + core_v1.list_pod_for_all_namespaces.return_value = MagicMock(items=[]) + agent = _make_agent(core_v1, device_count=4) + agent._reconcile_gpu = MagicMock() + + agent.reconcile_once() + + self.assertEqual(agent._reconcile_gpu.call_count, 4) + + +if __name__ == "__main__": + unittest.main() diff --git a/deploy/power-agent/tests/test_shutdown.py b/deploy/power-agent/tests/test_shutdown.py new file mode 100644 index 000000000000..3b7613bf0e79 --- /dev/null +++ b/deploy/power-agent/tests/test_shutdown.py @@ -0,0 +1,72 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Unit tests for the SIGTERM handler (§6.5). + +Verifies that: + - SIGTERM handler restores default TGP for every managed GPU. + - nvmlShutdown is called exactly once. + - _shutdown event is set after the handler completes. +""" + +import signal +import unittest +from unittest.mock import MagicMock, patch + +import power_agent + + +class TestSigtermHandler(unittest.TestCase): + def setUp(self): + # Reset module-level state between tests + power_agent._managed_gpu_indices.clear() + power_agent._shutdown.clear() + + def test_restores_tgp_for_all_managed_gpus(self): + power_agent._managed_gpu_indices.update([0, 1, 2]) + + mock_nvml = MagicMock() + mock_nvml.nvmlDeviceGetHandleByIndex.side_effect = lambda idx: f"handle_{idx}" + mock_nvml.nvmlDeviceGetPowerManagementDefaultLimit.return_value = 700_000 + mock_nvml.NVMLError = Exception + + with patch.dict("sys.modules", {"pynvml": mock_nvml}): + with patch.object(power_agent, "pynvml", mock_nvml): + power_agent._handle_sigterm(signal.SIGTERM, None) + + # Three GPUs managed → three SetPowerManagementLimit calls + self.assertEqual(mock_nvml.nvmlDeviceSetPowerManagementLimit.call_count, 3) + mock_nvml.nvmlDeviceSetPowerManagementLimit.assert_any_call("handle_0", 700_000) + mock_nvml.nvmlDeviceSetPowerManagementLimit.assert_any_call("handle_1", 700_000) + mock_nvml.nvmlDeviceSetPowerManagementLimit.assert_any_call("handle_2", 700_000) + # nvmlShutdown called once + mock_nvml.nvmlShutdown.assert_called_once() + # _shutdown event is set + self.assertTrue(power_agent._shutdown.is_set()) + + def test_no_managed_gpus_still_shuts_down(self): + mock_nvml = MagicMock() + with patch.object(power_agent, "pynvml", mock_nvml): + power_agent._handle_sigterm(signal.SIGTERM, None) + + mock_nvml.nvmlDeviceSetPowerManagementLimit.assert_not_called() + mock_nvml.nvmlShutdown.assert_called_once() + self.assertTrue(power_agent._shutdown.is_set()) + + def test_nvml_error_does_not_prevent_shutdown(self): + power_agent._managed_gpu_indices.add(0) + + mock_nvml = MagicMock() + mock_nvml.nvmlDeviceGetHandleByIndex.side_effect = Exception("nvml error") + mock_nvml.NVMLError = Exception + + with patch.object(power_agent, "pynvml", mock_nvml): + power_agent._handle_sigterm(signal.SIGTERM, None) + + # Despite the error, nvmlShutdown and _shutdown.set() still execute + mock_nvml.nvmlShutdown.assert_called_once() + self.assertTrue(power_agent._shutdown.is_set()) + + +if __name__ == "__main__": + unittest.main() diff --git a/pyproject.toml b/pyproject.toml index 501f12d1caf9..afbf02ec1c4e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -178,6 +178,10 @@ addopts = [ "--ignore-glob=*_inc.py", "--ignore-glob=*/llm/tensorrtllm*", "--ignore-glob=docs/*", + # power-agent ships a standalone suite run in-image (deploy/power-agent + # Dockerfile `test` stage + its own pytest.ini); its tests/ package name + # collides with the repo-root tests/ package during repo-wide collection. + "--ignore-glob=deploy/power-agent/tests/*", "--ignore-glob=components/src/dynamo/sglang/request_handlers/*", "--ignore-glob=components/src/dynamo/sglang/multimodal_utils/*", "--ignore-glob=components/src/dynamo/vllm/multimodal_utils/*",