From c735aeff12854cece9ff28e7e02b5507140001ed Mon Sep 17 00:00:00 2001 From: Anant Sharma Date: Wed, 24 Jun 2026 09:33:52 -0700 Subject: [PATCH 1/4] ci: collapse power-agent into one job and build test target via the action Merge power-agent, power-agent-test, and power-agent-helm-tests into a single power-agent job (build -> unit tests -> helm lint/render) so the PR shows one "Power Agent" check. Three same-named inline jobs do not nest the way reusable-workflow jobs (e.g. vllm-runtime) do, so they previously rendered as three separate checks. Add an optional push input (default true) to build-deploy-component so the test step builds the Dockerfile test target through the same action: routes the base image through the ECR Docker Hub mirror and reuses the runtime builder cache, while skipping the push for the never-shipped test stage. Drop the removed jobs from backend-status-check needs. Signed-off-by: Anant Sharma --- .../actions/build-deploy-component/action.yml | 32 +++++++++++++------ .github/workflows/post-merge-ci.yml | 28 +++++++--------- .github/workflows/pr.yaml | 32 +++++++------------ 3 files changed, 45 insertions(+), 47 deletions(-) diff --git a/.github/actions/build-deploy-component/action.yml b/.github/actions/build-deploy-component/action.yml index 9ee843c9b84f..f17d45652e06 100644 --- a/.github/actions/build-deploy-component/action.yml +++ b/.github/actions/build-deploy-component/action.yml @@ -15,6 +15,10 @@ inputs: description: 'Dockerfile target to build. Defaults to agent for snapshot and the default target for operator.' required: false default: '' + push: + description: 'Push the built image. Set false to build only (e.g. a test target whose RUN executes the suite but is never shipped).' + required: false + default: 'true' builder_name: description: 'Buildx builder name' required: true @@ -115,6 +119,13 @@ runs: TARGET_FLAG="--target ${TARGET}" fi + # When push is false the build still runs every RUN (so a test target's + # suite executes and gates the job); the image is just not exported. + PUSH_FLAG="--push" + if [[ "${{ inputs.push }}" != "true" ]]; then + PUSH_FLAG="" + fi + BUILD_ARGS=("--build-arg" "DOCKER_PROXY=${ECR_HOSTNAME}/dockerhub/") while IFS= read -r build_arg; do if [[ -n "${build_arg}" ]]; then @@ -130,18 +141,21 @@ runs: BUILD_CONTEXT_FLAG="--build-context snapshot=../snapshot" fi - docker buildx build --push \ + docker buildx build ${PUSH_FLAG} \ --platform ${{ steps.settings.outputs.build_platforms }} \ "${BUILD_ARGS[@]}" \ ${BUILD_CONTEXT_FLAG} \ ${TARGET_FLAG} \ ${TAGGING_FLAGS} -f Dockerfile . - DISPLAY_NAME="${{ inputs.component }}" - echo "### 🐳 ${DISPLAY_NAME^} Container Images" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "| Image URI |" >> $GITHUB_STEP_SUMMARY - echo "|-----|" >> $GITHUB_STEP_SUMMARY - for image_uri in "${IMAGE_URIS[@]}"; do - echo "| \`${image_uri}\` |" >> $GITHUB_STEP_SUMMARY - done + # Only advertise image URIs that were actually pushed. + if [[ "${{ inputs.push }}" == "true" ]]; then + DISPLAY_NAME="${{ inputs.component }}" + echo "### 🐳 ${DISPLAY_NAME^} Container Images" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "| Image URI |" >> $GITHUB_STEP_SUMMARY + echo "|-----|" >> $GITHUB_STEP_SUMMARY + for image_uri in "${IMAGE_URIS[@]}"; do + echo "| \`${image_uri}\` |" >> $GITHUB_STEP_SUMMARY + done + fi diff --git a/.github/workflows/post-merge-ci.yml b/.github/workflows/post-merge-ci.yml index ca664e97829f..08586c5a9978 100644 --- a/.github/workflows/post-merge-ci.yml +++ b/.github/workflows/post-merge-ci.yml @@ -141,27 +141,21 @@ jobs: azure_acr_password: ${{ secrets.AZURE_ACR_PASSWORD }} extra_tags: | ${{ secrets.AZURE_ACR_HOSTNAME }}/ai-dynamo/dynamo:${{ github.sha }}-power-agent - - power-agent-test: - name: Power Agent # group under the same UI umbrella as the build job - if: github.event_name != 'workflow_dispatch' - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Run unit tests inside the container image # Build the Dockerfile `test` stage (FROM runtime): its RUN pytest runs # the suite against the exact shipped image (Python 3.12 + baked deps), # validating the container itself rather than a pip-installed runner env. - run: docker build --target test -f deploy/power-agent/Dockerfile deploy/power-agent - - power-agent-helm-tests: - name: Power Agent # group under the same UI umbrella as the build job - if: github.event_name != 'workflow_dispatch' - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + # push: false — the test stage is never shipped; the build still runs the + # suite. Same builder as above, so the runtime layers are a cache hit. + uses: ./.github/actions/build-deploy-component + with: + component: power-agent + target: test + push: false + image_tag: ${{ github.sha }}-power-agent-test + builder_name: b-${{ github.run_id }}-${{ github.run_attempt }} + aws_default_region: ${{ secrets.AWS_DEFAULT_REGION }} + aws_account_id: ${{ secrets.AWS_ACCOUNT_ID }} - name: Set up Helm uses: azure/setup-helm@b9e51907a09c216f16ebe8536097933489208112 # v4.3.0 with: diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index dc2bf5d7243e..38a6ee5865cd 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -61,8 +61,6 @@ jobs: - snapshot-placeholder-vllm - snapshot-placeholder-sglang - power-agent - - power-agent-test - - power-agent-helm-tests - vllm-build - vllm-dev-build - vllm-test @@ -229,29 +227,21 @@ jobs: builder_name: ${{ needs.changed-files.outputs.builder_name }} aws_default_region: ${{ secrets.AWS_DEFAULT_REGION }} aws_account_id: ${{ secrets.AWS_ACCOUNT_ID }} - - power-agent-test: - needs: changed-files - if: needs.changed-files.outputs.power_agent == 'true' - name: Power Agent # group under the same UI umbrella as the build job - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Run unit tests inside the container image # Build the Dockerfile `test` stage (FROM runtime): its RUN pytest runs # the suite against the exact shipped image (Python 3.12 + baked deps), # validating the container itself rather than a pip-installed runner env. - run: docker build --target test -f deploy/power-agent/Dockerfile deploy/power-agent - - power-agent-helm-tests: - needs: changed-files - if: needs.changed-files.outputs.power_agent == 'true' - name: Power Agent # group under the same UI umbrella as the build job - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + # push: false — the test stage is never shipped; the build still runs the + # suite. Same builder as above, so the runtime layers are a cache hit. + uses: ./.github/actions/build-deploy-component + with: + component: power-agent + target: test + push: false + image_tag: ${{ github.sha }}-power-agent-test + builder_name: ${{ needs.changed-files.outputs.builder_name }} + aws_default_region: ${{ secrets.AWS_DEFAULT_REGION }} + aws_account_id: ${{ secrets.AWS_ACCOUNT_ID }} - name: Set up Helm uses: azure/setup-helm@b9e51907a09c216f16ebe8536097933489208112 # v4.3.0 with: From 3743a2ad4db93582c56dfe8308f843551be478d5 Mon Sep 17 00:00:00 2001 From: Anant Sharma Date: Wed, 24 Jun 2026 09:48:10 -0700 Subject: [PATCH 2/4] ci: build power-agent test target on the existing builder Calling build-deploy-component a second time for the test re-ran the builder bootstrap and failed with "existing instance ... no append mode". Build the test target directly with docker buildx build on the builder the runtime build already set up, routing the base image through the ECR Docker Hub mirror and reusing its layer cache. Insert a builder-refresher step between the runtime and test builds (matching shared-build-image.yml) so a stale remote BuildKit connection is repaired before the test build. Revert the unused push input added to build-deploy-component. Signed-off-by: Anant Sharma --- .../actions/build-deploy-component/action.yml | 32 ++++++------------- .github/workflows/post-merge-ci.yml | 29 ++++++++++------- .github/workflows/pr.yaml | 29 ++++++++++------- 3 files changed, 45 insertions(+), 45 deletions(-) diff --git a/.github/actions/build-deploy-component/action.yml b/.github/actions/build-deploy-component/action.yml index f17d45652e06..9ee843c9b84f 100644 --- a/.github/actions/build-deploy-component/action.yml +++ b/.github/actions/build-deploy-component/action.yml @@ -15,10 +15,6 @@ inputs: description: 'Dockerfile target to build. Defaults to agent for snapshot and the default target for operator.' required: false default: '' - push: - description: 'Push the built image. Set false to build only (e.g. a test target whose RUN executes the suite but is never shipped).' - required: false - default: 'true' builder_name: description: 'Buildx builder name' required: true @@ -119,13 +115,6 @@ runs: TARGET_FLAG="--target ${TARGET}" fi - # When push is false the build still runs every RUN (so a test target's - # suite executes and gates the job); the image is just not exported. - PUSH_FLAG="--push" - if [[ "${{ inputs.push }}" != "true" ]]; then - PUSH_FLAG="" - fi - BUILD_ARGS=("--build-arg" "DOCKER_PROXY=${ECR_HOSTNAME}/dockerhub/") while IFS= read -r build_arg; do if [[ -n "${build_arg}" ]]; then @@ -141,21 +130,18 @@ runs: BUILD_CONTEXT_FLAG="--build-context snapshot=../snapshot" fi - docker buildx build ${PUSH_FLAG} \ + docker buildx build --push \ --platform ${{ steps.settings.outputs.build_platforms }} \ "${BUILD_ARGS[@]}" \ ${BUILD_CONTEXT_FLAG} \ ${TARGET_FLAG} \ ${TAGGING_FLAGS} -f Dockerfile . - # Only advertise image URIs that were actually pushed. - if [[ "${{ inputs.push }}" == "true" ]]; then - DISPLAY_NAME="${{ inputs.component }}" - echo "### 🐳 ${DISPLAY_NAME^} Container Images" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "| Image URI |" >> $GITHUB_STEP_SUMMARY - echo "|-----|" >> $GITHUB_STEP_SUMMARY - for image_uri in "${IMAGE_URIS[@]}"; do - echo "| \`${image_uri}\` |" >> $GITHUB_STEP_SUMMARY - done - fi + DISPLAY_NAME="${{ inputs.component }}" + echo "### 🐳 ${DISPLAY_NAME^} Container Images" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "| Image URI |" >> $GITHUB_STEP_SUMMARY + echo "|-----|" >> $GITHUB_STEP_SUMMARY + for image_uri in "${IMAGE_URIS[@]}"; do + echo "| \`${image_uri}\` |" >> $GITHUB_STEP_SUMMARY + done diff --git a/.github/workflows/post-merge-ci.yml b/.github/workflows/post-merge-ci.yml index 08586c5a9978..e89c3b09595f 100644 --- a/.github/workflows/post-merge-ci.yml +++ b/.github/workflows/post-merge-ci.yml @@ -141,21 +141,28 @@ jobs: azure_acr_password: ${{ secrets.AZURE_ACR_PASSWORD }} extra_tags: | ${{ secrets.AZURE_ACR_HOSTNAME }}/ai-dynamo/dynamo:${{ github.sha }}-power-agent + - name: Refresh BuildKit builder + # The runtime build above can leave the remote BuildKit connection stale; + # re-establish it (re-routing only if unhealthy) before the test build. + uses: ./.github/actions/builder-refresher + with: + builder_name: b-${{ github.run_id }}-${{ github.run_attempt }} + flavor: general + arch: linux/amd64 - name: Run unit tests inside the container image # Build the Dockerfile `test` stage (FROM runtime): its RUN pytest runs # the suite against the exact shipped image (Python 3.12 + baked deps), # validating the container itself rather than a pip-installed runner env. - # push: false — the test stage is never shipped; the build still runs the - # suite. Same builder as above, so the runtime layers are a cache hit. - uses: ./.github/actions/build-deploy-component - with: - component: power-agent - target: test - push: false - image_tag: ${{ github.sha }}-power-agent-test - builder_name: b-${{ github.run_id }}-${{ github.run_attempt }} - aws_default_region: ${{ secrets.AWS_DEFAULT_REGION }} - aws_account_id: ${{ secrets.AWS_ACCOUNT_ID }} + # Reuses the builder + ECR Docker Hub mirror the build step set up above + # (runtime layers are a cache hit); no --push/--load runs pytest without + # exporting the never-shipped test image. + env: + ECR_HOSTNAME: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_DEFAULT_REGION }}.amazonaws.com + run: | + docker buildx build --builder b-${{ github.run_id }}-${{ github.run_attempt }} \ + --target test \ + --build-arg DOCKER_PROXY=${ECR_HOSTNAME}/dockerhub/ \ + -f deploy/power-agent/Dockerfile deploy/power-agent - name: Set up Helm uses: azure/setup-helm@b9e51907a09c216f16ebe8536097933489208112 # v4.3.0 with: diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 38a6ee5865cd..e5d949c7921b 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -227,21 +227,28 @@ jobs: builder_name: ${{ needs.changed-files.outputs.builder_name }} aws_default_region: ${{ secrets.AWS_DEFAULT_REGION }} aws_account_id: ${{ secrets.AWS_ACCOUNT_ID }} + - name: Refresh BuildKit builder + # The runtime build above can leave the remote BuildKit connection stale; + # re-establish it (re-routing only if unhealthy) before the test build. + uses: ./.github/actions/builder-refresher + with: + builder_name: ${{ needs.changed-files.outputs.builder_name }} + flavor: general + arch: linux/amd64 - name: Run unit tests inside the container image # Build the Dockerfile `test` stage (FROM runtime): its RUN pytest runs # the suite against the exact shipped image (Python 3.12 + baked deps), # validating the container itself rather than a pip-installed runner env. - # push: false — the test stage is never shipped; the build still runs the - # suite. Same builder as above, so the runtime layers are a cache hit. - uses: ./.github/actions/build-deploy-component - with: - component: power-agent - target: test - push: false - image_tag: ${{ github.sha }}-power-agent-test - builder_name: ${{ needs.changed-files.outputs.builder_name }} - aws_default_region: ${{ secrets.AWS_DEFAULT_REGION }} - aws_account_id: ${{ secrets.AWS_ACCOUNT_ID }} + # Reuses the builder + ECR Docker Hub mirror the build step set up above + # (runtime layers are a cache hit); no --push/--load runs pytest without + # exporting the never-shipped test image. + env: + ECR_HOSTNAME: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_DEFAULT_REGION }}.amazonaws.com + run: | + docker buildx build --builder ${{ needs.changed-files.outputs.builder_name }} \ + --target test \ + --build-arg DOCKER_PROXY=${ECR_HOSTNAME}/dockerhub/ \ + -f deploy/power-agent/Dockerfile deploy/power-agent - name: Set up Helm uses: azure/setup-helm@b9e51907a09c216f16ebe8536097933489208112 # v4.3.0 with: From 89ced1eeb8b80840616c7039752ff019a7bfc67b Mon Sep 17 00:00:00 2001 From: Anant Sharma Date: Wed, 24 Jun 2026 09:51:01 -0700 Subject: [PATCH 3/4] test: drop unused markers from power-agent tests The power-agent CI runs `pytest tests` with no -m filter, so the pre_merge/gpu_0/unit marks never selected anything. Remove the pytestmark lines (and the now-unused pytest import) and the marker registry from pytest.ini, and fix a stale power-agent-test job reference in the pytest.ini comment. Signed-off-by: Anant Sharma --- deploy/power-agent/pytest.ini | 6 +----- deploy/power-agent/tests/test_annotation_scope.py | 3 --- deploy/power-agent/tests/test_reconcile_failsafe.py | 3 --- 3 files changed, 1 insertion(+), 11 deletions(-) diff --git a/deploy/power-agent/pytest.ini b/deploy/power-agent/pytest.ini index ff5f1113022e..8b3702db49e5 100644 --- a/deploy/power-agent/pytest.ini +++ b/deploy/power-agent/pytest.ini @@ -7,7 +7,7 @@ # filterwarnings=error with TRTLLM/pydantic-specific filters (and extra plugins) # that are absent from the power-agent CI image, which installs only # pytest + pynvml + kubernetes + prometheus-client (see .github/workflows/pr.yaml -# and post-merge-ci.yml power-agent-test job). +# and post-merge-ci.yml power-agent job). [pytest] testpaths = tests # rootdir is deploy/power-agent (this file); "." puts it on sys.path so @@ -15,7 +15,3 @@ testpaths = tests # image's test stage (WORKDIR /app, where power_agent.py + this file live). pythonpath = . addopts = -ra --strict-markers --strict-config -markers = - pre_merge: marks tests to run before merging - gpu_0: marks tests that don't require GPU - unit: marks tests as unit tests diff --git a/deploy/power-agent/tests/test_annotation_scope.py b/deploy/power-agent/tests/test_annotation_scope.py index ef646b0c41ba..54e365e18adc 100644 --- a/deploy/power-agent/tests/test_annotation_scope.py +++ b/deploy/power-agent/tests/test_annotation_scope.py @@ -18,11 +18,8 @@ from unittest.mock import MagicMock, patch import power_agent -import pytest from power_agent import POWER_ANNOTATION_KEY, PowerAgent -pytestmark = [pytest.mark.pre_merge, pytest.mark.gpu_0, pytest.mark.unit] - SAFE_DEFAULT = 500 diff --git a/deploy/power-agent/tests/test_reconcile_failsafe.py b/deploy/power-agent/tests/test_reconcile_failsafe.py index 3ddf367afa03..a6aade186650 100644 --- a/deploy/power-agent/tests/test_reconcile_failsafe.py +++ b/deploy/power-agent/tests/test_reconcile_failsafe.py @@ -19,11 +19,8 @@ import unittest from unittest.mock import MagicMock -import pytest from power_agent import PowerAgent -pytestmark = [pytest.mark.pre_merge, pytest.mark.gpu_0, pytest.mark.unit] - def _make_agent(core_v1, device_count: int = 2) -> PowerAgent: """Build a PowerAgent without touching NVML / K8s (bypass __init__).""" From 17a866bcb6b5f6c27b46005e14361c7e3d62261d Mon Sep 17 00:00:00 2001 From: Anant Sharma Date: Wed, 24 Jun 2026 10:04:38 -0700 Subject: [PATCH 4/4] test: exclude power-agent from monorepo pytest collection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The monorepo pytest run (dynamo-runtime test job) recursively collected deploy/power-agent/tests, whose 'tests' package name collides with the repo-root 'tests' package, producing ModuleNotFoundError collection errors. power-agent ships its own pytest.ini and runs its suite inside its container, so ignore the subtree in the root pytest addopts — same rationale as the existing duplicate-collection ignores. Signed-off-by: Anant Sharma --- pyproject.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 501f12d1caf9..2249848c1f07 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -182,6 +182,10 @@ addopts = [ "--ignore-glob=components/src/dynamo/sglang/multimodal_utils/*", "--ignore-glob=components/src/dynamo/vllm/multimodal_utils/*", "--ignore-glob=examples/backends/sglang/slurm_jobs/*", + # power-agent is a self-contained deployable with its own pytest.ini; its + # `tests` package is tested inside its container, and collecting it here + # collides with the repo-root `tests` package. + "--ignore-glob=deploy/power-agent/*", # FIXME: Get relative/generic blob paths to work here ] xfail_strict = true