diff --git a/.github/workflows/docker-pr-tag-cleanup.yml b/.github/workflows/docker-pr-tag-cleanup.yml new file mode 100644 index 00000000000..0f2c9046655 --- /dev/null +++ b/.github/workflows/docker-pr-tag-cleanup.yml @@ -0,0 +1,27 @@ +name: Docker PR tag cleanup + +on: + pull_request: + types: [closed] + +permissions: + contents: read + +jobs: + delete-pr-tag: + if: github.event.pull_request.head.repo.full_name == github.repository + runs-on: ubuntu-latest + steps: + - shell: bash + run: | + TOKEN=$(python3 -c " + import json, urllib.request + req = urllib.request.Request('https://hub.docker.com/v2/users/login/', + data=json.dumps({'username': '${{ secrets.DOCKERHUB_USERNAME }}', + 'password': '${{ secrets.DOCKERHUB_TOKEN }}'}).encode(), + headers={'Content-Type': 'application/json'}) + print(json.load(urllib.request.urlopen(req))['token'])") + curl -s -X DELETE \ + -H "Authorization: JWT ${TOKEN}" \ + "https://hub.docker.com/v2/repositories/radixark/miles/tags/pr-${{ github.event.pull_request.number }}/" \ + || true diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index 01571e85016..0652e7cf5cd 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -70,7 +70,60 @@ jobs: PR_LABELS_JSON: ${{ toJSON(github.event.pull_request.labels.*.name) }} run: python -m tests.ci.ci_policy + docker-paths: + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + outputs: + changed: ${{ steps.diff.outputs.changed }} + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - id: diff + shell: bash + run: | + git fetch --depth 1 origin "${{ github.event.pull_request.base.sha }}" + # Dockerfile.rocm doesn't feed the cu13-x86 build this pipeline produces. + if git diff --name-only "${{ github.event.pull_request.base.sha }}" HEAD \ + | grep -v '^docker/Dockerfile\.rocm$' \ + | grep -qE '^(docker/(Dockerfile|build\.py|patch/)|requirements\.txt)'; then + echo "changed=true" >> "$GITHUB_OUTPUT" + else + echo "changed=false" >> "$GITHUB_OUTPUT" + fi + + # Docker PRs: build the image first and run every suite inside it. + docker-build: + needs: [docker-paths] + if: needs.docker-paths.outputs.changed == 'true' && github.event.pull_request.head.repo.full_name == github.repository + runs-on: ["h200", "2gpu"] + timeout-minutes: 180 + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + with: + driver-opts: | + image=moby/buildkit:latest + network=host + - name: Install Python + dependencies + run: | + apt-get update && apt-get install -y python3 python3-pip + pip3 install --break-system-packages typer + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Build and push PR tag + run: | + python3 docker/build.py --variant cu13-x86 --image-tag custom \ + --custom-tag pr-${{ github.event.pull_request.number }} --push + resolve-ci-image: + needs: [docker-paths, docker-build] + if: always() && !cancelled() && (needs.docker-build.result == 'success' || needs.docker-build.result == 'skipped') runs-on: ubuntu-latest outputs: container_image: ${{ steps.resolve.outputs.container_image }} @@ -81,8 +134,11 @@ jobs: env: PR_BODY: ${{ github.event.pull_request.body || '' }} INPUT_CI_IMAGE_TAG: ${{ github.event.inputs.ci_image_tag || '' }} + DOCKER_BUILT: ${{ needs.docker-build.result == 'success' }} + PR_NUMBER: ${{ github.event.pull_request.number || '' }} run: | CI_IMAGE_TAG="${INPUT_CI_IMAGE_TAG}" + [ -z "$CI_IMAGE_TAG" ] && [ "$DOCKER_BUILT" = "true" ] && CI_IMAGE_TAG="pr-${PR_NUMBER}" if [ -n "$PR_BODY" ]; then PR_CI_IMAGE_TAG=$(echo "$PR_BODY" | grep -m1 -oP '^ci-image-tag:\s+\K\S+' || true) [ -z "$CI_IMAGE_TAG" ] && [ -n "$PR_CI_IMAGE_TAG" ] && CI_IMAGE_TAG="$PR_CI_IMAGE_TAG" diff --git a/docker/Dockerfile b/docker/Dockerfile index 7d5d4f11bbd..23badbacd5d 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -120,7 +120,7 @@ RUN if [ "${ENABLE_CUDA_13}" = "1" ]; then \ # https://github.com/NVIDIA/TransformerEngine/pull/3141; drop it after TE v2.18. COPY docker/patch/ /tmp/patches/ RUN if [ "${ENABLE_CUDA_13}" = "1" ] && [ -d /tmp/patches/cu13 ]; then \ - TE_DIR=$(python -c 'import transformer_engine; print(transformer_engine.__path__[0])') && \ + TE_DIR=$(python -c 'import importlib.util; print(importlib.util.find_spec("transformer_engine").submodule_search_locations[0])') && \ for p in /tmp/patches/cu13/*.patch; do \ echo "Applying $(basename $p) to $TE_DIR" && \ patch -d "$TE_DIR" -p1 < "$p"; \ diff --git a/docs/ci/02-docker-build.md b/docs/ci/02-docker-build.md index 5dda33668da..ddd88700f3f 100644 --- a/docs/ci/02-docker-build.md +++ b/docs/ci/02-docker-build.md @@ -5,7 +5,7 @@ description: The Dockerfiles, the build script, the remote build workflow, and h # Docker build -GPU CI runs inside `radixark/miles`. This doc maps which Dockerfiles exist, the script that builds them, how the remote build is triggered, and how to build & push manually. +GPU CI runs inside `radixark/miles`. This doc maps which Dockerfiles exist, the script that builds them, the PR-side build check, how the remote build is triggered, and how to build & push manually. ## Dockerfiles @@ -57,6 +57,20 @@ The **Tag** column is for `--image-tag dev`, which also pushes a timestamped `de A multi-arch build (`cu13`) needs Buildx's `docker-container` driver and is push-only — buildx writes the manifest straight to the registry, it can't load into the local image store. Use `cu13-x86` / `cu13-aarch64` (single-platform; the arm64 one cross-builds via QEMU on an x86 host) for local single-arch iteration. Other flags: `--push`, `--dry-run`, `--dockerfile`, `--custom-tag`. +## PR build check (in `pr-test.yml`) + +Dockerfile changes are build-tested on the PR itself, before merge — `docker-build.yml` only runs after a push to `main`, so without this breakage lands on `main` first. + +When a PR touches `docker/Dockerfile`, `docker/build.py`, `docker/patch/**`, or `requirements.txt` (detected by the `docker-paths` job), `pr-test.yml` inserts a build in front of the test matrix: + +| Job | What it does | +| --- | --- | +| `docker-build` | builds `cu13-x86` and pushes a PR-scoped `radixark/miles:pr-` tag (same-repo PRs; fork PRs skip it and test on `dev`) | +| `resolve-ci-image` | waits for the build and resolves the CI image to `pr-`, so **every GPU suite runs inside the freshly built image**; a failed build stops the matrix instead of testing the stale image. The fresh build outranks a `ci-image-tag:` PR-body directive — the directive applies only when no PR image was built (non-docker or fork PRs) | +| `delete-pr-tag` (`docker-pr-tag-cleanup.yml`) | removes the `pr-` tag when the PR closes; the tag stays available for re-runs while the PR is open | + +Non-docker PRs are untouched: `docker-paths` reports no change, `docker-build` skips, and the matrix runs on `dev` as before. + ## Remote docker build (`docker-build.yml`) The only automated builder of `radixark/miles`. Two jobs: