From b89cba4b8b8f2431973a51c792ee92da0b96b51c Mon Sep 17 00:00:00 2001 From: JavaPythonAIForBAT Date: Wed, 9 Sep 2026 14:53:13 +0800 Subject: [PATCH 1/3] fix(ci): build with oci-mediatypes=false to produce Docker manifest list for SWR SWR rejects OCI image index (application/vnd.oci.image.index.v1+json) produced by 'docker buildx imagetools create' when combining OCI-format single-arch images, returning 400 Bad Request 'Invalid image, fail to parse manifest.json'. Root cause: buildx imagetools create selects output media type based on source manifests. When all sources are Docker v2 schema2, it produces a Docker manifest list (application/vnd.docker.distribution.manifest.list.v2+json) which SWR accepts. With OCI sources, it produces OCI image index which SWR rejects. Fix: add oci-mediatypes=false to the build output, so single-arch images are built as Docker v2 schema2. This allows docker buildx imagetools create to produce a Docker manifest list directly, eliminating the need for skopeo-based format conversion (which required sudo/root not available on the self-hosted runner). Changes: - Build output: add oci-mediatypes=false - build-push-digest 'Tag digest to prevent GC': replace skopeo copy with docker buildx imagetools create - merge-image: replace skopeo copy + docker manifest create/push with direct docker buildx imagetools create - merge-image-temp: same as merge-image - All 'Clean up temp tags': remove sudo apt-get install skopeo, guard skopeo delete with command -v check (no hard failure if unavailable) Signed-off-by: JavaPythonAIForBAT --- .github/workflows/_schedule_image_build.yaml | 123 +++++++------------ 1 file changed, 47 insertions(+), 76 deletions(-) diff --git a/.github/workflows/_schedule_image_build.yaml b/.github/workflows/_schedule_image_build.yaml index 6b574796ac58..3a39e6e035e8 100644 --- a/.github/workflows/_schedule_image_build.yaml +++ b/.github/workflows/_schedule_image_build.yaml @@ -334,7 +334,7 @@ jobs: file: ${{ inputs.dockerfile || 'Dockerfile' }} # only trigger when tag, branch/main push push: ${{ inputs.should_push }} - outputs: type=image,name=${{ inputs.swr_temp_repo }},push-by-digest=true,name-canonical=true,push=${{ inputs.should_push }} + outputs: type=image,name=${{ inputs.swr_temp_repo }},push-by-digest=true,name-canonical=true,push=${{ inputs.should_push }},oci-mediatypes=false # Image builds can compile inside Docker and therefore keep a fallback; # NPU test jobs use fail-on-cache-miss instead. build-args: | @@ -390,16 +390,12 @@ jobs: fi echo "Creating temp tag: ${{ inputs.swr_temp_repo }}:$TAG" - # Install skopeo if not present (image is Ubuntu-based). - if ! command -v skopeo >/dev/null 2>&1; then - apt-get update -qq && apt-get install -y -qq skopeo - fi - CREDS="${{ secrets.SWR_USERNAME }}:${{ secrets.SWR_PASSWORD }}" - skopeo copy \ - --src-creds "$CREDS" \ - --dest-creds "$CREDS" \ - "docker://${{ inputs.swr_temp_repo }}@${{ steps.build.outputs.digest }}" \ - "docker://${{ inputs.swr_temp_repo }}:$TAG" + # Copy the digest to a temp tag to prevent GC. The build uses + # oci-mediatypes=false so the image is Docker v2s2, and this produces + # a Docker manifest list which SWR accepts (no skopeo/sudo needed). + docker buildx imagetools create \ + -t "${{ inputs.swr_temp_repo }}:$TAG" \ + "${{ inputs.swr_temp_repo }}@${{ steps.build.outputs.digest }}" - name: Export digest run: | @@ -536,11 +532,6 @@ jobs: DIGEST_DIR="${{ runner.temp }}/digests" - # Install skopeo if not present. - if ! command -v skopeo >/dev/null 2>&1; then - apt-get update -qq && apt-get install -y -qq skopeo - fi - echo "Digests:" ls -la "$DIGEST_DIR" echo "Current tags:" @@ -557,51 +548,42 @@ jobs: fi echo "Creating tag $tag" - # SWR rejects OCI image index (application/vnd.oci.image.index.v1+json) - # produced by "docker buildx imagetools create". To work around this, - # copy each single-arch digest to the destination with a per-arch - # temp tag, converting to Docker v2 schema2 format via skopeo, then - # create a Docker manifest list which SWR accepts. - DIGEST_REFS="" + # The build step outputs Docker v2 schema2 images (oci-mediatypes=false), + # so "docker buildx imagetools create" produces a Docker manifest list + # (application/vnd.docker.distribution.manifest.list.v2+json) which SWR + # accepts, instead of an OCI image index (application/vnd.oci.image.index.v1+json). + DIGESTS="" for digest_file in "$DIGEST_DIR"/*; do digest=$(basename "$digest_file") - short_digest=$(echo "$digest" | cut -c1-12) - ARCH_TAG="${tag}-t${short_digest}" - echo " Copying $SOURCE_IMAGE@sha256:$digest → $ARCH_TAG (format v2s2)" - skopeo copy \ - --format v2s2 \ - "docker://${SOURCE_IMAGE}@sha256:${digest}" \ - "docker://${ARCH_TAG}" - DIGEST_REFS="$DIGEST_REFS $ARCH_TAG" + DIGESTS="$DIGESTS $SOURCE_IMAGE@sha256:$digest" done # shellcheck disable=SC2086 - docker manifest create "$tag" $DIGEST_REFS - docker manifest push "$tag" - - # Clean up per-arch temp tags - for digest_ref in $DIGEST_REFS; do - echo " Cleaning up $digest_ref" - skopeo delete "docker://$digest_ref" || true - done + docker buildx imagetools create \ + -t "$tag" \ + $DIGESTS done - name: Clean up temp tags if: ${{ inputs.build_type == 'daily' }} run: | - sudo apt-get update -qq && sudo apt-get install -y -qq skopeo SUFFIX="" if [ -n "${{ inputs.suffix }}" ]; then SUFFIX="-${{ inputs.suffix }}" fi DATE=$(date +'%Y%m%d') # Temp tags were created in SWR by build-push-digest, so clean them up there. - for arch in amd64 arm64; do - TAG="nightly-${{ inputs.schedule_tag_pattern }}${SUFFIX}-${arch}-${DATE}-temp" - skopeo delete \ - --creds "${{ secrets.SWR_USERNAME }}:${{ secrets.SWR_PASSWORD }}" \ - "docker://${{ inputs.swr_temp_repo }}:${TAG}" || true - done + # Skip cleanup if skopeo is not available (no sudo/root on this runner). + if command -v skopeo >/dev/null 2>&1; then + for arch in amd64 arm64; do + TAG="nightly-${{ inputs.schedule_tag_pattern }}${SUFFIX}-${arch}-${DATE}-temp" + skopeo delete \ + --creds "${{ secrets.SWR_USERNAME }}:${{ secrets.SWR_PASSWORD }}" \ + "docker://${{ inputs.swr_temp_repo }}:${TAG}" || true + done + else + echo "skopeo not available, skip temp tag cleanup" + fi merge-image-temp: name: merge-image-temp @@ -642,11 +624,6 @@ jobs: DIGEST_DIR="${{ runner.temp }}/digests" - # Install skopeo if not present. - if ! command -v skopeo >/dev/null 2>&1; then - apt-get update -qq && apt-get install -y -qq skopeo - fi - SUFFIX="" if [ -n "${{ inputs.suffix }}" ]; then SUFFIX="-${{ inputs.suffix }}" @@ -659,32 +636,22 @@ jobs: ls -la "$DIGEST_DIR" echo "Creating multi-arch manifest tag: $TAG" - # SWR rejects OCI image index produced by "docker buildx imagetools create". - # Copy each single-arch digest to a per-arch temp tag in Docker v2 schema2 - # format via skopeo, then create a Docker manifest list which SWR accepts. + # The build step outputs Docker v2 schema2 images (oci-mediatypes=false), + # so "docker buildx imagetools create" produces a Docker manifest list + # (application/vnd.docker.distribution.manifest.list.v2+json) which SWR + # accepts, instead of an OCI image index (application/vnd.oci.image.index.v1+json). merge_and_push() { local dest_tag="$1" - local digest_refs="" + local digests="" for digest_file in "$DIGEST_DIR"/*; do digest=$(basename "$digest_file") - short_digest=$(echo "$digest" | cut -c1-12) - ARCH_TAG="${dest_tag}-t${short_digest}" - echo " Copying $SOURCE_IMAGE@sha256:$digest → $ARCH_TAG (format v2s2)" - skopeo copy \ - --format v2s2 \ - "docker://${SOURCE_IMAGE}@sha256:${digest}" \ - "docker://${ARCH_TAG}" - digest_refs="$digest_refs $ARCH_TAG" + digests="$digests $SOURCE_IMAGE@sha256:$digest" done # shellcheck disable=SC2086 - docker manifest create "$dest_tag" $digest_refs - docker manifest push "$dest_tag" - - for digest_ref in $digest_refs; do - echo " Cleaning up $digest_ref" - skopeo delete "docker://$digest_ref" || true - done + docker buildx imagetools create \ + -t "$dest_tag" \ + $digests } merge_and_push "$TAG" @@ -706,7 +673,6 @@ jobs: - name: Clean up temp tags if: ${{ inputs.build_type == 'daily' }} run: | - sudo apt-get update -qq && sudo apt-get install -y -qq skopeo SUFFIX="" if [ -n "${{ inputs.suffix }}" ]; then SUFFIX="-${{ inputs.suffix }}" @@ -714,9 +680,14 @@ jobs: VLLM_SHORT=$(echo "${{ inputs.vllm_commit }}" | cut -c1-7) ASCEND_SHORT=$(echo "${{ inputs.vllm_ascend_commit }}" | cut -c1-7) # Temp tags were created in SWR by build-push-digest, so clean them up there. - for arch in amd64 arm64; do - TAG="vllm-ascend-${VLLM_SHORT}-${ASCEND_SHORT}${SUFFIX}-${arch}-temp" - skopeo delete \ - --creds "${{ secrets.SWR_USERNAME }}:${{ secrets.SWR_PASSWORD }}" \ - "docker://${{ inputs.swr_temp_repo }}:${TAG}" || true - done + # Skip cleanup if skopeo is not available (no sudo/root on this runner). + if command -v skopeo >/dev/null 2>&1; then + for arch in amd64 arm64; do + TAG="vllm-ascend-${VLLM_SHORT}-${ASCEND_SHORT}${SUFFIX}-${arch}-temp" + skopeo delete \ + --creds "${{ secrets.SWR_USERNAME }}:${{ secrets.SWR_PASSWORD }}" \ + "docker://${{ inputs.swr_temp_repo }}:${TAG}" || true + done + else + echo "skopeo not available, skip temp tag cleanup" + fi From 896b869e8adb6e388c927f1d1ad9021b45d56182 Mon Sep 17 00:00:00 2001 From: JavaPythonAIForBAT Date: Wed, 9 Sep 2026 15:30:36 +0800 Subject: [PATCH 2/3] fix(ci): derive tag pattern from workflow_dispatch inputs The schedule_tag_pattern was hardcoded to 'main' in the parent workflow (schedule_image_build_and_push.yaml:161), so the final image tag was always 'nightly-main--' regardless of the branch/tag passed via workflow_dispatch. Add a 'Prepare tag pattern' step in all three jobs (build-push-digest, merge-image, merge-image-temp) that dynamically computes the pattern: - Push tag / schedule events: use schedule_tag_pattern default ('main') - workflow_dispatch with tag: use the tag itself - workflow_dispatch with branch: use branch_ref with '/' replaced by '-' All references to inputs.schedule_tag_pattern in tag generation, Docker meta, GC temp tags, and cleanup are updated to use the dynamic steps.tag_pattern.outputs.pattern. Signed-off-by: JavaPythonAIForBAT --- .github/workflows/_schedule_image_build.yaml | 58 +++++++++++++++++--- 1 file changed, 50 insertions(+), 8 deletions(-) diff --git a/.github/workflows/_schedule_image_build.yaml b/.github/workflows/_schedule_image_build.yaml index 3a39e6e035e8..9cc8e3328987 100644 --- a/.github/workflows/_schedule_image_build.yaml +++ b/.github/workflows/_schedule_image_build.yaml @@ -325,6 +325,20 @@ jobs: username: ${{ secrets.SWR_USERNAME }} password: ${{ secrets.SWR_PASSWORD }} + - name: Prepare tag pattern + id: tag_pattern + run: | + PATTERN="${{ inputs.schedule_tag_pattern }}" + if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then + if [ -n "${{ inputs.workflow_dispatch_tag }}" ]; then + PATTERN="${{ inputs.workflow_dispatch_tag }}" + elif [ -n "${{ inputs.branch_ref }}" ]; then + PATTERN="${{ inputs.branch_ref }}" + PATTERN="${PATTERN//\//-}" + fi + fi + echo "pattern=${PATTERN}" >> $GITHUB_OUTPUT + - name: Build and push uses: docker/build-push-action@v7 id: build @@ -382,10 +396,10 @@ jobs: ASCEND_SHORT=$(echo "${{ inputs.vllm_ascend_commit }}" | cut -c1-7) TAG="vllm-ascend-${VLLM_SHORT}-${ASCEND_SHORT}${SUFFIX}-${{ matrix.runner_info.tag }}-temp" else - TAG="nightly-${{ inputs.schedule_tag_pattern }}${SUFFIX}-${{ matrix.runner_info.tag }}-temp" + TAG="nightly-${{ steps.tag_pattern.outputs.pattern }}${SUFFIX}-${{ matrix.runner_info.tag }}-temp" if [ "${{ inputs.build_type }}" = "daily" ]; then DATE=$(date +'%Y%m%d') - TAG="nightly-${{ inputs.schedule_tag_pattern }}${SUFFIX}-${{ matrix.runner_info.tag }}-${DATE}-temp" + TAG="nightly-${{ steps.tag_pattern.outputs.pattern }}${SUFFIX}-${{ matrix.runner_info.tag }}-${DATE}-temp" fi fi echo "Creating temp tag: ${{ inputs.swr_temp_repo }}:$TAG" @@ -444,6 +458,20 @@ jobs: echo "SUFFIX=" >> $GITHUB_ENV fi + - name: Prepare tag pattern + id: tag_pattern + run: | + PATTERN="${{ inputs.schedule_tag_pattern }}" + if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then + if [ -n "${{ inputs.workflow_dispatch_tag }}" ]; then + PATTERN="${{ inputs.workflow_dispatch_tag }}" + elif [ -n "${{ inputs.branch_ref }}" ]; then + PATTERN="${{ inputs.branch_ref }}" + PATTERN="${PATTERN//\//-}" + fi + fi + echo "pattern=${PATTERN}" >> $GITHUB_OUTPUT + # ===== New: Generate custom tags for daily scenario ===== - name: Generate daily tags id: daily_tags @@ -454,7 +482,7 @@ jobs: # Get CANN version and convert to lowercase (e.g., 2026.08.03 -> 2026.08.03) CANN_VERSION=$(echo "${{ inputs.cann_version }}" | tr '[:upper:]' '[:lower:]') # Get the base tag pattern (e.g., main) - PATTERN="${{ inputs.schedule_tag_pattern }}" + PATTERN="${{ steps.tag_pattern.outputs.pattern }}" # Get the suffix SUFFIX="${{ inputs.suffix }}" @@ -499,9 +527,9 @@ jobs: # (e.g. main-310p, nightly-main-310p) without date info, so they are disabled # for daily builds only. type=pep440,pattern={{raw}},suffix=${{ env.SUFFIX }},enable=${{ inputs.build_type != 'daily' }} - type=schedule,pattern=${{ inputs.schedule_tag_pattern }},prefix=nightly-,suffix=${{ env.SUFFIX }},enable=${{ inputs.build_type != 'daily' }} + type=schedule,pattern=${{ steps.tag_pattern.outputs.pattern }},prefix=nightly-,suffix=${{ env.SUFFIX }},enable=${{ inputs.build_type != 'daily' }} type=raw,value=${{ inputs.workflow_dispatch_tag }},enable=${{ github.event_name == 'workflow_dispatch' && inputs.workflow_dispatch_tag != '' && inputs.build_type != 'daily' }},suffix=${{ env.SUFFIX }} - type=raw,value=${{ inputs.branch_ref }},enable=${{ github.event_name == 'workflow_dispatch' && inputs.workflow_dispatch_tag == '' }},prefix=nightly-,suffix=${{ env.SUFFIX }} + type=raw,value=${{ steps.tag_pattern.outputs.pattern }},enable=${{ github.event_name == 'workflow_dispatch' && inputs.workflow_dispatch_tag == '' }},prefix=nightly-,suffix=${{ env.SUFFIX }} # ===== New: Daily scenario specific tags ===== type=raw,value=${{ steps.daily_tags.outputs.daily_tag }},enable=${{ inputs.build_type == 'daily' }} @@ -525,7 +553,7 @@ jobs: SOURCE_IMAGE: ${{ inputs.swr_temp_repo }} TAGS: ${{ steps.meta.outputs.tags }} BUILD_TYPE: ${{ inputs.build_type }} - SCHEDULE_TAG_PATTERN: ${{ inputs.schedule_tag_pattern }} + SCHEDULE_TAG_PATTERN: ${{ steps.tag_pattern.outputs.pattern }} SUFFIX: ${{ env.SUFFIX }} run: | set -euo pipefail @@ -576,7 +604,7 @@ jobs: # Skip cleanup if skopeo is not available (no sudo/root on this runner). if command -v skopeo >/dev/null 2>&1; then for arch in amd64 arm64; do - TAG="nightly-${{ inputs.schedule_tag_pattern }}${SUFFIX}-${arch}-${DATE}-temp" + TAG="nightly-${{ steps.tag_pattern.outputs.pattern }}${SUFFIX}-${arch}-${DATE}-temp" skopeo delete \ --creds "${{ secrets.SWR_USERNAME }}:${{ secrets.SWR_PASSWORD }}" \ "docker://${{ inputs.swr_temp_repo }}:${TAG}" || true @@ -613,6 +641,20 @@ jobs: username: ${{ secrets.SWR_USERNAME }} password: ${{ secrets.SWR_PASSWORD }} + - name: Prepare tag pattern + id: tag_pattern + run: | + PATTERN="${{ inputs.schedule_tag_pattern }}" + if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then + if [ -n "${{ inputs.workflow_dispatch_tag }}" ]; then + PATTERN="${{ inputs.workflow_dispatch_tag }}" + elif [ -n "${{ inputs.branch_ref }}" ]; then + PATTERN="${{ inputs.branch_ref }}" + PATTERN="${PATTERN//\//-}" + fi + fi + echo "pattern=${PATTERN}" >> $GITHUB_OUTPUT + - name: Merge and push multi-arch manifest to SWR temp env: # IMAGE is the destination (SWR temp); SOURCE_IMAGE is where the @@ -660,7 +702,7 @@ jobs: if [ "${{ inputs.build_type }}" = "daily" ]; then DATE=$(date +'%Y%m%d') CANN_VERSION=$(echo "${{ inputs.cann_version }}" | tr '[:upper:]' '[:lower:]') - PATTERN="${{ inputs.schedule_tag_pattern }}" + PATTERN="${{ steps.tag_pattern.outputs.pattern }}" if [ -n "${{ inputs.suffix }}" ]; then DAILY_TAG="${IMAGE}:nightly-${PATTERN}-${{ inputs.suffix }}-cann${CANN_VERSION}-${DATE}" else From b7ebb74f2b52a19bce1622d5efb587f1361b7bcc Mon Sep 17 00:00:00 2001 From: JavaPythonAIForBAT Date: Wed, 9 Sep 2026 16:30:16 +0800 Subject: [PATCH 3/3] fix(ci): skip quay.io sync on pull_request events When a pull_request with the image-build label runs the image build, image_build succeeds (build without push), which incorrectly triggers the trigger-image-sync job to dispatch the quay.io sync. Add github.event_name != 'pull_request' so sync only fires for non-PR events (push tag, branch/main push, workflow_dispatch, schedule), matching the intended behavior of syncing only after merge to main. Signed-off-by: JavaPythonAIForBAT --- .github/workflows/schedule_image_build_and_push.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/schedule_image_build_and_push.yaml b/.github/workflows/schedule_image_build_and_push.yaml index 78f3af678f42..8f058c223b55 100644 --- a/.github/workflows/schedule_image_build_and_push.yaml +++ b/.github/workflows/schedule_image_build_and_push.yaml @@ -226,7 +226,7 @@ jobs: trigger-image-sync: name: Trigger quay.io sync - if: ${{ !cancelled() && (needs.image_build.result == 'success' || needs.image_build_by_commit.result == 'success') }} + if: ${{ !cancelled() && github.event_name != 'pull_request' && (needs.image_build.result == 'success' || needs.image_build_by_commit.result == 'success') }} needs: [image_build, image_build_by_commit] runs-on: linux-amd64-cpu-4 steps: