Skip to content

[BugFix][CI] Build with oci-mediatypes=false to produce Docker manifest list for SWR #16130

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
wangxiyuan merged 3 commits into from
Sep 10, 2026
Merged

[BugFix][CI] Build with oci-mediatypes=false to produce Docker manifest list for SWR #16130

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
b89cba4
fix(ci): build with oci-mediatypes=false to produce Docker manifest l…
JavaPythonAIForBAT Sep 9, 2026
896b869
fix(ci): derive tag pattern from workflow_dispatch inputs
JavaPythonAIForBAT Sep 9, 2026
b7ebb74
fix(ci): skip quay.io sync on pull_request events
JavaPythonAIForBAT Sep 9, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
179 changes: 96 additions & 83 deletions .github/workflows/_schedule_image_build.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -325,6 +325,20 @@ jobs:
username: ${{ secrets.SWR_USERNAME }}
password: ${{ secrets.SWR_PASSWORD }}

- name: Prepare tag pattern
id: tag_pattern
run: |
PATTERN="${{ inputs.schedule_tag_pattern }}"
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
if [ -n "${{ inputs.workflow_dispatch_tag }}" ]; then
PATTERN="${{ inputs.workflow_dispatch_tag }}"
elif [ -n "${{ inputs.branch_ref }}" ]; then
PATTERN="${{ inputs.branch_ref }}"
PATTERN="${PATTERN//\//-}"
fi
fi
echo "pattern=${PATTERN}" >> $GITHUB_OUTPUT

- name: Build and push
uses: docker/build-push-action@v7
id: build
Expand All @@ -334,7 +348,7 @@ jobs:
file: ${{ inputs.dockerfile || 'Dockerfile' }}
# only trigger when tag, branch/main push
push: ${{ inputs.should_push }}
outputs: type=image,name=${{ inputs.swr_temp_repo }},push-by-digest=true,name-canonical=true,push=${{ inputs.should_push }}
outputs: type=image,name=${{ inputs.swr_temp_repo }},push-by-digest=true,name-canonical=true,push=${{ inputs.should_push }},oci-mediatypes=false
# Image builds can compile inside Docker and therefore keep a fallback;
# NPU test jobs use fail-on-cache-miss instead.
build-args: |
Expand Down Expand Up @@ -382,24 +396,20 @@ jobs:
ASCEND_SHORT=$(echo "${{ inputs.vllm_ascend_commit }}" | cut -c1-7)
TAG="vllm-ascend-${VLLM_SHORT}-${ASCEND_SHORT}${SUFFIX}-${{ matrix.runner_info.tag }}-temp"
else
TAG="nightly-${{ inputs.schedule_tag_pattern }}${SUFFIX}-${{ matrix.runner_info.tag }}-temp"
TAG="nightly-${{ steps.tag_pattern.outputs.pattern }}${SUFFIX}-${{ matrix.runner_info.tag }}-temp"
if [ "${{ inputs.build_type }}" = "daily" ]; then
DATE=$(date +'%Y%m%d')
TAG="nightly-${{ inputs.schedule_tag_pattern }}${SUFFIX}-${{ matrix.runner_info.tag }}-${DATE}-temp"
TAG="nightly-${{ steps.tag_pattern.outputs.pattern }}${SUFFIX}-${{ matrix.runner_info.tag }}-${DATE}-temp"
fi
fi
echo "Creating temp tag: ${{ inputs.swr_temp_repo }}:$TAG"

# Install skopeo if not present (image is Ubuntu-based).
if ! command -v skopeo >/dev/null 2>&1; then
apt-get update -qq && apt-get install -y -qq skopeo
fi
CREDS="${{ secrets.SWR_USERNAME }}:${{ secrets.SWR_PASSWORD }}"
skopeo copy \
--src-creds "$CREDS" \
--dest-creds "$CREDS" \
"docker://${{ inputs.swr_temp_repo }}@${{ steps.build.outputs.digest }}" \
"docker://${{ inputs.swr_temp_repo }}:$TAG"
# Copy the digest to a temp tag to prevent GC. The build uses
# oci-mediatypes=false so the image is Docker v2s2, and this produces
# a Docker manifest list which SWR accepts (no skopeo/sudo needed).
docker buildx imagetools create \
-t "${{ inputs.swr_temp_repo }}:$TAG" \
"${{ inputs.swr_temp_repo }}@${{ steps.build.outputs.digest }}"

- name: Export digest
run: |
Expand Down Expand Up @@ -448,6 +458,20 @@ jobs:
echo "SUFFIX=" >> $GITHUB_ENV
fi

- name: Prepare tag pattern
id: tag_pattern
run: |
PATTERN="${{ inputs.schedule_tag_pattern }}"
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
if [ -n "${{ inputs.workflow_dispatch_tag }}" ]; then
PATTERN="${{ inputs.workflow_dispatch_tag }}"
elif [ -n "${{ inputs.branch_ref }}" ]; then
PATTERN="${{ inputs.branch_ref }}"
PATTERN="${PATTERN//\//-}"
fi
fi
echo "pattern=${PATTERN}" >> $GITHUB_OUTPUT

# ===== New: Generate custom tags for daily scenario =====
- name: Generate daily tags
id: daily_tags
Expand All @@ -458,7 +482,7 @@ jobs:
# Get CANN version and convert to lowercase (e.g., 2026.08.03 -> 2026.08.03)
CANN_VERSION=$(echo "${{ inputs.cann_version }}" | tr '[:upper:]' '[:lower:]')
# Get the base tag pattern (e.g., main)
PATTERN="${{ inputs.schedule_tag_pattern }}"
PATTERN="${{ steps.tag_pattern.outputs.pattern }}"
# Get the suffix
SUFFIX="${{ inputs.suffix }}"

Expand Down Expand Up @@ -503,9 +527,9 @@ jobs:
# (e.g. main-310p, nightly-main-310p) without date info, so they are disabled
# for daily builds only.
type=pep440,pattern={{raw}},suffix=${{ env.SUFFIX }},enable=${{ inputs.build_type != 'daily' }}
type=schedule,pattern=${{ inputs.schedule_tag_pattern }},prefix=nightly-,suffix=${{ env.SUFFIX }},enable=${{ inputs.build_type != 'daily' }}
type=schedule,pattern=${{ steps.tag_pattern.outputs.pattern }},prefix=nightly-,suffix=${{ env.SUFFIX }},enable=${{ inputs.build_type != 'daily' }}
type=raw,value=${{ inputs.workflow_dispatch_tag }},enable=${{ github.event_name == 'workflow_dispatch' && inputs.workflow_dispatch_tag != '' && inputs.build_type != 'daily' }},suffix=${{ env.SUFFIX }}
type=raw,value=${{ inputs.branch_ref }},enable=${{ github.event_name == 'workflow_dispatch' && inputs.workflow_dispatch_tag == '' }},prefix=nightly-,suffix=${{ env.SUFFIX }}
type=raw,value=${{ steps.tag_pattern.outputs.pattern }},enable=${{ github.event_name == 'workflow_dispatch' && inputs.workflow_dispatch_tag == '' }},prefix=nightly-,suffix=${{ env.SUFFIX }}

# ===== New: Daily scenario specific tags =====
type=raw,value=${{ steps.daily_tags.outputs.daily_tag }},enable=${{ inputs.build_type == 'daily' }}
Expand All @@ -529,18 +553,13 @@ jobs:
SOURCE_IMAGE: ${{ inputs.swr_temp_repo }}
TAGS: ${{ steps.meta.outputs.tags }}
BUILD_TYPE: ${{ inputs.build_type }}
SCHEDULE_TAG_PATTERN: ${{ inputs.schedule_tag_pattern }}
SCHEDULE_TAG_PATTERN: ${{ steps.tag_pattern.outputs.pattern }}
SUFFIX: ${{ env.SUFFIX }}
run: |
set -euo pipefail

DIGEST_DIR="${{ runner.temp }}/digests"

# Install skopeo if not present.
if ! command -v skopeo >/dev/null 2>&1; then
apt-get update -qq && apt-get install -y -qq skopeo
fi

echo "Digests:"
ls -la "$DIGEST_DIR"
echo "Current tags:"
Expand All @@ -557,51 +576,42 @@ jobs:
fi
echo "Creating tag $tag"

# SWR rejects OCI image index (application/vnd.oci.image.index.v1+json)
# produced by "docker buildx imagetools create". To work around this,
# copy each single-arch digest to the destination with a per-arch
# temp tag, converting to Docker v2 schema2 format via skopeo, then
# create a Docker manifest list which SWR accepts.
DIGEST_REFS=""
# The build step outputs Docker v2 schema2 images (oci-mediatypes=false),
# so "docker buildx imagetools create" produces a Docker manifest list
# (application/vnd.docker.distribution.manifest.list.v2+json) which SWR
# accepts, instead of an OCI image index (application/vnd.oci.image.index.v1+json).
DIGESTS=""
for digest_file in "$DIGEST_DIR"/*; do
digest=$(basename "$digest_file")
short_digest=$(echo "$digest" | cut -c1-12)
ARCH_TAG="${tag}-t${short_digest}"
echo " Copying $SOURCE_IMAGE@sha256:$digest → $ARCH_TAG (format v2s2)"
skopeo copy \
--format v2s2 \
"docker://${SOURCE_IMAGE}@sha256:${digest}" \
"docker://${ARCH_TAG}"
DIGEST_REFS="$DIGEST_REFS $ARCH_TAG"
DIGESTS="$DIGESTS $SOURCE_IMAGE@sha256:$digest"
done

# shellcheck disable=SC2086
docker manifest create "$tag" $DIGEST_REFS
docker manifest push "$tag"

# Clean up per-arch temp tags
for digest_ref in $DIGEST_REFS; do
echo " Cleaning up $digest_ref"
skopeo delete "docker://$digest_ref" || true
done
docker buildx imagetools create \
-t "$tag" \
$DIGESTS
done

- name: Clean up temp tags
if: ${{ inputs.build_type == 'daily' }}
run: |
sudo apt-get update -qq && sudo apt-get install -y -qq skopeo
SUFFIX=""
if [ -n "${{ inputs.suffix }}" ]; then
SUFFIX="-${{ inputs.suffix }}"
fi
DATE=$(date +'%Y%m%d')
# Temp tags were created in SWR by build-push-digest, so clean them up there.
for arch in amd64 arm64; do
TAG="nightly-${{ inputs.schedule_tag_pattern }}${SUFFIX}-${arch}-${DATE}-temp"
skopeo delete \
--creds "${{ secrets.SWR_USERNAME }}:${{ secrets.SWR_PASSWORD }}" \
"docker://${{ inputs.swr_temp_repo }}:${TAG}" || true
done
# Skip cleanup if skopeo is not available (no sudo/root on this runner).
if command -v skopeo >/dev/null 2>&1; then
for arch in amd64 arm64; do
TAG="nightly-${{ steps.tag_pattern.outputs.pattern }}${SUFFIX}-${arch}-${DATE}-temp"
skopeo delete \
--creds "${{ secrets.SWR_USERNAME }}:${{ secrets.SWR_PASSWORD }}" \
"docker://${{ inputs.swr_temp_repo }}:${TAG}" || true
done
else
echo "skopeo not available, skip temp tag cleanup"
fi

merge-image-temp:
name: merge-image-temp
Expand Down Expand Up @@ -631,6 +641,20 @@ jobs:
username: ${{ secrets.SWR_USERNAME }}
password: ${{ secrets.SWR_PASSWORD }}

- name: Prepare tag pattern
id: tag_pattern
run: |
PATTERN="${{ inputs.schedule_tag_pattern }}"
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
if [ -n "${{ inputs.workflow_dispatch_tag }}" ]; then
PATTERN="${{ inputs.workflow_dispatch_tag }}"
elif [ -n "${{ inputs.branch_ref }}" ]; then
PATTERN="${{ inputs.branch_ref }}"
PATTERN="${PATTERN//\//-}"
fi
fi
echo "pattern=${PATTERN}" >> $GITHUB_OUTPUT

- name: Merge and push multi-arch manifest to SWR temp
env:
# IMAGE is the destination (SWR temp); SOURCE_IMAGE is where the
Expand All @@ -642,11 +666,6 @@ jobs:

DIGEST_DIR="${{ runner.temp }}/digests"

# Install skopeo if not present.
if ! command -v skopeo >/dev/null 2>&1; then
apt-get update -qq && apt-get install -y -qq skopeo
fi

SUFFIX=""
if [ -n "${{ inputs.suffix }}" ]; then
SUFFIX="-${{ inputs.suffix }}"
Expand All @@ -659,32 +678,22 @@ jobs:
ls -la "$DIGEST_DIR"
echo "Creating multi-arch manifest tag: $TAG"

# SWR rejects OCI image index produced by "docker buildx imagetools create".
# Copy each single-arch digest to a per-arch temp tag in Docker v2 schema2
# format via skopeo, then create a Docker manifest list which SWR accepts.
# The build step outputs Docker v2 schema2 images (oci-mediatypes=false),
# so "docker buildx imagetools create" produces a Docker manifest list
# (application/vnd.docker.distribution.manifest.list.v2+json) which SWR
# accepts, instead of an OCI image index (application/vnd.oci.image.index.v1+json).
merge_and_push() {
local dest_tag="$1"
local digest_refs=""
local digests=""
for digest_file in "$DIGEST_DIR"/*; do
digest=$(basename "$digest_file")
short_digest=$(echo "$digest" | cut -c1-12)
ARCH_TAG="${dest_tag}-t${short_digest}"
echo " Copying $SOURCE_IMAGE@sha256:$digest → $ARCH_TAG (format v2s2)"
skopeo copy \
--format v2s2 \
"docker://${SOURCE_IMAGE}@sha256:${digest}" \
"docker://${ARCH_TAG}"
digest_refs="$digest_refs $ARCH_TAG"
digests="$digests $SOURCE_IMAGE@sha256:$digest"
done

# shellcheck disable=SC2086
docker manifest create "$dest_tag" $digest_refs
docker manifest push "$dest_tag"

for digest_ref in $digest_refs; do
echo " Cleaning up $digest_ref"
skopeo delete "docker://$digest_ref" || true
done
docker buildx imagetools create \
-t "$dest_tag" \
$digests
}

merge_and_push "$TAG"
Expand All @@ -693,7 +702,7 @@ jobs:
if [ "${{ inputs.build_type }}" = "daily" ]; then
DATE=$(date +'%Y%m%d')
CANN_VERSION=$(echo "${{ inputs.cann_version }}" | tr '[:upper:]' '[:lower:]')
PATTERN="${{ inputs.schedule_tag_pattern }}"
PATTERN="${{ steps.tag_pattern.outputs.pattern }}"
if [ -n "${{ inputs.suffix }}" ]; then
DAILY_TAG="${IMAGE}:nightly-${PATTERN}-${{ inputs.suffix }}-cann${CANN_VERSION}-${DATE}"
else
Expand All @@ -706,17 +715,21 @@ jobs:
- name: Clean up temp tags
if: ${{ inputs.build_type == 'daily' }}
run: |
sudo apt-get update -qq && sudo apt-get install -y -qq skopeo
SUFFIX=""
if [ -n "${{ inputs.suffix }}" ]; then
SUFFIX="-${{ inputs.suffix }}"
fi
VLLM_SHORT=$(echo "${{ inputs.vllm_commit }}" | cut -c1-7)
ASCEND_SHORT=$(echo "${{ inputs.vllm_ascend_commit }}" | cut -c1-7)
# Temp tags were created in SWR by build-push-digest, so clean them up there.
for arch in amd64 arm64; do
TAG="vllm-ascend-${VLLM_SHORT}-${ASCEND_SHORT}${SUFFIX}-${arch}-temp"
skopeo delete \
--creds "${{ secrets.SWR_USERNAME }}:${{ secrets.SWR_PASSWORD }}" \
"docker://${{ inputs.swr_temp_repo }}:${TAG}" || true
done
# Skip cleanup if skopeo is not available (no sudo/root on this runner).
if command -v skopeo >/dev/null 2>&1; then
for arch in amd64 arm64; do
TAG="vllm-ascend-${VLLM_SHORT}-${ASCEND_SHORT}${SUFFIX}-${arch}-temp"
skopeo delete \
--creds "${{ secrets.SWR_USERNAME }}:${{ secrets.SWR_PASSWORD }}" \
"docker://${{ inputs.swr_temp_repo }}:${TAG}" || true
done
else
echo "skopeo not available, skip temp tag cleanup"
fi
2 changes: 1 addition & 1 deletion .github/workflows/schedule_image_build_and_push.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -226,7 +226,7 @@ jobs:

trigger-image-sync:
name: Trigger quay.io sync
if: ${{ !cancelled() && (needs.image_build.result == 'success' || needs.image_build_by_commit.result == 'success') }}
if: ${{ !cancelled() && github.event_name != 'pull_request' && (needs.image_build.result == 'success' || needs.image_build_by_commit.result == 'success') }}
needs: [image_build, image_build_by_commit]
runs-on: linux-amd64-cpu-4
steps:
Expand Down
Loading