From 8ed061fd5f2e116b1e6e02bfd620f4ca3f86b97d Mon Sep 17 00:00:00 2001 From: sebastianliebscher Date: Sun, 8 Oct 2023 09:04:24 +0200 Subject: [PATCH 1/6] build: Parallelize the CI image builds - replaces custom script to set tags with docker/metadata-action GitHub Action - replaces custom script to sequentially build images with docker/build-push-action GitHub Action - moves docker-release.yml logic into docker.yml by utilizing docker/metadata-action 'tags: type=pep440,pattern={{version}}' - removes docker buildx local cache usage as every build runs on its own job hence on different machines (docker buildx registry cache will be a follow-up PR) --- .github/workflows/docker-release.yml | 42 ------ .github/workflows/docker.yml | 207 +++++++++++++++++++++++++-- scripts/docker_build_push.sh | 189 ------------------------ 3 files changed, 192 insertions(+), 246 deletions(-) delete mode 100644 .github/workflows/docker-release.yml delete mode 100755 scripts/docker_build_push.sh diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml deleted file mode 100644 index 7cfba73299b4..000000000000 --- a/.github/workflows/docker-release.yml +++ /dev/null @@ -1,42 +0,0 @@ -name: Docker - -on: - release: - types: [published] -jobs: - config: - runs-on: "ubuntu-latest" - outputs: - has-secrets: ${{ steps.check.outputs.has-secrets }} - steps: - - name: "Check for secrets" - id: check - shell: bash - run: | - if [ -n "${{ (secrets.DOCKERHUB_USER != '' && secrets.DOCKERHUB_TOKEN != '') || '' }}" ]; then - echo "has-secrets=1" >> "$GITHUB_OUTPUT" - fi - - docker-release: - needs: config - if: needs.config.outputs.has-secrets - name: docker-release - runs-on: ubuntu-latest - steps: - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v3 - with: - persist-credentials: false - submodules: recursive - ref: ${{ github.ref }} - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - shell: bash - env: - DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - run: | - GITHUB_RELEASE_TAG_NAME="${{ github.event.release.tag_name }}" - ./scripts/docker_build_push.sh "$GITHUB_RELEASE_TAG_NAME" diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 6160d3cc1f59..1cfa1232ae14 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -1,12 +1,15 @@ name: Docker on: + release: + types: [ published ] push: branches: - 'master' pull_request: types: [synchronize, opened, reopened, ready_for_review] + jobs: config: runs-on: "ubuntu-latest" @@ -25,29 +28,205 @@ jobs: echo "no secrets!" fi - docker-build: + + build-lean-image: + name: Build ${{ matrix.image.version }} lean image needs: config if: needs.config.outputs.has-secrets - name: docker-build runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + image: + - {version: "3.9-slim-bookworm", suffix: ""} + - {version: "3.10-slim-bookworm", suffix: "-py310"} steps: - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v3 + - name: Checkout + uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ github.repository }} + flavor: | + latest=false + suffix=${{ matrix.image.suffix }} + tags: | + type=sha,prefix=,format=long + type=ref,event=pr + type=raw,value=master,enable={{is_default_branch}} + type=raw,value=latest,enable={{is_default_branch}} + type=pep440,pattern={{version}} + labels: | + target=lean + build_actor=${{ github.actor }} + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Build and push + uses: docker/build-push-action@v5 + with: + context: . + platforms: linux/amd64 + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + target: lean + build-args: | + PY_VER=${{ matrix.image.version }} + + + build-dev-image: + name: Build dev image + needs: config + if: needs.config.outputs.has-secrets + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ github.repository }} + flavor: | + latest=false + suffix=dev + tags: | + type=sha,prefix=,format=long + type=ref,event=pr + type=raw,value=master,enable={{is_default_branch}} + type=raw,value=latest,enable={{is_default_branch}} + type=pep440,pattern={{version}} + labels: | + target=dev + build_actor=${{ github.actor }} + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Build and push + uses: docker/build-push-action@v5 + with: + context: . + platforms: linux/amd64 + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + target: dev + + + build-websocket-image: + name: Build websocket image + needs: config + if: needs.config.outputs.has-secrets + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 with: persist-credentials: false - name: Set up QEMU - uses: docker/setup-qemu-action@v1 + uses: docker/setup-qemu-action@v3 + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ github.repository }} + flavor: | + latest=false + suffix=websocket + tags: | + type=sha,prefix=,format=long + type=ref,event=pr + type=raw,value=master,enable={{is_default_branch}} + type=raw,value=latest,enable={{is_default_branch}} + type=pep440,pattern={{version}} + labels: | + build_actor=${{ github.actor }} - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - shell: bash - env: - DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - run: | - ./scripts/docker_build_push.sh + uses: docker/setup-buildx-action@v3 + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Build and push + uses: docker/build-push-action@v5 + with: + context: ./superset-websocket + platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + + build-dockerize-image: + name: Build Dockerize image + needs: config + if: needs.config.outputs.has-secrets + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ github.repository }} + flavor: | + latest=false + tags: | + type=raw,value=dockerize,enable={{is_default_branch}} + labels: | + build_actor=${{ github.actor }} + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Build and push + uses: docker/build-push-action@v5 + with: + context: . + file: dockerize.Dockerfile + platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + + + # build Dockerfile 'ci' target, save to archive and upload as artifact + build-ephemeral-image: + name: Build ephemeral env image + needs: config + if: needs.config.outputs.has-secrets && github.event_name == 'pull_request' + runs-on: ubuntu-latest + steps: + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@v3 + with: + persist-credentials: false + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 - name: Build ephemeral env image - if: github.event_name == 'pull_request' run: | mkdir -p ./build echo ${{ github.sha }} > ./build/SHA @@ -61,9 +240,7 @@ jobs: --label "build_actor=${GITHUB_ACTOR}" \ . docker save ${{ github.sha }} | gzip > ./build/${{ github.sha }}.tar.gz - - name: Upload build artifacts - if: github.event_name == 'pull_request' uses: actions/upload-artifact@v3 with: name: build diff --git a/scripts/docker_build_push.sh b/scripts/docker_build_push.sh deleted file mode 100755 index 8ae82faaeb5e..000000000000 --- a/scripts/docker_build_push.sh +++ /dev/null @@ -1,189 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -set -eo pipefail - -GITHUB_RELEASE_TAG_NAME="$1" - -SHA=$(git rev-parse HEAD) -REPO_NAME="apache/superset" - -if [[ "${GITHUB_EVENT_NAME}" == "pull_request" ]]; then - REFSPEC=$(echo "${GITHUB_HEAD_REF}" | sed 's/[^a-zA-Z0-9]/-/g' | head -c 40) - PR_NUM=$(echo "${GITHUB_REF}" | sed 's:refs/pull/::' | sed 's:/merge::') - LATEST_TAG="pr-${PR_NUM}" -elif [[ "${GITHUB_EVENT_NAME}" == "release" ]]; then - REFSPEC=$(echo "${GITHUB_REF}" | sed 's:refs/tags/::' | head -c 40) - LATEST_TAG="${REFSPEC}" -else - REFSPEC=$(echo "${GITHUB_REF}" | sed 's:refs/heads/::' | sed 's/[^a-zA-Z0-9]/-/g' | head -c 40) - LATEST_TAG="${REFSPEC}" -fi - - -if [[ "${REFSPEC}" == "master" ]]; then - LATEST_TAG="master" -fi - -# get the latest release tag -if [ -n "${GITHUB_RELEASE_TAG_NAME}" ]; then - output=$(source ./scripts/tag_latest_release.sh "${GITHUB_RELEASE_TAG_NAME}" --dry-run) || true - SKIP_TAG=$(echo "${output}" | grep "SKIP_TAG" | cut -d'=' -f2) - if [[ "${SKIP_TAG}" == "SKIP_TAG::false" ]]; then - LATEST_TAG="latest" - fi -fi - -if [[ "${TEST_ENV}" == "true" ]]; then - # don't run the build in test environment - echo "LATEST_TAG is ${LATEST_TAG}" - exit 0 -fi - - -cat< Date: Sun, 8 Oct 2023 12:01:32 +0200 Subject: [PATCH 2/6] build always but push only if secrets available --- .github/workflows/docker.yml | 56 ++++++++++++++++++++++++++++++++---- 1 file changed, 50 insertions(+), 6 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 1cfa1232ae14..b86141c70cb6 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -24,7 +24,6 @@ jobs: echo "has-secrets=1" >> "$GITHUB_OUTPUT" echo "has secrets!" else - echo "has-secrets=0" >> "$GITHUB_OUTPUT" echo "no secrets!" fi @@ -64,12 +63,25 @@ jobs: build_actor=${{ github.actor }} - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + - name: Build + uses: docker/build-push-action@v5 + with: + context: . + platforms: linux/amd64 + load: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + target: lean + build-args: | + PY_VER=${{ matrix.image.version }} - name: Login to Docker Hub + if: needs.config.outputs.has-secrets uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Build and push + - name: Push + if: needs.config.outputs.has-secrets uses: docker/build-push-action@v5 with: context: . @@ -111,12 +123,23 @@ jobs: build_actor=${{ github.actor }} - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + - name: Build + uses: docker/build-push-action@v5 + with: + context: . + platforms: linux/amd64 + load: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + target: dev - name: Login to Docker Hub + if: needs.config.outputs.has-secrets uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Build and push + - name: Push + if: needs.config.outputs.has-secrets uses: docker/build-push-action@v5 with: context: . @@ -157,12 +180,22 @@ jobs: build_actor=${{ github.actor }} - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + - name: Build + uses: docker/build-push-action@v5 + with: + context: ./superset-websocket + platforms: linux/amd64,linux/arm64 + load: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} - name: Login to Docker Hub + if: needs.config.outputs.has-secrets uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Build and push + - name: Push + if: needs.config.outputs.has-secrets uses: docker/build-push-action@v5 with: context: ./superset-websocket @@ -197,12 +230,23 @@ jobs: build_actor=${{ github.actor }} - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + - name: Build + uses: docker/build-push-action@v5 + with: + context: . + file: dockerize.Dockerfile + platforms: linux/amd64,linux/arm64 + load: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} - name: Login to Docker Hub + if: needs.config.outputs.has-secrets uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Build and push + - name: Push + if: needs.config.outputs.has-secrets uses: docker/build-push-action@v5 with: context: . @@ -217,7 +261,7 @@ jobs: build-ephemeral-image: name: Build ephemeral env image needs: config - if: needs.config.outputs.has-secrets && github.event_name == 'pull_request' + if: github.event_name == 'pull_request' runs-on: ubuntu-latest steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" From 9788c13a94a8dc73ad413151493ec62a6b5efdbd Mon Sep 17 00:00:00 2001 From: sebastianliebscher Date: Sun, 8 Oct 2023 12:15:39 +0200 Subject: [PATCH 3/6] build always but push only if secrets available + rename --- .github/workflows/docker.yml | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index b86141c70cb6..b2dee9481080 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -1,4 +1,4 @@ -name: Docker +name: Build on: release: @@ -29,9 +29,8 @@ jobs: build-lean-image: - name: Build ${{ matrix.image.version }} lean image + name: ${{ matrix.image.version }} lean image needs: config - if: needs.config.outputs.has-secrets runs-on: ubuntu-latest strategy: fail-fast: false @@ -95,9 +94,8 @@ jobs: build-dev-image: - name: Build dev image + name: dev image needs: config - if: needs.config.outputs.has-secrets runs-on: ubuntu-latest steps: - name: Checkout @@ -151,9 +149,8 @@ jobs: build-websocket-image: - name: Build websocket image + name: websocket image needs: config - if: needs.config.outputs.has-secrets runs-on: ubuntu-latest steps: - name: Checkout @@ -206,9 +203,8 @@ jobs: build-dockerize-image: - name: Build Dockerize image + name: dockerize image needs: config - if: needs.config.outputs.has-secrets runs-on: ubuntu-latest steps: - name: Checkout @@ -259,7 +255,7 @@ jobs: # build Dockerfile 'ci' target, save to archive and upload as artifact build-ephemeral-image: - name: Build ephemeral env image + name: ephemeral env image needs: config if: github.event_name == 'pull_request' runs-on: ubuntu-latest From 8d61e42046917b036d80150a8c029e45af5d3814 Mon Sep 17 00:00:00 2001 From: sebastianliebscher Date: Sun, 8 Oct 2023 12:28:49 +0200 Subject: [PATCH 4/6] build single-platform + export multi-platform --- .github/workflows/docker.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index b2dee9481080..9e8e7379a99f 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -181,7 +181,7 @@ jobs: uses: docker/build-push-action@v5 with: context: ./superset-websocket - platforms: linux/amd64,linux/arm64 + platforms: linux/amd64 load: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} @@ -231,7 +231,7 @@ jobs: with: context: . file: dockerize.Dockerfile - platforms: linux/amd64,linux/arm64 + platforms: linux/amd64 load: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} From 359fee9ef8046b8a8942d40e75c450c5f89480ad Mon Sep 17 00:00:00 2001 From: sebastianliebscher Date: Sun, 8 Oct 2023 12:48:05 +0200 Subject: [PATCH 5/6] correct tag suffix --- .github/workflows/docker.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 9e8e7379a99f..0d0bc45851c4 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -109,7 +109,7 @@ jobs: images: ${{ github.repository }} flavor: | latest=false - suffix=dev + suffix=-dev tags: | type=sha,prefix=,format=long type=ref,event=pr @@ -166,7 +166,7 @@ jobs: images: ${{ github.repository }} flavor: | latest=false - suffix=websocket + suffix=-websocket tags: | type=sha,prefix=,format=long type=ref,event=pr From fb4633776cb3e0864847560788aad02ecc7ab203 Mon Sep 17 00:00:00 2001 From: sebastianliebscher Date: Sun, 8 Oct 2023 13:28:30 +0200 Subject: [PATCH 6/6] configure required checks --- .asf.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.asf.yaml b/.asf.yaml index ab980e0c3fb2..8a735cea56f7 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -64,7 +64,6 @@ github: - cypress-matrix (1, chrome) - cypress-matrix (2, chrome) - cypress-matrix (3, chrome) - - docker-build - frontend-build - pre-commit (3.9) - python-lint (3.9) @@ -72,6 +71,13 @@ github: - test-postgres (3.9) - test-postgres (3.10) - test-sqlite (3.9) + # Build workflow jobs - docker.yml + - 3.9-slim-bookworm lean image + - 3.10-slim-bookworm lean image + - dev image + - dockerize image + - ephemeral env image + - websocket image required_pull_request_reviews: dismiss_stale_reviews: false