From 412a71d7582fe83a29687d19ecf26b4513b338f1 Mon Sep 17 00:00:00 2001
From: Maxime Beauchemin <maximebeauchemin@gmail.com>
Date: Mon, 22 Jan 2024 12:08:57 -0800
Subject: [PATCH 01/19] changing the approach

---
 .github/workflows/docker-release.yml |  12 ++-
 .github/workflows/docker.yml         |  14 ++-
 scripts/docker_build_push.sh         | 150 +++++++++------------------
 3 files changed, 71 insertions(+), 105 deletions(-)

diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml
index 7cfba73299b4..5b50dc702065 100644
--- a/.github/workflows/docker-release.yml
+++ b/.github/workflows/docker-release.yml
@@ -22,6 +22,11 @@ jobs:
     if: needs.config.outputs.has-secrets
     name: docker-release
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        target: ["dev", "lean", "lean310", "lean39", "websocket", "dockerize"]
+        platform: ["linux/amd64", "linux/arm64"]
+      fail-fast: false
     steps:
       - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
         uses: actions/checkout@v3
@@ -29,14 +34,17 @@ jobs:
           persist-credentials: false
           submodules: recursive
           ref: ${{ github.ref }}
+
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v1
+
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v1
-      - shell: bash
+
+      - name: Build Docker Image
         env:
           DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
           DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
         run: |
           GITHUB_RELEASE_TAG_NAME="${{ github.event.release.tag_name }}"
-          ./scripts/docker_build_push.sh "$GITHUB_RELEASE_TAG_NAME"
+          ./scripts/docker_build_push.sh "$GITHUB_RELEASE_TAG_NAME" ${{ matrix.target }} ${{ matrix.platform }}
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index 6160d3cc1f59..87a7a50e962d 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -24,27 +24,35 @@ jobs:
             echo "has-secrets=0" >> "$GITHUB_OUTPUT"
             echo "no secrets!"
           fi
-
   docker-build:
     needs: config
     if: needs.config.outputs.has-secrets
     name: docker-build
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        target: ["dev", "lean", "lean310", "lean39", "websocket", "dockerize"]
+        platform: ["linux/amd64", "linux/arm64"]
+      fail-fast: false
     steps:
       - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
         uses: actions/checkout@v3
         with:
           persist-credentials: false
+
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v1
+
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v1
-      - shell: bash
+
+      - name: Build Docker Image
+        shell: bash
         env:
           DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
           DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
         run: |
-          ./scripts/docker_build_push.sh
+          ./scripts/docker_build_push.sh "" ${{ matrix.target }} ${{ matrix.platform }}
 
       - name: Build ephemeral env image
         if: github.event_name == 'pull_request'
diff --git a/scripts/docker_build_push.sh b/scripts/docker_build_push.sh
index 8ae82faaeb5e..ca63e7248267 100755
--- a/scripts/docker_build_push.sh
+++ b/scripts/docker_build_push.sh
@@ -18,9 +18,51 @@
 set -eo pipefail
 
 GITHUB_RELEASE_TAG_NAME="$1"
+TARGET="$2"
+BUILD_PLATFORM="$3" # should be either 'linux/amd64' or 'linux/arm64'
 
+# Common variables
 SHA=$(git rev-parse HEAD)
 REPO_NAME="apache/superset"
+DOCKER_ARGS="--load" # default args, change as needed
+DOCKER_CONTEXT="."
+
+
+case "${TARGET}" in
+  "dev")
+    DOCKER_TAGS="-t ${REPO_NAME}:${SHA}-dev -t ${REPO_NAME}:${REFSPEC}-dev -t ${DEV_TAG}"
+    BUILD_TARGET="dev"
+    ;;
+  "lean")
+    DOCKER_TAGS="-t ${REPO_NAME}:${SHA} -t ${REPO_NAME}:${REFSPEC} -t ${REPO_NAME}:${LATEST_TAG}"
+    BUILD_TARGET="lean"
+    ;;
+  "lean310")
+    DOCKER_TAGS="-t ${REPO_NAME}:${SHA}-py310 -t ${REPO_NAME}:${REFSPEC}-py310 -t ${REPO_NAME}:${LATEST_TAG}-py310"
+    BUILD_TARGET="lean"
+    BUILD_ARG="3.10-slim-bookworm"
+    ;;
+  "lean39")
+    DOCKER_TAGS="-t ${REPO_NAME}:${SHA}-py39 -t ${REPO_NAME}:${REFSPEC}-py39 -t ${REPO_NAME}:${LATEST_TAG}-py39"
+    BUILD_TARGET="lean"
+    BUILD_ARG="3.9-slim-bullseye"
+    ;;
+  "websocket")
+    DOCKER_TAGS="-t ${REPO_NAME}:${SHA}-websocket -t ${REPO_NAME}:${REFSPEC}-websocket -t ${REPO_NAME}:${LATEST_TAG}-websocket"
+    BUILD_TARGET="websocket"
+	DOCKER_CONTEXT="superset-websocket"
+    ;;
+  "dockerize")
+    DOCKER_TAGS="-t ${REPO_NAME}:dockerize"
+    BUILD_TARGET="dockerize"
+	DOCKER_CONTEXT="-f dockerize.Dockerfile ."
+    ;;
+  *)
+    echo "Invalid TARGET: ${TARGET}"
+    exit 1
+    ;;
+esac
+
 
 if [[ "${GITHUB_EVENT_NAME}" == "pull_request" ]]; then
   REFSPEC=$(echo "${GITHUB_HEAD_REF}" | sed 's/[^a-zA-Z0-9]/-/g' | head -c 40)
@@ -67,13 +109,11 @@ if [ -z "${DOCKERHUB_TOKEN}" ]; then
   echo "Skipping Docker push"
   # By default load it back
   DOCKER_ARGS="--load"
-  ARCHITECTURE_FOR_BUILD="linux/amd64 linux/arm64"
 else
   # Login and push
   docker logout
   docker login --username "${DOCKERHUB_USER}" --password "${DOCKERHUB_TOKEN}"
   DOCKER_ARGS="--push"
-  ARCHITECTURE_FOR_BUILD="linux/amd64,linux/arm64"
 fi
 set -x
 
@@ -85,105 +125,15 @@ else
   DEV_TAG="${REPO_NAME}:${LATEST_TAG}-dev"
 fi
 
-for BUILD_PLATFORM in $ARCHITECTURE_FOR_BUILD; do
-#
-# Build the dev image
-#
-docker buildx build --target dev \
-  $DOCKER_ARGS \
-  --cache-from=type=registry,ref=apache/superset:master-dev \
-  --cache-from=type=local,src=/tmp/superset \
-  --cache-to=type=local,ignore-error=true,dest=/tmp/superset \
-  -t "${REPO_NAME}:${SHA}-dev" \
-  -t "${REPO_NAME}:${REFSPEC}-dev" \
-  -t "${DEV_TAG}" \
-  --platform ${BUILD_PLATFORM} \
-  --label "sha=${SHA}" \
-  --label "built_at=$(date)" \
-  --label "target=dev" \
-  --label "build_actor=${GITHUB_ACTOR}" \
-  .
-
-#
-# Build the "lean" image
-#
-docker buildx build --target lean \
-  $DOCKER_ARGS \
-  --cache-from=type=local,src=/tmp/superset \
-  --cache-to=type=local,ignore-error=true,dest=/tmp/superset \
-  -t "${REPO_NAME}:${SHA}" \
-  -t "${REPO_NAME}:${REFSPEC}" \
-  -t "${REPO_NAME}:${LATEST_TAG}" \
-  --platform ${BUILD_PLATFORM} \
-  --label "sha=${SHA}" \
-  --label "built_at=$(date)" \
-  --label "target=lean" \
-  --label "build_actor=${GITHUB_ACTOR}" \
-  .
-
-#
-# Build the "lean310" image
-#
-docker buildx build --target lean \
-  $DOCKER_ARGS \
-  --cache-from=type=local,src=/tmp/superset \
-  --cache-to=type=local,ignore-error=true,dest=/tmp/superset \
-  -t "${REPO_NAME}:${SHA}-py310" \
-  -t "${REPO_NAME}:${REFSPEC}-py310" \
-  -t "${REPO_NAME}:${LATEST_TAG}-py310" \
-  --platform ${BUILD_PLATFORM} \
-  --build-arg PY_VER="3.10-slim-bookworm"\
-  --label "sha=${SHA}" \
-  --label "built_at=$(date)" \
-  --label "target=lean310" \
-  --label "build_actor=${GITHUB_ACTOR}" \
-  .
-
-#
-# Build the "lean39" image
-#
-docker buildx build --target lean \
-  $DOCKER_ARGS \
-  --cache-from=type=local,src=/tmp/superset \
-  --cache-to=type=local,ignore-error=true,dest=/tmp/superset \
-  -t "${REPO_NAME}:${SHA}-py39" \
-  -t "${REPO_NAME}:${REFSPEC}-py39" \
-  -t "${REPO_NAME}:${LATEST_TAG}-py39" \
-  --platform ${BUILD_PLATFORM} \
-  --build-arg PY_VER="3.9-slim-bullseye"\
-  --label "sha=${SHA}" \
-  --label "built_at=$(date)" \
-  --label "target=lean39" \
-  --label "build_actor=${GITHUB_ACTOR}" \
-  .
-
-#
-# Build the "websocket" image
-#
-docker buildx build \
-  $DOCKER_ARGS \
-  --cache-from=type=registry,ref=apache/superset:master-websocket \
-  -t "${REPO_NAME}:${SHA}-websocket" \
-  -t "${REPO_NAME}:${REFSPEC}-websocket" \
-  -t "${REPO_NAME}:${LATEST_TAG}-websocket" \
-  --platform ${BUILD_PLATFORM} \
-  --label "sha=${SHA}" \
-  --label "built_at=$(date)" \
-  --label "target=websocket" \
-  --label "build_actor=${GITHUB_ACTOR}" \
-  superset-websocket
-
-#
-# Build the dockerize image
-#
-docker buildx build \
-  $DOCKER_ARGS \
-  --cache-from=type=registry,ref=apache/superset:dockerize \
-  -t "${REPO_NAME}:dockerize" \
+docker buildx build --target ${BUILD_TARGET} \
+  ${DOCKER_ARGS} \
+  --cache-from=type=registry,ref=apache/superset:${BUILD_TARGET} \
+  --cache-to=type=registry,mode=max,ref=apache/superset:${BUILD_TARGET} \
+  ${DOCKER_TAGS} \
   --platform ${BUILD_PLATFORM} \
   --label "sha=${SHA}" \
   --label "built_at=$(date)" \
+  --label "target=${BUILD_TARGET}" \
   --label "build_actor=${GITHUB_ACTOR}" \
-  -f dockerize.Dockerfile \
-  .
-done
+  ${BUILD_ARG:+--build-arg PY_VER="${BUILD_ARG}"} \
+  $DOCKER_CONTEXT

From d9565fa4891a9dbd3af2734d6c745377a80bf5fc Mon Sep 17 00:00:00 2001
From: Maxime Beauchemin <maximebeauchemin@gmail.com>
Date: Mon, 22 Jan 2024 12:13:04 -0800
Subject: [PATCH 02/19] typo

---
 scripts/docker_build_push.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/docker_build_push.sh b/scripts/docker_build_push.sh
index ca63e7248267..86c2de8536b9 100755
--- a/scripts/docker_build_push.sh
+++ b/scripts/docker_build_push.sh
@@ -136,4 +136,4 @@ docker buildx build --target ${BUILD_TARGET} \
   --label "target=${BUILD_TARGET}" \
   --label "build_actor=${GITHUB_ACTOR}" \
   ${BUILD_ARG:+--build-arg PY_VER="${BUILD_ARG}"} \
-  $DOCKER_CONTEXT
+  ${DOCKER_CONTEXT}

From 83d2e7f903be35dcae70ff503b3030416b89a860 Mon Sep 17 00:00:00 2001
From: Maxime Beauchemin <maximebeauchemin@gmail.com>
Date: Mon, 22 Jan 2024 12:46:57 -0800
Subject: [PATCH 03/19] fix

---
 tests/unit_tests/scripts/docker_build_push_test.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tests/unit_tests/scripts/docker_build_push_test.py b/tests/unit_tests/scripts/docker_build_push_test.py
index 9f4c84318ed7..2359a845bc21 100644
--- a/tests/unit_tests/scripts/docker_build_push_test.py
+++ b/tests/unit_tests/scripts/docker_build_push_test.py
@@ -33,8 +33,11 @@ def test_tag_latest_release(tag, expected_output, branch):
     ) as subprocess_mock:
         result = BashMock.docker_build_push(tag, branch)
 
+        params = f"{tag} lean linux/amd64"
+        cmd = f"./scripts/docker_build_push.sh {params}"
+        print(cmd)
         subprocess_mock.assert_called_once_with(
-            f"./scripts/docker_build_push.sh {tag}",
+            cmd,
             shell=True,
             capture_output=True,
             text=True,

From 5af05976d8e86f7a07e0c4c15d294acdb3917194 Mon Sep 17 00:00:00 2001
From: Maxime Beauchemin <maximebeauchemin@gmail.com>
Date: Mon, 22 Jan 2024 13:30:44 -0800
Subject: [PATCH 04/19] temp chec

---
 scripts/docker_build_push.sh                  | 66 +++++++++----------
 tests/unit_tests/fixtures/bash_mock.py        |  6 +-
 .../scripts/docker_build_push_test.py         | 35 +++++-----
 3 files changed, 54 insertions(+), 53 deletions(-)

diff --git a/scripts/docker_build_push.sh b/scripts/docker_build_push.sh
index 86c2de8536b9..a5263bb2075f 100755
--- a/scripts/docker_build_push.sh
+++ b/scripts/docker_build_push.sh
@@ -28,6 +28,38 @@ DOCKER_ARGS="--load" # default args, change as needed
 DOCKER_CONTEXT="."
 
 
+if [[ "${GITHUB_EVENT_NAME}" == "pull_request" ]]; then
+  REFSPEC=$(echo "${GITHUB_HEAD_REF}" | sed 's/[^a-zA-Z0-9]/-/g' | head -c 40)
+  PR_NUM=$(echo "${GITHUB_REF}" | sed 's:refs/pull/::' | sed 's:/merge::')
+  LATEST_TAG="pr-${PR_NUM}"
+elif [[ "${GITHUB_EVENT_NAME}" == "release" ]]; then
+  REFSPEC=$(echo "${GITHUB_REF}" | sed 's:refs/tags/::' | head -c 40)
+  LATEST_TAG="${REFSPEC}"
+else
+  REFSPEC=$(echo "${GITHUB_REF}" | sed 's:refs/heads/::' | sed 's/[^a-zA-Z0-9]/-/g' | head -c 40)
+  LATEST_TAG="${REFSPEC}"
+fi
+
+
+if [[ "${REFSPEC}" == "master" ]]; then
+  LATEST_TAG="master"
+fi
+
+# get the latest release tag
+if [ -n "${GITHUB_RELEASE_TAG_NAME}" ]; then
+  output=$(source ./scripts/tag_latest_release.sh "${GITHUB_RELEASE_TAG_NAME}" --dry-run) || true
+  SKIP_TAG=$(echo "${output}" | grep "SKIP_TAG" | cut -d'=' -f2)
+  if [[ "${SKIP_TAG}" == "SKIP_TAG::false" ]]; then
+    LATEST_TAG="latest"
+  fi
+fi
+
+if [[ "${TEST_ENV}" == "true" ]]; then
+  # don't run the build in test environment
+  echo "LATEST_TAG is ${LATEST_TAG}"
+  exit 0
+fi
+
 case "${TARGET}" in
   "dev")
     DOCKER_TAGS="-t ${REPO_NAME}:${SHA}-dev -t ${REPO_NAME}:${REFSPEC}-dev -t ${DEV_TAG}"
@@ -63,40 +95,6 @@ case "${TARGET}" in
     ;;
 esac
 
-
-if [[ "${GITHUB_EVENT_NAME}" == "pull_request" ]]; then
-  REFSPEC=$(echo "${GITHUB_HEAD_REF}" | sed 's/[^a-zA-Z0-9]/-/g' | head -c 40)
-  PR_NUM=$(echo "${GITHUB_REF}" | sed 's:refs/pull/::' | sed 's:/merge::')
-  LATEST_TAG="pr-${PR_NUM}"
-elif [[ "${GITHUB_EVENT_NAME}" == "release" ]]; then
-  REFSPEC=$(echo "${GITHUB_REF}" | sed 's:refs/tags/::' | head -c 40)
-  LATEST_TAG="${REFSPEC}"
-else
-  REFSPEC=$(echo "${GITHUB_REF}" | sed 's:refs/heads/::' | sed 's/[^a-zA-Z0-9]/-/g' | head -c 40)
-  LATEST_TAG="${REFSPEC}"
-fi
-
-
-if [[ "${REFSPEC}" == "master" ]]; then
-  LATEST_TAG="master"
-fi
-
-# get the latest release tag
-if [ -n "${GITHUB_RELEASE_TAG_NAME}" ]; then
-  output=$(source ./scripts/tag_latest_release.sh "${GITHUB_RELEASE_TAG_NAME}" --dry-run) || true
-  SKIP_TAG=$(echo "${output}" | grep "SKIP_TAG" | cut -d'=' -f2)
-  if [[ "${SKIP_TAG}" == "SKIP_TAG::false" ]]; then
-    LATEST_TAG="latest"
-  fi
-fi
-
-if [[ "${TEST_ENV}" == "true" ]]; then
-  # don't run the build in test environment
-  echo "LATEST_TAG is ${LATEST_TAG}"
-  exit 0
-fi
-
-
 cat<<EOF
   Rolling with tags:
   - ${REPO_NAME}:${SHA}
diff --git a/tests/unit_tests/fixtures/bash_mock.py b/tests/unit_tests/fixtures/bash_mock.py
index 91194de6bf07..c6bc132b1000 100644
--- a/tests/unit_tests/fixtures/bash_mock.py
+++ b/tests/unit_tests/fixtures/bash_mock.py
@@ -32,10 +32,10 @@ def tag_latest_release(tag):
         return result
 
     @staticmethod
-    def docker_build_push(tag, branch):
-        bash_command = f"./scripts/docker_build_push.sh {tag}"
+    def docker_build_push(tag, target, platform, branch):
+        cmd = f'./scripts/docker_build_push.sh "{tag}" "{target}" "{platform}"'
         result = subprocess.run(
-            bash_command,
+            cmd,
             shell=True,
             capture_output=True,
             text=True,
diff --git a/tests/unit_tests/scripts/docker_build_push_test.py b/tests/unit_tests/scripts/docker_build_push_test.py
index 2359a845bc21..1fffa3d17d45 100644
--- a/tests/unit_tests/scripts/docker_build_push_test.py
+++ b/tests/unit_tests/scripts/docker_build_push_test.py
@@ -15,27 +15,31 @@ def wrapped(*args, **kwargs):
 
 
 @pytest.mark.parametrize(
-    "tag, expected_output, branch",
+    "tag, target, platform, expected_output, branch",
     [
-        ("1.0.0", "LATEST_TAG is master", "master"),
-        ("2.1.0", "LATEST_TAG is master", "master"),
-        ("2.1.1", "LATEST_TAG is latest", "master"),
-        ("3.0.0", "LATEST_TAG is latest", "master"),
-        ("2.1.0rc1", "LATEST_TAG is 2.1.0", "2.1.0"),
-        ("", "LATEST_TAG is foo", "foo"),
-        ("2.1", "LATEST_TAG is 2.1", "2.1"),
-        ("does_not_exist", "LATEST_TAG is does-not-exist", "does_not_exist"),
+        ("1.0.0", "lean", "linux/amd64", "LATEST_TAG is master", "master"),
+        ("2.1.0", "lean", "linux/amd64", "LATEST_TAG is master", "master"),
+        # ("2.1.1", "lean", "linux/amd64", "LATEST_TAG is master", "master"),
+        # ("3.0.0", "lean", "linux/amd64", "LATEST_TAG is master", "master"),
+        ("2.1.0rc1", "lean", "linux/amd64", "LATEST_TAG is 2.1.0", "2.1.0"),
+        ("", "lean", "linux/amd64", "LATEST_TAG is foo", "foo"),
+        ("2.1", "lean", "linux/amd64", "LATEST_TAG is 2.1", "2.1"),
+        (
+            "does_not_exist",
+            "lean",
+            "linux/amd64",
+            "LATEST_TAG is does-not-exist",
+            "does_not_exist",
+        ),
     ],
 )
-def test_tag_latest_release(tag, expected_output, branch):
+def test_tag_latest_release(tag, target, platform, expected_output, branch):
     with mock.patch(
         "tests.unit_tests.fixtures.bash_mock.subprocess.run", wraps=wrapped
     ) as subprocess_mock:
-        result = BashMock.docker_build_push(tag, branch)
+        result = BashMock.docker_build_push(tag, target, platform, branch)
 
-        params = f"{tag} lean linux/amd64"
-        cmd = f"./scripts/docker_build_push.sh {params}"
-        print(cmd)
+        cmd = f'./scripts/docker_build_push.sh "{tag}" "{target}" "{platform}"'
         subprocess_mock.assert_called_once_with(
             cmd,
             shell=True,
@@ -43,5 +47,4 @@ def test_tag_latest_release(tag, expected_output, branch):
             text=True,
             env={"TEST_ENV": "true", "GITHUB_REF": f"refs/heads/{branch}"},
         )
-
-        assert re.search(expected_output, result.stdout, re.MULTILINE)
+        assert re.search(expected_output, result.stdout.strip(), re.MULTILINE)

From 88642067a1056018afaa59a57f187f3b3d205c01 Mon Sep 17 00:00:00 2001
From: Maxime Beauchemin <maximebeauchemin@gmail.com>
Date: Mon, 22 Jan 2024 13:32:13 -0800
Subject: [PATCH 05/19] fix unit test

---
 tests/unit_tests/scripts/docker_build_push_test.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/unit_tests/scripts/docker_build_push_test.py b/tests/unit_tests/scripts/docker_build_push_test.py
index 1fffa3d17d45..edfd339bbc8f 100644
--- a/tests/unit_tests/scripts/docker_build_push_test.py
+++ b/tests/unit_tests/scripts/docker_build_push_test.py
@@ -19,8 +19,8 @@ def wrapped(*args, **kwargs):
     [
         ("1.0.0", "lean", "linux/amd64", "LATEST_TAG is master", "master"),
         ("2.1.0", "lean", "linux/amd64", "LATEST_TAG is master", "master"),
-        # ("2.1.1", "lean", "linux/amd64", "LATEST_TAG is master", "master"),
-        # ("3.0.0", "lean", "linux/amd64", "LATEST_TAG is master", "master"),
+        ("2.1.1", "lean", "linux/amd64", "LATEST_TAG is latest", "master"),
+        ("3.0.0", "lean", "linux/amd64", "LATEST_TAG is latest", "master"),
         ("2.1.0rc1", "lean", "linux/amd64", "LATEST_TAG is 2.1.0", "2.1.0"),
         ("", "lean", "linux/amd64", "LATEST_TAG is foo", "foo"),
         ("2.1", "lean", "linux/amd64", "LATEST_TAG is 2.1", "2.1"),

From ecc27050744c9a379203dbcbfcc875d7772b4320 Mon Sep 17 00:00:00 2001
From: Maxime Beauchemin <maximebeauchemin@gmail.com>
Date: Mon, 22 Jan 2024 13:44:31 -0800
Subject: [PATCH 06/19] skipping --target for dockerize and websocket

---
 scripts/docker_build_push.sh | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/scripts/docker_build_push.sh b/scripts/docker_build_push.sh
index a5263bb2075f..81d7791fc29e 100755
--- a/scripts/docker_build_push.sh
+++ b/scripts/docker_build_push.sh
@@ -81,12 +81,12 @@ case "${TARGET}" in
     ;;
   "websocket")
     DOCKER_TAGS="-t ${REPO_NAME}:${SHA}-websocket -t ${REPO_NAME}:${REFSPEC}-websocket -t ${REPO_NAME}:${LATEST_TAG}-websocket"
-    BUILD_TARGET="websocket"
+    BUILD_TARGET=""
 	DOCKER_CONTEXT="superset-websocket"
     ;;
   "dockerize")
     DOCKER_TAGS="-t ${REPO_NAME}:dockerize"
-    BUILD_TARGET="dockerize"
+    BUILD_TARGET=""
 	DOCKER_CONTEXT="-f dockerize.Dockerfile ."
     ;;
   *)
@@ -123,7 +123,13 @@ else
   DEV_TAG="${REPO_NAME}:${LATEST_TAG}-dev"
 fi
 
-docker buildx build --target ${BUILD_TARGET} \
+TARGET_ARGUMENT=""
+if [[ -n "${BUILD_TARGET}" ]]; then
+  TARGET_ARGUMENT="--target ${TARGET}"
+fi
+
+docker buildx build \
+  ${TARGET_ARGUMENT}
   ${DOCKER_ARGS} \
   --cache-from=type=registry,ref=apache/superset:${BUILD_TARGET} \
   --cache-to=type=registry,mode=max,ref=apache/superset:${BUILD_TARGET} \

From d0784e7094311251952a56e30d5ad425b906073c Mon Sep 17 00:00:00 2001
From: Maxime Beauchemin <maximebeauchemin@gmail.com>
Date: Mon, 22 Jan 2024 13:47:25 -0800
Subject: [PATCH 07/19] missed a thing

---
 scripts/docker_build_push.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/docker_build_push.sh b/scripts/docker_build_push.sh
index 81d7791fc29e..061722bf2a57 100755
--- a/scripts/docker_build_push.sh
+++ b/scripts/docker_build_push.sh
@@ -129,7 +129,7 @@ if [[ -n "${BUILD_TARGET}" ]]; then
 fi
 
 docker buildx build \
-  ${TARGET_ARGUMENT}
+  ${TARGET_ARGUMENT} \
   ${DOCKER_ARGS} \
   --cache-from=type=registry,ref=apache/superset:${BUILD_TARGET} \
   --cache-to=type=registry,mode=max,ref=apache/superset:${BUILD_TARGET} \

From b4f10b08b13ebce0a77c0de2e5c4441a7513f255 Mon Sep 17 00:00:00 2001
From: Maxime Beauchemin <maximebeauchemin@gmail.com>
Date: Mon, 22 Jan 2024 13:49:36 -0800
Subject: [PATCH 08/19] missed a thing

---
 scripts/docker_build_push.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/docker_build_push.sh b/scripts/docker_build_push.sh
index 061722bf2a57..44259d0759a5 100755
--- a/scripts/docker_build_push.sh
+++ b/scripts/docker_build_push.sh
@@ -125,7 +125,7 @@ fi
 
 TARGET_ARGUMENT=""
 if [[ -n "${BUILD_TARGET}" ]]; then
-  TARGET_ARGUMENT="--target ${TARGET}"
+  TARGET_ARGUMENT="--target ${BUILD_TARGET}"
 fi
 
 docker buildx build \

From c8b707d9b7bcf8a4c00112c1a94b1fdb58e78589 Mon Sep 17 00:00:00 2001
From: Maxime Beauchemin <maximebeauchemin@gmail.com>
Date: Mon, 22 Jan 2024 13:53:29 -0800
Subject: [PATCH 09/19] missed a thing

---
 scripts/docker_build_push.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/docker_build_push.sh b/scripts/docker_build_push.sh
index 44259d0759a5..4383c33b86d6 100755
--- a/scripts/docker_build_push.sh
+++ b/scripts/docker_build_push.sh
@@ -131,7 +131,7 @@ fi
 docker buildx build \
   ${TARGET_ARGUMENT} \
   ${DOCKER_ARGS} \
-  --cache-from=type=registry,ref=apache/superset:${BUILD_TARGET} \
+  --cache-from=type=registry,ref=apache/superset:${TARGET} \
   --cache-to=type=registry,mode=max,ref=apache/superset:${BUILD_TARGET} \
   ${DOCKER_TAGS} \
   --platform ${BUILD_PLATFORM} \

From a9894304b0255aed24d8781dc34efaa34527fc44 Mon Sep 17 00:00:00 2001
From: Maxime Beauchemin <maximebeauchemin@gmail.com>
Date: Mon, 22 Jan 2024 14:05:36 -0800
Subject: [PATCH 10/19] move DEV_TAG up

---
 scripts/docker_build_push.sh | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/scripts/docker_build_push.sh b/scripts/docker_build_push.sh
index 4383c33b86d6..b6f0546b9e8e 100755
--- a/scripts/docker_build_push.sh
+++ b/scripts/docker_build_push.sh
@@ -60,6 +60,14 @@ if [[ "${TEST_ENV}" == "true" ]]; then
   exit 0
 fi
 
+# for the dev image, it's ok to tag master as latest-dev
+# for production, we only want to tag the latest official release as latest
+if [ "${LATEST_TAG}" = "master" ]; then
+  DEV_TAG="${REPO_NAME}:latest-dev"
+else
+  DEV_TAG="${REPO_NAME}:${LATEST_TAG}-dev"
+fi
+
 case "${TARGET}" in
   "dev")
     DOCKER_TAGS="-t ${REPO_NAME}:${SHA}-dev -t ${REPO_NAME}:${REFSPEC}-dev -t ${DEV_TAG}"
@@ -115,14 +123,6 @@ else
 fi
 set -x
 
-# for the dev image, it's ok to tag master as latest-dev
-# for production, we only want to tag the latest official release as latest
-if [ "${LATEST_TAG}" = "master" ]; then
-  DEV_TAG="${REPO_NAME}:latest-dev"
-else
-  DEV_TAG="${REPO_NAME}:${LATEST_TAG}-dev"
-fi
-
 TARGET_ARGUMENT=""
 if [[ -n "${BUILD_TARGET}" ]]; then
   TARGET_ARGUMENT="--target ${BUILD_TARGET}"

From 6bf409b429fc3e669c0f3e63a6645eb8b188e591 Mon Sep 17 00:00:00 2001
From: Maxime Beauchemin <maximebeauchemin@gmail.com>
Date: Mon, 22 Jan 2024 14:08:25 -0800
Subject: [PATCH 11/19] missed a thing

---
 scripts/docker_build_push.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/docker_build_push.sh b/scripts/docker_build_push.sh
index b6f0546b9e8e..e77d2e4ae9d2 100755
--- a/scripts/docker_build_push.sh
+++ b/scripts/docker_build_push.sh
@@ -132,12 +132,12 @@ docker buildx build \
   ${TARGET_ARGUMENT} \
   ${DOCKER_ARGS} \
   --cache-from=type=registry,ref=apache/superset:${TARGET} \
-  --cache-to=type=registry,mode=max,ref=apache/superset:${BUILD_TARGET} \
+  --cache-to=type=registry,mode=max,ref=apache/superset:${TARGET} \
   ${DOCKER_TAGS} \
   --platform ${BUILD_PLATFORM} \
   --label "sha=${SHA}" \
   --label "built_at=$(date)" \
-  --label "target=${BUILD_TARGET}" \
+  --label "target=${TARGET}" \
   --label "build_actor=${GITHUB_ACTOR}" \
   ${BUILD_ARG:+--build-arg PY_VER="${BUILD_ARG}"} \
   ${DOCKER_CONTEXT}

From 686c13b6f4b17a7092d701699e69bb51440b9455 Mon Sep 17 00:00:00 2001
From: Maxime Beauchemin <maximebeauchemin@gmail.com>
Date: Mon, 22 Jan 2024 14:45:04 -0800
Subject: [PATCH 12/19] bumping actions versions

---
 .github/workflows/docker-release.yml | 6 +++---
 .github/workflows/docker.yml         | 6 +++---
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml
index 5b50dc702065..1b538e9a4b2a 100644
--- a/.github/workflows/docker-release.yml
+++ b/.github/workflows/docker-release.yml
@@ -29,17 +29,17 @@ jobs:
       fail-fast: false
     steps:
       - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           persist-credentials: false
           submodules: recursive
           ref: ${{ github.ref }}
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v3
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v3
 
       - name: Build Docker Image
         env:
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index 87a7a50e962d..efbd0af469ce 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -36,15 +36,15 @@ jobs:
       fail-fast: false
     steps:
       - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           persist-credentials: false
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v3
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v3
 
       - name: Build Docker Image
         shell: bash

From bee3014c376985d4a49df4ea870d2869815e423b Mon Sep 17 00:00:00 2001
From: Maxime Beauchemin <maximebeauchemin@gmail.com>
Date: Mon, 22 Jan 2024 14:53:43 -0800
Subject: [PATCH 13/19] use the right cache for ephemeral env

---
 .github/workflows/docker.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index efbd0af469ce..998806df4d78 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -62,7 +62,7 @@ jobs:
           echo ${{ github.event.pull_request.number }} > ./build/PR-NUM
           docker buildx build --target ci \
             --load \
-            --cache-from=type=local,src=/tmp/superset \
+            --cache-from=type=registry,ref=apache/superset:lean \
             -t ${{ github.sha }} \
             -t "pr-${{ github.event.pull_request.number }}" \
             --platform linux/amd64 \

From 0182457970bc41734d8d03fa4319b43253f2b10a Mon Sep 17 00:00:00 2001
From: Maxime Beauchemin <maximebeauchemin@gmail.com>
Date: Mon, 22 Jan 2024 15:12:39 -0800
Subject: [PATCH 14/19] trying v4

---
 .github/workflows/docker.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index 998806df4d78..a2f3f8b3390a 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -72,7 +72,7 @@ jobs:
 
       - name: Upload build artifacts
         if: github.event_name == 'pull_request'
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: build
           path: build/

From 442520a535224631b46ff1f6a55bbc5d93bdc8e2 Mon Sep 17 00:00:00 2001
From: Maxime Beauchemin <maximebeauchemin@gmail.com>
Date: Mon, 22 Jan 2024 15:19:36 -0800
Subject: [PATCH 15/19] bring ephemeral build to top level

---
 .github/workflows/docker.yml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index a2f3f8b3390a..d5c85b356c73 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -54,6 +54,12 @@ jobs:
         run: |
           ./scripts/docker_build_push.sh "" ${{ matrix.target }} ${{ matrix.platform }}
 
+  ephemeral-docker-build:
+    needs: config
+    if: needs.config.outputs.has-secrets
+    name: docker-build
+    runs-on: ubuntu-latest
+    steps:
       - name: Build ephemeral env image
         if: github.event_name == 'pull_request'
         run: |

From 5d1b4439393dec8ac1a7796e7d4cd575a879c410 Mon Sep 17 00:00:00 2001
From: Maxime Beauchemin <maximebeauchemin@gmail.com>
Date: Mon, 22 Jan 2024 15:21:37 -0800
Subject: [PATCH 16/19] bring ephemeral build to top level

---
 .github/workflows/docker.yml | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index d5c85b356c73..b0cb2e9e5b31 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -60,6 +60,17 @@ jobs:
     name: docker-build
     runs-on: ubuntu-latest
     steps:
+      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
       - name: Build ephemeral env image
         if: github.event_name == 'pull_request'
         run: |

From d40614e0ab9a27742add8716826020c1a171d34c Mon Sep 17 00:00:00 2001
From: Maxime Beauchemin <maximebeauchemin@gmail.com>
Date: Mon, 22 Jan 2024 15:54:12 -0800
Subject: [PATCH 17/19] moving COPY after pip install

---
 Dockerfile | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index fc3e66703783..ad58076d881c 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -85,14 +85,15 @@ RUN --mount=type=bind,target=./requirements/local.txt,src=./requirements/local.t
     --mount=type=cache,target=/root/.cache/pip \
     pip install -r requirements/local.txt
 
-COPY --chown=superset:superset --from=superset-node /app/superset/static/assets superset/static/assets
-## Lastly, let's install superset itself
-COPY --chown=superset:superset superset superset
 RUN --mount=type=cache,target=/root/.cache/pip \
     pip install -e . \
     && flask fab babel-compile --target superset/translations \
     && chown -R superset:superset superset/translations
 
+COPY --chown=superset:superset --from=superset-node /app/superset/static/assets superset/static/assets
+## Lastly, let's install superset itself
+COPY --chown=superset:superset superset superset
+
 COPY --chmod=755 ./docker/run-server.sh /usr/bin/
 USER superset
 

From 5dff290a5377371e127708fe6b3c988d1041dfa3 Mon Sep 17 00:00:00 2001
From: Maxime Beauchemin <maximebeauchemin@gmail.com>
Date: Mon, 22 Jan 2024 16:01:43 -0800
Subject: [PATCH 18/19] fix dockerfile

---
 Dockerfile | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index ad58076d881c..fc3e66703783 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -85,15 +85,14 @@ RUN --mount=type=bind,target=./requirements/local.txt,src=./requirements/local.t
     --mount=type=cache,target=/root/.cache/pip \
     pip install -r requirements/local.txt
 
+COPY --chown=superset:superset --from=superset-node /app/superset/static/assets superset/static/assets
+## Lastly, let's install superset itself
+COPY --chown=superset:superset superset superset
 RUN --mount=type=cache,target=/root/.cache/pip \
     pip install -e . \
     && flask fab babel-compile --target superset/translations \
     && chown -R superset:superset superset/translations
 
-COPY --chown=superset:superset --from=superset-node /app/superset/static/assets superset/static/assets
-## Lastly, let's install superset itself
-COPY --chown=superset:superset superset superset
-
 COPY --chmod=755 ./docker/run-server.sh /usr/bin/
 USER superset
 

From 49a6f9bcbea3b76576596872297e7c7b76f2c22d Mon Sep 17 00:00:00 2001
From: Maxime Beauchemin <maximebeauchemin@gmail.com>
Date: Tue, 23 Jan 2024 09:57:47 -0800
Subject: [PATCH 19/19] removing lean39 from matrix

---
 .github/workflows/docker-release.yml | 2 +-
 .github/workflows/docker.yml         | 2 +-
 scripts/docker_build_push.sh         | 8 +++-----
 3 files changed, 5 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml
index 1b538e9a4b2a..3e205fdeffaa 100644
--- a/.github/workflows/docker-release.yml
+++ b/.github/workflows/docker-release.yml
@@ -24,7 +24,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        target: ["dev", "lean", "lean310", "lean39", "websocket", "dockerize"]
+        target: ["dev", "lean", "lean310", "websocket", "dockerize"]
         platform: ["linux/amd64", "linux/arm64"]
       fail-fast: false
     steps:
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index b0cb2e9e5b31..b0d12be103c1 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -31,7 +31,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        target: ["dev", "lean", "lean310", "lean39", "websocket", "dockerize"]
+        target: ["dev", "lean", "lean310", "websocket", "dockerize"]
         platform: ["linux/amd64", "linux/arm64"]
       fail-fast: false
     steps:
diff --git a/scripts/docker_build_push.sh b/scripts/docker_build_push.sh
index e77d2e4ae9d2..10737447eaa6 100755
--- a/scripts/docker_build_push.sh
+++ b/scripts/docker_build_push.sh
@@ -68,6 +68,8 @@ else
   DEV_TAG="${REPO_NAME}:${LATEST_TAG}-dev"
 fi
 
+BUILD_ARG="3.9-slim-bookworm"
+
 case "${TARGET}" in
   "dev")
     DOCKER_TAGS="-t ${REPO_NAME}:${SHA}-dev -t ${REPO_NAME}:${REFSPEC}-dev -t ${DEV_TAG}"
@@ -82,11 +84,6 @@ case "${TARGET}" in
     BUILD_TARGET="lean"
     BUILD_ARG="3.10-slim-bookworm"
     ;;
-  "lean39")
-    DOCKER_TAGS="-t ${REPO_NAME}:${SHA}-py39 -t ${REPO_NAME}:${REFSPEC}-py39 -t ${REPO_NAME}:${LATEST_TAG}-py39"
-    BUILD_TARGET="lean"
-    BUILD_ARG="3.9-slim-bullseye"
-    ;;
   "websocket")
     DOCKER_TAGS="-t ${REPO_NAME}:${SHA}-websocket -t ${REPO_NAME}:${REFSPEC}-websocket -t ${REPO_NAME}:${LATEST_TAG}-websocket"
     BUILD_TARGET=""
@@ -138,6 +135,7 @@ docker buildx build \
   --label "sha=${SHA}" \
   --label "built_at=$(date)" \
   --label "target=${TARGET}" \
+  --label "base=${PY_VER}" \
   --label "build_actor=${GITHUB_ACTOR}" \
   ${BUILD_ARG:+--build-arg PY_VER="${BUILD_ARG}"} \
   ${DOCKER_CONTEXT}