Yikun · Yikun · Nov 25, 2022 · Nov 26, 2022 · Nov 28, 2022
diff --git a/.github/workflows/build_3.3.0.yaml b/.github/workflows/build_3.3.0.yaml
@@ -30,10 +30,14 @@ on:
 
 jobs:
   run-build:
+    strategy:
+      matrix:
+        image-type: ["all", "python", "scala", "r"]
     name: Run
     secrets: inherit
     uses: ./.github/workflows/main.yml
     with:
       spark: 3.3.0
       scala: 2.12
       java: 11
+      image-type: ${{ matrix.image-type }}
diff --git a/.github/workflows/build_3.3.1.yaml b/.github/workflows/build_3.3.1.yaml
@@ -30,10 +30,14 @@ on:
 
 jobs:
   run-build:
+    strategy:
+      matrix:
+        image-type: ["all", "python", "scala", "r"]
     name: Run
     secrets: inherit
     uses: ./.github/workflows/main.yml
     with:
       spark: 3.3.1
       scala: 2.12
       java: 11
+      image-type: ${{ matrix.image-type }}
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -37,16 +37,31 @@ on:
         required: true
         type: string
         default: 11
+      build:
+        description: Build the image or not.
+        required: false
+        type: boolean
+        default: true
       publish:
         description: Publish the image or not.
         required: false
         type: boolean
         default: false
       repository:
-        description: The registry to be published (Avaliable only when publish is selected).
+        description: The registry to be published (Avaliable only in publish/test workflow).
         required: false
         type: string
         default: ghcr.io/apache/spark-docker
+      image-type:
+        description: The image type of the image (all, python, scala, r).
+        required: false
+        type: string
+        default: python
+      image-tag:
+        type: string
+        description: The image tag to be validated. (Avaliable only in test workflow)
+        required: false
+        default: latest
 
 jobs:
   main:
@@ -60,41 +75,40 @@ jobs:
         image: registry:2
         ports:
           - 5000:5000
-    strategy:
-      matrix:
-        spark_version:
-          - ${{ inputs.spark }}
-        scala_version:
-          - ${{ inputs.scala }}
-        java_version:
-          - ${{ inputs.java }}
-        image_suffix: [python3-ubuntu, ubuntu, r-ubuntu, python3-r-ubuntu]
     steps:
       - name: Checkout Spark Docker repository
         uses: actions/checkout@v3
 
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-        with:
-          # This required by local registry
-          driver-opts: network=host
-
-      - name: Generate tags
+      - name: Prepare - Generate tags
         run: |
-          TAG=scala${{ matrix.scala_version }}-java${{ matrix.java_version }}-${{ matrix.image_suffix }}
+          case "${{ inputs.image-type }}" in
+              all) SUFFIX=python3-r-ubuntu
+                  ;;
+              python) SUFFIX=python3-ubuntu
+                  ;;
+              r) SUFFIX=r-ubuntu
+                  ;;
+              scala) SUFFIX=ubuntu
+                  ;;
+          esac
+          TAG=scala${{ inputs.scala }}-java${{ inputs.java }}-$SUFFIX
 
-          REPO_OWNER=$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]')
-          TEST_REPO=localhost:5000/$REPO_OWNER/spark-docker
           IMAGE_NAME=spark
-          IMAGE_PATH=${{ matrix.spark_version }}/$TAG
-          UNIQUE_IMAGE_TAG=${{ matrix.spark_version }}-$TAG
+          IMAGE_PATH=${{ inputs.spark }}/$TAG
+          if [ "${{ inputs.build }}" == "true" ]; then
+            # Use the local registry to build and test
+            REPO_OWNER=$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]')
+            TEST_REPO=localhost:5000/$REPO_OWNER/spark-docker
+            UNIQUE_IMAGE_TAG=${{ inputs.spark }}-$TAG
+          else
+            # Use specified {repository}/spark:{image-tag} image to test
+            TEST_REPO=${{ inputs.repository }}
+            UNIQUE_IMAGE_TAG=${{ inputs.image-tag }}
+          fi
           IMAGE_URL=$TEST_REPO/$IMAGE_NAME:$UNIQUE_IMAGE_TAG
 
           PUBLISH_REPO=${{ inputs.repository }}
-          PUBLISH_IMAGE_URL=`tools/manifest.py tags -i ${PUBLISH_REPO}/${IMAGE_NAME} -p ${{ matrix.spark_version }}/${TAG}`
+          PUBLISH_IMAGE_URL=`tools/manifest.py tags -i ${PUBLISH_REPO}/${IMAGE_NAME} -p ${{ inputs.spark }}/${TAG}`
 
           # Unique image tag in each version: 3.3.0-scala2.12-java11-python3-ubuntu
           echo "UNIQUE_IMAGE_TAG=${UNIQUE_IMAGE_TAG}" >> $GITHUB_ENV
@@ -110,7 +124,7 @@ jobs:
           echo "PUBLISH_REPO=${PUBLISH_REPO}" >> $GITHUB_ENV
           echo "PUBLISH_IMAGE_URL=${PUBLISH_IMAGE_URL}" >> $GITHUB_ENV
 
-      - name: Print Image tags
+      - name: Prepare - Print Image tags
         run: |
           echo "UNIQUE_IMAGE_TAG: "${UNIQUE_IMAGE_TAG}
           echo "TEST_REPO: "${TEST_REPO}
@@ -121,7 +135,19 @@ jobs:
           echo "PUBLISH_REPO:"${PUBLISH_REPO}
           echo "PUBLISH_IMAGE_URL:"${PUBLISH_IMAGE_URL}
 
-      - name: Build and push test image
+      - name: Build - Set up QEMU
+        if: ${{ inputs.build }}
+        uses: docker/setup-qemu-action@v2
+
+      - name: Build - Set up Docker Buildx
+        if: ${{ inputs.build }}
+        uses: docker/setup-buildx-action@v2
+        with:
+          # This required by local registry
+          driver-opts: network=host
+
+      - name: Build - Build and push test image
+        if: ${{ inputs.build }}
         uses: docker/build-push-action@v3
         with:
           context: ${{ env.IMAGE_PATH }}
@@ -130,20 +156,20 @@ jobs:
           push: true
 
       - name : Test - Run spark application for standalone cluster on docker
-        run: testing/run_tests.sh --image-url $IMAGE_URL --scala-version ${{ matrix.scala_version }} --spark-version ${{ matrix.spark_version }}
+        run: testing/run_tests.sh --image-url $IMAGE_URL --scala-version ${{ inputs.scala }} --spark-version ${{ inputs.spark }}
 
       - name: Test - Checkout Spark repository
         uses: actions/checkout@v3
         with:
           fetch-depth: 0
           repository: apache/spark
-          ref: v${{ matrix.spark_version }}
+          ref: v${{ inputs.spark }}
           path: ${{ github.workspace }}/spark
 
       - name: Test - Cherry pick commits
         # Apache Spark enable resource limited k8s IT since v3.3.1, cherry-pick patches for old release
         # https://github.com/apache/spark/pull/36087#issuecomment-1251756266
-        if: matrix.spark_version == '3.3.0'
+        if: inputs.spark == '3.3.0'
         working-directory: ${{ github.workspace }}/spark
         run: |
           # SPARK-38802: Add driverRequestCores/executorRequestCores supported
@@ -159,7 +185,7 @@ jobs:
           # This is required after v2, now just keep same distribution with v1
           # https://github.com/actions/setup-java/releases/tag/v2.0.0
           distribution: 'zulu'
-          java-version: ${{ matrix.java_version }}
+          java-version: ${{ inputs.java }}
 
       - name: Test - Cache Scala, SBT and Maven
         uses: actions/cache@v3
@@ -169,13 +195,13 @@ jobs:
             build/scala-*
             build/*.jar
             ~/.sbt
-          key: build-${{ matrix.spark_version }}-scala${{ matrix.scala_version }}-java${{ matrix.java_version }}
+          key: build-${{ inputs.spark }}-scala${{ inputs.scala }}-java${{ inputs.java }}
 
       - name: Test - Cache Coursier local repository
         uses: actions/cache@v3
         with:
           path: ~/.cache/coursier
-          key: build-${{ matrix.spark_version }}-scala${{ matrix.scala_version }}-java${{ matrix.java_version }}-coursier
+          key: build-${{ inputs.spark }}-scala${{ inputs.scala }}-java${{ inputs.java }}-coursier
 
       - name: Test - Start minikube
         run: |
@@ -205,11 +231,11 @@ jobs:
           OPTS+="-Dspark.kubernetes.test.pythonImage=${IMAGE_NAME} "
           OPTS+="-Dspark.kubernetes.test.rImage=${IMAGE_NAME} "
 
-          if echo ${{ matrix.image_suffix }} | grep -q "python3-r-ubuntu"; then
+          if [ "${{ inputs.image-type }}" = "all" ]; then
             # Prepare test jar for client tests
             CONTAINER_TMP_NAME=spark-example-image
             docker create -ti --name $CONTAINER_TMP_NAME ${{ env.IMAGE_URL }} bash
-            docker cp $CONTAINER_TMP_NAME:/opt/spark/examples/jars/spark-examples_${{ matrix.scala_version }}-${{ matrix.spark_version }}.jar .
+            docker cp $CONTAINER_TMP_NAME:/opt/spark/examples/jars/spark-examples_${{ inputs.scala }}-${{ inputs.spark }}.jar .
             docker rm -f $CONTAINER_TMP_NAME
             # Prepare PV test
             PVC_TMP_DIR=$(mktemp -d)
@@ -223,12 +249,12 @@ jobs:
             build/sbt $OPTS 'kubernetes-integration-tests/testOnly -- -z "Run SparkPi"'
 
             # Run basic test for PySpark image
-            if echo ${{ matrix.image_suffix }} | grep -q "python"; then
+            if [ "${{ inputs.image-type }}" = "python" ]; then
               build/sbt $OPTS 'kubernetes-integration-tests/testOnly -- -z "Run PySpark"'
             fi
 
             # Run basic test for SparkR image
-            if echo ${{ matrix.image_suffix }} | grep -q "r-"; then
+            if [ "${{ inputs.image-type }}" = "r" ]; then
               OPTS+="-Psparkr -Dtest.include.tags=r "
               build/sbt $OPTS 'kubernetes-integration-tests/testOnly'
             fi

diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
@@ -38,9 +38,10 @@ on:
       repository:
         description: The registry to be published (Avaliable only when publish is true).
         required: false
-        default: ghcr.io/apache/spark-docker
+        default: ghcr.io/yikun/spark-docker
         type: choice
         options:
+        - ghcr.io/yikun/spark-docker
         # GHCR: This required the write permission of apache/spark-docker (Spark Committer)
         - ghcr.io/apache/spark-docker
         # Dockerhub: This required the DOCKERHUB_TOKEN and DOCKERHUB_USER (Spark Committer)
@@ -53,6 +54,7 @@ jobs:
       matrix:
         scala: [2.12]
         java: [11]
+        image-type: ["all", "python", "scala", "r"]
     permissions:
       packages: write
     name: Run
@@ -64,3 +66,4 @@ jobs:
       java: ${{ matrix.java }}
       publish: ${{ inputs.publish }}
       repository: ${{ inputs.repository }}
+      image-type: ${{ matrix.image-type }}
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -0,0 +1,76 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+name: "Test"
+
+on:
+  workflow_dispatch:
+    inputs:
+      spark:
+        description: 'The Spark version of Spark image.'
+        required: true
+        default: '3.3.1'
+        type: choice
+        options:
+        - 3.3.0
+        - 3.3.1
+      java:
+        description: 'The Java version of Spark image.'
+        default: 11
+        type: string
+        required: true
+      scala:
+        description: 'The Scala version of Spark image.'
+        default: 2.12
+        type: string
+        required: true
+      repository:
+        description: The registry to be validated.
+        required: false
+        type: string
+        default: ghcr.io/apache/spark-docker
+      image-type:
+        description: 'The image type of Spark image.'
+        required: true
+        default: 'python'
+        type: choice
+        options:
+        - all
+        - python
+        - scala
+        - r
+      image-tag:
+        description: 'The image tag of Spark image.'
+        default: latest
+        type: string
+        required: true
+
+jobs:
+  run-build:
+    name: Test ${{ inputs.repository }}/spark:${{ inputs.image-tag }}
+    secrets: inherit
+    uses: ./.github/workflows/main.yml
+    with:
+      spark: ${{ inputs.spark }}
+      scala: ${{ inputs.scala }}
+      java: ${{ inputs.java }}
+      repository: ${{ inputs.repository }}
+      image-tag: ${{ inputs.image-tag }}
+      image-type: ${{ inputs.image-type }}
+      build: false