Test infra change

.github/workflows/build_and_test.yml

@@ -56,10 +56,14 @@ jobs:
       GITHUB_PREV_SHA: ${{ github.event.before }}
     outputs:
       required: ${{ steps.set-outputs.outputs.required }}
+      # For branch3.2, branch3.3, still use previous image tag.
+      # If infra image has changes, use dynamic infra image URL.
+      # If infra image hasn't changed, use pre-build image directly.
       image_url: >-
         ${{
-          (inputs.branch == 'master' && steps.infra-image-outputs.outputs.image_url)
-          || 'dongjoon/apache-spark-github-action-image:20220207'
+          (inputs.branch == 'branch-3.2' || inputs.branch == 'branch-3.3') && 'dongjoon/apache-spark-github-action-image:20220207'
+          || (fromJson(steps.set-outputs.outputs.required).infra-image == 'true' && steps.infra-image-outputs.outputs.image_url)
+          || format('ghcr.io/apache/spark/apache-spark-github-action-image-cache:{0}-static', inputs.branch)
         }}
     steps:
     - name: Checkout Spark repository
@@ -87,6 +91,7 @@ jobs:
             sparkr=`./dev/is-changed.py -m sparkr`
             tpcds=`./dev/is-changed.py -m sql`
             docker=`./dev/is-changed.py -m docker-integration-tests`
+            infra_image=`./dev/is-changed.py -m infra-image`
           fi
           # 'build', 'scala-213', and 'java-11-17' are always true for now.
           # It does not save significant time and most of PRs trigger the build.
@@ -97,6 +102,7 @@ jobs:
               \"sparkr\": \"$sparkr\",
               \"tpcds-1g\": \"$tpcds\",
               \"docker-integration-tests\": \"$docker\",
+              \"infra-image\": \"$infra_image\",
               \"scala-213\": \"true\",
               \"java-11-17\": \"true\",
               \"lint\" : \"true\",
@@ -116,6 +122,7 @@ jobs:
         fi
     - name: Generate infra image URL
       id: infra-image-outputs
+      if: fromJson(steps.set-outputs.outputs.required).infra-image == 'true'
       run: |
         # Convert to lowercase to meet Docker repo name requirement
         REPO_OWNER=$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]')
@@ -270,6 +277,7 @@ jobs:
     needs: precondition
     # Currently, only enable docker build from cache for `master` branch jobs
     if: >-
+      fromJson(needs.precondition.outputs.required).infra-image == 'true' &&
       (fromJson(needs.precondition.outputs.required).pyspark == 'true' ||
       fromJson(needs.precondition.outputs.required).lint == 'true' ||
       fromJson(needs.precondition.outputs.required).sparkr == 'true') &&

dev/infra/Dockerfile

@@ -16,7 +16,7 @@
 #
 
 # Image for building and testing Spark branches. Based on Ubuntu 20.04.
-# See also in https://hub.docker.com/_/ubuntu
+# See also in https://hub.docker.com/_/ubuntu test
 FROM ubuntu:focal-20220801
 
 ENV FULL_REFRESH_DATE 20220706

dev/is-changed.py

@@ -71,7 +71,10 @@ def main():
         print("false")
         if opts.fail:
             sys.exit(1)
-    elif "root" in test_modules or modules.root in changed_modules:
+    # `./dev/is-changed.py -m infra-image` == True only when changing the infra dockerfile
+    elif ("root" in test_modules or modules.root in changed_modules) and (
+        ["infra-image"] != test_modules
+    ):
         print("true")
     elif len(set(test_modules).intersection(module_names)) == 0:
         print("false")

dev/sparktestsupport/modules.py

@@ -781,6 +781,14 @@ def __hash__(self):
     test_tags=["org.apache.spark.tags.DockerTest"],
 )
 
+infra_image = Module(
+    name="infra-image",
+    dependencies=[],
+    source_file_regexes=[
+        "dev/infra/",
+    ],
+)
+
 # The root module is a dummy module which is used to run all of the tests.
 # No other modules should directly depend on this module.
 root = Module(