diff --git a/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/Dockerfile b/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/Dockerfile
index a1fc63789bc61..6ed37fc637b31 100644
--- a/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/Dockerfile
+++ b/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/Dockerfile
@@ -14,8 +14,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
+ARG java_image_tag=8-jre-slim
-FROM openjdk:8-jre-slim
+FROM openjdk:${java_image_tag}
ARG spark_uid=185
diff --git a/resource-managers/kubernetes/integration-tests/README.md b/resource-managers/kubernetes/integration-tests/README.md
index d7ad35a175a61..18b91916208d6 100644
--- a/resource-managers/kubernetes/integration-tests/README.md
+++ b/resource-managers/kubernetes/integration-tests/README.md
@@ -6,13 +6,17 @@ title: Spark on Kubernetes Integration Tests
# Running the Kubernetes Integration Tests
Note that the integration test framework is currently being heavily revised and
-is subject to change. Note that currently the integration tests only run with Java 8.
+is subject to change.
The simplest way to run the integration tests is to install and run Minikube, then run the following from this
directory:
./dev/dev-run-integration-tests.sh
+To run tests with Java 11 instead of Java 8, use `--java-image-tag` to specify the base image.
+
+ ./dev/dev-run-integration-tests.sh --java-image-tag 11-jre-slim
+
The minimum tested version of Minikube is 0.23.0. The kube-dns addon must be enabled. Minikube should
run with a minimum of 4 CPUs and 6G of memory:
@@ -183,7 +187,14 @@ to the wrapper scripts and using the wrapper scripts will simply set these appro
A specific image tag to use, when set assumes images with those tags are already built and available in the
specified image repository. When set to N/A (the default) fresh images will be built.
-
N/A
+ | N/A |
+
+
+ spark.kubernetes.test.javaImageTag |
+
+ A specific OpenJDK base image tag to use, when set uses it instead of 8-jre-slim.
+ |
+ 8-jre-slim |
spark.kubernetes.test.imageTagFile |
diff --git a/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh b/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh
index 1f0a8035cea7b..76d6e1c1e8499 100755
--- a/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh
+++ b/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh
@@ -23,6 +23,7 @@ DEPLOY_MODE="minikube"
IMAGE_REPO="docker.io/kubespark"
SPARK_TGZ="N/A"
IMAGE_TAG="N/A"
+JAVA_IMAGE_TAG=
BASE_IMAGE_NAME=
JVM_IMAGE_NAME=
PYTHON_IMAGE_NAME=
@@ -52,6 +53,10 @@ while (( "$#" )); do
IMAGE_TAG="$2"
shift
;;
+ --java-image-tag)
+ JAVA_IMAGE_TAG="$2"
+ shift
+ ;;
--deploy-mode)
DEPLOY_MODE="$2"
shift
@@ -120,6 +125,11 @@ properties=(
-Dtest.include.tags=$INCLUDE_TAGS
)
+if [ -n "$JAVA_IMAGE_TAG" ];
+then
+ properties=( ${properties[@]} -Dspark.kubernetes.test.javaImageTag=$JAVA_IMAGE_TAG )
+fi
+
if [ -n $NAMESPACE ];
then
properties=( ${properties[@]} -Dspark.kubernetes.test.namespace=$NAMESPACE )
diff --git a/resource-managers/kubernetes/integration-tests/pom.xml b/resource-managers/kubernetes/integration-tests/pom.xml
index 8e1043f77db6d..369dfd491826c 100644
--- a/resource-managers/kubernetes/integration-tests/pom.xml
+++ b/resource-managers/kubernetes/integration-tests/pom.xml
@@ -39,6 +39,7 @@
${project.build.directory}/spark-dist-unpacked
N/A
+ 8-jre-slim
${project.build.directory}/imageTag.txt
minikube
docker.io/kubespark
@@ -109,6 +110,9 @@
--image-tag
${spark.kubernetes.test.imageTag}
+ --java-image-tag
+ ${spark.kubernetes.test.javaImageTag}
+
--image-tag-output-file
${spark.kubernetes.test.imageTagFile}
diff --git a/resource-managers/kubernetes/integration-tests/scripts/setup-integration-test-env.sh b/resource-managers/kubernetes/integration-tests/scripts/setup-integration-test-env.sh
index 9e04b963fc40e..ab906604fce06 100755
--- a/resource-managers/kubernetes/integration-tests/scripts/setup-integration-test-env.sh
+++ b/resource-managers/kubernetes/integration-tests/scripts/setup-integration-test-env.sh
@@ -23,6 +23,7 @@ IMAGE_TAG_OUTPUT_FILE="$TEST_ROOT_DIR/target/image-tag.txt"
DEPLOY_MODE="minikube"
IMAGE_REPO="docker.io/kubespark"
IMAGE_TAG="N/A"
+JAVA_IMAGE_TAG="8-jre-slim"
SPARK_TGZ="N/A"
# Parse arguments
@@ -40,6 +41,10 @@ while (( "$#" )); do
IMAGE_TAG="$2"
shift
;;
+ --java-image-tag)
+ JAVA_IMAGE_TAG="$2"
+ shift
+ ;;
--image-tag-output-file)
IMAGE_TAG_OUTPUT_FILE="$2"
shift
@@ -82,6 +87,9 @@ then
IMAGE_TAG=$(uuidgen);
cd $SPARK_INPUT_DIR
+ # OpenJDK base-image tag (e.g. 8-jre-slim, 11-jre-slim)
+ JAVA_IMAGE_TAG_BUILD_ARG="-b java_image_tag=$JAVA_IMAGE_TAG"
+
# Build PySpark image
LANGUAGE_BINDING_BUILD_ARGS="-p $DOCKER_FILE_BASE_PATH/bindings/python/Dockerfile"
@@ -95,7 +103,7 @@ then
case $DEPLOY_MODE in
cloud)
# Build images
- $SPARK_INPUT_DIR/bin/docker-image-tool.sh -r $IMAGE_REPO -t $IMAGE_TAG $LANGUAGE_BINDING_BUILD_ARGS build
+ $SPARK_INPUT_DIR/bin/docker-image-tool.sh -r $IMAGE_REPO -t $IMAGE_TAG $JAVA_IMAGE_TAG_BUILD_ARG $LANGUAGE_BINDING_BUILD_ARGS build
# Push images appropriately
if [[ $IMAGE_REPO == gcr.io* ]] ;
@@ -109,13 +117,13 @@ then
docker-for-desktop)
# Only need to build as this will place it in our local Docker repo which is all
# we need for Docker for Desktop to work so no need to also push
- $SPARK_INPUT_DIR/bin/docker-image-tool.sh -r $IMAGE_REPO -t $IMAGE_TAG $LANGUAGE_BINDING_BUILD_ARGS build
+ $SPARK_INPUT_DIR/bin/docker-image-tool.sh -r $IMAGE_REPO -t $IMAGE_TAG $JAVA_IMAGE_TAG_BUILD_ARG $LANGUAGE_BINDING_BUILD_ARGS build
;;
minikube)
# Only need to build and if we do this with the -m option for minikube we will
# build the images directly using the minikube Docker daemon so no need to push
- $SPARK_INPUT_DIR/bin/docker-image-tool.sh -m -r $IMAGE_REPO -t $IMAGE_TAG $LANGUAGE_BINDING_BUILD_ARGS build
+ $SPARK_INPUT_DIR/bin/docker-image-tool.sh -m -r $IMAGE_REPO -t $IMAGE_TAG $JAVA_IMAGE_TAG_BUILD_ARG $LANGUAGE_BINDING_BUILD_ARGS build
;;
*)
echo "Unrecognized deploy mode $DEPLOY_MODE" && exit 1