From 27174e59fe85cb3add536cb153d64cb75ea96e42 Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Sun, 12 Mar 2023 23:21:42 -0700 Subject: [PATCH 1/2] [SPARK-42769][K8S] Add ENV_DRIVER_POD_IP env variable to executor pods --- .../scala/org/apache/spark/deploy/k8s/Constants.scala | 1 + .../docker/src/main/dockerfiles/spark/entrypoint.sh | 1 + .../deploy/k8s/integrationtest/BasicTestsSuite.scala | 9 +++++++++ 3 files changed, 11 insertions(+) diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Constants.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Constants.scala index 9da11047b314b..385734c557a38 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Constants.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Constants.scala @@ -54,6 +54,7 @@ private[spark] object Constants { val UI_PORT_NAME = "spark-ui" // Environment Variables + val ENV_DRIVER_POD_IP = "SPARK_DRIVER_POD_IP" val ENV_DRIVER_URL = "SPARK_DRIVER_URL" val ENV_EXECUTOR_CORES = "SPARK_EXECUTOR_CORES" val ENV_EXECUTOR_MEMORY = "SPARK_EXECUTOR_MEMORY" diff --git a/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/entrypoint.sh b/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/entrypoint.sh index cae004715391b..42f4df88f3da9 100755 --- a/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/entrypoint.sh +++ b/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/entrypoint.sh @@ -81,6 +81,7 @@ case "$1" in CMD=( "$SPARK_HOME/bin/spark-submit" --conf "spark.driver.bindAddress=$SPARK_DRIVER_BIND_ADDRESS" + --conf "spark.executorEnv.SPARK_DRIVER_POD_IP=$SPARK_DRIVER_BIND_ADDRESS" --deploy-mode client "$@" ) diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/BasicTestsSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/BasicTestsSuite.scala index ed38f8d70d7a0..a93f10606a091 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/BasicTestsSuite.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/BasicTestsSuite.scala @@ -159,6 +159,15 @@ private[spark] trait BasicTestsSuite { k8sSuite: KubernetesSuite => runSparkRemoteCheckAndVerifyCompletion(appArgs = Array(REMOTE_PAGE_RANK_FILE_NAME)) } } + + test("SPARK-42769: All executor pods have ENV_DRIVER_POD_IP env variable", k8sTestTag) { + runSparkPiAndVerifyCompletion( + executorPodChecker = (executorPod: Pod) => { + doBasicExecutorPodCheck(executorPod) + executorPod.getSpec.getContainers.get(0).getEnv.asScala + .exists(envVar => envVar.getName == "SPARK_DRIVER_POD_IP") + }) + } } private[spark] object BasicTestsSuite extends SparkFunSuite { From c153ba24c9ec894ee016ebad0743f722588454f5 Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Mon, 13 Mar 2023 00:32:30 -0700 Subject: [PATCH 2/2] Revise test case name --- .../spark/deploy/k8s/integrationtest/BasicTestsSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/BasicTestsSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/BasicTestsSuite.scala index a93f10606a091..af74032694db5 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/BasicTestsSuite.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/BasicTestsSuite.scala @@ -160,7 +160,7 @@ private[spark] trait BasicTestsSuite { k8sSuite: KubernetesSuite => } } - test("SPARK-42769: All executor pods have ENV_DRIVER_POD_IP env variable", k8sTestTag) { + test("SPARK-42769: All executor pods have SPARK_DRIVER_POD_IP env variable", k8sTestTag) { runSparkPiAndVerifyCompletion( executorPodChecker = (executorPod: Pod) => { doBasicExecutorPodCheck(executorPod)