diff --git a/docs/running-on-kubernetes.md b/docs/running-on-kubernetes.md
index e9c292d21fd47..3bd1c410e8433 100644
--- a/docs/running-on-kubernetes.md
+++ b/docs/running-on-kubernetes.md
@@ -1087,7 +1087,7 @@ See the [configuration page](configuration.html) for information on Spark config
spark.kubernetes.pyspark.pythonVersion |
"3" |
- This sets the major Python version of the docker image used to run the driver and executor containers. Can either be 2 or 3.
+ This sets the major Python version of the docker image used to run the driver and executor containers. Can be 3.
|
2.4.0 |
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala
index d6dc56f9d9d1b..00eaff452ba45 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala
@@ -285,11 +285,11 @@ private[spark] object Config extends Logging {
val PYSPARK_MAJOR_PYTHON_VERSION =
ConfigBuilder("spark.kubernetes.pyspark.pythonVersion")
- .doc("This sets the major Python version. Either 2 or 3. (Python2 or Python3)")
+ .doc("This sets the major Python version. Only 3 is available for Python3.")
.version("2.4.0")
.stringConf
- .checkValue(pv => List("2", "3").contains(pv),
- "Ensure that major Python version is either Python2 or Python3")
+ .checkValue(pv => List("3").contains(pv),
+ "Ensure that major Python version is Python3")
.createWithDefault("3")
val KUBERNETES_KERBEROS_KRB5_FILE =
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/DriverCommandFeatureStepSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/DriverCommandFeatureStepSuite.scala
index 829943f16beac..6a7366e9c6b7a 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/DriverCommandFeatureStepSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/DriverCommandFeatureStepSuite.scala
@@ -43,7 +43,6 @@ class DriverCommandFeatureStepSuite extends SparkFunSuite {
test("python resource") {
val mainResource = "local:/main.py"
val sparkConf = new SparkConf(false)
- .set(PYSPARK_MAJOR_PYTHON_VERSION, "2")
val spec = applyFeatureStep(
PythonMainAppResource(mainResource),
conf = sparkConf,
@@ -58,7 +57,7 @@ class DriverCommandFeatureStepSuite extends SparkFunSuite {
val envs = spec.pod.container.getEnv.asScala
.map { env => (env.getName, env.getValue) }
.toMap
- val expected = Map(ENV_PYSPARK_MAJOR_PYTHON_VERSION -> "2")
+ val expected = Map(ENV_PYSPARK_MAJOR_PYTHON_VERSION -> "3")
assert(envs === expected)
}
@@ -93,7 +92,6 @@ class DriverCommandFeatureStepSuite extends SparkFunSuite {
test("SPARK-25355: python resource args with proxy-user") {
val mainResource = "local:/main.py"
val sparkConf = new SparkConf(false)
- .set(PYSPARK_MAJOR_PYTHON_VERSION, "2")
val spec = applyFeatureStep(
PythonMainAppResource(mainResource),
conf = sparkConf,
diff --git a/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/entrypoint.sh b/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/entrypoint.sh
index 813a70c6e7ec3..d605ae43c024f 100755
--- a/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/entrypoint.sh
+++ b/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/entrypoint.sh
@@ -44,12 +44,7 @@ if [ -n "$SPARK_EXTRA_CLASSPATH" ]; then
SPARK_CLASSPATH="$SPARK_CLASSPATH:$SPARK_EXTRA_CLASSPATH"
fi
-if [ "$PYSPARK_MAJOR_PYTHON_VERSION" == "2" ]; then
- pyv="$(python -V 2>&1)"
- export PYTHON_VERSION="${pyv:7}"
- export PYSPARK_PYTHON="python"
- export PYSPARK_DRIVER_PYTHON="python"
-elif [ "$PYSPARK_MAJOR_PYTHON_VERSION" == "3" ]; then
+if [ "$PYSPARK_MAJOR_PYTHON_VERSION" == "3" ]; then
pyv3="$(python3 -V 2>&1)"
export PYTHON_VERSION="${pyv3:7}"
export PYSPARK_PYTHON="python3"
diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/DecommissionSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/DecommissionSuite.scala
index 6e42819b1779e..fd14b12b112d3 100644
--- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/DecommissionSuite.scala
+++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/DecommissionSuite.scala
@@ -26,7 +26,6 @@ private[spark] trait DecommissionSuite { k8sSuite: KubernetesSuite =>
test("Test basic decommissioning", k8sTestTag) {
sparkAppConf
.set(config.DECOMMISSION_ENABLED.key, "true")
- .set("spark.kubernetes.pyspark.pythonVersion", "3")
.set("spark.kubernetes.container.image", pyImage)
.set(config.STORAGE_DECOMMISSION_ENABLED.key, "true")
.set(config.STORAGE_DECOMMISSION_SHUFFLE_BLOCKS_ENABLED.key, "true")
diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PythonTestsSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PythonTestsSuite.scala
index b16ccb429074f..bad6f1c1021ba 100644
--- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PythonTestsSuite.scala
+++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PythonTestsSuite.scala
@@ -35,10 +35,9 @@ private[spark] trait PythonTestsSuite { k8sSuite: KubernetesSuite =>
isJVM = false)
}
- test("Run PySpark with Python3 to test a pyfiles example", k8sTestTag) {
+ test("Run PySpark to test a pyfiles example", k8sTestTag) {
sparkAppConf
.set("spark.kubernetes.container.image", pyImage)
- .set("spark.kubernetes.pyspark.pythonVersion", "3")
runSparkApplicationAndVerifyCompletion(
appResource = PYSPARK_FILES,
mainClass = "",
@@ -57,7 +56,6 @@ private[spark] trait PythonTestsSuite { k8sSuite: KubernetesSuite =>
test("Run PySpark with memory customization", k8sTestTag) {
sparkAppConf
.set("spark.kubernetes.container.image", pyImage)
- .set("spark.kubernetes.pyspark.pythonVersion", "3")
.set("spark.kubernetes.memoryOverheadFactor", s"$memOverheadConstant")
.set("spark.executor.pyspark.memory", s"${additionalMemory}m")
runSparkApplicationAndVerifyCompletion(
diff --git a/resource-managers/kubernetes/integration-tests/tests/pyfiles.py b/resource-managers/kubernetes/integration-tests/tests/pyfiles.py
index 51c0160554866..73c53be482c03 100644
--- a/resource-managers/kubernetes/integration-tests/tests/pyfiles.py
+++ b/resource-managers/kubernetes/integration-tests/tests/pyfiles.py
@@ -31,7 +31,7 @@
from py_container_checks import version_check
# Begin of Python container checks
- version_check(sys.argv[1], 2 if sys.argv[1] == "python" else 3)
+ version_check(sys.argv[1], 3)
# Check python executable at executors
spark.udf.register("get_sys_ver",