diff --git a/docs/running-on-kubernetes.md b/docs/running-on-kubernetes.md index e9c292d21fd47..3bd1c410e8433 100644 --- a/docs/running-on-kubernetes.md +++ b/docs/running-on-kubernetes.md @@ -1087,7 +1087,7 @@ See the [configuration page](configuration.html) for information on Spark config spark.kubernetes.pyspark.pythonVersion "3" - This sets the major Python version of the docker image used to run the driver and executor containers. Can either be 2 or 3. + This sets the major Python version of the docker image used to run the driver and executor containers. Can be 3. 2.4.0 diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala index d6dc56f9d9d1b..00eaff452ba45 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala @@ -285,11 +285,11 @@ private[spark] object Config extends Logging { val PYSPARK_MAJOR_PYTHON_VERSION = ConfigBuilder("spark.kubernetes.pyspark.pythonVersion") - .doc("This sets the major Python version. Either 2 or 3. (Python2 or Python3)") + .doc("This sets the major Python version. Only 3 is available for Python3.") .version("2.4.0") .stringConf - .checkValue(pv => List("2", "3").contains(pv), - "Ensure that major Python version is either Python2 or Python3") + .checkValue(pv => List("3").contains(pv), + "Ensure that major Python version is Python3") .createWithDefault("3") val KUBERNETES_KERBEROS_KRB5_FILE = diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/DriverCommandFeatureStepSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/DriverCommandFeatureStepSuite.scala index 829943f16beac..6a7366e9c6b7a 100644 --- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/DriverCommandFeatureStepSuite.scala +++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/DriverCommandFeatureStepSuite.scala @@ -43,7 +43,6 @@ class DriverCommandFeatureStepSuite extends SparkFunSuite { test("python resource") { val mainResource = "local:/main.py" val sparkConf = new SparkConf(false) - .set(PYSPARK_MAJOR_PYTHON_VERSION, "2") val spec = applyFeatureStep( PythonMainAppResource(mainResource), conf = sparkConf, @@ -58,7 +57,7 @@ class DriverCommandFeatureStepSuite extends SparkFunSuite { val envs = spec.pod.container.getEnv.asScala .map { env => (env.getName, env.getValue) } .toMap - val expected = Map(ENV_PYSPARK_MAJOR_PYTHON_VERSION -> "2") + val expected = Map(ENV_PYSPARK_MAJOR_PYTHON_VERSION -> "3") assert(envs === expected) } @@ -93,7 +92,6 @@ class DriverCommandFeatureStepSuite extends SparkFunSuite { test("SPARK-25355: python resource args with proxy-user") { val mainResource = "local:/main.py" val sparkConf = new SparkConf(false) - .set(PYSPARK_MAJOR_PYTHON_VERSION, "2") val spec = applyFeatureStep( PythonMainAppResource(mainResource), conf = sparkConf, diff --git a/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/entrypoint.sh b/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/entrypoint.sh index 813a70c6e7ec3..d605ae43c024f 100755 --- a/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/entrypoint.sh +++ b/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/entrypoint.sh @@ -44,12 +44,7 @@ if [ -n "$SPARK_EXTRA_CLASSPATH" ]; then SPARK_CLASSPATH="$SPARK_CLASSPATH:$SPARK_EXTRA_CLASSPATH" fi -if [ "$PYSPARK_MAJOR_PYTHON_VERSION" == "2" ]; then - pyv="$(python -V 2>&1)" - export PYTHON_VERSION="${pyv:7}" - export PYSPARK_PYTHON="python" - export PYSPARK_DRIVER_PYTHON="python" -elif [ "$PYSPARK_MAJOR_PYTHON_VERSION" == "3" ]; then +if [ "$PYSPARK_MAJOR_PYTHON_VERSION" == "3" ]; then pyv3="$(python3 -V 2>&1)" export PYTHON_VERSION="${pyv3:7}" export PYSPARK_PYTHON="python3" diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/DecommissionSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/DecommissionSuite.scala index 6e42819b1779e..fd14b12b112d3 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/DecommissionSuite.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/DecommissionSuite.scala @@ -26,7 +26,6 @@ private[spark] trait DecommissionSuite { k8sSuite: KubernetesSuite => test("Test basic decommissioning", k8sTestTag) { sparkAppConf .set(config.DECOMMISSION_ENABLED.key, "true") - .set("spark.kubernetes.pyspark.pythonVersion", "3") .set("spark.kubernetes.container.image", pyImage) .set(config.STORAGE_DECOMMISSION_ENABLED.key, "true") .set(config.STORAGE_DECOMMISSION_SHUFFLE_BLOCKS_ENABLED.key, "true") diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PythonTestsSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PythonTestsSuite.scala index b16ccb429074f..bad6f1c1021ba 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PythonTestsSuite.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PythonTestsSuite.scala @@ -35,10 +35,9 @@ private[spark] trait PythonTestsSuite { k8sSuite: KubernetesSuite => isJVM = false) } - test("Run PySpark with Python3 to test a pyfiles example", k8sTestTag) { + test("Run PySpark to test a pyfiles example", k8sTestTag) { sparkAppConf .set("spark.kubernetes.container.image", pyImage) - .set("spark.kubernetes.pyspark.pythonVersion", "3") runSparkApplicationAndVerifyCompletion( appResource = PYSPARK_FILES, mainClass = "", @@ -57,7 +56,6 @@ private[spark] trait PythonTestsSuite { k8sSuite: KubernetesSuite => test("Run PySpark with memory customization", k8sTestTag) { sparkAppConf .set("spark.kubernetes.container.image", pyImage) - .set("spark.kubernetes.pyspark.pythonVersion", "3") .set("spark.kubernetes.memoryOverheadFactor", s"$memOverheadConstant") .set("spark.executor.pyspark.memory", s"${additionalMemory}m") runSparkApplicationAndVerifyCompletion( diff --git a/resource-managers/kubernetes/integration-tests/tests/pyfiles.py b/resource-managers/kubernetes/integration-tests/tests/pyfiles.py index 51c0160554866..73c53be482c03 100644 --- a/resource-managers/kubernetes/integration-tests/tests/pyfiles.py +++ b/resource-managers/kubernetes/integration-tests/tests/pyfiles.py @@ -31,7 +31,7 @@ from py_container_checks import version_check # Begin of Python container checks - version_check(sys.argv[1], 2 if sys.argv[1] == "python" else 3) + version_check(sys.argv[1], 3) # Check python executable at executors spark.udf.register("get_sys_ver",