[SPARK-52561][PYTHON][DOC][FOLLOW-UP] Further upgrade the minimum python version

dev/create-release/spark-rm/Dockerfile

@@ -114,31 +114,19 @@ RUN python3.10 -m pip install $BASIC_PIP_PKGS unittest-xml-reporting $CONNECT_PI
     python3.10 -m pip install deepspeed torcheval && \
     python3.10 -m pip cache purge
 
-# Install Python 3.9
-RUN add-apt-repository ppa:deadsnakes/ppa
-RUN apt-get update && apt-get install -y \
-    python3.9 python3.9-distutils \
-    && rm -rf /var/lib/apt/lists/*
-RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python3.9
-RUN python3.9 -m pip install --ignore-installed blinker>=1.6.2 # mlflow needs this
-RUN python3.9 -m pip install --force $BASIC_PIP_PKGS unittest-xml-reporting $CONNECT_PIP_PKGS && \
-    python3.9 -m pip install 'torch<2.6.0' torchvision --index-url https://download.pytorch.org/whl/cpu && \
-    python3.9 -m pip install torcheval && \
-    python3.9 -m pip cache purge
-
 # Should unpin 'sphinxcontrib-*' after upgrading sphinx>5
 # See 'ipython_genutils' in SPARK-38517
 # See 'docutils<0.18.0' in SPARK-39421
-RUN python3.9 -m pip install 'sphinx==4.5.0' mkdocs 'pydata_sphinx_theme>=0.13' sphinx-copybutton nbsphinx numpydoc jinja2 markupsafe 'pyzmq<24.0.0' \
+RUN python3.10 -m pip install 'sphinx==4.5.0' mkdocs 'pydata_sphinx_theme>=0.13' sphinx-copybutton nbsphinx numpydoc jinja2 markupsafe 'pyzmq<24.0.0' \
 ipython ipython_genutils sphinx_plotly_directive 'numpy>=1.20.0' pyarrow pandas 'plotly>=4.8' 'docutils<0.18.0' \
 'flake8==3.9.0' 'mypy==1.8.0' 'pytest==7.1.3' 'pytest-mypy-plugins==1.9.3' 'black==23.12.1' \
 'pandas-stubs==1.2.0.53' 'grpcio==1.67.0' 'grpc-stubs==1.24.11' 'googleapis-common-protos-stubs==2.2.0' \
 'sphinxcontrib-applehelp==1.0.4' 'sphinxcontrib-devhelp==1.0.2' 'sphinxcontrib-htmlhelp==2.0.1' 'sphinxcontrib-qthelp==1.0.3' 'sphinxcontrib-serializinghtml==1.1.5'
-RUN python3.9 -m pip list
+RUN python3.10 -m pip list
 
 RUN gem install --no-document "bundler:2.4.22"
-RUN ln -s "$(which python3.9)" "/usr/local/bin/python"
-RUN ln -s "$(which python3.9)" "/usr/local/bin/python3"
+RUN ln -s "$(which python3.10)" "/usr/local/bin/python"
+RUN ln -s "$(which python3.10)" "/usr/local/bin/python3"
 
 WORKDIR /opt/spark-rm/output
 

dev/run-pip-tests

@@ -55,7 +55,7 @@ if hash virtualenv 2>/dev/null && [ ! -n "$USE_CONDA" ]; then
   fi
 elif hash conda 2>/dev/null; then
   echo "Using conda virtual environments"
-  PYTHON_EXECS=('3.9')
+  PYTHON_EXECS=('3.10')
   USE_CONDA=1
 else
   echo "Missing virtualenv & conda, skipping pip installability tests"

docs/index.md

@@ -34,7 +34,7 @@ source, visit [Building Spark](building-spark.html).
 
 Spark runs on both Windows and UNIX-like systems (e.g. Linux, Mac OS), and it should run on any platform that runs a supported version of Java. This should include JVMs on x86_64 and ARM64. It's easy to run locally on one machine --- all you need is to have `java` installed on your system `PATH`, or the `JAVA_HOME` environment variable pointing to a Java installation.
 
-Spark runs on Java 17/21, Scala 2.13, Python 3.9+, and R 3.5+ (Deprecated).
+Spark runs on Java 17/21, Scala 2.13, Python 3.10+, and R 3.5+ (Deprecated).
 When using the Scala API, it is necessary for applications to use the same version of Scala that Spark was compiled for. Since Spark 4.0.0, it's Scala 2.13.
 
 # Running the Examples and Shell

docs/rdd-programming-guide.md

@@ -39,7 +39,7 @@ along with if you launch Spark's interactive shell -- either `bin/spark-shell` f
 
 <div data-lang="python"  markdown="1">
 
-Spark {{site.SPARK_VERSION}} works with Python 3.9+. It can use the standard CPython interpreter,
+Spark {{site.SPARK_VERSION}} works with Python 3.10+. It can use the standard CPython interpreter,
 so C libraries like NumPy can be used. It also works with PyPy 7.3.6+.
 
 Spark applications in Python can either be run with the `bin/spark-submit` script which includes Spark at runtime, or by including it in your setup.py as:

python/docs/source/development/contributing.rst

@@ -129,8 +129,8 @@ If you are using Conda, the development environment can be set as follows.
 
 .. code-block:: bash
 
-    # Python 3.9+ is required
-    conda create --name pyspark-dev-env python=3.9
+    # Python 3.10+ is required
+    conda create --name pyspark-dev-env python=3.10
     conda activate pyspark-dev-env
     pip install --upgrade -r dev/requirements.txt
 
@@ -145,7 +145,7 @@ Now, you can start developing and `running the tests <testing.rst>`_.
 pip
 ~~~
 
-With Python 3.9+, pip can be used as below to install and set up the development environment.
+With Python 3.10+, pip can be used as below to install and set up the development environment.
 
 .. code-block:: bash
 

python/docs/source/tutorial/pandas_on_spark/typehints.rst

@@ -62,7 +62,7 @@ it as a Spark schema. As an example, you can specify the return type hint as bel
 Notice that the function ``pandas_div`` actually takes and outputs a pandas DataFrame instead of
 pandas-on-Spark :class:`DataFrame`. So, technically the correct types should be of pandas.
 
-With Python 3.9+, you can specify the type hints by using pandas instances as follows:
+With Python 3.10+, you can specify the type hints by using pandas instances as follows:
 
 .. code-block:: python
 

python/packaging/classic/setup.py

@@ -377,11 +377,10 @@ def run(self):
                 "pyyaml>=%s" % _minimum_pyyaml_version,
             ],
         },
-        python_requires=">=3.9",
+        python_requires=">=3.10",
         classifiers=[
             "Development Status :: 5 - Production/Stable",
             "License :: OSI Approved :: Apache Software License",
-            "Programming Language :: Python :: 3.9",
             "Programming Language :: Python :: 3.10",
             "Programming Language :: Python :: 3.11",
             "Programming Language :: Python :: 3.12",

python/packaging/client/setup.py

@@ -214,11 +214,10 @@
             "numpy>=%s" % _minimum_numpy_version,
             "pyyaml>=%s" % _minimum_pyyaml_version,
         ],
-        python_requires=">=3.9",
+        python_requires=">=3.10",
         classifiers=[
             "Development Status :: 5 - Production/Stable",
             "License :: OSI Approved :: Apache Software License",
-            "Programming Language :: Python :: 3.9",
             "Programming Language :: Python :: 3.10",
             "Programming Language :: Python :: 3.11",
             "Programming Language :: Python :: 3.12",

python/run-tests

@@ -21,9 +21,9 @@
 FWDIR="$(cd "`dirname $0`"/..; pwd)"
 cd "$FWDIR"
 
-PYTHON_VERSION_CHECK=$(python3 -c 'import sys; print(sys.version_info < (3, 9, 0))')
+PYTHON_VERSION_CHECK=$(python3 -c 'import sys; print(sys.version_info < (3, 10, 0))')
 if [[ "$PYTHON_VERSION_CHECK" == "True" ]]; then
-  echo "Python versions prior to 3.9 are not supported."
+  echo "Python versions prior to 3.10 are not supported."
   exit -1
 fi