diff --git a/dev/create-release/spark-rm/Dockerfile b/dev/create-release/spark-rm/Dockerfile index f7835b3b34052..7ef6f7b9ab1d8 100644 --- a/dev/create-release/spark-rm/Dockerfile +++ b/dev/create-release/spark-rm/Dockerfile @@ -114,31 +114,19 @@ RUN python3.10 -m pip install $BASIC_PIP_PKGS unittest-xml-reporting $CONNECT_PI python3.10 -m pip install deepspeed torcheval && \ python3.10 -m pip cache purge -# Install Python 3.9 -RUN add-apt-repository ppa:deadsnakes/ppa -RUN apt-get update && apt-get install -y \ - python3.9 python3.9-distutils \ - && rm -rf /var/lib/apt/lists/* -RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python3.9 -RUN python3.9 -m pip install --ignore-installed blinker>=1.6.2 # mlflow needs this -RUN python3.9 -m pip install --force $BASIC_PIP_PKGS unittest-xml-reporting $CONNECT_PIP_PKGS && \ - python3.9 -m pip install 'torch<2.6.0' torchvision --index-url https://download.pytorch.org/whl/cpu && \ - python3.9 -m pip install torcheval && \ - python3.9 -m pip cache purge - # Should unpin 'sphinxcontrib-*' after upgrading sphinx>5 # See 'ipython_genutils' in SPARK-38517 # See 'docutils<0.18.0' in SPARK-39421 -RUN python3.9 -m pip install 'sphinx==4.5.0' mkdocs 'pydata_sphinx_theme>=0.13' sphinx-copybutton nbsphinx numpydoc jinja2 markupsafe 'pyzmq<24.0.0' \ +RUN python3.10 -m pip install 'sphinx==4.5.0' mkdocs 'pydata_sphinx_theme>=0.13' sphinx-copybutton nbsphinx numpydoc jinja2 markupsafe 'pyzmq<24.0.0' \ ipython ipython_genutils sphinx_plotly_directive 'numpy>=1.20.0' pyarrow pandas 'plotly>=4.8' 'docutils<0.18.0' \ 'flake8==3.9.0' 'mypy==1.8.0' 'pytest==7.1.3' 'pytest-mypy-plugins==1.9.3' 'black==23.12.1' \ 'pandas-stubs==1.2.0.53' 'grpcio==1.67.0' 'grpc-stubs==1.24.11' 'googleapis-common-protos-stubs==2.2.0' \ 'sphinxcontrib-applehelp==1.0.4' 'sphinxcontrib-devhelp==1.0.2' 'sphinxcontrib-htmlhelp==2.0.1' 'sphinxcontrib-qthelp==1.0.3' 'sphinxcontrib-serializinghtml==1.1.5' -RUN python3.9 -m pip list +RUN python3.10 -m pip list RUN gem install --no-document "bundler:2.4.22" -RUN ln -s "$(which python3.9)" "/usr/local/bin/python" -RUN ln -s "$(which python3.9)" "/usr/local/bin/python3" +RUN ln -s "$(which python3.10)" "/usr/local/bin/python" +RUN ln -s "$(which python3.10)" "/usr/local/bin/python3" WORKDIR /opt/spark-rm/output diff --git a/dev/run-pip-tests b/dev/run-pip-tests index 7740481e4d832..426ec329d915d 100755 --- a/dev/run-pip-tests +++ b/dev/run-pip-tests @@ -55,7 +55,7 @@ if hash virtualenv 2>/dev/null && [ ! -n "$USE_CONDA" ]; then fi elif hash conda 2>/dev/null; then echo "Using conda virtual environments" - PYTHON_EXECS=('3.9') + PYTHON_EXECS=('3.10') USE_CONDA=1 else echo "Missing virtualenv & conda, skipping pip installability tests" diff --git a/docs/index.md b/docs/index.md index b2e6e4a2a8936..cb32ddcde7e2b 100644 --- a/docs/index.md +++ b/docs/index.md @@ -34,7 +34,7 @@ source, visit [Building Spark](building-spark.html). Spark runs on both Windows and UNIX-like systems (e.g. Linux, Mac OS), and it should run on any platform that runs a supported version of Java. This should include JVMs on x86_64 and ARM64. It's easy to run locally on one machine --- all you need is to have `java` installed on your system `PATH`, or the `JAVA_HOME` environment variable pointing to a Java installation. -Spark runs on Java 17/21, Scala 2.13, Python 3.9+, and R 3.5+ (Deprecated). +Spark runs on Java 17/21, Scala 2.13, Python 3.10+, and R 3.5+ (Deprecated). When using the Scala API, it is necessary for applications to use the same version of Scala that Spark was compiled for. Since Spark 4.0.0, it's Scala 2.13. # Running the Examples and Shell diff --git a/docs/rdd-programming-guide.md b/docs/rdd-programming-guide.md index 400f8a512e7a7..6e69cec23a40d 100644 --- a/docs/rdd-programming-guide.md +++ b/docs/rdd-programming-guide.md @@ -39,7 +39,7 @@ along with if you launch Spark's interactive shell -- either `bin/spark-shell` f
-Spark {{site.SPARK_VERSION}} works with Python 3.9+. It can use the standard CPython interpreter, +Spark {{site.SPARK_VERSION}} works with Python 3.10+. It can use the standard CPython interpreter, so C libraries like NumPy can be used. It also works with PyPy 7.3.6+. Spark applications in Python can either be run with the `bin/spark-submit` script which includes Spark at runtime, or by including it in your setup.py as: diff --git a/python/docs/source/development/contributing.rst b/python/docs/source/development/contributing.rst index fc2a1cc586040..4103a1e650930 100644 --- a/python/docs/source/development/contributing.rst +++ b/python/docs/source/development/contributing.rst @@ -129,8 +129,8 @@ If you are using Conda, the development environment can be set as follows. .. code-block:: bash - # Python 3.9+ is required - conda create --name pyspark-dev-env python=3.9 + # Python 3.10+ is required + conda create --name pyspark-dev-env python=3.10 conda activate pyspark-dev-env pip install --upgrade -r dev/requirements.txt @@ -145,7 +145,7 @@ Now, you can start developing and `running the tests `_. pip ~~~ -With Python 3.9+, pip can be used as below to install and set up the development environment. +With Python 3.10+, pip can be used as below to install and set up the development environment. .. code-block:: bash diff --git a/python/docs/source/tutorial/pandas_on_spark/typehints.rst b/python/docs/source/tutorial/pandas_on_spark/typehints.rst index 23126664d78a6..7045ce7f75987 100644 --- a/python/docs/source/tutorial/pandas_on_spark/typehints.rst +++ b/python/docs/source/tutorial/pandas_on_spark/typehints.rst @@ -62,7 +62,7 @@ it as a Spark schema. As an example, you can specify the return type hint as bel Notice that the function ``pandas_div`` actually takes and outputs a pandas DataFrame instead of pandas-on-Spark :class:`DataFrame`. So, technically the correct types should be of pandas. -With Python 3.9+, you can specify the type hints by using pandas instances as follows: +With Python 3.10+, you can specify the type hints by using pandas instances as follows: .. code-block:: python diff --git a/python/packaging/classic/setup.py b/python/packaging/classic/setup.py index 8f22779e0546a..9b5dcfc06756f 100755 --- a/python/packaging/classic/setup.py +++ b/python/packaging/classic/setup.py @@ -377,11 +377,10 @@ def run(self): "pyyaml>=%s" % _minimum_pyyaml_version, ], }, - python_requires=">=3.9", + python_requires=">=3.10", classifiers=[ "Development Status :: 5 - Production/Stable", "License :: OSI Approved :: Apache Software License", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", diff --git a/python/packaging/client/setup.py b/python/packaging/client/setup.py index ac80e3a1ff585..79b87d85884ad 100755 --- a/python/packaging/client/setup.py +++ b/python/packaging/client/setup.py @@ -214,11 +214,10 @@ "numpy>=%s" % _minimum_numpy_version, "pyyaml>=%s" % _minimum_pyyaml_version, ], - python_requires=">=3.9", + python_requires=">=3.10", classifiers=[ "Development Status :: 5 - Production/Stable", "License :: OSI Approved :: Apache Software License", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", diff --git a/python/run-tests b/python/run-tests index b5492a080d6a5..c0b0f1c9e5da4 100755 --- a/python/run-tests +++ b/python/run-tests @@ -21,9 +21,9 @@ FWDIR="$(cd "`dirname $0`"/..; pwd)" cd "$FWDIR" -PYTHON_VERSION_CHECK=$(python3 -c 'import sys; print(sys.version_info < (3, 9, 0))') +PYTHON_VERSION_CHECK=$(python3 -c 'import sys; print(sys.version_info < (3, 10, 0))') if [[ "$PYTHON_VERSION_CHECK" == "True" ]]; then - echo "Python versions prior to 3.9 are not supported." + echo "Python versions prior to 3.10 are not supported." exit -1 fi