Add support for Python 3.10 (#1246)

Release announcement:
https://www.python.org/downloads/release/python-3100/
https://blog.python.org/2021/10/python-3100-is-available.html

More detailsed "What's new":
https://docs.python.org/3.10/whatsnew/3.10.html

For this release, a new version of the buildpack's Python build script has been
added, which includes some long-overdue improvements:
- enables PGO (`--enable-optimizations`)
- enables loadable SQLite extensions (`--enable-loadable-sqlite-extensions`)
- enables strict configure option checking (`--enable-option-checking=fatal`)
- the make invocation now uses concurrency (!!)
- the test module cleanup step now also removes the `idle_test` directory

These bring our Python builds closer in-line with those used elsewhere, for example:
https://github.com/docker-library/python/blob/d65295b73bc2c711de96caeba17501d1a81b6a5c/3.9/buster/Dockerfile#L43-L45

For docs on the configure options, see:
https://docs.python.org/3.10/using/configure.html

Fixes #488.
GUS-W-8060029.
GUS-W-9093281.

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## Unreleased
 
+- Add support for Python 3.10 ([#1246](https://github.com/heroku/heroku-buildpack-python/pull/1246)).
 
 ## v199 (2021-09-05)
 

README.md

@@ -60,6 +60,7 @@ Specify a Python Runtime
 
 Supported runtime options include:
 
+- `python-3.10.0`
 - `python-3.9.7`
 - `python-3.8.12`
 - `python-3.7.12`

bin/compile

@@ -47,6 +47,7 @@ S3_BASE_URL="${BUILDPACK_S3_BASE_URL:-"https://heroku-buildpack-python.s3.amazon
 source "$BIN_DIR/default_pythons"
 
 # Supported Python Branches
+PY310="python-3.10"
 PY39="python-3.9"
 PY38="python-3.8"
 PY37="python-3.7"

bin/default_pythons

@@ -5,6 +5,7 @@
 # the env vars to subprocesses.
 # shellcheck disable=2034
 
+LATEST_310="python-3.10.0"
 LATEST_39="python-3.9.7"
 LATEST_38="python-3.8.12"
 LATEST_37="python-3.7.12"

bin/steps/pipenv-python-version

@@ -36,6 +36,9 @@ if [[ -f $BUILD_DIR/Pipfile ]]; then
                 if [ "$PYTHON" = 3.9 ]; then
                     echo "$LATEST_39" > "$BUILD_DIR/runtime.txt"
                 fi
+                if [ "$PYTHON" = "3.10" ]; then
+                    echo "$LATEST_310" > "$BUILD_DIR/runtime.txt"
+                fi
             fi
 
 

bin/steps/python

@@ -16,6 +16,12 @@ PYTHON_2_EOL_UPDATE="Python 2 has reached its community EOL. Upgrade your Python
 
 # check if runtime exists
 if curl --output /dev/null --silent --head --fail "$VENDORED_PYTHON"; then
+  if [[ "$PYTHON_VERSION" == $PY310* ]]; then
+    if [ "$PYTHON_VERSION" != "$LATEST_310" ]; then
+      puts-warn "$SECURITY_UPDATE" "$LATEST_310"
+      echo "       Learn More: https://devcenter.heroku.com/articles/python-runtimes"
+    fi
+  fi
   if [[ "$PYTHON_VERSION" == $PY39* ]]; then
     if [ "$PYTHON_VERSION" != "$LATEST_39" ]; then
       puts-warn "$SECURITY_UPDATE" "$LATEST_39"

builds/runtimes/python-3.10.0

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+source $(dirname $0)/python3-new

builds/runtimes/python3-new

@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+
+# This file will eventually be merged into the main `python3` build script,
+# however for Python 3.10 we are using some new configure options that we wish
+# to trial only in the Python 3.10 builds first.
+
+set -euo pipefail
+
+OUT_PREFIX=$1
+BIN_DIR="$(cd "$(dirname "$0")"/../.. || exit; pwd)/bin"
+export BIN_DIR
+
+# Orient ourselves and build download link
+dep_formula=${0#$WORKSPACE_DIR/} # the path to the original script (e.g. "builds/runtimes/python-3.10.0")
+BASE=${dep_formula##*/} # just the filename (e.g. "python-3.10.0")
+python_version=${BASE^} # the filename with the first character uppercased (e.g. "Python-3.10.0")
+version_number=$(echo "$python_version" | cut -d- -f2) # this version component (e.g. "3.10.0")
+dep_url=https://python.org/ftp/python/${version_number}/${python_version}.tgz
+
+echo "Building Python ${version_number}..."
+echo "Pulling from source: ${dep_url}"
+
+mkdir src
+curl -fL "${dep_url}" | tar -xz --strip-components=1 -C src/
+cd src
+
+./configure \
+  --enable-loadable-sqlite-extensions \
+  --enable-optimizations \
+  --enable-option-checking=fatal \
+  --prefix=$OUT_PREFIX \
+  --with-ensurepip=no
+
+make -j "$(nproc)"
+make install
+
+# Remove unneeded test directories, similar to the official Docker Python images:
+# https://github.com/docker-library/python
+# Not using the new --disable-test-modules since it breaks PGO which runs the Python test suite.
+find "${OUT_PREFIX}" \( -type d -a \( -name test -o -name tests -o -name idle_test \) \) -exec rm -rf '{}' +
+
+# Remove spare /
+LOCATION=${OUT_PREFIX%?}
+
+ln $LOCATION/bin/python3 $LOCATION/bin/python

spec/fixtures/pipenv_python_3.10/Pipfile

@@ -0,0 +1,12 @@
+[[source]]
+url = "https://pypi.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[packages]
+urllib3 = "*"
+
+[dev-packages]
+
+[requires]
+python_version = "3.10"

spec/fixtures/python_3.10/requirements.txt

@@ -0,0 +1 @@
+urllib3

spec/fixtures/python_3.10/runtime.txt

@@ -0,0 +1 @@
+python-3.10.0

spec/fixtures/python_3.10_outdated/runtime.txt

@@ -0,0 +1 @@
+python-3.10.0

spec/hatchet/pipenv_spec.rb

@@ -135,6 +135,12 @@
     include_examples 'builds using Pipenv with the requested Python version', LATEST_PYTHON_3_9
   end
 
+  context 'with a Pipfile.lock containing python_version 3.10' do
+    let(:app) { Hatchet::Runner.new('spec/fixtures/pipenv_python_3.10') }
+
+    include_examples 'builds using Pipenv with the requested Python version', LATEST_PYTHON_3_10
+  end
+
   context 'with a Pipfile.lock containing python_full_version 3.9.1' do
     let(:app) { Hatchet::Runner.new('spec/fixtures/pipenv_python_full_version') }
 

spec/hatchet/python_update_warning_spec.rb

@@ -111,6 +111,12 @@
     include_examples 'warns there is a Python update available', '3.9.0', LATEST_PYTHON_3_9
   end
 
+  context 'with a runtime.txt containing python-3.10.0', skip: 'requires that Python 3.10.1+ exists' do
+    let(:app) { Hatchet::Runner.new('spec/fixtures/python_3.10_outdated') }
+
+    include_examples 'warns there is a Python update available', '3.10.0', LATEST_PYTHON_3_10
+  end
+
   context 'with a runtime.txt containing pypy2.7-7.3.1' do
     let(:app) { Hatchet::Runner.new('spec/fixtures/pypy_2.7_outdated') }
 

spec/hatchet/python_version_spec.rb

@@ -180,6 +180,12 @@
     include_examples 'builds with the requested Python version', LATEST_PYTHON_3_9
   end
 
+  context 'when runtime.txt contains python-3.10.0' do
+    let(:app) { Hatchet::Runner.new('spec/fixtures/python_3.10') }
+
+    include_examples 'builds with the requested Python version', LATEST_PYTHON_3_10
+  end
+
   context 'when runtime.txt contains pypy2.7-7.3.2' do
     let(:app) { Hatchet::Runner.new('spec/fixtures/pypy_2.7') }
 

spec/spec_helper.rb

@@ -13,6 +13,7 @@
 LATEST_PYTHON_3_7 = '3.7.12'
 LATEST_PYTHON_3_8 = '3.8.12'
 LATEST_PYTHON_3_9 = '3.9.7'
+LATEST_PYTHON_3_10 = '3.10.0'
 LATEST_PYPY_2_7 = '7.3.2'
 LATEST_PYPY_3_6 = '7.3.2'
 DEFAULT_PYTHON_VERSION = LATEST_PYTHON_3_9