Improve pip/setuptools/wheel installation

CHANGELOG.md

@@ -2,6 +2,13 @@
 
 # Master
 
+- For repeat builds, also manage the installed versions of setuptools/wheel, rather than just that of pip (#1007).
+- Install an explicit version of wheel rather than the latest release at the time (#1007).
+- Output the installed version of pip, setuptools and wheel in the build log (#1007).
+- Errors installing pip/setuptools/wheel are now displayed in the build output and fail the build early (#1007).
+- Install pip using itself rather than `get-pip.py` (#1007).
+- Disable pip's version check + cache when installing pip/setuptools/wheel (#1007).
+- Install setuptools from PyPI rather than a vendored copy (#1007).
 - Reduce the number of environment variables exposed to `bin/{pre,post}_compile` and other subprocesses (#1011)
 
 # 173 (2020-07-21)

bin/compile

@@ -65,26 +65,6 @@ PYPY36="pypy3.6"
 
 # Which stack is used (for binary downloading), if none is provided (e.g. outside of Heroku)?
 DEFAULT_PYTHON_STACK="cedar-14"
-# If pip doesn't match this version (the version we install), run the installer.
-PIP_UPDATE="20.0.2"
-
-for file in "$BUILD_DIR/runtime.txt" "$CACHE_DIR/.heroku/python-version" ; do
-    [ -f "$file" ] || continue
-
-    version=$(tr -d '[:space:]' < "$file")
-
-    case "$version" in "$PY34"*)
-        # Python 3.4 support was dropped in pip >= 19.2.
-        PIP_UPDATE="19.1.1"
-        break
-        ;;
-    esac
-done
-
-if [[ -f "$BUILD_DIR/Pipfile" ]]; then
-    # Do not force pipenv users to re-install pipenv locally.
-    PIP_UPDATE="9.0.2"
-fi
 
 # Common Problem Warnings:
 # This section creates a temporary file in which to stick the output of `pip install`.

bin/steps/README.MD

@@ -2,13 +2,6 @@
 
 TODO: Add context on Python install steps, such as why symlinking vs copying
 
-## Installing the Pip tool
-
-The Python Buildpack uses a tool called `get-pip` to install the pip tool. This
-is done in the `python` script.
-
-This is in part because Python historically did not come with pip by default.
-
 ## Installing Python packages using Pip
 
 ### Convention: Use `python` process to invoke Pip

bin/steps/python

@@ -131,34 +131,52 @@ if [ ! "$SKIP_INSTALL" ]; then
   # Record for future reference.
   echo "$PYTHON_VERSION" > .heroku/python-version
   echo "$STACK" > .heroku/python-stack
-  FRESH_PYTHON=true
 
   hash -r
 fi
 
-# Heroku uses the get-pip utility maintained by the Python community to vendor Pip.
-# https://github.com/pypa/get-pip
-GETPIP_URL="https://lang-python.s3.amazonaws.com/etc/get-pip.py"
-GETPIP_PY="${TMPDIR:-/tmp}/get-pip.py"
+set -e
+
+PIP_VERSION='20.0.2'
+SETUPTOOLS_VERSION='39.0.1'
+WHEEL_VERSION='0.34.2'
 
-if ! curl -s "${GETPIP_URL}" -o "$GETPIP_PY" &> /dev/null; then
-    mcount "failure.python.get-pip"
-    echo "Failed to pull down get-pip"
-    exit 1
+if [[ "${PYTHON_VERSION}" == ${PY34}* ]]; then
+  # Python 3.4 support was dropped in pip 19.2+ and wheel 0.34.0+.
+  PIP_VERSION='19.1.1'
+  WHEEL_VERSION='0.33.6'
 fi
 
-# If a new Python has been installed or Pip isn't up to date:
-if [ "$FRESH_PYTHON" ] || [[ ! $(pip --version) == *$PIP_UPDATE* ]]; then
+# We don't use get-pip.py, since:
+#  - it uses `--force-reinstall`, which is unnecessary here and slows down repeat builds
+#  - it means downloading pip twice (once embedded in get-pip.py, and again during
+#    the install, since get-pip.py can't install the embedded version directly)
+#  - we would still have to manage several versions of get-pip.py, to support older Pythons.
+# Instead, we use the pip wheel to install itself, using the method described here:
+# https://github.com/pypa/pip/issues/2351#issuecomment-69994524
+PIP_WHEEL_FILENAME="pip-${PIP_VERSION}-py2.py3-none-any.whl"
+PIP_WHEEL_URL="https://lang-python.s3.amazonaws.com/common/${PIP_WHEEL_FILENAME}"
+PIP_WHEEL="${TMPDIR:-/tmp}/${PIP_WHEEL_FILENAME}"
+
+if ! curl -sSf "${PIP_WHEEL_URL}" -o "$PIP_WHEEL"; then
+  mcount "failure.python.download-pip"
+  puts-warn "Failed to download pip"
+  exit 1
+fi
 
-  puts-step "Installing pip"
+if [[ -f "$BUILD_DIR/Pipfile" ]]; then
+  # The buildpack is pinned to old pipenv, which requires older pip.
+  # Pip 9.0.2 doesn't support installing itself from a wheel, so we have to use split
+  # versions here (ie: installer pip version different from target pip version).
+  PIP_VERSION='9.0.2'
+  PIP_TO_INSTALL="pip==${PIP_VERSION}"
+else
+  PIP_TO_INSTALL="${PIP_WHEEL}"
+fi
 
-  # Remove old installations.
-  rm -fr /app/.heroku/python/lib/python*/site-packages/pip-*
-  rm -fr /app/.heroku/python/lib/python*/site-packages/setuptools-*
+puts-step "Installing pip ${PIP_VERSION}, setuptools ${SETUPTOOLS_VERSION} and wheel ${WHEEL_VERSION}"
 
-  /app/.heroku/python/bin/python "$GETPIP_PY" pip=="$PIP_UPDATE" &> /dev/null
-  /app/.heroku/python/bin/pip install "$ROOT_DIR/vendor/setuptools-39.0.1-py2.py3-none-any.whl" &> /dev/null
-fi
+/app/.heroku/python/bin/python "${PIP_WHEEL}/pip" install --quiet --disable-pip-version-check --no-cache \
+  "${PIP_TO_INSTALL}" "setuptools==${SETUPTOOLS_VERSION}" "wheel==${WHEEL_VERSION}"
 
-set -e
 hash -r

test/run-features

@@ -44,6 +44,7 @@ testStandardRequirements() {
 
 testPipenv() {
   compile "pipenv"
+  assertCaptured "Installing pip 9.0.2, setuptools 39.0.1 and wheel 0.34.2"
   assertCapturedSuccess
 }
 

test/run-versions

@@ -9,6 +9,7 @@ testPythonDefault() {
   compile "pythonDefault"
   assertCaptured $DEFAULT_PYTHON_VERSION
   assertNotCaptured "security update"
+  assertCaptured "Installing pip 20.0.2, setuptools 39.0.1 and wheel 0.34.2"
   assertCaptured "Installing SQLite3"
   assertCapturedSuccess
 }
@@ -24,6 +25,7 @@ testPython2() {
       assertNotCaptured "python-2-7-eol-faq";
     fi
     assertNotCaptured "security update"
+    assertCaptured "Installing pip 20.0.2, setuptools 39.0.1 and wheel 0.34.2"
     assertCaptured "Installing SQLite3"
     assertCapturedSuccess
 }
@@ -51,6 +53,7 @@ testPython3_4() {
   compile "python3_4"
   assertCaptured $LATEST_34
   assertNotCaptured "security update"
+  assertCaptured "Installing pip 19.1.1, setuptools 39.0.1 and wheel 0.33.6"
   # if cedar 14 and legacy binaries, fail. if cedar 14 and staging, succeed.
   if [[ ! -n $USE_STAGING_BINARIES ]] && [[ $STACK == "cedar-14" ]]; then
       assertCapturedError
@@ -86,6 +89,7 @@ testPython3_5() {
   compile "python3_5"
   assertCaptured $LATEST_35
   assertNotCaptured "security update"
+  assertCaptured "Installing pip 20.0.2, setuptools 39.0.1 and wheel 0.34.2"
   assertCaptured "Installing SQLite3"
   assertCapturedSuccess
 }
@@ -108,6 +112,7 @@ testPython3_6() {
   compile "python3_6"
   assertCaptured $LATEST_36
   assertNotCaptured "security update"
+  assertCaptured "Installing pip 20.0.2, setuptools 39.0.1 and wheel 0.34.2"
   assertCaptured "Installing SQLite3"
   assertCapturedSuccess
 }
@@ -134,6 +139,7 @@ testPython3_7() {
   else
     assertNotCaptured "security update"
     assertCaptured $LATEST_37
+    assertCaptured "Installing pip 20.0.2, setuptools 39.0.1 and wheel 0.34.2"
     assertCaptured "Installing SQLite3"
     assertCapturedSuccess
   fi
@@ -177,6 +183,7 @@ testPython3_8() {
   else
     assertNotCaptured "security update"
     assertCaptured $LATEST_38
+    assertCaptured "Installing pip 20.0.2, setuptools 39.0.1 and wheel 0.34.2"
     assertCaptured "Installing SQLite3"
     assertCapturedSuccess
   fi
@@ -195,6 +202,7 @@ testPypy3_6() {
   else
     assertCaptured "Installing pypy"
     assertCaptured "$PYPY_36"
+    assertCaptured "Installing pip 20.0.2, setuptools 39.0.1 and wheel 0.34.2"
     assertCapturedSuccess
   fi
 }
@@ -206,6 +214,7 @@ testPypy2_7() {
   else
     assertCaptured "Installing pypy"
     assertCaptured "$PYPY_27"
+    assertCaptured "Installing pip 20.0.2, setuptools 39.0.1 and wheel 0.34.2"
     assertCapturedSuccess
   fi
 }