From df1556ceecf11eaf1e60f297b4b7943fdcbf9355 Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Tue, 28 Jul 2020 21:02:13 +0100 Subject: [PATCH 01/11] Output the installed version of pip in the build log (#1007) Since: * "explicit is better than implicit" * we'll soon be upgrading pip, and debugging breakage caused by upgrades will be easier if versions are visible in the build log Closes #939. --- CHANGELOG.md | 1 + bin/steps/python | 2 +- test/run-features | 1 + test/run-versions | 9 +++++++++ 4 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6275cba2e..c2b7b74ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ # Master +- Output the installed version of pip in the build log (#1007). - Reduce the number of environment variables exposed to `bin/{pre,post}_compile` and other subprocesses (#1011) # 173 (2020-07-21) diff --git a/bin/steps/python b/bin/steps/python index 068e147cc..23a1850b9 100755 --- a/bin/steps/python +++ b/bin/steps/python @@ -150,7 +150,7 @@ fi # If a new Python has been installed or Pip isn't up to date: if [ "$FRESH_PYTHON" ] || [[ ! $(pip --version) == *$PIP_UPDATE* ]]; then - puts-step "Installing pip" + puts-step "Installing pip ${PIP_UPDATE}" # Remove old installations. rm -fr /app/.heroku/python/lib/python*/site-packages/pip-* diff --git a/test/run-features b/test/run-features index c50a8c33d..170a8a937 100755 --- a/test/run-features +++ b/test/run-features @@ -44,6 +44,7 @@ testStandardRequirements() { testPipenv() { compile "pipenv" + assertCaptured "Installing pip 9.0.2" assertCapturedSuccess } diff --git a/test/run-versions b/test/run-versions index 75f165b94..6fe7c1316 100755 --- a/test/run-versions +++ b/test/run-versions @@ -9,6 +9,7 @@ testPythonDefault() { compile "pythonDefault" assertCaptured $DEFAULT_PYTHON_VERSION assertNotCaptured "security update" + assertCaptured "Installing pip 20.0.2" assertCaptured "Installing SQLite3" assertCapturedSuccess } @@ -24,6 +25,7 @@ testPython2() { assertNotCaptured "python-2-7-eol-faq"; fi assertNotCaptured "security update" + assertCaptured "Installing pip 20.0.2" assertCaptured "Installing SQLite3" assertCapturedSuccess } @@ -51,6 +53,7 @@ testPython3_4() { compile "python3_4" assertCaptured $LATEST_34 assertNotCaptured "security update" + assertCaptured "Installing pip 19.1.1" # if cedar 14 and legacy binaries, fail. if cedar 14 and staging, succeed. if [[ ! -n $USE_STAGING_BINARIES ]] && [[ $STACK == "cedar-14" ]]; then assertCapturedError @@ -86,6 +89,7 @@ testPython3_5() { compile "python3_5" assertCaptured $LATEST_35 assertNotCaptured "security update" + assertCaptured "Installing pip 20.0.2" assertCaptured "Installing SQLite3" assertCapturedSuccess } @@ -108,6 +112,7 @@ testPython3_6() { compile "python3_6" assertCaptured $LATEST_36 assertNotCaptured "security update" + assertCaptured "Installing pip 20.0.2" assertCaptured "Installing SQLite3" assertCapturedSuccess } @@ -134,6 +139,7 @@ testPython3_7() { else assertNotCaptured "security update" assertCaptured $LATEST_37 + assertCaptured "Installing pip 20.0.2" assertCaptured "Installing SQLite3" assertCapturedSuccess fi @@ -177,6 +183,7 @@ testPython3_8() { else assertNotCaptured "security update" assertCaptured $LATEST_38 + assertCaptured "Installing pip 20.0.2" assertCaptured "Installing SQLite3" assertCapturedSuccess fi @@ -195,6 +202,7 @@ testPypy3_6() { else assertCaptured "Installing pypy" assertCaptured "$PYPY_36" + assertCaptured "Installing pip 20.0.2" assertCapturedSuccess fi } @@ -206,6 +214,7 @@ testPypy2_7() { else assertCaptured "Installing pypy" assertCaptured "$PYPY_27" + assertCaptured "Installing pip 20.0.2" assertCapturedSuccess fi } From eb3514d2f9df1719d08da50ea6ed9cd6204b9842 Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Tue, 28 Jul 2020 21:07:59 +0100 Subject: [PATCH 02/11] Move Pip version handling to bin/steps/python (#1007) And use the `$PYTHON_VERSION` calculated in `bin/steps/python` instead of re-implementing the Python version handling. --- bin/compile | 20 -------------------- bin/steps/python | 18 +++++++++++++++--- 2 files changed, 15 insertions(+), 23 deletions(-) diff --git a/bin/compile b/bin/compile index dd270d802..9a0a2d6f6 100755 --- a/bin/compile +++ b/bin/compile @@ -65,26 +65,6 @@ PYPY36="pypy3.6" # Which stack is used (for binary downloading), if none is provided (e.g. outside of Heroku)? DEFAULT_PYTHON_STACK="cedar-14" -# If pip doesn't match this version (the version we install), run the installer. -PIP_UPDATE="20.0.2" - -for file in "$BUILD_DIR/runtime.txt" "$CACHE_DIR/.heroku/python-version" ; do - [ -f "$file" ] || continue - - version=$(tr -d '[:space:]' < "$file") - - case "$version" in "$PY34"*) - # Python 3.4 support was dropped in pip >= 19.2. - PIP_UPDATE="19.1.1" - break - ;; - esac -done - -if [[ -f "$BUILD_DIR/Pipfile" ]]; then - # Do not force pipenv users to re-install pipenv locally. - PIP_UPDATE="9.0.2" -fi # Common Problem Warnings: # This section creates a temporary file in which to stick the output of `pip install`. diff --git a/bin/steps/python b/bin/steps/python index 23a1850b9..5129f5b36 100755 --- a/bin/steps/python +++ b/bin/steps/python @@ -136,6 +136,18 @@ if [ ! "$SKIP_INSTALL" ]; then hash -r fi +PIP_VERSION='20.0.2' + +if [[ "${PYTHON_VERSION}" == ${PY34}* ]]; then + # Python 3.4 support was dropped in pip 19.2+. + PIP_VERSION='19.1.1' +fi + +if [[ -f "$BUILD_DIR/Pipfile" ]]; then + # The buildpack is pinned to old pipenv, which requires older pip. + PIP_VERSION='9.0.2' +fi + # Heroku uses the get-pip utility maintained by the Python community to vendor Pip. # https://github.com/pypa/get-pip GETPIP_URL="https://lang-python.s3.amazonaws.com/etc/get-pip.py" @@ -148,15 +160,15 @@ if ! curl -s "${GETPIP_URL}" -o "$GETPIP_PY" &> /dev/null; then fi # If a new Python has been installed or Pip isn't up to date: -if [ "$FRESH_PYTHON" ] || [[ ! $(pip --version) == *$PIP_UPDATE* ]]; then +if [ "$FRESH_PYTHON" ] || [[ ! $(pip --version) == *${PIP_VERSION}* ]]; then - puts-step "Installing pip ${PIP_UPDATE}" + puts-step "Installing pip ${PIP_VERSION}" # Remove old installations. rm -fr /app/.heroku/python/lib/python*/site-packages/pip-* rm -fr /app/.heroku/python/lib/python*/site-packages/setuptools-* - /app/.heroku/python/bin/python "$GETPIP_PY" pip=="$PIP_UPDATE" &> /dev/null + /app/.heroku/python/bin/python "$GETPIP_PY" pip=="${PIP_VERSION}" &> /dev/null /app/.heroku/python/bin/pip install "$ROOT_DIR/vendor/setuptools-39.0.1-py2.py3-none-any.whl" &> /dev/null fi From 568c0b752b08b9a15d8865764b66eb191c8fdde8 Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Tue, 28 Jul 2020 21:17:41 +0100 Subject: [PATCH 03/11] Output the installed version of setuptools in the build log (#1007) Since: * "explicit is better than implicit" * we'll soon be upgrading setuptools, and debugging breakage caused by upgrades will be easier if versions are visible in the build log --- CHANGELOG.md | 2 +- bin/steps/python | 5 +++-- test/run-features | 2 +- test/run-versions | 18 +++++++++--------- 4 files changed, 14 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c2b7b74ae..ef490fa94 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ # Master -- Output the installed version of pip in the build log (#1007). +- Output the installed version of pip and setuptools in the build log (#1007). - Reduce the number of environment variables exposed to `bin/{pre,post}_compile` and other subprocesses (#1011) # 173 (2020-07-21) diff --git a/bin/steps/python b/bin/steps/python index 5129f5b36..cab5f7907 100755 --- a/bin/steps/python +++ b/bin/steps/python @@ -137,6 +137,7 @@ if [ ! "$SKIP_INSTALL" ]; then fi PIP_VERSION='20.0.2' +SETUPTOOLS_VERSION='39.0.1' if [[ "${PYTHON_VERSION}" == ${PY34}* ]]; then # Python 3.4 support was dropped in pip 19.2+. @@ -162,14 +163,14 @@ fi # If a new Python has been installed or Pip isn't up to date: if [ "$FRESH_PYTHON" ] || [[ ! $(pip --version) == *${PIP_VERSION}* ]]; then - puts-step "Installing pip ${PIP_VERSION}" + puts-step "Installing pip ${PIP_VERSION} and setuptools ${SETUPTOOLS_VERSION}" # Remove old installations. rm -fr /app/.heroku/python/lib/python*/site-packages/pip-* rm -fr /app/.heroku/python/lib/python*/site-packages/setuptools-* /app/.heroku/python/bin/python "$GETPIP_PY" pip=="${PIP_VERSION}" &> /dev/null - /app/.heroku/python/bin/pip install "$ROOT_DIR/vendor/setuptools-39.0.1-py2.py3-none-any.whl" &> /dev/null + /app/.heroku/python/bin/pip install "$ROOT_DIR/vendor/setuptools-${SETUPTOOLS_VERSION}-py2.py3-none-any.whl" &> /dev/null fi set -e diff --git a/test/run-features b/test/run-features index 170a8a937..1c059c8f8 100755 --- a/test/run-features +++ b/test/run-features @@ -44,7 +44,7 @@ testStandardRequirements() { testPipenv() { compile "pipenv" - assertCaptured "Installing pip 9.0.2" + assertCaptured "Installing pip 9.0.2 and setuptools 39.0.1" assertCapturedSuccess } diff --git a/test/run-versions b/test/run-versions index 6fe7c1316..49775d85a 100755 --- a/test/run-versions +++ b/test/run-versions @@ -9,7 +9,7 @@ testPythonDefault() { compile "pythonDefault" assertCaptured $DEFAULT_PYTHON_VERSION assertNotCaptured "security update" - assertCaptured "Installing pip 20.0.2" + assertCaptured "Installing pip 20.0.2 and setuptools 39.0.1" assertCaptured "Installing SQLite3" assertCapturedSuccess } @@ -25,7 +25,7 @@ testPython2() { assertNotCaptured "python-2-7-eol-faq"; fi assertNotCaptured "security update" - assertCaptured "Installing pip 20.0.2" + assertCaptured "Installing pip 20.0.2 and setuptools 39.0.1" assertCaptured "Installing SQLite3" assertCapturedSuccess } @@ -53,7 +53,7 @@ testPython3_4() { compile "python3_4" assertCaptured $LATEST_34 assertNotCaptured "security update" - assertCaptured "Installing pip 19.1.1" + assertCaptured "Installing pip 19.1.1 and setuptools 39.0.1" # if cedar 14 and legacy binaries, fail. if cedar 14 and staging, succeed. if [[ ! -n $USE_STAGING_BINARIES ]] && [[ $STACK == "cedar-14" ]]; then assertCapturedError @@ -89,7 +89,7 @@ testPython3_5() { compile "python3_5" assertCaptured $LATEST_35 assertNotCaptured "security update" - assertCaptured "Installing pip 20.0.2" + assertCaptured "Installing pip 20.0.2 and setuptools 39.0.1" assertCaptured "Installing SQLite3" assertCapturedSuccess } @@ -112,7 +112,7 @@ testPython3_6() { compile "python3_6" assertCaptured $LATEST_36 assertNotCaptured "security update" - assertCaptured "Installing pip 20.0.2" + assertCaptured "Installing pip 20.0.2 and setuptools 39.0.1" assertCaptured "Installing SQLite3" assertCapturedSuccess } @@ -139,7 +139,7 @@ testPython3_7() { else assertNotCaptured "security update" assertCaptured $LATEST_37 - assertCaptured "Installing pip 20.0.2" + assertCaptured "Installing pip 20.0.2 and setuptools 39.0.1" assertCaptured "Installing SQLite3" assertCapturedSuccess fi @@ -183,7 +183,7 @@ testPython3_8() { else assertNotCaptured "security update" assertCaptured $LATEST_38 - assertCaptured "Installing pip 20.0.2" + assertCaptured "Installing pip 20.0.2 and setuptools 39.0.1" assertCaptured "Installing SQLite3" assertCapturedSuccess fi @@ -202,7 +202,7 @@ testPypy3_6() { else assertCaptured "Installing pypy" assertCaptured "$PYPY_36" - assertCaptured "Installing pip 20.0.2" + assertCaptured "Installing pip 20.0.2 and setuptools 39.0.1" assertCapturedSuccess fi } @@ -214,7 +214,7 @@ testPypy2_7() { else assertCaptured "Installing pypy" assertCaptured "$PYPY_27" - assertCaptured "Installing pip 20.0.2" + assertCaptured "Installing pip 20.0.2 and setuptools 39.0.1" assertCapturedSuccess fi } From 43fff31efed16bcf57541a39de48b54df0165716 Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Tue, 28 Jul 2020 21:26:51 +0100 Subject: [PATCH 04/11] Install setuptools from PyPI rather than a vendored copy (#1007) Since: * we'll be updating setuptools soon, and newer setuptools has dropped support for Python versions this buildpack needs to support. As such if we continued to vendor setuptools, we would need to vendor at least three different versions. * we want to try and update setuptools more frequently than we have in the past, which will mean more repo bloat from binary churn. * we're still pinning to a specific version, meaning vendoring doesn't have determinism benefits. * setuptools is only fetched from PyPI for new installs (or where versions have changed), so this doesn't increase build time, load on PyPI, or reliance on PyPI in the common case. * setuptools is already being inadvertently installed from PyPI prior to being installed from the vendored copy (see #1001), so we're in effect already using/depending on PyPI here. * switching to storing setuptools on S3 wouldn't help reliability as much as it would appear at first glance, since the later `pip install` of customer dependencies will fail if PyPI is down anyway. --- CHANGELOG.md | 1 + bin/steps/python | 2 +- vendor/setuptools-39.0.1-py2.py3-none-any.whl | Bin 569436 -> 0 bytes 3 files changed, 2 insertions(+), 1 deletion(-) delete mode 100644 vendor/setuptools-39.0.1-py2.py3-none-any.whl diff --git a/CHANGELOG.md b/CHANGELOG.md index ef490fa94..a359d8f32 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ # Master - Output the installed version of pip and setuptools in the build log (#1007). +- Install setuptools from PyPI rather than a vendored copy (#1007). - Reduce the number of environment variables exposed to `bin/{pre,post}_compile` and other subprocesses (#1011) # 173 (2020-07-21) diff --git a/bin/steps/python b/bin/steps/python index cab5f7907..381cc2188 100755 --- a/bin/steps/python +++ b/bin/steps/python @@ -170,7 +170,7 @@ if [ "$FRESH_PYTHON" ] || [[ ! $(pip --version) == *${PIP_VERSION}* ]]; then rm -fr /app/.heroku/python/lib/python*/site-packages/setuptools-* /app/.heroku/python/bin/python "$GETPIP_PY" pip=="${PIP_VERSION}" &> /dev/null - /app/.heroku/python/bin/pip install "$ROOT_DIR/vendor/setuptools-${SETUPTOOLS_VERSION}-py2.py3-none-any.whl" &> /dev/null + /app/.heroku/python/bin/pip install "setuptools==${SETUPTOOLS_VERSION}" &> /dev/null fi set -e diff --git a/vendor/setuptools-39.0.1-py2.py3-none-any.whl b/vendor/setuptools-39.0.1-py2.py3-none-any.whl deleted file mode 100644 index edc3ca2d8ec3e0af7a305c5b9cf0fe574069ae90..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 569436 zcmV)0K+eBVO9KQH0000806BGXOk7X8MQH#40Db@f01p5F0A*owd0%O6b97;BY%Xwl zEzB_vgFp;K(SA>1xq&nsAQEMYv|M0hkwsZ)Z6nqcxjhgu)im!j^RyeqH`7bn?`