diff --git a/CHANGELOG.md b/CHANGELOG.md index a32fb0a58..345c1119e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ # Master +- Update pip, setuptools and wheel and refactor the way they are installed (#1007) # 173 (2020-07-21) diff --git a/NOTICE b/NOTICE index 8aa7a74ad..337f265dd 100644 --- a/NOTICE +++ b/NOTICE @@ -86,31 +86,6 @@ Each version is given a distinguishing version number. If the Library as you rec If the Library as you received it specifies that a proxy can decide whether future versions of the GNU Lesser General Public License shall apply, that proxy's public statement of acceptance of any version is permanent authorization for you to choose that version for the Library. -get-pip.py license ------------------- - -Copyright (c) 2008-2016 The pip developers (see AUTHORS.txt file) - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - - pip-pop license --------------- diff --git a/bin/compile b/bin/compile index 89c405121..05ceac3b5 100755 --- a/bin/compile +++ b/bin/compile @@ -63,28 +63,8 @@ PYPY36="pypy3.6" # Which stack is used (for binary downloading), if none is provided (e.g. outside of Heroku)? DEFAULT_PYTHON_STACK="cedar-14" -# If pip doesn't match this version (the version we install), run the installer. -PIP_UPDATE="20.0.2" -for file in "$BUILD_DIR/runtime.txt" "$CACHE_DIR/.heroku/python-version" ; do - [ -f "$file" ] || continue - - version=$(tr -d '[:space:]' < "$file") - - case "$version" in "$PY34"*) - # Python 3.4 support was dropped in pip >= 19.2. - PIP_UPDATE="19.1.1" - break - ;; - esac -done - -if [[ -f "$BUILD_DIR/Pipfile" ]]; then - # Do not force pipenv users to re-install pipenv locally. - PIP_UPDATE="9.0.2" -fi - -export DEFAULT_PYTHON_STACK PIP_UPDATE +export DEFAULT_PYTHON_STACK export PY37 PY36 PY35 PY27 PY34 # Common Problem Warnings: diff --git a/bin/steps/README.MD b/bin/steps/README.MD index 3c23cf78d..6bf9885f7 100644 --- a/bin/steps/README.MD +++ b/bin/steps/README.MD @@ -2,13 +2,6 @@ TODO: Add context on Python install steps, such as why symlinking vs copying -## Installing the Pip tool - -The Python Buildpack uses a tool called `get-pip` to install the pip tool. This -is done in the `python` script. - -This is in part because Python historically did not come with pip by default. - ## Installing Python packages using Pip ### Convention: Use `python` process to invoke Pip diff --git a/bin/steps/pipenv b/bin/steps/pipenv index c799c7d2e..f52f07d21 100755 --- a/bin/steps/pipenv +++ b/bin/steps/pipenv @@ -53,12 +53,16 @@ if [ ! "$SKIP_PIPENV_INSTALL" ]; then fi export PIPENV_VERSION="2018.5.18" + # TODO: Stop downgrading pip once pipenv is updated to a modern release. + PIP_VERSION='9.0.2' + + puts-step "Installing pipenv ${PIPENV_VERSION} and downgrading pip to ${PIP_VERSION} for compatibility" # Install pipenv. # Due to weird old pip behavior and pipenv behavior, pipenv upgrades pip # to latest if only --upgrade is specified. Specify upgrade strategy to # avoid this eager behavior. - /app/.heroku/python/bin/pip install pipenv==$PIPENV_VERSION --upgrade --upgrade-strategy only-if-needed &> /dev/null + /app/.heroku/python/bin/pip install "pipenv==${PIPENV_VERSION}" "pip==${PIP_VERSION}" --upgrade --upgrade-strategy only-if-needed &> /dev/null # Install the test dependencies, for CI. if [ "$INSTALL_TEST" ]; then diff --git a/bin/steps/python b/bin/steps/python index 068e147cc..1e997d5d2 100755 --- a/bin/steps/python +++ b/bin/steps/python @@ -131,34 +131,48 @@ if [ ! "$SKIP_INSTALL" ]; then # Record for future reference. echo "$PYTHON_VERSION" > .heroku/python-version echo "$STACK" > .heroku/python-stack - FRESH_PYTHON=true hash -r fi -# Heroku uses the get-pip utility maintained by the Python community to vendor Pip. -# https://github.com/pypa/get-pip -GETPIP_URL="https://lang-python.s3.amazonaws.com/etc/get-pip.py" -GETPIP_PY="${TMPDIR:-/tmp}/get-pip.py" +set -e -if ! curl -s "${GETPIP_URL}" -o "$GETPIP_PY" &> /dev/null; then - mcount "failure.python.get-pip" - echo "Failed to pull down get-pip" - exit 1 +PIP_VERSION='20.1.1' +# Must use setuptools <47.2.0 until we fix: https://github.com/heroku/heroku-buildpack-python/issues/1006 +SETUPTOOLS_VERSION='47.1.1' +WHEEL_VERSION='0.34.2' + +if [[ "${PYTHON_VERSION}" == ${PY34}* ]]; then + # Python 3.4 support was dropped in pip 19.2, setuptools 44.0.0 and wheel 0.34.0. + PIP_VERSION='19.1.1' + SETUPTOOLS_VERSION='43.0.0' + WHEEL_VERSION='0.33.6' +elif [[ "${PYTHON_VERSION}" == ${PY27}* || "${PYTHON_VERSION}" == ${PYPY27}* ]]; then + # Python 2.7 support was dropped in setuptools 45.0.0. + SETUPTOOLS_VERSION='44.1.1' fi -# If a new Python has been installed or Pip isn't up to date: -if [ "$FRESH_PYTHON" ] || [[ ! $(pip --version) == *$PIP_UPDATE* ]]; then +puts-step "Installing pip ${PIP_VERSION}, setuptools ${SETUPTOOLS_VERSION} and wheel ${WHEEL_VERSION}" - puts-step "Installing pip" +# We don't use get-pip.py, since: +# - it uses `--force-reinstall`, which is unnecessary here and slows down repeat builds +# - it means downloading pip twice (once embedded in get-pip.py, and again during +# the install, since get-pip.py can't install the embedded version directly) +# - we would still have to manage several versions of get-pip.py, to support older Pythons. +# Instead, we use the pip wheel to install itself, using the method described here: +# https://github.com/pypa/pip/issues/2351#issuecomment-69994524 - # Remove old installations. - rm -fr /app/.heroku/python/lib/python*/site-packages/pip-* - rm -fr /app/.heroku/python/lib/python*/site-packages/setuptools-* +PIP_WHEEL_FILENAME="pip-${PIP_VERSION}-py2.py3-none-any.whl" +PIP_WHEEL_URL="https://lang-python.s3.amazonaws.com/common/${PIP_WHEEL_FILENAME}" +PIP_WHEEL="${TMPDIR:-/tmp}/${PIP_WHEEL_FILENAME}" - /app/.heroku/python/bin/python "$GETPIP_PY" pip=="$PIP_UPDATE" &> /dev/null - /app/.heroku/python/bin/pip install "$ROOT_DIR/vendor/setuptools-39.0.1-py2.py3-none-any.whl" &> /dev/null +if ! curl -sSf "${PIP_WHEEL_URL}" -o "$PIP_WHEEL"; then + mcount "failure.python.download-pip" + puts-warn "Failed to download pip" + exit 1 fi -set -e +/app/.heroku/python/bin/python "${PIP_WHEEL}/pip" install -q --disable-pip-version-check --no-cache \ + "${PIP_WHEEL}" "setuptools==${SETUPTOOLS_VERSION}" "wheel==${WHEEL_VERSION}" + hash -r diff --git a/vendor/setuptools-39.0.1-py2.py3-none-any.whl b/vendor/setuptools-39.0.1-py2.py3-none-any.whl deleted file mode 100644 index edc3ca2d8..000000000 Binary files a/vendor/setuptools-39.0.1-py2.py3-none-any.whl and /dev/null differ