From f710fdbf12761db28aaca779c9c4d8ac1c1a30c7 Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Sun, 14 Apr 2024 21:21:23 +0100 Subject: [PATCH] Drop run-time version constraints for modern Pip Modern Pip (v9.0.0+) supports `Requires-Python` and so automatically takes the current Python version into account, when determining the latest version of packages that can be installed. As such, for modern Pip we don't need to specify version range constraints for the pip/setuptools/wheel versions passed to the `pip install` at `get-pip.py` run-time. This is the first step towards being able to remove `SCRIPT_CONSTRAINTS` and rely purely on `Requires-Python` metadata, per: https://github.com/pypa/get-pip/issues/88#issuecomment-798966926 I've intentionally left the "figure out what Pip version to embed in the template" part of template generation alone for now to keep the PR smaller, and have only changed the run-time `pip install` parts. I had to add a new `pre-9.py` template file (created as a copy of `pre-10.py`), since it's only pip<9 that needs the various version constraint references in the template (and otherwise anything else that used `pre-10.py`, such as Python 2.6, would have had redundant version constraints). The new `pre-9.py` template is only used for Python 3.2. --- public/2.6/get-pip.py | 6 +- public/2.7/get-pip.py | 4 +- public/3.3/get-pip.py | 4 +- public/3.4/get-pip.py | 2 +- public/3.5/get-pip.py | 2 +- public/3.6/get-pip.py | 2 +- scripts/generate.py | 27 ++---- templates/default.py | 8 +- templates/pre-10.py | 8 +- templates/pre-18.1.py | 8 +- templates/pre-19.3.py | 8 +- templates/pre-21.0.py | 8 +- templates/pre-9.py | 207 ++++++++++++++++++++++++++++++++++++++++++ 13 files changed, 245 insertions(+), 49 deletions(-) create mode 100644 templates/pre-9.py diff --git a/public/2.6/get-pip.py b/public/2.6/get-pip.py index b601fccb..b83a2ee6 100644 --- a/public/2.6/get-pip.py +++ b/public/2.6/get-pip.py @@ -147,11 +147,11 @@ def parse_args(self, args): # Add any implicit installations to the end of our args if implicit_pip: - args += ["pip<10"] + args += ["pip"] if implicit_setuptools: - args += ["setuptools<37"] + args += ["setuptools"] if implicit_wheel: - args += ["wheel<0.30"] + args += ["wheel"] delete_tmpdir = False try: diff --git a/public/2.7/get-pip.py b/public/2.7/get-pip.py index 3ef3d705..c2064943 100644 --- a/public/2.7/get-pip.py +++ b/public/2.7/get-pip.py @@ -149,9 +149,9 @@ def cert_parse_args(self, args): # Add any implicit installations to the end of our args if implicit_pip: - args += ["pip<21.0"] + args += ["pip"] if implicit_setuptools: - args += ["setuptools<45"] + args += ["setuptools"] if implicit_wheel: args += ["wheel"] diff --git a/public/3.3/get-pip.py b/public/3.3/get-pip.py index 77be6b0f..c5fb4a97 100644 --- a/public/3.3/get-pip.py +++ b/public/3.3/get-pip.py @@ -147,11 +147,11 @@ def parse_args(self, args): # Add any implicit installations to the end of our args if implicit_pip: - args += ["pip<18"] + args += ["pip"] if implicit_setuptools: args += ["setuptools"] if implicit_wheel: - args += ["wheel<0.30"] + args += ["wheel"] # Add our default arguments args = ["install", "--upgrade", "--force-reinstall"] + args diff --git a/public/3.4/get-pip.py b/public/3.4/get-pip.py index a52adebc..759d72e6 100644 --- a/public/3.4/get-pip.py +++ b/public/3.4/get-pip.py @@ -147,7 +147,7 @@ def parse_args(self, args): # Add any implicit installations to the end of our args if implicit_pip: - args += ["pip<19.2"] + args += ["pip"] if implicit_setuptools: args += ["setuptools"] if implicit_wheel: diff --git a/public/3.5/get-pip.py b/public/3.5/get-pip.py index 0761be8a..c2064943 100644 --- a/public/3.5/get-pip.py +++ b/public/3.5/get-pip.py @@ -149,7 +149,7 @@ def cert_parse_args(self, args): # Add any implicit installations to the end of our args if implicit_pip: - args += ["pip<21.0"] + args += ["pip"] if implicit_setuptools: args += ["setuptools"] if implicit_wheel: diff --git a/public/3.6/get-pip.py b/public/3.6/get-pip.py index 98456767..3f1f233f 100644 --- a/public/3.6/get-pip.py +++ b/public/3.6/get-pip.py @@ -69,7 +69,7 @@ def determine_pip_install_arguments(): pre_parser.add_argument("--no-wheel", action="store_true") pre, args = pre_parser.parse_known_args() - args.append("pip<22.0") + args.append("pip") if include_setuptools(pre): args.append("setuptools") diff --git a/scripts/generate.py b/scripts/generate.py index c03c10cd..27b0ef8d 100644 --- a/scripts/generate.py +++ b/scripts/generate.py @@ -23,43 +23,31 @@ SCRIPT_CONSTRAINTS = { "default": { "pip": "", - "setuptools": "", - "wheel": "", }, "2.6": { "pip": "<10", - "setuptools": "<37", - "wheel": "<0.30", }, "2.7": { "pip": "<21.0", - "setuptools": "<45", - "wheel": "", }, "3.2": { + # Pip older than v9.0.0 does not support Requires-Python so we have to manually + # constrain the pip, setuptools and wheel versions that are installed at runtime. "pip": "<8", "setuptools": "<30", "wheel": "<0.30", }, "3.3": { "pip": "<18", - "setuptools": "", - "wheel": "<0.30", }, "3.4": { "pip": "<19.2", - "setuptools": "", - "wheel": "", }, "3.5": { "pip": "<21.0", - "setuptools": "", - "wheel": "", }, "3.6": { "pip": "<22.0", - "setuptools": "", - "wheel": "", }, } @@ -243,7 +231,7 @@ def detect_newline(f: TextIO) -> str: def generate_one(variant, mapping, *, console, pip_versions): - # Determing the correct wheel to download + # Determine the correct wheel to download pip_version = determine_latest(pip_versions.keys(), constraint=mapping["pip"]) wheel_url, wheel_hash = pip_versions[pip_version] @@ -259,10 +247,11 @@ def generate_one(variant, mapping, *, console, pip_versions): newline = detect_newline(f) rendered_template = f.read().format( zipfile=encoded_wheel, - installed_version=pip_version, - pip_version=mapping["pip"], - setuptools_version=mapping["setuptools"], - wheel_version=mapping["wheel"], + bundled_pip_version=pip_version, + # These constraints are only set for pip versions that don't support Requires-Python. + pip_version_constraint=mapping.get("pip"), + setuptools_version_constraint=mapping.get("setuptools"), + wheel_version_constraint=mapping.get("wheel"), minimum_supported_version=mapping["minimum_supported_version"], ) # Write the script to the correct location diff --git a/templates/default.py b/templates/default.py index 46e96beb..d244ac39 100644 --- a/templates/default.py +++ b/templates/default.py @@ -5,7 +5,7 @@ # You may be wondering what this giant blob of binary data here is, you might # even be worried that we're up to something nefarious (good for you for being # paranoid!). This is a base85 encoding of a zip file, this zip file contains -# an entire copy of pip (version {installed_version}). +# an entire copy of pip (version {bundled_pip_version}). # # Pip is a thing that installs packages, pip itself is a package that someone # might want to install, especially if they're looking to run this get-pip.py @@ -69,13 +69,13 @@ def determine_pip_install_arguments(): pre_parser.add_argument("--no-wheel", action="store_true") pre, args = pre_parser.parse_known_args() - args.append("pip{pip_version}") + args.append("pip") if include_setuptools(pre): - args.append("setuptools{setuptools_version}") + args.append("setuptools") if include_wheel(pre): - args.append("wheel{wheel_version}") + args.append("wheel") return ["install", "--upgrade", "--force-reinstall"] + args diff --git a/templates/pre-10.py b/templates/pre-10.py index acbea9d8..280a1d8c 100644 --- a/templates/pre-10.py +++ b/templates/pre-10.py @@ -5,7 +5,7 @@ # You may be wondering what this giant blob of binary data here is, you might # even be worried that we're up to something nefarious (good for you for being # paranoid!). This is a base85 encoding of a zip file, this zip file contains -# an entire copy of pip (version {installed_version}). +# an entire copy of pip (version {bundled_pip_version}). # # Pip is a thing that installs packages, pip itself is a package that someone # might want to install, especially if they're looking to run this get-pip.py @@ -147,11 +147,11 @@ def parse_args(self, args): # Add any implicit installations to the end of our args if implicit_pip: - args += ["pip{pip_version}"] + args += ["pip"] if implicit_setuptools: - args += ["setuptools{setuptools_version}"] + args += ["setuptools"] if implicit_wheel: - args += ["wheel{wheel_version}"] + args += ["wheel"] delete_tmpdir = False try: diff --git a/templates/pre-18.1.py b/templates/pre-18.1.py index dd10d083..dfe760f8 100644 --- a/templates/pre-18.1.py +++ b/templates/pre-18.1.py @@ -5,7 +5,7 @@ # You may be wondering what this giant blob of binary data here is, you might # even be worried that we're up to something nefarious (good for you for being # paranoid!). This is a base85 encoding of a zip file, this zip file contains -# an entire copy of pip (version {installed_version}). +# an entire copy of pip (version {bundled_pip_version}). # # Pip is a thing that installs packages, pip itself is a package that someone # might want to install, especially if they're looking to run this get-pip.py @@ -147,11 +147,11 @@ def parse_args(self, args): # Add any implicit installations to the end of our args if implicit_pip: - args += ["pip{pip_version}"] + args += ["pip"] if implicit_setuptools: - args += ["setuptools{setuptools_version}"] + args += ["setuptools"] if implicit_wheel: - args += ["wheel{wheel_version}"] + args += ["wheel"] # Add our default arguments args = ["install", "--upgrade", "--force-reinstall"] + args diff --git a/templates/pre-19.3.py b/templates/pre-19.3.py index 38fb75c4..1d198f4c 100644 --- a/templates/pre-19.3.py +++ b/templates/pre-19.3.py @@ -5,7 +5,7 @@ # You may be wondering what this giant blob of binary data here is, you might # even be worried that we're up to something nefarious (good for you for being # paranoid!). This is a base85 encoding of a zip file, this zip file contains -# an entire copy of pip (version {installed_version}). +# an entire copy of pip (version {bundled_pip_version}). # # Pip is a thing that installs packages, pip itself is a package that someone # might want to install, especially if they're looking to run this get-pip.py @@ -147,11 +147,11 @@ def parse_args(self, args): # Add any implicit installations to the end of our args if implicit_pip: - args += ["pip{pip_version}"] + args += ["pip"] if implicit_setuptools: - args += ["setuptools{setuptools_version}"] + args += ["setuptools"] if implicit_wheel: - args += ["wheel{wheel_version}"] + args += ["wheel"] # Add our default arguments args = ["install", "--upgrade", "--force-reinstall"] + args diff --git a/templates/pre-21.0.py b/templates/pre-21.0.py index 038c6249..6006bfdf 100644 --- a/templates/pre-21.0.py +++ b/templates/pre-21.0.py @@ -5,7 +5,7 @@ # You may be wondering what this giant blob of binary data here is, you might # even be worried that we're up to something nefarious (good for you for being # paranoid!). This is a base85 encoding of a zip file, this zip file contains -# an entire copy of pip (version {installed_version}). +# an entire copy of pip (version {bundled_pip_version}). # # Pip is a thing that installs packages, pip itself is a package that someone # might want to install, especially if they're looking to run this get-pip.py @@ -149,11 +149,11 @@ def cert_parse_args(self, args): # Add any implicit installations to the end of our args if implicit_pip: - args += ["pip{pip_version}"] + args += ["pip"] if implicit_setuptools: - args += ["setuptools{setuptools_version}"] + args += ["setuptools"] if implicit_wheel: - args += ["wheel{wheel_version}"] + args += ["wheel"] # Add our default arguments args = ["install", "--upgrade", "--force-reinstall"] + args diff --git a/templates/pre-9.py b/templates/pre-9.py new file mode 100644 index 00000000..232f48c6 --- /dev/null +++ b/templates/pre-9.py @@ -0,0 +1,207 @@ +#!/usr/bin/env python +# +# Hi There! +# +# You may be wondering what this giant blob of binary data here is, you might +# even be worried that we're up to something nefarious (good for you for being +# paranoid!). This is a base85 encoding of a zip file, this zip file contains +# an entire copy of pip (version {bundled_pip_version}). +# +# Pip is a thing that installs packages, pip itself is a package that someone +# might want to install, especially if they're looking to run this get-pip.py +# script. Pip has a lot of code to deal with the security of installing +# packages, various edge cases on various platforms, and other such sort of +# "tribal knowledge" that has been encoded in its code base. Because of this +# we basically include an entire copy of pip inside this blob. We do this +# because the alternatives are attempt to implement a "minipip" that probably +# doesn't do things correctly and has weird edge cases, or compress pip itself +# down into a single file. +# +# If you're wondering how this is created, it is generated using +# `scripts/generate.py` in https://github.com/pypa/get-pip. + +import os.path +import pkgutil +import shutil +import sys +import struct +import tempfile + +# Useful for very coarse version differentiation. +PY2 = sys.version_info[0] == 2 +PY3 = sys.version_info[0] == 3 + +if PY3: + iterbytes = iter +else: + def iterbytes(buf): + return (ord(byte) for byte in buf) + +try: + from base64 import b85decode +except ImportError: + _b85alphabet = (b"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ" + b"abcdefghijklmnopqrstuvwxyz!#$%&()*+-;<=>?@^_`{{|}}~") + + def b85decode(b): + _b85dec = [None] * 256 + for i, c in enumerate(iterbytes(_b85alphabet)): + _b85dec[c] = i + + padding = (-len(b)) % 5 + b = b + b'~' * padding + out = [] + packI = struct.Struct('!I').pack + for i in range(0, len(b), 5): + chunk = b[i:i + 5] + acc = 0 + try: + for c in iterbytes(chunk): + acc = acc * 85 + _b85dec[c] + except TypeError: + for j, c in enumerate(iterbytes(chunk)): + if _b85dec[c] is None: + raise ValueError( + 'bad base85 character at position %d' % (i + j) + ) + raise + try: + out.append(packI(acc)) + except struct.error: + raise ValueError('base85 overflow in hunk starting at byte %d' + % i) + + result = b''.join(out) + if padding: + result = result[:-padding] + return result + + +def bootstrap(tmpdir=None): + # Import pip so we can use it to install pip and maybe setuptools too + import pip + from pip.commands.install import InstallCommand + from pip.req import InstallRequirement + + # Wrapper to provide default certificate with the lowest priority + class CertInstallCommand(InstallCommand): + def parse_args(self, args): + # If cert isn't specified in config or environment, we provide our + # own certificate through defaults. + # This allows user to specify custom cert anywhere one likes: + # config, environment variable or argv. + if not self.parser.get_default_values().cert: + self.parser.defaults["cert"] = cert_path # calculated below + return super(CertInstallCommand, self).parse_args(args) + + pip.commands_dict["install"] = CertInstallCommand + + implicit_pip = True + implicit_setuptools = True + implicit_wheel = True + + # Check if the user has requested us not to install setuptools + if "--no-setuptools" in sys.argv or os.environ.get("PIP_NO_SETUPTOOLS"): + args = [x for x in sys.argv[1:] if x != "--no-setuptools"] + implicit_setuptools = False + else: + args = sys.argv[1:] + + # Check if the user has requested us not to install wheel + if "--no-wheel" in args or os.environ.get("PIP_NO_WHEEL"): + args = [x for x in args if x != "--no-wheel"] + implicit_wheel = False + + # We only want to implicitly install setuptools and wheel if they don't + # already exist on the target platform. + if implicit_setuptools: + try: + import setuptools # noqa + implicit_setuptools = False + except ImportError: + pass + if implicit_wheel: + try: + import wheel # noqa + implicit_wheel = False + except ImportError: + pass + + # We want to support people passing things like 'pip<8' to get-pip.py which + # will let them install a specific version. However because of the dreaded + # DoubleRequirement error if any of the args look like they might be a + # specific for one of our packages, then we'll turn off the implicit + # install of them. + for arg in args: + try: + req = InstallRequirement.from_line(arg) + except Exception: + continue + + if implicit_pip and req.name == "pip": + implicit_pip = False + elif implicit_setuptools and req.name == "setuptools": + implicit_setuptools = False + elif implicit_wheel and req.name == "wheel": + implicit_wheel = False + + # Add any implicit installations to the end of our args + if implicit_pip: + args += ["pip{pip_version_constraint}"] + if implicit_setuptools: + args += ["setuptools{setuptools_version_constraint}"] + if implicit_wheel: + args += ["wheel{wheel_version_constraint}"] + + delete_tmpdir = False + try: + # Create a temporary directory to act as a working directory if we were + # not given one. + if tmpdir is None: + tmpdir = tempfile.mkdtemp() + delete_tmpdir = True + + # We need to extract the SSL certificates from requests so that they + # can be passed to --cert + cert_path = os.path.join(tmpdir, "cacert.pem") + with open(cert_path, "wb") as cert: + cert.write(pkgutil.get_data("pip._vendor.requests", "cacert.pem")) + + # Execute the included pip and use it to install the latest pip and + # setuptools from PyPI + sys.exit(pip.main(["install", "--upgrade"] + args)) + finally: + # Remove our temporary directory + if delete_tmpdir and tmpdir: + shutil.rmtree(tmpdir, ignore_errors=True) + + +def main(): + tmpdir = None + try: + # Create a temporary working directory + tmpdir = tempfile.mkdtemp() + + # Unpack the zipfile into the temporary directory + pip_zip = os.path.join(tmpdir, "pip.zip") + with open(pip_zip, "wb") as fp: + fp.write(b85decode(DATA.replace(b"\n", b""))) + + # Add the zipfile to sys.path so that we can import it + sys.path.insert(0, pip_zip) + + # Run the bootstrap + bootstrap(tmpdir=tmpdir) + finally: + # Clean up our temporary working directory + if tmpdir: + shutil.rmtree(tmpdir, ignore_errors=True) + + +DATA = b""" +{zipfile} +""" + + +if __name__ == "__main__": + main()