Merge branch 'master' into refactor/reduce-action-at-distance

Author: pradyunsg

Diff for: .gitattributes

@@ -0,0 +1,2 @@
+# Patches must have Unix-style line endings, even on Windows
+tasks/vendoring/patches/* eol=lf

Diff for: NEWS.rst

@@ -7,6 +7,22 @@
 
 .. towncrier release notes start
 
+
+9.0.3 (2018-03-21)
+==================
+
+- Fix an error where the vendored requests was not correctly containing itself
+  to only the internal vendored prefix.
+- Restore compatibility with 2.6.
+
+
+9.0.2 (2018-03-16)
+==================
+
+- Fallback to using SecureTransport on macOS when the linked OpenSSL is too old
+  to support TLSv1.2.
+
+
 9.0.1 (2016-11-06)
 ==================
 

Diff for: docs/reference/pip.rst

@@ -72,14 +72,16 @@ Build System Interface
 ======================
 
 Pip builds packages by invoking the build system. Presently, the only supported
-build system is ``setuptools``, but future developments to the Python packaging
-infrastructure are expected to include support for other build systems.  As
-well as package building, the build system is also invoked to install packages
-direct from source.
+build system is ``setuptools``, but in the future, pip will support `PEP517`_
+which allows projects to specify an alternative build system in a
+``pyproject.toml`` file.  As well as package building, the build system is also
+invoked to install packages direct from source.  This is handled by invoking
+the build system to build a wheel, and then installing from that wheel.  The
+built wheel is cached locally by pip to avoid repeated identical builds.
 
-The interface to the build system is via the ``setup.py`` command line script -
-all build actions are defined in terms of the specific ``setup.py`` command
-line that will be run to invoke the required action.
+The current interface to the build system is via the ``setup.py`` command line
+script - all build actions are defined in terms of the specific ``setup.py``
+command line that will be run to invoke the required action.
 
 Setuptools Injection
 ~~~~~~~~~~~~~~~~~~~~
@@ -111,6 +113,54 @@ unexpected byte sequences to Python-style hexadecimal escape sequences
 (``"\x80\xff"``, etc). However, it is still possible for output to be displayed
 using an incorrect encoding (mojibake).
 
+PEP 518 Support
+~~~~~~~~~~~~~~~
+
+Pip supports projects declaring dependencies that are required at install time
+using a ``pyproject.toml`` file, in the form described in `PEP518`_. When
+building a project, pip will install the required dependencies locally, and
+make them available to the build process.
+
+As noted in the PEP, the minimum requirements for pip to be able to build a
+project are::
+
+    [build-system]
+    # Minimum requirements for the build system to execute.
+    requires = ["setuptools", "wheel"]
+
+``setuptools`` and ``wheel`` **must** be included in any ``pyproject.toml``
+provided by a project - pip will assume these as a default, but will not add
+them to an explicitly supplied list in a project supplied ``pyproject.toml``
+file. Once `PEP517`_ support is added, this restriction will be lifted and
+alternative build tools will be allowed.
+
+When making build requirements available, pip does so in an *isolated
+environment*. That is, pip does not install those requirements into the user's
+``site-packages``, but rather installs them in a temporary directory which it
+adds to the user's ``sys.path`` for the duration of the build. This ensures
+that build requirements are handled independently of the user's runtime
+environment. For example, a project that needs a recent version of setuptools
+to build can still be installed, even if the user has an older version
+installed (and without silently replacing that version).
+
+In certain cases, projects (or redistributors) may have workflows that
+explicitly manage the build environment. For such workflows, build isolation
+can be problematic. If this is the case, pip provides a
+``--no-build-isolation`` flag to disable build isolation. Users supplying this
+flag are responsible for ensuring the build environment is managed
+appropriately.
+
+The current implementation of `PEP518`_ in pip requires that any dependencies
+specified in ``pyproject.toml`` are available as wheels. This is a technical
+limitation of the implementation - dependencies only available as source would
+require a build step of their own, which would recursively invoke the `PEP518`_
+dependency installation process. The potentially unbounded recursion involved
+was not considered acceptable, and so installation of build dependencies from
+source has been disabled until a safe resolution of this issue has been found.
+
+.. _PEP517: http://www.python.org/dev/peps/pep-0517/
+.. _PEP518: http://www.python.org/dev/peps/pep-0518/
+
 Future Developments
 ~~~~~~~~~~~~~~~~~~~
 

Diff for: news/36f6f515-fa9e-4d45-84fd-740a5c09752e.trivial

@@ -0,0 +1 @@
+Keep install options in requirements.txt from leaking.

Diff for: news/3763.bugfix

@@ -0,0 +1 @@
+Add auto completion of short options.

Diff for: news/4454.bugfix

@@ -0,0 +1 @@
+Terminal size is now correctly inferred when using Python 3 on Windows.

Diff for: news/4954.feature

@@ -0,0 +1 @@
+Add a ``--no-user`` option and use it when installing build dependencies.

Diff for: news/4966.bugfix

@@ -105,9 +105,11 @@ def autocomplete():
                 sys.exit(1)
 
         subcommand = commands_dict[subcommand_name]()
-        options += [(opt.get_opt_string(), opt.nargs)
-                    for opt in subcommand.parser.option_list_all
-                    if opt.help != optparse.SUPPRESS_HELP]
+
+        for opt in subcommand.parser.option_list_all:
+            if opt.help != optparse.SUPPRESS_HELP:
+                for opt_str in opt._long_opts + opt._short_opts:
+                    options.append((opt_str, opt.nargs))
 
         # filter out previously specified options from available options
         prev_opts = [x.split('=')[0] for x in cwords[1:cword - 1]]
@@ -117,7 +119,7 @@ def autocomplete():
         for option in options:
             opt_label = option[0]
             # append '=' to options which require args
-            if option[1]:
+            if option[1] and option[0][:2] == "--":
                 opt_label += '='
             print(opt_label)
     else:
@@ -127,8 +129,9 @@ def autocomplete():
             opts.append(parser.option_list)
             opts = (o for it in opts for o in it)
 
-            subcommands += [i.get_opt_string() for i in opts
-                            if i.help != optparse.SUPPRESS_HELP]
+            for opt in opts:
+                if opt.help != optparse.SUPPRESS_HELP:
+                    subcommands += opt._long_opts + opt._short_opts
 
         print(' '.join([x for x in subcommands if x.startswith(current)]))
     sys.exit(1)

Diff for: news/5085.bugfix

@@ -9,8 +9,8 @@
 
 from pip._vendor.six import string_types
 
+from pip._internal.compat import get_terminal_size
 from pip._internal.configuration import Configuration, ConfigurationError
-from pip._internal.utils.misc import get_terminal_size
 
 logger = logging.getLogger(__name__)
 

Diff for: src/pip/_internal/__init__.py

@@ -24,6 +24,7 @@ def __enter__(self):
 
         self.save_path = os.environ.get('PATH', None)
         self.save_pythonpath = os.environ.get('PYTHONPATH', None)
+        self.save_nousersite = os.environ.get('PYTHONNOUSERSITE', None)
 
         install_scheme = 'nt' if (os.name == 'nt') else 'posix_prefix'
         install_dirs = get_paths(install_scheme, vars={
@@ -48,20 +49,23 @@ def __enter__(self):
         else:
             os.environ['PYTHONPATH'] = lib_dirs
 
+        os.environ['PYTHONNOUSERSITE'] = '1'
+
         return self.path
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         if not self._no_clean:
             self._temp_dir.cleanup()
-        if self.save_path is None:
-            os.environ.pop('PATH', None)
-        else:
-            os.environ['PATH'] = self.save_path
 
-        if self.save_pythonpath is None:
-            os.environ.pop('PYTHONPATH', None)
-        else:
-            os.environ['PYTHONPATH'] = self.save_pythonpath
+        def restore_var(varname, old_value):
+            if old_value is None:
+                os.environ.pop(varname, None)
+            else:
+                os.environ[varname] = old_value
+
+        restore_var('PATH', self.save_path)
+        restore_var('PYTHONPATH', self.save_pythonpath)
+        restore_var('PYTHONNOUSERSITE', self.save_nousersite)
 
     def cleanup(self):
         self._temp_dir.cleanup()

Diff for: src/pip/_internal/baseparser.py

@@ -429,7 +429,7 @@ def only_binary():
     dest='ignore_dependencies',
     action='store_true',
     default=False,
-    help="Don't install package dependencies)."
+    help="Don't install package dependencies.",
 )  # type: Any
 
 build_dir = partial(
@@ -451,6 +451,17 @@ def only_binary():
     help='Ignore the Requires-Python information.'
 )  # type: Any
 
+no_build_isolation = partial(
+    Option,
+    '--no-build-isolation',
+    dest='build_isolation',
+    action='store_false',
+    default=True,
+    help='Disable isolation when building a modern source distribution. '
+         'Build dependencies specified by PEP 518 must be already installed '
+         'if this option is used.'
+)  # type: Any
+
 install_options = partial(
     Option,
     '--install-option',

Diff for: src/pip/_internal/build_env.py

@@ -13,19 +13,7 @@
 
 
 class ConfigurationCommand(Command):
-    """Manage local and global configuration."""
-    name = 'config'
-    usage = """
-        %prog [<file-option>] list
-        %prog [<file-option>] [--editor <editor-path>] edit
-
-        %prog [<file-option>] get name
-        %prog [<file-option>] set name value
-        %prog [<file-option>] unset name
-    """
-
-    summary = """
-        Manage local and global configuration.
+    """Manage local and global configuration.
 
         Subcommands:
 
@@ -41,6 +29,18 @@ class ConfigurationCommand(Command):
         default.
     """
 
+    name = 'config'
+    usage = """
+        %prog [<file-option>] list
+        %prog [<file-option>] [--editor <editor-path>] edit
+
+        %prog [<file-option>] get name
+        %prog [<file-option>] set name value
+        %prog [<file-option>] unset name
+    """
+
+    summary = "Manage local and global configuration."
+
     def __init__(self, *args, **kwargs):
         super(ConfigurationCommand, self).__init__(*args, **kwargs)
 

Diff for: src/pip/_internal/cmdoptions.py

@@ -57,6 +57,7 @@ def __init__(self, *args, **kw):
         cmd_opts.add_option(cmdoptions.no_clean())
         cmd_opts.add_option(cmdoptions.require_hashes())
         cmd_opts.add_option(cmdoptions.progress_bar())
+        cmd_opts.add_option(cmdoptions.no_build_isolation())
 
         cmd_opts.add_option(
             '-d', '--dest', '--destination-dir', '--destination-directory',
@@ -201,6 +202,7 @@ def run(self, options, args):
                     download_dir=options.download_dir,
                     wheel_download_dir=None,
                     progress_bar=options.progress_bar,
+                    build_isolation=options.build_isolation,
                 )
 
                 resolver = Resolver(

Diff for: src/pip/_internal/commands/configuration.py

@@ -5,6 +5,7 @@
 import operator
 import os
 import shutil
+from optparse import SUPPRESS_HELP
 
 from pip._internal import cmdoptions
 from pip._internal.basecommand import RequirementCommand
@@ -14,7 +15,7 @@
 )
 from pip._internal.locations import distutils_scheme, virtualenv_no_global
 from pip._internal.operations.prepare import RequirementPreparer
-from pip._internal.req import RequirementSet
+from pip._internal.req import RequirementSet, install_given_reqs
 from pip._internal.resolve import Resolver
 from pip._internal.status_codes import ERROR
 from pip._internal.utils.filesystem import check_path_owner
@@ -83,6 +84,11 @@ def __init__(self, *args, **kw):
                  "platform. Typically ~/.local/, or %APPDATA%\\Python on "
                  "Windows. (See the Python documentation for site.USER_BASE "
                  "for full details.)")
+        cmd_opts.add_option(
+            '--no-user',
+            dest='use_user_site',
+            action='store_false',
+            help=SUPPRESS_HELP)
         cmd_opts.add_option(
             '--root',
             dest='root_path',
@@ -139,6 +145,7 @@ def __init__(self, *args, **kw):
             help='Ignore the installed packages (reinstalling instead).')
 
         cmd_opts.add_option(cmdoptions.ignore_requires_python())
+        cmd_opts.add_option(cmdoptions.no_build_isolation())
 
         cmd_opts.add_option(cmdoptions.install_options())
         cmd_opts.add_option(cmdoptions.global_options())
@@ -257,6 +264,7 @@ def run(self, options, args):
                         download_dir=None,
                         wheel_download_dir=None,
                         progress_bar=options.progress_bar,
+                        build_isolation=options.build_isolation,
                     )
 
                     resolver = Resolver(
@@ -289,7 +297,11 @@ def run(self, options, args):
                             session=session, autobuilding=True
                         )
 
-                    installed = requirement_set.install(
+                    to_install = resolver.get_installation_order(
+                        requirement_set
+                    )
+                    installed = install_given_reqs(
+                        to_install,
                         install_options,
                         global_options,
                         root=options.root_path,

Diff for: src/pip/_internal/commands/download.py

@@ -12,12 +12,12 @@
 from pip._vendor.six.moves import xmlrpc_client  # type: ignore
 
 from pip._internal.basecommand import SUCCESS, Command
+from pip._internal.compat import get_terminal_size
 from pip._internal.download import PipXmlrpcTransport
 from pip._internal.exceptions import CommandError
 from pip._internal.models import PyPI
 from pip._internal.status_codes import NO_MATCHES_FOUND
 from pip._internal.utils.logging import indent_log
-from pip._internal.utils.misc import get_terminal_size
 
 logger = logging.getLogger(__name__)
 

Diff for: src/pip/_internal/commands/install.py

@@ -2,7 +2,7 @@
 
 import logging
 import os
-from email.parser import FeedParser
+from email.parser import FeedParser  # type: ignore
 
 from pip._vendor import pkg_resources
 from pip._vendor.packaging.utils import canonicalize_name

Diff for: src/pip/_internal/commands/search.py

@@ -63,7 +63,9 @@ def __init__(self, *args, **kw):
             dest='build_options',
             metavar='options',
             action='append',
-            help="Extra arguments to be supplied to 'setup.py bdist_wheel'.")
+            help="Extra arguments to be supplied to 'setup.py bdist_wheel'.",
+        )
+        cmd_opts.add_option(cmdoptions.no_build_isolation())
         cmd_opts.add_option(cmdoptions.constraints())
         cmd_opts.add_option(cmdoptions.editable())
         cmd_opts.add_option(cmdoptions.requirements())
@@ -158,6 +160,7 @@ def run(self, options, args):
                         download_dir=None,
                         wheel_download_dir=options.wheel_dir,
                         progress_bar=options.progress_bar,
+                        build_isolation=options.build_isolation,
                     )
 
                     resolver = Resolver(