env: make pip/setuptools configurable

This change introduces the following configuration options:

- `virtualenvs.options.no-pip` (Default: `false`)
- `virtualenvs.options.no-wheel` (Default: `false`)
- `virtualenvs.options.no-setuptools` (Default: `false`)

With these options, we allow, the user to control the inclusion of
`pip`, `wheel` and `setuptools` packages in new virtual environments.
The defaults used retain existing behaviour as of `1.1.5`.

In addition the following now holds true.

- projects can define `pip`, `wheel` and `setuptools` as dependencies
- if any of these packages are in the project's dependency tree, poetry
  will manage their versions
- all poetry internal commands will use pip embedded in virtualenv
  irrespective of pip beig available in the environment
- if `poetry run pip` is executed, pip installed in the virtualenv will
  be prioritsed
- `poetry install --remove-untracked` will ignore these packages if
  they are not present in the project's dependency tree

Relates-to: python-poetry#2826
Relates-to: python-poetry#3916

docs/docs/configuration.md

@@ -35,6 +35,9 @@ cache-dir = "/path/to/cache/directory"
 virtualenvs.create = true
 virtualenvs.in-project = null
 virtualenvs.options.always-copy = true
+virtualenvs.options.no-pip = true
+virtualenvs.options.no-setuptools = false
+virtualenvs.options.no-wheel = false
 virtualenvs.options.system-site-packages = false
 virtualenvs.path = "{cache-dir}/virtualenvs"  # /path/to/cache/directory/virtualenvs
 ```
@@ -146,6 +149,25 @@ Defaults to `{cache-dir}/virtualenvs` (`{cache-dir}\virtualenvs` on Windows).
 If set to `true` the `--always-copy` parameter is passed to `virtualenv` on creation of the venv. Thus all needed files are copied into the venv instead of symlinked.
 Defaults to `false`.
 
+### `virtualenvs.options.no-pip`: boolean
+
+If set to `true` the `--no-pip` parameter is passed to `virtualenv` on creation of the venv. Thus all needed files are copied into the venv instead of symlinked.
+Defaults to `false`.
+
+!!!note
+
+    If set to `true`, `poetry run pip` will use pip embedded in poetry's virtualenv dependency.
+
+### `virtualenvs.options.no-setuptools`: boolean
+
+If set to `true` the `--no-setuptools` parameter is passed to `virtualenv` on creation of the venv. Thus all needed files are copied into the venv instead of symlinked.
+Defaults to `false`.
+
+### `virtualenvs.options.no-wheel`: boolean
+
+If set to `true` the `--no-wheel parameter is passed to `virtualenv` on creation of the venv. Thus all needed files are copied into the venv instead of symlinked.
+Defaults to `false`.
+
 ### `virtualenvs.options.system-site-packages`: boolean
 
 Give the virtual environment access to the system site-packages directory.

poetry/config/config.py

@@ -33,7 +33,13 @@ class Config:
             "create": True,
             "in-project": None,
             "path": os.path.join("{cache-dir}", "virtualenvs"),
-            "options": {"always-copy": False, "system-site-packages": False},
+            "options": {
+                "always-copy": False,
+                "system-site-packages": False,
+                "no-pip": False,
+                "no-setuptools": False,
+                "no-wheel": False,
+            },
         },
         "experimental": {"new-installer": True},
         "installer": {"parallel": True},
@@ -139,6 +145,9 @@ def _get_normalizer(self, name: str) -> Callable:
             "virtualenvs.in-project",
             "virtualenvs.options.always-copy",
             "virtualenvs.options.system-site-packages",
+            "virtualenvs.options.no-pip",
+            "virtualenvs.options.no-setuptools",
+            "virtualenvs.options.no-wheel",
             "installer.parallel",
         }:
             return boolean_normalizer

poetry/console/commands/config.py

@@ -72,6 +72,21 @@ def unique_config_values(self) -> Dict[str, Tuple[Any, Any, Any]]:
                 boolean_normalizer,
                 False,
             ),
+            "virtualenvs.options.no-pip": (
+                boolean_validator,
+                boolean_normalizer,
+                False,
+            ),
+            "virtualenvs.options.no-setuptools": (
+                boolean_validator,
+                boolean_normalizer,
+                False,
+            ),
+            "virtualenvs.options.no-wheel": (
+                boolean_validator,
+                boolean_normalizer,
+                False,
+            ),
             "virtualenvs.path": (
                 str,
                 lambda val: str(Path(val)),

poetry/console/commands/debug/resolve.py

@@ -84,7 +84,14 @@ def handle(self) -> Optional[int]:
 
         pool = self.poetry.pool
 
-        solver = Solver(package, pool, Repository(), Repository(), self._io)
+        solver = Solver(
+            package,
+            pool,
+            Repository(),
+            Repository(),
+            self._io,
+            config=self.poetry.config,
+        )
 
         ops = solver.solve()
 
@@ -121,7 +128,9 @@ def handle(self) -> Optional[int]:
 
             pool.add_repository(locked_repository)
 
-            solver = Solver(package, pool, Repository(), Repository(), NullIO())
+            solver = Solver(
+                package, pool, Repository(), Repository(), NullIO(), poetry=self.poetry
+            )
             with solver.use_environment(env):
                 ops = solver.solve()
 

poetry/console/commands/show.py

@@ -90,6 +90,7 @@ def handle(self) -> Optional[int]:
             installed=Repository(),
             locked=locked_repo,
             io=NullIO(),
+            config=self.poetry.config,
         )
         solver.provider.load_deferred(False)
         with solver.use_environment(self.env):

poetry/installation/installer.py

@@ -74,6 +74,7 @@ def __init__(
             installed = self._get_installed()
 
         self._installed_repository = installed
+        self._poetry_config = config
 
     @property
     def executor(self) -> Executor:
@@ -209,6 +210,7 @@ def _do_refresh(self) -> int:
             locked_repository,
             locked_repository,
             self._io,
+            config=self._poetry_config,
         )
 
         ops = solver.solve(use_latest=[])
@@ -247,6 +249,7 @@ def _do_install(self, local_repo: Repository) -> int:
                 locked_repository,
                 self._io,
                 remove_untracked=self._remove_untracked,
+                config=self._poetry_config,
             )
 
             ops = solver.solve(use_latest=self._whitelist)
@@ -316,6 +319,7 @@ def _do_install(self, local_repo: Repository) -> int:
             locked_repository,
             NullIO(),
             remove_untracked=self._remove_untracked,
+            config=self._poetry_config,
         )
         # Everything is resolved at this point, so we no longer need
         # to load deferred dependencies (i.e. VCS, URL and path dependencies)

poetry/puzzle/solver.py

@@ -31,6 +31,7 @@
 
 
 if TYPE_CHECKING:
+    from poetry.config.config import Config
     from poetry.core.packages.dependency import Dependency
     from poetry.core.packages.directory_dependency import DirectoryDependency
     from poetry.core.packages.file_dependency import FileDependency
@@ -49,6 +50,7 @@ def __init__(
         io: IO,
         remove_untracked: bool = False,
         provider: Optional[Provider] = None,
+        config: Optional["Config"] = None,
     ):
         self._package = package
         self._pool = pool
@@ -63,10 +65,30 @@ def __init__(
         self._overrides = []
         self._remove_untracked = remove_untracked
 
+        self._poetry_config = config
+        self._preserved_package_names = None
+
     @property
     def provider(self) -> Provider:
         return self._provider
 
+    @property
+    def preserved_package_names(self):
+        if self._preserved_package_names is None:
+            self._preserved_package_names = {
+                self._package.name,
+                *Provider.UNSAFE_PACKAGES,
+            }
+            if self._poetry_config is not None:
+                deps = {package.name for package in self._locked.packages}
+
+                for name in {"pip", "wheel", "setuptools"}:
+                    if not self._poetry_config.get(f"virtualenvs.options.no-{name}"):
+                        if name not in deps:
+                            self._preserved_package_names.add(name)
+
+        return self._preserved_package_names
+
     @contextmanager
     def use_environment(self, env: Env) -> None:
         with self.provider.use_environment(env):
@@ -190,11 +212,9 @@ def solve(self, use_latest: List[str] = None) -> List["OperationTypes"]:
             locked_names = {locked.name for locked in self._locked.packages}
 
             for installed in self._installed.packages:
-                if installed.name == self._package.name:
-                    continue
-                if installed.name in Provider.UNSAFE_PACKAGES:
-                    # Never remove pip, setuptools etc.
+                if installed.name in self.preserved_package_names:
                     continue
+
                 if installed.name not in locked_names:
                     operations.append(Uninstall(installed))
 

poetry/utils/env.py

@@ -927,12 +927,29 @@ def build_venv(
         path: Union[Path, str],
         executable: Optional[Union[str, Path]] = None,
         flags: Dict[str, bool] = None,
-        with_pip: bool = False,
+        with_pip: Optional[bool] = None,
         with_wheel: Optional[bool] = None,
         with_setuptools: Optional[bool] = None,
     ) -> virtualenv.run.session.Session:
         flags = flags or {}
 
+        flags["no-pip"] = (
+            not with_pip if with_pip is not None else flags.pop("no-pip", True)
+        )
+
+        flags["no-setuptools"] = (
+            not with_setuptools
+            if with_setuptools is not None
+            else flags.pop("no-setuptools", True)
+        )
+
+        # we want wheels to be enabled when pip is required and it has not been explicitly disabled
+        flags["no-wheel"] = (
+            not with_wheel
+            if with_wheel is not None
+            else flags.pop("no-wheel", flags["no-pip"])
+        )
+
         if isinstance(executable, Path):
             executable = executable.resolve().as_posix()
 
@@ -943,20 +960,6 @@ def build_venv(
             executable or sys.executable,
         ]
 
-        if not with_pip:
-            args.append("--no-pip")
-        else:
-            if with_wheel is None:
-                # we want wheels to be enabled when pip is required and it has
-                # not been explicitly disabled
-                with_wheel = True
-
-        if with_wheel is None or not with_wheel:
-            args.append("--no-wheel")
-
-        if with_setuptools is None or not with_setuptools:
-            args.append("--no-setuptools")
-
         for flag, value in flags.items():
             if value is True:
                 args.append(f"--{flag}")
@@ -1039,6 +1042,8 @@ def __init__(self, path: Path, base: Optional[Path] = None) -> None:
         self._platlib = None
         self._script_dirs = None
 
+        self._embedded_pip_path = None
+
     @property
     def path(self) -> Path:
         return self._path
@@ -1074,6 +1079,12 @@ def get_embedded_wheel(self, distribution):
             distribution, "{}.{}".format(self.version_info[0], self.version_info[1])
         ).path
 
+    @property
+    def pip_embedded(self) -> str:
+        if self._embedded_pip_path is None:
+            self._embedded_pip_path = str(self.get_embedded_wheel("pip") / "pip")
+        return self._embedded_pip_path
+
     @property
     def pip(self) -> str:
         """
@@ -1082,7 +1093,7 @@ def pip(self) -> str:
         # we do not use as_posix() here due to issues with windows pathlib2 implementation
         path = self._bin("pip")
         if not Path(path).exists():
-            return str(self.get_embedded_wheel("pip") / "pip")
+            return str(self.pip_embedded)
         return path
 
     @property
@@ -1187,7 +1198,7 @@ def get_python_implementation(self) -> str:
     def get_marker_env(self) -> Dict[str, Any]:
         raise NotImplementedError()
 
-    def get_pip_command(self) -> List[str]:
+    def get_pip_command(self, embedded: bool = False) -> List[str]:
         raise NotImplementedError()
 
     def get_supported_tags(self) -> List[Tag]:
@@ -1208,16 +1219,20 @@ def is_sane(self) -> bool:
         """
         return True
 
-    def run(self, bin: str, *args: str, **kwargs: Any) -> Union[str, int]:
+    def get_command_from_bin(self, bin: str) -> List[str]:
         if bin == "pip":
-            return self.run_pip(*args, **kwargs)
+            # when pip is required we need to ensure that we fallback to
+            # embedded pip when pip is not available in the environment
+            return self.get_pip_command()
 
-        bin = self._bin(bin)
-        cmd = [bin] + list(args)
+        return [self._bin(bin)]
+
+    def run(self, bin: str, *args: str, **kwargs: Any) -> Union[str, int]:
+        cmd = self.get_command_from_bin(bin) + list(args)
         return self._run(cmd, **kwargs)
 
     def run_pip(self, *args: str, **kwargs: Any) -> Union[int, str]:
-        pip = self.get_pip_command()
+        pip = self.get_pip_command(embedded=True)
         cmd = pip + list(args)
         return self._run(cmd, **kwargs)
 
@@ -1259,17 +1274,13 @@ def _run(self, cmd: List[str], **kwargs: Any) -> Union[int, str]:
         return decode(output)
 
     def execute(self, bin: str, *args: str, **kwargs: Any) -> Optional[int]:
-        if bin == "pip":
-            return self.run_pip(*args, **kwargs)
-
-        bin = self._bin(bin)
+        command = self.get_command_from_bin(bin) + list(args)
         env = kwargs.pop("env", {k: v for k, v in os.environ.items()})
 
         if not self._is_windows:
-            args = [bin] + list(args)
-            return os.execvpe(bin, args, env=env)
+            return os.execvpe(command[0], command, env=env)
         else:
-            exe = subprocess.Popen([bin] + list(args), env=env, **kwargs)
+            exe = subprocess.Popen([command[0]] + command[1:], env=env, **kwargs)
             exe.communicate()
             return exe.returncode
 
@@ -1337,10 +1348,10 @@ def get_version_info(self) -> Tuple[int]:
     def get_python_implementation(self) -> str:
         return platform.python_implementation()
 
-    def get_pip_command(self) -> List[str]:
+    def get_pip_command(self, embedded: bool = False) -> List[str]:
         # If we're not in a venv, assume the interpreter we're running on
         # has a pip and use that
-        return [sys.executable, self.pip]
+        return [sys.executable, self.pip_embedded if embedded else self.pip]
 
     def get_paths(self) -> Dict[str, str]:
         # We can't use sysconfig.get_paths() because
@@ -1444,10 +1455,10 @@ def get_version_info(self) -> Tuple[int]:
     def get_python_implementation(self) -> str:
         return self.marker_env["platform_python_implementation"]
 
-    def get_pip_command(self) -> List[str]:
+    def get_pip_command(self, embedded: bool = False) -> List[str]:
         # We're in a virtualenv that is known to be sane,
         # so assume that we have a functional pip
-        return [self._bin("python"), self.pip]
+        return [self._bin("python"), self.pip_embedded if embedded else self.pip]
 
     def get_supported_tags(self) -> List[Tag]:
         file_path = Path(packaging.tags.__file__)
@@ -1559,8 +1570,8 @@ def __init__(
         self._execute = execute
         self.executed = []
 
-    def get_pip_command(self) -> List[str]:
-        return [self._bin("python"), self.pip]
+    def get_pip_command(self, embedded: bool = False) -> List[str]:
+        return [self._bin("python"), self.pip_embedded if embedded else self.pip]
 
     def _run(self, cmd: List[str], **kwargs: Any) -> int:
         self.executed.append(cmd)

poetry/utils/pip.py

@@ -53,7 +53,7 @@ def pip_install(
                 executable=environment.python, with_pip=True, with_setuptools=True
             ) as env:
                 return environment.run(
-                    env._bin("pip"),
+                    *env.get_pip_command(),
                     *args,
                     env={**os.environ, "PYTHONPATH": str(env.purelib)},
                 )