diff --git a/docs/cli.md b/docs/cli.md index 0f954fa6f59..1a8fb764725 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -233,6 +233,7 @@ option is used. When `--only` is specified, `--with` and `--without` options are ignored. {{% /note %}} + ## update In order to get the latest versions of the dependencies and to update the `poetry.lock` file, diff --git a/docs/configuration.md b/docs/configuration.md index 6204d450e9b..edd1c18aa3b 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -141,6 +141,54 @@ the number of maximum workers is still limited at `number_of_cores + 4`. This configuration will be ignored when `installer.parallel` is set to false. {{% /note %}} +### `installer.no-binary` + +**Type**: string | bool + +*Introduced in 1.2.0* + +When set this configuration allows users to configure package distribution format policy for all or +specific packages. + +| Configuration | Description | +|------------------------|------------------------------------------------------------| +| `:all:` or `true` | Disallow binary distributions for all packages. | +| `:none:` or `false` | Allow binary distributions for all packages. | +| `package[,package,..]` | Disallow binary distributions for specified packages only. | + +{{% note %}} +This configuration is only respected when using the new installer. If you have disabled it please +consider re-enabling it. + +As with all configurations described here, this is a user specific configuration. This means that this +is not taken into consideration when a lockfile is generated or dependencies are resolved. This is +applied only when selecting which distribution for dependency should be installed into a Poetry managed +environment. +{{% /note %}} + +{{% note %}} +For project specific usage, it is recommended that this be configured with the `--local`. + +```bash +poetry config --local installer.no-binary :all: +``` +{{% /note %}} + +{{% note %}} +For CI or container environments using [environment variable](#using-environment-variables) +to configure this might be useful. + +```bash +export POETRY_INSTALLER_NO_BINARY=:all: +``` +{{% /note %}} + +{{% warning %}} +Unless this is required system-wide, if configured globally, you could encounter slower install times +across all your projects if incorrectly set. +{{% /warning %}} + + ### `virtualenvs.create` **Type**: boolean diff --git a/pyproject.toml b/pyproject.toml index 88ede12c255..935964642fd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -123,11 +123,22 @@ module = [ 'poetry.installation.executor', 'poetry.installation.installer', 'poetry.installation.pip_installer', - 'poetry.repositories.installed_repository', 'poetry.utils.env', ] ignore_errors = true +# use of importlib-metadata backport at python3.7 makes it impossible to +# satisfy mypy without some ignores: but we get a different set of ignores at +# different python versions. +# +# , meanwhile suppress that +# warning. +[[tool.mypy.overrides]] +module = [ + 'poetry.repositories.installed_repository', +] +warn_unused_ignores = false + [[tool.mypy.overrides]] module = [ 'cachecontrol.*', diff --git a/src/poetry/config/config.py b/src/poetry/config/config.py index 693520766ca..51de8388e37 100644 --- a/src/poetry/config/config.py +++ b/src/poetry/config/config.py @@ -1,5 +1,6 @@ from __future__ import annotations +import dataclasses import logging import os import re @@ -11,6 +12,7 @@ from typing import Callable from poetry.core.toml import TOMLFile +from poetry.core.utils.helpers import canonicalize_name from poetry.config.dict_config_source import DictConfigSource from poetry.config.file_config_source import FileConfigSource @@ -34,6 +36,67 @@ def int_normalizer(val: str) -> int: return int(val) +@dataclasses.dataclass +class PackageFilterPolicy: + policy: dataclasses.InitVar[str | list[str] | None] + packages: list[str] = dataclasses.field(init=False) + + def __post_init__(self, policy: str | list[str] | None) -> None: + if not policy: + policy = [] + elif isinstance(policy, str): + policy = self.normalize(policy) + self.packages = policy + + def allows(self, package_name: str) -> bool: + if ":all:" in self.packages: + return False + + return ( + not self.packages + or ":none:" in self.packages + or canonicalize_name(package_name) not in self.packages + ) + + @classmethod + def is_reserved(cls, name: str) -> bool: + return bool(re.match(r":(all|none):", name)) + + @classmethod + def normalize(cls, policy: str) -> list[str]: + if boolean_validator(policy): + if boolean_normalizer(policy): + return [":all:"] + else: + return [":none:"] + + return list( + { + name.strip() if cls.is_reserved(name) else canonicalize_name(name) + for name in policy.strip().split(",") + if name + } + ) + + @classmethod + def validator(cls, policy: str) -> bool: + if boolean_validator(policy): + return True + + names = policy.strip().split(",") + + for name in names: + if ( + not name + or (cls.is_reserved(name) and len(names) == 1) + or re.match(r"^[a-zA-Z\d_-]+$", name) + ): + continue + return False + + return True + + logger = logging.getLogger(__name__) @@ -61,7 +124,7 @@ class Config: "prefer-active-python": False, }, "experimental": {"new-installer": True, "system-git-client": False}, - "installer": {"parallel": True, "max-workers": None}, + "installer": {"parallel": True, "max-workers": None, "no-binary": None}, } def __init__( @@ -196,6 +259,9 @@ def _get_normalizer(name: str) -> Callable[[str], Any]: if name == "installer.max-workers": return int_normalizer + if name == "installer.no-binary": + return PackageFilterPolicy.normalize + return lambda val: val @classmethod diff --git a/src/poetry/console/commands/config.py b/src/poetry/console/commands/config.py index 537f684040b..8a7ae790a4b 100644 --- a/src/poetry/console/commands/config.py +++ b/src/poetry/console/commands/config.py @@ -10,6 +10,7 @@ from cleo.helpers import argument from cleo.helpers import option +from poetry.config.config import PackageFilterPolicy from poetry.console.commands.command import Command @@ -107,6 +108,11 @@ def unique_config_values(self) -> dict[str, tuple[Any, Any, Any]]: int_normalizer, None, ), + "installer.no-binary": ( + PackageFilterPolicy.validator, + PackageFilterPolicy.normalize, + None, + ), } return unique_config_values diff --git a/src/poetry/installation/chooser.py b/src/poetry/installation/chooser.py index e51610f455e..82659d444eb 100644 --- a/src/poetry/installation/chooser.py +++ b/src/poetry/installation/chooser.py @@ -7,6 +7,8 @@ from packaging.tags import Tag +from poetry.config.config import Config +from poetry.config.config import PackageFilterPolicy from poetry.utils.patterns import wheel_file_re @@ -57,9 +59,13 @@ class Chooser: A Chooser chooses an appropriate release archive for packages. """ - def __init__(self, pool: Pool, env: Env) -> None: + def __init__(self, pool: Pool, env: Env, config: Config | None = None) -> None: self._pool = pool self._env = env + self._config = config or Config.create() + self._no_binary_policy: PackageFilterPolicy = PackageFilterPolicy( + self._config.get("installer.no-binary", []) + ) def choose_for(self, package: Package) -> Link: """ @@ -67,15 +73,23 @@ def choose_for(self, package: Package) -> Link: """ links = [] for link in self._get_links(package): - if link.is_wheel and not Wheel(link.filename).is_supported_by_environment( - self._env - ): - logger.debug( - "Skipping wheel %s as this is not supported by the current" - " environment", - link.filename, - ) - continue + if link.is_wheel: + if not self._no_binary_policy.allows(package.name): + logger.debug( + "Skipping wheel for %s as requested in no binary policy for" + " package (%s)", + link.filename, + package.name, + ) + continue + + if not Wheel(link.filename).is_supported_by_environment(self._env): + logger.debug( + "Skipping wheel %s as this is not supported by the current" + " environment", + link.filename, + ) + continue if link.ext in {".egg", ".exe", ".msi", ".rpm", ".srpm"}: logger.debug("Skipping unsupported distribution %s", link.filename) diff --git a/src/poetry/installation/executor.py b/src/poetry/installation/executor.py index b890dc30b80..8e1f18f78fa 100644 --- a/src/poetry/installation/executor.py +++ b/src/poetry/installation/executor.py @@ -58,7 +58,7 @@ def __init__( self._verbose = False self._authenticator = Authenticator(config, self._io) self._chef = Chef(config, self._env) - self._chooser = Chooser(pool, self._env) + self._chooser = Chooser(pool, self._env, config) if parallel is None: parallel = config.get("installer.parallel", True) diff --git a/src/poetry/repositories/installed_repository.py b/src/poetry/repositories/installed_repository.py index d7bea9ed652..d7a6d9abb6e 100644 --- a/src/poetry/repositories/installed_repository.py +++ b/src/poetry/repositories/installed_repository.py @@ -103,7 +103,7 @@ def create_package_from_distribution( ) -> Package: # We first check for a direct_url.json file to determine # the type of package. - path = Path(str(distribution._path)) + path = Path(str(distribution._path)) # type: ignore[attr-defined] if ( path.name.endswith(".dist-info") @@ -163,13 +163,17 @@ def create_package_from_distribution( source_reference=source_reference, source_resolved_reference=source_resolved_reference, ) - package.description = distribution.metadata.get("summary", "") + + package.description = distribution.metadata.get( # type: ignore[attr-defined] + "summary", + "", + ) return package @classmethod def create_package_from_pep610(cls, distribution: metadata.Distribution) -> Package: - path = Path(str(distribution._path)) + path = Path(str(distribution._path)) # type: ignore[attr-defined] source_type = None source_url = None source_reference = None @@ -213,7 +217,10 @@ def create_package_from_pep610(cls, distribution: metadata.Distribution) -> Pack develop=develop, ) - package.description = distribution.metadata.get("summary", "") + package.description = distribution.metadata.get( # type: ignore[attr-defined] + "summary", + "", + ) return package @@ -229,15 +236,17 @@ def load(cls, env: Env, with_dependencies: bool = False) -> InstalledRepository: for entry in reversed(env.sys_path): for distribution in sorted( - metadata.distributions(path=[entry]), - key=lambda d: str(d._path), + metadata.distributions( # type: ignore[no-untyped-call] + path=[entry], + ), + key=lambda d: str(d._path), # type: ignore[attr-defined] ): name = canonicalize_name(distribution.metadata["name"]) if name in seen: continue - path = Path(str(distribution._path)) + path = Path(str(distribution._path)) # type: ignore[attr-defined] try: path.relative_to(_VENDORS) diff --git a/tests/console/commands/test_config.py b/tests/console/commands/test_config.py index 7c0059b9918..bf5fb06ab67 100644 --- a/tests/console/commands/test_config.py +++ b/tests/console/commands/test_config.py @@ -7,10 +7,12 @@ import pytest +from deepdiff import DeepDiff from poetry.core.pyproject.exceptions import PyProjectException from poetry.config.config_source import ConfigSource from poetry.factory import Factory +from tests.conftest import Config if TYPE_CHECKING: @@ -20,7 +22,6 @@ from pytest_mock import MockerFixture from poetry.config.dict_config_source import DictConfigSource - from tests.conftest import Config from tests.types import CommandTesterFactory from tests.types import FixtureDirGetter @@ -53,6 +54,7 @@ def test_list_displays_default_value_if_not_set( experimental.new-installer = true experimental.system-git-client = false installer.max-workers = null +installer.no-binary = null installer.parallel = true virtualenvs.create = true virtualenvs.in-project = null @@ -80,6 +82,7 @@ def test_list_displays_set_get_setting( experimental.new-installer = true experimental.system-git-client = false installer.max-workers = null +installer.no-binary = null installer.parallel = true virtualenvs.create = false virtualenvs.in-project = null @@ -131,6 +134,7 @@ def test_list_displays_set_get_local_setting( experimental.new-installer = true experimental.system-git-client = false installer.max-workers = null +installer.no-binary = null installer.parallel = true virtualenvs.create = false virtualenvs.in-project = null @@ -200,3 +204,33 @@ def test_config_installer_parallel( "install" )._command._installer._executor._max_workers assert workers == 1 + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + ("true", [":all:"]), + ("1", [":all:"]), + ("false", [":none:"]), + ("0", [":none:"]), + ("pytest", ["pytest"]), + ("PyTest", ["pytest"]), + ("pytest,black", ["pytest", "black"]), + ("", []), + ], +) +def test_config_installer_no_binary( + tester: CommandTester, value: str, expected: list[str] +) -> None: + setting = "installer.no-binary" + + tester.execute(setting) + assert tester.io.fetch_output().strip() == "null" + + config = Config.create() + assert not config.get(setting) + + tester.execute(f"{setting} '{value}'") + + config = Config.create(reload=True) + assert not DeepDiff(config.get(setting), expected, ignore_order=True) diff --git a/tests/installation/test_chooser.py b/tests/installation/test_chooser.py index 973e4842ff6..51c8b66c2ab 100644 --- a/tests/installation/test_chooser.py +++ b/tests/installation/test_chooser.py @@ -23,6 +23,8 @@ from httpretty.core import HTTPrettyRequest + from tests.conftest import Config + JSON_FIXTURES = ( Path(__file__).parent.parent / "repositories" / "fixtures" / "pypi.org" / "json" @@ -121,6 +123,46 @@ def test_chooser_chooses_universal_wheel_link_if_available( assert link.filename == "pytest-3.5.0-py2.py3-none-any.whl" +@pytest.mark.parametrize( + ("policy", "filename"), + [ + (":all:", "pytest-3.5.0.tar.gz"), + (":none:", "pytest-3.5.0-py2.py3-none-any.whl"), + ("black", "pytest-3.5.0-py2.py3-none-any.whl"), + ("pytest", "pytest-3.5.0.tar.gz"), + ("pytest,black", "pytest-3.5.0.tar.gz"), + ], +) +@pytest.mark.parametrize("source_type", ["", "legacy"]) +def test_chooser_no_binary_policy( + env: MockEnv, + mock_pypi: None, + mock_legacy: None, + source_type: str, + pool: Pool, + policy: str, + filename: str, + config: Config, +): + config.merge({"installer": {"no-binary": policy.split(",")}}) + + chooser = Chooser(pool, env, config) + + package = Package("pytest", "3.5.0") + if source_type == "legacy": + package = Package( + package.name, + package.version.text, + source_type="legacy", + source_reference="foo", + source_url="https://foo.bar/simple/", + ) + + link = chooser.choose_for(package) + + assert link.filename == filename + + @pytest.mark.parametrize("source_type", ["", "legacy"]) def test_chooser_chooses_specific_python_universal_wheel_link_if_available( env: MockEnv, mock_pypi: None, mock_legacy: None, source_type: str, pool: Pool