From 42f8138b64b9493faf8d823016ce85c28c42c527 Mon Sep 17 00:00:00 2001 From: Arun Babu Neelicattu Date: Mon, 5 Oct 2020 19:26:59 +0200 Subject: [PATCH 01/10] config: add support for installer.parallel This change allows users to disable parallel execution while using the new installer. Resolves: #3087 --- docs/docs/configuration.md | 9 +++++++++ poetry/config/config.py | 3 +++ poetry/console/commands/config.py | 1 + poetry/installation/executor.py | 5 ++++- tests/config/test_config.py | 27 ++++++++++++++++++++------- tests/console/commands/test_config.py | 27 +++++++++++++++++++++++++++ 6 files changed, 64 insertions(+), 8 deletions(-) diff --git a/docs/docs/configuration.md b/docs/docs/configuration.md index 782c2f9ac87..72310886ca7 100644 --- a/docs/docs/configuration.md +++ b/docs/docs/configuration.md @@ -103,6 +103,15 @@ Defaults to one of the following directories: - Windows: `C:\Users\\AppData\Local\pypoetry\Cache` - Unix: `~/.cache/pypoetry` +### `installer.parallel`: boolean + +Use parallel execution when using the new (`>=1.1.0`) installer. +Defaults to `true`. + +!!!note: + This configuration will be ignored, and parallel execution disabled when running + Python 2.7 under Windows. + ### `virtualenvs.create`: boolean Create a new virtual environment if one doesn't already exist. diff --git a/poetry/config/config.py b/poetry/config/config.py index d1fb421e53c..6f18127e8ff 100644 --- a/poetry/config/config.py +++ b/poetry/config/config.py @@ -39,6 +39,7 @@ class Config(object): "options": {"always-copy": False}, }, "experimental": {"new-installer": True}, + "installer": {"parallel": True}, } def __init__( @@ -140,6 +141,7 @@ def _get_validator(self, name): # type: (str) -> Callable "virtualenvs.create", "virtualenvs.in-project", "virtualenvs.options.always-copy", + "installer.parallel", }: return boolean_validator @@ -151,6 +153,7 @@ def _get_normalizer(self, name): # type: (str) -> Callable "virtualenvs.create", "virtualenvs.in-project", "virtualenvs.options.always-copy", + "installer.parallel", }: return boolean_normalizer diff --git a/poetry/console/commands/config.py b/poetry/console/commands/config.py index 52e1cf125ab..934b2c818eb 100644 --- a/poetry/console/commands/config.py +++ b/poetry/console/commands/config.py @@ -69,6 +69,7 @@ def unique_config_values(self): boolean_normalizer, False, ), + "installer.parallel": (boolean_validator, boolean_normalizer, True,), } return unique_config_values diff --git a/poetry/installation/executor.py b/poetry/installation/executor.py index a3184b718d5..a65dbb4a0e1 100644 --- a/poetry/installation/executor.py +++ b/poetry/installation/executor.py @@ -32,7 +32,7 @@ class Executor(object): - def __init__(self, env, pool, config, io, parallel=True): + def __init__(self, env, pool, config, io, parallel=None): self._env = env self._io = io self._dry_run = False @@ -42,6 +42,9 @@ def __init__(self, env, pool, config, io, parallel=True): self._chef = Chef(config, self._env) self._chooser = Chooser(pool, self._env) + if parallel is None: + parallel = config.get("installer.parallel", True) + if parallel and not (PY2 and WINDOWS): # This should be directly handled by ThreadPoolExecutor # however, on some systems the number of CPUs cannot be determined diff --git a/tests/config/test_config.py b/tests/config/test_config.py index 07373ade0b4..4bd0cd048da 100644 --- a/tests/config/test_config.py +++ b/tests/config/test_config.py @@ -1,16 +1,29 @@ import os +import pytest -def test_config_get_default_value(config): - assert config.get("virtualenvs.create") is True + +@pytest.mark.parametrize( + ("name", "value"), [("installer.parallel", True), ("virtualenvs.create", True)] +) +def test_config_get_default_value(config, name, value): + assert config.get(name) is value def test_config_get_processes_depended_on_values(config): assert os.path.join("/foo", "virtualenvs") == config.get("virtualenvs.path") -def test_config_get_from_environment_variable(config, environ): - assert config.get("virtualenvs.create") - - os.environ["POETRY_VIRTUALENVS_CREATE"] = "false" - assert not config.get("virtualenvs.create") +@pytest.mark.parametrize( + ("name", "env_value", "value"), + [ + ("installer.parallel", "true", True), + ("installer.parallel", "false", False), + ("virtualenvs.create", "true", True), + ("virtualenvs.create", "false", False), + ], +) +def test_config_get_from_environment_variable(config, environ, name, env_value, value): + env_var = "POETRY_{}".format("_".join(k.upper() for k in name.split("."))) + os.environ[env_var] = env_value + assert config.get(name) is value diff --git a/tests/console/commands/test_config.py b/tests/console/commands/test_config.py index 9923f1b77af..4d0b9ada0c8 100644 --- a/tests/console/commands/test_config.py +++ b/tests/console/commands/test_config.py @@ -6,6 +6,8 @@ from poetry.config.config_source import ConfigSource from poetry.core.pyproject import PyProjectException from poetry.factory import Factory +from poetry.utils._compat import PY2 +from poetry.utils._compat import WINDOWS @pytest.fixture() @@ -28,6 +30,7 @@ def test_list_displays_default_value_if_not_set(tester, config): expected = """cache-dir = "/foo" experimental.new-installer = true +installer.parallel = true virtualenvs.create = true virtualenvs.in-project = null virtualenvs.options.always-copy = false @@ -46,6 +49,7 @@ def test_list_displays_set_get_setting(tester, config): expected = """cache-dir = "/foo" experimental.new-installer = true +installer.parallel = true virtualenvs.create = false virtualenvs.in-project = null virtualenvs.options.always-copy = false @@ -86,6 +90,7 @@ def test_list_displays_set_get_local_setting(tester, config): expected = """cache-dir = "/foo" experimental.new-installer = true +installer.parallel = true virtualenvs.create = false virtualenvs.in-project = null virtualenvs.options.always-copy = false @@ -122,3 +127,25 @@ def test_set_cert(tester, auth_config_source, mocker): tester.execute("certificates.foo.cert path/to/ca.pem") assert "path/to/ca.pem" == auth_config_source.config["certificates"]["foo"]["cert"] + + +def test_config_installer_parallel(tester, command_tester_factory): + serial_enforced = PY2 and WINDOWS + + tester.execute("--local installer.parallel") + assert tester.io.fetch_output().strip() == "true" + + workers = command_tester_factory( + "install" + )._command._installer._executor._max_workers + assert workers > 1 or (serial_enforced and workers == 1) + + tester.io.clear_output() + tester.execute("--local installer.parallel false") + tester.execute("--local installer.parallel") + assert tester.io.fetch_output().strip() == "false" + + workers = command_tester_factory( + "install" + )._command._installer._executor._max_workers + assert workers == 1 From 5cee6d0b11230c832a7fae3dc0830fb7a044c68d Mon Sep 17 00:00:00 2001 From: Arun Babu Neelicattu Date: Tue, 6 Oct 2020 21:07:04 +0200 Subject: [PATCH 02/10] utils/env: better support system site packages dir This change improves handling of site-packages under system env, by gracefully handling fallbacks to user site when required and possible. Resolves: #3079 --- poetry/installation/pip_installer.py | 4 +- poetry/masonry/builders/editable.py | 61 +++++---- poetry/utils/env.py | 129 +++++++++++++++++- poetry/utils/helpers.py | 5 + tests/installation/test_pip_installer.py | 4 +- .../masonry/builders/test_editable_builder.py | 10 +- tests/utils/test_env.py | 4 +- tests/utils/test_env_site.py | 41 ++++++ 8 files changed, 218 insertions(+), 40 deletions(-) create mode 100644 tests/utils/test_env_site.py diff --git a/poetry/installation/pip_installer.py b/poetry/installation/pip_installer.py index f5de6642f2d..e44d7a61297 100644 --- a/poetry/installation/pip_installer.py +++ b/poetry/installation/pip_installer.py @@ -113,7 +113,9 @@ def remove(self, package): raise # This is a workaround for https://github.com/pypa/pip/issues/4176 - nspkg_pth_file = self._env.site_packages / "{}-nspkg.pth".format(package.name) + nspkg_pth_file = self._env.site_packages.path / "{}-nspkg.pth".format( + package.name + ) if nspkg_pth_file.exists(): nspkg_pth_file.unlink() diff --git a/poetry/masonry/builders/editable.py b/poetry/masonry/builders/editable.py index 99136ecbd3e..c31f657b656 100644 --- a/poetry/masonry/builders/editable.py +++ b/poetry/masonry/builders/editable.py @@ -94,7 +94,6 @@ def _setup_build(self): os.remove(str(setup)) def _add_pth(self): - pth_file = Path(self._module.name).with_suffix(".pth") paths = set() for include in self._module.includes: if isinstance(include, PackageInclude) and ( @@ -106,29 +105,25 @@ def _add_pth(self): for path in paths: content += decode(path + os.linesep) - for site_package in [self._env.site_packages, self._env.usersite]: - if not site_package: - continue - - try: - site_package.mkdir(parents=True, exist_ok=True) - path = site_package.joinpath(pth_file) - self._debug( - " - Adding {} to {} for {}".format( - path.name, site_package, self._poetry.file.parent - ) + pth_file = Path(self._module.name).with_suffix(".pth") + try: + pth_file = self._env.site_packages.write_text( + pth_file, content, encoding="utf-8" + ) + self._debug( + " - Adding {} to {} for {}".format( + pth_file.name, pth_file.parent, self._poetry.file.parent ) - path.write_text(content, encoding="utf-8") - return [path] - except PermissionError: - self._debug("- {} is not writable trying next available site") - - self._io.error_line( - " - Failed to create {} for {}".format( - pth_file.name, self._poetry.file.parent ) - ) - return [] + return [pth_file] + except OSError: + # TODO: Replace with PermissionError + self._io.error_line( + " - Failed to create {} for {}".format( + pth_file.name, self._poetry.file.parent + ) + ) + return [] def _add_scripts(self): added = [] @@ -187,19 +182,27 @@ def _add_dist_info(self, added_files): added_files = added_files[:] builder = WheelBuilder(self._poetry) - dist_info = self._env.site_packages.joinpath(builder.dist_info) + + dist_info_path = Path(builder.dist_info) + for dist_info in self._env.site_packages.find( + dist_info_path, writable_only=True + ): + if dist_info.exists(): + self._debug( + " - Removing existing {} directory from {}".format( + dist_info.name, dist_info.parent + ) + ) + shutil.rmtree(str(dist_info)) + + dist_info = self._env.site_packages.mkdir(dist_info_path) self._debug( " - Adding the {} directory to {}".format( - dist_info.name, self._env.site_packages + dist_info.name, dist_info.parent ) ) - if dist_info.exists(): - shutil.rmtree(str(dist_info)) - - dist_info.mkdir() - with dist_info.joinpath("METADATA").open("w", encoding="utf-8") as f: builder._write_metadata_file(f) diff --git a/poetry/utils/env.py b/poetry/utils/env.py index 0e3df320d6a..a81de75458d 100644 --- a/poetry/utils/env.py +++ b/poetry/utils/env.py @@ -7,6 +7,7 @@ import shutil import sys import sysconfig +import tempfile import textwrap from contextlib import contextmanager @@ -39,6 +40,7 @@ from poetry.utils._compat import encode from poetry.utils._compat import list_to_shell_command from poetry.utils._compat import subprocess +from poetry.utils.helpers import paths_csv GET_ENVIRONMENT_INFO = """\ @@ -143,6 +145,125 @@ def _version_nodot(version): """ +class SitePackages: + def __init__( + self, path, fallbacks=None, skip_write_checks=False + ): # type: (Path, List[Path], bool) -> None + self._path = path + self._fallbacks = fallbacks or [] + self._skip_write_checks = skip_write_checks + self._candidates = [self._path] + self._fallbacks + self._writable_candidates = None if not skip_write_checks else self._candidates + + @property + def path(self): # type: () -> Path + return self._path + + @property + def candidates(self): # type: () -> List[Path] + return self._candidates + + @property + def writable_candidates(self): # type: () -> List[Path] + if self._writable_candidates is not None: + return self._writable_candidates + + self._writable_candidates = [] + for candidate in self._candidates: + try: + if not candidate.exists(): + continue + + with tempfile.TemporaryFile(dir=str(candidate)): + self._writable_candidates.append(candidate) + except (IOError, OSError): + pass + + return self._writable_candidates + + def make_candidates( + self, path, writable_only=False + ): # type: (Path, bool) -> List[Path] + candidates = self._candidates if not writable_only else self.writable_candidates + if path.is_absolute(): + for candidate in candidates: + try: + path.relative_to(candidate) + return [path] + except ValueError: + pass + else: + raise ValueError( + "{} is not relative to any discovered {}sites".format( + path, "writable " if writable_only else "" + ) + ) + + return [candidate / path for candidate in candidates if candidate] + + def _path_method_wrapper( + self, path, method, *args, **kwargs + ): # type: (Path, str, *Any, **Any) -> Union[Tuple[Path, Any], List[Tuple[Path, Any]]] + + # TODO: Move to parameters after dropping Python 2.7 + return_first = kwargs.pop("return_first", True) + writable_only = kwargs.pop("writable_only", False) + + candidates = self.make_candidates(path, writable_only=writable_only) + + if not candidates: + raise RuntimeError( + 'Unable to find a suitable destination for "{}" in {}'.format( + str(path), paths_csv(self._candidates) + ) + ) + + results = [] + + for candidate in candidates: + try: + result = candidate, getattr(candidate, method)(*args, **kwargs) + if return_first: + return result + else: + results.append(result) + except (IOError, OSError): + # TODO: Replace with PermissionError + pass + + if results: + return results + + raise OSError("Unable to access any of {}".format(paths_csv(candidates))) + + def write_text(self, path, *args, **kwargs): # type: (Path, *Any, **Any) -> Path + return self._path_method_wrapper(path, "write_text", *args, **kwargs)[0] + + def mkdir(self, path, *args, **kwargs): # type: (Path, *Any, **Any) -> Path + return self._path_method_wrapper(path, "mkdir", *args, **kwargs)[0] + + def exists(self, path): # type: (Path) -> bool + return any( + value[-1] + for value in self._path_method_wrapper(path, "exists", return_first=False) + ) + + def find(self, path, writable_only=False): # type: (Path, bool) -> List[Path] + return [ + value[0] + for value in self._path_method_wrapper( + path, "exists", return_first=False, writable_only=writable_only + ) + if value[-1] is True + ] + + def __getattr__(self, item): + try: + return super(SitePackages, self).__getattribute__(item) + except AttributeError: + return getattr(self.path, item) + + class EnvError(Exception): pass @@ -825,9 +946,13 @@ def pip_version(self): return self._pip_version @property - def site_packages(self): # type: () -> Path + def site_packages(self): # type: () -> SitePackages if self._site_packages is None: - self._site_packages = self.purelib + # we disable write checks if no user site exist + fallbacks = [self.usersite] if self.usersite else [] + self._site_packages = SitePackages( + self.purelib, fallbacks, skip_write_checks=False if fallbacks else True + ) return self._site_packages @property diff --git a/poetry/utils/helpers.py b/poetry/utils/helpers.py index 180a90d50f3..e6162d508db 100644 --- a/poetry/utils/helpers.py +++ b/poetry/utils/helpers.py @@ -5,6 +5,7 @@ import tempfile from contextlib import contextmanager +from typing import List from typing import Optional import requests @@ -113,3 +114,7 @@ def get_package_version_display_string( ) return package.full_pretty_version + + +def paths_csv(paths): # type: (List[Path]) -> str + return ", ".join('"{}"'.format(str(c)) for c in paths) diff --git a/tests/installation/test_pip_installer.py b/tests/installation/test_pip_installer.py index 206bcce59cc..d0e2e5a4dcd 100644 --- a/tests/installation/test_pip_installer.py +++ b/tests/installation/test_pip_installer.py @@ -189,7 +189,9 @@ def test_uninstall_git_package_nspkg_pth_cleanup(mocker, tmp_venv, pool): ) # we do this here because the virtual env might not be usable if failure case is triggered - pth_file_candidate = tmp_venv.site_packages / "{}-nspkg.pth".format(package.name) + pth_file_candidate = tmp_venv.site_packages.path / "{}-nspkg.pth".format( + package.name + ) # in order to reproduce the scenario where the git source is removed prior to proper # clean up of nspkg.pth file, we need to make sure the fixture is copied and not diff --git a/tests/masonry/builders/test_editable_builder.py b/tests/masonry/builders/test_editable_builder.py index 3aee74e7c77..daeff0e7777 100644 --- a/tests/masonry/builders/test_editable_builder.py +++ b/tests/masonry/builders/test_editable_builder.py @@ -76,14 +76,14 @@ def test_builder_installs_proper_files_for_standard_packages(simple_poetry, tmp_ builder.build() assert tmp_venv._bin_dir.joinpath("foo").exists() - assert tmp_venv.site_packages.joinpath("simple_project.pth").exists() - assert simple_poetry.file.parent.resolve().as_posix() == tmp_venv.site_packages.joinpath( + assert tmp_venv.site_packages.path.joinpath("simple_project.pth").exists() + assert simple_poetry.file.parent.resolve().as_posix() == tmp_venv.site_packages.path.joinpath( "simple_project.pth" ).read_text().strip( os.linesep ) - dist_info = tmp_venv.site_packages.joinpath("simple_project-1.2.3.dist-info") + dist_info = tmp_venv.site_packages.path.joinpath("simple_project-1.2.3.dist-info") assert dist_info.exists() assert dist_info.joinpath("INSTALLER").exists() assert dist_info.joinpath("METADATA").exists() @@ -130,7 +130,7 @@ def test_builder_installs_proper_files_for_standard_packages(simple_poetry, tmp_ assert metadata == dist_info.joinpath("METADATA").read_text(encoding="utf-8") records = dist_info.joinpath("RECORD").read_text() - assert str(tmp_venv.site_packages.joinpath("simple_project.pth")) in records + assert str(tmp_venv.site_packages.path.joinpath("simple_project.pth")) in records assert str(tmp_venv._bin_dir.joinpath("foo")) in records assert str(tmp_venv._bin_dir.joinpath("baz")) in records assert str(dist_info.joinpath("METADATA")) in records @@ -202,7 +202,7 @@ def test_builder_installs_proper_files_when_packages_configured( builder = EditableBuilder(project_with_include, tmp_venv, NullIO()) builder.build() - pth_file = tmp_venv.site_packages.joinpath("with_include.pth") + pth_file = tmp_venv.site_packages.path.joinpath("with_include.pth") assert pth_file.is_file() paths = set() diff --git a/tests/utils/test_env.py b/tests/utils/test_env.py index 8c816a2d32a..8934ae43427 100644 --- a/tests/utils/test_env.py +++ b/tests/utils/test_env.py @@ -866,7 +866,7 @@ def test_system_env_has_correct_paths(): assert paths.get("purelib") is not None assert paths.get("platlib") is not None assert paths.get("scripts") is not None - assert env.site_packages == Path(paths["purelib"]) + assert env.site_packages.path == Path(paths["purelib"]) @pytest.mark.parametrize( @@ -886,4 +886,4 @@ def test_venv_has_correct_paths(tmp_venv): assert paths.get("purelib") is not None assert paths.get("platlib") is not None assert paths.get("scripts") is not None - assert tmp_venv.site_packages == Path(paths["purelib"]) + assert tmp_venv.site_packages.path == Path(paths["purelib"]) diff --git a/tests/utils/test_env_site.py b/tests/utils/test_env_site.py new file mode 100644 index 00000000000..a13089160e6 --- /dev/null +++ b/tests/utils/test_env_site.py @@ -0,0 +1,41 @@ +import uuid + +from poetry.utils._compat import Path +from poetry.utils._compat import decode +from poetry.utils.env import SitePackages + + +def test_env_site_simple(tmp_dir): + site_packages = SitePackages(Path("/non-existent"), fallbacks=[Path(tmp_dir)]) + candidates = site_packages.make_candidates(Path("hello.txt"), writable_only=True) + hello = Path(tmp_dir) / "hello.txt" + + assert len(candidates) == 1 + assert candidates[0].as_posix() == hello.as_posix() + + content = decode(str(uuid.uuid4())) + site_packages.write_text(Path("hello.txt"), content, encoding="utf-8") + + assert hello.read_text(encoding="utf-8") == content + + assert not (site_packages.path / "hello.txt").exists() + + +def test_env_site_select_first(tmp_dir): + path = Path(tmp_dir) + fallback = path / "fallback" + fallback.mkdir(parents=True) + + site_packages = SitePackages(path, fallbacks=[fallback]) + candidates = site_packages.make_candidates(Path("hello.txt"), writable_only=True) + + assert len(candidates) == 2 + assert len(site_packages.find(Path("hello.txt"))) == 0 + + content = decode(str(uuid.uuid4())) + site_packages.write_text(Path("hello.txt"), content, encoding="utf-8") + + assert (site_packages.path / "hello.txt").exists() + assert not (fallback / "hello.txt").exists() + + assert len(site_packages.find(Path("hello.txt"))) == 1 From aca82c615afba052e48463676583484cba3becd4 Mon Sep 17 00:00:00 2001 From: Arun Babu Neelicattu Date: Fri, 23 Oct 2020 03:23:06 +0200 Subject: [PATCH 03/10] editable: use writable script dir for system env This change ensures that, When using system environment, poetry falls back to `userbase` if default location is not writable. --- poetry/masonry/builders/editable.py | 19 +++++++++++---- poetry/utils/env.py | 38 +++++++++++++++++++++-------- poetry/utils/helpers.py | 15 ++++++++++++ 3 files changed, 57 insertions(+), 15 deletions(-) diff --git a/poetry/masonry/builders/editable.py b/poetry/masonry/builders/editable.py index c31f657b656..0493423cdd2 100644 --- a/poetry/masonry/builders/editable.py +++ b/poetry/masonry/builders/editable.py @@ -13,6 +13,7 @@ from poetry.utils._compat import WINDOWS from poetry.utils._compat import Path from poetry.utils._compat import decode +from poetry.utils.helpers import is_dir_writable SCRIPT_TEMPLATE = """\ @@ -128,7 +129,17 @@ def _add_pth(self): def _add_scripts(self): added = [] entry_points = self.convert_entry_points() - scripts_path = Path(self._env.paths["scripts"]) + + for scripts_path in self._env.script_dirs: + if is_dir_writable(scripts_path): + break + else: + self._io.error_line( + " - Failed to find a suitable script installation directory for {}".format( + self._poetry.file.parent + ) + ) + return [] scripts = entry_points.get("console_scripts", []) for script in scripts: @@ -146,7 +157,7 @@ def _add_scripts(self): f.write( decode( SCRIPT_TEMPLATE.format( - python=self._env._bin("python"), + python=self._env.python, module=module, callable_holder=callable_holder, callable_=callable_, @@ -160,9 +171,7 @@ def _add_scripts(self): if WINDOWS: cmd_script = script_file.with_suffix(".cmd") - cmd = WINDOWS_CMD_TEMPLATE.format( - python=self._env._bin("python"), script=name - ) + cmd = WINDOWS_CMD_TEMPLATE.format(python=self._env.python, script=name) self._debug( " - Adding the {} script wrapper to {}".format( cmd_script.name, scripts_path diff --git a/poetry/utils/env.py b/poetry/utils/env.py index a81de75458d..33886926d4b 100644 --- a/poetry/utils/env.py +++ b/poetry/utils/env.py @@ -7,7 +7,6 @@ import shutil import sys import sysconfig -import tempfile import textwrap from contextlib import contextmanager @@ -40,6 +39,7 @@ from poetry.utils._compat import encode from poetry.utils._compat import list_to_shell_command from poetry.utils._compat import subprocess +from poetry.utils.helpers import is_dir_writable from poetry.utils.helpers import paths_csv @@ -170,14 +170,9 @@ def writable_candidates(self): # type: () -> List[Path] self._writable_candidates = [] for candidate in self._candidates: - try: - if not candidate.exists(): - continue - - with tempfile.TemporaryFile(dir=str(candidate)): - self._writable_candidates.append(candidate) - except (IOError, OSError): - pass + if not is_dir_writable(candidate): + continue + self._writable_candidates.append(candidate) return self._writable_candidates @@ -892,6 +887,7 @@ def __init__(self, path, base=None): # type: (Path, Optional[Path]) -> None self._supported_tags = None self._purelib = None self._platlib = None + self._script_dirs = None @property def path(self): # type: () -> Path @@ -960,6 +956,11 @@ def usersite(self): # type: () -> Optional[Path] if "usersite" in self.paths: return Path(self.paths["usersite"]) + @property + def userbase(self): # type: () -> Optional[Path] + if "userbase" in self.paths: + return Path(self.paths["userbase"]) + @property def purelib(self): # type: () -> Path if self._purelib is None: @@ -1106,6 +1107,18 @@ def execute(self, bin, *args, **kwargs): def is_venv(self): # type: () -> bool raise NotImplementedError() + @property + def script_dirs(self): # type: () -> List[Path] + if self._script_dirs is None: + self._script_dirs = ( + [Path(self.paths["scripts"])] + if "scripts" in self.paths + else self._bin_dir + ) + if self.userbase: + self._script_dirs.append(self.userbase / self._script_dirs[0].name) + return self._script_dirs + def _bin(self, bin): # type: (str) -> str """ Return path to the given executable. @@ -1141,6 +1154,10 @@ class SystemEnv(Env): A system (i.e. not a virtualenv) Python environment. """ + @property + def python(self): # type: () -> str + return sys.executable + @property def sys_path(self): # type: () -> List[str] return sys.path @@ -1181,6 +1198,7 @@ def get_paths(self): # type: () -> Dict[str, str] if site.check_enableusersite() and hasattr(obj, "install_usersite"): paths["usersite"] = getattr(obj, "install_usersite") + paths["userbase"] = getattr(obj, "install_userbase") return paths @@ -1316,7 +1334,7 @@ def is_venv(self): # type: () -> bool def is_sane(self): # A virtualenv is considered sane if both "python" and "pip" exist. - return os.path.exists(self._bin("python")) and os.path.exists(self._bin("pip")) + return os.path.exists(self.python) and os.path.exists(self._bin("pip")) def _run(self, cmd, **kwargs): with self.temp_environ(): diff --git a/poetry/utils/helpers.py b/poetry/utils/helpers.py index e6162d508db..232e65b7d44 100644 --- a/poetry/utils/helpers.py +++ b/poetry/utils/helpers.py @@ -118,3 +118,18 @@ def get_package_version_display_string( def paths_csv(paths): # type: (List[Path]) -> str return ", ".join('"{}"'.format(str(c)) for c in paths) + + +def is_dir_writable(path, create=False): # type: (Path, bool) -> bool + try: + if not path.exists(): + if not create: + return False + path.mkdir(parents=True, exist_ok=True) + + with tempfile.TemporaryFile(dir=str(path)): + pass + except (IOError, OSError): + return False + else: + return True From fea47ecd657f53cc6da0376a221e7077314c47b8 Mon Sep 17 00:00:00 2001 From: Arun Babu Neelicattu Date: Fri, 23 Oct 2020 03:34:53 +0200 Subject: [PATCH 04/10] utils/env: ensure user directories are created --- poetry/masonry/builders/editable.py | 2 +- poetry/utils/env.py | 2 +- tests/utils/test_env_site.py | 4 +++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/poetry/masonry/builders/editable.py b/poetry/masonry/builders/editable.py index 0493423cdd2..74d1f69c886 100644 --- a/poetry/masonry/builders/editable.py +++ b/poetry/masonry/builders/editable.py @@ -131,7 +131,7 @@ def _add_scripts(self): entry_points = self.convert_entry_points() for scripts_path in self._env.script_dirs: - if is_dir_writable(scripts_path): + if is_dir_writable(path=scripts_path, create=True): break else: self._io.error_line( diff --git a/poetry/utils/env.py b/poetry/utils/env.py index 33886926d4b..c247bf014f7 100644 --- a/poetry/utils/env.py +++ b/poetry/utils/env.py @@ -170,7 +170,7 @@ def writable_candidates(self): # type: () -> List[Path] self._writable_candidates = [] for candidate in self._candidates: - if not is_dir_writable(candidate): + if not is_dir_writable(path=candidate, create=True): continue self._writable_candidates.append(candidate) diff --git a/tests/utils/test_env_site.py b/tests/utils/test_env_site.py index a13089160e6..f25e2142193 100644 --- a/tests/utils/test_env_site.py +++ b/tests/utils/test_env_site.py @@ -5,7 +5,9 @@ from poetry.utils.env import SitePackages -def test_env_site_simple(tmp_dir): +def test_env_site_simple(tmp_dir, mocker): + # emulate permission error when creating directory + mocker.patch("poetry.utils._compat.Path.mkdir", side_effect=OSError()) site_packages = SitePackages(Path("/non-existent"), fallbacks=[Path(tmp_dir)]) candidates = site_packages.make_candidates(Path("hello.txt"), writable_only=True) hello = Path(tmp_dir) / "hello.txt" From 8099f9162ceeef5c9374dc06ba88016aad92c45b Mon Sep 17 00:00:00 2001 From: Arun Babu Neelicattu Date: Thu, 15 Oct 2020 01:52:30 +0200 Subject: [PATCH 05/10] Handle non-editable packages with pth files When detecting installed packages, this change ensures that packages with .pth files are not incorrectly marked as editable. A package is considered editable only if at least one of the paths detected is not in the environment site. Resolves: #3077 --- poetry/repositories/installed_repository.py | 12 ++++++++-- .../standard-1.2.3.dist-info/METADATA | 22 +++++++++++++++++++ .../lib/python3.7/site-packages/standard.pth | 1 + .../repositories/test_installed_repository.py | 11 ++++++++++ 4 files changed, 44 insertions(+), 2 deletions(-) create mode 100644 tests/repositories/fixtures/installed/lib/python3.7/site-packages/standard-1.2.3.dist-info/METADATA create mode 100644 tests/repositories/fixtures/installed/lib/python3.7/site-packages/standard.pth diff --git a/poetry/repositories/installed_repository.py b/poetry/repositories/installed_repository.py index a0630116f53..1320fdd6698 100644 --- a/poetry/repositories/installed_repository.py +++ b/poetry/repositories/installed_repository.py @@ -138,14 +138,22 @@ def load(cls, env): # type: (Env) -> InstalledRepository if path.name.endswith(".dist-info"): paths = cls.get_package_paths(env=env, name=package.pretty_name) if paths: + is_editable_package = False for src in paths: if cls.is_vcs_package(src, env): cls.set_package_vcs_properties(package, env) break + + if not ( + is_editable_package + or env.is_path_relative_to_lib(src) + ): + is_editable_package = True else: # TODO: handle multiple source directories? - package._source_type = "directory" - package._source_url = paths.pop().as_posix() + if is_editable_package: + package._source_type = "directory" + package._source_url = paths.pop().as_posix() continue if cls.is_vcs_package(path, env): diff --git a/tests/repositories/fixtures/installed/lib/python3.7/site-packages/standard-1.2.3.dist-info/METADATA b/tests/repositories/fixtures/installed/lib/python3.7/site-packages/standard-1.2.3.dist-info/METADATA new file mode 100644 index 00000000000..245121d496d --- /dev/null +++ b/tests/repositories/fixtures/installed/lib/python3.7/site-packages/standard-1.2.3.dist-info/METADATA @@ -0,0 +1,22 @@ +Metadata-Version: 2.1 +Name: standard +Version: 1.2.3 +Summary: Standard description. +License: MIT +Keywords: cli,commands +Author: Foo Bar +Author-email: foo@bar.com +Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.* +Classifier: License :: OSI Approved :: MIT License +Classifier: Programming Language :: Python :: 2 +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.4 +Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: 3.7 +Classifier: Programming Language :: Python :: 3.8 +Description-Content-Type: text/x-rst + +Editable +#### diff --git a/tests/repositories/fixtures/installed/lib/python3.7/site-packages/standard.pth b/tests/repositories/fixtures/installed/lib/python3.7/site-packages/standard.pth new file mode 100644 index 00000000000..aa0bc074b62 --- /dev/null +++ b/tests/repositories/fixtures/installed/lib/python3.7/site-packages/standard.pth @@ -0,0 +1 @@ +standard \ No newline at end of file diff --git a/tests/repositories/test_installed_repository.py b/tests/repositories/test_installed_repository.py index 79ac262671c..3caa702a09c 100644 --- a/tests/repositories/test_installed_repository.py +++ b/tests/repositories/test_installed_repository.py @@ -26,6 +26,7 @@ zipp.Path(str(SITE_PURELIB / "foo-0.1.0-py3.8.egg"), "EGG-INFO") ), metadata.PathDistribution(VENDOR_DIR / "attrs-19.3.0.dist-info"), + metadata.PathDistribution(SITE_PURELIB / "standard-1.2.3.dist-info"), metadata.PathDistribution(SITE_PURELIB / "editable-2.3.4.dist-info"), metadata.PathDistribution(SITE_PURELIB / "editable-with-import-2.3.4.dist-info"), metadata.PathDistribution(SITE_PLATLIB / "lib64-2.3.4.dist-info"), @@ -158,3 +159,13 @@ def test_load_editable_with_import_package(repository): assert editable.version.text == "2.3.4" assert editable.source_type is None assert editable.source_url is None + + +def test_load_standard_package_with_pth_file(repository): + # test standard packages with .pth file is not treated as editable + standard = get_package_from_repository("standard", repository) + assert standard is not None + assert standard.name == "standard" + assert standard.version.text == "1.2.3" + assert standard.source_type is None + assert standard.source_url is None From ff3e57e91e16475c31442cec5b0bc87ee533c175 Mon Sep 17 00:00:00 2001 From: Arun Babu Neelicattu Date: Fri, 16 Oct 2020 23:42:10 +0200 Subject: [PATCH 06/10] locker: handle nested extras requirement Previously, when using locked repository, incorrect dependency instance was created when a dependency's extra requirement activated a nested extra. This change ensures that these are correctly loaded. As part of this change new lock files write PEP 508 serialised form of extra dependencies in order to reuse core logic to parse specification of extra requirement. Resolves: #3224 --- poetry/packages/locker.py | 23 +++- .../fixtures/with-dependencies-extras.test | 2 +- .../with-dependencies-nested-extras.test | 45 ++++++ tests/installation/test_installer.py | 29 ++++ tests/packages/test_locker.py | 130 ++++++++++++++++++ 5 files changed, 222 insertions(+), 7 deletions(-) create mode 100644 tests/installation/fixtures/with-dependencies-nested-extras.test diff --git a/poetry/packages/locker.py b/poetry/packages/locker.py index 9dd75e66519..ac791537450 100644 --- a/poetry/packages/locker.py +++ b/poetry/packages/locker.py @@ -24,12 +24,14 @@ import poetry.repositories +from poetry.core.packages import dependency_from_pep_508 from poetry.core.packages.package import Dependency from poetry.core.packages.package import Package from poetry.core.semver import parse_constraint from poetry.core.semver.version import Version from poetry.core.toml.file import TOMLFile from poetry.core.version.markers import parse_marker +from poetry.core.version.requirements import InvalidRequirement from poetry.packages import DependencyPackage from poetry.utils._compat import OrderedDict from poetry.utils._compat import Path @@ -142,11 +144,18 @@ def locked_repository( package.extras[name] = [] for dep in deps: - m = re.match(r"^(.+?)(?:\s+\((.+)\))?$", dep) - dep_name = m.group(1) - constraint = m.group(2) or "*" - - package.extras[name].append(Dependency(dep_name, constraint)) + try: + dependency = dependency_from_pep_508(dep) + except InvalidRequirement: + # handle lock files with invalid PEP 508 + m = re.match(r"^(.+?)(?:\[(.+?)])?(?:\s+\((.+)\))?$", dep) + dep_name = m.group(1) + extras = m.group(2) or "" + constraint = m.group(3) or "*" + dependency = Dependency( + dep_name, constraint, extras=extras.split(",") + ) + package.extras[name].append(dependency) if "marker" in info: package.marker = parse_marker(info["marker"]) @@ -543,8 +552,10 @@ def _dump_package(self, package): # type: (Package) -> dict if package.extras: extras = {} for name, deps in package.extras.items(): + # TODO: This should use dep.to_pep_508() once this is fixed + # https://github.com/python-poetry/poetry-core/pull/102 extras[name] = [ - str(dep) if not dep.constraint.is_any() else dep.name + dep.base_pep_508_name if not dep.constraint.is_any() else dep.name for dep in deps ] diff --git a/tests/installation/fixtures/with-dependencies-extras.test b/tests/installation/fixtures/with-dependencies-extras.test index 63560bb4793..042e29670e1 100644 --- a/tests/installation/fixtures/with-dependencies-extras.test +++ b/tests/installation/fixtures/with-dependencies-extras.test @@ -18,7 +18,7 @@ python-versions = "*" C = {version = "^1.0", optional = true} [package.extras] -foo = ["C (^1.0)"] +foo = ["C (>=1.0,<2.0)"] [[package]] name = "C" diff --git a/tests/installation/fixtures/with-dependencies-nested-extras.test b/tests/installation/fixtures/with-dependencies-nested-extras.test new file mode 100644 index 00000000000..48a22a7c7f3 --- /dev/null +++ b/tests/installation/fixtures/with-dependencies-nested-extras.test @@ -0,0 +1,45 @@ +[[package]] +name = "A" +version = "1.0" +description = "" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +B = {version = "^1.0", optional = true, extras = ["C"]} + +[package.extras] +B = ["B[C] (>=1.0,<2.0)"] + +[[package]] +name = "B" +version = "1.0" +description = "" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +C = {version = "^1.0", optional = true} + +[package.extras] +C = ["C (>=1.0,<2.0)"] + +[[package]] +name = "C" +version = "1.0" +description = "" +category = "main" +optional = false +python-versions = "*" + +[metadata] +python-versions = "*" +lock-version = "1.1" +content-hash = "123456789" + +[metadata.files] +"A" = [] +"B" = [] +"C" = [] diff --git a/tests/installation/test_installer.py b/tests/installation/test_installer.py index 7a1660670ee..106efde6e9c 100644 --- a/tests/installation/test_installer.py +++ b/tests/installation/test_installer.py @@ -639,6 +639,35 @@ def test_run_with_dependencies_extras(installer, locker, repo, package): assert locker.written_data == expected +def test_run_with_dependencies_nested_extras(installer, locker, repo, package): + package_a = get_package("A", "1.0") + package_b = get_package("B", "1.0") + package_c = get_package("C", "1.0") + + dependency_c = Factory.create_dependency("C", {"version": "^1.0", "optional": True}) + dependency_b = Factory.create_dependency( + "B", {"version": "^1.0", "optional": True, "extras": ["C"]} + ) + dependency_a = Factory.create_dependency("A", {"version": "^1.0", "extras": ["B"]}) + + package_b.extras = {"C": [dependency_c]} + package_b.add_dependency(dependency_c) + + package_a.add_dependency(dependency_b) + package_a.extras = {"B": [dependency_b]} + + repo.add_package(package_a) + repo.add_package(package_b) + repo.add_package(package_c) + + package.add_dependency(dependency_a) + + installer.run() + expected = fixture("with-dependencies-nested-extras") + + assert locker.written_data == expected + + def test_run_does_not_install_extras_if_not_requested(installer, locker, repo, package): package.extras["foo"] = [get_dependency("D")] package_a = get_package("A", "1.0") diff --git a/tests/packages/test_locker.py b/tests/packages/test_locker.py index 35c584f2767..a4aa17971f8 100644 --- a/tests/packages/test_locker.py +++ b/tests/packages/test_locker.py @@ -142,6 +142,136 @@ def test_locker_properly_loads_extras(locker): assert lockfile_dep.name == "lockfile" +def test_locker_properly_loads_nested_extras(locker): + content = """\ +[[package]] +name = "a" +version = "1.0" +description = "" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +b = {version = "^1.0", optional = true, extras = "c"} + +[package.extras] +b = ["b[c] (>=1.0,<2.0)"] + +[[package]] +name = "b" +version = "1.0" +description = "" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +c = {version = "^1.0", optional = true} + +[package.extras] +c = ["c (>=1.0,<2.0)"] + +[[package]] +name = "c" +version = "1.0" +description = "" +category = "main" +optional = false +python-versions = "*" + +[metadata] +python-versions = "*" +lock-version = "1.1" +content-hash = "123456789" + +[metadata.files] +"a" = [] +"b" = [] +"c" = [] +""" + + locker.lock.write(tomlkit.parse(content)) + + repository = locker.locked_repository() + assert 3 == len(repository.packages) + + packages = repository.find_packages(get_dependency("a", "1.0")) + assert len(packages) == 1 + + package = packages[0] + assert len(package.requires) == 1 + assert len(package.extras) == 1 + + dependency_b = package.extras["b"][0] + assert dependency_b.name == "b" + assert dependency_b.extras == frozenset({"c"}) + + packages = repository.find_packages(dependency_b) + assert len(packages) == 1 + + package = packages[0] + assert len(package.requires) == 1 + assert len(package.extras) == 1 + + dependency_c = package.extras["c"][0] + assert dependency_c.name == "c" + assert dependency_c.extras == frozenset() + + packages = repository.find_packages(dependency_c) + assert len(packages) == 1 + + +def test_locker_properly_loads_extras_legacy(locker): + content = """\ +[[package]] +name = "a" +version = "1.0" +description = "" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +b = {version = "^1.0", optional = true} + +[package.extras] +b = ["b (^1.0)"] + +[[package]] +name = "b" +version = "1.0" +description = "" +category = "main" +optional = false +python-versions = "*" + +[metadata] +python-versions = "*" +lock-version = "1.1" +content-hash = "123456789" + +[metadata.files] +"a" = [] +"b" = [] +""" + + locker.lock.write(tomlkit.parse(content)) + + repository = locker.locked_repository() + assert 2 == len(repository.packages) + + packages = repository.find_packages(get_dependency("a", "1.0")) + assert len(packages) == 1 + + package = packages[0] + assert len(package.requires) == 1 + assert len(package.extras) == 1 + + dependency_b = package.extras["b"][0] + assert dependency_b.name == "b" + + def test_lock_packages_with_null_description(locker, root): package_a = get_package("A", "1.0.0") package_a.description = None From 3f4ad8397476797011df3835d6a1b26dea345b8b Mon Sep 17 00:00:00 2001 From: Arun Babu Neelicattu Date: Sun, 18 Oct 2020 00:14:48 +0200 Subject: [PATCH 07/10] locker: handle cyclic dependencies during walk Resolves: #3213 --- poetry/packages/locker.py | 52 ++++++++++++++++----------------- tests/utils/test_exporter.py | 56 ++++++++++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+), 27 deletions(-) diff --git a/poetry/packages/locker.py b/poetry/packages/locker.py index ac791537450..d0d03a56779 100644 --- a/poetry/packages/locker.py +++ b/poetry/packages/locker.py @@ -226,45 +226,43 @@ def __walk_dependency_level( next_level_dependencies = [] for requirement in dependencies: + key = (requirement.name, requirement.pretty_constraint) locked_package = cls.__get_locked_package(requirement, packages_by_name) if locked_package: - for require in locked_package.requires: - if require.marker.is_empty(): - require.marker = requirement.marker - else: - require.marker = require.marker.intersect(requirement.marker) + # create dependency from locked package to retain dependency metadata + # if this is not done, we can end-up with incorrect nested dependencies + marker = requirement.marker + requirement = locked_package.to_dependency() + requirement.marker = requirement.marker.intersect(marker) + + key = (requirement.name, requirement.pretty_constraint) + + if pinned_versions: + requirement.set_constraint( + locked_package.to_dependency().constraint + ) + + if key not in nested_dependencies: + for require in locked_package.requires: + if require.marker.is_empty(): + require.marker = requirement.marker + else: + require.marker = require.marker.intersect( + requirement.marker + ) - require.marker = require.marker.intersect(locked_package.marker) - next_level_dependencies.append(require) + require.marker = require.marker.intersect(locked_package.marker) + next_level_dependencies.append(require) if requirement.name in project_level_dependencies and level == 0: # project level dependencies take precedence continue - if locked_package: - # create dependency from locked package to retain dependency metadata - # if this is not done, we can end-up with incorrect nested dependencies - marker = requirement.marker - requirement = locked_package.to_dependency() - requirement.marker = requirement.marker.intersect(marker) - else: + if not locked_package: # we make a copy to avoid any side-effects requirement = deepcopy(requirement) - if pinned_versions: - requirement.set_constraint( - cls.__get_locked_package(requirement, packages_by_name) - .to_dependency() - .constraint - ) - - # dependencies use extra to indicate that it was activated via parent - # package's extras, this is not required for nested exports as we assume - # the resolver already selected this dependency - requirement.marker = requirement.marker.without_extras() - - key = (requirement.name, requirement.pretty_constraint) if key not in nested_dependencies: nested_dependencies[key] = requirement else: diff --git a/tests/utils/test_exporter.py b/tests/utils/test_exporter.py index c5ce0214ff7..1e660ce69cd 100644 --- a/tests/utils/test_exporter.py +++ b/tests/utils/test_exporter.py @@ -696,6 +696,62 @@ def test_exporter_can_export_requirements_txt_with_nested_packages(tmp_dir, poet assert expected == content +def test_exporter_can_export_requirements_txt_with_nested_packages_cyclic( + tmp_dir, poetry +): + poetry.locker.mock_lock_data( + { + "package": [ + { + "name": "foo", + "version": "1.2.3", + "category": "main", + "optional": False, + "python-versions": "*", + "dependencies": {"bar": {"version": "4.5.6"}}, + }, + { + "name": "bar", + "version": "4.5.6", + "category": "main", + "optional": False, + "python-versions": "*", + "dependencies": {"baz": {"version": "7.8.9"}}, + }, + { + "name": "baz", + "version": "7.8.9", + "category": "main", + "optional": False, + "python-versions": "*", + "dependencies": {"foo": {"version": "1.2.3"}}, + }, + ], + "metadata": { + "python-versions": "*", + "content-hash": "123456789", + "hashes": {"foo": [], "bar": [], "baz": []}, + }, + } + ) + set_package_requires(poetry, skip={"bar", "baz"}) + + exporter = Exporter(poetry) + + exporter.export("requirements.txt", Path(tmp_dir), "requirements.txt") + + with (Path(tmp_dir) / "requirements.txt").open(encoding="utf-8") as f: + content = f.read() + + expected = """\ +bar==4.5.6 +baz==7.8.9 +foo==1.2.3 +""" + + assert expected == content + + def test_exporter_can_export_requirements_txt_with_git_packages_and_markers( tmp_dir, poetry ): From 36b8c67620e09a2e22e9e1bf85fb5b1a2c77c6f3 Mon Sep 17 00:00:00 2001 From: Arun Babu Neelicattu Date: Tue, 20 Oct 2020 01:42:39 +0200 Subject: [PATCH 08/10] pool: ensure sources are prioritised over PyPI When a project specifies non default sources, PyPI gets added as the default source. This will prioritise packages available in PyPI when the package exists in both index. This change ensures that PyPI is only used as a default when no other sources are provided. Resolves: #1677 #2564 #3238 --- poetry/factory.py | 6 +++-- .../with_non_default_source/pyproject.toml | 18 +++++++++++++ tests/test_factory.py | 27 +++++++++++++++++++ 3 files changed, 49 insertions(+), 2 deletions(-) create mode 100644 tests/fixtures/with_non_default_source/pyproject.toml diff --git a/poetry/factory.py b/poetry/factory.py index fd863ba5991..08a6faca170 100644 --- a/poetry/factory.py +++ b/poetry/factory.py @@ -70,7 +70,8 @@ def create_poetry( ) # Configuring sources - for source in poetry.local_config.get("source", []): + sources = poetry.local_config.get("source", []) + for source in sources: repository = self.create_legacy_repository(source, config) is_default = source.get("default", False) is_secondary = source.get("secondary", False) @@ -90,7 +91,8 @@ def create_poetry( # Always put PyPI last to prefer private repositories # but only if we have no other default source if not poetry.pool.has_default(): - poetry.pool.add_repository(PyPiRepository(), True) + has_sources = bool(sources) + poetry.pool.add_repository(PyPiRepository(), not has_sources, has_sources) else: if io.is_debug(): io.write_line("Deactivating the PyPI repository") diff --git a/tests/fixtures/with_non_default_source/pyproject.toml b/tests/fixtures/with_non_default_source/pyproject.toml new file mode 100644 index 00000000000..d36abb55a25 --- /dev/null +++ b/tests/fixtures/with_non_default_source/pyproject.toml @@ -0,0 +1,18 @@ +[tool.poetry] +name = "my-package" +version = "1.2.3" +description = "Some description." +authors = [ + "Your Name " +] +license = "MIT" + +# Requirements +[tool.poetry.dependencies] +python = "~2.7 || ^3.6" + +[tool.poetry.dev-dependencies] + +[[tool.poetry.source]] +name = "foo" +url = "https://foo.bar/simple/" diff --git a/tests/test_factory.py b/tests/test_factory.py index d7758e8deaa..b2c232b20e7 100644 --- a/tests/test_factory.py +++ b/tests/test_factory.py @@ -6,6 +6,8 @@ from poetry.core.toml.file import TOMLFile from poetry.factory import Factory +from poetry.repositories.legacy_repository import LegacyRepository +from poetry.repositories.pypi_repository import PyPiRepository from poetry.utils._compat import PY2 from poetry.utils._compat import Path @@ -150,6 +152,31 @@ def test_poetry_with_default_source(): assert 1 == len(poetry.pool.repositories) +def test_poetry_with_non_default_source(): + poetry = Factory().create_poetry(fixtures_dir / "with_non_default_source") + + assert len(poetry.pool.repositories) == 2 + + assert not poetry.pool.has_default() + + assert poetry.pool.repositories[0].name == "foo" + assert isinstance(poetry.pool.repositories[0], LegacyRepository) + + assert poetry.pool.repositories[1].name == "PyPI" + assert isinstance(poetry.pool.repositories[1], PyPiRepository) + + +def test_poetry_with_no_default_source(): + poetry = Factory().create_poetry(fixtures_dir / "sample_project") + + assert len(poetry.pool.repositories) == 1 + + assert poetry.pool.has_default() + + assert poetry.pool.repositories[0].name == "PyPI" + assert isinstance(poetry.pool.repositories[0], PyPiRepository) + + def test_poetry_with_two_default_sources(): with pytest.raises(ValueError) as e: Factory().create_poetry(fixtures_dir / "with_two_default_sources") From 4dced2c66ba0a982924245a636ed61990a2539a1 Mon Sep 17 00:00:00 2001 From: Arun Babu Neelicattu Date: Wed, 21 Oct 2020 21:59:42 +0200 Subject: [PATCH 09/10] Ensure vcs dependencies preserve editable flag Resolves: #3263 --- poetry/packages/locker.py | 2 +- poetry/puzzle/provider.py | 1 + tests/packages/test_locker.py | 1 + tests/puzzle/test_provider.py | 9 +++++++++ 4 files changed, 12 insertions(+), 1 deletion(-) diff --git a/poetry/packages/locker.py b/poetry/packages/locker.py index d0d03a56779..f1637407068 100644 --- a/poetry/packages/locker.py +++ b/poetry/packages/locker.py @@ -582,7 +582,7 @@ def _dump_package(self, package): # type: (Package) -> dict if package.source_resolved_reference: data["source"]["resolved_reference"] = package.source_resolved_reference - if package.source_type == "directory": + if package.source_type in ["directory", "git"]: data["develop"] = package.develop return data diff --git a/poetry/puzzle/provider.py b/poetry/puzzle/provider.py index e2838370a79..c05efbd6844 100644 --- a/poetry/puzzle/provider.py +++ b/poetry/puzzle/provider.py @@ -168,6 +168,7 @@ def search_for_vcs(self, dependency): # type: (VCSDependency) -> List[Package] rev=dependency.rev, name=dependency.name, ) + package.develop = dependency.develop dependency._constraint = package.version dependency._pretty_constraint = package.version.text diff --git a/tests/packages/test_locker.py b/tests/packages/test_locker.py index a4aa17971f8..7fa89a44962 100644 --- a/tests/packages/test_locker.py +++ b/tests/packages/test_locker.py @@ -73,6 +73,7 @@ def test_lock_file_data_is_ordered(locker, root): category = "main" optional = false python-versions = "*" +develop = true [package.source] type = "git" diff --git a/tests/puzzle/test_provider.py b/tests/puzzle/test_provider.py index 693470156f4..ecab7f3ab2d 100644 --- a/tests/puzzle/test_provider.py +++ b/tests/puzzle/test_provider.py @@ -47,6 +47,15 @@ def provider(root, pool): return Provider(root, pool, NullIO()) +@pytest.mark.parametrize("value", [True, False]) +def test_search_for_vcs_retains_develop_flag(provider, value): + dependency = VCSDependency( + "demo", "git", "https://github.com/demo/demo.git", develop=value + ) + package = provider.search_for_vcs(dependency)[0] + assert package.develop == value + + def test_search_for_vcs_setup_egg_info(provider): dependency = VCSDependency("demo", "git", "https://github.com/demo/demo.git") From ab3e00f010f461a6efc3818f0dd0a1741fc3151d Mon Sep 17 00:00:00 2001 From: Arun Babu Neelicattu Date: Fri, 23 Oct 2020 23:08:33 +0200 Subject: [PATCH 10/10] Add 1.1.4 changelog --- CHANGELOG.md | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ec01b75abde..168944d8f4b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,25 @@ # Change Log +## [1.1.4] - 2020-10-23 + +### Added + +- Added `installer.parallel` boolean flag (defaults to `true`) configuration to enable/disable parallel execution of operations when using the new installer. ([#3088](https://github.com/python-poetry/poetry/pull/3088)) + +### Changed + +- When using system environments as an unprivileged user, user site and bin directories are created if they do not already exist. ([#3107](https://github.com/python-poetry/poetry/pull/3107)) + +### Fixed + +- Fixed editable installation of poetry projects when using system environments. ([#3107](https://github.com/python-poetry/poetry/pull/3107)) +- Fixed locking of nested extra activations. If you were affected by this issue, you will need to regenerate the lock file using `poetry lock --no-update`. ([#3229](https://github.com/python-poetry/poetry/pull/3229)) +- Fixed prioritisation of non-default custom package sources. ([#3251](https://github.com/python-poetry/poetry/pull/3251)) +- Fixed detection of installed editable packages when non-poetry managed `.pth` file exists. ([#3210](https://github.com/python-poetry/poetry/pull/3210)) +- Fixed scripts generated by editable builder to use valid import statements. ([#3214](https://github.com/python-poetry/poetry/pull/3214)) +- Fixed recursion error when locked dependencies contain cyclic dependencies. ([#3237](https://github.com/python-poetry/poetry/pull/3237)) +- Fixed propagation of editable flag for VCS dependencies. ([#3264](https://github.com/python-poetry/poetry/pull/3264)) + ## [1.1.3] - 2020-10-14 ### Changed @@ -1063,7 +1083,8 @@ Initial release -[Unreleased]: https://github.com/python-poetry/poetry/compare/1.1.3...master +[Unreleased]: https://github.com/python-poetry/poetry/compare/1.1.4...master +[1.1.3]: https://github.com/python-poetry/poetry/compare/1.1.4 [1.1.3]: https://github.com/python-poetry/poetry/compare/1.1.3 [1.1.2]: https://github.com/python-poetry/poetry/releases/tag/1.1.2 [1.1.1]: https://github.com/python-poetry/poetry/releases/tag/1.1.1