Release 1.1.4 forward port to master

CHANGELOG.md

@@ -1,5 +1,25 @@
 # Change Log
 
+## [1.1.4] - 2020-10-23
+
+### Added
+
+- Added `installer.parallel` boolean flag (defaults to `true`) configuration to enable/disable parallel execution of operations when using the new installer. ([#3088](https://github.com/python-poetry/poetry/pull/3088))
+
+### Changed
+
+- When using system environments as an unprivileged user, user site and bin directories are created if they do not already exist. ([#3107](https://github.com/python-poetry/poetry/pull/3107))
+
+### Fixed
+
+- Fixed editable installation of poetry projects when using system environments. ([#3107](https://github.com/python-poetry/poetry/pull/3107))
+- Fixed locking of nested extra activations. If you were affected by this issue, you will need to regenerate the lock file using `poetry lock --no-update`.  ([#3229](https://github.com/python-poetry/poetry/pull/3229))
+- Fixed prioritisation of non-default custom package sources. ([#3251](https://github.com/python-poetry/poetry/pull/3251))
+- Fixed detection of installed editable packages when non-poetry managed `.pth` file exists. ([#3210](https://github.com/python-poetry/poetry/pull/3210))
+- Fixed scripts generated by editable builder to use valid import statements. ([#3214](https://github.com/python-poetry/poetry/pull/3214))
+- Fixed recursion error when locked dependencies contain cyclic dependencies. ([#3237](https://github.com/python-poetry/poetry/pull/3237))
+- Fixed propagation of editable flag for VCS dependencies. ([#3264](https://github.com/python-poetry/poetry/pull/3264))
+
 ## [1.1.3] - 2020-10-14
 
 ### Changed
@@ -1063,7 +1083,8 @@ Initial release
 
 
 
-[Unreleased]: https://github.com/python-poetry/poetry/compare/1.1.3...master
+[Unreleased]: https://github.com/python-poetry/poetry/compare/1.1.4...master
+[1.1.3]: https://github.com/python-poetry/poetry/compare/1.1.4
 [1.1.3]: https://github.com/python-poetry/poetry/compare/1.1.3
 [1.1.2]: https://github.com/python-poetry/poetry/releases/tag/1.1.2
 [1.1.1]: https://github.com/python-poetry/poetry/releases/tag/1.1.1

docs/docs/configuration.md

@@ -103,6 +103,15 @@ Defaults to one of the following directories:
 - Windows: `C:\Users\<username>\AppData\Local\pypoetry\Cache`
 - Unix:    `~/.cache/pypoetry`
 
+### `installer.parallel`: boolean
+
+Use parallel execution when using the new (`>=1.1.0`) installer.
+Defaults to `true`.
+
+!!!note:
+        This configuration will be ignored, and parallel execution disabled when running
+        Python 2.7 under Windows.
+
 ### `virtualenvs.create`: boolean
 
 Create a new virtual environment if one doesn't already exist.

poetry/config/config.py

@@ -39,6 +39,7 @@ class Config(object):
             "options": {"always-copy": False},
         },
         "experimental": {"new-installer": True},
+        "installer": {"parallel": True},
     }
 
     def __init__(
@@ -140,6 +141,7 @@ def _get_validator(self, name):  # type: (str) -> Callable
             "virtualenvs.create",
             "virtualenvs.in-project",
             "virtualenvs.options.always-copy",
+            "installer.parallel",
         }:
             return boolean_validator
 
@@ -151,6 +153,7 @@ def _get_normalizer(self, name):  # type: (str) -> Callable
             "virtualenvs.create",
             "virtualenvs.in-project",
             "virtualenvs.options.always-copy",
+            "installer.parallel",
         }:
             return boolean_normalizer
 

poetry/console/commands/config.py

@@ -69,6 +69,7 @@ def unique_config_values(self):
                 boolean_normalizer,
                 False,
             ),
+            "installer.parallel": (boolean_validator, boolean_normalizer, True,),
         }
 
         return unique_config_values

poetry/factory.py

@@ -70,7 +70,8 @@ def create_poetry(
         )
 
         # Configuring sources
-        for source in poetry.local_config.get("source", []):
+        sources = poetry.local_config.get("source", [])
+        for source in sources:
             repository = self.create_legacy_repository(source, config)
             is_default = source.get("default", False)
             is_secondary = source.get("secondary", False)
@@ -90,7 +91,8 @@ def create_poetry(
         # Always put PyPI last to prefer private repositories
         # but only if we have no other default source
         if not poetry.pool.has_default():
-            poetry.pool.add_repository(PyPiRepository(), True)
+            has_sources = bool(sources)
+            poetry.pool.add_repository(PyPiRepository(), not has_sources, has_sources)
         else:
             if io.is_debug():
                 io.write_line("Deactivating the PyPI repository")

poetry/installation/executor.py

@@ -32,7 +32,7 @@
 
 
 class Executor(object):
-    def __init__(self, env, pool, config, io, parallel=True):
+    def __init__(self, env, pool, config, io, parallel=None):
         self._env = env
         self._io = io
         self._dry_run = False
@@ -42,6 +42,9 @@ def __init__(self, env, pool, config, io, parallel=True):
         self._chef = Chef(config, self._env)
         self._chooser = Chooser(pool, self._env)
 
+        if parallel is None:
+            parallel = config.get("installer.parallel", True)
+
         if parallel and not (PY2 and WINDOWS):
             # This should be directly handled by ThreadPoolExecutor
             # however, on some systems the number of CPUs cannot be determined

poetry/installation/pip_installer.py

@@ -113,7 +113,9 @@ def remove(self, package):
             raise
 
         # This is a workaround for https://github.com/pypa/pip/issues/4176
-        nspkg_pth_file = self._env.site_packages / "{}-nspkg.pth".format(package.name)
+        nspkg_pth_file = self._env.site_packages.path / "{}-nspkg.pth".format(
+            package.name
+        )
         if nspkg_pth_file.exists():
             nspkg_pth_file.unlink()
 

poetry/masonry/builders/editable.py

@@ -13,6 +13,7 @@
 from poetry.utils._compat import WINDOWS
 from poetry.utils._compat import Path
 from poetry.utils._compat import decode
+from poetry.utils.helpers import is_dir_writable
 
 
 SCRIPT_TEMPLATE = """\
@@ -94,7 +95,6 @@ def _setup_build(self):
                 os.remove(str(setup))
 
     def _add_pth(self):
-        pth_file = Path(self._module.name).with_suffix(".pth")
         paths = set()
         for include in self._module.includes:
             if isinstance(include, PackageInclude) and (
@@ -106,34 +106,40 @@ def _add_pth(self):
         for path in paths:
             content += decode(path + os.linesep)
 
-        for site_package in [self._env.site_packages, self._env.usersite]:
-            if not site_package:
-                continue
-
-            try:
-                site_package.mkdir(parents=True, exist_ok=True)
-                path = site_package.joinpath(pth_file)
-                self._debug(
-                    "  - Adding <c2>{}</c2> to <b>{}</b> for {}".format(
-                        path.name, site_package, self._poetry.file.parent
-                    )
+        pth_file = Path(self._module.name).with_suffix(".pth")
+        try:
+            pth_file = self._env.site_packages.write_text(
+                pth_file, content, encoding="utf-8"
+            )
+            self._debug(
+                "  - Adding <c2>{}</c2> to <b>{}</b> for {}".format(
+                    pth_file.name, pth_file.parent, self._poetry.file.parent
                 )
-                path.write_text(content, encoding="utf-8")
-                return [path]
-            except PermissionError:
-                self._debug("- <b>{}</b> is not writable trying next available site")
-
-        self._io.error_line(
-            "  - Failed to create <c2>{}</c2> for {}".format(
-                pth_file.name, self._poetry.file.parent
             )
-        )
-        return []
+            return [pth_file]
+        except OSError:
+            # TODO: Replace with PermissionError
+            self._io.error_line(
+                "  - Failed to create <c2>{}</c2> for {}".format(
+                    pth_file.name, self._poetry.file.parent
+                )
+            )
+            return []
 
     def _add_scripts(self):
         added = []
         entry_points = self.convert_entry_points()
-        scripts_path = Path(self._env.paths["scripts"])
+
+        for scripts_path in self._env.script_dirs:
+            if is_dir_writable(path=scripts_path, create=True):
+                break
+        else:
+            self._io.error_line(
+                "  - Failed to find a suitable script installation directory for {}".format(
+                    self._poetry.file.parent
+                )
+            )
+            return []
 
         scripts = entry_points.get("console_scripts", [])
         for script in scripts:
@@ -151,7 +157,7 @@ def _add_scripts(self):
                 f.write(
                     decode(
                         SCRIPT_TEMPLATE.format(
-                            python=self._env._bin("python"),
+                            python=self._env.python,
                             module=module,
                             callable_holder=callable_holder,
                             callable_=callable_,
@@ -165,9 +171,7 @@ def _add_scripts(self):
 
             if WINDOWS:
                 cmd_script = script_file.with_suffix(".cmd")
-                cmd = WINDOWS_CMD_TEMPLATE.format(
-                    python=self._env._bin("python"), script=name
-                )
+                cmd = WINDOWS_CMD_TEMPLATE.format(python=self._env.python, script=name)
                 self._debug(
                     "  - Adding the <c2>{}</c2> script wrapper to <b>{}</b>".format(
                         cmd_script.name, scripts_path
@@ -187,19 +191,27 @@ def _add_dist_info(self, added_files):
         added_files = added_files[:]
 
         builder = WheelBuilder(self._poetry)
-        dist_info = self._env.site_packages.joinpath(builder.dist_info)
+
+        dist_info_path = Path(builder.dist_info)
+        for dist_info in self._env.site_packages.find(
+            dist_info_path, writable_only=True
+        ):
+            if dist_info.exists():
+                self._debug(
+                    "  - Removing existing <c2>{}</c2> directory from <b>{}</b>".format(
+                        dist_info.name, dist_info.parent
+                    )
+                )
+                shutil.rmtree(str(dist_info))
+
+        dist_info = self._env.site_packages.mkdir(dist_info_path)
 
         self._debug(
             "  - Adding the <c2>{}</c2> directory to <b>{}</b>".format(
-                dist_info.name, self._env.site_packages
+                dist_info.name, dist_info.parent
             )
         )
 
-        if dist_info.exists():
-            shutil.rmtree(str(dist_info))
-
-        dist_info.mkdir()
-
         with dist_info.joinpath("METADATA").open("w", encoding="utf-8") as f:
             builder._write_metadata_file(f)
 

poetry/packages/locker.py

@@ -24,12 +24,14 @@
 
 import poetry.repositories
 
+from poetry.core.packages import dependency_from_pep_508
 from poetry.core.packages.package import Dependency
 from poetry.core.packages.package import Package
 from poetry.core.semver import parse_constraint
 from poetry.core.semver.version import Version
 from poetry.core.toml.file import TOMLFile
 from poetry.core.version.markers import parse_marker
+from poetry.core.version.requirements import InvalidRequirement
 from poetry.packages import DependencyPackage
 from poetry.utils._compat import OrderedDict
 from poetry.utils._compat import Path
@@ -142,11 +144,18 @@ def locked_repository(
                     package.extras[name] = []
 
                     for dep in deps:
-                        m = re.match(r"^(.+?)(?:\s+\((.+)\))?$", dep)
-                        dep_name = m.group(1)
-                        constraint = m.group(2) or "*"
-
-                        package.extras[name].append(Dependency(dep_name, constraint))
+                        try:
+                            dependency = dependency_from_pep_508(dep)
+                        except InvalidRequirement:
+                            # handle lock files with invalid PEP 508
+                            m = re.match(r"^(.+?)(?:\[(.+?)])?(?:\s+\((.+)\))?$", dep)
+                            dep_name = m.group(1)
+                            extras = m.group(2) or ""
+                            constraint = m.group(3) or "*"
+                            dependency = Dependency(
+                                dep_name, constraint, extras=extras.split(",")
+                            )
+                        package.extras[name].append(dependency)
 
             if "marker" in info:
                 package.marker = parse_marker(info["marker"])
@@ -217,45 +226,43 @@ def __walk_dependency_level(
         next_level_dependencies = []
 
         for requirement in dependencies:
+            key = (requirement.name, requirement.pretty_constraint)
             locked_package = cls.__get_locked_package(requirement, packages_by_name)
 
             if locked_package:
-                for require in locked_package.requires:
-                    if require.marker.is_empty():
-                        require.marker = requirement.marker
-                    else:
-                        require.marker = require.marker.intersect(requirement.marker)
+                # create dependency from locked package to retain dependency metadata
+                # if this is not done, we can end-up with incorrect nested dependencies
+                marker = requirement.marker
+                requirement = locked_package.to_dependency()
+                requirement.marker = requirement.marker.intersect(marker)
+
+                key = (requirement.name, requirement.pretty_constraint)
+
+                if pinned_versions:
+                    requirement.set_constraint(
+                        locked_package.to_dependency().constraint
+                    )
+
+                if key not in nested_dependencies:
+                    for require in locked_package.requires:
+                        if require.marker.is_empty():
+                            require.marker = requirement.marker
+                        else:
+                            require.marker = require.marker.intersect(
+                                requirement.marker
+                            )
 
-                    require.marker = require.marker.intersect(locked_package.marker)
-                    next_level_dependencies.append(require)
+                        require.marker = require.marker.intersect(locked_package.marker)
+                        next_level_dependencies.append(require)
 
             if requirement.name in project_level_dependencies and level == 0:
                 # project level dependencies take precedence
                 continue
 
-            if locked_package:
-                # create dependency from locked package to retain dependency metadata
-                # if this is not done, we can end-up with incorrect nested dependencies
-                marker = requirement.marker
-                requirement = locked_package.to_dependency()
-                requirement.marker = requirement.marker.intersect(marker)
-            else:
+            if not locked_package:
                 # we make a copy to avoid any side-effects
                 requirement = deepcopy(requirement)
 
-            if pinned_versions:
-                requirement.set_constraint(
-                    cls.__get_locked_package(requirement, packages_by_name)
-                    .to_dependency()
-                    .constraint
-                )
-
-            # dependencies use extra to indicate that it was activated via parent
-            # package's extras, this is not required for nested exports as we assume
-            # the resolver already selected this dependency
-            requirement.marker = requirement.marker.without_extras()
-
-            key = (requirement.name, requirement.pretty_constraint)
             if key not in nested_dependencies:
                 nested_dependencies[key] = requirement
             else:
@@ -543,8 +550,10 @@ def _dump_package(self, package):  # type: (Package) -> dict
         if package.extras:
             extras = {}
             for name, deps in package.extras.items():
+                # TODO: This should use dep.to_pep_508() once this is fixed
+                # https://github.com/python-poetry/poetry-core/pull/102
                 extras[name] = [
-                    str(dep) if not dep.constraint.is_any() else dep.name
+                    dep.base_pep_508_name if not dep.constraint.is_any() else dep.name
                     for dep in deps
                 ]
 
@@ -573,7 +582,7 @@ def _dump_package(self, package):  # type: (Package) -> dict
             if package.source_resolved_reference:
                 data["source"]["resolved_reference"] = package.source_resolved_reference
 
-            if package.source_type == "directory":
+            if package.source_type in ["directory", "git"]:
                 data["develop"] = package.develop
 
         return data