Convert exported requirements to constraints format (#308)

* test: Use Poetry 1.0 compatible lock file in test data

* test: Run tests againsts Poetry 1.0 as well

* test: Fix missing subdependencies in Project.dependencies

* test: Replace list_packages fixture by a plain function

* test: Replace run_nox_with_noxfile fixture by plain function

* test: Remove unused fixture run_nox

* test: Remove unused fixture write_noxfile

* style: Reformat test_functional

* test: Handle URL and path dependencies in list_packages fixture

* test: Handle URL dependencies in Project.get_dependency

* test: Add test data for URL dependencies

* test: Add functional test for URL dependencies

Add a failing test for URL dependencies.

Test output below:

  nox > Command python -m pip install
  --constraint=.nox/test/tmp/requirements.txt
  file:///…/url-dependency/dist/url_dependency-0.1.0-py3-none-any.whl failed
  with exit code 1:

  DEPRECATION: Constraints are only allowed to take the form of a package name
  and a version specifier. Other forms were originally permitted as an accident
  of the implementation, but were undocumented. The new implementation of the
  resolver no longer supports these forms. A possible replacement is replacing
  the constraint with a requirement.. You can find discussion regarding this at
  pypa/pip#8210.

  ERROR: Links are not allowed as constraints

* build: Add dependency on packaging >= 20.9

* refactor(poetry): Do not write exported requirements to disk

* fix: Convert exported requirements to constraints format

* test: Add unit tests for to_constraints

* test: Use canonicalize_name from packaging.utils

* test: Add test data for path dependencies

* test: Add functional test for path dependency

* test: Mark test for path dependencies as XFAIL

Author: cjolowicz

noxfile.py

@@ -3,6 +3,7 @@
 import sys
 from pathlib import Path
 from textwrap import dedent
+from typing import Optional
 
 import nox
 
@@ -117,7 +118,8 @@ def mypy(session: Session) -> None:
 
 
 @session(python=python_versions)
-def tests(session: Session) -> None:
+@nox.parametrize("poetry", ["1.0.10", None])
+def tests(session: Session, poetry: Optional[str]) -> None:
     """Run the test suite."""
     session.install(".")
     session.install(
@@ -130,6 +132,14 @@ def tests(session: Session) -> None:
     if session.python == "3.6":
         session.install("dataclasses")
 
+    if poetry is not None:
+        if session.python != python_versions[0]:
+            session.skip()
+
+        session.run_always(
+            "python", "-m", "pip", "install", f"poetry=={poetry}", silent=True
+        )
+
     try:
         session.run("coverage", "run", "--parallel", "-m", "pytest", *session.posargs)
     finally:

poetry.lock

@@ -23,6 +23,7 @@ Changelog = "https://github.com/cjolowicz/nox-poetry/releases"
 python = "^3.6.1"
 nox = ">=2020.8.22"
 tomlkit = "^0.7.0"
+packaging = ">=20.9"
 
 [tool.poetry.dev-dependencies]
 pytest = "^6.2.2"

pyproject.toml

@@ -61,22 +61,25 @@ def config(self) -> Config:
             self._config = Config(Path.cwd())
         return self._config
 
-    def export(self, path: Path) -> None:
+    def export(self) -> str:
         """Export the lock file to requirements format.
 
-        Args:
-            path: The destination path.
+        Returns:
+            The generated requirements as text.
         """
-        self.session.run_always(
+        output = self.session.run_always(
             "poetry",
             "export",
             "--format=requirements.txt",
-            f"--output={path}",
             "--dev",
             *[f"--extras={extra}" for extra in self.config.extras],
             "--without-hashes",
             external=True,
+            silent=True,
+            stderr=None,
         )
+        assert isinstance(output, str)  # noqa: S101
+        return output
 
     def build(self, *, format: str) -> str:
         """Build the package.

src/nox_poetry/poetry.py

@@ -5,10 +5,13 @@
 from pathlib import Path
 from typing import Any
 from typing import Iterable
+from typing import Iterator
 from typing import Optional
 from typing import Tuple
 
 import nox
+from packaging.requirements import InvalidRequirement
+from packaging.requirements import Requirement
 
 from nox_poetry.poetry import DistributionFormat
 from nox_poetry.poetry import Poetry
@@ -52,6 +55,39 @@ def _split_extras(arg: str) -> Tuple[str, Optional[str]]:
     return arg, None
 
 
+def to_constraint(requirement_string: str, line: int) -> Optional[str]:
+    """Convert requirement to constraint."""
+    if any(
+        requirement_string.startswith(prefix)
+        for prefix in ("-e ", "file://", "git+https://", "http://", "https://")
+    ):
+        return None
+
+    try:
+        requirement = Requirement(requirement_string)
+    except InvalidRequirement as error:
+        raise RuntimeError(f"line {line}: {requirement_string!r}: {error}")
+
+    if not (requirement.name and requirement.specifier):
+        return None
+
+    constraint = f"{requirement.name}{requirement.specifier}"
+    return f"{constraint}; {requirement.marker}" if requirement.marker else constraint
+
+
+def to_constraints(requirements: str) -> str:
+    """Convert requirements to constraints."""
+
+    def _to_constraints() -> Iterator[str]:
+        lines = requirements.strip().splitlines()
+        for line, requirement in enumerate(lines, start=1):
+            constraint = to_constraint(requirement, line)
+            if constraint is not None:
+                yield constraint
+
+    return "\n".join(_to_constraints())
+
+
 class _PoetrySession:
     """Poetry-related utilities for session functions."""
 
@@ -170,7 +206,8 @@ def export_requirements(self) -> Path:
         digest = hashlib.blake2b(lockdata).hexdigest()
 
         if not hashfile.is_file() or hashfile.read_text() != digest:
-            self.poetry.export(path)
+            constraints = to_constraints(self.poetry.export())
+            path.write_text(constraints)
             hashfile.write_text(digest)
 
         return path

src/nox_poetry/sessions.py

@@ -1,8 +1,6 @@
 """Fixtures for functional tests."""
-import functools
 import inspect
 import os
-import re
 import subprocess  # noqa: S404
 import sys
 from dataclasses import dataclass
@@ -17,6 +15,7 @@
 
 import pytest
 import tomlkit
+from packaging.utils import canonicalize_name
 
 
 if TYPE_CHECKING:
@@ -53,6 +52,10 @@ def get_dependency(self, name: str) -> Package:
         data = self._read_toml("poetry.lock")
         for package in data["package"]:
             if package["name"] == name:
+                url = package.get("source", {}).get("url")
+                if url is not None:
+                    # Abuse Package.version to store the URL (for ``list_packages``).
+                    return Package(name, url)
                 return Package(name, package["version"])
         raise ValueError(f"{name}: package not found")
 
@@ -66,15 +69,11 @@ def package(self) -> Package:
     @property
     def dependencies(self) -> List[Package]:
         """Return the package dependencies."""
-        table = self._get_config("dependencies")
+        data = self._read_toml("poetry.lock")
         dependencies: List[str] = [
-            package
-            for package, info in table.items()
-            if not (
-                package == "python"
-                or isinstance(info, dict)
-                and info.get("optional", None)
-            )
+            package["name"]
+            for package in data["package"]
+            if package["category"] == "main" and not package["optional"]
         ]
         return [self.get_dependency(package) for package in dependencies]
 
@@ -109,15 +108,6 @@ def _run_nox(project: Project) -> CompletedProcess:
         raise RuntimeError(f"{error}\n{error.stderr}")
 
 
-RunNox = Callable[[], CompletedProcess]
-
-
-@pytest.fixture
-def run_nox(project: Project) -> RunNox:
-    """Invoke Nox in the project."""
-    return functools.partial(_run_nox, project)
-
-
 SessionFunction = Callable[..., Any]
 
 
@@ -134,54 +124,18 @@ def _write_noxfile(
     path.write_text(text)
 
 
-WriteNoxfile = Callable[
-    [
-        Iterable[SessionFunction],
-        Iterable[ModuleType],
-    ],
-    None,
-]
-
-
-@pytest.fixture
-def write_noxfile(project: Project) -> WriteNoxfile:
-    """Write a noxfile with the given session functions."""
-    return functools.partial(_write_noxfile, project)
-
-
-def _run_nox_with_noxfile(
+def run_nox_with_noxfile(
     project: Project,
     sessions: Iterable[SessionFunction],
     imports: Iterable[ModuleType],
 ) -> None:
+    """Write a noxfile and run Nox in the project."""
     _write_noxfile(project, sessions, imports)
     _run_nox(project)
 
 
-RunNoxWithNoxfile = Callable[
-    [
-        Iterable[SessionFunction],
-        Iterable[ModuleType],
-    ],
-    None,
-]
-
-
-@pytest.fixture
-def run_nox_with_noxfile(project: Project) -> RunNoxWithNoxfile:
-    """Write a noxfile and run Nox in the project."""
-    return functools.partial(_run_nox_with_noxfile, project)
-
-
-_CANONICALIZE_PATTERN = re.compile(r"[-_.]+")
-
-
-def _canonicalize_name(name: str) -> str:
-    # From ``packaging.utils.canonicalize_name`` (PEP 503)
-    return _CANONICALIZE_PATTERN.sub("-", name).lower()
-
-
-def _list_packages(project: Project, session: SessionFunction) -> List[Package]:
+def list_packages(project: Project, session: SessionFunction) -> List[Package]:
+    """List the installed packages for a session in the given project."""
     bindir = "Scripts" if sys.platform == "win32" else "bin"
     pip = project.path / ".nox" / session.__name__ / bindir / "pip"
     process = subprocess.run(  # noqa: S603
@@ -194,19 +148,15 @@ def _list_packages(project: Project, session: SessionFunction) -> List[Package]:
 
     def parse(line: str) -> Package:
         name, _, version = line.partition("==")
-        name = _canonicalize_name(name)
-        if not version and name.startswith(f"{project.package.name} @ file://"):
-            # Local package is listed without version, but it does not matter.
-            return project.package
-        return Package(name, version)
-
-    return [parse(line) for line in process.stdout.splitlines()]
-
+        if not version and " @ " in line:
+            # Abuse Package.version to store the URL or path.
+            name, _, version = line.partition(" @ ")
 
-ListPackages = Callable[[SessionFunction], List[Package]]
+            if name == project.package.name:
+                # But use the known version for the local package.
+                return project.package
 
+        name = canonicalize_name(name)
+        return Package(name, version)
 
-@pytest.fixture
-def list_packages(project: Project) -> ListPackages:
-    """Return a function that lists the installed packages for a session."""
-    return functools.partial(_list_packages, project)
+    return [parse(line) for line in process.stdout.splitlines()]