Avoid the deprecated JSON API

Author: dimbleby

src/poetry/repositories/legacy_repository.py

@@ -34,59 +34,62 @@ def __init__(
         super().__init__(name, url.rstrip("/"), config, disable_cache)
 
     def find_packages(self, dependency: Dependency) -> list[Package]:
+        """
+        Find packages on the remote server.
+        """
         packages = []
         constraint, allow_prereleases = self._get_constraints_from_dependency(
             dependency
         )
+        ignored_pre_release_packages = []
+
+        versions: list[Version]
 
         key = dependency.name
         if not constraint.is_any():
             key = f"{key}:{constraint!s}"
 
-        ignored_pre_release_versions = []
-
         if self._cache.store("matches").has(key):
             versions = self._cache.store("matches").get(key)
         else:
-            page = self._get_page(f"/{dependency.name.replace('.', '-')}/")
+            page = self._get_page(f"/{dependency.name}/")
             if page is None:
+                self._log(
+                    f"No packages found for {dependency.name}",
+                    level="debug",
+                )
                 return []
 
-            versions = []
-            for version in page.versions(dependency.name):
-                if version.is_unstable() and not allow_prereleases:
-                    if constraint.is_any():
-                        # we need this when all versions of the package are pre-releases
-                        ignored_pre_release_versions.append(version)
-                    continue
-
-                if constraint.allows(version):
-                    versions.append(version)
-
+            versions = [
+                version
+                for version in page.versions(dependency.name)
+                if constraint.allows(version)
+            ]
             self._cache.store("matches").put(key, versions, 5)
 
-        for package_versions in (versions, ignored_pre_release_versions):
-            for version in package_versions:
-                package = Package(
-                    dependency.name,
-                    version,
-                    source_type="legacy",
-                    source_reference=self.name,
-                    source_url=self._url,
-                )
+        for version in versions:
+            package = Package(
+                dependency.name,
+                version,
+                source_type="legacy",
+                source_reference=self.name,
+                source_url=self._url,
+            )
 
-                packages.append(package)
+            if package.is_prerelease() and not allow_prereleases:
+                if constraint.is_any():
+                    # we need this when all versions of the package are pre-releases
+                    ignored_pre_release_packages.append(package)
+                continue
 
-            self._log(
-                f"{len(packages)} packages found for {dependency.name} {constraint!s}",
-                level="debug",
-            )
+            packages.append(package)
 
-            if packages or not constraint.is_any():
-                # we have matching packages, or constraint is not (*)
-                break
+        self._log(
+            f"{len(packages)} packages found for {dependency.name} {constraint!s}",
+            level="debug",
+        )
 
-        return packages
+        return packages or ignored_pre_release_packages
 
     def package(
         self, name: str, version: str, extras: list[str] | None = None
@@ -115,14 +118,14 @@ def package(
             return package
 
     def find_links_for_package(self, package: Package) -> list[Link]:
-        page = self._get_page(f"/{package.name.replace('.', '-')}/")
+        page = self._get_page(f"/{package.name}/")
         if page is None:
             return []
 
         return list(page.links_for_version(package.name, package.version))
 
     def _get_release_info(self, name: str, version: str) -> dict[str, Any]:
-        page = self._get_page(f"/{canonicalize_name(name).replace('.', '-')}/")
+        page = self._get_page(f"/{canonicalize_name(name)}/")
         if page is None:
             raise PackageNotFound(f'No package named "{name}"')
 

src/poetry/repositories/link_sources/json.py

@@ -0,0 +1,31 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+from typing import Any
+
+from poetry.core.packages.utils.link import Link
+
+from poetry.repositories.link_sources.base import LinkSource
+
+
+if TYPE_CHECKING:
+    from collections.abc import Iterator
+
+
+class SimpleJsonPage(LinkSource):
+    """Links as returned by PEP 691 compatible JSON-based Simple API."""
+
+    def __init__(self, url: str, content: dict[str, Any]) -> None:
+        super().__init__(url=url)
+        self.content = content
+
+    @property
+    def links(self) -> Iterator[Link]:
+        for file in self.content["files"]:
+            url = file["url"]
+            requires_python = file.get("requires-python")
+            link = Link(url, requires_python=requires_python)
+            if link.ext not in self.SUPPORTED_FORMATS:
+                continue
+
+            yield link

src/poetry/repositories/pypi_repository.py

@@ -16,6 +16,7 @@
 
 from poetry.repositories.exceptions import PackageNotFound
 from poetry.repositories.http import HTTPRepository
+from poetry.repositories.link_sources.json import SimpleJsonPage
 from poetry.utils._compat import to_str
 from poetry.utils.constants import REQUESTS_TIMEOUT
 
@@ -27,6 +28,7 @@
 
 if TYPE_CHECKING:
     from poetry.core.packages.dependency import Dependency
+    from poetry.core.semver.version import Version
 
 
 class PyPiRepository(HTTPRepository):
@@ -47,50 +49,49 @@ def find_packages(self, dependency: Dependency) -> list[Package]:
         """
         Find packages on the remote server.
         """
-        constraint, allow_prereleases = self._get_constraints_from_dependency(
-            dependency
-        )
-
         try:
-            info = self.get_package_info(dependency.name)
+            json_page = self.get_json_page(dependency.name)
         except PackageNotFound:
             self._log(
-                f"No packages found for {dependency.name} {constraint!s}",
+                f"No packages found for {dependency.name}",
                 level="debug",
             )
             return []
 
+        pretty_name = json_page.content["name"]
+
         packages = []
+        constraint, allow_prereleases = self._get_constraints_from_dependency(
+            dependency
+        )
         ignored_pre_release_packages = []
 
-        for version, release in info["releases"].items():
-            if not release:
-                # Bad release
-                self._log(
-                    f"No release information found for {dependency.name}-{version},"
-                    " skipping",
-                    level="debug",
-                )
-                continue
+        versions: list[Version]
 
-            try:
-                package = Package(info["info"]["name"], version)
-            except InvalidVersion:
-                self._log(
-                    f'Unable to parse version "{version}" for the'
-                    f" {dependency.name} package, skipping",
-                    level="debug",
-                )
-                continue
+        key = dependency.name
+        if not constraint.is_any():
+            key = f"{key}:{constraint!s}"
+
+        if self._cache.store("matches").has(key):
+            versions = self._cache.store("matches").get(key)
+        else:
+            versions = [
+                version
+                for version in json_page.versions(dependency.name)
+                if constraint.allows(version)
+            ]
+            self._cache.store("matches").put(key, versions, 5)
+
+        for version in versions:
+            package = Package(pretty_name, version)
 
             if package.is_prerelease() and not allow_prereleases:
                 if constraint.is_any():
                     # we need this when all versions of the package are pre-releases
                     ignored_pre_release_packages.append(package)
                 continue
 
-            if constraint.allows(package.version):
-                packages.append(package)
+            packages.append(package)
 
         self._log(
             f"{len(packages)} packages found for {dependency.name} {constraint!s}",
@@ -161,11 +162,12 @@ def get_package_info(self, name: str) -> dict[str, Any]:
         return package_info
 
     def _get_package_info(self, name: str) -> dict[str, Any]:
-        data = self._get(f"pypi/{name}/json")
-        if data is None:
+        headers = {"Accept": "application/vnd.pypi.simple.v1+json"}
+        info = self._get(f"simple/{name}/", headers=headers)
+        if info is None:
             raise PackageNotFound(f"Package [{name}] not found.")
 
-        return data
+        return info
 
     def find_links_for_package(self, package: Package) -> list[Link]:
         json_data = self._get(f"pypi/{package.name}/{package.version}/json")
@@ -244,12 +246,20 @@ def _get_release_info(
 
         return data.asdict()
 
-    def _get(self, endpoint: str) -> dict[str, Any] | None:
+    def get_json_page(self, name: str) -> SimpleJsonPage:
+        source = self._base_url + f"simple/{name}/"
+        info = self.get_package_info(name)
+        return SimpleJsonPage(source, info)
+
+    def _get(
+        self, endpoint: str, headers: dict[str, str] | None = None
+    ) -> dict[str, Any] | None:
         try:
             json_response = self.session.get(
                 self._base_url + endpoint,
                 raise_for_status=False,
                 timeout=REQUESTS_TIMEOUT,
+                headers=headers,
             )
         except requests.exceptions.TooManyRedirects:
             # Cache control redirect loop.
@@ -259,6 +269,7 @@ def _get(self, endpoint: str) -> dict[str, Any] | None:
                 self._base_url + endpoint,
                 raise_for_status=False,
                 timeout=REQUESTS_TIMEOUT,
+                headers=headers,
             )
 
         if json_response.status_code != 200:

src/poetry/utils/authenticator.py

@@ -191,7 +191,8 @@ def authenticated_url(self, url: str) -> str:
     def request(
         self, method: str, url: str, raise_for_status: bool = True, **kwargs: Any
     ) -> requests.Response:
-        request = requests.Request(method, url)
+        headers = kwargs.get("headers")
+        request = requests.Request(method, url, headers=headers)
         credential = self.get_credentials_for_url(url)
 
         if credential.username is not None or credential.password is not None:

tests/installation/test_executor.py

@@ -146,7 +146,7 @@ def test_execute_executes_a_batch_of_operations(
 
     return_code = executor.execute(
         [
-            Install(Package("pytest", "3.5.2")),
+            Install(Package("pytest", "3.5.1")),
             Uninstall(Package("attrs", "17.4.0")),
             Update(Package("requests", "2.18.3"), Package("requests", "2.18.4")),
             Uninstall(Package("clikit", "0.2.3")).skip("Not currently installed"),
@@ -159,7 +159,7 @@ def test_execute_executes_a_batch_of_operations(
     expected = f"""
 Package operations: 4 installs, 1 update, 1 removal
 
-  • Installing pytest (3.5.2)
+  • Installing pytest (3.5.1)
   • Removing attrs (17.4.0)
   • Updating requests (2.18.3 -> 2.18.4)
   • Installing demo (0.1.0 {file_package.source_url})
@@ -243,18 +243,18 @@ def test_execute_works_with_ansi_output(
     mocker.patch.object(env, "_run", return_value=install_output)
     return_code = executor.execute(
         [
-            Install(Package("pytest", "3.5.2")),
+            Install(Package("pytest", "3.5.1")),
         ]
     )
     env._run.assert_called_once()
 
     # fmt: off
     expected = [
         "\x1b[39;1mPackage operations\x1b[39;22m: \x1b[34m1\x1b[39m install, \x1b[34m0\x1b[39m updates, \x1b[34m0\x1b[39m removals",  # noqa: E501
-        "\x1b[34;1m•\x1b[39;22m \x1b[39mInstalling \x1b[39m\x1b[36mpytest\x1b[39m\x1b[39m (\x1b[39m\x1b[39;1m3.5.2\x1b[39;22m\x1b[39m)\x1b[39m: \x1b[34mPending...\x1b[39m",  # noqa: E501
-        "\x1b[34;1m•\x1b[39;22m \x1b[39mInstalling \x1b[39m\x1b[36mpytest\x1b[39m\x1b[39m (\x1b[39m\x1b[39;1m3.5.2\x1b[39;22m\x1b[39m)\x1b[39m: \x1b[34mDownloading...\x1b[39m",  # noqa: E501
-        "\x1b[34;1m•\x1b[39;22m \x1b[39mInstalling \x1b[39m\x1b[36mpytest\x1b[39m\x1b[39m (\x1b[39m\x1b[39;1m3.5.2\x1b[39;22m\x1b[39m)\x1b[39m: \x1b[34mInstalling...\x1b[39m",  # noqa: E501
-        "\x1b[32;1m•\x1b[39;22m \x1b[39mInstalling \x1b[39m\x1b[36mpytest\x1b[39m\x1b[39m (\x1b[39m\x1b[32m3.5.2\x1b[39m\x1b[39m)\x1b[39m",  # finished  # noqa: E501
+        "\x1b[34;1m•\x1b[39;22m \x1b[39mInstalling \x1b[39m\x1b[36mpytest\x1b[39m\x1b[39m (\x1b[39m\x1b[39;1m3.5.1\x1b[39;22m\x1b[39m)\x1b[39m: \x1b[34mPending...\x1b[39m",  # noqa: E501
+        "\x1b[34;1m•\x1b[39;22m \x1b[39mInstalling \x1b[39m\x1b[36mpytest\x1b[39m\x1b[39m (\x1b[39m\x1b[39;1m3.5.1\x1b[39;22m\x1b[39m)\x1b[39m: \x1b[34mDownloading...\x1b[39m",  # noqa: E501
+        "\x1b[34;1m•\x1b[39;22m \x1b[39mInstalling \x1b[39m\x1b[36mpytest\x1b[39m\x1b[39m (\x1b[39m\x1b[39;1m3.5.1\x1b[39;22m\x1b[39m)\x1b[39m: \x1b[34mInstalling...\x1b[39m",  # noqa: E501
+        "\x1b[32;1m•\x1b[39;22m \x1b[39mInstalling \x1b[39m\x1b[36mpytest\x1b[39m\x1b[39m (\x1b[39m\x1b[32m3.5.1\x1b[39m\x1b[39m)\x1b[39m",  # finished  # noqa: E501
     ]
     # fmt: on
 
@@ -285,15 +285,15 @@ def test_execute_works_with_no_ansi_output(
     mocker.patch.object(env, "_run", return_value=install_output)
     return_code = executor.execute(
         [
-            Install(Package("pytest", "3.5.2")),
+            Install(Package("pytest", "3.5.1")),
         ]
     )
     env._run.assert_called_once()
 
     expected = """
 Package operations: 1 install, 0 updates, 0 removals
 
-  • Installing pytest (3.5.2)
+  • Installing pytest (3.5.1)
 """
     expected = set(expected.splitlines())
     output = set(io_not_decorated.fetch_output().splitlines())