Avoid the deprecated JSON API

Author: dimbleby

src/poetry/repositories/link_sources/json.py

@@ -0,0 +1,39 @@
+from __future__ import annotations
+
+from collections import defaultdict
+from typing import TYPE_CHECKING
+from typing import Any
+
+from poetry.core.packages.utils.link import Link
+
+from poetry.repositories.link_sources.base import LinkSource
+
+
+if TYPE_CHECKING:
+    from poetry.repositories.link_sources.base import LinkCache
+
+
+class SimpleJsonPage(LinkSource):
+    """Links as returned by PEP 691 compatible JSON-based Simple API."""
+
+    def __init__(self, url: str, content: dict[str, Any]) -> None:
+        super().__init__(url=url)
+        self.content = content
+
+    @property
+    def _link_cache(self) -> LinkCache:
+        links: LinkCache = defaultdict(lambda: defaultdict(list))
+        for file in self.content["files"]:
+            url = file["url"]
+            requires_python = file.get("requires-python")
+            yanked = file.get("yanked", False)
+            link = Link(url, requires_python=requires_python, yanked=yanked)
+
+            if link.ext not in self.SUPPORTED_FORMATS:
+                continue
+
+            pkg = self.link_package_data(link)
+            if pkg:
+                links[pkg.name][pkg.version].append(link)
+
+        return links

src/poetry/repositories/pypi_repository.py

@@ -12,11 +12,11 @@
 from html5lib.html5parser import parse
 from poetry.core.packages.package import Package
 from poetry.core.packages.utils.link import Link
-from poetry.core.semver.version import Version
 from poetry.core.version.exceptions import InvalidVersion
 
 from poetry.repositories.exceptions import PackageNotFound
 from poetry.repositories.http import HTTPRepository
+from poetry.repositories.link_sources.json import SimpleJsonPage
 from poetry.utils._compat import to_str
 from poetry.utils.constants import REQUESTS_TIMEOUT
 
@@ -27,6 +27,7 @@
 
 if TYPE_CHECKING:
     from packaging.utils import NormalizedName
+    from poetry.core.semver.version import Version
     from poetry.core.semver.version_constraint import VersionConstraint
 
 SUPPORTED_PACKAGE_TYPES = {"sdist", "bdist_wheel"}
@@ -114,50 +115,44 @@ def _find_packages(
         Find packages on the remote server.
         """
         try:
-            info = self.get_package_info(name)
+            json_page = self.get_json_page(name)
         except PackageNotFound:
             self._log(
-                f"No packages found for {name} {constraint!s}",
+                f"No packages found for {name}",
                 level="debug",
             )
             return []
 
-        packages = []
+        versions: list[tuple[Version, str | bool]]
 
-        for version_string, release in info["releases"].items():
-            if not release:
-                # Bad release
-                self._log(
-                    f"No release information found for {name}-{version_string},"
-                    " skipping",
-                    level="debug",
-                )
-                continue
+        key: str = name
+        if not constraint.is_any():
+            key = f"{key}:{constraint!s}"
 
-            try:
-                version = Version.parse(version_string)
-            except InvalidVersion:
-                self._log(
-                    f'Unable to parse version "{version_string}" for the'
-                    f" {name} package, skipping",
-                    level="debug",
-                )
-                continue
+        if self._cache.store("matches").has(key):
+            versions = self._cache.store("matches").get(key)
+        else:
+            versions = [
+                (version, json_page.yanked(name, version))
+                for version in json_page.versions(name)
+                if constraint.allows(version)
+            ]
+            self._cache.store("matches").put(key, versions, 5)
 
-            if constraint.allows(version):
-                # PEP 592: PyPI always yanks entire releases, not individual files,
-                # so we just have to look for the first file
-                yanked = self._get_yanked(release[0])
-                packages.append(Package(info["info"]["name"], version, yanked=yanked))
+        pretty_name = json_page.content["name"]
+        packages = [
+            Package(pretty_name, version, yanked=yanked) for version, yanked in versions
+        ]
 
         return packages
 
-    def _get_package_info(self, name: NormalizedName) -> dict[str, Any]:
-        data = self._get(f"pypi/{name}/json")
-        if data is None:
+    def _get_package_info(self, name: str) -> dict[str, Any]:
+        headers = {"Accept": "application/vnd.pypi.simple.v1+json"}
+        info = self._get(f"simple/{name}/", headers=headers)
+        if info is None:
             raise PackageNotFound(f"Package [{name}] not found.")
 
-        return data
+        return info
 
     def find_links_for_package(self, package: Package) -> list[Link]:
         json_data = self._get(f"pypi/{package.name}/{package.version}/json")
@@ -239,12 +234,20 @@ def _get_release_info(
 
         return data.asdict()
 
-    def _get(self, endpoint: str) -> dict[str, Any] | None:
+    def get_json_page(self, name: NormalizedName) -> SimpleJsonPage:
+        source = self._base_url + f"simple/{name}/"
+        info = self.get_package_info(name)
+        return SimpleJsonPage(source, info)
+
+    def _get(
+        self, endpoint: str, headers: dict[str, str] | None = None
+    ) -> dict[str, Any] | None:
         try:
             json_response = self.session.get(
                 self._base_url + endpoint,
                 raise_for_status=False,
                 timeout=REQUESTS_TIMEOUT,
+                headers=headers,
             )
         except requests.exceptions.TooManyRedirects:
             # Cache control redirect loop.
@@ -254,6 +257,7 @@ def _get(self, endpoint: str) -> dict[str, Any] | None:
                 self._base_url + endpoint,
                 raise_for_status=False,
                 timeout=REQUESTS_TIMEOUT,
+                headers=headers,
             )
 
         if json_response.status_code != 200:

src/poetry/utils/authenticator.py

@@ -183,7 +183,8 @@ def authenticated_url(self, url: str) -> str:
     def request(
         self, method: str, url: str, raise_for_status: bool = True, **kwargs: Any
     ) -> requests.Response:
-        request = requests.Request(method, url)
+        headers = kwargs.get("headers")
+        request = requests.Request(method, url, headers=headers)
         credential = self.get_credentials_for_url(url)
 
         if credential.username is not None or credential.password is not None:

tests/installation/test_executor.py

@@ -147,7 +147,7 @@ def test_execute_executes_a_batch_of_operations(
 
     return_code = executor.execute(
         [
-            Install(Package("pytest", "3.5.2")),
+            Install(Package("pytest", "3.5.1")),
             Uninstall(Package("attrs", "17.4.0")),
             Update(Package("requests", "2.18.3"), Package("requests", "2.18.4")),
             Uninstall(Package("clikit", "0.2.3")).skip("Not currently installed"),
@@ -160,7 +160,7 @@ def test_execute_executes_a_batch_of_operations(
     expected = f"""
 Package operations: 4 installs, 1 update, 1 removal
 
-  • Installing pytest (3.5.2)
+  • Installing pytest (3.5.1)
   • Removing attrs (17.4.0)
   • Updating requests (2.18.3 -> 2.18.4)
   • Installing demo (0.1.0 {file_package.source_url})
@@ -182,20 +182,20 @@ def test_execute_executes_a_batch_of_operations(
     "operations, has_warning",
     [
         (
-            [Install(Package("black", "21.11b0")), Install(Package("pytest", "3.5.2"))],
+            [Install(Package("black", "21.11b0")), Install(Package("pytest", "3.5.1"))],
             True,
         ),
         (
             [
                 Uninstall(Package("black", "21.11b0")),
-                Uninstall(Package("pytest", "3.5.2")),
+                Uninstall(Package("pytest", "3.5.1")),
             ],
             False,
         ),
         (
             [
                 Update(Package("black", "19.10b0"), Package("black", "21.11b0")),
-                Update(Package("pytest", "3.5.1"), Package("pytest", "3.5.2")),
+                Update(Package("pytest", "3.5.0"), Package("pytest", "3.5.1")),
             ],
             True,
         ),
@@ -299,18 +299,18 @@ def test_execute_works_with_ansi_output(
     mocker.patch.object(env, "_run", return_value=install_output)
     return_code = executor.execute(
         [
-            Install(Package("pytest", "3.5.2")),
+            Install(Package("pytest", "3.5.1")),
         ]
     )
     env._run.assert_called_once()
 
     # fmt: off
     expected = [
         "\x1b[39;1mPackage operations\x1b[39;22m: \x1b[34m1\x1b[39m install, \x1b[34m0\x1b[39m updates, \x1b[34m0\x1b[39m removals",  # noqa: E501
-        "\x1b[34;1m•\x1b[39;22m \x1b[39mInstalling \x1b[39m\x1b[36mpytest\x1b[39m\x1b[39m (\x1b[39m\x1b[39;1m3.5.2\x1b[39;22m\x1b[39m)\x1b[39m: \x1b[34mPending...\x1b[39m",  # noqa: E501
-        "\x1b[34;1m•\x1b[39;22m \x1b[39mInstalling \x1b[39m\x1b[36mpytest\x1b[39m\x1b[39m (\x1b[39m\x1b[39;1m3.5.2\x1b[39;22m\x1b[39m)\x1b[39m: \x1b[34mDownloading...\x1b[39m",  # noqa: E501
-        "\x1b[34;1m•\x1b[39;22m \x1b[39mInstalling \x1b[39m\x1b[36mpytest\x1b[39m\x1b[39m (\x1b[39m\x1b[39;1m3.5.2\x1b[39;22m\x1b[39m)\x1b[39m: \x1b[34mInstalling...\x1b[39m",  # noqa: E501
-        "\x1b[32;1m•\x1b[39;22m \x1b[39mInstalling \x1b[39m\x1b[36mpytest\x1b[39m\x1b[39m (\x1b[39m\x1b[32m3.5.2\x1b[39m\x1b[39m)\x1b[39m",  # finished  # noqa: E501
+        "\x1b[34;1m•\x1b[39;22m \x1b[39mInstalling \x1b[39m\x1b[36mpytest\x1b[39m\x1b[39m (\x1b[39m\x1b[39;1m3.5.1\x1b[39;22m\x1b[39m)\x1b[39m: \x1b[34mPending...\x1b[39m",  # noqa: E501
+        "\x1b[34;1m•\x1b[39;22m \x1b[39mInstalling \x1b[39m\x1b[36mpytest\x1b[39m\x1b[39m (\x1b[39m\x1b[39;1m3.5.1\x1b[39;22m\x1b[39m)\x1b[39m: \x1b[34mDownloading...\x1b[39m",  # noqa: E501
+        "\x1b[34;1m•\x1b[39;22m \x1b[39mInstalling \x1b[39m\x1b[36mpytest\x1b[39m\x1b[39m (\x1b[39m\x1b[39;1m3.5.1\x1b[39;22m\x1b[39m)\x1b[39m: \x1b[34mInstalling...\x1b[39m",  # noqa: E501
+        "\x1b[32;1m•\x1b[39;22m \x1b[39mInstalling \x1b[39m\x1b[36mpytest\x1b[39m\x1b[39m (\x1b[39m\x1b[32m3.5.1\x1b[39m\x1b[39m)\x1b[39m",  # finished  # noqa: E501
     ]
     # fmt: on
 
@@ -341,15 +341,15 @@ def test_execute_works_with_no_ansi_output(
     mocker.patch.object(env, "_run", return_value=install_output)
     return_code = executor.execute(
         [
-            Install(Package("pytest", "3.5.2")),
+            Install(Package("pytest", "3.5.1")),
         ]
     )
     env._run.assert_called_once()
 
     expected = """
 Package operations: 1 install, 0 updates, 0 removals
 
-  • Installing pytest (3.5.2)
+  • Installing pytest (3.5.1)
 """
     expected = set(expected.splitlines())
     output = set(io_not_decorated.fetch_output().splitlines())