Avoid the deprecated JSON API

python-poetry · Aug 15, 2022 · a7cc7f0 · a7cc7f0
1 parent 7bb0a87
commit a7cc7f0
Show file tree

Hide file tree

Showing 41 changed files with 3,695 additions and 6,767 deletions.
diff --git a/src/poetry/repositories/link_sources/json.py b/src/poetry/repositories/link_sources/json.py
@@ -0,0 +1,31 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+from typing import Any
+
+from poetry.core.packages.utils.link import Link
+
+from poetry.repositories.link_sources.base import LinkSource
+
+
+if TYPE_CHECKING:
+    from collections.abc import Iterator
+
+
+class SimpleJsonPage(LinkSource):
+    """Links as returned by PEP 691 compatible JSON-based Simple API."""
+
+    def __init__(self, url: str, content: dict[str, Any]) -> None:
+        super().__init__(url=url)
+        self.content = content
+
+    @property
+    def links(self) -> Iterator[Link]:
+        for file in self.content["files"]:
+            url = file["url"]
+            requires_python = file.get("requires-python")
+            link = Link(url, requires_python=requires_python)
+            if link.ext not in self.SUPPORTED_FORMATS:
+                continue
+
+            yield link
diff --git a/src/poetry/repositories/pypi_repository.py b/src/poetry/repositories/pypi_repository.py
@@ -12,11 +12,11 @@
 from html5lib.html5parser import parse
 from poetry.core.packages.package import Package
 from poetry.core.packages.utils.link import Link
-from poetry.core.semver.version import Version
 from poetry.core.version.exceptions import InvalidVersion
 
 from poetry.repositories.exceptions import PackageNotFound
 from poetry.repositories.http import HTTPRepository
+from poetry.repositories.link_sources.json import SimpleJsonPage
 from poetry.utils._compat import to_str
 from poetry.utils.constants import REQUESTS_TIMEOUT
 
@@ -27,6 +27,7 @@
 
 
 if TYPE_CHECKING:
+    from poetry.core.semver.version import Version
     from poetry.core.semver.version_constraint import VersionConstraint
 
 
@@ -110,47 +111,40 @@ def _find_packages(self, name: str, constraint: VersionConstraint) -> list[Packa
         Find packages on the remote server.
         """
         try:
-            info = self.get_package_info(name)
+            json_page = self.get_json_page(name)
         except PackageNotFound:
             self._log(
-                f"No packages found for {name} {constraint!s}",
+                f"No packages found for {name}",
                 level="debug",
             )
             return []
 
-        packages = []
+        versions: list[Version]
 
-        for version_string, release in info["releases"].items():
-            if not release:
-                # Bad release
-                self._log(
-                    f"No release information found for {name}-{version_string},"
-                    " skipping",
-                    level="debug",
-                )
-                continue
+        key = name
+        if not constraint.is_any():
+            key = f"{key}:{constraint!s}"
 
-            try:
-                version = Version.parse(version_string)
-            except InvalidVersion:
-                self._log(
-                    f'Unable to parse version "{version_string}" for the'
-                    f" {name} package, skipping",
-                    level="debug",
-                )
-                continue
-
-            if constraint.allows(version):
-                packages.append(Package(info["info"]["name"], version))
+        if self._cache.store("matches").has(key):
+            versions = self._cache.store("matches").get(key)
+        else:
+            versions = [
+                version
+                for version in json_page.versions(name)
+                if constraint.allows(version)
+            ]
+            self._cache.store("matches").put(key, versions, 5)
 
-        return packages
+        pretty_name = json_page.content["name"]
+        return [Package(pretty_name, version) for version in versions]
 
     def _get_package_info(self, name: str) -> dict[str, Any]:
-        data = self._get(f"pypi/{name}/json")
-        if data is None:
+        headers = {"Accept": "application/vnd.pypi.simple.v1+json"}
+        info = self._get(f"simple/{name}/", headers=headers)
+        if info is None:
             raise PackageNotFound(f"Package [{name}] not found.")
 
-        return data
+        return info
 
     def find_links_for_package(self, package: Package) -> list[Link]:
         json_data = self._get(f"pypi/{package.name}/{package.version}/json")
@@ -229,12 +223,20 @@ def _get_release_info(
 
         return data.asdict()
 
-    def _get(self, endpoint: str) -> dict[str, Any] | None:
+    def get_json_page(self, name: str) -> SimpleJsonPage:
+        source = self._base_url + f"simple/{name}/"
+        info = self.get_package_info(name)
+        return SimpleJsonPage(source, info)
+
+    def _get(
+        self, endpoint: str, headers: dict[str, str] | None = None
+    ) -> dict[str, Any] | None:
         try:
             json_response = self.session.get(
                 self._base_url + endpoint,
                 raise_for_status=False,
                 timeout=REQUESTS_TIMEOUT,
+                headers=headers,
             )
         except requests.exceptions.TooManyRedirects:
             # Cache control redirect loop.
@@ -244,6 +246,7 @@ def _get(self, endpoint: str) -> dict[str, Any] | None:
                 self._base_url + endpoint,
                 raise_for_status=False,
                 timeout=REQUESTS_TIMEOUT,
+                headers=headers,
             )
 
         if json_response.status_code != 200:

diff --git a/src/poetry/utils/authenticator.py b/src/poetry/utils/authenticator.py
@@ -191,7 +191,8 @@ def authenticated_url(self, url: str) -> str:
     def request(
         self, method: str, url: str, raise_for_status: bool = True, **kwargs: Any
     ) -> requests.Response:
-        request = requests.Request(method, url)
+        headers = kwargs.get("headers")
+        request = requests.Request(method, url, headers=headers)
         credential = self.get_credentials_for_url(url)
 
         if credential.username is not None or credential.password is not None:

diff --git a/tests/installation/test_executor.py b/tests/installation/test_executor.py
@@ -146,7 +146,7 @@ def test_execute_executes_a_batch_of_operations(
 
     return_code = executor.execute(
         [
-            Install(Package("pytest", "3.5.2")),
+            Install(Package("pytest", "3.5.1")),
             Uninstall(Package("attrs", "17.4.0")),
             Update(Package("requests", "2.18.3"), Package("requests", "2.18.4")),
             Uninstall(Package("clikit", "0.2.3")).skip("Not currently installed"),
@@ -159,7 +159,7 @@ def test_execute_executes_a_batch_of_operations(
     expected = f"""
 Package operations: 4 installs, 1 update, 1 removal
 
-  • Installing pytest (3.5.2)
+  • Installing pytest (3.5.1)
   • Removing attrs (17.4.0)
   • Updating requests (2.18.3 -> 2.18.4)
   • Installing demo (0.1.0 {file_package.source_url})
@@ -243,18 +243,18 @@ def test_execute_works_with_ansi_output(
     mocker.patch.object(env, "_run", return_value=install_output)
     return_code = executor.execute(
         [
-            Install(Package("pytest", "3.5.2")),
+            Install(Package("pytest", "3.5.1")),
         ]
     )
     env._run.assert_called_once()
 
     # fmt: off
     expected = [
         "\x1b[39;1mPackage operations\x1b[39;22m: \x1b[34m1\x1b[39m install, \x1b[34m0\x1b[39m updates, \x1b[34m0\x1b[39m removals",  # noqa: E501
-        "\x1b[34;1m•\x1b[39;22m \x1b[39mInstalling \x1b[39m\x1b[36mpytest\x1b[39m\x1b[39m (\x1b[39m\x1b[39;1m3.5.2\x1b[39;22m\x1b[39m)\x1b[39m: \x1b[34mPending...\x1b[39m",  # noqa: E501
-        "\x1b[34;1m•\x1b[39;22m \x1b[39mInstalling \x1b[39m\x1b[36mpytest\x1b[39m\x1b[39m (\x1b[39m\x1b[39;1m3.5.2\x1b[39;22m\x1b[39m)\x1b[39m: \x1b[34mDownloading...\x1b[39m",  # noqa: E501
-        "\x1b[34;1m•\x1b[39;22m \x1b[39mInstalling \x1b[39m\x1b[36mpytest\x1b[39m\x1b[39m (\x1b[39m\x1b[39;1m3.5.2\x1b[39;22m\x1b[39m)\x1b[39m: \x1b[34mInstalling...\x1b[39m",  # noqa: E501
-        "\x1b[32;1m•\x1b[39;22m \x1b[39mInstalling \x1b[39m\x1b[36mpytest\x1b[39m\x1b[39m (\x1b[39m\x1b[32m3.5.2\x1b[39m\x1b[39m)\x1b[39m",  # finished  # noqa: E501
+        "\x1b[34;1m•\x1b[39;22m \x1b[39mInstalling \x1b[39m\x1b[36mpytest\x1b[39m\x1b[39m (\x1b[39m\x1b[39;1m3.5.1\x1b[39;22m\x1b[39m)\x1b[39m: \x1b[34mPending...\x1b[39m",  # noqa: E501
+        "\x1b[34;1m•\x1b[39;22m \x1b[39mInstalling \x1b[39m\x1b[36mpytest\x1b[39m\x1b[39m (\x1b[39m\x1b[39;1m3.5.1\x1b[39;22m\x1b[39m)\x1b[39m: \x1b[34mDownloading...\x1b[39m",  # noqa: E501
+        "\x1b[34;1m•\x1b[39;22m \x1b[39mInstalling \x1b[39m\x1b[36mpytest\x1b[39m\x1b[39m (\x1b[39m\x1b[39;1m3.5.1\x1b[39;22m\x1b[39m)\x1b[39m: \x1b[34mInstalling...\x1b[39m",  # noqa: E501
+        "\x1b[32;1m•\x1b[39;22m \x1b[39mInstalling \x1b[39m\x1b[36mpytest\x1b[39m\x1b[39m (\x1b[39m\x1b[32m3.5.1\x1b[39m\x1b[39m)\x1b[39m",  # finished  # noqa: E501
     ]
     # fmt: on
 
@@ -285,15 +285,15 @@ def test_execute_works_with_no_ansi_output(
     mocker.patch.object(env, "_run", return_value=install_output)
     return_code = executor.execute(
         [
-            Install(Package("pytest", "3.5.2")),
+            Install(Package("pytest", "3.5.1")),
         ]
     )
     env._run.assert_called_once()
 
     expected = """
 Package operations: 1 install, 0 updates, 0 removals
 
-  • Installing pytest (3.5.2)
+  • Installing pytest (3.5.1)
 """
     expected = set(expected.splitlines())
     output = set(io_not_decorated.fetch_output().splitlines())

diff --git a/tests/repositories/fixtures/pypi.org/json/attrs.json b/tests/repositories/fixtures/pypi.org/json/attrs.json
@@ -1,134 +1,25 @@
 {
-  "info": {
-    "author": "Hynek Schlawack",
-    "author_email": "[email protected]",
-    "bugtrack_url": null,
-    "classifiers": [
-      "Development Status :: 5 - Production/Stable",
-      "Intended Audience :: Developers",
-      "License :: OSI Approved :: MIT License",
-      "Natural Language :: English",
-      "Operating System :: OS Independent",
-      "Programming Language :: Python",
-      "Programming Language :: Python :: 2",
-      "Programming Language :: Python :: 2.7",
-      "Programming Language :: Python :: 3",
-      "Programming Language :: Python :: 3.4",
-      "Programming Language :: Python :: 3.5",
-      "Programming Language :: Python :: 3.6",
-      "Programming Language :: Python :: Implementation :: CPython",
-      "Programming Language :: Python :: Implementation :: PyPy",
-      "Topic :: Software Development :: Libraries :: Python Modules"
-    ],
-    "description": "",
-    "docs_url": null,
-    "download_url": "",
-    "downloads": {
-      "last_day": -1,
-      "last_month": -1,
-      "last_week": -1
-    },
-    "home_page": "http://www.attrs.org/",
-    "keywords": "class,attribute,boilerplate",
-    "license": "MIT",
-    "maintainer": "",
-    "maintainer_email": "",
-    "name": "attrs",
-    "package_url": "https://pypi.org/project/attrs/",
-    "platform": "",
-    "project_url": "https://pypi.org/project/attrs/",
-    "release_url": "https://pypi.org/project/attrs/17.4.0/",
-    "requires_dist": [
-      "coverage; extra == 'dev'",
-      "hypothesis; extra == 'dev'",
-      "pympler; extra == 'dev'",
-      "pytest; extra == 'dev'",
-      "six; extra == 'dev'",
-      "zope.interface; extra == 'dev'",
-      "sphinx; extra == 'dev'",
-      "zope.interface; extra == 'dev'",
-      "sphinx; extra == 'docs'",
-      "zope.interface; extra == 'docs'",
-      "coverage; extra == 'tests'",
-      "hypothesis; extra == 'tests'",
-      "pympler; extra == 'tests'",
-      "pytest; extra == 'tests'",
-      "six; extra == 'tests'",
-      "zope.interface; extra == 'tests'"
-    ],
-    "requires_python": "",
-    "summary": "Classes Without Boilerplate",
-    "version": "17.4.0"
-  },
-  "last_serial": 3451237,
-  "releases": {
-    "17.4.0": [
-      {
-        "comment_text": "",
-        "digests": {
-          "md5": "5835a573b3f0316e1602dac3fd9c1daf",
-          "sha256": "a17a9573a6f475c99b551c0e0a812707ddda1ec9653bed04c13841404ed6f450"
-        },
-        "downloads": -1,
-        "filename": "attrs-17.4.0-py2.py3-none-any.whl",
-        "has_sig": true,
-        "md5_digest": "5835a573b3f0316e1602dac3fd9c1daf",
-        "packagetype": "bdist_wheel",
-        "python_version": "py2.py3",
-        "size": 31658,
-        "upload_time": "2017-12-30T08:20:05",
-        "url": "https://files.pythonhosted.org/packages/b5/60/4e178c1e790fd60f1229a9b3cb2f8bc2f4cc6ff2c8838054c142c70b5adc/attrs-17.4.0-py2.py3-none-any.whl"
-      },
-      {
-        "comment_text": "",
-        "digests": {
-          "md5": "d7a89063b2e0fd36bd82389c4d82821d",
-          "sha256": "1c7960ccfd6a005cd9f7ba884e6316b5e430a3f1a6c37c5f87d8b43f83b54ec9"
-        },
-        "downloads": -1,
-        "filename": "attrs-17.4.0.tar.gz",
-        "has_sig": true,
-        "md5_digest": "d7a89063b2e0fd36bd82389c4d82821d",
-        "packagetype": "sdist",
-        "python_version": "source",
-        "size": 97071,
-        "upload_time": "2017-12-30T08:20:08",
-        "url": "https://files.pythonhosted.org/packages/8b/0b/a06cfcb69d0cb004fde8bc6f0fd192d96d565d1b8aa2829f0f20adb796e5/attrs-17.4.0.tar.gz"
-      }
-    ]
-  },
-  "urls": [
+  "name": "attrs",
+  "files": [
     {
-      "comment_text": "",
-      "digests": {
+      "filename": "attrs-17.4.0-py2.py3-none-any.whl",
+      "url": "https://files.pythonhosted.org/packages/b5/60/4e178c1e790fd60f1229a9b3cb2f8bc2f4cc6ff2c8838054c142c70b5adc/attrs-17.4.0-py2.py3-none-any.whl",
+      "hashes": {
         "md5": "5835a573b3f0316e1602dac3fd9c1daf",
         "sha256": "a17a9573a6f475c99b551c0e0a812707ddda1ec9653bed04c13841404ed6f450"
-      },
-      "downloads": -1,
-      "filename": "attrs-17.4.0-py2.py3-none-any.whl",
-      "has_sig": true,
-      "md5_digest": "5835a573b3f0316e1602dac3fd9c1daf",
-      "packagetype": "bdist_wheel",
-      "python_version": "py2.py3",
-      "size": 31658,
-      "upload_time": "2017-12-30T08:20:05",
-      "url": "https://files.pythonhosted.org/packages/b5/60/4e178c1e790fd60f1229a9b3cb2f8bc2f4cc6ff2c8838054c142c70b5adc/attrs-17.4.0-py2.py3-none-any.whl"
+      }
     },
     {
-      "comment_text": "",
-      "digests": {
+      "filename": "attrs-17.4.0.tar.gz",
+      "url": "https://files.pythonhosted.org/packages/8b/0b/a06cfcb69d0cb004fde8bc6f0fd192d96d565d1b8aa2829f0f20adb796e5/attrs-17.4.0.tar.gz",
+      "hashes": {
         "md5": "d7a89063b2e0fd36bd82389c4d82821d",
         "sha256": "1c7960ccfd6a005cd9f7ba884e6316b5e430a3f1a6c37c5f87d8b43f83b54ec9"
-      },
-      "downloads": -1,
-      "filename": "attrs-17.4.0.tar.gz",
-      "has_sig": true,
-      "md5_digest": "d7a89063b2e0fd36bd82389c4d82821d",
-      "packagetype": "sdist",
-      "python_version": "source",
-      "size": 97071,
-      "upload_time": "2017-12-30T08:20:08",
-      "url": "https://files.pythonhosted.org/packages/8b/0b/a06cfcb69d0cb004fde8bc6f0fd192d96d565d1b8aa2829f0f20adb796e5/attrs-17.4.0.tar.gz"
+      }
     }
-  ]
+  ],
+  "meta": {
+    "api-version": "1.0",
+    "_last-serial": 3451237
+  }
 }