From 8e3c1cbcf482a6d0e69573150cf1cbf73be9fc28 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Thu, 20 Oct 2022 11:28:04 -0700 Subject: [PATCH 1/4] remove requests dependency from PyPIClient --- eng/tox/tox.ini | 3 +- tools/azure-sdk-tools/pypi_tools/pypi.py | 20 ++++----- tools/azure-sdk-tools/setup.py | 1 + .../azure-sdk-tools/tests/test_pypi_client.py | 45 +++++++++++++++++++ 4 files changed, 58 insertions(+), 11 deletions(-) create mode 100644 tools/azure-sdk-tools/tests/test_pypi_client.py diff --git a/eng/tox/tox.ini b/eng/tox/tox.ini index 33f1861acb95..c90255e1a8dd 100644 --- a/eng/tox/tox.ini +++ b/eng/tox/tox.ini @@ -222,10 +222,11 @@ deps = {[packaging]pkgs} requests cryptography<4 + urllib3 commands = {envbindir}/python -m pip install {toxinidir}/../../../tools/azure-sdk-tools --no-deps {envbindir}/python {toxinidir}/../../../eng/tox/sanitize_setup.py -t {toxinidir} - {envbindir}/python {toxinidir}/../../../eng/tox/create_package_and_install.py \ + {envbindir}/python {toxinidir}/../../../eng/tox/create_package_anlk,,,,,,,,,,,,,,,,,,p;[[[[[[[[[[[[[[[[[[[[[[d_install.py \ -d {envtmpdir} \ -p {toxinidir} \ -w {envtmpdir} \ diff --git a/tools/azure-sdk-tools/pypi_tools/pypi.py b/tools/azure-sdk-tools/pypi_tools/pypi.py index 03f64ef831ca..08fa20691755 100644 --- a/tools/azure-sdk-tools/pypi_tools/pypi.py +++ b/tools/azure-sdk-tools/pypi_tools/pypi.py @@ -1,6 +1,8 @@ from packaging.version import parse as Version import sys - +import pdb +from urllib3 import Retry, PoolManager +import json def get_pypi_xmlrpc_client(): """This is actually deprecated client.""" @@ -11,26 +13,24 @@ def get_pypi_xmlrpc_client(): class PyPIClient: def __init__(self, host="https://pypi.org"): - import requests - self._host = host - self._session = requests.Session() + self._http = PoolManager(retries=Retry(raise_on_status=True)) def project(self, package_name): - response = self._session.get( + response = self._http.request( + 'get', "{host}/pypi/{project_name}/json".format(host=self._host, project_name=package_name) ) - response.raise_for_status() - return response.json() + return json.loads(response.data.decode('utf-8')) def project_release(self, package_name, version): - response = self._session.get( + response = self._http.request( + 'get', "{host}/pypi/{project_name}/{version}/json".format( host=self._host, project_name=package_name, version=version ) ) - response.raise_for_status() - return response.json() + return json.loads(response.data.decode('utf-8')) def filter_packages_for_compatibility(self, package_name, version_set): # only need the packaging.specifiers import if we're actually executing this filter. diff --git a/tools/azure-sdk-tools/setup.py b/tools/azure-sdk-tools/setup.py index c57a230929be..e3d8928b23f3 100644 --- a/tools/azure-sdk-tools/setup.py +++ b/tools/azure-sdk-tools/setup.py @@ -20,6 +20,7 @@ "pyopenssl", "python-dotenv", "PyYAML", + "urllib3" ] setup( diff --git a/tools/azure-sdk-tools/tests/test_pypi_client.py b/tools/azure-sdk-tools/tests/test_pypi_client.py new file mode 100644 index 000000000000..cde13d5b1eb0 --- /dev/null +++ b/tools/azure-sdk-tools/tests/test_pypi_client.py @@ -0,0 +1,45 @@ +from pypi_tools.pypi import PyPIClient +from unittest.mock import patch +import os +import pytest +import pdb + + +class TestPyPiClient: + @pytest.mark.skipif( + os.environ.get("TF_BUILD", "None") == True, + reason=f"This test isn't worth recording and could be flaky. Skipping in CI.", + ) + def test_package_retrieve(self): + client = PyPIClient() + result = client.project("azure-core") + + # we won't _exhaustively_ check this, but we can sanity check a few proxy values to ensure we haven't broken anything + assert result["info"]["name"] == "azure-core" + assert len(result["releases"].keys()) > 47 + assert "1.25.1" in result["releases"] + assert "1.10.0" in result["releases"] + + @pytest.mark.skipif( + os.environ.get("TF_BUILD", "None") == True, + reason=f"This test isn't worth recording and could be flaky. Skipping in CI.", + ) + def test_package_version_retrieve(self): + client = PyPIClient() + result = client.project_release("azure-core", "1.8.0") + + assert result["info"]["name"] == "azure-core" + assert result["info"]["release_url"] == "https://pypi.org/project/azure-core/1.8.0/" + + @pytest.mark.skipif( + os.environ.get("TF_BUILD", "None") == True, + reason=f"This test isn't worth recording and could be flaky. Skipping in CI.", + ) + @patch("pypi_tools.pypi.sys") + def test_package_filter_for_compatibility(self, mock_sys): + mock_sys.version_info = (2, 7, 0) + client = PyPIClient() + result = client.get_ordered_versions("azure-core", True) + unfiltered_results = client.get_ordered_versions("azure-core", False) + + assert len(result) < len(unfiltered_results) From dfa9dae4e022edd0ee75b864b9beabdcec029b40 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Thu, 20 Oct 2022 11:29:36 -0700 Subject: [PATCH 2/4] repair tox file --- eng/tox/tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/tox/tox.ini b/eng/tox/tox.ini index c90255e1a8dd..a04db70173fa 100644 --- a/eng/tox/tox.ini +++ b/eng/tox/tox.ini @@ -226,7 +226,7 @@ deps = commands = {envbindir}/python -m pip install {toxinidir}/../../../tools/azure-sdk-tools --no-deps {envbindir}/python {toxinidir}/../../../eng/tox/sanitize_setup.py -t {toxinidir} - {envbindir}/python {toxinidir}/../../../eng/tox/create_package_anlk,,,,,,,,,,,,,,,,,,p;[[[[[[[[[[[[[[[[[[[[[[d_install.py \ + {envbindir}/python {toxinidir}/../../../eng/tox/create_package_and_install.py \ -d {envtmpdir} \ -p {toxinidir} \ -w {envtmpdir} \ From 58f8e9dabe6b7d4c3bd5f8d3e580fc9a77cf9436 Mon Sep 17 00:00:00 2001 From: Scott Beddall <45376673+scbedd@users.noreply.github.com> Date: Thu, 20 Oct 2022 11:32:44 -0700 Subject: [PATCH 3/4] Update eng/tox/tox.ini --- eng/tox/tox.ini | 1 - 1 file changed, 1 deletion(-) diff --git a/eng/tox/tox.ini b/eng/tox/tox.ini index a04db70173fa..da894de48deb 100644 --- a/eng/tox/tox.ini +++ b/eng/tox/tox.ini @@ -220,7 +220,6 @@ setenv = PROXY_URL=http://localhost:5008 deps = {[packaging]pkgs} - requests cryptography<4 urllib3 commands = From 300a30752d1d3b19dba1e0f92166b42af55caff5 Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Thu, 20 Oct 2022 12:32:16 -0700 Subject: [PATCH 4/4] everywhere we use the packaging set, we can hit this issue with missing requests, or rather, missing urllib3 after this PR. azure-sdk-tools itself will carry the dependency for the standard tox environments --- eng/tox/tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/tox/tox.ini b/eng/tox/tox.ini index da894de48deb..4823c957a390 100644 --- a/eng/tox/tox.ini +++ b/eng/tox/tox.ini @@ -38,6 +38,7 @@ deps = pkgs = wheel==0.37.0 packaging==20.4 + urllib3==1.26.12 [testenv] @@ -221,7 +222,6 @@ setenv = deps = {[packaging]pkgs} cryptography<4 - urllib3 commands = {envbindir}/python -m pip install {toxinidir}/../../../tools/azure-sdk-tools --no-deps {envbindir}/python {toxinidir}/../../../eng/tox/sanitize_setup.py -t {toxinidir}