Se 1513 implement retry (#95)

* SE-1513 Implement retry logic.

* SE-1513 Drop support for Python 3.7 and add support for Python 3.12

* SE-1513 Update github workflows to exclude Python 3.7 and include Python 3.12

* SE-1513 Update unit tests for the client module and add tests for retries.

* SE-1513 Add Python version to x-user-agent

* SE-1513 Bump major version due to dropping Python 3.7

* SE-1513 Fix typos.

* SE-1513 Only retry for 429, 503 and 504

.github/workflows/publish-to-test-pypi.yml

@@ -18,11 +18,11 @@ jobs:
       fail-fast: false
       matrix:
         python-version:
-          - "3.7"
           - "3.8"
           - "3.9"
           - "3.10"
           - "3.11"
+          - "3.12"
         include:
           - os: "ubuntu-latest"
     steps:

.github/workflows/run-tests.yml

@@ -17,11 +17,11 @@ jobs:
       fail-fast: false
       matrix:
         python-version:
-          - "3.7"
           - "3.8"
           - "3.9"
           - "3.10"
           - "3.11"
+          - "3.12"
         include:
           - os: "ubuntu-latest"
     steps:

README.md

@@ -25,7 +25,9 @@ pip install predicthq
 
 ## Migrating from version <= 2.4.0
 
-If you are migrating to version 3.0.0 or above from an earlier version, please check the [breaking changes details](./docs/V3_Breaking_Changes.md).
+If you are migrating to version 3.0.0 or above from an earlier version, please check the [V3 breaking changes details](./docs/V3_Breaking_Changes.md).
+
+If you are migrating to version 4.0.0 or above from an earlier version, please check the [V4 breaking changes details](./docs/V4_Breaking_Changes.md).
 
 ## Usage
 

docs/V3_Breaking_Changes.md

@@ -1,6 +1,6 @@
 # V3 breaking changes details
 
-V3 introduces support for Python versions above 3.9 and drops support for Python3.6.
+V3 introduces support for Python versions above 3.9 and drops support for Python 3.6.
 
 In order to allow the following, we replaced the [Schematics](https://schematics.readthedocs.io/en/latest/) package (which is not maintained anymore and incompatible with python3.10+) by [Pydantic](https://docs.pydantic.dev/latest/).
 

docs/V4_Breaking_Changes.md

@@ -0,0 +1,5 @@
+# V4 breaking changes details
+
+V4 introduces support for Python version 3.12 and drops support for Python 3.7.
+
+This change was needed to implement retry logic using the [Stamina](https://stamina.hynek.me/en/stable/) retry library.

predicthq/client.py

@@ -1,12 +1,14 @@
 import json
 import logging
+from platform import python_version
 from urllib.parse import urljoin, urlparse, urlunparse
 from weakref import proxy
 
 import requests
+import stamina
 
 from .config import config
-from .exceptions import ClientError, ServerError
+from .exceptions import ClientError, RetriableError, ServerError
 from .version import __version__
 
 
@@ -35,7 +37,10 @@ def initialize_endpoints(self):
         self.radius = endpoints.SuggestedRadiusEndpoint(proxy(self))
 
     def get_headers(self, headers):
-        _headers = {"Accept": "application/json", "x-user-agent": f"PHQ-Py-SDK/{__version__}"}
+        _headers = {
+            "Accept": "application/json",
+            "x-user-agent": f"PHQ-Py-SDK/{__version__} (python/{python_version()})",
+        }
         if self.access_token:
             _headers.update(
                 {
@@ -45,6 +50,7 @@ def get_headers(self, headers):
         _headers.update(**headers)
         return _headers
 
+    @stamina.retry(on=RetriableError, attempts=3)
     def request(self, method, path, **kwargs):
         headers = self.get_headers(kwargs.pop("headers", {}))
         response = requests.request(method, self.build_url(path), headers=headers, **kwargs)
@@ -57,7 +63,10 @@ def request(self, method, path, **kwargs):
             except ValueError:
                 error = response.content
 
-            if 400 <= response.status_code <= 499:
+            if response.status_code in (429, 503, 504):
+                # We want to retry for these http status codes
+                raise RetriableError(error)
+            elif 400 <= response.status_code <= 499:
                 raise ClientError(error)
             else:
                 raise ServerError(error)

predicthq/exceptions.py

@@ -20,3 +20,7 @@ class ServerError(APIError):
 
 class ValidationError(PredictHQError):
     pass
+
+
+class RetriableError(APIError):
+    pass

predicthq/version.py

@@ -1 +1 @@
-__version__ = "3.6.0"
+__version__ = "4.0.0"

requirements.txt

@@ -3,3 +3,4 @@ pytest>=6.2.5,<7.0
 python-dateutil>=2.4.2,<3.0
 requests>=2.7.0,<3.0
 responses>=0.10.8,<1.0
+stamina>=24.3.0,<25

setup.py

@@ -36,12 +36,13 @@ def read(*parts):
     author_email="[email protected]",
     url=REPO_URL,
     packages=find_packages(exclude=("tests*",)),
-    python_requires=">=3.7",
+    python_requires=">=3.8",
     install_requires=[
         "pydantic>=2,<3",
         "requests>=2.7.0",
         "python-dateutil>=2.4.2",
         "pytz>=2017.2,<=2021.1",
+        "stamina>=24.3.0,<25",
     ],
     classifiers=[
         "Development Status :: 4 - Beta",
@@ -51,11 +52,11 @@ def read(*parts):
         "Operating System :: OS Independent",
         "Programming Language :: Python",
         "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
         "Programming Language :: Python :: 3.11",
+        "Programming Language :: Python :: 3.12",
         "Topic :: Software Development :: Libraries :: Python Modules",
     ]
 )

tests/test_client.py

@@ -1,16 +1,25 @@
+import unittest
+from unittest import mock
+
 import pytest
+import requests
+import stamina
 
 import predicthq
 from predicthq import endpoints
 from predicthq.endpoints.oauth2.schemas import AccessToken
-from predicthq.exceptions import ClientError, ServerError
-from tests import with_mock_responses, with_config, with_mock_client, load_fixture
+from predicthq.exceptions import ClientError, RetriableError, ServerError
+from tests import load_fixture, with_config, with_mock_client, with_mock_responses
 
 
-class ClientTest(object):
+class ClientTest(unittest.TestCase):
     def setUp(self):
+        stamina.set_testing(True)
         self.client = predicthq.Client()
 
+    def tearDown(self):
+        stamina.set_testing(False)
+
     @with_config(ENDPOINT_URL="https://api.predicthq.com")
     def test_build_url(self):
         assert self.client.build_url("v1") == "https://api.predicthq.com/v1/"
@@ -92,10 +101,10 @@ def test_authenticate(self, client):
             "/oauth2/token/",
             auth=("client_id", "client_secret"),
             data={"scope": "account events", "grant_type": "client_credentials"},
+            verify=True,
         )
 
         assert isinstance(token, AccessToken)
-        assert token.to_primitive() == client.request.return_value
 
         assert self.client.access_token == "token123"
 
@@ -107,3 +116,38 @@ def test_construct_with_access_token(self):
     def test_construct_with_access_token_config(self):
         client = predicthq.Client()
         assert client.access_token == "token123"
+
+
+# New tests are intentionally left out of the ClientTest class because pytest does not support
+# parametrized fixtures inside unittest.TestCase classes as documented at
+# https://docs.pytest.org/en/stable/how-to/unittest.html#pytest-features-in-unittest-testcase-subclasses
+# We should gradually migrate all tests to pytest style and remove unittest.TestCase classes
+
+
+@pytest.fixture
+def client():
+    return predicthq.Client()
+
+
+@pytest.mark.parametrize("status_code", (429, 503, 504))
+@mock.patch("predicthq.client.requests.request")
+def test_retries(mock_request, client, status_code):
+    stamina.set_testing(True, attempts=3)  # Disable stamina backoff waiting so the tests can run faster
+
+    mock_request.return_value.status_code = status_code
+    mock_request.return_value.raise_for_status.side_effect = requests.HTTPError
+
+    with pytest.raises(RetriableError):
+        client.get("/get/")
+
+    # Check that the request was retried 3 times
+    mock_request.assert_has_calls(
+        [
+            mock.call("get", client.build_url("/get/"), headers=client.get_headers({})),
+            mock.call().raise_for_status(),
+            mock.call().json(),
+        ]
+        * 3
+    )
+
+    stamina.set_testing(False)  # Enable stamina backoff waiting after the test is done

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = py37,py38,py39,py310,py311
+envlist = py38,py39,py310,py311,py312
 
 [testenv]
 deps = -rrequirements.txt