Attempt to retry 403 errors when they are actually secondary throttling limit

NOTICE.txt

@@ -3644,84 +3644,6 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.
 
 
-exceptiongroup
-1.2.2
-MIT License
-The MIT License (MIT)
-
-Copyright (c) 2022 Alex Grönholm
-
-Permission is hereby granted, free of charge, to any person obtaining a copy of
-this software and associated documentation files (the "Software"), to deal in
-the Software without restriction, including without limitation the rights to
-use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
-the Software, and to permit persons to whom the Software is furnished to do so,
-subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
-FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
-COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
-IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-
-
-This project contains code copied from the Python standard library.
-The following is the required license notice for those parts.
-
-PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
---------------------------------------------
-
-1. This LICENSE AGREEMENT is between the Python Software Foundation
-("PSF"), and the Individual or Organization ("Licensee") accessing and
-otherwise using this software ("Python") in source or binary form and
-its associated documentation.
-
-2. Subject to the terms and conditions of this License Agreement, PSF hereby
-grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce,
-analyze, test, perform and/or display publicly, prepare derivative works,
-distribute, and otherwise use Python alone or in any derivative version,
-provided, however, that PSF's License Agreement and PSF's notice of copyright,
-i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010,
-2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022 Python Software Foundation;
-All Rights Reserved" are retained in Python alone or in any derivative version
-prepared by Licensee.
-
-3. In the event Licensee prepares a derivative work that is based on
-or incorporates Python or any part thereof, and wants to make
-the derivative work available to others as provided herein, then
-Licensee hereby agrees to include in any such work a brief summary of
-the changes made to Python.
-
-4. PSF is making Python available to Licensee on an "AS IS"
-basis.  PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR
-IMPLIED.  BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND
-DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS
-FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT
-INFRINGE ANY THIRD PARTY RIGHTS.
-
-5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON
-FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS
-A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON,
-OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF.
-
-6. This License Agreement will automatically terminate upon a material
-breach of its terms and conditions.
-
-7. Nothing in this License Agreement shall be deemed to create any
-relationship of agency, partnership, or joint venture between PSF and
-Licensee.  This License Agreement does not grant permission to use PSF
-trademarks or trade name in a trademark sense to endorse or promote
-products or services of Licensee, or any third party.
-
-8. By copying, installing or otherwise using Python, Licensee
-agrees to be bound by the terms and conditions of this License
-Agreement.
-
-
 exchangelib
 5.4.0
 BSD License
@@ -7510,11 +7432,11 @@ made under the terms of *both* these licenses.
 
 
 soupsieve
-2.6
+2.7
 MIT License
 MIT License
 
-Copyright (c) 2018 - 2024 Isaac Muse <[email protected]>
+Copyright (c) 2018 - 2025 Isaac Muse <[email protected]>
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

connectors/sources/github.py

@@ -12,8 +12,8 @@
 
 import aiohttp
 import fastjsonschema
-from aiohttp.client_exceptions import ClientResponseError
-from gidgethub import QueryError, RateLimitExceeded, sansio
+import gidgethub
+from gidgethub import QueryError, sansio
 from gidgethub.abc import (
     BadGraphQLRequest,
     GraphQLAuthorizationFailure,
@@ -687,9 +687,6 @@ def __init__(
     def set_logger(self, logger_):
         self._logger = logger_
 
-    def get_rate_limit_encountered(self, status_code, rate_limit_remaining):
-        return status_code == FORBIDDEN and not int(rate_limit_remaining)
-
     async def _get_retry_after(self, resource_type):
         current_time = time.time()
         response = await self.get_github_item("/rate_limit")
@@ -737,7 +734,7 @@ async def _update_installation_access_token(self):
                 private_key=self.private_key,
             )
             self._installation_access_token = access_token_response["token"]
-        except RateLimitExceeded:
+        except gidgethub.RateLimitExceeded:
             await self._put_to_sleep("core")
         except Exception:
             self._logger.exception(
@@ -752,7 +749,7 @@ def _get_session(self):
         timeout = aiohttp.ClientTimeout(total=None)
         return aiohttp.ClientSession(
             timeout=timeout,
-            raise_for_status=True,
+            raise_for_status=False,
             connector=connector,
         )
 
@@ -844,8 +841,10 @@ async def get_github_item(self, resource):
             return await self._get_client.getitem(
                 url=resource, oauth_token=self._access_token()
             )
-        except ClientResponseError as exception:
-            if exception.status == UNAUTHORIZED:
+        except gidgethub.RateLimitExceeded:
+            await self._put_to_sleep("core")
+        except gidgethub.HTTPException as exception:
+            if exception.status_code == UNAUTHORIZED:
                 if self.auth_method == GITHUB_APP:
                     self._logger.debug(
                         f"The access token for installation #{self._installation_id} expired, Regenerating a new token."
@@ -854,17 +853,11 @@ async def get_github_item(self, resource):
                     raise
                 msg = "Your Github token is either expired or revoked. Please check again."
                 raise UnauthorizedException(msg) from exception
-            elif self.get_rate_limit_encountered(
-                exception.status, exception.headers.get("X-RateLimit-Remaining")
-            ):
-                await self._put_to_sleep(resource_type="core")
-            elif exception.status == FORBIDDEN:
+            elif exception.status_code == FORBIDDEN:
                 msg = f"Provided GitHub token does not have the necessary permissions to perform the request for the URL: {resource}."
                 raise ForbiddenException(msg) from exception
             else:
                 raise
-        except RateLimitExceeded:
-            await self._put_to_sleep("core")
         except Exception as e:
             self._logger.debug(
                 f"An unexpected error occurred while getting GitHub item: {resource}. Error: {e}"
@@ -918,16 +911,11 @@ async def get_personal_access_token_scopes(self):
                 )
                 return set()
             return {scope.strip() for scope in scopes.split(",")}
-        except ClientResponseError as exception:
-            if exception.status == FORBIDDEN:
-                if self.get_rate_limit_encountered(
-                    exception.status, exception.headers.get("X-RateLimit-Remaining")
-                ):
-                    await self._put_to_sleep("graphql")
-                else:
-                    msg = f"Provided GitHub token does not have the necessary permissions to perform the request for the URL: {self.base_url}."
-                    raise ForbiddenException(msg) from exception
-            elif exception.status == UNAUTHORIZED:
+        except gidgethub.HTTPException as exception:
+            if exception.status_code == FORBIDDEN:
+                msg = f"Provided GitHub token does not have the necessary permissions to perform the request for the URL: {self.base_url}."
+                raise ForbiddenException(msg) from exception
+            elif exception.status_code == UNAUTHORIZED:
                 msg = "Your Github token is either expired or revoked. Please check again."
                 raise UnauthorizedException(msg) from exception
             else:
@@ -950,7 +938,7 @@ async def _github_app_get(self, url):
                 get_jwt(app_id=self.app_id, private_key=self.private_key),
             )
         # we don't expect any 401 error as the jwt is freshly generated
-        except RateLimitExceeded:
+        except gidgethub.RateLimitExceeded:
             await self._put_to_sleep("core")
         except Exception:
             raise

tests/sources/test_github.py

@@ -10,9 +10,8 @@
 from http import HTTPStatus
 from unittest.mock import ANY, AsyncMock, Mock, patch
 
-import aiohttp
+import gidgethub
 import pytest
-from aiohttp.client_exceptions import ClientResponseError
 from gidgethub.abc import BadGraphQLRequest, GraphQLAuthorizationFailure, QueryError
 
 from connectors.access_control import DLS_QUERY
@@ -960,13 +959,7 @@ async def test_get_response_with_rate_limit_exceeded():
         with patch.object(
             source.github_client._get_client,
             "getitem",
-            side_effect=ClientResponseError(
-                status=403,
-                request_info=aiohttp.RequestInfo(
-                    real_url="", method=None, headers=None, url=""
-                ),
-                history=None,
-            ),
+            side_effect=gidgethub.HTTPException(status_code=HTTPStatus.FORBIDDEN),
         ):
             with pytest.raises(Exception):
                 source.github_client._get_retry_after = AsyncMock(return_value=0)
@@ -2094,37 +2087,20 @@ async def test_get_personal_access_token_scopes(scopes, expected_scopes):
     "exception, raises",
     [
         (
-            ClientResponseError(
-                status=401,
-                request_info=aiohttp.RequestInfo(
-                    real_url="", method=None, headers=None, url=""
-                ),
-                history=None,
-                headers={"X-RateLimit-Remaining": 5000},
-            ),
+            gidgethub.HTTPException(status_code=HTTPStatus.UNAUTHORIZED),
             UnauthorizedException,
         ),
         (
-            ClientResponseError(
-                status=403,
-                request_info=aiohttp.RequestInfo(
-                    real_url="", method=None, headers=None, url=""
-                ),
-                history=None,
-                headers={"X-RateLimit-Remaining": 2500},
+            gidgethub.HTTPException(
+                status_code=HTTPStatus.FORBIDDEN,
             ),
             ForbiddenException,
         ),
         (
-            ClientResponseError(
-                status=404,
-                request_info=aiohttp.RequestInfo(
-                    real_url="", method=None, headers=None, url=""
-                ),
-                history=None,
-                headers={"X-RateLimit-Remaining": 2500},
+            gidgethub.HTTPException(
+                status_code=HTTPStatus.NOT_FOUND,
             ),
-            ClientResponseError,
+            gidgethub.HTTPException,
         ),
     ],
 )
@@ -2310,37 +2286,20 @@ async def test_update_installation_access_token_when_error_occurs():
     "exceptions, raises",
     [
         (
-            ClientResponseError(
-                status=403,
-                request_info=aiohttp.RequestInfo(
-                    real_url="", method=None, headers=None, url=""
-                ),
-                headers={"X-RateLimit-Remaining": 4000},
-                history=None,
+            gidgethub.HTTPException(
+                status_code=HTTPStatus.FORBIDDEN,
             ),
             ForbiddenException,
         ),
         (
-            ClientResponseError(
-                status=401,
-                request_info=aiohttp.RequestInfo(
-                    real_url="", method=None, headers=None, url=""
-                ),
-                headers={"X-RateLimit-Remaining": 4000},
-                history=None,
-            ),
+            gidgethub.HTTPException(status_code=HTTPStatus.UNAUTHORIZED),
             UnauthorizedException,
         ),
         (
-            ClientResponseError(
-                status=404,
-                request_info=aiohttp.RequestInfo(
-                    real_url="", method=None, headers=None, url=""
-                ),
-                headers={"X-RateLimit-Remaining": 4000},
-                history=None,
+            gidgethub.HTTPException(
+                status_code=HTTPStatus.NOT_FOUND,
             ),
-            ClientResponseError,
+            gidgethub.HTTPException,
         ),
         (Exception(), Exception),
     ],