diff --git a/bbot/modules/base.py b/bbot/modules/base.py index e085c99c93..ed54a34723 100644 --- a/bbot/modules/base.py +++ b/bbot/modules/base.py @@ -103,6 +103,8 @@ class BaseModule: # disable the module after this many failed attempts in a row _api_failure_abort_threshold = 3 + # whether to retry on 429s when first pinging the API at scan start + _ping_retry_on_http_429 = False default_discovery_context = "{module} discovered {event.type}: {event.data}" @@ -381,8 +383,9 @@ async def ping(self): """ if url is None: url = getattr(self, "ping_url", "") + retry_on_http_429 = getattr(self, "_ping_retry_on_http_429", False) if url: - r = await self.api_request(url) + r = await self.api_request(url, retry_on_http_429=retry_on_http_429) if getattr(r, "status_code", 0) != 200: response_text = getattr(r, "text", "no response from server") raise ValueError(response_text) @@ -1200,6 +1203,7 @@ async def api_request(self, *args, **kwargs): - cancelling after too many failed attempts """ url = args[0] if args else kwargs.pop("url", "") + retry_on_http_429 = kwargs.pop("retry_on_http_429", True) # loop until we have a successful request for _ in range(self.api_retries): @@ -1225,7 +1229,7 @@ async def api_request(self, *args, **kwargs): else: # sleep for a bit if we're being rate limited retry_after = self._get_retry_after(r) - if retry_after or status_code == 429: + if (retry_after or status_code == 429) and retry_on_http_429: sleep_interval = int(retry_after) if retry_after is not None else self._429_sleep_interval if retry_after and retry_after > self._429_max_sleep_interval: self.verbose( diff --git a/bbot/modules/c99.py b/bbot/modules/c99.py index 17fea87a13..5f3792b45f 100644 --- a/bbot/modules/c99.py +++ b/bbot/modules/c99.py @@ -19,7 +19,7 @@ class c99(subdomain_enum_apikey): async def ping(self): url = f"{self.base_url}/randomnumber?key={{api_key}}&between=1,100&json" - response = await self.api_request(url) + response = await self.api_request(url, retry_on_http_429=False) assert response.json()["success"] is True, getattr(response, "text", "no response from server") async def request_url(self, query): diff --git a/bbot/modules/censys.py b/bbot/modules/censys.py index 69e1f6f945..67aaa37306 100644 --- a/bbot/modules/censys.py +++ b/bbot/modules/censys.py @@ -29,7 +29,7 @@ async def setup(self): async def ping(self): url = f"{self.base_url}/v1/account" - resp = await self.api_request(url) + resp = await self.api_request(url, retry_on_http_429=False) d = resp.json() assert isinstance(d, dict), f"Invalid response from {url}: {resp}" quota = d.get("quota", {}) diff --git a/bbot/modules/fullhunt.py b/bbot/modules/fullhunt.py index 85106e5827..13a3de5f56 100644 --- a/bbot/modules/fullhunt.py +++ b/bbot/modules/fullhunt.py @@ -22,7 +22,7 @@ async def setup(self): async def ping(self): url = f"{self.base_url}/auth/status" - j = (await self.api_request(url)).json() + j = (await self.api_request(url, retry_on_http_429=False)).json() remaining = j["user_credits"]["remaining_credits"] assert remaining > 0, "No credits remaining" diff --git a/bbot/modules/passivetotal.py b/bbot/modules/passivetotal.py index b20c7bbac0..b72f43cefd 100644 --- a/bbot/modules/passivetotal.py +++ b/bbot/modules/passivetotal.py @@ -21,7 +21,7 @@ async def setup(self): async def ping(self): url = f"{self.base_url}/account/quota" - j = (await self.api_request(url)).json() + j = (await self.api_request(url, retry_on_http_429=False)).json() limit = j["user"]["limits"]["search_api"] used = j["user"]["counts"]["search_api"] assert used < limit, "No quota remaining" diff --git a/bbot/modules/zoomeye.py b/bbot/modules/zoomeye.py index c6cb256df0..3c4c68d3e0 100644 --- a/bbot/modules/zoomeye.py +++ b/bbot/modules/zoomeye.py @@ -31,7 +31,7 @@ def prepare_api_request(self, url, kwargs): async def ping(self): url = f"{self.base_url}/resources-info" - r = await self.api_request(url) + r = await self.api_request(url, retry_on_http_429=False) assert int(r.json()["quota_info"]["remain_total_quota"]) > 0, "No quota remaining" async def handle_event(self, event):