Don't retry securitytrails ping API on HTTP 429

bbot/modules/base.py

@@ -103,6 +103,8 @@ class BaseModule:
 
     # disable the module after this many failed attempts in a row
     _api_failure_abort_threshold = 3
+    # whether to retry on 429s when first pinging the API at scan start
+    _ping_retry_on_http_429 = False
 
     default_discovery_context = "{module} discovered {event.type}: {event.data}"
 
@@ -381,8 +383,9 @@ async def ping(self):
         """
         if url is None:
             url = getattr(self, "ping_url", "")
+        retry_on_http_429 = getattr(self, "_ping_retry_on_http_429", False)
         if url:
-            r = await self.api_request(url)
+            r = await self.api_request(url, retry_on_http_429=retry_on_http_429)
             if getattr(r, "status_code", 0) != 200:
                 response_text = getattr(r, "text", "no response from server")
                 raise ValueError(response_text)
@@ -1200,6 +1203,7 @@ async def api_request(self, *args, **kwargs):
             - cancelling after too many failed attempts
         """
         url = args[0] if args else kwargs.pop("url", "")
+        retry_on_http_429 = kwargs.pop("retry_on_http_429", True)
 
         # loop until we have a successful request
         for _ in range(self.api_retries):
@@ -1225,7 +1229,7 @@ async def api_request(self, *args, **kwargs):
                 else:
                     # sleep for a bit if we're being rate limited
                     retry_after = self._get_retry_after(r)
-                    if retry_after or status_code == 429:
+                    if (retry_after or status_code == 429) and retry_on_http_429:
                         sleep_interval = int(retry_after) if retry_after is not None else self._429_sleep_interval
                         if retry_after and retry_after > self._429_max_sleep_interval:
                             self.verbose(

bbot/modules/c99.py

@@ -19,7 +19,7 @@ class c99(subdomain_enum_apikey):
 
     async def ping(self):
         url = f"{self.base_url}/randomnumber?key={{api_key}}&between=1,100&json"
-        response = await self.api_request(url)
+        response = await self.api_request(url, retry_on_http_429=False)
         assert response.json()["success"] is True, getattr(response, "text", "no response from server")
 
     async def request_url(self, query):

bbot/modules/censys.py

@@ -29,7 +29,7 @@ async def setup(self):
 
     async def ping(self):
         url = f"{self.base_url}/v1/account"
-        resp = await self.api_request(url)
+        resp = await self.api_request(url, retry_on_http_429=False)
         d = resp.json()
         assert isinstance(d, dict), f"Invalid response from {url}: {resp}"
         quota = d.get("quota", {})

bbot/modules/fullhunt.py

@@ -22,7 +22,7 @@ async def setup(self):
 
     async def ping(self):
         url = f"{self.base_url}/auth/status"
-        j = (await self.api_request(url)).json()
+        j = (await self.api_request(url, retry_on_http_429=False)).json()
         remaining = j["user_credits"]["remaining_credits"]
         assert remaining > 0, "No credits remaining"
 

bbot/modules/passivetotal.py

@@ -21,7 +21,7 @@ async def setup(self):
 
     async def ping(self):
         url = f"{self.base_url}/account/quota"
-        j = (await self.api_request(url)).json()
+        j = (await self.api_request(url, retry_on_http_429=False)).json()
         limit = j["user"]["limits"]["search_api"]
         used = j["user"]["counts"]["search_api"]
         assert used < limit, "No quota remaining"

bbot/modules/zoomeye.py

@@ -31,7 +31,7 @@ def prepare_api_request(self, url, kwargs):
 
     async def ping(self):
         url = f"{self.base_url}/resources-info"
-        r = await self.api_request(url)
+        r = await self.api_request(url, retry_on_http_429=False)
         assert int(r.json()["quota_info"]["remain_total_quota"]) > 0, "No quota remaining"
 
     async def handle_event(self, event):