Reduce event loop saturation for scan throughput

bbot/core/helpers/diff.py

@@ -233,9 +233,21 @@ async def compare(
                         if item in subject_response.text:
                             reflection = True
                             break
+        diff_reasons = await self.parent_helper.run_in_executor_cpu(
+            self._compare_sync,
+            subject_response,
+            subject,
+        )
+
+        if not diff_reasons:
+            return (True, [], reflection, subject_response)
+        else:
+            return (False, diff_reasons, reflection, subject_response)
+
+    def _compare_sync(self, subject_response, subject):
+        """CPU-bound comparison work offloaded from the event loop."""
         try:
             subject_json = xmltodict.parse(subject_response.text)
-
         except ExpatError:
             log.debug(f"Can't HTML parse for {subject.split('?')[0]}. Switching to text parsing as a backup")
             subject_json = subject_response.text.split("\n")
@@ -255,13 +267,9 @@ async def compare(
 
         if self.compare_body(self.baseline_json, subject_json) is False:
             log.debug("difference in HTML body, no match")
-
             diff_reasons.append("body")
 
-        if not diff_reasons:
-            return (True, [], reflection, subject_response)
-        else:
-            return (False, diff_reasons, reflection, subject_response)
+        return diff_reasons
 
     async def canary_check(self, url, mode, rounds=3):
         """

bbot/core/helpers/web/web.py

@@ -299,72 +299,88 @@ async def request(self, *args, **kwargs):
 
     async def request_batch(self, urls, threads=10, **kwargs):
         """
-        Given a list of URLs, request them in parallel and yield responses as they come in.
+        Request multiple URLs in parallel via blasthttp's native Rust batch engine.
+
+        Applies the same header/cookie/proxy/timeout logic as ``request()`` — each
+        entry is translated into a ``blasthttp.BatchConfig`` and sent to Rust in one
+        shot.  Results are returned as a list (not streamed).
 
         Each entry in ``urls`` can be:
             - A plain URL string (uses shared ``**kwargs`` for all requests)
             - A ``(url, per_request_kwargs)`` tuple for per-request options
             - A ``(url, per_request_kwargs, tracker)`` tuple to attach arbitrary
-              tracking data that is yielded back alongside the response
+              tracking data that is returned alongside the response
 
-        When entries are plain strings, yields ``(url, response)``.
-        When entries include a tracker, yields ``(url, response, tracker)``.
+        Returns:
+            When entries are plain strings: ``list[(url, response)]``
+            When any entry includes a tracker: ``list[(url, response, tracker)]``
 
         Args:
             urls: URLs to visit — strings or ``(url, kwargs[, tracker])`` tuples.
-            threads (int): Max concurrent requests. Defaults to 10.
-            **kwargs: Default keyword arguments passed to ``request()``.
+            threads (int): Concurrency passed to blasthttp. Defaults to 10.
+            **kwargs: Default keyword arguments (same as ``request()``).
                 Overridden by per-request kwargs when entries are tuples.
 
         Examples:
-            Simple (shared kwargs):
+            Simple (shared kwargs)::
 
-            >>> async for url, response in self.helpers.request_batch(urls, headers={"X-Test": "Test"}):
-            >>>     ...
+                results = await self.helpers.request_batch(urls, headers={"X-Test": "Test"})
+                for url, response in results:
+                    ...
 
-            Per-request kwargs with tracker:
+            Per-request kwargs with tracker::
 
-            >>> reqs = [("http://example.com", {"method": "POST"}, "my-tracker")]
-            >>> async for url, response, tracker in self.helpers.request_batch(reqs):
-            >>>     ...
+                reqs = [("http://example.com", {"method": "POST"}, "my-tracker")]
+                results = await self.helpers.request_batch(reqs)
+                for url, response, tracker in results:
+                    ...
         """
-        tasks = {}
-        semaphore = asyncio.Semaphore(threads)
-        has_tracker = False
-
-        async def _do_request(_url, _kwargs, _tracker):
-            async with semaphore:
-                return _url, await self.request(_url, **_kwargs), _tracker
+        import blasthttp
 
+        # Parse entries into uniform (url, req_kwargs, tracker) tuples
+        entries = []
+        has_tracker = False
         for entry in urls:
             if isinstance(entry, str):
-                url, req_kwargs, tracker = entry, kwargs, None
+                entries.append((entry, kwargs, None))
             elif isinstance(entry, tuple):
                 url = entry[0]
                 req_kwargs = entry[1] if len(entry) > 1 and isinstance(entry[1], dict) else kwargs
                 tracker = entry[2] if len(entry) > 2 else None
                 if tracker is not None:
                     has_tracker = True
+                entries.append((url, req_kwargs, tracker))
             else:
-                url, req_kwargs, tracker = str(entry), kwargs, None
-            task = asyncio.create_task(_do_request(url, req_kwargs, tracker))
-            tasks[task] = True
-
-        try:
-            while tasks:
-                finished, _ = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED)
-                for task in finished:
-                    del tasks[task]
-                    url, response, tracker = task.result()
-                    if has_tracker:
-                        yield url, response, tracker
-                    else:
-                        yield url, response
-        finally:
-            for task in tasks:
-                task.cancel()
-            if tasks:
-                await asyncio.gather(*tasks, return_exceptions=True)
+                entries.append((str(entry), kwargs, None))
+
+        if not entries:
+            return []
+
+        # Build BatchConfig objects using the same logic as request()
+        configs = []
+        tracker_by_url = {}
+        for url, req_kwargs, tracker in entries:
+            url, method, blast_kwargs = self._build_blasthttp_kwargs(url, **req_kwargs)
+            config = blasthttp.BatchConfig(url, **blast_kwargs)
+            configs.append(config)
+            if tracker is not None:
+                tracker_by_url[url] = tracker
+
+        # Send to Rust — all I/O happens here
+        batch_results = await self.client.request_batch(configs, concurrency=threads)
+
+        # Convert to (url, response[, tracker]) tuples
+        results = []
+        for br in batch_results:
+            if br.response is not None:
+                response = BlasthttpResponse(br.response, request_url=br.url, method="GET")
+            else:
+                response = None
+            if has_tracker:
+                results.append((br.url, response, tracker_by_url.get(br.url)))
+            else:
+                results.append((br.url, response))
+        return results
 
     async def download(self, url, **kwargs):
         """

bbot/modules/badsecrets.py

@@ -34,7 +34,7 @@ async def setup(self):
 
     @property
     def _module_threads(self):
-        return max(1, multiprocessing.cpu_count() - 1)
+        return min(4, max(1, multiprocessing.cpu_count() - 1))
 
     async def handle_event(self, event):
         resp_body = event.data.get("body", None)

bbot/modules/base.py

@@ -1857,16 +1857,15 @@ async def _worker(self):
                         continue
 
                     acceptable = True
-                    async with self._task_counter.count(f"event_precheck({event})"):
-                        precheck_pass, reason = self._event_precheck(event)
+                    precheck_pass, reason = self._event_precheck(event)
                     if not precheck_pass:
                         self.debug(f"Not intercepting {event} because precheck failed ({reason})")
                         acceptable = False
-                    async with self._task_counter.count(f"event_postcheck({event})"):
+                    else:
                         postcheck_pass, reason = await self._event_postcheck(event)
-                    if not postcheck_pass:
-                        self.debug(f"Not intercepting {event} because postcheck failed ({reason})")
-                        acceptable = False
+                        if not postcheck_pass:
+                            self.debug(f"Not intercepting {event} because postcheck failed ({reason})")
+                            acceptable = False
 
                     # whether to pass the event on to the rest of the scan
                     # defaults to true, unless handle_event returns False

bbot/modules/git.py

@@ -24,7 +24,7 @@ async def handle_event(self, event):
             self.helpers.urljoin(base_url, ".git/config"),
             self.helpers.urljoin(f"{base_url}/", ".git/config"),
         }
-        async for url, response in self.helpers.request_batch(urls):
+        for url, response in await self.helpers.request_batch(urls):
             text = getattr(response, "text", "")
             if not text:
                 text = ""

bbot/modules/iis_shortnames.py

@@ -30,7 +30,7 @@ class iis_shortnames(BaseModule):
     }
     in_scope_only = True
 
-    _module_threads = 8
+    _module_threads = 4
 
     async def detect(self, target):
         technique = None
@@ -132,7 +132,7 @@ async def solve_valid_chars(self, method, target, affirmative_status_code):
                 url = f"{target}{payload}{suffix}"
                 urls_and_kwargs.append((url, kwargs, (c, file_part)))
 
-        async for url, response, (c, file_part) in self.helpers.request_batch(urls_and_kwargs):
+        for url, response, (c, file_part) in await self.helpers.request_batch(urls_and_kwargs):
             if response is not None:
                 if response.status_code == affirmative_status_code:
                     if file_part == "stem":
@@ -173,7 +173,7 @@ async def solve_shortname_recursive(
             kwargs = {"method": method}
             urls_and_kwargs.append((url, kwargs, c))
 
-        async for url, response, c in self.helpers.request_batch(urls_and_kwargs):
+        for url, response, c in await self.helpers.request_batch(urls_and_kwargs):
             if response is not None:
                 if response.status_code == affirmative_status_code:
                     found_results = True

bbot/modules/internal/excavate.py

@@ -348,7 +348,7 @@ class excavateTestRule(ExcavateRule):
     scope_distance_modifier = None
     accept_dupes = False
 
-    _module_threads = 8
+    _module_threads = 6
 
     yara_rule_name_regex = re.compile(r"rule\s(\w+)\s{")
     yara_rule_regex = re.compile(r"(?s)((?:rule\s+\w+\s*{[^{}]*(?:{[^{}]*}[^{}]*)*[^{}]*(?:/\S*?}[^/]*?/)*)*})")

bbot/modules/ntlm.py

@@ -96,10 +96,10 @@ async def handle_event(self, event):
                 urls.add(f"{event.parsed_url.scheme}://{event.parsed_url.netloc}/{endpoint}")
 
         num_urls = len(urls)
-        agen = self.helpers.request_batch(
+        results = await self.helpers.request_batch(
             urls, headers=NTLM_test_header, allow_redirects=False, timeout=self.http_timeout
         )
-        async for url, response in agen:
+        for url, response in results:
             ntlm_resp = response.headers.get("WWW-Authenticate", "")
             if not ntlm_resp:
                 continue
@@ -109,7 +109,6 @@ async def handle_event(self, event):
                 if not ntlm_resp_decoded:
                     continue
 
-                await agen.aclose()
                 self.found.add(found_hash)
                 fqdn = ntlm_resp_decoded.get("FQDN", "")
                 await self.emit_event(

bbot/modules/paramminer_cookies.py

@@ -27,7 +27,7 @@ class paramminer_cookies(paramminer_headers):
     options_desc = {"wordlist": "Define the wordlist to be used to derive cookies"}
     scanned_hosts = []
     boring_words = set()
-    _module_threads = 12
+    _module_threads = 4
     in_scope_only = True
     compare_mode = "cookie"
     default_wordlist = "paramminer_parameters.txt"
@@ -36,11 +36,8 @@ async def check_batch(self, compare_helper, url, cookie_list):
         cookies = {p: self.rand_string(14) for p in cookie_list}
         return await compare_helper.compare(url, cookies=cookies, check_reflection=(len(cookie_list) == 1))
 
-    def gen_count_args(self, url):
-        cookie_count = 40
-        while 1:
-            if cookie_count < 0:
-                break
-            fake_cookies = {self.rand_string(14): self.rand_string(14) for _ in range(0, cookie_count)}
-            yield cookie_count, (url,), {"cookies": fake_cookies}
-            cookie_count -= 5
+    max_count = 40
+
+    def build_count_test_request(self, url, count):
+        fake_cookies = {self.rand_string(14): self.rand_string(14) for _ in range(count)}
+        return (url,), {"cookies": fake_cookies}

bbot/modules/paramminer_getparams.py

@@ -36,11 +36,8 @@ async def check_batch(self, compare_helper, url, getparam_list):
             self.helpers.add_get_params(url, test_getparams).geturl(), check_reflection=(len(getparam_list) == 1)
         )
 
-    def gen_count_args(self, url):
-        getparam_count = 40
-        while 1:
-            if getparam_count < 0:
-                break
-            fake_getparams = {self.rand_string(14): self.rand_string(14) for _ in range(0, getparam_count)}
-            yield getparam_count, (self.helpers.add_get_params(url, fake_getparams).geturl(),), {}
-            getparam_count -= 5
+    max_count = 40
+
+    def build_count_test_request(self, url, count):
+        fake_getparams = {self.rand_string(14): self.rand_string(14) for _ in range(count)}
+        return (self.helpers.add_get_params(url, fake_getparams).geturl(),), {}

bbot/modules/paramminer_headers.py

@@ -75,7 +75,7 @@ class paramminer_headers(BaseModule):
         "zx-request-id",
         "zx-timer",
     }
-    _module_threads = 12
+    _module_threads = 4
     in_scope_only = True
     compare_mode = "header"
     default_wordlist = "paramminer_headers.txt"
@@ -199,27 +199,35 @@ async def handle_event(self, event):
                 self.debug(f"Encountered HttpCompareError: [{e}] for URL [{event.url}]")
             await self.process_results(event, results)
 
+    max_count = 95
+
     async def count_test(self, url):
         baseline = await self.helpers.request(url)
         if baseline is None:
             return
         if str(baseline.status_code)[0] in {"4", "5"}:
             return
-        for count, args, kwargs in self.gen_count_args(url):
+
+        # Binary search for the maximum count the server accepts
+        lo, hi = 0, self.max_count
+        result = None
+        while lo <= hi:
+            mid = (lo + hi) // 2
+            if mid == 0:
+                break
+            args, kwargs = self.build_count_test_request(url, mid)
             r = await self.helpers.request(*args, **kwargs)
             if r is not None and str(r.status_code)[0] not in {"4", "5"}:
-                return count
+                result = mid
+                lo = mid + 1
+            else:
+                hi = mid - 1
+        return result
 
-    def gen_count_args(self, url):
-        header_count = 95
-        while 1:
-            if header_count < 0:
-                break
-            fake_headers = {}
-            for i in range(0, header_count):
-                fake_headers[self.rand_string(14)] = self.rand_string(14)
-            yield header_count, (url,), {"headers": fake_headers}
-            header_count -= 5
+    def build_count_test_request(self, url, count):
+        """Build a test request with `count` fake parameters. Returns (args, kwargs) for helpers.request()."""
+        fake_headers = {self.rand_string(14): self.rand_string(14) for _ in range(count)}
+        return (url,), {"headers": fake_headers}
 
     async def binary_search(self, compare_helper, url, group, reasons=None, reflection=False):
         if reasons is None:

bbot/modules/pgp.py

@@ -37,7 +37,7 @@ async def query(self, query):
         results = set()
         urls = self.config.get("search_urls", [])
         urls = [url.replace("<query>", self.helpers.quote(query)) for url in urls]
-        async for url, response in self.helpers.request_batch(urls):
+        for url, response in await self.helpers.request_batch(urls):
             keyserver = self.helpers.urlparse(url).netloc
             if response is not None:
                 for email in await self.helpers.re.extract_emails(response.text):