From dccc9684fb98e0c676970f18ec16cb939d58484a Mon Sep 17 00:00:00 2001
From: liquidsec <paul.mueller08@gmail.com>
Date: Thu, 7 May 2026 23:10:50 -0400
Subject: [PATCH 1/6] spill HTTP_RESPONSE bodies to disk via per-scan LRU +
 zstd
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Bodies are the dominant memory tenant in long-running scans (in our
benchmarks, mid-scan body bytes peak at ~640 MB on a wide-and-busy
workload). Holding them in event objects until _minimize() fires
keeps Python's working set above what processing actually needs.

The per-scan BodySpillStore lives at ${scan.temp_dir}/bodies/, writes
zstd-compressed bytes per event UUID, and serves reads from a bounded
LRU (default 512 MB, scan-config-overridable). Eviction is
biggest-first with FIFO tie-break — pure LRU is wrong for BBOT's
roughly-FIFO pipeline, where the oldest cache entry is typically the
next event to be processed. _minimize() drops both the cache entry
and the file; the existing scan-end temp_dir cleanup mops up anything
that escapes.

HTTP_RESPONSE.__init__ pops body from _data and routes it through
the store; reads go through a new event.body property. Six module
read sites migrated (telerik, ajaxpro, badsecrets, dotnetnuke,
excavate, newsletters) plus the raw_response property.

Measured savings on the existing scan_memory benchmarks:

  Cache=512MB (default):  961 -> 868 MB peak RSS  (-93 MB)
  Cache=32MB  (stress):   961 -> 395 MB peak RSS  (-566 MB), 78% hit rate

Hit rate stays high even with an aggressively small cache because
most module reads happen close in time to body emission; the
biggest-first eviction policy preserves the recent working set.
---
 bbot/core/event/base.py                  |  46 ++++-
 bbot/core/event/spill.py                 | 225 +++++++++++++++++++++++
 bbot/core/helpers/web/web.py             |   2 +-
 bbot/defaults.yml                        |  10 +
 bbot/modules/ajaxpro.py                  |   2 +-
 bbot/modules/badsecrets.py               |   2 +-
 bbot/modules/dotnetnuke.py               |   2 +-
 bbot/modules/internal/excavate.py        |   2 +-
 bbot/modules/newsletters.py              |   8 +-
 bbot/modules/telerik.py                  |   2 +-
 bbot/scanner/scanner.py                  |  18 ++
 bbot/test/test_step_1/test_body_spill.py | 222 ++++++++++++++++++++++
 pyproject.toml                           |   1 +
 uv.lock                                  |  92 +++++++++
 14 files changed, 622 insertions(+), 12 deletions(-)
 create mode 100644 bbot/core/event/spill.py
 create mode 100644 bbot/test/test_step_1/test_body_spill.py

diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py
index 4ffbe7dc40..d87c920c6b 100644
--- a/bbot/core/event/base.py
+++ b/bbot/core/event/base.py
@@ -1613,6 +1613,45 @@ def __init__(self, *args, **kwargs):
         if str(self.http_status).startswith("3"):
             self.num_redirects += 1
 
+        # Spill body to disk if a per-scan store is available. The body
+        # is removed from `_data` so JSON / human renderers don't see it;
+        # readers use the `.body` property which lazy-loads from the store.
+        # When spill is disabled (no store on the scan), body stays in
+        # `_data` and `.body` falls back to reading it from there.
+        store = getattr(getattr(self, "scan", None), "body_spill_store", None)
+        if store is not None and isinstance(self._data, dict) and "body" in self._data:
+            body = self._data.pop("body")
+            if isinstance(body, str):
+                body_bytes = body.encode("utf-8", errors="replace")
+            elif isinstance(body, (bytes, bytearray, memoryview)):
+                body_bytes = bytes(body)
+            else:
+                body_bytes = str(body).encode("utf-8", errors="replace")
+            if body_bytes:
+                store.write(str(self._uuid), body_bytes)
+
+    @property
+    def body(self):
+        """
+        The HTTP response body as a string.
+
+        With spill enabled, the body lives in the per-scan ``BodySpillStore``
+        (LRU cache + disk file). Cache hits are instant; misses re-read
+        from disk. After ``_minimize()`` fires the body is gone and this
+        property returns ``""``.
+
+        With spill disabled, falls back to ``self._data["body"]``.
+        """
+        store = getattr(getattr(self, "scan", None), "body_spill_store", None)
+        if store is None:
+            if isinstance(self._data, dict):
+                return self._data.get("body", "") or ""
+            return ""
+        body_bytes = store.read(str(self._uuid))
+        if not body_bytes:
+            return ""
+        return body_bytes.decode("utf-8", errors="replace")
+
     def _data_id(self):
         return self.data["method"] + "|" + self.data["url"]
 
@@ -1675,6 +1714,10 @@ def _data_human(self):
     def _minimize(self):
         super()._minimize()
         if self._module_consumers <= 0:
+            store = getattr(getattr(self, "scan", None), "body_spill_store", None)
+            if store is not None:
+                store.evict_and_delete(str(self._uuid))
+            # `body` may still be in _data if spill was disabled at creation
             self._data.pop("body", None)
             self._data.pop("raw_header", None)
             self._data.pop("header", None)
@@ -1687,8 +1730,7 @@ def raw_response(self):
         Formats the status code, headers, and body into a single string formatted as an HTTP/1.1 response.
         """
         raw_header = self.data.get("raw_header", "")
-        body = self.data.get("body", "")
-        return f"{raw_header}{body}"
+        return f"{raw_header}{self.body}"
 
     @property
     def http_status(self):
diff --git a/bbot/core/event/spill.py b/bbot/core/event/spill.py
new file mode 100644
index 0000000000..f5ac39bb0d
--- /dev/null
+++ b/bbot/core/event/spill.py
@@ -0,0 +1,225 @@
+"""
+Per-scan disk-spill store for HTTP_RESPONSE bodies.
+
+Bodies are the dominant memory tenant in long-running scans (in our
+benchmarks, mid-scan body bytes peak at ~640 MB on a wide-and-busy
+workload). Holding them in event objects until ``_minimize()`` fires
+keeps Python's working set above what the actual processing pipeline
+needs.
+
+This store keeps bodies on disk under ``${scan.temp_dir}/bodies/`` and
+serves reads from a bounded LRU cache. The cache absorbs the
+processing-window working set; reads only touch disk when the working
+set exceeds the cache.
+
+Design (locked-in):
+
+  - **Trigger**: every body always spills (never threshold-based).
+  - **Storage**: file-per-event, ``${scan.temp_dir}/bodies/{uuid}.body[.zst]``.
+  - **Compression**: zstd level 1 (configurable on/off).
+  - **Cache**: bounded by total bytes (default 512 MB), keyed by event
+    UUID, value is decompressed body bytes.
+  - **Eviction**: biggest-first, FIFO tie-break. (Pure LRU is wrong for
+    BBOT's pipeline — "least recently used" tends to be the next event
+    to be processed, not a candidate for eviction.)
+  - **Reads**: synchronous. Cache hit is instant; misses do a small
+    blocking read from page cache (typically microseconds).
+  - **Writes**: synchronous to OS page cache (a few hundred microseconds
+    for a typical body; OS handles actual disk flush async).
+  - **Lifecycle**: file + cache entry are deleted when the event's
+    ``_minimize()`` fires. Scan-end ``rm_rf`` of ``temp_dir`` mops up
+    anything that escaped.
+"""
+
+import logging
+from pathlib import Path
+from typing import Optional
+
+import zstandard as zstd
+
+log = logging.getLogger("bbot.spill")
+
+
+class BodySpillStore:
+    """
+    LRU + on-disk store for HTTP response bodies.
+
+    Bodies are always written to disk on insert. The LRU is a working-set
+    cache — hits are fast, misses re-read from disk. Eviction is
+    biggest-first with FIFO tie-break, which respects BBOT's roughly-FIFO
+    pipeline order (the oldest cache entry is typically the next event
+    to be processed, so the smaller-newer entries are safer to evict).
+
+    Thread-safety: not thread-safe. Designed for single-event-loop use
+    inside a Scanner. Calls cross the event loop boundary only via
+    cooperative ``await``, but I/O itself is synchronous (page cache).
+
+    Hit/miss accounting is exposed via ``stats()`` for benchmark and
+    operational visibility.
+    """
+
+    # Module-internal sentinel for evicted-but-not-deleted entries during
+    # iteration. Currently unused — left here for future async-write work.
+    _UNSET = object()
+
+    def __init__(
+        self,
+        base_dir: Path,
+        cache_bytes: int = 512 * 1024 * 1024,
+        compress: bool = True,
+        compress_level: int = 1,
+    ):
+        self.base_dir = Path(base_dir)
+        self.base_dir.mkdir(parents=True, exist_ok=True)
+        self.cache_bytes = int(cache_bytes)
+        self.compress = bool(compress)
+        # Compressors are cheap to construct but reusable.
+        self._cctx = zstd.ZstdCompressor(level=compress_level) if compress else None
+        self._dctx = zstd.ZstdDecompressor() if compress else None
+
+        # Cache value: dict with bytes + insertion sequence (for FIFO tie-break).
+        # {uuid: {"body": bytes, "seq": int}}
+        self._cache: dict[str, dict] = {}
+        self._cache_total_bytes = 0
+        self._seq = 0  # monotonic insertion counter
+
+        # Stats
+        self._hits = 0
+        self._misses = 0
+        self._writes = 0
+        self._evictions = 0
+
+    # ── Public API ────────────────────────────────────────────────────
+
+    def write(self, event_uuid: str, body: bytes) -> None:
+        """
+        Spill a body to disk and seed the cache. The body remains
+        immediately readable (cache hit) until evicted; thereafter,
+        reads pull from disk.
+
+        ``body`` must be ``bytes``. Callers passing ``str`` should
+        ``.encode("utf-8", errors="replace")`` first.
+        """
+        if not isinstance(body, (bytes, bytearray, memoryview)):
+            raise TypeError(f"body must be bytes-like, got {type(body).__name__}")
+        body_bytes = bytes(body)
+
+        path = self._path_for(event_uuid)
+        if self.compress:
+            on_disk = self._cctx.compress(body_bytes)
+        else:
+            on_disk = body_bytes
+
+        # Synchronous write to page cache. Fast (memcpy), the OS does
+        # the real disk flush asynchronously.
+        path.write_bytes(on_disk)
+        self._writes += 1
+
+        self._insert_cache(event_uuid, body_bytes)
+
+    def read(self, event_uuid: str) -> Optional[bytes]:
+        """
+        Return the body for ``event_uuid``, or ``None`` if neither
+        cache nor disk has it.
+
+        Cache hit: instant. Cache miss + file present: small blocking
+        disk read (page-cached after first hit).
+        """
+        entry = self._cache.get(event_uuid)
+        if entry is not None:
+            self._hits += 1
+            return entry["body"]
+
+        # Miss — try disk.
+        path = self._path_for(event_uuid)
+        if not path.exists():
+            self._misses += 1
+            return None
+
+        on_disk = path.read_bytes()
+        body_bytes = self._dctx.decompress(on_disk) if self.compress else on_disk
+
+        self._misses += 1
+        self._insert_cache(event_uuid, body_bytes)
+        return body_bytes
+
+    def evict_and_delete(self, event_uuid: str) -> None:
+        """
+        Drop ``event_uuid`` from the cache and delete the file.
+
+        Called from ``HTTP_RESPONSE._minimize()`` when the event is no
+        longer needed by any module.
+        """
+        entry = self._cache.pop(event_uuid, None)
+        if entry is not None:
+            self._cache_total_bytes -= len(entry["body"])
+
+        path = self._path_for(event_uuid)
+        try:
+            path.unlink()
+        except FileNotFoundError:
+            pass
+        except Exception as e:  # pragma: no cover — defensive
+            log.debug(f"failed to unlink spill file {path}: {e}")
+
+    def stats(self) -> dict:
+        """Hit/miss/eviction counters and current cache fill."""
+        total = self._hits + self._misses
+        hit_rate = (self._hits / total) if total else 0.0
+        return {
+            "hits": self._hits,
+            "misses": self._misses,
+            "writes": self._writes,
+            "evictions": self._evictions,
+            "hit_rate": round(hit_rate, 4),
+            "cache_entries": len(self._cache),
+            "cache_bytes": self._cache_total_bytes,
+            "cache_bytes_limit": self.cache_bytes,
+        }
+
+    # ── Internal ──────────────────────────────────────────────────────
+
+    def _path_for(self, event_uuid: str) -> Path:
+        suffix = ".body.zst" if self.compress else ".body"
+        return self.base_dir / f"{event_uuid}{suffix}"
+
+    def _insert_cache(self, event_uuid: str, body_bytes: bytes) -> None:
+        """Insert into cache, evicting biggest-first / oldest-first as needed."""
+        body_size = len(body_bytes)
+
+        # Replace existing entry if present (keeps semantics simple if a
+        # body is rewritten — currently never happens, but cheap insurance).
+        existing = self._cache.pop(event_uuid, None)
+        if existing is not None:
+            self._cache_total_bytes -= len(existing["body"])
+
+        # If a single body exceeds the entire cache budget, don't try
+        # to cache it — disk reads will be the only path.
+        if body_size > self.cache_bytes:
+            return
+
+        # Evict until there's room.
+        while self._cache_total_bytes + body_size > self.cache_bytes and self._cache:
+            self._evict_one()
+
+        self._seq += 1
+        self._cache[event_uuid] = {"body": body_bytes, "seq": self._seq}
+        self._cache_total_bytes += body_size
+
+    def _evict_one(self) -> None:
+        """Pick a victim by ``(-size, seq)`` — biggest-first, FIFO tie-break."""
+        # Single-pass min: maximize size, minimize seq among ties.
+        victim_uuid = None
+        victim_size = -1
+        victim_seq = float("inf")
+        for u, entry in self._cache.items():
+            sz = len(entry["body"])
+            if sz > victim_size or (sz == victim_size and entry["seq"] < victim_seq):
+                victim_uuid = u
+                victim_size = sz
+                victim_seq = entry["seq"]
+        if victim_uuid is None:  # pragma: no cover — empty cache, defensive
+            return
+        entry = self._cache.pop(victim_uuid)
+        self._cache_total_bytes -= len(entry["body"])
+        self._evictions += 1
diff --git a/bbot/core/helpers/web/web.py b/bbot/core/helpers/web/web.py
index 7d9bb18ff1..aded0c0de0 100644
--- a/bbot/core/helpers/web/web.py
+++ b/bbot/core/helpers/web/web.py
@@ -597,7 +597,7 @@ def beautifulsoup(
             - Write tests for this function
 
         Examples:
-            >>> soup = self.helpers.beautifulsoup(event.data["body"], "html.parser")
+            >>> soup = self.helpers.beautifulsoup(event.body, "html.parser")
             Perform an html parse of the 'markup' argument and return a soup instance
 
             >>> email_type = soup.find(type="email")
diff --git a/bbot/defaults.yml b/bbot/defaults.yml
index c1394f135a..9a2272c963 100644
--- a/bbot/defaults.yml
+++ b/bbot/defaults.yml
@@ -128,6 +128,16 @@ web:
   # Maximum HTTP requests per second (0 = unlimited)
   # Applies globally across all blasthttp consumers (http probing, web brute, etc.)
   http_rate_limit: 0
+  # HTTP_RESPONSE body disk-spill — keeps body bytes off the Python heap.
+  # Bodies are written to ${scan_home}/temp/bodies/{uuid}.body.zst and served
+  # from a bounded LRU cache; only cache misses touch disk.
+  body_spill:
+    # Master switch
+    enabled: true
+    # In-memory LRU cache budget for hot bodies, in MB
+    cache_mb: 512
+    # zstd level-1 compression on the way to disk
+    compress: true
 
 ### ENGINE ###
 
diff --git a/bbot/modules/ajaxpro.py b/bbot/modules/ajaxpro.py
index bdbebdab52..846133cc5c 100644
--- a/bbot/modules/ajaxpro.py
+++ b/bbot/modules/ajaxpro.py
@@ -36,7 +36,7 @@ async def check_url_event(self, event):
                     await self.confirm_exploitability(probe_url, event)
 
     async def check_http_response_event(self, event):
-        resp_body = event.data.get("body")
+        resp_body = event.body
         if resp_body:
             match = await self.helpers.re.search(self.ajaxpro_regex, resp_body)
             if match:
diff --git a/bbot/modules/badsecrets.py b/bbot/modules/badsecrets.py
index ec673cce12..582fb45b99 100644
--- a/bbot/modules/badsecrets.py
+++ b/bbot/modules/badsecrets.py
@@ -37,7 +37,7 @@ def _module_threads(self):
         return min(4, max(1, multiprocessing.cpu_count() - 1))
 
     async def handle_event(self, event):
-        resp_body = event.data.get("body", None)
+        resp_body = event.body
         resp_headers = event.data.get("header", None)
         resp_cookies = {}
         if resp_headers:
diff --git a/bbot/modules/dotnetnuke.py b/bbot/modules/dotnetnuke.py
index 539eab69a5..b99a7eeca0 100644
--- a/bbot/modules/dotnetnuke.py
+++ b/bbot/modules/dotnetnuke.py
@@ -83,7 +83,7 @@ async def handle_event(self, event):
                     )
                     detected = True
                     break
-        resp_body = event.data.get("body", None)
+        resp_body = event.body
         if resp_body:
             for body_signature in self.DNN_signatures_body:
                 if body_signature in resp_body:
diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py
index 893dcc4300..856c86ea4b 100644
--- a/bbot/modules/internal/excavate.py
+++ b/bbot/modules/internal/excavate.py
@@ -1206,7 +1206,7 @@ async def handle_event(self, event, **kwargs):
                             )
 
             # process response data
-            body = event.data.get("body", "")
+            body = event.body
             headers = event.data.get("header-dict", {})
             if body == "" and headers == {}:
                 return
diff --git a/bbot/modules/newsletters.py b/bbot/modules/newsletters.py
index 409c295689..988bac001c 100644
--- a/bbot/modules/newsletters.py
+++ b/bbot/modules/newsletters.py
@@ -38,12 +38,12 @@ def find_type(self, soup):
     async def handle_event(self, event):
         _event = event
 
-        # Call find_type Function if Webpage return Status Code 200 && "body" is found in event.data
+        # Call find_type Function if Webpage return Status Code 200 && body is non-empty
         # Ex: 'bbot -m blasthttp newsletters -t https://apf-api.eng.vn.cloud.tesla.com' returns
-        #     Status Code 200 but does NOT have event.data["body"]
+        #     Status Code 200 but does NOT have a response body
         if _event.data["status_code"] == 200:
-            if "body" in _event.data:
-                body = _event.data["body"]
+            body = _event.body
+            if body:
                 soup = self.helpers.beautifulsoup(body, "html.parser")
                 if soup is False:
                     self.debug("BeautifulSoup returned False")
diff --git a/bbot/modules/telerik.py b/bbot/modules/telerik.py
index 7cfc761d4d..af795ba71e 100644
--- a/bbot/modules/telerik.py
+++ b/bbot/modules/telerik.py
@@ -378,7 +378,7 @@ async def handle_event(self, event):
                     )
 
         elif event.type == "HTTP_RESPONSE":
-            resp_body = event.data.get("body", None)
+            resp_body = event.body
             url = event.url
             if resp_body:
                 if '":{"SerializedParameters":"' in resp_body:
diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py
index edfe5ca8db..bdedce36dd 100644
--- a/bbot/scanner/scanner.py
+++ b/bbot/scanner/scanner.py
@@ -250,6 +250,15 @@ def __init__(
                 "You have enabled custom HTTP cookies. These will be attached to all in-scope requests and all requests made by blasthttp."
             )
 
+        # HTTP_RESPONSE body disk-spill — keeps body bytes off the Python heap.
+        # See bbot/core/event/spill.py for design notes. Created in _prep()
+        # once temp_dir exists.
+        body_spill_config = web_config.get("body_spill", {})
+        self.body_spill_enabled = bool(body_spill_config.get("enabled", True))
+        self.body_spill_cache_mb = int(body_spill_config.get("cache_mb", 512))
+        self.body_spill_compress = bool(body_spill_config.get("compress", True))
+        self.body_spill_store = None
+
         # url file extensions
         self.url_extension_special = {e.lower() for e in self.config.get("url_extension_special", [])}
         self.url_extension_blacklist = {e.lower() for e in self.config.get("url_extension_blacklist", [])}
@@ -306,6 +315,15 @@ async def _prep(self):
         self.helpers.mkdir(self.home)
         self.helpers.mkdir(self.temp_dir)
 
+        if self.body_spill_enabled and self.body_spill_store is None:
+            from bbot.core.event.spill import BodySpillStore
+
+            self.body_spill_store = BodySpillStore(
+                self.temp_dir / "bodies",
+                cache_bytes=self.body_spill_cache_mb * 1024 * 1024,
+                compress=self.body_spill_compress,
+            )
+
         if not self._modules_loaded:
             self.modules = OrderedDict({})
             self.dummy_modules = {}
diff --git a/bbot/test/test_step_1/test_body_spill.py b/bbot/test/test_step_1/test_body_spill.py
new file mode 100644
index 0000000000..bf6c49dd66
--- /dev/null
+++ b/bbot/test/test_step_1/test_body_spill.py
@@ -0,0 +1,222 @@
+"""
+Tests for HTTP_RESPONSE body disk-spill (``bbot.core.event.spill``).
+
+Two layers:
+  - ``TestBodySpillStore``: direct unit tests of the LRU + disk store.
+  - ``TestHTTPResponseSpill``: integration tests asserting that
+    HTTP_RESPONSE events route bodies through the spill store when one
+    is attached to the scan, and fall back to in-memory data when not.
+"""
+
+import pytest
+
+from bbot.core.event.spill import BodySpillStore
+
+from ..bbot_fixtures import *  # noqa: F401, F403
+
+
+# ── BodySpillStore unit tests ─────────────────────────────────────────
+
+
+class TestBodySpillStore:
+    def test_roundtrip_compressed(self, tmp_path):
+        store = BodySpillStore(tmp_path, cache_bytes=4096, compress=True)
+        # Every byte value plus some recognizable binary magic. If a UTF-8
+        # round-trip ever sneaks back in, this body would be corrupted.
+        raw = bytes(range(256)) + b"PK\x03\x04hello"
+        store.write("uuid-1", raw)
+        assert store.read("uuid-1") == raw
+
+    def test_roundtrip_uncompressed(self, tmp_path):
+        store = BodySpillStore(tmp_path, cache_bytes=4096, compress=False)
+        raw = b"hello world\x00\xff"
+        store.write("uuid-1", raw)
+        assert store.read("uuid-1") == raw
+
+    def test_cache_hit_does_not_touch_disk(self, tmp_path):
+        # After write, body is in cache. Delete the file. Read should still
+        # succeed because the cache holds the body.
+        store = BodySpillStore(tmp_path, cache_bytes=4096)
+        store.write("uuid-1", b"hello")
+        for f in tmp_path.iterdir():
+            f.unlink()
+        assert store.read("uuid-1") == b"hello"
+
+    def test_cache_miss_loads_from_disk(self, tmp_path):
+        # Force eviction by inserting another body that pushes the first out.
+        store = BodySpillStore(tmp_path, cache_bytes=200, compress=False)
+        store.write("a", b"X" * 150)
+        store.write("b", b"Y" * 150)  # would total 300 — must evict 'a'
+        assert "a" not in store._cache
+        # 'a' is still on disk; reading should miss the cache and fetch it.
+        prior_misses = store.stats()["misses"]
+        assert store.read("a") == b"X" * 150
+        assert store.stats()["misses"] == prior_misses + 1
+
+    def test_eviction_biggest_first(self, tmp_path):
+        """
+        Cache budget of 300; insert small (50), medium (100), then big (200).
+        big triggers eviction. Among current cache contents (small=50,
+        medium=100), biggest-first picks medium.
+        """
+        store = BodySpillStore(tmp_path, cache_bytes=300, compress=False)
+        store.write("small", b"X" * 50)
+        store.write("medium", b"Y" * 100)
+        store.write("big", b"Z" * 200)
+        assert "small" in store._cache
+        assert "big" in store._cache
+        assert "medium" not in store._cache
+        assert store.stats()["evictions"] == 1
+
+    def test_eviction_fifo_tiebreak(self, tmp_path):
+        """
+        Two equal-size entries. Inserting a third forces eviction; the
+        older one (smaller seq) goes first.
+        """
+        store = BodySpillStore(tmp_path, cache_bytes=200, compress=False)
+        store.write("first", b"A" * 100)
+        store.write("second", b"B" * 100)  # cache full at 200
+        store.write("new", b"C" * 50)  # forces eviction of 'first'
+        assert "first" not in store._cache
+        assert "second" in store._cache
+        assert "new" in store._cache
+
+    def test_oversized_body_skips_cache(self, tmp_path):
+        """A body bigger than the entire cache budget is written to disk
+        but never enters the cache (would force-evict everything else
+        for nothing)."""
+        store = BodySpillStore(tmp_path, cache_bytes=100, compress=False)
+        store.write("huge", b"X" * 500)
+        assert "huge" not in store._cache
+        # Still readable from disk
+        assert store.read("huge") == b"X" * 500
+
+    def test_evict_and_delete_removes_file_and_cache(self, tmp_path):
+        store = BodySpillStore(tmp_path, cache_bytes=4096)
+        store.write("uuid-1", b"hello")
+        assert any(tmp_path.iterdir())
+        store.evict_and_delete("uuid-1")
+        assert "uuid-1" not in store._cache
+        assert not list(tmp_path.iterdir())
+        # Subsequent reads return None
+        assert store.read("uuid-1") is None
+
+    def test_stats(self, tmp_path):
+        store = BodySpillStore(tmp_path, cache_bytes=4096)
+        store.write("a", b"hello")
+        store.read("a")  # hit
+        store.read("nonexistent")  # miss + nothing on disk
+        s = store.stats()
+        assert s["hits"] == 1
+        assert s["misses"] == 1
+        assert s["writes"] == 1
+        assert s["hit_rate"] == 0.5
+
+    def test_empty_body_not_written(self, tmp_path):
+        # Caller is expected to skip empty bodies; the store doesn't enforce
+        # but should at least roundtrip cleanly.
+        store = BodySpillStore(tmp_path, cache_bytes=4096)
+        store.write("empty", b"")
+        # zstd-compressed empty bytes is still a small frame; we should
+        # be able to round-trip to b"".
+        assert store.read("empty") == b""
+
+    def test_rejects_non_bytes(self, tmp_path):
+        store = BodySpillStore(tmp_path, cache_bytes=4096)
+        with pytest.raises(TypeError):
+            store.write("uuid-1", "string-not-bytes")  # type: ignore[arg-type]
+
+
+# ── HTTP_RESPONSE event integration tests ────────────────────────────
+
+
+@pytest.fixture
+def fake_response_data():
+    """Minimal valid HTTP_RESPONSE data dict."""
+    return {
+        "url": "http://example.com:80/",
+        "input": "http://example.com:80/",
+        "method": "GET",
+        "path": "/",
+        "host": "example.com",
+        "status_code": 200,
+        "title": "Example",
+        "raw_header": "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n\r\n",
+        "header": {"content_type": "text/html"},
+        "content_type": "text/html",
+        "content_length": 23,
+        "body": "<html><body>hi</body></html>",
+        "location": "",
+        "hash": {
+            "body_md5": "x" * 32,
+            "body_mmh3": "0",
+            "body_sha256": "x" * 64,
+            "header_md5": "x" * 32,
+            "header_mmh3": "0",
+            "header_sha256": "x" * 64,
+        },
+    }
+
+
+class TestHTTPResponseSpill:
+    @pytest.mark.asyncio
+    async def test_body_routed_through_spill_store(self, bbot_scanner, fake_response_data):
+        """When a scan has a body_spill_store, body is removed from _data
+        and accessed via event.body which reads from the store."""
+        scan = bbot_scanner("http://example.com")
+        await scan._prep()
+        assert scan.body_spill_store is not None, "body_spill_store should be created in _prep"
+
+        event = scan.make_event(fake_response_data, "HTTP_RESPONSE", parent=scan.root_event)
+        # body should NOT be in _data (it's been spilled)
+        assert "body" not in event._data
+        # but event.body should still return the original body
+        assert event.body == "<html><body>hi</body></html>"
+        # And the spill store has it
+        assert scan.body_spill_store.stats()["writes"] >= 1
+
+    @pytest.mark.asyncio
+    async def test_minimize_evicts_and_deletes(self, bbot_scanner, fake_response_data):
+        """When _minimize() drops _module_consumers to 0, the body file
+        and cache entry should be removed."""
+        scan = bbot_scanner("http://example.com")
+        await scan._prep()
+
+        event = scan.make_event(fake_response_data, "HTTP_RESPONSE", parent=scan.root_event)
+        # Force a normal lifecycle: bump consumers up then drain to 0
+        event._module_consumers = 1
+        event._minimize()  # drops to 0 → triggers eviction
+        # After minimize, body should be gone
+        assert event.body == ""
+        # File should be deleted
+        bodies_dir = scan.temp_dir / "bodies"
+        assert not list(bodies_dir.iterdir()) or all(str(event._uuid) not in p.name for p in bodies_dir.iterdir())
+
+    @pytest.mark.asyncio
+    async def test_disabled_falls_back_to_in_memory(self, bbot_scanner, fake_response_data):
+        """When body_spill is disabled in config, body stays in _data and
+        event.body falls through to it."""
+        scan = bbot_scanner(
+            "http://example.com",
+            config={"web": {"body_spill": {"enabled": False}}},
+        )
+        await scan._prep()
+        assert scan.body_spill_store is None, "store should not be created when disabled"
+
+        event = scan.make_event(fake_response_data, "HTTP_RESPONSE", parent=scan.root_event)
+        # body remains in _data
+        assert event._data.get("body") == "<html><body>hi</body></html>"
+        # event.body still works
+        assert event.body == "<html><body>hi</body></html>"
+
+    @pytest.mark.asyncio
+    async def test_raw_response_uses_spilled_body(self, bbot_scanner, fake_response_data):
+        """The raw_response property should reconstruct the body from
+        the spill store, not from _data."""
+        scan = bbot_scanner("http://example.com")
+        await scan._prep()
+
+        event = scan.make_event(fake_response_data, "HTTP_RESPONSE", parent=scan.root_event)
+        raw = event.raw_response
+        assert "<html><body>hi</body></html>" in raw
+        assert "HTTP/1.1 200 OK" in raw
diff --git a/pyproject.toml b/pyproject.toml
index 12a3ecc755..3eed1ee0c0 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -50,6 +50,7 @@ dependencies = [
     "cloudcheck>=9.2.0,<10",
     "blasthttp>=0.5.1",
     "blastdns>=1.9.0,<2",
+    "zstandard>=0.22,<1",
 ]
 
 [project.urls]
diff --git a/uv.lock b/uv.lock
index 6a6836d936..017946c72b 100644
--- a/uv.lock
+++ b/uv.lock
@@ -226,6 +226,7 @@ dependencies = [
     { name = "xmltojson" },
     { name = "xxhash" },
     { name = "yara-python" },
+    { name = "zstandard" },
 ]
 
 [package.dev-dependencies]
@@ -298,6 +299,7 @@ requires-dist = [
     { name = "xmltojson", specifier = ">=2.0.2,<3" },
     { name = "xxhash", specifier = ">=3.5.0,<4" },
     { name = "yara-python", specifier = "==4.5.2" },
+    { name = "zstandard", specifier = ">=0.22,<1" },
 ]
 
 [package.metadata.requires-dev]
@@ -3461,3 +3463,93 @@ sdist = { url = "https://files.pythonhosted.org/packages/e3/02/0f2892c661036d50e
 wheels = [
     { url = "https://files.pythonhosted.org/packages/2e/54/647ade08bf0db230bfea292f893923872fd20be6ac6f53b2b936ba839d75/zipp-3.23.0-py3-none-any.whl", hash = "sha256:071652d6115ed432f5ce1d34c336c0adfd6a884660d1e9712a256d3d3bd4b14e", size = 10276, upload-time = "2025-06-08T17:06:38.034Z" },
 ]
+
+[[package]]
+name = "zstandard"
+version = "0.25.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/fd/aa/3e0508d5a5dd96529cdc5a97011299056e14c6505b678fd58938792794b1/zstandard-0.25.0.tar.gz", hash = "sha256:7713e1179d162cf5c7906da876ec2ccb9c3a9dcbdffef0cc7f70c3667a205f0b", size = 711513, upload-time = "2025-09-14T22:15:54.002Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/56/7a/28efd1d371f1acd037ac64ed1c5e2b41514a6cc937dd6ab6a13ab9f0702f/zstandard-0.25.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e59fdc271772f6686e01e1b3b74537259800f57e24280be3f29c8a0deb1904dd", size = 795256, upload-time = "2025-09-14T22:15:56.415Z" },
+    { url = "https://files.pythonhosted.org/packages/96/34/ef34ef77f1ee38fc8e4f9775217a613b452916e633c4f1d98f31db52c4a5/zstandard-0.25.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4d441506e9b372386a5271c64125f72d5df6d2a8e8a2a45a0ae09b03cb781ef7", size = 640565, upload-time = "2025-09-14T22:15:58.177Z" },
+    { url = "https://files.pythonhosted.org/packages/9d/1b/4fdb2c12eb58f31f28c4d28e8dc36611dd7205df8452e63f52fb6261d13e/zstandard-0.25.0-cp310-cp310-manylinux2010_i686.manylinux2014_i686.manylinux_2_12_i686.manylinux_2_17_i686.whl", hash = "sha256:ab85470ab54c2cb96e176f40342d9ed41e58ca5733be6a893b730e7af9c40550", size = 5345306, upload-time = "2025-09-14T22:16:00.165Z" },
+    { url = "https://files.pythonhosted.org/packages/73/28/a44bdece01bca027b079f0e00be3b6bd89a4df180071da59a3dd7381665b/zstandard-0.25.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e05ab82ea7753354bb054b92e2f288afb750e6b439ff6ca78af52939ebbc476d", size = 5055561, upload-time = "2025-09-14T22:16:02.22Z" },
+    { url = "https://files.pythonhosted.org/packages/e9/74/68341185a4f32b274e0fc3410d5ad0750497e1acc20bd0f5b5f64ce17785/zstandard-0.25.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:78228d8a6a1c177a96b94f7e2e8d012c55f9c760761980da16ae7546a15a8e9b", size = 5402214, upload-time = "2025-09-14T22:16:04.109Z" },
+    { url = "https://files.pythonhosted.org/packages/8b/67/f92e64e748fd6aaffe01e2b75a083c0c4fd27abe1c8747fee4555fcee7dd/zstandard-0.25.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:2b6bd67528ee8b5c5f10255735abc21aa106931f0dbaf297c7be0c886353c3d0", size = 5449703, upload-time = "2025-09-14T22:16:06.312Z" },
+    { url = "https://files.pythonhosted.org/packages/fd/e5/6d36f92a197c3c17729a2125e29c169f460538a7d939a27eaaa6dcfcba8e/zstandard-0.25.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4b6d83057e713ff235a12e73916b6d356e3084fd3d14ced499d84240f3eecee0", size = 5556583, upload-time = "2025-09-14T22:16:08.457Z" },
+    { url = "https://files.pythonhosted.org/packages/d7/83/41939e60d8d7ebfe2b747be022d0806953799140a702b90ffe214d557638/zstandard-0.25.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:9174f4ed06f790a6869b41cba05b43eeb9a35f8993c4422ab853b705e8112bbd", size = 5045332, upload-time = "2025-09-14T22:16:10.444Z" },
+    { url = "https://files.pythonhosted.org/packages/b3/87/d3ee185e3d1aa0133399893697ae91f221fda79deb61adbe998a7235c43f/zstandard-0.25.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:25f8f3cd45087d089aef5ba3848cd9efe3ad41163d3400862fb42f81a3a46701", size = 5572283, upload-time = "2025-09-14T22:16:12.128Z" },
+    { url = "https://files.pythonhosted.org/packages/0a/1d/58635ae6104df96671076ac7d4ae7816838ce7debd94aecf83e30b7121b0/zstandard-0.25.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:3756b3e9da9b83da1796f8809dd57cb024f838b9eeafde28f3cb472012797ac1", size = 4959754, upload-time = "2025-09-14T22:16:14.225Z" },
+    { url = "https://files.pythonhosted.org/packages/75/d6/57e9cb0a9983e9a229dd8fd2e6e96593ef2aa82a3907188436f22b111ccd/zstandard-0.25.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:81dad8d145d8fd981b2962b686b2241d3a1ea07733e76a2f15435dfb7fb60150", size = 5266477, upload-time = "2025-09-14T22:16:16.343Z" },
+    { url = "https://files.pythonhosted.org/packages/d1/a9/ee891e5edf33a6ebce0a028726f0bbd8567effe20fe3d5808c42323e8542/zstandard-0.25.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:a5a419712cf88862a45a23def0ae063686db3d324cec7edbe40509d1a79a0aab", size = 5440914, upload-time = "2025-09-14T22:16:18.453Z" },
+    { url = "https://files.pythonhosted.org/packages/58/08/a8522c28c08031a9521f27abc6f78dbdee7312a7463dd2cfc658b813323b/zstandard-0.25.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:e7360eae90809efd19b886e59a09dad07da4ca9ba096752e61a2e03c8aca188e", size = 5819847, upload-time = "2025-09-14T22:16:20.559Z" },
+    { url = "https://files.pythonhosted.org/packages/6f/11/4c91411805c3f7b6f31c60e78ce347ca48f6f16d552fc659af6ec3b73202/zstandard-0.25.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:75ffc32a569fb049499e63ce68c743155477610532da1eb38e7f24bf7cd29e74", size = 5363131, upload-time = "2025-09-14T22:16:22.206Z" },
+    { url = "https://files.pythonhosted.org/packages/ef/d6/8c4bd38a3b24c4c7676a7a3d8de85d6ee7a983602a734b9f9cdefb04a5d6/zstandard-0.25.0-cp310-cp310-win32.whl", hash = "sha256:106281ae350e494f4ac8a80470e66d1fe27e497052c8d9c3b95dc4cf1ade81aa", size = 436469, upload-time = "2025-09-14T22:16:25.002Z" },
+    { url = "https://files.pythonhosted.org/packages/93/90/96d50ad417a8ace5f841b3228e93d1bb13e6ad356737f42e2dde30d8bd68/zstandard-0.25.0-cp310-cp310-win_amd64.whl", hash = "sha256:ea9d54cc3d8064260114a0bbf3479fc4a98b21dffc89b3459edd506b69262f6e", size = 506100, upload-time = "2025-09-14T22:16:23.569Z" },
+    { url = "https://files.pythonhosted.org/packages/2a/83/c3ca27c363d104980f1c9cee1101cc8ba724ac8c28a033ede6aab89585b1/zstandard-0.25.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:933b65d7680ea337180733cf9e87293cc5500cc0eb3fc8769f4d3c88d724ec5c", size = 795254, upload-time = "2025-09-14T22:16:26.137Z" },
+    { url = "https://files.pythonhosted.org/packages/ac/4d/e66465c5411a7cf4866aeadc7d108081d8ceba9bc7abe6b14aa21c671ec3/zstandard-0.25.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a3f79487c687b1fc69f19e487cd949bf3aae653d181dfb5fde3bf6d18894706f", size = 640559, upload-time = "2025-09-14T22:16:27.973Z" },
+    { url = "https://files.pythonhosted.org/packages/12/56/354fe655905f290d3b147b33fe946b0f27e791e4b50a5f004c802cb3eb7b/zstandard-0.25.0-cp311-cp311-manylinux2010_i686.manylinux2014_i686.manylinux_2_12_i686.manylinux_2_17_i686.whl", hash = "sha256:0bbc9a0c65ce0eea3c34a691e3c4b6889f5f3909ba4822ab385fab9057099431", size = 5348020, upload-time = "2025-09-14T22:16:29.523Z" },
+    { url = "https://files.pythonhosted.org/packages/3b/13/2b7ed68bd85e69a2069bcc72141d378f22cae5a0f3b353a2c8f50ef30c1b/zstandard-0.25.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:01582723b3ccd6939ab7b3a78622c573799d5d8737b534b86d0e06ac18dbde4a", size = 5058126, upload-time = "2025-09-14T22:16:31.811Z" },
+    { url = "https://files.pythonhosted.org/packages/c9/dd/fdaf0674f4b10d92cb120ccff58bbb6626bf8368f00ebfd2a41ba4a0dc99/zstandard-0.25.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:5f1ad7bf88535edcf30038f6919abe087f606f62c00a87d7e33e7fc57cb69fcc", size = 5405390, upload-time = "2025-09-14T22:16:33.486Z" },
+    { url = "https://files.pythonhosted.org/packages/0f/67/354d1555575bc2490435f90d67ca4dd65238ff2f119f30f72d5cde09c2ad/zstandard-0.25.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:06acb75eebeedb77b69048031282737717a63e71e4ae3f77cc0c3b9508320df6", size = 5452914, upload-time = "2025-09-14T22:16:35.277Z" },
+    { url = "https://files.pythonhosted.org/packages/bb/1f/e9cfd801a3f9190bf3e759c422bbfd2247db9d7f3d54a56ecde70137791a/zstandard-0.25.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9300d02ea7c6506f00e627e287e0492a5eb0371ec1670ae852fefffa6164b072", size = 5559635, upload-time = "2025-09-14T22:16:37.141Z" },
+    { url = "https://files.pythonhosted.org/packages/21/88/5ba550f797ca953a52d708c8e4f380959e7e3280af029e38fbf47b55916e/zstandard-0.25.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:bfd06b1c5584b657a2892a6014c2f4c20e0db0208c159148fa78c65f7e0b0277", size = 5048277, upload-time = "2025-09-14T22:16:38.807Z" },
+    { url = "https://files.pythonhosted.org/packages/46/c0/ca3e533b4fa03112facbe7fbe7779cb1ebec215688e5df576fe5429172e0/zstandard-0.25.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:f373da2c1757bb7f1acaf09369cdc1d51d84131e50d5fa9863982fd626466313", size = 5574377, upload-time = "2025-09-14T22:16:40.523Z" },
+    { url = "https://files.pythonhosted.org/packages/12/9b/3fb626390113f272abd0799fd677ea33d5fc3ec185e62e6be534493c4b60/zstandard-0.25.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6c0e5a65158a7946e7a7affa6418878ef97ab66636f13353b8502d7ea03c8097", size = 4961493, upload-time = "2025-09-14T22:16:43.3Z" },
+    { url = "https://files.pythonhosted.org/packages/cb/d3/23094a6b6a4b1343b27ae68249daa17ae0651fcfec9ed4de09d14b940285/zstandard-0.25.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:c8e167d5adf59476fa3e37bee730890e389410c354771a62e3c076c86f9f7778", size = 5269018, upload-time = "2025-09-14T22:16:45.292Z" },
+    { url = "https://files.pythonhosted.org/packages/8c/a7/bb5a0c1c0f3f4b5e9d5b55198e39de91e04ba7c205cc46fcb0f95f0383c1/zstandard-0.25.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:98750a309eb2f020da61e727de7d7ba3c57c97cf6213f6f6277bb7fb42a8e065", size = 5443672, upload-time = "2025-09-14T22:16:47.076Z" },
+    { url = "https://files.pythonhosted.org/packages/27/22/503347aa08d073993f25109c36c8d9f029c7d5949198050962cb568dfa5e/zstandard-0.25.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:22a086cff1b6ceca18a8dd6096ec631e430e93a8e70a9ca5efa7561a00f826fa", size = 5822753, upload-time = "2025-09-14T22:16:49.316Z" },
+    { url = "https://files.pythonhosted.org/packages/e2/be/94267dc6ee64f0f8ba2b2ae7c7a2df934a816baaa7291db9e1aa77394c3c/zstandard-0.25.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:72d35d7aa0bba323965da807a462b0966c91608ef3a48ba761678cb20ce5d8b7", size = 5366047, upload-time = "2025-09-14T22:16:51.328Z" },
+    { url = "https://files.pythonhosted.org/packages/7b/a3/732893eab0a3a7aecff8b99052fecf9f605cf0fb5fb6d0290e36beee47a4/zstandard-0.25.0-cp311-cp311-win32.whl", hash = "sha256:f5aeea11ded7320a84dcdd62a3d95b5186834224a9e55b92ccae35d21a8b63d4", size = 436484, upload-time = "2025-09-14T22:16:55.005Z" },
+    { url = "https://files.pythonhosted.org/packages/43/a3/c6155f5c1cce691cb80dfd38627046e50af3ee9ddc5d0b45b9b063bfb8c9/zstandard-0.25.0-cp311-cp311-win_amd64.whl", hash = "sha256:daab68faadb847063d0c56f361a289c4f268706b598afbf9ad113cbe5c38b6b2", size = 506183, upload-time = "2025-09-14T22:16:52.753Z" },
+    { url = "https://files.pythonhosted.org/packages/8c/3e/8945ab86a0820cc0e0cdbf38086a92868a9172020fdab8a03ac19662b0e5/zstandard-0.25.0-cp311-cp311-win_arm64.whl", hash = "sha256:22a06c5df3751bb7dc67406f5374734ccee8ed37fc5981bf1ad7041831fa1137", size = 462533, upload-time = "2025-09-14T22:16:53.878Z" },
+    { url = "https://files.pythonhosted.org/packages/82/fc/f26eb6ef91ae723a03e16eddb198abcfce2bc5a42e224d44cc8b6765e57e/zstandard-0.25.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7b3c3a3ab9daa3eed242d6ecceead93aebbb8f5f84318d82cee643e019c4b73b", size = 795738, upload-time = "2025-09-14T22:16:56.237Z" },
+    { url = "https://files.pythonhosted.org/packages/aa/1c/d920d64b22f8dd028a8b90e2d756e431a5d86194caa78e3819c7bf53b4b3/zstandard-0.25.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:913cbd31a400febff93b564a23e17c3ed2d56c064006f54efec210d586171c00", size = 640436, upload-time = "2025-09-14T22:16:57.774Z" },
+    { url = "https://files.pythonhosted.org/packages/53/6c/288c3f0bd9fcfe9ca41e2c2fbfd17b2097f6af57b62a81161941f09afa76/zstandard-0.25.0-cp312-cp312-manylinux2010_i686.manylinux2014_i686.manylinux_2_12_i686.manylinux_2_17_i686.whl", hash = "sha256:011d388c76b11a0c165374ce660ce2c8efa8e5d87f34996aa80f9c0816698b64", size = 5343019, upload-time = "2025-09-14T22:16:59.302Z" },
+    { url = "https://files.pythonhosted.org/packages/1e/15/efef5a2f204a64bdb5571e6161d49f7ef0fffdbca953a615efbec045f60f/zstandard-0.25.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:6dffecc361d079bb48d7caef5d673c88c8988d3d33fb74ab95b7ee6da42652ea", size = 5063012, upload-time = "2025-09-14T22:17:01.156Z" },
+    { url = "https://files.pythonhosted.org/packages/b7/37/a6ce629ffdb43959e92e87ebdaeebb5ac81c944b6a75c9c47e300f85abdf/zstandard-0.25.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:7149623bba7fdf7e7f24312953bcf73cae103db8cae49f8154dd1eadc8a29ecb", size = 5394148, upload-time = "2025-09-14T22:17:03.091Z" },
+    { url = "https://files.pythonhosted.org/packages/e3/79/2bf870b3abeb5c070fe2d670a5a8d1057a8270f125ef7676d29ea900f496/zstandard-0.25.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:6a573a35693e03cf1d67799fd01b50ff578515a8aeadd4595d2a7fa9f3ec002a", size = 5451652, upload-time = "2025-09-14T22:17:04.979Z" },
+    { url = "https://files.pythonhosted.org/packages/53/60/7be26e610767316c028a2cbedb9a3beabdbe33e2182c373f71a1c0b88f36/zstandard-0.25.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5a56ba0db2d244117ed744dfa8f6f5b366e14148e00de44723413b2f3938a902", size = 5546993, upload-time = "2025-09-14T22:17:06.781Z" },
+    { url = "https://files.pythonhosted.org/packages/85/c7/3483ad9ff0662623f3648479b0380d2de5510abf00990468c286c6b04017/zstandard-0.25.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:10ef2a79ab8e2974e2075fb984e5b9806c64134810fac21576f0668e7ea19f8f", size = 5046806, upload-time = "2025-09-14T22:17:08.415Z" },
+    { url = "https://files.pythonhosted.org/packages/08/b3/206883dd25b8d1591a1caa44b54c2aad84badccf2f1de9e2d60a446f9a25/zstandard-0.25.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:aaf21ba8fb76d102b696781bddaa0954b782536446083ae3fdaa6f16b25a1c4b", size = 5576659, upload-time = "2025-09-14T22:17:10.164Z" },
+    { url = "https://files.pythonhosted.org/packages/9d/31/76c0779101453e6c117b0ff22565865c54f48f8bd807df2b00c2c404b8e0/zstandard-0.25.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1869da9571d5e94a85a5e8d57e4e8807b175c9e4a6294e3b66fa4efb074d90f6", size = 4953933, upload-time = "2025-09-14T22:17:11.857Z" },
+    { url = "https://files.pythonhosted.org/packages/18/e1/97680c664a1bf9a247a280a053d98e251424af51f1b196c6d52f117c9720/zstandard-0.25.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:809c5bcb2c67cd0ed81e9229d227d4ca28f82d0f778fc5fea624a9def3963f91", size = 5268008, upload-time = "2025-09-14T22:17:13.627Z" },
+    { url = "https://files.pythonhosted.org/packages/1e/73/316e4010de585ac798e154e88fd81bb16afc5c5cb1a72eeb16dd37e8024a/zstandard-0.25.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:f27662e4f7dbf9f9c12391cb37b4c4c3cb90ffbd3b1fb9284dadbbb8935fa708", size = 5433517, upload-time = "2025-09-14T22:17:16.103Z" },
+    { url = "https://files.pythonhosted.org/packages/5b/60/dd0f8cfa8129c5a0ce3ea6b7f70be5b33d2618013a161e1ff26c2b39787c/zstandard-0.25.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:99c0c846e6e61718715a3c9437ccc625de26593fea60189567f0118dc9db7512", size = 5814292, upload-time = "2025-09-14T22:17:17.827Z" },
+    { url = "https://files.pythonhosted.org/packages/fc/5f/75aafd4b9d11b5407b641b8e41a57864097663699f23e9ad4dbb91dc6bfe/zstandard-0.25.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:474d2596a2dbc241a556e965fb76002c1ce655445e4e3bf38e5477d413165ffa", size = 5360237, upload-time = "2025-09-14T22:17:19.954Z" },
+    { url = "https://files.pythonhosted.org/packages/ff/8d/0309daffea4fcac7981021dbf21cdb2e3427a9e76bafbcdbdf5392ff99a4/zstandard-0.25.0-cp312-cp312-win32.whl", hash = "sha256:23ebc8f17a03133b4426bcc04aabd68f8236eb78c3760f12783385171b0fd8bd", size = 436922, upload-time = "2025-09-14T22:17:24.398Z" },
+    { url = "https://files.pythonhosted.org/packages/79/3b/fa54d9015f945330510cb5d0b0501e8253c127cca7ebe8ba46a965df18c5/zstandard-0.25.0-cp312-cp312-win_amd64.whl", hash = "sha256:ffef5a74088f1e09947aecf91011136665152e0b4b359c42be3373897fb39b01", size = 506276, upload-time = "2025-09-14T22:17:21.429Z" },
+    { url = "https://files.pythonhosted.org/packages/ea/6b/8b51697e5319b1f9ac71087b0af9a40d8a6288ff8025c36486e0c12abcc4/zstandard-0.25.0-cp312-cp312-win_arm64.whl", hash = "sha256:181eb40e0b6a29b3cd2849f825e0fa34397f649170673d385f3598ae17cca2e9", size = 462679, upload-time = "2025-09-14T22:17:23.147Z" },
+    { url = "https://files.pythonhosted.org/packages/35/0b/8df9c4ad06af91d39e94fa96cc010a24ac4ef1378d3efab9223cc8593d40/zstandard-0.25.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ec996f12524f88e151c339688c3897194821d7f03081ab35d31d1e12ec975e94", size = 795735, upload-time = "2025-09-14T22:17:26.042Z" },
+    { url = "https://files.pythonhosted.org/packages/3f/06/9ae96a3e5dcfd119377ba33d4c42a7d89da1efabd5cb3e366b156c45ff4d/zstandard-0.25.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a1a4ae2dec3993a32247995bdfe367fc3266da832d82f8438c8570f989753de1", size = 640440, upload-time = "2025-09-14T22:17:27.366Z" },
+    { url = "https://files.pythonhosted.org/packages/d9/14/933d27204c2bd404229c69f445862454dcc101cd69ef8c6068f15aaec12c/zstandard-0.25.0-cp313-cp313-manylinux2010_i686.manylinux2014_i686.manylinux_2_12_i686.manylinux_2_17_i686.whl", hash = "sha256:e96594a5537722fdfb79951672a2a63aec5ebfb823e7560586f7484819f2a08f", size = 5343070, upload-time = "2025-09-14T22:17:28.896Z" },
+    { url = "https://files.pythonhosted.org/packages/6d/db/ddb11011826ed7db9d0e485d13df79b58586bfdec56e5c84a928a9a78c1c/zstandard-0.25.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:bfc4e20784722098822e3eee42b8e576b379ed72cca4a7cb856ae733e62192ea", size = 5063001, upload-time = "2025-09-14T22:17:31.044Z" },
+    { url = "https://files.pythonhosted.org/packages/db/00/87466ea3f99599d02a5238498b87bf84a6348290c19571051839ca943777/zstandard-0.25.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:457ed498fc58cdc12fc48f7950e02740d4f7ae9493dd4ab2168a47c93c31298e", size = 5394120, upload-time = "2025-09-14T22:17:32.711Z" },
+    { url = "https://files.pythonhosted.org/packages/2b/95/fc5531d9c618a679a20ff6c29e2b3ef1d1f4ad66c5e161ae6ff847d102a9/zstandard-0.25.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:fd7a5004eb1980d3cefe26b2685bcb0b17989901a70a1040d1ac86f1d898c551", size = 5451230, upload-time = "2025-09-14T22:17:34.41Z" },
+    { url = "https://files.pythonhosted.org/packages/63/4b/e3678b4e776db00f9f7b2fe58e547e8928ef32727d7a1ff01dea010f3f13/zstandard-0.25.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8e735494da3db08694d26480f1493ad2cf86e99bdd53e8e9771b2752a5c0246a", size = 5547173, upload-time = "2025-09-14T22:17:36.084Z" },
+    { url = "https://files.pythonhosted.org/packages/4e/d5/ba05ed95c6b8ec30bd468dfeab20589f2cf709b5c940483e31d991f2ca58/zstandard-0.25.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:3a39c94ad7866160a4a46d772e43311a743c316942037671beb264e395bdd611", size = 5046736, upload-time = "2025-09-14T22:17:37.891Z" },
+    { url = "https://files.pythonhosted.org/packages/50/d5/870aa06b3a76c73eced65c044b92286a3c4e00554005ff51962deef28e28/zstandard-0.25.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:172de1f06947577d3a3005416977cce6168f2261284c02080e7ad0185faeced3", size = 5576368, upload-time = "2025-09-14T22:17:40.206Z" },
+    { url = "https://files.pythonhosted.org/packages/5d/35/398dc2ffc89d304d59bc12f0fdd931b4ce455bddf7038a0a67733a25f550/zstandard-0.25.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3c83b0188c852a47cd13ef3bf9209fb0a77fa5374958b8c53aaa699398c6bd7b", size = 4954022, upload-time = "2025-09-14T22:17:41.879Z" },
+    { url = "https://files.pythonhosted.org/packages/9a/5c/36ba1e5507d56d2213202ec2b05e8541734af5f2ce378c5d1ceaf4d88dc4/zstandard-0.25.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:1673b7199bbe763365b81a4f3252b8e80f44c9e323fc42940dc8843bfeaf9851", size = 5267889, upload-time = "2025-09-14T22:17:43.577Z" },
+    { url = "https://files.pythonhosted.org/packages/70/e8/2ec6b6fb7358b2ec0113ae202647ca7c0e9d15b61c005ae5225ad0995df5/zstandard-0.25.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:0be7622c37c183406f3dbf0cba104118eb16a4ea7359eeb5752f0794882fc250", size = 5433952, upload-time = "2025-09-14T22:17:45.271Z" },
+    { url = "https://files.pythonhosted.org/packages/7b/01/b5f4d4dbc59ef193e870495c6f1275f5b2928e01ff5a81fecb22a06e22fb/zstandard-0.25.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:5f5e4c2a23ca271c218ac025bd7d635597048b366d6f31f420aaeb715239fc98", size = 5814054, upload-time = "2025-09-14T22:17:47.08Z" },
+    { url = "https://files.pythonhosted.org/packages/b2/e5/fbd822d5c6f427cf158316d012c5a12f233473c2f9c5fe5ab1ae5d21f3d8/zstandard-0.25.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4f187a0bb61b35119d1926aee039524d1f93aaf38a9916b8c4b78ac8514a0aaf", size = 5360113, upload-time = "2025-09-14T22:17:48.893Z" },
+    { url = "https://files.pythonhosted.org/packages/8e/e0/69a553d2047f9a2c7347caa225bb3a63b6d7704ad74610cb7823baa08ed7/zstandard-0.25.0-cp313-cp313-win32.whl", hash = "sha256:7030defa83eef3e51ff26f0b7bfb229f0204b66fe18e04359ce3474ac33cbc09", size = 436936, upload-time = "2025-09-14T22:17:52.658Z" },
+    { url = "https://files.pythonhosted.org/packages/d9/82/b9c06c870f3bd8767c201f1edbdf9e8dc34be5b0fbc5682c4f80fe948475/zstandard-0.25.0-cp313-cp313-win_amd64.whl", hash = "sha256:1f830a0dac88719af0ae43b8b2d6aef487d437036468ef3c2ea59c51f9d55fd5", size = 506232, upload-time = "2025-09-14T22:17:50.402Z" },
+    { url = "https://files.pythonhosted.org/packages/d4/57/60c3c01243bb81d381c9916e2a6d9e149ab8627c0c7d7abb2d73384b3c0c/zstandard-0.25.0-cp313-cp313-win_arm64.whl", hash = "sha256:85304a43f4d513f5464ceb938aa02c1e78c2943b29f44a750b48b25ac999a049", size = 462671, upload-time = "2025-09-14T22:17:51.533Z" },
+    { url = "https://files.pythonhosted.org/packages/3d/5c/f8923b595b55fe49e30612987ad8bf053aef555c14f05bb659dd5dbe3e8a/zstandard-0.25.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:e29f0cf06974c899b2c188ef7f783607dbef36da4c242eb6c82dcd8b512855e3", size = 795887, upload-time = "2025-09-14T22:17:54.198Z" },
+    { url = "https://files.pythonhosted.org/packages/8d/09/d0a2a14fc3439c5f874042dca72a79c70a532090b7ba0003be73fee37ae2/zstandard-0.25.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:05df5136bc5a011f33cd25bc9f506e7426c0c9b3f9954f056831ce68f3b6689f", size = 640658, upload-time = "2025-09-14T22:17:55.423Z" },
+    { url = "https://files.pythonhosted.org/packages/5d/7c/8b6b71b1ddd517f68ffb55e10834388d4f793c49c6b83effaaa05785b0b4/zstandard-0.25.0-cp314-cp314-manylinux2010_i686.manylinux_2_12_i686.manylinux_2_28_i686.whl", hash = "sha256:f604efd28f239cc21b3adb53eb061e2a205dc164be408e553b41ba2ffe0ca15c", size = 5379849, upload-time = "2025-09-14T22:17:57.372Z" },
+    { url = "https://files.pythonhosted.org/packages/a4/86/a48e56320d0a17189ab7a42645387334fba2200e904ee47fc5a26c1fd8ca/zstandard-0.25.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:223415140608d0f0da010499eaa8ccdb9af210a543fac54bce15babbcfc78439", size = 5058095, upload-time = "2025-09-14T22:17:59.498Z" },
+    { url = "https://files.pythonhosted.org/packages/f8/ad/eb659984ee2c0a779f9d06dbfe45e2dc39d99ff40a319895df2d3d9a48e5/zstandard-0.25.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2e54296a283f3ab5a26fc9b8b5d4978ea0532f37b231644f367aa588930aa043", size = 5551751, upload-time = "2025-09-14T22:18:01.618Z" },
+    { url = "https://files.pythonhosted.org/packages/61/b3/b637faea43677eb7bd42ab204dfb7053bd5c4582bfe6b1baefa80ac0c47b/zstandard-0.25.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ca54090275939dc8ec5dea2d2afb400e0f83444b2fc24e07df7fdef677110859", size = 6364818, upload-time = "2025-09-14T22:18:03.769Z" },
+    { url = "https://files.pythonhosted.org/packages/31/dc/cc50210e11e465c975462439a492516a73300ab8caa8f5e0902544fd748b/zstandard-0.25.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e09bb6252b6476d8d56100e8147b803befa9a12cea144bbe629dd508800d1ad0", size = 5560402, upload-time = "2025-09-14T22:18:05.954Z" },
+    { url = "https://files.pythonhosted.org/packages/c9/ae/56523ae9c142f0c08efd5e868a6da613ae76614eca1305259c3bf6a0ed43/zstandard-0.25.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a9ec8c642d1ec73287ae3e726792dd86c96f5681eb8df274a757bf62b750eae7", size = 4955108, upload-time = "2025-09-14T22:18:07.68Z" },
+    { url = "https://files.pythonhosted.org/packages/98/cf/c899f2d6df0840d5e384cf4c4121458c72802e8bda19691f3b16619f51e9/zstandard-0.25.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:a4089a10e598eae6393756b036e0f419e8c1d60f44a831520f9af41c14216cf2", size = 5269248, upload-time = "2025-09-14T22:18:09.753Z" },
+    { url = "https://files.pythonhosted.org/packages/1b/c0/59e912a531d91e1c192d3085fc0f6fb2852753c301a812d856d857ea03c6/zstandard-0.25.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:f67e8f1a324a900e75b5e28ffb152bcac9fbed1cc7b43f99cd90f395c4375344", size = 5430330, upload-time = "2025-09-14T22:18:11.966Z" },
+    { url = "https://files.pythonhosted.org/packages/a0/1d/7e31db1240de2df22a58e2ea9a93fc6e38cc29353e660c0272b6735d6669/zstandard-0.25.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:9654dbc012d8b06fc3d19cc825af3f7bf8ae242226df5f83936cb39f5fdc846c", size = 5811123, upload-time = "2025-09-14T22:18:13.907Z" },
+    { url = "https://files.pythonhosted.org/packages/f6/49/fac46df5ad353d50535e118d6983069df68ca5908d4d65b8c466150a4ff1/zstandard-0.25.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:4203ce3b31aec23012d3a4cf4a2ed64d12fea5269c49aed5e4c3611b938e4088", size = 5359591, upload-time = "2025-09-14T22:18:16.465Z" },
+    { url = "https://files.pythonhosted.org/packages/c2/38/f249a2050ad1eea0bb364046153942e34abba95dd5520af199aed86fbb49/zstandard-0.25.0-cp314-cp314-win32.whl", hash = "sha256:da469dc041701583e34de852d8634703550348d5822e66a0c827d39b05365b12", size = 444513, upload-time = "2025-09-14T22:18:20.61Z" },
+    { url = "https://files.pythonhosted.org/packages/3a/43/241f9615bcf8ba8903b3f0432da069e857fc4fd1783bd26183db53c4804b/zstandard-0.25.0-cp314-cp314-win_amd64.whl", hash = "sha256:c19bcdd826e95671065f8692b5a4aa95c52dc7a02a4c5a0cac46deb879a017a2", size = 516118, upload-time = "2025-09-14T22:18:17.849Z" },
+    { url = "https://files.pythonhosted.org/packages/f0/ef/da163ce2450ed4febf6467d77ccb4cd52c4c30ab45624bad26ca0a27260c/zstandard-0.25.0-cp314-cp314-win_arm64.whl", hash = "sha256:d7541afd73985c630bafcd6338d2518ae96060075f9463d7dc14cfb33514383d", size = 476940, upload-time = "2025-09-14T22:18:19.088Z" },
+]

From 2d75eb44c862da353321614f04144680c5567a43 Mon Sep 17 00:00:00 2001
From: liquidsec <paul.mueller08@gmail.com>
Date: Fri, 8 May 2026 12:25:00 -0400
Subject: [PATCH 2/6] fix _module_consumers leak in python output module
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The python output module's _worker is a no-op — the scan loop drains
its incoming queue directly via _events_waiting and yields to
async_start callers. But BaseModule.queue_event still ran the
counter-bump (event._module_consumers += 1) on its way in, with no
matching _minimize() call to pair it. Every event flowing through
async_start leaked +1 forever, so _minimize()'s strip block (which
gates on `_module_consumers <= 0`) never fired for any of them.

Effects of the leak: heavy fields (body, raw_header, header, hash,
cert_info, original_value, additional_params, assigned_cookies)
stayed in event._data for the entire scan, even after every module
that wanted them was done. With body-spill enabled, spill cache
entries and on-disk body files also never got released mid-scan
(temp_dir cleanup at scan end still mopped up, so this was
never a permanent leak — but it defeated the purpose of mid-scan
release).

The fix factors the counter bump out of BaseModule.queue_event into
an overridable hook (_increment_consumer_count) and overrides it as
a no-op in the python output module. Other modules unchanged.

Module tests routinely assert on heavy fields after the scan — they
run check() on a collected event list — so the test framework's
_execute_scan now restores BaseModule._increment_consumer_count on
each test scan's python module instance, which keeps events pinned
through check(). Scoped per-scan, doesn't affect production.
---
 bbot/modules/base.py                       | 13 ++++++++++++-
 bbot/modules/output/python.py              | 19 +++++++++++++++++++
 bbot/test/test_step_2/module_tests/base.py | 19 ++++++++++++++++++-
 3 files changed, 49 insertions(+), 2 deletions(-)

diff --git a/bbot/modules/base.py b/bbot/modules/base.py
index 39b4064c88..085b268350 100644
--- a/bbot/modules/base.py
+++ b/bbot/modules/base.py
@@ -1030,7 +1030,7 @@ async def queue_event(self, event):
                 self.debug(f"Queueing {event} because {reason}")
             try:
                 self.incoming_event_queue.put_nowait(event)
-                event._module_consumers += 1
+                self._increment_consumer_count(event)
                 async with self.event_received:
                     self.event_received.notify()
                 if event.type != "FINISHED":
@@ -1038,6 +1038,17 @@ async def queue_event(self, event):
             except AttributeError:
                 self.debug("Not in an acceptable state to queue incoming event")
 
+    def _increment_consumer_count(self, event):
+        """Increment the event's consumer count when it lands in this module's queue.
+
+        Paired with the matching ``_minimize()`` call when the worker
+        finishes processing. Modules that have no real worker (e.g. the
+        ``python`` output module backing ``Scanner.async_start``)
+        override this to skip the increment — otherwise the count
+        leaks +1 forever and ``_minimize()``'s strip block never fires.
+        """
+        event._module_consumers += 1
+
     async def queue_outgoing_event(self, event, **kwargs):
         """
         Queues an outgoing event to the module's outgoing event queue for further processing.
diff --git a/bbot/modules/output/python.py b/bbot/modules/output/python.py
index 81ceb360ed..346cc7e53b 100644
--- a/bbot/modules/output/python.py
+++ b/bbot/modules/output/python.py
@@ -2,8 +2,27 @@
 
 
 class python(BaseOutputModule):
+    """
+    Pseudo-output module backing ``Scanner.async_start()``. The scan loop
+    drains its incoming queue directly via ``_events_waiting`` and yields
+    events to the API caller — there is no real worker.
+
+    Because the worker is a no-op, we override ``queue_event`` so the
+    standard ``_module_consumers`` increment is skipped. Without this,
+    every event leaks +1 on its consumer count (worker would normally
+    pair the increment with a ``_minimize()`` call in its ``finally``
+    block, but there is no worker here). The leak prevents
+    ``_minimize()``'s ``<= 0`` block from ever firing — bodies stay in
+    memory / spill files stay on disk for the entire scan.
+    """
+
     watched_events = ["*"]
     meta = {"description": "Output via Python API", "created_date": "2022-09-13", "author": "@TheTechromancer"}
 
     async def _worker(self):
         pass
+
+    def _increment_consumer_count(self, event):
+        # No-op: see class docstring. The standard increment would leak
+        # because there's no worker to pair it with a `_minimize()` call.
+        pass
diff --git a/bbot/test/test_step_2/module_tests/base.py b/bbot/test/test_step_2/module_tests/base.py
index 35829e217f..df707f7b3d 100644
--- a/bbot/test/test_step_2/module_tests/base.py
+++ b/bbot/test/test_step_2/module_tests/base.py
@@ -125,7 +125,24 @@ async def module_test(
             await asyncio.gather(*tasks, return_exceptions=True)
 
     async def _execute_scan(self, module_test):
-        """Execute the scan and collect events. Can be overridden by benchmark classes."""
+        """Execute the scan and collect events. Can be overridden by benchmark classes.
+
+        Models ``check()`` as a downstream event consumer: tests routinely
+        assert on heavy fields (body, raw_header, original_value, etc.)
+        in ``check()`` after iteration completes. To prevent
+        ``_minimize()`` from stripping those fields before the test sees
+        them, we restore the standard ``_increment_consumer_count``
+        behavior on the ``python`` output module (which normally no-ops
+        the bump because it has no real worker — see
+        ``bbot/modules/output/python.py``). The extra +1 keeps each
+        event's consumer count above zero through scan completion, so
+        the strip block never fires. Events GC at test teardown.
+        """
+        py_mod = module_test.scan.modules.get("python")
+        if py_mod is not None:
+            from bbot.modules.base import BaseModule
+
+            py_mod._increment_consumer_count = BaseModule._increment_consumer_count.__get__(py_mod)
         module_test.events = [e async for e in module_test.scan.async_start()]
 
     @pytest.mark.asyncio

From c1f2677c71cb89decf1cc08596f32bf0941b585f Mon Sep 17 00:00:00 2001
From: liquidsec <paul.mueller08@gmail.com>
Date: Fri, 8 May 2026 14:14:26 -0400
Subject: [PATCH 3/6] fix python-module consumer-counter restore for non-module
 tests
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The previous fix (per-scan restore inside ModuleTestBase._execute_scan)
only covered module tests in test_step_2/module_tests/. Other tests
that collect events with `events = [e async for e in scan.async_start()]`
and read heavy fields (body, resolved_hosts, etc.) after iteration —
e.g. test_step_1/test_dns.py::test_wildcards — still hit the strip
because their scans use the production no-op _increment_consumer_count.

Move the restore to a session-wide loader hook in conftest.py.
ModuleLoader.load_module exec's a fresh module spec each call, so
each Scanner gets a brand-new `python` class object — patching the
imported class via `from bbot.modules.output.python import python;
python._increment_consumer_count = ...` doesn't reach those instances
(verified: `is same as imported python? False`). Wrapping the loader
catches every freshly-materialized class at load time.
---
 bbot/test/conftest.py                      | 38 ++++++++++++++++++++++
 bbot/test/test_step_2/module_tests/base.py | 19 +----------
 2 files changed, 39 insertions(+), 18 deletions(-)

diff --git a/bbot/test/conftest.py b/bbot/test/conftest.py
index b3b1b909f1..40b7180ece 100644
--- a/bbot/test/conftest.py
+++ b/bbot/test/conftest.py
@@ -49,6 +49,44 @@ def silence_live_logging():
             handler.setLevel(logging.CRITICAL)
 
 
+def _patch_python_module_loader():
+    """Restore the standard ``_module_consumers`` bump on the ``python`` output
+    module for every test scan.
+
+    In production, ``python._increment_consumer_count`` is a no-op (its
+    ``_worker`` is also no-op — see ``bbot/modules/output/python.py``),
+    so events flowing through ``async_start`` don't pin themselves and
+    ``_minimize()`` correctly strips heavy fields when their pipeline
+    finishes. But module + integration tests routinely do
+    ``events = [e async for e in scan.async_start()]`` and assert on
+    ``event.tags`` / ``event.resolved_hosts`` / ``event.body`` / etc.
+    *after* the scan completes — so for tests we want the standard
+    increment to fire, which keeps events pinned through assertion time.
+
+    BBOT's ``ModuleLoader.load_module`` exec's a fresh module spec each
+    call, so each Scanner gets a brand-new ``python`` *class object*,
+    not the one we'd see by importing ``bbot.modules.output.python``
+    statically. Patching the imported class therefore has no effect.
+    Instead we wrap the loader: every time a fresh ``python`` class is
+    materialized, restore the increment on it.
+    """
+    from bbot.core.modules import ModuleLoader
+    from bbot.modules.base import BaseModule
+
+    orig = ModuleLoader.load_module
+
+    def patched(self, module_name):
+        cls = orig(self, module_name)
+        if module_name == "python":
+            cls._increment_consumer_count = BaseModule._increment_consumer_count
+        return cls
+
+    ModuleLoader.load_module = patched
+
+
+_patch_python_module_loader()
+
+
 def stop_server(server):
     server.stop()
     while server.is_running():
diff --git a/bbot/test/test_step_2/module_tests/base.py b/bbot/test/test_step_2/module_tests/base.py
index df707f7b3d..35829e217f 100644
--- a/bbot/test/test_step_2/module_tests/base.py
+++ b/bbot/test/test_step_2/module_tests/base.py
@@ -125,24 +125,7 @@ async def module_test(
             await asyncio.gather(*tasks, return_exceptions=True)
 
     async def _execute_scan(self, module_test):
-        """Execute the scan and collect events. Can be overridden by benchmark classes.
-
-        Models ``check()`` as a downstream event consumer: tests routinely
-        assert on heavy fields (body, raw_header, original_value, etc.)
-        in ``check()`` after iteration completes. To prevent
-        ``_minimize()`` from stripping those fields before the test sees
-        them, we restore the standard ``_increment_consumer_count``
-        behavior on the ``python`` output module (which normally no-ops
-        the bump because it has no real worker — see
-        ``bbot/modules/output/python.py``). The extra +1 keeps each
-        event's consumer count above zero through scan completion, so
-        the strip block never fires. Events GC at test teardown.
-        """
-        py_mod = module_test.scan.modules.get("python")
-        if py_mod is not None:
-            from bbot.modules.base import BaseModule
-
-            py_mod._increment_consumer_count = BaseModule._increment_consumer_count.__get__(py_mod)
+        """Execute the scan and collect events. Can be overridden by benchmark classes."""
         module_test.events = [e async for e in module_test.scan.async_start()]
 
     @pytest.mark.asyncio

From 78eea8f88b165180e11c671969d3c4c744123de4 Mon Sep 17 00:00:00 2001
From: liquidsec <paul.mueller08@gmail.com>
Date: Fri, 8 May 2026 14:57:51 -0400
Subject: [PATCH 4/6] trim per-event memory footprint via lazy-init heavy
 fields
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The four heavy event containers (_tags, _resolved_hosts, dns_children,
raw_dns_records) were eagerly allocated as empty set/dict in __init__ —
~560 bytes per event regardless of whether they ever held anything.

Switch to None-by-default with public properties returning shared empty
frozenset/dict singletons. Mutations go through new helpers
(add_resolved_host, update_resolved_hosts, add_dns_child,
set_raw_dns_record) that lazy-allocate real containers on first write.

Slots renamed dns_children → _dns_children and raw_dns_records →
_raw_dns_records so the public attribute names can be properties.

Per-event size: 1626 → 769 bytes for plain DNS_NAME (53% reduction),
1283 for tagged DNS_NAME, 1547 for URL_UNVERIFIED. Benchmark savings
range 5-11 MB depending on event count; scales linearly to ~50-100 MB
on a 100K-event production scan.

Migrated mutation sites: dnsresolve, http, gowitness, asset_inventory.
Also fixed a pre-existing typo in asset_inventory that was reading
_dns_children (didn't exist) instead of dns_children.
---
 bbot/core/event/base.py                | 82 ++++++++++++++++++++++----
 bbot/modules/gowitness.py              |  2 +-
 bbot/modules/http.py                   |  2 +-
 bbot/modules/internal/dnsresolve.py    | 11 ++--
 bbot/modules/output/asset_inventory.py |  3 +-
 5 files changed, 77 insertions(+), 23 deletions(-)

diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py
index d87c920c6b..35c21de8d0 100644
--- a/bbot/core/event/base.py
+++ b/bbot/core/event/base.py
@@ -50,6 +50,16 @@
 log = logging.getLogger("bbot.core.event")
 
 
+# Shared empty defaults for lazy-init slots. Returned from property accessors
+# when the underlying slot is None — saves ~560 bytes per event compared to
+# allocating real empty containers (set/dict) at __init__ time. Mutating
+# helpers (add_tag, add_resolved_host, etc.) replace the slot with a real
+# container before mutating, so callers never see the singletons in a
+# mutable position.
+_EMPTY_FROZENSET: "frozenset[str]" = frozenset()
+_EMPTY_DICT: "dict" = {}
+
+
 class BaseEvent:
     """
     Represents a piece of data discovered during a BBOT scan.
@@ -161,9 +171,12 @@ class BaseEvent:
         "_internal",
         "_dummy",
         "_module",
-        # DNS-related attributes
-        "dns_children",
-        "raw_dns_records",
+        # DNS-related attributes — backing slots; public access via the
+        # ``dns_children`` / ``raw_dns_records`` properties so a None
+        # underlying slot transparently reads as an empty dict (lazy-init
+        # saves ~128 bytes per event).
+        "_dns_children",
+        "_raw_dns_records",
         "dns_resolve_distance",
         # Host metadata (cloud providers, ASN, whois, etc.)
         "_host_metadata",
@@ -213,7 +226,10 @@ def __init__(
         self._hash = None
         self._data = None
         self.__host = None
-        self._tags = set()
+        # Lazy-init: replaced with a real set/dict on first mutation.
+        # Reading via the property returns a shared empty frozenset/dict
+        # so callers never see None.
+        self._tags = None
         self._port = None
         self._omit = False
         self.__words = None
@@ -225,9 +241,9 @@ def __init__(
         self._scope_distance = None
         self._module_priority = None
         self._graph_important = False
-        self._resolved_hosts = set()
-        self.dns_children = {}
-        self.raw_dns_records = {}
+        self._resolved_hosts = None
+        self._dns_children = None
+        self._raw_dns_records = None
         self._discovery_context = ""
         self._module_consumers = 0
 
@@ -292,7 +308,44 @@ def resolved_hosts(self):
             return {
                 self.host,
             }
-        return self._resolved_hosts
+        return self._resolved_hosts if self._resolved_hosts is not None else _EMPTY_FROZENSET
+
+    def add_resolved_host(self, host):
+        """Add a host to ``_resolved_hosts``, lazy-allocating the set."""
+        if self._resolved_hosts is None or isinstance(self._resolved_hosts, frozenset):
+            # promote shared singleton / empty to a real mutable set
+            self._resolved_hosts = set(self._resolved_hosts) if self._resolved_hosts else set()
+        self._resolved_hosts.add(host)
+
+    def update_resolved_hosts(self, hosts):
+        """Add multiple hosts to ``_resolved_hosts``, lazy-allocating the set."""
+        if self._resolved_hosts is None or isinstance(self._resolved_hosts, frozenset):
+            self._resolved_hosts = set(self._resolved_hosts) if self._resolved_hosts else set()
+        self._resolved_hosts.update(hosts)
+
+    @property
+    def dns_children(self):
+        return self._dns_children if self._dns_children is not None else _EMPTY_DICT
+
+    @property
+    def raw_dns_records(self):
+        return self._raw_dns_records if self._raw_dns_records is not None else _EMPTY_DICT
+
+    def add_dns_child(self, rdtype, host):
+        """Record a DNS child host under ``rdtype``, lazy-allocating dict + child set."""
+        if self._dns_children is None:
+            self._dns_children = {}
+        existing = self._dns_children.get(rdtype)
+        if existing is None:
+            self._dns_children[rdtype] = {host}
+        else:
+            existing.add(host)
+
+    def set_raw_dns_record(self, rdtype, answers):
+        """Store the raw DNS answer list for ``rdtype``, lazy-allocating the dict."""
+        if self._raw_dns_records is None:
+            self._raw_dns_records = {}
+        self._raw_dns_records[rdtype] = answers
 
     @data.setter
     def data(self, data):
@@ -474,7 +527,7 @@ def _words(self):
 
     @property
     def tags(self):
-        return self._tags
+        return self._tags if self._tags is not None else _EMPTY_FROZENSET
 
     @tags.setter
     def tags(self, tags):
@@ -485,6 +538,8 @@ def tags(self, tags):
             self.add_tag(tag)
 
     def add_tag(self, tag):
+        if self._tags is None:
+            self._tags = set()
         self._tags.add(sys.intern(tagify(tag)))
 
     def add_tags(self, tags):
@@ -492,6 +547,8 @@ def add_tags(self, tags):
             self.add_tag(tag)
 
     def remove_tag(self, tag):
+        if not self._tags:
+            return
         with suppress(KeyError):
             self._tags.remove(sys.intern(tagify(tag)))
 
@@ -704,9 +761,10 @@ def _minimize(self):
         """
         self._module_consumers = max(0, self._module_consumers - 1)
         if self._module_consumers <= 0:
-            self.dns_children = {}
-            self.raw_dns_records = {}
-            self._resolved_hosts = set()
+            # release container slots; lazy-init pattern means None == empty
+            self._dns_children = None
+            self._raw_dns_records = None
+            self._resolved_hosts = None
 
     def clone(self):
         # Create a shallow copy of the event first
diff --git a/bbot/modules/gowitness.py b/bbot/modules/gowitness.py
index b26018d58c..147981b0cb 100644
--- a/bbot/modules/gowitness.py
+++ b/bbot/modules/gowitness.py
@@ -244,7 +244,7 @@ async def handle_batch(self, *events):
                     context=f"{{module}} visited {{event.type}}: {url}",
                 )
                 if url_event and ip:
-                    url_event._resolved_hosts.add(sys.intern(ip))
+                    url_event.add_resolved_host(sys.intern(ip))
                 await self.emit_event(url_event)
 
         # emit technologies
diff --git a/bbot/modules/http.py b/bbot/modules/http.py
index 04406399d8..cb8d9c4438 100644
--- a/bbot/modules/http.py
+++ b/bbot/modules/http.py
@@ -212,7 +212,7 @@ async def _process_result(self, result, parent_event):
         if url_event:
             response_ip = j.get("host", "")
             if response_ip:
-                url_event._resolved_hosts.add(response_ip)
+                url_event.add_resolved_host(response_ip)
             title = j.get("title", "")
             if title:
                 url_event.http_title = title
diff --git a/bbot/modules/internal/dnsresolve.py b/bbot/modules/internal/dnsresolve.py
index 2e90c5f9f8..a509f87518 100644
--- a/bbot/modules/internal/dnsresolve.py
+++ b/bbot/modules/internal/dnsresolve.py
@@ -243,11 +243,11 @@ async def emit_dns_children_raw(self, event, dns_tags):
     def check_scope(self, event):
         in_target = False
         blacklisted = False
-        dns_children = getattr(event, "dns_children", {})
+        dns_children = event.dns_children
         for rdtype in ("A", "AAAA", "CNAME"):
             hosts = dns_children.get(rdtype, [])
             # update resolved hosts
-            event.resolved_hosts.update(sys.intern(h) for h in hosts)
+            event.update_resolved_hosts(sys.intern(h) for h in hosts)
             for host in hosts:
                 # having a CNAME to an in-scope host doesn't make you in-scope
                 if rdtype != "CNAME":
@@ -286,15 +286,12 @@ async def resolve_event(self, event, types):
 
             event.add_tag(f"{rdtype}-record")
             # blastdns hands us an already-unique list[Record] -- store as-is, no copy
-            event.raw_dns_records[rdtype] = answers
+            event.set_raw_dns_record(rdtype, answers)
             for answer in answers:
                 for _rdtype, host in extract_targets(answer):
                     _rdtype = sys.intern(_rdtype)
                     host = sys.intern(host)
-                    try:
-                        event.dns_children[_rdtype].add(host)
-                    except KeyError:
-                        event.dns_children[_rdtype] = {host}
+                    event.add_dns_child(_rdtype, host)
                     # check for private IPs
                     try:
                         ip = ipaddress.ip_address(host)
diff --git a/bbot/modules/output/asset_inventory.py b/bbot/modules/output/asset_inventory.py
index 7901d6ac8b..bd3b54370b 100644
--- a/bbot/modules/output/asset_inventory.py
+++ b/bbot/modules/output/asset_inventory.py
@@ -291,8 +291,7 @@ def absorb_event(self, event):
         if not is_ip(event.host):
             self.host = event.host
 
-        dns_children = getattr(event, "_dns_children", {})
-        for rdtype, records in sorted(dns_children.items(), key=lambda x: x[0]):
+        for rdtype, records in sorted(event.dns_children.items(), key=lambda x: x[0]):
             for record in sorted([str(r) for r in records]):
                 self.dns_records.add(f"{rdtype}:{record}")
 

From 694ea643c55d7d32060a0ed94522c6941ddcd63b Mon Sep 17 00:00:00 2001
From: liquidsec <paul.mueller08@gmail.com>
Date: Mon, 11 May 2026 11:44:28 -0400
Subject: [PATCH 5/6] throttle ingress under memory pressure
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

When system memory exceeds max_mem_percent (default 90%), ScanIngress
sleeps a per-event delay before pulling the next event. Delay scales
linearly from 0s at the threshold to 5s at threshold+5 (capped at 95%),
recomputed each status tick from current memory.

Slows the front of the pipeline so already-running modules have time
to drain and free memory. No hard pause, no hysteresis, no state
machine — when memory drops back below threshold the delay naturally
returns to 0.

Edge-triggered logs on engage/clear.
---
 bbot/defaults.yml                  |  4 +++
 bbot/scanner/manager.py            |  4 +++
 bbot/scanner/scanner.py            | 33 ++++++++++++++++++----
 bbot/test/test_step_1/test_scan.py | 44 ++++++++++++++++++++++++++++++
 docs/scanning/configuration.md     |  4 +++
 5 files changed, 83 insertions(+), 6 deletions(-)

diff --git a/bbot/defaults.yml b/bbot/defaults.yml
index 9a2272c963..d102d97ac2 100644
--- a/bbot/defaults.yml
+++ b/bbot/defaults.yml
@@ -15,6 +15,10 @@ home: ~/.bbot
 keep_scans: 20
 # Interval for displaying status messages
 status_frequency: 15
+# When system memory exceeds this percentage, ingress is throttled with a per-event sleep
+# that scales linearly from 0s at the threshold up to 5s at threshold+5 (capped at 95%).
+# Last-ditch effort to give the pipeline a chance to drain before OOM.
+max_mem_percent: 90
 # Include the raw data of files (i.e. PDFs, web screenshots) as base64 in the event
 file_blobs: false
 # Include the raw data of directories (i.e. git repos) as tar.gz base64 in the event
diff --git a/bbot/scanner/manager.py b/bbot/scanner/manager.py
index 0ad7f59c85..ddf5048be4 100644
--- a/bbot/scanner/manager.py
+++ b/bbot/scanner/manager.py
@@ -145,6 +145,10 @@ def module_priority_weights(self):
         return self._module_priority_weights
 
     async def get_incoming_event(self):
+        # memory-pressure backpressure: scan-level delay scales 0s→5s as memory crosses 90→95%.
+        # delay is 0 in the common case so this is a near-free check.
+        if self.scan._ingress_delay > 0:
+            await asyncio.sleep(self.scan._ingress_delay)
         for q in self.helpers.weighted_shuffle(self.incoming_queues, self.module_priority_weights):
             try:
                 return q.get_nowait()
diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py
index bdedce36dd..c7a7aa0a1e 100644
--- a/bbot/scanner/scanner.py
+++ b/bbot/scanner/scanner.py
@@ -273,6 +273,12 @@ def __init__(
         # how often to print scan status
         self.status_frequency = self.config.get("status_frequency", 15)
 
+        # memory-pressure ingress throttle: when system memory exceeds max_mem_percent,
+        # ScanIngress sleeps _ingress_delay seconds before pulling each event.
+        # delay is recomputed every status tick from current memory.
+        self.max_mem_percent = self.config.get("max_mem_percent", 90)
+        self._ingress_delay = 0.0
+
         from .stats import ScanStats
 
         self.stats = ScanStats(self)
@@ -746,14 +752,20 @@ def modules_status(self, _log=False, detailed=False):
 
         modules_errored = [m for m, s in status["modules"].items() if s["errored"]]
 
-        max_mem_percent = 90
         mem_status = self.helpers.memory_status()
-        # abort if we don't have the memory
         mem_percent = mem_status.percent
-        if mem_percent > max_mem_percent:
-            free_memory = mem_status.available
-            free_memory_human = self.helpers.bytes_to_human(free_memory)
-            self.warning(f"System memory is at {mem_percent:.1f}% ({free_memory_human} remaining)")
+        prev_delay = self._ingress_delay
+        new_delay = self._compute_ingress_delay(mem_percent)
+        self._ingress_delay = new_delay
+        if mem_percent > self.max_mem_percent:
+            free_memory_human = self.helpers.bytes_to_human(mem_status.available)
+            if prev_delay == 0.0 and new_delay > 0.0:
+                self.warning(
+                    f"System memory is at {mem_percent:.1f}% ({free_memory_human} remaining); "
+                    f"throttling ingress ({new_delay:.1f}s/event) to let the pipeline drain"
+                )
+        elif prev_delay > 0.0 and new_delay == 0.0:
+            self.hugesuccess(f"System memory dropped to {mem_percent:.1f}%; ingress throttle cleared")
 
         if _log:
             modules_status = []
@@ -826,6 +838,15 @@ def modules_status(self, _log=False, detailed=False):
 
         return status
 
+    def _compute_ingress_delay(self, mem_percent):
+        # 0s at threshold, scaling linearly to 5s at threshold+5 (capped at 95%).
+        if mem_percent <= self.max_mem_percent:
+            return 0.0
+        cap = min(self.max_mem_percent + 5, 95)
+        overshoot_range = max(cap - self.max_mem_percent, 1)
+        overshoot = min(mem_percent - self.max_mem_percent, overshoot_range)
+        return (overshoot / overshoot_range) * 5.0
+
     async def async_stop(self):
         """Stops the in-progress scan and performs necessary cleanup.
 
diff --git a/bbot/test/test_step_1/test_scan.py b/bbot/test/test_step_1/test_scan.py
index 995d1e9c2a..45f97bd97d 100644
--- a/bbot/test/test_step_1/test_scan.py
+++ b/bbot/test/test_step_1/test_scan.py
@@ -431,3 +431,47 @@ async def test_scan_name(bbot_scanner):
     await scan._prep()
     assert scan.name == "test_scan_name"
     assert scan.preset.scan_name == "test_scan_name"
+
+
+@pytest.mark.asyncio
+async def test_memory_backpressure_throttle(bbot_scanner, monkeypatch):
+    """Ingress delay scales linearly with memory overshoot, and the scan still completes under pressure."""
+    from types import SimpleNamespace
+
+    mem_percent = [50.0]
+
+    def mock_memory_status():
+        return SimpleNamespace(percent=mem_percent[0], available=1_000_000_000)
+
+    scan = bbot_scanner("127.0.0.1", config={"max_mem_percent": 90})
+    await scan._prep()
+    await scan.helpers.dns._mock_dns({"1.1.1.1.in-addr.arpa": {"PTR": ["one.one.one.one"]}})
+    monkeypatch.setattr("bbot.core.helpers.misc.memory_status", mock_memory_status)
+
+    # delay curve — pure function of memory percent
+    assert scan._compute_ingress_delay(50.0) == 0.0
+    assert scan._compute_ingress_delay(90.0) == 0.0
+    assert scan._compute_ingress_delay(91.0) == pytest.approx(1.0)
+    assert scan._compute_ingress_delay(92.5) == pytest.approx(2.5)
+    assert scan._compute_ingress_delay(95.0) == pytest.approx(5.0)
+    # capped above threshold+5
+    assert scan._compute_ingress_delay(99.0) == pytest.approx(5.0)
+
+    # status loop wires memory_status -> _ingress_delay
+    assert scan._ingress_delay == 0.0
+
+    mem_percent[0] = 93.0
+    scan.modules_status(_log=False)
+    assert scan._ingress_delay == pytest.approx(3.0), "delay should engage above threshold"
+
+    mem_percent[0] = 97.0
+    scan.modules_status(_log=False)
+    assert scan._ingress_delay == pytest.approx(5.0), "delay should clamp at the cap"
+
+    mem_percent[0] = 80.0
+    scan.modules_status(_log=False)
+    assert scan._ingress_delay == 0.0, "delay should clear once memory drops back"
+
+    # scan still produces events with the throttle disengaged
+    events = [e async for e in scan.async_start()]
+    assert any(e.type == "IP_ADDRESS" for e in events), "scan should still produce events"
diff --git a/docs/scanning/configuration.md b/docs/scanning/configuration.md
index 221f22a860..3fefd0e999 100644
--- a/docs/scanning/configuration.md
+++ b/docs/scanning/configuration.md
@@ -74,6 +74,10 @@ home: ~/.bbot
 keep_scans: 20
 # Interval for displaying status messages
 status_frequency: 15
+# When system memory exceeds this percentage, ingress is throttled with a per-event sleep
+# that scales linearly from 0s at the threshold up to 5s at threshold+5 (capped at 95%).
+# Last-ditch effort to give the pipeline a chance to drain before OOM.
+max_mem_percent: 90
 # Include the raw data of files (i.e. PDFs, web screenshots) as base64 in the event
 file_blobs: false
 # Include the raw data of directories (i.e. git repos) as tar.gz base64 in the event

From c171610fa0447264a6d26e721989c3eb682f7cb4 Mon Sep 17 00:00:00 2001
From: liquidsec <paul.mueller08@gmail.com>
Date: Tue, 12 May 2026 21:31:11 -0400
Subject: [PATCH 6/6] use MappingProxyType for _EMPTY_DICT to prevent singleton
 poisoning

---
 bbot/core/event/base.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py
index 35c21de8d0..2bf59e62a8 100644
--- a/bbot/core/event/base.py
+++ b/bbot/core/event/base.py
@@ -14,6 +14,7 @@
 from typing import Optional
 from zoneinfo import ZoneInfo
 from copy import copy, deepcopy
+from types import MappingProxyType
 from contextlib import suppress
 from radixtarget import RadixTarget
 from pydantic import BaseModel, field_validator
@@ -57,7 +58,7 @@
 # container before mutating, so callers never see the singletons in a
 # mutable position.
 _EMPTY_FROZENSET: "frozenset[str]" = frozenset()
-_EMPTY_DICT: "dict" = {}
+_EMPTY_DICT = MappingProxyType({})
 
 
 class BaseEvent: