From cb5940b0eeb7a0ff8a425146a1ac8433d470f8d3 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Tue, 31 Mar 2026 09:52:20 -0400 Subject: [PATCH 1/5] Add scan memory benchmarks and MB support in benchmark report --- bbot/scripts/benchmark_report.py | 15 ++- bbot/test/benchmarks/test_scan_memory.py | 162 +++++++++++++++++++++++ 2 files changed, 174 insertions(+), 3 deletions(-) create mode 100644 bbot/test/benchmarks/test_scan_memory.py diff --git a/bbot/scripts/benchmark_report.py b/bbot/scripts/benchmark_report.py index 675c99bc64..12a1d23f86 100644 --- a/bbot/scripts/benchmark_report.py +++ b/bbot/scripts/benchmark_report.py @@ -245,10 +245,16 @@ def generate_comparison_table(current_data: Dict, base_data: Dict, current_branc else: base_ops = 1 / base_mean # Default: single operation - # Use per-event memory if available, otherwise use time + # Use memory metrics if available, otherwise use time + current_mb = current_extra.get("total_memory_mb") + base_mb = base_extra.get("total_memory_mb") current_peb = current_extra.get("per_event_bytes") base_peb = base_extra.get("per_event_bytes") - if current_peb is not None and base_peb is not None: + if current_mb is not None and base_mb is not None and current_peb is None: + change_percent, emoji = calculate_change_percentage(base_mb, current_mb) + base_label = f"{base_mb:.1f} MB" + current_label = f"{current_mb:.1f} MB" + elif current_peb is not None and base_peb is not None: change_percent, emoji = calculate_change_percentage(base_peb, current_peb) base_label = f"{base_peb:.0f} B/event" current_label = f"{current_peb:.0f} B/event" @@ -269,7 +275,10 @@ def generate_comparison_table(current_data: Dict, base_data: Dict, current_branc # Track significant changes if abs(change_percent) > 10: - is_memory = current_extra.get("per_event_bytes") is not None + is_memory = ( + current_extra.get("per_event_bytes") is not None + or current_extra.get("total_memory_mb") is not None + ) if is_memory: direction = "🐌 more memory" if change_percent > 0 else "🚀 less memory" else: diff --git a/bbot/test/benchmarks/test_scan_memory.py b/bbot/test/benchmarks/test_scan_memory.py new file mode 100644 index 0000000000..c25864fc4b --- /dev/null +++ b/bbot/test/benchmarks/test_scan_memory.py @@ -0,0 +1,162 @@ +""" +Memory benchmarks for BBOT scan patterns. + +Runs real scans against a local HTTP server and measures peak traced +memory via tracemalloc. The key metric is `total_memory_mb` in extra_info, +which the benchmark report script picks up and displays as MB. +""" + +import gc +import asyncio +import threading +import tracemalloc +from http.server import HTTPServer, BaseHTTPRequestHandler + +import pytest + +from bbot.scanner import Scanner + + +NUM_PAGES = 500 +BODY_SIZE = 500_000 # 500 KB per page + + +class _BenchmarkHTTPHandler(BaseHTTPRequestHandler): + """Serves an index page linking to sub-pages with large bodies.""" + + def do_GET(self): + if self.path == "/": + links = "".join(f'page{i}\n' for i in range(NUM_PAGES)) + body = f"{links}" + elif self.path.startswith("/page"): + i = self.path.replace("/page", "") + links = f'infodetails' + body = f"

Page {i}

{links}{'A' * BODY_SIZE}" + elif self.path.startswith("/data"): + body = "data endpoint" + else: + self.send_response(404) + self.end_headers() + return + self.send_response(200) + self.send_header("Content-Type", "text/html") + self.end_headers() + self.wfile.write(body.encode()) + + def log_message(self, *args): + pass + + +def _start_server(): + server = HTTPServer(("127.0.0.1", 0), _BenchmarkHTTPHandler) + port = server.server_address[1] + threading.Thread(target=server.serve_forever, daemon=True).start() + return server, port + + +def _measure_peak(coro_func): + """Run an async scan under tracemalloc and return peak memory in MB.""" + gc.collect() + if tracemalloc.is_tracing(): + tracemalloc.stop() + tracemalloc.start() + asyncio.run(coro_func()) + _, peak = tracemalloc.get_traced_memory() + tracemalloc.stop() + return round(peak / 1024 / 1024, 2) + + +# --------------------------------------------------------------------------- +# 1) Web crawl — httpx visits many pages, excavate processes bodies +# --------------------------------------------------------------------------- + + +async def _web_crawl_scan(): + server, port = _start_server() + try: + scan = Scanner( + f"http://127.0.0.1:{port}/", + modules=["httpx"], + output_modules=["python"], + config={ + "dns": {"disable": True}, + "scope": {"search_distance": 0}, + "web": {"spider_distance": 10, "spider_depth": 10, "spider_links_per_page": NUM_PAGES}, + "speculate": True, + "excavate": True, + "aggregate": False, + "cloudcheck": False, + "modules": {"httpx": {"batch_size": 25}}, + }, + force_start=True, + ) + events = [] + async for event in scan.async_start(): + events.append(event) + finally: + server.shutdown() + + +class TestWebCrawlMemory: + """Measures peak memory during a realistic web crawl with large pages.""" + + @pytest.mark.benchmark(group="memory_scan_patterns") + def test_web_crawl(self, benchmark): + peak_mb = _measure_peak(_web_crawl_scan) + benchmark.extra_info["total_memory_mb"] = peak_mb + benchmark.extra_info["num_pages"] = NUM_PAGES + benchmark.pedantic(lambda: None, iterations=1, rounds=1, warmup_rounds=0) + + +# --------------------------------------------------------------------------- +# 2) Subdomain enum — many DNS_NAME events, no heavy bodies +# --------------------------------------------------------------------------- + +SUBDOMAIN_ENUM_COUNT = 5000 + + +async def _subdomain_enum_scan(): + scan = Scanner( + "blacklanternsecurity.com", + modules=[], + output_modules=["python"], + config={ + "dns": {"disable": True}, + "scope": {"search_distance": 0}, + "web": {"spider_distance": 0, "spider_depth": 0}, + "speculate": False, + "excavate": True, + "aggregate": False, + "cloudcheck": False, + }, + force_start=True, + ) + events = [] + injected = False + + async for event in scan.async_start(): + events.append(event) + if event.type == "SCAN" and not injected: + injected = True + root_event = scan.root_event + for i in range(SUBDOMAIN_ENUM_COUNT): + dns_event = scan.make_event( + f"sub{i}.blacklanternsecurity.com", + "DNS_NAME", + parent=root_event, + context=f"benchmark DNS_NAME {i}", + ) + await scan.ingress_module.queue_event(dns_event, {}) + + return events + + +class TestSubdomainEnumMemory: + """Measures peak memory during a large subdomain enumeration.""" + + @pytest.mark.benchmark(group="memory_scan_patterns") + def test_subdomain_enum(self, benchmark): + peak_mb = _measure_peak(_subdomain_enum_scan) + benchmark.extra_info["total_memory_mb"] = peak_mb + benchmark.extra_info["num_subdomains"] = SUBDOMAIN_ENUM_COUNT + benchmark.pedantic(lambda: None, iterations=1, rounds=1, warmup_rounds=0) From 15a68e3d58ce37ed967a30f940e69afe01e45646 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Tue, 31 Mar 2026 11:34:11 -0400 Subject: [PATCH 2/5] Fix scan memory benchmarks to exclude Scanner init from tracemalloc Scanner construction allocates 400+ MB in pytest (presets, module loading, etc.) which was setting the tracemalloc peak before any scan events existed, masking real differences between branches. Split scanner init out of the tracemalloc window so we measure only scan execution memory. Also separate "new tests" from "significant changes" in benchmark report output. --- bbot/scripts/benchmark_report.py | 12 +++- bbot/test/benchmarks/test_scan_memory.py | 84 ++++++++++++++---------- 2 files changed, 60 insertions(+), 36 deletions(-) diff --git a/bbot/scripts/benchmark_report.py b/bbot/scripts/benchmark_report.py index 12a1d23f86..50ca6f3384 100644 --- a/bbot/scripts/benchmark_report.py +++ b/bbot/scripts/benchmark_report.py @@ -180,6 +180,7 @@ def generate_comparison_table(current_data: Dict, base_data: Dict, current_branc |--------------|---------|------------|-----------|-----------|""" significant_changes = [] + new_tests = [] performance_summary = [] for current_bench in current_benchmarks: @@ -304,9 +305,7 @@ def generate_comparison_table(current_data: Dict, base_data: Dict, current_branc else: table += f"\n| **{test_name}** | `-` | `{format_time(current_mean)}` | **New** 🆕 | 🆕 |" - significant_changes.append( - f"- **{test_name}**: New test 🆕 ({format_time(current_mean)}, {format_ops(current_ops)})" - ) + new_tests.append(f"- **{test_name}**: {format_time(current_mean)}, {format_ops(current_ops)}") table += "\n\n\n\n" @@ -332,6 +331,13 @@ def generate_comparison_table(current_data: Dict, base_data: Dict, current_branc table += f"{change}\n" table += "\n" + # Add new tests section + if new_tests: + table += "### 🆕 New Tests\n\n" + for new_test in new_tests: + table += f"{new_test}\n" + table += "\n" + return table diff --git a/bbot/test/benchmarks/test_scan_memory.py b/bbot/test/benchmarks/test_scan_memory.py index c25864fc4b..a7e4653d64 100644 --- a/bbot/test/benchmarks/test_scan_memory.py +++ b/bbot/test/benchmarks/test_scan_memory.py @@ -4,6 +4,11 @@ Runs real scans against a local HTTP server and measures peak traced memory via tracemalloc. The key metric is `total_memory_mb` in extra_info, which the benchmark report script picks up and displays as MB. + +Scanner construction is done outside the tracemalloc window because it +pulls in presets, module loading, and other heavy one-time setup that can +exceed 400 MB in a pytest process -- far more than the actual scan -- and +would set the tracemalloc peak before a single event is created. """ import gc @@ -54,47 +59,56 @@ def _start_server(): return server, port -def _measure_peak(coro_func): - """Run an async scan under tracemalloc and return peak memory in MB.""" +def _measure_peak(scan_factory, run_scan): + """Build the scanner untraced, then measure only scan execution memory.""" + # Phase 1: build the scanner (untraced) + scan, cleanup = scan_factory() + + # Phase 2: trace only the scan execution gc.collect() if tracemalloc.is_tracing(): tracemalloc.stop() tracemalloc.start() - asyncio.run(coro_func()) - _, peak = tracemalloc.get_traced_memory() - tracemalloc.stop() + try: + asyncio.run(run_scan(scan)) + finally: + _, peak = tracemalloc.get_traced_memory() + tracemalloc.stop() + if cleanup: + cleanup() return round(peak / 1024 / 1024, 2) # --------------------------------------------------------------------------- -# 1) Web crawl — httpx visits many pages, excavate processes bodies +# 1) Web crawl -- httpx visits many pages, excavate processes bodies # --------------------------------------------------------------------------- -async def _web_crawl_scan(): +def _web_crawl_factory(): server, port = _start_server() - try: - scan = Scanner( - f"http://127.0.0.1:{port}/", - modules=["httpx"], - output_modules=["python"], - config={ - "dns": {"disable": True}, - "scope": {"search_distance": 0}, - "web": {"spider_distance": 10, "spider_depth": 10, "spider_links_per_page": NUM_PAGES}, - "speculate": True, - "excavate": True, - "aggregate": False, - "cloudcheck": False, - "modules": {"httpx": {"batch_size": 25}}, - }, - force_start=True, - ) - events = [] - async for event in scan.async_start(): - events.append(event) - finally: - server.shutdown() + scan = Scanner( + f"http://127.0.0.1:{port}/", + modules=["httpx"], + output_modules=["python"], + config={ + "dns": {"disable": True}, + "scope": {"search_distance": 0}, + "web": {"spider_distance": 10, "spider_depth": 10, "spider_links_per_page": NUM_PAGES}, + "speculate": True, + "excavate": True, + "aggregate": False, + "cloudcheck": False, + "modules": {"httpx": {"batch_size": 25}}, + }, + force_start=True, + ) + return scan, server.shutdown + + +async def _web_crawl_run(scan): + events = [] + async for event in scan.async_start(): + events.append(event) class TestWebCrawlMemory: @@ -102,20 +116,20 @@ class TestWebCrawlMemory: @pytest.mark.benchmark(group="memory_scan_patterns") def test_web_crawl(self, benchmark): - peak_mb = _measure_peak(_web_crawl_scan) + peak_mb = _measure_peak(_web_crawl_factory, _web_crawl_run) benchmark.extra_info["total_memory_mb"] = peak_mb benchmark.extra_info["num_pages"] = NUM_PAGES benchmark.pedantic(lambda: None, iterations=1, rounds=1, warmup_rounds=0) # --------------------------------------------------------------------------- -# 2) Subdomain enum — many DNS_NAME events, no heavy bodies +# 2) Subdomain enum -- many DNS_NAME events, no heavy bodies # --------------------------------------------------------------------------- SUBDOMAIN_ENUM_COUNT = 5000 -async def _subdomain_enum_scan(): +def _subdomain_enum_factory(): scan = Scanner( "blacklanternsecurity.com", modules=[], @@ -131,6 +145,10 @@ async def _subdomain_enum_scan(): }, force_start=True, ) + return scan, None + + +async def _subdomain_enum_run(scan): events = [] injected = False @@ -156,7 +174,7 @@ class TestSubdomainEnumMemory: @pytest.mark.benchmark(group="memory_scan_patterns") def test_subdomain_enum(self, benchmark): - peak_mb = _measure_peak(_subdomain_enum_scan) + peak_mb = _measure_peak(_subdomain_enum_factory, _subdomain_enum_run) benchmark.extra_info["total_memory_mb"] = peak_mb benchmark.extra_info["num_subdomains"] = SUBDOMAIN_ENUM_COUNT benchmark.pedantic(lambda: None, iterations=1, rounds=1, warmup_rounds=0) From 590e979bbb2b618cac749e6f51adcdced797a199 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Tue, 31 Mar 2026 15:01:25 -0400 Subject: [PATCH 3/5] Run scan memory benchmarks as subprocesses for clean tracemalloc pytest's own allocations (~200 MB) contaminate tracemalloc peak measurements when scans run in-process, masking real differences between branches. Run each benchmark scan as a subprocess instead so measurements reflect only the scan's own memory use. Also rename tests to test_memory_use_* for clarity. --- bbot/test/benchmarks/test_scan_memory.py | 205 +++++++++++------------ 1 file changed, 99 insertions(+), 106 deletions(-) diff --git a/bbot/test/benchmarks/test_scan_memory.py b/bbot/test/benchmarks/test_scan_memory.py index a7e4653d64..82c236edff 100644 --- a/bbot/test/benchmarks/test_scan_memory.py +++ b/bbot/test/benchmarks/test_scan_memory.py @@ -1,122 +1,111 @@ """ Memory benchmarks for BBOT scan patterns. -Runs real scans against a local HTTP server and measures peak traced -memory via tracemalloc. The key metric is `total_memory_mb` in extra_info, -which the benchmark report script picks up and displays as MB. - -Scanner construction is done outside the tracemalloc window because it -pulls in presets, module loading, and other heavy one-time setup that can -exceed 400 MB in a pytest process -- far more than the actual scan -- and -would set the tracemalloc peak before a single event is created. +Each benchmark launches a scan as a subprocess so tracemalloc measurements +are not contaminated by pytest's own allocations. The subprocess writes +peak memory (MB) to stdout, which the test reads and stores in +benchmark extra_info["total_memory_mb"]. """ -import gc -import asyncio -import threading -import tracemalloc -from http.server import HTTPServer, BaseHTTPRequestHandler +import subprocess +import sys import pytest -from bbot.scanner import Scanner - NUM_PAGES = 500 BODY_SIZE = 500_000 # 500 KB per page +SUBDOMAIN_ENUM_COUNT = 5000 + + +def _run_scan_subprocess(script: str) -> float: + """Run a scan script in a clean subprocess, return peak memory in MB.""" + result = subprocess.run( + [sys.executable, "-c", script], + capture_output=True, + text=True, + timeout=600, + ) + if result.returncode != 0: + raise RuntimeError(f"Scan subprocess failed:\n{result.stderr[-2000:]}") + for line in result.stdout.strip().splitlines(): + if line.startswith("PEAK_MB:"): + return float(line.split(":", 1)[1]) + raise RuntimeError(f"No PEAK_MB in subprocess output:\n{result.stdout[-2000:]}") + + +# --------------------------------------------------------------------------- +# 1) Web crawl -- httpx visits many pages, excavate processes bodies +# --------------------------------------------------------------------------- +_WEB_CRAWL_SCRIPT = """ +import gc, asyncio, threading, tracemalloc +from http.server import HTTPServer, BaseHTTPRequestHandler +from bbot.scanner import Scanner -class _BenchmarkHTTPHandler(BaseHTTPRequestHandler): - """Serves an index page linking to sub-pages with large bodies.""" +NUM_PAGES = NUM_PAGES_PLACEHOLDER +BODY_SIZE = BODY_SIZE_PLACEHOLDER +class H(BaseHTTPRequestHandler): def do_GET(self): if self.path == "/": - links = "".join(f'page{i}\n' for i in range(NUM_PAGES)) - body = f"{links}" + links = "".join(f'page{i}' for i in range(NUM_PAGES)) + body = "" + links + "" elif self.path.startswith("/page"): i = self.path.replace("/page", "") links = f'infodetails' - body = f"

Page {i}

{links}{'A' * BODY_SIZE}" + body = "

Page " + i + "

" + links + "A" * BODY_SIZE + "" elif self.path.startswith("/data"): body = "data endpoint" else: - self.send_response(404) - self.end_headers() - return + self.send_response(404); self.end_headers(); return self.send_response(200) self.send_header("Content-Type", "text/html") self.end_headers() self.wfile.write(body.encode()) - - def log_message(self, *args): - pass - - -def _start_server(): - server = HTTPServer(("127.0.0.1", 0), _BenchmarkHTTPHandler) - port = server.server_address[1] - threading.Thread(target=server.serve_forever, daemon=True).start() - return server, port - - -def _measure_peak(scan_factory, run_scan): - """Build the scanner untraced, then measure only scan execution memory.""" - # Phase 1: build the scanner (untraced) - scan, cleanup = scan_factory() - - # Phase 2: trace only the scan execution + def log_message(self, *a): pass + +server = HTTPServer(("127.0.0.1", 0), H) +port = server.server_address[1] +threading.Thread(target=server.serve_forever, daemon=True).start() + +scan = Scanner( + f"http://127.0.0.1:{port}/", + modules=["httpx"], output_modules=["python"], + config={ + "dns": {"disable": True}, + "scope": {"search_distance": 0}, + "web": {"spider_distance": 10, "spider_depth": 10, "spider_links_per_page": NUM_PAGES}, + "speculate": True, "excavate": True, "aggregate": False, "cloudcheck": False, + "modules": {"httpx": {"batch_size": 25}}, + }, + force_start=True, +) + +async def run(): + await scan._prep() gc.collect() if tracemalloc.is_tracing(): tracemalloc.stop() tracemalloc.start() - try: - asyncio.run(run_scan(scan)) - finally: - _, peak = tracemalloc.get_traced_memory() - tracemalloc.stop() - if cleanup: - cleanup() - return round(peak / 1024 / 1024, 2) - - -# --------------------------------------------------------------------------- -# 1) Web crawl -- httpx visits many pages, excavate processes bodies -# --------------------------------------------------------------------------- - - -def _web_crawl_factory(): - server, port = _start_server() - scan = Scanner( - f"http://127.0.0.1:{port}/", - modules=["httpx"], - output_modules=["python"], - config={ - "dns": {"disable": True}, - "scope": {"search_distance": 0}, - "web": {"spider_distance": 10, "spider_depth": 10, "spider_links_per_page": NUM_PAGES}, - "speculate": True, - "excavate": True, - "aggregate": False, - "cloudcheck": False, - "modules": {"httpx": {"batch_size": 25}}, - }, - force_start=True, - ) - return scan, server.shutdown - - -async def _web_crawl_run(scan): events = [] async for event in scan.async_start(): events.append(event) +asyncio.run(run()) +_, peak = tracemalloc.get_traced_memory() +tracemalloc.stop() +server.shutdown() +print(f"PEAK_MB:{round(peak / 1024 / 1024, 2)}") +""".replace("NUM_PAGES_PLACEHOLDER", str(NUM_PAGES)).replace("BODY_SIZE_PLACEHOLDER", str(BODY_SIZE)) + class TestWebCrawlMemory: """Measures peak memory during a realistic web crawl with large pages.""" @pytest.mark.benchmark(group="memory_scan_patterns") - def test_web_crawl(self, benchmark): - peak_mb = _measure_peak(_web_crawl_factory, _web_crawl_run) + def test_memory_use_web_crawl(self, benchmark): + peak_mb = _run_scan_subprocess(_WEB_CRAWL_SCRIPT) benchmark.extra_info["total_memory_mb"] = peak_mb benchmark.extra_info["num_pages"] = NUM_PAGES benchmark.pedantic(lambda: None, iterations=1, rounds=1, warmup_rounds=0) @@ -126,32 +115,32 @@ def test_web_crawl(self, benchmark): # 2) Subdomain enum -- many DNS_NAME events, no heavy bodies # --------------------------------------------------------------------------- -SUBDOMAIN_ENUM_COUNT = 5000 - - -def _subdomain_enum_factory(): - scan = Scanner( - "blacklanternsecurity.com", - modules=[], - output_modules=["python"], - config={ - "dns": {"disable": True}, - "scope": {"search_distance": 0}, - "web": {"spider_distance": 0, "spider_depth": 0}, - "speculate": False, - "excavate": True, - "aggregate": False, - "cloudcheck": False, - }, - force_start=True, - ) - return scan, None - +_SUBDOMAIN_ENUM_SCRIPT = """ +import gc, asyncio, tracemalloc +from bbot.scanner import Scanner -async def _subdomain_enum_run(scan): +SUBDOMAIN_ENUM_COUNT = SUBDOMAIN_COUNT_PLACEHOLDER + +scan = Scanner( + "blacklanternsecurity.com", + modules=[], output_modules=["python"], + config={ + "dns": {"disable": True}, + "scope": {"search_distance": 0}, + "web": {"spider_distance": 0, "spider_depth": 0}, + "speculate": False, "excavate": True, "aggregate": False, "cloudcheck": False, + }, + force_start=True, +) + +async def run(): + await scan._prep() + gc.collect() + if tracemalloc.is_tracing(): + tracemalloc.stop() + tracemalloc.start() events = [] injected = False - async for event in scan.async_start(): events.append(event) if event.type == "SCAN" and not injected: @@ -166,15 +155,19 @@ async def _subdomain_enum_run(scan): ) await scan.ingress_module.queue_event(dns_event, {}) - return events +asyncio.run(run()) +_, peak = tracemalloc.get_traced_memory() +tracemalloc.stop() +print(f"PEAK_MB:{round(peak / 1024 / 1024, 2)}") +""".replace("SUBDOMAIN_COUNT_PLACEHOLDER", str(SUBDOMAIN_ENUM_COUNT)) class TestSubdomainEnumMemory: """Measures peak memory during a large subdomain enumeration.""" @pytest.mark.benchmark(group="memory_scan_patterns") - def test_subdomain_enum(self, benchmark): - peak_mb = _measure_peak(_subdomain_enum_factory, _subdomain_enum_run) + def test_memory_use_subdomain_enum(self, benchmark): + peak_mb = _run_scan_subprocess(_SUBDOMAIN_ENUM_SCRIPT) benchmark.extra_info["total_memory_mb"] = peak_mb benchmark.extra_info["num_subdomains"] = SUBDOMAIN_ENUM_COUNT benchmark.pedantic(lambda: None, iterations=1, rounds=1, warmup_rounds=0) From c6d6ac41f0f8c72e7112db78ae3f68626963e48e Mon Sep 17 00:00:00 2001 From: liquidsec Date: Tue, 31 Mar 2026 16:51:33 -0400 Subject: [PATCH 4/5] Intern repeated strings in resolved_hosts and dns_children IP addresses and DNS record type strings (A, AAAA, CNAME, etc.) repeat heavily across events. sys.intern() deduplicates them so all events sharing the same IPs/rdtypes reference the same string object, reducing memory ~10-30% on those fields. --- bbot/modules/gowitness.py | 3 ++- bbot/modules/httpx.py | 3 ++- bbot/modules/internal/dnsresolve.py | 6 +++++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/bbot/modules/gowitness.py b/bbot/modules/gowitness.py index 5d0538d834..b26018d58c 100644 --- a/bbot/modules/gowitness.py +++ b/bbot/modules/gowitness.py @@ -1,4 +1,5 @@ import os +import sys import asyncio import aiosqlite import multiprocessing @@ -243,7 +244,7 @@ async def handle_batch(self, *events): context=f"{{module}} visited {{event.type}}: {url}", ) if url_event and ip: - url_event._resolved_hosts.add(ip) + url_event._resolved_hosts.add(sys.intern(ip)) await self.emit_event(url_event) # emit technologies diff --git a/bbot/modules/httpx.py b/bbot/modules/httpx.py index 628ef4bb66..dd996a2b58 100644 --- a/bbot/modules/httpx.py +++ b/bbot/modules/httpx.py @@ -1,4 +1,5 @@ import re +import sys import orjson import tempfile import subprocess @@ -200,7 +201,7 @@ async def handle_batch(self, *events): if url_event: httpx_ip = j.get("host", "") if httpx_ip: - url_event._resolved_hosts.add(httpx_ip) + url_event._resolved_hosts.add(sys.intern(httpx_ip)) url_event.data["status_code"] = status_code title = j.get("title", "") if title: diff --git a/bbot/modules/internal/dnsresolve.py b/bbot/modules/internal/dnsresolve.py index 680cedc605..f23a3bea76 100644 --- a/bbot/modules/internal/dnsresolve.py +++ b/bbot/modules/internal/dnsresolve.py @@ -1,3 +1,4 @@ +import sys import ipaddress from contextlib import suppress @@ -231,7 +232,7 @@ def check_scope(self, event): for rdtype in ("A", "AAAA", "CNAME"): hosts = dns_children.get(rdtype, []) # update resolved hosts - event.resolved_hosts.update(hosts) + event.resolved_hosts.update(sys.intern(h) for h in hosts) for host in hosts: # having a CNAME to an in-scope host doesn't make you in-scope if rdtype != "CNAME": @@ -258,6 +259,7 @@ async def resolve_event(self, event, types): queries = [(event_host, rdtype) for rdtype in types] dns_errors = {} async for (query, rdtype), (answers, errors) in self.helpers.dns.resolve_raw_batch(queries): + rdtype = sys.intern(rdtype) # errors try: dns_errors[rdtype].update(errors) @@ -272,6 +274,8 @@ async def resolve_event(self, event, types): event.raw_dns_records[rdtype] = {answer} # hosts for _rdtype, host in extract_targets(answer): + _rdtype = sys.intern(_rdtype) + host = sys.intern(host) try: event.dns_children[_rdtype].add(host) except KeyError: From 32aa5a1784d7197064b165c0dcb5fbf1cbc087ac Mon Sep 17 00:00:00 2001 From: liquidsec Date: Wed, 1 Apr 2026 11:58:33 -0400 Subject: [PATCH 5/5] Extract embedded benchmark scripts into standalone files --- .../benchmarks/_scan_memory_subdomain_enum.py | 62 ++++++++ .../test/benchmarks/_scan_memory_web_crawl.py | 86 +++++++++++ bbot/test/benchmarks/test_scan_memory.py | 135 ++---------------- 3 files changed, 162 insertions(+), 121 deletions(-) create mode 100644 bbot/test/benchmarks/_scan_memory_subdomain_enum.py create mode 100644 bbot/test/benchmarks/_scan_memory_web_crawl.py diff --git a/bbot/test/benchmarks/_scan_memory_subdomain_enum.py b/bbot/test/benchmarks/_scan_memory_subdomain_enum.py new file mode 100644 index 0000000000..33b4b997f6 --- /dev/null +++ b/bbot/test/benchmarks/_scan_memory_subdomain_enum.py @@ -0,0 +1,62 @@ +""" +Subprocess script for subdomain enumeration memory benchmark. + +Injects SUBDOMAIN_ENUM_COUNT synthetic DNS_NAME events into a scan +and prints peak tracemalloc memory to stdout. + +Invoked by test_scan_memory.py — not meant to be run directly. +""" + +import gc +import sys +import asyncio +import tracemalloc + +from bbot.scanner import Scanner + +SUBDOMAIN_ENUM_COUNT = int(sys.argv[1]) + +scan = Scanner( + "blacklanternsecurity.com", + modules=[], + output_modules=["python"], + config={ + "dns": {"disable": True}, + "scope": {"search_distance": 0}, + "web": {"spider_distance": 0, "spider_depth": 0}, + "speculate": False, + "excavate": True, + "aggregate": False, + "cloudcheck": False, + }, + force_start=True, +) + + +async def run(): + await scan._prep() + gc.collect() + if tracemalloc.is_tracing(): + tracemalloc.stop() + tracemalloc.start() + events = [] + injected = False + async for event in scan.async_start(): + events.append(event) + if event.type == "SCAN" and not injected: + injected = True + root_event = scan.root_event + for i in range(SUBDOMAIN_ENUM_COUNT): + dns_event = scan.make_event( + f"sub{i}.blacklanternsecurity.com", + "DNS_NAME", + parent=root_event, + context=f"benchmark DNS_NAME {i}", + ) + await scan.ingress_module.queue_event(dns_event, {}) + + +asyncio.run(run()) +_, peak = tracemalloc.get_traced_memory() +tracemalloc.stop() +print(f"PEAK_MB:{round(peak / 1024 / 1024, 2)}") diff --git a/bbot/test/benchmarks/_scan_memory_web_crawl.py b/bbot/test/benchmarks/_scan_memory_web_crawl.py new file mode 100644 index 0000000000..e609220c74 --- /dev/null +++ b/bbot/test/benchmarks/_scan_memory_web_crawl.py @@ -0,0 +1,86 @@ +""" +Subprocess script for web crawl memory benchmark. + +Launches a local HTTP server with NUM_PAGES pages (each BODY_SIZE bytes), +runs a BBOT scan against it, and prints peak tracemalloc memory to stdout. + +Invoked by test_scan_memory.py — not meant to be run directly. +""" + +import gc +import sys +import asyncio +import threading +import tracemalloc +import importlib.util +from http.server import HTTPServer, BaseHTTPRequestHandler + +from bbot.scanner import Scanner + +NUM_PAGES = int(sys.argv[1]) +BODY_SIZE = int(sys.argv[2]) + +HTTP_MODULE = "httpx" if importlib.util.find_spec("bbot.modules.httpx") else "http" + + +class H(BaseHTTPRequestHandler): + def do_GET(self): + if self.path == "/": + links = "".join(f'page{i}' for i in range(NUM_PAGES)) + body = "" + links + "" + elif self.path.startswith("/page"): + i = self.path.replace("/page", "") + links = f'infodetails' + body = "

Page " + i + "

" + links + "A" * BODY_SIZE + "" + elif self.path.startswith("/data"): + body = "data endpoint" + else: + self.send_response(404) + self.end_headers() + return + self.send_response(200) + self.send_header("Content-Type", "text/html") + self.end_headers() + self.wfile.write(body.encode()) + + def log_message(self, *a): + pass + + +server = HTTPServer(("127.0.0.1", 0), H) +port = server.server_address[1] +threading.Thread(target=server.serve_forever, daemon=True).start() + +scan = Scanner( + f"http://127.0.0.1:{port}/", + modules=[HTTP_MODULE], + output_modules=["python"], + config={ + "dns": {"disable": True}, + "scope": {"search_distance": 0}, + "web": {"spider_distance": 10, "spider_depth": 10, "spider_links_per_page": NUM_PAGES}, + "speculate": True, + "excavate": True, + "aggregate": False, + "cloudcheck": False, + }, + force_start=True, +) + + +async def run(): + await scan._prep() + gc.collect() + if tracemalloc.is_tracing(): + tracemalloc.stop() + tracemalloc.start() + events = [] + async for event in scan.async_start(): + events.append(event) + + +asyncio.run(run()) +_, peak = tracemalloc.get_traced_memory() +tracemalloc.stop() +server.shutdown() +print(f"PEAK_MB:{round(peak / 1024 / 1024, 2)}") diff --git a/bbot/test/benchmarks/test_scan_memory.py b/bbot/test/benchmarks/test_scan_memory.py index 82c236edff..af4f76dfa6 100644 --- a/bbot/test/benchmarks/test_scan_memory.py +++ b/bbot/test/benchmarks/test_scan_memory.py @@ -9,6 +9,7 @@ import subprocess import sys +from pathlib import Path import pytest @@ -17,11 +18,13 @@ BODY_SIZE = 500_000 # 500 KB per page SUBDOMAIN_ENUM_COUNT = 5000 +_BENCHMARKS_DIR = Path(__file__).parent -def _run_scan_subprocess(script: str) -> float: + +def _run_scan_subprocess(script_path: Path, *args: str) -> float: """Run a scan script in a clean subprocess, return peak memory in MB.""" result = subprocess.run( - [sys.executable, "-c", script], + [sys.executable, str(script_path), *args], capture_output=True, text=True, timeout=600, @@ -34,140 +37,30 @@ def _run_scan_subprocess(script: str) -> float: raise RuntimeError(f"No PEAK_MB in subprocess output:\n{result.stdout[-2000:]}") -# --------------------------------------------------------------------------- -# 1) Web crawl -- httpx visits many pages, excavate processes bodies -# --------------------------------------------------------------------------- - -_WEB_CRAWL_SCRIPT = """ -import gc, asyncio, threading, tracemalloc -from http.server import HTTPServer, BaseHTTPRequestHandler -from bbot.scanner import Scanner - -NUM_PAGES = NUM_PAGES_PLACEHOLDER -BODY_SIZE = BODY_SIZE_PLACEHOLDER - -class H(BaseHTTPRequestHandler): - def do_GET(self): - if self.path == "/": - links = "".join(f'page{i}' for i in range(NUM_PAGES)) - body = "" + links + "" - elif self.path.startswith("/page"): - i = self.path.replace("/page", "") - links = f'infodetails' - body = "

Page " + i + "

" + links + "A" * BODY_SIZE + "" - elif self.path.startswith("/data"): - body = "data endpoint" - else: - self.send_response(404); self.end_headers(); return - self.send_response(200) - self.send_header("Content-Type", "text/html") - self.end_headers() - self.wfile.write(body.encode()) - def log_message(self, *a): pass - -server = HTTPServer(("127.0.0.1", 0), H) -port = server.server_address[1] -threading.Thread(target=server.serve_forever, daemon=True).start() - -scan = Scanner( - f"http://127.0.0.1:{port}/", - modules=["httpx"], output_modules=["python"], - config={ - "dns": {"disable": True}, - "scope": {"search_distance": 0}, - "web": {"spider_distance": 10, "spider_depth": 10, "spider_links_per_page": NUM_PAGES}, - "speculate": True, "excavate": True, "aggregate": False, "cloudcheck": False, - "modules": {"httpx": {"batch_size": 25}}, - }, - force_start=True, -) - -async def run(): - await scan._prep() - gc.collect() - if tracemalloc.is_tracing(): - tracemalloc.stop() - tracemalloc.start() - events = [] - async for event in scan.async_start(): - events.append(event) - -asyncio.run(run()) -_, peak = tracemalloc.get_traced_memory() -tracemalloc.stop() -server.shutdown() -print(f"PEAK_MB:{round(peak / 1024 / 1024, 2)}") -""".replace("NUM_PAGES_PLACEHOLDER", str(NUM_PAGES)).replace("BODY_SIZE_PLACEHOLDER", str(BODY_SIZE)) - - class TestWebCrawlMemory: """Measures peak memory during a realistic web crawl with large pages.""" @pytest.mark.benchmark(group="memory_scan_patterns") def test_memory_use_web_crawl(self, benchmark): - peak_mb = _run_scan_subprocess(_WEB_CRAWL_SCRIPT) + peak_mb = _run_scan_subprocess( + _BENCHMARKS_DIR / "_scan_memory_web_crawl.py", + str(NUM_PAGES), + str(BODY_SIZE), + ) benchmark.extra_info["total_memory_mb"] = peak_mb benchmark.extra_info["num_pages"] = NUM_PAGES benchmark.pedantic(lambda: None, iterations=1, rounds=1, warmup_rounds=0) -# --------------------------------------------------------------------------- -# 2) Subdomain enum -- many DNS_NAME events, no heavy bodies -# --------------------------------------------------------------------------- - -_SUBDOMAIN_ENUM_SCRIPT = """ -import gc, asyncio, tracemalloc -from bbot.scanner import Scanner - -SUBDOMAIN_ENUM_COUNT = SUBDOMAIN_COUNT_PLACEHOLDER - -scan = Scanner( - "blacklanternsecurity.com", - modules=[], output_modules=["python"], - config={ - "dns": {"disable": True}, - "scope": {"search_distance": 0}, - "web": {"spider_distance": 0, "spider_depth": 0}, - "speculate": False, "excavate": True, "aggregate": False, "cloudcheck": False, - }, - force_start=True, -) - -async def run(): - await scan._prep() - gc.collect() - if tracemalloc.is_tracing(): - tracemalloc.stop() - tracemalloc.start() - events = [] - injected = False - async for event in scan.async_start(): - events.append(event) - if event.type == "SCAN" and not injected: - injected = True - root_event = scan.root_event - for i in range(SUBDOMAIN_ENUM_COUNT): - dns_event = scan.make_event( - f"sub{i}.blacklanternsecurity.com", - "DNS_NAME", - parent=root_event, - context=f"benchmark DNS_NAME {i}", - ) - await scan.ingress_module.queue_event(dns_event, {}) - -asyncio.run(run()) -_, peak = tracemalloc.get_traced_memory() -tracemalloc.stop() -print(f"PEAK_MB:{round(peak / 1024 / 1024, 2)}") -""".replace("SUBDOMAIN_COUNT_PLACEHOLDER", str(SUBDOMAIN_ENUM_COUNT)) - - class TestSubdomainEnumMemory: """Measures peak memory during a large subdomain enumeration.""" @pytest.mark.benchmark(group="memory_scan_patterns") def test_memory_use_subdomain_enum(self, benchmark): - peak_mb = _run_scan_subprocess(_SUBDOMAIN_ENUM_SCRIPT) + peak_mb = _run_scan_subprocess( + _BENCHMARKS_DIR / "_scan_memory_subdomain_enum.py", + str(SUBDOMAIN_ENUM_COUNT), + ) benchmark.extra_info["total_memory_mb"] = peak_mb benchmark.extra_info["num_subdomains"] = SUBDOMAIN_ENUM_COUNT benchmark.pedantic(lambda: None, iterations=1, rounds=1, warmup_rounds=0)