Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions bbot/core/helpers/diff.py
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,7 @@ async def _baseline(self):
"date",
"last-modified",
"content-length",
"connection",
"ETag",
"X-Pad",
"X-Backside-Transport",
Expand Down
15 changes: 15 additions & 0 deletions bbot/modules/paramminer_headers.py
Original file line number Diff line number Diff line change
Expand Up @@ -328,6 +328,21 @@ async def finish(self):
continue
await self.process_results(event, results)

def _incoming_dedup_hash(self, event):
# dedup by endpoint structure, not full URL string -- value mutations
# of the same parameter set (e.g. from lightfuzz probes) are one test surface
p = getattr(event, "parsed_url", None)
if p is None:
return hash(event), ""
if event.type == "WEB_PARAMETER":
name = event.data.get("name", "")
additional_params = event.data.get("additional_params") or {}
param_keys = tuple(sorted(additional_params.keys()))
else:
name = ""
param_keys = ()
return hash((event.type, p.scheme, p.netloc, p.path, name, param_keys)), "per_endpoint+keys"

async def filter_event(self, event):
if await self._is_http_wildcard_host(event) is True:
return False, "host is an HTTP wildcard responder"
Expand Down
270 changes: 106 additions & 164 deletions bbot/modules/webbrute.py

Large diffs are not rendered by default.

24 changes: 16 additions & 8 deletions bbot/modules/webbrute_shortnames.py
Original file line number Diff line number Diff line change
Expand Up @@ -161,6 +161,8 @@ def find_class(self, module, name):

self.per_host_collection = {}
self.shortname_to_event = {}
self._host_timeouts = {}
self._blocked_hosts = set()

return True

Expand Down Expand Up @@ -221,6 +223,7 @@ async def handle_event(self, event):

root_stub = "/".join(event.parsed_url.path.split("/")[:-1])
root_url = f"{event.parsed_url.scheme}://{event.parsed_url.netloc}{root_stub}/"
netloc = event.parsed_url.netloc

if shortname_type == "endpoint":
used_extensions = self.build_extension_list(event)
Expand All @@ -235,7 +238,7 @@ async def handle_event(self, event):
if words_len > 0:
if shortname_type == "endpoint":
for ext in used_extensions:
async for r in self.execute_fuzz(words, root_url, suffix=f".{ext}"):
async for r in self.execute_fuzz(words, root_url, netloc, suffix=f".{ext}"):
await self.emit_event(
r["url"],
"URL_UNVERIFIED",
Expand All @@ -245,7 +248,7 @@ async def handle_event(self, event):
)

elif shortname_type == "directory":
async for r in self.execute_fuzz(words, root_url, exts=["/"]):
async for r in self.execute_fuzz(words, root_url, netloc, exts=["/"]):
r_url = f"{r['url'].rstrip('/')}/"
await self.emit_event(
r_url,
Expand All @@ -263,7 +266,7 @@ async def handle_event(self, event):
self.verbose(f"Detected delimiter [{delimiter}] in hint [{filename_hint}]")
words, words_len = await self.generate_templist(partial_hint, "directory")
fuzz_prefix = f"{prefix}{delimiter}"
async for r in self.execute_fuzz(words, root_url, prefix=fuzz_prefix, exts=["/"]):
async for r in self.execute_fuzz(words, root_url, netloc, prefix=fuzz_prefix, exts=["/"]):
await self.emit_event(
r["url"],
"URL_UNVERIFIED",
Expand All @@ -280,7 +283,9 @@ async def handle_event(self, event):
self.verbose(f"Detected delimiter [{delimiter}] in hint [{filename_hint}]")
words, words_len = await self.generate_templist(partial_hint, "endpoint")
fuzz_prefix = f"{prefix}{delimiter}"
async for r in self.execute_fuzz(words, root_url, prefix=fuzz_prefix, suffix=f".{ext}"):
async for r in self.execute_fuzz(
words, root_url, netloc, prefix=fuzz_prefix, suffix=f".{ext}"
):
await self.emit_event(
r["url"],
"URL_UNVERIFIED",
Expand All @@ -294,7 +299,7 @@ async def handle_event(self, event):
if subword:
if "shortname-directory" in event.tags:
words, words_len = await self.generate_templist(suffix, "directory")
async for r in self.execute_fuzz(words, root_url, prefix=subword, exts=["/"]):
async for r in self.execute_fuzz(words, root_url, netloc, prefix=subword, exts=["/"]):
await self.emit_event(
r["url"],
"URL_UNVERIFIED",
Expand All @@ -305,7 +310,7 @@ async def handle_event(self, event):
elif "shortname-endpoint" in event.tags:
for ext in used_extensions:
words, words_len = await self.generate_templist(suffix, "endpoint")
async for r in self.execute_fuzz(words, root_url, prefix=subword, suffix=f".{ext}"):
async for r in self.execute_fuzz(words, root_url, netloc, prefix=subword, suffix=f".{ext}"):
await self.emit_event(
r["url"],
"URL_UNVERIFIED",
Expand All @@ -327,6 +332,7 @@ async def finish(self):
self.verbose(f"Found common prefix: [{prefix}] for host [{host}]")
for hint_tuple in hint_tuple_list:
hint, url = hint_tuple
netloc = self.shortname_to_event[hint].parsed_url.netloc
if hint.startswith(prefix):
if "shortname-endpoint" in self.shortname_to_event[hint].tags:
shortname_type = "endpoint"
Expand All @@ -347,7 +353,7 @@ async def finish(self):
f"Running common prefix check for URL_HINT: {hint} with prefix: {prefix} and partial_hint: {partial_hint}"
)

async for r in self.execute_fuzz(words, url, prefix=prefix, exts=["/"]):
async for r in self.execute_fuzz(words, url, netloc, prefix=prefix, exts=["/"]):
await self.emit_event(
r["url"],
"URL_UNVERIFIED",
Expand All @@ -362,7 +368,9 @@ async def finish(self):
self.verbose(
f"Running common prefix check for URL_HINT: {hint} with prefix: {prefix}, extension: .{ext}, and partial_hint: {partial_hint}"
)
async for r in self.execute_fuzz(words, url, prefix=prefix, suffix=f".{ext}"):
async for r in self.execute_fuzz(
words, url, netloc, prefix=prefix, suffix=f".{ext}"
):
await self.emit_event(
r["url"],
"URL_UNVERIFIED",
Expand Down
2 changes: 2 additions & 0 deletions bbot/presets/kitchen-sink.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@ config:
recursive_mutations: true
dnscommonsrv:
recursive_mutations: true
webbrute:
avoid_wafs: False
wayback:
urls: True
parameters: True
Expand Down
1 change: 1 addition & 0 deletions bbot/presets/web/dirbust-heavy.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ config:
# we exploit the shortnames vulnerability to produce URL_HINTs which are consumed by webbrute_shortnames
detect_only: False
webbrute:
avoid_wafs: False
max_depth: 3
lines: 5000
extensions:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -221,3 +221,88 @@ async def mock_wildcard(scheme, host, port):
def check(self, module_test, events):
web_params = [e for e in events if e.type == "WEB_PARAMETER" and str(e.module) == "paramminer_headers"]
assert len(web_params) == 0, f"paramminer_headers should not fuzz wildcard hosts, but emitted: {web_params}"


class TestParamminerHeadersDedupValueMutations(Paramminer_Headers):
"""Value mutations of the same parameter set on the same endpoint should collapse to one test surface."""

async def setup_after_prep(self, module_test):
await super().setup_after_prep(module_test)
pm = module_test.scan.modules["paramminer_headers"]

def make_wp(**kwargs):
defaults = {"host": "127.0.0.1:8888", "type": "GETPARAM", "original_value": "x"}
defaults.update(kwargs)
return module_test.scan.make_event(defaults, "WEB_PARAMETER", dummy=True)

def make_hr(url):
return module_test.scan.make_event(
{
"url": url,
"status_code": 200,
"header-dict": {},
"body": "",
"raw_header": "HTTP/1.1 200 OK\r\n\r\n",
"method": "GET",
},
"HTTP_RESPONSE",
dummy=True,
)

# same endpoint, same param keys, different values (simulates lightfuzz probe cycling)
hash_a, _ = pm._incoming_dedup_hash(
make_wp(
url="http://127.0.0.1:8888/page?culture=probe_a&page=1",
name="culture",
additional_params={"page": "1"},
)
)
hash_b, _ = pm._incoming_dedup_hash(
make_wp(
url="http://127.0.0.1:8888/page?culture=probe_b&page=2",
name="culture",
additional_params={"page": "2"},
)
)
assert hash_a == hash_b, "Value mutations of same param keys should produce same dedup hash"

# different param keys on same path -> distinct
hash_c, _ = pm._incoming_dedup_hash(
make_wp(
url="http://127.0.0.1:8888/page?culture=x&page=1&csrf=tok",
name="culture",
additional_params={"page": "1", "csrf": "tok"},
)
)
assert hash_a != hash_c, "Different param key sets should produce different dedup hashes"

# different path, same keys -> distinct
hash_d, _ = pm._incoming_dedup_hash(
make_wp(
url="http://127.0.0.1:8888/other?culture=x&page=1",
name="culture",
additional_params={"page": "1"},
)
)
assert hash_a != hash_d, "Same param keys on different paths should produce different dedup hashes"

# different focus name, same sibling keys (excavate emits one WP per param) -> distinct
hash_e, _ = pm._incoming_dedup_hash(
make_wp(
url="http://127.0.0.1:8888/page?culture=x&page=1",
name="page",
additional_params={"culture": "x"},
)
)
assert hash_a != hash_e, "Different focus name with same siblings should produce different dedup hashes"

# HTTP_RESPONSE events: same endpoint collapses regardless
hash_hr_a, _ = pm._incoming_dedup_hash(make_hr("http://127.0.0.1:8888/page"))
hash_hr_b, _ = pm._incoming_dedup_hash(make_hr("http://127.0.0.1:8888/page"))
assert hash_hr_a == hash_hr_b, "HTTP_RESPONSE events for same endpoint should produce same dedup hash"

# WEB_PARAMETER and HTTP_RESPONSE for same URL should NOT collide
assert hash_a != hash_hr_a, "WEB_PARAMETER and HTTP_RESPONSE should not share dedup hashes"

def check(self, module_test, events):
super().check(module_test, events)
151 changes: 151 additions & 0 deletions bbot/test/test_step_2/module_tests/test_module_webbrute.py
Original file line number Diff line number Diff line change
Expand Up @@ -168,6 +168,157 @@ def check(self, module_test, events):
assert len(waf_hits) == 0, f"webbrute should filter WAF block pages, but got: {[e.url for e in waf_hits]}"


class TestWebBruteDynamicContentFilter(ModuleTestBase):
"""Server returns 200 with a per-request CSRF token for unknown paths.
Without HttpCompare, the baseline would fall through to status-only filtering
and report every 200 as a hit. HttpCompare should detect the token as a dynamic
position and filter it out, only reporting /admin (genuinely different content)."""

targets = ["http://127.0.0.1:8888"]
module_name = "webbrute"
test_wordlist = ["admin", "junkword1", "zzzjunkword2"]
config_overrides = {"modules": {"webbrute": {"wordlist": tempwordlist(test_wordlist)}}}
modules_overrides = ["webbrute", "http"]

_counter = 0

def request_handler(self, request):
uri = request.path
if uri == "/":
return Response("<html>Home</html>", status=200)
if uri.lstrip("/").startswith("admin"):
return Response("<html><body>Admin Panel - Secret Content</body></html>", status=200)
TestWebBruteDynamicContentFilter._counter += 1
body = (
"<html>\n"
"<head><title>Page Not Found</title></head>\n"
"<body>\n"
"<p>The page you requested could not be found.</p>\n"
f'<input type="hidden" name="csrf" value="tok{TestWebBruteDynamicContentFilter._counter:06d}"/>\n'
"</body>\n"
"</html>"
)
return Response(body, status=200)

async def setup_before_prep(self, module_test):
module_test.set_expect_requests_handler(expect_args=re.compile("/.*"), request_handler=self.request_handler)

def check(self, module_test, events):
webbrute_urls = [e for e in events if e.type == "URL_UNVERIFIED" and str(e.module) == "webbrute"]
assert any("admin" in e.url for e in webbrute_urls), (
f"webbrute should find /admin (genuinely different), but got: {[e.url for e in webbrute_urls]}"
)
junk_hits = [e for e in webbrute_urls if "junkword" in e.url]
assert len(junk_hits) == 0, (
f"webbrute should filter dynamic-content false positives, but got: {[e.url for e in junk_hits]}"
)


class TestWebBruteHitCap(ModuleTestBase):
"""Server returns unique content for every path, causing all words to pass
HttpCompare. The sqrt-scaled hit cap should detect this as a filtering
failure and discard everything before emission."""

targets = ["http://127.0.0.1:8888"]
module_name = "webbrute"
# 50 words → cap of int(4 * sqrt(50)) = 28. All 50 will "hit", exceeding the cap.
test_wordlist = [f"word{i:03d}" for i in range(50)]
config_overrides = {"modules": {"webbrute": {"wordlist": tempwordlist(test_wordlist)}}}
modules_overrides = ["webbrute", "http"]

_counter = 0

def request_handler(self, request):
uri = request.path
if uri == "/":
return Response("<html>Home</html>", status=200)
# Return unique content only for wordlist words (word000-word049).
# Random strings (like the canary) get a normal 404 so the canary
# doesn't mask the hit cap.
segment = uri.strip("/")
if segment.startswith("word"):
TestWebBruteHitCap._counter += 1
body = f"<html><body>Unique page {TestWebBruteHitCap._counter} for {uri}</body></html>"
return Response(body, status=200)
return Response("Not Found", status=404)

async def setup_before_prep(self, module_test):
module_test.set_expect_requests_handler(expect_args=re.compile("/.*"), request_handler=self.request_handler)

def check(self, module_test, events):
webbrute_urls = [e for e in events if e.type == "URL_UNVERIFIED" and str(e.module) == "webbrute"]
assert len(webbrute_urls) == 0, (
f"Hit cap should have discarded all hits, but got: {[e.url for e in webbrute_urls]}"
)


class TestWebBruteCanaryDefense(ModuleTestBase):
"""Server returns unique content for EVERY path including random strings.
The canary fires and aborts, discarding all hits."""

targets = ["http://127.0.0.1:8888"]
module_name = "webbrute"
test_wordlist = ["admin", "secret", "login"]
config_overrides = {"modules": {"webbrute": {"wordlist": tempwordlist(test_wordlist)}}}
modules_overrides = ["webbrute", "http"]

_counter = 0

def request_handler(self, request):
uri = request.path
if uri == "/":
return Response("<html>Home</html>", status=200)
# Every path (including random canary strings) gets unique content
# that differs structurally from the baseline, not just in a filterable position.
TestWebBruteCanaryDefense._counter += 1
n = TestWebBruteCanaryDefense._counter
extra = "".join(f"<p>Section {i}</p>\n" for i in range(n % 5 + 1))
body = f"<html><body><h1>Custom page #{n} for {uri}</h1>\n{extra}</body></html>"
return Response(body, status=200)

async def setup_before_prep(self, module_test):
module_test.set_expect_requests_handler(expect_args=re.compile("/.*"), request_handler=self.request_handler)

def check(self, module_test, events):
webbrute_urls = [e for e in events if e.type == "URL_UNVERIFIED" and str(e.module) == "webbrute"]
assert len(webbrute_urls) == 0, (
f"Canary defense should have aborted, but webbrute emitted: {[e.url for e in webbrute_urls]}"
)


class TestWebBruteMidScanBaselineDrift(ModuleTestBase):
"""Server matches baseline initially, then drifts (WAF kicks in after N requests).
Mid-scan baseline check should detect the drift and abort."""

targets = ["http://127.0.0.1:8888"]
module_name = "webbrute"
test_wordlist = ["admin", "secret", "login"]
config_overrides = {"modules": {"webbrute": {"wordlist": tempwordlist(test_wordlist)}}}
modules_overrides = ["webbrute", "http"]

_request_count = 0

def request_handler(self, request):
uri = request.path
if uri == "/":
return Response("<html>Home</html>", status=200)
TestWebBruteMidScanBaselineDrift._request_count += 1
# First 4 requests: normal 404 (baseline probes + early fuzz).
# After that: WAF kicks in, returns block page for everything.
if TestWebBruteMidScanBaselineDrift._request_count <= 4:
return Response("Not Found", status=404)
return Response("<html>Access Denied by WAF</html>", status=200)

async def setup_before_prep(self, module_test):
module_test.set_expect_requests_handler(expect_args=re.compile("/.*"), request_handler=self.request_handler)

def check(self, module_test, events):
webbrute_urls = [e for e in events if e.type == "URL_UNVERIFIED" and str(e.module) == "webbrute"]
assert len(webbrute_urls) == 0, (
f"Mid-scan baseline drift should have aborted, but webbrute emitted: {[e.url for e in webbrute_urls]}"
)


class TestWebBruteWildcardSkip(ModuleTestBase):
"""When the host is an HTTP wildcard, webbrute should skip fuzzing entirely."""

Expand Down
Loading
Loading