diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py index 215c9f61f9..2602f4debe 100644 --- a/bbot/core/event/base.py +++ b/bbot/core/event/base.py @@ -790,7 +790,6 @@ def _minimize(self): # release container slots; lazy-init pattern means None == empty self._dns_children = None self._raw_dns_records = None - self._resolved_hosts = None def clone(self): # Create a shallow copy of the event first diff --git a/bbot/modules/virtualhost.py b/bbot/modules/virtualhost.py index bc8262beb2..5b5a33fbb8 100644 --- a/bbot/modules/virtualhost.py +++ b/bbot/modules/virtualhost.py @@ -1019,10 +1019,10 @@ async def finish(self): async def filter_event(self, event): if ( - "cdn-cloudflare" in event.tags - or "cdn-imperva" in event.tags - or "cdn-akamai" in event.tags - or "cdn-cloudfront" in event.tags + "cloudflare" in event.tags + or "imperva" in event.tags + or "akamai" in event.tags + or "cloudfront" in event.tags ): self.debug(f"Not processing URL {event.url} because it's behind a WAF or CDN, and that's pointless") return False diff --git a/bbot/modules/waf_bypass.py b/bbot/modules/waf_bypass.py index 1ddae0aad4..fc2a6dc9b9 100644 --- a/bbot/modules/waf_bypass.py +++ b/bbot/modules/waf_bypass.py @@ -26,11 +26,16 @@ class Config(BaseModuleConfig): similarity_threshold: float = Field(0.90, description="Similarity threshold for content matching") search_ip_neighbors: bool = Field(True, description="Also check IP neighbors of the target domain") neighbor_cidr: int = Field( - 24, + 28, ge=24, le=31, description="CIDR mask (24-31) used for neighbor enumeration when search_ip_neighbors is true", ) + max_concurrent_checks: int = Field( + 100, + ge=1, + description="Maximum number of concurrent bypass-attempt HTTP checks in finish()", + ) meta = { "description": "Detects potential WAF bypasses", @@ -77,15 +82,15 @@ async def handle_event(self, event): self.cloud_ips.add(ip_str) self.debug(f"Added cloud-ip {ip_str} to cloud_ips") else: - self.warning(f"DNS resolution for {domain} returned non-IP result: {ip_str}") + self.verbose(f"DNS resolution for {domain} returned non-IP result: {ip_str}") else: self.warning(f"DNS resolution failed for {domain}") # Detect WAF/CDN protection based on tags provider_name = None - if "cdn-cloudflare" in event.tags or "waf-cloudflare" in event.tags: + if "cloudflare" in event.tags: provider_name = "CloudFlare" - elif "cdn-imperva" in event.tags: + elif "imperva" in event.tags: provider_name = "Imperva" is_protected = provider_name is not None @@ -189,8 +194,10 @@ async def finish(self): continue if ip not in waf_ips: # And IP isn't a known WAF IP - ip_bypass_candidates[ip] = domain - self.debug(f"Added potential bypass IP {ip} from domain {domain}") + # first attribution wins so logs are stable across scans + if ip not in ip_bypass_candidates: + ip_bypass_candidates[ip] = f"{domain} (direct)" + self.debug(f"Added potential bypass IP {ip} from domain {domain}") # if we have IP neighbors searching enabled, and the IP isn't a cloud IP, we can add the IP neighbors to our list of potential bypasses if self.search_ip_neighbors and ip not in self.cloud_ips: @@ -221,7 +228,9 @@ async def finish(self): self.debug( f"Added Neighbor IP ({ip} -> {neighbor_ip_str}) as potential bypass IP derived from {domain}" ) - ip_bypass_candidates[neighbor_ip_str] = domain + ip_bypass_candidates[neighbor_ip_str] = ( + f"{domain} (neighbor of {ip}/{self.neighbor_cidr})" + ) else: self.debug(f"IP {ip} is in WAF IPS so we don't check as potential bypass") @@ -245,7 +254,9 @@ async def finish(self): ) self.debug(f"about to start {len(coros)} coroutines") - async for completed in self.helpers.as_completed(coros): + async for completed in self.helpers.as_completed( + coros, max_concurrent=int(self.config.get("max_concurrent_checks") or 100) + ): result = await completed if result: confirmed_bypasses.append(result) diff --git a/bbot/presets/waf-bypass.yml b/bbot/presets/waf-bypass.yml index 4a4b7e06e5..fe2caf89c0 100644 --- a/bbot/presets/waf-bypass.yml +++ b/bbot/presets/waf-bypass.yml @@ -16,4 +16,4 @@ config: waf_bypass: similarity_threshold: 0.90 search_ip_neighbors: true - neighbor_cidr: 24 \ No newline at end of file + neighbor_cidr: 28 diff --git a/bbot/test/test_step_2/module_tests/test_module_cloudcheck.py b/bbot/test/test_step_2/module_tests/test_module_cloudcheck.py index 024d0dc75a..7407bc9061 100644 --- a/bbot/test/test_step_2/module_tests/test_module_cloudcheck.py +++ b/bbot/test/test_step_2/module_tests/test_module_cloudcheck.py @@ -123,6 +123,59 @@ async def setup_after_prep(self, module_test): assert "google" in child_disjoint.tags assert "cloud" in child_disjoint.tags + # ── _minimize() must not wipe resolved_hosts ─────────────────── + # Real scan flow: a DNS_NAME gets processed by dnsresolve + cloudcheck, + # then Event._minimize() decrements its consumer count. Later, child + # events (URL, OPEN_TCP_PORT) that share the DNS_NAME's host reach + # dnsresolve and copy the parent's resolved_hosts (line 156). If + # _minimize() wiped resolved_hosts, that copy carries nothing and + # cloudcheck's per-host filter has no IPs to match against, so the + # cloud tag never inherits. + minimized = scan.make_event("minimize.evilcorp.com", parent=scan.root_event) + minimized.resolved_hosts = ("asdf.amazonaws.com", "10.0.0.1") + await module.handle_event(minimized) + # drive _module_consumers below 0 so the wipe block runs + minimized._minimize() + minimized._minimize() + assert minimized.resolved_hosts, ( + "_minimize() must preserve resolved_hosts — dnsresolve line 156 copies it to child " + "URL/OPEN_TCP_PORT events, and cloudcheck's per-host inheritance needs those IPs" + ) + # cloud host_metadata already survives minimization; confirm + assert "asdf.amazonaws.com" in minimized.host_metadata + + # ── URL/OPEN_TCP_PORT propagation end-to-end ────────────────── + # Together with the _minimize() preservation above, this exercises + # the real flow: parent DNS_NAME cloud-tagged, then minimized, then a + # URL child gets its resolved_hosts populated (mirroring dnsresolve), + # and cloudcheck's existing subset-gate + per-host filter should copy + # the cloud tag onto the child. + url_child = scan.make_event( + "http://minimize.evilcorp.com/", + "URL", + parent=minimized, + tags=["status-200", "in-scope"], + ) + # simulate dnsresolve intercept: child inherits parent's resolved_hosts + url_child._resolved_hosts = minimized.resolved_hosts + await module.handle_event(url_child) + assert "amazon" in url_child.tags, ( + "URL child of amazon-tagged DNS_NAME must inherit the cloud tag " + "once its resolved_hosts carries the parent's IPs" + ) + assert "cloud" in url_child.tags + assert "asdf.amazonaws.com" in url_child.host_metadata + + port_child = scan.make_event( + "minimize.evilcorp.com:443", + "OPEN_TCP_PORT", + parent=minimized, + ) + port_child._resolved_hosts = minimized.resolved_hosts + await module.handle_event(port_child) + assert "amazon" in port_child.tags, "OPEN_TCP_PORT child must inherit cloud tag from parent" + assert "asdf.amazonaws.com" in port_child.host_metadata + # ── YARA prefilter short-circuit ─────────────────────────────── # When no host could possibly match any bucket regex (mismatched # suffix) the prefilter returns no matches and we skip the Python diff --git a/bbot/test/test_step_2/module_tests/test_module_virtualhost.py b/bbot/test/test_step_2/module_tests/test_module_virtualhost.py index 2c452b0bd2..8ef3ad6e0e 100644 --- a/bbot/test/test_step_2/module_tests/test_module_virtualhost.py +++ b/bbot/test/test_step_2/module_tests/test_module_virtualhost.py @@ -1070,3 +1070,36 @@ def check(self, module_test, events): f"SAN analyzer received {type(san_arg).__name__}, expected str. Value: {san_arg!r}" ) assert san_arg.startswith("https://"), f"Expected HTTPS URL, got {san_arg!r}" + + +class TestVirtualhostSkipsCdnWaf(VirtualhostTestBase): + """filter_event must reject URLs tagged with any of the flat cloud-provider tags + that cloudcheck emits (post-`Migrate cloudcheck to host_metadata` refactor).""" + + targets = ["http://localhost:8888"] + modules_overrides = ["virtualhost"] + + async def setup_after_prep(self, module_test): + pass + + async def check(self, module_test, events): + vh_module = module_test.scan.modules["virtualhost"] + + for tag in ("cloudflare", "imperva", "akamai", "cloudfront"): + url_event = module_test.scan.make_event( + "http://cdn-test.local:8888/", + "URL", + parent=module_test.scan.root_event, + tags=[tag, "in-scope", "status-200"], + ) + result = await vh_module.filter_event(url_event) + assert result is False, f"virtualhost must skip URL tagged {tag!r} (got {result!r})" + + untagged_event = module_test.scan.make_event( + "http://plain.local:8888/", + "URL", + parent=module_test.scan.root_event, + tags=["in-scope", "status-200"], + ) + result = await vh_module.filter_event(untagged_event) + assert result is True, f"virtualhost must not skip an untagged URL (got {result!r})" diff --git a/bbot/test/test_step_2/module_tests/test_module_waf_bypass.py b/bbot/test/test_step_2/module_tests/test_module_waf_bypass.py index 6e000732e5..45bdbbb135 100644 --- a/bbot/test/test_step_2/module_tests/test_module_waf_bypass.py +++ b/bbot/test/test_step_2/module_tests/test_module_waf_bypass.py @@ -43,7 +43,7 @@ async def handle_event(self, event): self.events_seen.append(event.data) url = "http://protected.test:8888/" url_event = self.scan.make_event( - url, "URL", parent=self.scan.root_event, tags=["cdn-cloudflare", "in-scope", "status-200"] + url, "URL", parent=self.scan.root_event, tags=["cloudflare", "in-scope", "status-200"] ) if url_event is not None: await self.emit_event(url_event) @@ -135,3 +135,58 @@ def check(self, module_test, events): in e.data["description"] ] assert correct_description, "Incorrect description" + + +class TestWAFBypassTagRecognition(ModuleTestBase): + """Regression: waf_bypass keys off the flat cloud-provider tags cloudcheck emits + post-`Migrate cloudcheck to host_metadata` refactor. Both `cloudflare` and `imperva` + tags must trigger protected-domain detection.""" + + targets = ["evilcorp.com"] + modules_overrides = ["waf_bypass"] + + async def setup_after_prep(self, module_test): + pass + + async def check(self, module_test, events): + waf_module = module_test.scan.modules["waf_bypass"] + + async def fake_resolve(host): + return [] + + module_test.monkeypatch.setattr(waf_module.helpers.dns, "resolve", fake_resolve) + + async def fake_get_url_content(url, ip=None): + return None + + import types + + module_test.monkeypatch.setattr( + waf_module, "get_url_content", types.MethodType(fake_get_url_content, waf_module), raising=True + ) + + for tag in ("cloudflare", "imperva"): + waf_module.protected_domains = {} + url_event = module_test.scan.make_event( + f"http://{tag}-protected.test/", + "URL", + parent=module_test.scan.root_event, + tags=[tag, "in-scope", "status-200"], + ) + await waf_module.handle_event(url_event) + assert f"{tag}-protected.test" in waf_module.protected_domains, ( + f"waf_bypass did not recognize {tag!r}-tagged URL as protected" + ) + + # sanity: an untagged URL is not treated as protected + waf_module.protected_domains = {} + untagged = module_test.scan.make_event( + "http://plain.test/", + "URL", + parent=module_test.scan.root_event, + tags=["in-scope", "status-200"], + ) + await waf_module.handle_event(untagged) + assert "plain.test" not in waf_module.protected_domains, ( + "waf_bypass should not flag untagged URLs as protected" + )