diff --git a/bbot/modules/paramminer_headers.py b/bbot/modules/paramminer_headers.py index eb86134785..aacebe698a 100644 --- a/bbot/modules/paramminer_headers.py +++ b/bbot/modules/paramminer_headers.py @@ -328,6 +328,21 @@ async def finish(self): continue await self.process_results(event, results) + def _incoming_dedup_hash(self, event): + # dedup by endpoint structure, not full URL string -- value mutations + # of the same parameter set (e.g. from lightfuzz probes) are one test surface + p = getattr(event, "parsed_url", None) + if p is None: + return hash(event), "" + if event.type == "WEB_PARAMETER": + name = event.data.get("name", "") + additional_params = event.data.get("additional_params") or {} + param_keys = tuple(sorted(additional_params.keys())) + else: + name = "" + param_keys = () + return hash((event.type, p.scheme, p.netloc, p.path, name, param_keys)), "per_endpoint+keys" + async def filter_event(self, event): if await self._is_http_wildcard_host(event) is True: return False, "host is an HTTP wildcard responder" diff --git a/bbot/test/test_step_2/module_tests/test_module_paramminer_headers.py b/bbot/test/test_step_2/module_tests/test_module_paramminer_headers.py index 17b353d58c..846efa177f 100644 --- a/bbot/test/test_step_2/module_tests/test_module_paramminer_headers.py +++ b/bbot/test/test_step_2/module_tests/test_module_paramminer_headers.py @@ -221,3 +221,88 @@ async def mock_wildcard(scheme, host, port): def check(self, module_test, events): web_params = [e for e in events if e.type == "WEB_PARAMETER" and str(e.module) == "paramminer_headers"] assert len(web_params) == 0, f"paramminer_headers should not fuzz wildcard hosts, but emitted: {web_params}" + + +class TestParamminerHeadersDedupValueMutations(Paramminer_Headers): + """Value mutations of the same parameter set on the same endpoint should collapse to one test surface.""" + + async def setup_after_prep(self, module_test): + await super().setup_after_prep(module_test) + pm = module_test.scan.modules["paramminer_headers"] + + def make_wp(**kwargs): + defaults = {"host": "127.0.0.1:8888", "type": "GETPARAM", "original_value": "x"} + defaults.update(kwargs) + return module_test.scan.make_event(defaults, "WEB_PARAMETER", dummy=True) + + def make_hr(url): + return module_test.scan.make_event( + { + "url": url, + "status_code": 200, + "header-dict": {}, + "body": "", + "raw_header": "HTTP/1.1 200 OK\r\n\r\n", + "method": "GET", + }, + "HTTP_RESPONSE", + dummy=True, + ) + + # same endpoint, same param keys, different values (simulates lightfuzz probe cycling) + hash_a, _ = pm._incoming_dedup_hash( + make_wp( + url="http://127.0.0.1:8888/page?culture=probe_a&page=1", + name="culture", + additional_params={"page": "1"}, + ) + ) + hash_b, _ = pm._incoming_dedup_hash( + make_wp( + url="http://127.0.0.1:8888/page?culture=probe_b&page=2", + name="culture", + additional_params={"page": "2"}, + ) + ) + assert hash_a == hash_b, "Value mutations of same param keys should produce same dedup hash" + + # different param keys on same path -> distinct + hash_c, _ = pm._incoming_dedup_hash( + make_wp( + url="http://127.0.0.1:8888/page?culture=x&page=1&csrf=tok", + name="culture", + additional_params={"page": "1", "csrf": "tok"}, + ) + ) + assert hash_a != hash_c, "Different param key sets should produce different dedup hashes" + + # different path, same keys -> distinct + hash_d, _ = pm._incoming_dedup_hash( + make_wp( + url="http://127.0.0.1:8888/other?culture=x&page=1", + name="culture", + additional_params={"page": "1"}, + ) + ) + assert hash_a != hash_d, "Same param keys on different paths should produce different dedup hashes" + + # different focus name, same sibling keys (excavate emits one WP per param) -> distinct + hash_e, _ = pm._incoming_dedup_hash( + make_wp( + url="http://127.0.0.1:8888/page?culture=x&page=1", + name="page", + additional_params={"culture": "x"}, + ) + ) + assert hash_a != hash_e, "Different focus name with same siblings should produce different dedup hashes" + + # HTTP_RESPONSE events: same endpoint collapses regardless + hash_hr_a, _ = pm._incoming_dedup_hash(make_hr("http://127.0.0.1:8888/page")) + hash_hr_b, _ = pm._incoming_dedup_hash(make_hr("http://127.0.0.1:8888/page")) + assert hash_hr_a == hash_hr_b, "HTTP_RESPONSE events for same endpoint should produce same dedup hash" + + # WEB_PARAMETER and HTTP_RESPONSE for same URL should NOT collide + assert hash_a != hash_hr_a, "WEB_PARAMETER and HTTP_RESPONSE should not share dedup hashes" + + def check(self, module_test, events): + super().check(module_test, events)