paramminer: dedup by endpoint + param keys, not full URL string

bbot/modules/paramminer_headers.py

@@ -328,6 +328,21 @@ async def finish(self):
                 continue
             await self.process_results(event, results)
 
+    def _incoming_dedup_hash(self, event):
+        # dedup by endpoint structure, not full URL string -- value mutations
+        # of the same parameter set (e.g. from lightfuzz probes) are one test surface
+        p = getattr(event, "parsed_url", None)
+        if p is None:
+            return hash(event), ""
+        if event.type == "WEB_PARAMETER":
+            name = event.data.get("name", "")
+            additional_params = event.data.get("additional_params") or {}
+            param_keys = tuple(sorted(additional_params.keys()))
+        else:
+            name = ""
+            param_keys = ()
+        return hash((event.type, p.scheme, p.netloc, p.path, name, param_keys)), "per_endpoint+keys"
+
     async def filter_event(self, event):
         if await self._is_http_wildcard_host(event) is True:
             return False, "host is an HTTP wildcard responder"

bbot/test/test_step_2/module_tests/test_module_paramminer_headers.py

@@ -221,3 +221,88 @@ async def mock_wildcard(scheme, host, port):
     def check(self, module_test, events):
         web_params = [e for e in events if e.type == "WEB_PARAMETER" and str(e.module) == "paramminer_headers"]
         assert len(web_params) == 0, f"paramminer_headers should not fuzz wildcard hosts, but emitted: {web_params}"
+
+
+class TestParamminerHeadersDedupValueMutations(Paramminer_Headers):
+    """Value mutations of the same parameter set on the same endpoint should collapse to one test surface."""
+
+    async def setup_after_prep(self, module_test):
+        await super().setup_after_prep(module_test)
+        pm = module_test.scan.modules["paramminer_headers"]
+
+        def make_wp(**kwargs):
+            defaults = {"host": "127.0.0.1:8888", "type": "GETPARAM", "original_value": "x"}
+            defaults.update(kwargs)
+            return module_test.scan.make_event(defaults, "WEB_PARAMETER", dummy=True)
+
+        def make_hr(url):
+            return module_test.scan.make_event(
+                {
+                    "url": url,
+                    "status_code": 200,
+                    "header-dict": {},
+                    "body": "",
+                    "raw_header": "HTTP/1.1 200 OK\r\n\r\n",
+                    "method": "GET",
+                },
+                "HTTP_RESPONSE",
+                dummy=True,
+            )
+
+        # same endpoint, same param keys, different values (simulates lightfuzz probe cycling)
+        hash_a, _ = pm._incoming_dedup_hash(
+            make_wp(
+                url="http://127.0.0.1:8888/page?culture=probe_a&page=1",
+                name="culture",
+                additional_params={"page": "1"},
+            )
+        )
+        hash_b, _ = pm._incoming_dedup_hash(
+            make_wp(
+                url="http://127.0.0.1:8888/page?culture=probe_b&page=2",
+                name="culture",
+                additional_params={"page": "2"},
+            )
+        )
+        assert hash_a == hash_b, "Value mutations of same param keys should produce same dedup hash"
+
+        # different param keys on same path -> distinct
+        hash_c, _ = pm._incoming_dedup_hash(
+            make_wp(
+                url="http://127.0.0.1:8888/page?culture=x&page=1&csrf=tok",
+                name="culture",
+                additional_params={"page": "1", "csrf": "tok"},
+            )
+        )
+        assert hash_a != hash_c, "Different param key sets should produce different dedup hashes"
+
+        # different path, same keys -> distinct
+        hash_d, _ = pm._incoming_dedup_hash(
+            make_wp(
+                url="http://127.0.0.1:8888/other?culture=x&page=1",
+                name="culture",
+                additional_params={"page": "1"},
+            )
+        )
+        assert hash_a != hash_d, "Same param keys on different paths should produce different dedup hashes"
+
+        # different focus name, same sibling keys (excavate emits one WP per param) -> distinct
+        hash_e, _ = pm._incoming_dedup_hash(
+            make_wp(
+                url="http://127.0.0.1:8888/page?culture=x&page=1",
+                name="page",
+                additional_params={"culture": "x"},
+            )
+        )
+        assert hash_a != hash_e, "Different focus name with same siblings should produce different dedup hashes"
+
+        # HTTP_RESPONSE events: same endpoint collapses regardless
+        hash_hr_a, _ = pm._incoming_dedup_hash(make_hr("http://127.0.0.1:8888/page"))
+        hash_hr_b, _ = pm._incoming_dedup_hash(make_hr("http://127.0.0.1:8888/page"))
+        assert hash_hr_a == hash_hr_b, "HTTP_RESPONSE events for same endpoint should produce same dedup hash"
+
+        # WEB_PARAMETER and HTTP_RESPONSE for same URL should NOT collide
+        assert hash_a != hash_hr_a, "WEB_PARAMETER and HTTP_RESPONSE should not share dedup hashes"
+
+    def check(self, module_test, events):
+        super().check(module_test, events)