dedup dns children by (rdtype, child) not parent host

bbot/modules/internal/dnsresolve.py

@@ -43,6 +43,8 @@ async def setup(self):
 
         self.host_module = self.HostModule(self.scan)
         self.children_emitted = set()
+        self.in_scope_children = set()
+        self.child_edges_emitted = set()
         self.children_emitted_raw = set()
         self.hosts_resolved = set()
 
@@ -132,10 +134,14 @@ async def handle_event(self, event, **kwargs):
             )
             main_host_event.add_tag(f"runaway-dns-{dns_resolve_distance}")
         else:
-            # emit dns children
-            await self.emit_dns_children_raw(main_host_event, dns_tags)
-            if not self.minimal:
-                await self.emit_dns_children(main_host_event)
+            # graph-important events are edge-only re-emissions of an already-processed host;
+            # skip re-walking their children (the canonical emission already did so, and
+            # re-walking would re-emit the same cross-parent edges once per parent)
+            if not event._graph_important:
+                # emit dns children
+                await self.emit_dns_children_raw(main_host_event, dns_tags)
+                if not self.minimal:
+                    await self.emit_dns_children(main_host_event)
 
             # emit the main DNS_NAME or IP_ADDRESS
             if (
@@ -210,17 +216,39 @@ async def emit_dns_children(self, event):
                 if rdtype == "PTR":
                     child_event.add_tag("ptr")
 
-                child_hash = hash(f"{event.host}:{module}:{child_host}")
+                child_hash = hash(f"{module}:{child_host}")
                 # if we haven't emitted this one before
                 if child_hash not in self.children_emitted:
+                    child_in_scope = self.preset.in_scope(child_host)
                     # and it's either in-scope or inside our dns search distance
-                    if self.preset.in_scope(child_host) or child_event.scope_distance <= self._dns_search_distance:
+                    if child_in_scope or child_event.scope_distance <= self._dns_search_distance:
                         self.children_emitted.add(child_hash)
+                        if child_in_scope:
+                            # remember in-scope children (so cross-parent dups are recognized without
+                            # a second scope lookup) and record this parent->child edge
+                            self.in_scope_children.add(child_hash)
+                            self.child_edges_emitted.add(hash(f"{event.host}:{module}:{child_host}"))
                         # if it's a hostname and it's only one hop away, mark it as affiliate
                         if child_event.type == "DNS_NAME" and child_event.scope_distance == 1:
                             child_event.add_tag("affiliate")
                         self.debug(f"Queueing DNS child for {event}: {child_event}")
                         await self.emit_event(child_event)
+                # the child entity was already emitted, but a genuinely new in-scope parent->child
+                # edge is worth preserving for graph outputs (neo4j/json) so shared in-scope
+                # infrastructure keeps every edge. out-of-scope dups and same-parent re-processing
+                # stay collapsed (out-of-scope children never enter in_scope_children).
+                elif child_hash in self.in_scope_children:
+                    # parent-aware key: tells a genuinely new parent->child edge apart from the same
+                    # host being re-processed (children_emitted drops the parent, so it can't)
+                    edge_hash = hash(f"{event.host}:{module}:{child_host}")
+                    if edge_hash not in self.child_edges_emitted:
+                        self.child_edges_emitted.add(edge_hash)
+                        # _graph_important must imply the event reaches output; an omitted type is
+                        # dropped there regardless, so don't flag it (keeps _graph_important => not _omit)
+                        if child_event.type not in self.scan.omitted_event_types:
+                            child_event._graph_important = True
+                            self.debug(f"Queueing graph-important DNS child edge for {event}: {child_event}")
+                            await self.emit_event(child_event)
 
     async def emit_dns_children_raw(self, event, dns_tags):
         for rdtype, answers in event.raw_dns_records.items():

bbot/scanner/manager.py

@@ -271,10 +271,16 @@ async def forward_event(self, event, kwargs):
         if -1 < event.scope_distance < 1:
             self.scan.word_cloud.absorb_event(event)
 
-        for mod in self.scan.modules.values():
+        for module in self.scan.modules.values():
             # don't distribute events to intercept modules
-            if not mod._intercept:
-                await mod.queue_event(event)
+            if module._intercept:
+                continue
+            # graph-important events are duplicates re-emitted only to preserve graph structure.
+            # a module that isn't graph-preserving and doesn't accept dupes would just drop them
+            # at its postcheck, so skip queueing entirely and avoid the churn (outcome unchanged)
+            if event._graph_important and not (module.preserve_graph or module.accept_dupes):
+                continue
+            await module.queue_event(event)
 
         # if no module accepted this event, minimize it now
         if event._module_consumers <= 0:

bbot/test/test_step_1/test_manager_deduplication.py

@@ -128,7 +128,7 @@ async def do_scan(*args, _config={}, _dns_mock={}, scan_callback=None, **kwargs)
     assert 1 == len([e for e in default_events if e.type == "DNS_NAME" and e.data == "per_hostport_only.test.notreal" and str(e.module) == "per_hostport_only"])
     assert 1 == len([e for e in default_events if e.type == "DNS_NAME" and e.data == "test.notreal" and str(e.module) == "SEED" and "SCAN:" in e.parent.data["id"]])
 
-    assert len(all_events) == 27
+    assert len(all_events) == 26
     assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "accept_dupes.test.notreal" and str(e.module) == "accept_dupes"])
     assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "default_module.test.notreal" and str(e.module) == "default_module"])
     assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes" and e.parent.data == "accept_dupes.test.notreal"])
@@ -140,7 +140,9 @@ async def do_scan(*args, _config={}, _dns_mock={}, scan_callback=None, **kwargs)
     assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "per_hostport_only.test.notreal" and str(e.module) == "per_hostport_only"])
     assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "test.notreal" and str(e.module) == "SEED" and "SCAN:" in e.parent.data["id"]])
     assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.3" and str(e.module) == "A" and e.parent.data == "test.notreal"])
-    assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.3" and str(e.module) == "A" and e.parent.data == "default_module.test.notreal"])
+    # the second 127.0.0.3 (parent=default_module.test.notreal) is now deduped at the DNSResolve layer
+    # because (rdtype, child) collides with the test.notreal->127.0.0.3 edge already emitted
+    assert 0 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.3" and str(e.module) == "A" and e.parent.data == "default_module.test.notreal"])
     assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.5" and str(e.module) == "A" and e.parent.data == "no_suppress_dupes.test.notreal"])
     assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.6" and str(e.module) == "A" and e.parent.data == "accept_dupes.test.notreal"])
     assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.7" and str(e.module) == "A" and e.parent.data == "per_hostport_only.test.notreal"])

bbot/test/test_step_2/module_tests/test_module_dnscommonsrv.py

@@ -75,9 +75,15 @@ def check(self, module_test, events):
                 and str(e.module) == "dnscommonsrv"
             ]
         ), "Failed to detect subdomain 2"
-        assert 2 == len([e for e in events if e.type == "DNS_NAME" and e.data == "asdf.blacklanternsecurity.com"]), (
-            "Failed to detect subdomain 3"
-        )
+        # asdf.blacklanternsecurity.com is in-scope shared infrastructure: the SRV target of two
+        # different in-scope _ldap records. both parent->child edges are preserved for graph output
+        # (the second as a graph-important re-emission), so it produces two DNS_NAME events.
+        asdf_events = [e for e in events if e.type == "DNS_NAME" and e.data == "asdf.blacklanternsecurity.com"]
+        assert 2 == len(asdf_events), "Failed to detect subdomain 3"
+        assert {str(e.parent.data) for e in asdf_events} == {
+            "_ldap._tcp.gc._msdcs.blacklanternsecurity.com",
+            "_ldap._tcp.gc._msdcs.api.blacklanternsecurity.com",
+        }, "in-scope shared SRV target should keep both cross-parent edges"
         assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "api.blacklanternsecurity.com"]), (
             "Failed to detect subdomain 4"
         )

bbot/test/test_step_2/module_tests/test_module_dnsresolve.py

@@ -94,6 +94,181 @@ def check(self, module_test, events):
         )
 
 
+class TestDNSResolveSharedNameserverDedup(ModuleTestBase):
+    """
+    Multiple in-scope parents that share the same NS/SOA records should not cause the
+    shared nameserver hostname to be re-emitted once per parent. The dedup in
+    DNSResolve.emit_dns_children must collapse identical (rdtype, child) pairs across
+    parents, not key on parent host.
+    """
+
+    module_name = "dnsresolve"
+    targets = ["domain-a.test", "domain-b.test", "domain-c.test"]
+    config_overrides = {"dns": {"minimal": False}, "scope": {"report_distance": 1}}
+
+    async def setup_after_prep(self, module_test):
+        shared_ns = ["shared-ns1.cloudprovider.test.", "shared-ns2.cloudprovider.test."]
+        shared_soa = ["shared-ns1.cloudprovider.test. admin.cloudprovider.test. 1 7200 3600 1209600 3600"]
+        await module_test.mock_dns(
+            {
+                "domain-a.test": {"A": ["192.168.0.1"], "NS": shared_ns, "SOA": shared_soa},
+                "domain-b.test": {"A": ["192.168.0.2"], "NS": shared_ns, "SOA": shared_soa},
+                "domain-c.test": {"A": ["192.168.0.3"], "NS": shared_ns, "SOA": shared_soa},
+                # resolve the shared nameservers so they keep the DNS_NAME type
+                "shared-ns1.cloudprovider.test": {"A": ["192.168.99.1"]},
+                "shared-ns2.cloudprovider.test": {"A": ["192.168.99.2"]},
+            }
+        )
+
+    def check(self, module_test, events):
+        from collections import Counter
+
+        ns_counts = Counter(
+            e.data for e in events if e.type in ("DNS_NAME", "DNS_NAME_UNRESOLVED") and str(e.module) == "NS"
+        )
+        soa_counts = Counter(
+            e.data for e in events if e.type in ("DNS_NAME", "DNS_NAME_UNRESOLVED") and str(e.module) == "SOA"
+        )
+
+        # each shared nameserver hostname is referenced by all three in-scope parents
+        # but should still be emitted exactly once
+        assert ns_counts == {
+            "shared-ns1.cloudprovider.test": 1,
+            "shared-ns2.cloudprovider.test": 1,
+        }, (
+            f"Expected each shared NS hostname to be emitted exactly once across parents, "
+            f"got: {dict(ns_counts)}. emit_dns_children dedup is keyed on "
+            f"(parent_host, rdtype, child) instead of (rdtype, child)."
+        )
+        assert soa_counts == {"shared-ns1.cloudprovider.test": 1}, (
+            f"Expected the shared SOA hostname to be emitted exactly once across parents, got: {dict(soa_counts)}."
+        )
+
+
+class TestDNSResolveInScopeSharedInfraGraphFidelity(ModuleTestBase):
+    """
+    Cross-parent graph fidelity for IN-SCOPE shared infrastructure, asserted against the
+    actual json graph output (the same parent->child edges neo4j builds).
+
+    Three in-scope parents share the same in-scope NS/SOA hosts (subdomains of an in-scope
+    target). Unlike the out-of-scope affiliate flood in TestDNSResolveSharedNameserverDedup,
+    each of these parent->child edges is a distinct, real graph edge that graph outputs
+    (neo4j/json) must preserve. The (rdtype, child) dedup correctly collapses the affiliate
+    flood, but it also drops these in-scope cross-parent edges, linking the shared host to
+    only the first-seen parent.
+
+    This reads output.json and asserts every in-scope parent->child edge survives. It FAILS
+    against the current dedup (only the first-seen edge is kept) and should pass once
+    cross-parent edges are preserved for graph outputs (e.g. re-emitted as graph-important).
+    """
+
+    module_name = "dnsresolve"
+    modules_overrides = ["dnsresolve", "json"]
+    targets = ["domain-a.test", "domain-b.test", "domain-c.test"]
+    config_overrides = {"dns": {"minimal": False}}
+
+    async def setup_after_prep(self, module_test):
+        # shared nameservers that are themselves in-scope (subdomains of an in-scope target)
+        shared_ns = ["ns1.domain-a.test.", "ns2.domain-a.test."]
+        shared_soa = ["ns1.domain-a.test. admin.domain-a.test. 1 7200 3600 1209600 3600"]
+        await module_test.mock_dns(
+            {
+                "domain-a.test": {"A": ["192.168.0.1"], "NS": shared_ns, "SOA": shared_soa},
+                "domain-b.test": {"A": ["192.168.0.2"], "NS": shared_ns, "SOA": shared_soa},
+                "domain-c.test": {"A": ["192.168.0.3"], "NS": shared_ns, "SOA": shared_soa},
+                # resolve the shared nameservers so they keep the DNS_NAME type
+                "ns1.domain-a.test": {"A": ["192.168.99.1"]},
+                "ns2.domain-a.test": {"A": ["192.168.99.2"]},
+            }
+        )
+
+    def check(self, module_test, events):
+        import json
+
+        output_json = module_test.scan.home / "output.json"
+        assert output_json.is_file(), f"json output not written to {output_json}"
+        graph_events = [json.loads(line) for line in output_json.read_text().splitlines() if line.strip()]
+        # map each event id to its data so we can resolve parent ids back to parent hosts (edges)
+        id_to_data = {e["id"]: e.get("data") for e in graph_events if "id" in e}
+
+        expected_parents = {"domain-a.test", "domain-b.test", "domain-c.test"}
+
+        def parents_of(child, module):
+            return {
+                id_to_data.get(e.get("parent"))
+                for e in graph_events
+                if e.get("type") in ("DNS_NAME", "DNS_NAME_UNRESOLVED")
+                and e.get("data") == child
+                and e.get("module") == module
+            }
+
+        ns1_parents = parents_of("ns1.domain-a.test", "NS")
+        ns2_parents = parents_of("ns2.domain-a.test", "NS")
+        soa_parents = parents_of("ns1.domain-a.test", "SOA")
+
+        assert ns1_parents == expected_parents, (
+            f"in-scope shared NS ns1.domain-a.test lost cross-parent edges in graph output: "
+            f"linked to {ns1_parents or set()}, expected {expected_parents}"
+        )
+        assert ns2_parents == expected_parents, (
+            f"in-scope shared NS ns2.domain-a.test lost cross-parent edges in graph output: "
+            f"linked to {ns2_parents or set()}, expected {expected_parents}"
+        )
+        assert soa_parents == expected_parents, (
+            f"in-scope shared SOA host lost cross-parent edges in graph output: "
+            f"linked to {soa_parents or set()}, expected {expected_parents}"
+        )
+
+
+class TestDNSResolveOmittedSharedInfraNotGraphImportant(ModuleTestBase):
+    """
+    Cross-parent edge re-emission must keep the rule that a graph-important event is never
+    omitted (graph outputs force these in; an omitted child is dropped at output regardless).
+
+    Same in-scope shared infra as TestDNSResolveInScopeSharedInfraGraphFidelity, but DNS_NAME is
+    omitted. dnsresolve must not flag the re-emitted cross-parent edges _graph_important, otherwise
+    they become the omit + graph-important combination the graph machinery assumes is unreachable.
+    """
+
+    module_name = "dnsresolve"
+    modules_overrides = ["dnsresolve", "json"]
+    targets = ["domain-a.test", "domain-b.test", "domain-c.test"]
+    config_overrides = {"dns": {"minimal": False}, "omit_event_types": ["DNS_NAME"]}
+
+    async def setup_after_prep(self, module_test):
+        shared_ns = ["ns1.domain-a.test.", "ns2.domain-a.test."]
+        shared_soa = ["ns1.domain-a.test. admin.domain-a.test. 1 7200 3600 1209600 3600"]
+        await module_test.mock_dns(
+            {
+                "domain-a.test": {"A": ["192.168.0.1"], "NS": shared_ns, "SOA": shared_soa},
+                "domain-b.test": {"A": ["192.168.0.2"], "NS": shared_ns, "SOA": shared_soa},
+                "domain-c.test": {"A": ["192.168.0.3"], "NS": shared_ns, "SOA": shared_soa},
+                "ns1.domain-a.test": {"A": ["192.168.99.1"]},
+                "ns2.domain-a.test": {"A": ["192.168.99.2"]},
+            }
+        )
+
+        # snapshot (type, graph_important) for every event dnsresolve emits, at emit time
+        self.emitted = []
+        dnsresolve = module_test.scan.modules["dnsresolve"]
+        original_emit = dnsresolve.emit_event
+
+        async def recording_emit(*args, **kwargs):
+            if args and hasattr(args[0], "_graph_important"):
+                self.emitted.append((args[0].type, args[0]._graph_important))
+            return await original_emit(*args, **kwargs)
+
+        module_test.monkeypatch.setattr(dnsresolve, "emit_event", recording_emit)
+
+    def check(self, module_test, events):
+        omitted = set(module_test.scan.omitted_event_types)
+        violations = [t for t, graph_important in self.emitted if graph_important and t in omitted]
+        assert not violations, (
+            f"dnsresolve emitted graph-important events of an omitted type {violations} "
+            f"(violates the _graph_important => not _omit rule)"
+        )
+
+
 class TestDNSResolveFilterPTRsDisabled(ModuleTestBase):
     """Test that PTR-derived hostnames ARE promoted to in-scope when filter_ptrs is disabled."""
 

bbot/test/test_step_2/template_tests/test_template_subdomain_enum.py

@@ -127,8 +127,10 @@ async def mock_query(query):
 
     def check(self, module_test, events):
         assert self.queries == ["walmart.cn"]
-        assert len(events) == 7
-        assert 2 == len(
+        assert len(events) == 6
+        # cross-parent (rdtype, child) dedup in DNSResolve.emit_dns_children collapses
+        # the three walmart.cn->127.0.0.1 edges into a single IP_ADDRESS event
+        assert 1 == len(
             [
                 e
                 for e in events