diff --git a/bbot/modules/internal/dnsresolve.py b/bbot/modules/internal/dnsresolve.py index aa952dfeb8..ba3bcb517b 100644 --- a/bbot/modules/internal/dnsresolve.py +++ b/bbot/modules/internal/dnsresolve.py @@ -43,6 +43,8 @@ async def setup(self): self.host_module = self.HostModule(self.scan) self.children_emitted = set() + self.in_scope_children = set() + self.child_edges_emitted = set() self.children_emitted_raw = set() self.hosts_resolved = set() @@ -132,10 +134,14 @@ async def handle_event(self, event, **kwargs): ) main_host_event.add_tag(f"runaway-dns-{dns_resolve_distance}") else: - # emit dns children - await self.emit_dns_children_raw(main_host_event, dns_tags) - if not self.minimal: - await self.emit_dns_children(main_host_event) + # graph-important events are edge-only re-emissions of an already-processed host; + # skip re-walking their children (the canonical emission already did so, and + # re-walking would re-emit the same cross-parent edges once per parent) + if not event._graph_important: + # emit dns children + await self.emit_dns_children_raw(main_host_event, dns_tags) + if not self.minimal: + await self.emit_dns_children(main_host_event) # emit the main DNS_NAME or IP_ADDRESS if ( @@ -210,17 +216,39 @@ async def emit_dns_children(self, event): if rdtype == "PTR": child_event.add_tag("ptr") - child_hash = hash(f"{event.host}:{module}:{child_host}") + child_hash = hash(f"{module}:{child_host}") # if we haven't emitted this one before if child_hash not in self.children_emitted: + child_in_scope = self.preset.in_scope(child_host) # and it's either in-scope or inside our dns search distance - if self.preset.in_scope(child_host) or child_event.scope_distance <= self._dns_search_distance: + if child_in_scope or child_event.scope_distance <= self._dns_search_distance: self.children_emitted.add(child_hash) + if child_in_scope: + # remember in-scope children (so cross-parent dups are recognized without + # a second scope lookup) and record this parent->child edge + self.in_scope_children.add(child_hash) + self.child_edges_emitted.add(hash(f"{event.host}:{module}:{child_host}")) # if it's a hostname and it's only one hop away, mark it as affiliate if child_event.type == "DNS_NAME" and child_event.scope_distance == 1: child_event.add_tag("affiliate") self.debug(f"Queueing DNS child for {event}: {child_event}") await self.emit_event(child_event) + # the child entity was already emitted, but a genuinely new in-scope parent->child + # edge is worth preserving for graph outputs (neo4j/json) so shared in-scope + # infrastructure keeps every edge. out-of-scope dups and same-parent re-processing + # stay collapsed (out-of-scope children never enter in_scope_children). + elif child_hash in self.in_scope_children: + # parent-aware key: tells a genuinely new parent->child edge apart from the same + # host being re-processed (children_emitted drops the parent, so it can't) + edge_hash = hash(f"{event.host}:{module}:{child_host}") + if edge_hash not in self.child_edges_emitted: + self.child_edges_emitted.add(edge_hash) + # _graph_important must imply the event reaches output; an omitted type is + # dropped there regardless, so don't flag it (keeps _graph_important => not _omit) + if child_event.type not in self.scan.omitted_event_types: + child_event._graph_important = True + self.debug(f"Queueing graph-important DNS child edge for {event}: {child_event}") + await self.emit_event(child_event) async def emit_dns_children_raw(self, event, dns_tags): for rdtype, answers in event.raw_dns_records.items(): diff --git a/bbot/scanner/manager.py b/bbot/scanner/manager.py index ddf5048be4..38fe5ae793 100644 --- a/bbot/scanner/manager.py +++ b/bbot/scanner/manager.py @@ -271,10 +271,16 @@ async def forward_event(self, event, kwargs): if -1 < event.scope_distance < 1: self.scan.word_cloud.absorb_event(event) - for mod in self.scan.modules.values(): + for module in self.scan.modules.values(): # don't distribute events to intercept modules - if not mod._intercept: - await mod.queue_event(event) + if module._intercept: + continue + # graph-important events are duplicates re-emitted only to preserve graph structure. + # a module that isn't graph-preserving and doesn't accept dupes would just drop them + # at its postcheck, so skip queueing entirely and avoid the churn (outcome unchanged) + if event._graph_important and not (module.preserve_graph or module.accept_dupes): + continue + await module.queue_event(event) # if no module accepted this event, minimize it now if event._module_consumers <= 0: diff --git a/bbot/test/test_step_1/test_manager_deduplication.py b/bbot/test/test_step_1/test_manager_deduplication.py index 9151ad7da8..0f7b0e48b4 100644 --- a/bbot/test/test_step_1/test_manager_deduplication.py +++ b/bbot/test/test_step_1/test_manager_deduplication.py @@ -128,7 +128,7 @@ async def do_scan(*args, _config={}, _dns_mock={}, scan_callback=None, **kwargs) assert 1 == len([e for e in default_events if e.type == "DNS_NAME" and e.data == "per_hostport_only.test.notreal" and str(e.module) == "per_hostport_only"]) assert 1 == len([e for e in default_events if e.type == "DNS_NAME" and e.data == "test.notreal" and str(e.module) == "SEED" and "SCAN:" in e.parent.data["id"]]) - assert len(all_events) == 27 + assert len(all_events) == 26 assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "accept_dupes.test.notreal" and str(e.module) == "accept_dupes"]) assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "default_module.test.notreal" and str(e.module) == "default_module"]) assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes" and e.parent.data == "accept_dupes.test.notreal"]) @@ -140,7 +140,9 @@ async def do_scan(*args, _config={}, _dns_mock={}, scan_callback=None, **kwargs) assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "per_hostport_only.test.notreal" and str(e.module) == "per_hostport_only"]) assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "test.notreal" and str(e.module) == "SEED" and "SCAN:" in e.parent.data["id"]]) assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.3" and str(e.module) == "A" and e.parent.data == "test.notreal"]) - assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.3" and str(e.module) == "A" and e.parent.data == "default_module.test.notreal"]) + # the second 127.0.0.3 (parent=default_module.test.notreal) is now deduped at the DNSResolve layer + # because (rdtype, child) collides with the test.notreal->127.0.0.3 edge already emitted + assert 0 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.3" and str(e.module) == "A" and e.parent.data == "default_module.test.notreal"]) assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.5" and str(e.module) == "A" and e.parent.data == "no_suppress_dupes.test.notreal"]) assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.6" and str(e.module) == "A" and e.parent.data == "accept_dupes.test.notreal"]) assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.7" and str(e.module) == "A" and e.parent.data == "per_hostport_only.test.notreal"]) diff --git a/bbot/test/test_step_2/module_tests/test_module_dnscommonsrv.py b/bbot/test/test_step_2/module_tests/test_module_dnscommonsrv.py index f1321ef896..b8b1255a55 100644 --- a/bbot/test/test_step_2/module_tests/test_module_dnscommonsrv.py +++ b/bbot/test/test_step_2/module_tests/test_module_dnscommonsrv.py @@ -75,9 +75,15 @@ def check(self, module_test, events): and str(e.module) == "dnscommonsrv" ] ), "Failed to detect subdomain 2" - assert 2 == len([e for e in events if e.type == "DNS_NAME" and e.data == "asdf.blacklanternsecurity.com"]), ( - "Failed to detect subdomain 3" - ) + # asdf.blacklanternsecurity.com is in-scope shared infrastructure: the SRV target of two + # different in-scope _ldap records. both parent->child edges are preserved for graph output + # (the second as a graph-important re-emission), so it produces two DNS_NAME events. + asdf_events = [e for e in events if e.type == "DNS_NAME" and e.data == "asdf.blacklanternsecurity.com"] + assert 2 == len(asdf_events), "Failed to detect subdomain 3" + assert {str(e.parent.data) for e in asdf_events} == { + "_ldap._tcp.gc._msdcs.blacklanternsecurity.com", + "_ldap._tcp.gc._msdcs.api.blacklanternsecurity.com", + }, "in-scope shared SRV target should keep both cross-parent edges" assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "api.blacklanternsecurity.com"]), ( "Failed to detect subdomain 4" ) diff --git a/bbot/test/test_step_2/module_tests/test_module_dnsresolve.py b/bbot/test/test_step_2/module_tests/test_module_dnsresolve.py index fb7d1b6224..002b1fb303 100644 --- a/bbot/test/test_step_2/module_tests/test_module_dnsresolve.py +++ b/bbot/test/test_step_2/module_tests/test_module_dnsresolve.py @@ -94,6 +94,181 @@ def check(self, module_test, events): ) +class TestDNSResolveSharedNameserverDedup(ModuleTestBase): + """ + Multiple in-scope parents that share the same NS/SOA records should not cause the + shared nameserver hostname to be re-emitted once per parent. The dedup in + DNSResolve.emit_dns_children must collapse identical (rdtype, child) pairs across + parents, not key on parent host. + """ + + module_name = "dnsresolve" + targets = ["domain-a.test", "domain-b.test", "domain-c.test"] + config_overrides = {"dns": {"minimal": False}, "scope": {"report_distance": 1}} + + async def setup_after_prep(self, module_test): + shared_ns = ["shared-ns1.cloudprovider.test.", "shared-ns2.cloudprovider.test."] + shared_soa = ["shared-ns1.cloudprovider.test. admin.cloudprovider.test. 1 7200 3600 1209600 3600"] + await module_test.mock_dns( + { + "domain-a.test": {"A": ["192.168.0.1"], "NS": shared_ns, "SOA": shared_soa}, + "domain-b.test": {"A": ["192.168.0.2"], "NS": shared_ns, "SOA": shared_soa}, + "domain-c.test": {"A": ["192.168.0.3"], "NS": shared_ns, "SOA": shared_soa}, + # resolve the shared nameservers so they keep the DNS_NAME type + "shared-ns1.cloudprovider.test": {"A": ["192.168.99.1"]}, + "shared-ns2.cloudprovider.test": {"A": ["192.168.99.2"]}, + } + ) + + def check(self, module_test, events): + from collections import Counter + + ns_counts = Counter( + e.data for e in events if e.type in ("DNS_NAME", "DNS_NAME_UNRESOLVED") and str(e.module) == "NS" + ) + soa_counts = Counter( + e.data for e in events if e.type in ("DNS_NAME", "DNS_NAME_UNRESOLVED") and str(e.module) == "SOA" + ) + + # each shared nameserver hostname is referenced by all three in-scope parents + # but should still be emitted exactly once + assert ns_counts == { + "shared-ns1.cloudprovider.test": 1, + "shared-ns2.cloudprovider.test": 1, + }, ( + f"Expected each shared NS hostname to be emitted exactly once across parents, " + f"got: {dict(ns_counts)}. emit_dns_children dedup is keyed on " + f"(parent_host, rdtype, child) instead of (rdtype, child)." + ) + assert soa_counts == {"shared-ns1.cloudprovider.test": 1}, ( + f"Expected the shared SOA hostname to be emitted exactly once across parents, got: {dict(soa_counts)}." + ) + + +class TestDNSResolveInScopeSharedInfraGraphFidelity(ModuleTestBase): + """ + Cross-parent graph fidelity for IN-SCOPE shared infrastructure, asserted against the + actual json graph output (the same parent->child edges neo4j builds). + + Three in-scope parents share the same in-scope NS/SOA hosts (subdomains of an in-scope + target). Unlike the out-of-scope affiliate flood in TestDNSResolveSharedNameserverDedup, + each of these parent->child edges is a distinct, real graph edge that graph outputs + (neo4j/json) must preserve. The (rdtype, child) dedup correctly collapses the affiliate + flood, but it also drops these in-scope cross-parent edges, linking the shared host to + only the first-seen parent. + + This reads output.json and asserts every in-scope parent->child edge survives. It FAILS + against the current dedup (only the first-seen edge is kept) and should pass once + cross-parent edges are preserved for graph outputs (e.g. re-emitted as graph-important). + """ + + module_name = "dnsresolve" + modules_overrides = ["dnsresolve", "json"] + targets = ["domain-a.test", "domain-b.test", "domain-c.test"] + config_overrides = {"dns": {"minimal": False}} + + async def setup_after_prep(self, module_test): + # shared nameservers that are themselves in-scope (subdomains of an in-scope target) + shared_ns = ["ns1.domain-a.test.", "ns2.domain-a.test."] + shared_soa = ["ns1.domain-a.test. admin.domain-a.test. 1 7200 3600 1209600 3600"] + await module_test.mock_dns( + { + "domain-a.test": {"A": ["192.168.0.1"], "NS": shared_ns, "SOA": shared_soa}, + "domain-b.test": {"A": ["192.168.0.2"], "NS": shared_ns, "SOA": shared_soa}, + "domain-c.test": {"A": ["192.168.0.3"], "NS": shared_ns, "SOA": shared_soa}, + # resolve the shared nameservers so they keep the DNS_NAME type + "ns1.domain-a.test": {"A": ["192.168.99.1"]}, + "ns2.domain-a.test": {"A": ["192.168.99.2"]}, + } + ) + + def check(self, module_test, events): + import json + + output_json = module_test.scan.home / "output.json" + assert output_json.is_file(), f"json output not written to {output_json}" + graph_events = [json.loads(line) for line in output_json.read_text().splitlines() if line.strip()] + # map each event id to its data so we can resolve parent ids back to parent hosts (edges) + id_to_data = {e["id"]: e.get("data") for e in graph_events if "id" in e} + + expected_parents = {"domain-a.test", "domain-b.test", "domain-c.test"} + + def parents_of(child, module): + return { + id_to_data.get(e.get("parent")) + for e in graph_events + if e.get("type") in ("DNS_NAME", "DNS_NAME_UNRESOLVED") + and e.get("data") == child + and e.get("module") == module + } + + ns1_parents = parents_of("ns1.domain-a.test", "NS") + ns2_parents = parents_of("ns2.domain-a.test", "NS") + soa_parents = parents_of("ns1.domain-a.test", "SOA") + + assert ns1_parents == expected_parents, ( + f"in-scope shared NS ns1.domain-a.test lost cross-parent edges in graph output: " + f"linked to {ns1_parents or set()}, expected {expected_parents}" + ) + assert ns2_parents == expected_parents, ( + f"in-scope shared NS ns2.domain-a.test lost cross-parent edges in graph output: " + f"linked to {ns2_parents or set()}, expected {expected_parents}" + ) + assert soa_parents == expected_parents, ( + f"in-scope shared SOA host lost cross-parent edges in graph output: " + f"linked to {soa_parents or set()}, expected {expected_parents}" + ) + + +class TestDNSResolveOmittedSharedInfraNotGraphImportant(ModuleTestBase): + """ + Cross-parent edge re-emission must keep the rule that a graph-important event is never + omitted (graph outputs force these in; an omitted child is dropped at output regardless). + + Same in-scope shared infra as TestDNSResolveInScopeSharedInfraGraphFidelity, but DNS_NAME is + omitted. dnsresolve must not flag the re-emitted cross-parent edges _graph_important, otherwise + they become the omit + graph-important combination the graph machinery assumes is unreachable. + """ + + module_name = "dnsresolve" + modules_overrides = ["dnsresolve", "json"] + targets = ["domain-a.test", "domain-b.test", "domain-c.test"] + config_overrides = {"dns": {"minimal": False}, "omit_event_types": ["DNS_NAME"]} + + async def setup_after_prep(self, module_test): + shared_ns = ["ns1.domain-a.test.", "ns2.domain-a.test."] + shared_soa = ["ns1.domain-a.test. admin.domain-a.test. 1 7200 3600 1209600 3600"] + await module_test.mock_dns( + { + "domain-a.test": {"A": ["192.168.0.1"], "NS": shared_ns, "SOA": shared_soa}, + "domain-b.test": {"A": ["192.168.0.2"], "NS": shared_ns, "SOA": shared_soa}, + "domain-c.test": {"A": ["192.168.0.3"], "NS": shared_ns, "SOA": shared_soa}, + "ns1.domain-a.test": {"A": ["192.168.99.1"]}, + "ns2.domain-a.test": {"A": ["192.168.99.2"]}, + } + ) + + # snapshot (type, graph_important) for every event dnsresolve emits, at emit time + self.emitted = [] + dnsresolve = module_test.scan.modules["dnsresolve"] + original_emit = dnsresolve.emit_event + + async def recording_emit(*args, **kwargs): + if args and hasattr(args[0], "_graph_important"): + self.emitted.append((args[0].type, args[0]._graph_important)) + return await original_emit(*args, **kwargs) + + module_test.monkeypatch.setattr(dnsresolve, "emit_event", recording_emit) + + def check(self, module_test, events): + omitted = set(module_test.scan.omitted_event_types) + violations = [t for t, graph_important in self.emitted if graph_important and t in omitted] + assert not violations, ( + f"dnsresolve emitted graph-important events of an omitted type {violations} " + f"(violates the _graph_important => not _omit rule)" + ) + + class TestDNSResolveFilterPTRsDisabled(ModuleTestBase): """Test that PTR-derived hostnames ARE promoted to in-scope when filter_ptrs is disabled.""" diff --git a/bbot/test/test_step_2/template_tests/test_template_subdomain_enum.py b/bbot/test/test_step_2/template_tests/test_template_subdomain_enum.py index 5104291cc3..5c5a442db7 100644 --- a/bbot/test/test_step_2/template_tests/test_template_subdomain_enum.py +++ b/bbot/test/test_step_2/template_tests/test_template_subdomain_enum.py @@ -127,8 +127,10 @@ async def mock_query(query): def check(self, module_test, events): assert self.queries == ["walmart.cn"] - assert len(events) == 7 - assert 2 == len( + assert len(events) == 6 + # cross-parent (rdtype, child) dedup in DNSResolve.emit_dns_children collapses + # the three walmart.cn->127.0.0.1 edges into a single IP_ADDRESS event + assert 1 == len( [ e for e in events