Scope Rework

bbot/core/event/base.py

@@ -108,7 +108,8 @@ class BaseEvent:
     # Always emit this event type even if it's not in scope
     _always_emit = False
     # Always emit events with these tags even if they're not in scope
-    _always_emit_tags = ["affiliate", "target"]
+
+    _always_emit_tags = ["affiliate", "seed"]
     # Bypass scope checking and dns resolution, distribute immediately to modules
     # This is useful for "end-of-line" events like FINDING and VULNERABILITY
     _quick_emit = False
@@ -596,6 +597,9 @@ def parent(self, parent):
                     new_scope_distance += 1
             self.scope_distance = new_scope_distance
             # inherit certain tags
+            # inherit seed tag from DNS_NAME_UNRESOLVED -> DNS_NAME only
+            if "seed" in parent.tags and parent.type == "DNS_NAME_UNRESOLVED" and self.type == "DNS_NAME":
+                self.add_tag("seed")
             if hosts_are_same:
                 # inherit web spider distance from parent
                 self.web_spider_distance = getattr(parent, "web_spider_distance", 0)
@@ -1214,6 +1218,9 @@ def _words(self):
 
 
 class OPEN_TCP_PORT(BaseEvent):
+    # we generally don't care about open ports on affiliates
+    _always_emit_tags = ["seed"]
+
     def sanitize_data(self, data):
         return validators.validate_open_port(data)
 
@@ -1719,7 +1726,7 @@ def __init__(self, *args, **kwargs):
 
 class RAW_DNS_RECORD(DictHostEvent, DnsEvent):
     # don't emit raw DNS records for affiliates
-    _always_emit_tags = ["target"]
+    _always_emit_tags = ["seed"]
 
 
 class MOBILE_APP(DictEvent):

bbot/core/helpers/web/client.py

@@ -90,9 +90,9 @@ def build_request(self, *args, **kwargs):
             kwargs["url"] = url
         url = kwargs["url"]
 
-        target_in_scope = self._target.in_scope(str(url))
+        in_target = self._target.in_target(str(url))
 
-        if target_in_scope:
+        if in_target:
             if not kwargs.get("cookies", None):
                 kwargs["cookies"] = {}
             for ck, cv in self._web_config.get("http_cookies", {}).items():
@@ -101,7 +101,7 @@ def build_request(self, *args, **kwargs):
 
         request = super().build_request(**kwargs)
 
-        if target_in_scope:
+        if in_target:
             for hk, hv in self._web_config.get("http_headers", {}).items():
                 hv = str(hv)
                 # don't clobber headers

bbot/models/pydantic.py

@@ -148,11 +148,11 @@ def from_scan(cls, scan):
 class Target(BBOTBaseModel):
     name: str = "Default Target"
     strict_dns_scope: bool = False
-    seeds: List = []
-    whitelist: Optional[List] = None
+    target: List = []
+    seeds: Optional[List] = None
     blacklist: List = []
     hash: Annotated[str, "indexed", "unique"]
     scope_hash: Annotated[str, "indexed"]
     seed_hash: Annotated[str, "indexed"]
-    whitelist_hash: Annotated[str, "indexed"]
+    target_hash: Annotated[str, "indexed"]
     blacklist_hash: Annotated[str, "indexed"]

bbot/models/sql.py

@@ -125,11 +125,11 @@ class Scan(BBOTBaseModel, table=True):
 class Target(BBOTBaseModel, table=True):
     name: str = "Default Target"
     strict_dns_scope: bool = False
-    seeds: List = Field(default=[], sa_type=JSON)
-    whitelist: Optional[List] = Field(default=None, sa_type=JSON)
+    target: List = Field(default=[], sa_type=JSON)
+    seeds: Optional[List] = Field(default=None, sa_type=JSON)
     blacklist: List = Field(default=[], sa_type=JSON)
     hash: str = Field(sa_column=Column("hash", String(length=255), unique=True, primary_key=True, index=True))
     scope_hash: str = Field(sa_column=Column("scope_hash", String(length=255), index=True))
-    seed_hash: str = Field(sa_column=Column("seed_hashhash", String(length=255), index=True))
-    whitelist_hash: str = Field(sa_column=Column("whitelist_hash", String(length=255), index=True))
+    seed_hash: str = Field(sa_column=Column("seed_hash", String(length=255), index=True))
+    target_hash: str = Field(sa_column=Column("target_hash", String(length=255), index=True))
     blacklist_hash: str = Field(sa_column=Column("blacklist_hash", String(length=255), index=True))

bbot/modules/base.py

@@ -52,6 +52,9 @@ class BaseModule:
 
         target_only (bool): Accept only the initial target event(s). Default is False.
 
+        accept_seeds (bool): Accept seed events (events from initial scan seeds).
+            Defaults to True for passive modules, False otherwise. Can be explicitly set to override the default.
+
         in_scope_only (bool): Accept only explicitly in-scope events, regardless of the scan's search distance. Default is False.
 
         accept_url_special (bool): Accept "special" URLs not typically distributed to web modules, e.g. JS URLs. Default is False.
@@ -791,6 +794,15 @@ async def _worker(self):
                 self.error(traceback.format_exc())
         self.log.trace("Worker stopped")
 
+    @property
+    def accept_seeds(self):
+        """
+        Returns whether the module accepts seed events.
+        Defaults to True for passive modules, False otherwise.
+        """
+        # Default to True for passive modules, False otherwise
+        return "passive" in self.flags
+
     @property
     def max_scope_distance(self):
         if self.in_scope_only or self.target_only:
@@ -834,11 +846,15 @@ def _event_precheck(self, event):
         if self.errored:
             return False, "module is in error state"
         # exclude non-watched types
-        if not any(t in self.get_watched_events() for t in ("*", event.type)):
+        watched_events = self.get_watched_events()
+        event_type_watched = any(t in watched_events for t in ("*", event.type))
+        # Check if module accepts seeds and event is a seed (only if event type is watched)
+        if self.accept_seeds and "seed" in event.tags and event_type_watched:
+            return True, "it is a seed event and module accepts seeds"
+        if not event_type_watched:
             return False, "its type is not in watched_events"
-        if self.target_only:
-            if "target" not in event.tags:
-                return False, "it did not meet target_only filter criteria"
+        if self.target_only and "target" not in event.tags:
+            return False, "it did not meet target_only filter criteria"
 
         # limit js URLs to modules that opt in to receive them
         if (not self.accept_url_special) and event.type.startswith("URL"):
@@ -913,6 +929,9 @@ async def _event_postcheck_inner(self, event):
         return True, ""
 
     def _scope_distance_check(self, event):
+        # Seeds bypass scope distance checks
+        if self.accept_seeds and "seed" in event.tags:
+            return True, "it is a seed event and module accepts seeds"
         if self.in_scope_only:
             if event.scope_distance > 0:
                 return False, "it did not meet in_scope_only filter criteria"

bbot/modules/github_org.py

@@ -90,7 +90,7 @@ async def handle_event(self, event):
             user = event.data
             self.verbose(f"Validating whether the organization {user} is within our scope...")
             is_org, in_scope = await self.validate_org(user)
-            if "target" in event.tags:
+            if "seed" in event.tags:
                 in_scope = True
             if not is_org or not in_scope:
                 self.verbose(f"Unable to validate that {user} is in-scope, skipping...")

bbot/modules/internal/dnsresolve.py

@@ -59,15 +59,15 @@ async def handle_event(self, event, **kwargs):
             non_minimal_rdtypes = self.non_minimal_rdtypes
 
         # first, we find or create the main DNS_NAME or IP_ADDRESS associated with this event
-        main_host_event, whitelisted, blacklisted, new_event = self.get_dns_parent(event)
+        main_host_event, in_target, blacklisted, new_event = self.get_dns_parent(event)
         original_tags = set(event.tags)
 
         # minimal resolution - first, we resolve A/AAAA records for scope purposes
         if new_event or event is main_host_event:
             await self.resolve_event(main_host_event, types=minimal_rdtypes)
-            # are any of its IPs whitelisted/blacklisted?
-            whitelisted, blacklisted = self.check_scope(main_host_event)
-            if whitelisted and event.scope_distance > 0:
+            # are any of its IPs in target scope or blacklisted?
+            in_target, blacklisted = self.check_scope(main_host_event)
+            if in_target and main_host_event.scope_distance > 0:
                 self.debug(f"Making {main_host_event} in-scope because it resolves to an in-scope resource (A/AAAA)")
                 main_host_event.scope_distance = 0
 
@@ -99,9 +99,11 @@ async def handle_event(self, event, **kwargs):
                                 )
 
         # if there weren't any DNS children and it's not an IP address, tag as unresolved
+        # Exception: don't convert seed events to DNS_NAME_UNRESOLVED so accept_seeds modules can process them
         if not main_host_event.raw_dns_records and not event_is_ip:
-            main_host_event.add_tag("unresolved")
-            main_host_event.type = "DNS_NAME_UNRESOLVED"
+            if "seed" not in main_host_event.tags:
+                main_host_event.add_tag("unresolved")
+                main_host_event.type = "DNS_NAME_UNRESOLVED"
 
         # main_host_event.add_tag(f"resolve-distance-{main_host_event.dns_resolve_distance}")
 
@@ -150,7 +152,7 @@ async def handle_wildcard_event(self, event):
             event.add_tag(f"{rdtype}-{wildcard_tag}")
 
         # wildcard event modification (www.evilcorp.com --> _wildcard.evilcorp.com)
-        if wildcard_rdtypes and "target" not in event.tags:
+        if wildcard_rdtypes and "seed" not in event.tags:
             # these are the rdtypes that have wildcards
             wildcard_rdtypes_set = set(wildcard_rdtypes)
             # consider the event a full wildcard if all its records are wildcards
@@ -219,7 +221,7 @@ async def emit_dns_children_raw(self, event, dns_tags):
                         )
 
     def check_scope(self, event):
-        whitelisted = False
+        in_target = False
         blacklisted = False
         dns_children = getattr(event, "dns_children", {})
         for rdtype in ("A", "AAAA", "CNAME"):
@@ -229,11 +231,11 @@ def check_scope(self, event):
             for host in hosts:
                 # having a CNAME to an in-scope host doesn't make you in-scope
                 if rdtype != "CNAME":
-                    if not whitelisted:
+                    if not in_target:
                         with suppress(ValidationError):
-                            if self.scan.whitelisted(host):
-                                whitelisted = True
-                                event.add_tag(f"dns-whitelisted-{rdtype}")
+                            if self.scan.in_target(host):
+                                in_target = True
+                                event.add_tag(f"dns-in-target-{rdtype}")
                 # but a CNAME to a blacklisted host means you're blacklisted
                 if not blacklisted:
                     with suppress(ValidationError):
@@ -242,8 +244,8 @@ def check_scope(self, event):
                             event.add_tag("blacklisted")
                             event.add_tag(f"dns-blacklisted-{rdtype}")
         if blacklisted:
-            whitelisted = False
-        return whitelisted, blacklisted
+            in_target = False
+        return in_target, blacklisted
 
     async def resolve_event(self, event, types):
         if not types:
@@ -287,16 +289,22 @@ async def resolve_event(self, event, types):
     def get_dns_parent(self, event):
         """
         Get the first parent DNS_NAME / IP_ADDRESS of an event. If one isn't found, create it.
+
+        Returns a 4-tuple of:
+        - the parent event
+        - whether the parent is in target
+        - whether the parent is blacklisted
+        - whether the parent is a new event, i.e. it is newly created or is the current event
         """
         for parent in event.get_parents(include_self=True):
             if parent.host == event.host and parent.type in ("IP_ADDRESS", "DNS_NAME", "DNS_NAME_UNRESOLVED"):
                 blacklisted = any(t.startswith("dns-blacklisted-") for t in parent.tags)
-                whitelisted = any(t.startswith("dns-whitelisted-") for t in parent.tags)
+                in_target = any(t.startswith("dns-in-target-") for t in parent.tags)
                 new_event = parent is event
-                return parent, whitelisted, blacklisted, new_event
+                return parent, in_target, blacklisted, new_event
         tags = set()
-        if "target" in event.tags:
-            tags.add("target")
+        if "seed" in event.tags:
+            tags.add("seed")
         return (
             self.scan.make_event(
                 event.host,

bbot/modules/internal/excavate.py

@@ -1163,8 +1163,8 @@ async def handle_event(self, event, **kwargs):
                 await self.emit_custom_parameters(event, "http_cookies", "COOKIE", "Custom Cookie")
                 await self.emit_custom_parameters(event, "http_headers", "HEADER", "Custom Header")
 
-                # if parameter extraction is enabled, and querystring removal is disabled, and the event is directly from the TARGET, create a WEB
-                if self.url_querystring_remove is False and str(event.parent.parent.module) == "TARGET":
+                # if parameter extraction is enabled, and querystring removal is disabled, and the event is directly from the SEED, create a WEB
+                if self.url_querystring_remove is False and str(event.parent.parent.module) == "SEED":
                     self.debug(f"Processing target URL [{urlunparse(event.parsed_url)}] for GET parameters")
                     for (
                         method,

bbot/modules/oauth.py

@@ -26,7 +26,7 @@ async def setup(self):
         return True
 
     async def filter_event(self, event):
-        if event.module == self or any(t in event.tags for t in ("target", "domain", "ms-auth-url")):
+        if event.module == self or any(t in event.tags for t in ("seed", "domain", "ms-auth-url")):
             return True
         elif self.try_all and event.scope_distance == 0:
             return True

bbot/modules/output/base.py

@@ -38,6 +38,9 @@ def _event_precheck(self, event):
         if self._is_graph_important(event):
             return True, "event is critical to the graph"
 
+        if event.always_emit:
+            return True, "event is always emitted"
+
         # omit certain event types
         if event._omit:
             if event.type in self.get_watched_events():

bbot/modules/output/mongo.py

@@ -1,4 +1,4 @@
-from motor.motor_asyncio import AsyncIOMotorClient
+from pymongo import AsyncMongoClient
 
 from bbot.models.pydantic import Event, Scan, Target
 from bbot.modules.output.base import BaseOutputModule
@@ -29,13 +29,13 @@ class Mongo(BaseOutputModule):
         "password": "The password to use to connect to the database",
         "collection_prefix": "Prefix the name of each collection with this string",
     }
-    deps_pip = ["motor~=3.6.0"]
+    deps_pip = ["pymongo~=4.15"]
 
     async def setup(self):
         self.uri = self.config.get("uri", "mongodb://localhost:27017")
         self.username = self.config.get("username", "")
         self.password = self.config.get("password", "")
-        self.db_client = AsyncIOMotorClient(self.uri, username=self.username, password=self.password)
+        self.db_client = AsyncMongoClient(self.uri, username=self.username, password=self.password)
 
         # Ping the server to confirm a successful connection
         try:

bbot/modules/templates/subdomain_enum.py

@@ -168,8 +168,8 @@ async def filter_event(self, event):
         is_cloud = False
         if any(t.startswith("cloud-") for t in event.tags):
             is_cloud = True
-        # reject if it's a cloud resource and not in our target
-        if is_cloud and event not in self.scan.target.whitelist:
+        # reject if it's a cloud resource and not in our target (unless it's a seed event)
+        if is_cloud and not self.scan.in_target(event) and "seed" not in event.tags:
             return False, "Event is a cloud resource and not a direct target"
         # optionally reject events with wildcards / errors
         if self.reject_wildcards:

bbot/scanner/manager.py

@@ -48,18 +48,21 @@ async def init_events(self, event_seeds=None):
             event_seeds = sorted(event_seeds, key=lambda e: (host_size_key(str(e.host)), e.data))
             # queue root scan event
             await self.queue_event(root_event, {})
-            target_module = self.scan._make_dummy_module(name="TARGET", _type="TARGET")
-            # queue each target in turn
+            target_module = self.scan._make_dummy_module(name="SEED", _type="SEED")
+            # queue each seed in turn
             for event_seed in event_seeds:
                 event = self.scan.make_event(
                     event_seed.data,
                     event_seed.type,
                     parent=root_event,
                     module=target_module,
                     context=f"Scan {self.scan.name} seeded with " + "{event.type}: {event.data}",
-                    tags=["target"],
+                    tags=["seed"],
                 )
-                self.verbose(f"Target: {event}")
+                # If the seed is also in the target scope, add the target tag
+                if self.scan.in_target(event):
+                    event.add_tag("target")
+                self.verbose(f"Seed: {event}")
                 # don't fill up the queue with too many events
                 while self.incoming_event_queue.qsize() > 100:
                     await asyncio.sleep(0.2)
@@ -113,9 +116,9 @@ async def handle_event(self, event, **kwargs):
 
         # Scope shepherding
         # here is where we make sure in-scope events are set to their proper scope distance
+
         if event.host:
-            event_whitelisted = self.scan.whitelisted(event)
-            if event_whitelisted:
+            if self.scan.in_target(event):
                 self.debug(f"Making {event} in-scope because its main host matches the scan target")
                 event.scope_distance = 0