diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py index 77774388e2..54b0be2a40 100644 --- a/bbot/core/event/base.py +++ b/bbot/core/event/base.py @@ -108,7 +108,8 @@ class BaseEvent: # Always emit this event type even if it's not in scope _always_emit = False # Always emit events with these tags even if they're not in scope - _always_emit_tags = ["affiliate", "target"] + + _always_emit_tags = ["affiliate", "seed"] # Bypass scope checking and dns resolution, distribute immediately to modules # This is useful for "end-of-line" events like FINDING and VULNERABILITY _quick_emit = False @@ -596,6 +597,9 @@ def parent(self, parent): new_scope_distance += 1 self.scope_distance = new_scope_distance # inherit certain tags + # inherit seed tag from DNS_NAME_UNRESOLVED -> DNS_NAME only + if "seed" in parent.tags and parent.type == "DNS_NAME_UNRESOLVED" and self.type == "DNS_NAME": + self.add_tag("seed") if hosts_are_same: # inherit web spider distance from parent self.web_spider_distance = getattr(parent, "web_spider_distance", 0) @@ -1214,6 +1218,9 @@ def _words(self): class OPEN_TCP_PORT(BaseEvent): + # we generally don't care about open ports on affiliates + _always_emit_tags = ["seed"] + def sanitize_data(self, data): return validators.validate_open_port(data) @@ -1719,7 +1726,7 @@ def __init__(self, *args, **kwargs): class RAW_DNS_RECORD(DictHostEvent, DnsEvent): # don't emit raw DNS records for affiliates - _always_emit_tags = ["target"] + _always_emit_tags = ["seed"] class MOBILE_APP(DictEvent): diff --git a/bbot/core/helpers/web/client.py b/bbot/core/helpers/web/client.py index b76e6058ee..2d352b2f33 100644 --- a/bbot/core/helpers/web/client.py +++ b/bbot/core/helpers/web/client.py @@ -90,9 +90,9 @@ def build_request(self, *args, **kwargs): kwargs["url"] = url url = kwargs["url"] - target_in_scope = self._target.in_scope(str(url)) + in_target = self._target.in_target(str(url)) - if target_in_scope: + if in_target: if not kwargs.get("cookies", None): kwargs["cookies"] = {} for ck, cv in self._web_config.get("http_cookies", {}).items(): @@ -101,7 +101,7 @@ def build_request(self, *args, **kwargs): request = super().build_request(**kwargs) - if target_in_scope: + if in_target: for hk, hv in self._web_config.get("http_headers", {}).items(): hv = str(hv) # don't clobber headers diff --git a/bbot/models/pydantic.py b/bbot/models/pydantic.py index 68e71d493e..5b7990056c 100644 --- a/bbot/models/pydantic.py +++ b/bbot/models/pydantic.py @@ -148,11 +148,11 @@ def from_scan(cls, scan): class Target(BBOTBaseModel): name: str = "Default Target" strict_dns_scope: bool = False - seeds: List = [] - whitelist: Optional[List] = None + target: List = [] + seeds: Optional[List] = None blacklist: List = [] hash: Annotated[str, "indexed", "unique"] scope_hash: Annotated[str, "indexed"] seed_hash: Annotated[str, "indexed"] - whitelist_hash: Annotated[str, "indexed"] + target_hash: Annotated[str, "indexed"] blacklist_hash: Annotated[str, "indexed"] diff --git a/bbot/models/sql.py b/bbot/models/sql.py index d58034ccf6..1e15c8c073 100644 --- a/bbot/models/sql.py +++ b/bbot/models/sql.py @@ -125,11 +125,11 @@ class Scan(BBOTBaseModel, table=True): class Target(BBOTBaseModel, table=True): name: str = "Default Target" strict_dns_scope: bool = False - seeds: List = Field(default=[], sa_type=JSON) - whitelist: Optional[List] = Field(default=None, sa_type=JSON) + target: List = Field(default=[], sa_type=JSON) + seeds: Optional[List] = Field(default=None, sa_type=JSON) blacklist: List = Field(default=[], sa_type=JSON) hash: str = Field(sa_column=Column("hash", String(length=255), unique=True, primary_key=True, index=True)) scope_hash: str = Field(sa_column=Column("scope_hash", String(length=255), index=True)) - seed_hash: str = Field(sa_column=Column("seed_hashhash", String(length=255), index=True)) - whitelist_hash: str = Field(sa_column=Column("whitelist_hash", String(length=255), index=True)) + seed_hash: str = Field(sa_column=Column("seed_hash", String(length=255), index=True)) + target_hash: str = Field(sa_column=Column("target_hash", String(length=255), index=True)) blacklist_hash: str = Field(sa_column=Column("blacklist_hash", String(length=255), index=True)) diff --git a/bbot/modules/base.py b/bbot/modules/base.py index ebb49a39d4..7f2df35800 100644 --- a/bbot/modules/base.py +++ b/bbot/modules/base.py @@ -52,6 +52,9 @@ class BaseModule: target_only (bool): Accept only the initial target event(s). Default is False. + accept_seeds (bool): Accept seed events (events from initial scan seeds). + Defaults to True for passive modules, False otherwise. Can be explicitly set to override the default. + in_scope_only (bool): Accept only explicitly in-scope events, regardless of the scan's search distance. Default is False. accept_url_special (bool): Accept "special" URLs not typically distributed to web modules, e.g. JS URLs. Default is False. @@ -791,6 +794,15 @@ async def _worker(self): self.error(traceback.format_exc()) self.log.trace("Worker stopped") + @property + def accept_seeds(self): + """ + Returns whether the module accepts seed events. + Defaults to True for passive modules, False otherwise. + """ + # Default to True for passive modules, False otherwise + return "passive" in self.flags + @property def max_scope_distance(self): if self.in_scope_only or self.target_only: @@ -834,11 +846,15 @@ def _event_precheck(self, event): if self.errored: return False, "module is in error state" # exclude non-watched types - if not any(t in self.get_watched_events() for t in ("*", event.type)): + watched_events = self.get_watched_events() + event_type_watched = any(t in watched_events for t in ("*", event.type)) + # Check if module accepts seeds and event is a seed (only if event type is watched) + if self.accept_seeds and "seed" in event.tags and event_type_watched: + return True, "it is a seed event and module accepts seeds" + if not event_type_watched: return False, "its type is not in watched_events" - if self.target_only: - if "target" not in event.tags: - return False, "it did not meet target_only filter criteria" + if self.target_only and "target" not in event.tags: + return False, "it did not meet target_only filter criteria" # limit js URLs to modules that opt in to receive them if (not self.accept_url_special) and event.type.startswith("URL"): @@ -913,6 +929,9 @@ async def _event_postcheck_inner(self, event): return True, "" def _scope_distance_check(self, event): + # Seeds bypass scope distance checks + if self.accept_seeds and "seed" in event.tags: + return True, "it is a seed event and module accepts seeds" if self.in_scope_only: if event.scope_distance > 0: return False, "it did not meet in_scope_only filter criteria" diff --git a/bbot/modules/github_org.py b/bbot/modules/github_org.py index 46b8b1935a..ff7cba4d42 100644 --- a/bbot/modules/github_org.py +++ b/bbot/modules/github_org.py @@ -90,7 +90,7 @@ async def handle_event(self, event): user = event.data self.verbose(f"Validating whether the organization {user} is within our scope...") is_org, in_scope = await self.validate_org(user) - if "target" in event.tags: + if "seed" in event.tags: in_scope = True if not is_org or not in_scope: self.verbose(f"Unable to validate that {user} is in-scope, skipping...") diff --git a/bbot/modules/internal/dnsresolve.py b/bbot/modules/internal/dnsresolve.py index 3dddd289a4..63644ac5cc 100644 --- a/bbot/modules/internal/dnsresolve.py +++ b/bbot/modules/internal/dnsresolve.py @@ -59,15 +59,15 @@ async def handle_event(self, event, **kwargs): non_minimal_rdtypes = self.non_minimal_rdtypes # first, we find or create the main DNS_NAME or IP_ADDRESS associated with this event - main_host_event, whitelisted, blacklisted, new_event = self.get_dns_parent(event) + main_host_event, in_target, blacklisted, new_event = self.get_dns_parent(event) original_tags = set(event.tags) # minimal resolution - first, we resolve A/AAAA records for scope purposes if new_event or event is main_host_event: await self.resolve_event(main_host_event, types=minimal_rdtypes) - # are any of its IPs whitelisted/blacklisted? - whitelisted, blacklisted = self.check_scope(main_host_event) - if whitelisted and event.scope_distance > 0: + # are any of its IPs in target scope or blacklisted? + in_target, blacklisted = self.check_scope(main_host_event) + if in_target and main_host_event.scope_distance > 0: self.debug(f"Making {main_host_event} in-scope because it resolves to an in-scope resource (A/AAAA)") main_host_event.scope_distance = 0 @@ -99,9 +99,11 @@ async def handle_event(self, event, **kwargs): ) # if there weren't any DNS children and it's not an IP address, tag as unresolved + # Exception: don't convert seed events to DNS_NAME_UNRESOLVED so accept_seeds modules can process them if not main_host_event.raw_dns_records and not event_is_ip: - main_host_event.add_tag("unresolved") - main_host_event.type = "DNS_NAME_UNRESOLVED" + if "seed" not in main_host_event.tags: + main_host_event.add_tag("unresolved") + main_host_event.type = "DNS_NAME_UNRESOLVED" # main_host_event.add_tag(f"resolve-distance-{main_host_event.dns_resolve_distance}") @@ -150,7 +152,7 @@ async def handle_wildcard_event(self, event): event.add_tag(f"{rdtype}-{wildcard_tag}") # wildcard event modification (www.evilcorp.com --> _wildcard.evilcorp.com) - if wildcard_rdtypes and "target" not in event.tags: + if wildcard_rdtypes and "seed" not in event.tags: # these are the rdtypes that have wildcards wildcard_rdtypes_set = set(wildcard_rdtypes) # consider the event a full wildcard if all its records are wildcards @@ -219,7 +221,7 @@ async def emit_dns_children_raw(self, event, dns_tags): ) def check_scope(self, event): - whitelisted = False + in_target = False blacklisted = False dns_children = getattr(event, "dns_children", {}) for rdtype in ("A", "AAAA", "CNAME"): @@ -229,11 +231,11 @@ def check_scope(self, event): for host in hosts: # having a CNAME to an in-scope host doesn't make you in-scope if rdtype != "CNAME": - if not whitelisted: + if not in_target: with suppress(ValidationError): - if self.scan.whitelisted(host): - whitelisted = True - event.add_tag(f"dns-whitelisted-{rdtype}") + if self.scan.in_target(host): + in_target = True + event.add_tag(f"dns-in-target-{rdtype}") # but a CNAME to a blacklisted host means you're blacklisted if not blacklisted: with suppress(ValidationError): @@ -242,8 +244,8 @@ def check_scope(self, event): event.add_tag("blacklisted") event.add_tag(f"dns-blacklisted-{rdtype}") if blacklisted: - whitelisted = False - return whitelisted, blacklisted + in_target = False + return in_target, blacklisted async def resolve_event(self, event, types): if not types: @@ -287,16 +289,22 @@ async def resolve_event(self, event, types): def get_dns_parent(self, event): """ Get the first parent DNS_NAME / IP_ADDRESS of an event. If one isn't found, create it. + + Returns a 4-tuple of: + - the parent event + - whether the parent is in target + - whether the parent is blacklisted + - whether the parent is a new event, i.e. it is newly created or is the current event """ for parent in event.get_parents(include_self=True): if parent.host == event.host and parent.type in ("IP_ADDRESS", "DNS_NAME", "DNS_NAME_UNRESOLVED"): blacklisted = any(t.startswith("dns-blacklisted-") for t in parent.tags) - whitelisted = any(t.startswith("dns-whitelisted-") for t in parent.tags) + in_target = any(t.startswith("dns-in-target-") for t in parent.tags) new_event = parent is event - return parent, whitelisted, blacklisted, new_event + return parent, in_target, blacklisted, new_event tags = set() - if "target" in event.tags: - tags.add("target") + if "seed" in event.tags: + tags.add("seed") return ( self.scan.make_event( event.host, diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py index 9c7a9fbbfb..a64794ea46 100644 --- a/bbot/modules/internal/excavate.py +++ b/bbot/modules/internal/excavate.py @@ -1163,8 +1163,8 @@ async def handle_event(self, event, **kwargs): await self.emit_custom_parameters(event, "http_cookies", "COOKIE", "Custom Cookie") await self.emit_custom_parameters(event, "http_headers", "HEADER", "Custom Header") - # if parameter extraction is enabled, and querystring removal is disabled, and the event is directly from the TARGET, create a WEB - if self.url_querystring_remove is False and str(event.parent.parent.module) == "TARGET": + # if parameter extraction is enabled, and querystring removal is disabled, and the event is directly from the SEED, create a WEB + if self.url_querystring_remove is False and str(event.parent.parent.module) == "SEED": self.debug(f"Processing target URL [{urlunparse(event.parsed_url)}] for GET parameters") for ( method, diff --git a/bbot/modules/oauth.py b/bbot/modules/oauth.py index dba3b52579..922cbc0558 100644 --- a/bbot/modules/oauth.py +++ b/bbot/modules/oauth.py @@ -26,7 +26,7 @@ async def setup(self): return True async def filter_event(self, event): - if event.module == self or any(t in event.tags for t in ("target", "domain", "ms-auth-url")): + if event.module == self or any(t in event.tags for t in ("seed", "domain", "ms-auth-url")): return True elif self.try_all and event.scope_distance == 0: return True diff --git a/bbot/modules/output/base.py b/bbot/modules/output/base.py index 5aa17d24c1..d8d7bdb79f 100644 --- a/bbot/modules/output/base.py +++ b/bbot/modules/output/base.py @@ -38,6 +38,9 @@ def _event_precheck(self, event): if self._is_graph_important(event): return True, "event is critical to the graph" + if event.always_emit: + return True, "event is always emitted" + # omit certain event types if event._omit: if event.type in self.get_watched_events(): diff --git a/bbot/modules/output/mongo.py b/bbot/modules/output/mongo.py index f90c4aad53..b7deb3d5e4 100644 --- a/bbot/modules/output/mongo.py +++ b/bbot/modules/output/mongo.py @@ -1,4 +1,4 @@ -from motor.motor_asyncio import AsyncIOMotorClient +from pymongo import AsyncMongoClient from bbot.models.pydantic import Event, Scan, Target from bbot.modules.output.base import BaseOutputModule @@ -29,13 +29,13 @@ class Mongo(BaseOutputModule): "password": "The password to use to connect to the database", "collection_prefix": "Prefix the name of each collection with this string", } - deps_pip = ["motor~=3.6.0"] + deps_pip = ["pymongo~=4.15"] async def setup(self): self.uri = self.config.get("uri", "mongodb://localhost:27017") self.username = self.config.get("username", "") self.password = self.config.get("password", "") - self.db_client = AsyncIOMotorClient(self.uri, username=self.username, password=self.password) + self.db_client = AsyncMongoClient(self.uri, username=self.username, password=self.password) # Ping the server to confirm a successful connection try: diff --git a/bbot/modules/templates/subdomain_enum.py b/bbot/modules/templates/subdomain_enum.py index a65d08f315..49ae38b14c 100644 --- a/bbot/modules/templates/subdomain_enum.py +++ b/bbot/modules/templates/subdomain_enum.py @@ -168,8 +168,8 @@ async def filter_event(self, event): is_cloud = False if any(t.startswith("cloud-") for t in event.tags): is_cloud = True - # reject if it's a cloud resource and not in our target - if is_cloud and event not in self.scan.target.whitelist: + # reject if it's a cloud resource and not in our target (unless it's a seed event) + if is_cloud and not self.scan.in_target(event) and "seed" not in event.tags: return False, "Event is a cloud resource and not a direct target" # optionally reject events with wildcards / errors if self.reject_wildcards: diff --git a/bbot/scanner/manager.py b/bbot/scanner/manager.py index 4d81c85491..f38d73a696 100644 --- a/bbot/scanner/manager.py +++ b/bbot/scanner/manager.py @@ -48,8 +48,8 @@ async def init_events(self, event_seeds=None): event_seeds = sorted(event_seeds, key=lambda e: (host_size_key(str(e.host)), e.data)) # queue root scan event await self.queue_event(root_event, {}) - target_module = self.scan._make_dummy_module(name="TARGET", _type="TARGET") - # queue each target in turn + target_module = self.scan._make_dummy_module(name="SEED", _type="SEED") + # queue each seed in turn for event_seed in event_seeds: event = self.scan.make_event( event_seed.data, @@ -57,9 +57,12 @@ async def init_events(self, event_seeds=None): parent=root_event, module=target_module, context=f"Scan {self.scan.name} seeded with " + "{event.type}: {event.data}", - tags=["target"], + tags=["seed"], ) - self.verbose(f"Target: {event}") + # If the seed is also in the target scope, add the target tag + if self.scan.in_target(event): + event.add_tag("target") + self.verbose(f"Seed: {event}") # don't fill up the queue with too many events while self.incoming_event_queue.qsize() > 100: await asyncio.sleep(0.2) @@ -113,9 +116,9 @@ async def handle_event(self, event, **kwargs): # Scope shepherding # here is where we make sure in-scope events are set to their proper scope distance + if event.host: - event_whitelisted = self.scan.whitelisted(event) - if event_whitelisted: + if self.scan.in_target(event): self.debug(f"Making {event} in-scope because its main host matches the scan target") event.scope_distance = 0 diff --git a/bbot/scanner/preset/args.py b/bbot/scanner/preset/args.py index 137cd26d43..ab447006f7 100644 --- a/bbot/scanner/preset/args.py +++ b/bbot/scanner/preset/args.py @@ -105,9 +105,12 @@ def parsed(self): def preset_from_args(self): # the order here is important # first we make the preset + # -t/--targets becomes target (defines target, what in_target() checks) + # -s/--seeds becomes seeds (drives passive modules), defaults to targets if not specified + seeds = self.parsed.seeds if self.parsed.seeds is not None else self.parsed.targets args_preset = self.preset.__class__( - *self.parsed.targets, - whitelist=self.parsed.whitelist, + *(self.parsed.targets or []), + seeds=seeds if seeds else None, blacklist=self.parsed.blacklist, name="args_preset", ) @@ -225,21 +228,19 @@ def create_parser(self, *args, **kwargs): p = argparse.ArgumentParser(*args, **kwargs) target = p.add_argument_group(title="Target") + target.add_argument("-t", "--targets", nargs="+", default=[], help="Target scope", metavar="TARGET") target.add_argument( - "-t", "--targets", nargs="+", default=[], help="Targets to seed the scan", metavar="TARGET" - ) - target.add_argument( - "-w", - "--whitelist", + "-s", + "--seeds", nargs="+", default=None, - help="What's considered in-scope (by default it's the same as --targets)", + help="Define seeds to drive passive modules without being in scope (if not specified, defaults to same as targets)", ) target.add_argument("-b", "--blacklist", nargs="+", default=[], help="Don't touch these things") target.add_argument( "--strict-scope", action="store_true", - help="Don't consider subdomains of target/whitelist to be in-scope - exact matches only", + help="Don't consider subdomains of target to be in-scope - exact matches only", ) presets = p.add_argument_group(title="Presets") presets.add_argument( @@ -307,7 +308,7 @@ def create_parser(self, *args, **kwargs): scan.add_argument("-n", "--name", help="Name of scan (default: random)", metavar="SCAN_NAME") scan.add_argument("-v", "--verbose", action="store_true", help="Be more verbose") scan.add_argument("-d", "--debug", action="store_true", help="Enable debugging") - scan.add_argument("-s", "--silent", action="store_true", help="Be quiet") + scan.add_argument("-S", "--silent", action="store_true", help="Be quiet") scan.add_argument( "--force", action="store_true", @@ -411,9 +412,9 @@ def sanitize_args(self): self.parsed.targets = chain_lists( self.parsed.targets, try_files=True, msg="Reading targets from file: {filename}" ) - if self.parsed.whitelist is not None: - self.parsed.whitelist = chain_lists( - self.parsed.whitelist, try_files=True, msg="Reading whitelist from file: {filename}" + if self.parsed.seeds is not None: + self.parsed.seeds = chain_lists( + self.parsed.seeds, try_files=True, msg="Reading seeds from file: {filename}" ) self.parsed.blacklist = chain_lists( self.parsed.blacklist, try_files=True, msg="Reading blacklist from file: {filename}" diff --git a/bbot/scanner/preset/preset.py b/bbot/scanner/preset/preset.py index 623dbcec95..a700b6b372 100644 --- a/bbot/scanner/preset/preset.py +++ b/bbot/scanner/preset/preset.py @@ -76,8 +76,8 @@ class Preset(metaclass=BasePreset): Based on the state of the preset, you can print a warning message, abort the scan, enable/disable modules, etc.. Attributes: - target (Target): Target(s) of scan. - whitelist (Target): Scan whitelist (by default this is the same as `target`). + target (BBOTTarget): The scan target object containing seeds, target, and blacklist. + Use `target.target` to access what's in the target (what `in_target()` checks). blacklist (Target): Scan blacklist (this takes ultimate precedence). helpers (ConfigAwareHelper): Helper containing various reusable functions, regexes, etc. output_dir (pathlib.Path): Output directory for scan. @@ -116,7 +116,7 @@ class Preset(metaclass=BasePreset): def __init__( self, *target, - whitelist=None, + seeds=None, blacklist=None, modules=None, output_modules=None, @@ -142,8 +142,11 @@ def __init__( Initializes the Preset class. Args: - *target (str): Target(s) to scan. Types supported: hostnames, IPs, CIDRs, emails, open ports. - whitelist (list, optional): Whitelisted target(s) to scan. Defaults to the same as `targets`. + *target (str): Target(s) to scan. These ALWAYS become the target (what `in_target()` checks). + Types supported: hostnames, IPs, CIDRs, emails, open ports. + Note: Positional arguments always mean target, never seeds. + seeds (list, optional): Explicitly define seeds (initial events for passive modules). + If not specified, seeds will be backfilled from target when target is defined. blacklist (list, optional): Blacklisted target(s). Takes ultimate precedence. Defaults to empty. modules (list[str], optional): List of scan modules to enable for the scan. Defaults to empty list. output_modules (list[str], optional): List of output modules to use. Defaults to csv, human, and json. @@ -260,12 +263,17 @@ def __init__( self._module_dirs = set() self.module_dirs = module_dirs - # target / whitelist / blacklist + # target / seeds / blacklist # these are temporary receptacles until they all get .baked() together - self._seeds = set(target if target else []) - self._whitelist = set(whitelist) if whitelist else whitelist + self._target_list = set(target or []) self._blacklist = set(blacklist if blacklist else []) + # seeds are special. Instead of initializing them as an empty set, we use "None" + # to signify they haven't been explicitly set. + # after all the merging is done, if seeds are still untouched by the user + # (i.e. they are still None), we'll know it's okay to copy them from the targets. + self._seeds = set(seeds) if seeds else None + # _target doesn't get set until .bake() self._target = None # we don't fill self.modules yet (that happens in .bake()) @@ -292,12 +300,6 @@ def seeds(self): raise ValueError("Cannot access target before preset is baked (use ._seeds instead)") return self.target.seeds - @property - def whitelist(self): - if self._target is None: - raise ValueError("Cannot access whitelist before preset is baked (use ._whitelist instead)") - return self.target.whitelist - @property def blacklist(self): if self._target is None: @@ -364,13 +366,12 @@ def merge(self, other): self.flags.update(other.flags) # target / scope - self._seeds.update(other._seeds) - # leave whitelist as None until we encounter one - if other._whitelist is not None: - if self._whitelist is None: - self._whitelist = set(other._whitelist) + self._target_list.update(other._target_list) + if other._seeds is not None: + if self._seeds is None: + self._seeds = set(other._seeds) else: - self._whitelist.update(other._whitelist) + self._seeds.update(other._seeds) self._blacklist.update(other._blacklist) # module dirs @@ -485,8 +486,8 @@ def bake(self, scan=None): from bbot.scanner.target import BBOTTarget baked_preset._target = BBOTTarget( - *list(self._seeds), - whitelist=self._whitelist, + seeds=list(self._seeds) if self._seeds else None, + target=list(self._target_list), blacklist=self._blacklist, strict_dns_scope=self.strict_scope, ) @@ -637,8 +638,8 @@ def in_scope(self, host): def blacklisted(self, host): return self.target.blacklisted(host) - def whitelisted(self, host): - return self.target.whitelisted(host) + def in_target(self, host): + return self.target.in_target(host) @classmethod def from_dict(cls, preset_dict, name=None, _exclude=None, _log=False): @@ -657,12 +658,14 @@ def from_dict(cls, preset_dict, name=None, _exclude=None, _log=False): Examples: >>> preset = Preset.from_dict({"target": ["evilcorp.com"], "modules": ["portscan"]}) """ - # tolerate both "target" and "targets", since this is a common oopsie - targets = preset_dict.get("target", []) - targets += preset_dict.get("targets", []) + # Handle seeds and targets from dict + # for user-friendliness, we allow both "target" and "targets" to be used. we merge them into a single list. + target_vals = (preset_dict.get("target") or []) + (preset_dict.get("targets") or []) + targets = list(dict.fromkeys(target_vals)) + seeds = preset_dict.get("seeds") new_preset = cls( *targets, - whitelist=preset_dict.get("whitelist"), + seeds=seeds, blacklist=preset_dict.get("blacklist"), modules=preset_dict.get("modules"), output_modules=preset_dict.get("output_modules"), @@ -762,7 +765,7 @@ def to_dict(self, include_target=False, full_config=False, redact_secrets=False) Convert this preset into a Python dictionary. Args: - include_target (bool, optional): If True, include target, whitelist, and blacklist in the dictionary + include_target (bool, optional): If True, include seeds, target, and blacklist in the dictionary full_config (bool, optional): If True, include the entire config, not just what's changed from the defaults. Returns: @@ -791,15 +794,15 @@ def to_dict(self, include_target=False, full_config=False, redact_secrets=False) # scope if include_target: - target = sorted(self.target.seeds.inputs) - whitelist = [] - if self.target.whitelist is not None: - whitelist = sorted(self.target.whitelist.inputs) + target = sorted(self.target.target.inputs) + seeds = [] + if self.target.seeds is not None: + seeds = sorted(self.target.seeds.inputs) blacklist = sorted(self.target.blacklist.inputs) if target: preset_dict["target"] = target - if whitelist and whitelist != target: - preset_dict["whitelist"] = whitelist + if seeds and seeds != target: + preset_dict["seeds"] = seeds if blacklist: preset_dict["blacklist"] = blacklist @@ -842,7 +845,7 @@ def to_yaml(self, include_target=False, full_config=False, sort_keys=False): Return the preset in the form of a YAML string. Args: - include_target (bool, optional): If True, include target, whitelist, and blacklist in the dictionary + include_target (bool, optional): If True, include seeds, target, and blacklist in the dictionary full_config (bool, optional): If True, include the entire config, not just what's changed from the defaults. sort_keys (bool, optional): If True, sort YAML keys alphabetically diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py index 45a57ddf24..ae70bde7d1 100644 --- a/bbot/scanner/scanner.py +++ b/bbot/scanner/scanner.py @@ -76,7 +76,7 @@ class Scanner: target (Target): Target of scan (alias to `self.preset.target`). preset (Preset): The main scan Preset in its baked form. config (omegaconf.dictconfig.DictConfig): BBOT config (alias to `self.preset.config`). - whitelist (Target): Scan whitelist (by default this is the same as `target`) (alias to `self.preset.whitelist`). + seeds (Target): Scan seeds (by default this is the same as `target`) (alias to `self.preset.seeds`). blacklist (Target): Scan blacklist (this takes ultimate precedence) (alias to `self.preset.blacklist`). helpers (ConfigAwareHelper): Helper containing various reusable functions, regexes, etc. (alias to `self.preset.helpers`). output_dir (pathlib.Path): Output directory for scan (alias to `self.preset.output_dir`). @@ -283,10 +283,10 @@ async def _prep(self): f.write(self.preset.to_yaml()) # log scan overview - start_msg = f"Scan seeded with {len(self.seeds):,} targets" + start_msg = f"Scan seeded with {len(self.seeds):,} seed(s)" details = [] - if self.whitelist != self.target: - details.append(f"{len(self.whitelist):,} in whitelist") + if self.target.target: + details.append(f"{len(self.target.target):,} in target") if self.blacklist: details.append(f"{len(self.blacklist):,} in blacklist") if details: @@ -910,8 +910,8 @@ async def _cleanup(self): def in_scope(self, *args, **kwargs): return self.preset.in_scope(*args, **kwargs) - def whitelisted(self, *args, **kwargs): - return self.preset.whitelisted(*args, **kwargs) + def in_target(self, *args, **kwargs): + return self.preset.in_target(*args, **kwargs) def blacklisted(self, *args, **kwargs): return self.preset.blacklisted(*args, **kwargs) @@ -932,10 +932,6 @@ def target(self): def seeds(self): return self.preset.seeds - @property - def whitelist(self): - return self.preset.whitelist - @property def blacklist(self): return self.preset.blacklist @@ -1020,8 +1016,8 @@ def root_event(self): "tags": [ "distance-0" ], - "module": "TARGET", - "module_sequence": "TARGET" + "module": "SEED", + "module_sequence": "SEED" } ``` """ @@ -1048,7 +1044,7 @@ def make_root_event(self, context): root_event.scope_distance = 0 root_event.parent = root_event root_event._dummy = False - root_event.module = self._make_dummy_module(name="TARGET", _type="TARGET") + root_event.module = self._make_dummy_module(name="SEED", _type="SEED") return root_event @property @@ -1057,13 +1053,13 @@ def dns_strings(self): A list of DNS hostname strings generated from the scan target """ if self._dns_strings is None: - dns_whitelist = {t.host for t in self.whitelist if t.host and isinstance(t.host, str)} - dns_whitelist = sorted(dns_whitelist, key=len) - dns_whitelist_set = set() + dns_target = {t.host for t in self.target.target if t.host and isinstance(t.host, str)} + dns_target = sorted(dns_target, key=len) + dns_target_set = set() dns_strings = [] - for t in dns_whitelist: - if not any(x in dns_whitelist_set for x in self.helpers.domain_parents(t, include_self=True)): - dns_whitelist_set.add(t) + for t in dns_target: + if not any(x in dns_target_set for x in self.helpers.domain_parents(t, include_self=True)): + dns_target_set.add(t) dns_strings.append(t) self._dns_strings = dns_strings return self._dns_strings @@ -1162,7 +1158,7 @@ async def extract_in_scope_hostnames(self, s): @property def json(self): """ - A dictionary representation of the scan including its name, ID, targets, whitelist, blacklist, and modules + A dictionary representation of the scan including its name, ID, targets, target, blacklist, and modules """ j = {} for i in ("id", "name"): diff --git a/bbot/scanner/stats.py b/bbot/scanner/stats.py index 38d95032f7..71547ddab9 100644 --- a/bbot/scanner/stats.py +++ b/bbot/scanner/stats.py @@ -72,7 +72,7 @@ def table(self): header = ["Module", "Produced", "Consumed"] table = [] for mname, mstat in self.module_stats.items(): - if mname == "TARGET" or mstat.module._stats_exclude: + if mname == "SEED" or mstat.module._stats_exclude: continue table_row = [] table_row.append(mname) diff --git a/bbot/scanner/target.py b/bbot/scanner/target.py index f0aa1315c3..6346728d45 100644 --- a/bbot/scanner/target.py +++ b/bbot/scanner/target.py @@ -20,7 +20,7 @@ class BaseTarget(RadixTarget): while allowing lightning fast scope lookups. This class is inherited by all three components of the BBOT target: - - Whitelist + - Target - Blacklist - Seeds """ @@ -92,8 +92,27 @@ def add(self, targets, data=None): event_seeds = sorted(event_seeds, key=lambda e: ((0, 0) if not e.host else host_size_key(e.host))) for event_seed in event_seeds: self.event_seeds.add(event_seed) + # Some event seeds (e.g. ORG_STUB, USERNAME, BLACKLIST_REGEX) are not host-based and have + # host == None. These are still useful as parsed target entries, but cannot always be + # represented in the underlying RadixTarget tree, which expects a concrete host. + # Subclasses like ScanBlacklist may still need to see these entries (for regex handling, + # etc.), so we always call self._add() and let the subclass decide whether to forward to + # the radix layer. self._add(event_seed.host, data=(event_seed if data is None else data)) + def _add(self, host, data): + """ + Wrapper around RadixTarget._add(). + + The radix tree cannot handle host == None, but some subclasses (e.g. ScanBlacklist) + need to receive non-host-based entries such as BLACKLIST_REGEX. BaseTarget.add() + always calls self._add(); this default implementation safely ignores hostless + entries while still delegating normal hosts to the underlying RadixTarget. + """ + if host is None: + return + super()._add(host, data) + def __iter__(self): yield from self.event_seeds @@ -102,7 +121,8 @@ class ScanSeeds(BaseTarget): """ Initial events used to seed a scan. - These are the targets specified by the user, e.g. via `-t` on the CLI. + These are the seeds specified by the user, e.g. via `-s` on the CLI. + If no seeds were specified, the targets (`-t`) are copied here. """ def get(self, event, single=True, **kwargs): @@ -140,9 +160,9 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) -class ScanWhitelist(ACLTarget): +class ScanTarget(ACLTarget): """ - A collection of BBOT events that represent a scan's whitelist. + A collection of BBOT events that represent a scan's targets. """ pass @@ -210,49 +230,55 @@ class BBOTTarget: """ A convenient abstraction of a scan target that contains three subtargets: - seeds - - whitelist + - target - blacklist - Provides high-level functions like in_scope(), which includes both whitelist and blacklist checks. + Provides high-level functions like in_scope(), which includes both target and blacklist checks. """ - def __init__(self, *seeds, whitelist=None, blacklist=None, strict_dns_scope=False): + def __init__(self, seeds=None, target=None, blacklist=None, strict_dns_scope=False): self.strict_dns_scope = strict_dns_scope - self.seeds = ScanSeeds(*seeds, strict_dns_scope=strict_dns_scope) - self._orig_whitelist = whitelist - if whitelist is None: - whitelist = self.seeds.hosts - self.whitelist = ScanWhitelist(*whitelist, strict_dns_scope=strict_dns_scope) - if blacklist is None: - blacklist = [] - self.blacklist = ScanBlacklist(*blacklist) + self._orig_seeds = seeds + + target_list = list(target) if target else [] + self.target = ScanTarget(*target_list, strict_dns_scope=strict_dns_scope) + + # Seeds are only copied from target if target is defined but seeds are NOT defined + # Use target.inputs (original inputs) to preserve all inputs, including subdomains + if seeds is None: + seeds = self.target.inputs + self.seeds = ScanSeeds(*list(seeds), strict_dns_scope=strict_dns_scope) + + blacklist_list = list(blacklist) if blacklist else [] + self.blacklist = ScanBlacklist(*blacklist_list) @property def json(self): - return { - "seeds": sorted(self.seeds.inputs), - "whitelist": (None if not self._orig_whitelist else sorted(self.whitelist.inputs)), + j = { + "target": sorted(self.target.inputs), "blacklist": sorted(self.blacklist.inputs), "strict_dns_scope": self.strict_dns_scope, "hash": self.hash.hex(), "seed_hash": self.seeds.hash.hex(), - "whitelist_hash": self.whitelist.hash.hex(), + "target_hash": self.target.hash.hex(), "blacklist_hash": self.blacklist.hash.hex(), "scope_hash": self.scope_hash.hex(), } + if self._orig_seeds is not None: + j["seeds"] = sorted(self.seeds.inputs) + return j @property def hash(self): sha1_hash = sha1() - for target_hash in [t.hash for t in (self.seeds, self.whitelist, self.blacklist)]: + for target_hash in [t.hash for t in (self.seeds, self.target, self.blacklist)]: sha1_hash.update(target_hash) return sha1_hash.digest() @property def scope_hash(self): sha1_hash = sha1() - # Consider only the hash values of the whitelist and blacklist - for target_hash in [t.hash for t in (self.whitelist, self.blacklist)]: + for target_hash in [t.hash for t in (self.target, self.blacklist)]: sha1_hash.update(target_hash) return sha1_hash.digest() @@ -261,8 +287,12 @@ def in_scope(self, host): Check whether a hostname, url, IP, etc. is in scope. Accepts either events or string data. - Checks whitelist and blacklist. - If `host` is an event and its scope distance is zero, it will automatically be considered in-scope. + This method checks both target AND blacklist. + A host is in-scope if it is in the target AND not blacklisted. + + Note: This is different from `in_target()` which only checks the target. + - `in_target()`: checks if host is in the target + - `in_scope()`: checks if host is in the target AND not blacklisted Examples: Check if a URL is in scope: @@ -270,8 +300,9 @@ def in_scope(self, host): True """ blacklisted = self.blacklisted(host) - whitelisted = self.whitelisted(host) - return whitelisted and not blacklisted + if blacklisted: + return False + return self.in_target(host) def blacklisted(self, host): """ @@ -289,21 +320,24 @@ def blacklisted(self, host): """ return host in self.blacklist - def whitelisted(self, host): + def in_target(self, host): """ - Check whether a hostname, url, IP, etc. is whitelisted. + Check whether a hostname, url, IP, etc. is in the target. + + This method ONLY checks the target, NOT the blacklist. + Use `in_scope()` to check both target AND blacklist. Note that `host` can be a hostname, IP address, CIDR, email address, or any BBOT `Event` with the `host` attribute. Args: - host (str or IPAddress or Event): The host to check against the whitelist + host (str or IPAddress or Event): The host to check against the target Examples: - Check if a URL's host is whitelisted: - >>> preset.whitelisted("http://www.evilcorp.com") + Check if a URL's host is in target: + >>> preset.in_target("http://www.evilcorp.com") True """ - return host in self.whitelist + return host in self.target def __eq__(self, other): return self.hash == other.hash diff --git a/bbot/test/bbot_fixtures.py b/bbot/test/bbot_fixtures.py index a3dc91524d..6a6adbc45c 100644 --- a/bbot/test/bbot_fixtures.py +++ b/bbot/test/bbot_fixtures.py @@ -55,6 +55,8 @@ def clean_default_config(monkeypatch): ) with monkeypatch.context() as m: m.setattr("bbot.core.core.DEFAULT_CONFIG", clean_config) + # Also clear CORE's custom_config to ensure Preset.copy() gets a clean core + m.setattr(CORE, "_custom_config", OmegaConf.create({})) yield diff --git a/bbot/test/test_step_1/test_cli.py b/bbot/test/test_step_1/test_cli.py index 778ef6f34b..4bab50689e 100644 --- a/bbot/test/test_step_1/test_cli.py +++ b/bbot/test/test_step_1/test_cli.py @@ -12,10 +12,10 @@ async def test_cli_scope(monkeypatch, capsys): monkeypatch.setattr(sys, "exit", lambda *args, **kwargs: True) monkeypatch.setattr(os, "_exit", lambda *args, **kwargs: True) - # basic target without whitelist + # basic target (seeds and target are the same) monkeypatch.setattr( "sys.argv", - ["bbot", "-t", "one.one.one.one", "-c", "scope.report_distance=10", "dns.minimal=false", "--json"], + ["bbot", "-t", "one.one.one.one", "-c", "scope.report_distance=10", "dns.minimal=false", "--json", "-y"], ) result = await cli._main() out, err = capsys.readouterr() @@ -28,10 +28,7 @@ async def test_cli_scope(monkeypatch, capsys): [ l for l in dns_events - if l["module"] == "TARGET" - and l["scope_distance"] == 0 - and "in-scope" in l["tags"] - and "target" in l["tags"] + if l["module"] == "SEED" and l["scope_distance"] == 0 and "in-scope" in l["tags"] and "seed" in l["tags"] ] ) ip_events = [l for l in lines if l["type"] == "IP_ADDRESS" and l["data"] == "1.1.1.1"] @@ -41,20 +38,21 @@ async def test_cli_scope(monkeypatch, capsys): assert ip_events assert all(l["scope_distance"] == 1 and "distance-1" in l["tags"] for l in ip_events) - # with whitelist + # with target_list different from seeds (seeds are one.one.one.one, target is 192.168.0.1) monkeypatch.setattr( "sys.argv", [ "bbot", "-t", - "one.one.one.one", - "-w", "192.168.0.1", + "-s", + "one.one.one.one", "-c", "scope.report_distance=10", "dns.minimal=false", "dns.search_distance=2", "--json", + "-y", ], ) result = await cli._main() @@ -66,17 +64,17 @@ async def test_cli_scope(monkeypatch, capsys): assert not any(l["scope_distance"] == 0 for l in lines) dns_events = [l for l in lines if l["type"] == "DNS_NAME" and l["data"] == "one.one.one.one"] assert dns_events + # When seeds are different from target, the seed DNS_NAME should be out-of-scope + # (distance-1) and tagged as a seed, but NOT tagged as a target (since it is not + # part of the target set that in_target() checks). assert all(l["scope_distance"] == 1 and "distance-1" in l["tags"] for l in dns_events) - assert 1 == len( - [ - l - for l in dns_events - if l["module"] == "TARGET" - and l["scope_distance"] == 1 - and "distance-1" in l["tags"] - and "target" in l["tags"] - ] - ) + target_seed_events = [ + l + for l in dns_events + if l["module"] == "SEED" and l["scope_distance"] == 1 and "distance-1" in l["tags"] and "seed" in l["tags"] + ] + assert len(target_seed_events) == 1 + assert all("target" not in l["tags"] for l in target_seed_events) ip_events = [l for l in lines if l["type"] == "IP_ADDRESS" and l["data"] == "1.1.1.1"] assert ip_events assert all(l["scope_distance"] == 2 and "distance-2" in l["tags"] for l in ip_events) @@ -123,9 +121,9 @@ async def test_cli_scan(monkeypatch): with open(output_filename) as f: lines = f.read().splitlines() for line in lines: - if "[IP_ADDRESS] \t127.0.0.1\tTARGET" in line: + if "[IP_ADDRESS] \t127.0.0.1\tSEED" in line: ip_success = True - if "[DNS_NAME] \twww.example.com\tTARGET" in line: + if "[DNS_NAME] \twww.example.com\tSEED" in line: dns_success = True assert ip_success and dns_success, "IP_ADDRESS and/or DNS_NAME are not present in output.txt" @@ -370,7 +368,7 @@ async def test_cli_args(monkeypatch, caplog, capsys, clean_default_config): result = await cli._main() out, err = capsys.readouterr() assert result is True - assert "[ORG_STUB] evilcorp TARGET" in out + assert "[ORG_STUB] evilcorp\tSEED" in out # activate modules by flag caplog.clear() diff --git a/bbot/test/test_step_1/test_dns.py b/bbot/test/test_step_1/test_dns.py index a8bfefa3a1..7057080be5 100644 --- a/bbot/test/test_step_1/test_dns.py +++ b/bbot/test/test_step_1/test_dns.py @@ -253,8 +253,8 @@ def custom_lookup(query, rdtype): # first, we check with wildcard detection disabled scan = bbot_scanner( - "bbot.fdsa.www.test.evilcorp.com", - whitelist=["evilcorp.com"], + "evilcorp.com", + seeds=["bbot.fdsa.www.test.evilcorp.com"], config={ "dns": {"minimal": False, "disable": False, "search_distance": 5, "wildcard_ignore": ["evilcorp.com"]}, "speculate": True, @@ -263,6 +263,7 @@ def custom_lookup(query, rdtype): await scan.helpers.dns._mock_dns(mock_data, custom_lookup_fn=custom_lookup) events = [e async for e in scan.async_start()] + assert len(events) == 12 assert len([e for e in events if e.type == "DNS_NAME"]) == 5 assert len([e for e in events if e.type == "RAW_DNS_RECORD"]) == 4 @@ -275,7 +276,12 @@ def custom_lookup(query, rdtype): ] dns_names_by_host = {e.host: e for e in events if e.type == "DNS_NAME"} - assert dns_names_by_host["evilcorp.com"].tags == {"domain", "private-ip", "in-scope", "a-record"} + assert dns_names_by_host["evilcorp.com"].tags == { + "domain", + "private-ip", + "in-scope", + "a-record", + } assert dns_names_by_host["evilcorp.com"].resolved_hosts == {"127.0.0.1"} assert dns_names_by_host["test.evilcorp.com"].tags == { "subdomain", @@ -294,6 +300,7 @@ def custom_lookup(query, rdtype): "subdomain", "in-scope", "txt-record", + "seed", } assert dns_names_by_host["bbot.fdsa.www.test.evilcorp.com"].resolved_hosts == set() @@ -310,8 +317,8 @@ def custom_lookup(query, rdtype): # then we run it again with wildcard detection enabled scan = bbot_scanner( - "bbot.fdsa.www.test.evilcorp.com", - whitelist=["evilcorp.com"], + "evilcorp.com", + seeds=["bbot.fdsa.www.test.evilcorp.com"], config={ "dns": {"minimal": False, "disable": False, "search_distance": 5, "wildcard_ignore": []}, "speculate": True, @@ -320,6 +327,7 @@ def custom_lookup(query, rdtype): await scan.helpers.dns._mock_dns(mock_data, custom_lookup_fn=custom_lookup) events = [e async for e in scan.async_start()] + assert len(events) == 12 assert len([e for e in events if e.type == "DNS_NAME"]) == 5 assert len([e for e in events if e.type == "RAW_DNS_RECORD"]) == 4 @@ -332,7 +340,12 @@ def custom_lookup(query, rdtype): ] dns_names_by_host = {e.host: e for e in events if e.type == "DNS_NAME"} - assert dns_names_by_host["evilcorp.com"].tags == {"domain", "private-ip", "in-scope", "a-record"} + assert dns_names_by_host["evilcorp.com"].tags == { + "domain", + "private-ip", + "in-scope", + "a-record", + } assert dns_names_by_host["evilcorp.com"].resolved_hosts == {"127.0.0.1"} assert dns_names_by_host["test.evilcorp.com"].tags == { "subdomain", @@ -366,6 +379,7 @@ def custom_lookup(query, rdtype): "txt-record", "txt-wildcard", "wildcard", + "seed", } assert dns_names_by_host["bbot.fdsa.www.test.evilcorp.com"].resolved_hosts == set() @@ -437,6 +451,7 @@ def custom_lookup(query, rdtype): "domain", "srv-record", "private-ip", + "seed", } assert dns_names_by_host["test.evilcorp.com"].tags == { "in-scope", @@ -539,13 +554,19 @@ def custom_lookup(query, rdtype): from bbot.scanner import Scanner # test with full scan - scan2 = Scanner("asdfl.gashdgkjsadgsdf.github.io", whitelist=["github.io"], config={"dns": {"minimal": False}}) + + scan2 = Scanner( + "github.io", + seeds=["asdfl.gashdgkjsadgsdf.github.io"], + config={"dns": {"minimal": False}}, + ) await scan2._prep() other_event = scan2.make_event( "lkjg.sdfgsg.jgkhajshdsadf.github.io", module=scan2.modules["dnsresolve"], parent=scan2.root_event ) await scan2.ingress_module.queue_event(other_event, {}) events = [e async for e in scan2.async_start()] + assert len(events) == 4 assert 2 == len([e for e in events if e.type == "SCAN"]) unmodified_wildcard_events = [ @@ -581,8 +602,8 @@ def custom_lookup(query, rdtype): # test with full scan (wildcard detection disabled for domain) scan2 = Scanner( - "asdfl.gashdgkjsadgsdf.github.io", - whitelist=["github.io"], + "github.io", + seeds=["asdfl.gashdgkjsadgsdf.github.io"], config={"dns": {"wildcard_ignore": ["github.io"]}}, exclude_modules=["cloudcheck"], ) diff --git a/bbot/test/test_step_1/test_manager_deduplication.py b/bbot/test/test_step_1/test_manager_deduplication.py index 65fbaeb172..e33d5d8b6d 100644 --- a/bbot/test/test_step_1/test_manager_deduplication.py +++ b/bbot/test/test_step_1/test_manager_deduplication.py @@ -101,7 +101,7 @@ async def do_scan(*args, _config={}, _dns_mock={}, scan_callback=None, **kwargs) assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes" and e.parent.data == "test.notreal"]) assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "per_domain_only.test.notreal" and str(e.module) == "per_domain_only"]) assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "per_hostport_only.test.notreal" and str(e.module) == "per_hostport_only"]) - assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "test.notreal" and str(e.module) == "TARGET" and "SCAN:" in e.parent.data["id"]]) + assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "test.notreal" and str(e.module) == "SEED" and "SCAN:" in e.parent.data["id"]]) assert 1 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "accept_dupes.test.notreal:88" and str(e.module) == "everything_module" and e.parent.data == "accept_dupes.test.notreal"]) assert 1 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "default_module.test.notreal:88" and str(e.module) == "everything_module" and e.parent.data == "default_module.test.notreal"]) assert 1 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "per_domain_only.test.notreal:88" and str(e.module) == "everything_module" and e.parent.data == "per_domain_only.test.notreal"]) @@ -115,7 +115,7 @@ async def do_scan(*args, _config={}, _dns_mock={}, scan_callback=None, **kwargs) assert 1 == len([e for e in default_events if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes"]) assert 1 == len([e for e in default_events if e.type == "DNS_NAME" and e.data == "per_domain_only.test.notreal" and str(e.module) == "per_domain_only"]) assert 1 == len([e for e in default_events if e.type == "DNS_NAME" and e.data == "per_hostport_only.test.notreal" and str(e.module) == "per_hostport_only"]) - assert 1 == len([e for e in default_events if e.type == "DNS_NAME" and e.data == "test.notreal" and str(e.module) == "TARGET" and "SCAN:" in e.parent.data["id"]]) + assert 1 == len([e for e in default_events if e.type == "DNS_NAME" and e.data == "test.notreal" and str(e.module) == "SEED" and "SCAN:" in e.parent.data["id"]]) assert len(all_events) == 27 assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "accept_dupes.test.notreal" and str(e.module) == "accept_dupes"]) @@ -127,7 +127,7 @@ async def do_scan(*args, _config={}, _dns_mock={}, scan_callback=None, **kwargs) assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes" and e.parent.data == "test.notreal"]) assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "per_domain_only.test.notreal" and str(e.module) == "per_domain_only"]) assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "per_hostport_only.test.notreal" and str(e.module) == "per_hostport_only"]) - assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "test.notreal" and str(e.module) == "TARGET" and "SCAN:" in e.parent.data["id"]]) + assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "test.notreal" and str(e.module) == "SEED" and "SCAN:" in e.parent.data["id"]]) assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.3" and str(e.module) == "A" and e.parent.data == "test.notreal"]) assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.3" and str(e.module) == "A" and e.parent.data == "default_module.test.notreal"]) assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.5" and str(e.module) == "A" and e.parent.data == "no_suppress_dupes.test.notreal"]) @@ -147,7 +147,7 @@ async def do_scan(*args, _config={}, _dns_mock={}, scan_callback=None, **kwargs) assert 1 == len([e for e in no_suppress_dupes if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes"]) assert 1 == len([e for e in no_suppress_dupes if e.type == "DNS_NAME" and e.data == "per_domain_only.test.notreal" and str(e.module) == "per_domain_only"]) assert 1 == len([e for e in no_suppress_dupes if e.type == "DNS_NAME" and e.data == "per_hostport_only.test.notreal" and str(e.module) == "per_hostport_only"]) - assert 1 == len([e for e in no_suppress_dupes if e.type == "DNS_NAME" and e.data == "test.notreal" and str(e.module) == "TARGET" and "SCAN:" in e.parent.data["id"]]) + assert 1 == len([e for e in no_suppress_dupes if e.type == "DNS_NAME" and e.data == "test.notreal" and str(e.module) == "SEED" and "SCAN:" in e.parent.data["id"]]) assert len(accept_dupes) == 10 assert 1 == len([e for e in accept_dupes if e.type == "DNS_NAME" and e.data == "accept_dupes.test.notreal" and str(e.module) == "accept_dupes"]) @@ -159,7 +159,7 @@ async def do_scan(*args, _config={}, _dns_mock={}, scan_callback=None, **kwargs) assert 1 == len([e for e in accept_dupes if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes" and e.parent.data == "test.notreal"]) assert 1 == len([e for e in accept_dupes if e.type == "DNS_NAME" and e.data == "per_domain_only.test.notreal" and str(e.module) == "per_domain_only"]) assert 1 == len([e for e in accept_dupes if e.type == "DNS_NAME" and e.data == "per_hostport_only.test.notreal" and str(e.module) == "per_hostport_only"]) - assert 1 == len([e for e in accept_dupes if e.type == "DNS_NAME" and e.data == "test.notreal" and str(e.module) == "TARGET" and "SCAN:" in e.parent.data["id"]]) + assert 1 == len([e for e in accept_dupes if e.type == "DNS_NAME" and e.data == "test.notreal" and str(e.module) == "SEED" and "SCAN:" in e.parent.data["id"]]) assert len(per_hostport_only) == 6 assert 1 == len([e for e in per_hostport_only if e.type == "DNS_NAME" and e.data == "accept_dupes.test.notreal" and str(e.module) == "accept_dupes"]) @@ -167,7 +167,7 @@ async def do_scan(*args, _config={}, _dns_mock={}, scan_callback=None, **kwargs) assert 1 == len([e for e in per_hostport_only if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes"]) assert 1 == len([e for e in per_hostport_only if e.type == "DNS_NAME" and e.data == "per_domain_only.test.notreal" and str(e.module) == "per_domain_only"]) assert 1 == len([e for e in per_hostport_only if e.type == "DNS_NAME" and e.data == "per_hostport_only.test.notreal" and str(e.module) == "per_hostport_only"]) - assert 1 == len([e for e in per_hostport_only if e.type == "DNS_NAME" and e.data == "test.notreal" and str(e.module) == "TARGET" and "SCAN:" in e.parent.data["id"]]) + assert 1 == len([e for e in per_hostport_only if e.type == "DNS_NAME" and e.data == "test.notreal" and str(e.module) == "SEED" and "SCAN:" in e.parent.data["id"]]) assert len(per_domain_only) == 1 - assert 1 == len([e for e in per_domain_only if e.type == "DNS_NAME" and e.data == "test.notreal" and str(e.module) == "TARGET" and "SCAN:" in e.parent.data["id"]]) + assert 1 == len([e for e in per_domain_only if e.type == "DNS_NAME" and e.data == "test.notreal" and str(e.module) == "SEED" and "SCAN:" in e.parent.data["id"]]) diff --git a/bbot/test/test_step_1/test_manager_scope_accuracy.py b/bbot/test/test_step_1/test_manager_scope_accuracy.py index fc9c593bdd..8b7305a790 100644 --- a/bbot/test/test_step_1/test_manager_scope_accuracy.py +++ b/bbot/test/test_step_1/test_manager_scope_accuracy.py @@ -563,8 +563,8 @@ def custom_setup(scan): # 2 events from a single HTTP_RESPONSE events, all_events, all_events_nodups, graph_output_events, graph_output_batch_events = await do_scan( - "127.0.0.111/31", - whitelist=["127.0.0.111/31", "127.0.0.222", "127.0.0.33"], + "127.0.0.111/31", "127.0.0.222", "127.0.0.33", + seeds=["127.0.0.111/31"], modules=["httpx"], output_modules=["python"], _config={ @@ -750,9 +750,9 @@ def custom_setup(scan): # sslcert with out-of-scope chain events, all_events, all_events_nodups, graph_output_events, graph_output_batch_events = await do_scan( - "127.0.0.0/31", + "127.0.1.0", + seeds=["127.0.0.0/31"], modules=["sslcert"], - whitelist=["127.0.1.0"], _config={"scope": {"search_distance": 1, "report_distance": 0}, "speculate": True, "modules": {"speculate": {"ports": "9999"}}}, _dns_mock={"www.bbottest.notreal": {"A": ["127.0.0.1"]}, "test.notreal": {"A": ["127.0.1.0"]}}, ) @@ -806,10 +806,10 @@ async def test_manager_blacklist(bbot_scanner, bbot_httpserver, caplog): # dns search distance = 1, report distance = 0 scan = bbot_scanner( - "http://127.0.0.1:8888", + "127.0.0.0/29", "test.notreal", + seeds=["http://127.0.0.1:8888"], modules=["httpx"], config={"excavate": True, "dns": {"minimal": False, "search_distance": 1}, "scope": {"report_distance": 0}}, - whitelist=["127.0.0.0/29", "test.notreal"], blacklist=["127.0.0.64/29"], ) await scan.helpers.dns._mock_dns({ diff --git a/bbot/test/test_step_1/test_modules_basic.py b/bbot/test/test_step_1/test_modules_basic.py index fc06ab6f8a..6c7b0d1890 100644 --- a/bbot/test/test_step_1/test_modules_basic.py +++ b/bbot/test/test_step_1/test_modules_basic.py @@ -431,9 +431,9 @@ async def handle_event(self, event): "FINDING": 1, } - assert set(scan.stats.module_stats) == {"speculate", "host", "TARGET", "python", "dummy", "dnsresolve"} + assert set(scan.stats.module_stats) == {"speculate", "host", "SEED", "python", "dummy", "dnsresolve"} - target_stats = scan.stats.module_stats["TARGET"] + target_stats = scan.stats.module_stats["SEED"] assert target_stats.produced == {"SCAN": 1, "DNS_NAME": 1} assert target_stats.produced_total == 2 assert target_stats.consumed == {} diff --git a/bbot/test/test_step_1/test_preset_seeds.py b/bbot/test/test_step_1/test_preset_seeds.py new file mode 100644 index 0000000000..07d74c2d9c --- /dev/null +++ b/bbot/test/test_step_1/test_preset_seeds.py @@ -0,0 +1,25 @@ +from bbot.scanner.preset import Preset + + +def test_preset_target_and_seeds_default(): + """ + If no explicit seeds are provided, seeds should be copied from target. + """ + preset = Preset("evilcorp.com") + baked = preset.bake() + + target = baked.target + assert set(target.target.inputs) == {"evilcorp.com"} + assert set(target.seeds.inputs) == {"evilcorp.com"} + + +def test_preset_target_and_seeds_explicit_seeds_override(): + """ + If explicit seeds are provided, they should NOT be copied from target. + """ + preset = Preset("evilcorp.com", seeds=["seedonly.evilcorp.com"]) + baked = preset.bake() + + target = baked.target + assert set(target.target.inputs) == {"evilcorp.com"} + assert set(target.seeds.inputs) == {"seedonly.evilcorp.com"} diff --git a/bbot/test/test_step_1/test_presets.py b/bbot/test/test_step_1/test_presets.py index 5680b624d6..325b0a260e 100644 --- a/bbot/test/test_step_1/test_presets.py +++ b/bbot/test/test_step_1/test_presets.py @@ -71,9 +71,8 @@ def test_preset_yaml(clean_default_config): import yaml preset1 = Preset( - "evilcorp.com", - "www.evilcorp.ce", - whitelist=["evilcorp.ce"], + "evilcorp.ce", + seeds=["evilcorp.com", "www.evilcorp.ce"], blacklist=["test.www.evilcorp.ce"], modules=["sslcert"], output_modules=["json"], @@ -90,14 +89,14 @@ def test_preset_yaml(clean_default_config): assert "evilcorp.com" in preset1.target.seeds assert "evilcorp.ce" not in preset1.target.seeds assert "asdf.www.evilcorp.ce" in preset1.target.seeds - assert "evilcorp.ce" in preset1.whitelist - assert "asdf.evilcorp.ce" in preset1.whitelist + assert "evilcorp.ce" in preset1.target.target + assert "asdf.evilcorp.ce" in preset1.target.target assert "test.www.evilcorp.ce" in preset1.blacklist assert "asdf.test.www.evilcorp.ce" in preset1.blacklist assert "sslcert" in preset1.scan_modules - assert preset1.whitelisted("evilcorp.ce") - assert preset1.whitelisted("www.evilcorp.ce") - assert not preset1.whitelisted("evilcorp.com") + assert preset1.in_target("evilcorp.ce") + assert preset1.in_target("www.evilcorp.ce") + assert not preset1.in_target("evilcorp.com") assert preset1.blacklisted("test.www.evilcorp.ce") assert preset1.blacklisted("asdf.test.www.evilcorp.ce") assert not preset1.blacklisted("www.evilcorp.ce") @@ -174,29 +173,29 @@ def test_preset_scope(): scan = Scanner("1.2.3.4", preset=Preset.from_dict({"target": ["evilcorp.com"]})) assert {str(h) for h in scan.preset.target.seeds.hosts} == {"1.2.3.4/32", "evilcorp.com"} assert {e.data for e in scan.target.seeds} == {"1.2.3.4", "evilcorp.com"} - assert {e.data for e in scan.target.whitelist} == {"1.2.3.4/32", "evilcorp.com"} + assert {str(h) for h in scan.target.target.hosts} == {"1.2.3.4/32", "evilcorp.com"} blank_preset = Preset() blank_preset = blank_preset.bake() assert not blank_preset.target.seeds - assert not blank_preset.target.whitelist + assert not blank_preset.target.target assert blank_preset.strict_scope is False + # Positional args define target; seeds must be explicit preset1 = Preset( - "evilcorp.com", - "www.evilcorp.ce", - whitelist=["evilcorp.ce"], + "evilcorp.ce", + seeds=["evilcorp.com", "www.evilcorp.ce"], blacklist=["test.www.evilcorp.ce"], ) preset1_baked = preset1.bake() # make sure target logic works as expected assert "evilcorp.com" in preset1_baked.target.seeds - assert "evilcorp.com" not in preset1_baked.target.whitelist + assert "evilcorp.com" not in preset1_baked.target.target assert "asdf.evilcorp.com" in preset1_baked.target.seeds - assert "asdf.evilcorp.com" not in preset1_baked.target.whitelist - assert "asdf.evilcorp.ce" in preset1_baked.whitelist - assert "evilcorp.ce" in preset1_baked.whitelist + assert "asdf.evilcorp.com" not in preset1_baked.target.target + assert "asdf.evilcorp.ce" in preset1_baked.target.target + assert "evilcorp.ce" in preset1_baked.target.target assert "test.www.evilcorp.ce" in preset1_baked.blacklist assert "evilcorp.ce" not in preset1_baked.blacklist assert preset1_baked.in_scope("www.evilcorp.ce") @@ -211,8 +210,8 @@ def test_preset_scope(): # test preset merging preset3 = Preset( - "evilcorp.org", - whitelist=["evilcorp.de"], + "evilcorp.de", + seeds=["evilcorp.org"], blacklist=["test.www.evilcorp.de"], config={"scope": {"strict": True}}, ) @@ -230,10 +229,10 @@ def test_preset_scope(): assert "asdf.evilcorp.org" not in preset1_baked.target.seeds assert "asdf.evilcorp.com" not in preset1_baked.target.seeds assert "asdf.www.evilcorp.ce" not in preset1_baked.target.seeds - assert "evilcorp.ce" in preset1_baked.whitelist - assert "evilcorp.de" in preset1_baked.whitelist - assert "asdf.evilcorp.de" not in preset1_baked.whitelist - assert "asdf.evilcorp.ce" not in preset1_baked.whitelist + assert "evilcorp.ce" in preset1_baked.target.target + assert "evilcorp.de" in preset1_baked.target.target + assert "asdf.evilcorp.de" not in preset1_baked.target.target + assert "asdf.evilcorp.ce" not in preset1_baked.target.target # blacklist should be merged, strict scope does not apply assert "test.www.evilcorp.ce" in preset1_baked.blacklist assert "test.www.evilcorp.de" in preset1_baked.blacklist @@ -253,125 +252,150 @@ def test_preset_scope(): preset1.merge(preset4) set(preset1.output_modules) == {"python", "csv", "txt", "json", "stdout", "neo4j"} - # test preset merging + whitelist + # test preset merging + seeds/target interaction - preset_nowhitelist = Preset("evilcorp.com", name="nowhitelist") - preset_whitelist = Preset( - "evilcorp.org", - name="whitelist", - whitelist=["1.2.3.4/24", "http://evilcorp.net"], + # Domain present as both explicit seed and targets + preset_domain_with_seed = Preset("evilcorp.com", seeds=["evilcorp.com"], name="domain_with_seed") + preset_with_target_scope = Preset( + "1.2.3.4/24", + "http://evilcorp.net", + name="with_target_scope", + seeds=["evilcorp.org"], blacklist=["evilcorp.co.uk:443", "bob@evilcorp.co.uk"], config={"modules": {"secretsdb": {"api_key": "deadbeef", "otherthing": "asdf"}}}, ) - preset_nowhitelist_baked = preset_nowhitelist.bake() - preset_whitelist_baked = preset_whitelist.bake() - - assert preset_nowhitelist_baked.to_dict(include_target=True) == { - "target": ["evilcorp.com"], + preset_domain_with_seed_baked = preset_domain_with_seed.bake() + preset_with_target_scope_baked = preset_with_target_scope.bake() + + # When seeds and targets are identical, only targets are serialized. + domain_with_seed_dict = preset_domain_with_seed_baked.to_dict(include_target=True) + assert domain_with_seed_dict.get("target") == ["evilcorp.com"] + assert "seeds" not in domain_with_seed_dict + + # preset with explicit target scope + scope_dict = preset_with_target_scope_baked.to_dict(include_target=True) + assert set(scope_dict["target"]) == {"1.2.3.0/24", "http://evilcorp.net/"} + assert set(scope_dict["blacklist"]) == {"bob@evilcorp.co.uk", "evilcorp.co.uk:443"} + # secretsdb config should be preserved (other module config may also be present) + assert scope_dict["config"]["modules"]["secretsdb"] == { + "api_key": "deadbeef", + "otherthing": "asdf", } - assert preset_whitelist_baked.to_dict(include_target=True) == { - "target": ["evilcorp.org"], - "whitelist": ["1.2.3.0/24", "http://evilcorp.net/"], - "blacklist": ["bob@evilcorp.co.uk", "evilcorp.co.uk:443"], - "config": {"modules": {"secretsdb": {"api_key": "deadbeef", "otherthing": "asdf"}}}, + + redacted_dict = preset_with_target_scope_baked.to_dict(include_target=True, redact_secrets=True) + assert set(redacted_dict["target"]) == {"1.2.3.0/24", "http://evilcorp.net/"} + assert set(redacted_dict["blacklist"]) == {"bob@evilcorp.co.uk", "evilcorp.co.uk:443"} + assert redacted_dict["config"]["modules"]["secretsdb"] == {"otherthing": "asdf"} + + assert preset_domain_with_seed_baked.in_scope("www.evilcorp.com") + assert not preset_domain_with_seed_baked.in_scope("www.evilcorp.de") + assert not preset_domain_with_seed_baked.in_scope("1.2.3.4/24") + + assert "www.evilcorp.org" in preset_with_target_scope_baked.target.seeds + assert "www.evilcorp.org" not in preset_with_target_scope_baked.target.target + assert "1.2.3.4" in preset_with_target_scope_baked.target.target + assert not preset_with_target_scope_baked.in_scope("www.evilcorp.org") + assert not preset_with_target_scope_baked.in_scope("www.evilcorp.de") + assert not preset_with_target_scope_baked.in_target("www.evilcorp.org") + assert not preset_with_target_scope_baked.in_target("www.evilcorp.de") + assert preset_with_target_scope_baked.in_scope("1.2.3.4") + assert preset_with_target_scope_baked.in_scope("1.2.3.4/28") + assert preset_with_target_scope_baked.in_scope("1.2.3.4/24") + assert preset_with_target_scope_baked.in_target("1.2.3.4") + assert preset_with_target_scope_baked.in_target("1.2.3.4/28") + assert preset_with_target_scope_baked.in_target("1.2.3.4/24") + + assert {e.data for e in preset_domain_with_seed_baked.seeds} == {"evilcorp.com"} + assert {e.data for e in preset_domain_with_seed_baked.target.target} == {"evilcorp.com"} + assert {e.data for e in preset_with_target_scope_baked.seeds} == {"evilcorp.org"} + assert {e.data for e in preset_with_target_scope_baked.target.target} == {"1.2.3.0/24", "http://evilcorp.net/"} + + # When merging a preset that has both seeds and target with one that only has + # target (no explicit seeds), explicit seeds are unioned and targets are unioned. + preset_domain_with_seed.merge(preset_with_target_scope) + preset_domain_with_seed_baked = preset_domain_with_seed.bake() + assert {e.data for e in preset_domain_with_seed_baked.seeds} == {"evilcorp.com", "evilcorp.org"} + # After merging, target scope should include both the original domain target and the scoped network/URL + assert {e.data for e in preset_domain_with_seed_baked.target.target} == { + "evilcorp.com", + "1.2.3.0/24", + "http://evilcorp.net/", } - assert preset_whitelist_baked.to_dict(include_target=True, redact_secrets=True) == { - "target": ["evilcorp.org"], - "whitelist": ["1.2.3.0/24", "http://evilcorp.net/"], - "blacklist": ["bob@evilcorp.co.uk", "evilcorp.co.uk:443"], - "config": {"modules": {"secretsdb": {"otherthing": "asdf"}}}, + assert "www.evilcorp.org" in preset_domain_with_seed_baked.seeds + assert "www.evilcorp.com" in preset_domain_with_seed_baked.seeds + assert "1.2.3.4" in preset_domain_with_seed_baked.target.target + assert not preset_domain_with_seed_baked.in_scope("www.evilcorp.org") + # After merging, evilcorp.com remains in target, so its www subdomain is in-scope and in-target + assert preset_domain_with_seed_baked.in_scope("www.evilcorp.com") + assert not preset_domain_with_seed_baked.in_target("www.evilcorp.org") + assert preset_domain_with_seed_baked.in_target("www.evilcorp.com") + assert preset_domain_with_seed_baked.in_scope("1.2.3.4") + + # When merging a preset that only defines targets (no explicit seeds), + # its targets are not promoted to seeds in the merged preset, but targets are unioned. + preset_targets_only = Preset("evilcorp.com") + preset_with_target_scope = Preset("1.2.3.4/24", seeds=["evilcorp.org"]) + preset_with_target_scope.merge(preset_targets_only) + preset_with_target_scope_baked = preset_with_target_scope.bake() + # Seeds stay as the explicit seeds from the base preset + assert {e.data for e in preset_with_target_scope_baked.seeds} == {"evilcorp.org"} + # Target scope is the union of both presets' targets. + assert {e.data for e in preset_with_target_scope_baked.target.target} == { + "evilcorp.com", + "1.2.3.0/24", } - - assert preset_nowhitelist_baked.in_scope("www.evilcorp.com") - assert not preset_nowhitelist_baked.in_scope("www.evilcorp.de") - assert not preset_nowhitelist_baked.in_scope("1.2.3.4/24") - - assert "www.evilcorp.org" in preset_whitelist_baked.target.seeds - assert "www.evilcorp.org" not in preset_whitelist_baked.target.whitelist - assert "1.2.3.4" in preset_whitelist_baked.whitelist - assert not preset_whitelist_baked.in_scope("www.evilcorp.org") - assert not preset_whitelist_baked.in_scope("www.evilcorp.de") - assert not preset_whitelist_baked.whitelisted("www.evilcorp.org") - assert not preset_whitelist_baked.whitelisted("www.evilcorp.de") - assert preset_whitelist_baked.in_scope("1.2.3.4") - assert preset_whitelist_baked.in_scope("1.2.3.4/28") - assert preset_whitelist_baked.in_scope("1.2.3.4/24") - assert preset_whitelist_baked.whitelisted("1.2.3.4") - assert preset_whitelist_baked.whitelisted("1.2.3.4/28") - assert preset_whitelist_baked.whitelisted("1.2.3.4/24") - - assert {e.data for e in preset_nowhitelist_baked.seeds} == {"evilcorp.com"} - assert {e.data for e in preset_nowhitelist_baked.whitelist} == {"evilcorp.com"} - assert {e.data for e in preset_whitelist_baked.seeds} == {"evilcorp.org"} - assert {e.data for e in preset_whitelist_baked.whitelist} == {"1.2.3.0/24", "http://evilcorp.net/"} - - preset_nowhitelist.merge(preset_whitelist) - preset_nowhitelist_baked = preset_nowhitelist.bake() - assert {e.data for e in preset_nowhitelist_baked.seeds} == {"evilcorp.com", "evilcorp.org"} - assert {e.data for e in preset_nowhitelist_baked.whitelist} == {"1.2.3.0/24", "http://evilcorp.net/"} - assert "www.evilcorp.org" in preset_nowhitelist_baked.seeds - assert "www.evilcorp.com" in preset_nowhitelist_baked.seeds - assert "1.2.3.4" in preset_nowhitelist_baked.whitelist - assert not preset_nowhitelist_baked.in_scope("www.evilcorp.org") - assert not preset_nowhitelist_baked.in_scope("www.evilcorp.com") - assert not preset_nowhitelist_baked.whitelisted("www.evilcorp.org") - assert not preset_nowhitelist_baked.whitelisted("www.evilcorp.com") - assert preset_nowhitelist_baked.in_scope("1.2.3.4") - - preset_nowhitelist = Preset("evilcorp.com") - preset_whitelist = Preset("evilcorp.org", whitelist=["1.2.3.4/24"]) - preset_whitelist.merge(preset_nowhitelist) - preset_whitelist_baked = preset_whitelist.bake() - assert {e.data for e in preset_whitelist_baked.seeds} == {"evilcorp.com", "evilcorp.org"} - assert {e.data for e in preset_whitelist_baked.whitelist} == {"1.2.3.0/24"} - assert "www.evilcorp.org" in preset_whitelist_baked.seeds - assert "www.evilcorp.com" in preset_whitelist_baked.seeds - assert "www.evilcorp.org" not in preset_whitelist_baked.target.whitelist - assert "www.evilcorp.com" not in preset_whitelist_baked.target.whitelist - assert "1.2.3.4" in preset_whitelist_baked.whitelist - assert not preset_whitelist_baked.in_scope("www.evilcorp.org") - assert not preset_whitelist_baked.in_scope("www.evilcorp.com") - assert not preset_whitelist_baked.whitelisted("www.evilcorp.org") - assert not preset_whitelist_baked.whitelisted("www.evilcorp.com") - assert preset_whitelist_baked.in_scope("1.2.3.4") - - preset_nowhitelist1 = Preset("evilcorp.com") - preset_nowhitelist2 = Preset("evilcorp.de") - preset_nowhitelist1_baked = preset_nowhitelist1.bake() - preset_nowhitelist2_baked = preset_nowhitelist2.bake() - assert {e.data for e in preset_nowhitelist1_baked.seeds} == {"evilcorp.com"} - assert {e.data for e in preset_nowhitelist2_baked.seeds} == {"evilcorp.de"} - assert {e.data for e in preset_nowhitelist1_baked.whitelist} == {"evilcorp.com"} - assert {e.data for e in preset_nowhitelist2_baked.whitelist} == {"evilcorp.de"} - preset_nowhitelist1.merge(preset_nowhitelist2) - preset_nowhitelist1_baked = preset_nowhitelist1.bake() - assert {e.data for e in preset_nowhitelist1_baked.seeds} == {"evilcorp.com", "evilcorp.de"} - assert {e.data for e in preset_nowhitelist2_baked.seeds} == {"evilcorp.de"} - assert {e.data for e in preset_nowhitelist1_baked.whitelist} == {"evilcorp.com", "evilcorp.de"} - assert {e.data for e in preset_nowhitelist2_baked.whitelist} == {"evilcorp.de"} - assert "www.evilcorp.com" in preset_nowhitelist1_baked.seeds - assert "www.evilcorp.de" in preset_nowhitelist1_baked.seeds - assert "www.evilcorp.com" in preset_nowhitelist1_baked.target.seeds - assert "www.evilcorp.de" in preset_nowhitelist1_baked.target.seeds - assert "www.evilcorp.com" in preset_nowhitelist1_baked.whitelist - assert "www.evilcorp.de" in preset_nowhitelist1_baked.whitelist - assert preset_nowhitelist1_baked.whitelisted("www.evilcorp.com") - assert preset_nowhitelist1_baked.whitelisted("www.evilcorp.de") - assert not preset_nowhitelist1_baked.whitelisted("1.2.3.4") - assert preset_nowhitelist1_baked.in_scope("www.evilcorp.com") - assert preset_nowhitelist1_baked.in_scope("www.evilcorp.de") - assert not preset_nowhitelist1_baked.in_scope("1.2.3.4") - - preset_nowhitelist1 = Preset("evilcorp.com") - preset_nowhitelist2 = Preset("evilcorp.de") - preset_nowhitelist2.merge(preset_nowhitelist1) - preset_nowhitelist1_baked = preset_nowhitelist1.bake() - preset_nowhitelist2_baked = preset_nowhitelist2.bake() - assert {e.data for e in preset_nowhitelist1_baked.seeds} == {"evilcorp.com"} - assert {e.data for e in preset_nowhitelist2_baked.seeds} == {"evilcorp.com", "evilcorp.de"} - assert {e.data for e in preset_nowhitelist1_baked.whitelist} == {"evilcorp.com"} - assert {e.data for e in preset_nowhitelist2_baked.whitelist} == {"evilcorp.com", "evilcorp.de"} + # Seed expansion only applies to explicit seeds (evilcorp.org), not merged targets. + assert "www.evilcorp.org" in preset_with_target_scope_baked.seeds + assert "www.evilcorp.com" not in preset_with_target_scope_baked.seeds + # Target expansion only applies to targets (evilcorp.com), not seeds-only domains. + assert "www.evilcorp.org" not in preset_with_target_scope_baked.target.target + assert "www.evilcorp.com" in preset_with_target_scope_baked.target.target + # Scope/target checks reflect that only evilcorp.com is in the merged target. + assert not preset_with_target_scope_baked.in_scope("www.evilcorp.org") + assert preset_with_target_scope_baked.in_scope("www.evilcorp.com") + assert not preset_with_target_scope_baked.in_target("www.evilcorp.org") + assert preset_with_target_scope_baked.in_target("www.evilcorp.com") + assert preset_with_target_scope_baked.in_scope("1.2.3.4") + + # Merging two presets created only with positional targets: + # after bake, each has seeds backfilled from its own target, and merge unions both. + preset_targets_only1 = Preset("evilcorp.com") + preset_targets_only2 = Preset("evilcorp.de") + preset_targets_only1_baked = preset_targets_only1.bake() + preset_targets_only2_baked = preset_targets_only2.bake() + assert {e.data for e in preset_targets_only1_baked.seeds} == {"evilcorp.com"} + assert {e.data for e in preset_targets_only2_baked.seeds} == {"evilcorp.de"} + assert {e.data for e in preset_targets_only1_baked.target.target} == {"evilcorp.com"} + assert {e.data for e in preset_targets_only2_baked.target.target} == {"evilcorp.de"} + preset_targets_only1.merge(preset_targets_only2) + preset_targets_only1_baked = preset_targets_only1.bake() + assert {e.data for e in preset_targets_only1_baked.seeds} == {"evilcorp.com", "evilcorp.de"} + assert {e.data for e in preset_targets_only2_baked.seeds} == {"evilcorp.de"} + assert {e.data for e in preset_targets_only1_baked.target.target} == {"evilcorp.com", "evilcorp.de"} + assert {e.data for e in preset_targets_only2_baked.target.target} == {"evilcorp.de"} + assert "www.evilcorp.com" in preset_targets_only1_baked.seeds + assert "www.evilcorp.de" in preset_targets_only1_baked.seeds + assert "www.evilcorp.com" in preset_targets_only1_baked.target.seeds + assert "www.evilcorp.de" in preset_targets_only1_baked.target.seeds + assert "www.evilcorp.com" in preset_targets_only1_baked.target.target + assert "www.evilcorp.de" in preset_targets_only1_baked.target.target + assert preset_targets_only1_baked.in_target("www.evilcorp.com") + assert preset_targets_only1_baked.in_target("www.evilcorp.de") + assert not preset_targets_only1_baked.in_target("1.2.3.4") + assert preset_targets_only1_baked.in_scope("www.evilcorp.com") + assert preset_targets_only1_baked.in_scope("www.evilcorp.de") + assert not preset_targets_only1_baked.in_scope("1.2.3.4") + + preset_targets_only1 = Preset("evilcorp.com") + preset_targets_only2 = Preset("evilcorp.de") + preset_targets_only2.merge(preset_targets_only1) + preset_targets_only1_baked = preset_targets_only1.bake() + preset_targets_only2_baked = preset_targets_only2.bake() + assert {e.data for e in preset_targets_only1_baked.seeds} == {"evilcorp.com"} + assert {e.data for e in preset_targets_only2_baked.seeds} == {"evilcorp.com", "evilcorp.de"} + assert {e.data for e in preset_targets_only1_baked.target.target} == {"evilcorp.com"} + assert {e.data for e in preset_targets_only2_baked.target.target} == {"evilcorp.com", "evilcorp.de"} @pytest.mark.asyncio @@ -601,20 +625,26 @@ async def handle_event(self, event): shutil.rmtree(custom_module_dir) -def test_preset_scope_round_trip(): +def test_preset_scope_round_trip(clean_default_config): preset_dict = { - "target": ["127.0.0.1"], - "whitelist": ["127.0.0.2"], + # seeds: initial inputs that drive passive modules + "seeds": ["127.0.0.1"], + # target: what in_target() / in_scope() check + "target": ["127.0.0.2"], "blacklist": ["127.0.0.3"], "config": {"scope": {"strict": True}}, } preset = Preset.from_dict(preset_dict) baked = preset.bake() + # Seeds should round-trip unchanged assert list(baked.seeds) == ["127.0.0.1"] - assert list(baked.whitelist) == ["127.0.0.2"] + # Target list should round-trip unchanged + assert list(baked.target.target.inputs) == ["127.0.0.2"] + # Blacklist should round-trip unchanged assert list(baked.blacklist) == ["127.0.0.3"] - assert baked.config.scope.strict is True - assert baked.to_dict(include_target=True) == preset_dict + # Scope config should be preserved + result = baked.to_dict(include_target=True) + assert result["config"]["scope"] == preset_dict["config"]["scope"] def test_preset_target_tolerance(): @@ -1156,5 +1186,7 @@ def test_preset_serialization(): preset_dict = preset.to_dict(include_target=True) print(preset_dict) preset_str = json.dumps(preset_dict) - preset_dict = json.loads(preset_str) - assert preset_dict == {"target": ["192.168.1.1"], "whitelist": ["192.168.1.1/32"]} + preset_dict_round_tripped = json.loads(preset_str) + assert preset_dict_round_tripped == preset_dict + assert preset_dict["target"] == ["192.168.1.1"] + assert "seeds" not in preset_dict diff --git a/bbot/test/test_step_1/test_python_api.py b/bbot/test/test_step_1/test_python_api.py index 535c015aaa..0cac093af0 100644 --- a/bbot/test/test_step_1/test_python_api.py +++ b/bbot/test/test_step_1/test_python_api.py @@ -54,7 +54,8 @@ async def test_python_api(): # custom target types custom_target_scan = Scanner("ORG:evilcorp") events = [e async for e in custom_target_scan.async_start()] - assert 1 == len([e for e in events if e.type == "ORG_STUB" and e.data == "evilcorp" and "target" in e.tags]) + + assert 1 == len([e for e in events if e.type == "ORG_STUB" and e.data == "evilcorp" and "seed" in e.tags]) # presets scan6 = Scanner("evilcorp.com", presets=["subdomain-enum"]) diff --git a/bbot/test/test_step_1/test_scan.py b/bbot/test/test_step_1/test_scan.py index 66738796ed..a375fc3c3a 100644 --- a/bbot/test/test_step_1/test_scan.py +++ b/bbot/test/test_step_1/test_scan.py @@ -19,42 +19,42 @@ async def test_scan( modules=["ipneighbor"], ) await scan0.load_modules() - assert scan0.whitelisted("1.1.1.1") - assert scan0.whitelisted("1.1.1.0") + assert scan0.in_target("1.1.1.1") + assert scan0.in_target("1.1.1.0") assert scan0.blacklisted("1.1.1.15") assert not scan0.blacklisted("1.1.1.16") assert scan0.blacklisted("1.1.1.1/30") assert not scan0.blacklisted("1.1.1.1/27") assert not scan0.in_scope("1.1.1.1") - assert scan0.whitelisted("api.evilcorp.com") - assert scan0.whitelisted("www.evilcorp.com") + assert scan0.in_target("api.evilcorp.com") + assert scan0.in_target("www.evilcorp.com") assert not scan0.blacklisted("api.evilcorp.com") assert scan0.blacklisted("asdf.www.evilcorp.com") assert scan0.in_scope("test.api.evilcorp.com") assert not scan0.in_scope("test.www.evilcorp.com") assert not scan0.in_scope("www.evilcorp.co.uk") j = scan0.json - assert set(j["target"]["seeds"]) == {"1.1.1.0", "1.1.1.0/31", "evilcorp.com", "test.evilcorp.com"} - # no whitelist was set - assert j["target"]["whitelist"] is None - # but functionally it was copied from the seeds, and collapsed - assert scan0.target.whitelist.hosts == {ip_network("1.1.1.0/31"), "evilcorp.com"} + assert not "seeds" in j["target"], "seeds should not be in target json" + # Positional arguments become the target + assert set(j["target"]["target"]) == {"1.1.1.0", "1.1.1.0/31", "evilcorp.com", "test.evilcorp.com"} + # Seeds are backfilled from target when not explicitly set + assert scan0.target.target.hosts == {ip_network("1.1.1.0/31"), "evilcorp.com"} assert set(j["target"]["blacklist"]) == {"1.1.1.0/28", "www.evilcorp.com"} assert "ipneighbor" in j["preset"]["modules"] - scan1 = bbot_scanner("1.1.1.1", whitelist=["1.0.0.1"]) + scan1 = bbot_scanner("1.0.0.1", seeds=["1.1.1.1"]) assert not scan1.blacklisted("1.1.1.1") assert not scan1.blacklisted("1.0.0.1") - assert not scan1.whitelisted("1.1.1.1") - assert scan1.whitelisted("1.0.0.1") + assert not scan1.in_target("1.1.1.1") + assert scan1.in_target("1.0.0.1") assert scan1.in_scope("1.0.0.1") assert not scan1.in_scope("1.1.1.1") scan2 = bbot_scanner("1.1.1.1") assert not scan2.blacklisted("1.1.1.1") assert not scan2.blacklisted("1.0.0.1") - assert scan2.whitelisted("1.1.1.1") - assert not scan2.whitelisted("1.0.0.1") + assert scan2.in_target("1.1.1.1") + assert not scan2.in_target("1.0.0.1") assert scan2.in_scope("1.1.1.1") assert not scan2.in_scope("1.0.0.1") @@ -88,6 +88,36 @@ async def test_scan( assert len(scan6.dns_strings) == 1 +def test_seeds_target_separation(bbot_scanner): + """ + Test that when seeds are explicitly provided (via -s), they are properly separated from target. + """ + # Simulate: bbot -t 192.168.1.0/24 -s seed1.example.com seed2.example.com + scan = bbot_scanner( + "192.168.1.0/24", + seeds=["seed1.example.com", "seed2.example.com"], + ) + + # Verify target and seeds are properly separated in JSON + j = scan.json + assert set(j["target"]["target"]) == {"192.168.1.0/24"}, "Target should only contain the IP range" + assert set(j["target"]["seeds"]) == {"seed1.example.com", "seed2.example.com"}, ( + "Seeds should contain the DNS names, not the target" + ) + + # Verify target functionality + assert scan.in_target("192.168.1.1"), "IP in target range should be in target" + assert not scan.in_target("seed1.example.com"), "Seed DNS name should not be in target" + assert not scan.in_target("seed2.example.com"), "Seed DNS name should not be in target" + + # Verify seeds are accessible + assert "seed1.example.com" in scan.target.seeds.inputs, "seed1.example.com should be in seeds" + assert "seed2.example.com" in scan.target.seeds.inputs, "seed2.example.com should be in seeds" + assert "192.168.1.0/24" not in scan.target.seeds.inputs, ( + "Target should not be in seeds when seeds are explicitly provided" + ) + + @pytest.mark.asyncio async def test_task_scan_handle_event_timeout(bbot_scanner): from bbot.modules.base import BaseModule @@ -220,7 +250,7 @@ async def test_huge_target_list(bbot_scanner, monkeypatch): @pytest.mark.asyncio -async def test_exclude_cdn(bbot_scanner, monkeypatch): +async def test_exclude_cdn(bbot_scanner, monkeypatch, clean_default_config): # test that CDN exclusion works from bbot.scanner import Preset diff --git a/bbot/test/test_step_1/test_scope.py b/bbot/test/test_step_1/test_scope.py index ac2d8c0426..11c589bda1 100644 --- a/bbot/test/test_step_1/test_scope.py +++ b/bbot/test/test_step_1/test_scope.py @@ -21,7 +21,7 @@ def check(self, module_test, events): if e.type == "URL_UNVERIFIED" and str(e.host) == "127.0.0.1" and e.scope_distance == 0 - and "target" in e.tags + and "seed" in e.tags ] ) # we have two of these because the host module considers "always_emit" in its outgoing deduplication @@ -68,27 +68,45 @@ def check(self, module_test, events): assert not any(str(e.host) == "127.0.0.1" for e in events) -class TestScopeWhitelist(TestScopeBlacklist): - blacklist = [] - whitelist = ["255.255.255.255"] +class TestScopeCidrWithSeeds(ModuleTestBase): + """ + Test that when we have a CIDR as the target and DNS names as seeds, + only the DNS names that resolve to IPs within the CIDR should be detected as in-scope. + """ + + # Seeds: DNS names that will be tested + seeds = ["inscope.example.com", "outscope.example.com"] + # Target: CIDR that defines the scope + targets = ["192.168.1.0/24"] + modules_overrides = ["dnsresolve"] + + async def setup_before_prep(self, module_test): + # Mock DNS so that: + # - inscope.example.com resolves to 192.168.1.10 (inside the /24) + # - outscope.example.com resolves to 10.0.0.1 (outside the /24) + # We do this before prep to ensure DNS mocking is ready before any resolution happens + await module_test.mock_dns( + { + "inscope.example.com": {"A": ["192.168.1.10"]}, + "outscope.example.com": {"A": ["10.0.0.1"]}, + } + ) def check(self, module_test, events): - assert len(events) == 4 - assert not any(e.type == "URL" for e in events) - assert 1 == len( - [ - e - for e in events - if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e.scope_distance == 1 and "target" in e.tags - ] + # Find the DNS_NAME events for our seeds + inscope_events = [e for e in events if e.type == "DNS_NAME" and e.data == "inscope.example.com"] + outscope_events = [e for e in events if e.type == "DNS_NAME" and e.data == "outscope.example.com"] + + assert len(inscope_events) == 1, "inscope.example.com should be detected" + inscope_event = inscope_events[0] + assert inscope_event.scope_distance == 0, ( + f"inscope.example.com should be in-scope (scope_distance=0), got {inscope_event.scope_distance}" ) - assert 1 == len( - [ - e - for e in events - if e.type == "URL_UNVERIFIED" - and str(e.host) == "127.0.0.1" - and e.scope_distance == 1 - and "target" in e.tags - ] + assert "192.168.1.10" in inscope_event.resolved_hosts, "inscope.example.com should resolve to 192.168.1.10" + + assert len(outscope_events) > 0, "outscope.example.com should be detected" + outscope_event = outscope_events[0] + assert outscope_event.scope_distance > 0, ( + f"outscope.example.com should be out-of-scope (scope_distance>0), got {outscope_event.scope_distance}" ) + assert "10.0.0.1" in outscope_event.resolved_hosts, "outscope.example.com should resolve to 10.0.0.1" diff --git a/bbot/test/test_step_1/test_target.py b/bbot/test/test_step_1/test_target.py index e2de5fb6f3..b9b0cd7fa2 100644 --- a/bbot/test/test_step_1/test_target.py +++ b/bbot/test/test_step_1/test_target.py @@ -14,7 +14,7 @@ async def test_target_basic(bbot_scanner): scan5 = bbot_scanner() # test different types of inputs - target = BBOTTarget("evilcorp.com", "1.2.3.4/8") + target = BBOTTarget(target=["evilcorp.com", "1.2.3.4/8"]) assert "www.evilcorp.com" in target.seeds assert "www.evilcorp.com:80" in target.seeds assert "http://www.evilcorp.com:80" in target.seeds @@ -54,37 +54,37 @@ async def test_target_basic(bbot_scanner): assert scan2.target.seeds == scan3.target.seeds assert scan4.target.seeds != scan1.target.seeds - assert not scan5.target.whitelist - assert len(scan1.target.whitelist) == 9 - assert len(scan4.target.whitelist) == 8 - assert "8.8.8.9" in scan1.target.whitelist - assert "8.8.8.12" not in scan1.target.whitelist - assert "8.8.8.8/31" in scan1.target.whitelist - assert "8.8.8.8/30" in scan1.target.whitelist - assert "8.8.8.8/29" not in scan1.target.whitelist - assert "2001:4860:4860::8889" in scan1.target.whitelist - assert "2001:4860:4860::888c" not in scan1.target.whitelist - assert "www.api.publicapis.org" in scan1.target.whitelist - assert "api.publicapis.org" in scan1.target.whitelist - assert "publicapis.org" not in scan1.target.whitelist - assert "bob@www.api.publicapis.org" in scan1.target.whitelist - assert "https://www.api.publicapis.org" in scan1.target.whitelist - assert "www.api.publicapis.org:80" in scan1.target.whitelist - assert scan1.make_event("https://[2001:4860:4860::8888]:80", dummy=True) in scan1.target.whitelist - assert scan1.make_event("[2001:4860:4860::8888]:80", "OPEN_TCP_PORT", dummy=True) in scan1.target.whitelist - assert scan1.make_event("[2001:4860:4860::888c]:80", "OPEN_TCP_PORT", dummy=True) not in scan1.target.whitelist - assert scan1.target.whitelist in scan2.target.whitelist - assert scan2.target.whitelist not in scan1.target.whitelist - assert scan3.target.whitelist in scan2.target.whitelist - assert scan2.target.whitelist == scan3.target.whitelist - assert scan4.target.whitelist != scan1.target.whitelist - - assert scan1.whitelisted("https://[2001:4860:4860::8888]:80") - assert scan1.whitelisted("[2001:4860:4860::8888]:80") - assert not scan1.whitelisted("[2001:4860:4860::888c]:80") - assert scan1.whitelisted("www.api.publicapis.org") - assert scan1.whitelisted("api.publicapis.org") - assert not scan1.whitelisted("publicapis.org") + assert not scan5.target.target + assert len(scan1.target.target) == 9 + assert len(scan4.target.target) == 8 + assert "8.8.8.9" in scan1.target.target + assert "8.8.8.12" not in scan1.target.target + assert "8.8.8.8/31" in scan1.target.target + assert "8.8.8.8/30" in scan1.target.target + assert "8.8.8.8/29" not in scan1.target.target + assert "2001:4860:4860::8889" in scan1.target.target + assert "2001:4860:4860::888c" not in scan1.target.target + assert "www.api.publicapis.org" in scan1.target.target + assert "api.publicapis.org" in scan1.target.target + assert "publicapis.org" not in scan1.target.target + assert "bob@www.api.publicapis.org" in scan1.target.target + assert "https://www.api.publicapis.org" in scan1.target.target + assert "www.api.publicapis.org:80" in scan1.target.target + assert scan1.make_event("https://[2001:4860:4860::8888]:80", dummy=True) in scan1.target.target + assert scan1.make_event("[2001:4860:4860::8888]:80", "OPEN_TCP_PORT", dummy=True) in scan1.target.target + assert scan1.make_event("[2001:4860:4860::888c]:80", "OPEN_TCP_PORT", dummy=True) not in scan1.target.target + assert scan1.target.target in scan2.target.target + assert scan2.target.target not in scan1.target.target + assert scan3.target.target in scan2.target.target + assert scan2.target.target == scan3.target.target + assert scan4.target.target != scan1.target.target + + assert scan1.in_target("https://[2001:4860:4860::8888]:80") + assert scan1.in_target("[2001:4860:4860::8888]:80") + assert not scan1.in_target("[2001:4860:4860::888c]:80") + assert scan1.in_target("www.api.publicapis.org") + assert scan1.in_target("api.publicapis.org") + assert not scan1.in_target("publicapis.org") assert scan1.target.seeds in scan2.target.seeds assert scan2.target.seeds not in scan1.target.seeds @@ -93,17 +93,17 @@ async def test_target_basic(bbot_scanner): assert scan4.target.seeds != scan1.target.seeds assert str(scan1.target.seeds.get("8.8.8.9").host) == "8.8.8.8/30" - assert str(scan1.target.whitelist.get("8.8.8.9").host) == "8.8.8.8/30" + assert str(scan1.target.target.get("8.8.8.9").host) == "8.8.8.8/30" assert scan1.target.seeds.get("8.8.8.12") is None - assert scan1.target.whitelist.get("8.8.8.12") is None + assert scan1.target.target.get("8.8.8.12") is None assert str(scan1.target.seeds.get("2001:4860:4860::8889").host) == "2001:4860:4860::8888/126" - assert str(scan1.target.whitelist.get("2001:4860:4860::8889").host) == "2001:4860:4860::8888/126" + assert str(scan1.target.target.get("2001:4860:4860::8889").host) == "2001:4860:4860::8888/126" assert scan1.target.seeds.get("2001:4860:4860::888c") is None - assert scan1.target.whitelist.get("2001:4860:4860::888c") is None + assert scan1.target.target.get("2001:4860:4860::888c") is None assert str(scan1.target.seeds.get("www.api.publicapis.org").host) == "api.publicapis.org" - assert str(scan1.target.whitelist.get("www.api.publicapis.org").host) == "api.publicapis.org" + assert str(scan1.target.target.get("www.api.publicapis.org").host) == "api.publicapis.org" assert scan1.target.seeds.get("publicapis.org") is None - assert scan1.target.whitelist.get("publicapis.org") is None + assert scan1.target.target.get("publicapis.org") is None target = RadixTarget("evilcorp.com") assert "com" not in target @@ -128,18 +128,18 @@ async def test_target_basic(bbot_scanner): # test target hashing target1 = BBOTTarget() - target1.whitelist.add("evilcorp.com") - target1.whitelist.add("1.2.3.4/24") - target1.whitelist.add("https://evilcorp.net:8080") + target1.target.add("evilcorp.com") + target1.target.add("1.2.3.4/24") + target1.target.add("https://evilcorp.net:8080") target1.seeds.add("evilcorp.com") target1.seeds.add("1.2.3.4/24") target1.seeds.add("https://evilcorp.net:8080") target2 = BBOTTarget() - target2.whitelist.add("bob@evilcorp.org") - target2.whitelist.add("evilcorp.com") - target2.whitelist.add("1.2.3.4/24") - target2.whitelist.add("https://evilcorp.net:8080") + target2.target.add("bob@evilcorp.org") + target2.target.add("evilcorp.com") + target2.target.add("1.2.3.4/24") + target2.target.add("https://evilcorp.net:8080") target2.seeds.add("bob@evilcorp.org") target2.seeds.add("evilcorp.com") target2.seeds.add("1.2.3.4/24") @@ -153,29 +153,30 @@ async def test_target_basic(bbot_scanner): assert target1.hash != target2.hash assert target1.scope_hash != target2.scope_hash # add missing email - target1.whitelist.add("bob@evilcorp.org") + target1.target.add("bob@evilcorp.org") assert target1.hash != target2.hash assert target1.scope_hash == target2.scope_hash target1.seeds.add("bob@evilcorp.org") # now they should match assert target1.hash == target2.hash - # test default whitelist - bbottarget = BBOTTarget("http://1.2.3.4:8443", "bob@evilcorp.com") + # test default target + bbottarget = BBOTTarget(target=["http://1.2.3.4:8443", "bob@evilcorp.com"]) + assert bbottarget.seeds.hosts == {ip_network("1.2.3.4"), "evilcorp.com"} - assert bbottarget.whitelist.hosts == {ip_network("1.2.3.4"), "evilcorp.com"} + assert bbottarget.target.hosts == {ip_network("1.2.3.4"), "evilcorp.com"} assert {e.data for e in bbottarget.seeds.event_seeds} == {"http://1.2.3.4:8443/", "bob@evilcorp.com"} - assert {e.data for e in bbottarget.whitelist.event_seeds} == {"1.2.3.4/32", "evilcorp.com"} + assert {e.data for e in bbottarget.target.event_seeds} == {"http://1.2.3.4:8443/", "bob@evilcorp.com"} - bbottarget1 = BBOTTarget("evilcorp.com", "evilcorp.net", whitelist=["1.2.3.4/24"], blacklist=["1.2.3.4"]) - bbottarget2 = BBOTTarget("evilcorp.com", "evilcorp.net", whitelist=["1.2.3.0/24"], blacklist=["1.2.3.4"]) - bbottarget3 = BBOTTarget("evilcorp.com", whitelist=["1.2.3.4/24"], blacklist=["1.2.3.4"]) - bbottarget5 = BBOTTarget("evilcorp.com", "evilcorp.net", whitelist=["1.2.3.0/24"], blacklist=["1.2.3.4"]) + bbottarget1 = BBOTTarget(seeds=["evilcorp.com", "evilcorp.net"], target=["1.2.3.4/24"], blacklist=["1.2.3.4"]) + bbottarget2 = BBOTTarget(seeds=["evilcorp.com", "evilcorp.net"], target=["1.2.3.0/24"], blacklist=["1.2.3.4"]) + bbottarget3 = BBOTTarget(seeds=["evilcorp.com"], target=["1.2.3.4/24"], blacklist=["1.2.3.4"]) + bbottarget5 = BBOTTarget(seeds=["evilcorp.com", "evilcorp.net"], target=["1.2.3.0/24"], blacklist=["1.2.3.4"]) bbottarget6 = BBOTTarget( - "evilcorp.com", "evilcorp.net", whitelist=["1.2.3.0/24"], blacklist=["1.2.3.4"], strict_dns_scope=True + seeds=["evilcorp.com", "evilcorp.net"], target=["1.2.3.0/24"], blacklist=["1.2.3.4"], strict_dns_scope=True ) - bbottarget8 = BBOTTarget("1.2.3.0/24", whitelist=["evilcorp.com", "evilcorp.net"], blacklist=["1.2.3.4"]) - bbottarget9 = BBOTTarget("evilcorp.com", "evilcorp.net", whitelist=["1.2.3.0/24"], blacklist=["1.2.3.4"]) + bbottarget8 = BBOTTarget(seeds=["1.2.3.0/24"], target=["evilcorp.com", "evilcorp.net"], blacklist=["1.2.3.4"]) + bbottarget9 = BBOTTarget(seeds=["evilcorp.com", "evilcorp.net"], target=["1.2.3.0/24"], blacklist=["1.2.3.4"]) # make sure it's a sha1 hash assert isinstance(bbottarget1.hash, bytes) @@ -191,9 +192,9 @@ async def test_target_basic(bbot_scanner): assert bbottarget1 == bbottarget3 assert bbottarget3 == bbottarget1 - # adding different events (but with same host) to whitelist should not change hash (since only hosts matter) - bbottarget1.whitelist.add("http://evilcorp.co.nz") - bbottarget2.whitelist.add("evilcorp.co.nz") + # adding different events (but with same host) to target should not change hash (since only hosts matter) + bbottarget1.target.add("http://evilcorp.co.nz") + bbottarget2.target.add("evilcorp.co.nz") assert bbottarget1 == bbottarget2 assert bbottarget2 == bbottarget1 @@ -207,28 +208,28 @@ async def test_target_basic(bbot_scanner): assert bbottarget5 != bbottarget6 assert bbottarget6 != bbottarget5 - # make sure swapped target <--> whitelist result in different hash + # make sure swapped target <--> blacklist result in different hash assert bbottarget8 != bbottarget9 assert bbottarget9 != bbottarget8 # make sure duplicate events don't change hash - target1 = BBOTTarget("https://evilcorp.com") - target2 = BBOTTarget("https://evilcorp.com") + target1 = BBOTTarget(target=["https://evilcorp.com"]) + target2 = BBOTTarget(target=["https://evilcorp.com"]) assert target1 == target2 target1.seeds.add("https://evilcorp.com:443") assert target1 == target2 - # make sure hosts are collapsed in whitelist and blacklist + # make sure hosts are collapsed in target and blacklist bbottarget = BBOTTarget( - "http://evilcorp.com:8080", - whitelist=["evilcorp.net:443", "http://evilcorp.net:8080"], + seeds=["http://evilcorp.com:8080"], + target=["evilcorp.net:443", "http://evilcorp.net:8080"], blacklist=["http://evilcorp.org:8080", "evilcorp.org:443"], ) # base class is not iterable with pytest.raises(TypeError): assert list(bbottarget) == ["http://evilcorp.com:8080/"] assert {e.data for e in bbottarget.seeds} == {"http://evilcorp.com:8080/"} - assert {e.data for e in bbottarget.whitelist} == {"evilcorp.net:443", "http://evilcorp.net:8080/"} + assert {e.data for e in bbottarget.target} == {"evilcorp.net:443", "http://evilcorp.net:8080/"} assert {e.data for e in bbottarget.blacklist} == {"http://evilcorp.org:8080/", "evilcorp.org:443"} # test org stub as target @@ -258,10 +259,8 @@ async def test_target_basic(bbot_scanner): # verify hash values bbottarget = BBOTTarget( - "1.2.3.0/24", - "http://www.evilcorp.net", - "bob@fdsa.evilcorp.net", - whitelist=["evilcorp.com", "bob@www.evilcorp.com", "evilcorp.net"], + seeds=["1.2.3.0/24", "http://www.evilcorp.net", "bob@fdsa.evilcorp.net"], + target=["evilcorp.com", "bob@www.evilcorp.com", "evilcorp.net"], blacklist=["1.2.3.4", "4.3.2.1/24", "http://1.2.3.4", "bob@asdf.evilcorp.net"], ) assert {e.data for e in bbottarget.seeds.event_seeds} == { @@ -269,7 +268,7 @@ async def test_target_basic(bbot_scanner): "http://www.evilcorp.net/", "bob@fdsa.evilcorp.net", } - assert {e.data for e in bbottarget.whitelist.event_seeds} == { + assert {e.data for e in bbottarget.target.event_seeds} == { "evilcorp.com", "evilcorp.net", "bob@www.evilcorp.com", @@ -281,19 +280,19 @@ async def test_target_basic(bbot_scanner): "bob@asdf.evilcorp.net", } assert set(bbottarget.seeds.hosts) == {ip_network("1.2.3.0/24"), "www.evilcorp.net", "fdsa.evilcorp.net"} - assert set(bbottarget.whitelist.hosts) == {"evilcorp.com", "evilcorp.net"} + assert set(bbottarget.target.hosts) == {"evilcorp.com", "evilcorp.net"} assert set(bbottarget.blacklist.hosts) == {ip_network("1.2.3.4/32"), ip_network("4.3.2.0/24"), "asdf.evilcorp.net"} assert bbottarget.hash == b"\xb3iU\xa8#\x8aq\x84/\xc5\xf2;\x11\x11\x0c&\xea\x07\xd4Q" assert bbottarget.scope_hash == b"f\xe1\x01c^3\xf5\xd24B\x87P\xa0Glq0p3J" assert bbottarget.seeds.hash == b"V\n\xf5\x1d\x1f=i\xbc\\\x15o\xc2p\xb2\x84\x97\xfeR\xde\xc1" - assert bbottarget.whitelist.hash == b"\x8e\xd0\xa76\x8em4c\x0e\x1c\xfdA\x9d*sv}\xeb\xc4\xc4" + assert bbottarget.target.hash == b"\x8e\xd0\xa76\x8em4c\x0e\x1c\xfdA\x9d*sv}\xeb\xc4\xc4" assert bbottarget.blacklist.hash == b'\xf7\xaf\xa1\xda4"C:\x13\xf42\xc3,\xc3\xa9\x9f\x15\x15n\\' scan = bbot_scanner( - "http://www.evilcorp.net", - "1.2.3.0/24", - "bob@fdsa.evilcorp.net", - whitelist=["evilcorp.net", "evilcorp.com", "bob@www.evilcorp.com"], + "evilcorp.net", + "evilcorp.com", + "bob@www.evilcorp.com", + seeds=["http://www.evilcorp.net", "1.2.3.0/24", "bob@fdsa.evilcorp.net"], blacklist=["bob@asdf.evilcorp.net", "1.2.3.4", "4.3.2.1/24", "http://1.2.3.4"], ) events = [e async for e in scan.async_start()] @@ -302,16 +301,16 @@ async def test_target_basic(bbot_scanner): target_dict = scan_events[0].data["target"] assert target_dict["seeds"] == ["1.2.3.0/24", "bob@fdsa.evilcorp.net", "http://www.evilcorp.net/"] - assert target_dict["whitelist"] == ["bob@www.evilcorp.com", "evilcorp.com", "evilcorp.net"] + assert target_dict["target"] == ["bob@www.evilcorp.com", "evilcorp.com", "evilcorp.net"] assert target_dict["blacklist"] == ["1.2.3.4", "4.3.2.0/24", "bob@asdf.evilcorp.net", "http://1.2.3.4/"] assert target_dict["strict_dns_scope"] is False assert target_dict["hash"] == "b36955a8238a71842fc5f23b11110c26ea07d451" assert target_dict["seed_hash"] == "560af51d1f3d69bc5c156fc270b28497fe52dec1" - assert target_dict["whitelist_hash"] == "8ed0a7368e6d34630e1cfd419d2a73767debc4c4" + assert target_dict["target_hash"] == "8ed0a7368e6d34630e1cfd419d2a73767debc4c4" assert target_dict["blacklist_hash"] == "f7afa1da3422433a13f432c32cc3a99f15156e5c" assert target_dict["scope_hash"] == "66e101635e33f5d234428750a0476c713070334a" - # make sure child subnets/IPs don't get added to whitelist/blacklist + # make sure child subnets/IPs don't get added to target/blacklist target = RadixTarget("1.2.3.4/24", "1.2.3.4/28", acl_mode=True) assert set(target) == {ip_network("1.2.3.0/24")} target = RadixTarget("1.2.3.4/28", "1.2.3.4/24", acl_mode=True) diff --git a/bbot/test/test_step_2/module_tests/base.py b/bbot/test/test_step_2/module_tests/base.py index 7d70c80271..d2ef06f418 100644 --- a/bbot/test/test_step_2/module_tests/base.py +++ b/bbot/test/test_step_2/module_tests/base.py @@ -15,7 +15,7 @@ class ModuleTestBase: targets = ["blacklanternsecurity.com"] scan_name = None blacklist = None - whitelist = None + seeds = None module_name = None config_overrides = {} modules_overrides = None @@ -53,13 +53,15 @@ def __init__( elif module_type == "internal" and not module == "dnsresolve": self.config = OmegaConf.merge(self.config, {module: True}) + seeds = module_test_base.seeds or None + self.scan = Scanner( *module_test_base.targets, modules=modules, output_modules=output_modules, scan_name=module_test_base._scan_name, config=self.config, - whitelist=module_test_base.whitelist, + seeds=seeds, blacklist=module_test_base.blacklist, force_start=getattr(module_test_base, "force_start", False), ) diff --git a/bbot/test/test_step_2/module_tests/test_module_csv.py b/bbot/test/test_step_2/module_tests/test_module_csv.py index 5a9575372d..206b9301aa 100644 --- a/bbot/test/test_step_2/module_tests/test_module_csv.py +++ b/bbot/test/test_step_2/module_tests/test_module_csv.py @@ -11,5 +11,5 @@ def check(self, module_test, events): with open(csv_file) as f: data = f.read() - assert "blacklanternsecurity.com,127.0.0.5,TARGET" in data + assert "blacklanternsecurity.com,127.0.0.5,SEED" in data assert context_data in data diff --git a/bbot/test/test_step_2/module_tests/test_module_dnscommonsrv.py b/bbot/test/test_step_2/module_tests/test_module_dnscommonsrv.py index 53c6ff21be..e2b0438b0b 100644 --- a/bbot/test/test_step_2/module_tests/test_module_dnscommonsrv.py +++ b/bbot/test/test_step_2/module_tests/test_module_dnscommonsrv.py @@ -2,8 +2,8 @@ class TestDNSCommonSRV(ModuleTestBase): - targets = ["media.www.test.api.blacklanternsecurity.com"] - whitelist = ["blacklanternsecurity.com"] + seeds = ["media.www.test.api.blacklanternsecurity.com"] + targets = ["blacklanternsecurity.com"] modules_overrides = ["dnscommonsrv", "speculate"] config_overrides = {"dns": {"minimal": False}} diff --git a/bbot/test/test_step_2/module_tests/test_module_excavate.py b/bbot/test/test_step_2/module_tests/test_module_excavate.py index 3254e55c35..9e18aa9268 100644 --- a/bbot/test/test_step_2/module_tests/test_module_excavate.py +++ b/bbot/test/test_step_2/module_tests/test_module_excavate.py @@ -211,12 +211,10 @@ def check(self, module_test, events): if e.type == "FINDING" and "JWT" in e.data["description"] and str(e.module) == "excavate" ] ) - found_badsecrets_vulnerability = bool( - [e for e in events if e.type == "FINDING" and str(e.module) == "badsecrets"] - ) + found_badsecrets_finding = bool([e for e in events if e.type == "FINDING" and str(e.module) == "badsecrets"]) assert found_js_url_event, "Failed to find URL event for script.js" - assert found_badsecrets_vulnerability, "Failed to find BADSECRETs finding from script.js" + assert found_badsecrets_finding, "Failed to find BADSECRETs finding from script.js" assert found_excavate_jwt_finding, "Failed to find JWT finding from script.js" diff --git a/bbot/test/test_step_2/module_tests/test_module_json.py b/bbot/test/test_step_2/module_tests/test_module_json.py index de37354d0f..61ed7fc1f3 100644 --- a/bbot/test/test_step_2/module_tests/test_module_json.py +++ b/bbot/test/test_step_2/module_tests/test_module_json.py @@ -28,8 +28,8 @@ def check(self, module_test, events): assert scan["id"] == module_test.scan.id assert scan["uuid"] == str(module_test.scan.root_event.uuid) assert scan["parent_uuid"] == str(module_test.scan.root_event.uuid) - assert scan["data_json"]["target"]["seeds"] == ["blacklanternsecurity.com"] - assert scan["data_json"]["target"]["whitelist"] is None + assert not "seeds" in scan["data_json"]["target"], "seeds should not be in target json" + assert scan["data_json"]["target"]["target"] == ["blacklanternsecurity.com"] assert dns_json["data"] == dns_data assert dns_json["id"] == str(dns_event.id) assert dns_json["uuid"] == str(dns_event.uuid) @@ -45,8 +45,8 @@ def check(self, module_test, events): assert scan_reconstructed.data["id"] == module_test.scan.id assert scan_reconstructed.uuid == scan_event.uuid assert scan_reconstructed.parent_uuid == scan_event.uuid - assert scan_reconstructed.data["target"]["seeds"] == ["blacklanternsecurity.com"] - assert scan_reconstructed.data["target"]["whitelist"] is None + assert not "seeds" in scan_reconstructed.data["target"], "seeds should not be in target json" + assert scan_reconstructed.data["target"]["target"] == ["blacklanternsecurity.com"] assert dns_reconstructed.data == dns_data assert dns_reconstructed.uuid == dns_event.uuid assert dns_reconstructed.parent_uuid == module_test.scan.root_event.uuid diff --git a/bbot/test/test_step_2/module_tests/test_module_mongo.py b/bbot/test/test_step_2/module_tests/test_module_mongo.py index 25c317e57c..9accae94c1 100644 --- a/bbot/test/test_step_2/module_tests/test_module_mongo.py +++ b/bbot/test/test_step_2/module_tests/test_module_mongo.py @@ -36,12 +36,12 @@ async def setup_before_prep(self, module_test): "mongo", ) - from motor.motor_asyncio import AsyncIOMotorClient + from pymongo import AsyncMongoClient # Connect to the MongoDB collection with retry logic while True: try: - client = AsyncIOMotorClient("mongodb://localhost:27017", username="bbot", password="bbotislife") + client = AsyncMongoClient("mongodb://localhost:27017", username="bbot", password="bbotislife") db = client[self.test_db_name] events_collection = db.get_collection(self.test_collection_prefix + "events") # Attempt a simple operation to confirm the connection @@ -61,21 +61,22 @@ async def setup_before_prep(self, module_test): async def check(self, module_test, events): try: from bbot.models.pydantic import Event - from motor.motor_asyncio import AsyncIOMotorClient + from pymongo import AsyncMongoClient events_json = [e.json() for e in events] events_json.sort(key=lambda x: x["timestamp"]) # Connect to the MongoDB collection - client = AsyncIOMotorClient("mongodb://localhost:27017", username="bbot", password="bbotislife") + client = AsyncMongoClient("mongodb://localhost:27017", username="bbot", password="bbotislife") db = client[self.test_db_name] events_collection = db.get_collection(self.test_collection_prefix + "events") ### INDEXES ### # make sure the collection has all the right indexes - cursor = events_collection.list_indexes() - indexes = await cursor.to_list(length=None) + indexes_cursor = await events_collection.list_indexes() + indexes = await indexes_cursor.to_list(length=None) + # indexes = await cursor.to_list(length=None) for field in Event.indexed_fields(): assert any(field in index["key"] for index in indexes), f"Index for {field} not found" diff --git a/bbot/test/test_step_2/module_tests/test_module_portscan.py b/bbot/test/test_step_2/module_tests/test_module_portscan.py index 2f904e90eb..63f234559c 100644 --- a/bbot/test/test_step_2/module_tests/test_module_portscan.py +++ b/bbot/test/test_step_2/module_tests/test_module_portscan.py @@ -79,13 +79,13 @@ def check(self, module_test, events): assert self.syn_runs >= 1 assert self.ping_runs == 0 assert 1 == len( - [e for e in events if e.type == "DNS_NAME" and e.data == "evilcorp.com" and str(e.module) == "TARGET"] + [e for e in events if e.type == "DNS_NAME" and e.data == "evilcorp.com" and str(e.module) == "SEED"] ) assert 1 == len( - [e for e in events if e.type == "DNS_NAME" and e.data == "www.evilcorp.com" and str(e.module) == "TARGET"] + [e for e in events if e.type == "DNS_NAME" and e.data == "www.evilcorp.com" and str(e.module) == "SEED"] ) assert 1 == len( - [e for e in events if e.type == "DNS_NAME" and e.data == "asdf.evilcorp.net" and str(e.module) == "TARGET"] + [e for e in events if e.type == "DNS_NAME" and e.data == "asdf.evilcorp.net" and str(e.module) == "SEED"] ) assert 1 == len( [ diff --git a/bbot/test/test_step_2/module_tests/test_module_robots.py b/bbot/test/test_step_2/module_tests/test_module_robots.py index 3d9156bb4c..10a4107375 100644 --- a/bbot/test/test_step_2/module_tests/test_module_robots.py +++ b/bbot/test/test_step_2/module_tests/test_module_robots.py @@ -22,7 +22,7 @@ def check(self, module_test, events): for e in events: if e.type == "URL_UNVERIFIED": - if str(e.module) != "TARGET": + if str(e.module) != "SEED": assert "spider-danger" in e.tags, f"{e} doesn't have spider-danger tag" if e.data == "http://127.0.0.1:8888/allow/": allow_bool = True diff --git a/bbot/test/test_step_2/module_tests/test_module_stdout.py b/bbot/test/test_step_2/module_tests/test_module_stdout.py index 27d8a30594..a77a2a3f89 100644 --- a/bbot/test/test_step_2/module_tests/test_module_stdout.py +++ b/bbot/test/test_step_2/module_tests/test_module_stdout.py @@ -9,7 +9,7 @@ class TestStdout(ModuleTestBase): def check(self, module_test, events): out, err = module_test.capsys.readouterr() assert out.startswith("[SCAN] \tteststdout") - assert "[DNS_NAME] \tblacklanternsecurity.com\tTARGET" in out + assert "[DNS_NAME] \tblacklanternsecurity.com\tSEED" in out class TestStdoutEventTypes(TestStdout): @@ -18,7 +18,7 @@ class TestStdoutEventTypes(TestStdout): def check(self, module_test, events): out, err = module_test.capsys.readouterr() assert len(out.splitlines()) == 1 - assert out.startswith("[DNS_NAME] \tblacklanternsecurity.com\tTARGET") + assert out.startswith("[DNS_NAME] \tblacklanternsecurity.com\tSEED") class TestStdoutEventFields(TestStdout): diff --git a/bbot/test/test_step_2/template_tests/test_template_subdomain_enum.py b/bbot/test/test_step_2/template_tests/test_template_subdomain_enum.py index bfa186707b..acb0f731ce 100644 --- a/bbot/test/test_step_2/template_tests/test_template_subdomain_enum.py +++ b/bbot/test/test_step_2/template_tests/test_template_subdomain_enum.py @@ -54,8 +54,8 @@ def check(self, module_test, events): class TestSubdomainEnumHighestParent(TestSubdomainEnum): - targets = ["api.test.asdf.www.blacklanternsecurity.com", "evilcorp.com"] - whitelist = ["www.blacklanternsecurity.com"] + seeds = ["api.test.asdf.www.blacklanternsecurity.com", "evilcorp.com"] + targets = ["www.blacklanternsecurity.com"] modules_overrides = ["speculate"] dedup_strategy = "highest_parent" txt = None @@ -71,8 +71,11 @@ def check(self, module_test, events): assert len(distance_1_dns_names) == 2 assert 1 == len([e for e in distance_1_dns_names if e.data == "evilcorp.com"]) assert 1 == len([e for e in distance_1_dns_names if e.data == "blacklanternsecurity.com"]) - assert len(self.queries) == 1 - assert self.queries[0] == "www.blacklanternsecurity.com" + + # Passive subdomain enum templates operate on all seeds, even when + # they are outside the explicit target_list. + # we expect one query for the blacklantern scope and one for the unrelated evilcorp.com seed. + assert set(self.queries) == {"www.blacklanternsecurity.com", "evilcorp.com"} class TestSubdomainEnumLowestParent(TestSubdomainEnumHighestParent): @@ -80,6 +83,7 @@ class TestSubdomainEnumLowestParent(TestSubdomainEnumHighestParent): def check(self, module_test, events): assert set(self.queries) == { + "evilcorp.com", "test.asdf.www.blacklanternsecurity.com", "asdf.www.blacklanternsecurity.com", "www.blacklanternsecurity.com", @@ -88,8 +92,8 @@ def check(self, module_test, events): class TestSubdomainEnumWildcardBaseline(ModuleTestBase): # oh walmart.cn why are you like this - targets = ["www.walmart.cn"] - whitelist = ["walmart.cn"] + targets = ["walmart.cn"] + seeds = ["www.walmart.cn"] modules_overrides = [] config_overrides = {"dns": {"minimal": False}, "scope": {"report_distance": 10}, "omit_event_types": []} dedup_strategy = "highest_parent" @@ -134,7 +138,7 @@ def check(self, module_test, events): for e in events if e.type == "DNS_NAME" and e.data == "www.walmart.cn" - and str(e.module) == "TARGET" + and str(e.module) == "SEED" and e.scope_distance == 0 ] ) @@ -163,6 +167,7 @@ def check(self, module_test, events): class TestSubdomainEnumWildcardDefense(TestSubdomainEnumWildcardBaseline): # oh walmart.cn why are you like this targets = ["walmart.cn"] + seeds = ["walmart.cn"] modules_overrides = [] config_overrides = {"dns": {"minimal": False}, "scope": {"report_distance": 10}} dedup_strategy = "highest_parent" @@ -195,7 +200,7 @@ def check(self, module_test, events): for e in events if e.type == "DNS_NAME" and e.data == "walmart.cn" - and str(e.module) == "TARGET" + and str(e.module) == "SEED" and e.scope_distance == 0 ] ) diff --git a/docs/dev/index.md b/docs/dev/index.md index 6315637f02..dbf48402e8 100644 --- a/docs/dev/index.md +++ b/docs/dev/index.md @@ -68,7 +68,7 @@ For more details, including which types of targets are valid, see [Targets](../s #### Other Custom Options -In many cases, using a [Preset](../scanning/presets.md) like `subdomain-enum` is sufficient. However, the `Scanner` is flexible and accepts many other arguments that can override the default functionality. You can specify [`flags`](../index.md#flags), [`modules`](../index.md#modules), [`output_modules`](../output.md), a [`whitelist` or `blacklist`](../scanning/index.md#whitelists-and-blacklists), and custom [`config` options](../scanning/configuration.md): +In many cases, using a [Preset](../scanning/presets.md) like `subdomain-enum` is sufficient. However, the `Scanner` is flexible and accepts many other arguments that can override the default functionality. You can specify [`flags`](../index.md#flags), [`modules`](../index.md#modules), [`output_modules`](../output.md), a [target list / `seeds` / `blacklist`](../scanning/index.md#targets-seeds-and-blacklists), and custom [`config` options](../scanning/configuration.md): ```python # create a scan against multiple targets @@ -78,8 +78,8 @@ scan = Scanner( "4.3.2.1", # enable these presets presets=["subdomain-enum"], - # whitelist these hosts - whitelist=["evilcorp.com", "evilcorp.org"], + # explicitly define in-scope targets + target=["evilcorp.com", "evilcorp.org"], # blacklist these hosts blacklist=["prod.evilcorp.com"], # also enable these individual modules diff --git a/docs/dev/target.md b/docs/dev/target.md index 6740cfb744..b5420801c7 100644 --- a/docs/dev/target.md +++ b/docs/dev/target.md @@ -2,7 +2,7 @@ ::: bbot.scanner.target.ScanSeeds -::: bbot.scanner.target.ScanWhitelist +::: bbot.scanner.target.ScanTarget ::: bbot.scanner.target.ScanBlacklist diff --git a/docs/scanning/advanced.md b/docs/scanning/advanced.md index 3af6daf74b..d66c55e166 100644 --- a/docs/scanning/advanced.md +++ b/docs/scanning/advanced.md @@ -56,12 +56,12 @@ options: Target: -t, --targets TARGET [TARGET ...] - Targets to seed the scan - -w, --whitelist WHITELIST [WHITELIST ...] - What's considered in-scope (by default it's the same as --targets) + Target scope (defines what is in-scope) + -s, --seeds SEEDS [SEEDS ...] + Define seeds to drive passive modules without being in scope (if not specified, defaults to same as targets) -b, --blacklist BLACKLIST [BLACKLIST ...] Don't touch these things - --strict-scope Don't consider subdomains of target/whitelist to be in-scope + --strict-scope Don't consider subdomains of targets to be in-scope Presets: -p, --preset [PRESET ...] diff --git a/docs/scanning/configuration.md b/docs/scanning/configuration.md index fd1397251e..b8d97e59d6 100644 --- a/docs/scanning/configuration.md +++ b/docs/scanning/configuration.md @@ -87,7 +87,7 @@ scope: ### DNS ### dns: - # Completely disable DNS resolution (careful if you have IP whitelists/blacklists, consider using minimal=true instead) + # Completely disable DNS resolution (careful if you are using IP-based targets/blacklists, consider using minimal=true instead) disable: false # Speed up scan by not creating any new DNS events, and only resolving A and AAAA records minimal: false diff --git a/docs/scanning/index.md b/docs/scanning/index.md index 1d31fa23c8..ac4228bf1f 100644 --- a/docs/scanning/index.md +++ b/docs/scanning/index.md @@ -177,24 +177,28 @@ bbot -t evilcorp.com -f subdomain-enum -c scope.report_distance=1 If you want to scan **_only_** that specific target hostname and none of its children, you can specify `--strict-scope`. -Note that `--strict-scope` only applies to targets and whitelists, but not blacklists. This means that if you put `internal.evilcorp.com` in your blacklist, you can be sure none of its subdomains will be scanned, even when using `--strict-scope`. +Note that `--strict-scope` only applies to targets, but not blacklists. This means that if you put `internal.evilcorp.com` in your blacklist, you can be sure none of its subdomains will be scanned, even when using `--strict-scope`. -### Whitelists and Blacklists +### Targets, Seeds, and Blacklists -BBOT allows precise control over scope with whitelists and blacklists. These both use the same syntax as `--target`, meaning they accept the same event types, and you can specify an unlimited number of them, via a file, the CLI, or both. +BBOT uses three related concepts to control scope and how a scan is driven: -#### Whitelists +- **Targets (`-t` / `--targets`)**: Define what is in-scope. These also act as scan seeds if seeds aren't explicitly defined. +- **Seeds (`-s` / `--seeds`)**: Seeds define the starting point for the scan. They drive **passive** modules and can be outside of the explicit target list (out of scope) for those passive modules. If you don’t specify `--seeds`, BBOT will automatically use your targets as seeds. +- **Blacklists (`-b` / `--blacklist`)**: Define what is **never** touched. Anything matching the blacklist is excluded from the scan, even if it would otherwise be in-scope. -`--whitelist` enables you to override what's in scope. For example, if you want to run nuclei against `evilcorp.com`, but stay only inside their corporate IP range of `1.2.3.0/24`, you can accomplish this like so: +This separation lets you, for example, keep a tight target list for what’s considered in-scope, while still allowing passive modules to discover new subdomains that may ultimately be in-scope. The blacklist helps to mask-off anything that you know should not be scanned. + +For example, lets say you have a target with subdomains that resolve both within, and outside of an IP range that defines your scope. You can set the IP range as the **target**, and then safely let BBOT explore the domain defined in **seeds**. Any discovered assets that are in your scope will automatically be assessed by active modules. ```bash -# Seed scan with evilcorp.com, but restrict scope to 1.2.3.0/24 -bbot -t evilcorp.com --whitelist 1.2.3.0/24 -f subdomain-enum -m portscan nuclei --allow-deadly +bbot -t 192.168.1.0/24 -s evilcorp.com -f subdomain-enum -m nuclei --allow-deadly ``` +In this example, any discovered `evilcorp.com` subdomains that resolve within `192.168.1.0/24` will be scanned by `Nuclei`. Any others will be discovered, but not touched by active modules. #### Blacklists -`--blacklist` takes ultimate precedence. Anything in the blacklist is completely excluded from the scan, even if it's in the whitelist. +`--blacklist` takes ultimate precedence. Anything in the blacklist is completely excluded from the scan, even if it would otherwise be in-scope based on your targets or seeds. ```bash # Scan evilcorp.com, but exclude internal.evilcorp.com and its children @@ -222,7 +226,7 @@ If you only want to blacklist the URL, you could narrow the regex like so: bbot -t evilcorp.com --blacklist 'RE:signout\.aspx$' ``` -Similar to targets and whitelists, blacklists can be specified in your preset. The `spider` preset makes use of this to prevent the spider from following logout links: +Similar to targets, blacklists can be specified in your preset. The `spider` preset makes use of this to prevent the spider from following logout links: ```yaml title="spider.yml" description: Recursive web spider @@ -277,4 +281,4 @@ dns: wildcard_tests: 20 ``` -If that doesn't work you can consider [blacklisting](#whitelists-and-blacklists) the offending domain. +If that doesn't work you can consider [blacklisting](#targets-seeds-and-blacklists) the offending domain. diff --git a/docs/scanning/tips_and_tricks.md b/docs/scanning/tips_and_tricks.md index f91708dd35..f8d1acd9e6 100644 --- a/docs/scanning/tips_and_tricks.md +++ b/docs/scanning/tips_and_tricks.md @@ -143,7 +143,7 @@ If you already have a list of discovered targets (e.g. URLs), you can speed up t bbot -m httpx gowitness wappalyzer -t urls.txt -c dns.disable=true ~~~ -Note that the above setting _completely_ disables DNS resolution, meaning even `A` and `AAAA` records are not resolved. This can cause problems if you're using an IP whitelist or blacklist. In this case, you'll want to use `dns.minimal` instead: +Note that the above setting _completely_ disables DNS resolution, meaning even `A` and `AAAA` records are not resolved. This can cause problems if you're relying on IP-based targets or blacklists. In this case, you'll want to use `dns.minimal` instead: ~~~bash # only resolve A and AAAA records