Validate Before Bake

bbot/cli.py

@@ -102,6 +102,7 @@ async def _main():
                 preset._default_internal_modules = []
 
             # Bake a temporary copy of the preset so that flags correctly enable their associated modules before listing them
+            preset.validate()
             preset = preset.bake()
 
             # --list-modules
@@ -156,6 +157,7 @@ async def _main():
                 print(row)
             return
 
+        preset.validate()
         baked_preset = preset.bake()
 
         # --current-preset / --current-preset-full

bbot/core/config/models.py

@@ -13,6 +13,7 @@
 
 from __future__ import annotations
 
+import os
 from typing import Annotated, Any, Literal, Optional
 
 from pydantic import BaseModel, BeforeValidator, ConfigDict, field_validator
@@ -223,7 +224,9 @@ def coerce_value(value, accepted):
     if "str" in accepted and not (accepted & _COLLECTION_NAMES):
         if is_raw:
             return value
-        return None if value is None else str(value)
+        if value is None or isinstance(value, (list, dict, set)):
+            return value
+        return str(value)
     if accepted == frozenset({"bool"}):
         v = _yaml_scalar(value) if is_raw else value
         if isinstance(v, bool):
@@ -237,7 +240,11 @@ def coerce_value(value, accepted):
             if low in _FALSE_WORDS:
                 return False
         return v
-    return _yaml_scalar(value) if is_raw else value
+    if is_raw:
+        return _yaml_scalar(value)
+    if "str" in accepted and isinstance(value, os.PathLike):
+        return str(value)
+    return value
 
 
 def coerce_config(config, index, prefix=""):
@@ -462,6 +469,7 @@ class BBOTConfig(BaseModel):
     aggregate: Optional[bool] = None
     dnsresolve: Optional[bool] = None
     cloudcheck: Optional[bool] = None
+    unarchive: Optional[bool] = None
 
     # URL handling
     url_querystring_remove: Optional[bool] = None

bbot/core/helpers/names_generator.py

@@ -237,6 +237,7 @@
     "rapid_unscheduled",
     "raving",
     "reckless",
+    "recursive",
     "reductive",
     "ripped",
     "ruthless",
@@ -720,6 +721,7 @@
     "theoden",
     "theon",
     "theresa",
+    "thetechromancer",
     "thomas",
     "tiffany",
     "timothy",

bbot/modules/bevigil.py

@@ -17,7 +17,7 @@ class bevigil(subdomain_enum_apikey):
     }
 
     class Config(BaseModuleConfig):
-        api_key: str = Field("", description="BeVigil OSINT API Key", sensitive=True, mandatory=True)
+        api_key: str | list[str] = Field("", description="BeVigil OSINT API Key", sensitive=True, mandatory=True)
         urls: bool = Field(False, description="Emit URLs in addition to DNS_NAMEs")
 
     base_url = "https://osint.bevigil.com/api"

bbot/modules/bufferoverrun.py

@@ -13,7 +13,7 @@ class BufferOverrun(subdomain_enum_apikey):
     }
 
     class Config(BaseModuleConfig):
-        api_key: str = Field("", description="BufferOverrun API key", sensitive=True, mandatory=True)
+        api_key: str | list[str] = Field("", description="BufferOverrun API key", sensitive=True, mandatory=True)
         commercial: bool = Field(False, description="Use commercial API")
 
     base_url = "https://tls.bufferover.run/dns"

bbot/modules/builtwith.py

@@ -25,7 +25,7 @@ class builtwith(subdomain_enum_apikey):
     }
 
     class Config(BaseModuleConfig):
-        api_key: str = Field("", description="Builtwith API key", sensitive=True, mandatory=True)
+        api_key: str | list[str] = Field("", description="Builtwith API key", sensitive=True, mandatory=True)
         redirects: bool = Field(True, description="Also look up inbound and outbound redirects")
 
     base_url = "https://api.builtwith.com"

bbot/modules/c99.py

@@ -13,7 +13,7 @@ class c99(subdomain_enum_apikey):
     }
 
     class Config(BaseModuleConfig):
-        api_key: str = Field("", description="c99.nl API key", sensitive=True, mandatory=True)
+        api_key: str | list[str] = Field("", description="c99.nl API key", sensitive=True, mandatory=True)
 
     base_url = "https://api.c99.nl"
     ping_url = f"{base_url}/randomnumber?key={{api_key}}&between=1,100&json"

bbot/modules/censys_dns.py

@@ -18,7 +18,7 @@ class censys_dns(censys):
     }
 
     class Config(BaseModuleConfig):
-        api_key: str = Field(
+        api_key: str | list[str] = Field(
             "", description="Censys.io API Key in the format of 'key:secret'", sensitive=True, mandatory=True
         )
         max_pages: int = Field(5, description="Maximum number of pages to fetch (100 results per page)")

bbot/modules/censys_ip.py

@@ -25,7 +25,7 @@ class censys_ip(censys):
     }
 
     class Config(BaseModuleConfig):
-        api_key: str = Field(
+        api_key: str | list[str] = Field(
             "", description="Censys.io API Key in the format of 'key:secret'", sensitive=True, mandatory=True
         )
         dns_names_limit: int = Field(

bbot/modules/chaos.py

@@ -13,7 +13,7 @@ class chaos(subdomain_enum_apikey):
     }
 
     class Config(BaseModuleConfig):
-        api_key: str = Field("", description="Chaos API key", sensitive=True, mandatory=True)
+        api_key: str | list[str] = Field("", description="Chaos API key", sensitive=True, mandatory=True)
 
     base_url = "https://dns.projectdiscovery.io/dns"
     ping_url = f"{base_url}/example.com"

bbot/modules/dehashed.py

@@ -15,7 +15,7 @@ class dehashed(subdomain_enum):
     }
 
     class Config(BaseModuleConfig):
-        api_key: str = Field("", description="DeHashed API Key", sensitive=True, mandatory=True)
+        api_key: str | list[str] = Field("", description="DeHashed API Key", sensitive=True, mandatory=True)
 
     target_only = True
 

bbot/modules/fullhunt.py

@@ -13,7 +13,7 @@ class fullhunt(subdomain_enum_apikey):
     }
 
     class Config(BaseModuleConfig):
-        api_key: str = Field("", description="FullHunt API Key", sensitive=True, mandatory=True)
+        api_key: str | list[str] = Field("", description="FullHunt API Key", sensitive=True, mandatory=True)
 
     base_url = "https://fullhunt.io/api/v1"
 

bbot/modules/git_clone.py

@@ -15,7 +15,7 @@ class git_clone(github):
     }
 
     class Config(BaseModuleConfig):
-        api_key: str = Field("", description="Github token", sensitive=True)
+        api_key: str | list[str] = Field("", description="Github token", sensitive=True)
         output_folder: str = Field(
             "",
             description="Folder to clone repositories to. If not specified, cloned repositories will be deleted when the scan completes, to minimize disk usage.",

bbot/modules/github_codesearch.py

@@ -14,7 +14,7 @@ class github_codesearch(github, subdomain_enum):
     }
 
     class Config(BaseModuleConfig):
-        api_key: str = Field("", description="Github token", sensitive=True, mandatory=True)
+        api_key: str | list[str] = Field("", description="Github token", sensitive=True, mandatory=True)
         limit: int = Field(100, description="Limit code search to this many results")
 
     github_raw_url = "https://raw.githubusercontent.com/"

bbot/modules/github_org.py

@@ -13,7 +13,7 @@ class github_org(github):
     }
 
     class Config(BaseModuleConfig):
-        api_key: str = Field("", description="Github token", sensitive=True)
+        api_key: str | list[str] = Field("", description="Github token", sensitive=True)
         include_members: bool = Field(True, description="Enumerate organization members")
         include_member_repos: bool = Field(False, description="Also enumerate organization members' repositories")
 

bbot/modules/github_usersearch.py

@@ -14,7 +14,7 @@ class github_usersearch(github, subdomain_enum):
     }
 
     class Config(BaseModuleConfig):
-        api_key: str = Field("", description="Github token", sensitive=True, mandatory=True)
+        api_key: str | list[str] = Field("", description="Github token", sensitive=True, mandatory=True)
 
     async def handle_event(self, event):
         self.verbose("Searching for users with emails matching in scope domains")

bbot/modules/github_workflows.py

@@ -17,7 +17,7 @@ class github_workflows(github):
     }
 
     class Config(BaseModuleConfig):
-        api_key: str = Field("", description="Github token", sensitive=True, mandatory=True)
+        api_key: str | list[str] = Field("", description="Github token", sensitive=True, mandatory=True)
         num_logs: int = Field(1, description="For each workflow fetch the last N successful runs logs (max 100)")
         output_folder: str = Field("", description="Folder to download workflow logs and artifacts to")
 

bbot/modules/gitlab_com.py

@@ -15,7 +15,9 @@ class gitlab_com(GitLabBaseModule):
     }
 
     class Config(BaseModuleConfig):
-        api_key: str = Field("", description="GitLab access token (for gitlab.com/org only)", sensitive=True)
+        api_key: str | list[str] = Field(
+            "", description="GitLab access token (for gitlab.com/org only)", sensitive=True
+        )
 
     # This is needed because we are consuming SOCIAL events, which aren't in scope
     scope_distance_modifier = 2

bbot/modules/gitlab_onprem.py

@@ -20,7 +20,9 @@ class gitlab_onprem(GitLabBaseModule):
     # Optional GitLab access token (only required for gitlab.com, but still
     # supported for on-prem installations that expose private projects).
     class Config(BaseModuleConfig):
-        api_key: str = Field("", description="GitLab access token (for self-hosted instances only)", sensitive=True)
+        api_key: str | list[str] = Field(
+            "", description="GitLab access token (for self-hosted instances only)", sensitive=True
+        )
 
     # Allow accepting events slightly beyond configured max distance so we can
     # discover repos on neighbouring infrastructure.

bbot/modules/hunterio.py

@@ -9,7 +9,7 @@ class hunterio(subdomain_enum_apikey):
     meta = {"description": "Query hunter.io for emails", "created_date": "2022-04-25", "author": "@TheTechromancer"}
 
     class Config(BaseModuleConfig):
-        api_key: str = Field("", description="Hunter.IO API key", sensitive=True, mandatory=True)
+        api_key: str | list[str] = Field("", description="Hunter.IO API key", sensitive=True, mandatory=True)
 
     base_url = "https://api.hunter.io/v2"
     ping_url = f"{base_url}/account?api_key={{api_key}}"

bbot/modules/ip2location.py

@@ -17,7 +17,7 @@ class IP2Location(BaseModule):
     }
 
     class Config(BaseModuleConfig):
-        api_key: str = Field("", description="IP2location.io API Key", sensitive=True, mandatory=True)
+        api_key: str | list[str] = Field("", description="IP2location.io API Key", sensitive=True, mandatory=True)
         lang: str = Field(
             "",
             description="Translation information(ISO639-1). The translation is only applicable for continent, country, region and city name.",

bbot/modules/ipstack.py

@@ -14,7 +14,7 @@ class Ipstack(BaseModule):
     meta = {"description": "Query IPStack's GeoIP API", "created_date": "2022-11-26", "author": "@tycoonslive"}
 
     class Config(BaseModuleConfig):
-        api_key: str = Field("", description="IPStack GeoIP API Key", sensitive=True, mandatory=True)
+        api_key: str | list[str] = Field("", description="IPStack GeoIP API Key", sensitive=True, mandatory=True)
 
     scope_distance_modifier = 1
     _priority = 2

bbot/modules/leakix.py

@@ -8,7 +8,7 @@ class leakix(subdomain_enum_apikey):
     flags = ["safe", "subdomain-enum", "passive"]
 
     class Config(BaseModuleConfig):
-        api_key: str = Field("", description="LeakIX API Key", sensitive=True, mandatory=True)
+        api_key: str | list[str] = Field("", description="LeakIX API Key", sensitive=True, mandatory=True)
 
     meta = {
         "description": "Query leakix.net for subdomains",

bbot/modules/otx.py

@@ -13,7 +13,7 @@ class otx(subdomain_enum_apikey):
     }
 
     class Config(BaseModuleConfig):
-        api_key: str = Field("", description="OTX API key", sensitive=True, mandatory=True)
+        api_key: str | list[str] = Field("", description="OTX API key", sensitive=True, mandatory=True)
 
     base_url = "https://otx.alienvault.com"
 

bbot/modules/postman.py

@@ -13,7 +13,7 @@ class postman(postman):
     }
 
     class Config(BaseModuleConfig):
-        api_key: str = Field("", description="Postman API Key", sensitive=True, mandatory=True)
+        api_key: str | list[str] = Field("", description="Postman API Key", sensitive=True, mandatory=True)
 
     reject_wildcards = False
 

bbot/modules/postman_download.py

@@ -20,7 +20,7 @@ class Config(BaseModuleConfig):
             "",
             description="Folder to download postman workspaces to. If not specified, downloaded workspaces will be deleted when the scan completes, to minimize disk usage.",
         )
-        api_key: str = Field("", description="Postman API Key", sensitive=True, mandatory=True)
+        api_key: str | list[str] = Field("", description="Postman API Key", sensitive=True, mandatory=True)
 
     scope_distance_modifier = 2
 

bbot/modules/securitytrails.py

@@ -13,7 +13,7 @@ class securitytrails(subdomain_enum_apikey):
     }
 
     class Config(BaseModuleConfig):
-        api_key: str = Field("", description="SecurityTrails API key", sensitive=True, mandatory=True)
+        api_key: str | list[str] = Field("", description="SecurityTrails API key", sensitive=True, mandatory=True)
 
     base_url = "https://api.securitytrails.com/v1"
     ping_url = f"{base_url}/ping?apikey={{api_key}}"

bbot/modules/shodan_dns.py

@@ -9,7 +9,7 @@ class shodan_dns(shodan):
     meta = {"description": "Query Shodan for subdomains", "created_date": "2022-07-03", "author": "@TheTechromancer"}
 
     class Config(BaseModuleConfig):
-        api_key: str = Field("", description="Shodan API key", sensitive=True, mandatory=True)
+        api_key: str | list[str] = Field("", description="Shodan API key", sensitive=True, mandatory=True)
 
     base_url = "https://api.shodan.io"
 

bbot/modules/shodan_enterprise.py

@@ -13,7 +13,7 @@ class shodan_enterprise(BaseModule):
     }
 
     class Config(BaseModuleConfig):
-        api_key: str = Field("", description="Shodan API Key", sensitive=True, mandatory=True)
+        api_key: str | list[str] = Field("", description="Shodan API Key", sensitive=True, mandatory=True)
         in_scope_only: bool = Field(
             True, description="Only query in-scope IPs. If False, will query up to distance 1."
         )

bbot/modules/subdomainradar.py

@@ -18,7 +18,7 @@ class SubdomainRadar(subdomain_enum_apikey):
     }
 
     class Config(BaseModuleConfig):
-        api_key: str = Field("", description="SubDomainRadar.io API key", sensitive=True, mandatory=True)
+        api_key: str | list[str] = Field("", description="SubDomainRadar.io API key", sensitive=True, mandatory=True)
         group: Literal["fast", "medium", "deep"] = Field(
             "fast", description="The enumeration group to use. Choose from fast, medium, deep"
         )

bbot/modules/trickest.py

@@ -9,7 +9,7 @@ class Trickest(subdomain_enum_apikey):
     meta = {"description": "Query Trickest's API for subdomains", "author": "@amiremami", "created_date": "2024-07-27"}
 
     class Config(BaseModuleConfig):
-        api_key: str = Field("", description="Trickest API key", sensitive=True, mandatory=True)
+        api_key: str | list[str] = Field("", description="Trickest API key", sensitive=True, mandatory=True)
 
     base_url = "https://api.trickest.io/solutions/v1/public/solution/a7cba1f1-df07-4a5c-876a-953f178996be"
     ping_url = f"{base_url}/dataset"

bbot/modules/virustotal.py

@@ -13,7 +13,7 @@ class virustotal(subdomain_enum_apikey):
     }
 
     class Config(BaseModuleConfig):
-        api_key: str = Field("", description="VirusTotal API Key", sensitive=True, mandatory=True)
+        api_key: str | list[str] = Field("", description="VirusTotal API Key", sensitive=True, mandatory=True)
 
     base_url = "https://www.virustotal.com/api/v3"
     api_page_iter_kwargs = {"json": False, "next_key": lambda r: r.json().get("links", {}).get("next", "")}

bbot/modules/wpscan.py

@@ -14,7 +14,7 @@ class wpscan(BaseModule):
     }
 
     class Config(BaseModuleConfig):
-        api_key: str = Field("", description="WPScan API Key", sensitive=True)
+        api_key: str | list[str] = Field("", description="WPScan API Key", sensitive=True)
         enumerate: str = Field(
             "vp,vt,cb,dbe", description="Enumeration Process see wpscan help documentation (default: vp,vt,cb,dbe)"
         )