diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c946199c9a..af196d5518 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -58,6 +58,11 @@ jobs: - uses: actions/checkout@v5 with: fetch-depth: 0 + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} - name: Set up Python uses: actions/setup-python@v6 with: @@ -83,24 +88,59 @@ jobs: uses: elgohr/Publish-Docker-Github-Action@v5 with: name: blacklanternsecurity/bbot - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} tags: "latest,dev,${{ steps.version.outputs.BBOT_VERSION }}" - name: Publish to Docker Hub (stable) if: github.event_name == 'push' && github.ref == 'refs/heads/stable' uses: elgohr/Publish-Docker-Github-Action@v5 with: name: blacklanternsecurity/bbot - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} tags: "stable,${{ steps.version.outputs.BBOT_VERSION }}" + - name: Publish Full Docker Image to Docker Hub (dev) + if: github.event_name == 'push' && github.ref == 'refs/heads/dev' + uses: elgohr/Publish-Docker-Github-Action@v5 + with: + name: blacklanternsecurity/bbot + dockerfile: Dockerfile.full + tags: "latest-full,dev-full,${{ steps.version.outputs.BBOT_VERSION }}-full" + - name: Publish Full Docker Image to Docker Hub (stable) + if: github.event_name == 'push' && github.ref == 'refs/heads/stable' + uses: elgohr/Publish-Docker-Github-Action@v5 + with: + name: blacklanternsecurity/bbot + dockerfile: Dockerfile.full + tags: "stable-full,${{ steps.version.outputs.BBOT_VERSION }}-full" - name: Docker Hub Description if: github.event_name == 'push' && github.ref == 'refs/heads/dev' - uses: peter-evans/dockerhub-description@v4 + uses: peter-evans/dockerhub-description@v5 with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} repository: blacklanternsecurity/bbot + - name: Clean up old Docker Hub tags (up to 50 most recent tags plus 'latest') + if: github.event_name == 'push' && github.ref == 'refs/heads/dev' + run: | + # Install jq for JSON processing + sudo apt-get update && sudo apt-get install -y jq + + IMAGE="blacklanternsecurity/bbot" + + # Clean up dev tags (keep 50 most recent) + for tag_pattern in "rc$" "rc-full$"; do + echo "Cleaning up tags ending with $tag_pattern..." + + tags_response=$(curl -s -H "Authorization: Bearer ${{ secrets.DOCKER_TOKEN }}" \ + "https://hub.docker.com/v2/repositories/$IMAGE/tags/?page_size=100") + + tags_to_delete=$(echo "$tags_response" | jq -r --arg pattern "$tag_pattern" \ + '.results[] | select(.name | test($pattern)) | [.last_updated, .name] | @tsv' | \ + sort -r | tail -n +51 | cut -f2) + + for tag in $tags_to_delete; do + echo "Deleting $IMAGE tag: $tag" + curl -X DELETE -H "Authorization: Bearer ${{ secrets.DOCKER_TOKEN }}" \ + "https://hub.docker.com/v2/repositories/$IMAGE/tags/$tag/" + done + + echo "Cleanup completed for tags ending with $tag_pattern. Kept 50 most recent." + done outputs: BBOT_VERSION: ${{ steps.version.outputs.BBOT_VERSION }} publish_docs: diff --git a/Dockerfile b/Dockerfile index e893c7fb1a..6e7c29c7c6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.10-slim +FROM python:3.11-slim ENV LANG=C.UTF-8 ENV LC_ALL=C.UTF-8 diff --git a/Dockerfile.full b/Dockerfile.full new file mode 100644 index 0000000000..776a2416ed --- /dev/null +++ b/Dockerfile.full @@ -0,0 +1,19 @@ +FROM python:3.11-slim + +ENV LANG=C.UTF-8 +ENV LC_ALL=C.UTF-8 +ENV PIP_NO_CACHE_DIR=off + +WORKDIR /usr/src/bbot + +RUN apt-get update && apt-get install -y openssl gcc git make unzip curl wget vim nano sudo + +COPY . . + +RUN pip install . + +RUN bbot --install-all-deps + +WORKDIR /root + +ENTRYPOINT [ "bbot" ] diff --git a/bbot/cli.py b/bbot/cli.py index 333ab8c202..23703edfd7 100755 --- a/bbot/cli.py +++ b/bbot/cli.py @@ -7,7 +7,7 @@ from bbot.errors import * from bbot import __version__ from bbot.logger import log_to_stderr -from bbot.core.helpers.misc import chain_lists +from bbot.core.helpers.misc import chain_lists, rm_rf if multiprocessing.current_process().name == "MainProcess": @@ -173,13 +173,27 @@ async def _main(): # --install-all-deps if options.install_all_deps: - all_modules = list(preset.module_loader.preloaded()) - scan.helpers.depsinstaller.force_deps = True - succeeded, failed = await scan.helpers.depsinstaller.install(*all_modules) - if failed: - log.hugewarning(f"Failed to install dependencies for the following modules: {', '.join(failed)}") + preloaded_modules = preset.module_loader.preloaded() + scan_modules = [k for k, v in preloaded_modules.items() if str(v.get("type", "")) == "scan"] + output_modules = [k for k, v in preloaded_modules.items() if str(v.get("type", "")) == "output"] + log.verbose("Creating dummy scan with all modules + output modules for deps installation") + dummy_scan = Scanner(preset=preset, modules=scan_modules, output_modules=output_modules) + dummy_scan.helpers.depsinstaller.force_deps = True + log.info("Installing module dependencies") + await dummy_scan.load_modules() + log.verbose("Running module setups") + succeeded, hard_failed, soft_failed = await dummy_scan.setup_modules(deps_only=True) + # remove any leftovers from the dummy scan + rm_rf(dummy_scan.home, ignore_errors=True) + rm_rf(dummy_scan.temp_dir, ignore_errors=True) + if succeeded: + log.success( + f"Successfully installed dependencies for {len(succeeded):,} modules: {','.join(succeeded)}" + ) + if soft_failed or hard_failed: + failed = soft_failed + hard_failed + log.warning(f"Failed to install dependencies for {len(failed):,} modules: {', '.join(failed)}") return False - log.hugesuccess(f"Successfully installed dependencies for the following modules: {', '.join(succeeded)}") return True scan_name = str(scan.name) diff --git a/bbot/core/helpers/web/web.py b/bbot/core/helpers/web/web.py index 5e86424049..60ff35dd59 100644 --- a/bbot/core/helpers/web/web.py +++ b/bbot/core/helpers/web/web.py @@ -267,7 +267,8 @@ async def wordlist(self, path, lines=None, zip=False, zip_filename=None, **kwarg if not path: raise WordlistError(f"Invalid wordlist: {path}") if "cache_hrs" not in kwargs: - kwargs["cache_hrs"] = 720 + # 4320 hrs = 180 days = 6 months + kwargs["cache_hrs"] = 4320 if self.parent_helper.is_url(path): filename = await self.download(str(path), **kwargs) if filename is None: diff --git a/bbot/core/modules.py b/bbot/core/modules.py index a691e57792..b6165aaef3 100644 --- a/bbot/core/modules.py +++ b/bbot/core/modules.py @@ -56,7 +56,6 @@ def __init__(self): self._shared_deps = dict(SHARED_DEPS) self.__preloaded = {} - self._modules = {} self._configs = {} self.flag_choices = set() self.all_module_choices = set() @@ -463,7 +462,6 @@ def load_modules(self, module_names): for module_name in module_names: module = self.load_module(module_name) modules[module_name] = module - self._modules[module_name] = module return modules def load_module(self, module_name): diff --git a/bbot/modules/base.py b/bbot/modules/base.py index 40da917cbe..63ae43f31b 100644 --- a/bbot/modules/base.py +++ b/bbot/modules/base.py @@ -213,6 +213,14 @@ async def setup(self): return True + async def setup_deps(self): + """ + Similar to setup(), but reserved for installing dependencies not covered by Ansible. + + This should always be used to install static dependencies like AI models, wordlists, etc. + """ + return True + async def handle_event(self, event, **kwargs): """Asynchronously handles incoming events that the module is configured to watch. @@ -620,39 +628,26 @@ def start(self): name=f"{self.scan.name}.{self.name}._event_handler_watchdog()", ) - async def _setup(self): - """ - Asynchronously sets up the module by invoking its 'setup()' method. - - This method catches exceptions during setup, sets the module's error state if necessary, and determines the - status code based on the result of the setup process. - - Args: - None - - Returns: - tuple: A tuple containing the module's name, status (True for success, False for hard-fail, None for soft-fail), - and an optional status message. - - Raises: - Exception: Captured exceptions from the 'setup()' method are logged, but not propagated. - - Notes: - - The 'setup()' method can return either a simple boolean status or a tuple of status and message. - - A WordlistError exception triggers a soft-fail status. - - The debug log will contain setup status information for the module. - """ + async def _setup(self, deps_only=False): + """ """ status_codes = {False: "hard-fail", None: "soft-fail", True: "success"} status = False self.debug(f"Setting up module {self.name}") try: - result = await self.setup() - if type(result) == tuple and len(result) == 2: - status, msg = result - else: - status = result - msg = status_codes[status] + funcs = [self.setup_deps] + if not deps_only: + funcs.append(self.setup) + for func in funcs: + self.debug(f"Running {self.name}.{func.__name__}()") + result = await func() + if type(result) == tuple and len(result) == 2: + status, msg = result + else: + status = result + msg = status_codes[status] + if status is False: + break self.debug(f"Finished setting up module {self.name}") except Exception as e: self.set_error_state(f"Unexpected error during module setup: {e}", critical=True) diff --git a/bbot/modules/dnsbrute.py b/bbot/modules/dnsbrute.py index 3b847933c4..4c57d1feda 100644 --- a/bbot/modules/dnsbrute.py +++ b/bbot/modules/dnsbrute.py @@ -23,9 +23,14 @@ class dnsbrute(subdomain_enum): dedup_strategy = "lowest_parent" _qsize = 10000 + async def setup_deps(self): + self.subdomain_file = await self.helpers.wordlist(self.config.get("wordlist")) + # tell the dnsbrute helper to fetch the resolver file + await self.helpers.dns.brute.resolver_file() + return True + async def setup(self): self.max_depth = max(1, self.config.get("max_depth", 5)) - self.subdomain_file = await self.helpers.wordlist(self.config.get("wordlist")) self.subdomain_list = set(self.helpers.read_file(self.subdomain_file)) self.wordlist_size = len(self.subdomain_list) return await super().setup() diff --git a/bbot/modules/ffuf.py b/bbot/modules/ffuf.py index 81e114c7e7..038aa5d268 100644 --- a/bbot/modules/ffuf.py +++ b/bbot/modules/ffuf.py @@ -37,12 +37,15 @@ class ffuf(BaseModule): in_scope_only = True + async def setup_deps(self): + self.wordlist = await self.helpers.wordlist(self.config.get("wordlist")) + return True + async def setup(self): self.proxy = self.scan.web_config.get("http_proxy", "") self.canary = "".join(random.choice(string.ascii_lowercase) for i in range(10)) wordlist_url = self.config.get("wordlist", "") self.debug(f"Using wordlist [{wordlist_url}]") - self.wordlist = await self.helpers.wordlist(wordlist_url) self.wordlist_lines = self.generate_wordlist(self.wordlist) self.tempfile, tempfile_len = self.generate_templist() self.rate = self.config.get("rate", 0) diff --git a/bbot/modules/ffuf_shortnames.py b/bbot/modules/ffuf_shortnames.py index db44ed1108..ee6e040095 100644 --- a/bbot/modules/ffuf_shortnames.py +++ b/bbot/modules/ffuf_shortnames.py @@ -87,14 +87,17 @@ def find_common_prefixes(strings, minimum_set_length=4): found_prefixes.add(prefix) return list(found_prefixes) - async def setup(self): - self.proxy = self.scan.web_config.get("http_proxy", "") - self.canary = "".join(random.choice(string.ascii_lowercase) for i in range(10)) + async def setup_deps(self): wordlist_extensions = self.config.get("wordlist_extensions", "") if not wordlist_extensions: wordlist_extensions = f"{self.helpers.wordlist_dir}/raft-small-extensions-lowercase_CLEANED.txt" self.debug(f"Using [{wordlist_extensions}] for shortname candidate extension list") self.wordlist_extensions = await self.helpers.wordlist(wordlist_extensions) + return True + + async def setup(self): + self.proxy = self.scan.web_config.get("http_proxy", "") + self.canary = "".join(random.choice(string.ascii_lowercase) for i in range(10)) self.ignore_redirects = self.config.get("ignore_redirects") self.max_predictions = self.config.get("max_predictions") self.find_subwords = self.config.get("find_subwords") diff --git a/bbot/modules/filedownload.py b/bbot/modules/filedownload.py index c5b0b17c18..cdf8deb163 100644 --- a/bbot/modules/filedownload.py +++ b/bbot/modules/filedownload.py @@ -94,6 +94,12 @@ class filedownload(BaseModule): scope_distance_modifier = 3 + async def setup_deps(self): + self.mime_db_file = await self.helpers.wordlist( + "https://raw.githubusercontent.com/jshttp/mime-db/master/db.json" + ) + return True + async def setup(self): self.extensions = list({e.lower().strip(".") for e in self.config.get("extensions", [])}) self.max_filesize = self.config.get("max_filesize", "10MB") @@ -105,9 +111,6 @@ async def setup(self): else: self.download_dir = self.scan.temp_dir / "filedownload" self.helpers.mkdir(self.download_dir) - self.mime_db_file = await self.helpers.wordlist( - "https://raw.githubusercontent.com/jshttp/mime-db/master/db.json" - ) self.mime_db = {} with open(self.mime_db_file) as f: mime_db = json.load(f) diff --git a/bbot/modules/medusa.py b/bbot/modules/medusa.py index 46fbc1ae2e..e8814b5c2b 100644 --- a/bbot/modules/medusa.py +++ b/bbot/modules/medusa.py @@ -1,6 +1,5 @@ import re from bbot.modules.base import BaseModule -from bbot.errors import WordlistError class medusa(BaseModule): @@ -102,13 +101,11 @@ class medusa(BaseModule): }, ] - async def setup(self): - # Try to cache wordlist - try: - self.snmp_wordlist_path = await self.helpers.wordlist(self.config.get("snmp_wordlist")) - except WordlistError as e: - return False, f"Error retrieving wordlist: {e}" + async def setup_deps(self): + self.snmp_wordlist_path = await self.helpers.wordlist(self.config.get("snmp_wordlist")) + return True + async def setup(self): self.password_match_regex = re.compile(r"Password:\s*(\S+)") self.success_indicator_match_regex = re.compile(r"\[([^\]]+)\]\s*$") diff --git a/bbot/modules/paramminer_headers.py b/bbot/modules/paramminer_headers.py index e3b3e4aa3f..d3cbeb0661 100644 --- a/bbot/modules/paramminer_headers.py +++ b/bbot/modules/paramminer_headers.py @@ -82,18 +82,21 @@ class paramminer_headers(BaseModule): header_regex = re.compile(r"^[!#$%&\'*+\-.^_`|~0-9a-zA-Z]+: [^\r\n]+$") - async def setup(self): - self.recycle_words = self.config.get("recycle_words", True) - self.event_dict = {} - self.already_checked = set() + async def setup_deps(self): wordlist = self.config.get("wordlist", "") if not wordlist: wordlist = f"{self.helpers.wordlist_dir}/{self.default_wordlist}" + self.wordlist_file = await self.helpers.wordlist(wordlist) self.debug(f"Using wordlist: [{wordlist}]") + return True + + async def setup(self): + self.recycle_words = self.config.get("recycle_words", True) + self.event_dict = {} + self.already_checked = set() + self.wl = { - h.strip().lower() - for h in self.helpers.read_file(await self.helpers.wordlist(wordlist)) - if len(h) > 0 and "%" not in h + h.strip().lower() for h in self.helpers.read_file(self.wordlist_file) if len(h) > 0 and "%" not in h } # check against the boring list (if the option is set) diff --git a/bbot/modules/trufflehog.py b/bbot/modules/trufflehog.py index 0fe9ee3e36..dcd71f6b07 100644 --- a/bbot/modules/trufflehog.py +++ b/bbot/modules/trufflehog.py @@ -41,11 +41,14 @@ class trufflehog(BaseModule): scope_distance_modifier = 2 - async def setup(self): - self.verified = self.config.get("only_verified", True) + async def setup_deps(self): self.config_file = self.config.get("config", "") if self.config_file: self.config_file = await self.helpers.wordlist(self.config_file) + return True + + async def setup(self): + self.verified = self.config.get("only_verified", True) self.concurrency = int(self.config.get("concurrency", 8)) self.deleted_forks = self.config.get("deleted_forks", False) diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py index b5269bf753..a7b59bb117 100644 --- a/bbot/scanner/scanner.py +++ b/bbot/scanner/scanner.py @@ -484,7 +484,7 @@ def _start_modules(self): for module in self.modules.values(): module.start() - async def setup_modules(self, remove_failed=True): + async def setup_modules(self, remove_failed=True, deps_only=False): """Asynchronously initializes all loaded modules by invoking their `setup()` methods. Args: @@ -509,7 +509,7 @@ async def setup_modules(self, remove_failed=True): hard_failed = [] soft_failed = [] - async for task in self.helpers.as_completed([m._setup() for m in self.modules.values()]): + async for task in self.helpers.as_completed([m._setup(deps_only=deps_only) for m in self.modules.values()]): module, status, msg = await task if status is True: self.debug(f"Setup succeeded for {module.name} ({msg})") diff --git a/bbot/test/test_step_1/test_modules_basic.py b/bbot/test/test_step_1/test_modules_basic.py index 07b4f6692d..529dd64d75 100644 --- a/bbot/test/test_step_1/test_modules_basic.py +++ b/bbot/test/test_step_1/test_modules_basic.py @@ -342,6 +342,31 @@ async def test_modules_basic_perdomainonly(bbot_scanner, monkeypatch): await per_domain_scan._cleanup() +@pytest.mark.asyncio +async def test_modules_basic_setup_deps(bbot_scanner): + from bbot.modules.base import BaseModule + + class dummy(BaseModule): + _name = "dummy" + deps_ran = False + setup_ran = False + + async def setup_deps(self): + self.deps_ran = True + return True + + async def setup(self): + self.setup_ran = True + return True + + scan = bbot_scanner() + scan.modules["dummy"] = dummy(scan) + await scan.setup_modules(deps_only=True) + assert scan.modules["dummy"].deps_ran + assert not scan.modules["dummy"].setup_ran + await scan._cleanup() + + @pytest.mark.asyncio async def test_modules_basic_stats(helpers, events, bbot_scanner, httpx_mock, monkeypatch): from bbot.modules.base import BaseModule diff --git a/docs/dev/module_howto.md b/docs/dev/module_howto.md index de5bdd90b2..ec6be65bd3 100644 --- a/docs/dev/module_howto.md +++ b/docs/dev/module_howto.md @@ -80,9 +80,13 @@ The `handle_event()` method is the most important part of the module. By overrid The `emit_event()` method is how modules return data. When you call `emit_event()`, it creates an [event](./scanning/events.md) and outputs it, sending it any modules that are interested in that data type. -## `setup()` +## `setup_deps()` and `setup()` -A module's `setup()` method is used for performing one-time setup at the start of the scan, like downloading a wordlist or checking to make sure an API key is valid. It needs to return either: +`setup_deps()` and `setup()` are used for performing one-time setup at the start of the scan. + +`setup_deps()` is reserved for downloading or installing any dependencies not covered by Ansible, i.e. AI models or wordlists. Any other one-time setup tasks can be put into `setup()`. + +These methods must return either: 1. `True` - module setup succeeded 2. `None` - module setup soft-failed (scan will continue but module will be disabled) @@ -95,16 +99,16 @@ async def setup(self): if not self.config.get("api_key"): # soft-fail return None, "No API key specified" + return True -async def setup(self): - try: - wordlist = self.helpers.wordlist("https://raw.githubusercontent.com/user/wordlist.txt") - except WordlistError as e: - # hard-fail - return False, f"Error downloading wordlist: {e}" +async def setup_deps(self): + self.wordlist = self.helpers.wordlist("https://raw.githubusercontent.com/user/wordlist.txt") + return True async def setup(self): self.timeout = self.config.get("timeout", 5) + if self.timeout <= 0: + return False, "Timeout must be greater than or equal to 0" # success return True ``` diff --git a/docs/index.md b/docs/index.md index 9161d34658..7418989009 100644 --- a/docs/index.md +++ b/docs/index.md @@ -32,16 +32,20 @@ bbot --help ### [Docker](https://hub.docker.com/r/blacklanternsecurity/bbot) -Docker images are provided, along with helper script `bbot-docker.sh` to persist your scan data. +Docker images are provided, along with helper script `bbot-docker.sh` to persist your scan data. Images come in four flavors: `dev`, `dev-full`, `stable`, and `stable-full`. `dev` is the latest bleeding edge version. `-full` images are larger and have all of BBOT's module dependencies preinstalled (wordlists, pip packages, etc.). Scans are output to `~/.bbot/scans` (the usual place for BBOT scan data). ```bash -# bleeding edge (dev) +# dev (bleeding edge) docker run -it blacklanternsecurity/bbot --help +# dev (bleeding edge - full) +docker run -it blacklanternsecurity/bbot:dev-full --help # stable docker run -it blacklanternsecurity/bbot:stable --help +# stable (full) +docker run -it blacklanternsecurity/bbot:stable-full --help # helper script git clone https://github.com/blacklanternsecurity/bbot && cd bbot