From e3c327038412d691a4a08ba5c769529e4a97d854 Mon Sep 17 00:00:00 2001 From: Mukesh Sai Kumar Date: Tue, 3 Jun 2025 19:18:27 +0530 Subject: [PATCH 1/8] Fix broken chromium on M4 Mac --- bbot/core/helpers/misc.py | 4 ++-- bbot/modules/gowitness.py | 8 ++++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/bbot/core/helpers/misc.py b/bbot/core/helpers/misc.py index a5bc8a2fca..0230e6f249 100644 --- a/bbot/core/helpers/misc.py +++ b/bbot/core/helpers/misc.py @@ -1300,7 +1300,7 @@ def make_netloc(host, port=None): return f"{host}:{port}" -def which(*executables): +def which(*executables, path=None): """Finds the full path of the first available executable from a list of executables. Args: @@ -1316,7 +1316,7 @@ def which(*executables): import shutil for e in executables: - location = shutil.which(e) + location = shutil.which(e, path=path) if location: return location diff --git a/bbot/modules/gowitness.py b/bbot/modules/gowitness.py index 6e3b2e19eb..d907d6e6f6 100644 --- a/bbot/modules/gowitness.py +++ b/bbot/modules/gowitness.py @@ -71,6 +71,10 @@ async def setup(self): custom_chrome_path = self.helpers.tools_dir / "chrome-linux" / "chrome" if custom_chrome_path.is_file(): self.chrome_path = custom_chrome_path + config_chrome_path = self.config.get("chrome_path") + if config_chrome_path: + custom_chrome_path = config_chrome_path + self.chrome_path = Path(config_chrome_path) # fix ubuntu-specific sandbox bug chrome_devel_sandbox = self.helpers.tools_dir / "chrome-linux" / "chrome_sandbox" @@ -79,8 +83,8 @@ async def setup(self): # make sure we have a working chrome install chrome_test_pass = False - for binary in ("chrome", "chromium", "chromium-browser", custom_chrome_path): - binary_path = self.helpers.which(binary) + for binary in ("Google Chrome", "chrome", "chromium", "chromium-browser", custom_chrome_path): + binary_path = self.helpers.which(binary, path=custom_chrome_path) if binary_path and Path(binary_path).is_file(): chrome_test_proc = await self.run_process([binary_path, "--version"]) if getattr(chrome_test_proc, "returncode", 1) == 0: From 624a1e3ef795e20ec0534f07b65eadb3a545d4a8 Mon Sep 17 00:00:00 2001 From: Mukesh Sai Kumar Date: Tue, 3 Jun 2025 22:58:17 +0530 Subject: [PATCH 2/8] Respect PATH when searching for chrom(e/ium) --- bbot/modules/gowitness.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/bbot/modules/gowitness.py b/bbot/modules/gowitness.py index d907d6e6f6..ed39fe023c 100644 --- a/bbot/modules/gowitness.py +++ b/bbot/modules/gowitness.py @@ -68,13 +68,13 @@ async def setup(self): else: self.base_path = self.scan.home / "gowitness" self.chrome_path = None - custom_chrome_path = self.helpers.tools_dir / "chrome-linux" / "chrome" - if custom_chrome_path.is_file(): - self.chrome_path = custom_chrome_path - config_chrome_path = self.config.get("chrome_path") - if config_chrome_path: - custom_chrome_path = config_chrome_path - self.chrome_path = Path(config_chrome_path) + + chrome_search_path = os.environ.get("PATH") + config_chrome_path = Path(self.config.get("chrome_path")) + if config_chrome_path.is_file(): + chrome_search_path = f"{config_chrome_path.parent}:{chrome_search_path}" + elif config_chrome_path.is_dir(): + chrome_search_path = f"{config_chrome_path}:{chrome_search_path}" # fix ubuntu-specific sandbox bug chrome_devel_sandbox = self.helpers.tools_dir / "chrome-linux" / "chrome_sandbox" @@ -83,12 +83,14 @@ async def setup(self): # make sure we have a working chrome install chrome_test_pass = False + custom_chrome_path = self.helpers.tools_dir / "chrome-linux" / "chrome" for binary in ("Google Chrome", "chrome", "chromium", "chromium-browser", custom_chrome_path): - binary_path = self.helpers.which(binary, path=custom_chrome_path) + binary_path = self.helpers.which(binary, path=chrome_search_path) if binary_path and Path(binary_path).is_file(): chrome_test_proc = await self.run_process([binary_path, "--version"]) if getattr(chrome_test_proc, "returncode", 1) == 0: self.verbose(f"Found chrome executable at {binary_path}") + self.chrome_path = binary_path chrome_test_pass = True break if not chrome_test_pass: From cf719dd59c338b52f92681cefa35be3f77bb4e49 Mon Sep 17 00:00:00 2001 From: Mukesh Sai Kumar Date: Thu, 5 Jun 2025 12:18:11 +0530 Subject: [PATCH 3/8] Handle case where chrome_path is None --- bbot/modules/gowitness.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/bbot/modules/gowitness.py b/bbot/modules/gowitness.py index ed39fe023c..b452562a71 100644 --- a/bbot/modules/gowitness.py +++ b/bbot/modules/gowitness.py @@ -70,11 +70,13 @@ async def setup(self): self.chrome_path = None chrome_search_path = os.environ.get("PATH") - config_chrome_path = Path(self.config.get("chrome_path")) - if config_chrome_path.is_file(): - chrome_search_path = f"{config_chrome_path.parent}:{chrome_search_path}" - elif config_chrome_path.is_dir(): - chrome_search_path = f"{config_chrome_path}:{chrome_search_path}" + config_chrome_path = self.config.get("chrome_path") + if config_chrome_path: + config_chrome_path = Path(config_chrome_path) + if config_chrome_path.is_file(): + chrome_search_path = f"{config_chrome_path.parent}:{chrome_search_path}" + elif config_chrome_path.is_dir(): + chrome_search_path = f"{config_chrome_path}:{chrome_search_path}" # fix ubuntu-specific sandbox bug chrome_devel_sandbox = self.helpers.tools_dir / "chrome-linux" / "chrome_sandbox" From dab7b8c3e346c6eb8e7557f8703b5f904f43dbff Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 5 Jun 2025 15:06:08 -0400 Subject: [PATCH 4/8] chrome path tweaks --- bbot/modules/gowitness.py | 56 ++++++++++++++++++++++++--------------- 1 file changed, 35 insertions(+), 21 deletions(-) diff --git a/bbot/modules/gowitness.py b/bbot/modules/gowitness.py index b452562a71..4a523e08ff 100644 --- a/bbot/modules/gowitness.py +++ b/bbot/modules/gowitness.py @@ -23,6 +23,7 @@ class gowitness(BaseModule): "output_path": "", "social": False, "idle_timeout": 1800, + "chrome_path": "", } options_desc = { "version": "Gowitness version", @@ -33,6 +34,7 @@ class gowitness(BaseModule): "output_path": "Where to save screenshots", "social": "Whether to screenshot social media webpages", "idle_timeout": "Skip the current gowitness batch if it stalls for longer than this many seconds", + "chrome_path": "Path to chrome executable", } deps_common = ["chromium"] deps_pip = ["aiosqlite"] @@ -67,37 +69,49 @@ async def setup(self): self.base_path = Path(output_path) / "gowitness" else: self.base_path = self.scan.home / "gowitness" - self.chrome_path = None - chrome_search_path = os.environ.get("PATH") + self.chrome_path = None config_chrome_path = self.config.get("chrome_path") if config_chrome_path: config_chrome_path = Path(config_chrome_path) - if config_chrome_path.is_file(): - chrome_search_path = f"{config_chrome_path.parent}:{chrome_search_path}" - elif config_chrome_path.is_dir(): - chrome_search_path = f"{config_chrome_path}:{chrome_search_path}" + if not config_chrome_path.is_file(): + return False, f"Could not find custom Chrome path at {config_chrome_path}" + self.chrome_path = config_chrome_path + else: + bbot_chrome_path = self.helpers.tools_dir / "chrome-linux" / "chrome" + if bbot_chrome_path.is_file(): + self.chrome_path = bbot_chrome_path + + # make sure our chrome path works + chrome_test_pass = False + if self.chrome_path and self.chrome_path.is_file(): + chrome_test_proc = await self.run_process([str(self.chrome_path), "--version"]) + if getattr(chrome_test_proc, "returncode", 1) == 0: + self.verbose(f"Found chrome executable at {self.chrome_path}") + chrome_test_pass = True + + if not chrome_test_pass: + # last resort - try to find a working chrome install + for binary in ("Google Chrome", "chrome", "chromium", "chromium-browser"): + binary_path = self.helpers.which(binary) + if binary_path and Path(binary_path).is_file(): + chrome_test_proc = await self.run_process([str(binary_path), "--version"]) + if getattr(chrome_test_proc, "returncode", 1) == 0: + self.verbose(f"Found chrome executable at {binary_path}") + chrome_test_pass = True + break + + if not chrome_test_pass: + return ( + False, + "Failed to set up Google chrome. Please install manually and set `chrome_path`, or try again with --force-deps.", + ) # fix ubuntu-specific sandbox bug chrome_devel_sandbox = self.helpers.tools_dir / "chrome-linux" / "chrome_sandbox" if chrome_devel_sandbox.is_file(): os.environ["CHROME_DEVEL_SANDBOX"] = str(chrome_devel_sandbox) - # make sure we have a working chrome install - chrome_test_pass = False - custom_chrome_path = self.helpers.tools_dir / "chrome-linux" / "chrome" - for binary in ("Google Chrome", "chrome", "chromium", "chromium-browser", custom_chrome_path): - binary_path = self.helpers.which(binary, path=chrome_search_path) - if binary_path and Path(binary_path).is_file(): - chrome_test_proc = await self.run_process([binary_path, "--version"]) - if getattr(chrome_test_proc, "returncode", 1) == 0: - self.verbose(f"Found chrome executable at {binary_path}") - self.chrome_path = binary_path - chrome_test_pass = True - break - if not chrome_test_pass: - return False, "Failed to set up Google chrome. Please install manually or try again with --force-deps." - self.db_path = self.base_path / "gowitness.sqlite3" self.screenshot_path = self.base_path / "screenshots" self.command = self.construct_command() From a87c83f9752f07f96ba3e58a0700b8abe96a598c Mon Sep 17 00:00:00 2001 From: Mukesh Sai Kumar Date: Sun, 8 Jun 2025 16:22:22 +0530 Subject: [PATCH 5/8] Respect --force-deps for common dependencies --- bbot/core/helpers/depsinstaller/installer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/core/helpers/depsinstaller/installer.py b/bbot/core/helpers/depsinstaller/installer.py index 5bea5f508a..ffb7e2fb12 100644 --- a/bbot/core/helpers/depsinstaller/installer.py +++ b/bbot/core/helpers/depsinstaller/installer.py @@ -196,7 +196,7 @@ async def install_module(self, module): deps_common = preloaded["deps"]["common"] if deps_common: for dep_common in deps_common: - if self.setup_status.get(dep_common, False) is True: + if self.setup_status.get(dep_common, False) is True and self.deps_behavior != "force_install": log.debug( f'Skipping installation of dependency "{dep_common}" for module "{module}" since it is already installed' ) From 7b2d7defbaa718296ec2fc4b1eacb28fa8493198 Mon Sep 17 00:00:00 2001 From: Mukesh Sai Kumar Date: Sun, 8 Jun 2025 16:22:46 +0530 Subject: [PATCH 6/8] Add chromium auto-install for Darwin --- bbot/core/shared_deps.py | 38 ++++++++++++++++++++++++++++++++++++++ bbot/modules/gowitness.py | 6 +++++- 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/bbot/core/shared_deps.py b/bbot/core/shared_deps.py index cd338093fc..c73c710da8 100644 --- a/bbot/core/shared_deps.py +++ b/bbot/core/shared_deps.py @@ -108,6 +108,24 @@ "when": "ansible_facts['os_family'] == 'Debian'", "ignore_errors": True, }, + { + "name": "Get latest Chromium version (Darwin x86_64)", + "uri": { + "url": "https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/Mac%2FLAST_CHANGE?alt=media", + "return_content": True, + }, + "register": "chromium_version", + "when": "ansible_facts['os_family'] == 'Darwin' and ansible_facts['architecture'] == 'x86_64'", + }, + { + "name": "Get latest Chromium version (Darwin arm64)", + "uri": { + "url": "https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/Mac_Arm%2FLAST_CHANGE?alt=media", + "return_content": True, + }, + "register": "chromium_version", + "when": "ansible_facts['os_family'] == 'Darwin' and ansible_facts['architecture'] == 'arm64'", + }, { "name": "Download Chromium (Debian)", "unarchive": { @@ -119,6 +137,26 @@ "when": "ansible_facts['os_family'] == 'Debian'", "ignore_errors": True, }, + { + "name": "Download Chromium (Darwin x86_64)", + "unarchive": { + "src": "https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/Mac%2F{{ chromium_version.content }}%2Fchrome-mac.zip?alt=media", + "remote_src": True, + "dest": "#{BBOT_TOOLS}", + "creates": "#{BBOT_TOOLS}/chrome-mac", + }, + "when": "ansible_facts['os_family'] == 'Darwin' and ansible_facts['architecture'] == 'x86_64'", + }, + { + "name": "Download Chromium (Darwin arm64)", + "unarchive": { + "src": "https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/Mac_Arm%2F{{ chromium_version.content }}%2Fchrome-mac.zip?alt=media", + "remote_src": True, + "dest": "#{BBOT_TOOLS}", + "creates": "#{BBOT_TOOLS}/chrome-mac", + }, + "when": "ansible_facts['os_family'] == 'Darwin' and ansible_facts['architecture'] == 'arm64'", + }, # Because Ubuntu is a special snowflake, we have to bend over backwards to fix the chrome sandbox # see https://chromium.googlesource.com/chromium/src/+/main/docs/security/apparmor-userns-restrictions.md { diff --git a/bbot/modules/gowitness.py b/bbot/modules/gowitness.py index 4a523e08ff..e845ea28b9 100644 --- a/bbot/modules/gowitness.py +++ b/bbot/modules/gowitness.py @@ -2,6 +2,7 @@ import asyncio import aiosqlite import multiprocessing +import platform from pathlib import Path from contextlib import suppress from shutil import copyfile, copymode @@ -78,7 +79,10 @@ async def setup(self): return False, f"Could not find custom Chrome path at {config_chrome_path}" self.chrome_path = config_chrome_path else: - bbot_chrome_path = self.helpers.tools_dir / "chrome-linux" / "chrome" + if platform.system() == "Darwin": + bbot_chrome_path = self.helpers.tools_dir / "chrome-mac" / "Chromium.app" / "Contents" / "MacOS" / "Chromium" + else: + bbot_chrome_path = self.helpers.tools_dir / "chrome-linux" / "chrome" if bbot_chrome_path.is_file(): self.chrome_path = bbot_chrome_path From a962e90f0a7c083a955e4d6869233ddbef5b45bd Mon Sep 17 00:00:00 2001 From: Mukesh Sai Kumar Date: Sun, 8 Jun 2025 16:56:11 +0530 Subject: [PATCH 7/8] Fix failing tests --- bbot/core/shared_deps.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bbot/core/shared_deps.py b/bbot/core/shared_deps.py index c73c710da8..013a8b4d67 100644 --- a/bbot/core/shared_deps.py +++ b/bbot/core/shared_deps.py @@ -114,7 +114,7 @@ "url": "https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/Mac%2FLAST_CHANGE?alt=media", "return_content": True, }, - "register": "chromium_version", + "register": "chromium_version_darwin_x86_64", "when": "ansible_facts['os_family'] == 'Darwin' and ansible_facts['architecture'] == 'x86_64'", }, { @@ -123,7 +123,7 @@ "url": "https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/Mac_Arm%2FLAST_CHANGE?alt=media", "return_content": True, }, - "register": "chromium_version", + "register": "chromium_version_darwin_arm64", "when": "ansible_facts['os_family'] == 'Darwin' and ansible_facts['architecture'] == 'arm64'", }, { @@ -140,7 +140,7 @@ { "name": "Download Chromium (Darwin x86_64)", "unarchive": { - "src": "https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/Mac%2F{{ chromium_version.content }}%2Fchrome-mac.zip?alt=media", + "src": "https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/Mac%2F{{ chromium_version_darwin_x86_64.content }}%2Fchrome-mac.zip?alt=media", "remote_src": True, "dest": "#{BBOT_TOOLS}", "creates": "#{BBOT_TOOLS}/chrome-mac", @@ -150,7 +150,7 @@ { "name": "Download Chromium (Darwin arm64)", "unarchive": { - "src": "https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/Mac_Arm%2F{{ chromium_version.content }}%2Fchrome-mac.zip?alt=media", + "src": "https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/Mac_Arm%2F{{ chromium_version_darwin_arm64.content }}%2Fchrome-mac.zip?alt=media", "remote_src": True, "dest": "#{BBOT_TOOLS}", "creates": "#{BBOT_TOOLS}/chrome-mac", From dfef2c95e468b984bcffe7fb48d7f8f2db7ed13e Mon Sep 17 00:00:00 2001 From: Mukesh Sai Kumar Date: Sun, 8 Jun 2025 17:02:20 +0530 Subject: [PATCH 8/8] Fix linting error --- bbot/modules/gowitness.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bbot/modules/gowitness.py b/bbot/modules/gowitness.py index e845ea28b9..ec1199bd37 100644 --- a/bbot/modules/gowitness.py +++ b/bbot/modules/gowitness.py @@ -80,7 +80,9 @@ async def setup(self): self.chrome_path = config_chrome_path else: if platform.system() == "Darwin": - bbot_chrome_path = self.helpers.tools_dir / "chrome-mac" / "Chromium.app" / "Contents" / "MacOS" / "Chromium" + bbot_chrome_path = ( + self.helpers.tools_dir / "chrome-mac" / "Chromium.app" / "Contents" / "MacOS" / "Chromium" + ) else: bbot_chrome_path = self.helpers.tools_dir / "chrome-linux" / "chrome" if bbot_chrome_path.is_file():