log: surface real failures in error.log

bbot/cli.py

@@ -337,6 +337,8 @@ def main():
     import traceback
     from bbot.core import CORE
 
+    log = logging.getLogger("bbot.cli")
+
     global scan_name
     try:
         asyncio.run(_main())
@@ -347,10 +349,18 @@ def main():
         msg = "Interrupted"
         if scan_name:
             msg = f"You killed {scan_name}"
+        log.warning(msg)
+        log.trace(traceback.format_exc())
         log_to_stderr(msg, level="WARNING")
         if CORE.logger.log_level <= logging.DEBUG:
             log_to_stderr(traceback.format_exc(), level="DEBUG")
         exit(1)
+    except Exception as e:
+        log.error(f"Unhandled exception: {e}")
+        log.trace(traceback.format_exc())
+        log_to_stderr(f"Unhandled exception: {e}", level="CRITICAL")
+        log_to_stderr(traceback.format_exc(), level="DEBUG")
+        exit(1)
 
 
 if __name__ == "__main__":

bbot/core/helpers/depsinstaller/installer.py

@@ -202,7 +202,7 @@ async def install(self, *modules):
                             log.debug(f'Setup succeeded for module "{m}"')
                             succeeded.append(m)
                         else:
-                            log.warning(f'Setup failed for module "{m}"')
+                            log.error(f'Setup failed for module "{m}"')
                             failed.append(m)
                     else:
                         if success or self.deps_behavior == "ignore_failed":
@@ -211,7 +211,7 @@ async def install(self, *modules):
                             )
                             succeeded.append(m)
                         else:
-                            log.warning(
+                            log.error(
                                 f'Skipping dependency install for module "{m}" because it failed previously (--retry-deps to retry or --ignore-failed-deps to ignore)'
                             )
                             failed.append(m)
@@ -288,7 +288,7 @@ async def pip_install(self, packages, constraints=None):
             log.info(message)
             return True
         except CalledProcessError as err:
-            log.warning(f"Failed to install pip packages {packages_str} (return code {err.returncode}): {err.stderr}")
+            log.error(f"Failed to install pip packages {packages_str} (return code {err.returncode}): {err.stderr}")
         return False
 
     def apt_install(self, packages):
@@ -300,11 +300,9 @@ def apt_install(self, packages):
         if success:
             log.info(f'Successfully installed OS packages "{",".join(sorted(packages))}"')
         else:
-            log.warning(
-                f"Failed to install OS packages ({err}). Recommend installing the following packages manually:"
-            )
+            log.error(f"Failed to install OS packages ({err}). Recommend installing the following packages manually:")
             for p in packages:
-                log.warning(f" - {p}")
+                log.error(f" - {p}")
         return success
 
     def _make_apt_ansible_args(self, packages):
@@ -341,7 +339,7 @@ def shell(self, module, commands):
         if success:
             log.info(f"Successfully ran {len(commands):,} shell commands")
         else:
-            log.warning("Failed to run shell dependencies")
+            log.error("Failed to run shell dependencies")
         return success
 
     def tasks(self, module, tasks):
@@ -350,7 +348,7 @@ def tasks(self, module, tasks):
         if success:
             log.info(f"Successfully ran {len(tasks):,} Ansible tasks for {module}")
         else:
-            log.warning(f"Failed to run Ansible tasks for {module}")
+            log.error(f"Failed to run Ansible tasks for {module}")
         return success
 
     def ansible_run(self, tasks=None, module=None, args=None, ansible_args=None):
@@ -440,7 +438,7 @@ def ensure_root(self, message=""):
                 try:
                     _sudo_password = getpass.getpass(prompt="[USER] Please enter sudo password: ")
                 except OSError:
-                    log.warning("Unable to read sudo password (no TTY). Set BBOT_SUDO_PASS env var.")
+                    log.error("Unable to read sudo password (no TTY). Set BBOT_SUDO_PASS env var.")
                     return
                 if self.parent_helper.verify_sudo_password(_sudo_password):
                     log.success("Authentication successful")

bbot/core/helpers/dns/dns.py

@@ -412,7 +412,7 @@ async def _connectivity_check(self, interval=5):
                     return True
 
         if time.time() - self._last_connectivity_warning > interval:
-            self.log.warning("DNS queries are failing, please check your internet connection")
+            self.log.error("DNS queries are failing, please check your internet connection")
             self._last_connectivity_warning = time.time()
         self._errors.clear()
         return False

bbot/modules/base.py

@@ -1069,36 +1069,30 @@ async def queue_outgoing_event(self, event, **kwargs):
         except AttributeError:
             self.debug("Not in an acceptable state to queue outgoing event")
 
-    def set_error_state(self, message=None, clear_outgoing_queue=False, critical=False):
+    def set_error_state(self, message=None, clear_outgoing_queue=False, critical=False, log_level="error"):
         """
-        Puts the module into an errored state where it cannot accept new events. Optionally logs a warning message.
-
-        The function sets the module's `errored` attribute to True and logs a warning with the optional message.
-        It also clears the incoming event queue to prevent further processing and updates its status to False.
+        Puts the module into an errored state where it cannot accept new events. Optionally logs a message.
 
         Args:
-            message (str, optional): Additional message to be logged along with the warning.
-
-        Returns:
-            None: The function doesn't return anything but updates the `errored` state and clears the incoming event queue.
+            message (str, optional): Additional message to log alongside the state transition.
+            clear_outgoing_queue (bool): Drain the outgoing event queue as well.
+            critical (bool): Log at CRITICAL severity (overrides log_level).
+            log_level (str): Severity to log at when not critical. Use "info" or "verbose" for intentional
+                stops (e.g. user-initiated kill) so they don't appear in error.log.
 
         Examples:
             >>> self.set_error_state()
             >>> self.set_error_state("Failed to connect to the server")
-
-        Notes:
-            - The function sets `self._incoming_event_queue` to False to prevent its further use.
-            - If the module was already in an errored state, the function will not reset the error state or the queue.
+            >>> self.set_error_state("killed by user", log_level="info")
         """
         if not self.errored:
             log_msg = "Setting error state"
             if message is not None:
                 log_msg += f": {message}"
             if critical:
-                log_fn = self.error
+                self.critical(log_msg, trace=False)
             else:
-                log_fn = self.warning
-            log_fn(log_msg)
+                getattr(self, log_level)(log_msg)
             self.errored = True
             # clear incoming queue
             if self.incoming_event_queue is not False:

bbot/modules/wayback.py

@@ -42,13 +42,13 @@ async def query(self, query):
         waybackurl = f"{self.base_url}/cdx/search/cdx?url={self.helpers.quote(query)}&matchType=domain&output=json&fl=original&collapse=original"
         r = await self.helpers.request(waybackurl, timeout=self.http_timeout + 10)
         if not r:
-            self.warning(f'Error connecting to archive.org for query "{query}"')
+            self.verbose(f'Error connecting to archive.org for query "{query}"')
             return results
         try:
             j = r.json()
             assert type(j) == list
         except Exception:
-            self.warning(f'Error JSON-decoding archive.org response for query "{query}"')
+            self.verbose(f'Error JSON-decoding archive.org response for query "{query}"')
             return results
 
         urls = []

bbot/scanner/scanner.py

@@ -588,7 +588,7 @@ async def setup_modules(self, remove_failed=True, deps_only=False):
                 self.debug(f"Setup succeeded for {module.name} ({msg})")
                 succeeded.append(module.name)
             elif status is False:
-                self.warning(f"Setup hard-failed for {module.name}: {msg}")
+                self.error(f"Setup hard-failed for {module.name}: {msg}")
                 self.modules[module.name].set_error_state()
                 hard_failed.append(module.name)
             else:
@@ -714,7 +714,7 @@ def kill_module(self, module_name, message=None):
         if module._intercept:
             self.warning(f'Cannot kill module "{module_name}" because it is critical to the scan')
             return
-        module.set_error_state(message=message, clear_outgoing_queue=True)
+        module.set_error_state(message=message, clear_outgoing_queue=True, log_level="info")
         for proc in module._proc_tracker:
             with contextlib.suppress(Exception):
                 proc.send_signal(SIGINT)
@@ -1402,9 +1402,9 @@ def _load_modules(self, modules):
                     self.verbose(f'Loaded module "{module_name}"')
                     continue
                 except Exception:
-                    self.warning(f"Failed to load module {module_class}")
+                    self.error(f"Failed to load module {module_class}")
             else:
-                self.warning(f'Failed to load unknown module "{module_name}"')
+                self.error(f'Failed to load unknown module "{module_name}"')
             failed.add(module_name)
         return loaded_modules, failed
 

bbot/test/test_step_1/test_cli.py

@@ -522,15 +522,15 @@ def test_cli_module_validation(monkeypatch, caplog):
         [
             l
             for l in lines
-            if l.startswith("WARNING  bbot.scanner:scanner.py")
+            if l.startswith("ERROR    bbot.scanner:scanner.py")
             and l.endswith("Setup hard-failed for websocket: Must set URL")
         ]
     )
     assert 1 == len(
         [
             l
             for l in lines
-            if l.startswith("WARNING  bbot.modules.output.websocket:base.py") and l.endswith("Setting error state")
+            if l.startswith("ERROR    bbot.modules.output.websocket:base.py") and l.endswith("Setting error state")
         ]
     )
     assert 1 == len(
@@ -556,15 +556,15 @@ def test_cli_module_validation(monkeypatch, caplog):
         [
             l
             for l in lines
-            if l.startswith("WARNING  bbot.scanner:scanner.py")
+            if l.startswith("ERROR    bbot.scanner:scanner.py")
             and l.endswith("Setup hard-failed for websocket: Must set URL")
         ]
     )
     assert 1 == len(
         [
             l
             for l in lines
-            if l.startswith("WARNING  bbot.modules.output.websocket:base.py") and l.endswith("Setting error state")
+            if l.startswith("ERROR    bbot.modules.output.websocket:base.py") and l.endswith("Setting error state")
         ]
     )
     assert 1 == len(