Make output_modules additive, add exclude_output_modules

bbot/core/config/models.py

@@ -391,6 +391,7 @@ class BBOTConfig(BaseModel):
     dnsresolve: Optional[bool] = None
     cloudcheck: Optional[bool] = None
     unarchive: Optional[bool] = None
+    python: Optional[bool] = None
 
     # URL handling
     url_querystring_remove: Optional[bool] = None
@@ -443,6 +444,7 @@ class PresetSchema(BaseModel):
     modules: Optional[list[str]] = None
     output_modules: Optional[list[str]] = None
     exclude_modules: Optional[list[str]] = None
+    exclude_output_modules: Optional[list[str]] = None
     flags: Optional[list[str]] = None
     require_flags: Optional[list[str]] = None
     exclude_flags: Optional[list[str]] = None

bbot/defaults.yml

@@ -186,6 +186,8 @@ aggregate: True
 dnsresolve: True
 # Cloud provider tagging
 cloudcheck: True
+# Python API event bridge
+python: True
 
 # Strip querystring from URLs by default
 url_querystring_remove: True

bbot/modules/output/python.py → bbot/modules/internal/python.py

@@ -3,26 +3,25 @@
 
 class python(BaseOutputModule):
     """
-    Pseudo-output module backing ``Scanner.async_start()``. The scan loop
+    Internal module backing ``Scanner.async_start()``. The scan loop
     drains its incoming queue directly via ``_events_waiting`` and yields
-    events to the API caller — there is no real worker.
+    events to the API caller -- there is no real worker.
 
-    Because the worker is a no-op, we override ``queue_event`` so the
-    standard ``_module_consumers`` increment is skipped. Without this,
+    Because the worker is a no-op, we override ``_increment_consumer_count``
+    so the standard ``_module_consumers`` increment is skipped. Without this,
     every event leaks +1 on its consumer count (worker would normally
     pair the increment with a ``_minimize()`` call in its ``finally``
     block, but there is no worker here). The leak prevents
-    ``_minimize()``'s ``<= 0`` block from ever firing — bodies stay in
+    ``_minimize()``'s ``<= 0`` block from ever firing -- bodies stay in
     memory / spill files stay on disk for the entire scan.
     """
 
+    _type = "internal"
     watched_events = ["*"]
     meta = {"description": "Output via Python API", "created_date": "2022-09-13", "author": "@TheTechromancer"}
 
     async def _worker(self):
         pass
 
     def _increment_consumer_count(self, event):
-        # No-op: see class docstring. The standard increment would leak
-        # because there's no worker to pair it with a `_minimize()` call.
         pass

bbot/scanner/preset/args.py

@@ -159,6 +159,7 @@ def preset_from_args(self):
 
         # modules + flags
         args_preset.exclude_modules.update(set(self.parsed.exclude_modules))
+        args_preset.exclude_output_modules.update(set(self.parsed.exclude_output_modules))
         args_preset.exclude_flags.update(set(self.parsed.exclude_flags))
         args_preset.require_flags.update(set(self.parsed.require_flags))
         args_preset.explicit_scan_modules.update(set(self.parsed.modules))
@@ -372,7 +373,15 @@ def create_parser(self, *args, **kwargs):
             "--output-modules",
             nargs="+",
             default=[],
-            help=f"Output module(s). Choices: {','.join(sorted(self.preset.module_loader.output_module_choices))}",
+            help=f"Add output module(s). Choices: {','.join(sorted(self.preset.module_loader.output_module_choices))}",
+            metavar="MODULE",
+        )
+        output.add_argument(
+            "-eom",
+            "--exclude-output-modules",
+            nargs="+",
+            default=[],
+            help="Exclude output module(s)",
             metavar="MODULE",
         )
         output.add_argument("-lo", "--list-output-modules", action="store_true", help="List available output modules")
@@ -440,6 +449,7 @@ def sanitize_args(self):
         self.parsed.modules = chain_lists(self.parsed.modules)
         self.parsed.exclude_modules = chain_lists(self.parsed.exclude_modules)
         self.parsed.output_modules = chain_lists(self.parsed.output_modules)
+        self.parsed.exclude_output_modules = chain_lists(self.parsed.exclude_output_modules)
         self.parsed.targets = chain_lists(
             self.parsed.targets, try_files=True, msg="Reading targets from file: {filename}", _strip_comments=True
         )

bbot/scanner/preset/preset.py

@@ -119,6 +119,7 @@ def __init__(
         modules=None,
         output_modules=None,
         exclude_modules=None,
+        exclude_output_modules=None,
         flags=None,
         require_flags=None,
         exclude_flags=None,
@@ -147,8 +148,9 @@ def __init__(
                 If not specified, seeds will be backfilled from target when target is defined.
             blacklist (list, optional): Blacklisted target(s). Takes ultimate precedence. Defaults to empty.
             modules (list[str], optional): List of scan modules to enable for the scan. Defaults to empty list.
-            output_modules (list[str], optional): List of output modules to use. Defaults to csv, human, and json.
+            output_modules (list[str], optional): Additional output modules to enable (additive on top of defaults).
             exclude_modules (list[str], optional): List of modules to exclude from the scan.
+            exclude_output_modules (list[str], optional): Output modules to exclude (e.g. to remove defaults).
             require_flags (list[str], optional): Only enable modules if they have these flags.
             exclude_flags (list[str], optional): Don't enable modules if they have any of these flags.
             module_dirs (list[str], optional): additional directories to load modules from.
@@ -186,6 +188,7 @@ def __init__(
         # modules / flags
         self.modules = set()
         self.exclude_modules = set()
+        self.exclude_output_modules = set()
         self.flags = set()
         self.exclude_flags = set()
         self.require_flags = set()
@@ -203,6 +206,10 @@ def __init__(
             exclude_modules = []
         if isinstance(exclude_modules, str):
             exclude_modules = [exclude_modules]
+        if exclude_output_modules is None:
+            exclude_output_modules = []
+        if isinstance(exclude_output_modules, str):
+            exclude_output_modules = [exclude_output_modules]
         if flags is None:
             flags = []
         if isinstance(flags, str):
@@ -280,6 +287,7 @@ def __init__(
         self.explicit_scan_modules.update(set(modules))
         self.explicit_output_modules.update(set(output_modules))
         self.exclude_modules.update(set(exclude_modules))
+        self.exclude_output_modules.update(set(exclude_output_modules))
         self.flags.update(set(flags))
         self.exclude_flags.update(set(exclude_flags))
         self.require_flags.update(set(require_flags))
@@ -315,7 +323,7 @@ def default_output_modules(self):
         if self._default_output_modules is not None:
             output_modules = self._default_output_modules
         else:
-            output_modules = ["python", "csv", "txt", "json"]
+            output_modules = ["csv", "txt", "json"]
             if self._cli:
                 output_modules.append("stdout")
         return output_modules
@@ -358,6 +366,7 @@ def merge(self, other):
         # modules + flags
         # establish requirements / exclusions first
         self.exclude_modules.update(other.exclude_modules)
+        self.exclude_output_modules.update(other.exclude_output_modules)
         self.require_flags.update(other.require_flags)
         self.exclude_flags.update(other.exclude_flags)
         # then it's okay to start enabling modules
@@ -450,13 +459,10 @@ def bake(self, scan=None):
         for module in baked_preset.explicit_scan_modules:
             baked_preset.add_module(module, module_type="scan")
 
-        # enable output modules
-        output_modules_to_enable = set(baked_preset.explicit_output_modules)
-        default_output_modules = self.default_output_modules
-        output_module_override = any(m in default_output_modules for m in output_modules_to_enable)
-        # if none of the default output modules have been explicitly specified, enable them all
-        if not output_module_override:
-            output_modules_to_enable.update(self.default_output_modules)
+        # enable output modules (always additive: defaults + explicit, minus excluded)
+        output_modules_to_enable = set(self.default_output_modules)
+        output_modules_to_enable.update(baked_preset.explicit_output_modules)
+        output_modules_to_enable -= baked_preset.exclude_output_modules
         for module in output_modules_to_enable:
             baked_preset.add_module(module, module_type="output", raise_error=False)
 
@@ -481,8 +487,8 @@ def bake(self, scan=None):
                     self.log_debug(f'Enabling module "{module}" because it has flag "{flag}"')
                     baked_preset.add_module(module, module_type, raise_error=False)
 
-        # ensure we have output modules
-        if not baked_preset.output_modules:
+        # ensure we have output modules (unless the user explicitly excluded them)
+        if not baked_preset.output_modules and not baked_preset.exclude_output_modules:
             for output_module in self.default_output_modules:
                 baked_preset.add_module(output_module, module_type="output", raise_error=False)
 
@@ -739,6 +745,7 @@ def from_dict(cls, preset_dict, name=None, _exclude=None, _log=False):
             modules=preset_dict.get("modules"),
             output_modules=preset_dict.get("output_modules"),
             exclude_modules=preset_dict.get("exclude_modules"),
+            exclude_output_modules=preset_dict.get("exclude_output_modules"),
             flags=preset_dict.get("flags"),
             require_flags=preset_dict.get("require_flags"),
             exclude_flags=preset_dict.get("exclude_flags"),
@@ -896,6 +903,8 @@ def to_dict(self, include_target=False, full_config=False, redact_secrets=False)
             preset_dict["exclude_flags"] = sorted(self.exclude_flags)
         if self.exclude_modules:
             preset_dict["exclude_modules"] = sorted(self.exclude_modules)
+        if self.exclude_output_modules:
+            preset_dict["exclude_output_modules"] = sorted(self.exclude_output_modules)
         if self.flags:
             preset_dict["flags"] = sorted(self.flags)
         if self.explicit_scan_modules:
@@ -1027,6 +1036,11 @@ def validate(self):
                 raise ValidationError(
                     get_closest_match(excluded_module, self.module_loader.all_module_choices, msg="module")
                 )
+        for excluded_module in self.exclude_output_modules:
+            if excluded_module not in self.module_loader.output_module_choices:
+                raise ValidationError(
+                    get_closest_match(excluded_module, self.module_loader.output_module_choices, msg="output module")
+                )
         # validate declared module names so typos fail early
         for scan_module in self.explicit_scan_modules:
             self._is_valid_module(scan_module, "scan", name_only=True)

bbot/scanner/preset/validate.py

@@ -275,7 +275,7 @@ def validate_preset(preset_dict: Any, module_loader=None) -> list[PresetValidati
     # nested mapping). Check them explicitly, with the same closest-match hint.
     # Skip non-list values; the schema pass above already flagged the type error,
     # and iterating a string here would yield bogus per-character lookups.
-    for key in ("modules", "output_modules", "exclude_modules"):
+    for key in ("modules", "output_modules", "exclude_modules", "exclude_output_modules"):
         value = preset_dict.get(key)
         if not isinstance(value, list):
             continue

bbot/scanner/scanner.py

@@ -397,9 +397,9 @@ async def _prep(self):
                 intercept_module._incoming_event_queue = interqueue
                 prev_intercept_module._outgoing_event_queue = interqueue
 
-            # abort if there are no output modules
+            # abort if there are no output modules (unless the user explicitly excluded them)
             num_output_modules = len([m for m in self.modules.values() if m._type == "output"])
-            if num_output_modules < 1:
+            if num_output_modules < 1 and not self.preset.exclude_output_modules:
                 raise ScanError("Failed to load output modules. Aborting.")
             # abort if any of the module .setup()s hard-failed (i.e. they errored or returned False)
             total_failed = len(hard_failed + soft_failed)
@@ -563,7 +563,7 @@ async def _mark_finished(self):
 
         if not self._stopping:
             # queue final scan event with output modules
-            output_modules = [m for m in self.modules.values() if m._type == "output" and m.name != "python"]
+            output_modules = [m for m in self.modules.values() if m._type == "output"]
             for m in output_modules:
                 await m.queue_event(scan_finish_event)
             # wait until output modules are flushed

bbot/test/benchmarks/_scan_memory_deep_chain.py

@@ -80,7 +80,7 @@ def log_message(self, *a):
 scan = Scanner(
     f"http://127.0.0.1:{port}/",
     modules=[HTTP_MODULE],
-    output_modules=["python"],
+    exclude_output_modules=["csv", "json", "txt"],
     config={
         "dns": {"minimal": True},
         "scope": {"search_distance": 0},

bbot/test/benchmarks/_scan_memory_parallel_chains.py

@@ -87,7 +87,7 @@ def log_message(self, *a):
 scan = Scanner(
     *targets,
     modules=[HTTP_MODULE],
-    output_modules=["python"],
+    exclude_output_modules=["csv", "json", "txt"],
     config={
         "dns": {"minimal": True},
         "scope": {"search_distance": 0},

bbot/test/benchmarks/_scan_memory_subdomain_enum.py

@@ -30,7 +30,7 @@
 scan = Scanner(
     "example.com",
     modules=[],
-    output_modules=["python"],
+    exclude_output_modules=["csv", "json", "txt"],
     config={
         "dns": {"disable": True},
         "scope": {"search_distance": 0},

bbot/test/benchmarks/_scan_memory_web_crawl.py

@@ -65,7 +65,7 @@ def log_message(self, *a):
 scan = Scanner(
     f"http://127.0.0.1:{port}/",
     modules=[HTTP_MODULE],
-    output_modules=["python"],
+    exclude_output_modules=["csv", "json", "txt"],
     config={
         "dns": {"minimal": True},
         "scope": {"search_distance": 0},

bbot/test/test_step_1/test_cli.py

@@ -322,44 +322,55 @@ async def test_cli_args(monkeypatch, caplog, capsys, clean_default_config):
     assert "| dnsbrute " not in out
     assert "| http " in out
 
-    # output modules override
+    # -om is additive (defaults stay)
     caplog.clear()
     assert not caplog.text
     monkeypatch.setattr("sys.argv", ["bbot", "-om", "csv,json", "-y"])
     result = await cli._main()
     assert result is True
-    assert "Loaded 2/2 output modules, (csv,json)" in caplog.text
+    assert "Loaded 4/4 output modules, (csv,json,stdout,txt)" in caplog.text
     caplog.clear()
     monkeypatch.setattr("sys.argv", ["bbot", "-em", "csv,json", "-y"])
     result = await cli._main()
     assert result is True
-    assert "Loaded 3/3 output modules, (python,stdout,txt)" in caplog.text
+    assert "Loaded 2/2 output modules, (stdout,txt)" in caplog.text
 
-    # output modules override
+    # -om adds non-default module on top of defaults
     caplog.clear()
     assert not caplog.text
     monkeypatch.setattr("sys.argv", ["bbot", "-om", "subdomains", "-y"])
     result = await cli._main()
     assert result is True
-    assert "Loaded 6/6 output modules, (csv,json,python,stdout,subdomains,txt)" in caplog.text
+    assert "Loaded 5/5 output modules, (csv,json,stdout,subdomains,txt)" in caplog.text
 
-    # internal modules override
+    # -eom removes output modules
+    caplog.clear()
+    assert not caplog.text
+    monkeypatch.setattr("sys.argv", ["bbot", "-eom", "csv,txt", "-y"])
+    result = await cli._main()
+    assert result is True
+    assert "Loaded 2/2 output modules, (json,stdout)" in caplog.text
+
+    # internal modules (python is now internal)
     caplog.clear()
     assert not caplog.text
     monkeypatch.setattr("sys.argv", ["bbot", "-y"])
     result = await cli._main()
     assert result is True
-    assert "Loaded 6/6 internal modules (aggregate,cloudcheck,dnsresolve,excavate,speculate,unarchive)" in caplog.text
+    assert (
+        "Loaded 7/7 internal modules (aggregate,cloudcheck,dnsresolve,excavate,python,speculate,unarchive)"
+        in caplog.text
+    )
     caplog.clear()
     monkeypatch.setattr("sys.argv", ["bbot", "-em", "excavate", "speculate", "-y"])
     result = await cli._main()
     assert result is True
-    assert "Loaded 4/4 internal modules (aggregate,cloudcheck,dnsresolve,unarchive)" in caplog.text
+    assert "Loaded 5/5 internal modules (aggregate,cloudcheck,dnsresolve,python,unarchive)" in caplog.text
     caplog.clear()
     monkeypatch.setattr("sys.argv", ["bbot", "-c", "speculate=false", "-y"])
     result = await cli._main()
     assert result is True
-    assert "Loaded 5/5 internal modules (aggregate,cloudcheck,dnsresolve,excavate,unarchive)" in caplog.text
+    assert "Loaded 6/6 internal modules (aggregate,cloudcheck,dnsresolve,excavate,python,unarchive)" in caplog.text
 
     # custom target type
     out, err = capsys.readouterr()
@@ -631,7 +642,7 @@ def test_cli_module_validation(monkeypatch, caplog):
     monkeypatch.setattr("sys.argv", ["bbot", "-om", "websocket", "-c", "modules.websocket.url=", "-y"])
     cli.main()
     lines = caplog.text.splitlines()
-    assert "Loaded 6/6 output modules, (csv,json,python,stdout,txt,websocket)" in caplog.text
+    assert "Loaded 5/5 output modules, (csv,json,stdout,txt,websocket)" in caplog.text
     assert 1 == len(
         [
             l

bbot/test/test_step_1/test_manager_scope_accuracy.py

@@ -540,7 +540,7 @@ def custom_setup(scan):
         "127.0.0.33",
         seeds=["127.0.0.111/31"],
         modules=["http"],
-        output_modules=["python"],
+        exclude_output_modules=["csv", "json", "txt"],
         _config={
             "dns": {"minimal": False, "search_distance": 2},
             "scope": {"search_distance": 0, "report_distance": 0},

bbot/test/test_step_1/test_modules_basic.py

@@ -432,7 +432,7 @@ async def handle_event(self, event):
     scan = bbot_scanner(
         "evilcorp.com",
         config={"speculate": True, "dns": {"minimal": False}},
-        output_modules=["python"],
+        exclude_output_modules=["csv", "json", "txt"],
         force_start=True,
     )