Merge pull request blacklanternsecurity#1119 from blacklanternsecurity/dev

Dev --> Stable

Author: TheTechromancer

.github/workflows/tests.yml

@@ -129,8 +129,7 @@ jobs:
           password: ${{ secrets.PYPI_API_TOKEN }}
       - name: Get BBOT version
         id: version
-        run: |
-          echo "::set-output name=BBOT_VERSION::$(poetry version | cut -d' ' -f2 | tr -d v)"
+        run: echo "BBOT_VERSION=$(poetry version | cut -d' ' -f2)" >> $GITHUB_OUTPUT
       - name: Publish to Docker Hub (dev)
         if: github.event_name == 'push' && github.ref == 'refs/heads/dev'
         uses: elgohr/Publish-Docker-Github-Action@v5
@@ -154,3 +153,28 @@ jobs:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
           repository: blacklanternsecurity/bbot
+    outputs:
+      BBOT_VERSION: ${{ steps.version.outputs.BBOT_VERSION }}
+  # tag_commit:
+  #   needs: publish_code
+  #   runs-on: ubuntu-latest
+  #   if: github.event_name == 'push' && github.ref == 'refs/heads/stable'
+  #   steps:
+  #     - uses: actions/checkout@v3
+  #       with:
+  #         ref: ${{ github.head_ref }}
+  #         fetch-depth: 0 # Fetch all history for all tags and branches
+  #     - name: Configure git
+  #       run: |
+  #         git config --local user.email "[email protected]"
+  #         git config --local user.name "GitHub Actions"
+  #     - name: Tag commit
+  #       run: |
+  #         VERSION="${{ needs.publish_code.outputs.BBOT_VERSION }}"
+  #         if [[ "${{ github.ref }}" == "refs/heads/dev" ]]; then
+  #           TAG_MESSAGE="Dev Release $VERSION"
+  #         elif [[ "${{ github.ref }}" == "refs/heads/stable" ]]; then
+  #           TAG_MESSAGE="Stable Release $VERSION"
+  #         fi
+  #         git tag -a $VERSION -m "$TAG_MESSAGE"
+  #         git push origin --tags

.gitignore

@@ -1,2 +1,4 @@
 __pycache__/
 .coverage*
+/data/
+/neo4j/

README.md

@@ -8,7 +8,6 @@
 import traceback
 from omegaconf import OmegaConf
 from contextlib import suppress
-from aioconsole import stream
 
 # fix tee buffering
 sys.stdout.reconfigure(line_buffering=True)
@@ -20,12 +19,10 @@
 from bbot import __version__
 from bbot.modules import module_loader
 from bbot.core.configurator.args import parser
-from bbot.core.helpers.misc import smart_decode
 from bbot.core.helpers.logger import log_to_stderr
 from bbot.core.configurator import ensure_config_files, check_cli_args, environ
 
 log = logging.getLogger("bbot.cli")
-sys.stdout.reconfigure(line_buffering=True)
 
 
 log_level = get_log_level()
@@ -303,7 +300,12 @@ async def _main():
                 if not options.dry_run:
                     log.trace(f"Command: {' '.join(sys.argv)}")
 
+                    # if we're on the terminal, enable keyboard interaction
                     if sys.stdin.isatty():
+
+                        import fcntl
+                        from bbot.core.helpers.misc import smart_decode
+
                         if not options.agent_mode and not options.yes:
                             log.hugesuccess(f"Scan ready. Press enter to execute {scanner.name}")
                             input()
@@ -324,11 +326,17 @@ def handle_keyboard_input(keyboard_input):
                                 toggle_log_level(logger=log)
                                 scanner.manager.modules_status(_log=True)
 
-                        # Reader
-                        reader = stream.StandardStreamReader()
-                        protocol = stream.StandardStreamReaderProtocol(reader)
+                        reader = asyncio.StreamReader()
+                        protocol = asyncio.StreamReaderProtocol(reader)
                         await asyncio.get_event_loop().connect_read_pipe(lambda: protocol, sys.stdin)
 
+                        # set stdout and stderr to blocking mode
+                        # this is needed to prevent BlockingIOErrors in logging etc.
+                        fds = [sys.stdout.fileno(), sys.stderr.fileno()]
+                        for fd in fds:
+                            flags = fcntl.fcntl(fd, fcntl.F_GETFL)
+                            fcntl.fcntl(fd, fcntl.F_SETFL, flags & ~os.O_NONBLOCK)
+
                         async def akeyboard_listen():
                             try:
                                 allowed_errors = 10

bbot/cli.py

@@ -1,2 +1,2 @@
-from .helpers import make_event_id, is_event_id
+from .helpers import make_event_id
 from .base import make_event, is_event, event_from_json

bbot/core/event/__init__.py

@@ -4,6 +4,7 @@
 import logging
 import ipaddress
 import traceback
+from copy import copy
 from typing import Optional
 from datetime import datetime
 from contextlib import suppress
@@ -29,7 +30,6 @@
     split_host_port,
     tagify,
     validators,
-    truncate_string,
 )
 
 
@@ -106,6 +106,8 @@ class BaseEvent:
     _dummy = False
     # Data validation, if data is a dictionary
     _data_validator = None
+    # Whether to increment scope distance if the child and parent hosts are the same
+    _scope_distance_increment_same_host = False
 
     def __init__(
         self,
@@ -211,6 +213,9 @@ def __init__(
         # an event indicating whether the event has undergone DNS resolution
         self._resolved = asyncio.Event()
 
+        # inherit web spider distance from parent
+        self.web_spider_distance = getattr(self.source, "web_spider_distance", 0)
+
     @property
     def data(self):
         return self._data
@@ -413,7 +418,7 @@ def source(self, source):
             if source.scope_distance >= 0:
                 new_scope_distance = int(source.scope_distance)
                 # only increment the scope distance if the host changes
-                if not hosts_are_same:
+                if self._scope_distance_increment_same_host or not hosts_are_same:
                     new_scope_distance += 1
                 self.scope_distance = new_scope_distance
             # inherit certain tags
@@ -490,7 +495,10 @@ def data_human(self):
         return self._data_human()
 
     def _data_human(self):
-        return truncate_string(str(self.data), n=2000)
+        if isinstance(self.data, (dict, list)):
+            with suppress(Exception):
+                return json.dumps(self.data, sort_keys=True)
+        return smart_decode(self.data)
 
     def _data_load(self, data):
         """
@@ -524,10 +532,7 @@ def pretty_string(self):
         return self._pretty_string()
 
     def _pretty_string(self):
-        if isinstance(self.data, dict):
-            with suppress(Exception):
-                return json.dumps(self.data, sort_keys=True)
-        return smart_decode(self.data)
+        return self._data_human()
 
     @property
     def data_graph(self):
@@ -753,9 +758,6 @@ def sanitize_data(self, data):
             self.parsed = validators.validate_url_parsed(url)
         return data
 
-    def _data_human(self):
-        return json.dumps(self.data, sort_keys=True)
-
     def _data_load(self, data):
         if isinstance(data, str):
             return json.loads(data)
@@ -880,7 +882,6 @@ class URL_UNVERIFIED(BaseEvent):
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
-        self.web_spider_distance = getattr(self.source, "web_spider_distance", 0)
         # increment the web spider distance
         if self.type == "URL_UNVERIFIED" and getattr(self.module, "name", "") != "TARGET":
             self.web_spider_distance += 1
@@ -967,6 +968,11 @@ class _data_validator(BaseModel):
         url: str
         _validate_url = field_validator("url")(validators.validate_url)
 
+    def sanitize_data(self, data):
+        data = super().sanitize_data(data)
+        data["name"] = data["name"].lower()
+        return data
+
     def _words(self):
         return self.data["name"]
 
@@ -999,6 +1005,7 @@ def __init__(self, *args, **kwargs):
     def sanitize_data(self, data):
         url = data.get("url", "")
         self.parsed = validators.validate_url_parsed(url)
+        data["url"] = self.parsed.geturl()
 
         header_dict = {}
         for i in data.get("raw_header", "").splitlines():
@@ -1164,9 +1171,10 @@ class USERNAME(BaseEvent):
     _quick_emit = True
 
 
-class SOCIAL(DictEvent):
+class SOCIAL(DictHostEvent):
     _always_emit = True
     _quick_emit = True
+    _scope_distance_increment_same_host = True
 
 
 class WEBSCREENSHOT(DictHostEvent):
@@ -1186,13 +1194,13 @@ class WAF(DictHostEvent):
     class _data_validator(BaseModel):
         url: str
         host: str
-        WAF: str
+        waf: str
         info: Optional[str] = None
         _validate_url = field_validator("url")(validators.validate_url)
         _validate_host = field_validator("host")(validators.validate_host)
 
     def _pretty_string(self):
-        return self.data["WAF"]
+        return self.data["waf"]
 
 
 def make_event(
@@ -1256,9 +1264,10 @@ def make_event(
         tags = []
     elif isinstance(tags, str):
         tags = [tags]
-    tags = list(tags)
+    tags = set(tags)
 
     if is_event(data):
+        data = copy(data)
         if scan is not None and not data.scan:
             data.scan = scan
         if scans is not None and not data.scans:
@@ -1269,6 +1278,8 @@ def make_event(
             data.source = source
         if internal == True:
             data.internal = True
+        if tags:
+            data.tags = tags.union(data.tags)
         event_type = data.type
         return data
     else:
@@ -1299,7 +1310,7 @@ def make_event(
         # USERNAME <--> EMAIL_ADDRESS confusion
         if event_type == "USERNAME" and validators.soft_validate(data, "email"):
             event_type = "EMAIL_ADDRESS"
-            tags.append("affiliate")
+            tags.add("affiliate")
 
         event_class = globals().get(event_type, DefaultEvent)
 

bbot/core/event/base.py

@@ -3,8 +3,8 @@
 from contextlib import suppress
 
 from bbot.core.errors import ValidationError
+from bbot.core.helpers.regexes import event_type_regexes
 from bbot.core.helpers import sha1, smart_decode, smart_encode_punycode
-from bbot.core.helpers.regexes import event_type_regexes, event_id_regex
 
 
 log = logging.getLogger("bbot.core.event.helpers")
@@ -52,11 +52,5 @@ def get_event_type(data):
     raise ValidationError(f'Unable to autodetect event type from "{data}"')
 
 
-def is_event_id(s):
-    if event_id_regex.match(str(s)):
-        return True
-    return False
-
-
 def make_event_id(data, event_type):
     return f"{event_type}:{sha1(data).hexdigest()}"

bbot/core/event/helpers.py

@@ -2,6 +2,7 @@
     "active": "Makes active connections to target systems",
     "affiliates": "Discovers affiliated hostnames/domains",
     "aggressive": "Generates a large amount of network traffic",
+    "baddns": "Runs all modules from the DNS auditing tool BadDNS",
     "cloud-enum": "Enumerates cloud resources",
     "deadly": "Highly aggressive",
     "email-enum": "Enumerates email addresses",
@@ -13,6 +14,7 @@
     "service-enum": "Identifies protocols running on open ports",
     "slow": "May take a long time to complete",
     "social-enum": "Enumerates social media",
+    "repo-enum": "Enumerates code repositories",
     "subdomain-enum": "Enumerates subdomains",
     "subdomain-hijack": "Detects hijackable subdomains",
     "web-basic": "Basic, non-intrusive web scan functionality",