Merge pull request #1380 from domwhewell-sage/code_repository_regex

TheTechromancer · web-flow · commit 9e6c83cdd788 · 2024-05-15T14:40:26.000-04:00
New Module: Code Repository
diff --git a/bbot/core/flags.py b/bbot/core/flags.py
@@ -14,6 +14,7 @@
     "service-enum": "Identifies protocols running on open ports",
     "slow": "May take a long time to complete",
     "social-enum": "Enumerates social media",
+    "repo-enum": "Enumerates code repositories",
     "subdomain-enum": "Enumerates subdomains",
     "subdomain-hijack": "Detects hijackable subdomains",
     "web-basic": "Basic, non-intrusive web scan functionality",
diff --git a/bbot/modules/code_repository.py b/bbot/modules/code_repository.py
@@ -0,0 +1,48 @@
+import re
+from bbot.modules.base import BaseModule
+
+
+class code_repository(BaseModule):
+    watched_events = ["URL_UNVERIFIED"]
+    produced_events = ["CODE_REPOSITORY"]
+    meta = {"description": "Look for code repository links in webpages"}
+    flags = ["passive", "safe", "repo-enum"]
+
+    # platform name : (regex, case_sensitive)
+    code_repositories = {
+        "git": [
+            (r"github.com/[a-zA-Z0-9_-]+/[a-zA-Z0-9_-]+", False),
+            (r"gitlab.(?:com|org)/[a-zA-Z0-9_-]+/[a-zA-Z0-9_-]+", False),
+        ],
+        "docker": (r"hub.docker.com/r/[a-zA-Z0-9_-]+/[a-zA-Z0-9_-]+", False),
+    }
+
+    scope_distance_modifier = 1
+
+    async def setup(self):
+        self.compiled_regexes = {}
+        for k, v in self.code_repositories.items():
+            if isinstance(v, list):
+                self.compiled_regexes[k] = [(re.compile(pattern), c) for pattern, c in v]
+            else:
+                pattern, c = v
+                self.compiled_regexes[k] = (re.compile(pattern), c)
+        return True
+
+    async def handle_event(self, event):
+        for platform, regexes in self.compiled_regexes.items():
+            if not isinstance(regexes, list):
+                regexes = [regexes]
+            for regex, case_sensitive in regexes:
+                for match in regex.finditer(event.data):
+                    url = match.group()
+                    if not case_sensitive:
+                        url = url.lower()
+                    repo_event = self.make_event(
+                        {"url": f"https://{url}"},
+                        "CODE_REPOSITORY",
+                        tags=platform,
+                        source=event,
+                    )
+                    repo_event.scope_distance = event.scope_distance
+                    await self.emit_event(repo_event)
diff --git a/bbot/test/test_step_2/module_tests/test_module_code_repository.py b/bbot/test/test_step_2/module_tests/test_module_code_repository.py
@@ -0,0 +1,59 @@
+from .base import ModuleTestBase
+
+
+class TestCodeRepository(ModuleTestBase):
+    targets = ["http://127.0.0.1:8888"]
+    modules_overrides = ["httpx", "excavate", "code_repository"]
+
+    async def setup_after_prep(self, module_test):
+        expect_args = {"method": "GET", "uri": "/"}
+        respond_args = {
+            "response_data": """
+            <html>
+                <a href="https://github.com/blacklanternsecurity/bbot"/>
+                <a href="https://gitlab.com/blacklanternsecurity/bbot"/>
+                <a href="https://gitlab.org/blacklanternsecurity/bbot"/>
+                <a href="https://hub.docker.com/r/blacklanternsecurity/bbot"/>
+            </html>
+            """
+        }
+        module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)
+
+    def check(self, module_test, events):
+        assert 4 == len([e for e in events if e.type == "CODE_REPOSITORY"])
+        assert 1 == len(
+            [
+                e
+                for e in events
+                if e.type == "CODE_REPOSITORY"
+                and "git" in e.tags
+                and e.data["url"] == "https://github.com/blacklanternsecurity/bbot"
+            ]
+        )
+        assert 1 == len(
+            [
+                e
+                for e in events
+                if e.type == "CODE_REPOSITORY"
+                and "git" in e.tags
+                and e.data["url"] == "https://gitlab.com/blacklanternsecurity/bbot"
+            ]
+        )
+        assert 1 == len(
+            [
+                e
+                for e in events
+                if e.type == "CODE_REPOSITORY"
+                and "git" in e.tags
+                and e.data["url"] == "https://gitlab.org/blacklanternsecurity/bbot"
+            ]
+        )
+        assert 1 == len(
+            [
+                e
+                for e in events
+                if e.type == "CODE_REPOSITORY"
+                and "docker" in e.tags
+                and e.data["url"] == "https://hub.docker.com/r/blacklanternsecurity/bbot"
+            ]
+        )
