Convert websocket URLs to HTTP instead of raising findings

bbot/modules/internal/excavate.py

@@ -817,6 +817,19 @@ async def process(self, yara_results, event, yara_rule_settings, discovery_conte
                     except ValueError as e:
                         self.excavate.debug(f"Failed to parse netloc: {e}")
                         continue
+                    # convert websocket URLs to their HTTP equivalents
+                    if parsed_url.scheme in ["ws", "wss"]:
+                        http_scheme = "https" if parsed_url.scheme == "wss" else "http"
+                        http_url = parsed_url._replace(scheme=http_scheme).geturl()
+                        await self.report(
+                            http_url,
+                            event,
+                            yara_rule_settings,
+                            discovery_context,
+                            event_type="URL_UNVERIFIED",
+                        )
+                        continue
+
                     if parsed_url.scheme in ["http", "https"]:
                         continue
 

bbot/test/test_step_2/module_tests/test_module_excavate.py

@@ -473,6 +473,8 @@ class TestExcavateNonHttpScheme(TestExcavate):
     <p>hxxp://test.notreal</p>
     <p>ftp://test.notreal</p>
     <p>nonsense://test.notreal</p>
+    <p>ws://test.notreal</p>
+    <p>wss://test.notreal</p>
     </body>
     </html>
     """
@@ -484,6 +486,9 @@ def check(self, module_test, events):
         found_hxxp_url = False
         found_ftp_url = False
         found_nonsense_url = False
+        found_ws_finding = False
+        found_ws_url = False
+        found_wss_url = False
 
         for e in events:
             if e.type == "FINDING":
@@ -493,9 +498,19 @@ def check(self, module_test, events):
                     found_ftp_url = True
                 if "nonsense" in e.data["description"]:
                     found_nonsense_url = True
+                if "ws://" in e.data.get("description", "") or "wss://" in e.data.get("description", ""):
+                    found_ws_finding = True
+            if e.type == "URL_UNVERIFIED":
+                if e.data.get("url", "") == "http://test.notreal/":
+                    found_ws_url = True
+                if e.data.get("url", "") == "https://test.notreal/":
+                    found_wss_url = True
         assert found_hxxp_url
         assert found_ftp_url
         assert not found_nonsense_url
+        assert not found_ws_finding, "ws:// should not produce a FINDING"
+        assert found_ws_url, "ws:// should be converted to http:// URL_UNVERIFIED"
+        assert found_wss_url, "wss:// should be converted to https:// URL_UNVERIFIED"
 
 
 class TestExcavateParameterExtraction(TestExcavate):