Two new modules, bugfixes and optimizations

bbot/core/event/base.py

@@ -26,8 +26,12 @@
 
 class BaseEvent:
 
-    _dummy = False
+    # Exclude from output modules
     _omit = False
+    # Priority, 1-5, lower numbers == higher priority
+    _priority = 3
+    # Disables certain data validations
+    _dummy = False
 
     def __init__(
         self,
@@ -291,9 +295,30 @@ def json(self):
                     j[k] = smart_decode(v)
         return j
 
+    @property
+    def priority(self):
+        self_priority = int(max(1, min(5, self._priority)))
+        mod_priority = int(max(1, min(5, getattr(self.module, "priority", 1))))
+        return self_priority + mod_priority
+
     def __iter__(self):
+        """
+        For dict(event)
+        """
         yield from self.json.items()
 
+    def __lt__(self, other):
+        """
+        For queue sorting
+        """
+        return self.priority < int(getattr(other, "priority", 5))
+
+    def __gt__(self, other):
+        """
+        For queue sorting
+        """
+        return self.priority > int(getattr(other, "priority", 5))
+
     def __eq__(self, other):
         try:
             other = make_event(other, dummy=True)

bbot/core/helpers/dns.py

@@ -7,7 +7,6 @@
 from concurrent.futures import ThreadPoolExecutor
 
 from .regexes import dns_name_regex
-from ...modules.base import BaseModule
 from bbot.core.errors import ValidationError
 from ..threadpool import ThreadPoolWrapper, NamedLock
 from .misc import is_ip, domain_parents, parent_domain, rand_string
@@ -104,9 +103,11 @@ def resolve_raw(self, query, **kwargs):
 
     def resolve_event(self, event):
         result = self._resolve_event(event)
+        # if it's a wildcard, go again with _wildcard.{domain}
         if len(result) == 1:
             event = result[0]
             return self._resolve_event(event, check_wildcard=False)
+        # else we're good
         else:
             return result
 
@@ -115,7 +116,7 @@ def _resolve_event(self, event, check_wildcard=True):
         Tag event with appropriate dns record types
         Optionally create child events from dns resolutions
         """
-        if not event.host:
+        if not event.host or event.type in ("IP_RANGE",):
             return [], set(), False
         children = []
         event_tags = set()
@@ -353,7 +354,7 @@ def _resolve_hostname(self, query, **kwargs):
             return results
         try:
             results = list(self._catch(self.resolver.resolve, query, **kwargs))
-        except dns.resolver.NoNameservers:
+        except (dns.resolver.NoNameservers, dns.exception.Timeout, dns.resolver.LifetimeTimeout):
             with self._error_lock:
                 try:
                     self._errors[hash(parent)] += 1
@@ -370,6 +371,8 @@ def _resolve_ip(self, query, **kwargs):
             results = list(self._catch(self.resolver.resolve_address, query, **kwargs))
         except dns.resolver.NoNameservers as e:
             self.debug(f"{e} (query={query}, kwargs={kwargs})")
+        except (dns.exception.Timeout, dns.resolver.LifetimeTimeout):
+            pass
         self.debug(f"Results for {query} with kwargs={kwargs}: {results}")
         return results
 
@@ -378,8 +381,9 @@ def _catch(self, callback, *args, **kwargs):
             return callback(*args, **kwargs)
         except dns.resolver.NoNameservers:
             raise
-        except dns.exception.Timeout:
+        except (dns.exception.Timeout, dns.resolver.LifetimeTimeout):
             log.debug(f"DNS query with args={args}, kwargs={kwargs} timed out after {self.timeout} seconds")
+            raise
         except dns.exception.DNSException as e:
             self.debug(f"{e} (args={args}, kwargs={kwargs})")
         except Exception:
@@ -457,14 +461,6 @@ def _get_dummy_module(self, name):
             try:
                 dummy_module = self._dummy_modules[name]
             except KeyError:
-                dummy_module = DNSDummyModule(name=name, scan=self.parent_helper.scan)
+                dummy_module = self.parent_helper._make_dummy_module(name=name, _type="DNS")
                 self._dummy_modules[name] = dummy_module
         return dummy_module
-
-
-class DNSDummyModule(BaseModule):
-    _type = "DNS"
-
-    def __init__(self, *args, **kwargs):
-        self._name = kwargs.pop("name")
-        super().__init__(*args, **kwargs)

bbot/core/helpers/helper.py

@@ -9,6 +9,7 @@
 from .diff import HttpCompare
 from .wordcloud import WordCloud
 from ..threadpool import as_completed
+from ...modules.base import BaseModule
 from .depsinstaller import DepsInstaller
 
 log = logging.getLogger("bbot.core.helpers")
@@ -66,6 +67,12 @@ def scan(self):
     def as_completed(*args, **kwargs):
         return as_completed(*args, **kwargs)
 
+    def _make_dummy_module(self, name, _type):
+        """
+        Construct a dummy module, for attachment to events
+        """
+        return DummyModule(scan=self.scan, name=name, _type=_type)
+
     def __getattribute__(self, attr):
         """
         Allow static functions from sub-helpers to be accessed from the main class
@@ -84,3 +91,10 @@ def __getattribute__(self, attr):
                 except AttributeError:
                     # then die
                     raise AttributeError(f'Helper has no attribute "{attr}"')
+
+
+class DummyModule(BaseModule):
+    def __init__(self, *args, **kwargs):
+        self._name = kwargs.pop("name")
+        self._type = kwargs.pop("_type")
+        super().__init__(*args, **kwargs)

bbot/core/helpers/misc.py

@@ -106,6 +106,21 @@ def domain_parents(d, include_self=False):
         break
 
 
+def tldextract(data):
+    """
+    "www.evilcorp.co.uk" --> ExtractResult(subdomain='www', domain='evilcorp', suffix='co.uk')
+    """
+    return _tldextract.extract(smart_decode(data))
+
+
+def split_domain(hostname):
+    """
+    "www.internal.evilcorp.co.uk" --> ("www.internal", "evilcorp.co.uk")
+    """
+    parsed = tldextract(hostname)
+    return (parsed.subdomain, parsed.registered_domain)
+
+
 def ip_network_parents(i, include_self=False):
     """
     "192.168.1.1" --> [192.168.1.0/31, 192.168.1.0/30 ... 128.0.0.0/1, 0.0.0.0/0]
@@ -228,21 +243,6 @@ def smart_encode(data):
     return str(data).encode("utf-8", errors="ignore")
 
 
-def tldextract(data):
-    """
-    "www.evilcorp.co.uk" --> ExtractResult(subdomain='www', domain='evilcorp', suffix='co.uk')
-    """
-    return _tldextract.extract(smart_decode(data))
-
-
-def split_domain(hostname):
-    """
-    "www.internal.evilcorp.co.uk" --> ("www.internal", "evilcorp.co.uk")
-    """
-    parsed = tldextract(hostname)
-    return (parsed.subdomain, parsed.registered_domain)
-
-
 rand_pool = string.ascii_lowercase + string.digits
 
 

bbot/modules/azure_tenant.py

@@ -0,0 +1,77 @@
+import re
+
+from .viewdns import viewdns
+
+
+class azure_tenant(viewdns):
+    watched_events = ["DNS_NAME"]
+    produced_events = ["DNS_NAME"]
+    flags = ["subdomain-enum", "passive"]
+    base_url = "https://autodiscover-s.outlook.com"
+    in_scope_only = True
+
+    def setup(self):
+        self.processed = set()
+        self.d_xml_regex = re.compile(r"<Domain>([^<>/]*)</Domain>", re.I)
+        return True
+
+    def handle_event(self, event):
+        _, query = self.helpers.split_domain(event.data)
+        if hash(query) in self.processed:
+            self.debug(f'Already processed "{query}", skipping')
+            return
+        self.processed.add(hash(query))
+        domains, _ = self.query(query)
+        if domains:
+            self.success(f'Found {len(domains):,} domains under tenant for "{query}')
+        for domain in domains:
+            self.emit_event(domain, "DNS_NAME", source=event)
+        # todo: tenants?
+
+    def query(self, domain):
+        url = f"{self.base_url}/autodiscover/autodiscover.svc"
+        data = f"""<?xml version="1.0" encoding="utf-8"?>
+<soap:Envelope xmlns:exm="http://schemas.microsoft.com/exchange/services/2006/messages" xmlns:ext="http://schemas.microsoft.com/exchange/services/2006/types" xmlns:a="http://www.w3.org/2005/08/addressing" xmlns:soap="http://schemas.xmlsoap.org/soap/envelope/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema">
+    <soap:Header>
+        <a:Action soap:mustUnderstand="1">http://schemas.microsoft.com/exchange/2010/Autodiscover/Autodiscover/GetFederationInformation</a:Action>
+        <a:To soap:mustUnderstand="1">https://autodiscover-s.outlook.com/autodiscover/autodiscover.svc</a:To>
+        <a:ReplyTo>
+            <a:Address>http://www.w3.org/2005/08/addressing/anonymous</a:Address>
+        </a:ReplyTo>
+    </soap:Header>
+    <soap:Body>
+        <GetFederationInformationRequestMessage xmlns="http://schemas.microsoft.com/exchange/2010/Autodiscover">
+            <Request>
+                <Domain>{domain}</Domain>
+            </Request>
+        </GetFederationInformationRequestMessage>
+    </soap:Body>
+</soap:Envelope>"""
+
+        headers = {
+            "Content-Type": "text/xml; charset=utf-8",
+            "SOAPAction": '"http://schemas.microsoft.com/exchange/2010/Autodiscover/Autodiscover/GetFederationInformation"',
+            "User-Agent": "AutodiscoverClient",
+            "Accept-Encoding": "identity",
+        }
+
+        self.debug(f"Retrieving tenant domains at {url}")
+
+        r = self.helpers.request(url, method="POST", headers=headers, data=data)
+        status_code = getattr(r, "status_code", 0)
+        if status_code not in (200,):
+            self.warning(f"Error retrieving reverse whois results (status code: {status_code})")
+        found_domains = list(set(self.d_xml_regex.findall(r.text)))
+        domains = set()
+        tenantnames = set()
+
+        for d in found_domains:
+            # tenant names
+            if d.lower().endswith(".onmicrosoft.com"):
+                tenantnames.add(d.split(".")[0].lower())
+            # make sure we don't make any unnecessary api calls
+            _, query = self.helpers.split_domain(d)
+            self.processed.add(hash(query))
+            domains.add(str(d).lower())
+
+        return domains, tenantnames

bbot/modules/base.py

@@ -15,7 +15,7 @@ class BaseModule:
     watched_events = []
     # Event types to produce
     produced_events = []
-    # Flags
+    # Flags, must include either "passive" or "active"
     flags = []
     # python dependencies (pip install ____)
     deps_pip = []
@@ -50,8 +50,8 @@ class BaseModule:
     batch_size = 1
     # Seconds to wait before force-submitting batch
     batch_wait = 10
-    # Priority, smaller numbers run first
-    _priority = 0
+    # Priority of events raised by this module, 1-5, lower numbers == higher priority
+    _priority = 3
     # Name, overridden automatically
     _name = "base"
     # Type, for differentiating between normal modules and output modules, etc.
@@ -156,9 +156,12 @@ def _handle_batch(self, force=False):
         return False
 
     def make_event(self, *args, **kwargs):
+        raise_error = kwargs.pop("raise_error", False)
         try:
             event = self.scan.make_event(*args, **kwargs)
         except ValidationError as e:
+            if raise_error:
+                raise
             self.warning(f"{e}")
             return
         if not event.module:
@@ -406,6 +409,10 @@ def event_queue(self):
             self._event_queue = queue.SimpleQueue()
         return self._event_queue
 
+    @property
+    def priority(self):
+        return int(max(1, min(5, self._priority)))
+
     @property
     def log(self):
         if self._log is None:

bbot/modules/crobat.py

@@ -27,21 +27,29 @@ def filter_event(self, event):
             if event not in self.scan.target:
                 valid = True
             query = self.helpers.parent_domain(event.data)
-        if valid and hash(query) not in self.processed:
+        if valid and not self.already_processed(query):
             is_wildcard, _ = self.helpers.is_wildcard(query)
             if not is_wildcard:
                 return True
 
+    def already_processed(self, hostname):
+        for parent in self.helpers.domain_parents(hostname, include_self=True):
+            if hash(parent) in self.processed:
+                return True
+        return False
+
     def handle_event(self, event):
         if "target" in event.tags:
             query = str(event.data)
         else:
             query = self.helpers.parent_domain(event.data).lower()
 
-        if hash(query) in self.processed:
+        if self.already_processed(query):
             self.debug(f'Already processed "{query}", skipping')
             return
 
+        self.processed.add(hash(query))
+
         results = self.query(query)
         if results:
             for hostname in results:

bbot/modules/dnszonetransfer.py

@@ -40,7 +40,7 @@ def handle_event(self, event):
                 else:
                     parent_data = f"{name}.{domain}"
                 parent_event = self.make_event(parent_data, "DNS_NAME", event)
-                if parent_event == event:
+                if not parent_event or parent_event == event:
                     parent_event = event
                 else:
                     self.emit_event(parent_event)