diff --git a/bbot/modules/dnsdumpster.py b/bbot/modules/dnsdumpster.py index 5c0ae29041..2a5b7539b0 100644 --- a/bbot/modules/dnsdumpster.py +++ b/bbot/modules/dnsdumpster.py @@ -1,4 +1,4 @@ -import re +import json from bbot.modules.templates.subdomain_enum import subdomain_enum @@ -15,78 +15,61 @@ class dnsdumpster(subdomain_enum): base_url = "https://dnsdumpster.com" + async def setup(self): + self.apikey_regex = self.helpers.re.compile(r']*data-form-id="mainform"[^>]*hx-headers=\'([^\']*)\'') + return True + async def query(self, domain): ret = [] - # first, get the CSRF tokens + # first, get the JWT token from the main page res1 = await self.api_request(self.base_url) status_code = getattr(res1, "status_code", 0) - if status_code in [429]: - self.verbose(f'Too many requests "{status_code}"') - return ret - elif status_code not in [200]: + if status_code not in [200]: self.verbose(f'Bad response code "{status_code}" from DNSDumpster') return ret - else: - self.debug(f'Valid response code "{status_code}" from DNSDumpster') - - html = self.helpers.beautifulsoup(res1.content, "html.parser") - if html is False: - self.verbose("BeautifulSoup returned False") - return ret - csrftoken = None - csrfmiddlewaretoken = None + # Extract JWT token from the form's hx-headers attribute using regex + jwt_token = None try: - for cookie in res1.headers.get("set-cookie", "").split(";"): - try: - k, v = cookie.split("=", 1) - except ValueError: - self.verbose("Error retrieving cookie") - return ret - if k == "csrftoken": - csrftoken = str(v) - csrfmiddlewaretoken = html.find("input", {"name": "csrfmiddlewaretoken"}).attrs.get("value", None) - except AttributeError: - pass + # Look for the form with data-form-id="mainform" and extract hx-headers + form_match = await self.helpers.re.search(self.apikey_regex, res1.text) + if form_match: + headers_json = form_match.group(1) + headers_data = json.loads(headers_json) + jwt_token = headers_data.get("Authorization") + except (AttributeError, json.JSONDecodeError, KeyError): + self.log.warning("Error obtaining JWT token") + return ret - # Abort if we didn't get the tokens - if not csrftoken or not csrfmiddlewaretoken: - self.verbose("Error obtaining CSRF tokens") + # Abort if we didn't get the JWT token + if not jwt_token: + self.verbose("Error obtaining JWT token") self.errorState = True return ret else: - self.debug("Successfully obtained CSRF tokens") + self.debug("Successfully obtained JWT token") if self.scan.stopping: - return + return ret - # Otherwise, do the needful - subdomains = set() + # Query the API with the JWT token res2 = await self.api_request( - f"{self.base_url}/", + "https://api.dnsdumpster.com/htmld/", method="POST", - cookies={"csrftoken": csrftoken}, - data={ - "csrfmiddlewaretoken": csrfmiddlewaretoken, - "targetip": str(domain).lower(), - "user": "free", - }, + data={"target": str(domain).lower()}, headers={ - "origin": "https://dnsdumpster.com", - "referer": "https://dnsdumpster.com/", + "Authorization": jwt_token, + "Content-Type": "application/x-www-form-urlencoded", + "Origin": "https://dnsdumpster.com", + "Referer": "https://dnsdumpster.com/", + "HX-Request": "true", + "HX-Target": "results", + "HX-Current-URL": "https://dnsdumpster.com/", }, ) status_code = getattr(res2, "status_code", 0) if status_code not in [200]: - self.verbose(f'Bad response code "{status_code}" from DNSDumpster') - return ret - html = self.helpers.beautifulsoup(res2.content, "html.parser") - if html is False: - self.verbose("BeautifulSoup returned False") + self.verbose(f'Bad response code "{status_code}" from DNSDumpster API') return ret - escaped_domain = re.escape(domain) - match_pattern = re.compile(r"^[\w\.-]+\." + escaped_domain + r"$") - for subdomain in html.findAll(text=match_pattern): - subdomains.add(str(subdomain).strip().lower()) - return list(subdomains) + return await self.scan.extract_in_scope_hostnames(res2.text) diff --git a/bbot/test/test_step_2/module_tests/test_module_dnsdumpster.py b/bbot/test/test_step_2/module_tests/test_module_dnsdumpster.py index 714a610c0c..8f5f99e838 100644 --- a/bbot/test/test_step_2/module_tests/test_module_dnsdumpster.py +++ b/bbot/test/test_step_2/module_tests/test_module_dnsdumpster.py @@ -5,13 +5,11 @@ class TestDNSDumpster(ModuleTestBase): async def setup_after_prep(self, module_test): module_test.httpx_mock.add_response( url="https://dnsdumpster.com", - headers={"Set-Cookie": "csrftoken=asdf"}, - content=b'\n\n \n\n \n \n\n \n \n \n \n DNSdumpster.com - dns recon and research, find and lookup dns records\n\n\n \n \n \n\n \n \n\n \n\n \n\n
\n
\n\n
\n
\n\n
\n
\n \n
\n
\n\n\n\n\n
\n
\n

dns recon & research, find & lookup dns records

\n

\n

\n
\n
\n
\n\n\n\n\n
\n
\n
\n
\n
Loading...
\n
\n
\n
\n

\n\n
\n\n
\n\n

DNSdumpster.com is a FREE domain research tool that can discover hosts related to a domain. Finding visible hosts from the attackers perspective is an important part of the security assessment process.

\n\n
\n\n

this is a project

\n\n\n
\n
\n
\n

\n

Open Source Intelligence for Networks

\n
\n
\n
\n
\n \n \n \n

Attack

\n

The ability to quickly identify the attack surface is essential. Whether you are penetration testing or chasing bug bounties.

\n
\n
\n \n \n \n

Defend

\n

Network defenders benefit from passive reconnaissance in a number of ways. With analysis informing information security strategy.

\n
\n
\n \n \n \n

Learn

\n

Understanding network based OSINT helps information technologists to better operate, assess and manage the network.

\n
\n
\n
\n\n\n\n\n
\n\n \n

Map an organizations attack surface with a virtual dumpster dive* of the DNS records associated with the target organization.

\n

*DUMPSTER DIVING: The practice of sifting refuse from an office or technical installation to extract confidential data, especially security-compromising information.

\n
\n\n\n
\n\n

Frequently Asked Questions

\n\n

How can I take my security assessments to the next level?

\n\n

The company behind DNSDumpster is hackertarget.com where we provide online hosted access to trusted open source security vulnerability scanners and network intelligence tools.

Save time and headaches by incorporating our attack surface discovery into your vulnerability assessment process.

HackerTarget.com | Online Security Testing and Open Source Intelligence

\n\n

What data does DNSDumpster use?

\n\n

No brute force subdomain enumeration is used as is common in dns recon tools that enumerate subdomains. We use open source intelligence resources to query for related domain data. It is then compiled into an actionable resource for both attackers and defenders of Internet facing systems.

\n

More than a simple DNS lookup this tool will discover those hard to find sub-domains and web hosts. The search relies on data from our crawls of the Alexa Top 1 Million sites, Search Engines, Common Crawl, Certificate Transparency, Max Mind, Team Cymru, Shodan and scans.io.

\n\n

I have hit the host limit, do you have a PRO option?

\n\n

Over at hackertarget.com there\'s a tool we call domain profiler. This compiles data similar to DNSDumpster; with additional data discovery. Queries available are based on the membership plan with the number of results (subdomains) being unlimited. With a STARTER membership you have access to the domain profiler tool for 12 months. Once the years membership expires you will revert to BASIC member status, however access to Domain Profiler and Basic Nmap scans continue. The BASIC access does not expire.

\n\n

What are some other resources and tools for learning more?

\n\n

There are some great open source recon frameworks that have been developed over the past couple of years. In addition tools such as Metasploit and Nmap include various modules for enumerating DNS. Check our Getting Started with Footprinting for more information.

\n\n
\n\n\n
\n\n\n\n \n \n \n \n\n\n\n\n\n\n \n \n \n \n\n\n\n\n\n\n\n\n\n \n\n', + content=b"""
""", ) module_test.httpx_mock.add_response( - url="https://dnsdumpster.com/", - method="POST", - content=b'\n\n \n\n \n \n\n \n \n \n \n DNSdumpster.com - dns recon and research, find and lookup dns records\n\n\n \n \n \n\n \n \n\n \n\n \n\n
\n
\n\n
\n
\n\n
\n
\n \n
\n
\n\n\n\n\n
\n
\n

dns recon & research, find & lookup dns records

\n

\n

\n\n
\n
\n\n\n\n\n
\n
\n
\n
\n
Loading...
\n
\n
\n
\n

\n\n
\n\n

Showing results for blacklanternsecurity.com

\n
\n
\n
\n
\n

Hosting (IP block owners)

\n
\n
\n

GeoIP of Host Locations

\n
\n
\n
\n\n

DNS Servers

\n
\n \n \n \n \n \n \n
ns01.domaincontrol.com.
\n\n\n \n
\n
\n
97.74.100.1
ns01.domaincontrol.com
GODADDY-DNS
United States
ns02.domaincontrol.com.
\n\n\n \n
\n
\n
173.201.68.1
ns02.domaincontrol.com
GODADDY-DNS
United States
\n
\n\n

MX Records ** This is where email for the domain goes...

\n
\n \n \n \n \n
asdf.blacklanternsecurity.com.mail.protection.outlook.com.
\n\n\n
\n
\n
104.47.55.138
mail-bn8nam120138.inbound.protection.outlook.com
MICROSOFT-CORP-MSN-AS-BLOCK
United States
\n
\n\n

TXT Records ** Find more hosts in Sender Policy Framework (SPF) configurations

\n
\n \n\n\n\n\n\n\n\n\n\n
"MS=ms26206678"
"v=spf1 ip4:50.240.76.25 include:spf.protection.outlook.com -all"
"google-site-verification=O_PoQFTGJ_hZ9LqfNT9OEc0KPFERKHQ_1t1m0YTx_1E"
"google-site-verification=7XKUMxJSTHBSzdvT7gH47jLRjNAS76nrEfXmzhR_DO4"
\n
\n\n\n

Host Records (A) ** this data may not be current as it uses a static database (updated monthly)

\n
\n \n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
blacklanternsecurity.com
\n\n\n\n
\n
\n\n\n
HTTP: \n GitHub.com\n\n\n\n\n\n\n\n\n
HTTP TECH: \n varnish\n\n\n\n
185.199.108.153
cdn-185-199-108-153.github.com
FASTLY
United States
asdf.blacklanternsecurity.com
\n\n\n\n
\n
\n\n\n\n\n\n
SSH: \n SSH-2.0-OpenSSH_8.2p1 Ubuntu-4ubuntu0.3\n\n\n\n\n\n\n\n
143.244.156.80
asdf.blacklanternsecurity.com
DIGITALOCEAN-ASN
United States
asdf.blacklanternsecurity.com
\n\n\n\n
\n
\n\n\n
HTTP: \n Apache/2.4.29 (Ubuntu)\n\n\n\n\n\n\n\n\n
HTTP TECH: \n Ubuntu
Apache,2.4.29
\n\n\n\n
64.227.8.231
asdf.blacklanternsecurity.com
DIGITALOCEAN-ASN
United States
asdf.blacklanternsecurity.com
\n\n\n\n
\n
\n\n\n\n\n\n\n\n\n\n\n\n
192.34.56.157
asdf.blacklanternsecurity.com
DIGITALOCEAN-ASN
United States
asdf.blacklanternsecurity.com
\n\n\n\n
\n
\n\n\n\n\n\n\n\n\n\n\n\n
192.241.216.208
asdf.blacklanternsecurity.com
DIGITALOCEAN-ASN
United States
asdf.blacklanternsecurity.com
\n\n\n\n
\n
\n\n\n\n\n\n\n\n\n\n\n\n
167.71.95.71
asdf.blacklanternsecurity.com
DIGITALOCEAN-ASN
United States
asdf.blacklanternsecurity.com
\n\n\n\n
\n
\n\n\n\n\n\n\n\n\n\n\n\n
157.245.247.197
asdf.blacklanternsecurity.com
DIGITALOCEAN-ASN
United States
\n
\n\n\n\n\n\n
\n

Mapping the domain ** click for full size image

\n

\n\n

\n
\n\n
\n\n

DNSdumpster.com is a FREE domain research tool that can discover hosts related to a domain. Finding visible hosts from the attackers perspective is an important part of the security assessment process.

\n\n
\n\n

this is a project

\n\n\n
\n
\n
\n

\n

Open Source Intelligence for Networks

\n
\n
\n
\n
\n \n \n \n

Attack

\n

The ability to quickly identify the attack surface is essential. Whether you are penetration testing or chasing bug bounties.

\n
\n
\n \n \n \n

Defend

\n

Network defenders benefit from passive reconnaissance in a number of ways. With analysis informing information security strategy.

\n
\n
\n \n \n \n

Learn

\n

Understanding network based OSINT helps information technologists to better operate, assess and manage the network.

\n
\n
\n
\n\n\n\n\n
\n\n \n

Map an organizations attack surface with a virtual dumpster dive* of the DNS records associated with the target organization.

\n

*DUMPSTER DIVING: The practice of sifting refuse from an office or technical installation to extract confidential data, especially security-compromising information.

\n
\n\n\n
\n\n

Frequently Asked Questions

\n\n

How can I take my security assessments to the next level?

\n\n

The company behind DNSDumpster is hackertarget.com where we provide online hosted access to trusted open source security vulnerability scanners and network intelligence tools.

Save time and headaches by incorporating our attack surface discovery into your vulnerability assessment process.

HackerTarget.com | Online Security Testing and Open Source Intelligence

\n\n

What data does DNSDumpster use?

\n\n

No brute force subdomain enumeration is used as is common in dns recon tools that enumerate subdomains. We use open source intelligence resources to query for related domain data. It is then compiled into an actionable resource for both attackers and defenders of Internet facing systems.

\n

More than a simple DNS lookup this tool will discover those hard to find sub-domains and web hosts. The search relies on data from our crawls of the Alexa Top 1 Million sites, Search Engines, Common Crawl, Certificate Transparency, Max Mind, Team Cymru, Shodan and scans.io.

\n\n

I have hit the host limit, do you have a PRO option?

\n\n

Over at hackertarget.com there\'s a tool we call domain profiler. This compiles data similar to DNSDumpster; with additional data discovery. Queries available are based on the membership plan with the number of results (subdomains) being unlimited. With a STARTER membership you have access to the domain profiler tool for 12 months. Once the years membership expires you will revert to BASIC member status, however access to Domain Profiler and Basic Nmap scans continue. The BASIC access does not expire.

\n\n

What are some other resources and tools for learning more?

\n\n

There are some great open source recon frameworks that have been developed over the past couple of years. In addition tools such as Metasploit and Nmap include various modules for enumerating DNS. Check our Getting Started with Footprinting for more information.

\n\n
\n\n\n\n\n\n\n
\n
\n\n\n \n \n \n \n\n\n\n\n\n\n \n \n \n \n\n\n\n \n \n \n\n \n\n\n\n\n\n\n\n\n\n\n\n\n \n\n', + url="https://api.dnsdumpster.com/htmld/", + content=b"asdf.blacklanternsecurity.com", ) def check(self, module_test, events):