Skip to content
Merged
Show file tree
Hide file tree
Changes from 9 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
92 changes: 57 additions & 35 deletions bbot/core/helpers/web/web.py
Original file line number Diff line number Diff line change
Expand Up @@ -500,8 +500,12 @@ async def wordlist(self, path, lines=None, zip=False, zip_filename=None, **kwarg
Allows for optional line-based truncation and caching. Returns the full path of the wordlist
file or a truncated version of it.

Also accepts a list of paths/URLs, in which case all wordlists are fetched and merged
into a single deduplicated file before being returned.

Args:
path (str): The local or remote path of the wordlist.
path (str | list): The local or remote path of the wordlist, or a list of paths/URLs
to merge into a single deduplicated wordlist.
lines (int, optional): Number of lines to read from the wordlist.
If specified, will return a truncated wordlist with this many lines.
zip (bool, optional): Whether to unzip the file after downloading. Defaults to False.
Expand All @@ -523,47 +527,65 @@ async def wordlist(self, path, lines=None, zip=False, zip_filename=None, **kwarg

Fetching and truncating to the first 100 lines
>>> wordlist_path = await self.helpers.wordlist("/root/rockyou.txt", lines=100)

Merging multiple wordlists into one
>>> wordlist_path = await self.helpers.wordlist(["/custom.txt", "https://example.com/wordlist.txt"])
"""
import zipfile

if not path:
raise WordlistError(f"Invalid wordlist: {path}")
if "cache_hrs" not in kwargs:
# 4320 hrs = 180 days = 6 months
kwargs["cache_hrs"] = 4320
if self.parent_helper.is_url(path):
filename = await self.download(str(path), **kwargs)
if filename is None:
raise WordlistError(f"Unable to retrieve wordlist from {path}")
# Handle list of wordlists - fetch each and merge into a single order-preserving deduplicated file,
# then fall through to the unified truncation logic below
Comment thread
liquidsec marked this conversation as resolved.
Resolved
if not isinstance(path, (str, Path)):
paths = list(path)
if not paths:
raise WordlistError("Wordlist list is empty")
all_words = []
for p in paths:
f = await self.wordlist(p, **kwargs)
all_words.extend(self.parent_helper.read_file(f))
cache_key = "merged_wordlist:" + ":".join(sorted(str(p) for p in paths))
filename = self.parent_helper.cache_filename(cache_key)
with open(filename, "w") as f:
for word in dict.fromkeys(all_words):
f.write(f"{word}\n")
else:
filename = Path(path).resolve()
if not filename.is_file():
raise WordlistError(f"Unable to find wordlist at {path}")

if zip:
if not zip_filename:
raise WordlistError("zip_filename must be specified when zip is True")
try:
with zipfile.ZipFile(filename, "r") as zip_ref:
if zip_filename not in zip_ref.namelist():
raise WordlistError(f"File {zip_filename} not found in the zip archive {filename}")
zip_ref.extract(zip_filename, filename.parent)
filename = filename.parent / zip_filename
except Exception as e:
raise WordlistError(f"Error unzipping file {filename}: {e}")
if not path:
raise WordlistError(f"Invalid wordlist: {path}")
if "cache_hrs" not in kwargs:
# 4320 hrs = 180 days = 6 months
kwargs["cache_hrs"] = 4320
if self.parent_helper.is_url(path):
filename = await self.download(str(path), **kwargs)
if filename is None:
raise WordlistError(f"Unable to retrieve wordlist from {path}")
else:
filename = Path(path).resolve()
if not filename.is_file():
raise WordlistError(f"Unable to find wordlist at {path}")

if zip:
if not zip_filename:
raise WordlistError("zip_filename must be specified when zip is True")
try:
with zipfile.ZipFile(filename, "r") as zip_ref:
if zip_filename not in zip_ref.namelist():
raise WordlistError(f"File {zip_filename} not found in the zip archive {filename}")
zip_ref.extract(zip_filename, filename.parent)
filename = filename.parent / zip_filename
except Exception as e:
raise WordlistError(f"Error unzipping file {filename}: {e}")

if lines is None:
return filename
else:
lines = int(lines)
with open(filename) as f:
read_lines = f.readlines()
cache_key = f"{filename}:{lines}"
truncated_filename = self.parent_helper.cache_filename(cache_key)
with open(truncated_filename, "w") as f:
for line in read_lines[:lines]:
f.write(line)
return truncated_filename
lines = int(lines)
with open(filename) as f:
read_lines = f.readlines()
cache_key = f"{filename}:{lines}"
truncated_filename = self.parent_helper.cache_filename(cache_key)
with open(truncated_filename, "w") as f:
for line in read_lines[:lines]:
f.write(line)
return truncated_filename

def beautifulsoup(
self,
Expand Down
2 changes: 1 addition & 1 deletion bbot/modules/dnsbrute.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ class dnsbrute(subdomain_enum):
"max_depth": 5,
}
options_desc = {
"wordlist": "Subdomain wordlist URL",
"wordlist": "Subdomain wordlist URL or file path. Accepts a list of URLs/paths to merge multiple wordlists (duplicates are removed).",
"max_depth": "How many subdomains deep to brute force, i.e. 5.4.3.2.1.evilcorp.com",
}
deps_common = ["massdns"]
Expand Down
1 change: 1 addition & 0 deletions bbot/modules/legba.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ class legba(BaseModule):
"version": "1.1.1",
}

# All *_wordlist options also accept a list of URLs/paths to merge multiple wordlists (duplicates are removed).
Comment thread
ausmaster marked this conversation as resolved.
Resolved
options_desc = {
"ssh_wordlist": "Wordlist URL for SSH combined username:password wordlist, newline separated",
"ftp_wordlist": "Wordlist URL for FTP combined username:password wordlist, newline separated",
Expand Down
2 changes: 1 addition & 1 deletion bbot/modules/medusa.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ class medusa(BaseModule):
}

options_desc = {
"snmp_wordlist": "Wordlist url for SNMP community strings, newline separated (default https://raw.githubusercontent.com/danielmiessler/SecLists/refs/heads/master/Discovery/SNMP/snmp.txt)",
"snmp_wordlist": "Wordlist url for SNMP community strings, newline separated (default https://raw.githubusercontent.com/danielmiessler/SecLists/refs/heads/master/Discovery/SNMP/snmp.txt). Accepts a list of URLs/paths to merge multiple wordlists (duplicates are removed).",
"snmp_versions": "List of SNMP versions to attempt against the SNMP server (default ['1', '2C'])",
"wait_microseconds": "Wait time after every SNMP request in microseconds (default 200)",
"timeout_s": "Wait time for the SNMP response(s) once at the end of all attempts (default 5)",
Expand Down
2 changes: 1 addition & 1 deletion bbot/modules/paramminer_cookies.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ class paramminer_cookies(paramminer_headers):
"skip_boring_words": True,
}
options_desc = {
"wordlist": "Define the wordlist to be used to derive cookies",
"wordlist": "Define the wordlist to be used to derive cookies. Accepts a list of URLs/paths to merge multiple wordlists (duplicates are removed).",
"recycle_words": "Attempt to use words found during the scan on all other endpoints",
"skip_boring_words": "Remove commonly uninteresting words from the wordlist",
}
Expand Down
2 changes: 1 addition & 1 deletion bbot/modules/paramminer_getparams.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ class paramminer_getparams(paramminer_headers):
"brute_short": False,
}
options_desc = {
"wordlist": "Define the wordlist to be used to derive headers",
"wordlist": "Define the wordlist to be used to derive headers. Accepts a list of URLs/paths to merge multiple wordlists (duplicates are removed).",
"recycle_words": "Attempt to use words found during the scan on all other endpoints",
"skip_boring_words": "Remove commonly uninteresting words from the wordlist",
"mutate_case": (
Expand Down
2 changes: 1 addition & 1 deletion bbot/modules/paramminer_headers.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ class paramminer_headers(BaseModule):
"skip_boring_words": True,
}
options_desc = {
"wordlist": "Define the wordlist to be used to derive headers",
"wordlist": "Define the wordlist to be used to derive headers. Accepts a list of URLs/paths to merge multiple wordlists (duplicates are removed).",
"recycle_words": "Attempt to use words found during the scan on all other endpoints",
"skip_boring_words": "Remove commonly uninteresting words from the wordlist",
}
Expand Down
2 changes: 1 addition & 1 deletion bbot/modules/webbrute.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ class webbrute(BaseModule):
}

options_desc = {
"wordlist": "Specify wordlist to use when finding directories",
"wordlist": "Specify wordlist to use when finding directories. Accepts a list of URLs/paths to merge multiple wordlists (duplicates are removed).",
"lines": "take only the first N lines from the wordlist when finding directories",
"max_depth": "the maximum directory depth to attempt to solve",
"extensions": "Optionally include a list of extensions to extend the keyword with (comma separated)",
Expand Down
2 changes: 1 addition & 1 deletion bbot/modules/webbrute_shortnames.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ class webbrute_shortnames(webbrute):
}

options_desc = {
"wordlist_extensions": "Specify wordlist to use when making extension lists",
"wordlist_extensions": "Specify wordlist to use when making extension lists. Accepts a list of URLs/paths to merge multiple wordlists (duplicates are removed).",
"max_depth": "the maximum directory depth to attempt to solve",
"extensions": "Optionally include a list of extensions to extend the keyword with (comma separated)",
"find_common_prefixes": "Attempt to automatically detect common prefixes and make additional runs against them",
Expand Down
40 changes: 40 additions & 0 deletions bbot/test/test_step_2/module_tests/test_module_dnsbrute.py
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,46 @@ def check(self, module_test, events):
)


class TestDnsbruteMultiWordlist(ModuleTestBase):
"""Test that a list of wordlists is merged and deduplicated via helpers.wordlist()."""

module_name = "dnsbrute"
wordlist_1 = tempwordlist(["www", "shared"])
wordlist_2 = tempwordlist(["asdf", "shared"])
config_overrides = {"modules": {"dnsbrute": {"wordlist": [str(wordlist_1), str(wordlist_2)], "max_depth": 3}}}

async def setup_after_prep(self, module_test):
old_run_live = module_test.scan.helpers.run_live

async def new_run_live(*command, check=False, text=True, **kwargs):
if "massdns" in command[:2]:
_input = [l async for l in kwargs["input"]]
if "asdf.blacklanternsecurity.com" in _input:
Comment thread
github-advanced-security[bot] marked this conversation as resolved.
Fixed
yield """{"name": "asdf.blacklanternsecurity.com.", "type": "A", "class": "IN", "status": "NOERROR", "rx_ts": 1713974911725326170, "data": {"answers": [{"ttl": 86400, "type": "A", "class": "IN", "name": "asdf.blacklanternsecurity.com.", "data": "1.2.3.4."}]}, "flags": ["rd", "ra"], "resolver": "195.226.187.130:53", "proto": "UDP"}"""
else:
async for _ in old_run_live(*command, check=False, text=True, **kwargs):
yield _

module_test.monkeypatch.setattr(module_test.scan.helpers, "run_live", new_run_live)

await module_test.mock_dns(
{
"blacklanternsecurity.com": {"A": ["4.3.2.1"]},
"asdf.blacklanternsecurity.com": {"A": ["1.2.3.4"]},
}
)

def check(self, module_test, events):
# "shared" appears in both wordlists - merged set should deduplicate it
assert module_test.module.wordlist_size == 3, (
f"Expected 3 unique words from 2 overlapping wordlists, got {module_test.module.wordlist_size}"
)
assert module_test.module.subdomain_list == {"www", "asdf", "shared"}
assert any(e.data == "asdf.blacklanternsecurity.com" and str(e.module) == "dnsbrute" for e in events), (
"Expected asdf.blacklanternsecurity.com from second wordlist"
)


class TestDnsbruteCanaryCheck(ModuleTestBase):
"""Test that the canary check correctly aborts brute-forcing on wildcard domains.

Expand Down
Loading