diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c3eadd4414..81dba55baf 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -35,6 +35,8 @@ jobs: uv run ruff check uv run ruff format --check - name: Run tests + env: + BBOT_IO_API_KEY: ${{ secrets.BBOT_IO_API_KEY }} run: | uv run pytest -vv --reruns 2 -o timeout_func_only=true --timeout 1200 --disable-warnings --log-cli-level=INFO --cov-config=bbot/test/coverage.cfg --cov-report xml:cov.xml --cov=bbot . - name: Upload Debug Logs diff --git a/README.md b/README.md index 58f5425db0..79610a4d33 100644 --- a/README.md +++ b/README.md @@ -219,12 +219,7 @@ include: - paramminer - dirbust-light - web-screenshots - - baddns-intense - -config: - modules: - baddns: - enable_references: True + - baddns-heavy ``` diff --git a/bbot/modules/baddns.py b/bbot/modules/baddns.py index c6142fcc5e..5b142f9043 100644 --- a/bbot/modules/baddns.py +++ b/bbot/modules/baddns.py @@ -4,6 +4,37 @@ import logging +SEVERITY_LEVELS = ("INFO", "LOW", "MEDIUM", "HIGH", "CRITICAL") +CONFIDENCE_LEVELS = ("UNKNOWN", "LOW", "MODERATE", "HIGH", "CONFIRMED") + +SUBMODULE_MAX_SEVERITY = { + "CNAME": "MEDIUM", + "NS": "MEDIUM", + "MX": "MEDIUM", + "TXT": "LOW", + "references": "MEDIUM", + "NSEC": "INFO", + "zonetransfer": "INFO", + "DMARC": "INFO", + "SPF": "MEDIUM", + "MTA-STS": "HIGH", + "WILDCARD": "HIGH", +} + +SUBMODULE_MAX_CONFIDENCE = { + "CNAME": "CONFIRMED", + "NS": "HIGH", + "MX": "CONFIRMED", + "TXT": "CONFIRMED", + "references": "CONFIRMED", + "NSEC": "CONFIRMED", + "zonetransfer": "CONFIRMED", + "DMARC": "CONFIRMED", + "SPF": "CONFIRMED", + "MTA-STS": "CONFIRMED", + "WILDCARD": "CONFIRMED", +} + class baddns(BaseModule): watched_events = ["DNS_NAME", "DNS_NAME_UNRESOLVED"] @@ -14,14 +45,15 @@ class baddns(BaseModule): "created_date": "2024-01-18", "author": "@liquidsec", } - options = {"custom_nameservers": [], "only_high_confidence": False, "enabled_submodules": []} + options = {"custom_nameservers": [], "min_severity": "LOW", "min_confidence": "MODERATE", "enabled_submodules": []} options_desc = { "custom_nameservers": "Force BadDNS to use a list of custom nameservers", - "only_high_confidence": "Do not emit low-confidence or generic detections", + "min_severity": "Minimum severity to emit (INFO, LOW, MEDIUM, HIGH, CRITICAL)", + "min_confidence": "Minimum confidence to emit (UNKNOWN, LOW, MODERATE, HIGH, CONFIRMED)", "enabled_submodules": "A list of submodules to enable. Empty list (default) enables CNAME, TXT and MX Only", } module_threads = 8 - deps_pip = ["baddns~=1.12.294"] + deps_pip = ["baddns~=2.0.0"] def select_modules(self): selected_submodules = [] @@ -35,14 +67,50 @@ def set_modules(self): if self.enabled_submodules == []: self.enabled_submodules = ["CNAME", "MX", "TXT"] + def _filter_submodules(self): + filtered = [] + for name in self.enabled_submodules: + max_sev = SUBMODULE_MAX_SEVERITY.get(name) + max_conf = SUBMODULE_MAX_CONFIDENCE.get(name) + if max_sev is None or max_conf is None: + filtered.append(name) + continue + sev_idx = SEVERITY_LEVELS.index(max_sev) if max_sev in SEVERITY_LEVELS else 0 + conf_idx = CONFIDENCE_LEVELS.index(max_conf) if max_conf in CONFIDENCE_LEVELS else 0 + if sev_idx < self._min_sev_idx or conf_idx < self._min_conf_idx: + self.verbose( + f"Auto-disabling submodule [{name}]: max_severity={max_sev}, max_confidence={max_conf} below configured thresholds" + ) + else: + filtered.append(name) + return filtered + + def _meets_threshold(self, severity, confidence): + sev_idx = SEVERITY_LEVELS.index(severity) if severity in SEVERITY_LEVELS else 0 + conf_idx = CONFIDENCE_LEVELS.index(confidence) if confidence in CONFIDENCE_LEVELS else 0 + return sev_idx >= self._min_sev_idx and conf_idx >= self._min_conf_idx + async def setup(self): self.preset.core.logger.include_logger(logging.getLogger("baddns")) self.custom_nameservers = self.config.get("custom_nameservers", []) or None if self.custom_nameservers: self.custom_nameservers = self.helpers.chain_lists(self.custom_nameservers) - self.only_high_confidence = self.config.get("only_high_confidence", False) + min_severity = self.config.get("min_severity", "LOW").upper() + min_confidence = self.config.get("min_confidence", "MODERATE").upper() + if min_severity not in SEVERITY_LEVELS: + self.warning(f"Invalid min_severity: {min_severity}, defaulting to LOW") + min_severity = "LOW" + if min_confidence not in CONFIDENCE_LEVELS: + self.warning(f"Invalid min_confidence: {min_confidence}, defaulting to MODERATE") + min_confidence = "MODERATE" + self._min_sev_idx = SEVERITY_LEVELS.index(min_severity) + self._min_conf_idx = CONFIDENCE_LEVELS.index(min_confidence) self.signatures = load_signatures() self.set_modules() + self.enabled_submodules = self._filter_submodules() + if not self.enabled_submodules: + self.warning("All submodules were disabled by severity/confidence thresholds") + return False all_submodules_list = [m.name for m in get_all_modules()] for m in self.enabled_submodules: if m not in all_submodules_list: @@ -62,11 +130,25 @@ async def _run_module(self, module_instance): self.warning(f"Task for {module_instance} raised an error: {e}") return module_instance, None + def _new_http_client(self, *args, **kwargs): + """Create a non-cached HTTP client for baddns submodules. + + baddns submodules close their HTTP clients during cleanup, so we can't + use the caching ``web.AsyncClient`` factory — that would let one + submodule close a client that another submodule is still using. + + TODO: revisit this when we switch to blasthttp — the caching/lifecycle + model will be different and this workaround may no longer be needed. + """ + from bbot.core.helpers.web.client import BBOTAsyncClient + + return BBOTAsyncClient.from_config(self.scan.config, self.scan.target, *args, persist_cookies=False, **kwargs) + async def handle_event(self, event): coroutines = [] for ModuleClass in self.select_modules(): kwargs = { - "http_client_class": self.scan.helpers.web.AsyncClient, + "http_client_class": self._new_http_client, "dns_client": self.scan.helpers.dns.resolver, "custom_nameservers": self.custom_nameservers, "signatures": self.signatures, @@ -96,46 +178,28 @@ async def handle_event(self, event): r_dict = r.to_dict() confidence = r_dict["confidence"] + severity = r_dict["severity"] - if confidence in ["CONFIRMED", "PROBABLE"]: - data = { - "severity": "MEDIUM", - "name": f"BadDNS {r_dict['signature']}", - "confidence": "HIGH", - "description": f"{r_dict['description']}. Confidence: [{confidence}] Signature: [{r_dict['signature']}] Indicator: [{r_dict['indicator']}] Trigger: [{r_dict['trigger']}] baddns Module: [{r_dict['module']}]", - "host": str(event.host), - } - await self.emit_event( - data, - "FINDING", - event, - tags=[f"baddns-{module_instance.name.lower()}"], - context=f'{{module}}\'s "{r_dict["module"]}" module found {{event.type}}: {r_dict["description"]}', + if not self._meets_threshold(severity, confidence): + self.debug( + f"Skipping result below threshold (severity={severity}, confidence={confidence})" ) - - elif confidence in ["UNLIKELY", "POSSIBLE"]: - if not self.only_high_confidence: - data = { - "name": f"BadDNS {r_dict['signature']}", - "description": f"{r_dict['description']} Confidence: [{confidence}] Signature: [{r_dict['signature']}] Indicator: [{r_dict['indicator']}] Trigger: [{r_dict['trigger']}] baddns Module: [{r_dict['module']}]", - "host": str(event.host), - "severity": "MEDIUM", - "confidence": "LOW", - } - await self.emit_event( - data, - "FINDING", - event, - tags=[f"baddns-{module_instance.name.lower()}"], - context=f'{{module}}\'s "{r_dict["module"]}" module found {{event.type}}: {r_dict["description"]}', - ) - else: - self.debug( - f"Skipping low-confidence result due to only_high_confidence setting: {confidence}" - ) - - else: - self.warning(f"Got unrecognized confidence level: {confidence}") + continue + + data = { + "severity": severity, + "name": f"BadDNS {r_dict['signature']}", + "confidence": confidence, + "description": f"{r_dict['description']}. Confidence: [{confidence}] Signature: [{r_dict['signature']}] Indicator: [{r_dict['indicator']}] Trigger: [{r_dict['trigger']}] baddns Module: [{r_dict['module']}]", + "host": str(event.host), + } + await self.emit_event( + data, + "FINDING", + event, + tags=[f"baddns-{module_instance.name.lower()}"], + context=f'{{module}}\'s "{r_dict["module"]}" module found {{event.type}}: {r_dict["description"]}', + ) found_domains = r_dict.get("found_domains", None) if found_domains: diff --git a/bbot/modules/baddns_direct.py b/bbot/modules/baddns_direct.py index 4c0e409d7d..0eac0f9d14 100644 --- a/bbot/modules/baddns_direct.py +++ b/bbot/modules/baddns_direct.py @@ -1,11 +1,7 @@ -from baddns.base import get_all_modules -from baddns.lib.loader import load_signatures -from .base import BaseModule +from .baddns import baddns as baddns_module -import logging - -class baddns_direct(BaseModule): +class baddns_direct(baddns_module): watched_events = ["URL", "STORAGE_BUCKET"] produced_events = ["FINDING"] flags = ["active", "safe", "subdomain-enum", "baddns", "cloud-enum"] @@ -14,30 +10,19 @@ class baddns_direct(BaseModule): "created_date": "2024-01-29", "author": "@liquidsec", } - options = {"custom_nameservers": []} + options = {"custom_nameservers": [], "min_severity": "LOW", "min_confidence": "MODERATE"} options_desc = { "custom_nameservers": "Force BadDNS to use a list of custom nameservers", + "min_severity": "Minimum severity to emit (INFO, LOW, MEDIUM, HIGH, CRITICAL)", + "min_confidence": "Minimum confidence to emit (UNKNOWN, LOW, MODERATE, HIGH, CONFIRMED)", } module_threads = 8 - deps_pip = ["baddns~=1.12.294"] + deps_pip = ["baddns~=2.0.0"] scope_distance_modifier = 1 - async def setup(self): - self.preset.core.logger.include_logger(logging.getLogger("baddns")) - self.custom_nameservers = self.config.get("custom_nameservers", []) or None - if self.custom_nameservers: - self.custom_nameservers = self.helpers.chain_lists(self.custom_nameservers) - self.only_high_confidence = self.config.get("only_high_confidence", False) - self.signatures = load_signatures() - return True - - def select_modules(self): - selected_modules = [] - for m in get_all_modules(): - if m.name in ["CNAME"]: - selected_modules.append(m) - return selected_modules + def set_modules(self): + self.enabled_submodules = ["CNAME"] async def handle_event(self, event): CNAME_direct_module = self.select_modules()[0] @@ -56,12 +41,19 @@ async def handle_event(self, event): for r in results: r_dict = r.to_dict() + severity = r_dict["severity"] + confidence = r_dict["confidence"] + + if not self._meets_threshold(severity, confidence): + self.debug(f"Skipping result below threshold (severity={severity}, confidence={confidence})") + continue + data = { "name": f"BadDNS {r_dict['signature']}", "description": f"Possible [{r_dict['signature']}] via direct BadDNS analysis. Indicator: [{r_dict['indicator']}] Trigger: [{r_dict['trigger']}] baddns Module: [{r_dict['module']}]", "host": str(event.host), - "severity": "HIGH", - "confidence": "MEDIUM", + "severity": severity, + "confidence": confidence, } await self.emit_event( diff --git a/bbot/modules/baddns_zone.py b/bbot/modules/baddns_zone.py index 7fcf900444..c948135efb 100644 --- a/bbot/modules/baddns_zone.py +++ b/bbot/modules/baddns_zone.py @@ -10,13 +10,14 @@ class baddns_zone(baddns_module): "created_date": "2024-01-29", "author": "@liquidsec", } - options = {"custom_nameservers": [], "only_high_confidence": False} + options = {"custom_nameservers": [], "min_severity": "INFO", "min_confidence": "MODERATE"} options_desc = { "custom_nameservers": "Force BadDNS to use a list of custom nameservers", - "only_high_confidence": "Do not emit low-confidence or generic detections", + "min_severity": "Minimum severity to emit (INFO, LOW, MEDIUM, HIGH, CRITICAL)", + "min_confidence": "Minimum confidence to emit (UNKNOWN, LOW, MODERATE, HIGH, CONFIRMED)", } module_threads = 8 - deps_pip = ["baddns~=1.12.294"] + deps_pip = ["baddns~=2.0.0"] def set_modules(self): self.enabled_submodules = ["NSEC", "zonetransfer"] diff --git a/bbot/modules/badsecrets.py b/bbot/modules/badsecrets.py index 3e3890eb46..b3a00668ba 100644 --- a/bbot/modules/badsecrets.py +++ b/bbot/modules/badsecrets.py @@ -17,7 +17,7 @@ class badsecrets(BaseModule): options_desc = { "custom_secrets": "Include custom secrets loaded from a local file", } - deps_pip = ["badsecrets~=0.13.47"] + deps_pip = ["badsecrets~=1.0.0"] async def setup(self): self.custom_secrets = None @@ -86,8 +86,8 @@ async def handle_event(self, event): # Only vulnerable (SecretFound) JWT results are worth emitting from badsecrets. if r["detecting_module"] == "Generic_JWT": continue - # There is little value to presenting a non-vulnerable asp.net viewstate, as it is not crackable without a Matrioshka brain. Just emit a technology instead. - if r["detecting_module"] == "ASPNET_Viewstate": + # There is little value to presenting a non-vulnerable asp.net viewstate/resource, as it is not crackable without a Matrioshka brain. Just emit a technology instead. + if r["detecting_module"] in ("ASPNET_Viewstate", "ASPNET_Resource"): technology = "microsoft asp.net" await self.emit_event( {"technology": technology, "url": event.data["url"], "host": str(event.host)}, diff --git a/bbot/presets/baddns-heavy.yml b/bbot/presets/baddns-heavy.yml new file mode 100644 index 0000000000..7171728864 --- /dev/null +++ b/bbot/presets/baddns-heavy.yml @@ -0,0 +1,21 @@ +description: Run all baddns modules and submodules. + +include: + - baddns + +modules: + - baddns_zone + - baddns_direct + +config: + modules: + baddns: + enabled_submodules: [CNAME, NS, MX, TXT, references, DMARC, SPF, MTA-STS, WILDCARD] + min_severity: INFORMATIONAL + min_confidence: UNKNOWN + baddns_zone: + min_severity: INFORMATIONAL + min_confidence: UNKNOWN + baddns_direct: + min_severity: INFORMATIONAL + min_confidence: UNKNOWN diff --git a/bbot/presets/baddns-intense.yml b/bbot/presets/baddns-intense.yml deleted file mode 100644 index 8afeebd3d9..0000000000 --- a/bbot/presets/baddns-intense.yml +++ /dev/null @@ -1,12 +0,0 @@ -description: Run all baddns modules and submodules. - - -modules: - - baddns - - baddns_zone - - baddns_direct - -config: - modules: - baddns: - enabled_submodules: [CNAME,references,MX,NS,TXT] diff --git a/bbot/presets/baddns.yml b/bbot/presets/baddns.yml new file mode 100644 index 0000000000..c737257011 --- /dev/null +++ b/bbot/presets/baddns.yml @@ -0,0 +1,11 @@ +description: Check for subdomain takeovers and other DNS issues. + +modules: + - baddns + +config: + modules: + baddns: + enabled_submodules: [CNAME, MX, TXT] + min_severity: LOW + min_confidence: MODERATE diff --git a/bbot/presets/kitchen-sink.yml b/bbot/presets/kitchen-sink.yml index c01039099e..7b98d2934e 100644 --- a/bbot/presets/kitchen-sink.yml +++ b/bbot/presets/kitchen-sink.yml @@ -10,9 +10,4 @@ include: - paramminer - dirbust-light - web-screenshots - - baddns-intense - -config: - modules: - baddns: - enable_references: True + - baddns-heavy diff --git a/bbot/presets/nuclei/nuclei-intense.yml b/bbot/presets/nuclei/nuclei-intense.yml index 27f833c387..6486ca8586 100644 --- a/bbot/presets/nuclei/nuclei-intense.yml +++ b/bbot/presets/nuclei/nuclei-intense.yml @@ -18,7 +18,7 @@ config: conditions: - | {% if config.web.spider_distance == 0 and config.modules.nuclei.directory_only == False %} - {{ warn("The 'nuclei-intense' preset turns the 'directory_only' limitation off on the nuclei module. To make the best use of this, you may want to enable spidering with 'spider' or 'spider-intense' preset.") }} + {{ warn("The 'nuclei-intense' preset turns the 'directory_only' limitation off on the nuclei module. To make the best use of this, you may want to enable spidering with 'spider' or 'spider-heavy' preset.") }} {% endif %} diff --git a/bbot/presets/spider-intense.yml b/bbot/presets/spider-heavy.yml similarity index 100% rename from bbot/presets/spider-intense.yml rename to bbot/presets/spider-heavy.yml diff --git a/bbot/presets/web/lightfuzz-light.yml b/bbot/presets/web/lightfuzz-light.yml index e562a5aa84..0292474185 100644 --- a/bbot/presets/web/lightfuzz-light.yml +++ b/bbot/presets/web/lightfuzz-light.yml @@ -17,5 +17,5 @@ config: conditions: - | {% if config.web.spider_distance == 0 %} - {{ warn("Lightfuzz works much better with spider enabled! Consider adding 'spider' or 'spider-intense' preset.") }} + {{ warn("Lightfuzz works much better with spider enabled! Consider adding 'spider' or 'spider-heavy' preset.") }} {% endif %} diff --git a/bbot/presets/web/lightfuzz-xss.yml b/bbot/presets/web/lightfuzz-xss.yml index 8eff0d0cc3..2a43bc039b 100644 --- a/bbot/presets/web/lightfuzz-xss.yml +++ b/bbot/presets/web/lightfuzz-xss.yml @@ -18,5 +18,5 @@ config: conditions: - | {% if config.web.spider_distance == 0 %} - {{ warn("The lightfuzz-xss preset works much better with spider enabled! Consider adding 'spider' or 'spider-intense' preset.") }} + {{ warn("The lightfuzz-xss preset works much better with spider enabled! Consider adding 'spider' or 'spider-heavy' preset.") }} {% endif %} diff --git a/bbot/presets/web/paramminer.yml b/bbot/presets/web/paramminer.yml index 3057a1b3a9..634e883742 100644 --- a/bbot/presets/web/paramminer.yml +++ b/bbot/presets/web/paramminer.yml @@ -11,5 +11,5 @@ modules: conditions: - | {% if config.web.spider_distance == 0 %} - {{ warn("The paramminer preset works much better with spider enabled! Consider adding 'spider' or 'spider-intense' preset.") }} + {{ warn("The paramminer preset works much better with spider enabled! Consider adding 'spider' or 'spider-heavy' preset.") }} {% endif %} \ No newline at end of file diff --git a/bbot/test/test_step_2/module_tests/test_module_baddns.py b/bbot/test/test_step_2/module_tests/test_module_baddns.py index 2d5d476d05..47d38d989b 100644 --- a/bbot/test/test_step_2/module_tests/test_module_baddns.py +++ b/bbot/test/test_step_2/module_tests/test_module_baddns.py @@ -1,6 +1,146 @@ +import ast +import inspect + +from baddns.base import get_all_modules +from baddns.lib.findings import SEVERITY_LEVELS, CONFIDENCE_LEVELS + +from bbot.modules.baddns import SUBMODULE_MAX_SEVERITY, SUBMODULE_MAX_CONFIDENCE + from .base import ModuleTestBase +def _extract_finding_values(module_cls): + """ + AST-parse a baddns submodule and extract all string-literal severity + and confidence values from Finding() constructor calls. + + Returns (set of severity strings, set of confidence strings). + """ + source = inspect.getsource(module_cls) + tree = ast.parse(source) + + severities = set() + confidences = set() + + for node in ast.walk(tree): + # Look for Finding({...}) calls + if not isinstance(node, ast.Call): + continue + func = node.func + if not (isinstance(func, ast.Name) and func.id == "Finding"): + continue + if not node.args: + continue + arg = node.args[0] + if not isinstance(arg, ast.Dict): + continue + for key, value in zip(arg.keys, arg.values): + if not isinstance(key, ast.Constant): + continue + if key.value == "severity" and isinstance(value, ast.Constant) and isinstance(value.value, str): + severities.add(value.value) + if key.value == "confidence" and isinstance(value, ast.Constant) and isinstance(value.value, str): + confidences.add(value.value) + + return severities, confidences + + +def _max_level(values, levels): + """Return the highest value from `values` according to the ordering in `levels`.""" + if not values: + return None + # levels is ordered high-to-low in baddns (CRITICAL, HIGH, MEDIUM, LOW, INFO) + for level in levels: + if level in values: + return level + return None + + +# Modules that wrap CNAME findings and may use variables for severity/confidence. +# When a field uses a variable instead of a literal, its max is bounded by +# CNAME's max for that field. +CNAME_DERIVED_MODULES = {"references", "txt", "wildcard", "mtasts"} + + +def test_baddns_max_severity_confidence(): + """ + Ensure SUBMODULE_MAX_SEVERITY and SUBMODULE_MAX_CONFIDENCE in the BBOT + baddns module match what the baddns library's source code actually contains. + """ + all_modules = {cls.name: cls for cls in get_all_modules()} + + # First, compute CNAME's maxes since other modules derive from it + cname_cls = all_modules.get("CNAME") + assert cname_cls is not None, "CNAME module not found in baddns" + cname_sevs, cname_confs = _extract_finding_values(cname_cls) + cname_max_sev = _max_level(cname_sevs, SEVERITY_LEVELS) + cname_max_conf = _max_level(cname_confs, CONFIDENCE_LEVELS) + + errors = [] + + for name, cls in all_modules.items(): + sevs, confs = _extract_finding_values(cls) + + # For CNAME-derived modules, if no literal severity/confidence was found, + # the values are inherited from CNAME findings + if ( + name.lower() in CNAME_DERIVED_MODULES + or cls.__name__.lower().replace("baddns_", "") in CNAME_DERIVED_MODULES + ): + if not sevs and cname_max_sev: + sevs = {cname_max_sev} + if not confs and cname_max_conf: + confs = {cname_max_conf} + + actual_max_sev = _max_level(sevs, SEVERITY_LEVELS) + actual_max_conf = _max_level(confs, CONFIDENCE_LEVELS) + + expected_sev = SUBMODULE_MAX_SEVERITY.get(name) + expected_conf = SUBMODULE_MAX_CONFIDENCE.get(name) + + if expected_sev is None and expected_conf is None: + errors.append(f"Module [{name}] is missing from both SUBMODULE_MAX_SEVERITY and SUBMODULE_MAX_CONFIDENCE") + continue + + if expected_sev != actual_max_sev: + errors.append( + f"Module [{name}] max_severity mismatch: BBOT says {expected_sev}, baddns source says {actual_max_sev}" + ) + if expected_conf != actual_max_conf: + errors.append( + f"Module [{name}] max_confidence mismatch: BBOT says {expected_conf}, baddns source says {actual_max_conf}" + ) + + assert not errors, "BBOT baddns max severity/confidence out of sync with baddns library:\n" + "\n".join(errors) + + +def test_baddns_submodule_coverage(): + """ + Ensure every baddns submodule is represented in both SUBMODULE_MAX dicts, + and no stale entries exist for modules that were removed. + """ + all_module_names = {cls.name for cls in get_all_modules()} + severity_names = set(SUBMODULE_MAX_SEVERITY.keys()) + confidence_names = set(SUBMODULE_MAX_CONFIDENCE.keys()) + + missing_severity = all_module_names - severity_names + missing_confidence = all_module_names - confidence_names + stale_severity = severity_names - all_module_names + stale_confidence = confidence_names - all_module_names + + errors = [] + if missing_severity: + errors.append(f"Modules missing from SUBMODULE_MAX_SEVERITY: {missing_severity}") + if missing_confidence: + errors.append(f"Modules missing from SUBMODULE_MAX_CONFIDENCE: {missing_confidence}") + if stale_severity: + errors.append(f"Stale entries in SUBMODULE_MAX_SEVERITY (module removed from baddns): {stale_severity}") + if stale_confidence: + errors.append(f"Stale entries in SUBMODULE_MAX_CONFIDENCE (module removed from baddns): {stale_confidence}") + + assert not errors, "\n".join(errors) + + class BaseTestBaddns(ModuleTestBase): modules_overrides = ["baddns"] targets = ["bad.dns"] diff --git a/bbot/test/test_step_2/module_tests/test_module_baddns_direct.py b/bbot/test/test_step_2/module_tests/test_module_baddns_direct.py index b2b49717c8..03b98b8dd0 100644 --- a/bbot/test/test_step_2/module_tests/test_module_baddns_direct.py +++ b/bbot/test/test_step_2/module_tests/test_module_baddns_direct.py @@ -44,7 +44,10 @@ def set_target(self, target): module_test.scan.modules["dummy_module"] = self.dummy_module expect_args = {"method": "GET", "uri": "/"} - respond_args = {"response_data": "The specified bucket does not exist", "status": 401} + respond_args = { + "response_data": "NoSuchBucketThe specified bucket does not existbad.dns", + "status": 401, + } module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) await module_test.mock_dns({"bad.dns": {"A": ["127.0.0.1"]}}) @@ -55,7 +58,7 @@ def set_target(self, target): def check(self, module_test, events): assert any( e.type == "FINDING" - and "Possible [AWS Bucket Takeover Detection] via direct BadDNS analysis. Indicator: [[Words: The specified bucket does not exist | Condition: and | Part: body] Matchers-Condition: and] Trigger: [self] baddns Module: [CNAME]" + and "Possible [AWS Bucket Takeover Detection] via direct BadDNS analysis. Indicator: [[Words: The specified bucket does not exist, BucketName | Condition: and | Part: body] Matchers-Condition: and] Trigger: [self] baddns Module: [CNAME]" in e.data["description"] for e in events ), "Failed to emit FINDING" diff --git a/bbot/test/test_step_2/module_tests/test_module_baddns_zone.py b/bbot/test/test_step_2/module_tests/test_module_baddns_zone.py index 10da342217..688f105ea5 100644 --- a/bbot/test/test_step_2/module_tests/test_module_baddns_zone.py +++ b/bbot/test/test_step_2/module_tests/test_module_baddns_zone.py @@ -43,20 +43,22 @@ def check(self, module_test, events): class TestBaddns_zone_nsec(BaseTestBaddns_zone): + targets = ["bad.com"] + async def setup_after_prep(self, module_test): from baddns.lib.whoismanager import WhoisManager await module_test.mock_dns( { - "bad.dns": {"A": ["127.0.0.5"], "NSEC": ["asdf.bad.dns"]}, - "asdf.bad.dns": {"NSEC": ["zzzz.bad.dns"]}, - "zzzz.bad.dns": {"NSEC": ["xyz.bad.dns"]}, + "bad.com": {"A": ["127.0.0.5"], "NSEC": ["asdf.bad.com"]}, + "asdf.bad.com": {"NSEC": ["zzzz.bad.com"]}, + "zzzz.bad.com": {"NSEC": ["xyz.bad.com"]}, } ) module_test.monkeypatch.setattr(WhoisManager, "dispatchWHOIS", self.dispatchWHOIS) def check(self, module_test, events): - assert any(e.data == "zzzz.bad.dns" for e in events), "NSEC Walk Failed (1)" - assert any(e.data == "xyz.bad.dns" for e in events), "NSEC Walk Failed (2)" + assert any(e.data == "zzzz.bad.com" for e in events), "NSEC Walk Failed (1)" + assert any(e.data == "xyz.bad.com" for e in events), "NSEC Walk Failed (2)" assert any(e.type == "FINDING" for e in events), "Failed to emit FINDING" assert any("baddns-nsec" in e.tags for e in events), "Failed to add baddns tag" diff --git a/pyproject.toml b/pyproject.toml index 4ac757167f..96ab0632f4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -77,6 +77,7 @@ dev = [ "pytest-httpx>=0.35", "pytest-benchmark>=4,<6", "ruff==0.15.2", + "baddns~=2.0.0", ] docs = [ "mkdocs>=1.5.2,<2", diff --git a/uv.lock b/uv.lock index 5601ab7479..d6a27e7f21 100644 --- a/uv.lock +++ b/uv.lock @@ -164,6 +164,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/02/e3/a4fa1946722c4c7b063cc25043a12d9ce9b4323777f89643be74cef2993c/backrefs-6.1-py39-none-any.whl", hash = "sha256:a9e99b8a4867852cad177a6430e31b0f6e495d65f8c6c134b68c14c3c95bf4b0", size = 381058, upload-time = "2025-11-15T14:52:06.698Z" }, ] +[[package]] +name = "baddns" +version = "2.0.452" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cloudcheck" }, + { name = "colorama" }, + { name = "dnspython" }, + { name = "httpx" }, + { name = "python-dateutil" }, + { name = "python-whois" }, + { name = "pyyaml" }, + { name = "tldextract" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/77/32/1d82b8781bd9285fabb4a03f342bfc8e27f219665af5dd184fc97bf9694a/baddns-2.0.452.tar.gz", hash = "sha256:193e3ff866986ddb626ec563991c3ea760985ca74f7ed59c0f3770c9f5543d9d", size = 61143, upload-time = "2026-03-20T18:39:14.505Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d6/cb/7ed49be7f8802adfcb57710548ed28239d899eaf5db7a49e57df8600385f/baddns-2.0.452-py3-none-any.whl", hash = "sha256:54e38e2b661de981f95454abea8b17a6bb3d1d5ff1e046738c319cfd7cfc018d", size = 117120, upload-time = "2026-03-20T18:39:13.376Z" }, +] + [[package]] name = "bbot" source = { editable = "." } @@ -181,6 +200,7 @@ dependencies = [ { name = "httpx" }, { name = "idna" }, { name = "jinja2" }, + { name = "kreuzberg" }, { name = "lxml" }, { name = "mmh3" }, { name = "omegaconf" }, @@ -190,6 +210,7 @@ dependencies = [ { name = "pycryptodome" }, { name = "pydantic" }, { name = "pyjwt" }, + { name = "pypdfium2" }, { name = "pyzmq" }, { name = "radixtarget" }, { name = "regex" }, @@ -207,6 +228,7 @@ dependencies = [ [package.dev-dependencies] dev = [ + { name = "baddns" }, { name = "fastapi" }, { name = "pre-commit" }, { name = "pytest" }, @@ -249,6 +271,7 @@ requires-dist = [ { name = "httpx", specifier = ">=0.28.1,<1" }, { name = "idna", specifier = ">=3.4,<4" }, { name = "jinja2", specifier = ">=3.1.3,<4" }, + { name = "kreuzberg", specifier = ">=4.3,<5" }, { name = "lxml", specifier = ">=4.9.2,<7.0.0" }, { name = "mmh3", specifier = ">=4.1,<6.0" }, { name = "omegaconf", specifier = ">=2.3.0,<3" }, @@ -258,6 +281,7 @@ requires-dist = [ { name = "pycryptodome", specifier = ">=3.17,<4" }, { name = "pydantic", specifier = ">=2.12.2,<3" }, { name = "pyjwt", specifier = ">=2.7.0,<3" }, + { name = "pypdfium2", specifier = ">=5.0,<6" }, { name = "pyzmq", specifier = ">=26.0.3,<28.0.0" }, { name = "radixtarget", specifier = ">=4.0.1,<5" }, { name = "regex", specifier = ">=2024.4.16,<2027.0.0" }, @@ -275,6 +299,7 @@ requires-dist = [ [package.metadata.requires-dev] dev = [ + { name = "baddns", specifier = "~=2.0.0" }, { name = "fastapi", specifier = ">=0.115.5,<0.129.0" }, { name = "pre-commit", specifier = ">=3.4,<5.0" }, { name = "pytest", specifier = ">=8.3.1,<9" }, @@ -993,6 +1018,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, ] +[[package]] +name = "kreuzberg" +version = "4.5.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/eb/99/e2b94a0ceece484390b6ec3888f505ecc889d1366dbf7d8bbf7dedd11e67/kreuzberg-4.5.2.tar.gz", hash = "sha256:d7f30f5699ef953acbbd32f228eaf938e850b7f0a767d671faad29e18d4fc9f2", size = 2000056, upload-time = "2026-03-21T23:40:37.87Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c0/2d/3d50764bd70d3adc6ea3e2daa5ce3eb46e359d7624e6aef0727384be5e60/kreuzberg-4.5.2-cp310-abi3-macosx_14_0_arm64.whl", hash = "sha256:d0b596b8371cdc654cf988dbdf9c129a45f76c00a52e0181fde770cbc39dbbf9", size = 49133340, upload-time = "2026-03-21T23:40:25.239Z" }, + { url = "https://files.pythonhosted.org/packages/16/f9/2b8994b855f717a2ea5a73a3ff7059b83dca98ce4dc5ff086024533eb3e4/kreuzberg-4.5.2-cp310-abi3-manylinux_2_39_aarch64.whl", hash = "sha256:e0b7c8822ca7a19c57f53eb76fc4a4481a4c4bcace917ea0a8556848c1b97bee", size = 57139635, upload-time = "2026-03-21T23:40:28.417Z" }, + { url = "https://files.pythonhosted.org/packages/56/06/2d1d2f9f8f586076d6b20239f7180cfa181b9868d64057f5b19a66ddb59e/kreuzberg-4.5.2-cp310-abi3-manylinux_2_39_x86_64.whl", hash = "sha256:10856a2cfc480a31dfeebf1c9c6b514f765f90566ff818d01e910ec8052ece37", size = 56747977, upload-time = "2026-03-21T23:40:31.618Z" }, + { url = "https://files.pythonhosted.org/packages/4b/33/2714045420ebbcd2c7a743b3b4d0bc529e00c92bbd0fe727e7cae0df891f/kreuzberg-4.5.2-cp310-abi3-win_amd64.whl", hash = "sha256:5f158326504d94a7d26ebb8f10f673cee9556fb238009842a86b3ff723867465", size = 56455168, upload-time = "2026-03-21T23:40:35.197Z" }, +] + [[package]] name = "libsass" version = "0.23.0" @@ -1992,6 +2029,35 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/10/bd/c038d7cc38edc1aa5bf91ab8068b63d4308c66c4c8bb3cbba7dfbc049f9c/pyparsing-3.3.2-py3-none-any.whl", hash = "sha256:850ba148bd908d7e2411587e247a1e4f0327839c40e2e5e6d05a007ecc69911d", size = 122781, upload-time = "2026-01-21T03:57:55.912Z" }, ] +[[package]] +name = "pypdfium2" +version = "5.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3b/01/be763b9081c7eb823196e7d13d9c145bf75ac43f3c1466de81c21c24b381/pypdfium2-5.6.0.tar.gz", hash = "sha256:bcb9368acfe3547054698abbdae68ba0cbd2d3bda8e8ee437e061deef061976d", size = 270714, upload-time = "2026-03-08T01:05:06.5Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9d/b1/129ed0177521a93a892f8a6a215dd3260093e30e77ef7035004bb8af7b6c/pypdfium2-5.6.0-py3-none-android_23_arm64_v8a.whl", hash = "sha256:fb7858c9707708555b4a719b5548a6e7f5d26bc82aef55ae4eb085d7a2190b11", size = 3346059, upload-time = "2026-03-08T01:04:21.37Z" }, + { url = "https://files.pythonhosted.org/packages/86/34/cbdece6886012180a7f2c7b2c360c415cf5e1f83f1973d2c9201dae3506a/pypdfium2-5.6.0-py3-none-android_23_armeabi_v7a.whl", hash = "sha256:6a7e1f4597317786f994bfb947eef480e53933f804a990193ab89eef8243f805", size = 2804418, upload-time = "2026-03-08T01:04:23.384Z" }, + { url = "https://files.pythonhosted.org/packages/6e/f6/9f9e190fe0e5a6b86b82f83bd8b5d3490348766062381140ca5cad8e00b1/pypdfium2-5.6.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:e468c38997573f0e86f03273c2c1fbdea999de52ba43fee96acaa2f6b2ad35f7", size = 3412541, upload-time = "2026-03-08T01:04:25.45Z" }, + { url = "https://files.pythonhosted.org/packages/ee/8d/e57492cb2228ba56ed57de1ff044c8ac114b46905f8b1445c33299ba0488/pypdfium2-5.6.0-py3-none-macosx_11_0_x86_64.whl", hash = "sha256:ad3abddc5805424f962e383253ccad6a0d1d2ebd86afa9a9e1b9ca659773cd0d", size = 3592320, upload-time = "2026-03-08T01:04:27.509Z" }, + { url = "https://files.pythonhosted.org/packages/f9/8a/8ab82e33e9c551494cbe1526ea250ca8cc4e9e98d6a4fc6b6f8d959aa1d1/pypdfium2-5.6.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f6b5eb9eae5c45076395454522ca26add72ba8bd1fe473e1e4721aa58521470c", size = 3596450, upload-time = "2026-03-08T01:04:29.183Z" }, + { url = "https://files.pythonhosted.org/packages/f5/b5/602a792282312ccb158cc63849528079d94b0a11efdc61f2a359edfb41e9/pypdfium2-5.6.0-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:258624da8ef45cdc426e11b33e9d83f9fb723c1c201c6e0f4ab5a85966c6b876", size = 3325442, upload-time = "2026-03-08T01:04:30.886Z" }, + { url = "https://files.pythonhosted.org/packages/81/1f/9e48ec05ed8d19d736c2d1f23c1bd0f20673f02ef846a2576c69e237f15d/pypdfium2-5.6.0-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e9367451c8a00931d6612db0822525a18c06f649d562cd323a719e46ac19c9bb", size = 3727434, upload-time = "2026-03-08T01:04:33.619Z" }, + { url = "https://files.pythonhosted.org/packages/33/90/0efd020928b4edbd65f4f3c2af0c84e20b43a3ada8fa6d04f999a97afe7a/pypdfium2-5.6.0-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a757869f891eac1cc1372e38a4aa01adac8abc8fe2a8a4e2ebf50595e3bf5937", size = 4139029, upload-time = "2026-03-08T01:04:36.08Z" }, + { url = "https://files.pythonhosted.org/packages/ff/49/a640b288a48dab1752281dd9b72c0679fccea107874e80a65a606b00efa9/pypdfium2-5.6.0-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:515be355222cc57ae9e62cd5c7c350b8e0c863efc539f80c7d75e2811ba45cb6", size = 3646387, upload-time = "2026-03-08T01:04:38.151Z" }, + { url = "https://files.pythonhosted.org/packages/b0/3b/a344c19c01021eeb5d830c102e4fc9b1602f19c04aa7d11abbe2d188fd8e/pypdfium2-5.6.0-py3-none-manylinux_2_27_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d1c4753c7caf7d004211d7f57a21f10d127f5e0e5510a14d24bc073e7220a3ea", size = 3097212, upload-time = "2026-03-08T01:04:40.776Z" }, + { url = "https://files.pythonhosted.org/packages/50/96/e48e13789ace22aeb9b7510904a1b1493ec588196e11bbacc122da330b3d/pypdfium2-5.6.0-py3-none-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c49729090281fdd85775fb8912c10bd19e99178efaa98f145ab06e7ce68554d2", size = 2965026, upload-time = "2026-03-08T01:04:42.857Z" }, + { url = "https://files.pythonhosted.org/packages/cb/06/3100e44d4935f73af8f5d633d3bd40f0d36d606027085a0ef1f0566a6320/pypdfium2-5.6.0-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:a4a1749a8d4afd62924a8d95cfa4f2e26fc32957ce34ac3b674be6f127ed252e", size = 4131431, upload-time = "2026-03-08T01:04:44.982Z" }, + { url = "https://files.pythonhosted.org/packages/64/ef/d8df63569ce9a66c8496057782eb8af78e0d28667922d62ec958434e3d4b/pypdfium2-5.6.0-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:36469ebd0fdffb7130ce45ed9c44f8232d91571c89eb851bd1633c64b6f6114f", size = 3747469, upload-time = "2026-03-08T01:04:46.702Z" }, + { url = "https://files.pythonhosted.org/packages/a6/47/fd2c6a67a49fade1acd719fbd11f7c375e7219912923ef2de0ea0ac1544e/pypdfium2-5.6.0-py3-none-musllinux_1_2_i686.whl", hash = "sha256:9da900df09be3cf546b637a127a7b6428fb22d705951d731269e25fd3adef457", size = 4337578, upload-time = "2026-03-08T01:04:49.007Z" }, + { url = "https://files.pythonhosted.org/packages/6b/f5/836c83e54b01e09478c4d6bf4912651d6053c932250fcee953f5c72d8e4a/pypdfium2-5.6.0-py3-none-musllinux_1_2_ppc64le.whl", hash = "sha256:45fccd5622233c5ec91a885770ae7dd4004d4320ac05a4ad8fa03a66dea40244", size = 4376104, upload-time = "2026-03-08T01:04:51.04Z" }, + { url = "https://files.pythonhosted.org/packages/6e/7f/b940b6a1664daf8f9bad87c6c99b84effa3611615b8708d10392dc33036c/pypdfium2-5.6.0-py3-none-musllinux_1_2_riscv64.whl", hash = "sha256:282dc030e767cd61bd0299f9d581052b91188e2b87561489057a8e7963e7e0cb", size = 3929824, upload-time = "2026-03-08T01:04:53.544Z" }, + { url = "https://files.pythonhosted.org/packages/88/79/00267d92a6a58c229e364d474f5698efe446e0c7f4f152f58d0138715e99/pypdfium2-5.6.0-py3-none-musllinux_1_2_s390x.whl", hash = "sha256:a1c1dfe950382c76a7bba1ba160ec5e40df8dd26b04a1124ae268fda55bc4cbe", size = 4270201, upload-time = "2026-03-08T01:04:55.81Z" }, + { url = "https://files.pythonhosted.org/packages/e1/ab/b127f38aba41746bdf9ace15ba08411d7ef6ecba1326d529ba414eb1ed50/pypdfium2-5.6.0-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:43b0341ca6feb6c92e4b7a9eb4813e5466f5f5e8b6baeb14df0a94d5f312c00b", size = 4180793, upload-time = "2026-03-08T01:04:57.961Z" }, + { url = "https://files.pythonhosted.org/packages/0e/8c/a01c8e4302448b614d25a85c08298b0d3e9dfbdac5bd1b2f32c9b02e83d9/pypdfium2-5.6.0-py3-none-win32.whl", hash = "sha256:9dfcd4ff49a2b9260d00e38539ab28190d59e785e83030b30ffaf7a29c42155d", size = 3596753, upload-time = "2026-03-08T01:05:00.566Z" }, + { url = "https://files.pythonhosted.org/packages/9b/5f/2d871adf46761bb002a62686545da6348afe838d19af03df65d1ece786a2/pypdfium2-5.6.0-py3-none-win_amd64.whl", hash = "sha256:c6bc8dd63d0568f4b592f3e03de756afafc0e44aa1fe8878cc4aba1b11ae7374", size = 3716526, upload-time = "2026-03-08T01:05:02.433Z" }, + { url = "https://files.pythonhosted.org/packages/3a/80/0d9b162098597fbe3ac2b269b1682c0c3e8db9ba87679603fdd9b19afaa6/pypdfium2-5.6.0-py3-none-win_arm64.whl", hash = "sha256:5538417b199bdcb3207370c88df61f2ba3dac7a3253f82e1aa2708e6376b6f90", size = 3515049, upload-time = "2026-03-08T01:05:04.587Z" }, +] + [[package]] name = "pytest" version = "8.4.2" @@ -2138,6 +2204,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, ] +[[package]] +name = "python-whois" +version = "0.9.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "python-dateutil" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f1/0c/537914eca91ee5ff281309a5ca71da23c0c975cd6658668a44d3fdcf1cc4/python_whois-0.9.6.tar.gz", hash = "sha256:2e6de7b6d70e305a85f4859cd17781ee3f0da3a02a8e94f23cb4cdcd2e400bfa", size = 125107, upload-time = "2025-10-07T04:36:14.913Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/46/53/d0ceb3ae30da8e8ec2d9af11050178f3b4114d5aa6a7f7074199db3c806f/python_whois-0.9.6-py3-none-any.whl", hash = "sha256:153261941a4d238b1278a4ca9b5b5e0590ed3b4d0c534ba111c4434d5d339410", size = 116976, upload-time = "2025-10-07T04:36:12.328Z" }, +] + [[package]] name = "pyyaml" version = "6.0.3"