diff --git a/bbot/core/helpers/files.py b/bbot/core/helpers/files.py index 5e7d2d88d4..982f55c38d 100644 --- a/bbot/core/helpers/files.py +++ b/bbot/core/helpers/files.py @@ -9,7 +9,7 @@ log = logging.getLogger("bbot.core.helpers.files") -def tempfile(self, content, pipe=True): +def tempfile(self, content, pipe=True, extension=None): """ Creates a temporary file or named pipe and populates it with content. @@ -29,7 +29,7 @@ def tempfile(self, content, pipe=True): >>> tempfile(["Another", "temp", "file"], pipe=False) '/home/user/.bbot/temp/someotherfile' """ - filename = self.temp_filename() + filename = self.temp_filename(extension) rm_at_exit(filename) try: if type(content) not in (set, list, tuple): diff --git a/bbot/defaults.yml b/bbot/defaults.yml index 1df926e460..64614d08e1 100644 --- a/bbot/defaults.yml +++ b/bbot/defaults.yml @@ -187,8 +187,10 @@ url_extension_blacklist: - mov - flv - webm -# Distribute URLs with these extensions only to httpx (these are omitted from output) -url_extension_httpx_only: + +# URLs with these extensions are not distributed to modules unless the module opts in via `accept_url_special = True` +# They are also excluded from output. If you want to see them in output, remove them from this list. +url_extension_special: - js # These url extensions are almost always static, so we exclude them from modules that fuzz things diff --git a/bbot/modules/base.py b/bbot/modules/base.py index ed54a34723..40da917cbe 100644 --- a/bbot/modules/base.py +++ b/bbot/modules/base.py @@ -53,6 +53,8 @@ class BaseModule: in_scope_only (bool): Accept only explicitly in-scope events, regardless of the scan's search distance. Default is False. + accept_url_special (bool): Accept "special" URLs not typically distributed to web modules, e.g. JS URLs. Default is False. + options (Dict): Customizable options for the module, e.g., {"api_key": ""}. Empty dict by default. options_desc (Dict): Descriptions for options, e.g., {"api_key": "API Key"}. Empty dict by default. @@ -97,7 +99,7 @@ class BaseModule: scope_distance_modifier = 0 target_only = False in_scope_only = False - + accept_url_special = False _module_threads = 1 _batch_size = 1 @@ -785,10 +787,14 @@ def _event_precheck(self, event): if "target" not in event.tags: return False, "it did not meet target_only filter criteria" - # exclude certain URLs (e.g. javascript): - # TODO: revisit this after httpx rework - if event.type.startswith("URL") and self.name != "httpx" and "httpx-only" in event.tags: - return False, "its extension was listed in url_extension_httpx_only" + # limit js URLs to modules that opt in to receive them + if (not self.accept_url_special) and event.type.startswith("URL"): + extension = getattr(event, "url_extension", "") + if extension in self.scan.url_extension_special: + return ( + False, + f"it is a special URL (extension {extension}) but the module does not opt in to receive special URLs", + ) return True, "precheck succeeded" diff --git a/bbot/modules/httpx.py b/bbot/modules/httpx.py index 21fa48d63d..5be9aacca7 100644 --- a/bbot/modules/httpx.py +++ b/bbot/modules/httpx.py @@ -50,6 +50,8 @@ class httpx(BaseModule): _shuffle_incoming_queue = False _batch_size = 500 _priority = 2 + # accept Javascript URLs + accept_url_special = True async def setup(self): self.threads = self.config.get("threads", 50) diff --git a/bbot/modules/output/base.py b/bbot/modules/output/base.py index da80d4d0aa..71956c25ab 100644 --- a/bbot/modules/output/base.py +++ b/bbot/modules/output/base.py @@ -38,11 +38,6 @@ def _event_precheck(self, event): if self._is_graph_important(event): return True, "event is critical to the graph" - # exclude certain URLs (e.g. javascript): - # TODO: revisit this after httpx rework - if event.type.startswith("URL") and self.name != "httpx" and "httpx-only" in event.tags: - return False, (f"Omitting {event} from output because it's marked as httpx-only") - # omit certain event types if event._omit: if "target" in event.tags: diff --git a/bbot/modules/retirejs.py b/bbot/modules/retirejs.py new file mode 100644 index 0000000000..b1b945fe03 --- /dev/null +++ b/bbot/modules/retirejs.py @@ -0,0 +1,238 @@ +import json +from enum import IntEnum +from bbot.modules.base import BaseModule + + +class RetireJSSeverity(IntEnum): + NONE = 0 + LOW = 1 + MEDIUM = 2 + HIGH = 3 + CRITICAL = 4 + + @classmethod + def from_string(cls, severity_str): + try: + return cls[severity_str.upper()] + except (KeyError, AttributeError): + return cls.NONE + + +class retirejs(BaseModule): + watched_events = ["URL_UNVERIFIED"] + produced_events = ["FINDING"] + flags = ["active", "safe", "web-thorough"] + meta = { + "description": "Detect vulnerable/out-of-date JavaScript libraries", + "created_date": "2025-08-19", + "author": "@liquidsec", + } + options = { + "version": "5.3.0", + "node_version": "18.19.1", + "severity": "medium", + } + options_desc = { + "version": "retire.js version", + "node_version": "Node.js version to install locally", + "severity": "Minimum severity level to report (none, low, medium, high, critical)", + } + + deps_ansible = [ + # Download Node.js binary (Linux x64) + { + "name": "Download Node.js binary (Linux x64)", + "get_url": { + "url": "https://nodejs.org/dist/v#{BBOT_MODULES_RETIREJS_NODE_VERSION}/node-v#{BBOT_MODULES_RETIREJS_NODE_VERSION}-linux-x64.tar.xz", + "dest": "#{BBOT_TEMP}/node-v#{BBOT_MODULES_RETIREJS_NODE_VERSION}-linux-x64.tar.xz", + "mode": "0644", + }, + }, + # Extract Node.js binary (x64) + { + "name": "Extract Node.js binary (x64)", + "unarchive": { + "src": "#{BBOT_TEMP}/node-v#{BBOT_MODULES_RETIREJS_NODE_VERSION}-linux-x64.tar.xz", + "dest": "#{BBOT_TOOLS}", + "remote_src": True, + }, + }, + # Remove existing node directory if it exists + { + "name": "Remove existing node directory", + "file": {"path": "#{BBOT_TOOLS}/node", "state": "absent"}, + }, + # Rename extracted directory to 'node' (x64) + { + "name": "Rename Node.js directory (x64)", + "command": "mv #{BBOT_TOOLS}/node-v#{BBOT_MODULES_RETIREJS_NODE_VERSION}-linux-x64 #{BBOT_TOOLS}/node", + }, + # Make Node.js binary executable + { + "name": "Make Node.js binary executable", + "file": {"path": "#{BBOT_TOOLS}/node/bin/node", "mode": "0755"}, + }, + # Make npm executable + { + "name": "Make npm executable", + "file": {"path": "#{BBOT_TOOLS}/node/bin/npm", "mode": "0755"}, + }, + # Remove existing retirejs directory if it exists + { + "name": "Remove existing retirejs directory", + "file": {"path": "#{BBOT_TOOLS}/retirejs", "state": "absent"}, + }, + # Create retire.js local directory + { + "name": "Create retire.js directory in BBOT_TOOLS", + "file": {"path": "#{BBOT_TOOLS}/retirejs", "state": "directory", "mode": "0755"}, + }, + # Install retire.js locally using local Node.js + { + "name": "Install retire.js locally", + "shell": "cd #{BBOT_TOOLS}/retirejs && #{BBOT_TOOLS}/node/bin/node #{BBOT_TOOLS}/node/lib/node_modules/npm/bin/npm-cli.js install retire@#{BBOT_MODULES_RETIREJS_VERSION} --no-fund --no-audit --silent --no-optional", + "args": {"creates": "#{BBOT_TOOLS}/retirejs/node_modules/.bin/retire"}, + "timeout": 600, + "ignore_errors": False, + }, + # Fix retire script shebang to use our local node binary + { + "name": "Fix retire script shebang", + "shell": "sed -i '1s|#!/usr/bin/env node|#!#{BBOT_TOOLS}/node/bin/node|' #{BBOT_TOOLS}/retirejs/node_modules/.bin/retire", + }, + # Make retire script executable + { + "name": "Make retire script executable", + "file": {"path": "#{BBOT_TOOLS}/retirejs/node_modules/.bin/retire", "mode": "0755"}, + }, + # Create retire cache directory + { + "name": "Create retire cache directory", + "file": {"path": "#{BBOT_CACHE}/retire_cache", "state": "directory", "mode": "0755"}, + }, + ] + + accept_url_special = True + scope_distance_modifier = 1 + _module_threads = 4 + + async def setup(self): + excavate_enabled = self.scan.config.get("excavate") + if not excavate_enabled: + return None, "retirejs will not function without excavate enabled" + + # Validate severity level + valid_severities = ["none", "low", "medium", "high", "critical"] + configured_severity = self.config.get("severity", "medium").lower() + if configured_severity not in valid_severities: + return ( + False, + f"Invalid severity level '{configured_severity}'. Valid options are: {', '.join(valid_severities)}", + ) + + self.repofile = await self.helpers.download( + "https://raw.githubusercontent.com/RetireJS/retire.js/master/repository/jsrepository-v4.json", cache_hrs=24 + ) + if not self.repofile: + return False, "failed to download retire.js repository file" + return True + + async def handle_event(self, event): + js_file = await self.helpers.request(event.data) + if js_file: + js_file_body = js_file.text + if js_file_body: + js_file_body_saved = self.helpers.tempfile(js_file_body, pipe=False, extension="js") + results = await self.execute_retirejs(js_file_body_saved) + if not results: + self.warning("no output from retire.js") + return + results_json = json.loads(results) + if results_json.get("data"): + for file_result in results_json["data"]: + for component_result in file_result.get("results", []): + component = component_result.get("component", "unknown") + version = component_result.get("version", "unknown") + vulnerabilities = component_result.get("vulnerabilities", []) + for vuln in vulnerabilities: + severity = vuln.get("severity", "unknown") + + # Filter by minimum severity level + min_severity = RetireJSSeverity.from_string(self.config.get("severity", "medium")) + vuln_severity = RetireJSSeverity.from_string(severity) + if vuln_severity < min_severity: + self.debug( + f"Skipping vulnerability with severity '{severity}' (below minimum '{min_severity.name.lower()}')" + ) + continue + + identifiers = vuln.get("identifiers", {}) + summary = identifiers.get("summary", "Unknown vulnerability") + cves = identifiers.get("CVE", []) + description_parts = [ + f"Vulnerable JavaScript library detected: {component} v{version}", + f"Severity: {severity.upper()}", + f"Summary: {summary}", + f"JavaScript URL: {event.data}", + ] + if cves: + description_parts.append(f"CVE(s): {', '.join(cves)}") + + below_version = vuln.get("below", "") + at_or_above = vuln.get("atOrAbove", "") + if at_or_above and below_version: + description_parts.append(f"Affected versions: [{at_or_above} to {below_version})") + elif below_version: + description_parts.append(f"Affected versions: [< {below_version}]") + elif at_or_above: + description_parts.append(f"Affected versions: [>= {at_or_above}]") + description = " ".join(description_parts) + data = { + "description": description, + "severity": severity, + "component": component, + "url": event.parent.data["url"], + } + await self.emit_event( + data, + "FINDING", + parent=event, + context=f"{{module}} identified vulnerable JavaScript library {component} v{version} ({severity} severity)", + ) + + async def filter_event(self, event): + url_extension = getattr(event, "url_extension", "") + if url_extension != "js": + return False, f"it is a {url_extension} URL but retirejs only accepts js URLs" + return True + + async def execute_retirejs(self, js_file): + cache_dir = self.helpers.cache_dir / "retire_cache" + retire_dir = self.scan.helpers.tools_dir / "retirejs" + + # Use the retire CLI script directly with our local node binary + local_node_dir = self.scan.helpers.tools_dir / "node" + retire_cli_script = retire_dir / "node_modules" / "retire" / "lib" / "cli.js" + + command = [ + str(local_node_dir / "bin" / "node"), + str(retire_cli_script), + "--outputformat", + "json", + "--cachedir", + str(cache_dir), + "--path", + js_file, + "--jsrepo", + str(self.repofile), + ] + + proxy = self.scan.web_config.get("http_proxy") + if proxy: + command.extend(["--proxy", proxy]) + + self.verbose(f"Running retire.js on {js_file}") + self.verbose(f"retire.js command: {command}") + + result = await self.run_process(command) + return result.stdout diff --git a/bbot/scanner/manager.py b/bbot/scanner/manager.py index e4739b20e7..47d91c545e 100644 --- a/bbot/scanner/manager.py +++ b/bbot/scanner/manager.py @@ -94,10 +94,6 @@ async def handle_event(self, event, **kwargs): # special handling of URL extensions url_extension = getattr(event, "url_extension", None) if url_extension is not None: - if url_extension in self.scan.url_extension_httpx_only: - event.add_tag("httpx-only") - event._omit = True - # blacklist by extension if url_extension in self.scan.url_extension_blacklist: self.debug( @@ -209,6 +205,13 @@ async def handle_event(self, event, **kwargs): ) event.internal = True + # mark special URLs (e.g. Javascript) as internal so they don't get output except when they're critical to the graph + if event.type.startswith("URL"): + extension = getattr(event, "url_extension", "") + if extension in self.scan.url_extension_special: + event.internal = True + self.debug(f"Making {event} internal because it is a special URL (extension {extension})") + if event.type in self.scan.omitted_event_types: self.debug(f"Omitting {event} because its type is omitted in the config") event._omit = True diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py index 84eced9324..b5269bf753 100644 --- a/bbot/scanner/scanner.py +++ b/bbot/scanner/scanner.py @@ -230,8 +230,8 @@ def __init__( ) # url file extensions + self.url_extension_special = {e.lower() for e in self.config.get("url_extension_special", [])} self.url_extension_blacklist = {e.lower() for e in self.config.get("url_extension_blacklist", [])} - self.url_extension_httpx_only = {e.lower() for e in self.config.get("url_extension_httpx_only", [])} # url querystring behavior self.url_querystring_remove = self.config.get("url_querystring_remove", True) diff --git a/bbot/test/test_step_1/test_bbot_fastapi.py b/bbot/test/test_step_1/test_bbot_fastapi.py index 1136963a3d..669ca827d9 100644 --- a/bbot/test/test_step_1/test_bbot_fastapi.py +++ b/bbot/test/test_step_1/test_bbot_fastapi.py @@ -22,8 +22,8 @@ def test_bbot_multiprocess(bbot_httpserver): queue = multiprocessing.Queue() events_process = multiprocessing.Process(target=run_bbot_multiprocess, args=(queue,)) events_process.start() - events_process.join() - events = queue.get() + events_process.join(timeout=300) + events = queue.get(timeout=10) assert len(events) >= 3 scan_events = [e for e in events if e["type"] == "SCAN"] assert len(scan_events) == 2 diff --git a/bbot/test/test_step_1/test_events.py b/bbot/test/test_step_1/test_events.py index 6c9d58003d..64bd060bf8 100644 --- a/bbot/test/test_step_1/test_events.py +++ b/bbot/test/test_step_1/test_events.py @@ -209,7 +209,6 @@ async def test_events(events, helpers): javascript_event = scan.make_event("http://evilcorp.com/asdf/a.js?b=c#d", "URL_UNVERIFIED", parent=scan.root_event) assert "extension-js" in javascript_event.tags await scan.ingress_module.handle_event(javascript_event) - assert "httpx-only" in javascript_event.tags # scope distance event1 = scan.make_event("1.2.3.4", dummy=True) diff --git a/bbot/test/test_step_1/test_scan.py b/bbot/test/test_step_1/test_scan.py index 1b4b30aafe..5e11561b33 100644 --- a/bbot/test/test_step_1/test_scan.py +++ b/bbot/test/test_step_1/test_scan.py @@ -147,24 +147,18 @@ async def handle_batch(self, *events): @pytest.mark.asyncio async def test_url_extension_handling(bbot_scanner): - scan = bbot_scanner(config={"url_extension_blacklist": ["css"], "url_extension_httpx_only": ["js"]}) + scan = bbot_scanner(config={"url_extension_blacklist": ["css"]}) await scan._prep() assert scan.url_extension_blacklist == {"css"} - assert scan.url_extension_httpx_only == {"js"} good_event = scan.make_event("https://evilcorp.com/a.txt", "URL", tags=["status-200"], parent=scan.root_event) bad_event = scan.make_event("https://evilcorp.com/a.css", "URL", tags=["status-200"], parent=scan.root_event) - httpx_event = scan.make_event("https://evilcorp.com/a.js", "URL", tags=["status-200"], parent=scan.root_event) assert "blacklisted" not in bad_event.tags - assert "httpx-only" not in httpx_event.tags result = await scan.ingress_module.handle_event(good_event) assert result is None result, reason = await scan.ingress_module.handle_event(bad_event) assert result is False assert reason == "event is blacklisted" assert "blacklisted" in bad_event.tags - result = await scan.ingress_module.handle_event(httpx_event) - assert result is None - assert "httpx-only" in httpx_event.tags await scan._cleanup() diff --git a/bbot/test/test_step_2/module_tests/base.py b/bbot/test/test_step_2/module_tests/base.py index addde29b47..1af3db5892 100644 --- a/bbot/test/test_step_2/module_tests/base.py +++ b/bbot/test/test_step_2/module_tests/base.py @@ -61,6 +61,7 @@ def __init__( config=self.config, whitelist=module_test_base.whitelist, blacklist=module_test_base.blacklist, + force_start=getattr(module_test_base, "force_start", False), ) self.events = [] self.log = logging.getLogger(f"bbot.test.{module_test_base.name}") diff --git a/bbot/test/test_step_2/module_tests/test_module_excavate.py b/bbot/test/test_step_2/module_tests/test_module_excavate.py index b7542d3beb..9fa4107c1c 100644 --- a/bbot/test/test_step_2/module_tests/test_module_excavate.py +++ b/bbot/test/test_step_2/module_tests/test_module_excavate.py @@ -24,14 +24,13 @@ async def setup_before_prep(self, module_test): \\x3dwww6.test.notreal %0awww7.test.notreal \\u000awww8.test.notreal - # these ones shouldn't get emitted because they're .js (url_extension_httpx_only) - - - # these ones should Help
  • 16x50 UART Driver
    • + # these ones should get emitted as URL_UNVERIFIED events (processed by httpx which has accept_js_url=True) + + """ expect_args = {"method": "GET", "uri": "/"} respond_args = {"response_data": response_data} @@ -63,8 +62,9 @@ def check(self, module_test, events): assert "www6.test.notreal" in event_data assert "www7.test.notreal" in event_data assert "www8.test.notreal" in event_data - assert "http://127.0.0.1:8888/a_relative.js" not in event_data - assert "http://127.0.0.1:8888/link_relative.js" not in event_data + # .js files should be emitted as URL_UNVERIFIED events (they are processed by httpx which has accept_js_url=True) + assert "http://127.0.0.1:8888/a_relative.js" in event_data + assert "http://127.0.0.1:8888/link_relative.js" in event_data assert "http://127.0.0.1:8888/a_relative.txt" in event_data assert "http://127.0.0.1:8888/link_relative.txt" in event_data diff --git a/bbot/test/test_step_2/module_tests/test_module_retirejs.py b/bbot/test/test_step_2/module_tests/test_module_retirejs.py new file mode 100644 index 0000000000..5cc6a8c212 --- /dev/null +++ b/bbot/test/test_step_2/module_tests/test_module_retirejs.py @@ -0,0 +1,159 @@ +from .base import ModuleTestBase + + +class TestRetireJS(ModuleTestBase): + targets = ["http://127.0.0.1:8888"] + modules_overrides = ["httpx", "excavate", "retirejs"] + + # HTML page with vulnerable JavaScript libraries + vulnerable_html = """ + + + + retire.js test page + + +

    retire.js test page

    +

    This page includes JavaScript libraries for testing.

    + + + + + + + + + + + + +""" + + # Sample jQuery 3.4.1 content + jquery_content = """/*! + * jQuery JavaScript Library v3.4.1 + * https://jquery.com/ + */ +(function( global, factory ) { + "use strict"; + factory( global ); +})(typeof window !== "undefined" ? window : this, function( window, noGlobal ) { + var jQuery = function( selector, context ) { + return new jQuery.fn.init( selector, context ); + }; + jQuery.fn = jQuery.prototype = {}; + jQuery.fn.jquery = "3.4.1"; + if ( typeof noGlobal === "undefined" ) { + window.jQuery = window.$ = jQuery; + } + return jQuery; +});""" + + # Sample Lodash 4.17.11 content + lodash_content = """/** + * @license + * Lodash lodash.com/license | Underscore.js 1.8.3 underscorejs.org/LICENSE + */ +;(function(){ +var i="4.17.11"; +var Mn={VERSION:i}; +if(typeof define=="function"&&define.amd)define(function(){return Mn});else if(typeof module=="object"&&module.exports)module.exports=Mn;else this._=Mn}());""" + + # Sample Handlebars 4.0.5 content + handlebars_content = """/*! + handlebars v4.0.5 +*/ +!function(a,b){"object"==typeof exports&&"object"==typeof module?module.exports=b():"function"==typeof define&&define.amd?define([],b):"object"==typeof exports?exports.Handlebars=b():a.Handlebars=b()}(this,function(){ +var Handlebars={}; +Handlebars.VERSION="4.0.5"; +return Handlebars; +});""" + + async def setup_after_prep(self, module_test): + expect_args = {"uri": "/"} + respond_args = {"response_data": self.vulnerable_html} + module_test.set_expect_requests(expect_args, respond_args) + + expect_args = {"uri": "/jquery-3.4.1.min.js"} + respond_args = {"response_data": self.jquery_content} + module_test.set_expect_requests(expect_args, respond_args) + + expect_args = {"uri": "/lodash.min.js"} + respond_args = {"response_data": self.lodash_content} + module_test.set_expect_requests(expect_args, respond_args) + + expect_args = {"uri": "/handlebars.min.js"} + respond_args = {"response_data": self.handlebars_content} + module_test.set_expect_requests(expect_args, respond_args) + + def check(self, module_test, events): + # Check that excavate found the JavaScript URLs + url_unverified_events = [e for e in events if e.type == "URL_UNVERIFIED"] + js_url_events = [e for e in url_unverified_events if "extension-js" in e.tags] + + # We should have found the JavaScript URLs + assert len(url_unverified_events) > 0, "No URL_UNVERIFIED events found - excavate may not be working" + assert len(js_url_events) >= 3, f"Expected at least 3 JavaScript URLs, found {len(js_url_events)}" + + # Check for FINDING events generated by retirejs + finding_events = [e for e in events if e.type == "FINDING"] + retirejs_findings = [ + e + for e in finding_events + if "vulnerable javascript library detected:" in e.data.get("description", "").lower() + ] + + # We should have at least some findings from our vulnerable libraries + assert len(retirejs_findings) > 0, ( + f"Expected retirejs to find vulnerabilities, but got {len(retirejs_findings)} findings" + ) + + # Check for specific expected vulnerability descriptions + descriptions = [finding.data.get("description", "") for finding in retirejs_findings] + all_descriptions = "\n".join(descriptions) + + # Look for specific vulnerabilities we expect to find + expected_handlebars_vuln = "Vulnerable JavaScript library detected: handlebars v4.0.5 Severity: HIGH Summary: Regular Expression Denial of Service in Handlebars JavaScript URL: http://127.0.0.1:8888/handlebars.min.js CVE(s): CVE-2019-20922 Affected versions: [4.0.0 to 4.4.5)" + expected_jquery_vuln = "Vulnerable JavaScript library detected: jquery v3.4.1 Severity: MEDIUM Summary: Regex in its jQuery.htmlPrefilter sometimes may introduce XSS JavaScript URL: http://127.0.0.1:8888/jquery-3.4.1.min.js CVE(s): CVE-2020-11022 Affected versions: [1.2.0 to 3.5.0)" + + # Verify at least one of the expected vulnerabilities is found + handlebars_found = expected_handlebars_vuln in all_descriptions + jquery_found = expected_jquery_vuln in all_descriptions + + assert handlebars_found and jquery_found, ( + f"Expected to find specific vulnerabilities but didn't find them. Found descriptions:\n{all_descriptions}" + ) + + # Basic validation of findings structure + for finding in retirejs_findings: + assert "description" in finding.data, "Finding should have description" + assert "url" in finding.data, "Finding should have url" + assert finding.parent.type == "URL_UNVERIFIED", "Parent should be URL_UNVERIFIED" + + +class TestRetireJSNoExcavate(ModuleTestBase): + targets = ["http://127.0.0.1:8888"] + modules_overrides = ["httpx", "retirejs"] + force_start = True # Allow scan to continue even if modules fail setup + config_overrides = { + "excavate": False, + } + + def check(self, module_test, events): + # When excavate is disabled, retirejs should fail setup but scan should still run + retirejs_module = module_test.scan.modules.get("retirejs") + + if retirejs_module: + # Check that the module exists but setup failed + setup_status = getattr(retirejs_module, "_setup_status", None) + if setup_status is not None: + success, error_msg = setup_status + assert success is False, "retirejs setup should have failed without excavate" + expected_error = "retirejs will not function without excavate enabled" + assert error_msg == expected_error, f"Expected error message '{expected_error}', but got '{error_msg}'" + + # No retirejs findings should be generated since setup failed + retirejs_findings = [e for e in events if e.type == "FINDING" and getattr(e, "module", None) == "retirejs"] + assert len(retirejs_findings) == 0, "retirejs should not generate findings when setup fails" diff --git a/docs/scanning/configuration.md b/docs/scanning/configuration.md index 82aff01aa2..4d51c91477 100644 --- a/docs/scanning/configuration.md +++ b/docs/scanning/configuration.md @@ -246,9 +246,6 @@ url_extension_blacklist: - mov - flv - webm -# Distribute URLs with these extensions only to httpx (these are omitted from output) -url_extension_httpx_only: - - js # These url extensions are almost always static, so we exclude them from modules that fuzz things url_extension_static: