From b138ceffb7e7d07e339f822e4e8ed1b3124264f9 Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Wed, 21 Aug 2024 11:23:27 +0530 Subject: [PATCH 1/4] Add util class to support both regex and string out of scope subdomains --- web/reNgine/utilities.py | 47 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/web/reNgine/utilities.py b/web/reNgine/utilities.py index c63fef975..a3101625f 100644 --- a/web/reNgine/utilities.py +++ b/web/reNgine/utilities.py @@ -1,3 +1,4 @@ +import re import os import validators @@ -113,4 +114,48 @@ def is_valid_url(url, validate_only_http_scheme=True): if validate_only_http_scheme: return url.startswith('http://') or url.startswith('https://') return True - return False \ No newline at end of file + return False + + +class SubdomainScopeChecker: + """ + SubdomainScopeChecker is a utility class to check if a subdomain is in scope or not. + it supports both regex and string matching. + """ + + def __init__(self, patterns): + self.regex_patterns = set() + self.plain_patterns = set() + self.load_patterns(patterns) + + def load_patterns(self, patterns): + """ + Load patterns into the checker. + + Args: + patterns (list): List of patterns to load. + Returns: + None + """ + for pattern in patterns: + # skip empty patterns + if not pattern: + continue + try: + self.regex_patterns.add(re.compile(pattern, re.IGNORECASE)) + except re.error: + self.plain_patterns.add(pattern.lower()) + + def is_out_of_scope(self, subdomain): + """ + Check if a subdomain is out of scope. + + Args: + subdomain (str): The subdomain to check. + Returns: + bool: True if the subdomain is out of scope, False otherwise. + """ + subdomain = subdomain.lower() # though we wont encounter this, but just in case + if subdomain in self.plain_patterns: + return True + return any(pattern.search(subdomain) for pattern in self.regex_patterns) From d44d043ba96e368a2a55db8ead88890c16ba187b Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Wed, 21 Aug 2024 11:23:49 +0530 Subject: [PATCH 2/4] add out of scope regex in ui --- .../startScan/_items/start_scan_wizard.html | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/web/startScan/templates/startScan/_items/start_scan_wizard.html b/web/startScan/templates/startScan/_items/start_scan_wizard.html index 1f2af3eda..cd0c8097f 100644 --- a/web/startScan/templates/startScan/_items/start_scan_wizard.html +++ b/web/startScan/templates/startScan/_items/start_scan_wizard.html @@ -28,23 +28,24 @@

Import/Ignore Subdomains

Import Subdomains(Optional)

- You can import subdomains for {{domain.name}} using your private recon tools. +

You can import subdomains for {{domain.name}} discovered through your private reconnaissance tools.


-

Out of Scope Subdomains(Optional)

- You can import subdomains for {{domain.name}} using your private recon tools. -
-
From 9a285758024521fca5c4d92f6fd305fe66861fce Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Wed, 21 Aug 2024 11:23:59 +0530 Subject: [PATCH 3/4] Refactor subdomain out-of-scope checking logic --- web/reNgine/tasks.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/web/reNgine/tasks.py b/web/reNgine/tasks.py index 33ecbae04..2e94b4daf 100644 --- a/web/reNgine/tasks.py +++ b/web/reNgine/tasks.py @@ -420,6 +420,7 @@ def subdomain_discovery( custom_subdomain_tools = [tool.name.lower() for tool in InstalledExternalTool.objects.filter(is_default=False).filter(is_subdomain_gathering=True)] send_subdomain_changes, send_interesting = False, False notif = Notification.objects.first() + subdomain_scope_checker = SubdomainScopeChecker(self.out_of_scope_subdomains) if notif: send_subdomain_changes = notif.send_subdomain_changes_notif send_interesting = notif.send_interesting_notif @@ -565,7 +566,7 @@ def subdomain_discovery( if valid_url: subdomain_name = urlparse(subdomain_name).netloc - if subdomain_name in self.out_of_scope_subdomains: + if subdomain_scope_checker.is_out_of_scope(subdomain_name): logger.error(f'Subdomain {subdomain_name} is out of scope. Skipping.') continue @@ -4421,6 +4422,7 @@ def save_subdomain(subdomain_name, ctx={}): scan_id = ctx.get('scan_history_id') subscan_id = ctx.get('subscan_id') out_of_scope_subdomains = ctx.get('out_of_scope_subdomains', []) + subdomain_checker = SubdomainScopeChecker(out_of_scope_subdomains) valid_domain = ( validators.domain(subdomain_name) or validators.ipv4(subdomain_name) or @@ -4430,7 +4432,7 @@ def save_subdomain(subdomain_name, ctx={}): logger.error(f'{subdomain_name} is not an invalid domain. Skipping.') return None, False - if subdomain_name in out_of_scope_subdomains: + if subdomain_checker.is_out_of_scope(subdomain_name): logger.error(f'{subdomain_name} is out-of-scope. Skipping.') return None, False From ab11e525e559d15a3a5fbd9f8d886a355bf2dc99 Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Wed, 21 Aug 2024 12:16:40 +0530 Subject: [PATCH 4/4] Improve UI for specifying out of scope subdomains in scheduled scans --- .../startScan/_items/schedule_scan_wizard.html | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/web/startScan/templates/startScan/_items/schedule_scan_wizard.html b/web/startScan/templates/startScan/_items/schedule_scan_wizard.html index d2dbf8816..7edf9d05a 100644 --- a/web/startScan/templates/startScan/_items/schedule_scan_wizard.html +++ b/web/startScan/templates/startScan/_items/schedule_scan_wizard.html @@ -75,14 +75,15 @@

Import Subdomains(Optional)

-

Out of Scope Subdomains(Optional)

- You can import subdomains for {{domain.name}} using your private recon tools. -
-

URL Scope and Exclusions