Issue #62: Report to file (--to-file parameter).

dirhunt/crawler.py

@@ -23,7 +23,7 @@
 class Crawler(ThreadPoolExecutor):
     def __init__(self, max_workers=None, interesting_extensions=None, interesting_files=None, std=None,
                  progress_enabled=True, timeout=10, depth=3, not_follow_subdomains=False, exclude_sources=(),
-                 not_allow_redirects=False, proxies=None, delay=0, limit=1000):
+                 not_allow_redirects=False, proxies=None, delay=0, limit=1000, to_file=None):
         if not max_workers and not delay:
             max_workers = (multiprocessing.cpu_count() or 1) * 5
         elif not max_workers and delay:
@@ -50,6 +50,7 @@ def __init__(self, max_workers=None, interesting_extensions=None, interesting_fi
         self.not_allow_redirects = not_allow_redirects
         self.limit = limit
         self.current_processed_count = 0
+        self.to_file = to_file
 
     def add_init_urls(self, *urls):
         """Add urls to queue.
@@ -188,4 +189,4 @@ def json(self):
             'processing': self.processing,
             'processed': self.processed,
             # TODO: self.results ?
-        }
+        }

dirhunt/management.py

@@ -8,7 +8,7 @@
 
 import sys
 
-from click import BadOptionUsage
+from click import BadOptionUsage, Path
 
 from dirhunt.crawler import Crawler
 from dirhunt.exceptions import DirHuntError, catch
@@ -125,11 +125,12 @@ def flags_range(flags):
               help='Delay between requests to avoid bans by the server')
 @click.option('--not-allow-redirects', is_flag=True, help='Redirectors will not be followed')
 @click.option('--limit', type=int, default=1000, help='Max number of pages processed to search for directories.')
+@click.option('--to-file', type=Path(writable=True), default=1000, help='Create a report file in JSON.')
 @click.option('--version', is_flag=True, callback=print_version,
               expose_value=False, is_eager=True)
 def hunt(urls, threads, exclude_flags, include_flags, interesting_extensions, interesting_files, stdout_flags,
          progress_enabled, timeout, max_depth, not_follow_subdomains, exclude_sources, proxies, delay,
-         not_allow_redirects, limit):
+         not_allow_redirects, limit, to_file):
     """Find web directories without bruteforce
     """
     if exclude_flags and include_flags:
@@ -147,7 +148,8 @@ def hunt(urls, threads, exclude_flags, include_flags, interesting_extensions, in
                       interesting_files=interesting_files, std=sys.stdout if sys.stdout.isatty() else sys.stderr,
                       progress_enabled=progress_enabled, timeout=timeout, depth=max_depth,
                       not_follow_subdomains=not_follow_subdomains, exclude_sources=exclude_sources,
-                      not_allow_redirects=not_allow_redirects, proxies=proxies, delay=delay, limit=limit)
+                      not_allow_redirects=not_allow_redirects, proxies=proxies, delay=delay, limit=limit,
+                      to_file=to_file)
     crawler.add_init_urls(*urls)
     try:
         catch_keyboard_interrupt(crawler.print_results, crawler.restart)(set(exclude_flags), set(include_flags))