Issue #62: Report to file (dump json object to file).

Author: Nekmo

dirhunt/crawler.py

@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
+import json
 import multiprocessing
 from concurrent.futures import ThreadPoolExecutor
 from concurrent.futures.thread import _python_exit
@@ -13,6 +14,7 @@
 from dirhunt.cli import random_spinner
 from dirhunt.crawler_url import CrawlerUrl
 from dirhunt.exceptions import EmptyError, RequestError, reraise_with_stack
+from dirhunt.json_report import JsonReportEncoder
 from dirhunt.sessions import Sessions
 from dirhunt.sources import Sources
 from dirhunt.url_info import UrlsInfo
@@ -181,6 +183,9 @@ def close(self):
         self.shutdown(False)
         atexit.unregister(_python_exit)
 
+    def create_report(self):
+        json.dump(self.json(), open(self.to_file, 'w'), cls=JsonReportEncoder, indent=4, sort_keys=True)
+
     def json(self):
         return {
             'current_processed_count': self.current_processed_count,

dirhunt/json_report.py

@@ -0,0 +1,10 @@
+from json import JSONEncoder
+
+
+class JsonReportEncoder(JSONEncoder):
+    def default(self, obj):
+        if isinstance(obj, (set, frozenset)):
+            return list(obj)
+        elif hasattr(obj, 'json'):
+            return obj.json()
+        return super(JsonReportEncoder, self).default(obj)

dirhunt/management.py

@@ -158,6 +158,8 @@ def hunt(urls, threads, exclude_flags, include_flags, interesting_extensions, in
     crawler.print_urls_info()
     if not sys.stdout.isatty():
         output_urls(crawler, stdout_flags)
+    if to_file:
+        crawler.create_report()
 
 
 def main():