Issue #69: Feature request: Check java script files for dirhunt (adde…

…d js to must be downloaded files).

dirhunt/crawler_url.py

@@ -107,7 +107,9 @@ def maybe_rewrite(self):
     def must_be_downloaded(self, response):
         """The file must be downloaded to obtain information.
         """
-        return self.maybe_directory() or (response.headers.get('Content-Type') in ['text/css'])
+        return self.maybe_directory() or (response.headers.get('Content-Type') in [
+            'text/css', 'application/javascript'
+        ])
 
     def maybe_directory(self):
         return self.type not in ['asset', 'document', 'rewrite'] or self.type in ['directory']