Issue #104: Improve performance restore session

Nekmo · Mar 7, 2022 · f820d0e · f820d0e
1 parent c000109
commit f820d0e
Showing 1 changed file with 4 additions and 2 deletions.
diff --git a/dirhunt/crawler_url.py b/dirhunt/crawler_url.py
@@ -50,14 +50,16 @@ def add_self_directories(self, exists=None, type_=None):
 
     def start(self):
         from dirhunt.processors import get_processor, GenericProcessor, Error, ProcessIndexOfRequest
-
+        if self.crawler.closing:
+            return self
         session = self.crawler.sessions.get_session()
         try:
             with session.get(self.url.url, stream=True, verify=False, timeout=self.timeout,
                              allow_redirects=False) as resp:
                 self.set_type(resp.headers.get('Content-Type'))
                 self.flags.add(str(resp.status_code))
-
+                if self.crawler.closing:
+                    return self
                 text = ''
                 soup = None
                 processor = None