Merge pull request #77 from OpenIsraeliSupermarkets/dev

always report summary

il_supermarket_scarper/engines/engine.py

@@ -251,6 +251,7 @@ def scrape(
             limit=limit, files_types=files_types, store_id=store_id
         )
         self.make_storage_path_dir()
+        completed_successfully = True
         results = []
         try:
             results = self._scrape(
@@ -264,12 +265,15 @@ def scrape(
                 suppress_exception=suppress_exception,
             )
             self.on_download_completed(results=results)
-            self.on_scrape_completed(self.get_storage_path())
         except Exception as e:  # pylint: disable=broad-exception-caught
             if not suppress_exception:
                 raise e
             Logger.warning(f"Suppressing exception! {e}")
+            completed_successfully = False
         finally:
+            self.on_scrape_completed(
+                self.get_storage_path(), completed_successfully=completed_successfully
+            )
             self._post_scraping()
 
         return results

il_supermarket_scarper/utils/scraper_status.py

@@ -111,10 +111,12 @@ def _add_downloaded_files_to_list(self, results, **_):
             self.database.insert_documents(self.VERIFIED_DOWNLOADS, documents)
 
     @lock_by_string()
-    def on_scrape_completed(self, folder_name):
+    def on_scrape_completed(self, folder_name, completed_successfully=True):
         """Report when scraping is completed."""
         self._insert_an_update(
-            ScraperStatus.ESTIMATED_SIZE, folder_size=log_folder_details(folder_name)
+            ScraperStatus.ESTIMATED_SIZE,
+            folder_size=log_folder_details(folder_name),
+            completed_successfully=completed_successfully,
         )
 
     @lock_by_string()

setup.py

@@ -29,7 +29,7 @@
     tests_require=dev_required,
     extras_require={"test": ["pytest", "pytest-xdist"]},
     # *strongly* suggested for sharing
-    version="0.5.2",
+    version="0.5.3",
     # The license can be anything you like
     license="MIT",
     description="python package that implement a scraping for israeli supermarket data",